I wrote a script that imports a CSV file into an applications database but the final file that its going to import has over 70k lines and my script is just far to slow. It hits the max execution time limit on their server before it has even imported a 10th of the lines.
Is there a way to make this script import lines in batches that will be run as separate executions without completely rewriting it?
$job_strings[] = 'ImportAccounts';
function ImportAccounts(){
global $sugar_config;
$cfg = new Configurator();
$emailAddress = $cfg->config['IA']['IA_email'];
$emailSubject = 'Accounts Importing: Error Report';
$import_file = $cfg->config['IA']['IA_path'];
try {
$GLOBALS['log']->fatal('The Account import works');
//Used to handle mac line endings
ini_set('auto_detect_line_endings', TRUE);
$errors = array();
//check file exists if not throw an error
if (!file_exists($import_file)) {
throw new Exception('The import file is not found.');
$GLOBALS['log']->fatal('The Account import csv file is not found.');
}
//Open the csv file
if ($file = fopen($import_file, "r")) {
$i = 1;
while (($data = fgetcsv($file, 100000, ',')) !== FALSE) {//Loop through all rows in the cvs file
if ($i == 1) {//Get column headers
$header_count = count($data);
}
if ($i > 1) {
$col_count = count($data);
// Bail out of the loop if columns are incorrect
if ($header_count != $col_count) {
$error_msg = 'Error: Incorrect number of columns in the .CVS file at line: ' . $i . '. Import halted no records were imported after this point.';
$errors[] = $error_msg;
$GLOBALS['log']->fatal('Incorrect number of columns in the .CVS file at line: \' . $i . \'. Import halted no records were imported after this point.');
break;
} else {
if (!IsNullOrEmptyString($data[21])) { //If ID is specified update the existing record
$account = new Account();
$e_account = $account->retrieve($data[21]);
//Log error if the ID is not recognised.
if (IsNullOrEmptyString($e_account)) {
$error_msg = 'Error: Record at line: ' . $i . ' does not have a valid ID field or the record does not exist. Record was not Imported!' . "\r\n";
$errors[] = $error_msg;
} else {//update record
$e_account->wg_accountnumber_c = $data[0];
$e_account->name = $data[1];
$e_account->billing_address_street = $data[2];
$e_account->billing_address_street_2 = $data[3];
$e_account->billing_address_street_3 = $data[4];
$e_account->billing_address_street_4 = $data[5];
$e_account->billing_address_state = $data[6];
$e_account->billing_address_postalcode = $data[7];
$e_account->phone_office = $data[8];
//check email
if (!IsNullOrEmptyString($data[9])) {
if (filter_var($data[9], FILTER_VALIDATE_EMAIL)) {
$e_account->email1 = $data[9];
} else {
$error_msg = 'Error: Record at line: ' . $i . '. The Record was imported successfully but may contain an invalid email address.' . "\r\n";
$errors[] = $error_msg;
}
}
$e_account->phone_fax = $data[10];
if (!IsNullOrEmptyString($data[11])) {
$e_account->website = $data[11];
}
$e_account->wg_gl_ordertype_c = $data[12];
$e_account->industry = $data[13];
$e_account->user_id_c = $data[14];
$e_account->wg_sellingcompany_c = $data[15];
$e_account->wg_creditterms_c = $data[16];
$e_account->wg_credithold_c = $data[17];
$e_account->wg_proforma_c = $data[18];
$e_account->wg_revenue_ytd_c = $data[19];
$e_account->annual_revenue = $data[20];
$e_account->save();
}
} else { //If no ID is specified create a new record
$e_account = new Account();
$e_account->wg_accountnumber_c = $data[0];
$e_account->name = $data[1];
$e_account->billing_address_street = $data[2];
$e_account->billing_address_street_2 = $data[3];
$e_account->billing_address_street_3 = $data[4];
$e_account->billing_address_street_4 = $data[5];
$e_account->billing_address_state = $data[6];
$e_account->billing_address_postalcode = $data[7];
$e_account->phone_office = $data[8];
//check email
if (!IsNullOrEmptyString($data[9])) {
if (filter_var($data[9], FILTER_VALIDATE_EMAIL)) {
$e_account->email1 = $data[9];
} else {
$error_msg = 'Error: Record at line: ' . $i . '. The Record was imported successfully but may contain an invalid email address.' . "\r\n";
$errors[] = $error_msg;
}
}
$e_account->phone_fax = $data[10];
if (!IsNullOrEmptyString($data[11])) {
$e_account->website = $data[11];
}
$e_account->wg_gl_ordertype_c = $data[12];
$e_account->industry = $data[13];
$e_account->user_id_c = $data[14];
$e_account->wg_sellingcompany_c = $data[15];
$e_account->wg_creditterms_c = $data[16];
$e_account->wg_credithold_c = $data[17];
$e_account->wg_proforma_c = $data[18];
$e_account->wg_revenue_ytd_c = $data[19];
$e_account->annual_revenue = $data[20];
$e_account->save();
}
}
}
$i++;
}
ini_set('auto_detect_line_endings', FALSE);
fclose($file);
//send error report email
if (!empty($errors)) {
$emailBody = 'The following errors occurred during Account import at: ' . date("Y-m-d H:i:s") . "\r\n";
foreach ($errors as $error) {
$emailBody .= "\r\n" . $error;
}
//Send email
sendEmail($emailAddress, $emailSubject, $emailBody);
}
} else {
throw new Exception('The Import file failed to open.');
$GLOBALS['log']->fatal('The import csv file failed to open.');
}
}
catch ( Exception $e ) { //Catch errors where the import file fails to open or is not found
$error_msg = 'Error: '.$e->getMessage(); ;
$errors[] = $error_msg;
$emailBody = 'The following errors occurred during Account import at: ' . date("Y-m-d H:i:s") . "\r\n";
foreach ($errors as $error) {
$emailBody .= "\r\n" . $error;
}
//Send email
sendEmail($emailAddress, $emailSubject, $emailBody);
}
return true;
}
