After much frustration, here's the answer that works:
Note that in this case I've configured the default Magento Dataflow Profile for Import All Products (ID: 3) to read in an XML import format, from a predefined file. This is a file I create as needed prior to running the import operations. (Refer to the screenshot in the question above)
Once you have your profile created you'll need 2 files:
- importer.php
- batch_importer_processor.php
You can place the files in the /magento/root/shell/ directory, or adjust the paths as needed if you're including in a separate location. Once in the directory, you can call trigger the operation via cron using:
php -f /path/to/magento/root/directory/shell/importer.php
The -f parameter above is to "parse and execute" the file being called.
Source for each file is:
importer.php
<?php
require_once '../app/Mage.php';
set_time_limit(0);
ini_set('memory_limit', '128M');
$root = "/path/to/your/magento/root/directory/";
$logFile = 'magento_import.log';
umask(0);
$app = Mage::app('default');
Mage::log("========================== BEGIN IMPORT ==========================", null, $logFile);
//echo "========================== BEGIN IMPORT ==========================\n";
// Login Admin User
Mage::getSingleton('core/session', array('name' => 'adminhtml'));
$user = Mage::getModel('admin/user')->loadByUsername($username);
if (Mage::getSingleton('adminhtml/url')->useSecretKey()) {
Mage::getSingleton('adminhtml/url')->renewSecretUrls();
}
$session = Mage::getSingleton('admin/session');
$session->setIsFirstVisit(true);
$session->setUser($user);
$session->setAcl(Mage::getResourceModel('admin/acl')->loadAcl());
Mage::dispatchEvent('admin_session_user_login_success',array('user'=>$user));
if ($session->isLoggedIn()) {
Mage::log("User '" . $username . "' logged in.", null, $logFile);
//echo "User '" . $username . "' logged in.\n";
} else {
Mage::log("ERROR: Could not login as user '" . $username . "'.", null, $logFile);
//echo "ERROR: Could not login as user '" . $username . "'.\n";
}
// Load DataFlow Profile
$profile_id = 3;
$profile = Mage::getModel('dataflow/profile');
$profile->load($profile_id);
if (!$profile->getId()) {
Mage::log("ERROR: Profile with ID #{$profile_id} does not exist.", null, $logFile);
//echo "ERROR: Profile with ID #{$profile_id} does not exist.\n";
exit;
}
Mage::register('current_convert_profile', $profile);
$profile->run();
// Begin Bactch Processing
// Limit of products per batch (max: 50)
$batchLimit = 50;
function convert($size)
{
$unit=array('b','kb','mb','gb','tb','pb');
return @round($size/pow(1024,($i=floor(log($size,1024)))),2).' '.$unit[$i];
}
$batchModel = Mage::getSingleton('dataflow/batch');
if (!$batchModel->getId()) {
Mage::log(convert(memory_get_usage()) . " - ERROR: Can't get batchModel", null, $logFile);
//echo convert(memory_get_usage()) . " - ERROR: Can't get batchModel\n";
exit;
}
if (!$batchModel->getAdapter()) {
Mage::log(convert(memory_get_usage()) . " - ERROR: Can't getAdapter", null, $logFile);
//echo convert(memory_get_usage()) . " - ERROR: Can't getAdapter\n";
exit;
}
$batchId = $batchModel->getId();
$batchImportModel = $batchModel->getBatchImportModel();
$importIds = $batchImportModel->getIdCollection();
$recordCount = null;
$totalproducts = count($importIds);
$saved = 0;
$batchArrayIds = array();
foreach ($importIds as $importId) {
$recordCount++;
$batchArrayIds[] = $importId;
if ($recordCount%$batchLimit == 0 || $recordCount == $totalproducts) {
$paramsArr = array('batchid' => $batchId, 'ids' => $batchArrayIds);
$params = json_encode($paramsArr);
$result = array();
exec("php -f {$root}shell/batch_import_processor.php '{$params}'", $result);
$saved += $result[0];
Mage::log(convert(memory_get_usage()) . " - processed {$recordCount}/$totalproducts. Saved {$result[0]} products.", null, $logFile);
//echo convert(memory_get_usage()) . " - processed {$recordCount}/$totalproducts. Saved {$result[0]} products.\n";
$batchArrayIds = array();
}
}
$batchModel = Mage::getModel('dataflow/batch')->load($batchId);
try {
$batchModel->beforeFinish();
} catch (Mage_Core_Exception $e) {
Mage::log(convert(memory_get_usage()) . " - ERROR: ". $e->getMessage(), null, $logFile);
//echo convert(memory_get_usage()) . " - ERROR: ". $e->getMessage() . "\n";
} catch (Exception $e) {
Mage::log(convert(memory_get_usage()) . " - ERROR: An error occurred while finishing process. Please refresh the cache" . $e->getMessage(), null, $logFile);
//echo convert(memory_get_usage()) . " - ERROR: An error occurred while finishing process. Please refresh the cache" . $e->getMessage() . "\n";
}
$batchModel->delete();
// Output Debugging Info
foreach ($profile->getExceptions() as $e) {
Mage::log(convert(memory_get_usage()) . " - " . $e->getMessage(), null, $logFile);
//echo convert(memory_get_usage()) . " - " . $e->getMessage() . "\n";
}
Mage::log("IMPORT COMPLETE.", null, $logFile);
//echo "IMPORT COMPLETE.\n";
?>
batch_import_processor.php
<?php
$root = '/your/path/to/magento/root/directory/';
$logFile = 'magento_import.log';
require_once $root . 'app/Mage.php';
set_time_limit(0);
ini_set('memory_limit', '128M');
ob_implicit_flush();
Mage::app()->setCurrentStore(Mage_Core_Model_App::ADMIN_STORE_ID);
$params = $argv[1];
$paramsArray = json_decode($params, true);
$batchId = $paramsArray['batchid'];
$importIds = $paramsArray['ids'];
$saved = 0;
$batchModel = Mage::getModel('dataflow/batch')->load($batchId);
$batchImportModel = $batchModel->getBatchImportModel();
$adapter = Mage::getModel($batchModel->getAdapter());
$adapter->setBatchParams($batchModel->getParams());
foreach ($importIds as $importId) {
$batchImportModel->load($importId);
if (!$batchImportModel->getId()) {
Mage::log(convert(memory_get_usage()) . " - ERROR: Skip undefined row {$importId}", null, $logFile);
//echo convert(memory_get_usage()) . " - ERROR: Skip undefined row {$importId}\n";
continue;
}
try {
$importData = $batchImportModel->getBatchData();
$adapter->saveRow($importData);
} catch (Exception $e) {
Mage::log("Exception : " . $e, null, $logFile);
//echo "Exception : " . $e;
continue;
}
$saved ++;
}
if (method_exists($adapter, 'getEventPrefix')) {
// Event to process rules relationships after import
Mage::dispatchEvent($adapter->getEventPrefix() . '_finish_before', array(
'adapter' => $adapter
));
// Clear affected ids for possible reuse
$adapter->clearAffectedEntityIds();
}
Mage::log("Total Products to Import: " . $saved, null, $logFile);
echo $saved;
?>
It's not too difficult to add to this code so you can run multiple profiles in sequence if that is something that you require. Otherwise, be sure to get the profile ID from the Magento backend and set the $profile_id variable with the desired value.
I've included calls to the Magento Log (Mage::log) with the respective echo statements below if you prefer that route. Edit as needed. The logs should be saved in the /magento/root/var/log/ directory. Be sure to enable this feature in the Magento backend by going to:
- System > Configuration > Advanced > Developer > Log Settings
And setting enabled to "Yes" see below:
After running the importer.php file, you should have 2 log files in the /magento/root/var/log/ directory:
- system.log
- magento_import.log
You may also have the exception.log file which is always enabled.
This is the only solution that I've found to work with Magento 1.9.1.0, thanks go out to Andrey for the inspiration.
Any thoughts or improvements are always welcome.
