0

I'm new to PhantomJS. Following this example, I tried to create a script which logs in to avito.ru, then opens up an edit page, changes the price, submits the form, and as the server sends back a confirmation page, script should submit a new form for changes to take place.

So the cycle should be as follows:

Log in
GET .../items/edit/678347092
POST to .../items/edit/678347092
GET .../stiralnaya_mashina_indesit_nws_7105_l_novaya_678347092/edit/confirm
POST to .../stiralnaya_mashina_indesit_nws_7105_l_novaya_678347092/edit/confirm

My code is below:

var steps=[];
var testindex = 0;
var loadInProgress = false;

/*********SETTINGS*********************/
var webPage = require('webpage');
var page = webPage.create();
page.viewportSize = { width: 1366, height: 768 };
page.settings.userAgent = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36';
page.settings.javascriptEnabled = true;
phantom.cookiesEnabled = true;
phantom.javascriptEnabled = true;
/*********SETTINGS END*****************/

console.log('All settings loaded, start with execution');
page.onConsoleMessage = function(msg) {
    console.log(msg);
};
/**********DEFINE STEPS THAT FANTOM SHOULD DO***********************/
steps = [

    //Step 1 - Open Avito login page
    function(){
        console.log('Step 1 - opening Avito');
        page.open("https://www.avito.ru/profile/login?next=%2Fprofile", function(status){

            });
    },

    //Step 2 - Populate and submit the login form
    function(){
        console.log('Step 2');
        console.log('URL: ' + page.url);
        console.log('Populate and submit the login form');
        page.evaluate(function(){
                $('input[type="email"]').val('myemail');
            $('input[type="password"]').val('mypassword');
            $('.js-form-login').submit();    
          });
    },
    //Step 3 Profile page
    function(){
        console.log('Step 3');
        console.log('URL: ' + page.url);
        page.render('3-avito.png');
        console.log("Opening edit page");
        page.open("https://www.avito.ru/items/edit/678347092", function(status){

            });
    },


    //Step 4 Edit the price
   function(){
        console.log('Step 4');
        console.log('URL: ' + page.url);
        page.render('4-edit-page.png');
        console.log("Changing the price and submitting the form");
            page.evaluate(function(){
            document.querySelector('#item-edit__price').value = '21999';
            document.forms[0].submit();
        });
        },
  //Step 5 Confirm changes
   function(){
        console.log('Step 5');
        console.log('URL: ' + page.url);
        page.render('5-edit-confirm.png'); 
        console.log("Final confirmation");
        page.evaluate(function(){ 
            document.forms[0].submit();
            });

    },
  //Step 6 Render the result
    function(){
        console.log('Step 6');
        console.log('URL: ' + page.url);
        console.log("Rendering final page");
        page.render('6-result.png'); 
    }
];

/**********END STEPS THAT FANTOM SHOULD DO***********************/

//Execute steps one by one
interval = setInterval(executeRequestsStepByStep,50);

function executeRequestsStepByStep(){
    if (loadInProgress == false && typeof steps[testindex] == "function") {
        steps[testindex]();
        testindex++;
    }
    if (typeof steps[testindex] != "function") {
        console.log("test complete!");
        phantom.exit();
    }
}

/**
 * These listeners are very important in order to phantom work properly. Using these listeners, we control loadInProgress marker which controls, weather a page is fully loaded.
 * Without this, we will get content of the page, even a page is not fully loaded.
 */
page.onLoadStarted = function() {
    loadInProgress = true;
    console.log('Loading started');
};
page.onLoadFinished = function() {
    loadInProgress = false;
    console.log('Loading finished');
};
page.onConsoleMessage = function(msg) {
    console.log(msg);
};

I tried to paste the code inside page.evaluate to the Chrome's console, and it works fine. But as I can see in Fiddler, this script doesn't perform a first POST request (the form is not submitted at step 4). Also I have rendered images indicating that the price is changed at step 4, but since then all other images remain the same.

Below is the output of the script to the console:

All settings loaded, start with execution
Step 1 - opening Avito
Loading started
Loading finished
Step 2
URL: https://www.avito.ru/profile/login?next=/profile
Populate and submit the login form
Loading started
Loading finished
Step 3
URL: https://www.avito.ru/profile
Opening edit page
Loading started
Loading finished
Step 4
URL: https://www.avito.ru/items/edit/678347092
Changing the price and submitting the form
Step 5
URL: https://www.avito.ru/items/edit/678347092
Final confirmation
Step 6
URL: https://www.avito.ru/items/edit/678347092
Rendering final page
test complete!

Any ideas how to make this script work?

Update: the script sometimes fails to submit the first login form, sometimes the second form, and sometimes it submits the second form instead of the third at the step 5, rather than at the step 4.

I've used onResourceRequested to see the headers of the requests, and in the example below it doesn't submit the second form at the step 4, but submits the same form at the step 5. (So I think it is not the problem with the form validation). Here are some extracts:

Step 1 Open login page

Request (#1): {"headers":[{"name":"User-Agent","value":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 

Safari/537.36"},{"name":"Accept","value":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}],"id":1,"method":"GET","time":"2015-11-

19T12:38:51.111Z","url":"https://www.avito.ru/profile/login?next=%2Fprofile"}

Loading started

...
Loading finished


Step 2
 - POST login and password
Request (#28): {"headers":[{"name":"Origin","value":"https://www.avito.ru"},{"name":"User-Agent","value":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 

(KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36"},{"name":"Content-Type","value":"multipart/form-data; boundary=----WebKitFormBoundaryrQFFuoCCfmZnvHH9"},

{"name":"Accept","value":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"},{"name":"Referer","value":"https://www.avito.ru/profile/login?next=

%2Fprofile"},{"name":"Content-Length","value":"355"}],"id":28,"method":"POST","time":"2015-11-19T12:38:57.595Z","url":"https://www.avito.ru/profile/login"}


Loading started

Request (#29): {"headers":[{"name":"Origin","value":"https://www.avito.ru"},{"name":"User-Agent","value":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 

(KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36"},{"name":"Accept","value":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"},

{"name":"Referer","value":"https://www.avito.ru/profile/login?next=%2Fprofile"}],"id":29,"method":"GET","time":"2015-11-

19T12:38:58.360Z","url":"https://www.avito.ru/profile"}
...
Loading finished


Step 3 Open new page
Request (#72): {"headers":[{"name":"User-Agent","value":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 

Safari/537.36"},{"name":"Accept","value":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}],"id":72,"method":"GET","time":"2015-11-

19T12:39:13.887Z","url":"https://www.avito.ru/items/edit/678347092"}

Loading started
...Loading finished

Step 4 Edit the price and submit changes

doesn't work

Step 5
 - Confirm changes (instead it's performed submit of the form from the step 4)

Request (#107): {"headers":[{"name":"Origin","value":"https://www.avito.ru"},{"name":"User-Agent","value":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 

(KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36"},{"name":"Content-Type","value":"application/x-www-form-urlencoded"},

{"name":"Accept","value":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"},{"name":"Referer","value":"https://www.avito.ru/items/edit/678347092"},

{"name":"Content-Length","value":"8110"}],"id":107,"method":"POST","time":"2015-11-19T12:39:25.950Z","url":"https://www.avito.ru/items/edit/678347092"}

Loading started


Request (#108): {"headers":[{"name":"Origin","value":"https://www.avito.ru"},{"name":"User-Agent","value":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 

(KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36"},{"name":"Accept","value":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"},

{"name":"Referer","value":"https://www.avito.ru/items/edit/678347092"}],"id":108,"method":"GET","time":"2015-11-

19T12:39:26.709Z","url":"https://www.avito.ru/ulyanovsk/bytovaya_tehnika/stiralnaya_mashina_indesit_nws_7105_l_novaya_678347092/edit/confirm"}

Loading finished

Loading started


Loading finished

Step 6 - Screenshot
Vladislav
  • 483
  • 2
  • 6
  • 17
  • Ah, Avito! :) You should add onError callback http://phantomjs.org/api/phantom/handler/on-error.html to see it there's anything going wrong. – Vaviloff Nov 19 '15 at 06:31
  • I've added phantom.onError = function(msg, trace) { var msgStack = ['PHANTOM ERROR: ' + msg]; if (trace && trace.length) { msgStack.push('TRACE:'); trace.forEach(function(t) { msgStack.push(' -> ' + (t.file || t.sourceURL) + ': ' + t.line + (t.function ? ' (in function ' + t.function +')' : '')); }); } console.error(msgStack.join('\n')); phantom.exit(1); }; But this didn't change neither the behavior nor the output of the script – Vladislav Nov 19 '15 at 07:09
  • Have you tried making a screenshot after submitting form in step 4? There probably are some form validation errors, Avito people keep changing the edit item page all the time. – Vaviloff Nov 19 '15 at 08:55
  • Yes, I've tried, but there are no errors displayed. I run script several times, and sometimes it fails to submit first login form, sometimes second form, and sometimes it submits second form not at the step 4, but at the step 5 (step 5 has the same command document.forms[0].submit();). I can't manage to submit the confirmatin form at the last step now. – Vladislav Nov 19 '15 at 13:31
  • 1
    If I were them, I would use (among other things) time-based protection. For example, if form was opened, edited and submitted VERY QUICKLY, I would consider user a bot and mess with him. How much time does your script spend from start to submitting in step 4? – Vaviloff Nov 19 '15 at 14:12
  • @Vaviloff You're right! Finally works after two days of mind-blowing. I've increased the interval. Thank you so much. – Vladislav Nov 19 '15 at 15:06

0 Answers0