Cookies not set with HttpUrlConnection

Question

I'm trying ton connect to a site, according to this page : http://www.mkyong.com/java/how-to-automate-login-a-website-java-example/

But it seems, that the cookies are not set. Cannot find the problem...

My aim is to authenticate to this sit in order to grab the Html. If there is another, more simple, solution, don't hesitate.

package htmlmKyongTest;

import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.CookieHandler;
import java.net.CookieManager;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;

import javax.net.ssl.HttpsURLConnection;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class HttpUrlConnectionExample {

  private List<String> cookies;
  private HttpsURLConnection conn;

  private final String USER_AGENT = "Mozilla/5.0";

  public static void main(String[] args) throws Exception {



    String url = "https://www.entea.fr/";
    String gmail = "https://www.entea.fr/etabs/0671688W/Pages/Accueil.aspx";

    HttpUrlConnectionExample http = new HttpUrlConnectionExample();

    // make sure cookies is turn on
    CookieHandler.setDefault(new CookieManager());

    // 1. Send a "GET" request, so that you can extract the form's data.
    String page = http.GetPageContent(url);
    String postParams = http.getFormParams(page, "myUsername", "myPassword");

    // 2. Construct above post's content and then send a POST request for
    // authentication
    http.sendPost(url, postParams);

    // 3. success
    String result = http.GetPageContent(gmail);
    System.out.println(result);
  }

  private void sendPost(String url, String postParams) throws Exception {

    URL obj = new URL(url);
    conn = (HttpsURLConnection) obj.openConnection();

    // Acts like a browser
    conn.setUseCaches(false);
    conn.setRequestMethod("POST");
    conn.setRequestProperty("Host", "www.entea.fr");
    conn.setRequestProperty("User-Agent", USER_AGENT);
    conn.setRequestProperty("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
    conn.setRequestProperty("Accept-Language", "fr-FR,fr;q=0.8,en-US;q=0.6,en;q=0.4");
    conn.setRequestProperty("Content-Type","application/x-www-form-urlencoded");
    for (String cookie : this.cookies) {
        conn.addRequestProperty("Set-Cookie", cookie.split(";", 1)[0]);
    }

    conn.setRequestProperty("Connection", "keep-alive");
    conn.setRequestProperty("Referer", "https://www.entea.fr/CookieAuth.dll?GetLogon?curl=Z2F&reason=0&formdir=7");
    conn.setRequestProperty("Accept-Encoding","gzip,deflate");
    conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
    conn.setRequestProperty("Content-Length", Integer.toString(postParams.length()));
    conn.setDoOutput(true);
    conn.setDoInput(true);

    // Send post request
    DataOutputStream wr = new DataOutputStream(conn.getOutputStream());
    wr.writeBytes(postParams);
    wr.flush();
    wr.close();

    int responseCode = conn.getResponseCode();
    System.out.println("\nSending 'POST' request to URL : " + url);
    System.out.println("Post parameters : " + postParams);
    System.out.println("Response Code : " + responseCode);

    BufferedReader in = 
             new BufferedReader(new InputStreamReader(conn.getInputStream()));
    String inputLine;
    StringBuffer response = new StringBuffer();

    while ((inputLine = in.readLine()) != null) {
        response.append(inputLine);
    }
    in.close();

  }

  private String GetPageContent(String url) throws Exception {

    URL obj = new URL(url);
    conn = (HttpsURLConnection) obj.openConnection();

    // default is GET
    conn.setRequestMethod("GET");
    conn.setUseCaches(false);

    // act like a browser
    conn.setRequestProperty("User-Agent", USER_AGENT);
    conn.setRequestProperty("Accept",
        "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
    conn.setRequestProperty("Accept-Language", "en-US,en;q=0.5");

    if (cookies != null) {
        for (String cookie : this.cookies) {
            conn.addRequestProperty("Set-Cookie", cookie.split(";", 1)[0]);
        }
    }
    int responseCode = conn.getResponseCode();
    System.out.println("\nSending 'GET' request to URL : " + url);
    System.out.println("Response Code : " + responseCode);

    BufferedReader in = 
            new BufferedReader(new InputStreamReader(conn.getInputStream()));
    String inputLine;
    StringBuffer response = new StringBuffer();

    while ((inputLine = in.readLine()) != null) {
        response.append(inputLine);
    }
    in.close();

    // Get the response cookies
    setCookies(conn.getHeaderFields().get("Set-Cookie"));
    return response.toString();

  }

  public String getFormParams(String html, String username, String password)
        throws UnsupportedEncodingException {

    System.out.println("Extracting form's data...");
    Document doc = Jsoup.parse(html);

    Element loginform = doc.getElementById("logonForm");
    Elements inputElements = loginform.getElementsByTag("input");
    List<String> paramList = new ArrayList<String>();
    for (Element inputElement : inputElements) {
        System.out.println("inputElement : " + inputElement.toString());
        String key = inputElement.attr("name");
        String value = inputElement.attr("value");

        if (key.equals("username"))
            value = username;
        else if (key.equals("password")) {
            value = password;
        }
        if (!(inputElement.attr("style").contains("none"))) paramList.add(key + "=" + URLEncoder.encode(value, "UTF-8"));
    }

    // build parameters list
    StringBuilder result = new StringBuilder();
    for (String param : paramList) {
        if (result.length() == 0) {
            result.append(param);
        } else {
            result.append("&" + param);
        }
    }

    System.out.println("result : " + result.toString().replace("trusted=4", "SubmitCreds.x=0&SubmitCreds.y=0"));
    return result.toString().replace("trusted=4", "SubmitCreds.x=0&SubmitCreds.y=0");
  }

  public List<String> getCookies() {
    return cookies;
  }

  public void setCookies(List<String> cookies) {
    this.cookies = cookies;
  }

}

the Output is the following :

Sending 'GET' request to URL : https://www.entea.fr/
Response Code : 200
Extracting form's data...
inputElement : <input type="password" id="autocompleteoff" style="display:none" />
inputElement : <input type="hidden" id="curl" name="curl" value="Z2F" />
inputElement : <input type="hidden" id="flags" name="flags" value="0" />
inputElement : <input type="hidden" id="forcedownlevel" name="forcedownlevel" value="0" />
inputElement : <input type="hidden" id="formdir" name="formdir" value="7" />
inputElement : <input type="text" id="username" name="username" />
inputElement : <input id="password" onfocus="g_fFcs=0" type="password" name="password" autocomplete="off" />
inputElement : <input type="radio" name="trusted" id="rdoPblc" value="0" onclick="clkSec()" checked="checked" />
inputElement : <input type="radio" name="trusted" id="rdoPrvt" value="4" onclick="clkSec()" />
inputElement : <input id="SubmitCreds" onclick="clkLgn()" type="image" value="Ouvrir une session" name="SubmitCreds" src="/CookieAuth.dll?GetPic?formdir=7&amp;image=encart_footer_01.png" alt="Connexion" />
result : curl=Z2F&flags=0&forcedownlevel=0&formdir=7&username=xxxxxxxxx&password=xxxxxxxxxxx*&trusted=0&SubmitCreds.x=0&SubmitCreds.y=0&SubmitCreds=Ouvrir+une+session
Exception in thread "main" java.lang.NullPointerException
    at htmlmKyongTest.HttpUrlConnectionExample.sendPost(HttpUrlConnectionExample.java:66)
    at htmlmKyongTest.HttpUrlConnectionExample.main(HttpUrlConnectionExample.java:46)

Alexis Hassler · Accepted Answer · 2014-11-03T20:28:14.653

The cookies field isn't initialized for the first sendPost. You should protect your for loop with a if (cookies != null).

EDIT : The actual solution is to split the initial request + redirect into 2 separate GET requests.

Try this :

import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.CookieHandler;
import java.net.CookieManager;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;

import javax.net.ssl.HttpsURLConnection;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class HttpUrlConnectionExample {

    private List<String> cookies;
    private HttpsURLConnection conn;

    private final String USER_AGENT = "Mozilla/5.0";

    public static void main(String[] args) throws Exception {
        String url = "https://www.entea.fr/";
        String gmail = "https://www.entea.fr/etabs/0671688W/Pages/Accueil.aspx";

        HttpUrlConnectionExample http = new HttpUrlConnectionExample();

        // make sure cookies is turn on
        CookieHandler.setDefault(new CookieManager());

        // 1. Send a "GET" request, so that you can extract the form's data.
        http.getPageContent(url);
        String page = http.getPageContent("https://www.entea.fr/CookieAuth.dll?GetLogon?curl=Z2F&reason=0&formdir=7");

        String postParams = http.getFormParams(page, "myUsername", "myPassword");

        // 2. Construct above post's content and then send a POST request for
        // authentication
        http.sendPost(url, postParams);

        // 3. success
        String result = http.getPageContent(gmail);
        System.out.println(result);
    }

    private void sendPost(String url, String postParams) throws Exception {

        URL obj = new URL(url);
        conn = (HttpsURLConnection) obj.openConnection();

        // Acts like a browser
        conn.setUseCaches(false);
        conn.setRequestMethod("POST");
        conn.setRequestProperty("Host", "www.entea.fr");
        conn.setRequestProperty("User-Agent", USER_AGENT);
        conn.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
        conn.setRequestProperty("Accept-Language", "fr-FR,fr;q=0.8,en-US;q=0.6,en;q=0.4");
        conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
        for (String cookie : this.cookies) {
            conn.addRequestProperty("Set-Cookie", cookie.split(";", 1)[0]);
        }

        conn.setRequestProperty("Connection", "keep-alive");
        conn.setRequestProperty("Referer", "https://www.entea.fr/CookieAuth.dll?GetLogon?curl=Z2F&reason=0&formdir=7");
        conn.setRequestProperty("Accept-Encoding", "gzip,deflate");
        conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
        conn.setRequestProperty("Content-Length", Integer.toString(postParams.length()));
        conn.setDoOutput(true);
        conn.setDoInput(true);

        // Send post request
        DataOutputStream wr = new DataOutputStream(conn.getOutputStream());
        wr.writeBytes(postParams);
        wr.flush();
        wr.close();

        int responseCode = conn.getResponseCode();
        System.out.println("\nSending 'POST' request to URL : " + url);
        System.out.println("Post parameters : " + postParams);
        System.out.println("Response Code : " + responseCode);

        BufferedReader in =
                new BufferedReader(new InputStreamReader(conn.getInputStream()));
        String inputLine;
        StringBuffer response = new StringBuffer();

        while ((inputLine = in.readLine()) != null) {
            response.append(inputLine);
        }
        in.close();

    }

    private String getPageContent(String url) throws Exception {

        URL obj = new URL(url);
        conn = (HttpsURLConnection) obj.openConnection();

        // default is GET
        conn.setRequestMethod("GET");
        conn.setUseCaches(false);

        // act like a browser
        conn.setRequestProperty("User-Agent", USER_AGENT);
        conn.setRequestProperty("Accept",
                "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
        conn.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
        conn.setInstanceFollowRedirects(false);

        if (cookies != null) {
            for (String cookie : this.cookies) {
                conn.addRequestProperty("Set-Cookie", cookie.split(";", 1)[0]);
            }
        }
        int responseCode = conn.getResponseCode();
        System.out.println("\nSending 'GET' request to URL : " + url);
        System.out.println("Response Code : " + responseCode);

        BufferedReader in =
                new BufferedReader(new InputStreamReader(conn.getInputStream()));
        String inputLine;
        StringBuffer response = new StringBuffer();

        while ((inputLine = in.readLine()) != null) {
            response.append(inputLine);
        }
        in.close();

        // Get the response cookies
        setCookies(conn.getHeaderFields().get("Set-Cookie"));
        return response.toString();

    }

    public String getFormParams(String html, String username, String password)
            throws UnsupportedEncodingException {

        System.out.println("Extracting form's data...");
        Document doc = Jsoup.parse(html);

        Element loginform = doc.getElementById("logonForm");
        Elements inputElements = loginform.getElementsByTag("input");
        List<String> paramList = new ArrayList<String>();
        for (Element inputElement : inputElements) {
            System.out.println("inputElement : " + inputElement.toString());
            String key = inputElement.attr("name");
            String value = inputElement.attr("value");

            if (key.equals("username"))
                value = username;
            else if (key.equals("password")) {
                value = password;
            }
            if (!(inputElement.attr("style").contains("none")))
                paramList.add(key + "=" + URLEncoder.encode(value, "UTF-8"));
        }

        // build parameters list
        StringBuilder result = new StringBuilder();
        for (String param : paramList) {
            if (result.length() == 0) {
                result.append(param);
            } else {
                result.append("&" + param);
            }
        }

        System.out.println("result : " + result.toString().replace("trusted=4", "SubmitCreds.x=0&SubmitCreds.y=0"));
        return result.toString().replace("trusted=4", "SubmitCreds.x=0&SubmitCreds.y=0");
    }

    public void setCookies(List<String> cookies) {
        if (cookies != null) {
            this.cookies = cookies;
        }
    }

}

There is only one SendPost. The problem is : why are the cookies null. — Stéphane GROSSMANN, Nov 03 '14 at 15:18
OK, I didn't see the previous GET request. Cookies should come from the previous response, and if you run you program in debug mode, you'll see that there's no cookie in response. — Alexis Hassler, Nov 03 '14 at 16:59
I see that the first request is sent to https://www.entea.fr/ wich respond a 302 to an other URL. The 302 response has a cookie, but not the redirected one. The solution would be to set the followRedirect property to false on the connection object. Be careful, you'll have to do the redirect request by your own. — Alexis Hassler, Nov 03 '14 at 17:19
Tell me if it works, so that I can update the answer with the most relevant part of the comments. — Alexis Hassler, Nov 03 '14 at 17:56
I set the folloWRedirect Property to false and did my SendPost on the new Url. No luck. Besides, when I put the new Url in a browser, an error page is launched. So strange .... — Stéphane GROSSMANN, Nov 03 '14 at 18:50
You have to send 2 GET requests : `http.getPageContent(url);` `String page = http.getPageContent("https://www.entea.fr/CookieAuth.dll?GetLogon?curl=Z2F&reason=0&formdir=7");` — Alexis Hassler, Nov 03 '14 at 19:26
One more change : call setCookies() only if response has cookies so that you don't loose the initial cookie. — Alexis Hassler, Nov 03 '14 at 19:59
I edited the answer with a new version of your code. It seems to work. — Alexis Hassler, Nov 03 '14 at 20:29
So great ! Thanks ! Instead of conn.setInstanceFollowRedirects(false), I did conn.setFollowRedirects(false); Merci :-) — Stéphane GROSSMANN, Nov 03 '14 at 20:43

Cookies not set with HttpUrlConnection

1 Answers1