Restify: Retrieve external url content (ISO-8859-1) return to requesting client (UTF-8)

Question

I have a very simple javascript using Node. The purpose of the script is to:

Open a listening socket
Add a handler for URL /test with HTTP function GET
When /test is requested another URL located on another external web site should be fetched. This web site is encoded with ISO-8859-1
The data returned from the external website should be packaged in a JSON structure and returned to the requesting client encoded with UTF-8

So far I have created the following code:

var Buffer = require('buffer').Buffer;
var iconv  = require('iconv-lite');
var urllib = require('url');
var restify = require('restify');
var server = restify.createServer();

server.use(restify.bodyParser());
server.get('/test', test);
server.listen(8080, function() {
  console.log('%s listening at %s', server.name, server.url);
});

function test(req, res, next) {
    console.log('TEST');

    var httpClient = restify.createStringClient({ url: "http://dl.dropboxusercontent.com" });
    httpClient.get("/u/815962/iso-8859-1.html", function(cerr, creq, cres, cdata) {
        cdata = iconv.decode(cdata, 'iso-8859-1');

        res.send(200, {"Data": cdata}); 
    });     
}

I have set up a test document used in the code above. The test document is in ISO-8859-1 encoding and has the national letters "ÅÄÖåäö" inside it. When returned to the client, if read like UTF-8 I receive "ýýýýýý"

score 0 · Answer 1 · answered Sep 27 '13 at 23:26

It really seem that this is a bug in Restify. The following example shows different results using Restify and Request lib:

var request = require('request');                                               
var iconv = require('iconv');                                                   
var restify = require('restify');
var ic = new iconv.Iconv('iso-8859-1', 'utf-8');                              

request.get({ url: 'http://dl.dropboxusercontent.com/u/815962/iso-8859-1.html', encoding: null, }, function(err, res, body) {      
    var buf = ic.convert(body);                                                   
    var utf8String = buf.toString('utf-8');  
    console.log(utf8String);
});  

var httpClient = restify.createStringClient({ url: "http://dl.dropboxusercontent.com" });
httpClient.get("/u/815962/iso-8859-1.html", function(cerr, creq, cres, cdata) {
    var buf = ic.convert(cdata);                                                   
    var utf8String = buf.toString('utf-8'); 
    console.log(utf8String);
});

I have browsed the code of Restify on github trying to find the issue, but I can't

Restify: Retrieve external url content (ISO-8859-1) return to requesting client (UTF-8)

1 Answers1