expoting big data to csv file - carshes

Question

Im trying to export big data to csv files. (above 20000 lines... can get to over 100000 lines easily). after I try download the file, it crashes - download failed due to network failures. (managed to download an 18000 lines file, that weighs 1.7MB, the code worked perfectly. above 20000 the download crashes)

heres my code... thanks!

edit- works on IE, doesnt work in chrome

var data2 =     [[data12]];
var csvContent2 = "";
data2.forEach(function (infoArray, index) {

    dataString = Array.prototype.join.call(infoArray, "");
    csvContent2 += index < data2.length ? dataString + '\n' : dataString;

});

var download = function(content, fileName, mimeType) {
var a = document.createElement('a');
    mimeType = mimeType || 'application/octet-stream';

    if (navigator.msSaveBlob) { // IE10
        return navigator.msSaveBlob(new Blob([content], { type: mimeType }), fileName);
    } else if ('download' in a) { //html5 A[download]
        a.href = 'data:' + mimeType + ',' + encodeURIComponent(content);
        a.setAttribute('download', fileName);
        document.body.appendChild(a);
        setTimeout(function() {
        a.click();
        document.body.removeChild(a);
        }, 66);
    return true;
} else { //do iframe dataURL download (old ch+FF):
    var f = document.createElement('iframe');
    document.body.appendChild(f);
    f.src = 'data:' + mimeType + ',' + encodeURIComponent(content);

    setTimeout(function() {
        document.body.removeChild(f);
    }, 333);
    return true;
    }
}

    download(csvContent2, 'GroupB.csv', 'text/csv');

try without removing the `` tag you create to attach the Blob... maybe there some kind of link between the tag and the blob ;) — Patrick Ferreira, Sep 22 '16 at 12:16
I think it is a good idea to add the error message you are getting, because "crashes" does not explain much by itself .. — Alexander, Sep 22 '16 at 12:16
I dont know... how can I check it out? anyway, it works in IE... but crashes in chrome... — badbuda, Sep 22 '16 at 12:25
Copy the link in source code and then paste it to a software which can count characters (like word, for example). — Aethyn, Sep 22 '16 at 12:32

score 0 · Answer 1 · answered Sep 22 '16 at 12:25

Warning, there are some limits with URL, for example total length of the URL including a GET, must not exceed 2000+ characters on IE, maybe you'll hit the maximum size with a large CSV file ?
How long is your URI ? See this link about IE URI limit.

Edit : Maybe you can try with this createObjectURL(blob), but this is NOT a stabilized feature :x

score 0 · Answer 2 · answered Sep 22 '16 at 15:20

never mind, just used this script, from this web- http://jsfiddle.net/4zv6r/

           function cloneAndConvert(){

var jsonData = [{firstName:"Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum", lastName:"Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum", age:46}];
for(i=0;i<90000;i++)
{     
   jsonData.push({firstName:"Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum"+i, lastName:"Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum"+i, age:46});
}

var filteredGridData = JSON.parse(JSON.stringify(jsonData))
JSONToCSVConvertor(filteredGridData, "UserReport.csv", true);

}

function JSONToCSVConvertor(JSONData, ReportTitle, ShowLabel) {

//If JSONData is not an object then JSON.parse will parse the JSON string in an Object
var arrData = typeof JSONData != 'object' ? JSON.parse(JSONData) : JSONData;
var CSV = '';    
//This condition will generate the Label/Header
if (ShowLabel) {
    var row = "";

    //This loop will extract the label from 1st index of on array
    for (var index in arrData[0]) {
        //Now convert each value to string and comma-seprated
        row += index + ',';
    }
    row = row.slice(0, -1);
    //append Label row with line break
    CSV += row + '\r\n';
}

//1st loop is to extract each row
for (var i = 0; i < arrData.length; i++) {
    var row = "";
    //2nd loop will extract each column and convert it in string comma-seprated
    for (var index in arrData[i]) {
        row += '"' + arrData[i][index] + '",';
    }
    row.slice(0, row.length - 1);
    //add a line break after each row
    CSV += row + '\r\n';
}

if (CSV == '') {        
    alert("Invalid data");
    return;
}   

//this trick will generate a temp "a" tag
var link = document.createElement("a");    
link.id="lnkDwnldLnk";

//this part will append the anchor tag and remove it after automatic click
document.body.appendChild(link);

var csv = CSV;  
blob = new Blob([csv], { type: 'text/csv' }); 
var csvUrl = window.webkitURL.createObjectURL(blob);
var filename = 'UserExport.csv';
$("#lnkDwnldLnk")
.attr({
    'download': filename,
    'href': csvUrl
}); 

$('#lnkDwnldLnk')[0].click();    
document.body.removeChild(link);

}

score 0 · Answer 3 · answered Oct 04 '16 at 20:16

If you are dealing with lots of data, then you maybe want to look for a way to handle the memory abit more friendlier by using StreamSaver

async function download(){
    const fileStream = streamSaver.createWriteStream('filename.csv')
    const writer = fileStream.getWriter()
    const encoder = new TextEncoder

    for(let row of rows){
        await writer.ready()
        let binary = encoder.encode(data + "\r\n")
        writer.write(uint8array)
    }

    writer.close()
}

expoting big data to csv file - carshes

3 Answers3