Convert ndjson to json for HTML table

Question

I would like to know if it is possible to convert the ndjson data from this API: https://lichess.org/api/team/overberg-chess-group/users and turn it into an HTML table. I have found some javascript snippets that will convert normal json into an html table but not ndjson. Any help would be appreciated

Answer 1

Looks like the API pulls a file that's very close to JSON. Assuming that the API data is in var sourceData, just a few tweaks are required...

// Add commas between the list of objects...
data = sourceData.split( '}\n{' ).join( '},{' );

// ...remove all carriage return line feeds...
data = data.split( '\r\n' ).join( '|' );

// ...and there's one instance of a double quoted string, 
// eg, "bio":""I am committed..."".  This appears to be
// the only occurrence of this anomaly, so the following
// currently works, but you might run into issues in the
// future, for example, if an attribute is a null string "".
data = data.split( '""' ).join( '"' );

let dataObject = JSON.parse( '[' + data + ']' );

At this point, dataObject contains the object data representing the ndjson data pulled via the API. Note that when you go to place this object into a HTML table, you'll need to convert the pipe characters "|" back into line feeds... This should help you along your way...

Answer 2

It works for me: https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API/Using_Fetch#processing_a_text_file_line_by_line

async function* makeNDIterator(url) {
    const utf8Decoder = new TextDecoder("utf-8");
    const response = await fetch(url, {cache: "no-store"});
    const reader = response.body.getReader();
    let {value: chunk, done: readerDone} = await reader.read();
    chunk = chunk ? utf8Decoder.decode(chunk) : "";

    const newline = /\r?\n/gm;
    let startIndex = 0;

    while (true) {
        const result = newline.exec(chunk);
        if (!result) {
            if (readerDone) break;
            const remainder = chunk.substring(startIndex);
            ({value: chunk, done: readerDone} = await reader.read());
            chunk = remainder + (chunk ? utf8Decoder.decode(chunk) : "");
            startIndex = newline.lastIndex = 0;
            continue;
        }
        yield chunk.substring(startIndex, result.index);
        startIndex = newline.lastIndex;
    }

    if (startIndex < chunk.length) {
        // Last line didn't end in a newline char
        yield chunk.substring(startIndex);
    }
}

async function run(url) {
  for await (const line of makeNDIterator(url)) {
    processLine(line); // implement it! 
  }
}

run("http://localhost:8080/nd-json-stream");