I'm getting this error I'm inserting the data into the Clickhouse and i get this error because some of fields are null. Unknown error field: Code: 117. DB::Exception: Unknown field found while parsing JSONEachRow format: SharesStats: (at row 1): While executing JSONEachRowRowInputFormat. (INCORRECT_DATA) (version 21.11.6.7 (official build))
const Clickhouse = require('@apla/clickhouse');
const { default: axios } = require('axios');
const stockCode = require('./stockCode');
const clickhouse = new Clickhouse({
host: 'localhost',
port: 8123,
username: 'default',
password: '',
dataObjects: true,
queryOptions: {
database: 'stocks'
}
});
const indexStream = clickhouse.query(
'INSERT INTO test2',
{
format: 'JSONEachRow'
},
(err) => {
if (err) {
return console.error(err);
}
}
);
const fetchDataFundamentals = async (stock) => {
try {
const { data: response } = await axios.get(
`https://eodhistoricaldata.com/api/fundamentals/${stock}?api_token=${apiKey}&fmt=json`
);
const nestedFields = [
'SharesStats',
'Technicals',
'SplitsDividends',
'outstandingShares',
'Earnings',
'Financials'
];
const data = {
...response.General,
...response.Highlights,
...response.Valuation,
...response.Holders,
...response.InsiderTransactions,
SharesStats: response.SharesStats,
Technicals: response.Technicals,
SplitsDividends: response.SplitsDividends,
outstandingShares: response.outstandingShares,
Earnings: response.Earnings,
Financials: response.Financials
};
Object.keys(data).forEach((key) => {
if (nestedFields.includes(key)) {
data[key] = Object.keys(data[key]).map((nestedKey) => {
if (!data[key][nestedKey]) {
data[key][nestedKey] = 0;
} else if (typeof data[key][nestedKey] === 'object') {
data[key][nestedKey] = JSON.stringify(data[key][nestedKey]);
}
return data[key][nestedKey];
});
} else if (!data[key]) {
delete data[key];
} else if (typeof data[key] === 'object') {
data[key] = JSON.stringify(data[key]);
}
});
console.log(data);
indexStream.write(data);
indexStream.end();
} catch (error) {
console.error(error.message);
}
};
// Promise.all(stockCode.map((code) => fetchDataFundamentals(code)))
// .then(() => {
// indexStream.end();
// console.log('Fundametals data inserted');
// })
// .catch((err) => {
// console.log(err.message);
// });
try {
fetchDataFundamentals('1010.SR');
} catch (error) {
console.error(error.message);
}