0

I'm getting this error I'm inserting the data into the Clickhouse and i get this error because some of fields are null. Unknown error field: Code: 117. DB::Exception: Unknown field found while parsing JSONEachRow format: SharesStats: (at row 1): While executing JSONEachRowRowInputFormat. (INCORRECT_DATA) (version 21.11.6.7 (official build))

        const Clickhouse = require('@apla/clickhouse');
        const { default: axios } = require('axios');
        
        const stockCode = require('./stockCode');
        
        const clickhouse = new Clickhouse({
          host: 'localhost',
          port: 8123,
          username: 'default',
          password: '',
          dataObjects: true,
          queryOptions: {
            database: 'stocks'
          }
        });
        
        const indexStream = clickhouse.query(
          'INSERT INTO test2',
          {
            format: 'JSONEachRow'
          },
          (err) => {
            if (err) {
              return console.error(err);
            }
          }
        );
        
        const fetchDataFundamentals = async (stock) => {
          try {
            const { data: response } = await axios.get(
              `https://eodhistoricaldata.com/api/fundamentals/${stock}?api_token=${apiKey}&fmt=json`
            );
        
            const nestedFields = [
              'SharesStats',
              'Technicals',
              'SplitsDividends',
              'outstandingShares',
              'Earnings',
              'Financials'
            ];
        
            const data = {
              ...response.General,
              ...response.Highlights,
              ...response.Valuation,
              ...response.Holders,
              ...response.InsiderTransactions,
              SharesStats: response.SharesStats,
              Technicals: response.Technicals,
              SplitsDividends: response.SplitsDividends,
              outstandingShares: response.outstandingShares,
              Earnings: response.Earnings,
              Financials: response.Financials
            };
        
            Object.keys(data).forEach((key) => {
              if (nestedFields.includes(key)) {
                data[key] = Object.keys(data[key]).map((nestedKey) => {
                  if (!data[key][nestedKey]) {
                    data[key][nestedKey] = 0;
                  } else if (typeof data[key][nestedKey] === 'object') {
                    data[key][nestedKey] = JSON.stringify(data[key][nestedKey]);
                  }
        
                  return data[key][nestedKey];
                });
              } else if (!data[key]) {
                delete data[key];
              } else if (typeof data[key] === 'object') {
                data[key] = JSON.stringify(data[key]);
              }
            });
        
            console.log(data);
        
            indexStream.write(data);
            indexStream.end();
          } catch (error) {
            console.error(error.message);
          }
        };
        
        // Promise.all(stockCode.map((code) => fetchDataFundamentals(code)))
        //   .then(() => {
        //     indexStream.end();
        
        //     console.log('Fundametals data inserted');
        //   })
        //   .catch((err) => {
        //     console.log(err.message);
        //   });
        
        try {
          fetchDataFundamentals('1010.SR');
        } catch (error) {
          console.error(error.message);
        }
LukStorms
  • 28,916
  • 5
  • 31
  • 45
Noman
  • 594
  • 1
  • 18

0 Answers0