0

I would like to do insert from a stream, like in the Microsoft docs below but with intermittent commits. Why do I need to add the lines with question mark to let it work? Are there better alternatives to write this code? not sure why I just can't commit multiple times on the same transaction.

https://learn.microsoft.com/en-us/dotnet/standard/data/sqlite/bulk-insert

private static void CreateDatabase(IEnumerable<Ticker> ticks)
{
    var currentDate = DateTime.Now.ToString("yyyyMMddHHmmss");

    using (var connection = new SqliteConnection(@"Data Source=D:\ticks_" + currentDate + ".db"))
    {

        connection.Open();


        var createCommandStr = "CREATE TABLE ticker (id INTEGER NOT NULL, exchange_id INTEGER NOT NULL, time INTEGER NOT NULL, " +
                    "name STRING NOT NULL, price NUMERIC NOT NULL, size NUMERIC NOT NULL, " +
                    "bid_price NUMERIC NOT NULL, bid_size NUMERIC NOT NULL, ask_price NUMERIC NOT NULL, " +
                    "ask_size NUMERIC NOT NULL, PRIMARY KEY(id))";

        using (var createCommand = new SqliteCommand(createCommandStr, connection))
        {
            createCommand.ExecuteNonQuery();
        }

        var insertCommandStr = "INSERT INTO ticker (id, exchange_id, time, name, price, size, bid_price, bid_size, ask_price, ask_size) " +
                "VALUES ($id, $exchange_id, $time, $name, $price, $size, $bid_price, $bid_size, $ask_price, $ask_size)";

        ////deferred: true
        //using (var transaction = connection.BeginTransaction(deferred: true))
        //{
        using (var insertCommand = new SqliteCommand(insertCommandStr, connection, connection.BeginTransaction()))
        {
            #region shitRegion
            var id = insertCommand.CreateParameter();
            id.ParameterName = "$id";
            insertCommand.Parameters.Add(id);

            var exchangeId = insertCommand.CreateParameter();
            exchangeId.ParameterName = "$exchange_id";
            insertCommand.Parameters.Add(exchangeId);

            var time = insertCommand.CreateParameter();
            time.ParameterName = "$time";
            insertCommand.Parameters.Add(time);

            var name = insertCommand.CreateParameter();
            name.ParameterName = "$name";
            insertCommand.Parameters.Add(name);

            var price = insertCommand.CreateParameter();
            price.ParameterName = "$price";
            insertCommand.Parameters.Add(price);

            var size = insertCommand.CreateParameter();
            size.ParameterName = "$size";
            insertCommand.Parameters.Add(size);

            var bid_price = insertCommand.CreateParameter();
            bid_price.ParameterName = "$bid_price";
            insertCommand.Parameters.Add(bid_price);

            var bid_size = insertCommand.CreateParameter();
            bid_size.ParameterName = "$bid_size";
            insertCommand.Parameters.Add(bid_size);

            var ask_price = insertCommand.CreateParameter();
            ask_price.ParameterName = "$ask_price";
            insertCommand.Parameters.Add(ask_price);

            var ask_size = insertCommand.CreateParameter();
            ask_size.ParameterName = "$ask_size";
            insertCommand.Parameters.Add(ask_size);

            #endregion

            insertCommand.Prepare();

            long count = 1L;

            foreach (var ticker in ticks)
            {
                id.Value = count;
                exchangeId.Value = ticker.ExchangeId;
                time.Value = ticker.Time;
                name.Value = ticker.Name;
                price.Value = ticker.Price;
                size.Value = ticker.Size;
                bid_price.Value = ticker.BidPrice;
                bid_size.Value = ticker.BidSize;
                ask_price.Value = ticker.AskPrice;
                ask_size.Value = ticker.AskSize;

                insertCommand.ExecuteNonQuery();

                if (count % 150000L == 0L)
                {
                    //transaction.Commit();
                    insertCommand.Transaction.Commit();
                    insertCommand.Connection = connection;//?
                    insertCommand.Transaction = connection.BeginTransaction();//?
                }

                count++;
            }
        }
    }
}
BigChief
  • 1,413
  • 4
  • 24
  • 37
  • 1
    Why not create a new command and transaction for every tick group? – Guru Stron Apr 16 '21 at 18:24
  • I'm not sure if I understand your comment? You think I shouldn't follow the steps in the msdn page? – BigChief Apr 16 '21 at 18:37
  • 1
    I'm saying that you can follow it but split you ticks in batches (for example like [here](https://stackoverflow.com/q/13709626/2501279)) and then use the code from msdn for every batch. – Guru Stron Apr 16 '21 at 18:43
  • ah yes.. I understand what you say but that would be less a continuous stream right, since you would wait for the 150k tick to receive before you start the transaction (and initialize the insert command etcetera again), would like to insert it more as a stream – BigChief Apr 16 '21 at 18:54
  • 1
    So when you ask "why can't we commit multiple times in the same transaction" does it mean you think of a transaction as like a Microsoft word document, that you open, edit a bit, save, edit a bit more, save again.. ? – Caius Jard Apr 16 '21 at 19:08
  • exactly that https://docs.sqlalchemy.org/en/14/faq/performance.html here the init_sqlite3 return object you can commit multiple times subsequently with insertions, how to in C#?? – BigChief Apr 16 '21 at 19:33
  • 1
    @BigChief are you sure that there is the same underlying db transaction? – Guru Stron Apr 16 '21 at 19:59
  • no I'm not really sure and also don't know how to figure that out – BigChief Apr 16 '21 at 20:57

0 Answers0