I would like to do insert from a stream, like in the Microsoft docs below but with intermittent commits. Why do I need to add the lines with question mark to let it work? Are there better alternatives to write this code? not sure why I just can't commit multiple times on the same transaction.
https://learn.microsoft.com/en-us/dotnet/standard/data/sqlite/bulk-insert
private static void CreateDatabase(IEnumerable<Ticker> ticks)
{
var currentDate = DateTime.Now.ToString("yyyyMMddHHmmss");
using (var connection = new SqliteConnection(@"Data Source=D:\ticks_" + currentDate + ".db"))
{
connection.Open();
var createCommandStr = "CREATE TABLE ticker (id INTEGER NOT NULL, exchange_id INTEGER NOT NULL, time INTEGER NOT NULL, " +
"name STRING NOT NULL, price NUMERIC NOT NULL, size NUMERIC NOT NULL, " +
"bid_price NUMERIC NOT NULL, bid_size NUMERIC NOT NULL, ask_price NUMERIC NOT NULL, " +
"ask_size NUMERIC NOT NULL, PRIMARY KEY(id))";
using (var createCommand = new SqliteCommand(createCommandStr, connection))
{
createCommand.ExecuteNonQuery();
}
var insertCommandStr = "INSERT INTO ticker (id, exchange_id, time, name, price, size, bid_price, bid_size, ask_price, ask_size) " +
"VALUES ($id, $exchange_id, $time, $name, $price, $size, $bid_price, $bid_size, $ask_price, $ask_size)";
////deferred: true
//using (var transaction = connection.BeginTransaction(deferred: true))
//{
using (var insertCommand = new SqliteCommand(insertCommandStr, connection, connection.BeginTransaction()))
{
#region shitRegion
var id = insertCommand.CreateParameter();
id.ParameterName = "$id";
insertCommand.Parameters.Add(id);
var exchangeId = insertCommand.CreateParameter();
exchangeId.ParameterName = "$exchange_id";
insertCommand.Parameters.Add(exchangeId);
var time = insertCommand.CreateParameter();
time.ParameterName = "$time";
insertCommand.Parameters.Add(time);
var name = insertCommand.CreateParameter();
name.ParameterName = "$name";
insertCommand.Parameters.Add(name);
var price = insertCommand.CreateParameter();
price.ParameterName = "$price";
insertCommand.Parameters.Add(price);
var size = insertCommand.CreateParameter();
size.ParameterName = "$size";
insertCommand.Parameters.Add(size);
var bid_price = insertCommand.CreateParameter();
bid_price.ParameterName = "$bid_price";
insertCommand.Parameters.Add(bid_price);
var bid_size = insertCommand.CreateParameter();
bid_size.ParameterName = "$bid_size";
insertCommand.Parameters.Add(bid_size);
var ask_price = insertCommand.CreateParameter();
ask_price.ParameterName = "$ask_price";
insertCommand.Parameters.Add(ask_price);
var ask_size = insertCommand.CreateParameter();
ask_size.ParameterName = "$ask_size";
insertCommand.Parameters.Add(ask_size);
#endregion
insertCommand.Prepare();
long count = 1L;
foreach (var ticker in ticks)
{
id.Value = count;
exchangeId.Value = ticker.ExchangeId;
time.Value = ticker.Time;
name.Value = ticker.Name;
price.Value = ticker.Price;
size.Value = ticker.Size;
bid_price.Value = ticker.BidPrice;
bid_size.Value = ticker.BidSize;
ask_price.Value = ticker.AskPrice;
ask_size.Value = ticker.AskSize;
insertCommand.ExecuteNonQuery();
if (count % 150000L == 0L)
{
//transaction.Commit();
insertCommand.Transaction.Commit();
insertCommand.Connection = connection;//?
insertCommand.Transaction = connection.BeginTransaction();//?
}
count++;
}
}
}
}