Save all data available from the ws stream.

writer.go

@@ -7,12 +7,31 @@ import (
 	"time"
 )
 
+const insertTickSQL = `
+INSERT OR IGNORE INTO btc_ticks (
+    seq,
+    trade_id,
+    trade_ts,
+    message_ts,
+    recv_ts,
+    symbol,
+    side,
+    price,
+    volume,
+    tick_dir,
+    block_trade,
+    rpi
+)
+VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+`
+
 // Writer drains the tick channel and batch-writes to hot_ticks.db.
 type Writer struct {
 	tickCh    <-chan Tick
 	hotDB     *sql.DB
 	batchSize int
 	flushMs   int
+	lastSeq int64
 }
 
 // NewWriter creates a Writer with its own hot DB connection.
@@ -76,12 +95,30 @@ func (w *Writer) flush(batch []Tick) {
 	if len(batch) == 0 {
 		return
 	}
+
 	tx, err := w.hotDB.Begin()
 	if err != nil {
 		log.Printf("[writer] begin tx failed: %v", err)
 		return
 	}
-	stmt, err := tx.Prepare("INSERT INTO btc_ticks (timestamp, price, volume, side) VALUES (?, ?, ?, ?)")
+
+	stmt, err := tx.Prepare(`
+		INSERT OR IGNORE INTO btc_ticks (
+			seq,
+			trade_id,
+			trade_ts,
+			message_ts,
+			recv_ts,
+			symbol,
+			side,
+			price,
+			volume,
+			tick_dir,
+			block_trade,
+			rpi
+		)
+		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+	`)
 	if err != nil {
 		log.Printf("[writer] prepare failed: %v", err)
 		tx.Rollback()
@@ -90,15 +127,43 @@ func (w *Writer) flush(batch []Tick) {
 	defer stmt.Close()
 
 	for _, t := range batch {
-		if _, err := stmt.Exec(t.Timestamp, t.Price, t.Volume, t.Side); err != nil {
+
+        if t.Seq < w.lastSeq {
+            log.Printf(
+                "[writer] OUT OF ORDER SEQ: prev=%d current=%d",
+                w.lastSeq,
+                t.Seq,
+            )
+        }
+
+		w.lastSeq = t.Seq
+
+		_, err := stmt.Exec(
+			t.Seq,
+			t.TradeID,
+			t.TradeTS,
+			t.MessageTS,
+			t.RecvTS,
+			t.Symbol,
+			t.Side,
+			t.Price,
+			t.Volume,
+			t.TickDir,
+			t.BlockTrade,
+			t.RPI,
+		)
+
+		if err != nil {
 			log.Printf("[writer] insert failed: %v", err)
 			tx.Rollback()
 			return
 		}
 	}
+
 	if err := tx.Commit(); err != nil {
 		log.Printf("[writer] commit failed: %v", err)
 		return
 	}
+
 	log.Printf("[writer] Flushed %d ticks to hot_ticks.db", len(batch))
 }