Added klines, liquidations, tickers and trades to be recorded. Bundled the stats as cli argument.

ticker_handler.go

@@ -0,0 +1,214 @@
+package main
+
+import (
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"log"
+	"strconv"
+	"sync"
+	"time"
+)
+
+// TickerHandler processes Bybit tickers stream data, snapshotting to hot_ticker.db and computing features for features.db.
+type TickerHandler struct {
+	cfg         Config
+	storage     *StreamStorage
+	mu          sync.Mutex
+	latest      TickerSnapshot
+	hasData     bool
+	prevOI      float64
+	hasPrevOI   bool
+	hotDB       *sql.DB
+	featDB      *sql.DB
+	stopChan    chan struct{}
+	wg          sync.WaitGroup
+}
+
+func NewTickerHandler(cfg Config, sm *StorageManager) (*TickerHandler, error) {
+	ss := sm.GetStreamStorage("ticker")
+	if ss == nil {
+		return nil, fmt.Errorf("ticker stream storage not found")
+	}
+
+	hotDB, err := OpenDBWithAutoVacuum(ss.DBPath("hot_ticker.db"))
+	if err != nil {
+		return nil, fmt.Errorf("open hot_ticker db: %w", err)
+	}
+
+	featDB, err := OpenDBWithAutoVacuum(ss.DBPath("features.db"))
+	if err != nil {
+		hotDB.Close()
+		return nil, fmt.Errorf("open ticker features db: %w", err)
+	}
+
+	th := &TickerHandler{
+		cfg:      cfg,
+		storage:  ss,
+		hotDB:    hotDB,
+		featDB:   featDB,
+		stopChan: make(chan struct{}),
+	}
+
+	// Load last open interest for continuity
+	var lastOI float64
+	err = featDB.QueryRow("SELECT open_interest FROM ticker_snapshots ORDER BY timestamp DESC LIMIT 1").Scan(&lastOI)
+	if err == nil {
+		th.prevOI = lastOI
+		th.hasPrevOI = true
+	}
+
+	// Start periodic snapshot worker
+	interval := time.Duration(cfg.Streams.Ticker.SnapshotIntervalMs) * time.Millisecond
+	if interval <= 0 {
+		interval = 5 * time.Second
+	}
+
+	th.wg.Add(1)
+	go th.runSnapshotLoop(interval)
+
+	return th, nil
+}
+
+func (th *TickerHandler) Topics() []string {
+	return []string{fmt.Sprintf("tickers.%s", th.cfg.Symbol)}
+}
+
+func (th *TickerHandler) HandleMessage(data []byte) {
+	var msg BybitTickerMessage
+	if err := json.Unmarshal(data, &msg); err != nil {
+		return
+	}
+
+	raw := msg.Data
+	if raw.Symbol == "" && th.cfg.Symbol != "" {
+		raw.Symbol = th.cfg.Symbol
+	}
+
+	th.mu.Lock()
+	defer th.mu.Unlock()
+
+	// Update existing state with non-empty delta fields
+	if p, err := strconv.ParseFloat(raw.LastPrice, 64); err == nil && p > 0 {
+		th.latest.LastPrice = p
+	}
+	if p, err := strconv.ParseFloat(raw.Bid1Price, 64); err == nil && p > 0 {
+		th.latest.Bid1Price = p
+	}
+	if s, err := strconv.ParseFloat(raw.Bid1Size, 64); err == nil && s >= 0 {
+		th.latest.Bid1Size = s
+	}
+	if p, err := strconv.ParseFloat(raw.Ask1Price, 64); err == nil && p > 0 {
+		th.latest.Ask1Price = p
+	}
+	if s, err := strconv.ParseFloat(raw.Ask1Size, 64); err == nil && s >= 0 {
+		th.latest.Ask1Size = s
+	}
+	if p, err := strconv.ParseFloat(raw.MarkPrice, 64); err == nil && p > 0 {
+		th.latest.MarkPrice = p
+	}
+	if p, err := strconv.ParseFloat(raw.IndexPrice, 64); err == nil && p > 0 {
+		th.latest.IndexPrice = p
+	}
+	if oi, err := strconv.ParseFloat(raw.OpenInterest, 64); err == nil && oi >= 0 {
+		th.latest.OpenInterest = oi
+	}
+	if fr, err := strconv.ParseFloat(raw.FundingRate, 64); err == nil {
+		th.latest.FundingRate = fr
+	}
+	if v, err := strconv.ParseFloat(raw.Volume24h, 64); err == nil && v >= 0 {
+		th.latest.Volume24h = v
+	}
+	if t, err := strconv.ParseFloat(raw.Turnover24h, 64); err == nil && t >= 0 {
+		th.latest.Turnover24h = t
+	}
+	th.latest.Timestamp = msg.TS
+	if th.latest.Timestamp == 0 {
+		th.latest.Timestamp = time.Now().UnixMilli()
+	}
+	th.hasData = true
+}
+
+func (th *TickerHandler) runSnapshotLoop(interval time.Duration) {
+	defer th.wg.Done()
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-th.stopChan:
+			th.takeSnapshot()
+			return
+		case <-ticker.C:
+			th.takeSnapshot()
+		}
+	}
+}
+
+func (th *TickerHandler) takeSnapshot() {
+	th.mu.Lock()
+	if !th.hasData {
+		th.mu.Unlock()
+		return
+	}
+	snap := th.latest
+	prevOI := th.prevOI
+	hasPrevOI := th.hasPrevOI
+	th.prevOI = snap.OpenInterest
+	th.hasPrevOI = true
+	th.mu.Unlock()
+
+	ts := (snap.Timestamp / 5000) * 5000
+
+	// Write snapshot to hot DB
+	_, err := th.hotDB.Exec(`
+		INSERT INTO ticker_snapshots (
+			timestamp, last_price, bid1_price, bid1_size, ask1_price, ask1_size,
+			mark_price, index_price, open_interest, funding_rate, volume_24h, turnover_24h
+		) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+	`, ts, snap.LastPrice, snap.Bid1Price, snap.Bid1Size, snap.Ask1Price, snap.Ask1Size,
+		snap.MarkPrice, snap.IndexPrice, snap.OpenInterest, snap.FundingRate, snap.Volume24h, snap.Turnover24h)
+	if err != nil {
+		log.Printf("[ticker_handler] hot db insert error: %v", err)
+	}
+
+	// Calculate and write feature
+	spread := snap.Ask1Price - snap.Bid1Price
+	midPrice := (snap.Bid1Price + snap.Ask1Price) / 2.0
+	spreadBps := 0.0
+	if midPrice > 0 {
+		spreadBps = (spread / midPrice) * 10000.0
+	}
+
+	oiChange := 0.0
+	if hasPrevOI {
+		oiChange = snap.OpenInterest - prevOI
+	}
+
+	markIndexBasis := snap.MarkPrice - snap.IndexPrice
+	totalSize := snap.Bid1Size + snap.Ask1Size
+	bidAskImbalance := 0.0
+	if totalSize > 0 {
+		bidAskImbalance = snap.Bid1Size / totalSize
+	}
+
+	_, err = th.featDB.Exec(`
+		INSERT OR IGNORE INTO ticker_features (
+			timestamp, spread, spread_bps, mid_price, oi_change, funding_rate, mark_index_basis, bid_ask_imbalance
+		) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+	`, ts, spread, spreadBps, midPrice, oiChange, snap.FundingRate, markIndexBasis, bidAskImbalance)
+	if err != nil {
+		log.Printf("[ticker_handler] features db insert error: %v", err)
+	}
+}
+
+func (th *TickerHandler) Close() {
+	close(th.stopChan)
+	th.wg.Wait()
+	if th.hotDB != nil {
+		th.hotDB.Close()
+	}
+	if th.featDB != nil {
+		th.featDB.Close()
+	}
+}