bybit_btcusdt_ingest/ticker_handler.go

package main

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"log"
	"strconv"
	"sync"
	"time"
)

// TickerHandler processes Bybit tickers stream data, snapshotting to hot_ticker.db and computing features for features.db.
type TickerHandler struct {
	cfg         Config
	storage     *StreamStorage
	mu          sync.Mutex
	latest      TickerSnapshot
	hasData     bool
	prevOI      float64
	hasPrevOI   bool
	hotDB       *sql.DB
	featDB      *sql.DB
	stopChan    chan struct{}
	wg          sync.WaitGroup
}

func NewTickerHandler(cfg Config, sm *StorageManager) (*TickerHandler, error) {
	ss := sm.GetStreamStorage("ticker")
	if ss == nil {
		return nil, fmt.Errorf("ticker stream storage not found")
	}

	hotDB, err := OpenDBWithAutoVacuum(ss.DBPath("hot_ticker.db"))
	if err != nil {
		return nil, fmt.Errorf("open hot_ticker db: %w", err)
	}

	featDB, err := OpenDBWithAutoVacuum(ss.DBPath("features.db"))
	if err != nil {
		hotDB.Close()
		return nil, fmt.Errorf("open ticker features db: %w", err)
	}

	th := &TickerHandler{
		cfg:      cfg,
		storage:  ss,
		hotDB:    hotDB,
		featDB:   featDB,
		stopChan: make(chan struct{}),
	}

	// Load last open interest for continuity
	var lastOI float64
	err = featDB.QueryRow("SELECT open_interest FROM ticker_snapshots ORDER BY timestamp DESC LIMIT 1").Scan(&lastOI)
	if err == nil {
		th.prevOI = lastOI
		th.hasPrevOI = true
	}

	// Start periodic snapshot worker
	interval := time.Duration(cfg.Streams.Ticker.SnapshotIntervalMs) * time.Millisecond
	if interval <= 0 {
		interval = 5 * time.Second
	}

	th.wg.Add(1)
	go th.runSnapshotLoop(interval)

	return th, nil
}

func (th *TickerHandler) Topics() []string {
	return []string{fmt.Sprintf("tickers.%s", th.cfg.Symbol)}
}

func (th *TickerHandler) HandleMessage(data []byte) {
	var msg BybitTickerMessage
	if err := json.Unmarshal(data, &msg); err != nil {
		return
	}

	raw := msg.Data
	if raw.Symbol == "" && th.cfg.Symbol != "" {
		raw.Symbol = th.cfg.Symbol
	}

	th.mu.Lock()
	defer th.mu.Unlock()

	// Update existing state with non-empty delta fields
	if p, err := strconv.ParseFloat(raw.LastPrice, 64); err == nil && p > 0 {
		th.latest.LastPrice = p
	}
	if p, err := strconv.ParseFloat(raw.Bid1Price, 64); err == nil && p > 0 {
		th.latest.Bid1Price = p
	}
	if s, err := strconv.ParseFloat(raw.Bid1Size, 64); err == nil && s >= 0 {
		th.latest.Bid1Size = s
	}
	if p, err := strconv.ParseFloat(raw.Ask1Price, 64); err == nil && p > 0 {
		th.latest.Ask1Price = p
	}
	if s, err := strconv.ParseFloat(raw.Ask1Size, 64); err == nil && s >= 0 {
		th.latest.Ask1Size = s
	}
	if p, err := strconv.ParseFloat(raw.MarkPrice, 64); err == nil && p > 0 {
		th.latest.MarkPrice = p
	}
	if p, err := strconv.ParseFloat(raw.IndexPrice, 64); err == nil && p > 0 {
		th.latest.IndexPrice = p
	}
	if oi, err := strconv.ParseFloat(raw.OpenInterest, 64); err == nil && oi >= 0 {
		th.latest.OpenInterest = oi
	}
	if fr, err := strconv.ParseFloat(raw.FundingRate, 64); err == nil {
		th.latest.FundingRate = fr
	}
	if v, err := strconv.ParseFloat(raw.Volume24h, 64); err == nil && v >= 0 {
		th.latest.Volume24h = v
	}
	if t, err := strconv.ParseFloat(raw.Turnover24h, 64); err == nil && t >= 0 {
		th.latest.Turnover24h = t
	}
	th.latest.Timestamp = msg.TS
	if th.latest.Timestamp == 0 {
		th.latest.Timestamp = time.Now().UnixMilli()
	}
	th.hasData = true
}

func (th *TickerHandler) runSnapshotLoop(interval time.Duration) {
	defer th.wg.Done()
	ticker := time.NewTicker(interval)
	defer ticker.Stop()

	for {
		select {
		case <-th.stopChan:
			th.takeSnapshot()
			return
		case <-ticker.C:
			th.takeSnapshot()
		}
	}
}

func (th *TickerHandler) takeSnapshot() {
	th.mu.Lock()
	if !th.hasData {
		th.mu.Unlock()
		return
	}
	snap := th.latest
	prevOI := th.prevOI
	hasPrevOI := th.hasPrevOI
	th.prevOI = snap.OpenInterest
	th.hasPrevOI = true
	th.mu.Unlock()

	ts := (snap.Timestamp / 5000) * 5000

	// Write snapshot to hot DB
	_, err := th.hotDB.Exec(`
		INSERT INTO ticker_snapshots (
			timestamp, last_price, bid1_price, bid1_size, ask1_price, ask1_size,
			mark_price, index_price, open_interest, funding_rate, volume_24h, turnover_24h
		) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
	`, ts, snap.LastPrice, snap.Bid1Price, snap.Bid1Size, snap.Ask1Price, snap.Ask1Size,
		snap.MarkPrice, snap.IndexPrice, snap.OpenInterest, snap.FundingRate, snap.Volume24h, snap.Turnover24h)
	if err != nil {
		log.Printf("[ticker_handler] hot db insert error: %v", err)
	}

	// Calculate and write feature
	spread := snap.Ask1Price - snap.Bid1Price
	midPrice := (snap.Bid1Price + snap.Ask1Price) / 2.0
	spreadBps := 0.0
	if midPrice > 0 {
		spreadBps = (spread / midPrice) * 10000.0
	}

	oiChange := 0.0
	if hasPrevOI {
		oiChange = snap.OpenInterest - prevOI
	}

	markIndexBasis := snap.MarkPrice - snap.IndexPrice
	totalSize := snap.Bid1Size + snap.Ask1Size
	bidAskImbalance := 0.0
	if totalSize > 0 {
		bidAskImbalance = snap.Bid1Size / totalSize
	}

	_, err = th.featDB.Exec(`
		INSERT OR IGNORE INTO ticker_features (
			timestamp, spread, spread_bps, mid_price, oi_change, funding_rate, mark_index_basis, bid_ask_imbalance
		) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
	`, ts, spread, spreadBps, midPrice, oiChange, snap.FundingRate, markIndexBasis, bidAskImbalance)
	if err != nil {
		log.Printf("[ticker_handler] features db insert error: %v", err)
	}
}

func (th *TickerHandler) Close() {
	close(th.stopChan)
	th.wg.Wait()
	if th.hotDB != nil {
		th.hotDB.Close()
	}
	if th.featDB != nil {
		th.featDB.Close()
	}
}