bybit_btcusdt_ingest/aggregator.go

package main

import (
	"database/sql"
	"log"
	"math"
	"sync"
)

const bucketDurationMs = 5000 // 5 seconds

// Aggregator accumulates ticks into 5-second buckets and writes completed
// feature rows to the features database.
type Aggregator struct {
	mu            sync.Mutex
	currentBucket int64     // epoch ms of current bucket start
	ticks         []Tick    // ticks within the current bucket
	featDB        *sql.DB   // connection to features.db
	lastPrice     float64   // closing price of the previous bucket (for log return)
}

// NewAggregator creates an aggregator with a connection to the features database.
func NewAggregator(sm *StorageManager) (*Aggregator, error) {
	db, err := sm.OpenFeaturesDB()
	if err != nil {
		return nil, err
	}

	// Attempt to load the last close price from features.db for continuity
	var lastPrice float64
	err = db.QueryRow(`
		SELECT close_price FROM five_second_features 
		ORDER BY timestamp DESC LIMIT 1
	`).Scan(&lastPrice)
	if err != nil {
		lastPrice = 0 // Will be set from first tick
	}

	return &Aggregator{
		featDB:    db,
		lastPrice: lastPrice,
	}, nil
}

// ProcessTick adds a tick to the current bucket. If the tick crosses a 5-second
// boundary, the previous bucket is aggregated and flushed to the database.
func (a *Aggregator) ProcessTick(tick Tick) {
	a.mu.Lock()
	defer a.mu.Unlock()

	// Determine which 5-second bucket this tick belongs to
	tickBucket := (tick.Timestamp / bucketDurationMs) * bucketDurationMs

	if a.currentBucket == 0 {
		// First tick ever — initialize the bucket
		a.currentBucket = tickBucket
	}

	if tickBucket > a.currentBucket {
		// This tick belongs to a new bucket — flush the previous one
		if len(a.ticks) > 0 {
			a.flushBucket()
		}
		a.currentBucket = tickBucket
		a.ticks = a.ticks[:0] // reset slice, keep backing array
	}

	a.ticks = append(a.ticks, tick)
}

// Close shuts down the aggregator, flushing any remaining bucket and closing the DB.
func (a *Aggregator) Close() {
	a.mu.Lock()
	defer a.mu.Unlock()

	if len(a.ticks) > 0 {
		a.flushBucket()
	}
	if a.featDB != nil {
		a.featDB.Close()
	}
}

// flushBucket computes features from the accumulated ticks and writes to features.db.
// Must be called while holding a.mu.
func (a *Aggregator) flushBucket() {
	if len(a.ticks) == 0 {
		return
	}

	bucket := a.computeFeatures(a.ticks)

	_, err := a.featDB.Exec(`
		INSERT OR IGNORE INTO five_second_features 
		(timestamp, log_return, realized_vol, ofi, volume_sum, close_price, vwap)
		VALUES (?, ?, ?, ?, ?, ?, ?)
	`, bucket.Timestamp, bucket.LogReturn, bucket.RealizedVol,
		bucket.OFI, bucket.VolumeSum, bucket.ClosePrice, bucket.VWAP)

	if err != nil {
		log.Printf("[aggregator] Failed to write feature bucket: %v", err)
	}

	// Update lastPrice for next bucket's log return calculation
	a.lastPrice = bucket.ClosePrice
}

// computeFeatures calculates all feature columns from a slice of ticks.
func (a *Aggregator) computeFeatures(ticks []Tick) FeatureBucket {
	n := len(ticks)
	startPrice := ticks[0].Price
	closePrice := ticks[n-1].Price

	// Log return: ln(close / start_of_bucket_or_prev_close)
	refPrice := a.lastPrice
	if refPrice == 0 {
		refPrice = startPrice
	}
	logReturn := math.Log(closePrice / refPrice)

	// Order Flow Imbalance (OFI), total volume, and sum of price * volume
	var buyVol, sellVol, volumeSum float64
	var priceVolumeSum float64
	for _, t := range ticks {
		volumeSum += t.Volume
		priceVolumeSum += t.Price * t.Volume
		if t.Side == "Buy" {
			buyVol += t.Volume
		} else {
			sellVol += t.Volume
		}
	}
	ofi := buyVol - sellVol

	vwap := closePrice
	if volumeSum > 0 {
		vwap = priceVolumeSum / volumeSum
	}

	// Realized volatility: standard deviation of tick-to-tick log returns
	realizedVol := 0.0
	if n > 1 {
		logReturns := make([]float64, n-1)
		for i := 1; i < n; i++ {
			if ticks[i-1].Price > 0 {
				logReturns[i-1] = math.Log(ticks[i].Price / ticks[i-1].Price)
			}
		}

		// Calculate mean
		var sum float64
		for _, lr := range logReturns {
			sum += lr
		}
		mean := sum / float64(len(logReturns))

		// Calculate variance
		var variance float64
		for _, lr := range logReturns {
			diff := lr - mean
			variance += diff * diff
		}
		variance /= float64(len(logReturns))
		realizedVol = math.Sqrt(variance)
	}

	return FeatureBucket{
		Timestamp:   a.currentBucket,
		LogReturn:   logReturn,
		RealizedVol: realizedVol,
		OFI:         ofi,
		VolumeSum:   volumeSum,
		ClosePrice:  closePrice,
		VWAP:        vwap,
	}
}