Initial commit.

aggregator.go

@@ -0,0 +1,168 @@
+package main
+
+import (
+	"database/sql"
+	"log"
+	"math"
+	"sync"
+)
+
+const bucketDurationMs = 5000 // 5 seconds
+
+// Aggregator accumulates ticks into 5-second buckets and writes completed
+// feature rows to the features database.
+type Aggregator struct {
+	mu            sync.Mutex
+	currentBucket int64     // epoch ms of current bucket start
+	ticks         []Tick    // ticks within the current bucket
+	featDB        *sql.DB   // connection to features.db
+	lastPrice     float64   // closing price of the previous bucket (for log return)
+}
+
+// NewAggregator creates an aggregator with a connection to the features database.
+func NewAggregator(sm *StorageManager) (*Aggregator, error) {
+	db, err := sm.OpenFeaturesDB()
+	if err != nil {
+		return nil, err
+	}
+
+	// Attempt to load the last close price from features.db for continuity
+	var lastPrice float64
+	err = db.QueryRow(`
+		SELECT close_price FROM five_second_features 
+		ORDER BY timestamp DESC LIMIT 1
+	`).Scan(&lastPrice)
+	if err != nil {
+		lastPrice = 0 // Will be set from first tick
+	}
+
+	return &Aggregator{
+		featDB:    db,
+		lastPrice: lastPrice,
+	}, nil
+}
+
+// ProcessTick adds a tick to the current bucket. If the tick crosses a 5-second
+// boundary, the previous bucket is aggregated and flushed to the database.
+func (a *Aggregator) ProcessTick(tick Tick) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
+	// Determine which 5-second bucket this tick belongs to
+	tickBucket := (tick.Timestamp / bucketDurationMs) * bucketDurationMs
+
+	if a.currentBucket == 0 {
+		// First tick ever — initialize the bucket
+		a.currentBucket = tickBucket
+	}
+
+	if tickBucket > a.currentBucket {
+		// This tick belongs to a new bucket — flush the previous one
+		if len(a.ticks) > 0 {
+			a.flushBucket()
+		}
+		a.currentBucket = tickBucket
+		a.ticks = a.ticks[:0] // reset slice, keep backing array
+	}
+
+	a.ticks = append(a.ticks, tick)
+}
+
+// Close shuts down the aggregator, flushing any remaining bucket and closing the DB.
+func (a *Aggregator) Close() {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
+	if len(a.ticks) > 0 {
+		a.flushBucket()
+	}
+	if a.featDB != nil {
+		a.featDB.Close()
+	}
+}
+
+// flushBucket computes features from the accumulated ticks and writes to features.db.
+// Must be called while holding a.mu.
+func (a *Aggregator) flushBucket() {
+	if len(a.ticks) == 0 {
+		return
+	}
+
+	bucket := a.computeFeatures(a.ticks)
+
+	_, err := a.featDB.Exec(`
+		INSERT OR IGNORE INTO five_second_features 
+		(timestamp, log_return, realized_vol, ofi, volume_sum, close_price)
+		VALUES (?, ?, ?, ?, ?, ?)
+	`, bucket.Timestamp, bucket.LogReturn, bucket.RealizedVol,
+		bucket.OFI, bucket.VolumeSum, bucket.ClosePrice)
+
+	if err != nil {
+		log.Printf("[aggregator] Failed to write feature bucket: %v", err)
+	}
+
+	// Update lastPrice for next bucket's log return calculation
+	a.lastPrice = bucket.ClosePrice
+}
+
+// computeFeatures calculates all feature columns from a slice of ticks.
+func (a *Aggregator) computeFeatures(ticks []Tick) FeatureBucket {
+	n := len(ticks)
+	startPrice := ticks[0].Price
+	closePrice := ticks[n-1].Price
+
+	// Log return: ln(close / start_of_bucket_or_prev_close)
+	refPrice := a.lastPrice
+	if refPrice == 0 {
+		refPrice = startPrice
+	}
+	logReturn := math.Log(closePrice / refPrice)
+
+	// Order Flow Imbalance (OFI) and total volume
+	var buyVol, sellVol, volumeSum float64
+	for _, t := range ticks {
+		volumeSum += t.Volume
+		if t.Side == "Buy" {
+			buyVol += t.Volume
+		} else {
+			sellVol += t.Volume
+		}
+	}
+	ofi := buyVol - sellVol
+
+	// Realized volatility: standard deviation of tick-to-tick log returns
+	realizedVol := 0.0
+	if n > 1 {
+		logReturns := make([]float64, n-1)
+		for i := 1; i < n; i++ {
+			if ticks[i-1].Price > 0 {
+				logReturns[i-1] = math.Log(ticks[i].Price / ticks[i-1].Price)
+			}
+		}
+
+		// Calculate mean
+		var sum float64
+		for _, lr := range logReturns {
+			sum += lr
+		}
+		mean := sum / float64(len(logReturns))
+
+		// Calculate variance
+		var variance float64
+		for _, lr := range logReturns {
+			diff := lr - mean
+			variance += diff * diff
+		}
+		variance /= float64(len(logReturns))
+		realizedVol = math.Sqrt(variance)
+	}
+
+	return FeatureBucket{
+		Timestamp:   a.currentBucket,
+		LogReturn:   logReturn,
+		RealizedVol: realizedVol,
+		OFI:         ofi,
+		VolumeSum:   volumeSum,
+		ClosePrice:  closePrice,
+	}
+}