Initial commit.

2026-07-14 20:56:34 +03:00
commit 6a58f1ba7e
11 changed files with 1114 additions and 0 deletions
@@ -0,0 +1,353 @@
+package main
+
+import (
+	"database/sql"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"time"
+
+	_ "modernc.org/sqlite"
+)
+
+// StorageManager handles all SQLite database operations: initialization,
+// raw tick writing, feature writing, hourly migration, and pruning.
+type StorageManager struct {
+	cfg        Config
+	hotDBPath  string
+	featDBPath string
+	archiveDir string
+}
+
+// NewStorageManager creates directories, initializes databases, and returns a ready manager.
+func NewStorageManager(cfg Config) (*StorageManager, error) {
+	archiveDir := filepath.Join(cfg.DataDir, "archive")
+	if err := os.MkdirAll(archiveDir, 0755); err != nil {
+		return nil, fmt.Errorf("create archive dir: %w", err)
+	}
+
+	sm := &StorageManager{
+		cfg:        cfg,
+		hotDBPath:  filepath.Join(cfg.DataDir, "hot_ticks.db"),
+		featDBPath: filepath.Join(cfg.DataDir, "features.db"),
+		archiveDir: archiveDir,
+	}
+
+	if err := sm.initHotDB(); err != nil {
+		return nil, fmt.Errorf("init hot db: %w", err)
+	}
+	if err := sm.initFeaturesDB(); err != nil {
+		return nil, fmt.Errorf("init features db: %w", err)
+	}
+
+	return sm, nil
+}
+
+// OpenHotDB returns a new connection to the hot ticks database.
+func (sm *StorageManager) OpenHotDB() (*sql.DB, error) {
+	db, err := sql.Open("sqlite", sm.hotDBPath)
+	if err != nil {
+		return nil, err
+	}
+	if _, err := db.Exec("PRAGMA journal_mode=WAL"); err != nil {
+		db.Close()
+		return nil, err
+	}
+	if _, err := db.Exec("PRAGMA synchronous=NORMAL"); err != nil {
+		db.Close()
+		return nil, err
+	}
+	return db, nil
+}
+
+// OpenFeaturesDB returns a new connection to the features database.
+func (sm *StorageManager) OpenFeaturesDB() (*sql.DB, error) {
+	db, err := sql.Open("sqlite", sm.featDBPath)
+	if err != nil {
+		return nil, err
+	}
+	if _, err := db.Exec("PRAGMA journal_mode=WAL"); err != nil {
+		db.Close()
+		return nil, err
+	}
+	if _, err := db.Exec("PRAGMA synchronous=NORMAL"); err != nil {
+		db.Close()
+		return nil, err
+	}
+	return db, nil
+}
+
+func (sm *StorageManager) initHotDB() error {
+	db, err := sm.OpenHotDB()
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	_, err = db.Exec(`
+		PRAGMA auto_vacuum=INCREMENTAL;
+
+		CREATE TABLE IF NOT EXISTS btc_ticks (
+			timestamp INTEGER NOT NULL,
+			price REAL NOT NULL,
+			volume REAL NOT NULL,
+			side TEXT NOT NULL
+		);
+
+		CREATE INDEX IF NOT EXISTS idx_timestamp ON btc_ticks(timestamp);
+	`)
+	return err
+}
+
+func (sm *StorageManager) initFeaturesDB() error {
+	db, err := sm.OpenFeaturesDB()
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	_, err = db.Exec(`
+		PRAGMA auto_vacuum=INCREMENTAL;
+
+		CREATE TABLE IF NOT EXISTS five_second_features (
+			timestamp INTEGER PRIMARY KEY,
+			log_return REAL NOT NULL,
+			realized_vol REAL NOT NULL,
+			ofi REAL NOT NULL,
+			volume_sum REAL NOT NULL,
+			close_price REAL NOT NULL
+		);
+	`)
+	return err
+}
+
+// weeklyArchivePath resolves the archive DB file path for a given timestamp.
+func (sm *StorageManager) weeklyArchivePath(timestampMs int64) string {
+	t := time.UnixMilli(timestampMs)
+	year, week := t.ISOWeek()
+	filename := fmt.Sprintf("btc_ticks_%d_W%02d.db", year, week)
+	return filepath.Join(sm.archiveDir, filename)
+}
+
+// initArchiveDB ensures the archive database has the correct schema.
+func (sm *StorageManager) initArchiveDB(path string) error {
+	db, err := sql.Open("sqlite", path)
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	_, err = db.Exec(`
+		PRAGMA journal_mode=WAL;
+		PRAGMA synchronous=NORMAL;
+
+		CREATE TABLE IF NOT EXISTS btc_ticks (
+			timestamp INTEGER NOT NULL,
+			price REAL NOT NULL,
+			volume REAL NOT NULL,
+			side TEXT NOT NULL
+		);
+
+		CREATE INDEX IF NOT EXISTS idx_timestamp ON btc_ticks(timestamp);
+	`)
+	return err
+}
+
+// RunHourlyMaintenance performs tick migration and feature pruning.
+func (sm *StorageManager) RunHourlyMaintenance() {
+	nowMs := time.Now().UnixMilli()
+	log.Println("[maintenance] Starting hourly database maintenance...")
+
+	// Part A: Migrate raw ticks older than retention window
+	cutoff12h := nowMs - int64(sm.cfg.HotRetentionHours)*60*60*1000
+	if err := sm.migrateRawTicks(cutoff12h); err != nil {
+		log.Printf("[maintenance] Raw tick migration failed: %v", err)
+	}
+
+	// Part B: Prune features older than retention window
+	cutoffFeatures := nowMs - int64(sm.cfg.FeatureRetentionDays)*24*60*60*1000
+	if err := sm.pruneFeatures(cutoffFeatures); err != nil {
+		log.Printf("[maintenance] Feature pruning failed: %v", err)
+	}
+
+	log.Println("[maintenance] Hourly maintenance complete.")
+}
+
+// StartupRecovery checks for stale ticks and migrates them before normal operation.
+func (sm *StorageManager) StartupRecovery() error {
+	log.Println("[startup] Checking for stale ticks in hot database...")
+
+	db, err := sm.OpenHotDB()
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	cutoffMs := time.Now().UnixMilli() - int64(sm.cfg.HotRetentionHours)*60*60*1000
+	var count int64
+	err = db.QueryRow("SELECT COUNT(*) FROM btc_ticks WHERE timestamp < ?", cutoffMs).Scan(&count)
+	if err != nil {
+		return fmt.Errorf("count stale ticks: %w", err)
+	}
+
+	if count > 0 {
+		log.Printf("[startup] Found %d stale ticks, triggering immediate migration...", count)
+		if err := sm.migrateRawTicks(cutoffMs); err != nil {
+			return fmt.Errorf("startup migration: %w", err)
+		}
+	} else {
+		log.Println("[startup] No stale ticks found, hot database is clean.")
+	}
+
+	return nil
+}
+
+func (sm *StorageManager) migrateRawTicks(cutoffMs int64) error {
+	db, err := sm.OpenHotDB()
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	// Find the range of ticks that need migration
+	var minTS, maxTS sql.NullInt64
+	err = db.QueryRow(`
+		SELECT MIN(timestamp), MAX(timestamp)
+		FROM btc_ticks
+		WHERE timestamp < ?
+	`, cutoffMs).Scan(&minTS, &maxTS)
+	if err != nil {
+		return fmt.Errorf("query migration range: %w", err)
+	}
+
+	if !minTS.Valid || !maxTS.Valid {
+		log.Println("[maintenance] No raw ticks older than cutoff to migrate.")
+		return nil
+	}
+
+	// Process ticks week by week to handle cross-week boundaries
+	currentTs := minTS.Int64
+	for currentTs <= maxTS.Int64 {
+		t := time.UnixMilli(currentTs)
+		year, week := t.ISOWeek()
+		archivePath := sm.weeklyArchivePath(currentTs)
+
+		log.Printf("[maintenance] Migrating ticks for %d W%02d -> %s",
+			year, week, filepath.Base(archivePath))
+
+		// Calculate the start of the next ISO week (Monday 00:00:00 UTC)
+		nextWeekStart := sm.nextISOWeekStartMs(currentTs)
+
+		// The upper bound for this chunk: either next week start or cutoff, whichever is smaller
+		chunkEnd := nextWeekStart
+		if cutoffMs < chunkEnd {
+			chunkEnd = cutoffMs
+		}
+
+		// Ensure archive DB exists and has schema
+		if err := sm.initArchiveDB(archivePath); err != nil {
+			return fmt.Errorf("init archive db: %w", err)
+		}
+
+		// Attach and atomically migrate
+		if err := sm.atomicMigrate(db, archivePath, currentTs, chunkEnd); err != nil {
+			return fmt.Errorf("atomic migrate: %w", err)
+		}
+
+		currentTs = nextWeekStart
+	}
+
+	// Reclaim space
+	if _, err := db.Exec("PRAGMA incremental_vacuum(500)"); err != nil {
+		log.Printf("[maintenance] incremental_vacuum warning: %v", err)
+	}
+
+	log.Println("[maintenance] Raw tick migration completed successfully.")
+	return nil
+}
+
+// nextISOWeekStartMs calculates the epoch ms of the next Monday 00:00:00 UTC
+// relative to the given timestamp.
+func (sm *StorageManager) nextISOWeekStartMs(timestampMs int64) int64 {
+	t := time.UnixMilli(timestampMs).UTC()
+	// Calculate days until next Monday
+	daysUntilMonday := (8 - int(t.Weekday())) % 7
+	if daysUntilMonday == 0 {
+		daysUntilMonday = 7
+	}
+	nextMonday := time.Date(t.Year(), t.Month(), t.Day()+daysUntilMonday, 0, 0, 0, 0, time.UTC)
+	return nextMonday.UnixMilli()
+}
+
+func (sm *StorageManager) atomicMigrate(hotDB *sql.DB, archivePath string, fromMs, toMs int64) error {
+	// Attach the archive database
+	_, err := hotDB.Exec("ATTACH DATABASE ? AS archive", archivePath)
+	if err != nil {
+		return fmt.Errorf("attach archive: %w", err)
+	}
+	defer hotDB.Exec("DETACH DATABASE archive")
+
+	tx, err := hotDB.Begin()
+	if err != nil {
+		return fmt.Errorf("begin migration tx: %w", err)
+	}
+
+	// Insert into archive
+	_, err = tx.Exec(`
+		INSERT INTO archive.btc_ticks (timestamp, price, volume, side)
+		SELECT timestamp, price, volume, side
+		FROM main.btc_ticks
+		WHERE timestamp >= ? AND timestamp < ?
+	`, fromMs, toMs)
+	if err != nil {
+		tx.Rollback()
+		return fmt.Errorf("insert into archive: %w", err)
+	}
+
+	// Delete from hot
+	result, err := tx.Exec(`
+		DELETE FROM main.btc_ticks
+		WHERE timestamp >= ? AND timestamp < ?
+	`, fromMs, toMs)
+	if err != nil {
+		tx.Rollback()
+		return fmt.Errorf("delete from hot: %w", err)
+	}
+
+	migrated, _ := result.RowsAffected()
+	if err := tx.Commit(); err != nil {
+		return fmt.Errorf("commit migration: %w", err)
+	}
+
+	log.Printf("[maintenance] Migrated %d ticks to archive.", migrated)
+	return nil
+}
+
+func (sm *StorageManager) pruneFeatures(cutoffMs int64) error {
+	db, err := sm.OpenFeaturesDB()
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	cutoffTime := time.UnixMilli(cutoffMs)
+	log.Printf("[maintenance] Pruning features older than %s", cutoffTime.Format(time.RFC3339))
+
+	result, err := db.Exec("DELETE FROM five_second_features WHERE timestamp < ?", cutoffMs)
+	if err != nil {
+		return fmt.Errorf("delete old features: %w", err)
+	}
+
+	deleted, _ := result.RowsAffected()
+	if deleted > 0 {
+		log.Printf("[maintenance] Pruned %d old feature rows.", deleted)
+		if _, err := db.Exec("PRAGMA incremental_vacuum(100)"); err != nil {
+			log.Printf("[maintenance] feature vacuum warning: %v", err)
+		}
+	} else {
+		log.Println("[maintenance] No feature rows needed pruning.")
+	}
+
+	return nil
+}