Added klines, liquidations, tickers and trades to be recorded. Bundled the stats as cli argument.

2026-07-22 15:35:33 +03:00
parent 270922fb78
commit f6765fdf07
15 changed files with 2719 additions and 495 deletions
@@ -11,10 +11,84 @@ import (
 	_ "modernc.org/sqlite"
 )

+// StreamStorage manages databases for a single stream (trades, ticker, etc.).
+// Each stream has its own subdirectory under the base data dir.
+type StreamStorage struct {
+	streamName string
+	dataDir    string // full path: e.g. "data/trades"
+	archiveDir string // full path: e.g. "data/trades/archive"
+}
+
+// NewStreamStorage creates directories for a stream and returns a StreamStorage.
+func NewStreamStorage(baseDir, streamName string) (*StreamStorage, error) {
+	dataDir := filepath.Join(baseDir, streamName)
+	archiveDir := filepath.Join(dataDir, "archive")
+
+	if err := os.MkdirAll(archiveDir, 0755); err != nil {
+		return nil, fmt.Errorf("create %s dirs: %w", streamName, err)
+	}
+
+	return &StreamStorage{
+		streamName: streamName,
+		dataDir:    dataDir,
+		archiveDir: archiveDir,
+	}, nil
+}
+
+// DBPath returns the full path for a database file within this stream's directory.
+func (ss *StreamStorage) DBPath(filename string) string {
+	return filepath.Join(ss.dataDir, filename)
+}
+
+// ArchivePath returns an archive file path for a given timestamp using ISO week format.
+func (ss *StreamStorage) ArchivePath(prefix string, timestampMs int64) string {
+	t := time.UnixMilli(timestampMs)
+	year, week := t.ISOWeek()
+	filename := fmt.Sprintf("%s_%d_W%02d.db", prefix, year, week)
+	return filepath.Join(ss.archiveDir, filename)
+}
+
+// OpenDB opens a SQLite database at the given path with WAL mode and performance pragmas.
+func OpenDB(path string) (*sql.DB, error) {
+	db, err := sql.Open("sqlite", path)
+	if err != nil {
+		return nil, err
+	}
+	if _, err := db.Exec("PRAGMA journal_mode=WAL"); err != nil {
+		db.Close()
+		return nil, err
+	}
+	if _, err := db.Exec("PRAGMA synchronous=NORMAL"); err != nil {
+		db.Close()
+		return nil, err
+	}
+	return db, nil
+}
+
+// OpenDBWithAutoVacuum opens a SQLite database with WAL mode and incremental auto-vacuum.
+func OpenDBWithAutoVacuum(path string) (*sql.DB, error) {
+	db, err := OpenDB(path)
+	if err != nil {
+		return nil, err
+	}
+	if _, err := db.Exec("PRAGMA auto_vacuum=INCREMENTAL"); err != nil {
+		db.Close()
+		return nil, err
+	}
+	return db, nil
+}
+
+// ----------------------------------------------------------------
+// StorageManager — coordinates per-stream storages and maintenance
+// ----------------------------------------------------------------
+
 // StorageManager handles all SQLite database operations: initialization,
 // raw tick writing, feature writing, hourly migration, and pruning.
 type StorageManager struct {
-	cfg        Config
+	cfg     Config
+	streams map[string]*StreamStorage
+
+	// Legacy paths (used for backward-compatible migration only)
 	hotDBPath  string
 	featDBPath string
 	archiveDir string
@@ -22,72 +96,112 @@ type StorageManager struct {

 // NewStorageManager creates directories, initializes databases, and returns a ready manager.
 func NewStorageManager(cfg Config) (*StorageManager, error) {
-	archiveDir := filepath.Join(cfg.DataDir, "archive")
-	if err := os.MkdirAll(archiveDir, 0755); err != nil {
-		return nil, fmt.Errorf("create archive dir: %w", err)
-	}
-
 	sm := &StorageManager{
-		cfg:        cfg,
-		hotDBPath:  filepath.Join(cfg.DataDir, "hot_ticks.db"),
-		featDBPath: filepath.Join(cfg.DataDir, "features.db"),
-		archiveDir: archiveDir,
+		cfg:     cfg,
+		streams: make(map[string]*StreamStorage),
 	}

-	if err := sm.initHotDB(); err != nil {
-		return nil, fmt.Errorf("init hot db: %w", err)
+	// Perform data migration from old flat layout to new per-stream layout
+	if err := sm.migrateOldLayout(); err != nil {
+		return nil, fmt.Errorf("migrate old layout: %w", err)
 	}
-	if err := sm.initFeaturesDB(); err != nil {
-		return nil, fmt.Errorf("init features db: %w", err)
+
+	// Initialize stream storages for enabled streams
+	streamNames := []string{}
+	if cfg.Streams.Trades.Enabled {
+		streamNames = append(streamNames, "trades")
+	}
+	if cfg.Streams.Ticker.Enabled {
+		streamNames = append(streamNames, "ticker")
+	}
+	if cfg.Streams.Klines.Enabled {
+		streamNames = append(streamNames, "klines")
+	}
+	if cfg.Streams.Orderbook.Enabled {
+		streamNames = append(streamNames, "orderbook")
+	}
+	if cfg.Streams.Liquidations.Enabled {
+		streamNames = append(streamNames, "liquidations")
+	}
+
+	for _, name := range streamNames {
+		ss, err := NewStreamStorage(cfg.DataDir, name)
+		if err != nil {
+			return nil, err
+		}
+		sm.streams[name] = ss
+	}
+
+	// Initialize databases per stream
+	if cfg.Streams.Trades.Enabled {
+		if err := sm.initTradesDBs(); err != nil {
+			return nil, fmt.Errorf("init trades dbs: %w", err)
+		}
+	}
+	if cfg.Streams.Ticker.Enabled {
+		if err := sm.initTickerDBs(); err != nil {
+			return nil, fmt.Errorf("init ticker dbs: %w", err)
+		}
+	}
+	if cfg.Streams.Klines.Enabled {
+		if err := sm.initKlineDBs(); err != nil {
+			return nil, fmt.Errorf("init kline dbs: %w", err)
+		}
+	}
+	if cfg.Streams.Orderbook.Enabled {
+		if err := sm.initOrderbookDBs(); err != nil {
+			return nil, fmt.Errorf("init orderbook dbs: %w", err)
+		}
+	}
+	if cfg.Streams.Liquidations.Enabled {
+		if err := sm.initLiquidationDBs(); err != nil {
+			return nil, fmt.Errorf("init liquidation dbs: %w", err)
+		}
+	}
+
+	// Set legacy paths for backward compatibility with trade-specific methods
+	if ss, ok := sm.streams["trades"]; ok {
+		sm.hotDBPath = ss.DBPath("hot_ticks.db")
+		sm.featDBPath = ss.DBPath("features.db")
+		sm.archiveDir = ss.archiveDir
 	}

 	return sm, nil
 }

+// GetStreamStorage returns the StreamStorage for a given stream name.
+func (sm *StorageManager) GetStreamStorage(name string) *StreamStorage {
+	return sm.streams[name]
+}
+
+// ----------------------------------------------------------------
+// Legacy compatibility methods (used by existing trade pipeline)
+// ----------------------------------------------------------------
+
 // OpenHotDB returns a new connection to the hot ticks database.
 func (sm *StorageManager) OpenHotDB() (*sql.DB, error) {
-	db, err := sql.Open("sqlite", sm.hotDBPath)
-	if err != nil {
-		return nil, err
-	}
-	if _, err := db.Exec("PRAGMA journal_mode=WAL"); err != nil {
-		db.Close()
-		return nil, err
-	}
-	if _, err := db.Exec("PRAGMA synchronous=NORMAL"); err != nil {
-		db.Close()
-		return nil, err
-	}
-	return db, nil
+	return OpenDB(sm.hotDBPath)
 }

 // OpenFeaturesDB returns a new connection to the features database.
 func (sm *StorageManager) OpenFeaturesDB() (*sql.DB, error) {
-	db, err := sql.Open("sqlite", sm.featDBPath)
-	if err != nil {
-		return nil, err
-	}
-	if _, err := db.Exec("PRAGMA journal_mode=WAL"); err != nil {
-		db.Close()
-		return nil, err
-	}
-	if _, err := db.Exec("PRAGMA synchronous=NORMAL"); err != nil {
-		db.Close()
-		return nil, err
-	}
-	return db, nil
+	return OpenDB(sm.featDBPath)
 }

-func (sm *StorageManager) initHotDB() error {
-	db, err := sm.OpenHotDB()
+// ----------------------------------------------------------------
+// Database initialization per stream
+// ----------------------------------------------------------------
+
+func (sm *StorageManager) initTradesDBs() error {
+	ss := sm.streams["trades"]
+
+	db, err := OpenDBWithAutoVacuum(ss.DBPath("hot_ticks.db"))
 	if err != nil {
 		return err
 	}
 	defer db.Close()

 	_, err = db.Exec(`
-		PRAGMA auto_vacuum=INCREMENTAL;
-
 		CREATE TABLE IF NOT EXISTS btc_ticks (
 			seq INTEGER PRIMARY KEY,

@@ -114,19 +228,17 @@ func (sm *StorageManager) initHotDB() error {
 		CREATE INDEX IF NOT EXISTS idx_seq
 		ON btc_ticks(seq);
 	`)
-	return err
-}
-
-func (sm *StorageManager) initFeaturesDB() error {
-	db, err := sm.OpenFeaturesDB()
 	if err != nil {
 		return err
 	}
-	defer db.Close()

-	_, err = db.Exec(`
-		PRAGMA auto_vacuum=INCREMENTAL;
+	featDB, err := OpenDBWithAutoVacuum(ss.DBPath("features.db"))
+	if err != nil {
+		return err
+	}
+	defer featDB.Close()

+	_, err = featDB.Exec(`
 		CREATE TABLE IF NOT EXISTS five_second_features (
 			timestamp INTEGER PRIMARY KEY,
 			log_return REAL NOT NULL,
@@ -140,15 +252,206 @@ func (sm *StorageManager) initFeaturesDB() error {
 	return err
 }

-// weeklyArchivePath resolves the archive DB file path for a given timestamp.
-func (sm *StorageManager) weeklyArchivePath(timestampMs int64) string {
-	t := time.UnixMilli(timestampMs)
-	year, week := t.ISOWeek()
-	filename := fmt.Sprintf("btc_ticks_%d_W%02d.db", year, week)
-	return filepath.Join(sm.archiveDir, filename)
+func (sm *StorageManager) initTickerDBs() error {
+	ss := sm.streams["ticker"]
+
+	db, err := OpenDBWithAutoVacuum(ss.DBPath("hot_ticker.db"))
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	_, err = db.Exec(`
+		CREATE TABLE IF NOT EXISTS ticker_snapshots (
+			timestamp       INTEGER NOT NULL,
+			last_price      REAL NOT NULL,
+			bid1_price      REAL NOT NULL,
+			bid1_size       REAL NOT NULL,
+			ask1_price      REAL NOT NULL,
+			ask1_size       REAL NOT NULL,
+			mark_price      REAL NOT NULL,
+			index_price     REAL NOT NULL,
+			open_interest   REAL NOT NULL,
+			funding_rate    REAL NOT NULL,
+			volume_24h      REAL NOT NULL,
+			turnover_24h    REAL NOT NULL
+		);
+		CREATE INDEX IF NOT EXISTS idx_ticker_ts ON ticker_snapshots(timestamp);
+	`)
+	if err != nil {
+		return err
+	}
+
+	featDB, err := OpenDBWithAutoVacuum(ss.DBPath("features.db"))
+	if err != nil {
+		return err
+	}
+	defer featDB.Close()
+
+	_, err = featDB.Exec(`
+		CREATE TABLE IF NOT EXISTS ticker_features (
+			timestamp         INTEGER PRIMARY KEY,
+			spread            REAL NOT NULL,
+			spread_bps        REAL NOT NULL,
+			mid_price         REAL NOT NULL,
+			oi_change         REAL NOT NULL,
+			funding_rate      REAL NOT NULL,
+			mark_index_basis  REAL NOT NULL,
+			bid_ask_imbalance REAL NOT NULL
+		);
+	`)
+	return err
 }

-// initArchiveDB ensures the archive database has the correct schema.
+func (sm *StorageManager) initKlineDBs() error {
+	ss := sm.streams["klines"]
+
+	for _, interval := range sm.cfg.Streams.Klines.Intervals {
+		dbName := fmt.Sprintf("kline_%s.db", interval)
+		db, err := OpenDBWithAutoVacuum(ss.DBPath(dbName))
+		if err != nil {
+			return fmt.Errorf("init kline_%s: %w", interval, err)
+		}
+
+		_, err = db.Exec(`
+			CREATE TABLE IF NOT EXISTS klines (
+				start_time  INTEGER PRIMARY KEY,
+				end_time    INTEGER NOT NULL,
+				interval    TEXT    NOT NULL,
+				open        REAL    NOT NULL,
+				high        REAL    NOT NULL,
+				low         REAL    NOT NULL,
+				close       REAL    NOT NULL,
+				volume      REAL    NOT NULL,
+				turnover    REAL    NOT NULL,
+				confirmed   INTEGER NOT NULL
+			);
+		`)
+		db.Close()
+		if err != nil {
+			return err
+		}
+	}
+
+	featDB, err := OpenDBWithAutoVacuum(ss.DBPath("features.db"))
+	if err != nil {
+		return err
+	}
+	defer featDB.Close()
+
+	_, err = featDB.Exec(`
+		CREATE TABLE IF NOT EXISTS kline_features (
+			timestamp       INTEGER NOT NULL,
+			interval        TEXT    NOT NULL,
+			body_ratio      REAL    NOT NULL,
+			upper_wick      REAL    NOT NULL,
+			lower_wick      REAL    NOT NULL,
+			log_return      REAL    NOT NULL,
+			volume          REAL    NOT NULL,
+			turnover        REAL    NOT NULL,
+			PRIMARY KEY (interval, timestamp)
+		);
+	`)
+	return err
+}
+
+func (sm *StorageManager) initOrderbookDBs() error {
+	ss := sm.streams["orderbook"]
+
+	db, err := OpenDBWithAutoVacuum(ss.DBPath("hot_snapshots.db"))
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	_, err = db.Exec(`
+		CREATE TABLE IF NOT EXISTS ob_snapshots (
+			timestamp   INTEGER NOT NULL,
+			level       INTEGER NOT NULL,
+			bid_price   REAL,
+			bid_size    REAL,
+			ask_price   REAL,
+			ask_size    REAL,
+			PRIMARY KEY (timestamp, level)
+		);
+	`)
+	if err != nil {
+		return err
+	}
+
+	featDB, err := OpenDBWithAutoVacuum(ss.DBPath("features.db"))
+	if err != nil {
+		return err
+	}
+	defer featDB.Close()
+
+	_, err = featDB.Exec(`
+		CREATE TABLE IF NOT EXISTS ob_features (
+			timestamp           INTEGER PRIMARY KEY,
+			spread              REAL NOT NULL,
+			mid_price           REAL NOT NULL,
+			bid_depth_5         REAL NOT NULL,
+			ask_depth_5         REAL NOT NULL,
+			bid_depth_20        REAL NOT NULL,
+			ask_depth_20        REAL NOT NULL,
+			depth_imbalance_5   REAL NOT NULL,
+			depth_imbalance_20  REAL NOT NULL,
+			weighted_mid        REAL NOT NULL,
+			vwap_10             REAL NOT NULL
+		);
+	`)
+	return err
+}
+
+func (sm *StorageManager) initLiquidationDBs() error {
+	ss := sm.streams["liquidations"]
+
+	db, err := OpenDBWithAutoVacuum(ss.DBPath("hot_liquidations.db"))
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	_, err = db.Exec(`
+		CREATE TABLE IF NOT EXISTS liquidations (
+			timestamp   INTEGER NOT NULL,
+			side        TEXT    NOT NULL,
+			price       REAL    NOT NULL,
+			quantity    REAL    NOT NULL,
+			value       REAL    NOT NULL
+		);
+		CREATE INDEX IF NOT EXISTS idx_liq_ts ON liquidations(timestamp);
+	`)
+	if err != nil {
+		return err
+	}
+
+	featDB, err := OpenDBWithAutoVacuum(ss.DBPath("features.db"))
+	if err != nil {
+		return err
+	}
+	defer featDB.Close()
+
+	_, err = featDB.Exec(`
+		CREATE TABLE IF NOT EXISTS liquidation_features (
+			timestamp          INTEGER PRIMARY KEY,
+			count_total        INTEGER NOT NULL,
+			count_long         INTEGER NOT NULL,
+			count_short        INTEGER NOT NULL,
+			volume_total       REAL    NOT NULL,
+			volume_long        REAL    NOT NULL,
+			volume_short       REAL    NOT NULL,
+			value_total        REAL    NOT NULL,
+			value_long         REAL    NOT NULL,
+			value_short        REAL    NOT NULL,
+			avg_price          REAL    NOT NULL,
+			net_value          REAL    NOT NULL
+		);
+	`)
+	return err
+}
+
+// initArchiveDB ensures an archive database has the btc_ticks schema.
 func (sm *StorageManager) initArchiveDB(path string) error {
 	db, err := sql.Open("sqlite", path)
 	if err != nil {
@@ -189,28 +492,429 @@ func (sm *StorageManager) initArchiveDB(path string) error {
 	return err
 }

-// RunHourlyMaintenance performs tick migration and feature pruning.
+// initLiquidationArchiveDB ensures a liquidation archive has the correct schema.
+func (sm *StorageManager) initLiquidationArchiveDB(path string) error {
+	db, err := sql.Open("sqlite", path)
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	_, err = db.Exec(`
+		PRAGMA journal_mode=WAL;
+		PRAGMA synchronous=NORMAL;
+
+		CREATE TABLE IF NOT EXISTS liquidations (
+			timestamp   INTEGER NOT NULL,
+			side        TEXT    NOT NULL,
+			price       REAL    NOT NULL,
+			quantity    REAL    NOT NULL,
+			value       REAL    NOT NULL
+		);
+		CREATE INDEX IF NOT EXISTS idx_liq_ts ON liquidations(timestamp);
+	`)
+	return err
+}
+
+// ----------------------------------------------------------------
+// Data migration from old flat layout
+// ----------------------------------------------------------------
+
+// migrateOldLayout moves data from the old flat data/ layout into data/trades/.
+func (sm *StorageManager) migrateOldLayout() error {
+	oldHotDB := filepath.Join(sm.cfg.DataDir, "hot_ticks.db")
+	tradesDir := filepath.Join(sm.cfg.DataDir, "trades")
+
+	// Check if old layout exists and new doesn't
+	if _, err := os.Stat(oldHotDB); err != nil {
+		return nil // No old layout, nothing to migrate
+	}
+	if _, err := os.Stat(tradesDir); err == nil {
+		return nil // New layout already exists
+	}
+
+	log.Println("[migration] Detected old flat data layout, migrating to data/trades/...")
+
+	// Create trades directory
+	tradesArchiveDir := filepath.Join(tradesDir, "archive")
+	if err := os.MkdirAll(tradesArchiveDir, 0755); err != nil {
+		return err
+	}
+
+	// Move hot_ticks.db and its WAL/SHM files
+	for _, suffix := range []string{"", "-wal", "-shm"} {
+		src := oldHotDB + suffix
+		dst := filepath.Join(tradesDir, "hot_ticks.db"+suffix)
+		if _, err := os.Stat(src); err == nil {
+			if err := os.Rename(src, dst); err != nil {
+				return fmt.Errorf("move %s: %w", src, err)
+			}
+			log.Printf("[migration] Moved %s -> %s", filepath.Base(src), dst)
+		}
+	}
+
+	// Move features.db and its WAL/SHM files
+	oldFeatDB := filepath.Join(sm.cfg.DataDir, "features.db")
+	for _, suffix := range []string{"", "-wal", "-shm"} {
+		src := oldFeatDB + suffix
+		dst := filepath.Join(tradesDir, "features.db"+suffix)
+		if _, err := os.Stat(src); err == nil {
+			if err := os.Rename(src, dst); err != nil {
+				return fmt.Errorf("move %s: %w", src, err)
+			}
+			log.Printf("[migration] Moved %s -> %s", filepath.Base(src), dst)
+		}
+	}
+
+	// Move archive directory contents
+	oldArchiveDir := filepath.Join(sm.cfg.DataDir, "archive")
+	if entries, err := os.ReadDir(oldArchiveDir); err == nil {
+		for _, entry := range entries {
+			src := filepath.Join(oldArchiveDir, entry.Name())
+			dst := filepath.Join(tradesArchiveDir, entry.Name())
+			if err := os.Rename(src, dst); err != nil {
+				return fmt.Errorf("move archive %s: %w", entry.Name(), err)
+			}
+			log.Printf("[migration] Moved archive/%s -> trades/archive/%s", entry.Name(), entry.Name())
+		}
+		// Remove empty old archive dir
+		os.Remove(oldArchiveDir)
+	}
+
+	log.Println("[migration] Data migration to data/trades/ completed successfully.")
+	return nil
+}
+
+// ----------------------------------------------------------------
+// Maintenance — trade stream (backward compatible)
+// ----------------------------------------------------------------
+
+// weeklyArchivePath resolves the archive DB file path for a given timestamp.
+func (sm *StorageManager) weeklyArchivePath(timestampMs int64) string {
+	t := time.UnixMilli(timestampMs)
+	year, week := t.ISOWeek()
+	filename := fmt.Sprintf("btc_ticks_%d_W%02d.db", year, week)
+	return filepath.Join(sm.archiveDir, filename)
+}
+
+// RunHourlyMaintenance performs maintenance across all enabled streams.
 func (sm *StorageManager) RunHourlyMaintenance() {
 	nowMs := time.Now().UnixMilli()
 	log.Println("[maintenance] Starting hourly database maintenance...")

-	// Part A: Migrate raw ticks older than retention window
-	cutoff12h := nowMs - int64(sm.cfg.HotRetentionHours)*60*60*1000
-	if err := sm.migrateRawTicks(cutoff12h); err != nil {
-		log.Printf("[maintenance] Raw tick migration failed: %v", err)
+	// --- Trades ---
+	if sm.cfg.Streams.Trades.Enabled {
+		retHours := sm.cfg.Streams.Trades.HotRetentionHours
+		if retHours == 0 {
+			retHours = sm.cfg.HotRetentionHours
+		}
+		cutoff := nowMs - int64(retHours)*60*60*1000
+		if err := sm.migrateRawTicks(cutoff); err != nil {
+			log.Printf("[maintenance] Trade tick migration failed: %v", err)
+		}
+
+		retDays := sm.cfg.Streams.Trades.FeatureRetentionDays
+		if retDays == 0 {
+			retDays = sm.cfg.FeatureRetentionDays
+		}
+		cutoffFeat := nowMs - int64(retDays)*24*60*60*1000
+		if err := sm.pruneFeatures(cutoffFeat); err != nil {
+			log.Printf("[maintenance] Trade feature pruning failed: %v", err)
+		}
 	}

-	// Part B: Prune features older than retention window
-	cutoffFeatures := nowMs - int64(sm.cfg.FeatureRetentionDays)*24*60*60*1000
-	if err := sm.pruneFeatures(cutoffFeatures); err != nil {
-		log.Printf("[maintenance] Feature pruning failed: %v", err)
+	// --- Ticker ---
+	if sm.cfg.Streams.Ticker.Enabled {
+		sm.maintenanceTicker(nowMs)
+	}
+
+	// --- Klines ---
+	if sm.cfg.Streams.Klines.Enabled {
+		sm.maintenanceKlines(nowMs)
+	}
+
+	// --- Orderbook ---
+	if sm.cfg.Streams.Orderbook.Enabled {
+		sm.maintenanceOrderbook(nowMs)
+	}
+
+	// --- Liquidations ---
+	if sm.cfg.Streams.Liquidations.Enabled {
+		sm.maintenanceLiquidations(nowMs)
 	}

 	log.Println("[maintenance] Hourly maintenance complete.")
 }

+// maintenanceTicker prunes old ticker snapshots and features.
+func (sm *StorageManager) maintenanceTicker(nowMs int64) {
+	ss := sm.streams["ticker"]
+	if ss == nil {
+		return
+	}
+
+	// Prune hot snapshots
+	cutoff := nowMs - int64(sm.cfg.Streams.Ticker.HotRetentionHours)*60*60*1000
+	db, err := OpenDB(ss.DBPath("hot_ticker.db"))
+	if err != nil {
+		log.Printf("[maintenance] ticker hot open: %v", err)
+		return
+	}
+	result, err := db.Exec("DELETE FROM ticker_snapshots WHERE timestamp < ?", cutoff)
+	if err != nil {
+		log.Printf("[maintenance] ticker prune: %v", err)
+	} else {
+		deleted, _ := result.RowsAffected()
+		if deleted > 0 {
+			log.Printf("[maintenance] Pruned %d old ticker snapshots.", deleted)
+			db.Exec("PRAGMA incremental_vacuum(100)")
+		}
+	}
+	db.Close()
+
+	// Prune features
+	cutoffFeat := nowMs - int64(sm.cfg.Streams.Ticker.FeatureRetentionDays)*24*60*60*1000
+	sm.pruneStreamFeatures(ss, "ticker_features", "timestamp", cutoffFeat)
+}
+
+// maintenanceKlines prunes old kline data per interval.
+func (sm *StorageManager) maintenanceKlines(nowMs int64) {
+	ss := sm.streams["klines"]
+	if ss == nil {
+		return
+	}
+
+	for _, interval := range sm.cfg.Streams.Klines.Intervals {
+		dbName := fmt.Sprintf("kline_%s.db", interval)
+		db, err := OpenDB(ss.DBPath(dbName))
+		if err != nil {
+			log.Printf("[maintenance] kline_%s open: %v", interval, err)
+			continue
+		}
+
+		if interval == "60" {
+			// 60m klines: weekly archive rotation
+			cutoff := nowMs - int64(sm.cfg.Streams.Klines.LongRetentionWeeks)*7*24*60*60*1000
+			sm.migrateKlinesToArchive(db, ss, interval, cutoff)
+		} else {
+			// Short klines (5m, 15m): simple deletion after retention
+			cutoff := nowMs - int64(sm.cfg.Streams.Klines.ShortRetentionHours)*60*60*1000
+			result, err := db.Exec("DELETE FROM klines WHERE start_time < ?", cutoff)
+			if err != nil {
+				log.Printf("[maintenance] kline_%s prune: %v", interval, err)
+			} else {
+				deleted, _ := result.RowsAffected()
+				if deleted > 0 {
+					log.Printf("[maintenance] Pruned %d old kline_%s rows.", deleted, interval)
+					db.Exec("PRAGMA incremental_vacuum(100)")
+				}
+			}
+		}
+		db.Close()
+	}
+
+	// Prune kline features
+	cutoffFeat := nowMs - int64(sm.cfg.Streams.Klines.FeatureRetentionDays)*24*60*60*1000
+	sm.pruneStreamFeatures(ss, "kline_features", "timestamp", cutoffFeat)
+}
+
+// migrateKlinesToArchive archives 60m kline data into weekly files.
+func (sm *StorageManager) migrateKlinesToArchive(db *sql.DB, ss *StreamStorage, interval string, cutoffMs int64) {
+	var minTS, maxTS sql.NullInt64
+	err := db.QueryRow("SELECT MIN(start_time), MAX(start_time) FROM klines WHERE start_time < ?", cutoffMs).Scan(&minTS, &maxTS)
+	if err != nil || !minTS.Valid {
+		return
+	}
+
+	currentTs := minTS.Int64
+	for currentTs <= maxTS.Int64 {
+		archivePath := ss.ArchivePath(fmt.Sprintf("kline_%s", interval), currentTs)
+		nextWeekStart := sm.nextISOWeekStartMs(currentTs)
+		chunkEnd := nextWeekStart
+		if cutoffMs < chunkEnd {
+			chunkEnd = cutoffMs
+		}
+
+		// Initialize archive
+		adb, err := sql.Open("sqlite", archivePath)
+		if err != nil {
+			log.Printf("[maintenance] kline archive open: %v", err)
+			break
+		}
+		adb.Exec("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL;")
+		adb.Exec(`CREATE TABLE IF NOT EXISTS klines (
+			start_time INTEGER PRIMARY KEY, end_time INTEGER NOT NULL,
+			interval TEXT NOT NULL, open REAL NOT NULL, high REAL NOT NULL,
+			low REAL NOT NULL, close REAL NOT NULL, volume REAL NOT NULL,
+			turnover REAL NOT NULL, confirmed INTEGER NOT NULL
+		)`)
+		adb.Close()
+
+		// ATTACH and migrate atomically
+		_, err = db.Exec("ATTACH DATABASE ? AS archive", archivePath)
+		if err != nil {
+			log.Printf("[maintenance] kline attach: %v", err)
+			break
+		}
+
+		tx, err := db.Begin()
+		if err != nil {
+			db.Exec("DETACH DATABASE archive")
+			break
+		}
+
+		tx.Exec(`INSERT OR IGNORE INTO archive.klines SELECT * FROM main.klines WHERE start_time >= ? AND start_time < ?`, currentTs, chunkEnd)
+		result, _ := tx.Exec(`DELETE FROM main.klines WHERE start_time >= ? AND start_time < ?`, currentTs, chunkEnd)
+		if err := tx.Commit(); err != nil {
+			log.Printf("[maintenance] kline archive commit: %v", err)
+		} else {
+			migrated, _ := result.RowsAffected()
+			log.Printf("[maintenance] Archived %d kline_%s rows.", migrated, interval)
+		}
+
+		db.Exec("DETACH DATABASE archive")
+		currentTs = nextWeekStart
+	}
+
+	db.Exec("PRAGMA incremental_vacuum(100)")
+}
+
+// maintenanceOrderbook prunes old orderbook snapshots and features.
+func (sm *StorageManager) maintenanceOrderbook(nowMs int64) {
+	ss := sm.streams["orderbook"]
+	if ss == nil {
+		return
+	}
+
+	cutoff := nowMs - int64(sm.cfg.Streams.Orderbook.HotRetentionHours)*60*60*1000
+	db, err := OpenDB(ss.DBPath("hot_snapshots.db"))
+	if err != nil {
+		log.Printf("[maintenance] orderbook hot open: %v", err)
+		return
+	}
+	result, err := db.Exec("DELETE FROM ob_snapshots WHERE timestamp < ?", cutoff)
+	if err != nil {
+		log.Printf("[maintenance] orderbook prune: %v", err)
+	} else {
+		deleted, _ := result.RowsAffected()
+		if deleted > 0 {
+			log.Printf("[maintenance] Pruned %d old orderbook snapshot rows.", deleted)
+			db.Exec("PRAGMA incremental_vacuum(500)")
+		}
+	}
+	db.Close()
+
+	cutoffFeat := nowMs - int64(sm.cfg.Streams.Orderbook.FeatureRetentionDays)*24*60*60*1000
+	sm.pruneStreamFeatures(ss, "ob_features", "timestamp", cutoffFeat)
+}
+
+// maintenanceLiquidations prunes and archives old liquidation data.
+func (sm *StorageManager) maintenanceLiquidations(nowMs int64) {
+	ss := sm.streams["liquidations"]
+	if ss == nil {
+		return
+	}
+
+	// Migrate old liquidations to weekly archives (like trades)
+	cutoff := nowMs - int64(sm.cfg.Streams.Liquidations.HotRetentionHours)*60*60*1000
+	if err := sm.migrateLiquidations(cutoff); err != nil {
+		log.Printf("[maintenance] liquidation migration: %v", err)
+	}
+
+	// Prune features
+	cutoffFeat := nowMs - int64(sm.cfg.Streams.Liquidations.FeatureRetentionDays)*24*60*60*1000
+	sm.pruneStreamFeatures(ss, "liquidation_features", "timestamp", cutoffFeat)
+}
+
+// migrateLiquidations archives old liquidation events to weekly files.
+func (sm *StorageManager) migrateLiquidations(cutoffMs int64) error {
+	ss := sm.streams["liquidations"]
+	db, err := OpenDB(ss.DBPath("hot_liquidations.db"))
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	var minTS, maxTS sql.NullInt64
+	err = db.QueryRow("SELECT MIN(timestamp), MAX(timestamp) FROM liquidations WHERE timestamp < ?", cutoffMs).Scan(&minTS, &maxTS)
+	if err != nil || !minTS.Valid {
+		return nil
+	}
+
+	currentTs := minTS.Int64
+	for currentTs <= maxTS.Int64 {
+		archivePath := ss.ArchivePath("liquidations", currentTs)
+		nextWeekStart := sm.nextISOWeekStartMs(currentTs)
+		chunkEnd := nextWeekStart
+		if cutoffMs < chunkEnd {
+			chunkEnd = cutoffMs
+		}
+
+		if err := sm.initLiquidationArchiveDB(archivePath); err != nil {
+			return fmt.Errorf("init liq archive: %w", err)
+		}
+
+		_, err = db.Exec("ATTACH DATABASE ? AS archive", archivePath)
+		if err != nil {
+			return fmt.Errorf("attach liq archive: %w", err)
+		}
+
+		tx, err := db.Begin()
+		if err != nil {
+			db.Exec("DETACH DATABASE archive")
+			return err
+		}
+
+		tx.Exec(`INSERT OR IGNORE INTO archive.liquidations SELECT * FROM main.liquidations WHERE timestamp >= ? AND timestamp < ?`, currentTs, chunkEnd)
+		result, _ := tx.Exec(`DELETE FROM main.liquidations WHERE timestamp >= ? AND timestamp < ?`, currentTs, chunkEnd)
+		if err := tx.Commit(); err != nil {
+			log.Printf("[maintenance] liq archive commit: %v", err)
+		} else {
+			migrated, _ := result.RowsAffected()
+			log.Printf("[maintenance] Archived %d liquidation events.", migrated)
+		}
+
+		db.Exec("DETACH DATABASE archive")
+		currentTs = nextWeekStart
+	}
+
+	db.Exec("PRAGMA incremental_vacuum(100)")
+	return nil
+}
+
+// pruneStreamFeatures deletes old rows from a stream's features DB.
+func (sm *StorageManager) pruneStreamFeatures(ss *StreamStorage, tableName, tsColumn string, cutoffMs int64) {
+	db, err := OpenDB(ss.DBPath("features.db"))
+	if err != nil {
+		log.Printf("[maintenance] %s features open: %v", ss.streamName, err)
+		return
+	}
+	defer db.Close()
+
+	query := fmt.Sprintf("DELETE FROM %s WHERE %s < ?", tableName, tsColumn)
+	result, err := db.Exec(query, cutoffMs)
+	if err != nil {
+		log.Printf("[maintenance] %s feature prune: %v", ss.streamName, err)
+		return
+	}
+
+	deleted, _ := result.RowsAffected()
+	if deleted > 0 {
+		log.Printf("[maintenance] Pruned %d old %s feature rows.", deleted, ss.streamName)
+		db.Exec("PRAGMA incremental_vacuum(100)")
+	}
+}
+
+// ----------------------------------------------------------------
+// Startup recovery (trades stream)
+// ----------------------------------------------------------------
+
 // StartupRecovery checks for stale ticks and migrates them before normal operation.
 func (sm *StorageManager) StartupRecovery() error {
+	if !sm.cfg.Streams.Trades.Enabled {
+		return nil
+	}
+
 	log.Println("[startup] Checking for stale ticks in hot database...")

 	db, err := sm.OpenHotDB()
@@ -219,7 +923,12 @@ func (sm *StorageManager) StartupRecovery() error {
 	}
 	defer db.Close()

-	cutoffMs := time.Now().UnixMilli() - int64(sm.cfg.HotRetentionHours)*60*60*1000
+	retHours := sm.cfg.Streams.Trades.HotRetentionHours
+	if retHours == 0 {
+		retHours = sm.cfg.HotRetentionHours
+	}
+	cutoffMs := time.Now().UnixMilli() - int64(retHours)*60*60*1000
+
 	var count int64
 	err = db.QueryRow("SELECT COUNT(*) FROM btc_ticks WHERE trade_ts < ?", cutoffMs).Scan(&count)
 	if err != nil {