1123 lines
31 KiB
Go
1123 lines
31 KiB
Go
package main
|
|
|
|
import (
|
|
"database/sql"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"time"
|
|
|
|
_ "modernc.org/sqlite"
|
|
)
|
|
|
|
// StreamStorage manages databases for a single stream (trades, ticker, etc.).
|
|
// Each stream has its own subdirectory under the base data dir.
|
|
type StreamStorage struct {
|
|
streamName string
|
|
dataDir string // full path: e.g. "data/trades"
|
|
archiveDir string // full path: e.g. "data/trades/archive"
|
|
}
|
|
|
|
// NewStreamStorage creates directories for a stream and returns a StreamStorage.
|
|
func NewStreamStorage(baseDir, streamName string) (*StreamStorage, error) {
|
|
dataDir := filepath.Join(baseDir, streamName)
|
|
archiveDir := filepath.Join(dataDir, "archive")
|
|
|
|
if err := os.MkdirAll(archiveDir, 0755); err != nil {
|
|
return nil, fmt.Errorf("create %s dirs: %w", streamName, err)
|
|
}
|
|
|
|
return &StreamStorage{
|
|
streamName: streamName,
|
|
dataDir: dataDir,
|
|
archiveDir: archiveDir,
|
|
}, nil
|
|
}
|
|
|
|
// DBPath returns the full path for a database file within this stream's directory.
|
|
func (ss *StreamStorage) DBPath(filename string) string {
|
|
return filepath.Join(ss.dataDir, filename)
|
|
}
|
|
|
|
// ArchivePath returns an archive file path for a given timestamp using ISO week format.
|
|
func (ss *StreamStorage) ArchivePath(prefix string, timestampMs int64) string {
|
|
t := time.UnixMilli(timestampMs)
|
|
year, week := t.ISOWeek()
|
|
filename := fmt.Sprintf("%s_%d_W%02d.db", prefix, year, week)
|
|
return filepath.Join(ss.archiveDir, filename)
|
|
}
|
|
|
|
// OpenDB opens a SQLite database at the given path with WAL mode and performance pragmas.
|
|
func OpenDB(path string) (*sql.DB, error) {
|
|
db, err := sql.Open("sqlite", path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := db.Exec("PRAGMA journal_mode=WAL"); err != nil {
|
|
db.Close()
|
|
return nil, err
|
|
}
|
|
if _, err := db.Exec("PRAGMA synchronous=NORMAL"); err != nil {
|
|
db.Close()
|
|
return nil, err
|
|
}
|
|
return db, nil
|
|
}
|
|
|
|
// OpenDBWithAutoVacuum opens a SQLite database with WAL mode and incremental auto-vacuum.
|
|
func OpenDBWithAutoVacuum(path string) (*sql.DB, error) {
|
|
db, err := OpenDB(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := db.Exec("PRAGMA auto_vacuum=INCREMENTAL"); err != nil {
|
|
db.Close()
|
|
return nil, err
|
|
}
|
|
return db, nil
|
|
}
|
|
|
|
// ----------------------------------------------------------------
|
|
// StorageManager — coordinates per-stream storages and maintenance
|
|
// ----------------------------------------------------------------
|
|
|
|
// StorageManager handles all SQLite database operations: initialization,
|
|
// raw tick writing, feature writing, hourly migration, and pruning.
|
|
type StorageManager struct {
|
|
cfg Config
|
|
streams map[string]*StreamStorage
|
|
|
|
// Legacy paths (used for backward-compatible migration only)
|
|
hotDBPath string
|
|
featDBPath string
|
|
archiveDir string
|
|
}
|
|
|
|
// NewStorageManager creates directories, initializes databases, and returns a ready manager.
|
|
func NewStorageManager(cfg Config) (*StorageManager, error) {
|
|
sm := &StorageManager{
|
|
cfg: cfg,
|
|
streams: make(map[string]*StreamStorage),
|
|
}
|
|
|
|
// Perform data migration from old flat layout to new per-stream layout
|
|
if err := sm.migrateOldLayout(); err != nil {
|
|
return nil, fmt.Errorf("migrate old layout: %w", err)
|
|
}
|
|
|
|
// Initialize stream storages for enabled streams
|
|
streamNames := []string{}
|
|
if cfg.Streams.Trades.Enabled {
|
|
streamNames = append(streamNames, "trades")
|
|
}
|
|
if cfg.Streams.Ticker.Enabled {
|
|
streamNames = append(streamNames, "ticker")
|
|
}
|
|
if cfg.Streams.Klines.Enabled {
|
|
streamNames = append(streamNames, "klines")
|
|
}
|
|
if cfg.Streams.Orderbook.Enabled {
|
|
streamNames = append(streamNames, "orderbook")
|
|
}
|
|
if cfg.Streams.Liquidations.Enabled {
|
|
streamNames = append(streamNames, "liquidations")
|
|
}
|
|
|
|
for _, name := range streamNames {
|
|
ss, err := NewStreamStorage(cfg.DataDir, name)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
sm.streams[name] = ss
|
|
}
|
|
|
|
// Initialize databases per stream
|
|
if cfg.Streams.Trades.Enabled {
|
|
if err := sm.initTradesDBs(); err != nil {
|
|
return nil, fmt.Errorf("init trades dbs: %w", err)
|
|
}
|
|
}
|
|
if cfg.Streams.Ticker.Enabled {
|
|
if err := sm.initTickerDBs(); err != nil {
|
|
return nil, fmt.Errorf("init ticker dbs: %w", err)
|
|
}
|
|
}
|
|
if cfg.Streams.Klines.Enabled {
|
|
if err := sm.initKlineDBs(); err != nil {
|
|
return nil, fmt.Errorf("init kline dbs: %w", err)
|
|
}
|
|
}
|
|
if cfg.Streams.Orderbook.Enabled {
|
|
if err := sm.initOrderbookDBs(); err != nil {
|
|
return nil, fmt.Errorf("init orderbook dbs: %w", err)
|
|
}
|
|
}
|
|
if cfg.Streams.Liquidations.Enabled {
|
|
if err := sm.initLiquidationDBs(); err != nil {
|
|
return nil, fmt.Errorf("init liquidation dbs: %w", err)
|
|
}
|
|
}
|
|
|
|
// Set legacy paths for backward compatibility with trade-specific methods
|
|
if ss, ok := sm.streams["trades"]; ok {
|
|
sm.hotDBPath = ss.DBPath("hot_ticks.db")
|
|
sm.featDBPath = ss.DBPath("features.db")
|
|
sm.archiveDir = ss.archiveDir
|
|
}
|
|
|
|
return sm, nil
|
|
}
|
|
|
|
// GetStreamStorage returns the StreamStorage for a given stream name.
|
|
func (sm *StorageManager) GetStreamStorage(name string) *StreamStorage {
|
|
return sm.streams[name]
|
|
}
|
|
|
|
// ----------------------------------------------------------------
|
|
// Legacy compatibility methods (used by existing trade pipeline)
|
|
// ----------------------------------------------------------------
|
|
|
|
// OpenHotDB returns a new connection to the hot ticks database.
|
|
func (sm *StorageManager) OpenHotDB() (*sql.DB, error) {
|
|
return OpenDB(sm.hotDBPath)
|
|
}
|
|
|
|
// OpenFeaturesDB returns a new connection to the features database.
|
|
func (sm *StorageManager) OpenFeaturesDB() (*sql.DB, error) {
|
|
return OpenDB(sm.featDBPath)
|
|
}
|
|
|
|
// ----------------------------------------------------------------
|
|
// Database initialization per stream
|
|
// ----------------------------------------------------------------
|
|
|
|
func (sm *StorageManager) initTradesDBs() error {
|
|
ss := sm.streams["trades"]
|
|
|
|
db, err := OpenDBWithAutoVacuum(ss.DBPath("hot_ticks.db"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer db.Close()
|
|
|
|
_, err = db.Exec(`
|
|
CREATE TABLE IF NOT EXISTS btc_ticks (
|
|
seq INTEGER PRIMARY KEY,
|
|
|
|
trade_id TEXT NOT NULL,
|
|
|
|
trade_ts INTEGER NOT NULL,
|
|
message_ts INTEGER NOT NULL,
|
|
recv_ts INTEGER NOT NULL,
|
|
|
|
symbol TEXT NOT NULL,
|
|
side TEXT NOT NULL,
|
|
|
|
price REAL NOT NULL,
|
|
volume REAL NOT NULL,
|
|
|
|
tick_dir TEXT,
|
|
block_trade INTEGER NOT NULL,
|
|
rpi INTEGER NOT NULL
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_trade_ts
|
|
ON btc_ticks(trade_ts);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_seq
|
|
ON btc_ticks(seq);
|
|
`)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
featDB, err := OpenDBWithAutoVacuum(ss.DBPath("features.db"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer featDB.Close()
|
|
|
|
_, err = featDB.Exec(`
|
|
CREATE TABLE IF NOT EXISTS five_second_features (
|
|
timestamp INTEGER PRIMARY KEY,
|
|
log_return REAL NOT NULL,
|
|
realized_vol REAL NOT NULL,
|
|
ofi REAL NOT NULL,
|
|
volume_sum REAL NOT NULL,
|
|
close_price REAL NOT NULL,
|
|
vwap REAL NOT NULL
|
|
);
|
|
`)
|
|
return err
|
|
}
|
|
|
|
func (sm *StorageManager) initTickerDBs() error {
|
|
ss := sm.streams["ticker"]
|
|
|
|
db, err := OpenDBWithAutoVacuum(ss.DBPath("hot_ticker.db"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer db.Close()
|
|
|
|
_, err = db.Exec(`
|
|
CREATE TABLE IF NOT EXISTS ticker_snapshots (
|
|
timestamp INTEGER NOT NULL,
|
|
last_price REAL NOT NULL,
|
|
bid1_price REAL NOT NULL,
|
|
bid1_size REAL NOT NULL,
|
|
ask1_price REAL NOT NULL,
|
|
ask1_size REAL NOT NULL,
|
|
mark_price REAL NOT NULL,
|
|
index_price REAL NOT NULL,
|
|
open_interest REAL NOT NULL,
|
|
funding_rate REAL NOT NULL,
|
|
volume_24h REAL NOT NULL,
|
|
turnover_24h REAL NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_ticker_ts ON ticker_snapshots(timestamp);
|
|
`)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
featDB, err := OpenDBWithAutoVacuum(ss.DBPath("features.db"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer featDB.Close()
|
|
|
|
_, err = featDB.Exec(`
|
|
CREATE TABLE IF NOT EXISTS ticker_features (
|
|
timestamp INTEGER PRIMARY KEY,
|
|
spread REAL NOT NULL,
|
|
spread_bps REAL NOT NULL,
|
|
mid_price REAL NOT NULL,
|
|
oi_change REAL NOT NULL,
|
|
funding_rate REAL NOT NULL,
|
|
mark_index_basis REAL NOT NULL,
|
|
bid_ask_imbalance REAL NOT NULL
|
|
);
|
|
`)
|
|
return err
|
|
}
|
|
|
|
func (sm *StorageManager) initKlineDBs() error {
|
|
ss := sm.streams["klines"]
|
|
|
|
for _, interval := range sm.cfg.Streams.Klines.Intervals {
|
|
dbName := fmt.Sprintf("kline_%s.db", interval)
|
|
db, err := OpenDBWithAutoVacuum(ss.DBPath(dbName))
|
|
if err != nil {
|
|
return fmt.Errorf("init kline_%s: %w", interval, err)
|
|
}
|
|
|
|
_, err = db.Exec(`
|
|
CREATE TABLE IF NOT EXISTS klines (
|
|
start_time INTEGER PRIMARY KEY,
|
|
end_time INTEGER NOT NULL,
|
|
interval TEXT NOT NULL,
|
|
open REAL NOT NULL,
|
|
high REAL NOT NULL,
|
|
low REAL NOT NULL,
|
|
close REAL NOT NULL,
|
|
volume REAL NOT NULL,
|
|
turnover REAL NOT NULL,
|
|
confirmed INTEGER NOT NULL
|
|
);
|
|
`)
|
|
db.Close()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
featDB, err := OpenDBWithAutoVacuum(ss.DBPath("features.db"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer featDB.Close()
|
|
|
|
_, err = featDB.Exec(`
|
|
CREATE TABLE IF NOT EXISTS kline_features (
|
|
timestamp INTEGER NOT NULL,
|
|
interval TEXT NOT NULL,
|
|
body_ratio REAL NOT NULL,
|
|
upper_wick REAL NOT NULL,
|
|
lower_wick REAL NOT NULL,
|
|
log_return REAL NOT NULL,
|
|
volume REAL NOT NULL,
|
|
turnover REAL NOT NULL,
|
|
PRIMARY KEY (interval, timestamp)
|
|
);
|
|
`)
|
|
return err
|
|
}
|
|
|
|
func (sm *StorageManager) initOrderbookDBs() error {
|
|
ss := sm.streams["orderbook"]
|
|
|
|
db, err := OpenDBWithAutoVacuum(ss.DBPath("hot_snapshots.db"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer db.Close()
|
|
|
|
_, err = db.Exec(`
|
|
CREATE TABLE IF NOT EXISTS ob_snapshots (
|
|
timestamp INTEGER NOT NULL,
|
|
level INTEGER NOT NULL,
|
|
bid_price REAL,
|
|
bid_size REAL,
|
|
ask_price REAL,
|
|
ask_size REAL,
|
|
PRIMARY KEY (timestamp, level)
|
|
);
|
|
`)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
featDB, err := OpenDBWithAutoVacuum(ss.DBPath("features.db"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer featDB.Close()
|
|
|
|
_, err = featDB.Exec(`
|
|
CREATE TABLE IF NOT EXISTS ob_features (
|
|
timestamp INTEGER PRIMARY KEY,
|
|
spread REAL NOT NULL,
|
|
mid_price REAL NOT NULL,
|
|
bid_depth_5 REAL NOT NULL,
|
|
ask_depth_5 REAL NOT NULL,
|
|
bid_depth_20 REAL NOT NULL,
|
|
ask_depth_20 REAL NOT NULL,
|
|
depth_imbalance_5 REAL NOT NULL,
|
|
depth_imbalance_20 REAL NOT NULL,
|
|
weighted_mid REAL NOT NULL,
|
|
vwap_10 REAL NOT NULL
|
|
);
|
|
`)
|
|
return err
|
|
}
|
|
|
|
func (sm *StorageManager) initLiquidationDBs() error {
|
|
ss := sm.streams["liquidations"]
|
|
|
|
db, err := OpenDBWithAutoVacuum(ss.DBPath("hot_liquidations.db"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer db.Close()
|
|
|
|
_, err = db.Exec(`
|
|
CREATE TABLE IF NOT EXISTS liquidations (
|
|
timestamp INTEGER NOT NULL,
|
|
side TEXT NOT NULL,
|
|
price REAL NOT NULL,
|
|
quantity REAL NOT NULL,
|
|
value REAL NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_liq_ts ON liquidations(timestamp);
|
|
`)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
featDB, err := OpenDBWithAutoVacuum(ss.DBPath("features.db"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer featDB.Close()
|
|
|
|
_, err = featDB.Exec(`
|
|
CREATE TABLE IF NOT EXISTS liquidation_features (
|
|
timestamp INTEGER PRIMARY KEY,
|
|
count_total INTEGER NOT NULL,
|
|
count_long INTEGER NOT NULL,
|
|
count_short INTEGER NOT NULL,
|
|
volume_total REAL NOT NULL,
|
|
volume_long REAL NOT NULL,
|
|
volume_short REAL NOT NULL,
|
|
value_total REAL NOT NULL,
|
|
value_long REAL NOT NULL,
|
|
value_short REAL NOT NULL,
|
|
avg_price REAL NOT NULL,
|
|
net_value REAL NOT NULL
|
|
);
|
|
`)
|
|
return err
|
|
}
|
|
|
|
// initArchiveDB ensures an archive database has the btc_ticks schema.
|
|
func (sm *StorageManager) initArchiveDB(path string) error {
|
|
db, err := sql.Open("sqlite", path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer db.Close()
|
|
|
|
_, err = db.Exec(`
|
|
PRAGMA journal_mode=WAL;
|
|
PRAGMA synchronous=NORMAL;
|
|
|
|
CREATE TABLE IF NOT EXISTS btc_ticks (
|
|
seq INTEGER PRIMARY KEY,
|
|
|
|
trade_id TEXT NOT NULL,
|
|
|
|
trade_ts INTEGER NOT NULL,
|
|
message_ts INTEGER NOT NULL,
|
|
recv_ts INTEGER NOT NULL,
|
|
|
|
symbol TEXT NOT NULL,
|
|
side TEXT NOT NULL,
|
|
|
|
price REAL NOT NULL,
|
|
volume REAL NOT NULL,
|
|
|
|
tick_dir TEXT,
|
|
block_trade INTEGER NOT NULL,
|
|
rpi INTEGER NOT NULL
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_trade_ts
|
|
ON btc_ticks(trade_ts);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_seq
|
|
ON btc_ticks(seq);
|
|
`)
|
|
return err
|
|
}
|
|
|
|
// initLiquidationArchiveDB ensures a liquidation archive has the correct schema.
|
|
func (sm *StorageManager) initLiquidationArchiveDB(path string) error {
|
|
db, err := sql.Open("sqlite", path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer db.Close()
|
|
|
|
_, err = db.Exec(`
|
|
PRAGMA journal_mode=WAL;
|
|
PRAGMA synchronous=NORMAL;
|
|
|
|
CREATE TABLE IF NOT EXISTS liquidations (
|
|
timestamp INTEGER NOT NULL,
|
|
side TEXT NOT NULL,
|
|
price REAL NOT NULL,
|
|
quantity REAL NOT NULL,
|
|
value REAL NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_liq_ts ON liquidations(timestamp);
|
|
`)
|
|
return err
|
|
}
|
|
|
|
// ----------------------------------------------------------------
|
|
// Data migration from old flat layout
|
|
// ----------------------------------------------------------------
|
|
|
|
// migrateOldLayout moves data from the old flat data/ layout into data/trades/.
|
|
func (sm *StorageManager) migrateOldLayout() error {
|
|
oldHotDB := filepath.Join(sm.cfg.DataDir, "hot_ticks.db")
|
|
tradesDir := filepath.Join(sm.cfg.DataDir, "trades")
|
|
|
|
// Check if old layout exists and new doesn't
|
|
if _, err := os.Stat(oldHotDB); err != nil {
|
|
return nil // No old layout, nothing to migrate
|
|
}
|
|
if _, err := os.Stat(tradesDir); err == nil {
|
|
return nil // New layout already exists
|
|
}
|
|
|
|
log.Println("[migration] Detected old flat data layout, migrating to data/trades/...")
|
|
|
|
// Create trades directory
|
|
tradesArchiveDir := filepath.Join(tradesDir, "archive")
|
|
if err := os.MkdirAll(tradesArchiveDir, 0755); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Move hot_ticks.db and its WAL/SHM files
|
|
for _, suffix := range []string{"", "-wal", "-shm"} {
|
|
src := oldHotDB + suffix
|
|
dst := filepath.Join(tradesDir, "hot_ticks.db"+suffix)
|
|
if _, err := os.Stat(src); err == nil {
|
|
if err := os.Rename(src, dst); err != nil {
|
|
return fmt.Errorf("move %s: %w", src, err)
|
|
}
|
|
log.Printf("[migration] Moved %s -> %s", filepath.Base(src), dst)
|
|
}
|
|
}
|
|
|
|
// Move features.db and its WAL/SHM files
|
|
oldFeatDB := filepath.Join(sm.cfg.DataDir, "features.db")
|
|
for _, suffix := range []string{"", "-wal", "-shm"} {
|
|
src := oldFeatDB + suffix
|
|
dst := filepath.Join(tradesDir, "features.db"+suffix)
|
|
if _, err := os.Stat(src); err == nil {
|
|
if err := os.Rename(src, dst); err != nil {
|
|
return fmt.Errorf("move %s: %w", src, err)
|
|
}
|
|
log.Printf("[migration] Moved %s -> %s", filepath.Base(src), dst)
|
|
}
|
|
}
|
|
|
|
// Move archive directory contents
|
|
oldArchiveDir := filepath.Join(sm.cfg.DataDir, "archive")
|
|
if entries, err := os.ReadDir(oldArchiveDir); err == nil {
|
|
for _, entry := range entries {
|
|
src := filepath.Join(oldArchiveDir, entry.Name())
|
|
dst := filepath.Join(tradesArchiveDir, entry.Name())
|
|
if err := os.Rename(src, dst); err != nil {
|
|
return fmt.Errorf("move archive %s: %w", entry.Name(), err)
|
|
}
|
|
log.Printf("[migration] Moved archive/%s -> trades/archive/%s", entry.Name(), entry.Name())
|
|
}
|
|
// Remove empty old archive dir
|
|
os.Remove(oldArchiveDir)
|
|
}
|
|
|
|
log.Println("[migration] Data migration to data/trades/ completed successfully.")
|
|
return nil
|
|
}
|
|
|
|
// ----------------------------------------------------------------
|
|
// Maintenance — trade stream (backward compatible)
|
|
// ----------------------------------------------------------------
|
|
|
|
// weeklyArchivePath resolves the archive DB file path for a given timestamp.
|
|
func (sm *StorageManager) weeklyArchivePath(timestampMs int64) string {
|
|
t := time.UnixMilli(timestampMs)
|
|
year, week := t.ISOWeek()
|
|
filename := fmt.Sprintf("btc_ticks_%d_W%02d.db", year, week)
|
|
return filepath.Join(sm.archiveDir, filename)
|
|
}
|
|
|
|
// RunHourlyMaintenance performs maintenance across all enabled streams.
|
|
func (sm *StorageManager) RunHourlyMaintenance() {
|
|
nowMs := time.Now().UnixMilli()
|
|
log.Println("[maintenance] Starting hourly database maintenance...")
|
|
|
|
// --- Trades ---
|
|
if sm.cfg.Streams.Trades.Enabled {
|
|
retHours := sm.cfg.Streams.Trades.HotRetentionHours
|
|
if retHours == 0 {
|
|
retHours = sm.cfg.HotRetentionHours
|
|
}
|
|
cutoff := nowMs - int64(retHours)*60*60*1000
|
|
if err := sm.migrateRawTicks(cutoff); err != nil {
|
|
log.Printf("[maintenance] Trade tick migration failed: %v", err)
|
|
}
|
|
|
|
retDays := sm.cfg.Streams.Trades.FeatureRetentionDays
|
|
if retDays == 0 {
|
|
retDays = sm.cfg.FeatureRetentionDays
|
|
}
|
|
cutoffFeat := nowMs - int64(retDays)*24*60*60*1000
|
|
if err := sm.pruneFeatures(cutoffFeat); err != nil {
|
|
log.Printf("[maintenance] Trade feature pruning failed: %v", err)
|
|
}
|
|
}
|
|
|
|
// --- Ticker ---
|
|
if sm.cfg.Streams.Ticker.Enabled {
|
|
sm.maintenanceTicker(nowMs)
|
|
}
|
|
|
|
// --- Klines ---
|
|
if sm.cfg.Streams.Klines.Enabled {
|
|
sm.maintenanceKlines(nowMs)
|
|
}
|
|
|
|
// --- Orderbook ---
|
|
if sm.cfg.Streams.Orderbook.Enabled {
|
|
sm.maintenanceOrderbook(nowMs)
|
|
}
|
|
|
|
// --- Liquidations ---
|
|
if sm.cfg.Streams.Liquidations.Enabled {
|
|
sm.maintenanceLiquidations(nowMs)
|
|
}
|
|
|
|
log.Println("[maintenance] Hourly maintenance complete.")
|
|
}
|
|
|
|
// maintenanceTicker prunes old ticker snapshots and features.
|
|
func (sm *StorageManager) maintenanceTicker(nowMs int64) {
|
|
ss := sm.streams["ticker"]
|
|
if ss == nil {
|
|
return
|
|
}
|
|
|
|
// Prune hot snapshots
|
|
cutoff := nowMs - int64(sm.cfg.Streams.Ticker.HotRetentionHours)*60*60*1000
|
|
db, err := OpenDB(ss.DBPath("hot_ticker.db"))
|
|
if err != nil {
|
|
log.Printf("[maintenance] ticker hot open: %v", err)
|
|
return
|
|
}
|
|
result, err := db.Exec("DELETE FROM ticker_snapshots WHERE timestamp < ?", cutoff)
|
|
if err != nil {
|
|
log.Printf("[maintenance] ticker prune: %v", err)
|
|
} else {
|
|
deleted, _ := result.RowsAffected()
|
|
if deleted > 0 {
|
|
log.Printf("[maintenance] Pruned %d old ticker snapshots.", deleted)
|
|
db.Exec("PRAGMA incremental_vacuum(100)")
|
|
}
|
|
}
|
|
db.Close()
|
|
|
|
// Prune features
|
|
cutoffFeat := nowMs - int64(sm.cfg.Streams.Ticker.FeatureRetentionDays)*24*60*60*1000
|
|
sm.pruneStreamFeatures(ss, "ticker_features", "timestamp", cutoffFeat)
|
|
}
|
|
|
|
// maintenanceKlines prunes old kline data per interval.
|
|
func (sm *StorageManager) maintenanceKlines(nowMs int64) {
|
|
ss := sm.streams["klines"]
|
|
if ss == nil {
|
|
return
|
|
}
|
|
|
|
for _, interval := range sm.cfg.Streams.Klines.Intervals {
|
|
dbName := fmt.Sprintf("kline_%s.db", interval)
|
|
db, err := OpenDB(ss.DBPath(dbName))
|
|
if err != nil {
|
|
log.Printf("[maintenance] kline_%s open: %v", interval, err)
|
|
continue
|
|
}
|
|
|
|
if interval == "60" {
|
|
// 60m klines: weekly archive rotation
|
|
cutoff := nowMs - int64(sm.cfg.Streams.Klines.LongRetentionWeeks)*7*24*60*60*1000
|
|
sm.migrateKlinesToArchive(db, ss, interval, cutoff)
|
|
} else {
|
|
// Short klines (5m, 15m): simple deletion after retention
|
|
cutoff := nowMs - int64(sm.cfg.Streams.Klines.ShortRetentionHours)*60*60*1000
|
|
result, err := db.Exec("DELETE FROM klines WHERE start_time < ?", cutoff)
|
|
if err != nil {
|
|
log.Printf("[maintenance] kline_%s prune: %v", interval, err)
|
|
} else {
|
|
deleted, _ := result.RowsAffected()
|
|
if deleted > 0 {
|
|
log.Printf("[maintenance] Pruned %d old kline_%s rows.", deleted, interval)
|
|
db.Exec("PRAGMA incremental_vacuum(100)")
|
|
}
|
|
}
|
|
}
|
|
db.Close()
|
|
}
|
|
|
|
// Prune kline features
|
|
cutoffFeat := nowMs - int64(sm.cfg.Streams.Klines.FeatureRetentionDays)*24*60*60*1000
|
|
sm.pruneStreamFeatures(ss, "kline_features", "timestamp", cutoffFeat)
|
|
}
|
|
|
|
// migrateKlinesToArchive archives 60m kline data into weekly files.
|
|
func (sm *StorageManager) migrateKlinesToArchive(db *sql.DB, ss *StreamStorage, interval string, cutoffMs int64) {
|
|
var minTS, maxTS sql.NullInt64
|
|
err := db.QueryRow("SELECT MIN(start_time), MAX(start_time) FROM klines WHERE start_time < ?", cutoffMs).Scan(&minTS, &maxTS)
|
|
if err != nil || !minTS.Valid {
|
|
return
|
|
}
|
|
|
|
currentTs := minTS.Int64
|
|
for currentTs <= maxTS.Int64 {
|
|
archivePath := ss.ArchivePath(fmt.Sprintf("kline_%s", interval), currentTs)
|
|
nextWeekStart := sm.nextISOWeekStartMs(currentTs)
|
|
chunkEnd := nextWeekStart
|
|
if cutoffMs < chunkEnd {
|
|
chunkEnd = cutoffMs
|
|
}
|
|
|
|
// Initialize archive
|
|
adb, err := sql.Open("sqlite", archivePath)
|
|
if err != nil {
|
|
log.Printf("[maintenance] kline archive open: %v", err)
|
|
break
|
|
}
|
|
adb.Exec("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL;")
|
|
adb.Exec(`CREATE TABLE IF NOT EXISTS klines (
|
|
start_time INTEGER PRIMARY KEY, end_time INTEGER NOT NULL,
|
|
interval TEXT NOT NULL, open REAL NOT NULL, high REAL NOT NULL,
|
|
low REAL NOT NULL, close REAL NOT NULL, volume REAL NOT NULL,
|
|
turnover REAL NOT NULL, confirmed INTEGER NOT NULL
|
|
)`)
|
|
adb.Close()
|
|
|
|
// ATTACH and migrate atomically
|
|
_, err = db.Exec("ATTACH DATABASE ? AS archive", archivePath)
|
|
if err != nil {
|
|
log.Printf("[maintenance] kline attach: %v", err)
|
|
break
|
|
}
|
|
|
|
tx, err := db.Begin()
|
|
if err != nil {
|
|
db.Exec("DETACH DATABASE archive")
|
|
break
|
|
}
|
|
|
|
tx.Exec(`INSERT OR IGNORE INTO archive.klines SELECT * FROM main.klines WHERE start_time >= ? AND start_time < ?`, currentTs, chunkEnd)
|
|
result, _ := tx.Exec(`DELETE FROM main.klines WHERE start_time >= ? AND start_time < ?`, currentTs, chunkEnd)
|
|
if err := tx.Commit(); err != nil {
|
|
log.Printf("[maintenance] kline archive commit: %v", err)
|
|
} else {
|
|
migrated, _ := result.RowsAffected()
|
|
log.Printf("[maintenance] Archived %d kline_%s rows.", migrated, interval)
|
|
}
|
|
|
|
db.Exec("DETACH DATABASE archive")
|
|
currentTs = nextWeekStart
|
|
}
|
|
|
|
db.Exec("PRAGMA incremental_vacuum(100)")
|
|
}
|
|
|
|
// maintenanceOrderbook prunes old orderbook snapshots and features.
|
|
func (sm *StorageManager) maintenanceOrderbook(nowMs int64) {
|
|
ss := sm.streams["orderbook"]
|
|
if ss == nil {
|
|
return
|
|
}
|
|
|
|
cutoff := nowMs - int64(sm.cfg.Streams.Orderbook.HotRetentionHours)*60*60*1000
|
|
db, err := OpenDB(ss.DBPath("hot_snapshots.db"))
|
|
if err != nil {
|
|
log.Printf("[maintenance] orderbook hot open: %v", err)
|
|
return
|
|
}
|
|
result, err := db.Exec("DELETE FROM ob_snapshots WHERE timestamp < ?", cutoff)
|
|
if err != nil {
|
|
log.Printf("[maintenance] orderbook prune: %v", err)
|
|
} else {
|
|
deleted, _ := result.RowsAffected()
|
|
if deleted > 0 {
|
|
log.Printf("[maintenance] Pruned %d old orderbook snapshot rows.", deleted)
|
|
db.Exec("PRAGMA incremental_vacuum(500)")
|
|
}
|
|
}
|
|
db.Close()
|
|
|
|
cutoffFeat := nowMs - int64(sm.cfg.Streams.Orderbook.FeatureRetentionDays)*24*60*60*1000
|
|
sm.pruneStreamFeatures(ss, "ob_features", "timestamp", cutoffFeat)
|
|
}
|
|
|
|
// maintenanceLiquidations prunes and archives old liquidation data.
|
|
func (sm *StorageManager) maintenanceLiquidations(nowMs int64) {
|
|
ss := sm.streams["liquidations"]
|
|
if ss == nil {
|
|
return
|
|
}
|
|
|
|
// Migrate old liquidations to weekly archives (like trades)
|
|
cutoff := nowMs - int64(sm.cfg.Streams.Liquidations.HotRetentionHours)*60*60*1000
|
|
if err := sm.migrateLiquidations(cutoff); err != nil {
|
|
log.Printf("[maintenance] liquidation migration: %v", err)
|
|
}
|
|
|
|
// Prune features
|
|
cutoffFeat := nowMs - int64(sm.cfg.Streams.Liquidations.FeatureRetentionDays)*24*60*60*1000
|
|
sm.pruneStreamFeatures(ss, "liquidation_features", "timestamp", cutoffFeat)
|
|
}
|
|
|
|
// migrateLiquidations archives old liquidation events to weekly files.
|
|
func (sm *StorageManager) migrateLiquidations(cutoffMs int64) error {
|
|
ss := sm.streams["liquidations"]
|
|
db, err := OpenDB(ss.DBPath("hot_liquidations.db"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer db.Close()
|
|
|
|
var minTS, maxTS sql.NullInt64
|
|
err = db.QueryRow("SELECT MIN(timestamp), MAX(timestamp) FROM liquidations WHERE timestamp < ?", cutoffMs).Scan(&minTS, &maxTS)
|
|
if err != nil || !minTS.Valid {
|
|
return nil
|
|
}
|
|
|
|
currentTs := minTS.Int64
|
|
for currentTs <= maxTS.Int64 {
|
|
archivePath := ss.ArchivePath("liquidations", currentTs)
|
|
nextWeekStart := sm.nextISOWeekStartMs(currentTs)
|
|
chunkEnd := nextWeekStart
|
|
if cutoffMs < chunkEnd {
|
|
chunkEnd = cutoffMs
|
|
}
|
|
|
|
if err := sm.initLiquidationArchiveDB(archivePath); err != nil {
|
|
return fmt.Errorf("init liq archive: %w", err)
|
|
}
|
|
|
|
_, err = db.Exec("ATTACH DATABASE ? AS archive", archivePath)
|
|
if err != nil {
|
|
return fmt.Errorf("attach liq archive: %w", err)
|
|
}
|
|
|
|
tx, err := db.Begin()
|
|
if err != nil {
|
|
db.Exec("DETACH DATABASE archive")
|
|
return err
|
|
}
|
|
|
|
tx.Exec(`INSERT OR IGNORE INTO archive.liquidations SELECT * FROM main.liquidations WHERE timestamp >= ? AND timestamp < ?`, currentTs, chunkEnd)
|
|
result, _ := tx.Exec(`DELETE FROM main.liquidations WHERE timestamp >= ? AND timestamp < ?`, currentTs, chunkEnd)
|
|
if err := tx.Commit(); err != nil {
|
|
log.Printf("[maintenance] liq archive commit: %v", err)
|
|
} else {
|
|
migrated, _ := result.RowsAffected()
|
|
log.Printf("[maintenance] Archived %d liquidation events.", migrated)
|
|
}
|
|
|
|
db.Exec("DETACH DATABASE archive")
|
|
currentTs = nextWeekStart
|
|
}
|
|
|
|
db.Exec("PRAGMA incremental_vacuum(100)")
|
|
return nil
|
|
}
|
|
|
|
// pruneStreamFeatures deletes old rows from a stream's features DB.
|
|
func (sm *StorageManager) pruneStreamFeatures(ss *StreamStorage, tableName, tsColumn string, cutoffMs int64) {
|
|
db, err := OpenDB(ss.DBPath("features.db"))
|
|
if err != nil {
|
|
log.Printf("[maintenance] %s features open: %v", ss.streamName, err)
|
|
return
|
|
}
|
|
defer db.Close()
|
|
|
|
query := fmt.Sprintf("DELETE FROM %s WHERE %s < ?", tableName, tsColumn)
|
|
result, err := db.Exec(query, cutoffMs)
|
|
if err != nil {
|
|
log.Printf("[maintenance] %s feature prune: %v", ss.streamName, err)
|
|
return
|
|
}
|
|
|
|
deleted, _ := result.RowsAffected()
|
|
if deleted > 0 {
|
|
log.Printf("[maintenance] Pruned %d old %s feature rows.", deleted, ss.streamName)
|
|
db.Exec("PRAGMA incremental_vacuum(100)")
|
|
}
|
|
}
|
|
|
|
// ----------------------------------------------------------------
|
|
// Startup recovery (trades stream)
|
|
// ----------------------------------------------------------------
|
|
|
|
// StartupRecovery checks for stale ticks and migrates them before normal operation.
|
|
func (sm *StorageManager) StartupRecovery() error {
|
|
if !sm.cfg.Streams.Trades.Enabled {
|
|
return nil
|
|
}
|
|
|
|
log.Println("[startup] Checking for stale ticks in hot database...")
|
|
|
|
db, err := sm.OpenHotDB()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer db.Close()
|
|
|
|
retHours := sm.cfg.Streams.Trades.HotRetentionHours
|
|
if retHours == 0 {
|
|
retHours = sm.cfg.HotRetentionHours
|
|
}
|
|
cutoffMs := time.Now().UnixMilli() - int64(retHours)*60*60*1000
|
|
|
|
var count int64
|
|
err = db.QueryRow("SELECT COUNT(*) FROM btc_ticks WHERE trade_ts < ?", cutoffMs).Scan(&count)
|
|
if err != nil {
|
|
return fmt.Errorf("count stale ticks: %w", err)
|
|
}
|
|
|
|
if count > 0 {
|
|
log.Printf("[startup] Found %d stale ticks, triggering immediate migration...", count)
|
|
if err := sm.migrateRawTicks(cutoffMs); err != nil {
|
|
return fmt.Errorf("startup migration: %w", err)
|
|
}
|
|
} else {
|
|
log.Println("[startup] No stale ticks found, hot database is clean.")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (sm *StorageManager) migrateRawTicks(cutoffMs int64) error {
|
|
db, err := sm.OpenHotDB()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer db.Close()
|
|
|
|
// Find the range of ticks that need migration
|
|
var minTS, maxTS sql.NullInt64
|
|
err = db.QueryRow(`
|
|
SELECT MIN(trade_ts), MAX(trade_ts)
|
|
FROM btc_ticks
|
|
WHERE trade_ts < ?
|
|
`, cutoffMs).Scan(&minTS, &maxTS)
|
|
if err != nil {
|
|
return fmt.Errorf("query migration range: %w", err)
|
|
}
|
|
|
|
if !minTS.Valid || !maxTS.Valid {
|
|
log.Println("[maintenance] No raw ticks older than cutoff to migrate.")
|
|
return nil
|
|
}
|
|
|
|
// Process ticks week by week to handle cross-week boundaries
|
|
currentTs := minTS.Int64
|
|
for currentTs <= maxTS.Int64 {
|
|
t := time.UnixMilli(currentTs)
|
|
year, week := t.ISOWeek()
|
|
archivePath := sm.weeklyArchivePath(currentTs)
|
|
|
|
log.Printf("[maintenance] Migrating ticks for %d W%02d -> %s",
|
|
year, week, filepath.Base(archivePath))
|
|
|
|
// Calculate the start of the next ISO week (Monday 00:00:00 UTC)
|
|
nextWeekStart := sm.nextISOWeekStartMs(currentTs)
|
|
|
|
// The upper bound for this chunk: either next week start or cutoff, whichever is smaller
|
|
chunkEnd := nextWeekStart
|
|
if cutoffMs < chunkEnd {
|
|
chunkEnd = cutoffMs
|
|
}
|
|
|
|
// Ensure archive DB exists and has schema
|
|
if err := sm.initArchiveDB(archivePath); err != nil {
|
|
return fmt.Errorf("init archive db: %w", err)
|
|
}
|
|
|
|
// Attach and atomically migrate
|
|
if err := sm.atomicMigrate(db, archivePath, currentTs, chunkEnd); err != nil {
|
|
return fmt.Errorf("atomic migrate: %w", err)
|
|
}
|
|
|
|
currentTs = nextWeekStart
|
|
}
|
|
|
|
// Reclaim space
|
|
if _, err := db.Exec("PRAGMA incremental_vacuum(500)"); err != nil {
|
|
log.Printf("[maintenance] incremental_vacuum warning: %v", err)
|
|
}
|
|
|
|
log.Println("[maintenance] Raw tick migration completed successfully.")
|
|
return nil
|
|
}
|
|
|
|
// nextISOWeekStartMs calculates the epoch ms of the next Monday 00:00:00 UTC
|
|
// relative to the given timestamp.
|
|
func (sm *StorageManager) nextISOWeekStartMs(timestampMs int64) int64 {
|
|
t := time.UnixMilli(timestampMs).UTC()
|
|
// Calculate days until next Monday
|
|
daysUntilMonday := (8 - int(t.Weekday())) % 7
|
|
if daysUntilMonday == 0 {
|
|
daysUntilMonday = 7
|
|
}
|
|
nextMonday := time.Date(t.Year(), t.Month(), t.Day()+daysUntilMonday, 0, 0, 0, 0, time.UTC)
|
|
return nextMonday.UnixMilli()
|
|
}
|
|
|
|
func (sm *StorageManager) atomicMigrate(hotDB *sql.DB, archivePath string, fromMs, toMs int64) error {
|
|
// Attach the archive database
|
|
_, err := hotDB.Exec("ATTACH DATABASE ? AS archive", archivePath)
|
|
if err != nil {
|
|
return fmt.Errorf("attach archive: %w", err)
|
|
}
|
|
defer hotDB.Exec("DETACH DATABASE archive")
|
|
|
|
tx, err := hotDB.Begin()
|
|
if err != nil {
|
|
return fmt.Errorf("begin migration tx: %w", err)
|
|
}
|
|
|
|
// Insert into archive
|
|
_, err = tx.Exec(`
|
|
INSERT OR IGNORE INTO archive.btc_ticks (
|
|
seq,
|
|
trade_id,
|
|
trade_ts,
|
|
message_ts,
|
|
recv_ts,
|
|
symbol,
|
|
side,
|
|
price,
|
|
volume,
|
|
tick_dir,
|
|
block_trade,
|
|
rpi
|
|
)
|
|
SELECT
|
|
seq,
|
|
trade_id,
|
|
trade_ts,
|
|
message_ts,
|
|
recv_ts,
|
|
symbol,
|
|
side,
|
|
price,
|
|
volume,
|
|
tick_dir,
|
|
block_trade,
|
|
rpi
|
|
FROM main.btc_ticks
|
|
WHERE trade_ts >= ? AND trade_ts < ?
|
|
`, fromMs, toMs)
|
|
if err != nil {
|
|
tx.Rollback()
|
|
return fmt.Errorf("insert into archive: %w", err)
|
|
}
|
|
|
|
// Delete from hot
|
|
result, err := tx.Exec(`
|
|
DELETE FROM main.btc_ticks
|
|
WHERE trade_ts >= ? AND trade_ts < ?
|
|
`, fromMs, toMs)
|
|
if err != nil {
|
|
tx.Rollback()
|
|
return fmt.Errorf("delete from hot: %w", err)
|
|
}
|
|
|
|
migrated, _ := result.RowsAffected()
|
|
if err := tx.Commit(); err != nil {
|
|
return fmt.Errorf("commit migration: %w", err)
|
|
}
|
|
|
|
log.Printf("[maintenance] Migrated %d ticks to archive.", migrated)
|
|
return nil
|
|
}
|
|
|
|
func (sm *StorageManager) pruneFeatures(cutoffMs int64) error {
|
|
db, err := sm.OpenFeaturesDB()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer db.Close()
|
|
|
|
cutoffTime := time.UnixMilli(cutoffMs)
|
|
log.Printf("[maintenance] Pruning features older than %s", cutoffTime.Format(time.RFC3339))
|
|
|
|
result, err := db.Exec("DELETE FROM five_second_features WHERE timestamp < ?", cutoffMs)
|
|
if err != nil {
|
|
return fmt.Errorf("delete old features: %w", err)
|
|
}
|
|
|
|
deleted, _ := result.RowsAffected()
|
|
if deleted > 0 {
|
|
log.Printf("[maintenance] Pruned %d old feature rows.", deleted)
|
|
if _, err := db.Exec("PRAGMA incremental_vacuum(100)"); err != nil {
|
|
log.Printf("[maintenance] feature vacuum warning: %v", err)
|
|
}
|
|
} else {
|
|
log.Println("[maintenance] No feature rows needed pruning.")
|
|
}
|
|
|
|
return nil
|
|
}
|