MrKalzu/bybit_btcusdt_ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Save all data available from the ws stream.

Browse Source
This commit is contained in:
Kalzu Rekku
2026-07-16 21:16:43 +03:00
parent e22dc05352
commit 7579c68524
6 changed files with 248 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
storage.go
+74 -15
View File
@@ -89,13 +89,30 @@ func (sm *StorageManager) initHotDB() error {
PRAGMA auto_vacuum=INCREMENTAL;
CREATE TABLE IF NOT EXISTS btc_ticks (
timestamp INTEGER NOT NULL,
price REAL NOT NULL,
seq INTEGER PRIMARY KEY,
trade_id TEXT NOT NULL,
trade_ts INTEGER NOT NULL,
message_ts INTEGER NOT NULL,
recv_ts INTEGER NOT NULL,
symbol TEXT NOT NULL,
side TEXT NOT NULL,
price REAL NOT NULL,
volume REAL NOT NULL,
side TEXT NOT NULL
tick_dir TEXT,
block_trade INTEGER NOT NULL,
rpi INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_timestamp ON btc_ticks(timestamp);
CREATE INDEX IF NOT EXISTS idx_trade_ts
ON btc_ticks(trade_ts);
CREATE INDEX IF NOT EXISTS idx_seq
ON btc_ticks(seq);
`)
return err
}
@@ -144,13 +161,30 @@ func (sm *StorageManager) initArchiveDB(path string) error {
PRAGMA synchronous=NORMAL;
CREATE TABLE IF NOT EXISTS btc_ticks (
timestamp INTEGER NOT NULL,
price REAL NOT NULL,
seq INTEGER PRIMARY KEY,
trade_id TEXT NOT NULL,
trade_ts INTEGER NOT NULL,
message_ts INTEGER NOT NULL,
recv_ts INTEGER NOT NULL,
symbol TEXT NOT NULL,
side TEXT NOT NULL,
price REAL NOT NULL,
volume REAL NOT NULL,
side TEXT NOT NULL
tick_dir TEXT,
block_trade INTEGER NOT NULL,
rpi INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_timestamp ON btc_ticks(timestamp);
CREATE INDEX IF NOT EXISTS idx_trade_ts
ON btc_ticks(trade_ts);
CREATE INDEX IF NOT EXISTS idx_seq
ON btc_ticks(seq);
`)
return err
}
@@ -187,7 +221,7 @@ func (sm *StorageManager) StartupRecovery() error {
cutoffMs := time.Now().UnixMilli() - int64(sm.cfg.HotRetentionHours)*60*60*1000
var count int64
err = db.QueryRow("SELECT COUNT(*) FROM btc_ticks WHERE timestamp < ?", cutoffMs).Scan(&count)
err = db.QueryRow("SELECT COUNT(*) FROM btc_ticks WHERE trade_ts < ?", cutoffMs).Scan(&count)
if err != nil {
return fmt.Errorf("count stale ticks: %w", err)
}
@@ -214,9 +248,9 @@ func (sm *StorageManager) migrateRawTicks(cutoffMs int64) error {
// Find the range of ticks that need migration
var minTS, maxTS sql.NullInt64
err = db.QueryRow(`
SELECT MIN(timestamp), MAX(timestamp)
SELECT MIN(trade_ts), MAX(trade_ts)
FROM btc_ticks
WHERE timestamp < ?
WHERE trade_ts < ?
`, cutoffMs).Scan(&minTS, &maxTS)
if err != nil {
return fmt.Errorf("query migration range: %w", err)
@@ -296,10 +330,35 @@ func (sm *StorageManager) atomicMigrate(hotDB *sql.DB, archivePath string, fromM
// Insert into archive
_, err = tx.Exec(`
INSERT INTO archive.btc_ticks (timestamp, price, volume, side)
SELECT timestamp, price, volume, side
INSERT OR IGNORE INTO archive.btc_ticks (
seq,
trade_id,
trade_ts,
message_ts,
recv_ts,
symbol,
side,
price,
volume,
tick_dir,
block_trade,
rpi
)
SELECT
seq,
trade_id,
trade_ts,
message_ts,
recv_ts,
symbol,
side,
price,
volume,
tick_dir,
block_trade,
rpi
FROM main.btc_ticks
WHERE timestamp >= ? AND timestamp < ?
WHERE trade_ts >= ? AND trade_ts < ?
`, fromMs, toMs)
if err != nil {
tx.Rollback()
@@ -309,7 +368,7 @@ func (sm *StorageManager) atomicMigrate(hotDB *sql.DB, archivePath string, fromM
// Delete from hot
result, err := tx.Exec(`
DELETE FROM main.btc_ticks
WHERE timestamp >= ? AND timestamp < ?
WHERE trade_ts >= ? AND trade_ts < ?
`, fromMs, toMs)
if err != nil {
tx.Rollback()