Initial commit.

2026-07-14 20:56:34 +03:00
commit 6a58f1ba7e
11 changed files with 1114 additions and 0 deletions
@@ -0,0 +1,112 @@
+package main
+
+import (
+	"context"
+	"log"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+	"time"
+)
+
+func main() {
+	log.SetFlags(log.LstdFlags | log.Lmicroseconds)
+	log.Println("=== Bybit BTC/USDT Tick Ingest Engine ===")
+
+	// Load configuration
+	cfg, err := LoadConfig("config.json")
+	if err != nil {
+		log.Fatalf("Config error: %v", err)
+	}
+	log.Printf("Config: symbol=%s, hot_retention=%dh, feature_retention=%dd",
+		cfg.Symbol, cfg.HotRetentionHours, cfg.FeatureRetentionDays)
+
+	// Initialize storage (creates dirs, databases, tables)
+	sm, err := NewStorageManager(cfg)
+	if err != nil {
+		log.Fatalf("Storage init error: %v", err)
+	}
+
+	// Startup recovery: migrate any stale ticks from previous runs
+	if err := sm.StartupRecovery(); err != nil {
+		log.Fatalf("Startup recovery error: %v", err)
+	}
+
+	// Create the tick channel (buffered to absorb bursts)
+	tickCh := make(chan Tick, cfg.TickChannelBuffer)
+
+	// Initialize aggregator (5-second feature bucketing)
+	agg, err := NewAggregator(sm)
+	if err != nil {
+		log.Fatalf("Aggregator init error: %v", err)
+	}
+
+	// Initialize batch writer
+	writer, err := NewWriter(sm, tickCh, cfg)
+	if err != nil {
+		log.Fatalf("Writer init error: %v", err)
+	}
+
+	// Create ingestor
+	ingestor := NewIngestor(cfg, tickCh, agg)
+
+	// Context for graceful shutdown
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// Catch OS signals
+	sigCh := make(chan os.Signal, 1)
+	signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
+
+	var wg sync.WaitGroup
+
+	// Goroutine 1: Batch Writer
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		writer.Run(ctx)
+	}()
+
+	// Goroutine 2: WebSocket Ingestor
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		ingestor.Run(ctx)
+	}()
+
+	// Goroutine 3: Hourly Maintenance
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		maintInterval := time.Duration(cfg.MaintenanceIntervalMinutes) * time.Minute
+		ticker := time.NewTicker(maintInterval)
+		defer ticker.Stop()
+
+		log.Printf("[maintenance] Scheduled every %d minutes.", cfg.MaintenanceIntervalMinutes)
+		for {
+			select {
+			case <-ctx.Done():
+				log.Println("[maintenance] Maintenance goroutine stopped.")
+				return
+			case <-ticker.C:
+				sm.RunHourlyMaintenance()
+			}
+		}
+	}()
+
+	// Wait for shutdown signal
+	sig := <-sigCh
+	log.Printf("Received signal %v, initiating graceful shutdown...", sig)
+	cancel()
+
+	// Close the tick channel so the writer drains remaining ticks
+	close(tickCh)
+
+	// Flush any remaining aggregator bucket
+	agg.Close()
+
+	// Wait for all goroutines to finish
+	wg.Wait()
+	log.Println("=== Engine shut down cleanly. ===")
+}