Added simple stats program to prope the ingest engine databases.

utils/stats/main.go

@@ -0,0 +1,151 @@
+package main
+
+import (
+	"database/sql"
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	_ "modernc.org/sqlite"
+)
+
+// ANSI Color Codes
+const (
+	ColorReset  = "\033[0m"
+	ColorGreen  = "\033[32m"
+	ColorYellow = "\033[33m"
+	ColorCyan   = "\033[36m"
+	ColorRed    = "\033[31m"
+	ColorGray   = "\033[90m"
+	ColorBold   = "\033[1m"
+)
+
+func main() {
+	dataDir := flag.String("data", "./data", "Path to the data directory")
+	noColor := flag.Bool("no-color", false, "Disable colored output")
+	flag.Parse()
+
+	// Disable colors if requested or if not a TTY (simplified check)
+	if *noColor {
+		disableColors()
+	}
+
+	fmt.Printf("%s%s--- Database Dashboard (%s) ---%s\n", ColorBold, ColorGreen, time.Now().Format(time.RFC822), ColorReset)
+
+	// 1. Hot Ticks Stats
+	hotPath := filepath.Join(*dataDir, "hot_ticks.db")
+	printTableStats("HOT STORAGE (RECENT TICKS)", hotPath, "btc_ticks", "trade_ts")
+
+	// 2. Feature Stats
+	featPath := filepath.Join(*dataDir, "features.db")
+	printTableStats("FEATURE STORAGE (ANALYTICS)", featPath, "five_second_features", "timestamp")
+
+	// 3. Archive Stats
+	archiveDir := filepath.Join(*dataDir, "archive")
+	files, _ := filepath.Glob(filepath.Join(archiveDir, "*.db"))
+
+	if len(files) > 0 {
+		fmt.Printf("\n%s%s[ ARCHIVE DIRECTORY: %d FILES ]%s\n", ColorBold, ColorCyan, len(files), ColorReset)
+		var totalArchived int64
+		var totalSize int64
+		
+		for _, f := range files {
+			count, minTs, maxTs, err := getBasicStats(f, "btc_ticks", "trade_ts")
+			size := getFullDBSize(f)
+			totalSize += size
+			
+			if err != nil {
+				fmt.Printf("  %s!%s %-25s: Error reading (%v)\n", ColorRed, ColorReset, filepath.Base(f), err)
+				continue
+			}
+			totalArchived += count
+			fmt.Printf("  %s•%s %-25s %s%10s%s | %s%12d rows%s | %s to %s\n",
+				ColorCyan, ColorReset, filepath.Base(f), 
+				ColorYellow, formatBytes(size), ColorReset,
+				ColorGray, count, ColorReset,
+				formatMs(minTs), formatMs(maxTs))
+		}
+		fmt.Printf("%sTotal Archived: %d rows across %s%s\n", ColorBold, totalArchived, formatBytes(totalSize), ColorReset)
+	} else {
+		fmt.Printf("\n%s[ ARCHIVE ] No weekly archive files found.%s\n", ColorGray, ColorReset)
+	}
+}
+
+func printTableStats(label, dbPath, table, tsCol string) {
+	fmt.Printf("\n%s%s[ %s ]%s\n", ColorBold, ColorCyan, label, ColorReset)
+	
+	if _, err := os.Stat(dbPath); os.IsNotExist(err) {
+		fmt.Printf("  %sFile not found: %s%s\n", ColorRed, dbPath, ColorReset)
+		return
+	}
+
+	size := getFullDBSize(dbPath)
+	count, minTs, maxTs, err := getBasicStats(dbPath, table, tsCol)
+	
+	if err != nil {
+		fmt.Printf("  %sDatabase Error: %v%s\n", ColorRed, err, ColorReset)
+		return
+	}
+
+	fmt.Printf("  %-12s %s%s%s\n", "Disk Usage:", ColorYellow, formatBytes(size), ColorReset)
+	fmt.Printf("  %-12s %s%d%s\n", "Row Count:", ColorYellow, count, ColorReset)
+	
+	if count > 0 {
+		fmt.Printf("  %-12s %s\n", "Time Range:", formatMs(minTs))
+		fmt.Printf("  %-12s %s\n", "", formatMs(maxTs))
+		span := time.UnixMilli(maxTs).Sub(time.UnixMilli(minTs)).Round(time.Second)
+		fmt.Printf("  %-12s %s%v%s\n", "Data Span:", ColorGreen, span, ColorReset)
+	}
+}
+
+func getBasicStats(dbPath, table, tsCol string) (count int64, minTs int64, maxTs int64, err error) {
+	// mode=ro allows reading while the heavy writer is working
+	// nolock=1 is an alternative, but mode=ro + WAL is safer for stats
+	dsn := fmt.Sprintf("file:%s?mode=ro&_journal_mode=WAL", dbPath)
+	db, err := sql.Open("sqlite", dsn)
+	if err != nil {
+		return 0, 0, 0, err
+	}
+	defer db.Close()
+
+	query := fmt.Sprintf("SELECT COUNT(*), COALESCE(MIN(%s),0), COALESCE(MAX(%s),0) FROM %s", tsCol, tsCol, table)
+	err = db.QueryRow(query).Scan(&count, &minTs, &maxTs)
+	return count, minTs, maxTs, err
+}
+
+// getFullDBSize sums the .db, .db-wal, and .db-shm files
+func getFullDBSize(path string) int64 {
+	var total int64
+	extensions := []string{"", "-wal", "-shm"}
+	for _, ext := range extensions {
+		if info, err := os.Stat(path + ext); err == nil {
+			total += info.Size()
+		}
+	}
+	return total
+}
+
+func formatBytes(b int64) string {
+	const unit = 1024
+	if b < unit {
+		return fmt.Sprintf("%d B", b)
+	}
+	div, exp := int64(unit), 0
+	for n := b / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.2f %cB", float64(b)/float64(div), "KMGTPE"[exp])
+}
+
+func formatMs(ms int64) string {
+	if ms == 0 {
+		return "N/A"
+	}
+	return time.UnixMilli(ms).UTC().Format("2006-01-02 15:04:05 MST")
+}
+
+func disableColors() {
+}