From 9834272df3118b091ee286a74406c79e43112ec5 Mon Sep 17 00:00:00 2001 From: Kalzu Rekku Date: Sat, 17 Jan 2026 21:42:28 +0200 Subject: [PATCH] Added mother.sh to ease the usage and installation. Created main readme file. --- README.md | 129 ++++++++++++++++++++++++++++++++++++++++++++ mother.sh | 159 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 288 insertions(+) create mode 100644 README.md create mode 100755 mother.sh diff --git a/README.md b/README.md new file mode 100644 index 0000000..768f4e9 --- /dev/null +++ b/README.md @@ -0,0 +1,129 @@ +This is a comprehensive `README.md` designed for the root of your repository. It ties together the Go and Python services, explains the data pipeline, and provides the first documentation for your `mother.sh` orchestration script. + +*** + +# Bybit BTCUSDT Signal Engine + +A high-performance, multi-stage pipeline for streaming, processing, and generating trading signals from Bybit's BTCUSDT public trade WebSocket. + +This project is architected as a series of decoupled microservices that move data from raw WebSocket packets to actionable trading signals via SQLite persistence and Unix Domain Sockets. + +## 🏗 Architecture & Data Flow + +The engine operates as a linear pipeline, ensuring data integrity and separation of concerns: + +1. **`input` (Go)**: Connects to Bybit V5 WS. Streams raw JSON payloads into hourly-rotated `.jsonl` files. +2. **`onramp` (Go)**: Tails the JSONL files. Aggregates individual trades into OHLCV candles across multiple timeframes (1m, 5m, 15m, 1h) and saves them to `candles.db`. +3. **`analysis` (Python)**: Reads `candles.db`. Computes technical indicators (EMA, RSI, MACD, Bollinger Bands) and saves them to `analysis.db`. +4. **`signals` (Python)**: Monitors both databases. Applies "Personalities" (Scalping or Swing logic) to generate signals, broadcasted via Unix Domain Sockets. +5. **`monitor` (Python)**: Provides a real-time dashboard of the system health and signal flow. + +```mermaid +graph LR + Bybit((Bybit WS)) --> input[input service] + input --> JSONL[(JSONL Files)] + JSONL --> onramp[onramp service] + onramp --> CDB[(candles.db)] + CDB --> analysis[analysis service] + analysis --> ADB[(analysis.db)] + ADB --> signals[signals service] + CDB --> signals + signals --> Socket((/tmp/signals.sock)) + Socket --> Bot[Trading Bot / Monitor] +``` + +--- + +## 🛠 The "Mother" Script (`mother.sh`) + +The entire stack is managed by `mother.sh`, a centralized orchestration script located in the root. It handles dependency management, system health checks, and process multiplexing using **tmux**. + +### Key Features +* **Dependency Management**: One-command setup for both Go modules and Python `pipenv` environments. +* **Tick Verification**: Before starting, the script polls `pool.ntp.org` to ensure system clock drift is `< 0.5s`, preventing timestamp rejection from Bybit. +* **Sequential Bootstrapping**: Starts services in the correct order (`input` -> `onramp` -> `analysis` -> `signals`) with specific delays to allow database initialization. +* **Process Isolation**: Each service runs in its own named window within a `tmux` session. + +### Usage +| Command | Description | +| :--- | :--- | +| `./mother.sh setup` | Installs all Go and Python dependencies in their respective subdirectories. | +| `./mother.sh start` | Verifies NTP time and launches the full pipeline in a tmux session. | +| `./mother.sh status` | Checks if the services are currently running. | +| `./mother.sh stop` | Gracefully kills the tmux session and all underlying processes. | +| `./mother.sh restart` | Full stop and sequential restart. | + +--- + +## 🚀 Getting Started + +### Prerequisites +* **Go** 1.20+ +* **Python** 3.10+ & **pipenv** +* **tmux** (for process management) +* **TA-Lib** (System library required for Python signal analysis) + ```bash + # Ubuntu/Debian + sudo apt-get install ntpdate bc + # See signals/README.md for TA-Lib compilation instructions + ``` + +### Installation & Launch +1. **Clone the repo**: + ```bash + git clone + cd bybitbtc + ``` +2. **Run Setup**: + ```bash + ./mother.sh setup + ``` +3. **Start the Engine**: + ```bash + ./mother.sh start + ``` +4. **Attach to Logs**: + ```bash + tmux attach -t bybitbtc + ``` + +--- + +## 📊 Service Details + +### 1. Ingestion (`input/`) +* **Language**: Go +* **Output**: `./input/output/publicTrade.BTCUSDT_TIMESTAMP.jsonl` +* **Rotation**: Hourly. +* **Compression**: Automatically gzips files older than 2 hours to save disk space. + +### 2. Aggregation (`onramp/`) +* **Language**: Go +* **Database**: `candles.db` (SQLite with WAL mode). +* **Features**: Real-time OHLCV calculation, Buy/Sell volume tracking, and a Janitor loop that prunes data older than 30 days. + +### 3. Analysis (`analysis/`) +* **Language**: Python (pandas, talib) +* **Database**: `analysis.db`. +* **Indicators**: EMA (9, 21), SMA (50, 200), RSI (14), MACD (12, 26, 9), Bollinger Bands with Squeeze detection. + +### 4. Signals (`signals/`) +* **Language**: Python +* **Personalities**: + * **Scalping**: High-frequency (1m/5m), EMA-cross and Stochastic focused. + * **Swing**: Medium-frequency (15m/1h), Trend regime and BB-Squeeze focused. +* **Delivery**: Streams JSON signals over `/tmp/signals.sock`. + +--- + +## 📡 Monitoring & Health + +Each service exposes a status interface: +* **Input**: `go run input.go -status` +* **Onramp**: `echo "status" | nc 127.0.0.1 9999` +* **Signals**: `python signals/signal_health_client.py` (via `/tmp/signals_health.sock`) + +--- + +## ⚖️ License + MIT \ No newline at end of file diff --git a/mother.sh b/mother.sh new file mode 100755 index 0000000..23952b2 --- /dev/null +++ b/mother.sh @@ -0,0 +1,159 @@ +#!/bin/bash + +# Configuration +SESSION_NAME="bybitbtc" +NTP_SERVER="pool.ntp.org" +MAX_TIME_OFFSET=0.5 # Seconds +PYTHON_VERSION="3.10" # Adjust as needed + +# Colors for output +G='\033[0;32m' +R='\033[0;31m' +Y='\033[1;33m' +NC='\033[0m' + +log() { echo -e "${G}[MOTHER]${NC} $1"; } +warn() { echo -e "${Y}[WARN]${NC} $1"; } +err() { echo -e "${R}[ERROR]${NC} $1"; } + +# 1. Dependency Check & Installation +setup() { + log "Starting dependency installation..." + + # Check for tmux + if ! command -v tmux &> /dev/null; then + err "tmux not found. Please install it." + exit 1 + fi + + # Go Services + for dir in "input" "onramp"; do + if [ -d "$dir" ]; then + log "Setting up Go service: $dir" + (cd "$dir" && go mod tidy) + fi + done + + # Python Services + for dir in "analysis" "signals" "monitor"; do + if [ -d "$dir" ]; then + log "Setting up Python service: $dir" + (cd "$dir" && pipenv install) + fi + done + + log "Setup complete." +} + +# 2. Tick Verification (NTP Sync Check) +verify_time() { + log "Verifying system clock sync with $NTP_SERVER..." + if ! command -v ntpdate &> /dev/null; then + warn "ntpdate not installed. Skipping precise tick verification." + return + fi + + # Query NTP offset without setting the clock (-q) + OFFSET=$(ntpdate -q $NTP_SERVER | tail -1 | awk '{print $6}' | tr -d '-') + + if [ -z "$OFFSET" ]; then + err "Could not reach NTP server." + return + fi + + log "Current time offset: $OFFSET seconds" + + # Compare offset using bc (bash doesn't do float comparison well) + IS_SYNCED=$(echo "$OFFSET < $MAX_TIME_OFFSET" | bc) + if [ "$IS_SYNCED" -eq 1 ]; then + log "Time sync is within healthy parameters." + else + err "Clock desync detected ($OFFSET s)! Trades might be rejected or delayed." + read -p "Continue anyway? (y/n) " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + exit 1 + fi + fi +} + +# 3. Start Services in Tmux +start() { + verify_time + + # Kill existing session if it exists + tmux kill-session -t $SESSION_NAME 2>/dev/null + + log "Launching services in tmux session: $SESSION_NAME" + + # 1. Start INPUT + log "Stage 1: Starting Input (Websocket)..." + tmux new-session -d -s $SESSION_NAME -n "input" + tmux send-keys -t $SESSION_NAME:input "cd input && go run input.go" C-m + sleep 3 # Wait for WS connection and first file creation + + # 2. Start ONRAMP + log "Stage 2: Starting Onramp (DB Writer)..." + tmux new-window -t $SESSION_NAME -n "onramp" + tmux send-keys -t $SESSION_NAME:onramp "cd onramp && go run onramp.go" C-m + sleep 5 # Wait for candles.db to be initialized and first rows inserted + + # 3. Start ANALYST + log "Stage 3: Starting Analyst (Indicators)..." + tmux new-window -t $SESSION_NAME -n "analysis" + tmux send-keys -t $SESSION_NAME:analysis "cd analysis && pipenv run python analyst.py" C-m + sleep 8 # Analyst needs time to calculate indicators for the first time + + # 4. Start SIGNALS + log "Stage 4: Starting Signals (Logic)..." + tmux new-window -t $SESSION_NAME -n "signals" + tmux send-keys -t $SESSION_NAME:signals "cd signals && pipenv run python signals.py" C-m + sleep 3 # Wait for the Unix Socket (/tmp/signals.sock) to be created + + # 5. Start MONITOR + log "Stage 5: Starting Monitor..." + tmux new-window -t $SESSION_NAME -n "monitor" + tmux send-keys -t $SESSION_NAME:monitor "cd monitor && pipenv run python monitor.py" C-m + + log "${G}All services started in sequence.${NC}" + log "Use 'tmux attach -t $SESSION_NAME' to view logs." + log "Use './mother.sh stop' to kill all services." +} + +stop() { + log "Stopping tmux session $SESSION_NAME..." + tmux kill-session -t $SESSION_NAME 2>/dev/null + log "Stopped." +} + +status() { + if tmux has-session -t $SESSION_NAME 2>/dev/null; then + log "Status: ${G}RUNNING${NC}" + tmux list-windows -t $SESSION_NAME + else + log "Status: ${R}STOPPED${NC}" + fi +} + +# Main Command Switch +case "$1" in + setup) + setup + ;; + start) + start + ;; + stop) + stop + ;; + status) + status + ;; + restart) + stop + start + ;; + *) + echo "Usage: $0 {setup|start|stop|status|restart}" + exit 1 +esac