MrKalzu/node-monitor
1
1
Fork 0
Code Issues Pull Requests Packages Wiki Activity

Remade the client in go, now named monitor-agent. The agents now support http basic auth

Browse Source
This commit is contained in:
Kalzu Rekku
2026-07-25 00:01:46 +03:00
parent c98081d360
commit 48bdab08b8
7 changed files with 578 additions and 1608 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client.py
+166 -65
View File
@@ -8,13 +8,17 @@ import json
import logging
from datetime import datetime, timezone
import platform
import socket # For getting local IP
import socket # For getting local IP
import sys
import base64
import urllib.parse
# --- Install necessary libraries if not already present ---
try:
import psutil # For system metrics
from pythonping import ping as python_ping # Renamed to avoid conflict with common 'ping'
import psutil # For system metrics
from pythonping import (
ping as python_ping,
) # Renamed to avoid conflict with common 'ping'
except ImportError:
print("Required libraries 'psutil' and 'pythonping' not found.")
print("Please install them: pip install psutil pythonping")
@@ -25,84 +29,95 @@ except ImportError:
NODE_UUID = os.environ.get("NODE_UUID", str(uuid.uuid4()))
# The UUID of the target monitoring service (the main.py server).
# IMPORTANT: This MUST match the SERVICE_UUID of your running FastAPI server.
# You can get this from the server's initial console output or by accessing its root endpoint ('/').
TARGET_SERVICE_UUID = os.environ.get(
"TARGET_SERVICE_UUID", "REPLACE_ME_WITH_YOUR_SERVER_SERVICE_UUID"
"TARGET_SERVICE_UUID", "ab73d00a-8169-46bb-997d-f13e5f760973"
)
# Optional basic auth for the server (UTF-8 compatible)
BASIC_AUTH_USERNAME = os.environ.get("BASIC_AUTH_USERNAME", "")
BASIC_AUTH_PASSWORD = os.environ.get("BASIC_AUTH_PASSWORD", "")
# The base URL of the FastAPI monitoring service
SERVER_BASE_URL = os.environ.get("SERVER_URL", "http://localhost:8000")
SERVER_BASE_URL = os.environ.get("SERVER_URL", "https://test.mystaginglab.net")
# How often to send status updates (in seconds)
UPDATE_INTERVAL_SECONDS = int(os.environ.get("UPDATE_INTERVAL_SECONDS", 5))
UPDATE_INTERVAL_SECONDS = int(os.environ.get("UPDATE_INTERVAL_SECONDS", 10))
# SSL verification (set to False if using self-signed certificates)
SSL_VERIFY = os.environ.get("SSL_VERIFY", "true").lower() == "true"
# File to store known peers' UUIDs and IPs for persistence
PEERS_FILE = os.environ.get("PEERS_FILE", f"known_peers_{NODE_UUID}.json")
# --- Logging Configuration ---
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger("NodeClient")
# --- Global state ---
uptime_seconds = 0 # Will be updated by psutil.boot_time() or incremented
uptime_seconds = 0 # Will be updated by psutil.boot_time() or incremented
# known_peers will store { "node_uuid_str": "ip_address_str" }
known_peers: dict[str, str] = {}
known_peers: dict[str, str] = {}
# Determine local IP for self-pinging and reporting to server
LOCAL_IP = "127.0.0.1" # Default fallback
LOCAL_IP = "127.0.0.1" # Default fallback
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(("8.8.8.8", 80)) # Connect to an external host (doesn't send data)
s.connect(("8.8.8.8", 80)) # Connect to an external host (doesn't send data)
LOCAL_IP = s.getsockname()[0]
s.close()
except Exception:
logger.warning("Could not determine local IP, defaulting to 127.0.0.1 for pings.")
# --- File Operations for Peers ---
def load_peers():
"""Loads known peers (UUID: IP) from a local JSON file."""
global known_peers
if os.path.exists(PEERS_FILE):
try:
with open(PEERS_FILE, 'r') as f:
with open(PEERS_FILE, "r") as f:
loaded_data = json.load(f)
# Ensure loaded peers are in the correct {uuid: ip} format
# Handle cases where the file might contain server's full peer info
temp_peers = {}
for k, v in loaded_data.items():
if isinstance(v, str): # Already in {uuid: ip} format
if isinstance(v, str): # Already in {uuid: ip} format
temp_peers[k] = v
elif isinstance(v, dict) and 'ip' in v: # Server's full peer info
temp_peers[k] = v['ip']
elif isinstance(v, dict) and "ip" in v: # Server's full peer info
temp_peers[k] = v["ip"]
known_peers = temp_peers
logger.info(f"Loaded {len(known_peers)} known peers from {PEERS_FILE}")
except json.JSONDecodeError as e:
logger.error(f"Error decoding JSON from {PEERS_FILE}: {e}. Starting with no known peers.")
known_peers = {} # Reset if file is corrupt
logger.error(
f"Error decoding JSON from {PEERS_FILE}: {e}. Starting with no known peers."
)
known_peers = {} # Reset if file is corrupt
except Exception as e:
logger.error(f"Error loading peers from {PEERS_FILE}: {e}. Starting with no known peers.")
logger.error(
f"Error loading peers from {PEERS_FILE}: {e}. Starting with no known peers."
)
known_peers = {}
else:
logger.info(f"No existing peers file found at {PEERS_FILE}.")
def save_peers():
"""Saves current known peers (UUID: IP) to a local JSON file."""
try:
with open(PEERS_FILE, 'w') as f:
with open(PEERS_FILE, "w") as f:
json.dump(known_peers, f, indent=2)
logger.debug(f"Saved {len(known_peers)} known peers to {PEERS_FILE}")
except Exception as e:
logger.error(f"Error saving peers to {PEERS_FILE}: {e}")
# --- System Metrics Collection ---
def get_system_metrics():
"""Collects actual system load and memory usage using psutil."""
global uptime_seconds
# Uptime
# psutil.boot_time() returns a timestamp in seconds since epoch
uptime_seconds = int(time.time() - psutil.boot_time())
@@ -112,14 +127,16 @@ def get_system_metrics():
# For cross-platform consistency, we'll use psutil.cpu_percent()
# and simulate 5/15 min averages if os.getloadavg is not available.
load_avg = [0.0, 0.0, 0.0]
if hasattr(os, 'getloadavg'):
if hasattr(os, "getloadavg"):
load_avg = list(os.getloadavg())
else: # Fallback for Windows or systems without getloadavg
else: # Fallback for Windows or systems without getloadavg
# psutil.cpu_percent() gives current CPU utilization over an interval.
# It's not true load average, but a reasonable proxy for monitoring.
# We'll use a short interval to get a "current" load.
cpu_percent = psutil.cpu_percent(interval=0.5) / 100.0 # CPU usage as a fraction
load_avg = [cpu_percent, cpu_percent * 0.9, cpu_percent * 0.8] # Simulate decay
cpu_percent = (
psutil.cpu_percent(interval=0.5) / 100.0
) # CPU usage as a fraction
load_avg = [cpu_percent, cpu_percent * 0.9, cpu_percent * 0.8] # Simulate decay
logger.debug(f"Using psutil.cpu_percent() for load_avg (non-Unix): {load_avg}")
# Memory Usage
@@ -129,9 +146,10 @@ def get_system_metrics():
return {
"uptime_seconds": uptime_seconds,
"load_avg": [round(l, 2) for l in load_avg],
"memory_usage_percent": round(memory_usage_percent, 2)
"memory_usage_percent": round(memory_usage_percent, 2),
}
# --- Ping Logic ---
def perform_pings(targets: dict[str, str]) -> dict[str, float]:
"""Performs actual pings to target IPs and returns latencies in ms."""
@@ -145,7 +163,7 @@ def perform_pings(targets: dict[str, str]) -> dict[str, float]:
# pythonping returns response_time in seconds, convert to milliseconds
pings_results[str(NODE_UUID)] = round(response_list.rtt_avg_ms, 2)
else:
pings_results[str(NODE_UUID)] = -1.0 # Indicate failure
pings_results[str(NODE_UUID)] = -1.0 # Indicate failure
logger.debug(f"Ping to self ({LOCAL_IP}): {pings_results[str(NODE_UUID)]}ms")
except Exception as e:
logger.warning(f"Failed to ping self ({LOCAL_IP}): {e}")
@@ -154,7 +172,7 @@ def perform_pings(targets: dict[str, str]) -> dict[str, float]:
# Ping other known peers
for peer_uuid, peer_ip in targets.items():
if peer_uuid == str(NODE_UUID):
continue # Already pinged self
continue # Already pinged self
try:
# Use a longer timeout for external pings
@@ -162,14 +180,37 @@ def perform_pings(targets: dict[str, str]) -> dict[str, float]:
if response_list.success:
pings_results[peer_uuid] = round(response_list.rtt_avg_ms, 2)
else:
pings_results[peer_uuid] = -1.0 # Indicate failure
logger.debug(f"Ping to {peer_uuid} ({peer_ip}): {pings_results[peer_uuid]}ms")
pings_results[peer_uuid] = -1.0 # Indicate failure
logger.debug(
f"Ping to {peer_uuid} ({peer_ip}): {pings_results[peer_uuid]}ms"
)
except Exception as e:
logger.warning(f"Failed to ping {peer_uuid} ({peer_ip}): {e}")
pings_results[peer_uuid] = -1.0
return pings_results
# --- Authentication Helper ---
def create_auth_headers():
"""Create authentication headers with UTF-8 support for Basic Auth."""
headers = {"Content-Type": "application/json"}
if BASIC_AUTH_USERNAME and BASIC_AUTH_PASSWORD:
try:
# Create Basic Auth header with UTF-8 encoding
credentials_str = f"{BASIC_AUTH_USERNAME}:{BASIC_AUTH_PASSWORD}"
credentials_b64 = base64.b64encode(credentials_str.encode("utf-8")).decode(
"ascii"
)
headers["Authorization"] = f"Basic {credentials_b64}"
logger.debug("Using HTTP Basic Authentication (UTF-8)")
except Exception as e:
logger.error(f"Failed to create authentication headers: {e}")
return headers
# --- Main Client Logic ---
def run_client():
global known_peers
@@ -179,15 +220,25 @@ def run_client():
logger.info(f"Target Service UUID: {TARGET_SERVICE_UUID}")
logger.info(f"Server URL: {SERVER_BASE_URL}")
logger.info(f"Update Interval: {UPDATE_INTERVAL_SECONDS} seconds")
logger.info(f"SSL Verification: {SSL_VERIFY}")
logger.info(f"Peers file: {PEERS_FILE}")
if BASIC_AUTH_USERNAME and BASIC_AUTH_PASSWORD:
logger.info(f"Basic Auth enabled for user: {BASIC_AUTH_USERNAME}")
else:
logger.info("No Basic Auth configured")
if TARGET_SERVICE_UUID == "REPLACE_ME_WITH_YOUR_SERVER_SERVICE_UUID":
logger.error("-" * 50)
logger.error("ERROR: TARGET_SERVICE_UUID is not set correctly!")
logger.error("Please replace 'REPLACE_ME_WITH_YOUR_SERVER_SERVICE_UUID' in the script")
logger.error(
"Please replace 'REPLACE_ME_WITH_YOUR_SERVER_SERVICE_UUID' in the script"
)
logger.error("or set the environment variable TARGET_SERVICE_UUID.")
logger.error("You can find the server's UUID by running main.py and checking its console output")
logger.error("or by visiting 'http://localhost:8000/' in your browser.")
logger.error(
"You can find the server's UUID by running main.py and checking its console output"
)
logger.error("or by visiting the server's root endpoint in your browser.")
logger.error("-" * 50)
return
@@ -198,74 +249,124 @@ def run_client():
try:
# 1. Get real system metrics
status_data = get_system_metrics()
# 2. Perform pings to known peers (and self)
ping_data = perform_pings(known_peers)
# 3. Construct the payload
payload = {
"node": str(NODE_UUID),
"timestamp": datetime.now(timezone.utc).isoformat(),
"timestamp": datetime.now(timezone.utc).isoformat(),
"status": status_data,
"pings": ping_data
"pings": ping_data,
}
# 4. Define the endpoint URL
endpoint_url = f"{SERVER_BASE_URL}/{TARGET_SERVICE_UUID}/{NODE_UUID}/"
# 5. Send the PUT request
# 5. Create headers with authentication
headers = create_auth_headers()
# 6. Send the PUT request
logger.info(
f"Sending update. Uptime: {status_data['uptime_seconds']}s, "
f"Load: {status_data['load_avg']}, Mem: {status_data['memory_usage_percent']}%, "
f"Pings: {len(ping_data)}"
)
response = requests.put(endpoint_url, json=payload, timeout=15) # Increased timeout
# 6. Process the response
response = requests.put(
endpoint_url,
json=payload,
headers=headers,
timeout=15,
verify=SSL_VERIFY,
)
# 7. Process the response
if response.status_code == 200:
response_data = response.json()
logger.info(f"Successfully sent update. Server message: '{response_data.get('message')}'")
if "peers" in response_data and isinstance(response_data["peers"], dict):
logger.info(
f"Successfully sent update. Server message: '{response_data.get('message')}'"
)
if "peers" in response_data and isinstance(
response_data["peers"], dict
):
# Update known_peers from server response
updated_peers = {}
# The server returns {uuid: {"last_seen": "...", "ip": "..."}}
# We only need the UUID and IP for pinging.
for peer_uuid, peer_info in response_data["peers"].items():
if 'ip' in peer_info:
updated_peers[peer_uuid] = peer_info['ip']
if "ip" in peer_info:
updated_peers[peer_uuid] = peer_info["ip"]
# Log newly discovered peers
newly_discovered = set(updated_peers.keys()) - set(known_peers.keys())
newly_discovered = set(updated_peers.keys()) - set(
known_peers.keys()
)
if newly_discovered:
logger.info(f"Discovered new peer(s): {', '.join(newly_discovered)}")
logger.info(
f"Discovered new peer(s): {', '.join(newly_discovered)}"
)
known_peers = updated_peers
save_peers() # Save updated peers to file for persistence
save_peers() # Save updated peers to file for persistence
logger.info(f"Total known peers for pinging: {len(known_peers)}")
else:
logger.warning("Server response did not contain a valid 'peers' field or it was empty.")
logger.warning(
"Server response did not contain a valid 'peers' field or it was empty."
)
else:
logger.error(f"Failed to send update. Status code: {response.status_code}, Response: {response.text}")
if response.status_code == 404:
logger.error("Hint: The TARGET_SERVICE_UUID might be incorrect, or the server isn't running at this endpoint.")
elif response.status_code == 422: # Pydantic validation error
logger.error(f"Server validation error (422 Unprocessable Entity): {response.json()}")
logger.error(
f"Failed to send update. Status code: {response.status_code}, Response: {response.text}"
)
if response.status_code == 401:
logger.error("Authentication failed (401 Unauthorized).")
logger.error(
"Please check your BASIC_AUTH_USERNAME and BASIC_AUTH_PASSWORD."
)
elif response.status_code == 404:
logger.error(
"Endpoint not found (404). The TARGET_SERVICE_UUID might be incorrect, or the server isn't running at this endpoint."
)
elif response.status_code == 422: # Pydantic validation error
try:
error_detail = response.json()
logger.error(
f"Server validation error (422 Unprocessable Entity): {error_detail}"
)
except:
logger.error(
f"Server validation error (422 Unprocessable Entity): {response.text}"
)
except requests.exceptions.SSLError as e:
logger.error(f"SSL Error: {e}")
logger.error(
"If using self-signed certificates, set SSL_VERIFY=false environment variable"
)
except requests.exceptions.Timeout:
logger.error(f"Request timed out after {15} seconds. Is the server running and responsive?")
logger.error(
f"Request timed out after 15 seconds. Is the server running and responsive?"
)
except requests.exceptions.ConnectionError as e:
logger.error(f"Connection error: {e}. Is the server running at {SERVER_BASE_URL}?")
logger.error(
f"Connection error: {e}. Is the server running at {SERVER_BASE_URL}?"
)
except requests.exceptions.RequestException as e:
logger.error(f"An unexpected request error occurred: {e}", exc_info=True)
except json.JSONDecodeError:
logger.error(f"Failed to decode JSON response: {response.text}. Is the server returning valid JSON?")
logger.error(
f"Failed to decode JSON response: {response.text}. Is the server returning valid JSON?"
)
except Exception as e:
logger.error(f"An unexpected error occurred in the client loop: {e}", exc_info=True)
logger.error(
f"An unexpected error occurred in the client loop: {e}", exc_info=True
)
# 7. Wait for the next update
# 8. Wait for the next update
time.sleep(UPDATE_INTERVAL_SECONDS)
if __name__ == "__main__":
run_client()
run_client()