From d90343b3797c57b32aace733599ec68f1cb1e5c7 Mon Sep 17 00:00:00 2001 From: ryyst Date: Wed, 5 Jun 2024 22:59:20 +0300 Subject: [PATCH] TEMP --- .nvmrc | 1 + README.md | 22 ++- app/collector.py | 103 ----------- app/db.py | 98 ++-------- app/main.py | 2 +- app/parser.py | 119 ++++++++++++ app/static/example.js | 184 +++++++++++++++++++ app/static/index.html | 3 - app/static/index.js | 419 ++++++++++++++++++++---------------------- app/static/index.ts | 240 ++++++++++++++++++++++++ tsconfig.json | 10 + 11 files changed, 785 insertions(+), 416 deletions(-) create mode 100644 .nvmrc delete mode 100755 app/collector.py create mode 100755 app/parser.py create mode 100644 app/static/example.js create mode 100644 app/static/index.ts create mode 100644 tsconfig.json diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 0000000..53d1c14 --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +v22 diff --git a/README.md b/README.md index 076e9d1..8fcaa2d 100644 --- a/README.md +++ b/README.md @@ -2,19 +2,39 @@ ## Development +### Backend + +Pre-requisites: `pipenv`, or whichever venv manager you prefer + ```sh # Install all dependencies. `pipenv` was used previously: pipenv --python 3.11 pipenv install -pipenv shell # Start dev server +pipenv shell fastapi dev app/main.py # Start posting data to the tracing endpoint: traceroute git.rauhala.info -q1 | http POST localhost:8000/trace/MYHOSTNAME ``` +### Frontend + +Pre-requisites: `nvm`, `tsc` / `typescript` + +```sh +# Install latest npm version with node version manager: +nvm install 22 + +# Install project packages: +npm install + +# Regular usage, activate project node version & start TS compiler in watch mode: +nvm use +tsc -w +``` + ## URLs of interest - http://localhost:8000/ diff --git a/app/collector.py b/app/collector.py deleted file mode 100755 index cb713c7..0000000 --- a/app/collector.py +++ /dev/null @@ -1,103 +0,0 @@ -import json -import uuid -import hashlib - -from datetime import datetime - -from .db import Database - - -def parse_traceroute_output(data: str, origin: str): - # TODO: data validation - - lines = data.strip().split("\n") - target = lines[0].split()[2] - - created = datetime.now().isoformat() - - trace = {"target": target, "created": created, "origin": origin, "hops": []} - - prev_latency = None - - for line in lines[1:]: - hop_info = line.split() - print("LINE:", hop_info) - try: - # Regular lines. - number, name, ip, latency, _ = hop_info - latency = float(latency) - hop = { - "created": created, - "number": number, - "name": name, - "ip": ip.strip("()"), - "latency": latency, - "link_latency": round(latency if prev_latency else latency, 3), - } - prev_latency = latency - except ValueError: - # Asterisks, no data found for hop. - number, name = hop_info - hop = { - "created": created, - "number": number, - "name": name, - "ip": None, - "latency": None, - "link_latency": "?", - } - - trace["hops"].append(hop) - - return trace - - -def store_traceroute(trace): - db = Database() - - # hops_json = json.dumps(trace['hops']) - - path_ids = {} - - previous_hop_ip = None - previous_hop_latency = None - for hop in trace["hops"]: - hop_number = hop["number"] - hop_name = hop.get("name") - hop_ip = hop.get("ip") - hop_latency = hop.get("latency") - link_id = None - - # insert links and get their id's - if previous_hop_ip: - link_id = db.create_link(previous_hop_ip, hop_ip) - path_ids[hop_number] = link_id - - previous_hop_ip = hop_ip - - # Save hop details - db.create_hop(hop_name, hop_ip, hop_latency) - - # calculate link latency if possible and store it - if link_id and previous_hop_latency: - link_latency = hop_latency - previous_hop_latency - db.create_latency(link_id, trace["created"], link_latency) - - # make entry to "Paths" table - if path_ids: - json_path_ids = json.dumps(path_ids) - db.create_path(node, trace["target"], json_path_ids) - - db.end() - - -def generate_node_id(): - mac = uuid.getnode() - mac_str = ":".join( - ["{:02x}".format((mac >> ele) & 0xFF) for ele in range(0, 8 * 6, 8)][::-1] - ) - - # Hash the MAC address using SHA-256 to generate a unique ID - unique_id = hashlib.sha256(mac_str.encode()).hexdigest() - - return unique_id diff --git a/app/db.py b/app/db.py index 228fe73..36b1cf1 100644 --- a/app/db.py +++ b/app/db.py @@ -25,20 +25,24 @@ class Database: id INTEGER PRIMARY KEY AUTOINCREMENT, created TEXT NOT NULL, origin TEXT NOT NULL, - target TEXT NOT NULL + target TEXT NOT NULL, + unparsed TEXT NOT NULL ); - CREATE TABLE IF NOT EXISTS Hops ( + CREATE TABLE IF NOT EXISTS Nodes ( id INTEGER PRIMARY KEY AUTOINCREMENT, - trace_id INTEGER, created TEXT NOT NULL, - number INTEGER NOT NULL, name TEXT, ip TEXT, - latency TEXT, - link_latency TEXT, + latency_ms REAL NOT NULL, + ); - FOREIGN KEY(trace_id) REFERENCES Traces(id) + CREATE TABLE IF NOT EXISTS Links ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + created TEXT NOT NULL, + source TEXT NOT NULL, + target TEXT NOT NULL, + latency_ms REAL NOT NULL, ); """ ) @@ -49,6 +53,7 @@ class Database: self.conn.commit() self.conn.close() + # TODO def list_traces(self): # TODO: time filter result = [] @@ -75,6 +80,7 @@ class Database: return result + # TODO def create_trace(self, trace): self.cursor.execute( "INSERT OR IGNORE INTO Traces (created, origin, target) VALUES (?, ?, ?)", @@ -87,7 +93,7 @@ class Database: "INSERT OR IGNORE INTO Hops (trace_id, created, number, name, ip, latency, link_latency) VALUES (?, ?, ?, ?, ?, ?, ?)", ( trace_id, - hop["created"], + hop["created"], # TODO: trace.created hop["number"], hop["name"], hop["ip"], @@ -109,79 +115,3 @@ def ensure_table_setup(): db = Database() db.create_tables() db.end() - - -#################################################################### -#################################################################### -#################################################################### -#################################################################### - - -def with_connection(func): - @wraps(func) - def wrapped(*args, **kwargs): - conn = sqlite3.connect(DB_FILE) - cursor = conn.cursor() - - result = func(cursor, *args, **kwargs) - - conn.commit() - conn.close() - - return result - - return wrapped - - -@with_connection -def init_db(cursor: Cursor): - cursor.executescript( - """ - CREATE TABLE IF NOT EXISTS Links ( - id INTEGER PRIMARY KEY, - source_ip TEXT NOT NULL, - destination_ip TEXT NOT NULL, - UNIQUE(source_ip, destination_ip) - ); - - CREATE TABLE IF NOT EXISTS Paths ( - id INTEGER PRIMARY KEY, - node TEXT NOT NULL, - target TEXT NOT NULL, - hops_json TEXT NOT NULL, - UNIQUE(node, target, hops_json) - ); - - CREATE TABLE IF NOT EXISTS Latency ( - id INTEGER PRIMARY KEY, - link_id INTEGER NOT NULL, - timestamp DATETIME DEFAULT CURRENT_TIMESTAMP, - latency_ms REAL NOT NULL, - FOREIGN KEY (link_id) REFERENCES Links(id) - ); - - CREATE TABLE IF NOT EXISTS HopDetails ( - id INTEGER PRIMARY KEY, - hop_name TEXT, - hop_ip TEXT, - hop_latency TEXT - ); - """ - ) - - -@with_connection -def insert_hop(cursor: Cursor, previous_hop_ip: str, hop_ip: str): - """Insert a new hop and return related Link id""" - - cursor.execute( - "INSERT OR IGNORE INTO Links (source_ip, destination_ip) VALUES (?, ?)", - (previous_hop_ip, hop_ip), - ) - - cursor.execute( - "SELECT id FROM Links WHERE source_ip = ? AND destination_ip = ?", - (previous_hop_ip, hop_ip), - ) - - return cursor.fetchone() diff --git a/app/main.py b/app/main.py index 598a097..015c838 100755 --- a/app/main.py +++ b/app/main.py @@ -1,7 +1,7 @@ from fastapi import Request, FastAPI from fastapi.staticfiles import StaticFiles -from .collector import parse_traceroute_output, store_traceroute +from .parser import parse_traceroute_output from .db import Database, ensure_table_setup from pprint import pprint as print diff --git a/app/parser.py b/app/parser.py new file mode 100755 index 0000000..0dd6b64 --- /dev/null +++ b/app/parser.py @@ -0,0 +1,119 @@ +import json +import uuid +import hashlib + +from datetime import datetime + +from .db import Database + + +def parse_node(hop_info: list[str], target, origin): + try: + # Regular lines. + number, name, ip, latency, _ = hop_info + latency = float(latency) + ip = (ip.strip("()"),) + return { + "render_id": ip, + "hop_number": number, + "name": name, + "ip": ip, + "latency_ms": latency, + } + except ValueError: + # Asterisks, no data found for hop. + number, name = hop_info + return { + "render_id": f"{origin}-{target}-{number}", + "hop_number": number, + "name": name, + "ip": None, + "latency_ms": None, + } + + +def parse_link(node, prev_node): + latency = ( + node["latency"] - prev_node["latency"] + if prev_node is not None + else node["latency"] + ) + return { + "source": prev_node.get("render_id"), + "target": node["render_id"], + "latency_ms": latency, + } + + +def parse_traceroute_output(data: str, origin: str): + # TODO: data validation + + lines = data.strip().split("\n") + target = lines[0].split()[2] + + created = datetime.now().isoformat() + + trace = {"target": target, "created": created, "origin": origin, "hops": []} + + prev_node = None + + for line in lines[1:]: + hop_info = line.split() + print("LINE:", hop_info) + + node = parse_node(hop_info, target, origin) + link = parse_link(node, prev_node) + + trace["nodes"].append(node) + trace["links"].append(link) + + prev_node = node + + return trace + + +# def store_traceroute(trace): +# # hops_json = json.dumps(trace['hops']) +# +# path_ids = {} +# +# previous_hop_ip = None +# previous_hop_latency = None +# for hop in trace["hops"]: +# hop_number = hop["number"] +# hop_name = hop.get("name") +# hop_ip = hop.get("ip") +# hop_latency = hop.get("latency") +# link_id = None +# +# # insert links and get their id's +# if previous_hop_ip: +# link_id = db.create_link(previous_hop_ip, hop_ip) +# path_ids[hop_number] = link_id +# +# previous_hop_ip = hop_ip +# +# # Save hop details +# db.create_hop(hop_name, hop_ip, hop_latency) +# +# # calculate link latency if possible and store it +# if link_id and previous_hop_latency: +# link_latency = hop_latency - previous_hop_latency +# db.create_latency(link_id, trace["created"], link_latency) +# +# # make entry to "Paths" table +# if path_ids: +# json_path_ids = json.dumps(path_ids) +# db.create_path(node, trace["target"], json_path_ids) + + +# def generate_node_id(): +# mac = uuid.getnode() +# mac_str = ":".join( +# ["{:02x}".format((mac >> ele) & 0xFF) for ele in range(0, 8 * 6, 8)][::-1] +# ) +# +# # Hash the MAC address using SHA-256 to generate a unique ID +# unique_id = hashlib.sha256(mac_str.encode()).hexdigest() +# +# return unique_id diff --git a/app/static/example.js b/app/static/example.js new file mode 100644 index 0000000..9838fdf --- /dev/null +++ b/app/static/example.js @@ -0,0 +1,184 @@ +/** + * This example showcases sigma's reducers, which aim to facilitate dynamically + * changing the appearance of nodes and edges, without actually changing the + * main graphology data. + */ +import Graph from "graphology"; +import Sigma from "sigma"; +import { Coordinates, EdgeDisplayData, NodeDisplayData } from "sigma/types"; + +import { onStoryDown } from "../utils"; +import data from "./data.json"; + +export default () => { + // Retrieve some useful DOM elements: + const container = document.getElementById("sigma-container") as HTMLElement; + const searchInput = document.getElementById("search-input") as HTMLInputElement; + const searchSuggestions = document.getElementById("suggestions") as HTMLDataListElement; + + // Instantiate sigma: + const graph = new Graph(); + graph.import(data); + const renderer = new Sigma(graph, container); + + // Type and declare internal state: + interface State { + hoveredNode?: string; + searchQuery: string; + + // State derived from query: + selectedNode?: string; + suggestions?: Set; + + // State derived from hovered node: + hoveredNeighbors?: Set; + } + const state: State = { searchQuery: "" }; + + // Feed the datalist autocomplete values: + searchSuggestions.innerHTML = graph + .nodes() + .map((node) => ``) + .join("\n"); + + // Actions: + function setSearchQuery(query: string) { + state.searchQuery = query; + + if (searchInput.value !== query) searchInput.value = query; + + if (query) { + const lcQuery = query.toLowerCase(); + const suggestions = graph + .nodes() + .map((n) => ({ id: n, label: graph.getNodeAttribute(n, "label") as string })) + .filter(({ label }) => label.toLowerCase().includes(lcQuery)); + + // If we have a single perfect match, them we remove the suggestions, and + // we consider the user has selected a node through the datalist + // autocomplete: + if (suggestions.length === 1 && suggestions[0].label === query) { + state.selectedNode = suggestions[0].id; + state.suggestions = undefined; + + // Move the camera to center it on the selected node: + const nodePosition = renderer.getNodeDisplayData(state.selectedNode) as Coordinates; + renderer.getCamera().animate(nodePosition, { + duration: 500, + }); + } + // Else, we display the suggestions list: + else { + state.selectedNode = undefined; + state.suggestions = new Set(suggestions.map(({ id }) => id)); + } + } + // If the query is empty, then we reset the selectedNode / suggestions state: + else { + state.selectedNode = undefined; + state.suggestions = undefined; + } + + // Refresh rendering + // You can directly call `renderer.refresh()`, but if you need performances + // you can provide some options to the refresh method. + // In this case, we don't touch the graph data so we can skip its reindexation + renderer.refresh({ + skipIndexation: true, + }); + } + function setHoveredNode(node?: string) { + if (node) { + state.hoveredNode = node; + state.hoveredNeighbors = new Set(graph.neighbors(node)); + } + + // Compute the partial that we need to re-render to optimize the refresh + const nodes = graph.filterNodes((n) => n !== state.hoveredNode && !state.hoveredNeighbors?.has(n)); + const nodesIndex = new Set(nodes); + const edges = graph.filterEdges((e) => graph.extremities(e).some((n) => nodesIndex.has(n))); + + if (!node) { + state.hoveredNode = undefined; + state.hoveredNeighbors = undefined; + } + + // Refresh rendering + renderer.refresh({ + partialGraph: { + nodes, + edges, + }, + // We don't touch the graph data so we can skip its reindexation + skipIndexation: true, + }); + } + + // Bind search input interactions: + searchInput.addEventListener("input", () => { + setSearchQuery(searchInput.value || ""); + }); + searchInput.addEventListener("blur", () => { + setSearchQuery(""); + }); + + // Bind graph interactions: + renderer.on("enterNode", ({ node }) => { + setHoveredNode(node); + }); + renderer.on("leaveNode", () => { + setHoveredNode(undefined); + }); + + // Render nodes accordingly to the internal state: + // 1. If a node is selected, it is highlighted + // 2. If there is query, all non-matching nodes are greyed + // 3. If there is a hovered node, all non-neighbor nodes are greyed + renderer.setSetting("nodeReducer", (node, data) => { + const res: Partial = { ...data }; + + if (state.hoveredNeighbors && !state.hoveredNeighbors.has(node) && state.hoveredNode !== node) { + res.label = ""; + res.color = "#f6f6f6"; + } + + if (state.selectedNode === node) { + res.highlighted = true; + } else if (state.suggestions) { + if (state.suggestions.has(node)) { + res.forceLabel = true; + } else { + res.label = ""; + res.color = "#f6f6f6"; + } + } + + return res; + }); + + // Render edges accordingly to the internal state: + // 1. If a node is hovered, the edge is hidden if it is not connected to the + // node + // 2. If there is a query, the edge is only visible if it connects two + // suggestions + renderer.setSetting("edgeReducer", (edge, data) => { + const res: Partial = { ...data }; + + if (state.hoveredNode && !graph.hasExtremity(edge, state.hoveredNode)) { + res.hidden = true; + } + + if ( + state.suggestions && + (!state.suggestions.has(graph.source(edge)) || !state.suggestions.has(graph.target(edge))) + ) { + res.hidden = true; + } + + return res; + }); + + onStoryDown(() => { + renderer.kill(); + }); +}; diff --git a/app/static/index.html b/app/static/index.html index 4c1c9da..273ddbf 100644 --- a/app/static/index.html +++ b/app/static/index.html @@ -4,9 +4,6 @@ Kalzu - - -