Merge pull request 'http-rewrite' (#2) from ryyst/traceroute_map:http-rewrite into ryyst_http_rewrite

Reviewed-on: #2

.gitignore

@@ -0,0 +1 @@
+*.db

Pipfile

@@ -0,0 +1,16 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+networkx = "*"
+pyvis = "*"
+matplotlib = "*"
+fastapi = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3.11"
+python_full_version = "3.11.8"

README.md

@@ -0,0 +1,27 @@
+# Traceroute Internet Mapper - TIM
+
+## Development
+
+```sh
+# Install all dependencies. `pipenv` was used previously:
+pipenv --python 3.11
+pipenv install
+pipenv shell
+
+# Start dev server
+fastapi dev app/main.py
+
+# Start posting data to the tracing endpoint:
+traceroute git.rauhala.info -q1 | http POST localhost:8000/trace/MYHOSTNAME
+```
+
+## URLs of interest
+
+- http://localhost:8000/
+  - API Index, nothing to see.
+
+- http://localhost:8000/docs/
+  - Autogenerated API docs, mildy interesting
+
+- http://localhost:8000/static/index.html
+  - Rendered HTML index, very interesting

app/collector.py

@@ -0,0 +1,103 @@
+import json
+import uuid
+import hashlib
+
+from datetime import datetime
+
+from .db import Database
+
+
+def parse_traceroute_output(data: str, origin: str):
+    # TODO: data validation
+
+    lines = data.strip().split("\n")
+    target = lines[0].split()[2]
+
+    created = datetime.now().isoformat()
+
+    trace = {"target": target, "created": created, "origin": origin, "hops": []}
+
+    prev_latency = None
+
+    for line in lines[1:]:
+        hop_info = line.split()
+        print("LINE:", hop_info)
+        try:
+            # Regular lines.
+            number, name, ip, latency, _ = hop_info
+            latency = float(latency)
+            hop = {
+                "created": created,
+                "number": number,
+                "name": name,
+                "ip": ip.strip("()"),
+                "latency": latency,
+                "link_latency": round(latency if prev_latency else latency, 3),
+            }
+            prev_latency = latency
+        except ValueError:
+            # Asterisks, no data found for hop.
+            number, name = hop_info
+            hop = {
+                "created": created,
+                "number": number,
+                "name": name,
+                "ip": None,
+                "latency": None,
+                "link_latency": "?",
+            }
+
+        trace["hops"].append(hop)
+
+    return trace
+
+
+def store_traceroute(trace):
+    db = Database()
+
+    # hops_json = json.dumps(trace['hops'])
+
+    path_ids = {}
+
+    previous_hop_ip = None
+    previous_hop_latency = None
+    for hop in trace["hops"]:
+        hop_number = hop["number"]
+        hop_name = hop.get("name")
+        hop_ip = hop.get("ip")
+        hop_latency = hop.get("latency")
+        link_id = None
+
+        # insert links and get their id's
+        if previous_hop_ip:
+            link_id = db.create_link(previous_hop_ip, hop_ip)
+            path_ids[hop_number] = link_id
+
+        previous_hop_ip = hop_ip
+
+        # Save hop details
+        db.create_hop(hop_name, hop_ip, hop_latency)
+
+        # calculate link latency if possible and store it
+        if link_id and previous_hop_latency:
+            link_latency = hop_latency - previous_hop_latency
+            db.create_latency(link_id, trace["created"], link_latency)
+
+    # make entry to "Paths" table
+    if path_ids:
+        json_path_ids = json.dumps(path_ids)
+        db.create_path(node, trace["target"], json_path_ids)
+
+    db.end()
+
+
+def generate_node_id():
+    mac = uuid.getnode()
+    mac_str = ":".join(
+        ["{:02x}".format((mac >> ele) & 0xFF) for ele in range(0, 8 * 6, 8)][::-1]
+    )
+
+    # Hash the MAC address using SHA-256 to generate a unique ID
+    unique_id = hashlib.sha256(mac_str.encode()).hexdigest()
+
+    return unique_id

app/db.py

@@ -0,0 +1,187 @@
+import sqlite3
+from functools import wraps
+
+# Type alias
+Cursor = sqlite3.Cursor
+
+# Configs
+DB_FILE = "./traceroute.db"
+
+
+class Database:
+    def __init__(self):
+        self.db_file = DB_FILE
+        self.conn = sqlite3.connect(self.db_file, check_same_thread=False)
+
+        # Return fetch() data as Row objects, instead of tuples.
+        self.conn.row_factory = sqlite3.Row
+
+        self.cursor = self.conn.cursor()
+
+    def create_tables(self):
+        self.cursor.executescript(
+            """
+            CREATE TABLE IF NOT EXISTS Traces (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                created TEXT NOT NULL,
+                origin TEXT NOT NULL,
+                target TEXT NOT NULL
+            );
+
+            CREATE TABLE IF NOT EXISTS Hops (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                trace_id INTEGER,
+                created TEXT NOT NULL,
+                number INTEGER NOT NULL,
+                name TEXT,
+                ip TEXT,
+                latency TEXT,
+                link_latency TEXT,
+
+                FOREIGN KEY(trace_id) REFERENCES Traces(id)
+            );
+        """
+        )
+
+    def end(self):
+        """Always call this after you're done with the connection / request."""
+
+        self.conn.commit()
+        self.conn.close()
+
+    def list_traces(self):
+        # TODO: time filter
+        result = []
+
+        self.cursor.execute("SELECT * FROM Traces")
+        traces = self.cursor.fetchall()
+
+        for t in traces:
+            trace = dict(t)
+
+            self.cursor.execute(
+                """
+                SELECT number, name, ip, latency, link_latency
+                FROM Hops
+                WHERE trace_id = ?
+                ORDER BY number ASC
+            """,
+                (trace["id"],),
+            )
+            hops = self.cursor.fetchall()
+            trace["hops"] = hops
+
+            result.append(trace)
+
+        return result
+
+    def create_trace(self, trace):
+        self.cursor.execute(
+            "INSERT OR IGNORE INTO Traces (created, origin, target) VALUES (?, ?, ?)",
+            (trace["created"], trace["origin"], trace["target"]),
+        )
+        trace_id = self.cursor.lastrowid
+
+        for hop in trace["hops"]:
+            self.cursor.execute(
+                "INSERT OR IGNORE INTO Hops (trace_id, created, number, name, ip, latency, link_latency) VALUES (?, ?, ?, ?, ?, ?, ?)",
+                (
+                    trace_id,
+                    hop["created"],
+                    hop["number"],
+                    hop["name"],
+                    hop["ip"],
+                    hop["latency"],
+                    hop["link_latency"],
+                ),
+            )
+
+        return self.cursor.fetchone()
+
+    def create_hop(self, name, ip, latency):
+        self.cursor.execute(
+            "INSERT INTO Hops (name, ip, latency) VALUES (?, ?, ?)",
+            (name, ip, latency),
+        )
+
+
+def ensure_table_setup():
+    db = Database()
+    db.create_tables()
+    db.end()
+
+
+####################################################################
+####################################################################
+####################################################################
+####################################################################
+
+
+def with_connection(func):
+    @wraps(func)
+    def wrapped(*args, **kwargs):
+        conn = sqlite3.connect(DB_FILE)
+        cursor = conn.cursor()
+
+        result = func(cursor, *args, **kwargs)
+
+        conn.commit()
+        conn.close()
+
+        return result
+
+    return wrapped
+
+
+@with_connection
+def init_db(cursor: Cursor):
+    cursor.executescript(
+        """
+        CREATE TABLE IF NOT EXISTS Links (
+            id INTEGER PRIMARY KEY,
+            source_ip TEXT NOT NULL,
+            destination_ip TEXT NOT NULL,
+            UNIQUE(source_ip, destination_ip)
+        );
+
+        CREATE TABLE IF NOT EXISTS Paths (
+            id INTEGER PRIMARY KEY,
+            node TEXT NOT NULL,
+            target TEXT NOT NULL,
+            hops_json TEXT NOT NULL,
+            UNIQUE(node, target, hops_json)
+        );
+
+        CREATE TABLE IF NOT EXISTS Latency (
+            id INTEGER PRIMARY KEY,
+            link_id INTEGER NOT NULL,
+            timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
+            latency_ms REAL NOT NULL,
+            FOREIGN KEY (link_id) REFERENCES Links(id)
+        );
+
+        CREATE TABLE IF NOT EXISTS HopDetails (
+            id INTEGER PRIMARY KEY,
+            hop_name TEXT,
+            hop_ip TEXT,
+            hop_latency TEXT
+        );
+    """
+    )
+
+
+@with_connection
+def insert_hop(cursor: Cursor, previous_hop_ip: str, hop_ip: str):
+    """Insert a new hop and return related Link id"""
+
+    cursor.execute(
+        "INSERT OR IGNORE INTO Links (source_ip, destination_ip) VALUES (?, ?)",
+        (previous_hop_ip, hop_ip),
+    )
+
+    cursor.execute(
+        "SELECT id FROM Links WHERE source_ip = ? AND destination_ip = ?",
+        (previous_hop_ip, hop_ip),
+    )
+
+    return cursor.fetchone()

app/main.py

@@ -0,0 +1,67 @@
+from fastapi import Request, FastAPI
+from fastapi.staticfiles import StaticFiles
+
+from .collector import parse_traceroute_output, store_traceroute
+from .db import Database, ensure_table_setup
+
+from pprint import pprint as print
+
+
+# Setup our SQLite before anything else.
+ensure_table_setup()
+
+# Setup web framework thingies
+app = FastAPI()
+app.mount("/static", StaticFiles(directory="app/static"), name="static")
+
+
+@app.get("/")
+def read_root():
+    return {
+        "greeting": "Hello, Kalzu!",
+        "instructions": [
+            "Try piping traceroute data directly to POST /trace/{hostname}.",
+            "{hostname} is for filtering data by sender.",
+            "For example the following command using HTTPie:",
+            "",
+            "  $ traceroute peitto.info | http POST localhost:8000/trace/rekku",
+            "",
+            "",
+            "Also take a look at /docs/ and /static/index.html",
+            "",
+            "",
+            "",
+            "END OF TRANSMISSION",
+        ]
+        + [None] * 800,
+    }
+
+
+@app.get("/trace/")
+def list_traces():
+    db = Database()
+
+    trace = db.list_traces()
+
+    db.end()
+    return trace
+
+
+@app.post("/trace/{origin}")
+async def create_trace(origin: str, request: Request):
+    raw_data = await request.body()
+    data = raw_data.decode("utf-8", "ignore")
+
+    print(f"Received data from {origin}:")
+    print(data)
+
+    trace = parse_traceroute_output(data, origin)
+
+    print("Parsed data:")
+    print(trace)
+
+    db = Database()
+    db.create_trace(trace)
+    db.end()
+
+    return {"status": "ok"}

app/static/index.html

@@ -0,0 +1,20 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <title>Kalzu</title>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/sigma.js/2.4.0/sigma.min.js" integrity="sha512-iiPEYww3QXZU5C795JnnINBRNgHqDnRHs9mA7aJoqx4pNE4u3CknCDGmePHFoHtKR/6C9aIcRFa+HJ6obtlteQ==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/graphology/0.25.4/graphology.umd.min.js" integrity="sha512-tjMBhL9fLMcqoccPOwpRiIQIOAyUh18lWUlUvE10zvG1UNMfxUC4qSERmUq+VF30iavIyqs/q6fSP2o475FAUw==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
+    <script src="/static/d3.v7.min.js"></script>
+    <style>
+      #container {
+        height: 95vh;
+      }
+    </style>
+  </head>
+  <body style="background: #eee">
+    <div id="container"></div>
+    <script src="/static/index.js"></script>
+  </body>
+</html>

app/static/index.js

@@ -0,0 +1,448 @@
+const linkArc = (d) => {
+  const r = Math.hypot(d.target.x - d.source.x, d.target.y - d.source.y) * 3;
+  return `
+    M${d.source.x},${d.source.y}
+    A${r},${r} 0 0,1 ${d.target.x},${d.target.y}
+  `;
+};
+
+const drag = (simulation) => {
+  function dragstarted(event, d) {
+    if (!event.active) simulation.alphaTarget(0.3).restart();
+    d.fx = d.x;
+    d.fy = d.y;
+  }
+
+  function dragged(event, d) {
+    d.fx = event.x;
+    d.fy = event.y;
+  }
+
+  function dragended(event, d) {
+    if (!event.active) simulation.alphaTarget(0);
+    d.fx = null;
+    d.fy = null;
+  }
+
+  return d3
+    .drag()
+    .on("start", dragstarted)
+    .on("drag", dragged)
+    .on("end", dragended);
+};
+
+// const drawChart = (data) => {
+//   // Specify the dimensions of the chart.
+//   const width = 1600;
+//   const height = 1200;
+//
+//   // Specify the color scale.
+//   const color = d3.scaleOrdinal(d3.schemeCategory10);
+//
+//   // The force simulation mutates links and nodes, so create a copy
+//   // so that re-evaluating this cell produces the same result.
+//   const links = data.links.map((d) => ({ ...d }));
+//   const nodes = data.nodes.map((d) => ({ ...d }));
+//
+//   // Create a simulation with several forces.
+//   const simulation = d3
+//     .forceSimulation(nodes)
+//     .force(
+//       "link",
+//       d3.forceLink(links).id((d) => d.id),
+//     )
+//     .force("charge", d3.forceManyBody())
+//     .force("x", d3.forceX())
+//     .force("y", d3.forceY());
+//
+//   // Create the SVG container.
+//   const svg = d3
+//     .create("svg")
+//     .attr("width", width)
+//     .attr("height", height)
+//     .attr("viewBox", [-width / 2, -height / 2, width, height])
+//     .attr("style", "max-width: 100%; height: auto;");
+//
+//   // Add a line for each link, and a circle for each node.
+//   const link = svg
+//     .append("g")
+//     .attr("stroke", "#999")
+//     .attr("stroke-opacity", 0.6)
+//     .selectAll("line")
+//     .data(links)
+//     .join("line")
+//     .attr("stroke-width", 1); // (d) => Math.sqrt(d.value));
+//
+//   const node = svg
+//     .append("g")
+//     .attr("stroke", "#fff")
+//     .attr("stroke-width", 1.5)
+//     .selectAll("circle")
+//     .data(nodes)
+//     .join("circle")
+//     .attr("r", 5)
+//     .attr("fill", (d) => color(d.group));
+//
+//   node.append("title").text((d) => d.id);
+//
+//   // Add a drag behavior.
+//   node.call(
+//     d3.drag().on("start", dragstarted).on("drag", dragged).on("end", dragended),
+//   );
+//
+//   // Set the position attributes of links and nodes each time the simulation ticks.
+//   simulation.on("tick", () => {
+//     link
+//       .attr("x1", (d) => d.source.x)
+//       .attr("y1", (d) => d.source.y)
+//       .attr("x2", (d) => d.target.x)
+//       .attr("y2", (d) => d.target.y);
+//
+//     node.attr("cx", (d) => d.x).attr("cy", (d) => d.y);
+//   });
+//
+//   // Reheat the simulation when drag starts, and fix the subject position.
+//   function dragstarted(event) {
+//     if (!event.active) simulation.alphaTarget(0.3).restart();
+//     event.subject.fx = event.subject.x;
+//     event.subject.fy = event.subject.y;
+//   }
+//
+//   // Update the subject (dragged node) position during drag.
+//   function dragged(event) {
+//     event.subject.fx = event.x;
+//     event.subject.fy = event.y;
+//   }
+//
+//   // Restore the target alpha so the simulation cools after dragging ends.
+//   // Unfix the subject position now that it’s no longer being dragged.
+//   function dragended(event) {
+//     if (!event.active) simulation.alphaTarget(0);
+//     event.subject.fx = null;
+//     event.subject.fy = null;
+//   }
+//
+//   // When this cell is re-run, stop the previous simulation. (This doesn’t
+//   // really matter since the target alpha is zero and the simulation will
+//   // stop naturally, but it’s a good practice.)
+//   // invalidation.then(() => simulation.stop());
+//
+//   return svg.node();
+// };
+
+// const drawChart2 = (data) => {
+//   // Data parsing
+//   const nodes = Array.from(
+//     new Set(data.flatMap((l) => [l.source, l.target])),
+//     (id) => ({ id }),
+//   );
+//   const links = data.map((d) => Object.create(d));
+//
+//   // Styles
+//   const width = window.visualViewport.width;
+//   const height = window.visualViewport.height - 10;
+//
+//   const colors = Array.from(new Set(data.map((d) => d.traceId)));
+//   const color = d3.scaleOrdinal(colors, d3.schemeCategory10);
+//
+//   const simulation = d3
+//     .forceSimulation(nodes)
+//     .force(
+//       "link",
+//       d3.forceLink(links).id((d) => d.id),
+//     )
+//     .force("charge", d3.forceManyBody().strength(-400))
+//     .force("x", d3.forceX())
+//     .force("y", d3.forceY());
+//
+//   // Canvas settings
+//   const svg = d3
+//     .create("svg")
+//     .attr("width", width)
+//     .attr("height", height)
+//     .attr("viewBox", [-width / 2, -height / 2, width, height])
+//     .attr("style", "max-width: 100%; height: auto; font: 14px monospace;");
+//
+//   // Pre-type arrowheads, as they don't inherit styles.
+//   svg
+//     .append("defs")
+//     .selectAll("marker")
+//     .data(colors)
+//     .join("marker")
+//     .attr("id", (d) => `arrow-${d}`)
+//     .attr("viewBox", "0 -5 10 10")
+//     .attr("refX", 15)
+//     .attr("refY", -0.5)
+//     .attr("markerWidth", 5)
+//     .attr("markerHeight", 5)
+//     .attr("orient", "auto")
+//     .append("path")
+//     .attr("fill", color)
+//     .attr("d", "M0,-5L10,0L0,5");
+//
+//   const link = svg
+//     .append("g")
+//     .attr("fill", "none")
+//     .attr("stroke-width", 1.5)
+//     .selectAll("path")
+//     .data(links)
+//     .join("path")
+//     .attr("stroke", (d) => color(d.traceId))
+//     .attr(
+//       "marker-end",
+//       (d) => `url(${new URL(`#arrow-${d.traceId}`, location)})`,
+//     );
+//
+//   const node = svg
+//     .append("g")
+//     .attr("fill", "currentColor")
+//     .attr("stroke-linecap", "round")
+//     .attr("stroke-linejoin", "round")
+//     .selectAll("g")
+//     .data(nodes)
+//     .join("g")
+//     .call(drag(simulation));
+//
+//   // Node "icon"
+//   node
+//     .append("circle")
+//     .attr("stroke", "white")
+//     .attr("stroke-width", 1.5)
+//     .attr("r", 5);
+//
+//   // Node text
+//   node
+//     .append("text")
+//     .attr("x", 8)
+//     .attr("y", 4)
+//     .text((d) => (d.id.endsWith("*") ? "*" : d.id))
+//     .clone(true)
+//     .lower()
+//     .attr("fill", "black")
+//     .attr("stroke", "white")
+//     .attr("stroke-width", 2);
+//
+//   simulation.on("tick", () => {
+//     link.attr("d", linkArc);
+//     // link
+//     //   .attr("x1", (d) => d.source.x)
+//     //   .attr("y1", (d) => d.source.y)
+//     //   .attr("x2", (d) => d.target.x)
+//     //   .attr("y2", (d) => d.target.y);
+//     node.attr("transform", (d) => `translate(${d.x},${d.y})`);
+//   });
+//
+//   // invalidation.then(() => simulation.stop());
+//
+//   return Object.assign(svg.node(), {
+//     scales: { color },
+//   });
+// };
+
+const state = {};
+
+const drawSigma = (data) => {
+  // Create a graphology graph
+  const graph = new graphology.MultiDirectedGraph();
+
+  const setHoveredNode = (node) => {
+    if (node) {
+      const props = graph.getNodeAttributes(node);
+      state.hoveredNode = node;
+      state.hoveredTrace = props.traceId;
+
+      // Compute the partial that we need to re-render to optimize the refresh
+      const nodes = graph.filterNodes((n) => {
+        const np = graph.getNodeAttributes(n);
+        return np.traceId === state.hoveredTrace;
+      });
+
+      console.log("Nämä?", nodes);
+
+      const nodesIndex = new Set(nodes);
+      const edges = graph.filterEdges((e) =>
+        graph.extremities(e).some((n) => nodesIndex.has(n)),
+      );
+
+      // Refresh rendering
+      renderer.refresh({
+        partialGraph: {
+          nodes,
+          edges,
+        },
+        // We don't touch the graph data so we can skip its reindexation
+        skipIndexation: true,
+      });
+    } else {
+      state.hoveredNode = undefined;
+      state.hoveredTrace = undefined;
+
+      // Refresh rendering
+      renderer.refresh({
+        // We don't touch the graph data so we can skip its reindexation
+        skipIndexation: true,
+      });
+    }
+  };
+
+  data.nodes.forEach((n) => {
+    try {
+      graph.addNode(n.id, n);
+    } catch (e) {
+      // Duplicate node found, which is correct in our data scenario.
+    }
+  });
+
+  data.links.forEach((l) => {
+    graph.addEdge(l.source, l.target, l);
+  });
+
+  // Instantiate sigma.js and render the graph
+  const renderer = new Sigma(graph, document.getElementById("container"), {
+    // labelThreshold: -10000,
+    renderEdgeLabels: true,
+  });
+
+  // Bind graph interactions:
+  renderer.on("enterNode", ({ node }) => {
+    console.log("enter");
+    setHoveredNode(node);
+  });
+  renderer.on("leaveNode", () => {
+    console.log("leave");
+    setHoveredNode(undefined);
+  });
+
+  // Render nodes accordingly to the internal state:
+  // 1. If a node is selected, it is highlighted
+  // 2. If there is query, all non-matching nodes are greyed
+  // 3. If there is a hovered node, all non-neighbor nodes are greyed
+  renderer.setSetting("nodeReducer", (node, data) => {
+    const res = { ...data };
+    const props = graph.getNodeAttributes(node);
+
+    if (state.hoveredTrace && props.traceId !== state.hoveredTrace) {
+      res.label = "";
+      res.color = "#f6f6f6";
+    }
+
+    return res;
+  });
+
+  // Render edges accordingly to the internal state:
+  // 1. If a node is hovered, the edge is hidden if it is not connected to the
+  //    node
+  // 2. If there is a query, the edge is only visible if it connects two
+  //    suggestions
+  renderer.setSetting("edgeReducer", (edge, data) => {
+    const res = { ...data };
+    const props = graph.getEdgeAttributes(edge);
+
+    if (state.hoveredTrace && props.traceId !== state.hoveredTrace) {
+      res.hidden = true;
+    }
+
+    return res;
+  });
+};
+
+const getNodeID = (hop, prevId, trace) => {
+  if (prevId === null) {
+    return trace.origin;
+  }
+  if (hop.name === "*") {
+    return `${trace.id}-${hop.number}-*`;
+  }
+
+  return hop.ip;
+};
+
+const parseNodesAndLinks = (traces) => {
+  const nodes = [];
+  const links = [];
+
+  const colors = Array.from(new Set(traces.map((t) => t.id)));
+  const color = d3.scaleOrdinal(colors, d3.schemeCategory10);
+
+  traces.forEach((trace) => {
+    let prevId = null;
+    let latestNumber = null;
+
+    trace.hops.forEach((hop) => {
+      const id = getNodeID(hop, prevId, trace);
+
+      // New node
+      nodes.push({
+        id: id,
+        label: id.endsWith("*") ? "*" : id,
+        x: trace.id,
+        y: hop.number / 2,
+        traceId: trace.id,
+        size: 9,
+        labelSize: 30,
+        color: color(trace.id),
+      });
+
+      if (prevId) {
+        // New link
+        links.push({
+          label: hop.link_latency,
+          source: prevId,
+          target: id,
+          traceId: trace.id,
+          origin: trace.origin,
+          size: 3,
+          color: color(trace.id),
+        });
+      }
+
+      prevId = id;
+      latestNumber = hop.number;
+    });
+
+    /**
+     * Last "destination" node, just for candy
+     */
+    nodes.push({
+      id: trace.id,
+      label: trace.target,
+      traceId: trace.id,
+      x: trace.id,
+      y: (latestNumber + 1) / 2,
+      size: 8,
+      color: "black",
+    });
+
+    links.push({
+      label: "-",
+      size: 8,
+      traceId: trace.id,
+      source: prevId,
+      target: trace.id,
+    });
+  });
+
+  // { id: ip, group: origin, radius: 2 }
+  // { source: prev.ip, target: ip, value: latency }
+  return {
+    nodes,
+    links,
+  };
+};
+
+async function main() {
+  const response = await fetch("/trace/");
+  const traces = await response.json();
+
+  console.log("Traces:", traces);
+  const data = parseNodesAndLinks(traces);
+
+  console.log("Data:", data);
+  // const chart = drawChart2(data.links);
+
+  const chart = drawSigma(data);
+
+  // container.append(chart);
+}
+
+main();

app/traceroute_visualization.py

@@ -9,8 +9,8 @@ def retrieve_traceroute_data():
     conn = sqlite3.connect('traceroute.db')
     cursor = conn.cursor()
     cursor.execute('''
-        SELECT l.source_ip, l.destination_ip, ll.latency, ll.timestamp
-        FROM link_latency ll
+        SELECT l.source_ip, l.destination_ip, ll.latency_ms, ll.timestamp
+        FROM Latency ll
         JOIN links l ON ll.link_id = l.id
         ORDER BY ll.timestamp
     ''')
@@ -25,33 +25,33 @@ def build_graph(data):
         destination_ip = row[1]
         latency = row[2]
         timestamp = row[3]
-        
+
         # Add nodes and edges to the graph
         G.add_node(source_ip)
         G.add_node(destination_ip)
         G.add_edge(source_ip, destination_ip, latency=latency, timestamp=timestamp)
-        
+
     return G
 
 #def visualize_graph_pyvis(G, output_file='network.html'):
 #    net = Network(height='750px', width='100%', directed=True)
 #    net.from_nx(G)
-#    
+#
 #    for edge in G.edges(data=True):
 #        src, dst, data = edge
 #        latency = data['latency']
 #        timestamp = data['timestamp']
 #        net.add_edge(src, dst, title=f'Latency: {latency} ms<br>Timestamp: {timestamp}', value=latency)
-#    
+#
 #    net.show(output_file)
 
 def main():
     # Retrieve data from the database
     traceroute_data = retrieve_traceroute_data()
-    
+
     # Build the network graph
     graph = build_graph(traceroute_data)
-    
+
     return graph
 
 if __name__ == '__main__':
@@ -59,9 +59,9 @@ if __name__ == '__main__':
 
     nx.draw_planar(graph, with_labels=True)
     plt.savefig("path.png")
-    
-    
+
+
     # Visualize the graph using pyvis
     #visualize_graph_pyvis(graph, 'traceroute_network.html')
 
-exit(0)
+#exit(0)

traceroute_collector.py

@@ -1,246 +0,0 @@
-#!/usr/bin/env python3.11
-
-
-import subprocess
-import sqlite3
-import re
-import json
-import ipaddress
-import uuid
-import hashlib
-from datetime import datetime
-
-
-def run_traceroute(host):
-    timestamp = datetime.now().timestamp()
-    result = subprocess.run(['traceroute', host], stdout=subprocess.PIPE)
-    return result.stdout.decode(), timestamp
-
-def parse_traceroute_output(output, timestamp):
-
-    lines = output.strip().split('\n')
-    trace = {}
-    hops = []
-
-    ip_regex = r"\((.*?)\)" # ipaddress are in ()
-
-    target = output.strip().split('\n')[0].split()[2]
-
-    for line in lines[1:]:
-        hop = {}
-        hop_info = line.split()
-        hop_number = int(hop_info[0])
-        hop_name = None
-        hop_ip = None
-        hop_latency = None
-
-        latencies = []
-
-        #print("##### "+str(hop_info))
-        count = 0
-        for part in hop_info[1:]:
-            count += 1
-
-            # source node drops or blocks icmp packages
-            # We will give funny to name to hop for not answering and move on.
-            if part == '*':
-                hop_name = 'unresponsive'
-                hop_ip = 'unresponsive'
-                break
-
-
-            # If first colum is either name or ip-address
-            if count == 1:
-                match = re.search(ip_regex, part)
-                if  match:
-                    hop_ip = part.strip('()')
-                else:
-                    hop_name = part
-
-            # Second colum is ip-address first latency reading
-            if count == 2:
-                if re.search(ip_regex, part):
-                    try:
-                        _ip = ipaddress.ip_address(part.strip('()'))
-                        hop_ip = part.strip('()')
-                    except ValueError:
-                        pass  # Ignore if it's not a valid IP address
-
-            # Rest of the input colums are either latency floats, 'ms' or 
-            # reruns of the hop_name and hop_ip...
-            # We only need the latency floats anymore.
-            else:
-                try:
-                    latency = float(part)
-                    latencies.append(latency)
-                except ValueError:
-                    pass
-
-        hop_latency = sum(latencies) / len(latencies) if latencies else None
-
-        hop['hop_number'] = hop_number
-        if not hop_name == None:
-            hop['hop_name'] = hop_name
-        hop['hop_ip'] = hop_ip
-        hop['hop_latency'] = hop_latency
-        hops.append(hop)
-        
-    trace['target'] = target
-    trace['timestamp'] = timestamp
-    trace['hops'] = hops
-
-    return trace
-
-
-def create_tables(databasefile):
-        # Connect to the SQLite database
-    conn = sqlite3.connect(databasefile)
-    cursor = conn.cursor()
-    
-    # SQL statements to create the tables
-    create_links_table = """
-    CREATE TABLE IF NOT EXISTS Links (
-        id INTEGER PRIMARY KEY,
-        source_ip TEXT NOT NULL,
-        destination_ip TEXT NOT NULL,
-        UNIQUE(source_ip, destination_ip)
-    );
-    """
-    
-    create_paths_table = """
-    CREATE TABLE IF NOT EXISTS Paths (
-        id INTEGER PRIMARY KEY,
-        node TEXT NOT NULL,
-        target TEXT NOT NULL,
-        hops_json TEXT NOT NULL,
-        UNIQUE(node, target, hops_json)
-    );
-    """
-    
-    create_latency_table = """
-    CREATE TABLE IF NOT EXISTS Latency (
-        id INTEGER PRIMARY KEY,
-        link_id INTEGER NOT NULL,
-        timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
-        latency_ms REAL NOT NULL,
-        FOREIGN KEY (link_id) REFERENCES Links(id)
-    );
-    """
-    
-    create_hopdetails_table = """
-    CREATE TABLE IF NOT EXISTS HopDetails (
-        id INTEGER PRIMARY KEY,
-        hop_name TEXT,
-        hop_ip TEXT,
-        hop_latency TEXT
-    );
-    """
-    
-    # Execute the SQL statements
-    cursor.execute(create_links_table)
-    cursor.execute(create_paths_table)
-    cursor.execute(create_latency_table)
-    cursor.execute(create_hopdetails_table)
-    
-    # Commit changes and close the connection
-    conn.commit()
-    conn.close()
-
-def store_traceroute(db_file, node, trace):
-    conn = sqlite3.connect(db_file)
-    cursor = conn.cursor()
-
-
-    hops_json = json.dumps(trace['hops'])
-    
-    path_ids = {}
-
-    previous_hop_ip = None
-    previous_hop_latency = None
-    for hop in trace['hops']:
-        hop_number = hop['hop_number']
-        hop_name = hop.get('hop_name')
-        hop_ip = hop.get('hop_ip')
-        hop_latency = hop.get('hop_latency')
-        link_id = None
-
-        # insert links and get their id's
-        if previous_hop_ip:
-            cursor.execute("""
-                INSERT OR IGNORE INTO Links (source_ip, destination_ip) VALUES (?, ?)
-            """, (previous_hop_ip, hop_ip))
-            cursor.execute("""
-                SELECT id FROM Links WHERE source_ip = ? AND destination_ip = ?
-            """, (previous_hop_ip, hop_ip))
-            result = cursor.fetchone()
-            link_id = result
-            path_ids[hop_number] = link_id
-        previous_hop_ip = hop_ip
-
-        # Save hop details
-        cursor.execute("INSERT INTO HopDetails (hop_name, hop_ip, hop_latency) VALUES (?, ?, ?)",
-                        (hop_name, hop_ip, hop_latency))
-        
-        # calculate link latency if possible and store it
-        if link_id and previous_hop_latency:
-            link_latency = hop_latency - previous_hop_latency
-            cursor.execute("INSERT INTO Latency (link_id, timestamp, latency_ms) VALUES (?, ?, ?)", 
-                           (link_id, trace['timestamp'], link_latency))
-
-    # make entry to "Paths" table
-    if path_ids:
-        json_path_ids = json.dumps(path_ids)
-        cursor.execute("INSERT OR IGNORE INTO Paths (node, target, hops_json) VALUES (?, ?, ?)",
-                       (node, trace['target'], json_path_ids))
-
-    conn.commit()
-    conn.close()
-
-
-def retrieve_traceroute(db_file):
-
-    retval = {}
-
-    conn = sqlite3.connect(db_file)
-    cursor = conn.cursor()
-    cursor.execute('''
-        SELECT target, hops_json
-        FROM Paths
-    ''')
-    retval['path'] = cursor.fetchall()
-
-    cursor.execute('''
-        SELECT source_ip, destination_ip
-        FROM Links
-    ''')
-    retval['links'] = cursor.fetchall()
-    conn.close()
-    return retval
-
-def generate_node_id():
-    mac = uuid.getnode()
-    mac_str = ':'.join(['{:02x}'.format((mac >> ele) & 0xff) for ele in range(0,8*6,8)][::-1])
-    
-    # Hash the MAC address using SHA-256 to generate a unique ID
-    unique_id = hashlib.sha256(mac_str.encode()).hexdigest()
-
-    return unique_id
-
-
-if __name__ == '__main__':
-
-    db_file="./traceroute.db"
-    create_tables(db_file)
-    my_id = generate_node_id()
-
-    target='1.1.1.1'
-    traceroute_output, timestamp = run_traceroute(target)
-    trace = parse_traceroute_output(traceroute_output, timestamp)
-
-    store_traceroute(db_file, my_id, trace)
-
-    print("#####")
-    print(retrieve_traceroute(db_file))
-    print("#####")
-
-exit(0)