last time, late modifications...

.gitignore

@@ -1 +0,0 @@
-*.db

Pipfile

@@ -1,16 +0,0 @@
-[[source]]
-url = "https://pypi.org/simple"
-verify_ssl = true
-name = "pypi"
-
-[packages]
-networkx = "*"
-pyvis = "*"
-matplotlib = "*"
-fastapi = "*"
-
-[dev-packages]
-
-[requires]
-python_version = "3.11"
-python_full_version = "3.11.8"

README.md

@@ -1,30 +0,0 @@
-# Traceroute Internet Mapper - TIM
-
-## Development
-
-```sh
-# Install all dependencies. `pipenv` was used previously:
-pipenv --python 3.11
-pipenv install
-pipenv shell
-
-# Start dev server
-fastapi dev app/main.py
-
-# Start posting data to the tracing endpoint:
-traceroute git.rauhala.info -q1 | http POST localhost:8000/trace/MYHOSTNAME
-or, with curl:
-traceroute tuoppi.dy.fi -q1 | curl -X POST http://localhost:8000/trace/MYHOSTNAME --data-binary @-
-
-```
-
-## URLs of interest
-
-- http://localhost:8000/
-  - API Index, nothing to see.
-
-- http://localhost:8000/docs/
-  - Autogenerated API docs, mildy interesting
-
-- http://localhost:8000/static/index.html
-  - Rendered HTML index, very interesting

app/collector.py

@@ -1,95 +0,0 @@
-import json
-import uuid
-import hashlib
-
-from datetime import datetime
-
-from .db import Database
-
-
-def parse_traceroute_output(data: str, origin: str):
-    lines = data.strip().split("\n")
-    target = lines[0].split()[2]
-
-    created = datetime.now().isoformat()
-
-    trace = {"target": target, "created": created, "origin": origin, "hops": []}
-
-    prev_latency = 0
-
-    for line in lines[1:]:
-        hop_info = line.split()
-        print("LINE:", hop_info)
-        try:
-            number, name, ip, latency, _ = hop_info
-            hop = {
-                "created": created,
-                "number": number,
-                "name": name,
-                "ip": ip.strip("()"),
-                "latency": float(latency),
-            }
-        except ValueError:
-            number, name = hop_info
-            hop = {
-                "created": created,
-                "number": number,
-                "name": name,
-                "ip": None,
-                "latency": None,
-            }
-
-        trace["hops"].append(hop)
-
-    return trace
-
-
-def store_traceroute(trace):
-    db = Database()
-
-    # hops_json = json.dumps(trace['hops'])
-
-    path_ids = {}
-
-    previous_hop_ip = None
-    previous_hop_latency = None
-    for hop in trace["hops"]:
-        hop_number = hop["number"]
-        hop_name = hop.get("name")
-        hop_ip = hop.get("ip")
-        hop_latency = hop.get("latency")
-        link_id = None
-
-        # insert links and get their id's
-        if previous_hop_ip:
-            link_id = db.create_link(previous_hop_ip, hop_ip)
-            path_ids[hop_number] = link_id
-
-        previous_hop_ip = hop_ip
-
-        # Save hop details
-        db.create_hop(hop_name, hop_ip, hop_latency)
-
-        # calculate link latency if possible and store it
-        if link_id and previous_hop_latency:
-            link_latency = hop_latency - previous_hop_latency
-            db.create_latency(link_id, trace["created"], link_latency)
-
-    # make entry to "Paths" table
-    if path_ids:
-        json_path_ids = json.dumps(path_ids)
-        db.create_path(node, trace["target"], json_path_ids)
-
-    db.end()
-
-
-def generate_node_id():
-    mac = uuid.getnode()
-    mac_str = ":".join(
-        ["{:02x}".format((mac >> ele) & 0xFF) for ele in range(0, 8 * 6, 8)][::-1]
-    )
-
-    # Hash the MAC address using SHA-256 to generate a unique ID
-    unique_id = hashlib.sha256(mac_str.encode()).hexdigest()
-
-    return unique_id

app/db.py

@@ -1,192 +0,0 @@
-import sqlite3
-from functools import wraps
-
-# Type alias
-Cursor = sqlite3.Cursor
-
-# Configs
-DB_FILE = "./traceroute.db"
-
-
-class Database:
-    def __init__(self):
-        self.db_file = DB_FILE
-        self.conn = sqlite3.connect(self.db_file, check_same_thread=False)
-
-        # Return fetch() data as Row objects, instead of tuples.
-        self.conn.row_factory = sqlite3.Row
-
-        self.cursor = self.conn.cursor()
-
-    def create_tables(self):
-        self.cursor.executescript(
-            """
-            CREATE TABLE IF NOT EXISTS Traces (
-                id INTEGER PRIMARY KEY AUTOINCREMENT,
-                created TEXT NOT NULL,
-                origin TEXT NOT NULL,
-                target TEXT NOT NULL
-            );
-
-            CREATE TABLE IF NOT EXISTS Hops (
-                id INTEGER PRIMARY KEY AUTOINCREMENT,
-                trace_id INTEGER,
-                created TEXT NOT NULL,
-                number INTEGER NOT NULL,
-                name TEXT,
-                ip TEXT,
-                latency TEXT,
-
-                FOREIGN KEY(trace_id) REFERENCES Traces(id)
-            );
-        """
-        )
-
-    def end(self):
-        """Always call this after you're done with the connection / request."""
-
-        self.conn.commit()
-        self.conn.close()
-
-    def list_traces(self):
-        # TODO: time filter
-        result = []
-
-        self.cursor.execute("SELECT * FROM Traces")
-        traces = self.cursor.fetchall()
-
-        for t in traces:
-            trace = dict(t)
-
-            self.cursor.execute(
-                "SELECT number, name, ip, latency FROM Hops WHERE trace_id = ? ORDER BY number ASC",
-                (trace["id"],),
-            )
-            hops = self.cursor.fetchall()
-            trace["hops"] = hops
-
-            result.append(trace)
-
-        return result
-
-    def create_trace(self, trace):
-        self.cursor.execute(
-            "INSERT OR IGNORE INTO Traces (created, origin, target) VALUES (?, ?, ?)",
-            (trace["created"], trace["origin"], trace["target"]),
-        )
-        trace_id = self.cursor.lastrowid
-
-        for hop in trace["hops"]:
-            self.cursor.execute(
-                "INSERT OR IGNORE INTO Hops (trace_id, created, number, name, ip, latency) VALUES (?, ?, ?, ?, ?, ?)",
-                (
-                    trace_id,
-                    hop["created"],
-                    hop["number"],
-                    hop["name"],
-                    hop["ip"],
-                    hop["latency"],
-                ),
-            )
-
-        return self.cursor.fetchone()
-
-    def create_hop(self, name, ip, latency):
-        self.cursor.execute(
-            "INSERT INTO Hops (name, ip, latency) VALUES (?, ?, ?)",
-            (name, ip, latency),
-        )
-
-    def create_latency(self, link_id, timestamp, link_latency):
-        self.cursor.execute(
-            "INSERT INTO Latency (link_id, timestamp, latency_ms) VALUES (?, ?, ?)",
-            (link_id, timestamp, link_latency),
-        )
-
-    def create_path(self, node, target, json):
-        self.cursor.execute(
-            "INSERT OR IGNORE INTO Paths (node, target, hops_json) VALUES (?, ?, ?)",
-            (node, target, json),
-        )
-
-
-def ensure_table_setup():
-    db = Database()
-    db.create_tables()
-    db.end()
-
-
-####################################################################
-####################################################################
-####################################################################
-####################################################################
-
-
-def with_connection(func):
-    @wraps(func)
-    def wrapped(*args, **kwargs):
-        conn = sqlite3.connect(DB_FILE)
-        cursor = conn.cursor()
-
-        result = func(cursor, *args, **kwargs)
-
-        conn.commit()
-        conn.close()
-
-        return result
-
-    return wrapped
-
-
-@with_connection
-def init_db(cursor: Cursor):
-    cursor.executescript(
-        """
-        CREATE TABLE IF NOT EXISTS Links (
-            id INTEGER PRIMARY KEY,
-            source_ip TEXT NOT NULL,
-            destination_ip TEXT NOT NULL,
-            UNIQUE(source_ip, destination_ip)
-        );
-
-        CREATE TABLE IF NOT EXISTS Paths (
-            id INTEGER PRIMARY KEY,
-            node TEXT NOT NULL,
-            target TEXT NOT NULL,
-            hops_json TEXT NOT NULL,
-            UNIQUE(node, target, hops_json)
-        );
-
-        CREATE TABLE IF NOT EXISTS Latency (
-            id INTEGER PRIMARY KEY,
-            link_id INTEGER NOT NULL,
-            timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
-            latency_ms REAL NOT NULL,
-            FOREIGN KEY (link_id) REFERENCES Links(id)
-        );
-
-        CREATE TABLE IF NOT EXISTS HopDetails (
-            id INTEGER PRIMARY KEY,
-            hop_name TEXT,
-            hop_ip TEXT,
-            hop_latency TEXT
-        );
-    """
-    )
-
-
-@with_connection
-def insert_hop(cursor: Cursor, previous_hop_ip: str, hop_ip: str):
-    """Insert a new hop and return related Link id"""
-
-    cursor.execute(
-        "INSERT OR IGNORE INTO Links (source_ip, destination_ip) VALUES (?, ?)",
-        (previous_hop_ip, hop_ip),
-    )
-
-    cursor.execute(
-        "SELECT id FROM Links WHERE source_ip = ? AND destination_ip = ?",
-        (previous_hop_ip, hop_ip),
-    )
-
-    return cursor.fetchone()

app/main.py

@@ -1,67 +0,0 @@
-from fastapi import Request, FastAPI
-from fastapi.staticfiles import StaticFiles
-
-from .collector import parse_traceroute_output, store_traceroute
-from .db import Database, ensure_table_setup
-
-from pprint import pprint as print
-
-
-# Setup our SQLite before anything else.
-ensure_table_setup()
-
-# Setup web framework thingies
-app = FastAPI()
-app.mount("/static", StaticFiles(directory="app/static"), name="static")
-
-
-@app.get("/")
-def read_root():
-    return {
-        "greeting": "Hello, Kalzu!",
-        "instructions": [
-            "Try piping traceroute data directly to POST /trace/{hostname}.",
-            "{hostname} is for filtering data by sender.",
-            "For example the following command using HTTPie:",
-            "",
-            "  $ traceroute peitto.info | http POST localhost:8000/trace/rekku",
-            "",
-            "",
-            "Also take a look at /docs/ and /static/index.html",
-            "",
-            "",
-            "",
-            "END OF TRANSMISSION",
-        ]
-        + [None] * 800,
-    }
-
-
-@app.get("/trace/")
-def list_traces():
-    db = Database()
-
-    trace = db.list_traces()
-
-    db.end()
-    return trace
-
-
-@app.post("/trace/{origin}")
-async def create_trace(origin: str, request: Request):
-    raw_data = await request.body()
-    data = raw_data.decode("utf-8", "ignore")
-
-    print(f"Received data from {origin}:")
-    print(data)
-
-    trace = parse_traceroute_output(data, origin)
-
-    print("Parsed data:")
-    print(trace)
-
-    db = Database()
-    db.create_trace(trace)
-    db.end()
-
-    return {"status": "ok"}

app/static/index.html

@@ -1,20 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-    <title>Kalzu</title>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1">
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/sigma.js/2.4.0/sigma.min.js" integrity="sha512-iiPEYww3QXZU5C795JnnINBRNgHqDnRHs9mA7aJoqx4pNE4u3CknCDGmePHFoHtKR/6C9aIcRFa+HJ6obtlteQ==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/graphology/0.25.4/graphology.umd.min.js" integrity="sha512-tjMBhL9fLMcqoccPOwpRiIQIOAyUh18lWUlUvE10zvG1UNMfxUC4qSERmUq+VF30iavIyqs/q6fSP2o475FAUw==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
-    <script src="/static/d3.v7.min.js"></script>
-    <style>
-      #container {
-        height: 95vh;
-      }
-    </style>
-  </head>
-  <body style="background: #eee">
-    <div id="container"></div>
-    <script src="/static/index.js"></script>
-  </body>
-</html>

app/static/index.js

@@ -1,357 +0,0 @@
-const linkArc = (d) => {
-  const r = Math.hypot(d.target.x - d.source.x, d.target.y - d.source.y) * 3;
-  return `
-    M${d.source.x},${d.source.y}
-    A${r},${r} 0 0,1 ${d.target.x},${d.target.y}
-  `;
-};
-
-const drag = (simulation) => {
-  function dragstarted(event, d) {
-    if (!event.active) simulation.alphaTarget(0.3).restart();
-    d.fx = d.x;
-    d.fy = d.y;
-  }
-
-  function dragged(event, d) {
-    d.fx = event.x;
-    d.fy = event.y;
-  }
-
-  function dragended(event, d) {
-    if (!event.active) simulation.alphaTarget(0);
-    d.fx = null;
-    d.fy = null;
-  }
-
-  return d3
-    .drag()
-    .on("start", dragstarted)
-    .on("drag", dragged)
-    .on("end", dragended);
-};
-
-// const drawChart = (data) => {
-//   // Specify the dimensions of the chart.
-//   const width = 1600;
-//   const height = 1200;
-//
-//   // Specify the color scale.
-//   const color = d3.scaleOrdinal(d3.schemeCategory10);
-//
-//   // The force simulation mutates links and nodes, so create a copy
-//   // so that re-evaluating this cell produces the same result.
-//   const links = data.links.map((d) => ({ ...d }));
-//   const nodes = data.nodes.map((d) => ({ ...d }));
-//
-//   // Create a simulation with several forces.
-//   const simulation = d3
-//     .forceSimulation(nodes)
-//     .force(
-//       "link",
-//       d3.forceLink(links).id((d) => d.id),
-//     )
-//     .force("charge", d3.forceManyBody())
-//     .force("x", d3.forceX())
-//     .force("y", d3.forceY());
-//
-//   // Create the SVG container.
-//   const svg = d3
-//     .create("svg")
-//     .attr("width", width)
-//     .attr("height", height)
-//     .attr("viewBox", [-width / 2, -height / 2, width, height])
-//     .attr("style", "max-width: 100%; height: auto;");
-//
-//   // Add a line for each link, and a circle for each node.
-//   const link = svg
-//     .append("g")
-//     .attr("stroke", "#999")
-//     .attr("stroke-opacity", 0.6)
-//     .selectAll("line")
-//     .data(links)
-//     .join("line")
-//     .attr("stroke-width", 1); // (d) => Math.sqrt(d.value));
-//
-//   const node = svg
-//     .append("g")
-//     .attr("stroke", "#fff")
-//     .attr("stroke-width", 1.5)
-//     .selectAll("circle")
-//     .data(nodes)
-//     .join("circle")
-//     .attr("r", 5)
-//     .attr("fill", (d) => color(d.group));
-//
-//   node.append("title").text((d) => d.id);
-//
-//   // Add a drag behavior.
-//   node.call(
-//     d3.drag().on("start", dragstarted).on("drag", dragged).on("end", dragended),
-//   );
-//
-//   // Set the position attributes of links and nodes each time the simulation ticks.
-//   simulation.on("tick", () => {
-//     link
-//       .attr("x1", (d) => d.source.x)
-//       .attr("y1", (d) => d.source.y)
-//       .attr("x2", (d) => d.target.x)
-//       .attr("y2", (d) => d.target.y);
-//
-//     node.attr("cx", (d) => d.x).attr("cy", (d) => d.y);
-//   });
-//
-//   // Reheat the simulation when drag starts, and fix the subject position.
-//   function dragstarted(event) {
-//     if (!event.active) simulation.alphaTarget(0.3).restart();
-//     event.subject.fx = event.subject.x;
-//     event.subject.fy = event.subject.y;
-//   }
-//
-//   // Update the subject (dragged node) position during drag.
-//   function dragged(event) {
-//     event.subject.fx = event.x;
-//     event.subject.fy = event.y;
-//   }
-//
-//   // Restore the target alpha so the simulation cools after dragging ends.
-//   // Unfix the subject position now that it’s no longer being dragged.
-//   function dragended(event) {
-//     if (!event.active) simulation.alphaTarget(0);
-//     event.subject.fx = null;
-//     event.subject.fy = null;
-//   }
-//
-//   // When this cell is re-run, stop the previous simulation. (This doesn’t
-//   // really matter since the target alpha is zero and the simulation will
-//   // stop naturally, but it’s a good practice.)
-//   // invalidation.then(() => simulation.stop());
-//
-//   return svg.node();
-// };
-
-const drawChart2 = (data) => {
-  // Data parsing
-  const nodes = Array.from(
-    new Set(data.flatMap((l) => [l.source, l.target])),
-    (id) => ({ id }),
-  );
-  const links = data.map((d) => Object.create(d));
-
-  // Styles
-  const width = window.visualViewport.width;
-  const height = window.visualViewport.height - 10;
-
-  const colors = Array.from(new Set(data.map((d) => d.traceId)));
-  const color = d3.scaleOrdinal(colors, d3.schemeCategory10);
-
-  const simulation = d3
-    .forceSimulation(nodes)
-    .force(
-      "link",
-      d3.forceLink(links).id((d) => d.id),
-    )
-    .force("charge", d3.forceManyBody().strength(-400))
-    .force("x", d3.forceX())
-    .force("y", d3.forceY());
-
-  // Canvas settings
-  const svg = d3
-    .create("svg")
-    .attr("width", width)
-    .attr("height", height)
-    .attr("viewBox", [-width / 2, -height / 2, width, height])
-    .attr("style", "max-width: 100%; height: auto; font: 14px monospace;");
-
-  // Pre-type arrowheads, as they don't inherit styles.
-  svg
-    .append("defs")
-    .selectAll("marker")
-    .data(colors)
-    .join("marker")
-    .attr("id", (d) => `arrow-${d}`)
-    .attr("viewBox", "0 -5 10 10")
-    .attr("refX", 15)
-    .attr("refY", -0.5)
-    .attr("markerWidth", 5)
-    .attr("markerHeight", 5)
-    .attr("orient", "auto")
-    .append("path")
-    .attr("fill", color)
-    .attr("d", "M0,-5L10,0L0,5");
-
-  const link = svg
-    .append("g")
-    .attr("fill", "none")
-    .attr("stroke-width", 1.5)
-    .selectAll("path")
-    .data(links)
-    .join("path")
-    .attr("stroke", (d) => color(d.traceId))
-    .attr(
-      "marker-end",
-      (d) => `url(${new URL(`#arrow-${d.traceId}`, location)})`,
-    );
-
-  const node = svg
-    .append("g")
-    .attr("fill", "currentColor")
-    .attr("stroke-linecap", "round")
-    .attr("stroke-linejoin", "round")
-    .selectAll("g")
-    .data(nodes)
-    .join("g")
-    .call(drag(simulation));
-
-  // Node "icon"
-  node
-    .append("circle")
-    .attr("stroke", "white")
-    .attr("stroke-width", 1.5)
-    .attr("r", 5);
-
-  // Node text
-  node
-    .append("text")
-    .attr("x", 8)
-    .attr("y", 4)
-    .text((d) => (d.id.endsWith("*") ? "*" : d.id))
-    .clone(true)
-    .lower()
-    .attr("fill", "black")
-    .attr("stroke", "white")
-    .attr("stroke-width", 2);
-
-  simulation.on("tick", () => {
-    link.attr("d", linkArc);
-    // link
-    //   .attr("x1", (d) => d.source.x)
-    //   .attr("y1", (d) => d.source.y)
-    //   .attr("x2", (d) => d.target.x)
-    //   .attr("y2", (d) => d.target.y);
-    node.attr("transform", (d) => `translate(${d.x},${d.y})`);
-  });
-
-  // invalidation.then(() => simulation.stop());
-
-  return Object.assign(svg.node(), { scales: { color } });
-};
-
-const drawSigma = (data) => {
-  // Create a graphology graph
-  const graph = new graphology.MultiDirectedGraph();
-
-  data.nodes.forEach((n, i) => {
-    console.log("Node:", n);
-    try {
-      graph.addNode(n.id, n);
-    } catch (e) {
-      console.log("Node add:", e);
-    }
-  });
-
-  data.links.forEach((l, i) => {
-    graph.addEdge(l.source, l.target, l);
-  });
-
-  // Instantiate sigma.js and render the graph
-  const renderer = new Sigma(graph, document.getElementById("container"), {
-    labelThreshold: -10000,
-  });
-};
-
-const getNodeID = (hop, prevId, trace) => {
-  if (prevId === null) {
-    return trace.origin;
-  }
-  if (hop.name === "*") {
-    return `${trace.id}-${hop.number}-*`;
-  }
-
-  return hop.ip;
-};
-
-const parseNodesAndLinks = (traces) => {
-  const nodes = [];
-  const links = [];
-
-  const colors = Array.from(new Set(traces.map((t) => t.id)));
-  const color = d3.scaleOrdinal(colors, d3.schemeCategory10);
-
-  traces.forEach((trace) => {
-    let prevId = null;
-    let latestNumber = null;
-
-    trace.hops.forEach((hop) => {
-      const id = getNodeID(hop, prevId, trace);
-
-      // New node
-      nodes.push({
-        id: id,
-        label: id.endsWith("*") ? "*" : id,
-        x: trace.id,
-        y: hop.number / 2,
-        size: 9,
-        labelSize: 30,
-        color: color(trace.id),
-      });
-
-      if (prevId) {
-        // New link
-        links.push({
-          label: "asd",
-          source: prevId,
-          target: id,
-          traceId: trace.id,
-          origin: trace.origin,
-          size: 1,
-          color: color(trace.id),
-        });
-      }
-
-      prevId = id;
-      latestNumber = hop.number;
-    });
-
-    /**
-     * Last "destination" node, just for candy
-     */
-    nodes.push({
-      id: trace.id,
-      label: trace.target,
-      x: trace.id,
-      y: (latestNumber + 1) / 2,
-      size: 8,
-      color: "black",
-    });
-
-    links.push({
-      source: prevId,
-      target: trace.id,
-    });
-  });
-
-  // { id: ip, group: origin, radius: 2 }
-  // { source: prev.ip, target: ip, value: latency }
-  return {
-    nodes,
-    links,
-  };
-};
-
-async function main() {
-  const response = await fetch("/trace/");
-  const traces = await response.json();
-
-  console.log("Traces:", traces);
-  const data = parseNodesAndLinks(traces);
-
-  console.log("Data:", data);
-  // const chart = drawChart2(data.links);
-
-  const chart = drawSigma(data);
-
-  // container.append(chart);
-}
-
-main();

test.py

@@ -7,10 +7,11 @@ from sys import hash_info
 import pprint
 
 def run_traceroute(host):
+    timestamp = datetime.now().timestamp()
     result = subprocess.run(['traceroute', host], stdout=subprocess.PIPE)
-    return result.stdout.decode()
+    return result.stdout.decode(), timestamp
 
-def parse_traceroute_output(output):
+def parse_traceroute_output(output, timestamp):
 
     lines = output.strip().split('\n')
     hops = []
@@ -20,7 +21,7 @@ def parse_traceroute_output(output):
     target = output.strip().split('\n')[0].split()[2]
 
     for line in lines[1:]:
-        hop = []
+        hop = {}
         hop_info = line.split()
         hop_number = int(hop_info[0])
         hop_name = None
@@ -43,17 +44,14 @@ def parse_traceroute_output(output):
 
             # If first colum is either name or ip-address
             if count == 1:
-                print(part)
                 match = re.search(ip_regex, part)
                 if  match:
                     hop_ip = part.strip('()')
                 else:
-                    print('do ever here?')
                     hop_name = part
 
             # Second colum is ip-address first latency reading
             if count == 2:
-                print(part)
                 if re.search(ip_regex, part):
                     try:
                         _ip = ipaddress.ip_address(part.strip('()'))
@@ -65,7 +63,6 @@ def parse_traceroute_output(output):
             # reruns of the hop_name and hop_ip...
             # We only need the latency floats anymore.
             else:
-                print(part)
                 try:
                     latency = float(part)
                     latencies.append(latency)
@@ -74,12 +71,12 @@ def parse_traceroute_output(output):
 
         hop_latency = sum(latencies) / len(latencies) if latencies else None
 
-        
+        hop['timestamp'] = timestamp
-        hop.append(hop_number)
+        hop['hop_number'] = hop_number
         if not hop_name == None:
-            hop.append(hop_name)
+            hop['hop_name'] = hop_name
-        hop.append(hop_ip)
+        hop['hop_ip'] = hop_ip
-        hop.append(hop_latency)
+        hop['hop_latency'] = hop_latency
         
 
         hops.append(hop)
@@ -89,8 +86,8 @@ def parse_traceroute_output(output):
 if __name__ == '__main__':
 
     target='8.8.8.8'
-    traceroute_output = run_traceroute(target)
+    traceroute_output, timestamp = run_traceroute(target)
     
-    target, hops = parse_traceroute_output(traceroute_output)
+    target, hops = parse_traceroute_output(traceroute_output, timestamp)
     print('>> '+target)
     pprint.pprint(hops)

traceroute_collector.py

@@ -0,0 +1,246 @@
+#!/usr/bin/env python3.11
+
+
+import subprocess
+import sqlite3
+import re
+import json
+import ipaddress
+import uuid
+import hashlib
+from datetime import datetime
+
+
+def run_traceroute(host):
+    timestamp = datetime.now().timestamp()
+    result = subprocess.run(['traceroute', host], stdout=subprocess.PIPE)
+    return result.stdout.decode(), timestamp
+
+def parse_traceroute_output(output, timestamp):
+
+    lines = output.strip().split('\n')
+    trace = {}
+    hops = []
+
+    ip_regex = r"\((.*?)\)" # ipaddress are in ()
+
+    target = output.strip().split('\n')[0].split()[2]
+
+    for line in lines[1:]:
+        hop = {}
+        hop_info = line.split()
+        hop_number = int(hop_info[0])
+        hop_name = None
+        hop_ip = None
+        hop_latency = None
+
+        latencies = []
+
+        #print("##### "+str(hop_info))
+        count = 0
+        for part in hop_info[1:]:
+            count += 1
+
+            # source node drops or blocks icmp packages
+            # We will give funny to name to hop for not answering and move on.
+            if part == '*':
+                hop_name = 'unresponsive'
+                hop_ip = 'unresponsive'
+                break
+
+
+            # If first colum is either name or ip-address
+            if count == 1:
+                match = re.search(ip_regex, part)
+                if  match:
+                    hop_ip = part.strip('()')
+                else:
+                    hop_name = part
+
+            # Second colum is ip-address first latency reading
+            if count == 2:
+                if re.search(ip_regex, part):
+                    try:
+                        _ip = ipaddress.ip_address(part.strip('()'))
+                        hop_ip = part.strip('()')
+                    except ValueError:
+                        pass  # Ignore if it's not a valid IP address
+
+            # Rest of the input colums are either latency floats, 'ms' or 
+            # reruns of the hop_name and hop_ip...
+            # We only need the latency floats anymore.
+            else:
+                try:
+                    latency = float(part)
+                    latencies.append(latency)
+                except ValueError:
+                    pass
+
+        hop_latency = sum(latencies) / len(latencies) if latencies else None
+
+        hop['hop_number'] = hop_number
+        if not hop_name == None:
+            hop['hop_name'] = hop_name
+        hop['hop_ip'] = hop_ip
+        hop['hop_latency'] = hop_latency
+        hops.append(hop)
+        
+    trace['target'] = target
+    trace['timestamp'] = timestamp
+    trace['hops'] = hops
+
+    return trace
+
+
+def create_tables(databasefile):
+        # Connect to the SQLite database
+    conn = sqlite3.connect(databasefile)
+    cursor = conn.cursor()
+    
+    # SQL statements to create the tables
+    create_links_table = """
+    CREATE TABLE IF NOT EXISTS Links (
+        id INTEGER PRIMARY KEY,
+        source_ip TEXT NOT NULL,
+        destination_ip TEXT NOT NULL,
+        UNIQUE(source_ip, destination_ip)
+    );
+    """
+    
+    create_paths_table = """
+    CREATE TABLE IF NOT EXISTS Paths (
+        id INTEGER PRIMARY KEY,
+        node TEXT NOT NULL,
+        target TEXT NOT NULL,
+        hops_json TEXT NOT NULL,
+        UNIQUE(node, target, hops_json)
+    );
+    """
+    
+    create_latency_table = """
+    CREATE TABLE IF NOT EXISTS Latency (
+        id INTEGER PRIMARY KEY,
+        link_id INTEGER NOT NULL,
+        timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
+        latency_ms REAL NOT NULL,
+        FOREIGN KEY (link_id) REFERENCES Links(id)
+    );
+    """
+    
+    create_hopdetails_table = """
+    CREATE TABLE IF NOT EXISTS HopDetails (
+        id INTEGER PRIMARY KEY,
+        hop_name TEXT,
+        hop_ip TEXT,
+        hop_latency TEXT
+    );
+    """
+    
+    # Execute the SQL statements
+    cursor.execute(create_links_table)
+    cursor.execute(create_paths_table)
+    cursor.execute(create_latency_table)
+    cursor.execute(create_hopdetails_table)
+    
+    # Commit changes and close the connection
+    conn.commit()
+    conn.close()
+
+def store_traceroute(db_file, node, trace):
+    conn = sqlite3.connect(db_file)
+    cursor = conn.cursor()
+
+
+    hops_json = json.dumps(trace['hops'])
+    
+    path_ids = {}
+
+    previous_hop_ip = None
+    previous_hop_latency = None
+    for hop in trace['hops']:
+        hop_number = hop['hop_number']
+        hop_name = hop.get('hop_name')
+        hop_ip = hop.get('hop_ip')
+        hop_latency = hop.get('hop_latency')
+        link_id = None
+
+        # insert links and get their id's
+        if previous_hop_ip:
+            cursor.execute("""
+                INSERT OR IGNORE INTO Links (source_ip, destination_ip) VALUES (?, ?)
+            """, (previous_hop_ip, hop_ip))
+            cursor.execute("""
+                SELECT id FROM Links WHERE source_ip = ? AND destination_ip = ?
+            """, (previous_hop_ip, hop_ip))
+            result = cursor.fetchone()
+            link_id = result
+            path_ids[hop_number] = link_id
+        previous_hop_ip = hop_ip
+
+        # Save hop details
+        cursor.execute("INSERT INTO HopDetails (hop_name, hop_ip, hop_latency) VALUES (?, ?, ?)",
+                        (hop_name, hop_ip, hop_latency))
+        
+        # calculate link latency if possible and store it
+        if link_id and previous_hop_latency:
+            link_latency = hop_latency - previous_hop_latency
+            cursor.execute("INSERT INTO Latency (link_id, timestamp, latency_ms) VALUES (?, ?, ?)", 
+                           (link_id, trace['timestamp'], link_latency))
+
+    # make entry to "Paths" table
+    if path_ids:
+        json_path_ids = json.dumps(path_ids)
+        cursor.execute("INSERT OR IGNORE INTO Paths (node, target, hops_json) VALUES (?, ?, ?)",
+                       (node, trace['target'], json_path_ids))
+
+    conn.commit()
+    conn.close()
+
+
+def retrieve_traceroute(db_file):
+
+    retval = {}
+
+    conn = sqlite3.connect(db_file)
+    cursor = conn.cursor()
+    cursor.execute('''
+        SELECT target, hops_json
+        FROM Paths
+    ''')
+    retval['path'] = cursor.fetchall()
+
+    cursor.execute('''
+        SELECT source_ip, destination_ip
+        FROM Links
+    ''')
+    retval['links'] = cursor.fetchall()
+    conn.close()
+    return retval
+
+def generate_node_id():
+    mac = uuid.getnode()
+    mac_str = ':'.join(['{:02x}'.format((mac >> ele) & 0xff) for ele in range(0,8*6,8)][::-1])
+    
+    # Hash the MAC address using SHA-256 to generate a unique ID
+    unique_id = hashlib.sha256(mac_str.encode()).hexdigest()
+
+    return unique_id
+
+
+if __name__ == '__main__':
+
+    db_file="./traceroute.db"
+    create_tables(db_file)
+    my_id = generate_node_id()
+
+    target='1.1.1.1'
+    traceroute_output, timestamp = run_traceroute(target)
+    trace = parse_traceroute_output(traceroute_output, timestamp)
+
+    store_traceroute(db_file, my_id, trace)
+
+    print("#####")
+    print(retrieve_traceroute(db_file))
+    print("#####")
+
+exit(0)

traceroute_visualization.py

@@ -9,8 +9,8 @@ def retrieve_traceroute_data():
     conn = sqlite3.connect('traceroute.db')
     cursor = conn.cursor()
     cursor.execute('''
-        SELECT l.source_ip, l.destination_ip, ll.latency_ms, ll.timestamp
+        SELECT l.source_ip, l.destination_ip, ll.latency, ll.timestamp
-        FROM Latency ll
+        FROM link_latency ll
         JOIN links l ON ll.link_id = l.id
         ORDER BY ll.timestamp
     ''')
@@ -64,4 +64,4 @@ if __name__ == '__main__':
     # Visualize the graph using pyvis
     #visualize_graph_pyvis(graph, 'traceroute_network.html')
 
-#exit(0)
+exit(0)