TEMP: Rewrite everything on top of FastAPI

Pipfile

@@ -0,0 +1,16 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+networkx = "*"
+pyvis = "*"
+matplotlib = "*"
+fastapi = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3.11"
+python_full_version = "3.11.8"

app/collector.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3.11
+
+import re
+import json
+import ipaddress
+import uuid
+import hashlib
+
+from .db import Database
+
+def parse_traceroute_output(output, timestamp):
+    lines = output.strip().split('\n')
+    trace = {}
+    hops = []
+
+    ip_regex = r"\((.*?)\)" # ipaddress are in ()
+
+    target = output.strip().split('\n')[0].split()[2]
+
+    for line in lines[1:]:
+        hop = {}
+        hop_info = line.split()
+        hop_number = int(hop_info[0])
+        hop_name = None
+        hop_ip = None
+        hop_latency = None
+
+        latencies = []
+
+        #print("##### "+str(hop_info))
+        count = 0
+        for part in hop_info[1:]:
+            count += 1
+
+            # source node drops or blocks icmp packages
+            # We will give funny to name to hop for not answering and move on.
+            if part == '*':
+                hop_name = 'unresponsive'
+                hop_ip = 'unresponsive'
+                break
+
+
+            # If first colum is either name or ip-address
+            if count == 1:
+                match = re.search(ip_regex, part)
+                if  match:
+                    hop_ip = part.strip('()')
+                else:
+                    hop_name = part
+
+            # Second colum is ip-address first latency reading
+            if count == 2:
+                if re.search(ip_regex, part):
+                    try:
+                        _ip = ipaddress.ip_address(part.strip('()'))
+                        hop_ip = part.strip('()')
+                    except ValueError:
+                        pass  # Ignore if it's not a valid IP address
+
+            # Rest of the input colums are either latency floats, 'ms' or
+            # reruns of the hop_name and hop_ip...
+            # We only need the latency floats anymore.
+            else:
+                try:
+                    latency = float(part)
+                    latencies.append(latency)
+                except ValueError:
+                    pass
+
+        hop_latency = sum(latencies) / len(latencies) if latencies else None
+
+        hop['hop_number'] = hop_number
+        if not hop_name == None:
+            hop['hop_name'] = hop_name
+        hop['hop_ip'] = hop_ip
+        hop['hop_latency'] = hop_latency
+        hops.append(hop)
+
+    trace['target'] = target
+    trace['timestamp'] = timestamp
+    trace['hops'] = hops
+
+    return trace
+
+
+def store_traceroute(node, trace):
+    db = Database()
+
+    #hops_json = json.dumps(trace['hops'])
+
+    path_ids = {}
+
+    previous_hop_ip = None
+    previous_hop_latency = None
+    for hop in trace['hops']:
+        hop_number = hop['hop_number']
+        hop_name = hop.get('hop_name')
+        hop_ip = hop.get('hop_ip')
+        hop_latency = hop.get('hop_latency')
+        link_id = None
+
+        # insert links and get their id's
+        if previous_hop_ip:
+            link_id = db.create_link(previous_hop_ip, hop_ip)
+            path_ids[hop_number] = link_id
+
+        previous_hop_ip = hop_ip
+
+        # Save hop details
+        db.create_hop(hop_name, hop_ip, hop_latency)
+
+        # calculate link latency if possible and store it
+        if link_id and previous_hop_latency:
+            link_latency = hop_latency - previous_hop_latency
+            db.create_latency(link_id, trace['timestamp'], link_latency)
+
+    # make entry to "Paths" table
+    if path_ids:
+        json_path_ids = json.dumps(path_ids)
+        db.create_path(node, trace['target'], json_path_ids)
+
+    db.end()
+
+
+
+def generate_node_id():
+    mac = uuid.getnode()
+    mac_str = ':'.join(['{:02x}'.format((mac >> ele) & 0xff) for ele in range(0,8*6,8)][::-1])
+
+    # Hash the MAC address using SHA-256 to generate a unique ID
+    unique_id = hashlib.sha256(mac_str.encode()).hexdigest()
+
+    return unique_id

app/db.py

@@ -0,0 +1,182 @@
+import sqlite3
+from functools import wraps
+
+# Type alias
+Cursor = sqlite3.Cursor
+
+# Configs
+DB_FILE = "./traceroute.db"
+
+
+class Database:
+    def __init__(self):
+        self.db_file = DB_FILE
+        self.conn = sqlite3.connect(self.db_file, check_same_thread=False)
+        self.cursor = self.conn.cursor()
+
+    def create_tables(self):
+        self.cursor.executescript("""
+            CREATE TABLE IF NOT EXISTS Links (
+                id INTEGER PRIMARY KEY,
+                source_ip TEXT NOT NULL,
+                destination_ip TEXT NOT NULL,
+                UNIQUE(source_ip, destination_ip)
+            );
+
+            CREATE TABLE IF NOT EXISTS Paths (
+                id INTEGER PRIMARY KEY,
+                node TEXT NOT NULL,
+                target TEXT NOT NULL,
+                hops_json TEXT NOT NULL,
+                UNIQUE(node, target, hops_json)
+            );
+
+            CREATE TABLE IF NOT EXISTS Latency (
+                id INTEGER PRIMARY KEY,
+                link_id INTEGER NOT NULL,
+                timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
+                latency_ms REAL NOT NULL,
+                FOREIGN KEY (link_id) REFERENCES Links(id)
+            );
+
+            CREATE TABLE IF NOT EXISTS HopDetails (
+                id INTEGER PRIMARY KEY,
+                hop_name TEXT,
+                hop_ip TEXT,
+                hop_latency TEXT
+            );
+        """)
+
+    def end(self):
+        """Always call this after you're done with the connection / request."""
+
+        self.conn.commit()
+        self.conn.close()
+
+    def get_traceroute(self):
+        retval = {}
+
+        self.cursor.execute('''
+            SELECT target, hops_json
+            FROM Paths
+        ''')
+        retval['path'] = self.cursor.fetchall()
+
+        self.cursor.execute('''
+            SELECT source_ip, destination_ip
+            FROM Links
+        ''')
+        retval['links'] = self.cursor.fetchall()
+
+        return retval
+
+    def create_link(self, previous_hop_ip, hop_ip):
+        self.cursor.execute(
+            "INSERT OR IGNORE INTO Links (source_ip, destination_ip) VALUES (?, ?)",
+            (previous_hop_ip, hop_ip)
+        )
+        self.cursor.execute(
+            "SELECT id FROM Links WHERE source_ip = ? AND destination_ip = ?",
+            (previous_hop_ip, hop_ip)
+        )
+
+        return self.cursor.fetchone()
+
+    def create_hop(self, name, ip, latency):
+        self.cursor.execute(
+            "INSERT INTO HopDetails (hop_name, hop_ip, hop_latency) VALUES (?, ?, ?)",
+            (name, ip, latency)
+        )
+
+    def create_latency(self, link_id, timestamp, link_latency):
+        self.cursor.execute(
+            "INSERT INTO Latency (link_id, timestamp, latency_ms) VALUES (?, NOW(), ?)",
+            (link_id, timestamp, link_latency)
+        )
+
+    def create_path(self, node, target, json):
+        self.cursor.execute(
+            "INSERT OR IGNORE INTO Paths (node, target, hops_json) VALUES (?, ?, ?)",
+            (node, target, json)
+        )
+
+
+
+
+def ensure_table_setup():
+    db = Database()
+    db.create_tables()
+    db.end()
+
+
+
+
+
+####################################################################
+####################################################################
+####################################################################
+####################################################################
+# Temp testing. Fancy decorator stuff.
+
+def with_connection(func):
+    @wraps(func)
+    def wrapped(*args, **kwargs):
+        conn = sqlite3.connect(DB_FILE)
+        cursor = conn.cursor()
+
+        result = func(cursor, *args, **kwargs)
+
+        conn.commit()
+        conn.close()
+
+        return result
+    return wrapped
+
+
+@with_connection
+def init_db(cursor: Cursor):
+    cursor.executescript("""
+        CREATE TABLE IF NOT EXISTS Links (
+            id INTEGER PRIMARY KEY,
+            source_ip TEXT NOT NULL,
+            destination_ip TEXT NOT NULL,
+            UNIQUE(source_ip, destination_ip)
+        );
+
+        CREATE TABLE IF NOT EXISTS Paths (
+            id INTEGER PRIMARY KEY,
+            node TEXT NOT NULL,
+            target TEXT NOT NULL,
+            hops_json TEXT NOT NULL,
+            UNIQUE(node, target, hops_json)
+        );
+
+        CREATE TABLE IF NOT EXISTS Latency (
+            id INTEGER PRIMARY KEY,
+            link_id INTEGER NOT NULL,
+            timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
+            latency_ms REAL NOT NULL,
+            FOREIGN KEY (link_id) REFERENCES Links(id)
+        );
+
+        CREATE TABLE IF NOT EXISTS HopDetails (
+            id INTEGER PRIMARY KEY,
+            hop_name TEXT,
+            hop_ip TEXT,
+            hop_latency TEXT
+        );
+    """)
+
+@with_connection
+def create_link(cursor: Cursor, previous_hop_ip: str, hop_ip: str):
+    """Insert a new hop and return related Link id"""
+
+    cursor.execute("""
+        INSERT OR IGNORE INTO Links (source_ip, destination_ip) VALUES (?, ?)
+    """, (previous_hop_ip, hop_ip))
+
+    cursor.execute("""
+        SELECT id FROM Links WHERE source_ip = ? AND destination_ip = ?
+    """, (previous_hop_ip, hop_ip))
+
+    return cursor.fetchone()

app/main.py

@@ -0,0 +1,55 @@
+from fastapi import Request, FastAPI
+from fastapi.staticfiles import StaticFiles
+
+from datetime import datetime
+
+from .collector import parse_traceroute_output, store_traceroute
+from .db import ensure_table_setup
+
+from pprint import pprint as print
+
+
+# Setup our SQLite before anything else.
+ensure_table_setup()
+
+# Setup web framework thingies
+app = FastAPI()
+app.mount("/static", StaticFiles(directory="static"), name="static")
+
+@app.get("/")
+def read_root():
+    return {
+        "greeting": "Hello, Kalzu!",
+        "instructions": [
+            "Try piping traceroute data directly to POST /trace/{hostname}.",
+            "{hostname} is for filtering data by sender.",
+            "For example the following command using HTTPie:",
+            "",
+            "  $ traceroute peitto.info | http POST localhost:8000/trace/rekku",
+            "",
+            "",
+            "Also take a look at /docs/ and /static/index.html",
+            "",
+            "",
+            "",
+            "END OF TRANSMISSION",
+        ] + [None]*800
+    }
+
+
+@app.post("/trace/{hostname}")
+async def create_trace(hostname: str, request: Request):
+    raw_data = await request.body()
+    data = raw_data.decode("utf-8", "ignore")
+
+    print(f"Received data from {hostname}:")
+    print(data)
+
+    trace = parse_traceroute_output(data, datetime.now())
+
+    print("Parsed data:")
+    print(trace)
+
+    store_traceroute(hostname, trace)
+
+    return {"status": "ok" }

app/static/index.html

@@ -0,0 +1,56 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <title>Kalzu</title>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/sigma.js/2.4.0/sigma.min.js" integrity="sha512-iiPEYww3QXZU5C795JnnINBRNgHqDnRHs9mA7aJoqx4pNE4u3CknCDGmePHFoHtKR/6C9aIcRFa+HJ6obtlteQ==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/graphology/0.25.4/graphology.umd.min.js" integrity="sha512-tjMBhL9fLMcqoccPOwpRiIQIOAyUh18lWUlUvE10zvG1UNMfxUC4qSERmUq+VF30iavIyqs/q6fSP2o475FAUw==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
+  </head>
+  <body style="background: lightgrey">
+    <div id="container" style="width: 97%; height: 95vh; background: white; margin: auto; margin-top: 32px"></div>
+    <script>
+
+      const hops = [
+        ["_gateway", "(192.168.0.1)  0.549 ms"],
+        ["83-148-246-178.static.lounea.fi", "(83.148.246.178)  1.200 ms"],
+        ["*"],
+        ["*"],
+        ["*"],
+        ["86-60-254-198.static.lounea.fi", "(86.60.254.198)  3.719 ms"],
+        ["72.14.194.142", " (72.14.194.142)  3.716 ms"],
+        ["142.251.53.71", " (142.251.53.71)  3.704 ms"],
+        ["142.250.229.87", " (142.250.229.87)  4.603 ms"],
+        ["mad06s09-in-f142.1e100.net", " (216.58.210.142)  3.700 ms"],
+      ];
+
+      console.log("Hops!", hops)
+
+
+      // Create a graphology graph
+      const graph = new graphology.Graph();
+
+      hops.forEach((hop, i) => {
+        const [ label, extra ] = hop;
+
+        const size = i === 0 ? 15 : 10;
+        const color = "green"
+
+        graph.addNode(i, { label, x: 0, y: i, size, color });
+        if (i > 0) {
+
+          graph.addEdge(i, i-1, { size: 2, color: "purple" });
+        }
+      });
+
+      // graph.addNode("1", { label: "Node 1", x: 0, y: 0, size: 10, color: "blue" });
+      // graph.addNode("2", { label: "Node 2", x: 1, y: 1, size: 20, color: "red" });
+      // graph.addEdge("1", "2", { size: 5, color: "purple" });
+
+      // Instantiate sigma.js and render the graph
+      const sigmaInstance = new Sigma(graph, document.getElementById("container"));
+    </script>
+  </body>
+</html>
+</html>

app/traceroute_visualization.py

@@ -9,8 +9,8 @@ def retrieve_traceroute_data():
     conn = sqlite3.connect('traceroute.db')
     cursor = conn.cursor()
     cursor.execute('''
-        SELECT l.source_ip, l.destination_ip, ll.latency, ll.timestamp
+        SELECT l.source_ip, l.destination_ip, ll.latency_ms, ll.timestamp
-        FROM link_latency ll
+        FROM Latency ll
         JOIN links l ON ll.link_id = l.id
         ORDER BY ll.timestamp
     ''')
@@ -25,33 +25,33 @@ def build_graph(data):
         destination_ip = row[1]
         latency = row[2]
         timestamp = row[3]
-        
+
         # Add nodes and edges to the graph
         G.add_node(source_ip)
         G.add_node(destination_ip)
         G.add_edge(source_ip, destination_ip, latency=latency, timestamp=timestamp)
-        
+
     return G
 
 #def visualize_graph_pyvis(G, output_file='network.html'):
 #    net = Network(height='750px', width='100%', directed=True)
 #    net.from_nx(G)
-#    
+#
 #    for edge in G.edges(data=True):
 #        src, dst, data = edge
 #        latency = data['latency']
 #        timestamp = data['timestamp']
 #        net.add_edge(src, dst, title=f'Latency: {latency} ms<br>Timestamp: {timestamp}', value=latency)
-#    
+#
 #    net.show(output_file)
 
 def main():
     # Retrieve data from the database
     traceroute_data = retrieve_traceroute_data()
-    
+
     # Build the network graph
     graph = build_graph(traceroute_data)
-    
+
     return graph
 
 if __name__ == '__main__':
@@ -59,9 +59,9 @@ if __name__ == '__main__':
 
     nx.draw_planar(graph, with_labels=True)
     plt.savefig("path.png")
-    
+
-    
+
     # Visualize the graph using pyvis
     #visualize_graph_pyvis(graph, 'traceroute_network.html')
 
-exit(0)
+#exit(0)

traceroute_collector.py

@@ -1,246 +0,0 @@
-#!/usr/bin/env python3.11
-
-
-import subprocess
-import sqlite3
-import re
-import json
-import ipaddress
-import uuid
-import hashlib
-from datetime import datetime
-
-
-def run_traceroute(host):
-    timestamp = datetime.now().timestamp()
-    result = subprocess.run(['traceroute', host], stdout=subprocess.PIPE)
-    return result.stdout.decode(), timestamp
-
-def parse_traceroute_output(output, timestamp):
-
-    lines = output.strip().split('\n')
-    trace = {}
-    hops = []
-
-    ip_regex = r"\((.*?)\)" # ipaddress are in ()
-
-    target = output.strip().split('\n')[0].split()[2]
-
-    for line in lines[1:]:
-        hop = {}
-        hop_info = line.split()
-        hop_number = int(hop_info[0])
-        hop_name = None
-        hop_ip = None
-        hop_latency = None
-
-        latencies = []
-
-        #print("##### "+str(hop_info))
-        count = 0
-        for part in hop_info[1:]:
-            count += 1
-
-            # source node drops or blocks icmp packages
-            # We will give funny to name to hop for not answering and move on.
-            if part == '*':
-                hop_name = 'unresponsive'
-                hop_ip = 'unresponsive'
-                break
-
-
-            # If first colum is either name or ip-address
-            if count == 1:
-                match = re.search(ip_regex, part)
-                if  match:
-                    hop_ip = part.strip('()')
-                else:
-                    hop_name = part
-
-            # Second colum is ip-address first latency reading
-            if count == 2:
-                if re.search(ip_regex, part):
-                    try:
-                        _ip = ipaddress.ip_address(part.strip('()'))
-                        hop_ip = part.strip('()')
-                    except ValueError:
-                        pass  # Ignore if it's not a valid IP address
-
-            # Rest of the input colums are either latency floats, 'ms' or 
-            # reruns of the hop_name and hop_ip...
-            # We only need the latency floats anymore.
-            else:
-                try:
-                    latency = float(part)
-                    latencies.append(latency)
-                except ValueError:
-                    pass
-
-        hop_latency = sum(latencies) / len(latencies) if latencies else None
-
-        hop['hop_number'] = hop_number
-        if not hop_name == None:
-            hop['hop_name'] = hop_name
-        hop['hop_ip'] = hop_ip
-        hop['hop_latency'] = hop_latency
-        hops.append(hop)
-        
-    trace['target'] = target
-    trace['timestamp'] = timestamp
-    trace['hops'] = hops
-
-    return trace
-
-
-def create_tables(databasefile):
-        # Connect to the SQLite database
-    conn = sqlite3.connect(databasefile)
-    cursor = conn.cursor()
-    
-    # SQL statements to create the tables
-    create_links_table = """
-    CREATE TABLE IF NOT EXISTS Links (
-        id INTEGER PRIMARY KEY,
-        source_ip TEXT NOT NULL,
-        destination_ip TEXT NOT NULL,
-        UNIQUE(source_ip, destination_ip)
-    );
-    """
-    
-    create_paths_table = """
-    CREATE TABLE IF NOT EXISTS Paths (
-        id INTEGER PRIMARY KEY,
-        node TEXT NOT NULL,
-        target TEXT NOT NULL,
-        hops_json TEXT NOT NULL,
-        UNIQUE(node, target, hops_json)
-    );
-    """
-    
-    create_latency_table = """
-    CREATE TABLE IF NOT EXISTS Latency (
-        id INTEGER PRIMARY KEY,
-        link_id INTEGER NOT NULL,
-        timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
-        latency_ms REAL NOT NULL,
-        FOREIGN KEY (link_id) REFERENCES Links(id)
-    );
-    """
-    
-    create_hopdetails_table = """
-    CREATE TABLE IF NOT EXISTS HopDetails (
-        id INTEGER PRIMARY KEY,
-        hop_name TEXT,
-        hop_ip TEXT,
-        hop_latency TEXT
-    );
-    """
-    
-    # Execute the SQL statements
-    cursor.execute(create_links_table)
-    cursor.execute(create_paths_table)
-    cursor.execute(create_latency_table)
-    cursor.execute(create_hopdetails_table)
-    
-    # Commit changes and close the connection
-    conn.commit()
-    conn.close()
-
-def store_traceroute(db_file, node, trace):
-    conn = sqlite3.connect(db_file)
-    cursor = conn.cursor()
-
-
-    hops_json = json.dumps(trace['hops'])
-    
-    path_ids = {}
-
-    previous_hop_ip = None
-    previous_hop_latency = None
-    for hop in trace['hops']:
-        hop_number = hop['hop_number']
-        hop_name = hop.get('hop_name')
-        hop_ip = hop.get('hop_ip')
-        hop_latency = hop.get('hop_latency')
-        link_id = None
-
-        # insert links and get their id's
-        if previous_hop_ip:
-            cursor.execute("""
-                INSERT OR IGNORE INTO Links (source_ip, destination_ip) VALUES (?, ?)
-            """, (previous_hop_ip, hop_ip))
-            cursor.execute("""
-                SELECT id FROM Links WHERE source_ip = ? AND destination_ip = ?
-            """, (previous_hop_ip, hop_ip))
-            result = cursor.fetchone()
-            link_id = result
-            path_ids[hop_number] = link_id
-        previous_hop_ip = hop_ip
-
-        # Save hop details
-        cursor.execute("INSERT INTO HopDetails (hop_name, hop_ip, hop_latency) VALUES (?, ?, ?)",
-                        (hop_name, hop_ip, hop_latency))
-        
-        # calculate link latency if possible and store it
-        if link_id and previous_hop_latency:
-            link_latency = hop_latency - previous_hop_latency
-            cursor.execute("INSERT INTO Latency (link_id, timestamp, latency_ms) VALUES (?, ?, ?)", 
-                           (link_id, trace['timestamp'], link_latency))
-
-    # make entry to "Paths" table
-    if path_ids:
-        json_path_ids = json.dumps(path_ids)
-        cursor.execute("INSERT OR IGNORE INTO Paths (node, target, hops_json) VALUES (?, ?, ?)",
-                       (node, trace['target'], json_path_ids))
-
-    conn.commit()
-    conn.close()
-
-
-def retrieve_traceroute(db_file):
-
-    retval = {}
-
-    conn = sqlite3.connect(db_file)
-    cursor = conn.cursor()
-    cursor.execute('''
-        SELECT target, hops_json
-        FROM Paths
-    ''')
-    retval['path'] = cursor.fetchall()
-
-    cursor.execute('''
-        SELECT source_ip, destination_ip
-        FROM Links
-    ''')
-    retval['links'] = cursor.fetchall()
-    conn.close()
-    return retval
-
-def generate_node_id():
-    mac = uuid.getnode()
-    mac_str = ':'.join(['{:02x}'.format((mac >> ele) & 0xff) for ele in range(0,8*6,8)][::-1])
-    
-    # Hash the MAC address using SHA-256 to generate a unique ID
-    unique_id = hashlib.sha256(mac_str.encode()).hexdigest()
-
-    return unique_id
-
-
-if __name__ == '__main__':
-
-    db_file="./traceroute.db"
-    create_tables(db_file)
-    my_id = generate_node_id()
-
-    target='1.1.1.1'
-    traceroute_output, timestamp = run_traceroute(target)
-    trace = parse_traceroute_output(traceroute_output, timestamp)
-
-    store_traceroute(db_file, my_id, trace)
-
-    print("#####")
-    print(retrieve_traceroute(db_file))
-    print("#####")
-
-exit(0)