last time, late modifications...

test.py

@@ -7,10 +7,11 @@ from sys import hash_info
 import pprint
 
 def run_traceroute(host):
+    timestamp = datetime.now().timestamp()
     result = subprocess.run(['traceroute', host], stdout=subprocess.PIPE)
-    return result.stdout.decode()
+    return result.stdout.decode(), timestamp
 
-def parse_traceroute_output(output):
+def parse_traceroute_output(output, timestamp):
 
     lines = output.strip().split('\n')
     hops = []
@@ -20,7 +21,7 @@ def parse_traceroute_output(output):
     target = output.strip().split('\n')[0].split()[2]
 
     for line in lines[1:]:
-        hop = []
+        hop = {}
         hop_info = line.split()
         hop_number = int(hop_info[0])
         hop_name = None
@@ -43,17 +44,14 @@ def parse_traceroute_output(output):
 
             # If first colum is either name or ip-address
             if count == 1:
-                print(part)
                 match = re.search(ip_regex, part)
                 if  match:
                     hop_ip = part.strip('()')
                 else:
-                    print('do ever here?')
                     hop_name = part
 
             # Second colum is ip-address first latency reading
             if count == 2:
-                print(part)
                 if re.search(ip_regex, part):
                     try:
                         _ip = ipaddress.ip_address(part.strip('()'))
@@ -65,7 +63,6 @@ def parse_traceroute_output(output):
             # reruns of the hop_name and hop_ip...
             # We only need the latency floats anymore.
             else:
-                print(part)
                 try:
                     latency = float(part)
                     latencies.append(latency)
@@ -74,12 +71,12 @@ def parse_traceroute_output(output):
 
         hop_latency = sum(latencies) / len(latencies) if latencies else None
 
-        
+        hop['timestamp'] = timestamp
-        hop.append(hop_number)
+        hop['hop_number'] = hop_number
         if not hop_name == None:
-            hop.append(hop_name)
+            hop['hop_name'] = hop_name
-        hop.append(hop_ip)
+        hop['hop_ip'] = hop_ip
-        hop.append(hop_latency)
+        hop['hop_latency'] = hop_latency
         
 
         hops.append(hop)
@@ -89,8 +86,8 @@ def parse_traceroute_output(output):
 if __name__ == '__main__':
 
     target='8.8.8.8'
-    traceroute_output = run_traceroute(target)
+    traceroute_output, timestamp = run_traceroute(target)
     
-    target, hops = parse_traceroute_output(traceroute_output)
+    target, hops = parse_traceroute_output(traceroute_output, timestamp)
     print('>> '+target)
     pprint.pprint(hops)

traceroute_collector.py

@@ -6,8 +6,11 @@ import sqlite3
 import re
 import json
 import ipaddress
+import uuid
+import hashlib
 from datetime import datetime
 
+
 def run_traceroute(host):
     timestamp = datetime.now().timestamp()
     result = subprocess.run(['traceroute', host], stdout=subprocess.PIPE)
@@ -16,6 +19,7 @@ def run_traceroute(host):
 def parse_traceroute_output(output, timestamp):
 
     lines = output.strip().split('\n')
+    trace = {}
     hops = []
 
     ip_regex = r"\((.*?)\)" # ipaddress are in ()
@@ -40,7 +44,8 @@ def parse_traceroute_output(output, timestamp):
             # source node drops or blocks icmp packages
             # We will give funny to name to hop for not answering and move on.
             if part == '*':
-                hop_name = 'hop.'+str(count)+'.'+str(target)
+                hop_name = 'unresponsive'
+                hop_ip = 'unresponsive'
                 break
 
 
@@ -73,17 +78,18 @@ def parse_traceroute_output(output, timestamp):
 
         hop_latency = sum(latencies) / len(latencies) if latencies else None
 
-        hop['timestamp'] = timestamp
         hop['hop_number'] = hop_number
         if not hop_name == None:
             hop['hop_name'] = hop_name
         hop['hop_ip'] = hop_ip
         hop['hop_latency'] = hop_latency
-        
-        hops.append(target)
         hops.append(hop)
         
-    return hops
+    trace['target'] = target
+    trace['timestamp'] = timestamp
+    trace['hops'] = hops
+
+    return trace
 
 
 def create_tables(databasefile):
@@ -104,10 +110,10 @@ def create_tables(databasefile):
     create_paths_table = """
     CREATE TABLE IF NOT EXISTS Paths (
         id INTEGER PRIMARY KEY,
-        start_ip TEXT NOT NULL,
+        node TEXT NOT NULL,
-        end_ip TEXT NOT NULL,
+        target TEXT NOT NULL,
         hops_json TEXT NOT NULL,
-        UNIQUE(start_ip, end_ip, hops_json)
+        UNIQUE(node, target, hops_json)
     );
     """
     
@@ -124,12 +130,9 @@ def create_tables(databasefile):
     create_hopdetails_table = """
     CREATE TABLE IF NOT EXISTS HopDetails (
         id INTEGER PRIMARY KEY,
-        link_id INTEGER NOT NULL,
+        hop_name TEXT,
-        ttl INTEGER,
+        hop_ip TEXT,
-        icmp_type INTEGER,
+        hop_latency TEXT
-        icmp_code INTEGER,
-        packet_loss REAL,
-        FOREIGN KEY (link_id) REFERENCES Links(id)
     );
     """
     
@@ -143,70 +146,101 @@ def create_tables(databasefile):
     conn.commit()
     conn.close()
 
-def store_traceroute(db_file, start_ip, end_ip, hops):
+def store_traceroute(db_file, node, trace):
     conn = sqlite3.connect(db_file)
     cursor = conn.cursor()
 
-    # Insert links and get their IDs
-    link_ids = []
-    for hop in hops:
-        print(hop)
-        source_ip = start_ip if not link_ids else hops[len(link_ids)-1][0]
-        destination_ip = hop[0]
-        latency = hop[1]
 
+    hops_json = json.dumps(trace['hops'])
+    
+    path_ids = {}
+
+    previous_hop_ip = None
+    previous_hop_latency = None
+    for hop in trace['hops']:
+        hop_number = hop['hop_number']
+        hop_name = hop.get('hop_name')
+        hop_ip = hop.get('hop_ip')
+        hop_latency = hop.get('hop_latency')
+        link_id = None
+
+        # insert links and get their id's
+        if previous_hop_ip:
             cursor.execute("""
                 INSERT OR IGNORE INTO Links (source_ip, destination_ip) VALUES (?, ?)
-        """, (source_ip, destination_ip))
+            """, (previous_hop_ip, hop_ip))
             cursor.execute("""
                 SELECT id FROM Links WHERE source_ip = ? AND destination_ip = ?
-        """, (source_ip, destination_ip))
+            """, (previous_hop_ip, hop_ip))
             result = cursor.fetchone()
-        if result is None:
+            link_id = result
-            print(hop)
+            path_ids[hop_number] = link_id
-            raise ValueError(f"Failed to insert of find link between {source_ip} and {destination_ip}")
+        previous_hop_ip = hop_ip
-        link_id = result[0]
-        link_ids.append(link_id)
 
-        cursor.execute("""
+        # Save hop details
-            INSERT INTO Latency (link_id, latency_ms) VALUES (?, ?)
+        cursor.execute("INSERT INTO HopDetails (hop_name, hop_ip, hop_latency) VALUES (?, ?, ?)",
-        """, (link_id, latency))
+                        (hop_name, hop_ip, hop_latency))
         
-    # Insert path
+        # calculate link latency if possible and store it
-    hops_json = json.dumps(link_ids)
+        if link_id and previous_hop_latency:
-    cursor.execute("""
+            link_latency = hop_latency - previous_hop_latency
-        INSERT INTO Paths (start_ip, end_ip, hops_json) VALUES (?, ?, ?)
+            cursor.execute("INSERT INTO Latency (link_id, timestamp, latency_ms) VALUES (?, ?, ?)", 
-    """, (start_ip, end_ip, hops_json))
+                           (link_id, trace['timestamp'], link_latency))
+
+    # make entry to "Paths" table
+    if path_ids:
+        json_path_ids = json.dumps(path_ids)
+        cursor.execute("INSERT OR IGNORE INTO Paths (node, target, hops_json) VALUES (?, ?, ?)",
+                       (node, trace['target'], json_path_ids))
 
     conn.commit()
     conn.close()
 
 
-def retrieve_traceroute():
+def retrieve_traceroute(db_file):
-    conn = sqlite3.connect('traceroute.db')
+
+    retval = {}
+
+    conn = sqlite3.connect(db_file)
     cursor = conn.cursor()
     cursor.execute('''
-        SELECT l.source_ip, l.destination_ip, ll.latency, ll.timestamp
+        SELECT target, hops_json
-        FROM link_latency ll
+        FROM Paths
-        JOIN links l ON ll.link_id = l.id
-        ORDER BY ll.timestamp
     ''')
-    rows = cursor.fetchall()
+    retval['path'] = cursor.fetchall()
+
+    cursor.execute('''
+        SELECT source_ip, destination_ip
+        FROM Links
+    ''')
+    retval['links'] = cursor.fetchall()
     conn.close()
-    return rows
+    return retval
+
+def generate_node_id():
+    mac = uuid.getnode()
+    mac_str = ':'.join(['{:02x}'.format((mac >> ele) & 0xff) for ele in range(0,8*6,8)][::-1])
+    
+    # Hash the MAC address using SHA-256 to generate a unique ID
+    unique_id = hashlib.sha256(mac_str.encode()).hexdigest()
+
+    return unique_id
+
 
 if __name__ == '__main__':
 
-    databasefile="./traceroute.db"
+    db_file="./traceroute.db"
-    create_tables(databasefile)
+    create_tables(db_file)
+    my_id = generate_node_id()
 
-
+    target='1.1.1.1'
-    target='vi.fi'
     traceroute_output, timestamp = run_traceroute(target)
-    hops = parse_traceroute_output(traceroute_output, timestamp)
+    trace = parse_traceroute_output(traceroute_output, timestamp)
+
+    store_traceroute(db_file, my_id, trace)
 
     print("#####")
-    print(hops)
+    print(retrieve_traceroute(db_file))
     print("#####")
 
 exit(0)