last time, late modifications...

test.py

@@ -0,0 +1,93 @@
+import subprocess
+import re
+import json
+import ipaddress
+from datetime import datetime
+from sys import hash_info
+import pprint
+
+def run_traceroute(host):
+    timestamp = datetime.now().timestamp()
+    result = subprocess.run(['traceroute', host], stdout=subprocess.PIPE)
+    return result.stdout.decode(), timestamp
+
+def parse_traceroute_output(output, timestamp):
+
+    lines = output.strip().split('\n')
+    hops = []
+
+    ip_regex = r"\((.*?)\)" # ipaddress are in ()
+
+    target = output.strip().split('\n')[0].split()[2]
+
+    for line in lines[1:]:
+        hop = {}
+        hop_info = line.split()
+        hop_number = int(hop_info[0])
+        hop_name = None
+        hop_ip = None
+        hop_latency = None
+
+        latencies = []
+
+        #print("##### "+str(hop_info))
+        count = 0
+        for part in hop_info[1:]:
+            count += 1
+
+            # source node drops or blocks icmp packages
+            # We will give funny to name to hop for not answering and move on.
+            if part == '*':
+                hop_name = 'hop.'+str(count)+'.'+str(target)
+                break
+
+
+            # If first colum is either name or ip-address
+            if count == 1:
+                match = re.search(ip_regex, part)
+                if  match:
+                    hop_ip = part.strip('()')
+                else:
+                    hop_name = part
+
+            # Second colum is ip-address first latency reading
+            if count == 2:
+                if re.search(ip_regex, part):
+                    try:
+                        _ip = ipaddress.ip_address(part.strip('()'))
+                        hop_ip = part.strip('()')
+                    except ValueError:
+                        pass  # Ignore if it's not a valid IP address
+
+            # Rest of the input colums are either latency floats, 'ms' or 
+            # reruns of the hop_name and hop_ip...
+            # We only need the latency floats anymore.
+            else:
+                try:
+                    latency = float(part)
+                    latencies.append(latency)
+                except ValueError:
+                    pass
+
+        hop_latency = sum(latencies) / len(latencies) if latencies else None
+
+        hop['timestamp'] = timestamp
+        hop['hop_number'] = hop_number
+        if not hop_name == None:
+            hop['hop_name'] = hop_name
+        hop['hop_ip'] = hop_ip
+        hop['hop_latency'] = hop_latency
+        
+
+        hops.append(hop)
+
+    return target, hops
+
+if __name__ == '__main__':
+
+    target='8.8.8.8'
+    traceroute_output, timestamp = run_traceroute(target)
+    
+    target, hops = parse_traceroute_output(traceroute_output, timestamp)
+    print('>> '+target)
+    pprint.pprint(hops)

traceroute_collector.py

@@ -4,123 +4,243 @@
 import subprocess
 import sqlite3
 import re
+import json
+import ipaddress
+import uuid
+import hashlib
 from datetime import datetime
 
+
 def run_traceroute(host):
+    timestamp = datetime.now().timestamp()
     result = subprocess.run(['traceroute', host], stdout=subprocess.PIPE)
-    return result.stdout.decode()
+    return result.stdout.decode(), timestamp
 
-def parse_traceroute_output(output):
+def parse_traceroute_output(output, timestamp):
+
+    lines = output.strip().split('\n')
+    trace = {}
     hops = []
-    lines = output.strip().split('\n')[1:]  # Skip the first line (traceroute to ...)
+
-    previous_ip = None
+    ip_regex = r"\((.*?)\)" # ipaddress are in ()
-    for line in lines:
+
-        parts = line.split()
+    target = output.strip().split('\n')[0].split()[2]
-        hop_number = int(parts[0])
+
-        ip_address = parts[1]
+    for line in lines[1:]:
+        hop = {}
+        hop_info = line.split()
+        hop_number = int(hop_info[0])
+        hop_name = None
+        hop_ip = None
+        hop_latency = None
+
         latencies = []
-        for part in parts[2:]:
+
-            if re.match(r'^[0-9+\s+ms$]', part):
+        #print("##### "+str(hop_info))
-                latency_str = re.sub(r'[^0-9.]', '', part)
+        count = 0
-                if latency_str and not latency_str == '':
+        for part in hop_info[1:]:
+            count += 1
+
+            # source node drops or blocks icmp packages
+            # We will give funny to name to hop for not answering and move on.
+            if part == '*':
+                hop_name = 'unresponsive'
+                hop_ip = 'unresponsive'
+                break
+
+
+            # If first colum is either name or ip-address
+            if count == 1:
+                match = re.search(ip_regex, part)
+                if  match:
+                    hop_ip = part.strip('()')
+                else:
+                    hop_name = part
+
+            # Second colum is ip-address first latency reading
+            if count == 2:
+                if re.search(ip_regex, part):
                     try:
-                        print(part)
+                        _ip = ipaddress.ip_address(part.strip('()'))
-                        print(latency_str)
+                        hop_ip = part.strip('()')
-                        latencies.append(float(latency_str))
                     except ValueError:
-                        print(f"Could not convert '{latency_str}' to float.")
+                        pass  # Ignore if it's not a valid IP address
 
-        avg_latency = sum(latencies) / len(latencies) if latencies else None
+            # Rest of the input colums are either latency floats, 'ms' or 
-        timestamp = datetime.now().isoformat()
+            # reruns of the hop_name and hop_ip...
-        if previous_ip:
+            # We only need the latency floats anymore.
-            hops.append({
+            else:
-                'hop_number': hop_number,
+                try:
-                'source_ip': previous_ip,
+                    latency = float(part)
-                'destination_ip': ip_address,
+                    latencies.append(latency)
-                'latency': avg_latency,
+                except ValueError:
-                'timestamp': timestamp,
+                    pass
-            })
-        previous_ip = ip_address
-    return hops
 
-def create_tables():
+        hop_latency = sum(latencies) / len(latencies) if latencies else None
-    conn = sqlite3.connect('traceroute.db')
+
+        hop['hop_number'] = hop_number
+        if not hop_name == None:
+            hop['hop_name'] = hop_name
+        hop['hop_ip'] = hop_ip
+        hop['hop_latency'] = hop_latency
+        hops.append(hop)
+        
+    trace['target'] = target
+    trace['timestamp'] = timestamp
+    trace['hops'] = hops
+
+    return trace
+
+
+def create_tables(databasefile):
+        # Connect to the SQLite database
+    conn = sqlite3.connect(databasefile)
     cursor = conn.cursor()
     
-    # Table to store unique links
+    # SQL statements to create the tables
-    cursor.execute('''
+    create_links_table = """
-        CREATE TABLE IF NOT EXISTS links (
+    CREATE TABLE IF NOT EXISTS Links (
         id INTEGER PRIMARY KEY,
-            source_ip TEXT,
+        source_ip TEXT NOT NULL,
-            destination_ip TEXT,
+        destination_ip TEXT NOT NULL,
         UNIQUE(source_ip, destination_ip)
-        )
+    );
-    ''')
+    """
     
-    # Table to store latency and timestamp for each link
+    create_paths_table = """
-    cursor.execute('''
+    CREATE TABLE IF NOT EXISTS Paths (
-        CREATE TABLE IF NOT EXISTS link_latency (
         id INTEGER PRIMARY KEY,
-            link_id INTEGER,
+        node TEXT NOT NULL,
-            latency REAL,
+        target TEXT NOT NULL,
-            timestamp TEXT,
+        hops_json TEXT NOT NULL,
-            FOREIGN KEY (link_id) REFERENCES links (id)
+        UNIQUE(node, target, hops_json)
-        )
+    );
-    ''')
+    """
+    
+    create_latency_table = """
+    CREATE TABLE IF NOT EXISTS Latency (
+        id INTEGER PRIMARY KEY,
+        link_id INTEGER NOT NULL,
+        timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
+        latency_ms REAL NOT NULL,
+        FOREIGN KEY (link_id) REFERENCES Links(id)
+    );
+    """
+    
+    create_hopdetails_table = """
+    CREATE TABLE IF NOT EXISTS HopDetails (
+        id INTEGER PRIMARY KEY,
+        hop_name TEXT,
+        hop_ip TEXT,
+        hop_latency TEXT
+    );
+    """
+    
+    # Execute the SQL statements
+    cursor.execute(create_links_table)
+    cursor.execute(create_paths_table)
+    cursor.execute(create_latency_table)
+    cursor.execute(create_hopdetails_table)
+    
+    # Commit changes and close the connection
+    conn.commit()
+    conn.close()
+
+def store_traceroute(db_file, node, trace):
+    conn = sqlite3.connect(db_file)
+    cursor = conn.cursor()
+
+
+    hops_json = json.dumps(trace['hops'])
+    
+    path_ids = {}
+
+    previous_hop_ip = None
+    previous_hop_latency = None
+    for hop in trace['hops']:
+        hop_number = hop['hop_number']
+        hop_name = hop.get('hop_name')
+        hop_ip = hop.get('hop_ip')
+        hop_latency = hop.get('hop_latency')
+        link_id = None
+
+        # insert links and get their id's
+        if previous_hop_ip:
+            cursor.execute("""
+                INSERT OR IGNORE INTO Links (source_ip, destination_ip) VALUES (?, ?)
+            """, (previous_hop_ip, hop_ip))
+            cursor.execute("""
+                SELECT id FROM Links WHERE source_ip = ? AND destination_ip = ?
+            """, (previous_hop_ip, hop_ip))
+            result = cursor.fetchone()
+            link_id = result
+            path_ids[hop_number] = link_id
+        previous_hop_ip = hop_ip
+
+        # Save hop details
+        cursor.execute("INSERT INTO HopDetails (hop_name, hop_ip, hop_latency) VALUES (?, ?, ?)",
+                        (hop_name, hop_ip, hop_latency))
+        
+        # calculate link latency if possible and store it
+        if link_id and previous_hop_latency:
+            link_latency = hop_latency - previous_hop_latency
+            cursor.execute("INSERT INTO Latency (link_id, timestamp, latency_ms) VALUES (?, ?, ?)", 
+                           (link_id, trace['timestamp'], link_latency))
+
+    # make entry to "Paths" table
+    if path_ids:
+        json_path_ids = json.dumps(path_ids)
+        cursor.execute("INSERT OR IGNORE INTO Paths (node, target, hops_json) VALUES (?, ?, ?)",
+                       (node, trace['target'], json_path_ids))
 
     conn.commit()
     conn.close()
 
-def store_traceroute(hops):
-    conn = sqlite3.connect('traceroute.db')
-    cursor = conn.cursor()
 
-    for hop in hops:
+def retrieve_traceroute(db_file):
-        # Insert or ignore link into links table
-        cursor.execute('''
-            INSERT OR IGNORE INTO links (source_ip, destination_ip)
-            VALUES (?, ?)
-        ''', (hop['source_ip'], hop['destination_ip']))
 
-        # Retrieve the link_id
+    retval = {}
-        cursor.execute('''
-            SELECT id FROM links WHERE source_ip = ? AND destination_ip = ?
-        ''', (hop['source_ip'], hop['destination_ip']))
-        link_id = cursor.fetchone()[0]
 
-        # Insert latency data into link_latency table
+    conn = sqlite3.connect(db_file)
-        cursor.execute('''
-            INSERT INTO link_latency (link_id, latency, timestamp)
-            VALUES (?, ?, ?)
-        ''', (link_id, hop['latency'], hop['timestamp']))
-    
-    conn.commit()
-    conn.close()
-
-def retrieve_traceroute():
-    conn = sqlite3.connect('traceroute.db')
     cursor = conn.cursor()
     cursor.execute('''
-        SELECT l.source_ip, l.destination_ip, ll.latency, ll.timestamp
+        SELECT target, hops_json
-        FROM link_latency ll
+        FROM Paths
-        JOIN links l ON ll.link_id = l.id
-        ORDER BY ll.timestamp
     ''')
-    rows = cursor.fetchall()
+    retval['path'] = cursor.fetchall()
-    conn.close()
+
-    return rows
+    cursor.execute('''
+        SELECT source_ip, destination_ip
+        FROM Links
+    ''')
+    retval['links'] = cursor.fetchall()
+    conn.close()
+    return retval
+
+def generate_node_id():
+    mac = uuid.getnode()
+    mac_str = ':'.join(['{:02x}'.format((mac >> ele) & 0xff) for ele in range(0,8*6,8)][::-1])
+    
+    # Hash the MAC address using SHA-256 to generate a unique ID
+    unique_id = hashlib.sha256(mac_str.encode()).hexdigest()
+
+    return unique_id
+
 
-# Usage
 if __name__ == '__main__':
-    create_tables()
-    traceroute_output = run_traceroute('vi.fi')
-    hops = parse_traceroute_output(traceroute_output)
-#    for hop in hops:
-#        print(hop)
-    store_traceroute(hops)
-    stored_hops = retrieve_traceroute()
-    for hop in stored_hops:
-        print(f"Link: {hop[0]} -> {hop[1]}, Latency: {hop[2]} ms, Timestamp: {hop[3]}")
 
+    db_file="./traceroute.db"
+    create_tables(db_file)
+    my_id = generate_node_id()
+
+    target='1.1.1.1'
+    traceroute_output, timestamp = run_traceroute(target)
+    trace = parse_traceroute_output(traceroute_output, timestamp)
+
+    store_traceroute(db_file, my_id, trace)
+
+    print("#####")
+    print(retrieve_traceroute(db_file))
+    print("#####")
 
 exit(0)