MrKalzu/traceroute_map
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

http-rewrite #1

Merged
MrKalzu merged 5 commits from ryyst/traceroute_map:http-rewrite into master 2024-06-02 19:21:29 +03:00
Conversation 1 Commits 5 Files Changed 15 +1796 -257
12 changed files with 1796 additions and 257 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit f1c3cf8758 - Show all commits

16
Pipfile Normal file
View File

@ -0,0 +1,16 @@
[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"
[packages]
networkx = "*"
pyvis = "*"
matplotlib = "*"
fastapi = "*"
[dev-packages]
[requires]
python_version = "3.11"
python_full_version = "3.11.8"

1343
Pipfile.lock generated Normal file
View File

File diff suppressed because it is too large Load Diff

0
app/__init__.py Executable file
View File

133
app/collector.py Executable file
View File

@ -0,0 +1,133 @@
#!/usr/bin/env python3.11
import re
import json
import ipaddress
import uuid
import hashlib
from .db import Database
def parse_traceroute_output(output, timestamp):
lines = output.strip().split('\n')
trace = {}
hops = []
ip_regex = r"\((.*?)\)" # ipaddress are in ()
target = output.strip().split('\n')[0].split()[2]
for line in lines[1:]:
hop = {}
hop_info = line.split()
hop_number = int(hop_info[0])
hop_name = None
hop_ip = None
hop_latency = None
latencies = []
#print("##### "+str(hop_info))
count = 0
for part in hop_info[1:]:
count += 1
# source node drops or blocks icmp packages
# We will give funny to name to hop for not answering and move on.
if part == '*':
hop_name = 'unresponsive'
hop_ip = 'unresponsive'
break
# If first colum is either name or ip-address
if count == 1:
match = re.search(ip_regex, part)
if match:
hop_ip = part.strip('()')
else:
hop_name = part
# Second colum is ip-address first latency reading
if count == 2:
if re.search(ip_regex, part):
try:
_ip = ipaddress.ip_address(part.strip('()'))
hop_ip = part.strip('()')
except ValueError:
pass # Ignore if it's not a valid IP address
# Rest of the input colums are either latency floats, 'ms' or
# reruns of the hop_name and hop_ip...
# We only need the latency floats anymore.
else:
try:
latency = float(part)
latencies.append(latency)
except ValueError:
pass
hop_latency = sum(latencies) / len(latencies) if latencies else None
hop['hop_number'] = hop_number
if not hop_name == None:
hop['hop_name'] = hop_name
hop['hop_ip'] = hop_ip
hop['hop_latency'] = hop_latency
hops.append(hop)
trace['target'] = target
trace['timestamp'] = timestamp
trace['hops'] = hops
return trace
def store_traceroute(node, trace):
db = Database()
#hops_json = json.dumps(trace['hops'])
path_ids = {}
previous_hop_ip = None
previous_hop_latency = None
for hop in trace['hops']:
hop_number = hop['hop_number']
hop_name = hop.get('hop_name')
hop_ip = hop.get('hop_ip')
hop_latency = hop.get('hop_latency')
link_id = None
# insert links and get their id's
if previous_hop_ip:
link_id = db.create_link(previous_hop_ip, hop_ip)
path_ids[hop_number] = link_id
previous_hop_ip = hop_ip
# Save hop details
db.create_hop(hop_name, hop_ip, hop_latency)
# calculate link latency if possible and store it
if link_id and previous_hop_latency:
link_latency = hop_latency - previous_hop_latency
db.create_latency(link_id, trace['timestamp'], link_latency)
# make entry to "Paths" table
if path_ids:
json_path_ids = json.dumps(path_ids)
db.create_path(node, trace['target'], json_path_ids)
db.end()
def generate_node_id():
mac = uuid.getnode()
mac_str = ':'.join(['{:02x}'.format((mac >> ele) & 0xff) for ele in range(0,8*6,8)][::-1])
# Hash the MAC address using SHA-256 to generate a unique ID
unique_id = hashlib.sha256(mac_str.encode()).hexdigest()
return unique_id

182
app/db.py Normal file
View File

@ -0,0 +1,182 @@
import sqlite3
from functools import wraps
# Type alias
Cursor = sqlite3.Cursor
# Configs
DB_FILE = "./traceroute.db"
class Database:
def __init__(self):
self.db_file = DB_FILE
self.conn = sqlite3.connect(self.db_file, check_same_thread=False)
self.cursor = self.conn.cursor()
def create_tables(self):
self.cursor.executescript("""
CREATE TABLE IF NOT EXISTS Links (
id INTEGER PRIMARY KEY,
source_ip TEXT NOT NULL,
destination_ip TEXT NOT NULL,
UNIQUE(source_ip, destination_ip)
);
CREATE TABLE IF NOT EXISTS Paths (
id INTEGER PRIMARY KEY,
node TEXT NOT NULL,
target TEXT NOT NULL,
hops_json TEXT NOT NULL,
UNIQUE(node, target, hops_json)
);
CREATE TABLE IF NOT EXISTS Latency (
id INTEGER PRIMARY KEY,
link_id INTEGER NOT NULL,
timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
latency_ms REAL NOT NULL,
FOREIGN KEY (link_id) REFERENCES Links(id)
);
CREATE TABLE IF NOT EXISTS HopDetails (
id INTEGER PRIMARY KEY,
hop_name TEXT,
hop_ip TEXT,
hop_latency TEXT
);
""")
def end(self):
"""Always call this after you're done with the connection / request."""
self.conn.commit()
self.conn.close()
def get_traceroute(self):
retval = {}
self.cursor.execute('''
SELECT target, hops_json
FROM Paths
''')
retval['path'] = self.cursor.fetchall()
self.cursor.execute('''
SELECT source_ip, destination_ip
FROM Links
''')
retval['links'] = self.cursor.fetchall()
return retval
def create_link(self, previous_hop_ip, hop_ip):
self.cursor.execute(
"INSERT OR IGNORE INTO Links (source_ip, destination_ip) VALUES (?, ?)",
(previous_hop_ip, hop_ip)
)
self.cursor.execute(
"SELECT id FROM Links WHERE source_ip = ? AND destination_ip = ?",
(previous_hop_ip, hop_ip)
)
return self.cursor.fetchone()
def create_hop(self, name, ip, latency):
self.cursor.execute(
"INSERT INTO HopDetails (hop_name, hop_ip, hop_latency) VALUES (?, ?, ?)",
(name, ip, latency)
)
def create_latency(self, link_id, timestamp, link_latency):
self.cursor.execute(
"INSERT INTO Latency (link_id, timestamp, latency_ms) VALUES (?, NOW(), ?)",
(link_id, timestamp, link_latency)
)
def create_path(self, node, target, json):
self.cursor.execute(
"INSERT OR IGNORE INTO Paths (node, target, hops_json) VALUES (?, ?, ?)",
(node, target, json)
)
def ensure_table_setup():
db = Database()
db.create_tables()
db.end()
####################################################################
####################################################################
####################################################################
####################################################################
# Temp testing. Fancy decorator stuff.
def with_connection(func):
@wraps(func)
def wrapped(*args, **kwargs):
conn = sqlite3.connect(DB_FILE)
cursor = conn.cursor()
result = func(cursor, *args, **kwargs)
conn.commit()
conn.close()
return result
return wrapped
@with_connection
def init_db(cursor: Cursor):
cursor.executescript("""
CREATE TABLE IF NOT EXISTS Links (
id INTEGER PRIMARY KEY,
source_ip TEXT NOT NULL,
destination_ip TEXT NOT NULL,
UNIQUE(source_ip, destination_ip)
);
CREATE TABLE IF NOT EXISTS Paths (
id INTEGER PRIMARY KEY,
node TEXT NOT NULL,
target TEXT NOT NULL,
hops_json TEXT NOT NULL,
UNIQUE(node, target, hops_json)
);
CREATE TABLE IF NOT EXISTS Latency (
id INTEGER PRIMARY KEY,
link_id INTEGER NOT NULL,
timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
latency_ms REAL NOT NULL,
FOREIGN KEY (link_id) REFERENCES Links(id)
);
CREATE TABLE IF NOT EXISTS HopDetails (
id INTEGER PRIMARY KEY,
hop_name TEXT,
hop_ip TEXT,
hop_latency TEXT
);
""")
@with_connection
def create_link(cursor: Cursor, previous_hop_ip: str, hop_ip: str):
"""Insert a new hop and return related Link id"""
cursor.execute("""
INSERT OR IGNORE INTO Links (source_ip, destination_ip) VALUES (?, ?)
""", (previous_hop_ip, hop_ip))
cursor.execute("""
SELECT id FROM Links WHERE source_ip = ? AND destination_ip = ?
""", (previous_hop_ip, hop_ip))
return cursor.fetchone()

55
app/main.py Executable file
View File

@ -0,0 +1,55 @@
from fastapi import Request, FastAPI
from fastapi.staticfiles import StaticFiles
from datetime import datetime
from .collector import parse_traceroute_output, store_traceroute
from .db import ensure_table_setup
from pprint import pprint as print
# Setup our SQLite before anything else.
ensure_table_setup()
# Setup web framework thingies
app = FastAPI()
app.mount("/static", StaticFiles(directory="static"), name="static")
@app.get("/")
def read_root():
return {
"greeting": "Hello, Kalzu!",
"instructions": [
"Try piping traceroute data directly to POST /trace/{hostname}.",
"{hostname} is for filtering data by sender.",
"For example the following command using HTTPie:",
"",
" $ traceroute peitto.info | http POST localhost:8000/trace/rekku",
"",
"",
"Also take a look at /docs/ and /static/index.html",
"",
"",
"",
"END OF TRANSMISSION",
] + [None]*800
}
@app.post("/trace/{hostname}")
async def create_trace(hostname: str, request: Request):
raw_data = await request.body()
data = raw_data.decode("utf-8", "ignore")
print(f"Received data from {hostname}:")
print(data)
trace = parse_traceroute_output(data, datetime.now())
print("Parsed data:")
print(trace)
store_traceroute(hostname, trace)
return {"status": "ok" }

0
netx_path_test.py → app/netx_path_test.py
View File

56
app/static/index.html Normal file
View File

@ -0,0 +1,56 @@
<!DOCTYPE html>
<html lang="en">
<head>
<title>Kalzu</title>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<script src="https://cdnjs.cloudflare.com/ajax/libs/sigma.js/2.4.0/sigma.min.js" integrity="sha512-iiPEYww3QXZU5C795JnnINBRNgHqDnRHs9mA7aJoqx4pNE4u3CknCDGmePHFoHtKR/6C9aIcRFa+HJ6obtlteQ==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/graphology/0.25.4/graphology.umd.min.js" integrity="sha512-tjMBhL9fLMcqoccPOwpRiIQIOAyUh18lWUlUvE10zvG1UNMfxUC4qSERmUq+VF30iavIyqs/q6fSP2o475FAUw==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
</head>
<body style="background: lightgrey">
<div id="container" style="width: 97%; height: 95vh; background: white; margin: auto; margin-top: 32px"></div>
<script>
const hops = [
["_gateway", "(192.168.0.1) 0.549 ms"],
["83-148-246-178.static.lounea.fi", "(83.148.246.178) 1.200 ms"],
["*"],
["*"],
["*"],
["86-60-254-198.static.lounea.fi", "(86.60.254.198) 3.719 ms"],
["72.14.194.142", " (72.14.194.142) 3.716 ms"],
["142.251.53.71", " (142.251.53.71) 3.704 ms"],
["142.250.229.87", " (142.250.229.87) 4.603 ms"],
["mad06s09-in-f142.1e100.net", " (216.58.210.142) 3.700 ms"],
];
console.log("Hops!", hops)
// Create a graphology graph
const graph = new graphology.Graph();
hops.forEach((hop, i) => {
const [ label, extra ] = hop;
const size = i === 0 ? 15 : 10;
const color = "green"
graph.addNode(i, { label, x: 0, y: i, size, color });
if (i > 0) {
graph.addEdge(i, i-1, { size: 2, color: "purple" });
}
});
// graph.addNode("1", { label: "Node 1", x: 0, y: 0, size: 10, color: "blue" });
// graph.addNode("2", { label: "Node 2", x: 1, y: 1, size: 20, color: "red" });
// graph.addEdge("1", "2", { size: 5, color: "purple" });
// Instantiate sigma.js and render the graph
const sigmaInstance = new Sigma(graph, document.getElementById("container"));
</script>
</body>
</html>
</html>

0
test.py → app/test.py
View File

BIN
app/traceroute.db Normal file
View File

Binary file not shown.

6
traceroute_visualization.py → app/traceroute_visualization.py
View File

@ -9,8 +9,8 @@ def retrieve_traceroute_data():
conn = sqlite3.connect('traceroute.db')
cursor = conn.cursor()
cursor.execute('''
SELECT l.source_ip, l.destination_ip, ll.latency, ll.timestamp
FROM link_latency ll
SELECT l.source_ip, l.destination_ip, ll.latency_ms, ll.timestamp
FROM Latency ll
JOIN links l ON ll.link_id = l.id
ORDER BY ll.timestamp
''')
@ -64,4 +64,4 @@ if __name__ == '__main__':
# Visualize the graph using pyvis
#visualize_graph_pyvis(graph, 'traceroute_network.html')
exit(0)
#exit(0)

246
traceroute_collector.py
View File

@ -1,246 +0,0 @@
#!/usr/bin/env python3.11
import subprocess
import sqlite3
import re
import json
import ipaddress
import uuid
import hashlib
from datetime import datetime
def run_traceroute(host):
timestamp = datetime.now().timestamp()
result = subprocess.run(['traceroute', host], stdout=subprocess.PIPE)
return result.stdout.decode(), timestamp
def parse_traceroute_output(output, timestamp):
lines = output.strip().split('\n')
trace = {}
hops = []
ip_regex = r"\((.*?)\)" # ipaddress are in ()
target = output.strip().split('\n')[0].split()[2]
for line in lines[1:]:
hop = {}
hop_info = line.split()
hop_number = int(hop_info[0])
hop_name = None
hop_ip = None
hop_latency = None
latencies = []
#print("##### "+str(hop_info))
count = 0
for part in hop_info[1:]:
count += 1
# source node drops or blocks icmp packages
# We will give funny to name to hop for not answering and move on.
if part == '*':
hop_name = 'unresponsive'
hop_ip = 'unresponsive'
break
# If first colum is either name or ip-address
if count == 1:
match = re.search(ip_regex, part)
if match:
hop_ip = part.strip('()')
else:
hop_name = part
# Second colum is ip-address first latency reading
if count == 2:
if re.search(ip_regex, part):
try:
_ip = ipaddress.ip_address(part.strip('()'))
hop_ip = part.strip('()')
except ValueError:
pass # Ignore if it's not a valid IP address
# Rest of the input colums are either latency floats, 'ms' or
# reruns of the hop_name and hop_ip...
# We only need the latency floats anymore.
else:
try:
latency = float(part)
latencies.append(latency)
except ValueError:
pass
hop_latency = sum(latencies) / len(latencies) if latencies else None
hop['hop_number'] = hop_number
if not hop_name == None:
hop['hop_name'] = hop_name
hop['hop_ip'] = hop_ip
hop['hop_latency'] = hop_latency
hops.append(hop)
trace['target'] = target
trace['timestamp'] = timestamp
trace['hops'] = hops
return trace
def create_tables(databasefile):
# Connect to the SQLite database
conn = sqlite3.connect(databasefile)
cursor = conn.cursor()
# SQL statements to create the tables
create_links_table = """
CREATE TABLE IF NOT EXISTS Links (
id INTEGER PRIMARY KEY,
source_ip TEXT NOT NULL,
destination_ip TEXT NOT NULL,
UNIQUE(source_ip, destination_ip)
);
"""
create_paths_table = """
CREATE TABLE IF NOT EXISTS Paths (
id INTEGER PRIMARY KEY,
node TEXT NOT NULL,
target TEXT NOT NULL,
hops_json TEXT NOT NULL,
UNIQUE(node, target, hops_json)
);
"""
create_latency_table = """
CREATE TABLE IF NOT EXISTS Latency (
id INTEGER PRIMARY KEY,
link_id INTEGER NOT NULL,
timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
latency_ms REAL NOT NULL,
FOREIGN KEY (link_id) REFERENCES Links(id)
);
"""
create_hopdetails_table = """
CREATE TABLE IF NOT EXISTS HopDetails (
id INTEGER PRIMARY KEY,
hop_name TEXT,
hop_ip TEXT,
hop_latency TEXT
);
"""
# Execute the SQL statements
cursor.execute(create_links_table)
cursor.execute(create_paths_table)
cursor.execute(create_latency_table)
cursor.execute(create_hopdetails_table)
# Commit changes and close the connection
conn.commit()
conn.close()
def store_traceroute(db_file, node, trace):
conn = sqlite3.connect(db_file)
cursor = conn.cursor()
hops_json = json.dumps(trace['hops'])
path_ids = {}
previous_hop_ip = None
previous_hop_latency = None
for hop in trace['hops']:
hop_number = hop['hop_number']
hop_name = hop.get('hop_name')
hop_ip = hop.get('hop_ip')
hop_latency = hop.get('hop_latency')
link_id = None
# insert links and get their id's
if previous_hop_ip:
cursor.execute("""
INSERT OR IGNORE INTO Links (source_ip, destination_ip) VALUES (?, ?)
""", (previous_hop_ip, hop_ip))
cursor.execute("""
SELECT id FROM Links WHERE source_ip = ? AND destination_ip = ?
""", (previous_hop_ip, hop_ip))
result = cursor.fetchone()
link_id = result
path_ids[hop_number] = link_id
previous_hop_ip = hop_ip
# Save hop details
cursor.execute("INSERT INTO HopDetails (hop_name, hop_ip, hop_latency) VALUES (?, ?, ?)",
(hop_name, hop_ip, hop_latency))
# calculate link latency if possible and store it
if link_id and previous_hop_latency:
link_latency = hop_latency - previous_hop_latency
cursor.execute("INSERT INTO Latency (link_id, timestamp, latency_ms) VALUES (?, ?, ?)",
(link_id, trace['timestamp'], link_latency))
# make entry to "Paths" table
if path_ids:
json_path_ids = json.dumps(path_ids)
cursor.execute("INSERT OR IGNORE INTO Paths (node, target, hops_json) VALUES (?, ?, ?)",
(node, trace['target'], json_path_ids))
conn.commit()
conn.close()
def retrieve_traceroute(db_file):
retval = {}
conn = sqlite3.connect(db_file)
cursor = conn.cursor()
cursor.execute('''
SELECT target, hops_json
FROM Paths
''')
retval['path'] = cursor.fetchall()
cursor.execute('''
SELECT source_ip, destination_ip
FROM Links
''')
retval['links'] = cursor.fetchall()
conn.close()
return retval
def generate_node_id():
mac = uuid.getnode()
mac_str = ':'.join(['{:02x}'.format((mac >> ele) & 0xff) for ele in range(0,8*6,8)][::-1])
# Hash the MAC address using SHA-256 to generate a unique ID
unique_id = hashlib.sha256(mac_str.encode()).hexdigest()
return unique_id
if __name__ == '__main__':
db_file="./traceroute.db"
create_tables(db_file)
my_id = generate_node_id()
target='1.1.1.1'
traceroute_output, timestamp = run_traceroute(target)
trace = parse_traceroute_output(traceroute_output, timestamp)
store_traceroute(db_file, my_id, trace)
print("#####")
print(retrieve_traceroute(db_file))
print("#####")
exit(0)