MrKalzu/traceroute_map
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

TEMP: Rewrite everything on top of FastAPI

Browse Source
This commit is contained in:
ryyst 2024-05-31 23:13:38 +03:00
parent 929a3523ba
commit f1c3cf8758
12 changed files with 1796 additions and 257 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
Pipfile Normal file
View File

@ -0,0 +1,16 @@
[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"
[packages]
networkx = "*"
pyvis = "*"
matplotlib = "*"
fastapi = "*"
[dev-packages]
[requires]
python_version = "3.11"
python_full_version = "3.11.8"

1343
Pipfile.lock generated Normal file
View File

File diff suppressed because it is too large Load Diff

0
app/__init__.py Executable file
View File

133
app/collector.py Executable file
View File

@ -0,0 +1,133 @@
#!/usr/bin/env python3.11
import re
import json
import ipaddress
import uuid
import hashlib
from .db import Database
def parse_traceroute_output(output, timestamp):
lines = output.strip().split('\n')
trace = {}
hops = []
ip_regex = r"\((.*?)\)" # ipaddress are in ()
target = output.strip().split('\n')[0].split()[2]
for line in lines[1:]:
hop = {}
hop_info = line.split()
hop_number = int(hop_info[0])
hop_name = None
hop_ip = None
hop_latency = None
latencies = []
#print("##### "+str(hop_info))
count = 0
for part in hop_info[1:]:
count += 1
# source node drops or blocks icmp packages
# We will give funny to name to hop for not answering and move on.
if part == '*':
hop_name = 'unresponsive'
hop_ip = 'unresponsive'
break
# If first colum is either name or ip-address
if count == 1:
match = re.search(ip_regex, part)
if match:
hop_ip = part.strip('()')
else:
hop_name = part
# Second colum is ip-address first latency reading
if count == 2:
if re.search(ip_regex, part):
try:
_ip = ipaddress.ip_address(part.strip('()'))
hop_ip = part.strip('()')
except ValueError:
pass # Ignore if it's not a valid IP address
# Rest of the input colums are either latency floats, 'ms' or
# reruns of the hop_name and hop_ip...
# We only need the latency floats anymore.
else:
try:
latency = float(part)
latencies.append(latency)
except ValueError:
pass
hop_latency = sum(latencies) / len(latencies) if latencies else None
hop['hop_number'] = hop_number
if not hop_name == None:
hop['hop_name'] = hop_name
hop['hop_ip'] = hop_ip
hop['hop_latency'] = hop_latency
hops.append(hop)
trace['target'] = target
trace['timestamp'] = timestamp
trace['hops'] = hops
return trace
def store_traceroute(node, trace):
db = Database()
#hops_json = json.dumps(trace['hops'])
path_ids = {}
previous_hop_ip = None
previous_hop_latency = None
for hop in trace['hops']:
hop_number = hop['hop_number']
hop_name = hop.get('hop_name')
hop_ip = hop.get('hop_ip')
hop_latency = hop.get('hop_latency')
link_id = None
# insert links and get their id's
if previous_hop_ip:
link_id = db.create_link(previous_hop_ip, hop_ip)
path_ids[hop_number] = link_id
previous_hop_ip = hop_ip
# Save hop details
db.create_hop(hop_name, hop_ip, hop_latency)
# calculate link latency if possible and store it
if link_id and previous_hop_latency:
link_latency = hop_latency - previous_hop_latency
db.create_latency(link_id, trace['timestamp'], link_latency)
# make entry to "Paths" table
if path_ids:
json_path_ids = json.dumps(path_ids)
db.create_path(node, trace['target'], json_path_ids)
db.end()
def generate_node_id():
mac = uuid.getnode()
mac_str = ':'.join(['{:02x}'.format((mac >> ele) & 0xff) for ele in range(0,8*6,8)][::-1])
# Hash the MAC address using SHA-256 to generate a unique ID
unique_id = hashlib.sha256(mac_str.encode()).hexdigest()
return unique_id

182
app/db.py Normal file
View File

@ -0,0 +1,182 @@
import sqlite3
from functools import wraps
# Type alias
Cursor = sqlite3.Cursor
# Configs
DB_FILE = "./traceroute.db"
class Database:
def __init__(self):
self.db_file = DB_FILE
self.conn = sqlite3.connect(self.db_file, check_same_thread=False)
self.cursor = self.conn.cursor()
def create_tables(self):
self.cursor.executescript("""
CREATE TABLE IF NOT EXISTS Links (
id INTEGER PRIMARY KEY,
source_ip TEXT NOT NULL,
destination_ip TEXT NOT NULL,
UNIQUE(source_ip, destination_ip)
);
CREATE TABLE IF NOT EXISTS Paths (
id INTEGER PRIMARY KEY,
node TEXT NOT NULL,
target TEXT NOT NULL,
hops_json TEXT NOT NULL,
UNIQUE(node, target, hops_json)
);
CREATE TABLE IF NOT EXISTS Latency (
id INTEGER PRIMARY KEY,
link_id INTEGER NOT NULL,
timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
latency_ms REAL NOT NULL,
FOREIGN KEY (link_id) REFERENCES Links(id)
);
CREATE TABLE IF NOT EXISTS HopDetails (
id INTEGER PRIMARY KEY,
hop_name TEXT,
hop_ip TEXT,
hop_latency TEXT
);
""")
def end(self):
"""Always call this after you're done with the connection / request."""
self.conn.commit()
self.conn.close()
def get_traceroute(self):
retval = {}
self.cursor.execute('''
SELECT target, hops_json
FROM Paths
''')
retval['path'] = self.cursor.fetchall()
self.cursor.execute('''
SELECT source_ip, destination_ip
FROM Links
''')
retval['links'] = self.cursor.fetchall()
return retval
def create_link(self, previous_hop_ip, hop_ip):
self.cursor.execute(
"INSERT OR IGNORE INTO Links (source_ip, destination_ip) VALUES (?, ?)",
(previous_hop_ip, hop_ip)
)
self.cursor.execute(
"SELECT id FROM Links WHERE source_ip = ? AND destination_ip = ?",
(previous_hop_ip, hop_ip)
)
return self.cursor.fetchone()
def create_hop(self, name, ip, latency):
self.cursor.execute(
"INSERT INTO HopDetails (hop_name, hop_ip, hop_latency) VALUES (?, ?, ?)",
(name, ip, latency)
)
def create_latency(self, link_id, timestamp, link_latency):
self.cursor.execute(
"INSERT INTO Latency (link_id, timestamp, latency_ms) VALUES (?, NOW(), ?)",
(link_id, timestamp, link_latency)
)
def create_path(self, node, target, json):
self.cursor.execute(
"INSERT OR IGNORE INTO Paths (node, target, hops_json) VALUES (?, ?, ?)",
(node, target, json)
)
def ensure_table_setup():
db = Database()
db.create_tables()
db.end()
####################################################################
####################################################################
####################################################################
####################################################################
# Temp testing. Fancy decorator stuff.
def with_connection(func):
@wraps(func)
def wrapped(*args, **kwargs):
conn = sqlite3.connect(DB_FILE)
cursor = conn.cursor()
result = func(cursor, *args, **kwargs)
conn.commit()
conn.close()
return result
return wrapped
@with_connection
def init_db(cursor: Cursor):
cursor.executescript("""
CREATE TABLE IF NOT EXISTS Links (
id INTEGER PRIMARY KEY,
source_ip TEXT NOT NULL,
destination_ip TEXT NOT NULL,
UNIQUE(source_ip, destination_ip)
);
CREATE TABLE IF NOT EXISTS Paths (
id INTEGER PRIMARY KEY,
node TEXT NOT NULL,
target TEXT NOT NULL,
hops_json TEXT NOT NULL,
UNIQUE(node, target, hops_json)
);
CREATE TABLE IF NOT EXISTS Latency (
id INTEGER PRIMARY KEY,
link_id INTEGER NOT NULL,
timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
latency_ms REAL NOT NULL,
FOREIGN KEY (link_id) REFERENCES Links(id)
);
CREATE TABLE IF NOT EXISTS HopDetails (
id INTEGER PRIMARY KEY,
hop_name TEXT,
hop_ip TEXT,
hop_latency TEXT
);
""")
@with_connection
def create_link(cursor: Cursor, previous_hop_ip: str, hop_ip: str):
"""Insert a new hop and return related Link id"""
cursor.execute("""
INSERT OR IGNORE INTO Links (source_ip, destination_ip) VALUES (?, ?)
""", (previous_hop_ip, hop_ip))
cursor.execute("""
SELECT id FROM Links WHERE source_ip = ? AND destination_ip = ?
""", (previous_hop_ip, hop_ip))
return cursor.fetchone()

55
app/main.py Executable file
View File

@ -0,0 +1,55 @@
from fastapi import Request, FastAPI
from fastapi.staticfiles import StaticFiles
from datetime import datetime
from .collector import parse_traceroute_output, store_traceroute
from .db import ensure_table_setup
from pprint import pprint as print
# Setup our SQLite before anything else.
ensure_table_setup()
# Setup web framework thingies
app = FastAPI()
app.mount("/static", StaticFiles(directory="static"), name="static")
@app.get("/")
def read_root():
return {
"greeting": "Hello, Kalzu!",
"instructions": [
"Try piping traceroute data directly to POST /trace/{hostname}.",
"{hostname} is for filtering data by sender.",
"For example the following command using HTTPie:",
"",
" $ traceroute peitto.info | http POST localhost:8000/trace/rekku",
"",
"",
"Also take a look at /docs/ and /static/index.html",
"",
"",
"",
"END OF TRANSMISSION",
] + [None]*800
}
@app.post("/trace/{hostname}")
async def create_trace(hostname: str, request: Request):
raw_data = await request.body()
data = raw_data.decode("utf-8", "ignore")
print(f"Received data from {hostname}:")
print(data)
trace = parse_traceroute_output(data, datetime.now())
print("Parsed data:")
print(trace)
store_traceroute(hostname, trace)
return {"status": "ok" }

0
netx_path_test.py → app/netx_path_test.py
View File

56
app/static/index.html Normal file
View File

@ -0,0 +1,56 @@
<!DOCTYPE html>
<html lang="en">
<head>
<title>Kalzu</title>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<script src="https://cdnjs.cloudflare.com/ajax/libs/sigma.js/2.4.0/sigma.min.js" integrity="sha512-iiPEYww3QXZU5C795JnnINBRNgHqDnRHs9mA7aJoqx4pNE4u3CknCDGmePHFoHtKR/6C9aIcRFa+HJ6obtlteQ==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/graphology/0.25.4/graphology.umd.min.js" integrity="sha512-tjMBhL9fLMcqoccPOwpRiIQIOAyUh18lWUlUvE10zvG1UNMfxUC4qSERmUq+VF30iavIyqs/q6fSP2o475FAUw==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
</head>
<body style="background: lightgrey">
<div id="container" style="width: 97%; height: 95vh; background: white; margin: auto; margin-top: 32px"></div>
<script>
const hops = [
["_gateway", "(192.168.0.1) 0.549 ms"],
["83-148-246-178.static.lounea.fi", "(83.148.246.178) 1.200 ms"],
["*"],
["*"],
["*"],
["86-60-254-198.static.lounea.fi", "(86.60.254.198) 3.719 ms"],
["72.14.194.142", " (72.14.194.142) 3.716 ms"],
["142.251.53.71", " (142.251.53.71) 3.704 ms"],
["142.250.229.87", " (142.250.229.87) 4.603 ms"],
["mad06s09-in-f142.1e100.net", " (216.58.210.142) 3.700 ms"],
];
console.log("Hops!", hops)
// Create a graphology graph
const graph = new graphology.Graph();
hops.forEach((hop, i) => {
const [ label, extra ] = hop;
const size = i === 0 ? 15 : 10;
const color = "green"
graph.addNode(i, { label, x: 0, y: i, size, color });
if (i > 0) {
graph.addEdge(i, i-1, { size: 2, color: "purple" });
}
});
// graph.addNode("1", { label: "Node 1", x: 0, y: 0, size: 10, color: "blue" });
// graph.addNode("2", { label: "Node 2", x: 1, y: 1, size: 20, color: "red" });
// graph.addEdge("1", "2", { size: 5, color: "purple" });
// Instantiate sigma.js and render the graph
const sigmaInstance = new Sigma(graph, document.getElementById("container"));
</script>
</body>
</html>
</html>

0
test.py → app/test.py
View File

BIN
app/traceroute.db Normal file
View File

Binary file not shown.

22
traceroute_visualization.py → app/traceroute_visualization.py
View File

@ -9,8 +9,8 @@ def retrieve_traceroute_data():
conn = sqlite3.connect('traceroute.db')
cursor = conn.cursor()
cursor.execute('''
SELECT l.source_ip, l.destination_ip, ll.latency, ll.timestamp
FROM link_latency ll
SELECT l.source_ip, l.destination_ip, ll.latency_ms, ll.timestamp
FROM Latency ll
JOIN links l ON ll.link_id = l.id
ORDER BY ll.timestamp
''')
@ -25,33 +25,33 @@ def build_graph(data):
destination_ip = row[1]
latency = row[2]
timestamp = row[3]
# Add nodes and edges to the graph
G.add_node(source_ip)
G.add_node(destination_ip)
G.add_edge(source_ip, destination_ip, latency=latency, timestamp=timestamp)
return G
#def visualize_graph_pyvis(G, output_file='network.html'):
# net = Network(height='750px', width='100%', directed=True)
# net.from_nx(G)
#
#
# for edge in G.edges(data=True):
# src, dst, data = edge
# latency = data['latency']
# timestamp = data['timestamp']
# net.add_edge(src, dst, title=f'Latency: {latency} ms<br>Timestamp: {timestamp}', value=latency)
#
#
# net.show(output_file)
def main():
# Retrieve data from the database
traceroute_data = retrieve_traceroute_data()
# Build the network graph
graph = build_graph(traceroute_data)
return graph
if __name__ == '__main__':
@ -59,9 +59,9 @@ if __name__ == '__main__':
nx.draw_planar(graph, with_labels=True)
plt.savefig("path.png")
# Visualize the graph using pyvis
#visualize_graph_pyvis(graph, 'traceroute_network.html')
exit(0)
#exit(0)

246
traceroute_collector.py
View File

@ -1,246 +0,0 @@
#!/usr/bin/env python3.11
import subprocess
import sqlite3
import re
import json
import ipaddress
import uuid
import hashlib
from datetime import datetime
def run_traceroute(host):
timestamp = datetime.now().timestamp()
result = subprocess.run(['traceroute', host], stdout=subprocess.PIPE)
return result.stdout.decode(), timestamp
def parse_traceroute_output(output, timestamp):
lines = output.strip().split('\n')
trace = {}
hops = []
ip_regex = r"\((.*?)\)" # ipaddress are in ()
target = output.strip().split('\n')[0].split()[2]
for line in lines[1:]:
hop = {}
hop_info = line.split()
hop_number = int(hop_info[0])
hop_name = None
hop_ip = None
hop_latency = None
latencies = []
#print("##### "+str(hop_info))
count = 0
for part in hop_info[1:]:
count += 1
# source node drops or blocks icmp packages
# We will give funny to name to hop for not answering and move on.
if part == '*':
hop_name = 'unresponsive'
hop_ip = 'unresponsive'
break
# If first colum is either name or ip-address
if count == 1:
match = re.search(ip_regex, part)
if match:
hop_ip = part.strip('()')
else:
hop_name = part
# Second colum is ip-address first latency reading
if count == 2:
if re.search(ip_regex, part):
try:
_ip = ipaddress.ip_address(part.strip('()'))
hop_ip = part.strip('()')
except ValueError:
pass # Ignore if it's not a valid IP address
# Rest of the input colums are either latency floats, 'ms' or
# reruns of the hop_name and hop_ip...
# We only need the latency floats anymore.
else:
try:
latency = float(part)
latencies.append(latency)
except ValueError:
pass
hop_latency = sum(latencies) / len(latencies) if latencies else None
hop['hop_number'] = hop_number
if not hop_name == None:
hop['hop_name'] = hop_name
hop['hop_ip'] = hop_ip
hop['hop_latency'] = hop_latency
hops.append(hop)
trace['target'] = target
trace['timestamp'] = timestamp
trace['hops'] = hops
return trace
def create_tables(databasefile):
# Connect to the SQLite database
conn = sqlite3.connect(databasefile)
cursor = conn.cursor()
# SQL statements to create the tables
create_links_table = """
CREATE TABLE IF NOT EXISTS Links (
id INTEGER PRIMARY KEY,
source_ip TEXT NOT NULL,
destination_ip TEXT NOT NULL,
UNIQUE(source_ip, destination_ip)
);
"""
create_paths_table = """
CREATE TABLE IF NOT EXISTS Paths (
id INTEGER PRIMARY KEY,
node TEXT NOT NULL,
target TEXT NOT NULL,
hops_json TEXT NOT NULL,
UNIQUE(node, target, hops_json)
);
"""
create_latency_table = """
CREATE TABLE IF NOT EXISTS Latency (
id INTEGER PRIMARY KEY,
link_id INTEGER NOT NULL,
timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
latency_ms REAL NOT NULL,
FOREIGN KEY (link_id) REFERENCES Links(id)
);
"""
create_hopdetails_table = """
CREATE TABLE IF NOT EXISTS HopDetails (
id INTEGER PRIMARY KEY,
hop_name TEXT,
hop_ip TEXT,
hop_latency TEXT
);
"""
# Execute the SQL statements
cursor.execute(create_links_table)
cursor.execute(create_paths_table)
cursor.execute(create_latency_table)
cursor.execute(create_hopdetails_table)
# Commit changes and close the connection
conn.commit()
conn.close()
def store_traceroute(db_file, node, trace):
conn = sqlite3.connect(db_file)
cursor = conn.cursor()
hops_json = json.dumps(trace['hops'])
path_ids = {}
previous_hop_ip = None
previous_hop_latency = None
for hop in trace['hops']:
hop_number = hop['hop_number']
hop_name = hop.get('hop_name')
hop_ip = hop.get('hop_ip')
hop_latency = hop.get('hop_latency')
link_id = None
# insert links and get their id's
if previous_hop_ip:
cursor.execute("""
INSERT OR IGNORE INTO Links (source_ip, destination_ip) VALUES (?, ?)
""", (previous_hop_ip, hop_ip))
cursor.execute("""
SELECT id FROM Links WHERE source_ip = ? AND destination_ip = ?
""", (previous_hop_ip, hop_ip))
result = cursor.fetchone()
link_id = result
path_ids[hop_number] = link_id
previous_hop_ip = hop_ip
# Save hop details
cursor.execute("INSERT INTO HopDetails (hop_name, hop_ip, hop_latency) VALUES (?, ?, ?)",
(hop_name, hop_ip, hop_latency))
# calculate link latency if possible and store it
if link_id and previous_hop_latency:
link_latency = hop_latency - previous_hop_latency
cursor.execute("INSERT INTO Latency (link_id, timestamp, latency_ms) VALUES (?, ?, ?)",
(link_id, trace['timestamp'], link_latency))
# make entry to "Paths" table
if path_ids:
json_path_ids = json.dumps(path_ids)
cursor.execute("INSERT OR IGNORE INTO Paths (node, target, hops_json) VALUES (?, ?, ?)",
(node, trace['target'], json_path_ids))
conn.commit()
conn.close()
def retrieve_traceroute(db_file):
retval = {}
conn = sqlite3.connect(db_file)
cursor = conn.cursor()
cursor.execute('''
SELECT target, hops_json
FROM Paths
''')
retval['path'] = cursor.fetchall()
cursor.execute('''
SELECT source_ip, destination_ip
FROM Links
''')
retval['links'] = cursor.fetchall()
conn.close()
return retval
def generate_node_id():
mac = uuid.getnode()
mac_str = ':'.join(['{:02x}'.format((mac >> ele) & 0xff) for ele in range(0,8*6,8)][::-1])
# Hash the MAC address using SHA-256 to generate a unique ID
unique_id = hashlib.sha256(mac_str.encode()).hexdigest()
return unique_id
if __name__ == '__main__':
db_file="./traceroute.db"
create_tables(db_file)
my_id = generate_node_id()
target='1.1.1.1'
traceroute_output, timestamp = run_traceroute(target)
trace = parse_traceroute_output(traceroute_output, timestamp)
store_traceroute(db_file, my_id, trace)
print("#####")
print(retrieve_traceroute(db_file))
print("#####")
exit(0)