5 Commits

Author SHA1 Message Date
018d6a71cf Merge pull request 'http-rewrite' (#2) from ryyst/traceroute_map:http-rewrite into ryyst_http_rewrite
Reviewed-on: MrKalzu/traceroute_map#2
2024-06-02 20:19:25 +03:00
2048f9c57d TEMP 2024-06-02 19:38:39 +03:00
c4244e86e6 TEMP 2024-06-02 00:22:00 +03:00
f82a95ab8d TEMP 2024-06-02 00:12:19 +03:00
83acd47845 TEMP 2024-06-01 19:43:18 +03:00
5 changed files with 407 additions and 231 deletions

27
README.md Normal file
View File

@ -0,0 +1,27 @@
# Traceroute Internet Mapper - TIM
## Development
```sh
# Install all dependencies. `pipenv` was used previously:
pipenv --python 3.11
pipenv install
pipenv shell
# Start dev server
fastapi dev app/main.py
# Start posting data to the tracing endpoint:
traceroute git.rauhala.info -q1 | http POST localhost:8000/trace/MYHOSTNAME
```
## URLs of interest
- http://localhost:8000/
- API Index, nothing to see.
- http://localhost:8000/docs/
- Autogenerated API docs, mildy interesting
- http://localhost:8000/static/index.html
- Rendered HTML index, very interesting

View File

@ -8,6 +8,8 @@ from .db import Database
def parse_traceroute_output(data: str, origin: str):
# TODO: data validation
lines = data.strip().split("\n")
target = lines[0].split()[2]
@ -15,21 +17,26 @@ def parse_traceroute_output(data: str, origin: str):
trace = {"target": target, "created": created, "origin": origin, "hops": []}
prev_latency = 0
prev_latency = None
for line in lines[1:]:
hop_info = line.split()
print("LINE:", hop_info)
try:
# Regular lines.
number, name, ip, latency, _ = hop_info
latency = float(latency)
hop = {
"created": created,
"number": number,
"name": name,
"ip": ip.strip("()"),
"latency": float(latency),
"latency": latency,
"link_latency": round(latency if prev_latency else latency, 3),
}
prev_latency = latency
except ValueError:
# Asterisks, no data found for hop.
number, name = hop_info
hop = {
"created": created,
@ -37,6 +44,7 @@ def parse_traceroute_output(data: str, origin: str):
"name": name,
"ip": None,
"latency": None,
"link_latency": "?",
}
trace["hops"].append(hop)

View File

@ -36,6 +36,7 @@ class Database:
name TEXT,
ip TEXT,
latency TEXT,
link_latency TEXT,
FOREIGN KEY(trace_id) REFERENCES Traces(id)
);
@ -59,7 +60,12 @@ class Database:
trace = dict(t)
self.cursor.execute(
"SELECT number, name, ip, latency FROM Hops WHERE trace_id = ? ORDER BY number ASC",
"""
SELECT number, name, ip, latency, link_latency
FROM Hops
WHERE trace_id = ?
ORDER BY number ASC
""",
(trace["id"],),
)
hops = self.cursor.fetchall()
@ -78,7 +84,7 @@ class Database:
for hop in trace["hops"]:
self.cursor.execute(
"INSERT OR IGNORE INTO Hops (trace_id, created, number, name, ip, latency) VALUES (?, ?, ?, ?, ?, ?)",
"INSERT OR IGNORE INTO Hops (trace_id, created, number, name, ip, latency, link_latency) VALUES (?, ?, ?, ?, ?, ?, ?)",
(
trace_id,
hop["created"],
@ -86,6 +92,7 @@ class Database:
hop["name"],
hop["ip"],
hop["latency"],
hop["link_latency"],
),
)
@ -97,18 +104,6 @@ class Database:
(name, ip, latency),
)
def create_latency(self, link_id, timestamp, link_latency):
self.cursor.execute(
"INSERT INTO Latency (link_id, timestamp, latency_ms) VALUES (?, ?, ?)",
(link_id, timestamp, link_latency),
)
def create_path(self, node, target, json):
self.cursor.execute(
"INSERT OR IGNORE INTO Paths (node, target, hops_json) VALUES (?, ?, ?)",
(node, target, json),
)
def ensure_table_setup():
db = Database()

View File

@ -4,11 +4,17 @@
<title>Kalzu</title>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<script src="https://cdnjs.cloudflare.com/ajax/libs/sigma.js/2.4.0/sigma.min.js" integrity="sha512-iiPEYww3QXZU5C795JnnINBRNgHqDnRHs9mA7aJoqx4pNE4u3CknCDGmePHFoHtKR/6C9aIcRFa+HJ6obtlteQ==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/graphology/0.25.4/graphology.umd.min.js" integrity="sha512-tjMBhL9fLMcqoccPOwpRiIQIOAyUh18lWUlUvE10zvG1UNMfxUC4qSERmUq+VF30iavIyqs/q6fSP2o475FAUw==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
<script src="/static/d3.v7.min.js"></script>
<style>
#container {
height: 95vh;
}
</style>
</head>
<body style="background: #EEE">
<body style="background: #eee">
<div id="container"></div>
<script src="/static/index.js"></script>
</body>
</html>
</html>

View File

@ -1,5 +1,5 @@
const linkArc = (d) => {
const r = Math.hypot(d.target.x - d.source.x, d.target.y - d.source.y);
const r = Math.hypot(d.target.x - d.source.x, d.target.y - d.source.y) * 3;
return `
M${d.source.x},${d.source.y}
A${r},${r} 0 0,1 ${d.target.x},${d.target.y}
@ -31,265 +31,403 @@ const drag = (simulation) => {
.on("end", dragended);
};
const drawChart = (data) => {
// Specify the dimensions of the chart.
const width = 1600;
const height = 1200;
// const drawChart = (data) => {
// // Specify the dimensions of the chart.
// const width = 1600;
// const height = 1200;
//
// // Specify the color scale.
// const color = d3.scaleOrdinal(d3.schemeCategory10);
//
// // The force simulation mutates links and nodes, so create a copy
// // so that re-evaluating this cell produces the same result.
// const links = data.links.map((d) => ({ ...d }));
// const nodes = data.nodes.map((d) => ({ ...d }));
//
// // Create a simulation with several forces.
// const simulation = d3
// .forceSimulation(nodes)
// .force(
// "link",
// d3.forceLink(links).id((d) => d.id),
// )
// .force("charge", d3.forceManyBody())
// .force("x", d3.forceX())
// .force("y", d3.forceY());
//
// // Create the SVG container.
// const svg = d3
// .create("svg")
// .attr("width", width)
// .attr("height", height)
// .attr("viewBox", [-width / 2, -height / 2, width, height])
// .attr("style", "max-width: 100%; height: auto;");
//
// // Add a line for each link, and a circle for each node.
// const link = svg
// .append("g")
// .attr("stroke", "#999")
// .attr("stroke-opacity", 0.6)
// .selectAll("line")
// .data(links)
// .join("line")
// .attr("stroke-width", 1); // (d) => Math.sqrt(d.value));
//
// const node = svg
// .append("g")
// .attr("stroke", "#fff")
// .attr("stroke-width", 1.5)
// .selectAll("circle")
// .data(nodes)
// .join("circle")
// .attr("r", 5)
// .attr("fill", (d) => color(d.group));
//
// node.append("title").text((d) => d.id);
//
// // Add a drag behavior.
// node.call(
// d3.drag().on("start", dragstarted).on("drag", dragged).on("end", dragended),
// );
//
// // Set the position attributes of links and nodes each time the simulation ticks.
// simulation.on("tick", () => {
// link
// .attr("x1", (d) => d.source.x)
// .attr("y1", (d) => d.source.y)
// .attr("x2", (d) => d.target.x)
// .attr("y2", (d) => d.target.y);
//
// node.attr("cx", (d) => d.x).attr("cy", (d) => d.y);
// });
//
// // Reheat the simulation when drag starts, and fix the subject position.
// function dragstarted(event) {
// if (!event.active) simulation.alphaTarget(0.3).restart();
// event.subject.fx = event.subject.x;
// event.subject.fy = event.subject.y;
// }
//
// // Update the subject (dragged node) position during drag.
// function dragged(event) {
// event.subject.fx = event.x;
// event.subject.fy = event.y;
// }
//
// // Restore the target alpha so the simulation cools after dragging ends.
// // Unfix the subject position now that its no longer being dragged.
// function dragended(event) {
// if (!event.active) simulation.alphaTarget(0);
// event.subject.fx = null;
// event.subject.fy = null;
// }
//
// // When this cell is re-run, stop the previous simulation. (This doesnt
// // really matter since the target alpha is zero and the simulation will
// // stop naturally, but its a good practice.)
// // invalidation.then(() => simulation.stop());
//
// return svg.node();
// };
// Specify the color scale.
const color = d3.scaleOrdinal(d3.schemeCategory10);
// const drawChart2 = (data) => {
// // Data parsing
// const nodes = Array.from(
// new Set(data.flatMap((l) => [l.source, l.target])),
// (id) => ({ id }),
// );
// const links = data.map((d) => Object.create(d));
//
// // Styles
// const width = window.visualViewport.width;
// const height = window.visualViewport.height - 10;
//
// const colors = Array.from(new Set(data.map((d) => d.traceId)));
// const color = d3.scaleOrdinal(colors, d3.schemeCategory10);
//
// const simulation = d3
// .forceSimulation(nodes)
// .force(
// "link",
// d3.forceLink(links).id((d) => d.id),
// )
// .force("charge", d3.forceManyBody().strength(-400))
// .force("x", d3.forceX())
// .force("y", d3.forceY());
//
// // Canvas settings
// const svg = d3
// .create("svg")
// .attr("width", width)
// .attr("height", height)
// .attr("viewBox", [-width / 2, -height / 2, width, height])
// .attr("style", "max-width: 100%; height: auto; font: 14px monospace;");
//
// // Pre-type arrowheads, as they don't inherit styles.
// svg
// .append("defs")
// .selectAll("marker")
// .data(colors)
// .join("marker")
// .attr("id", (d) => `arrow-${d}`)
// .attr("viewBox", "0 -5 10 10")
// .attr("refX", 15)
// .attr("refY", -0.5)
// .attr("markerWidth", 5)
// .attr("markerHeight", 5)
// .attr("orient", "auto")
// .append("path")
// .attr("fill", color)
// .attr("d", "M0,-5L10,0L0,5");
//
// const link = svg
// .append("g")
// .attr("fill", "none")
// .attr("stroke-width", 1.5)
// .selectAll("path")
// .data(links)
// .join("path")
// .attr("stroke", (d) => color(d.traceId))
// .attr(
// "marker-end",
// (d) => `url(${new URL(`#arrow-${d.traceId}`, location)})`,
// );
//
// const node = svg
// .append("g")
// .attr("fill", "currentColor")
// .attr("stroke-linecap", "round")
// .attr("stroke-linejoin", "round")
// .selectAll("g")
// .data(nodes)
// .join("g")
// .call(drag(simulation));
//
// // Node "icon"
// node
// .append("circle")
// .attr("stroke", "white")
// .attr("stroke-width", 1.5)
// .attr("r", 5);
//
// // Node text
// node
// .append("text")
// .attr("x", 8)
// .attr("y", 4)
// .text((d) => (d.id.endsWith("*") ? "*" : d.id))
// .clone(true)
// .lower()
// .attr("fill", "black")
// .attr("stroke", "white")
// .attr("stroke-width", 2);
//
// simulation.on("tick", () => {
// link.attr("d", linkArc);
// // link
// // .attr("x1", (d) => d.source.x)
// // .attr("y1", (d) => d.source.y)
// // .attr("x2", (d) => d.target.x)
// // .attr("y2", (d) => d.target.y);
// node.attr("transform", (d) => `translate(${d.x},${d.y})`);
// });
//
// // invalidation.then(() => simulation.stop());
//
// return Object.assign(svg.node(), {
// scales: { color },
// });
// };
// The force simulation mutates links and nodes, so create a copy
// so that re-evaluating this cell produces the same result.
const links = data.links.map((d) => ({ ...d }));
const nodes = data.nodes.map((d) => ({ ...d }));
const state = {};
// Create a simulation with several forces.
const simulation = d3
.forceSimulation(nodes)
.force(
"link",
d3.forceLink(links).id((d) => d.id),
)
.force("charge", d3.forceManyBody())
.force("x", d3.forceX())
.force("y", d3.forceY());
const drawSigma = (data) => {
// Create a graphology graph
const graph = new graphology.MultiDirectedGraph();
// Create the SVG container.
const svg = d3
.create("svg")
.attr("width", width)
.attr("height", height)
.attr("viewBox", [-width / 2, -height / 2, width, height])
.attr("style", "max-width: 100%; height: auto;");
const setHoveredNode = (node) => {
if (node) {
const props = graph.getNodeAttributes(node);
state.hoveredNode = node;
state.hoveredTrace = props.traceId;
// Add a line for each link, and a circle for each node.
const link = svg
.append("g")
.attr("stroke", "#999")
.attr("stroke-opacity", 0.6)
.selectAll("line")
.data(links)
.join("line")
.attr("stroke-width", 1); // (d) => Math.sqrt(d.value));
// Compute the partial that we need to re-render to optimize the refresh
const nodes = graph.filterNodes((n) => {
const np = graph.getNodeAttributes(n);
return np.traceId === state.hoveredTrace;
});
const node = svg
.append("g")
.attr("stroke", "#fff")
.attr("stroke-width", 1.5)
.selectAll("circle")
.data(nodes)
.join("circle")
.attr("r", 5)
.attr("fill", (d) => color(d.group));
console.log("Nämä?", nodes);
node.append("title").text((d) => d.id);
const nodesIndex = new Set(nodes);
const edges = graph.filterEdges((e) =>
graph.extremities(e).some((n) => nodesIndex.has(n)),
);
// Add a drag behavior.
node.call(
d3.drag().on("start", dragstarted).on("drag", dragged).on("end", dragended),
);
// Refresh rendering
renderer.refresh({
partialGraph: {
nodes,
edges,
},
// We don't touch the graph data so we can skip its reindexation
skipIndexation: true,
});
} else {
state.hoveredNode = undefined;
state.hoveredTrace = undefined;
// Set the position attributes of links and nodes each time the simulation ticks.
simulation.on("tick", () => {
link
.attr("x1", (d) => d.source.x)
.attr("y1", (d) => d.source.y)
.attr("x2", (d) => d.target.x)
.attr("y2", (d) => d.target.y);
// Refresh rendering
renderer.refresh({
// We don't touch the graph data so we can skip its reindexation
skipIndexation: true,
});
}
};
node.attr("cx", (d) => d.x).attr("cy", (d) => d.y);
data.nodes.forEach((n) => {
try {
graph.addNode(n.id, n);
} catch (e) {
// Duplicate node found, which is correct in our data scenario.
}
});
// Reheat the simulation when drag starts, and fix the subject position.
function dragstarted(event) {
if (!event.active) simulation.alphaTarget(0.3).restart();
event.subject.fx = event.subject.x;
event.subject.fy = event.subject.y;
}
data.links.forEach((l) => {
graph.addEdge(l.source, l.target, l);
});
// Update the subject (dragged node) position during drag.
function dragged(event) {
event.subject.fx = event.x;
event.subject.fy = event.y;
}
// Instantiate sigma.js and render the graph
const renderer = new Sigma(graph, document.getElementById("container"), {
// labelThreshold: -10000,
renderEdgeLabels: true,
});
// Restore the target alpha so the simulation cools after dragging ends.
// Unfix the subject position now that its no longer being dragged.
function dragended(event) {
if (!event.active) simulation.alphaTarget(0);
event.subject.fx = null;
event.subject.fy = null;
}
// Bind graph interactions:
renderer.on("enterNode", ({ node }) => {
console.log("enter");
setHoveredNode(node);
});
renderer.on("leaveNode", () => {
console.log("leave");
setHoveredNode(undefined);
});
// When this cell is re-run, stop the previous simulation. (This doesnt
// really matter since the target alpha is zero and the simulation will
// stop naturally, but its a good practice.)
// invalidation.then(() => simulation.stop());
// Render nodes accordingly to the internal state:
// 1. If a node is selected, it is highlighted
// 2. If there is query, all non-matching nodes are greyed
// 3. If there is a hovered node, all non-neighbor nodes are greyed
renderer.setSetting("nodeReducer", (node, data) => {
const res = { ...data };
const props = graph.getNodeAttributes(node);
return svg.node();
if (state.hoveredTrace && props.traceId !== state.hoveredTrace) {
res.label = "";
res.color = "#f6f6f6";
}
return res;
});
// Render edges accordingly to the internal state:
// 1. If a node is hovered, the edge is hidden if it is not connected to the
// node
// 2. If there is a query, the edge is only visible if it connects two
// suggestions
renderer.setSetting("edgeReducer", (edge, data) => {
const res = { ...data };
const props = graph.getEdgeAttributes(edge);
if (state.hoveredTrace && props.traceId !== state.hoveredTrace) {
res.hidden = true;
}
return res;
});
};
const drawChart2 = (data) => {
const width = 1600;
const height = 1200;
const types = Array.from(new Set(data.map((d) => d.type)));
const nodes = Array.from(
new Set(data.flatMap((l) => [l.source, l.target])),
(id) => ({ id }),
);
const links = data.map((d) => Object.create(d));
const getNodeID = (hop, prevId, trace) => {
if (prevId === null) {
return trace.origin;
}
if (hop.name === "*") {
return `${trace.id}-${hop.number}-*`;
}
const color = d3.scaleOrdinal(types, d3.schemeCategory10);
const simulation = d3
.forceSimulation(nodes)
.force(
"link",
d3.forceLink(links).id((d) => d.id),
)
.force("charge", d3.forceManyBody().strength(-400))
.force("x", d3.forceX())
.force("y", d3.forceY());
const svg = d3
.create("svg")
.attr("viewBox", [-width / 2, -height / 2, width, height])
.attr("width", width)
.attr("height", height)
.attr("style", "max-width: 100%; height: auto; font: 12px sans-serif;");
// Per-type markers, as they don't inherit styles.
svg
.append("defs")
.selectAll("marker")
.data(types)
.join("marker")
.attr("id", (d) => `arrow-${d}`)
.attr("viewBox", "0 -5 10 10")
.attr("refX", 15)
.attr("refY", -0.5)
.attr("markerWidth", 6)
.attr("markerHeight", 6)
.attr("orient", "auto")
.append("path")
.attr("fill", color)
.attr("d", "M0,-5L10,0L0,5");
const link = svg
.append("g")
.attr("fill", "none")
.attr("stroke-width", 1.5)
.selectAll("path")
.data(links)
.join("path")
.attr("stroke", (d) => color(d.type))
.attr("marker-end", (d) => `url(${new URL(`#arrow-${d.type}`, location)})`);
const node = svg
.append("g")
.attr("fill", "currentColor")
.attr("stroke-linecap", "round")
.attr("stroke-linejoin", "round")
.selectAll("g")
.data(nodes)
.join("g")
.call(drag(simulation));
node
.append("circle")
.attr("stroke", "white")
.attr("stroke-width", 1.5)
.attr("r", 4);
node
.append("text")
.attr("x", 8)
.attr("y", "0.31em")
.text((d) => (d.id.endsWith("*") ? "*" : d.id))
.clone(true)
.lower()
.attr("fill", "none")
.attr("stroke", "white")
.attr("stroke-width", 3);
simulation.on("tick", () => {
link.attr("d", linkArc);
node.attr("transform", (d) => `translate(${d.x},${d.y})`);
});
// invalidation.then(() => simulation.stop());
return Object.assign(svg.node(), { scales: { color } });
return hop.ip;
};
const parseNodesAndLinks = (traces) => {
const result = {
nodes: [],
links: [],
};
const nodes = [];
const links = [];
const colors = Array.from(new Set(traces.map((t) => t.id)));
const color = d3.scaleOrdinal(colors, d3.schemeCategory10);
traces.forEach((trace) => {
let prevId = null;
const getId = (hop) => {
if (prevId === null) {
return trace.origin;
}
if (hop.name === "*") {
return `${trace.id}-${hop.number}-*`;
}
return hop.ip;
};
let latestNumber = null;
trace.hops.forEach((hop) => {
const id = getId(hop);
const id = getNodeID(hop, prevId, trace);
// New node
result.nodes.push({
nodes.push({
id: id,
group: trace.origin,
radius: 8,
value: hop.name || "name?",
origin: trace.origin,
label: id.endsWith("*") ? "*" : id,
x: trace.id,
y: hop.number / 2,
traceId: trace.id,
size: 9,
labelSize: 30,
color: color(trace.id),
});
if (prevId) {
// New link
result.links.push({
links.push({
label: hop.link_latency,
source: prevId,
target: id,
type: trace.origin,
group: trace.origin,
traceId: trace.id,
origin: trace.origin,
size: 3,
color: color(trace.id),
});
}
prevId = id;
latestNumber = hop.number;
});
// Last "destination" node
result.nodes.push({
/**
* Last "destination" node, just for candy
*/
nodes.push({
id: trace.id,
group: trace.origin,
radius: 8,
value: trace.target,
label: trace.target,
traceId: trace.id,
x: trace.id,
y: (latestNumber + 1) / 2,
size: 8,
color: "black",
});
if (prevId) {
// New link
result.links.push({
source: prevId,
target: trace.target,
type: trace.origin,
group: trace.origin,
});
}
links.push({
label: "-",
size: 8,
traceId: trace.id,
source: prevId,
target: trace.id,
});
});
// { id: ip, group: origin, radius: 2 }
// { source: prev.ip, target: ip, value: latency }
return result;
return {
nodes,
links,
};
};
async function main() {
@ -300,9 +438,11 @@ async function main() {
const data = parseNodesAndLinks(traces);
console.log("Data:", data);
const chart = drawChart2(data.links);
// const chart = drawChart2(data.links);
container.append(chart);
const chart = drawSigma(data);
// container.append(chart);
}
main();