Added curl example to README.

Reviewed-on: #1

README.md

@@ -13,6 +13,9 @@ fastapi dev app/main.py
 
 # Start posting data to the tracing endpoint:
 traceroute git.rauhala.info -q1 | http POST localhost:8000/trace/MYHOSTNAME
+or, with curl:
+traceroute tuoppi.dy.fi -q1 | curl -X POST http://localhost:8000/trace/MYHOSTNAME --data-binary @-
+
 ```
 
 ## URLs of interest

app/collector.py

@@ -8,8 +8,6 @@ from .db import Database
 
 
 def parse_traceroute_output(data: str, origin: str):
-    # TODO: data validation
-
     lines = data.strip().split("\n")
     target = lines[0].split()[2]
 
@@ -17,26 +15,21 @@ def parse_traceroute_output(data: str, origin: str):
 
     trace = {"target": target, "created": created, "origin": origin, "hops": []}
 
-    prev_latency = None
+    prev_latency = 0
 
     for line in lines[1:]:
         hop_info = line.split()
         print("LINE:", hop_info)
         try:
-            # Regular lines.
             number, name, ip, latency, _ = hop_info
-            latency = float(latency)
             hop = {
                 "created": created,
                 "number": number,
                 "name": name,
                 "ip": ip.strip("()"),
-                "latency": latency,
+                "latency": float(latency),
-                "link_latency": round(latency if prev_latency else latency, 3),
             }
-            prev_latency = latency
         except ValueError:
-            # Asterisks, no data found for hop.
             number, name = hop_info
             hop = {
                 "created": created,
@@ -44,7 +37,6 @@ def parse_traceroute_output(data: str, origin: str):
                 "name": name,
                 "ip": None,
                 "latency": None,
-                "link_latency": "?",
             }
 
         trace["hops"].append(hop)

app/db.py

@@ -36,7 +36,6 @@ class Database:
                 name TEXT,
                 ip TEXT,
                 latency TEXT,
-                link_latency TEXT,
 
                 FOREIGN KEY(trace_id) REFERENCES Traces(id)
             );
@@ -60,12 +59,7 @@ class Database:
             trace = dict(t)
 
             self.cursor.execute(
-                """
+                "SELECT number, name, ip, latency FROM Hops WHERE trace_id = ? ORDER BY number ASC",
-                SELECT number, name, ip, latency, link_latency
-                FROM Hops
-                WHERE trace_id = ?
-                ORDER BY number ASC
-            """,
                 (trace["id"],),
             )
             hops = self.cursor.fetchall()
@@ -84,7 +78,7 @@ class Database:
 
         for hop in trace["hops"]:
             self.cursor.execute(
-                "INSERT OR IGNORE INTO Hops (trace_id, created, number, name, ip, latency, link_latency) VALUES (?, ?, ?, ?, ?, ?, ?)",
+                "INSERT OR IGNORE INTO Hops (trace_id, created, number, name, ip, latency) VALUES (?, ?, ?, ?, ?, ?)",
                 (
                     trace_id,
                     hop["created"],
@@ -92,7 +86,6 @@ class Database:
                     hop["name"],
                     hop["ip"],
                     hop["latency"],
-                    hop["link_latency"],
                 ),
             )
 
@@ -104,6 +97,18 @@ class Database:
             (name, ip, latency),
         )
 
+    def create_latency(self, link_id, timestamp, link_latency):
+        self.cursor.execute(
+            "INSERT INTO Latency (link_id, timestamp, latency_ms) VALUES (?, ?, ?)",
+            (link_id, timestamp, link_latency),
+        )
+
+    def create_path(self, node, target, json):
+        self.cursor.execute(
+            "INSERT OR IGNORE INTO Paths (node, target, hops_json) VALUES (?, ?, ?)",
+            (node, target, json),
+        )
+
 
 def ensure_table_setup():
     db = Database()

app/static/index.js

@@ -130,219 +130,133 @@ const drag = (simulation) => {
 //   return svg.node();
 // };
 
-// const drawChart2 = (data) => {
+const drawChart2 = (data) => {
-//   // Data parsing
+  // Data parsing
-//   const nodes = Array.from(
+  const nodes = Array.from(
-//     new Set(data.flatMap((l) => [l.source, l.target])),
+    new Set(data.flatMap((l) => [l.source, l.target])),
-//     (id) => ({ id }),
+    (id) => ({ id }),
-//   );
+  );
-//   const links = data.map((d) => Object.create(d));
+  const links = data.map((d) => Object.create(d));
-//
-//   // Styles
-//   const width = window.visualViewport.width;
-//   const height = window.visualViewport.height - 10;
-//
-//   const colors = Array.from(new Set(data.map((d) => d.traceId)));
-//   const color = d3.scaleOrdinal(colors, d3.schemeCategory10);
-//
-//   const simulation = d3
-//     .forceSimulation(nodes)
-//     .force(
-//       "link",
-//       d3.forceLink(links).id((d) => d.id),
-//     )
-//     .force("charge", d3.forceManyBody().strength(-400))
-//     .force("x", d3.forceX())
-//     .force("y", d3.forceY());
-//
-//   // Canvas settings
-//   const svg = d3
-//     .create("svg")
-//     .attr("width", width)
-//     .attr("height", height)
-//     .attr("viewBox", [-width / 2, -height / 2, width, height])
-//     .attr("style", "max-width: 100%; height: auto; font: 14px monospace;");
-//
-//   // Pre-type arrowheads, as they don't inherit styles.
-//   svg
-//     .append("defs")
-//     .selectAll("marker")
-//     .data(colors)
-//     .join("marker")
-//     .attr("id", (d) => `arrow-${d}`)
-//     .attr("viewBox", "0 -5 10 10")
-//     .attr("refX", 15)
-//     .attr("refY", -0.5)
-//     .attr("markerWidth", 5)
-//     .attr("markerHeight", 5)
-//     .attr("orient", "auto")
-//     .append("path")
-//     .attr("fill", color)
-//     .attr("d", "M0,-5L10,0L0,5");
-//
-//   const link = svg
-//     .append("g")
-//     .attr("fill", "none")
-//     .attr("stroke-width", 1.5)
-//     .selectAll("path")
-//     .data(links)
-//     .join("path")
-//     .attr("stroke", (d) => color(d.traceId))
-//     .attr(
-//       "marker-end",
-//       (d) => `url(${new URL(`#arrow-${d.traceId}`, location)})`,
-//     );
-//
-//   const node = svg
-//     .append("g")
-//     .attr("fill", "currentColor")
-//     .attr("stroke-linecap", "round")
-//     .attr("stroke-linejoin", "round")
-//     .selectAll("g")
-//     .data(nodes)
-//     .join("g")
-//     .call(drag(simulation));
-//
-//   // Node "icon"
-//   node
-//     .append("circle")
-//     .attr("stroke", "white")
-//     .attr("stroke-width", 1.5)
-//     .attr("r", 5);
-//
-//   // Node text
-//   node
-//     .append("text")
-//     .attr("x", 8)
-//     .attr("y", 4)
-//     .text((d) => (d.id.endsWith("*") ? "*" : d.id))
-//     .clone(true)
-//     .lower()
-//     .attr("fill", "black")
-//     .attr("stroke", "white")
-//     .attr("stroke-width", 2);
-//
-//   simulation.on("tick", () => {
-//     link.attr("d", linkArc);
-//     // link
-//     //   .attr("x1", (d) => d.source.x)
-//     //   .attr("y1", (d) => d.source.y)
-//     //   .attr("x2", (d) => d.target.x)
-//     //   .attr("y2", (d) => d.target.y);
-//     node.attr("transform", (d) => `translate(${d.x},${d.y})`);
-//   });
-//
-//   // invalidation.then(() => simulation.stop());
-//
-//   return Object.assign(svg.node(), {
-//     scales: { color },
-//   });
-// };
 
-const state = {};
+  // Styles
+  const width = window.visualViewport.width;
+  const height = window.visualViewport.height - 10;
+
+  const colors = Array.from(new Set(data.map((d) => d.traceId)));
+  const color = d3.scaleOrdinal(colors, d3.schemeCategory10);
+
+  const simulation = d3
+    .forceSimulation(nodes)
+    .force(
+      "link",
+      d3.forceLink(links).id((d) => d.id),
+    )
+    .force("charge", d3.forceManyBody().strength(-400))
+    .force("x", d3.forceX())
+    .force("y", d3.forceY());
+
+  // Canvas settings
+  const svg = d3
+    .create("svg")
+    .attr("width", width)
+    .attr("height", height)
+    .attr("viewBox", [-width / 2, -height / 2, width, height])
+    .attr("style", "max-width: 100%; height: auto; font: 14px monospace;");
+
+  // Pre-type arrowheads, as they don't inherit styles.
+  svg
+    .append("defs")
+    .selectAll("marker")
+    .data(colors)
+    .join("marker")
+    .attr("id", (d) => `arrow-${d}`)
+    .attr("viewBox", "0 -5 10 10")
+    .attr("refX", 15)
+    .attr("refY", -0.5)
+    .attr("markerWidth", 5)
+    .attr("markerHeight", 5)
+    .attr("orient", "auto")
+    .append("path")
+    .attr("fill", color)
+    .attr("d", "M0,-5L10,0L0,5");
+
+  const link = svg
+    .append("g")
+    .attr("fill", "none")
+    .attr("stroke-width", 1.5)
+    .selectAll("path")
+    .data(links)
+    .join("path")
+    .attr("stroke", (d) => color(d.traceId))
+    .attr(
+      "marker-end",
+      (d) => `url(${new URL(`#arrow-${d.traceId}`, location)})`,
+    );
+
+  const node = svg
+    .append("g")
+    .attr("fill", "currentColor")
+    .attr("stroke-linecap", "round")
+    .attr("stroke-linejoin", "round")
+    .selectAll("g")
+    .data(nodes)
+    .join("g")
+    .call(drag(simulation));
+
+  // Node "icon"
+  node
+    .append("circle")
+    .attr("stroke", "white")
+    .attr("stroke-width", 1.5)
+    .attr("r", 5);
+
+  // Node text
+  node
+    .append("text")
+    .attr("x", 8)
+    .attr("y", 4)
+    .text((d) => (d.id.endsWith("*") ? "*" : d.id))
+    .clone(true)
+    .lower()
+    .attr("fill", "black")
+    .attr("stroke", "white")
+    .attr("stroke-width", 2);
+
+  simulation.on("tick", () => {
+    link.attr("d", linkArc);
+    // link
+    //   .attr("x1", (d) => d.source.x)
+    //   .attr("y1", (d) => d.source.y)
+    //   .attr("x2", (d) => d.target.x)
+    //   .attr("y2", (d) => d.target.y);
+    node.attr("transform", (d) => `translate(${d.x},${d.y})`);
+  });
+
+  // invalidation.then(() => simulation.stop());
+
+  return Object.assign(svg.node(), { scales: { color } });
+};
 
 const drawSigma = (data) => {
   // Create a graphology graph
   const graph = new graphology.MultiDirectedGraph();
 
-  const setHoveredNode = (node) => {
+  data.nodes.forEach((n, i) => {
-    if (node) {
+    console.log("Node:", n);
-      const props = graph.getNodeAttributes(node);
-      state.hoveredNode = node;
-      state.hoveredTrace = props.traceId;
-
-      // Compute the partial that we need to re-render to optimize the refresh
-      const nodes = graph.filterNodes((n) => {
-        const np = graph.getNodeAttributes(n);
-        return np.traceId === state.hoveredTrace;
-      });
-
-      console.log("Nämä?", nodes);
-
-      const nodesIndex = new Set(nodes);
-      const edges = graph.filterEdges((e) =>
-        graph.extremities(e).some((n) => nodesIndex.has(n)),
-      );
-
-      // Refresh rendering
-      renderer.refresh({
-        partialGraph: {
-          nodes,
-          edges,
-        },
-        // We don't touch the graph data so we can skip its reindexation
-        skipIndexation: true,
-      });
-    } else {
-      state.hoveredNode = undefined;
-      state.hoveredTrace = undefined;
-
-      // Refresh rendering
-      renderer.refresh({
-        // We don't touch the graph data so we can skip its reindexation
-        skipIndexation: true,
-      });
-    }
-  };
-
-  data.nodes.forEach((n) => {
     try {
       graph.addNode(n.id, n);
     } catch (e) {
-      // Duplicate node found, which is correct in our data scenario.
+      console.log("Node add:", e);
     }
   });
 
-  data.links.forEach((l) => {
+  data.links.forEach((l, i) => {
     graph.addEdge(l.source, l.target, l);
   });
 
   // Instantiate sigma.js and render the graph
   const renderer = new Sigma(graph, document.getElementById("container"), {
-    // labelThreshold: -10000,
+    labelThreshold: -10000,
-    renderEdgeLabels: true,
-  });
-
-  // Bind graph interactions:
-  renderer.on("enterNode", ({ node }) => {
-    console.log("enter");
-    setHoveredNode(node);
-  });
-  renderer.on("leaveNode", () => {
-    console.log("leave");
-    setHoveredNode(undefined);
-  });
-
-  // Render nodes accordingly to the internal state:
-  // 1. If a node is selected, it is highlighted
-  // 2. If there is query, all non-matching nodes are greyed
-  // 3. If there is a hovered node, all non-neighbor nodes are greyed
-  renderer.setSetting("nodeReducer", (node, data) => {
-    const res = { ...data };
-    const props = graph.getNodeAttributes(node);
-
-    if (state.hoveredTrace && props.traceId !== state.hoveredTrace) {
-      res.label = "";
-      res.color = "#f6f6f6";
-    }
-
-    return res;
-  });
-
-  // Render edges accordingly to the internal state:
-  // 1. If a node is hovered, the edge is hidden if it is not connected to the
-  //    node
-  // 2. If there is a query, the edge is only visible if it connects two
-  //    suggestions
-  renderer.setSetting("edgeReducer", (edge, data) => {
-    const res = { ...data };
-    const props = graph.getEdgeAttributes(edge);
-
-    if (state.hoveredTrace && props.traceId !== state.hoveredTrace) {
-      res.hidden = true;
-    }
-
-    return res;
   });
 };
 
@@ -377,7 +291,6 @@ const parseNodesAndLinks = (traces) => {
         label: id.endsWith("*") ? "*" : id,
         x: trace.id,
         y: hop.number / 2,
-        traceId: trace.id,
         size: 9,
         labelSize: 30,
         color: color(trace.id),
@@ -386,12 +299,12 @@ const parseNodesAndLinks = (traces) => {
       if (prevId) {
         // New link
         links.push({
-          label: hop.link_latency,
+          label: "asd",
           source: prevId,
           target: id,
           traceId: trace.id,
           origin: trace.origin,
-          size: 3,
+          size: 1,
           color: color(trace.id),
         });
       }
@@ -406,7 +319,6 @@ const parseNodesAndLinks = (traces) => {
     nodes.push({
       id: trace.id,
       label: trace.target,
-      traceId: trace.id,
       x: trace.id,
       y: (latestNumber + 1) / 2,
       size: 8,
@@ -414,9 +326,6 @@ const parseNodesAndLinks = (traces) => {
     });
 
     links.push({
-      label: "-",
-      size: 8,
-      traceId: trace.id,
       source: prevId,
       target: trace.id,
     });