Merge pull request 'http-rewrite' (#2) from ryyst/traceroute_map:http-rewrite into ryyst_http_rewrite

Reviewed-on: #2

README.md

@@ -13,9 +13,6 @@ fastapi dev app/main.py
 
 # Start posting data to the tracing endpoint:
 traceroute git.rauhala.info -q1 | http POST localhost:8000/trace/MYHOSTNAME
-or, with curl:
-traceroute tuoppi.dy.fi -q1 | curl -X POST http://localhost:8000/trace/MYHOSTNAME --data-binary @-
-
 ```
 
 ## URLs of interest

app/collector.py

@@ -8,6 +8,8 @@ from .db import Database
 
 
 def parse_traceroute_output(data: str, origin: str):
+    # TODO: data validation
+
     lines = data.strip().split("\n")
     target = lines[0].split()[2]
 
@@ -15,21 +17,26 @@ def parse_traceroute_output(data: str, origin: str):
 
     trace = {"target": target, "created": created, "origin": origin, "hops": []}
 
-    prev_latency = 0
+    prev_latency = None
 
     for line in lines[1:]:
         hop_info = line.split()
         print("LINE:", hop_info)
         try:
+            # Regular lines.
             number, name, ip, latency, _ = hop_info
+            latency = float(latency)
             hop = {
                 "created": created,
                 "number": number,
                 "name": name,
                 "ip": ip.strip("()"),
-                "latency": float(latency),
+                "latency": latency,
+                "link_latency": round(latency if prev_latency else latency, 3),
             }
+            prev_latency = latency
         except ValueError:
+            # Asterisks, no data found for hop.
             number, name = hop_info
             hop = {
                 "created": created,
@@ -37,6 +44,7 @@ def parse_traceroute_output(data: str, origin: str):
                 "name": name,
                 "ip": None,
                 "latency": None,
+                "link_latency": "?",
             }
 
         trace["hops"].append(hop)

app/db.py

@@ -36,6 +36,7 @@ class Database:
                 name TEXT,
                 ip TEXT,
                 latency TEXT,
+                link_latency TEXT,
 
                 FOREIGN KEY(trace_id) REFERENCES Traces(id)
             );
@@ -59,7 +60,12 @@ class Database:
             trace = dict(t)
 
             self.cursor.execute(
-                "SELECT number, name, ip, latency FROM Hops WHERE trace_id = ? ORDER BY number ASC",
+                """
+                SELECT number, name, ip, latency, link_latency
+                FROM Hops
+                WHERE trace_id = ?
+                ORDER BY number ASC
+            """,
                 (trace["id"],),
             )
             hops = self.cursor.fetchall()
@@ -78,7 +84,7 @@ class Database:
 
         for hop in trace["hops"]:
             self.cursor.execute(
-                "INSERT OR IGNORE INTO Hops (trace_id, created, number, name, ip, latency) VALUES (?, ?, ?, ?, ?, ?)",
+                "INSERT OR IGNORE INTO Hops (trace_id, created, number, name, ip, latency, link_latency) VALUES (?, ?, ?, ?, ?, ?, ?)",
                 (
                     trace_id,
                     hop["created"],
@@ -86,6 +92,7 @@ class Database:
                     hop["name"],
                     hop["ip"],
                     hop["latency"],
+                    hop["link_latency"],
                 ),
             )
 
@@ -97,18 +104,6 @@ class Database:
             (name, ip, latency),
         )
 
-    def create_latency(self, link_id, timestamp, link_latency):
-        self.cursor.execute(
-            "INSERT INTO Latency (link_id, timestamp, latency_ms) VALUES (?, ?, ?)",
-            (link_id, timestamp, link_latency),
-        )
-
-    def create_path(self, node, target, json):
-        self.cursor.execute(
-            "INSERT OR IGNORE INTO Paths (node, target, hops_json) VALUES (?, ?, ?)",
-            (node, target, json),
-        )
-
 
 def ensure_table_setup():
     db = Database()

app/static/index.js

@@ -130,133 +130,219 @@ const drag = (simulation) => {
 //   return svg.node();
 // };
 
-const drawChart2 = (data) => {
+// const drawChart2 = (data) => {
-  // Data parsing
+//   // Data parsing
-  const nodes = Array.from(
+//   const nodes = Array.from(
-    new Set(data.flatMap((l) => [l.source, l.target])),
+//     new Set(data.flatMap((l) => [l.source, l.target])),
-    (id) => ({ id }),
+//     (id) => ({ id }),
-  );
+//   );
-  const links = data.map((d) => Object.create(d));
+//   const links = data.map((d) => Object.create(d));
+//
+//   // Styles
+//   const width = window.visualViewport.width;
+//   const height = window.visualViewport.height - 10;
+//
+//   const colors = Array.from(new Set(data.map((d) => d.traceId)));
+//   const color = d3.scaleOrdinal(colors, d3.schemeCategory10);
+//
+//   const simulation = d3
+//     .forceSimulation(nodes)
+//     .force(
+//       "link",
+//       d3.forceLink(links).id((d) => d.id),
+//     )
+//     .force("charge", d3.forceManyBody().strength(-400))
+//     .force("x", d3.forceX())
+//     .force("y", d3.forceY());
+//
+//   // Canvas settings
+//   const svg = d3
+//     .create("svg")
+//     .attr("width", width)
+//     .attr("height", height)
+//     .attr("viewBox", [-width / 2, -height / 2, width, height])
+//     .attr("style", "max-width: 100%; height: auto; font: 14px monospace;");
+//
+//   // Pre-type arrowheads, as they don't inherit styles.
+//   svg
+//     .append("defs")
+//     .selectAll("marker")
+//     .data(colors)
+//     .join("marker")
+//     .attr("id", (d) => `arrow-${d}`)
+//     .attr("viewBox", "0 -5 10 10")
+//     .attr("refX", 15)
+//     .attr("refY", -0.5)
+//     .attr("markerWidth", 5)
+//     .attr("markerHeight", 5)
+//     .attr("orient", "auto")
+//     .append("path")
+//     .attr("fill", color)
+//     .attr("d", "M0,-5L10,0L0,5");
+//
+//   const link = svg
+//     .append("g")
+//     .attr("fill", "none")
+//     .attr("stroke-width", 1.5)
+//     .selectAll("path")
+//     .data(links)
+//     .join("path")
+//     .attr("stroke", (d) => color(d.traceId))
+//     .attr(
+//       "marker-end",
+//       (d) => `url(${new URL(`#arrow-${d.traceId}`, location)})`,
+//     );
+//
+//   const node = svg
+//     .append("g")
+//     .attr("fill", "currentColor")
+//     .attr("stroke-linecap", "round")
+//     .attr("stroke-linejoin", "round")
+//     .selectAll("g")
+//     .data(nodes)
+//     .join("g")
+//     .call(drag(simulation));
+//
+//   // Node "icon"
+//   node
+//     .append("circle")
+//     .attr("stroke", "white")
+//     .attr("stroke-width", 1.5)
+//     .attr("r", 5);
+//
+//   // Node text
+//   node
+//     .append("text")
+//     .attr("x", 8)
+//     .attr("y", 4)
+//     .text((d) => (d.id.endsWith("*") ? "*" : d.id))
+//     .clone(true)
+//     .lower()
+//     .attr("fill", "black")
+//     .attr("stroke", "white")
+//     .attr("stroke-width", 2);
+//
+//   simulation.on("tick", () => {
+//     link.attr("d", linkArc);
+//     // link
+//     //   .attr("x1", (d) => d.source.x)
+//     //   .attr("y1", (d) => d.source.y)
+//     //   .attr("x2", (d) => d.target.x)
+//     //   .attr("y2", (d) => d.target.y);
+//     node.attr("transform", (d) => `translate(${d.x},${d.y})`);
+//   });
+//
+//   // invalidation.then(() => simulation.stop());
+//
+//   return Object.assign(svg.node(), {
+//     scales: { color },
+//   });
+// };
 
-  // Styles
+const state = {};
-  const width = window.visualViewport.width;
-  const height = window.visualViewport.height - 10;
-
-  const colors = Array.from(new Set(data.map((d) => d.traceId)));
-  const color = d3.scaleOrdinal(colors, d3.schemeCategory10);
-
-  const simulation = d3
-    .forceSimulation(nodes)
-    .force(
-      "link",
-      d3.forceLink(links).id((d) => d.id),
-    )
-    .force("charge", d3.forceManyBody().strength(-400))
-    .force("x", d3.forceX())
-    .force("y", d3.forceY());
-
-  // Canvas settings
-  const svg = d3
-    .create("svg")
-    .attr("width", width)
-    .attr("height", height)
-    .attr("viewBox", [-width / 2, -height / 2, width, height])
-    .attr("style", "max-width: 100%; height: auto; font: 14px monospace;");
-
-  // Pre-type arrowheads, as they don't inherit styles.
-  svg
-    .append("defs")
-    .selectAll("marker")
-    .data(colors)
-    .join("marker")
-    .attr("id", (d) => `arrow-${d}`)
-    .attr("viewBox", "0 -5 10 10")
-    .attr("refX", 15)
-    .attr("refY", -0.5)
-    .attr("markerWidth", 5)
-    .attr("markerHeight", 5)
-    .attr("orient", "auto")
-    .append("path")
-    .attr("fill", color)
-    .attr("d", "M0,-5L10,0L0,5");
-
-  const link = svg
-    .append("g")
-    .attr("fill", "none")
-    .attr("stroke-width", 1.5)
-    .selectAll("path")
-    .data(links)
-    .join("path")
-    .attr("stroke", (d) => color(d.traceId))
-    .attr(
-      "marker-end",
-      (d) => `url(${new URL(`#arrow-${d.traceId}`, location)})`,
-    );
-
-  const node = svg
-    .append("g")
-    .attr("fill", "currentColor")
-    .attr("stroke-linecap", "round")
-    .attr("stroke-linejoin", "round")
-    .selectAll("g")
-    .data(nodes)
-    .join("g")
-    .call(drag(simulation));
-
-  // Node "icon"
-  node
-    .append("circle")
-    .attr("stroke", "white")
-    .attr("stroke-width", 1.5)
-    .attr("r", 5);
-
-  // Node text
-  node
-    .append("text")
-    .attr("x", 8)
-    .attr("y", 4)
-    .text((d) => (d.id.endsWith("*") ? "*" : d.id))
-    .clone(true)
-    .lower()
-    .attr("fill", "black")
-    .attr("stroke", "white")
-    .attr("stroke-width", 2);
-
-  simulation.on("tick", () => {
-    link.attr("d", linkArc);
-    // link
-    //   .attr("x1", (d) => d.source.x)
-    //   .attr("y1", (d) => d.source.y)
-    //   .attr("x2", (d) => d.target.x)
-    //   .attr("y2", (d) => d.target.y);
-    node.attr("transform", (d) => `translate(${d.x},${d.y})`);
-  });
-
-  // invalidation.then(() => simulation.stop());
-
-  return Object.assign(svg.node(), { scales: { color } });
-};
 
 const drawSigma = (data) => {
   // Create a graphology graph
   const graph = new graphology.MultiDirectedGraph();
 
-  data.nodes.forEach((n, i) => {
+  const setHoveredNode = (node) => {
-    console.log("Node:", n);
+    if (node) {
+      const props = graph.getNodeAttributes(node);
+      state.hoveredNode = node;
+      state.hoveredTrace = props.traceId;
+
+      // Compute the partial that we need to re-render to optimize the refresh
+      const nodes = graph.filterNodes((n) => {
+        const np = graph.getNodeAttributes(n);
+        return np.traceId === state.hoveredTrace;
+      });
+
+      console.log("Nämä?", nodes);
+
+      const nodesIndex = new Set(nodes);
+      const edges = graph.filterEdges((e) =>
+        graph.extremities(e).some((n) => nodesIndex.has(n)),
+      );
+
+      // Refresh rendering
+      renderer.refresh({
+        partialGraph: {
+          nodes,
+          edges,
+        },
+        // We don't touch the graph data so we can skip its reindexation
+        skipIndexation: true,
+      });
+    } else {
+      state.hoveredNode = undefined;
+      state.hoveredTrace = undefined;
+
+      // Refresh rendering
+      renderer.refresh({
+        // We don't touch the graph data so we can skip its reindexation
+        skipIndexation: true,
+      });
+    }
+  };
+
+  data.nodes.forEach((n) => {
     try {
       graph.addNode(n.id, n);
     } catch (e) {
-      console.log("Node add:", e);
+      // Duplicate node found, which is correct in our data scenario.
     }
   });
 
-  data.links.forEach((l, i) => {
+  data.links.forEach((l) => {
     graph.addEdge(l.source, l.target, l);
   });
 
   // Instantiate sigma.js and render the graph
   const renderer = new Sigma(graph, document.getElementById("container"), {
-    labelThreshold: -10000,
+    // labelThreshold: -10000,
+    renderEdgeLabels: true,
+  });
+
+  // Bind graph interactions:
+  renderer.on("enterNode", ({ node }) => {
+    console.log("enter");
+    setHoveredNode(node);
+  });
+  renderer.on("leaveNode", () => {
+    console.log("leave");
+    setHoveredNode(undefined);
+  });
+
+  // Render nodes accordingly to the internal state:
+  // 1. If a node is selected, it is highlighted
+  // 2. If there is query, all non-matching nodes are greyed
+  // 3. If there is a hovered node, all non-neighbor nodes are greyed
+  renderer.setSetting("nodeReducer", (node, data) => {
+    const res = { ...data };
+    const props = graph.getNodeAttributes(node);
+
+    if (state.hoveredTrace && props.traceId !== state.hoveredTrace) {
+      res.label = "";
+      res.color = "#f6f6f6";
+    }
+
+    return res;
+  });
+
+  // Render edges accordingly to the internal state:
+  // 1. If a node is hovered, the edge is hidden if it is not connected to the
+  //    node
+  // 2. If there is a query, the edge is only visible if it connects two
+  //    suggestions
+  renderer.setSetting("edgeReducer", (edge, data) => {
+    const res = { ...data };
+    const props = graph.getEdgeAttributes(edge);
+
+    if (state.hoveredTrace && props.traceId !== state.hoveredTrace) {
+      res.hidden = true;
+    }
+
+    return res;
   });
 };
 
@@ -291,6 +377,7 @@ const parseNodesAndLinks = (traces) => {
         label: id.endsWith("*") ? "*" : id,
         x: trace.id,
         y: hop.number / 2,
+        traceId: trace.id,
         size: 9,
         labelSize: 30,
         color: color(trace.id),
@@ -299,12 +386,12 @@ const parseNodesAndLinks = (traces) => {
       if (prevId) {
         // New link
         links.push({
-          label: "asd",
+          label: hop.link_latency,
           source: prevId,
           target: id,
           traceId: trace.id,
           origin: trace.origin,
-          size: 1,
+          size: 3,
           color: color(trace.id),
         });
       }
@@ -319,6 +406,7 @@ const parseNodesAndLinks = (traces) => {
     nodes.push({
       id: trace.id,
       label: trace.target,
+      traceId: trace.id,
       x: trace.id,
       y: (latestNumber + 1) / 2,
       size: 8,
@@ -326,6 +414,9 @@ const parseNodesAndLinks = (traces) => {
     });
 
     links.push({
+      label: "-",
+      size: 8,
+      traceId: trace.id,
       source: prevId,
       target: trace.id,
     });