feat(vis-network): use scanner extractors

package.json

@@ -99,7 +99,7 @@
     "@nodesecure/ossf-scorecard-sdk": "^3.2.1",
     "@nodesecure/rc": "^5.0.0",
     "@nodesecure/report": "4.1.1",
-    "@nodesecure/scanner": "8.2.0",
+    "@nodesecure/scanner": "9.0.0",
     "@nodesecure/mama": "2.0.2",
     "@nodesecure/server": "1.0.0",
     "@nodesecure/utils": "^2.2.0",

workspaces/cache/package.json

@@ -23,7 +23,7 @@
   "dependencies": {
     "@nodesecure/flags": "3.0.3",
     "@nodesecure/js-x-ray": "11.1.0",
-    "@nodesecure/scanner": "8.2.0",
+    "@nodesecure/scanner": "9.0.0",
     "cacache": "20.0.3"
   }
 }

workspaces/vis-network/package.json

@@ -30,6 +30,6 @@
   },
   "devDependencies": {
     "@nodesecure/flags": "^3.0.3",
-    "@nodesecure/scanner": "8.2.0"
+    "@nodesecure/scanner": "9.0.0"
   }
 }

workspaces/vis-network/src/dataset.js

@@ -1,4 +1,5 @@
 // Import Third-party Dependencies
+import { Extractors } from "@nodesecure/scanner/extractors";
 import prettyBytes from "pretty-bytes";
 import { DataSet } from "vis-data";
 
@@ -93,87 +94,97 @@ export default class NodeSecureDataSet extends EventTarget {
         return acc;
       }, { names: new Set(), emails: new Set() });
 
-    const dataEntries = Object.entries(data.dependencies);
-    this.dependenciesCount = dataEntries.length;
+    const dependencies = Object.entries(data.dependencies);
+    this.dependenciesCount = dependencies.length;
 
     this.rawEdgesData = [];
     this.rawNodesData = [];
 
-    const rootDependency = dataEntries.find(([name]) => name === data.rootDependency.name);
-    const rootContributors = [
+    const rootDependency = dependencies.find(([name]) => name === data.rootDependency.name);
+    this.rootContributors = [
       rootDependency[1].metadata.author,
       ...rootDependency[1].metadata.maintainers,
       ...rootDependency[1].metadata.publishers
     ];
-    for (const [packageName, descriptor] of dataEntries) {
-      const contributors = [descriptor.metadata.author, ...descriptor.metadata.maintainers, ...descriptor.metadata.publishers];
-      for (const [currVersion, opt] of Object.entries(descriptor.versions)) {
-        const { id, usedBy, flags, size, uniqueLicenseIds, author, composition, warnings, links } = opt;
-        const filteredWarnings = warnings
-          .filter((row) => !this.warningsToIgnore.has(row.kind));
-        const hasWarnings = filteredWarnings.length > 0;
-
-        opt.name = packageName;
-        opt.version = currVersion;
-        opt.hidden = false;
-        opt.hasWarnings = hasWarnings;
-
-        this.computeExtension(composition.extensions);
-        this.computeLicense(uniqueLicenseIds);
-        this.computeAuthor(author, `${packageName}@${currVersion}`, contributors);
-
-        if (flags.includes("hasIndirectDependencies")) {
-          this.indirectDependencies++;
-        }
-        this.size += size;
-
-        const flagStr = utils.getFlagsEmojisInlined(
-          flags,
-          hasWarnings ? this.flagsToIgnore : new Set([...this.flagsToIgnore, "hasWarnings"])
-        );
-        const isFriendly = window.settings.config.showFriendlyDependencies & rootContributors.some(
-          (rootContributor) => contributors.some((contributor) => {
-            if (contributor === null || rootContributor === null) {
-              return false;
-            }
-            else if (contributor.email && contributor.email === rootContributor.email) {
-              return true;
-            }
-            else if (contributor.name && contributor.name === rootContributor.name) {
-              return true;
-            }
 
+    const extractor = new Extractors.Payload(data, [
+      new Extractors.Probes.Licenses(),
+      new Extractors.Probes.Extensions()
+    ]);
+
+    extractor.on("manifest", (currVersion, opt, { name, dependency }) => {
+      const contributors = [dependency.metadata.author, ...dependency.metadata.maintainers, ...dependency.metadata.publishers];
+      const packageName = name;
+      const { id, usedBy, flags, size, author, warnings, links } = opt;
+      const filteredWarnings = warnings
+        .filter((row) => !this.warningsToIgnore.has(row.kind));
+      const hasWarnings = filteredWarnings.length > 0;
+
+      opt.name = packageName;
+      opt.version = currVersion;
+      opt.hidden = false;
+      opt.hasWarnings = hasWarnings;
+
+      this.computeAuthor(author, `${packageName}@${currVersion}`, contributors);
+
+      if (flags.includes("hasIndirectDependencies")) {
+        this.indirectDependencies++;
+      }
+      this.size += size;
+
+      const flagStr = utils.getFlagsEmojisInlined(
+        flags,
+        hasWarnings ? this.flagsToIgnore : new Set([...this.flagsToIgnore, "hasWarnings"])
+      );
+      const isFriendly = window.settings.config.showFriendlyDependencies & this.rootContributors.some(
+        (rootContributor) => contributors.some((contributor) => {
+          if (contributor === null || rootContributor === null) {
             return false;
-          })
-        );
-        opt.isFriendly = isFriendly;
-        this.packages.push({
-          id,
-          name: packageName,
-          version: currVersion,
-          hasWarnings,
-          flags: flagStr.replace(/\s/g, ""),
-          links,
-          isFriendly
-        });
-
-        const label = `<b>${packageName}@${currVersion}</b>${flagStr}\n<b>[${prettyBytes(size)}]</b>`;
-        const color = utils.getNodeColor({
-          id,
-          hasWarnings,
-          isFriendly,
-          theme: this.theme.toUpperCase()
-        });
-        color.font.multi = "html";
-
-        this.linker.set(Number(id), opt);
-        this.rawNodesData.push(Object.assign({ id, label }, color));
-
-        for (const [name, version] of Object.entries(usedBy)) {
-          this.rawEdgesData.push({ from: id, to: data.dependencies[name].versions[version].id });
-        }
+          }
+          else if (contributor.email && contributor.email === rootContributor.email) {
+            return true;
+          }
+          else if (contributor.name && contributor.name === rootContributor.name) {
+            return true;
+          }
+
+          return false;
+        })
+      );
+      opt.isFriendly = isFriendly;
+
+      this.packages.push({
+        id,
+        name: packageName,
+        version: currVersion,
+        hasWarnings,
+        flags: flagStr.replace(/\s/g, ""),
+        links,
+        isFriendly
+      });
+
+      const label = `<b>${packageName}@${currVersion}</b>${flagStr}\n<b>[${prettyBytes(size)}]</b>`;
+      const color = utils.getNodeColor({
+        id,
+        hasWarnings,
+        isFriendly,
+        theme: this.theme.toUpperCase()
+      });
+      color.font.multi = "html";
+
+      this.linker.set(Number(id), opt);
+      this.rawNodesData.push(Object.assign({ id, label }, color));
+
+      for (const [name, version] of Object.entries(usedBy)) {
+        this.rawEdgesData.push({ from: id, to: this.data.dependencies[name].versions[version].id });
       }
-    }
+    });
+
+    const { extensions, licenses } = extractor.extractAndMerge();
+
+    this.extensions = extensions;
+    this.licenses = licenses;
+
     console.log("[NodeSecureDataSet] Initialization done!");
   }
 
@@ -187,20 +198,6 @@ export default class NodeSecureDataSet extends EventTarget {
     return null;
   }
 
-  computeExtension(extensions) {
-    for (const extName of extensions) {
-      if (extName !== "") {
-        this.extensions[extName] = Reflect.has(this.extensions, extName) ? ++this.extensions[extName] : 1;
-      }
-    }
-  }
-
-  computeLicense(uniqueLicenseIds) {
-    for (const licenseName of uniqueLicenseIds) {
-      this.licenses[licenseName] = Reflect.has(this.licenses, licenseName) ? ++this.licenses[licenseName] : 1;
-    }
-  }
-
   computeAuthor(author, spec, contributors = []) {
     if (author === null) {
       return;

workspaces/vis-network/test/dataset-payload.json

@@ -64,7 +64,7 @@
           },
           "licenses": [],
           "uniqueLicenseIds": [
-            "Unlicense"
+            "MIT"
           ],
           "name": "pkg2",
           "version": "1.0.3"
@@ -87,7 +87,7 @@
           },
           "licenses": [],
           "uniqueLicenseIds": [
-            "Unlicense"
+            "MIT"
           ],
           "name": "pkg2",
           "version": "1.0.4"
@@ -127,7 +127,7 @@
           },
           "licenses": [],
           "uniqueLicenseIds": [
-            "Licence1"
+            "RND"
           ]
         }
       }
@@ -149,7 +149,8 @@
           ],
           "size": 200,
           "author": {
-            "name": "john doe"
+            "name": "john doe",
+            "email": "dummy@email.com"
           },
           "composition": {
             "extensions": [

workspaces/vis-network/test/dataset.test.js

@@ -1,6 +1,6 @@
 // Import Node.js Dependencies
-import { test } from "node:test";
 import assert from "node:assert";
+import { test } from "node:test";
 
 // Import Internal Dependencies
 import NodeSecureDataSet from "../src/dataset.js";
@@ -32,20 +32,27 @@ test("NodeSecureDataSet.init should fetch data & flags from the network", async(
   assert.equal(nsDataSet.FLAGS, "FLAG_01", "should fetch flags");
 });
 
-test("NodeSecureDataSet.prettySize", () => {
-  const nsDataSet = new NodeSecureDataSet();
-  nsDataSet.size = 1337;
-  assert.equal(nsDataSet.prettySize, "1.34 kB", "should convert bytes to human readable string");
-});
-
-test("NodeSecureDataSet.computeExtensions", () => {
+test("NodeSecureDataSet.init should compute extensions", async() => {
   const nsDataSet = new NodeSecureDataSet();
   assert.equal(Object.keys(nsDataSet.extensions).length, 0, "should have 0 extensions");
 
-  nsDataSet.computeExtension([".js", ".js", ".json"]);
+  await nsDataSet.init();
 
-  assert.equal(Object.keys(nsDataSet.extensions).length, 2, "should have 2 extension (js and json)");
-  assert.equal(nsDataSet.extensions[".js"], 2, "should have 2 '.js' extensions'");
+  assert.equal(Object.keys(nsDataSet.extensions).length, 5, "should have 2 extension (js and json)");
+  assert.equal(nsDataSet.extensions[".js"], 4, "should have 2 '.js' extensions'");
+});
+
+test("NodeSecureDataSet.init should compute licenses", async() => {
+  const nsDataSet = new NodeSecureDataSet();
+  await nsDataSet.init();
+  assert.equal(Object.keys(nsDataSet.licenses).length, 2, "should have 2 licenses (MIT, RND)");
+  assert.equal(nsDataSet.licenses.MIT, 2, "should have 2 MIT licenses");
+  assert.equal(nsDataSet.licenses.RND, 1, "should have 1 RND license");
+});
+test("NodeSecureDataSet.prettySize", () => {
+  const nsDataSet = new NodeSecureDataSet();
+  nsDataSet.size = 1337;
+  assert.equal(nsDataSet.prettySize, "1.34 kB", "should convert bytes to human readable string");
 });
 
 test("NodeSecureDataSet.isHighlighted", async() => {
@@ -63,13 +70,6 @@ test("NodeSecureDataSet.isHighlighted", async() => {
     "email: gentilhomme.thomas@gmail.com should be hightlighted");
 });
 
-test("NodeSecureDataSet.computeLicenses", () => {
-  const nsDataSet = new NodeSecureDataSet();
-  nsDataSet.computeLicense(["MIT", "MIT", "RND"]);
-  assert.equal(Object.keys(nsDataSet.licenses).length, 3, "should have 3 licenses (MIT, RND & 1 unknown)");
-  assert.equal(nsDataSet.licenses.MIT, 2, "should have 2 MIT licenses");
-});
-
 test("NodeSecureDataSet.computeAuthors", () => {
   const nsDataSet = new NodeSecureDataSet();
   nsDataSet.computeAuthor({ name: "John Doe" }, "pkg@1.1");