Render Arrow record batches progressively

R/html_dependencies.R

@@ -32,7 +32,10 @@ helpersDependency = function() {
       "rdeckglgeoarrowHelpers"
       , '0.0.1'
       , src = system.file("htmlwidgets", package = "deckglgeoarrow")
-      , script = "rdeckglgeoarrowHelpers.js"
+      , script = c(
+        "rdeckglgeoarrowHelpers.js"
+        , "progressiveArrowBatches.js"
+      )
       , stylesheet = 'css/deckglgeoarrow.css'
     )
   )

inst/htmlwidgets/addGeoArrowDeckglPathLayer.js

@@ -35,37 +35,8 @@ addGeoArrowDeckglPathLayer = function(map, opts) {
 
   opts.extension_type = guessExtension(data_fl.href);
 
-  fetch(data_fl.href)
-    .then(result => {
-      return parse2ArrowTable(result, opts.extension_type);
-    })
-    .then(arrow_table => {
-
-      const pathlayers = [];
-      let len = arrow_table.batches.length;
-      let batch = {};
-      let id = [];
-
-      for (let i = 0; i < len; i++) {
-
-        batch = arrow_table.batches[i];
-        id = `${opts.layerId}-${i}`;
-        pathlayers.push(pathLayer(map, opts, batch, id));
-
-      }
-
-     // does the mapboxoverlay already have layer(s)?
-      if (deckoverlay._props.layers.length ===  0) {
-        deckoverlay.setProps({ layers: pathlayers })
-      } else {
-        let lrs = deckoverlay._props.layers.concat(pathlayers);
-        lrs = lrs.sort(function(a, b) {
-          return a.props.zIndex - b.props.zIndex;
-        });
-        deckoverlay.setProps({ layers: lrs });
-      }
-
-    });
+  loadGeoArrowLayers(deckoverlay, data_fl.href, opts,
+    (batch, id) => pathLayer(map, opts, batch, id));
 
   map.on("projectiontransition", () => {
     deckoverlay._updateViewState();

inst/htmlwidgets/addGeoArrowDeckglPolygonLayer.js

@@ -41,37 +41,8 @@ addGeoArrowDeckglPolygonLayer = function(map, opts) {
 
   opts.extension_type = guessExtension(data_fl.href);
 
-  fetch(data_fl.href)
-    .then(result => {
-      return parse2ArrowTable(result, opts.extension_type);
-    })
-    .then(arrow_table => {
-
-      const polygonlayers = [];
-      let len = arrow_table.batches.length;
-      let batch = {};
-      let id = [];
-
-      for (let i = 0; i < len; i++) {
-
-        batch = arrow_table.batches[i];
-        id = `${opts.layerId}-${i}`;
-        polygonlayers.push(polygonLayer(map, opts, batch, id));
-
-      }
-
-      // does the mapboxoverlay already have layer(s)?
-      if (deckoverlay._props.layers.length ===  0) {
-        deckoverlay.setProps({ layers: polygonlayers })
-      } else {
-        let lrs = deckoverlay._props.layers.concat(polygonlayers);
-        lrs = lrs.sort(function(a, b) {
-          return a.props.zIndex - b.props.zIndex;
-        });
-        deckoverlay.setProps({ layers: lrs });
-      }
-
-    });
+  loadGeoArrowLayers(deckoverlay, data_fl.href, opts,
+    (batch, id) => polygonLayer(map, opts, batch, id));
 
   map.on("projectiontransition", () => {
     deckoverlay._updateViewState();

inst/htmlwidgets/addGeoArrowDeckglS2Layer.js

@@ -41,37 +41,8 @@ addGeoArrowDeckglS2Layer = function(map, opts) {
 
   opts.extension_type = guessExtension(data_fl.href);
 
-  fetch(data_fl.href)
-    .then(result => {
-      return parse2ArrowTable(result, opts.extension_type);
-    })
-    .then(arrow_table => {
-
-      const s2layers = [];
-      let len = arrow_table.batches.length;
-      let batch = {};
-      let id = [];
-
-      for (let i = 0; i < len; i++) {
-
-        batch = arrow_table.batches[i];
-        id = `${opts.layerId}-${i}`;
-        s2layers.push(s2Layer(map, opts, batch, id));
-
-      }
-
-      // does the mapboxoverlay already have layer(s)?
-      if (deckoverlay._props.layers.length ===  0) {
-        deckoverlay.setProps({ layers: s2layers });
-      } else {
-        let lrs = deckoverlay._props.layers.concat(s2layers);
-        lrs = lrs.sort(function(a, b) {
-          return a.props.zIndex - b.props.zIndex;
-        });
-        deckoverlay.setProps({ layers: lrs });
-      }
-
-    });
+  loadGeoArrowLayers(deckoverlay, data_fl.href, opts,
+    (batch, id) => s2Layer(map, opts, batch, id));
 
 
   map.on("projectiontransition", () => {

inst/htmlwidgets/addGeoArrowDeckglScatterplotLayer.js

@@ -42,37 +42,8 @@ addGeoArrowDeckglScatterplotLayer = function(map, opts) {
 
   opts.extension_type = guessExtension(data_fl.href);
 
-  fetch(data_fl.href)
-    .then(result => {
-      return parse2ArrowTable(result, opts.extension_type);
-    })
-    .then(arrow_table => {
-
-      const scatterlayers = [];
-      let len = arrow_table.batches.length;
-      let batch = {};
-      let id = [];
-
-      for (let i = 0; i < len; i++) {
-
-        batch = arrow_table.batches[i];
-        id = `${opts.layerId}-${i}`;
-        scatterlayers.push(scatterplotLayer(map, opts, batch, id));
-
-      }
-
-      // does the mapboxoverlay already have layer(s)?
-      if (deckoverlay._props.layers.length ===  0) {
-        deckoverlay.setProps({ layers: scatterlayers });
-      } else {
-        let lrs = deckoverlay._props.layers.concat(scatterlayers);
-        lrs = lrs.sort(function(a, b) {
-          return a.props.zIndex - b.props.zIndex;
-        });
-        deckoverlay.setProps({ layers: lrs });
-      }
-
-    });
+  loadGeoArrowLayers(deckoverlay, data_fl.href, opts,
+    (batch, id) => scatterplotLayer(map, opts, batch, id));
 
   map.on("projectiontransition", () => {
     deckoverlay._updateViewState();

inst/htmlwidgets/progressiveArrowBatches.js

@@ -0,0 +1,42 @@
+appendStableGeoArrowLayer = function(deckoverlay, layer) {
+  const existing = deckoverlay._props.layers || [];
+  if (existing.some(candidate => candidate.props.id === layer.props.id)) {
+    return false;
+  }
+
+  const layers = existing.concat([layer]).sort(
+    (left, right) => left.props.zIndex - right.props.zIndex
+  );
+  deckoverlay.setProps({layers: layers});
+  return true;
+};
+
+progressiveGeoArrowLayers = async function(deckoverlay, url, opts, layerFactory) {
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new Error(`Failed to fetch ${url}: HTTP ${response.status}`);
+  }
+
+  const reader = await Arrow.RecordBatchReader.from(response.body);
+  let index = 0;
+  for await (const batch of reader) {
+    const id = `${opts.layerId}-${index}`;
+    appendStableGeoArrowLayer(deckoverlay, layerFactory(batch, id));
+    index += 1;
+  }
+};
+
+loadGeoArrowLayers = async function(deckoverlay, url, opts, layerFactory) {
+  if (opts.extension_type === "arrow") {
+    return progressiveGeoArrowLayers(deckoverlay, url, opts, layerFactory);
+  }
+
+  const response = await fetch(url);
+  const table = await parse2ArrowTable(response, opts.extension_type);
+  if (table === null) return;
+
+  table.batches.forEach((batch, index) => {
+    const id = `${opts.layerId}-${index}`;
+    appendStableGeoArrowLayer(deckoverlay, layerFactory(batch, id));
+  });
+};

tests-js/progressive-arrow-batches.test.mjs

@@ -0,0 +1,80 @@
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import vm from "node:vm";
+
+globalThis.window = globalThis;
+
+const firstBatch = {name: "first"};
+const secondBatch = {name: "second"};
+const responseBody = {name: "response stream"};
+const original = {props: {id: "existing", zIndex: 0}};
+const overlay = {
+  _props: {layers: [original]},
+  history: [],
+  setProps(value) {
+    this._props = {...this._props, ...value};
+    this.history.push(value.layers.slice());
+  }
+};
+
+globalThis.fetch = async () => ({ok: true, body: responseBody});
+globalThis.Arrow = {
+  RecordBatchReader: {
+    from: async body => {
+      assert.strictEqual(body, responseBody);
+      return {
+        async *[Symbol.asyncIterator]() {
+          yield firstBatch;
+          assert.equal(overlay.history.length, 1);
+          yield secondBatch;
+        }
+      };
+    }
+  }
+};
+
+vm.runInThisContext(
+  fs.readFileSync("inst/htmlwidgets/progressiveArrowBatches.js", "utf8")
+);
+
+await loadGeoArrowLayers(
+  overlay,
+  "http://example/data.arrow",
+  {layerId: "points", extension_type: "arrow"},
+  (batch, id) => ({props: {id, zIndex: 1}, batch})
+);
+
+assert.equal(overlay.history.length, 2);
+assert.strictEqual(overlay.history[1][0], original);
+assert.deepEqual(
+  overlay.history[1].map(layer => layer.props.id),
+  ["existing", "points-0", "points-1"]
+);
+assert.strictEqual(overlay.history[0][1].batch, firstBatch);
+assert.strictEqual(overlay.history[1][2].batch, secondBatch);
+
+const fgbBatch = {name: "flatgeobuf"};
+const fgbOverlay = {
+  _props: {layers: []},
+  setProps(value) { this._props = {...this._props, ...value}; }
+};
+globalThis.parse2ArrowTable = async (response, extensionType) => {
+  assert.equal(response.ok, true);
+  assert.equal(extensionType, "fgb");
+  return {batches: [fgbBatch]};
+};
+
+await loadGeoArrowLayers(
+  fgbOverlay,
+  "http://example/data.fgb",
+  {layerId: "polygons", extension_type: "fgb"},
+  (batch, id) => ({props: {id, zIndex: 1}, batch})
+);
+
+assert.deepEqual(
+  fgbOverlay._props.layers.map(layer => layer.props.id),
+  ["polygons-0"]
+);
+assert.strictEqual(fgbOverlay._props.layers[0].batch, fgbBatch);
+
+console.log("progressive Arrow batch tests passed");