diff --git a/R/html_dependencies.R b/R/html_dependencies.R
index a927e16..5738c67 100644
--- a/R/html_dependencies.R
+++ b/R/html_dependencies.R
@@ -32,7 +32,10 @@ helpersDependency = function() {
"rdeckglgeoarrowHelpers"
, '0.0.1'
, src = system.file("htmlwidgets", package = "deckglgeoarrow")
- , script = "rdeckglgeoarrowHelpers.js"
+ , script = c(
+ "rdeckglgeoarrowHelpers.js"
+ , "progressiveArrowBatches.js"
+ )
, stylesheet = 'css/deckglgeoarrow.css'
)
)
diff --git a/inst/htmlwidgets/addGeoArrowDeckglPathLayer.js b/inst/htmlwidgets/addGeoArrowDeckglPathLayer.js
index 59bb5f9..93ad079 100644
--- a/inst/htmlwidgets/addGeoArrowDeckglPathLayer.js
+++ b/inst/htmlwidgets/addGeoArrowDeckglPathLayer.js
@@ -35,37 +35,8 @@ addGeoArrowDeckglPathLayer = function(map, opts) {
opts.extension_type = guessExtension(data_fl.href);
- fetch(data_fl.href)
- .then(result => {
- return parse2ArrowTable(result, opts.extension_type);
- })
- .then(arrow_table => {
-
- const pathlayers = [];
- let len = arrow_table.batches.length;
- let batch = {};
- let id = [];
-
- for (let i = 0; i < len; i++) {
-
- batch = arrow_table.batches[i];
- id = `${opts.layerId}-${i}`;
- pathlayers.push(pathLayer(map, opts, batch, id));
-
- }
-
- // does the mapboxoverlay already have layer(s)?
- if (deckoverlay._props.layers.length === 0) {
- deckoverlay.setProps({ layers: pathlayers })
- } else {
- let lrs = deckoverlay._props.layers.concat(pathlayers);
- lrs = lrs.sort(function(a, b) {
- return a.props.zIndex - b.props.zIndex;
- });
- deckoverlay.setProps({ layers: lrs });
- }
-
- });
+ loadGeoArrowLayers(deckoverlay, data_fl.href, opts,
+ (batch, id) => pathLayer(map, opts, batch, id));
map.on("projectiontransition", () => {
deckoverlay._updateViewState();
diff --git a/inst/htmlwidgets/addGeoArrowDeckglPolygonLayer.js b/inst/htmlwidgets/addGeoArrowDeckglPolygonLayer.js
index ef6160c..b4b7aee 100644
--- a/inst/htmlwidgets/addGeoArrowDeckglPolygonLayer.js
+++ b/inst/htmlwidgets/addGeoArrowDeckglPolygonLayer.js
@@ -41,37 +41,8 @@ addGeoArrowDeckglPolygonLayer = function(map, opts) {
opts.extension_type = guessExtension(data_fl.href);
- fetch(data_fl.href)
- .then(result => {
- return parse2ArrowTable(result, opts.extension_type);
- })
- .then(arrow_table => {
-
- const polygonlayers = [];
- let len = arrow_table.batches.length;
- let batch = {};
- let id = [];
-
- for (let i = 0; i < len; i++) {
-
- batch = arrow_table.batches[i];
- id = `${opts.layerId}-${i}`;
- polygonlayers.push(polygonLayer(map, opts, batch, id));
-
- }
-
- // does the mapboxoverlay already have layer(s)?
- if (deckoverlay._props.layers.length === 0) {
- deckoverlay.setProps({ layers: polygonlayers })
- } else {
- let lrs = deckoverlay._props.layers.concat(polygonlayers);
- lrs = lrs.sort(function(a, b) {
- return a.props.zIndex - b.props.zIndex;
- });
- deckoverlay.setProps({ layers: lrs });
- }
-
- });
+ loadGeoArrowLayers(deckoverlay, data_fl.href, opts,
+ (batch, id) => polygonLayer(map, opts, batch, id));
map.on("projectiontransition", () => {
deckoverlay._updateViewState();
diff --git a/inst/htmlwidgets/addGeoArrowDeckglS2Layer.js b/inst/htmlwidgets/addGeoArrowDeckglS2Layer.js
index 787910e..efec793 100644
--- a/inst/htmlwidgets/addGeoArrowDeckglS2Layer.js
+++ b/inst/htmlwidgets/addGeoArrowDeckglS2Layer.js
@@ -41,37 +41,8 @@ addGeoArrowDeckglS2Layer = function(map, opts) {
opts.extension_type = guessExtension(data_fl.href);
- fetch(data_fl.href)
- .then(result => {
- return parse2ArrowTable(result, opts.extension_type);
- })
- .then(arrow_table => {
-
- const s2layers = [];
- let len = arrow_table.batches.length;
- let batch = {};
- let id = [];
-
- for (let i = 0; i < len; i++) {
-
- batch = arrow_table.batches[i];
- id = `${opts.layerId}-${i}`;
- s2layers.push(s2Layer(map, opts, batch, id));
-
- }
-
- // does the mapboxoverlay already have layer(s)?
- if (deckoverlay._props.layers.length === 0) {
- deckoverlay.setProps({ layers: s2layers });
- } else {
- let lrs = deckoverlay._props.layers.concat(s2layers);
- lrs = lrs.sort(function(a, b) {
- return a.props.zIndex - b.props.zIndex;
- });
- deckoverlay.setProps({ layers: lrs });
- }
-
- });
+ loadGeoArrowLayers(deckoverlay, data_fl.href, opts,
+ (batch, id) => s2Layer(map, opts, batch, id));
map.on("projectiontransition", () => {
diff --git a/inst/htmlwidgets/addGeoArrowDeckglScatterplotLayer.js b/inst/htmlwidgets/addGeoArrowDeckglScatterplotLayer.js
index 139ca1d..b625e88 100644
--- a/inst/htmlwidgets/addGeoArrowDeckglScatterplotLayer.js
+++ b/inst/htmlwidgets/addGeoArrowDeckglScatterplotLayer.js
@@ -42,37 +42,8 @@ addGeoArrowDeckglScatterplotLayer = function(map, opts) {
opts.extension_type = guessExtension(data_fl.href);
- fetch(data_fl.href)
- .then(result => {
- return parse2ArrowTable(result, opts.extension_type);
- })
- .then(arrow_table => {
-
- const scatterlayers = [];
- let len = arrow_table.batches.length;
- let batch = {};
- let id = [];
-
- for (let i = 0; i < len; i++) {
-
- batch = arrow_table.batches[i];
- id = `${opts.layerId}-${i}`;
- scatterlayers.push(scatterplotLayer(map, opts, batch, id));
-
- }
-
- // does the mapboxoverlay already have layer(s)?
- if (deckoverlay._props.layers.length === 0) {
- deckoverlay.setProps({ layers: scatterlayers });
- } else {
- let lrs = deckoverlay._props.layers.concat(scatterlayers);
- lrs = lrs.sort(function(a, b) {
- return a.props.zIndex - b.props.zIndex;
- });
- deckoverlay.setProps({ layers: lrs });
- }
-
- });
+ loadGeoArrowLayers(deckoverlay, data_fl.href, opts,
+ (batch, id) => scatterplotLayer(map, opts, batch, id));
map.on("projectiontransition", () => {
deckoverlay._updateViewState();
diff --git a/inst/htmlwidgets/progressiveArrowBatches.js b/inst/htmlwidgets/progressiveArrowBatches.js
new file mode 100644
index 0000000..1894e45
--- /dev/null
+++ b/inst/htmlwidgets/progressiveArrowBatches.js
@@ -0,0 +1,42 @@
+appendStableGeoArrowLayer = function(deckoverlay, layer) {
+ const existing = deckoverlay._props.layers || [];
+ if (existing.some(candidate => candidate.props.id === layer.props.id)) {
+ return false;
+ }
+
+ const layers = existing.concat([layer]).sort(
+ (left, right) => left.props.zIndex - right.props.zIndex
+ );
+ deckoverlay.setProps({layers: layers});
+ return true;
+};
+
+progressiveGeoArrowLayers = async function(deckoverlay, url, opts, layerFactory) {
+ const response = await fetch(url);
+ if (!response.ok) {
+ throw new Error(`Failed to fetch ${url}: HTTP ${response.status}`);
+ }
+
+ const reader = await Arrow.RecordBatchReader.from(response.body);
+ let index = 0;
+ for await (const batch of reader) {
+ const id = `${opts.layerId}-${index}`;
+ appendStableGeoArrowLayer(deckoverlay, layerFactory(batch, id));
+ index += 1;
+ }
+};
+
+loadGeoArrowLayers = async function(deckoverlay, url, opts, layerFactory) {
+ if (opts.extension_type === "arrow") {
+ return progressiveGeoArrowLayers(deckoverlay, url, opts, layerFactory);
+ }
+
+ const response = await fetch(url);
+ const table = await parse2ArrowTable(response, opts.extension_type);
+ if (table === null) return;
+
+ table.batches.forEach((batch, index) => {
+ const id = `${opts.layerId}-${index}`;
+ appendStableGeoArrowLayer(deckoverlay, layerFactory(batch, id));
+ });
+};
diff --git a/tests-js/progressive-arrow-batches.test.mjs b/tests-js/progressive-arrow-batches.test.mjs
new file mode 100644
index 0000000..ee7c3c5
--- /dev/null
+++ b/tests-js/progressive-arrow-batches.test.mjs
@@ -0,0 +1,80 @@
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import vm from "node:vm";
+
+globalThis.window = globalThis;
+
+const firstBatch = {name: "first"};
+const secondBatch = {name: "second"};
+const responseBody = {name: "response stream"};
+const original = {props: {id: "existing", zIndex: 0}};
+const overlay = {
+ _props: {layers: [original]},
+ history: [],
+ setProps(value) {
+ this._props = {...this._props, ...value};
+ this.history.push(value.layers.slice());
+ }
+};
+
+globalThis.fetch = async () => ({ok: true, body: responseBody});
+globalThis.Arrow = {
+ RecordBatchReader: {
+ from: async body => {
+ assert.strictEqual(body, responseBody);
+ return {
+ async *[Symbol.asyncIterator]() {
+ yield firstBatch;
+ assert.equal(overlay.history.length, 1);
+ yield secondBatch;
+ }
+ };
+ }
+ }
+};
+
+vm.runInThisContext(
+ fs.readFileSync("inst/htmlwidgets/progressiveArrowBatches.js", "utf8")
+);
+
+await loadGeoArrowLayers(
+ overlay,
+ "http://example/data.arrow",
+ {layerId: "points", extension_type: "arrow"},
+ (batch, id) => ({props: {id, zIndex: 1}, batch})
+);
+
+assert.equal(overlay.history.length, 2);
+assert.strictEqual(overlay.history[1][0], original);
+assert.deepEqual(
+ overlay.history[1].map(layer => layer.props.id),
+ ["existing", "points-0", "points-1"]
+);
+assert.strictEqual(overlay.history[0][1].batch, firstBatch);
+assert.strictEqual(overlay.history[1][2].batch, secondBatch);
+
+const fgbBatch = {name: "flatgeobuf"};
+const fgbOverlay = {
+ _props: {layers: []},
+ setProps(value) { this._props = {...this._props, ...value}; }
+};
+globalThis.parse2ArrowTable = async (response, extensionType) => {
+ assert.equal(response.ok, true);
+ assert.equal(extensionType, "fgb");
+ return {batches: [fgbBatch]};
+};
+
+await loadGeoArrowLayers(
+ fgbOverlay,
+ "http://example/data.fgb",
+ {layerId: "polygons", extension_type: "fgb"},
+ (batch, id) => ({props: {id, zIndex: 1}, batch})
+);
+
+assert.deepEqual(
+ fgbOverlay._props.layers.map(layer => layer.props.id),
+ ["polygons-0"]
+);
+assert.strictEqual(fgbOverlay._props.layers[0].batch, fgbBatch);
+
+console.log("progressive Arrow batch tests passed");