From ad4631a63d3249723033d42236de48175d059cca Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Thu, 17 Sep 2026 16:02:31 +0200 Subject: [PATCH] feat: render Arrow record batches progressively --- R/html_dependencies.R | 5 +- .../htmlwidgets/addGeoArrowDeckglPathLayer.js | 33 +------- .../addGeoArrowDeckglPolygonLayer.js | 33 +------- inst/htmlwidgets/addGeoArrowDeckglS2Layer.js | 33 +------- .../addGeoArrowDeckglScatterplotLayer.js | 33 +------- inst/htmlwidgets/progressiveArrowBatches.js | 42 ++++++++++ tests-js/progressive-arrow-batches.test.mjs | 80 +++++++++++++++++++ 7 files changed, 134 insertions(+), 125 deletions(-) create mode 100644 inst/htmlwidgets/progressiveArrowBatches.js create mode 100644 tests-js/progressive-arrow-batches.test.mjs diff --git a/R/html_dependencies.R b/R/html_dependencies.R index a927e16..5738c67 100644 --- a/R/html_dependencies.R +++ b/R/html_dependencies.R @@ -32,7 +32,10 @@ helpersDependency = function() { "rdeckglgeoarrowHelpers" , '0.0.1' , src = system.file("htmlwidgets", package = "deckglgeoarrow") - , script = "rdeckglgeoarrowHelpers.js" + , script = c( + "rdeckglgeoarrowHelpers.js" + , "progressiveArrowBatches.js" + ) , stylesheet = 'css/deckglgeoarrow.css' ) ) diff --git a/inst/htmlwidgets/addGeoArrowDeckglPathLayer.js b/inst/htmlwidgets/addGeoArrowDeckglPathLayer.js index 59bb5f9..93ad079 100644 --- a/inst/htmlwidgets/addGeoArrowDeckglPathLayer.js +++ b/inst/htmlwidgets/addGeoArrowDeckglPathLayer.js @@ -35,37 +35,8 @@ addGeoArrowDeckglPathLayer = function(map, opts) { opts.extension_type = guessExtension(data_fl.href); - fetch(data_fl.href) - .then(result => { - return parse2ArrowTable(result, opts.extension_type); - }) - .then(arrow_table => { - - const pathlayers = []; - let len = arrow_table.batches.length; - let batch = {}; - let id = []; - - for (let i = 0; i < len; i++) { - - batch = arrow_table.batches[i]; - id = `${opts.layerId}-${i}`; - pathlayers.push(pathLayer(map, opts, batch, id)); - - } - - // does the mapboxoverlay already have layer(s)? - if (deckoverlay._props.layers.length === 0) { - deckoverlay.setProps({ layers: pathlayers }) - } else { - let lrs = deckoverlay._props.layers.concat(pathlayers); - lrs = lrs.sort(function(a, b) { - return a.props.zIndex - b.props.zIndex; - }); - deckoverlay.setProps({ layers: lrs }); - } - - }); + loadGeoArrowLayers(deckoverlay, data_fl.href, opts, + (batch, id) => pathLayer(map, opts, batch, id)); map.on("projectiontransition", () => { deckoverlay._updateViewState(); diff --git a/inst/htmlwidgets/addGeoArrowDeckglPolygonLayer.js b/inst/htmlwidgets/addGeoArrowDeckglPolygonLayer.js index ef6160c..b4b7aee 100644 --- a/inst/htmlwidgets/addGeoArrowDeckglPolygonLayer.js +++ b/inst/htmlwidgets/addGeoArrowDeckglPolygonLayer.js @@ -41,37 +41,8 @@ addGeoArrowDeckglPolygonLayer = function(map, opts) { opts.extension_type = guessExtension(data_fl.href); - fetch(data_fl.href) - .then(result => { - return parse2ArrowTable(result, opts.extension_type); - }) - .then(arrow_table => { - - const polygonlayers = []; - let len = arrow_table.batches.length; - let batch = {}; - let id = []; - - for (let i = 0; i < len; i++) { - - batch = arrow_table.batches[i]; - id = `${opts.layerId}-${i}`; - polygonlayers.push(polygonLayer(map, opts, batch, id)); - - } - - // does the mapboxoverlay already have layer(s)? - if (deckoverlay._props.layers.length === 0) { - deckoverlay.setProps({ layers: polygonlayers }) - } else { - let lrs = deckoverlay._props.layers.concat(polygonlayers); - lrs = lrs.sort(function(a, b) { - return a.props.zIndex - b.props.zIndex; - }); - deckoverlay.setProps({ layers: lrs }); - } - - }); + loadGeoArrowLayers(deckoverlay, data_fl.href, opts, + (batch, id) => polygonLayer(map, opts, batch, id)); map.on("projectiontransition", () => { deckoverlay._updateViewState(); diff --git a/inst/htmlwidgets/addGeoArrowDeckglS2Layer.js b/inst/htmlwidgets/addGeoArrowDeckglS2Layer.js index 787910e..efec793 100644 --- a/inst/htmlwidgets/addGeoArrowDeckglS2Layer.js +++ b/inst/htmlwidgets/addGeoArrowDeckglS2Layer.js @@ -41,37 +41,8 @@ addGeoArrowDeckglS2Layer = function(map, opts) { opts.extension_type = guessExtension(data_fl.href); - fetch(data_fl.href) - .then(result => { - return parse2ArrowTable(result, opts.extension_type); - }) - .then(arrow_table => { - - const s2layers = []; - let len = arrow_table.batches.length; - let batch = {}; - let id = []; - - for (let i = 0; i < len; i++) { - - batch = arrow_table.batches[i]; - id = `${opts.layerId}-${i}`; - s2layers.push(s2Layer(map, opts, batch, id)); - - } - - // does the mapboxoverlay already have layer(s)? - if (deckoverlay._props.layers.length === 0) { - deckoverlay.setProps({ layers: s2layers }); - } else { - let lrs = deckoverlay._props.layers.concat(s2layers); - lrs = lrs.sort(function(a, b) { - return a.props.zIndex - b.props.zIndex; - }); - deckoverlay.setProps({ layers: lrs }); - } - - }); + loadGeoArrowLayers(deckoverlay, data_fl.href, opts, + (batch, id) => s2Layer(map, opts, batch, id)); map.on("projectiontransition", () => { diff --git a/inst/htmlwidgets/addGeoArrowDeckglScatterplotLayer.js b/inst/htmlwidgets/addGeoArrowDeckglScatterplotLayer.js index 139ca1d..b625e88 100644 --- a/inst/htmlwidgets/addGeoArrowDeckglScatterplotLayer.js +++ b/inst/htmlwidgets/addGeoArrowDeckglScatterplotLayer.js @@ -42,37 +42,8 @@ addGeoArrowDeckglScatterplotLayer = function(map, opts) { opts.extension_type = guessExtension(data_fl.href); - fetch(data_fl.href) - .then(result => { - return parse2ArrowTable(result, opts.extension_type); - }) - .then(arrow_table => { - - const scatterlayers = []; - let len = arrow_table.batches.length; - let batch = {}; - let id = []; - - for (let i = 0; i < len; i++) { - - batch = arrow_table.batches[i]; - id = `${opts.layerId}-${i}`; - scatterlayers.push(scatterplotLayer(map, opts, batch, id)); - - } - - // does the mapboxoverlay already have layer(s)? - if (deckoverlay._props.layers.length === 0) { - deckoverlay.setProps({ layers: scatterlayers }); - } else { - let lrs = deckoverlay._props.layers.concat(scatterlayers); - lrs = lrs.sort(function(a, b) { - return a.props.zIndex - b.props.zIndex; - }); - deckoverlay.setProps({ layers: lrs }); - } - - }); + loadGeoArrowLayers(deckoverlay, data_fl.href, opts, + (batch, id) => scatterplotLayer(map, opts, batch, id)); map.on("projectiontransition", () => { deckoverlay._updateViewState(); diff --git a/inst/htmlwidgets/progressiveArrowBatches.js b/inst/htmlwidgets/progressiveArrowBatches.js new file mode 100644 index 0000000..1894e45 --- /dev/null +++ b/inst/htmlwidgets/progressiveArrowBatches.js @@ -0,0 +1,42 @@ +appendStableGeoArrowLayer = function(deckoverlay, layer) { + const existing = deckoverlay._props.layers || []; + if (existing.some(candidate => candidate.props.id === layer.props.id)) { + return false; + } + + const layers = existing.concat([layer]).sort( + (left, right) => left.props.zIndex - right.props.zIndex + ); + deckoverlay.setProps({layers: layers}); + return true; +}; + +progressiveGeoArrowLayers = async function(deckoverlay, url, opts, layerFactory) { + const response = await fetch(url); + if (!response.ok) { + throw new Error(`Failed to fetch ${url}: HTTP ${response.status}`); + } + + const reader = await Arrow.RecordBatchReader.from(response.body); + let index = 0; + for await (const batch of reader) { + const id = `${opts.layerId}-${index}`; + appendStableGeoArrowLayer(deckoverlay, layerFactory(batch, id)); + index += 1; + } +}; + +loadGeoArrowLayers = async function(deckoverlay, url, opts, layerFactory) { + if (opts.extension_type === "arrow") { + return progressiveGeoArrowLayers(deckoverlay, url, opts, layerFactory); + } + + const response = await fetch(url); + const table = await parse2ArrowTable(response, opts.extension_type); + if (table === null) return; + + table.batches.forEach((batch, index) => { + const id = `${opts.layerId}-${index}`; + appendStableGeoArrowLayer(deckoverlay, layerFactory(batch, id)); + }); +}; diff --git a/tests-js/progressive-arrow-batches.test.mjs b/tests-js/progressive-arrow-batches.test.mjs new file mode 100644 index 0000000..ee7c3c5 --- /dev/null +++ b/tests-js/progressive-arrow-batches.test.mjs @@ -0,0 +1,80 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import vm from "node:vm"; + +globalThis.window = globalThis; + +const firstBatch = {name: "first"}; +const secondBatch = {name: "second"}; +const responseBody = {name: "response stream"}; +const original = {props: {id: "existing", zIndex: 0}}; +const overlay = { + _props: {layers: [original]}, + history: [], + setProps(value) { + this._props = {...this._props, ...value}; + this.history.push(value.layers.slice()); + } +}; + +globalThis.fetch = async () => ({ok: true, body: responseBody}); +globalThis.Arrow = { + RecordBatchReader: { + from: async body => { + assert.strictEqual(body, responseBody); + return { + async *[Symbol.asyncIterator]() { + yield firstBatch; + assert.equal(overlay.history.length, 1); + yield secondBatch; + } + }; + } + } +}; + +vm.runInThisContext( + fs.readFileSync("inst/htmlwidgets/progressiveArrowBatches.js", "utf8") +); + +await loadGeoArrowLayers( + overlay, + "http://example/data.arrow", + {layerId: "points", extension_type: "arrow"}, + (batch, id) => ({props: {id, zIndex: 1}, batch}) +); + +assert.equal(overlay.history.length, 2); +assert.strictEqual(overlay.history[1][0], original); +assert.deepEqual( + overlay.history[1].map(layer => layer.props.id), + ["existing", "points-0", "points-1"] +); +assert.strictEqual(overlay.history[0][1].batch, firstBatch); +assert.strictEqual(overlay.history[1][2].batch, secondBatch); + +const fgbBatch = {name: "flatgeobuf"}; +const fgbOverlay = { + _props: {layers: []}, + setProps(value) { this._props = {...this._props, ...value}; } +}; +globalThis.parse2ArrowTable = async (response, extensionType) => { + assert.equal(response.ok, true); + assert.equal(extensionType, "fgb"); + return {batches: [fgbBatch]}; +}; + +await loadGeoArrowLayers( + fgbOverlay, + "http://example/data.fgb", + {layerId: "polygons", extension_type: "fgb"}, + (batch, id) => ({props: {id, zIndex: 1}, batch}) +); + +assert.deepEqual( + fgbOverlay._props.layers.map(layer => layer.props.id), + ["polygons-0"] +); +assert.strictEqual(fgbOverlay._props.layers[0].batch, fgbBatch); + +console.log("progressive Arrow batch tests passed");