feat: Add GeoArrow WKB/WKT to GeoJSON decoding.

.github/workflows/test.yml

@@ -12,7 +12,7 @@ jobs:
 
     strategy:
       matrix:
-        node: [24, 25]
+        node: [24, 26]
 
     name: Node ${{ matrix.node }}
 

README.md

@@ -118,6 +118,7 @@ const table = tableFromIPC(ipc, {
   useDecimalInt: true,      // use BigInt for decimals, do not coerce to number
   useBigInt: true,          // use BigInt for 64-bit ints, do not coerce to number
   useBigIntTimestamp: true, // use BigInt for timestamps, do not coerce to millisecond number
+  useGeoJSON: true,         // decode GeoArrow WKB/WKT geometry columns to GeoJSON objects
   useMap: true,             // create Map objects for [key, value] pair lists
   useProxy: true            // use zero-copy proxies for struct and table row objects
 });

docs/api/data-types.md

@@ -15,8 +15,8 @@ The table below provides an overview of all data types supported by the Apache A
 |   1 | [Null](#null)                       | ✅ | ✅ | ✅ | `null` |
 |   2 | [Int](#int)                         | ✅ | ✅ | ✅ | `number`, or `bigint` for 64-bit values via the `useBigInt` flag |
 |   3 | [Float](#float)                     | ✅ | ✅ | ✅ | `number` |
-|   4 | [Binary](#binary)                   | ✅ | ✅ | ✅ | `Uint8Array` |
-|   5 | [Utf8](#utf8)                       | ✅ | ✅ | ✅ | `string` |
+|   4 | [Binary](#binary)                   | ✅ | ✅ | ✅ | `Uint8Array`, or GeoJSON for GeoArrow geometries via the `useGeoJSON` flag |
+|   5 | [Utf8](#utf8)                       | ✅ | ✅ | ✅ | `string`, or GeoJSON for GeoArrow geometries via the `useGeoJSON` flag |
 |   6 | [Bool](#bool)                       | ✅ | ✅ | ✅ | `boolean` |
 |   7 | [Decimal](#decimal)                 | ✅ | ✅ | ✅ | `number`, or scaled integers via the `useDecimalInt` flag |
 |   8 | [Date](#date)                       | ✅ | ✅ | ✅ | `number`, or `Date` via the `useDate` flag. |
@@ -30,12 +30,12 @@ The table below provides an overview of all data types supported by the Apache A
 |  16 | [FixedSizeList](#fixedSizeList)     | ✅ | ✅ | ✅ | `Array` or `TypedArray` of child type |
 |  17 | [Map](#map)                         | ✅ | ✅ | ✅ | `[key, value][]`, or `Map` via the `useMap` flag |
 |  18 | [Duration](#duration)               | ✅ | ✅ | ✅ | `number`, or `bigint` via the `useBigInt` flag |
-|  19 | [LargeBinary](#largeBinary)         | ✅ | ✅ | ✅ | `Uint8Array` |
-|  20 | [LargeUtf8](#largeUtf8)             | ✅ | ✅ | ✅ | `string` |
+|  19 | [LargeBinary](#largeBinary)         | ✅ | ✅ | ✅ | `Uint8Array`, or GeoJSON for GeoArrow geometries via the `useGeoJSON` flag |
+|  20 | [LargeUtf8](#largeUtf8)             | ✅ | ✅ | ✅ | `string`, or GeoJSON for GeoArrow geometries via the `useGeoJSON` flag |
 |  21 | [LargeList](#largeList)             | ✅ | ✅ | ✅ | `Array` or `TypedArray` of child type |
 |  22 | [RunEndEncoded](#runEndEncoded)     | ✅ | ✅ | ✅ | depends on child type |
-|  23 | [BinaryView](#binaryView)           | ✅ | ✅ | ❌ | `Uint8Array` |
-|  24 | [Utf8View](#utf8View)               | ✅ | ✅ | ❌ | `string` |
+|  23 | [BinaryView](#binaryView)           | ✅ | ✅ | ❌ | `Uint8Array`, or GeoJSON for GeoArrow geometries via the `useGeoJSON` flag |
+|  24 | [Utf8View](#utf8View)               | ✅ | ✅ | ❌ | `string`, or GeoJSON for GeoArrow geometries via the `useGeoJSON` flag |
 |  25 | [ListView](#listView)               | ✅ | ✅ | ❌ | `Array` or `TypedArray` of child type |
 |  26 | [LargeListView](#largeListView)     | ✅ | ✅ | ❌ | `Array` or `TypedArray` of child type |
 

docs/api/index.md

@@ -28,6 +28,7 @@ By default Flechette assumes input data is uncompressed. If input IPC data conta
   * *useBigIntTimestamp* (`boolean`): If true, extract timestamps as JavaScript `BigInt` values. Otherwise, coerce timestamps to millisecond numbers. This option takes precedence over the *useDate* option.
   * *useDate* (`boolean`): If true, extract dates and timestamps as JavaScript `Date` objects. Otherwise, return numerical timestamp values (default `false`). If *useBigIntTimestamp* is true, this option does not apply to timestamps.
   * *useDecimalInt* (`boolean`): If true, extract decimal-type data as scaled integer values, where fractional digits are scaled to integer positions. Returned integers are `BigInt` values for decimal bit widths of 64 bits or higher and 32-bit integers (as JavaScript `number`) otherwise. If false, decimals are lossily converted to floating-point numbers (default).
+  * *useGeoJSON* (`boolean`): If true, extract geometry data in [GeoArrow](https://geoarrow.org/) well-known binary (WKB) or well-known text (WKT) formats as [GeoJSON](https://datatracker.ietf.org/doc/html/rfc7946) geometry objects. Geometry columns are identified by the field metadata key `ARROW:extension:name` with value `geoarrow.wkb` or `geoarrow.wkt`. Otherwise, return raw WKB byte arrays or WKT strings (default `false`).
   * *useMap* (`boolean`): If true, extract Arrow 'Map' values as JavaScript `Map` instances. Otherwise, return an array of [key, value] pairs compatible with both `Map` and `Object.fromEntries` (default `false`).
   * *useProxy* (`boolean`): If true, extract Arrow 'Struct' values and table row objects using zero-copy proxy objects that extract data from underlying Arrow batches. The proxy objects can improve performance and reduce memory usage, but do not support property enumeration (`Object.keys`, `Object.values`, `Object.entries`) or spreading (`{ ...object }`). Otherwise (default `false`), use standard JS objects for structs and table rows.
 

docs/index.md

@@ -118,6 +118,7 @@ const table = tableFromIPC(ipc, {
   useDecimalInt: true,      // use scaled ints for decimals, not floating point
   useBigInt: true,          // use BigInt for 64-bit ints, do not coerce to number
   useBigIntTimestamp: true, // use BigInt for timestamps, do not coerce to millisecond number
+  useGeoJSON: true,         // decode GeoArrow WKB/WKT geometry columns to GeoJSON objects
   useMap: true,             // create Map objects for [key, value] pair lists
   useProxy: true            // use zero-copy proxies for struct and table row objects
 });

package.json

@@ -34,6 +34,9 @@
     "prepublishOnly": "pnpm run test && pnpm run lint && pnpm run build",
     "prepare": "pnpm run build"
   },
+  "dependencies": {
+    "@uwdata/wk-geojson": "^0.0.1"
+  },
   "devDependencies": {
     "apache-arrow": "^21.2.0",
     "lz4js": "0.2.0",

pnpm-workspace.yaml

@@ -0,0 +1,2 @@
+minimumReleaseAgeExclude:
+  - '@uwdata/wk-geojson'

src/batch-type.js

@@ -1,18 +1,23 @@
 /**
- * @import { DataType, ExtractionOptions } from './types.js';
+ * @import { DataType, ExtractionOptions, Metadata } from './types.js';
  */
-import { BinaryBatch, BinaryViewBatch, BoolBatch, DateBatch, DateDayBatch, DateDayMillisecondBatch, Decimal32NumberBatch, DecimalBigIntBatch, DecimalNumberBatch, DenseUnionBatch, DictionaryBatch, DirectBatch, FixedBinaryBatch, FixedListBatch, Float16Batch, Int64Batch, IntervalDayTimeBatch, IntervalMonthDayNanoBatch, LargeBinaryBatch, LargeListBatch, LargeListViewBatch, LargeUtf8Batch, ListBatch, ListViewBatch, MapBatch, MapEntryBatch, NullBatch, RunEndEncodedBatch, SparseUnionBatch, StructBatch, StructProxyBatch, TimestampMicrosecondBatch, TimestampMillisecondBatch, TimestampNanosecondBatch, TimestampSecondBatch, Utf8Batch, Utf8ViewBatch } from './batch.js';
+import { BinaryBatch, BinaryViewBatch, BoolBatch, DateBatch, DateDayBatch, DateDayMillisecondBatch, Decimal32NumberBatch, DecimalBigIntBatch, DecimalNumberBatch, DenseUnionBatch, DictionaryBatch, DirectBatch, FixedBinaryBatch, FixedListBatch, Float16Batch, Int64Batch, IntervalDayTimeBatch, IntervalMonthDayNanoBatch, LargeBinaryBatch, LargeListBatch, LargeListViewBatch, LargeUtf8Batch, ListBatch, ListViewBatch, MapBatch, MapEntryBatch, NullBatch, RunEndEncodedBatch, SparseUnionBatch, StructBatch, StructProxyBatch, TimestampMicrosecondBatch, TimestampMillisecondBatch, TimestampNanosecondBatch, TimestampSecondBatch, Utf8Batch, Utf8ViewBatch, WKBGeoJSONBatch, WKTGeoJSONBatch } from './batch.js';
 import { DateUnit, IntervalUnit, TimeUnit, Type } from './constants.js';
 import { invalidDataType } from './data-types.js';
 
 /**
  * Return a batch constructor for the given data type and extraction options.
  * @param {DataType} type The data type.
  * @param {ExtractionOptions} options The extraction options.
+ * @param {Metadata | null} [metadata] Custom field metadata, used to
+ *  identify extension types such as GeoArrow WKB and WKT geometries.
  */
-export function batchType(type, options = {}) {
+export function batchType(type, options = {}, metadata = null) {
   const { typeId, bitWidth, mode, precision, unit } = /** @type {any} */(type);
-  const { useBigInt, useBigIntTimestamp, useDate, useDecimalInt, useMap, useProxy } = options;
+  const { useBigInt, useBigIntTimestamp, useDate, useDecimalInt, useGeoJSON, useMap, useProxy } = options;
+  const ext = useGeoJSON ? metadata?.get('ARROW:extension:name') : null;
+  const wkb = ext === 'geoarrow.wkb' && WKBGeoJSONBatch;
+  const wkt = ext === 'geoarrow.wkt' && WKTGeoJSONBatch;
 
   switch (typeId) {
     case Type.Null: return NullBatch;
@@ -45,12 +50,12 @@ export function batchType(type, options = {}) {
         : unit === IntervalUnit.YEAR_MONTH ? DirectBatch
         : IntervalMonthDayNanoBatch;
     case Type.FixedSizeBinary: return FixedBinaryBatch;
-    case Type.Utf8: return Utf8Batch;
-    case Type.LargeUtf8: return LargeUtf8Batch;
-    case Type.Binary: return BinaryBatch;
-    case Type.LargeBinary: return LargeBinaryBatch;
-    case Type.BinaryView: return BinaryViewBatch;
-    case Type.Utf8View: return Utf8ViewBatch;
+    case Type.Utf8: return wrap(Utf8Batch, wkt);
+    case Type.LargeUtf8: return wrap(LargeUtf8Batch, wkt);
+    case Type.Binary: return wrap(BinaryBatch, wkb);
+    case Type.LargeBinary: return wrap(LargeBinaryBatch, wkb);
+    case Type.BinaryView: return wrap(BinaryViewBatch, wkb);
+    case Type.Utf8View: return wrap(Utf8ViewBatch, wkt);
     case Type.List: return ListBatch;
     case Type.LargeList: return LargeListBatch;
     case Type.Map: return useMap ? MapBatch : MapEntryBatch;

src/batch.js

@@ -1,7 +1,9 @@
 /**
+ * @import { Geometry } from '@uwdata/wk-geojson'
  * @import { Column } from './column.js'
  * @import { DataType, DecimalType, IntegerArray, OffsetArray, TypedArray, TypedArrayConstructor, ValueArray } from './types.js'
  */
+import { wkbToGeoJSON, wktToGeoJSON } from '@uwdata/wk-geojson';
 import { bisect, float64Array } from './util/arrays.js';
 import { divide, fromDecimal128, fromDecimal256, fromDecimal64, toNumber } from './util/numbers.js';
 import { decodeBit, readInt32, readInt64 } from './util/read.js';
@@ -515,6 +517,54 @@ export class LargeUtf8Batch extends ArrayBatch {
   }
 }
 
+/**
+ * A batch of geometry values in well-known binary (WKB) format that are
+ * decoded and returned as GeoJSON geometry objects. This batch wraps a
+ * source batch that provides WKB byte arrays.
+ * @extends {ArrayBatch<Geometry>}
+ */
+export class WKBGeoJSONBatch extends ArrayBatch {
+  /**
+   * Create a new WKB to GeoJSON batch.
+   * @param {Batch<Uint8Array>} batch A batch of WKB byte arrays.
+   */
+  constructor(batch) {
+    super(batch);
+    this.source = batch;
+  }
+
+  /**
+   * @param {number} index The value index
+   */
+  value(index) {
+    return wkbToGeoJSON(this.source.value(index));
+  }
+}
+
+/**
+ * A batch of geometry values in well-known text (WKT) format that are
+ * decoded and returned as GeoJSON geometry objects. This batch wraps a
+ * source batch that provides WKT strings.
+ * @extends {ArrayBatch<Geometry>}
+ */
+export class WKTGeoJSONBatch extends ArrayBatch {
+  /**
+   * Create a new WKT to GeoJSON batch.
+   * @param {Batch<string>} batch A batch of WKT strings.
+   */
+  constructor(batch) {
+    super(batch);
+    this.source = batch;
+  }
+
+  /**
+   * @param {number} index The value index
+   */
+  value(index) {
+    return wktToGeoJSON(this.source.value(index));
+  }
+}
+
 /**
  * A batch of list (array) values of variable length. The list offsets are
  * 32-bit ints.

src/decode/table-from-ipc.js

@@ -108,7 +108,7 @@ export function createTable(data, options = {}) {
     }
 
     const ctx = context(records[i]);
-    fields.forEach((f, idx) => cols[idx].add(visit(f.type, ctx)));
+    fields.forEach((f, idx) => cols[idx].add(visit(f.type, ctx, f.metadata)));
   }
   // Drain any dictionary batches that come after the last record batch in
   // the stream. These can't affect any already-decoded record batch, but
@@ -166,7 +166,7 @@ function contextGenerator(options, version, dictionaryMap) {
           : bytes.subarray(offset, offset + length)
       },
       variadic: () => variadic[++variadicIndex],
-      visit(children) { return children.map(f => visit(f.type, this)); }
+      visit(children) { return children.map(f => visit(f.type, this, f.metadata)); }
     };
   };
 }
@@ -193,10 +193,10 @@ function maybeDecompress(body, region, compression) {
 /**
  * Visit a field, instantiating views of buffer regions.
  */
-function visit(type, ctx) {
+function visit(type, ctx, metadata = null) {
   const { typeId } = type;
   const { options, node, buffer, variadic, version } = ctx;
-  const BatchType = batchType(type, options);
+  const BatchType = batchType(type, options, metadata);
 
   // extract the next { length, nullCount } field node - ALL fields have field nodes
   const base = { ...node(), type };

src/types.ts

@@ -373,6 +373,14 @@ export interface ExtractionOptions {
    * Otherwise, coerce timestamps to float milliseconds.
    */
   useBigIntTimestamp?: boolean;
+  /**
+   * If true, extract geometry data in GeoArrow well-known binary (WKB) or
+   * well-known text (WKT) formats as GeoJSON geometry objects. Geometry
+   * columns are identified by the field metadata key
+   * `ARROW:extension:name` with value `geoarrow.wkb` or `geoarrow.wkt`.
+   * Otherwise, return raw WKB byte arrays or WKT strings (default).
+   */
+  useGeoJSON?: boolean;
   /**
    * If true, extract Arrow 'Map' values as JavaScript `Map` instances.
    * Otherwise, return an array of [key, value] pairs compatible with

test/geoarrow.test.js

@@ -0,0 +1,64 @@
+import { describe, it, expect } from "vitest";
+import { geoJSONToWKB, geoJSONToWKT } from '@uwdata/wk-geojson';
+import { Table, binary, columnFromArray, field, largeBinary, largeUtf8, struct, tableFromIPC, tableToIPC, utf8 } from '../src/index.js';
+
+const geoms = [
+  { type: 'Point', coordinates: [1, 2] },
+  null,
+  { type: 'LineString', coordinates: [[0, 0], [1, 1], [2, 0]] },
+  { type: 'Polygon', coordinates: [[[30, 10], [40, 40], [20, 40], [10, 20], [30, 10]]] }
+];
+const wkb = geoms.map(g => g && geoJSONToWKB(g));
+const wkt = geoms.map(g => g && geoJSONToWKT(g));
+
+const extension = name => new Map([['ARROW:extension:name', name]]);
+
+/**
+ * Round-trip a single-column table through IPC bytes.
+ */
+function roundTrip(values, type, metadata, options) {
+  const schema = { fields: [field('geom', type, true, metadata)] };
+  const table = new Table(schema, [columnFromArray(values, type)]);
+  const bytes = tableToIPC(table, { format: 'stream' });
+  return tableFromIPC(bytes, options).getChild('geom');
+}
+
+describe('GeoArrow', () => {
+  for (const [name, type] of [['binary', binary()], ['largeBinary', largeBinary()]]) {
+    it(`decodes geoarrow.wkb ${name} data to GeoJSON`, () => {
+      const col = roundTrip(wkb, type, extension('geoarrow.wkb'), { useGeoJSON: true });
+      expect(col.toArray()).toStrictEqual(geoms);
+      expect([...col]).toStrictEqual(geoms);
+      expect(geoms.map((_, i) => col.at(i))).toStrictEqual(geoms);
+    });
+  }
+
+  for (const [name, type] of [['utf8', utf8()], ['largeUtf8', largeUtf8()]]) {
+    it(`decodes geoarrow.wkt ${name} data to GeoJSON`, () => {
+      const col = roundTrip(wkt, type, extension('geoarrow.wkt'), { useGeoJSON: true });
+      expect(col.toArray()).toStrictEqual(geoms);
+      expect([...col]).toStrictEqual(geoms);
+    });
+  }
+
+  it('returns raw WKB and WKT values by default', () => {
+    const b = roundTrip(wkb, binary(), extension('geoarrow.wkb'));
+    expect(b.toArray()).toStrictEqual(wkb);
+    const t = roundTrip(wkt, utf8(), extension('geoarrow.wkt'));
+    expect(t.toArray()).toStrictEqual(wkt);
+  });
+
+  it('ignores fields without geoarrow metadata', () => {
+    expect(roundTrip(wkb, binary(), null, { useGeoJSON: true }).toArray()).toStrictEqual(wkb);
+    expect(roundTrip(wkt, utf8(), extension('other'), { useGeoJSON: true }).toArray()).toStrictEqual(wkt);
+    // mismatched storage type: WKB extension on a string column
+    expect(roundTrip(wkt, utf8(), extension('geoarrow.wkb'), { useGeoJSON: true }).toArray()).toStrictEqual(wkt);
+  });
+
+  it('decodes geoarrow data nested in structs', () => {
+    const type = struct([field('geom', binary(), true, extension('geoarrow.wkb'))]);
+    const values = wkb.map(geom => ({ geom }));
+    const col = roundTrip(values, type, null, { useGeoJSON: true });
+    expect(col.toArray()).toStrictEqual(geoms.map(geom => ({ geom })));
+  });
+});