diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 378c28c..b26dd1f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: - node: [24, 25] + node: [24, 26] name: Node ${{ matrix.node }} diff --git a/README.md b/README.md index 541fd0b..d323bac 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,7 @@ const table = tableFromIPC(ipc, { useDecimalInt: true, // use BigInt for decimals, do not coerce to number useBigInt: true, // use BigInt for 64-bit ints, do not coerce to number useBigIntTimestamp: true, // use BigInt for timestamps, do not coerce to millisecond number + useGeoJSON: true, // decode GeoArrow WKB/WKT geometry columns to GeoJSON objects useMap: true, // create Map objects for [key, value] pair lists useProxy: true // use zero-copy proxies for struct and table row objects }); diff --git a/docs/api/data-types.md b/docs/api/data-types.md index 7fab8d0..05c2f8b 100644 --- a/docs/api/data-types.md +++ b/docs/api/data-types.md @@ -15,8 +15,8 @@ The table below provides an overview of all data types supported by the Apache A | 1 | [Null](#null) | ✅ | ✅ | ✅ | `null` | | 2 | [Int](#int) | ✅ | ✅ | ✅ | `number`, or `bigint` for 64-bit values via the `useBigInt` flag | | 3 | [Float](#float) | ✅ | ✅ | ✅ | `number` | -| 4 | [Binary](#binary) | ✅ | ✅ | ✅ | `Uint8Array` | -| 5 | [Utf8](#utf8) | ✅ | ✅ | ✅ | `string` | +| 4 | [Binary](#binary) | ✅ | ✅ | ✅ | `Uint8Array`, or GeoJSON for GeoArrow geometries via the `useGeoJSON` flag | +| 5 | [Utf8](#utf8) | ✅ | ✅ | ✅ | `string`, or GeoJSON for GeoArrow geometries via the `useGeoJSON` flag | | 6 | [Bool](#bool) | ✅ | ✅ | ✅ | `boolean` | | 7 | [Decimal](#decimal) | ✅ | ✅ | ✅ | `number`, or scaled integers via the `useDecimalInt` flag | | 8 | [Date](#date) | ✅ | ✅ | ✅ | `number`, or `Date` via the `useDate` flag. | @@ -30,12 +30,12 @@ The table below provides an overview of all data types supported by the Apache A | 16 | [FixedSizeList](#fixedSizeList) | ✅ | ✅ | ✅ | `Array` or `TypedArray` of child type | | 17 | [Map](#map) | ✅ | ✅ | ✅ | `[key, value][]`, or `Map` via the `useMap` flag | | 18 | [Duration](#duration) | ✅ | ✅ | ✅ | `number`, or `bigint` via the `useBigInt` flag | -| 19 | [LargeBinary](#largeBinary) | ✅ | ✅ | ✅ | `Uint8Array` | -| 20 | [LargeUtf8](#largeUtf8) | ✅ | ✅ | ✅ | `string` | +| 19 | [LargeBinary](#largeBinary) | ✅ | ✅ | ✅ | `Uint8Array`, or GeoJSON for GeoArrow geometries via the `useGeoJSON` flag | +| 20 | [LargeUtf8](#largeUtf8) | ✅ | ✅ | ✅ | `string`, or GeoJSON for GeoArrow geometries via the `useGeoJSON` flag | | 21 | [LargeList](#largeList) | ✅ | ✅ | ✅ | `Array` or `TypedArray` of child type | | 22 | [RunEndEncoded](#runEndEncoded) | ✅ | ✅ | ✅ | depends on child type | -| 23 | [BinaryView](#binaryView) | ✅ | ✅ | ❌ | `Uint8Array` | -| 24 | [Utf8View](#utf8View) | ✅ | ✅ | ❌ | `string` | +| 23 | [BinaryView](#binaryView) | ✅ | ✅ | ❌ | `Uint8Array`, or GeoJSON for GeoArrow geometries via the `useGeoJSON` flag | +| 24 | [Utf8View](#utf8View) | ✅ | ✅ | ❌ | `string`, or GeoJSON for GeoArrow geometries via the `useGeoJSON` flag | | 25 | [ListView](#listView) | ✅ | ✅ | ❌ | `Array` or `TypedArray` of child type | | 26 | [LargeListView](#largeListView) | ✅ | ✅ | ❌ | `Array` or `TypedArray` of child type | diff --git a/docs/api/index.md b/docs/api/index.md index 81159a3..b0cea70 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -28,6 +28,7 @@ By default Flechette assumes input data is uncompressed. If input IPC data conta * *useBigIntTimestamp* (`boolean`): If true, extract timestamps as JavaScript `BigInt` values. Otherwise, coerce timestamps to millisecond numbers. This option takes precedence over the *useDate* option. * *useDate* (`boolean`): If true, extract dates and timestamps as JavaScript `Date` objects. Otherwise, return numerical timestamp values (default `false`). If *useBigIntTimestamp* is true, this option does not apply to timestamps. * *useDecimalInt* (`boolean`): If true, extract decimal-type data as scaled integer values, where fractional digits are scaled to integer positions. Returned integers are `BigInt` values for decimal bit widths of 64 bits or higher and 32-bit integers (as JavaScript `number`) otherwise. If false, decimals are lossily converted to floating-point numbers (default). + * *useGeoJSON* (`boolean`): If true, extract geometry data in [GeoArrow](https://geoarrow.org/) well-known binary (WKB) or well-known text (WKT) formats as [GeoJSON](https://datatracker.ietf.org/doc/html/rfc7946) geometry objects. Geometry columns are identified by the field metadata key `ARROW:extension:name` with value `geoarrow.wkb` or `geoarrow.wkt`. Otherwise, return raw WKB byte arrays or WKT strings (default `false`). * *useMap* (`boolean`): If true, extract Arrow 'Map' values as JavaScript `Map` instances. Otherwise, return an array of [key, value] pairs compatible with both `Map` and `Object.fromEntries` (default `false`). * *useProxy* (`boolean`): If true, extract Arrow 'Struct' values and table row objects using zero-copy proxy objects that extract data from underlying Arrow batches. The proxy objects can improve performance and reduce memory usage, but do not support property enumeration (`Object.keys`, `Object.values`, `Object.entries`) or spreading (`{ ...object }`). Otherwise (default `false`), use standard JS objects for structs and table rows. diff --git a/docs/index.md b/docs/index.md index de2e5ac..809e817 100644 --- a/docs/index.md +++ b/docs/index.md @@ -118,6 +118,7 @@ const table = tableFromIPC(ipc, { useDecimalInt: true, // use scaled ints for decimals, not floating point useBigInt: true, // use BigInt for 64-bit ints, do not coerce to number useBigIntTimestamp: true, // use BigInt for timestamps, do not coerce to millisecond number + useGeoJSON: true, // decode GeoArrow WKB/WKT geometry columns to GeoJSON objects useMap: true, // create Map objects for [key, value] pair lists useProxy: true // use zero-copy proxies for struct and table row objects }); diff --git a/package.json b/package.json index 0d69c05..a255a08 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,9 @@ "prepublishOnly": "pnpm run test && pnpm run lint && pnpm run build", "prepare": "pnpm run build" }, + "dependencies": { + "@uwdata/wk-geojson": "^0.0.1" + }, "devDependencies": { "apache-arrow": "^21.2.0", "lz4js": "0.2.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 126465f..4f6f882 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -165,6 +165,10 @@ settings: importers: .: + dependencies: + '@uwdata/wk-geojson': + specifier: ^0.0.1 + version: 0.0.1 devDependencies: apache-arrow: specifier: ^21.2.0 @@ -559,6 +563,9 @@ packages: cpu: [x64] os: [win32] + '@uwdata/wk-geojson@0.0.1': + resolution: {integrity: sha512-4RpnKbdrphwzP+BwgzuGRqA/s03oT1rzHAKdU43Ozn8J1BCTwS+2iAGQ7YQAnrCkCi213WtQ7VYaFjVtvZotPQ==} + '@vitest/mocker@5.0.3': resolution: {integrity: sha512-T8sWAIbkSyAjkwTcaEc3Iu0o9A27X1/kdXrizhZkGuSKScRQtRzclfAMpOTcGdXCsqxeWlpGy3XjqaW8CpLORg==} peerDependencies: @@ -1085,6 +1092,8 @@ snapshots: '@typescript/typescript-win32-x64@7.0.2': optional: true + '@uwdata/wk-geojson@0.0.1': {} + '@vitest/mocker@5.0.3(vite@8.3.3(@types/node@25.9.9))': dependencies: '@jridgewell/trace-mapping': 0.3.31 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml new file mode 100644 index 0000000..2b421aa --- /dev/null +++ b/pnpm-workspace.yaml @@ -0,0 +1,2 @@ +minimumReleaseAgeExclude: + - '@uwdata/wk-geojson' diff --git a/src/batch-type.js b/src/batch-type.js index 2a05266..8052ca3 100644 --- a/src/batch-type.js +++ b/src/batch-type.js @@ -1,7 +1,7 @@ /** - * @import { DataType, ExtractionOptions } from './types.js'; + * @import { DataType, ExtractionOptions, Metadata } from './types.js'; */ -import { BinaryBatch, BinaryViewBatch, BoolBatch, DateBatch, DateDayBatch, DateDayMillisecondBatch, Decimal32NumberBatch, DecimalBigIntBatch, DecimalNumberBatch, DenseUnionBatch, DictionaryBatch, DirectBatch, FixedBinaryBatch, FixedListBatch, Float16Batch, Int64Batch, IntervalDayTimeBatch, IntervalMonthDayNanoBatch, LargeBinaryBatch, LargeListBatch, LargeListViewBatch, LargeUtf8Batch, ListBatch, ListViewBatch, MapBatch, MapEntryBatch, NullBatch, RunEndEncodedBatch, SparseUnionBatch, StructBatch, StructProxyBatch, TimestampMicrosecondBatch, TimestampMillisecondBatch, TimestampNanosecondBatch, TimestampSecondBatch, Utf8Batch, Utf8ViewBatch } from './batch.js'; +import { BinaryBatch, BinaryViewBatch, BoolBatch, DateBatch, DateDayBatch, DateDayMillisecondBatch, Decimal32NumberBatch, DecimalBigIntBatch, DecimalNumberBatch, DenseUnionBatch, DictionaryBatch, DirectBatch, FixedBinaryBatch, FixedListBatch, Float16Batch, Int64Batch, IntervalDayTimeBatch, IntervalMonthDayNanoBatch, LargeBinaryBatch, LargeListBatch, LargeListViewBatch, LargeUtf8Batch, ListBatch, ListViewBatch, MapBatch, MapEntryBatch, NullBatch, RunEndEncodedBatch, SparseUnionBatch, StructBatch, StructProxyBatch, TimestampMicrosecondBatch, TimestampMillisecondBatch, TimestampNanosecondBatch, TimestampSecondBatch, Utf8Batch, Utf8ViewBatch, WKBGeoJSONBatch, WKTGeoJSONBatch } from './batch.js'; import { DateUnit, IntervalUnit, TimeUnit, Type } from './constants.js'; import { invalidDataType } from './data-types.js'; @@ -9,10 +9,15 @@ import { invalidDataType } from './data-types.js'; * Return a batch constructor for the given data type and extraction options. * @param {DataType} type The data type. * @param {ExtractionOptions} options The extraction options. + * @param {Metadata | null} [metadata] Custom field metadata, used to + * identify extension types such as GeoArrow WKB and WKT geometries. */ -export function batchType(type, options = {}) { +export function batchType(type, options = {}, metadata = null) { const { typeId, bitWidth, mode, precision, unit } = /** @type {any} */(type); - const { useBigInt, useBigIntTimestamp, useDate, useDecimalInt, useMap, useProxy } = options; + const { useBigInt, useBigIntTimestamp, useDate, useDecimalInt, useGeoJSON, useMap, useProxy } = options; + const ext = useGeoJSON ? metadata?.get('ARROW:extension:name') : null; + const wkb = ext === 'geoarrow.wkb' && WKBGeoJSONBatch; + const wkt = ext === 'geoarrow.wkt' && WKTGeoJSONBatch; switch (typeId) { case Type.Null: return NullBatch; @@ -45,12 +50,12 @@ export function batchType(type, options = {}) { : unit === IntervalUnit.YEAR_MONTH ? DirectBatch : IntervalMonthDayNanoBatch; case Type.FixedSizeBinary: return FixedBinaryBatch; - case Type.Utf8: return Utf8Batch; - case Type.LargeUtf8: return LargeUtf8Batch; - case Type.Binary: return BinaryBatch; - case Type.LargeBinary: return LargeBinaryBatch; - case Type.BinaryView: return BinaryViewBatch; - case Type.Utf8View: return Utf8ViewBatch; + case Type.Utf8: return wrap(Utf8Batch, wkt); + case Type.LargeUtf8: return wrap(LargeUtf8Batch, wkt); + case Type.Binary: return wrap(BinaryBatch, wkb); + case Type.LargeBinary: return wrap(LargeBinaryBatch, wkb); + case Type.BinaryView: return wrap(BinaryViewBatch, wkb); + case Type.Utf8View: return wrap(Utf8ViewBatch, wkt); case Type.List: return ListBatch; case Type.LargeList: return LargeListBatch; case Type.Map: return useMap ? MapBatch : MapEntryBatch; diff --git a/src/batch.js b/src/batch.js index e9b160e..41bfc72 100644 --- a/src/batch.js +++ b/src/batch.js @@ -1,7 +1,9 @@ /** + * @import { Geometry } from '@uwdata/wk-geojson' * @import { Column } from './column.js' * @import { DataType, DecimalType, IntegerArray, OffsetArray, TypedArray, TypedArrayConstructor, ValueArray } from './types.js' */ +import { wkbToGeoJSON, wktToGeoJSON } from '@uwdata/wk-geojson'; import { bisect, float64Array } from './util/arrays.js'; import { divide, fromDecimal128, fromDecimal256, fromDecimal64, toNumber } from './util/numbers.js'; import { decodeBit, readInt32, readInt64 } from './util/read.js'; @@ -515,6 +517,54 @@ export class LargeUtf8Batch extends ArrayBatch { } } +/** + * A batch of geometry values in well-known binary (WKB) format that are + * decoded and returned as GeoJSON geometry objects. This batch wraps a + * source batch that provides WKB byte arrays. + * @extends {ArrayBatch} + */ +export class WKBGeoJSONBatch extends ArrayBatch { + /** + * Create a new WKB to GeoJSON batch. + * @param {Batch} batch A batch of WKB byte arrays. + */ + constructor(batch) { + super(batch); + this.source = batch; + } + + /** + * @param {number} index The value index + */ + value(index) { + return wkbToGeoJSON(this.source.value(index)); + } +} + +/** + * A batch of geometry values in well-known text (WKT) format that are + * decoded and returned as GeoJSON geometry objects. This batch wraps a + * source batch that provides WKT strings. + * @extends {ArrayBatch} + */ +export class WKTGeoJSONBatch extends ArrayBatch { + /** + * Create a new WKT to GeoJSON batch. + * @param {Batch} batch A batch of WKT strings. + */ + constructor(batch) { + super(batch); + this.source = batch; + } + + /** + * @param {number} index The value index + */ + value(index) { + return wktToGeoJSON(this.source.value(index)); + } +} + /** * A batch of list (array) values of variable length. The list offsets are * 32-bit ints. diff --git a/src/decode/table-from-ipc.js b/src/decode/table-from-ipc.js index 376b2bb..d11bc0c 100644 --- a/src/decode/table-from-ipc.js +++ b/src/decode/table-from-ipc.js @@ -108,7 +108,7 @@ export function createTable(data, options = {}) { } const ctx = context(records[i]); - fields.forEach((f, idx) => cols[idx].add(visit(f.type, ctx))); + fields.forEach((f, idx) => cols[idx].add(visit(f.type, ctx, f.metadata))); } // Drain any dictionary batches that come after the last record batch in // the stream. These can't affect any already-decoded record batch, but @@ -166,7 +166,7 @@ function contextGenerator(options, version, dictionaryMap) { : bytes.subarray(offset, offset + length) }, variadic: () => variadic[++variadicIndex], - visit(children) { return children.map(f => visit(f.type, this)); } + visit(children) { return children.map(f => visit(f.type, this, f.metadata)); } }; }; } @@ -193,10 +193,10 @@ function maybeDecompress(body, region, compression) { /** * Visit a field, instantiating views of buffer regions. */ -function visit(type, ctx) { +function visit(type, ctx, metadata = null) { const { typeId } = type; const { options, node, buffer, variadic, version } = ctx; - const BatchType = batchType(type, options); + const BatchType = batchType(type, options, metadata); // extract the next { length, nullCount } field node - ALL fields have field nodes const base = { ...node(), type }; diff --git a/src/types.ts b/src/types.ts index d33256c..f0e4165 100644 --- a/src/types.ts +++ b/src/types.ts @@ -373,6 +373,14 @@ export interface ExtractionOptions { * Otherwise, coerce timestamps to float milliseconds. */ useBigIntTimestamp?: boolean; + /** + * If true, extract geometry data in GeoArrow well-known binary (WKB) or + * well-known text (WKT) formats as GeoJSON geometry objects. Geometry + * columns are identified by the field metadata key + * `ARROW:extension:name` with value `geoarrow.wkb` or `geoarrow.wkt`. + * Otherwise, return raw WKB byte arrays or WKT strings (default). + */ + useGeoJSON?: boolean; /** * If true, extract Arrow 'Map' values as JavaScript `Map` instances. * Otherwise, return an array of [key, value] pairs compatible with diff --git a/test/geoarrow.test.js b/test/geoarrow.test.js new file mode 100644 index 0000000..93233ca --- /dev/null +++ b/test/geoarrow.test.js @@ -0,0 +1,64 @@ +import { describe, it, expect } from "vitest"; +import { geoJSONToWKB, geoJSONToWKT } from '@uwdata/wk-geojson'; +import { Table, binary, columnFromArray, field, largeBinary, largeUtf8, struct, tableFromIPC, tableToIPC, utf8 } from '../src/index.js'; + +const geoms = [ + { type: 'Point', coordinates: [1, 2] }, + null, + { type: 'LineString', coordinates: [[0, 0], [1, 1], [2, 0]] }, + { type: 'Polygon', coordinates: [[[30, 10], [40, 40], [20, 40], [10, 20], [30, 10]]] } +]; +const wkb = geoms.map(g => g && geoJSONToWKB(g)); +const wkt = geoms.map(g => g && geoJSONToWKT(g)); + +const extension = name => new Map([['ARROW:extension:name', name]]); + +/** + * Round-trip a single-column table through IPC bytes. + */ +function roundTrip(values, type, metadata, options) { + const schema = { fields: [field('geom', type, true, metadata)] }; + const table = new Table(schema, [columnFromArray(values, type)]); + const bytes = tableToIPC(table, { format: 'stream' }); + return tableFromIPC(bytes, options).getChild('geom'); +} + +describe('GeoArrow', () => { + for (const [name, type] of [['binary', binary()], ['largeBinary', largeBinary()]]) { + it(`decodes geoarrow.wkb ${name} data to GeoJSON`, () => { + const col = roundTrip(wkb, type, extension('geoarrow.wkb'), { useGeoJSON: true }); + expect(col.toArray()).toStrictEqual(geoms); + expect([...col]).toStrictEqual(geoms); + expect(geoms.map((_, i) => col.at(i))).toStrictEqual(geoms); + }); + } + + for (const [name, type] of [['utf8', utf8()], ['largeUtf8', largeUtf8()]]) { + it(`decodes geoarrow.wkt ${name} data to GeoJSON`, () => { + const col = roundTrip(wkt, type, extension('geoarrow.wkt'), { useGeoJSON: true }); + expect(col.toArray()).toStrictEqual(geoms); + expect([...col]).toStrictEqual(geoms); + }); + } + + it('returns raw WKB and WKT values by default', () => { + const b = roundTrip(wkb, binary(), extension('geoarrow.wkb')); + expect(b.toArray()).toStrictEqual(wkb); + const t = roundTrip(wkt, utf8(), extension('geoarrow.wkt')); + expect(t.toArray()).toStrictEqual(wkt); + }); + + it('ignores fields without geoarrow metadata', () => { + expect(roundTrip(wkb, binary(), null, { useGeoJSON: true }).toArray()).toStrictEqual(wkb); + expect(roundTrip(wkt, utf8(), extension('other'), { useGeoJSON: true }).toArray()).toStrictEqual(wkt); + // mismatched storage type: WKB extension on a string column + expect(roundTrip(wkt, utf8(), extension('geoarrow.wkb'), { useGeoJSON: true }).toArray()).toStrictEqual(wkt); + }); + + it('decodes geoarrow data nested in structs', () => { + const type = struct([field('geom', binary(), true, extension('geoarrow.wkb'))]); + const values = wkb.map(geom => ({ geom })); + const col = roundTrip(values, type, null, { useGeoJSON: true }); + expect(col.toArray()).toStrictEqual(geoms.map(geom => ({ geom }))); + }); +});