diff --git a/docs/user/general/datatypes.rst b/docs/user/general/datatypes.rst index adefd59397b..203a2891aa9 100644 --- a/docs/user/general/datatypes.rst +++ b/docs/user/general/datatypes.rst @@ -101,6 +101,8 @@ The table below list the mapping between OpenSearch Data Type, OpenSearch SQL Da +-----------------+---------------------+-----------+ | object | struct | STRUCT | +-----------------+---------------------+-----------+ +| flat_object | struct | STRUCT | ++-----------------+---------------------+-----------+ | nested | array | STRUCT | +-----------------+---------------------+-----------+ diff --git a/docs/user/ppl/general/datatypes.md b/docs/user/ppl/general/datatypes.md index b2f38418ea8..d3d37bb2635 100644 --- a/docs/user/ppl/general/datatypes.md +++ b/docs/user/ppl/general/datatypes.md @@ -49,6 +49,7 @@ The table below list the mapping between OpenSearch Data Type, PPL Data Type and | ip | ip | VARCHAR | | binary | binary | VARBINARY | | object | struct | STRUCT | +| flat_object | struct | STRUCT | | nested | array | STRUCT | Notes: Not all the PPL Type has correspond OpenSearch Type. e.g. data and time. To use function which required such data type, user should explicit convert the data type. diff --git a/docs/user/ppl/limitations/limitations.md b/docs/user/ppl/limitations/limitations.md index e532f64a790..a434288698b 100644 --- a/docs/user/ppl/limitations/limitations.md +++ b/docs/user/ppl/limitations/limitations.md @@ -13,7 +13,7 @@ PPL does not support all [OpenSearch data types](https://docs.opensearch.org/lat | --- | --- | | knn_vector | Ignored | | Range field types | Ignored | -| Object - flat_object | Ignored | +| Object - flat_object | Partial — see [Flat Object Field Behavior](#flat-object-field-behavior) | | Object - join | Ignored | | String - Match-only text | Ignored | | String - Wildcard | Ignored | @@ -37,6 +37,37 @@ For a field to be queryable in PPL, the following index settings must be enabled | index: true | Enables field indexing | Required for filtering, search, and aggregations | | doc_values: true | Enables columnar access for aggregations/sorting | Required for `stats`, `sort` | +## Flat Object Field Behavior + +`flat_object` fields are returned in query results and their top-level keys are addressable, but the +type carries limitations that come from how OpenSearch indexes it and from how PPL resolves dotted +paths. + +Supported: + +* The field is listed by `describe` (as type `struct`) and returned by `source` and `fields`. +* A top-level key is addressable and keeps its `_source` type — given + `{"flat": {"s": "x", "n": 7}}`, `fields flat.s` returns the string `x` and `fields flat.n` returns + the integer `7`. +* Filtering on a top-level key works, for example `where flat.s = 'x'`. + +Not supported: + +* **Subfield paths deeper than one level resolve to `NULL`.** For `{"flat": {"a": {"b": 1}}}`, + `fields flat.a.b` returns `NULL` rather than `1`, and no error is raised. +* **A nested value is returned as JSON text**, not as a structure — `fields flat.a` returns the + string `{"b":1}`. +* **Aggregating on a subfield cannot be pushed down.** OpenSearch reports `flat_object` as + non-aggregatable, so `stats ... by flat.a` cannot use a native aggregation. +* **Sorting on a subfield is not meaningful.** OpenSearch sorts on an internal representation that + spans every subfield of the object. +* `flatten` and `expand` cannot enumerate a `flat_object`, because subfield names are not present in + the index mapping. + +Subfield names are resolved lazily by OpenSearch per query and are absent from the mapping, so PPL +cannot discover them at planning time; a `flat_object` therefore contributes exactly one column to +the schema. Use `spath` to reach deeper paths within the returned JSON. + ## Nested Field Behavior * There are [limitations](https://github.com/opensearch-project/sql/issues/4625) regarding the nested levels and query types that need improvement. diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/1604.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/1604.yml new file mode 100644 index 00000000000..a37e74994d9 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/1604.yml @@ -0,0 +1,160 @@ +# Issue: https://github.com/opensearch-project/sql/issues/1604 +# https://github.com/opensearch-project/sql/issues/3067 +# flat_object fields used to be dropped by OpenSearchDataType.parseMapping ("unknown type, skip +# it"), so they vanished from query results entirely and could not be referenced at all. This +# covers the projection contract: the field is visible, selectable, and round-trips its _source +# JSON, on both the Calcite and the v2 engine. +setup: + - do: + indices.create: + index: issue1604 + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + eventName: + type: keyword + requestParameters: + type: flat_object + + - do: + bulk: + refresh: true + body: + - '{"index": {"_index": "issue1604", "_id": "1"}}' + - '{"eventName": "CreateInstanceExportTask", "requestParameters": {"instanceId": "i-123", "nextToken": "abc"}}' + - '{"index": {"_index": "issue1604", "_id": "2"}}' + - '{"eventName": "DescribeInstances", "requestParameters": {"instanceId": "i-456"}}' + - '{"index": {"_index": "issue1604", "_id": "3"}}' + - '{"eventName": "RunInstances", "requestParameters": {"instanceId": "i-789", "filter": {"state": "running"}}}' + +--- +teardown: + - do: + indices.delete: + index: issue1604 + ignore_unavailable: true + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: false + +--- +"Issue 1604: flat_object field is projected with Calcite enabled": + - skip: + features: + - headers + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: true + plugins.calcite.fallback.allowed: false + + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: 'source=issue1604 | where eventName = ''DescribeInstances'' | fields requestParameters' + - match: {"total": 1} + - match: {"schema": [{"name": "requestParameters", "type": "struct"}]} + - match: {"datarows": [[{"instanceId": "i-456"}]]} + +--- +"Issue 1604: flat_object field is projected with Calcite disabled": + - skip: + features: + - headers + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: false + + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: 'source=issue1604 | where eventName = ''DescribeInstances'' | fields requestParameters' + - match: {"total": 1} + - match: {"schema": [{"name": "requestParameters", "type": "struct"}]} + - match: {"datarows": [[{"instanceId": "i-456"}]]} + +--- +"Issue 1604: flat_object subfield is addressable": + - skip: + features: + - headers + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: true + plugins.calcite.fallback.allowed: false + + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: 'source=issue1604 | sort eventName | fields requestParameters.instanceId' + - match: {"total": 3} + - match: {"datarows": [["i-123"], ["i-456"], ["i-789"]]} + +--- +"Issue 1604: flat_object field appears in describe": + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: 'describe issue1604 | where COLUMN_NAME = ''requestParameters'' | fields COLUMN_NAME, TYPE_NAME' + - match: {"total": 1} + # PPL reports the PPL type name (struct), matching how object fields are described. + # The SQL surface reports legacyTypeName() = flat_object; that divergence predates this change. + - match: {"datarows": [["requestParameters", "struct"]]} + +--- +# Known limitations of this first increment, pinned so any future change is deliberate rather +# than accidental. Tracked by https://github.com/opensearch-project/sql/issues/1604 item 2. +# +# * A nested value is stringified by OpenSearchExprValueFactory#parseContent, which has no +# object branch (the Content interface has no isObject() predicate). See issue 3751. +# * A subfield path deeper than one level resolves to NULL: QualifiedNameResolver#resolveFieldAccess +# joins the remaining parts into a single ITEM key ('filter.state'), which cannot match. +"Issue 1604: nested values are stringified and depth-2 paths resolve to null": + - skip: + features: + - headers + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: true + plugins.calcite.fallback.allowed: false + + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: 'source=issue1604 | where eventName = ''RunInstances'' | fields requestParameters.filter' + - match: {"total": 1} + - match: {"datarows": [["{\"state\":\"running\"}"]]} + + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: 'source=issue1604 | where eventName = ''RunInstances'' | fields requestParameters.filter.state' + - match: {"total": 1} + - match: {"datarows": [[null]]} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchAliasType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchAliasType.java index 1954134c11b..e20c8d34615 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchAliasType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchAliasType.java @@ -20,7 +20,7 @@ public class OpenSearchAliasType extends OpenSearchDataType { public static final String typeName = "alias"; public static final String pathPropertyName = "path"; public static final Set objectFieldTypes = - Set.of(MappingType.Object, MappingType.Nested); + Set.of(MappingType.Object, MappingType.Nested, MappingType.FlatObject); private final String path; @Getter private final OpenSearchDataType originalType; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java index 2a70502f392..a5aae6be1d4 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java @@ -38,6 +38,7 @@ public enum MappingType { Date("date", ExprCoreType.TIMESTAMP), DateNanos("date_nanos", ExprCoreType.TIMESTAMP), Object("object", ExprCoreType.STRUCT), + FlatObject("flat_object", ExprCoreType.STRUCT), Nested("nested", ExprCoreType.ARRAY), Byte("byte", ExprCoreType.BYTE), Short("short", ExprCoreType.SHORT), @@ -149,6 +150,10 @@ public static OpenSearchDataType of(MappingType mappingType, Map OpenSearchDataType res = instances.getOrDefault(mappingType.toString(), new OpenSearchDataType(mappingType)); switch (mappingType) { + // flat_object has no declared `properties`; it shares the Object arm so that it gets a + // fresh instance with a mutable (empty) property map rather than the cached singleton, + // whose immutable map would break cross-index merging in DESCRIBE. + case FlatObject: case Object: // TODO: use Object type once it has been added case Nested: