Support flat_object field type in PPL projection

docs/user/general/datatypes.rst

@@ -101,6 +101,8 @@ The table below list the mapping between OpenSearch Data Type, OpenSearch SQL Da
 +-----------------+---------------------+-----------+
 | object          | struct              | STRUCT    |
 +-----------------+---------------------+-----------+
+| flat_object     | struct              | STRUCT    |
++-----------------+---------------------+-----------+
 | nested          | array               | STRUCT    |
 +-----------------+---------------------+-----------+
 

docs/user/ppl/general/datatypes.md

@@ -49,6 +49,7 @@ The table below list the mapping between OpenSearch Data Type, PPL Data Type and
 | ip | ip | VARCHAR |
 | binary | binary | VARBINARY |
 | object | struct | STRUCT |
+| flat_object | struct | STRUCT |
 | nested | array | STRUCT |
 
 Notes: Not all the PPL Type has correspond OpenSearch Type. e.g. data and time. To use function which required such data type, user should explicit convert the data type.

docs/user/ppl/limitations/limitations.md

@@ -13,7 +13,7 @@ PPL does not support all [OpenSearch data types](https://docs.opensearch.org/lat
 | --- | --- |
 | knn_vector | Ignored |
 | Range field types | Ignored |
-| Object - flat_object | Ignored |
+| Object - flat_object | Partial — see [Flat Object Field Behavior](#flat-object-field-behavior) |
 | Object - join | Ignored |
 | String - Match-only text | Ignored |
 | String - Wildcard | Ignored |
@@ -37,6 +37,37 @@ For a field to be queryable in PPL, the following index settings must be enabled
 | index: true | Enables field indexing | Required for filtering, search, and aggregations |
 | doc_values: true | Enables columnar access for aggregations/sorting | Required for `stats`, `sort` |
 
+## Flat Object Field Behavior
+
+`flat_object` fields are returned in query results and their top-level keys are addressable, but the
+type carries limitations that come from how OpenSearch indexes it and from how PPL resolves dotted
+paths.
+
+Supported:
+
+* The field is listed by `describe` (as type `struct`) and returned by `source` and `fields`.
+* A top-level key is addressable and keeps its `_source` type — given
+  `{"flat": {"s": "x", "n": 7}}`, `fields flat.s` returns the string `x` and `fields flat.n` returns
+  the integer `7`.
+* Filtering on a top-level key works, for example `where flat.s = 'x'`.
+
+Not supported:
+
+* **Subfield paths deeper than one level resolve to `NULL`.** For `{"flat": {"a": {"b": 1}}}`,
+  `fields flat.a.b` returns `NULL` rather than `1`, and no error is raised.
+* **A nested value is returned as JSON text**, not as a structure — `fields flat.a` returns the
+  string `{"b":1}`.
+* **Aggregating on a subfield cannot be pushed down.** OpenSearch reports `flat_object` as
+  non-aggregatable, so `stats ... by flat.a` cannot use a native aggregation.
+* **Sorting on a subfield is not meaningful.** OpenSearch sorts on an internal representation that
+  spans every subfield of the object.
+* `flatten` and `expand` cannot enumerate a `flat_object`, because subfield names are not present in
+  the index mapping.
+
+Subfield names are resolved lazily by OpenSearch per query and are absent from the mapping, so PPL
+cannot discover them at planning time; a `flat_object` therefore contributes exactly one column to
+the schema. Use `spath` to reach deeper paths within the returned JSON.
+
 ## Nested Field Behavior  
 
 * There are [limitations](https://github.com/opensearch-project/sql/issues/4625) regarding the nested levels and query types that need improvement.  

integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/1604.yml

@@ -0,0 +1,160 @@
+# Issue: https://github.com/opensearch-project/sql/issues/1604
+#        https://github.com/opensearch-project/sql/issues/3067
+# flat_object fields used to be dropped by OpenSearchDataType.parseMapping ("unknown type, skip
+# it"), so they vanished from query results entirely and could not be referenced at all. This
+# covers the projection contract: the field is visible, selectable, and round-trips its _source
+# JSON, on both the Calcite and the v2 engine.
+setup:
+  - do:
+      indices.create:
+        index: issue1604
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            properties:
+              eventName:
+                type: keyword
+              requestParameters:
+                type: flat_object
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{"index": {"_index": "issue1604", "_id": "1"}}'
+          - '{"eventName": "CreateInstanceExportTask", "requestParameters": {"instanceId": "i-123", "nextToken": "abc"}}'
+          - '{"index": {"_index": "issue1604", "_id": "2"}}'
+          - '{"eventName": "DescribeInstances", "requestParameters": {"instanceId": "i-456"}}'
+          - '{"index": {"_index": "issue1604", "_id": "3"}}'
+          - '{"eventName": "RunInstances", "requestParameters": {"instanceId": "i-789", "filter": {"state": "running"}}}'
+
+---
+teardown:
+  - do:
+      indices.delete:
+        index: issue1604
+        ignore_unavailable: true
+  - do:
+      query.settings:
+        body:
+          transient:
+            plugins.calcite.enabled: false
+
+---
+"Issue 1604: flat_object field is projected with Calcite enabled":
+  - skip:
+      features:
+        - headers
+  - do:
+      query.settings:
+        body:
+          transient:
+            plugins.calcite.enabled: true
+            plugins.calcite.fallback.allowed: false
+
+  - do:
+      headers:
+        Content-Type: 'application/json'
+      ppl:
+        body:
+          query: 'source=issue1604 | where eventName = ''DescribeInstances'' | fields requestParameters'
+  - match: {"total": 1}
+  - match: {"schema": [{"name": "requestParameters", "type": "struct"}]}
+  - match: {"datarows": [[{"instanceId": "i-456"}]]}
+
+---
+"Issue 1604: flat_object field is projected with Calcite disabled":
+  - skip:
+      features:
+        - headers
+  - do:
+      query.settings:
+        body:
+          transient:
+            plugins.calcite.enabled: false
+
+  - do:
+      headers:
+        Content-Type: 'application/json'
+      ppl:
+        body:
+          query: 'source=issue1604 | where eventName = ''DescribeInstances'' | fields requestParameters'
+  - match: {"total": 1}
+  - match: {"schema": [{"name": "requestParameters", "type": "struct"}]}
+  - match: {"datarows": [[{"instanceId": "i-456"}]]}
+
+---
+"Issue 1604: flat_object subfield is addressable":
+  - skip:
+      features:
+        - headers
+  - do:
+      query.settings:
+        body:
+          transient:
+            plugins.calcite.enabled: true
+            plugins.calcite.fallback.allowed: false
+
+  - do:
+      headers:
+        Content-Type: 'application/json'
+      ppl:
+        body:
+          query: 'source=issue1604 | sort eventName | fields requestParameters.instanceId'
+  - match: {"total": 3}
+  - match: {"datarows": [["i-123"], ["i-456"], ["i-789"]]}
+
+---
+"Issue 1604: flat_object field appears in describe":
+  - skip:
+      features:
+        - headers
+  - do:
+      headers:
+        Content-Type: 'application/json'
+      ppl:
+        body:
+          query: 'describe issue1604 | where COLUMN_NAME = ''requestParameters'' | fields COLUMN_NAME, TYPE_NAME'
+  - match: {"total": 1}
+  # PPL reports the PPL type name (struct), matching how object fields are described.
+  # The SQL surface reports legacyTypeName() = flat_object; that divergence predates this change.
+  - match: {"datarows": [["requestParameters", "struct"]]}
+
+---
+# Known limitations of this first increment, pinned so any future change is deliberate rather
+# than accidental. Tracked by https://github.com/opensearch-project/sql/issues/1604 item 2.
+#
+#  * A nested value is stringified by OpenSearchExprValueFactory#parseContent, which has no
+#    object branch (the Content interface has no isObject() predicate). See issue 3751.
+#  * A subfield path deeper than one level resolves to NULL: QualifiedNameResolver#resolveFieldAccess
+#    joins the remaining parts into a single ITEM key ('filter.state'), which cannot match.
+"Issue 1604: nested values are stringified and depth-2 paths resolve to null":
+  - skip:
+      features:
+        - headers
+  - do:
+      query.settings:
+        body:
+          transient:
+            plugins.calcite.enabled: true
+            plugins.calcite.fallback.allowed: false
+
+  - do:
+      headers:
+        Content-Type: 'application/json'
+      ppl:
+        body:
+          query: 'source=issue1604 | where eventName = ''RunInstances'' | fields requestParameters.filter'
+  - match: {"total": 1}
+  - match: {"datarows": [["{\"state\":\"running\"}"]]}
+
+  - do:
+      headers:
+        Content-Type: 'application/json'
+      ppl:
+        body:
+          query: 'source=issue1604 | where eventName = ''RunInstances'' | fields requestParameters.filter.state'
+  - match: {"total": 1}
+  - match: {"datarows": [[null]]}

opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchAliasType.java

@@ -20,7 +20,7 @@ public class OpenSearchAliasType extends OpenSearchDataType {
   public static final String typeName = "alias";
   public static final String pathPropertyName = "path";
   public static final Set<MappingType> objectFieldTypes =
-      Set.of(MappingType.Object, MappingType.Nested);
+      Set.of(MappingType.Object, MappingType.Nested, MappingType.FlatObject);
   private final String path;
   @Getter private final OpenSearchDataType originalType;
 

opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java

@@ -38,6 +38,7 @@ public enum MappingType {
     Date("date", ExprCoreType.TIMESTAMP),
     DateNanos("date_nanos", ExprCoreType.TIMESTAMP),
     Object("object", ExprCoreType.STRUCT),
+    FlatObject("flat_object", ExprCoreType.STRUCT),
     Nested("nested", ExprCoreType.ARRAY),
     Byte("byte", ExprCoreType.BYTE),
     Short("short", ExprCoreType.SHORT),
@@ -149,6 +150,10 @@ public static OpenSearchDataType of(MappingType mappingType, Map<String, Object>
     OpenSearchDataType res =
         instances.getOrDefault(mappingType.toString(), new OpenSearchDataType(mappingType));
     switch (mappingType) {
+      // flat_object has no declared `properties`; it shares the Object arm so that it gets a
+      // fresh instance with a mutable (empty) property map rather than the cached singleton,
+      // whose immutable map would break cross-index merging in DESCRIBE.
+      case FlatObject:
       case Object:
       // TODO: use Object type once it has been added
       case Nested: