feat(lapis): support scalar functions in aggregation fields via dot notation

lapis-docs/astro.config.mjs

@@ -102,6 +102,10 @@ export default defineConfig({
                             label: 'Request ID',
                             link: '/concepts/request-id',
                         },
+                        {
+                            label: 'Computed fields',
+                            link: '/concepts/computed-fields',
+                        },
                         {
                             label: 'Filters',
                             items: [

lapis-docs/src/content/docs/concepts/computed-fields.mdx

@@ -0,0 +1,35 @@
+---
+title: Computed fields
+description: Computed fields
+---
+
+In addition to plain metadata field names, the `fields` parameter of `/sample/aggregated` accepts computed fields
+using the syntax `<field>.<function>`, e.g. `date.isoWeek`.
+A computed field applies a scalar function to a metadata field and groups by the result,
+instead of grouping by the raw field value.
+
+**Example:** to count sequences per ISO week instead of per exact date:
+
+```
+[URL to LAPIS instance]/sample/aggregated?fields=date.isoWeek
+```
+
+```json
+{
+    "fields": ["date.isoWeek"]
+}
+```
+
+The response uses the full `<field>.<function>` string as the column/property key, e.g. `date.isoWeek`.
+
+## Available functions
+
+| Function  | Applicable field types | Description                          |
+| --------- | ---------------------- | ------------------------------------ |
+| `isoWeek` | `date`                 | The ISO 8601 week of the date field. |
+
+:::note
+Computed fields are only supported in the `fields` parameter of `/sample/aggregated`
+(and the analogous endpoints for other sequence types).
+They are not supported by endpoints that return the underlying data rows, such as `/sample/details`.
+:::

lapis-docs/src/content/docs/references/introduction.mdx

@@ -29,3 +29,4 @@ Fields allow you to dictate the grouping of the returned data.
 For example, to find out the number of sequences from each country,you would use `GET /sample/aggregated?fields=country`.
 The available fields are documented in [Fields](../references/fields),
 and you can test them at [Open API / Swagger](../references/open-api-definition).
+Some fields also support [computed fields](../concepts/computed-fields), e.g. `fields=date.isoWeek`.

lapis-docs/tests/docs.spec.ts

@@ -47,6 +47,7 @@ const conceptsPages = prependToRelativeUrl(
         { title: 'Customizable FASTA headers', relativeUrl: '/customizable-fasta-headers' },
         { title: 'Data versions', relativeUrl: '/data-versions' },
         { title: 'Request ID', relativeUrl: '/request-id' },
+        { title: 'Computed fields', relativeUrl: '/computed-fields' },
         { title: 'Mutation filters', relativeUrl: '/mutation-filters' },
         { title: 'Insertion filters', relativeUrl: '/insertion-filters' },
         { title: 'Ambiguous symbols', relativeUrl: '/ambiguous-symbols' },

lapis-e2e/test/aggregated.spec.ts

@@ -172,6 +172,56 @@ describe('The /aggregated endpoint', () => {
     expect(resultJson.error.detail).to.include("Unknown field: 'notAField', known values are [primaryKey,");
   });
 
+  it('should stratify by a computed field using dot notation', async () => {
+    const result = await lapisClient.postAggregated({
+      aggregatedPostRequest: {
+        date: '2021-06-05',
+        fields: ['date.isoWeek'],
+      },
+    });
+
+    expect(result.data).to.have.length(1);
+    expect(result.data[0]).to.have.property('count', 1);
+    expect(result.data[0]).to.have.property('date.isoWeek', 22);
+  });
+
+  it('should order by a computed field using dot notation', async () => {
+    const result = await lapisClient.postAggregated({
+      aggregatedPostRequest: {
+        date: '2021-06-05',
+        fields: ['date.isoWeek'],
+        orderBy: [{ field: 'date.isoWeek', type: 'ascending' }],
+      },
+    });
+
+    expect(result.data).to.have.length(1);
+    expect(result.data[0]).to.have.property('date.isoWeek', 22);
+  });
+
+  it('should return bad request for an unknown scalar function', async () => {
+    const urlParams = new URLSearchParams({
+      fields: 'date.notAFunction',
+    });
+
+    const result = await getAggregated(urlParams);
+
+    expect(result.status).equals(400);
+    const resultJson = await result.json();
+    expect(resultJson.error.detail).to.include("Unknown scalar function 'notAFunction'");
+  });
+
+  it('should return bad request for a scalar function applied to a field of the wrong type', async () => {
+    const urlParams = new URLSearchParams({
+      fields: 'country.isoWeek',
+    });
+
+    const result = await getAggregated(urlParams);
+
+    expect(result.status).equals(400);
+    const resultJson = await result.json();
+    expect(resultJson.error.detail).to.include("is not valid for field 'country'");
+  });
+
   it('should return bad request for invalid variant query', async () => {
     const urlParams = new URLSearchParams({
       variantQuery: 'not a valid variant query',

lapis-e2e/test/details.spec.ts

@@ -32,6 +32,18 @@ describe('The /details endpoint', () => {
     });
   });
 
+  it('should return bad request for a computed field', async () => {
+    const result = await fetch(
+      basePath + '/sample/details?' + new URLSearchParams({ fields: 'date.isoWeek' })
+    );
+
+    expect(result.status).equals(400);
+    const resultJson = await result.json();
+    expect(resultJson.error.detail).to.include(
+      'Scalar functions are not supported in fields for this endpoint: date.isoWeek'
+    );
+  });
+
   it('should fetch the correct key for "isNull" filter', async () => {
     const result = await lapisClient.postDetails({
       detailsPostRequest: {

lapis/src/main/kotlin/org/genspectrum/lapis/controller/ControllerDescriptions.kt

@@ -69,11 +69,14 @@ const val AGGREGATED_GROUP_BY_FIELDS_DESCRIPTION =
     """The fields to stratify by.
 If empty, only the overall count is returned.
 If requesting CSV or TSV data, the columns are ordered in the same order as the fields are specified here.
-You can provide metadata fields."""
+You can provide metadata fields.
+A field can also be a computed field of the form "<field>.<function>" (e.g. "date.isoWeek"), which applies a
+scalar function to the field and stratifies by the result. The response uses the full "<field>.<function>" string
+as the field name. Currently, the only supported function is "isoWeek", which is only valid for date fields."""
 const val AGGREGATED_ORDER_BY_FIELDS_DESCRIPTION =
-    """The fields of the response to order by. 
-    Fields specified here must either be \"count\" or also be present in \"fields\".
-    You can also use \"random\" or \"random(SEED)\" where SEED is an integer."""
+    """The fields of the response to order by.
+    Fields specified here must either be "count" or also be present in "fields".
+    You can also use "random" or "random(SEED)" where SEED is an integer."""
 const val DETAILS_FIELDS_DESCRIPTION =
     """The fields that the response items should contain.
 If empty, all fields are returned.

lapis/src/main/kotlin/org/genspectrum/lapis/model/SiloQueryModel.kt

@@ -4,6 +4,7 @@ import org.genspectrum.lapis.config.DatabaseConfig
 import org.genspectrum.lapis.config.ReferenceGenomeSchema
 import org.genspectrum.lapis.request.AggregatedFiltersRequest
 import org.genspectrum.lapis.request.CommonSequenceFilters
+import org.genspectrum.lapis.request.ComputedField
 import org.genspectrum.lapis.request.DetailsFiltersRequest
 import org.genspectrum.lapis.request.MRCASequenceFiltersRequest
 import org.genspectrum.lapis.request.MutationProportionsRequest
@@ -46,6 +47,7 @@ class SiloQueryModel(
                     limit = sequenceFilters.limit,
                     offset = sequenceFilters.offset,
                     sequencePositionFields = sequenceFilters.fields.filterIsInstance<SequencePositionField>(),
+                    computedFields = sequenceFilters.fields.filterIsInstance<ComputedField>(),
                 ),
                 siloFilterExpressionMapper.map(sequenceFilters),
             ),

lapis/src/main/kotlin/org/genspectrum/lapis/model/mutationsOverTime/QueriesOverTimeModel.kt

@@ -339,10 +339,10 @@ class QueriesOverTimeModel(
         siloClient.sendQueryAndGetDataVersion(
             SiloQuery(
                 SiloAction.aggregated(
-                    listOf(dateField),
-                    OrderBySpec.EMPTY,
-                    null,
-                    null,
+                    groupByFields = listOf(dateField),
+                    orderByFields = OrderBySpec.EMPTY,
+                    limit = null,
+                    offset = null,
                 ),
                 And(
                     children = listOfNotNull(

lapis/src/main/kotlin/org/genspectrum/lapis/request/Field.kt

@@ -18,3 +18,10 @@ data class SequencePositionField(
     /** Used both as the SaneQL alias and as the response column key, e.g. `S[501]` or `[501]` for shorthand. */
     override val outputColumnName: String get() = if (isSingleSegment) "[$position]" else "$sequenceName[$position]"
 }
+
+data class ComputedField(
+    val sourceField: String,
+    val function: ScalarFunction,
+) : Field {
+    override val outputColumnName: String get() = "$sourceField.${function.saneQlMethodName}"
+}

lapis/src/main/kotlin/org/genspectrum/lapis/request/PhyloTreeSequenceFiltersRequest.kt

@@ -60,7 +60,7 @@ class PhyloTreeSequenceFiltersRequestDeserializer(
             parsedCommonFields.orderByFields,
             parsedCommonFields.limit,
             parsedCommonFields.offset,
-            phyloTreeField.fieldName,
+            phyloTreeField.outputColumnName,
         )
     }
 }
@@ -89,7 +89,7 @@ class MRCASequenceFiltersRequestDeserializer(
             parsedCommonFields.orderByFields,
             parsedCommonFields.limit,
             parsedCommonFields.offset,
-            phyloTreeField.fieldName,
+            phyloTreeField.outputColumnName,
             printNodesNotInTree = printNodesNotInTree,
         )
     }

lapis/src/main/kotlin/org/genspectrum/lapis/request/ScalarFunction.kt

@@ -0,0 +1,14 @@
+package org.genspectrum.lapis.request
+
+import org.genspectrum.lapis.config.MetadataType
+
+/**
+ * An enum of scalar functions supported by SILO.
+ * New functions need to be whitelisted here explicitly.
+ */
+enum class ScalarFunction(
+    val saneQlMethodName: String,
+    val validForTypes: Set<MetadataType>,
+) {
+    ISO_WEEK("isoWeek", setOf(MetadataType.DATE)),
+}

lapis/src/main/kotlin/org/genspectrum/lapis/request/converter/AggregatedFieldConverter.kt

@@ -3,12 +3,15 @@ package org.genspectrum.lapis.request.converter
 import org.genspectrum.lapis.request.Field
 import org.springframework.stereotype.Component
 
-/** Resolves a field that may be a plain metadata field or a sequence position field. Used by `/aggregated`. */
+/** Resolves a field that may be a plain metadata field, sequence position field, or scalar function field. Used by `/aggregated`. */
 @Component
 class AggregatedFieldConverter(
     private val sequencePositionFieldConverter: SequencePositionFieldConverter,
+    private val scalarFunctionFieldConverter: ScalarFunctionFieldConverter,
     private val metadataFieldConverter: MetadataFieldConverter,
 ) : FieldConverter<Field> {
     override fun convert(source: String): Field =
-        sequencePositionFieldConverter.tryConvert(source) ?: metadataFieldConverter.convert(source)
+        sequencePositionFieldConverter.tryConvert(source)
+            ?: scalarFunctionFieldConverter.tryConvert(source)
+            ?: metadataFieldConverter.convert(source)
 }

lapis/src/main/kotlin/org/genspectrum/lapis/request/converter/OrderByFieldsCleaner.kt

@@ -1,10 +1,19 @@
 package org.genspectrum.lapis.request.converter
 
+import org.genspectrum.lapis.controller.BadRequestException
 import org.springframework.stereotype.Component
 
 @Component
 class OrderByFieldsCleaner(
     private val caseInsensitiveFieldsCleaner: CaseInsensitiveFieldsCleaner,
+    private val scalarFunctionFieldConverter: ScalarFunctionFieldConverter,
 ) {
-    fun clean(fieldName: String): String = caseInsensitiveFieldsCleaner.clean(fieldName) ?: fieldName
+    fun clean(fieldName: String): String =
+        try {
+            scalarFunctionFieldConverter.tryConvert(fieldName)?.outputColumnName
+                ?: caseInsensitiveFieldsCleaner.clean(fieldName)
+                ?: fieldName
+        } catch (e: BadRequestException) {
+            fieldName
+        }
 }

lapis/src/main/kotlin/org/genspectrum/lapis/request/converter/PlainFieldConverter.kt

@@ -17,6 +17,11 @@ class PlainFieldConverter(
                 "Sequence position fields are not supported here: ${it.outputColumnName}",
             )
         }
+        if ('.' in source) {
+            throw BadRequestException(
+                "Scalar functions are not supported in fields for this endpoint: $source",
+            )
+        }
         return metadataFieldConverter.convert(source)
     }
 }

lapis/src/main/kotlin/org/genspectrum/lapis/request/converter/ScalarFunctionFieldConverter.kt

@@ -0,0 +1,46 @@
+package org.genspectrum.lapis.request.converter
+
+import org.genspectrum.lapis.config.DatabaseConfig
+import org.genspectrum.lapis.config.MetadataType
+import org.genspectrum.lapis.controller.BadRequestException
+import org.genspectrum.lapis.request.ComputedField
+import org.genspectrum.lapis.request.ScalarFunction
+import org.springframework.stereotype.Component
+
+@Component
+class ScalarFunctionFieldConverter(
+    private val caseInsensitiveFieldsCleaner: CaseInsensitiveFieldsCleaner,
+    private val databaseConfig: DatabaseConfig,
+) {
+    private val fieldTypesByLowercaseName: Map<String, MetadataType> =
+        databaseConfig.schema.metadata.associateBy({ it.name.lowercase() }, { it.type })
+
+    fun tryConvert(source: String): ComputedField? {
+        if ('.' !in source) return null
+        val dotIndex = source.lastIndexOf('.')
+        val rawField = source.substring(0, dotIndex)
+        val rawFunction = source.substring(dotIndex + 1)
+
+        val cleanedField = caseInsensitiveFieldsCleaner.clean(rawField)
+            ?: throw BadRequestException(
+                "Unknown field '$rawField' in '$source'. " +
+                    "Known fields: ${caseInsensitiveFieldsCleaner.getKnownFields()}",
+            )
+
+        val function = ScalarFunction.entries.find { it.saneQlMethodName.equals(rawFunction, ignoreCase = true) }
+            ?: throw BadRequestException(
+                "Unknown scalar function '$rawFunction' in '$source'. " +
+                    "Available functions: ${ScalarFunction.entries.joinToString { it.saneQlMethodName }}",
+            )
+
+        val fieldType = fieldTypesByLowercaseName[cleanedField.lowercase()]!!
+        if (fieldType !in function.validForTypes) {
+            throw BadRequestException(
+                "Scalar function '${function.saneQlMethodName}' is not valid for field '$cleanedField' of type " +
+                    "$fieldType. Valid types: ${function.validForTypes.joinToString()}",
+            )
+        }
+
+        return ComputedField(cleanedField, function)
+    }
+}

lapis/src/main/kotlin/org/genspectrum/lapis/silo/SaneQlAst.kt

@@ -62,7 +62,7 @@ data class SaneQlIdentifier(
 
 /**
  * A `{item1, item2, ...}` list literal. Acts as a record when its [items] are [SaneQlAssignment]s,
- * e.g. `{count:=count()}`.
+ * e.g. `{"count":=count()}`.
  */
 data class SaneQlList(
     val items: List<SaneQlExpression>,