Feature/8941 improve search ranking consistency

server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLElasticSetting.java

@@ -6,7 +6,9 @@
 public enum CQLElasticSetting {
     score("_score"),
     page_size("page_size"),
-    search_after("search_after");
+    search_after("search_after"),
+    // Elasticsearch search preference, the same value routes identical searches to the same shard copies so scoring stays consistent
+    preference("preference");
 
     private final String setting;
 

server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactory.java

@@ -96,6 +96,7 @@ public static Map<CQLElasticSetting, String> getDefaultSetting() {
         defaultSetting.put(CQLElasticSetting.score, null);
         defaultSetting.put(CQLElasticSetting.page_size, null);
         defaultSetting.put(CQLElasticSetting.search_after, null);
+        defaultSetting.put(CQLElasticSetting.preference, null);
         return defaultSetting;
     }
     /**

server/src/main/java/au/org/aodn/ogcapi/server/core/service/CacheNoLandGeometry.java

@@ -38,6 +38,7 @@ public Map<String, StacCollectionModel> getAllNoLandGeometry() {
                 null,
                 null,
                 null,
+                null,
                 null);
 
         return result.collections

server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java

@@ -394,6 +394,7 @@ protected ElasticSearchBase.SearchResult<StacCollectionModel> searchCollectionsB
                 null,
                 createSortOptions(sortBy, CQLFields.class),
                 null,
+                null,
                 null);
     }
 
@@ -494,6 +495,7 @@ protected Supplier<SearchRequest.Builder> buildParameterSearchRequestSupplier(
                     null,
                     createSortOptions(sortBy, CQLFields.class),
                     null,
+                    null,
                     null);
         }
 
@@ -569,7 +571,8 @@ protected Supplier<SearchRequest.Builder> buildParameterSearchRequestSupplier(
                 searchAfter,
                 sortOptions,
                 score,
-                maxSize);
+                maxSize,
+                setting.get(CQLElasticSetting.preference));
     }
 
     @Override
@@ -739,7 +742,8 @@ public ElasticSearchBase.SearchResult<StacCollectionModel> searchByParameters(Li
                     searchAfter,
                     sortOptions,
                     score,
-                    maxSize
+                    maxSize,
+                    setting.get(CQLElasticSetting.preference)
             );
         }
     }

server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java

@@ -180,6 +180,7 @@ protected SearchRequest buildSearchAsYouTypeRequest(
      * @param should - This query will use in the should block of elastic search
      * @param filters - The Query coming from CQL parser
      * @param properties - The fields you want to return in the search, you can search a field but not include in the return
+     * @param preference - Elasticsearch search preference, a fixed value routes identical searches to the same shard copies so the score is consistent, null to leave routing to Elasticsearch
      * @return - The constructed builderSupplier
      */
     protected Supplier<SearchRequest.Builder> buildCollectionSearchRequestSupplier(final List<Query> queries,
@@ -189,9 +190,13 @@ protected Supplier<SearchRequest.Builder> buildCollectionSearchRequestSupplier(f
                                                                                    final List<FieldValue> searchAfter,
                                                                                    final List<SortOptions> sortOptions,
                                                                                    final Double score,
-                                                                                   final Long maxSize) {
+                                                                                   final Long maxSize,
+                                                                                   final String preference) {
         return () -> {
             SearchRequest.Builder builder = new SearchRequest.Builder();
+            if (preference != null && !preference.isBlank()) {
+                builder.preference(preference);
+            }
             builder.index(indexName)
                     // If user query request a page that is smaller then the internal default, then
                     // we use the smaller one. The internal page size is used to get the result by
@@ -344,6 +349,7 @@ else if (should != null && !should.isEmpty()) {
      * @param should - This query will use in the should block of elastic search
      * @param filters - The Query coming from CQL parser
      * @param properties - The fields you want to return in the search, you can search a field but not include in the return
+     * @param preference - Elasticsearch search preference, see buildCollectionSearchRequestSupplier
      * @return - The search result from Elastic query and format in StacCollectionModel
      */
     protected SearchResult<StacCollectionModel> searchCollectionBy(final List<Query> queries,
@@ -353,9 +359,10 @@ protected SearchResult<StacCollectionModel> searchCollectionBy(final List<Query>
                                                                    final List<FieldValue> searchAfter,
                                                                    final List<SortOptions> sortOptions,
                                                                    final Double score,
-                                                                   final Long maxSize) {
+                                                                   final Long maxSize,
+                                                                   final String preference) {
         Supplier<SearchRequest.Builder> builderSupplier = buildCollectionSearchRequestSupplier(
-                queries, should, filters, properties, searchAfter, sortOptions, score, maxSize
+                queries, should, filters, properties, searchAfter, sortOptions, score, maxSize, preference
         );
 
         List<StacCollectionModel> collections = new ArrayList<>();
@@ -486,6 +493,8 @@ protected JsonNode explainCollectionById(String id, Supplier<SearchRequest.Build
                 .index(indexName)
                 .id(id)
                 .query(searchRequest.query())
+                // Same shard copy as the search, otherwise the explained score can differ from the search score
+                .preference(searchRequest.preference())
                 .source(s -> s.fetch(false))
                 .build();
 

server/src/test/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactoryTest.java

@@ -73,6 +73,17 @@ public void prioritySortMetadataIsCollectedAlongsideQuerySettings() throws CQLEx
         assertTrue(factory.isPlatformPrioritySort());
     }
 
+    @Test
+    public void preferenceIsCollectedAsQuerySetting() throws CQLException {
+        CQLToElasticFilterFactory<CQLFields> factory = parse(
+                "preference='search-ranking-test' AND page_size=11 AND "
+                        + "(BBOX(geometry,104,-43,163,-8) OR geometry IS NULL)");
+
+        assertEquals("search-ranking-test", factory.getQuerySetting().get(CQLElasticSetting.preference),
+                "quoted literal arrives without its quotes");
+        assertEquals("11", factory.getQuerySetting().get(CQLElasticSetting.page_size));
+    }
+
     @Test
     public void temporalDuringUsesOverlapRangeQueryAndIncludesOngoingRecords() throws CQLException {
         Filter filter = CompilerUtil.parseFilter(

server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java

@@ -294,6 +294,52 @@ public void cqlPageSizeStillCapsLightweightSearch() throws Exception {
                 "CQL page_size remains the upper bound even for lightweight property lists");
     }
 
+    @Test
+    public void cqlPreferenceIsSetOnNormalAndExplainRequests() throws Exception {
+        CapturingElasticSearch capturingSearch = new CapturingElasticSearch(mockClient);
+        List<String> keywords = List.of("temperature");
+        String cql = "preference='search-ranking-test' AND page_size=3";
+
+        capturingSearch.searchByParameters(
+                keywords,
+                cql,
+                List.of("title"),
+                "-score,-rank",
+                CQLCrsType.EPSG4326);
+        SearchRequest normalRequest = capturingSearch.normalRequest;
+
+        capturingSearch.explainByParameters(
+                keywords,
+                cql,
+                List.of("title"),
+                "-score,-rank",
+                CQLCrsType.EPSG4326,
+                false);
+        SearchRequest explainRequest = capturingSearch.explainRequest;
+
+        assertEquals("search-ranking-test", normalRequest.preference(),
+                "the same preference routes identical searches to the same shard copies");
+        assertEquals(normalRequest.toString(), explainRequest.toString(),
+                "explain must be routed the same way as the search it explains");
+        assertTrue(normalRequest.query().scriptScore().query().bool().filter().isEmpty(),
+                "preference is a setting, not a filter");
+    }
+
+    @Test
+    public void noPreferenceUnlessCqlSetsIt() throws Exception {
+        CapturingElasticSearch capturingSearch = new CapturingElasticSearch(mockClient);
+
+        capturingSearch.searchByParameters(
+                List.of("temperature"),
+                "page_size=3",
+                List.of("title"),
+                "-score,-rank",
+                CQLCrsType.EPSG4326);
+
+        assertNull(capturingSearch.normalRequest.preference(),
+                "routing is left to Elasticsearch when the caller does not opt in");
+    }
+
     private record SearchArguments(
             List<Query> queries,
             List<Query> should,
@@ -302,7 +348,8 @@ private record SearchArguments(
             List<FieldValue> searchAfter,
             List<SortOptions> sortOptions,
             Double score,
-            Long maxSize) {
+            Long maxSize,
+            String preference) {
     }
 
     private static class CapturingElasticSearch extends ElasticSearch {
@@ -325,7 +372,8 @@ protected SearchResult<au.org.aodn.stac.model.StacCollectionModel> searchCollect
                 List<FieldValue> searchAfter,
                 List<SortOptions> sortOptions,
                 Double score,
-                Long maxSize) {
+                Long maxSize,
+                String preference) {
             this.should = should;
             this.arguments = new SearchArguments(
                     queries,
@@ -335,7 +383,8 @@ protected SearchResult<au.org.aodn.stac.model.StacCollectionModel> searchCollect
                     searchAfter,
                     sortOptions,
                     score,
-                    maxSize);
+                    maxSize,
+                    preference);
             this.normalRequest = buildCollectionSearchRequestSupplier(
                     queries,
                     should,
@@ -344,7 +393,8 @@ protected SearchResult<au.org.aodn.stac.model.StacCollectionModel> searchCollect
                     searchAfter,
                     sortOptions,
                     score,
-                    maxSize).get().build();
+                    maxSize,
+                    preference).get().build();
             SearchResult<au.org.aodn.stac.model.StacCollectionModel> result = new SearchResult<>();
             result.setCollections(List.of());
             return result;