Feature/9002 optimise reranking algorithm

server/src/main/java/au/org/aodn/ogcapi/server/core/model/ExplainSimplifiedResponse.java

@@ -33,8 +33,8 @@ public static class Total {
     @Builder
     @JsonInclude(JsonInclude.Include.NON_NULL)
     @JsonPropertyOrder({
-            "rank", "id", "title", "final_score",
-            "es_relevance", "internal_score", "quality_multiplier", "matched_terms", "filters"
+            "rank", "id", "title", "final_score", "es_relevance", "internal_score",
+            "quality_multiplier", "sort_values", "matched_terms", "filters"
     })
     public static class Hit {
         private Integer rank;
@@ -56,6 +56,12 @@ public static class Hit {
         @JsonProperty("quality_multiplier")
         private Double qualityMultiplier;
 
+        /**
+         * The sort key values that decided this hit's rank, strongest first, e.g. the dataset_group priority ahead of _score.
+         */
+        @JsonProperty("sort_values")
+        private List<SortValue> sortValues;
+
         /** The per matched field and term and score which contributed to the esRelevance */
         @JsonProperty("matched_terms")
         private List<MatchedTerm> matchedTerms;
@@ -67,6 +73,15 @@ public static class Hit {
         private List<MatchedFilter> filters;
     }
 
+    @Data
+    @Builder
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    public static class SortValue {
+        /** The sort key that produced the value, e.g. "summaries.dataset_group" or "_score" */
+        private String field;
+        private Object value;
+    }
+
     @Data
     @Builder
     @JsonInclude(JsonInclude.Include.NON_NULL)

server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java

@@ -8,6 +8,7 @@
 
 import java.io.StringReader;
 import java.util.List;
+import java.util.Map;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.Arrays;
@@ -355,6 +356,7 @@ private static String normalizeDatasetGroupTerm(String value) {
                 return value.toLowerCase(Locale.ROOT).trim();
         }
 
+        // used by the dataset_group CQL filter only without boost
         private static Query datasetGroupTermsQuery(List<String> values) {
                 List<FieldValue> terms = values.stream()
                         .filter(value -> !value.isBlank())
@@ -364,19 +366,16 @@ private static Query datasetGroupTermsQuery(List<String> values) {
 
                 return TermsQuery.of(query -> query
                                 .field(StacSummeries.DatasetGroup.searchField)
-                                .terms(field -> field.value(terms))
-                                .boost(100.0F))
+                                .terms(field -> field.value(terms)))
                         ._toQuery();
         }
 
         /**
-         * Builds a dataset-group query specifically for free-text search.
-         * Unquoted input includes both the complete normalized input and its individual words.
-         * For example, "csiro temperature" => ["csiro temperature", "csiro", "temperature"].
-         * Double-quoted free-text input is treated as one exact value.
+         * Expands a free-text term into dataset-group candidate values. For example, "csiro temperature" => ["csiro temperature", "csiro", "temperature"].
+         * Unquoted input includes both the complete normalized input and its individual words. Double-quoted free-text input is treated as one exact value.
          * Dataset-group CQL filters continue to use getPropertyEqualToQuery().
          */
-        public static Query getDatasetGroupTextSearchQuery(
+        public static List<String> getDatasetGroupCandidates(
                 String literal,
                 boolean isExact) {
 
@@ -389,7 +388,30 @@ public static Query getDatasetGroupTextSearchQuery(
                                 Arrays.stream(normalized.split("\\s+")))
                         .toList();
 
-                return datasetGroupTermsQuery(candidates);
+                return candidates.stream()
+                        .filter(value -> !value.isBlank())
+                        .distinct()
+                        .toList();
+        }
+
+        /**
+         * Priority sort for dataset_group: records whose group matches any of the search terms rank first.
+         */
+        public static SortOptions getDatasetGroupPrioritySort(List<String> candidates) {
+                String field = StacSummeries.DatasetGroup.searchField;
+                return new SortOptions.Builder().script(s -> s
+                                .type(ScriptSortType.Number)
+                                .script(sc -> sc
+                                                .lang("painless")
+                                                .params(Map.of("groups", JsonData.of(candidates)))
+                                                .source("if (!doc.containsKey('" + field + "') || doc['" + field
+                                                                + "'].empty) { return 0; } "
+                                                                + "for (g in params.groups) { "
+                                                                + "if (doc['" + field + "'].contains(g)) { return 1; } "
+                                                                + "} "
+                                                                + "return 0;"))
+                                .order(SortOrder.Desc))
+                        .build();
         }
 
         @Override

server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java

@@ -423,6 +423,55 @@ public ElasticSearchBase.SearchResult<StacCollectionModel> searchAllCollections(
     }
 
 
+    /**
+     * Builds the relevance should clauses for each search keyword. Shared by searchByParameters (search) and buildParameterSearchRequestSupplier (explain) so the two align with each other.
+     * These should clauses contribute to the Elasticsearch BM25 relevance score.
+     */
+    private static List<Query> createKeywordShouldClauses(List<String> keywords) {
+        List<Query> should = new ArrayList<>();
+        for (String t : keywords) {
+            // If user's input (keywords) wrapped with double quot", and the text is not empty, treat the user intend to search with the exact term, so fuzzy matching not applied on title and description
+            boolean isExact = t.startsWith("\"") && t.endsWith("\"") && t.length() > 2;
+            // If search text with double quote, remove quotee, otherwise keeps same
+            String term = isExact ? t.substring(1, t.length() - 1) : t;
+
+            if (isExact) {
+                // Match phrase in original title and description, not use fuzzy fields
+                should.add(CQLFields.title.getPropertyEqualToQuery(term));
+                should.add(CQLFields.description.getPropertyEqualToQuery(term));
+            }
+            else {
+                should.add(CQLFields.fuzzy_title.getPropertyEqualToQuery(term));
+                should.add(CQLFields.fuzzy_desc.getPropertyEqualToQuery(term));
+            }
+            should.add(CQLFields.parameter_vocabs.getPropertyEqualToQuery(term));
+            should.add(CQLFields.organisation_vocabs.getPropertyEqualToQuery(term));
+            should.add(CQLFields.platform_vocabs.getPropertyEqualToQuery(term));
+            should.add(CQLFields.id.getPropertyEqualToQuery(term));
+            // Acronym match on the *.synonyms sub-fields, e.g. "SOOP" -> "ships of opportunity".
+            should.add(CQLFields.acronym_title.getPropertyEqualToQuery(term));
+            should.add(CQLFields.acronym_desc.getPropertyEqualToQuery(term));
+            // credit_contains uses match query by default, exact match is not applied here
+            should.add(CQLFields.credit_contains.getPropertyEqualToQuery(term));
+        }
+        return should;
+    }
+
+    /**
+     * Dataset-group candidate values for the priority sort, expanded from the search keywords with the same exact/quoted handling as the should clauses.
+     */
+    private static List<String> collectDatasetGroupCandidates(List<String> keywords) {
+        return keywords.stream()
+                .map(t -> {
+                    boolean isExact = t.startsWith("\"") && t.endsWith("\"") && t.length() > 2;
+                    String term = isExact ? t.substring(1, t.length() - 1) : t;
+                    return CQLFields.getDatasetGroupCandidates(term, isExact);
+                })
+                .flatMap(List::stream)
+                .distinct()
+                .toList();
+    }
+
     /**
      * Build SearchRequest for searchByParameters and explainByParameters
      * */
@@ -451,34 +500,10 @@ protected Supplier<SearchRequest.Builder> buildParameterSearchRequestSupplier(
         }
 
         List<Query> should = null;
+        List<String> datasetGroupCandidates = List.of();
         if (keywords != null && !keywords.isEmpty()) {
-            should = new ArrayList<>();
-
-            for (String t : keywords) {
-                boolean isExact = t.startsWith("\"") && t.endsWith("\"") && t.length() > 2;
-                String term = isExact ? t.substring(1, t.length() - 1) : t;
-
-                if (isExact) {
-                    should.add(CQLFields.title.getPropertyEqualToQuery(term));
-                    should.add(CQLFields.description.getPropertyEqualToQuery(term));
-                }
-                else {
-                    should.add(CQLFields.fuzzy_title.getPropertyEqualToQuery(term));
-                    should.add(CQLFields.fuzzy_desc.getPropertyEqualToQuery(term));
-                }
-
-                should.add(CQLFields.parameter_vocabs.getPropertyEqualToQuery(term));
-                should.add(CQLFields.organisation_vocabs.getPropertyEqualToQuery(term));
-                should.add(CQLFields.platform_vocabs.getPropertyEqualToQuery(term));
-                should.add(CQLFields.id.getPropertyEqualToQuery(term));
-                // Acronym match on the *.synonyms sub-fields, e.g. "SOOP" -> "ships of opportunity".
-                should.add(CQLFields.acronym_title.getPropertyEqualToQuery(term));
-                should.add(CQLFields.acronym_desc.getPropertyEqualToQuery(term));
-                // credit_contains uses match query by default, exact match is not applied here
-                should.add(CQLFields.credit_contains.getPropertyEqualToQuery(term));
-                // match the acronym for AODN partner organisations
-                should.add(CQLFields.getDatasetGroupTextSearchQuery(term, isExact));
-            }
+            should = createKeywordShouldClauses(keywords);
+            datasetGroupCandidates = collectDatasetGroupCandidates(keywords);
         }
 
         List<Query> filters = new ArrayList<>();
@@ -532,6 +557,12 @@ protected Supplier<SearchRequest.Builder> buildParameterSearchRequestSupplier(
             sortOptions.add(0, CQLFields.platform_vocabs.getSortBuilder().apply(SortOrder.Desc).build());
         }
 
+        // Records whose dataset_group matches a search term rank first among recalled records; prepended last so it is the strongest sort key.
+        if (should != null && !should.isEmpty() && !datasetGroupCandidates.isEmpty()) {
+            if (sortOptions == null) sortOptions = new ArrayList<>();
+            sortOptions.add(0, CQLFields.getDatasetGroupPrioritySort(datasetGroupCandidates));
+        }
+
         return buildCollectionSearchRequestSupplier(
                 null,
                 should,
@@ -552,41 +583,10 @@ public ElasticSearchBase.SearchResult<StacCollectionModel> searchByParameters(Li
         else {
 
             List<Query> should = null;
+            List<String> datasetGroupCandidates = List.of();
             if(keywords != null && !keywords.isEmpty()) {
-                should = new ArrayList<>();
-
-                for (String t : keywords) {
-                    // If user's input (keywords) starts and ends with quote ", and the text is not empty
-                    // treat the user intend to search with the exact term,
-                    // instead of searching in fuzzy fields i.e., fuzzy_title and fuzzy_desc,
-                    // search in the original title and description fields
-                    // other fields are searched with the same term regardless of exact match or not, as they do not use fuzzy matching.
-                    boolean isExact = t.startsWith("\"") && t.endsWith("\"") && t.length() > 2;
-                    // If search text with double quote, remove quotes,
-                    // otherwise keeps same
-                    String term = isExact ? t.substring(1, t.length() - 1) : t;
-
-                    if (isExact) {
-                        // Match phrase in original title and description, not use fuzzy fields
-                        should.add(CQLFields.title.getPropertyEqualToQuery(term));
-                        should.add(CQLFields.description.getPropertyEqualToQuery(term));
-                    }
-                    else {
-                        should.add(CQLFields.fuzzy_title.getPropertyEqualToQuery(term));
-                        should.add(CQLFields.fuzzy_desc.getPropertyEqualToQuery(term));
-                    }
-                    should.add(CQLFields.parameter_vocabs.getPropertyEqualToQuery(term));
-                    should.add(CQLFields.organisation_vocabs.getPropertyEqualToQuery(term));
-                    should.add(CQLFields.platform_vocabs.getPropertyEqualToQuery(term));
-                    should.add(CQLFields.id.getPropertyEqualToQuery(term));
-                    // Acronym match on the *.synonyms sub-fields, e.g. "SOOP" -> "ships of opportunity".
-                    should.add(CQLFields.acronym_title.getPropertyEqualToQuery(term));
-                    should.add(CQLFields.acronym_desc.getPropertyEqualToQuery(term));
-                    // credit_contains uses match query by default, exact match is not applied here
-                    should.add(CQLFields.credit_contains.getPropertyEqualToQuery(term));
-                    // match the acronym for AODN partner organisations
-                    should.add(CQLFields.getDatasetGroupTextSearchQuery(term, isExact));
-                }
+                should = createKeywordShouldClauses(keywords);
+                datasetGroupCandidates = collectDatasetGroupCandidates(keywords);
             }
 
             List<Query> filters = new ArrayList<>();
@@ -678,6 +678,14 @@ public ElasticSearchBase.SearchResult<StacCollectionModel> searchByParameters(Li
                 sortOptions.add(0, CQLFields.platform_vocabs.getSortBuilder().apply(SortOrder.Desc).build());
             }
 
+            // Records whose dataset_group matches a search term rank first among recalled records; prepended last so it is the strongest sort key.
+            if (should != null && !should.isEmpty() && !datasetGroupCandidates.isEmpty()) {
+                if (sortOptions == null) {
+                    sortOptions = new ArrayList<>();
+                }
+                sortOptions.add(0, CQLFields.getDatasetGroupPrioritySort(datasetGroupCandidates));
+            }
+
             return searchCollectionBy(
                     null,
                     should,

server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java

@@ -427,8 +427,8 @@ protected JsonNode explainCollectionBy(Supplier<SearchRequest.Builder> requestSu
         SearchResponse<ObjectNode> response = esClient.search(request, ObjectNode.class);
 
         if (simplified) {
-            // keep the request which carries the exact values of every terms clause
-            return mapper.valueToTree(ExplainSimplifier.from(response, toJsonNode(request)));
+            // the request carries the names of the sort keys, the response their values only so pass the request to explain as well
+            return mapper.valueToTree(ExplainSimplifier.from(response, request.sort()));
         }
 
         ObjectNode result = mapper.createObjectNode();