diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ExplainSimplifiedResponse.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ExplainSimplifiedResponse.java index 2aad03b6..2e5fe460 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ExplainSimplifiedResponse.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ExplainSimplifiedResponse.java @@ -33,8 +33,8 @@ public static class Total { @Builder @JsonInclude(JsonInclude.Include.NON_NULL) @JsonPropertyOrder({ - "rank", "id", "title", "final_score", - "es_relevance", "internal_score", "quality_multiplier", "matched_terms", "filters" + "rank", "id", "title", "final_score", "es_relevance", "internal_score", + "quality_multiplier", "sort_values", "matched_terms", "filters" }) public static class Hit { private Integer rank; @@ -56,6 +56,12 @@ public static class Hit { @JsonProperty("quality_multiplier") private Double qualityMultiplier; + /** + * The sort key values that decided this hit's rank, strongest first, e.g. the dataset_group priority ahead of _score. + */ + @JsonProperty("sort_values") + private List sortValues; + /** The per matched field and term and score which contributed to the esRelevance */ @JsonProperty("matched_terms") private List matchedTerms; @@ -67,6 +73,15 @@ public static class Hit { private List filters; } + @Data + @Builder + @JsonInclude(JsonInclude.Include.NON_NULL) + public static class SortValue { + /** The sort key that produced the value, e.g. "summaries.dataset_group" or "_score" */ + private String field; + private Object value; + } + @Data @Builder @JsonInclude(JsonInclude.Include.NON_NULL) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java index c6c46d00..4b9df62d 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java @@ -8,6 +8,7 @@ import java.io.StringReader; import java.util.List; +import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; import java.util.Arrays; @@ -355,6 +356,7 @@ private static String normalizeDatasetGroupTerm(String value) { return value.toLowerCase(Locale.ROOT).trim(); } + // used by the dataset_group CQL filter only without boost private static Query datasetGroupTermsQuery(List values) { List terms = values.stream() .filter(value -> !value.isBlank()) @@ -364,19 +366,16 @@ private static Query datasetGroupTermsQuery(List values) { return TermsQuery.of(query -> query .field(StacSummeries.DatasetGroup.searchField) - .terms(field -> field.value(terms)) - .boost(100.0F)) + .terms(field -> field.value(terms))) ._toQuery(); } /** - * Builds a dataset-group query specifically for free-text search. - * Unquoted input includes both the complete normalized input and its individual words. - * For example, "csiro temperature" => ["csiro temperature", "csiro", "temperature"]. - * Double-quoted free-text input is treated as one exact value. + * Expands a free-text term into dataset-group candidate values. For example, "csiro temperature" => ["csiro temperature", "csiro", "temperature"]. + * Unquoted input includes both the complete normalized input and its individual words. Double-quoted free-text input is treated as one exact value. * Dataset-group CQL filters continue to use getPropertyEqualToQuery(). */ - public static Query getDatasetGroupTextSearchQuery( + public static List getDatasetGroupCandidates( String literal, boolean isExact) { @@ -389,7 +388,30 @@ public static Query getDatasetGroupTextSearchQuery( Arrays.stream(normalized.split("\\s+"))) .toList(); - return datasetGroupTermsQuery(candidates); + return candidates.stream() + .filter(value -> !value.isBlank()) + .distinct() + .toList(); + } + + /** + * Priority sort for dataset_group: records whose group matches any of the search terms rank first. + */ + public static SortOptions getDatasetGroupPrioritySort(List candidates) { + String field = StacSummeries.DatasetGroup.searchField; + return new SortOptions.Builder().script(s -> s + .type(ScriptSortType.Number) + .script(sc -> sc + .lang("painless") + .params(Map.of("groups", JsonData.of(candidates))) + .source("if (!doc.containsKey('" + field + "') || doc['" + field + + "'].empty) { return 0; } " + + "for (g in params.groups) { " + + "if (doc['" + field + "'].contains(g)) { return 1; } " + + "} " + + "return 0;")) + .order(SortOrder.Desc)) + .build(); } @Override diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index 6ee28549..868b477e 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -423,6 +423,55 @@ public ElasticSearchBase.SearchResult searchAllCollections( } + /** + * Builds the relevance should clauses for each search keyword. Shared by searchByParameters (search) and buildParameterSearchRequestSupplier (explain) so the two align with each other. + * These should clauses contribute to the Elasticsearch BM25 relevance score. + */ + private static List createKeywordShouldClauses(List keywords) { + List should = new ArrayList<>(); + for (String t : keywords) { + // If user's input (keywords) wrapped with double quot", and the text is not empty, treat the user intend to search with the exact term, so fuzzy matching not applied on title and description + boolean isExact = t.startsWith("\"") && t.endsWith("\"") && t.length() > 2; + // If search text with double quote, remove quotee, otherwise keeps same + String term = isExact ? t.substring(1, t.length() - 1) : t; + + if (isExact) { + // Match phrase in original title and description, not use fuzzy fields + should.add(CQLFields.title.getPropertyEqualToQuery(term)); + should.add(CQLFields.description.getPropertyEqualToQuery(term)); + } + else { + should.add(CQLFields.fuzzy_title.getPropertyEqualToQuery(term)); + should.add(CQLFields.fuzzy_desc.getPropertyEqualToQuery(term)); + } + should.add(CQLFields.parameter_vocabs.getPropertyEqualToQuery(term)); + should.add(CQLFields.organisation_vocabs.getPropertyEqualToQuery(term)); + should.add(CQLFields.platform_vocabs.getPropertyEqualToQuery(term)); + should.add(CQLFields.id.getPropertyEqualToQuery(term)); + // Acronym match on the *.synonyms sub-fields, e.g. "SOOP" -> "ships of opportunity". + should.add(CQLFields.acronym_title.getPropertyEqualToQuery(term)); + should.add(CQLFields.acronym_desc.getPropertyEqualToQuery(term)); + // credit_contains uses match query by default, exact match is not applied here + should.add(CQLFields.credit_contains.getPropertyEqualToQuery(term)); + } + return should; + } + + /** + * Dataset-group candidate values for the priority sort, expanded from the search keywords with the same exact/quoted handling as the should clauses. + */ + private static List collectDatasetGroupCandidates(List keywords) { + return keywords.stream() + .map(t -> { + boolean isExact = t.startsWith("\"") && t.endsWith("\"") && t.length() > 2; + String term = isExact ? t.substring(1, t.length() - 1) : t; + return CQLFields.getDatasetGroupCandidates(term, isExact); + }) + .flatMap(List::stream) + .distinct() + .toList(); + } + /** * Build SearchRequest for searchByParameters and explainByParameters * */ @@ -451,34 +500,10 @@ protected Supplier buildParameterSearchRequestSupplier( } List should = null; + List datasetGroupCandidates = List.of(); if (keywords != null && !keywords.isEmpty()) { - should = new ArrayList<>(); - - for (String t : keywords) { - boolean isExact = t.startsWith("\"") && t.endsWith("\"") && t.length() > 2; - String term = isExact ? t.substring(1, t.length() - 1) : t; - - if (isExact) { - should.add(CQLFields.title.getPropertyEqualToQuery(term)); - should.add(CQLFields.description.getPropertyEqualToQuery(term)); - } - else { - should.add(CQLFields.fuzzy_title.getPropertyEqualToQuery(term)); - should.add(CQLFields.fuzzy_desc.getPropertyEqualToQuery(term)); - } - - should.add(CQLFields.parameter_vocabs.getPropertyEqualToQuery(term)); - should.add(CQLFields.organisation_vocabs.getPropertyEqualToQuery(term)); - should.add(CQLFields.platform_vocabs.getPropertyEqualToQuery(term)); - should.add(CQLFields.id.getPropertyEqualToQuery(term)); - // Acronym match on the *.synonyms sub-fields, e.g. "SOOP" -> "ships of opportunity". - should.add(CQLFields.acronym_title.getPropertyEqualToQuery(term)); - should.add(CQLFields.acronym_desc.getPropertyEqualToQuery(term)); - // credit_contains uses match query by default, exact match is not applied here - should.add(CQLFields.credit_contains.getPropertyEqualToQuery(term)); - // match the acronym for AODN partner organisations - should.add(CQLFields.getDatasetGroupTextSearchQuery(term, isExact)); - } + should = createKeywordShouldClauses(keywords); + datasetGroupCandidates = collectDatasetGroupCandidates(keywords); } List filters = new ArrayList<>(); @@ -532,6 +557,12 @@ protected Supplier buildParameterSearchRequestSupplier( sortOptions.add(0, CQLFields.platform_vocabs.getSortBuilder().apply(SortOrder.Desc).build()); } + // Records whose dataset_group matches a search term rank first among recalled records; prepended last so it is the strongest sort key. + if (should != null && !should.isEmpty() && !datasetGroupCandidates.isEmpty()) { + if (sortOptions == null) sortOptions = new ArrayList<>(); + sortOptions.add(0, CQLFields.getDatasetGroupPrioritySort(datasetGroupCandidates)); + } + return buildCollectionSearchRequestSupplier( null, should, @@ -552,41 +583,10 @@ public ElasticSearchBase.SearchResult searchByParameters(Li else { List should = null; + List datasetGroupCandidates = List.of(); if(keywords != null && !keywords.isEmpty()) { - should = new ArrayList<>(); - - for (String t : keywords) { - // If user's input (keywords) starts and ends with quote ", and the text is not empty - // treat the user intend to search with the exact term, - // instead of searching in fuzzy fields i.e., fuzzy_title and fuzzy_desc, - // search in the original title and description fields - // other fields are searched with the same term regardless of exact match or not, as they do not use fuzzy matching. - boolean isExact = t.startsWith("\"") && t.endsWith("\"") && t.length() > 2; - // If search text with double quote, remove quotes, - // otherwise keeps same - String term = isExact ? t.substring(1, t.length() - 1) : t; - - if (isExact) { - // Match phrase in original title and description, not use fuzzy fields - should.add(CQLFields.title.getPropertyEqualToQuery(term)); - should.add(CQLFields.description.getPropertyEqualToQuery(term)); - } - else { - should.add(CQLFields.fuzzy_title.getPropertyEqualToQuery(term)); - should.add(CQLFields.fuzzy_desc.getPropertyEqualToQuery(term)); - } - should.add(CQLFields.parameter_vocabs.getPropertyEqualToQuery(term)); - should.add(CQLFields.organisation_vocabs.getPropertyEqualToQuery(term)); - should.add(CQLFields.platform_vocabs.getPropertyEqualToQuery(term)); - should.add(CQLFields.id.getPropertyEqualToQuery(term)); - // Acronym match on the *.synonyms sub-fields, e.g. "SOOP" -> "ships of opportunity". - should.add(CQLFields.acronym_title.getPropertyEqualToQuery(term)); - should.add(CQLFields.acronym_desc.getPropertyEqualToQuery(term)); - // credit_contains uses match query by default, exact match is not applied here - should.add(CQLFields.credit_contains.getPropertyEqualToQuery(term)); - // match the acronym for AODN partner organisations - should.add(CQLFields.getDatasetGroupTextSearchQuery(term, isExact)); - } + should = createKeywordShouldClauses(keywords); + datasetGroupCandidates = collectDatasetGroupCandidates(keywords); } List filters = new ArrayList<>(); @@ -678,6 +678,14 @@ public ElasticSearchBase.SearchResult searchByParameters(Li sortOptions.add(0, CQLFields.platform_vocabs.getSortBuilder().apply(SortOrder.Desc).build()); } + // Records whose dataset_group matches a search term rank first among recalled records; prepended last so it is the strongest sort key. + if (should != null && !should.isEmpty() && !datasetGroupCandidates.isEmpty()) { + if (sortOptions == null) { + sortOptions = new ArrayList<>(); + } + sortOptions.add(0, CQLFields.getDatasetGroupPrioritySort(datasetGroupCandidates)); + } + return searchCollectionBy( null, should, diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java index 09c02f3a..c7373599 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java @@ -427,8 +427,8 @@ protected JsonNode explainCollectionBy(Supplier requestSu SearchResponse response = esClient.search(request, ObjectNode.class); if (simplified) { - // keep the request which carries the exact values of every terms clause - return mapper.valueToTree(ExplainSimplifier.from(response, toJsonNode(request))); + // the request carries the names of the sort keys, the response their values only so pass the request to explain as well + return mapper.valueToTree(ExplainSimplifier.from(response, request.sort())); } ObjectNode result = mapper.createObjectNode(); diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java index ee415e4f..6a6b6ec3 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java @@ -3,6 +3,8 @@ import au.org.aodn.ogcapi.server.core.model.ExplainSimplifiedResponse; import au.org.aodn.ogcapi.server.core.model.enumeration.StacBasicField; import au.org.aodn.ogcapi.server.core.model.enumeration.StacSummeries; +import co.elastic.clients.elasticsearch._types.FieldValue; +import co.elastic.clients.elasticsearch._types.SortOptions; import co.elastic.clients.elasticsearch.core.SearchResponse; import co.elastic.clients.elasticsearch.core.explain.Explanation; import co.elastic.clients.elasticsearch.core.explain.ExplanationDetail; @@ -13,9 +15,7 @@ import java.util.ArrayList; import java.util.Comparator; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -31,19 +31,11 @@ public class ExplainSimplifier { */ protected static final Pattern WEIGHT_PATTERN = Pattern.compile("^weight\\((.*) in \\d+\\)"); - /** - * A terms set leaf. E.g. "summaries.dataset_group:(csiro csiro temperature temperature)^100.0" - */ - protected static final Pattern TERMS_SET_PATTERN = - Pattern.compile("^([^\\s:()]+):\\((.*)\\)(?:\\^([\\d.eE+-]+))?$"); - - protected static final String TERMS_CLAUSE = "terms"; - - /** A run two different clauses could have produced, so it cannot be separated safely */ - protected static final String AMBIGUOUS = ""; - protected static final String WEIGHT_PREFIX = "weight("; + /** The first quoted token of a painless sort script is the doc field it reads */ + protected static final Pattern SCRIPT_FIELD_PATTERN = Pattern.compile("'([^']+)'"); + protected static final String SYNONYM_PREFIX = "Synonym("; protected static final String RELEVANCE_DESCRIPTION_PREFIX = "_score:"; @@ -57,31 +49,11 @@ private ExplainSimplifier() { } /** - * Lucene joins the values of a terms query with a space, "field:(a b c)", and a value can - * itself hold a space, e.g. "csiro oceans and atmosphere", so the rendered run cannot be - * split on its own. The request still holds the exact values, so pair every rendered leaf - * back with the clause that produced it and re-join those values with a comma. - * - * @param request the search request that produced the response, serialised + * @param sortOptions the sort of the request that produced the response, which carries the + * names of the sort keys, the response holds their values only */ - public static ExplainSimplifiedResponse from(SearchResponse response, JsonNode request) { - ExplainSimplifiedResponse simplified = from(response); - Map byRendering = termsClausesOf(request); - - if (byRendering.isEmpty()) { - return simplified; - } - - for (ExplainSimplifiedResponse.Hit hit : simplified.getHits()) { - for (ExplainSimplifiedResponse.MatchedFilter filter : hit.getFilters()) { - filter.setDescription(separateTermsValues(filter.getDescription(), byRendering)); - } - } - - return simplified; - } - - public static ExplainSimplifiedResponse from(SearchResponse response) { + public static ExplainSimplifiedResponse from(SearchResponse response, + List sortOptions) { ExplainSimplifiedResponse.Total total = null; if (response.hits().total() != null) { @@ -95,7 +67,7 @@ public static ExplainSimplifiedResponse from(SearchResponse response int rank = 1; for (Hit hit : response.hits().hits()) { - hits.add(toSimplifiedHit(hit, rank++)); + hits.add(toSimplifiedHit(hit, rank++, sortOptions)); } return ExplainSimplifiedResponse.builder() @@ -104,7 +76,8 @@ public static ExplainSimplifiedResponse from(SearchResponse response .build(); } - protected static ExplainSimplifiedResponse.Hit toSimplifiedHit(Hit hit, int rank) { + protected static ExplainSimplifiedResponse.Hit toSimplifiedHit(Hit hit, int rank, + List sortOptions) { Explanation explanation = hit.explanation(); // hit.score() keeps the precision elastic search reported, the explanation values are floats @@ -148,11 +121,60 @@ else if (explanation != null) { .esRelevance(esRelevance) .internalScore(doubleField(hit.source(), StacSummeries.Score.searchField)) .qualityMultiplier(qualityMultiplier) + .sortValues(sortValuesOf(hit.sort(), sortOptions)) .matchedTerms(terms) .filters(filters) .build(); } + /** + * The sort key values that decided the hit's rank, e.g. the dataset_group priority ahead of _score. + * So that a priority sort can rank a lower scoring hit above a higher scoring one. + * Elastic search emits one value per sort option in order, which pairs every value with the + * name of its key; values are kept unnamed when the options do not line up. + */ + protected static List sortValuesOf(List sort, + List options) { + if (sort == null || sort.isEmpty()) { + return null; + } + + boolean aligned = options != null && options.size() == sort.size(); + + List values = new ArrayList<>(sort.size()); + for (int i = 0; i < sort.size(); i++) { + values.add(ExplainSimplifiedResponse.SortValue.builder() + .field(aligned ? sortFieldOf(options.get(i)) : null) + .value(sort.get(i)._get()) + .build()); + } + return values; + } + + /** + * Name the sort key of one sort option. A script sort carries no name, so the field the script reads is lifted from its source, e.g. "doc['summaries.dataset_group']" or the + * "doc.containsKey('summaries.dataset_group')" guard names the dataset_group priority. + */ + protected static String sortFieldOf(SortOptions option) { + if (option.isScore()) { + return "_score"; + } + if (option.isField()) { + return option.field().field(); + } + if (option.isScript()) { + String source = option.script().script().source(); + if (source != null) { + Matcher matcher = SCRIPT_FIELD_PATTERN.matcher(source); + if (matcher.find()) { + return matcher.group(1); + } + } + return "script"; + } + return option._kind().jsonValue(); + } + /** * Locate the "_score: " node, the untouched relevance elastic search produced before the * painless script applied the quality multiplier. It is not a direct child of the root, @@ -223,78 +245,6 @@ protected static void collectScoreParts(List details, } } - protected static Map termsClausesOf(JsonNode request) { - Map byRendering = new HashMap<>(); - collectTermsClauses(request, byRendering); - return byRendering; - } - - protected static void collectTermsClauses(JsonNode node, Map byRendering) { - if (node == null || node.isValueNode()) { - return; - } - - JsonNode terms = node.get(TERMS_CLAUSE); - if (terms != null && terms.isObject()) { - // one field per clause, the other properties are the query options, e.g. "boost" - terms.forEach(values -> indexTermsClause(values, byRendering)); - } - - // an object iterates over its values, an array over its elements - node.forEach(child -> collectTermsClauses(child, byRendering)); - } - - protected static void indexTermsClause(JsonNode values, Map byRendering) { - if (!values.isArray() || values.isEmpty()) { - return; - } - - List written = new ArrayList<>(); - - for (JsonNode value : values) { - if (!value.isTextual()) { - // a numeric value or a terms lookup does not render as a run of words - return; - } - written.add(value.asText()); - } - - index(written.stream().sorted().toList(), byRendering); - index(written, byRendering); - } - - protected static void index(List values, Map byRendering) { - byRendering.merge( - String.join(" ", values), - String.join(",", values), - (existing, added) -> existing.equals(added) ? existing : AMBIGUOUS); - } - - /** - * Report a terms set leaf with its values separated, keeping the compiled lucene shape, e.g. - * summaries.dataset_group:(csiro csiro temperature temperature)^100.0 - * becomes - * summaries.dataset_group:(csiro,csiro temperature,temperature)^100.0 - * Any other description, e.g. a bbox filter or the match_all, is passed through untouche. - */ - protected static String separateTermsValues(String description, Map byRendering) { - Matcher matcher = TERMS_SET_PATTERN.matcher(description); - - if (!matcher.matches()) { - return description; - } - - String separated = byRendering.get(matcher.group(2)); - - if (separated == null || separated.equals(AMBIGUOUS)) { - return description; - } - - String boost = matcher.group(3); - - return matcher.group(1) + ":(" + separated + ")" + (boost == null ? "" : "^" + boost); - } - /** * Interpret the query text of a leaf scoring node, it is one of * title:network a term match, exact or fuzzy diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/common/RestAdminApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/common/RestAdminApiTest.java index b6f30146..8294865d 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/common/RestAdminApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/common/RestAdminApiTest.java @@ -198,27 +198,28 @@ public void explainSimplifiedFormatReportsMultiWordMatches() throws IOException } @Test - public void explainSimplifiedFormatSeparatesTheDatasetGroupTerms() throws IOException { - // the record sits in the "imos" dataset group, the free text search adds a terms clause - // holding the whole input plus each of its words + public void explainSimplifiedFormatReportsTheDatasetGroupPriorityInTheSortValues() throws IOException { + // the record sits in the "imos" dataset group and its credits name IMOS, so the search recalls it on the text and the group then ranks it insertRecordsWithExplicitIds("7709f541-fc0c-4318-b5b9-9053aa474e0e.json"); - URI simpleUri = explainUri("q", "imos temperature", "format", "simple"); + URI simpleUri = explainUri("q", "imos", "format", "simple"); ResponseEntity response = testRestTemplate.getForEntity(simpleUri, JsonNode.class); assertTrue(response.getStatusCode().is2xxSuccessful()); - List descriptions = fieldValues( - Objects.requireNonNull(response.getBody()).path("hits").path(0).path("filters"), - "description"); - - // elastic search joins the values with a space, which reads as one run of words - assertTrue(descriptions.stream() - .noneMatch(d -> d.contains("summaries.dataset_group:(imos imos temperature temperature"))); - // the boost is left out of the assertion, its value is still being tuned - assertTrue(descriptions.stream() - .anyMatch(d -> d.startsWith("summaries.dataset_group:(imos,imos temperature,temperature)^")), - "the dataset group values must be separated, got " + descriptions); + JsonNode top = Objects.requireNonNull(response.getBody()).path("hits").path(0); + + // dataset_group priority, _score, summaries.score, uuid + assertEquals(4, top.path("sort_values").size(), + "a text search ranks by four sort keys, got " + top.path("sort_values")); + assertEquals("summaries.dataset_group", top.path("sort_values").path(0).path("field").asText(), + "the strongest sort key is the dataset_group priority"); + assertEquals(1, top.path("sort_values").path(0).path("value").asInt(), + "the dataset group of the record matches the search term"); + // dataset_group no longer contributes to the score + List descriptions = fieldValues(top.path("filters"), "description"); + assertTrue(descriptions.stream().noneMatch(d -> d.contains("summaries.dataset_group")), + "dataset_group must not appear as a scored clause, got " + descriptions); } @Test diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java index e02a132a..43db66ee 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java @@ -1,40 +1,95 @@ package au.org.aodn.ogcapi.server.core.util; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; +import au.org.aodn.ogcapi.server.core.model.ExplainSimplifiedResponse; +import co.elastic.clients.elasticsearch._types.FieldValue; +import co.elastic.clients.elasticsearch._types.ScriptSortType; +import co.elastic.clients.elasticsearch._types.SortOptions; +import co.elastic.clients.elasticsearch._types.SortOrder; +import co.elastic.clients.elasticsearch.core.search.Hit; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.jupiter.api.Test; -import java.util.Map; +import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; public class ExplainSimplifierTest { - private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final String UUID = "7709f541-fc0c-4318-b5b9-9053aa474e0e"; + + private static Hit hitSortedBy(FieldValue... sort) { + return Hit.of(h -> h + .index("records") + .id(UUID) + .score(2.5) + .sort(List.of(sort))); + } + + /** The sort of the request that ranks a text search: dataset_group priority, _score, uuid */ + private static List textSearchSort() { + return List.of( + SortOptions.of(so -> so.script(s -> s + .type(ScriptSortType.Number) + .script(sc -> sc + .lang("painless") + .source("if (!doc.containsKey('summaries.dataset_group')) { return 0; } return 1;")) + .order(SortOrder.Desc))), + SortOptions.of(so -> so.score(sc -> sc.order(SortOrder.Desc))), + SortOptions.of(so -> so.field(f -> f.field("id.keyword").order(SortOrder.Asc)))); + } /** - * A free text search of "csiro temperature", the dataset group clause holds the whole input - * alongside each of its words. Elastic search renders it as one run of words, - * "summaries.dataset_group:(csiro csiro temperature temperature)^100.0", which the request - * is needed to separate again. + * The sort values decide the rank ahead of the score, e.g. the dataset_group priority + * sits first, so the simplified hit reports them, each named by its sort key. A script + * sort carries no name, the field its source reads names it. */ - private Map datasetGroupRequest() throws JsonProcessingException { - JsonNode request = MAPPER.readTree(""" - {"query":{"bool":{"should":[ - {"match":{"title":{"query":"csiro temperature"}}}, - {"terms":{"summaries.dataset_group":["csiro temperature","csiro","temperature"], - "boost":100.0}}]}}}"""); - - return ExplainSimplifier.termsClausesOf(request); + @Test + public void simplifiedHitNamesEverySortValueByItsSortKey() { + Hit hit = hitSortedBy(FieldValue.of(1), FieldValue.of(2.5), FieldValue.of(UUID)); + + ExplainSimplifiedResponse.Hit simplified = + ExplainSimplifier.toSimplifiedHit(hit, 1, textSearchSort()); + + assertEquals( + List.of( + sortValue("summaries.dataset_group", 1L), + sortValue("_score", 2.5), + sortValue("id.keyword", UUID)), + simplified.getSortValues()); } @Test - public void separateTermsValuesCommaSeparatesTheValuesOfTheClause() throws JsonProcessingException { + public void sortValuesAreKeptUnnamedWhenTheSortOptionsDoNotLineUp() { + Hit hit = hitSortedBy(FieldValue.of(2.5), FieldValue.of(UUID)); + + // one option for two values, so no pairing is safe + ExplainSimplifiedResponse.Hit simplified = ExplainSimplifier.toSimplifiedHit( + hit, 1, + List.of(SortOptions.of(so -> so.score(sc -> sc.order(SortOrder.Desc))))); + assertEquals( - "summaries.dataset_group:(csiro,csiro temperature,temperature)^100.0", - ExplainSimplifier.separateTermsValues( - "summaries.dataset_group:(csiro csiro temperature temperature)^100.0", - datasetGroupRequest())); + List.of(sortValue(null, 2.5), sortValue(null, UUID)), + simplified.getSortValues()); + } + + @Test + public void sortValuesAreLeftOutWhenTheHitCarriesNone() { + Hit hit = Hit.of(h -> h + .index("records") + .id(UUID) + .score(2.5)); + + ExplainSimplifiedResponse.Hit simplified = + ExplainSimplifier.toSimplifiedHit(hit, 1, textSearchSort()); + + assertNull(simplified.getSortValues()); + } + + private static ExplainSimplifiedResponse.SortValue sortValue(String field, Object value) { + return ExplainSimplifiedResponse.SortValue.builder() + .field(field) + .value(value) + .build(); } } diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index 2c42be09..5ea97cf7 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -256,18 +256,20 @@ public void verifyCorrectPageSizeDataReturnWithQuery() throws IOException { assertEquals(4, collections.getBody().getTotal(), "Get total works"); // The search after give you the value to go to next batch - assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after have three values"); + // 4 values with a text query: dataset_group priority sort, _score, summaries.score, uuid + assertEquals(4, collections.getBody().getSearchAfter().size(), "search_after have four values"); // Ranking depends on BM25 _score (varies by env); assert the cursor is one of the matching docs - assertTrue(DATASET_MATCH_IDS.contains(collections.getBody().getSearchAfter().get(2)), - "search_after cursor should be a matching doc id, got: " + collections.getBody().getSearchAfter().get(2)); + assertTrue(DATASET_MATCH_IDS.contains(collections.getBody().getSearchAfter().get(3)), + "search_after cursor should be a matching doc id, got: " + collections.getBody().getSearchAfter().get(3)); // Now the same search, same page but search_after the actual cursor returned above collections = testRestTemplate.exchange( getBasePath() + "/collections?q=dataset&filter=page_size=1 AND search_after=" + - String.format("'%s||%s||%s'", + String.format("'%s||%s||%s||%s'", collections.getBody().getSearchAfter().get(0), collections.getBody().getSearchAfter().get(1), - collections.getBody().getSearchAfter().get(2).replace("str:", "")), + collections.getBody().getSearchAfter().get(2), + collections.getBody().getSearchAfter().get(3).replace("str:", "")), HttpMethod.GET, null, new ParameterizedTypeReference<>() { @@ -282,19 +284,20 @@ public void verifyCorrectPageSizeDataReturnWithQuery() throws IOException { assertEquals(4, collections.getBody().getTotal(), "Get total works"); // The search after give you the value to go to next batch - assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after have three values"); + assertEquals(4, collections.getBody().getSearchAfter().size(), "search_after have four values"); // Ranking depends on BM25 _score (varies by env); assert the cursor is one of the matching docs - assertTrue(DATASET_MATCH_IDS.contains(collections.getBody().getSearchAfter().get(2)), - "search_after cursor should be a matching doc id, got: " + collections.getBody().getSearchAfter().get(2)); + assertTrue(DATASET_MATCH_IDS.contains(collections.getBody().getSearchAfter().get(3)), + "search_after cursor should be a matching doc id, got: " + collections.getBody().getSearchAfter().get(3)); // Now the same search, diff page but search_after the actual cursor returned above // set a bigger page size (4) which exceed more than remaining record hit as negative test collections = testRestTemplate.exchange( getBasePath() + "/collections?q=dataset&filter=page_size=4 AND search_after=" + - String.format("'%s||%s ||%s'", + String.format("'%s||%s||%s ||%s'", collections.getBody().getSearchAfter().get(0), collections.getBody().getSearchAfter().get(1), - collections.getBody().getSearchAfter().get(2).replace("str:", "")), + collections.getBody().getSearchAfter().get(2), + collections.getBody().getSearchAfter().get(3).replace("str:", "")), HttpMethod.GET, null, new ParameterizedTypeReference<>() { @@ -309,11 +312,11 @@ public void verifyCorrectPageSizeDataReturnWithQuery() throws IOException { assertEquals(4, collections.getBody().getTotal(), "Get total works"); // The search after give you the value to go to next batch - assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields"); + assertEquals(4, collections.getBody().getSearchAfter().size(), "search_after four fields"); // Ranking of remaining records depends on BM25 _score (varies by env), so assert the cursor is // one of the matching docs instead of a specific value. - String lastCursor = collections.getBody().getSearchAfter().get(2); + String lastCursor = collections.getBody().getSearchAfter().get(3); assertTrue( DATASET_MATCH_IDS.contains(lastCursor), "search_after cursor should be one of the matching doc ids, got: " + lastCursor @@ -372,22 +375,23 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { assertEquals(4, collections.getBody().getTotal(), "Get total works"); // The search after give you the value to go to next batch - assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields"); + assertEquals(4, collections.getBody().getSearchAfter().size(), "search_after four fields"); log.info("verifyCorrectPageSizeAndScoreWithQuery - uuid return {}", collections.getBody().getCollections().get(0).getId()); log.info("verifyCorrectPageSizeAndScoreWithQuery - search after {}", collections.getBody().getSearchAfter()); // Ranking depends on BM25 _score (varies by env); assert the cursor is one of the matching docs - assertTrue(DATASET_MATCH_IDS.contains(collections.getBody().getSearchAfter().get(2)), - "search_after cursor should be a matching doc id, got: " + collections.getBody().getSearchAfter().get(2)); + assertTrue(DATASET_MATCH_IDS.contains(collections.getBody().getSearchAfter().get(3)), + "search_after cursor should be a matching doc id, got: " + collections.getBody().getSearchAfter().get(3)); // Now the same search, same page but search_after the actual cursor returned above collections = testRestTemplate.exchange( getBasePath() + "/collections?q=dataset&filter=page_size=6 AND score>=1.3 AND search_after=" + - String.format("'%s|| %s || %s'", + String.format("'%s|| %s || %s || %s'", collections.getBody().getSearchAfter().get(0), collections.getBody().getSearchAfter().get(1), - collections.getBody().getSearchAfter().get(2).replace("str:", "")), + collections.getBody().getSearchAfter().get(2), + collections.getBody().getSearchAfter().get(3).replace("str:", "")), HttpMethod.GET, null, new ParameterizedTypeReference<>() { @@ -407,12 +411,12 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { assertEquals(4, collections.getBody().getTotal(), "Get total works"); // The search after give you the value to go to next batch - assertEquals(3, collections.getBody().getSearchAfter().size(), "search_after three fields"); + assertEquals(4, collections.getBody().getSearchAfter().size(), "search_after four fields"); // Note: relative ordering of the remaining docs depends on BM25 _score, // which can vary slightly between environments. // So we assert that the cursor is one of them instead of expecting a specific exact value. - String lastCursor = collections.getBody().getSearchAfter().get(2); + String lastCursor = collections.getBody().getSearchAfter().get(3); assertTrue( DATASET_MATCH_IDS.contains(lastCursor), "search_after cursor should be one of the matching doc ids, got: " + lastCursor diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java index e76924a3..f5a262b7 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java @@ -37,10 +37,12 @@ public void searchByParametersWithDoubleQuote() throws Exception { "-score,-rank", CQLCrsType.EPSG4326); - assertEquals(10, capturingSearch.should.size(), - "Exact match should produce 10 queries (title + description + other fields)"); + assertEquals(9, capturingSearch.should.size(), + "Exact match should produce 9 queries (title + description + other fields, no dataset_group)"); assertTrue(capturingSearch.should.get(0).isMatchPhrase(), "Title query should be MatchPhraseQuery"); assertTrue(capturingSearch.should.get(1).isMatchPhrase(), "Description query should be MatchPhraseQuery"); + assertTrue(capturingSearch.arguments.sortOptions().get(0).isScript(), + "dataset_group priority sort should be the first sort key"); } @Test @@ -54,8 +56,10 @@ public void searchByParametersWithoutDoubleQuote() throws Exception { "-score,-rank", CQLCrsType.EPSG4326); - assertEquals(10, capturingSearch.should.size(), "Fuzzy match should produce 10 queries"); + assertEquals(9, capturingSearch.should.size(), "Fuzzy match should produce 9 queries"); assertTrue(capturingSearch.should.get(0).isMatch(), "fuzzy_title should be MatchQuery"); + assertTrue(capturingSearch.arguments.sortOptions().get(0).isScript(), + "dataset_group priority sort should be the first sort key"); } // The portal SEO pipeline requests these in bulk to build Dataset JSON-LD @@ -118,7 +122,7 @@ public void explainByParametersUsesScriptScoreRequestForKeywords() throws Except "title-only _source is lightweight so the larger search_after batch is used"); assertNotNull(capturingSearch.explainRequest.query()); assertTrue(capturingSearch.explainRequest.query().isScriptScore()); - assertEquals(10, capturingSearch.explainRequest.query().scriptScore() + assertEquals(9, capturingSearch.explainRequest.query().scriptScore() .query().bool().should().size()); assertNotNull(capturingSearch.explainRequest.source()); assertTrue(capturingSearch.explainRequest.source().isFilter());