diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchConfig.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchConfig.java index 30f42b73..10934756 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchConfig.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchConfig.java @@ -54,7 +54,8 @@ public ElasticsearchClient geoNetworkElasticsearchClient(RestClientTransport tra * @param client - The elastic search client * @param mapper - Object mapper for string to object transformation * @param indexName - The elastic index name that store the STAC from es-indexer - * @param pageSize - Do not set this value too high, say 5000 will crash elastic search + * @param pageSize - Do not set this value too high, say 5000 will crash elastic search when _source is the full document + * @param lightweightPageSize - Larger search_after batch for small property lists (id, temporal, title, ...) * @param searchAsYouTypeSize - The number of search result return for search as you type * @return The search object */ @@ -64,8 +65,9 @@ public Search createElasticSearch(ElasticsearchClient client, ObjectMapper mapper, @Value("${elasticsearch.index.name}") String indexName, @Value("${elasticsearch.index.pageSize:2200}") Integer pageSize, + @Value("${elasticsearch.index.lightweightPageSize:10000}") Integer lightweightPageSize, @Value("${elasticsearch.search_as_you_type.size:10}") Integer searchAsYouTypeSize) { - return new ElasticSearch(client, cacheNoLandGeometry, mapper, indexName, pageSize, searchAsYouTypeSize); + return new ElasticSearch(client, cacheNoLandGeometry, mapper, indexName, pageSize, lightweightPageSize, searchAsYouTypeSize); } } diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/Converter.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/Converter.java index 680b83df..bc661611 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/Converter.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/Converter.java @@ -123,13 +123,12 @@ default Collection getCollection(D m, Filter fil if (m.getExtent() != null) { extent.setSpatial(new ExtentSpatial()); + // Do not show WARN for missing bbox here because some query intentionally do not have bbox if (m.getExtent().getBbox() != null && !m.getExtent().getBbox().isEmpty()) { // The first item is the overall bbox, this is STAC spec requirement and it ok with ogc api extent.getSpatial().bbox(m.getExtent().getBbox()); collection.setExtent(extent); - } else { - logger.warn("BBOX is missing for this UUID {}", m.getUuid()); } extent.setTemporal(new ExtentTemporal()); diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java index 17a41318..c6c46d00 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java @@ -469,4 +469,35 @@ public static List findInvalidEnum(List args) { }) .collect(Collectors.toList()); } + + /** + * Fields whose _source payload is large enough that Elasticsearch should keep the + * conservative search_after batch size. + */ + public boolean isHeavySourceField() { + return switch (this) { + case geometry, bbox, centroid, centroid_nocache, links, themes -> true; + default -> false; + }; + } + + /** + * True when the caller asked for a non-empty property list that contains none of the + * heavy source fields. Full-document requests (null/empty properties) are not lightweight. + */ + public static boolean requestsLightweightSource(List properties) { + if (properties == null || properties.isEmpty()) { + return false; + } + for (String property : properties) { + try { + if (valueOf(property).isHeavySourceField()) { + return false; + } + } catch (IllegalArgumentException e) { + return false; + } + } + return true; + } } diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/AfterImpl.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/AfterImpl.java index 4fc19139..2aa7eb2b 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/AfterImpl.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/AfterImpl.java @@ -39,6 +39,7 @@ public AfterImpl(Expression expression1, Expression expression2, Class enumTy this.query = NestedQuery.of(n -> n .path(StacSummeries.Temporal.searchField) + .scoreMode(ChildScoreMode.None) .query(q1 -> q1 .range(r -> r .date(d -> d diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/BeforeImpl.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/BeforeImpl.java index 671d9603..238eb01b 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/BeforeImpl.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/BeforeImpl.java @@ -3,6 +3,7 @@ import au.org.aodn.ogcapi.server.core.model.enumeration.CQLFields; import au.org.aodn.ogcapi.server.core.model.enumeration.CQLFieldsInterface; import au.org.aodn.ogcapi.server.core.model.enumeration.StacSummeries; +import co.elastic.clients.elasticsearch._types.query_dsl.ChildScoreMode; import co.elastic.clients.elasticsearch._types.query_dsl.NestedQuery; import org.geotools.filter.AttributeExpressionImpl; import org.geotools.filter.LiteralExpressionImpl; @@ -39,6 +40,7 @@ public BeforeImpl(Expression expression1, Expression expression2, Class enumT this.query = NestedQuery.of(n -> n .path(StacSummeries.Temporal.searchField) + .scoreMode(ChildScoreMode.None) .query(q1 -> q1 .range(r -> r .date(d -> d diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/DuringImpl.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/DuringImpl.java index f1cb2b16..2339a44d 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/DuringImpl.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/DuringImpl.java @@ -4,6 +4,7 @@ import au.org.aodn.ogcapi.server.core.model.enumeration.CQLFieldsInterface; import au.org.aodn.ogcapi.server.core.model.enumeration.StacSummeries; import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery; +import co.elastic.clients.elasticsearch._types.query_dsl.ChildScoreMode; import co.elastic.clients.elasticsearch._types.query_dsl.ExistsQuery; import co.elastic.clients.elasticsearch._types.query_dsl.NestedQuery; import co.elastic.clients.elasticsearch._types.query_dsl.Query; @@ -91,6 +92,7 @@ public DuringImpl(Expression expression1, Expression expression2, Class enumT this.query = NestedQuery.of(n -> n .path(StacSummeries.Temporal.searchField) + .scoreMode(ChildScoreMode.None) .query(BoolQuery.of(q -> q .must(endAfterFilterStartOrOngoing, startBeforeOrAtFilterEnd))._toQuery() ) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index 236ff806..6ee28549 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -79,12 +79,14 @@ public ElasticSearch(ElasticsearchClient client, ObjectMapper mapper, String indexName, Integer pageSize, + Integer lightweightPageSize, Integer searchAsYouTypeSize) { this.setEsClient(client); this.setMapper(mapper); this.setIndexName(indexName); this.setPageSize(pageSize); + this.setLightweightPageSize(lightweightPageSize); this.setSearchAsYouTypeSize(searchAsYouTypeSize); this.setCacheNoLandGeometry(cacheNoLandGeometry); this.defaultElasticSetting = CQLToElasticFilterFactory.getDefaultSetting(); diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java index 18e33453..09c02f3a 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java @@ -54,6 +54,11 @@ public abstract class ElasticSearchBase { protected Integer searchAsYouTypeSize; protected String indexName; protected Integer pageSize; + /** + * Larger search_after batch used when {@code properties} is a small field list (no geometries/links). + * Full-document queries keep {@link #pageSize} because a large _source batch can overwhelm Elasticsearch. + */ + protected Integer lightweightPageSize; protected ElasticsearchClient esClient; protected ObjectMapper mapper; protected CacheNoLandGeometry cacheNoLandGeometry; @@ -103,6 +108,24 @@ protected & CQLFieldsInterface> List createSortO } return sos; } + + /** + * Choose the Elasticsearch {@code size} for one search_after page. + * Lightweight property lists can use a larger batch; CQL {@code page_size} still caps it. + */ + protected int resolveSearchPageSize(List properties, Long maxSize) { + int batchSize = pageSize; + if (CQLFields.requestsLightweightSource(properties) + && lightweightPageSize != null + && lightweightPageSize > pageSize) { + batchSize = lightweightPageSize; + } + if (maxSize != null && maxSize < batchSize) { + return maxSize.intValue(); + } + return batchSize; + } + /** * Construct the skeleton of in the elastic query and fill in values * @param must - The must portion of Elastic query @@ -172,8 +195,10 @@ protected Supplier buildCollectionSearchRequestSupplier(f // If user query request a page that is smaller then the internal default, then // we use the smaller one. The internal page size is used to get the result by // batch, lets say page is 20 and internal is 10, then we do it in two batch. - // But if we request 5 only, then there is no point to load 10 - .size(maxSize != null && maxSize < pageSize ? maxSize.intValue() : pageSize); + // But if we request 5 only, then there is no point to load 10. + // Lightweight property lists (id, temporal, title, ...) use a larger batch so + // unbounded catalog queries need fewer serial search_after round-trips. + .size(resolveSearchPageSize(properties, maxSize)); // use script score if search with text, in such case, the final score depends on both relevance and metadata quality // put query in script block diff --git a/server/src/main/resources/application.yaml b/server/src/main/resources/application.yaml index b2be0494..cf0b896e 100644 --- a/server/src/main/resources/application.yaml +++ b/server/src/main/resources/application.yaml @@ -12,6 +12,10 @@ ogcapi: elasticsearch: index: name: dev_portal_records + # search_after batch for full-document / geometry / links queries + pageSize: 2200 + # search_after batch when properties is a small field list (id, temporal, title, ...) + lightweightPageSize: 7000 vocabs_index: name: vocabs_index cloud_optimized_index: diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactoryTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactoryTest.java index 448dfb4b..29738d70 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactoryTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactoryTest.java @@ -4,6 +4,7 @@ import au.org.aodn.ogcapi.server.core.model.enumeration.CQLElasticSetting; import au.org.aodn.ogcapi.server.core.model.enumeration.CQLFields; import au.org.aodn.ogcapi.server.core.model.enumeration.StacSummeries; +import co.elastic.clients.elasticsearch._types.query_dsl.ChildScoreMode; import co.elastic.clients.elasticsearch._types.query_dsl.Query; import org.geotools.filter.text.commons.CompilerUtil; import org.geotools.filter.text.commons.Language; @@ -12,12 +13,9 @@ import org.opengis.filter.Filter; import java.util.List; +import java.util.Objects; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertInstanceOf; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; public class CQLToElasticFilterFactoryTest { @@ -36,7 +34,7 @@ public void parameterVocabFilterEnablesPrioritySort() throws CQLException { assertTrue(parameterFilter.getQuery().isBool()); assertEquals(6, parameterFilter.getQuery().bool().should().size()); assertTrue( - parameterFilter.getQuery().bool().should().stream().noneMatch(query -> query.isBool()), + parameterFilter.getQuery().bool().should().stream().noneMatch(Query::isBool), "Parameter vocabulary clauses should be flattened into one should list"); } @@ -54,7 +52,7 @@ public void platformVocabFilterEnablesPrioritySort() throws CQLException { assertTrue(platformFilter.getQuery().isBool()); assertEquals(4, platformFilter.getQuery().bool().should().size()); assertTrue( - platformFilter.getQuery().bool().should().stream().noneMatch(query -> query.isBool()), + platformFilter.getQuery().bool().should().stream().noneMatch(Query::isBool), "Grouped platform vocabulary clauses should be flattened into one should list"); } @@ -80,13 +78,14 @@ public void temporalDuringUsesOverlapRangeQueryAndIncludesOngoingRecords() throw DuringImpl duringFilter = assertInstanceOf(DuringImpl.class, filter); assertTrue(duringFilter.getQuery().isNested()); + assertEquals(ChildScoreMode.None, duringFilter.getQuery().nested().scoreMode()); List must = duringFilter.getQuery().nested().query().bool().must(); assertEquals(2, must.size()); Query startRange = findDateRange(must, StacSummeries.TemporalStart.searchField); assertEquals("strict_date_optional_time", startRange.range().date().format()); - assertTrue(startRange.range().date().lte().contains("2026-06-25")); + assertTrue(Objects.requireNonNull(startRange.range().date().lte()).contains("2026-06-25")); Query endAfterFilterStartOrOngoing = must.stream() .filter(Query::isBool) @@ -97,7 +96,7 @@ public void temporalDuringUsesOverlapRangeQueryAndIncludesOngoingRecords() throw List should = endAfterFilterStartOrOngoing.bool().should(); Query endRange = findDateRange(should, StacSummeries.TemporalEnd.searchField); assertEquals("strict_date_optional_time", endRange.range().date().format()); - assertTrue(endRange.range().date().gte().contains("2025-06-25")); + assertTrue(Objects.requireNonNull(endRange.range().date().gte()).contains("2025-06-25")); assertTrue(should.stream() .filter(Query::isBool) @@ -106,6 +105,24 @@ public void temporalDuringUsesOverlapRangeQueryAndIncludesOngoingRecords() throw && StacSummeries.TemporalEnd.searchField.equals(query.exists().field()))); } + @Test + public void temporalAfterUsesNestedRangeWithoutScoring() throws CQLException { + Filter filter = CompilerUtil.parseFilter( + Language.ECQL, + "temporal AFTER 1970-01-01T00:00:00Z", + newFactory()); + + AfterImpl afterFilter = assertInstanceOf(AfterImpl.class, filter); + assertTrue(afterFilter.getQuery().isNested()); + assertEquals(ChildScoreMode.None, afterFilter.getQuery().nested().scoreMode()); + assertEquals(StacSummeries.Temporal.searchField, afterFilter.getQuery().nested().path()); + + Query range = afterFilter.getQuery().nested().query(); + assertTrue(range.isRange()); + assertEquals(StacSummeries.TemporalStart.searchField, range.range().date().field()); + assertTrue(Objects.requireNonNull(range.range().date().gte()).contains("1970-01-01")); + } + @Test public void querySettingsCannotBeCombinedWithOr() { IllegalArgumentException settingFirst = assertThrows( diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java index 7cb3dbef..e76924a3 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java @@ -114,7 +114,8 @@ public void explainByParametersUsesScriptScoreRequestForKeywords() throws Except false); assertEquals("captured", result.path("status").asText()); - assertEquals(100, capturingSearch.explainRequest.size()); + assertEquals(10000, capturingSearch.explainRequest.size(), + "title-only _source is lightweight so the larger search_after batch is used"); assertNotNull(capturingSearch.explainRequest.query()); assertTrue(capturingSearch.explainRequest.query().isScriptScore()); assertEquals(10, capturingSearch.explainRequest.query().scriptScore() @@ -179,6 +180,73 @@ public void normalAndExplainRequestsMatchForKeywordAndCqlSettings() throws Excep assertFalse(explainRequest.query().scriptScore().query().bool().filter().isEmpty()); } + @Test + public void lightweightPropertiesUseLargerSearchPageSize() throws Exception { + CapturingElasticSearch capturingSearch = new CapturingElasticSearch(mockClient); + + capturingSearch.searchByParameters( + null, + "temporal AFTER 1970-01-01T00:00:00Z", + List.of("id", "temporal"), + "id", + CQLCrsType.EPSG4326); + + assertEquals(10000, capturingSearch.normalRequest.size(), + "id,temporal is a small _source so the catalog query can fetch 10000 hits per page"); + + capturingSearch.searchByParameters( + null, + "temporal AFTER 1970-01-01T00:00:00Z", + List.of("id", "providers"), + "id", + CQLCrsType.EPSG4326); + + assertEquals(10000, capturingSearch.normalRequest.size(), + "id,providers is also a small _source"); + } + + @Test + public void heavyOrMissingPropertiesKeepDefaultSearchPageSize() throws Exception { + CapturingElasticSearch capturingSearch = new CapturingElasticSearch(mockClient); + + capturingSearch.searchByParameters( + null, + "temporal AFTER 1970-01-01T00:00:00Z", + List.of("id", "centroid"), + "id", + CQLCrsType.EPSG4326); + assertEquals(100, capturingSearch.normalRequest.size(), + "centroid pulls no-land geometry so the conservative batch size is kept"); + + capturingSearch.searchByParameters( + null, + "temporal AFTER 1970-01-01T00:00:00Z", + List.of("id", "links"), + "id", + CQLCrsType.EPSG4326); + assertEquals(100, capturingSearch.normalRequest.size(), + "links payloads stay on the conservative batch size"); + + capturingSearch.searchAllCollections("id"); + assertEquals(100, capturingSearch.normalRequest.size(), + "full-document searches keep the conservative batch size"); + } + + @Test + public void cqlPageSizeStillCapsLightweightSearch() throws Exception { + CapturingElasticSearch capturingSearch = new CapturingElasticSearch(mockClient); + + capturingSearch.searchByParameters( + null, + "temporal AFTER 1970-01-01T00:00:00Z AND page_size=3", + List.of("id", "temporal"), + "id", + CQLCrsType.EPSG4326); + + assertEquals(3, capturingSearch.normalRequest.size(), + "CQL page_size remains the upper bound even for lightweight property lists"); + } + private record SearchArguments( List queries, List should, @@ -197,7 +265,7 @@ private static class CapturingElasticSearch extends ElasticSearch { private SearchRequest explainRequest; private CapturingElasticSearch(ElasticsearchClient client) { - super(client, null, new ObjectMapper(), "test-index", 100, 10); + super(client, null, new ObjectMapper(), "test-index", 100, 10000, 10); this.searchAfterSplitRegex = "\\|\\|"; } diff --git a/server/src/test/resources/application-test.yaml b/server/src/test/resources/application-test.yaml index 93ff6007..80c43603 100644 --- a/server/src/test/resources/application-test.yaml +++ b/server/src/test/resources/application-test.yaml @@ -8,6 +8,7 @@ elasticsearch: index: name: testing_index pageSize: 4 + lightweightPageSize: 4 vocabs_index: name: test_vocabs_index