Bugfix/8994 improve performance

server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchConfig.java

@@ -54,7 +54,8 @@ public ElasticsearchClient geoNetworkElasticsearchClient(RestClientTransport tra
      * @param client - The elastic search client
      * @param mapper - Object mapper for string to object transformation
      * @param indexName - The elastic index name that store the STAC from es-indexer
-     * @param pageSize - Do not set this value too high, say 5000 will crash elastic search
+     * @param pageSize - Do not set this value too high, say 5000 will crash elastic search when _source is the full document
+     * @param lightweightPageSize - Larger search_after batch for small property lists (id, temporal, title, ...)
      * @param searchAsYouTypeSize - The number of search result return for search as you type
      * @return The search object
      */
@@ -64,8 +65,9 @@ public Search createElasticSearch(ElasticsearchClient client,
                                       ObjectMapper mapper,
                                       @Value("${elasticsearch.index.name}") String indexName,
                                       @Value("${elasticsearch.index.pageSize:2200}") Integer pageSize,
+                                      @Value("${elasticsearch.index.lightweightPageSize:10000}") Integer lightweightPageSize,
                                       @Value("${elasticsearch.search_as_you_type.size:10}") Integer searchAsYouTypeSize) {
 
-        return new ElasticSearch(client, cacheNoLandGeometry, mapper, indexName, pageSize, searchAsYouTypeSize);
+        return new ElasticSearch(client, cacheNoLandGeometry, mapper, indexName, pageSize, lightweightPageSize, searchAsYouTypeSize);
     }
 }

server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/Converter.java

@@ -123,13 +123,12 @@ default <D extends StacCollectionModel> Collection getCollection(D m, Filter fil
         if (m.getExtent() != null) {
             extent.setSpatial(new ExtentSpatial());
 
+            // Do not show WARN for missing bbox here because some query intentionally do not have bbox
             if (m.getExtent().getBbox() != null
                     && !m.getExtent().getBbox().isEmpty()) {
                 // The first item is the overall bbox, this is STAC spec requirement and it ok with ogc api
                 extent.getSpatial().bbox(m.getExtent().getBbox());
                 collection.setExtent(extent);
-            } else {
-                logger.warn("BBOX is missing for this UUID {}", m.getUuid());
             }
 
             extent.setTemporal(new ExtentTemporal());

server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java

@@ -469,4 +469,35 @@ public static List<String> findInvalidEnum(List<String> args) {
                                 })
                                 .collect(Collectors.toList());
         }
+
+        /**
+         * Fields whose _source payload is large enough that Elasticsearch should keep the
+         * conservative search_after batch size.
+         */
+        public boolean isHeavySourceField() {
+                return switch (this) {
+                        case geometry, bbox, centroid, centroid_nocache, links, themes -> true;
+                        default -> false;
+                };
+        }
+
+        /**
+         * True when the caller asked for a non-empty property list that contains none of the
+         * heavy source fields. Full-document requests (null/empty properties) are not lightweight.
+         */
+        public static boolean requestsLightweightSource(List<String> properties) {
+                if (properties == null || properties.isEmpty()) {
+                        return false;
+                }
+                for (String property : properties) {
+                        try {
+                                if (valueOf(property).isHeavySourceField()) {
+                                        return false;
+                                }
+                        } catch (IllegalArgumentException e) {
+                                return false;
+                        }
+                }
+                return true;
+        }
 }

server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/AfterImpl.java

@@ -39,6 +39,7 @@ public AfterImpl(Expression expression1, Expression expression2, Class<T> enumTy
 
                     this.query = NestedQuery.of(n -> n
                             .path(StacSummeries.Temporal.searchField)
+                            .scoreMode(ChildScoreMode.None)
                             .query(q1 -> q1
                                     .range(r -> r
                                             .date(d -> d

server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/BeforeImpl.java

@@ -3,6 +3,7 @@
 import au.org.aodn.ogcapi.server.core.model.enumeration.CQLFields;
 import au.org.aodn.ogcapi.server.core.model.enumeration.CQLFieldsInterface;
 import au.org.aodn.ogcapi.server.core.model.enumeration.StacSummeries;
+import co.elastic.clients.elasticsearch._types.query_dsl.ChildScoreMode;
 import co.elastic.clients.elasticsearch._types.query_dsl.NestedQuery;
 import org.geotools.filter.AttributeExpressionImpl;
 import org.geotools.filter.LiteralExpressionImpl;
@@ -39,6 +40,7 @@ public BeforeImpl(Expression expression1, Expression expression2, Class<T> enumT
 
                     this.query = NestedQuery.of(n -> n
                             .path(StacSummeries.Temporal.searchField)
+                            .scoreMode(ChildScoreMode.None)
                             .query(q1 -> q1
                                     .range(r -> r
                                             .date(d -> d

server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/DuringImpl.java

@@ -4,6 +4,7 @@
 import au.org.aodn.ogcapi.server.core.model.enumeration.CQLFieldsInterface;
 import au.org.aodn.ogcapi.server.core.model.enumeration.StacSummeries;
 import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery;
+import co.elastic.clients.elasticsearch._types.query_dsl.ChildScoreMode;
 import co.elastic.clients.elasticsearch._types.query_dsl.ExistsQuery;
 import co.elastic.clients.elasticsearch._types.query_dsl.NestedQuery;
 import co.elastic.clients.elasticsearch._types.query_dsl.Query;
@@ -91,6 +92,7 @@ public DuringImpl(Expression expression1, Expression expression2, Class<T> enumT
 
                     this.query = NestedQuery.of(n -> n
                             .path(StacSummeries.Temporal.searchField)
+                            .scoreMode(ChildScoreMode.None)
                             .query(BoolQuery.of(q -> q
                                     .must(endAfterFilterStartOrOngoing, startBeforeOrAtFilterEnd))._toQuery()
                             )

server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java

@@ -79,12 +79,14 @@ public ElasticSearch(ElasticsearchClient client,
                          ObjectMapper mapper,
                          String indexName,
                          Integer pageSize,
+                         Integer lightweightPageSize,
                          Integer searchAsYouTypeSize) {
 
         this.setEsClient(client);
         this.setMapper(mapper);
         this.setIndexName(indexName);
         this.setPageSize(pageSize);
+        this.setLightweightPageSize(lightweightPageSize);
         this.setSearchAsYouTypeSize(searchAsYouTypeSize);
         this.setCacheNoLandGeometry(cacheNoLandGeometry);
         this.defaultElasticSetting = CQLToElasticFilterFactory.getDefaultSetting();

server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java

@@ -54,6 +54,11 @@ public abstract class ElasticSearchBase {
     protected Integer searchAsYouTypeSize;
     protected String indexName;
     protected Integer pageSize;
+    /**
+     * Larger search_after batch used when {@code properties} is a small field list (no geometries/links).
+     * Full-document queries keep {@link #pageSize} because a large _source batch can overwhelm Elasticsearch.
+     */
+    protected Integer lightweightPageSize;
     protected ElasticsearchClient esClient;
     protected ObjectMapper mapper;
     protected CacheNoLandGeometry cacheNoLandGeometry;
@@ -103,6 +108,24 @@ protected <E extends Enum<E> & CQLFieldsInterface> List<SortOptions> createSortO
         }
         return sos;
     }
+
+    /**
+     * Choose the Elasticsearch {@code size} for one search_after page.
+     * Lightweight property lists can use a larger batch; CQL {@code page_size} still caps it.
+     */
+    protected int resolveSearchPageSize(List<String> properties, Long maxSize) {
+        int batchSize = pageSize;
+        if (CQLFields.requestsLightweightSource(properties)
+                && lightweightPageSize != null
+                && lightweightPageSize > pageSize) {
+            batchSize = lightweightPageSize;
+        }
+        if (maxSize != null && maxSize < batchSize) {
+            return maxSize.intValue();
+        }
+        return batchSize;
+    }
+
     /**
      * Construct the skeleton of in the elastic query and fill in values
      * @param must - The must portion of Elastic query
@@ -172,8 +195,10 @@ protected Supplier<SearchRequest.Builder> buildCollectionSearchRequestSupplier(f
                     // If user query request a page that is smaller then the internal default, then
                     // we use the smaller one. The internal page size is used to get the result by
                     // batch, lets say page is 20 and internal is 10, then we do it in two batch.
-                    // But if we request 5 only, then there is no point to load 10
-                    .size(maxSize != null && maxSize < pageSize ? maxSize.intValue() : pageSize);
+                    // But if we request 5 only, then there is no point to load 10.
+                    // Lightweight property lists (id, temporal, title, ...) use a larger batch so
+                    // unbounded catalog queries need fewer serial search_after round-trips.
+                    .size(resolveSearchPageSize(properties, maxSize));
 
             // use script score if search with text, in such case, the final score depends on both relevance and metadata quality
             // put query in script block

server/src/main/resources/application.yaml

@@ -12,6 +12,10 @@ ogcapi:
 elasticsearch:
   index:
     name: dev_portal_records
+    # search_after batch for full-document / geometry / links queries
+    pageSize: 2200
+    # search_after batch when properties is a small field list (id, temporal, title, ...)
+    lightweightPageSize: 7000
   vocabs_index:
     name: vocabs_index
   cloud_optimized_index:

server/src/test/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactoryTest.java

@@ -4,6 +4,7 @@
 import au.org.aodn.ogcapi.server.core.model.enumeration.CQLElasticSetting;
 import au.org.aodn.ogcapi.server.core.model.enumeration.CQLFields;
 import au.org.aodn.ogcapi.server.core.model.enumeration.StacSummeries;
+import co.elastic.clients.elasticsearch._types.query_dsl.ChildScoreMode;
 import co.elastic.clients.elasticsearch._types.query_dsl.Query;
 import org.geotools.filter.text.commons.CompilerUtil;
 import org.geotools.filter.text.commons.Language;
@@ -12,12 +13,9 @@
 import org.opengis.filter.Filter;
 
 import java.util.List;
+import java.util.Objects;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertInstanceOf;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.*;
 
 public class CQLToElasticFilterFactoryTest {
 
@@ -36,7 +34,7 @@ public void parameterVocabFilterEnablesPrioritySort() throws CQLException {
         assertTrue(parameterFilter.getQuery().isBool());
         assertEquals(6, parameterFilter.getQuery().bool().should().size());
         assertTrue(
-                parameterFilter.getQuery().bool().should().stream().noneMatch(query -> query.isBool()),
+                parameterFilter.getQuery().bool().should().stream().noneMatch(Query::isBool),
                 "Parameter vocabulary clauses should be flattened into one should list");
     }
 
@@ -54,7 +52,7 @@ public void platformVocabFilterEnablesPrioritySort() throws CQLException {
         assertTrue(platformFilter.getQuery().isBool());
         assertEquals(4, platformFilter.getQuery().bool().should().size());
         assertTrue(
-                platformFilter.getQuery().bool().should().stream().noneMatch(query -> query.isBool()),
+                platformFilter.getQuery().bool().should().stream().noneMatch(Query::isBool),
                 "Grouped platform vocabulary clauses should be flattened into one should list");
     }
 
@@ -80,13 +78,14 @@ public void temporalDuringUsesOverlapRangeQueryAndIncludesOngoingRecords() throw
 
         DuringImpl<?> duringFilter = assertInstanceOf(DuringImpl.class, filter);
         assertTrue(duringFilter.getQuery().isNested());
+        assertEquals(ChildScoreMode.None, duringFilter.getQuery().nested().scoreMode());
 
         List<Query> must = duringFilter.getQuery().nested().query().bool().must();
         assertEquals(2, must.size());
 
         Query startRange = findDateRange(must, StacSummeries.TemporalStart.searchField);
         assertEquals("strict_date_optional_time", startRange.range().date().format());
-        assertTrue(startRange.range().date().lte().contains("2026-06-25"));
+        assertTrue(Objects.requireNonNull(startRange.range().date().lte()).contains("2026-06-25"));
 
         Query endAfterFilterStartOrOngoing = must.stream()
                 .filter(Query::isBool)
@@ -97,7 +96,7 @@ public void temporalDuringUsesOverlapRangeQueryAndIncludesOngoingRecords() throw
         List<Query> should = endAfterFilterStartOrOngoing.bool().should();
         Query endRange = findDateRange(should, StacSummeries.TemporalEnd.searchField);
         assertEquals("strict_date_optional_time", endRange.range().date().format());
-        assertTrue(endRange.range().date().gte().contains("2025-06-25"));
+        assertTrue(Objects.requireNonNull(endRange.range().date().gte()).contains("2025-06-25"));
 
         assertTrue(should.stream()
                 .filter(Query::isBool)
@@ -106,6 +105,24 @@ public void temporalDuringUsesOverlapRangeQueryAndIncludesOngoingRecords() throw
                         && StacSummeries.TemporalEnd.searchField.equals(query.exists().field())));
     }
 
+    @Test
+    public void temporalAfterUsesNestedRangeWithoutScoring() throws CQLException {
+        Filter filter = CompilerUtil.parseFilter(
+                Language.ECQL,
+                "temporal AFTER 1970-01-01T00:00:00Z",
+                newFactory());
+
+        AfterImpl<?> afterFilter = assertInstanceOf(AfterImpl.class, filter);
+        assertTrue(afterFilter.getQuery().isNested());
+        assertEquals(ChildScoreMode.None, afterFilter.getQuery().nested().scoreMode());
+        assertEquals(StacSummeries.Temporal.searchField, afterFilter.getQuery().nested().path());
+
+        Query range = afterFilter.getQuery().nested().query();
+        assertTrue(range.isRange());
+        assertEquals(StacSummeries.TemporalStart.searchField, range.range().date().field());
+        assertTrue(Objects.requireNonNull(range.range().date().gte()).contains("1970-01-01"));
+    }
+
     @Test
     public void querySettingsCannotBeCombinedWithOr() {
         IllegalArgumentException settingFirst = assertThrows(

server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java

@@ -114,7 +114,8 @@ public void explainByParametersUsesScriptScoreRequestForKeywords() throws Except
                 false);
 
         assertEquals("captured", result.path("status").asText());
-        assertEquals(100, capturingSearch.explainRequest.size());
+        assertEquals(10000, capturingSearch.explainRequest.size(),
+                "title-only _source is lightweight so the larger search_after batch is used");
         assertNotNull(capturingSearch.explainRequest.query());
         assertTrue(capturingSearch.explainRequest.query().isScriptScore());
         assertEquals(10, capturingSearch.explainRequest.query().scriptScore()
@@ -179,6 +180,73 @@ public void normalAndExplainRequestsMatchForKeywordAndCqlSettings() throws Excep
         assertFalse(explainRequest.query().scriptScore().query().bool().filter().isEmpty());
     }
 
+    @Test
+    public void lightweightPropertiesUseLargerSearchPageSize() throws Exception {
+        CapturingElasticSearch capturingSearch = new CapturingElasticSearch(mockClient);
+
+        capturingSearch.searchByParameters(
+                null,
+                "temporal AFTER 1970-01-01T00:00:00Z",
+                List.of("id", "temporal"),
+                "id",
+                CQLCrsType.EPSG4326);
+
+        assertEquals(10000, capturingSearch.normalRequest.size(),
+                "id,temporal is a small _source so the catalog query can fetch 10000 hits per page");
+
+        capturingSearch.searchByParameters(
+                null,
+                "temporal AFTER 1970-01-01T00:00:00Z",
+                List.of("id", "providers"),
+                "id",
+                CQLCrsType.EPSG4326);
+
+        assertEquals(10000, capturingSearch.normalRequest.size(),
+                "id,providers is also a small _source");
+    }
+
+    @Test
+    public void heavyOrMissingPropertiesKeepDefaultSearchPageSize() throws Exception {
+        CapturingElasticSearch capturingSearch = new CapturingElasticSearch(mockClient);
+
+        capturingSearch.searchByParameters(
+                null,
+                "temporal AFTER 1970-01-01T00:00:00Z",
+                List.of("id", "centroid"),
+                "id",
+                CQLCrsType.EPSG4326);
+        assertEquals(100, capturingSearch.normalRequest.size(),
+                "centroid pulls no-land geometry so the conservative batch size is kept");
+
+        capturingSearch.searchByParameters(
+                null,
+                "temporal AFTER 1970-01-01T00:00:00Z",
+                List.of("id", "links"),
+                "id",
+                CQLCrsType.EPSG4326);
+        assertEquals(100, capturingSearch.normalRequest.size(),
+                "links payloads stay on the conservative batch size");
+
+        capturingSearch.searchAllCollections("id");
+        assertEquals(100, capturingSearch.normalRequest.size(),
+                "full-document searches keep the conservative batch size");
+    }
+
+    @Test
+    public void cqlPageSizeStillCapsLightweightSearch() throws Exception {
+        CapturingElasticSearch capturingSearch = new CapturingElasticSearch(mockClient);
+
+        capturingSearch.searchByParameters(
+                null,
+                "temporal AFTER 1970-01-01T00:00:00Z AND page_size=3",
+                List.of("id", "temporal"),
+                "id",
+                CQLCrsType.EPSG4326);
+
+        assertEquals(3, capturingSearch.normalRequest.size(),
+                "CQL page_size remains the upper bound even for lightweight property lists");
+    }
+
     private record SearchArguments(
             List<Query> queries,
             List<Query> should,
@@ -197,7 +265,7 @@ private static class CapturingElasticSearch extends ElasticSearch {
         private SearchRequest explainRequest;
 
         private CapturingElasticSearch(ElasticsearchClient client) {
-            super(client, null, new ObjectMapper(), "test-index", 100, 10);
+            super(client, null, new ObjectMapper(), "test-index", 100, 10000, 10);
             this.searchAfterSplitRegex = "\\|\\|";
         }
 

server/src/test/resources/application-test.yaml

@@ -8,6 +8,7 @@ elasticsearch:
   index:
     name: testing_index
     pageSize: 4
+    lightweightPageSize: 4
 
   vocabs_index:
     name: test_vocabs_index