diff --git a/pom.xml b/pom.xml index a3cdebc8..fabcdac9 100644 --- a/pom.xml +++ b/pom.xml @@ -192,7 +192,7 @@ au.org.aodn stacmodel - 0.0.63 + 0.0.66 diff --git a/server/pom.xml b/server/pom.xml index ab375258..fd130d0b 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -205,10 +205,6 @@ au.org.aodn stacmodel - - org.openapitools - jackson-databind-nullable - org.springframework.boot spring-boot-starter-test diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/common/RestAdminApi.java b/server/src/main/java/au/org/aodn/ogcapi/server/common/RestAdminApi.java index addbbbfc..d7dd1bf5 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/common/RestAdminApi.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/common/RestAdminApi.java @@ -1,6 +1,7 @@ package au.org.aodn.ogcapi.server.common; +import au.org.aodn.ogcapi.server.core.configuration.OgcApiProperties; import au.org.aodn.ogcapi.server.core.model.enumeration.CQLCrsType; import com.fasterxml.jackson.databind.JsonNode; import io.swagger.v3.oas.annotations.Parameter; @@ -28,6 +29,9 @@ public class RestAdminApi { @Autowired protected RestAdminService restAdminService; + @Autowired + protected OgcApiProperties ogcApiProperties; + /** * Explain the detail relevance score of a search query * Internal debugging/troubleshooting usage only @@ -51,7 +55,7 @@ public ResponseEntity getExplainByParameters( @Parameter(in = ParameterIn.QUERY, description = "Response format, simple for the flattened score breakdown, anything else returns the full elastic search explanation") @RequestParam(value = "format", required = false) String format ) throws Exception { - if (!restAdminService.isElasticsearchExplainEnabled()) { + if (!ogcApiProperties.debug().elasticsearchExplainEnabled()) { //return 404 NotFound error if elasticsearch-explain-enabled is set as false return ResponseEntity.notFound().build(); } @@ -88,7 +92,7 @@ public ResponseEntity getExplainByParametersUuid( @Parameter(in = ParameterIn.QUERY, description = "Filter language") @RequestParam(value = "filter-lang", required = false, defaultValue = "cql-text") String filterLang ) throws Exception { - if (!restAdminService.isElasticsearchExplainEnabled()) { + if (!ogcApiProperties.debug().elasticsearchExplainEnabled()) { //return 404 NotFound error if elasticsearch-explain-enabled is set as false return ResponseEntity.notFound().build(); } diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/common/RestAdminService.java b/server/src/main/java/au/org/aodn/ogcapi/server/common/RestAdminService.java index 40fb452c..599fe9b1 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/common/RestAdminService.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/common/RestAdminService.java @@ -5,7 +5,6 @@ import com.fasterxml.jackson.databind.JsonNode; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import java.util.List; @@ -13,20 +12,10 @@ @Service @Slf4j public class RestAdminService { - @Value("${ogcapi.debug.elasticsearch-explain-enabled:false}") - protected boolean elasticsearchExplainEnabled; @Autowired protected Search searchService; - /** - * Value defined in application-*.yml, set as true for dev, edge, staging, production and test. - * The default is false, so any environment that does not set it explicitly has explain disabled. - */ - public boolean isElasticsearchExplainEnabled() { - return elasticsearchExplainEnabled; - } - public JsonNode explainByParameters( List q, String filter, diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/Config.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/Config.java index 0101b041..f0c438f7 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/Config.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/Config.java @@ -34,7 +34,8 @@ IndexerProperties.class, GNProperties.class, DasProperties.class, - BatchJobProperties.class + BatchJobProperties.class, + OgcApiProperties.class }) public class Config { diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/CustomWebMvcConfigurer.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/CustomWebMvcConfigurer.java index 0b0af1cd..e322f6ed 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/CustomWebMvcConfigurer.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/CustomWebMvcConfigurer.java @@ -8,6 +8,7 @@ @Component public class CustomWebMvcConfigurer implements WebMvcConfigurer { + /** * In springboot, parameter and path variable conversion isn't done via @JsonCreator but Converter, here * we define additional generic converter for the Enum types. diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/OgcApiProperties.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/OgcApiProperties.java new file mode 100644 index 00000000..ca2fc0ee --- /dev/null +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/OgcApiProperties.java @@ -0,0 +1,66 @@ +package au.org.aodn.ogcapi.server.core.configuration; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.bind.DefaultValue; + +import java.util.List; +import java.util.Map; + +@ConfigurationProperties(prefix = "ogcapi") +public record OgcApiProperties( + @DefaultValue Debug debug, + @DefaultValue HttpCache httpCache +) { + public record Debug( + @DefaultValue("false") boolean elasticsearchExplainEnabled + ) { + } + + public record HttpCache( + @DefaultValue("false") boolean enabled, + List mappings + ) { + public HttpCache { + if (mappings == null) { + mappings = List.of(); + } + } + } + + public record Mapping( + String path, + int maxAgeHours, + Map expectedParams + ) { + public Mapping { + if (expectedParams == null) { + expectedParams = Map.of(); + } + } + + /** + * True when the request path equals {@code path} and the query string contains + * exactly the keys and decoded values in {@code expectedParams} (no extras). + */ + public boolean matches(String requestPath, Map queryParams) { + if (path == null || !path.equals(requestPath)) { + return false; + } + Map params = queryParams == null ? Map.of() : queryParams; + if (expectedParams.size() != params.size()) { + return false; + } + for (Map.Entry expected : expectedParams.entrySet()) { + String[] values = params.get(expected.getKey()); + if (values == null || values.length != 1 || !expected.getValue().equals(values[0])) { + return false; + } + } + return true; + } + + public String cacheControlHeader() { + return "public, max-age=" + Math.max(0, maxAgeHours) * 3600L; + } + } +} diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/http/HttpCacheControlFilter.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/http/HttpCacheControlFilter.java new file mode 100644 index 00000000..064d82a0 --- /dev/null +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/http/HttpCacheControlFilter.java @@ -0,0 +1,111 @@ +package au.org.aodn.ogcapi.server.core.http; + +import au.org.aodn.ogcapi.server.core.configuration.OgcApiProperties; +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletOutputStream; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletResponseWrapper; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.lang.Nullable; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; + +import java.io.IOException; +import java.io.PrintWriter; + +/** + * Sets {@code Cache-Control} only when HTTP cache is enabled and the request path plus + * query parameters match a mapping exactly. Unmatched requests get no cache header. + *

+ * Implemented as a filter (not a {@code HandlerInterceptor}) so the header is applied + * before the response body is written. {@code postHandle} runs after {@code @ResponseBody} + * conversion, by which time compression can already have committed the response. + */ +@Component +public class HttpCacheControlFilter extends OncePerRequestFilter { + + private final OgcApiProperties ogcApiProperties; + + public HttpCacheControlFilter(OgcApiProperties ogcApiProperties) { + this.ogcApiProperties = ogcApiProperties; + } + + @Override + protected void doFilterInternal( + @Nullable HttpServletRequest request, + @Nullable HttpServletResponse response, + @Nullable FilterChain filterChain) throws ServletException, IOException { + + OgcApiProperties.HttpCache httpCache = ogcApiProperties.httpCache(); + if (httpCache == null || !httpCache.enabled() || (request != null && !HttpMethod.GET.matches(request.getMethod()))) { + if (filterChain != null) { + filterChain.doFilter(request, response); + } + return; + } + + if (filterChain != null) { + CacheControlResponseWrapper wrapped = new CacheControlResponseWrapper(request, response, httpCache); + filterChain.doFilter(request, wrapped); + wrapped.applyIfEligible(); + } + } + + static final class CacheControlResponseWrapper extends HttpServletResponseWrapper { + + private final HttpServletRequest request; + private final OgcApiProperties.HttpCache httpCache; + private boolean applied; + + CacheControlResponseWrapper( + HttpServletRequest request, + HttpServletResponse response, + OgcApiProperties.HttpCache httpCache) { + super(response); + this.request = request; + this.httpCache = httpCache; + } + + void applyIfEligible() { + if (applied) { + return; + } + applied = true; + if (getStatus() != HttpStatus.OK.value()) { + return; + } + if (containsHeader(HttpHeaders.CACHE_CONTROL)) { + return; + } + String path = request.getRequestURI(); + for (OgcApiProperties.Mapping mapping : httpCache.mappings()) { + if (mapping.matches(path, request.getParameterMap())) { + setHeader(HttpHeaders.CACHE_CONTROL, mapping.cacheControlHeader()); + return; + } + } + } + + @Override + public ServletOutputStream getOutputStream() throws IOException { + applyIfEligible(); + return super.getOutputStream(); + } + + @Override + public PrintWriter getWriter() throws IOException { + applyIfEligible(); + return super.getWriter(); + } + + @Override + public void flushBuffer() throws IOException { + applyIfEligible(); + super.flushBuffer(); + } + } +} diff --git a/server/src/main/resources/application.yaml b/server/src/main/resources/application.yaml index cf0b896e..f3dadb08 100644 --- a/server/src/main/resources/application.yaml +++ b/server/src/main/resources/application.yaml @@ -8,6 +8,15 @@ server: ogcapi: debug: elasticsearch-explain-enabled: false + http-cache: + enabled: true + mappings: + - path: "/api/v1/ogc/collections" + max-age-hours: 24 + expected-params: + properties: "id,temporal" + filter: "temporal after 1970-01-01T00:00:00Z" # Decoded space format + sortby: "id" elasticsearch: index: diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/common/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/common/RestApiTest.java index a6573258..cba6e690 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/common/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/common/RestApiTest.java @@ -805,4 +805,26 @@ public void verifyQueryByIdWorks() throws IOException { assertEquals(1, Objects.requireNonNull(collections.getBody()).getCollections().size(), "hit 1"); assertEquals("516811d7-cd1e-207a-e0440003ba8c79dd", Objects.requireNonNull(collections.getBody()).getCollections().get(0).getId(), "id correct"); } + /** + * A config set for this particular api call, we need to set the cache control header to signal cloud-front caching + * @throws IOException Not expected + */ + @Test + public void verifyCacheHeaderSetForQuery() throws IOException { + super.insertJsonToElasticRecordIndex( + "073fde5a-bff3-1c1f-e053-08114f8c5588.json", + "5c418118-2581-4936-b6fd-d6bedfe74f62.json", + "19da2ce7-138f-4427-89de-a50c724f5f54.json", + "516811d7-cd1e-207a-e0440003ba8c79dd.json", + "35234913-aa3c-48ec-b9a4-77f822f66ef8.json" // This one have cloud optimized index, that is assets.summary value + ); + + ResponseEntity collections = testRestTemplate.exchange( + getBasePath() + "/collections?properties=id,temporal&filter=temporal after 1970-01-01T00:00:00Z&sortby=id", + HttpMethod.GET, + null, + new ParameterizedTypeReference<>() {}); + + assertNotNull(collections.getHeaders().getFirst("Cache-Control")); + } } diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchTestConfig.java b/server/src/test/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchTestConfig.java index 014725e2..18d1050d 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchTestConfig.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchTestConfig.java @@ -8,7 +8,11 @@ import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; @@ -19,12 +23,16 @@ import org.testcontainers.elasticsearch.ElasticsearchContainer; import org.testcontainers.utility.DockerImageName; +import java.io.IOException; + /** * We use test container with docker image throughout the testing. */ @Configuration public class ElasticSearchTestConfig { + private static final Logger log = LoggerFactory.getLogger(ElasticSearchTestConfig.class); + @Lazy @Autowired protected ElasticsearchContainer container; @@ -53,6 +61,7 @@ public ElasticsearchContainer createElasticDockerTestContainer( .allowInsecure(); ElasticsearchContainer container = new ElasticsearchContainer(ELASTICSEARCH_IMAGE) + .withEnv("xpack.license.self_generated.type", "trial") .waitingFor(httpsWaitStrategy); container.start(); @@ -83,7 +92,24 @@ public RestClientTransport testRestClientTransport() { }) .build(); + startTrialLicense(client); + // Create the transport with a Jackson mapper return new RestClientTransport(client, new JacksonJsonpMapper()); } + + /** + * Testcontainers ships a basic licence. {@code semantic_text} needs the {@code inference} + * feature, which a 30-day self-generated trial enables. The container is discarded after tests. + */ + private static void startTrialLicense(RestClient client) { + try { + Request request = new Request("POST", "/_license/start_trial"); + request.addParameter("acknowledge", "true"); + Response response = client.performRequest(request); + log.info("Elasticsearch trial licence start returned {}", response.getStatusLine()); + } catch (IOException e) { + log.warn("Could not start Elasticsearch trial licence (may already be trial): {}", e.getMessage()); + } + } } diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/core/http/HttpCacheControlFilterTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/core/http/HttpCacheControlFilterTest.java new file mode 100644 index 00000000..c582426c --- /dev/null +++ b/server/src/test/java/au/org/aodn/ogcapi/server/core/http/HttpCacheControlFilterTest.java @@ -0,0 +1,139 @@ +package au.org.aodn.ogcapi.server.core.http; + +import au.org.aodn.ogcapi.server.core.configuration.OgcApiProperties; +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletResponse; +import org.junit.jupiter.api.Test; +import org.springframework.http.HttpHeaders; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +public class HttpCacheControlFilterTest { + + private static final String PATH = "/api/v1/ogc/collections"; + private static final Map EXPECTED = Map.of( + "properties", "id,temporal", + "filter", "temporal after 1970-01-01T00:00:00Z", + "sortby", "id" + ); + + private static OgcApiProperties properties(boolean enabled) { + return new OgcApiProperties( + new OgcApiProperties.Debug(false), + new OgcApiProperties.HttpCache( + enabled, + List.of(new OgcApiProperties.Mapping(PATH, 1, EXPECTED)) + ) + ); + } + + private static MockHttpServletRequest matchingGet() { + MockHttpServletRequest request = new MockHttpServletRequest("GET", PATH); + request.setParameter("properties", "id,temporal"); + request.setParameter("filter", "temporal after 1970-01-01T00:00:00Z"); + request.setParameter("sortby", "id"); + return request; + } + + private static void run(HttpCacheControlFilter filter, MockHttpServletRequest request, MockHttpServletResponse response, FilterChain chain) + throws ServletException, IOException { + filter.doFilter(request, response, chain); + } + + private static FilterChain statusThenWrite(int status) { + return (request, response) -> { + HttpServletResponse http = (HttpServletResponse) response; + http.setStatus(status); + http.getOutputStream().write(new byte[2048]); + }; + } + + @Test + void noHeaderWhenDisabled() throws Exception { + HttpCacheControlFilter filter = new HttpCacheControlFilter(properties(false)); + MockHttpServletResponse response = new MockHttpServletResponse(); + + run(filter, matchingGet(), response, statusThenWrite(200)); + + assertNull(response.getHeader(HttpHeaders.CACHE_CONTROL)); + } + + @Test + void setsHeaderOnExactMatchBeforeBodyWrite() throws Exception { + HttpCacheControlFilter filter = new HttpCacheControlFilter(properties(true)); + MockHttpServletResponse response = new MockHttpServletResponse(); + + run(filter, matchingGet(), response, statusThenWrite(200)); + + assertEquals("public, max-age=3600", response.getHeader(HttpHeaders.CACHE_CONTROL)); + } + + @Test + void noHeaderWhenPathDiffers() throws Exception { + HttpCacheControlFilter filter = new HttpCacheControlFilter(properties(true)); + MockHttpServletRequest request = matchingGet(); + request.setRequestURI("/api/v1/ogc/collections/other"); + MockHttpServletResponse response = new MockHttpServletResponse(); + + run(filter, request, response, statusThenWrite(200)); + + assertNull(response.getHeader(HttpHeaders.CACHE_CONTROL)); + } + + @Test + void noHeaderWhenExtraQueryParamPresent() throws Exception { + HttpCacheControlFilter filter = new HttpCacheControlFilter(properties(true)); + MockHttpServletRequest request = matchingGet(); + request.setParameter("q", "fish"); + MockHttpServletResponse response = new MockHttpServletResponse(); + + run(filter, request, response, statusThenWrite(200)); + + assertNull(response.getHeader(HttpHeaders.CACHE_CONTROL)); + } + + @Test + void noHeaderWhenExpectedParamValueDiffers() throws Exception { + HttpCacheControlFilter filter = new HttpCacheControlFilter(properties(true)); + MockHttpServletRequest request = matchingGet(); + request.setParameter("sortby", "-score"); + MockHttpServletResponse response = new MockHttpServletResponse(); + + run(filter, request, response, statusThenWrite(200)); + + assertNull(response.getHeader(HttpHeaders.CACHE_CONTROL)); + } + + @Test + void noHeaderWhenNotOk() throws Exception { + HttpCacheControlFilter filter = new HttpCacheControlFilter(properties(true)); + MockHttpServletResponse response = new MockHttpServletResponse(); + + run(filter, matchingGet(), response, statusThenWrite(500)); + + assertNull(response.getHeader(HttpHeaders.CACHE_CONTROL)); + } + + @Test + void doesNotOverrideExistingCacheControl() throws Exception { + HttpCacheControlFilter filter = new HttpCacheControlFilter(properties(true)); + MockHttpServletResponse response = new MockHttpServletResponse(); + + run(filter, matchingGet(), response, (request, resp) -> { + HttpServletResponse http = (HttpServletResponse) resp; + http.setStatus(200); + http.setHeader(HttpHeaders.CACHE_CONTROL, "no-store"); + http.getOutputStream().write(new byte[2048]); + }); + + assertEquals("no-store", response.getHeader(HttpHeaders.CACHE_CONTROL)); + } +}