From 9cf9065b0a5c9468c0b4b154b3382c65c91edff4 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Sun, 19 Jul 2026 12:53:52 +0200 Subject: [PATCH 01/61] Consolidate raster handling and use std::expected --- src/CMakeLists.txt | 6 +- src/dag_builder/encoded.h | 6 +- src/dag_builder/thread_safe_storage.h | 6 +- src/mesh_convert/main.cpp | 6 +- src/sf_builder/CMakeLists.txt | 2 +- src/sf_builder/mesh_builder.cpp | 55 ++++--- src/sf_builder/mesh_builder.h | 4 +- src/sf_builder/raster.h | 145 ------------------ src/sf_builder/raw_dataset_reader.h | 14 +- src/sf_builder/terrainbuilder.cpp | 2 +- src/sf_merger/mask.h | 14 +- src/terrainlib/CMakeLists.txt | 1 - src/terrainlib/io/bytes.cpp | 14 +- src/terrainlib/io/bytes.h | 6 +- src/terrainlib/io/serialize.h | 10 +- src/terrainlib/io/serialize.inl | 20 +-- src/terrainlib/mesh/encode.h | 24 +-- src/terrainlib/mesh/io.cpp | 8 +- src/terrainlib/mesh/io.h | 6 +- src/terrainlib/mesh/io/gltf.cpp | 34 ++-- src/terrainlib/mesh/io/gltf.h | 12 +- src/terrainlib/mesh/io/terrain.cpp | 40 ++--- src/terrainlib/mesh/io/terrain.h | 10 +- .../octree/storage/IndexedStorage.h | 2 +- src/terrainlib/octree/storage/RawStorage.h | 22 +-- src/terrainlib/octree/storage/Storage.h | 16 +- src/terrainlib/octree/storage/codec/Codec.h | 6 +- .../octree/storage/codec/DefaultCodec.h | 6 +- .../octree/storage/codec/MeshCodec.h | 6 +- src/terrainlib/octree/storage/helpers.cpp | 6 +- src/terrainlib/octree/storage/helpers.h | 4 +- src/terrainlib/octree/storage/open.h | 4 +- src/terrainlib/octree/storage/open.inl | 6 +- src/terrainlib/pch.h | 2 +- src/terrainlib/srs.h | 10 +- src/terrainlib/uv/unwrap.cpp | 8 +- src/terrainlib/uv/unwrap.h | 10 +- src/tile_builder/CMakeLists.txt | 2 +- src/tile_builder/DatasetReader.cpp | 12 +- src/tile_builder/DatasetReader.h | 8 +- src/tile_builder/Image.cpp | 20 --- src/tile_builder/Image.h | 105 ------------- src/tile_builder/ParallelTileGenerator.cpp | 2 +- src/tile_builder/ParallelTileGenerator.h | 6 +- src/tile_builder/algorithms/primitives.h | 103 ------------- .../algorithms/raster_triangle_scanline.h | 101 ------------ src/tile_builder/alpine_raster.cpp | 8 +- src/tile_builder/alpine_raster.h | 4 +- src/tile_builder/image_writer.cpp | 46 ++++++ src/tile_builder/image_writer.h | 51 ++++++ unittests/CMakeLists.txt | 2 +- unittests/terrainlib/mesh_io.cpp | 12 +- .../tilebuilder/alpine_raster_format.cpp | 8 +- unittests/tilebuilder/dataset_reading.cpp | 17 +- unittests/tilebuilder/image.cpp | 66 -------- unittests/tilebuilder/image_writer.cpp | 86 +++++++++++ .../tilebuilder/parallel_tile_generator.cpp | 2 +- 57 files changed, 431 insertions(+), 783 deletions(-) delete mode 100644 src/sf_builder/raster.h delete mode 100644 src/tile_builder/Image.cpp delete mode 100644 src/tile_builder/Image.h delete mode 100644 src/tile_builder/algorithms/primitives.h delete mode 100644 src/tile_builder/algorithms/raster_triangle_scanline.h create mode 100644 src/tile_builder/image_writer.cpp create mode 100644 src/tile_builder/image_writer.h delete mode 100644 unittests/tilebuilder/image.cpp create mode 100644 unittests/tilebuilder/image_writer.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5ece70b8..95f856ed 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -35,10 +35,6 @@ alp_add_git_repository(stb URL https://github.com/nothings/stb COMMITISH f4a71b1 add_library(stb INTERFACE) target_include_directories(stb SYSTEM INTERFACE ${stb_SOURCE_DIR}) -alp_add_git_repository(tl_expected URL https://github.com/TartanLlama/expected.git COMMITISH v1.1.0 DO_NOT_ADD_SUBPROJECT) -add_library(tl_expected INTERFACE) -target_include_directories(tl_expected SYSTEM INTERFACE ${tl_expected_SOURCE_DIR}/include) - set(SPDLOG_FMT_EXTERNAL ON CACHE BOOL "SPDLOG_FMT_EXTERNAL" FORCE) alp_add_git_repository(spdlog URL https://github.com/gabime/spdlog.git COMMITISH v1.17.0) @@ -54,7 +50,7 @@ set(LIBIGL_COPYLEFT_CORE ON) set(LIBIGL_RESTRICTED_TRIANGLE ON) alp_add_git_repository(libigl URL https://github.com/libigl/libigl.git COMMITISH v2.6.0) -alp_add_git_repository(radix URL https://github.com/AlpineMapsOrg/radix.git COMMITISH 2ce3484513b826bb11e5433f868b048eaffe3e5d NOT_SYSTEM DEEP_CLONE) +alp_add_git_repository(radix URL https://github.com/AlpineMapsOrg/radix.git COMMITISH 9720bce6399f4581e35edc88f87651f020348b34 NOT_SYSTEM DEEP_CLONE) set(ALP_CLI_COMPONENT_OPTIONS ALP_BUILD_SF_MERGER diff --git a/src/dag_builder/encoded.h b/src/dag_builder/encoded.h index 769e567f..2ccee0e9 100644 --- a/src/dag_builder/encoded.h +++ b/src/dag_builder/encoded.h @@ -6,7 +6,7 @@ #include #include -#include +#include #include #include #include @@ -165,14 +165,14 @@ auto serialize(Archive &archive, Clustering &clustering) { } } -inline tl::expected +inline std::expected save_clustering(const Clustering &clustering, const std::filesystem::path &path, const bool make_dirs = true) { return ::io::write_to_path(clustering, path, make_dirs); } -inline tl::expected +inline std::expected load_clustering(const std::filesystem::path &path) { return ::io::read_from_path(path); } diff --git a/src/dag_builder/thread_safe_storage.h b/src/dag_builder/thread_safe_storage.h index 4190acac..b0d25998 100644 --- a/src/dag_builder/thread_safe_storage.h +++ b/src/dag_builder/thread_safe_storage.h @@ -4,7 +4,7 @@ #include #include -#include +#include #include "octree/Id.h" @@ -33,7 +33,7 @@ class ThreadSafeStorage { return std::move(this->_storage); } - tl::expected load(const octree::Id &id) const { + std::expected load(const octree::Id &id) const { std::shared_lock lock(this->_mutex); return this->_storage.load(id); } @@ -47,7 +47,7 @@ class ThreadSafeStorage { return this->_storage.base_path(); } - tl::expected save(const octree::Id &id, const value_type &value) const { + std::expected save(const octree::Id &id, const value_type &value) const { std::unique_lock lock(this->_mutex); return this->_storage.save(id, value); } diff --git a/src/mesh_convert/main.cpp b/src/mesh_convert/main.cpp index ef258319..a396ce50 100644 --- a/src/mesh_convert/main.cpp +++ b/src/mesh_convert/main.cpp @@ -1,7 +1,7 @@ #include #include -#include +#include #include "mesh/SimpleMesh.h" #include "mesh/io.h" @@ -11,7 +11,7 @@ void run(const cli::Args& args) { LOG_INFO("Loading input mesh..."); - const tl::expected load_result = mesh::io::load_from_path(args.input_path); + const std::expected load_result = mesh::io::load_from_path(args.input_path); if (!load_result.has_value()) { LOG_ERROR("Failed to load mesh: {}", load_result.error().description()); return; @@ -30,7 +30,7 @@ void run(const cli::Args& args) { } LOG_INFO("Writing output mesh..."); - const tl::expected save_result = mesh::io::save_to_path(mesh, args.output_path); + const std::expected save_result = mesh::io::save_to_path(mesh, args.output_path); if (!save_result.has_value()) { LOG_ERROR("Failed to save mesh: {}", save_result.error().description()); return; diff --git a/src/sf_builder/CMakeLists.txt b/src/sf_builder/CMakeLists.txt index c50cae1c..cfdabb70 100644 --- a/src/sf_builder/CMakeLists.txt +++ b/src/sf_builder/CMakeLists.txt @@ -3,7 +3,7 @@ add_library(sfbuilderlib mesh_builder.cpp ) target_include_directories(sfbuilderlib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(sfbuilderlib PUBLIC terrainlib spdlog tl_expected) +target_link_libraries(sfbuilderlib PUBLIC terrainlib spdlog) add_executable(sf-builder main.cpp) target_link_libraries(sf-builder PRIVATE sfbuilderlib CLI11::CLI11) diff --git a/src/sf_builder/mesh_builder.cpp b/src/sf_builder/mesh_builder.cpp index 364d2368..cba54e72 100644 --- a/src/sf_builder/mesh_builder.cpp +++ b/src/sf_builder/mesh_builder.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -15,7 +16,7 @@ #include "mesh/SimpleMesh.h" #include "mesh/cleanup.h" #include "mesh_builder.h" -#include "raster.h" +#include #include "raw_dataset_reader.h" #include "srs.h" #include "mesh/clip.h" @@ -50,7 +51,7 @@ glm::dvec2 apply_transform(std::array transform, const glm::tvec2 return result; } -glm::dvec3 convert_pixel_to_vertex(const float height, const raster::Coords pixel_coords, const RawDatasetReader& reader, const PixelBounds& pixel_bounds) { +glm::dvec3 convert_pixel_to_vertex(const float height, const glm::uvec2 pixel_coords, const RawDatasetReader& reader, const PixelBounds& pixel_bounds) { const glm::dvec2 point_offset_in_raster(0.5); // Convert pixel coordinates into a point in the dataset's srs. const glm::dvec2 coords_raster_relative = glm::dvec2(pixel_coords) + point_offset_in_raster; const glm::dvec2 coords_raster_absolute = coords_raster_relative + glm::dvec2(pixel_bounds.min); @@ -58,9 +59,9 @@ glm::dvec3 convert_pixel_to_vertex(const float height, const raster::Coords pixe return coords_source; } -SimpleMesh meshify(const raster::Raster& source_points, const raster::Mask& mask) { +SimpleMesh meshify(const radix::Raster& source_points, const radix::RasterMask& mask) { // Compact the vertex grid into a list of valid ones. - const size_t valid_vertex_count = std::reduce(mask.begin(), mask.end(), 0); + const size_t valid_vertex_count = std::reduce(mask.begin(), mask.end(), size_t(0)); // Check if we even have any valid vertices. Can happen if all of the region is padding. if (valid_vertex_count == 0) { return SimpleMesh(); @@ -69,11 +70,15 @@ SimpleMesh meshify(const raster::Raster& source_points, const raster std::vector positions; positions.reserve(valid_vertex_count); - const raster::Raster vertex_index_map = raster::transform(source_points, mask, [&](const glm::dvec3 &point) -> size_t { + auto vertex_index_map_result = radix::raster::transform(source_points, mask, [&](const glm::dvec3& point) -> size_t { const size_t index = positions.size(); positions.push_back(point); return index; }); + DEBUG_ASSERT(vertex_index_map_result.has_value()); + if (!vertex_index_map_result.has_value()) + return {}; + const auto vertex_index_map = std::move(*vertex_index_map_result); DEBUG_ASSERT(positions.size() == valid_vertex_count); // Allocate triangle vector @@ -81,13 +86,13 @@ SimpleMesh meshify(const raster::Raster& source_points, const raster std::vector triangles; triangles.reserve(max_triangle_count); - for (size_t y = 0; y < source_points.height() - 1; y++) { - for (size_t x = 0; x < source_points.width() - 1; x++) { - const std::array quad { - raster::Coords{x, y}, - raster::Coords{x + 1, y}, - raster::Coords{x + 1, y + 1}, - raster::Coords{x, y + 1}}; + for (unsigned y = 0; y < source_points.height() - 1; y++) { + for (unsigned x = 0; x < source_points.width() - 1; x++) { + const std::array quad { + glm::uvec2{x, y}, + glm::uvec2{x + 1, y}, + glm::uvec2{x + 1, y + 1}, + glm::uvec2{x, y + 1}}; for (uint32_t i = 0; i < 4; i++) { const auto& v0 = quad[i]; @@ -142,7 +147,7 @@ radix::geometry::Aabb3d extend_bounds_to_3d(radix::geometry::Aabb2d bounds2d) { } } -tl::expected build_reference_mesh_tile( +std::expected build_reference_mesh_tile( Dataset &dataset, const OGRSpatialReference &mesh_srs, const OGRSpatialReference &tile_srs, const radix::tile::SrsBounds &tile_bounds, @@ -150,7 +155,7 @@ tl::expected build_reference_mesh_tile( return build_reference_mesh_patch(dataset, mesh_srs, tile_srs, extend_bounds_to_3d(tile_bounds), texture_srs, texture_bounds); } -tl::expected build_reference_mesh_patch( +std::expected build_reference_mesh_patch( Dataset &dataset, const OGRSpatialReference &mesh_srs, const OGRSpatialReference &clip_srs, const radix::geometry::Aabb3d &clip_bounds, @@ -171,35 +176,39 @@ tl::expected build_reference_mesh_patch( radix::geometry::Aabb2i pixel_bounds = reader.transform_srs_bounds_to_pixel_bounds(target_bounds_in_source_srs); add_border_to_aabb(pixel_bounds, Border(1)); LOG_TRACE("Reading pixels [({}, {})-({}, {})] from dataset", pixel_bounds.min.x, pixel_bounds.min.y, pixel_bounds.max.x, pixel_bounds.max.y); - const std::optional read_result = reader.read_data_in_pixel_bounds_clamped(pixel_bounds); - if (!read_result.has_value() || read_result->size() == 0) { - return tl::unexpected(BuildMeshError::OutOfBounds); + auto read_result = reader.read_data_in_pixel_bounds_clamped(pixel_bounds); + if (!read_result.has_value() || read_result->buffer().empty()) { + return std::unexpected(BuildMeshError::OutOfBounds); } - const raster::HeightMap height_map = read_result.value(); + const radix::Raster height_map = std::move(*read_result); LOG_TRACE("Finding valid pixels"); const float no_data_value = reader.get_no_data_value(); - const raster::Mask valid_mask = raster::transform(height_map, [=](const float height) { + const radix::RasterMask valid_mask = radix::raster::transform(height_map, [=](const float height) { return height != no_data_value; }); LOG_TRACE("Transforming pixels to vertices"); - const raster::Raster source_points = raster::transform(height_map, valid_mask, [&](const float height, const raster::Coords& coords) { + auto source_points_result = radix::raster::transform(height_map, valid_mask, [&](const float height, const glm::uvec2& coords) { return convert_pixel_to_vertex(height, coords, reader, pixel_bounds); }); + DEBUG_ASSERT(source_points_result.has_value()); + if (!source_points_result.has_value()) + return std::unexpected(BuildMeshError::EmptyRegion); + const auto source_points = std::move(*source_points_result); LOG_TRACE("Generating triangles"); SimpleMesh mesh_in_source_srs = meshify(source_points, valid_mask); // Check if we even have any valid vertices. Can happen if all of the region is padding. if (mesh_in_source_srs.vertex_count() == 0 || mesh_in_source_srs.face_count() == 0) { - return tl::unexpected(BuildMeshError::EmptyRegion); + return std::unexpected(BuildMeshError::EmptyRegion); } // Fast check if all vertices will be clipped const radix::geometry::Aabb3d actual_source_bounds = calculate_bounds(mesh_in_source_srs); const radix::geometry::Aabb3d approx_clip_bounds = srs::encompassing_bounds_transfer(source_srs, clip_srs, actual_source_bounds); if (!radix::geometry::intersect(approx_clip_bounds, clip_bounds)) { - return tl::unexpected(BuildMeshError::EmptyRegion); + return std::unexpected(BuildMeshError::EmptyRegion); } LOG_TRACE("Clipping mesh based on target bounds"); @@ -207,7 +216,7 @@ tl::expected build_reference_mesh_patch( SimpleMesh clipped_mesh = mesh::clip_on_bounds(mesh_in_clip_srs, clip_bounds); // Check if there are any vertices left if (clipped_mesh.vertex_count() == 0 || clipped_mesh.face_count() == 0) { - return tl::unexpected(BuildMeshError::EmptyRegion); + return std::unexpected(BuildMeshError::EmptyRegion); } // TODO: move this to another function? diff --git a/src/sf_builder/mesh_builder.h b/src/sf_builder/mesh_builder.h index acf3e4b8..2979042d 100644 --- a/src/sf_builder/mesh_builder.h +++ b/src/sf_builder/mesh_builder.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include "Dataset.h" #include "srs.h" @@ -17,7 +17,7 @@ enum class BuildMeshError { std::ostream &operator<<(std::ostream &os, BuildMeshError error); /// Builds a mesh from the given height dataset. -tl::expected build_reference_mesh_patch( +std::expected build_reference_mesh_patch( Dataset &dataset, const OGRSpatialReference &mesh_srs, const OGRSpatialReference &clip_srs, const radix::geometry::Aabb3d &clip_bounds, diff --git a/src/sf_builder/raster.h b/src/sf_builder/raster.h deleted file mode 100644 index fe9ee5a6..00000000 --- a/src/sf_builder/raster.h +++ /dev/null @@ -1,145 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -namespace raster { - -using Index = size_t; -using Coords = glm::vec<2, size_t>; - -template -class Raster { -public: - Raster() = default; - Raster(Index width, Index height) - : _width(width), _height(height), _data(width * height) { - } - - [[nodiscard]] Index width() const { - return this->_width; - } - [[nodiscard]] Index height() const { - return this->_height; - } - [[nodiscard]] decltype(auto) pixel(const Coords &coords) { - DEBUG_ASSERT(coords.x < this->_width); - DEBUG_ASSERT(coords.y < this->_height); - const Index pixel_index = this->index(coords); - if constexpr (std::is_same_v) { - return static_cast(this->_data[pixel_index]); - } else { - return static_cast(this->_data[pixel_index]); - } - } - [[nodiscard]] decltype(auto) pixel(const Coords &coords) const { - DEBUG_ASSERT(coords.x < this->_width); - DEBUG_ASSERT(coords.y < this->_height); - const Index pixel_index = this->index(coords); - if constexpr (std::is_same_v) { - return static_cast(this->_data[pixel_index]); - } else { - return static_cast(this->_data[pixel_index]); - } - } - - [[nodiscard]] Index index(const Coords& coords) const { - return coords.y * this->_width + coords.x; - } - - [[nodiscard]] T *data() { - return this->_data.data(); - } - [[nodiscard]] const T *data() const { - return this->_data.data(); - } - - [[nodiscard]] Index size() const { - return this->_data.size(); - } - [[nodiscard]] auto begin() { - return this->_data.begin(); - } - [[nodiscard]] auto end() { - return this->_data.end(); - } - [[nodiscard]] auto begin() const { - return this->_data.begin(); - } - [[nodiscard]] auto end() const { - return this->_data.end(); - } - -private: - unsigned _width = 0; - unsigned _height = 0; - std::vector _data; -}; - -using Mask = Raster; -using HeightMap = Raster; - -template -concept TransformFn = requires(F f, In in) { - { f(in) }; -}; - -template -concept TransformFnWithCoords = requires(F f, In in, Coords coord) { - { f(in, coord) }; -}; - -template F> -[[nodiscard]] auto transform(const Raster &input, F &&f) { - using Out = decltype(f(input.pixel(Coords(0, 0)))); - Raster output(input.width(), input.height()); - std::transform(input.begin(), input.end(), output.begin(), std::forward(f)); - return output; -} - -template F> -[[nodiscard]] auto transform(const Raster &input, F &&f) { - using Out = decltype(f(input.pixel(Coords(0, 0)), Coords(0, 0))); - Raster output(input.width(), input.height()); - for (Index y = 0; y < input.height(); ++y) { - for (Index x = 0; x < input.width(); ++x) { - Coords coords(x, y); - output.pixel(coords) = f(input.pixel(coords), coords); - } - } - return output; -} - -template F> -[[nodiscard]] auto transform(const Raster &input, const Mask &mask, F &&f) { - using Out = decltype(f(input.pixel(Coords(0, 0)))); - Raster output(input.width(), input.height()); - for (Index y = 0; y < input.height(); ++y) { - for (Index x = 0; x < input.width(); ++x) { - Coords coords(x, y); - if (mask.pixel(coords)) { - output.pixel(coords) = f(input.pixel(coords)); - } - } - } - return output; -} - -template F> -[[nodiscard]] auto transform(const Raster &input, const Mask &mask, F &&f) { - using Out = decltype(f(input.pixel(Coords(0, 0)), Coords(0, 0))); - Raster output(input.width(), input.height()); - for (Index y = 0; y < input.height(); ++y) { - for (Index x = 0; x < input.width(); ++x) { - Coords coords(x, y); - if (mask.pixel(coords)) { - output.pixel(coords) = f(input.pixel(coords), coords); - } - } - } - return output; -} -} diff --git a/src/sf_builder/raw_dataset_reader.h b/src/sf_builder/raw_dataset_reader.h index 9eed966e..88e4d9cb 100644 --- a/src/sf_builder/raw_dataset_reader.h +++ b/src/sf_builder/raw_dataset_reader.h @@ -7,7 +7,7 @@ #include #include "log.h" -#include "raster.h" +#include namespace terrainbuilder { @@ -46,7 +46,7 @@ class RawDatasetReader { } // TODO: support reading other data types - std::optional read_data_in_pixel_bounds(const radix::geometry::Aabb2i& bounds) { + std::optional> read_data_in_pixel_bounds(const radix::geometry::Aabb2i& bounds) { DEBUG_ASSERT(glm::all(glm::greaterThanEqual(bounds.min, glm::ivec2(0)))); DEBUG_ASSERT(glm::all(glm::greaterThanEqual(bounds.max, glm::ivec2(0)))); DEBUG_ASSERT(glm::all(glm::lessThan(bounds.min, glm::ivec2(this->dataset_size())))); @@ -56,7 +56,7 @@ class RawDatasetReader { GDALRasterBand *height_band = this->dataset->GetRasterBand(1); // non-owning pointer // Initialize the HeightData for reading - raster::HeightMap height_data(bounds.width(), bounds.height()); + radix::Raster height_data(glm::uvec2(bounds.width(), bounds.height())); if (bounds.width() == 0 || bounds.height() == 0) { LOG_WARN("Target dataset bounds are empty"); return height_data; @@ -65,7 +65,7 @@ class RawDatasetReader { // Read data from the heights band into heights_data const int read_result = height_band->RasterIO( GF_Read, bounds.min.x, bounds.min.y, bounds.width(), bounds.height(), - static_cast(height_data.data()), bounds.width(), bounds.height(), GDT_Float32, 0, 0); + static_cast(height_data.buffer().data()), bounds.width(), bounds.height(), GDT_Float32, 0, 0); if (read_result != CE_None) { const char * message = CPLGetLastErrorMsg(); @@ -75,7 +75,7 @@ class RawDatasetReader { return height_data; } - std::optional read_data_in_pixel_bounds_clamped(radix::geometry::Aabb2i &bounds) { + std::optional> read_data_in_pixel_bounds_clamped(radix::geometry::Aabb2i &bounds) { const auto original_bounds = bounds; const glm::ivec2 max_in_bounds = glm::ivec2(this->dataset_size()) - glm::ivec2(1); @@ -90,13 +90,13 @@ class RawDatasetReader { if (bounds.width() == 0 || bounds.height() == 0) { LOG_WARN("Target dataset bounds are empty (clamped)"); - return raster::HeightMap(0, 0); + return radix::Raster(); } return this->read_data_in_pixel_bounds(bounds); } - std::optional read_data_in_srs_bounds(const radix::tile::SrsBounds &bounds) { + std::optional> read_data_in_srs_bounds(const radix::tile::SrsBounds &bounds) { // Transform the SrsBounds to pixel space radix::geometry::Aabb2i pixel_bounds = this->transform_srs_bounds_to_pixel_bounds(bounds); diff --git a/src/sf_builder/terrainbuilder.cpp b/src/sf_builder/terrainbuilder.cpp index ca5a74db..09d47079 100644 --- a/src/sf_builder/terrainbuilder.cpp +++ b/src/sf_builder/terrainbuilder.cpp @@ -56,7 +56,7 @@ std::optional build_patch( std::chrono::high_resolution_clock::time_point start; start = std::chrono::high_resolution_clock::now(); LOG_INFO("Building mesh..."); - tl::expected mesh_result = build_reference_mesh_patch( + std::expected mesh_result = build_reference_mesh_patch( dataset, mesh_srs, target_bounds_srs, target_bounds, diff --git a/src/sf_merger/mask.h b/src/sf_merger/mask.h index 6ca6ccd3..47f2b875 100644 --- a/src/sf_merger/mask.h +++ b/src/sf_merger/mask.h @@ -253,7 +253,7 @@ auto length2(const Vec &v) -> decltype(glm::dot(v, v)) { } // namespace -inline tl::expected load_referenced_from_dataset(Dataset& mask_dataset) { +inline std::expected load_referenced_from_dataset(Dataset& mask_dataset) { GDALDataset *dataset = mask_dataset.gdalDataset(); MultipolygonWithHoles2 polygons; @@ -265,7 +265,7 @@ inline tl::expected load_referenced_from_datas if (polygons.is_empty()) { LOG_ERROR("No valid polygons found in mask dataset '{}'", mask_dataset.name()); - return tl::unexpected(LoadErrorKind::EmptySource); + return std::unexpected(LoadErrorKind::EmptySource); } OGRSpatialReference srs; @@ -438,25 +438,25 @@ inline MeshMask extrude( return extrude(mask, padded_radius_range); } -inline tl::expected load_referenced_from_path(const std::filesystem::path &path) { +inline std::expected load_referenced_from_path(const std::filesystem::path &path) { if (!std::filesystem::exists(path)) { LOG_ERROR("Mask file does not exist: {}", path); - return tl::unexpected(LoadErrorKind::FileNotFound); + return std::unexpected(LoadErrorKind::FileNotFound); } auto ds_opt = Dataset::open_vector(path); if (!ds_opt.has_value()) { LOG_ERROR("Failed to load mask datset: {}", path); - return tl::unexpected(LoadErrorKind::FileNotFound); + return std::unexpected(LoadErrorKind::FileNotFound); } Dataset dataset = std::move(ds_opt.value()); return load_referenced_from_dataset(dataset); } -inline tl::expected load_from_path(const std::filesystem::path &path, const glm::dvec2& radius_range) { +inline std::expected load_from_path(const std::filesystem::path &path, const glm::dvec2& radius_range) { auto ref_mask_res = load_referenced_from_path(path); if (!ref_mask_res.has_value()) { - return tl::unexpected(ref_mask_res.error()); + return std::unexpected(ref_mask_res.error()); } ReferencedPolygonMask ref_polygon_mask = std::move(ref_mask_res.value()); SpherePolygonMask sphere_polygon_mask = project_onto_sphere(std::move(ref_polygon_mask), radius_range.x); diff --git a/src/terrainlib/CMakeLists.txt b/src/terrainlib/CMakeLists.txt index 79dd32d6..2e321547 100644 --- a/src/terrainlib/CMakeLists.txt +++ b/src/terrainlib/CMakeLists.txt @@ -71,7 +71,6 @@ target_link_libraries(terrainlib PUBLIC zpp_bits cgltf TBB::tbb - tl_expected opencv_core opencv_imgproc opencv_imgcodecs diff --git a/src/terrainlib/io/bytes.cpp b/src/terrainlib/io/bytes.cpp index 499b0ee9..1412ce33 100644 --- a/src/terrainlib/io/bytes.cpp +++ b/src/terrainlib/io/bytes.cpp @@ -6,7 +6,7 @@ namespace io { -tl::expected write_bytes_to_path(const std::span bytes, const std::filesystem::path &path, bool make_dirs) { +std::expected write_bytes_to_path(const std::span bytes, const std::filesystem::path &path, bool make_dirs) { LOG_TRACE("Writing bytes to path {}", path); if (make_dirs) { @@ -16,31 +16,31 @@ tl::expected write_bytes_to_path(const std::span byt std::ofstream file(path, std::ios::binary); if (!file.is_open()) { LOG_DEBUG("Failed to open file for writing {}", path); - return tl::unexpected(Error::OpenFile); + return std::unexpected(Error::OpenFile); } file.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); if (!file.good()) { LOG_ERROR("Failed to write bytes to file {}", path); - return tl::unexpected(Error::WriteBytes); + return std::unexpected(Error::WriteBytes); } return {}; } -tl::expected, Error> read_bytes_from_path(const std::filesystem::path& path) { +std::expected, Error> read_bytes_from_path(const std::filesystem::path& path) { LOG_TRACE("Reading bytes from path {}", path); std::ifstream file(path, std::ios::binary | std::ios::ate); if (!file.is_open()) { LOG_DEBUG("Failed to open file for reading {}", path); - return tl::unexpected(Error::OpenFile); + return std::unexpected(Error::OpenFile); } const std::streamsize size = file.tellg(); if (size < 0) { LOG_ERROR("Failed to determine size for file {}", path); - return tl::unexpected(Error::DetermineSize); + return std::unexpected(Error::DetermineSize); } std::vector buffer(static_cast(size)); @@ -49,7 +49,7 @@ tl::expected, Error> read_bytes_from_path(const std::filesy if (!file.good()) { LOG_ERROR("Failed to read bytes from file {}", path); - return tl::unexpected(Error::ReadBytes); + return std::unexpected(Error::ReadBytes); } return buffer; diff --git a/src/terrainlib/io/bytes.h b/src/terrainlib/io/bytes.h index 5c156cf4..be3852dc 100644 --- a/src/terrainlib/io/bytes.h +++ b/src/terrainlib/io/bytes.h @@ -4,13 +4,13 @@ #include #include -#include +#include #include "io/Error.h" namespace io { -tl::expected write_bytes_to_path(const std::span bytes, const std::filesystem::path &path, bool make_dirs = true); -tl::expected, Error> read_bytes_from_path(const std::filesystem::path &path); +std::expected write_bytes_to_path(const std::span bytes, const std::filesystem::path &path, bool make_dirs = true); +std::expected, Error> read_bytes_from_path(const std::filesystem::path &path); } diff --git a/src/terrainlib/io/serialize.h b/src/terrainlib/io/serialize.h index 55919e23..ffda2d1f 100644 --- a/src/terrainlib/io/serialize.h +++ b/src/terrainlib/io/serialize.h @@ -4,21 +4,21 @@ #include #include -#include +#include #include "io/Error.h" namespace io { template -tl::expected, Error> write_to_bytes(const T &value); +std::expected, Error> write_to_bytes(const T &value); template -tl::expected read_from_bytes(const std::span bytes); +std::expected read_from_bytes(const std::span bytes); template -tl::expected write_to_path(const T &value, const std::filesystem::path &path, bool make_dirs = true); +std::expected write_to_path(const T &value, const std::filesystem::path &path, bool make_dirs = true); template -tl::expected read_from_path(const std::filesystem::path &path); +std::expected read_from_path(const std::filesystem::path &path); } diff --git a/src/terrainlib/io/serialize.inl b/src/terrainlib/io/serialize.inl index 90d7080b..80324d03 100644 --- a/src/terrainlib/io/serialize.inl +++ b/src/terrainlib/io/serialize.inl @@ -9,7 +9,7 @@ namespace io { template -tl::expected, Error> write_to_bytes(const T &value) { +std::expected, Error> write_to_bytes(const T &value) { std::vector data; zpp::bits::out out(data); const auto result = out(value); @@ -21,11 +21,11 @@ tl::expected, Error> write_to_bytes(const T &value) { case std::errc::no_buffer_space: // growing buffer would grow beyond the allocation limits or overflow. case std::errc::message_size: // message size is beyond the user defined allocation limits. case std::errc::result_out_of_range: // attempting to write or read from a too short buffer. - return tl::unexpected(Error::OutOfMemory); + return std::unexpected(Error::OutOfMemory); case std::errc::value_too_large: // varint (variable length integer) encoding is beyond the representation limits. case std::errc::bad_message: // attempt to read a variant of unrecognized type. case std::errc::invalid_argument: // attempting to serialize null pointer or a value-less variant. - return tl::unexpected(Error::Serialize); + return std::unexpected(Error::Serialize); case std::errc::protocol_error: // attempt to deserialize an invalid protocol message. case std::errc::not_supported: // attempt to call an RPC that is not listed as supported. UNREACHABLE(); @@ -37,7 +37,7 @@ tl::expected, Error> write_to_bytes(const T &value) { } template -tl::expected read_from_bytes(const std::span bytes) { +std::expected read_from_bytes(const std::span bytes) { zpp::bits::in in(bytes); T value; const auto result = in(value); @@ -49,11 +49,11 @@ tl::expected read_from_bytes(const std::span bytes) { case std::errc::no_buffer_space: // growing buffer would grow beyond the allocation limits or overflow. case std::errc::message_size: // message size is beyond the user defined allocation limits. case std::errc::result_out_of_range: // attempting to write or read from a too short buffer. - return tl::unexpected(Error::OutOfMemory); + return std::unexpected(Error::OutOfMemory); case std::errc::value_too_large: // varint (variable length integer) encoding is beyond the representation limits. case std::errc::bad_message: // attempt to read a variant of unrecognized type. case std::errc::invalid_argument: // attempting to serialize null pointer or a value-less variant. - return tl::unexpected(Error::Deserialize); + return std::unexpected(Error::Deserialize); case std::errc::protocol_error: // attempt to deserialize an invalid protocol message. case std::errc::not_supported: // attempt to call an RPC that is not listed as supported. UNREACHABLE(); @@ -65,20 +65,20 @@ tl::expected read_from_bytes(const std::span bytes) { } template -tl::expected read_from_path(const std::filesystem::path &path) { +std::expected read_from_path(const std::filesystem::path &path) { const auto result = read_bytes_from_path(path); if (!result.has_value()) { - return tl::unexpected(result.error()); + return std::unexpected(result.error()); } const std::vector bytes = result.value(); return read_from_bytes(bytes); } template -tl::expected write_to_path(const T &value, const std::filesystem::path &path, bool make_dirs) { +std::expected write_to_path(const T &value, const std::filesystem::path &path, bool make_dirs) { const auto result = write_to_bytes(value); if (!result.has_value()) { - return tl::unexpected(result.error()); + return std::unexpected(result.error()); } const std::vector bytes = result.value(); diff --git a/src/terrainlib/mesh/encode.h b/src/terrainlib/mesh/encode.h index 302fd016..eca66508 100644 --- a/src/terrainlib/mesh/encode.h +++ b/src/terrainlib/mesh/encode.h @@ -4,7 +4,7 @@ #include #include -#include +#include #include "log.h" #include "mesh/EncodedMesh.h" @@ -81,7 +81,7 @@ inline std::ostream& operator<<(std::ostream& os, const EncodeError& err) { } template -tl::expected encode(const Simple_& mesh, const EncodeOptions options = {}) { +std::expected encode(const Simple_& mesh, const EncodeOptions options = {}) { using Mesh = Simple_; const size_t vertex_count = mesh.vertex_count(); @@ -96,7 +96,7 @@ tl::expected encode(const Simple_& mesh, const position_buf.resize(meshopt_encodeVertexBufferBound(vertex_count, position_size)); const size_t pos_written = meshopt_encodeVertexBuffer(position_buf.data(), position_buf.size(), mesh.positions.data(), vertex_count, position_size); if (pos_written == 0) { - return tl::unexpected(EncodeError::PositionEncode); + return std::unexpected(EncodeError::PositionEncode); } position_buf.resize(pos_written); } @@ -107,7 +107,7 @@ tl::expected encode(const Simple_& mesh, const uv_buf.resize(meshopt_encodeVertexBufferBound(vertex_count, uv_size)); const size_t uv_written = meshopt_encodeVertexBuffer(uv_buf.data(), uv_buf.size(), mesh.uvs.data(), vertex_count, uv_size); if (uv_written == 0) { - return tl::unexpected(EncodeError::UvEncode); + return std::unexpected(EncodeError::UvEncode); } uv_buf.resize(uv_written); } @@ -121,7 +121,7 @@ tl::expected encode(const Simple_& mesh, const reinterpret_cast(mesh.triangles.data()), index_count); if (index_written == 0) { - return tl::unexpected(EncodeError::TriangleEncode); + return std::unexpected(EncodeError::TriangleEncode); } index_buf.resize(index_written); } @@ -133,7 +133,7 @@ tl::expected encode(const Simple_& mesh, const texture_buf = mesh::io::write_texture_to_encoded_buffer(mesh.texture.value(), options.texture_format); } catch (const cv::Exception &e) { LOG_ERROR("Failed while encoding texture {}", e.what()); - return tl::unexpected(EncodeError::TextureEncode); + return std::unexpected(EncodeError::TextureEncode); } } @@ -189,13 +189,13 @@ inline std::ostream &operator<<(std::ostream &os, const DecodeError &err) { template -tl::expected, DecodeError> decode(const Encoded &encoded, const DecodeOptions = {}) { +std::expected, DecodeError> decode(const Encoded &encoded, const DecodeOptions = {}) { const uint32_t expected_component_type = component_type_id(); const Encoded::Header& header = encoded.header; if (header.version != 1 || header.n_dims != n_dims || header.component_type != expected_component_type) { - return tl::unexpected(DecodeError::IncompatibleData); + return std::unexpected(DecodeError::IncompatibleData); } using Mesh = Simple_; @@ -208,7 +208,7 @@ tl::expected, DecodeError> decode(const Encoded &encoded, con mesh.positions.resize(vertex_count); result = meshopt_decodeVertexBuffer(mesh.positions.data(), vertex_count, position_size, encoded.positions.data(), encoded.positions.size()); if (result != 0) { - return tl::unexpected(DecodeError::PositionDecode); + return std::unexpected(DecodeError::PositionDecode); } // Decode uvs @@ -217,7 +217,7 @@ tl::expected, DecodeError> decode(const Encoded &encoded, con mesh.uvs.resize(vertex_count); result = meshopt_decodeVertexBuffer(mesh.uvs.data(), vertex_count, uv_size, encoded.uvs.data(), encoded.uvs.size()); if (result != 0) { - return tl::unexpected(DecodeError::UvDecode); + return std::unexpected(DecodeError::UvDecode); } } @@ -228,7 +228,7 @@ tl::expected, DecodeError> decode(const Encoded &encoded, con mesh.triangles.resize(face_count); result = meshopt_decodeIndexBuffer(mesh.triangles.data(), index_count, index_size, encoded.triangles.data(), encoded.triangles.size()); if (result != 0) { - return tl::unexpected(DecodeError::TriangleDecode); + return std::unexpected(DecodeError::TriangleDecode); } // Decode texture @@ -237,7 +237,7 @@ tl::expected, DecodeError> decode(const Encoded &encoded, con mesh.texture = mesh::io::read_texture_from_encoded_bytes(encoded.texture); } catch (const cv::Exception &e) { LOG_ERROR("Failed while decoding texture {}", e.what()); - return tl::unexpected(DecodeError::TextureDecode); + return std::unexpected(DecodeError::TextureDecode); } } diff --git a/src/terrainlib/mesh/io.cpp b/src/terrainlib/mesh/io.cpp index 6d56c2c1..467e0537 100644 --- a/src/terrainlib/mesh/io.cpp +++ b/src/terrainlib/mesh/io.cpp @@ -6,7 +6,7 @@ namespace mesh::io { -tl::expected load_from_path( +std::expected load_from_path( const std::filesystem::path &path, const LoadOptions& options) { const std::filesystem::path extension = path.extension(); @@ -15,11 +15,11 @@ tl::expected load_from_path( } else if (extension == ".terrain") { return terrain::load_from_path(path, options); } else { - return tl::unexpected(LoadMeshErrorKind::UnsupportedFormat); + return std::unexpected(LoadMeshErrorKind::UnsupportedFormat); } } -tl::expected save_to_path( +std::expected save_to_path( const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions &options) { @@ -33,7 +33,7 @@ tl::expected save_to_path( } else if (extension == ".terrain") { return terrain::save_to_path(mesh, path, options); } else { - return tl::unexpected(SaveMeshErrorKind::UnsupportedFormat); + return std::unexpected(SaveMeshErrorKind::UnsupportedFormat); } } diff --git a/src/terrainlib/mesh/io.h b/src/terrainlib/mesh/io.h index 81fb076e..ea474c73 100644 --- a/src/terrainlib/mesh/io.h +++ b/src/terrainlib/mesh/io.h @@ -2,7 +2,7 @@ #include -#include +#include #include "mesh/SimpleMesh.h" #include "mesh/io/options.h" @@ -10,11 +10,11 @@ namespace mesh::io { -tl::expected load_from_path( +std::expected load_from_path( const std::filesystem::path &path, const LoadOptions& options = {}); -tl::expected save_to_path( +std::expected save_to_path( const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions& options = {}); diff --git a/src/terrainlib/mesh/io/gltf.cpp b/src/terrainlib/mesh/io/gltf.cpp index bb01bb45..61c0fec1 100644 --- a/src/terrainlib/mesh/io/gltf.cpp +++ b/src/terrainlib/mesh/io/gltf.cpp @@ -212,7 +212,7 @@ std::optional load_texture_from_material(const cgltf_material &material #define GET_OR_INVALID_FORMAT(var, opt) \ do { \ if (!(opt).has_value()) { \ - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); \ + return std::unexpected(LoadMeshErrorKind::InvalidFormat); \ } else { \ var = opt.value(); \ } \ @@ -277,56 +277,56 @@ static std::string image_ext_to_mime(std::string_view extension) { } } -tl::expected load_raw_from_path(const std::filesystem::path &path) { +std::expected load_raw_from_path(const std::filesystem::path &path) { cgltf_options options = {}; cgltf_data *data = NULL; const std::string path_str = path.string(); const char *path_ptr = path_str.c_str(); cgltf_result result = cgltf_parse_file(&options, path_ptr, &data); if (result != cgltf_result::cgltf_result_success) { - return tl::unexpected(result); + return std::unexpected(result); } result = cgltf_load_buffers(&options, data, path_ptr); if (result != cgltf_result::cgltf_result_success) { cgltf_free(data); - return tl::unexpected(result); + return std::unexpected(result); } result = cgltf_validate(data); if (result != cgltf_result_success) { cgltf_free(data); - return tl::unexpected(result); + return std::unexpected(result); } return RawMesh(data, cgltf_free); } -tl::expected load_from_raw(const RawMesh &raw, const LoadOptions& /* options */) { +std::expected load_from_raw(const RawMesh &raw, const LoadOptions& /* options */) { LOG_TRACE("Loading mesh from gltf data"); const cgltf_data &data = *raw; const auto mesh_opt = get_single_element("mesh", data.meshes_count, data.meshes); if (!mesh_opt.has_value()) { - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); + return std::unexpected(LoadMeshErrorKind::InvalidFormat); } const cgltf_mesh &mesh = mesh_opt.value(); const auto mesh_primitive_opt = get_single_element("mesh primitive", mesh.primitives_count, mesh.primitives); if (!mesh_primitive_opt.has_value()) { - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); + return std::unexpected(LoadMeshErrorKind::InvalidFormat); } const cgltf_primitive &mesh_primitive = mesh_primitive_opt.value(); if (mesh_primitive.type != cgltf_primitive_type::cgltf_primitive_type_triangles) { LOG_ERROR("mesh has invalid primitive type"); - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); + return std::unexpected(LoadMeshErrorKind::InvalidFormat); } // indices if (mesh_primitive.indices == nullptr) { LOG_ERROR("mesh primitive has no indices"); - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); + return std::unexpected(LoadMeshErrorKind::InvalidFormat); } cgltf_accessor &index_accessor = *mesh_primitive.indices; std::vector indices; @@ -342,13 +342,13 @@ tl::expected load_from_raw(const RawMesh &raw, const cgltf_attribute *position_attr = find_attribute_with_type(mesh_primitive.attributes, mesh_primitive.attributes_count, cgltf_attribute_type_position); if (position_attr == nullptr) { LOG_ERROR("mesh has no position attribute"); - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); + return std::unexpected(LoadMeshErrorKind::InvalidFormat); } cgltf_accessor &position_accessor = *position_attr->data; if (position_accessor.type != cgltf_type_vec3) { LOG_WARN("mesh positions are not vec3"); - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); + return std::unexpected(LoadMeshErrorKind::InvalidFormat); } std::vector positions; positions.resize(position_accessor.count); @@ -363,7 +363,7 @@ tl::expected load_from_raw(const RawMesh &raw, const cgltf_accessor &uv_accessor = *uv_attr->data; if (uv_accessor.type != cgltf_type_vec2) { LOG_WARN("mesh uvss are not vec2"); - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); + return std::unexpected(LoadMeshErrorKind::InvalidFormat); } uvs.resize(uv_accessor.count); cgltf_accessor_unpack_floats(&uv_accessor, reinterpret_cast(uvs.data()), uvs.size() * 2); @@ -393,7 +393,7 @@ tl::expected load_from_raw(const RawMesh &raw, const } /// Saves the mesh as a .gltf or .glb file at the given path. -tl::expected save_to_path( +std::expected save_to_path( const SimpleMesh &terrain_mesh, const std::filesystem::path &path, const SaveOptions& options) { @@ -731,10 +731,10 @@ tl::expected save_to_path( return {}; } -tl::expected load_from_path(const std::filesystem::path &path, const LoadOptions &options) { - tl::expected raw_mesh = load_raw_from_path(path); +std::expected load_from_path(const std::filesystem::path &path, const LoadOptions &options) { + std::expected raw_mesh = load_raw_from_path(path); if (!raw_mesh) { - return tl::unexpected(map_cgltf_error(raw_mesh.error())); + return std::unexpected(map_cgltf_error(raw_mesh.error())); } return load_from_raw(*raw_mesh, options); } diff --git a/src/terrainlib/mesh/io/gltf.h b/src/terrainlib/mesh/io/gltf.h index 60757382..20333e93 100644 --- a/src/terrainlib/mesh/io/gltf.h +++ b/src/terrainlib/mesh/io/gltf.h @@ -5,7 +5,7 @@ #include #include -#include +#include #include "mesh/SimpleMesh.h" #include "mesh/io/error.h" @@ -15,19 +15,19 @@ namespace mesh::io::gltf { using RawMesh = std::unique_ptr; -tl::expected load_from_path( +std::expected load_from_path( const std::filesystem::path &path, const LoadOptions &options = {}); -tl::expected load_from_raw( +std::expected load_from_raw( const RawMesh &mesh, const LoadOptions &options = {}); -tl::expected save_to_path( +std::expected save_to_path( const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions &options = {}); -// tl::expected save_to_raw(const RawMesh &mesh, const SaveOptions &options = {}); +// std::expected save_to_raw(const RawMesh &mesh, const SaveOptions &options = {}); -tl::expected load_raw_from_path(const std::filesystem::path &path); +std::expected load_raw_from_path(const std::filesystem::path &path); } // namespace mesh::io::gltf diff --git a/src/terrainlib/mesh/io/terrain.cpp b/src/terrainlib/mesh/io/terrain.cpp index 1068a991..d443d6f8 100644 --- a/src/terrainlib/mesh/io/terrain.cpp +++ b/src/terrainlib/mesh/io/terrain.cpp @@ -34,24 +34,24 @@ SaveMeshError save_error_from_io_error(::io::Error error) { } } -tl::expected write_bytes_to_path( +std::expected write_bytes_to_path( const std::span bytes, const std::filesystem::path &path) { const auto result = ::io::write_bytes_to_path(bytes, path); if (!result.has_value()) { - return tl::unexpected(save_error_from_io_error(result.error())); + return std::unexpected(save_error_from_io_error(result.error())); } return {}; } -tl::expected, LoadMeshError> read_bytes_from_path(const std::filesystem::path &path) { +std::expected, LoadMeshError> read_bytes_from_path(const std::filesystem::path &path) { const auto result = ::io::read_bytes_from_path(path); if (!result.has_value()) { - return tl::unexpected(load_error_from_io_error(result.error())); + return std::unexpected(load_error_from_io_error(result.error())); } return result.value(); } -tl::expected, SaveMeshError> save_encoded_to_buffer(const mesh::Encoded &mesh) { +std::expected, SaveMeshError> save_encoded_to_buffer(const mesh::Encoded &mesh) { LOG_TRACE("Serializing mesh to buffer"); // TODO: this ignores the texture format in SaveOptions @@ -66,7 +66,7 @@ tl::expected, SaveMeshError> save_encoded_to_buffer(const m case std::errc::no_buffer_space: case std::errc::message_size: case std::errc::result_out_of_range: - return tl::unexpected(SaveMeshErrorKind::OutOfMemory); + return std::unexpected(SaveMeshErrorKind::OutOfMemory); break; default: UNREACHABLE(); @@ -77,7 +77,7 @@ tl::expected, SaveMeshError> save_encoded_to_buffer(const m return data; } -tl::expected load_encoded_from_buffer(const std::span bytes) { +std::expected load_encoded_from_buffer(const std::span bytes) { LOG_TRACE("Deserializing mesh from buffer"); zpp::bits::in in(bytes); @@ -90,12 +90,12 @@ tl::expected load_encoded_from_buffer(const std::s switch (result) { case std::errc::no_buffer_space: case std::errc::message_size: - return tl::unexpected(LoadMeshErrorKind::OutOfMemory); + return std::unexpected(LoadMeshErrorKind::OutOfMemory); case std::errc::value_too_large: case std::errc::bad_message: case std::errc::protocol_error: case std::errc::result_out_of_range: - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); + return std::unexpected(LoadMeshErrorKind::InvalidFormat); case std::errc::not_supported: case std::errc::invalid_argument: UNREACHABLE(); @@ -109,61 +109,61 @@ tl::expected load_encoded_from_buffer(const std::s } } -tl::expected, SaveMeshError> save_to_buffer(const SimpleMesh &mesh, const SaveOptions& options) { +std::expected, SaveMeshError> save_to_buffer(const SimpleMesh &mesh, const SaveOptions& options) { const auto encode_result = mesh::encode(mesh, mesh::EncodeOptions{ .texture_format = options.texture_format}); if (!encode_result.has_value()) { - return tl::unexpected(SaveMeshErrorKind::UnsupportedFormat); + return std::unexpected(SaveMeshErrorKind::UnsupportedFormat); } const mesh::Encoded encoded = encode_result.value(); const auto deser_result = save_encoded_to_buffer(encoded); if (!deser_result.has_value()) { - return tl::unexpected(deser_result.error()); + return std::unexpected(deser_result.error()); } const std::vector buffer = deser_result.value(); return buffer; } -tl::expected load_from_buffer(const std::span bytes, const LoadOptions & /* options */) { +std::expected load_from_buffer(const std::span bytes, const LoadOptions & /* options */) { const auto deser_result = load_encoded_from_buffer(bytes); if (!deser_result.has_value()) { - return tl::unexpected(deser_result.error()); + return std::unexpected(deser_result.error()); } const mesh::Encoded encoded = deser_result.value(); const auto decode_result = mesh::decode(encoded); if (!decode_result.has_value()) { - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); + return std::unexpected(LoadMeshErrorKind::InvalidFormat); } const mesh::Simple mesh = decode_result.value(); return mesh; } -tl::expected load_from_path(const std::filesystem::path &path, const LoadOptions & /* options */) { +std::expected load_from_path(const std::filesystem::path &path, const LoadOptions & /* options */) { const auto bytes_result = read_bytes_from_path(path); if (!bytes_result.has_value()) { - return tl::unexpected(bytes_result.error()); + return std::unexpected(bytes_result.error()); } const std::vector bytes = bytes_result.value(); return load_from_buffer(bytes); } -tl::expected save_to_path(const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions& options) { +std::expected save_to_path(const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions& options) { LOG_TRACE("Saving mesh as high precision terrain"); const auto result = save_to_buffer(mesh, options); if (!result.has_value()) { - return tl::unexpected(result.error()); + return std::unexpected(result.error()); } const std::vector bytes = result.value(); const auto write_result = write_bytes_to_path(bytes, path); if (!write_result.has_value()) { - return tl::unexpected(write_result.error()); + return std::unexpected(write_result.error()); } return {}; diff --git a/src/terrainlib/mesh/io/terrain.h b/src/terrainlib/mesh/io/terrain.h index 9c505c14..fd097a24 100644 --- a/src/terrainlib/mesh/io/terrain.h +++ b/src/terrainlib/mesh/io/terrain.h @@ -3,7 +3,7 @@ #include #include -#include +#include #include "mesh/SimpleMesh.h" #include "mesh/io/error.h" @@ -11,10 +11,10 @@ namespace mesh::io::terrain { -tl::expected load_from_path(const std::filesystem::path &path, const LoadOptions &options = {}); -tl::expected load_from_buffer(const std::span buffer, const LoadOptions &options = {}); +std::expected load_from_path(const std::filesystem::path &path, const LoadOptions &options = {}); +std::expected load_from_buffer(const std::span buffer, const LoadOptions &options = {}); -tl::expected save_to_path(const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions &options = {}); -tl::expected, SaveMeshError> save_to_buffer(const SimpleMesh &mesh, const SaveOptions &options = {}); +std::expected save_to_path(const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions &options = {}); +std::expected, SaveMeshError> save_to_buffer(const SimpleMesh &mesh, const SaveOptions &options = {}); } // namespace mesh::io::terrain diff --git a/src/terrainlib/octree/storage/IndexedStorage.h b/src/terrainlib/octree/storage/IndexedStorage.h index 70076cd3..3ba94e8f 100644 --- a/src/terrainlib/octree/storage/IndexedStorage.h +++ b/src/terrainlib/octree/storage/IndexedStorage.h @@ -45,7 +45,7 @@ class IndexedStorage_ : public Storage_ { void update_index() noexcept { Storage_::update_index(); } - tl::expected save_index() const noexcept { + std::expected save_index() const noexcept { if (!this->is_index_dirty()) { return {}; } diff --git a/src/terrainlib/octree/storage/RawStorage.h b/src/terrainlib/octree/storage/RawStorage.h index 679efb38..39482250 100644 --- a/src/terrainlib/octree/storage/RawStorage.h +++ b/src/terrainlib/octree/storage/RawStorage.h @@ -3,7 +3,7 @@ #include #include -#include +#include #include "octree/Id.h" #include "octree/disk/Layout.h" @@ -29,23 +29,23 @@ class RawStorage_ { RawStorage_(RawStorage_ &&) = default; RawStorage_ &operator=(RawStorage_ &&) = default; - tl::expected load(const Id &id) const noexcept { + std::expected load(const Id &id) const noexcept { const auto path = this->path_for(id); return Codec::load_from_path(path); } - tl::expected save(const Id &id, const T &node) const noexcept { + std::expected save(const Id &id, const T &node) const noexcept { const auto path = this->path_for(id); return Codec::save_to_path(node, path); } - tl::expected copy_to(const Id &id, RawStorage_ &target) const noexcept { + std::expected copy_to(const Id &id, RawStorage_ &target) const noexcept { return target.copy_from(id, *this); } - tl::expected copy_from(const Id &id, const RawStorage_ &source) noexcept { + std::expected copy_from(const Id &id, const RawStorage_ &source) noexcept { if (!source.has(id)) { - return tl::unexpected(CopyErrorKind::FileNotFound); + return std::unexpected(CopyErrorKind::FileNotFound); } const auto source_path = source.path_for(id); @@ -54,13 +54,13 @@ class RawStorage_ { // TODO: should this error instead? const auto load_result = source.load(id); if (!load_result.has_value()) { - return tl::unexpected(CopyErrorKind::Read); + return std::unexpected(CopyErrorKind::Read); } const value_type node = load_result.value(); const auto save_result = this->save(id, node); if (!save_result.has_value()) { - return tl::unexpected(CopyErrorKind::Write); + return std::unexpected(CopyErrorKind::Write); } return {}; } @@ -68,18 +68,18 @@ class RawStorage_ { std::error_code ec; if (std::filesystem::remove(target_path, ec)) { if (ec) { - return tl::unexpected(CopyErrorKind::RemoveOld); + return std::unexpected(CopyErrorKind::RemoveOld); } } std::filesystem::create_directories(target_path.parent_path(), ec); if (ec) { - return tl::unexpected(CopyErrorKind::CreateDirectories); + return std::unexpected(CopyErrorKind::CreateDirectories); } std::filesystem::create_hard_link(source_path, target_path, ec); if (ec) { - return tl::unexpected(CopyErrorKind::CreateLink); + return std::unexpected(CopyErrorKind::CreateLink); } return {}; diff --git a/src/terrainlib/octree/storage/Storage.h b/src/terrainlib/octree/storage/Storage.h index 7fbfc9eb..782a9c03 100644 --- a/src/terrainlib/octree/storage/Storage.h +++ b/src/terrainlib/octree/storage/Storage.h @@ -7,7 +7,7 @@ #include #include -#include +#include #include "mesh/io.h" #include "octree/Id.h" @@ -143,13 +143,13 @@ class Storage_ { } } - tl::expected load(const Id &id) const noexcept { + std::expected load(const Id &id) const noexcept { if (const auto value_opt = this->_cache.get(id)) { return value_opt.value(); } if (!this->_index.contains(id, true)) { - return tl::unexpected(Codec::file_not_found()); + return std::unexpected(Codec::file_not_found()); } const auto result = this->_inner.load(id); @@ -159,7 +159,7 @@ class Storage_ { return result; } - tl::expected save(const Id &id, const value_type &value) noexcept { + std::expected save(const Id &id, const value_type &value) noexcept { if (this->check_overwrite(id)) { LOG_ERROR_AND_EXIT("tried to overwrite value when not allowed"); } @@ -172,9 +172,9 @@ class Storage_ { return result; } - tl::expected copy_from(const Id &id, const Storage_ &source) noexcept { + std::expected copy_from(const Id &id, const Storage_ &source) noexcept { if (!source._index.contains(id, true)) { - return tl::unexpected(CopyErrorKind::FileNotFound); + return std::unexpected(CopyErrorKind::FileNotFound); } if (this->check_overwrite(id)) { @@ -189,7 +189,7 @@ class Storage_ { return result; } - tl::expected copy_to(const Id &id, Storage_ &target) const noexcept { + std::expected copy_to(const Id &id, Storage_ &target) const noexcept { return target.copy_from(id, *this); } @@ -252,7 +252,7 @@ class Storage_ { } } - tl::expected save_or_create_index() noexcept { + std::expected save_or_create_index() noexcept { if (this->is_indexed() && !this->_index.dirty) { return {}; } diff --git a/src/terrainlib/octree/storage/codec/Codec.h b/src/terrainlib/octree/storage/codec/Codec.h index b4c3bc47..f31b6f45 100644 --- a/src/terrainlib/octree/storage/codec/Codec.h +++ b/src/terrainlib/octree/storage/codec/Codec.h @@ -4,7 +4,7 @@ #include #include -#include +#include template concept CodecFor = @@ -16,10 +16,10 @@ concept CodecFor = requires std::same_as; { Codec::load_from_path(path) } noexcept - -> std::same_as>; + -> std::same_as>; { Codec::save_to_path(value, path) } noexcept - -> std::same_as>; + -> std::same_as>; { Codec::file_not_found() } noexcept -> std::same_as; diff --git a/src/terrainlib/octree/storage/codec/DefaultCodec.h b/src/terrainlib/octree/storage/codec/DefaultCodec.h index dd70a27b..dc4acd5a 100644 --- a/src/terrainlib/octree/storage/codec/DefaultCodec.h +++ b/src/terrainlib/octree/storage/codec/DefaultCodec.h @@ -4,7 +4,7 @@ #include #include -#include +#include #include "Codec.h" #include "io/serialize.h" @@ -18,11 +18,11 @@ struct ZppBitsCodec { using load_error = io::Error; using save_error = io::Error; - static tl::expected load_from_path(const std::filesystem::path& path) noexcept { + static std::expected load_from_path(const std::filesystem::path& path) noexcept { return io::read_from_path(path); } - static tl::expected save_to_path(const value_type& value, const std::filesystem::path& path) noexcept { + static std::expected save_to_path(const value_type& value, const std::filesystem::path& path) noexcept { return io::write_to_path(value, path); } diff --git a/src/terrainlib/octree/storage/codec/MeshCodec.h b/src/terrainlib/octree/storage/codec/MeshCodec.h index 2dc65fb6..7a15400a 100644 --- a/src/terrainlib/octree/storage/codec/MeshCodec.h +++ b/src/terrainlib/octree/storage/codec/MeshCodec.h @@ -4,7 +4,7 @@ #include #include -#include +#include #include "mesh/io.h" @@ -15,11 +15,11 @@ struct MeshCodec { using load_error = mesh::io::LoadMeshError; using save_error = mesh::io::SaveMeshError; - static tl::expected load_from_path(const std::filesystem::path& path) noexcept { + static std::expected load_from_path(const std::filesystem::path& path) noexcept { return mesh::io::load_from_path(path); } - static tl::expected save_to_path(const value_type& value, const std::filesystem::path& path) noexcept { + static std::expected save_to_path(const value_type& value, const std::filesystem::path& path) noexcept { return mesh::io::save_to_path(value, path); } diff --git a/src/terrainlib/octree/storage/helpers.cpp b/src/terrainlib/octree/storage/helpers.cpp index 7366be55..ed817515 100644 --- a/src/terrainlib/octree/storage/helpers.cpp +++ b/src/terrainlib/octree/storage/helpers.cpp @@ -5,7 +5,7 @@ #include #include -#include +#include #include "octree/storage/helpers.h" #include "io/serialize.h" @@ -90,7 +90,7 @@ std::optional guess_layout_strategy( return std::nullopt; } -tl::expected save_index_map(const IndexMap& index, const disk::Layout& layout) { +std::expected save_index_map(const IndexMap& index, const disk::Layout& layout) { const auto index_path = layout.base_path() / disk::v1::index_file_name(); LOG_TRACE("Saving octree storage index to {}", index_path); @@ -102,7 +102,7 @@ tl::expected save_index_map(const IndexMap& index, const disk:: const auto result = io::write_to_path(index_file, index_path); if (!result.has_value()) { LOG_ERROR("Failed to save octree storage index to {}", index_path); - return tl::unexpected(result.error()); + return std::unexpected(result.error()); } return {}; } diff --git a/src/terrainlib/octree/storage/helpers.h b/src/terrainlib/octree/storage/helpers.h index a7dff810..288096cf 100644 --- a/src/terrainlib/octree/storage/helpers.h +++ b/src/terrainlib/octree/storage/helpers.h @@ -4,7 +4,7 @@ #include #include -#include +#include #include "io/Error.h" #include "octree/IndexMap.h" @@ -22,7 +22,7 @@ struct LayoutWithoutBase { std::optional guess_layout_strategy( const std::filesystem::path &base_path, size_t max_files_to_check = 100); -tl::expected save_index_map(const IndexMap &index, const disk::Layout &layout); +std::expected save_index_map(const IndexMap &index, const disk::Layout &layout); void update_index_map(IndexMap &index, const disk::Layout &layout); } diff --git a/src/terrainlib/octree/storage/open.h b/src/terrainlib/octree/storage/open.h index ec212e88..b21ede08 100644 --- a/src/terrainlib/octree/storage/open.h +++ b/src/terrainlib/octree/storage/open.h @@ -4,7 +4,7 @@ #include #include -#include +#include #include "io/Error.h" #include "octree/disk/layout/strategy/Default.h" @@ -21,7 +21,7 @@ struct OpenOptions { }; template Codec = DefaultCodecFor> -tl::expected, io::Error> open_index(const std::filesystem::path &index_path); +std::expected, io::Error> open_index(const std::filesystem::path &index_path); template Codec = DefaultCodecFor> Storage_ open_folder( const std::filesystem::path &base_path, diff --git a/src/terrainlib/octree/storage/open.inl b/src/terrainlib/octree/storage/open.inl index a7cfdb99..a948f4ac 100644 --- a/src/terrainlib/octree/storage/open.inl +++ b/src/terrainlib/octree/storage/open.inl @@ -4,7 +4,7 @@ #include #include -#include +#include #include "io/serialize.h" #include "log.h" @@ -19,13 +19,13 @@ namespace octree { template Codec> -tl::expected, io::Error> open_index(const std::filesystem::path &index_path) { +std::expected, io::Error> open_index(const std::filesystem::path &index_path) { LOG_TRACE("Opening storage index {}", index_path); const auto result = io::read_from_path(index_path); if (!result.has_value()) { LOG_TRACE("Failed to open storage index due to {}", result.error()); - return tl::unexpected(result.error()); + return std::unexpected(result.error()); } auto index_file = result.value(); LOG_TRACE("Successfully read storage index with {} entries.", index_file.map.size()); diff --git a/src/terrainlib/pch.h b/src/terrainlib/pch.h index dce9cf57..dfbd471c 100644 --- a/src/terrainlib/pch.h +++ b/src/terrainlib/pch.h @@ -34,7 +34,7 @@ #include #include #include -#include +#include #include // Internal headers diff --git a/src/terrainlib/srs.h b/src/terrainlib/srs.h index af8308f5..a5e5ce5c 100644 --- a/src/terrainlib/srs.h +++ b/src/terrainlib/srs.h @@ -32,7 +32,7 @@ #include #include #include -#include +#include #include namespace srs { @@ -352,19 +352,19 @@ inline radix::geometry::Aabb3d encompassing_bounds_transfer( return encompassing_bounds_transfer(transform.get(), source_bounds, intermediate_points_edges, intermediate_points_faces); } -inline tl::expected from_epsg(const uint32_t epsg) { +inline std::expected from_epsg(const uint32_t epsg) { OGRSpatialReference srs; if (srs.importFromEPSG(epsg) != OGRERR_NONE) { - return tl::unexpected(fmt::format("Failed to import spatial reference from EPSG code: {}", epsg)); + return std::unexpected(fmt::format("Failed to import spatial reference from EPSG code: {}", epsg)); } srs.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); return srs; } -inline tl::expected from_user_input(const std::string &user_input) { +inline std::expected from_user_input(const std::string &user_input) { OGRSpatialReference srs; if (srs.SetFromUserInput(user_input.c_str()) != OGRERR_NONE) { - return tl::unexpected(fmt::format("Failed to set spatial reference from user input: {}", user_input)); + return std::unexpected(fmt::format("Failed to set spatial reference from user input: {}", user_input)); } srs.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); return srs; diff --git a/src/terrainlib/uv/unwrap.cpp b/src/terrainlib/uv/unwrap.cpp index 5e61092b..206eb285 100644 --- a/src/terrainlib/uv/unwrap.cpp +++ b/src/terrainlib/uv/unwrap.cpp @@ -124,7 +124,7 @@ void clamp_uv_map(UvMap &map, size_t vertex_count) { } } -tl::expected parameterize_mesh(cgal::Mesh &mesh, Algorithm algorithm, Border border) { +std::expected parameterize_mesh(cgal::Mesh &mesh, Algorithm algorithm, Border border) { const cgal::HalfedgeDescriptor bhd = CGAL::Polygon_mesh_processing::longest_border(mesh).first; DEBUG_ASSERT(bhd != boost::graph_traits::null_halfedge()); @@ -173,7 +173,7 @@ tl::expected parameterize_mesh(cgal::Mesh &mesh, Algorit } if (result != CGAL::Surface_mesh_parameterization::OK) { - return tl::unexpected(UnwrapError(result)); + return std::unexpected(UnwrapError(result)); } const auto vertex_count = CGAL::num_vertices(mesh); @@ -201,7 +201,7 @@ std::vector decode_uv_map(const UvMap &map, size_t vertex_count) { } } -tl::expected unwrap( +std::expected unwrap( const std::span triangles, const std::span positions, Algorithm algorithm, @@ -213,7 +213,7 @@ tl::expected unwrap( cgal::Mesh cgal_mesh = convert::to_cgal_mesh(mesh); auto result = parameterize_mesh(cgal_mesh, algorithm, border); if (!result) { - return tl::unexpected(result.error()); + return std::unexpected(result.error()); } const CgalUvMap cgal_uv_map = result.value(); const Map uv_map = decode_uv_map(cgal_uv_map, mesh.vertex_count()); diff --git a/src/terrainlib/uv/unwrap.h b/src/terrainlib/uv/unwrap.h index 53e0fc3a..8a9a564d 100644 --- a/src/terrainlib/uv/unwrap.h +++ b/src/terrainlib/uv/unwrap.h @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include @@ -51,13 +51,13 @@ using Texture = cv::Mat; inline constexpr Algorithm DEFAULT_ALGORITHM = Algorithm::TutteBarycentricMapping; inline constexpr Border DEFAULT_BORDER = Border::Circle; -tl::expected unwrap( +std::expected unwrap( const std::span triangles, const std::span positions, Algorithm algorithm = DEFAULT_ALGORITHM, Border border = DEFAULT_BORDER); -inline tl::expected unwrap( +inline std::expected unwrap( const std::vector& triangles, const std::vector& positions, Algorithm algorithm = DEFAULT_ALGORITHM, @@ -69,7 +69,7 @@ inline tl::expected unwrap( border); } -inline tl::expected unwrap( +inline std::expected unwrap( const mesh::View &mesh, Algorithm algorithm = DEFAULT_ALGORITHM, Border border = DEFAULT_BORDER) { @@ -80,7 +80,7 @@ inline tl::expected unwrap( border); } -inline tl::expected unwrap( +inline std::expected unwrap( const mesh::Simple &mesh, Algorithm algorithm = DEFAULT_ALGORITHM, Border border = DEFAULT_BORDER) { diff --git a/src/tile_builder/CMakeLists.txt b/src/tile_builder/CMakeLists.txt index d4464342..576df41c 100644 --- a/src/tile_builder/CMakeLists.txt +++ b/src/tile_builder/CMakeLists.txt @@ -1,7 +1,7 @@ add_library(tilebuilderlib alpine_raster.cpp DatasetReader.cpp - Image.cpp + image_writer.cpp ParallelTileGenerator.cpp ParallelTiler.cpp TileHeightsGenerator.cpp diff --git a/src/tile_builder/DatasetReader.cpp b/src/tile_builder/DatasetReader.cpp index bad96340..a472ae54 100644 --- a/src/tile_builder/DatasetReader.cpp +++ b/src/tile_builder/DatasetReader.cpp @@ -29,7 +29,7 @@ #include "Dataset.h" #include "Exception.h" -#include "Image.h" +#include #include "ctb/types.hpp" #include "log.h" @@ -185,12 +185,12 @@ DatasetReader::DatasetReader(const std::shared_ptr& dataset, const OGRS throw Exception(fmt::format("Dataset does not contain band number {} (there are {} bands).", band, dataset->n_bands())); } -HeightData DatasetReader::read(const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const +radix::Raster DatasetReader::read(const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const { return readFrom(m_dataset, bounds, width, height); } -HeightData DatasetReader::readWithOverviews(const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const +radix::Raster DatasetReader::readWithOverviews(const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const { #ifdef ALP_ENABLE_OVERVIEW_READING auto transformer_args = make_image_transform_args(*this, m_dataset.get(), bounds, width, height); @@ -202,7 +202,7 @@ HeightData DatasetReader::readWithOverviews(const radix::tile::SrsBounds& bounds #endif } -HeightData DatasetReader::readFrom(const std::shared_ptr& source_dataset, const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const +radix::Raster DatasetReader::readFrom(const std::shared_ptr& source_dataset, const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const { // if we have performance problems with the warping, it'd still be possible to approximate the warping operation with a linear transform (mostly when zoomed in / on higher zoom levels). // CTB does this in GDALTiler.cpp around line 375 ("// Decide if we are doing an approximate or exact transformation"). @@ -212,9 +212,9 @@ HeightData DatasetReader::readFrom(const std::shared_ptr& source_datase auto warped_dataset = Dataset(static_cast(GDALCreateWarpedVRT(source_dataset->gdalDataset(), int(width), int(height), adfGeoTransform.data(), warp_options.first.get()))); auto* heights_band = warped_dataset.gdalDataset()->GetRasterBand(1); // non-owning pointer - auto heights_data = HeightData(width, height); + auto heights_data = radix::Raster({ width, height }); if (heights_band->RasterIO(GF_Read, 0, 0, int(width), int(height), - static_cast(heights_data.data()), int(width), int(height), GDT_Float32, 0, 0) + static_cast(heights_data.buffer().data()), int(width), int(height), GDT_Float32, 0, 0) != CE_None) throw Exception("couldn't read data"); diff --git a/src/tile_builder/DatasetReader.h b/src/tile_builder/DatasetReader.h index e75b1ff8..a60400c6 100644 --- a/src/tile_builder/DatasetReader.h +++ b/src/tile_builder/DatasetReader.h @@ -23,7 +23,7 @@ #include #include -#include "Image.h" +#include #include class Dataset; @@ -33,8 +33,8 @@ class DatasetReader { public: DatasetReader(const std::shared_ptr& dataset, const OGRSpatialReference& targetSRS, unsigned band, bool warn_on_missing_overviews = true); - HeightData read(const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const; - HeightData readWithOverviews(const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const; + radix::Raster read(const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const; + radix::Raster readWithOverviews(const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const; unsigned dataset_band() const { return m_band; } bool isReprojecting() const { return m_requires_reprojection; } @@ -42,7 +42,7 @@ class DatasetReader { std::string target_srs_wkt() const { return m_target_srs_wkt; } protected: - HeightData readFrom(const std::shared_ptr& dataset, const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const; + radix::Raster readFrom(const std::shared_ptr& dataset, const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const; private: std::shared_ptr m_dataset; diff --git a/src/tile_builder/Image.cpp b/src/tile_builder/Image.cpp deleted file mode 100644 index 3b33465b..00000000 --- a/src/tile_builder/Image.cpp +++ /dev/null @@ -1,20 +0,0 @@ -#include -#include "Image.h" -#include - -void image::saveImageAsPng(const Image& inputImage, const std::string& path) -{ - const int width = static_cast(inputImage.width()); - const int height = static_cast(inputImage.height()); - - cv::Mat image(height, width, CV_8UC3); - - for (int row = 0; row < height; ++row) { - for (int col = 0; col < width; ++col) { - const glm::u8vec3& pixel = inputImage.pixel(height - row - 1, col); // Flip vertically - image.at(row, col) = cv::Vec3b(pixel.z, pixel.y, pixel.x); // RGB to BGR - } - } - - cv::imwrite(path, image); -} diff --git a/src/tile_builder/Image.h b/src/tile_builder/Image.h deleted file mode 100644 index 5785329d..00000000 --- a/src/tile_builder/Image.h +++ /dev/null @@ -1,105 +0,0 @@ -/***************************************************************************** - * Alpine Terrain Builder - * Copyright (C) 2022 alpinemaps.org - * Copyright (C) 2022 Adam Celarek - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *****************************************************************************/ - -#ifndef IMAGE_H -#define IMAGE_H - -#include -#include -#include -#include -#include -#include - -#include - -namespace tntn { -template -class Raster; -} - -template -class Image { -public: - Image() = default; - Image(unsigned width, unsigned height) - : m_width(width) - , m_height(height) - , m_data(size_t(m_width * m_height)) - { - } - - [[nodiscard]] unsigned width() const { return m_width; } - [[nodiscard]] unsigned height() const { return m_height; } - [[nodiscard]] T pixel(unsigned row, unsigned column) const - { - assert(column < m_width); - assert(row < m_height); - assert(m_data.size() == size_t(m_width * m_height)); - return m_data[row * m_width + column]; - } - - [[nodiscard]] float* data() { return m_data.data(); } - - [[nodiscard]] auto size() const { return m_data.size(); } - [[nodiscard]] auto begin() { return m_data.begin(); } - [[nodiscard]] auto end() { return m_data.end(); } - [[nodiscard]] auto begin() const { return m_data.begin(); } - [[nodiscard]] auto end() const { return m_data.end(); } - -private: - unsigned m_width = 0; - unsigned m_height = 0; - std::vector m_data; - - friend class tntn::Raster; -}; - -using HeightData = Image; -using RgbImage = Image; -using uchar = unsigned char; - -namespace image { -void saveImageAsPng(const Image& image, const std::string& path); - -template -[[nodiscard]] auto transformImage(const Image& i, Fun conversion_fun) -> Image -{ - using T2 = decltype(conversion_fun(*i.begin())); - Image i2(i.width(), i.height()); - std::transform(i.begin(), i.end(), i2.begin(), conversion_fun); - return i2; -} - -template -void debugOut(const Image& image, const std::string& path) -{ - auto [min, max] = std::ranges::minmax(image); - - // [min=min, ..] is required for cpp correctness. min/max from the capture are not variables, we need to copy them: - // https://stackoverflow.com/questions/50799719/reference-to-local-binding-declared-in-enclosing-function?noredirect=1&lq=1 - saveImageAsPng(transformImage(image, [min = min, max = max](auto v) { - const auto c = uchar(255.F * (float(v) - float(min)) / float(max - min)); - return glm::u8vec3(c, c, c); - }), - path); -} -} - -#endif // HEIGHTDATA_H diff --git a/src/tile_builder/ParallelTileGenerator.cpp b/src/tile_builder/ParallelTileGenerator.cpp index e0d7e731..70924640 100644 --- a/src/tile_builder/ParallelTileGenerator.cpp +++ b/src/tile_builder/ParallelTileGenerator.cpp @@ -64,7 +64,7 @@ const ctb::Grid& ParallelTileGenerator::grid() const return m_grid; } -void ParallelTileGenerator::write(const radix::tile::Descriptor& tile, const HeightData& heights) const +void ParallelTileGenerator::write(const radix::tile::Descriptor& tile, const radix::Raster& heights) const { const auto dir_path = fmt::format("{}/{}/{}", m_output_data_path, tile.id.zoom_level, tile.id.coords.x); const auto file_path = fmt::format("{}/{}.{}", dir_path, tile.id.coords.y, m_tile_writer->formatFileEnding()); diff --git a/src/tile_builder/ParallelTileGenerator.h b/src/tile_builder/ParallelTileGenerator.h index f95a412f..f9fa61e0 100644 --- a/src/tile_builder/ParallelTileGenerator.h +++ b/src/tile_builder/ParallelTileGenerator.h @@ -22,7 +22,7 @@ #include #include -#include "Image.h" +#include #include "ParallelTiler.h" #include "ctb/Grid.hpp" #include "ctb/types.hpp" @@ -49,7 +49,7 @@ class ParallelTileGenerator { void setWarnOnMissingOverviews(bool flag) { m_warn_on_missing_overviews = flag; } [[nodiscard]] const ParallelTiler& tiler() const; [[nodiscard]] const ctb::Grid& grid() const; - void write(const radix::tile::Descriptor& tile, const HeightData& heights) const; + void write(const radix::tile::Descriptor& tile, const radix::Raster& heights) const; void process(const std::pair& zoom_range, bool progress_bar_on_console = false, bool generate_world_wide_tiles = false) const; }; @@ -68,7 +68,7 @@ class ParallelTileWriterInterface { virtual ~ParallelTileWriterInterface() = default; ParallelTileWriterInterface& operator=(const ParallelTileWriterInterface&) = default; ParallelTileWriterInterface& operator=(ParallelTileWriterInterface&&) = default; - virtual void write(const std::string& file_path, const radix::tile::Descriptor& tile, const HeightData& heights) const = 0; + virtual void write(const std::string& file_path, const radix::tile::Descriptor& tile, const radix::Raster& heights) const = 0; [[nodiscard]] radix::tile::Border formatRequiresBorder() const; [[nodiscard]] const std::string& formatFileEnding() const; }; diff --git a/src/tile_builder/algorithms/primitives.h b/src/tile_builder/algorithms/primitives.h deleted file mode 100644 index a6101364..00000000 --- a/src/tile_builder/algorithms/primitives.h +++ /dev/null @@ -1,103 +0,0 @@ -#include -#include -#include - -#ifndef ALGORITHMS_PRIMITIVES_H -#define ALGORITHMS_PRIMITIVES_H - -namespace primitives { - -// two times the area of the ccw triangle with vertices a, b, and c. -// negative, if the triangle is cw -template -inline T triAreaX2(const glm::tvec2& a, const glm::tvec2& b, const glm::tvec2& c) -{ - // doesn't work for unsigned types. if you need that, come up with something :) - static_assert(std::is_signed_v); - return (b.x - a.x) * (c.y - a.y) - (b.y - a.y) * (c.x - a.x); -} - -enum class Winding : char { - CW = -1, - Undefined = 0, - CCW = 1 -}; - -template -inline Winding winding(const glm::tvec2& a, const glm::tvec2& b, const glm::tvec2& c) -{ - constexpr bool is_integral = std::is_integral_v; - if constexpr (is_integral) { - using Signed = typename std::conditional::type, T>::type; - auto v = (Signed(b.x) - Signed(a.x)) * (Signed(c.y) - Signed(a.y)) - (Signed(b.y) - Signed(a.y)) * (Signed(c.x) - Signed(a.x)); - if (v == 0) - return Winding::Undefined; - if (v < 0) - return Winding::CW; - return Winding::CCW; - } else { - auto v = (b.x - a.x) * (c.y - a.y) - (b.y - a.y) * (c.x - a.x); - if (std::abs(v) < 0.0000000000000001) - return Winding::Undefined; - if (v < 0) - return Winding::CW; - return Winding::CCW; - } -} - -template -inline bool ccw(const glm::tvec2& a, const glm::tvec2& b, const glm::tvec2& c) -{ - constexpr bool is_integral = std::is_integral_v; - if constexpr (is_integral) { - using Signed = typename std::conditional::type, T>::type; - if constexpr (include_border) - return (Signed(b.x) - Signed(a.x)) * (Signed(c.y) - Signed(a.y)) >= (Signed(b.y) - Signed(a.y)) * (Signed(c.x) - Signed(a.x)); - return (Signed(b.x) - Signed(a.x)) * (Signed(c.y) - Signed(a.y)) > (Signed(b.y) - Signed(a.y)) * (Signed(c.x) - Signed(a.x)); - } - if constexpr (include_border) - return (b.x - a.x) * (c.y - a.y) >= (b.y - a.y) * (c.x - a.x); - return (b.x - a.x) * (c.y - a.y) > (b.y - a.y) * (c.x - a.x); -} - -template -inline bool rightOf(const glm::tvec2& x, const glm::tvec2& org, const glm::tvec2& dest) -{ - return ccw(x, dest, org); -} - -template -inline bool leftOf(const glm::tvec2& x, const glm::tvec2& org, const glm::tvec2& dest) -{ - return ccw(x, org, dest); -} - -// https://www.scratchapixel.com/lessons/3d-basic-rendering/rasterization-practical-implementation/rasterization-stage -// the bottom edge of the raster is not included! -template -inline bool inside(const glm::tvec2& x, const glm::tvec2& a, const glm::tvec2& b, const glm::tvec2& c) -{ - constexpr bool is_integral = std::is_integral_v; - using Signed = typename std::conditional::type, T>::type; - using sVec = glm::tvec2; - // return leftOf(x, a, b) && leftOf(x, b, c) && leftOf(x, c, a); - const auto ccw = [](Winding w) { return w == Winding::CCW; }; - const auto undef = [](Winding w) { return w == Winding::Undefined; }; - const auto topleftedge = [](const sVec& edge) { return (edge.y == 0 && edge.x < 0) || edge.y < 0; }; - - const auto w_ab = winding(x, a, b); - const auto w_bc = winding(x, b, c); - const auto w_ca = winding(x, c, a); - const auto e_ab = sVec(b) - sVec(a); - const auto e_bc = sVec(c) - sVec(b); - const auto e_ca = sVec(a) - sVec(c); - - bool overlap = true; - overlap &= ccw(w_ab) || (undef(w_ab) && topleftedge(e_ab)); - overlap &= ccw(w_bc) || (undef(w_bc) && topleftedge(e_bc)); - overlap &= ccw(w_ca) || (undef(w_ca) && topleftedge(e_ca)); - return overlap; -} -} - -#endif diff --git a/src/tile_builder/algorithms/raster_triangle_scanline.h b/src/tile_builder/algorithms/raster_triangle_scanline.h deleted file mode 100644 index 7e22fdca..00000000 --- a/src/tile_builder/algorithms/raster_triangle_scanline.h +++ /dev/null @@ -1,101 +0,0 @@ -/***************************************************************************** - * Alpine Terrain Builder - * Copyright (C) 2022 alpinemaps.org - * Copyright (C) 2022 Adam Celarek - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *****************************************************************************/ - -#ifndef ALGORITHMS_RASTER_TRIANGLE_SCANLINE_H -#define ALGORITHMS_RASTER_TRIANGLE_SCANLINE_H - -#include "primitives.h" -#include "tntn/Raster.h" -#include -#include -#include - -namespace raster { - -template -void triangle_scanline(const tntn::Raster& raster, const glm::uvec2& a, const glm::uvec2& b, const glm::uvec2& c, const Lambda& fun) -{ - assert(primitives::winding(a, b, c) == primitives::Winding::CCW); - - const auto min = glm::min(glm::min(a, b), c); - const auto max = glm::max(glm::max(a, b), c); - for (auto y = min.y; y <= max.y; ++y) { - for (auto x = min.x; x <= max.x; ++x) { - const auto coord = glm::uvec2(x, y); - if (primitives::inside(coord, a, b, c)) - fun(coord, raster.value(coord.y, coord.x)); // raster is row / column - } - } - if (min.y == 0) { - // bottom row of raster is not included in primitives::inside, so check for it and make an extrawurscht. - const auto walk_bottom = [&](const glm::uvec2& a, const glm::uvec2& b) { - if ((b - a).y == 0 && a.y == 0) { - const auto end_x = std::max(a.x, b.x); - for (auto x = std::min(a.x, b.x); x < end_x; ++x) { - const auto coord = glm::uvec2(x, 0); - fun(coord, raster.value(coord.y, coord.x)); // raster is row / column - } - } - }; - walk_bottom(a, b); - walk_bottom(b, c); - walk_bottom(c, a); - } - const auto last_x = raster.get_width() - 1; - if (max.x == last_x) { - // similar with the rightmost column - const auto walk_right = [&](const glm::uvec2& a, const glm::uvec2& b) { - if ((b - a).x == 0 && a.x == last_x) { - const auto end_y = std::max(a.y, b.y); - for (auto y = std::min(a.y, b.y); y < end_y; ++y) { - const auto coord = glm::uvec2(last_x, y); - fun(coord, raster.value(coord.y, coord.x)); // raster is row / column - } - } - }; - walk_right(a, b); - walk_right(b, c); - walk_right(c, a); - } - // if (max.x == last_x && min.y == 0) { - // const auto check_br = [&](const glm::uvec2& a, const glm::uvec2& b) { - // if ((b - a).y == 0 && a.y == 0 && b.x == last_x) { - // const auto coord = glm::uvec2(last_x, 0); - // fun(coord, raster.value(coord.y, coord.x)); // raster is row / column - // } - // }; - // check_br(a, b); - // check_br(b, c); - // check_br(c, a); - // } - const auto last_y = raster.get_height() - 1; - if (max.x == last_x && max.y == last_y) { - const auto check_tr = [&](const glm::uvec2& a, const glm::uvec2& b) { - if ((b - a).x == 0 && b.y == last_y && b.x == last_x) { - const auto coord = glm::uvec2(last_x, last_y); - fun(coord, raster.value(coord.y, coord.x)); // raster is row / column - } - }; - check_tr(a, b); - check_tr(b, c); - check_tr(c, a); - } -} -} -#endif diff --git a/src/tile_builder/alpine_raster.cpp b/src/tile_builder/alpine_raster.cpp index b9ce5353..be4e0e1d 100644 --- a/src/tile_builder/alpine_raster.cpp +++ b/src/tile_builder/alpine_raster.cpp @@ -25,9 +25,10 @@ #include #include -#include "Image.h" #include "ParallelTileGenerator.h" #include "ctb/Grid.hpp" +#include "image_writer.h" +#include #include ParallelTileGenerator alpine_raster::make_generator(const std::string& input_data_path, const std::string& output_data_path, ctb::Grid::Srs srs, radix::tile::Scheme tiling_scheme, radix::tile::Border border, unsigned grid_resolution) @@ -35,8 +36,7 @@ ParallelTileGenerator alpine_raster::make_generator(const std::string& input_dat return ParallelTileGenerator::make(input_data_path, srs, tiling_scheme, std::make_unique(border), output_data_path, grid_resolution); } -void alpine_raster::TileWriter::write(const std::string& file_path, const radix::tile::Descriptor&, const HeightData& heights) const +void alpine_raster::TileWriter::write(const std::string& file_path, const radix::tile::Descriptor&, const radix::Raster& heights) const { - image::saveImageAsPng(image::transformImage(heights, radix::height_encoding::to_rgb), - file_path); + image::saveImageAsPng(radix::raster::transform(heights, radix::height_encoding::to_rgb), file_path); } diff --git a/src/tile_builder/alpine_raster.h b/src/tile_builder/alpine_raster.h index 966d55fb..22eee044 100644 --- a/src/tile_builder/alpine_raster.h +++ b/src/tile_builder/alpine_raster.h @@ -25,7 +25,7 @@ #include #include -#include "Image.h" +#include #include "ParallelTileGenerator.h" #include #include "ctb/Grid.hpp" @@ -38,7 +38,7 @@ class TileWriter : public ParallelTileWriterInterface { : ParallelTileWriterInterface(border, "png") { } - void write(const std::string& base_path, const radix::tile::Descriptor& tile, const HeightData& heights) const override; + void write(const std::string& base_path, const radix::tile::Descriptor& tile, const radix::Raster& heights) const override; }; [[nodiscard]] ParallelTileGenerator make_generator( const std::string& input_data_path, diff --git a/src/tile_builder/image_writer.cpp b/src/tile_builder/image_writer.cpp new file mode 100644 index 00000000..725cdfae --- /dev/null +++ b/src/tile_builder/image_writer.cpp @@ -0,0 +1,46 @@ +/***************************************************************************** + * Alpine Terrain Builder + * Copyright (C) 2022 alpinemaps.org + * Copyright (C) 2022 Adam Celarek + * Copyright (C) 2025 Martin Braunsperger + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *****************************************************************************/ + +#include "image_writer.h" + +#include +#include + +void image::saveImageAsPng(const radix::Raster& input_image, const std::string& path) +{ + const int width = static_cast(input_image.width()); + const int height = static_cast(input_image.height()); + + cv::Mat image(height, width, CV_8UC3); + + for (int row = 0; row < height; ++row) { + for (int column = 0; column < width; ++column) { + const auto& pixel = input_image.pixel({ static_cast(column), static_cast(height - row - 1) }); + image.at(row, column) = cv::Vec3b(pixel.z, pixel.y, pixel.x); + } + } + + try { + if (!cv::imwrite(path, image)) + throw std::runtime_error("Failed to write PNG image to " + path); + } catch (const cv::Exception& error) { + throw std::runtime_error("Failed to write PNG image to " + path + ": " + error.what()); + } +} diff --git a/src/tile_builder/image_writer.h b/src/tile_builder/image_writer.h new file mode 100644 index 00000000..db8ff8f9 --- /dev/null +++ b/src/tile_builder/image_writer.h @@ -0,0 +1,51 @@ +/***************************************************************************** + * Alpine Terrain Builder + * Copyright (C) 2022 alpinemaps.org + * Copyright (C) 2022 Adam Celarek + * Copyright (C) 2025 Martin Braunsperger + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *****************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include + +#include +#include + +namespace image { + +void saveImageAsPng(const radix::Raster& image, const std::string& path); + +template +void debugOut(const radix::Raster& image, const std::string& path) +{ + if (image.buffer().empty()) + throw std::invalid_argument("Can't write an empty raster to " + path); + + const auto [min, max] = std::ranges::minmax(image); + const auto range = float(max) - float(min); + saveImageAsPng(radix::raster::transform(image, [min, range](const auto value) { + const auto intensity = range == 0.F ? std::uint8_t(0) : std::uint8_t(255.F * (float(value) - float(min)) / range); + return glm::u8vec3(intensity); + }), + path); +} + +} // namespace image diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index cbf59ea8..647863fe 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -66,7 +66,7 @@ if(TARGET tilebuilderlib) tilebuilder/alpine_raster_format.cpp tilebuilder/dataset_reading.cpp tilebuilder/depth_first_tile_traverser.cpp - tilebuilder/image.cpp + tilebuilder/image_writer.cpp tilebuilder/parallel_tile_generator.cpp tilebuilder/parallel_tiler.cpp tilebuilder/tile_heights_generator.cpp diff --git a/unittests/terrainlib/mesh_io.cpp b/unittests/terrainlib/mesh_io.cpp index bac6225b..cb117366 100644 --- a/unittests/terrainlib/mesh_io.cpp +++ b/unittests/terrainlib/mesh_io.cpp @@ -44,14 +44,14 @@ TEST_CASE("transcode roundtrip") { mesh.texture = cv::Mat3b(100, 100); cv::randu(*mesh.texture, cv::Scalar(0, 0, 0), cv::Scalar(256, 256, 256)); - const tl::expected encode_result = + const std::expected encode_result = mesh::encode(mesh, mesh::EncodeOptions{.texture_format = ".png"}); if (!encode_result.has_value()) { FAIL(encode_result.error()); } const mesh::Encoded encoded = encode_result.value(); - const tl::expected decode_result = + const std::expected decode_result = mesh::decode(encoded, mesh::DecodeOptions{}); if (!decode_result.has_value()) { FAIL(decode_result.error()); @@ -93,7 +93,7 @@ TEST_CASE("io roundtrip") { mesh::io::save_to_path(mesh, mesh_path, mesh::io::SaveOptions{.texture_format = ".png"}); CHECK(std::filesystem::exists(mesh_path)); - const tl::expected result = mesh::io::load_from_path(mesh_path); + const std::expected result = mesh::io::load_from_path(mesh_path); if (!result.has_value()) { FAIL(result.error().description()); } @@ -136,7 +136,7 @@ TEST_CASE("io roundtrip high precision") { mesh::io::save_to_path(mesh, mesh_path, mesh::io::SaveOptions{.texture_format = ".png"}); CHECK(std::filesystem::exists(mesh_path)); - const tl::expected result = mesh::io::load_from_path(mesh_path); + const std::expected result = mesh::io::load_from_path(mesh_path); if (!result.has_value()) { FAIL(result.error().description()); } @@ -179,7 +179,7 @@ TEST_CASE("io roundtrip no texture") { mesh::io::save_to_path(mesh, mesh_path); CHECK(std::filesystem::exists(mesh_path)); - const tl::expected result = mesh::io::load_from_path(mesh_path); + const std::expected result = mesh::io::load_from_path(mesh_path); if (!result.has_value()) { FAIL(result.error().description()); } @@ -216,7 +216,7 @@ TEST_CASE("io roundtrip no texture and uvs") { mesh::io::save_to_path(mesh, mesh_path); CHECK(std::filesystem::exists(mesh_path)); - const tl::expected result = mesh::io::load_from_path(mesh_path); + const std::expected result = mesh::io::load_from_path(mesh_path); if (!result.has_value()) { FAIL(result.error().description()); } diff --git a/unittests/tilebuilder/alpine_raster_format.cpp b/unittests/tilebuilder/alpine_raster_format.cpp index 00ec9bf4..5c5cafd4 100644 --- a/unittests/tilebuilder/alpine_raster_format.cpp +++ b/unittests/tilebuilder/alpine_raster_format.cpp @@ -24,7 +24,7 @@ #include #include -#include "Image.h" +#include #include "alpine_raster.h" #include "ctb/Grid.hpp" @@ -42,14 +42,14 @@ TEMPLATE_TEST_CASE("alpine raster format, border ", "", std::true_type, std::fal SECTION("raste write") { const auto generator = alpine_raster::make_generator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", "./unittest_tiles/", ctb::Grid::Srs::SphericalMercator, radix::tile::Scheme::Tms, radix::tile::Border::Yes); - generator.write(radix::tile::Descriptor { {0, glm::uvec2(0, 0)}, {}, int(ctb::Grid::Srs::SphericalMercator), 256, 257 }, HeightData(257, 257)); + generator.write(radix::tile::Descriptor { {0, glm::uvec2(0, 0)}, {}, int(ctb::Grid::Srs::SphericalMercator), 256, 257 }, radix::Raster({ 257, 257 })); CHECK(std::filesystem::exists("./unittest_tiles/0/0/0.png")); - generator.write(radix::tile::Descriptor { {1, glm::uvec2(2, 3)}, {}, int(ctb::Grid::Srs::SphericalMercator), 256, 257 }, HeightData(257, 257)); + generator.write(radix::tile::Descriptor { {1, glm::uvec2(2, 3)}, {}, int(ctb::Grid::Srs::SphericalMercator), 256, 257 }, radix::Raster({ 257, 257 })); CHECK(std::filesystem::exists("./unittest_tiles/1/2/3.png")); // check that a second write doesn't crash - generator.write(radix::tile::Descriptor { {1, glm::uvec2(2, 3)}, {}, int(ctb::Grid::Srs::SphericalMercator), 256, 257 }, HeightData(257, 257)); + generator.write(radix::tile::Descriptor { {1, glm::uvec2(2, 3)}, {}, int(ctb::Grid::Srs::SphericalMercator), 256, 257 }, radix::Raster({ 257, 257 })); CHECK(std::filesystem::exists("./unittest_tiles/1/2/3.png")); // in the best case, we would read back the data and check it. but that's too much work for now. diff --git a/unittests/tilebuilder/dataset_reading.cpp b/unittests/tilebuilder/dataset_reading.cpp index 17c714e9..ba2804db 100644 --- a/unittests/tilebuilder/dataset_reading.cpp +++ b/unittests/tilebuilder/dataset_reading.cpp @@ -27,6 +27,7 @@ #include "Dataset.h" #include "DatasetReader.h" #include "ctb/types.hpp" +#include "image_writer.h" #include "srs.h" using namespace radix; @@ -184,7 +185,7 @@ TEST_CASE("reading") if (ALP_UNITTESTS_DEBUG_IMAGES) { image::debugOut(ref_heights, fmt::format("./heights_{}_{}.png", test_name, dataset_name.substr(s, l))); - auto height_diffs = HeightData(render_width, render_height); + auto height_diffs = radix::Raster({ render_width, render_height }); std::transform(ref_heights.begin(), ref_heights.end(), heights.begin(), height_diffs.begin(), [](auto a, auto b) { return std::abs(a - b); }); const auto path = fmt::format("./diffs_{}_{}.png", test_name, dataset_name.substr(s, l)); image::debugOut(height_diffs, path); @@ -194,7 +195,7 @@ TEST_CASE("reading") const auto t = std::abs(double(a) - double(b)); largest_abs_diff = std::max(t, largest_abs_diff); return t * t; - }) / double(ref_heights.size()); + }) / double(ref_heights.buffer_length()); // fmt::print("{} | {}; mse: {}, largest_abs_diff: {}\n", test_name, dataset_name.substr(s, l), mse, largest_abs_diff); CHECK(largest_abs_diff < double(max_abs_diff)); CHECK(mse < max_mse); @@ -248,7 +249,7 @@ TEST_CASE("reading") image::debugOut(low_res_heights, fmt::format("./low_res_heights.png")); image::debugOut(high_res_heights, fmt::format("./high_res_heights.png")); - auto height_diffs = HeightData(render_width, render_height); + auto height_diffs = radix::Raster({ render_width, render_height }); std::transform(low_res_heights.begin(), low_res_heights.end(), high_res_heights.begin(), height_diffs.begin(), [](auto a, auto b) { return std::abs(a - b); }); image::debugOut(height_diffs, "./diff_low_res_high_res.png"); } @@ -257,7 +258,7 @@ TEST_CASE("reading") const auto t = std::abs(double(a) - double(b)); largest_abs_diff = std::max(t, largest_abs_diff); return t * t; - }) / double(low_res_heights.size()); + }) / double(low_res_heights.buffer_length()); // fmt::print("mse: {}, largest_abs_diff: {}\n", mse, largest_abs_diff); CHECK(largest_abs_diff < double(max_abs_diff)); CHECK(mse < max_mse); @@ -290,7 +291,7 @@ TEST_CASE("reading") image::debugOut(low_res_heights, fmt::format("./ov_with_warping_low_res_heights.png")); image::debugOut(high_res_heights, fmt::format("./ov_with_warping_high_res_heights.png")); - auto height_diffs = HeightData(render_width, render_height); + auto height_diffs = radix::Raster({ render_width, render_height }); std::transform(low_res_heights.begin(), low_res_heights.end(), high_res_heights.begin(), height_diffs.begin(), [](auto a, auto b) { return std::abs(a - b); }); image::debugOut(height_diffs, "./ov_with_warping_diff_low_res_high_res.png"); } @@ -299,7 +300,7 @@ TEST_CASE("reading") const auto t = std::abs(double(a) - double(b)); largest_abs_diff = std::max(t, largest_abs_diff); return t * t; - }) / double(low_res_heights.size()); + }) / double(low_res_heights.buffer_length()); // fmt::print("mse: {}, largest_abs_diff: {}\n", mse, largest_abs_diff); CHECK(largest_abs_diff < double(max_abs_diff)); CHECK(mse < max_mse); @@ -332,7 +333,7 @@ TEST_CASE("reading") image::debugOut(low_res_heights, fmt::format("./lowres_ov_with_warping_low_res_heights.png")); image::debugOut(high_res_heights, fmt::format("./lowres_ov_with_warping_high_res_heights.png")); - auto height_diffs = HeightData(render_width, render_height); + auto height_diffs = radix::Raster({ render_width, render_height }); std::transform(low_res_heights.begin(), low_res_heights.end(), high_res_heights.begin(), height_diffs.begin(), [](auto a, auto b) { return std::abs(a - b); }); image::debugOut(height_diffs, "./lowres_ov_with_warping_diff_low_res_high_res.png"); } @@ -341,7 +342,7 @@ TEST_CASE("reading") const auto t = std::abs(double(a) - double(b)); largest_abs_diff = std::max(t, largest_abs_diff); return t * t; - }) / double(low_res_heights.size()); + }) / double(low_res_heights.buffer_length()); // fmt::print("render w/h: {}/{}, mse: {}, largest_abs_diff: {}\n", render_width, render_height, mse, largest_abs_diff); CHECK(largest_abs_diff < double(max_abs_diff)); CHECK(mse < max_mse); diff --git a/unittests/tilebuilder/image.cpp b/unittests/tilebuilder/image.cpp deleted file mode 100644 index 448aef27..00000000 --- a/unittests/tilebuilder/image.cpp +++ /dev/null @@ -1,66 +0,0 @@ -/***************************************************************************** - * Alpine Terrain Builder - * Copyright (C) 2022 alpinemaps.org - * Copyright (C) 2022 Adam Celarek - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *****************************************************************************/ - -#include -#include -#include -#include "Image.h" - -using Catch::Approx; - -TEST_CASE("image") -{ - SECTION("iteration") - { - HeightData d { 40, 60 }; - REQUIRE(d.size() == 40 * 60); - REQUIRE(d.height() == 60); - REQUIRE(d.width() == 40); - int v = 0; - std::ranges::for_each(d, [&](auto& d) { d = float(v++); }); - - v = 0; - for (unsigned r = 0; r < d.height(); ++r) { - for (unsigned c = 0; c < d.width(); ++c) { - REQUIRE(d.pixel(r, c) == Approx(float(v++))); - } - } - } - - SECTION("conversion") - { - HeightData d { 40, 60 }; - int v_init = 0; - std::ranges::for_each(d, [&](auto& d) { d = float(v_init++); }); - - Image image = image::transformImage(d, [&](auto v) { const auto b = uchar(255.F * v / float(v_init)); return glm::u8vec3(b, b, b); }); - const auto max = float(v_init); - v_init = 0; - for (unsigned r = 0; r < d.height(); ++r) { - for (unsigned c = 0; c < d.width(); ++c) { - const auto t = uchar(float(v_init) * 255.F / max); - REQUIRE(image.pixel(r, c).x == t); - REQUIRE(image.pixel(r, c).y == t); - REQUIRE(image.pixel(r, c).z == t); - v_init++; - } - } -// image::saveImageAsPng(image, "/home/madam/Documents/work/tuw/alpinemaps/tmp/test.png"); - } -} diff --git a/unittests/tilebuilder/image_writer.cpp b/unittests/tilebuilder/image_writer.cpp new file mode 100644 index 00000000..aeddf2f2 --- /dev/null +++ b/unittests/tilebuilder/image_writer.cpp @@ -0,0 +1,86 @@ +/***************************************************************************** + * Alpine Terrain Builder + * Copyright (C) 2022 alpinemaps.org + * Copyright (C) 2022 Adam Celarek + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *****************************************************************************/ + +#include +#include +#include + +#include +#include +#include + +#include "image_writer.h" + +TEST_CASE("tile builder writes radix rasters as PNG images") +{ + const auto output_path = std::filesystem::temp_directory_path() / "alpine_terrain_builder_image_writer_test.png"; + std::filesystem::remove(output_path); + + radix::Raster raster({ 2, 2 }); + raster.pixel({ 0, 0 }) = { 255, 0, 0 }; + raster.pixel({ 1, 0 }) = { 0, 255, 0 }; + raster.pixel({ 0, 1 }) = { 0, 0, 255 }; + raster.pixel({ 1, 1 }) = { 255, 255, 255 }; + + image::saveImageAsPng(raster, output_path.string()); + + const auto decoded = cv::imread(output_path.string(), cv::IMREAD_COLOR); + REQUIRE(decoded.rows == 2); + REQUIRE(decoded.cols == 2); + CHECK(decoded.at(0, 0) == cv::Vec3b(255, 0, 0)); + CHECK(decoded.at(0, 1) == cv::Vec3b(255, 255, 255)); + CHECK(decoded.at(1, 0) == cv::Vec3b(0, 0, 255)); + CHECK(decoded.at(1, 1) == cv::Vec3b(0, 255, 0)); + + std::filesystem::remove(output_path); +} + +TEST_CASE("tile builder handles image writer edge cases") +{ + SECTION("empty debug raster is rejected") + { + CHECK_THROWS_AS(image::debugOut(radix::Raster {}, "empty.png"), std::invalid_argument); + } + + SECTION("constant debug raster is written as black") + { + const auto output_path = std::filesystem::temp_directory_path() / "alpine_terrain_builder_constant_raster_test.png"; + std::filesystem::remove(output_path); + + radix::Raster raster({ 2, 2 }); + std::ranges::fill(raster, 42.F); + image::debugOut(raster, output_path.string()); + + const auto decoded = cv::imread(output_path.string(), cv::IMREAD_COLOR); + REQUIRE(decoded.rows == 2); + REQUIRE(decoded.cols == 2); + CHECK(cv::countNonZero(decoded.reshape(1)) == 0); + + std::filesystem::remove(output_path); + } + + SECTION("PNG write failure is reported") + { + const auto missing_directory = std::filesystem::temp_directory_path() / "alpine_terrain_builder_missing_directory"; + std::filesystem::remove_all(missing_directory); + const auto output_path = missing_directory / "image.png"; + + CHECK_THROWS_AS(image::saveImageAsPng(radix::Raster({ 1, 1 }), output_path.string()), std::runtime_error); + } +} diff --git a/unittests/tilebuilder/parallel_tile_generator.cpp b/unittests/tilebuilder/parallel_tile_generator.cpp index ee57262d..3fb7a7c4 100644 --- a/unittests/tilebuilder/parallel_tile_generator.cpp +++ b/unittests/tilebuilder/parallel_tile_generator.cpp @@ -43,7 +43,7 @@ TEST_CASE("parallel tile generator") , m_validation_error_counter(validation_error_counter) { } - void write(const std::string& file_path, const radix::tile::Descriptor& tile, const HeightData& heights) const override + void write(const std::string& file_path, const radix::tile::Descriptor& tile, const radix::Raster& heights) const override { if (file_path.empty() || tile.gridSize != 256 || heights.width() != 256 || heights.height() != 256) (*m_validation_error_counter)++; From 64794bb38b8cc39ee5da4423d765f1e8f0fc2f99 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Sun, 19 Jul 2026 13:32:22 +0200 Subject: [PATCH 02/61] Document raster store design --- docs/raster-store/README.md | 33 ++ docs/raster-store/architecture.md | 272 ++++++++++++++++ .../decisions-and-open-questions.md | 196 ++++++++++++ docs/raster-store/requirements.md | 191 +++++++++++ docs/raster-store/sampling-and-generation.md | 249 +++++++++++++++ docs/raster-store/status-quo.md | 274 ++++++++++++++++ docs/raster-store/storage-format.md | 296 ++++++++++++++++++ 7 files changed, 1511 insertions(+) create mode 100644 docs/raster-store/README.md create mode 100644 docs/raster-store/architecture.md create mode 100644 docs/raster-store/decisions-and-open-questions.md create mode 100644 docs/raster-store/requirements.md create mode 100644 docs/raster-store/sampling-and-generation.md create mode 100644 docs/raster-store/status-quo.md create mode 100644 docs/raster-store/storage-format.md diff --git a/docs/raster-store/README.md b/docs/raster-store/README.md new file mode 100644 index 00000000..61662c83 --- /dev/null +++ b/docs/raster-store/README.md @@ -0,0 +1,33 @@ +# Raster store design + +This directory describes a proposed authoritative raster store and the +generation of delivery tile pyramids from it. The documents are a design +baseline, not a finalized binary-format specification. + +The documents distinguish three kinds of statement: + +- **Confirmed** records a requirement or decision established in the design + discussion. +- **Proposed** records the current recommendation and is subject to review. +- **Open** records a question that must be resolved before the affected part + is implemented. + +## Documents + +- [Requirements and terminology](requirements.md) +- [Status quo and reuse assessment](status-quo.md) +- [Architecture](architecture.md) +- [Storage format](storage-format.md) +- [Sampling and pyramid generation](sampling-and-generation.md) +- [Decisions and open questions](decisions-and-open-questions.md) + +## Scope + +The authoritative store holds Web Mercator raster data at the best available +quality, together with exact per-pixel source attribution. It is separate from +delivery tile pyramids, which are filtered, formatted, and regenerated for a +particular consumer. + +The current scope includes raster imagery of different kinds, including +height rasters used to generate geometry. It does not define vector storage, +rendering styles, mesh formats, or a final source-ranking policy. diff --git a/docs/raster-store/architecture.md b/docs/raster-store/architecture.md new file mode 100644 index 00000000..d778b83a --- /dev/null +++ b/docs/raster-store/architecture.md @@ -0,0 +1,272 @@ +# Architecture + +## System boundary + +The design separates authoritative data management from delivery generation: + +```text +Input rasters + │ + │ inspect, transform to Web Mercator, select source per pixel + ▼ +Raster-store builder + │ + │ writes a new immutable snapshot + ▼ +Authoritative raster store + │ + ├── payload chunks + ├── per-pixel source maps + ├── source catalog + └── sparse quadtree index + │ + │ reconstruct, filter, sample, encode + ▼ +Pyramid generators + ├── area-pixel texture pyramids + └── vertex-pixel height/geometry pyramids +``` + +The store resolves which source owns each stored pixel. Generators may blend +those selected payload values but do not modify the authoritative snapshot. + +## Dataset organization + +### Proposed + +A store root contains immutable snapshots. Each snapshot contains a manifest, +a source catalog, and one or more named raster layers: + +```text +store/ +└── snapshots/ + └── / + ├── manifest + ├── sources + └── layers/ + ├── heights/ + │ ├── layer + │ ├── raster.index + │ └── chunks/... + └── orthophoto/ + ├── layer + ├── raster.index + └── chunks/... +``` + +The names and exact hierarchy are illustrative. The important boundaries are: + +- a source catalog has snapshot-wide identity and metadata; +- each layer has a homogeneous pixel schema; +- each layer owns a sparse quadtree index; and +- each physical quadtree node maps to one atomic chunk container. + +The store manifest records the format version, snapshot identity, parent +snapshot when applicable, publication state, Web Mercator definition, and +layer list. + +## Spatial model + +### Confirmed + +The hierarchy uses `radix::tile::Id` semantics: zoom, `x`, and `y`, with four +children per node. Inputs are transformed into Web Mercator during ingestion. + +Physical payloads may occur at several quadtree levels. A physical parent can +coexist with physical descendants so that the parent remains a coherent +fallback representation. + +### Proposed + +The persistent store uses one canonical `y` convention. Slippy/XYZ is the +current recommendation because it matches common web-map paths, but this is +not yet confirmed. API callers may convert from TMS before lookup. + +A layer chooses one fixed chunk pixel dimension. Every physical node in that +layer covers the Web Mercator bounds of its tile ID with that many stored +pixels per side. Deeper nodes therefore provide twice the linear spatial +resolution at each level. + +Store chunks are non-overlapping. Any halo required for filtering or any +shared border required by a delivery format is assembled by a generator. + +## Sparse quadtree index + +The index uses the same four logical states as the octree index: + +| State | Chunk exists | Indexed descendants | +|---|---:|---:| +| `Leaf` | yes | no | +| `Inner` | yes | yes | +| `Virtual` | no | yes | +| `Missing` | no | no | + +`Missing` is represented by absence from the index, not serialized as an +entry. + +The index answers structural questions only. It does not claim that every +child exists and does not mark a virtual subtree as spatially complete. + +### Required operations + +- Look up a node without probing the filesystem. +- Add and remove physical nodes while maintaining virtual ancestors. +- Traverse only indexed branches. +- Enumerate physical descendants of a subtree. +- Find the nearest physical ancestor for fallback. +- Determine whether descendants may improve a requested output. +- Serialize and validate a versioned 2D topology. + +The last two operations may require aggregate metadata beyond +`Leaf/Inner/Virtual`, such as best descendant resolution or coverage. Such +metadata is an optimization and should be derivable from authoritative +entries. + +## Chunk model + +Each physical node owns one logical chunk: + +```text +Chunk +├── identity and schema reference +├── local source table +├── payload raster +└── source-index raster +``` + +The payload and source raster have identical dimensions. Every source-index +element names exactly one local source entry. + +### Why one atomic container is proposed + +An atomic container keeps payload and provenance inseparable, makes one +hard-link represent one unchanged chunk, and prevents a snapshot from pairing +a new payload with an old source map after interruption. + +The container may internally use independently compressed blocks to support +bounded window reads. Atomicity does not require monolithic decompression. + +## Source catalog and local tables + +### Proposed + +The source catalog assigns a stable `SourceId` to every ingested source and +records metadata such as: + +- source URI or durable identity; +- content/version identity where available; +- original CRS and transform; +- original and effective resolution; +- acquisition or publication time; +- explicit priority or quality fields; +- bands, data type, colour interpretation, and NoData information; and +- ingestion software and parameters. + +Each chunk stores a compact local source table: + +```text +local index 0 → SourceId 918 +local index 1 → SourceId 42 +local index 2 → SourceId 7001 +``` + +The source map stores local indexes rather than full global IDs. Its integer +width may be selected per chunk if the container records that width. + +This arrangement provides exact per-pixel attribution while keeping common +single- or few-source chunks compact. + +`SourceId` values referenced by a hard-linked chunk must remain stable across +the complete snapshot lineage in which that chunk is reused. A child snapshot +must preserve every catalog record referenced by inherited chunks; it cannot +renumber the catalog independently. + +## Source selection and fallback + +Source selection is a builder policy, not a property of the raster container. +Its initial comparison is expected to prioritize effective pixel resolution. + +For a candidate input, the builder conceptually performs: + +```text +for each affected stored pixel: + compare candidate source with selected source + write one winning payload value + write the winning source ID +``` + +The physical hierarchy supports a separate generator decision: + +```text +physical node is sufficiently accurate and coherent + → use it directly + +physical node is insufficient and better descendants exist + → refine and combine descendants, falling back to ancestors for gaps +``` + +These policies must not be conflated. Pixel ownership determines the contents +of one physical chunk. Refinement determines which physical chunks contribute +to a requested derived tile. + +## Snapshot lifecycle + +### Proposed + +A build never mutates a published snapshot: + +1. Create a private staging snapshot. +2. Load the parent snapshot's manifest and indexes when updating. +3. Identify affected chunks from candidate source bounds and resolution. +4. Hard-link unchanged chunk containers into staging. +5. Rebuild changed chunk containers. +6. Write new source catalogs, layer metadata, and indexes. +7. Validate references, dimensions, checksums, and index/file agreement. +8. Atomically publish the completed snapshot. + +Because hard links share inodes, a linked container must never be opened for +in-place modification. Changed chunks are written to new temporary paths and +renamed into place. + +Cross-filesystem hard links cannot be created. The builder must expose this as +a clear configuration or error rather than discovering it after a long build. + +## Pyramid generator interface + +A generator requests a layer over a target tile and sampling specification. +The store reader supplies selected authoritative values and provenance over a +window large enough for the generator's filter support. + +The generator owns: + +- target output zoom and dimensions; +- vertex-pixel or area-pixel placement; +- low-pass/reconstruction filter; +- NoData normalization during filtering; +- colour-space and alpha treatment; +- border construction; +- output codec; and +- tile-level contributing-source metadata. + +The store owns: + +- chunk location and decoding; +- sparse hierarchy and physical fallback; +- exact stored source map; +- source catalog lookup; and +- consistent window access across chunk boundaries. + +## Scaling implications + +The 10-100 MB chunk target reduces filesystem entry and index counts but makes +whole-chunk rewrites expensive. The builder should therefore: + +- determine affected chunks before decoding parent data; +- bound the number of resident payload/source-map pairs; +- stream source windows rather than load complete input datasets; +- use internal block compression if partial reads are frequent; and +- avoid materializing world-scale tile vectors before parallel execution. + +The first implementation should prefer correctness and measurable behaviour. +Compression blocks, caches, aggregate index metadata, and scheduling should be +driven by profiles and representative datasets. diff --git a/docs/raster-store/decisions-and-open-questions.md b/docs/raster-store/decisions-and-open-questions.md new file mode 100644 index 00000000..02371b4f --- /dev/null +++ b/docs/raster-store/decisions-and-open-questions.md @@ -0,0 +1,196 @@ +# Decisions and open questions + +This register prevents confirmed requirements, recommendations, and unresolved +questions from being mixed together. A future architectural decision record +may replace an entry when implementation requires a durable choice. + +## Confirmed decisions + +### D1: Separate authoritative and derived layers + +The raster store is authoritative. Delivery tile pyramids are derived and may +be regenerated with different sampling, filtering, dimensions, or codecs. + +### D2: Canonical Web Mercator ingestion + +Inputs are transformed into Web Mercator while building the authoritative +store, analogous to the SF builder transforming geometry into its canonical +ECEF space. + +### D3: Use a quadtree and Radix tile identity + +The hierarchy is two-dimensional and uses `radix::tile::Id`, not +`octree::Id`. + +### D4: Preserve physical coarse fallbacks + +A physical chunk may coexist with descendants. Generators may use the parent +as a coherent coarse representation or refine into more accurate descendants. + +### D5: Store chunks are non-overlapping + +The authoritative store does not duplicate height borders. Overlapping +rendering borders are generated from a consistent filtered signal. + +### D6: One selected value and source per stored pixel + +Every stored pixel contains exactly one payload value and one source +attribution. The store does not keep alternate candidate values for that +pixel. + +### D7: Generated provenance is tile-level + +Derived filtering may blend payload values from several sources. Generated +tiles retain provenance per tile, not per output pixel. + +### D8: Generator terminology + +Use **vertex pixel** for generated values on grid vertices and **area pixel** +for generated values associated with raster cells. The names describe required +output placement, not source measurement history. + +### D9: Large authoritative chunks + +Store tiles should be approximately 10-100 MB. They are storage chunks, not +ordinary web delivery tiles. + +### D10: Radix owns the generic raster primitive + +`radix::Raster` is the common in-memory raster representation. Store +georeferencing, provenance, persistence, and snapshot lifecycle remain outside +Radix. + +## Current recommendations + +These are proposed defaults, not confirmed decisions. + +### R1: Immutable snapshots + +Publish immutable snapshots, hard-link unchanged chunk containers, and write +changed chunks to new inodes. This avoids accidental mutation through hard +links and supports discardable failed builds. + +### R2: One atomic chunk container + +Keep payload, local source table, and source map in one container so they +cannot become inconsistent and can be reused with one hard link. + +### R3: Fixed power-of-two chunk dimension per layer + +Choose a fixed side length from a logical byte budget. Do not split spatial +identities dynamically based on compressed size. + +### R4: Snapshot-wide catalog plus local source tables + +Store full metadata once in a snapshot catalog. Let each chunk map compact +local source indexes to stable catalog IDs. + +### R5: Canonical persistent tile scheme + +Persist one scheme and normalize at API boundaries. Slippy/XYZ is the current +candidate, but TMS has not been rejected. + +### R6: Explicit versioning and validation + +New format files should have magic values, versions, checksums, strict tile-ID +validation, and a rebuildable index. The current octree index binary should +not be reused as an unlabelled quadtree format. + +### R7: Global filtering across chunk boundaries + +Generators request filter halos and operate on logical global windows. They do +not clamp at internal chunk or delivery-tile boundaries. + +## Open questions + +### Spatial and layer model + +1. Are raster kinds always separate named layers with independent indexes? +2. Which exact Web Mercator definition and world extent are canonical? +3. Is the persistent scheme Slippy/XYZ or TMS? +4. What is the maximum supported zoom? +5. What exact spatial interpretation does the store assign to its own raster + elements? This is independent of vertex-pixel/area-pixel delivery policy + but must be defined for georeferencing and reconstruction. +6. Does source accuracy vary only by source, or can it vary spatially within a + source? + +### Chunk size and physical format + +7. Does 10-100 MB mean logical bytes, compressed disk bytes, or constraints on + both? +8. Are units decimal MB or binary MiB? +9. Is chunk dimension fixed globally or selected per layer? +10. Which representative payload schemas must be used to choose between + 2,048², 4,096², or another size? +11. Does the source map use `uint8_t`, `uint16_t`, or adaptive width? +12. Which container serialization, compression, block size, and checksum are + suitable for representative imagery and source maps? +13. Is `//` sufficient for filesystem scaling, or is prefix sharding + required? + +### Source metadata and selection + +14. What creates a stable `SourceId`: catalog sequence, content hash, external + ID, or a combination? +15. Which source metadata fields are mandatory? +16. How is a pixel with no valid source represented? +17. What is the first complete source-ranking policy after pixel resolution? +18. How much resolution loss is acceptable to prefer one coherent source over + a finer mosaic? +19. How are acquisition time, quality flags, explicit priority, and ties + handled? +20. Can an update remove or invalidate a previously selected source, and if + so, where does replacement data come from when alternates are not stored? + +### Filtering and generation + +21. Which reconstruction and low-pass filters are required for heights? +22. Is terrain reduction optimized only for anti-aliasing, or also for + geometric error and preservation of extrema? +23. Which colour space and alpha convention are authoritative for imagery? +24. Which semantic raster kinds require categorical or conservative + reduction rather than linear filtering? +25. What are the exact NoData normalization rules? +26. Is tile-level provenance only a source-ID set, or does it include + contribution fractions? +27. How are Web Mercator horizontal wrap and north/south boundaries filtered? +28. Must shared generated height edges be bit-identical across separate runs + and execution orders? + +### Index and implementation structure + +29. Should the octree index/storage be generalized over hierarchical ID type, + or should the raster store receive a separate 2D implementation? +30. Should `radix::tile::Id` itself gain checked root/zoom/coordinate and + serialization APIs, or should storage use a checked adapter? +31. Does the index need aggregate coverage or best-descendant-resolution data, + or can the first version derive it during traversal? +32. How is an existing index verified against chunk files without making every + open operation scan the filesystem? + +### Snapshots and operation + +33. What atomically publishes a completed snapshot? +34. Are cross-filesystem updates rejected or allowed with a copy fallback? +35. What is the retention and garbage-collection policy? +36. How does a long build resume, and how are staged chunks validated before + being trusted? +37. Which corruption and compatibility guarantees are required for long-term + archival use? + +## Suggested order for resolving questions + +The questions do not need to be answered all at once. A practical order is: + +1. Fix the store's spatial sampling interpretation and layer model. +2. Select representative payload formats and datasets. +3. Prototype chunk dimensions, source-index widths, and block compression. +4. Decide generic versus separate index/storage implementation. +5. Specify source identity, NoData, and the initial resolution ranking. +6. Implement a single-source area-pixel path with golden grid tests. +7. Add multi-source ownership and immutable snapshots. +8. Prototype vertex-pixel height filtering and seam tests. +9. Finalize operational publication, recovery, and compatibility rules. + +Each step should produce a small approved plan before code changes begin. diff --git a/docs/raster-store/requirements.md b/docs/raster-store/requirements.md new file mode 100644 index 00000000..e31bf841 --- /dev/null +++ b/docs/raster-store/requirements.md @@ -0,0 +1,191 @@ +# Requirements and terminology + +## Goals + +### Confirmed + +The system has two distinct layers: + +1. An authoritative raster store that retains the best representation + available for every covered pixel. +2. Derived tile pyramids generated from that store for rendering or other + delivery formats. + +Inputs may use arbitrary supported coordinate reference systems. Ingestion +transforms them into Web Mercator before placing their data in the store. + +The store must support multiple raster kinds. Height data is one raster kind; +orthophotos and other image-like data are others. Pixel format, NoData +handling, filtering, and output encoding may differ by kind. + +The store must support mixed accuracy over space. A physical payload at a +coarser quadtree node may remain available as a coherent fallback while more +accurate payloads exist in descendants. + +### Proposed + +Each named raster layer has a homogeneous value schema and its own sparse +quadtree index. Layers share the same Web Mercator tile addressing and source +catalog infrastructure but do not mix incompatible pixel types in one +payload. + +The authoritative store is output-format independent. It does not contain +rendering borders or require the sampling placement of a particular delivery +format. + +## Stored pixels and provenance + +### Confirmed + +Every stored pixel has exactly: + +- one payload value; and +- one source attribution. + +A tile carries a local source table and a source map. Each source-map element +indexes one entry in the local table. A local entry identifies source metadata +held by the store. + +The store does not blend source identifiers and does not retain several +candidate payload values for one stored pixel. Source selection is resolved +before the pixel is committed. + +Generated tiles do not retain per-pixel provenance. They record provenance at +tile granularity, for example as the set of sources that contributed to the +filtered output tile. Filtering may blend payload values across source +boundaries without changing the authoritative store's provenance model. + +### Open + +- Whether a source map refers to a dataset-wide source catalog by stable ID, + or contains complete source metadata in each tile. +- The reserved representation for a pixel with no valid source. +- Whether generated tile provenance needs only a set of source IDs or also + approximate contribution fractions. + +## Tile and chunk requirements + +### Confirmed + +Store tiles are large data chunks, not ordinary 256-pixel web delivery tiles. +Their target size is approximately 10-100 MB. + +Store tiles do not duplicate rendering overlap. In particular, overlapping +height borders are generated later and are not authoritative duplicated data. + +Tiles are addressed by a two-dimensional quadtree using +`radix::tile::Id`, not `octree::Id`. + +### Proposed + +The size target should initially be interpreted as an uncompressed logical +byte budget for payload plus source map. That gives predictable peak memory, +I/O, and rewrite cost. Compressed on-disk size varies too much with content to +be the only sizing invariant. + +The pixel side length should be a power of two. Illustrative uncompressed +sizes are: + +| Side length | Pixels | RGB8 + uint16 source | float32 + uint16 source | +|---:|---:|---:|---:| +| 2,048 | 4,194,304 | 20 MiB | 24 MiB | +| 4,096 | 16,777,216 | 80 MiB | 96 MiB | + +These calculations exclude headers, the local source table, alignment, and +compression. They demonstrate that a fixed 2,048- or 4,096-pixel chunk is in +the intended range for common formats; they do not decide the final chunk +dimension. + +### Open + +- Whether the 10-100 MB target refers to logical, compressed, or both sizes. +- Whether all layers use one fixed side length or choose it from their bytes + per pixel and source-index width. +- Whether the source map uses a fixed or per-tile adaptive integer width. +- Whether payload and provenance are one atomic container or coordinated + files. + +## Accuracy and fallback + +### Confirmed + +Pixel resolution is expected to be the most important source-accuracy signal, +but the final ranking rule is not yet known. + +A coarse, coherent source should be usable directly for a coarse output tile +even when more accurate fragments exist below it. Generators must not be +forced to patch every output from the deepest available descendants. + +### Proposed + +Source metadata records effective ground resolution after reprojection as +well as original resolution where meaningful. Source selection is a policy +interface rather than a fixed comparison embedded in the storage format. + +A pyramid generator refines into descendants only when the current physical +node does not satisfy the requested output resolution or another configured +quality criterion. + +### Open + +- The resolution tolerance that permits use of one coherent source instead + of a finer mosaic. +- Tie-breaking by acquisition time, explicit priority, quality, or source + identity. +- Whether source accuracy is scalar per source or may vary spatially. + +## Updates and reuse + +### Proposed + +Published store versions are immutable snapshots. Building a new snapshot +hard-links unchanged tile containers from an earlier snapshot and writes new +containers only for changed tiles. The snapshot owns a new index and manifest. + +This is safer than mutating hard-linked files in place and makes a failed build +discardable without damaging an earlier published store. + +Hard-link reuse requires source and destination snapshots to reside on the +same filesystem. The implementation must either require that condition or +define an explicit copy fallback. + +### Open + +- Snapshot publication and atomic rename rules. +- Retention and garbage-collection policy. +- Whether cross-filesystem builds fail or copy unchanged chunks. + +## Terminology + +**Raster store tile** or **chunk** +: A large authoritative file associated with a quadtree tile ID. It is not a + delivery tile and is expected to contain roughly 10-100 MB. + +**Payload** +: The selected raster value array stored by a chunk. + +**Source map** +: A raster aligned one-to-one with the payload. Each element selects exactly + one entry in the chunk's local source table. + +**Physical node** +: A quadtree node with an authoritative chunk on disk. + +**Virtual node** +: An index-only quadtree node with descendants but no chunk of its own. + +**Vertex pixel** +: A generated value located on a grid vertex. Height tiles for mesh generation + require vertex pixels, including shared boundary positions. + +**Area pixel** +: A generated value associated with a raster cell. Ordinary texture outputs + use area pixels whose cell boundaries align with tile boundaries. + +The terms vertex pixel and area pixel describe generator outputs. They do not +assert how an original sensor or source raster produced its values. + +**Derived tile** +: A filtered and encoded output tile generated from the authoritative store. +It may have different dimensions, sampling placement, encoding, and +provenance granularity from a store chunk. diff --git a/docs/raster-store/sampling-and-generation.md b/docs/raster-store/sampling-and-generation.md new file mode 100644 index 00000000..83309521 --- /dev/null +++ b/docs/raster-store/sampling-and-generation.md @@ -0,0 +1,249 @@ +# Sampling and pyramid generation + +This document defines generator-facing sampling terminology and invariants. +It deliberately separates output sampling from how an original source raster +was measured or produced. + +## Terminology + +### Vertex pixel + +A vertex pixel is a generated value located on a grid vertex. For a tile with +`N` intervals per side, vertex positions are: + +```text +x(i) = left + i × tile_width / N, i = 0 … N +``` + +The output has `N+1` pixels per side: + +```text +tile boundary tile boundary +●---------●---------●---------●---------● +``` + +Height maps used to form mesh vertices require this placement. Adjacent +rendering tiles contain overlapping copies of their shared edge and corner +vertex pixels. + +### Area pixel + +An area pixel is a generated value associated with one raster cell. For a tile +with `N` cells per side, effective cell-centre positions are: + +```text +x(i) = left + (i + 1/2) × tile_width / N, i = 0 … N-1 +``` + +The output has `N` pixels per side: + +```text +tile boundary tile boundary +│ × × × × │ +``` + +The term describes placement and support in the generated grid. It does not +assert that the input value was a physical area integral. + +## Source semantics versus output placement + +An input height raster may have been produced from LiDAR points through +gridding, interpolation, fitting, or averaging. An orthophoto may already have +passed through sensor integration, reconstruction, reprojection, and +resampling. Those histories do not decide where a delivery format requires +its output values. + +Generation is modelled as: + +```text +stored discrete raster + ↓ reconstruct its implied field +continuous or evaluable field + ↓ low-pass for target resolution +filtered field + ↓ evaluate on requested output grid +vertex pixels or area pixels +``` + +The source interpretation and reconstruction rule are layer/generator policy. +Vertex-pixel and area-pixel placement are output requirements. + +## Reduction by two + +### Vertex pixels + +At fine spacing `Δ`, fine vertex positions are `nΔ`. Coarse positions are +`2mΔ`, coinciding with every second fine location: + +```text +fine: ●---●---●---●---● +coarse: ●-------●-------● +``` + +Copying every second value would be unfiltered decimation and is unacceptable +because frequencies above the new Nyquist limit would alias. The generator +must low-pass first, using a kernel centred on each retained vertex position: + +```text +coarse[m] = Σ h[k] × fine[2m - k] +``` + +The spatial centre stays in place; the value generally changes because it is +sampled from the filtered signal. + +### Area pixels + +Fine area-pixel centres are `(n+1/2)Δ`. A coarse cell spans two fine cells and +has its centre at `(2m+1)Δ`, halfway between two fine centres: + +```text +fine cells: |---- × ----|---- × ----| +coarse cell: |---------- × ----------| +``` + +The simplest reduction is the average of each 2x2 fine block. If fine values +are exact equal-area averages, that produces the exact average over the union +of the four cells. A box filter is not an ideal anti-aliasing filter, however, +and may be insufficient for visual imagery or other signals. + +A higher-quality area-pixel reduction applies a low-pass filter with the +correct half-sample phase, centred on the coarse cell centre. Its support may +extend beyond the four cells geometrically covered by the coarse cell. + +## No duplicated height borders in the store + +The authoritative store does not persist overlapping rendering borders. +Pyramid generation constructs a vertex-pixel output only after reconstruction +and filtering. + +The implementation may obtain a requested `(N+1) × (N+1)` output window by +reading non-overlapping store chunks plus the filter halo required on every +side. The generated shared vertices must be computed from the same global +coordinates and source data for both neighbouring output tiles. + +The generator must not independently clamp its filter at each tile edge. +Clamping would make an internal tile boundary behave like a data boundary and +could produce seams. + +Two implementation strategies can satisfy the invariant: + +1. Evaluate shared global vertex coordinates deterministically from a common + window reader; or +2. Generate a metatile, filter it once, and split it into overlapping output + tiles. + +The first gives execution-order independence. The second may reduce repeated +I/O. They can coexist if tests establish identical results. + +## Filter halos and chunk boundaries + +Any nontrivial low-pass filter needs samples outside the exact output bounds. +The required halo is determined by the reconstruction and reduction filters, +not by a fixed one-pixel border flag. + +The store reader should expose a logical raster window over the quadtree. It +resolves: + +- physical chunks selected for the requested accuracy; +- ancestor fallback where finer data is absent; +- chunk and source-map decoding; and +- neighbouring data needed by the window. + +The generator determines the requested halo and applies boundary conditions +only at true dataset/world boundaries or NoData boundaries. + +## Mixed sources + +The raster store contains exactly one payload and one source ID for each +stored pixel. A generator filter may span pixels attributed to several +sources: + +```text +store pixels: A A A B B +filter support: [-------] +output value: blend of A and B payloads +``` + +This is allowed. A generated output pixel does not retain a source ID. The +generated tile records tile-level provenance, at minimum the set of source IDs +whose payload values contributed nonzero filter weight to any output pixel. + +Source IDs are categorical and are never averaged. Payload filtering and +provenance collection are parallel operations: + +```text +numeric payload samples → weighted filtered value +source IDs → contributing-source set +``` + +The exact handling of invalid/NoData samples requires a policy. A common +continuous-raster rule is to normalize by the total weight of valid samples, +but that must not be applied automatically to categorical data. + +## Layer-specific filtering + +Sampling placement alone does not determine a correct filter: + +| Semantic kind | Relevant considerations | +|---|---| +| Height | low-pass before decimation; terrain error and peak loss | +| Orthophoto | linear-light filtering; alpha premultiplication | +| Categorical | mode, coverage, or another categorical policy | +| Probability/coverage | conservative area averaging may be appropriate | +| Vector/normal | component filtering followed by normalization where needed | +| Mask/NoData | validity-aware weights and explicit coverage rules | + +The current `radix::raster::generate_mipmap` performs a component-wise 2x2 +box average. It may be a reference for simple area-pixel aggregation, but it +does not implement these policies or vertex-pixel filtering. + +## Coherent coarse-source selection + +The generator need not always filter the deepest available descendants. If a +physical chunk at the requested scale is sufficiently accurate, using that +single coherent source may be preferable to composing several finer sources. + +The selection process is conceptually: + +```text +choose physical representation(s) for the requested output and quality policy + ↓ +read a continuous window with fallback and required halo + ↓ +filter for the target resolution + ↓ +evaluate vertex pixels or area pixels +``` + +Source-selection/refinement policy precedes filtering. Filtering does not +change the authoritative hierarchy. + +## World and dataset boundaries + +The generator needs explicit rules for: + +- horizontal wrapping at the Web Mercator antimeridian; +- north/south limits of the Web Mercator world; +- areas with no physical ancestor or descendant; +- NoData holes inside otherwise covered chunks; and +- filters whose support crosses a layer's coverage boundary. + +These rules are not yet decided. Tests must distinguish true boundaries from +ordinary internal chunk and delivery-tile boundaries. + +## Required golden tests + +Before production filtering is implemented, synthetic fixtures should prove: + +1. A constant raster remains constant across chunks and pyramid levels. +2. An impulse or frequency sweep demonstrates the chosen anti-alias response. +3. Two adjacent area-pixel tiles match a single equivalent metatile result. +4. Two adjacent vertex-pixel tiles produce bit-identical shared edges. +5. Filtering is unchanged when a store window is split into different chunks. +6. A source boundary blends payloads but reports both tile-level sources. +7. A NoData boundary follows the configured validity rule. +8. A coherent physical parent can be chosen instead of finer descendants. +9. TMS and Slippy input IDs normalize to the same canonical spatial tile. + +The filter coefficients and acceptable numeric tolerances remain open design +decisions. The tests should lock them only after representative evaluation. diff --git a/docs/raster-store/status-quo.md b/docs/raster-store/status-quo.md new file mode 100644 index 00000000..9d2b5701 --- /dev/null +++ b/docs/raster-store/status-quo.md @@ -0,0 +1,274 @@ +# Status quo and reuse assessment + +This document evaluates the current repository after commit `9cf9065` +(`Consolidate raster handling and use std::expected`). It distinguishes +reusable mechanisms from interfaces that encode assumptions unsuitable for +the raster store. + +## Summary + +The project already contains most low-level ingredients: + +- `radix::Raster` for contiguous typed raster memory; +- `radix::tile::Id` for quadtree addressing; +- Web Mercator grid calculations and GDAL reprojection in `tile_builder`; +- sparse topology, indexed traversal, codecs, layouts, and hard-link reuse in + the octree storage code; and +- the SF merger's snapshot-like reuse of unchanged subtrees. + +There is no existing component that should become the raster store unchanged. +The best path is to compose the Radix raster and tile primitives with a new +2D storage layer, while extracting or adapting selected octree-storage ideas. + +## Reuse matrix + +| Component | Assessment | Intended use | +|---|---|---| +| `radix::Raster` | Reuse directly | In-memory payloads and source maps | +| `radix::RasterMask` | Reuse directly | Temporary validity/selection masks | +| `radix::raster::transform` | Reuse directly | Typed pixel transformations | +| `radix::raster::generate_mipmap` | Do not use as general generator | Only a 2x2 component-wise box average | +| `radix::tile::Id` | Reuse after hardening or through an adapter | Persistent quadtree keys | +| `radix::quad_tree::Node` | Do not reuse for disk index | Dense in-memory ownership model | +| CTB `GlobalMercator` and grid bounds | Reuse initially | Tile bounds and Web Mercator resolution | +| `Dataset` and GDAL setup | Reuse/adapt | Input discovery and reprojection | +| `DatasetReader` | Adapt substantially | Windowed, typed, multi-band ingestion | +| `Tiler` / `ParallelTiler` | Reuse calculations, replace orchestration | Candidate chunk enumeration | +| `ParallelTileGenerator` | Do not use as store builder | Small-file writer with incompatible lifecycle | +| Octree `IndexMap` algorithm | Reuse design; generalize or port | Sparse physical/virtual topology | +| Octree `Storage_` / `RawStorage_` | Reuse design and selected code | Codec boundary, indexing, hard-link copy | +| Octree disk layouts | Do not use as-is | They encode `octree::Id` and 3D paths | +| SF merger visitors and geometry code | Do not reuse | Mesh- and ECEF-specific semantics | +| SF merger unchanged-subtree copy | Reuse design | Snapshot construction and hard links | +| `zpp_bits` serialization helpers | Reuse cautiously | Versioned metadata/index serialization | + +## Radix raster + +### What can be reused + +`extern/radix/src/radix/raster.h` now provides a shared, value-typed raster: + +- rectangular `glm::uvec2` dimensions; +- contiguous row-major `std::vector` storage; +- element and byte spans; +- typed pixel access; +- move construction from an existing vector; +- a contiguous byte-valued `RasterMask`; +- dimension-checked concatenation; and +- masked and unmasked transforms. + +These properties fit both store arrays: + +```cpp +radix::Raster payload; +radix::Raster source_map; +``` + +The class correctly remains independent of Web Mercator, tile IDs, source +metadata, compression, and file I/O. Those belong to higher layers. + +### What needs adaptation around it + +Large chunks and filtered generation will benefit from operations not +currently supplied by `Raster`: + +- non-owning raster views and subwindows; +- explicit row stride where external codecs require it; +- halo/window assembly across neighbouring chunks; +- checked construction that reports allocation/dimension errors without + relying on assertions; and +- streaming or block processing when a full set of input chunks would exceed + the memory budget. + +These should be introduced only when required. They are not reasons to embed +store concepts into `Raster`. + +### What cannot be reused for final filtering + +`radix::raster::generate_mipmap` requires a square, power-of-two raster and +reduces each 2x2 group by component-wise averaging. It does not provide: + +- a selectable reconstruction or low-pass filter; +- the phase difference between vertex pixels and area pixels; +- halo samples across tile boundaries; +- linear-light colour and premultiplied-alpha handling; +- NoData-aware normalization; +- categorical reduction; or +- source-contribution tracking. + +It is therefore a useful simple raster utility, not the raster-store pyramid +generator. + +## Radix tile addressing + +### What can be reused + +`radix::tile::Id` already contains the required 2D identity: + +- zoom level; +- `x/y` coordinates; +- parent and four-child relationships; +- TMS/Slippy conversion; and +- hashing and ordering support. + +### Required hardening + +Persistent storage needs stronger invariants than the current convenience +type supplies: + +- Calling `parent()` at zoom zero currently underflows. +- Construction does not reject coordinates outside `[0, 2^z)`. +- Shifting `1u << zoom_level` limits valid conversion at high zooms. +- The scheme participates in identity, so the same spatial tile in TMS and + Slippy form becomes two keys. +- There is no persistent serialization contract or format version. + +The store should choose one canonical scheme and normalize all IDs at its API +boundary. Whether the hardening belongs in Radix or in a checked store adapter +is open. + +`radix::quad_tree::Node` is not a replacement for the index. It owns a +fully allocated group of four children whenever refined, represents no +missing child within such a group, and has no persistence or physical/virtual +status. The store requires a sparse map keyed by tile ID. + +## Tile builder and GDAL path + +### Reusable foundations + +The current tile builder already demonstrates: + +- opening GDAL raster datasets; +- reading dataset bounds; +- selecting Web Mercator or geodetic CTB grids; +- transforming arbitrary source SRS data during reads; +- calculating tile bounds and resolutions; +- enumerating intersecting `radix::tile::Id` values; and +- parallel per-tile processing. + +`ctb::GlobalMercator`, `Tiler::tile_for`, and `ParallelTiler` are useful +references and may be reused initially for grid math. + +### Required changes + +`DatasetReader` currently reads one band into a float raster, uses cubic GDAL +warping, and constructs a warped VRT for a requested output rectangle. The +store needs typed and multi-band reads, explicit alpha/NoData handling, +controlled resampling, source metadata, and deterministic alignment with the +store chunk grid. + +`Tiler` models delivery-oriented dimensions through `Border::Yes/No` and a +south/east extra pixel. Store chunks have no rendering overlap, and generated +vertex pixels require an explicit global sampling/filtering model. The border +boolean should not define store geometry. + +`ParallelTileGenerator` writes individual image files for explicitly +enumerated tiles. It is not suitable as the snapshot builder because it lacks: + +- source-map generation; +- old-snapshot reuse; +- atomic chunk containers; +- index transactions and publication; +- resumability validation; and +- bounded enumeration/streaming for very large tile sets. + +Its parallel work pattern and progress reporting may still inform the new +builder. + +## Octree index and storage + +### Reusable design + +The octree index captures the required topology semantics: + +- `Leaf`: physical payload without indexed descendants; +- `Inner`: physical payload with indexed descendants; +- `Virtual`: no physical payload, but indexed descendants; and +- absence from the map: missing node and subtree. + +Adding a physical node creates virtual ancestors and promotes a physical +ancestor from leaf to inner. Removing nodes collapses unused virtual chains. +Traversal follows only present index entries. + +The storage stack also has useful separation between: + +- logical indexed storage; +- raw path-based storage; +- disk layout; +- payload codec; and +- optional cache. + +`RawStorage_::copy_from` demonstrates hard-link reuse when source and target +extensions match. The SF merger demonstrates copying unchanged physical +subtrees and writing a new output index. + +### Why it cannot be reused unchanged + +The entire stack uses concrete `octree::Id` types. `IndexMap`, cache APIs, +layouts, filesystem path parsing, traversal, storage, serialization, and +formatting all embed this type. The coordinate-directory layout is +`level/x/y/z`, and the default codec is a mesh codec. + +The existing index file also records no tree kind. Reinterpreting its +serialized `(level, index)` octree IDs as web tiles would be unsafe. + +A 2D implementation can either: + +1. generalize the hierarchy/storage stack over an ID and layout policy; or +2. create a raster-store-specific 2D implementation using the same algorithms. + +Generalization avoids duplicate infrastructure but has a larger blast radius +in mature octree code. A separate implementation is initially safer but risks +long-term duplication. This requires an explicit decision before coding. + +### Behaviours that should not be copied blindly + +- Hard-link failure currently has no copy fallback. +- Existing indexes are trusted rather than reconciled against filesystem + contents when opened. +- An unindexed output discovers files through a final recursive directory + scan. +- Some error paths use assertions or process termination. +- The binary index lacks an explicit magic/version/topology header suitable + for a new durable format. + +The raster store should retain the useful topology and codec boundaries while +specifying stronger snapshot, validation, and error-handling rules. + +## SF builder and merger + +The SF builder's ECEF octree placement, mesh construction, texture atlases, +mask clipping, and mesh visitors are not reusable for a Web Mercator raster +store. + +The reusable ideas are architectural: + +- transform source data into a canonical spatial system during build; +- preserve physical ancestors as fallback representations; +- use an index to avoid per-node filesystem probing; +- stop refinement when a coherent representation is sufficient; and +- hard-link unchanged chunks into a new output dataset. + +Those ideas should be reimplemented against Radix tile IDs and raster payloads +rather than adapted through mesh abstractions. + +## Recommended component boundary + +The current components suggest the following dependency direction: + +```text +Radix + Raster, RasterMask, tile::Id, geometry + ↓ +Terrain library + GDAL dataset access, Web Mercator grid math, filtering primitives + ↓ +Raster store + source catalog, chunk container, sparse index, snapshots + ↓ +Builders and generators + ingestion, source selection, texture pyramids, height/geometry pyramids +``` + +The raster store should consume `radix::Raster`; Radix should not depend on +the store's source catalog, file format, or snapshot lifecycle. diff --git a/docs/raster-store/storage-format.md b/docs/raster-store/storage-format.md new file mode 100644 index 00000000..56a639b9 --- /dev/null +++ b/docs/raster-store/storage-format.md @@ -0,0 +1,296 @@ +# Storage format + +This document specifies the logical format and required invariants. Exact +binary encodings, field widths, compression libraries, and filenames remain +open unless explicitly marked confirmed. + +## Format principles + +### Confirmed + +- The persistent hierarchy is a Web Mercator quadtree keyed by Radix tile IDs. +- Store chunks are non-overlapping and approximately 10-100 MB. +- Every stored pixel has one payload value and one source attribution. +- A coarse physical chunk may coexist with more accurate descendants. +- Rendering overlap and delivery encodings are not stored authoritatively. + +### Proposed + +- All files have explicit magic, format version, and byte-order declarations. +- Payload and source map are held in one atomic chunk container. +- Published files are immutable. +- Checksums cover metadata and independently compressed data sections. +- Unknown optional fields can be skipped by length; incompatible required + fields cause a clear load failure. + +## Store manifest + +The root manifest should contain: + +```text +format magic and version +snapshot ID +optional parent snapshot ID +creation metadata +canonical CRS identifier and definition +canonical tile scheme +source-catalog reference +layer descriptors +``` + +The CRS must be specified more strongly than a human-readable name. The exact +EPSG/Web Mercator definition and world extent used by tile calculations must +be unambiguous. + +## Layer descriptor + +A layer descriptor should contain: + +```text +layer ID and name +semantic kind +payload pixel format +band/component interpretation +chunk width and height +NoData representation +index reference +chunk layout/codec identifier +``` + +The descriptor defines stored data. Vertex-pixel versus area-pixel delivery is +generator policy and is not required as a layer field unless the eventual +store permits more than one stored spatial placement. + +The semantic kind is still useful because it constrains valid operations. For +example, height scalars, linear colour, gamma-encoded colour, categorical +values, masks, and vectors require different filtering. + +## Source catalog + +A source-catalog record should contain a stable ID and sufficient provenance +to understand or reproduce ingestion. Candidate fields are: + +```text +SourceId +source identity and version/content fingerprint +human-readable name +original URI or path, if durable and safe to persist +original CRS and geotransform +original dimensions, bands, and pixel type +original NoData/alpha information +original nominal resolution +effective Web Mercator resolution +acquisition/publication time +priority and quality metadata +ingestion timestamp +ingestion transform and resampling policy +``` + +Pathnames may contain deployment-specific or sensitive information. The +format should support opaque source identities without requiring an original +local path. + +Source IDs are persistent across related snapshots. A snapshot that hard-links +an older chunk must retain the same meaning for every global source ID in that +chunk's local table. Snapshot-local catalog compaction may not renumber IDs +referenced by inherited containers. + +## Quadtree index file + +The index should contain: + +```text +magic +format version +tree kind = quadtree +canonical scheme +chunk layout identifier +entry count +entries: TileId → NodeStatus +optional aggregate metadata +checksum +``` + +`TileId` must be validated when decoded: + +```text +zoom is supported +x < 2^zoom +y < 2^zoom +scheme is canonical or omitted because the file declares it globally +``` + +Node status has the following invariant: + +- `Leaf` and `Inner` have a chunk file. +- `Virtual` has no chunk file. +- `Inner` and `Virtual` have at least one indexed descendant. +- `Leaf` has no indexed descendant. +- Every non-root entry has all required ancestors represented. + +The index must be reconstructible by scanning valid chunk paths. A rebuild +tool should write a new index rather than silently changing a published one. + +## Chunk container + +### Logical structure + +```text +Chunk header +├── magic and version +├── tile ID +├── layer/schema identity +├── raster dimensions +├── payload encoding +├── source-index encoding +└── section directory + +Local source table +├── local source index +└── global SourceId + +Payload sections +└── typed raster data + +Source-map sections +└── local source indexes, one per payload pixel +``` + +Payload and source-map dimensions must match exactly. A decoder rejects the +container if a source-map value is outside the local source table. + +### Source-index width + +Possible representations include: + +- `uint8_t` for at most 256 local entries; +- `uint16_t` for at most 65,536 local entries; or +- a width selected per chunk and recorded in the header. + +Adaptive width is attractive because most chunks are expected to use few +sources, but it adds codec branches. This is an open implementation decision. + +A reserved local entry may represent no valid source. If this is adopted, its +payload validity and filtering semantics must be defined rather than inferred +from an arbitrary payload value. + +### Compression and random access + +The chunk-size target makes whole-file decompression undesirable for small +windows. A proposed container divides payload and source map into matching +independently compressed blocks and records their offsets in the section +directory. + +Block dimensions should support: + +- bounded memory; +- halo reads across chunks; +- checksumming damaged regions; +- skipping source-map blocks when provenance is not requested; and +- skipping payload blocks during provenance-only inspection. + +The payload and source-map blocks need not use the same compression algorithm. +Source maps are categorical and may benefit from run-length, palette, or +general-purpose compression. The initial implementation should benchmark +representative data before fixing a codec. + +## Chunk sizing + +### Proposed sizing rule + +Choose a power-of-two side length from a logical byte budget: + +```text +logical bytes ≈ width × height × + (payload bytes per pixel + source-index bytes per pixel) +``` + +The target interval is 10-100 MB, including both large raster arrays but +excluding small metadata. For example: + +```text +4096² × (RGB8 + uint16 source) ≈ 80 MiB +4096² × (float32 + uint16 source) ≈ 96 MiB +2048² × (RGBA8 + uint16 source) ≈ 24 MiB +``` + +The examples use binary MiB despite the conversational 10-100 MB requirement; +the final specification must choose units explicitly. + +If a layer's pixel type would place 4,096² outside the budget, the layer may +choose another fixed power-of-two dimension. Varying dimensions between nodes +within one layer is not recommended because it complicates resolution and +window calculations. + +### Why compressed size should not drive identity + +Compressed size depends on terrain, imagery, source-map fragmentation, and +codec settings. Splitting chunks dynamically to hit a compressed-byte target +would make spatial identities content-dependent and destabilize updates. +A fixed spatial grid with a logical size budget gives predictable identity; +compression is an optimization. + +## Filesystem layout + +A proposed chunk path is: + +```text +layers//chunks///.rst +``` + +This is familiar and reconstructible, but very large datasets may place too +many entries in intermediate directories. Alternative layouts may shard by +Morton/Hilbert prefix. The layout identifier in layer metadata allows this to +change without changing tile identity. + +Layout parsing must accept only canonical chunk files. Auxiliary previews, +temporary files, and generated delivery images must not accidentally become +index entries during a rebuild scan. + +## Immutability and hard links + +An unchanged chunk in a child snapshot may be a hard link to the parent's +container. The following rules are mandatory if hard links are used: + +- Published containers are never modified in place. +- A changed container is written to a distinct temporary inode. +- Completion uses an atomic rename within the target filesystem. +- Snapshot deletion removes names, not shared content still referenced by + another snapshot. +- Validation may inspect inode/link counts for diagnostics but format + correctness never depends on them. + +The index, manifest, and source catalog are snapshot-specific and normally new +files even when many chunks are linked. + +## Consistency and recovery + +A snapshot is publishable only if: + +- every physical index entry resolves to one valid chunk; +- no virtual entry has a chunk; +- all chunk tile IDs and layer IDs match their paths and index entries; +- source maps reference valid local entries; +- local entries reference valid catalog records; +- dimensions and formats match the layer descriptor; and +- checksums pass. + +Temporary and incomplete files use names excluded from layout scanning. +Publication should make the complete snapshot visible in one atomic metadata +operation. The exact mechanism is open. + +## Explicitly not finalized + +This document does not yet choose: + +- binary serialization library; +- integer field widths; +- compression codecs and block dimensions; +- exact source metadata fields; +- checksum algorithm; +- file extensions and directory names; +- snapshot publication mechanism; or +- generic versus raster-specific index implementation. + +Those choices require prototypes and representative datasets, and should not +be implied by a first implementation. From ec0384784232efcc8d9c4f1b13e01e98ce6d4416 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Sun, 19 Jul 2026 15:51:21 +0200 Subject: [PATCH 03/61] Use Slippy tile coordinates throughout --- README.md | 15 +- docs/tile-downloader/download.sh | 6 +- docs/tile-downloader/root_tile_list | 728 +++++++++--------- src/CMakeLists.txt | 2 +- src/sf_builder/main.cpp | 23 +- src/sf_builder/texture_assembler.h | 3 +- src/sf_builder/tile_provider.h | 21 +- src/terrainlib/CMakeLists.txt | 1 + src/terrainlib/ctb/GlobalGeodetic.hpp | 5 +- src/terrainlib/ctb/GlobalMercator.hpp | 5 +- src/terrainlib/ctb/Grid.hpp | 27 +- src/terrainlib/tile_path.h | 14 + src/tile_builder/ParallelTileGenerator.cpp | 4 +- src/tile_builder/ParallelTileGenerator.h | 2 +- src/tile_builder/ParallelTiler.cpp | 17 +- src/tile_builder/ParallelTiler.h | 2 +- src/tile_builder/TileHeightsGenerator.cpp | 9 +- src/tile_builder/TileHeightsGenerator.h | 4 +- src/tile_builder/Tiler.cpp | 9 +- src/tile_builder/Tiler.h | 5 +- src/tile_builder/TopDownTiler.cpp | 7 +- src/tile_builder/TopDownTiler.h | 2 +- src/tile_builder/alpine_raster.cpp | 4 +- src/tile_builder/alpine_raster.h | 1 - src/tile_builder/main.cpp | 10 +- src/tile_downloader/TileDownloader.h | 24 +- src/tile_downloader/TileUrlBuilder.h | 57 +- src/tile_downloader/cli.cpp | 34 +- src/tile_downloader/cli.h | 7 +- src/tile_downloader/main.cpp | 6 +- unittests/CMakeLists.txt | 9 + unittests/sf_builder/texture.cpp | 104 +-- unittests/tile_downloader/url_builder.cpp | 45 ++ .../tilebuilder/alpine_raster_format.cpp | 6 +- .../depth_first_tile_traverser.cpp | 42 +- .../tilebuilder/parallel_tile_generator.cpp | 14 +- unittests/tilebuilder/parallel_tiler.cpp | 78 +- .../tilebuilder/tile_heights_generator.cpp | 8 +- unittests/tilebuilder/top_down_tiler.cpp | 30 +- 39 files changed, 732 insertions(+), 658 deletions(-) create mode 100644 src/terrainlib/tile_path.h create mode 100644 unittests/tile_downloader/url_builder.cpp diff --git a/README.md b/README.md index e4af0bdb..20d2d8d8 100644 --- a/README.md +++ b/README.md @@ -24,15 +24,20 @@ The tools typically only handle one tile per command which makes it infeasible t In this example, we'll show how to build the hierarchy for Vienna's city center (Zoom: 13, X: 4468, Y:2840). ### 1. Downloading tiles -The following command will download the basemap tiles from our mirror with the following format: -https://gataki.cg.tuwien.ac.at/raw/basemap/tiles/{zoom}/{Y}/{X}.jpeg +Tile coordinates always use the Google/Mapbox/XYZ convention internally: the origin is north-west, X grows east, and Y grows south. Google Maps, Mapbox, OpenStreetMap, and most XYZ services use the common URL order `{zoom}/{x}/{y}`; this is the downloader default. + +Some providers use `{zoom}/{y}/{x}` instead. Both built-in Austrian providers currently require `--url-coordinate-order yx`; for example, the Gataki mirror uses: + +`https://gataki.cg.tuwien.ac.at/raw/basemap/tiles/{zoom}/{y}/{x}.jpeg` Example for the root tile: https://gataki.cg.tuwien.ac.at/raw/basemap/tiles/13/2840/4468.jpeg -The tiles will be downloaded into the folder `./tiles/`. +Select URL coordinate order with `--url-coordinate-order xy|yx`. Select the URL Y direction with `--url-y-direction down|up`; `down` is the common Google/Mapbox convention and `up` is the legacy TMS convention. These options affect only remote URLs. Downloaded files always use the common Google/Mapbox layout `{zoom}/{x}/{y}.jpeg`. + +The following command downloads the mirror's `{zoom}/{y}/{x}` URLs and writes the root tile to `./tiles/13/4468/2840.jpeg`: ``` -./tile-downloader --provider gataki --zoom 13 --row 2840 --col 4468 --max-zoom-level 19 +./tile-downloader --provider gataki --zoom 13 --x 4468 --y 2840 --url-coordinate-order yx --max-zoom-level 19 ``` ### 2. Download heightmap dataset @@ -84,4 +89,4 @@ In order to build, you need to install: - tbb (intel threading building blocks) sudo apt-get install libcgal-dev libopencv-dev libfmt-dev libglm-dev libgdal-dev catch2 libfreeimage-dev libtbb-dev libcurl4-openssl-dev -(libgmp-dev libmpfr-dev libsqlite3-dev) \ No newline at end of file +(libgmp-dev libmpfr-dev libsqlite3-dev) diff --git a/docs/tile-downloader/download.sh b/docs/tile-downloader/download.sh index b375ad75..d5aef40a 100755 --- a/docs/tile-downloader/download.sh +++ b/docs/tile-downloader/download.sh @@ -3,7 +3,7 @@ build_path="/home/madam/Documents/work/tuw/alpinemaps/build-terrain-builder-Desktop_Qt_6_2_3_GCC_64bit-Release/src" while read p; do - read zoom row col <<<${p//[^0-9]/ } - echo -e "nice -10 \$build_path/tile-downloader --provider basemap --zoom ${zoom} --row ${row} --col ${col} --verbosity 0&" -# nice -10 $build_path/tile-downloader --provider basemap --zoom ${zoom} --row ${row} --col ${col} --verbosity 0& + read zoom x y <<<${p//[^0-9]/ } + echo -e "nice -10 \$build_path/tile-downloader --provider basemap --zoom ${zoom} --x ${x} --y ${y} --url-coordinate-order yx --verbosity 0&" +# nice -10 $build_path/tile-downloader --provider basemap --zoom ${zoom} --x ${x} --y ${y} --url-coordinate-order yx --verbosity 0& done &data radix::geometry::Aabb3d parse_bounds_from_tile( const std::vector &data, - radix::tile::Scheme scheme, const OGRSpatialReference &srs) { // Determine the correct Grid type based on SRS @@ -86,7 +85,7 @@ radix::geometry::Aabb3d parse_bounds_from_tile( const unsigned int zoom_level = data[0]; const glm::uvec2 tile_coords(data[1], data[2]); - const radix::tile::Id target_tile(zoom_level, tile_coords, scheme); + const radix::tile::Id target_tile(zoom_level, tile_coords); return extend_bounds_to_3d(grid->srsBounds(target_tile, false)); } @@ -117,7 +116,6 @@ radix::geometry::Aabb3d parse_target_bounds( const std::vector &bounds_data, const std::vector &node_data, const std::vector &tile_data, - radix::tile::Scheme tile_scheme, OGRSpatialReference &srs) { if (!bounds_data.empty()) { @@ -125,7 +123,7 @@ radix::geometry::Aabb3d parse_target_bounds( } if (!tile_data.empty()) { - return parse_bounds_from_tile(tile_data, tile_scheme, srs); + return parse_bounds_from_tile(tile_data, srs); } if (!node_data.empty()) { @@ -221,7 +219,6 @@ int run(std::span args) { std::vector target_tile_data; std::vector target_node_data; std::string target_srs_input; - radix::tile::Scheme target_tile_scheme; auto *target = single->add_option_group("target"); target->add_option("--bounds", target_bounds_data, "Target bounds for the reference mesh as \"{xmin} {width} {ymin} {height} [{zmin} {depth}]\"") @@ -235,15 +232,6 @@ int run(std::span args) { single->add_option("--output", output_path, "Output path were the mesh is written to (.terrain, .gltf or .glb)") ->required(); - std::map scheme_str_map{ - {"slippymap", radix::tile::Scheme::SlippyMap}, - {"google", radix::tile::Scheme::SlippyMap}, - {"tms", radix::tile::Scheme::Tms}}; - single->add_option("--scheme", target_tile_scheme, "Tile scheme") - ->default_val(radix::tile::Scheme::SlippyMap) - ->needs("--tile") - ->transform(CLI::CheckedTransformer(scheme_str_map, CLI::ignore_case)); - single->add_option("--srs", target_srs_input, "EPSG code of the srs of the target bounds or id"); single->callback([&]() { if (target_srs_input.empty()) { @@ -288,14 +276,14 @@ int run(std::span args) { std::unique_ptr tile_provider; if (texture_base_path.has_value()) { - BasemapSchemeTilePathProvider basemap_provider(texture_base_path.value()); + GoogleMapboxTilePathProvider basemap_provider(texture_base_path.value()); if (min_texture_level.has_value() || max_texture_level.has_value()) { - tile_provider = std::make_unique>( + tile_provider = std::make_unique>( std::move(basemap_provider), min_texture_level, max_texture_level); } else { - tile_provider = std::make_unique(std::move(basemap_provider)); + tile_provider = std::make_unique(std::move(basemap_provider)); } } @@ -305,7 +293,6 @@ int run(std::span args) { target_bounds_data, target_node_data, target_tile_data, - target_tile_scheme, target_srs); terrainbuilder::build_and_save_patch( diff --git a/src/sf_builder/texture_assembler.h b/src/sf_builder/texture_assembler.h index 90ee1ac2..d51d02bd 100644 --- a/src/sf_builder/texture_assembler.h +++ b/src/sf_builder/texture_assembler.h @@ -148,7 +148,6 @@ namespace terrainbuilder { const radix::tile::Id root_tile, const glm::uvec2 tile_image_pixel_size, const unsigned int max_zoom_level) { - tile = tile.to(radix::tile::Scheme::SlippyMap); const size_t relative_zoom_level = tile.zoom_level - root_tile.zoom_level; const glm::uvec2 tile_size_factor = glm::uvec2(std::pow(2, max_zoom_level - tile.zoom_level)); const glm::uvec2 tile_size = tile_image_pixel_size * tile_size_factor; @@ -379,7 +378,7 @@ inline std::optional try_get_tile_path(const radix::tile: // Start by transforming the input bounds into the srs the tiles are in. const radix::tile::SrsBounds encompassing_bounds = srs::encompassing_bounds_transfer(target_srs, grid.getSRS(), target_bounds); // Then we find the smallest tile (id) that encompasses these bounds. - const radix::tile::Id smallest_encompassing_tile = grid.findSmallestEncompassingTile(encompassing_bounds).value().to(radix::tile::Scheme::SlippyMap); + const radix::tile::Id smallest_encompassing_tile = grid.findSmallestEncompassingTile(encompassing_bounds).value(); LOG_TRACE("Smallest encompassing tile for texture bounds is {}", radix::tile::to_string(smallest_encompassing_tile)); if (max_zoom.has_value() && smallest_encompassing_tile.zoom_level > max_zoom.value()) { diff --git a/src/sf_builder/tile_provider.h b/src/sf_builder/tile_provider.h index da90c0e2..c709536d 100644 --- a/src/sf_builder/tile_provider.h +++ b/src/sf_builder/tile_provider.h @@ -6,6 +6,8 @@ #include #include +#include "tile_path.h" + class TileProvider { public: virtual ~TileProvider() = default; @@ -52,16 +54,13 @@ class StaticTileProvider : public TileProvider { public: std::unordered_map tiles; - StaticTileProvider(const std::unordered_map& tiles) { - // TODO: remove this once the == operator of tile::Id is updated. - for (const auto& tile : tiles) { - const radix::tile::Id tile_id = tile.first.to(radix::tile::Scheme::SlippyMap); - this->tiles[tile_id] = tile.second; - } + StaticTileProvider(const std::unordered_map& tiles) + : tiles(tiles) + { } virtual std::optional get_tile(const radix::tile::Id tile_id) const override { - const auto tile = this->tiles.find(tile_id.to(radix::tile::Scheme::SlippyMap)); + const auto tile = this->tiles.find(tile_id); if (tile != this->tiles.end()) { return tile->second; } else { @@ -70,7 +69,7 @@ class StaticTileProvider : public TileProvider { } virtual bool has_tile(const radix::tile::Id tile_id) const override { - return this->tiles.find(tile_id.to(radix::tile::Scheme::SlippyMap)) != this->tiles.cend(); + return this->tiles.find(tile_id) != this->tiles.cend(); } }; @@ -139,13 +138,13 @@ class ZoomRangeTileProvider final : public TileProvider { uint32_t _max_zoom; }; -class BasemapSchemeTilePathProvider : public TilePathProvider { +class GoogleMapboxTilePathProvider : public TilePathProvider { public: - BasemapSchemeTilePathProvider(std::filesystem::path base_path) + GoogleMapboxTilePathProvider(std::filesystem::path base_path) : base_path(base_path) {} std::optional get_tile_path(const radix::tile::Id tile_id) const override { - return base_path / std::to_string(tile_id.zoom_level) / std::to_string(tile_id.coords.y) / (std::to_string(tile_id.coords.x) + ".jpeg"); + return google_tile_path(base_path, tile_id, ".jpeg"); } private: diff --git a/src/terrainlib/CMakeLists.txt b/src/terrainlib/CMakeLists.txt index 2e321547..c799f067 100644 --- a/src/terrainlib/CMakeLists.txt +++ b/src/terrainlib/CMakeLists.txt @@ -48,6 +48,7 @@ add_library(terrainlib init.cpp log.cpp ProgressIndicator.cpp + tile_path.h ) target_compile_features(terrainlib PUBLIC cxx_std_23) diff --git a/src/terrainlib/ctb/GlobalGeodetic.hpp b/src/terrainlib/ctb/GlobalGeodetic.hpp index 5ddeecc5..917690b9 100644 --- a/src/terrainlib/ctb/GlobalGeodetic.hpp +++ b/src/terrainlib/ctb/GlobalGeodetic.hpp @@ -28,10 +28,11 @@ namespace ctb { /** - * @brief An implementation of the TMS Global Geodetic Profile + * @brief An implementation of the global Geodetic grid profile * * This class models the [Tile Mapping Service Global Geodetic * Profile](http://wiki.osgeo.org/wiki/Tile_Map_Service_Specification#global-geodetic). + * Its public tile identifiers use Google/Mapbox/XYZ coordinates. */ class GlobalGeodetic : public Grid { public: @@ -42,7 +43,7 @@ class GlobalGeodetic : public Grid { cSRS, 4326, // global geodetic has 2 root tiles: https://wiki.osgeo.org/wiki/Tile_Map_Service_Specification#global-geodetic - std::vector{radix::tile::Id{0, {0, 0}, radix::tile::Scheme::Tms}, radix::tile::Id{0, {1, 0}, radix::tile::Scheme::Tms}}, + std::vector{radix::tile::Id{0, {0, 0}}, radix::tile::Id{0, {1, 0}}}, 2) { } diff --git a/src/terrainlib/ctb/GlobalMercator.hpp b/src/terrainlib/ctb/GlobalMercator.hpp index 2da275a4..9de4af42 100644 --- a/src/terrainlib/ctb/GlobalMercator.hpp +++ b/src/terrainlib/ctb/GlobalMercator.hpp @@ -29,10 +29,11 @@ class GlobalMercator; } /** - * @brief An implementation of the TMS Global Mercator Profile + * @brief An implementation of the global Mercator grid profile * * This class models the [Tile Mapping Service Global Mercator * Profile](http://wiki.osgeo.org/wiki/Tile_Map_Service_Specification#global-mercator). + * Its public tile identifiers use Google/Mapbox/XYZ coordinates. */ class ctb::GlobalMercator : public Grid { public: @@ -41,7 +42,7 @@ class ctb::GlobalMercator : public Grid { radix::tile::SrsBounds{{-cOriginShift, -cOriginShift}, {cOriginShift, cOriginShift}}, cSRS, 3857, - std::vector{radix::tile::Id{0, {0, 0}, radix::tile::Scheme::Tms}}, + std::vector{radix::tile::Id{0, {0, 0}}}, 2) { } diff --git a/src/terrainlib/ctb/Grid.hpp b/src/terrainlib/ctb/Grid.hpp index a4338843..8d81488c 100644 --- a/src/terrainlib/ctb/Grid.hpp +++ b/src/terrainlib/ctb/Grid.hpp @@ -51,15 +51,8 @@ class Grid; * The code here generalises the logic in the `gdal2tiles.py` script available * with the GDAL library. * - * Warning: The y directino is dangerous. Sometimes the positve y axis points north, sometimes not. - * - GlobalMercator has y positive pointing north - * - GlobalGeodetic as well. - * - https://www.maptiler.com/google-maps-coordinates-tile-bounds-projection/#1/175.75/56.27 - * - google webmercator has tile coordinates where y=0 is the northern most tile. - * - tms webmercator has tile coordinates with y=0 being southern most. - * - * Effectively, ctb::Grid is always positive pointing north. Support for google webmercator / - * slippyMap is done in Tile.h + * Tile identifiers use Google/Mapbox/XYZ coordinates: the origin is north-west and y grows south. + * CRS and pixel coordinates inside Grid retain their conventional positive-north orientation. */ class ctb::Grid { public: @@ -143,23 +136,25 @@ class ctb::Grid { /// Get the tile coordinate in which a location falls at a specific zoom level [[nodiscard]] inline radix::tile::Id crsToTile(const CRSPoint &coord, i_zoom zoom) const { const PixelPoint pixel = crsToPixels(coord, zoom); - TilePoint tile = pixelsToTile(pixel); + const TilePoint tile = pixelsToTile(pixel); + const auto tile_count = i_tile(1u << zoom); - return {zoom, tile, radix::tile::Scheme::Tms}; + return {zoom, {tile.x, tile_count - tile.y - 1}}; } /// Get the CRS bounds of a particular tile /// border_se should be true if a border should be included on the south eastern corner /// e.g., for the cesium raster terrain format (https://github.com/CesiumGS/cesium/wiki/heightmap-1%2E0) [[nodiscard]] inline radix::tile::SrsBounds srsBounds(const radix::tile::Id &tile_id, bool border_se) const { - const auto tms_tile_id = tile_id.to(radix::tile::Scheme::Tms); + const auto tile_count = i_tile(1u << tile_id.zoom_level); + const auto grid_y = tile_count - tile_id.coords.y - 1; // get the pixels coordinates representing the tile bounds - const PixelPoint pxMinLeft(tms_tile_id.coords.x * mGridSize, tms_tile_id.coords.y * mGridSize); - const PixelPoint pxMaxRight((tms_tile_id.coords.x + 1) * mGridSize + border_se, (tms_tile_id.coords.y + 1) * mGridSize + border_se); + const PixelPoint pxMinLeft(tile_id.coords.x * mGridSize, grid_y * mGridSize); + const PixelPoint pxMaxRight((tile_id.coords.x + 1) * mGridSize + border_se, (grid_y + 1) * mGridSize + border_se); // convert pixels to native coordinates - const CRSPoint minLeft = pixelsToCrs(pxMinLeft, tms_tile_id.zoom_level); - const CRSPoint maxRight = pixelsToCrs(pxMaxRight, tms_tile_id.zoom_level); + const CRSPoint minLeft = pixelsToCrs(pxMinLeft, tile_id.zoom_level); + const CRSPoint maxRight = pixelsToCrs(pxMaxRight, tile_id.zoom_level); return { minLeft, maxRight }; } diff --git a/src/terrainlib/tile_path.h b/src/terrainlib/tile_path.h new file mode 100644 index 00000000..8f866d80 --- /dev/null +++ b/src/terrainlib/tile_path.h @@ -0,0 +1,14 @@ +#pragma once + +#include +#include + +#include + +[[nodiscard]] inline std::filesystem::path google_tile_path( + const std::filesystem::path& base_path, + const radix::tile::Id& tile_id, + const std::string& extension) +{ + return base_path / std::to_string(tile_id.zoom_level) / std::to_string(tile_id.coords.x) / (std::to_string(tile_id.coords.y) + extension); +} diff --git a/src/tile_builder/ParallelTileGenerator.cpp b/src/tile_builder/ParallelTileGenerator.cpp index 70924640..6a52a497 100644 --- a/src/tile_builder/ParallelTileGenerator.cpp +++ b/src/tile_builder/ParallelTileGenerator.cpp @@ -41,7 +41,7 @@ ParallelTileGenerator::ParallelTileGenerator(const std::string& input_data_path, } ParallelTileGenerator ParallelTileGenerator::make(const std::string& input_data_path, - ctb::Grid::Srs srs, radix::tile::Scheme tiling_scheme, + ctb::Grid::Srs srs, std::unique_ptr tile_writer, const std::string& output_data_path, unsigned grid_resolution) @@ -51,7 +51,7 @@ ParallelTileGenerator ParallelTileGenerator::make(const std::string& input_data_ if (srs == ctb::Grid::Srs::SphericalMercator) grid = ctb::GlobalMercator(grid_resolution); const auto border = tile_writer->formatRequiresBorder(); - return { input_data_path, grid, ParallelTiler(grid, dataset->bounds(grid.getSRS()), border, tiling_scheme), std::move(tile_writer), output_data_path }; + return { input_data_path, grid, ParallelTiler(grid, dataset->bounds(grid.getSRS()), border), std::move(tile_writer), output_data_path }; } const ParallelTiler& ParallelTileGenerator::tiler() const diff --git a/src/tile_builder/ParallelTileGenerator.h b/src/tile_builder/ParallelTileGenerator.h index f9fa61e0..cd6f6240 100644 --- a/src/tile_builder/ParallelTileGenerator.h +++ b/src/tile_builder/ParallelTileGenerator.h @@ -41,7 +41,7 @@ class ParallelTileGenerator { public: ParallelTileGenerator(const std::string& input_data_path, const ctb::Grid& grid, const ParallelTiler& tiler, std::unique_ptr tile_writer, const std::string& output_data_path); [[nodiscard]] static ParallelTileGenerator make(const std::string& input_data_path, - ctb::Grid::Srs srs, radix::tile::Scheme tiling_scheme, + ctb::Grid::Srs srs, std::unique_ptr tile_writer, const std::string& output_data_path, unsigned grid_resolution = 256); diff --git a/src/tile_builder/ParallelTiler.cpp b/src/tile_builder/ParallelTiler.cpp index 3f1c2958..fe6b2330 100644 --- a/src/tile_builder/ParallelTiler.cpp +++ b/src/tile_builder/ParallelTiler.cpp @@ -22,32 +22,31 @@ #include "Exception.h" #include -ParallelTiler::ParallelTiler(const ctb::Grid& grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border, radix::tile::Scheme scheme) : Tiler(grid, bounds, border, scheme) +ParallelTiler::ParallelTiler(const ctb::Grid& grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border) : Tiler(grid, bounds, border) { } radix::tile::Id ParallelTiler::southWestTile(unsigned zoom_level) const { - return grid().crsToTile(bounds().min, zoom_level).to(scheme()); + return grid().crsToTile(bounds().min, zoom_level); } radix::tile::Id ParallelTiler::northEastTile(unsigned zoom_level) const { const auto epsilon = grid().resolution(zoom_level) / 100; - return grid().crsToTile(bounds().max - epsilon, zoom_level).to(scheme()); + return grid().crsToTile(bounds().max - epsilon, zoom_level); } std::vector ParallelTiler::generateTiles(unsigned zoom_level) const { - // in the tms scheme south west corresponds to the smaller numbers. hence we can iterate from sw to ne - const auto sw = southWestTile(zoom_level).to(radix::tile::Scheme::Tms).coords; - const auto ne = northEastTile(zoom_level).to(radix::tile::Scheme::Tms).coords; + const auto sw = southWestTile(zoom_level).coords; + const auto ne = northEastTile(zoom_level).coords; std::vector tiles; - tiles.reserve((ne.y - sw.y + 1) * (ne.x - sw.x + 1)); - for (auto ty = sw.y; ty <= ne.y; ++ty) { + tiles.reserve((sw.y - ne.y + 1) * (ne.x - sw.x + 1)); + for (auto ty = ne.y; ty <= sw.y; ++ty) { for (auto tx = sw.x; tx <= ne.x; ++tx) { - const auto tile_id = radix::tile::Id { zoom_level, { tx, ty }, radix::tile::Scheme::Tms }.to(scheme()); + const auto tile_id = radix::tile::Id { zoom_level, { tx, ty } }; tiles.emplace_back(tile_for(tile_id)); if (tiles.size() >= 1'000'000'000) // think about creating an on the fly tile generator. storing so many tiles takes a lot of memory. diff --git a/src/tile_builder/ParallelTiler.h b/src/tile_builder/ParallelTiler.h index 7d01f0de..62cd07ac 100644 --- a/src/tile_builder/ParallelTiler.h +++ b/src/tile_builder/ParallelTiler.h @@ -23,7 +23,7 @@ class ParallelTiler : public Tiler { public: - ParallelTiler(const ctb::Grid& grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border, radix::tile::Scheme scheme); + ParallelTiler(const ctb::Grid& grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border); [[nodiscard]] std::vector generateTiles(unsigned zoom_level) const; [[nodiscard]] std::vector generateTiles(const std::pair& zoom_range) const; diff --git a/src/tile_builder/TileHeightsGenerator.cpp b/src/tile_builder/TileHeightsGenerator.cpp index 6c816dde..0e535fab 100644 --- a/src/tile_builder/TileHeightsGenerator.cpp +++ b/src/tile_builder/TileHeightsGenerator.cpp @@ -28,10 +28,9 @@ #include "depth_first_tile_traverser.h" #include -TileHeightsGenerator::TileHeightsGenerator(std::string input_data_path, ctb::Grid::Srs srs, radix::tile::Scheme scheme, radix::tile::Border border, std::filesystem::path output_path) +TileHeightsGenerator::TileHeightsGenerator(std::string input_data_path, ctb::Grid::Srs srs, radix::tile::Border border, std::filesystem::path output_path) : m_input_data_path(std::move(input_data_path)) , m_srs(srs) - , m_scheme(scheme) , m_border(border) , m_output_path(std::move(output_path)) { @@ -51,7 +50,7 @@ void TileHeightsGenerator::run(unsigned max_zoom_level) const grid = ctb::GlobalMercator(64); const auto bounds = dataset->bounds(grid.getSRS()); const auto tile_reader = DatasetReader(dataset, grid.getSRS(), 1, false); - const auto tiler = TopDownTiler(grid, bounds, m_border, m_scheme); + const auto tiler = TopDownTiler(grid, bounds, m_border); auto tile_heights = radix::TileHeights(); const auto read_function = [&](const radix::tile::Descriptor& tile) -> MinMaxData { @@ -74,10 +73,10 @@ void TileHeightsGenerator::run(unsigned max_zoom_level) const }; - traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, { 0, { 0, 0 }, m_scheme }, max_zoom_level); + traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, { 0, { 0, 0 } }, max_zoom_level); if (m_srs == ctb::Grid::Srs::WGS84) { // two root tiles - traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, { 0, { 1, 0 }, m_scheme }, max_zoom_level); + traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, { 0, { 1, 0 } }, max_zoom_level); } tile_heights.write_to(m_output_path); diff --git a/src/tile_builder/TileHeightsGenerator.h b/src/tile_builder/TileHeightsGenerator.h index 0d64af56..703a1227 100644 --- a/src/tile_builder/TileHeightsGenerator.h +++ b/src/tile_builder/TileHeightsGenerator.h @@ -29,11 +29,9 @@ class TileHeightsGenerator { std::string m_input_data_path; ctb::Grid::Srs m_srs; - radix::tile::Scheme m_scheme; radix::tile::Border m_border; std::filesystem::path m_output_path; public: - TileHeightsGenerator(std::string input_data_path, ctb::Grid::Srs srs, radix::tile::Scheme scheme, radix::tile::Border border, std::filesystem::path output_path); + TileHeightsGenerator(std::string input_data_path, ctb::Grid::Srs srs, radix::tile::Border border, std::filesystem::path output_path); void run(unsigned max_zoom_level) const; }; - diff --git a/src/tile_builder/Tiler.cpp b/src/tile_builder/Tiler.cpp index 00153c6c..fafa3ba7 100644 --- a/src/tile_builder/Tiler.cpp +++ b/src/tile_builder/Tiler.cpp @@ -20,11 +20,10 @@ #include -Tiler::Tiler(ctb::Grid grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border, radix::tile::Scheme scheme) +Tiler::Tiler(ctb::Grid grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border) : m_grid(std::move(grid)) , m_bounds(bounds) , m_border_south_east(border) - , m_scheme(scheme) { } @@ -55,11 +54,6 @@ radix::tile::Descriptor Tiler::tile_for(const radix::tile::Id& tile_id) const return {tile_id, srs_bounds, grid().getEpsgCode(), grid_size(), tile_size()}; } -radix::tile::Scheme Tiler::scheme() const -{ - return m_scheme; -} - const radix::tile::SrsBounds& Tiler::bounds() const { return m_bounds; @@ -69,4 +63,3 @@ void Tiler::setBounds(const radix::tile::SrsBounds& newBounds) { m_bounds = newBounds; } - diff --git a/src/tile_builder/Tiler.h b/src/tile_builder/Tiler.h index d477ce3b..70b5049e 100644 --- a/src/tile_builder/Tiler.h +++ b/src/tile_builder/Tiler.h @@ -25,9 +25,8 @@ class Tiler { public: - Tiler(ctb::Grid grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border, radix::tile::Scheme scheme); + Tiler(ctb::Grid grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border); - [[nodiscard]] radix::tile::Scheme scheme() const; [[nodiscard]] const radix::tile::SrsBounds& bounds() const; void setBounds(const radix::tile::SrsBounds& newBounds); [[nodiscard]] radix::tile::Descriptor tile_for(const radix::tile::Id& tile_id) const; @@ -43,6 +42,4 @@ class Tiler const ctb::Grid m_grid; radix::tile::SrsBounds m_bounds; const radix::tile::Border m_border_south_east; - const radix::tile::Scheme m_scheme; }; - diff --git a/src/tile_builder/TopDownTiler.cpp b/src/tile_builder/TopDownTiler.cpp index 9962962e..ca5ead02 100644 --- a/src/tile_builder/TopDownTiler.cpp +++ b/src/tile_builder/TopDownTiler.cpp @@ -18,15 +18,14 @@ #include "TopDownTiler.h" -TopDownTiler::TopDownTiler(const ctb::Grid& grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border, radix::tile::Scheme scheme) - : Tiler(grid, bounds, border, scheme) +TopDownTiler::TopDownTiler(const ctb::Grid& grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border) + : Tiler(grid, bounds, border) { } std::vector TopDownTiler::generateTiles(const radix::tile::Id& parent_id) const { - assert(parent_id.scheme == scheme()); - const auto tile_ids = parent_id.to(scheme()).children(); + const auto tile_ids = parent_id.children(); std::vector tiles; for (const auto& tile_id : tile_ids) { radix::tile::Descriptor t = tile_for(tile_id); diff --git a/src/tile_builder/TopDownTiler.h b/src/tile_builder/TopDownTiler.h index a2e065d1..f078ca89 100644 --- a/src/tile_builder/TopDownTiler.h +++ b/src/tile_builder/TopDownTiler.h @@ -24,7 +24,7 @@ class TopDownTiler : public Tiler { public: - TopDownTiler(const ctb::Grid& grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border, radix::tile::Scheme scheme); + TopDownTiler(const ctb::Grid& grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border); [[nodiscard]] std::vector generateTiles(const radix::tile::Id& parent_id) const; }; diff --git a/src/tile_builder/alpine_raster.cpp b/src/tile_builder/alpine_raster.cpp index be4e0e1d..c5d8a6b0 100644 --- a/src/tile_builder/alpine_raster.cpp +++ b/src/tile_builder/alpine_raster.cpp @@ -31,9 +31,9 @@ #include #include -ParallelTileGenerator alpine_raster::make_generator(const std::string& input_data_path, const std::string& output_data_path, ctb::Grid::Srs srs, radix::tile::Scheme tiling_scheme, radix::tile::Border border, unsigned grid_resolution) +ParallelTileGenerator alpine_raster::make_generator(const std::string& input_data_path, const std::string& output_data_path, ctb::Grid::Srs srs, radix::tile::Border border, unsigned grid_resolution) { - return ParallelTileGenerator::make(input_data_path, srs, tiling_scheme, std::make_unique(border), output_data_path, grid_resolution); + return ParallelTileGenerator::make(input_data_path, srs, std::make_unique(border), output_data_path, grid_resolution); } void alpine_raster::TileWriter::write(const std::string& file_path, const radix::tile::Descriptor&, const radix::Raster& heights) const diff --git a/src/tile_builder/alpine_raster.h b/src/tile_builder/alpine_raster.h index 22eee044..d82f05a6 100644 --- a/src/tile_builder/alpine_raster.h +++ b/src/tile_builder/alpine_raster.h @@ -44,7 +44,6 @@ class TileWriter : public ParallelTileWriterInterface { const std::string& input_data_path, const std::string& output_data_path, ctb::Grid::Srs srs, - radix::tile::Scheme tiling_scheme, radix::tile::Border border, unsigned grid_resolution = 256); }; diff --git a/src/tile_builder/main.cpp b/src/tile_builder/main.cpp index c2f0fe05..06ec1c93 100644 --- a/src/tile_builder/main.cpp +++ b/src/tile_builder/main.cpp @@ -12,28 +12,26 @@ int main() { // const std::string input_raster = "/home/madam/rajaton/raw/Oe_2020/OeRect_01m_gs_31287.img"; // const std::string output_path = "/home/madam/rajaton/tiles/atb_terrain/"; - //// const auto generator = alpine_raster::make_generator("./test_tiles/", "/home/madam/valtava/raw/Oe_2020/OeRect_01m_gs_31287.img", ctb::Grid::Srs::SphericalMercator, Tiler::Scheme::SlippyMap, Tiler::Border::No); + //// const auto generator = alpine_raster::make_generator("./test_tiles/", "/home/madam/valtava/raw/Oe_2020/OeRect_01m_gs_31287.img", ctb::Grid::Srs::SphericalMercator, radix::tile::Border::No); //// generator.process({16, 16}); - // const auto generator = cesium_tin_terra::make_generator(input_raster, output_path, ctb::Grid::Srs::WGS84, Tiler::Scheme::Tms, Tiler::Border::Yes); // generator.process({0, 5}, true, true); // generator.process({6, 16}, true, false); const std::string input_raster = "/home/madam/valtava/raw/Oe_2020/OeRect_01m_gs_31287.img"; // const std::string input_raster = "/home/madam/valtava/raw/vienna/innenstadt_gs_1m_mgi.tif"; const std::string output_path = "/home/madam/valtava/tiles/alpine_png2"; - // const auto generator = alpine_raster::make_generator("./test_tiles/", "/home/madam/valtava/raw/Oe_2020/OeRect_01m_gs_31287.img", ctb::Grid::Srs::SphericalMercator, Tiler::Scheme::SlippyMap, Tiler::Border::No); + // const auto generator = alpine_raster::make_generator("./test_tiles/", "/home/madam/valtava/raw/Oe_2020/OeRect_01m_gs_31287.img", ctb::Grid::Srs::SphericalMercator, radix::tile::Border::No); // generator.process({16, 16}); - const auto generator = alpine_raster::make_generator(input_raster, output_path, ctb::Grid::Srs::SphericalMercator, radix::tile::Scheme::Tms, radix::tile::Border::Yes, 64); + const auto generator = alpine_raster::make_generator(input_raster, output_path, ctb::Grid::Srs::SphericalMercator, radix::tile::Border::Yes, 64); // generator.process({0, 5}, true, true); generator.process({ 15, 16 }, true, false); - // const auto metadata = MetaDataGenerator::make(input_raster, ctb::Grid::Srs::WGS84, Tiler::Scheme::Tms); // const auto json = layer_json_writer::process(metadata); //// generate height data (min and max) for tiles up to level 13 // const auto base_path = std::filesystem::path(output_path); // constexpr auto file_name = "height_data.atb"; -// const auto generator = TileHeightsGenerator(input_raster, ctb::Grid::Srs::SphericalMercator, radix::tile::Scheme::Tms, radix::tile::Border::Yes, base_path / file_name); +// const auto generator = TileHeightsGenerator(input_raster, ctb::Grid::Srs::SphericalMercator, radix::tile::Border::Yes, base_path / file_name); // generator.run(13); return 0; diff --git a/src/tile_downloader/TileDownloader.h b/src/tile_downloader/TileDownloader.h index ceb5103b..94e24af9 100644 --- a/src/tile_downloader/TileDownloader.h +++ b/src/tile_downloader/TileDownloader.h @@ -12,13 +12,14 @@ #include "HttpClient.h" #include "TileLogger.h" #include "TileUrlBuilder.h" +#include "tile_path.h" class TileDownloader { public: - TileDownloader(const TileUrlBuilder &url_builder, std::string output_pattern, + TileDownloader(const TileUrlBuilder &url_builder, std::filesystem::path output_directory, bool early_skip, std::optional max_zoom_level) : _url_builder(url_builder), - _output_pattern(std::move(output_pattern)), + _output_directory(std::move(output_directory)), _early_skip(early_skip), _max_zoom_level(max_zoom_level) {} @@ -51,7 +52,7 @@ class TileDownloader { private: const TileUrlBuilder &_url_builder; - std::string _output_pattern; + std::filesystem::path _output_directory; HttpClient _http; TileLogger _logger; bool _early_skip; @@ -69,21 +70,8 @@ class TileDownloader { return std::filesystem::exists(this->tile_path(tile)); } - std::string tile_path(const radix::tile::Id &tile) const { - std::string path = this->_output_pattern; - replace_all(path, "{zoom}", std::to_string(tile.zoom_level)); - replace_all(path, "{x}", std::to_string(tile.coords.x)); - replace_all(path, "{y}", std::to_string(tile.coords.y)); - replace_all(path, "{ext}", "jpeg"); - return path; - } - - static void replace_all(std::string &s, std::string_view find, std::string_view replace) { - size_t pos = 0; - while ((pos = s.find(find, pos)) != std::string::npos) { - s.replace(pos, find.length(), replace); - pos += replace.length(); - } + std::filesystem::path tile_path(const radix::tile::Id &tile) const { + return google_tile_path(_output_directory, tile, ".jpeg"); } static void ensure_parent_dirs(const std::filesystem::path &path) { diff --git a/src/tile_downloader/TileUrlBuilder.h b/src/tile_downloader/TileUrlBuilder.h index 328cf110..2e0b22d3 100644 --- a/src/tile_downloader/TileUrlBuilder.h +++ b/src/tile_downloader/TileUrlBuilder.h @@ -1,10 +1,42 @@ #pragma once +#include +#include #include +#include #include #include +enum class TileCoordinateOrder { + Xy, + Yx +}; + +enum class TileYDirection { + Down, + Up +}; + +struct TileUrlFormat { + TileCoordinateOrder coordinate_order = TileCoordinateOrder::Xy; + TileYDirection y_direction = TileYDirection::Down; + + [[nodiscard]] std::pair coordinates(const radix::tile::Id& tile_id) const + { + auto y = tile_id.coords.y; + if (y_direction == TileYDirection::Up) { + if (tile_id.zoom_level >= std::numeric_limits::digits) + throw std::invalid_argument("tile zoom level is too large for legacy TMS coordinates"); + y = (1u << tile_id.zoom_level) - y - 1; + } + + if (coordinate_order == TileCoordinateOrder::Xy) + return { tile_id.coords.x, y }; + return { y, tile_id.coords.x }; + } +}; + class TileUrlBuilder { public: virtual ~TileUrlBuilder() = default; @@ -13,27 +45,40 @@ class TileUrlBuilder { class BasemapTileUrlBuilder : public TileUrlBuilder { public: - BasemapTileUrlBuilder(std::string layer, std::string style) - : _layer(std::move(layer)), _style(std::move(style)) {} + BasemapTileUrlBuilder(std::string layer, std::string style, TileUrlFormat format = {}) + : _layer(std::move(layer)) + , _style(std::move(style)) + , _format(format) + { + } std::string build_url(const radix::tile::Id &tile_id) const override { - const auto t = tile_id.to(radix::tile::Scheme::SlippyMap); + const auto [first_coordinate, second_coordinate] = _format.coordinates(tile_id); return fmt::format( "https://mapsneu.wien.gv.at/basemap/{}/{}/google3857/{}/{}/{}.jpeg", - this->_layer, this->_style, t.zoom_level, t.coords.y, t.coords.x); + _layer, _style, tile_id.zoom_level, first_coordinate, second_coordinate); } private: std::string _layer; std::string _style; + TileUrlFormat _format; }; class GatakiTileUrlBuilder : public TileUrlBuilder { public: + explicit GatakiTileUrlBuilder(TileUrlFormat format = {}) + : _format(format) + { + } + std::string build_url(const radix::tile::Id &tile_id) const override { - const auto t = tile_id.to(radix::tile::Scheme::SlippyMap); + const auto [first_coordinate, second_coordinate] = _format.coordinates(tile_id); return fmt::format( "https://gataki.cg.tuwien.ac.at/raw/basemap/tiles/{}/{}/{}.jpeg", - t.zoom_level, t.coords.y, t.coords.x); + tile_id.zoom_level, first_coordinate, second_coordinate); } + +private: + TileUrlFormat _format; }; diff --git a/src/tile_downloader/cli.cpp b/src/tile_downloader/cli.cpp index b83d4c36..4c5acb95 100644 --- a/src/tile_downloader/cli.cpp +++ b/src/tile_downloader/cli.cpp @@ -18,24 +18,30 @@ Args parse(int argc, const char *const *argv) { ->check(CLI::IsMember({"basemap", "gataki"}, CLI::ignore_case)); app.add_option("--zoom", args.zoom, "Root tile zoom level")->required(); - app.add_option("--x,--row", args.x, "Root tile x coordinate")->required(); - app.add_option("--y,--col", args.y, "Root tile y coordinate")->required(); - - const std::map scheme_map{ - {"slippymap", radix::tile::Scheme::SlippyMap}, - {"google", radix::tile::Scheme::SlippyMap}, - {"xyz", radix::tile::Scheme::SlippyMap}, - {"tms", radix::tile::Scheme::Tms}}; - args.scheme = radix::tile::Scheme::SlippyMap; - app.add_option("--scheme", args.scheme, "Tile scheme") - ->default_val(radix::tile::Scheme::SlippyMap) - ->transform(CLI::CheckedTransformer(scheme_map, CLI::ignore_case)); + app.add_option("--x,--col", args.x, "Root tile x/column in Google/Mapbox coordinates")->required(); + app.add_option("--y,--row", args.y, "Root tile y/row in Google/Mapbox coordinates")->required(); + + const std::map coordinate_order_map{ + {"xy", TileCoordinateOrder::Xy}, + {"yx", TileCoordinateOrder::Yx}}; + args.url_coordinate_order = TileCoordinateOrder::Xy; + app.add_option("--url-coordinate-order", args.url_coordinate_order, "URL coordinate order: xy (common Google/Mapbox format) or yx") + ->default_str("xy") + ->transform(CLI::CheckedTransformer(coordinate_order_map, CLI::ignore_case)); + + const std::map y_direction_map{ + {"down", TileYDirection::Down}, + {"up", TileYDirection::Up}}; + args.url_y_direction = TileYDirection::Down; + app.add_option("--url-y-direction", args.url_y_direction, "URL y direction: down (common Google/Mapbox format) or up (legacy TMS)") + ->default_str("down") + ->transform(CLI::CheckedTransformer(y_direction_map, CLI::ignore_case)); args.srs = 3857; app.add_option("--srs", args.srs, "Spatial reference system EPSG code")->default_val(3857); - args.output = "tiles/{zoom}/{y}/{x}.{ext}"; - app.add_option("--output", args.output, "Output path template")->default_val(args.output); + args.output = "tiles"; + app.add_option("--output", args.output, "Output directory; files use Google/Mapbox zoom/x/y.jpeg layout")->default_val(args.output.string()); const std::map log_level_names{ {"off", spdlog::level::off}, diff --git a/src/tile_downloader/cli.h b/src/tile_downloader/cli.h index 03edd30f..bc667f98 100644 --- a/src/tile_downloader/cli.h +++ b/src/tile_downloader/cli.h @@ -7,6 +7,8 @@ #include #include +#include "TileUrlBuilder.h" + namespace cli { struct Args { @@ -14,9 +16,10 @@ struct Args { unsigned int zoom; unsigned int x; unsigned int y; - radix::tile::Scheme scheme; + TileCoordinateOrder url_coordinate_order; + TileYDirection url_y_direction; unsigned int srs; - std::string output; + std::filesystem::path output; spdlog::level::level_enum log_level; bool early_skip; std::optional max_zoom_level; diff --git a/src/tile_downloader/main.cpp b/src/tile_downloader/main.cpp index 79102590..b2973a29 100644 --- a/src/tile_downloader/main.cpp +++ b/src/tile_downloader/main.cpp @@ -20,12 +20,12 @@ int main(int argc, char *argv[]) { std::transform(provider.begin(), provider.end(), provider.begin(), [](unsigned char c) { return std::tolower(c); }); if (provider == "basemap") { - url_builder = std::make_unique(args.layer, args.style); + url_builder = std::make_unique(args.layer, args.style, TileUrlFormat { args.url_coordinate_order, args.url_y_direction }); } else { - url_builder = std::make_unique(); + url_builder = std::make_unique(TileUrlFormat { args.url_coordinate_order, args.url_y_direction }); } - const radix::tile::Id root_id = {args.zoom, {args.x, args.y}, args.scheme}; + const radix::tile::Id root_id = {args.zoom, {args.x, args.y}}; TileDownloader downloader(*url_builder, args.output, args.early_skip, args.max_zoom_level); downloader.download_recursive(root_id); diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 647863fe..909360fb 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -86,6 +86,15 @@ if(TARGET sfbuilderlib) atb_configure_test(unittests_sfbuilder) endif() +if(TARGET tile-downloader) + add_executable(unittests_tile_downloader + tile_downloader/url_builder.cpp + ) + target_include_directories(unittests_tile_downloader PRIVATE ${CMAKE_SOURCE_DIR}/src/tile_downloader) + target_link_libraries(unittests_tile_downloader PRIVATE terrainlib Catch2::Catch2WithMain) + atb_configure_test(unittests_tile_downloader) +endif() + if(TARGET dagbuilderlib) add_executable(unittests_dagbuilder catch2_helpers.h diff --git a/unittests/sf_builder/texture.cpp b/unittests/sf_builder/texture.cpp index 4f33e016..b32188d4 100644 --- a/unittests/sf_builder/texture.cpp +++ b/unittests/sf_builder/texture.cpp @@ -34,7 +34,7 @@ TEST_CASE("estimate_zoom_level", "[terrainbuilder]") { const ctb::Grid grid = ctb::GlobalMercator(); - const radix::tile::Id tile(20, glm::uvec2(0, 1), radix::tile::Scheme::SlippyMap); + const radix::tile::Id tile(20, glm::uvec2(0, 1)); const radix::tile::SrsBounds tile_bounds = grid.srsBounds(tile, false); const radix::tile::SrsBounds shifted_bounds(tile_bounds.min + glm::dvec2(-100, 420), tile_bounds.max + glm::dvec2(-100, 420)); @@ -67,11 +67,11 @@ class AlwaysEmptyTileProvider : public TileProvider { }; TEST_CASE("texture assembler takes root tile if only available ", "[terrainbuilder]") { - const radix::tile::Id root_tile(3, glm::uvec2(5, 4), radix::tile::Scheme::SlippyMap); + const radix::tile::Id root_tile(3, glm::uvec2(5, 4)); const std::set available_tiles = { - {3, {5, 4}, radix::tile::Scheme::SlippyMap}}; + {3, {5, 4}}}; const std::set expected_tiles = { - {3, {5, 4}, radix::tile::Scheme::SlippyMap}}; + {3, {5, 4}}}; const AvailabilityListEmptyTileProvider tile_provider(available_tiles); @@ -88,18 +88,18 @@ TEST_CASE("texture assembler takes root tile if only available ", "[terrainbuild } TEST_CASE("texture assembler ignores parent if all children are present", "[terrainbuilder]") { - const radix::tile::Id root_tile(3, glm::uvec2(5, 4), radix::tile::Scheme::SlippyMap); + const radix::tile::Id root_tile(3, glm::uvec2(5, 4)); const std::set available_tiles = { - {3, {5, 4}, radix::tile::Scheme::SlippyMap}, - {4, {10, 8}, radix::tile::Scheme::SlippyMap}, - {4, {11, 8}, radix::tile::Scheme::SlippyMap}, - {4, {10, 9}, radix::tile::Scheme::SlippyMap}, - {4, {11, 9}, radix::tile::Scheme::SlippyMap}}; + {3, {5, 4}}, + {4, {10, 8}}, + {4, {11, 8}}, + {4, {10, 9}}, + {4, {11, 9}}}; const std::set expected_tiles = { - {4, {10, 8}, radix::tile::Scheme::SlippyMap}, - {4, {11, 8}, radix::tile::Scheme::SlippyMap}, - {4, {10, 9}, radix::tile::Scheme::SlippyMap}, - {4, {11, 9}, radix::tile::Scheme::SlippyMap}}; + {4, {10, 8}}, + {4, {11, 8}}, + {4, {10, 9}}, + {4, {11, 9}}}; const AvailabilityListEmptyTileProvider tile_provider(available_tiles); @@ -116,15 +116,15 @@ TEST_CASE("texture assembler ignores parent if all children are present", "[terr } TEST_CASE("texture assembler considers max zoom level", "[terrainbuilder]") { - const radix::tile::Id root_tile(3, glm::uvec2(5, 4), radix::tile::Scheme::SlippyMap); + const radix::tile::Id root_tile(3, glm::uvec2(5, 4)); const std::set available_tiles = { - {3, {5, 4}, radix::tile::Scheme::SlippyMap}, - {4, {10, 8}, radix::tile::Scheme::SlippyMap}, - {4, {11, 8}, radix::tile::Scheme::SlippyMap}, - {4, {10, 9}, radix::tile::Scheme::SlippyMap}, - {4, {11, 9}, radix::tile::Scheme::SlippyMap}}; + {3, {5, 4}}, + {4, {10, 8}}, + {4, {11, 8}}, + {4, {10, 9}}, + {4, {11, 9}}}; const std::set expected_tiles = { - {3, {5, 4}, radix::tile::Scheme::SlippyMap}}; + {3, {5, 4}}}; const AvailabilityListEmptyTileProvider tile_provider(available_tiles); @@ -143,29 +143,29 @@ TEST_CASE("texture assembler considers max zoom level", "[terrainbuilder]") { TEST_CASE("texture assembler works for arbitrary bounds", "[terrainbuilder]") { const std::set available_tiles = { - // {21, {1048576, 1048576}, radix::tile::Scheme::Tms}, - {21, {1048577, 1048576}, radix::tile::Scheme::Tms}, - {21, {1048578, 1048576}, radix::tile::Scheme::Tms}, - {21, {1048579, 1048576}, radix::tile::Scheme::Tms}, - {21, {1048580, 1048576}, radix::tile::Scheme::Tms}, - // {21, {1048581, 1048576}, radix::tile::Scheme::Tms}, - {20, {524288, 524288}, radix::tile::Scheme::Tms}, - {20, {524289, 524288}, radix::tile::Scheme::Tms}, - {20, {524290, 524288}, radix::tile::Scheme::Tms}, - // {19, {262144, 262144}, radix::tile::Scheme::Tms}, - {19, {262145, 262144}, radix::tile::Scheme::Tms}}; + // {21, {1048576, 1048575}}, + {21, {1048577, 1048575}}, + {21, {1048578, 1048575}}, + {21, {1048579, 1048575}}, + {21, {1048580, 1048575}}, + // {21, {1048581, 1048575}}, + {20, {524288, 524287}}, + {20, {524289, 524287}}, + {20, {524290, 524287}}, + // {19, {262144, 262143}}, + {19, {262145, 262143}}}; const std::set expected_tiles = { - // {21, {1048576, 1048576}, radix::tile::Scheme::Tms}, - {21, {1048577, 1048576}, radix::tile::Scheme::Tms}, - {21, {1048578, 1048576}, radix::tile::Scheme::Tms}, - {21, {1048579, 1048576}, radix::tile::Scheme::Tms}, - {21, {1048580, 1048576}, radix::tile::Scheme::Tms}, - // {21, {1048581, 1048576}, radix::tile::Scheme::Tms}, - {20, {524288, 524288}, radix::tile::Scheme::Tms}, - // {20, {524289, 524288}, radix::tile::Scheme::Tms}, - {20, {524290, 524288}, radix::tile::Scheme::Tms}, - // {19, {262144, 262144}, radix::tile::Scheme::Tms}, - // {19, {262145, 262144}, radix::tile::Scheme::Tms} + // {21, {1048576, 1048575}}, + {21, {1048577, 1048575}}, + {21, {1048578, 1048575}}, + {21, {1048579, 1048575}}, + {21, {1048580, 1048575}}, + // {21, {1048581, 1048575}}, + {20, {524288, 524287}}, + // {20, {524289, 524287}}, + {20, {524290, 524287}}, + // {19, {262144, 262143}}, + // {19, {262145, 262143}} }; const AvailabilityListEmptyTileProvider tile_provider(available_tiles); @@ -208,7 +208,7 @@ TEST_CASE("texture assembler does not fail if there are not tiles", "[terrainbui TEST_CASE("texture assembler assembles single tile", "[terrainbuilder]") { const std::unordered_map tiles_to_texture = { - {radix::tile::Id(0, {0, 0}, radix::tile::Scheme::SlippyMap), cv::Mat(1, 1, CV_8UC3, cv::Vec3b(0, 0, 255))}, + {radix::tile::Id(0, {0, 0}), cv::Mat(1, 1, CV_8UC3, cv::Vec3b(0, 0, 255))}, }; std::vector tiles_to_splatter; std::transform(tiles_to_texture.begin(), tiles_to_texture.end(), std::back_inserter(tiles_to_splatter), @@ -217,7 +217,7 @@ TEST_CASE("texture assembler assembles single tile", "[terrainbuilder]") { const StaticTileProvider tile_provider(tiles_to_texture); const ctb::Grid grid = ctb::GlobalMercator(); - const radix::tile::Id root_tile(0, {0, 0}, radix::tile::Scheme::SlippyMap); + const radix::tile::Id root_tile(0, {0, 0}); cv::Mat assembled_texture = terrainbuilder::splatter_tiles_to_texture( root_tile, grid, @@ -234,8 +234,8 @@ TEST_CASE("texture assembler assembles single tile", "[terrainbuilder]") { TEST_CASE("texture assembler assembles two tiles", "[terrainbuilder]") { const std::unordered_map tiles_to_texture = { - {radix::tile::Id(1, {0, 0}, radix::tile::Scheme::SlippyMap), cv::Mat(1, 1, CV_8UC3, cv::Vec3b(0, 0, 255))}, - {radix::tile::Id(1, {0, 1}, radix::tile::Scheme::SlippyMap), cv::Mat(1, 1, CV_8UC3, cv::Vec3b(0, 255, 0))}, + {radix::tile::Id(1, {0, 0}), cv::Mat(1, 1, CV_8UC3, cv::Vec3b(0, 0, 255))}, + {radix::tile::Id(1, {0, 1}), cv::Mat(1, 1, CV_8UC3, cv::Vec3b(0, 255, 0))}, }; std::vector tiles_to_splatter; std::transform(tiles_to_texture.begin(), tiles_to_texture.end(), std::back_inserter(tiles_to_splatter), @@ -244,7 +244,7 @@ TEST_CASE("texture assembler assembles two tiles", "[terrainbuilder]") { const StaticTileProvider tile_provider(tiles_to_texture); const ctb::Grid grid = ctb::GlobalMercator(); - const radix::tile::Id root_tile(0, {0, 0}, radix::tile::Scheme::SlippyMap); + const radix::tile::Id root_tile(0, {0, 0}); cv::Mat assembled_texture = terrainbuilder::splatter_tiles_to_texture( root_tile, grid, @@ -262,9 +262,9 @@ TEST_CASE("texture assembler assembles two tiles", "[terrainbuilder]") { TEST_CASE("texture assembler correct order of texture writes", "[terrainbuilder]") { const std::unordered_map tiles_to_texture = { - {radix::tile::Id(0, {0, 0}, radix::tile::Scheme::SlippyMap), cv::Mat(1, 1, CV_8UC1, uint8_t(1))}, - {radix::tile::Id(1, {0, 1}, radix::tile::Scheme::Tms), cv::Mat(1, 1, CV_8UC1, uint8_t(2))}, - {radix::tile::Id(1, {0, 1}, radix::tile::Scheme::SlippyMap), cv::Mat(1, 1, CV_8UC1, uint8_t(3))}, + {radix::tile::Id(0, {0, 0}), cv::Mat(1, 1, CV_8UC1, uint8_t(1))}, + {radix::tile::Id(1, {0, 0}), cv::Mat(1, 1, CV_8UC1, uint8_t(2))}, + {radix::tile::Id(1, {0, 1}), cv::Mat(1, 1, CV_8UC1, uint8_t(3))}, }; std::vector tiles_to_splatter; std::transform(tiles_to_texture.begin(), tiles_to_texture.end(), std::back_inserter(tiles_to_splatter), @@ -275,7 +275,7 @@ TEST_CASE("texture assembler correct order of texture writes", "[terrainbuilder] const StaticTileProvider tile_provider(tiles_to_texture); const ctb::Grid grid = ctb::GlobalMercator(); - const radix::tile::Id root_tile(0, {0, 0}, radix::tile::Scheme::SlippyMap); + const radix::tile::Id root_tile(0, {0, 0}); cv::Mat assembled_texture = terrainbuilder::splatter_tiles_to_texture( root_tile, grid, diff --git a/unittests/tile_downloader/url_builder.cpp b/unittests/tile_downloader/url_builder.cpp new file mode 100644 index 00000000..dd4ade75 --- /dev/null +++ b/unittests/tile_downloader/url_builder.cpp @@ -0,0 +1,45 @@ +#include + +#include "TileUrlBuilder.h" +#include "tile_path.h" + +namespace { +constexpr radix::tile::Id tile { 3, { 1, 2 } }; + +void check_urls(const TileUrlFormat format, const std::string& coordinate_path) +{ + const BasemapTileUrlBuilder basemap("layer", "style", format); + CHECK(basemap.build_url(tile) == "https://mapsneu.wien.gv.at/basemap/layer/style/google3857/" + coordinate_path + ".jpeg"); + + const GatakiTileUrlBuilder gataki(format); + CHECK(gataki.build_url(tile) == "https://gataki.cg.tuwien.ac.at/raw/basemap/tiles/" + coordinate_path + ".jpeg"); +} +} + +TEST_CASE("tile URL coordinate formats") +{ + SECTION("xy with downward y") + { + check_urls({ TileCoordinateOrder::Xy, TileYDirection::Down }, "3/1/2"); + } + + SECTION("yx with downward y") + { + check_urls({ TileCoordinateOrder::Yx, TileYDirection::Down }, "3/2/1"); + } + + SECTION("xy with upward legacy TMS y") + { + check_urls({ TileCoordinateOrder::Xy, TileYDirection::Up }, "3/1/5"); + } + + SECTION("yx with upward legacy TMS y") + { + check_urls({ TileCoordinateOrder::Yx, TileYDirection::Up }, "3/5/1"); + } +} + +TEST_CASE("downloaded tile path uses Google and Mapbox layout") +{ + CHECK(google_tile_path("tiles", tile, ".jpeg") == std::filesystem::path("tiles/3/1/2.jpeg")); +} diff --git a/unittests/tilebuilder/alpine_raster_format.cpp b/unittests/tilebuilder/alpine_raster_format.cpp index 5c5cafd4..a0d98223 100644 --- a/unittests/tilebuilder/alpine_raster_format.cpp +++ b/unittests/tilebuilder/alpine_raster_format.cpp @@ -41,7 +41,7 @@ TEMPLATE_TEST_CASE("alpine raster format, border ", "", std::true_type, std::fal SECTION("raste write") { - const auto generator = alpine_raster::make_generator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", "./unittest_tiles/", ctb::Grid::Srs::SphericalMercator, radix::tile::Scheme::Tms, radix::tile::Border::Yes); + const auto generator = alpine_raster::make_generator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", "./unittest_tiles/", ctb::Grid::Srs::SphericalMercator, radix::tile::Border::Yes); generator.write(radix::tile::Descriptor { {0, glm::uvec2(0, 0)}, {}, int(ctb::Grid::Srs::SphericalMercator), 256, 257 }, radix::Raster({ 257, 257 })); CHECK(std::filesystem::exists("./unittest_tiles/0/0/0.png")); @@ -58,7 +58,7 @@ TEMPLATE_TEST_CASE("alpine raster format, border ", "", std::true_type, std::fal SECTION("process all tiles") { - auto generator = alpine_raster::make_generator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", "./unittest_tiles/", ctb::Grid::Srs::SphericalMercator, radix::tile::Scheme::Tms, testTypeValue2Border(TestType::value)); + auto generator = alpine_raster::make_generator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", "./unittest_tiles/", ctb::Grid::Srs::SphericalMercator, testTypeValue2Border(TestType::value)); generator.setWarnOnMissingOverviews(false); generator.process({ 0, 7 }); const auto tiles = generator.tiler().generateTiles({ 0, 7 }); @@ -71,7 +71,7 @@ TEMPLATE_TEST_CASE("alpine raster format, border ", "", std::true_type, std::fal #if defined(ALP_UNITTESTS_EXTENDED) && ALP_UNITTESTS_EXTENDED SECTION("process all tiles with max zoom") { - auto generator = alpine_raster::make_generator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", "./unittest_tiles/", ctb::Grid::Srs::SphericalMercator, radix::tile::Scheme::Tms, testTypeValue2Border(TestType::value)); + auto generator = alpine_raster::make_generator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", "./unittest_tiles/", ctb::Grid::Srs::SphericalMercator, testTypeValue2Border(TestType::value)); generator.setWarnOnMissingOverviews(false); generator.process({ 4, 8 }); const auto tiles = generator.tiler().generateTiles({ 4, 8 }); diff --git a/unittests/tilebuilder/depth_first_tile_traverser.cpp b/unittests/tilebuilder/depth_first_tile_traverser.cpp index ef753a3a..5cf7fb5e 100644 --- a/unittests/tilebuilder/depth_first_tile_traverser.cpp +++ b/unittests/tilebuilder/depth_first_tile_traverser.cpp @@ -37,8 +37,8 @@ TEST_CASE("depth_first_tile_traverser interface") const auto aggregate_function = [](std::vector) { return ReadType {}; }; const auto grid = ctb::GlobalMercator(); - const auto tiler = TopDownTiler(grid, grid.getExtent(), radix::tile::Border::No, radix::tile::Scheme::Tms); - const radix::tile::Id root_id = { 0, { 0, 0 }, tiler.scheme() }; + const auto tiler = TopDownTiler(grid, grid.getExtent(), radix::tile::Border::No); + const radix::tile::Id root_id = { 0, { 0, 0 } }; const unsigned max_zoom_level = 3; traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, root_id, max_zoom_level); @@ -68,15 +68,15 @@ TEST_CASE("depth_first_tile_traverser basics") }; const auto grid = ctb::GlobalMercator(); - const auto tiler = TopDownTiler(grid, grid.getExtent(), radix::tile::Border::No, radix::tile::Scheme::Tms); - const radix::tile::Id root_id = { 0, { 0, 0 }, tiler.scheme() }; + const auto tiler = TopDownTiler(grid, grid.getExtent(), radix::tile::Border::No); + const radix::tile::Id root_id = { 0, { 0, 0 } }; SECTION("reads root tile #1") { const auto result = traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, root_id, 0); REQUIRE(read_tiles.size() == 1); CHECK(aggregate_calls.size() == 0); - CHECK(read_tiles.contains(radix::tile::Id{0, {0, 0}, tiler.scheme()})); + CHECK(read_tiles.contains(radix::tile::Id{0, {0, 0}})); CHECK(result.d == glm::uvec2 { 0, 0 }); } @@ -84,7 +84,7 @@ TEST_CASE("depth_first_tile_traverser basics") { const auto result = traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, { 2, { 1, 3 } }, 2); REQUIRE(read_tiles.size() == 1); - CHECK(read_tiles.contains(radix::tile::Id{2, {1, 3}, tiler.scheme()})); + CHECK(read_tiles.contains(radix::tile::Id{2, {1, 3}})); CHECK(result.d == glm::uvec2 { 1, 3 }); } @@ -92,10 +92,10 @@ TEST_CASE("depth_first_tile_traverser basics") { traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, { 0, { 0, 0 } }, 1); REQUIRE(read_tiles.size() == 4); - CHECK(read_tiles.contains(radix::tile::Id{1, {0, 0}, tiler.scheme()})); - CHECK(read_tiles.contains(radix::tile::Id{1, {0, 1}, tiler.scheme()})); - CHECK(read_tiles.contains(radix::tile::Id{1, {1, 0}, tiler.scheme()})); - CHECK(read_tiles.contains(radix::tile::Id{1, {1, 1}, tiler.scheme()})); + CHECK(read_tiles.contains(radix::tile::Id{1, {0, 0}})); + CHECK(read_tiles.contains(radix::tile::Id{1, {0, 1}})); + CHECK(read_tiles.contains(radix::tile::Id{1, {1, 0}})); + CHECK(read_tiles.contains(radix::tile::Id{1, {1, 1}})); } SECTION("aggregate is called correctly") @@ -119,7 +119,7 @@ TEST_CASE("depth_first_tile_traverser austrian heights") const auto dataset = Dataset::open_shared_raster(ALP_TEST_DATA_DIR "/austria/at_100m_mgi.tif").value(); // const auto dataset = Dataset::open_shared_raster(ALP_TEST_DATA_DIR "/austria/at_mgi.tif"); const auto bounds = dataset->bounds(grid.getSRS()); - const auto tiler = TopDownTiler(grid, bounds, radix::tile::Border::No, radix::tile::Scheme::Tms); + const auto tiler = TopDownTiler(grid, bounds, radix::tile::Border::No); const auto tile_reader = DatasetReader(dataset, grid.getSRS(), 1, false); // const auto dataset_reader = DatasetReader() std::set read_tiles; @@ -142,14 +142,14 @@ TEST_CASE("depth_first_tile_traverser austrian heights") return aggr; }; - const radix::tile::Id root_id = { 0, { 0, 0 }, tiler.scheme() }; + const radix::tile::Id root_id = { 0, { 0, 0 } }; SECTION("reads root tile") { const auto result = traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, root_id, 0); REQUIRE(read_tiles.size() == 1); CHECK(aggregate_calls.size() == 0); - CHECK(read_tiles.contains(radix::tile::Id{0, {0, 0}, tiler.scheme()})); + CHECK(read_tiles.contains(radix::tile::Id{0, {0, 0}})); CHECK(result.first >= 0); CHECK(result.first <= 4000); CHECK(result.second >= 0); @@ -161,14 +161,14 @@ TEST_CASE("depth_first_tile_traverser austrian heights") const auto result = traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, root_id, 6); CHECK(read_tiles.size() == 6); CHECK(aggregate_calls.size() == 9); - CHECK(read_tiles.contains(radix::tile::Id{6, {33, 41}, tiler.scheme()})); - CHECK(read_tiles.contains(radix::tile::Id{6, {33, 42}, tiler.scheme()})); + CHECK(read_tiles.contains(radix::tile::Id{6, {33, 22}})); + CHECK(read_tiles.contains(radix::tile::Id{6, {33, 21}})); - CHECK(read_tiles.contains(radix::tile::Id{6, {34, 41}, tiler.scheme()})); - CHECK(read_tiles.contains(radix::tile::Id{6, {34, 42}, tiler.scheme()})); + CHECK(read_tiles.contains(radix::tile::Id{6, {34, 22}})); + CHECK(read_tiles.contains(radix::tile::Id{6, {34, 21}})); - CHECK(read_tiles.contains(radix::tile::Id{6, {35, 41}, tiler.scheme()})); - CHECK(read_tiles.contains(radix::tile::Id{6, {35, 42}, tiler.scheme()})); + CHECK(read_tiles.contains(radix::tile::Id{6, {35, 22}})); + CHECK(read_tiles.contains(radix::tile::Id{6, {35, 21}})); CHECK(result.first >= 0); CHECK(result.first <= 500); CHECK(result.second >= 2000); @@ -187,8 +187,8 @@ TEST_CASE("depth_first_tile_traverser aggregate is not called with an empty vect // parent tile is produced, because its border overlaps the extents // child tiles have smaller pixels -> their border does not overlap the extents any more. bounds.min.x = (bounds.width() / 256) / 4; - const auto tiler = TopDownTiler(grid, bounds, radix::tile::Border::Yes, radix::tile::Scheme::Tms); - const radix::tile::Id root_id = { 0, { 0, 0 }, tiler.scheme() }; + const auto tiler = TopDownTiler(grid, bounds, radix::tile::Border::Yes); + const radix::tile::Id root_id = { 0, { 0, 0 } }; const auto read_function = [&](const radix::tile::Descriptor&) -> int { return 0; diff --git a/unittests/tilebuilder/parallel_tile_generator.cpp b/unittests/tilebuilder/parallel_tile_generator.cpp index 3fb7a7c4..f9326334 100644 --- a/unittests/tilebuilder/parallel_tile_generator.cpp +++ b/unittests/tilebuilder/parallel_tile_generator.cpp @@ -55,7 +55,7 @@ TEST_CASE("parallel tile generator") }; std::filesystem::path base_path = "./unittest_tiles/"; - auto generator = ParallelTileGenerator::make(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", ctb::Grid::Srs::SphericalMercator, radix::tile::Scheme::Tms, std::make_unique(&tile_counter, &validation_error_counter), base_path); + auto generator = ParallelTileGenerator::make(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", ctb::Grid::Srs::SphericalMercator, std::make_unique(&tile_counter, &validation_error_counter), base_path); generator.setWarnOnMissingOverviews(false); SECTION("dataset tiles only") { @@ -63,12 +63,12 @@ TEST_CASE("parallel tile generator") CHECK(validation_error_counter == 0); CHECK(tile_counter == 27); CHECK(std::filesystem::exists(base_path / "0" / "0" / "0.empty")); - CHECK(std::filesystem::exists(base_path / "1" / "1" / "1.empty")); - CHECK(std::filesystem::exists(base_path / "2" / "2" / "2.empty")); - CHECK(std::filesystem::exists(base_path / "3" / "4" / "5.empty")); - CHECK(std::filesystem::exists(base_path / "4" / "8" / "10.empty")); - CHECK(std::filesystem::exists(base_path / "5" / "16" / "20.empty")); - CHECK(std::filesystem::exists(base_path / "7" / "70" / "84.empty")); + CHECK(std::filesystem::exists(base_path / "1" / "1" / "0.empty")); + CHECK(std::filesystem::exists(base_path / "2" / "2" / "1.empty")); + CHECK(std::filesystem::exists(base_path / "3" / "4" / "2.empty")); + CHECK(std::filesystem::exists(base_path / "4" / "8" / "5.empty")); + CHECK(std::filesystem::exists(base_path / "5" / "16" / "11.empty")); + CHECK(std::filesystem::exists(base_path / "7" / "70" / "43.empty")); } SECTION("world wide tiles") { diff --git a/unittests/tilebuilder/parallel_tiler.cpp b/unittests/tilebuilder/parallel_tiler.cpp index 815b48ba..b18124cf 100644 --- a/unittests/tilebuilder/parallel_tiler.cpp +++ b/unittests/tilebuilder/parallel_tiler.cpp @@ -23,23 +23,21 @@ #include "ctb/GlobalGeodetic.hpp" #include "ctb/GlobalMercator.hpp" #include -#include #include #include #include #include -#include using Catch::Approx; using namespace radix; -TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::false_type) +TEST_CASE("ParallelTiler") { // const auto bounds = radix::tile::SrsBounds(1'000'000, 6'000'000, 2'000'000, 6'700'000); // in m SECTION("mercator / level 0") { const auto grid = ctb::GlobalMercator(); - const auto tiler = ParallelTiler(grid, grid.getExtent(), radix::tile::Border::No, TestType::value ? radix::tile::Scheme::Tms : radix::tile::Scheme::SlippyMap); + const auto tiler = ParallelTiler(grid, grid.getExtent(), radix::tile::Border::No); CHECK(tiler.northEastTile(0).coords == glm::uvec2(0, 0)); CHECK(tiler.southWestTile(0).coords == glm::uvec2(0, 0)); @@ -59,26 +57,25 @@ TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::f CHECK(t.srsBounds.max.x == Approx(grid.getExtent().max.x)); } - SECTION("mercator tms / level 1 and 2") + SECTION("mercator / level 1 and 2") { const auto grid = ctb::GlobalMercator(); auto dataset = Dataset::open_shared_raster(ALP_TEST_DATA_DIR "/austria/at_mgi.tif").value(); - const auto tiler = ParallelTiler(grid, dataset->bounds(grid.getSRS()), radix::tile::Border::No, TestType::value ? radix::tile::Scheme::Tms : radix::tile::Scheme::SlippyMap); + const auto tiler = ParallelTiler(grid, dataset->bounds(grid.getSRS()), radix::tile::Border::No); - CHECK(tiler.northEastTile(1).coords == glm::uvec2(1, TestType::value ? 1 : 0)); - CHECK(tiler.southWestTile(1).coords == glm::uvec2(1, TestType::value ? 1 : 0)); + CHECK(tiler.northEastTile(1).coords == glm::uvec2(1, 0)); + CHECK(tiler.southWestTile(1).coords == glm::uvec2(1, 0)); - CHECK(tiler.northEastTile(2).coords == glm::uvec2(2, TestType::value ? 2 : 1)); - CHECK(tiler.southWestTile(2).coords == glm::uvec2(2, TestType::value ? 2 : 1)); + CHECK(tiler.northEastTile(2).coords == glm::uvec2(2, 1)); + CHECK(tiler.southWestTile(2).coords == glm::uvec2(2, 1)); { // https://www.maptiler.com/google-maps-coordinates-tile-bounds-projection/#1/-5.80/62.29 - // this code is for TMS mapping, i.e., tile y = 0 is south const auto l1_tiles = tiler.generateTiles(1); REQUIRE(l1_tiles.size() == 1); const auto t = l1_tiles.front(); CHECK(t.id.zoom_level == 1); - CHECK(t.id.coords == glm::uvec2(1, TestType::value ? 1 : 0)); + CHECK(t.id.coords == glm::uvec2(1, 0)); CHECK(t.gridSize == 256); CHECK(t.tileSize == 256); @@ -92,7 +89,7 @@ TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::f REQUIRE(l2_tiles.size() == 1); const auto t = l2_tiles.front(); CHECK(t.id.zoom_level == 2); - CHECK(t.id.coords == glm::uvec2(2, TestType::value ? 2 : 1)); + CHECK(t.id.coords == glm::uvec2(2, 1)); CHECK(t.gridSize == 256); CHECK(t.tileSize == 256); @@ -103,10 +100,10 @@ TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::f } } - SECTION("geodetic tms / level 0") + SECTION("geodetic / level 0") { const auto grid = ctb::GlobalGeodetic(64); - const auto tiler = ParallelTiler(grid, grid.getExtent(), radix::tile::Border::Yes, TestType::value ? radix::tile::Scheme::Tms : radix::tile::Scheme::SlippyMap); + const auto tiler = ParallelTiler(grid, grid.getExtent(), radix::tile::Border::Yes); CHECK(tiler.northEastTile(0).coords == glm::uvec2(1, 0)); CHECK(tiler.southWestTile(0).coords == glm::uvec2(0, 0)); @@ -140,24 +137,24 @@ TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::f CHECK(t1.srsBounds.max.x == Approx(grid.getExtent().max.x + grid.resolution(0))); } - SECTION("geodetic tms / level 1 and 2") + SECTION("geodetic / level 1 and 2") { const auto grid = ctb::GlobalGeodetic(64); auto dataset = Dataset::open_shared_raster(ALP_TEST_DATA_DIR "/austria/at_mgi.tif").value(); - const auto tiler = ParallelTiler(grid, dataset->bounds(grid.getSRS()), radix::tile::Border::Yes, TestType::value ? radix::tile::Scheme::Tms : radix::tile::Scheme::SlippyMap); + const auto tiler = ParallelTiler(grid, dataset->bounds(grid.getSRS()), radix::tile::Border::Yes); - CHECK(tiler.northEastTile(1).coords == glm::uvec2(2, TestType::value ? 1 : 0)); - CHECK(tiler.southWestTile(1).coords == glm::uvec2(2, TestType::value ? 1 : 0)); + CHECK(tiler.northEastTile(1).coords == glm::uvec2(2, 0)); + CHECK(tiler.southWestTile(1).coords == glm::uvec2(2, 0)); - CHECK(tiler.northEastTile(2).coords == glm::uvec2(4, TestType::value ? 3 : 0)); - CHECK(tiler.southWestTile(2).coords == glm::uvec2(4, TestType::value ? 3 : 0)); + CHECK(tiler.northEastTile(2).coords == glm::uvec2(4, 0)); + CHECK(tiler.southWestTile(2).coords == glm::uvec2(4, 0)); { const auto l1_tiles = tiler.generateTiles(1); REQUIRE(l1_tiles.size() == 1); const auto t = l1_tiles.front(); CHECK(t.id.zoom_level == 1); - CHECK(t.id.coords == glm::uvec2(2, TestType::value ? 1 : 0)); + CHECK(t.id.coords == glm::uvec2(2, 0)); CHECK(t.gridSize == 64); CHECK(t.tileSize == 65); @@ -173,7 +170,7 @@ TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::f REQUIRE(l2_tiles.size() == 1); const auto t = l2_tiles.front(); CHECK(t.id.zoom_level == 2); - CHECK(t.id.coords == glm::uvec2(4, TestType::value ? 3 : 0)); + CHECK(t.id.coords == glm::uvec2(4, 0)); CHECK(t.gridSize == 64); CHECK(t.tileSize == 65); @@ -186,26 +183,25 @@ TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::f } } - SECTION("mercator tms / level 1 and 2 (test with cape horn, on southern and western hemisphere)") + SECTION("mercator / level 1 and 2 (test with cape horn, on southern and western hemisphere)") { const auto grid = ctb::GlobalMercator(); auto dataset = Dataset::open_shared_raster(ALP_TEST_DATA_DIR "/capehorn/small.tif").value(); - const auto tiler = ParallelTiler(grid, dataset->bounds(grid.getSRS()), radix::tile::Border::No, TestType::value ? radix::tile::Scheme::Tms : radix::tile::Scheme::SlippyMap); + const auto tiler = ParallelTiler(grid, dataset->bounds(grid.getSRS()), radix::tile::Border::No); - CHECK(tiler.northEastTile(1).coords == glm::uvec2(0, TestType::value ? 0 : 1)); - CHECK(tiler.southWestTile(1).coords == glm::uvec2(0, TestType::value ? 0 : 1)); + CHECK(tiler.northEastTile(1).coords == glm::uvec2(0, 1)); + CHECK(tiler.southWestTile(1).coords == glm::uvec2(0, 1)); - CHECK(tiler.northEastTile(2).coords == glm::uvec2(1, TestType::value ? 1 : 2)); - CHECK(tiler.southWestTile(2).coords == glm::uvec2(1, TestType::value ? 1 : 2)); + CHECK(tiler.northEastTile(2).coords == glm::uvec2(1, 2)); + CHECK(tiler.southWestTile(2).coords == glm::uvec2(1, 2)); { // https://www.maptiler.com/google-maps-coordinates-tile-bounds-projection/#1/-5.80/62.29 - // this code is for TMS mapping, i.e., tile y = 0 is south const auto l1_tiles = tiler.generateTiles(1); REQUIRE(l1_tiles.size() == 1); const auto t = l1_tiles.front(); CHECK(t.id.zoom_level == 1); - CHECK(t.id.coords == glm::uvec2(0, TestType::value ? 0 : 1)); + CHECK(t.id.coords == glm::uvec2(0, 1)); CHECK(t.gridSize == 256); CHECK(t.tileSize == 256); @@ -219,7 +215,7 @@ TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::f REQUIRE(l2_tiles.size() == 1); const auto t = l2_tiles.front(); CHECK(t.id.zoom_level == 2); - CHECK(t.id.coords == glm::uvec2(1, TestType::value ? 1 : 2)); + CHECK(t.id.coords == glm::uvec2(1, 2)); CHECK(t.gridSize == 256); CHECK(t.tileSize == 256); @@ -230,24 +226,24 @@ TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::f } } - SECTION("geodetic tms / level 1 and 2 (test with cape horn, on southern and western hemisphere)") + SECTION("geodetic / level 1 and 2 (test with cape horn, on southern and western hemisphere)") { const auto grid = ctb::GlobalGeodetic(64); auto dataset = Dataset::open_shared_raster(ALP_TEST_DATA_DIR "/capehorn/small.tif").value(); - const auto tiler = ParallelTiler(grid, dataset->bounds(grid.getSRS()), radix::tile::Border::Yes, TestType::value ? radix::tile::Scheme::Tms : radix::tile::Scheme::SlippyMap); + const auto tiler = ParallelTiler(grid, dataset->bounds(grid.getSRS()), radix::tile::Border::Yes); - CHECK(tiler.northEastTile(1).coords == glm::uvec2(1, TestType::value ? 0 : 1)); - CHECK(tiler.southWestTile(1).coords == glm::uvec2(1, TestType::value ? 0 : 1)); + CHECK(tiler.northEastTile(1).coords == glm::uvec2(1, 1)); + CHECK(tiler.southWestTile(1).coords == glm::uvec2(1, 1)); - CHECK(tiler.northEastTile(2).coords == glm::uvec2(2, TestType::value ? 0 : 3)); - CHECK(tiler.southWestTile(2).coords == glm::uvec2(2, TestType::value ? 0 : 3)); + CHECK(tiler.northEastTile(2).coords == glm::uvec2(2, 3)); + CHECK(tiler.southWestTile(2).coords == glm::uvec2(2, 3)); { const auto l1_tiles = tiler.generateTiles(1); REQUIRE(l1_tiles.size() == 1); const auto t = l1_tiles.front(); CHECK(t.id.zoom_level == 1); - CHECK(t.id.coords == glm::uvec2(1, TestType::value ? 0 : 1)); + CHECK(t.id.coords == glm::uvec2(1, 1)); CHECK(t.gridSize == 64); CHECK(t.tileSize == 65); @@ -263,7 +259,7 @@ TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::f REQUIRE(l2_tiles.size() == 1); const auto t = l2_tiles.front(); CHECK(t.id.zoom_level == 2); - CHECK(t.id.coords == glm::uvec2(2, TestType::value ? 0 : 3)); + CHECK(t.id.coords == glm::uvec2(2, 3)); CHECK(t.gridSize == 64); CHECK(t.tileSize == 65); @@ -281,7 +277,7 @@ TEST_CASE("ParallelTiler returns tiles for several zoom levels") { const auto dataset = Dataset(ALP_TEST_DATA_DIR "/austria/at_mgi.tif"); const auto grid = ctb::GlobalMercator(256); - const auto tiler = ParallelTiler(grid, dataset.bounds(grid.getSRS()), radix::tile::Border::Yes, radix::tile::Scheme::Tms); + const auto tiler = ParallelTiler(grid, dataset.bounds(grid.getSRS()), radix::tile::Border::Yes); SECTION("generate from 0 to 7") { diff --git a/unittests/tilebuilder/tile_heights_generator.cpp b/unittests/tilebuilder/tile_heights_generator.cpp index 43f74237..89c532ae 100644 --- a/unittests/tilebuilder/tile_heights_generator.cpp +++ b/unittests/tilebuilder/tile_heights_generator.cpp @@ -30,7 +30,7 @@ TEST_CASE("TileHeightsGenerator") constexpr auto file_name = "height_data.atb"; SECTION("mercator") { - const auto generator = TileHeightsGenerator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", ctb::Grid::Srs::SphericalMercator, radix::tile::Scheme::Tms, radix::tile::Border::Yes, base_path / file_name); + const auto generator = TileHeightsGenerator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", ctb::Grid::Srs::SphericalMercator, radix::tile::Border::Yes, base_path / file_name); generator.run(8); const auto heights = TileHeights::read_from(base_path / file_name); @@ -41,7 +41,7 @@ TEST_CASE("TileHeightsGenerator") } { - auto [min, max] = heights.query({ 8, { 138, 166 } }); // part of styria, lower and upper austria (https://www.maptiler.com/google-maps-coordinates-tile-bounds-projection/#8/15.69/47.75) + auto [min, max] = heights.query({ 8, { 138, 89 } }); // part of styria, lower and upper austria (https://www.maptiler.com/google-maps-coordinates-tile-bounds-projection/#8/15.69/47.75) CHECK(min > 300); CHECK(min < 400); CHECK(max > 1500); @@ -50,7 +50,7 @@ TEST_CASE("TileHeightsGenerator") } SECTION("geodetic") { - const auto generator = TileHeightsGenerator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", ctb::Grid::Srs::WGS84, radix::tile::Scheme::Tms, radix::tile::Border::Yes, base_path / file_name); + const auto generator = TileHeightsGenerator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", ctb::Grid::Srs::WGS84, radix::tile::Border::Yes, base_path / file_name); generator.run(8); const auto heights = TileHeights::read_from(base_path / file_name); @@ -61,7 +61,7 @@ TEST_CASE("TileHeightsGenerator") } { - auto [min, max] = heights.query({ 8, { 270, 194 } }); // can't check the address easily, because there is no web service showing geodetic tile names. + auto [min, max] = heights.query({ 8, { 270, 61 } }); // can't check the address easily, because there is no web service showing geodetic tile names. CHECK(min > 500); CHECK(min < 700); CHECK(max > 3600); diff --git a/unittests/tilebuilder/top_down_tiler.cpp b/unittests/tilebuilder/top_down_tiler.cpp index cd1e33b5..61645361 100644 --- a/unittests/tilebuilder/top_down_tiler.cpp +++ b/unittests/tilebuilder/top_down_tiler.cpp @@ -21,7 +21,7 @@ #include "TopDownTiler.h" #include "ctb/GlobalGeodetic.hpp" #include "ctb/GlobalMercator.hpp" -#include +#include #include using namespace radix; @@ -49,18 +49,16 @@ void compare_tile_lists(const std::vector& a_tiles, std } -TEMPLATE_TEST_CASE("BottomUpTiler, using tms scheme", "", std::true_type, std::false_type) +TEST_CASE("TopDownTiler") { - const auto scheme = TestType::value ? radix::tile::Scheme::Tms : radix::tile::Scheme::SlippyMap; - SECTION("mercator / level 0 all") { const auto grid = ctb::GlobalMercator(); - const auto tiler = TopDownTiler(grid, grid.getExtent(), radix::tile::Border::No, scheme); + const auto tiler = TopDownTiler(grid, grid.getExtent(), radix::tile::Border::No); - const auto tiles = tiler.generateTiles({0, { 0, 0 }, scheme}); + const auto tiles = tiler.generateTiles({0, { 0, 0 }}); REQUIRE(tiles.size() == 4); - const auto parallel_tiler = ParallelTiler(grid, grid.getExtent(), radix::tile::Border::No, scheme); + const auto parallel_tiler = ParallelTiler(grid, grid.getExtent(), radix::tile::Border::No); compare_tile_lists(tiles, parallel_tiler.generateTiles(1)); } @@ -69,22 +67,22 @@ TEMPLATE_TEST_CASE("BottomUpTiler, using tms scheme", "", std::true_type, std::f const auto grid = ctb::GlobalMercator(); auto dataset = Dataset::open_shared_raster(ALP_TEST_DATA_DIR "/austria/at_mgi.tif").value(); const auto bounds = dataset->bounds(grid.getSRS()); - const auto tiler = TopDownTiler(grid, bounds, radix::tile::Border::No, scheme); + const auto tiler = TopDownTiler(grid, bounds, radix::tile::Border::No); - const auto tiles = tiler.generateTiles({0, { 0, 0 }, scheme}); + const auto tiles = tiler.generateTiles({0, { 0, 0 }}); REQUIRE(tiles.size() == 1); - const auto parallel_tiler = ParallelTiler(grid, bounds, radix::tile::Border::No, scheme); + const auto parallel_tiler = ParallelTiler(grid, bounds, radix::tile::Border::No); compare_tile_lists(tiles, parallel_tiler.generateTiles(1)); } SECTION("geodetic / level 0 east half") { const auto grid = ctb::GlobalGeodetic(); - const auto tiler = TopDownTiler(grid, grid.getExtent(), radix::tile::Border::No, scheme); + const auto tiler = TopDownTiler(grid, grid.getExtent(), radix::tile::Border::No); - const auto tiles = tiler.generateTiles({0, { 1, 0 }, scheme}); + const auto tiles = tiler.generateTiles({0, { 1, 0 }}); REQUIRE(tiles.size() == 4); - const auto parallel_tiler = ParallelTiler(grid, {{0, -90}, {180, 90}}, radix::tile::Border::No, scheme); + const auto parallel_tiler = ParallelTiler(grid, {{0, -90}, {180, 90}}, radix::tile::Border::No); compare_tile_lists(tiles, parallel_tiler.generateTiles(1)); } @@ -93,11 +91,11 @@ TEMPLATE_TEST_CASE("BottomUpTiler, using tms scheme", "", std::true_type, std::f const auto grid = ctb::GlobalGeodetic(); auto dataset = Dataset::open_shared_raster(ALP_TEST_DATA_DIR "/austria/at_mgi.tif").value(); const auto bounds = dataset->bounds(grid.getSRS()); - const auto tiler = TopDownTiler(grid, bounds, radix::tile::Border::No, scheme); + const auto tiler = TopDownTiler(grid, bounds, radix::tile::Border::No); - const auto tiles = tiler.generateTiles({0, { 1, 0 }, scheme}); + const auto tiles = tiler.generateTiles({0, { 1, 0 }}); REQUIRE(tiles.size() == 1); - const auto parallel_tiler = ParallelTiler(grid, bounds, radix::tile::Border::No, scheme); + const auto parallel_tiler = ParallelTiler(grid, bounds, radix::tile::Border::No); compare_tile_lists(tiles, parallel_tiler.generateTiles(1)); } } From 7a384ff72924113ceb2392838e79e8b645c376ea Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Sun, 19 Jul 2026 13:46:18 +0200 Subject: [PATCH 04/61] ci: allow warnings on raster-store branch --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1c36786..e96f76f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,7 @@ on: env: CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 1G + ALP_CI_WARNINGS_AS_ERRORS: ${{ (github.ref_name == 'raster-store' || github.head_ref == 'raster-store') && 'OFF' || 'ON' }} jobs: build-and-test: @@ -122,7 +123,7 @@ jobs: -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - -DALP_WARNINGS_AS_ERRORS=ON + -DALP_WARNINGS_AS_ERRORS=${{ env.ALP_CI_WARNINGS_AS_ERRORS }} ${{ matrix.cmake_flags }} - name: Save installed CMake dependencies From 39d36be9306f0e543e6831294eacfce11a1f03db Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Mon, 20 Jul 2026 04:18:16 +0200 Subject: [PATCH 05/61] Make tile downloader URL formats provider-driven Bundle URL patterns and Y direction with the built-in Basemap and Gataki providers. Replace layer, style, and coordinate-order flags with custom placeholder URLs and an optional legacy TMS Y direction. Keep downloaded files in Google/Mapbox Slippy layout and update URL-generation tests and documentation. --- README.md | 10 ++- docs/tile-downloader/download.sh | 4 +- src/tile_downloader/TileUrlBuilder.h | 105 +++++++++++----------- src/tile_downloader/cli.cpp | 28 +++--- src/tile_downloader/cli.h | 7 +- src/tile_downloader/main.cpp | 19 ++-- unittests/tile_downloader/url_builder.cpp | 43 +++++---- 7 files changed, 104 insertions(+), 112 deletions(-) diff --git a/README.md b/README.md index 20d2d8d8..011b8cd6 100644 --- a/README.md +++ b/README.md @@ -24,22 +24,24 @@ The tools typically only handle one tile per command which makes it infeasible t In this example, we'll show how to build the hierarchy for Vienna's city center (Zoom: 13, X: 4468, Y:2840). ### 1. Downloading tiles -Tile coordinates always use the Google/Mapbox/XYZ convention internally: the origin is north-west, X grows east, and Y grows south. Google Maps, Mapbox, OpenStreetMap, and most XYZ services use the common URL order `{zoom}/{x}/{y}`; this is the downloader default. +Tile coordinates always use the Google/Mapbox/XYZ convention internally: the origin is north-west, X grows east, and Y grows south. Google Maps, Mapbox, OpenStreetMap, and most XYZ services use the common URL order `{zoom}/{x}/{y}`. -Some providers use `{zoom}/{y}/{x}` instead. Both built-in Austrian providers currently require `--url-coordinate-order yx`; for example, the Gataki mirror uses: +The `basemap` and `gataki` providers select their complete URL pattern and Y direction automatically. Both currently use downward Y with the URL order `{zoom}/{y}/{x}`. The `basemap` provider downloads the basemap.at orthophoto, and the Gataki mirror uses: `https://gataki.cg.tuwien.ac.at/raw/basemap/tiles/{zoom}/{y}/{x}.jpeg` Example for the root tile: https://gataki.cg.tuwien.ac.at/raw/basemap/tiles/13/2840/4468.jpeg -Select URL coordinate order with `--url-coordinate-order xy|yx`. Select the URL Y direction with `--url-y-direction down|up`; `down` is the common Google/Mapbox convention and `up` is the legacy TMS convention. These options affect only remote URLs. Downloaded files always use the common Google/Mapbox layout `{zoom}/{x}/{y}.jpeg`. +For another service, pass a quoted `--url` pattern containing `{zoom}`, `{x}`, and `{y}`. Placeholder placement selects URL coordinate order. Custom URLs default to downward Y; use `--url-y-direction up` for legacy TMS. Downloaded files always use the common Google/Mapbox layout `{zoom}/{x}/{y}.jpeg`, independently of the remote URL. The following command downloads the mirror's `{zoom}/{y}/{x}` URLs and writes the root tile to `./tiles/13/4468/2840.jpeg`: ``` -./tile-downloader --provider gataki --zoom 13 --x 4468 --y 2840 --url-coordinate-order yx --max-zoom-level 19 +./tile-downloader --provider gataki --zoom 13 --x 4468 --y 2840 --max-zoom-level 19 ``` +For example, a custom Google/Mapbox URL can be selected with `--url 'https://example.test/{zoom}/{x}/{y}.jpeg'`. + ### 2. Download heightmap dataset The meshes are built from a heightmap dataset, therefore we need to download one. For this example we'll only use a small part of the complete dataset for the whole of austria (available at https://gataki.cg.tuwien.ac.at/raw/Oe_2020/, 268 GB to 1.1 TB). The part we're gonna use contains Vienna's city center and it is available at https://gataki.cg.tuwien.ac.at/raw/vienna/innenstadt_gs_1m_mgi.tif (228 MB). diff --git a/docs/tile-downloader/download.sh b/docs/tile-downloader/download.sh index d5aef40a..dcd168f8 100755 --- a/docs/tile-downloader/download.sh +++ b/docs/tile-downloader/download.sh @@ -4,6 +4,6 @@ build_path="/home/madam/Documents/work/tuw/alpinemaps/build-terrain-builder-Desk while read p; do read zoom x y <<<${p//[^0-9]/ } - echo -e "nice -10 \$build_path/tile-downloader --provider basemap --zoom ${zoom} --x ${x} --y ${y} --url-coordinate-order yx --verbosity 0&" -# nice -10 $build_path/tile-downloader --provider basemap --zoom ${zoom} --x ${x} --y ${y} --url-coordinate-order yx --verbosity 0& + echo -e "nice -10 \$build_path/tile-downloader --provider basemap --zoom ${zoom} --x ${x} --y ${y} --verbosity 0&" +# nice -10 $build_path/tile-downloader --provider basemap --zoom ${zoom} --x ${x} --y ${y} --verbosity 0& done #include #include +#include #include -#include #include -enum class TileCoordinateOrder { - Xy, - Yx +enum class TileDownloadProvider { + Basemap, + Gataki }; enum class TileYDirection { @@ -18,67 +18,68 @@ enum class TileYDirection { Up }; -struct TileUrlFormat { - TileCoordinateOrder coordinate_order = TileCoordinateOrder::Xy; +struct TileProviderConfig { + std::string url_pattern; TileYDirection y_direction = TileYDirection::Down; +}; - [[nodiscard]] std::pair coordinates(const radix::tile::Id& tile_id) const - { - auto y = tile_id.coords.y; - if (y_direction == TileYDirection::Up) { - if (tile_id.zoom_level >= std::numeric_limits::digits) - throw std::invalid_argument("tile zoom level is too large for legacy TMS coordinates"); - y = (1u << tile_id.zoom_level) - y - 1; - } - - if (coordinate_order == TileCoordinateOrder::Xy) - return { tile_id.coords.x, y }; - return { y, tile_id.coords.x }; +[[nodiscard]] inline TileProviderConfig tile_provider_config(TileDownloadProvider provider) +{ + switch (provider) { + case TileDownloadProvider::Basemap: + return { + "https://mapsneu.wien.gv.at/basemap/bmaporthofoto30cm/normal/google3857/{zoom}/{y}/{x}.jpeg", + TileYDirection::Down + }; + case TileDownloadProvider::Gataki: + return { + "https://gataki.cg.tuwien.ac.at/raw/basemap/tiles/{zoom}/{y}/{x}.jpeg", + TileYDirection::Down + }; } -}; -class TileUrlBuilder { -public: - virtual ~TileUrlBuilder() = default; - virtual std::string build_url(const radix::tile::Id &tile_id) const = 0; -}; + throw std::invalid_argument("unknown tile download provider"); +} -class BasemapTileUrlBuilder : public TileUrlBuilder { +class TileUrlBuilder { public: - BasemapTileUrlBuilder(std::string layer, std::string style, TileUrlFormat format = {}) - : _layer(std::move(layer)) - , _style(std::move(style)) - , _format(format) + explicit TileUrlBuilder(TileProviderConfig config) + : _url_pattern(std::move(config.url_pattern)) + , _y_direction(config.y_direction) { + if (_url_pattern.find("{zoom}") == std::string::npos + || _url_pattern.find("{x}") == std::string::npos + || _url_pattern.find("{y}") == std::string::npos) { + throw std::invalid_argument("tile URL pattern must contain {zoom}, {x}, and {y}"); + } } - std::string build_url(const radix::tile::Id &tile_id) const override { - const auto [first_coordinate, second_coordinate] = _format.coordinates(tile_id); - return fmt::format( - "https://mapsneu.wien.gv.at/basemap/{}/{}/google3857/{}/{}/{}.jpeg", - _layer, _style, tile_id.zoom_level, first_coordinate, second_coordinate); + [[nodiscard]] std::string build_url(const radix::tile::Id& tile_id) const + { + auto y = tile_id.coords.y; + if (_y_direction == TileYDirection::Up) { + if (tile_id.zoom_level >= std::numeric_limits::digits) + throw std::invalid_argument("tile zoom level is too large for legacy TMS coordinates"); + y = (1u << tile_id.zoom_level) - y - 1; + } + + auto url = _url_pattern; + replace_all(url, "{zoom}", std::to_string(tile_id.zoom_level)); + replace_all(url, "{x}", std::to_string(tile_id.coords.x)); + replace_all(url, "{y}", std::to_string(y)); + return url; } private: - std::string _layer; - std::string _style; - TileUrlFormat _format; -}; - -class GatakiTileUrlBuilder : public TileUrlBuilder { -public: - explicit GatakiTileUrlBuilder(TileUrlFormat format = {}) - : _format(format) + static void replace_all(std::string& value, std::string_view placeholder, const std::string& replacement) { + size_t position = 0; + while ((position = value.find(placeholder, position)) != std::string::npos) { + value.replace(position, placeholder.size(), replacement); + position += replacement.size(); + } } - std::string build_url(const radix::tile::Id &tile_id) const override { - const auto [first_coordinate, second_coordinate] = _format.coordinates(tile_id); - return fmt::format( - "https://gataki.cg.tuwien.ac.at/raw/basemap/tiles/{}/{}/{}.jpeg", - tile_id.zoom_level, first_coordinate, second_coordinate); - } - -private: - TileUrlFormat _format; + std::string _url_pattern; + TileYDirection _y_direction; }; diff --git a/src/tile_downloader/cli.cpp b/src/tile_downloader/cli.cpp index 4c5acb95..567d197b 100644 --- a/src/tile_downloader/cli.cpp +++ b/src/tile_downloader/cli.cpp @@ -13,28 +13,26 @@ Args parse(int argc, const char *const *argv) { Args args; - app.add_option("--provider", args.provider, "Tile provider (basemap or gataki)") - ->required() - ->check(CLI::IsMember({"basemap", "gataki"}, CLI::ignore_case)); + const std::map provider_map{ + {"basemap", TileDownloadProvider::Basemap}, + {"gataki", TileDownloadProvider::Gataki}}; + auto* source_group = app.add_option_group("Tile source"); + source_group->add_option("--provider", args.provider, "Configured tile provider (basemap or gataki)") + ->transform(CLI::CheckedTransformer(provider_map, CLI::ignore_case)); + auto* url_option = source_group->add_option("--url", args.url_pattern, "Custom tile URL pattern containing {zoom}, {x}, and {y}"); + source_group->require_option(1); app.add_option("--zoom", args.zoom, "Root tile zoom level")->required(); app.add_option("--x,--col", args.x, "Root tile x/column in Google/Mapbox coordinates")->required(); app.add_option("--y,--row", args.y, "Root tile y/row in Google/Mapbox coordinates")->required(); - const std::map coordinate_order_map{ - {"xy", TileCoordinateOrder::Xy}, - {"yx", TileCoordinateOrder::Yx}}; - args.url_coordinate_order = TileCoordinateOrder::Xy; - app.add_option("--url-coordinate-order", args.url_coordinate_order, "URL coordinate order: xy (common Google/Mapbox format) or yx") - ->default_str("xy") - ->transform(CLI::CheckedTransformer(coordinate_order_map, CLI::ignore_case)); - const std::map y_direction_map{ {"down", TileYDirection::Down}, {"up", TileYDirection::Up}}; args.url_y_direction = TileYDirection::Down; - app.add_option("--url-y-direction", args.url_y_direction, "URL y direction: down (common Google/Mapbox format) or up (legacy TMS)") + app.add_option("--url-y-direction", args.url_y_direction, "Custom URL y direction: down (Google/Mapbox) or up (legacy TMS)") ->default_str("down") + ->needs(url_option) ->transform(CLI::CheckedTransformer(y_direction_map, CLI::ignore_case)); args.srs = 3857; @@ -62,12 +60,6 @@ Args parse(int argc, const char *const *argv) { app.add_option("--max-zoom-level", args.max_zoom_level, "Maximum zoom level to descend to"); - args.layer = "bmaporthofoto30cm"; - app.add_option("--layer", args.layer, "Basemap layer name")->default_val(args.layer); - - args.style = "normal"; - app.add_option("--style", args.style, "Basemap style")->default_val(args.style); - try { app.parse(argc, argv); } catch (const CLI::ParseError &e) { diff --git a/src/tile_downloader/cli.h b/src/tile_downloader/cli.h index bc667f98..80fa1d7c 100644 --- a/src/tile_downloader/cli.h +++ b/src/tile_downloader/cli.h @@ -4,7 +4,6 @@ #include #include -#include #include #include "TileUrlBuilder.h" @@ -12,19 +11,17 @@ namespace cli { struct Args { - std::string provider; + std::optional provider; + std::optional url_pattern; unsigned int zoom; unsigned int x; unsigned int y; - TileCoordinateOrder url_coordinate_order; TileYDirection url_y_direction; unsigned int srs; std::filesystem::path output; spdlog::level::level_enum log_level; bool early_skip; std::optional max_zoom_level; - std::string layer; - std::string style; }; Args parse(int argc, const char *const *argv); diff --git a/src/tile_downloader/main.cpp b/src/tile_downloader/main.cpp index b2973a29..18fbf28d 100644 --- a/src/tile_downloader/main.cpp +++ b/src/tile_downloader/main.cpp @@ -1,7 +1,3 @@ -#include -#include -#include - #include "TileDownloader.h" #include "TileUrlBuilder.h" #include "cli.h" @@ -15,19 +11,14 @@ int main(int argc, char *argv[]) { LOG_ERROR_AND_EXIT("unsupported srs EPSG \"{}\"", args.srs); } - std::unique_ptr url_builder; - std::string provider = args.provider; - std::transform(provider.begin(), provider.end(), provider.begin(), - [](unsigned char c) { return std::tolower(c); }); - if (provider == "basemap") { - url_builder = std::make_unique(args.layer, args.style, TileUrlFormat { args.url_coordinate_order, args.url_y_direction }); - } else { - url_builder = std::make_unique(TileUrlFormat { args.url_coordinate_order, args.url_y_direction }); - } + const auto provider_config = args.provider.has_value() + ? tile_provider_config(*args.provider) + : TileProviderConfig { *args.url_pattern, args.url_y_direction }; + const TileUrlBuilder url_builder(provider_config); const radix::tile::Id root_id = {args.zoom, {args.x, args.y}}; - TileDownloader downloader(*url_builder, args.output, args.early_skip, args.max_zoom_level); + TileDownloader downloader(url_builder, args.output, args.early_skip, args.max_zoom_level); downloader.download_recursive(root_id); return 0; diff --git a/unittests/tile_downloader/url_builder.cpp b/unittests/tile_downloader/url_builder.cpp index dd4ade75..e6bdaccf 100644 --- a/unittests/tile_downloader/url_builder.cpp +++ b/unittests/tile_downloader/url_builder.cpp @@ -5,38 +5,47 @@ namespace { constexpr radix::tile::Id tile { 3, { 1, 2 } }; +} -void check_urls(const TileUrlFormat format, const std::string& coordinate_path) +TEST_CASE("configured tile provider URLs") { - const BasemapTileUrlBuilder basemap("layer", "style", format); - CHECK(basemap.build_url(tile) == "https://mapsneu.wien.gv.at/basemap/layer/style/google3857/" + coordinate_path + ".jpeg"); + { + const TileUrlBuilder builder(tile_provider_config(TileDownloadProvider::Basemap)); + CHECK(builder.build_url(tile) == "https://mapsneu.wien.gv.at/basemap/bmaporthofoto30cm/normal/google3857/3/2/1.jpeg"); + } - const GatakiTileUrlBuilder gataki(format); - CHECK(gataki.build_url(tile) == "https://gataki.cg.tuwien.ac.at/raw/basemap/tiles/" + coordinate_path + ".jpeg"); -} + { + const TileUrlBuilder builder(tile_provider_config(TileDownloadProvider::Gataki)); + CHECK(builder.build_url(tile) == "https://gataki.cg.tuwien.ac.at/raw/basemap/tiles/3/2/1.jpeg"); + } } -TEST_CASE("tile URL coordinate formats") +TEST_CASE("custom tile URL patterns") { - SECTION("xy with downward y") + SECTION("zoom/x/y with downward y") { - check_urls({ TileCoordinateOrder::Xy, TileYDirection::Down }, "3/1/2"); + const TileUrlBuilder builder({ "https://example.test/{zoom}/{x}/{y}.png", TileYDirection::Down }); + CHECK(builder.build_url(tile) == "https://example.test/3/1/2.png"); } - SECTION("yx with downward y") + SECTION("zoom/y/x with downward y") { - check_urls({ TileCoordinateOrder::Yx, TileYDirection::Down }, "3/2/1"); + const TileUrlBuilder builder({ "https://example.test/{zoom}/{y}/{x}.png", TileYDirection::Down }); + CHECK(builder.build_url(tile) == "https://example.test/3/2/1.png"); } - SECTION("xy with upward legacy TMS y") + SECTION("upward legacy TMS y") { - check_urls({ TileCoordinateOrder::Xy, TileYDirection::Up }, "3/1/5"); + const TileUrlBuilder builder({ "https://example.test/{zoom}/{x}/{y}.png", TileYDirection::Up }); + CHECK(builder.build_url(tile) == "https://example.test/3/1/5.png"); } +} - SECTION("yx with upward legacy TMS y") - { - check_urls({ TileCoordinateOrder::Yx, TileYDirection::Up }, "3/5/1"); - } +TEST_CASE("tile URL patterns require all coordinate placeholders") +{ + CHECK_THROWS_AS(TileUrlBuilder({ "https://example.test/{x}/{y}.png", TileYDirection::Down }), std::invalid_argument); + CHECK_THROWS_AS(TileUrlBuilder({ "https://example.test/{zoom}/{y}.png", TileYDirection::Down }), std::invalid_argument); + CHECK_THROWS_AS(TileUrlBuilder({ "https://example.test/{zoom}/{x}.png", TileYDirection::Down }), std::invalid_argument); } TEST_CASE("downloaded tile path uses Google and Mapbox layout") From 9b3ad878bc07afeb212837cbbaa0a5f79572800a Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Mon, 20 Jul 2026 20:47:30 +0200 Subject: [PATCH 06/61] Revert "ci: allow warnings on raster-store branch" This reverts commit 7a384ff72924113ceb2392838e79e8b645c376ea. --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e96f76f3..c1c36786 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,6 @@ on: env: CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 1G - ALP_CI_WARNINGS_AS_ERRORS: ${{ (github.ref_name == 'raster-store' || github.head_ref == 'raster-store') && 'OFF' || 'ON' }} jobs: build-and-test: @@ -123,7 +122,7 @@ jobs: -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - -DALP_WARNINGS_AS_ERRORS=${{ env.ALP_CI_WARNINGS_AS_ERRORS }} + -DALP_WARNINGS_AS_ERRORS=ON ${{ matrix.cmake_flags }} - name: Save installed CMake dependencies From 79898583649974c1bb7391550745e2a20b1a04ab Mon Sep 17 00:00:00 2001 From: Adam <5292991+adam-ce@users.noreply.github.com> Date: Mon, 20 Jul 2026 21:02:55 +0200 Subject: [PATCH 07/61] Update storage-format.md --- docs/raster-store/storage-format.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/raster-store/storage-format.md b/docs/raster-store/storage-format.md index 56a639b9..2c13ce99 100644 --- a/docs/raster-store/storage-format.md +++ b/docs/raster-store/storage-format.md @@ -9,8 +9,8 @@ open unless explicitly marked confirmed. ### Confirmed - The persistent hierarchy is a Web Mercator quadtree keyed by Radix tile IDs. -- Store chunks are non-overlapping and approximately 10-100 MB. -- Every stored pixel has one payload value and one source attribution. +- Store chunks are non-overlapping and approximately 10-100 MB after compression. +- Every stored pixel has one payload value (can be vector type) and one source attribution. - A coarse physical chunk may coexist with more accurate descendants. - Rendering overlap and delivery encodings are not stored authoritatively. From 617dd1d600d41f6277eb181cf62bb2ac0384bb0a Mon Sep 17 00:00:00 2001 From: Adam Celarek <5292991+adam-ce@users.noreply.github.com> Date: Sun, 26 Jul 2026 11:22:37 +0200 Subject: [PATCH 08/61] define part of the storage format --- docs/raster-store/storage-format.md | 381 +++++++--------------------- 1 file changed, 87 insertions(+), 294 deletions(-) diff --git a/docs/raster-store/storage-format.md b/docs/raster-store/storage-format.md index 2c13ce99..8be6b5eb 100644 --- a/docs/raster-store/storage-format.md +++ b/docs/raster-store/storage-format.md @@ -1,296 +1,89 @@ # Storage format -This document specifies the logical format and required invariants. Exact -binary encodings, field widths, compression libraries, and filenames remain -open unless explicitly marked confirmed. - -## Format principles - -### Confirmed - -- The persistent hierarchy is a Web Mercator quadtree keyed by Radix tile IDs. -- Store chunks are non-overlapping and approximately 10-100 MB after compression. -- Every stored pixel has one payload value (can be vector type) and one source attribution. -- A coarse physical chunk may coexist with more accurate descendants. -- Rendering overlap and delivery encodings are not stored authoritatively. - -### Proposed - -- All files have explicit magic, format version, and byte-order declarations. -- Payload and source map are held in one atomic chunk container. -- Published files are immutable. -- Checksums cover metadata and independently compressed data sections. -- Unknown optional fields can be skipped by length; incompatible required - fields cause a clear load failure. - -## Store manifest - -The root manifest should contain: - -```text -format magic and version -snapshot ID -optional parent snapshot ID -creation metadata -canonical CRS identifier and definition -canonical tile scheme -source-catalog reference -layer descriptors -``` - -The CRS must be specified more strongly than a human-readable name. The exact -EPSG/Web Mercator definition and world extent used by tile calculations must -be unambiguous. - -## Layer descriptor - -A layer descriptor should contain: - -```text -layer ID and name -semantic kind -payload pixel format -band/component interpretation -chunk width and height -NoData representation -index reference -chunk layout/codec identifier -``` - -The descriptor defines stored data. Vertex-pixel versus area-pixel delivery is -generator policy and is not required as a layer field unless the eventual -store permits more than one stored spatial placement. - -The semantic kind is still useful because it constrains valid operations. For -example, height scalars, linear colour, gamma-encoded colour, categorical -values, masks, and vectors require different filtering. - -## Source catalog - -A source-catalog record should contain a stable ID and sufficient provenance -to understand or reproduce ingestion. Candidate fields are: - -```text -SourceId -source identity and version/content fingerprint -human-readable name -original URI or path, if durable and safe to persist -original CRS and geotransform -original dimensions, bands, and pixel type -original NoData/alpha information -original nominal resolution -effective Web Mercator resolution -acquisition/publication time -priority and quality metadata -ingestion timestamp -ingestion transform and resampling policy -``` - -Pathnames may contain deployment-specific or sensitive information. The -format should support opaque source identities without requiring an original -local path. - -Source IDs are persistent across related snapshots. A snapshot that hard-links -an older chunk must retain the same meaning for every global source ID in that -chunk's local table. Snapshot-local catalog compaction may not renumber IDs -referenced by inherited containers. - -## Quadtree index file - -The index should contain: - -```text -magic -format version -tree kind = quadtree -canonical scheme -chunk layout identifier -entry count -entries: TileId → NodeStatus -optional aggregate metadata -checksum -``` - -`TileId` must be validated when decoded: - -```text -zoom is supported -x < 2^zoom -y < 2^zoom -scheme is canonical or omitted because the file declares it globally -``` - -Node status has the following invariant: - -- `Leaf` and `Inner` have a chunk file. -- `Virtual` has no chunk file. -- `Inner` and `Virtual` have at least one indexed descendant. -- `Leaf` has no indexed descendant. -- Every non-root entry has all required ancestors represented. - -The index must be reconstructible by scanning valid chunk paths. A rebuild -tool should write a new index rather than silently changing a published one. - -## Chunk container - -### Logical structure - -```text -Chunk header -├── magic and version -├── tile ID -├── layer/schema identity -├── raster dimensions -├── payload encoding -├── source-index encoding -└── section directory - -Local source table -├── local source index -└── global SourceId - -Payload sections -└── typed raster data - -Source-map sections -└── local source indexes, one per payload pixel -``` - -Payload and source-map dimensions must match exactly. A decoder rejects the -container if a source-map value is outside the local source table. - -### Source-index width - -Possible representations include: - -- `uint8_t` for at most 256 local entries; -- `uint16_t` for at most 65,536 local entries; or -- a width selected per chunk and recorded in the header. - -Adaptive width is attractive because most chunks are expected to use few -sources, but it adds codec branches. This is an open implementation decision. - -A reserved local entry may represent no valid source. If this is adopted, its -payload validity and filtering semantics must be defined rather than inferred -from an arbitrary payload value. - -### Compression and random access - -The chunk-size target makes whole-file decompression undesirable for small -windows. A proposed container divides payload and source map into matching -independently compressed blocks and records their offsets in the section -directory. - -Block dimensions should support: - -- bounded memory; -- halo reads across chunks; -- checksumming damaged regions; -- skipping source-map blocks when provenance is not requested; and -- skipping payload blocks during provenance-only inspection. - -The payload and source-map blocks need not use the same compression algorithm. -Source maps are categorical and may benefit from run-length, palette, or -general-purpose compression. The initial implementation should benchmark -representative data before fixing a codec. - -## Chunk sizing - -### Proposed sizing rule - -Choose a power-of-two side length from a logical byte budget: - -```text -logical bytes ≈ width × height × - (payload bytes per pixel + source-index bytes per pixel) -``` - -The target interval is 10-100 MB, including both large raster arrays but -excluding small metadata. For example: - -```text -4096² × (RGB8 + uint16 source) ≈ 80 MiB -4096² × (float32 + uint16 source) ≈ 96 MiB -2048² × (RGBA8 + uint16 source) ≈ 24 MiB -``` - -The examples use binary MiB despite the conversational 10-100 MB requirement; -the final specification must choose units explicitly. - -If a layer's pixel type would place 4,096² outside the budget, the layer may -choose another fixed power-of-two dimension. Varying dimensions between nodes -within one layer is not recommended because it complicates resolution and -window calculations. - -### Why compressed size should not drive identity - -Compressed size depends on terrain, imagery, source-map fragmentation, and -codec settings. Splitting chunks dynamically to hit a compressed-byte target -would make spatial identities content-dependent and destabilize updates. -A fixed spatial grid with a logical size budget gives predictable identity; -compression is an optimization. - -## Filesystem layout - -A proposed chunk path is: - -```text -layers//chunks///.rst -``` - -This is familiar and reconstructible, but very large datasets may place too -many entries in intermediate directories. Alternative layouts may shard by -Morton/Hilbert prefix. The layout identifier in layer metadata allows this to -change without changing tile identity. - -Layout parsing must accept only canonical chunk files. Auxiliary previews, -temporary files, and generated delivery images must not accidentally become -index entries during a rebuild scan. - -## Immutability and hard links - -An unchanged chunk in a child snapshot may be a hard link to the parent's -container. The following rules are mandatory if hard links are used: - -- Published containers are never modified in place. -- A changed container is written to a distinct temporary inode. -- Completion uses an atomic rename within the target filesystem. -- Snapshot deletion removes names, not shared content still referenced by - another snapshot. -- Validation may inspect inode/link counts for diagnostics but format - correctness never depends on them. - -The index, manifest, and source catalog are snapshot-specific and normally new -files even when many chunks are linked. - -## Consistency and recovery - -A snapshot is publishable only if: - -- every physical index entry resolves to one valid chunk; -- no virtual entry has a chunk; -- all chunk tile IDs and layer IDs match their paths and index entries; -- source maps reference valid local entries; -- local entries reference valid catalog records; -- dimensions and formats match the layer descriptor; and -- checksums pass. - -Temporary and incomplete files use names excluded from layout scanning. -Publication should make the complete snapshot visible in one atomic metadata -operation. The exact mechanism is open. - -## Explicitly not finalized - -This document does not yet choose: - -- binary serialization library; -- integer field widths; -- compression codecs and block dimensions; -- exact source metadata fields; -- checksum algorithm; -- file extensions and directory names; -- snapshot publication mechanism; or -- generic versus raster-specific index implementation. - -Those choices require prototypes and representative datasets, and should not -be implied by a first implementation. +This document specifies the logical format and required invariants for the following data stores: +- raster-fundamentalis (our authoritative raster store) +- tile-base (the tile-pyramid for our tile-server) +- the source-attribution table (for correct copyright attribution and data selection) + +## Source-attribution table +The source attribution table is used in raster-fundamentalis, tile-base, and in abbreviated format in the delivered tiles. +- stores a vector of structs (called Table), each struct describing one source +- the struct (called Entity) contains: + - the spatial resolution in pixel-width at the equator (EPSG:3857) + - the date of data acquisition + - the date of ingestion + - a copyright string + - a copyright link string + - a license string +- the index 0 is reserved for "no-data" +- the index must be checked to be smaller than 2^16-1, and we throw unsupported if it becomes larger (this is because of the storage format of tiles). +- the source attribution table is implemented in src/terrainlib/source_attribution.h, in the namespace source_attribution::* +- We need a `struct` declaration and a `using Table = .. ` (both versioned) +- the source-attributino table is stored in a file named source_attribution_table.ard (.alpine raster data) +- there is one per directory tree (it's valid for all tiles stored within the same directory tree, all siblings and children). +- given a tile (either rf or tb), the lookup of the source attribution table is first in the same directory and then in all parrent dirs, until a source_attribution_table.ard is found (or a failure is thrown). + +## raster-fundamentalis (rf) format +raster-fundamentalis is our authoritative raster-store, containing only the data and no overviews / downsampled version. +- The hierarchy is a Web Mercator (EPSG:3857) quadtree keyed by tile IDs (radix::tile::ID, https://docs.maptiler.com/google-maps-coordinates-tile-bounds-projection/). +- unlike a tile pyramid, not every level is occupied (there is no downsampled versions of the data). +- stored tiles have a resolution of 4096x4096 pixels +- Every stored pixel has one payload value (can be vector type) and one source attribution index. +- Coarse physical tiles (e.g. zoom level 10) may coexist with more accurate descendants (e.g. zoom level 15). +- the source attribution index is stored as uint16, and indexes into a global source attribution table (see above). +- data is stored in radix::Raster objects (one for source attribution index, one for the actual data) +- raster-fundamentalis is implemented in src/terrainlib/raster/fundamentalis.h, in the namespace raster::fundamentalis::* +- tiles are stored in .arft files (alpine raster fundamentalis tile) + +### unclear +- exact quadtree format / how to reuse structura fundamentalis code +- file names / paths / index file name + +## Tile-base Format (tb) +tile-base is a hierarchy build from raster-fundamentalis, containing all data and its overviews / downsampled versions. it is used directly by the tile-server to generate tiles at the requested resolution and format. +- The hierarchy is a Web Mercator (EPSG:3857) quadtree keyed by tile IDs (radix::tile::ID, https://docs.maptiler.com/google-maps-coordinates-tile-bounds-projection/). +- every level is occupied, and every level selects an adequate data source +- stored tiles have a resolution of 4096x4096 pixels (?) +- Every stored pixel has one payload value (can be vector type) and one source attribution index. +- the source attribution index is stored as uint16, and indexes into a global source attribution table (see above) +- data is stored in radix::Raster objects (one for source attribution index, one for the actual data) +- tile-base is implemented in src/terrainlib/raster/tile_base.h, in the namespace raster::tile_base::* +- tiles are stored in .artb files (alpine raster tile base) + +### tile-server +- should generate tiles of requested resolution and pixel type (vertex|area) on the fly +- requests by url, e.g.: layer/vertex|area/resolution/z/x/y.ending + +### unclear +- how to build tile-base from raster-fundamentalis +- details of the tile server +- file names / index file name + +## serialization / deserialization and versioning +- all files are serialised with zpp::bits in two levels + - first level contains: + - an uint64 long file type specific random magic number, generated once at coding time, as an definitive file type identifier + - a version number (uint32) + - a checksum for the payload, computed from the compressed data + - an enum for the compression algorithm + - a payload (byte vector), the second level + - the second level is a compressed byte array. the compressed payload is deserialised directly into the respective versioned data classes (tile or source attribution table) +- data structs are stored in versioned namespaces, e.g.: raster::fundamentalis::v1::Tile +- outside the versioned namespace, there is a using declaration for the newest version +- outside the versioned namespace, there is serialization function, taking only the newest version +- outside the versioned namespace, there is a deserialization function, taking a byte stream, and returning the newest version (convert to the newest version, if the payload encodes an older version) +- conversion to newer versions is done by the constructor, e.g. the v2::Tile constructor shall take a v1::Tile, and convert it to v2. once we have a v3, it would take a v2. this way we would have a conversion trail from v1 to v3. +- we have clear fails if + - the magic or version is unsupported + - if the checksum check fails + - if the compression algorithm is missing or unsupported. + - deserialization fails +- we fail by throwing + + +## to be defined +- semantic layer kind (height scalars, linear colour, gamma-encoded colour, categorical values), should be used for filtering +- human readable description? +- enumeration of layers etc? + +For now, these things will be defined in code, we will have one terrain-elevation, one surface-elevation and one ortho-photo store. later probably also a percentage store (for the snow layer) From 9350bb326ea88ed22b59ec692485d9e0b55dfae2 Mon Sep 17 00:00:00 2001 From: Adam Celarek <5292991+adam-ce@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:18:45 +0200 Subject: [PATCH 09/61] work on docs --- docs/raster-store/README.md | 20 +- docs/raster-store/architecture.md | 220 +++--------------- .../decisions-and-open-questions.md | 196 ---------------- docs/raster-store/requirements.md | 191 --------------- docs/raster-store/terminology.md | 35 +++ 5 files changed, 73 insertions(+), 589 deletions(-) delete mode 100644 docs/raster-store/decisions-and-open-questions.md delete mode 100644 docs/raster-store/requirements.md create mode 100644 docs/raster-store/terminology.md diff --git a/docs/raster-store/README.md b/docs/raster-store/README.md index 61662c83..e757e417 100644 --- a/docs/raster-store/README.md +++ b/docs/raster-store/README.md @@ -4,30 +4,14 @@ This directory describes a proposed authoritative raster store and the generation of delivery tile pyramids from it. The documents are a design baseline, not a finalized binary-format specification. -The documents distinguish three kinds of statement: - -- **Confirmed** records a requirement or decision established in the design - discussion. -- **Proposed** records the current recommendation and is subject to review. -- **Open** records a question that must be resolved before the affected part - is implemented. - ## Documents -- [Requirements and terminology](requirements.md) +- [Terminology](terminology.md) - [Status quo and reuse assessment](status-quo.md) - [Architecture](architecture.md) - [Storage format](storage-format.md) - [Sampling and pyramid generation](sampling-and-generation.md) -- [Decisions and open questions](decisions-and-open-questions.md) ## Scope -The authoritative store holds Web Mercator raster data at the best available -quality, together with exact per-pixel source attribution. It is separate from -delivery tile pyramids, which are filtered, formatted, and regenerated for a -particular consumer. - -The current scope includes raster imagery of different kinds, including -height rasters used to generate geometry. It does not define vector storage, -rendering styles, mesh formats, or a final source-ranking policy. +The documents mostly hold format information, and no tool descriptions. We will be writing tools that query and transform these formats. Tool descriptions will be given later. diff --git a/docs/raster-store/architecture.md b/docs/raster-store/architecture.md index d778b83a..356cdc25 100644 --- a/docs/raster-store/architecture.md +++ b/docs/raster-store/architecture.md @@ -5,101 +5,49 @@ The design separates authoritative data management from delivery generation: ```text -Input rasters +Input rasters (GDAL) │ - │ inspect, transform to Web Mercator, select source per pixel + │ inspect, transform to Web Mercator, define source, one source per pixel -> rf_builder ▼ -Raster-store builder +raster-fundamentalis (one rf per source at the beginning) │ - │ writes a new immutable snapshot + │ rf_merger: merge two rf stores based on (vecrtor) mask, take one tile if no overlap / far from vector border, merge strategy otherwise + │ at the beginning, have xor strategy, and exact border, later we may implement linear blending ▼ -Authoritative raster store +Authoritative rf raster store │ - ├── payload chunks - ├── per-pixel source maps - ├── source catalog - └── sparse quadtree index - │ - │ reconstruct, filter, sample, encode + │ generate overviews using defined filtering strategy, all area pixels. for now simple averaging, later maybe larger filter sizes / more complex filters ▼ -Pyramid generators - ├── area-pixel texture pyramids - └── vertex-pixel height/geometry pyramids +tile-base store (one per layer, one per data version. user visible server should only need one version per layer) + ├── read by tile-server + ├── area/vertex pixel tile generation + └── select resolution, type etc by url ``` -The store resolves which source owns each stored pixel. Generators may blend -those selected payload values but do not modify the authoritative snapshot. - ## Dataset organization ### Proposed -A store root contains immutable snapshots. Each snapshot contains a manifest, -a source catalog, and one or more named raster layers: +A store root contains immutable snapshots. Each snapshot contains an index, a source attribution table and the data: ```text store/ -└── snapshots/ - └── / - ├── manifest - ├── sources - └── layers/ - ├── heights/ - │ ├── layer - │ ├── raster.index - │ └── chunks/... - └── orthophoto/ - ├── layer - ├── raster.index - └── chunks/... +└── snapshot-id/ + ├── source_attribution_table.ard + ├── index + └── chunks/...[.arft|.artb] (or, z/x/y.[arft|artb]) ``` -The names and exact hierarchy are illustrative. The important boundaries are: - -- a source catalog has snapshot-wide identity and metadata; -- each layer has a homogeneous pixel schema; -- each layer owns a sparse quadtree index; and -- each physical quadtree node maps to one atomic chunk container. - -The store manifest records the format version, snapshot identity, parent -snapshot when applicable, publication state, Web Mercator definition, and -layer list. - -## Spatial model - -### Confirmed - -The hierarchy uses `radix::tile::Id` semantics: zoom, `x`, and `y`, with four -children per node. Inputs are transformed into Web Mercator during ingestion. - -Physical payloads may occur at several quadtree levels. A physical parent can -coexist with physical descendants so that the parent remains a coherent -fallback representation. - -### Proposed - -The persistent store uses one canonical `y` convention. Slippy/XYZ is the -current recommendation because it matches common web-map paths, but this is -not yet confirmed. API callers may convert from TMS before lookup. - -A layer chooses one fixed chunk pixel dimension. Every physical node in that -layer covers the Web Mercator bounds of its tile ID with that many stored -pixels per side. Deeper nodes therefore provide twice the linear spatial -resolution at each level. - -Store chunks are non-overlapping. Any halo required for filtering or any -shared border required by a delivery format is assembled by a generator. - ## Sparse quadtree index The index uses the same four logical states as the octree index: -| State | Chunk exists | Indexed descendants | -|---|---:|---:| -| `Leaf` | yes | no | -| `Inner` | yes | yes | -| `Virtual` | no | yes | -| `Missing` | no | no | +| State | Chunk exists | Indexed descendants | +|-----------|-------------:|--------------------:| +| `Leaf` | yes | no | +| `Inner` | yes | yes | +| `Virtual` | no | yes | +| `Missing` | no | no | `Missing` is represented by absence from the index, not serialized as an entry. @@ -124,114 +72,33 @@ entries. ## Chunk model -Each physical node owns one logical chunk: - -```text -Chunk -├── identity and schema reference -├── local source table -├── payload raster -└── source-index raster -``` - -The payload and source raster have identical dimensions. Every source-index -element names exactly one local source entry. - -### Why one atomic container is proposed - -An atomic container keeps payload and provenance inseparable, makes one -hard-link represent one unchanged chunk, and prevents a snapshot from pairing -a new payload with an old source map after interruption. - -The container may internally use independently compressed blocks to support -bounded window reads. Atomicity does not require monolithic decompression. - -## Source catalog and local tables - -### Proposed - -The source catalog assigns a stable `SourceId` to every ingested source and -records metadata such as: - -- source URI or durable identity; -- content/version identity where available; -- original CRS and transform; -- original and effective resolution; -- acquisition or publication time; -- explicit priority or quality fields; -- bands, data type, colour interpretation, and NoData information; and -- ingestion software and parameters. - -Each chunk stores a compact local source table: - -```text -local index 0 → SourceId 918 -local index 1 → SourceId 42 -local index 2 → SourceId 7001 -``` - -The source map stores local indexes rather than full global IDs. Its integer -width may be selected per chunk if the container records that width. - -This arrangement provides exact per-pixel attribution while keeping common -single- or few-source chunks compact. - -`SourceId` values referenced by a hard-linked chunk must remain stable across -the complete snapshot lineage in which that chunk is reused. A child snapshot -must preserve every catalog record referenced by inherited chunks; it cannot -renumber the catalog independently. +Each physical node owns one logical chunk, see storage-format.md ## Source selection and fallback -Source selection is a builder policy, not a property of the raster container. -Its initial comparison is expected to prioritize effective pixel resolution. - -For a candidate input, the builder conceptually performs: - -```text -for each affected stored pixel: - compare candidate source with selected source - write one winning payload value - write the winning source ID -``` - -The physical hierarchy supports a separate generator decision: - -```text -physical node is sufficiently accurate and coherent - → use it directly - -physical node is insufficient and better descendants exist - → refine and combine descendants, falling back to ancestors for gaps -``` +Source selection is a policy of the tools, not a property of the raster container. +Its initial comparison is expected to prioritize effective pixel resolution, and can be represented as a per zoom level or global ordered vector of attribution indices. -These policies must not be conflated. Pixel ownership determines the contents -of one physical chunk. Refinement determines which physical chunks contribute -to a requested derived tile. +At the beginning the selection will be binary, later we may introduce blending (over pixels of one zoom level, or several zoom levels). ## Snapshot lifecycle -### Proposed - -A build never mutates a published snapshot: +A snapshot is never mutated, instead, operations build new snapshots, while reducing disk usage by using hard links: -1. Create a private staging snapshot. -2. Load the parent snapshot's manifest and indexes when updating. -3. Identify affected chunks from candidate source bounds and resolution. -4. Hard-link unchanged chunk containers into staging. -5. Rebuild changed chunk containers. -6. Write new source catalogs, layer metadata, and indexes. -7. Validate references, dimensions, checksums, and index/file agreement. -8. Atomically publish the completed snapshot. +when adding data, we would: +1. create a new sf from the new data +2. define a validity mask for the new data +3. define a source merging priority (an ordering of sources) +4. using these two, a new snapshot is generated from the new data and the existing / authoritative snapshot, hardlinking tiles without change. Because hard links share inodes, a linked container must never be opened for -in-place modification. Changed chunks are written to new temporary paths and -renamed into place. +in-place modification. Existing snapshots are considered immutable. Obsolete snapshots can be deleted, the data will be preserved if necessary due to reference counting in the inodes. +When merging, we need to create new hardlinks for unchanged rf tiles (taken completely from either snapshot), and we need to create new rf tiles if the new tile shares information from both. Cross-filesystem hard links cannot be created. The builder must expose this as -a clear configuration or error rather than discovering it after a long build. +a clear configuration error rather than discovering it after a long build. -## Pyramid generator interface +## Pyramid generator interface (to be confirmed, LLM, do not use the following without consultation) A generator requests a layer over a target tile and sampling specification. The store reader supplies selected authoritative values and provenance over a @@ -255,18 +122,3 @@ The store owns: - exact stored source map; - source catalog lookup; and - consistent window access across chunk boundaries. - -## Scaling implications - -The 10-100 MB chunk target reduces filesystem entry and index counts but makes -whole-chunk rewrites expensive. The builder should therefore: - -- determine affected chunks before decoding parent data; -- bound the number of resident payload/source-map pairs; -- stream source windows rather than load complete input datasets; -- use internal block compression if partial reads are frequent; and -- avoid materializing world-scale tile vectors before parallel execution. - -The first implementation should prefer correctness and measurable behaviour. -Compression blocks, caches, aggregate index metadata, and scheduling should be -driven by profiles and representative datasets. diff --git a/docs/raster-store/decisions-and-open-questions.md b/docs/raster-store/decisions-and-open-questions.md deleted file mode 100644 index 02371b4f..00000000 --- a/docs/raster-store/decisions-and-open-questions.md +++ /dev/null @@ -1,196 +0,0 @@ -# Decisions and open questions - -This register prevents confirmed requirements, recommendations, and unresolved -questions from being mixed together. A future architectural decision record -may replace an entry when implementation requires a durable choice. - -## Confirmed decisions - -### D1: Separate authoritative and derived layers - -The raster store is authoritative. Delivery tile pyramids are derived and may -be regenerated with different sampling, filtering, dimensions, or codecs. - -### D2: Canonical Web Mercator ingestion - -Inputs are transformed into Web Mercator while building the authoritative -store, analogous to the SF builder transforming geometry into its canonical -ECEF space. - -### D3: Use a quadtree and Radix tile identity - -The hierarchy is two-dimensional and uses `radix::tile::Id`, not -`octree::Id`. - -### D4: Preserve physical coarse fallbacks - -A physical chunk may coexist with descendants. Generators may use the parent -as a coherent coarse representation or refine into more accurate descendants. - -### D5: Store chunks are non-overlapping - -The authoritative store does not duplicate height borders. Overlapping -rendering borders are generated from a consistent filtered signal. - -### D6: One selected value and source per stored pixel - -Every stored pixel contains exactly one payload value and one source -attribution. The store does not keep alternate candidate values for that -pixel. - -### D7: Generated provenance is tile-level - -Derived filtering may blend payload values from several sources. Generated -tiles retain provenance per tile, not per output pixel. - -### D8: Generator terminology - -Use **vertex pixel** for generated values on grid vertices and **area pixel** -for generated values associated with raster cells. The names describe required -output placement, not source measurement history. - -### D9: Large authoritative chunks - -Store tiles should be approximately 10-100 MB. They are storage chunks, not -ordinary web delivery tiles. - -### D10: Radix owns the generic raster primitive - -`radix::Raster` is the common in-memory raster representation. Store -georeferencing, provenance, persistence, and snapshot lifecycle remain outside -Radix. - -## Current recommendations - -These are proposed defaults, not confirmed decisions. - -### R1: Immutable snapshots - -Publish immutable snapshots, hard-link unchanged chunk containers, and write -changed chunks to new inodes. This avoids accidental mutation through hard -links and supports discardable failed builds. - -### R2: One atomic chunk container - -Keep payload, local source table, and source map in one container so they -cannot become inconsistent and can be reused with one hard link. - -### R3: Fixed power-of-two chunk dimension per layer - -Choose a fixed side length from a logical byte budget. Do not split spatial -identities dynamically based on compressed size. - -### R4: Snapshot-wide catalog plus local source tables - -Store full metadata once in a snapshot catalog. Let each chunk map compact -local source indexes to stable catalog IDs. - -### R5: Canonical persistent tile scheme - -Persist one scheme and normalize at API boundaries. Slippy/XYZ is the current -candidate, but TMS has not been rejected. - -### R6: Explicit versioning and validation - -New format files should have magic values, versions, checksums, strict tile-ID -validation, and a rebuildable index. The current octree index binary should -not be reused as an unlabelled quadtree format. - -### R7: Global filtering across chunk boundaries - -Generators request filter halos and operate on logical global windows. They do -not clamp at internal chunk or delivery-tile boundaries. - -## Open questions - -### Spatial and layer model - -1. Are raster kinds always separate named layers with independent indexes? -2. Which exact Web Mercator definition and world extent are canonical? -3. Is the persistent scheme Slippy/XYZ or TMS? -4. What is the maximum supported zoom? -5. What exact spatial interpretation does the store assign to its own raster - elements? This is independent of vertex-pixel/area-pixel delivery policy - but must be defined for georeferencing and reconstruction. -6. Does source accuracy vary only by source, or can it vary spatially within a - source? - -### Chunk size and physical format - -7. Does 10-100 MB mean logical bytes, compressed disk bytes, or constraints on - both? -8. Are units decimal MB or binary MiB? -9. Is chunk dimension fixed globally or selected per layer? -10. Which representative payload schemas must be used to choose between - 2,048², 4,096², or another size? -11. Does the source map use `uint8_t`, `uint16_t`, or adaptive width? -12. Which container serialization, compression, block size, and checksum are - suitable for representative imagery and source maps? -13. Is `//` sufficient for filesystem scaling, or is prefix sharding - required? - -### Source metadata and selection - -14. What creates a stable `SourceId`: catalog sequence, content hash, external - ID, or a combination? -15. Which source metadata fields are mandatory? -16. How is a pixel with no valid source represented? -17. What is the first complete source-ranking policy after pixel resolution? -18. How much resolution loss is acceptable to prefer one coherent source over - a finer mosaic? -19. How are acquisition time, quality flags, explicit priority, and ties - handled? -20. Can an update remove or invalidate a previously selected source, and if - so, where does replacement data come from when alternates are not stored? - -### Filtering and generation - -21. Which reconstruction and low-pass filters are required for heights? -22. Is terrain reduction optimized only for anti-aliasing, or also for - geometric error and preservation of extrema? -23. Which colour space and alpha convention are authoritative for imagery? -24. Which semantic raster kinds require categorical or conservative - reduction rather than linear filtering? -25. What are the exact NoData normalization rules? -26. Is tile-level provenance only a source-ID set, or does it include - contribution fractions? -27. How are Web Mercator horizontal wrap and north/south boundaries filtered? -28. Must shared generated height edges be bit-identical across separate runs - and execution orders? - -### Index and implementation structure - -29. Should the octree index/storage be generalized over hierarchical ID type, - or should the raster store receive a separate 2D implementation? -30. Should `radix::tile::Id` itself gain checked root/zoom/coordinate and - serialization APIs, or should storage use a checked adapter? -31. Does the index need aggregate coverage or best-descendant-resolution data, - or can the first version derive it during traversal? -32. How is an existing index verified against chunk files without making every - open operation scan the filesystem? - -### Snapshots and operation - -33. What atomically publishes a completed snapshot? -34. Are cross-filesystem updates rejected or allowed with a copy fallback? -35. What is the retention and garbage-collection policy? -36. How does a long build resume, and how are staged chunks validated before - being trusted? -37. Which corruption and compatibility guarantees are required for long-term - archival use? - -## Suggested order for resolving questions - -The questions do not need to be answered all at once. A practical order is: - -1. Fix the store's spatial sampling interpretation and layer model. -2. Select representative payload formats and datasets. -3. Prototype chunk dimensions, source-index widths, and block compression. -4. Decide generic versus separate index/storage implementation. -5. Specify source identity, NoData, and the initial resolution ranking. -6. Implement a single-source area-pixel path with golden grid tests. -7. Add multi-source ownership and immutable snapshots. -8. Prototype vertex-pixel height filtering and seam tests. -9. Finalize operational publication, recovery, and compatibility rules. - -Each step should produce a small approved plan before code changes begin. diff --git a/docs/raster-store/requirements.md b/docs/raster-store/requirements.md deleted file mode 100644 index e31bf841..00000000 --- a/docs/raster-store/requirements.md +++ /dev/null @@ -1,191 +0,0 @@ -# Requirements and terminology - -## Goals - -### Confirmed - -The system has two distinct layers: - -1. An authoritative raster store that retains the best representation - available for every covered pixel. -2. Derived tile pyramids generated from that store for rendering or other - delivery formats. - -Inputs may use arbitrary supported coordinate reference systems. Ingestion -transforms them into Web Mercator before placing their data in the store. - -The store must support multiple raster kinds. Height data is one raster kind; -orthophotos and other image-like data are others. Pixel format, NoData -handling, filtering, and output encoding may differ by kind. - -The store must support mixed accuracy over space. A physical payload at a -coarser quadtree node may remain available as a coherent fallback while more -accurate payloads exist in descendants. - -### Proposed - -Each named raster layer has a homogeneous value schema and its own sparse -quadtree index. Layers share the same Web Mercator tile addressing and source -catalog infrastructure but do not mix incompatible pixel types in one -payload. - -The authoritative store is output-format independent. It does not contain -rendering borders or require the sampling placement of a particular delivery -format. - -## Stored pixels and provenance - -### Confirmed - -Every stored pixel has exactly: - -- one payload value; and -- one source attribution. - -A tile carries a local source table and a source map. Each source-map element -indexes one entry in the local table. A local entry identifies source metadata -held by the store. - -The store does not blend source identifiers and does not retain several -candidate payload values for one stored pixel. Source selection is resolved -before the pixel is committed. - -Generated tiles do not retain per-pixel provenance. They record provenance at -tile granularity, for example as the set of sources that contributed to the -filtered output tile. Filtering may blend payload values across source -boundaries without changing the authoritative store's provenance model. - -### Open - -- Whether a source map refers to a dataset-wide source catalog by stable ID, - or contains complete source metadata in each tile. -- The reserved representation for a pixel with no valid source. -- Whether generated tile provenance needs only a set of source IDs or also - approximate contribution fractions. - -## Tile and chunk requirements - -### Confirmed - -Store tiles are large data chunks, not ordinary 256-pixel web delivery tiles. -Their target size is approximately 10-100 MB. - -Store tiles do not duplicate rendering overlap. In particular, overlapping -height borders are generated later and are not authoritative duplicated data. - -Tiles are addressed by a two-dimensional quadtree using -`radix::tile::Id`, not `octree::Id`. - -### Proposed - -The size target should initially be interpreted as an uncompressed logical -byte budget for payload plus source map. That gives predictable peak memory, -I/O, and rewrite cost. Compressed on-disk size varies too much with content to -be the only sizing invariant. - -The pixel side length should be a power of two. Illustrative uncompressed -sizes are: - -| Side length | Pixels | RGB8 + uint16 source | float32 + uint16 source | -|---:|---:|---:|---:| -| 2,048 | 4,194,304 | 20 MiB | 24 MiB | -| 4,096 | 16,777,216 | 80 MiB | 96 MiB | - -These calculations exclude headers, the local source table, alignment, and -compression. They demonstrate that a fixed 2,048- or 4,096-pixel chunk is in -the intended range for common formats; they do not decide the final chunk -dimension. - -### Open - -- Whether the 10-100 MB target refers to logical, compressed, or both sizes. -- Whether all layers use one fixed side length or choose it from their bytes - per pixel and source-index width. -- Whether the source map uses a fixed or per-tile adaptive integer width. -- Whether payload and provenance are one atomic container or coordinated - files. - -## Accuracy and fallback - -### Confirmed - -Pixel resolution is expected to be the most important source-accuracy signal, -but the final ranking rule is not yet known. - -A coarse, coherent source should be usable directly for a coarse output tile -even when more accurate fragments exist below it. Generators must not be -forced to patch every output from the deepest available descendants. - -### Proposed - -Source metadata records effective ground resolution after reprojection as -well as original resolution where meaningful. Source selection is a policy -interface rather than a fixed comparison embedded in the storage format. - -A pyramid generator refines into descendants only when the current physical -node does not satisfy the requested output resolution or another configured -quality criterion. - -### Open - -- The resolution tolerance that permits use of one coherent source instead - of a finer mosaic. -- Tie-breaking by acquisition time, explicit priority, quality, or source - identity. -- Whether source accuracy is scalar per source or may vary spatially. - -## Updates and reuse - -### Proposed - -Published store versions are immutable snapshots. Building a new snapshot -hard-links unchanged tile containers from an earlier snapshot and writes new -containers only for changed tiles. The snapshot owns a new index and manifest. - -This is safer than mutating hard-linked files in place and makes a failed build -discardable without damaging an earlier published store. - -Hard-link reuse requires source and destination snapshots to reside on the -same filesystem. The implementation must either require that condition or -define an explicit copy fallback. - -### Open - -- Snapshot publication and atomic rename rules. -- Retention and garbage-collection policy. -- Whether cross-filesystem builds fail or copy unchanged chunks. - -## Terminology - -**Raster store tile** or **chunk** -: A large authoritative file associated with a quadtree tile ID. It is not a - delivery tile and is expected to contain roughly 10-100 MB. - -**Payload** -: The selected raster value array stored by a chunk. - -**Source map** -: A raster aligned one-to-one with the payload. Each element selects exactly - one entry in the chunk's local source table. - -**Physical node** -: A quadtree node with an authoritative chunk on disk. - -**Virtual node** -: An index-only quadtree node with descendants but no chunk of its own. - -**Vertex pixel** -: A generated value located on a grid vertex. Height tiles for mesh generation - require vertex pixels, including shared boundary positions. - -**Area pixel** -: A generated value associated with a raster cell. Ordinary texture outputs - use area pixels whose cell boundaries align with tile boundaries. - -The terms vertex pixel and area pixel describe generator outputs. They do not -assert how an original sensor or source raster produced its values. - -**Derived tile** -: A filtered and encoded output tile generated from the authoritative store. -It may have different dimensions, sampling placement, encoding, and -provenance granularity from a store chunk. diff --git a/docs/raster-store/terminology.md b/docs/raster-store/terminology.md new file mode 100644 index 00000000..cff4ab31 --- /dev/null +++ b/docs/raster-store/terminology.md @@ -0,0 +1,35 @@ +# Terminology +** payload raster ** +: A square data matrix (image), containing the actual data (e.g. elevation, colour..) + +** attribution raster ** +: A square data matrix (image), containing indices into the source-attribution table + +** source-attribution table ** +: A table of data sources, including meta data like resolution, dates etc. + +**raster-fundamentalis (rf)** +: The authoritative raster store + +**tile-base (tb)** +: basically rf with overviews, used to generate derived tiles + +**Vertex pixel** +: A generated value located on a grid vertex. Height tiles for mesh generation + require vertex pixels, including shared boundary positions. + +**Area pixel** +: A generated value associated with a raster cell. Ordinary texture outputs + use area pixels whose cell boundaries align with tile boundaries. + +The terms vertex pixel and area pixel describe generator outputs. They do not +assert how an original sensor or source raster produced its values. + + +**tile** +: A chunk of raster data with an tile ID. Rf is a store for tiles, tb is a store for tiles, and we generate derived tiles / output tiles for the client. + +**Derived / output tile** +: A filtered and encoded output tile generated from the tile base store. +It may have different dimensions, sampling placement, encoding, and +provenance granularity from a store chunk. From f6dfb159b956d4c060bda0d5e09f92f5590d3b26 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Sun, 26 Jul 2026 18:12:01 +0200 Subject: [PATCH 10/61] Document pre-refactor SF architecture --- docs/raster-store/before-refactor/README.md | 433 ++++++++++++++++++++ 1 file changed, 433 insertions(+) create mode 100644 docs/raster-store/before-refactor/README.md diff --git a/docs/raster-store/before-refactor/README.md b/docs/raster-store/before-refactor/README.md new file mode 100644 index 00000000..f9d3d9f0 --- /dev/null +++ b/docs/raster-store/before-refactor/README.md @@ -0,0 +1,433 @@ +# SF storage and merge architecture before the 2D/3D refactor + +This report describes the existing Structura Fundamentalis (SF) builder, +sparse octree index, storage stack, and mask-based dataset merger. Its purpose +is to identify the mechanisms that can be generalised from 3D octrees to a +shared 2D/3D hierarchy without carrying mesh- and ECEF-specific behaviour into +the common storage layer. + +The central conclusion is: + +> Generalise the sparse hierarchy, storage, traversal, merge decisions, and +> unchanged-subtree reuse. Keep GDAL ingestion, raster processing, mesh +> construction, spatial transforms, and mask geometry in dimension-specific +> adapters. + +The current storage templates are already payload-generic, but they are not +hierarchy-generic: `octree::Id` is embedded throughout indexing, traversal, +caches, disk paths, and serialization. + +## Current architecture + +```mermaid +flowchart TB + subgraph Applications + SFB[sf-builder] + SFM[sf-merger] + end + + subgraph Input + GDALR[GDAL raster datasource] + GDALV[GDAL vector mask] + OLD[Base SF dataset] + NEW[New SF dataset] + end + + subgraph Domain3D["3D and mesh policy"] + SPACE[octree::Space and ECEF bounds] + MESHBUILD[Height raster to SimpleMesh] + MESHMASK[Polygon to spherical extruded MeshMask] + MESHMERGE[Clip, combine, and texture atlas] + FALLBACK[Reconstruct child by clipping ancestor mesh] + end + + subgraph GenericCandidate["Mostly generalisable mechanisms"] + INDEX[IndexMap and NodeStatus] + WALK[Depth-first and breadth-first traversal] + DRIVER[Merger and merge result actions] + STORAGE[Storage and IndexedStorage] + RAW[RawStorage] + CODEC[Codec policy] + LAYOUT[Layout strategy] + LINK[Hard-link unchanged payload] + end + + GDALR --> SFB + SFB --> SPACE + SFB --> MESHBUILD + MESHBUILD --> STORAGE + + OLD --> SFM + NEW --> SFM + GDALV --> MESHMASK + MESHMASK --> SFM + SFM --> DRIVER + DRIVER --> FALLBACK + DRIVER --> MESHMERGE + DRIVER --> STORAGE + + STORAGE --> INDEX + STORAGE --> RAW + RAW --> CODEC + RAW --> LAYOUT + RAW --> LINK + WALK --> INDEX + DRIVER --> WALK +``` + +The significant code boundaries are: + +- hierarchy identity: [`octree::Id`](../../../src/terrainlib/octree/Id.h); +- sparse topology: [`IndexMap`](../../../src/terrainlib/octree/IndexMap.h); +- traversal: [`octree::traverse`](../../../src/terrainlib/octree/traverse.h); +- logical storage: + [`Storage_`](../../../src/terrainlib/octree/storage/Storage.h); +- raw file and hard-link storage: + [`RawStorage_`](../../../src/terrainlib/octree/storage/RawStorage.h); +- disk paths: [`disk::Layout`](../../../src/terrainlib/octree/disk/Layout.h); +- merge driver: [`Merger`](../../../src/sf_merger/merge.h). + +## Sparse index model + +The index has four logical states: + +| State | Physical payload | Indexed descendants | +|---|---:|---:| +| `Missing` | no | no | +| `Virtual` | no | yes | +| `Leaf` | yes | no | +| `Inner` | yes | yes | + +`Missing` is represented by absence from `IndexMap`. The other states are +defined by +[`NodeStatus`](../../../src/terrainlib/octree/NodeStatus.h). + +```mermaid +stateDiagram-v2 + [*] --> Missing + + Missing --> Leaf: add physical root or node + Missing --> Virtual: add deeper descendant + + Virtual --> Inner: add physical payload here + Leaf --> Inner: add physical descendant + + Inner --> Leaf: remove final descendant + Leaf --> Missing: remove payload and no descendants + Virtual --> Missing: remove final descendant + Inner --> Virtual: remove payload but keep descendants +``` + +This state machine is dimension-independent. What is currently 3D-specific is +how a node finds its parent and children: `octree::Id` uses three +Morton-interleaved coordinates and eight children. + +Traversal is also reusable in concept. It: + +1. looks up the root in the sparse index; +2. visits only entries that exist; +3. enumerates children through the concrete ID type; and +4. supports depth-first or breadth-first order plus a refinement predicate. + +The only reason +[`traverse()`](../../../src/terrainlib/octree/traverse.h) +is 3D is its dependency on `octree::Id` and `octree::Id::children()`. + +## Creating an SF dataset from a GDAL datasource + +The current SF builder creates physical mesh nodes at one requested octree +level. + +```mermaid +sequenceDiagram + participant CLI as sf-builder CLI + participant DS as Dataset and GDAL + participant Space as octree::Space + participant Build as terrainbuilder + participant Reader as RawDatasetReader + participant Raster as Raster and RasterMask + participant Mesh as mesh operations + participant Store as Storage + participant Index as IndexMap and terrain.index + + CLI->>DS: Open raster datasource + DS-->>CLI: SRS, 2D bounds, and height range + + CLI->>Space: Transform bounds to ECEF + Space-->>CLI: Smallest enclosing octree node + + loop Intersecting children until target level + CLI->>Space: Node bounds and intersection tests + end + + loop Each target-level node + CLI->>Build: build_patch using node bounds + Build->>Reader: Map SRS bounds to source pixels + Reader->>DS: RasterIO band 1 as float + Reader-->>Raster: Height raster + Raster->>Raster: Build NoData validity mask + Raster->>Mesh: Generate positions and triangle grid + Mesh->>Mesh: Clip to node volume + Mesh->>Mesh: Generate UVs and transform output SRS + Mesh-->>Store: SimpleMesh with optional texture + Store->>Store: MeshCodec writes node file + end + + Store->>Index: Scan output paths + Index->>Index: Add leaves and virtual ancestors + Index->>Store: Write terrain.index +``` + +### Current components + +| Responsibility | Current component | +|---|---| +| Own and open a GDAL datasource | [`Dataset`](../../../src/terrainlib/Dataset.h) | +| One-time GDAL registration | [`initialize_gdal_once()`](../../../src/terrainlib/init.cpp) | +| ECEF octree geometry | [`octree::Space`](../../../src/terrainlib/octree/Space.h) | +| Batch enumeration and orchestration | [`build_all_patches()`](../../../src/sf_builder/terrainbuilder.cpp) | +| Direct source-window reading | [`RawDatasetReader`](../../../src/sf_builder/raw_dataset_reader.h) | +| Height-to-mesh conversion | [`build_reference_mesh_patch()`](../../../src/sf_builder/mesh_builder.cpp) | +| Optional imagery | [`TileProvider`](../../../src/sf_builder/tile_provider.h) and [`texture_assembler.h`](../../../src/sf_builder/texture_assembler.h) | +| Mesh persistence | `Storage_` | +| Sparse topology | `IndexMap` | +| Index creation and serialization | `save_or_create_index()` and `terrain.index` | + +For a new output, `build_all_patches()` opens unindexed storage. Individual +saves therefore do not build the index incrementally. The final +`save_or_create_index()` recursively scans the output directory, parses node +paths, and calls `IndexMap::add()`. + +### More suitable 2D GDAL path + +The separate +[`DatasetReader`](../../../src/tile_builder/DatasetReader.h) +is closer to what a 2D SF or raster-store builder needs: + +- it accepts requested target-SRS bounds and output dimensions; +- it creates a GDAL warped VRT; +- it reprojects into the target grid; and +- it returns `radix::Raster`. + +[`Tiler`](../../../src/tile_builder/Tiler.h) and +[`ParallelTiler`](../../../src/tile_builder/ParallelTiler.h) +already enumerate `radix::tile::Id` values over dataset bounds. + +Their grid calculations are useful, but `ParallelTileGenerator` is a +delivery-file writer rather than an SF snapshot builder. + +## Mask-based SF dataset merging + +```mermaid +flowchart TD + START[Open base and new datasets as IndexedStorage] + MASK[Read vector mask through GDAL and OGR] + PREP[Polygon repair, sphere projection, triangulation, radial extrusion] + ROOT[Start at octree root] + STATUS[Read left and right NodeStatus] + POLICY[Masked visitor evaluates node] + RECURSE[Clip mask to node bounds and recurse into 8 children] + LEFT[Keep base subtree] + RIGHT[Keep new subtree] + MERGE[Clip base outside mask and new inside mask] + ATLAS[Combine meshes and rebuild texture atlas] + WRITE[Write new physical node] + COPY[Traverse unchanged source subtree] + LINK[Hard-link each physical payload] + INDEX[Scan and save new output index] + + START --> ROOT + MASK --> PREP --> POLICY + ROOT --> STATUS --> POLICY + + POLICY -->|virtual or refinement required| RECURSE + RECURSE --> STATUS + + POLICY -->|unchanged left| LEFT --> COPY + POLICY -->|unchanged right| RIGHT --> COPY + COPY --> LINK + + POLICY -->|boundary node| MERGE --> ATLAS --> WRITE + POLICY -->|no retained data| INDEX + + LINK --> INDEX + WRITE --> INDEX +``` + +### Merge components + +- [`Merger`](../../../src/sf_merger/merge.h) is the recursive dispatcher. +- [`NodeLoader`](../../../src/sf_merger/NodeLoader.h) reports status and loads + payloads. +- When an exact payload is missing, `NodeLoader` searches physical ancestors + and reconstructs the requested child by clipping the ancestor mesh. That + reconstruction is 3D-specific. +- [`NodeData`](../../../src/sf_merger/merge/NodeData.h) lazily exposes a node + payload to a visitor. +- [`Result`](../../../src/sf_merger/merge/Result.h) gives the driver four + actions: recurse, ignore, preserve one source unchanged, or write a merged + payload. +- [`Masked`](../../../src/sf_merger/merge/visitor/Masked.h) implements + mesh-and-mask policy. +- [`NodeWriter`](../../../src/sf_merger/NodeWriter.h) writes changed nodes or + copies unchanged subtrees. + +The vector-mask pipeline in +[`mask.h`](../../../src/sf_merger/mask.h) +is almost entirely 3D and Earth-specific after OGR polygon loading: + +```text +OGR polygons + -> referenced 2D polygon mask + -> ECEF and spherical projection + -> triangulated surface + -> extrusion over an Earth-radius interval + -> closed 3D MeshMask +``` + +For 2D raster merging, only the OGR polygon loading and CRS transformation +ideas remain relevant. Triangulation, spherical projection, extrusion, 3D +clipping, and texture atlases should not enter the generic core. + +## Hard-link behaviour + +Hard links are implemented at the lowest storage layer in +[`RawStorage_::copy_from()`](../../../src/terrainlib/octree/storage/RawStorage.h): + +```mermaid +flowchart TD + COPY[copy_from node] + EXISTS{Source payload exists?} + EXT{Source and target extensions match?} + DECODE[Decode source payload] + ENCODE[Encode into target format] + REMOVE[Remove existing target] + DIRS[Create parent directories] + HARDLINK[Create hard link] + DONE[Add physical node to target index] + + COPY --> EXISTS + EXISTS -->|no| ERROR1[FileNotFound] + EXISTS -->|yes| EXT + EXT -->|no| DECODE --> ENCODE --> DONE + EXT -->|yes| REMOVE --> DIRS --> HARDLINK --> DONE +``` + +Properties: + +- Hard linking is payload-agnostic and fully generalisable. +- It happens only when source and target filename extensions match. +- Different formats cause decode and re-encode through the codec. +- There is no copy fallback if hard-link creation fails, including across + filesystems. +- Snapshot immutability is not enforced by `Storage_`; it is a caller-level + convention. +- The existing target file is removed before the link is created. +- The new output receives a fresh index; `terrain.index` itself is not linked. + +## Generalisation assessment + +| Mechanism | Assessment | Required change | +|---|---|---| +| `NodeStatus` state model | Reuse essentially unchanged | Move outside `octree` naming | +| `IndexMap` state transitions | Generalise | Template over node key and tree traits | +| Sparse traversal | Generalise | Obtain children through tree traits | +| `Storage_` | Good starting point | Also template over node key and layout | +| `RawStorage_` hard-link behaviour | Generalise | Key-neutral paths and explicit fallback policy | +| Codec concept | Reuse | No dimensional dependency | +| Cache interface | Generalise | Key type is currently `octree::Id` | +| Layout strategy | Generalise | Key-neutral path API | +| Index file | Replace or version | Record topology kind, format version, and validated key | +| Merge result algebra | Generalise | `Merged` must hold generic payload, not `SimpleMesh` | +| Recursive merge driver | Generalise | Tree traits, payload, loader/writer, and policy | +| Unchanged-subtree reuse | Reuse | Enumerate every physical state, including `Inner` | +| GDAL datasource ownership | Reuse or adapt | Typed, multi-band reads and explicit NoData | +| Mask loading through OGR | Reuse or adapt | Produce dimension-specific mask representation | +| `octree::Space` | Keep 3D | Add a sibling 2D grid or space policy | +| Mesh clipping, atlas, and UV code | Keep 3D | Do not place in generic storage | +| ECEF and spherical mask conversion | Keep 3D | 2D uses polygon classification or rasterisation | +| `MeshCodec` | Keep 3D | Add a raster chunk codec or container | + +## Existing `Inner` limitation + +The index supports `Inner`, but the SF merger effectively does not: + +- `Merger::call_merge()` dispatches only `Missing`, `Leaf`, and `Virtual` + combinations. +- `Masked::visit()` declares `Inner` unreachable. +- `NodeWriter::copy_subtree_to_output()` skips `Virtual` and asserts every + other visited node is `Leaf`. + +This matters because a physical coarse node coexisting with physical +descendants is exactly the `Inner` case. A general 2D/3D implementation should +make "has a physical payload" and "has indexed descendants" independent +properties and handle all four states throughout traversal and merging. + +## Recommended target architecture + +```mermaid +flowchart TB + subgraph Core["Dimension-neutral hierarchical store"] + TRAITS["TreeTraits<Key>
root, parent, children, validation"] + SI["SparseIndex<Key>"] + TR["Traversal<Key>"] + ST["Storage<Key, Payload, Codec>"] + DL["DiskLayout<Key>"] + ME["MergeEngine<Key, Payload, Policy>"] + SNAP[Snapshot and subtree reuse] + end + + subgraph D2["2D adapter"] + TILE[radix::tile::Id] + GRID[Web Mercator grid] + RASTER[RasterChunk] + GDAL[GDAL warped-window reader] + MASK2[2D polygon or raster mask policy] + RC[Raster codec and container] + end + + subgraph D3["3D adapter"] + OCT[octree::Id] + ECEF[ECEF octree space] + MESH[SimpleMesh] + MASK3[Extruded mesh-mask policy] + MC[MeshCodec] + end + + TILE --> TRAITS + OCT --> TRAITS + + RASTER --> ST + MESH --> ST + RC --> ST + MC --> ST + + MASK2 --> ME + MASK3 --> ME + + SI --> TR + SI --> ST + DL --> ST + TR --> ME + ST --> SNAP + ME --> SNAP +``` + +The central abstractions should therefore be: + +1. `TreeTraits`: root, parent, children, maximum depth, and key + validation. +2. `SparseIndex`: the current `IndexMap` algorithm. +3. `Traversal`: DFS, BFS, and refinement independent of child + count. +4. `DiskLayout`: key-to-path and path-to-key conversion. +5. `Storage`: logical storage and index maintenance. +6. `MergeEngine`: paired sparse-tree walking. +7. Generic merge actions such as `Recurse`, `Ignore`, `KeepLeft`, `KeepRight`, + and `Write`. +8. A snapshot copier that hard-links unchanged physical nodes without knowing + their payload type. +9. Separate 2D and 3D spatial, mask, and fallback policies. + +The GDAL builder and mask merger should be clients of this core, not part of +it. From 037501c9d4f307a15d195410bd7c7a9bc15ac479 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Mon, 27 Jul 2026 02:09:33 +0200 Subject: [PATCH 11/61] docs: add 2D and 3D store refactor plan --- docs/raster-store/refactor-plan.md | 577 +++++++++++++++++++++++++++++ 1 file changed, 577 insertions(+) create mode 100644 docs/raster-store/refactor-plan.md diff --git a/docs/raster-store/refactor-plan.md b/docs/raster-store/refactor-plan.md new file mode 100644 index 00000000..026ef6e1 --- /dev/null +++ b/docs/raster-store/refactor-plan.md @@ -0,0 +1,577 @@ +# 2D/3D hierarchical store refactor plan + +Status: proposal for review. This document is an implementation plan, not a +record of completed work. + +## Decisions already made + +- The shared implementation will live in `src/terrainlib/store` and use the + `store` namespace. +- The 2D implementation will live in `src/terrainlib/raster_store` and use the + `raster_store` namespace. +- Existing 3D Structura Fundamentalis datasets must remain readable and + writable without changing their on-disk contract. +- 3D compatibility includes both existing path layouts: + `flat` and `level_and_coordinate_directories`. +- The 2D raster-fundamentalis format is a separate format. Requirements in + this directory apply to that 2D format and must not be retrofitted onto + existing 3D datasets. +- A path layout strategy should contain a stable identifier and two + operations: key to relative path, and relative path to key. It should not + require an inheritance hierarchy, RTTI, global self-registration, or heap + allocation. +- 3D geometry, ECEF bounds, mesh codecs, mesh reconstruction, mask geometry, + and raster-specific processing remain outside the shared store. + +## Goals + +1. Use one sparse hierarchy implementation for `octree::Id` and + `radix::tile::Id`. +2. Use one traversal, storage, cache, codec boundary, and unchanged-subtree + copier for 2D and 3D. +3. Make paired-tree walking reusable without putting mesh or raster merge + policy into the shared layer. +4. Replace the current layout-strategy class hierarchy with small path-mapping + values backed by function pairs. +5. Preserve all valid existing 3D index files and payload paths. +6. Introduce the 2D storage adapter without inventing unspecified raster file + details. +7. Land the refactor in small, testable steps. Every phase should build and + pass tests before the next phase begins. + +## Non-goals + +- Changing `octree::Id`, `octree::Space`, `IdRect`, `OddLevelShifted`, or + other 3D spatial calculations. +- Defining or implementing GDAL ingestion, raster resampling, filtering, + source selection, or mask rasterisation. +- Defining the `.arft` payload or source-attribution-table serialization + beyond the requirements already in [storage-format.md](storage-format.md). +- Implementing an `rf_builder`, `rf_merger`, tile-base generator, or tile + server in this refactor. +- Changing the existing 3D hard-link policy by adding a silent file-copy + fallback. +- Refactoring unrelated octree, DAG, mesh, or tile-builder code. + +## Compatibility contract + +Before moving code, tests must lock down the following 3D behaviour: + +| Item | Required compatibility | +|---|---| +| Index filename | `terrain.index` | +| Index field order | layout ID, preferred extension, index map | +| Node-key encoding | existing `octree::Id` level/index serialization | +| Node-status encoding | `Leaf = 0`, `Inner = 1`, `Virtual = 2` | +| Flat layout ID | `flat` | +| Flat path | `-` | +| Coordinate layout ID | `level_and_coordinate_directories` | +| Coordinate path | `///` | +| Default layout | existing level/coordinate layout | +| Layout detection | both existing layouts remain detectable | +| Copy with equal extensions | hard link, or an explicit error | +| Copy with different extensions | decode and encode through the codec | + +Compatibility means that the refactored code can open datasets written before +the refactor and produces datasets that the pre-refactor code can open. Exact +byte-for-byte rewriting of an unordered index map is not required, but the +serialized schema and values must remain compatible. + +The 3D index disk type should remain a versioned 3D adapter. The shared store +must not add a topology field, new header, checksum, or compression layer to +`terrain.index`. + +## Proposed source boundary + +```text +src/terrainlib/ +├── store/ +│ ├── NodeStatus.h +│ ├── NodeStatusOrMissing.h +│ ├── Traits.h +│ ├── Index.h +│ ├── traverse.h +│ ├── PathMapping.h +│ ├── Layout.h +│ ├── Codec.h +│ ├── CopyError.h +│ ├── RawStorage.h +│ ├── Storage.h +│ ├── IndexedStorage.h +│ ├── copy_subtree.h +│ ├── cache/ +│ │ ├── Interface.h +│ │ ├── Dummy.h +│ │ └── Lru.h +│ └── merge/ +│ ├── Action.h +│ └── walk.h +├── octree/ +│ ├── Id.h +│ ├── StoreTraits.h +│ ├── store_layout/ +│ │ ├── Flat.h +│ │ ├── LevelAndCoordinateDirectories.h +│ │ └── Mappings.h +│ └── storage/ +│ ├── IndexFile.h +│ └── open.h +└── raster_store/ + ├── StoreTraits.h + ├── IndexFile.h + ├── Storage.h + └── store_layout/ + └── Zxy.h +``` + +The exact file grouping may be collapsed if a file would only contain a few +lines. The important boundaries are: + +- `store` contains dimension- and payload-neutral mechanisms; +- `octree` contains the legacy 3D format and key adapters; +- `raster_store` contains the new 2D format and key adapters; and +- subdirectory names match their namespaces where a subnamespace is used. + +Temporary forwarding headers and aliases under `octree` are allowed during +migration. They must not contain a second implementation. + +## Shared interfaces + +The names below are the intended shape, not signatures that must be copied +verbatim without testing. + +### Hierarchy traits + +The store should be parameterized by one traits type rather than assuming that +all key classes expose identical member functions: + +```cpp +template +concept HierarchyTraits = requires(typename Traits::Key key) { + typename Traits::Key; + typename Traits::Hasher; + { Traits::root() } -> std::same_as; + { Traits::parent(key) }; + { Traits::children(key) }; + { Traits::is_valid(key) } -> std::same_as; +}; +``` + +The concrete names should be: + +```cpp +store::Index +store::Index +``` + +`octree::StoreTraits` adapts the existing optional parent/children API without +changing `octree::Id`. + +`raster_store::StoreTraits` adapts `radix::tile::Id` and must: + +- treat zoom zero as the only root; +- never call `radix::tile::Id::parent()` at zoom zero, where it underflows; +- reject coordinates outside `[0, 2^zoom)`; +- define a supported maximum zoom without an overflowing shift; +- use `radix::tile::Id::Hasher`; and +- accept only one canonical XYZ/Slippy interpretation at the persistent + boundary. + +The shared code must obtain roots, parents, children, validation, and hashing +through the traits. It must not use dimension checks or specialize behaviour +on key types internally. + +### Sparse index and traversal + +Move the existing four-state model to: + +```cpp +store::NodeStatus +store::NodeStatusOrMissing +store::Index +store::traverse(index, visitor, refine, root, order) +``` + +The index algorithm remains the current one: + +- adding a physical descendant creates virtual ancestors; +- adding a payload to a virtual node makes it `Inner`; +- removing a payload from an `Inner` node makes it `Virtual`; +- removing the final descendant collapses virtual ancestors; and +- a physical parent becomes `Leaf` after its last descendant is removed. + +Traversal must follow only indexed nodes and use `Traits::children`. Child +order is the order supplied by the traits and is therefore deterministic per +hierarchy, not universally fixed by `store`. + +### Path mappings + +Replace `octree::disk::layout::Strategy` and +`octree::disk::layout::StrategyRegister` with a value similar to: + +```cpp +template +struct store::PathMapping { + std::string_view id; + std::filesystem::path (*key_to_path)( + const Key&, std::string_view extension_with_dot); + std::optional (*path_to_key)( + const std::filesystem::path& relative_path); +}; +``` + +`store::Layout` owns the base directory, preferred extension, and one +`PathMapping`. It only adds/removes the base directory around the two +mapping functions. + +The stable ID is format metadata, not a third strategy operation. The +dimension adapters provide ordinary lookup functions: + +```cpp +octree::store_layout::flat() +octree::store_layout::level_and_coordinate_directories() +octree::store_layout::from_id(id) +octree::store_layout::all() + +raster_store::store_layout::zxy() +raster_store::store_layout::from_id(id) +``` + +This retains runtime selection from an index file while removing virtual +dispatch, RTTI type-to-ID lookup, static registration, and ownership through +`unique_ptr`. + +Path parsers must validate the complete relative path and the expected file +extension at the `Layout` boundary. They must return an error or `nullopt`; +they must not assert on input read from disk. + +### Storage and format adapters + +Generalize these mechanisms over traits, payload, and codec: + +```cpp +store::RawStorage +store::Storage +store::IndexedStorage +store::cache::Interface +``` + +The payload codec remains path-based and key-neutral. Mesh and raster codecs +remain with their payload domains. + +Index serialization is not a responsibility of `store::Index`. Opening and +saving a dataset must receive a dimension-specific format adapter which +provides: + +- the index filename; +- index read/write conversion; +- mapping lookup by stable ID; +- the default mapping; and +- the mappings considered during legacy directory scanning. + +This adapter may be a compile-time policy or a small value of function +pointers. Choose the smaller implementation after the Phase 0 tests exist. +It must not reintroduce a class hierarchy or global registration. + +For 3D, the adapter reads and writes the current `octree` index DTO unchanged. +For 2D, it reads and writes a separately versioned +`raster_store::v1` DTO using the serialization envelope required by +[storage-format.md](storage-format.md). + +Automatic dirty-index saving currently happens in the 3D storage destructor. +Preserve that behaviour for existing 3D entry points during the migration. +The new 2D snapshot API should require an explicit finalization/publication +step; a destructor must not make an incomplete snapshot authoritative. + +### Unchanged-subtree reuse + +Move payload-neutral subtree copying out of `sf_merger::NodeWriter`. The +shared operation should: + +1. traverse an indexed source subtree; +2. skip `Virtual` nodes; +3. copy physical payloads for both `Leaf` and `Inner`; +4. use `Storage::copy_from` so equal-format payloads are hard-linked; +5. update the target index incrementally; and +6. return an error instead of asserting or terminating. + +This fixes the current incorrect assumption that every non-virtual visited +node is a `Leaf`. + +Hard-link rules: + +- never modify an existing linked payload in place; +- matching extensions require hard-link creation; +- different extensions retain the current decode/re-encode path; +- hard-link failure is explicit; +- 2D snapshot tools must preflight that source and destination support hard + links before a long operation starts; and +- no silent copy fallback is introduced by this refactor. + +### Paired hierarchy walking + +Extract only the dimension-neutral control flow from `sf_merger::Merger`. +The shared walker obtains the left and right status for a key and asks a +policy for one of: + +```cpp +store::merge::Recurse +store::merge::Ignore +store::merge::KeepLeft +store::merge::KeepRight +store::merge::Write +``` + +The walker owns recursion and unchanged-subtree reuse. The policy owns +selection and payload combination. + +All 16 combinations of `Missing`, `Leaf`, `Inner`, and `Virtual` must be +handled. Unsupported combinations may return a typed error, but they must not +fall into `UNREACHABLE()`. + +The existing 3D adapter retains: + +- `NodeLoader` ancestor mesh reconstruction; +- ECEF node bounds; +- mesh masks and clipping; +- mesh combination and texture atlas generation; and +- mesh validation and auxiliary texture writes. + +A future 2D merger can supply a raster policy without changing the shared +walker. Implementing that policy is outside this refactor. + +## Implementation phases + +Each phase should be one reviewable commit unless the tests and implementation +are clearer as two commits. Do not begin a later phase while the current phase +has failing tests. + +### Phase 0 — Capture current compatibility + +No production behaviour changes. + +1. Add golden 3D fixtures created by the current code: + - one `terrain.index` using `flat`; + - one using `level_and_coordinate_directories`; + - physical payload paths for a root, child, and deeper descendant; and + - an index containing `Leaf`, `Virtual`, and `Inner`. +2. Test that both fixtures open, resolve the expected IDs and extensions, and + traverse the expected sparse nodes. +3. Add path round-trip tests for boundary IDs and both layouts. +4. Add storage tests for: + - matching-extension hard links; + - different-extension decode/re-encode; + - overwrite rejection; + - indexed and unindexed opens; and + - final index creation by directory scan. +5. Record the pre-refactor public aliases used by `sf_builder`, `sf_merger`, + `sf_index_browser`, `dag_builder`, and `dag_convert_debug`. + +Exit criterion: the compatibility tests pass against the untouched +implementation and fail when any stable filename, layout ID, path encoding, +status value, or index field order is deliberately changed. + +### Phase 1 — Extract topology into `store` + +1. Move `NodeStatus` and `NodeStatusOrMissing` to `store`, preserving their + underlying values and serialization. +2. Introduce the hierarchy-traits concept and `octree::StoreTraits`. +3. Convert `IndexMap` into `store::Index`. +4. Convert traversal into `store::traverse`. +5. Add `raster_store::StoreTraits` for `radix::tile::Id`. +6. Run the same index-transition and DFS/BFS tests with both trait types. +7. Provide temporary `octree` aliases so downstream migration is separate + from the algorithm extraction. + +Exit criterion: 2D and 3D keys pass the same topology suite; existing 3D +callers still build through aliases; no filesystem code has changed. + +### Phase 2 — Replace disk layout strategies + +1. Add `store::PathMapping` and `store::Layout`. +2. Port the two existing 3D layouts to ordinary function pairs without + changing paths or stable IDs. +3. Replace the singleton strategy registry with explicit `from_id()` and + `all()` functions in the 3D adapter. +4. Port layout guessing to consume a span of mappings supplied by the format + adapter. +5. Add the proposed 2D `z/x/y.arft` mapping only after the review decision + listed below is resolved. +6. Switch path and layout-detection tests to the new implementation. +7. Delete the old strategy base class, registration machinery, and concrete + strategy classes once no call site uses them. + +Exit criterion: both legacy fixtures resolve to identical payload paths; +there is no layout inheritance, RTTI lookup, static registrar, or owning +strategy pointer. + +### Phase 3 — Generalize storage and index lifecycle + +1. Move the codec concept, copy error, raw storage, caches, logical storage, + and indexed storage into `store`. +2. Replace every embedded `octree::Id` with `Traits::Key`. +3. Keep payload codecs outside the shared module: + `octree::MeshCodec` remains 3D, and the future `.arft` codec remains under + `raster_store`. +4. Split generic directory scanning from 3D index serialization. +5. Implement the small format-adapter boundary described above. +6. Keep the current 3D `terrain.index` DTO and open functions as compatibility + adapters over the shared storage. +7. Migrate the existing octree storage aliases and all application callers. +8. Preserve the current 3D destructor-save behaviour until all callers have + explicit index finalization. + +Exit criterion: all existing applications build and all Phase 0 fixtures pass +through the shared storage implementation. No second storage implementation +remains under `octree`. + +### Phase 4 — Generalize subtree reuse and paired walking + +1. Add the shared unchanged-subtree copier. +2. Test copies containing `Leaf`, `Virtual`, and `Inner` nodes. +3. Add the paired hierarchy walker and typed actions. +4. Cover all 16 status pairs with table-driven tests. +5. Adapt the 3D merger to the shared walker while keeping mesh policy in + `sf_merger`. +6. Remove generic recursion and copy logic from `sf_merger::Merger` and + `NodeWriter`. +7. Add a 3D integration test proving an unchanged subtree is hard-linked and + a changed boundary node is newly written. + +Exit criterion: the existing 3D merger behaviour is preserved, `Inner` no +longer reaches `UNREACHABLE()`, and the shared walker contains no mesh, ECEF, +GDAL, OpenCV, or raster dependencies. + +### Phase 5 — Add the 2D raster-fundamentalis adapter + +This phase starts only after the unresolved 2D format decisions below are +edited into decisions in this document. + +1. Add the checked 2D persistent-key conversion around `radix::tile::Id`. +2. Add the chosen 2D path mapping and stable ID. +3. Define the versioned 2D index DTO and index filename. +4. Use the required magic/version/checksum/compression serialization envelope. +5. Add the 2D format adapter and storage aliases under + `raster_store`. +6. Exercise storage with a small test codec if the final `.arft` codec is not + implemented yet; do not make `.arft` claims from a placeholder codec. +7. Test: + - invalid and boundary tile IDs; + - index serialization and validation; + - `Leaf`/`Inner` coexistence; + - sparse traversal and ancestor lookup; + - path round trips; + - snapshot hard-link reuse; and + - explicit cross-filesystem/preflight failure. + +Exit criterion: the same shared store can create, open, traverse, and reuse a +minimal 2D raster-fundamentalis fixture without changing the 3D fixtures. + +### Phase 6 — Cleanup and documentation + +1. Remove temporary forwarding headers that no repository caller needs. +2. Remove obsolete files under `octree/disk` and the old generic + implementation under `octree/storage`. +3. Keep only 3D key, format, codec, and compatibility adapters under + `octree`. +4. Update includes, CMake source lists, and precompiled-header includes. +5. Update [architecture.md](architecture.md), + [status-quo.md](status-quo.md), and the before-refactor report with links to + the implemented boundary. Preserve the before-refactor report as history; + do not rewrite it as if it described the new code. +6. Document the final public names and a minimal 2D/3D opening example. + +Exit criterion: repository search finds no generic implementation tied to +`octree::Id`; all tests pass; the old layout strategy hierarchy is gone. + +## Test and verification plan + +Tests should live in the existing `unittests_terrainlib` target. Suggested +files: + +```text +unittests/terrainlib/store_index.cpp +unittests/terrainlib/store_traverse.cpp +unittests/terrainlib/store_layout.cpp +unittests/terrainlib/store_storage.cpp +unittests/terrainlib/store_compatibility.cpp +unittests/terrainlib/store_merge_walk.cpp +``` + +During implementation: + +1. Build in `$source_dir/build/$config_name`. +2. Run unit tests from that build directory. +3. Run the focused store tests after each edit. +4. Run the full `unittests_terrainlib` target at every phase boundary. +5. Build `sf_builder`, `sf_merger`, `sf_index_browser`, `dag_builder`, and + `dag_convert_debug` after their storage aliases move. +6. Run any existing merger integration fixture after Phase 4. +7. Inspect `git diff --check` and the final worktree before each commit. + +No formatting-only pass or unrelated refactor belongs in these commits. + +## Expected migration map + +| Current code | Target | +|---|---| +| `octree/NodeStatus.h` | `store/NodeStatus.h` plus temporary alias | +| `octree/NodeStatusOrMissing.h` | `store/NodeStatusOrMissing.h` plus temporary alias | +| `octree/IndexMap.*` | `store/Index.h` | +| `octree/traverse.h` | `store/traverse.h` | +| `octree/disk/Layout.h` | `store/Layout.h` | +| `octree/disk/layout/Strategy.h` | `store/PathMapping.h` | +| `StrategyRegister.h` | explicit dimension-adapter lookup functions | +| `strategy/Flat.h` | `octree/store_layout/Flat.h` | +| `strategy/LevelAndCoordinateDirectories.h` | `octree/store_layout/LevelAndCoordinateDirectories.h` | +| `octree/storage/cache/*` | `store/cache/*` | +| `octree/storage/codec/Codec.h` | `store/Codec.h` | +| `octree/storage/RawStorage.h` | `store/RawStorage.h` | +| `octree/storage/Storage.h` | `store/Storage.h` | +| `octree/storage/IndexedStorage.h` | `store/IndexedStorage.h` | +| `octree/storage/helpers.*` | generic scan helpers plus 3D format adapter | +| `octree/disk/IndexFile.h` | versioned 3D format adapter under `octree` | +| `sf_merger::NodeWriter` subtree loop | `store/copy_subtree.h` | +| `sf_merger::Merger` recursion | `store/merge/walk.h` | + +## Risks and controls + +| Risk | Control | +|---|---| +| Existing indexes stop loading | Golden pre-refactor fixtures and unchanged 3D DTO | +| Valid legacy paths are parsed differently | Characterization and round-trip tests before replacement | +| Template migration creates a large unreviewable diff | Compatibility aliases and phase-by-phase caller migration | +| `radix::tile::Id` root underflows | Traits intercept root parent lookup | +| Invalid 2D coordinates become persistent | Validate on every disk/API boundary | +| `Inner` payloads are lost during subtree reuse | Copy every physical status and test mixed-depth fixtures | +| Linked snapshots are modified in place | Immutable snapshot API and overwrite-disabled output | +| Hard-link failure appears late | 2D operation preflight and explicit errors | +| Shared code accumulates mesh/raster policy | Dependency tests/review against the source boundary | +| Generic index accidentally dictates both disk formats | Separate 3D and 2D format adapters | + +## Decisions required before Phase 5 + +These are intentionally unresolved because the current raster-store documents +mark them as unclear: + +1. **2D index filename:** choose the filename used inside a + raster-fundamentalis snapshot. +2. **2D payload path:** confirm `z/x/y.arft`, including whether chunks live + directly under the snapshot or below a `chunks/` directory. +3. **2D layout ID:** choose the stable string serialized in the index. +4. **Maximum zoom:** choose the supported persistent range and integer widths + for zoom, x, and y. +5. **Index contents:** decide whether v1 stores only sparse status entries or + also derived aggregate metadata. The first implementation should omit + derivable metadata unless a concrete query requires it. +6. **Index envelope constants:** assign the 2D index magic number, compression + choice, and version according to the common serialization rules. +7. **Publication:** define whether snapshot completion uses an atomic rename, + a manifest marker, or an external store-level operation. The generic + storage layer should expose finalization but not invent store-level + lifecycle policy. + +Until these decisions are made, Phases 0–4 can complete and the shared +implementation can be proven with `radix::tile::Id` in memory and with +temporary-directory tests. No provisional 2D disk format should escape into +production data. From 3c6c9f8fe7c7ff9737f6ab7ecec81eb72d1445d2 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Mon, 27 Jul 2026 02:26:21 +0200 Subject: [PATCH 12/61] Document NodeLoader cache hazard --- src/terrainlib/octree/storage/cache/LruCache.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/terrainlib/octree/storage/cache/LruCache.h b/src/terrainlib/octree/storage/cache/LruCache.h index fcf4f20f..1c705e05 100644 --- a/src/terrainlib/octree/storage/cache/LruCache.h +++ b/src/terrainlib/octree/storage/cache/LruCache.h @@ -11,6 +11,9 @@ namespace octree::cache { // TODO: UNTESTED +// Do not use this cache for NodeLoader's ancestor lookup until cached ancestors +// are clipped to the requested node bounds. NodeLoader currently returns a +// cached ancestor unchanged, producing incorrect geometry for child requests. template class Lru_ : public ICache { public: From fa3282c03451479c3c60629447ba124ef312ca2b Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Mon, 27 Jul 2026 02:54:11 +0200 Subject: [PATCH 13/61] Make progress monitoring cancellable --- src/terrainlib/ProgressIndicator.cpp | 38 ++++++++++++------- src/terrainlib/ProgressIndicator.h | 4 ++ .../terrainlib/progress_indicator_test.cpp | 27 +++++++++++++ 3 files changed, 56 insertions(+), 13 deletions(-) diff --git a/src/terrainlib/ProgressIndicator.cpp b/src/terrainlib/ProgressIndicator.cpp index 565f7ee8..a9f49272 100644 --- a/src/terrainlib/ProgressIndicator.cpp +++ b/src/terrainlib/ProgressIndicator.cpp @@ -39,6 +39,7 @@ void ProgressIndicator::task_finished() { if (m_step > m_n_steps) { throw std::runtime_error("Too many steps reported."); } + m_monitor_condition.notify_all(); } std::jthread ProgressIndicator::start_monitoring() const { @@ -50,19 +51,30 @@ std::jthread ProgressIndicator::start_monitoring() const { }; const auto t0 = std::chrono::steady_clock::now(); - std::jthread thread([=, this]() { - const auto delta_t = 500ms; - auto v_t_minus_1 = this->m_step.load(); - do { - auto v_t = this->m_step.load(); - const auto delta_v = v_t - v_t_minus_1; - v_t_minus_1 = v_t; - print(delta_v, delta_t); - std::this_thread::sleep_for(delta_t); - } while (this->m_step < this->m_n_steps); - const auto t1 = std::chrono::steady_clock::now(); - print(this->m_n_steps, std::chrono::duration_cast(t1 - t0)); - std::cout << std::endl; + std::jthread thread([=, this](std::stop_token stop_token) noexcept { + try { + const auto delta_t = 500ms; + auto v_t_minus_1 = this->m_step.load(); + while (!stop_token.stop_requested() && this->m_step < this->m_n_steps) { + const auto v_t = this->m_step.load(); + const auto delta_v = v_t - v_t_minus_1; + v_t_minus_1 = v_t; + print(delta_v, delta_t); + + std::unique_lock lock(this->m_monitor_mutex); + this->m_monitor_condition.wait_for( + lock, stop_token, delta_t, [this]() { return this->m_step >= this->m_n_steps; }); + } + + if (!stop_token.stop_requested()) { + const auto t1 = std::chrono::steady_clock::now(); + print(this->m_n_steps, std::chrono::duration_cast(t1 - t0)); + std::cout << std::endl; + } + } catch (...) { + // Console output and formatting must not terminate the process from + // inside the monitoring thread. + } }); return thread; } diff --git a/src/terrainlib/ProgressIndicator.h b/src/terrainlib/ProgressIndicator.h index fcf34eb0..ca16e36e 100644 --- a/src/terrainlib/ProgressIndicator.h +++ b/src/terrainlib/ProgressIndicator.h @@ -21,12 +21,16 @@ #include #include +#include +#include #include #include class ProgressIndicator { const size_t m_n_steps; std::atomic m_step = 0; + mutable std::condition_variable_any m_monitor_condition; + mutable std::mutex m_monitor_mutex; public: ProgressIndicator(size_t n_steps); diff --git a/unittests/terrainlib/progress_indicator_test.cpp b/unittests/terrainlib/progress_indicator_test.cpp index b994ddb7..826c6e8d 100644 --- a/unittests/terrainlib/progress_indicator_test.cpp +++ b/unittests/terrainlib/progress_indicator_test.cpp @@ -19,7 +19,10 @@ #include "ProgressIndicator.h" #include +#include +#include #include +#include #include #include @@ -100,4 +103,28 @@ TEST_CASE("progress indicator") monitoring_thread.join(); CHECK_THROWS(pi.task_finished()); } + + SECTION("monitoring can be stopped before all tasks finish") + { + ProgressIndicator pi(1); + auto monitoring_thread = pi.start_monitoring(); + + std::condition_variable_any fallback_condition; + std::mutex fallback_mutex; + std::jthread fallback([&](std::stop_token stop_token) { + std::unique_lock lock(fallback_mutex); + fallback_condition.wait_for(lock, stop_token, 2s, []() { return false; }); + if (!stop_token.stop_requested()) { + pi.task_finished(); + } + }); + + const auto before_stop = std::chrono::steady_clock::now(); + monitoring_thread.request_stop(); + monitoring_thread.join(); + const auto stop_duration = std::chrono::steady_clock::now() - before_stop; + fallback.request_stop(); + + CHECK(stop_duration < 250ms); + } } From bec6c7b5e0379e275784672825c21cb5aea8d8ac Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Mon, 27 Jul 2026 02:55:51 +0200 Subject: [PATCH 14/61] Guard tile progress with an RAII session --- src/tile_downloader/TileDownloader.h | 4 +- src/tile_downloader/TileLogger.h | 77 +++++++++++++++++++++++----- unittests/CMakeLists.txt | 1 + unittests/tile_downloader/logger.cpp | 33 ++++++++++++ 4 files changed, 99 insertions(+), 16 deletions(-) create mode 100644 unittests/tile_downloader/logger.cpp diff --git a/src/tile_downloader/TileDownloader.h b/src/tile_downloader/TileDownloader.h index 1af9a9ba..3b863d40 100644 --- a/src/tile_downloader/TileDownloader.h +++ b/src/tile_downloader/TileDownloader.h @@ -25,9 +25,9 @@ class TileDownloader { _max_zoom_level(max_zoom_level) {} void download_recursive(const radix::tile::Id &root_id) { - this->_logger.start(); + auto progress_session = this->_logger.start(); this->download_recursive_core(root_id); - this->_logger.finish(); + progress_session.finish(); } private: diff --git a/src/tile_downloader/TileLogger.h b/src/tile_downloader/TileLogger.h index 3919e74c..af30a4e0 100644 --- a/src/tile_downloader/TileLogger.h +++ b/src/tile_downloader/TileLogger.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -31,13 +32,12 @@ struct TileResult { class TileLogger { public: + class Session; + explicit TileLogger(unsigned root_zoom_level) : _root_zoom_level(root_zoom_level), _progress(PROGRESS_RESOLUTION) {} - // Starts the progress display; pair with a call to finish(). - void start() { - this->_progress_thread = this->_progress.start_monitoring(); - } + [[nodiscard]] Session start(); // The tile turned out not to exist (or errored out), so its branch stops here. void missing(const radix::tile::Id &tile) { @@ -64,9 +64,20 @@ class TileLogger { }, status); } - // Fills in any steps still missing due to floating point rounding, so the - // progress indicator reaches its total, then joins the thread started by start(). - void finish() { +private: + static constexpr size_t PROGRESS_RESOLUTION = 10'000; + + unsigned _root_zoom_level; + ProgressIndicator _progress; + std::jthread _progress_thread; + size_t _steps_done = 0; + std::map _level_counts; + + void start_monitoring() { + this->_progress_thread = this->_progress.start_monitoring(); + } + + void finish_monitoring() { while (this->_steps_done < PROGRESS_RESOLUTION) { this->_progress.task_finished(); this->_steps_done++; @@ -76,14 +87,18 @@ class TileLogger { } } -private: - static constexpr size_t PROGRESS_RESOLUTION = 10'000; + void cancel_monitoring() noexcept { + if (!this->_progress_thread.joinable()) { + return; + } - unsigned _root_zoom_level; - ProgressIndicator _progress; - std::jthread _progress_thread; - size_t _steps_done = 0; - std::map _level_counts; + this->_progress_thread.request_stop(); + try { + this->_progress_thread.join(); + } catch (...) { + // Session cleanup must not replace the active exception. + } + } static std::string format(const radix::tile::Id &tile) { return fmt::format("Tile[Zoom={}, X={}, Y={}]", tile.zoom_level, tile.coords.x, tile.coords.y); @@ -114,3 +129,37 @@ class TileLogger { } } }; + +class TileLogger::Session { +public: + explicit Session(TileLogger &logger) + : _logger(&logger) { + this->_logger->start_monitoring(); + } + + Session(const Session &) = delete; + Session &operator=(const Session &) = delete; + + Session(Session &&other) noexcept + : _logger(std::exchange(other._logger, nullptr)) {} + + Session &operator=(Session &&) = delete; + + ~Session() { + if (this->_logger) { + this->_logger->cancel_monitoring(); + } + } + + void finish() { + this->_logger->finish_monitoring(); + this->_logger = nullptr; + } + +private: + TileLogger *_logger; +}; + +inline TileLogger::Session TileLogger::start() { + return Session(*this); +} diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 41117671..4fe1abd4 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -89,6 +89,7 @@ endif() if(TARGET tile-downloader) add_executable(unittests_tile_downloader + tile_downloader/logger.cpp tile_downloader/url_builder.cpp ) target_include_directories(unittests_tile_downloader PRIVATE ${CMAKE_SOURCE_DIR}/src/tile_downloader) diff --git a/unittests/tile_downloader/logger.cpp b/unittests/tile_downloader/logger.cpp new file mode 100644 index 00000000..c41ecb4c --- /dev/null +++ b/unittests/tile_downloader/logger.cpp @@ -0,0 +1,33 @@ +#include "TileLogger.h" + +#include +#include + +#include + +using namespace std::literals; + +TEST_CASE("tile logger session stops monitoring during exceptional unwinding") +{ + TileLogger logger(0); + + const auto before_throw = std::chrono::steady_clock::now(); + CHECK_THROWS_AS( + [&]() { + auto session = logger.start(); + throw std::runtime_error("download failed"); + }(), + std::runtime_error); + const auto unwind_duration = std::chrono::steady_clock::now() - before_throw; + + CHECK(unwind_duration < 250ms); +} + +TEST_CASE("tile logger session can finish normally") +{ + TileLogger logger(0); + auto session = logger.start(); + + logger.skipped(radix::tile::Id{0, {0, 0}}); + session.finish(); +} From 5171513d5f3fb794607e75a221fcef631169f0b9 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Mon, 27 Jul 2026 02:58:36 +0200 Subject: [PATCH 15/61] Propagate exceptions from curl callbacks --- src/tile_downloader/HttpClient.h | 72 ++++++++++++++++++----- unittests/CMakeLists.txt | 4 +- unittests/tile_downloader/http_client.cpp | 62 +++++++++++++++++++ 3 files changed, 121 insertions(+), 17 deletions(-) create mode 100644 unittests/tile_downloader/http_client.cpp diff --git a/src/tile_downloader/HttpClient.h b/src/tile_downloader/HttpClient.h index 4db7e230..d5ef1ca4 100644 --- a/src/tile_downloader/HttpClient.h +++ b/src/tile_downloader/HttpClient.h @@ -1,8 +1,10 @@ #pragma once +#include #include #include #include +#include #include #include @@ -18,7 +20,10 @@ using ProgressFn = std::function; class HttpClient { public: - HttpClient() : _curl(curl_easy_init()) { + using WriteFn = std::function &, const char *, size_t)>; + + explicit HttpClient(WriteFn write = {}) + : _curl(curl_easy_init()), _write(std::move(write)) { if (!_curl) { throw std::runtime_error("failed to init cURL"); } @@ -35,10 +40,12 @@ class HttpClient { HttpResponse get(const std::string &url, const ProgressFn &on_progress = {}) const { HttpResponse response; + WriteContext write_context{&response.body, &this->_write, {}}; + ProgressContext progress_context{&on_progress, {}}; curl_easy_setopt(this->_curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(this->_curl, CURLOPT_WRITEFUNCTION, write_cb); - curl_easy_setopt(this->_curl, CURLOPT_WRITEDATA, &response.body); + curl_easy_setopt(this->_curl, CURLOPT_WRITEDATA, &write_context); curl_easy_setopt(this->_curl, CURLOPT_TIMEOUT, 5L); curl_easy_setopt(this->_curl, CURLOPT_CONNECTTIMEOUT, 5L); curl_easy_setopt(this->_curl, CURLOPT_FAILONERROR, 1L); @@ -46,12 +53,18 @@ class HttpClient { if (on_progress) { curl_easy_setopt(this->_curl, CURLOPT_NOPROGRESS, 0L); curl_easy_setopt(this->_curl, CURLOPT_XFERINFOFUNCTION, progress_cb); - curl_easy_setopt(this->_curl, CURLOPT_XFERINFODATA, &on_progress); + curl_easy_setopt(this->_curl, CURLOPT_XFERINFODATA, &progress_context); } else { curl_easy_setopt(this->_curl, CURLOPT_NOPROGRESS, 1L); } response.curl_code = curl_easy_perform(this->_curl); + if (write_context.exception) { + std::rethrow_exception(write_context.exception); + } + if (progress_context.exception) { + std::rethrow_exception(progress_context.exception); + } char *ct = nullptr; if (curl_easy_getinfo(this->_curl, CURLINFO_CONTENT_TYPE, &ct) == CURLE_OK && ct) { @@ -68,23 +81,50 @@ class HttpClient { } private: - CURL *_curl; + struct WriteContext { + std::vector *buffer; + const WriteFn *write; + std::exception_ptr exception; + }; - static size_t write_cb(void *ptr, size_t size, size_t nmemb, void *userdata) { - auto &buf = *static_cast *>(userdata); - size_t total = size * nmemb; - buf.insert(buf.end(), static_cast(ptr), static_cast(ptr) + total); - return total; + struct ProgressContext { + const ProgressFn *progress; + std::exception_ptr exception; + }; + + CURL *_curl; + WriteFn _write; + + static size_t write_cb(void *ptr, size_t size, size_t nmemb, void *userdata) noexcept { + auto &context = *static_cast(userdata); + const size_t total = size * nmemb; + try { + const auto *data = static_cast(ptr); + if (*context.write) { + (*context.write)(*context.buffer, data, total); + } else { + context.buffer->insert(context.buffer->end(), data, data + total); + } + return total; + } catch (...) { + context.exception = std::current_exception(); + return 0; + } } static int progress_cb(void *clientp, curl_off_t dltotal, curl_off_t dlnow, - curl_off_t /*ultotal*/, curl_off_t /*ulnow*/) { - const auto &fn = *static_cast(clientp); - if (dltotal > 0) { - fn(double(dlnow) / double(dltotal)); - } else { - fn(-1.0); + curl_off_t /*ultotal*/, curl_off_t /*ulnow*/) noexcept { + auto &context = *static_cast(clientp); + try { + if (dltotal > 0) { + (*context.progress)(double(dlnow) / double(dltotal)); + } else { + (*context.progress)(-1.0); + } + return 0; + } catch (...) { + context.exception = std::current_exception(); + return 1; } - return 0; } }; diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 4fe1abd4..c24810b2 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -88,12 +88,14 @@ if(TARGET sfbuilderlib) endif() if(TARGET tile-downloader) + find_package(CURL REQUIRED) add_executable(unittests_tile_downloader + tile_downloader/http_client.cpp tile_downloader/logger.cpp tile_downloader/url_builder.cpp ) target_include_directories(unittests_tile_downloader PRIVATE ${CMAKE_SOURCE_DIR}/src/tile_downloader) - target_link_libraries(unittests_tile_downloader PRIVATE terrainlib Catch2::Catch2WithMain) + target_link_libraries(unittests_tile_downloader PRIVATE terrainlib Catch2::Catch2WithMain CURL::libcurl) atb_configure_test(unittests_tile_downloader) endif() diff --git a/unittests/tile_downloader/http_client.cpp b/unittests/tile_downloader/http_client.cpp new file mode 100644 index 00000000..61311a8c --- /dev/null +++ b/unittests/tile_downloader/http_client.cpp @@ -0,0 +1,62 @@ +#include "HttpClient.h" + +#include +#include +#include +#include + +#include + +namespace { + +class TemporaryFile { +public: + TemporaryFile() + : _path(std::filesystem::temp_directory_path() / "atb-http-client-test.txt") { + std::ofstream output(_path, std::ios::binary); + output << "response body"; + REQUIRE(output); + } + + ~TemporaryFile() { + std::error_code error; + std::filesystem::remove(_path, error); + } + + [[nodiscard]] std::string url() const { + return "file://" + _path.string(); + } + +private: + std::filesystem::path _path; +}; + +class CallbackError : public std::runtime_error { +public: + CallbackError() + : std::runtime_error("callback failed") {} +}; + +} + +TEST_CASE("http client propagates response writer exceptions") +{ + const TemporaryFile source; + HttpClient client([](std::vector &, const char *, size_t) { + throw CallbackError(); + }); + + CHECK_THROWS_AS(client.get(source.url()), CallbackError); +} + +TEST_CASE("http client propagates progress callback exceptions") +{ + const TemporaryFile source; + HttpClient client; + + CHECK_THROWS_AS( + client.get(source.url(), [](double) { + throw CallbackError(); + }), + CallbackError); +} From 817c7d74bca002e16d32493d55a2f416b09ce310 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Mon, 27 Jul 2026 03:00:09 +0200 Subject: [PATCH 16/61] Detect tile output write failures --- src/tile_downloader/TileDownloader.h | 12 +----- src/tile_downloader/write_file.h | 26 ++++++++++++ unittests/CMakeLists.txt | 1 + unittests/tile_downloader/write_file.cpp | 53 ++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 10 deletions(-) create mode 100644 src/tile_downloader/write_file.h create mode 100644 unittests/tile_downloader/write_file.cpp diff --git a/src/tile_downloader/TileDownloader.h b/src/tile_downloader/TileDownloader.h index 3b863d40..32372ec7 100644 --- a/src/tile_downloader/TileDownloader.h +++ b/src/tile_downloader/TileDownloader.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include #include @@ -13,6 +12,7 @@ #include "TileLogger.h" #include "TileUrlBuilder.h" #include "tile_path.h" +#include "write_file.h" class TileDownloader { public: @@ -89,14 +89,6 @@ class TileDownloader { } } - static void write_file(const std::filesystem::path &path, const std::vector &data) { - std::ofstream out(path, std::ios::binary); - if (!out) { - throw std::runtime_error(fmt::format("failed to open \"{}\" for writing", path.string())); - } - out.write(data.data(), data.size()); - } - TileResult::Status download_tile(const radix::tile::Id &tile) { const auto path = std::filesystem::absolute(this->tile_path(tile)); @@ -112,7 +104,7 @@ class TileDownloader { HttpResponse response = this->_http.get(url); if (response.curl_code == CURLE_OK && this->_http.is_image(response)) { - write_file(path, response.body); + write_file_checked(path, response.body); return TileResult::Downloaded{}; } diff --git a/src/tile_downloader/write_file.h b/src/tile_downloader/write_file.h new file mode 100644 index 00000000..61883228 --- /dev/null +++ b/src/tile_downloader/write_file.h @@ -0,0 +1,26 @@ +#pragma once + +#include +#include +#include +#include + +#include + +inline void write_file_checked(const std::filesystem::path &path, const std::vector &data) +{ + std::ofstream output(path, std::ios::binary); + if (!output) { + throw std::runtime_error(fmt::format("failed to open \"{}\" for writing", path.string())); + } + + output.write(data.data(), static_cast(data.size())); + if (!output) { + throw std::runtime_error(fmt::format("failed to write \"{}\"", path.string())); + } + + output.close(); + if (!output) { + throw std::runtime_error(fmt::format("failed to finish writing \"{}\"", path.string())); + } +} diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index c24810b2..569d95be 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -93,6 +93,7 @@ if(TARGET tile-downloader) tile_downloader/http_client.cpp tile_downloader/logger.cpp tile_downloader/url_builder.cpp + tile_downloader/write_file.cpp ) target_include_directories(unittests_tile_downloader PRIVATE ${CMAKE_SOURCE_DIR}/src/tile_downloader) target_link_libraries(unittests_tile_downloader PRIVATE terrainlib Catch2::Catch2WithMain CURL::libcurl) diff --git a/unittests/tile_downloader/write_file.cpp b/unittests/tile_downloader/write_file.cpp new file mode 100644 index 00000000..57efbde3 --- /dev/null +++ b/unittests/tile_downloader/write_file.cpp @@ -0,0 +1,53 @@ +#include "write_file.h" + +#include +#include +#include +#include +#include + +#include + +namespace { + +class TemporaryOutput { +public: + TemporaryOutput() + : _path(std::filesystem::temp_directory_path() / "atb-write-file-test.bin") {} + + ~TemporaryOutput() { + std::error_code error; + std::filesystem::remove(_path, error); + } + + [[nodiscard]] const std::filesystem::path &path() const { + return _path; + } + +private: + std::filesystem::path _path; +}; + +} + +TEST_CASE("checked file writer persists the complete response") +{ + const TemporaryOutput output; + const std::vector expected{'t', 'i', 'l', 'e'}; + + write_file_checked(output.path(), expected); + + std::ifstream input(output.path(), std::ios::binary); + const auto begin = std::istreambuf_iterator(input); + const std::vector actual(begin, std::istreambuf_iterator{}); + CHECK(actual == expected); +} + +#if defined(__linux__) +TEST_CASE("checked file writer reports persistence failures") +{ + const std::vector data(64 * 1024, 'x'); + + CHECK_THROWS_AS(write_file_checked("/dev/full", data), std::runtime_error); +} +#endif From fcade5498904ea1d069020c0772a6977d31d2b78 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Mon, 27 Jul 2026 08:36:45 +0200 Subject: [PATCH 17/61] Publish tile files atomically --- src/tile_downloader/write_file.h | 21 +++++++++++++- unittests/tile_downloader/write_file.cpp | 36 +++++++++++++++++++++--- 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/src/tile_downloader/write_file.h b/src/tile_downloader/write_file.h index 61883228..7d7d2d20 100644 --- a/src/tile_downloader/write_file.h +++ b/src/tile_downloader/write_file.h @@ -7,7 +7,9 @@ #include -inline void write_file_checked(const std::filesystem::path &path, const std::vector &data) +namespace tile_downloader_detail { + +inline void write_file_checked_direct(const std::filesystem::path &path, const std::vector &data) { std::ofstream output(path, std::ios::binary); if (!output) { @@ -24,3 +26,20 @@ inline void write_file_checked(const std::filesystem::path &path, const std::vec throw std::runtime_error(fmt::format("failed to finish writing \"{}\"", path.string())); } } + +} + +inline void write_file_checked(const std::filesystem::path &path, const std::vector &data) +{ + auto staging_path = path; + staging_path += ".part"; + + try { + tile_downloader_detail::write_file_checked_direct(staging_path, data); + std::filesystem::rename(staging_path, path); + } catch (...) { + std::error_code cleanup_error; + std::filesystem::remove(staging_path, cleanup_error); + throw; + } +} diff --git a/unittests/tile_downloader/write_file.cpp b/unittests/tile_downloader/write_file.cpp index 57efbde3..0463a389 100644 --- a/unittests/tile_downloader/write_file.cpp +++ b/unittests/tile_downloader/write_file.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -12,18 +13,29 @@ namespace { class TemporaryOutput { public: - TemporaryOutput() - : _path(std::filesystem::temp_directory_path() / "atb-write-file-test.bin") {} + explicit TemporaryOutput(std::string_view name) + : _path(std::filesystem::temp_directory_path() / name) { + std::error_code error; + std::filesystem::remove_all(_path, error); + std::filesystem::remove(staging_path(), error); + } ~TemporaryOutput() { std::error_code error; - std::filesystem::remove(_path, error); + std::filesystem::remove_all(_path, error); + std::filesystem::remove(staging_path(), error); } [[nodiscard]] const std::filesystem::path &path() const { return _path; } + [[nodiscard]] std::filesystem::path staging_path() const { + auto staging = _path; + staging += ".part"; + return staging; + } + private: std::filesystem::path _path; }; @@ -32,7 +44,7 @@ class TemporaryOutput { TEST_CASE("checked file writer persists the complete response") { - const TemporaryOutput output; + const TemporaryOutput output("atb-write-file-success.bin"); const std::vector expected{'t', 'i', 'l', 'e'}; write_file_checked(output.path(), expected); @@ -41,6 +53,20 @@ TEST_CASE("checked file writer persists the complete response") const auto begin = std::istreambuf_iterator(input); const std::vector actual(begin, std::istreambuf_iterator{}); CHECK(actual == expected); + CHECK_FALSE(std::filesystem::exists(output.staging_path())); +} + +TEST_CASE("checked file writer preserves the final path when promotion fails") +{ + const TemporaryOutput output("atb-write-file-promotion-failure"); + REQUIRE(std::filesystem::create_directory(output.path())); + + CHECK_THROWS_AS( + write_file_checked(output.path(), std::vector{'t', 'i', 'l', 'e'}), + std::filesystem::filesystem_error); + + CHECK(std::filesystem::is_directory(output.path())); + CHECK_FALSE(std::filesystem::exists(output.staging_path())); } #if defined(__linux__) @@ -49,5 +75,7 @@ TEST_CASE("checked file writer reports persistence failures") const std::vector data(64 * 1024, 'x'); CHECK_THROWS_AS(write_file_checked("/dev/full", data), std::runtime_error); + CHECK(std::filesystem::is_character_file("/dev/full")); + CHECK_FALSE(std::filesystem::exists("/dev/full.part")); } #endif From b7e7c0e6258a561a9850e1f4968ab9af705d35d2 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Mon, 27 Jul 2026 09:25:37 +0200 Subject: [PATCH 18/61] Finalize tile parents after descendants --- src/tile_downloader/TileDownloader.h | 77 ++++++++++++----- src/tile_downloader/TileLogger.h | 3 +- src/tile_downloader/cli.cpp | 4 - src/tile_downloader/cli.h | 1 - src/tile_downloader/main.cpp | 6 +- src/tile_downloader/write_file.h | 35 ++++++-- unittests/CMakeLists.txt | 1 + unittests/tile_downloader/downloader.cpp | 105 +++++++++++++++++++++++ unittests/tile_downloader/write_file.cpp | 39 ++++++--- 9 files changed, 221 insertions(+), 50 deletions(-) create mode 100644 unittests/tile_downloader/downloader.cpp diff --git a/src/tile_downloader/TileDownloader.h b/src/tile_downloader/TileDownloader.h index 32372ec7..7cc0252a 100644 --- a/src/tile_downloader/TileDownloader.h +++ b/src/tile_downloader/TileDownloader.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -17,17 +18,17 @@ class TileDownloader { public: TileDownloader(const TileUrlBuilder &url_builder, std::filesystem::path output_directory, - bool early_skip, std::optional max_zoom_level, unsigned root_zoom_level) + std::optional max_zoom_level, unsigned root_zoom_level) : _url_builder(url_builder), _output_directory(std::move(output_directory)), _logger(root_zoom_level), - _early_skip(early_skip), _max_zoom_level(max_zoom_level) {} - void download_recursive(const radix::tile::Id &root_id) { + [[nodiscard]] bool download_recursive(const radix::tile::Id &root_id) { auto progress_session = this->_logger.start(); - this->download_recursive_core(root_id); + const bool complete = this->download_recursive_core(root_id); progress_session.finish(); + return complete; } private: @@ -35,52 +36,81 @@ class TileDownloader { std::filesystem::path _output_directory; HttpClient _http; TileLogger _logger; - bool _early_skip; std::optional _max_zoom_level; - void download_recursive_core(const radix::tile::Id &root_id) { + [[nodiscard]] bool download_recursive_core(const radix::tile::Id &root_id) { auto result = this->download_tile(root_id); this->_logger.report_error(root_id, result); + if (std::holds_alternative(result)) { + this->_logger.skipped(root_id); + return true; + } + + if (std::holds_alternative(result)) { + this->_logger.missing(root_id); + return true; + } + if (is_failure(result)) { this->_logger.missing(root_id); - return; + return false; } const auto children = root_id.children(); + bool children_complete = true; for (size_t i = 0; i < children.size(); i++) { - if (this->_early_skip && i + 1 < children.size()) { - if (this->tile_exists(children[i + 1])) { - this->_logger.skipped(children[i]); - continue; - } - } - if (this->_max_zoom_level.has_value() && children[i].zoom_level > *this->_max_zoom_level) { this->_logger.skipped(children[i]); continue; } - this->download_recursive_core(children[i]); + if (!this->download_recursive_core(children[i])) { + children_complete = false; + } + } + + if (!children_complete) { + return false; } + + mark_tile_children_complete(this->tile_path(root_id), this->completion_time(children)); + return true; } static bool is_failure(const TileResult::Status &result) { - return std::holds_alternative(result) - || std::holds_alternative(result) + return std::holds_alternative(result) || std::holds_alternative(result) || std::holds_alternative(result) || std::holds_alternative(result); } - bool tile_exists(const radix::tile::Id &tile) const { - return std::filesystem::exists(this->tile_path(tile)); - } - std::filesystem::path tile_path(const radix::tile::Id &tile) const { return google_tile_path(_output_directory, tile, ".jpeg"); } + std::filesystem::file_time_type completion_time( + const std::array &children) const + { + auto completion_time = std::filesystem::file_time_type::clock::now(); + for (const auto &child : children) { + if (this->_max_zoom_level.has_value() && child.zoom_level > *this->_max_zoom_level) { + continue; + } + + const auto child_path = this->tile_path(child); + if (!std::filesystem::exists(child_path)) { + continue; + } + + const auto child_time = std::filesystem::last_write_time(child_path); + if (child_time >= completion_time) { + completion_time = child_time + std::filesystem::file_time_type::duration{1}; + } + } + return completion_time; + } + static void ensure_parent_dirs(const std::filesystem::path &path) { std::error_code ec; std::filesystem::create_directories(path.parent_path(), ec); @@ -95,6 +125,9 @@ class TileDownloader { if (std::filesystem::exists(path)) { return TileResult::Skipped{}; } + if (std::filesystem::exists(children_pending_tile_path(path))) { + return TileResult::ChildrenPending{}; + } ensure_parent_dirs(path); @@ -104,7 +137,7 @@ class TileDownloader { HttpResponse response = this->_http.get(url); if (response.curl_code == CURLE_OK && this->_http.is_image(response)) { - write_file_checked(path, response.body); + write_file_children_pending(path, response.body); return TileResult::Downloaded{}; } diff --git a/src/tile_downloader/TileLogger.h b/src/tile_downloader/TileLogger.h index af30a4e0..c9d6c81b 100644 --- a/src/tile_downloader/TileLogger.h +++ b/src/tile_downloader/TileLogger.h @@ -20,6 +20,7 @@ struct TileResult { struct Downloaded {}; + struct ChildrenPending {}; struct Skipped {}; struct Absent {}; struct HttpError { long status_code; }; @@ -27,7 +28,7 @@ struct TileResult { struct CurlError { CURLcode code; }; struct TimedOut {}; - using Status = std::variant; + using Status = std::variant; }; class TileLogger { diff --git a/src/tile_downloader/cli.cpp b/src/tile_downloader/cli.cpp index 567d197b..bd1ca8af 100644 --- a/src/tile_downloader/cli.cpp +++ b/src/tile_downloader/cli.cpp @@ -54,10 +54,6 @@ Args parse(int argc, const char *const *argv) { ->transform(CLI::CheckedTransformer(log_level_names, CLI::ignore_case)) ->default_val(spdlog::level::info); - args.early_skip = true; - app.add_option("--early-skip", args.early_skip, "Resume optimization: skip completed subtrees") - ->default_val(true); - app.add_option("--max-zoom-level", args.max_zoom_level, "Maximum zoom level to descend to"); try { diff --git a/src/tile_downloader/cli.h b/src/tile_downloader/cli.h index 80fa1d7c..8e2380f8 100644 --- a/src/tile_downloader/cli.h +++ b/src/tile_downloader/cli.h @@ -20,7 +20,6 @@ struct Args { unsigned int srs; std::filesystem::path output; spdlog::level::level_enum log_level; - bool early_skip; std::optional max_zoom_level; }; diff --git a/src/tile_downloader/main.cpp b/src/tile_downloader/main.cpp index eb0fa8d1..17c25314 100644 --- a/src/tile_downloader/main.cpp +++ b/src/tile_downloader/main.cpp @@ -18,8 +18,6 @@ int main(int argc, char *argv[]) { const radix::tile::Id root_id = {args.zoom, {args.x, args.y}}; - TileDownloader downloader(url_builder, args.output, args.early_skip, args.max_zoom_level, root_id.zoom_level); - downloader.download_recursive(root_id); - - return 0; + TileDownloader downloader(url_builder, args.output, args.max_zoom_level, root_id.zoom_level); + return downloader.download_recursive(root_id) ? 0 : 1; } diff --git a/src/tile_downloader/write_file.h b/src/tile_downloader/write_file.h index 7d7d2d20..0a0cffa5 100644 --- a/src/tile_downloader/write_file.h +++ b/src/tile_downloader/write_file.h @@ -29,17 +29,40 @@ inline void write_file_checked_direct(const std::filesystem::path &path, const s } -inline void write_file_checked(const std::filesystem::path &path, const std::vector &data) +[[nodiscard]] inline std::filesystem::path partial_tile_path(const std::filesystem::path &path) { - auto staging_path = path; - staging_path += ".part"; + auto partial_path = path; + partial_path += ".part"; + return partial_path; +} + +[[nodiscard]] inline std::filesystem::path children_pending_tile_path(const std::filesystem::path &path) +{ + auto pending_path = path; + pending_path += ".children-pending"; + return pending_path; +} + +inline void write_file_children_pending(const std::filesystem::path &path, const std::vector &data) +{ + const auto partial_path = partial_tile_path(path); + const auto pending_path = children_pending_tile_path(path); try { - tile_downloader_detail::write_file_checked_direct(staging_path, data); - std::filesystem::rename(staging_path, path); + tile_downloader_detail::write_file_checked_direct(partial_path, data); + std::filesystem::rename(partial_path, pending_path); } catch (...) { std::error_code cleanup_error; - std::filesystem::remove(staging_path, cleanup_error); + std::filesystem::remove(partial_path, cleanup_error); throw; } } + +inline void mark_tile_children_complete( + const std::filesystem::path &path, + std::filesystem::file_time_type completion_time) +{ + const auto pending_path = children_pending_tile_path(path); + std::filesystem::last_write_time(pending_path, completion_time); + std::filesystem::rename(pending_path, path); +} diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 569d95be..27d24f95 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -90,6 +90,7 @@ endif() if(TARGET tile-downloader) find_package(CURL REQUIRED) add_executable(unittests_tile_downloader + tile_downloader/downloader.cpp tile_downloader/http_client.cpp tile_downloader/logger.cpp tile_downloader/url_builder.cpp diff --git a/unittests/tile_downloader/downloader.cpp b/unittests/tile_downloader/downloader.cpp new file mode 100644 index 00000000..2d599d47 --- /dev/null +++ b/unittests/tile_downloader/downloader.cpp @@ -0,0 +1,105 @@ +#include "TileDownloader.h" + +#include +#include +#include + +#include + +namespace { + +class TemporaryPyramid { +public: + explicit TemporaryPyramid(std::string_view name) + : _path(std::filesystem::temp_directory_path() / name) { + std::error_code error; + std::filesystem::remove_all(_path, error); + std::filesystem::create_directories(_path); + } + + ~TemporaryPyramid() { + std::error_code error; + std::filesystem::remove_all(_path, error); + } + + [[nodiscard]] const std::filesystem::path &path() const { + return _path; + } + + [[nodiscard]] std::filesystem::path tile_path(const radix::tile::Id &tile) const { + return google_tile_path(_path, tile, ".jpeg"); + } + + void create_pending(const radix::tile::Id &tile) const { + const auto path = tile_path(tile); + std::filesystem::create_directories(path.parent_path()); + write_file_children_pending(path, std::vector{'t', 'i', 'l', 'e'}); + } + + void create_complete(const radix::tile::Id &tile) const { + create_pending(tile); + mark_tile_children_complete( + tile_path(tile), std::filesystem::file_time_type::clock::now()); + } + +private: + std::filesystem::path _path; +}; + +const TileUrlBuilder missing_file_url({ + "file:///definitely-missing-atb-tile/{zoom}/{x}/{y}.jpeg", + TileYDirection::Down +}); + +} + +TEST_CASE("tile downloader promotes parents after completed children") +{ + const TemporaryPyramid pyramid("atb-downloader-complete-pyramid"); + const radix::tile::Id root{0, {0, 0}}; + const auto children = root.children(); + + pyramid.create_pending(root); + for (const auto &child : children) { + pyramid.create_pending(child); + } + + TileDownloader downloader(missing_file_url, pyramid.path(), 1u, root.zoom_level); + REQUIRE(downloader.download_recursive(root)); + + REQUIRE(std::filesystem::exists(pyramid.tile_path(root))); + CHECK_FALSE(std::filesystem::exists(children_pending_tile_path(pyramid.tile_path(root)))); + const auto root_time = std::filesystem::last_write_time(pyramid.tile_path(root)); + + for (const auto &child : children) { + CHECK(std::filesystem::exists(pyramid.tile_path(child))); + CHECK_FALSE(std::filesystem::exists(children_pending_tile_path(pyramid.tile_path(child)))); + CHECK(std::filesystem::last_write_time(pyramid.tile_path(child)) < root_time); + } + + REQUIRE(std::filesystem::remove(pyramid.tile_path(children.front()))); + TileDownloader resumed_downloader(missing_file_url, pyramid.path(), 1u, root.zoom_level); + CHECK(resumed_downloader.download_recursive(root)); + CHECK_FALSE(std::filesystem::exists(pyramid.tile_path(children.front()))); +} + +TEST_CASE("tile downloader leaves ancestors pending after a child failure") +{ + const TemporaryPyramid pyramid("atb-downloader-failed-pyramid"); + const radix::tile::Id root{0, {0, 0}}; + const auto children = root.children(); + + pyramid.create_pending(root); + for (size_t i = 1; i < children.size(); ++i) { + pyramid.create_complete(children[i]); + } + + TileDownloader downloader(missing_file_url, pyramid.path(), 1u, root.zoom_level); + CHECK_FALSE(downloader.download_recursive(root)); + + CHECK_FALSE(std::filesystem::exists(pyramid.tile_path(root))); + CHECK(std::filesystem::exists(children_pending_tile_path(pyramid.tile_path(root)))); + for (size_t i = 1; i < children.size(); ++i) { + CHECK(std::filesystem::exists(pyramid.tile_path(children[i]))); + } +} diff --git a/unittests/tile_downloader/write_file.cpp b/unittests/tile_downloader/write_file.cpp index 0463a389..7918b22d 100644 --- a/unittests/tile_downloader/write_file.cpp +++ b/unittests/tile_downloader/write_file.cpp @@ -17,23 +17,27 @@ class TemporaryOutput { : _path(std::filesystem::temp_directory_path() / name) { std::error_code error; std::filesystem::remove_all(_path, error); - std::filesystem::remove(staging_path(), error); + std::filesystem::remove(partial_path(), error); + std::filesystem::remove(pending_path(), error); } ~TemporaryOutput() { std::error_code error; std::filesystem::remove_all(_path, error); - std::filesystem::remove(staging_path(), error); + std::filesystem::remove(partial_path(), error); + std::filesystem::remove(pending_path(), error); } [[nodiscard]] const std::filesystem::path &path() const { return _path; } - [[nodiscard]] std::filesystem::path staging_path() const { - auto staging = _path; - staging += ".part"; - return staging; + [[nodiscard]] std::filesystem::path partial_path() const { + return partial_tile_path(_path); + } + + [[nodiscard]] std::filesystem::path pending_path() const { + return children_pending_tile_path(_path); } private: @@ -47,26 +51,36 @@ TEST_CASE("checked file writer persists the complete response") const TemporaryOutput output("atb-write-file-success.bin"); const std::vector expected{'t', 'i', 'l', 'e'}; - write_file_checked(output.path(), expected); + write_file_children_pending(output.path(), expected); - std::ifstream input(output.path(), std::ios::binary); + CHECK_FALSE(std::filesystem::exists(output.path())); + CHECK_FALSE(std::filesystem::exists(output.partial_path())); + REQUIRE(std::filesystem::exists(output.pending_path())); + + std::ifstream input(output.pending_path(), std::ios::binary); const auto begin = std::istreambuf_iterator(input); const std::vector actual(begin, std::istreambuf_iterator{}); CHECK(actual == expected); - CHECK_FALSE(std::filesystem::exists(output.staging_path())); + + mark_tile_children_complete( + output.path(), std::filesystem::file_time_type::clock::now()); + CHECK(std::filesystem::exists(output.path())); + CHECK_FALSE(std::filesystem::exists(output.pending_path())); } TEST_CASE("checked file writer preserves the final path when promotion fails") { const TemporaryOutput output("atb-write-file-promotion-failure"); REQUIRE(std::filesystem::create_directory(output.path())); + write_file_children_pending(output.path(), std::vector{'t', 'i', 'l', 'e'}); CHECK_THROWS_AS( - write_file_checked(output.path(), std::vector{'t', 'i', 'l', 'e'}), + mark_tile_children_complete( + output.path(), std::filesystem::file_time_type::clock::now()), std::filesystem::filesystem_error); CHECK(std::filesystem::is_directory(output.path())); - CHECK_FALSE(std::filesystem::exists(output.staging_path())); + CHECK(std::filesystem::exists(output.pending_path())); } #if defined(__linux__) @@ -74,8 +88,9 @@ TEST_CASE("checked file writer reports persistence failures") { const std::vector data(64 * 1024, 'x'); - CHECK_THROWS_AS(write_file_checked("/dev/full", data), std::runtime_error); + CHECK_THROWS_AS(write_file_children_pending("/dev/full", data), std::runtime_error); CHECK(std::filesystem::is_character_file("/dev/full")); CHECK_FALSE(std::filesystem::exists("/dev/full.part")); + CHECK_FALSE(std::filesystem::exists("/dev/full.children-pending")); } #endif From f0c296161e95a332b2d5c86890fc9b0f2af8feed Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Mon, 27 Jul 2026 09:41:00 +0200 Subject: [PATCH 19/61] Remove tile completion timestamp rewriting --- src/tile_downloader/TileDownloader.h | 25 +----------------------- src/tile_downloader/write_file.h | 5 +---- unittests/tile_downloader/downloader.cpp | 5 +---- unittests/tile_downloader/write_file.cpp | 6 ++---- 4 files changed, 5 insertions(+), 36 deletions(-) diff --git a/src/tile_downloader/TileDownloader.h b/src/tile_downloader/TileDownloader.h index 7cc0252a..e1f49bf4 100644 --- a/src/tile_downloader/TileDownloader.h +++ b/src/tile_downloader/TileDownloader.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include #include @@ -74,7 +73,7 @@ class TileDownloader { return false; } - mark_tile_children_complete(this->tile_path(root_id), this->completion_time(children)); + mark_tile_children_complete(this->tile_path(root_id)); return true; } @@ -89,28 +88,6 @@ class TileDownloader { return google_tile_path(_output_directory, tile, ".jpeg"); } - std::filesystem::file_time_type completion_time( - const std::array &children) const - { - auto completion_time = std::filesystem::file_time_type::clock::now(); - for (const auto &child : children) { - if (this->_max_zoom_level.has_value() && child.zoom_level > *this->_max_zoom_level) { - continue; - } - - const auto child_path = this->tile_path(child); - if (!std::filesystem::exists(child_path)) { - continue; - } - - const auto child_time = std::filesystem::last_write_time(child_path); - if (child_time >= completion_time) { - completion_time = child_time + std::filesystem::file_time_type::duration{1}; - } - } - return completion_time; - } - static void ensure_parent_dirs(const std::filesystem::path &path) { std::error_code ec; std::filesystem::create_directories(path.parent_path(), ec); diff --git a/src/tile_downloader/write_file.h b/src/tile_downloader/write_file.h index 0a0cffa5..ab2d07f2 100644 --- a/src/tile_downloader/write_file.h +++ b/src/tile_downloader/write_file.h @@ -58,11 +58,8 @@ inline void write_file_children_pending(const std::filesystem::path &path, const } } -inline void mark_tile_children_complete( - const std::filesystem::path &path, - std::filesystem::file_time_type completion_time) +inline void mark_tile_children_complete(const std::filesystem::path &path) { const auto pending_path = children_pending_tile_path(path); - std::filesystem::last_write_time(pending_path, completion_time); std::filesystem::rename(pending_path, path); } diff --git a/unittests/tile_downloader/downloader.cpp b/unittests/tile_downloader/downloader.cpp index 2d599d47..083ecd49 100644 --- a/unittests/tile_downloader/downloader.cpp +++ b/unittests/tile_downloader/downloader.cpp @@ -38,8 +38,7 @@ class TemporaryPyramid { void create_complete(const radix::tile::Id &tile) const { create_pending(tile); - mark_tile_children_complete( - tile_path(tile), std::filesystem::file_time_type::clock::now()); + mark_tile_children_complete(tile_path(tile)); } private: @@ -69,12 +68,10 @@ TEST_CASE("tile downloader promotes parents after completed children") REQUIRE(std::filesystem::exists(pyramid.tile_path(root))); CHECK_FALSE(std::filesystem::exists(children_pending_tile_path(pyramid.tile_path(root)))); - const auto root_time = std::filesystem::last_write_time(pyramid.tile_path(root)); for (const auto &child : children) { CHECK(std::filesystem::exists(pyramid.tile_path(child))); CHECK_FALSE(std::filesystem::exists(children_pending_tile_path(pyramid.tile_path(child)))); - CHECK(std::filesystem::last_write_time(pyramid.tile_path(child)) < root_time); } REQUIRE(std::filesystem::remove(pyramid.tile_path(children.front()))); diff --git a/unittests/tile_downloader/write_file.cpp b/unittests/tile_downloader/write_file.cpp index 7918b22d..2773c4ed 100644 --- a/unittests/tile_downloader/write_file.cpp +++ b/unittests/tile_downloader/write_file.cpp @@ -62,8 +62,7 @@ TEST_CASE("checked file writer persists the complete response") const std::vector actual(begin, std::istreambuf_iterator{}); CHECK(actual == expected); - mark_tile_children_complete( - output.path(), std::filesystem::file_time_type::clock::now()); + mark_tile_children_complete(output.path()); CHECK(std::filesystem::exists(output.path())); CHECK_FALSE(std::filesystem::exists(output.pending_path())); } @@ -75,8 +74,7 @@ TEST_CASE("checked file writer preserves the final path when promotion fails") write_file_children_pending(output.path(), std::vector{'t', 'i', 'l', 'e'}); CHECK_THROWS_AS( - mark_tile_children_complete( - output.path(), std::filesystem::file_time_type::clock::now()), + mark_tile_children_complete(output.path()), std::filesystem::filesystem_error); CHECK(std::filesystem::is_directory(output.path())); From 6c5da1341841556449c0a5695f2bbde1881cad2e Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Mon, 27 Jul 2026 16:53:50 +0200 Subject: [PATCH 20/61] work on refactor plan --- docs/raster-store/refactor-plan.md | 391 +++++++++++++++++++++------- docs/raster-store/storage-format.md | 30 +-- docs/raster-store/terminology.md | 3 - 3 files changed, 311 insertions(+), 113 deletions(-) diff --git a/docs/raster-store/refactor-plan.md b/docs/raster-store/refactor-plan.md index 026ef6e1..f6f9bda9 100644 --- a/docs/raster-store/refactor-plan.md +++ b/docs/raster-store/refactor-plan.md @@ -3,6 +3,10 @@ Status: proposal for review. This document is an implementation plan, not a record of completed work. +## Purpose + +This plan describes how to extract the existing octree-specific index, traversal, storage, codec, and subtree-reuse code into a shared 2D/3D store. The refactor must preserve existing 3D datasets while enabling raster-fundamentalis storage without duplicating infrastructure. + ## Decisions already made - The shared implementation will live in `src/terrainlib/store` and use the @@ -13,13 +17,21 @@ record of completed work. writable without changing their on-disk contract. - 3D compatibility includes both existing path layouts: `flat` and `level_and_coordinate_directories`. -- The 2D raster-fundamentalis format is a separate format. Requirements in +- The 2D raster tile format is a separate format. Requirements in this directory apply to that 2D format and must not be retrofitted onto - existing 3D datasets. + existing 3D datasets. The storage format for 2d tiles is defined in storage-format.md - A path layout strategy should contain a stable identifier and two - operations: key to relative path, and relative path to key. It should not - require an inheritance hierarchy, RTTI, global self-registration, or heap - allocation. + operations: key to extensionless `NodePath`, and `NodePath` to key. It + should not require an inheritance hierarchy, RTTI, global + self-registration, or heap allocation. +- A configured codec owns all filename endings and maps one `NodePath` to one + or more physical files. `Codec::paths()` must not need a payload. +- Codecs are stateful runtime objects behind a small interface. They may + support reading, writing, or both; an unsupported operation throws. reading and writing must be reentrant (callable concurrently from different threads). +- `copy_from()` hard-links every file when the input and output codecs return + the same path list for a common dummy `NodePath`. Otherwise it decodes with + the input codec and encodes with the output codec. Callers can force + re-encoding. - 3D geometry, ECEF bounds, mesh codecs, mesh reconstruction, mask geometry, and raster-specific processing remain outside the shared store. @@ -27,16 +39,18 @@ record of completed work. 1. Use one sparse hierarchy implementation for `octree::Id` and `radix::tile::Id`. -2. Use one traversal, storage, cache, codec boundary, and unchanged-subtree - copier for 2D and 3D. +2. Use one traversal, storage, cache, runtime codec boundary, and + unchanged-subtree copier for 2D and 3D. 3. Make paired-tree walking reusable without putting mesh or raster merge policy into the shared layer. 4. Replace the current layout-strategy class hierarchy with small path-mapping values backed by function pairs. -5. Preserve all valid existing 3D index files and payload paths. -6. Introduce the 2D storage adapter without inventing unspecified raster file +5. Allow one logical node payload to consist of multiple files without making + layouts aware of those files. +6. Preserve all valid existing 3D index files and payload paths. +7. Introduce the 2D storage adapter without inventing unspecified raster file details. -7. Land the refactor in small, testable steps. Every phase should build and +8. Land the refactor in small, testable steps. Every phase should build and pass tests before the next phase begins. ## Non-goals @@ -45,7 +59,7 @@ record of completed work. other 3D spatial calculations. - Defining or implementing GDAL ingestion, raster resampling, filtering, source selection, or mask rasterisation. -- Defining the `.arft` payload or source-attribution-table serialization +- Defining the `.amort` payload or source-attribution-table serialization beyond the requirements already in [storage-format.md](storage-format.md). - Implementing an `rf_builder`, `rf_merger`, tile-base generator, or tile server in this refactor. @@ -69,8 +83,9 @@ Before moving code, tests must lock down the following 3D behaviour: | Coordinate path | `///` | | Default layout | existing level/coordinate layout | | Layout detection | both existing layouts remain detectable | -| Copy with equal extensions | hard link, or an explicit error | -| Copy with different extensions | decode and encode through the codec | +| Codec selection | legacy preferred extension selects terrain or configured glTF codec | +| Equal codec path lists | hard-link every file, or report an explicit error | +| Different codec path lists | decode with input codec and encode with output codec | Compatibility means that the refactored code can open datasets written before the refactor and produces datasets that the pre-refactor code can open. Exact @@ -91,6 +106,7 @@ src/terrainlib/ │ ├── Traits.h │ ├── Index.h │ ├── traverse.h +│ ├── NodePath.h │ ├── PathMapping.h │ ├── Layout.h │ ├── Codec.h @@ -106,6 +122,10 @@ src/terrainlib/ │ └── merge/ │ ├── Action.h │ └── walk.h +├── mesh/ +│ └── codec/ +│ ├── Terrain.h +│ └── Gltf.h ├── octree/ │ ├── Id.h │ ├── StoreTraits.h @@ -120,6 +140,9 @@ src/terrainlib/ ├── StoreTraits.h ├── IndexFile.h ├── Storage.h + ├── codec/ + │ ├── Arft.h + │ └── Debug.h └── store_layout/ └── Zxy.h ``` @@ -128,8 +151,10 @@ The exact file grouping may be collapsed if a file would only contain a few lines. The important boundaries are: - `store` contains dimension- and payload-neutral mechanisms; -- `octree` contains the legacy 3D format and key adapters; -- `raster_store` contains the new 2D format and key adapters; and +- `mesh::codec` contains the separately configured terrain and glTF codecs; +- `octree` contains the 3D format and key adapters; +- `raster_store` contains the new 2D format, key adapters, and raster codecs; + and - subdirectory names match their namespaces where a subnamespace is used. Temporary forwarding headers and aliases under `octree` are allowed during @@ -204,25 +229,33 @@ Traversal must follow only indexed nodes and use `Traits::children`. Child order is the order supplied by the traits and is therefore deterministic per hierarchy, not universally fixed by `store`. -### Path mappings +### Node paths and path mappings + +`store::NodePath` is an extensionless logical location for one hierarchy +node. For example: + +```text +octree flat 12-123456 +octree coordinates 12/34/56/78 +raster-store ZXY 12/2200/1400 +``` -Replace `octree::disk::layout::Strategy` and +It does not necessarily name a physical file. Replace +`octree::disk::layout::Strategy` and `octree::disk::layout::StrategyRegister` with a value similar to: ```cpp template struct store::PathMapping { std::string_view id; - std::filesystem::path (*key_to_path)( - const Key&, std::string_view extension_with_dot); - std::optional (*path_to_key)( - const std::filesystem::path& relative_path); + NodePath (*key_to_node_path)(const Key&); + std::optional (*node_path_to_key)(const NodePath&); }; ``` -`store::Layout` owns the base directory, preferred extension, and one -`PathMapping`. It only adds/removes the base directory around the two -mapping functions. +`store::Layout` owns the base directory and one `PathMapping`. It +does not own a preferred extension. A configured codec expands the +extensionless `NodePath` into the physical file or files. The stable ID is format metadata, not a third strategy operation. The dimension adapters provide ordinary lookup functions: @@ -241,40 +274,132 @@ This retains runtime selection from an index file while removing virtual dispatch, RTTI type-to-ID lookup, static registration, and ownership through `unique_ptr`. -Path parsers must validate the complete relative path and the expected file -extension at the `Layout` boundary. They must return an error or `nullopt`; -they must not assert on input read from disk. +Path parsers validate the complete logical `NodePath`, not a file ending. +Codec or format-adapter code removes and validates physical file endings +before asking the layout to recover a key. Invalid disk input returns an error +or `nullopt`; it must not trigger an assertion. + +### Codec interface + +A codec is a configured runtime object for one logical payload type. It owns +all physical filename endings and may map one `NodePath` to several files: + +```cpp +template +class store::Codec { +public: + virtual ~Codec() = default; + + virtual std::vector + paths(const NodePath& node_path) const = 0; + + virtual NodeData read(const NodePath&) const { + throw UnsupportedCodecOperation{"read"}; + } + + virtual void write( + const NodePath&, + const NodeData&) const { + throw UnsupportedCodecOperation{"write"}; + } +}; +``` + +Concrete codecs contain their configuration and are constructed before +storage use. Unsupported read or write operations may use the base +implementation and throw at runtime. + +`Codec::paths()` has the following contract: + +- it needs no NodeData payload and performs no filesystem access; +- it returns every physical file belonging to the logical node; +- results depend only on codec configuration and the supplied `NodePath`; +- result order is stable and pairs corresponding input/output files; +- two codecs returning the same path list for the same `NodePath` must produce + mutually compatible files; and +- different artifact counts or filename endings produce different lists. + +Examples: + +```text +Terrain codec + 12/34/56/78 + -> 12/34/56/78.terrain + +glTF codec configured for binary output + 12/34/56/78 + -> 12/34/56/78.glb + +glTF codec configured for JSON output + 12/34/56/78 + -> 12/34/56/78.gltf + +Debug raster codec configured for JPEG data and PNG attribution + 12/2200/1400 + -> 12/2200/1400.data.jpg + -> 12/2200/1400.attribution.png +``` + +The mesh side has separate terrain and glTF codecs because they use different +format implementations. Binary `.glb` and JSON `.gltf` remain configurations +of one glTF codec because both use the same `cgltf` implementation. + +The raster side has similarly shaped codecs specialized on PixelType: + +```cpp +template struct raster_store::codec::Amort : store::Codec> { .. }; +template struct raster_store::codec::Debug : store::Codec> { .. }; +``` + +`Amort` supports reading and writing. `Debug` supports writing only and may +contain runtime options for data format, attribution format, JPEG quality, or +similar debugging choices. It is not template-composed from separate image +codec types. + ### Storage and format adapters -Generalize these mechanisms over traits, payload, and codec: +Generalize storage over traits and NodeData. It owns a configured codec through +the runtime interface: ```cpp -store::RawStorage -store::Storage -store::IndexedStorage -store::cache::Interface +store::RawStorage +store::Storage +store::IndexedStorage +store::cache::Interface ``` -The payload codec remains path-based and key-neutral. Mesh and raster codecs -remain with their payload domains. +The NodeData codecs remain with their domains under `mesh::codec` and +`raster_store::codec`. Index serialization is not a responsibility of `store::Index`. Opening and -saving a dataset must receive a dimension-specific format adapter which -provides: +saving a dataset receives a dimension-specific format adapter which provides: - the index filename; - index read/write conversion; - mapping lookup by stable ID; - the default mapping; and -- the mappings considered during legacy directory scanning. +- legacy directory discovery where it is required. This adapter may be a compile-time policy or a small value of function pointers. Choose the smaller implementation after the Phase 0 tests exist. -It must not reintroduce a class hierarchy or global registration. +It must not reintroduce a layout class hierarchy or global registration. For 3D, the adapter reads and writes the current `octree` index DTO unchanged. -For 2D, it reads and writes a separately versioned +Its legacy `preferred_extension` field selects the configured codec: + +```text +.terrain -> terrain codec +.glb -> glTF codec with binary container +.gltf -> glTF codec with JSON container +``` + +Legacy unindexed-directory discovery remains in the 3D adapter: it recognizes +the known codec endings, removes them to obtain a `NodePath`, and then invokes +the selected layout parser. The generic layout does not recover keys directly +from codec-owned file paths. + +For 2D, the adapter reads and writes a separately versioned `raster_store::v1` DTO using the serialization envelope required by [storage-format.md](storage-format.md). @@ -283,17 +408,48 @@ Preserve that behaviour for existing 3D entry points during the migration. The new 2D snapshot API should require an explicit finalization/publication step; a destructor must not make an incomplete snapshot authoritative. -### Unchanged-subtree reuse +### Copying and unchanged-subtree reuse + +Keep `Storage::copy_from()` close to its current role. Add: + +```cpp +struct CopyOptions { + bool force_reencode = false; +}; +``` + +For one key, `copy_from()`: + +1. calls the input and output `Codec::paths()` with the same fixed dummy + `NodePath`; +2. when the lists are equal and `force_reencode` is false, calls both codecs + again with their actual source and target `NodePath` values and hard-links + every source path to the corresponding target path; +3. when the dummy lists differ or re-encoding is forced, reads the payload + with the input codec and writes it with the output codec; and +4. updates the target index only after all links or the write complete. + +The dummy path must be fixed and collision-free, for example +`__codec_probe__/node`. Path lists are compared exactly, including count, +order, and filename endings. + +If linking several files fails partway through, remove the target links +created by that call before returning the error. There is no silent copy +fallback. An unsupported read or write needed for re-encoding throws through +the codec interface. + +Codec settings that do not change `paths()`, such as compression level or +JPEG quality, do not force re-encoding by default. A caller that needs those +settings applied to every node passes `force_reencode = true`. Move payload-neutral subtree copying out of `sf_merger::NodeWriter`. The -shared operation should: +shared operation: -1. traverse an indexed source subtree; -2. skip `Virtual` nodes; -3. copy physical payloads for both `Leaf` and `Inner`; -4. use `Storage::copy_from` so equal-format payloads are hard-linked; -5. update the target index incrementally; and -6. return an error instead of asserting or terminating. +1. traverses an indexed source subtree; +2. skips `Virtual` nodes; +3. calls `copy_from()` for physical payloads in both `Leaf` and `Inner` + states; and +4. propagates errors instead of asserting or terminating. This fixes the current incorrect assumption that every non-virtual visited node is a `Leaf`. @@ -301,12 +457,14 @@ node is a `Leaf`. Hard-link rules: - never modify an existing linked payload in place; -- matching extensions require hard-link creation; -- different extensions retain the current decode/re-encode path; +- a matching codec path list hard-links every file; +- a different path list decodes with the input codec and encodes with the + output codec; +- `force_reencode` always selects decode/encode; - hard-link failure is explicit; -- 2D snapshot tools must preflight that source and destination support hard - links before a long operation starts; and -- no silent copy fallback is introduced by this refactor. +- 2D snapshot tools preflight that source and destination support hard links + before a long operation starts; and +- no silent file-copy fallback is introduced. ### Paired hierarchy walking @@ -361,6 +519,7 @@ No production behaviour changes. 4. Add storage tests for: - matching-extension hard links; - different-extension decode/re-encode; + - `.terrain`, `.glb`, and `.gltf` dispatch; - overwrite rejection; - indexed and unindexed opens; and - final index creation by directory scan. @@ -388,59 +547,95 @@ callers still build through aliases; no filesystem code has changed. ### Phase 2 — Replace disk layout strategies -1. Add `store::PathMapping` and `store::Layout`. +1. Add `store::NodePath`, `store::PathMapping`, and + `store::Layout`. 2. Port the two existing 3D layouts to ordinary function pairs without - changing paths or stable IDs. + changing stable IDs. The mappings return `level-index` and + `level/x/y/z` without file endings. 3. Replace the singleton strategy registry with explicit `from_id()` and `all()` functions in the 3D adapter. -4. Port layout guessing to consume a span of mappings supplied by the format - adapter. -5. Add the proposed 2D `z/x/y.arft` mapping only after the review decision - listed below is resolved. -6. Switch path and layout-detection tests to the new implementation. -7. Delete the old strategy base class, registration machinery, and concrete +4. Move the legacy preferred extension out of generic `Layout` and retain it + in the 3D format adapter. +5. Port legacy layout discovery so it strips recognized 3D file endings before + calling `node_path_to_key()`. +6. Add the proposed 2D `z/x/y` mapping only after the review decision listed + below is resolved. The raster codec, not the mapping, adds `.arft` or debug + endings. +7. Switch node-path and layout-detection tests to the new implementation. +8. Delete the old strategy base class, registration machinery, and concrete strategy classes once no call site uses them. -Exit criterion: both legacy fixtures resolve to identical payload paths; +Exit criterion: the legacy 3D adapter plus codec resolves both fixtures to +identical physical payload paths; generic `Layout` contains no extension; there is no layout inheritance, RTTI lookup, static registrar, or owning strategy pointer. ### Phase 3 — Generalize storage and index lifecycle -1. Move the codec concept, copy error, raw storage, caches, logical storage, - and indexed storage into `store`. -2. Replace every embedded `octree::Id` with `Traits::Key`. -3. Keep payload codecs outside the shared module: - `octree::MeshCodec` remains 3D, and the future `.arft` codec remains under - `raster_store`. -4. Split generic directory scanning from 3D index serialization. -5. Implement the small format-adapter boundary described above. -6. Keep the current 3D `terrain.index` DTO and open functions as compatibility - adapters over the shared storage. -7. Migrate the existing octree storage aliases and all application callers. -8. Preserve the current 3D destructor-save behaviour until all callers have - explicit index finalization. +1. Add the stateful `store::Codec` interface with `paths()`, `read()`, + and `write()`. +2. Split the current extension-dispatching `octree::MeshCodec` into: + - a terrain codec; and + - one glTF codec configured for binary `.glb` or JSON `.gltf`. +3. Move copy error, raw storage, caches, logical storage, and indexed storage + into `store`. +4. Make storage own a configured `std::unique_ptr>`; remove the + codec template parameter from storage. +5. Replace every embedded `octree::Id` with `Traits::Key`. +6. Make every raw file operation obtain its complete file list through + `Codec::paths()`. `has()` requires every listed file, and `remove()` removes + every listed file. +7. Keep payload codecs outside the shared module under `mesh::codec` and + `raster_store::codec`. +8. Split generic index maintenance from 3D index serialization and legacy + folder discovery. +9. Keep the current 3D `terrain.index` DTO and open functions as compatibility + adapters over the shared storage. Map its preferred extension to a + configured terrain or glTF codec. +10. Migrate the existing octree storage aliases and all application callers. +11. Preserve the current 3D destructor-save behaviour until all callers have + explicit index finalization. + +Add focused codec tests using single-file, multi-file, read/write, and +write-only test codecs before depending on the raster payload implementation. Exit criterion: all existing applications build and all Phase 0 fixtures pass -through the shared storage implementation. No second storage implementation -remains under `octree`. +through the shared runtime codec and storage implementation. No +extension-dispatching mesh codec or second storage implementation remains +under `octree`. ### Phase 4 — Generalize subtree reuse and paired walking -1. Add the shared unchanged-subtree copier. -2. Test copies containing `Leaf`, `Virtual`, and `Inner` nodes. -3. Add the paired hierarchy walker and typed actions. -4. Cover all 16 status pairs with table-driven tests. -5. Adapt the 3D merger to the shared walker while keeping mesh policy in +1. Change `Storage::copy_from()` to compare input and output codec path lists + for the fixed dummy `NodePath`. +2. Hard-link all actual files when the lists match, with cleanup of links + created by a partially failed call. +3. Decode with the input codec and encode with the output codec when lists + differ. +4. Add `CopyOptions::force_reencode`, defaulting to false. +5. Test: + - one-file hard linking; + - multi-file hard linking; + - different path counts and endings; + - forced re-encoding with otherwise equal paths; + - conversion between terrain and glTF; + - conversion into a write-only codec; and + - runtime failure when a required codec operation is unsupported. +6. Add the shared unchanged-subtree copier. +7. Test copies containing `Leaf`, `Virtual`, and `Inner` nodes. +8. Add the paired hierarchy walker and typed actions. +9. Cover all 16 status pairs with table-driven tests. +10. Adapt the 3D merger to the shared walker while keeping mesh policy in `sf_merger`. -6. Remove generic recursion and copy logic from `sf_merger::Merger` and +11. Remove generic recursion and copy logic from `sf_merger::Merger` and `NodeWriter`. -7. Add a 3D integration test proving an unchanged subtree is hard-linked and +12. Add a 3D integration test proving an unchanged subtree is hard-linked and a changed boundary node is newly written. Exit criterion: the existing 3D merger behaviour is preserved, `Inner` no -longer reaches `UNREACHABLE()`, and the shared walker contains no mesh, ECEF, -GDAL, OpenCV, or raster dependencies. +longer reaches `UNREACHABLE()`, multi-file reuse works through +`Codec::paths()`, and the shared walker contains no mesh, ECEF, GDAL, OpenCV, +or raster dependencies. ### Phase 5 — Add the 2D raster-fundamentalis adapter @@ -453,15 +648,20 @@ edited into decisions in this document. 4. Use the required magic/version/checksum/compression serialization envelope. 5. Add the 2D format adapter and storage aliases under `raster_store`. -6. Exercise storage with a small test codec if the final `.arft` codec is not - implemented yet; do not make `.arft` claims from a placeholder codec. -7. Test: +6. Add `raster_store::codec::Arft` with runtime format options when + the final `.arft` serialization is available. +7. Add the output-only `raster_store::codec::Debug` with runtime + options for its data and attribution files. +8. If the final `.arft` serialization is not available, exercise storage with + the Phase 3 test codec and do not make `.arft` claims from it. +9. Test: - invalid and boundary tile IDs; - index serialization and validation; - `Leaf`/`Inner` coexistence; - sparse traversal and ancestor lookup; - path round trips; - - snapshot hard-link reuse; and + - snapshot hard-link reuse; + - ARFT-to-debug output conversion; and - explicit cross-filesystem/preflight failure. Exit criterion: the same shared store can create, open, traverse, and reuse a @@ -493,6 +693,7 @@ files: unittests/terrainlib/store_index.cpp unittests/terrainlib/store_traverse.cpp unittests/terrainlib/store_layout.cpp +unittests/terrainlib/store_codec.cpp unittests/terrainlib/store_storage.cpp unittests/terrainlib/store_compatibility.cpp unittests/terrainlib/store_merge_walk.cpp @@ -519,13 +720,15 @@ No formatting-only pass or unrelated refactor belongs in these commits. | `octree/NodeStatusOrMissing.h` | `store/NodeStatusOrMissing.h` plus temporary alias | | `octree/IndexMap.*` | `store/Index.h` | | `octree/traverse.h` | `store/traverse.h` | +| complete node paths embedded in layouts | extensionless `store/NodePath.h` plus codec endings | | `octree/disk/Layout.h` | `store/Layout.h` | | `octree/disk/layout/Strategy.h` | `store/PathMapping.h` | | `StrategyRegister.h` | explicit dimension-adapter lookup functions | | `strategy/Flat.h` | `octree/store_layout/Flat.h` | | `strategy/LevelAndCoordinateDirectories.h` | `octree/store_layout/LevelAndCoordinateDirectories.h` | | `octree/storage/cache/*` | `store/cache/*` | -| `octree/storage/codec/Codec.h` | `store/Codec.h` | +| `octree/storage/codec/Codec.h` | runtime `store/Codec.h` | +| `octree/storage/codec/MeshCodec.h` | `mesh/codec/Terrain.h` and configured `mesh/codec/Gltf.h` | | `octree/storage/RawStorage.h` | `store/RawStorage.h` | | `octree/storage/Storage.h` | `store/Storage.h` | | `octree/storage/IndexedStorage.h` | `store/IndexedStorage.h` | @@ -545,6 +748,9 @@ No formatting-only pass or unrelated refactor belongs in these commits. | Invalid 2D coordinates become persistent | Validate on every disk/API boundary | | `Inner` payloads are lost during subtree reuse | Copy every physical status and test mixed-depth fixtures | | Linked snapshots are modified in place | Immutable snapshot API and overwrite-disabled output | +| Multi-file hard linking fails partway through | Remove links created by the failed `copy_from()` before returning | +| Incompatible codecs return the same path list | Treat path-list equality as a codec contract and test every concrete codec pairing | +| Output-only codec is selected for required input | Throw a clear unsupported-operation error at runtime | | Hard-link failure appears late | 2D operation preflight and explicit errors | | Shared code accumulates mesh/raster policy | Dependency tests/review against the source boundary | | Generic index accidentally dictates both disk formats | Separate 3D and 2D format adapters | @@ -556,8 +762,9 @@ mark them as unclear: 1. **2D index filename:** choose the filename used inside a raster-fundamentalis snapshot. -2. **2D payload path:** confirm `z/x/y.arft`, including whether chunks live - directly under the snapshot or below a `chunks/` directory. +2. **2D node and payload paths:** confirm the extensionless `z/x/y` + `NodePath`, the ARFT codec's resulting `z/x/y.arft` file, and whether + chunks live directly under the snapshot or below a `chunks/` directory. 3. **2D layout ID:** choose the stable string serialized in the index. 4. **Maximum zoom:** choose the supported persistent range and integer widths for zoom, x, and y. diff --git a/docs/raster-store/storage-format.md b/docs/raster-store/storage-format.md index 8be6b5eb..673bbfd5 100644 --- a/docs/raster-store/storage-format.md +++ b/docs/raster-store/storage-format.md @@ -23,36 +23,30 @@ The source attribution table is used in raster-fundamentalis, tile-base, and in - there is one per directory tree (it's valid for all tiles stored within the same directory tree, all siblings and children). - given a tile (either rf or tb), the lookup of the source attribution table is first in the same directory and then in all parrent dirs, until a source_attribution_table.ard is found (or a failure is thrown). +## alpine maps raster store format +this is the binary format used to store rf and tb tiles. both share the same basic tile format, but use it in different ways. +- The hierarchy is a Web Mercator (EPSG:3857) quadtree keyed by tile IDs (radix::tile::ID, https://docs.maptiler.com/google-maps-coordinates-tile-bounds-projection/). +- stored tiles have a resolution of 4096x4096 pixels +- Every stored pixel has one data value (can be vector type) and one source attribution index. +- the source attribution index is stored as uint16, and indexes into a global source attribution table (see above) +- data is stored in radix::Raster objects (one for source attribution index, one for the actual data), i.e. template struct raster_store::Tile { radix::Raster data; radix::Raster source_attribution; }; +- the file ending is .amort (AlpineMapsOrg raster tile), it is serialised used the principles outlined below. + ## raster-fundamentalis (rf) format raster-fundamentalis is our authoritative raster-store, containing only the data and no overviews / downsampled version. -- The hierarchy is a Web Mercator (EPSG:3857) quadtree keyed by tile IDs (radix::tile::ID, https://docs.maptiler.com/google-maps-coordinates-tile-bounds-projection/). - unlike a tile pyramid, not every level is occupied (there is no downsampled versions of the data). -- stored tiles have a resolution of 4096x4096 pixels -- Every stored pixel has one payload value (can be vector type) and one source attribution index. - Coarse physical tiles (e.g. zoom level 10) may coexist with more accurate descendants (e.g. zoom level 15). -- the source attribution index is stored as uint16, and indexes into a global source attribution table (see above). -- data is stored in radix::Raster objects (one for source attribution index, one for the actual data) -- raster-fundamentalis is implemented in src/terrainlib/raster/fundamentalis.h, in the namespace raster::fundamentalis::* -- tiles are stored in .arft files (alpine raster fundamentalis tile) - -### unclear -- exact quadtree format / how to reuse structura fundamentalis code -- file names / paths / index file name +- the raster-fundamentalis builder is implemented in src/rf-builder/*, it consumes raw gdal data. ## Tile-base Format (tb) tile-base is a hierarchy build from raster-fundamentalis, containing all data and its overviews / downsampled versions. it is used directly by the tile-server to generate tiles at the requested resolution and format. -- The hierarchy is a Web Mercator (EPSG:3857) quadtree keyed by tile IDs (radix::tile::ID, https://docs.maptiler.com/google-maps-coordinates-tile-bounds-projection/). - every level is occupied, and every level selects an adequate data source -- stored tiles have a resolution of 4096x4096 pixels (?) -- Every stored pixel has one payload value (can be vector type) and one source attribution index. -- the source attribution index is stored as uint16, and indexes into a global source attribution table (see above) -- data is stored in radix::Raster objects (one for source attribution index, one for the actual data) -- tile-base is implemented in src/terrainlib/raster/tile_base.h, in the namespace raster::tile_base::* -- tiles are stored in .artb files (alpine raster tile base) +- the tile-base builder is implemented in src/tb-builder, it consumes rf ### tile-server - should generate tiles of requested resolution and pixel type (vertex|area) on the fly - requests by url, e.g.: layer/vertex|area/resolution/z/x/y.ending +- live in src/tile-server/* ### unclear - how to build tile-base from raster-fundamentalis diff --git a/docs/raster-store/terminology.md b/docs/raster-store/terminology.md index cff4ab31..eb98976b 100644 --- a/docs/raster-store/terminology.md +++ b/docs/raster-store/terminology.md @@ -1,7 +1,4 @@ # Terminology -** payload raster ** -: A square data matrix (image), containing the actual data (e.g. elevation, colour..) - ** attribution raster ** : A square data matrix (image), containing indices into the source-attribution table From b52879ec66fb4e3b0e87680fc6067f68b968572b Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:10:05 +0200 Subject: [PATCH 21/61] plan++ --- docs/raster-store/refactor-plan.md | 148 +++++++++++++++++++++++++++-- 1 file changed, 141 insertions(+), 7 deletions(-) diff --git a/docs/raster-store/refactor-plan.md b/docs/raster-store/refactor-plan.md index f6f9bda9..60b31bcf 100644 --- a/docs/raster-store/refactor-plan.md +++ b/docs/raster-store/refactor-plan.md @@ -410,7 +410,40 @@ step; a destructor must not make an incomplete snapshot authoritative. ### Copying and unchanged-subtree reuse -Keep `Storage::copy_from()` close to its current role. Add: +There are two separate responsibilities in the current implementation: + +1. `sf_merger::NodeWriter` traverses a source subtree and decides which + indexed nodes to reuse. +2. `octree::Storage::copy_from()` delegates to + `octree::RawStorage::copy_from()`, where + `std::filesystem::create_hard_link()` performs the actual hard link and the + target index is updated on success. + +The filesystem hard-link implementation is therefore already in terrainlib. +This refactor shall move the payload-neutral subtree traversal and copy +orchestration out of `sf_merger`. + +The current call chain is: + +```text +sf_merger decides to keep a source subtree unchanged + -> NodeWriter traverses the source index + -> octree::Storage::copy_from() + -> octree::RawStorage::copy_from() + -> create_hard_link(), or decode/encode when formats differ +``` + +The target call chain becomes: + +```text +merge policy decides to keep a source subtree unchanged + -> store::copy_subtree() + -> store::Storage::copy_from() + -> hard-link every codec path, or decode/encode +``` + +`Storage::copy_from()` remains the operation for copying one logical node. +Add: ```cpp struct CopyOptions { @@ -442,17 +475,118 @@ Codec settings that do not change `paths()`, such as compression level or JPEG quality, do not force re-encoding by default. A caller that needs those settings applied to every node passes `force_reencode = true`. -Move payload-neutral subtree copying out of `sf_merger::NodeWriter`. The -shared operation: +#### Shared subtree copier + +Move the payload-neutral traversal in +`sf_merger::NodeWriter::copy_subtree_to_output()` to a shared operation such +as: + +```cpp +std::expected +store::copy_subtree( + const IndexedStorage& source, + Storage& target, + const Key& root, + CopyOptions options = {}); +``` + +The shared operation: 1. traverses an indexed source subtree; 2. skips `Virtual` nodes; 3. calls `copy_from()` for physical payloads in both `Leaf` and `Inner` - states; and -4. propagates errors instead of asserting or terminating. + states; +4. continues traversal below `Inner` nodes; and +5. returns copy failures to its caller instead of converting them into an + assertion or immediate process termination. + +The operation is payload-neutral because it only interprets hierarchy status +and delegates each physical node to `Storage::copy_from()`. It does not know +about meshes, rasters, masks, attribution, or their encodings. + +#### `Leaf`, `Inner`, and `Virtual` + +The sparse index distinguishes payload presence from descendant presence: + +| Status | Physical payload | Indexed descendants | +|---|---:|---:| +| `Leaf` | yes | no | +| `Inner` | yes | yes | +| `Virtual` | no | yes | + +`Missing` is represented by absence from the sparse index and is not visited +by subtree traversal. + +The current `NodeWriter` callback effectively does: + +```cpp +if (status == NodeStatus::Virtual) { + return; +} + +DEBUG_ASSERT(status == NodeStatus::Leaf); +DEBUG_ASSERT_VAL(target.copy_from(id, source)); +``` + +The `DEBUG_ASSERT(status == Leaf)` is an incorrect assumption. `Inner` is +also a physical state and its payload must be copied. The existing +`IndexMap::add()` creates an `Inner` node whenever a physical `Leaf` gains a +physical descendant. Such a hierarchy is valid in 3D and is explicitly +required for raster-fundamentalis, where a coarse physical tile may coexist +with more accurate descendants. + +For example: + +```text +zoom 10 physical tile -> Inner +└── zoom 11 physical tile -> Leaf +``` + +Reusing this subtree must preserve both payloads. Skipping the `Inner` +payload would lose the coarse fallback; asserting on `Inner` rejects a valid +hierarchy. + +The shared copier shall instead handle status as: + +```cpp +switch (status) { +case NodeStatus::Virtual: + break; +case NodeStatus::Leaf: +case NodeStatus::Inner: + target.copy_from(id, source, options); + break; +} +``` -This fixes the current incorrect assumption that every non-virtual visited -node is a `Leaf`. +Traversal shall continue below the `Inner` node. Copying the parent first adds it +to the target as a `Leaf`; copying its descendant then promotes the parent to +`Inner`, reconstructing the source topology through the normal index +transitions. + +The existing paired merger has a related limitation: its dispatcher only +handles `Missing`, `Leaf`, and `Virtual` pairs and sends any pair containing +`Inner` to `UNREACHABLE()`. The paired hierarchy-walking work below must +handle or explicitly reject all 16 status combinations without treating +valid input data as an impossible program state. + +#### Error propagation + +The lower storage layer already represents ordinary copy failures, including +missing source files, directory creation failure, hard-link failure, decode +failure, and encode failure. The current `NodeWriter` consumes +`Storage::copy_from()` with `DEBUG_ASSERT_VAL`, while some overwrite paths +terminate through `LOG_ERROR_AND_EXIT()`. Because +`copy_subtree_to_output()` returns `void`, the merge caller cannot report or +handle these failures. + +The shared copier must return the failure through `copy_subtree()` and the +merge call chain until the application boundary can report it with the +affected key and path. An unsupported codec read or write may propagate as +the codec's runtime exception. Assertions remain appropriate for internal +invariants, but filesystem conditions, unsupported conversions, malformed +datasets, and overwrite conflicts are operational errors rather than +assertion failures. Hard-link rules: From a2a9aa4ce3f0bd05be5363e132fb4fa30e996a2c Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:19:11 +0200 Subject: [PATCH 22/61] plan++ --- docs/raster-store/refactor-plan.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/raster-store/refactor-plan.md b/docs/raster-store/refactor-plan.md index 60b31bcf..27787697 100644 --- a/docs/raster-store/refactor-plan.md +++ b/docs/raster-store/refactor-plan.md @@ -475,6 +475,18 @@ Codec settings that do not change `paths()`, such as compression level or JPEG quality, do not force re-encoding by default. A caller that needs those settings applied to every node passes `force_reencode = true`. +Hard-link rules: + +- never modify an existing linked payload in place; +- a matching codec path list hard-links every file; +- a different path list decodes with the input codec and encodes with the + output codec; +- `force_reencode` always selects decode/encode; +- hard-link failure is explicit; +- 2D snapshot tools preflight that source and destination support hard links + before a long operation starts; and +- no silent file-copy fallback is introduced. + #### Shared subtree copier Move the payload-neutral traversal in @@ -588,18 +600,6 @@ invariants, but filesystem conditions, unsupported conversions, malformed datasets, and overwrite conflicts are operational errors rather than assertion failures. -Hard-link rules: - -- never modify an existing linked payload in place; -- a matching codec path list hard-links every file; -- a different path list decodes with the input codec and encodes with the - output codec; -- `force_reencode` always selects decode/encode; -- hard-link failure is explicit; -- 2D snapshot tools preflight that source and destination support hard links - before a long operation starts; and -- no silent file-copy fallback is introduced. - ### Paired hierarchy walking Extract only the dimension-neutral control flow from `sf_merger::Merger`. From 37ffeaccb7130aae424a9c5c6f54fa29ffffef9c Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Mon, 27 Jul 2026 18:13:11 +0200 Subject: [PATCH 23/61] plan++ --- docs/raster-store/refactor-plan.md | 7 +++++++ docs/raster-store/storage-format.md | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/raster-store/refactor-plan.md b/docs/raster-store/refactor-plan.md index 27787697..5af2530a 100644 --- a/docs/raster-store/refactor-plan.md +++ b/docs/raster-store/refactor-plan.md @@ -896,21 +896,28 @@ mark them as unclear: 1. **2D index filename:** choose the filename used inside a raster-fundamentalis snapshot. + => call it raster_store.index. 2. **2D node and payload paths:** confirm the extensionless `z/x/y` `NodePath`, the ARFT codec's resulting `z/x/y.arft` file, and whether chunks live directly under the snapshot or below a `chunks/` directory. + => I already renamed .arft into .amort (alpine maps org raster tile) in several places. look for places i missed and rename it. the chunks / tiles shall live under a path defined by the store_layout, the default 2d store layout shall put them into z/x/y, not in chunks/z/x/y. 3. **2D layout ID:** choose the stable string serialized in the index. + => "zoom/x/y_google" 4. **Maximum zoom:** choose the supported persistent range and integer widths for zoom, x, and y. + => use the tile id from radix. x and y are unsigned 32, that would give maximum zoom 31 or 32? that should be well enough. 5. **Index contents:** decide whether v1 stores only sparse status entries or also derived aggregate metadata. The first implementation should omit derivable metadata unless a concrete query requires it. + => is this a question? 6. **Index envelope constants:** assign the 2D index magic number, compression choice, and version according to the common serialization rules. + => erm, question? 7. **Publication:** define whether snapshot completion uses an atomic rename, a manifest marker, or an external store-level operation. The generic storage layer should expose finalization but not invent store-level lifecycle policy. + => use an atomic rename, but don't hide the file. use a .part extension to the new directory name, and then remove it. make sure to flush and close all files before renaming. Until these decisions are made, Phases 0–4 can complete and the shared implementation can be proven with `radix::tile::Id` in memory and with diff --git a/docs/raster-store/storage-format.md b/docs/raster-store/storage-format.md index 673bbfd5..c476517d 100644 --- a/docs/raster-store/storage-format.md +++ b/docs/raster-store/storage-format.md @@ -72,7 +72,8 @@ tile-base is a hierarchy build from raster-fundamentalis, containing all data an - if the checksum check fails - if the compression algorithm is missing or unsupported. - deserialization fails -- we fail by throwing +- we fail by throwing in these cases +- compression: use libzstd with best compression. libzstd must be imported via the projects cmake install facility from https://github.com/AlpineMapsOrgDependencies/zstd ## to be defined From db795cbd0c0268e7229d65abe1cd384883adf7e2 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Mon, 27 Jul 2026 18:54:36 +0200 Subject: [PATCH 24/61] plan++ --- docs/raster-store/architecture.md | 21 +++- docs/raster-store/refactor-plan.md | 153 +++++++++++++++++----------- docs/raster-store/storage-format.md | 15 ++- 3 files changed, 124 insertions(+), 65 deletions(-) diff --git a/docs/raster-store/architecture.md b/docs/raster-store/architecture.md index 356cdc25..10a62418 100644 --- a/docs/raster-store/architecture.md +++ b/docs/raster-store/architecture.md @@ -34,10 +34,14 @@ A store root contains immutable snapshots. Each snapshot contains an index, a so store/ └── snapshot-id/ ├── source_attribution_table.ard - ├── index - └── chunks/...[.arft|.artb] (or, z/x/y.[arft|artb]) + ├── raster_store.index + └── //.amort ``` +The shown payload path is the default `zoom/x/y_google` layout. Other layouts +may map the same tile IDs differently; there is no mandatory `chunks/` +directory. + ## Sparse quadtree index The index uses the same four logical states as the octree index: @@ -98,6 +102,19 @@ When merging, we need to create new hardlinks for unchanged rf tiles (taken comp Cross-filesystem hard links cannot be created. The builder must expose this as a clear configuration error rather than discovering it after a long build. +A new snapshot is assembled in a sibling directory named +`.part`. The builder writes the index last, validates the +snapshot, flushes and closes all files, and atomically renames the directory +to `` on the same filesystem. The final destination must not +already exist. A `.part` directory is incomplete and must not be exposed as a +published snapshot during normal operation. + +Publication does not guarantee durability or safe recovery across a power +failure, operating-system crash, or storage failure. The implementation does +not perform platform-specific filesystem syncing. After such a failure, +either a `.part` directory or a final snapshot may be unusable and must be +validated and rebuilt. + ## Pyramid generator interface (to be confirmed, LLM, do not use the following without consultation) A generator requests a layer over a target tile and sampling specification. diff --git a/docs/raster-store/refactor-plan.md b/docs/raster-store/refactor-plan.md index 5af2530a..31da1ef5 100644 --- a/docs/raster-store/refactor-plan.md +++ b/docs/raster-store/refactor-plan.md @@ -141,10 +141,10 @@ src/terrainlib/ ├── IndexFile.h ├── Storage.h ├── codec/ - │ ├── Arft.h + │ ├── Amort.h │ └── Debug.h └── store_layout/ - └── Zxy.h + └── ZoomXYGoogle.h ``` The exact file grouping may be collapsed if a file would only contain a few @@ -197,10 +197,15 @@ changing `octree::Id`. - treat zoom zero as the only root; - never call `radix::tile::Id::parent()` at zoom zero, where it underflows; - reject coordinates outside `[0, 2^zoom)`; -- define a supported maximum zoom without an overflowing shift; +- accept zoom levels 0 through + `std::numeric_limits::digits`, inclusive; +- treat that maximum zoom as terminal because a child cannot be represented + by the `uint32_t` x/y coordinates; +- validate the maximum zoom without evaluating an overflowing + `uint32_t{1} << 32`; - use `radix::tile::Id::Hasher`; and -- accept only one canonical XYZ/Slippy interpretation at the persistent - boundary. +- use the Google/XYZ convention, with the origin at the north-west, at the + persistent boundary. The shared code must obtain roots, parents, children, validation, and hashing through the traits. It must not use dimension checks or specialize behaviour @@ -266,7 +271,7 @@ octree::store_layout::level_and_coordinate_directories() octree::store_layout::from_id(id) octree::store_layout::all() -raster_store::store_layout::zxy() +raster_store::store_layout::zoom_x_y_google() raster_store::store_layout::from_id(id) ``` @@ -399,9 +404,11 @@ the known codec endings, removes them to obtain a `NodePath`, and then invokes the selected layout parser. The generic layout does not recover keys directly from codec-owned file paths. -For 2D, the adapter reads and writes a separately versioned -`raster_store::v1` DTO using the serialization envelope required by -[storage-format.md](storage-format.md). +For 2D, the adapter reads and writes `raster_store.index` as a separately +versioned `raster_store::v1` DTO using the serialization envelope required by +[storage-format.md](storage-format.md). Version 1 stores the layout ID and +sparse key/status entries. `Missing` is represented by absence; derivable +aggregate metadata is not stored. Automatic dirty-index saving currently happens in the 3D storage destructor. Preserve that behaviour for existing 3D entry points during the migration. @@ -692,9 +699,8 @@ callers still build through aliases; no filesystem code has changed. in the 3D format adapter. 5. Port legacy layout discovery so it strips recognized 3D file endings before calling `node_path_to_key()`. -6. Add the proposed 2D `z/x/y` mapping only after the review decision listed - below is resolved. The raster codec, not the mapping, adds `.arft` or debug - endings. +6. Add the 2D `z/x/y` mapping with stable ID `zoom/x/y_google`. The raster + codec, not the mapping, adds `.amort` or debug endings. 7. Switch node-path and layout-detection tests to the new implementation. 8. Delete the old strategy base class, registration machinery, and concrete strategy classes once no call site uses them. @@ -773,30 +779,42 @@ or raster dependencies. ### Phase 5 — Add the 2D raster-fundamentalis adapter -This phase starts only after the unresolved 2D format decisions below are -edited into decisions in this document. - -1. Add the checked 2D persistent-key conversion around `radix::tile::Id`. -2. Add the chosen 2D path mapping and stable ID. -3. Define the versioned 2D index DTO and index filename. -4. Use the required magic/version/checksum/compression serialization envelope. +All required 2D format decisions are resolved below. + +1. Add the checked 2D persistent-key conversion around `radix::tile::Id`, + supporting zoom 0 through `std::numeric_limits::digits` and + treating that maximum zoom as terminal. +2. Add the default `//` path mapping with stable ID + `zoom/x/y_google`. +3. Define `raster_store.index` with a versioned 2D DTO containing the layout + ID and sparse key/status entries, without derived aggregate metadata. Use + fixed-width `uint32_t` zoom/x/y fields and serialize entries in + lexicographic `(zoom, x, y)` order. +4. Implement the version-1 magic/version/checksum/compression envelope. Store + zlib CRC-32 as `uint32_t`, computed over the compressed payload, and use + zstd at its best-compression setting. Import zstd through the project's + CMake dependency facility; use the existing `ZLIB::ZLIB` dependency for + CRC-32. 5. Add the 2D format adapter and storage aliases under `raster_store`. -6. Add `raster_store::codec::Arft` with runtime format options when - the final `.arft` serialization is available. +6. Add `raster_store::codec::Amort` with runtime format options when + the final `.amort` serialization is available. 7. Add the output-only `raster_store::codec::Debug` with runtime options for its data and attribution files. -8. If the final `.arft` serialization is not available, exercise storage with - the Phase 3 test codec and do not make `.arft` claims from it. +8. If the final `.amort` serialization is not available, exercise storage + with the Phase 3 test codec and do not make `.amort` claims from it. 9. Test: - - invalid and boundary tile IDs; + - invalid and boundary tile IDs, including the maximum zoom and rejected + children beyond it; - index serialization and validation; - `Leaf`/`Inner` coexistence; - sparse traversal and ancestor lookup; - path round trips; - snapshot hard-link reuse; - - ARFT-to-debug output conversion; and - - explicit cross-filesystem/preflight failure. + - AMORT-to-debug output conversion; + - explicit cross-filesystem/preflight failure; + - publication from `.part` to ``; and + - publication rejection when the destination already exists. Exit criterion: the same shared store can create, open, traverse, and reuse a minimal 2D raster-fundamentalis fixture without changing the 3D fixtures. @@ -891,35 +909,54 @@ No formatting-only pass or unrelated refactor belongs in these commits. ## Decisions required before Phase 5 -These are intentionally unresolved because the current raster-store documents -mark them as unclear: - -1. **2D index filename:** choose the filename used inside a - raster-fundamentalis snapshot. - => call it raster_store.index. -2. **2D node and payload paths:** confirm the extensionless `z/x/y` - `NodePath`, the ARFT codec's resulting `z/x/y.arft` file, and whether - chunks live directly under the snapshot or below a `chunks/` directory. - => I already renamed .arft into .amort (alpine maps org raster tile) in several places. look for places i missed and rename it. the chunks / tiles shall live under a path defined by the store_layout, the default 2d store layout shall put them into z/x/y, not in chunks/z/x/y. -3. **2D layout ID:** choose the stable string serialized in the index. - => "zoom/x/y_google" -4. **Maximum zoom:** choose the supported persistent range and integer widths - for zoom, x, and y. - => use the tile id from radix. x and y are unsigned 32, that would give maximum zoom 31 or 32? that should be well enough. -5. **Index contents:** decide whether v1 stores only sparse status entries or - also derived aggregate metadata. The first implementation should omit - derivable metadata unless a concrete query requires it. - => is this a question? -6. **Index envelope constants:** assign the 2D index magic number, compression - choice, and version according to the common serialization rules. - => erm, question? -7. **Publication:** define whether snapshot completion uses an atomic rename, - a manifest marker, or an external store-level operation. The generic - storage layer should expose finalization but not invent store-level - lifecycle policy. - => use an atomic rename, but don't hide the file. use a .part extension to the new directory name, and then remove it. make sure to flush and close all files before renaming. - -Until these decisions are made, Phases 0–4 can complete and the shared -implementation can be proven with `radix::tile::Id` in memory and with -temporary-directory tests. No provisional 2D disk format should escape into -production data. +The review resolved the layout, key, index-content, checksum, and publication +mechanism questions. One detail remains before Phase 5. + +### Resolved decisions + +1. **Index filename and location:** each snapshot stores + `raster_store.index` directly at its root. +2. **Node and payload paths:** `store_layout` determines the extensionless + node path. The default 2D mapping produces `//` directly below + the snapshot; there is no fixed `chunks/` directory. The AMORT codec + expands it to `//.amort`. +3. **Layout identity and coordinates:** the stable layout ID serialized in the + index is `zoom/x/y_google`. It uses Google/XYZ coordinates with the origin + at the north-west. +4. **Persistent key and zoom range:** use `radix::tile::Id`, whose x and y + coordinates must fit in `uint32_t`. The maximum zoom is therefore + `std::numeric_limits::digits` (32): at that zoom every + `uint32_t` x/y value is representable and valid. A maximum-zoom tile is + terminal because its children would require another coordinate bit. + Validation must special-case this boundary rather than evaluate an + overflowing `uint32_t{1} << 32`. The versioned disk DTO uses fixed-width + `uint32_t` fields for zoom, x, and y rather than serializing the platform + `unsigned` type directly. +5. **Version-1 index contents:** store the layout ID and sparse key/status + entries. `Leaf`, `Inner`, and `Virtual` are serialized; `Missing` is + represented by absence. Do not store derived aggregate metadata until a + concrete query demonstrates that it is required. Serialize entries in + lexicographic `(zoom, x, y)` order and reject duplicate keys while reading. +6. **Serialization envelope:** version 1 uses a fixed, file-type-specific + 64-bit magic value generated once during implementation, a 32-bit version, + zlib CRC-32 stored as `uint32_t` and computed over the compressed payload, + a compression enum, and the compressed payload. Use zstd with its + best-compression setting. The exact magic value is an implementation + constant, not a further design decision. +7. **Publication:** build a snapshot in a sibling directory named + `.part`. Write the index last, validate the completed snapshot, + flush and close every file, and atomically rename the directory to + `` on the same filesystem. The destination must not already + exist. A `.part` directory is never considered published; the rename + removes the suffix without a separate marker or manifest. This guarantees + atomic visibility to concurrent readers during normal operation. +8. **Crash behavior:** the store does not guarantee durability or safe + recovery across a power failure, operating-system crash, or storage + failure. Do not add `fsync()`, `fdatasync()`, `FlushFileBuffers()`, or + equivalent platform-specific synchronization to this refactor. After such + a failure, either a `.part` directory or a final snapshot may be unusable + and must be validated and rebuilt. Flushing and closing files remains + required before the rename for normal-operation correctness, but must not + be documented as a crash-durability guarantee. + +There are no remaining decisions blocking Phase 5. diff --git a/docs/raster-store/storage-format.md b/docs/raster-store/storage-format.md index c476517d..54832f91 100644 --- a/docs/raster-store/storage-format.md +++ b/docs/raster-store/storage-format.md @@ -31,6 +31,9 @@ this is the binary format used to store rf and tb tiles. both share the same bas - the source attribution index is stored as uint16, and indexes into a global source attribution table (see above) - data is stored in radix::Raster objects (one for source attribution index, one for the actual data), i.e. template struct raster_store::Tile { radix::Raster data; radix::Raster source_attribution; }; - the file ending is .amort (AlpineMapsOrg raster tile), it is serialised used the principles outlined below. +- each snapshot stores its index as `raster_store.index`. +- the default `zoom/x/y_google` layout stores a tile as + `//.amort`, directly below the snapshot root. ## raster-fundamentalis (rf) format raster-fundamentalis is our authoritative raster-store, containing only the data and no overviews / downsampled version. @@ -51,18 +54,19 @@ tile-base is a hierarchy build from raster-fundamentalis, containing all data an ### unclear - how to build tile-base from raster-fundamentalis - details of the tile server -- file names / index file name ## serialization / deserialization and versioning - all files are serialised with zpp::bits in two levels - first level contains: - an uint64 long file type specific random magic number, generated once at coding time, as an definitive file type identifier - a version number (uint32) - - a checksum for the payload, computed from the compressed data - - an enum for the compression algorithm + - an enum for the checksum, default to HandledByCompressionLib, other option None + - a string checksum for the payload, computed from the uncompressed data (default empty) + - an enum for the compression algorithm, default to to ZStd_BestCompression, other option None. - a payload (byte vector), the second level - the second level is a compressed byte array. the compressed payload is deserialised directly into the respective versioned data classes (tile or source attribution table) -- data structs are stored in versioned namespaces, e.g.: raster::fundamentalis::v1::Tile +- data structs are stored in versioned namespaces, e.g.: + `raster_store::v1::Tile` - outside the versioned namespace, there is a using declaration for the newest version - outside the versioned namespace, there is serialization function, taking only the newest version - outside the versioned namespace, there is a deserialization function, taking a byte stream, and returning the newest version (convert to the newest version, if the payload encodes an older version) @@ -73,7 +77,8 @@ tile-base is a hierarchy build from raster-fundamentalis, containing all data an - if the compression algorithm is missing or unsupported. - deserialization fails - we fail by throwing in these cases -- compression: use libzstd with best compression. libzstd must be imported via the projects cmake install facility from https://github.com/AlpineMapsOrgDependencies/zstd +- compression: use libzstd with best compression. libzstd must be imported via the projects cmake install facility from https://github.com/AlpineMapsOrgDependencies/zstd. the context must be configured to compute and check a checksum. do compression and decompression in functions with an std::byte interface (and error handling). dispatch is done via a simple switch case. +- checksum: assert the enum is either None or HandledByCompressionLib. ## to be defined From cfc4e13e853b82b9ff79dcad32faff9d9cfc55db Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Mon, 27 Jul 2026 21:54:39 +0200 Subject: [PATCH 25/61] Clarify refactor plan error handling --- docs/raster-store/refactor-plan.md | 41 +++++++++++++++++------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/docs/raster-store/refactor-plan.md b/docs/raster-store/refactor-plan.md index 31da1ef5..51792bbd 100644 --- a/docs/raster-store/refactor-plan.md +++ b/docs/raster-store/refactor-plan.md @@ -27,7 +27,10 @@ This plan describes how to extract the existing octree-specific index, traversal - A configured codec owns all filename endings and maps one `NodePath` to one or more physical files. `Codec::paths()` must not need a payload. - Codecs are stateful runtime objects behind a small interface. They may - support reading, writing, or both; an unsupported operation throws. reading and writing must be reentrant (callable concurrently from different threads). + support reading, writing, or both. Reading and writing return + `std::expected`; unsupported operations and other operational failures are + reported as error values. Reading and writing must be reentrant (callable + concurrently from different threads). - `copy_from()` hard-links every file when the input and output codecs return the same path list for a common dummy `NodePath`. Otherwise it decodes with the input codec and encodes with the output codec. Callers can force @@ -298,21 +301,26 @@ public: virtual std::vector paths(const NodePath& node_path) const = 0; - virtual NodeData read(const NodePath&) const { - throw UnsupportedCodecOperation{"read"}; + virtual std::expected + read(const NodePath&) const { + return std::unexpected( + CodecError::unsupported_operation("read")); } - virtual void write( + virtual std::expected write( const NodePath&, const NodeData&) const { - throw UnsupportedCodecOperation{"write"}; + return std::unexpected( + CodecError::unsupported_operation("write")); } }; ``` Concrete codecs contain their configuration and are constructed before -storage use. Unsupported read or write operations may use the base -implementation and throw at runtime. +storage use. `CodecError` is a payload-neutral operational error that records +the failed operation, an error category, and a diagnostic message. Concrete +codecs convert their domain errors to it. Unsupported read or write operations +may use the base implementation and return its `UnsupportedOperation` error. `Codec::paths()` has the following contract: @@ -475,8 +483,8 @@ order, and filename endings. If linking several files fails partway through, remove the target links created by that call before returning the error. There is no silent copy -fallback. An unsupported read or write needed for re-encoding throws through -the codec interface. +fallback. An unsupported read or write needed for re-encoding is returned +through `CopyError`, retaining the underlying `CodecError`. Codec settings that do not change `paths()`, such as compression level or JPEG quality, do not force re-encoding by default. A caller that needs those @@ -601,11 +609,11 @@ handle these failures. The shared copier must return the failure through `copy_subtree()` and the merge call chain until the application boundary can report it with the -affected key and path. An unsupported codec read or write may propagate as -the codec's runtime exception. Assertions remain appropriate for internal -invariants, but filesystem conditions, unsupported conversions, malformed -datasets, and overwrite conflicts are operational errors rather than -assertion failures. +affected key and path. Codec, filesystem, unsupported-conversion, malformed +dataset, and overwrite failures are propagated with `std::expected`; +`CopyError` retains any underlying `CodecError`. Assertions remain appropriate +for internal invariants, but operational failures are not assertion failures +or intentionally thrown exceptions. ### Paired hierarchy walking @@ -902,16 +910,13 @@ No formatting-only pass or unrelated refactor belongs in these commits. | Linked snapshots are modified in place | Immutable snapshot API and overwrite-disabled output | | Multi-file hard linking fails partway through | Remove links created by the failed `copy_from()` before returning | | Incompatible codecs return the same path list | Treat path-list equality as a codec contract and test every concrete codec pairing | -| Output-only codec is selected for required input | Throw a clear unsupported-operation error at runtime | +| Output-only codec is selected for required input | Return a clear `UnsupportedOperation` error | | Hard-link failure appears late | 2D operation preflight and explicit errors | | Shared code accumulates mesh/raster policy | Dependency tests/review against the source boundary | | Generic index accidentally dictates both disk formats | Separate 3D and 2D format adapters | ## Decisions required before Phase 5 -The review resolved the layout, key, index-content, checksum, and publication -mechanism questions. One detail remains before Phase 5. - ### Resolved decisions 1. **Index filename and location:** each snapshot stores From 060ca21c455c99cca3de7b469d19540ba81c37cb Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:05:19 +0200 Subject: [PATCH 26/61] Document DAG storage compatibility --- docs/raster-store/refactor-plan.md | 150 ++++++++++++++++++++++------- 1 file changed, 114 insertions(+), 36 deletions(-) diff --git a/docs/raster-store/refactor-plan.md b/docs/raster-store/refactor-plan.md index 51792bbd..fa3c2944 100644 --- a/docs/raster-store/refactor-plan.md +++ b/docs/raster-store/refactor-plan.md @@ -15,6 +15,9 @@ This plan describes how to extract the existing octree-specific index, traversal `raster_store` namespace. - Existing 3D Structura Fundamentalis datasets must remain readable and writable without changing their on-disk contract. +- Existing DAG datasets stored in the 3D hierarchy with `.bin` payloads must + also remain readable and writable without changing their index or payload + serialization. - 3D compatibility includes both existing path layouts: `flat` and `level_and_coordinate_directories`. - The 2D raster tile format is a separate format. Requirements in @@ -31,6 +34,9 @@ This plan describes how to extract the existing octree-specific index, traversal `std::expected`; unsupported operations and other operational failures are reported as error values. Reading and writing must be reentrant (callable concurrently from different threads). +- Legacy index metadata selects a codec through an explicit, caller-supplied + resolver. The octree format adapter does not contain a global codec registry + or depend on mesh or DAG payload types. - `copy_from()` hard-links every file when the input and output codecs return the same path list for a common dummy `NodePath`. Otherwise it decodes with the input codec and encodes with the output codec. Callers can force @@ -69,6 +75,7 @@ This plan describes how to extract the existing octree-specific index, traversal - Changing the existing 3D hard-link policy by adding a silent file-copy fallback. - Refactoring unrelated octree, DAG, mesh, or tile-builder code. +- Changing the serialized schema of existing DAG `.bin` payloads. ## Compatibility contract @@ -86,7 +93,9 @@ Before moving code, tests must lock down the following 3D behaviour: | Coordinate path | `///` | | Default layout | existing level/coordinate layout | | Layout detection | both existing layouts remain detectable | -| Codec selection | legacy preferred extension selects terrain or configured glTF codec | +| Mesh codec selection | legacy preferred extension selects terrain or configured glTF codec | +| DAG codec selection | legacy `.bin` preferred extension selects the ZPP Bits codec | +| DAG payload encoding | existing `dag::ClusterBatch` ZPP Bits serialization | | Equal codec path lists | hard-link every file, or report an explicit error | | Different codec path lists | decode with input codec and encode with output codec | @@ -122,6 +131,8 @@ src/terrainlib/ │ │ ├── Interface.h │ │ ├── Dummy.h │ │ └── Lru.h +│ ├── codec/ +│ │ └── ZppBits.h │ └── merge/ │ ├── Action.h │ └── walk.h @@ -154,6 +165,8 @@ The exact file grouping may be collapsed if a file would only contain a few lines. The important boundaries are: - `store` contains dimension- and payload-neutral mechanisms; +- `store::codec::ZppBits` is the reusable concrete codec for payload types + that provide ZPP Bits serialization; - `mesh::codec` contains the separately configured terrain and glTF codecs; - `octree` contains the 3D format and key adapters; - `raster_store` contains the new 2D format, key adapters, and raster codecs; @@ -163,6 +176,12 @@ lines. The important boundaries are: Temporary forwarding headers and aliases under `octree` are allowed during migration. They must not contain a second implementation. +DAG serialization remains owned by `dag_builder`. Consolidate the existing +`dag::Id`, `dag::ClusterBatch`, `Clustering`, `Cluster`, and `TextureSet` +serialization functions in `src/dag_builder/serialization.h`. The DAG storage +adapter includes that header explicitly so template instantiation does not +depend on callers including `encoded.h` in the correct order. + ## Shared interfaces The names below are the intended shape, not signatures that must be copied @@ -357,6 +376,19 @@ The mesh side has separate terrain and glTF codecs because they use different format implementations. Binary `.glb` and JSON `.gltf` remain configurations of one glTF codec because both use the same `cgltf` implementation. +The shared module also provides: + +```cpp +template +struct store::codec::ZppBits : store::Codec { .. }; +``` + +It uses the existing `io::read_from_path()` and `io::write_to_path()` +functions, maps one node to `.bin`, and converts `io::Error` to +`CodecError`. It contains no DAG-specific serialization logic. DAG payload +serialization remains in `dag_builder/serialization.h`, and its field order +and meshoptimizer/JPEG encoding remain unchanged. + The raster side has similarly shaped codecs specialized on PixelType: ```cpp @@ -382,8 +414,9 @@ store::IndexedStorage store::cache::Interface ``` -The NodeData codecs remain with their domains under `mesh::codec` and -`raster_store::codec`. +Domain-specific NodeData codecs remain under `mesh::codec` and +`raster_store::codec`. The reusable ZPP Bits codec remains under +`store::codec`. Index serialization is not a responsibility of `store::Index`. Opening and saving a dataset receives a dimension-specific format adapter which provides: @@ -399,18 +432,33 @@ pointers. Choose the smaller implementation after the Phase 0 tests exist. It must not reintroduce a layout class hierarchy or global registration. For 3D, the adapter reads and writes the current `octree` index DTO unchanged. -Its legacy `preferred_extension` field selects the configured codec: +When opening indexed storage or discovering a legacy unindexed directory, it +passes the legacy `preferred_extension` to a caller-supplied codec resolver. +The resolver is an ordinary callable and returns +`std::expected>, CodecError>`. It is not +a global registry. + +The payload domains provide ordinary resolver functions: ```text -.terrain -> terrain codec -.glb -> glTF codec with binary container -.gltf -> glTF codec with JSON container +mesh::codec::from_extension + .terrain -> terrain codec + .glb -> glTF codec with binary container + .gltf -> glTF codec with JSON container + +dag::codec::from_extension + .bin -> store::codec::ZppBits ``` +An unknown extension returns an explicit `UnsupportedCodec` error. Opening a +new empty store may receive an already constructed codec instead of a resolver. +Convenience functions in `src/dag_builder/storage.h` supply the DAG resolver so +application call sites do not handle it directly. + Legacy unindexed-directory discovery remains in the 3D adapter: it recognizes -the known codec endings, removes them to obtain a `NodePath`, and then invokes -the selected layout parser. The generic layout does not recover keys directly -from codec-owned file paths. +candidate endings by asking the supplied resolver, removes an accepted ending +to obtain a `NodePath`, and then invokes the selected layout parser. The +generic layout does not recover keys directly from codec-owned file paths. For 2D, the adapter reads and writes `raster_store.index` as a separately versioned `raster_store::v1` DTO using the serialization envelope required by @@ -660,15 +708,19 @@ No production behaviour changes. 1. Add golden 3D fixtures created by the current code: - one `terrain.index` using `flat`; - one using `level_and_coordinate_directories`; + - one DAG dataset whose index selects `.bin` and whose payload contains a + valid serialized `dag::ClusterBatch`; - physical payload paths for a root, child, and deeper descendant; and - an index containing `Leaf`, `Virtual`, and `Inner`. -2. Test that both fixtures open, resolve the expected IDs and extensions, and +2. Test that all fixtures open, resolve the expected IDs and extensions, and traverse the expected sparse nodes. 3. Add path round-trip tests for boundary IDs and both layouts. 4. Add storage tests for: - matching-extension hard links; - different-extension decode/re-encode; - - `.terrain`, `.glb`, and `.gltf` dispatch; + - `.terrain`, `.glb`, `.gltf`, and `.bin` dispatch through the appropriate + domain resolver; + - explicit failure for an unknown preferred extension; - overwrite rejection; - indexed and unindexed opens; and - final index creation by directory scan. @@ -677,7 +729,8 @@ No production behaviour changes. Exit criterion: the compatibility tests pass against the untouched implementation and fail when any stable filename, layout ID, path encoding, -status value, or index field order is deliberately changed. +status value, index field order, or DAG payload serialization is deliberately +changed. ### Phase 1 — Extract topology into `store` @@ -713,8 +766,8 @@ callers still build through aliases; no filesystem code has changed. 8. Delete the old strategy base class, registration machinery, and concrete strategy classes once no call site uses them. -Exit criterion: the legacy 3D adapter plus codec resolves both fixtures to -identical physical payload paths; generic `Layout` contains no extension; +Exit criterion: the legacy 3D adapter plus codecs resolve all Phase 0 fixtures +to identical physical payload paths; generic `Layout` contains no extension; there is no layout inheritance, RTTI lookup, static registrar, or owning strategy pointer. @@ -722,35 +775,48 @@ strategy pointer. 1. Add the stateful `store::Codec` interface with `paths()`, `read()`, and `write()`. -2. Split the current extension-dispatching `octree::MeshCodec` into: +2. Move the current generic `octree::ZppBitsCodec` to the runtime + `store::codec::ZppBits`, preserving its `.bin` paths and serialized + payload bytes. +3. Consolidate the DAG serialization functions in + `src/dag_builder/serialization.h` without changing their serialized field + order, meshoptimizer encoding, or JPEG texture encoding. +4. Split the current extension-dispatching `octree::MeshCodec` into: - a terrain codec; and - one glTF codec configured for binary `.glb` or JSON `.gltf`. -3. Move copy error, raw storage, caches, logical storage, and indexed storage +5. Move copy error, raw storage, caches, logical storage, and indexed storage into `store`. -4. Make storage own a configured `std::unique_ptr>`; remove the +6. Make storage own a configured `std::unique_ptr>`; remove the codec template parameter from storage. -5. Replace every embedded `octree::Id` with `Traits::Key`. -6. Make every raw file operation obtain its complete file list through +7. Replace every embedded `octree::Id` with `Traits::Key`. +8. Make every raw file operation obtain its complete file list through `Codec::paths()`. `has()` requires every listed file, and `remove()` removes every listed file. -7. Keep payload codecs outside the shared module under `mesh::codec` and - `raster_store::codec`. -8. Split generic index maintenance from 3D index serialization and legacy +9. Keep domain-specific payload codecs outside the shared module under + `mesh::codec` and `raster_store::codec`. +10. Split generic index maintenance from 3D index serialization and legacy folder discovery. -9. Keep the current 3D `terrain.index` DTO and open functions as compatibility - adapters over the shared storage. Map its preferred extension to a - configured terrain or glTF codec. -10. Migrate the existing octree storage aliases and all application callers. -11. Preserve the current 3D destructor-save behaviour until all callers have +11. Keep the current 3D `terrain.index` DTO and open functions as compatibility + adapters over the shared storage. Resolve its preferred extension through + the caller-supplied mesh or DAG resolver. +12. Add DAG storage convenience functions that supply + `dag::codec::from_extension`, and migrate `dag_builder` and + `dag_convert_debug` to them. +13. Migrate the existing octree storage aliases and all other application + callers. +14. Preserve the current 3D destructor-save behaviour until all callers have explicit index finalization. Add focused codec tests using single-file, multi-file, read/write, and write-only test codecs before depending on the raster payload implementation. +Test that a pre-refactor DAG fixture opens through the new resolver and that a +new deterministic `.bin` payload matches the Phase 0 golden bytes and remains +readable through the unchanged ZPP serialization functions. Exit criterion: all existing applications build and all Phase 0 fixtures pass -through the shared runtime codec and storage implementation. No -extension-dispatching mesh codec or second storage implementation remains -under `octree`. +through the shared runtime codec and storage implementation. Existing DAG +payload bytes and `.bin` paths remain compatible. No extension-dispatching +mesh codec or second storage implementation remains under `octree`. ### Phase 4 — Generalize subtree reuse and paired walking @@ -846,8 +912,8 @@ Exit criterion: repository search finds no generic implementation tied to ## Test and verification plan -Tests should live in the existing `unittests_terrainlib` target. Suggested -files: +Generic store tests should live in the existing `unittests_terrainlib` target. +Suggested files: ```text unittests/terrainlib/store_index.cpp @@ -859,16 +925,22 @@ unittests/terrainlib/store_compatibility.cpp unittests/terrainlib/store_merge_walk.cpp ``` +The DAG payload-compatibility fixture and resolver integration test belong in +`unittests_dagbuilder`, because `terrainlib` must not depend on +`dag::ClusterBatch` or its serializers. + During implementation: 1. Build in `$source_dir/build/$config_name`. 2. Run unit tests from that build directory. 3. Run the focused store tests after each edit. 4. Run the full `unittests_terrainlib` target at every phase boundary. -5. Build `sf_builder`, `sf_merger`, `sf_index_browser`, `dag_builder`, and +5. Run `unittests_dagbuilder` after the ZPP codec, DAG serialization header, or + DAG resolver changes. +6. Build `sf_builder`, `sf_merger`, `sf_index_browser`, `dag_builder`, and `dag_convert_debug` after their storage aliases move. -6. Run any existing merger integration fixture after Phase 4. -7. Inspect `git diff --check` and the final worktree before each commit. +7. Run any existing merger integration fixture after Phase 4. +8. Inspect `git diff --check` and the final worktree before each commit. No formatting-only pass or unrelated refactor belongs in these commits. @@ -888,12 +960,15 @@ No formatting-only pass or unrelated refactor belongs in these commits. | `strategy/LevelAndCoordinateDirectories.h` | `octree/store_layout/LevelAndCoordinateDirectories.h` | | `octree/storage/cache/*` | `store/cache/*` | | `octree/storage/codec/Codec.h` | runtime `store/Codec.h` | +| `octree/storage/codec/DefaultCodec.h` | runtime `store/codec/ZppBits.h` | | `octree/storage/codec/MeshCodec.h` | `mesh/codec/Terrain.h` and configured `mesh/codec/Gltf.h` | | `octree/storage/RawStorage.h` | `store/RawStorage.h` | | `octree/storage/Storage.h` | `store/Storage.h` | | `octree/storage/IndexedStorage.h` | `store/IndexedStorage.h` | | `octree/storage/helpers.*` | generic scan helpers plus 3D format adapter | | `octree/disk/IndexFile.h` | versioned 3D format adapter under `octree` | +| DAG serializers in `dag_node.h` and `encoded.h` | `dag_builder/serialization.h` | +| `dag_builder/storage.h` aliases | DAG storage aliases plus codec resolver convenience functions | | `sf_merger::NodeWriter` subtree loop | `store/copy_subtree.h` | | `sf_merger::Merger` recursion | `store/merge/walk.h` | @@ -902,6 +977,9 @@ No formatting-only pass or unrelated refactor belongs in these commits. | Risk | Control | |---|---| | Existing indexes stop loading | Golden pre-refactor fixtures and unchanged 3D DTO | +| Existing DAG `.bin` datasets stop loading | Golden DAG fixture, unchanged serializers, and explicit `.bin` resolver | +| Octree format adapter gains DAG dependencies | Caller-supplied resolver owned by `dag_builder` | +| Unknown legacy extension silently selects the wrong codec | Return an explicit `UnsupportedCodec` error | | Valid legacy paths are parsed differently | Characterization and round-trip tests before replacement | | Template migration creates a large unreviewable diff | Compatibility aliases and phase-by-phase caller migration | | `radix::tile::Id` root underflows | Traits intercept root parent lookup | From fb918f0ac25ca2722774447a58586fde197f8fdb Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Wed, 29 Jul 2026 22:19:39 +0200 Subject: [PATCH 27/61] Refine raster store refactor scope --- docs/raster-store/README.md | 6 +- docs/raster-store/architecture.md | 35 +- docs/raster-store/refactor-plan.md | 635 +++++++++++------------------ docs/raster-store/rf_builder.md | 202 +++++++++ docs/raster-store/rf_merger.md | 231 +++++++++++ 5 files changed, 707 insertions(+), 402 deletions(-) create mode 100644 docs/raster-store/rf_builder.md create mode 100644 docs/raster-store/rf_merger.md diff --git a/docs/raster-store/README.md b/docs/raster-store/README.md index e757e417..54006330 100644 --- a/docs/raster-store/README.md +++ b/docs/raster-store/README.md @@ -11,7 +11,11 @@ baseline, not a finalized binary-format specification. - [Architecture](architecture.md) - [Storage format](storage-format.md) - [Sampling and pyramid generation](sampling-and-generation.md) +- [DRAFT rf_builder idea notes](rf_builder.md) +- [DRAFT rf_merger idea notes](rf_merger.md) ## Scope -The documents mostly hold format information, and no tool descriptions. We will be writing tools that query and transform these formats. Tool descriptions will be given later. +The documents mostly hold format information. The `rf_builder` and +`rf_merger` documents are explicitly non-authoritative idea parking lots, not +tool specifications or implementation plans. diff --git a/docs/raster-store/architecture.md b/docs/raster-store/architecture.md index 10a62418..204c7ac7 100644 --- a/docs/raster-store/architecture.md +++ b/docs/raster-store/architecture.md @@ -99,21 +99,34 @@ Because hard links share inodes, a linked container must never be opened for in-place modification. Existing snapshots are considered immutable. Obsolete snapshots can be deleted, the data will be preserved if necessary due to reference counting in the inodes. When merging, we need to create new hardlinks for unchanged rf tiles (taken completely from either snapshot), and we need to create new rf tiles if the new tile shares information from both. -Cross-filesystem hard links cannot be created. The builder must expose this as -a clear configuration error rather than discovering it after a long build. +### Publication A new snapshot is assembled in a sibling directory named -`.part`. The builder writes the index last, validates the -snapshot, flushes and closes all files, and atomically renames the directory -to `` on the same filesystem. The final destination must not -already exist. A `.part` directory is incomplete and must not be exposed as a -published snapshot during normal operation. +`.part`. Publication follows this protocol: + +1. Write all payload and metadata files into the `.part` directory. +2. Write the index last and validate the completed snapshot. +3. Flush and close every file. +4. Atomically rename the directory to `` on the same filesystem. + +The final destination must not already exist. A `.part` directory is +incomplete and is never considered published. The rename removes the suffix; +there is no separate marker or manifest. During normal operation this gives +readers atomic visibility: they see either no final snapshot or the completed +one. + +Cross-filesystem publication is unsupported because the final rename and any +hard links must remain on one filesystem. A builder or merger must reject that +configuration before starting a long operation. Publication does not guarantee durability or safe recovery across a power -failure, operating-system crash, or storage failure. The implementation does -not perform platform-specific filesystem syncing. After such a failure, -either a `.part` directory or a final snapshot may be unusable and must be -validated and rebuilt. +failure, operating-system crash, or storage failure. Flushing and closing +files before the rename is required for normal-operation correctness, but is +not a crash-durability guarantee. The implementation does not require +`fsync()`, `fdatasync()`, `FlushFileBuffers()`, or equivalent +platform-specific synchronization. After such a failure, either a `.part` +directory or a final snapshot may be unusable and must be validated and +rebuilt. ## Pyramid generator interface (to be confirmed, LLM, do not use the following without consultation) diff --git a/docs/raster-store/refactor-plan.md b/docs/raster-store/refactor-plan.md index fa3c2944..a6689376 100644 --- a/docs/raster-store/refactor-plan.md +++ b/docs/raster-store/refactor-plan.md @@ -5,24 +5,39 @@ record of completed work. ## Purpose -This plan describes how to extract the existing octree-specific index, traversal, storage, codec, and subtree-reuse code into a shared 2D/3D store. The refactor must preserve existing 3D datasets while enabling raster-fundamentalis storage without duplicating infrastructure. +This plan describes how to extract the existing octree-specific index, +traversal, storage, and codec code into a shared 2D/3D store. The refactor +must preserve existing 3D datasets and prove, using test-only mappings and +codecs where necessary, that the shared mechanisms work with a 2D key. +Persistent raster-fundamentalis formats, adapters, and tools are later work. ## Decisions already made - The shared implementation will live in `src/terrainlib/store` and use the `store` namespace. -- The 2D implementation will live in `src/terrainlib/raster_store` and use the - `raster_store` namespace. +- The minimal 2D traits adapter used to exercise the shared hierarchy will + live in `src/terrainlib/raster_store` and use the `raster_store` namespace. +- Common Structura Fundamentalis validation and errors will live in + `src/terrainlib/sf` and use the `sf` namespace. - Existing 3D Structura Fundamentalis datasets must remain readable and writable without changing their on-disk contract. - Existing DAG datasets stored in the 3D hierarchy with `.bin` payloads must also remain readable and writable without changing their index or payload serialization. +- `Inner` is a valid shared topology state and is supported by DAG, + raster-fundamentalis, and tile-base datasets. It is not valid in Structura + Fundamentalis datasets. +- SF producer and processing boundaries validate indexed SF data and return a + typed `sf::InvalidTopology` error containing an offending key when `Inner` + is present. The diagnostic `sf_index_browser` is exempt. This is an SF data + invariant, not a generic store or octree-format rule. - 3D compatibility includes both existing path layouts: `flat` and `level_and_coordinate_directories`. -- The 2D raster tile format is a separate format. Requirements in - this directory apply to that 2D format and must not be retrofitted onto - existing 3D datasets. The storage format for 2d tiles is defined in storage-format.md +- A persistent 2D raster tile format is not defined by this refactor. + [architecture.md](architecture.md) and + [storage-format.md](storage-format.md) describe intended direction and + provisional requirements that will be finalized in later RF work. They must + not be retrofitted onto existing 3D datasets. - A path layout strategy should contain a stable identifier and two operations: key to extensionless `NodePath`, and `NodePath` to key. It should not require an inheritance hierarchy, RTTI, global @@ -39,8 +54,7 @@ This plan describes how to extract the existing octree-specific index, traversal or depend on mesh or DAG payload types. - `copy_from()` hard-links every file when the input and output codecs return the same path list for a common dummy `NodePath`. Otherwise it decodes with - the input codec and encodes with the output codec. Callers can force - re-encoding. + the input codec and encodes with the output codec. - 3D geometry, ECEF bounds, mesh codecs, mesh reconstruction, mask geometry, and raster-specific processing remain outside the shared store. @@ -48,18 +62,17 @@ This plan describes how to extract the existing octree-specific index, traversal 1. Use one sparse hierarchy implementation for `octree::Id` and `radix::tile::Id`. -2. Use one traversal, storage, cache, runtime codec boundary, and - unchanged-subtree copier for 2D and 3D. -3. Make paired-tree walking reusable without putting mesh or raster merge - policy into the shared layer. -4. Replace the current layout-strategy class hierarchy with small path-mapping +2. Make traversal, storage, cache, and the runtime codec boundary + dimension-neutral while keeping production format adapters 3D-only in this + refactor. +3. Replace the current layout-strategy class hierarchy with small path-mapping values backed by function pairs. -5. Allow one logical node payload to consist of multiple files without making +4. Allow one logical node payload to consist of multiple files without making layouts aware of those files. -6. Preserve all valid existing 3D index files and payload paths. -7. Introduce the 2D storage adapter without inventing unspecified raster file - details. -8. Land the refactor in small, testable steps. Every phase should build and +5. Preserve all valid existing 3D index files and payload paths. +6. Prove the shared topology and traversal with `radix::tile::Id` without + defining a persistent 2D adapter or format. +7. Land the refactor in small, testable steps. Every phase should build and pass tests before the next phase begins. ## Non-goals @@ -68,10 +81,18 @@ This plan describes how to extract the existing octree-specific index, traversal other 3D spatial calculations. - Defining or implementing GDAL ingestion, raster resampling, filtering, source selection, or mask rasterisation. -- Defining the `.amort` payload or source-attribution-table serialization - beyond the requirements already in [storage-format.md](storage-format.md). -- Implementing an `rf_builder`, `rf_merger`, tile-base generator, or tile - server in this refactor. +- Defining or implementing a raster-fundamentalis index, payload format, + layout ID, codec, publication lifecycle, or persistent storage adapter. +- Implementing an `rf_builder`, `rf_merger`, tile-base generator, tile server, + snapshot-reuse operation, or other RF tool in this refactor. +- Defining the raster-fundamentalis merge policy or the final paired-hierarchy + walker/action algebra. That work is deferred until `rf_merger` requirements + are defined. +- Defining a shared subtree-copy abstraction, `Inner` subtree-copy behaviour, + or forced re-encoding policy for RF. Those decisions are deferred until + `rf_merger`. +- Adding merge semantics for `Inner` nodes in SF. Such nodes are invalid SF + input and must be rejected before merge dispatch. - Changing the existing 3D hard-link policy by adding a silent file-copy fallback. - Refactoring unrelated octree, DAG, mesh, or tile-builder code. @@ -87,6 +108,8 @@ Before moving code, tests must lock down the following 3D behaviour: | Index field order | layout ID, preferred extension, index map | | Node-key encoding | existing `octree::Id` level/index serialization | | Node-status encoding | `Leaf = 0`, `Inner = 1`, `Virtual = 2` | +| Valid SF statuses | `Leaf` and `Virtual`; reject `Inner` with `sf::InvalidTopology` | +| Valid DAG statuses | `Leaf`, `Inner`, and `Virtual` | | Flat layout ID | `flat` | | Flat path | `-` | | Coordinate layout ID | `level_and_coordinate_directories` | @@ -126,20 +149,19 @@ src/terrainlib/ │ ├── RawStorage.h │ ├── Storage.h │ ├── IndexedStorage.h -│ ├── copy_subtree.h │ ├── cache/ │ │ ├── Interface.h │ │ ├── Dummy.h │ │ └── Lru.h -│ ├── codec/ -│ │ └── ZppBits.h -│ └── merge/ -│ ├── Action.h -│ └── walk.h +│ └── codec/ +│ └── ZppBits.h ├── mesh/ │ └── codec/ │ ├── Terrain.h │ └── Gltf.h +├── sf/ +│ ├── InvalidTopology.h +│ └── validate_index.h ├── octree/ │ ├── Id.h │ ├── StoreTraits.h @@ -151,14 +173,7 @@ src/terrainlib/ │ ├── IndexFile.h │ └── open.h └── raster_store/ - ├── StoreTraits.h - ├── IndexFile.h - ├── Storage.h - ├── codec/ - │ ├── Amort.h - │ └── Debug.h - └── store_layout/ - └── ZoomXYGoogle.h + └── StoreTraits.h ``` The exact file grouping may be collapsed if a file would only contain a few @@ -168,11 +183,25 @@ lines. The important boundaries are: - `store::codec::ZppBits` is the reusable concrete codec for payload types that provide ZPP Bits serialization; - `mesh::codec` contains the separately configured terrain and glTF codecs; +- `sf` contains SF-specific topology validation and errors shared by + `sf_builder`, `sf_merger`, and `dag_builder`; - `octree` contains the 3D format and key adapters; -- `raster_store` contains the new 2D format, key adapters, and raster codecs; - and +- `raster_store` contains only the minimal 2D hierarchy traits adapter in this + refactor; and - subdirectory names match their namespaces where a subnamespace is used. +Shared topology and octree-format adapters accept `Inner`. The SF restriction +is enforced by `sf::validate_index()` at SF producer/consumer boundaries and +reported as `sf::InvalidTopology`. It must not be embedded in `store::Index`, +traversal, or the generic 3D disk adapter. `sf_index_browser` is a diagnostic +tool and intentionally does not apply SF validation, so it can display invalid +trees including `Inner`. + +`sf::validate_index()` returns +`std::expected`. SF application-level error types +must retain this error and `CopyError` when propagating failures; neither is +reduced to a log message, assertion, or generic boolean. + Temporary forwarding headers and aliases under `octree` are allowed during migration. They must not contain a second implementation. @@ -225,9 +254,10 @@ changing `octree::Id`. by the `uint32_t` x/y coordinates; - validate the maximum zoom without evaluating an overflowing `uint32_t{1} << 32`; -- use `radix::tile::Id::Hasher`; and -- use the Google/XYZ convention, with the origin at the north-west, at the - persistent boundary. +- use `radix::tile::Id::Hasher`. + +This traits adapter defines only hierarchy operations. It does not define +persistent coordinates, a path layout, or an RF disk format. The shared code must obtain roots, parents, children, validation, and hashing through the traits. It must not use dimension checks or specialize behaviour @@ -264,7 +294,6 @@ node. For example: ```text octree flat 12-123456 octree coordinates 12/34/56/78 -raster-store ZXY 12/2200/1400 ``` It does not necessarily name a physical file. Replace @@ -292,9 +321,6 @@ octree::store_layout::flat() octree::store_layout::level_and_coordinate_directories() octree::store_layout::from_id(id) octree::store_layout::all() - -raster_store::store_layout::zoom_x_y_google() -raster_store::store_layout::from_id(id) ``` This retains runtime selection from an index file while removing virtual @@ -366,10 +392,10 @@ glTF codec configured for JSON output 12/34/56/78 -> 12/34/56/78.gltf -Debug raster codec configured for JPEG data and PNG attribution - 12/2200/1400 - -> 12/2200/1400.data.jpg - -> 12/2200/1400.attribution.png +Multi-file test codec + 12/34/56/78 + -> 12/34/56/78.data + -> 12/34/56/78.metadata ``` The mesh side has separate terrain and glTF codecs because they use different @@ -389,19 +415,6 @@ functions, maps one node to `.bin`, and converts `io::Error` to serialization remains in `dag_builder/serialization.h`, and its field order and meshoptimizer/JPEG encoding remain unchanged. -The raster side has similarly shaped codecs specialized on PixelType: - -```cpp -template struct raster_store::codec::Amort : store::Codec> { .. }; -template struct raster_store::codec::Debug : store::Codec> { .. }; -``` - -`Amort` supports reading and writing. `Debug` supports writing only and may -contain runtime options for data format, attribution format, JPEG quality, or -similar debugging choices. It is not template-composed from separate image -codec types. - - ### Storage and format adapters Generalize storage over traits and NodeData. It owns a configured codec through @@ -414,9 +427,8 @@ store::IndexedStorage store::cache::Interface ``` -Domain-specific NodeData codecs remain under `mesh::codec` and -`raster_store::codec`. The reusable ZPP Bits codec remains under -`store::codec`. +Domain-specific mesh codecs remain under `mesh::codec`. The reusable ZPP Bits +codec remains under `store::codec`. RF codecs are deferred. Index serialization is not a responsibility of `store::Index`. Opening and saving a dataset receives a dimension-specific format adapter which provides: @@ -460,69 +472,49 @@ candidate endings by asking the supplied resolver, removes an accepted ending to obtain a `NodePath`, and then invokes the selected layout parser. The generic layout does not recover keys directly from codec-owned file paths. -For 2D, the adapter reads and writes `raster_store.index` as a separately -versioned `raster_store::v1` DTO using the serialization envelope required by -[storage-format.md](storage-format.md). Version 1 stores the layout ID and -sparse key/status entries. `Missing` is represented by absence; derivable -aggregate metadata is not stored. - Automatic dirty-index saving currently happens in the 3D storage destructor. Preserve that behaviour for existing 3D entry points during the migration. -The new 2D snapshot API should require an explicit finalization/publication -step; a destructor must not make an incomplete snapshot authoritative. -### Copying and unchanged-subtree reuse +### Copying one node and SF subtree reuse There are two separate responsibilities in the current implementation: -1. `sf_merger::NodeWriter` traverses a source subtree and decides which - indexed nodes to reuse. +1. `sf_merger::NodeWriter` traverses a source subtree and + `sf_merger::cut_leaf_node()` identifies an unchanged leaf. 2. `octree::Storage::copy_from()` delegates to `octree::RawStorage::copy_from()`, where `std::filesystem::create_hard_link()` performs the actual hard link and the target index is updated on success. The filesystem hard-link implementation is therefore already in terrainlib. -This refactor shall move the payload-neutral subtree traversal and copy -orchestration out of `sf_merger`. +Move the one-node storage operation into the shared store, but keep subtree +selection and traversal in `sf_merger`. There is no current DAG caller, and RF +subtree-copy requirements will be defined with `rf_merger`. -The current call chain is: +The migrated call chains are: ```text sf_merger decides to keep a source subtree unchanged -> NodeWriter traverses the source index - -> octree::Storage::copy_from() - -> octree::RawStorage::copy_from() - -> create_hard_link(), or decode/encode when formats differ -``` - -The target call chain becomes: + -> store::Storage::copy_from() + -> hard-link every codec path, or decode/encode -```text -merge policy decides to keep a source subtree unchanged - -> store::copy_subtree() +sf_merger determines that a cut leaf is unchanged + -> cut_leaf_node() -> store::Storage::copy_from() -> hard-link every codec path, or decode/encode ``` `Storage::copy_from()` remains the operation for copying one logical node. -Add: - -```cpp -struct CopyOptions { - bool force_reencode = false; -}; -``` - For one key, `copy_from()`: 1. calls the input and output `Codec::paths()` with the same fixed dummy `NodePath`; -2. when the lists are equal and `force_reencode` is false, calls both codecs - again with their actual source and target `NodePath` values and hard-links - every source path to the corresponding target path; -3. when the dummy lists differ or re-encoding is forced, reads the payload - with the input codec and writes it with the output codec; and +2. when the lists are equal, calls both codecs again with their actual source + and target `NodePath` values and hard-links every source path to the + corresponding target path; +3. when the dummy lists differ, reads the payload with the input codec and + writes it with the output codec; and 4. updates the target index only after all links or the write complete. The dummy path must be fixed and collision-free, for example @@ -534,166 +526,73 @@ created by that call before returning the error. There is no silent copy fallback. An unsupported read or write needed for re-encoding is returned through `CopyError`, retaining the underlying `CodecError`. -Codec settings that do not change `paths()`, such as compression level or -JPEG quality, do not force re-encoding by default. A caller that needs those -settings applied to every node passes `force_reencode = true`. - Hard-link rules: - never modify an existing linked payload in place; - a matching codec path list hard-links every file; - a different path list decodes with the input codec and encodes with the output codec; -- `force_reencode` always selects decode/encode; - hard-link failure is explicit; -- 2D snapshot tools preflight that source and destination support hard links - before a long operation starts; and - no silent file-copy fallback is introduced. -#### Shared subtree copier - -Move the payload-neutral traversal in -`sf_merger::NodeWriter::copy_subtree_to_output()` to a shared operation such -as: - -```cpp -std::expected -store::copy_subtree( - const IndexedStorage& source, - Storage& target, - const Key& root, - CopyOptions options = {}); -``` - -The shared operation: - -1. traverses an indexed source subtree; -2. skips `Virtual` nodes; -3. calls `copy_from()` for physical payloads in both `Leaf` and `Inner` - states; -4. continues traversal below `Inner` nodes; and -5. returns copy failures to its caller instead of converting them into an - assertion or immediate process termination. - -The operation is payload-neutral because it only interprets hierarchy status -and delegates each physical node to `Storage::copy_from()`. It does not know -about meshes, rasters, masks, attribution, or their encodings. +#### SF-local subtree traversal -#### `Leaf`, `Inner`, and `Virtual` +`sf_merger::NodeWriter::copy_subtree_to_output()` remains in `sf_merger`. It +uses shared traversal and `Storage::copy_from()`, but it is not promoted to a +generic store API during this refactor. -The sparse index distinguishes payload presence from descendant presence: +SF validation guarantees that its indexed inputs contain only `Leaf` and +`Virtual` nodes. The SF-local traversal skips `Virtual`, copies `Leaf`, and +does not define behaviour for `Inner`. An `Inner` node is rejected before +merge or cut processing with `sf::InvalidTopology` containing the offending +key. -| Status | Physical payload | Indexed descendants | -|---|---:|---:| -| `Leaf` | yes | no | -| `Inner` | yes | yes | -| `Virtual` | no | yes | - -`Missing` is represented by absence from the sparse index and is not visited -by subtree traversal. - -The current `NodeWriter` callback effectively does: - -```cpp -if (status == NodeStatus::Virtual) { - return; -} - -DEBUG_ASSERT(status == NodeStatus::Leaf); -DEBUG_ASSERT_VAL(target.copy_from(id, source)); -``` - -The `DEBUG_ASSERT(status == Leaf)` is an incorrect assumption. `Inner` is -also a physical state and its payload must be copied. The existing -`IndexMap::add()` creates an `Inner` node whenever a physical `Leaf` gains a -physical descendant. Such a hierarchy is valid in 3D and is explicitly -required for raster-fundamentalis, where a coarse physical tile may coexist -with more accurate descendants. - -For example: - -```text -zoom 10 physical tile -> Inner -└── zoom 11 physical tile -> Leaf -``` - -Reusing this subtree must preserve both payloads. Skipping the `Inner` -payload would lose the coarse fallback; asserting on `Inner` rejects a valid -hierarchy. - -The shared copier shall instead handle status as: - -```cpp -switch (status) { -case NodeStatus::Virtual: - break; -case NodeStatus::Leaf: -case NodeStatus::Inner: - target.copy_from(id, source, options); - break; -} -``` - -Traversal shall continue below the `Inner` node. Copying the parent first adds it -to the target as a `Leaf`; copying its descendant then promotes the parent to -`Inner`, reconstructing the source topology through the normal index -transitions. - -The existing paired merger has a related limitation: its dispatcher only -handles `Missing`, `Leaf`, and `Virtual` pairs and sends any pair containing -`Inner` to `UNREACHABLE()`. The paired hierarchy-walking work below must -handle or explicitly reject all 16 status combinations without treating -valid input data as an impossible program state. +When `rf_merger` is designed, it can initially compose `store::traverse` and +`Storage::copy_from()`. At that point, the SF and RF implementations provide +enough evidence to decide whether a shared subtree copier is useful and how it +must handle RF `Inner` nodes. #### Error propagation The lower storage layer already represents ordinary copy failures, including missing source files, directory creation failure, hard-link failure, decode failure, and encode failure. The current `NodeWriter` consumes -`Storage::copy_from()` with `DEBUG_ASSERT_VAL`, while some overwrite paths -terminate through `LOG_ERROR_AND_EXIT()`. Because -`copy_subtree_to_output()` returns `void`, the merge caller cannot report or -handle these failures. - -The shared copier must return the failure through `copy_subtree()` and the -merge call chain until the application boundary can report it with the -affected key and path. Codec, filesystem, unsupported-conversion, malformed -dataset, and overwrite failures are propagated with `std::expected`; -`CopyError` retains any underlying `CodecError`. Assertions remain appropriate -for internal invariants, but operational failures are not assertion failures -or intentionally thrown exceptions. - -### Paired hierarchy walking - -Extract only the dimension-neutral control flow from `sf_merger::Merger`. -The shared walker obtains the left and right status for a key and asks a -policy for one of: - -```cpp -store::merge::Recurse -store::merge::Ignore -store::merge::KeepLeft -store::merge::KeepRight -store::merge::Write -``` - -The walker owns recursion and unchanged-subtree reuse. The policy owns -selection and payload combination. - -All 16 combinations of `Missing`, `Leaf`, `Inner`, and `Virtual` must be -handled. Unsupported combinations may return a typed error, but they must not -fall into `UNREACHABLE()`. - -The existing 3D adapter retains: - -- `NodeLoader` ancestor mesh reconstruction; -- ECEF node bounds; -- mesh masks and clipping; -- mesh combination and texture atlas generation; and -- mesh validation and auxiliary texture writes. - -A future 2D merger can supply a raster policy without changing the shared -walker. Implementing that policy is outside this refactor. +`Storage::copy_from()` with `DEBUG_ASSERT_VAL`, as does the unchanged-leaf path +in `cut_leaf_node()`, while some overwrite paths terminate through +`LOG_ERROR_AND_EXIT()`. Their `void` call chains prevent the application from +reporting or handling these failures. + +Return failures from the SF-local subtree and cut functions through their +callers until the application boundary can report the affected key and path. +Codec, filesystem, unsupported-conversion, malformed-dataset, and overwrite +failures are propagated with `std::expected`; `CopyError` retains any +underlying `CodecError`. Assertions remain appropriate for internal +invariants, but operational failures are not assertion failures or +intentionally thrown exceptions. + +### Paired hierarchy walking is deferred + +This refactor does not extract `sf_merger::Merger` into a shared paired-tree +walker. SF only permits `Missing`, `Leaf`, and `Virtual`, and its current +recursion does not provide enough evidence to define the `Inner` behaviour +needed by raster-fundamentalis and tile-base merging. + +The previously proposed mutually exclusive actions `Recurse`, `Ignore`, +`KeepLeft`, `KeepRight`, and `Write` cannot express both an action for the +current physical payload and recursion into descendants. `Inner` merging may +require both. Whether the future interface uses a combined `WriteAndRecurse` +action or independent current-node and descendant decisions belongs to the +`rf_merger` design. + +For this refactor: + +- keep recursion, subtree traversal, and mesh policy in `sf_merger`; +- validate SF inputs and reject `Inner` through `std::expected`; +- move only the one-node `Storage::copy_from()` mechanism into `store`; and +- do not add a shared `store::merge` namespace. + +A future `rf_merger` task will define the paired-tree action algebra from the +2D requirements and may migrate `sf_merger` once both use cases are known. ## Implementation phases @@ -705,26 +604,30 @@ has failing tests. No production behaviour changes. -1. Add golden 3D fixtures created by the current code: +1. Add golden SF fixtures created by the current code: - one `terrain.index` using `flat`; - one using `level_and_coordinate_directories`; - - one DAG dataset whose index selects `.bin` and whose payload contains a - valid serialized `dag::ClusterBatch`; - - physical payload paths for a root, child, and deeper descendant; and - - an index containing `Leaf`, `Virtual`, and `Inner`. -2. Test that all fixtures open, resolve the expected IDs and extensions, and + - across the fixtures, physical payload paths for a root, child, and deeper + descendant, without placing physical payloads at ancestor and descendant + keys in the same index; and + - index entries containing `Leaf` and `Virtual`, but no `Inner`. +2. Add one golden DAG dataset whose index selects `.bin`, whose payload + contains a valid serialized `dag::ClusterBatch`, and whose index contains + `Leaf`, `Virtual`, and `Inner`. +3. Test that all fixtures open, resolve the expected IDs and extensions, and traverse the expected sparse nodes. -3. Add path round-trip tests for boundary IDs and both layouts. -4. Add storage tests for: +4. Add path round-trip tests for boundary IDs and both layouts. +5. Add storage tests for: - matching-extension hard links; - different-extension decode/re-encode; - - `.terrain`, `.glb`, `.gltf`, and `.bin` dispatch through the appropriate - domain resolver; + - `.terrain`, `.glb`, `.gltf`, and `.bin` open/read dispatch through the + appropriate domain resolver; the `.bin` case does not exercise + `copy_from()`; - explicit failure for an unknown preferred extension; - overwrite rejection; - indexed and unindexed opens; and - final index creation by directory scan. -5. Record the pre-refactor public aliases used by `sf_builder`, `sf_merger`, +6. Record the pre-refactor public aliases used by `sf_builder`, `sf_merger`, `sf_index_browser`, `dag_builder`, and `dag_convert_debug`. Exit criterion: the compatibility tests pass against the untouched @@ -760,10 +663,8 @@ callers still build through aliases; no filesystem code has changed. in the 3D format adapter. 5. Port legacy layout discovery so it strips recognized 3D file endings before calling `node_path_to_key()`. -6. Add the 2D `z/x/y` mapping with stable ID `zoom/x/y_google`. The raster - codec, not the mapping, adds `.amort` or debug endings. -7. Switch node-path and layout-detection tests to the new implementation. -8. Delete the old strategy base class, registration machinery, and concrete +6. Switch node-path and layout-detection tests to the new implementation. +7. Delete the old strategy base class, registration machinery, and concrete strategy classes once no call site uses them. Exit criterion: the legacy 3D adapter plus codecs resolve all Phase 0 fixtures @@ -792,8 +693,8 @@ strategy pointer. 8. Make every raw file operation obtain its complete file list through `Codec::paths()`. `has()` requires every listed file, and `remove()` removes every listed file. -9. Keep domain-specific payload codecs outside the shared module under - `mesh::codec` and `raster_store::codec`. +9. Keep domain-specific mesh codecs outside the shared module under + `mesh::codec`. 10. Split generic index maintenance from 3D index serialization and legacy folder discovery. 11. Keep the current 3D `terrain.index` DTO and open functions as compatibility @@ -806,6 +707,10 @@ strategy pointer. callers. 14. Preserve the current 3D destructor-save behaviour until all callers have explicit index finalization. +15. Instantiate the shared storage tests with `raster_store::StoreTraits` + using a test-only path mapping and codec. This proves the storage templates + contain no hidden `octree::Id` dependency without defining a stable RF + layout, codec, or disk format. Add focused codec tests using single-file, multi-file, read/write, and write-only test codecs before depending on the raster payload implementation. @@ -818,7 +723,7 @@ through the shared runtime codec and storage implementation. Existing DAG payload bytes and `.bin` paths remain compatible. No extension-dispatching mesh codec or second storage implementation remains under `octree`. -### Phase 4 — Generalize subtree reuse and paired walking +### Phase 4 — Harden node reuse and enforce SF topology 1. Change `Storage::copy_from()` to compare input and output codec path lists for the fixed dummy `NodePath`. @@ -826,74 +731,37 @@ mesh codec or second storage implementation remains under `octree`. created by a partially failed call. 3. Decode with the input codec and encode with the output codec when lists differ. -4. Add `CopyOptions::force_reencode`, defaulting to false. -5. Test: +4. Test: - one-file hard linking; - multi-file hard linking; - different path counts and endings; - - forced re-encoding with otherwise equal paths; + - cleanup after a partially failed multi-file hard link; - conversion between terrain and glTF; - - conversion into a write-only codec; and - - runtime failure when a required codec operation is unsupported. -6. Add the shared unchanged-subtree copier. -7. Test copies containing `Leaf`, `Virtual`, and `Inner` nodes. -8. Add the paired hierarchy walker and typed actions. -9. Cover all 16 status pairs with table-driven tests. -10. Adapt the 3D merger to the shared walker while keeping mesh policy in - `sf_merger`. -11. Remove generic recursion and copy logic from `sf_merger::Merger` and - `NodeWriter`. -12. Add a 3D integration test proving an unchanged subtree is hard-linked and - a changed boundary node is newly written. - -Exit criterion: the existing 3D merger behaviour is preserved, `Inner` no -longer reaches `UNREACHABLE()`, multi-file reuse works through -`Codec::paths()`, and the shared walker contains no mesh, ECEF, GDAL, OpenCV, -or raster dependencies. - -### Phase 5 — Add the 2D raster-fundamentalis adapter - -All required 2D format decisions are resolved below. - -1. Add the checked 2D persistent-key conversion around `radix::tile::Id`, - supporting zoom 0 through `std::numeric_limits::digits` and - treating that maximum zoom as terminal. -2. Add the default `//` path mapping with stable ID - `zoom/x/y_google`. -3. Define `raster_store.index` with a versioned 2D DTO containing the layout - ID and sparse key/status entries, without derived aggregate metadata. Use - fixed-width `uint32_t` zoom/x/y fields and serialize entries in - lexicographic `(zoom, x, y)` order. -4. Implement the version-1 magic/version/checksum/compression envelope. Store - zlib CRC-32 as `uint32_t`, computed over the compressed payload, and use - zstd at its best-compression setting. Import zstd through the project's - CMake dependency facility; use the existing `ZLIB::ZLIB` dependency for - CRC-32. -5. Add the 2D format adapter and storage aliases under - `raster_store`. -6. Add `raster_store::codec::Amort` with runtime format options when - the final `.amort` serialization is available. -7. Add the output-only `raster_store::codec::Debug` with runtime - options for its data and attribution files. -8. If the final `.amort` serialization is not available, exercise storage - with the Phase 3 test codec and do not make `.amort` claims from it. -9. Test: - - invalid and boundary tile IDs, including the maximum zoom and rejected - children beyond it; - - index serialization and validation; - - `Leaf`/`Inner` coexistence; - - sparse traversal and ancestor lookup; - - path round trips; - - snapshot hard-link reuse; - - AMORT-to-debug output conversion; - - explicit cross-filesystem/preflight failure; - - publication from `.part` to ``; and - - publication rejection when the destination already exists. - -Exit criterion: the same shared store can create, open, traverse, and reuse a -minimal 2D raster-fundamentalis fixture without changing the 3D fixtures. - -### Phase 6 — Cleanup and documentation + - overwrite rejection; and + - missing-file, hard-link, decode, and encode error propagation. +5. Add `sf::validate_index()`, returning `sf::InvalidTopology` with the + offending key when it encounters `Inner`. +6. Apply the validator to SF-builder output finalization, SF-merger merge and + cut inputs, SF-merger output, and the DAG builder's SF input. Do not apply + it when opening DAG datasets, through generic octree/store adapters, or in + the diagnostic `sf_index_browser`. +7. Keep SF recursion, subtree traversal, and mesh policy in `sf_merger`. + Change its subtree and cut call chains to propagate validation and + `copy_from()` failures through `std::expected` to the application boundary. +8. Add integration tests proving: + - valid `Leaf`/`Virtual` SF merge behaviour is unchanged; + - an SF input containing `Inner` fails validation before merge dispatch; + - an unchanged SF subtree is hard-linked and a changed boundary node is + newly written; and + - an unchanged leaf in the SF cut path is hard-linked while a clipped leaf + is newly written. + +Exit criterion: one-node copying works through `Codec::paths()`, SF consumers +reject `Inner` with a typed error before processing, existing valid SF merge +and cut behaviour is preserved, and neither a shared subtree copier nor a +paired-tree walker has been introduced. + +### Phase 5 — Cleanup and documentation 1. Remove temporary forwarding headers that no repository caller needs. 2. Remove obsolete files under `octree/disk` and the old generic @@ -902,10 +770,15 @@ minimal 2D raster-fundamentalis fixture without changing the 3D fixtures. `octree`. 4. Update includes, CMake source lists, and precompiled-header includes. 5. Update [architecture.md](architecture.md), - [status-quo.md](status-quo.md), and the before-refactor report with links to - the implemented boundary. Preserve the before-refactor report as history; - do not rewrite it as if it described the new code. -6. Document the final public names and a minimal 2D/3D opening example. + [storage-format.md](storage-format.md), [status-quo.md](status-quo.md), and + the before-refactor report with links to the implemented boundary. State + clearly that RF formats and tools remain future work and that architecture + requirements are not acceptance criteria for this refactor. Preserve the + before-refactor report as history; do not rewrite it as if it described the + new code. +6. Document the final public names, a legacy 3D opening example, and an + in-memory `radix::tile::Id` topology example. Do not document a persistent + RF format or opening API. Exit criterion: repository search finds no generic implementation tied to `octree::Id`; all tests pass; the old layout strategy hierarchy is gone. @@ -922,13 +795,19 @@ unittests/terrainlib/store_layout.cpp unittests/terrainlib/store_codec.cpp unittests/terrainlib/store_storage.cpp unittests/terrainlib/store_compatibility.cpp -unittests/terrainlib/store_merge_walk.cpp +unittests/terrainlib/sf_validate_index.cpp ``` The DAG payload-compatibility fixture and resolver integration test belong in `unittests_dagbuilder`, because `terrainlib` must not depend on `dag::ClusterBatch` or its serializers. +The validator's unit tests belong in `unittests_terrainlib`. Boundary tests +belong with their consumers: SF-builder output validation in +`unittests_sfbuilder`, merge and cut validation/error propagation in +`unittests_sfmerger`, and DAG-builder SF-input validation in +`unittests_dagbuilder`. `sf_index_browser` remains unvalidated by design. + During implementation: 1. Build in `$source_dir/build/$config_name`. @@ -939,7 +818,9 @@ During implementation: DAG resolver changes. 6. Build `sf_builder`, `sf_merger`, `sf_index_browser`, `dag_builder`, and `dag_convert_debug` after their storage aliases move. -7. Run any existing merger integration fixture after Phase 4. +7. Run `unittests_sfbuilder`, `unittests_sfmerger`, and + `unittests_dagbuilder`, plus any existing merger integration fixture, after + the Phase 4 validation changes. 8. Inspect `git diff --check` and the final worktree before each commit. No formatting-only pass or unrelated refactor belongs in these commits. @@ -969,8 +850,9 @@ No formatting-only pass or unrelated refactor belongs in these commits. | `octree/disk/IndexFile.h` | versioned 3D format adapter under `octree` | | DAG serializers in `dag_node.h` and `encoded.h` | `dag_builder/serialization.h` | | `dag_builder/storage.h` aliases | DAG storage aliases plus codec resolver convenience functions | -| `sf_merger::NodeWriter` subtree loop | `store/copy_subtree.h` | -| `sf_merger::Merger` recursion | `store/merge/walk.h` | +| `sf_merger::NodeWriter` subtree loop | remains in `sf_merger`; return copy failures through `std::expected` | +| `sf_merger::cut_leaf_node()` copy path | remains in `sf_merger`; return copy failures through `std::expected` | +| SF merger `Inner` `UNREACHABLE()` path | `sf::validate_index()` returning `sf::InvalidTopology` | ## Risks and controls @@ -983,63 +865,36 @@ No formatting-only pass or unrelated refactor belongs in these commits. | Valid legacy paths are parsed differently | Characterization and round-trip tests before replacement | | Template migration creates a large unreviewable diff | Compatibility aliases and phase-by-phase caller migration | | `radix::tile::Id` root underflows | Traits intercept root parent lookup | -| Invalid 2D coordinates become persistent | Validate on every disk/API boundary | -| `Inner` payloads are lost during subtree reuse | Copy every physical status and test mixed-depth fixtures | -| Linked snapshots are modified in place | Immutable snapshot API and overwrite-disabled output | +| Invalid `radix::tile::Id` values enter the shared index | Validate through `raster_store::StoreTraits` and test boundary zooms | +| Invalid `Inner` nodes reach SF merge dispatch | Validate every SF input first and return the offending key in a typed error | +| SF subtree copying is generalized before RF requirements exist | Keep it in `sf_merger`; reconsider extraction with `rf_merger` | +| A paired-walker API is fixed before RF semantics are known | Defer its action algebra until `rf_merger` requirements are defined | | Multi-file hard linking fails partway through | Remove links created by the failed `copy_from()` before returning | | Incompatible codecs return the same path list | Treat path-list equality as a codec contract and test every concrete codec pairing | | Output-only codec is selected for required input | Return a clear `UnsupportedOperation` error | -| Hard-link failure appears late | 2D operation preflight and explicit errors | -| Shared code accumulates mesh/raster policy | Dependency tests/review against the source boundary | -| Generic index accidentally dictates both disk formats | Separate 3D and 2D format adapters | - -## Decisions required before Phase 5 - -### Resolved decisions - -1. **Index filename and location:** each snapshot stores - `raster_store.index` directly at its root. -2. **Node and payload paths:** `store_layout` determines the extensionless - node path. The default 2D mapping produces `//` directly below - the snapshot; there is no fixed `chunks/` directory. The AMORT codec - expands it to `//.amort`. -3. **Layout identity and coordinates:** the stable layout ID serialized in the - index is `zoom/x/y_google`. It uses Google/XYZ coordinates with the origin - at the north-west. -4. **Persistent key and zoom range:** use `radix::tile::Id`, whose x and y - coordinates must fit in `uint32_t`. The maximum zoom is therefore - `std::numeric_limits::digits` (32): at that zoom every - `uint32_t` x/y value is representable and valid. A maximum-zoom tile is - terminal because its children would require another coordinate bit. - Validation must special-case this boundary rather than evaluate an - overflowing `uint32_t{1} << 32`. The versioned disk DTO uses fixed-width - `uint32_t` fields for zoom, x, and y rather than serializing the platform - `unsigned` type directly. -5. **Version-1 index contents:** store the layout ID and sparse key/status - entries. `Leaf`, `Inner`, and `Virtual` are serialized; `Missing` is - represented by absence. Do not store derived aggregate metadata until a - concrete query demonstrates that it is required. Serialize entries in - lexicographic `(zoom, x, y)` order and reject duplicate keys while reading. -6. **Serialization envelope:** version 1 uses a fixed, file-type-specific - 64-bit magic value generated once during implementation, a 32-bit version, - zlib CRC-32 stored as `uint32_t` and computed over the compressed payload, - a compression enum, and the compressed payload. Use zstd with its - best-compression setting. The exact magic value is an implementation - constant, not a further design decision. -7. **Publication:** build a snapshot in a sibling directory named - `.part`. Write the index last, validate the completed snapshot, - flush and close every file, and atomically rename the directory to - `` on the same filesystem. The destination must not already - exist. A `.part` directory is never considered published; the rename - removes the suffix without a separate marker or manifest. This guarantees - atomic visibility to concurrent readers during normal operation. -8. **Crash behavior:** the store does not guarantee durability or safe - recovery across a power failure, operating-system crash, or storage - failure. Do not add `fsync()`, `fdatasync()`, `FlushFileBuffers()`, or - equivalent platform-specific synchronization to this refactor. After such - a failure, either a `.part` directory or a final snapshot may be unusable - and must be validated and rebuilt. Flushing and closing files remains - required before the rename for normal-operation correctness, but must not - be documented as a crash-durability guarantee. - -There are no remaining decisions blocking Phase 5. +| Shared code accumulates mesh or provisional RF policy | Dependency tests/review against the source boundary | +| The refactor accidentally fixes the future RF disk format | Do not add a persistent 2D format adapter, layout, or codec | + +## Deferred raster-fundamentalis work + +This refactor is a prerequisite for, not an implementation of, the system +described in [architecture.md](architecture.md). That document and +[storage-format.md](storage-format.md) remain useful design input, but their RF +details are not acceptance criteria for this refactor and are not declared +final here. + +A later RF design phase must resolve and test at least: + +- the persistent index filename, schema, versioning, and serialization + envelope; +- persistent tile keys, coordinate convention, layout IDs, and node paths; +- tile and source-attribution payload formats and codecs; +- snapshot construction, validation, publication, and crash expectations; +- hard-link preflight and unchanged-tile reuse; +- `rf_merger` policy, paired-tree walking, and `Inner` copy behaviour; and +- RF builders, converters, debugging outputs, and other tools. + +None of those decisions blocks completion of this refactor. +The removed ideas are retained, without implementation-plan status, in the +explicitly draft [rf_builder notes](rf_builder.md) and +[rf_merger notes](rf_merger.md). diff --git a/docs/raster-store/rf_builder.md b/docs/raster-store/rf_builder.md new file mode 100644 index 00000000..7006b981 --- /dev/null +++ b/docs/raster-store/rf_builder.md @@ -0,0 +1,202 @@ +# DRAFT — `rf_builder` + +Status: **draft archive of removed ideas**. + +This is not a current implementation plan, accepted format specification, or +statement that the choices below are correct. It is a lightly reformatted +archive of RF-builder and persistent-2D material removed from +[refactor-plan.md](refactor-plan.md). The details are retained so they are not +lost; they require a separate design pass after the shared-store refactor. + +## Proposed names and source boundary + +The removed proposal placed the 2D implementation in +`src/terrainlib/raster_store` and used the `raster_store` namespace: + +```text +src/terrainlib/raster_store/ +├── StoreTraits.h +├── IndexFile.h +├── Storage.h +├── codec/ +│ ├── Amort.h +│ └── Debug.h +└── store_layout/ + └── ZoomXYGoogle.h +``` + +The proposed boundary was: + +- `store` contains dimension- and payload-neutral mechanisms; +- `raster_store` contains the 2D format, key adapters, and raster codecs; and +- subdirectory names match their namespaces where a subnamespace is used. + +The proposed concrete index type was: + +```cpp +store::Index +``` + +## `raster_store::StoreTraits` + +The removed proposal adapted `radix::tile::Id` through +`raster_store::StoreTraits` and specified that it: + +- treats zoom zero as the only root; +- never calls `radix::tile::Id::parent()` at zoom zero, where it underflows; +- rejects coordinates outside `[0, 2^zoom)`; +- accepts zoom levels 0 through + `std::numeric_limits::digits`, inclusive; +- treats that maximum zoom as terminal because a child cannot be represented + by the `uint32_t` x/y coordinates; +- validates the maximum zoom without evaluating an overflowing + `uint32_t{1} << 32`; +- uses `radix::tile::Id::Hasher`; and +- uses the Google/XYZ convention, with the origin at the north-west, at the + persistent boundary. + +The shared code was expected to obtain roots, parents, children, validation, +and hashing through the traits without specializing on the key type. + +## Node paths and layout + +The removed example mapped a raster node to an extensionless `store::NodePath`: + +```text +raster-store ZXY 12/2200/1400 +``` + +The proposed lookup functions were: + +```cpp +raster_store::store_layout::zoom_x_y_google() +raster_store::store_layout::from_id(id) +``` + +The stable layout ID was `zoom/x/y_google`. The default mapping produced +`//` directly below the snapshot root; there was no fixed +`chunks/` directory. A codec, rather than the layout, added `.amort` or debug +file endings. + +## Payload and codec sketches + +The removed proposal used the shared runtime codec interface and placed +raster codecs in `raster_store::codec`: + +```cpp +template +struct raster_store::codec::Amort + : store::Codec> { .. }; + +template +struct raster_store::codec::Debug + : store::Codec> { .. }; +``` + +`Amort` was proposed as readable and writable. `Debug` was proposed as +write-only, with runtime options for data format, attribution format, JPEG +quality, or similar debugging choices. It was not to be template-composed +from separate image codec types. + +The multi-file debug example was: + +```text +Debug raster codec configured for JPEG data and PNG attribution + 12/2200/1400 + -> 12/2200/1400.data.jpg + -> 12/2200/1400.attribution.png +``` + +The proposed final tile path was `//.amort`. The `.amort` payload +and source-attribution-table serialization were explicitly not defined beyond +the separate storage-format notes. + +## Index and format-adapter sketch + +The removed proposal used a separately versioned `raster_store::v1` DTO stored +as `raster_store.index`. + +The proposed version-1 contents were: + +- a layout ID; +- sparse key/status entries; +- serialized `Leaf`, `Inner`, and `Virtual` values; +- `Missing` represented by absence; +- no derived aggregate metadata until a concrete query requires it; +- fixed-width `uint32_t` zoom/x/y fields instead of platform `unsigned`; +- entries ordered lexicographically by `(zoom, x, y)`; and +- duplicate keys rejected while reading. + +The proposed serialization envelope contained: + +- a fixed, file-type-specific 64-bit magic value generated during + implementation; +- a 32-bit version; +- zlib CRC-32 stored as `uint32_t` and computed over the compressed payload; +- a compression enum; and +- a zstd-compressed payload using zstd's best-compression setting. + +The proposal imported zstd through the project's CMake dependency facility and +used the existing `ZLIB::ZLIB` dependency for CRC-32. + +Index serialization was not a responsibility of `store::Index`. The proposed +2D format adapter supplied the index filename, index conversion, mapping +lookup, and default mapping. + +## Snapshot publication sketch + +The removed proposal required explicit finalization/publication for the 2D +snapshot API; a destructor was not to make an incomplete snapshot +authoritative. + +The publication details are now retained in +[architecture.md](architecture.md#publication). The removed proposal used a +sibling `.part` directory, wrote the index last, validated and +closed all files, and atomically renamed it to `` on the same +filesystem. The destination could not already exist. + +The removed text explicitly did not promise durability or safe recovery after +a power failure, operating-system crash, or storage failure, and did not add +`fsync()`, `fdatasync()`, `FlushFileBuffers()`, or equivalent synchronization. + +## Removed implementation and verification ideas + +The removed 2D-adapter phase contained these items: + +- add the checked persistent-key conversion around `radix::tile::Id`; +- add the `//` mapping with stable ID `zoom/x/y_google`; +- define the versioned `raster_store.index` DTO; +- implement the magic/version/checksum/compression envelope; +- add the 2D format adapter and storage aliases under `raster_store`; +- add `raster_store::codec::Amort` when final `.amort` + serialization is available; +- add the output-only `raster_store::codec::Debug`; +- use a test codec instead of making `.amort` claims if final serialization is + unavailable; and +- keep raster-specific processing outside the shared store. + +The removed verification list contained: + +- invalid and boundary tile IDs, including maximum zoom and rejected children; +- index serialization and validation; +- `Leaf`/`Inner` coexistence; +- sparse traversal and ancestor lookup; +- path round trips; +- snapshot hard-link reuse; +- AMORT-to-debug output conversion; +- explicit cross-filesystem/preflight failure; +- publication from `.part` to ``; and +- publication rejection when the destination already exists. + +## Removed risks and controls + +| Removed risk | Removed control | +|---|---| +| `radix::tile::Id` root underflows | Traits intercept root parent lookup | +| Invalid 2D coordinates become persistent | Validate on every disk/API boundary | +| Linked snapshots are modified in place | Immutable snapshot API and overwrite-disabled output | +| Hard-link failure appears late | 2D operation preflight and explicit errors | +| Generic index dictates both disk formats | Separate 3D and 2D format adapters | + +All material in this document remains provisional despite the concrete names +preserved above. diff --git a/docs/raster-store/rf_merger.md b/docs/raster-store/rf_merger.md new file mode 100644 index 00000000..3fc97d0a --- /dev/null +++ b/docs/raster-store/rf_merger.md @@ -0,0 +1,231 @@ +# DRAFT — `rf_merger` + +Status: **draft archive of removed ideas**. + +This is not a current implementation plan or accepted RF merge specification. +It is a lightly reformatted archive of subtree-copy and paired-walker material +removed from [refactor-plan.md](refactor-plan.md). The details are retained so +they are not lost; later discussion already established that RF semantics must +be designed before choosing these abstractions. + +## Proposed names and source boundary + +The removed proposal added these dimension-neutral files: + +```text +src/terrainlib/store/ +├── copy_subtree.h +└── merge/ + ├── Action.h + └── walk.h +``` + +The proposed migration map was: + +| Existing code | Removed target | +|---|---| +| `sf_merger::NodeWriter` subtree loop | `store/copy_subtree.h` | +| `sf_merger::Merger` recursion | `store/merge/walk.h` | + +The proposal intended shared mechanisms to contain no mesh or raster merge +policy. + +## One-node copy proposal + +The proposed one-node API added: + +```cpp +struct CopyOptions { + bool force_reencode = false; +}; +``` + +For one key, the proposed `Storage::copy_from()` behaviour was: + +1. Call input and output `Codec::paths()` with the same fixed dummy + `NodePath`, proposed as `__codec_probe__/node`. +2. Compare path lists exactly, including count, order, and filename endings. +3. When the lists match and `force_reencode` is false, call both codecs with + the actual source and target `NodePath` and hard-link every corresponding + file. +4. When lists differ or re-encoding is forced, read with the input codec and + write with the output codec. +5. Update the target index only after all links or the write complete. + +If a multi-file link failed partway through, the proposal removed target links +created by that call before returning `CopyError`. `CopyError` retained any +underlying `CodecError`. There was no silent file-copy fallback. + +Codec settings that did not change `paths()`, such as compression level or +JPEG quality, did not force re-encoding by default. Callers could pass +`force_reencode = true`. + +The removed hard-link rules were: + +- never modify an existing linked payload in place; +- matching codec path lists hard-link every file; +- different path lists decode with the input codec and encode with the output + codec; +- `force_reencode` selects decode/encode; +- hard-link failure is explicit; +- 2D snapshot tools preflight hard-link support before a long operation; and +- no silent file-copy fallback. + +## Shared subtree-copy proposal + +The removed API sketch was: + +```cpp +std::expected +store::copy_subtree( + const IndexedStorage& source, + Storage& target, + const Key& root, + CopyOptions options = {}); +``` + +The proposed operation: + +1. traversed an indexed source subtree; +2. skipped `Virtual` nodes; +3. called `copy_from()` for physical payloads in `Leaf` and `Inner` states; +4. continued traversal below `Inner`; and +5. returned copy failures rather than asserting or terminating. + +The operation was intended to be payload-neutral and know nothing about +meshes, rasters, masks, attribution, or their encodings. + +The removed target call chain was: + +```text +merge policy decides to keep a source subtree unchanged + -> store::copy_subtree() + -> store::Storage::copy_from() + -> hard-link every codec path, or decode/encode +``` + +## Removed `Inner` copying rationale + +The removed proposal used this topology model: + +| Status | Physical payload | Indexed descendants | +|---|---:|---:| +| `Leaf` | yes | no | +| `Inner` | yes | yes | +| `Virtual` | no | yes | + +It used this RF example: + +```text +zoom 10 physical tile -> Inner +└── zoom 11 physical tile -> Leaf +``` + +The proposal said subtree reuse must preserve both payloads: copy the parent +payload, continue below it, and allow insertion of the descendant to promote +the copied parent from `Leaf` to `Inner` through normal index transitions. + +The proposed status handling was: + +```cpp +switch (status) { +case NodeStatus::Virtual: + break; +case NodeStatus::Leaf: +case NodeStatus::Inner: + target.copy_from(id, source, options); + break; +} +``` + +This described unchanged-subtree copying only. It did not define how two RF +trees should be merged. + +## Paired-tree walker proposal + +The removed proposal extracted dimension-neutral recursion from +`sf_merger::Merger` and introduced these policy results: + +```cpp +store::merge::Recurse +store::merge::Ignore +store::merge::KeepLeft +store::merge::KeepRight +store::merge::Write +``` + +The walker owned recursion and unchanged-subtree reuse. The policy owned +selection and payload combination. + +The removed proposal required all 16 combinations of `Missing`, `Leaf`, +`Inner`, and `Virtual` to be handled or rejected with a typed error rather than +falling into `UNREACHABLE()`. + +It kept these concerns outside the shared walker: + +- `NodeLoader` ancestor mesh reconstruction; +- ECEF node bounds; +- mesh masks and clipping; +- mesh combination and texture atlas generation; and +- mesh validation and auxiliary texture writes. + +The proposal suggested that a future 2D merger could supply a raster policy. +Later discussion identified that the mutually exclusive action list cannot +express both an action for an `Inner` payload and recursion into descendants. +Ideas mentioned after that were a combined `WriteAndRecurse` result or +independent current-node and descendant decisions. None is selected. + +## Removed implementation and verification ideas + +The removed subtree/walker phase contained: + +- compare input and output codec path lists for a fixed dummy `NodePath`; +- hard-link all files when lists match, including partial-failure cleanup; +- decode with the input codec and encode with the output codec when lists + differ; +- add `CopyOptions::force_reencode`; +- add the shared unchanged-subtree copier; +- add the paired hierarchy walker and typed actions; +- cover all 16 status pairs with table-driven tests; +- adapt the 3D merger while keeping mesh policy in `sf_merger`; +- remove generic recursion and copy logic from `sf_merger::Merger` and + `NodeWriter`; and +- add a 3D integration test for an unchanged hard-linked subtree and a newly + written changed boundary node. + +The removed focused tests included: + +- one-file hard linking; +- multi-file hard linking; +- different path counts and endings; +- forced re-encoding with otherwise equal paths; +- conversion between terrain and glTF; +- conversion into a write-only codec; +- runtime failure for an unsupported codec operation; +- copies containing `Leaf`, `Virtual`, and `Inner`; and +- all 16 paired status combinations. + +The proposed generic test file was: + +```text +unittests/terrainlib/store_merge_walk.cpp +``` + +## Removed error and risk notes + +The removed error-propagation proposal passed copy failures through +`copy_subtree()` and the merge call chain to the application boundary with the +affected key and path. Codec, filesystem, unsupported-conversion, +malformed-dataset, and overwrite failures used `std::expected`; operational +failures were not assertions or intentional exceptions. + +| Removed risk | Removed control | +|---|---| +| `Inner` payloads are lost during subtree reuse | Copy every physical status and test mixed-depth fixtures | +| Multi-file hard linking fails partway through | Remove links created by the failed `copy_from()` before returning | +| Incompatible codecs return the same path list | Treat path-list equality as a codec contract and test every concrete codec pairing | +| Output-only codec is selected for required input | Return a clear `UnsupportedOperation` error | +| Shared code accumulates mesh/raster policy | Dependency tests/review against the source boundary | + +All material in this document remains provisional despite the concrete names +preserved above. From b4e80a85a9e74ce355215e4c8a824979247668f3 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Thu, 30 Jul 2026 00:21:11 +0200 Subject: [PATCH 28/61] Clarify raster store refactor guarantees --- docs/raster-store/README.md | 8 ++++++-- docs/raster-store/refactor-plan.md | 25 ++++++++++++++++++------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/docs/raster-store/README.md b/docs/raster-store/README.md index 54006330..5ebae8f5 100644 --- a/docs/raster-store/README.md +++ b/docs/raster-store/README.md @@ -11,8 +11,12 @@ baseline, not a finalized binary-format specification. - [Architecture](architecture.md) - [Storage format](storage-format.md) - [Sampling and pyramid generation](sampling-and-generation.md) -- [DRAFT rf_builder idea notes](rf_builder.md) -- [DRAFT rf_merger idea notes](rf_merger.md) + +## Plans + +- [Store refactoring plan](refactor-plan.md) +- [DRAFT RF builder plan archive](rf_builder.md) +- [DRAFT RF merger plan archive](rf_merger.md) ## Scope diff --git a/docs/raster-store/refactor-plan.md b/docs/raster-store/refactor-plan.md index a6689376..45ae114a 100644 --- a/docs/raster-store/refactor-plan.md +++ b/docs/raster-store/refactor-plan.md @@ -49,12 +49,20 @@ Persistent raster-fundamentalis formats, adapters, and tools are later work. `std::expected`; unsupported operations and other operational failures are reported as error values. Reading and writing must be reentrant (callable concurrently from different threads). +- This refactor does not add synchronization to storage, indexes, or caches + and does not change their concurrency guarantees. Existing caller-side + synchronization and concurrency behaviour are preserved; any concurrency + bug fix is separate work. - Legacy index metadata selects a codec through an explicit, caller-supplied resolver. The octree format adapter does not contain a global codec registry or depend on mesh or DAG payload types. - `copy_from()` hard-links every file when the input and output codecs return the same path list for a common dummy `NodePath`. Otherwise it decodes with the input codec and encodes with the output codec. +- The `.png` written beside changed meshes by `sf_merger::NodeWriter` is an + unmanaged debug artifact. It is not part of a logical node, is not returned + by `Codec::paths()`, and is not indexed or copied by storage. Its existing + application-local behaviour is preserved. - 3D geometry, ECEF bounds, mesh codecs, mesh reconstruction, mask geometry, and raster-specific processing remain outside the shared store. @@ -521,10 +529,11 @@ The dummy path must be fixed and collision-free, for example `__codec_probe__/node`. Path lists are compared exactly, including count, order, and filename endings. -If linking several files fails partway through, remove the target links -created by that call before returning the error. There is no silent copy -fallback. An unsupported read or write needed for re-encoding is returned -through `CopyError`, retaining the underlying `CodecError`. +If linking several files fails partway through, return the error without a +transactional rollback guarantee; target links already created by the call +may remain. There is no silent copy fallback. An unsupported read or write +needed for re-encoding is returned through `CopyError`, retaining the +underlying `CodecError`. Hard-link rules: @@ -727,15 +736,16 @@ mesh codec or second storage implementation remains under `octree`. 1. Change `Storage::copy_from()` to compare input and output codec path lists for the fixed dummy `NodePath`. -2. Hard-link all actual files when the lists match, with cleanup of links - created by a partially failed call. +2. Hard-link all actual files when the lists match. A partially failed + multi-file operation returns an error without rolling back links already + created. 3. Decode with the input codec and encode with the output codec when lists differ. 4. Test: - one-file hard linking; - multi-file hard linking; - different path counts and endings; - - cleanup after a partially failed multi-file hard link; + - error propagation after a partially failed multi-file hard link; - conversion between terrain and glTF; - overwrite rejection; and - missing-file, hard-link, decode, and encode error propagation. @@ -851,6 +861,7 @@ No formatting-only pass or unrelated refactor belongs in these commits. | DAG serializers in `dag_node.h` and `encoded.h` | `dag_builder/serialization.h` | | `dag_builder/storage.h` aliases | DAG storage aliases plus codec resolver convenience functions | | `sf_merger::NodeWriter` subtree loop | remains in `sf_merger`; return copy failures through `std::expected` | +| `sf_merger::NodeWriter` auxiliary `.png` write | remains an unmanaged, application-local debug artifact | | `sf_merger::cut_leaf_node()` copy path | remains in `sf_merger`; return copy failures through `std::expected` | | SF merger `Inner` `UNREACHABLE()` path | `sf::validate_index()` returning `sf::InvalidTopology` | From f463e159b237a0cd0c61a156c4c91ca27fea5df6 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Thu, 30 Jul 2026 10:46:11 +0200 Subject: [PATCH 29/61] Finalize raster store refactor plan --- docs/raster-store/refactor-plan.md | 179 +++++++++++++++++++++++------ 1 file changed, 147 insertions(+), 32 deletions(-) diff --git a/docs/raster-store/refactor-plan.md b/docs/raster-store/refactor-plan.md index 45ae114a..66574ef4 100644 --- a/docs/raster-store/refactor-plan.md +++ b/docs/raster-store/refactor-plan.md @@ -49,16 +49,33 @@ Persistent raster-fundamentalis formats, adapters, and tools are later work. `std::expected`; unsupported operations and other operational failures are reported as error values. Reading and writing must be reentrant (callable concurrently from different threads). +- `store::RawStorage` exclusively owns the configured + `std::unique_ptr>`. `store::Storage` owns the raw + storage and therefore owns the codec transitively. Storage consumers do not + receive a codec template parameter and application call sites do not manage + codec objects. - This refactor does not add synchronization to storage, indexes, or caches and does not change their concurrency guarantees. Existing caller-side synchronization and concurrency behaviour are preserved; any concurrency bug fix is separate work. - Legacy index metadata selects a codec through an explicit, caller-supplied - resolver. The octree format adapter does not contain a global codec registry - or depend on mesh or DAG payload types. + resolver supplied by the payload-domain opening function. The octree format + adapter does not contain a global codec registry or depend on mesh or DAG + payload types. Application-level storage consumers do not call the resolver + or handle the resulting codec object. +- Dimension-specific index persistence uses a small runtime + `store::IndexFormat` value containing ordinary function pointers. It + is not an inheritance hierarchy and does not use global registration. - `copy_from()` hard-links every file when the input and output codecs return the same path list for a common dummy `NodePath`. Otherwise it decodes with the input codec and encodes with the output codec. +- Public store operations reject invalid hierarchy keys through + `std::expected`; invalid keys are not represented by assertions or generic + booleans. +- Preserve `StorageSettings::allow_overwrite`. It defaults to `false`; + rejected overwrites return `AlreadyExists` through `std::expected`, and + enabling it preserves the existing DAG-builder overwrite and debug-export + behaviour. - The `.png` written beside changed meshes by `sf_merger::NodeWriter` is an unmanaged debug artifact. It is not part of a logical node, is not returned by `Codec::paths()`, and is not indexed or copied by storage. Its existing @@ -129,6 +146,7 @@ Before moving code, tests must lock down the following 3D behaviour: | DAG payload encoding | existing `dag::ClusterBatch` ZPP Bits serialization | | Equal codec path lists | hard-link every file, or report an explicit error | | Different codec path lists | decode with input codec and encode with output codec | +| Overwrite setting | `StorageSettings::allow_overwrite`, default `false`; enabled writes replace existing payloads | Compatibility means that the refactored code can open datasets written before the refactor and produces datasets that the pre-refactor code can open. Exact @@ -147,13 +165,18 @@ src/terrainlib/ │ ├── NodeStatus.h │ ├── NodeStatusOrMissing.h │ ├── Traits.h +│ ├── InvalidKey.h │ ├── Index.h │ ├── traverse.h │ ├── NodePath.h │ ├── PathMapping.h │ ├── Layout.h +│ ├── IndexFormat.h │ ├── Codec.h +│ ├── CodecError.h +│ ├── OpenError.h │ ├── CopyError.h +│ ├── StorageSettings.h │ ├── RawStorage.h │ ├── Storage.h │ ├── IndexedStorage.h @@ -262,7 +285,9 @@ changing `octree::Id`. by the `uint32_t` x/y coordinates; - validate the maximum zoom without evaluating an overflowing `uint32_t{1} << 32`; -- use `radix::tile::Id::Hasher`. +- use `radix::tile::Id::Hasher`; and +- return children in the deterministic order produced by + `radix::tile::Id::children()`. This traits adapter defines only hierarchy operations. It does not define persistent coordinates, a path layout, or an RF disk format. @@ -271,6 +296,14 @@ The shared code must obtain roots, parents, children, validation, and hashing through the traits. It must not use dimension checks or specialize behaviour on key types internally. +Operations accepting a key validate it through `Traits::is_valid()`. Index +lookup and mutation, traversal with an explicit root, and storage operations +return an `std::expected` retaining an `InvalidKey` when validation fails. +Keys produced internally by `Traits::root()`, `Traits::parent()`, and +`Traits::children()` are trusted only after trait-specific tests establish that +they preserve validity. The child order affects traversal order and is locked +down by the 2D and 3D trait tests; it is not serialized as separate metadata. + ### Sparse index and traversal Move the existing four-state model to: @@ -374,6 +407,10 @@ storage use. `CodecError` is a payload-neutral operational error that records the failed operation, an error category, and a diagnostic message. Concrete codecs convert their domain errors to it. Unsupported read or write operations may use the base implementation and return its `UnsupportedOperation` error. +Codec writes preserve the current directory-creation behaviour: they create +the parent directories required by their output paths before writing. The +storage hard-link path continues to create its target parent directories before +linking. `Codec::paths()` has the following contract: @@ -435,23 +472,65 @@ store::IndexedStorage store::cache::Interface ``` +`RawStorage` owns the +`std::unique_ptr>`. `Storage` owns `RawStorage`, and +`IndexedStorage` owns or derives from `Storage`; no other layer shares codec +ownership. Moving storage transfers ownership. Caches, layouts, index formats, +resolvers, and application consumers never own the codec. + Domain-specific mesh codecs remain under `mesh::codec`. The reusable ZPP Bits codec remains under `store::codec`. RF codecs are deferred. Index serialization is not a responsibility of `store::Index`. Opening and -saving a dataset receives a dimension-specific format adapter which provides: +saving a dataset receives the following small runtime values: + +```cpp +template +struct store::IndexMetadata { + store::Index index; + std::string layout_id; + std::string codec_selector; +}; + +template +struct store::IndexFormat { + std::string_view index_filename; + + std::expected, IndexFormatError> + (*read)(const std::filesystem::path& index_path); + + std::expected + (*write)( + const std::filesystem::path& index_path, + const IndexMetadata& metadata); + + std::optional> + (*mapping_from_id)(std::string_view id); + + PathMapping + (*default_mapping)(); +}; +``` + +The value provides: - the index filename; - index read/write conversion; - mapping lookup by stable ID; -- the default mapping; and -- legacy directory discovery where it is required. +- the default mapping. -This adapter may be a compile-time policy or a small value of function -pointers. Choose the smaller implementation after the Phase 0 tests exist. -It must not reintroduce a layout class hierarchy or global registration. +Legacy directory discovery is an octree opening helper which composes the +octree index format, the supplied payload-domain codec resolver, and the known +octree mappings. It is not a generic `IndexFormat` operation. Neither the +format value nor discovery may reintroduce a layout class hierarchy or global +registration. For 3D, the adapter reads and writes the current `octree` index DTO unchanged. +Its `codec_selector` is exactly the legacy `preferred_extension`, including +the leading dot. Storage retains the selected `IndexFormat`, index path, +layout ID, and codec selector as its index-persistence state, so explicit and +destructor-triggered index saves can reproduce the same metadata. + When opening indexed storage or discovering a legacy unindexed directory, it passes the legacy `preferred_extension` to a caller-supplied codec resolver. The resolver is an ordinary callable and returns @@ -471,9 +550,28 @@ dag::codec::from_extension ``` An unknown extension returns an explicit `UnsupportedCodec` error. Opening a -new empty store may receive an already constructed codec instead of a resolver. -Convenience functions in `src/dag_builder/storage.h` supply the DAG resolver so -application call sites do not handle it directly. +new empty store receives an explicit legacy codec selector plus an already +constructed codec at the payload-domain opening boundary; the generic storage +does not infer persistent metadata from `Codec::paths()`. Mesh and DAG +convenience functions select or resolve the codec and pass ownership into raw +storage, so application storage consumers do not handle codec objects. +Convenience functions in `src/dag_builder/storage.h` supply the DAG resolver. + +Opening functions return their requested storage type through +`std::expected<..., OpenError>`. `OpenError` is a typed sum which retains the +failing path and the underlying error where applicable: + +- index I/O or malformed index metadata (`IndexFormatError`); +- filesystem failure; +- unknown layout ID; +- unsupported codec selector or codec construction failure (`CodecError`); and +- invalid hierarchy key (`InvalidKey`). + +Loading and saving likewise return storage-level expected errors which retain +an invalid key, an underlying `CodecError`, and `AlreadyExists` for a rejected +save. `CopyError` retains invalid-key, missing-source, overwrite, filesystem, +and codec failures. Domain/application boundaries may add context, but must not +discard these errors. Legacy unindexed-directory discovery remains in the 3D adapter: it recognizes candidate endings by asking the supplied resolver, removes an accepted ending @@ -531,9 +629,12 @@ order, and filename endings. If linking several files fails partway through, return the error without a transactional rollback guarantee; target links already created by the call -may remain. There is no silent copy fallback. An unsupported read or write -needed for re-encoding is returned through `CopyError`, retaining the -underlying `CodecError`. +may remain. The copy operation stops immediately, leaves the target index +unchanged for that logical node, and propagates the failure until the +application aborts the overall operation. There is no journal, rollback, +cleanup guarantee, or silent copy fallback. An unsupported read or write needed +for re-encoding is returned through `CopyError`, retaining the underlying +`CodecError`. Hard-link rules: @@ -629,11 +730,8 @@ No production behaviour changes. 5. Add storage tests for: - matching-extension hard links; - different-extension decode/re-encode; - - `.terrain`, `.glb`, `.gltf`, and `.bin` open/read dispatch through the - appropriate domain resolver; the `.bin` case does not exercise - `copy_from()`; - - explicit failure for an unknown preferred extension; - overwrite rejection; + - overwrite-enabled replacement; - indexed and unindexed opens; and - final index creation by directory scan. 6. Record the pre-refactor public aliases used by `sf_builder`, `sf_merger`, @@ -652,7 +750,9 @@ changed. 3. Convert `IndexMap` into `store::Index`. 4. Convert traversal into `store::traverse`. 5. Add `raster_store::StoreTraits` for `radix::tile::Id`. -6. Run the same index-transition and DFS/BFS tests with both trait types. +6. Run the same index-transition and DFS/BFS tests with both trait types, + including deterministic child order, invalid-key errors through + `std::expected`, maximum-depth children, and explicit traversal roots. 7. Provide temporary `octree` aliases so downstream migration is separate from the algorithm extraction. @@ -696,27 +796,38 @@ strategy pointer. - one glTF codec configured for binary `.glb` or JSON `.gltf`. 5. Move copy error, raw storage, caches, logical storage, and indexed storage into `store`. -6. Make storage own a configured `std::unique_ptr>`; remove the - codec template parameter from storage. +6. Make `RawStorage` exclusively own a configured + `std::unique_ptr>`. `Storage` owns it transitively through + raw storage; remove the codec template parameter from every storage type. 7. Replace every embedded `octree::Id` with `Traits::Key`. 8. Make every raw file operation obtain its complete file list through `Codec::paths()`. `has()` requires every listed file, and `remove()` removes every listed file. 9. Keep domain-specific mesh codecs outside the shared module under `mesh::codec`. -10. Split generic index maintenance from 3D index serialization and legacy - folder discovery. +10. Add the function-pointer-based `IndexFormat`, `IndexMetadata`, and + typed format/open errors. Split generic index maintenance from 3D index + serialization and legacy folder discovery. 11. Keep the current 3D `terrain.index` DTO and open functions as compatibility - adapters over the shared storage. Resolve its preferred extension through - the caller-supplied mesh or DAG resolver. + adapters over the shared storage. Retain its exact preferred extension as + `codec_selector`, resolve it through the payload-domain mesh or DAG + resolver, and retain the format metadata required by automatic saving. 12. Add DAG storage convenience functions that supply - `dag::codec::from_extension`, and migrate `dag_builder` and - `dag_convert_debug` to them. + `dag::codec::from_extension`, and mesh storage convenience functions that + select the configured terrain or glTF codec. Migrate `dag_builder`, + `dag_convert_debug`, and mesh storage consumers without exposing codec + objects at application call sites. 13. Migrate the existing octree storage aliases and all other application callers. 14. Preserve the current 3D destructor-save behaviour until all callers have explicit index finalization. -15. Instantiate the shared storage tests with `raster_store::StoreTraits` +15. Preserve `StorageSettings::allow_overwrite`, including the DAG builder's + overwrite mode and repeat debug export. Replace process termination on a + rejected overwrite with `AlreadyExists` in the storage-level expected + error. +16. Add resolver tests for `.terrain`, `.glb`, `.gltf`, and `.bin` open/read + dispatch, plus explicit failure for an unknown preferred extension. +17. Instantiate the shared storage tests with `raster_store::StoreTraits` using a test-only path mapping and codec. This proves the storage templates contain no hidden `octree::Id` dependency without defining a stable RF layout, codec, or disk format. @@ -725,7 +836,9 @@ Add focused codec tests using single-file, multi-file, read/write, and write-only test codecs before depending on the raster payload implementation. Test that a pre-refactor DAG fixture opens through the new resolver and that a new deterministic `.bin` payload matches the Phase 0 golden bytes and remains -readable through the unchanged ZPP serialization functions. +readable through the unchanged ZPP serialization functions. Test unknown +layout IDs, malformed index metadata, invalid hierarchy keys, and retained +underlying open/codec errors through `std::expected`. Exit criterion: all existing applications build and all Phase 0 fixtures pass through the shared runtime codec and storage implementation. Existing DAG @@ -747,7 +860,8 @@ mesh codec or second storage implementation remains under `octree`. - different path counts and endings; - error propagation after a partially failed multi-file hard link; - conversion between terrain and glTF; - - overwrite rejection; and + - overwrite rejection; + - overwrite-enabled replacement; and - missing-file, hard-link, decode, and encode error propagation. 5. Add `sf::validate_index()`, returning `sf::InvalidTopology` with the offending key when it encounters `Inner`. @@ -856,6 +970,7 @@ No formatting-only pass or unrelated refactor belongs in these commits. | `octree/storage/RawStorage.h` | `store/RawStorage.h` | | `octree/storage/Storage.h` | `store/Storage.h` | | `octree/storage/IndexedStorage.h` | `store/IndexedStorage.h` | +| `octree::StorageSettings::allow_overwrite` | `store::StorageSettings::allow_overwrite`, preserving default and enabled behaviour | | `octree/storage/helpers.*` | generic scan helpers plus 3D format adapter | | `octree/disk/IndexFile.h` | versioned 3D format adapter under `octree` | | DAG serializers in `dag_node.h` and `encoded.h` | `dag_builder/serialization.h` | @@ -880,7 +995,7 @@ No formatting-only pass or unrelated refactor belongs in these commits. | Invalid `Inner` nodes reach SF merge dispatch | Validate every SF input first and return the offending key in a typed error | | SF subtree copying is generalized before RF requirements exist | Keep it in `sf_merger`; reconsider extraction with `rf_merger` | | A paired-walker API is fixed before RF semantics are known | Defer its action algebra until `rf_merger` requirements are defined | -| Multi-file hard linking fails partway through | Remove links created by the failed `copy_from()` before returning | +| Multi-file hard linking fails partway through | Stop immediately, leave the node unindexed, propagate the error, and abort the overall operation; partial files may remain | | Incompatible codecs return the same path list | Treat path-list equality as a codec contract and test every concrete codec pairing | | Output-only codec is selected for required input | Return a clear `UnsupportedOperation` error | | Shared code accumulates mesh or provisional RF policy | Dependency tests/review against the source boundary | From c6ef9e5a1f75b13a0678d86ecfad9ee1d3aae51a Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Thu, 30 Jul 2026 11:00:37 +0200 Subject: [PATCH 30/61] plan++ --- docs/raster-store/README.md | 1 + docs/raster-store/refactor-plan.md | 45 +++++++++++++++++++++--------- docs/raster-store/todo.md | 5 ++++ 3 files changed, 38 insertions(+), 13 deletions(-) create mode 100644 docs/raster-store/todo.md diff --git a/docs/raster-store/README.md b/docs/raster-store/README.md index 5ebae8f5..04429875 100644 --- a/docs/raster-store/README.md +++ b/docs/raster-store/README.md @@ -15,6 +15,7 @@ baseline, not a finalized binary-format specification. ## Plans - [Store refactoring plan](refactor-plan.md) +- [Raster store TODO](todo.md) - [DRAFT RF builder plan archive](rf_builder.md) - [DRAFT RF merger plan archive](rf_merger.md) diff --git a/docs/raster-store/refactor-plan.md b/docs/raster-store/refactor-plan.md index 66574ef4..f881a114 100644 --- a/docs/raster-store/refactor-plan.md +++ b/docs/raster-store/refactor-plan.md @@ -76,6 +76,10 @@ Persistent raster-fundamentalis formats, adapters, and tools are later work. rejected overwrites return `AlreadyExists` through `std::expected`, and enabling it preserves the existing DAG-builder overwrite and debug-export behaviour. +- The existing cache implementations are currently non-functional. Port their + public API where it remains useful, keep application call sites building, and + provide compile coverage only. Cache behaviour is not a compatibility + requirement of this refactor. - The `.png` written beside changed meshes by `sf_merger::NodeWriter` is an unmanaged debug artifact. It is not part of a logical node, is not returned by `Codec::paths()`, and is not indexed or copied by storage. Its existing @@ -149,7 +153,7 @@ Before moving code, tests must lock down the following 3D behaviour: | Overwrite setting | `StorageSettings::allow_overwrite`, default `false`; enabled writes replace existing payloads | Compatibility means that the refactored code can open datasets written before -the refactor and produces datasets that the pre-refactor code can open. Exact +the refactor. Pre-refactor readers are not tested against post-refactor outpus. Exact byte-for-byte rewriting of an unordered index map is not required, but the serialized schema and values must remain compatible. @@ -233,6 +237,10 @@ trees including `Inner`. must retain this error and `CopyError` when propagating failures; neither is reduced to a log message, assertion, or generic boolean. +For this refactor, `sf::validate_index()` checks only for `Inner`. It does not +validate other structural or disk/filesystem invariants. Possible future +extensions are recorded in [todo.md](todo.md). + Temporary forwarding headers and aliases under `octree` are allowed during migration. They must not contain a second implementation. @@ -753,8 +761,7 @@ changed. 6. Run the same index-transition and DFS/BFS tests with both trait types, including deterministic child order, invalid-key errors through `std::expected`, maximum-depth children, and explicit traversal roots. -7. Provide temporary `octree` aliases so downstream migration is separate - from the algorithm extraction. +7. Provide temporary `octree` aliases so downstream migration is separate where this makes sense, otherwise migrate downstream immediately. Exit criterion: 2D and 3D keys pass the same topology suite; existing 3D callers still build through aliases; no filesystem code has changed. @@ -794,8 +801,9 @@ strategy pointer. 4. Split the current extension-dispatching `octree::MeshCodec` into: - a terrain codec; and - one glTF codec configured for binary `.glb` or JSON `.gltf`. -5. Move copy error, raw storage, caches, logical storage, and indexed storage - into `store`. +5. Move copy error, raw storage, logical storage, and indexed storage into + `store`. Port the existing cache API where useful for source compatibility, + but do not require cache behaviour tests. 6. Make `RawStorage` exclusively own a configured `std::unique_ptr>`. `Storage` owns it transitively through raw storage; remove the codec template parameter from every storage type. @@ -864,17 +872,23 @@ mesh codec or second storage implementation remains under `octree`. - overwrite-enabled replacement; and - missing-file, hard-link, decode, and encode error propagation. 5. Add `sf::validate_index()`, returning `sf::InvalidTopology` with the - offending key when it encounters `Inner`. -6. Apply the validator to SF-builder output finalization, SF-merger merge and - cut inputs, SF-merger output, and the DAG builder's SF input. Do not apply - it when opening DAG datasets, through generic octree/store adapters, or in - the diagnostic `sf_index_browser`. + offending key when it encounters `Inner`. Do not add other validation rules + in this refactor. +6. Apply the validator to SF-merger merge and cut inputs and the DAG builder's + SF input before processing. Apply it to SF-builder and SF-merger output + after the completed `terrain.index` has been written. If output validation + fails, retain the written index and payloads for diagnosis, propagate + `sf::InvalidTopology` to the command line, and report that the output is + invalid. Do not apply the validator when opening DAG datasets, through + generic octree/store adapters, or in the diagnostic `sf_index_browser`. 7. Keep SF recursion, subtree traversal, and mesh policy in `sf_merger`. Change its subtree and cut call chains to propagate validation and `copy_from()` failures through `std::expected` to the application boundary. 8. Add integration tests proving: - valid `Leaf`/`Virtual` SF merge behaviour is unchanged; - an SF input containing `Inner` fails validation before merge dispatch; + - invalid SF output writes `terrain.index` for diagnosis and returns + `sf::InvalidTopology` to the application boundary; - an unchanged SF subtree is hard-linked and a changed boundary node is newly written; and - an unchanged leaf in the SF cut path is hard-linked while a clipped leaf @@ -882,8 +896,9 @@ mesh codec or second storage implementation remains under `octree`. Exit criterion: one-node copying works through `Codec::paths()`, SF consumers reject `Inner` with a typed error before processing, existing valid SF merge -and cut behaviour is preserved, and neither a shared subtree copier nor a -paired-tree walker has been introduced. +and cut behaviour is preserved, invalid SF output remains inspectable with a +written `terrain.index`, and neither a shared subtree copier nor a paired-tree +walker has been introduced. ### Phase 5 — Cleanup and documentation @@ -932,6 +947,10 @@ belong with their consumers: SF-builder output validation in `unittests_sfmerger`, and DAG-builder SF-input validation in `unittests_dagbuilder`. `sf_index_browser` remains unvalidated by design. +Cache migration has compile coverage only. Existing cache application call +sites must continue to build where the API remains meaningful, but no cache +behaviour or compatibility test is required. + During implementation: 1. Build in `$source_dir/build/$config_name`. @@ -963,7 +982,7 @@ No formatting-only pass or unrelated refactor belongs in these commits. | `StrategyRegister.h` | explicit dimension-adapter lookup functions | | `strategy/Flat.h` | `octree/store_layout/Flat.h` | | `strategy/LevelAndCoordinateDirectories.h` | `octree/store_layout/LevelAndCoordinateDirectories.h` | -| `octree/storage/cache/*` | `store/cache/*` | +| `octree/storage/cache/*` | `store/cache/*` where useful; compile compatibility only | | `octree/storage/codec/Codec.h` | runtime `store/Codec.h` | | `octree/storage/codec/DefaultCodec.h` | runtime `store/codec/ZppBits.h` | | `octree/storage/codec/MeshCodec.h` | `mesh/codec/Terrain.h` and configured `mesh/codec/Gltf.h` | diff --git a/docs/raster-store/todo.md b/docs/raster-store/todo.md new file mode 100644 index 00000000..fe9bf14a --- /dev/null +++ b/docs/raster-store/todo.md @@ -0,0 +1,5 @@ +# Raster store TODO + +- Consider extending `sf::validate_index()` beyond rejecting `Inner` after + additional SF invariants and their required error reporting are defined. + This is not part of the shared-store refactor. From 7bc6a3503afaf93c466f3b9c2f2acfe6057b5df4 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Thu, 30 Jul 2026 13:12:20 +0200 Subject: [PATCH 31/61] Combine path mapping and codec phases --- docs/raster-store/refactor-plan.md | 110 ++++++++++++++++------------- 1 file changed, 62 insertions(+), 48 deletions(-) diff --git a/docs/raster-store/refactor-plan.md b/docs/raster-store/refactor-plan.md index f881a114..a0a9d995 100644 --- a/docs/raster-store/refactor-plan.md +++ b/docs/raster-store/refactor-plan.md @@ -766,82 +766,95 @@ changed. Exit criterion: 2D and 3D keys pass the same topology suite; existing 3D callers still build through aliases; no filesystem code has changed. -### Phase 2 — Replace disk layout strategies +### Phase 2 — Introduce path mappings and runtime codecs + +This phase introduces the extensionless layout and runtime codec pieces +together. It does not cut production storage over to them yet. The existing +storage, layout strategies, and static codecs remain temporarily as the +working compatibility path until Phase 3 can replace the complete +layout-plus-codec path construction in one step. 1. Add `store::NodePath`, `store::PathMapping`, and `store::Layout`. -2. Port the two existing 3D layouts to ordinary function pairs without +2. Add the stateful `store::Codec` interface with `paths()`, `read()`, + and `write()`, plus `CodecError`. +3. Add the runtime `store::codec::ZppBits`, preserving the existing `.bin` + path and serialized payload bytes. +4. Consolidate the DAG serialization functions in + `src/dag_builder/serialization.h` without changing their serialized field + order, meshoptimizer encoding, or JPEG texture encoding. +5. Add the terrain codec and one glTF codec configured for binary `.glb` or + JSON `.gltf`. Keep the current extension-dispatching `octree::MeshCodec` + only as temporary production compatibility glue until the Phase 3 cutover. +6. Port the two existing 3D layouts to ordinary function pairs without changing stable IDs. The mappings return `level-index` and `level/x/y/z` without file endings. -3. Replace the singleton strategy registry with explicit `from_id()` and - `all()` functions in the 3D adapter. -4. Move the legacy preferred extension out of generic `Layout` and retain it - in the 3D format adapter. -5. Port legacy layout discovery so it strips recognized 3D file endings before - calling `node_path_to_key()`. -6. Switch node-path and layout-detection tests to the new implementation. -7. Delete the old strategy base class, registration machinery, and concrete - strategy classes once no call site uses them. - -Exit criterion: the legacy 3D adapter plus codecs resolve all Phase 0 fixtures -to identical physical payload paths; generic `Layout` contains no extension; -there is no layout inheritance, RTTI lookup, static registrar, or owning -strategy pointer. +7. Add explicit `from_id()` and `all()` lookup functions in the 3D adapter. + Keep the singleton strategy registry only for the old production storage + path until Phase 3. +8. Compose each new 3D mapping with each applicable runtime codec in tests and + prove that they resolve all Phase 0 fixtures to identical physical payload + paths. +9. Add focused codec tests using single-file, multi-file, read/write, and + write-only test codecs. Test stable path ordering, unsupported operations, + directory creation, and conversion of domain errors to `CodecError`. + +Exit criterion: the extensionless mappings and runtime codecs together resolve +all Phase 0 fixtures to their existing physical payload paths; generic +`Layout` contains no extension; the new codec tests pass; existing production +storage and all callers still build unchanged through the temporary legacy +path. ### Phase 3 — Generalize storage and index lifecycle -1. Add the stateful `store::Codec` interface with `paths()`, `read()`, - and `write()`. -2. Move the current generic `octree::ZppBitsCodec` to the runtime - `store::codec::ZppBits`, preserving its `.bin` paths and serialized - payload bytes. -3. Consolidate the DAG serialization functions in - `src/dag_builder/serialization.h` without changing their serialized field - order, meshoptimizer encoding, or JPEG texture encoding. -4. Split the current extension-dispatching `octree::MeshCodec` into: - - a terrain codec; and - - one glTF codec configured for binary `.glb` or JSON `.gltf`. -5. Move copy error, raw storage, logical storage, and indexed storage into +1. Cut the production path construction over to the Phase 2 + `store::Layout` and runtime codecs. Move the legacy preferred extension + out of layout state and retain it as the 3D format adapter's codec selector. +2. Port legacy layout discovery so it recognizes codec endings through the + supplied resolver, strips the accepted ending, and then calls + `node_path_to_key()`. +3. Move copy error, raw storage, logical storage, and indexed storage into `store`. Port the existing cache API where useful for source compatibility, but do not require cache behaviour tests. -6. Make `RawStorage` exclusively own a configured +4. Make `RawStorage` exclusively own a configured `std::unique_ptr>`. `Storage` owns it transitively through raw storage; remove the codec template parameter from every storage type. -7. Replace every embedded `octree::Id` with `Traits::Key`. -8. Make every raw file operation obtain its complete file list through +5. Replace every embedded `octree::Id` with `Traits::Key`. +6. Make every raw file operation obtain its complete file list through `Codec::paths()`. `has()` requires every listed file, and `remove()` removes every listed file. -9. Keep domain-specific mesh codecs outside the shared module under +7. Keep domain-specific mesh codecs outside the shared module under `mesh::codec`. -10. Add the function-pointer-based `IndexFormat`, `IndexMetadata`, and - typed format/open errors. Split generic index maintenance from 3D index - serialization and legacy folder discovery. -11. Keep the current 3D `terrain.index` DTO and open functions as compatibility - adapters over the shared storage. Retain its exact preferred extension as - `codec_selector`, resolve it through the payload-domain mesh or DAG - resolver, and retain the format metadata required by automatic saving. -12. Add DAG storage convenience functions that supply +8. Add the function-pointer-based `IndexFormat`, `IndexMetadata`, and + typed format/open errors. Split generic index maintenance from 3D index + serialization and legacy folder discovery. +9. Keep the current 3D `terrain.index` DTO and open functions as compatibility + adapters over the shared storage. Retain its exact preferred extension as + `codec_selector`, resolve it through the payload-domain mesh or DAG + resolver, and retain the format metadata required by automatic saving. +10. Add DAG storage convenience functions that supply `dag::codec::from_extension`, and mesh storage convenience functions that select the configured terrain or glTF codec. Migrate `dag_builder`, `dag_convert_debug`, and mesh storage consumers without exposing codec objects at application call sites. -13. Migrate the existing octree storage aliases and all other application +11. Migrate the existing octree storage aliases and all other application callers. -14. Preserve the current 3D destructor-save behaviour until all callers have +12. Preserve the current 3D destructor-save behaviour until all callers have explicit index finalization. -15. Preserve `StorageSettings::allow_overwrite`, including the DAG builder's +13. Preserve `StorageSettings::allow_overwrite`, including the DAG builder's overwrite mode and repeat debug export. Replace process termination on a rejected overwrite with `AlreadyExists` in the storage-level expected error. -16. Add resolver tests for `.terrain`, `.glb`, `.gltf`, and `.bin` open/read +14. Add resolver tests for `.terrain`, `.glb`, `.gltf`, and `.bin` open/read dispatch, plus explicit failure for an unknown preferred extension. -17. Instantiate the shared storage tests with `raster_store::StoreTraits` +15. Instantiate the shared storage tests with `raster_store::StoreTraits` using a test-only path mapping and codec. This proves the storage templates contain no hidden `octree::Id` dependency without defining a stable RF layout, codec, or disk format. +16. Delete the old strategy base class, strategy registry, concrete strategy + classes, static codec concept and codecs, and their temporary compatibility + glue once no call site uses them. -Add focused codec tests using single-file, multi-file, read/write, and -write-only test codecs before depending on the raster payload implementation. Test that a pre-refactor DAG fixture opens through the new resolver and that a new deterministic `.bin` payload matches the Phase 0 golden bytes and remains readable through the unchanged ZPP serialization functions. Test unknown @@ -851,7 +864,8 @@ underlying open/codec errors through `std::expected`. Exit criterion: all existing applications build and all Phase 0 fixtures pass through the shared runtime codec and storage implementation. Existing DAG payload bytes and `.bin` paths remain compatible. No extension-dispatching -mesh codec or second storage implementation remains under `octree`. +mesh codec, layout inheritance, RTTI lookup, static registrar, owning strategy +pointer, or second storage implementation remains under `octree`. ### Phase 4 — Harden node reuse and enforce SF topology From 60de87a6d133f9403cc5c905b5654ea4956933cc Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Thu, 30 Jul 2026 13:25:44 +0200 Subject: [PATCH 32/61] Clarify codec and overwrite verification --- docs/raster-store/refactor-plan.md | 8 ++++---- docs/raster-store/todo.md | 6 ++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/raster-store/refactor-plan.md b/docs/raster-store/refactor-plan.md index a0a9d995..379fb7a6 100644 --- a/docs/raster-store/refactor-plan.md +++ b/docs/raster-store/refactor-plan.md @@ -738,7 +738,6 @@ No production behaviour changes. 5. Add storage tests for: - matching-extension hard links; - different-extension decode/re-encode; - - overwrite rejection; - overwrite-enabled replacement; - indexed and unindexed opens; and - final index creation by directory scan. @@ -844,7 +843,8 @@ path. 13. Preserve `StorageSettings::allow_overwrite`, including the DAG builder's overwrite mode and repeat debug export. Replace process termination on a rejected overwrite with `AlreadyExists` in the storage-level expected - error. + error. Test that a rejected save returns `AlreadyExists` and that enabling + overwrite replaces the existing payload. 14. Add resolver tests for `.terrain`, `.glb`, `.gltf`, and `.bin` open/read dispatch, plus explicit failure for an unknown preferred extension. 15. Instantiate the shared storage tests with `raster_store::StoreTraits` @@ -882,8 +882,8 @@ pointer, or second storage implementation remains under `octree`. - different path counts and endings; - error propagation after a partially failed multi-file hard link; - conversion between terrain and glTF; - - overwrite rejection; - - overwrite-enabled replacement; and + - `copy_from()` overwrite rejection; + - `copy_from()` overwrite-enabled replacement; and - missing-file, hard-link, decode, and encode error propagation. 5. Add `sf::validate_index()`, returning `sf::InvalidTopology` with the offending key when it encounters `Inner`. Do not add other validation rules diff --git a/docs/raster-store/todo.md b/docs/raster-store/todo.md index fe9bf14a..fecdfba6 100644 --- a/docs/raster-store/todo.md +++ b/docs/raster-store/todo.md @@ -3,3 +3,9 @@ - Consider extending `sf::validate_index()` beyond rejecting `Inner` after additional SF invariants and their required error reporting are defined. This is not part of the shared-store refactor. +- Resolve the existing glTF write exception path before implementing the + runtime codec error contract. `mesh::io::gltf::save_to_path()` currently + throws when `cgltf_write_file()` fails, while codec operational failures are + intended to be returned through `std::expected`. Decide whether mesh I/O + should return a typed error or the glTF codec should catch and translate the + exception, and explicitly confirm whether changing mesh I/O is in scope. From 68983a9849435c2ac83e9e27d1213012be790a2f Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Thu, 30 Jul 2026 23:43:05 +0200 Subject: [PATCH 33/61] Enable HFA and read vector layer CRS --- cmake/SetupGDAL.cmake | 1 + src/terrainlib/Dataset.cpp | 12 +++++++++--- unittests/terrainlib/dataset.cpp | 20 ++++++++++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/cmake/SetupGDAL.cmake b/cmake/SetupGDAL.cmake index c1e2185c..a5c51da6 100644 --- a/cmake/SetupGDAL.cmake +++ b/cmake/SetupGDAL.cmake @@ -38,6 +38,7 @@ function(alp_setup_gdal) COMMITISH ${ARG_GDAL_VERSION} CMAKE_ARGUMENTS -DGDAL_BUILD_OPTIONAL_DRIVERS=OFF + -DGDAL_ENABLE_DRIVER_HFA=ON -DOGR_BUILD_OPTIONAL_DRIVERS=OFF -DBUILD_APPS=OFF -DBUILD_TESTING=OFF diff --git a/src/terrainlib/Dataset.cpp b/src/terrainlib/Dataset.cpp index 43098966..38cca735 100644 --- a/src/terrainlib/Dataset.cpp +++ b/src/terrainlib/Dataset.cpp @@ -27,6 +27,7 @@ #include #include +#include #include "ctb/Grid.hpp" #include "init.h" @@ -213,11 +214,16 @@ radix::tile::SrsBounds Dataset::bounds(const OGRSpatialReference &targetSrs) con } OGRSpatialReference Dataset::srs() const { - const char *srcWKT = m_gdal_dataset->GetProjectionRef(); - if (!strlen(srcWKT)) { + const OGRSpatialReference *source_srs = m_gdal_dataset->GetSpatialRef(); + for (int layer_index = 0; source_srs == nullptr && layer_index < m_gdal_dataset->GetLayerCount(); ++layer_index) { + if (OGRLayer *layer = m_gdal_dataset->GetLayer(layer_index)) { + source_srs = layer->GetSpatialRef(); + } + } + if (source_srs == nullptr) { throw std::runtime_error("The source dataset does not have a spatial reference system assigned"); } - auto srs = OGRSpatialReference(srcWKT); + auto srs = *source_srs; srs.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); return srs; } diff --git a/unittests/terrainlib/dataset.cpp b/unittests/terrainlib/dataset.cpp index 0e67eb54..c7f359d6 100644 --- a/unittests/terrainlib/dataset.cpp +++ b/unittests/terrainlib/dataset.cpp @@ -20,11 +20,13 @@ #include "../catch2_helpers.h" #include +#include #include "Dataset.h" #include "ctb/GlobalGeodetic.hpp" #include "ctb/GlobalMercator.hpp" #include "ctb/types.hpp" +#include "init.h" #include "srs.h" using namespace radix; @@ -106,6 +108,24 @@ TEST_CASE("datasets are as expected") { } } +TEST_CASE("vector datasets use the layer spatial reference") { + initialize_gdal_once(); + GDALDriver *driver = GetGDALDriverManager()->GetDriverByName("Memory"); + REQUIRE(driver != nullptr); + + GDALDataset *raw_dataset = driver->Create("", 0, 0, 0, GDT_Unknown, nullptr); + REQUIRE(raw_dataset != nullptr); + + OGRSpatialReference expected_srs; + REQUIRE(expected_srs.importFromEPSG(31287) == OGRERR_NONE); + expected_srs.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); + REQUIRE(raw_dataset->CreateLayer("mask", &expected_srs, wkbPolygon, nullptr) != nullptr); + + Dataset dataset(raw_dataset); + const OGRSpatialReference actual_srs = dataset.srs(); + CHECK(actual_srs.IsSame(&expected_srs)); +} + TEST_CASE("bbox width pixels") { auto d_mgi = Dataset(ALP_TEST_DATA_DIR "/austria/at_mgi.tif"); auto d_wgs84 = Dataset(ALP_TEST_DATA_DIR "/austria/at_wgs84.tif"); From 3edc572941650813634e7fba20f59e3535ae2c12 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Fri, 31 Jul 2026 00:08:28 +0200 Subject: [PATCH 34/61] Enable GeoPackage vector support --- cmake/SetupGDAL.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmake/SetupGDAL.cmake b/cmake/SetupGDAL.cmake index a5c51da6..d3d111ab 100644 --- a/cmake/SetupGDAL.cmake +++ b/cmake/SetupGDAL.cmake @@ -40,6 +40,8 @@ function(alp_setup_gdal) -DGDAL_BUILD_OPTIONAL_DRIVERS=OFF -DGDAL_ENABLE_DRIVER_HFA=ON -DOGR_BUILD_OPTIONAL_DRIVERS=OFF + -DOGR_ENABLE_DRIVER_GPKG=ON + -DOGR_ENABLE_DRIVER_SQLITE=ON -DBUILD_APPS=OFF -DBUILD_TESTING=OFF -DBUILD_PYTHON_BINDINGS=OFF @@ -47,6 +49,7 @@ function(alp_setup_gdal) -DBUILD_CSHARP_BINDINGS=OFF -DGDAL_USE_ICONV=OFF -DGDAL_USE_EXTERNAL_LIBS=OFF + -DGDAL_USE_SQLITE3=ON "-DCMAKE_INSTALL_RPATH=\$ORIGIN/../../proj/lib" ) From ddde549cb99b373f51a214f6466e15b51f2b5d2c Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Sat, 1 Aug 2026 21:46:31 +0200 Subject: [PATCH 35/61] Simplify vector masks before triangulation --- cmake/SetupGDAL.cmake | 16 +- docs/raster-store/golden-e2e.sh | 288 ++++++++++++++++++++++++++++++++ src/CMakeLists.txt | 2 +- src/sf_merger/mask.h | 100 +++++++++-- unittests/CMakeLists.txt | 1 + unittests/sf_merger/mask.cpp | 57 +++++++ 6 files changed, 447 insertions(+), 17 deletions(-) create mode 100755 docs/raster-store/golden-e2e.sh create mode 100644 unittests/sf_merger/mask.cpp diff --git a/cmake/SetupGDAL.cmake b/cmake/SetupGDAL.cmake index d3d111ab..6be25b74 100644 --- a/cmake/SetupGDAL.cmake +++ b/cmake/SetupGDAL.cmake @@ -21,17 +21,24 @@ if(NOT COMMAND alp_setup_cmake_project) endif() function(alp_setup_gdal) - set(oneValueArgs GDAL_VERSION PROJ_VERSION) + set(oneValueArgs GDAL_VERSION GEOS_VERSION PROJ_VERSION) cmake_parse_arguments(ARG "" "${oneValueArgs}" "" ${ARGN}) - if(NOT ARG_GDAL_VERSION OR NOT ARG_PROJ_VERSION) - message(FATAL_ERROR "alp_setup_gdal() needs: GDAL_VERSION PROJ_VERSION ") + if(NOT ARG_GDAL_VERSION OR NOT ARG_GEOS_VERSION OR NOT ARG_PROJ_VERSION) + message(FATAL_ERROR "alp_setup_gdal() needs: GDAL_VERSION GEOS_VERSION PROJ_VERSION ") endif() alp_setup_cmake_project(proj URL https://github.com/OSGeo/PROJ.git COMMITISH ${ARG_PROJ_VERSION} CMAKE_ARGUMENTS -DBUILD_TESTING=OFF -DBUILD_APPS=OFF) find_package(PROJ CONFIG REQUIRED) - set(_proj_install "${ALP_PROJ_INSTALL_DIR}") + alp_setup_cmake_project(geos + URL https://github.com/libgeos/geos.git + COMMITISH ${ARG_GEOS_VERSION} + CMAKE_ARGUMENTS + -DBUILD_SHARED_LIBS=OFF + -DBUILD_TESTING=OFF + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + ) alp_setup_cmake_project(gdal URL https://github.com/OSGeo/gdal.git @@ -49,6 +56,7 @@ function(alp_setup_gdal) -DBUILD_CSHARP_BINDINGS=OFF -DGDAL_USE_ICONV=OFF -DGDAL_USE_EXTERNAL_LIBS=OFF + -DGDAL_USE_GEOS=ON -DGDAL_USE_SQLITE3=ON "-DCMAKE_INSTALL_RPATH=\$ORIGIN/../../proj/lib" ) diff --git a/docs/raster-store/golden-e2e.sh b/docs/raster-store/golden-e2e.sh new file mode 100755 index 00000000..208ecb92 --- /dev/null +++ b/docs/raster-store/golden-e2e.sh @@ -0,0 +1,288 @@ +#!/usr/bin/env bash + +# Repeatable, non-unit end-to-end verification for the current 3D SF/DAG path. +# Inputs are prepared once under INPUT_ROOT. Completed steps are skipped through +# explicit state markers, so an interrupted SF or DAG run can be resumed. + +set -Eeuo pipefail + +readonly SOURCE_DIR="/home/codex/Documents/alpine-terrain-builder/terrain-builder-raster-store" +readonly BUILD_DIR="${SOURCE_DIR}/build/hfa-vector-srs" +readonly RUN_ROOT="/data/scratch/codex/alpine-terrain-builder-golden-e2e" +readonly INPUT_ROOT="${RUN_ROOT}/inputs" +readonly REFERENCE_ROOT="${RUN_ROOT}/reference" +readonly LOG_ROOT="${RUN_ROOT}/logs" +readonly STATE_ROOT="${RUN_ROOT}/state" + +readonly SF_BUILDER="${BUILD_DIR}/src/sf_builder/sf-builder" +readonly SF_MERGER="${BUILD_DIR}/src/sf_merger/sf-merger" +readonly DAG_BUILDER="${BUILD_DIR}/src/dag_builder/dag-builder" + +# This VRT is a 4x4 km, native-resolution window into the original HFA dataset. +readonly GS_VRT="${INPUT_ROOT}/grossglockner-gs-4km/grossglockner-gs-4km.vrt" +readonly BASEMAP_TILES="${INPUT_ROOT}/basemap" +readonly GATAKI_TILES="${INPUT_ROOT}/gataki" +# The exact Tirol border exercises the sf-merger's built-in 0.1 m +# topology-preserving mask simplification. +readonly TIROL_MASK="${INPUT_ROOT}/tirol-boundary/benchmark-shape/exact/tirol.shp" + +readonly SF_ROOT="${REFERENCE_ROOT}/sf" +readonly DAG_ROOT="${REFERENCE_ROOT}/dag" +readonly SF_BASEMAP="${SF_ROOT}/grossglockner-gs-basemap-terrain" +readonly SF_GATAKI="${SF_ROOT}/grossglockner-gs-gataki-terrain" +readonly SF_MERGED="${SF_ROOT}/grossglockner-gs-merged-terrain" +readonly DAG_MERGED="${DAG_ROOT}/grossglockner-gs-merged-terrain" + +readonly SF_TARGET_LEVEL="${SF_TARGET_LEVEL:-15}" +readonly SF_THREADS="${SF_THREADS:-12}" +readonly MIN_TEXTURE_LEVEL=12 +readonly MAX_TEXTURE_LEVEL=19 + +mkdir -p "${SF_ROOT}" "${DAG_ROOT}" "${LOG_ROOT}" "${STATE_ROOT}" +exec > >(tee -a "${LOG_ROOT}/golden-e2e.log") 2>&1 + +timestamp() +{ + date --iso-8601=seconds +} + +run_timed() +{ + local name="$1" + shift + + local start_epoch end_epoch elapsed status + start_epoch="$(date +%s)" + printf '[%s] START %s\n' "$(timestamp)" "${name}" + + set +e + "$@" + status=$? + set -e + + end_epoch="$(date +%s)" + elapsed=$((end_epoch - start_epoch)) + printf '%s\t%s\t%s\t%s\t%s\n' \ + "${name}" "${start_epoch}" "${end_epoch}" "${elapsed}" "${status}" \ + >> "${LOG_ROOT}/timings.tsv" + printf '[%s] END %s status=%s elapsed_seconds=%s\n' \ + "$(timestamp)" "${name}" "${status}" "${elapsed}" + + if ((status != 0)); then + return "${status}" + fi +} + +require_file() +{ + if [[ ! -f "$1" ]]; then + printf 'Required file is missing: %s\n' "$1" >&2 + exit 2 + fi +} + +require_directory() +{ + if [[ ! -d "$1" ]]; then + printf 'Required directory is missing: %s\n' "$1" >&2 + exit 2 + fi +} + +require_executable() +{ + if [[ ! -x "$1" ]]; then + printf 'Required executable is missing: %s\n' "$1" >&2 + exit 2 + fi +} + +verify_snapshot() +{ + local snapshot="$1" + local extension="$2" + local payload_count + + require_file "${snapshot}/terrain.index" + if [[ ! -s "${snapshot}/terrain.index" ]]; then + printf 'Index is empty: %s\n' "${snapshot}/terrain.index" >&2 + return 1 + fi + + payload_count="$(find "${snapshot}" -type f -name "*${extension}" | wc -l)" + if ((payload_count == 0)); then + printf 'No %s payloads found in %s\n' "${extension}" "${snapshot}" >&2 + return 1 + fi + printf 'Verified %s: %s payloads\n' "${snapshot}" "${payload_count}" +} + +build_sf() +{ + local name="$1" + local textures="$2" + local output="$3" + local marker="${STATE_ROOT}/${name}.complete" + + if [[ -f "${marker}" ]]; then + printf '[%s] SKIP %s: completion marker exists\n' "$(timestamp)" "${name}" + verify_snapshot "${output}" ".terrain" + return + fi + + mkdir -p "${output}" + run_timed "${name}" \ + "${SF_BUILDER}" \ + --dataset "${GS_VRT}" \ + --textures "${textures}" \ + --min-texture-level "${MIN_TEXTURE_LEVEL}" \ + --max-texture-level "${MAX_TEXTURE_LEVEL}" \ + --mesh-srs EPSG:4978 \ + --verbosity info \ + batch \ + --target-level "${SF_TARGET_LEVEL}" \ + --output "${output}" \ + --format .terrain \ + --threads "${SF_THREADS}" + + verify_snapshot "${output}" ".terrain" + touch "${marker}" +} + +merge_sf() +{ + local marker="${STATE_ROOT}/merge_sf.complete" + + if [[ -f "${marker}" ]]; then + printf '[%s] SKIP merge_sf: completion marker exists\n' "$(timestamp)" + verify_snapshot "${SF_MERGED}" ".terrain" + return + fi + + if [[ -d "${SF_MERGED}" ]] && [[ -n "$(find "${SF_MERGED}" -mindepth 1 -print -quit)" ]]; then + printf 'Partial merge output exists at %s.\n' "${SF_MERGED}" >&2 + printf 'The current sf-merger cannot safely resume this output; refusing to overwrite it.\n' >&2 + return 1 + fi + + mkdir -p "${SF_MERGED}" + run_timed merge_sf \ + "${SF_MERGER}" merge \ + --base "${SF_GATAKI}" \ + --new "${SF_BASEMAP}" \ + --mask "${TIROL_MASK}" \ + --output "${SF_MERGED}" \ + --verbosity info + + verify_snapshot "${SF_MERGED}" ".terrain" + touch "${marker}" +} + +build_dag() +{ + local marker="${STATE_ROOT}/build_dag.complete" + local continuation=(--overwrite) + + if [[ -f "${marker}" ]]; then + printf '[%s] SKIP build_dag: completion marker exists\n' "$(timestamp)" + verify_snapshot "${DAG_MERGED}" ".bin" + return + fi + + mkdir -p "${DAG_MERGED}" + if [[ -s "${DAG_MERGED}/terrain.index" ]]; then + continuation=(--resume) + fi + + run_timed build_dag \ + "${DAG_BUILDER}" \ + --input "${SF_MERGED}" \ + --output "${DAG_MERGED}" \ + "${continuation[@]}" \ + --verbosity info + + verify_snapshot "${DAG_MERGED}" ".bin" + touch "${marker}" +} + +write_manifest() +{ + local name="$1" + local snapshot="$2" + local output="${LOG_ROOT}/${name}.sha256" + + ( + cd "${snapshot}" + find . -type f ! -name terrain.index -print0 \ + | sort -z \ + | xargs -0 -r sha256sum + ) > "${output}" +} + +record_hard_links() +{ + local merged_count=0 + local linked_to_gataki=0 + local linked_to_basemap=0 + local newly_written=0 + local merged_file relative merged_inode candidate_inode + + while IFS= read -r -d '' merged_file; do + relative="${merged_file#"${SF_MERGED}/"}" + merged_inode="$(stat -c '%d:%i' "${merged_file}")" + ((merged_count += 1)) + + if [[ -f "${SF_GATAKI}/${relative}" ]]; then + candidate_inode="$(stat -c '%d:%i' "${SF_GATAKI}/${relative}")" + if [[ "${merged_inode}" == "${candidate_inode}" ]]; then + ((linked_to_gataki += 1)) + continue + fi + fi + + if [[ -f "${SF_BASEMAP}/${relative}" ]]; then + candidate_inode="$(stat -c '%d:%i' "${SF_BASEMAP}/${relative}")" + if [[ "${merged_inode}" == "${candidate_inode}" ]]; then + ((linked_to_basemap += 1)) + continue + fi + fi + + ((newly_written += 1)) + done < <(find "${SF_MERGED}" -type f -name '*.terrain' -print0) + + printf 'merged_payloads=%s\nlinked_to_gataki=%s\nlinked_to_basemap=%s\nnewly_written=%s\n' \ + "${merged_count}" "${linked_to_gataki}" "${linked_to_basemap}" "${newly_written}" \ + | tee "${LOG_ROOT}/merge-hard-links.txt" +} + +require_file "${GS_VRT}" +require_file "${TIROL_MASK}" +require_directory "${BASEMAP_TILES}" +require_directory "${GATAKI_TILES}" +require_executable "${SF_BUILDER}" +require_executable "${SF_MERGER}" +require_executable "${DAG_BUILDER}" + +{ + printf 'run_started=%s\n' "$(timestamp)" + printf 'git_commit=%s\n' "$(git -C "${SOURCE_DIR}" rev-parse HEAD)" + printf 'git_status=%q\n' "$(git -C "${SOURCE_DIR}" status --porcelain=v1 --branch)" + printf 'sf_target_level=%s\n' "${SF_TARGET_LEVEL}" + printf 'sf_threads=%s\n' "${SF_THREADS}" + printf 'min_texture_level=%s\n' "${MIN_TEXTURE_LEVEL}" + printf 'max_texture_level=%s\n' "${MAX_TEXTURE_LEVEL}" + printf 'cpu_count=%s\n' "$(nproc)" +} >> "${LOG_ROOT}/run-metadata.txt" + +build_sf build_sf_basemap_terrain "${BASEMAP_TILES}" "${SF_BASEMAP}" +build_sf build_sf_gataki_terrain "${GATAKI_TILES}" "${SF_GATAKI}" +merge_sf +record_hard_links +build_dag +run_timed manifest_sf_basemap write_manifest sf-basemap "${SF_BASEMAP}" +run_timed manifest_sf_gataki write_manifest sf-gataki "${SF_GATAKI}" +run_timed manifest_sf_merged write_manifest sf-merged "${SF_MERGED}" +run_timed manifest_dag_merged write_manifest dag-merged "${DAG_MERGED}" + +printf '[%s] Golden end-to-end run complete\n' "$(timestamp)" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a33beb5c..b419708d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -14,7 +14,7 @@ alp_setup_cmake_project(tbb URL https://github.com/uxlfoundation/oneTBB.git COMM find_package(TBB REQUIRED) include(../cmake/SetupGDAL.cmake) -alp_setup_gdal(GDAL_VERSION v3.10.3 PROJ_VERSION 9.6.0) +alp_setup_gdal(GDAL_VERSION v3.10.3 GEOS_VERSION 3.13.1 PROJ_VERSION 9.6.0) alp_setup_cmake_project(cgal URL https://github.com/CGAL/cgal.git COMMITISH "v6.0.1" CMAKE_ARGUMENTS -DBUILD_TESTING=OFF -DWITH_examples=OFF) find_package(CGAL REQUIRED) diff --git a/src/sf_merger/mask.h b/src/sf_merger/mask.h index 47f2b875..aac4a7ba 100644 --- a/src/sf_merger/mask.h +++ b/src/sf_merger/mask.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -68,7 +69,9 @@ namespace { enum class LoadErrorKind { UnsupportedFormat, FileNotFound, - EmptySource + EmptySource, + InvalidGeometry, + UnsupportedSpatialReference }; class LoadError { @@ -95,6 +98,10 @@ class LoadError { return "file not found"; case LoadErrorKind::EmptySource: return "empty input source"; + case LoadErrorKind::InvalidGeometry: + return "invalid geometry"; + case LoadErrorKind::UnsupportedSpatialReference: + return "unsupported spatial reference"; default: return "unknown error"; } @@ -109,6 +116,58 @@ class LoadError { }; namespace { +constexpr double simplification_tolerance_metres = 0.1; + +std::optional simplification_tolerance(const OGRSpatialReference &srs) { + if (srs.IsProjected()) { + const double metres_per_unit = srs.GetLinearUnits(); + if (metres_per_unit > 0) { + return simplification_tolerance_metres / metres_per_unit; + } + } + + if (srs.IsGeographic()) { + OGRErr error = OGRERR_NONE; + const double semi_major_axis_metres = srs.GetSemiMajor(&error); + const double radians_per_unit = srs.GetAngularUnits(); + if (error == OGRERR_NONE && semi_major_axis_metres > 0 && radians_per_unit > 0) { + return simplification_tolerance_metres / semi_major_axis_metres / radians_per_unit; + } + } + + return std::nullopt; +} + +uint64_t point_count(const OGRGeometry &geometry) { + const OGRwkbGeometryType geometry_type = wkbFlatten(geometry.getGeometryType()); + if (geometry_type == wkbPolygon) { + const OGRPolygon *polygon = geometry.toPolygon(); + uint64_t count = polygon->getExteriorRing()->getNumPoints(); + for (int i = 0; i < polygon->getNumInteriorRings(); ++i) { + count += polygon->getInteriorRing(i)->getNumPoints(); + } + return count; + } + + if (geometry_type == wkbMultiPolygon || geometry_type == wkbGeometryCollection) { + const OGRGeometryCollection *collection = geometry.toGeometryCollection(); + uint64_t count = 0; + for (int i = 0; i < collection->getNumGeometries(); ++i) { + count += point_count(*collection->getGeometryRef(i)); + } + return count; + } + + return 0; +} + +std::unique_ptr simplify_geometry( + const OGRGeometry &geometry, + const double tolerance +) { + return std::unique_ptr(geometry.SimplifyPreserveTopology(tolerance)); +} + std::optional convert_ring(const OGRLinearRing &ring, bool is_outer) { uint32_t num_points = ring.getNumPoints(); if (ring.get_IsClosed()) { @@ -256,11 +315,38 @@ auto length2(const Vec &v) -> decltype(glm::dot(v, v)) { inline std::expected load_referenced_from_dataset(Dataset& mask_dataset) { GDALDataset *dataset = mask_dataset.gdalDataset(); + OGRSpatialReference srs; + // TODO: remove this try catch + try { + srs = mask_dataset.srs(); + } catch (std::runtime_error &e) { + LOG_WARN("Mask does not reference an srs, assuming WGS84"); + srs = srs::wgs84(); + // srs.SetAxisMappingStrategy(OAMS_AUTHORITY_COMPLIANT); + } + + const std::optional tolerance = simplification_tolerance(srs); + if (!tolerance) { + LOG_ERROR("Cannot express the {} m mask simplification tolerance in the source SRS", + simplification_tolerance_metres); + return std::unexpected(LoadErrorKind::UnsupportedSpatialReference); + } + MultipolygonWithHoles2 polygons; for (auto &&feature_layer_pair : dataset->GetFeatures()) { OGRGeometry *geometry = feature_layer_pair.feature->GetGeometryRef(); DEBUG_ASSERT(geometry); - process_geometry(*geometry, polygons); + + const uint64_t original_point_count = point_count(*geometry); + std::unique_ptr simplified = simplify_geometry(*geometry, *tolerance); + if (!simplified || simplified->IsEmpty() || !simplified->IsValid()) { + LOG_ERROR("Failed to simplify mask geometry while preserving its topology"); + return std::unexpected(LoadErrorKind::InvalidGeometry); + } + + LOG_DEBUG("Simplified mask with {} m tolerance from {} to {} points", + simplification_tolerance_metres, original_point_count, point_count(*simplified)); + process_geometry(*simplified, polygons); } if (polygons.is_empty()) { @@ -268,16 +354,6 @@ inline std::expected load_referenced_from_data return std::unexpected(LoadErrorKind::EmptySource); } - OGRSpatialReference srs; - // TODO: remove this try catch - try { - srs = mask_dataset.srs(); - } catch (std::runtime_error &e) { - LOG_WARN("Mask does not reference an srs, assuming WGS84"); - srs = srs::wgs84(); - // srs.SetAxisMappingStrategy(OAMS_AUTHORITY_COMPLIANT); - } - return ReferencedPolygonMask{.polygons = std::move(polygons), .srs = std::move(srs)}; } diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 27d24f95..0ea40bcc 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -116,6 +116,7 @@ endif() if(TARGET sfmergerlib) add_executable(unittests_sfmerger catch2_helpers.h + sf_merger/mask.cpp sf_merger/sphere_projector.cpp ) target_link_libraries(unittests_sfmerger PRIVATE sfmergerlib Catch2::Catch2WithMain) diff --git a/unittests/sf_merger/mask.cpp b/unittests/sf_merger/mask.cpp new file mode 100644 index 00000000..48f0ed63 --- /dev/null +++ b/unittests/sf_merger/mask.cpp @@ -0,0 +1,57 @@ +#include "../catch2_helpers.h" + +#include + +#include "mask.h" + +TEST_CASE("Mask simplification uses a fixed tenth of a metre") { + OGRSpatialReference projected; + REQUIRE(projected.importFromEPSG(31287) == OGRERR_NONE); + + const std::optional projected_tolerance = mask::simplification_tolerance(projected); + REQUIRE(projected_tolerance); + CHECK(*projected_tolerance == Catch::Approx(0.1)); + + OGRSpatialReference geographic; + REQUIRE(geographic.SetWellKnownGeogCS("WGS84") == OGRERR_NONE); + + const std::optional geographic_tolerance = mask::simplification_tolerance(geographic); + REQUIRE(geographic_tolerance); + CHECK(*geographic_tolerance == Catch::Approx( + 0.1 / geographic.GetSemiMajor() / geographic.GetAngularUnits())); +} + +TEST_CASE("Mask simplification preserves polygon topology") { + OGRLinearRing outer; + outer.addPoint(0.0, 0.0); + outer.addPoint(1.0, 0.01); + outer.addPoint(2.0, 0.0); + outer.addPoint(2.0, 2.0); + outer.addPoint(0.0, 2.0); + outer.addPoint(0.0, 0.0); + + OGRLinearRing hole; + hole.addPoint(0.5, 0.5); + hole.addPoint(1.0, 0.51); + hole.addPoint(1.5, 0.5); + hole.addPoint(1.5, 1.5); + hole.addPoint(0.5, 1.5); + hole.addPoint(0.5, 0.5); + + OGRPolygon polygon; + REQUIRE(polygon.addRing(&outer) == OGRERR_NONE); + REQUIRE(polygon.addRing(&hole) == OGRERR_NONE); + REQUIRE(polygon.IsValid()); + + const uint64_t original_point_count = mask::point_count(polygon); + std::unique_ptr simplified = mask::simplify_geometry(polygon, 0.1); + + REQUIRE(simplified); + REQUIRE_FALSE(simplified->IsEmpty()); + REQUIRE(simplified->IsValid()); + CHECK(mask::point_count(*simplified) < original_point_count); + + const OGRPolygon *simplified_polygon = simplified->toPolygon(); + REQUIRE(simplified_polygon); + CHECK(simplified_polygon->getNumInteriorRings() == 1); +} From ea6cb53d55cc18ae96c18272708b51b5d619f2ac Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Sat, 1 Aug 2026 21:48:33 +0200 Subject: [PATCH 36/61] Preserve SF format when merging --- src/sf_merger/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sf_merger/main.cpp b/src/sf_merger/main.cpp index bd20a16b..6ff4896a 100644 --- a/src/sf_merger/main.cpp +++ b/src/sf_merger/main.cpp @@ -41,7 +41,7 @@ void run(const cli::MergeArgs& args) { LOG_TRACE("Creating output dataset at {}", args.output_path); std::filesystem::create_directories(args.output_path); - octree::Storage output_dataset = octree::open_folder(args.output_path, false, octree::OpenOptions{.preferred_extension_with_dot = ".glb"}); + octree::Storage output_dataset = octree::open_folder(args.output_path, false, octree::OpenOptions{.preferred_extension_with_dot = std::string(base_dataset.layout().extension_with_dot())}); std::optional mask = flatten(map(args.mask_path, load_mask_from_path)); From 19589e3b9528717196dae2804c4637d7fa19985f Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Sun, 2 Aug 2026 11:48:55 +0200 Subject: [PATCH 37/61] Update raster storage format design --- docs/raster-store/storage-format.md | 31 +++++++++++++++-------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/docs/raster-store/storage-format.md b/docs/raster-store/storage-format.md index 54832f91..c70d706b 100644 --- a/docs/raster-store/storage-format.md +++ b/docs/raster-store/storage-format.md @@ -50,35 +50,36 @@ tile-base is a hierarchy build from raster-fundamentalis, containing all data an - should generate tiles of requested resolution and pixel type (vertex|area) on the fly - requests by url, e.g.: layer/vertex|area/resolution/z/x/y.ending - live in src/tile-server/* +- the delivery tile format is not yet defined -### unclear -- how to build tile-base from raster-fundamentalis -- details of the tile server - -## serialization / deserialization and versioning -- all files are serialised with zpp::bits in two levels - - first level contains: - - an uint64 long file type specific random magic number, generated once at coding time, as an definitive file type identifier +## serialization / deserialization envelope and versioning +-zpp::bits is a library that takes a cpp object and (de)serialises it (from)into a byte stream, +- we serialise objects with zpp::bits in two levels + - first level (the envelope) contains: + - a c++ class identifier (string like type) + - an uint64 long file type specific magic number, F5FBD3EF919428CA, as an definitive file format identifier - a version number (uint32) - an enum for the checksum, default to HandledByCompressionLib, other option None - - a string checksum for the payload, computed from the uncompressed data (default empty) + - a string checksum for the payload, computed from the uncompressed data (default empty, this is unused for now) - an enum for the compression algorithm, default to to ZStd_BestCompression, other option None. - a payload (byte vector), the second level - - the second level is a compressed byte array. the compressed payload is deserialised directly into the respective versioned data classes (tile or source attribution table) + - the second level is a compressed byte vector. the compressed payload is deserialised directly into the respective versioned data classes - data structs are stored in versioned namespaces, e.g.: `raster_store::v1::Tile` - outside the versioned namespace, there is a using declaration for the newest version -- outside the versioned namespace, there is serialization function, taking only the newest version +- outside the versioned namespace, there is a using declaration for the serialization function, taking only the newest version - outside the versioned namespace, there is a deserialization function, taking a byte stream, and returning the newest version (convert to the newest version, if the payload encodes an older version) - conversion to newer versions is done by the constructor, e.g. the v2::Tile constructor shall take a v1::Tile, and convert it to v2. once we have a v3, it would take a v2. this way we would have a conversion trail from v1 to v3. +- serialization/deserialization should be done in a templated function, receiving the class type, version, and name (string like type) as template parameters, and the object as function parameter. the functions above should be implemented in terms of the templated function, e.g. by try reading with V3, then V2, then v1. a more elegant design with a few more functions may be proposed, if it generalises nicely. I can imagine, that something with a variadic template (versioned classes) could work. - we have clear fails if - - the magic or version is unsupported + - the classname or magic is wrong, or the version is unsupported - if the checksum check fails - if the compression algorithm is missing or unsupported. - deserialization fails -- we fail by throwing in these cases -- compression: use libzstd with best compression. libzstd must be imported via the projects cmake install facility from https://github.com/AlpineMapsOrgDependencies/zstd. the context must be configured to compute and check a checksum. do compression and decompression in functions with an std::byte interface (and error handling). dispatch is done via a simple switch case. -- checksum: assert the enum is either None or HandledByCompressionLib. +- we fail by returning an unexpected in these cases +- compression: use libzstd with best compression. libzstd must be imported via the projects cmake install facility from https://github.com/AlpineMapsOrgDependencies/zstd. the context must be configured to compute and check a checksum. +- an API for compression and decompression should be created. data should be passed as std:: vector, error handling via std expected, dispatch between compression algorithms is done via a simple switch case. +- checksum: verify the enum is either None or HandledByCompressionLib. ## to be defined From 304333a7f1bc041780dd1a0f2879965de218795a Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Sun, 2 Aug 2026 16:17:26 +0200 Subject: [PATCH 38/61] Add versioned serialization envelope --- docs/raster-store/storage-format.md | 40 ++-- src/CMakeLists.txt | 12 ++ src/terrainlib/CMakeLists.txt | 2 + src/terrainlib/io/compression.cpp | 156 ++++++++++++++ src/terrainlib/io/compression.h | 67 ++++++ src/terrainlib/io/envelope.h | 140 +++++++++++++ src/terrainlib/io/envelope.inl | 132 ++++++++++++ unittests/CMakeLists.txt | 1 + unittests/terrainlib/envelope.cpp | 313 ++++++++++++++++++++++++++++ 9 files changed, 845 insertions(+), 18 deletions(-) create mode 100644 src/terrainlib/io/compression.cpp create mode 100644 src/terrainlib/io/compression.h create mode 100644 src/terrainlib/io/envelope.h create mode 100644 src/terrainlib/io/envelope.inl create mode 100644 unittests/terrainlib/envelope.cpp diff --git a/docs/raster-store/storage-format.md b/docs/raster-store/storage-format.md index c70d706b..df26fe7e 100644 --- a/docs/raster-store/storage-format.md +++ b/docs/raster-store/storage-format.md @@ -52,34 +52,38 @@ tile-base is a hierarchy build from raster-fundamentalis, containing all data an - live in src/tile-server/* - the delivery tile format is not yet defined -## serialization / deserialization envelope and versioning --zpp::bits is a library that takes a cpp object and (de)serialises it (from)into a byte stream, -- we serialise objects with zpp::bits in two levels - - first level (the envelope) contains: - - a c++ class identifier (string like type) - - an uint64 long file type specific magic number, F5FBD3EF919428CA, as an definitive file format identifier - - a version number (uint32) - - an enum for the checksum, default to HandledByCompressionLib, other option None - - a string checksum for the payload, computed from the uncompressed data (default empty, this is unused for now) - - an enum for the compression algorithm, default to to ZStd_BestCompression, other option None. - - a payload (byte vector), the second level - - the second level is a compressed byte vector. the compressed payload is deserialised directly into the respective versioned data classes +## serialization / deserialization envelope and versioning +- `zpp::bits` serialises C++ objects to byte streams and deserialises them again. +- The envelope is generic and is not specific to the raster store. +- We serialise objects with `zpp::bits` in two levels. + - The first level is an aggregate with the following fields, in this order: + - `uint64 magic`, always `F5FBD3EF919428CA`, identifying this envelope format; + - `string class_name`, identifying the payload type; + - `uint32 class_version`, identifying the versioned payload type; + - `ChecksumAlgorithm checksum_algorithm`, default `HandledByCompressionLib`, alternatively `None`; + - `string checksum`, empty when the checksum is handled by the compression library; + - `CompressionAlgorithm compression_algorithm`, default `ZstdBestCompressionWithChecksum`, alternatively `None`; + - `Bytes compressed_data`, containing the second level. + - The second level is a compressed byte vector. It is deserialised directly into the selected versioned payload class. +- The magic is shared by all payload types. `class_name` distinguishes payload types. An incompatible future envelope layout requires a new magic. - data structs are stored in versioned namespaces, e.g.: `raster_store::v1::Tile` - outside the versioned namespace, there is a using declaration for the newest version -- outside the versioned namespace, there is a using declaration for the serialization function, taking only the newest version +- outside the versioned namespace, there is a serialization wrapper function taking only the newest version - outside the versioned namespace, there is a deserialization function, taking a byte stream, and returning the newest version (convert to the newest version, if the payload encodes an older version) -- conversion to newer versions is done by the constructor, e.g. the v2::Tile constructor shall take a v1::Tile, and convert it to v2. once we have a v3, it would take a v2. this way we would have a conversion trail from v1 to v3. -- serialization/deserialization should be done in a templated function, receiving the class type, version, and name (string like type) as template parameters, and the object as function parameter. the functions above should be implemented in terms of the templated function, e.g. by try reading with V3, then V2, then v1. a more elegant design with a few more functions may be proposed, if it generalises nicely. I can imagine, that something with a variadic template (versioned classes) could work. +- Newer versions provide a static `from_previous` function, e.g. `v2::Tile::from_previous(v1::Tile)`. Static conversion functions keep the payload types aggregates, allowing `zpp::bits` to serialise them without per-type serialisation declarations. A conversion trail upgrades v1 to v2 and then v3. +- `Version` pairs version numbers with payload types. `PayloadSchema` defines the class name, supported versions, latest type, and conversion trail. +- The generic serialisation function receives the schema and version as template parameters. Deserialisation reads `class_version`, deserialises exactly that payload type, and follows the conversion trail to the latest type. It does not speculatively try other payload versions. - we have clear fails if - the classname or magic is wrong, or the version is unsupported - if the checksum check fails - if the compression algorithm is missing or unsupported. - deserialization fails - we fail by returning an unexpected in these cases -- compression: use libzstd with best compression. libzstd must be imported via the projects cmake install facility from https://github.com/AlpineMapsOrgDependencies/zstd. the context must be configured to compute and check a checksum. -- an API for compression and decompression should be created. data should be passed as std:: vector, error handling via std expected, dispatch between compression algorithms is done via a simple switch case. -- checksum: verify the enum is either None or HandledByCompressionLib. +- Compression uses libzstd at its best compression level. Libzstd is imported through the project's CMake install facility from https://github.com/AlpineMapsOrgDependencies/zstd. `ZstdBestCompressionWithChecksum` writes an embedded zstd frame checksum, which libzstd verifies while decompressing. +- `compress_with_checksum` accepts a `vector` and returns the compressed bytes plus the external checksum string. `checked_decompress` accepts both and returns the decompressed bytes. Both use `std::expected` and dispatch with a switch. +- `None` compression must be paired with `None` checksum. `ZstdBestCompressionWithChecksum` must be paired with `HandledByCompressionLib`; its external checksum string must be empty because the checksum is embedded in the zstd frame. +- Decompression rejects output larger than 1 GiB. ## to be defined diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b419708d..13dd4474 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -42,6 +42,18 @@ alp_add_git_repository(zpp_bits URL https://github.com/eyalz800/zpp_bits.git COM add_library(zpp_bits INTERFACE) target_include_directories(zpp_bits SYSTEM INTERFACE ${zpp_bits_SOURCE_DIR}) +alp_setup_cmake_project(zstd + URL https://github.com/AlpineMapsOrgDependencies/zstd.git + COMMITISH 5c7b7bad26808e6b40ac3b3d0075466e27738a9d + CMAKE_ARGUMENTS + -DZSTD_BUILD_PROGRAMS=OFF + -DZSTD_BUILD_TESTS=OFF + -DZSTD_BUILD_CONTRIB=OFF + -DZSTD_BUILD_SHARED=OFF + -DZSTD_BUILD_STATIC=ON +) +find_package(zstd CONFIG REQUIRED) + alp_add_git_repository(libassert URL https://github.com/jeremy-rifkin/libassert.git COMMITISH v2.1.0) alp_add_git_repository(magic_enum URL https://github.com/Neargye/magic_enum.git COMMITISH v0.9.7) alp_add_git_repository(meshoptimizer URL https://github.com/zeux/meshoptimizer.git COMMITISH v1.0.1) diff --git a/src/terrainlib/CMakeLists.txt b/src/terrainlib/CMakeLists.txt index c799f067..a258d045 100644 --- a/src/terrainlib/CMakeLists.txt +++ b/src/terrainlib/CMakeLists.txt @@ -7,6 +7,7 @@ add_library(terrainlib ctb/types.hpp io/bytes.cpp + io/compression.cpp io/utils.cpp mesh/igl/igl.cpp @@ -70,6 +71,7 @@ target_link_libraries(terrainlib PUBLIC spdlog fmt zpp_bits + zstd::libzstd_static cgltf TBB::tbb opencv_core diff --git a/src/terrainlib/io/compression.cpp b/src/terrainlib/io/compression.cpp new file mode 100644 index 00000000..9a2b3bad --- /dev/null +++ b/src/terrainlib/io/compression.cpp @@ -0,0 +1,156 @@ +#define ZSTD_STATIC_LINKING_ONLY +#include +#include + +#include "io/compression.h" + +#include +#include +#include +#include + +namespace io::envelope { +namespace { + +using CompressionContext = std::unique_ptr; + +std::expected validate_algorithms( + const CompressionAlgorithm compression_algorithm, + const ChecksumAlgorithm checksum_algorithm) +{ + switch (checksum_algorithm) { + case ChecksumAlgorithm::None: + case ChecksumAlgorithm::HandledByCompressionLib: + break; + default: + return std::unexpected(Error{ErrorCode::UnsupportedChecksumAlgorithm}); + } + + switch (compression_algorithm) { + case CompressionAlgorithm::None: + case CompressionAlgorithm::ZstdBestCompressionWithChecksum: + break; + default: + return std::unexpected(Error{ErrorCode::UnsupportedCompressionAlgorithm}); + } + + const bool no_compression = compression_algorithm == CompressionAlgorithm::None + && checksum_algorithm == ChecksumAlgorithm::None; + const bool zstd_with_checksum = + compression_algorithm == CompressionAlgorithm::ZstdBestCompressionWithChecksum + && checksum_algorithm == ChecksumAlgorithm::HandledByCompressionLib; + if (!no_compression && !zstd_with_checksum) { + return std::unexpected(Error{ErrorCode::InvalidAlgorithmCombination}); + } + + return {}; +} + +} // namespace + +std::expected compress_with_checksum( + const Bytes &uncompressed_data, + const CompressionAlgorithm compression_algorithm, + const ChecksumAlgorithm checksum_algorithm) +{ + if (const auto validation = validate_algorithms(compression_algorithm, checksum_algorithm); !validation) { + return std::unexpected(validation.error()); + } + if (uncompressed_data.size() > default_max_decompressed_size) { + return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); + } + + if (compression_algorithm == CompressionAlgorithm::None) { + return CompressedData{uncompressed_data, {}}; + } + + CompressionContext context{ZSTD_createCCtx(), &ZSTD_freeCCtx}; + if (!context) { + return std::unexpected(Error{ErrorCode::CompressionFailed}); + } + + if (ZSTD_isError(ZSTD_CCtx_setParameter(context.get(), ZSTD_c_compressionLevel, ZSTD_maxCLevel())) + || ZSTD_isError(ZSTD_CCtx_setParameter(context.get(), ZSTD_c_checksumFlag, 1))) { + return std::unexpected(Error{ErrorCode::CompressionFailed}); + } + + const std::size_t capacity = ZSTD_compressBound(uncompressed_data.size()); + Bytes compressed_data(capacity); + const std::size_t compressed_size = ZSTD_compress2( + context.get(), + compressed_data.data(), + compressed_data.size(), + uncompressed_data.data(), + uncompressed_data.size()); + if (ZSTD_isError(compressed_size)) { + return std::unexpected(Error{ErrorCode::CompressionFailed}); + } + + compressed_data.resize(compressed_size); + return CompressedData{std::move(compressed_data), {}}; +} + +std::expected checked_decompress( + const Bytes &compressed_data, + const CompressionAlgorithm compression_algorithm, + const ChecksumAlgorithm checksum_algorithm, + const std::string_view checksum, + const std::size_t max_decompressed_size) +{ + if (const auto validation = validate_algorithms(compression_algorithm, checksum_algorithm); !validation) { + return std::unexpected(validation.error()); + } + if (!checksum.empty()) { + return std::unexpected(Error{ErrorCode::InvalidAlgorithmCombination}); + } + + const std::size_t effective_max_decompressed_size = + std::min(max_decompressed_size, default_max_decompressed_size); + + if (compression_algorithm == CompressionAlgorithm::None) { + if (compressed_data.size() > effective_max_decompressed_size) { + return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); + } + return compressed_data; + } + + ZSTD_frameHeader frame_header{}; + const std::size_t frame_header_result = ZSTD_getFrameHeader( + &frame_header, compressed_data.data(), compressed_data.size()); + if (ZSTD_isError(frame_header_result) || frame_header_result != 0) { + return std::unexpected(Error{ErrorCode::DecompressionFailed}); + } + if (frame_header.checksumFlag == 0) { + return std::unexpected(Error{ErrorCode::ChecksumMismatch}); + } + + const unsigned long long content_size = + ZSTD_getFrameContentSize(compressed_data.data(), compressed_data.size()); + if (content_size == ZSTD_CONTENTSIZE_ERROR || content_size == ZSTD_CONTENTSIZE_UNKNOWN + || content_size > std::numeric_limits::max()) { + return std::unexpected(Error{ErrorCode::DecompressionFailed}); + } + if (content_size > effective_max_decompressed_size) { + return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); + } + + Bytes uncompressed_data(static_cast(content_size)); + const std::size_t decompressed_size = ZSTD_decompress( + uncompressed_data.data(), + uncompressed_data.size(), + compressed_data.data(), + compressed_data.size()); + if (ZSTD_isError(decompressed_size)) { + if (ZSTD_getErrorCode(decompressed_size) == ZSTD_error_checksum_wrong) { + return std::unexpected(Error{ErrorCode::ChecksumMismatch}); + } + return std::unexpected(Error{ErrorCode::DecompressionFailed}); + } + if (decompressed_size != uncompressed_data.size()) { + return std::unexpected(Error{ErrorCode::DecompressionFailed}); + } + + return uncompressed_data; +} + +} // namespace io::envelope diff --git a/src/terrainlib/io/compression.h b/src/terrainlib/io/compression.h new file mode 100644 index 00000000..acaca8d1 --- /dev/null +++ b/src/terrainlib/io/compression.h @@ -0,0 +1,67 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +namespace io::envelope { + +using Bytes = std::vector; + +inline constexpr std::size_t default_max_decompressed_size = std::size_t{1} << 30; + +enum class ChecksumAlgorithm : std::uint8_t { + None, + HandledByCompressionLib, +}; + +enum class CompressionAlgorithm : std::uint8_t { + None, + ZstdBestCompressionWithChecksum, +}; + +enum class ErrorCode : std::uint8_t { + SerializationFailed, + DeserializationFailed, + InvalidMagic, + WrongClassName, + UnsupportedClassVersion, + UnsupportedChecksumAlgorithm, + UnsupportedCompressionAlgorithm, + InvalidAlgorithmCombination, + ChecksumMismatch, + CompressionFailed, + DecompressionFailed, + SizeLimitExceeded, +}; + +struct Error { + ErrorCode code; + std::errc serialization_error{}; + + constexpr bool operator==(const Error &) const = default; +}; + +struct CompressedData { + Bytes compressed_data; + std::string checksum; +}; + +std::expected compress_with_checksum( + const Bytes &uncompressed_data, + CompressionAlgorithm compression_algorithm, + ChecksumAlgorithm checksum_algorithm); + +std::expected checked_decompress( + const Bytes &compressed_data, + CompressionAlgorithm compression_algorithm, + ChecksumAlgorithm checksum_algorithm, + std::string_view checksum, + std::size_t max_decompressed_size = default_max_decompressed_size); + +} // namespace io::envelope diff --git a/src/terrainlib/io/envelope.h b/src/terrainlib/io/envelope.h new file mode 100644 index 00000000..2c17e004 --- /dev/null +++ b/src/terrainlib/io/envelope.h @@ -0,0 +1,140 @@ +#pragma once + +#include "io/compression.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace io::envelope { + +inline constexpr std::uint64_t magic = 0xF5FBD3EF919428CAULL; + +struct Envelope { + std::uint64_t magic; + std::string class_name; + std::uint32_t class_version; + ChecksumAlgorithm checksum_algorithm; + std::string checksum; + CompressionAlgorithm compression_algorithm; + Bytes compressed_data; +}; + +template +struct FixedString { + char value[Size]; + + constexpr FixedString(const char (&text)[Size]) + { + std::copy_n(text, Size, value); + } + + constexpr auto operator<=>(const FixedString &) const = default; +}; + +template +struct Version { + static constexpr std::uint32_t number = Number; + using payload_type = VersionedPayloadType; +}; + +namespace detail { + +template +struct FindVersion; + +template +struct FindVersion { + using type = std::conditional_t::type>; +}; + +template +struct FindVersion { + using type = void; +}; + +template +consteval bool versions_are_strictly_increasing() +{ + constexpr std::uint32_t numbers[] = {Versions::number...}; + for (std::size_t index = 1; index < sizeof...(Versions); ++index) { + if (numbers[index - 1] >= numbers[index]) { + return false; + } + } + return true; +} + +template +concept ConvertsFromPrevious = requires(From previous) { + { To::from_previous(std::move(previous)) } -> std::same_as; +}; + +template +consteval bool conversions_are_valid(std::index_sequence) +{ + return (ConvertsFromPrevious< + typename std::tuple_element_t::payload_type, + typename std::tuple_element_t::payload_type> + && ...); +} + +} // namespace detail + +template +struct PayloadSchema { + static_assert(sizeof...(Versions) > 0, "a payload schema requires at least one version"); + static_assert(detail::versions_are_strictly_increasing(), + "payload versions must be strictly increasing"); + + using version_tuple = std::tuple; + static constexpr std::size_t version_count = sizeof...(Versions); + + template + using version_at = std::tuple_element_t; + + using latest_version_descriptor = version_at; + using latest_type = typename latest_version_descriptor::payload_type; + + static constexpr std::string_view class_name{ClassName.value, sizeof(ClassName.value) - 1}; + static constexpr std::uint32_t latest_version = latest_version_descriptor::number; + + template + static constexpr bool supports_version = + !std::is_void_v::type>; + + static constexpr bool supports_version_number(const std::uint32_t number) + { + return ((number == Versions::number) || ...); + } + + template + using payload_type = typename detail::FindVersion::type::payload_type; + + static_assert(version_count == 1 + || detail::conversions_are_valid( + std::make_index_sequence{}), + "each payload version must provide from_previous for the preceding version"); +}; + +template +std::expected serialize( + const typename Schema::template payload_type &payload, + CompressionAlgorithm compression_algorithm = CompressionAlgorithm::ZstdBestCompressionWithChecksum, + ChecksumAlgorithm checksum_algorithm = ChecksumAlgorithm::HandledByCompressionLib); + +template +std::expected deserialize( + std::span bytes, + std::size_t max_decompressed_size = default_max_decompressed_size); + +} // namespace io::envelope + +#include "io/envelope.inl" diff --git a/src/terrainlib/io/envelope.inl b/src/terrainlib/io/envelope.inl new file mode 100644 index 00000000..ebb1f3bf --- /dev/null +++ b/src/terrainlib/io/envelope.inl @@ -0,0 +1,132 @@ +#pragma once + +#include + +namespace io::envelope { +namespace detail { + +template +std::expected serialize_to_bytes(const Value &value) +{ + Bytes bytes; + zpp::bits::out output(bytes, zpp::bits::alloc_limit()); + const zpp::bits::errc result = output(value); + if (zpp::bits::failure(result)) { + return std::unexpected(Error{ErrorCode::SerializationFailed, result.code}); + } + return bytes; +} + +template +std::expected deserialize_from_bytes(const std::span bytes) +{ + Value value{}; + zpp::bits::in input(bytes, zpp::bits::alloc_limit()); + const zpp::bits::errc result = input(value); + if (zpp::bits::failure(result) || input.position() != bytes.size()) { + return std::unexpected(Error{ + ErrorCode::DeserializationFailed, + zpp::bits::failure(result) ? result.code : std::errc::bad_message, + }); + } + return value; +} + +template +typename Schema::latest_type convert_to_latest(Current current) +{ + if constexpr (Index + 1 == Schema::version_count) { + return current; + } else { + using Next = typename Schema::template version_at::payload_type; + return convert_to_latest(Next::from_previous(std::move(current))); + } +} + +template +std::expected deserialize_version( + const std::uint32_t class_version, + const std::span payload_bytes) +{ + using CurrentVersion = typename Schema::template version_at; + if (class_version == CurrentVersion::number) { + auto payload = deserialize_from_bytes(payload_bytes); + if (!payload) { + return std::unexpected(payload.error()); + } + return convert_to_latest(std::move(*payload)); + } + + if constexpr (Index + 1 < Schema::version_count) { + return deserialize_version(class_version, payload_bytes); + } else { + return std::unexpected(Error{ErrorCode::UnsupportedClassVersion}); + } +} + +} // namespace detail + +template +std::expected serialize( + const typename Schema::template payload_type &payload, + const CompressionAlgorithm compression_algorithm, + const ChecksumAlgorithm checksum_algorithm) +{ + static_assert(Schema::template supports_version, + "the requested payload version is not part of the schema"); + + auto payload_bytes = detail::serialize_to_bytes(payload); + if (!payload_bytes) { + return std::unexpected(payload_bytes.error()); + } + + auto compressed = compress_with_checksum(*payload_bytes, compression_algorithm, checksum_algorithm); + if (!compressed) { + return std::unexpected(compressed.error()); + } + + const Envelope envelope{ + .magic = magic, + .class_name = std::string{Schema::class_name}, + .class_version = VersionNumber, + .checksum_algorithm = checksum_algorithm, + .checksum = std::move(compressed->checksum), + .compression_algorithm = compression_algorithm, + .compressed_data = std::move(compressed->compressed_data), + }; + return detail::serialize_to_bytes(envelope); +} + +template +std::expected deserialize( + const std::span bytes, + const std::size_t max_decompressed_size) +{ + auto envelope = detail::deserialize_from_bytes(bytes); + if (!envelope) { + return std::unexpected(envelope.error()); + } + if (envelope->magic != magic) { + return std::unexpected(Error{ErrorCode::InvalidMagic}); + } + if (envelope->class_name != Schema::class_name) { + return std::unexpected(Error{ErrorCode::WrongClassName}); + } + if (!Schema::supports_version_number(envelope->class_version)) { + return std::unexpected(Error{ErrorCode::UnsupportedClassVersion}); + } + + auto payload_bytes = checked_decompress( + envelope->compressed_data, + envelope->compression_algorithm, + envelope->checksum_algorithm, + envelope->checksum, + max_decompressed_size); + if (!payload_bytes) { + return std::unexpected(payload_bytes.error()); + } + + return detail::deserialize_version(envelope->class_version, *payload_bytes); +} + +} // namespace io::envelope diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 0ea40bcc..d43ee019 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -20,6 +20,7 @@ add_executable(unittests_terrainlib terrainlib/cow.cpp terrainlib/dataset.cpp terrainlib/enumerate.cpp + terrainlib/envelope.cpp terrainlib/fixed_vector.cpp terrainlib/grid.cpp terrainlib/hash_utils.cpp diff --git a/unittests/terrainlib/envelope.cpp b/unittests/terrainlib/envelope.cpp new file mode 100644 index 00000000..999e5b24 --- /dev/null +++ b/unittests/terrainlib/envelope.cpp @@ -0,0 +1,313 @@ +#include "../catch2_helpers.h" + +#include "io/envelope.h" + +#include + +namespace { + +namespace v1 { + +struct Payload { + std::uint32_t id; + std::string name; + + bool operator==(const Payload &) const = default; +}; + +} // namespace v1 + +namespace v2 { + +struct Payload { + std::uint64_t id; + std::string name; + bool enabled; + + static Payload from_previous(v1::Payload previous) + { + return { + .id = previous.id, + .name = std::move(previous.name), + .enabled = true, + }; + } + + bool operator==(const Payload &) const = default; +}; + +} // namespace v2 + +namespace v3 { + +struct Payload { + std::uint64_t id; + std::string label; + bool enabled; + std::vector samples; + + static Payload from_previous(v2::Payload previous) + { + return { + .id = previous.id, + .label = std::move(previous.name), + .enabled = previous.enabled, + .samples = {}, + }; + } + + bool operator==(const Payload &) const = default; +}; + +} // namespace v3 + +using Schema = io::envelope::PayloadSchema< + "test.Payload", + io::envelope::Version<1, v1::Payload>, + io::envelope::Version<2, v2::Payload>, + io::envelope::Version<3, v3::Payload>>; + +static_assert(std::is_aggregate_v); +static_assert(std::is_aggregate_v); +static_assert(std::is_aggregate_v); +static_assert(Schema::class_name == "test.Payload"); +static_assert(Schema::latest_version == 3); +static_assert(std::same_as); +static_assert(std::same_as, v1::Payload>); + +io::envelope::Bytes encode_envelope(const io::envelope::Envelope &envelope) +{ + io::envelope::Bytes bytes; + zpp::bits::out output(bytes); + output(envelope).or_throw(); + return bytes; +} + +io::envelope::Envelope decode_envelope(const io::envelope::Bytes &bytes) +{ + io::envelope::Envelope envelope{}; + zpp::bits::in input(bytes); + input(envelope).or_throw(); + return envelope; +} + +template +void check_error(const Result &result, const io::envelope::ErrorCode expected) +{ + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code == expected); +} + +} // namespace + +TEST_CASE("Envelope round trips the latest payload version") +{ + const v3::Payload expected{ + .id = 42, + .label = "latest", + .enabled = false, + .samples = {1, 2, 3}, + }; + + const auto bytes = io::envelope::serialize(expected); + REQUIRE(bytes.has_value()); + + const auto envelope = decode_envelope(*bytes); + CHECK(envelope.magic == io::envelope::magic); + CHECK(envelope.class_name == Schema::class_name); + CHECK(envelope.class_version == 3); + CHECK(envelope.checksum_algorithm == io::envelope::ChecksumAlgorithm::HandledByCompressionLib); + CHECK(envelope.checksum.empty()); + CHECK(envelope.compression_algorithm + == io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum); + + const auto result = io::envelope::deserialize(*bytes); + REQUIRE(result.has_value()); + CHECK(*result == expected); +} + +TEST_CASE("Envelope upgrades older payload versions") +{ + SECTION("version 1 is upgraded through every subsequent version") + { + const v1::Payload original{.id = 7, .name = "version one"}; + const auto bytes = io::envelope::serialize(original); + REQUIRE(bytes.has_value()); + + const auto result = io::envelope::deserialize(*bytes); + REQUIRE(result.has_value()); + CHECK(*result == v3::Payload{ + .id = 7, + .label = "version one", + .enabled = true, + .samples = {}, + }); + } + + SECTION("version 2 is upgraded to the latest version") + { + const v2::Payload original{.id = 9, .name = "version two", .enabled = false}; + const auto bytes = io::envelope::serialize(original); + REQUIRE(bytes.has_value()); + + const auto result = io::envelope::deserialize(*bytes); + REQUIRE(result.has_value()); + CHECK(*result == v3::Payload{ + .id = 9, + .label = "version two", + .enabled = false, + .samples = {}, + }); + } +} + +TEST_CASE("Envelope supports uncompressed data without a checksum") +{ + const v3::Payload expected{ + .id = 11, + .label = "plain", + .enabled = true, + .samples = {5, 8}, + }; + const auto bytes = io::envelope::serialize( + expected, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::None); + REQUIRE(bytes.has_value()); + + const auto envelope = decode_envelope(*bytes); + CHECK(envelope.compression_algorithm == io::envelope::CompressionAlgorithm::None); + CHECK(envelope.checksum_algorithm == io::envelope::ChecksumAlgorithm::None); + CHECK(envelope.checksum.empty()); + + const auto result = io::envelope::deserialize(*bytes); + REQUIRE(result.has_value()); + CHECK(*result == expected); +} + +TEST_CASE("Checked compression round trips and validates its checksum") +{ + const io::envelope::Bytes original(4096, std::byte{0x2a}); + const auto compressed = io::envelope::compress_with_checksum( + original, + io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib); + REQUIRE(compressed.has_value()); + CHECK(compressed->compressed_data.size() < original.size()); + CHECK(compressed->checksum.empty()); + + const auto result = io::envelope::checked_decompress( + compressed->compressed_data, + io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib, + compressed->checksum); + REQUIRE(result.has_value()); + CHECK(*result == original); + + auto corrupted = compressed->compressed_data; + corrupted.back() ^= std::byte{0x01}; + check_error( + io::envelope::checked_decompress( + corrupted, + io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib, + {}), + io::envelope::ErrorCode::ChecksumMismatch); + + check_error( + io::envelope::checked_decompress( + compressed->compressed_data, + io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib, + {}, + original.size() - 1), + io::envelope::ErrorCode::SizeLimitExceeded); +} + +TEST_CASE("Envelope rejects incompatible metadata") +{ + const v3::Payload payload{.id = 1, .label = "metadata", .enabled = true, .samples = {}}; + const auto serialized = io::envelope::serialize(payload); + REQUIRE(serialized.has_value()); + + SECTION("magic") + { + auto envelope = decode_envelope(*serialized); + ++envelope.magic; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::InvalidMagic); + } + + SECTION("class name") + { + auto envelope = decode_envelope(*serialized); + envelope.class_name = "other.Payload"; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::WrongClassName); + } + + SECTION("class version") + { + auto envelope = decode_envelope(*serialized); + envelope.class_version = 99; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::UnsupportedClassVersion); + } + + SECTION("checksum algorithm") + { + auto envelope = decode_envelope(*serialized); + envelope.checksum_algorithm = static_cast(99); + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::UnsupportedChecksumAlgorithm); + } + + SECTION("compression algorithm") + { + auto envelope = decode_envelope(*serialized); + envelope.compression_algorithm = static_cast(99); + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::UnsupportedCompressionAlgorithm); + } + + SECTION("algorithm combination") + { + auto envelope = decode_envelope(*serialized); + envelope.checksum_algorithm = io::envelope::ChecksumAlgorithm::None; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::InvalidAlgorithmCombination); + } + + SECTION("external checksum") + { + auto envelope = decode_envelope(*serialized); + envelope.checksum = "not used by zstd"; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::InvalidAlgorithmCombination); + } +} + +TEST_CASE("Envelope reports malformed serialized data") +{ + SECTION("envelope") + { + const io::envelope::Bytes malformed{std::byte{0x01}, std::byte{0x02}}; + check_error(io::envelope::deserialize(malformed), + io::envelope::ErrorCode::DeserializationFailed); + } + + SECTION("payload") + { + const io::envelope::Envelope envelope{ + .magic = io::envelope::magic, + .class_name = std::string{Schema::class_name}, + .class_version = 3, + .checksum_algorithm = io::envelope::ChecksumAlgorithm::None, + .checksum = {}, + .compression_algorithm = io::envelope::CompressionAlgorithm::None, + .compressed_data = {std::byte{0x01}}, + }; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::DeserializationFailed); + } +} From 0634ae1d897872c1420bfd6313f3802e002afc16 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Sun, 2 Aug 2026 22:27:02 +0200 Subject: [PATCH 39/61] Store uncompressed envelope size --- docs/raster-store/storage-format.md | 5 +- src/terrainlib/io/compression.cpp | 23 +++-- src/terrainlib/io/envelope.h | 1 + src/terrainlib/io/envelope.inl | 18 +++- unittests/terrainlib/envelope.cpp | 146 +++++++++++++++++++++++++++- 5 files changed, 183 insertions(+), 10 deletions(-) diff --git a/docs/raster-store/storage-format.md b/docs/raster-store/storage-format.md index df26fe7e..29103995 100644 --- a/docs/raster-store/storage-format.md +++ b/docs/raster-store/storage-format.md @@ -63,6 +63,7 @@ tile-base is a hierarchy build from raster-fundamentalis, containing all data an - `ChecksumAlgorithm checksum_algorithm`, default `HandledByCompressionLib`, alternatively `None`; - `string checksum`, empty when the checksum is handled by the compression library; - `CompressionAlgorithm compression_algorithm`, default `ZstdBestCompressionWithChecksum`, alternatively `None`; + - `uint64 uncompressed_size`, the exact size of the uncompressed second level; - `Bytes compressed_data`, containing the second level. - The second level is a compressed byte vector. It is deserialised directly into the selected versioned payload class. - The magic is shared by all payload types. `class_name` distinguishes payload types. An incompatible future envelope layout requires a new magic. @@ -82,8 +83,10 @@ tile-base is a hierarchy build from raster-fundamentalis, containing all data an - we fail by returning an unexpected in these cases - Compression uses libzstd at its best compression level. Libzstd is imported through the project's CMake install facility from https://github.com/AlpineMapsOrgDependencies/zstd. `ZstdBestCompressionWithChecksum` writes an embedded zstd frame checksum, which libzstd verifies while decompressing. - `compress_with_checksum` accepts a `vector` and returns the compressed bytes plus the external checksum string. `checked_decompress` accepts both and returns the decompressed bytes. Both use `std::expected` and dispatch with a switch. +- `checked_decompress` accepts a maximum decompressed size, defaulting to 1 GiB. It uses a size reported by the compression format when available, otherwise it uses the maximum as its allocation bound and shrinks the result to the produced size. +- Envelope deserialisation passes `uncompressed_size` as that maximum and requires the produced size to match it exactly. A mismatch is a decompression failure. A declared size above the caller's limit or the hard 1 GiB limit is a size-limit failure. - `None` compression must be paired with `None` checksum. `ZstdBestCompressionWithChecksum` must be paired with `HandledByCompressionLib`; its external checksum string must be empty because the checksum is embedded in the zstd frame. -- Decompression rejects output larger than 1 GiB. +- Decompression never allocates or produces output larger than 1 GiB. ## to be defined diff --git a/src/terrainlib/io/compression.cpp b/src/terrainlib/io/compression.cpp index 9a2b3bad..c137988c 100644 --- a/src/terrainlib/io/compression.cpp +++ b/src/terrainlib/io/compression.cpp @@ -124,17 +124,23 @@ std::expected checked_decompress( return std::unexpected(Error{ErrorCode::ChecksumMismatch}); } - const unsigned long long content_size = + const unsigned long long frame_content_size = ZSTD_getFrameContentSize(compressed_data.data(), compressed_data.size()); - if (content_size == ZSTD_CONTENTSIZE_ERROR || content_size == ZSTD_CONTENTSIZE_UNKNOWN - || content_size > std::numeric_limits::max()) { + if (frame_content_size == ZSTD_CONTENTSIZE_ERROR + || (frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN + && frame_content_size > std::numeric_limits::max())) { return std::unexpected(Error{ErrorCode::DecompressionFailed}); } - if (content_size > effective_max_decompressed_size) { + + const bool content_size_is_known = frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN; + if (content_size_is_known && frame_content_size > effective_max_decompressed_size) { return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); } - Bytes uncompressed_data(static_cast(content_size)); + const std::size_t allocation_size = content_size_is_known + ? static_cast(frame_content_size) + : effective_max_decompressed_size; + Bytes uncompressed_data(allocation_size); const std::size_t decompressed_size = ZSTD_decompress( uncompressed_data.data(), uncompressed_data.size(), @@ -144,12 +150,17 @@ std::expected checked_decompress( if (ZSTD_getErrorCode(decompressed_size) == ZSTD_error_checksum_wrong) { return std::unexpected(Error{ErrorCode::ChecksumMismatch}); } + if (!content_size_is_known + && ZSTD_getErrorCode(decompressed_size) == ZSTD_error_dstSize_tooSmall) { + return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); + } return std::unexpected(Error{ErrorCode::DecompressionFailed}); } - if (decompressed_size != uncompressed_data.size()) { + if (content_size_is_known && decompressed_size != uncompressed_data.size()) { return std::unexpected(Error{ErrorCode::DecompressionFailed}); } + uncompressed_data.resize(decompressed_size); return uncompressed_data; } diff --git a/src/terrainlib/io/envelope.h b/src/terrainlib/io/envelope.h index 2c17e004..4b36065b 100644 --- a/src/terrainlib/io/envelope.h +++ b/src/terrainlib/io/envelope.h @@ -24,6 +24,7 @@ struct Envelope { ChecksumAlgorithm checksum_algorithm; std::string checksum; CompressionAlgorithm compression_algorithm; + std::uint64_t uncompressed_size; Bytes compressed_data; }; diff --git a/src/terrainlib/io/envelope.inl b/src/terrainlib/io/envelope.inl index ebb1f3bf..c9055894 100644 --- a/src/terrainlib/io/envelope.inl +++ b/src/terrainlib/io/envelope.inl @@ -2,6 +2,8 @@ #include +#include + namespace io::envelope { namespace detail { @@ -92,6 +94,7 @@ std::expected serialize( .checksum_algorithm = checksum_algorithm, .checksum = std::move(compressed->checksum), .compression_algorithm = compression_algorithm, + .uncompressed_size = payload_bytes->size(), .compressed_data = std::move(compressed->compressed_data), }; return detail::serialize_to_bytes(envelope); @@ -116,15 +119,28 @@ std::expected deserialize( return std::unexpected(Error{ErrorCode::UnsupportedClassVersion}); } + const std::size_t effective_max_decompressed_size = + std::min(max_decompressed_size, default_max_decompressed_size); + if (envelope->uncompressed_size > std::numeric_limits::max() + || envelope->uncompressed_size > effective_max_decompressed_size) { + return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); + } + auto payload_bytes = checked_decompress( envelope->compressed_data, envelope->compression_algorithm, envelope->checksum_algorithm, envelope->checksum, - max_decompressed_size); + static_cast(envelope->uncompressed_size)); if (!payload_bytes) { + if (payload_bytes.error().code == ErrorCode::SizeLimitExceeded) { + return std::unexpected(Error{ErrorCode::DecompressionFailed}); + } return std::unexpected(payload_bytes.error()); } + if (payload_bytes->size() != envelope->uncompressed_size) { + return std::unexpected(Error{ErrorCode::DecompressionFailed}); + } return detail::deserialize_version(envelope->class_version, *payload_bytes); } diff --git a/unittests/terrainlib/envelope.cpp b/unittests/terrainlib/envelope.cpp index 999e5b24..a8b82273 100644 --- a/unittests/terrainlib/envelope.cpp +++ b/unittests/terrainlib/envelope.cpp @@ -2,8 +2,12 @@ #include "io/envelope.h" +#include #include +#include +#include + namespace { namespace v1 { @@ -75,14 +79,20 @@ static_assert(Schema::latest_version == 3); static_assert(std::same_as); static_assert(std::same_as, v1::Payload>); -io::envelope::Bytes encode_envelope(const io::envelope::Envelope &envelope) +template +io::envelope::Bytes encode_value(const Value &value) { io::envelope::Bytes bytes; zpp::bits::out output(bytes); - output(envelope).or_throw(); + output(value).or_throw(); return bytes; } +io::envelope::Bytes encode_envelope(const io::envelope::Envelope &envelope) +{ + return encode_value(envelope); +} + io::envelope::Envelope decode_envelope(const io::envelope::Bytes &bytes) { io::envelope::Envelope envelope{}; @@ -91,6 +101,30 @@ io::envelope::Envelope decode_envelope(const io::envelope::Bytes &bytes) return envelope; } +io::envelope::Bytes compress_without_content_size(const io::envelope::Bytes &uncompressed_data) +{ + const std::unique_ptr context{ + ZSTD_createCCtx(), &ZSTD_freeCCtx}; + if (!context + || ZSTD_isError(ZSTD_CCtx_setParameter(context.get(), ZSTD_c_contentSizeFlag, 0)) + || ZSTD_isError(ZSTD_CCtx_setParameter(context.get(), ZSTD_c_checksumFlag, 1))) { + throw std::runtime_error{"could not configure zstd test context"}; + } + + io::envelope::Bytes compressed_data(ZSTD_compressBound(uncompressed_data.size())); + const std::size_t compressed_size = ZSTD_compress2( + context.get(), + compressed_data.data(), + compressed_data.size(), + uncompressed_data.data(), + uncompressed_data.size()); + if (ZSTD_isError(compressed_size)) { + throw std::runtime_error{"could not create zstd test data"}; + } + compressed_data.resize(compressed_size); + return compressed_data; +} + template void check_error(const Result &result, const io::envelope::ErrorCode expected) { @@ -120,6 +154,7 @@ TEST_CASE("Envelope round trips the latest payload version") CHECK(envelope.checksum.empty()); CHECK(envelope.compression_algorithm == io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum); + CHECK(envelope.uncompressed_size == encode_value(expected).size()); const auto result = io::envelope::deserialize(*bytes); REQUIRE(result.has_value()); @@ -179,6 +214,7 @@ TEST_CASE("Envelope supports uncompressed data without a checksum") CHECK(envelope.compression_algorithm == io::envelope::CompressionAlgorithm::None); CHECK(envelope.checksum_algorithm == io::envelope::ChecksumAlgorithm::None); CHECK(envelope.checksum.empty()); + CHECK(envelope.uncompressed_size == envelope.compressed_data.size()); const auto result = io::envelope::deserialize(*bytes); REQUIRE(result.has_value()); @@ -204,6 +240,19 @@ TEST_CASE("Checked compression round trips and validates its checksum") REQUIRE(result.has_value()); CHECK(*result == original); + const auto empty_compressed = io::envelope::compress_with_checksum( + {}, + io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib); + REQUIRE(empty_compressed.has_value()); + const auto empty_result = io::envelope::checked_decompress( + empty_compressed->compressed_data, + io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib, + empty_compressed->checksum); + REQUIRE(empty_result.has_value()); + CHECK(empty_result->empty()); + auto corrupted = compressed->compressed_data; corrupted.back() ^= std::byte{0x01}; check_error( @@ -224,6 +273,30 @@ TEST_CASE("Checked compression round trips and validates its checksum") io::envelope::ErrorCode::SizeLimitExceeded); } +TEST_CASE("Checked decompression uses its maximum when the format omits the content size") +{ + const io::envelope::Bytes original(4096, std::byte{0x37}); + const auto compressed_data = compress_without_content_size(original); + + const auto result = io::envelope::checked_decompress( + compressed_data, + io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib, + {}, + original.size()); + REQUIRE(result.has_value()); + CHECK(*result == original); + + check_error( + io::envelope::checked_decompress( + compressed_data, + io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib, + {}, + original.size() - 1), + io::envelope::ErrorCode::SizeLimitExceeded); +} + TEST_CASE("Envelope rejects incompatible metadata") { const v3::Payload payload{.id = 1, .label = "metadata", .enabled = true, .samples = {}}; @@ -285,6 +358,74 @@ TEST_CASE("Envelope rejects incompatible metadata") check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::InvalidAlgorithmCombination); } + + SECTION("uncompressed size is smaller than the payload") + { + auto envelope = decode_envelope(*serialized); + --envelope.uncompressed_size; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::DecompressionFailed); + } + + SECTION("uncompressed size is larger than the payload") + { + auto envelope = decode_envelope(*serialized); + ++envelope.uncompressed_size; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::DecompressionFailed); + } + + SECTION("zero uncompressed size does not mean unspecified") + { + auto envelope = decode_envelope(*serialized); + envelope.uncompressed_size = 0; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::DecompressionFailed); + } + + SECTION("uncompressed size exceeds the hard limit") + { + auto envelope = decode_envelope(*serialized); + envelope.uncompressed_size = io::envelope::default_max_decompressed_size + 1; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::SizeLimitExceeded); + } + + SECTION("uncompressed size exceeds a caller limit") + { + auto envelope = decode_envelope(*serialized); + check_error( + io::envelope::deserialize( + encode_envelope(envelope), + static_cast(envelope.uncompressed_size - 1)), + io::envelope::ErrorCode::SizeLimitExceeded); + } +} + +TEST_CASE("Envelope validates the size of uncompressed data") +{ + const v3::Payload payload{.id = 2, .label = "plain", .enabled = true, .samples = {1}}; + const auto serialized = io::envelope::serialize( + payload, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::None); + REQUIRE(serialized.has_value()); + + SECTION("declared size is smaller") + { + auto envelope = decode_envelope(*serialized); + --envelope.uncompressed_size; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::DecompressionFailed); + } + + SECTION("declared size is larger") + { + auto envelope = decode_envelope(*serialized); + ++envelope.uncompressed_size; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::DecompressionFailed); + } } TEST_CASE("Envelope reports malformed serialized data") @@ -305,6 +446,7 @@ TEST_CASE("Envelope reports malformed serialized data") .checksum_algorithm = io::envelope::ChecksumAlgorithm::None, .checksum = {}, .compression_algorithm = io::envelope::CompressionAlgorithm::None, + .uncompressed_size = 1, .compressed_data = {std::byte{0x01}}, }; check_error(io::envelope::deserialize(encode_envelope(envelope)), From cf6d4a6bce4c68843a4dec27dfa48e153d3d07e2 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:28:04 +0200 Subject: [PATCH 40/61] Add CRC-32C envelope checksums --- docs/raster-store/storage-format.md | 8 +- src/terrainlib/io/compression.cpp | 150 +++++++++++++++-------- src/terrainlib/io/compression.h | 1 + unittests/terrainlib/envelope.cpp | 181 ++++++++++++++++++++++++++++ 4 files changed, 289 insertions(+), 51 deletions(-) diff --git a/docs/raster-store/storage-format.md b/docs/raster-store/storage-format.md index 29103995..5767e0f8 100644 --- a/docs/raster-store/storage-format.md +++ b/docs/raster-store/storage-format.md @@ -60,8 +60,8 @@ tile-base is a hierarchy build from raster-fundamentalis, containing all data an - `uint64 magic`, always `F5FBD3EF919428CA`, identifying this envelope format; - `string class_name`, identifying the payload type; - `uint32 class_version`, identifying the versioned payload type; - - `ChecksumAlgorithm checksum_algorithm`, default `HandledByCompressionLib`, alternatively `None`; - - `string checksum`, empty when the checksum is handled by the compression library; + - `ChecksumAlgorithm checksum_algorithm`, default `HandledByCompressionLib`, alternatively `Crc32c` or `None`; + - `string checksum`, empty when no external checksum is used, otherwise the CRC-32C value; - `CompressionAlgorithm compression_algorithm`, default `ZstdBestCompressionWithChecksum`, alternatively `None`; - `uint64 uncompressed_size`, the exact size of the uncompressed second level; - `Bytes compressed_data`, containing the second level. @@ -83,9 +83,11 @@ tile-base is a hierarchy build from raster-fundamentalis, containing all data an - we fail by returning an unexpected in these cases - Compression uses libzstd at its best compression level. Libzstd is imported through the project's CMake install facility from https://github.com/AlpineMapsOrgDependencies/zstd. `ZstdBestCompressionWithChecksum` writes an embedded zstd frame checksum, which libzstd verifies while decompressing. - `compress_with_checksum` accepts a `vector` and returns the compressed bytes plus the external checksum string. `checked_decompress` accepts both and returns the decompressed bytes. Both use `std::expected` and dispatch with a switch. +- `Crc32c` is an external CRC-32C (Castagnoli) checksum of the complete uncompressed serialised payload. It uses the reflected polynomial `82F63B78`, an initial value of `FFFFFFFF`, and a final XOR of `FFFFFFFF`. The checksum string contains exactly eight lowercase hexadecimal characters. The empty payload has checksum `00000000`; the ASCII test vector `123456789` has checksum `e3069283`. CRC-32C detects accidental corruption but is not cryptographic. - `checked_decompress` accepts a maximum decompressed size, defaulting to 1 GiB. It uses a size reported by the compression format when available, otherwise it uses the maximum as its allocation bound and shrinks the result to the produced size. - Envelope deserialisation passes `uncompressed_size` as that maximum and requires the produced size to match it exactly. A mismatch is a decompression failure. A declared size above the caller's limit or the hard 1 GiB limit is a size-limit failure. -- `None` compression must be paired with `None` checksum. `ZstdBestCompressionWithChecksum` must be paired with `HandledByCompressionLib`; its external checksum string must be empty because the checksum is embedded in the zstd frame. +- `None` compression may be paired with `None` or `Crc32c`. `ZstdBestCompressionWithChecksum` may be paired with `HandledByCompressionLib` or `Crc32c`; with `Crc32c`, both the embedded zstd checksum and the external CRC-32C are checked. `HandledByCompressionLib` is invalid with `None` compression, and `None` checksum is invalid with `ZstdBestCompressionWithChecksum`. +- `None` and `HandledByCompressionLib` require an empty checksum string. `Crc32c` requires its canonical eight-character checksum. Deserialisation verifies it after bounded decompression and before parsing the payload with `zpp::bits`; envelope deserialisation also requires the resulting size to exactly match `uncompressed_size`. - Decompression never allocates or produces output larger than 1 GiB. diff --git a/src/terrainlib/io/compression.cpp b/src/terrainlib/io/compression.cpp index c137988c..ba18e638 100644 --- a/src/terrainlib/io/compression.cpp +++ b/src/terrainlib/io/compression.cpp @@ -5,6 +5,7 @@ #include "io/compression.h" #include +#include #include #include #include @@ -14,6 +15,38 @@ namespace { using CompressionContext = std::unique_ptr; +std::string crc32c_checksum(const Bytes &data) +{ + static constexpr auto table = [] { + constexpr std::uint32_t polynomial = 0x82f63b78u; + + std::array values{}; + for (std::size_t index = 0; index < values.size(); ++index) { + auto crc = static_cast(index); + for (int bit = 0; bit < 8; ++bit) { + crc = (crc >> 1u) ^ ((crc & 1u) != 0u ? polynomial : 0u); + } + values[index] = crc; + } + return values; + }(); + + std::uint32_t crc = 0xffffffffu; + for (const auto value : data) { + const auto index = (crc ^ std::to_integer(value)) & 0xffu; + crc = table[index] ^ (crc >> 8u); + } + crc = ~crc; + + constexpr char hex_digits[] = "0123456789abcdef"; + std::string checksum(8, '0'); + for (std::size_t index = 0; index < checksum.size(); ++index) { + const auto shift = static_cast((checksum.size() - index - 1) * 4); + checksum[index] = hex_digits[(crc >> shift) & 0x0fu]; + } + return checksum; +} + std::expected validate_algorithms( const CompressionAlgorithm compression_algorithm, const ChecksumAlgorithm checksum_algorithm) @@ -21,6 +54,7 @@ std::expected validate_algorithms( switch (checksum_algorithm) { case ChecksumAlgorithm::None: case ChecksumAlgorithm::HandledByCompressionLib: + case ChecksumAlgorithm::Crc32c: break; default: return std::unexpected(Error{ErrorCode::UnsupportedChecksumAlgorithm}); @@ -35,10 +69,12 @@ std::expected validate_algorithms( } const bool no_compression = compression_algorithm == CompressionAlgorithm::None - && checksum_algorithm == ChecksumAlgorithm::None; + && (checksum_algorithm == ChecksumAlgorithm::None + || checksum_algorithm == ChecksumAlgorithm::Crc32c); const bool zstd_with_checksum = compression_algorithm == CompressionAlgorithm::ZstdBestCompressionWithChecksum - && checksum_algorithm == ChecksumAlgorithm::HandledByCompressionLib; + && (checksum_algorithm == ChecksumAlgorithm::HandledByCompressionLib + || checksum_algorithm == ChecksumAlgorithm::Crc32c); if (!no_compression && !zstd_with_checksum) { return std::unexpected(Error{ErrorCode::InvalidAlgorithmCombination}); } @@ -60,8 +96,12 @@ std::expected compress_with_checksum( return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); } + const std::string checksum = checksum_algorithm == ChecksumAlgorithm::Crc32c + ? crc32c_checksum(uncompressed_data) + : std::string{}; + if (compression_algorithm == CompressionAlgorithm::None) { - return CompressedData{uncompressed_data, {}}; + return CompressedData{uncompressed_data, checksum}; } CompressionContext context{ZSTD_createCCtx(), &ZSTD_freeCCtx}; @@ -87,7 +127,7 @@ std::expected compress_with_checksum( } compressed_data.resize(compressed_size); - return CompressedData{std::move(compressed_data), {}}; + return CompressedData{std::move(compressed_data), checksum}; } std::expected checked_decompress( @@ -100,67 +140,81 @@ std::expected checked_decompress( if (const auto validation = validate_algorithms(compression_algorithm, checksum_algorithm); !validation) { return std::unexpected(validation.error()); } - if (!checksum.empty()) { + if (checksum_algorithm != ChecksumAlgorithm::Crc32c && !checksum.empty()) { return std::unexpected(Error{ErrorCode::InvalidAlgorithmCombination}); } + if (checksum_algorithm == ChecksumAlgorithm::Crc32c + && (checksum.size() != 8 + || !std::all_of(checksum.begin(), checksum.end(), [](const char character) { + return (character >= '0' && character <= '9') + || (character >= 'a' && character <= 'f'); + }))) { + return std::unexpected(Error{ErrorCode::ChecksumMismatch}); + } const std::size_t effective_max_decompressed_size = std::min(max_decompressed_size, default_max_decompressed_size); + Bytes uncompressed_data; if (compression_algorithm == CompressionAlgorithm::None) { if (compressed_data.size() > effective_max_decompressed_size) { return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); } - return compressed_data; - } - - ZSTD_frameHeader frame_header{}; - const std::size_t frame_header_result = ZSTD_getFrameHeader( - &frame_header, compressed_data.data(), compressed_data.size()); - if (ZSTD_isError(frame_header_result) || frame_header_result != 0) { - return std::unexpected(Error{ErrorCode::DecompressionFailed}); - } - if (frame_header.checksumFlag == 0) { - return std::unexpected(Error{ErrorCode::ChecksumMismatch}); - } + uncompressed_data = compressed_data; + } else { + ZSTD_frameHeader frame_header{}; + const std::size_t frame_header_result = ZSTD_getFrameHeader( + &frame_header, compressed_data.data(), compressed_data.size()); + if (ZSTD_isError(frame_header_result) || frame_header_result != 0) { + return std::unexpected(Error{ErrorCode::DecompressionFailed}); + } + if (frame_header.checksumFlag == 0) { + return std::unexpected(Error{ErrorCode::ChecksumMismatch}); + } - const unsigned long long frame_content_size = - ZSTD_getFrameContentSize(compressed_data.data(), compressed_data.size()); - if (frame_content_size == ZSTD_CONTENTSIZE_ERROR - || (frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN - && frame_content_size > std::numeric_limits::max())) { - return std::unexpected(Error{ErrorCode::DecompressionFailed}); - } + const unsigned long long frame_content_size = + ZSTD_getFrameContentSize(compressed_data.data(), compressed_data.size()); + if (frame_content_size == ZSTD_CONTENTSIZE_ERROR + || (frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN + && frame_content_size > std::numeric_limits::max())) { + return std::unexpected(Error{ErrorCode::DecompressionFailed}); + } - const bool content_size_is_known = frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN; - if (content_size_is_known && frame_content_size > effective_max_decompressed_size) { - return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); - } + const bool content_size_is_known = frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN; + if (content_size_is_known && frame_content_size > effective_max_decompressed_size) { + return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); + } - const std::size_t allocation_size = content_size_is_known - ? static_cast(frame_content_size) - : effective_max_decompressed_size; - Bytes uncompressed_data(allocation_size); - const std::size_t decompressed_size = ZSTD_decompress( - uncompressed_data.data(), - uncompressed_data.size(), - compressed_data.data(), - compressed_data.size()); - if (ZSTD_isError(decompressed_size)) { - if (ZSTD_getErrorCode(decompressed_size) == ZSTD_error_checksum_wrong) { - return std::unexpected(Error{ErrorCode::ChecksumMismatch}); + const std::size_t allocation_size = content_size_is_known + ? static_cast(frame_content_size) + : effective_max_decompressed_size; + uncompressed_data.resize(allocation_size); + const std::size_t decompressed_size = ZSTD_decompress( + uncompressed_data.data(), + uncompressed_data.size(), + compressed_data.data(), + compressed_data.size()); + if (ZSTD_isError(decompressed_size)) { + if (ZSTD_getErrorCode(decompressed_size) == ZSTD_error_checksum_wrong) { + return std::unexpected(Error{ErrorCode::ChecksumMismatch}); + } + if (!content_size_is_known + && ZSTD_getErrorCode(decompressed_size) == ZSTD_error_dstSize_tooSmall) { + return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); + } + return std::unexpected(Error{ErrorCode::DecompressionFailed}); } - if (!content_size_is_known - && ZSTD_getErrorCode(decompressed_size) == ZSTD_error_dstSize_tooSmall) { - return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); + if (content_size_is_known && decompressed_size != uncompressed_data.size()) { + return std::unexpected(Error{ErrorCode::DecompressionFailed}); } - return std::unexpected(Error{ErrorCode::DecompressionFailed}); - } - if (content_size_is_known && decompressed_size != uncompressed_data.size()) { - return std::unexpected(Error{ErrorCode::DecompressionFailed}); + + uncompressed_data.resize(decompressed_size); } - uncompressed_data.resize(decompressed_size); + if (checksum_algorithm == ChecksumAlgorithm::Crc32c + && crc32c_checksum(uncompressed_data) != checksum) { + return std::unexpected(Error{ErrorCode::ChecksumMismatch}); + } return uncompressed_data; } diff --git a/src/terrainlib/io/compression.h b/src/terrainlib/io/compression.h index acaca8d1..104f9edd 100644 --- a/src/terrainlib/io/compression.h +++ b/src/terrainlib/io/compression.h @@ -18,6 +18,7 @@ inline constexpr std::size_t default_max_decompressed_size = std::size_t{1} << 3 enum class ChecksumAlgorithm : std::uint8_t { None, HandledByCompressionLib, + Crc32c, }; enum class CompressionAlgorithm : std::uint8_t { diff --git a/unittests/terrainlib/envelope.cpp b/unittests/terrainlib/envelope.cpp index a8b82273..109f2b1b 100644 --- a/unittests/terrainlib/envelope.cpp +++ b/unittests/terrainlib/envelope.cpp @@ -7,6 +7,7 @@ #include #include +#include namespace { @@ -132,6 +133,16 @@ void check_error(const Result &result, const io::envelope::ErrorCode expected) CHECK(result.error().code == expected); } +io::envelope::Bytes bytes_from_string(const std::string_view text) +{ + io::envelope::Bytes bytes; + bytes.reserve(text.size()); + for (const char character : text) { + bytes.push_back(static_cast(static_cast(character))); + } + return bytes; +} + } // namespace TEST_CASE("Envelope round trips the latest payload version") @@ -221,6 +232,158 @@ TEST_CASE("Envelope supports uncompressed data without a checksum") CHECK(*result == expected); } +TEST_CASE("CRC-32C uses its canonical hexadecimal representation") +{ + SECTION("standard test vector") + { + const auto compressed = io::envelope::compress_with_checksum( + bytes_from_string("123456789"), + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::Crc32c); + REQUIRE(compressed.has_value()); + CHECK(compressed->checksum == "e3069283"); + } + + SECTION("empty input") + { + const auto compressed = io::envelope::compress_with_checksum( + {}, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::Crc32c); + REQUIRE(compressed.has_value()); + CHECK(compressed->checksum == "00000000"); + } +} + +TEST_CASE("CRC-32C protects independently compressed data") +{ + const auto original = bytes_from_string("payload protected by CRC-32C"); + + SECTION("without compression") + { + const auto compressed = io::envelope::compress_with_checksum( + original, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::Crc32c); + REQUIRE(compressed.has_value()); + + const auto result = io::envelope::checked_decompress( + compressed->compressed_data, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::Crc32c, + compressed->checksum); + REQUIRE(result.has_value()); + CHECK(*result == original); + + auto corrupted = compressed->compressed_data; + corrupted.front() ^= std::byte{0x01}; + check_error( + io::envelope::checked_decompress( + corrupted, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::Crc32c, + compressed->checksum), + io::envelope::ErrorCode::ChecksumMismatch); + } + + SECTION("with zstd compression") + { + const auto compressed = io::envelope::compress_with_checksum( + original, + io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, + io::envelope::ChecksumAlgorithm::Crc32c); + REQUIRE(compressed.has_value()); + REQUIRE(compressed->checksum.size() == 8); + + const auto result = io::envelope::checked_decompress( + compressed->compressed_data, + io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, + io::envelope::ChecksumAlgorithm::Crc32c, + compressed->checksum); + REQUIRE(result.has_value()); + CHECK(*result == original); + + auto wrong_checksum = compressed->checksum; + wrong_checksum.front() = wrong_checksum.front() == '0' ? '1' : '0'; + check_error( + io::envelope::checked_decompress( + compressed->compressed_data, + io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, + io::envelope::ChecksumAlgorithm::Crc32c, + wrong_checksum), + io::envelope::ErrorCode::ChecksumMismatch); + } + + SECTION("malformed checksum") + { + check_error( + io::envelope::checked_decompress( + original, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::Crc32c, + "E3069283"), + io::envelope::ErrorCode::ChecksumMismatch); + check_error( + io::envelope::checked_decompress( + original, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::Crc32c, + {}), + io::envelope::ErrorCode::ChecksumMismatch); + } +} + +TEST_CASE("Envelope round trips and upgrades payloads protected by CRC-32C") +{ + SECTION("latest version") + { + const v3::Payload expected{ + .id = 12, + .label = "crc", + .enabled = true, + .samples = {3, 5, 8}, + }; + const auto bytes = io::envelope::serialize( + expected, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::Crc32c); + REQUIRE(bytes.has_value()); + + const auto envelope = decode_envelope(*bytes); + CHECK(envelope.checksum_algorithm == io::envelope::ChecksumAlgorithm::Crc32c); + CHECK(envelope.checksum.size() == 8); + CHECK(envelope.compression_algorithm == io::envelope::CompressionAlgorithm::None); + + const auto result = io::envelope::deserialize(*bytes); + REQUIRE(result.has_value()); + CHECK(*result == expected); + + auto corrupted = envelope; + corrupted.compressed_data.front() ^= std::byte{0x01}; + check_error(io::envelope::deserialize(encode_envelope(corrupted)), + io::envelope::ErrorCode::ChecksumMismatch); + } + + SECTION("older version") + { + const v1::Payload original{.id = 13, .name = "crc version one"}; + const auto bytes = io::envelope::serialize( + original, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::Crc32c); + REQUIRE(bytes.has_value()); + + const auto result = io::envelope::deserialize(*bytes); + REQUIRE(result.has_value()); + CHECK(*result == v3::Payload{ + .id = 13, + .label = "crc version one", + .enabled = true, + .samples = {}, + }); + } +} + TEST_CASE("Checked compression round trips and validates its checksum") { const io::envelope::Bytes original(4096, std::byte{0x2a}); @@ -359,6 +522,24 @@ TEST_CASE("Envelope rejects incompatible metadata") io::envelope::ErrorCode::InvalidAlgorithmCombination); } + SECTION("malformed CRC-32C checksum") + { + auto envelope = decode_envelope(*serialized); + envelope.checksum_algorithm = io::envelope::ChecksumAlgorithm::Crc32c; + envelope.checksum = "1234"; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::ChecksumMismatch); + } + + SECTION("incorrect CRC-32C checksum") + { + auto envelope = decode_envelope(*serialized); + envelope.checksum_algorithm = io::envelope::ChecksumAlgorithm::Crc32c; + envelope.checksum = "00000000"; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::ChecksumMismatch); + } + SECTION("uncompressed size is smaller than the payload") { auto envelope = decode_envelope(*serialized); From b424e204a4fddd1db02ebebc6f29aa11404cc182 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Tue, 11 Aug 2026 16:01:39 +0200 Subject: [PATCH 41/61] update plan with new DAG code --- docs/raster-store/refactor-plan.md | 196 ++++++++++++++++++++--------- docs/raster-store/todo.md | 6 - 2 files changed, 139 insertions(+), 63 deletions(-) diff --git a/docs/raster-store/refactor-plan.md b/docs/raster-store/refactor-plan.md index 379fb7a6..f783e5d0 100644 --- a/docs/raster-store/refactor-plan.md +++ b/docs/raster-store/refactor-plan.md @@ -21,9 +21,9 @@ Persistent raster-fundamentalis formats, adapters, and tools are later work. `src/terrainlib/sf` and use the `sf` namespace. - Existing 3D Structura Fundamentalis datasets must remain readable and writable without changing their on-disk contract. -- Existing DAG datasets stored in the 3D hierarchy with `.bin` payloads must - also remain readable and writable without changing their index or payload - serialization. +- DAG datasets written by the current code when Phase 0 begins must remain + readable and writable throughout the refactor without changing their index + or payload serialization. Older DAG payload schemas are not supported. - `Inner` is a valid shared topology state and is supported by DAG, raster-fundamentalis, and tile-base datasets. It is not valid in Structura Fundamentalis datasets. @@ -49,6 +49,8 @@ Persistent raster-fundamentalis formats, adapters, and tools are later work. `std::expected`; unsupported operations and other operational failures are reported as error values. Reading and writing must be reentrant (callable concurrently from different threads). +- The runtime glTF codec catches exceptions from the existing glTF mesh writer + and converts them to `CodecError`; changing the mesh I/O API is out of scope. - `store::RawStorage` exclusively owns the configured `std::unique_ptr>`. `store::Storage` owns the raw storage and therefore owns the codec transitively. Storage consumers do not @@ -58,6 +60,9 @@ Persistent raster-fundamentalis formats, adapters, and tools are later work. and does not change their concurrency guarantees. Existing caller-side synchronization and concurrency behaviour are preserved; any concurrency bug fix is separate work. +- Preserve `dag::ThreadSafeStorage` as the caller-side synchronization around + DAG output storage. DAG storage remains cacheless while shared-lock reads are + used. - Legacy index metadata selects a codec through an explicit, caller-supplied resolver supplied by the payload-domain opening function. The octree format adapter does not contain a global codec registry or depend on mesh or DAG @@ -125,7 +130,7 @@ Persistent raster-fundamentalis formats, adapters, and tools are later work. - Changing the existing 3D hard-link policy by adding a silent file-copy fallback. - Refactoring unrelated octree, DAG, mesh, or tile-builder code. -- Changing the serialized schema of existing DAG `.bin` payloads. +- Changing the serialized schema of the current DAG `.bin` payloads. ## Compatibility contract @@ -147,15 +152,16 @@ Before moving code, tests must lock down the following 3D behaviour: | Layout detection | both existing layouts remain detectable | | Mesh codec selection | legacy preferred extension selects terrain or configured glTF codec | | DAG codec selection | legacy `.bin` preferred extension selects the ZPP Bits codec | -| DAG payload encoding | existing `dag::ClusterBatch` ZPP Bits serialization | +| DAG payload encoding | current `dag::ClusterBatch` ZPP Bits serialization: metadata, then clustering | | Equal codec path lists | hard-link every file, or report an explicit error | | Different codec path lists | decode with input codec and encode with output codec | | Overwrite setting | `StorageSettings::allow_overwrite`, default `false`; enabled writes replace existing payloads | -Compatibility means that the refactored code can open datasets written before -the refactor. Pre-refactor readers are not tested against post-refactor outpus. Exact +Compatibility means that each refactor phase can open the Phase 0 fixtures +written by the current code. DAG formats older than the Phase 0 baseline and +pre-refactor readers opening post-refactor output are not tested. Exact byte-for-byte rewriting of an unordered index map is not required, but the -serialized schema and values must remain compatible. +serialized schema and values must remain compatible during the migration. The 3D index disk type should remain a versioned 3D adapter. The shared store must not add a topology field, new header, checksum, or compression layer to @@ -244,11 +250,19 @@ extensions are recorded in [todo.md](todo.md). Temporary forwarding headers and aliases under `octree` are allowed during migration. They must not contain a second implementation. -DAG serialization remains owned by `dag_builder`. Consolidate the existing -`dag::Id`, `dag::ClusterBatch`, `Clustering`, `Cluster`, and `TextureSet` -serialization functions in `src/dag_builder/serialization.h`. The DAG storage -adapter includes that header explicitly so template instantiation does not -depend on callers including `encoded.h` in the correct order. +DAG serialization remains owned by `dag_builder`. Consolidate the serializers +for `dag::Id`, `dag::Group`, `dag::NodeMetadata`, `dag::ClusterBatch`, +`radix::geometry::Aabb3d`, GLM vectors, `Clustering`, `Cluster`, and +`TextureSet` in `src/dag_builder/serialization.h`. The DAG storage adapter +includes that header explicitly so template instantiation does not depend on +caller include order. Preserve the current tuples exactly: + +- `ClusterBatch`: metadata, clustering; +- `NodeMetadata`: group assignment, groups; and +- `Group`: children, error, bounds. + +`Group::child_errors` is currently neither populated nor serialized. It remains +non-persistent in this refactor and the Phase 0 fixture locks down its omission. ## Shared interfaces @@ -312,6 +326,20 @@ Keys produced internally by `Traits::root()`, `Traits::parent()`, and they preserve validity. The child order affects traversal order and is locked down by the 2D and 3D trait tests; it is not serialized as separate metadata. +The API result shapes are: + +- index lookup returns `expected, InvalidKey>`; +- index mutation and predicates return `expected>`; +- traversal returns `expected>`; and +- storage `load`, `save`, `has`, `remove`, path lookup, and `copy_from` return + operation-specific `expected` types which retain `InvalidKey` and any + applicable codec, filesystem, missing-source, or overwrite error. + +During Phase 1, a forwarding `octree::IndexMap` compatibility wrapper preserves +the old optional/bool API for existing 3D callers with already-valid +`octree::Id` values. It delegates to `store::Index`, is not +a second implementation, and is removed after callers migrate. + ### Sparse index and traversal Move the existing four-state model to: @@ -486,6 +514,11 @@ store::cache::Interface ownership. Moving storage transfers ownership. Caches, layouts, index formats, resolvers, and application consumers never own the codec. +A move disarms the source so its destructor cannot save moved-out index state. +Move assignment must finalize a displaced dirty destination through the +existing destructor-save policy rather than silently discard it. Tests cover +the DAG builder's move into and release from `dag::ThreadSafeStorage`. + Domain-specific mesh codecs remain under `mesh::codec`. The reusable ZPP Bits codec remains under `store::codec`. RF codecs are deferred. @@ -555,6 +588,9 @@ mesh::codec::from_extension dag::codec::from_extension .bin -> store::codec::ZppBits + +dag::codec::metadata_from_extension + .bin -> read-only dag::codec::MetadataView ``` An unknown extension returns an explicit `UnsupportedCodec` error. Opening a @@ -563,7 +599,12 @@ constructed codec at the payload-domain opening boundary; the generic storage does not infer persistent metadata from `Codec::paths()`. Mesh and DAG convenience functions select or resolve the codec and pass ownership into raw storage, so application storage consumers do not handle codec objects. -Convenience functions in `src/dag_builder/storage.h` supply the DAG resolver. +Convenience functions in `src/dag_builder/storage.h` supply the DAG resolvers. +Preserve `DagMetaStorage` and `IndexedDagMetaStorage` as independently readable +views of the metadata prefix in each `ClusterBatch` `.bin` file. Their codec +returns `UnsupportedOperation` from writes instead of terminating or replacing +a full batch with metadata alone. `dag::codec::MetadataView` implements this +domain-specific read-only prefix view over the generic ZPP Bits codec. Opening functions return their requested storage type through `std::expected<..., OpenError>`. `OpenError` is a typed sum which retains the @@ -578,13 +619,18 @@ failing path and the underlying error where applicable: Loading and saving likewise return storage-level expected errors which retain an invalid key, an underlying `CodecError`, and `AlreadyExists` for a rejected save. `CopyError` retains invalid-key, missing-source, overwrite, filesystem, -and codec failures. Domain/application boundaries may add context, but must not -discard these errors. +and codec failures. The SF call chains changed in Phase 4 retain these errors +to their application boundary. DAG callers preserve their existing per-node +log-and-continue and command-line policies; making the DAG builder fail-fast is +not part of this refactor. Legacy unindexed-directory discovery remains in the 3D adapter: it recognizes candidate endings by asking the supplied resolver, removes an accepted ending to obtain a `NodePath`, and then invokes the selected layout parser. The generic layout does not recover keys directly from codec-owned file paths. +Only a missing index triggers this discovery path. An unreadable or malformed +index, unknown layout, or unsupported codec selector returns `OpenError` and +must not silently fall back to directory discovery. Automatic dirty-index saving currently happens in the 3D storage destructor. Preserve that behaviour for existing 3D entry points during the migration. @@ -635,14 +681,15 @@ The dummy path must be fixed and collision-free, for example `__codec_probe__/node`. Path lists are compared exactly, including count, order, and filename endings. -If linking several files fails partway through, return the error without a -transactional rollback guarantee; target links already created by the call -may remain. The copy operation stops immediately, leaves the target index -unchanged for that logical node, and propagates the failure until the -application aborts the overall operation. There is no journal, rollback, -cleanup guarantee, or silent copy fallback. An unsupported read or write needed -for re-encoding is returned through `CopyError`, retaining the underlying -`CodecError`. +If overwrite is enabled for an indexed target, remove the target index entry +immediately before the first target file is modified. If linking several files +then fails partway through, return the error without a transactional rollback +guarantee; target links or old files may remain, but the logical node stays +unindexed. The copy operation stops immediately and propagates the failure +until the application aborts the overall operation. There is no journal, +rollback, cleanup guarantee, or silent copy fallback. An unsupported read or +write needed for re-encoding is returned through `CopyError`, retaining the +underlying `CodecError`. Hard-link rules: @@ -731,7 +778,10 @@ No production behaviour changes. - index entries containing `Leaf` and `Virtual`, but no `Inner`. 2. Add one golden DAG dataset whose index selects `.bin`, whose payload contains a valid serialized `dag::ClusterBatch`, and whose index contains - `Leaf`, `Virtual`, and `Inner`. + `Leaf`, `Virtual`, and `Inner`. Use the current metadata-then-clustering + format, include non-trivial group metadata, and prove that the same file can + be opened through the read-only `NodeMetadata` view. Lock down that + `Group::child_errors` is not serialized. 3. Test that all fixtures open, resolve the expected IDs and extensions, and traverse the expected sparse nodes. 4. Add path round-trip tests for boundary IDs and both layouts. @@ -742,7 +792,8 @@ No production behaviour changes. - indexed and unindexed opens; and - final index creation by directory scan. 6. Record the pre-refactor public aliases used by `sf_builder`, `sf_merger`, - `sf_index_browser`, `dag_builder`, and `dag_convert_debug`. + `sf_index_browser`, `dag_builder`, and `dag_convert_debug`, including + `DagMetaStorage`, `IndexedDagMetaStorage`, and `dag::ThreadSafeStorage`. Exit criterion: the compatibility tests pass against the untouched implementation and fail when any stable filename, layout ID, path encoding, @@ -760,7 +811,8 @@ changed. 6. Run the same index-transition and DFS/BFS tests with both trait types, including deterministic child order, invalid-key errors through `std::expected`, maximum-depth children, and explicit traversal roots. -7. Provide temporary `octree` aliases so downstream migration is separate where this makes sense, otherwise migrate downstream immediately. +7. Provide the temporary forwarding `octree::IndexMap` wrapper and aliases so + downstream migration is separate; otherwise migrate downstream immediately. Exit criterion: 2D and 3D keys pass the same topology suite; existing 3D callers still build through aliases; no filesystem code has changed. @@ -785,6 +837,7 @@ layout-plus-codec path construction in one step. 5. Add the terrain codec and one glTF codec configured for binary `.glb` or JSON `.gltf`. Keep the current extension-dispatching `octree::MeshCodec` only as temporary production compatibility glue until the Phase 3 cutover. + Test that glTF writer exceptions become `CodecError` values. 6. Port the two existing 3D layouts to ordinary function pairs without changing stable IDs. The mappings return `level-index` and `level/x/y/z` without file endings. @@ -796,7 +849,9 @@ layout-plus-codec path construction in one step. paths. 9. Add focused codec tests using single-file, multi-file, read/write, and write-only test codecs. Test stable path ordering, unsupported operations, - directory creation, and conversion of domain errors to `CodecError`. + directory creation, conversion of domain errors to `CodecError`, and + concurrent reads and writes. Exercise reentrancy of every production codec + used by the parallel DAG builder. Exit criterion: the extensionless mappings and runtime codecs together resolve all Phase 0 fixtures to their existing physical payload paths; generic @@ -826,27 +881,31 @@ path. `mesh::codec`. 8. Add the function-pointer-based `IndexFormat`, `IndexMetadata`, and typed format/open errors. Split generic index maintenance from 3D index - serialization and legacy folder discovery. + serialization and legacy folder discovery. Test that only a missing index + starts discovery; all other index errors are returned. 9. Keep the current 3D `terrain.index` DTO and open functions as compatibility adapters over the shared storage. Retain its exact preferred extension as `codec_selector`, resolve it through the payload-domain mesh or DAG resolver, and retain the format metadata required by automatic saving. -10. Add DAG storage convenience functions that supply - `dag::codec::from_extension`, and mesh storage convenience functions that - select the configured terrain or glTF codec. Migrate `dag_builder`, - `dag_convert_debug`, and mesh storage consumers without exposing codec - objects at application call sites. +10. Add DAG storage convenience functions that supply the writable batch and + read-only metadata resolvers, and mesh storage convenience functions that + select the configured terrain or glTF codec. Preserve the DAG metadata + storage aliases. Migrate `dag_builder`, `dag_convert_debug`, and mesh + storage consumers without exposing codec objects at application call sites. 11. Migrate the existing octree storage aliases and all other application - callers. + callers, including adapting `dag::ThreadSafeStorage` to the new key and + expected-returning APIs while preserving its shared/exclusive locking. 12. Preserve the current 3D destructor-save behaviour until all callers have - explicit index finalization. + explicit index finalization. Test move construction, move assignment over + dirty state, moved-from destruction, and the DAG move/release path. 13. Preserve `StorageSettings::allow_overwrite`, including the DAG builder's overwrite mode and repeat debug export. Replace process termination on a rejected overwrite with `AlreadyExists` in the storage-level expected error. Test that a rejected save returns `AlreadyExists` and that enabling overwrite replaces the existing payload. -14. Add resolver tests for `.terrain`, `.glb`, `.gltf`, and `.bin` open/read - dispatch, plus explicit failure for an unknown preferred extension. +14. Add resolver tests for `.terrain`, `.glb`, `.gltf`, writable + `ClusterBatch` `.bin`, and read-only `NodeMetadata` `.bin` dispatch, plus + explicit failure for an unknown preferred extension and metadata writes. 15. Instantiate the shared storage tests with `raster_store::StoreTraits` using a test-only path mapping and codec. This proves the storage templates contain no hidden `octree::Id` dependency without defining a stable RF @@ -855,15 +914,15 @@ path. classes, static codec concept and codecs, and their temporary compatibility glue once no call site uses them. -Test that a pre-refactor DAG fixture opens through the new resolver and that a -new deterministic `.bin` payload matches the Phase 0 golden bytes and remains -readable through the unchanged ZPP serialization functions. Test unknown -layout IDs, malformed index metadata, invalid hierarchy keys, and retained -underlying open/codec errors through `std::expected`. +Test that the Phase 0 DAG fixture opens through both new resolvers and that a +new deterministic `.bin` payload matches its golden bytes and remains readable +through the unchanged ZPP serialization functions. Test unknown layout IDs, +malformed index metadata, invalid hierarchy keys, and retained underlying +open/codec errors through `std::expected`. Exit criterion: all existing applications build and all Phase 0 fixtures pass -through the shared runtime codec and storage implementation. Existing DAG -payload bytes and `.bin` paths remain compatible. No extension-dispatching +through the shared runtime codec and storage implementation. Phase 0 DAG +payload bytes and `.bin` paths remain unchanged. No extension-dispatching mesh codec, layout inheritance, RTTI lookup, static registrar, owning strategy pointer, or second storage implementation remains under `octree`. @@ -881,6 +940,8 @@ pointer, or second storage implementation remains under `octree`. - multi-file hard linking; - different path counts and endings; - error propagation after a partially failed multi-file hard link; + - overwrite failure leaving an existing logical node unindexed even when + old or partial files remain; - conversion between terrain and glTF; - `copy_from()` overwrite rejection; - `copy_from()` overwrite-enabled replacement; and @@ -895,6 +956,9 @@ pointer, or second storage implementation remains under `octree`. `sf::InvalidTopology` to the command line, and report that the output is invalid. Do not apply the validator when opening DAG datasets, through generic octree/store adapters, or in the diagnostic `sf_index_browser`. + Extract an SF-builder finalization boundary into `sfbuilderlib` which writes + the index, validates it, and returns the typed result so output validation + and command-line propagation are directly testable. 7. Keep SF recursion, subtree traversal, and mesh policy in `sf_merger`. Change its subtree and cut call chains to propagate validation and `copy_from()` failures through `std::expected` to the application boundary. @@ -936,6 +1000,18 @@ walker has been introduced. Exit criterion: repository search finds no generic implementation tied to `octree::Id`; all tests pass; the old layout strategy hierarchy is gone. +### Phase 6 — Remove migration-only compatibility fixtures + +1. Delete the Phase 0 golden SF and DAG datasets and the tests whose only + purpose is opening or byte-comparing data written before the completed + refactor. +2. Keep the format round-trip, path, resolver, topology, storage, error, and + application integration tests which exercise the final implementation + without pre-refactor fixture files. + +Exit criterion: no pre-refactor dataset fixture remains, and the retained test +suite passes against data produced by the final implementation. + ## Test and verification plan Generic store tests should live in the existing `unittests_terrainlib` target. @@ -947,13 +1023,14 @@ unittests/terrainlib/store_traverse.cpp unittests/terrainlib/store_layout.cpp unittests/terrainlib/store_codec.cpp unittests/terrainlib/store_storage.cpp -unittests/terrainlib/store_compatibility.cpp +unittests/terrainlib/store_compatibility.cpp # temporary through Phase 5 unittests/terrainlib/sf_validate_index.cpp ``` -The DAG payload-compatibility fixture and resolver integration test belong in -`unittests_dagbuilder`, because `terrainlib` must not depend on -`dag::ClusterBatch` or its serializers. +The temporary DAG payload-compatibility fixture and resolver integration test +belong in `unittests_dagbuilder`, because `terrainlib` must not depend on +`dag::ClusterBatch` or its serializers. Phase 6 removes the fixture and its +byte-comparison test while retaining resolver tests built from current data. The validator's unit tests belong in `unittests_terrainlib`. Boundary tests belong with their consumers: SF-builder output validation in @@ -973,8 +1050,11 @@ During implementation: 4. Run the full `unittests_terrainlib` target at every phase boundary. 5. Run `unittests_dagbuilder` after the ZPP codec, DAG serialization header, or DAG resolver changes. -6. Build `sf_builder`, `sf_merger`, `sf_index_browser`, `dag_builder`, and - `dag_convert_debug` after their storage aliases move. +6. Configure with `ALP_BUILD_SF_BUILDER`, `ALP_BUILD_SF_MERGER`, + `ALP_BUILD_SF_INDEX_BROWSER`, `ALP_BUILD_DAG_BUILDER`, and + `ALP_BUILD_DAG_CONVERT_DEBUG` enabled, then build `sf-builder`, `sf-merger`, + `sf-index-browser`, `dag-builder`, and `dag-convert-debug` after their + storage aliases move. 7. Run `unittests_sfbuilder`, `unittests_sfmerger`, and `unittests_dagbuilder`, plus any existing merger integration fixture, after the Phase 4 validation changes. @@ -1000,14 +1080,16 @@ No formatting-only pass or unrelated refactor belongs in these commits. | `octree/storage/codec/Codec.h` | runtime `store/Codec.h` | | `octree/storage/codec/DefaultCodec.h` | runtime `store/codec/ZppBits.h` | | `octree/storage/codec/MeshCodec.h` | `mesh/codec/Terrain.h` and configured `mesh/codec/Gltf.h` | +| `octree/storage/codec/ReadOnlyCodec.h` metadata specialization | `dag::codec::MetadataView` | | `octree/storage/RawStorage.h` | `store/RawStorage.h` | | `octree/storage/Storage.h` | `store/Storage.h` | | `octree/storage/IndexedStorage.h` | `store/IndexedStorage.h` | | `octree::StorageSettings::allow_overwrite` | `store::StorageSettings::allow_overwrite`, preserving default and enabled behaviour | | `octree/storage/helpers.*` | generic scan helpers plus 3D format adapter | | `octree/disk/IndexFile.h` | versioned 3D format adapter under `octree` | -| DAG serializers in `dag_node.h` and `encoded.h` | `dag_builder/serialization.h` | -| `dag_builder/storage.h` aliases | DAG storage aliases plus codec resolver convenience functions | +| DAG serializers in `dag_node.h`, `dag_id.h`, `metadata.h`, `encoded.h`, and `zpp_bits_glm.h` | `dag_builder/serialization.h` | +| `dag_builder/storage.h` aliases | Batch and read-only metadata storage aliases plus codec resolver convenience functions | +| `dag_builder/thread_safe_storage.h` | Adapted caller-side wrapper over shared storage | | `sf_merger::NodeWriter` subtree loop | remains in `sf_merger`; return copy failures through `std::expected` | | `sf_merger::NodeWriter` auxiliary `.png` write | remains an unmanaged, application-local debug artifact | | `sf_merger::cut_leaf_node()` copy path | remains in `sf_merger`; return copy failures through `std::expected` | @@ -1018,7 +1100,7 @@ No formatting-only pass or unrelated refactor belongs in these commits. | Risk | Control | |---|---| | Existing indexes stop loading | Golden pre-refactor fixtures and unchanged 3D DTO | -| Existing DAG `.bin` datasets stop loading | Golden DAG fixture, unchanged serializers, and explicit `.bin` resolver | +| The refactor changes current DAG `.bin` bytes | Temporary golden DAG fixture, unchanged serializers, and explicit `.bin` resolvers | | Octree format adapter gains DAG dependencies | Caller-supplied resolver owned by `dag_builder` | | Unknown legacy extension silently selects the wrong codec | Return an explicit `UnsupportedCodec` error | | Valid legacy paths are parsed differently | Characterization and round-trip tests before replacement | @@ -1028,7 +1110,7 @@ No formatting-only pass or unrelated refactor belongs in these commits. | Invalid `Inner` nodes reach SF merge dispatch | Validate every SF input first and return the offending key in a typed error | | SF subtree copying is generalized before RF requirements exist | Keep it in `sf_merger`; reconsider extraction with `rf_merger` | | A paired-walker API is fixed before RF semantics are known | Defer its action algebra until `rf_merger` requirements are defined | -| Multi-file hard linking fails partway through | Stop immediately, leave the node unindexed, propagate the error, and abort the overall operation; partial files may remain | +| Multi-file hard linking fails partway through | Remove any existing index entry before modifying target files, stop immediately, leave the node unindexed, propagate the error, and abort the overall operation; old or partial files may remain | | Incompatible codecs return the same path list | Treat path-list equality as a codec contract and test every concrete codec pairing | | Output-only codec is selected for required input | Return a clear `UnsupportedOperation` error | | Shared code accumulates mesh or provisional RF policy | Dependency tests/review against the source boundary | @@ -1044,8 +1126,8 @@ final here. A later RF design phase must resolve and test at least: -- the persistent index filename, schema, versioning, and serialization - envelope; +- the persistent index filename and schema, and how they use the existing + generic serialization envelope and versioning support; - persistent tile keys, coordinate convention, layout IDs, and node paths; - tile and source-attribution payload formats and codecs; - snapshot construction, validation, publication, and crash expectations; diff --git a/docs/raster-store/todo.md b/docs/raster-store/todo.md index fecdfba6..fe9bf14a 100644 --- a/docs/raster-store/todo.md +++ b/docs/raster-store/todo.md @@ -3,9 +3,3 @@ - Consider extending `sf::validate_index()` beyond rejecting `Inner` after additional SF invariants and their required error reporting are defined. This is not part of the shared-store refactor. -- Resolve the existing glTF write exception path before implementing the - runtime codec error contract. `mesh::io::gltf::save_to_path()` currently - throws when `cgltf_write_file()` fails, while codec operational failures are - intended to be returned through `std::expected`. Decide whether mesh I/O - should return a typed error or the glTF codec should catch and translate the - exception, and explicitly confirm whether changing mesh I/O is in scope. From cfe96d9aef138d14fce67c04608ed8509226126a Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:12:01 +0200 Subject: [PATCH 42/61] Update Radix dependency --- src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d8488333..0cdac5df 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -69,7 +69,7 @@ set(LIBIGL_COPYLEFT_CORE ON) set(LIBIGL_RESTRICTED_TRIANGLE ON) alp_add_git_repository(libigl URL https://github.com/libigl/libigl.git COMMITISH v2.6.0) -alp_add_git_repository(radix URL https://github.com/AlpineMapsOrg/radix.git COMMITISH 2ce3484513b826bb11e5433f868b048eaffe3e5d NOT_SYSTEM) +alp_add_git_repository(radix URL https://github.com/AlpineMapsOrg/radix.git COMMITISH 6babc632401cb95a701ce431cb68adf91e2c1b71 NOT_SYSTEM) set(ALP_CLI_COMPONENT_OPTIONS ALP_BUILD_SF_MERGER From 2c133b82929b502d710d9ed516738b06f9b984b0 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Wed, 12 Aug 2026 21:44:41 +0200 Subject: [PATCH 43/61] Disable GEOS developer warnings --- cmake/SetupGDAL.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/SetupGDAL.cmake b/cmake/SetupGDAL.cmake index 6be25b74..fdc85178 100644 --- a/cmake/SetupGDAL.cmake +++ b/cmake/SetupGDAL.cmake @@ -37,6 +37,7 @@ function(alp_setup_gdal) CMAKE_ARGUMENTS -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTING=OFF + -DGEOS_BUILD_DEVELOPER=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON ) From 6660990788e950aff36b9a3e94a404d2a520da52 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Thu, 13 Aug 2026 07:29:49 +0200 Subject: [PATCH 44/61] Handle storage write failures --- src/dag_builder/build.cpp | 5 ++++- src/sf_merger/cut.h | 5 ++++- src/sf_merger/merge.h | 5 ++++- src/terrainlib/octree/storage/codec/ReadOnlyCodec.h | 5 ++--- src/terrainlib/octree/storage/open.inl | 5 ++++- 5 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/dag_builder/build.cpp b/src/dag_builder/build.cpp index c7e453be..a176aa28 100644 --- a/src/dag_builder/build.cpp +++ b/src/dag_builder/build.cpp @@ -529,7 +529,10 @@ std::unordered_set build_level( const auto save_result = ctx.output_storage.save(target, *result); if (save_result) { if (debug_storage) { - debug_storage->save(target, clustering_to_mesh(result->clustering)); + const auto debug_save_result = debug_storage->save(target, clustering_to_mesh(result->clustering)); + if (!debug_save_result.has_value()) { + LOG_ERROR_AND_EXIT("Failed to save debug mesh for node {}: {}", target, debug_save_result.error()); + } } saved_ids.push_back(target); } else { diff --git a/src/sf_merger/cut.h b/src/sf_merger/cut.h index b33d0a37..411d47ce 100644 --- a/src/sf_merger/cut.h +++ b/src/sf_merger/cut.h @@ -114,7 +114,10 @@ inline void cut_dataset( const bool keep_inside) { Context ctx(input, output, octree::Space::earth(), keep_inside); cut_node(ctx, octree::Id::root(), mask); - output.save_or_create_index(); + const auto index_result = output.save_or_create_index(); + if (!index_result.has_value()) { + LOG_ERROR_AND_EXIT("Failed to save output index in {}: {}", output.base_path(), index_result.error()); + } } inline void cut_dataset( diff --git a/src/sf_merger/merge.h b/src/sf_merger/merge.h index ea3a1b18..180dbb59 100644 --- a/src/sf_merger/merge.h +++ b/src/sf_merger/merge.h @@ -177,5 +177,8 @@ inline void merge_datasets( merger.merge_root(); } - output_dataset.save_or_create_index(); + const auto index_result = output_dataset.save_or_create_index(); + if (!index_result.has_value()) { + LOG_ERROR_AND_EXIT("Failed to save output index in {}: {}", output_dataset.base_path(), index_result.error()); + } } diff --git a/src/terrainlib/octree/storage/codec/ReadOnlyCodec.h b/src/terrainlib/octree/storage/codec/ReadOnlyCodec.h index 2d397cb2..3ea68ba9 100644 --- a/src/terrainlib/octree/storage/codec/ReadOnlyCodec.h +++ b/src/terrainlib/octree/storage/codec/ReadOnlyCodec.h @@ -1,9 +1,8 @@ #pragma once +#include #include -#include - #include "log.h" namespace octree { @@ -12,7 +11,7 @@ namespace octree { // read-only (e.g. a view that aliases another storage's files). template struct ReadOnlyCodec : Codec { - static tl::expected save_to_path( + static std::expected save_to_path( const typename Codec::value_type &, const std::filesystem::path &) noexcept { LOG_ERROR_AND_EXIT("Attempted to write through a read-only codec"); } diff --git a/src/terrainlib/octree/storage/open.inl b/src/terrainlib/octree/storage/open.inl index 473308c2..c60c3699 100644 --- a/src/terrainlib/octree/storage/open.inl +++ b/src/terrainlib/octree/storage/open.inl @@ -93,7 +93,10 @@ Storage_ open_folder( IndexMap map; helpers::update_index_map(map, layout); if (!map.empty()) { - helpers::save_index_map(map, layout); + const auto save_result = helpers::save_index_map(map, layout); + if (!save_result.has_value()) { + LOG_ERROR_AND_EXIT("Failed to create storage index in {}: {}", base_path, save_result.error()); + } } return Storage_(RawStorage_(std::move(layout)), std::move(map)); } From 3d47287ff610e5b6b20090cfef114dbebcdf20a5 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Thu, 13 Aug 2026 07:31:19 +0200 Subject: [PATCH 45/61] Cancel parallel terrain builds after write failures --- src/sf_builder/terrainbuilder.cpp | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/sf_builder/terrainbuilder.cpp b/src/sf_builder/terrainbuilder.cpp index 3e7bde4b..2e30b2b4 100644 --- a/src/sf_builder/terrainbuilder.cpp +++ b/src/sf_builder/terrainbuilder.cpp @@ -275,6 +275,7 @@ void build_all_patches( logger->set_level(new_level); } + tbb::task_group_context context; tbb::parallel_for(size_t(0), target_nodes.size(), [&](size_t i) { const auto &node = target_nodes[i]; if (!overwrite_existing && storage.has(node)) { @@ -299,16 +300,33 @@ void build_all_patches( if (mesh_result.has_value()) { const auto mesh = std::move(mesh_result.value()); mesh::validate(mesh); - storage.save(node, mesh); + const auto save_result = storage.save(node, mesh); + if (!save_result.has_value()) { + LOG_ERROR("Failed to save mesh for node {}: {}", node, save_result.error()); + progress.task_finished(); + context.cancel_group_execution(); + return; + } } progress.task_finished(); - }); + }, context); // Restore original level logger->set_level(original_level); + if (context.is_group_execution_cancelled()) { + progress_thread.request_stop(); + } progress_thread.join(); - storage.save_or_create_index(); + + if (context.is_group_execution_cancelled()) { + LOG_ERROR_AND_EXIT("Failed to build all terrain patches"); + } + + const auto index_result = storage.save_or_create_index(); + if (!index_result.has_value()) { + LOG_ERROR_AND_EXIT("Failed to save output index in {}: {}", storage.base_path(), index_result.error()); + } } } From f999a0b6e6162aa8e97010bec961f29b8803d061 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Thu, 13 Aug 2026 07:40:53 +0200 Subject: [PATCH 46/61] Handle DAG index write failure --- src/dag_builder/main.cpp | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/dag_builder/main.cpp b/src/dag_builder/main.cpp index e247e119..16a57874 100644 --- a/src/dag_builder/main.cpp +++ b/src/dag_builder/main.cpp @@ -39,14 +39,18 @@ int main(int argc, char **argv) { // If the user specified neither target, fall back to a default error. if (!args.target_ratio && !args.target_error) { options.relative_target_error = 0.001f; - } - - dag::build_levels(input_storage, output_storage, options, args.level_range); - output_storage.save_index(); - - return EXIT_SUCCESS; + } + + dag::build_levels(input_storage, output_storage, options, args.level_range); + const auto index_result = output_storage.save_index(); + if (!index_result.has_value()) { + LOG_ERROR("Failed to save output index in {}: {}", args.output_path, index_result.error()); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; } catch (const std::exception &e) { LOG_ERROR("{}", e.what()); return EXIT_FAILURE; } -} \ No newline at end of file +} From 47c42a185db9bc03f113be5020bdec43ed6ffc57 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Thu, 13 Aug 2026 07:41:40 +0200 Subject: [PATCH 47/61] Preserve DAG main line endings --- src/dag_builder/main.cpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/dag_builder/main.cpp b/src/dag_builder/main.cpp index 16a57874..1827cca5 100644 --- a/src/dag_builder/main.cpp +++ b/src/dag_builder/main.cpp @@ -39,18 +39,18 @@ int main(int argc, char **argv) { // If the user specified neither target, fall back to a default error. if (!args.target_ratio && !args.target_error) { options.relative_target_error = 0.001f; - } - - dag::build_levels(input_storage, output_storage, options, args.level_range); - const auto index_result = output_storage.save_index(); - if (!index_result.has_value()) { - LOG_ERROR("Failed to save output index in {}: {}", args.output_path, index_result.error()); - return EXIT_FAILURE; - } - - return EXIT_SUCCESS; + } + + dag::build_levels(input_storage, output_storage, options, args.level_range); + const auto index_result = output_storage.save_index(); + if (!index_result.has_value()) { + LOG_ERROR("Failed to save output index in {}: {}", args.output_path, index_result.error()); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; } catch (const std::exception &e) { LOG_ERROR("{}", e.what()); return EXIT_FAILURE; } -} +} From c7b6be231a90d6c2afb6d2a3796c16d42a7704b9 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Thu, 13 Aug 2026 07:51:54 +0200 Subject: [PATCH 48/61] Check mesh fixture writes --- unittests/terrainlib/mesh_io.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/unittests/terrainlib/mesh_io.cpp b/unittests/terrainlib/mesh_io.cpp index cb117366..fd886bac 100644 --- a/unittests/terrainlib/mesh_io.cpp +++ b/unittests/terrainlib/mesh_io.cpp @@ -90,7 +90,7 @@ TEST_CASE("io roundtrip") { std::filesystem::remove(mesh_path); CHECK(!std::filesystem::exists(mesh_path)); - mesh::io::save_to_path(mesh, mesh_path, mesh::io::SaveOptions{.texture_format = ".png"}); + REQUIRE(mesh::io::save_to_path(mesh, mesh_path, mesh::io::SaveOptions{.texture_format = ".png"}).has_value()); CHECK(std::filesystem::exists(mesh_path)); const std::expected result = mesh::io::load_from_path(mesh_path); @@ -133,7 +133,7 @@ TEST_CASE("io roundtrip high precision") { std::filesystem::remove(mesh_path); CHECK(!std::filesystem::exists(mesh_path)); - mesh::io::save_to_path(mesh, mesh_path, mesh::io::SaveOptions{.texture_format = ".png"}); + REQUIRE(mesh::io::save_to_path(mesh, mesh_path, mesh::io::SaveOptions{.texture_format = ".png"}).has_value()); CHECK(std::filesystem::exists(mesh_path)); const std::expected result = mesh::io::load_from_path(mesh_path); @@ -176,7 +176,7 @@ TEST_CASE("io roundtrip no texture") { std::filesystem::remove(mesh_path); CHECK(!std::filesystem::exists(mesh_path)); - mesh::io::save_to_path(mesh, mesh_path); + REQUIRE(mesh::io::save_to_path(mesh, mesh_path).has_value()); CHECK(std::filesystem::exists(mesh_path)); const std::expected result = mesh::io::load_from_path(mesh_path); @@ -213,7 +213,7 @@ TEST_CASE("io roundtrip no texture and uvs") { std::filesystem::remove(mesh_path); CHECK(!std::filesystem::exists(mesh_path)); - mesh::io::save_to_path(mesh, mesh_path); + REQUIRE(mesh::io::save_to_path(mesh, mesh_path).has_value()); CHECK(std::filesystem::exists(mesh_path)); const std::expected result = mesh::io::load_from_path(mesh_path); From 6cac18f5ea2ff863fefd9ca164e9b67218dea7b1 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Fri, 14 Aug 2026 09:58:25 +0200 Subject: [PATCH 49/61] Add repeatable raster-store golden test --- docs/raster-store/golden-e2e.sh | 68 ++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 23 deletions(-) diff --git a/docs/raster-store/golden-e2e.sh b/docs/raster-store/golden-e2e.sh index 208ecb92..b919938c 100755 --- a/docs/raster-store/golden-e2e.sh +++ b/docs/raster-store/golden-e2e.sh @@ -7,39 +7,47 @@ set -Eeuo pipefail readonly SOURCE_DIR="/home/codex/Documents/alpine-terrain-builder/terrain-builder-raster-store" -readonly BUILD_DIR="${SOURCE_DIR}/build/hfa-vector-srs" +readonly BUILD_DIR="${SOURCE_DIR}/build/golden-e2e" readonly RUN_ROOT="/data/scratch/codex/alpine-terrain-builder-golden-e2e" readonly INPUT_ROOT="${RUN_ROOT}/inputs" readonly REFERENCE_ROOT="${RUN_ROOT}/reference" readonly LOG_ROOT="${RUN_ROOT}/logs" readonly STATE_ROOT="${RUN_ROOT}/state" +readonly RUN_ID="${RUN_ID:-$(date +%Y%m%dT%H%M%S)}" +readonly RUN_LOG_ROOT="${LOG_ROOT}/runs/${RUN_ID}" +readonly TIMINGS_FILE="${RUN_LOG_ROOT}/timings.tsv" readonly SF_BUILDER="${BUILD_DIR}/src/sf_builder/sf-builder" readonly SF_MERGER="${BUILD_DIR}/src/sf_merger/sf-merger" readonly DAG_BUILDER="${BUILD_DIR}/src/dag_builder/dag-builder" -# This VRT is a 4x4 km, native-resolution window into the original HFA dataset. -readonly GS_VRT="${INPUT_ROOT}/grossglockner-gs-4km/grossglockner-gs-4km.vrt" +# These VRTs are identical 4x4 km, native-resolution windows into the GS and +# GT rasters. They are recreated from the raw datasets at the start of a run. +readonly RAW_GS="/data/raw/raster_data/Oe_2020/OeRect_01m_gs_31287.img" +readonly RAW_GT="/data/raw/raster_data/Oe_2020/OeRect_01m_gt_31287.img" +readonly ELEVATION_ROOT="${INPUT_ROOT}/grossglockner-elevation-4km" +readonly GS_VRT="${ELEVATION_ROOT}/grossglockner-gs-4km.vrt" +readonly GT_VRT="${ELEVATION_ROOT}/grossglockner-gt-4km.vrt" readonly BASEMAP_TILES="${INPUT_ROOT}/basemap" readonly GATAKI_TILES="${INPUT_ROOT}/gataki" -# The exact Tirol border exercises the sf-merger's built-in 0.1 m -# topology-preserving mask simplification. -readonly TIROL_MASK="${INPUT_ROOT}/tirol-boundary/benchmark-shape/exact/tirol.shp" +# A 0.01 m simplification preserves the Tirol border to sub-centimetre area +# accuracy while avoiding the exact geometry's pathological merge time. +readonly TIROL_MASK="${INPUT_ROOT}/tirol-boundary/benchmark-shape/0_01m/tirol.shp" readonly SF_ROOT="${REFERENCE_ROOT}/sf" readonly DAG_ROOT="${REFERENCE_ROOT}/dag" -readonly SF_BASEMAP="${SF_ROOT}/grossglockner-gs-basemap-terrain" -readonly SF_GATAKI="${SF_ROOT}/grossglockner-gs-gataki-terrain" -readonly SF_MERGED="${SF_ROOT}/grossglockner-gs-merged-terrain" -readonly DAG_MERGED="${DAG_ROOT}/grossglockner-gs-merged-terrain" +readonly SF_BASEMAP="${SF_ROOT}/grossglockner-basemap-gs-terrain" +readonly SF_GATAKI="${SF_ROOT}/grossglockner-gataki-gt-terrain" +readonly SF_MERGED="${SF_ROOT}/grossglockner-merged-terrain" +readonly DAG_MERGED="${DAG_ROOT}/grossglockner-merged-terrain" readonly SF_TARGET_LEVEL="${SF_TARGET_LEVEL:-15}" readonly SF_THREADS="${SF_THREADS:-12}" readonly MIN_TEXTURE_LEVEL=12 readonly MAX_TEXTURE_LEVEL=19 -mkdir -p "${SF_ROOT}" "${DAG_ROOT}" "${LOG_ROOT}" "${STATE_ROOT}" -exec > >(tee -a "${LOG_ROOT}/golden-e2e.log") 2>&1 +mkdir -p "${SF_ROOT}" "${DAG_ROOT}" "${RUN_LOG_ROOT}" "${STATE_ROOT}" +exec > >(tee -a "${RUN_LOG_ROOT}/golden-e2e.log") 2>&1 timestamp() { @@ -64,7 +72,7 @@ run_timed() elapsed=$((end_epoch - start_epoch)) printf '%s\t%s\t%s\t%s\t%s\n' \ "${name}" "${start_epoch}" "${end_epoch}" "${elapsed}" "${status}" \ - >> "${LOG_ROOT}/timings.tsv" + >> "${TIMINGS_FILE}" printf '[%s] END %s status=%s elapsed_seconds=%s\n' \ "$(timestamp)" "${name}" "${status}" "${elapsed}" @@ -120,8 +128,9 @@ verify_snapshot() build_sf() { local name="$1" - local textures="$2" - local output="$3" + local dataset="$2" + local textures="$3" + local output="$4" local marker="${STATE_ROOT}/${name}.complete" if [[ -f "${marker}" ]]; then @@ -133,7 +142,7 @@ build_sf() mkdir -p "${output}" run_timed "${name}" \ "${SF_BUILDER}" \ - --dataset "${GS_VRT}" \ + --dataset "${dataset}" \ --textures "${textures}" \ --min-texture-level "${MIN_TEXTURE_LEVEL}" \ --max-texture-level "${MAX_TEXTURE_LEVEL}" \ @@ -168,8 +177,8 @@ merge_sf() mkdir -p "${SF_MERGED}" run_timed merge_sf \ "${SF_MERGER}" merge \ - --base "${SF_GATAKI}" \ - --new "${SF_BASEMAP}" \ + --base "${SF_BASEMAP}" \ + --new "${SF_GATAKI}" \ --mask "${TIROL_MASK}" \ --output "${SF_MERGED}" \ --verbosity info @@ -209,7 +218,7 @@ write_manifest() { local name="$1" local snapshot="$2" - local output="${LOG_ROOT}/${name}.sha256" + local output="${RUN_LOG_ROOT}/${name}.sha256" ( cd "${snapshot}" @@ -253,10 +262,23 @@ record_hard_links() printf 'merged_payloads=%s\nlinked_to_gataki=%s\nlinked_to_basemap=%s\nnewly_written=%s\n' \ "${merged_count}" "${linked_to_gataki}" "${linked_to_basemap}" "${newly_written}" \ - | tee "${LOG_ROOT}/merge-hard-links.txt" + | tee "${RUN_LOG_ROOT}/merge-hard-links.txt" } +prepare_elevation_vrts() +{ + mkdir -p "${ELEVATION_ROOT}" + gdal_translate -q -of VRT -srcwin 259507 245043 4000 4000 "${RAW_GS}" "${GS_VRT}" + gdal_translate -q -of VRT -srcwin 259507 245043 4000 4000 "${RAW_GT}" "${GT_VRT}" + gdal_edit.py -units m "${GS_VRT}" + gdal_edit.py -units m "${GT_VRT}" +} + +require_file "${RAW_GS}" +require_file "${RAW_GT}" +run_timed prepare_elevation_vrts prepare_elevation_vrts require_file "${GS_VRT}" +require_file "${GT_VRT}" require_file "${TIROL_MASK}" require_directory "${BASEMAP_TILES}" require_directory "${GATAKI_TILES}" @@ -273,10 +295,10 @@ require_executable "${DAG_BUILDER}" printf 'min_texture_level=%s\n' "${MIN_TEXTURE_LEVEL}" printf 'max_texture_level=%s\n' "${MAX_TEXTURE_LEVEL}" printf 'cpu_count=%s\n' "$(nproc)" -} >> "${LOG_ROOT}/run-metadata.txt" +} >> "${RUN_LOG_ROOT}/run-metadata.txt" -build_sf build_sf_basemap_terrain "${BASEMAP_TILES}" "${SF_BASEMAP}" -build_sf build_sf_gataki_terrain "${GATAKI_TILES}" "${SF_GATAKI}" +build_sf build_sf_basemap_gs_terrain "${GS_VRT}" "${BASEMAP_TILES}" "${SF_BASEMAP}" +build_sf build_sf_gataki_gt_terrain "${GT_VRT}" "${GATAKI_TILES}" "${SF_GATAKI}" merge_sf record_hard_links build_dag From d1309f54cfcd5756786735c6a2c849419c187ed7 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:44:24 +0200 Subject: [PATCH 50/61] Update raster store refactor plan for DAG changes --- docs/raster-store/refactor-plan.md | 31 +++++++++++------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/docs/raster-store/refactor-plan.md b/docs/raster-store/refactor-plan.md index f783e5d0..63b17e2c 100644 --- a/docs/raster-store/refactor-plan.md +++ b/docs/raster-store/refactor-plan.md @@ -57,9 +57,11 @@ Persistent raster-fundamentalis formats, adapters, and tools are later work. receive a codec template parameter and application call sites do not manage codec objects. - This refactor does not add synchronization to storage, indexes, or caches - and does not change their concurrency guarantees. Existing caller-side - synchronization and concurrency behaviour are preserved; any concurrency - bug fix is separate work. + and does not change their concurrency guarantees. Any concurrency bug fix is + separate work. +- Application callers retain their existing storage lifecycle, + synchronization, and failure-handling behaviour unless explicitly changed + by this plan. - Preserve `dag::ThreadSafeStorage` as the caller-side synchronization around DAG output storage. DAG storage remains cacheless while shared-lock reads are used. @@ -79,8 +81,7 @@ Persistent raster-fundamentalis formats, adapters, and tools are later work. booleans. - Preserve `StorageSettings::allow_overwrite`. It defaults to `false`; rejected overwrites return `AlreadyExists` through `std::expected`, and - enabling it preserves the existing DAG-builder overwrite and debug-export - behaviour. + enabling it replaces existing payloads. - The existing cache implementations are currently non-functional. Port their public API where it remains useful, keep application call sites building, and provide compile coverage only. Cache behaviour is not a compatibility @@ -261,9 +262,6 @@ caller include order. Preserve the current tuples exactly: - `NodeMetadata`: group assignment, groups; and - `Group`: children, error, bounds. -`Group::child_errors` is currently neither populated nor serialized. It remains -non-persistent in this refactor and the Phase 0 fixture locks down its omission. - ## Shared interfaces The names below are the intended shape, not signatures that must be copied @@ -620,9 +618,7 @@ Loading and saving likewise return storage-level expected errors which retain an invalid key, an underlying `CodecError`, and `AlreadyExists` for a rejected save. `CopyError` retains invalid-key, missing-source, overwrite, filesystem, and codec failures. The SF call chains changed in Phase 4 retain these errors -to their application boundary. DAG callers preserve their existing per-node -log-and-continue and command-line policies; making the DAG builder fail-fast is -not part of this refactor. +to their application boundary. Legacy unindexed-directory discovery remains in the 3D adapter: it recognizes candidate endings by asking the supplied resolver, removes an accepted ending @@ -632,9 +628,6 @@ Only a missing index triggers this discovery path. An unreadable or malformed index, unknown layout, or unsupported codec selector returns `OpenError` and must not silently fall back to directory discovery. -Automatic dirty-index saving currently happens in the 3D storage destructor. -Preserve that behaviour for existing 3D entry points during the migration. - ### Copying one node and SF subtree reuse There are two separate responsibilities in the current implementation: @@ -780,8 +773,7 @@ No production behaviour changes. contains a valid serialized `dag::ClusterBatch`, and whose index contains `Leaf`, `Virtual`, and `Inner`. Use the current metadata-then-clustering format, include non-trivial group metadata, and prove that the same file can - be opened through the read-only `NodeMetadata` view. Lock down that - `Group::child_errors` is not serialized. + be opened through the read-only `NodeMetadata` view. 3. Test that all fixtures open, resolve the expected IDs and extensions, and traverse the expected sparse nodes. 4. Add path round-trip tests for boundary IDs and both layouts. @@ -894,13 +886,12 @@ path. storage consumers without exposing codec objects at application call sites. 11. Migrate the existing octree storage aliases and all other application callers, including adapting `dag::ThreadSafeStorage` to the new key and - expected-returning APIs while preserving its shared/exclusive locking. + expected-returning APIs. 12. Preserve the current 3D destructor-save behaviour until all callers have explicit index finalization. Test move construction, move assignment over dirty state, moved-from destruction, and the DAG move/release path. -13. Preserve `StorageSettings::allow_overwrite`, including the DAG builder's - overwrite mode and repeat debug export. Replace process termination on a - rejected overwrite with `AlreadyExists` in the storage-level expected +13. Preserve `StorageSettings::allow_overwrite`. Replace process termination + on a rejected overwrite with `AlreadyExists` in the storage-level expected error. Test that a rejected save returns `AlreadyExists` and that enabling overwrite replaces the existing payload. 14. Add resolver tests for `.terrain`, `.glb`, `.gltf`, writable From 75277f89e5def61f97a8f7ee810b7c26c1c8c53b Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:17:57 +0200 Subject: [PATCH 51/61] Capture raster store compatibility baseline --- docs/raster-store/refactor-status.md | 59 +++++ unittests/CMakeLists.txt | 2 + .../dag_builder/storage_compatibility.cpp | 58 +++++ .../raster-store-refactor/dag/0/0/0/0.bin | Bin 0 -> 276 bytes .../raster-store-refactor/dag/2/2/3/1.bin | Bin 0 -> 276 bytes .../raster-store-refactor/dag/terrain.index | Bin 0 -> 78 bytes .../sf-coordinates/terrain.index | Bin 0 -> 92 bytes .../sf-flat/terrain.index | Bin 0 -> 34 bytes unittests/terrainlib/store_compatibility.cpp | 212 ++++++++++++++++++ 9 files changed, 331 insertions(+) create mode 100644 docs/raster-store/refactor-status.md create mode 100644 unittests/dag_builder/storage_compatibility.cpp create mode 100644 unittests/data/raster-store-refactor/dag/0/0/0/0.bin create mode 100644 unittests/data/raster-store-refactor/dag/2/2/3/1.bin create mode 100644 unittests/data/raster-store-refactor/dag/terrain.index create mode 100644 unittests/data/raster-store-refactor/sf-coordinates/terrain.index create mode 100644 unittests/data/raster-store-refactor/sf-flat/terrain.index create mode 100644 unittests/terrainlib/store_compatibility.cpp diff --git a/docs/raster-store/refactor-status.md b/docs/raster-store/refactor-status.md new file mode 100644 index 00000000..cf951dad --- /dev/null +++ b/docs/raster-store/refactor-status.md @@ -0,0 +1,59 @@ +# Raster store refactor status + +This file is the resumable implementation log for +[refactor-plan.md](refactor-plan.md). + +## Baseline + +- Branch: `raster-store` +- Baseline commit: `d1309f54cfcd5756786735c6a2c849419c187ed7` +- Baseline tag: `refactor_before` +- Started: 2026-08-14 +- Worktree was clean before implementation began. +- `dev_driver.py doctor` rejects this repository because its root + `CMakeLists.txt` is not the alpine-renderer project. Builds therefore use + this repository's own CMake configuration and build directories. + +## Current position + +- Phase: 0 — Capture current compatibility +- State: in progress +- Next action: run the full Phase 0 verification, inspect the diff, commit, + and tag the phase boundary. + +## Phase log + +### Phase 0 — Capture current compatibility + +- Added pre-refactor SF fixtures for `flat` and + `level_and_coordinate_directories` layouts. +- Added a pre-refactor DAG fixture using `.bin`, with `Leaf`, `Virtual`, and + `Inner` index states and non-trivial metadata. +- Added characterization coverage for fixture opening, metadata-prefix reads, + payload byte stability, layout/path round trips, hard links, + decode/re-encode, enabled overwrites, indexed/unindexed opens, and index + creation by directory scan. +- Recorded pre-refactor public aliases: + - SF builder, SF merger, and SF index browser use `octree::MeshStorage` and + `octree::IndexedStorage` (the latter aliases `IndexedMeshStorage`). + - DAG builder and DAG convert debug use `octree::DagStorage`, + `octree::IndexedDagStorage`, `octree::DagMetaStorage`, and + `octree::IndexedDagMetaStorage`. + - DAG output synchronization uses + `dag::ThreadSafeStorage`. +- State: complete. + +## Verification log + +- Baseline `unittests_terrainlib`: 23,676 assertions in 391 test cases passed. +- Baseline `unittests_dagbuilder`: 400 assertions in 69 test cases passed. +- Focused Phase 0 terrainlib compatibility tests: 51 assertions in 6 test + cases passed before warning cleanup. +- Focused Phase 0 DAG compatibility test: 23 assertions passed. +- Phase 0 `unittests_terrainlib`: 23,732 assertions in 397 test cases passed. +- Phase 0 `unittests_dagbuilder`: 423 assertions in 70 test cases passed. +- Phase 0 `git diff --check`: passed. + +## Commit and tag log + +- `refactor_before` — annotated baseline tag at `d1309f5`. diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 6e694422..7d89ca6f 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -30,6 +30,7 @@ add_executable(unittests_terrainlib terrainlib/offset_table.cpp terrainlib/segmented_buffer.cpp terrainlib/stamp_set.cpp + terrainlib/store_compatibility.cpp terrainlib/index_map.cpp terrainlib/index_traverse.cpp terrainlib/mesh_boundary.cpp @@ -120,6 +121,7 @@ if(TARGET dagbuilderlib) dag_builder/fixed_point.cpp dag_builder/texture_sizing.cpp dag_builder/texture_reuse.cpp + dag_builder/storage_compatibility.cpp ) target_link_libraries(unittests_dagbuilder PUBLIC dagbuilderlib Catch2::Catch2WithMain) atb_configure_test(unittests_dagbuilder) diff --git a/unittests/dag_builder/storage_compatibility.cpp b/unittests/dag_builder/storage_compatibility.cpp new file mode 100644 index 00000000..e68750bf --- /dev/null +++ b/unittests/dag_builder/storage_compatibility.cpp @@ -0,0 +1,58 @@ +#include + +#include + +#include "encoded.h" +#include "io/bytes.h" +#include "io/serialize.h" +#include "octree/disk/IndexFile.h" +#include "octree/storage/open.h" +#include "octree/traverse.h" +#include "storage.h" + +TEST_CASE("pre-refactor DAG fixture preserves index and payload contracts") { + const std::filesystem::path path = + std::filesystem::path(ALP_TEST_DATA_DIR) / "raster-store-refactor" / "dag"; + const octree::Id root = octree::Id::root(); + const octree::Id parent = root.child(3).value(); + const octree::Id leaf = parent.child(6).value(); + + const auto index_file = io::read_from_path(path / "terrain.index"); + REQUIRE(index_file.has_value()); + CHECK(index_file->layout_strategy_id == "level_and_coordinate_directories"); + CHECK(index_file->preferred_extension == ".bin"); + CHECK(index_file->map.is(octree::NodeStatus::Inner, root)); + CHECK(index_file->map.is(octree::NodeStatus::Virtual, parent)); + CHECK(index_file->map.is(octree::NodeStatus::Leaf, leaf)); + CHECK(std::filesystem::is_regular_file(path / "0/0/0/0.bin")); + CHECK(std::filesystem::is_regular_file(path / "2/2/3/1.bin")); + + std::vector visited; + octree::traverse(index_file->map, [&](const octree::Id id, const octree::NodeStatus) { + visited.push_back(id); + }); + CHECK(visited == std::vector{root, parent, leaf}); + + const auto batch_storage = octree::open_folder_indexed(path); + const auto batch = batch_storage.load(root); + REQUIRE(batch.has_value()); + CHECK(batch->metadata.group_assignment == std::vector{0}); + REQUIRE(batch->metadata.groups.size() == 1); + CHECK(batch->metadata.groups.front().error == 12.5); + CHECK(batch->metadata.groups.front().children == std::vector{{root.child(4).value(), 3}}); + REQUIRE(batch->clustering.clusters.size() == 1); + CHECK(batch->clustering.clusters.front().id == 17); + + const auto metadata_storage = octree::open_folder_indexed(path); + const auto metadata = metadata_storage.load(root); + REQUIRE(metadata.has_value()); + CHECK(metadata->group_assignment == batch->metadata.group_assignment); + REQUIRE(metadata->groups.size() == batch->metadata.groups.size()); + CHECK(metadata->groups.front().error == batch->metadata.groups.front().error); + + const auto encoded = io::write_to_bytes(batch.value()); + const auto fixture_bytes = io::read_bytes_from_path(path / "0/0/0/0.bin"); + REQUIRE(encoded.has_value()); + REQUIRE(fixture_bytes.has_value()); + CHECK(encoded.value() == fixture_bytes.value()); +} diff --git a/unittests/data/raster-store-refactor/dag/0/0/0/0.bin b/unittests/data/raster-store-refactor/dag/0/0/0/0.bin new file mode 100644 index 0000000000000000000000000000000000000000..7c8b1d28a199bea0ce758cdcc3ae51b44eed5bea GIT binary patch literal 276 zcmZQ%fB;4y1;#8;0cI%8z@X^>VSd;T6*>T+IiR$P1B7n^rR^MGdi8)33s>y|K?Vm2 z1`P=Y1rdna5B3n60jh_?0c<46;Dc dg{v8um|)7#X-SX*bU_$j5XN9&xBzn!0|33_7i<6k literal 0 HcmV?d00001 diff --git a/unittests/data/raster-store-refactor/dag/2/2/3/1.bin b/unittests/data/raster-store-refactor/dag/2/2/3/1.bin new file mode 100644 index 0000000000000000000000000000000000000000..7c8b1d28a199bea0ce758cdcc3ae51b44eed5bea GIT binary patch literal 276 zcmZQ%fB;4y1;#8;0cI%8z@X^>VSd;T6*>T+IiR$P1B7n^rR^MGdi8)33s>y|K?Vm2 z1`P=Y1rdna5B3n60jh_?0c<46;Dc dg{v8um|)7#X-SX*bU_$j5XN9&xBzn!0|33_7i<6k literal 0 HcmV?d00001 diff --git a/unittests/data/raster-store-refactor/dag/terrain.index b/unittests/data/raster-store-refactor/dag/terrain.index new file mode 100644 index 0000000000000000000000000000000000000000..e3f1d112fe64250be60fa4d32d0b944f214aaca5 GIT binary patch literal 78 zcmY#jU|`5eElbUbPs~e+PtMOTO3BPiEJ=+|$t+4uF3B&-Of6;ss?kfz%wq=9OmaX1 P3>ZON5MY8(K#~yvLw642 literal 0 HcmV?d00001 diff --git a/unittests/data/raster-store-refactor/sf-coordinates/terrain.index b/unittests/data/raster-store-refactor/sf-coordinates/terrain.index new file mode 100644 index 0000000000000000000000000000000000000000..546dd8bfb81ff88d2e18776d6159eb841eac8f5a GIT binary patch literal 92 zcmY#jU|`5eElbUbPs~e+PtMOTO3BPiEJ=+|$t+4uF3B&-OfBXBs?jS+EhSd6SNA(#jQ698sS5W4^X literal 0 HcmV?d00001 diff --git a/unittests/data/raster-store-refactor/sf-flat/terrain.index b/unittests/data/raster-store-refactor/sf-flat/terrain.index new file mode 100644 index 0000000000000000000000000000000000000000..c581a3f6f1d99d289ca268fad4d3421bf7cf191a GIT binary patch literal 34 ecmZQ!U|>kgNi5+2(t0JSMMa63d5l0d0ssJM*#y`C literal 0 HcmV?d00001 diff --git a/unittests/terrainlib/store_compatibility.cpp b/unittests/terrainlib/store_compatibility.cpp new file mode 100644 index 00000000..84e28a85 --- /dev/null +++ b/unittests/terrainlib/store_compatibility.cpp @@ -0,0 +1,212 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include "io/serialize.h" +#include "mesh/SimpleMesh.h" +#include "octree/Storage.h" +#include "octree/disk/IndexFile.h" +#include "octree/disk/layout/strategy/Flat.h" +#include "octree/disk/layout/strategy/LevelAndCoordinateDirectories.h" +#include "octree/storage/RawStorage.h" +#include "octree/storage/open.h" +#include "octree/traverse.h" + +namespace { + +std::filesystem::path fixture_path(const std::string_view name) { + return std::filesystem::path(ALP_TEST_DATA_DIR) / "raster-store-refactor" / name; +} + +class TemporaryDirectory { +public: + explicit TemporaryDirectory(const std::string_view label) { + static std::atomic_uint64_t counter = 0; + const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); + _path = std::filesystem::temp_directory_path() + / ("atb-raster-store-" + std::string(label) + "-" + + std::to_string(timestamp) + "-" + std::to_string(counter++)); + REQUIRE(std::filesystem::create_directories(_path)); + } + + ~TemporaryDirectory() { + std::error_code error; + std::filesystem::remove_all(_path, error); + } + + const std::filesystem::path &path() const { + return _path; + } + +private: + std::filesystem::path _path; +}; + +mesh::Simple triangle_mesh(const double offset) { + return mesh::Simple( + {{0, 1, 2}}, + {{offset, 0.0, 0.0}, {offset + 1.0, 0.0, 0.0}, {offset, 1.0, 0.0}}); +} + +octree::IndexedStorage make_storage( + const std::filesystem::path &path, + std::unique_ptr strategy, + const std::string &extension) { + octree::disk::Layout layout(path, std::move(strategy), extension); + return octree::IndexedStorage(octree::RawStorage_(std::move(layout)), octree::IndexMap{}); +} + +} // namespace + +TEST_CASE("pre-refactor SF fixtures preserve index and path contracts") { + const octree::Id root = octree::Id::root(); + + SECTION("flat") { + const std::filesystem::path path = fixture_path("sf-flat"); + const auto index_file = io::read_from_path(path / "terrain.index"); + REQUIRE(index_file.has_value()); + CHECK(index_file->layout_strategy_id == "flat"); + CHECK(index_file->preferred_extension == ".terrain"); + CHECK(index_file->map.size() == 1); + CHECK(index_file->map.is(octree::NodeStatus::Leaf, root)); + CHECK(std::filesystem::is_regular_file(path / "0-0.terrain")); + + const octree::IndexedStorage storage = octree::open_folder_indexed(path); + const auto mesh = storage.load(root); + REQUIRE(mesh.has_value()); + CHECK(mesh->face_count() == 1); + } + + SECTION("level and coordinate directories") { + const std::filesystem::path path = fixture_path("sf-coordinates"); + const octree::Id child = root.child(2).value(); + const octree::Id deep = root.child(5).value().child(7).value(); + const octree::Id deep_parent = deep.parent().value(); + + const auto index_file = io::read_from_path(path / "terrain.index"); + REQUIRE(index_file.has_value()); + CHECK(index_file->layout_strategy_id == "level_and_coordinate_directories"); + CHECK(index_file->preferred_extension == ".terrain"); + CHECK(index_file->map.is(octree::NodeStatus::Virtual, root)); + CHECK(index_file->map.is(octree::NodeStatus::Leaf, child)); + CHECK(index_file->map.is(octree::NodeStatus::Virtual, deep_parent)); + CHECK(index_file->map.is(octree::NodeStatus::Leaf, deep)); + CHECK(std::filesystem::is_regular_file(path / "1/0/1/0.terrain")); + CHECK(std::filesystem::is_regular_file(path / "2/3/1/3.terrain")); + + std::vector visited; + octree::traverse(index_file->map, [&](const octree::Id id, const octree::NodeStatus) { + visited.push_back(id); + }); + CHECK(visited == std::vector{octree::Id{root}, child, deep_parent, deep}); + } +} + +TEST_CASE("pre-refactor layouts round-trip boundary IDs") { + const std::vector ids{ + octree::Id::root(), + octree::Id(octree::Id::max_level(), octree::Id::Index{0}), + octree::Id(octree::Id::max_level(), octree::Id::max_index_on_level(octree::Id::max_level())), + }; + + octree::disk::layout::strategy::Flat flat; + octree::disk::layout::strategy::LevelAndCoordinateDirectories coordinates; + for (const octree::Id id : ids) { + const auto flat_path = flat.get_relative_node_path(id, ".terrain"); + CHECK(flat.get_id_from_relative_node_path(flat_path) == id); + + const auto coordinate_path = coordinates.get_relative_node_path(id, ".terrain"); + CHECK(coordinates.get_id_from_relative_node_path(coordinate_path) == id); + } +} + +TEST_CASE("pre-refactor storage hard-links matching payload formats") { + TemporaryDirectory source_directory("hard-link-source"); + TemporaryDirectory target_directory("hard-link-target"); + auto source = make_storage( + source_directory.path(), + std::make_unique(), + ".terrain"); + auto target = make_storage( + target_directory.path(), + std::make_unique(), + ".terrain"); + + const octree::Id root = octree::Id::root(); + REQUIRE(source.save(root, triangle_mesh(0.0)).has_value()); + REQUIRE(target.copy_from(root, source).has_value()); + CHECK(std::filesystem::equivalent(source.path_for(root), target.path_for(root))); + REQUIRE(source.save_index().has_value()); + REQUIRE(target.save_index().has_value()); +} + +TEST_CASE("pre-refactor storage re-encodes differing payload formats") { + TemporaryDirectory source_directory("reencode-source"); + TemporaryDirectory target_directory("reencode-target"); + auto source = make_storage( + source_directory.path(), + std::make_unique(), + ".terrain"); + auto target = make_storage( + target_directory.path(), + std::make_unique(), + ".glb"); + + const octree::Id root = octree::Id::root(); + REQUIRE(source.save(root, triangle_mesh(4.0)).has_value()); + REQUIRE(target.copy_from(root, source).has_value()); + CHECK_FALSE(std::filesystem::equivalent(source.path_for(root), target.path_for(root))); + const auto loaded = target.load(root); + REQUIRE(loaded.has_value()); + CHECK(loaded->face_count() == 1); + REQUIRE(source.save_index().has_value()); + REQUIRE(target.save_index().has_value()); +} + +TEST_CASE("pre-refactor storage supports enabled overwrites") { + TemporaryDirectory directory("overwrite"); + auto storage = make_storage( + directory.path(), + std::make_unique(), + ".terrain"); + storage.settings().allow_overwrite = true; + + const octree::Id root = octree::Id::root(); + REQUIRE(storage.save(root, triangle_mesh(1.0)).has_value()); + REQUIRE(storage.save(root, triangle_mesh(9.0)).has_value()); + const auto loaded = storage.load(root); + REQUIRE(loaded.has_value()); + REQUIRE_FALSE(loaded->positions.empty()); + CHECK(loaded->positions.front().x == 9.0); + REQUIRE(storage.save_index().has_value()); +} + +TEST_CASE("pre-refactor folder opening scans payloads and creates an index") { + TemporaryDirectory directory("folder-open"); + octree::OpenOptions options; + options.default_layout_strategy = std::make_unique(); + options.preferred_extension_with_dot = ".terrain"; + + { + auto storage = octree::open_folder(directory.path(), false, std::move(options)); + CHECK_FALSE(storage.is_indexed()); + REQUIRE(storage.save(octree::Id::root(), triangle_mesh(2.0)).has_value()); + CHECK_FALSE(std::filesystem::exists(directory.path() / "terrain.index")); + } + + { + const auto storage = octree::open_folder_indexed(directory.path()); + CHECK(storage.is_indexed()); + CHECK(storage.has(octree::Id::root())); + CHECK(std::filesystem::is_regular_file(directory.path() / "terrain.index")); + } + + const auto reopened = octree::open_index<>(directory.path() / "terrain.index"); + REQUIRE(reopened.has_value()); + CHECK(reopened->has(octree::Id::root())); +} From d756a77b51aa7d6cd133546a96fed68b02c686e0 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:18:21 +0200 Subject: [PATCH 52/61] Add ignored SF compatibility payloads --- docs/raster-store/refactor-status.md | 6 ++++++ .../sf-coordinates/1/0/1/0.terrain | Bin 0 -> 100 bytes .../sf-coordinates/2/3/1/3.terrain | Bin 0 -> 100 bytes .../raster-store-refactor/sf-flat/0-0.terrain | Bin 0 -> 103 bytes 4 files changed, 6 insertions(+) create mode 100644 unittests/data/raster-store-refactor/sf-coordinates/1/0/1/0.terrain create mode 100644 unittests/data/raster-store-refactor/sf-coordinates/2/3/1/3.terrain create mode 100644 unittests/data/raster-store-refactor/sf-flat/0-0.terrain diff --git a/docs/raster-store/refactor-status.md b/docs/raster-store/refactor-status.md index cf951dad..5493f513 100644 --- a/docs/raster-store/refactor-status.md +++ b/docs/raster-store/refactor-status.md @@ -57,3 +57,9 @@ This file is the resumable implementation log for ## Commit and tag log - `refactor_before` — annotated baseline tag at `d1309f5`. +- `75277f8` — Phase 0 compatibility tests, indexes, DAG payloads, and status + log. +- Phase 0 fixture-payload follow-up — force-adds the intentionally committed + `.terrain` golden payloads ignored by the repository's general artifact + rule. +- `refactor_phase_0` — annotated Phase 0 completion tag. diff --git a/unittests/data/raster-store-refactor/sf-coordinates/1/0/1/0.terrain b/unittests/data/raster-store-refactor/sf-coordinates/1/0/1/0.terrain new file mode 100644 index 0000000000000000000000000000000000000000..8b7a579c003dd10778221743462a9aab1ae525ff GIT binary patch literal 100 zcmZQ%U|?VdVlEK~%?M-*0rA5R3}x+M=@l#6QadwdFlI6^=mA9+uG+Qg*Q!+vEI=*t Q3=DNZAs|q308=;#08EYz_5c6? literal 0 HcmV?d00001 diff --git a/unittests/data/raster-store-refactor/sf-coordinates/2/3/1/3.terrain b/unittests/data/raster-store-refactor/sf-coordinates/2/3/1/3.terrain new file mode 100644 index 0000000000000000000000000000000000000000..bebf97b788532b2663c16a2becfe57e15b2126d4 GIT binary patch literal 100 zcmZQ%U|?VdVlEK~%?M-*0rA5R3}x+M=@l#6QadwdFlI6^=mA9+uG+Qg*Q!+vOh9Gw Q3=DNZAs{eu08=;#08Q8q0ssI2 literal 0 HcmV?d00001 diff --git a/unittests/data/raster-store-refactor/sf-flat/0-0.terrain b/unittests/data/raster-store-refactor/sf-flat/0-0.terrain new file mode 100644 index 0000000000000000000000000000000000000000..8bc0949026e08a52b88a5f372d02ddf2a804eb07 GIT binary patch literal 103 zcmZQ%U|?VdVlEK~%?M-*0rA5R3}x+M=@l#6QadwdFlI6^7y?BXuKERps~F@J80u;n N7~~lk>VTp|006Aq4+sDN literal 0 HcmV?d00001 From 4c171c881ce67344fa6a2119e3ae965eaaba05df Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:27:47 +0200 Subject: [PATCH 53/61] Extract shared hierarchy topology --- docs/raster-store/refactor-status.md | 29 ++- src/terrainlib/CMakeLists.txt | 1 - src/terrainlib/octree/IndexMap.cpp | 174 -------------- src/terrainlib/octree/IndexMap.h | 87 +++++-- src/terrainlib/octree/NodeStatus.h | 78 +------ src/terrainlib/octree/NodeStatusOrMissing.h | 135 +---------- src/terrainlib/octree/StoreTraits.h | 28 +++ src/terrainlib/octree/traverse.h | 80 ++----- src/terrainlib/raster_store/StoreTraits.h | 45 ++++ src/terrainlib/store/Index.h | 239 ++++++++++++++++++++ src/terrainlib/store/InvalidKey.h | 12 + src/terrainlib/store/NodeStatus.h | 85 +++++++ src/terrainlib/store/NodeStatusOrMissing.h | 134 +++++++++++ src/terrainlib/store/Traits.h | 17 ++ src/terrainlib/store/traverse.h | 99 ++++++++ unittests/CMakeLists.txt | 2 + unittests/terrainlib/store_index.cpp | 81 +++++++ unittests/terrainlib/store_traverse.cpp | 68 ++++++ 18 files changed, 920 insertions(+), 474 deletions(-) delete mode 100644 src/terrainlib/octree/IndexMap.cpp create mode 100644 src/terrainlib/octree/StoreTraits.h create mode 100644 src/terrainlib/raster_store/StoreTraits.h create mode 100644 src/terrainlib/store/Index.h create mode 100644 src/terrainlib/store/InvalidKey.h create mode 100644 src/terrainlib/store/NodeStatus.h create mode 100644 src/terrainlib/store/NodeStatusOrMissing.h create mode 100644 src/terrainlib/store/Traits.h create mode 100644 src/terrainlib/store/traverse.h create mode 100644 unittests/terrainlib/store_index.cpp create mode 100644 unittests/terrainlib/store_traverse.cpp diff --git a/docs/raster-store/refactor-status.md b/docs/raster-store/refactor-status.md index 5493f513..68cfdf06 100644 --- a/docs/raster-store/refactor-status.md +++ b/docs/raster-store/refactor-status.md @@ -16,10 +16,10 @@ This file is the resumable implementation log for ## Current position -- Phase: 0 — Capture current compatibility +- Phase: 1 — Extract topology into `store` - State: in progress -- Next action: run the full Phase 0 verification, inspect the diff, commit, - and tag the phase boundary. +- Next action: build the shared topology extraction and run focused topology + plus compatibility tests. ## Phase log @@ -43,6 +43,17 @@ This file is the resumable implementation log for `dag::ThreadSafeStorage`. - State: complete. +### Phase 1 — Extract topology into `store` + +- Added shared node-status types, hierarchy traits, invalid-key errors, + sparse index, and traversal under `terrainlib/store`. +- Added `octree::StoreTraits` and `raster_store::StoreTraits`. +- Converted `octree::IndexMap` and `octree::traverse` to compatibility + wrappers over the shared implementation. +- Added common 2D/3D topology and traversal coverage plus 2D boundary and + invalid-key tests. +- State: complete. + ## Verification log - Baseline `unittests_terrainlib`: 23,676 assertions in 391 test cases passed. @@ -53,6 +64,18 @@ This file is the resumable implementation log for - Phase 0 `unittests_terrainlib`: 23,732 assertions in 397 test cases passed. - Phase 0 `unittests_dagbuilder`: 423 assertions in 70 test cases passed. - Phase 0 `git diff --check`: passed. +- Phase 1 focused shared-store tests: 67 assertions in 7 test cases passed. +- Phase 1 pre-refactor terrain compatibility tests: 56 assertions in 6 test + cases passed. +- Phase 1 `unittests_terrainlib`: 23,799 assertions in 404 test cases passed. +- Phase 1 `unittests_dagbuilder`: 423 assertions in 70 test cases passed. +- Phase 1 application builds passed for `sf-builder`, `dag-builder`, and + `dag-convert-debug`. The current Debug configuration has `sf-merger` and + `sf-index-browser` disabled. +- The all-target build remains blocked by a pre-existing, unrelated call to + removed `radix::tile::Scheme` in `unittests/sf_builder/texture.cpp`; the + affected refactor targets and suites build independently. +- Phase 1 `git diff --check`: passed. ## Commit and tag log diff --git a/src/terrainlib/CMakeLists.txt b/src/terrainlib/CMakeLists.txt index a51a7821..4046109b 100644 --- a/src/terrainlib/CMakeLists.txt +++ b/src/terrainlib/CMakeLists.txt @@ -37,7 +37,6 @@ add_library(terrainlib mesh/validate.cpp octree/storage/helpers.cpp - octree/IndexMap.cpp octree/Space.cpp octree/OddLevelShifted.cpp diff --git a/src/terrainlib/octree/IndexMap.cpp b/src/terrainlib/octree/IndexMap.cpp deleted file mode 100644 index 8bb7e53c..00000000 --- a/src/terrainlib/octree/IndexMap.cpp +++ /dev/null @@ -1,174 +0,0 @@ -#include - -#include "octree/IndexMap.h" -#include "log.h" - -namespace octree { - -std::optional IndexMap::get(Id id) const { - const NodeStatus* status = this->get_raw(id); - if (status != nullptr) { - return *status; - } else { - return std::nullopt; - } -} - -bool IndexMap::add(Id id) { - NodeStatus* status = this->get_raw(id); - if (status != nullptr) { - switch (*status) { - case NodeStatus::Inner: - case NodeStatus::Leaf: - // Nothing to do - return false; - case NodeStatus::Virtual: - this->set_raw(id, NodeStatus::Inner); - return true; - } - } - - const auto parent_opt = id.parent(); - if (!parent_opt.has_value()) { - DEBUG_ASSERT(id.is_root()); - if (this->empty()) { - this->set_raw(id, NodeStatus::Leaf); - return true; - } else { - UNREACHABLE(); - return true; - } - } - - const auto parent = parent_opt.value(); - const NodeStatus* parent_status = this->get_raw(parent); - if (parent_status == nullptr) { - this->add(parent); - this->set_raw(parent, NodeStatus::Virtual); - this->set_raw(id, NodeStatus::Leaf); - return true; - } - - switch (*parent_status) { - case NodeStatus::Leaf: - this->set_raw(parent, NodeStatus::Inner); - this->set_raw(id, NodeStatus::Leaf); - break; - case NodeStatus::Inner: - case NodeStatus::Virtual: - this->set_raw(id, NodeStatus::Leaf); - break; - } - - return true; -} - -bool IndexMap::remove(Id id) { - NodeStatus* status = this->get_raw(id); - if (status == nullptr) { - return false; - } - - switch (*status) { - case NodeStatus::Inner: - *status = NodeStatus::Virtual; - return true; - case NodeStatus::Virtual: - // Nothing to do - return false; - case NodeStatus::Leaf: - this->remove_raw(id); - this->update_parent_after_remove(id); - return true; - } - - UNREACHABLE(); -} - -bool IndexMap::is_present(Id id) const { - auto it = this->_index.find(id); - return it != this->_index.end(); -} -bool IndexMap::is_absent(Id id) const { - return !this->is_present(id); -} -bool IndexMap::is(NodeStatus status, Id id) const { - return this->get(id) == status; -} - -void IndexMap::clear() { - this->_index.clear(); -} -bool IndexMap::empty() const { - return this->_index.empty(); -} -size_t IndexMap::size() const { - return this->_index.size(); -} - -IndexMap::const_iterator IndexMap::begin() const { - return this->_index.begin(); -} -IndexMap::const_iterator IndexMap::end() const { - return this->_index.end(); -} -IndexMap::const_iterator IndexMap::cbegin() const { - return this->_index.cbegin(); -} -IndexMap::const_iterator IndexMap::cend() const { - return this->_index.cend(); -} - -NodeStatus* IndexMap::get_raw(Id id) { - if (auto it = this->_index.find(id); it != this->_index.end()) { - return &it->second; - } - return nullptr; -} -const NodeStatus* IndexMap::get_raw(Id id) const { - if (auto it = this->_index.find(id); it != this->_index.end()) { - return &it->second; - } - return nullptr; -} -void IndexMap::set_raw(Id id, NodeStatus status) { - this->_index[id] = status; -} -void IndexMap::remove_raw(Id id) { - this->_index.erase(id); -} - -void IndexMap::update_parent_after_remove(Id id) { - const auto parent_opt = id.parent(); - if (!parent_opt.has_value()) { - return; - } - - const auto parent = parent_opt.value(); - NodeStatus* parent_status = this->get_raw(parent); - DEBUG_ASSERT(parent_status != nullptr); - - // We know there are children since we just removed one - DEBUG_ASSERT(parent.has_children()); - const auto siblings = parent.children().value(); - for (const auto& sibling : siblings) { - if (this->is_present(sibling) && sibling != id) { - return; - } - } - - switch (*parent_status) { - case NodeStatus::Inner: - *parent_status = NodeStatus::Leaf; - break; - case NodeStatus::Virtual: - this->remove_raw(parent); - this->update_parent_after_remove(parent); - break; - case NodeStatus::Leaf: - UNREACHABLE(); - break; - } -} - -} diff --git a/src/terrainlib/octree/IndexMap.h b/src/terrainlib/octree/IndexMap.h index c5dbf535..c231e763 100644 --- a/src/terrainlib/octree/IndexMap.h +++ b/src/terrainlib/octree/IndexMap.h @@ -1,49 +1,88 @@ #pragma once -#include #include #include -#include "octree/Id.h" #include "octree/NodeStatus.h" +#include "octree/StoreTraits.h" +#include "store/Index.h" namespace octree { class IndexMap { public: - using Container = std::unordered_map; - using iterator = Container::iterator; - using const_iterator = Container::const_iterator; + using SharedIndex = store::Index; + using iterator = SharedIndex::iterator; + using const_iterator = SharedIndex::const_iterator; - std::optional get(Id id) const; - bool add(Id id); - bool remove(Id id); + std::optional get(const Id id) const { + return _index.get(id).value(); + } + bool add(const Id id) { + return _index.add(id).value(); + } + bool remove(const Id id) { + return _index.remove(id).value(); + } + bool is_present(const Id id) const { + return _index.is_present(id).value(); + } + bool is_absent(const Id id) const { + return _index.is_absent(id).value(); + } + bool is(const NodeStatus status, const Id id) const { + return _index.is(status, id).value(); + } - bool is_present(Id id) const; - bool is_absent(Id id) const; - bool is(NodeStatus status, Id id) const; + void clear() { + _index.clear(); + } + bool empty() const { + return _index.empty(); + } + size_t size() const { + return _index.size(); + } - void clear(); - bool empty() const; - size_t size() const; + const_iterator begin() const { + return _index.begin(); + } + const_iterator end() const { + return _index.end(); + } + const_iterator cbegin() const { + return _index.cbegin(); + } + const_iterator cend() const { + return _index.cend(); + } - const_iterator begin() const; - const_iterator end() const; - const_iterator cbegin() const; - const_iterator cend() const; + NodeStatus *get_raw(const Id id) { + return _index.get_raw(id).value(); + } + const NodeStatus *get_raw(const Id id) const { + return _index.get_raw(id).value(); + } + void set_raw(const Id id, const NodeStatus status) { + _index.set_raw(id, status).value(); + } + void remove_raw(const Id id) { + _index.remove_raw(id).value(); + } - NodeStatus* get_raw(Id id); - const NodeStatus* get_raw(Id id) const; - void set_raw(Id id, NodeStatus status); - void remove_raw(Id id); + SharedIndex &shared() { + return _index; + } + const SharedIndex &shared() const { + return _index; + } using serialize = zpp::bits::members<1>; friend zpp::bits::access; private: - Container _index; - void update_parent_after_remove(Id id); + SharedIndex _index; }; } // namespace octree diff --git a/src/terrainlib/octree/NodeStatus.h b/src/terrainlib/octree/NodeStatus.h index 9e00d97c..185f4fdd 100644 --- a/src/terrainlib/octree/NodeStatus.h +++ b/src/terrainlib/octree/NodeStatus.h @@ -1,81 +1,7 @@ #pragma once -#include - -#include "log.h" +#include "store/NodeStatus.h" namespace octree { - -class NodeStatus { -public: - using Underlying = uint8_t; - enum Value : Underlying { - Leaf = 0, - Inner = 1, - Virtual = 2 // not present on disk but has children - }; - - constexpr NodeStatus() = default; - constexpr NodeStatus(Value value) : _value(value) {} - - constexpr operator Value() const { - return this->_value; - } - explicit operator bool() const = delete; - constexpr bool operator==(NodeStatus other) const { - return this->_value == other._value; - } - constexpr bool operator!=(NodeStatus other) const { - return !(*this == other); - } - constexpr bool operator==(Value other_value) const { - return this->_value == other_value; - } - constexpr bool operator!=(Value other_value) const { - return this->_value != other_value; - } - - std::string to_string() const; - -private: - Value _value; - -public: - using serialize = zpp::bits::members<1>; - friend zpp::bits::access; -}; - -} - -#include -template <> -struct fmt::formatter { - template - constexpr auto parse(ParseContext &ctx) { - return ctx.begin(); - } - - template - auto format(const octree::NodeStatus &status, FormatContext &ctx) const { - switch (status) { - case octree::NodeStatus::Leaf: - return fmt::format_to(ctx.out(), "Leaf"); - case octree::NodeStatus::Inner: - return fmt::format_to(ctx.out(), "Inner"); - case octree::NodeStatus::Virtual: - return fmt::format_to(ctx.out(), "Virtual"); - default: - UNREACHABLE(); - } - } -}; - -#include -#include -inline std::string octree::NodeStatus::to_string() const { - return fmt::format("{}", *this); -} -inline std::ostream &operator<<(std::ostream &os, const octree::NodeStatus &status) { - fmt::print(os, "{}", status); - return os; +using NodeStatus = store::NodeStatus; } diff --git a/src/terrainlib/octree/NodeStatusOrMissing.h b/src/terrainlib/octree/NodeStatusOrMissing.h index 0868cb93..1c71a67d 100644 --- a/src/terrainlib/octree/NodeStatusOrMissing.h +++ b/src/terrainlib/octree/NodeStatusOrMissing.h @@ -1,138 +1,7 @@ #pragma once -#include - -#include - -#include "octree/NodeStatus.h" - -namespace octree { - -class NodeStatusOrMissing { -public: - using Underlying = NodeStatus::Underlying; - enum Value : Underlying { - Leaf = 0, - Inner = 1, - Virtual = 2, // not present on disk but has children - Missing = 3 - }; - - constexpr NodeStatusOrMissing() = default; - constexpr NodeStatusOrMissing(Value value) : _value(value) {} - NodeStatusOrMissing(NodeStatus opt) : _value(from_status(opt)) {} - NodeStatusOrMissing(std::optional opt) : _value(from_optional_status(opt)) {} - - constexpr operator Value() const { - return this->_value; - } - constexpr operator std::optional() const { - return this->to_optional_status(); - } - explicit operator bool() const = delete; - constexpr bool operator==(NodeStatusOrMissing other) const { - return this->_value == other._value; - } - constexpr bool operator!=(NodeStatusOrMissing other) const { - return !(*this == other); - } - constexpr bool operator==(Value other_value) const { - return this->_value == other_value; - } - constexpr bool operator!=(Value other_value) const { - return this->_value != other_value; - } - - std::string to_string() const; - -private: - constexpr static NodeStatusOrMissing from_status(NodeStatus status) noexcept { - return static_cast(static_cast(status)); - } - constexpr static NodeStatusOrMissing from_optional_status(std::optional opt) noexcept { - if (opt.has_value()) { - return from_status(opt.value()); - } else { - return NodeStatusOrMissing::Missing; - } - } - constexpr std::optional to_optional_status() const noexcept { - if (this->_value == NodeStatusOrMissing::Missing) { - return std::nullopt; - } else { - return static_cast(this->_value); - } - } - -public: - Value _value; -}; - -} - -#include -template <> -struct fmt::formatter { - template - constexpr auto parse(ParseContext &ctx) { - return ctx.begin(); - } - - template - auto format(const octree::NodeStatusOrMissing &status, FormatContext &ctx) const { - switch (status) { - case octree::NodeStatusOrMissing::Leaf: - return fmt::format_to(ctx.out(), "Leaf"); - case octree::NodeStatusOrMissing::Inner: - return fmt::format_to(ctx.out(), "Inner"); - case octree::NodeStatusOrMissing::Virtual: - return fmt::format_to(ctx.out(), "Virtual"); - case octree::NodeStatusOrMissing::Missing: - return fmt::format_to(ctx.out(), "Missing"); - default: - UNREACHABLE(); - } - } -}; - -#include -#include -inline std::string octree::NodeStatusOrMissing::to_string() const { - return fmt::format("{}", *this); -} -inline std::ostream &operator<<(std::ostream &os, const octree::NodeStatusOrMissing &status) { - fmt::print(os, "{}", status); - return os; -} +#include "store/NodeStatusOrMissing.h" namespace octree { -namespace { -template -consteval bool _verify_enum_values(std::index_sequence) { - constexpr auto ns_values = magic_enum::enum_values(); - constexpr auto nsom_values = magic_enum::enum_values(); - - return ((magic_enum::enum_underlying(ns_values[Is]) == - magic_enum::enum_underlying(nsom_values[Is])) && - ...); -} - -consteval bool _verify_enum() { - static_assert( - std::is_same_v, - magic_enum::underlying_type_t>, - "Mismatched underlying type"); - - constexpr auto ns_values = magic_enum::enum_values(); - constexpr auto nsom_values = magic_enum::enum_values(); - - static_assert(nsom_values.size() == ns_values.size() + 1, "Mismatched enum counts"); - - static_assert(_verify_enum_values(std::make_index_sequence()), - "Enum value mismatch"); - - return true; -} -static_assert(_verify_enum()); -} +using NodeStatusOrMissing = store::NodeStatusOrMissing; } diff --git a/src/terrainlib/octree/StoreTraits.h b/src/terrainlib/octree/StoreTraits.h new file mode 100644 index 00000000..655656a4 --- /dev/null +++ b/src/terrainlib/octree/StoreTraits.h @@ -0,0 +1,28 @@ +#pragma once + +#include + +#include "octree/Id.h" + +namespace octree { + +struct StoreTraits { + using Key = Id; + using Hasher = std::hash; + + static constexpr Key root() { + return Key::root(); + } + static constexpr auto parent(const Key &key) { + return key.parent(); + } + static constexpr auto children(const Key &key) { + return key.children(); + } + static constexpr bool is_valid(const Key &key) { + return key.level() <= Key::max_level() + && key.index_on_level() <= Key::max_index_on_level(key.level()); + } +}; + +} // namespace octree diff --git a/src/terrainlib/octree/traverse.h b/src/terrainlib/octree/traverse.h index b912a0fa..6d0af8db 100644 --- a/src/terrainlib/octree/traverse.h +++ b/src/terrainlib/octree/traverse.h @@ -1,81 +1,35 @@ #pragma once #include -#include +#include -#include "octree/Id.h" -#include "octree/Storage.h" -#include "log.h" +#include + +#include "octree/IndexMap.h" +#include "store/traverse.h" namespace octree { -enum class TraversalOrder { - DepthFirst, - BreadthFirst -}; +using TraversalOrder = store::TraversalOrder; constexpr bool always_refine(const Id &) { return true; } -template < - typename VisitFn, - typename RefineFn = std::function> +template> void traverse( const IndexMap &index, - VisitFn &&visit_fn, - RefineFn &&refine_fn = always_refine, + VisitFn &&visit, + RefineFn &&refine = always_refine, const Id &root = Id::root(), - TraversalOrder order = TraversalOrder::DepthFirst) { - if (!index.is_present(root)) { - return; - } - - if (order == TraversalOrder::DepthFirst) { - std::function dfs; - dfs = [&](const Id& current) { - auto current_status_opt = index.get(current); - if (!current_status_opt) { - return; - } - const auto current_status = current_status_opt.value(); - - visit_fn(current, current_status); - - if (current.has_children() && refine_fn(current)) { - const auto children = current.children().value(); - for (const auto& child : children) { - dfs(child); - } - } - }; - dfs(root); - } else if (order == TraversalOrder::BreadthFirst) { - std::queue queue; - queue.push(root); - - while (!queue.empty()) { - Id current = queue.front(); - queue.pop(); - - auto current_status_opt = index.get(current); - if (!current_status_opt) { - continue; - } - const auto current_status = current_status_opt.value(); - - visit_fn(current, current_status); - - if (current.has_children() && refine_fn(current)) { - const auto children = current.children().value(); - for (const auto& child : children) { - queue.push(child); - } - } - } - } else { - UNREACHABLE(); - } + const TraversalOrder order = TraversalOrder::DepthFirst) { + const auto result = store::traverse( + index.shared(), + std::forward(visit), + std::forward(refine), + root, + order); + DEBUG_ASSERT(result.has_value()); } } // namespace octree diff --git a/src/terrainlib/raster_store/StoreTraits.h b/src/terrainlib/raster_store/StoreTraits.h new file mode 100644 index 00000000..a6ba4a8b --- /dev/null +++ b/src/terrainlib/raster_store/StoreTraits.h @@ -0,0 +1,45 @@ +#pragma once + +#include +#include +#include +#include + +#include + +namespace raster_store { + +struct StoreTraits { + using Key = radix::tile::Id; + using Hasher = Key::Hasher; + + static constexpr unsigned max_zoom_level = std::numeric_limits::digits; + + static Key root() { + return {0, {0, 0}}; + } + static std::optional parent(const Key &key) { + if (key.zoom_level == 0) { + return std::nullopt; + } + return key.parent(); + } + static std::optional> children(const Key &key) { + if (key.zoom_level >= max_zoom_level) { + return std::nullopt; + } + return key.children(); + } + static bool is_valid(const Key &key) { + if (key.zoom_level > max_zoom_level) { + return false; + } + if (key.zoom_level == max_zoom_level) { + return true; + } + const uint32_t extent = uint32_t{1} << key.zoom_level; + return key.coords.x < extent && key.coords.y < extent; + } +}; + +} // namespace raster_store diff --git a/src/terrainlib/store/Index.h b/src/terrainlib/store/Index.h new file mode 100644 index 00000000..22664de1 --- /dev/null +++ b/src/terrainlib/store/Index.h @@ -0,0 +1,239 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include + +#include "log.h" +#include "store/InvalidKey.h" +#include "store/NodeStatus.h" +#include "store/Traits.h" + +namespace store { + +template +class Index { +public: + using Key = typename Traits::Key; + using Error = InvalidKey; + using Container = std::unordered_map; + using iterator = typename Container::iterator; + using const_iterator = typename Container::const_iterator; + + std::expected, Error> get(const Key &key) const { + if (!Traits::is_valid(key)) { + return std::unexpected(Error{key}); + } + const NodeStatus *status = get_raw_unchecked(key); + if (status == nullptr) { + return std::nullopt; + } + return *status; + } + + std::expected add(const Key &key) { + if (!Traits::is_valid(key)) { + return std::unexpected(Error{key}); + } + + NodeStatus *status = get_raw_unchecked(key); + if (status != nullptr) { + switch (*status) { + case NodeStatus::Inner: + case NodeStatus::Leaf: + return false; + case NodeStatus::Virtual: + set_raw_unchecked(key, NodeStatus::Inner); + return true; + } + } + + const auto parent = Traits::parent(key); + if (!parent.has_value()) { + DEBUG_ASSERT(key == Traits::root()); + if (empty()) { + set_raw_unchecked(key, NodeStatus::Leaf); + return true; + } + UNREACHABLE(); + } + + if (!Traits::is_valid(parent.value())) { + return std::unexpected(Error{parent.value()}); + } + NodeStatus *parent_status = get_raw_unchecked(parent.value()); + if (parent_status == nullptr) { + const auto parent_result = add(parent.value()); + if (!parent_result.has_value()) { + return std::unexpected(parent_result.error()); + } + set_raw_unchecked(parent.value(), NodeStatus::Virtual); + set_raw_unchecked(key, NodeStatus::Leaf); + return true; + } + + switch (*parent_status) { + case NodeStatus::Leaf: + set_raw_unchecked(parent.value(), NodeStatus::Inner); + set_raw_unchecked(key, NodeStatus::Leaf); + break; + case NodeStatus::Inner: + case NodeStatus::Virtual: + set_raw_unchecked(key, NodeStatus::Leaf); + break; + } + return true; + } + + std::expected remove(const Key &key) { + if (!Traits::is_valid(key)) { + return std::unexpected(Error{key}); + } + + NodeStatus *status = get_raw_unchecked(key); + if (status == nullptr) { + return false; + } + switch (*status) { + case NodeStatus::Inner: + *status = NodeStatus::Virtual; + return true; + case NodeStatus::Virtual: + return false; + case NodeStatus::Leaf: + remove_raw_unchecked(key); + return update_parent_after_remove(key); + } + UNREACHABLE(); + } + + std::expected is_present(const Key &key) const { + if (!Traits::is_valid(key)) { + return std::unexpected(Error{key}); + } + return _index.contains(key); + } + std::expected is_absent(const Key &key) const { + const auto present = is_present(key); + if (!present.has_value()) { + return std::unexpected(present.error()); + } + return !present.value(); + } + std::expected is(const NodeStatus status, const Key &key) const { + const auto result = get(key); + if (!result.has_value()) { + return std::unexpected(result.error()); + } + return result.value() == status; + } + + std::expected get_raw(const Key &key) { + if (!Traits::is_valid(key)) { + return std::unexpected(Error{key}); + } + return get_raw_unchecked(key); + } + std::expected get_raw(const Key &key) const { + if (!Traits::is_valid(key)) { + return std::unexpected(Error{key}); + } + return get_raw_unchecked(key); + } + std::expected set_raw(const Key &key, const NodeStatus status) { + if (!Traits::is_valid(key)) { + return std::unexpected(Error{key}); + } + set_raw_unchecked(key, status); + return {}; + } + std::expected remove_raw(const Key &key) { + if (!Traits::is_valid(key)) { + return std::unexpected(Error{key}); + } + remove_raw_unchecked(key); + return {}; + } + + void clear() { + _index.clear(); + } + bool empty() const { + return _index.empty(); + } + size_t size() const { + return _index.size(); + } + + const_iterator begin() const { + return _index.begin(); + } + const_iterator end() const { + return _index.end(); + } + const_iterator cbegin() const { + return _index.cbegin(); + } + const_iterator cend() const { + return _index.cend(); + } + + using serialize = zpp::bits::members<1>; + friend zpp::bits::access; + +private: + NodeStatus *get_raw_unchecked(const Key &key) { + const auto iterator = _index.find(key); + return iterator == _index.end() ? nullptr : &iterator->second; + } + const NodeStatus *get_raw_unchecked(const Key &key) const { + const auto iterator = _index.find(key); + return iterator == _index.end() ? nullptr : &iterator->second; + } + void set_raw_unchecked(const Key &key, const NodeStatus status) { + _index[key] = status; + } + void remove_raw_unchecked(const Key &key) { + _index.erase(key); + } + + std::expected update_parent_after_remove(const Key &key) { + const auto parent = Traits::parent(key); + if (!parent.has_value()) { + return true; + } + if (!Traits::is_valid(parent.value())) { + return std::unexpected(Error{parent.value()}); + } + + NodeStatus *parent_status = get_raw_unchecked(parent.value()); + DEBUG_ASSERT(parent_status != nullptr); + const auto siblings = Traits::children(parent.value()); + DEBUG_ASSERT(siblings.has_value()); + for (const Key &sibling : siblings.value()) { + if (sibling != key && get_raw_unchecked(sibling) != nullptr) { + return true; + } + } + + switch (*parent_status) { + case NodeStatus::Inner: + *parent_status = NodeStatus::Leaf; + break; + case NodeStatus::Virtual: + remove_raw_unchecked(parent.value()); + return update_parent_after_remove(parent.value()); + case NodeStatus::Leaf: + UNREACHABLE(); + } + return true; + } + + Container _index; +}; + +} // namespace store diff --git a/src/terrainlib/store/InvalidKey.h b/src/terrainlib/store/InvalidKey.h new file mode 100644 index 00000000..d8caad27 --- /dev/null +++ b/src/terrainlib/store/InvalidKey.h @@ -0,0 +1,12 @@ +#pragma once + +namespace store { + +template +struct InvalidKey { + Key key; + + bool operator==(const InvalidKey &) const = default; +}; + +} // namespace store diff --git a/src/terrainlib/store/NodeStatus.h b/src/terrainlib/store/NodeStatus.h new file mode 100644 index 00000000..775cf246 --- /dev/null +++ b/src/terrainlib/store/NodeStatus.h @@ -0,0 +1,85 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include + +#include "log.h" + +namespace store { + +class NodeStatus { +public: + using Underlying = uint8_t; + enum Value : Underlying { + Leaf = 0, + Inner = 1, + Virtual = 2, + }; + + constexpr NodeStatus() = default; + constexpr NodeStatus(const Value value) : _value(value) {} + + constexpr operator Value() const { + return _value; + } + explicit operator bool() const = delete; + constexpr bool operator==(const NodeStatus other) const { + return _value == other._value; + } + constexpr bool operator!=(const NodeStatus other) const { + return !(*this == other); + } + constexpr bool operator==(const Value other) const { + return _value == other; + } + constexpr bool operator!=(const Value other) const { + return _value != other; + } + + std::string to_string() const; + +private: + Value _value; + +public: + using serialize = zpp::bits::members<1>; + friend zpp::bits::access; +}; + +} // namespace store + +template<> +struct fmt::formatter { + template + constexpr auto parse(ParseContext &context) { + return context.begin(); + } + + template + auto format(const store::NodeStatus &status, FormatContext &context) const { + switch (status) { + case store::NodeStatus::Leaf: + return fmt::format_to(context.out(), "Leaf"); + case store::NodeStatus::Inner: + return fmt::format_to(context.out(), "Inner"); + case store::NodeStatus::Virtual: + return fmt::format_to(context.out(), "Virtual"); + default: + UNREACHABLE(); + } + } +}; + +inline std::string store::NodeStatus::to_string() const { + return fmt::format("{}", *this); +} + +inline std::ostream &operator<<(std::ostream &stream, const store::NodeStatus &status) { + fmt::print(stream, "{}", status); + return stream; +} diff --git a/src/terrainlib/store/NodeStatusOrMissing.h b/src/terrainlib/store/NodeStatusOrMissing.h new file mode 100644 index 00000000..4157b57d --- /dev/null +++ b/src/terrainlib/store/NodeStatusOrMissing.h @@ -0,0 +1,134 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "log.h" +#include "store/NodeStatus.h" + +namespace store { + +class NodeStatusOrMissing { +public: + using Underlying = NodeStatus::Underlying; + enum Value : Underlying { + Leaf = 0, + Inner = 1, + Virtual = 2, + Missing = 3, + }; + + constexpr NodeStatusOrMissing() = default; + constexpr NodeStatusOrMissing(const Value value) : _value(value) {} + NodeStatusOrMissing(const NodeStatus status) : _value(from_status(status)) {} + NodeStatusOrMissing(const std::optional status) : _value(from_optional_status(status)) {} + + constexpr operator Value() const { + return _value; + } + constexpr operator std::optional() const { + return to_optional_status(); + } + explicit operator bool() const = delete; + constexpr bool operator==(const NodeStatusOrMissing other) const { + return _value == other._value; + } + constexpr bool operator!=(const NodeStatusOrMissing other) const { + return !(*this == other); + } + constexpr bool operator==(const Value other) const { + return _value == other; + } + constexpr bool operator!=(const Value other) const { + return _value != other; + } + + std::string to_string() const; + +private: + static constexpr NodeStatusOrMissing from_status(const NodeStatus status) noexcept { + return static_cast(static_cast(status)); + } + static constexpr NodeStatusOrMissing from_optional_status(const std::optional status) noexcept { + return status.has_value() + ? from_status(status.value()) + : NodeStatusOrMissing(NodeStatusOrMissing::Missing); + } + constexpr std::optional to_optional_status() const noexcept { + if (_value == Missing) { + return std::nullopt; + } + return static_cast(_value); + } + +public: + Value _value; +}; + +namespace detail { +template +consteval bool verify_node_status_values(std::index_sequence) { + constexpr auto status_values = magic_enum::enum_values(); + constexpr auto optional_values = magic_enum::enum_values(); + return ((magic_enum::enum_underlying(status_values[Indices]) + == magic_enum::enum_underlying(optional_values[Indices])) + && ...); +} + +consteval bool verify_node_status_enums() { + static_assert( + std::is_same_v< + magic_enum::underlying_type_t, + magic_enum::underlying_type_t>); + constexpr auto status_values = magic_enum::enum_values(); + constexpr auto optional_values = magic_enum::enum_values(); + static_assert(optional_values.size() == status_values.size() + 1); + static_assert(verify_node_status_values(std::make_index_sequence())); + return true; +} + +static_assert(verify_node_status_enums()); +} // namespace detail + +} // namespace store + +template<> +struct fmt::formatter { + template + constexpr auto parse(ParseContext &context) { + return context.begin(); + } + + template + auto format(const store::NodeStatusOrMissing &status, FormatContext &context) const { + switch (status) { + case store::NodeStatusOrMissing::Leaf: + return fmt::format_to(context.out(), "Leaf"); + case store::NodeStatusOrMissing::Inner: + return fmt::format_to(context.out(), "Inner"); + case store::NodeStatusOrMissing::Virtual: + return fmt::format_to(context.out(), "Virtual"); + case store::NodeStatusOrMissing::Missing: + return fmt::format_to(context.out(), "Missing"); + default: + UNREACHABLE(); + } + } +}; + +inline std::string store::NodeStatusOrMissing::to_string() const { + return fmt::format("{}", *this); +} + +inline std::ostream &operator<<(std::ostream &stream, const store::NodeStatusOrMissing &status) { + fmt::print(stream, "{}", status); + return stream; +} diff --git a/src/terrainlib/store/Traits.h b/src/terrainlib/store/Traits.h new file mode 100644 index 00000000..80fe88a7 --- /dev/null +++ b/src/terrainlib/store/Traits.h @@ -0,0 +1,17 @@ +#pragma once + +#include + +namespace store { + +template +concept HierarchyTraits = requires(typename Traits::Key key) { + typename Traits::Key; + typename Traits::Hasher; + { Traits::root() } -> std::same_as; + Traits::parent(key); + Traits::children(key); + { Traits::is_valid(key) } -> std::same_as; +}; + +} // namespace store diff --git a/src/terrainlib/store/traverse.h b/src/terrainlib/store/traverse.h new file mode 100644 index 00000000..8dbfe948 --- /dev/null +++ b/src/terrainlib/store/traverse.h @@ -0,0 +1,99 @@ +#pragma once + +#include +#include +#include + +#include "store/Index.h" + +namespace store { + +enum class TraversalOrder { + DepthFirst, + BreadthFirst, +}; + +struct AlwaysRefine { + template + constexpr bool operator()(const Key &) const { + return true; + } +}; + +template +std::expected> traverse( + const Index &index, + VisitFn &&visit, + RefineFn &&refine = {}, + const typename Traits::Key &root = Traits::root(), + const TraversalOrder order = TraversalOrder::DepthFirst) { + using Key = typename Traits::Key; + using Error = InvalidKey; + + if (!Traits::is_valid(root)) { + return std::unexpected(Error{root}); + } + const auto root_status = index.get(root); + if (!root_status.has_value()) { + return std::unexpected(root_status.error()); + } + if (!root_status.value().has_value()) { + return {}; + } + + if (order == TraversalOrder::DepthFirst) { + std::function(const Key &)> depth_first; + depth_first = [&](const Key ¤t) -> std::expected { + const auto status = index.get(current); + if (!status.has_value()) { + return std::unexpected(status.error()); + } + if (!status.value().has_value()) { + return {}; + } + visit(current, status.value().value()); + + if (refine(current)) { + const auto children = Traits::children(current); + if (children.has_value()) { + for (const Key &child : children.value()) { + const auto result = depth_first(child); + if (!result.has_value()) { + return result; + } + } + } + } + return {}; + }; + return depth_first(root); + } + + std::queue queue; + queue.push(root); + while (!queue.empty()) { + const Key current = queue.front(); + queue.pop(); + + const auto status = index.get(current); + if (!status.has_value()) { + return std::unexpected(status.error()); + } + if (!status.value().has_value()) { + continue; + } + visit(current, status.value().value()); + + if (refine(current)) { + const auto children = Traits::children(current); + if (children.has_value()) { + for (const Key &child : children.value()) { + queue.push(child); + } + } + } + } + return {}; +} + +} // namespace store diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 7d89ca6f..3ebf239c 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -31,6 +31,8 @@ add_executable(unittests_terrainlib terrainlib/segmented_buffer.cpp terrainlib/stamp_set.cpp terrainlib/store_compatibility.cpp + terrainlib/store_index.cpp + terrainlib/store_traverse.cpp terrainlib/index_map.cpp terrainlib/index_traverse.cpp terrainlib/mesh_boundary.cpp diff --git a/unittests/terrainlib/store_index.cpp b/unittests/terrainlib/store_index.cpp new file mode 100644 index 00000000..454d7948 --- /dev/null +++ b/unittests/terrainlib/store_index.cpp @@ -0,0 +1,81 @@ +#include +#include + +#include +#include + +#include "octree/StoreTraits.h" +#include "raster_store/StoreTraits.h" +#include "store/Index.h" + +static_assert(store::HierarchyTraits); +static_assert(store::HierarchyTraits); + +TEMPLATE_TEST_CASE( + "shared hierarchy index transitions", + "[store][index]", + octree::StoreTraits, + raster_store::StoreTraits) { + using Traits = TestType; + using Index = store::Index; + + Index index; + const auto root = Traits::root(); + const auto children = Traits::children(root).value(); + const auto first = children[0]; + const auto second = children[1]; + + REQUIRE(index.add(first).value()); + CHECK(index.is(store::NodeStatus::Virtual, root).value()); + CHECK(index.is(store::NodeStatus::Leaf, first).value()); + + REQUIRE(index.add(root).value()); + CHECK(index.is(store::NodeStatus::Inner, root).value()); + CHECK_FALSE(index.add(root).value()); + + REQUIRE(index.add(second).value()); + REQUIRE(index.remove(first).value()); + CHECK(index.is(store::NodeStatus::Inner, root).value()); + REQUIRE(index.remove(second).value()); + CHECK(index.is(store::NodeStatus::Leaf, root).value()); + REQUIRE(index.remove(root).value()); + CHECK(index.empty()); +} + +TEST_CASE("raster store traits validate tile boundaries", "[store][index][raster]") { + using Traits = raster_store::StoreTraits; + using Key = Traits::Key; + + CHECK(Traits::is_valid(Key{0, {0, 0}})); + CHECK_FALSE(Traits::is_valid(Key{0, {1, 0}})); + CHECK(Traits::is_valid(Key{31, {uint32_t{0x7fffffff}, uint32_t{0x7fffffff}}})); + CHECK_FALSE(Traits::is_valid(Key{31, {uint32_t{0x80000000}, 0}})); + CHECK(Traits::is_valid(Key{32, {0, 0}})); + CHECK(Traits::is_valid(Key{32, {UINT32_MAX, UINT32_MAX}})); + CHECK_FALSE(Traits::is_valid(Key{33, {0, 0}})); + + CHECK_FALSE(Traits::parent(Traits::root()).has_value()); + CHECK_FALSE(Traits::children(Key{32, {UINT32_MAX, UINT32_MAX}}).has_value()); + + const Key level_31{31, {uint32_t{0x7fffffff}, uint32_t{0x7fffffff}}}; + const auto children = Traits::children(level_31); + REQUIRE(children.has_value()); + CHECK(children->at(0) == Key{32, {UINT32_MAX - 1, UINT32_MAX - 1}}); + CHECK(children->at(1) == Key{32, {UINT32_MAX, UINT32_MAX - 1}}); + CHECK(children->at(2) == Key{32, {UINT32_MAX - 1, UINT32_MAX}}); + CHECK(children->at(3) == Key{32, {UINT32_MAX, UINT32_MAX}}); +} + +TEST_CASE("shared index rejects invalid keys", "[store][index][raster]") { + using Traits = raster_store::StoreTraits; + using Key = Traits::Key; + store::Index index; + const Key invalid{4, {16, 0}}; + + CHECK(index.get(invalid) == std::unexpected(store::InvalidKey{invalid})); + CHECK(index.add(invalid) == std::unexpected(store::InvalidKey{invalid})); + CHECK(index.remove(invalid) == std::unexpected(store::InvalidKey{invalid})); + CHECK(index.is_present(invalid) == std::unexpected(store::InvalidKey{invalid})); + CHECK(index.is_absent(invalid) == std::unexpected(store::InvalidKey{invalid})); + CHECK(index.is(store::NodeStatus::Leaf, invalid) == std::unexpected(store::InvalidKey{invalid})); +} diff --git a/unittests/terrainlib/store_traverse.cpp b/unittests/terrainlib/store_traverse.cpp new file mode 100644 index 00000000..43992ed1 --- /dev/null +++ b/unittests/terrainlib/store_traverse.cpp @@ -0,0 +1,68 @@ +#include + +#include +#include + +#include "octree/StoreTraits.h" +#include "raster_store/StoreTraits.h" +#include "store/traverse.h" + +TEMPLATE_TEST_CASE( + "shared hierarchy traversal preserves trait child order", + "[store][traverse]", + octree::StoreTraits, + raster_store::StoreTraits) { + using Traits = TestType; + using Key = typename Traits::Key; + + store::Index index; + const Key root = Traits::root(); + const auto children = Traits::children(root).value(); + const Key first = children[0]; + const Key second = children[1]; + const Key grandchild = Traits::children(first).value()[0]; + REQUIRE(index.add(root).has_value()); + REQUIRE(index.add(first).has_value()); + REQUIRE(index.add(second).has_value()); + REQUIRE(index.add(grandchild).has_value()); + + std::vector depth_first; + REQUIRE(store::traverse( + index, + [&](const Key &key, const store::NodeStatus) { depth_first.push_back(key); }) + .has_value()); + CHECK(depth_first == std::vector{root, first, grandchild, second}); + + std::vector breadth_first; + REQUIRE(store::traverse( + index, + [&](const Key &key, const store::NodeStatus) { breadth_first.push_back(key); }, + store::AlwaysRefine{}, + root, + store::TraversalOrder::BreadthFirst) + .has_value()); + CHECK(breadth_first == std::vector{root, first, second, grandchild}); + + std::vector explicit_root; + REQUIRE(store::traverse( + index, + [&](const Key &key, const store::NodeStatus) { explicit_root.push_back(key); }, + store::AlwaysRefine{}, + first) + .has_value()); + CHECK(explicit_root == std::vector{first, grandchild}); +} + +TEST_CASE("shared traversal rejects an invalid explicit root", "[store][traverse][raster]") { + using Traits = raster_store::StoreTraits; + using Key = Traits::Key; + const Key invalid{1, {2, 0}}; + store::Index index; + + const auto result = store::traverse( + index, + [](const Key &, const store::NodeStatus) {}, + store::AlwaysRefine{}, + invalid); + CHECK(result == std::unexpected(store::InvalidKey{invalid})); +} From fbc02d08543fdcfc3cca210e04acc80b9f961a60 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:37:15 +0200 Subject: [PATCH 54/61] Add runtime store layouts and codecs --- docs/raster-store/refactor-status.md | 31 ++- src/dag_builder/dag_id.h | 16 -- src/dag_builder/dag_node.h | 14 -- src/dag_builder/encoded.h | 173 +------------- src/dag_builder/metadata.h | 32 --- src/dag_builder/serialization.h | 212 ++++++++++++++++++ src/dag_builder/storage.h | 3 +- src/dag_builder/zpp_bits_glm.h | 20 -- src/terrainlib/mesh/codec/Gltf.h | 81 +++++++ src/terrainlib/mesh/codec/Terrain.h | 49 ++++ src/terrainlib/octree/store_layout/Flat.h | 39 ++++ .../LevelAndCoordinateDirectories.h | 49 ++++ src/terrainlib/octree/store_layout/Mappings.h | 38 ++++ src/terrainlib/store/Codec.h | 31 +++ src/terrainlib/store/CodecError.h | 38 ++++ src/terrainlib/store/Layout.h | 44 ++++ src/terrainlib/store/NodePath.h | 23 ++ src/terrainlib/store/PathMapping.h | 19 ++ src/terrainlib/store/codec/ZppBits.h | 57 +++++ unittests/CMakeLists.txt | 2 + .../dag_builder/storage_compatibility.cpp | 70 ++++++ unittests/terrainlib/store_codec.cpp | 202 +++++++++++++++++ unittests/terrainlib/store_layout.cpp | 70 ++++++ 23 files changed, 1061 insertions(+), 252 deletions(-) create mode 100644 src/dag_builder/serialization.h delete mode 100644 src/dag_builder/zpp_bits_glm.h create mode 100644 src/terrainlib/mesh/codec/Gltf.h create mode 100644 src/terrainlib/mesh/codec/Terrain.h create mode 100644 src/terrainlib/octree/store_layout/Flat.h create mode 100644 src/terrainlib/octree/store_layout/LevelAndCoordinateDirectories.h create mode 100644 src/terrainlib/octree/store_layout/Mappings.h create mode 100644 src/terrainlib/store/Codec.h create mode 100644 src/terrainlib/store/CodecError.h create mode 100644 src/terrainlib/store/Layout.h create mode 100644 src/terrainlib/store/NodePath.h create mode 100644 src/terrainlib/store/PathMapping.h create mode 100644 src/terrainlib/store/codec/ZppBits.h create mode 100644 unittests/terrainlib/store_codec.cpp create mode 100644 unittests/terrainlib/store_layout.cpp diff --git a/docs/raster-store/refactor-status.md b/docs/raster-store/refactor-status.md index 68cfdf06..6a2e1e83 100644 --- a/docs/raster-store/refactor-status.md +++ b/docs/raster-store/refactor-status.md @@ -16,10 +16,10 @@ This file is the resumable implementation log for ## Current position -- Phase: 1 — Extract topology into `store` +- Phase: 2 — Introduce path mappings and runtime codecs - State: in progress -- Next action: build the shared topology extraction and run focused topology - plus compatibility tests. +- Next action: run the complete Phase 2 verification, inspect the diff, + commit, and tag the phase boundary. ## Phase log @@ -54,6 +54,20 @@ This file is the resumable implementation log for invalid-key tests. - State: complete. +### Phase 2 — Introduce path mappings and runtime codecs + +- Added extensionless `NodePath`, `PathMapping`, and `Layout` values. +- Added the stateful runtime `Codec` interface and typed `CodecError`. +- Added runtime ZPP Bits, terrain, binary glTF, and JSON glTF codecs. +- Added ordinary `flat` and `level_and_coordinate_directories` mapping + function pairs plus explicit lookup. +- Consolidated DAG serialization in `dag_builder/serialization.h` and made + the legacy DAG storage adapter include it explicitly. +- Added mapping/codec composition, parser validation, single-/multi-file, + unsupported-operation, write-only, directory-creation, error-conversion, + and concurrent codec tests. +- State: complete. + ## Verification log - Baseline `unittests_terrainlib`: 23,676 assertions in 391 test cases passed. @@ -76,6 +90,15 @@ This file is the resumable implementation log for removed `radix::tile::Scheme` in `unittests/sf_builder/texture.cpp`; the affected refactor targets and suites build independently. - Phase 1 `git diff --check`: passed. +- Phase 2 focused shared-store tests: 160 assertions in 17 test cases passed. +- Phase 2 DAG golden fixture: 30 assertions passed, including runtime codec + output byte equality. +- Phase 2 concurrent DAG runtime codec: 10 assertions passed. +- Phase 2 `unittests_terrainlib`: 23,892 assertions in 414 test cases passed. +- Phase 2 `unittests_dagbuilder`: 440 assertions in 71 test cases passed. +- Phase 2 application builds passed for `sf-builder`, `dag-builder`, and + `dag-convert-debug`. +- Phase 2 `git diff --check`: passed. ## Commit and tag log @@ -86,3 +109,5 @@ This file is the resumable implementation log for `.terrain` golden payloads ignored by the repository's general artifact rule. - `refactor_phase_0` — annotated Phase 0 completion tag. +- `4c171c8` / `refactor_phase_1` — shared topology extraction and Phase 1 + completion tag. diff --git a/src/dag_builder/dag_id.h b/src/dag_builder/dag_id.h index 232f66fd..3370d1bf 100644 --- a/src/dag_builder/dag_id.h +++ b/src/dag_builder/dag_id.h @@ -2,8 +2,6 @@ #include -#include - #include "octree/Id.h" #include "hash_utils.h" @@ -33,17 +31,3 @@ struct fmt::formatter { return fmt::format_to(ctx.out(), "{}:{}", id.source_batch, id.cluster_index); } }; - -namespace zpp::bits { - -template -auto serialize(Archive &archive, const dag::Id &id) { - return archive(id.source_batch, id.cluster_index); -} - -template -auto serialize(Archive &archive, dag::Id &id) { - return archive(id.source_batch, id.cluster_index); -} - -} diff --git a/src/dag_builder/dag_node.h b/src/dag_builder/dag_node.h index c276a27a..2b46e6ec 100644 --- a/src/dag_builder/dag_node.h +++ b/src/dag_builder/dag_node.h @@ -18,17 +18,3 @@ inline ClusterBatch make_leaf_batch(Clustering clustering) { } } - -namespace dag { - -template -auto serialize(Archive &archive, const dag::ClusterBatch &node) { - return archive(node.metadata, node.clustering); -} - -template -auto serialize(Archive &archive, dag::ClusterBatch &node) { - return archive(node.metadata, node.clustering); -} - -} // namespace dag diff --git a/src/dag_builder/encoded.h b/src/dag_builder/encoded.h index 2ccee0e9..e9b3510a 100644 --- a/src/dag_builder/encoded.h +++ b/src/dag_builder/encoded.h @@ -1,178 +1,21 @@ #pragma once -#include #include -#include -#include -#include #include -#include -#include -#include -#include "TextureSet.h" -#include "cluster.h" -#include "io/bytes.h" +#include "io/Error.h" #include "io/serialize.h" -#include "mesh/io/texture.h" -#include "meshopt.h" +#include "serialization.h" -template -auto serialize(Archive &archive, const TextureSet &textures) { - size_t size = textures.size(); - auto result = archive(size); - using Result = std::remove_cvref_t; - if constexpr (!std::is_same_v && !std::is_same_v) { - return result; - } else { - if (zpp::bits::failure(result)) { - return result; - } - - for (const auto &texture : textures) { - const mesh::io::ImageAndExt item{texture, ".jpeg"}; - result = archive(item); - if (zpp::bits::failure(result)) { - return result; - } - } - - return result; - } -} -template -auto serialize(Archive &archive, TextureSet &textures) { - size_t size; - auto result = archive(size); - using Result = std::remove_cvref_t; - if constexpr (!std::is_same_v && !std::is_same_v) { - return result; - } else { - if (zpp::bits::failure(result)) { - return result; - } - - std::vector images; - images.reserve(size); - for (size_t i = 0; i < size; i++) { - mesh::io::ImageAndExt item; - result = archive(item); - if (zpp::bits::failure(result)) { - return result; - } - - images.push_back(item.image); - } - textures = TextureSet(images); - - return result; - } -} - -template -auto serialize(Archive &archive, const Cluster &cluster) { - const size_t triangle_count = cluster.triangle_count(); - const size_t vertex_count = cluster.vertex_count(); - const size_t uv_count = cluster.uvs.size(); - - auto encoded_triangles = meshopt::encode_index_buffer(cluster.local_triangles); - auto encoded_vertex_indices = meshopt::encode_vertex_buffer(cluster.vertex_indices); - auto encoded_uvs = meshopt::encode_vertex_buffer(cluster.uvs); - - return archive( - triangle_count, - vertex_count, - uv_count, - encoded_triangles, - encoded_vertex_indices, - encoded_uvs, - cluster.id, - cluster.texture_id, - cluster.absolute_error); -} -template -auto serialize(Archive &archive, Cluster &cluster) { - size_t triangle_count; - size_t vertex_count; - size_t uv_count; - - std::vector encoded_triangles; - std::vector encoded_vertex_indices; - std::vector encoded_uvs; - - auto result = archive( - triangle_count, - vertex_count, - uv_count, - encoded_triangles, - encoded_vertex_indices, - encoded_uvs, - cluster.id, - cluster.texture_id, - cluster.absolute_error); - - using Result = std::remove_cvref_t; - if constexpr (!std::is_same_v && !std::is_same_v) { - return result; - } else { - if (zpp::bits::failure(result)) { - return result; - } - - meshopt::decode_index_buffer(cluster.local_triangles, triangle_count, encoded_triangles); - meshopt::decode_vertex_buffer(cluster.vertex_indices, vertex_count, encoded_vertex_indices); - meshopt::decode_vertex_buffer(cluster.uvs, uv_count, encoded_uvs); - - return result; - } -} - -template -auto serialize(Archive &archive, const Clustering &clustering) { - const size_t vertex_count = clustering.vertex_count(); - auto encoded_positions = meshopt::encode_vertex_buffer(clustering.positions); - - return archive( - vertex_count, - encoded_positions, - clustering.clusters, - clustering.textures); -} - -template -auto serialize(Archive &archive, Clustering &clustering) { - size_t vertex_count; - std::vector encoded_positions; - - auto result = archive( - vertex_count, - encoded_positions, - clustering.clusters, - clustering.textures); - - using Result = std::remove_cvref_t; - if constexpr (!std::is_same_v && !std::is_same_v) { - return result; - } else { - if (zpp::bits::failure(result)) { - return result; - } - - meshopt::decode_vertex_buffer(clustering.positions, vertex_count, encoded_positions); - - return result; - } -} - -inline std::expected -save_clustering(const Clustering &clustering, - const std::filesystem::path &path, - const bool make_dirs = true) { +inline std::expected save_clustering( + const Clustering &clustering, + const std::filesystem::path &path, + const bool make_dirs = true) { return ::io::write_to_path(clustering, path, make_dirs); } -inline std::expected -load_clustering(const std::filesystem::path &path) { +inline std::expected load_clustering( + const std::filesystem::path &path) { return ::io::read_from_path(path); } diff --git a/src/dag_builder/metadata.h b/src/dag_builder/metadata.h index 5527fe3a..6c12b2bf 100644 --- a/src/dag_builder/metadata.h +++ b/src/dag_builder/metadata.h @@ -8,7 +8,6 @@ #include "cluster.h" #include "dag_id.h" #include "utils.h" -#include "zpp_bits_glm.h" #include "octree/storage/defaults.h" #include "octree/storage/codec/ZppBitsCodec.h" #include "octree/storage/codec/ReadOnlyCodec.h" @@ -80,34 +79,3 @@ struct DefaultCodec { using type = ReadOnlyCodec>; }; } - -namespace zpp::bits { - -template -auto serialize(Archive &archive, const radix::geometry::Aabb3d &bounds) { - return archive(bounds.min, bounds.max); -} -template -auto serialize(Archive &archive, radix::geometry::Aabb3d &bounds) { - return archive(bounds.min, bounds.max); -} - -template -auto serialize(Archive &archive, const dag::Group &group) { - return archive(group.children, group.error, group.bounds); -} -template -auto serialize(Archive &archive, dag::Group &group) { - return archive(group.children, group.error, group.bounds); -} - -template -auto serialize(Archive &archive, const dag::NodeMetadata &metadata) { - return archive(metadata.group_assignment, metadata.groups); -} -template -auto serialize(Archive &archive, dag::NodeMetadata &metadata) { - return archive(metadata.group_assignment, metadata.groups); -} - -} // namespace zpp::bits diff --git a/src/dag_builder/serialization.h b/src/dag_builder/serialization.h new file mode 100644 index 00000000..b5c58481 --- /dev/null +++ b/src/dag_builder/serialization.h @@ -0,0 +1,212 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include + +#include "dag_node.h" +#include "glm_utils.h" +#include "io/bytes.h" +#include "mesh/io/texture.h" +#include "meshopt.h" + +namespace zpp::bits { + +template +auto serialize(Archive &archive, const glm::vec &vector) { + return archive(as_span(vector)); +} + +template +auto serialize(Archive &archive, glm::vec &vector) { + return archive(as_span(vector)); +} + +template +auto serialize(Archive &archive, const dag::Id &id) { + return archive(id.source_batch, id.cluster_index); +} + +template +auto serialize(Archive &archive, dag::Id &id) { + return archive(id.source_batch, id.cluster_index); +} + +template +auto serialize(Archive &archive, const radix::geometry::Aabb3d &bounds) { + return archive(bounds.min, bounds.max); +} + +template +auto serialize(Archive &archive, radix::geometry::Aabb3d &bounds) { + return archive(bounds.min, bounds.max); +} + +template +auto serialize(Archive &archive, const dag::Group &group) { + return archive(group.children, group.error, group.bounds); +} + +template +auto serialize(Archive &archive, dag::Group &group) { + return archive(group.children, group.error, group.bounds); +} + +template +auto serialize(Archive &archive, const dag::NodeMetadata &metadata) { + return archive(metadata.group_assignment, metadata.groups); +} + +template +auto serialize(Archive &archive, dag::NodeMetadata &metadata) { + return archive(metadata.group_assignment, metadata.groups); +} + +template +auto serialize(Archive &archive, const dag::ClusterBatch &batch) { + return archive(batch.metadata, batch.clustering); +} + +template +auto serialize(Archive &archive, dag::ClusterBatch &batch) { + return archive(batch.metadata, batch.clustering); +} + +} // namespace zpp::bits + +template +auto serialize(Archive &archive, const TextureSet &textures) { + size_t size = textures.size(); + auto result = archive(size); + using Result = std::remove_cvref_t; + if constexpr (!std::is_same_v + && !std::is_same_v) { + return result; + } else { + if (zpp::bits::failure(result)) { + return result; + } + for (const auto &texture : textures) { + const mesh::io::ImageAndExt item{texture, ".jpeg"}; + result = archive(item); + if (zpp::bits::failure(result)) { + return result; + } + } + return result; + } +} + +template +auto serialize(Archive &archive, TextureSet &textures) { + size_t size; + auto result = archive(size); + using Result = std::remove_cvref_t; + if constexpr (!std::is_same_v + && !std::is_same_v) { + return result; + } else { + if (zpp::bits::failure(result)) { + return result; + } + std::vector images; + images.reserve(size); + for (size_t index = 0; index < size; ++index) { + mesh::io::ImageAndExt item; + result = archive(item); + if (zpp::bits::failure(result)) { + return result; + } + images.push_back(item.image); + } + textures = TextureSet(images); + return result; + } +} + +template +auto serialize(Archive &archive, const Cluster &cluster) { + const size_t triangle_count = cluster.triangle_count(); + const size_t vertex_count = cluster.vertex_count(); + const size_t uv_count = cluster.uvs.size(); + auto encoded_triangles = meshopt::encode_index_buffer(cluster.local_triangles); + auto encoded_vertex_indices = meshopt::encode_vertex_buffer(cluster.vertex_indices); + auto encoded_uvs = meshopt::encode_vertex_buffer(cluster.uvs); + return archive( + triangle_count, + vertex_count, + uv_count, + encoded_triangles, + encoded_vertex_indices, + encoded_uvs, + cluster.id, + cluster.texture_id, + cluster.absolute_error); +} + +template +auto serialize(Archive &archive, Cluster &cluster) { + size_t triangle_count; + size_t vertex_count; + size_t uv_count; + std::vector encoded_triangles; + std::vector encoded_vertex_indices; + std::vector encoded_uvs; + auto result = archive( + triangle_count, + vertex_count, + uv_count, + encoded_triangles, + encoded_vertex_indices, + encoded_uvs, + cluster.id, + cluster.texture_id, + cluster.absolute_error); + using Result = std::remove_cvref_t; + if constexpr (!std::is_same_v + && !std::is_same_v) { + return result; + } else { + if (zpp::bits::failure(result)) { + return result; + } + meshopt::decode_index_buffer(cluster.local_triangles, triangle_count, encoded_triangles); + meshopt::decode_vertex_buffer(cluster.vertex_indices, vertex_count, encoded_vertex_indices); + meshopt::decode_vertex_buffer(cluster.uvs, uv_count, encoded_uvs); + return result; + } +} + +template +auto serialize(Archive &archive, const Clustering &clustering) { + const size_t vertex_count = clustering.vertex_count(); + auto encoded_positions = meshopt::encode_vertex_buffer(clustering.positions); + return archive(vertex_count, encoded_positions, clustering.clusters, clustering.textures); +} + +template +auto serialize(Archive &archive, Clustering &clustering) { + size_t vertex_count; + std::vector encoded_positions; + auto result = archive( + vertex_count, + encoded_positions, + clustering.clusters, + clustering.textures); + using Result = std::remove_cvref_t; + if constexpr (!std::is_same_v + && !std::is_same_v) { + return result; + } else { + if (zpp::bits::failure(result)) { + return result; + } + meshopt::decode_vertex_buffer(clustering.positions, vertex_count, encoded_positions); + return result; + } +} diff --git a/src/dag_builder/storage.h b/src/dag_builder/storage.h index 8c520529..7fb24242 100644 --- a/src/dag_builder/storage.h +++ b/src/dag_builder/storage.h @@ -1,7 +1,6 @@ #pragma once -#include "dag_node.h" -#include "metadata.h" +#include "serialization.h" #include "octree/storage/Storage.h" #include "octree/storage/IndexedStorage.h" namespace octree { diff --git a/src/dag_builder/zpp_bits_glm.h b/src/dag_builder/zpp_bits_glm.h deleted file mode 100644 index 0dade30d..00000000 --- a/src/dag_builder/zpp_bits_glm.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include -#include - -#include "glm_utils.h" - -namespace zpp::bits { - -template -auto serialize(Archive &archive, const glm::vec &v) { - return archive(as_span(v)); -} - -template -auto serialize(Archive &archive, glm::vec &v) { - return archive(as_span(v)); -} - -} diff --git a/src/terrainlib/mesh/codec/Gltf.h b/src/terrainlib/mesh/codec/Gltf.h new file mode 100644 index 00000000..9db26fe9 --- /dev/null +++ b/src/terrainlib/mesh/codec/Gltf.h @@ -0,0 +1,81 @@ +#pragma once + +#include +#include +#include +#include + +#include "mesh/SimpleMesh.h" +#include "mesh/io/gltf.h" +#include "store/Codec.h" +#include "store/codec/ZppBits.h" + +namespace mesh::codec { + +enum class GltfContainer { + Binary, + Json, +}; + +class Gltf final : public store::Codec { +public: + explicit Gltf(const GltfContainer container) : _container(container) {} + + std::vector paths(const store::NodePath &node_path) const override { + return {store::codec::append_extension( + node_path, + _container == GltfContainer::Binary ? ".glb" : ".gltf")}; + } + + std::expected read( + const store::NodePath &node_path) const override { + const auto result = mesh::io::gltf::load_from_path(paths(node_path).front()); + if (!result.has_value()) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + result.error() == mesh::io::LoadMeshErrorKind::FileNotFound + ? store::CodecErrorCategory::FileNotFound + : store::CodecErrorCategory::InvalidData, + result.error().description(), + }); + } + return result.value(); + } + + std::expected write( + const store::NodePath &node_path, + const mesh::Simple &mesh) const override { + try { + const auto result = mesh::io::gltf::save_to_path(mesh, paths(node_path).front()); + if (!result.has_value()) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Domain, + result.error().description(), + }); + } + return {}; + } catch (const std::exception &error) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Domain, + error.what(), + }); + } catch (...) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Domain, + "unknown glTF writer exception", + }); + } + } + + GltfContainer container() const { + return _container; + } + +private: + GltfContainer _container; +}; + +} // namespace mesh::codec diff --git a/src/terrainlib/mesh/codec/Terrain.h b/src/terrainlib/mesh/codec/Terrain.h new file mode 100644 index 00000000..d1356e76 --- /dev/null +++ b/src/terrainlib/mesh/codec/Terrain.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include + +#include "mesh/SimpleMesh.h" +#include "mesh/io/terrain.h" +#include "store/Codec.h" +#include "store/codec/ZppBits.h" + +namespace mesh::codec { + +class Terrain final : public store::Codec { +public: + std::vector paths(const store::NodePath &node_path) const override { + return {store::codec::append_extension(node_path, ".terrain")}; + } + + std::expected read( + const store::NodePath &node_path) const override { + const auto result = mesh::io::terrain::load_from_path(paths(node_path).front()); + if (!result.has_value()) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + result.error() == mesh::io::LoadMeshErrorKind::FileNotFound + ? store::CodecErrorCategory::FileNotFound + : store::CodecErrorCategory::InvalidData, + result.error().description(), + }); + } + return result.value(); + } + + std::expected write( + const store::NodePath &node_path, + const mesh::Simple &mesh) const override { + const auto result = mesh::io::terrain::save_to_path(mesh, paths(node_path).front()); + if (!result.has_value()) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Domain, + result.error().description(), + }); + } + return {}; + } +}; + +} // namespace mesh::codec diff --git a/src/terrainlib/octree/store_layout/Flat.h b/src/terrainlib/octree/store_layout/Flat.h new file mode 100644 index 00000000..c7f58a7f --- /dev/null +++ b/src/terrainlib/octree/store_layout/Flat.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include +#include +#include + +#include + +#include "octree/Id.h" +#include "store/NodePath.h" +#include "string_utils.h" + +namespace octree::store_layout { + +inline store::NodePath flat_key_to_node_path(const Id &id) { + return store::NodePath(fmt::format("{}-{}", id.level(), id.index_on_level())); +} + +inline std::optional flat_node_path_to_key(const store::NodePath &node_path) { + const std::filesystem::path &path = node_path.path(); + if (path.empty() || path.is_absolute() || path.has_parent_path() || path.has_extension()) { + return std::nullopt; + } + + const std::string text = path.string(); + const size_t separator = text.find('-'); + if (separator == std::string::npos || separator != text.rfind('-')) { + return std::nullopt; + } + const auto level = from_chars(std::string_view(text).substr(0, separator)); + const auto index = from_chars(std::string_view(text).substr(separator + 1)); + if (!level.has_value() || !index.has_value()) { + return std::nullopt; + } + return Id::try_make(level.value(), index.value()); +} + +} // namespace octree::store_layout diff --git a/src/terrainlib/octree/store_layout/LevelAndCoordinateDirectories.h b/src/terrainlib/octree/store_layout/LevelAndCoordinateDirectories.h new file mode 100644 index 00000000..7036443a --- /dev/null +++ b/src/terrainlib/octree/store_layout/LevelAndCoordinateDirectories.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include +#include + +#include + +#include "octree/Id.h" +#include "store/NodePath.h" +#include "string_utils.h" + +namespace octree::store_layout { + +inline store::NodePath level_and_coordinate_key_to_node_path(const Id &id) { + const Id::Coords coordinates = id.coords(); + return store::NodePath(fmt::format( + "{}/{}/{}/{}", + id.level(), + coordinates.x, + coordinates.y, + coordinates.z)); +} + +inline std::optional level_and_coordinate_node_path_to_key( + const store::NodePath &node_path) { + const std::filesystem::path &path = node_path.path(); + if (path.empty() || path.is_absolute() || std::distance(path.begin(), path.end()) != 4) { + return std::nullopt; + } + + auto part = path.begin(); + const auto level = from_chars(part->string()); + ++part; + const auto x = from_chars(part->string()); + ++part; + const auto y = from_chars(part->string()); + ++part; + if (part->has_extension()) { + return std::nullopt; + } + const auto z = from_chars(part->string()); + if (!level.has_value() || !x.has_value() || !y.has_value() || !z.has_value()) { + return std::nullopt; + } + return Id::try_make(level.value(), {x.value(), y.value(), z.value()}); +} + +} // namespace octree::store_layout diff --git a/src/terrainlib/octree/store_layout/Mappings.h b/src/terrainlib/octree/store_layout/Mappings.h new file mode 100644 index 00000000..1b46bed8 --- /dev/null +++ b/src/terrainlib/octree/store_layout/Mappings.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include +#include + +#include "octree/store_layout/Flat.h" +#include "octree/store_layout/LevelAndCoordinateDirectories.h" +#include "store/PathMapping.h" + +namespace octree::store_layout { + +inline store::PathMapping flat() { + return {"flat", flat_key_to_node_path, flat_node_path_to_key}; +} + +inline store::PathMapping level_and_coordinate_directories() { + return { + "level_and_coordinate_directories", + level_and_coordinate_key_to_node_path, + level_and_coordinate_node_path_to_key, + }; +} + +inline std::array, 2> all() { + return {flat(), level_and_coordinate_directories()}; +} + +inline std::optional> from_id(const std::string_view id) { + for (const auto mapping : all()) { + if (mapping.id == id) { + return mapping; + } + } + return std::nullopt; +} + +} // namespace octree::store_layout diff --git a/src/terrainlib/store/Codec.h b/src/terrainlib/store/Codec.h new file mode 100644 index 00000000..c9611589 --- /dev/null +++ b/src/terrainlib/store/Codec.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include + +#include + +#include "store/CodecError.h" +#include "store/NodePath.h" + +namespace store { + +template +class Codec { +public: + virtual ~Codec() = default; + + virtual std::vector paths(const NodePath &node_path) const = 0; + + virtual std::expected read(const NodePath &) const { + return std::unexpected( + CodecError::unsupported_operation(CodecOperation::Read, "read")); + } + + virtual std::expected write(const NodePath &, const NodeData &) const { + return std::unexpected( + CodecError::unsupported_operation(CodecOperation::Write, "write")); + } +}; + +} // namespace store diff --git a/src/terrainlib/store/CodecError.h b/src/terrainlib/store/CodecError.h new file mode 100644 index 00000000..ba4adeb8 --- /dev/null +++ b/src/terrainlib/store/CodecError.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include +#include + +namespace store { + +enum class CodecOperation { + Read, + Write, + Resolve, +}; + +enum class CodecErrorCategory { + UnsupportedOperation, + UnsupportedCodec, + FileNotFound, + InvalidData, + Io, + Domain, +}; + +struct CodecError { + CodecOperation operation; + CodecErrorCategory category; + std::string message; + + static CodecError unsupported_operation( + const CodecOperation operation, + const std::string_view name) { + return {operation, CodecErrorCategory::UnsupportedOperation, std::string(name)}; + } + + bool operator==(const CodecError &) const = default; +}; + +} // namespace store diff --git a/src/terrainlib/store/Layout.h b/src/terrainlib/store/Layout.h new file mode 100644 index 00000000..b987c535 --- /dev/null +++ b/src/terrainlib/store/Layout.h @@ -0,0 +1,44 @@ +#pragma once + +#include +#include +#include + +#include "store/NodePath.h" +#include "store/PathMapping.h" + +namespace store { + +template +class Layout { +public: + Layout(std::filesystem::path base_path, PathMapping mapping) + : _base_path(std::move(base_path)), _mapping(mapping) {} + + NodePath node_path(const Key &key) const { + return NodePath(_base_path / _mapping.key_to_node_path(key).path()); + } + + std::optional key_from_node_path(const NodePath &node_path) const { + std::error_code error; + const std::filesystem::path relative = + std::filesystem::relative(node_path.path(), _base_path, error); + if (error || relative.empty() || relative.is_absolute()) { + return std::nullopt; + } + return _mapping.node_path_to_key(NodePath(relative)); + } + + const std::filesystem::path &base_path() const { + return _base_path; + } + PathMapping mapping() const { + return _mapping; + } + +private: + std::filesystem::path _base_path; + PathMapping _mapping; +}; + +} // namespace store diff --git a/src/terrainlib/store/NodePath.h b/src/terrainlib/store/NodePath.h new file mode 100644 index 00000000..31673862 --- /dev/null +++ b/src/terrainlib/store/NodePath.h @@ -0,0 +1,23 @@ +#pragma once + +#include +#include + +namespace store { + +class NodePath { +public: + NodePath() = default; + explicit NodePath(std::filesystem::path path) : _path(std::move(path)) {} + + const std::filesystem::path &path() const { + return _path; + } + + bool operator==(const NodePath &) const = default; + +private: + std::filesystem::path _path; +}; + +} // namespace store diff --git a/src/terrainlib/store/PathMapping.h b/src/terrainlib/store/PathMapping.h new file mode 100644 index 00000000..0a2fe725 --- /dev/null +++ b/src/terrainlib/store/PathMapping.h @@ -0,0 +1,19 @@ +#pragma once + +#include +#include + +#include "store/NodePath.h" + +namespace store { + +template +struct PathMapping { + std::string_view id; + NodePath (*key_to_node_path)(const Key &); + std::optional (*node_path_to_key)(const NodePath &); + + bool operator==(const PathMapping &) const = default; +}; + +} // namespace store diff --git a/src/terrainlib/store/codec/ZppBits.h b/src/terrainlib/store/codec/ZppBits.h new file mode 100644 index 00000000..1a37153f --- /dev/null +++ b/src/terrainlib/store/codec/ZppBits.h @@ -0,0 +1,57 @@ +#pragma once + +#include +#include + +#include + +#include "io/serialize.h" +#include "store/Codec.h" + +namespace store::codec { + +inline std::filesystem::path append_extension( + const NodePath &node_path, + const std::string_view extension) { + std::filesystem::path result = node_path.path(); + result += extension; + return result; +} + +template +class ZppBits final : public Codec { +public: + std::vector paths(const NodePath &node_path) const override { + return {append_extension(node_path, ".bin")}; + } + + std::expected read(const NodePath &node_path) const override { + const auto result = io::read_from_path(paths(node_path).front()); + if (!result.has_value()) { + return std::unexpected(CodecError{ + CodecOperation::Read, + result.error() == io::Error(io::Error::OpenFile) + ? CodecErrorCategory::FileNotFound + : CodecErrorCategory::Io, + fmt::format("{}", result.error()), + }); + } + return result.value(); + } + + std::expected write( + const NodePath &node_path, + const NodeData &data) const override { + const auto result = io::write_to_path(data, paths(node_path).front()); + if (!result.has_value()) { + return std::unexpected(CodecError{ + CodecOperation::Write, + CodecErrorCategory::Io, + fmt::format("{}", result.error()), + }); + } + return {}; + } +}; + +} // namespace store::codec diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 3ebf239c..72f787cf 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -31,7 +31,9 @@ add_executable(unittests_terrainlib terrainlib/segmented_buffer.cpp terrainlib/stamp_set.cpp terrainlib/store_compatibility.cpp + terrainlib/store_codec.cpp terrainlib/store_index.cpp + terrainlib/store_layout.cpp terrainlib/store_traverse.cpp terrainlib/index_map.cpp terrainlib/index_traverse.cpp diff --git a/unittests/dag_builder/storage_compatibility.cpp b/unittests/dag_builder/storage_compatibility.cpp index e68750bf..1b53b9fa 100644 --- a/unittests/dag_builder/storage_compatibility.cpp +++ b/unittests/dag_builder/storage_compatibility.cpp @@ -1,4 +1,7 @@ +#include +#include #include +#include #include @@ -9,6 +12,33 @@ #include "octree/storage/open.h" #include "octree/traverse.h" #include "storage.h" +#include "store/codec/ZppBits.h" + +namespace { + +class TemporaryDirectory { +public: + TemporaryDirectory() { + static std::atomic_uint64_t counter = 0; + const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); + _path = std::filesystem::temp_directory_path() + / ("atb-dag-codec-" + std::to_string(timestamp) + "-" + + std::to_string(counter++)); + REQUIRE(std::filesystem::create_directories(_path)); + } + ~TemporaryDirectory() { + std::error_code error; + std::filesystem::remove_all(_path, error); + } + const std::filesystem::path &path() const { + return _path; + } + +private: + std::filesystem::path _path; +}; + +} // namespace TEST_CASE("pre-refactor DAG fixture preserves index and payload contracts") { const std::filesystem::path path = @@ -55,4 +85,44 @@ TEST_CASE("pre-refactor DAG fixture preserves index and payload contracts") { REQUIRE(encoded.has_value()); REQUIRE(fixture_bytes.has_value()); CHECK(encoded.value() == fixture_bytes.value()); + + store::codec::ZppBits runtime_codec; + const auto runtime_batch = runtime_codec.read(store::NodePath(path / "0/0/0/0")); + REQUIRE(runtime_batch.has_value()); + CHECK(runtime_batch->metadata.group_assignment == batch->metadata.group_assignment); + CHECK(runtime_codec.paths(store::NodePath(path / "0/0/0/0")) + == std::vector{path / "0/0/0/0.bin"}); + + TemporaryDirectory output; + const store::NodePath output_path(output.path() / "nested/node"); + REQUIRE(runtime_codec.write(output_path, batch.value()).has_value()); + const auto runtime_bytes = io::read_bytes_from_path(output.path() / "nested/node.bin"); + REQUIRE(runtime_bytes.has_value()); + CHECK(runtime_bytes.value() == fixture_bytes.value()); +} + +TEST_CASE("runtime DAG ZPP Bits codec is reentrant") { + const std::filesystem::path fixture = + std::filesystem::path(ALP_TEST_DATA_DIR) + / "raster-store-refactor/dag/0/0/0/0.bin"; + const auto batch = io::read_from_path(fixture); + REQUIRE(batch.has_value()); + + TemporaryDirectory output; + store::codec::ZppBits codec; + std::vector> operations; + for (int index = 0; index < 8; ++index) { + operations.push_back(std::async(std::launch::async, [&codec, &batch, &output, index] { + const store::NodePath path(output.path() / std::to_string(index) / "node"); + if (!codec.write(path, batch.value()).has_value()) { + return false; + } + const auto loaded = codec.read(path); + return loaded.has_value() + && loaded->metadata.group_assignment == batch->metadata.group_assignment; + })); + } + for (auto &operation : operations) { + REQUIRE(operation.get()); + } } diff --git a/unittests/terrainlib/store_codec.cpp b/unittests/terrainlib/store_codec.cpp new file mode 100644 index 00000000..5b4c2fca --- /dev/null +++ b/unittests/terrainlib/store_codec.cpp @@ -0,0 +1,202 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include "io/bytes.h" +#include "mesh/codec/Gltf.h" +#include "mesh/codec/Terrain.h" +#include "store/Codec.h" +#include "store/codec/ZppBits.h" + +namespace { + +class TemporaryDirectory { +public: + explicit TemporaryDirectory(const std::string_view label) { + static std::atomic_uint64_t counter = 0; + const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); + _path = std::filesystem::temp_directory_path() + / ("atb-store-codec-" + std::string(label) + "-" + + std::to_string(timestamp) + "-" + std::to_string(counter++)); + REQUIRE(std::filesystem::create_directories(_path)); + } + ~TemporaryDirectory() { + std::error_code error; + std::filesystem::remove_all(_path, error); + } + const std::filesystem::path &path() const { + return _path; + } + +private: + std::filesystem::path _path; +}; + +class MultiFileCodec final : public store::Codec { +public: + std::vector paths(const store::NodePath &node_path) const override { + std::filesystem::path data = node_path.path(); + std::filesystem::path metadata = node_path.path(); + data += ".data"; + metadata += ".metadata"; + return {data, metadata}; + } +}; + +class WriteOnlyCodec final : public store::Codec { +public: + std::vector paths(const store::NodePath &node_path) const override { + std::filesystem::path path = node_path.path(); + path += ".out"; + return {path}; + } + std::expected write( + const store::NodePath &, + const int &) const override { + ++writes; + return {}; + } + + mutable std::atomic_uint32_t writes = 0; +}; + +mesh::Simple triangle_mesh(const double offset) { + return mesh::Simple( + {{0, 1, 2}}, + {{offset, 0.0, 0.0}, {offset + 1.0, 0.0, 0.0}, {offset, 1.0, 0.0}}); +} + +template +void check_mesh_codec_reentrancy(const Codec &codec, const std::filesystem::path &base) { + std::vector> writes; + for (int index = 0; index < 4; ++index) { + writes.push_back(std::async(std::launch::async, [&codec, base, index] { + return codec.write( + store::NodePath(base / std::to_string(index) / "node"), + triangle_mesh(index)) + .has_value(); + })); + } + for (auto &write : writes) { + REQUIRE(write.get()); + } + + std::vector> reads; + for (int index = 0; index < 4; ++index) { + reads.push_back(std::async(std::launch::async, [&codec, base, index] { + const auto result = codec.read( + store::NodePath(base / std::to_string(index) / "node")); + return result.has_value() && result->face_count() == 1; + })); + } + for (auto &read : reads) { + REQUIRE(read.get()); + } +} + +} // namespace + +TEST_CASE("runtime codec defaults report unsupported operations", "[store][codec]") { + MultiFileCodec codec; + const store::NodePath path("12/34/56/78"); + CHECK(codec.paths(path) + == std::vector{ + "12/34/56/78.data", + "12/34/56/78.metadata", + }); + + const auto read = codec.read(path); + REQUIRE_FALSE(read.has_value()); + CHECK(read.error().operation == store::CodecOperation::Read); + CHECK(read.error().category == store::CodecErrorCategory::UnsupportedOperation); + + const auto write = codec.write(path, 42); + REQUIRE_FALSE(write.has_value()); + CHECK(write.error().operation == store::CodecOperation::Write); + CHECK(write.error().category == store::CodecErrorCategory::UnsupportedOperation); +} + +TEST_CASE("runtime write-only codec remains explicitly unreadable", "[store][codec]") { + WriteOnlyCodec codec; + REQUIRE(codec.write(store::NodePath("node"), 42).has_value()); + CHECK(codec.writes == 1); + const auto read = codec.read(store::NodePath("node")); + REQUIRE_FALSE(read.has_value()); + CHECK(read.error().category == store::CodecErrorCategory::UnsupportedOperation); +} + +TEST_CASE("ZPP Bits runtime codec creates directories and converts errors", "[store][codec]") { + TemporaryDirectory directory("zpp"); + const store::NodePath path(directory.path() / "nested/deeper/node"); + store::codec::ZppBits codec; + + REQUIRE(codec.write(path, 12345).has_value()); + CHECK(codec.paths(path) == std::vector{directory.path() / "nested/deeper/node.bin"}); + REQUIRE(std::filesystem::is_regular_file(codec.paths(path).front())); + const auto value = codec.read(path); + REQUIRE(value.has_value()); + CHECK(value.value() == 12345); + + REQUIRE(io::write_bytes_to_path( + std::vector{0xff}, + directory.path() / "invalid.bin") + .has_value()); + const auto invalid = codec.read(store::NodePath(directory.path() / "invalid")); + REQUIRE_FALSE(invalid.has_value()); + CHECK(invalid.error().operation == store::CodecOperation::Read); + CHECK(invalid.error().category == store::CodecErrorCategory::Io); +} + +TEST_CASE("ZPP Bits runtime codec is reentrant", "[store][codec]") { + TemporaryDirectory directory("zpp-reentrant"); + store::codec::ZppBits codec; + std::vector> operations; + for (int index = 0; index < 16; ++index) { + operations.push_back(std::async(std::launch::async, [&codec, &directory, index] { + const store::NodePath path(directory.path() / std::to_string(index) / "node"); + if (!codec.write(path, index).has_value()) { + return false; + } + const auto value = codec.read(path); + return value.has_value() && value.value() == index; + })); + } + for (auto &operation : operations) { + REQUIRE(operation.get()); + } +} + +TEST_CASE("runtime mesh codecs are reentrant", "[store][codec]") { + TemporaryDirectory directory("mesh-reentrant"); + + SECTION("terrain") { + check_mesh_codec_reentrancy(mesh::codec::Terrain{}, directory.path() / "terrain"); + } + SECTION("binary glTF") { + check_mesh_codec_reentrancy( + mesh::codec::Gltf(mesh::codec::GltfContainer::Binary), + directory.path() / "glb"); + } + SECTION("JSON glTF") { + check_mesh_codec_reentrancy( + mesh::codec::Gltf(mesh::codec::GltfContainer::Json), + directory.path() / "gltf"); + } +} + +TEST_CASE("runtime glTF codec converts writer exceptions", "[store][codec]") { + TemporaryDirectory directory("gltf-error"); + mesh::codec::Gltf codec(mesh::codec::GltfContainer::Binary); + const store::NodePath node_path(directory.path() / "blocked"); + REQUIRE(std::filesystem::create_directory(codec.paths(node_path).front())); + + const auto result = codec.write(node_path, triangle_mesh(0.0)); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().operation == store::CodecOperation::Write); + CHECK(result.error().category == store::CodecErrorCategory::Domain); +} diff --git a/unittests/terrainlib/store_layout.cpp b/unittests/terrainlib/store_layout.cpp new file mode 100644 index 00000000..ec4eaa4b --- /dev/null +++ b/unittests/terrainlib/store_layout.cpp @@ -0,0 +1,70 @@ +#include + +#include + +#include "mesh/codec/Gltf.h" +#include "mesh/codec/Terrain.h" +#include "octree/store_layout/Mappings.h" +#include "store/Layout.h" + +TEST_CASE("extensionless octree mappings preserve physical fixture paths", "[store][layout]") { + const std::filesystem::path fixtures = + std::filesystem::path(ALP_TEST_DATA_DIR) / "raster-store-refactor"; + mesh::codec::Terrain terrain; + + SECTION("flat") { + const store::Layout layout(fixtures / "sf-flat", octree::store_layout::flat()); + const store::NodePath node_path = layout.node_path(octree::Id::root()); + CHECK(node_path.path() == fixtures / "sf-flat/0-0"); + CHECK(terrain.paths(node_path) == std::vector{fixtures / "sf-flat/0-0.terrain"}); + CHECK(layout.key_from_node_path(node_path) == octree::Id::root()); + } + + SECTION("level and coordinate directories") { + const store::Layout layout( + fixtures / "sf-coordinates", + octree::store_layout::level_and_coordinate_directories()); + const octree::Id child = octree::Id::root().child(2).value(); + const octree::Id deep = octree::Id::root().child(5).value().child(7).value(); + CHECK(terrain.paths(layout.node_path(child)) + == std::vector{fixtures / "sf-coordinates/1/0/1/0.terrain"}); + CHECK(terrain.paths(layout.node_path(deep)) + == std::vector{fixtures / "sf-coordinates/2/3/1/3.terrain"}); + CHECK(layout.key_from_node_path(layout.node_path(child)) == child); + CHECK(layout.key_from_node_path(layout.node_path(deep)) == deep); + } +} + +TEST_CASE("octree mapping lookup is explicit", "[store][layout]") { + REQUIRE(octree::store_layout::from_id("flat").has_value()); + CHECK(octree::store_layout::from_id("flat")->id == "flat"); + REQUIRE(octree::store_layout::from_id("level_and_coordinate_directories").has_value()); + CHECK(octree::store_layout::from_id("level_and_coordinate_directories")->id + == "level_and_coordinate_directories"); + CHECK_FALSE(octree::store_layout::from_id("unknown").has_value()); + CHECK(octree::store_layout::all().size() == 2); +} + +TEST_CASE("extensionless octree mappings validate complete paths", "[store][layout]") { + const auto flat = octree::store_layout::flat(); + CHECK_FALSE(flat.node_path_to_key(store::NodePath("1-2.terrain")).has_value()); + CHECK_FALSE(flat.node_path_to_key(store::NodePath("nested/1-2")).has_value()); + CHECK_FALSE(flat.node_path_to_key(store::NodePath("1-2-extra")).has_value()); + + const auto coordinates = octree::store_layout::level_and_coordinate_directories(); + CHECK_FALSE(coordinates.node_path_to_key(store::NodePath("1/0/0/0.terrain")).has_value()); + CHECK_FALSE(coordinates.node_path_to_key(store::NodePath("1/0/0")).has_value()); + CHECK_FALSE(coordinates.node_path_to_key(store::NodePath("1/0/0/0/extra")).has_value()); + CHECK_FALSE(coordinates.node_path_to_key(store::NodePath("22/0/0/0")).has_value()); +} + +TEST_CASE("runtime mesh codecs own their filename endings", "[store][layout][codec]") { + const store::NodePath path("12/34/56/78"); + mesh::codec::Terrain terrain; + mesh::codec::Gltf binary(mesh::codec::GltfContainer::Binary); + mesh::codec::Gltf json(mesh::codec::GltfContainer::Json); + + CHECK(terrain.paths(path) == std::vector{"12/34/56/78.terrain"}); + CHECK(binary.paths(path) == std::vector{"12/34/56/78.glb"}); + CHECK(json.paths(path) == std::vector{"12/34/56/78.gltf"}); +} From 61acb743b03843a4200f23d1221043c2cb7840cc Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:43:03 +0200 Subject: [PATCH 55/61] Add shared runtime-codec storage core --- docs/raster-store/refactor-status.md | 19 +- src/terrainlib/store/IndexFormat.h | 49 +++++ src/terrainlib/store/IndexedStorage.h | 36 +++ src/terrainlib/store/RawStorage.h | 181 +++++++++++++++ src/terrainlib/store/Storage.h | 292 +++++++++++++++++++++++++ src/terrainlib/store/StorageError.h | 52 +++++ src/terrainlib/store/StorageSettings.h | 9 + unittests/CMakeLists.txt | 1 + unittests/terrainlib/store_storage.cpp | 251 +++++++++++++++++++++ 9 files changed, 887 insertions(+), 3 deletions(-) create mode 100644 src/terrainlib/store/IndexFormat.h create mode 100644 src/terrainlib/store/IndexedStorage.h create mode 100644 src/terrainlib/store/RawStorage.h create mode 100644 src/terrainlib/store/Storage.h create mode 100644 src/terrainlib/store/StorageError.h create mode 100644 src/terrainlib/store/StorageSettings.h create mode 100644 unittests/terrainlib/store_storage.cpp diff --git a/docs/raster-store/refactor-status.md b/docs/raster-store/refactor-status.md index 6a2e1e83..b5158ed2 100644 --- a/docs/raster-store/refactor-status.md +++ b/docs/raster-store/refactor-status.md @@ -16,10 +16,10 @@ This file is the resumable implementation log for ## Current position -- Phase: 2 — Introduce path mappings and runtime codecs +- Phase: 3 — Generalize storage and index lifecycle - State: in progress -- Next action: run the complete Phase 2 verification, inspect the diff, - commit, and tag the phase boundary. +- Next action: build and verify the shared storage core, then make the first + Phase 3 commit before migrating production adapters. ## Phase log @@ -68,6 +68,19 @@ This file is the resumable implementation log for and concurrent codec tests. - State: complete. +### Phase 3 — Generalize storage and index lifecycle + +- In progress. +- Added typed storage, filesystem, overwrite, and index-format errors. +- Added shared `RawStorage`, `Storage`, and `IndexedStorage` with exclusive + runtime-codec ownership, trait-keyed operations, multi-file `has`/`remove`, + overwrite handling, index persistence, and move finalization. +- Added shared storage instantiations for both 2D and 3D traits plus move, + overwrite, invalid-key, and multi-file tests. +- Shared storage core focused tests: 48 assertions in 6 test cases passed. +- Shared storage core build and `git diff --check`: passed. +- Next: migrate 3D format/opening adapters and application callers. + ## Verification log - Baseline `unittests_terrainlib`: 23,676 assertions in 391 test cases passed. diff --git a/src/terrainlib/store/IndexFormat.h b/src/terrainlib/store/IndexFormat.h new file mode 100644 index 00000000..7e0c3b8c --- /dev/null +++ b/src/terrainlib/store/IndexFormat.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include +#include +#include + +#include + +#include "store/Index.h" +#include "store/PathMapping.h" + +namespace store { + +enum class IndexFormatErrorCategory { + Open, + Write, + Malformed, +}; + +struct IndexFormatError { + IndexFormatErrorCategory category; + std::filesystem::path path; + std::string message; + + bool operator==(const IndexFormatError &) const = default; +}; + +template +struct IndexMetadata { + Index index; + std::string layout_id; + std::string codec_selector; +}; + +template +struct IndexFormat { + std::string_view index_filename; + + std::expected, IndexFormatError> (*read)( + const std::filesystem::path &index_path); + std::expected (*write)( + const std::filesystem::path &index_path, + const IndexMetadata &metadata); + std::optional> (*mapping_from_id)(std::string_view id); + PathMapping (*default_mapping)(); +}; + +} // namespace store diff --git a/src/terrainlib/store/IndexedStorage.h b/src/terrainlib/store/IndexedStorage.h new file mode 100644 index 00000000..9a963a10 --- /dev/null +++ b/src/terrainlib/store/IndexedStorage.h @@ -0,0 +1,36 @@ +#pragma once + +#include + +#include "store/Storage.h" + +namespace store { + +template +class IndexedStorage : public Storage { +public: + using Base = Storage; + using typename Base::Persistence; + + explicit IndexedStorage(Storage storage) + : Base(std::move(storage)) { + this->ensure_indexed(); + } + + IndexedStorage( + RawStorage raw, + Index index, + Persistence persistence) + : Base(std::move(raw), std::move(index), std::move(persistence)) {} + + IndexedStorage(const IndexedStorage &) = delete; + IndexedStorage &operator=(const IndexedStorage &) = delete; + IndexedStorage(IndexedStorage &&) noexcept = default; + IndexedStorage &operator=(IndexedStorage &&) noexcept = default; + + const Index &index() const { + return this->index_ref(); + } +}; + +} // namespace store diff --git a/src/terrainlib/store/RawStorage.h b/src/terrainlib/store/RawStorage.h new file mode 100644 index 00000000..487a3e97 --- /dev/null +++ b/src/terrainlib/store/RawStorage.h @@ -0,0 +1,181 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +#include "store/Codec.h" +#include "store/Layout.h" +#include "store/StorageError.h" +#include "store/Traits.h" + +namespace store { + +template +class RawStorage { +public: + using Key = typename Traits::Key; + using value_type = NodeData; + + RawStorage(Layout layout, std::unique_ptr> codec) + : _layout(std::move(layout)), _codec(std::move(codec)) {} + + RawStorage(const RawStorage &) = delete; + RawStorage &operator=(const RawStorage &) = delete; + RawStorage(RawStorage &&) noexcept = default; + RawStorage &operator=(RawStorage &&) noexcept = default; + + std::expected> load(const Key &key) const { + if (!Traits::is_valid(key)) { + return std::unexpected(LoadError(InvalidKey{key})); + } + const auto result = _codec->read(_layout.node_path(key)); + if (!result.has_value()) { + return std::unexpected(LoadError(result.error())); + } + return result.value(); + } + + std::expected> save(const Key &key, const NodeData &data) const { + if (!Traits::is_valid(key)) { + return std::unexpected(SaveError(InvalidKey{key})); + } + const auto result = _codec->write(_layout.node_path(key), data); + if (!result.has_value()) { + return std::unexpected(SaveError(result.error())); + } + return {}; + } + + std::expected, InvalidKey> paths( + const Key &key) const { + if (!Traits::is_valid(key)) { + return std::unexpected(InvalidKey{key}); + } + return _codec->paths(_layout.node_path(key)); + } + + std::expected> has(const Key &key) const { + const auto node_paths = paths(key); + if (!node_paths.has_value()) { + return std::unexpected(FileOperationError(node_paths.error())); + } + if (node_paths->empty()) { + return false; + } + for (const auto &path : node_paths.value()) { + std::error_code error; + const bool exists = std::filesystem::exists(path, error); + if (error) { + return std::unexpected(FileOperationError(FilesystemError{ + path, + "exists", + error, + })); + } + if (!exists) { + return false; + } + } + return true; + } + + std::expected> remove(const Key &key) const { + const auto node_paths = paths(key); + if (!node_paths.has_value()) { + return std::unexpected(FileOperationError(node_paths.error())); + } + bool removed = false; + for (const auto &path : node_paths.value()) { + std::error_code error; + removed = std::filesystem::remove(path, error) || removed; + if (error) { + return std::unexpected(FileOperationError(FilesystemError{ + path, + "remove", + error, + })); + } + } + return removed; + } + + std::expected> copy_from( + const Key &key, + const RawStorage &source) const { + if (!Traits::is_valid(key)) { + return std::unexpected(CopyError(InvalidKey{key})); + } + const auto source_exists = source.has(key); + if (!source_exists.has_value()) { + return std::unexpected(std::visit( + [](const auto &error) -> CopyError { return error; }, + source_exists.error())); + } + if (!source_exists.value()) { + return std::unexpected(CopyError(MissingSource{key})); + } + + const NodePath probe("__codec_probe__/node"); + if (_codec->paths(probe) != source._codec->paths(probe)) { + const auto loaded = source._codec->read(source._layout.node_path(key)); + if (!loaded.has_value()) { + return std::unexpected(CopyError(loaded.error())); + } + const auto written = _codec->write(_layout.node_path(key), loaded.value()); + if (!written.has_value()) { + return std::unexpected(CopyError(written.error())); + } + return {}; + } + + const auto source_paths = source._codec->paths(source._layout.node_path(key)); + const auto target_paths = _codec->paths(_layout.node_path(key)); + for (size_t index = 0; index < source_paths.size(); ++index) { + std::error_code error; + std::filesystem::remove(target_paths[index], error); + if (error) { + return std::unexpected(CopyError(FilesystemError{ + target_paths[index], + "remove", + error, + })); + } + std::filesystem::create_directories(target_paths[index].parent_path(), error); + if (error) { + return std::unexpected(CopyError(FilesystemError{ + target_paths[index].parent_path(), + "create_directories", + error, + })); + } + std::filesystem::create_hard_link(source_paths[index], target_paths[index], error); + if (error) { + return std::unexpected(CopyError(FilesystemError{ + target_paths[index], + "create_hard_link", + error, + })); + } + } + return {}; + } + + const Layout &layout() const { + return _layout; + } + const Codec &codec() const { + return *_codec; + } + +private: + Layout _layout; + std::unique_ptr> _codec; +}; + +} // namespace store diff --git a/src/terrainlib/store/Storage.h b/src/terrainlib/store/Storage.h new file mode 100644 index 00000000..fa99b6a4 --- /dev/null +++ b/src/terrainlib/store/Storage.h @@ -0,0 +1,292 @@ +#pragma once + +#include +#include +#include +#include + +#include + +#include "log.h" +#include "store/IndexFormat.h" +#include "store/RawStorage.h" +#include "store/StorageSettings.h" + +namespace store { + +template +class Storage { +public: + using Key = typename Traits::Key; + using value_type = NodeData; + + struct Persistence { + IndexFormat format; + std::filesystem::path index_path; + std::string layout_id; + std::string codec_selector; + }; + + explicit Storage(RawStorage raw) : _raw(std::move(raw)) {} + + Storage( + RawStorage raw, + Index index, + Persistence persistence) + : _raw(std::move(raw)), + _index(std::move(index)), + _persistence(std::move(persistence)) {} + + virtual ~Storage() { + finalize_displaced_state(); + } + + Storage(const Storage &) = delete; + Storage &operator=(const Storage &) = delete; + + Storage(Storage &&other) noexcept + : _raw(std::move(other._raw)), + _index(std::move(other._index)), + _persistence(std::move(other._persistence)), + _settings(other._settings), + _dirty(std::exchange(other._dirty, false)) { + other._index.reset(); + other._persistence.reset(); + } + + Storage &operator=(Storage &&other) noexcept { + if (this == &other) { + return *this; + } + finalize_displaced_state(); + _raw = std::move(other._raw); + _index = std::move(other._index); + _persistence = std::move(other._persistence); + _settings = other._settings; + _dirty = std::exchange(other._dirty, false); + other._index.reset(); + other._persistence.reset(); + return *this; + } + + std::expected> load(const Key &key) const { + if (!Traits::is_valid(key)) { + return std::unexpected(LoadError(InvalidKey{key})); + } + if (_index.has_value()) { + const auto status = _index->get(key); + if (!status.has_value()) { + return std::unexpected(LoadError(status.error())); + } + if (!status->has_value() + || status->value() == NodeStatus::Virtual) { + return std::unexpected(LoadError(CodecError{ + CodecOperation::Read, + CodecErrorCategory::FileNotFound, + "node is not physically present in the index", + })); + } + } + return _raw.load(key); + } + + std::expected> save(const Key &key, const NodeData &data) { + if (!Traits::is_valid(key)) { + return std::unexpected(SaveError(InvalidKey{key})); + } + const auto exists = has(key); + if (!exists.has_value()) { + return std::unexpected(std::visit( + [](const auto &error) -> SaveError { return error; }, + exists.error())); + } + if (exists.value() && !_settings.allow_overwrite) { + const auto node_paths = _raw.paths(key).value(); + return std::unexpected(SaveError(AlreadyExists{ + node_paths.empty() ? _raw.layout().node_path(key).path() : node_paths.front(), + })); + } + + const auto result = _raw.save(key, data); + if (!result.has_value()) { + return result; + } + if (_index.has_value()) { + const auto added = _index->add(key); + if (!added.has_value()) { + return std::unexpected(SaveError(added.error())); + } + _dirty = _dirty || added.value(); + } + return {}; + } + + std::expected> copy_from( + const Key &key, + const Storage &source) { + if (!Traits::is_valid(key)) { + return std::unexpected(CopyError(InvalidKey{key})); + } + const auto exists = has(key); + if (!exists.has_value()) { + return std::unexpected(std::visit( + [](const auto &error) -> CopyError { return error; }, + exists.error())); + } + if (exists.value() && !_settings.allow_overwrite) { + const auto node_paths = _raw.paths(key).value(); + return std::unexpected(CopyError(AlreadyExists{ + node_paths.empty() ? _raw.layout().node_path(key).path() : node_paths.front(), + })); + } + if (exists.value() && _index.has_value()) { + const auto removed = _index->remove(key); + if (!removed.has_value()) { + return std::unexpected(CopyError(removed.error())); + } + _dirty = _dirty || removed.value(); + } + + const auto result = _raw.copy_from(key, source._raw); + if (!result.has_value()) { + return result; + } + if (_index.has_value()) { + const auto added = _index->add(key); + if (!added.has_value()) { + return std::unexpected(CopyError(added.error())); + } + _dirty = _dirty || added.value(); + } + return {}; + } + + std::expected> remove(const Key &key) { + const auto removed = _raw.remove(key); + if (!removed.has_value()) { + return removed; + } + if (_index.has_value()) { + const auto index_removed = _index->remove(key); + if (!index_removed.has_value()) { + return std::unexpected(FileOperationError(index_removed.error())); + } + _dirty = _dirty || index_removed.value(); + } + return removed.value(); + } + + std::expected> has(const Key &key) const { + if (!Traits::is_valid(key)) { + return std::unexpected(FileOperationError(InvalidKey{key})); + } + if (!_index.has_value()) { + return _raw.has(key); + } + const auto status = _index->get(key); + if (!status.has_value()) { + return std::unexpected(FileOperationError(status.error())); + } + return status->has_value() && status->value() != NodeStatus::Virtual; + } + + std::expected, InvalidKey> paths( + const Key &key) const { + return _raw.paths(key); + } + + const std::filesystem::path &base_path() const { + return _raw.layout().base_path(); + } + const Layout &layout() const { + return _raw.layout(); + } + const Codec &codec() const { + return _raw.codec(); + } + + bool is_indexed() const { + return _index.has_value(); + } + void ensure_indexed() { + if (!_index.has_value()) { + _index.emplace(); + _dirty = true; + } + } + + std::optional>> index() const { + if (!_index.has_value()) { + return std::nullopt; + } + return std::cref(_index.value()); + } + + std::expected save_index() const { + if (!_dirty) { + return {}; + } + if (!_index.has_value() || !_persistence.has_value()) { + return std::unexpected(IndexFormatError{ + IndexFormatErrorCategory::Write, + {}, + "indexed storage has no persistence configuration", + }); + } + const IndexMetadata metadata{ + _index.value(), + _persistence->layout_id, + _persistence->codec_selector, + }; + const auto result = _persistence->format.write( + _persistence->index_path, + metadata); + if (result.has_value()) { + _dirty = false; + } + return result; + } + + const StorageSettings &settings() const { + return _settings; + } + StorageSettings &settings() { + return _settings; + } + +protected: + Index &index_mut() { + return _index.value(); + } + const Index &index_ref() const { + return _index.value(); + } + bool is_index_dirty() const { + return _dirty; + } + void set_index_dirty(const bool dirty = true) const { + _dirty = dirty; + } + +private: + void finalize_displaced_state() noexcept { + if (!_dirty || !_index.has_value()) { + return; + } + const auto result = save_index(); + if (!result.has_value()) { + LOG_ERROR( + "Failed to automatically save index {}: {}", + result.error().path, + result.error().message); + } + } + + RawStorage _raw; + std::optional> _index; + std::optional _persistence; + StorageSettings _settings; + mutable bool _dirty = false; +}; + +} // namespace store diff --git a/src/terrainlib/store/StorageError.h b/src/terrainlib/store/StorageError.h new file mode 100644 index 00000000..ffd6d69b --- /dev/null +++ b/src/terrainlib/store/StorageError.h @@ -0,0 +1,52 @@ +#pragma once + +#include +#include +#include +#include + +#include "store/CodecError.h" +#include "store/InvalidKey.h" + +namespace store { + +struct AlreadyExists { + std::filesystem::path path; + bool operator==(const AlreadyExists &) const = default; +}; + +struct FilesystemError { + std::filesystem::path path; + std::string operation; + std::error_code error; + bool operator==(const FilesystemError &) const = default; +}; + +template +struct MissingSource { + Key key; + bool operator==(const MissingSource &) const = default; +}; + +template +using LoadError = std::variant, CodecError>; + +template +using SaveError = std::variant< + InvalidKey, + CodecError, + AlreadyExists, + FilesystemError>; + +template +using FileOperationError = std::variant, FilesystemError>; + +template +using CopyError = std::variant< + InvalidKey, + MissingSource, + AlreadyExists, + FilesystemError, + CodecError>; + +} // namespace store diff --git a/src/terrainlib/store/StorageSettings.h b/src/terrainlib/store/StorageSettings.h new file mode 100644 index 00000000..2a883ee1 --- /dev/null +++ b/src/terrainlib/store/StorageSettings.h @@ -0,0 +1,9 @@ +#pragma once + +namespace store { + +struct StorageSettings { + bool allow_overwrite = false; +}; + +} // namespace store diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 72f787cf..26db9b5d 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -34,6 +34,7 @@ add_executable(unittests_terrainlib terrainlib/store_codec.cpp terrainlib/store_index.cpp terrainlib/store_layout.cpp + terrainlib/store_storage.cpp terrainlib/store_traverse.cpp terrainlib/index_map.cpp terrainlib/index_traverse.cpp diff --git a/unittests/terrainlib/store_storage.cpp b/unittests/terrainlib/store_storage.cpp new file mode 100644 index 00000000..04125969 --- /dev/null +++ b/unittests/terrainlib/store_storage.cpp @@ -0,0 +1,251 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "io/serialize.h" +#include "octree/StoreTraits.h" +#include "octree/store_layout/Mappings.h" +#include "raster_store/StoreTraits.h" +#include "store/IndexedStorage.h" +#include "store/codec/ZppBits.h" +#include "string_utils.h" + +namespace { + +class TemporaryDirectory { +public: + explicit TemporaryDirectory(const std::string_view label) { + static std::atomic_uint64_t counter = 0; + const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); + _path = std::filesystem::temp_directory_path() + / ("atb-store-storage-" + std::string(label) + "-" + + std::to_string(timestamp) + "-" + std::to_string(counter++)); + REQUIRE(std::filesystem::create_directories(_path)); + } + ~TemporaryDirectory() { + std::error_code error; + std::filesystem::remove_all(_path, error); + } + const std::filesystem::path &path() const { + return _path; + } + +private: + std::filesystem::path _path; +}; + +store::NodePath tile_to_path(const radix::tile::Id &key) { + return store::NodePath( + std::to_string(key.zoom_level) + "/" + std::to_string(key.coords.x) + + "/" + std::to_string(key.coords.y)); +} + +std::optional path_to_tile(const store::NodePath &node_path) { + const std::filesystem::path &path = node_path.path(); + if (path.is_absolute() || std::distance(path.begin(), path.end()) != 3) { + return std::nullopt; + } + auto part = path.begin(); + const auto zoom = from_chars(part->string()); + ++part; + const auto x = from_chars(part->string()); + ++part; + const auto y = from_chars(part->string()); + if (!zoom.has_value() || !x.has_value() || !y.has_value()) { + return std::nullopt; + } + const radix::tile::Id key{zoom.value(), {x.value(), y.value()}}; + return raster_store::StoreTraits::is_valid(key) + ? std::optional(key) + : std::nullopt; +} + +template +store::PathMapping test_mapping(); + +template<> +store::PathMapping test_mapping() { + return octree::store_layout::flat(); +} + +template<> +store::PathMapping test_mapping() { + return {"test_tiles", tile_to_path, path_to_tile}; +} + +template +store::Storage make_storage(const std::filesystem::path &path) { + return store::Storage(store::RawStorage( + store::Layout(path, test_mapping()), + std::make_unique>())); +} + +class MultiFileCodec final : public store::Codec { +public: + std::vector paths(const store::NodePath &node_path) const override { + std::filesystem::path first = node_path.path(); + std::filesystem::path second = node_path.path(); + first += ".data"; + second += ".metadata"; + return {first, second}; + } +}; + +std::atomic_uint32_t index_writes = 0; + +std::expected, store::IndexFormatError> +unused_index_read(const std::filesystem::path &path) { + return std::unexpected(store::IndexFormatError{ + store::IndexFormatErrorCategory::Open, + path, + "not used", + }); +} + +std::expected count_index_write( + const std::filesystem::path &, + const store::IndexMetadata &) { + ++index_writes; + return {}; +} + +std::optional> fake_mapping_from_id( + const std::string_view id) { + return id == "flat" + ? std::optional>(octree::store_layout::flat()) + : std::nullopt; +} + +store::PathMapping fake_default_mapping() { + return octree::store_layout::flat(); +} + +store::IndexFormat counting_index_format() { + return { + "terrain.index", + unused_index_read, + count_index_write, + fake_mapping_from_id, + fake_default_mapping, + }; +} + +store::IndexedStorage make_indexed_storage( + const std::filesystem::path &path) { + using Storage = store::Storage; + return store::IndexedStorage( + store::RawStorage( + store::Layout(path, octree::store_layout::flat()), + std::make_unique>()), + store::Index{}, + Storage::Persistence{ + counting_index_format(), + path / "terrain.index", + "flat", + ".bin", + }); +} + +} // namespace + +TEMPLATE_TEST_CASE( + "shared raw storage has no hidden octree dependency", + "[store][storage]", + octree::StoreTraits, + raster_store::StoreTraits) { + using Traits = TestType; + TemporaryDirectory directory("traits"); + auto storage = make_storage(directory.path()); + const auto root = Traits::root(); + + CHECK_FALSE(storage.has(root).value()); + REQUIRE(storage.save(root, 42).has_value()); + CHECK(storage.has(root).value()); + REQUIRE(storage.load(root).has_value()); + CHECK(storage.load(root).value() == 42); + REQUIRE(storage.remove(root).has_value()); + CHECK_FALSE(storage.has(root).value()); +} + +TEST_CASE("shared storage rejects invalid 2D keys", "[store][storage]") { + TemporaryDirectory directory("invalid-key"); + auto storage = make_storage(directory.path()); + const radix::tile::Id invalid{3, {8, 0}}; + + CHECK(std::holds_alternative>( + storage.load(invalid).error())); + CHECK(std::holds_alternative>( + storage.save(invalid, 1).error())); + CHECK(std::holds_alternative>( + storage.has(invalid).error())); + CHECK(std::holds_alternative>( + storage.remove(invalid).error())); +} + +TEST_CASE("shared storage preserves overwrite settings", "[store][storage]") { + TemporaryDirectory directory("overwrite"); + auto storage = make_storage(directory.path()); + const octree::Id root = octree::Id::root(); + + REQUIRE(storage.save(root, 1).has_value()); + const auto rejected = storage.save(root, 2); + REQUIRE_FALSE(rejected.has_value()); + CHECK(std::holds_alternative(rejected.error())); + CHECK(storage.load(root).value() == 1); + + storage.settings().allow_overwrite = true; + REQUIRE(storage.save(root, 2).has_value()); + CHECK(storage.load(root).value() == 2); +} + +TEST_CASE("raw storage requires and removes every codec path", "[store][storage]") { + TemporaryDirectory directory("multi-file"); + store::RawStorage storage( + store::Layout(directory.path(), octree::store_layout::flat()), + std::make_unique()); + const octree::Id root = octree::Id::root(); + const auto paths = storage.paths(root).value(); + REQUIRE(paths.size() == 2); + + REQUIRE(io::write_to_path(1, paths[0]).has_value()); + CHECK_FALSE(storage.has(root).value()); + REQUIRE(io::write_to_path(2, paths[1]).has_value()); + CHECK(storage.has(root).value()); + REQUIRE(storage.remove(root).value()); + CHECK_FALSE(std::filesystem::exists(paths[0])); + CHECK_FALSE(std::filesystem::exists(paths[1])); +} + +TEST_CASE("storage moves transfer and finalize dirty index state", "[store][storage]") { + index_writes = 0; + TemporaryDirectory first_directory("move-first"); + TemporaryDirectory second_directory("move-second"); + + { + auto original = make_indexed_storage(first_directory.path()); + REQUIRE(original.save(octree::Id::root(), 1).has_value()); + auto moved = std::move(original); + CHECK(index_writes == 0); + REQUIRE(moved.save_index().has_value()); + CHECK(index_writes == 1); + } + CHECK(index_writes == 1); + + { + auto destination = make_indexed_storage(first_directory.path()); + destination.settings().allow_overwrite = true; + REQUIRE(destination.save(octree::Id::root(), 2).has_value()); + auto source = make_indexed_storage(second_directory.path()); + REQUIRE(source.save(octree::Id::root(), 3).has_value()); + destination = std::move(source); + CHECK(index_writes == 2); + } + CHECK(index_writes == 3); +} From 4ffcdeadbe34f16ed40f6ac7452ea331b6c9587e Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Fri, 14 Aug 2026 17:10:25 +0200 Subject: [PATCH 56/61] Migrate applications to shared storage --- docs/raster-store/refactor-status.md | 31 +- src/dag_builder/build.cpp | 41 ++- src/dag_builder/codec/MetadataView.h | 38 +++ src/dag_builder/codec/from_extension.h | 40 +++ src/dag_builder/main.cpp | 28 +- src/dag_builder/metadata.h | 11 - src/dag_builder/storage.h | 65 +++- src/dag_builder/thread_safe_storage.h | 9 +- src/dag_convert_debug/main.cpp | 45 ++- src/sf_builder/terrainbuilder.cpp | 41 ++- src/sf_index_browser/main.cpp | 39 ++- src/sf_merger/NodeLoader.h | 11 +- src/sf_merger/NodeWriter.h | 6 +- src/sf_merger/cut.h | 33 +- src/sf_merger/main.cpp | 40 ++- src/sf_merger/merge.h | 6 +- src/terrainlib/CMakeLists.txt | 1 - src/terrainlib/mesh/codec/from_extension.h | 33 ++ src/terrainlib/mesh/storage.h | 49 +++ src/terrainlib/octree/IndexMap.h | 6 + src/terrainlib/octree/Storage.h | 11 +- src/terrainlib/octree/disk/Layout.h | 49 --- src/terrainlib/octree/disk/layout/Strategy.h | 19 -- .../octree/disk/layout/StrategyRegister.h | 84 ----- .../octree/disk/layout/strategy/Default.h | 15 - .../octree/disk/layout/strategy/Flat.h | 50 --- .../strategy/LevelAndCoordinateDirectories.h | 60 ---- src/terrainlib/octree/storage/CopyError.h | 59 ---- src/terrainlib/octree/storage/IndexFile.h | 68 ++++ .../octree/storage/IndexedStorage.h | 67 ---- src/terrainlib/octree/storage/MeshStorage.h | 11 +- src/terrainlib/octree/storage/RawStorage.h | 118 ------- src/terrainlib/octree/storage/Storage.h | 312 ------------------ src/terrainlib/octree/storage/cache/Dummy.h | 26 +- src/terrainlib/octree/storage/cache/ICache.h | 19 +- .../octree/storage/cache/LruCache.h | 70 +--- src/terrainlib/octree/storage/codec/Codec.h | 32 -- .../octree/storage/codec/MeshCodec.h | 35 -- .../octree/storage/codec/ReadOnlyCodec.h | 20 -- .../octree/storage/codec/ZppBitsCodec.h | 38 --- src/terrainlib/octree/storage/defaults.h | 25 -- src/terrainlib/octree/storage/helpers.cpp | 136 -------- src/terrainlib/octree/storage/helpers.h | 28 -- src/terrainlib/octree/storage/open.h | 44 ++- src/terrainlib/octree/storage/open.inl | 111 ------- src/terrainlib/octree/storage/open_runtime.h | 196 +++++++++++ src/terrainlib/octree/traverse.h | 17 + src/terrainlib/store/OpenError.h | 24 ++ src/terrainlib/store/Storage.h | 116 +++++++ src/terrainlib/store/cache/Dummy.h | 25 ++ src/terrainlib/store/cache/Interface.h | 21 ++ src/terrainlib/store/cache/Lru.h | 73 ++++ src/terrainlib/store/describe_error.h | 50 +++ src/terrainlib/store/open.h | 93 ++++++ .../dag_builder/storage_compatibility.cpp | 52 ++- unittests/terrainlib/store_compatibility.cpp | 123 +++++-- 56 files changed, 1347 insertions(+), 1523 deletions(-) create mode 100644 src/dag_builder/codec/MetadataView.h create mode 100644 src/dag_builder/codec/from_extension.h create mode 100644 src/terrainlib/mesh/codec/from_extension.h create mode 100644 src/terrainlib/mesh/storage.h delete mode 100644 src/terrainlib/octree/disk/Layout.h delete mode 100644 src/terrainlib/octree/disk/layout/Strategy.h delete mode 100644 src/terrainlib/octree/disk/layout/StrategyRegister.h delete mode 100644 src/terrainlib/octree/disk/layout/strategy/Default.h delete mode 100644 src/terrainlib/octree/disk/layout/strategy/Flat.h delete mode 100644 src/terrainlib/octree/disk/layout/strategy/LevelAndCoordinateDirectories.h delete mode 100644 src/terrainlib/octree/storage/CopyError.h create mode 100644 src/terrainlib/octree/storage/IndexFile.h delete mode 100644 src/terrainlib/octree/storage/IndexedStorage.h delete mode 100644 src/terrainlib/octree/storage/RawStorage.h delete mode 100644 src/terrainlib/octree/storage/Storage.h delete mode 100644 src/terrainlib/octree/storage/codec/Codec.h delete mode 100644 src/terrainlib/octree/storage/codec/MeshCodec.h delete mode 100644 src/terrainlib/octree/storage/codec/ReadOnlyCodec.h delete mode 100644 src/terrainlib/octree/storage/codec/ZppBitsCodec.h delete mode 100644 src/terrainlib/octree/storage/defaults.h delete mode 100644 src/terrainlib/octree/storage/helpers.cpp delete mode 100644 src/terrainlib/octree/storage/helpers.h delete mode 100644 src/terrainlib/octree/storage/open.inl create mode 100644 src/terrainlib/octree/storage/open_runtime.h create mode 100644 src/terrainlib/store/OpenError.h create mode 100644 src/terrainlib/store/cache/Dummy.h create mode 100644 src/terrainlib/store/cache/Interface.h create mode 100644 src/terrainlib/store/cache/Lru.h create mode 100644 src/terrainlib/store/describe_error.h create mode 100644 src/terrainlib/store/open.h diff --git a/docs/raster-store/refactor-status.md b/docs/raster-store/refactor-status.md index b5158ed2..1bfd3b4a 100644 --- a/docs/raster-store/refactor-status.md +++ b/docs/raster-store/refactor-status.md @@ -17,9 +17,9 @@ This file is the resumable implementation log for ## Current position - Phase: 3 — Generalize storage and index lifecycle -- State: in progress -- Next action: build and verify the shared storage core, then make the first - Phase 3 commit before migrating production adapters. +- State: complete; completion commit pending. +- Next action: commit and tag Phase 3, then begin Phase 4 copy hardening and + SF topology validation. ## Phase log @@ -70,7 +70,6 @@ This file is the resumable implementation log for ### Phase 3 — Generalize storage and index lifecycle -- In progress. - Added typed storage, filesystem, overwrite, and index-format errors. - Added shared `RawStorage`, `Storage`, and `IndexedStorage` with exclusive runtime-codec ownership, trait-keyed operations, multi-file `has`/`remove`, @@ -79,7 +78,18 @@ This file is the resumable implementation log for overwrite, invalid-key, and multi-file tests. - Shared storage core focused tests: 48 assertions in 6 test cases passed. - Shared storage core build and `git diff --check`: passed. -- Next: migrate 3D format/opening adapters and application callers. +- Added the legacy 3D `terrain.index` format adapter, explicit mesh and DAG + codec resolvers, folder discovery, and expected-returning opening APIs. +- Preserved independently readable, read-only DAG metadata views and migrated + DAG output synchronization to the shared storage types. +- Migrated SF builder, SF merger, SF index browser, DAG builder, and DAG + convert debug without exposing codec ownership at application call sites. +- Ported cache API aliases for compile compatibility. +- Deleted the legacy layout class hierarchy and registry, static codec stack, + and duplicate octree storage implementation. +- Added resolver, unsupported metadata write, unknown layout/codec, malformed + index no-fallback, and DAG synchronized move/release coverage. +- State: complete. ## Verification log @@ -112,6 +122,14 @@ This file is the resumable implementation log for - Phase 2 application builds passed for `sf-builder`, `dag-builder`, and `dag-convert-debug`. - Phase 2 `git diff --check`: passed. +- Phase 3 focused terrain store/open tests: 233 assertions in 25 test cases + passed. +- Phase 3 focused DAG store tests: 17 assertions in 2 test cases passed. +- Phase 3 `unittests_terrainlib`: 23,973 assertions in 422 test cases passed. +- Phase 3 `unittests_dagbuilder`: 459 assertions in 73 test cases passed. +- Phase 3 all-application configuration built `sf-builder`, `sf-merger`, + `sf-index-browser`, `dag-builder`, and `dag-convert-debug` successfully. +- Phase 3 `git diff --check`: passed. ## Commit and tag log @@ -124,3 +142,6 @@ This file is the resumable implementation log for - `refactor_phase_0` — annotated Phase 0 completion tag. - `4c171c8` / `refactor_phase_1` — shared topology extraction and Phase 1 completion tag. +- `fbc02d0` / `refactor_phase_2` — runtime layouts/codecs and Phase 2 + completion tag. +- `61acb74` — Phase 3 shared runtime-codec storage core. diff --git a/src/dag_builder/build.cpp b/src/dag_builder/build.cpp index 567fe005..203dca83 100644 --- a/src/dag_builder/build.cpp +++ b/src/dag_builder/build.cpp @@ -36,6 +36,7 @@ #include "range_utils.h" #include "storage.h" #include "thread_safe_storage.h" +#include "store/describe_error.h" #include "utils.h" #include "vertex_lock.h" #include "parallel.h" @@ -52,8 +53,13 @@ std::optional load_and_clusterize_mesh( const auto result = storage.load(id); if (!result) { - if (result.error() != mesh::io::LoadMeshErrorKind::FileNotFound) { - LOG_ERROR("Failed to read node {}: {}", id, result.error()); + const auto *codec_error = std::get_if(&result.error()); + if (codec_error == nullptr + || codec_error->category != store::CodecErrorCategory::FileNotFound) { + LOG_ERROR( + "Failed to read node {}: {}", + id, + store::describe_error(result.error())); } return std::nullopt; } @@ -448,7 +454,7 @@ LevelWorkplan build_level_workplan( // Pre-filter input nodes to only those that intersect the target root bounds. std::vector> gather_relevant_input_leaves( - const octree::IndexMap &index, + const store::Index &index, const octree::Space &space, const radix::geometry::Aabb3d &root_bounds) { @@ -500,7 +506,7 @@ std::unordered_set build_level( std::unordered_set already_built; if (ctx.options.continuation_mode != ContinuationMode::Overwrite) { for (const octree::Id &target : targets) { - if (ctx.output_storage.has(target)) { + if (DEBUG_ASSERT_VAL(ctx.output_storage.has(target)).value()) { already_built.insert(target); } } @@ -513,10 +519,18 @@ std::unordered_set build_level( // Initialize debug storage if requested (contains .glb meshes) std::optional debug_storage; if (ctx.options.write_debug_meshes) { - debug_storage = octree::open_folder( + octree::OpenOptions options; + options.preferred_extension = ".glb"; + auto debug_result = octree::open_folder( ctx.output_storage.base_path().string() + "-debug", false, - octree::OpenOptions{.preferred_extension_with_dot = ".glb"}); + std::move(options)); + if (!debug_result.has_value()) { + LOG_ERROR_AND_EXIT( + "Failed to open debug mesh storage: {}", + store::describe_error(debug_result.error())); + } + debug_storage = std::move(debug_result.value()); } tbb::concurrent_vector saved_ids; @@ -540,12 +554,18 @@ std::unordered_set build_level( if (debug_storage) { const auto debug_save_result = debug_storage->save(target, clustering_to_mesh(result->clustering)); if (!debug_save_result.has_value()) { - LOG_ERROR_AND_EXIT("Failed to save debug mesh for node {}: {}", target, debug_save_result.error()); + LOG_ERROR_AND_EXIT( + "Failed to save debug mesh for node {}: {}", + target, + store::describe_error(debug_save_result.error())); } } saved_ids.push_back(target); } else { - LOG_ERROR("Failed to save node {}: {}", target, save_result.error()); + LOG_ERROR( + "Failed to save node {}: {}", + target, + store::describe_error(save_result.error())); } } progress.task_finished(); @@ -610,7 +630,10 @@ void build_levels( // Persist per level so a finished level can be read back before the whole run completes if (const auto result = ctx.output_storage.save_or_create_index(); !result.has_value()) { - LOG_WARN("Could not save index after level {}: {}", level, result.error()); + LOG_WARN( + "Could not save index after level {}: {}", + level, + store::describe_error(result.error())); } } diff --git a/src/dag_builder/codec/MetadataView.h b/src/dag_builder/codec/MetadataView.h new file mode 100644 index 00000000..9df6b856 --- /dev/null +++ b/src/dag_builder/codec/MetadataView.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include + +#include + +#include "io/serialize.h" +#include "serialization.h" +#include "store/Codec.h" +#include "store/codec/ZppBits.h" + +namespace dag::codec { + +class MetadataView final : public store::Codec { +public: + std::vector paths( + const store::NodePath &node_path) const override { + return {store::codec::append_extension(node_path, ".bin")}; + } + + std::expected read( + const store::NodePath &node_path) const override { + const auto result = io::read_from_path(paths(node_path).front()); + if (!result.has_value()) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + result.error() == io::Error(io::Error::OpenFile) + ? store::CodecErrorCategory::FileNotFound + : store::CodecErrorCategory::Io, + fmt::format("{}", result.error()), + }); + } + return result.value(); + } +}; + +} // namespace dag::codec diff --git a/src/dag_builder/codec/from_extension.h b/src/dag_builder/codec/from_extension.h new file mode 100644 index 00000000..3233f5af --- /dev/null +++ b/src/dag_builder/codec/from_extension.h @@ -0,0 +1,40 @@ +#pragma once + +#include +#include +#include + +#include + +#include "codec/MetadataView.h" +#include "dag_node.h" +#include "serialization.h" +#include "store/codec/ZppBits.h" + +namespace dag::codec { + +inline std::expected>, store::CodecError> +from_extension(const std::string_view extension) { + if (extension == ".bin") { + return std::make_unique>(); + } + return std::unexpected(store::CodecError{ + store::CodecOperation::Resolve, + store::CodecErrorCategory::UnsupportedCodec, + "unsupported DAG codec selector: " + std::string(extension), + }); +} + +inline std::expected>, store::CodecError> +metadata_from_extension(const std::string_view extension) { + if (extension == ".bin") { + return std::make_unique(); + } + return std::unexpected(store::CodecError{ + store::CodecOperation::Resolve, + store::CodecErrorCategory::UnsupportedCodec, + "unsupported DAG metadata codec selector: " + std::string(extension), + }); +} + +} // namespace dag::codec diff --git a/src/dag_builder/main.cpp b/src/dag_builder/main.cpp index acce9d48..495bbf09 100644 --- a/src/dag_builder/main.cpp +++ b/src/dag_builder/main.cpp @@ -5,10 +5,9 @@ #include "cli.h" #include "build.h" #include "log.h" -#include "octree/storage/Storage.h" -#include "octree/storage/MeshStorage.h" #include "octree/storage/open.h" #include "storage.h" +#include "store/describe_error.h" #include "ContinuationMode.h" int main(int argc, char **argv) { @@ -17,8 +16,24 @@ int main(int argc, char **argv) { Log::init(args.log_level); try { - const octree::IndexedMeshStorage input_storage = octree::open_folder_indexed(args.input_path); - octree::IndexedDagStorage output_storage = octree::open_folder_indexed(args.output_path); + auto input_result = octree::open_folder_indexed(args.input_path); + if (!input_result.has_value()) { + LOG_ERROR( + "Failed to open input dataset {}: {}", + args.input_path, + store::describe_error(input_result.error())); + return EXIT_FAILURE; + } + auto output_result = dag::storage::open_folder_indexed(args.output_path); + if (!output_result.has_value()) { + LOG_ERROR( + "Failed to open output dataset {}: {}", + args.output_path, + store::describe_error(output_result.error())); + return EXIT_FAILURE; + } + const octree::IndexedMeshStorage input_storage = std::move(input_result.value()); + octree::IndexedDagStorage output_storage = std::move(output_result.value()); output_storage.settings().allow_overwrite = args.continuation_mode == ContinuationMode::Overwrite; dag::BuildOptions options{ @@ -46,7 +61,10 @@ int main(int argc, char **argv) { dag::build_levels(input_storage, output_storage, options, args.level_range); const auto index_result = output_storage.save_index(); if (!index_result.has_value()) { - LOG_ERROR("Failed to save output index in {}: {}", args.output_path, index_result.error()); + LOG_ERROR( + "Failed to save output index in {}: {}", + args.output_path, + store::describe_error(index_result.error())); return EXIT_FAILURE; } diff --git a/src/dag_builder/metadata.h b/src/dag_builder/metadata.h index 6c12b2bf..e5c5d35e 100644 --- a/src/dag_builder/metadata.h +++ b/src/dag_builder/metadata.h @@ -8,9 +8,6 @@ #include "cluster.h" #include "dag_id.h" #include "utils.h" -#include "octree/storage/defaults.h" -#include "octree/storage/codec/ZppBitsCodec.h" -#include "octree/storage/codec/ReadOnlyCodec.h" namespace dag { @@ -71,11 +68,3 @@ inline const radix::geometry::Aabb3d &get_group_bounds(const NodeMetadata &metad } } // namespace dag - -// Set default codec for metadata. -namespace octree { -template <> -struct DefaultCodec { - using type = ReadOnlyCodec>; -}; -} diff --git a/src/dag_builder/storage.h b/src/dag_builder/storage.h index 7fb24242..92bccec9 100644 --- a/src/dag_builder/storage.h +++ b/src/dag_builder/storage.h @@ -1,14 +1,67 @@ #pragma once +#include +#include + +#include "codec/from_extension.h" +#include "octree/StoreTraits.h" +#include "octree/storage/IndexFile.h" +#include "octree/storage/open_runtime.h" #include "serialization.h" -#include "octree/storage/Storage.h" -#include "octree/storage/IndexedStorage.h" +#include "store/IndexedStorage.h" + namespace octree { -using DagStorage = Storage_; -using IndexedDagStorage = IndexedStorage_; +using DagStorage = store::Storage; +using IndexedDagStorage = store::IndexedStorage; +using DagMetaStorage = store::Storage; +using IndexedDagMetaStorage = store::IndexedStorage; + +} // namespace octree -using DagMetaStorage = Storage_; -using IndexedDagMetaStorage = IndexedStorage_; +namespace dag::storage { + +using OpenOptions = octree::storage::OpenOptions; + +inline std::expected> open_index( + const std::filesystem::path &path) { + return store::open_index( + path, + octree::storage::index_format(), + dag::codec::from_extension); +} +inline std::expected> open_folder( + const std::filesystem::path &path, + const bool create_index = false, + OpenOptions options = {}) { + return octree::storage::open_folder( + path, + create_index, + ".bin", + dag::codec::from_extension, + std::move(options)); } + +inline std::expected> +open_folder_indexed(const std::filesystem::path &path, OpenOptions options = {}) { + return octree::storage::open_folder_indexed( + path, + ".bin", + dag::codec::from_extension, + std::move(options)); +} + +inline std::expected> +open_metadata_indexed(const std::filesystem::path &path) { + const std::filesystem::path index_path = + path.filename() == octree::disk::v1::index_file_name() + ? path + : path / octree::disk::v1::index_file_name(); + return store::open_index( + index_path, + octree::storage::index_format(), + dag::codec::metadata_from_extension); +} + +} // namespace dag::storage diff --git a/src/dag_builder/thread_safe_storage.h b/src/dag_builder/thread_safe_storage.h index 9b5e6120..da54f8a0 100644 --- a/src/dag_builder/thread_safe_storage.h +++ b/src/dag_builder/thread_safe_storage.h @@ -6,8 +6,6 @@ #include -#include "octree/Id.h" - namespace dag { // A mutex-based thread-safe wrapper around a Storage_ instance. @@ -18,6 +16,7 @@ class ThreadSafeStorage { public: using value_type = typename Storage::value_type; + using key_type = typename Storage::key_type; using load_error = typename Storage::load_error; using save_error = typename Storage::save_error; @@ -33,12 +32,12 @@ class ThreadSafeStorage { return std::move(this->_storage); } - std::expected load(const octree::Id &id) const { + std::expected load(const key_type &id) const { std::shared_lock lock(this->_mutex); return this->_storage.load(id); } - bool has(const octree::Id &id) const { + auto has(const key_type &id) const { std::shared_lock lock(this->_mutex); return this->_storage.has(id); } @@ -47,7 +46,7 @@ class ThreadSafeStorage { return this->_storage.base_path(); } - std::expected save(const octree::Id &id, const value_type &value) const { + std::expected save(const key_type &id, const value_type &value) const { std::unique_lock lock(this->_mutex); return this->_storage.save(id, value); } diff --git a/src/dag_convert_debug/main.cpp b/src/dag_convert_debug/main.cpp index 88f776fa..d72be3a2 100644 --- a/src/dag_convert_debug/main.cpp +++ b/src/dag_convert_debug/main.cpp @@ -4,19 +4,21 @@ #include "cli.h" #include "dag_node.h" #include "encoded.h" +#include "io/serialize.h" #include "log.h" #include "mesh/io.h" #include "octree/storage/MeshStorage.h" -#include "octree/storage/codec/ZppBitsCodec.h" #include "octree/storage/open.h" #include "ProgressIndicator.h" #include "storage.h" #include "utils.h" +#include "serialization.h" +#include "store/describe_error.h" namespace { void export_node(const cli::Args &args) { - const auto load_result = octree::ZppBitsCodec::load_from_path(args.input_path); + const auto load_result = io::read_from_path(args.input_path); if (!load_result.has_value()) { LOG_ERROR("Failed to load node from {}: {}", args.input_path, load_result.error()); return; @@ -31,12 +33,30 @@ void export_node(const cli::Args &args) { } void export_storage(const cli::Args &args) { - const octree::IndexedDagStorage input_storage = octree::open_folder_indexed(args.input_path); + auto input_result = dag::storage::open_folder_indexed(args.input_path); + if (!input_result.has_value()) { + LOG_ERROR( + "Failed to open input storage {}: {}", + args.input_path, + store::describe_error(input_result.error())); + return; + } + const octree::IndexedDagStorage input_storage = std::move(input_result.value()); - octree::MeshStorage output_storage = octree::open_folder( + octree::OpenOptions options; + options.preferred_extension = ".glb"; + auto output_result = octree::open_folder( args.output_path, false, - octree::OpenOptions{.preferred_extension_with_dot = ".glb"}); + std::move(options)); + if (!output_result.has_value()) { + LOG_ERROR( + "Failed to open output storage {}: {}", + args.output_path, + store::describe_error(output_result.error())); + return; + } + octree::MeshStorage output_storage = std::move(output_result.value()); output_storage.settings().allow_overwrite = true; size_t exported_count = 0; @@ -52,7 +72,10 @@ void export_storage(const cli::Args &args) { const auto load_result = input_storage.load(id); if (!load_result.has_value()) { - LOG_ERROR("Failed to load node {}: {}", id, load_result.error()); + LOG_ERROR( + "Failed to load node {}: {}", + id, + store::describe_error(load_result.error())); progress.task_finished(); continue; } @@ -61,7 +84,10 @@ void export_storage(const cli::Args &args) { const auto save_result = output_storage.save(id, mesh); if (!save_result.has_value()) { - LOG_ERROR("Failed to save mesh for node {}: {}", id, save_result.error().description()); + LOG_ERROR( + "Failed to save mesh for node {}: {}", + id, + store::describe_error(save_result.error())); progress.task_finished(); continue; } @@ -74,7 +100,10 @@ void export_storage(const cli::Args &args) { const auto index_result = output_storage.save_or_create_index(); if (!index_result.has_value()) { - LOG_ERROR("Failed to save index for {}: {}", args.output_path, index_result.error()); + LOG_ERROR( + "Failed to save index for {}: {}", + args.output_path, + store::describe_error(index_result.error())); } LOG_INFO("Exported {} debug meshes to {}", exported_count, args.output_path); diff --git a/src/sf_builder/terrainbuilder.cpp b/src/sf_builder/terrainbuilder.cpp index 083ab6ba..a5780999 100644 --- a/src/sf_builder/terrainbuilder.cpp +++ b/src/sf_builder/terrainbuilder.cpp @@ -42,8 +42,9 @@ #include "octree/Id.h" #include "octree/Space.h" #include "octree/Storage.h" -#include "octree/disk/layout/strategy/LevelAndCoordinateDirectories.h" +#include "octree/storage/open.h" #include "octree/utils.h" +#include "store/describe_error.h" namespace terrainbuilder { @@ -189,13 +190,19 @@ void build_all_patches( LOG_ERROR_AND_EXIT("Output base path {} exists but is not a directory", output_base_path); } - octree::Storage storage = octree::open_folder( + octree::OpenOptions open_options; + open_options.preferred_extension = output_format; + auto storage_result = octree::open_folder( output_base_path, false, - octree::OpenOptions { - .preferred_extension_with_dot = output_format - } - ); + std::move(open_options)); + if (!storage_result.has_value()) { + LOG_ERROR_AND_EXIT( + "Failed to open output dataset {}: {}", + output_base_path, + store::describe_error(storage_result.error())); + } + octree::Storage storage = std::move(storage_result.value()); storage.settings().allow_overwrite = overwrite_existing; const auto dataset_srs = dataset.srs(); @@ -281,7 +288,17 @@ void build_all_patches( tbb::task_group_context context; tbb::parallel_for(size_t(0), target_nodes.size(), [&](size_t i) { const auto &node = target_nodes[i]; - if (!overwrite_existing && storage.has(node)) { + const auto already_exists = storage.has(node); + if (!already_exists.has_value()) { + LOG_ERROR( + "Failed to inspect node {}: {}", + node, + store::describe_error(already_exists.error())); + progress.task_finished(); + context.cancel_group_execution(); + return; + } + if (!overwrite_existing && already_exists.value()) { progress.task_finished(); // TODO: correctly handle virtual nodes return; } @@ -305,7 +322,10 @@ void build_all_patches( mesh::validate(mesh); const auto save_result = storage.save(node, mesh); if (!save_result.has_value()) { - LOG_ERROR("Failed to save mesh for node {}: {}", node, save_result.error()); + LOG_ERROR( + "Failed to save mesh for node {}: {}", + node, + store::describe_error(save_result.error())); progress.task_finished(); context.cancel_group_execution(); return; @@ -329,7 +349,10 @@ void build_all_patches( const auto index_result = storage.save_or_create_index(); if (!index_result.has_value()) { - LOG_ERROR_AND_EXIT("Failed to save output index in {}: {}", storage.base_path(), index_result.error()); + LOG_ERROR_AND_EXIT( + "Failed to save output index in {}: {}", + storage.base_path(), + store::describe_error(index_result.error())); } } } diff --git a/src/sf_index_browser/main.cpp b/src/sf_index_browser/main.cpp index 74c8f304..0c6e6ddd 100644 --- a/src/sf_index_browser/main.cpp +++ b/src/sf_index_browser/main.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -15,8 +16,10 @@ #include "octree/NodeStatus.h" #include "octree/NodeStatusOrMissing.h" #include "octree/Storage.h" +#include "octree/storage/open.h" #include "octree/traverse.h" #include "cli.h" +#include "store/describe_error.h" namespace { struct IndexNode { @@ -42,7 +45,8 @@ class TreeView { using const_iterator = std::list::const_iterator; explicit TreeView(const octree::IndexedStorage &storage, const octree::Id root) : root(root), _storage(storage) { - const octree::NodeStatusOrMissing status = this->_storage.index().get(root); + const octree::NodeStatusOrMissing status( + DEBUG_ASSERT_VAL(this->_storage.index().get(root)).value()); this->_view.emplace_back(IndexNode{root, status, false}); } @@ -85,7 +89,7 @@ class TreeView { const auto &mesh = result.value(); entry = MeshNode{parent.id, mesh.vertex_count(), mesh.face_count()}; } else { - entry = ErrorNode{parent.id, result.error().description()}; + entry = ErrorNode{parent.id, store::describe_error(result.error())}; } it++; it = this->_view.emplace(it, entry); @@ -97,11 +101,12 @@ class TreeView { } const auto children = parent.id.children().value(); for (const auto &child_id : children) { - auto status_opt = this->_storage.index().get(child_id); - if (!status_opt.has_value()) { + auto status_result = this->_storage.index().get(child_id); + DEBUG_ASSERT(status_result.has_value()); + if (!status_result->has_value()) { continue; } - const octree::NodeStatus status = status_opt.value(); + const octree::NodeStatus status = status_result->value(); IndexNode child_node{child_id, status, false}; it++; it = this->_view.emplace(it, child_node); @@ -148,8 +153,10 @@ class TreeView { const octree::IndexedStorage& _storage; }; -const octree::Id find_deepest_root(const octree::IndexMap &index, const octree::Id &root = octree::Id::root()) { - switch (octree::NodeStatusOrMissing(index.get(root))) { +const octree::Id find_deepest_root( + const store::Index &index, + const octree::Id &root = octree::Id::root()) { + switch (octree::NodeStatusOrMissing(DEBUG_ASSERT_VAL(index.get(root)).value())) { case octree::NodeStatusOrMissing::Missing: return octree::Id::root(); case octree::NodeStatusOrMissing::Leaf: @@ -163,7 +170,8 @@ const octree::Id find_deepest_root(const octree::IndexMap &index, const octree:: const auto children = current.children().value(); std::optional next; for (const octree::Id &child_id : children) { - if (!index.get(child_id).has_value()) { + const auto status = DEBUG_ASSERT_VAL(index.get(child_id)).value(); + if (!status.has_value()) { continue; } if (next.has_value()) { @@ -184,11 +192,16 @@ const octree::Id find_deepest_root(const octree::IndexMap &index, const octree:: } // namespace octree::IndexedStorage open_path_indexed(const std::filesystem::path& path) { - if (std::filesystem::is_directory(path)) { - return octree::open_folder_indexed(path); - } else { - return octree::open_index(path).value(); + auto result = std::filesystem::is_directory(path) + ? octree::open_folder_indexed(path) + : octree::open_index(path); + if (!result.has_value()) { + LOG_ERROR_AND_EXIT( + "Failed to open dataset {}: {}", + path, + store::describe_error(result.error())); } + return std::move(result.value()); } std::string render_line_content(const DisplayEntry &entry) { @@ -220,7 +233,7 @@ ftxui::Element render_line(const DisplayEntry &entry, size_t base_level, bool is int run(const cli::Args &args) { const octree::IndexedStorage storage = open_path_indexed(args.dataset_path); - const octree::IndexMap &index = storage.index(); + const store::Index &index = storage.index(); const octree::Id root_id = args.full_view ? octree::Id::root() : find_deepest_root(index); TreeView tree_view(storage, root_id); diff --git a/src/sf_merger/NodeLoader.h b/src/sf_merger/NodeLoader.h index c6ac4027..c7c44b90 100644 --- a/src/sf_merger/NodeLoader.h +++ b/src/sf_merger/NodeLoader.h @@ -11,21 +11,18 @@ #include "octree/Id.h" #include "octree/NodeStatusOrMissing.h" #include "octree/Space.h" -#include "octree/storage/IndexedStorage.h" +#include "octree/Storage.h" #include "octree/storage/cache/Dummy.h" #include "octree/storage/cache/ICache.h" class NodeLoader { public: NodeLoader(const octree::IndexedStorage &storage, octree::cache::ICache &cache, octree::Space space) - : _storage(storage), _cache(cache), _space(space) { - if (storage.cache().has_value() && dynamic_cast *>(&cache) != nullptr) { - LOG_WARN("Backing storage for NodeLoader instance is cached, but another cache was provided leading to double caching."); - } - } + : _storage(storage), _cache(cache), _space(space) {} octree::NodeStatusOrMissing get_status(const octree::Id &id) const noexcept { - return this->_storage.index().get(id); + return octree::NodeStatusOrMissing( + DEBUG_ASSERT_VAL(this->_storage.index().get(id)).value()); } // Try to retrieve or reconstruct node mesh by ID diff --git a/src/sf_merger/NodeWriter.h b/src/sf_merger/NodeWriter.h index f160a8ad..6c5bf290 100644 --- a/src/sf_merger/NodeWriter.h +++ b/src/sf_merger/NodeWriter.h @@ -6,7 +6,7 @@ #include "mesh/SimpleMesh.h" #include "octree/Id.h" #include "octree/NodeStatus.h" -#include "octree/storage/Storage.h" +#include "octree/Storage.h" #include "octree/traverse.h" // TODO: make thread safe @@ -15,13 +15,13 @@ class NodeWriter { NodeWriter(octree::Storage &storage) : _storage(storage) {} bool has_node(const octree::Id &id) { - return this->_storage.has(id); + return DEBUG_ASSERT_VAL(this->_storage.has(id)).value(); } void write_node(const octree::Id &id, const SimpleMesh &mesh) { mesh::validate(mesh); DEBUG_ASSERT_VAL(this->_storage.save(id, mesh)); - auto p = this->_storage.path_for(id); + auto p = DEBUG_ASSERT_VAL(this->_storage.path_for(id)).value(); // change extension to .png p.replace_extension(".png"); cv::imwrite(p, mesh.texture.value_or(cv::Mat())); diff --git a/src/sf_merger/cut.h b/src/sf_merger/cut.h index 6f61d554..1e785618 100644 --- a/src/sf_merger/cut.h +++ b/src/sf_merger/cut.h @@ -7,12 +7,13 @@ #include "mask.h" #include "octree/Id.h" #include "octree/NodeStatus.h" -#include "octree/storage/IndexedStorage.h" +#include "octree/Storage.h" #include "octree/storage/open.h" #include "octree/Space.h" #include "octree/traverse.h" #include "utils.h" #include "containers/Cow.h" +#include "store/describe_error.h" struct Context { const octree::IndexedStorage& input; @@ -31,7 +32,8 @@ inline void cut_leaf_node( const octree::Id& id, const MeshMask& mask ) { - DEBUG_ASSERT(ctx.input.index().is(octree::NodeStatus::Leaf, id)); + DEBUG_ASSERT(DEBUG_ASSERT_VAL( + ctx.input.index().is(octree::NodeStatus::Leaf, id)).value()); const SimpleMesh mesh = DEBUG_ASSERT_VAL(ctx.input.load(id)).value(); LOG_TRACE("Cutting mesh at {} using mask with {} vertices and {} triangles", @@ -57,7 +59,8 @@ inline void cut_virtual_node( const octree::Id& id, const MeshMask& mask ) { - DEBUG_ASSERT(ctx.input.index().is(octree::NodeStatus::Virtual, id)); + DEBUG_ASSERT(DEBUG_ASSERT_VAL( + ctx.input.index().is(octree::NodeStatus::Virtual, id)).value()); DEBUG_ASSERT(id.has_children()); const auto children = id.children().value(); @@ -88,12 +91,13 @@ inline void cut_node( mesh::io::save_to_path(mask.mesh, new_path); */ - const auto status_opt = ctx.input.index().get(id); - if (!status_opt.has_value()) { + const auto status_result = ctx.input.index().get(id); + DEBUG_ASSERT(status_result.has_value()); + if (!status_result->has_value()) { return; } - const octree::NodeStatus status = status_opt.value(); + const octree::NodeStatus status = status_result->value(); switch (status) { case octree::NodeStatus::Virtual: cut_virtual_node(ctx, id, mask); @@ -116,7 +120,10 @@ inline void cut_dataset( cut_node(ctx, octree::Id::root(), mask); const auto index_result = output.save_or_create_index(); if (!index_result.has_value()) { - LOG_ERROR_AND_EXIT("Failed to save output index in {}: {}", output.base_path(), index_result.error()); + LOG_ERROR_AND_EXIT( + "Failed to save output index in {}: {}", + output.base_path(), + store::describe_error(index_result.error())); } } @@ -128,7 +135,17 @@ inline void cut_dataset( LOG_TRACE("Creating output dataset at {}", output_path); std::filesystem::create_directories(output_path); - octree::IndexedStorage output_dataset = octree::open_folder_indexed(output_path, octree::OpenOptions(octree::disk::layout::strategy::make_default(), ".glb")); + octree::OpenOptions options; + options.default_mapping = octree::store_layout::level_and_coordinate_directories(); + options.preferred_extension = ".glb"; + auto output_result = octree::open_folder_indexed(output_path, std::move(options)); + if (!output_result.has_value()) { + LOG_ERROR_AND_EXIT( + "Failed to open output dataset {}: {}", + output_path, + store::describe_error(output_result.error())); + } + octree::IndexedStorage output_dataset = std::move(output_result.value()); if (!output_dataset.index().empty()) { LOG_ERROR_AND_EXIT("Output dataset has to be empty."); } diff --git a/src/sf_merger/main.cpp b/src/sf_merger/main.cpp index 6ff4896a..76894680 100644 --- a/src/sf_merger/main.cpp +++ b/src/sf_merger/main.cpp @@ -11,6 +11,7 @@ #include "octree/Storage.h" #include "optional_utils.h" #include "earth.h" +#include "store/describe_error.h" std::optional load_mask_from_path(const std::filesystem::path& path) { if (std::filesystem::exists(path)) { @@ -34,14 +35,38 @@ std::optional load_mask_from_path(const std::filesystem::path& path) { void run(const cli::MergeArgs& args) { LOG_TRACE("Loading base dataset from {}", args.base_path); - octree::IndexedStorage base_dataset = octree::open_folder_indexed(args.base_path); + auto base_result = octree::open_folder_indexed(args.base_path); + if (!base_result.has_value()) { + LOG_ERROR_AND_EXIT( + "Failed to open base dataset {}: {}", + args.base_path, + store::describe_error(base_result.error())); + } + octree::IndexedStorage base_dataset = std::move(base_result.value()); LOG_TRACE("Loading new dataset from {}", args.new_path); - octree::IndexedStorage new_dataset = octree::open_folder_indexed(args.new_path); + auto new_result = octree::open_folder_indexed(args.new_path); + if (!new_result.has_value()) { + LOG_ERROR_AND_EXIT( + "Failed to open new dataset {}: {}", + args.new_path, + store::describe_error(new_result.error())); + } + octree::IndexedStorage new_dataset = std::move(new_result.value()); LOG_TRACE("Creating output dataset at {}", args.output_path); std::filesystem::create_directories(args.output_path); - octree::Storage output_dataset = octree::open_folder(args.output_path, false, octree::OpenOptions{.preferred_extension_with_dot = std::string(base_dataset.layout().extension_with_dot())}); + octree::OpenOptions options; + options.default_mapping = base_dataset.layout().mapping(); + options.preferred_extension = std::string(base_dataset.codec_selector().value_or(".terrain")); + auto output_result = octree::open_folder(args.output_path, false, std::move(options)); + if (!output_result.has_value()) { + LOG_ERROR_AND_EXIT( + "Failed to open output dataset {}: {}", + args.output_path, + store::describe_error(output_result.error())); + } + octree::Storage output_dataset = std::move(output_result.value()); std::optional mask = flatten(map(args.mask_path, load_mask_from_path)); @@ -50,7 +75,14 @@ void run(const cli::MergeArgs& args) { void run(const cli::CutArgs& args) { LOG_TRACE("Loading input dataset from {}", args.input_path); - const octree::IndexedStorage input_dataset = octree::open_folder_indexed(args.input_path); + auto input_result = octree::open_folder_indexed(args.input_path); + if (!input_result.has_value()) { + LOG_ERROR_AND_EXIT( + "Failed to open input dataset {}: {}", + args.input_path, + store::describe_error(input_result.error())); + } + const octree::IndexedStorage input_dataset = std::move(input_result.value()); const MeshMask mask = DEBUG_ASSERT_VAL(load_mask_from_path(args.mask_path)).value(); cut_dataset(input_dataset, mask, args.output_path, args.keep_inside); } diff --git a/src/sf_merger/merge.h b/src/sf_merger/merge.h index 180dbb59..b84fe38d 100644 --- a/src/sf_merger/merge.h +++ b/src/sf_merger/merge.h @@ -14,6 +14,7 @@ #include "octree/Id.h" #include "octree/NodeStatusOrMissing.h" #include "octree/storage/cache/Dummy.h" +#include "store/describe_error.h" inline std::string get_dataset_name(const octree::Storage &storage) { return storage.layout().base_path().filename().string(); @@ -179,6 +180,9 @@ inline void merge_datasets( const auto index_result = output_dataset.save_or_create_index(); if (!index_result.has_value()) { - LOG_ERROR_AND_EXIT("Failed to save output index in {}: {}", output_dataset.base_path(), index_result.error()); + LOG_ERROR_AND_EXIT( + "Failed to save output index in {}: {}", + output_dataset.base_path(), + store::describe_error(index_result.error())); } } diff --git a/src/terrainlib/CMakeLists.txt b/src/terrainlib/CMakeLists.txt index 4046109b..5ed36a37 100644 --- a/src/terrainlib/CMakeLists.txt +++ b/src/terrainlib/CMakeLists.txt @@ -36,7 +36,6 @@ add_library(terrainlib mesh/texture_trim.cpp mesh/validate.cpp - octree/storage/helpers.cpp octree/Space.cpp octree/OddLevelShifted.cpp diff --git a/src/terrainlib/mesh/codec/from_extension.h b/src/terrainlib/mesh/codec/from_extension.h new file mode 100644 index 00000000..abc18813 --- /dev/null +++ b/src/terrainlib/mesh/codec/from_extension.h @@ -0,0 +1,33 @@ +#pragma once + +#include +#include +#include + +#include + +#include "mesh/codec/Gltf.h" +#include "mesh/codec/Terrain.h" +#include "store/Codec.h" + +namespace mesh::codec { + +inline std::expected>, store::CodecError> +from_extension(const std::string_view extension) { + if (extension == ".terrain") { + return std::make_unique(); + } + if (extension == ".glb") { + return std::make_unique(GltfContainer::Binary); + } + if (extension == ".gltf") { + return std::make_unique(GltfContainer::Json); + } + return std::unexpected(store::CodecError{ + store::CodecOperation::Resolve, + store::CodecErrorCategory::UnsupportedCodec, + "unsupported mesh codec selector: " + std::string(extension), + }); +} + +} // namespace mesh::codec diff --git a/src/terrainlib/mesh/storage.h b/src/terrainlib/mesh/storage.h new file mode 100644 index 00000000..102f6afc --- /dev/null +++ b/src/terrainlib/mesh/storage.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include + +#include "mesh/SimpleMesh.h" +#include "mesh/codec/from_extension.h" +#include "octree/StoreTraits.h" +#include "octree/storage/IndexFile.h" +#include "octree/storage/open_runtime.h" +#include "store/IndexedStorage.h" + +namespace mesh::storage { + +using Storage = store::Storage; +using IndexedStorage = store::IndexedStorage; +using OpenOptions = octree::storage::OpenOptions; + +inline std::expected> open_index( + const std::filesystem::path &path) { + return store::open_index( + path, + octree::storage::index_format(), + mesh::codec::from_extension); +} + +inline std::expected> open_folder( + const std::filesystem::path &path, + const bool create_index = false, + OpenOptions options = {}) { + return octree::storage::open_folder( + path, + create_index, + ".terrain", + mesh::codec::from_extension, + std::move(options)); +} + +inline std::expected> open_folder_indexed( + const std::filesystem::path &path, + OpenOptions options = {}) { + return octree::storage::open_folder_indexed( + path, + ".terrain", + mesh::codec::from_extension, + std::move(options)); +} + +} // namespace mesh::storage diff --git a/src/terrainlib/octree/IndexMap.h b/src/terrainlib/octree/IndexMap.h index c231e763..c45d7da1 100644 --- a/src/terrainlib/octree/IndexMap.h +++ b/src/terrainlib/octree/IndexMap.h @@ -16,6 +16,9 @@ class IndexMap { using iterator = SharedIndex::iterator; using const_iterator = SharedIndex::const_iterator; + IndexMap() = default; + explicit IndexMap(SharedIndex index) : _index(std::move(index)) {} + std::optional get(const Id id) const { return _index.get(id).value(); } @@ -77,6 +80,9 @@ class IndexMap { const SharedIndex &shared() const { return _index; } + SharedIndex take_shared() && { + return std::move(_index); + } using serialize = zpp::bits::members<1>; friend zpp::bits::access; diff --git a/src/terrainlib/octree/Storage.h b/src/terrainlib/octree/Storage.h index e6776555..adb5181e 100644 --- a/src/terrainlib/octree/Storage.h +++ b/src/terrainlib/octree/Storage.h @@ -1,14 +1,13 @@ #pragma once -#include "octree/storage/IndexedStorage.h" -#include "octree/storage/MeshStorage.h" -#include "octree/storage/RawStorage.h" -#include "octree/storage/Storage.h" -#include "octree/storage/open.h" +#include "mesh/storage.h" namespace octree { +using MeshStorage = mesh::storage::Storage; +using IndexedMeshStorage = mesh::storage::IndexedStorage; using Storage = MeshStorage; using IndexedStorage = IndexedMeshStorage; +using StorageSettings = store::StorageSettings; -} +} // namespace octree diff --git a/src/terrainlib/octree/disk/Layout.h b/src/terrainlib/octree/disk/Layout.h deleted file mode 100644 index ba56f40b..00000000 --- a/src/terrainlib/octree/disk/Layout.h +++ /dev/null @@ -1,49 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -#include - -#include "octree/Id.h" -#include "octree/disk/layout/Strategy.h" - -namespace octree::disk { - -class Layout { -public: - Layout(std::filesystem::path base_path, std::unique_ptr strategy, std::string extension_with_dot) - : _base_path(std::move(base_path)), _strategy(std::move(strategy)), _extension_with_dot(std::move(extension_with_dot)) { - DEBUG_ASSERT(!_extension_with_dot.empty() && _extension_with_dot[0] == '.'); - } - - std::filesystem::path get_node_path(Id id) const { - return this->_base_path / this->_strategy->get_relative_node_path(id, this->_extension_with_dot); - } - std::optional get_id_from_node_path(const std::filesystem::path& path) const { - const std::filesystem::path relative_path = std::filesystem::relative(path, _base_path); - return this->_strategy->get_id_from_relative_node_path(relative_path); - } - - const std::filesystem::path& base_path() const { - return this->_base_path; - } - const layout::Strategy &strategy() const { - return *this->_strategy; - } - std::string_view extension_with_dot() const { - return this->_extension_with_dot; - } - std::string_view extension() const { - return this->extension_with_dot().substr(1); - } - -private: - std::filesystem::path _base_path; - std::unique_ptr _strategy; - std::string _extension_with_dot; -}; - -} diff --git a/src/terrainlib/octree/disk/layout/Strategy.h b/src/terrainlib/octree/disk/layout/Strategy.h deleted file mode 100644 index a1d5fb70..00000000 --- a/src/terrainlib/octree/disk/layout/Strategy.h +++ /dev/null @@ -1,19 +0,0 @@ -#pragma once - -#include -#include -#include - -#include "octree/Id.h" - -namespace octree::disk::layout { - -class Strategy { -public: - virtual ~Strategy() = default; - - virtual std::filesystem::path get_relative_node_path(Id id, std::string_view extension_with_dot) const = 0; - virtual std::optional get_id_from_relative_node_path(const std::filesystem::path &relative_path) const = 0; -}; - -} diff --git a/src/terrainlib/octree/disk/layout/StrategyRegister.h b/src/terrainlib/octree/disk/layout/StrategyRegister.h deleted file mode 100644 index 026a44f7..00000000 --- a/src/terrainlib/octree/disk/layout/StrategyRegister.h +++ /dev/null @@ -1,84 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -#include "octree/disk/layout/Strategy.h" -#include "log.h" - -namespace octree::disk::layout { - -class StrategyRegister { -public: - using Factory = std::function()>; - - static StrategyRegister &instance() { - static StrategyRegister inst; - return inst; - } - - template - void register_strategy(const std::string &id) { - this->_factories[id] = [] { return std::make_unique(); }; - this->_type_to_id[std::type_index(typeid(T))] = id; - } - - std::unique_ptr create(const std::string &id) const { - auto it = this->_factories.find(id); - if (it != this->_factories.end()) { - return (it->second)(); - } - return nullptr; - } - - const std::unordered_map &factories() const { - return this->_factories; - } - - template - std::string get_id() const { - return this->_get_id(typeid(T)); - } - - std::string get_id(const Strategy &s) const { - return this->_get_id(typeid(s)); - } - -private: - std::unordered_map _factories; - std::unordered_map _type_to_id; - - // Private ctor/dtor to enforce singleton - StrategyRegister() = default; - ~StrategyRegister() = default; - StrategyRegister(const StrategyRegister &) = delete; - StrategyRegister &operator=(const StrategyRegister &) = delete; - - std::string _get_id(std::type_index type_index) const { - auto it = this->_type_to_id.find(type_index); - if (it != this->_type_to_id.end()) { - return it->second; - } - LOG_ERROR_AND_EXIT("Encountered octree layout strategy not registered"); - } -}; - -} // namespace octree::disk::layout - -#include "octree/disk/layout/strategy/Flat.h" -#include "octree/disk/layout/strategy/LevelAndCoordinateDirectories.h" - -namespace { -template -struct Registrar { - Registrar(const std::string &id) { - octree::disk::layout::StrategyRegister::instance().register_strategy(id); - } -}; - -const Registrar reg_flat("flat"); -const Registrar reg_lacd("level_and_coordinate_directories"); -} diff --git a/src/terrainlib/octree/disk/layout/strategy/Default.h b/src/terrainlib/octree/disk/layout/strategy/Default.h deleted file mode 100644 index 45c7842d..00000000 --- a/src/terrainlib/octree/disk/layout/strategy/Default.h +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once - -#include - -#include "octree/disk/layout/strategy/LevelAndCoordinateDirectories.h" - -namespace octree::disk::layout::strategy { - -using Default = LevelAndCoordinateDirectories; - -inline std::unique_ptr make_default() { - return std::make_unique(); -} - -} // namespace octree::disk::layout::strategy diff --git a/src/terrainlib/octree/disk/layout/strategy/Flat.h b/src/terrainlib/octree/disk/layout/strategy/Flat.h deleted file mode 100644 index 0a11a941..00000000 --- a/src/terrainlib/octree/disk/layout/strategy/Flat.h +++ /dev/null @@ -1,50 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include "octree/Id.h" -#include "octree/disk/layout/Strategy.h" -#include "string_utils.h" - -namespace octree::disk::layout::strategy { - -class Flat : public Strategy { -public: - std::filesystem::path get_relative_node_path(Id id, std::string_view extension_with_dot) const override { - return fmt::format("{}-{}{}", id.level(), id.index_on_level(), extension_with_dot); - } - - std::optional get_id_from_relative_node_path(const std::filesystem::path &relative_path) const override { - DEBUG_ASSERT(relative_path.is_relative()); - - const std::string _filestem = relative_path.stem().string(); - const std::string_view filestem = _filestem; - - const size_t dash_pos = filestem.find('-'); - if (dash_pos == std::string::npos) { - return std::nullopt; - } - const std::string_view level_str = filestem.substr(0, dash_pos); - const std::string_view index_str = filestem.substr(dash_pos + 1); - - const auto level_opt = from_chars(level_str); - if (!level_opt) { - return std::nullopt; - } - - const auto index_opt = from_chars(index_str); - if (!index_opt) { - return std::nullopt; - } - - const Id::Level level = *level_opt; - const Id::Index index = *index_opt; - return Id::try_make(level, index); - } -}; - -} // namespace octree::disk::layout::strategy diff --git a/src/terrainlib/octree/disk/layout/strategy/LevelAndCoordinateDirectories.h b/src/terrainlib/octree/disk/layout/strategy/LevelAndCoordinateDirectories.h deleted file mode 100644 index 6a5ffa68..00000000 --- a/src/terrainlib/octree/disk/layout/strategy/LevelAndCoordinateDirectories.h +++ /dev/null @@ -1,60 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include "octree/Id.h" -#include "octree/disk/layout/Strategy.h" -#include "string_utils.h" - -namespace octree::disk::layout::strategy { - -class LevelAndCoordinateDirectories : public Strategy { -public: - std::filesystem::path get_relative_node_path(Id id, std::string_view extension_with_dot) const override { - const Id::Coords coords = id.coords(); - return fmt::format("{}/{}/{}/{}{}", id.level(), coords.x, coords.y, coords.z, extension_with_dot); - } - std::optional get_id_from_relative_node_path(const std::filesystem::path &relative_path) const override { - DEBUG_ASSERT(relative_path.is_relative()); - - auto it = relative_path.begin(); - if (std::distance(it, relative_path.end()) < 4) { - return std::nullopt; - } - - const auto level_opt = from_chars(it->native()); - if (!level_opt) { - return std::nullopt; - } - const Id::Level level = *level_opt; - ++it; - - const auto x_opt = from_chars(it->native()); - if (!x_opt) { - return std::nullopt; - } - const Id::Coord x = *x_opt; - ++it; - - const auto y_opt = from_chars(it->native()); - if (!y_opt) { - return std::nullopt; - } - const Id::Coord y = *y_opt; - ++it; - - const auto z_opt = from_chars(it->stem().native()); - if (!z_opt) { - return std::nullopt; - } - const Id::Coord z = *z_opt; - - return Id::try_make(level, {x, y, z}); - } -}; - -} // namespace octree::disk::layout::strategy diff --git a/src/terrainlib/octree/storage/CopyError.h b/src/terrainlib/octree/storage/CopyError.h deleted file mode 100644 index 7bd6c278..00000000 --- a/src/terrainlib/octree/storage/CopyError.h +++ /dev/null @@ -1,59 +0,0 @@ -#pragma once - -#include -#include - -namespace octree { - -enum class CopyErrorKind { - FileNotFound, - CreateLink, - CreateDirectories, - RemoveOld, - Read, - Write -}; - -class CopyError { -public: - constexpr CopyError(CopyErrorKind kind) - : kind(kind) {} - - operator CopyErrorKind() const { - return this->kind; - } - constexpr bool operator==(CopyError other) const { - return this->kind == other.kind; - } - constexpr bool operator!=(CopyError other) const { - return this->kind != other.kind; - } - - std::string description() const { - switch (kind) { - case CopyErrorKind::FileNotFound: - return "file not found"; - case CopyErrorKind::CreateLink: - return "cannot create hard link"; - case CopyErrorKind::CreateDirectories: - return "cannot create directories"; - case CopyErrorKind::RemoveOld: - return "cannot remove current file"; - case CopyErrorKind::Read: - return "cannot read file"; - case CopyErrorKind::Write: - return "cannot write file"; - default: - return "undefined error"; - } - } - - friend std::ostream &operator<<(std::ostream &os, const CopyError &err) { - return os << err.description(); - } - -private: - CopyErrorKind kind; -}; - -} // namespace octree diff --git a/src/terrainlib/octree/storage/IndexFile.h b/src/terrainlib/octree/storage/IndexFile.h new file mode 100644 index 00000000..7d70fbde --- /dev/null +++ b/src/terrainlib/octree/storage/IndexFile.h @@ -0,0 +1,68 @@ +#pragma once + +#include + +#include + +#include "io/serialize.h" +#include "octree/IndexMap.h" +#include "octree/StoreTraits.h" +#include "octree/disk/IndexFile.h" +#include "octree/store_layout/Mappings.h" +#include "store/IndexFormat.h" + +namespace octree::storage { + +inline std::expected, store::IndexFormatError> +read_index_file(const std::filesystem::path &path) { + auto result = io::read_from_path(path); + if (!result.has_value()) { + return std::unexpected(store::IndexFormatError{ + result.error() == io::Error(io::Error::OpenFile) + ? store::IndexFormatErrorCategory::Open + : store::IndexFormatErrorCategory::Malformed, + path, + fmt::format("{}", result.error()), + }); + } + disk::v1::IndexFile file = std::move(result.value()); + return store::IndexMetadata{ + std::move(file.map).take_shared(), + std::move(file.layout_strategy_id), + std::move(file.preferred_extension), + }; +} + +inline std::expected write_index_file( + const std::filesystem::path &path, + const store::IndexMetadata &metadata) { + disk::v1::IndexFile file; + file.layout_strategy_id = metadata.layout_id; + file.preferred_extension = metadata.codec_selector; + file.map = IndexMap(metadata.index); + const auto result = io::write_to_path(file, path); + if (!result.has_value()) { + return std::unexpected(store::IndexFormatError{ + store::IndexFormatErrorCategory::Write, + path, + fmt::format("{}", result.error()), + }); + } + return {}; +} + +inline store::PathMapping default_mapping() { + return store_layout::level_and_coordinate_directories(); +} + +inline store::IndexFormat index_format() { + return { + disk::v1::index_file_name(), + read_index_file, + write_index_file, + store_layout::from_id, + default_mapping, + }; +} + +} // namespace octree::storage diff --git a/src/terrainlib/octree/storage/IndexedStorage.h b/src/terrainlib/octree/storage/IndexedStorage.h deleted file mode 100644 index 3ba94e8f..00000000 --- a/src/terrainlib/octree/storage/IndexedStorage.h +++ /dev/null @@ -1,67 +0,0 @@ -#pragma once - -#include - -#include - -#include "octree/Id.h" -#include "octree/IndexMap.h" -#include "octree/disk/Layout.h" -#include "octree/storage/Storage.h" -#include "octree/storage/codec/Codec.h" -#include "octree/storage/defaults.h" -#include "octree/storage/helpers.h" - -namespace octree { - -template Codec = DefaultCodecFor> -class IndexedStorage_ : public Storage_ { -public: - explicit IndexedStorage_(Storage_ inner) noexcept - : Storage_(std::move(inner)) { - this->ensure_indexed(); - DEBUG_ASSERT(this->is_indexed()); - } - explicit IndexedStorage_(RawStorage_ inner, IndexMap map) noexcept - : Storage_(std::move(inner), std::move(map)) { - DEBUG_ASSERT(this->is_indexed()); - } - - IndexedStorage_ &operator=(const IndexedStorage_ &) = delete; - IndexedStorage_(const IndexedStorage_ &) = delete; - IndexedStorage_(IndexedStorage_ &&) = default; - IndexedStorage_ &operator=(IndexedStorage_ &&) = default; - - ~IndexedStorage_() override { - if (this->is_index_dirty()) { - LOG_WARN("Index was not saved upon IndexedStorage_ destruction, use save_index()."); - } - } - - const IndexMap& index() const noexcept { - DEBUG_ASSERT(this->is_indexed()); - return Storage_::index().value(); - } - void update_index() noexcept { - Storage_::update_index(); - } - std::expected save_index() const noexcept { - if (!this->is_index_dirty()) { - return {}; - } - - auto result = helpers::save_index_map(this->index(), this->layout()); - if (result.has_value()) { - this->set_index_dirty(false); - } - return result; - } - -private: - IndexMap& index_mut() noexcept { - DEBUG_ASSERT(this->is_indexed()); - return Storage_::index_mut().value(); - } -}; - -} // namespace octree diff --git a/src/terrainlib/octree/storage/MeshStorage.h b/src/terrainlib/octree/storage/MeshStorage.h index 46b69813..e2d8b585 100644 --- a/src/terrainlib/octree/storage/MeshStorage.h +++ b/src/terrainlib/octree/storage/MeshStorage.h @@ -1,12 +1,3 @@ #pragma once -#include "mesh/SimpleMesh.h" -#include "octree/storage/Storage.h" -#include "octree/storage/IndexedStorage.h" - -namespace octree { - -using MeshStorage = Storage_; -using IndexedMeshStorage = IndexedStorage_; - -} +#include "octree/Storage.h" diff --git a/src/terrainlib/octree/storage/RawStorage.h b/src/terrainlib/octree/storage/RawStorage.h deleted file mode 100644 index 39482250..00000000 --- a/src/terrainlib/octree/storage/RawStorage.h +++ /dev/null @@ -1,118 +0,0 @@ -#pragma once - -#include -#include - -#include - -#include "octree/Id.h" -#include "octree/disk/Layout.h" -#include "octree/storage/codec/Codec.h" -#include "octree/storage/CopyError.h" -#include "octree/storage/defaults.h" - -namespace octree { - -template Codec = DefaultCodecFor> -class RawStorage_ { -public: - using value_type = T; - using codec_type = Codec; - using load_error = typename Codec::load_error; - using save_error = typename Codec::save_error; - - explicit RawStorage_(disk::Layout layout) noexcept : _layout(std::move(layout)) {} - - ~RawStorage_() = default; - RawStorage_ &operator=(const RawStorage_ &) = delete; - RawStorage_(const RawStorage_ &) = delete; - RawStorage_(RawStorage_ &&) = default; - RawStorage_ &operator=(RawStorage_ &&) = default; - - std::expected load(const Id &id) const noexcept { - const auto path = this->path_for(id); - return Codec::load_from_path(path); - } - - std::expected save(const Id &id, const T &node) const noexcept { - const auto path = this->path_for(id); - return Codec::save_to_path(node, path); - } - - std::expected copy_to(const Id &id, RawStorage_ &target) const noexcept { - return target.copy_from(id, *this); - } - - std::expected copy_from(const Id &id, const RawStorage_ &source) noexcept { - if (!source.has(id)) { - return std::unexpected(CopyErrorKind::FileNotFound); - } - - const auto source_path = source.path_for(id); - const auto target_path = this->path_for(id); - if (source_path.extension() != target_path.extension()) { - // TODO: should this error instead? - const auto load_result = source.load(id); - if (!load_result.has_value()) { - return std::unexpected(CopyErrorKind::Read); - } - const value_type node = load_result.value(); - - const auto save_result = this->save(id, node); - if (!save_result.has_value()) { - return std::unexpected(CopyErrorKind::Write); - } - return {}; - } - - std::error_code ec; - if (std::filesystem::remove(target_path, ec)) { - if (ec) { - return std::unexpected(CopyErrorKind::RemoveOld); - } - } - - std::filesystem::create_directories(target_path.parent_path(), ec); - if (ec) { - return std::unexpected(CopyErrorKind::CreateDirectories); - } - - std::filesystem::create_hard_link(source_path, target_path, ec); - if (ec) { - return std::unexpected(CopyErrorKind::CreateLink); - } - - return {}; - } - - bool remove(const Id &id) const noexcept { - const auto path = this->path_for(id); - std::error_code ec; - const auto removed = std::filesystem::remove(path, ec); - return !ec && removed; - } - - bool has(const Id &id) const noexcept { - const auto path = this->path_for(id); - std::error_code ec; - const auto exists = std::filesystem::exists(path, ec); - return !ec && exists; - } - - std::filesystem::path path_for(const Id &id) const noexcept { - return this->_layout.get_node_path(id); - } - - std::filesystem::path base_path() const noexcept { - return this->_layout.base_path(); - } - - const disk::Layout &layout() const noexcept { - return this->_layout; - } - -private: - disk::Layout _layout; -}; - -} // namespace octree diff --git a/src/terrainlib/octree/storage/Storage.h b/src/terrainlib/octree/storage/Storage.h deleted file mode 100644 index 782a9c03..00000000 --- a/src/terrainlib/octree/storage/Storage.h +++ /dev/null @@ -1,312 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include - -#include - -#include "mesh/io.h" -#include "octree/Id.h" -#include "octree/IndexMap.h" -#include "octree/disk/Layout.h" -#include "octree/disk/layout/strategy/Default.h" -#include "octree/NodeStatusOrMissing.h" -#include "octree/storage/RawStorage.h" -#include "octree/storage/cache/ICache.h" -#include "octree/storage/codec/Codec.h" -#include "octree/storage/defaults.h" -#include "octree/storage/helpers.h" - -namespace octree { - -namespace detail { -struct MaybeIndex { - std::optional map; - mutable bool dirty = false; - - explicit MaybeIndex() : map(std::nullopt) {} - explicit MaybeIndex(IndexMap map) : map(std::move(map)) {} - - MaybeIndex(const MaybeIndex &) = default; - MaybeIndex &operator=(const MaybeIndex &) = default; - - MaybeIndex(MaybeIndex &&other) noexcept - : map(std::move(other.map)), dirty(std::exchange(other.dirty, false)) { - other.map.reset(); - } - - MaybeIndex &operator=(MaybeIndex &&other) noexcept { - this->map = std::move(other.map); - this->dirty = std::exchange(other.dirty, false); - other.map.reset(); - return *this; - } - - bool add(const Id &id) noexcept { - if (!map.has_value()) { - return false; - } - - const bool changed = map->add(id); - dirty = dirty || changed; - return changed; - } - - bool remove(const Id &id) noexcept { - if (!map.has_value()) { - return false; - } - - const bool changed = map->remove(id); - dirty = dirty || changed; - return changed; - } - - bool contains(const Id &id, const bool def = false) const noexcept { - if (this->map.has_value()) { - const octree::NodeStatusOrMissing status = this->map->get(id); - return status == octree::NodeStatusOrMissing::Leaf || status == NodeStatusOrMissing::Inner; - } - return def; - } -}; - -template -struct MaybeCache : public cache::ICache { - std::optional>> cache; - - explicit MaybeCache() : cache(std::nullopt) {} - explicit MaybeCache(std::unique_ptr> cache) : cache(std::move(cache)) {} - - std::optional get(const Id& id) noexcept override { - if (this->cache.has_value()) { - return this->cache->get()->get(id); - } - return std::nullopt; - } - - bool put(const Id &id, const T &value) noexcept override { - if (this->cache.has_value()) { - return this->cache->get()->put(id, value); - } - return false; - } - - bool remove(const Id &id) noexcept override { - if (this->cache.has_value()) { - return this->cache->get()->remove(id); - } - return false; - } - - bool contains(const Id &id) const noexcept override { - if (this->cache.has_value()) { - return this->cache->get()->contains(id); - } - return false; - } -}; -} - -struct StorageSettings { - bool allow_overwrite = false; -}; - -template Codec = DefaultCodecFor> -class Storage_ { -public: - using value_type = T; - using codec_type = Codec; - using load_error = typename Codec::load_error; - using save_error = typename Codec::save_error; - - explicit Storage_(RawStorage_ inner) - : _inner(std::move(inner)) {} - explicit Storage_(RawStorage_ inner, IndexMap index) - : _inner(std::move(inner)), _index(detail::MaybeIndex(std::move(index))) {} - - Storage_ &operator=(const Storage_ &) = delete; - Storage_(const Storage_ &) = delete; - Storage_(Storage_ &&) = default; - Storage_ &operator=(Storage_ &&) = default; - - virtual ~Storage_() { - if (this->_index.map.has_value() && this->_index.dirty) { - auto result = helpers::save_index_map(this->_index.map.value(), this->_inner.layout()); - if (!result.has_value()) { - LOG_ERROR("Failed to automatically save index when closing storage: {}", result.error()); - } - } - } - - std::expected load(const Id &id) const noexcept { - if (const auto value_opt = this->_cache.get(id)) { - return value_opt.value(); - } - - if (!this->_index.contains(id, true)) { - return std::unexpected(Codec::file_not_found()); - } - - const auto result = this->_inner.load(id); - if (result.has_value()) { - this->_cache.put(id, result.value()); - } - return result; - } - - std::expected save(const Id &id, const value_type &value) noexcept { - if (this->check_overwrite(id)) { - LOG_ERROR_AND_EXIT("tried to overwrite value when not allowed"); - } - - const auto result = this->_inner.save(id, value); - if (result.has_value()) { - this->_cache.put(id, value); - this->_index.add(id); - } - return result; - } - - std::expected copy_from(const Id &id, const Storage_ &source) noexcept { - if (!source._index.contains(id, true)) { - return std::unexpected(CopyErrorKind::FileNotFound); - } - - if (this->check_overwrite(id)) { - // TODO: proper error - LOG_ERROR_AND_EXIT("tried to overwrite value when not allowed"); - } - - const auto result = this->_inner.copy_from(id, source._inner); - if (result.has_value()) { - this->_index.add(id); - } - return result; - } - - std::expected copy_to(const Id &id, Storage_ &target) const noexcept { - return target.copy_from(id, *this); - } - - bool remove(const Id &id) noexcept { - this->_cache.remove(id); - this->_index.remove(id); - return this->_inner.remove(id); - } - - bool has(const Id &id) const noexcept { - if (this->is_indexed()) { - return this->_index.contains(id, false); - } else { - return this->_inner.has(id); - } - } - - std::filesystem::path path_for(const Id &id) const noexcept { - return this->_inner.path_for(id); - } - - std::filesystem::path base_path() const noexcept { - return this->_inner.base_path(); - } - - bool is_indexed() const noexcept { - return this->_index.map.has_value(); - } - - void ensure_indexed() noexcept { - if (this->is_indexed()) { - return; - } - - LOG_TRACE("Index not present, creating empty index"); - IndexMap map; - helpers::update_index_map(map, this->_inner.layout()); - LOG_TRACE("Index created with {} entries", map.size()); - this->_index.map = std::move(map); - } - - std::optional> index() const noexcept { - // I miss Option::as_ref - if (this->_index.map.has_value()) { - return this->_index.map.value(); - } else { - return std::nullopt; - } - } - - std::optional>> &cache() noexcept { - return this->_cache.cache; - } - - std::optional>> cache() const noexcept { - if (this->_cache.cache.has_value()) { - return *this->_cache.cache.value(); - } else { - return std::nullopt; - } - } - - std::expected save_or_create_index() noexcept { - if (this->is_indexed() && !this->_index.dirty) { - return {}; - } - - this->ensure_indexed(); - - auto result = helpers::save_index_map(this->index().value(), this->_inner.layout()); - if (result.has_value()) { - this->_index.dirty = false; - } - return result; - } - - const octree::disk::Layout &layout() const noexcept { - return this->_inner.layout(); - } - - const StorageSettings& settings() const noexcept { - return this->_settings; - } - StorageSettings& settings() noexcept { - return this->_settings; - } - -protected: - bool check_overwrite(const Id &id) noexcept { - return !this->_settings.allow_overwrite && this->has(id); - } - - std::optional &index_mut() noexcept { - return this->_index.map; - } - - bool is_index_dirty() const noexcept { - return this->_index.dirty; - } - void set_index_dirty(const bool val = true) const noexcept { - this->_index.dirty = val; - } - - void update_index() noexcept { - if (!this->is_indexed()) { - return; - } - - helpers::update_index_map(this->index_mut().value(), this->_inner.layout()); - this->_index.dirty = false; - } - -private: - RawStorage_ _inner; - detail::MaybeIndex _index = detail::MaybeIndex(); - mutable detail::MaybeCache _cache = detail::MaybeCache(); - StorageSettings _settings = {}; -}; - -} // namespace octree diff --git a/src/terrainlib/octree/storage/cache/Dummy.h b/src/terrainlib/octree/storage/cache/Dummy.h index ecf6cff8..159bc32d 100644 --- a/src/terrainlib/octree/storage/cache/Dummy.h +++ b/src/terrainlib/octree/storage/cache/Dummy.h @@ -1,27 +1,11 @@ #pragma once -#include - -#include "octree/Id.h" -#include "octree/storage/cache/ICache.h" +#include "octree/StoreTraits.h" +#include "store/cache/Dummy.h" namespace octree::cache { - template -class Dummy : public ICache { -public: - std::optional get(const Id &) noexcept override { - return std::nullopt; - } - bool put(const Id &, const T &) noexcept override { - return false; - } - bool remove(const Id &) noexcept override { - return false; - } - bool contains(const Id &) const noexcept override { - return false; - } -}; +template +using Dummy = store::cache::Dummy; -} +} // namespace octree::cache diff --git a/src/terrainlib/octree/storage/cache/ICache.h b/src/terrainlib/octree/storage/cache/ICache.h index bd0258bc..b14bd0b0 100644 --- a/src/terrainlib/octree/storage/cache/ICache.h +++ b/src/terrainlib/octree/storage/cache/ICache.h @@ -1,20 +1,11 @@ #pragma once -#include - -#include "octree/Id.h" +#include "octree/StoreTraits.h" +#include "store/cache/Interface.h" namespace octree::cache { -template -class ICache { -public: - virtual ~ICache() = default; +template +using ICache = store::cache::Interface; - virtual std::optional get(const Id& id) noexcept = 0; - virtual bool put(const Id &id, const T &value) noexcept = 0; - virtual bool remove(const Id &id) noexcept = 0; - virtual bool contains(const Id &id) const noexcept = 0; -}; - -} +} // namespace octree::cache diff --git a/src/terrainlib/octree/storage/cache/LruCache.h b/src/terrainlib/octree/storage/cache/LruCache.h index 1c705e05..167f19d2 100644 --- a/src/terrainlib/octree/storage/cache/LruCache.h +++ b/src/terrainlib/octree/storage/cache/LruCache.h @@ -1,71 +1,11 @@ #pragma once -#include -#include -#include - -#include "octree/Id.h" -#include "octree/RawStorage.h" -#include "octree/cache/ICache.h" +#include "octree/StoreTraits.h" +#include "store/cache/Lru.h" namespace octree::cache { -// TODO: UNTESTED -// Do not use this cache for NodeLoader's ancestor lookup until cached ancestors -// are clipped to the requested node bounds. NodeLoader currently returns a -// cached ancestor unchanged, producing incorrect geometry for child requests. -template -class Lru_ : public ICache { -public: - explicit Lru_(const size_t capacity) : _capacity(capacity) {} - - std::optional get(const Key& key) { - auto it = this->_map.find(key); - if (it == this->_map.end()) return std::nullopt; - - // Move to front (most recently used) - this->_usage.splice(this->_usage.begin(), this->_usage, it->second.second); - return it->second.first; - } - - void put(const Key& key, const Value& value) { - auto it = this->_map.find(key); - if (it != this->_map.end()) { - // Update value and move to front - it->second.first = value; - this->_usage.splice(this->_usage.begin(), this->_usage, it->second.second); - } else { - // Insert new entry - if (this->_map.size() == _capacity) { - // Evict least recently used - const Key& lru_key = this->_usage.back(); - this->_map.erase(lru_key); - this->_usage.pop_back(); - } - - this->_usage.push_front(key); - this->_map[key] = { value, this->_usage.begin() }; - } - } - - void remove(const Key& key) { - auto it = this->_map.find(key); - if (it != this->_map.end()) { - this->_usage.erase(it->second.second); - this->_map.erase(it); - } - } - - bool contains(const Key& key) const { - return this->_map.contains(key); - } - -private: - size_t _capacity; - std::list _usage; - std::unordered_map::iterator>> _map; -}; - -using Lru = Lru; +template +using Lru = store::cache::Lru; -} +} // namespace octree::cache diff --git a/src/terrainlib/octree/storage/codec/Codec.h b/src/terrainlib/octree/storage/codec/Codec.h deleted file mode 100644 index f31b6f45..00000000 --- a/src/terrainlib/octree/storage/codec/Codec.h +++ /dev/null @@ -1,32 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -template -concept CodecFor = - requires(const std::filesystem::path &path, const T &value) { - typename Codec::value_type; - typename Codec::load_error; - typename Codec::save_error; - - requires std::same_as; - - { Codec::load_from_path(path) } noexcept - -> std::same_as>; - - { Codec::save_to_path(value, path) } noexcept - -> std::same_as>; - - { Codec::file_not_found() } noexcept - -> std::same_as; - - { Codec::supported_extensions() } -> std::ranges::range; - - requires std::convertible_to< - std::ranges::range_value_t, - std::string_view>; - }; diff --git a/src/terrainlib/octree/storage/codec/MeshCodec.h b/src/terrainlib/octree/storage/codec/MeshCodec.h deleted file mode 100644 index 7a15400a..00000000 --- a/src/terrainlib/octree/storage/codec/MeshCodec.h +++ /dev/null @@ -1,35 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include "mesh/io.h" - -namespace octree { - -struct MeshCodec { - using value_type = mesh::Simple; - using load_error = mesh::io::LoadMeshError; - using save_error = mesh::io::SaveMeshError; - - static std::expected load_from_path(const std::filesystem::path& path) noexcept { - return mesh::io::load_from_path(path); - } - - static std::expected save_to_path(const value_type& value, const std::filesystem::path& path) noexcept { - return mesh::io::save_to_path(value, path); - } - - static load_error file_not_found() noexcept { - return mesh::io::LoadMeshErrorKind::FileNotFound; - } - - static std::vector supported_extensions() { - return {".terrain", ".glb", ".gltf"}; - } -}; - -} // namespace octree diff --git a/src/terrainlib/octree/storage/codec/ReadOnlyCodec.h b/src/terrainlib/octree/storage/codec/ReadOnlyCodec.h deleted file mode 100644 index 3ea68ba9..00000000 --- a/src/terrainlib/octree/storage/codec/ReadOnlyCodec.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include -#include - -#include "log.h" - -namespace octree { - -// Wraps a codec so saving always fails loudly instead of writing, for storages that must stay -// read-only (e.g. a view that aliases another storage's files). -template -struct ReadOnlyCodec : Codec { - static std::expected save_to_path( - const typename Codec::value_type &, const std::filesystem::path &) noexcept { - LOG_ERROR_AND_EXIT("Attempted to write through a read-only codec"); - } -}; - -} // namespace octree diff --git a/src/terrainlib/octree/storage/codec/ZppBitsCodec.h b/src/terrainlib/octree/storage/codec/ZppBitsCodec.h deleted file mode 100644 index dc4acd5a..00000000 --- a/src/terrainlib/octree/storage/codec/ZppBitsCodec.h +++ /dev/null @@ -1,38 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include "Codec.h" -#include "io/serialize.h" -#include "io/Error.h" - -namespace octree { - -template -struct ZppBitsCodec { - using value_type = T; - using load_error = io::Error; - using save_error = io::Error; - - static std::expected load_from_path(const std::filesystem::path& path) noexcept { - return io::read_from_path(path); - } - - static std::expected save_to_path(const value_type& value, const std::filesystem::path& path) noexcept { - return io::write_to_path(value, path); - } - - static load_error file_not_found() noexcept { - return io::Error::Value::OpenFile; - } - - static std::vector supported_extensions() { - return {".bin"}; - } -}; - -} // namespace octree diff --git a/src/terrainlib/octree/storage/defaults.h b/src/terrainlib/octree/storage/defaults.h deleted file mode 100644 index fb4981c4..00000000 --- a/src/terrainlib/octree/storage/defaults.h +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once - -#include "mesh/SimpleMesh.h" -#include "codec/Codec.h" -#include "codec/MeshCodec.h" -#include "codec/ZppBitsCodec.h" - -namespace octree { - -using DefaultT = mesh::Simple; - -template -struct DefaultCodec { - using type = ZppBitsCodec; -}; - -template <> -struct DefaultCodec { - using type = MeshCodec; -}; - -template -using DefaultCodecFor = typename DefaultCodec>::type; - -} // namespace octree \ No newline at end of file diff --git a/src/terrainlib/octree/storage/helpers.cpp b/src/terrainlib/octree/storage/helpers.cpp deleted file mode 100644 index ed817515..00000000 --- a/src/terrainlib/octree/storage/helpers.cpp +++ /dev/null @@ -1,136 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include - -#include "octree/storage/helpers.h" -#include "io/serialize.h" -#include "octree/NodeStatus.h" -#include "octree/IndexMap.h" -#include "octree/disk/IndexFile.h" -#include "octree/disk/Layout.h" -#include "octree/disk/layout/Strategy.h" -#include "octree/disk/layout/StrategyRegister.h" - -namespace octree::helpers { -namespace { -template -std::optional find_max_key(const std::unordered_map& map) { - if (map.empty()) { - return std::nullopt; - } - - auto max_it = map.begin(); - for (auto it = std::next(max_it); it != map.end(); it++) { - if (it->second > max_it->second) { - max_it = it; - } - } - - return max_it->first; -} -} - -std::optional guess_layout_strategy( - const std::filesystem::path &base_path, - size_t max_files_to_check) { - - if (!std::filesystem::is_directory(base_path)) { - return std::nullopt; - } - - std::vector candidate_paths; - std::unordered_map extension_counters; - for (const auto &entry : std::filesystem::recursive_directory_iterator(base_path)) { - if (max_files_to_check == 0) { - break; - } - if (!entry.is_regular_file() && !entry.is_symlink()) { - continue; - } - - const auto ext = entry.path().extension(); - if (ext == disk::v1::index_extension()) { - continue; - } - extension_counters[ext]++; - - candidate_paths.emplace_back(std::filesystem::relative(entry.path(), base_path)); - max_files_to_check--; - } - std::optional most_common_ext = find_max_key(extension_counters); - - std::unique_ptr best_strategy; - size_t best_match_count = 0; - const auto &factories = disk::layout::StrategyRegister::instance().factories(); - for (const auto &[_, make_strategy] : factories) { - auto strategy = make_strategy(); // assume returns unique_ptr - - size_t match_count = 0; - for (const auto &rel_path : candidate_paths) { - if (strategy->get_id_from_relative_node_path(rel_path).has_value()) { - match_count++; - } - } - - if (match_count > best_match_count) { - best_match_count = match_count; - best_strategy = std::move(strategy); - } - } - - if (best_strategy && most_common_ext.has_value()) { - return LayoutWithoutBase(std::move(best_strategy), most_common_ext.value()); - } - - return std::nullopt; -} - -std::expected save_index_map(const IndexMap& index, const disk::Layout& layout) { - const auto index_path = layout.base_path() / disk::v1::index_file_name(); - LOG_TRACE("Saving octree storage index to {}", index_path); - - disk::v1::IndexFile index_file; - index_file.map = index; - index_file.preferred_extension = layout.extension_with_dot(); - index_file.layout_strategy_id = disk::layout::StrategyRegister::instance().get_id(layout.strategy()); - - const auto result = io::write_to_path(index_file, index_path); - if (!result.has_value()) { - LOG_ERROR("Failed to save octree storage index to {}", index_path); - return std::unexpected(result.error()); - } - return {}; -} - -void update_index_map(IndexMap &index, const disk::Layout &layout) { - index.clear(); - const auto &base_path = layout.base_path(); - - std::filesystem::create_directories(base_path); - - for (const auto &entry : std::filesystem::recursive_directory_iterator(base_path)) { - if (!entry.is_regular_file() && !entry.is_symlink()) { - continue; - } - - const auto ext = entry.path().extension(); - if (ext == disk::v1::index_extension()) { - continue; - } - - auto id_opt = layout.get_id_from_node_path(entry.path()); - if (!id_opt) { - continue; - } - - const Id id = *id_opt; - index.add(id); - } -} - -} diff --git a/src/terrainlib/octree/storage/helpers.h b/src/terrainlib/octree/storage/helpers.h deleted file mode 100644 index 288096cf..00000000 --- a/src/terrainlib/octree/storage/helpers.h +++ /dev/null @@ -1,28 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include "io/Error.h" -#include "octree/IndexMap.h" -#include "octree/disk/IndexFile.h" -#include "octree/disk/Layout.h" -#include "octree/disk/layout/Strategy.h" - -namespace octree::helpers { - -struct LayoutWithoutBase { - std::unique_ptr strategy; - std::string extension_with_dot; -}; - -std::optional guess_layout_strategy( - const std::filesystem::path &base_path, - size_t max_files_to_check = 100); -std::expected save_index_map(const IndexMap &index, const disk::Layout &layout); -void update_index_map(IndexMap &index, const disk::Layout &layout); - -} diff --git a/src/terrainlib/octree/storage/open.h b/src/terrainlib/octree/storage/open.h index b21ede08..b8f1a6d0 100644 --- a/src/terrainlib/octree/storage/open.h +++ b/src/terrainlib/octree/storage/open.h @@ -1,37 +1,31 @@ #pragma once #include -#include -#include +#include -#include - -#include "io/Error.h" -#include "octree/disk/layout/strategy/Default.h" -#include "octree/storage/IndexedStorage.h" -#include "octree/storage/Storage.h" -#include "octree/storage/codec/Codec.h" -#include "octree/storage/defaults.h" +#include "mesh/storage.h" +#include "octree/Storage.h" namespace octree { -struct OpenOptions { - std::unique_ptr default_layout_strategy = {}; - std::optional preferred_extension_with_dot = {}; -}; +using OpenOptions = mesh::storage::OpenOptions; + +inline std::expected> open_index( + const std::filesystem::path &path) { + return mesh::storage::open_index(path); +} -template Codec = DefaultCodecFor> -std::expected, io::Error> open_index(const std::filesystem::path &index_path); -template Codec = DefaultCodecFor> -Storage_ open_folder( - const std::filesystem::path &base_path, +inline std::expected> open_folder( + const std::filesystem::path &path, const bool create_index = false, - OpenOptions options = {}); -template Codec = DefaultCodecFor> -IndexedStorage_ open_folder_indexed( - const std::filesystem::path &base_path, - OpenOptions options = {}); + OpenOptions options = {}) { + return mesh::storage::open_folder(path, create_index, std::move(options)); } -#include "open.inl" +inline std::expected> open_folder_indexed( + const std::filesystem::path &path, + OpenOptions options = {}) { + return mesh::storage::open_folder_indexed(path, std::move(options)); +} +} // namespace octree diff --git a/src/terrainlib/octree/storage/open.inl b/src/terrainlib/octree/storage/open.inl deleted file mode 100644 index c60c3699..00000000 --- a/src/terrainlib/octree/storage/open.inl +++ /dev/null @@ -1,111 +0,0 @@ -#include -#include -#include -#include -#include - -#include - -#include "io/serialize.h" -#include "log.h" -#include "io/Error.h" -#include "octree/disk/IndexFile.h" -#include "octree/disk/layout/StrategyRegister.h" -#include "octree/storage/IndexedStorage.h" -#include "octree/storage/helpers.h" -#include "octree/storage/defaults.h" -#include "range_utils.h" - -namespace octree { - -template Codec> -std::expected, io::Error> open_index(const std::filesystem::path &index_path) { - LOG_TRACE("Opening storage index {}", index_path); - - const auto result = io::read_from_path(index_path); - if (!result.has_value()) { - LOG_TRACE("Failed to open storage index due to {}", result.error()); - return std::unexpected(result.error()); - } - auto index_file = result.value(); - LOG_TRACE("Successfully read storage index with {} entries.", index_file.map.size()); - auto index_map = std::move(index_file.map); - const auto base_path = index_path.parent_path(); - auto layout_strategy = disk::layout::StrategyRegister::instance().create(index_file.layout_strategy_id); - disk::Layout layout(base_path, std::move(layout_strategy), index_file.preferred_extension); - RawStorage_ raw_storage(std::move(layout)); - return IndexedStorage_(std::move(raw_storage), std::move(index_map)); -} - -template Codec> -Storage_ open_folder( - const std::filesystem::path &base_path, - bool create_index, - OpenOptions options) { - LOG_TRACE("Opening storage folder {}", base_path); - - if (!std::filesystem::is_directory(base_path)) { - if (std::filesystem::exists(base_path)) { - LOG_ERROR_AND_EXIT("Base path {} exists but is not a directory", base_path); - } - - LOG_TRACE("Base path {} does not exist, creating it", base_path); - std::filesystem::create_directories(base_path); - } - - const std::filesystem::path index_path = base_path / disk::v1::index_file_name(); - auto storage_opt = open_index(index_path); - if (storage_opt.has_value()) { - LOG_TRACE("Loaded existing index"); - return Storage_(std::move(storage_opt.value())); - } - - auto layout_info_opt = helpers::guess_layout_strategy(base_path); - if (layout_info_opt.has_value()) { - const octree::helpers::LayoutWithoutBase &layout_info = layout_info_opt.value(); - LOG_TRACE("Guessed layout strategy of dataset as {} and extension as {}", - disk::layout::StrategyRegister::instance().get_id(*layout_info.strategy), - layout_info.extension_with_dot); - } else { - auto default_layout_strategy = std::move(options.default_layout_strategy); - if (!default_layout_strategy) { - default_layout_strategy = disk::layout::strategy::make_default(); - } - std::string default_extension_with_dot; - const std::vector supported_extensions = Codec::supported_extensions(); - if (options.preferred_extension_with_dot.has_value()) { - ASSERT(contains(supported_extensions, options.preferred_extension_with_dot.value())); - default_extension_with_dot = options.preferred_extension_with_dot.value(); - } else { - default_extension_with_dot = supported_extensions[0]; - } - LOG_WARN("Unable to determine layout of dataset, using layout strategy {} and extension {}", - disk::layout::StrategyRegister::instance().get_id(*default_layout_strategy), - default_extension_with_dot); - layout_info_opt = octree::helpers::LayoutWithoutBase(std::move(default_layout_strategy), default_extension_with_dot); - } - - disk::Layout layout(base_path, std::move(layout_info_opt->strategy), layout_info_opt->extension_with_dot); - if (!create_index) { - return Storage_(RawStorage_(std::move(layout))); - } - - IndexMap map; - helpers::update_index_map(map, layout); - if (!map.empty()) { - const auto save_result = helpers::save_index_map(map, layout); - if (!save_result.has_value()) { - LOG_ERROR_AND_EXIT("Failed to create storage index in {}: {}", base_path, save_result.error()); - } - } - return Storage_(RawStorage_(std::move(layout)), std::move(map)); -} - -template Codec> -IndexedStorage_ open_folder_indexed( - const std::filesystem::path &base_path, - OpenOptions options) { - return IndexedStorage_(open_folder(base_path, true, std::move(options))); -} - -} // namespace octree diff --git a/src/terrainlib/octree/storage/open_runtime.h b/src/terrainlib/octree/storage/open_runtime.h new file mode 100644 index 00000000..64070609 --- /dev/null +++ b/src/terrainlib/octree/storage/open_runtime.h @@ -0,0 +1,196 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "octree/StoreTraits.h" +#include "octree/storage/IndexFile.h" +#include "store/open.h" + +namespace octree::storage { + +struct OpenOptions { + std::optional> default_mapping; + std::optional preferred_extension; +}; + +struct DiscoveredLayout { + store::PathMapping mapping; + std::string codec_selector; +}; + +template +std::optional discover_layout( + const std::filesystem::path &base_path, + CodecResolver &resolve_codec) { + struct Candidate { + std::filesystem::path path; + std::string extension; + }; + std::vector candidates; + std::unordered_map extension_counts; + std::error_code error; + for (std::filesystem::recursive_directory_iterator iterator(base_path, error), end; + !error && iterator != end; + iterator.increment(error)) { + if (!iterator->is_regular_file(error) && !iterator->is_symlink(error)) { + if (error) { + break; + } + continue; + } + if (iterator->path().filename() == disk::v1::index_file_name()) { + continue; + } + const std::string extension = iterator->path().extension().string(); + if (extension.empty() || !resolve_codec(extension).has_value()) { + continue; + } + candidates.push_back({iterator->path(), extension}); + ++extension_counts[extension]; + } + if (error || candidates.empty()) { + return std::nullopt; + } + + std::optional> best_mapping; + size_t best_match_count = 0; + for (const auto mapping : store_layout::all()) { + const store::Layout layout(base_path, mapping); + size_t matches = 0; + for (const Candidate &candidate : candidates) { + std::filesystem::path node_path = candidate.path; + node_path.replace_extension(); + matches += layout.key_from_node_path(store::NodePath(node_path)).has_value(); + } + if (matches > best_match_count) { + best_mapping = mapping; + best_match_count = matches; + } + } + if (!best_mapping.has_value()) { + return std::nullopt; + } + + const auto extension = std::max_element( + extension_counts.begin(), + extension_counts.end(), + [](const auto &left, const auto &right) { return left.second < right.second; }); + return DiscoveredLayout{best_mapping.value(), extension->first}; +} + +template +std::expected, store::OpenError> open_folder( + const std::filesystem::path &base_path, + const bool create_index, + std::string default_codec_selector, + CodecResolver resolve_codec, + OpenOptions options = {}) { + std::error_code error; + const bool base_exists = std::filesystem::exists(base_path, error); + if (error) { + return std::unexpected(store::OpenError(store::FilesystemError{ + base_path, + "exists", + error, + })); + } + if (base_exists && !std::filesystem::is_directory(base_path, error)) { + return std::unexpected(store::OpenError(store::FilesystemError{ + base_path, + "is_directory", + error ? error : std::make_error_code(std::errc::not_a_directory), + })); + } + if (!base_exists) { + std::filesystem::create_directories(base_path, error); + if (error) { + return std::unexpected(store::OpenError(store::FilesystemError{ + base_path, + "create_directories", + error, + })); + } + } + + const auto format = index_format(); + const std::filesystem::path index_path = base_path / format.index_filename; + const bool index_exists = std::filesystem::exists(index_path, error); + if (error) { + return std::unexpected(store::OpenError(store::FilesystemError{ + index_path, + "exists", + error, + })); + } + if (index_exists) { + auto indexed = store::open_index( + index_path, + format, + resolve_codec); + if (!indexed.has_value()) { + return std::unexpected(indexed.error()); + } + return store::Storage(std::move(indexed.value())); + } + + const auto discovered = discover_layout(base_path, resolve_codec); + const store::PathMapping mapping = discovered.has_value() + ? discovered->mapping + : options.default_mapping.value_or(format.default_mapping()); + std::string codec_selector = discovered.has_value() + ? discovered->codec_selector + : options.preferred_extension.value_or(std::move(default_codec_selector)); + auto codec = resolve_codec(codec_selector); + if (!codec.has_value()) { + return std::unexpected(store::OpenError(codec.error())); + } + + auto storage_result = store::make_storage( + base_path, + format, + mapping, + std::move(codec_selector), + std::move(codec.value()), + false); + if (!storage_result.has_value()) { + return std::unexpected(storage_result.error()); + } + auto storage = std::move(storage_result.value()); + if (create_index) { + const auto index_result = storage.save_or_create_index(); + if (!index_result.has_value()) { + return std::unexpected(store::OpenError(index_result.error())); + } + } + return storage; +} + +template +std::expected, store::OpenError> +open_folder_indexed( + const std::filesystem::path &base_path, + std::string default_codec_selector, + CodecResolver resolve_codec, + OpenOptions options = {}) { + auto result = open_folder( + base_path, + true, + std::move(default_codec_selector), + std::move(resolve_codec), + std::move(options)); + if (!result.has_value()) { + return std::unexpected(result.error()); + } + return store::IndexedStorage(std::move(result.value())); +} + +} // namespace octree::storage diff --git a/src/terrainlib/octree/traverse.h b/src/terrainlib/octree/traverse.h index 6d0af8db..cc5b9435 100644 --- a/src/terrainlib/octree/traverse.h +++ b/src/terrainlib/octree/traverse.h @@ -6,6 +6,7 @@ #include #include "octree/IndexMap.h" +#include "octree/StoreTraits.h" #include "store/traverse.h" namespace octree { @@ -32,4 +33,20 @@ void traverse( DEBUG_ASSERT(result.has_value()); } +template> +void traverse( + const store::Index &index, + VisitFn &&visit, + RefineFn &&refine = always_refine, + const Id &root = Id::root(), + const TraversalOrder order = TraversalOrder::DepthFirst) { + const auto result = store::traverse( + index, + std::forward(visit), + std::forward(refine), + root, + order); + DEBUG_ASSERT(result.has_value()); +} + } // namespace octree diff --git a/src/terrainlib/store/OpenError.h b/src/terrainlib/store/OpenError.h new file mode 100644 index 00000000..2622b8a5 --- /dev/null +++ b/src/terrainlib/store/OpenError.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + +#include "store/IndexFormat.h" +#include "store/StorageError.h" + +namespace store { + +struct UnknownLayout { + std::string id; + bool operator==(const UnknownLayout &) const = default; +}; + +template +using OpenError = std::variant< + IndexFormatError, + FilesystemError, + UnknownLayout, + CodecError, + InvalidKey>; + +} // namespace store diff --git a/src/terrainlib/store/Storage.h b/src/terrainlib/store/Storage.h index fa99b6a4..e3cf3a85 100644 --- a/src/terrainlib/store/Storage.h +++ b/src/terrainlib/store/Storage.h @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include #include @@ -18,7 +20,10 @@ template class Storage { public: using Key = typename Traits::Key; + using key_type = Key; using value_type = NodeData; + using load_error = LoadError; + using save_error = SaveError; struct Persistence { IndexFormat format; @@ -29,6 +34,9 @@ class Storage { explicit Storage(RawStorage raw) : _raw(std::move(raw)) {} + Storage(RawStorage raw, Persistence persistence) + : _raw(std::move(raw)), _persistence(std::move(persistence)) {} + Storage( RawStorage raw, Index index, @@ -195,6 +203,17 @@ class Storage { return _raw.paths(key); } + std::expected> path_for( + const Key &key) const { + const auto node_paths = paths(key); + if (!node_paths.has_value()) { + return std::unexpected(node_paths.error()); + } + return node_paths->empty() + ? _raw.layout().node_path(key).path() + : node_paths->front(); + } + const std::filesystem::path &base_path() const { return _raw.layout().base_path(); } @@ -204,6 +223,12 @@ class Storage { const Codec &codec() const { return _raw.codec(); } + std::optional codec_selector() const { + if (!_persistence.has_value()) { + return std::nullopt; + } + return _persistence->codec_selector; + } bool is_indexed() const { return _index.has_value(); @@ -247,6 +272,18 @@ class Storage { return result; } + std::expected save_or_create_index() { + if (!_index.has_value()) { + const auto scan_result = scan_index(); + if (!scan_result.has_value()) { + return std::unexpected(scan_result.error()); + } + _index = std::move(scan_result.value()); + _dirty = true; + } + return save_index(); + } + const StorageSettings &settings() const { return _settings; } @@ -269,6 +306,85 @@ class Storage { } private: + std::expected, IndexFormatError> scan_index() const { + std::error_code error; + std::filesystem::create_directories(base_path(), error); + if (error) { + return std::unexpected(IndexFormatError{ + IndexFormatErrorCategory::Open, + base_path(), + error.message(), + }); + } + + const NodePath probe("__codec_probe__/node"); + const std::string probe_text = probe.path().generic_string(); + std::vector suffixes; + for (const auto &codec_path : _raw.codec().paths(probe)) { + const std::string text = codec_path.generic_string(); + if (text.starts_with(probe_text)) { + suffixes.push_back(text.substr(probe_text.size())); + } + } + + std::unordered_set candidates; + for (std::filesystem::recursive_directory_iterator iterator(base_path(), error), end; + !error && iterator != end; + iterator.increment(error)) { + if (!iterator->is_regular_file(error) && !iterator->is_symlink(error)) { + if (error) { + break; + } + continue; + } + const std::string path_text = iterator->path().generic_string(); + for (const std::string &suffix : suffixes) { + if (path_text.size() < suffix.size() + || !path_text.ends_with(suffix)) { + continue; + } + const NodePath node_path( + std::filesystem::path(path_text.substr(0, path_text.size() - suffix.size()))); + const auto key = _raw.layout().key_from_node_path(node_path); + if (key.has_value() && Traits::is_valid(key.value())) { + candidates.insert(key.value()); + } + } + } + if (error) { + return std::unexpected(IndexFormatError{ + IndexFormatErrorCategory::Open, + base_path(), + error.message(), + }); + } + + Index result; + for (const Key &key : candidates) { + const auto present = _raw.has(key); + if (!present.has_value()) { + const FilesystemError &filesystem_error = + std::get(present.error()); + return std::unexpected(IndexFormatError{ + IndexFormatErrorCategory::Open, + filesystem_error.path, + filesystem_error.error.message(), + }); + } + if (present.value()) { + const auto added = result.add(key); + if (!added.has_value()) { + return std::unexpected(IndexFormatError{ + IndexFormatErrorCategory::Malformed, + base_path(), + "codec path resolved to an invalid hierarchy key", + }); + } + } + } + return result; + } + void finalize_displaced_state() noexcept { if (!_dirty || !_index.has_value()) { return; diff --git a/src/terrainlib/store/cache/Dummy.h b/src/terrainlib/store/cache/Dummy.h new file mode 100644 index 00000000..bcb7c7e7 --- /dev/null +++ b/src/terrainlib/store/cache/Dummy.h @@ -0,0 +1,25 @@ +#pragma once + +#include "store/cache/Interface.h" + +namespace store::cache { + +template +class Dummy final : public Interface { +public: + using Key = typename Traits::Key; + std::optional get(const Key &) noexcept override { + return std::nullopt; + } + bool put(const Key &, const NodeData &) noexcept override { + return false; + } + bool remove(const Key &) noexcept override { + return false; + } + bool contains(const Key &) const noexcept override { + return false; + } +}; + +} // namespace store::cache diff --git a/src/terrainlib/store/cache/Interface.h b/src/terrainlib/store/cache/Interface.h new file mode 100644 index 00000000..4ffd070d --- /dev/null +++ b/src/terrainlib/store/cache/Interface.h @@ -0,0 +1,21 @@ +#pragma once + +#include + +#include "store/Traits.h" + +namespace store::cache { + +template +class Interface { +public: + using Key = typename Traits::Key; + virtual ~Interface() = default; + + virtual std::optional get(const Key &key) noexcept = 0; + virtual bool put(const Key &key, const NodeData &value) noexcept = 0; + virtual bool remove(const Key &key) noexcept = 0; + virtual bool contains(const Key &key) const noexcept = 0; +}; + +} // namespace store::cache diff --git a/src/terrainlib/store/cache/Lru.h b/src/terrainlib/store/cache/Lru.h new file mode 100644 index 00000000..0e8d8abf --- /dev/null +++ b/src/terrainlib/store/cache/Lru.h @@ -0,0 +1,73 @@ +#pragma once + +#include +#include +#include + +#include "store/cache/Interface.h" + +namespace store::cache { + +template +class Lru final : public Interface { +public: + using Key = typename Traits::Key; + + explicit Lru(const size_t capacity) : _capacity(capacity) {} + + std::optional get(const Key &key) noexcept override { + const auto iterator = _values.find(key); + if (iterator == _values.end()) { + return std::nullopt; + } + _usage.splice(_usage.begin(), _usage, iterator->second.second); + return iterator->second.first; + } + + bool put(const Key &key, const NodeData &value) noexcept override { + const auto iterator = _values.find(key); + if (iterator != _values.end()) { + iterator->second.first = value; + _usage.splice(_usage.begin(), _usage, iterator->second.second); + return false; + } + if (_capacity == 0) { + return false; + } + if (_values.size() == _capacity) { + _values.erase(_usage.back()); + _usage.pop_back(); + } + _usage.push_front(key); + _values.emplace(key, std::pair::iterator>{ + value, + _usage.begin(), + }); + return true; + } + + bool remove(const Key &key) noexcept override { + const auto iterator = _values.find(key); + if (iterator == _values.end()) { + return false; + } + _usage.erase(iterator->second.second); + _values.erase(iterator); + return true; + } + + bool contains(const Key &key) const noexcept override { + return _values.contains(key); + } + +private: + size_t _capacity; + std::list _usage; + std::unordered_map< + Key, + std::pair::iterator>, + typename Traits::Hasher> + _values; +}; + +} // namespace store::cache diff --git a/src/terrainlib/store/describe_error.h b/src/terrainlib/store/describe_error.h new file mode 100644 index 00000000..aa61d2a1 --- /dev/null +++ b/src/terrainlib/store/describe_error.h @@ -0,0 +1,50 @@ +#pragma once + +#include +#include + +#include "store/IndexFormat.h" +#include "store/OpenError.h" + +namespace store { + +inline std::string describe_error(const CodecError &error) { + return error.message; +} + +inline std::string describe_error(const IndexFormatError &error) { + return error.path.empty() + ? error.message + : error.path.string() + ": " + error.message; +} + +inline std::string describe_error(const FilesystemError &error) { + return error.operation + " " + error.path.string() + ": " + error.error.message(); +} + +inline std::string describe_error(const UnknownLayout &error) { + return "unknown layout: " + error.id; +} + +inline std::string describe_error(const AlreadyExists &error) { + return "path already exists: " + error.path.string(); +} + +template +std::string describe_error(const InvalidKey &) { + return "invalid hierarchy key"; +} + +template +std::string describe_error(const MissingSource &) { + return "source node is missing"; +} + +template +std::string describe_error(const std::variant &error) { + return std::visit( + [](const auto &value) { return describe_error(value); }, + error); +} + +} // namespace store diff --git a/src/terrainlib/store/open.h b/src/terrainlib/store/open.h new file mode 100644 index 00000000..70bf3647 --- /dev/null +++ b/src/terrainlib/store/open.h @@ -0,0 +1,93 @@ +#pragma once + +#include +#include +#include + +#include + +#include "store/IndexedStorage.h" +#include "store/OpenError.h" + +namespace store { + +template +std::expected, OpenError> open_index( + const std::filesystem::path &index_path, + const IndexFormat format, + CodecResolver &&resolve_codec) { + using Key = typename Traits::Key; + const auto metadata_result = format.read(index_path); + if (!metadata_result.has_value()) { + return std::unexpected(OpenError(metadata_result.error())); + } + IndexMetadata metadata = std::move(metadata_result.value()); + for (const auto &[key, status] : metadata.index) { + static_cast(status); + if (!Traits::is_valid(key)) { + return std::unexpected(OpenError(InvalidKey{key})); + } + } + + const auto mapping = format.mapping_from_id(metadata.layout_id); + if (!mapping.has_value()) { + return std::unexpected(OpenError(UnknownLayout{metadata.layout_id})); + } + auto codec = resolve_codec(metadata.codec_selector); + if (!codec.has_value()) { + return std::unexpected(OpenError(codec.error())); + } + + const std::filesystem::path base_path = index_path.parent_path(); + using Storage = store::Storage; + return IndexedStorage( + RawStorage( + Layout(base_path, mapping.value()), + std::move(codec.value())), + std::move(metadata.index), + typename Storage::Persistence{ + format, + index_path, + std::move(metadata.layout_id), + std::move(metadata.codec_selector), + }); +} + +template +std::expected, OpenError> make_storage( + const std::filesystem::path &base_path, + const IndexFormat format, + const PathMapping mapping, + std::string codec_selector, + std::unique_ptr> codec, + const bool indexed) { + using Key = typename Traits::Key; + std::error_code error; + std::filesystem::create_directories(base_path, error); + if (error) { + return std::unexpected(OpenError(FilesystemError{ + base_path, + "create_directories", + error, + })); + } + + typename Storage::Persistence persistence{ + format, + base_path / format.index_filename, + std::string(mapping.id), + std::move(codec_selector), + }; + RawStorage raw( + Layout(base_path, mapping), + std::move(codec)); + if (indexed) { + return Storage( + std::move(raw), + Index{}, + std::move(persistence)); + } + return Storage(std::move(raw), std::move(persistence)); +} + +} // namespace store diff --git a/unittests/dag_builder/storage_compatibility.cpp b/unittests/dag_builder/storage_compatibility.cpp index 1b53b9fa..45a3d351 100644 --- a/unittests/dag_builder/storage_compatibility.cpp +++ b/unittests/dag_builder/storage_compatibility.cpp @@ -13,6 +13,7 @@ #include "octree/traverse.h" #include "storage.h" #include "store/codec/ZppBits.h" +#include "thread_safe_storage.h" namespace { @@ -63,7 +64,9 @@ TEST_CASE("pre-refactor DAG fixture preserves index and payload contracts") { }); CHECK(visited == std::vector{root, parent, leaf}); - const auto batch_storage = octree::open_folder_indexed(path); + auto batch_storage_result = dag::storage::open_folder_indexed(path); + REQUIRE(batch_storage_result.has_value()); + const auto batch_storage = std::move(batch_storage_result.value()); const auto batch = batch_storage.load(root); REQUIRE(batch.has_value()); CHECK(batch->metadata.group_assignment == std::vector{0}); @@ -73,7 +76,9 @@ TEST_CASE("pre-refactor DAG fixture preserves index and payload contracts") { REQUIRE(batch->clustering.clusters.size() == 1); CHECK(batch->clustering.clusters.front().id == 17); - const auto metadata_storage = octree::open_folder_indexed(path); + auto metadata_storage_result = dag::storage::open_metadata_indexed(path); + REQUIRE(metadata_storage_result.has_value()); + const auto metadata_storage = std::move(metadata_storage_result.value()); const auto metadata = metadata_storage.load(root); REQUIRE(metadata.has_value()); CHECK(metadata->group_assignment == batch->metadata.group_assignment); @@ -126,3 +131,46 @@ TEST_CASE("runtime DAG ZPP Bits codec is reentrant") { REQUIRE(operation.get()); } } + +TEST_CASE("DAG resolvers expose writable batches and read-only metadata", "[store][open]") { + const auto batch_codec = dag::codec::from_extension(".bin"); + REQUIRE(batch_codec.has_value()); + CHECK(batch_codec.value()->paths(store::NodePath("node")) + == std::vector{"node.bin"}); + + const auto metadata_codec = dag::codec::metadata_from_extension(".bin"); + REQUIRE(metadata_codec.has_value()); + const auto write_result = metadata_codec.value()->write( + store::NodePath("node"), + dag::NodeMetadata{}); + REQUIRE_FALSE(write_result.has_value()); + CHECK(write_result.error().operation == store::CodecOperation::Write); + CHECK(write_result.error().category == store::CodecErrorCategory::UnsupportedOperation); + + const auto unknown = dag::codec::from_extension(".unknown"); + REQUIRE_FALSE(unknown.has_value()); + CHECK(unknown.error().category == store::CodecErrorCategory::UnsupportedCodec); +} + +TEST_CASE("DAG indexed storage survives ThreadSafeStorage move and release", "[store][storage]") { + const std::filesystem::path fixture = + std::filesystem::path(ALP_TEST_DATA_DIR) + / "raster-store-refactor/dag/0/0/0/0.bin"; + const auto batch = io::read_from_path(fixture); + REQUIRE(batch.has_value()); + + TemporaryDirectory output; + auto storage_result = dag::storage::open_folder_indexed(output.path()); + REQUIRE(storage_result.has_value()); + dag::ThreadSafeStorage synchronized(std::move(storage_result.value())); + REQUIRE(synchronized.save(octree::Id::root(), batch.value()).has_value()); + + auto released = std::move(synchronized).release(); + REQUIRE(released.save_index().has_value()); + CHECK(std::filesystem::is_regular_file(output.path() / "terrain.index")); + + auto reopened_result = dag::storage::open_folder_indexed(output.path()); + REQUIRE(reopened_result.has_value()); + CHECK(reopened_result->has(octree::Id::root()).value()); + CHECK(reopened_result->load(octree::Id::root()).has_value()); +} diff --git a/unittests/terrainlib/store_compatibility.cpp b/unittests/terrainlib/store_compatibility.cpp index 84e28a85..40a3f61f 100644 --- a/unittests/terrainlib/store_compatibility.cpp +++ b/unittests/terrainlib/store_compatibility.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -8,12 +9,14 @@ #include #include "io/serialize.h" +#include "io/bytes.h" #include "mesh/SimpleMesh.h" #include "octree/Storage.h" #include "octree/disk/IndexFile.h" -#include "octree/disk/layout/strategy/Flat.h" -#include "octree/disk/layout/strategy/LevelAndCoordinateDirectories.h" -#include "octree/storage/RawStorage.h" +#include "mesh/codec/from_extension.h" +#include "mesh/storage.h" +#include "octree/store_layout/Mappings.h" +#include "octree/storage/IndexFile.h" #include "octree/storage/open.h" #include "octree/traverse.h" @@ -55,10 +58,21 @@ mesh::Simple triangle_mesh(const double offset) { octree::IndexedStorage make_storage( const std::filesystem::path &path, - std::unique_ptr strategy, + const store::PathMapping mapping, const std::string &extension) { - octree::disk::Layout layout(path, std::move(strategy), extension); - return octree::IndexedStorage(octree::RawStorage_(std::move(layout)), octree::IndexMap{}); + auto codec = mesh::codec::from_extension(extension); + REQUIRE(codec.has_value()); + return octree::IndexedStorage( + store::RawStorage( + store::Layout(path, mapping), + std::move(codec.value())), + store::Index{}, + octree::Storage::Persistence{ + octree::storage::index_format(), + path / "terrain.index", + std::string(mapping.id), + extension, + }); } } // namespace @@ -76,7 +90,9 @@ TEST_CASE("pre-refactor SF fixtures preserve index and path contracts") { CHECK(index_file->map.is(octree::NodeStatus::Leaf, root)); CHECK(std::filesystem::is_regular_file(path / "0-0.terrain")); - const octree::IndexedStorage storage = octree::open_folder_indexed(path); + auto storage_result = octree::open_folder_indexed(path); + REQUIRE(storage_result.has_value()); + const octree::IndexedStorage storage = std::move(storage_result.value()); const auto mesh = storage.load(root); REQUIRE(mesh.has_value()); CHECK(mesh->face_count() == 1); @@ -114,14 +130,14 @@ TEST_CASE("pre-refactor layouts round-trip boundary IDs") { octree::Id(octree::Id::max_level(), octree::Id::max_index_on_level(octree::Id::max_level())), }; - octree::disk::layout::strategy::Flat flat; - octree::disk::layout::strategy::LevelAndCoordinateDirectories coordinates; + const auto flat = octree::store_layout::flat(); + const auto coordinates = octree::store_layout::level_and_coordinate_directories(); for (const octree::Id id : ids) { - const auto flat_path = flat.get_relative_node_path(id, ".terrain"); - CHECK(flat.get_id_from_relative_node_path(flat_path) == id); + const auto flat_path = flat.key_to_node_path(id); + CHECK(flat.node_path_to_key(flat_path) == id); - const auto coordinate_path = coordinates.get_relative_node_path(id, ".terrain"); - CHECK(coordinates.get_id_from_relative_node_path(coordinate_path) == id); + const auto coordinate_path = coordinates.key_to_node_path(id); + CHECK(coordinates.node_path_to_key(coordinate_path) == id); } } @@ -130,17 +146,17 @@ TEST_CASE("pre-refactor storage hard-links matching payload formats") { TemporaryDirectory target_directory("hard-link-target"); auto source = make_storage( source_directory.path(), - std::make_unique(), + octree::store_layout::flat(), ".terrain"); auto target = make_storage( target_directory.path(), - std::make_unique(), + octree::store_layout::flat(), ".terrain"); const octree::Id root = octree::Id::root(); REQUIRE(source.save(root, triangle_mesh(0.0)).has_value()); REQUIRE(target.copy_from(root, source).has_value()); - CHECK(std::filesystem::equivalent(source.path_for(root), target.path_for(root))); + CHECK(std::filesystem::equivalent(source.path_for(root).value(), target.path_for(root).value())); REQUIRE(source.save_index().has_value()); REQUIRE(target.save_index().has_value()); } @@ -150,17 +166,19 @@ TEST_CASE("pre-refactor storage re-encodes differing payload formats") { TemporaryDirectory target_directory("reencode-target"); auto source = make_storage( source_directory.path(), - std::make_unique(), + octree::store_layout::flat(), ".terrain"); auto target = make_storage( target_directory.path(), - std::make_unique(), + octree::store_layout::flat(), ".glb"); const octree::Id root = octree::Id::root(); REQUIRE(source.save(root, triangle_mesh(4.0)).has_value()); REQUIRE(target.copy_from(root, source).has_value()); - CHECK_FALSE(std::filesystem::equivalent(source.path_for(root), target.path_for(root))); + CHECK_FALSE(std::filesystem::equivalent( + source.path_for(root).value(), + target.path_for(root).value())); const auto loaded = target.load(root); REQUIRE(loaded.has_value()); CHECK(loaded->face_count() == 1); @@ -172,7 +190,7 @@ TEST_CASE("pre-refactor storage supports enabled overwrites") { TemporaryDirectory directory("overwrite"); auto storage = make_storage( directory.path(), - std::make_unique(), + octree::store_layout::flat(), ".terrain"); storage.settings().allow_overwrite = true; @@ -189,24 +207,75 @@ TEST_CASE("pre-refactor storage supports enabled overwrites") { TEST_CASE("pre-refactor folder opening scans payloads and creates an index") { TemporaryDirectory directory("folder-open"); octree::OpenOptions options; - options.default_layout_strategy = std::make_unique(); - options.preferred_extension_with_dot = ".terrain"; + options.default_mapping = octree::store_layout::flat(); + options.preferred_extension = ".terrain"; { - auto storage = octree::open_folder(directory.path(), false, std::move(options)); + auto storage_result = octree::open_folder(directory.path(), false, std::move(options)); + REQUIRE(storage_result.has_value()); + auto storage = std::move(storage_result.value()); CHECK_FALSE(storage.is_indexed()); REQUIRE(storage.save(octree::Id::root(), triangle_mesh(2.0)).has_value()); CHECK_FALSE(std::filesystem::exists(directory.path() / "terrain.index")); } { - const auto storage = octree::open_folder_indexed(directory.path()); + auto storage_result = octree::open_folder_indexed(directory.path()); + REQUIRE(storage_result.has_value()); + const auto storage = std::move(storage_result.value()); CHECK(storage.is_indexed()); - CHECK(storage.has(octree::Id::root())); + CHECK(storage.has(octree::Id::root()).value()); CHECK(std::filesystem::is_regular_file(directory.path() / "terrain.index")); } - const auto reopened = octree::open_index<>(directory.path() / "terrain.index"); + const auto reopened = octree::open_index(directory.path() / "terrain.index"); REQUIRE(reopened.has_value()); - CHECK(reopened->has(octree::Id::root())); + CHECK(reopened->has(octree::Id::root()).value()); +} + +TEST_CASE("mesh codec resolver dispatches every supported legacy selector", "[store][open]") { + for (const std::string extension : {".terrain", ".glb", ".gltf"}) { + const auto codec = mesh::codec::from_extension(extension); + REQUIRE(codec.has_value()); + const auto paths = codec.value()->paths(store::NodePath("node")); + REQUIRE_FALSE(paths.empty()); + CHECK(paths.front().extension() == extension); + } + + const auto unknown = mesh::codec::from_extension(".unknown"); + REQUIRE_FALSE(unknown.has_value()); + CHECK(unknown.error().category == store::CodecErrorCategory::UnsupportedCodec); + CHECK(unknown.error().operation == store::CodecOperation::Resolve); +} + +TEST_CASE("octree opening retains index failures without directory fallback", "[store][open]") { + TemporaryDirectory directory("open-errors"); + const std::filesystem::path index_path = directory.path() / "terrain.index"; + + octree::disk::v1::IndexFile index_file; + index_file.layout_strategy_id = "unknown-layout"; + index_file.preferred_extension = ".terrain"; + REQUIRE(io::write_to_path(index_file, index_path).has_value()); + + const auto unknown_layout = octree::open_folder_indexed(directory.path()); + REQUIRE_FALSE(unknown_layout.has_value()); + REQUIRE(std::holds_alternative(unknown_layout.error())); + CHECK(std::get(unknown_layout.error()).id == "unknown-layout"); + + index_file.layout_strategy_id = "flat"; + index_file.preferred_extension = ".unknown"; + REQUIRE(io::write_to_path(index_file, index_path).has_value()); + const auto unknown_codec = octree::open_folder_indexed(directory.path()); + REQUIRE_FALSE(unknown_codec.has_value()); + REQUIRE(std::holds_alternative(unknown_codec.error())); + CHECK(std::get(unknown_codec.error()).category + == store::CodecErrorCategory::UnsupportedCodec); + + const std::array malformed_bytes{0xff, 0x00, 0x01}; + REQUIRE(io::write_bytes_to_path(malformed_bytes, index_path).has_value()); + const auto malformed = octree::open_folder_indexed(directory.path()); + REQUIRE_FALSE(malformed.has_value()); + REQUIRE(std::holds_alternative(malformed.error())); + CHECK(std::get(malformed.error()).category + == store::IndexFormatErrorCategory::Malformed); } From 30863369c92be2967867a6144a9784b36cf882b7 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Fri, 14 Aug 2026 17:29:56 +0200 Subject: [PATCH 57/61] Harden SF storage boundaries --- docs/raster-store/refactor-status.md | 45 ++- src/dag_builder/build.cpp | 16 +- src/dag_builder/build.h | 6 +- src/dag_builder/main.cpp | 11 +- src/sf_builder/main.cpp | 9 +- src/sf_builder/terrainbuilder.cpp | 15 +- src/sf_builder/terrainbuilder.h | 7 +- src/sf_merger/NodeWriter.h | 45 ++- src/sf_merger/cut.h | 78 ++++-- src/sf_merger/main.cpp | 19 +- src/sf_merger/merge.h | 86 ++++-- src/terrainlib/sf/Error.h | 42 +++ src/terrainlib/sf/InvalidTopology.h | 13 + src/terrainlib/sf/finalize_storage.h | 28 ++ src/terrainlib/sf/validate_index.h | 21 ++ src/terrainlib/store/RawStorage.h | 25 ++ src/terrainlib/store/Storage.h | 10 +- unittests/CMakeLists.txt | 11 + .../dag_builder/storage_compatibility.cpp | 34 +++ unittests/sf_builder/storage_finalization.cpp | 80 ++++++ unittests/sf_merger/storage_boundaries.cpp | 196 +++++++++++++ unittests/terrainlib/sf_validate_index.cpp | 24 ++ unittests/terrainlib/store_storage.cpp | 265 +++++++++++++++++- 23 files changed, 998 insertions(+), 88 deletions(-) create mode 100644 src/terrainlib/sf/Error.h create mode 100644 src/terrainlib/sf/InvalidTopology.h create mode 100644 src/terrainlib/sf/finalize_storage.h create mode 100644 src/terrainlib/sf/validate_index.h create mode 100644 unittests/sf_builder/storage_finalization.cpp create mode 100644 unittests/sf_merger/storage_boundaries.cpp create mode 100644 unittests/terrainlib/sf_validate_index.cpp diff --git a/docs/raster-store/refactor-status.md b/docs/raster-store/refactor-status.md index 1bfd3b4a..132225d9 100644 --- a/docs/raster-store/refactor-status.md +++ b/docs/raster-store/refactor-status.md @@ -16,10 +16,10 @@ This file is the resumable implementation log for ## Current position -- Phase: 3 — Generalize storage and index lifecycle +- Phase: 4 — Harden node reuse and enforce SF topology - State: complete; completion commit pending. -- Next action: commit and tag Phase 3, then begin Phase 4 copy hardening and - SF topology validation. +- Next action: commit and tag Phase 4, then begin Phase 5 cleanup and + documentation. ## Phase log @@ -91,6 +91,29 @@ This file is the resumable implementation log for index no-fallback, and DAG synchronized move/release coverage. - State: complete. +### Phase 4 — Harden node reuse and enforce SF topology + +- Hardened `copy_from()` around the fixed codec probe path: matching path + lists hard-link every physical file, differing lists decode/re-encode, and + actual path-count mismatches return a typed codec error. +- Moved overwrite index removal to the mutation boundary. Missing-source and + decode failures leave the old logical target indexed; encode and partial + multi-file hard-link failures leave it unindexed for safety. +- Added one-file and multi-file hard-link, re-encode, overwrite, + missing-source, decode, encode, and partial-hard-link failure coverage. +- Added `sf::InvalidTopology`, `sf::validate_index()`, typed SF processing and + finalization errors, and an SF finalization boundary that writes the index + before validating it. +- Applied SF validation before SF merge/cut and DAG processing and after SF + builder/merger output finalization. The diagnostic index browser remains + exempt. +- Propagated SF merger save, copy, open, index, and validation failures through + `std::expected` to the command-line boundary. +- Added SF builder, SF merger, SF cut, and DAG builder boundary tests, + including invalid-input rejection, inspectable invalid output, unchanged + hard links, and newly written changed/clipped nodes. +- State: complete. + ## Verification log - Baseline `unittests_terrainlib`: 23,676 assertions in 391 test cases passed. @@ -130,6 +153,20 @@ This file is the resumable implementation log for - Phase 3 all-application configuration built `sf-builder`, `sf-merger`, `sf-index-browser`, `dag-builder`, and `dag-convert-debug` successfully. - Phase 3 `git diff --check`: passed. +- Phase 4 focused storage-copy tests: 70 assertions in 5 test cases passed. +- Phase 4 SF validator tests: 6 assertions in 2 test cases passed. +- Phase 4 DAG SF-input boundary test: 12 assertions passed. +- Phase 4 SF merger boundary tests: 66 assertions in 4 test cases passed. +- Phase 4 SF builder finalization tests: 19 assertions in 2 test cases passed. +- Phase 4 `unittests_terrainlib`: 24,049 assertions in 429 test cases passed. +- Phase 4 `unittests_dagbuilder`: 471 assertions in 74 test cases passed. +- Phase 4 `unittests_sfmerger`: 121 assertions in 8 test cases passed. +- Phase 4 all-application configuration built `sf-builder`, `sf-merger`, + `sf-index-browser`, `dag-builder`, and `dag-convert-debug` successfully. +- The pre-existing `unittests_sfbuilder` compile failure involving removed + `radix::tile::Scheme` remains isolated; the new finalization tests build and + pass in `unittests_sfbuilder_finalization`. +- Phase 4 `git diff --check`: passed. ## Commit and tag log @@ -145,3 +182,5 @@ This file is the resumable implementation log for - `fbc02d0` / `refactor_phase_2` — runtime layouts/codecs and Phase 2 completion tag. - `61acb74` — Phase 3 shared runtime-codec storage core. +- `4ffcdea` / `refactor_phase_3` — Phase 3 application migration and + completion tag. diff --git a/src/dag_builder/build.cpp b/src/dag_builder/build.cpp index 203dca83..7653df1b 100644 --- a/src/dag_builder/build.cpp +++ b/src/dag_builder/build.cpp @@ -41,6 +41,7 @@ #include "vertex_lock.h" #include "parallel.h" #include "ContinuationMode.h" +#include "sf/validate_index.h" namespace dag { @@ -584,11 +585,15 @@ std::unordered_set build_level( // Builds the DAG from input_storage into output_storage, restricted to levels within level_range. // Iterates octree levels from finest to coarsest, simplifying and re-clustering geometry at each // level from its children. -void build_levels( +std::expected build_levels( const octree::IndexedMeshStorage &input_storage, octree::IndexedDagStorage &output_storage, const BuildOptions &options, const AnyRange &level_range) { + const auto validation = sf::validate_index(input_storage.index()); + if (!validation.has_value()) { + return std::unexpected(validation.error()); + } const octree::OddLevelShifted shifted_space = octree::OddLevelShifted::earth(); const octree::Space space = octree::Space::earth(); const octree::Id root_node = options.root_node; @@ -599,7 +604,7 @@ void build_levels( auto max_input_level_opt = find_max_input_level(input_by_level); if (!max_input_level_opt.has_value()) { LOG_WARN("No input nodes found for root {}", root_node); - return; + return {}; } const uint32_t max_input_level = max_input_level_opt.value(); @@ -607,7 +612,7 @@ void build_levels( const Range range = valid_range.intersect(level_range).to_range(max_input_level + 1); if (range.is_empty()) { LOG_WARN("Requested level range does not overlap with buildable levels {}-{}", root_node.level(), max_input_level); - return; + return {}; } // Seed prev_level_built with any already-built nodes one level finer than the first level. @@ -638,14 +643,15 @@ void build_levels( } output_storage = std::move(ctx.output_storage).release(); + return {}; } // Builds the complete DAG from input_storage into output_storage. -void build_full( +std::expected build_full( const octree::IndexedMeshStorage &input_storage, octree::IndexedDagStorage &output_storage, const BuildOptions &options) { - build_levels(input_storage, output_storage, options, RangeFull{}); + return build_levels(input_storage, output_storage, options, RangeFull{}); } } // namespace dag diff --git a/src/dag_builder/build.h b/src/dag_builder/build.h index d39491d6..b87528e7 100644 --- a/src/dag_builder/build.h +++ b/src/dag_builder/build.h @@ -2,6 +2,7 @@ #include #include +#include #include "ContinuationMode.h" #include "Range.h" @@ -9,6 +10,7 @@ #include "texturing.h" #include "octree/storage/MeshStorage.h" #include "storage.h" +#include "sf/InvalidTopology.h" namespace dag { @@ -30,12 +32,12 @@ struct BuildOptions { ContinuationMode continuation_mode = ContinuationMode::Error; }; -void build_full( +std::expected build_full( const octree::IndexedMeshStorage &input_storage, octree::IndexedDagStorage &output_storage, const BuildOptions &options); -void build_levels( +std::expected build_levels( const octree::IndexedMeshStorage &input_storage, octree::IndexedDagStorage &output_storage, const BuildOptions &options, diff --git a/src/dag_builder/main.cpp b/src/dag_builder/main.cpp index 495bbf09..4fb1d70a 100644 --- a/src/dag_builder/main.cpp +++ b/src/dag_builder/main.cpp @@ -8,6 +8,7 @@ #include "octree/storage/open.h" #include "storage.h" #include "store/describe_error.h" +#include "sf/Error.h" #include "ContinuationMode.h" int main(int argc, char **argv) { @@ -58,7 +59,15 @@ int main(int argc, char **argv) { options.relative_target_error = 0.001f; } - dag::build_levels(input_storage, output_storage, options, args.level_range); + const auto build_result = dag::build_levels( + input_storage, + output_storage, + options, + args.level_range); + if (!build_result.has_value()) { + LOG_ERROR("Invalid Structura Fundamentalis input: {}", sf::describe_error(build_result.error())); + return EXIT_FAILURE; + } const auto index_result = output_storage.save_index(); if (!index_result.has_value()) { LOG_ERROR( diff --git a/src/sf_builder/main.cpp b/src/sf_builder/main.cpp index e7216346..6bef9d68 100644 --- a/src/sf_builder/main.cpp +++ b/src/sf_builder/main.cpp @@ -18,6 +18,7 @@ #include "terrainbuilder.h" #include "octree/Space.h" #include "tile_provider.h" +#include "sf/Error.h" #include "ctb/GlobalGeodetic.hpp" #include "ctb/GlobalMercator.hpp" @@ -317,7 +318,7 @@ int run(std::span args) { const auto maybe_s = actual_num_threads == 1 ? "" : "s"; LOG_INFO("Using {} thread{} for batch processing.", actual_num_threads, maybe_s); - terrainbuilder::build_all_patches( + const auto build_result = terrainbuilder::build_all_patches( dataset, target_level, texture_srs, @@ -326,6 +327,12 @@ int run(std::span args) { output_base_path, output_format, overwrite_existing); + if (!build_result.has_value()) { + LOG_ERROR( + "Failed to finalize Structura Fundamentalis output: {}", + sf::describe_error(build_result.error())); + return 1; + } } return 0; diff --git a/src/sf_builder/terrainbuilder.cpp b/src/sf_builder/terrainbuilder.cpp index a5780999..438da793 100644 --- a/src/sf_builder/terrainbuilder.cpp +++ b/src/sf_builder/terrainbuilder.cpp @@ -45,6 +45,7 @@ #include "octree/storage/open.h" #include "octree/utils.h" #include "store/describe_error.h" +#include "sf/finalize_storage.h" namespace terrainbuilder { @@ -173,7 +174,11 @@ T expect(const std::optional &opt, const std::string &msg) { } } -void build_all_patches( +std::expected finalize_storage(octree::Storage &storage) { + return sf::finalize_storage(storage); +} + +std::expected build_all_patches( Dataset &dataset, const octree::Id::Level target_level, const OGRSpatialReference &texture_srs, @@ -347,12 +352,6 @@ void build_all_patches( LOG_ERROR_AND_EXIT("Failed to build all terrain patches"); } - const auto index_result = storage.save_or_create_index(); - if (!index_result.has_value()) { - LOG_ERROR_AND_EXIT( - "Failed to save output index in {}: {}", - storage.base_path(), - store::describe_error(index_result.error())); - } + return finalize_storage(storage); } } diff --git a/src/sf_builder/terrainbuilder.h b/src/sf_builder/terrainbuilder.h index b4c0f581..35d51b29 100644 --- a/src/sf_builder/terrainbuilder.h +++ b/src/sf_builder/terrainbuilder.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include @@ -8,6 +9,8 @@ #include "octree/Id.h" #include "mesh/SimpleMesh.h" #include "tile_provider.h" +#include "octree/Storage.h" +#include "sf/Error.h" namespace terrainbuilder { @@ -28,7 +31,9 @@ std::optional build_patch( const TileProvider *tile_provider, const OGRSpatialReference &mesh_srs); -void build_all_patches( +std::expected finalize_storage(octree::Storage &storage); + +std::expected build_all_patches( Dataset &dataset, const octree::Id::Level target_level, const OGRSpatialReference &texture_srs, diff --git a/src/sf_merger/NodeWriter.h b/src/sf_merger/NodeWriter.h index 6c5bf290..9c51359e 100644 --- a/src/sf_merger/NodeWriter.h +++ b/src/sf_merger/NodeWriter.h @@ -1,5 +1,8 @@ #pragma once +#include +#include + #include #include "NodeLoader.h" @@ -14,34 +17,58 @@ class NodeWriter { public: NodeWriter(octree::Storage &storage) : _storage(storage) {} - bool has_node(const octree::Id &id) { - return DEBUG_ASSERT_VAL(this->_storage.has(id)).value(); + std::expected> has_node( + const octree::Id &id) { + return this->_storage.has(id); } - void write_node(const octree::Id &id, const SimpleMesh &mesh) { + std::expected> write_node( + const octree::Id &id, + const SimpleMesh &mesh) { mesh::validate(mesh); - DEBUG_ASSERT_VAL(this->_storage.save(id, mesh)); - auto p = DEBUG_ASSERT_VAL(this->_storage.path_for(id)).value(); + const auto save_result = this->_storage.save(id, mesh); + if (!save_result.has_value()) { + return save_result; + } + const auto path_result = this->_storage.path_for(id); + if (!path_result.has_value()) { + return std::unexpected(store::SaveError(path_result.error())); + } + auto p = path_result.value(); // change extension to .png p.replace_extension(".png"); - cv::imwrite(p, mesh.texture.value_or(cv::Mat())); + if (mesh.texture.has_value()) { + cv::imwrite(p, mesh.texture.value()); + } + return {}; } - void copy_subtree_to_output( + std::expected> copy_subtree_to_output( const octree::Id &id, const NodeLoader &loader) { + std::optional> error; octree::traverse( loader.storage().index(), [&](const octree::Id &child_id, const octree::NodeStatus &status) { + if (error.has_value()) { + return; + } if (status == octree::NodeStatus::Virtual) { return; } DEBUG_ASSERT(status == octree::NodeStatus::Leaf); - DEBUG_ASSERT_VAL(this->_storage.copy_from(child_id, loader.storage())); + const auto result = this->_storage.copy_from(child_id, loader.storage()); + if (!result.has_value()) { + error = result.error(); + } }, - octree::always_refine, + [&](const octree::Id &) { return !error.has_value(); }, id); + if (error.has_value()) { + return std::unexpected(std::move(error.value())); + } + return {}; } private: diff --git a/src/sf_merger/cut.h b/src/sf_merger/cut.h index 1e785618..3c6ce4d6 100644 --- a/src/sf_merger/cut.h +++ b/src/sf_merger/cut.h @@ -1,5 +1,7 @@ #pragma once +#include + #include #include "mesh/SimpleMesh.h" @@ -14,6 +16,9 @@ #include "utils.h" #include "containers/Cow.h" #include "store/describe_error.h" +#include "sf/Error.h" +#include "sf/finalize_storage.h" +#include "sf/validate_index.h" struct Context { const octree::IndexedStorage& input; @@ -22,12 +27,12 @@ struct Context { const bool keep_inside; }; -void cut_node( +std::expected cut_node( Context& ctx, const octree::Id& id, const MeshMask& mask); -inline void cut_leaf_node( +inline std::expected cut_leaf_node( Context& ctx, const octree::Id& id, const MeshMask& mask @@ -41,20 +46,27 @@ inline void cut_leaf_node( const Cow clipped = clip_on_mask(mesh, mask, ctx.keep_inside); if (clipped.is_ref()) { LOG_TRACE("Mesh was fully inside the mask"); - DEBUG_ASSERT_VAL(ctx.output.copy_from(id, ctx.input)); + const auto copy_result = ctx.output.copy_from(id, ctx.input); + if (!copy_result.has_value()) { + return std::unexpected(sf::ProcessingError(copy_result.error())); + } } else { const SimpleMesh &clipped_mesh = clipped; if (!clipped_mesh.is_empty()) { LOG_TRACE("Mesh was clipped from {} vertices and {} triangles to {} vertices and {} triangles", mesh.vertex_count(), mesh.face_count(), clipped_mesh.vertex_count(), clipped_mesh.face_count()); - DEBUG_ASSERT_VAL(ctx.output.save(id, clipped_mesh)); + const auto save_result = ctx.output.save(id, clipped_mesh); + if (!save_result.has_value()) { + return std::unexpected(sf::ProcessingError(save_result.error())); + } } else { LOG_TRACE("Mesh was fully outside the mask"); } } + return {}; } -inline void cut_virtual_node( +inline std::expected cut_virtual_node( Context& ctx, const octree::Id& id, const MeshMask& mask @@ -69,18 +81,22 @@ inline void cut_virtual_node( const auto child_bounds = geometry::pad_bounds_relative(ctx.space.get_node_bounds(child_id), 0.125); LOG_TRACE("Clipping mask for {}", child_id); const auto child_mask = MeshMask(mesh::clip_on_bounds_and_cap(mask.mesh, child_bounds)); - cut_node(ctx, child_id, child_mask); + const auto child_result = cut_node(ctx, child_id, child_mask); + if (!child_result.has_value()) { + return child_result; + } } + return {}; } -inline void cut_node( +inline std::expected cut_node( Context& ctx, const octree::Id& id, const MeshMask& mask ) { if (ctx.keep_inside && mask.mesh.is_empty()) { LOG_TRACE("Mesh was fully outside the mask"); - return; + return {}; } /* @@ -94,40 +110,46 @@ inline void cut_node( const auto status_result = ctx.input.index().get(id); DEBUG_ASSERT(status_result.has_value()); if (!status_result->has_value()) { - return; + return {}; } const octree::NodeStatus status = status_result->value(); switch (status) { case octree::NodeStatus::Virtual: - cut_virtual_node(ctx, id, mask); - break; + return cut_virtual_node(ctx, id, mask); case octree::NodeStatus::Leaf: - cut_leaf_node(ctx, id, mask); - break; + return cut_leaf_node(ctx, id, mask); default: UNREACHABLE(); - break; + return {}; } + return {}; } -inline void cut_dataset( +inline std::expected cut_dataset( const octree::IndexedStorage &input, const MeshMask& mask, octree::Storage &output, const bool keep_inside) { + const auto validation = sf::validate_index(input.index()); + if (!validation.has_value()) { + return std::unexpected(sf::ProcessingError(validation.error())); + } Context ctx(input, output, octree::Space::earth(), keep_inside); - cut_node(ctx, octree::Id::root(), mask); - const auto index_result = output.save_or_create_index(); - if (!index_result.has_value()) { - LOG_ERROR_AND_EXIT( - "Failed to save output index in {}: {}", - output.base_path(), - store::describe_error(index_result.error())); + const auto cut_result = cut_node(ctx, octree::Id::root(), mask); + if (!cut_result.has_value()) { + return cut_result; + } + const auto finalization = sf::finalize_storage(output); + if (!finalization.has_value()) { + return std::unexpected(std::visit( + [](const auto &error) -> sf::ProcessingError { return error; }, + finalization.error())); } + return {}; } -inline void cut_dataset( +inline std::expected cut_dataset( const octree::IndexedStorage &input_dataset, const MeshMask& mask, const std::filesystem::path &output_path, @@ -140,15 +162,13 @@ inline void cut_dataset( options.preferred_extension = ".glb"; auto output_result = octree::open_folder_indexed(output_path, std::move(options)); if (!output_result.has_value()) { - LOG_ERROR_AND_EXIT( - "Failed to open output dataset {}: {}", - output_path, - store::describe_error(output_result.error())); + return std::unexpected(sf::ProcessingError(output_result.error())); } octree::IndexedStorage output_dataset = std::move(output_result.value()); if (!output_dataset.index().empty()) { - LOG_ERROR_AND_EXIT("Output dataset has to be empty."); + return std::unexpected(sf::ProcessingError(store::SaveError( + store::AlreadyExists{output_path}))); } - cut_dataset(input_dataset, mask, output_dataset, keep_inside); + return cut_dataset(input_dataset, mask, output_dataset, keep_inside); } diff --git a/src/sf_merger/main.cpp b/src/sf_merger/main.cpp index 76894680..bae30669 100644 --- a/src/sf_merger/main.cpp +++ b/src/sf_merger/main.cpp @@ -12,6 +12,7 @@ #include "optional_utils.h" #include "earth.h" #include "store/describe_error.h" +#include "sf/Error.h" std::optional load_mask_from_path(const std::filesystem::path& path) { if (std::filesystem::exists(path)) { @@ -70,7 +71,14 @@ void run(const cli::MergeArgs& args) { std::optional mask = flatten(map(args.mask_path, load_mask_from_path)); - return merge_datasets(base_dataset, new_dataset, output_dataset, mask); + const auto merge_result = merge_datasets( + base_dataset, + new_dataset, + output_dataset, + mask); + if (!merge_result.has_value()) { + LOG_ERROR_AND_EXIT("Failed to merge datasets: {}", sf::describe_error(merge_result.error())); + } } void run(const cli::CutArgs& args) { @@ -84,7 +92,14 @@ void run(const cli::CutArgs& args) { } const octree::IndexedStorage input_dataset = std::move(input_result.value()); const MeshMask mask = DEBUG_ASSERT_VAL(load_mask_from_path(args.mask_path)).value(); - cut_dataset(input_dataset, mask, args.output_path, args.keep_inside); + const auto cut_result = cut_dataset( + input_dataset, + mask, + args.output_path, + args.keep_inside); + if (!cut_result.has_value()) { + LOG_ERROR_AND_EXIT("Failed to cut dataset: {}", sf::describe_error(cut_result.error())); + } } void run(const cli::Args &args) { diff --git a/src/sf_merger/merge.h b/src/sf_merger/merge.h index b84fe38d..6987d14d 100644 --- a/src/sf_merger/merge.h +++ b/src/sf_merger/merge.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -15,6 +16,9 @@ #include "octree/NodeStatusOrMissing.h" #include "octree/storage/cache/Dummy.h" #include "store/describe_error.h" +#include "sf/Error.h" +#include "sf/finalize_storage.h" +#include "sf/validate_index.h" inline std::string get_dataset_name(const octree::Storage &storage) { return storage.layout().base_path().filename().string(); @@ -33,6 +37,7 @@ class Merger { using Status = octree::NodeStatusOrMissing; using Context = Visitor::Context; using Result = merge::Result; + using Expected = std::expected; Merger( Visitor& visitor, @@ -41,62 +46,86 @@ class Merger { NodeWriter output) : _visitor(visitor), _left(left), _right(right), _output(output) { } - void merge_root() { + Expected merge_root() { const octree::Id id = octree::Id::root(); const Context ctx = this->_visitor.make_root_context(); - merge_node(id, ctx); + return merge_node(id, ctx); } - void merge_node(const octree::Id &id, const Context& ctx) { + Expected merge_node(const octree::Id &id, const Context& ctx) { const Status left_status = this->_left.get_status(id); const Status right_status = this->_right.get_status(id); return this->merge_node(id, left_status, right_status, ctx); } - void merge_node( + Expected merge_node( const octree::Id &id, const Status left_status, const Status right_status, const Context& ctx ) { LOG_DEBUG("[{}] Start merging (left = {}, right = {})", id, left_status, right_status); - if (this->_output.has_node(id)) { + const auto has_result = this->_output.has_node(id); + if (!has_result.has_value()) { + return std::unexpected(sf::ProcessingError(has_result.error())); + } + if (has_result.value()) { LOG_DEBUG("[{}] Already merged, skipping...", id); - return; // Already merged previously + return {}; // Already merged previously } const auto merge_result = this->call_merge(id, left_status, right_status, ctx); - std::visit([&](const auto &result) { + return std::visit([&](const auto &result) -> Expected { using Result = std::decay_t; if constexpr (std::is_same_v>) { LOG_DEBUG("[{}] needs recursion", id); DEBUG_ASSERT(id.has_children()); const auto children = id.children().value(); for (const auto &child_id : children) { - this->merge_node(child_id, result.context); + const auto child_result = this->merge_node(child_id, result.context); + if (!child_result.has_value()) { + return child_result; + } } + return {}; } else if constexpr (std::is_same_v) { LOG_DEBUG("[{}] remains unchanged (same as {})", id, (result.source == merge::Source::Left ? "left" : "right")); // If the node should remain unchanged, but its not present on disk we cannot copy it. if (result.source == merge::Source::Left && left_status == Status::Missing) { auto mesh_opt = this->_left.load_node(id); if (mesh_opt.has_value()) { - this->_output.write_node(id, mesh_opt.value()); + const auto write_result = this->_output.write_node(id, mesh_opt.value()); + if (!write_result.has_value()) { + return std::unexpected(sf::ProcessingError(write_result.error())); + } } } else if (result.source == merge::Source::Right && right_status == Status::Missing) { auto mesh_opt = this->_right.load_node(id); if (mesh_opt.has_value()) { - this->_output.write_node(id, mesh_opt.value()); + const auto write_result = this->_output.write_node(id, mesh_opt.value()); + if (!write_result.has_value()) { + return std::unexpected(sf::ProcessingError(write_result.error())); + } } } else { - this->_output.copy_subtree_to_output(id, result.source == merge::Source::Left ? this->_left : this->_right); + const auto copy_result = this->_output.copy_subtree_to_output( + id, + result.source == merge::Source::Left ? this->_left : this->_right); + if (!copy_result.has_value()) { + return std::unexpected(sf::ProcessingError(copy_result.error())); + } } + return {}; } else if constexpr (std::is_same_v) { LOG_DEBUG("[{}] was merged", id); - this->_output.write_node(id, result.mesh); + const auto write_result = this->_output.write_node(id, result.mesh); + if (!write_result.has_value()) { + return std::unexpected(sf::ProcessingError(write_result.error())); + } + return {}; } else if constexpr (std::is_same_v) { LOG_DEBUG("[{}] was ignored", id); - // do nothing + return {}; } }, merge_result); } @@ -152,11 +181,18 @@ class Merger { NodeWriter _output; }; -inline void merge_datasets( +inline std::expected merge_datasets( const octree::IndexedStorage &left_dataset, const octree::IndexedStorage &right_dataset, octree::Storage &output_dataset, const std::optional mask = std::nullopt) { + for (const octree::IndexedStorage *input : {&left_dataset, &right_dataset}) { + const auto validation = sf::validate_index(input->index()); + if (!validation.has_value()) { + return std::unexpected(sf::ProcessingError(validation.error())); + } + } + LOG_TRACE("Merging {} and {} into {}", get_dataset_name(left_dataset), get_dataset_name(right_dataset), @@ -171,18 +207,24 @@ inline void merge_datasets( if (mask.has_value()) { merge::visitor::Masked visitor {mask.value(), space}; Merger merger(visitor, left, right, output); - merger.merge_root(); + const auto result = merger.merge_root(); + if (!result.has_value()) { + return result; + } } else { merge::visitor::Simple visitor; Merger merger(visitor, left, right, output); - merger.merge_root(); + const auto result = merger.merge_root(); + if (!result.has_value()) { + return result; + } } - const auto index_result = output_dataset.save_or_create_index(); - if (!index_result.has_value()) { - LOG_ERROR_AND_EXIT( - "Failed to save output index in {}: {}", - output_dataset.base_path(), - store::describe_error(index_result.error())); + const auto finalization = sf::finalize_storage(output_dataset); + if (!finalization.has_value()) { + return std::unexpected(std::visit( + [](const auto &error) -> sf::ProcessingError { return error; }, + finalization.error())); } + return {}; } diff --git a/src/terrainlib/sf/Error.h b/src/terrainlib/sf/Error.h new file mode 100644 index 00000000..a8274c72 --- /dev/null +++ b/src/terrainlib/sf/Error.h @@ -0,0 +1,42 @@ +#pragma once + +#include +#include +#include + +#include "octree/Id.h" +#include "sf/InvalidTopology.h" +#include "store/OpenError.h" +#include "store/describe_error.h" + +namespace sf { + +using FinalizeError = std::variant; + +using ProcessingError = std::variant< + InvalidTopology, + store::OpenError, + store::FileOperationError, + store::SaveError, + store::CopyError, + store::IndexFormatError>; + +inline std::string describe_error(const InvalidTopology &error) { + return "Structura Fundamentalis topology contains Inner node " + + error.key.to_string(); +} + +template +std::string describe_error(const std::variant &error) { + return std::visit( + [](const auto &value) -> std::string { + if constexpr (std::is_same_v, InvalidTopology>) { + return describe_error(value); + } else { + return store::describe_error(value); + } + }, + error); +} + +} // namespace sf diff --git a/src/terrainlib/sf/InvalidTopology.h b/src/terrainlib/sf/InvalidTopology.h new file mode 100644 index 00000000..643d769d --- /dev/null +++ b/src/terrainlib/sf/InvalidTopology.h @@ -0,0 +1,13 @@ +#pragma once + +#include "octree/Id.h" + +namespace sf { + +struct InvalidTopology { + octree::Id key; + + bool operator==(const InvalidTopology &) const = default; +}; + +} // namespace sf diff --git a/src/terrainlib/sf/finalize_storage.h b/src/terrainlib/sf/finalize_storage.h new file mode 100644 index 00000000..8ee9a06c --- /dev/null +++ b/src/terrainlib/sf/finalize_storage.h @@ -0,0 +1,28 @@ +#pragma once + +#include + +#include + +#include "octree/Storage.h" +#include "sf/Error.h" +#include "sf/validate_index.h" + +namespace sf { + +inline std::expected finalize_storage(octree::Storage &storage) { + const auto save_result = storage.save_or_create_index(); + if (!save_result.has_value()) { + return std::unexpected(FinalizeError(save_result.error())); + } + + const auto index = storage.index(); + DEBUG_ASSERT(index.has_value()); + const auto validation = validate_index(index->get()); + if (!validation.has_value()) { + return std::unexpected(FinalizeError(validation.error())); + } + return {}; +} + +} // namespace sf diff --git a/src/terrainlib/sf/validate_index.h b/src/terrainlib/sf/validate_index.h new file mode 100644 index 00000000..1cf33719 --- /dev/null +++ b/src/terrainlib/sf/validate_index.h @@ -0,0 +1,21 @@ +#pragma once + +#include + +#include "octree/StoreTraits.h" +#include "sf/InvalidTopology.h" +#include "store/Index.h" + +namespace sf { + +inline std::expected validate_index( + const store::Index &index) { + for (const auto &[key, status] : index) { + if (status == store::NodeStatus::Inner) { + return std::unexpected(InvalidTopology{key}); + } + } + return {}; +} + +} // namespace sf diff --git a/src/terrainlib/store/RawStorage.h b/src/terrainlib/store/RawStorage.h index 487a3e97..c4bb1abc 100644 --- a/src/terrainlib/store/RawStorage.h +++ b/src/terrainlib/store/RawStorage.h @@ -108,6 +108,16 @@ class RawStorage { std::expected> copy_from( const Key &key, const RawStorage &source) const { + return copy_from(key, source, []() -> std::expected> { + return {}; + }); + } + + template + std::expected> copy_from( + const Key &key, + const RawStorage &source, + BeforeModify &&before_modify) const { if (!Traits::is_valid(key)) { return std::unexpected(CopyError(InvalidKey{key})); } @@ -127,6 +137,10 @@ class RawStorage { if (!loaded.has_value()) { return std::unexpected(CopyError(loaded.error())); } + const auto prepared = before_modify(); + if (!prepared.has_value()) { + return prepared; + } const auto written = _codec->write(_layout.node_path(key), loaded.value()); if (!written.has_value()) { return std::unexpected(CopyError(written.error())); @@ -136,6 +150,17 @@ class RawStorage { const auto source_paths = source._codec->paths(source._layout.node_path(key)); const auto target_paths = _codec->paths(_layout.node_path(key)); + if (source_paths.size() != target_paths.size()) { + return std::unexpected(CopyError(CodecError{ + CodecOperation::Write, + CodecErrorCategory::Domain, + "matching codec probes produced different actual path counts", + })); + } + const auto prepared = before_modify(); + if (!prepared.has_value()) { + return prepared; + } for (size_t index = 0; index < source_paths.size(); ++index) { std::error_code error; std::filesystem::remove(target_paths[index], error); diff --git a/src/terrainlib/store/Storage.h b/src/terrainlib/store/Storage.h index e3cf3a85..e6a4f211 100644 --- a/src/terrainlib/store/Storage.h +++ b/src/terrainlib/store/Storage.h @@ -147,15 +147,19 @@ class Storage { node_paths.empty() ? _raw.layout().node_path(key).path() : node_paths.front(), })); } - if (exists.value() && _index.has_value()) { + const auto prepare_target = [&]() -> std::expected> { + if (!exists.value() || !_index.has_value()) { + return {}; + } const auto removed = _index->remove(key); if (!removed.has_value()) { return std::unexpected(CopyError(removed.error())); } _dirty = _dirty || removed.value(); - } + return {}; + }; - const auto result = _raw.copy_from(key, source._raw); + const auto result = _raw.copy_from(key, source._raw, prepare_target); if (!result.has_value()) { return result; } diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 26db9b5d..3cb01c33 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -29,6 +29,7 @@ add_executable(unittests_terrainlib terrainlib/hybrid_vector.cpp terrainlib/offset_table.cpp terrainlib/segmented_buffer.cpp + terrainlib/sf_validate_index.cpp terrainlib/stamp_set.cpp terrainlib/store_compatibility.cpp terrainlib/store_codec.cpp @@ -97,6 +98,15 @@ if(TARGET sfbuilderlib) ) target_link_libraries(unittests_sfbuilder PRIVATE sfbuilderlib Catch2::Catch2WithMain) atb_configure_test(unittests_sfbuilder) + + add_executable(unittests_sfbuilder_finalization + catch2_helpers.h + sf_builder/storage_finalization.cpp + ) + target_link_libraries( + unittests_sfbuilder_finalization + PRIVATE sfbuilderlib Catch2::Catch2WithMain) + atb_configure_test(unittests_sfbuilder_finalization) endif() if(TARGET tile-downloader) @@ -137,6 +147,7 @@ if(TARGET sfmergerlib) catch2_helpers.h sf_merger/mask.cpp sf_merger/sphere_projector.cpp + sf_merger/storage_boundaries.cpp ) target_link_libraries(unittests_sfmerger PRIVATE sfmergerlib Catch2::Catch2WithMain) atb_configure_test(unittests_sfmerger) diff --git a/unittests/dag_builder/storage_compatibility.cpp b/unittests/dag_builder/storage_compatibility.cpp index 45a3d351..c2854a22 100644 --- a/unittests/dag_builder/storage_compatibility.cpp +++ b/unittests/dag_builder/storage_compatibility.cpp @@ -6,6 +6,7 @@ #include #include "encoded.h" +#include "build.h" #include "io/bytes.h" #include "io/serialize.h" #include "octree/disk/IndexFile.h" @@ -174,3 +175,36 @@ TEST_CASE("DAG indexed storage survives ThreadSafeStorage move and release", "[s CHECK(reopened_result->has(octree::Id::root()).value()); CHECK(reopened_result->load(octree::Id::root()).has_value()); } + +TEST_CASE("DAG builder rejects invalid SF topology before processing", "[dag][sf][validation]") { + const std::filesystem::path fixture_path = + std::filesystem::path(ALP_TEST_DATA_DIR) / "raster-store-refactor/sf-flat"; + auto fixture_result = octree::open_folder_indexed(fixture_path); + REQUIRE(fixture_result.has_value()); + const auto mesh_result = fixture_result->load(octree::Id::root()); + REQUIRE(mesh_result.has_value()); + + TemporaryDirectory input_directory; + TemporaryDirectory output_directory; + auto input_result = octree::open_folder_indexed(input_directory.path()); + REQUIRE(input_result.has_value()); + auto input = std::move(input_result.value()); + const octree::Id root = octree::Id::root(); + REQUIRE(input.save(root, mesh_result.value()).has_value()); + REQUIRE(input.save(root.child(0).value(), mesh_result.value()).has_value()); + REQUIRE(input.index().is(store::NodeStatus::Inner, root).value()); + + auto output_result = dag::storage::open_folder_indexed(output_directory.path()); + REQUIRE(output_result.has_value()); + auto output = std::move(output_result.value()); + const dag::BuildOptions options{ + .clusters_per_partition = 1, + .target_ratio = 1.0f, + .relative_target_error = std::nullopt, + }; + + const auto built = dag::build_full(input, output, options); + REQUIRE_FALSE(built.has_value()); + CHECK(built.error().key == root); + CHECK(output.index().empty()); +} diff --git a/unittests/sf_builder/storage_finalization.cpp b/unittests/sf_builder/storage_finalization.cpp new file mode 100644 index 00000000..3ef11068 --- /dev/null +++ b/unittests/sf_builder/storage_finalization.cpp @@ -0,0 +1,80 @@ +#include +#include +#include + +#include + +#include "octree/storage/open.h" +#include "terrainbuilder.h" + +namespace { + +class TemporaryDirectory { +public: + TemporaryDirectory() { + static std::atomic_uint64_t counter = 0; + const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); + _path = std::filesystem::temp_directory_path() + / ("atb-sfbuilder-finalize-" + std::to_string(timestamp) + "-" + + std::to_string(counter++)); + REQUIRE(std::filesystem::create_directories(_path)); + } + + ~TemporaryDirectory() { + std::error_code error; + std::filesystem::remove_all(_path, error); + } + + const std::filesystem::path &path() const { + return _path; + } + +private: + std::filesystem::path _path; +}; + +mesh::Simple fixture_mesh() { + const std::filesystem::path fixture_path = + std::filesystem::path(ALP_TEST_DATA_DIR) / "raster-store-refactor/sf-flat"; + auto fixture = octree::open_folder_indexed(fixture_path); + REQUIRE(fixture.has_value()); + auto loaded = fixture->load(octree::Id::root()); + REQUIRE(loaded.has_value()); + return std::move(loaded.value()); +} + +} // namespace + +TEST_CASE("SF builder finalization writes and validates a valid index", "[sfbuilder][sf]") { + TemporaryDirectory directory; + auto storage_result = octree::open_folder(directory.path()); + REQUIRE(storage_result.has_value()); + auto storage = std::move(storage_result.value()); + REQUIRE(storage.save(octree::Id::root(), fixture_mesh()).has_value()); + + CHECK(terrainbuilder::finalize_storage(storage).has_value()); + CHECK(std::filesystem::is_regular_file(directory.path() / "terrain.index")); +} + +TEST_CASE( + "SF builder finalization retains an invalid written index for diagnosis", + "[sfbuilder][sf]") { + TemporaryDirectory directory; + auto storage_result = octree::open_folder(directory.path()); + REQUIRE(storage_result.has_value()); + auto storage = std::move(storage_result.value()); + const octree::Id root = octree::Id::root(); + const mesh::Simple mesh = fixture_mesh(); + REQUIRE(storage.save(root, mesh).has_value()); + REQUIRE(storage.save(root.child(0).value(), mesh).has_value()); + + const auto finalized = terrainbuilder::finalize_storage(storage); + REQUIRE_FALSE(finalized.has_value()); + REQUIRE(std::holds_alternative(finalized.error())); + CHECK(std::get(finalized.error()).key == root); + CHECK(std::filesystem::is_regular_file(directory.path() / "terrain.index")); + + auto reopened = octree::open_index(directory.path() / "terrain.index"); + REQUIRE(reopened.has_value()); + CHECK(reopened->index().is(store::NodeStatus::Inner, root).value()); +} diff --git a/unittests/sf_merger/storage_boundaries.cpp b/unittests/sf_merger/storage_boundaries.cpp new file mode 100644 index 00000000..cbc96ed6 --- /dev/null +++ b/unittests/sf_merger/storage_boundaries.cpp @@ -0,0 +1,196 @@ +#include +#include +#include + +#include + +#include "cut.h" +#include "merge.h" +#include "octree/storage/open.h" + +namespace { + +class TemporaryDirectory { +public: + explicit TemporaryDirectory(const std::string_view label) { + static std::atomic_uint64_t counter = 0; + const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); + _path = std::filesystem::temp_directory_path() + / ("atb-sfmerger-" + std::string(label) + "-" + + std::to_string(timestamp) + "-" + std::to_string(counter++)); + REQUIRE(std::filesystem::create_directories(_path)); + } + + ~TemporaryDirectory() { + std::error_code error; + std::filesystem::remove_all(_path, error); + } + + const std::filesystem::path &path() const { + return _path; + } + +private: + std::filesystem::path _path; +}; + +mesh::Simple triangle(const double x_offset = 0.0) { + return mesh::Simple( + {{0, 1, 2}}, + { + {-1.0 + x_offset, 0.0, 0.0}, + {1.0 + x_offset, 0.0, 0.0}, + {x_offset, 2.0, 0.0}, + }); +} + +mesh::Simple clipping_box() { + return mesh::Simple( + { + {0, 1, 2}, {0, 2, 3}, + {1, 5, 6}, {1, 6, 2}, + {5, 4, 7}, {5, 7, 6}, + {4, 0, 3}, {4, 3, 7}, + {3, 2, 6}, {3, 6, 7}, + {4, 5, 1}, {4, 1, 0}, + }, + { + {0.0, -1.0, -1.0}, + {2.0, -1.0, -1.0}, + {2.0, 3.0, -1.0}, + {0.0, 3.0, -1.0}, + {0.0, -1.0, 1.0}, + {2.0, -1.0, 1.0}, + {2.0, 3.0, 1.0}, + {0.0, 3.0, 1.0}, + }); +} + +octree::IndexedStorage indexed_storage(const std::filesystem::path &path) { + auto result = octree::open_folder_indexed(path); + REQUIRE(result.has_value()); + return std::move(result.value()); +} + +octree::Storage unindexed_storage(const std::filesystem::path &path) { + auto result = octree::open_folder(path); + REQUIRE(result.has_value()); + return std::move(result.value()); +} + +} // namespace + +TEST_CASE("SF merge rejects Inner input before dispatch", "[sfmerger][sf][validation]") { + TemporaryDirectory left_directory("invalid-left"); + TemporaryDirectory right_directory("invalid-right"); + TemporaryDirectory output_directory("invalid-output"); + auto left = indexed_storage(left_directory.path()); + auto right = indexed_storage(right_directory.path()); + auto output = unindexed_storage(output_directory.path()); + const octree::Id root = octree::Id::root(); + REQUIRE(left.save(root, triangle()).has_value()); + REQUIRE(left.save(root.child(0).value(), triangle()).has_value()); + + const auto merged = merge_datasets(left, right, output); + REQUIRE_FALSE(merged.has_value()); + REQUIRE(std::holds_alternative(merged.error())); + CHECK(std::get(merged.error()).key == root); + CHECK_FALSE(std::filesystem::exists(output_directory.path() / "terrain.index")); +} + +TEST_CASE( + "SF merge retains an invalid output index for diagnosis", + "[sfmerger][sf][validation]") { + TemporaryDirectory left_directory("valid-left"); + TemporaryDirectory right_directory("valid-right"); + TemporaryDirectory output_directory("bad-final-output"); + auto left = indexed_storage(left_directory.path()); + auto right = indexed_storage(right_directory.path()); + auto output = unindexed_storage(output_directory.path()); + const octree::Id root = octree::Id::root(); + REQUIRE(output.save(root, triangle()).has_value()); + REQUIRE(output.save(root.child(0).value(), triangle()).has_value()); + + const auto merged = merge_datasets(left, right, output); + REQUIRE_FALSE(merged.has_value()); + REQUIRE(std::holds_alternative(merged.error())); + CHECK(std::get(merged.error()).key == root); + CHECK(std::filesystem::is_regular_file(output_directory.path() / "terrain.index")); +} + +TEST_CASE("SF merge hard-links unchanged nodes and writes changed nodes", "[sfmerger][sf][copy]") { + const octree::Id root = octree::Id::root(); + + SECTION("unchanged") { + TemporaryDirectory left_directory("unchanged-left"); + TemporaryDirectory right_directory("unchanged-right"); + TemporaryDirectory output_directory("unchanged-output"); + auto left = indexed_storage(left_directory.path()); + auto right = indexed_storage(right_directory.path()); + auto output = unindexed_storage(output_directory.path()); + REQUIRE(left.save(root, triangle()).has_value()); + + REQUIRE(merge_datasets(left, right, output).has_value()); + REQUIRE(output.has(root).value()); + CHECK(std::filesystem::equivalent( + left.paths(root)->front(), + output.paths(root)->front())); + } + + SECTION("changed") { + TemporaryDirectory left_directory("changed-left"); + TemporaryDirectory right_directory("changed-right"); + TemporaryDirectory output_directory("changed-output"); + auto left = indexed_storage(left_directory.path()); + auto right = indexed_storage(right_directory.path()); + auto output = unindexed_storage(output_directory.path()); + REQUIRE(left.save(root, triangle()).has_value()); + REQUIRE(right.save(root, triangle(10.0)).has_value()); + + REQUIRE(merge_datasets(left, right, output).has_value()); + REQUIRE(output.has(root).value()); + CHECK_FALSE(std::filesystem::equivalent( + left.paths(root)->front(), + output.paths(root)->front())); + CHECK_FALSE(std::filesystem::equivalent( + right.paths(root)->front(), + output.paths(root)->front())); + REQUIRE(output.load(root).has_value()); + CHECK(output.load(root)->face_count() == 2); + } +} + +TEST_CASE("SF cut hard-links unchanged leaves and writes clipped leaves", "[sfmerger][sf][copy]") { + const octree::Id root = octree::Id::root(); + + SECTION("unchanged") { + TemporaryDirectory input_directory("cut-unchanged-input"); + TemporaryDirectory output_directory("cut-unchanged-output"); + auto input = indexed_storage(input_directory.path()); + auto output = unindexed_storage(output_directory.path()); + REQUIRE(input.save(root, triangle()).has_value()); + + REQUIRE(cut_dataset(input, MeshMask{}, output, false).has_value()); + REQUIRE(output.has(root).value()); + CHECK(std::filesystem::equivalent( + input.paths(root)->front(), + output.paths(root)->front())); + } + + SECTION("clipped") { + TemporaryDirectory input_directory("cut-clipped-input"); + TemporaryDirectory output_directory("cut-clipped-output"); + auto input = indexed_storage(input_directory.path()); + auto output = unindexed_storage(output_directory.path()); + REQUIRE(input.save(root, triangle()).has_value()); + + REQUIRE(cut_dataset(input, MeshMask{clipping_box()}, output, true).has_value()); + REQUIRE(output.has(root).value()); + CHECK_FALSE(std::filesystem::equivalent( + input.paths(root)->front(), + output.paths(root)->front())); + const auto clipped = output.load(root); + REQUIRE(clipped.has_value()); + CHECK_FALSE(clipped->is_empty()); + } +} diff --git a/unittests/terrainlib/sf_validate_index.cpp b/unittests/terrainlib/sf_validate_index.cpp new file mode 100644 index 00000000..b8f7e2ec --- /dev/null +++ b/unittests/terrainlib/sf_validate_index.cpp @@ -0,0 +1,24 @@ +#include + +#include "octree/StoreTraits.h" +#include "sf/validate_index.h" + +TEST_CASE("Structura Fundamentalis validation accepts Leaf and Virtual nodes", "[sf][store]") { + store::Index index; + const octree::Id leaf = octree::Id::root().child(2).value().child(5).value(); + REQUIRE(index.add(leaf).has_value()); + + CHECK(sf::validate_index(index).has_value()); +} + +TEST_CASE("Structura Fundamentalis validation reports the first Inner node", "[sf][store]") { + store::Index index; + const octree::Id root = octree::Id::root(); + const octree::Id child = root.child(3).value(); + REQUIRE(index.add(root).has_value()); + REQUIRE(index.add(child).has_value()); + + const auto result = sf::validate_index(index); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().key == root); +} diff --git a/unittests/terrainlib/store_storage.cpp b/unittests/terrainlib/store_storage.cpp index 04125969..3065a6d2 100644 --- a/unittests/terrainlib/store_storage.cpp +++ b/unittests/terrainlib/store_storage.cpp @@ -96,6 +96,91 @@ class MultiFileCodec final : public store::Codec { second += ".metadata"; return {first, second}; } + + std::expected read( + const store::NodePath &node_path) const override { + const auto value = io::read_from_path(paths(node_path).front()); + if (!value.has_value()) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + store::CodecErrorCategory::Io, + "multi-file read failed", + }); + } + return value.value(); + } + + std::expected write( + const store::NodePath &node_path, + const int &value) const override { + const auto node_paths = paths(node_path); + if (!io::write_to_path(value, node_paths[0]).has_value() + || !io::write_to_path(value + 1, node_paths[1]).has_value()) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Io, + "multi-file write failed", + }); + } + return {}; + } +}; + +class ConfigurableCodec final : public store::Codec { +public: + explicit ConfigurableCodec(std::string extension) + : extension(std::move(extension)) {} + + std::vector paths( + const store::NodePath &node_path) const override { + std::filesystem::path path = node_path.path(); + path += extension; + return {path}; + } + + std::expected read( + const store::NodePath &node_path) const override { + if (fail_read) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + store::CodecErrorCategory::Domain, + "injected decode failure", + }); + } + const auto value = io::read_from_path(paths(node_path).front()); + if (!value.has_value()) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + store::CodecErrorCategory::Io, + "configurable read failed", + }); + } + return value.value(); + } + + std::expected write( + const store::NodePath &node_path, + const int &value) const override { + if (fail_write) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Domain, + "injected encode failure", + }); + } + if (!io::write_to_path(value, paths(node_path).front()).has_value()) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Io, + "configurable write failed", + }); + } + return {}; + } + + std::string extension; + bool fail_read = false; + bool fail_write = false; }; std::atomic_uint32_t index_writes = 0; @@ -138,12 +223,14 @@ store::IndexFormat counting_index_format() { } store::IndexedStorage make_indexed_storage( - const std::filesystem::path &path) { + const std::filesystem::path &path, + std::unique_ptr> codec = + std::make_unique>()) { using Storage = store::Storage; return store::IndexedStorage( store::RawStorage( store::Layout(path, octree::store_layout::flat()), - std::make_unique>()), + std::move(codec)), store::Index{}, Storage::Persistence{ counting_index_format(), @@ -249,3 +336,177 @@ TEST_CASE("storage moves transfer and finalize dirty index state", "[store][stor } CHECK(index_writes == 3); } + +TEST_CASE("copy_from hard-links every matching codec file", "[store][storage][copy]") { + const octree::Id root = octree::Id::root(); + + SECTION("one file") { + TemporaryDirectory source_directory("copy-one-source"); + TemporaryDirectory target_directory("copy-one-target"); + auto source = make_indexed_storage(source_directory.path()); + auto target = make_indexed_storage(target_directory.path()); + REQUIRE(source.save(root, 41).has_value()); + + REQUIRE(target.copy_from(root, source).has_value()); + REQUIRE(target.has(root).value()); + CHECK(target.load(root).value() == 41); + CHECK(std::filesystem::equivalent( + source.paths(root)->front(), + target.paths(root)->front())); + } + + SECTION("multiple files") { + TemporaryDirectory source_directory("copy-many-source"); + TemporaryDirectory target_directory("copy-many-target"); + auto source = make_indexed_storage( + source_directory.path(), std::make_unique()); + auto target = make_indexed_storage( + target_directory.path(), std::make_unique()); + REQUIRE(source.save(root, 42).has_value()); + + REQUIRE(target.copy_from(root, source).has_value()); + const auto source_paths = source.paths(root).value(); + const auto target_paths = target.paths(root).value(); + REQUIRE(source_paths.size() == 2); + REQUIRE(target_paths.size() == 2); + CHECK(std::filesystem::equivalent(source_paths[0], target_paths[0])); + CHECK(std::filesystem::equivalent(source_paths[1], target_paths[1])); + } +} + +TEST_CASE("copy_from re-encodes differing path lists", "[store][storage][copy]") { + const octree::Id root = octree::Id::root(); + TemporaryDirectory source_directory("copy-reencode-source"); + TemporaryDirectory target_directory("copy-reencode-target"); + auto source = make_indexed_storage( + source_directory.path(), std::make_unique()); + auto target = make_indexed_storage(target_directory.path()); + REQUIRE(source.save(root, 43).has_value()); + + REQUIRE(target.copy_from(root, source).has_value()); + CHECK(target.load(root).value() == 43); + CHECK(target.paths(root)->size() == 1); + CHECK(source.paths(root)->size() == 2); + CHECK_FALSE(std::filesystem::equivalent( + source.paths(root)->front(), + target.paths(root)->front())); +} + +TEST_CASE("copy_from enforces overwrite settings", "[store][storage][copy]") { + const octree::Id root = octree::Id::root(); + TemporaryDirectory source_directory("copy-overwrite-source"); + TemporaryDirectory target_directory("copy-overwrite-target"); + auto source = make_indexed_storage(source_directory.path()); + auto target = make_indexed_storage(target_directory.path()); + REQUIRE(source.save(root, 44).has_value()); + REQUIRE(target.save(root, 1).has_value()); + + const auto rejected = target.copy_from(root, source); + REQUIRE_FALSE(rejected.has_value()); + CHECK(std::holds_alternative(rejected.error())); + CHECK(target.load(root).value() == 1); + + target.settings().allow_overwrite = true; + REQUIRE(target.copy_from(root, source).has_value()); + CHECK(target.load(root).value() == 44); + CHECK(std::filesystem::equivalent( + source.paths(root)->front(), + target.paths(root)->front())); +} + +TEST_CASE( + "failed multi-file overwrite leaves the target node unindexed", + "[store][storage][copy]") { + const octree::Id root = octree::Id::root(); + TemporaryDirectory source_directory("copy-partial-source"); + TemporaryDirectory target_directory("copy-partial-target"); + auto source = make_indexed_storage( + source_directory.path(), std::make_unique()); + auto target = make_indexed_storage( + target_directory.path(), std::make_unique()); + REQUIRE(source.save(root, 45).has_value()); + REQUIRE(target.save(root, 1).has_value()); + target.settings().allow_overwrite = true; + + const auto target_paths = target.paths(root).value(); + REQUIRE(std::filesystem::remove(target_paths[1])); + REQUIRE(std::filesystem::create_directory(target_paths[1])); + REQUIRE(io::write_to_path(9, target_paths[1] / "blocker.bin").has_value()); + + const auto copied = target.copy_from(root, source); + REQUIRE_FALSE(copied.has_value()); + CHECK(std::holds_alternative(copied.error())); + CHECK_FALSE(target.has(root).value()); + CHECK_FALSE(target.index().get(root).value().has_value()); + CHECK(std::filesystem::equivalent( + source.paths(root)->front(), + target_paths.front())); + CHECK(std::filesystem::is_directory(target_paths[1])); +} + +TEST_CASE("copy_from propagates source and codec failures", "[store][storage][copy]") { + const octree::Id root = octree::Id::root(); + + SECTION("missing source") { + TemporaryDirectory source_directory("copy-missing-source"); + TemporaryDirectory target_directory("copy-missing-target"); + auto source = make_indexed_storage(source_directory.path()); + auto target = make_indexed_storage(target_directory.path()); + REQUIRE(target.save(root, 7).has_value()); + target.settings().allow_overwrite = true; + + const auto copied = target.copy_from(root, source); + REQUIRE_FALSE(copied.has_value()); + CHECK(std::holds_alternative>( + copied.error())); + CHECK(target.has(root).value()); + CHECK(target.load(root).value() == 7); + } + + SECTION("decode failure") { + TemporaryDirectory source_directory("copy-decode-source"); + TemporaryDirectory target_directory("copy-decode-target"); + auto source_codec = std::make_unique(".source"); + ConfigurableCodec *source_codec_ptr = source_codec.get(); + auto source = make_indexed_storage( + source_directory.path(), std::move(source_codec)); + auto target = make_indexed_storage( + target_directory.path(), + std::make_unique(".target")); + REQUIRE(source.save(root, 46).has_value()); + REQUIRE(target.save(root, 7).has_value()); + target.settings().allow_overwrite = true; + source_codec_ptr->fail_read = true; + + const auto copied = target.copy_from(root, source); + REQUIRE_FALSE(copied.has_value()); + REQUIRE(std::holds_alternative(copied.error())); + CHECK(std::get(copied.error()).operation + == store::CodecOperation::Read); + CHECK(target.has(root).value()); + CHECK(target.load(root).value() == 7); + } + + SECTION("encode failure") { + TemporaryDirectory source_directory("copy-encode-source"); + TemporaryDirectory target_directory("copy-encode-target"); + auto source = make_indexed_storage( + source_directory.path(), + std::make_unique(".source")); + auto target_codec = std::make_unique(".target"); + ConfigurableCodec *target_codec_ptr = target_codec.get(); + auto target = make_indexed_storage( + target_directory.path(), std::move(target_codec)); + REQUIRE(source.save(root, 47).has_value()); + REQUIRE(target.save(root, 7).has_value()); + target.settings().allow_overwrite = true; + target_codec_ptr->fail_write = true; + + const auto copied = target.copy_from(root, source); + REQUIRE_FALSE(copied.has_value()); + REQUIRE(std::holds_alternative(copied.error())); + CHECK(std::get(copied.error()).operation + == store::CodecOperation::Write); + CHECK_FALSE(target.has(root).value()); + } +} From bca0b5cd591e4e0af13cfcd27c702bd202ba4406 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Fri, 14 Aug 2026 17:40:03 +0200 Subject: [PATCH 58/61] Remove octree migration shims --- docs/raster-store/README.md | 1 + docs/raster-store/architecture.md | 59 ++++++++ docs/raster-store/before-refactor/README.md | 7 + docs/raster-store/refactor-status.md | 34 ++++- docs/raster-store/status-quo.md | 13 ++ docs/raster-store/storage-format.md | 12 ++ src/dag_builder/build.cpp | 30 ++-- src/dag_builder/build.h | 10 +- src/dag_builder/main.cpp | 4 +- src/dag_builder/storage.h | 22 ++- src/dag_builder/thread_safe_storage.h | 4 +- src/dag_convert_debug/main.cpp | 8 +- src/sf_builder/terrainbuilder.cpp | 5 +- src/sf_builder/terrainbuilder.h | 4 +- src/sf_index_browser/main.cpp | 30 ++-- src/sf_merger/NodeLoader.h | 22 +-- src/sf_merger/NodeWriter.h | 20 +-- src/sf_merger/cut.h | 26 ++-- src/sf_merger/main.cpp | 9 +- src/sf_merger/merge.h | 21 +-- src/sf_merger/merge/NodeData.h | 15 +- src/sf_merger/merge/visitor/Masked.h | 4 +- src/sf_merger/merge/visitor/Simple.h | 4 +- src/sf_merger/merge/visitor/Visitor.h | 8 +- src/terrainlib/octree/IndexMap.h | 94 ------------- src/terrainlib/octree/NodeStatus.h | 7 - src/terrainlib/octree/NodeStatusOrMissing.h | 7 - src/terrainlib/octree/Storage.h | 13 -- src/terrainlib/octree/disk/IndexFile.h | 5 +- src/terrainlib/octree/storage/IndexFile.h | 5 +- src/terrainlib/octree/storage/MeshStorage.h | 3 - src/terrainlib/octree/storage/cache/Dummy.h | 11 -- src/terrainlib/octree/storage/cache/ICache.h | 11 -- .../octree/storage/cache/LruCache.h | 11 -- src/terrainlib/octree/storage/open.h | 7 +- src/terrainlib/octree/traverse.h | 52 ------- src/terrainlib/sf/finalize_storage.h | 4 +- unittests/CMakeLists.txt | 2 - .../dag_builder/storage_compatibility.cpp | 16 ++- unittests/sf_merger/storage_boundaries.cpp | 4 +- unittests/terrainlib/index_map.cpp | 100 ------------- unittests/terrainlib/index_traverse.cpp | 132 ------------------ unittests/terrainlib/store_compatibility.cpp | 29 ++-- 43 files changed, 279 insertions(+), 606 deletions(-) delete mode 100644 src/terrainlib/octree/IndexMap.h delete mode 100644 src/terrainlib/octree/NodeStatus.h delete mode 100644 src/terrainlib/octree/NodeStatusOrMissing.h delete mode 100644 src/terrainlib/octree/Storage.h delete mode 100644 src/terrainlib/octree/storage/MeshStorage.h delete mode 100644 src/terrainlib/octree/storage/cache/Dummy.h delete mode 100644 src/terrainlib/octree/storage/cache/ICache.h delete mode 100644 src/terrainlib/octree/storage/cache/LruCache.h delete mode 100644 src/terrainlib/octree/traverse.h delete mode 100644 unittests/terrainlib/index_map.cpp delete mode 100644 unittests/terrainlib/index_traverse.cpp diff --git a/docs/raster-store/README.md b/docs/raster-store/README.md index 04429875..e78c6fd0 100644 --- a/docs/raster-store/README.md +++ b/docs/raster-store/README.md @@ -11,6 +11,7 @@ baseline, not a finalized binary-format specification. - [Architecture](architecture.md) - [Storage format](storage-format.md) - [Sampling and pyramid generation](sampling-and-generation.md) +- [Implemented refactor status](refactor-status.md) ## Plans diff --git a/docs/raster-store/architecture.md b/docs/raster-store/architecture.md index 204c7ac7..9373d754 100644 --- a/docs/raster-store/architecture.md +++ b/docs/raster-store/architecture.md @@ -1,5 +1,64 @@ # Architecture +## Implementation status + +This document describes the intended raster-fundamentalis and tile-base +architecture. Its RF format, snapshot-publication, builder, merger, pyramid, +and server requirements are future work; they were not acceptance criteria +for the completed 2D/3D hierarchy-store refactor. The implemented boundary is +recorded in [refactor-status.md](refactor-status.md), and the original plan is +kept in [refactor-plan.md](refactor-plan.md). + +The refactor delivered the dimension-neutral mechanisms needed by that future +work: + +- `store::Index`, `store::traverse`, layouts, runtime codecs, storage, + typed errors, and cache interfaces; +- `octree::StoreTraits` plus legacy 3D layout/index/open adapters; +- `raster_store::StoreTraits` for in-memory topology keyed by + `radix::tile::Id`; and +- SF-only topology validation under `sf`. + +It deliberately did not define a persistent RF index, RF payload codec, RF +opening API, `rf_builder`, `rf_merger`, tile-base generator, or tile server. + +### Final public names + +The shared API uses `store::NodeStatus`, `store::NodeStatusOrMissing`, +`store::Index`, `store::traverse`, `store::RawStorage`, +`store::Storage`, and `store::IndexedStorage`. Mesh storage aliases live under +`mesh::storage`; DAG batch and metadata aliases live under `dag::storage`. +The `octree` namespace retains the 3D key, traits, path mappings, legacy index +DTO, and mesh-opening compatibility functions. + +A legacy 3D mesh dataset is opened through the 3D adapter: + +```cpp +#include "octree/storage/open.h" + +auto opened = octree::open_folder_indexed(dataset_path); +if (!opened.has_value()) { + return std::unexpected(opened.error()); +} +mesh::storage::IndexedStorage storage = std::move(opened.value()); +``` + +The 2D traits adapter can exercise the shared topology without implying a +persistent RF format: + +```cpp +#include "raster_store/StoreTraits.h" +#include "store/Index.h" +#include "store/traverse.h" + +store::Index index; +const radix::tile::Id tile{2, {1, 3}}; +auto added = index.add(tile); +auto walked = store::traverse(index, [](const auto &id, store::NodeStatus status) { + // In-memory hierarchy processing only. +}); +``` + ## System boundary The design separates authoritative data management from delivery generation: diff --git a/docs/raster-store/before-refactor/README.md b/docs/raster-store/before-refactor/README.md index f9d3d9f0..5a5e146d 100644 --- a/docs/raster-store/before-refactor/README.md +++ b/docs/raster-store/before-refactor/README.md @@ -1,5 +1,12 @@ # SF storage and merge architecture before the 2D/3D refactor +> **Historical report:** this file describes the pre-refactor tree and keeps +> its original names, paths, diagrams, and conclusions for review history. +> Some linked source paths were intentionally removed by the completed +> migration. For the implemented boundary and verification results, see +> [refactor-status.md](../refactor-status.md); for current public names and +> examples, see [architecture.md](../architecture.md#implementation-status). + This report describes the existing Structura Fundamentalis (SF) builder, sparse octree index, storage stack, and mask-based dataset merger. Its purpose is to identify the mechanisms that can be generalised from 3D octrees to a diff --git a/docs/raster-store/refactor-status.md b/docs/raster-store/refactor-status.md index 132225d9..7f4f4b27 100644 --- a/docs/raster-store/refactor-status.md +++ b/docs/raster-store/refactor-status.md @@ -16,10 +16,10 @@ This file is the resumable implementation log for ## Current position -- Phase: 4 — Harden node reuse and enforce SF topology +- Phase: 5 — Cleanup and documentation - State: complete; completion commit pending. -- Next action: commit and tag Phase 4, then begin Phase 5 cleanup and - documentation. +- Next action: commit and tag Phase 5, then remove migration-only fixtures in + Phase 6. ## Phase log @@ -114,6 +114,23 @@ This file is the resumable implementation log for hard links, and newly written changed/clipped nodes. - State: complete. +### Phase 5 — Cleanup and documentation + +- Migrated repository callers from the temporary octree topology, traversal, + storage, and cache aliases to the final `store`, `mesh::storage`, and + `dag::storage` public names. +- Removed the temporary `octree::IndexMap`, status, traversal, mesh-storage, + and cache forwarding headers and their wrapper-only unit tests. +- Changed the unchanged legacy `terrain.index` DTO to serialize the shared + `store::Index` directly. +- Kept only 3D identity/geometry, traits, mappings, legacy index-format, and + opening adapters under `octree`. +- Updated the architecture, storage-format, status-quo, historical + before-refactor report, and raster-store index documentation. The docs now + distinguish the implemented shared boundary from future RF formats and + tools and include final public names plus 3D-open and in-memory 2D examples. +- State: complete. + ## Verification log - Baseline `unittests_terrainlib`: 23,676 assertions in 391 test cases passed. @@ -167,6 +184,15 @@ This file is the resumable implementation log for `radix::tile::Scheme` remains isolated; the new finalization tests build and pass in `unittests_sfbuilder_finalization`. - Phase 4 `git diff --check`: passed. +- Phase 5 `unittests_terrainlib`: 24,017 assertions in 426 test cases passed. +- Phase 5 `unittests_dagbuilder`: 472 assertions in 74 test cases passed. +- Phase 5 `unittests_sfmerger`: 121 assertions in 8 test cases passed. +- Phase 5 SF builder finalization: 19 assertions in 2 test cases passed. +- Phase 5 application builds passed for `sf-builder`, `sf-merger`, + `sf-index-browser`, `dag-builder`, and `dag-convert-debug`. +- Phase 5 repository searches found no remaining caller of a removed octree + forwarding name and no `octree::Id` dependency under `terrainlib/store`. +- Phase 5 `git diff --check`: passed. ## Commit and tag log @@ -184,3 +210,5 @@ This file is the resumable implementation log for - `61acb74` — Phase 3 shared runtime-codec storage core. - `4ffcdea` / `refactor_phase_3` — Phase 3 application migration and completion tag. +- `3086336` / `refactor_phase_4` — hardened node reuse, SF topology + enforcement, and Phase 4 completion tag. diff --git a/docs/raster-store/status-quo.md b/docs/raster-store/status-quo.md index 9d2b5701..461faf46 100644 --- a/docs/raster-store/status-quo.md +++ b/docs/raster-store/status-quo.md @@ -1,5 +1,18 @@ # Status quo and reuse assessment +> **Historical assessment:** this document records the repository before the +> shared-store refactor and is intentionally not rewritten as current code +> documentation. The completed implementation and verification record is in +> [refactor-status.md](refactor-status.md); the resulting public boundary is +> summarized in [architecture.md](architecture.md#implementation-status). + +The refactor chose the generalization option discussed below. Sparse topology, +traversal, layouts, runtime codecs, storage, index lifecycle, and cache +interfaces now live under `store` and are parameterized by hierarchy traits. +The legacy 3D format remains an adapter under `octree`, while +`raster_store::StoreTraits` proves the shared in-memory mechanisms with +`radix::tile::Id`. Persistent RF formats and RF tools remain future work. + This document evaluates the current repository after commit `9cf9065` (`Consolidate raster handling and use std::expected`). It distinguishes reusable mechanisms from interfaces that encode assumptions unsuitable for diff --git a/docs/raster-store/storage-format.md b/docs/raster-store/storage-format.md index 5767e0f8..0332e8e5 100644 --- a/docs/raster-store/storage-format.md +++ b/docs/raster-store/storage-format.md @@ -1,5 +1,17 @@ # Storage format +> **Status:** future raster-store design. The completed 2D/3D store refactor +> did not implement or finalize the RF/TB formats described below. These +> requirements were not acceptance criteria for that refactor. See +> [refactor-status.md](refactor-status.md) for the implemented shared boundary. + +The refactor preserves the existing 3D `terrain.index` and payload contracts +through the adapters under `octree` and `mesh::storage`. It supplies only an +in-memory `raster_store::StoreTraits` adapter for `radix::tile::Id`; there is +currently no persistent RF index, `.amort` codec, snapshot publisher, or RF +opening API. Names and formats in this document remain proposals until their +own implementation plans approve them. + This document specifies the logical format and required invariants for the following data stores: - raster-fundamentalis (our authoritative raster store) - tile-base (the tile-pyramid for our tile-server) diff --git a/src/dag_builder/build.cpp b/src/dag_builder/build.cpp index 7653df1b..d8e265a6 100644 --- a/src/dag_builder/build.cpp +++ b/src/dag_builder/build.cpp @@ -26,9 +26,8 @@ #include "octree/IdRect.h" #include "octree/OddLevelShifted.h" #include "octree/Space.h" -#include "octree/Storage.h" #include "octree/storage/open.h" -#include "octree/traverse.h" +#include "store/traverse.h" #include "ProgressIndicator.h" #include "partition.h" #include "simplify.h" @@ -49,7 +48,7 @@ namespace { // Load a mesh from storage and clusterize it. std::optional load_and_clusterize_mesh( - const octree::MeshStorage &storage, + const mesh::storage::Storage &storage, const octree::Id &id) { const auto result = storage.load(id); @@ -127,7 +126,7 @@ LodResult build_lod( // Load input meshes, clusterize, and filter them to the target region. std::vector load_input_clusters( const std::span input_ids, - const octree::MeshStorage &input_storage, + const mesh::storage::Storage &input_storage, const RegionFilter& filter) { std::vector result; @@ -154,8 +153,8 @@ std::vector load_input_clusters( // Dependencies shared across the whole dag building pipeline. struct BuildContext { - const octree::IndexedMeshStorage &input_storage; - ThreadSafeStorage output_storage; + const mesh::storage::IndexedStorage &input_storage; + ThreadSafeStorage output_storage; const BuildOptions &options; const octree::Space &space; const octree::OddLevelShifted &shifted_space; @@ -462,10 +461,10 @@ std::vector> gather_relevant_input_leaves( const auto start = space.find_smallest_node_encompassing_bounds(root_bounds) .value_or(octree::Id::root()); std::vector> result(octree::Id::max_level() + 1); - octree::traverse( + const auto traversal = store::traverse( index, - [&](const octree::Id &id, const octree::NodeStatus status) { - if (status == octree::NodeStatus::Leaf && radix::geometry::intersect(root_bounds, space.get_node_bounds(id))) { + [&](const octree::Id &id, const store::NodeStatus status) { + if (status == store::NodeStatus::Leaf && radix::geometry::intersect(root_bounds, space.get_node_bounds(id))) { result[id.level()].push_back(id); } }, @@ -473,6 +472,7 @@ std::vector> gather_relevant_input_leaves( return radix::geometry::intersect(root_bounds, space.get_node_bounds(id)); }, start); + DEBUG_ASSERT(traversal.has_value()); return result; } @@ -518,7 +518,7 @@ std::unordered_set build_level( } // Initialize debug storage if requested (contains .glb meshes) - std::optional debug_storage; + std::optional debug_storage; if (ctx.options.write_debug_meshes) { octree::OpenOptions options; options.preferred_extension = ".glb"; @@ -586,8 +586,8 @@ std::unordered_set build_level( // Iterates octree levels from finest to coarsest, simplifying and re-clustering geometry at each // level from its children. std::expected build_levels( - const octree::IndexedMeshStorage &input_storage, - octree::IndexedDagStorage &output_storage, + const mesh::storage::IndexedStorage &input_storage, + dag::storage::IndexedStorage &output_storage, const BuildOptions &options, const AnyRange &level_range) { const auto validation = sf::validate_index(input_storage.index()); @@ -619,7 +619,7 @@ std::expected build_levels( // TODO: use hierachical lookup here based on root_bounds and IdRect std::unordered_set prev_level_built; for (const auto &[id, status] : output_storage.index()) { - if (id.level() == range.end && status != octree::NodeStatus::Virtual) { + if (id.level() == range.end && status != store::NodeStatus::Virtual) { prev_level_built.insert(id); } } @@ -648,8 +648,8 @@ std::expected build_levels( // Builds the complete DAG from input_storage into output_storage. std::expected build_full( - const octree::IndexedMeshStorage &input_storage, - octree::IndexedDagStorage &output_storage, + const mesh::storage::IndexedStorage &input_storage, + dag::storage::IndexedStorage &output_storage, const BuildOptions &options) { return build_levels(input_storage, output_storage, options, RangeFull{}); } diff --git a/src/dag_builder/build.h b/src/dag_builder/build.h index b87528e7..a2b774a4 100644 --- a/src/dag_builder/build.h +++ b/src/dag_builder/build.h @@ -8,7 +8,7 @@ #include "Range.h" #include "build_config.h" #include "texturing.h" -#include "octree/storage/MeshStorage.h" +#include "mesh/storage.h" #include "storage.h" #include "sf/InvalidTopology.h" @@ -33,13 +33,13 @@ struct BuildOptions { }; std::expected build_full( - const octree::IndexedMeshStorage &input_storage, - octree::IndexedDagStorage &output_storage, + const mesh::storage::IndexedStorage &input_storage, + dag::storage::IndexedStorage &output_storage, const BuildOptions &options); std::expected build_levels( - const octree::IndexedMeshStorage &input_storage, - octree::IndexedDagStorage &output_storage, + const mesh::storage::IndexedStorage &input_storage, + dag::storage::IndexedStorage &output_storage, const BuildOptions &options, const AnyRange &level_range); diff --git a/src/dag_builder/main.cpp b/src/dag_builder/main.cpp index 4fb1d70a..b733c3c6 100644 --- a/src/dag_builder/main.cpp +++ b/src/dag_builder/main.cpp @@ -33,8 +33,8 @@ int main(int argc, char **argv) { store::describe_error(output_result.error())); return EXIT_FAILURE; } - const octree::IndexedMeshStorage input_storage = std::move(input_result.value()); - octree::IndexedDagStorage output_storage = std::move(output_result.value()); + const mesh::storage::IndexedStorage input_storage = std::move(input_result.value()); + dag::storage::IndexedStorage output_storage = std::move(output_result.value()); output_storage.settings().allow_overwrite = args.continuation_mode == ContinuationMode::Overwrite; dag::BuildOptions options{ diff --git a/src/dag_builder/storage.h b/src/dag_builder/storage.h index 92bccec9..b3638172 100644 --- a/src/dag_builder/storage.h +++ b/src/dag_builder/storage.h @@ -10,20 +10,16 @@ #include "serialization.h" #include "store/IndexedStorage.h" -namespace octree { - -using DagStorage = store::Storage; -using IndexedDagStorage = store::IndexedStorage; -using DagMetaStorage = store::Storage; -using IndexedDagMetaStorage = store::IndexedStorage; - -} // namespace octree - namespace dag::storage { using OpenOptions = octree::storage::OpenOptions; +using Storage = store::Storage; +using IndexedStorage = store::IndexedStorage; +using MetadataStorage = store::Storage; +using IndexedMetadataStorage = + store::IndexedStorage; -inline std::expected> open_index( +inline std::expected> open_index( const std::filesystem::path &path) { return store::open_index( path, @@ -31,7 +27,7 @@ inline std::expected> op dag::codec::from_extension); } -inline std::expected> open_folder( +inline std::expected> open_folder( const std::filesystem::path &path, const bool create_index = false, OpenOptions options = {}) { @@ -43,7 +39,7 @@ inline std::expected> open_fold std::move(options)); } -inline std::expected> +inline std::expected> open_folder_indexed(const std::filesystem::path &path, OpenOptions options = {}) { return octree::storage::open_folder_indexed( path, @@ -52,7 +48,7 @@ open_folder_indexed(const std::filesystem::path &path, OpenOptions options = {}) std::move(options)); } -inline std::expected> +inline std::expected> open_metadata_indexed(const std::filesystem::path &path) { const std::filesystem::path index_path = path.filename() == octree::disk::v1::index_file_name() diff --git a/src/dag_builder/thread_safe_storage.h b/src/dag_builder/thread_safe_storage.h index da54f8a0..9a4c483d 100644 --- a/src/dag_builder/thread_safe_storage.h +++ b/src/dag_builder/thread_safe_storage.h @@ -8,8 +8,8 @@ namespace dag { -// A mutex-based thread-safe wrapper around a Storage_ instance. -// TODO: move this locking into Storage_ itself and remove this wrapper once that lands. +// A mutex-based thread-safe wrapper around a Storage instance. +// TODO: move this locking into Storage itself and remove this wrapper once that lands. template class ThreadSafeStorage { static_assert(!std::is_const_v, "ThreadSafeStorage requires a non-const Storage"); diff --git a/src/dag_convert_debug/main.cpp b/src/dag_convert_debug/main.cpp index d72be3a2..f4901974 100644 --- a/src/dag_convert_debug/main.cpp +++ b/src/dag_convert_debug/main.cpp @@ -7,7 +7,7 @@ #include "io/serialize.h" #include "log.h" #include "mesh/io.h" -#include "octree/storage/MeshStorage.h" +#include "mesh/storage.h" #include "octree/storage/open.h" #include "ProgressIndicator.h" #include "storage.h" @@ -41,7 +41,7 @@ void export_storage(const cli::Args &args) { store::describe_error(input_result.error())); return; } - const octree::IndexedDagStorage input_storage = std::move(input_result.value()); + const dag::storage::IndexedStorage input_storage = std::move(input_result.value()); octree::OpenOptions options; options.preferred_extension = ".glb"; @@ -56,7 +56,7 @@ void export_storage(const cli::Args &args) { store::describe_error(output_result.error())); return; } - octree::MeshStorage output_storage = std::move(output_result.value()); + mesh::storage::Storage output_storage = std::move(output_result.value()); output_storage.settings().allow_overwrite = true; size_t exported_count = 0; @@ -65,7 +65,7 @@ void export_storage(const cli::Args &args) { auto progress_thread = progress.start_monitoring(); for (const auto &[id, status] : input_storage.index()) { - if (status == octree::NodeStatus::Virtual) { + if (status == store::NodeStatus::Virtual) { progress.task_finished(); continue; } diff --git a/src/sf_builder/terrainbuilder.cpp b/src/sf_builder/terrainbuilder.cpp index 438da793..310e384d 100644 --- a/src/sf_builder/terrainbuilder.cpp +++ b/src/sf_builder/terrainbuilder.cpp @@ -41,7 +41,6 @@ #include "octree/Id.h" #include "octree/Space.h" -#include "octree/Storage.h" #include "octree/storage/open.h" #include "octree/utils.h" #include "store/describe_error.h" @@ -174,7 +173,7 @@ T expect(const std::optional &opt, const std::string &msg) { } } -std::expected finalize_storage(octree::Storage &storage) { +std::expected finalize_storage(mesh::storage::Storage &storage) { return sf::finalize_storage(storage); } @@ -207,7 +206,7 @@ std::expected build_all_patches( output_base_path, store::describe_error(storage_result.error())); } - octree::Storage storage = std::move(storage_result.value()); + mesh::storage::Storage storage = std::move(storage_result.value()); storage.settings().allow_overwrite = overwrite_existing; const auto dataset_srs = dataset.srs(); diff --git a/src/sf_builder/terrainbuilder.h b/src/sf_builder/terrainbuilder.h index 35d51b29..8ddbeba0 100644 --- a/src/sf_builder/terrainbuilder.h +++ b/src/sf_builder/terrainbuilder.h @@ -9,7 +9,7 @@ #include "octree/Id.h" #include "mesh/SimpleMesh.h" #include "tile_provider.h" -#include "octree/Storage.h" +#include "mesh/storage.h" #include "sf/Error.h" namespace terrainbuilder { @@ -31,7 +31,7 @@ std::optional build_patch( const TileProvider *tile_provider, const OGRSpatialReference &mesh_srs); -std::expected finalize_storage(octree::Storage &storage); +std::expected finalize_storage(mesh::storage::Storage &storage); std::expected build_all_patches( Dataset &dataset, diff --git a/src/sf_index_browser/main.cpp b/src/sf_index_browser/main.cpp index 0c6e6ddd..f1008ff3 100644 --- a/src/sf_index_browser/main.cpp +++ b/src/sf_index_browser/main.cpp @@ -13,18 +13,16 @@ #include "log.h" #include "octree/Id.h" -#include "octree/NodeStatus.h" -#include "octree/NodeStatusOrMissing.h" -#include "octree/Storage.h" +#include "mesh/storage.h" #include "octree/storage/open.h" -#include "octree/traverse.h" #include "cli.h" #include "store/describe_error.h" +#include "store/NodeStatusOrMissing.h" namespace { struct IndexNode { octree::Id id; - octree::NodeStatusOrMissing status; + store::NodeStatusOrMissing status; bool expanded = false; }; struct MeshNode { @@ -44,8 +42,8 @@ class TreeView { using iterator = std::list::iterator; using const_iterator = std::list::const_iterator; - explicit TreeView(const octree::IndexedStorage &storage, const octree::Id root) : root(root), _storage(storage) { - const octree::NodeStatusOrMissing status( + explicit TreeView(const mesh::storage::IndexedStorage &storage, const octree::Id root) : root(root), _storage(storage) { + const store::NodeStatusOrMissing status( DEBUG_ASSERT_VAL(this->_storage.index().get(root)).value()); this->_view.emplace_back(IndexNode{root, status, false}); } @@ -82,7 +80,7 @@ class TreeView { } parent.expanded = true; - if (parent.status == octree::NodeStatusOrMissing::Leaf) { + if (parent.status == store::NodeStatusOrMissing::Leaf) { auto result = this->_storage.load(parent.id); DisplayEntry entry; if (result.has_value()) { @@ -106,7 +104,7 @@ class TreeView { if (!status_result->has_value()) { continue; } - const octree::NodeStatus status = status_result->value(); + const store::NodeStatus status = status_result->value(); IndexNode child_node{child_id, status, false}; it++; it = this->_view.emplace(it, child_node); @@ -128,7 +126,7 @@ class TreeView { } parent.expanded = false; - if (parent.status == octree::NodeStatusOrMissing::Leaf) { + if (parent.status == store::NodeStatusOrMissing::Leaf) { it++; it = this->_view.erase(it); return; @@ -150,16 +148,16 @@ class TreeView { private: std::list _view; - const octree::IndexedStorage& _storage; + const mesh::storage::IndexedStorage& _storage; }; const octree::Id find_deepest_root( const store::Index &index, const octree::Id &root = octree::Id::root()) { - switch (octree::NodeStatusOrMissing(DEBUG_ASSERT_VAL(index.get(root)).value())) { - case octree::NodeStatusOrMissing::Missing: + switch (store::NodeStatusOrMissing(DEBUG_ASSERT_VAL(index.get(root)).value())) { + case store::NodeStatusOrMissing::Missing: return octree::Id::root(); - case octree::NodeStatusOrMissing::Leaf: + case store::NodeStatusOrMissing::Leaf: return root; default: break; @@ -191,7 +189,7 @@ const octree::Id find_deepest_root( } // namespace -octree::IndexedStorage open_path_indexed(const std::filesystem::path& path) { +mesh::storage::IndexedStorage open_path_indexed(const std::filesystem::path& path) { auto result = std::filesystem::is_directory(path) ? octree::open_folder_indexed(path) : octree::open_index(path); @@ -232,7 +230,7 @@ ftxui::Element render_line(const DisplayEntry &entry, size_t base_level, bool is } int run(const cli::Args &args) { - const octree::IndexedStorage storage = open_path_indexed(args.dataset_path); + const mesh::storage::IndexedStorage storage = open_path_indexed(args.dataset_path); const store::Index &index = storage.index(); const octree::Id root_id = args.full_view ? octree::Id::root() : find_deepest_root(index); diff --git a/src/sf_merger/NodeLoader.h b/src/sf_merger/NodeLoader.h index c7c44b90..23a2cbd1 100644 --- a/src/sf_merger/NodeLoader.h +++ b/src/sf_merger/NodeLoader.h @@ -9,19 +9,21 @@ #include "mesh/clip.h" #include "mesh/texture_trim.h" #include "octree/Id.h" -#include "octree/NodeStatusOrMissing.h" #include "octree/Space.h" -#include "octree/Storage.h" -#include "octree/storage/cache/Dummy.h" -#include "octree/storage/cache/ICache.h" +#include "mesh/storage.h" +#include "store/NodeStatusOrMissing.h" +#include "store/cache/Interface.h" class NodeLoader { public: - NodeLoader(const octree::IndexedStorage &storage, octree::cache::ICache &cache, octree::Space space) + NodeLoader( + const mesh::storage::IndexedStorage &storage, + store::cache::Interface &cache, + octree::Space space) : _storage(storage), _cache(cache), _space(space) {} - octree::NodeStatusOrMissing get_status(const octree::Id &id) const noexcept { - return octree::NodeStatusOrMissing( + store::NodeStatusOrMissing get_status(const octree::Id &id) const noexcept { + return store::NodeStatusOrMissing( DEBUG_ASSERT_VAL(this->_storage.index().get(id)).value()); } @@ -66,12 +68,12 @@ class NodeLoader { return std::nullopt; } - const octree::IndexedStorage &storage() const { + const mesh::storage::IndexedStorage &storage() const { return this->_storage; } private: - const octree::IndexedStorage &_storage; - octree::cache::ICache &_cache; + const mesh::storage::IndexedStorage &_storage; + store::cache::Interface &_cache; const octree::Space _space; }; diff --git a/src/sf_merger/NodeWriter.h b/src/sf_merger/NodeWriter.h index 9c51359e..6a29a556 100644 --- a/src/sf_merger/NodeWriter.h +++ b/src/sf_merger/NodeWriter.h @@ -8,14 +8,13 @@ #include "NodeLoader.h" #include "mesh/SimpleMesh.h" #include "octree/Id.h" -#include "octree/NodeStatus.h" -#include "octree/Storage.h" -#include "octree/traverse.h" +#include "mesh/storage.h" +#include "store/traverse.h" // TODO: make thread safe class NodeWriter { public: - NodeWriter(octree::Storage &storage) : _storage(storage) {} + NodeWriter(mesh::storage::Storage &storage) : _storage(storage) {} std::expected> has_node( const octree::Id &id) { @@ -47,16 +46,16 @@ class NodeWriter { const octree::Id &id, const NodeLoader &loader) { std::optional> error; - octree::traverse( + const auto traversal = store::traverse( loader.storage().index(), - [&](const octree::Id &child_id, const octree::NodeStatus &status) { + [&](const octree::Id &child_id, const store::NodeStatus &status) { if (error.has_value()) { return; } - if (status == octree::NodeStatus::Virtual) { + if (status == store::NodeStatus::Virtual) { return; } - DEBUG_ASSERT(status == octree::NodeStatus::Leaf); + DEBUG_ASSERT(status == store::NodeStatus::Leaf); const auto result = this->_storage.copy_from(child_id, loader.storage()); if (!result.has_value()) { @@ -65,6 +64,9 @@ class NodeWriter { }, [&](const octree::Id &) { return !error.has_value(); }, id); + if (!traversal.has_value()) { + return std::unexpected(store::CopyError(traversal.error())); + } if (error.has_value()) { return std::unexpected(std::move(error.value())); } @@ -72,5 +74,5 @@ class NodeWriter { } private: - octree::Storage &_storage; + mesh::storage::Storage &_storage; }; diff --git a/src/sf_merger/cut.h b/src/sf_merger/cut.h index 3c6ce4d6..6ecca34c 100644 --- a/src/sf_merger/cut.h +++ b/src/sf_merger/cut.h @@ -8,11 +8,9 @@ #include "log.h" #include "mask.h" #include "octree/Id.h" -#include "octree/NodeStatus.h" -#include "octree/Storage.h" +#include "mesh/storage.h" #include "octree/storage/open.h" #include "octree/Space.h" -#include "octree/traverse.h" #include "utils.h" #include "containers/Cow.h" #include "store/describe_error.h" @@ -21,8 +19,8 @@ #include "sf/validate_index.h" struct Context { - const octree::IndexedStorage& input; - octree::Storage& output; + const mesh::storage::IndexedStorage& input; + mesh::storage::Storage& output; const octree::Space space; const bool keep_inside; }; @@ -38,7 +36,7 @@ inline std::expected cut_leaf_node( const MeshMask& mask ) { DEBUG_ASSERT(DEBUG_ASSERT_VAL( - ctx.input.index().is(octree::NodeStatus::Leaf, id)).value()); + ctx.input.index().is(store::NodeStatus::Leaf, id)).value()); const SimpleMesh mesh = DEBUG_ASSERT_VAL(ctx.input.load(id)).value(); LOG_TRACE("Cutting mesh at {} using mask with {} vertices and {} triangles", @@ -72,7 +70,7 @@ inline std::expected cut_virtual_node( const MeshMask& mask ) { DEBUG_ASSERT(DEBUG_ASSERT_VAL( - ctx.input.index().is(octree::NodeStatus::Virtual, id)).value()); + ctx.input.index().is(store::NodeStatus::Virtual, id)).value()); DEBUG_ASSERT(id.has_children()); const auto children = id.children().value(); @@ -113,11 +111,11 @@ inline std::expected cut_node( return {}; } - const octree::NodeStatus status = status_result->value(); + const store::NodeStatus status = status_result->value(); switch (status) { - case octree::NodeStatus::Virtual: + case store::NodeStatus::Virtual: return cut_virtual_node(ctx, id, mask); - case octree::NodeStatus::Leaf: + case store::NodeStatus::Leaf: return cut_leaf_node(ctx, id, mask); default: UNREACHABLE(); @@ -127,9 +125,9 @@ inline std::expected cut_node( } inline std::expected cut_dataset( - const octree::IndexedStorage &input, + const mesh::storage::IndexedStorage &input, const MeshMask& mask, - octree::Storage &output, + mesh::storage::Storage &output, const bool keep_inside) { const auto validation = sf::validate_index(input.index()); if (!validation.has_value()) { @@ -150,7 +148,7 @@ inline std::expected cut_dataset( } inline std::expected cut_dataset( - const octree::IndexedStorage &input_dataset, + const mesh::storage::IndexedStorage &input_dataset, const MeshMask& mask, const std::filesystem::path &output_path, const bool keep_inside) { @@ -164,7 +162,7 @@ inline std::expected cut_dataset( if (!output_result.has_value()) { return std::unexpected(sf::ProcessingError(output_result.error())); } - octree::IndexedStorage output_dataset = std::move(output_result.value()); + mesh::storage::IndexedStorage output_dataset = std::move(output_result.value()); if (!output_dataset.index().empty()) { return std::unexpected(sf::ProcessingError(store::SaveError( store::AlreadyExists{output_path}))); diff --git a/src/sf_merger/main.cpp b/src/sf_merger/main.cpp index bae30669..3d4f5bcd 100644 --- a/src/sf_merger/main.cpp +++ b/src/sf_merger/main.cpp @@ -8,7 +8,6 @@ #include "log.h" #include "mask.h" #include "merge.h" -#include "octree/Storage.h" #include "optional_utils.h" #include "earth.h" #include "store/describe_error.h" @@ -43,7 +42,7 @@ void run(const cli::MergeArgs& args) { args.base_path, store::describe_error(base_result.error())); } - octree::IndexedStorage base_dataset = std::move(base_result.value()); + mesh::storage::IndexedStorage base_dataset = std::move(base_result.value()); LOG_TRACE("Loading new dataset from {}", args.new_path); auto new_result = octree::open_folder_indexed(args.new_path); @@ -53,7 +52,7 @@ void run(const cli::MergeArgs& args) { args.new_path, store::describe_error(new_result.error())); } - octree::IndexedStorage new_dataset = std::move(new_result.value()); + mesh::storage::IndexedStorage new_dataset = std::move(new_result.value()); LOG_TRACE("Creating output dataset at {}", args.output_path); std::filesystem::create_directories(args.output_path); @@ -67,7 +66,7 @@ void run(const cli::MergeArgs& args) { args.output_path, store::describe_error(output_result.error())); } - octree::Storage output_dataset = std::move(output_result.value()); + mesh::storage::Storage output_dataset = std::move(output_result.value()); std::optional mask = flatten(map(args.mask_path, load_mask_from_path)); @@ -90,7 +89,7 @@ void run(const cli::CutArgs& args) { args.input_path, store::describe_error(input_result.error())); } - const octree::IndexedStorage input_dataset = std::move(input_result.value()); + const mesh::storage::IndexedStorage input_dataset = std::move(input_result.value()); const MeshMask mask = DEBUG_ASSERT_VAL(load_mask_from_path(args.mask_path)).value(); const auto cut_result = cut_dataset( input_dataset, diff --git a/src/sf_merger/merge.h b/src/sf_merger/merge.h index 6987d14d..bef384ac 100644 --- a/src/sf_merger/merge.h +++ b/src/sf_merger/merge.h @@ -13,14 +13,15 @@ #include "merge/visitor/Simple.h" #include "merge/visitor/Visitor.h" #include "octree/Id.h" -#include "octree/NodeStatusOrMissing.h" -#include "octree/storage/cache/Dummy.h" +#include "mesh/storage.h" +#include "store/NodeStatusOrMissing.h" +#include "store/cache/Dummy.h" #include "store/describe_error.h" #include "sf/Error.h" #include "sf/finalize_storage.h" #include "sf/validate_index.h" -inline std::string get_dataset_name(const octree::Storage &storage) { +inline std::string get_dataset_name(const mesh::storage::Storage &storage) { return storage.layout().base_path().filename().string(); } @@ -34,7 +35,7 @@ using smallest_uint_t = template class Merger { public: - using Status = octree::NodeStatusOrMissing; + using Status = store::NodeStatusOrMissing; using Context = Visitor::Context; using Result = merge::Result; using Expected = std::expected; @@ -182,11 +183,11 @@ class Merger { }; inline std::expected merge_datasets( - const octree::IndexedStorage &left_dataset, - const octree::IndexedStorage &right_dataset, - octree::Storage &output_dataset, + const mesh::storage::IndexedStorage &left_dataset, + const mesh::storage::IndexedStorage &right_dataset, + mesh::storage::Storage &output_dataset, const std::optional mask = std::nullopt) { - for (const octree::IndexedStorage *input : {&left_dataset, &right_dataset}) { + for (const mesh::storage::IndexedStorage *input : {&left_dataset, &right_dataset}) { const auto validation = sf::validate_index(input->index()); if (!validation.has_value()) { return std::unexpected(sf::ProcessingError(validation.error())); @@ -199,9 +200,9 @@ inline std::expected merge_datasets( get_dataset_name(output_dataset)); octree::Space space = octree::Space::earth(); - octree::cache::Dummy left_cache; + store::cache::Dummy left_cache; NodeLoader left(left_dataset, left_cache, space); - octree::cache::Dummy right_cache; + store::cache::Dummy right_cache; NodeLoader right(right_dataset, right_cache, space); NodeWriter output(output_dataset); if (mask.has_value()) { diff --git a/src/sf_merger/merge/NodeData.h b/src/sf_merger/merge/NodeData.h index c3c9de13..2bdd72f1 100644 --- a/src/sf_merger/merge/NodeData.h +++ b/src/sf_merger/merge/NodeData.h @@ -6,18 +6,17 @@ #include "log.h" #include "mesh/SimpleMesh.h" #include "octree/Id.h" -#include "octree/NodeStatusOrMissing.h" -#include "octree/Storage.h" #include "optional_utils.h" +#include "store/NodeStatusOrMissing.h" namespace merge { -template +template class NodeData { public: constexpr explicit NodeData(octree::Id id, const NodeLoader &loader) : _id(id), _loader(loader) {} - static constexpr octree::NodeStatusOrMissing status() { + static constexpr store::NodeStatusOrMissing status() { return Status; } @@ -26,8 +25,8 @@ class NodeData { } // Status::Leaf or Status::Inner guarantuess a node is present - template - std::enable_if_t + template + std::enable_if_t mesh() const { auto mesh = this->load_mesh(); if (mesh.has_value()) { @@ -38,8 +37,8 @@ class NodeData { } // Status::Missing and Status::Virtual do not exist on disk, but may be the child of a leaf or inner node - template - std::enable_if_t>> + template + std::enable_if_t>> mesh() const { return this->load_mesh(); } diff --git a/src/sf_merger/merge/visitor/Masked.h b/src/sf_merger/merge/visitor/Masked.h index 69936bfd..3c095c40 100644 --- a/src/sf_merger/merge/visitor/Masked.h +++ b/src/sf_merger/merge/visitor/Masked.h @@ -11,7 +11,7 @@ #include "mesh/merging/helpers.h" #include "mesh/merging/mapping.h" #include "octree/Id.h" -#include "octree/NodeStatusOrMissing.h" +#include "store/NodeStatusOrMissing.h" #include "polygon/Polygon.h" #include "polygon/triangulate.h" #include "spatial_lookup/Hashmap.h" @@ -26,7 +26,7 @@ namespace merge::visitor { class Masked { public: - using Status = octree::NodeStatusOrMissing; + using Status = store::NodeStatusOrMissing; struct Context { MeshMask mask; bool has_left_parent; diff --git a/src/sf_merger/merge/visitor/Simple.h b/src/sf_merger/merge/visitor/Simple.h index 531cd686..9305ce7b 100644 --- a/src/sf_merger/merge/visitor/Simple.h +++ b/src/sf_merger/merge/visitor/Simple.h @@ -5,13 +5,13 @@ #include "merge/visitor/Visitor.h" #include "mesh/merge.h" #include "octree/Id.h" -#include "octree/NodeStatusOrMissing.h" +#include "store/NodeStatusOrMissing.h" namespace merge::visitor { class Simple { public: - using Status = octree::NodeStatusOrMissing; + using Status = store::NodeStatusOrMissing; struct Context{}; using Result = merge::Result; diff --git a/src/sf_merger/merge/visitor/Visitor.h b/src/sf_merger/merge/visitor/Visitor.h index 35206fe4..6607e222 100644 --- a/src/sf_merger/merge/visitor/Visitor.h +++ b/src/sf_merger/merge/visitor/Visitor.h @@ -3,7 +3,7 @@ #include "merge/NodeData.h" #include "merge/Result.h" #include "octree/Id.h" -#include "octree/NodeStatusOrMissing.h" +#include "store/NodeStatusOrMissing.h" namespace merge { template @@ -15,10 +15,10 @@ concept Visitor = requires(T t, const octree::Id &id) { { t.make_root_context() } -> std::same_as; { - t.template visit( + t.template visit( id, - std::declval &>(), - std::declval &>(), + std::declval &>(), + std::declval &>(), std::declval()) } -> std::same_as>; }; diff --git a/src/terrainlib/octree/IndexMap.h b/src/terrainlib/octree/IndexMap.h deleted file mode 100644 index c45d7da1..00000000 --- a/src/terrainlib/octree/IndexMap.h +++ /dev/null @@ -1,94 +0,0 @@ -#pragma once - -#include - -#include - -#include "octree/NodeStatus.h" -#include "octree/StoreTraits.h" -#include "store/Index.h" - -namespace octree { - -class IndexMap { -public: - using SharedIndex = store::Index; - using iterator = SharedIndex::iterator; - using const_iterator = SharedIndex::const_iterator; - - IndexMap() = default; - explicit IndexMap(SharedIndex index) : _index(std::move(index)) {} - - std::optional get(const Id id) const { - return _index.get(id).value(); - } - bool add(const Id id) { - return _index.add(id).value(); - } - bool remove(const Id id) { - return _index.remove(id).value(); - } - bool is_present(const Id id) const { - return _index.is_present(id).value(); - } - bool is_absent(const Id id) const { - return _index.is_absent(id).value(); - } - bool is(const NodeStatus status, const Id id) const { - return _index.is(status, id).value(); - } - - void clear() { - _index.clear(); - } - bool empty() const { - return _index.empty(); - } - size_t size() const { - return _index.size(); - } - - const_iterator begin() const { - return _index.begin(); - } - const_iterator end() const { - return _index.end(); - } - const_iterator cbegin() const { - return _index.cbegin(); - } - const_iterator cend() const { - return _index.cend(); - } - - NodeStatus *get_raw(const Id id) { - return _index.get_raw(id).value(); - } - const NodeStatus *get_raw(const Id id) const { - return _index.get_raw(id).value(); - } - void set_raw(const Id id, const NodeStatus status) { - _index.set_raw(id, status).value(); - } - void remove_raw(const Id id) { - _index.remove_raw(id).value(); - } - - SharedIndex &shared() { - return _index; - } - const SharedIndex &shared() const { - return _index; - } - SharedIndex take_shared() && { - return std::move(_index); - } - - using serialize = zpp::bits::members<1>; - friend zpp::bits::access; - -private: - SharedIndex _index; -}; - -} // namespace octree diff --git a/src/terrainlib/octree/NodeStatus.h b/src/terrainlib/octree/NodeStatus.h deleted file mode 100644 index 185f4fdd..00000000 --- a/src/terrainlib/octree/NodeStatus.h +++ /dev/null @@ -1,7 +0,0 @@ -#pragma once - -#include "store/NodeStatus.h" - -namespace octree { -using NodeStatus = store::NodeStatus; -} diff --git a/src/terrainlib/octree/NodeStatusOrMissing.h b/src/terrainlib/octree/NodeStatusOrMissing.h deleted file mode 100644 index 1c71a67d..00000000 --- a/src/terrainlib/octree/NodeStatusOrMissing.h +++ /dev/null @@ -1,7 +0,0 @@ -#pragma once - -#include "store/NodeStatusOrMissing.h" - -namespace octree { -using NodeStatusOrMissing = store::NodeStatusOrMissing; -} diff --git a/src/terrainlib/octree/Storage.h b/src/terrainlib/octree/Storage.h deleted file mode 100644 index adb5181e..00000000 --- a/src/terrainlib/octree/Storage.h +++ /dev/null @@ -1,13 +0,0 @@ -#pragma once - -#include "mesh/storage.h" - -namespace octree { - -using MeshStorage = mesh::storage::Storage; -using IndexedMeshStorage = mesh::storage::IndexedStorage; -using Storage = MeshStorage; -using IndexedStorage = IndexedMeshStorage; -using StorageSettings = store::StorageSettings; - -} // namespace octree diff --git a/src/terrainlib/octree/disk/IndexFile.h b/src/terrainlib/octree/disk/IndexFile.h index 478be69f..92baacd9 100644 --- a/src/terrainlib/octree/disk/IndexFile.h +++ b/src/terrainlib/octree/disk/IndexFile.h @@ -4,7 +4,8 @@ #include -#include "octree/IndexMap.h" +#include "octree/StoreTraits.h" +#include "store/Index.h" namespace octree::disk::v1 { @@ -20,6 +21,6 @@ struct IndexFile { std::string layout_strategy_id; std::string preferred_extension; - IndexMap map; + store::Index map; }; } diff --git a/src/terrainlib/octree/storage/IndexFile.h b/src/terrainlib/octree/storage/IndexFile.h index 7d70fbde..b296a755 100644 --- a/src/terrainlib/octree/storage/IndexFile.h +++ b/src/terrainlib/octree/storage/IndexFile.h @@ -5,7 +5,6 @@ #include #include "io/serialize.h" -#include "octree/IndexMap.h" #include "octree/StoreTraits.h" #include "octree/disk/IndexFile.h" #include "octree/store_layout/Mappings.h" @@ -27,7 +26,7 @@ read_index_file(const std::filesystem::path &path) { } disk::v1::IndexFile file = std::move(result.value()); return store::IndexMetadata{ - std::move(file.map).take_shared(), + std::move(file.map), std::move(file.layout_strategy_id), std::move(file.preferred_extension), }; @@ -39,7 +38,7 @@ inline std::expected write_index_file( disk::v1::IndexFile file; file.layout_strategy_id = metadata.layout_id; file.preferred_extension = metadata.codec_selector; - file.map = IndexMap(metadata.index); + file.map = metadata.index; const auto result = io::write_to_path(file, path); if (!result.has_value()) { return std::unexpected(store::IndexFormatError{ diff --git a/src/terrainlib/octree/storage/MeshStorage.h b/src/terrainlib/octree/storage/MeshStorage.h deleted file mode 100644 index e2d8b585..00000000 --- a/src/terrainlib/octree/storage/MeshStorage.h +++ /dev/null @@ -1,3 +0,0 @@ -#pragma once - -#include "octree/Storage.h" diff --git a/src/terrainlib/octree/storage/cache/Dummy.h b/src/terrainlib/octree/storage/cache/Dummy.h deleted file mode 100644 index 159bc32d..00000000 --- a/src/terrainlib/octree/storage/cache/Dummy.h +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once - -#include "octree/StoreTraits.h" -#include "store/cache/Dummy.h" - -namespace octree::cache { - -template -using Dummy = store::cache::Dummy; - -} // namespace octree::cache diff --git a/src/terrainlib/octree/storage/cache/ICache.h b/src/terrainlib/octree/storage/cache/ICache.h deleted file mode 100644 index b14bd0b0..00000000 --- a/src/terrainlib/octree/storage/cache/ICache.h +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once - -#include "octree/StoreTraits.h" -#include "store/cache/Interface.h" - -namespace octree::cache { - -template -using ICache = store::cache::Interface; - -} // namespace octree::cache diff --git a/src/terrainlib/octree/storage/cache/LruCache.h b/src/terrainlib/octree/storage/cache/LruCache.h deleted file mode 100644 index 167f19d2..00000000 --- a/src/terrainlib/octree/storage/cache/LruCache.h +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once - -#include "octree/StoreTraits.h" -#include "store/cache/Lru.h" - -namespace octree::cache { - -template -using Lru = store::cache::Lru; - -} // namespace octree::cache diff --git a/src/terrainlib/octree/storage/open.h b/src/terrainlib/octree/storage/open.h index b8f1a6d0..c15b1a0b 100644 --- a/src/terrainlib/octree/storage/open.h +++ b/src/terrainlib/octree/storage/open.h @@ -4,25 +4,24 @@ #include #include "mesh/storage.h" -#include "octree/Storage.h" namespace octree { using OpenOptions = mesh::storage::OpenOptions; -inline std::expected> open_index( +inline std::expected> open_index( const std::filesystem::path &path) { return mesh::storage::open_index(path); } -inline std::expected> open_folder( +inline std::expected> open_folder( const std::filesystem::path &path, const bool create_index = false, OpenOptions options = {}) { return mesh::storage::open_folder(path, create_index, std::move(options)); } -inline std::expected> open_folder_indexed( +inline std::expected> open_folder_indexed( const std::filesystem::path &path, OpenOptions options = {}) { return mesh::storage::open_folder_indexed(path, std::move(options)); diff --git a/src/terrainlib/octree/traverse.h b/src/terrainlib/octree/traverse.h deleted file mode 100644 index cc5b9435..00000000 --- a/src/terrainlib/octree/traverse.h +++ /dev/null @@ -1,52 +0,0 @@ -#pragma once - -#include -#include - -#include - -#include "octree/IndexMap.h" -#include "octree/StoreTraits.h" -#include "store/traverse.h" - -namespace octree { - -using TraversalOrder = store::TraversalOrder; - -constexpr bool always_refine(const Id &) { - return true; -} - -template> -void traverse( - const IndexMap &index, - VisitFn &&visit, - RefineFn &&refine = always_refine, - const Id &root = Id::root(), - const TraversalOrder order = TraversalOrder::DepthFirst) { - const auto result = store::traverse( - index.shared(), - std::forward(visit), - std::forward(refine), - root, - order); - DEBUG_ASSERT(result.has_value()); -} - -template> -void traverse( - const store::Index &index, - VisitFn &&visit, - RefineFn &&refine = always_refine, - const Id &root = Id::root(), - const TraversalOrder order = TraversalOrder::DepthFirst) { - const auto result = store::traverse( - index, - std::forward(visit), - std::forward(refine), - root, - order); - DEBUG_ASSERT(result.has_value()); -} - -} // namespace octree diff --git a/src/terrainlib/sf/finalize_storage.h b/src/terrainlib/sf/finalize_storage.h index 8ee9a06c..157d9852 100644 --- a/src/terrainlib/sf/finalize_storage.h +++ b/src/terrainlib/sf/finalize_storage.h @@ -4,13 +4,13 @@ #include -#include "octree/Storage.h" +#include "mesh/storage.h" #include "sf/Error.h" #include "sf/validate_index.h" namespace sf { -inline std::expected finalize_storage(octree::Storage &storage) { +inline std::expected finalize_storage(mesh::storage::Storage &storage) { const auto save_result = storage.save_or_create_index(); if (!save_result.has_value()) { return std::unexpected(FinalizeError(save_result.error())); diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 3cb01c33..9853fe1a 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -37,8 +37,6 @@ add_executable(unittests_terrainlib terrainlib/store_layout.cpp terrainlib/store_storage.cpp terrainlib/store_traverse.cpp - terrainlib/index_map.cpp - terrainlib/index_traverse.cpp terrainlib/mesh_boundary.cpp terrainlib/mesh_bounds.cpp terrainlib/mesh_cleanup.cpp diff --git a/unittests/dag_builder/storage_compatibility.cpp b/unittests/dag_builder/storage_compatibility.cpp index c2854a22..22545e2f 100644 --- a/unittests/dag_builder/storage_compatibility.cpp +++ b/unittests/dag_builder/storage_compatibility.cpp @@ -11,7 +11,7 @@ #include "io/serialize.h" #include "octree/disk/IndexFile.h" #include "octree/storage/open.h" -#include "octree/traverse.h" +#include "store/traverse.h" #include "storage.h" #include "store/codec/ZppBits.h" #include "thread_safe_storage.h" @@ -53,16 +53,18 @@ TEST_CASE("pre-refactor DAG fixture preserves index and payload contracts") { REQUIRE(index_file.has_value()); CHECK(index_file->layout_strategy_id == "level_and_coordinate_directories"); CHECK(index_file->preferred_extension == ".bin"); - CHECK(index_file->map.is(octree::NodeStatus::Inner, root)); - CHECK(index_file->map.is(octree::NodeStatus::Virtual, parent)); - CHECK(index_file->map.is(octree::NodeStatus::Leaf, leaf)); + CHECK(index_file->map.is(store::NodeStatus::Inner, root).value()); + CHECK(index_file->map.is(store::NodeStatus::Virtual, parent).value()); + CHECK(index_file->map.is(store::NodeStatus::Leaf, leaf).value()); CHECK(std::filesystem::is_regular_file(path / "0/0/0/0.bin")); CHECK(std::filesystem::is_regular_file(path / "2/2/3/1.bin")); std::vector visited; - octree::traverse(index_file->map, [&](const octree::Id id, const octree::NodeStatus) { - visited.push_back(id); - }); + REQUIRE(store::traverse( + index_file->map, + [&](const octree::Id id, const store::NodeStatus) { + visited.push_back(id); + }).has_value()); CHECK(visited == std::vector{root, parent, leaf}); auto batch_storage_result = dag::storage::open_folder_indexed(path); diff --git a/unittests/sf_merger/storage_boundaries.cpp b/unittests/sf_merger/storage_boundaries.cpp index cbc96ed6..26f5eba7 100644 --- a/unittests/sf_merger/storage_boundaries.cpp +++ b/unittests/sf_merger/storage_boundaries.cpp @@ -66,13 +66,13 @@ mesh::Simple clipping_box() { }); } -octree::IndexedStorage indexed_storage(const std::filesystem::path &path) { +mesh::storage::IndexedStorage indexed_storage(const std::filesystem::path &path) { auto result = octree::open_folder_indexed(path); REQUIRE(result.has_value()); return std::move(result.value()); } -octree::Storage unindexed_storage(const std::filesystem::path &path) { +mesh::storage::Storage unindexed_storage(const std::filesystem::path &path) { auto result = octree::open_folder(path); REQUIRE(result.has_value()); return std::move(result.value()); diff --git a/unittests/terrainlib/index_map.cpp b/unittests/terrainlib/index_map.cpp deleted file mode 100644 index 5d595c23..00000000 --- a/unittests/terrainlib/index_map.cpp +++ /dev/null @@ -1,100 +0,0 @@ -#include "../catch2_helpers.h" - -#include "octree/IndexMap.h" - -using namespace octree; - -TEST_CASE("IndexMap basic operations") { - IndexMap index; - - SECTION("Add root node") { - Id root = Id::root(); - index.add(root); - CHECK(index.is_present(root)); - CHECK(index.is(NodeStatus::Leaf, root)); - CHECK(index.get(root) == std::optional(NodeStatus::Leaf)); - } - - SECTION("Add child node creates virtual parent") { - Id root = Id::root(); - Id child = root.child(3).value(); - index.add(child); - CHECK(index.is_present(root)); - CHECK(index.is(NodeStatus::Virtual, root)); - CHECK(index.is_present(child)); - CHECK(index.is(NodeStatus::Leaf, child)); - } - - SECTION("Add child note promotes parent to inner") { - Id root = Id::root(); - Id child1 = root.child(0).value(); - index.add(child1); - index.add(root); - CHECK(index.is(NodeStatus::Leaf, child1)); - CHECK(index.is(NodeStatus::Inner, root)); - } - - SECTION("Removing a leaf node cleans up correctly") { - Id root = Id::root(); - Id child = root.child(2).value(); - index.add(child); - CHECK(index.is_present(child)); - index.remove(child); - CHECK(index.is_absent(child)); - CHECK(index.is_absent(root)); - } - - SECTION("Removing one of two children of an inner keeps it inner") { - Id root = Id::root(); - Id child1 = root.child(0).value(); - Id child2 = root.child(1).value(); - index.add(root); - index.add(child1); - index.add(child2); - index.remove(child1); - - CHECK(index.is_absent(child1)); - CHECK(index.is(NodeStatus::Leaf, child2)); - CHECK(index.is(NodeStatus::Inner, root)); - } - - SECTION("Removing last child of inner converts it to leaf") { - Id root = Id::root(); - Id child1 = root.child(0).value(); - index.add(child1); - index.add(root); - - index.remove(child1); - - CHECK(index.is_absent(child1)); - CHECK(index.is(NodeStatus::Leaf, root)); - } - - SECTION("Clear works") { - Id a = Id::root().child(1).value(); - Id b = Id::root().child(2).value(); - index.add(a); - index.add(b); - CHECK(index.size() == 3); // root + a + b - index.clear(); - CHECK(index.empty()); - } - - SECTION("Serialization round-trip") { - IndexMap original; - original.add(Id::root().child(3).value()); - - const std::errc success {}; - - std::vector buffer; - zpp::bits::out out(buffer); - CHECK(out(original).code == success); - - IndexMap loaded; - zpp::bits::in in(buffer); - CHECK(in(loaded).code == success); - - CHECK(original.size() == loaded.size()); - CHECK(loaded.is_present(Id::root().child(3).value())); - } -} diff --git a/unittests/terrainlib/index_traverse.cpp b/unittests/terrainlib/index_traverse.cpp deleted file mode 100644 index 5ada7575..00000000 --- a/unittests/terrainlib/index_traverse.cpp +++ /dev/null @@ -1,132 +0,0 @@ -#include -#include -#include -#include - -#include "octree/Id.h" -#include "octree/IndexMap.h" -#include "octree/traverse.h" - -using namespace octree; - -TEST_CASE("octree::traverse basic") { - IndexMap index; - - const Id root = Id::root(); - index.add(root); - - const auto children = root.children().value(); - for (const auto& child : children) { - index.add(child); - } - - SECTION("traversal visits root then children") { - std::vector visited; - traverse( - index, - [&](const Id &id, const NodeStatus &) { - visited.push_back(id); - }, - [](const Id &) { return true; }, // refine always - root, - TraversalOrder::DepthFirst); - - CHECK(visited.front() == root); - CHECK(visited.size() == 9); // root + 8 children - std::vector expected(children.begin(), children.end()); - visited.erase(visited.begin()); - CHECK(visited == expected); - } - - SECTION("Refine function disables recursion") { - std::vector visited; - traverse( - index, - [&](const Id &id, const NodeStatus &) { - visited.push_back(id); - }, - [](const Id &) { return false; }, // don't refine - root, - TraversalOrder::DepthFirst); - - CHECK(visited.size() == 1); - CHECK(visited.front() == root); - } - - SECTION("Traversal skips when root is missing") { - IndexMap empty; - bool visited = false; - - traverse( - empty, - [&](const Id&, const NodeStatus&) { - visited = true; - }, - [](const Id&) { return true; }, - root, - TraversalOrder::DepthFirst - ); - - CHECK_FALSE(visited); - } -} - -TEST_CASE("octree::traverse with depth") { - IndexMap index; - - const Id root = Id::root(); - index.add(root); - - const auto children = root.children().value(); - const Id c0 = children[0]; - const Id c1 = children[1]; - const Id c2 = children[2]; - - index.add(c0); - index.add(c1); - index.add(c2); - - // Add grandchildren - const Id gc0 = c0.child(0).value(); // grandchild of c0 - const Id gc1 = c2.child(1).value(); // grandchild of c2 - - index.add(gc0); - index.add(gc1); - - SECTION("DFS visits deeply before siblings") { - std::vector visited; - traverse( - index, - [&](const Id& id, const NodeStatus&) { - visited.push_back(id); - }, - [](const Id&) { return true; }, - root, - TraversalOrder::DepthFirst - ); - - CHECK(visited.size() == 6); - - // DFS order should be: root, c0, gc0, c1, c2, gc1 - std::vector expected{root, c0, gc0, c1, c2, gc1}; - CHECK(visited == expected); - } - - SECTION("BFS visits all nodes level by level") { - std::vector visited; - traverse( - index, - [&](const Id &id, const NodeStatus &) { - visited.push_back(id); - }, - [](const Id &) { return true; }, - root, - TraversalOrder::BreadthFirst); - - CHECK(visited.size() == 6); - - // BFS order should be: root, c0, c1, c2, gc0, gc1 - std::vector expected{root, c0, c1, c2, gc0, gc1}; - CHECK(visited == expected); - } -} diff --git a/unittests/terrainlib/store_compatibility.cpp b/unittests/terrainlib/store_compatibility.cpp index 40a3f61f..574ef8fc 100644 --- a/unittests/terrainlib/store_compatibility.cpp +++ b/unittests/terrainlib/store_compatibility.cpp @@ -11,14 +11,13 @@ #include "io/serialize.h" #include "io/bytes.h" #include "mesh/SimpleMesh.h" -#include "octree/Storage.h" #include "octree/disk/IndexFile.h" #include "mesh/codec/from_extension.h" #include "mesh/storage.h" #include "octree/store_layout/Mappings.h" #include "octree/storage/IndexFile.h" #include "octree/storage/open.h" -#include "octree/traverse.h" +#include "store/traverse.h" namespace { @@ -56,18 +55,18 @@ mesh::Simple triangle_mesh(const double offset) { {{offset, 0.0, 0.0}, {offset + 1.0, 0.0, 0.0}, {offset, 1.0, 0.0}}); } -octree::IndexedStorage make_storage( +mesh::storage::IndexedStorage make_storage( const std::filesystem::path &path, const store::PathMapping mapping, const std::string &extension) { auto codec = mesh::codec::from_extension(extension); REQUIRE(codec.has_value()); - return octree::IndexedStorage( + return mesh::storage::IndexedStorage( store::RawStorage( store::Layout(path, mapping), std::move(codec.value())), store::Index{}, - octree::Storage::Persistence{ + mesh::storage::Storage::Persistence{ octree::storage::index_format(), path / "terrain.index", std::string(mapping.id), @@ -87,12 +86,12 @@ TEST_CASE("pre-refactor SF fixtures preserve index and path contracts") { CHECK(index_file->layout_strategy_id == "flat"); CHECK(index_file->preferred_extension == ".terrain"); CHECK(index_file->map.size() == 1); - CHECK(index_file->map.is(octree::NodeStatus::Leaf, root)); + CHECK(index_file->map.is(store::NodeStatus::Leaf, root).value()); CHECK(std::filesystem::is_regular_file(path / "0-0.terrain")); auto storage_result = octree::open_folder_indexed(path); REQUIRE(storage_result.has_value()); - const octree::IndexedStorage storage = std::move(storage_result.value()); + const mesh::storage::IndexedStorage storage = std::move(storage_result.value()); const auto mesh = storage.load(root); REQUIRE(mesh.has_value()); CHECK(mesh->face_count() == 1); @@ -108,17 +107,19 @@ TEST_CASE("pre-refactor SF fixtures preserve index and path contracts") { REQUIRE(index_file.has_value()); CHECK(index_file->layout_strategy_id == "level_and_coordinate_directories"); CHECK(index_file->preferred_extension == ".terrain"); - CHECK(index_file->map.is(octree::NodeStatus::Virtual, root)); - CHECK(index_file->map.is(octree::NodeStatus::Leaf, child)); - CHECK(index_file->map.is(octree::NodeStatus::Virtual, deep_parent)); - CHECK(index_file->map.is(octree::NodeStatus::Leaf, deep)); + CHECK(index_file->map.is(store::NodeStatus::Virtual, root).value()); + CHECK(index_file->map.is(store::NodeStatus::Leaf, child).value()); + CHECK(index_file->map.is(store::NodeStatus::Virtual, deep_parent).value()); + CHECK(index_file->map.is(store::NodeStatus::Leaf, deep).value()); CHECK(std::filesystem::is_regular_file(path / "1/0/1/0.terrain")); CHECK(std::filesystem::is_regular_file(path / "2/3/1/3.terrain")); std::vector visited; - octree::traverse(index_file->map, [&](const octree::Id id, const octree::NodeStatus) { - visited.push_back(id); - }); + REQUIRE(store::traverse( + index_file->map, + [&](const octree::Id id, const store::NodeStatus) { + visited.push_back(id); + }).has_value()); CHECK(visited == std::vector{octree::Id{root}, child, deep_parent, deep}); } } From ae87580b9eec19cebdc62a7da903c105bcabca6f Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Fri, 14 Aug 2026 17:46:29 +0200 Subject: [PATCH 59/61] Remove migration compatibility fixtures --- docs/raster-store/refactor-status.md | 29 +++- .../dag_builder/storage_compatibility.cpp | 139 ++++++------------ .../raster-store-refactor/dag/0/0/0/0.bin | Bin 276 -> 0 bytes .../raster-store-refactor/dag/2/2/3/1.bin | Bin 276 -> 0 bytes .../raster-store-refactor/dag/terrain.index | Bin 78 -> 0 bytes .../sf-coordinates/1/0/1/0.terrain | Bin 100 -> 0 bytes .../sf-coordinates/2/3/1/3.terrain | Bin 100 -> 0 bytes .../sf-coordinates/terrain.index | Bin 92 -> 0 bytes .../raster-store-refactor/sf-flat/0-0.terrain | Bin 103 -> 0 bytes .../sf-flat/terrain.index | Bin 34 -> 0 bytes unittests/sf_builder/storage_finalization.cpp | 16 +- unittests/terrainlib/store_compatibility.cpp | 63 +------- unittests/terrainlib/store_layout.cpp | 17 +-- 13 files changed, 91 insertions(+), 173 deletions(-) delete mode 100644 unittests/data/raster-store-refactor/dag/0/0/0/0.bin delete mode 100644 unittests/data/raster-store-refactor/dag/2/2/3/1.bin delete mode 100644 unittests/data/raster-store-refactor/dag/terrain.index delete mode 100644 unittests/data/raster-store-refactor/sf-coordinates/1/0/1/0.terrain delete mode 100644 unittests/data/raster-store-refactor/sf-coordinates/2/3/1/3.terrain delete mode 100644 unittests/data/raster-store-refactor/sf-coordinates/terrain.index delete mode 100644 unittests/data/raster-store-refactor/sf-flat/0-0.terrain delete mode 100644 unittests/data/raster-store-refactor/sf-flat/terrain.index diff --git a/docs/raster-store/refactor-status.md b/docs/raster-store/refactor-status.md index 7f4f4b27..3a82dc46 100644 --- a/docs/raster-store/refactor-status.md +++ b/docs/raster-store/refactor-status.md @@ -16,10 +16,10 @@ This file is the resumable implementation log for ## Current position -- Phase: 5 — Cleanup and documentation +- Phase: 6 — Remove migration-only compatibility fixtures - State: complete; completion commit pending. -- Next action: commit and tag Phase 5, then remove migration-only fixtures in - Phase 6. +- Next action: commit and tag Phase 6, then perform the final clean-worktree + and tag audit. ## Phase log @@ -131,6 +131,19 @@ This file is the resumable implementation log for tools and include final public names plus 3D-open and in-memory 2D examples. - State: complete. +### Phase 6 — Remove migration-only compatibility fixtures + +- Removed the golden pre-refactor SF and DAG indexes and payload files. +- Removed fixture-only SF/DAG opening and byte-comparison assertions. +- Converted DAG codec reentrancy, metadata-prefix reading, synchronized + storage, SF-builder finalization, and DAG validation tests to generate + current data in temporary directories. +- Retained layout/path, resolver, topology, storage, error, merge/cut, and + producer/consumer boundary coverage. +- Repository search and filesystem inspection find no remaining pre-refactor + fixture reference or fixture file. +- State: complete. + ## Verification log - Baseline `unittests_terrainlib`: 23,676 assertions in 391 test cases passed. @@ -193,6 +206,12 @@ This file is the resumable implementation log for - Phase 5 repository searches found no remaining caller of a removed octree forwarding name and no `octree::Id` dependency under `terrainlib/store`. - Phase 5 `git diff --check`: passed. +- Phase 6 `unittests_terrainlib`: 23,997 assertions in 425 test cases passed. +- Phase 6 `unittests_dagbuilder`: 439 assertions in 73 test cases passed. +- Phase 6 `unittests_sfmerger`: 121 assertions in 8 test cases passed. +- Phase 6 SF builder finalization: 15 assertions in 2 test cases passed. +- Phase 6 fixture-reference search, fixture-file inspection, and + `git diff --check`: passed. ## Commit and tag log @@ -212,3 +231,7 @@ This file is the resumable implementation log for completion tag. - `3086336` / `refactor_phase_4` — hardened node reuse, SF topology enforcement, and Phase 4 completion tag. +- `bca0b5c` / `refactor_phase_5` — removed migration shims, documented final + public boundaries, and tagged Phase 5 completion. +- Phase 6 completion commit / `refactor_phase_6` — removed migration-only + fixtures while retaining current-data coverage. diff --git a/unittests/dag_builder/storage_compatibility.cpp b/unittests/dag_builder/storage_compatibility.cpp index 22545e2f..bde98acb 100644 --- a/unittests/dag_builder/storage_compatibility.cpp +++ b/unittests/dag_builder/storage_compatibility.cpp @@ -5,13 +5,10 @@ #include -#include "encoded.h" #include "build.h" -#include "io/bytes.h" -#include "io/serialize.h" -#include "octree/disk/IndexFile.h" +#include "dag_node.h" +#include "mesh/SimpleMesh.h" #include "octree/storage/open.h" -#include "store/traverse.h" #include "storage.h" #include "store/codec/ZppBits.h" #include "thread_safe_storage.h" @@ -40,81 +37,40 @@ class TemporaryDirectory { std::filesystem::path _path; }; -} // namespace - -TEST_CASE("pre-refactor DAG fixture preserves index and payload contracts") { - const std::filesystem::path path = - std::filesystem::path(ALP_TEST_DATA_DIR) / "raster-store-refactor" / "dag"; - const octree::Id root = octree::Id::root(); - const octree::Id parent = root.child(3).value(); - const octree::Id leaf = parent.child(6).value(); - - const auto index_file = io::read_from_path(path / "terrain.index"); - REQUIRE(index_file.has_value()); - CHECK(index_file->layout_strategy_id == "level_and_coordinate_directories"); - CHECK(index_file->preferred_extension == ".bin"); - CHECK(index_file->map.is(store::NodeStatus::Inner, root).value()); - CHECK(index_file->map.is(store::NodeStatus::Virtual, parent).value()); - CHECK(index_file->map.is(store::NodeStatus::Leaf, leaf).value()); - CHECK(std::filesystem::is_regular_file(path / "0/0/0/0.bin")); - CHECK(std::filesystem::is_regular_file(path / "2/2/3/1.bin")); - - std::vector visited; - REQUIRE(store::traverse( - index_file->map, - [&](const octree::Id id, const store::NodeStatus) { - visited.push_back(id); - }).has_value()); - CHECK(visited == std::vector{root, parent, leaf}); - - auto batch_storage_result = dag::storage::open_folder_indexed(path); - REQUIRE(batch_storage_result.has_value()); - const auto batch_storage = std::move(batch_storage_result.value()); - const auto batch = batch_storage.load(root); - REQUIRE(batch.has_value()); - CHECK(batch->metadata.group_assignment == std::vector{0}); - REQUIRE(batch->metadata.groups.size() == 1); - CHECK(batch->metadata.groups.front().error == 12.5); - CHECK(batch->metadata.groups.front().children == std::vector{{root.child(4).value(), 3}}); - REQUIRE(batch->clustering.clusters.size() == 1); - CHECK(batch->clustering.clusters.front().id == 17); - - auto metadata_storage_result = dag::storage::open_metadata_indexed(path); - REQUIRE(metadata_storage_result.has_value()); - const auto metadata_storage = std::move(metadata_storage_result.value()); - const auto metadata = metadata_storage.load(root); - REQUIRE(metadata.has_value()); - CHECK(metadata->group_assignment == batch->metadata.group_assignment); - REQUIRE(metadata->groups.size() == batch->metadata.groups.size()); - CHECK(metadata->groups.front().error == batch->metadata.groups.front().error); - - const auto encoded = io::write_to_bytes(batch.value()); - const auto fixture_bytes = io::read_bytes_from_path(path / "0/0/0/0.bin"); - REQUIRE(encoded.has_value()); - REQUIRE(fixture_bytes.has_value()); - CHECK(encoded.value() == fixture_bytes.value()); - - store::codec::ZppBits runtime_codec; - const auto runtime_batch = runtime_codec.read(store::NodePath(path / "0/0/0/0")); - REQUIRE(runtime_batch.has_value()); - CHECK(runtime_batch->metadata.group_assignment == batch->metadata.group_assignment); - CHECK(runtime_codec.paths(store::NodePath(path / "0/0/0/0")) - == std::vector{path / "0/0/0/0.bin"}); +dag::ClusterBatch sample_batch() { + dag::ClusterBatch batch; + batch.metadata.group_assignment = {0}; + batch.metadata.groups = {{ + .children = {{octree::Id::root().child(4).value(), 3}}, + .error = 12.5, + .bounds = {}, + }}; + batch.clustering.positions = { + {0.0, 0.0, 0.0}, + {1.0, 0.0, 0.0}, + {0.0, 1.0, 0.0}, + }; + batch.clustering.clusters = {{ + .id = 17, + .vertex_indices = {0, 1, 2}, + .local_triangles = {{0, 1, 2}}, + .uvs = {}, + .texture_id = std::nullopt, + .absolute_error = 0.0, + }}; + return batch; +} - TemporaryDirectory output; - const store::NodePath output_path(output.path() / "nested/node"); - REQUIRE(runtime_codec.write(output_path, batch.value()).has_value()); - const auto runtime_bytes = io::read_bytes_from_path(output.path() / "nested/node.bin"); - REQUIRE(runtime_bytes.has_value()); - CHECK(runtime_bytes.value() == fixture_bytes.value()); +mesh::Simple sample_mesh() { + return mesh::Simple( + {{0, 1, 2}}, + {{0.0, 0.0, 0.0}, {1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}}); } +} // namespace + TEST_CASE("runtime DAG ZPP Bits codec is reentrant") { - const std::filesystem::path fixture = - std::filesystem::path(ALP_TEST_DATA_DIR) - / "raster-store-refactor/dag/0/0/0/0.bin"; - const auto batch = io::read_from_path(fixture); - REQUIRE(batch.has_value()); + const dag::ClusterBatch batch = sample_batch(); TemporaryDirectory output; store::codec::ZppBits codec; @@ -122,12 +78,12 @@ TEST_CASE("runtime DAG ZPP Bits codec is reentrant") { for (int index = 0; index < 8; ++index) { operations.push_back(std::async(std::launch::async, [&codec, &batch, &output, index] { const store::NodePath path(output.path() / std::to_string(index) / "node"); - if (!codec.write(path, batch.value()).has_value()) { + if (!codec.write(path, batch).has_value()) { return false; } const auto loaded = codec.read(path); return loaded.has_value() - && loaded->metadata.group_assignment == batch->metadata.group_assignment; + && loaded->metadata.group_assignment == batch.metadata.group_assignment; })); } for (auto &operation : operations) { @@ -156,17 +112,13 @@ TEST_CASE("DAG resolvers expose writable batches and read-only metadata", "[stor } TEST_CASE("DAG indexed storage survives ThreadSafeStorage move and release", "[store][storage]") { - const std::filesystem::path fixture = - std::filesystem::path(ALP_TEST_DATA_DIR) - / "raster-store-refactor/dag/0/0/0/0.bin"; - const auto batch = io::read_from_path(fixture); - REQUIRE(batch.has_value()); + const dag::ClusterBatch batch = sample_batch(); TemporaryDirectory output; auto storage_result = dag::storage::open_folder_indexed(output.path()); REQUIRE(storage_result.has_value()); dag::ThreadSafeStorage synchronized(std::move(storage_result.value())); - REQUIRE(synchronized.save(octree::Id::root(), batch.value()).has_value()); + REQUIRE(synchronized.save(octree::Id::root(), batch).has_value()); auto released = std::move(synchronized).release(); REQUIRE(released.save_index().has_value()); @@ -176,24 +128,25 @@ TEST_CASE("DAG indexed storage survives ThreadSafeStorage move and release", "[s REQUIRE(reopened_result.has_value()); CHECK(reopened_result->has(octree::Id::root()).value()); CHECK(reopened_result->load(octree::Id::root()).has_value()); + + auto metadata_result = dag::storage::open_metadata_indexed(output.path()); + REQUIRE(metadata_result.has_value()); + const auto metadata = metadata_result->load(octree::Id::root()); + REQUIRE(metadata.has_value()); + CHECK(metadata->group_assignment == batch.metadata.group_assignment); + CHECK(metadata->groups.front().error == batch.metadata.groups.front().error); } TEST_CASE("DAG builder rejects invalid SF topology before processing", "[dag][sf][validation]") { - const std::filesystem::path fixture_path = - std::filesystem::path(ALP_TEST_DATA_DIR) / "raster-store-refactor/sf-flat"; - auto fixture_result = octree::open_folder_indexed(fixture_path); - REQUIRE(fixture_result.has_value()); - const auto mesh_result = fixture_result->load(octree::Id::root()); - REQUIRE(mesh_result.has_value()); - TemporaryDirectory input_directory; TemporaryDirectory output_directory; auto input_result = octree::open_folder_indexed(input_directory.path()); REQUIRE(input_result.has_value()); auto input = std::move(input_result.value()); const octree::Id root = octree::Id::root(); - REQUIRE(input.save(root, mesh_result.value()).has_value()); - REQUIRE(input.save(root.child(0).value(), mesh_result.value()).has_value()); + const mesh::Simple mesh = sample_mesh(); + REQUIRE(input.save(root, mesh).has_value()); + REQUIRE(input.save(root.child(0).value(), mesh).has_value()); REQUIRE(input.index().is(store::NodeStatus::Inner, root).value()); auto output_result = dag::storage::open_folder_indexed(output_directory.path()); diff --git a/unittests/data/raster-store-refactor/dag/0/0/0/0.bin b/unittests/data/raster-store-refactor/dag/0/0/0/0.bin deleted file mode 100644 index 7c8b1d28a199bea0ce758cdcc3ae51b44eed5bea..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 276 zcmZQ%fB;4y1;#8;0cI%8z@X^>VSd;T6*>T+IiR$P1B7n^rR^MGdi8)33s>y|K?Vm2 z1`P=Y1rdna5B3n60jh_?0c<46;Dc dg{v8um|)7#X-SX*bU_$j5XN9&xBzn!0|33_7i<6k diff --git a/unittests/data/raster-store-refactor/dag/2/2/3/1.bin b/unittests/data/raster-store-refactor/dag/2/2/3/1.bin deleted file mode 100644 index 7c8b1d28a199bea0ce758cdcc3ae51b44eed5bea..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 276 zcmZQ%fB;4y1;#8;0cI%8z@X^>VSd;T6*>T+IiR$P1B7n^rR^MGdi8)33s>y|K?Vm2 z1`P=Y1rdna5B3n60jh_?0c<46;Dc dg{v8um|)7#X-SX*bU_$j5XN9&xBzn!0|33_7i<6k diff --git a/unittests/data/raster-store-refactor/dag/terrain.index b/unittests/data/raster-store-refactor/dag/terrain.index deleted file mode 100644 index e3f1d112fe64250be60fa4d32d0b944f214aaca5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 78 zcmY#jU|`5eElbUbPs~e+PtMOTO3BPiEJ=+|$t+4uF3B&-Of6;ss?kfz%wq=9OmaX1 P3>ZON5MY8(K#~yvLw642 diff --git a/unittests/data/raster-store-refactor/sf-coordinates/1/0/1/0.terrain b/unittests/data/raster-store-refactor/sf-coordinates/1/0/1/0.terrain deleted file mode 100644 index 8b7a579c003dd10778221743462a9aab1ae525ff..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 100 zcmZQ%U|?VdVlEK~%?M-*0rA5R3}x+M=@l#6QadwdFlI6^=mA9+uG+Qg*Q!+vEI=*t Q3=DNZAs|q308=;#08EYz_5c6? diff --git a/unittests/data/raster-store-refactor/sf-coordinates/2/3/1/3.terrain b/unittests/data/raster-store-refactor/sf-coordinates/2/3/1/3.terrain deleted file mode 100644 index bebf97b788532b2663c16a2becfe57e15b2126d4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 100 zcmZQ%U|?VdVlEK~%?M-*0rA5R3}x+M=@l#6QadwdFlI6^=mA9+uG+Qg*Q!+vOh9Gw Q3=DNZAs{eu08=;#08Q8q0ssI2 diff --git a/unittests/data/raster-store-refactor/sf-coordinates/terrain.index b/unittests/data/raster-store-refactor/sf-coordinates/terrain.index deleted file mode 100644 index 546dd8bfb81ff88d2e18776d6159eb841eac8f5a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 92 zcmY#jU|`5eElbUbPs~e+PtMOTO3BPiEJ=+|$t+4uF3B&-OfBXBs?jS+EhSd6SNA(#jQ698sS5W4^X diff --git a/unittests/data/raster-store-refactor/sf-flat/0-0.terrain b/unittests/data/raster-store-refactor/sf-flat/0-0.terrain deleted file mode 100644 index 8bc0949026e08a52b88a5f372d02ddf2a804eb07..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 103 zcmZQ%U|?VdVlEK~%?M-*0rA5R3}x+M=@l#6QadwdFlI6^7y?BXuKERps~F@J80u;n N7~~lk>VTp|006Aq4+sDN diff --git a/unittests/data/raster-store-refactor/sf-flat/terrain.index b/unittests/data/raster-store-refactor/sf-flat/terrain.index deleted file mode 100644 index c581a3f6f1d99d289ca268fad4d3421bf7cf191a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 34 ecmZQ!U|>kgNi5+2(t0JSMMa63d5l0d0ssJM*#y`C diff --git a/unittests/sf_builder/storage_finalization.cpp b/unittests/sf_builder/storage_finalization.cpp index 3ef11068..229b2e3b 100644 --- a/unittests/sf_builder/storage_finalization.cpp +++ b/unittests/sf_builder/storage_finalization.cpp @@ -33,14 +33,10 @@ class TemporaryDirectory { std::filesystem::path _path; }; -mesh::Simple fixture_mesh() { - const std::filesystem::path fixture_path = - std::filesystem::path(ALP_TEST_DATA_DIR) / "raster-store-refactor/sf-flat"; - auto fixture = octree::open_folder_indexed(fixture_path); - REQUIRE(fixture.has_value()); - auto loaded = fixture->load(octree::Id::root()); - REQUIRE(loaded.has_value()); - return std::move(loaded.value()); +mesh::Simple sample_mesh() { + return mesh::Simple( + {{0, 1, 2}}, + {{0.0, 0.0, 0.0}, {1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}}); } } // namespace @@ -50,7 +46,7 @@ TEST_CASE("SF builder finalization writes and validates a valid index", "[sfbuil auto storage_result = octree::open_folder(directory.path()); REQUIRE(storage_result.has_value()); auto storage = std::move(storage_result.value()); - REQUIRE(storage.save(octree::Id::root(), fixture_mesh()).has_value()); + REQUIRE(storage.save(octree::Id::root(), sample_mesh()).has_value()); CHECK(terrainbuilder::finalize_storage(storage).has_value()); CHECK(std::filesystem::is_regular_file(directory.path() / "terrain.index")); @@ -64,7 +60,7 @@ TEST_CASE( REQUIRE(storage_result.has_value()); auto storage = std::move(storage_result.value()); const octree::Id root = octree::Id::root(); - const mesh::Simple mesh = fixture_mesh(); + const mesh::Simple mesh = sample_mesh(); REQUIRE(storage.save(root, mesh).has_value()); REQUIRE(storage.save(root.child(0).value(), mesh).has_value()); diff --git a/unittests/terrainlib/store_compatibility.cpp b/unittests/terrainlib/store_compatibility.cpp index 574ef8fc..79a1d3ad 100644 --- a/unittests/terrainlib/store_compatibility.cpp +++ b/unittests/terrainlib/store_compatibility.cpp @@ -17,14 +17,9 @@ #include "octree/store_layout/Mappings.h" #include "octree/storage/IndexFile.h" #include "octree/storage/open.h" -#include "store/traverse.h" namespace { -std::filesystem::path fixture_path(const std::string_view name) { - return std::filesystem::path(ALP_TEST_DATA_DIR) / "raster-store-refactor" / name; -} - class TemporaryDirectory { public: explicit TemporaryDirectory(const std::string_view label) { @@ -76,55 +71,7 @@ mesh::storage::IndexedStorage make_storage( } // namespace -TEST_CASE("pre-refactor SF fixtures preserve index and path contracts") { - const octree::Id root = octree::Id::root(); - - SECTION("flat") { - const std::filesystem::path path = fixture_path("sf-flat"); - const auto index_file = io::read_from_path(path / "terrain.index"); - REQUIRE(index_file.has_value()); - CHECK(index_file->layout_strategy_id == "flat"); - CHECK(index_file->preferred_extension == ".terrain"); - CHECK(index_file->map.size() == 1); - CHECK(index_file->map.is(store::NodeStatus::Leaf, root).value()); - CHECK(std::filesystem::is_regular_file(path / "0-0.terrain")); - - auto storage_result = octree::open_folder_indexed(path); - REQUIRE(storage_result.has_value()); - const mesh::storage::IndexedStorage storage = std::move(storage_result.value()); - const auto mesh = storage.load(root); - REQUIRE(mesh.has_value()); - CHECK(mesh->face_count() == 1); - } - - SECTION("level and coordinate directories") { - const std::filesystem::path path = fixture_path("sf-coordinates"); - const octree::Id child = root.child(2).value(); - const octree::Id deep = root.child(5).value().child(7).value(); - const octree::Id deep_parent = deep.parent().value(); - - const auto index_file = io::read_from_path(path / "terrain.index"); - REQUIRE(index_file.has_value()); - CHECK(index_file->layout_strategy_id == "level_and_coordinate_directories"); - CHECK(index_file->preferred_extension == ".terrain"); - CHECK(index_file->map.is(store::NodeStatus::Virtual, root).value()); - CHECK(index_file->map.is(store::NodeStatus::Leaf, child).value()); - CHECK(index_file->map.is(store::NodeStatus::Virtual, deep_parent).value()); - CHECK(index_file->map.is(store::NodeStatus::Leaf, deep).value()); - CHECK(std::filesystem::is_regular_file(path / "1/0/1/0.terrain")); - CHECK(std::filesystem::is_regular_file(path / "2/3/1/3.terrain")); - - std::vector visited; - REQUIRE(store::traverse( - index_file->map, - [&](const octree::Id id, const store::NodeStatus) { - visited.push_back(id); - }).has_value()); - CHECK(visited == std::vector{octree::Id{root}, child, deep_parent, deep}); - } -} - -TEST_CASE("pre-refactor layouts round-trip boundary IDs") { +TEST_CASE("octree layouts round-trip boundary IDs") { const std::vector ids{ octree::Id::root(), octree::Id(octree::Id::max_level(), octree::Id::Index{0}), @@ -142,7 +89,7 @@ TEST_CASE("pre-refactor layouts round-trip boundary IDs") { } } -TEST_CASE("pre-refactor storage hard-links matching payload formats") { +TEST_CASE("storage hard-links matching payload formats") { TemporaryDirectory source_directory("hard-link-source"); TemporaryDirectory target_directory("hard-link-target"); auto source = make_storage( @@ -162,7 +109,7 @@ TEST_CASE("pre-refactor storage hard-links matching payload formats") { REQUIRE(target.save_index().has_value()); } -TEST_CASE("pre-refactor storage re-encodes differing payload formats") { +TEST_CASE("storage re-encodes differing payload formats") { TemporaryDirectory source_directory("reencode-source"); TemporaryDirectory target_directory("reencode-target"); auto source = make_storage( @@ -187,7 +134,7 @@ TEST_CASE("pre-refactor storage re-encodes differing payload formats") { REQUIRE(target.save_index().has_value()); } -TEST_CASE("pre-refactor storage supports enabled overwrites") { +TEST_CASE("storage supports enabled overwrites") { TemporaryDirectory directory("overwrite"); auto storage = make_storage( directory.path(), @@ -205,7 +152,7 @@ TEST_CASE("pre-refactor storage supports enabled overwrites") { REQUIRE(storage.save_index().has_value()); } -TEST_CASE("pre-refactor folder opening scans payloads and creates an index") { +TEST_CASE("folder opening scans payloads and creates an index") { TemporaryDirectory directory("folder-open"); octree::OpenOptions options; options.default_mapping = octree::store_layout::flat(); diff --git a/unittests/terrainlib/store_layout.cpp b/unittests/terrainlib/store_layout.cpp index ec4eaa4b..736df506 100644 --- a/unittests/terrainlib/store_layout.cpp +++ b/unittests/terrainlib/store_layout.cpp @@ -7,29 +7,28 @@ #include "octree/store_layout/Mappings.h" #include "store/Layout.h" -TEST_CASE("extensionless octree mappings preserve physical fixture paths", "[store][layout]") { - const std::filesystem::path fixtures = - std::filesystem::path(ALP_TEST_DATA_DIR) / "raster-store-refactor"; +TEST_CASE("extensionless octree mappings preserve physical paths", "[store][layout]") { + const std::filesystem::path dataset = "dataset"; mesh::codec::Terrain terrain; SECTION("flat") { - const store::Layout layout(fixtures / "sf-flat", octree::store_layout::flat()); + const store::Layout layout(dataset / "sf-flat", octree::store_layout::flat()); const store::NodePath node_path = layout.node_path(octree::Id::root()); - CHECK(node_path.path() == fixtures / "sf-flat/0-0"); - CHECK(terrain.paths(node_path) == std::vector{fixtures / "sf-flat/0-0.terrain"}); + CHECK(node_path.path() == dataset / "sf-flat/0-0"); + CHECK(terrain.paths(node_path) == std::vector{dataset / "sf-flat/0-0.terrain"}); CHECK(layout.key_from_node_path(node_path) == octree::Id::root()); } SECTION("level and coordinate directories") { const store::Layout layout( - fixtures / "sf-coordinates", + dataset / "sf-coordinates", octree::store_layout::level_and_coordinate_directories()); const octree::Id child = octree::Id::root().child(2).value(); const octree::Id deep = octree::Id::root().child(5).value().child(7).value(); CHECK(terrain.paths(layout.node_path(child)) - == std::vector{fixtures / "sf-coordinates/1/0/1/0.terrain"}); + == std::vector{dataset / "sf-coordinates/1/0/1/0.terrain"}); CHECK(terrain.paths(layout.node_path(deep)) - == std::vector{fixtures / "sf-coordinates/2/3/1/3.terrain"}); + == std::vector{dataset / "sf-coordinates/2/3/1/3.terrain"}); CHECK(layout.key_from_node_path(layout.node_path(child)) == child); CHECK(layout.key_from_node_path(layout.node_path(deep)) == deep); } From c1f4bb48e88f9ca36b4d5ee9973b582b04304098 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Sun, 16 Aug 2026 15:53:00 +0200 Subject: [PATCH 60/61] Parameterize golden E2E paths --- docs/raster-store/golden-e2e.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/raster-store/golden-e2e.sh b/docs/raster-store/golden-e2e.sh index b919938c..894622cf 100755 --- a/docs/raster-store/golden-e2e.sh +++ b/docs/raster-store/golden-e2e.sh @@ -6,13 +6,13 @@ set -Eeuo pipefail -readonly SOURCE_DIR="/home/codex/Documents/alpine-terrain-builder/terrain-builder-raster-store" -readonly BUILD_DIR="${SOURCE_DIR}/build/golden-e2e" -readonly RUN_ROOT="/data/scratch/codex/alpine-terrain-builder-golden-e2e" -readonly INPUT_ROOT="${RUN_ROOT}/inputs" -readonly REFERENCE_ROOT="${RUN_ROOT}/reference" -readonly LOG_ROOT="${RUN_ROOT}/logs" -readonly STATE_ROOT="${RUN_ROOT}/state" +readonly SOURCE_DIR="${SOURCE_DIR:-/home/codex/Documents/alpine-terrain-builder/terrain-builder-raster-store}" +readonly BUILD_DIR="${BUILD_DIR:-${SOURCE_DIR}/build/golden-e2e}" +readonly RUN_ROOT="${RUN_ROOT:-/data/scratch/codex/alpine-terrain-builder-golden-e2e}" +readonly INPUT_ROOT="${INPUT_ROOT:-${RUN_ROOT}/inputs}" +readonly REFERENCE_ROOT="${REFERENCE_ROOT:-${RUN_ROOT}/reference}" +readonly LOG_ROOT="${LOG_ROOT:-${RUN_ROOT}/logs}" +readonly STATE_ROOT="${STATE_ROOT:-${RUN_ROOT}/state}" readonly RUN_ID="${RUN_ID:-$(date +%Y%m%dT%H%M%S)}" readonly RUN_LOG_ROOT="${LOG_ROOT}/runs/${RUN_ID}" readonly TIMINGS_FILE="${RUN_LOG_ROOT}/timings.tsv" @@ -25,7 +25,7 @@ readonly DAG_BUILDER="${BUILD_DIR}/src/dag_builder/dag-builder" # GT rasters. They are recreated from the raw datasets at the start of a run. readonly RAW_GS="/data/raw/raster_data/Oe_2020/OeRect_01m_gs_31287.img" readonly RAW_GT="/data/raw/raster_data/Oe_2020/OeRect_01m_gt_31287.img" -readonly ELEVATION_ROOT="${INPUT_ROOT}/grossglockner-elevation-4km" +readonly ELEVATION_ROOT="${ELEVATION_ROOT:-${INPUT_ROOT}/grossglockner-elevation-4km}" readonly GS_VRT="${ELEVATION_ROOT}/grossglockner-gs-4km.vrt" readonly GT_VRT="${ELEVATION_ROOT}/grossglockner-gt-4km.vrt" readonly BASEMAP_TILES="${INPUT_ROOT}/basemap" From b90a7a81a7e9aa13e038c44e317e18373e35c943 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Mon, 17 Aug 2026 09:30:57 +0200 Subject: [PATCH 61/61] Fix stale SF tile ID test --- unittests/sf_builder/texture.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unittests/sf_builder/texture.cpp b/unittests/sf_builder/texture.cpp index 06118437..3da052b4 100644 --- a/unittests/sf_builder/texture.cpp +++ b/unittests/sf_builder/texture.cpp @@ -351,7 +351,7 @@ TEST_CASE("texture assembler reports the content region", "[terrainbuilder]") { tile_image.row(row).setTo(uint8_t(row)); } - const radix::tile::Id root_tile(0, {0, 0}, radix::tile::Scheme::SlippyMap); + const radix::tile::Id root_tile(0, {0, 0}); const StaticTileProvider tile_provider({{root_tile, tile_image}}); const std::vector tiles_to_splatter = {root_tile};