diff --git a/README.md b/README.md index e4af0bdb..011b8cd6 100644 --- a/README.md +++ b/README.md @@ -24,17 +24,24 @@ The tools typically only handle one tile per command which makes it infeasible t In this example, we'll show how to build the hierarchy for Vienna's city center (Zoom: 13, X: 4468, Y:2840). ### 1. Downloading tiles -The following command will download the basemap tiles from our mirror with the following format: -https://gataki.cg.tuwien.ac.at/raw/basemap/tiles/{zoom}/{Y}/{X}.jpeg +Tile coordinates always use the Google/Mapbox/XYZ convention internally: the origin is north-west, X grows east, and Y grows south. Google Maps, Mapbox, OpenStreetMap, and most XYZ services use the common URL order `{zoom}/{x}/{y}`. + +The `basemap` and `gataki` providers select their complete URL pattern and Y direction automatically. Both currently use downward Y with the URL order `{zoom}/{y}/{x}`. The `basemap` provider downloads the basemap.at orthophoto, and the Gataki mirror uses: + +`https://gataki.cg.tuwien.ac.at/raw/basemap/tiles/{zoom}/{y}/{x}.jpeg` Example for the root tile: https://gataki.cg.tuwien.ac.at/raw/basemap/tiles/13/2840/4468.jpeg -The tiles will be downloaded into the folder `./tiles/`. +For another service, pass a quoted `--url` pattern containing `{zoom}`, `{x}`, and `{y}`. Placeholder placement selects URL coordinate order. Custom URLs default to downward Y; use `--url-y-direction up` for legacy TMS. Downloaded files always use the common Google/Mapbox layout `{zoom}/{x}/{y}.jpeg`, independently of the remote URL. + +The following command downloads the mirror's `{zoom}/{y}/{x}` URLs and writes the root tile to `./tiles/13/4468/2840.jpeg`: ``` -./tile-downloader --provider gataki --zoom 13 --row 2840 --col 4468 --max-zoom-level 19 +./tile-downloader --provider gataki --zoom 13 --x 4468 --y 2840 --max-zoom-level 19 ``` +For example, a custom Google/Mapbox URL can be selected with `--url 'https://example.test/{zoom}/{x}/{y}.jpeg'`. + ### 2. Download heightmap dataset The meshes are built from a heightmap dataset, therefore we need to download one. For this example we'll only use a small part of the complete dataset for the whole of austria (available at https://gataki.cg.tuwien.ac.at/raw/Oe_2020/, 268 GB to 1.1 TB). The part we're gonna use contains Vienna's city center and it is available at https://gataki.cg.tuwien.ac.at/raw/vienna/innenstadt_gs_1m_mgi.tif (228 MB). @@ -84,4 +91,4 @@ In order to build, you need to install: - tbb (intel threading building blocks) sudo apt-get install libcgal-dev libopencv-dev libfmt-dev libglm-dev libgdal-dev catch2 libfreeimage-dev libtbb-dev libcurl4-openssl-dev -(libgmp-dev libmpfr-dev libsqlite3-dev) \ No newline at end of file +(libgmp-dev libmpfr-dev libsqlite3-dev) diff --git a/cmake/SetupGDAL.cmake b/cmake/SetupGDAL.cmake index a5c51da6..fdc85178 100644 --- a/cmake/SetupGDAL.cmake +++ b/cmake/SetupGDAL.cmake @@ -21,17 +21,25 @@ if(NOT COMMAND alp_setup_cmake_project) endif() function(alp_setup_gdal) - set(oneValueArgs GDAL_VERSION PROJ_VERSION) + set(oneValueArgs GDAL_VERSION GEOS_VERSION PROJ_VERSION) cmake_parse_arguments(ARG "" "${oneValueArgs}" "" ${ARGN}) - if(NOT ARG_GDAL_VERSION OR NOT ARG_PROJ_VERSION) - message(FATAL_ERROR "alp_setup_gdal() needs: GDAL_VERSION PROJ_VERSION ") + if(NOT ARG_GDAL_VERSION OR NOT ARG_GEOS_VERSION OR NOT ARG_PROJ_VERSION) + message(FATAL_ERROR "alp_setup_gdal() needs: GDAL_VERSION GEOS_VERSION PROJ_VERSION ") endif() alp_setup_cmake_project(proj URL https://github.com/OSGeo/PROJ.git COMMITISH ${ARG_PROJ_VERSION} CMAKE_ARGUMENTS -DBUILD_TESTING=OFF -DBUILD_APPS=OFF) find_package(PROJ CONFIG REQUIRED) - set(_proj_install "${ALP_PROJ_INSTALL_DIR}") + alp_setup_cmake_project(geos + URL https://github.com/libgeos/geos.git + COMMITISH ${ARG_GEOS_VERSION} + CMAKE_ARGUMENTS + -DBUILD_SHARED_LIBS=OFF + -DBUILD_TESTING=OFF + -DGEOS_BUILD_DEVELOPER=OFF + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + ) alp_setup_cmake_project(gdal URL https://github.com/OSGeo/gdal.git @@ -40,6 +48,8 @@ function(alp_setup_gdal) -DGDAL_BUILD_OPTIONAL_DRIVERS=OFF -DGDAL_ENABLE_DRIVER_HFA=ON -DOGR_BUILD_OPTIONAL_DRIVERS=OFF + -DOGR_ENABLE_DRIVER_GPKG=ON + -DOGR_ENABLE_DRIVER_SQLITE=ON -DBUILD_APPS=OFF -DBUILD_TESTING=OFF -DBUILD_PYTHON_BINDINGS=OFF @@ -47,6 +57,8 @@ function(alp_setup_gdal) -DBUILD_CSHARP_BINDINGS=OFF -DGDAL_USE_ICONV=OFF -DGDAL_USE_EXTERNAL_LIBS=OFF + -DGDAL_USE_GEOS=ON + -DGDAL_USE_SQLITE3=ON "-DCMAKE_INSTALL_RPATH=\$ORIGIN/../../proj/lib" ) diff --git a/docs/raster-store/README.md b/docs/raster-store/README.md new file mode 100644 index 00000000..e78c6fd0 --- /dev/null +++ b/docs/raster-store/README.md @@ -0,0 +1,27 @@ +# Raster store design + +This directory describes a proposed authoritative raster store and the +generation of delivery tile pyramids from it. The documents are a design +baseline, not a finalized binary-format specification. + +## Documents + +- [Terminology](terminology.md) +- [Status quo and reuse assessment](status-quo.md) +- [Architecture](architecture.md) +- [Storage format](storage-format.md) +- [Sampling and pyramid generation](sampling-and-generation.md) +- [Implemented refactor status](refactor-status.md) + +## Plans + +- [Store refactoring plan](refactor-plan.md) +- [Raster store TODO](todo.md) +- [DRAFT RF builder plan archive](rf_builder.md) +- [DRAFT RF merger plan archive](rf_merger.md) + +## Scope + +The documents mostly hold format information. The `rf_builder` and +`rf_merger` documents are explicitly non-authoritative idea parking lots, not +tool specifications or implementation plans. diff --git a/docs/raster-store/architecture.md b/docs/raster-store/architecture.md new file mode 100644 index 00000000..9373d754 --- /dev/null +++ b/docs/raster-store/architecture.md @@ -0,0 +1,213 @@ +# Architecture + +## Implementation status + +This document describes the intended raster-fundamentalis and tile-base +architecture. Its RF format, snapshot-publication, builder, merger, pyramid, +and server requirements are future work; they were not acceptance criteria +for the completed 2D/3D hierarchy-store refactor. The implemented boundary is +recorded in [refactor-status.md](refactor-status.md), and the original plan is +kept in [refactor-plan.md](refactor-plan.md). + +The refactor delivered the dimension-neutral mechanisms needed by that future +work: + +- `store::Index`, `store::traverse`, layouts, runtime codecs, storage, + typed errors, and cache interfaces; +- `octree::StoreTraits` plus legacy 3D layout/index/open adapters; +- `raster_store::StoreTraits` for in-memory topology keyed by + `radix::tile::Id`; and +- SF-only topology validation under `sf`. + +It deliberately did not define a persistent RF index, RF payload codec, RF +opening API, `rf_builder`, `rf_merger`, tile-base generator, or tile server. + +### Final public names + +The shared API uses `store::NodeStatus`, `store::NodeStatusOrMissing`, +`store::Index`, `store::traverse`, `store::RawStorage`, +`store::Storage`, and `store::IndexedStorage`. Mesh storage aliases live under +`mesh::storage`; DAG batch and metadata aliases live under `dag::storage`. +The `octree` namespace retains the 3D key, traits, path mappings, legacy index +DTO, and mesh-opening compatibility functions. + +A legacy 3D mesh dataset is opened through the 3D adapter: + +```cpp +#include "octree/storage/open.h" + +auto opened = octree::open_folder_indexed(dataset_path); +if (!opened.has_value()) { + return std::unexpected(opened.error()); +} +mesh::storage::IndexedStorage storage = std::move(opened.value()); +``` + +The 2D traits adapter can exercise the shared topology without implying a +persistent RF format: + +```cpp +#include "raster_store/StoreTraits.h" +#include "store/Index.h" +#include "store/traverse.h" + +store::Index index; +const radix::tile::Id tile{2, {1, 3}}; +auto added = index.add(tile); +auto walked = store::traverse(index, [](const auto &id, store::NodeStatus status) { + // In-memory hierarchy processing only. +}); +``` + +## System boundary + +The design separates authoritative data management from delivery generation: + +```text +Input rasters (GDAL) + │ + │ inspect, transform to Web Mercator, define source, one source per pixel -> rf_builder + ▼ +raster-fundamentalis (one rf per source at the beginning) + │ + │ rf_merger: merge two rf stores based on (vecrtor) mask, take one tile if no overlap / far from vector border, merge strategy otherwise + │ at the beginning, have xor strategy, and exact border, later we may implement linear blending + ▼ +Authoritative rf raster store + │ + │ generate overviews using defined filtering strategy, all area pixels. for now simple averaging, later maybe larger filter sizes / more complex filters + ▼ +tile-base store (one per layer, one per data version. user visible server should only need one version per layer) + ├── read by tile-server + ├── area/vertex pixel tile generation + └── select resolution, type etc by url +``` + +## Dataset organization + +### Proposed + +A store root contains immutable snapshots. Each snapshot contains an index, a source attribution table and the data: + +```text +store/ +└── snapshot-id/ + ├── source_attribution_table.ard + ├── raster_store.index + └── //.amort +``` + +The shown payload path is the default `zoom/x/y_google` layout. Other layouts +may map the same tile IDs differently; there is no mandatory `chunks/` +directory. + +## Sparse quadtree index + +The index uses the same four logical states as the octree index: + +| State | Chunk exists | Indexed descendants | +|-----------|-------------:|--------------------:| +| `Leaf` | yes | no | +| `Inner` | yes | yes | +| `Virtual` | no | yes | +| `Missing` | no | no | + +`Missing` is represented by absence from the index, not serialized as an +entry. + +The index answers structural questions only. It does not claim that every +child exists and does not mark a virtual subtree as spatially complete. + +### Required operations + +- Look up a node without probing the filesystem. +- Add and remove physical nodes while maintaining virtual ancestors. +- Traverse only indexed branches. +- Enumerate physical descendants of a subtree. +- Find the nearest physical ancestor for fallback. +- Determine whether descendants may improve a requested output. +- Serialize and validate a versioned 2D topology. + +The last two operations may require aggregate metadata beyond +`Leaf/Inner/Virtual`, such as best descendant resolution or coverage. Such +metadata is an optimization and should be derivable from authoritative +entries. + +## Chunk model + +Each physical node owns one logical chunk, see storage-format.md + +## Source selection and fallback + +Source selection is a policy of the tools, not a property of the raster container. +Its initial comparison is expected to prioritize effective pixel resolution, and can be represented as a per zoom level or global ordered vector of attribution indices. + +At the beginning the selection will be binary, later we may introduce blending (over pixels of one zoom level, or several zoom levels). + +## Snapshot lifecycle + +A snapshot is never mutated, instead, operations build new snapshots, while reducing disk usage by using hard links: + +when adding data, we would: +1. create a new sf from the new data +2. define a validity mask for the new data +3. define a source merging priority (an ordering of sources) +4. using these two, a new snapshot is generated from the new data and the existing / authoritative snapshot, hardlinking tiles without change. + +Because hard links share inodes, a linked container must never be opened for +in-place modification. Existing snapshots are considered immutable. Obsolete snapshots can be deleted, the data will be preserved if necessary due to reference counting in the inodes. +When merging, we need to create new hardlinks for unchanged rf tiles (taken completely from either snapshot), and we need to create new rf tiles if the new tile shares information from both. + +### Publication + +A new snapshot is assembled in a sibling directory named +`.part`. Publication follows this protocol: + +1. Write all payload and metadata files into the `.part` directory. +2. Write the index last and validate the completed snapshot. +3. Flush and close every file. +4. Atomically rename the directory to `` on the same filesystem. + +The final destination must not already exist. A `.part` directory is +incomplete and is never considered published. The rename removes the suffix; +there is no separate marker or manifest. During normal operation this gives +readers atomic visibility: they see either no final snapshot or the completed +one. + +Cross-filesystem publication is unsupported because the final rename and any +hard links must remain on one filesystem. A builder or merger must reject that +configuration before starting a long operation. + +Publication does not guarantee durability or safe recovery across a power +failure, operating-system crash, or storage failure. Flushing and closing +files before the rename is required for normal-operation correctness, but is +not a crash-durability guarantee. The implementation does not require +`fsync()`, `fdatasync()`, `FlushFileBuffers()`, or equivalent +platform-specific synchronization. After such a failure, either a `.part` +directory or a final snapshot may be unusable and must be validated and +rebuilt. + +## Pyramid generator interface (to be confirmed, LLM, do not use the following without consultation) + +A generator requests a layer over a target tile and sampling specification. +The store reader supplies selected authoritative values and provenance over a +window large enough for the generator's filter support. + +The generator owns: + +- target output zoom and dimensions; +- vertex-pixel or area-pixel placement; +- low-pass/reconstruction filter; +- NoData normalization during filtering; +- colour-space and alpha treatment; +- border construction; +- output codec; and +- tile-level contributing-source metadata. + +The store owns: + +- chunk location and decoding; +- sparse hierarchy and physical fallback; +- exact stored source map; +- source catalog lookup; and +- consistent window access across chunk boundaries. diff --git a/docs/raster-store/before-refactor/README.md b/docs/raster-store/before-refactor/README.md new file mode 100644 index 00000000..5a5e146d --- /dev/null +++ b/docs/raster-store/before-refactor/README.md @@ -0,0 +1,440 @@ +# SF storage and merge architecture before the 2D/3D refactor + +> **Historical report:** this file describes the pre-refactor tree and keeps +> its original names, paths, diagrams, and conclusions for review history. +> Some linked source paths were intentionally removed by the completed +> migration. For the implemented boundary and verification results, see +> [refactor-status.md](../refactor-status.md); for current public names and +> examples, see [architecture.md](../architecture.md#implementation-status). + +This report describes the existing Structura Fundamentalis (SF) builder, +sparse octree index, storage stack, and mask-based dataset merger. Its purpose +is to identify the mechanisms that can be generalised from 3D octrees to a +shared 2D/3D hierarchy without carrying mesh- and ECEF-specific behaviour into +the common storage layer. + +The central conclusion is: + +> Generalise the sparse hierarchy, storage, traversal, merge decisions, and +> unchanged-subtree reuse. Keep GDAL ingestion, raster processing, mesh +> construction, spatial transforms, and mask geometry in dimension-specific +> adapters. + +The current storage templates are already payload-generic, but they are not +hierarchy-generic: `octree::Id` is embedded throughout indexing, traversal, +caches, disk paths, and serialization. + +## Current architecture + +```mermaid +flowchart TB + subgraph Applications + SFB[sf-builder] + SFM[sf-merger] + end + + subgraph Input + GDALR[GDAL raster datasource] + GDALV[GDAL vector mask] + OLD[Base SF dataset] + NEW[New SF dataset] + end + + subgraph Domain3D["3D and mesh policy"] + SPACE[octree::Space and ECEF bounds] + MESHBUILD[Height raster to SimpleMesh] + MESHMASK[Polygon to spherical extruded MeshMask] + MESHMERGE[Clip, combine, and texture atlas] + FALLBACK[Reconstruct child by clipping ancestor mesh] + end + + subgraph GenericCandidate["Mostly generalisable mechanisms"] + INDEX[IndexMap and NodeStatus] + WALK[Depth-first and breadth-first traversal] + DRIVER[Merger and merge result actions] + STORAGE[Storage and IndexedStorage] + RAW[RawStorage] + CODEC[Codec policy] + LAYOUT[Layout strategy] + LINK[Hard-link unchanged payload] + end + + GDALR --> SFB + SFB --> SPACE + SFB --> MESHBUILD + MESHBUILD --> STORAGE + + OLD --> SFM + NEW --> SFM + GDALV --> MESHMASK + MESHMASK --> SFM + SFM --> DRIVER + DRIVER --> FALLBACK + DRIVER --> MESHMERGE + DRIVER --> STORAGE + + STORAGE --> INDEX + STORAGE --> RAW + RAW --> CODEC + RAW --> LAYOUT + RAW --> LINK + WALK --> INDEX + DRIVER --> WALK +``` + +The significant code boundaries are: + +- hierarchy identity: [`octree::Id`](../../../src/terrainlib/octree/Id.h); +- sparse topology: [`IndexMap`](../../../src/terrainlib/octree/IndexMap.h); +- traversal: [`octree::traverse`](../../../src/terrainlib/octree/traverse.h); +- logical storage: + [`Storage_`](../../../src/terrainlib/octree/storage/Storage.h); +- raw file and hard-link storage: + [`RawStorage_`](../../../src/terrainlib/octree/storage/RawStorage.h); +- disk paths: [`disk::Layout`](../../../src/terrainlib/octree/disk/Layout.h); +- merge driver: [`Merger`](../../../src/sf_merger/merge.h). + +## Sparse index model + +The index has four logical states: + +| State | Physical payload | Indexed descendants | +|---|---:|---:| +| `Missing` | no | no | +| `Virtual` | no | yes | +| `Leaf` | yes | no | +| `Inner` | yes | yes | + +`Missing` is represented by absence from `IndexMap`. The other states are +defined by +[`NodeStatus`](../../../src/terrainlib/octree/NodeStatus.h). + +```mermaid +stateDiagram-v2 + [*] --> Missing + + Missing --> Leaf: add physical root or node + Missing --> Virtual: add deeper descendant + + Virtual --> Inner: add physical payload here + Leaf --> Inner: add physical descendant + + Inner --> Leaf: remove final descendant + Leaf --> Missing: remove payload and no descendants + Virtual --> Missing: remove final descendant + Inner --> Virtual: remove payload but keep descendants +``` + +This state machine is dimension-independent. What is currently 3D-specific is +how a node finds its parent and children: `octree::Id` uses three +Morton-interleaved coordinates and eight children. + +Traversal is also reusable in concept. It: + +1. looks up the root in the sparse index; +2. visits only entries that exist; +3. enumerates children through the concrete ID type; and +4. supports depth-first or breadth-first order plus a refinement predicate. + +The only reason +[`traverse()`](../../../src/terrainlib/octree/traverse.h) +is 3D is its dependency on `octree::Id` and `octree::Id::children()`. + +## Creating an SF dataset from a GDAL datasource + +The current SF builder creates physical mesh nodes at one requested octree +level. + +```mermaid +sequenceDiagram + participant CLI as sf-builder CLI + participant DS as Dataset and GDAL + participant Space as octree::Space + participant Build as terrainbuilder + participant Reader as RawDatasetReader + participant Raster as Raster and RasterMask + participant Mesh as mesh operations + participant Store as Storage + participant Index as IndexMap and terrain.index + + CLI->>DS: Open raster datasource + DS-->>CLI: SRS, 2D bounds, and height range + + CLI->>Space: Transform bounds to ECEF + Space-->>CLI: Smallest enclosing octree node + + loop Intersecting children until target level + CLI->>Space: Node bounds and intersection tests + end + + loop Each target-level node + CLI->>Build: build_patch using node bounds + Build->>Reader: Map SRS bounds to source pixels + Reader->>DS: RasterIO band 1 as float + Reader-->>Raster: Height raster + Raster->>Raster: Build NoData validity mask + Raster->>Mesh: Generate positions and triangle grid + Mesh->>Mesh: Clip to node volume + Mesh->>Mesh: Generate UVs and transform output SRS + Mesh-->>Store: SimpleMesh with optional texture + Store->>Store: MeshCodec writes node file + end + + Store->>Index: Scan output paths + Index->>Index: Add leaves and virtual ancestors + Index->>Store: Write terrain.index +``` + +### Current components + +| Responsibility | Current component | +|---|---| +| Own and open a GDAL datasource | [`Dataset`](../../../src/terrainlib/Dataset.h) | +| One-time GDAL registration | [`initialize_gdal_once()`](../../../src/terrainlib/init.cpp) | +| ECEF octree geometry | [`octree::Space`](../../../src/terrainlib/octree/Space.h) | +| Batch enumeration and orchestration | [`build_all_patches()`](../../../src/sf_builder/terrainbuilder.cpp) | +| Direct source-window reading | [`RawDatasetReader`](../../../src/sf_builder/raw_dataset_reader.h) | +| Height-to-mesh conversion | [`build_reference_mesh_patch()`](../../../src/sf_builder/mesh_builder.cpp) | +| Optional imagery | [`TileProvider`](../../../src/sf_builder/tile_provider.h) and [`texture_assembler.h`](../../../src/sf_builder/texture_assembler.h) | +| Mesh persistence | `Storage_` | +| Sparse topology | `IndexMap` | +| Index creation and serialization | `save_or_create_index()` and `terrain.index` | + +For a new output, `build_all_patches()` opens unindexed storage. Individual +saves therefore do not build the index incrementally. The final +`save_or_create_index()` recursively scans the output directory, parses node +paths, and calls `IndexMap::add()`. + +### More suitable 2D GDAL path + +The separate +[`DatasetReader`](../../../src/tile_builder/DatasetReader.h) +is closer to what a 2D SF or raster-store builder needs: + +- it accepts requested target-SRS bounds and output dimensions; +- it creates a GDAL warped VRT; +- it reprojects into the target grid; and +- it returns `radix::Raster`. + +[`Tiler`](../../../src/tile_builder/Tiler.h) and +[`ParallelTiler`](../../../src/tile_builder/ParallelTiler.h) +already enumerate `radix::tile::Id` values over dataset bounds. + +Their grid calculations are useful, but `ParallelTileGenerator` is a +delivery-file writer rather than an SF snapshot builder. + +## Mask-based SF dataset merging + +```mermaid +flowchart TD + START[Open base and new datasets as IndexedStorage] + MASK[Read vector mask through GDAL and OGR] + PREP[Polygon repair, sphere projection, triangulation, radial extrusion] + ROOT[Start at octree root] + STATUS[Read left and right NodeStatus] + POLICY[Masked visitor evaluates node] + RECURSE[Clip mask to node bounds and recurse into 8 children] + LEFT[Keep base subtree] + RIGHT[Keep new subtree] + MERGE[Clip base outside mask and new inside mask] + ATLAS[Combine meshes and rebuild texture atlas] + WRITE[Write new physical node] + COPY[Traverse unchanged source subtree] + LINK[Hard-link each physical payload] + INDEX[Scan and save new output index] + + START --> ROOT + MASK --> PREP --> POLICY + ROOT --> STATUS --> POLICY + + POLICY -->|virtual or refinement required| RECURSE + RECURSE --> STATUS + + POLICY -->|unchanged left| LEFT --> COPY + POLICY -->|unchanged right| RIGHT --> COPY + COPY --> LINK + + POLICY -->|boundary node| MERGE --> ATLAS --> WRITE + POLICY -->|no retained data| INDEX + + LINK --> INDEX + WRITE --> INDEX +``` + +### Merge components + +- [`Merger`](../../../src/sf_merger/merge.h) is the recursive dispatcher. +- [`NodeLoader`](../../../src/sf_merger/NodeLoader.h) reports status and loads + payloads. +- When an exact payload is missing, `NodeLoader` searches physical ancestors + and reconstructs the requested child by clipping the ancestor mesh. That + reconstruction is 3D-specific. +- [`NodeData`](../../../src/sf_merger/merge/NodeData.h) lazily exposes a node + payload to a visitor. +- [`Result`](../../../src/sf_merger/merge/Result.h) gives the driver four + actions: recurse, ignore, preserve one source unchanged, or write a merged + payload. +- [`Masked`](../../../src/sf_merger/merge/visitor/Masked.h) implements + mesh-and-mask policy. +- [`NodeWriter`](../../../src/sf_merger/NodeWriter.h) writes changed nodes or + copies unchanged subtrees. + +The vector-mask pipeline in +[`mask.h`](../../../src/sf_merger/mask.h) +is almost entirely 3D and Earth-specific after OGR polygon loading: + +```text +OGR polygons + -> referenced 2D polygon mask + -> ECEF and spherical projection + -> triangulated surface + -> extrusion over an Earth-radius interval + -> closed 3D MeshMask +``` + +For 2D raster merging, only the OGR polygon loading and CRS transformation +ideas remain relevant. Triangulation, spherical projection, extrusion, 3D +clipping, and texture atlases should not enter the generic core. + +## Hard-link behaviour + +Hard links are implemented at the lowest storage layer in +[`RawStorage_::copy_from()`](../../../src/terrainlib/octree/storage/RawStorage.h): + +```mermaid +flowchart TD + COPY[copy_from node] + EXISTS{Source payload exists?} + EXT{Source and target extensions match?} + DECODE[Decode source payload] + ENCODE[Encode into target format] + REMOVE[Remove existing target] + DIRS[Create parent directories] + HARDLINK[Create hard link] + DONE[Add physical node to target index] + + COPY --> EXISTS + EXISTS -->|no| ERROR1[FileNotFound] + EXISTS -->|yes| EXT + EXT -->|no| DECODE --> ENCODE --> DONE + EXT -->|yes| REMOVE --> DIRS --> HARDLINK --> DONE +``` + +Properties: + +- Hard linking is payload-agnostic and fully generalisable. +- It happens only when source and target filename extensions match. +- Different formats cause decode and re-encode through the codec. +- There is no copy fallback if hard-link creation fails, including across + filesystems. +- Snapshot immutability is not enforced by `Storage_`; it is a caller-level + convention. +- The existing target file is removed before the link is created. +- The new output receives a fresh index; `terrain.index` itself is not linked. + +## Generalisation assessment + +| Mechanism | Assessment | Required change | +|---|---|---| +| `NodeStatus` state model | Reuse essentially unchanged | Move outside `octree` naming | +| `IndexMap` state transitions | Generalise | Template over node key and tree traits | +| Sparse traversal | Generalise | Obtain children through tree traits | +| `Storage_` | Good starting point | Also template over node key and layout | +| `RawStorage_` hard-link behaviour | Generalise | Key-neutral paths and explicit fallback policy | +| Codec concept | Reuse | No dimensional dependency | +| Cache interface | Generalise | Key type is currently `octree::Id` | +| Layout strategy | Generalise | Key-neutral path API | +| Index file | Replace or version | Record topology kind, format version, and validated key | +| Merge result algebra | Generalise | `Merged` must hold generic payload, not `SimpleMesh` | +| Recursive merge driver | Generalise | Tree traits, payload, loader/writer, and policy | +| Unchanged-subtree reuse | Reuse | Enumerate every physical state, including `Inner` | +| GDAL datasource ownership | Reuse or adapt | Typed, multi-band reads and explicit NoData | +| Mask loading through OGR | Reuse or adapt | Produce dimension-specific mask representation | +| `octree::Space` | Keep 3D | Add a sibling 2D grid or space policy | +| Mesh clipping, atlas, and UV code | Keep 3D | Do not place in generic storage | +| ECEF and spherical mask conversion | Keep 3D | 2D uses polygon classification or rasterisation | +| `MeshCodec` | Keep 3D | Add a raster chunk codec or container | + +## Existing `Inner` limitation + +The index supports `Inner`, but the SF merger effectively does not: + +- `Merger::call_merge()` dispatches only `Missing`, `Leaf`, and `Virtual` + combinations. +- `Masked::visit()` declares `Inner` unreachable. +- `NodeWriter::copy_subtree_to_output()` skips `Virtual` and asserts every + other visited node is `Leaf`. + +This matters because a physical coarse node coexisting with physical +descendants is exactly the `Inner` case. A general 2D/3D implementation should +make "has a physical payload" and "has indexed descendants" independent +properties and handle all four states throughout traversal and merging. + +## Recommended target architecture + +```mermaid +flowchart TB + subgraph Core["Dimension-neutral hierarchical store"] + TRAITS["TreeTraits<Key>
root, parent, children, validation"] + SI["SparseIndex<Key>"] + TR["Traversal<Key>"] + ST["Storage<Key, Payload, Codec>"] + DL["DiskLayout<Key>"] + ME["MergeEngine<Key, Payload, Policy>"] + SNAP[Snapshot and subtree reuse] + end + + subgraph D2["2D adapter"] + TILE[radix::tile::Id] + GRID[Web Mercator grid] + RASTER[RasterChunk] + GDAL[GDAL warped-window reader] + MASK2[2D polygon or raster mask policy] + RC[Raster codec and container] + end + + subgraph D3["3D adapter"] + OCT[octree::Id] + ECEF[ECEF octree space] + MESH[SimpleMesh] + MASK3[Extruded mesh-mask policy] + MC[MeshCodec] + end + + TILE --> TRAITS + OCT --> TRAITS + + RASTER --> ST + MESH --> ST + RC --> ST + MC --> ST + + MASK2 --> ME + MASK3 --> ME + + SI --> TR + SI --> ST + DL --> ST + TR --> ME + ST --> SNAP + ME --> SNAP +``` + +The central abstractions should therefore be: + +1. `TreeTraits`: root, parent, children, maximum depth, and key + validation. +2. `SparseIndex`: the current `IndexMap` algorithm. +3. `Traversal`: DFS, BFS, and refinement independent of child + count. +4. `DiskLayout`: key-to-path and path-to-key conversion. +5. `Storage`: logical storage and index maintenance. +6. `MergeEngine`: paired sparse-tree walking. +7. Generic merge actions such as `Recurse`, `Ignore`, `KeepLeft`, `KeepRight`, + and `Write`. +8. A snapshot copier that hard-links unchanged physical nodes without knowing + their payload type. +9. Separate 2D and 3D spatial, mask, and fallback policies. + +The GDAL builder and mask merger should be clients of this core, not part of +it. diff --git a/docs/raster-store/envelope-migration-plan.md b/docs/raster-store/envelope-migration-plan.md new file mode 100644 index 00000000..0767b6be --- /dev/null +++ b/docs/raster-store/envelope-migration-plan.md @@ -0,0 +1,348 @@ +# Versioned storage-envelope migration plan + +Status: implemented in the `raster-store` worktree; awaiting commit. + +## Purpose + +This plan replaces the repository's remaining private raw ZPP Bits disk +formats with the generic versioned envelope in `terrainlib/io`. It also moves +dataset-wide format selection out of the octree index and into an independent +metadata file, splits DAG data from its cheaply readable metadata, and removes +domain transformations and validation from ZPP serialization callbacks. + +The migration intentionally breaks compatibility with the pre-envelope +`terrain.index`, `.terrain`, and `.bin` formats. Every changed format receives +a new filename ending. Once an enveloped format is introduced, later readers +must continue to read its older envelope payload versions and upgrade them to +the latest in-memory type. + +## Decisions already made + +- The generic `io::envelope` magic, class name, class version, bounded + decompression, checksum, and compression fields are used for every new + private binary format. +- Default zstd compression with its embedded checksum is used for indexes, + metadata, SF meshes, DAG data, and DAG metadata, including already encoded + meshoptimizer and JPEG byte streams. +- The new persistent names are: + - `octree.storemeta` for dataset-wide metadata; + - `octree.storeindex` for the sparse hierarchy index; + - `.sfmesh` for enveloped SF mesh nodes; + - `.dag` for enveloped DAG clustering data; and + - `.dagmeta` for enveloped DAG node metadata. +- Standard `.glb` and `.gltf` files remain unchanged. They are debugging + formats and are not wrapped in the private envelope. +- Filename endings continue to identify formats. A format change receives a + new ending; versions that remain readable by the same schema retain the + ending. +- The dataset metadata file, not the index, is authoritative for the layout + ID, payload class, and codec selector. It remains readable when the index is + missing or malformed. +- The index continues to persist the complete `store::Index` state, including + `Leaf`, `Inner`, and `Virtual`. Deterministic byte order is not required. +- A DAG logical node consists of two files. The writable batch codec writes + both `.dag` and `.dagmeta`; the read-only metadata codec reads only + `.dagmeta`. +- Hard-link behavior remains path-list based. Identical new formats use the + same endings and can be linked; changing format or compression policy uses + a different ending and therefore decodes and rewrites. +- Domain-specific ZPP serialization callbacks are removed. ZPP serializes + versioned aggregate wire types mechanically. Explicit free functions encode, + validate, and decode those wire types. +- The old `io/serialize.h` and `io/serialize.inl` API is removed after all + production and test callers migrate. +- Avoidable copies are removed with spans and moves. Compression necessarily + requires owned uncompressed and compressed buffers; zero-copy decoding is + not a goal. + +## Goals + +1. Give every private binary file envelope identity, versioning, checksum, + compression, bounded allocation, and complete-input validation. +2. Keep persistence schemas explicit and versioned without putting disk + concerns into processing types. +3. Make validation independently callable and testable. +4. Preserve cheap DAG metadata reads without reading or decompressing `.dag`. +5. Remove layout/codec guessing from directory contents. +6. Keep standard glTF behavior unchanged. +7. Preserve current application semantics, topology rules, hard-link reuse, + and error propagation apart from the deliberate format break. + +## Non-goals + +- Reading pre-envelope `terrain.index`, `.terrain`, or `.bin` files. +- Changing GLB or JSON glTF serialization. +- Defining raster-fundamentalis `.amort`, source-attribution, or raster index + payloads. +- Changing hierarchy algorithms, SF merge policy, DAG construction, mesh + processing, or snapshot publication. +- Making index serialization deterministic. +- Adding a hard-link compatibility registry or implicit transcoding between + formats with the same ending. +- Refactoring unrelated image, tile-builder, or downloader formats. + +## Format boundary + +```text +octree dataset root +├── octree.storemeta envelope: octree.StoreMetadata +├── octree.storeindex envelope: octree.StoreIndex +└── + ├── .sfmesh envelope: mesh.SfMesh + ├── .dag envelope: dag.Clustering + ├── .dagmeta envelope: dag.NodeMetadata + ├── .glb unchanged standard debug output + └── .gltf unchanged standard debug output +``` + +The files shown under one node are alternatives except that `.dag` and +`.dagmeta` form one logical DAG node. + +## Versioned payloads + +Each format defines `v1` aggregate payloads, a schema alias, a latest-type +alias, and small read/write wrappers. Future `v2` types add +`from_previous(v1::Type)` as demonstrated by the envelope tests. + +### Dataset metadata + +`octree::storage::v1::StoreMetadata` contains: + +- layout ID; +- payload class ID; and +- codec selector/format ending. + +Its validator rejects empty identifiers and unknown layouts at the adapter +boundary. Payload-specific open functions additionally require the expected +payload class and codec. + +### Sparse index + +The versioned index payload contains the complete runtime +`store::Index`. To remove the custom `octree::Id` ZPP +callback, the disk representation uses aggregate key/status entries and +explicit conversion to and from the runtime index. Conversion validates IDs, +status values, duplicates, and topology consistency. + +The conversion is allowed to retain the runtime index's semantic state and +non-deterministic iteration order. It must not silently normalize malformed +serialized status combinations. + +### SF mesh + +`mesh::sf::v1::Payload` contains fixed-width counts and encoded byte buffers +for triangles, positions, UVs, and the optional texture. The envelope version +replaces `mesh::Encoded::Header::version`. Dimension and component-type fields +that are invariant for `mesh::Simple3d` are not used for runtime dispatch. + +Free functions perform: + +```text +mesh::Simple + -> encode meshoptimizer buffers and image + -> validate v1 payload + -> envelope write + +envelope read + -> validate v1 payload + -> decode meshoptimizer buffers and image + -> mesh::Simple +``` + +### DAG clustering and metadata + +The `.dag` payload contains the encoded clustering only. Its aggregate DTOs +use fixed-width counts, primitive coordinate aggregates, encoded +meshoptimizer buffers, encoded JPEG byte vectors, IDs, texture references, +and errors. The `.dagmeta` payload independently contains group assignment, +groups, child IDs, bounds, and errors. + +The writable `Codec` returns both paths, encodes the two +payloads, and writes both envelopes. Its read path reads both envelopes and +combines their decoded latest types. `Codec` returns and +reads only `.dagmeta`, with writes unsupported. + +Validation rejects malformed encoded buffers, invalid octree IDs, invalid +cluster/group/texture references, inconsistent counts, and dimensions that +would overflow allocations before constructing processing objects. + +## Serialization and I/O cleanup + +One bounded ZPP byte API supports the envelope implementation. Public path +helpers read and write `io::envelope::Bytes`; path wrappers compose byte I/O +with `io::envelope::serialize` and `deserialize`. All error conversion occurs +at the format/codec boundary. + +The migration removes: + +- raw `io::write_to_bytes`, `read_from_bytes`, `write_to_path`, and + `read_from_path` serialization; +- the `octree::Id` custom serializer and its embedded `try_make` validation; +- `store::Index` and `NodeStatus` persistence hooks; +- `mesh::Encoded` and `ImageAndExt` persistence hooks; +- DAG GLM, ID, metadata, texture, cluster, clustering, and batch serializers; + and +- unused standalone clustering persistence helpers. + +`io::bytes` remains the filesystem byte layer. Envelope path helpers expose +its `uint8_t` buffers as `std::byte` spans without copying. + +## Opening and creation lifecycle + +Opening an existing private dataset requires both fixed root files: + +1. Read and validate `octree.storemeta`. +2. Resolve the layout and payload codec from metadata. +3. Read and validate `octree.storeindex`. +4. Construct indexed storage. + +An unreadable metadata or index file returns a typed open error. There is no +directory scan fallback. + +Creating a dataset receives an explicit layout and codec, creates an indexed +in-memory store immediately, and writes node payloads while updating that +index. Finalization writes dataset metadata and the index. The implementation +does not infer layout or codec from existing node filenames. + +## Implementation phases + +### Phase 0 — Plan and semantic baseline + +- Record this approved design. +- Retain semantic round-trip, store, merge, and builder tests rather than raw + legacy bytes. +- Confirm the clean raster-store worktree and current local commits. + +Exit criterion: format names, compatibility boundary, split DAG layout, and +metadata authority are explicit. + +### Phase 1 — Consolidate envelope I/O + +- Add envelope path read/write helpers and zero-copy byte-span bridging to the + filesystem I/O layer. +- Provide a latest-version serialization overload so ordinary writers do not + repeat the latest version number. +- Remove avoidable intermediate copies. +- Add path I/O and latest-version tests. + +Exit criterion: production codecs can use the envelope without the legacy raw +serialization API. + +### Phase 2 — Version dataset metadata and index + +- Add versioned dataset metadata and index DTOs and schemas. +- Add free validation/conversion functions. +- Write `octree.storemeta` and `octree.storeindex` through the envelope. +- Separate dataset metadata persistence from index persistence in the shared + storage plumbing. +- Remove directory discovery and index reconstruction by extension scanning. +- Start new outputs indexed in memory. + +Exit criterion: a new empty or populated store opens only through its metadata +and index, and corrupt/mismatched files return typed failures. + +### Phase 3 — Migrate SF mesh payloads + +- Add the versioned `.sfmesh` wire payload. +- Move meshoptimizer and texture work into explicit encode/decode functions. +- Add free validation for counts, buffers, texture data, and mesh semantics. +- Update the SF mesh codec resolver and CLI defaults to `.sfmesh`. +- Retain `.glb`/`.gltf` unchanged. + +Exit criterion: SF builder/merger/store tests round-trip `.sfmesh`; standard +debug codecs still behave as before; no `.terrain` production path remains. + +### Phase 4 — Split and migrate DAG payloads + +- Add versioned `.dag` clustering and `.dagmeta` metadata payloads. +- Move all DAG encoding/decoding out of ZPP callbacks. +- Add a two-file writable/readable batch codec. +- Add a separate read-only `.dagmeta` codec. +- Update DAG builder and debug converter callers. + +Exit criterion: full batches round-trip through two files, metadata reads touch +only `.dagmeta`, and malformed encoded data returns errors rather than +asserting. + +### Phase 5 — Remove legacy serialization and inference + +- Delete domain-specific serializers and `io/serialize.h/.inl`. +- Delete unused standalone clustering persistence. +- Remove `.terrain`, `.bin`, and `terrain.index` private-format selectors and + tests. +- Remove unindexed directory discovery and suffix-probe index scans. +- Update documentation, golden scripts, test expectations, and error text. + +Exit criterion: repository search finds no production raw ZPP persistence or +legacy private extension and no format inference from directory contents. + +### Phase 6 — Verification + +- Run focused envelope, index, codec, storage, SF, and DAG tests after their + phases. +- Run the complete terrainlib, DAG builder, SF builder finalization, and SF + merger suites. +- Build `sf-builder`, `sf-merger`, `sf-index-browser`, `dag-builder`, and + `dag-convert-debug` in the repository build configuration. +- Run the golden end-to-end workflow with new expected endings where its data + prerequisites are available. +- Inspect `git diff --check` and the final worktree. + +## Required tests + +- Every production schema round-trips v1 through default zstd. +- Each schema rejects the wrong envelope class and unsupported versions. +- Corrupt compressed data and malformed aggregate payloads return typed + errors. +- Dataset metadata remains readable when `octree.storeindex` is corrupt. +- Payload class, codec, and layout mismatches fail before node processing. +- Invalid octree IDs, node statuses, duplicates, and inconsistent topology are + rejected by free validation/conversion functions. +- SF mesh encode/decode preserves current semantic mesh equality and rejects + malformed meshoptimizer/image data. +- DAG full-batch reads require `.dag` and `.dagmeta`. +- DAG metadata reads succeed with only `.dagmeta` present and do not access + `.dag`. +- Same-format node copies hard-link both DAG files; differing formats + decode/re-encode. +- GLB/GLTF tests remain unchanged. +- New dataset construction never scans extensions to decide its format. + +## Risks and controls + +| Risk | Control | +|---|---| +| A future reader cannot load v1 | Versioned namespaces, schema conversion trail, and retained v1 tests | +| Dataset metadata and index disagree | Metadata is authoritative; index contains topology only | +| Broken index hides format information | Independent `octree.storemeta` is read and tested separately | +| DAG metadata reads regress to full data reads | Separate `.dagmeta` file and read-only codec tests | +| Custom ZPP callbacks continue to hide failures | Aggregate DTOs plus explicit expected-returning validation/decode | +| Meshoptimizer corruption aborts through assertions | Checked decode wrappers return typed errors | +| New compression adds excessive copies | Shared byte type, spans, moves, and copy-focused review | +| New format is accidentally linked to an old one | Every changed format uses a new ending | +| Standard debug formats are accidentally wrapped | Dedicated unchanged GLB/GLTF codecs and existing tests | +| Incomplete output becomes authoritative | Keep explicit finalization boundaries and write root metadata/index last | + +## Expected migration map + +| Current | Target | +|---|---| +| `terrain.index` raw ZPP DTO | `octree.storeindex` envelope schema | +| layout and preferred extension inside index | `octree.storemeta` envelope schema | +| `.terrain` raw encoded mesh | `.sfmesh` envelope schema | +| `.bin` DAG batch | `.dag` clustering plus `.dagmeta` metadata envelopes | +| `.bin` metadata-prefix view | read-only `.dagmeta` codec | +| extension/layout directory discovery | explicit metadata-driven open | +| unindexed save then recursive scan | indexed in-memory creation and final write | +| domain `serialize(Archive&, T&)` callbacks | aggregate DTO plus free encode/validate/decode | +| `io/serialize.h/.inl` | envelope path helpers | +| `.glb` / `.gltf` | unchanged | + +## Verification result + +The implementation builds the SF builder/merger, DAG builder/debug converter, +and SF index browser targets. The terrainlib, DAG builder, SF finalization, SF +merger, and SF builder/merger integration tests pass with the new formats. The +golden end-to-end script has been migrated to the new filenames and passes +shell syntax validation; its external-data workflow was not run as part of +this implementation. diff --git a/docs/raster-store/golden-e2e.sh b/docs/raster-store/golden-e2e.sh new file mode 100755 index 00000000..df44f53e --- /dev/null +++ b/docs/raster-store/golden-e2e.sh @@ -0,0 +1,321 @@ +#!/usr/bin/env bash + +# Repeatable, non-unit end-to-end verification for the current 3D SF/DAG path. +# Inputs are prepared once under INPUT_ROOT. Completed steps are skipped through +# explicit state markers, so an interrupted SF or DAG run can be resumed. + +set -Eeuo pipefail + +readonly SOURCE_DIR="${SOURCE_DIR:-/home/codex/Documents/alpine-terrain-builder/terrain-builder-raster-store}" +readonly BUILD_DIR="${BUILD_DIR:-${SOURCE_DIR}/build/golden-e2e}" +readonly RUN_ROOT="${RUN_ROOT:-/data/scratch/codex/alpine-terrain-builder-golden-e2e}" +readonly INPUT_ROOT="${INPUT_ROOT:-${RUN_ROOT}/inputs}" +readonly REFERENCE_ROOT="${REFERENCE_ROOT:-${RUN_ROOT}/reference}" +readonly LOG_ROOT="${LOG_ROOT:-${RUN_ROOT}/logs}" +readonly STATE_ROOT="${STATE_ROOT:-${RUN_ROOT}/state}" +readonly RUN_ID="${RUN_ID:-$(date +%Y%m%dT%H%M%S)}" +readonly RUN_LOG_ROOT="${LOG_ROOT}/runs/${RUN_ID}" +readonly TIMINGS_FILE="${RUN_LOG_ROOT}/timings.tsv" + +readonly SF_BUILDER="${BUILD_DIR}/src/sf_builder/sf-builder" +readonly SF_MERGER="${BUILD_DIR}/src/sf_merger/sf-merger" +readonly DAG_BUILDER="${BUILD_DIR}/src/dag_builder/dag-builder" + +# These VRTs are identical 4x4 km, native-resolution windows into the GS and +# GT rasters. They are recreated from the raw datasets at the start of a run. +readonly RAW_GS="/data/raw/raster_data/Oe_2020/OeRect_01m_gs_31287.img" +readonly RAW_GT="/data/raw/raster_data/Oe_2020/OeRect_01m_gt_31287.img" +readonly ELEVATION_ROOT="${ELEVATION_ROOT:-${INPUT_ROOT}/grossglockner-elevation-4km}" +readonly GS_VRT="${ELEVATION_ROOT}/grossglockner-gs-4km.vrt" +readonly GT_VRT="${ELEVATION_ROOT}/grossglockner-gt-4km.vrt" +readonly BASEMAP_TILES="${INPUT_ROOT}/basemap" +readonly GATAKI_TILES="${INPUT_ROOT}/gataki" +# A 0.01 m simplification preserves the Tirol border to sub-centimetre area +# accuracy while avoiding the exact geometry's pathological merge time. +readonly TIROL_MASK="${INPUT_ROOT}/tirol-boundary/benchmark-shape/0_01m/tirol.shp" + +readonly SF_ROOT="${REFERENCE_ROOT}/sf" +readonly DAG_ROOT="${REFERENCE_ROOT}/dag" +readonly SF_BASEMAP="${SF_ROOT}/grossglockner-basemap-gs-terrain" +readonly SF_GATAKI="${SF_ROOT}/grossglockner-gataki-gt-terrain" +readonly SF_MERGED="${SF_ROOT}/grossglockner-merged-terrain" +readonly DAG_MERGED="${DAG_ROOT}/grossglockner-merged-terrain" + +readonly SF_TARGET_LEVEL="${SF_TARGET_LEVEL:-15}" +readonly SF_THREADS="${SF_THREADS:-12}" +readonly MIN_TEXTURE_LEVEL=12 +readonly MAX_TEXTURE_LEVEL=19 + +mkdir -p "${SF_ROOT}" "${DAG_ROOT}" "${RUN_LOG_ROOT}" "${STATE_ROOT}" +exec > >(tee -a "${RUN_LOG_ROOT}/golden-e2e.log") 2>&1 + +timestamp() +{ + date --iso-8601=seconds +} + +run_timed() +{ + local name="$1" + shift + + local start_epoch end_epoch elapsed status + start_epoch="$(date +%s)" + printf '[%s] START %s\n' "$(timestamp)" "${name}" + + set +e + "$@" + status=$? + set -e + + end_epoch="$(date +%s)" + elapsed=$((end_epoch - start_epoch)) + printf '%s\t%s\t%s\t%s\t%s\n' \ + "${name}" "${start_epoch}" "${end_epoch}" "${elapsed}" "${status}" \ + >> "${TIMINGS_FILE}" + printf '[%s] END %s status=%s elapsed_seconds=%s\n' \ + "$(timestamp)" "${name}" "${status}" "${elapsed}" + + if ((status != 0)); then + return "${status}" + fi +} + +require_file() +{ + if [[ ! -f "$1" ]]; then + printf 'Required file is missing: %s\n' "$1" >&2 + exit 2 + fi +} + +require_directory() +{ + if [[ ! -d "$1" ]]; then + printf 'Required directory is missing: %s\n' "$1" >&2 + exit 2 + fi +} + +require_executable() +{ + if [[ ! -x "$1" ]]; then + printf 'Required executable is missing: %s\n' "$1" >&2 + exit 2 + fi +} + +verify_snapshot() +{ + local snapshot="$1" + local extension="$2" + local companion_extension="${3:-}" + local payload_count + + require_file "${snapshot}/octree.storemeta" + require_file "${snapshot}/octree.storeindex" + if [[ ! -s "${snapshot}/octree.storemeta" ]] || [[ ! -s "${snapshot}/octree.storeindex" ]]; then + printf 'Octree metadata or index is empty: %s\n' "${snapshot}" >&2 + return 1 + fi + + payload_count="$(find "${snapshot}" -type f -name "*${extension}" | wc -l)" + if ((payload_count == 0)); then + printf 'No %s payloads found in %s\n' "${extension}" "${snapshot}" >&2 + return 1 + fi + if [[ -n "${companion_extension}" ]]; then + local companion_count + companion_count="$(find "${snapshot}" -type f -name "*${companion_extension}" | wc -l)" + if [[ "${companion_count}" != "${payload_count}" ]]; then + printf '%s payload count (%s) does not match %s count (%s) in %s\n' \ + "${extension}" "${payload_count}" "${companion_extension}" "${companion_count}" "${snapshot}" >&2 + return 1 + fi + fi + printf 'Verified %s: %s payloads\n' "${snapshot}" "${payload_count}" +} + +build_sf() +{ + local name="$1" + local dataset="$2" + local textures="$3" + local output="$4" + local marker="${STATE_ROOT}/${name}.complete" + + if [[ -f "${marker}" ]]; then + printf '[%s] SKIP %s: completion marker exists\n' "$(timestamp)" "${name}" + verify_snapshot "${output}" ".sfmesh" + return + fi + + mkdir -p "${output}" + run_timed "${name}" \ + "${SF_BUILDER}" \ + --dataset "${dataset}" \ + --textures "${textures}" \ + --min-texture-level "${MIN_TEXTURE_LEVEL}" \ + --max-texture-level "${MAX_TEXTURE_LEVEL}" \ + --mesh-srs EPSG:4978 \ + --verbosity info \ + batch \ + --target-level "${SF_TARGET_LEVEL}" \ + --output "${output}" \ + --format .sfmesh \ + --threads "${SF_THREADS}" + + verify_snapshot "${output}" ".sfmesh" + touch "${marker}" +} + +merge_sf() +{ + local marker="${STATE_ROOT}/merge_sf.complete" + + if [[ -f "${marker}" ]]; then + printf '[%s] SKIP merge_sf: completion marker exists\n' "$(timestamp)" + verify_snapshot "${SF_MERGED}" ".sfmesh" + return + fi + + if [[ -d "${SF_MERGED}" ]] && [[ -n "$(find "${SF_MERGED}" -mindepth 1 -print -quit)" ]]; then + printf 'Partial merge output exists at %s.\n' "${SF_MERGED}" >&2 + printf 'The current sf-merger cannot safely resume this output; refusing to overwrite it.\n' >&2 + return 1 + fi + + mkdir -p "${SF_MERGED}" + run_timed merge_sf \ + "${SF_MERGER}" merge \ + --base "${SF_BASEMAP}" \ + --new "${SF_GATAKI}" \ + --mask "${TIROL_MASK}" \ + --output "${SF_MERGED}" \ + --verbosity info + + verify_snapshot "${SF_MERGED}" ".sfmesh" + touch "${marker}" +} + +build_dag() +{ + local marker="${STATE_ROOT}/build_dag.complete" + local continuation=(--overwrite) + + if [[ -f "${marker}" ]]; then + printf '[%s] SKIP build_dag: completion marker exists\n' "$(timestamp)" + verify_snapshot "${DAG_MERGED}" ".dag" ".dagmeta" + return + fi + + mkdir -p "${DAG_MERGED}" + if [[ -s "${DAG_MERGED}/octree.storeindex" ]]; then + continuation=(--resume) + fi + + run_timed build_dag \ + "${DAG_BUILDER}" \ + --input "${SF_MERGED}" \ + --output "${DAG_MERGED}" \ + "${continuation[@]}" \ + --verbosity info + + verify_snapshot "${DAG_MERGED}" ".dag" ".dagmeta" + touch "${marker}" +} + +write_manifest() +{ + local name="$1" + local snapshot="$2" + local output="${RUN_LOG_ROOT}/${name}.sha256" + + ( + cd "${snapshot}" + find . -type f ! -name octree.storeindex ! -name octree.storemeta -print0 \ + | sort -z \ + | xargs -0 -r sha256sum + ) > "${output}" +} + +record_hard_links() +{ + local merged_count=0 + local linked_to_gataki=0 + local linked_to_basemap=0 + local newly_written=0 + local merged_file relative merged_inode candidate_inode + + while IFS= read -r -d '' merged_file; do + relative="${merged_file#"${SF_MERGED}/"}" + merged_inode="$(stat -c '%d:%i' "${merged_file}")" + ((merged_count += 1)) + + if [[ -f "${SF_GATAKI}/${relative}" ]]; then + candidate_inode="$(stat -c '%d:%i' "${SF_GATAKI}/${relative}")" + if [[ "${merged_inode}" == "${candidate_inode}" ]]; then + ((linked_to_gataki += 1)) + continue + fi + fi + + if [[ -f "${SF_BASEMAP}/${relative}" ]]; then + candidate_inode="$(stat -c '%d:%i' "${SF_BASEMAP}/${relative}")" + if [[ "${merged_inode}" == "${candidate_inode}" ]]; then + ((linked_to_basemap += 1)) + continue + fi + fi + + ((newly_written += 1)) + done < <(find "${SF_MERGED}" -type f -name '*.sfmesh' -print0) + + printf 'merged_payloads=%s\nlinked_to_gataki=%s\nlinked_to_basemap=%s\nnewly_written=%s\n' \ + "${merged_count}" "${linked_to_gataki}" "${linked_to_basemap}" "${newly_written}" \ + | tee "${RUN_LOG_ROOT}/merge-hard-links.txt" +} + +prepare_elevation_vrts() +{ + mkdir -p "${ELEVATION_ROOT}" + gdal_translate -q -of VRT -srcwin 259507 245043 4000 4000 "${RAW_GS}" "${GS_VRT}" + gdal_translate -q -of VRT -srcwin 259507 245043 4000 4000 "${RAW_GT}" "${GT_VRT}" + gdal_edit.py -units m "${GS_VRT}" + gdal_edit.py -units m "${GT_VRT}" +} + +require_file "${RAW_GS}" +require_file "${RAW_GT}" +run_timed prepare_elevation_vrts prepare_elevation_vrts +require_file "${GS_VRT}" +require_file "${GT_VRT}" +require_file "${TIROL_MASK}" +require_directory "${BASEMAP_TILES}" +require_directory "${GATAKI_TILES}" +require_executable "${SF_BUILDER}" +require_executable "${SF_MERGER}" +require_executable "${DAG_BUILDER}" + +{ + printf 'run_started=%s\n' "$(timestamp)" + printf 'git_commit=%s\n' "$(git -C "${SOURCE_DIR}" rev-parse HEAD)" + printf 'git_status=%q\n' "$(git -C "${SOURCE_DIR}" status --porcelain=v1 --branch)" + printf 'sf_target_level=%s\n' "${SF_TARGET_LEVEL}" + printf 'sf_threads=%s\n' "${SF_THREADS}" + printf 'min_texture_level=%s\n' "${MIN_TEXTURE_LEVEL}" + printf 'max_texture_level=%s\n' "${MAX_TEXTURE_LEVEL}" + printf 'cpu_count=%s\n' "$(nproc)" +} >> "${RUN_LOG_ROOT}/run-metadata.txt" + +build_sf build_sf_basemap_gs_terrain "${GS_VRT}" "${BASEMAP_TILES}" "${SF_BASEMAP}" +build_sf build_sf_gataki_gt_terrain "${GT_VRT}" "${GATAKI_TILES}" "${SF_GATAKI}" +merge_sf +record_hard_links +build_dag +run_timed manifest_sf_basemap write_manifest sf-basemap "${SF_BASEMAP}" +run_timed manifest_sf_gataki write_manifest sf-gataki "${SF_GATAKI}" +run_timed manifest_sf_merged write_manifest sf-merged "${SF_MERGED}" +run_timed manifest_dag_merged write_manifest dag-merged "${DAG_MERGED}" + +printf '[%s] Golden end-to-end run complete\n' "$(timestamp)" diff --git a/docs/raster-store/refactor-plan.md b/docs/raster-store/refactor-plan.md new file mode 100644 index 00000000..63b17e2c --- /dev/null +++ b/docs/raster-store/refactor-plan.md @@ -0,0 +1,1132 @@ +# 2D/3D hierarchical store refactor plan + +Status: proposal for review. This document is an implementation plan, not a +record of completed work. + +## Purpose + +This plan describes how to extract the existing octree-specific index, +traversal, storage, and codec code into a shared 2D/3D store. The refactor +must preserve existing 3D datasets and prove, using test-only mappings and +codecs where necessary, that the shared mechanisms work with a 2D key. +Persistent raster-fundamentalis formats, adapters, and tools are later work. + +## Decisions already made + +- The shared implementation will live in `src/terrainlib/store` and use the + `store` namespace. +- The minimal 2D traits adapter used to exercise the shared hierarchy will + live in `src/terrainlib/raster_store` and use the `raster_store` namespace. +- Common Structura Fundamentalis validation and errors will live in + `src/terrainlib/sf` and use the `sf` namespace. +- Existing 3D Structura Fundamentalis datasets must remain readable and + writable without changing their on-disk contract. +- DAG datasets written by the current code when Phase 0 begins must remain + readable and writable throughout the refactor without changing their index + or payload serialization. Older DAG payload schemas are not supported. +- `Inner` is a valid shared topology state and is supported by DAG, + raster-fundamentalis, and tile-base datasets. It is not valid in Structura + Fundamentalis datasets. +- SF producer and processing boundaries validate indexed SF data and return a + typed `sf::InvalidTopology` error containing an offending key when `Inner` + is present. The diagnostic `sf_index_browser` is exempt. This is an SF data + invariant, not a generic store or octree-format rule. +- 3D compatibility includes both existing path layouts: + `flat` and `level_and_coordinate_directories`. +- A persistent 2D raster tile format is not defined by this refactor. + [architecture.md](architecture.md) and + [storage-format.md](storage-format.md) describe intended direction and + provisional requirements that will be finalized in later RF work. They must + not be retrofitted onto existing 3D datasets. +- A path layout strategy should contain a stable identifier and two + operations: key to extensionless `NodePath`, and `NodePath` to key. It + should not require an inheritance hierarchy, RTTI, global + self-registration, or heap allocation. +- A configured codec owns all filename endings and maps one `NodePath` to one + or more physical files. `Codec::paths()` must not need a payload. +- Codecs are stateful runtime objects behind a small interface. They may + support reading, writing, or both. Reading and writing return + `std::expected`; unsupported operations and other operational failures are + reported as error values. Reading and writing must be reentrant (callable + concurrently from different threads). +- The runtime glTF codec catches exceptions from the existing glTF mesh writer + and converts them to `CodecError`; changing the mesh I/O API is out of scope. +- `store::RawStorage` exclusively owns the configured + `std::unique_ptr>`. `store::Storage` owns the raw + storage and therefore owns the codec transitively. Storage consumers do not + receive a codec template parameter and application call sites do not manage + codec objects. +- This refactor does not add synchronization to storage, indexes, or caches + and does not change their concurrency guarantees. Any concurrency bug fix is + separate work. +- Application callers retain their existing storage lifecycle, + synchronization, and failure-handling behaviour unless explicitly changed + by this plan. +- Preserve `dag::ThreadSafeStorage` as the caller-side synchronization around + DAG output storage. DAG storage remains cacheless while shared-lock reads are + used. +- Legacy index metadata selects a codec through an explicit, caller-supplied + resolver supplied by the payload-domain opening function. The octree format + adapter does not contain a global codec registry or depend on mesh or DAG + payload types. Application-level storage consumers do not call the resolver + or handle the resulting codec object. +- Dimension-specific index persistence uses a small runtime + `store::IndexFormat` value containing ordinary function pointers. It + is not an inheritance hierarchy and does not use global registration. +- `copy_from()` hard-links every file when the input and output codecs return + the same path list for a common dummy `NodePath`. Otherwise it decodes with + the input codec and encodes with the output codec. +- Public store operations reject invalid hierarchy keys through + `std::expected`; invalid keys are not represented by assertions or generic + booleans. +- Preserve `StorageSettings::allow_overwrite`. It defaults to `false`; + rejected overwrites return `AlreadyExists` through `std::expected`, and + enabling it replaces existing payloads. +- The existing cache implementations are currently non-functional. Port their + public API where it remains useful, keep application call sites building, and + provide compile coverage only. Cache behaviour is not a compatibility + requirement of this refactor. +- The `.png` written beside changed meshes by `sf_merger::NodeWriter` is an + unmanaged debug artifact. It is not part of a logical node, is not returned + by `Codec::paths()`, and is not indexed or copied by storage. Its existing + application-local behaviour is preserved. +- 3D geometry, ECEF bounds, mesh codecs, mesh reconstruction, mask geometry, + and raster-specific processing remain outside the shared store. + +## Goals + +1. Use one sparse hierarchy implementation for `octree::Id` and + `radix::tile::Id`. +2. Make traversal, storage, cache, and the runtime codec boundary + dimension-neutral while keeping production format adapters 3D-only in this + refactor. +3. Replace the current layout-strategy class hierarchy with small path-mapping + values backed by function pairs. +4. Allow one logical node payload to consist of multiple files without making + layouts aware of those files. +5. Preserve all valid existing 3D index files and payload paths. +6. Prove the shared topology and traversal with `radix::tile::Id` without + defining a persistent 2D adapter or format. +7. Land the refactor in small, testable steps. Every phase should build and + pass tests before the next phase begins. + +## Non-goals + +- Changing `octree::Id`, `octree::Space`, `IdRect`, `OddLevelShifted`, or + other 3D spatial calculations. +- Defining or implementing GDAL ingestion, raster resampling, filtering, + source selection, or mask rasterisation. +- Defining or implementing a raster-fundamentalis index, payload format, + layout ID, codec, publication lifecycle, or persistent storage adapter. +- Implementing an `rf_builder`, `rf_merger`, tile-base generator, tile server, + snapshot-reuse operation, or other RF tool in this refactor. +- Defining the raster-fundamentalis merge policy or the final paired-hierarchy + walker/action algebra. That work is deferred until `rf_merger` requirements + are defined. +- Defining a shared subtree-copy abstraction, `Inner` subtree-copy behaviour, + or forced re-encoding policy for RF. Those decisions are deferred until + `rf_merger`. +- Adding merge semantics for `Inner` nodes in SF. Such nodes are invalid SF + input and must be rejected before merge dispatch. +- Changing the existing 3D hard-link policy by adding a silent file-copy + fallback. +- Refactoring unrelated octree, DAG, mesh, or tile-builder code. +- Changing the serialized schema of the current DAG `.bin` payloads. + +## Compatibility contract + +Before moving code, tests must lock down the following 3D behaviour: + +| Item | Required compatibility | +|---|---| +| Index filename | `terrain.index` | +| Index field order | layout ID, preferred extension, index map | +| Node-key encoding | existing `octree::Id` level/index serialization | +| Node-status encoding | `Leaf = 0`, `Inner = 1`, `Virtual = 2` | +| Valid SF statuses | `Leaf` and `Virtual`; reject `Inner` with `sf::InvalidTopology` | +| Valid DAG statuses | `Leaf`, `Inner`, and `Virtual` | +| Flat layout ID | `flat` | +| Flat path | `-` | +| Coordinate layout ID | `level_and_coordinate_directories` | +| Coordinate path | `///` | +| Default layout | existing level/coordinate layout | +| Layout detection | both existing layouts remain detectable | +| Mesh codec selection | legacy preferred extension selects terrain or configured glTF codec | +| DAG codec selection | legacy `.bin` preferred extension selects the ZPP Bits codec | +| DAG payload encoding | current `dag::ClusterBatch` ZPP Bits serialization: metadata, then clustering | +| Equal codec path lists | hard-link every file, or report an explicit error | +| Different codec path lists | decode with input codec and encode with output codec | +| Overwrite setting | `StorageSettings::allow_overwrite`, default `false`; enabled writes replace existing payloads | + +Compatibility means that each refactor phase can open the Phase 0 fixtures +written by the current code. DAG formats older than the Phase 0 baseline and +pre-refactor readers opening post-refactor output are not tested. Exact +byte-for-byte rewriting of an unordered index map is not required, but the +serialized schema and values must remain compatible during the migration. + +The 3D index disk type should remain a versioned 3D adapter. The shared store +must not add a topology field, new header, checksum, or compression layer to +`terrain.index`. + +## Proposed source boundary + +```text +src/terrainlib/ +├── store/ +│ ├── NodeStatus.h +│ ├── NodeStatusOrMissing.h +│ ├── Traits.h +│ ├── InvalidKey.h +│ ├── Index.h +│ ├── traverse.h +│ ├── NodePath.h +│ ├── PathMapping.h +│ ├── Layout.h +│ ├── IndexFormat.h +│ ├── Codec.h +│ ├── CodecError.h +│ ├── OpenError.h +│ ├── CopyError.h +│ ├── StorageSettings.h +│ ├── RawStorage.h +│ ├── Storage.h +│ ├── IndexedStorage.h +│ ├── cache/ +│ │ ├── Interface.h +│ │ ├── Dummy.h +│ │ └── Lru.h +│ └── codec/ +│ └── ZppBits.h +├── mesh/ +│ └── codec/ +│ ├── Terrain.h +│ └── Gltf.h +├── sf/ +│ ├── InvalidTopology.h +│ └── validate_index.h +├── octree/ +│ ├── Id.h +│ ├── StoreTraits.h +│ ├── store_layout/ +│ │ ├── Flat.h +│ │ ├── LevelAndCoordinateDirectories.h +│ │ └── Mappings.h +│ └── storage/ +│ ├── IndexFile.h +│ └── open.h +└── raster_store/ + └── StoreTraits.h +``` + +The exact file grouping may be collapsed if a file would only contain a few +lines. The important boundaries are: + +- `store` contains dimension- and payload-neutral mechanisms; +- `store::codec::ZppBits` is the reusable concrete codec for payload types + that provide ZPP Bits serialization; +- `mesh::codec` contains the separately configured terrain and glTF codecs; +- `sf` contains SF-specific topology validation and errors shared by + `sf_builder`, `sf_merger`, and `dag_builder`; +- `octree` contains the 3D format and key adapters; +- `raster_store` contains only the minimal 2D hierarchy traits adapter in this + refactor; and +- subdirectory names match their namespaces where a subnamespace is used. + +Shared topology and octree-format adapters accept `Inner`. The SF restriction +is enforced by `sf::validate_index()` at SF producer/consumer boundaries and +reported as `sf::InvalidTopology`. It must not be embedded in `store::Index`, +traversal, or the generic 3D disk adapter. `sf_index_browser` is a diagnostic +tool and intentionally does not apply SF validation, so it can display invalid +trees including `Inner`. + +`sf::validate_index()` returns +`std::expected`. SF application-level error types +must retain this error and `CopyError` when propagating failures; neither is +reduced to a log message, assertion, or generic boolean. + +For this refactor, `sf::validate_index()` checks only for `Inner`. It does not +validate other structural or disk/filesystem invariants. Possible future +extensions are recorded in [todo.md](todo.md). + +Temporary forwarding headers and aliases under `octree` are allowed during +migration. They must not contain a second implementation. + +DAG serialization remains owned by `dag_builder`. Consolidate the serializers +for `dag::Id`, `dag::Group`, `dag::NodeMetadata`, `dag::ClusterBatch`, +`radix::geometry::Aabb3d`, GLM vectors, `Clustering`, `Cluster`, and +`TextureSet` in `src/dag_builder/serialization.h`. The DAG storage adapter +includes that header explicitly so template instantiation does not depend on +caller include order. Preserve the current tuples exactly: + +- `ClusterBatch`: metadata, clustering; +- `NodeMetadata`: group assignment, groups; and +- `Group`: children, error, bounds. + +## Shared interfaces + +The names below are the intended shape, not signatures that must be copied +verbatim without testing. + +### Hierarchy traits + +The store should be parameterized by one traits type rather than assuming that +all key classes expose identical member functions: + +```cpp +template +concept HierarchyTraits = requires(typename Traits::Key key) { + typename Traits::Key; + typename Traits::Hasher; + { Traits::root() } -> std::same_as; + { Traits::parent(key) }; + { Traits::children(key) }; + { Traits::is_valid(key) } -> std::same_as; +}; +``` + +The concrete names should be: + +```cpp +store::Index +store::Index +``` + +`octree::StoreTraits` adapts the existing optional parent/children API without +changing `octree::Id`. + +`raster_store::StoreTraits` adapts `radix::tile::Id` and must: + +- treat zoom zero as the only root; +- never call `radix::tile::Id::parent()` at zoom zero, where it underflows; +- reject coordinates outside `[0, 2^zoom)`; +- accept zoom levels 0 through + `std::numeric_limits::digits`, inclusive; +- treat that maximum zoom as terminal because a child cannot be represented + by the `uint32_t` x/y coordinates; +- validate the maximum zoom without evaluating an overflowing + `uint32_t{1} << 32`; +- use `radix::tile::Id::Hasher`; and +- return children in the deterministic order produced by + `radix::tile::Id::children()`. + +This traits adapter defines only hierarchy operations. It does not define +persistent coordinates, a path layout, or an RF disk format. + +The shared code must obtain roots, parents, children, validation, and hashing +through the traits. It must not use dimension checks or specialize behaviour +on key types internally. + +Operations accepting a key validate it through `Traits::is_valid()`. Index +lookup and mutation, traversal with an explicit root, and storage operations +return an `std::expected` retaining an `InvalidKey` when validation fails. +Keys produced internally by `Traits::root()`, `Traits::parent()`, and +`Traits::children()` are trusted only after trait-specific tests establish that +they preserve validity. The child order affects traversal order and is locked +down by the 2D and 3D trait tests; it is not serialized as separate metadata. + +The API result shapes are: + +- index lookup returns `expected, InvalidKey>`; +- index mutation and predicates return `expected>`; +- traversal returns `expected>`; and +- storage `load`, `save`, `has`, `remove`, path lookup, and `copy_from` return + operation-specific `expected` types which retain `InvalidKey` and any + applicable codec, filesystem, missing-source, or overwrite error. + +During Phase 1, a forwarding `octree::IndexMap` compatibility wrapper preserves +the old optional/bool API for existing 3D callers with already-valid +`octree::Id` values. It delegates to `store::Index`, is not +a second implementation, and is removed after callers migrate. + +### Sparse index and traversal + +Move the existing four-state model to: + +```cpp +store::NodeStatus +store::NodeStatusOrMissing +store::Index +store::traverse(index, visitor, refine, root, order) +``` + +The index algorithm remains the current one: + +- adding a physical descendant creates virtual ancestors; +- adding a payload to a virtual node makes it `Inner`; +- removing a payload from an `Inner` node makes it `Virtual`; +- removing the final descendant collapses virtual ancestors; and +- a physical parent becomes `Leaf` after its last descendant is removed. + +Traversal must follow only indexed nodes and use `Traits::children`. Child +order is the order supplied by the traits and is therefore deterministic per +hierarchy, not universally fixed by `store`. + +### Node paths and path mappings + +`store::NodePath` is an extensionless logical location for one hierarchy +node. For example: + +```text +octree flat 12-123456 +octree coordinates 12/34/56/78 +``` + +It does not necessarily name a physical file. Replace +`octree::disk::layout::Strategy` and +`octree::disk::layout::StrategyRegister` with a value similar to: + +```cpp +template +struct store::PathMapping { + std::string_view id; + NodePath (*key_to_node_path)(const Key&); + std::optional (*node_path_to_key)(const NodePath&); +}; +``` + +`store::Layout` owns the base directory and one `PathMapping`. It +does not own a preferred extension. A configured codec expands the +extensionless `NodePath` into the physical file or files. + +The stable ID is format metadata, not a third strategy operation. The +dimension adapters provide ordinary lookup functions: + +```cpp +octree::store_layout::flat() +octree::store_layout::level_and_coordinate_directories() +octree::store_layout::from_id(id) +octree::store_layout::all() +``` + +This retains runtime selection from an index file while removing virtual +dispatch, RTTI type-to-ID lookup, static registration, and ownership through +`unique_ptr`. + +Path parsers validate the complete logical `NodePath`, not a file ending. +Codec or format-adapter code removes and validates physical file endings +before asking the layout to recover a key. Invalid disk input returns an error +or `nullopt`; it must not trigger an assertion. + +### Codec interface + +A codec is a configured runtime object for one logical payload type. It owns +all physical filename endings and may map one `NodePath` to several files: + +```cpp +template +class store::Codec { +public: + virtual ~Codec() = default; + + virtual std::vector + paths(const NodePath& node_path) const = 0; + + virtual std::expected + read(const NodePath&) const { + return std::unexpected( + CodecError::unsupported_operation("read")); + } + + virtual std::expected write( + const NodePath&, + const NodeData&) const { + return std::unexpected( + CodecError::unsupported_operation("write")); + } +}; +``` + +Concrete codecs contain their configuration and are constructed before +storage use. `CodecError` is a payload-neutral operational error that records +the failed operation, an error category, and a diagnostic message. Concrete +codecs convert their domain errors to it. Unsupported read or write operations +may use the base implementation and return its `UnsupportedOperation` error. +Codec writes preserve the current directory-creation behaviour: they create +the parent directories required by their output paths before writing. The +storage hard-link path continues to create its target parent directories before +linking. + +`Codec::paths()` has the following contract: + +- it needs no NodeData payload and performs no filesystem access; +- it returns every physical file belonging to the logical node; +- results depend only on codec configuration and the supplied `NodePath`; +- result order is stable and pairs corresponding input/output files; +- two codecs returning the same path list for the same `NodePath` must produce + mutually compatible files; and +- different artifact counts or filename endings produce different lists. + +Examples: + +```text +Terrain codec + 12/34/56/78 + -> 12/34/56/78.terrain + +glTF codec configured for binary output + 12/34/56/78 + -> 12/34/56/78.glb + +glTF codec configured for JSON output + 12/34/56/78 + -> 12/34/56/78.gltf + +Multi-file test codec + 12/34/56/78 + -> 12/34/56/78.data + -> 12/34/56/78.metadata +``` + +The mesh side has separate terrain and glTF codecs because they use different +format implementations. Binary `.glb` and JSON `.gltf` remain configurations +of one glTF codec because both use the same `cgltf` implementation. + +The shared module also provides: + +```cpp +template +struct store::codec::ZppBits : store::Codec { .. }; +``` + +It uses the existing `io::read_from_path()` and `io::write_to_path()` +functions, maps one node to `.bin`, and converts `io::Error` to +`CodecError`. It contains no DAG-specific serialization logic. DAG payload +serialization remains in `dag_builder/serialization.h`, and its field order +and meshoptimizer/JPEG encoding remain unchanged. + +### Storage and format adapters + +Generalize storage over traits and NodeData. It owns a configured codec through +the runtime interface: + +```cpp +store::RawStorage +store::Storage +store::IndexedStorage +store::cache::Interface +``` + +`RawStorage` owns the +`std::unique_ptr>`. `Storage` owns `RawStorage`, and +`IndexedStorage` owns or derives from `Storage`; no other layer shares codec +ownership. Moving storage transfers ownership. Caches, layouts, index formats, +resolvers, and application consumers never own the codec. + +A move disarms the source so its destructor cannot save moved-out index state. +Move assignment must finalize a displaced dirty destination through the +existing destructor-save policy rather than silently discard it. Tests cover +the DAG builder's move into and release from `dag::ThreadSafeStorage`. + +Domain-specific mesh codecs remain under `mesh::codec`. The reusable ZPP Bits +codec remains under `store::codec`. RF codecs are deferred. + +Index serialization is not a responsibility of `store::Index`. Opening and +saving a dataset receives the following small runtime values: + +```cpp +template +struct store::IndexMetadata { + store::Index index; + std::string layout_id; + std::string codec_selector; +}; + +template +struct store::IndexFormat { + std::string_view index_filename; + + std::expected, IndexFormatError> + (*read)(const std::filesystem::path& index_path); + + std::expected + (*write)( + const std::filesystem::path& index_path, + const IndexMetadata& metadata); + + std::optional> + (*mapping_from_id)(std::string_view id); + + PathMapping + (*default_mapping)(); +}; +``` + +The value provides: + +- the index filename; +- index read/write conversion; +- mapping lookup by stable ID; +- the default mapping. + +Legacy directory discovery is an octree opening helper which composes the +octree index format, the supplied payload-domain codec resolver, and the known +octree mappings. It is not a generic `IndexFormat` operation. Neither the +format value nor discovery may reintroduce a layout class hierarchy or global +registration. + +For 3D, the adapter reads and writes the current `octree` index DTO unchanged. +Its `codec_selector` is exactly the legacy `preferred_extension`, including +the leading dot. Storage retains the selected `IndexFormat`, index path, +layout ID, and codec selector as its index-persistence state, so explicit and +destructor-triggered index saves can reproduce the same metadata. + +When opening indexed storage or discovering a legacy unindexed directory, it +passes the legacy `preferred_extension` to a caller-supplied codec resolver. +The resolver is an ordinary callable and returns +`std::expected>, CodecError>`. It is not +a global registry. + +The payload domains provide ordinary resolver functions: + +```text +mesh::codec::from_extension + .terrain -> terrain codec + .glb -> glTF codec with binary container + .gltf -> glTF codec with JSON container + +dag::codec::from_extension + .bin -> store::codec::ZppBits + +dag::codec::metadata_from_extension + .bin -> read-only dag::codec::MetadataView +``` + +An unknown extension returns an explicit `UnsupportedCodec` error. Opening a +new empty store receives an explicit legacy codec selector plus an already +constructed codec at the payload-domain opening boundary; the generic storage +does not infer persistent metadata from `Codec::paths()`. Mesh and DAG +convenience functions select or resolve the codec and pass ownership into raw +storage, so application storage consumers do not handle codec objects. +Convenience functions in `src/dag_builder/storage.h` supply the DAG resolvers. +Preserve `DagMetaStorage` and `IndexedDagMetaStorage` as independently readable +views of the metadata prefix in each `ClusterBatch` `.bin` file. Their codec +returns `UnsupportedOperation` from writes instead of terminating or replacing +a full batch with metadata alone. `dag::codec::MetadataView` implements this +domain-specific read-only prefix view over the generic ZPP Bits codec. + +Opening functions return their requested storage type through +`std::expected<..., OpenError>`. `OpenError` is a typed sum which retains the +failing path and the underlying error where applicable: + +- index I/O or malformed index metadata (`IndexFormatError`); +- filesystem failure; +- unknown layout ID; +- unsupported codec selector or codec construction failure (`CodecError`); and +- invalid hierarchy key (`InvalidKey`). + +Loading and saving likewise return storage-level expected errors which retain +an invalid key, an underlying `CodecError`, and `AlreadyExists` for a rejected +save. `CopyError` retains invalid-key, missing-source, overwrite, filesystem, +and codec failures. The SF call chains changed in Phase 4 retain these errors +to their application boundary. + +Legacy unindexed-directory discovery remains in the 3D adapter: it recognizes +candidate endings by asking the supplied resolver, removes an accepted ending +to obtain a `NodePath`, and then invokes the selected layout parser. The +generic layout does not recover keys directly from codec-owned file paths. +Only a missing index triggers this discovery path. An unreadable or malformed +index, unknown layout, or unsupported codec selector returns `OpenError` and +must not silently fall back to directory discovery. + +### Copying one node and SF subtree reuse + +There are two separate responsibilities in the current implementation: + +1. `sf_merger::NodeWriter` traverses a source subtree and + `sf_merger::cut_leaf_node()` identifies an unchanged leaf. +2. `octree::Storage::copy_from()` delegates to + `octree::RawStorage::copy_from()`, where + `std::filesystem::create_hard_link()` performs the actual hard link and the + target index is updated on success. + +The filesystem hard-link implementation is therefore already in terrainlib. +Move the one-node storage operation into the shared store, but keep subtree +selection and traversal in `sf_merger`. There is no current DAG caller, and RF +subtree-copy requirements will be defined with `rf_merger`. + +The migrated call chains are: + +```text +sf_merger decides to keep a source subtree unchanged + -> NodeWriter traverses the source index + -> store::Storage::copy_from() + -> hard-link every codec path, or decode/encode + +sf_merger determines that a cut leaf is unchanged + -> cut_leaf_node() + -> store::Storage::copy_from() + -> hard-link every codec path, or decode/encode +``` + +`Storage::copy_from()` remains the operation for copying one logical node. +For one key, `copy_from()`: + +1. calls the input and output `Codec::paths()` with the same fixed dummy + `NodePath`; +2. when the lists are equal, calls both codecs again with their actual source + and target `NodePath` values and hard-links every source path to the + corresponding target path; +3. when the dummy lists differ, reads the payload with the input codec and + writes it with the output codec; and +4. updates the target index only after all links or the write complete. + +The dummy path must be fixed and collision-free, for example +`__codec_probe__/node`. Path lists are compared exactly, including count, +order, and filename endings. + +If overwrite is enabled for an indexed target, remove the target index entry +immediately before the first target file is modified. If linking several files +then fails partway through, return the error without a transactional rollback +guarantee; target links or old files may remain, but the logical node stays +unindexed. The copy operation stops immediately and propagates the failure +until the application aborts the overall operation. There is no journal, +rollback, cleanup guarantee, or silent copy fallback. An unsupported read or +write needed for re-encoding is returned through `CopyError`, retaining the +underlying `CodecError`. + +Hard-link rules: + +- never modify an existing linked payload in place; +- a matching codec path list hard-links every file; +- a different path list decodes with the input codec and encodes with the + output codec; +- hard-link failure is explicit; +- no silent file-copy fallback is introduced. + +#### SF-local subtree traversal + +`sf_merger::NodeWriter::copy_subtree_to_output()` remains in `sf_merger`. It +uses shared traversal and `Storage::copy_from()`, but it is not promoted to a +generic store API during this refactor. + +SF validation guarantees that its indexed inputs contain only `Leaf` and +`Virtual` nodes. The SF-local traversal skips `Virtual`, copies `Leaf`, and +does not define behaviour for `Inner`. An `Inner` node is rejected before +merge or cut processing with `sf::InvalidTopology` containing the offending +key. + +When `rf_merger` is designed, it can initially compose `store::traverse` and +`Storage::copy_from()`. At that point, the SF and RF implementations provide +enough evidence to decide whether a shared subtree copier is useful and how it +must handle RF `Inner` nodes. + +#### Error propagation + +The lower storage layer already represents ordinary copy failures, including +missing source files, directory creation failure, hard-link failure, decode +failure, and encode failure. The current `NodeWriter` consumes +`Storage::copy_from()` with `DEBUG_ASSERT_VAL`, as does the unchanged-leaf path +in `cut_leaf_node()`, while some overwrite paths terminate through +`LOG_ERROR_AND_EXIT()`. Their `void` call chains prevent the application from +reporting or handling these failures. + +Return failures from the SF-local subtree and cut functions through their +callers until the application boundary can report the affected key and path. +Codec, filesystem, unsupported-conversion, malformed-dataset, and overwrite +failures are propagated with `std::expected`; `CopyError` retains any +underlying `CodecError`. Assertions remain appropriate for internal +invariants, but operational failures are not assertion failures or +intentionally thrown exceptions. + +### Paired hierarchy walking is deferred + +This refactor does not extract `sf_merger::Merger` into a shared paired-tree +walker. SF only permits `Missing`, `Leaf`, and `Virtual`, and its current +recursion does not provide enough evidence to define the `Inner` behaviour +needed by raster-fundamentalis and tile-base merging. + +The previously proposed mutually exclusive actions `Recurse`, `Ignore`, +`KeepLeft`, `KeepRight`, and `Write` cannot express both an action for the +current physical payload and recursion into descendants. `Inner` merging may +require both. Whether the future interface uses a combined `WriteAndRecurse` +action or independent current-node and descendant decisions belongs to the +`rf_merger` design. + +For this refactor: + +- keep recursion, subtree traversal, and mesh policy in `sf_merger`; +- validate SF inputs and reject `Inner` through `std::expected`; +- move only the one-node `Storage::copy_from()` mechanism into `store`; and +- do not add a shared `store::merge` namespace. + +A future `rf_merger` task will define the paired-tree action algebra from the +2D requirements and may migrate `sf_merger` once both use cases are known. + +## Implementation phases + +Each phase should be one reviewable commit unless the tests and implementation +are clearer as two commits. Do not begin a later phase while the current phase +has failing tests. + +### Phase 0 — Capture current compatibility + +No production behaviour changes. + +1. Add golden SF fixtures created by the current code: + - one `terrain.index` using `flat`; + - one using `level_and_coordinate_directories`; + - across the fixtures, physical payload paths for a root, child, and deeper + descendant, without placing physical payloads at ancestor and descendant + keys in the same index; and + - index entries containing `Leaf` and `Virtual`, but no `Inner`. +2. Add one golden DAG dataset whose index selects `.bin`, whose payload + contains a valid serialized `dag::ClusterBatch`, and whose index contains + `Leaf`, `Virtual`, and `Inner`. Use the current metadata-then-clustering + format, include non-trivial group metadata, and prove that the same file can + be opened through the read-only `NodeMetadata` view. +3. Test that all fixtures open, resolve the expected IDs and extensions, and + traverse the expected sparse nodes. +4. Add path round-trip tests for boundary IDs and both layouts. +5. Add storage tests for: + - matching-extension hard links; + - different-extension decode/re-encode; + - overwrite-enabled replacement; + - indexed and unindexed opens; and + - final index creation by directory scan. +6. Record the pre-refactor public aliases used by `sf_builder`, `sf_merger`, + `sf_index_browser`, `dag_builder`, and `dag_convert_debug`, including + `DagMetaStorage`, `IndexedDagMetaStorage`, and `dag::ThreadSafeStorage`. + +Exit criterion: the compatibility tests pass against the untouched +implementation and fail when any stable filename, layout ID, path encoding, +status value, index field order, or DAG payload serialization is deliberately +changed. + +### Phase 1 — Extract topology into `store` + +1. Move `NodeStatus` and `NodeStatusOrMissing` to `store`, preserving their + underlying values and serialization. +2. Introduce the hierarchy-traits concept and `octree::StoreTraits`. +3. Convert `IndexMap` into `store::Index`. +4. Convert traversal into `store::traverse`. +5. Add `raster_store::StoreTraits` for `radix::tile::Id`. +6. Run the same index-transition and DFS/BFS tests with both trait types, + including deterministic child order, invalid-key errors through + `std::expected`, maximum-depth children, and explicit traversal roots. +7. Provide the temporary forwarding `octree::IndexMap` wrapper and aliases so + downstream migration is separate; otherwise migrate downstream immediately. + +Exit criterion: 2D and 3D keys pass the same topology suite; existing 3D +callers still build through aliases; no filesystem code has changed. + +### Phase 2 — Introduce path mappings and runtime codecs + +This phase introduces the extensionless layout and runtime codec pieces +together. It does not cut production storage over to them yet. The existing +storage, layout strategies, and static codecs remain temporarily as the +working compatibility path until Phase 3 can replace the complete +layout-plus-codec path construction in one step. + +1. Add `store::NodePath`, `store::PathMapping`, and + `store::Layout`. +2. Add the stateful `store::Codec` interface with `paths()`, `read()`, + and `write()`, plus `CodecError`. +3. Add the runtime `store::codec::ZppBits`, preserving the existing `.bin` + path and serialized payload bytes. +4. Consolidate the DAG serialization functions in + `src/dag_builder/serialization.h` without changing their serialized field + order, meshoptimizer encoding, or JPEG texture encoding. +5. Add the terrain codec and one glTF codec configured for binary `.glb` or + JSON `.gltf`. Keep the current extension-dispatching `octree::MeshCodec` + only as temporary production compatibility glue until the Phase 3 cutover. + Test that glTF writer exceptions become `CodecError` values. +6. Port the two existing 3D layouts to ordinary function pairs without + changing stable IDs. The mappings return `level-index` and + `level/x/y/z` without file endings. +7. Add explicit `from_id()` and `all()` lookup functions in the 3D adapter. + Keep the singleton strategy registry only for the old production storage + path until Phase 3. +8. Compose each new 3D mapping with each applicable runtime codec in tests and + prove that they resolve all Phase 0 fixtures to identical physical payload + paths. +9. Add focused codec tests using single-file, multi-file, read/write, and + write-only test codecs. Test stable path ordering, unsupported operations, + directory creation, conversion of domain errors to `CodecError`, and + concurrent reads and writes. Exercise reentrancy of every production codec + used by the parallel DAG builder. + +Exit criterion: the extensionless mappings and runtime codecs together resolve +all Phase 0 fixtures to their existing physical payload paths; generic +`Layout` contains no extension; the new codec tests pass; existing production +storage and all callers still build unchanged through the temporary legacy +path. + +### Phase 3 — Generalize storage and index lifecycle + +1. Cut the production path construction over to the Phase 2 + `store::Layout` and runtime codecs. Move the legacy preferred extension + out of layout state and retain it as the 3D format adapter's codec selector. +2. Port legacy layout discovery so it recognizes codec endings through the + supplied resolver, strips the accepted ending, and then calls + `node_path_to_key()`. +3. Move copy error, raw storage, logical storage, and indexed storage into + `store`. Port the existing cache API where useful for source compatibility, + but do not require cache behaviour tests. +4. Make `RawStorage` exclusively own a configured + `std::unique_ptr>`. `Storage` owns it transitively through + raw storage; remove the codec template parameter from every storage type. +5. Replace every embedded `octree::Id` with `Traits::Key`. +6. Make every raw file operation obtain its complete file list through + `Codec::paths()`. `has()` requires every listed file, and `remove()` removes + every listed file. +7. Keep domain-specific mesh codecs outside the shared module under + `mesh::codec`. +8. Add the function-pointer-based `IndexFormat`, `IndexMetadata`, and + typed format/open errors. Split generic index maintenance from 3D index + serialization and legacy folder discovery. Test that only a missing index + starts discovery; all other index errors are returned. +9. Keep the current 3D `terrain.index` DTO and open functions as compatibility + adapters over the shared storage. Retain its exact preferred extension as + `codec_selector`, resolve it through the payload-domain mesh or DAG + resolver, and retain the format metadata required by automatic saving. +10. Add DAG storage convenience functions that supply the writable batch and + read-only metadata resolvers, and mesh storage convenience functions that + select the configured terrain or glTF codec. Preserve the DAG metadata + storage aliases. Migrate `dag_builder`, `dag_convert_debug`, and mesh + storage consumers without exposing codec objects at application call sites. +11. Migrate the existing octree storage aliases and all other application + callers, including adapting `dag::ThreadSafeStorage` to the new key and + expected-returning APIs. +12. Preserve the current 3D destructor-save behaviour until all callers have + explicit index finalization. Test move construction, move assignment over + dirty state, moved-from destruction, and the DAG move/release path. +13. Preserve `StorageSettings::allow_overwrite`. Replace process termination + on a rejected overwrite with `AlreadyExists` in the storage-level expected + error. Test that a rejected save returns `AlreadyExists` and that enabling + overwrite replaces the existing payload. +14. Add resolver tests for `.terrain`, `.glb`, `.gltf`, writable + `ClusterBatch` `.bin`, and read-only `NodeMetadata` `.bin` dispatch, plus + explicit failure for an unknown preferred extension and metadata writes. +15. Instantiate the shared storage tests with `raster_store::StoreTraits` + using a test-only path mapping and codec. This proves the storage templates + contain no hidden `octree::Id` dependency without defining a stable RF + layout, codec, or disk format. +16. Delete the old strategy base class, strategy registry, concrete strategy + classes, static codec concept and codecs, and their temporary compatibility + glue once no call site uses them. + +Test that the Phase 0 DAG fixture opens through both new resolvers and that a +new deterministic `.bin` payload matches its golden bytes and remains readable +through the unchanged ZPP serialization functions. Test unknown layout IDs, +malformed index metadata, invalid hierarchy keys, and retained underlying +open/codec errors through `std::expected`. + +Exit criterion: all existing applications build and all Phase 0 fixtures pass +through the shared runtime codec and storage implementation. Phase 0 DAG +payload bytes and `.bin` paths remain unchanged. No extension-dispatching +mesh codec, layout inheritance, RTTI lookup, static registrar, owning strategy +pointer, or second storage implementation remains under `octree`. + +### Phase 4 — Harden node reuse and enforce SF topology + +1. Change `Storage::copy_from()` to compare input and output codec path lists + for the fixed dummy `NodePath`. +2. Hard-link all actual files when the lists match. A partially failed + multi-file operation returns an error without rolling back links already + created. +3. Decode with the input codec and encode with the output codec when lists + differ. +4. Test: + - one-file hard linking; + - multi-file hard linking; + - different path counts and endings; + - error propagation after a partially failed multi-file hard link; + - overwrite failure leaving an existing logical node unindexed even when + old or partial files remain; + - conversion between terrain and glTF; + - `copy_from()` overwrite rejection; + - `copy_from()` overwrite-enabled replacement; and + - missing-file, hard-link, decode, and encode error propagation. +5. Add `sf::validate_index()`, returning `sf::InvalidTopology` with the + offending key when it encounters `Inner`. Do not add other validation rules + in this refactor. +6. Apply the validator to SF-merger merge and cut inputs and the DAG builder's + SF input before processing. Apply it to SF-builder and SF-merger output + after the completed `terrain.index` has been written. If output validation + fails, retain the written index and payloads for diagnosis, propagate + `sf::InvalidTopology` to the command line, and report that the output is + invalid. Do not apply the validator when opening DAG datasets, through + generic octree/store adapters, or in the diagnostic `sf_index_browser`. + Extract an SF-builder finalization boundary into `sfbuilderlib` which writes + the index, validates it, and returns the typed result so output validation + and command-line propagation are directly testable. +7. Keep SF recursion, subtree traversal, and mesh policy in `sf_merger`. + Change its subtree and cut call chains to propagate validation and + `copy_from()` failures through `std::expected` to the application boundary. +8. Add integration tests proving: + - valid `Leaf`/`Virtual` SF merge behaviour is unchanged; + - an SF input containing `Inner` fails validation before merge dispatch; + - invalid SF output writes `terrain.index` for diagnosis and returns + `sf::InvalidTopology` to the application boundary; + - an unchanged SF subtree is hard-linked and a changed boundary node is + newly written; and + - an unchanged leaf in the SF cut path is hard-linked while a clipped leaf + is newly written. + +Exit criterion: one-node copying works through `Codec::paths()`, SF consumers +reject `Inner` with a typed error before processing, existing valid SF merge +and cut behaviour is preserved, invalid SF output remains inspectable with a +written `terrain.index`, and neither a shared subtree copier nor a paired-tree +walker has been introduced. + +### Phase 5 — Cleanup and documentation + +1. Remove temporary forwarding headers that no repository caller needs. +2. Remove obsolete files under `octree/disk` and the old generic + implementation under `octree/storage`. +3. Keep only 3D key, format, codec, and compatibility adapters under + `octree`. +4. Update includes, CMake source lists, and precompiled-header includes. +5. Update [architecture.md](architecture.md), + [storage-format.md](storage-format.md), [status-quo.md](status-quo.md), and + the before-refactor report with links to the implemented boundary. State + clearly that RF formats and tools remain future work and that architecture + requirements are not acceptance criteria for this refactor. Preserve the + before-refactor report as history; do not rewrite it as if it described the + new code. +6. Document the final public names, a legacy 3D opening example, and an + in-memory `radix::tile::Id` topology example. Do not document a persistent + RF format or opening API. + +Exit criterion: repository search finds no generic implementation tied to +`octree::Id`; all tests pass; the old layout strategy hierarchy is gone. + +### Phase 6 — Remove migration-only compatibility fixtures + +1. Delete the Phase 0 golden SF and DAG datasets and the tests whose only + purpose is opening or byte-comparing data written before the completed + refactor. +2. Keep the format round-trip, path, resolver, topology, storage, error, and + application integration tests which exercise the final implementation + without pre-refactor fixture files. + +Exit criterion: no pre-refactor dataset fixture remains, and the retained test +suite passes against data produced by the final implementation. + +## Test and verification plan + +Generic store tests should live in the existing `unittests_terrainlib` target. +Suggested files: + +```text +unittests/terrainlib/store_index.cpp +unittests/terrainlib/store_traverse.cpp +unittests/terrainlib/store_layout.cpp +unittests/terrainlib/store_codec.cpp +unittests/terrainlib/store_storage.cpp +unittests/terrainlib/store_compatibility.cpp # temporary through Phase 5 +unittests/terrainlib/sf_validate_index.cpp +``` + +The temporary DAG payload-compatibility fixture and resolver integration test +belong in `unittests_dagbuilder`, because `terrainlib` must not depend on +`dag::ClusterBatch` or its serializers. Phase 6 removes the fixture and its +byte-comparison test while retaining resolver tests built from current data. + +The validator's unit tests belong in `unittests_terrainlib`. Boundary tests +belong with their consumers: SF-builder output validation in +`unittests_sfbuilder`, merge and cut validation/error propagation in +`unittests_sfmerger`, and DAG-builder SF-input validation in +`unittests_dagbuilder`. `sf_index_browser` remains unvalidated by design. + +Cache migration has compile coverage only. Existing cache application call +sites must continue to build where the API remains meaningful, but no cache +behaviour or compatibility test is required. + +During implementation: + +1. Build in `$source_dir/build/$config_name`. +2. Run unit tests from that build directory. +3. Run the focused store tests after each edit. +4. Run the full `unittests_terrainlib` target at every phase boundary. +5. Run `unittests_dagbuilder` after the ZPP codec, DAG serialization header, or + DAG resolver changes. +6. Configure with `ALP_BUILD_SF_BUILDER`, `ALP_BUILD_SF_MERGER`, + `ALP_BUILD_SF_INDEX_BROWSER`, `ALP_BUILD_DAG_BUILDER`, and + `ALP_BUILD_DAG_CONVERT_DEBUG` enabled, then build `sf-builder`, `sf-merger`, + `sf-index-browser`, `dag-builder`, and `dag-convert-debug` after their + storage aliases move. +7. Run `unittests_sfbuilder`, `unittests_sfmerger`, and + `unittests_dagbuilder`, plus any existing merger integration fixture, after + the Phase 4 validation changes. +8. Inspect `git diff --check` and the final worktree before each commit. + +No formatting-only pass or unrelated refactor belongs in these commits. + +## Expected migration map + +| Current code | Target | +|---|---| +| `octree/NodeStatus.h` | `store/NodeStatus.h` plus temporary alias | +| `octree/NodeStatusOrMissing.h` | `store/NodeStatusOrMissing.h` plus temporary alias | +| `octree/IndexMap.*` | `store/Index.h` | +| `octree/traverse.h` | `store/traverse.h` | +| complete node paths embedded in layouts | extensionless `store/NodePath.h` plus codec endings | +| `octree/disk/Layout.h` | `store/Layout.h` | +| `octree/disk/layout/Strategy.h` | `store/PathMapping.h` | +| `StrategyRegister.h` | explicit dimension-adapter lookup functions | +| `strategy/Flat.h` | `octree/store_layout/Flat.h` | +| `strategy/LevelAndCoordinateDirectories.h` | `octree/store_layout/LevelAndCoordinateDirectories.h` | +| `octree/storage/cache/*` | `store/cache/*` where useful; compile compatibility only | +| `octree/storage/codec/Codec.h` | runtime `store/Codec.h` | +| `octree/storage/codec/DefaultCodec.h` | runtime `store/codec/ZppBits.h` | +| `octree/storage/codec/MeshCodec.h` | `mesh/codec/Terrain.h` and configured `mesh/codec/Gltf.h` | +| `octree/storage/codec/ReadOnlyCodec.h` metadata specialization | `dag::codec::MetadataView` | +| `octree/storage/RawStorage.h` | `store/RawStorage.h` | +| `octree/storage/Storage.h` | `store/Storage.h` | +| `octree/storage/IndexedStorage.h` | `store/IndexedStorage.h` | +| `octree::StorageSettings::allow_overwrite` | `store::StorageSettings::allow_overwrite`, preserving default and enabled behaviour | +| `octree/storage/helpers.*` | generic scan helpers plus 3D format adapter | +| `octree/disk/IndexFile.h` | versioned 3D format adapter under `octree` | +| DAG serializers in `dag_node.h`, `dag_id.h`, `metadata.h`, `encoded.h`, and `zpp_bits_glm.h` | `dag_builder/serialization.h` | +| `dag_builder/storage.h` aliases | Batch and read-only metadata storage aliases plus codec resolver convenience functions | +| `dag_builder/thread_safe_storage.h` | Adapted caller-side wrapper over shared storage | +| `sf_merger::NodeWriter` subtree loop | remains in `sf_merger`; return copy failures through `std::expected` | +| `sf_merger::NodeWriter` auxiliary `.png` write | remains an unmanaged, application-local debug artifact | +| `sf_merger::cut_leaf_node()` copy path | remains in `sf_merger`; return copy failures through `std::expected` | +| SF merger `Inner` `UNREACHABLE()` path | `sf::validate_index()` returning `sf::InvalidTopology` | + +## Risks and controls + +| Risk | Control | +|---|---| +| Existing indexes stop loading | Golden pre-refactor fixtures and unchanged 3D DTO | +| The refactor changes current DAG `.bin` bytes | Temporary golden DAG fixture, unchanged serializers, and explicit `.bin` resolvers | +| Octree format adapter gains DAG dependencies | Caller-supplied resolver owned by `dag_builder` | +| Unknown legacy extension silently selects the wrong codec | Return an explicit `UnsupportedCodec` error | +| Valid legacy paths are parsed differently | Characterization and round-trip tests before replacement | +| Template migration creates a large unreviewable diff | Compatibility aliases and phase-by-phase caller migration | +| `radix::tile::Id` root underflows | Traits intercept root parent lookup | +| Invalid `radix::tile::Id` values enter the shared index | Validate through `raster_store::StoreTraits` and test boundary zooms | +| Invalid `Inner` nodes reach SF merge dispatch | Validate every SF input first and return the offending key in a typed error | +| SF subtree copying is generalized before RF requirements exist | Keep it in `sf_merger`; reconsider extraction with `rf_merger` | +| A paired-walker API is fixed before RF semantics are known | Defer its action algebra until `rf_merger` requirements are defined | +| Multi-file hard linking fails partway through | Remove any existing index entry before modifying target files, stop immediately, leave the node unindexed, propagate the error, and abort the overall operation; old or partial files may remain | +| Incompatible codecs return the same path list | Treat path-list equality as a codec contract and test every concrete codec pairing | +| Output-only codec is selected for required input | Return a clear `UnsupportedOperation` error | +| Shared code accumulates mesh or provisional RF policy | Dependency tests/review against the source boundary | +| The refactor accidentally fixes the future RF disk format | Do not add a persistent 2D format adapter, layout, or codec | + +## Deferred raster-fundamentalis work + +This refactor is a prerequisite for, not an implementation of, the system +described in [architecture.md](architecture.md). That document and +[storage-format.md](storage-format.md) remain useful design input, but their RF +details are not acceptance criteria for this refactor and are not declared +final here. + +A later RF design phase must resolve and test at least: + +- the persistent index filename and schema, and how they use the existing + generic serialization envelope and versioning support; +- persistent tile keys, coordinate convention, layout IDs, and node paths; +- tile and source-attribution payload formats and codecs; +- snapshot construction, validation, publication, and crash expectations; +- hard-link preflight and unchanged-tile reuse; +- `rf_merger` policy, paired-tree walking, and `Inner` copy behaviour; and +- RF builders, converters, debugging outputs, and other tools. + +None of those decisions blocks completion of this refactor. +The removed ideas are retained, without implementation-plan status, in the +explicitly draft [rf_builder notes](rf_builder.md) and +[rf_merger notes](rf_merger.md). diff --git a/docs/raster-store/refactor-status.md b/docs/raster-store/refactor-status.md new file mode 100644 index 00000000..3a82dc46 --- /dev/null +++ b/docs/raster-store/refactor-status.md @@ -0,0 +1,237 @@ +# Raster store refactor status + +This file is the resumable implementation log for +[refactor-plan.md](refactor-plan.md). + +## Baseline + +- Branch: `raster-store` +- Baseline commit: `d1309f54cfcd5756786735c6a2c849419c187ed7` +- Baseline tag: `refactor_before` +- Started: 2026-08-14 +- Worktree was clean before implementation began. +- `dev_driver.py doctor` rejects this repository because its root + `CMakeLists.txt` is not the alpine-renderer project. Builds therefore use + this repository's own CMake configuration and build directories. + +## Current position + +- Phase: 6 — Remove migration-only compatibility fixtures +- State: complete; completion commit pending. +- Next action: commit and tag Phase 6, then perform the final clean-worktree + and tag audit. + +## Phase log + +### Phase 0 — Capture current compatibility + +- Added pre-refactor SF fixtures for `flat` and + `level_and_coordinate_directories` layouts. +- Added a pre-refactor DAG fixture using `.bin`, with `Leaf`, `Virtual`, and + `Inner` index states and non-trivial metadata. +- Added characterization coverage for fixture opening, metadata-prefix reads, + payload byte stability, layout/path round trips, hard links, + decode/re-encode, enabled overwrites, indexed/unindexed opens, and index + creation by directory scan. +- Recorded pre-refactor public aliases: + - SF builder, SF merger, and SF index browser use `octree::MeshStorage` and + `octree::IndexedStorage` (the latter aliases `IndexedMeshStorage`). + - DAG builder and DAG convert debug use `octree::DagStorage`, + `octree::IndexedDagStorage`, `octree::DagMetaStorage`, and + `octree::IndexedDagMetaStorage`. + - DAG output synchronization uses + `dag::ThreadSafeStorage`. +- State: complete. + +### Phase 1 — Extract topology into `store` + +- Added shared node-status types, hierarchy traits, invalid-key errors, + sparse index, and traversal under `terrainlib/store`. +- Added `octree::StoreTraits` and `raster_store::StoreTraits`. +- Converted `octree::IndexMap` and `octree::traverse` to compatibility + wrappers over the shared implementation. +- Added common 2D/3D topology and traversal coverage plus 2D boundary and + invalid-key tests. +- State: complete. + +### Phase 2 — Introduce path mappings and runtime codecs + +- Added extensionless `NodePath`, `PathMapping`, and `Layout` values. +- Added the stateful runtime `Codec` interface and typed `CodecError`. +- Added runtime ZPP Bits, terrain, binary glTF, and JSON glTF codecs. +- Added ordinary `flat` and `level_and_coordinate_directories` mapping + function pairs plus explicit lookup. +- Consolidated DAG serialization in `dag_builder/serialization.h` and made + the legacy DAG storage adapter include it explicitly. +- Added mapping/codec composition, parser validation, single-/multi-file, + unsupported-operation, write-only, directory-creation, error-conversion, + and concurrent codec tests. +- State: complete. + +### Phase 3 — Generalize storage and index lifecycle + +- Added typed storage, filesystem, overwrite, and index-format errors. +- Added shared `RawStorage`, `Storage`, and `IndexedStorage` with exclusive + runtime-codec ownership, trait-keyed operations, multi-file `has`/`remove`, + overwrite handling, index persistence, and move finalization. +- Added shared storage instantiations for both 2D and 3D traits plus move, + overwrite, invalid-key, and multi-file tests. +- Shared storage core focused tests: 48 assertions in 6 test cases passed. +- Shared storage core build and `git diff --check`: passed. +- Added the legacy 3D `terrain.index` format adapter, explicit mesh and DAG + codec resolvers, folder discovery, and expected-returning opening APIs. +- Preserved independently readable, read-only DAG metadata views and migrated + DAG output synchronization to the shared storage types. +- Migrated SF builder, SF merger, SF index browser, DAG builder, and DAG + convert debug without exposing codec ownership at application call sites. +- Ported cache API aliases for compile compatibility. +- Deleted the legacy layout class hierarchy and registry, static codec stack, + and duplicate octree storage implementation. +- Added resolver, unsupported metadata write, unknown layout/codec, malformed + index no-fallback, and DAG synchronized move/release coverage. +- State: complete. + +### Phase 4 — Harden node reuse and enforce SF topology + +- Hardened `copy_from()` around the fixed codec probe path: matching path + lists hard-link every physical file, differing lists decode/re-encode, and + actual path-count mismatches return a typed codec error. +- Moved overwrite index removal to the mutation boundary. Missing-source and + decode failures leave the old logical target indexed; encode and partial + multi-file hard-link failures leave it unindexed for safety. +- Added one-file and multi-file hard-link, re-encode, overwrite, + missing-source, decode, encode, and partial-hard-link failure coverage. +- Added `sf::InvalidTopology`, `sf::validate_index()`, typed SF processing and + finalization errors, and an SF finalization boundary that writes the index + before validating it. +- Applied SF validation before SF merge/cut and DAG processing and after SF + builder/merger output finalization. The diagnostic index browser remains + exempt. +- Propagated SF merger save, copy, open, index, and validation failures through + `std::expected` to the command-line boundary. +- Added SF builder, SF merger, SF cut, and DAG builder boundary tests, + including invalid-input rejection, inspectable invalid output, unchanged + hard links, and newly written changed/clipped nodes. +- State: complete. + +### Phase 5 — Cleanup and documentation + +- Migrated repository callers from the temporary octree topology, traversal, + storage, and cache aliases to the final `store`, `mesh::storage`, and + `dag::storage` public names. +- Removed the temporary `octree::IndexMap`, status, traversal, mesh-storage, + and cache forwarding headers and their wrapper-only unit tests. +- Changed the unchanged legacy `terrain.index` DTO to serialize the shared + `store::Index` directly. +- Kept only 3D identity/geometry, traits, mappings, legacy index-format, and + opening adapters under `octree`. +- Updated the architecture, storage-format, status-quo, historical + before-refactor report, and raster-store index documentation. The docs now + distinguish the implemented shared boundary from future RF formats and + tools and include final public names plus 3D-open and in-memory 2D examples. +- State: complete. + +### Phase 6 — Remove migration-only compatibility fixtures + +- Removed the golden pre-refactor SF and DAG indexes and payload files. +- Removed fixture-only SF/DAG opening and byte-comparison assertions. +- Converted DAG codec reentrancy, metadata-prefix reading, synchronized + storage, SF-builder finalization, and DAG validation tests to generate + current data in temporary directories. +- Retained layout/path, resolver, topology, storage, error, merge/cut, and + producer/consumer boundary coverage. +- Repository search and filesystem inspection find no remaining pre-refactor + fixture reference or fixture file. +- State: complete. + +## Verification log + +- Baseline `unittests_terrainlib`: 23,676 assertions in 391 test cases passed. +- Baseline `unittests_dagbuilder`: 400 assertions in 69 test cases passed. +- Focused Phase 0 terrainlib compatibility tests: 51 assertions in 6 test + cases passed before warning cleanup. +- Focused Phase 0 DAG compatibility test: 23 assertions passed. +- Phase 0 `unittests_terrainlib`: 23,732 assertions in 397 test cases passed. +- Phase 0 `unittests_dagbuilder`: 423 assertions in 70 test cases passed. +- Phase 0 `git diff --check`: passed. +- Phase 1 focused shared-store tests: 67 assertions in 7 test cases passed. +- Phase 1 pre-refactor terrain compatibility tests: 56 assertions in 6 test + cases passed. +- Phase 1 `unittests_terrainlib`: 23,799 assertions in 404 test cases passed. +- Phase 1 `unittests_dagbuilder`: 423 assertions in 70 test cases passed. +- Phase 1 application builds passed for `sf-builder`, `dag-builder`, and + `dag-convert-debug`. The current Debug configuration has `sf-merger` and + `sf-index-browser` disabled. +- The all-target build remains blocked by a pre-existing, unrelated call to + removed `radix::tile::Scheme` in `unittests/sf_builder/texture.cpp`; the + affected refactor targets and suites build independently. +- Phase 1 `git diff --check`: passed. +- Phase 2 focused shared-store tests: 160 assertions in 17 test cases passed. +- Phase 2 DAG golden fixture: 30 assertions passed, including runtime codec + output byte equality. +- Phase 2 concurrent DAG runtime codec: 10 assertions passed. +- Phase 2 `unittests_terrainlib`: 23,892 assertions in 414 test cases passed. +- Phase 2 `unittests_dagbuilder`: 440 assertions in 71 test cases passed. +- Phase 2 application builds passed for `sf-builder`, `dag-builder`, and + `dag-convert-debug`. +- Phase 2 `git diff --check`: passed. +- Phase 3 focused terrain store/open tests: 233 assertions in 25 test cases + passed. +- Phase 3 focused DAG store tests: 17 assertions in 2 test cases passed. +- Phase 3 `unittests_terrainlib`: 23,973 assertions in 422 test cases passed. +- Phase 3 `unittests_dagbuilder`: 459 assertions in 73 test cases passed. +- Phase 3 all-application configuration built `sf-builder`, `sf-merger`, + `sf-index-browser`, `dag-builder`, and `dag-convert-debug` successfully. +- Phase 3 `git diff --check`: passed. +- Phase 4 focused storage-copy tests: 70 assertions in 5 test cases passed. +- Phase 4 SF validator tests: 6 assertions in 2 test cases passed. +- Phase 4 DAG SF-input boundary test: 12 assertions passed. +- Phase 4 SF merger boundary tests: 66 assertions in 4 test cases passed. +- Phase 4 SF builder finalization tests: 19 assertions in 2 test cases passed. +- Phase 4 `unittests_terrainlib`: 24,049 assertions in 429 test cases passed. +- Phase 4 `unittests_dagbuilder`: 471 assertions in 74 test cases passed. +- Phase 4 `unittests_sfmerger`: 121 assertions in 8 test cases passed. +- Phase 4 all-application configuration built `sf-builder`, `sf-merger`, + `sf-index-browser`, `dag-builder`, and `dag-convert-debug` successfully. +- The pre-existing `unittests_sfbuilder` compile failure involving removed + `radix::tile::Scheme` remains isolated; the new finalization tests build and + pass in `unittests_sfbuilder_finalization`. +- Phase 4 `git diff --check`: passed. +- Phase 5 `unittests_terrainlib`: 24,017 assertions in 426 test cases passed. +- Phase 5 `unittests_dagbuilder`: 472 assertions in 74 test cases passed. +- Phase 5 `unittests_sfmerger`: 121 assertions in 8 test cases passed. +- Phase 5 SF builder finalization: 19 assertions in 2 test cases passed. +- Phase 5 application builds passed for `sf-builder`, `sf-merger`, + `sf-index-browser`, `dag-builder`, and `dag-convert-debug`. +- Phase 5 repository searches found no remaining caller of a removed octree + forwarding name and no `octree::Id` dependency under `terrainlib/store`. +- Phase 5 `git diff --check`: passed. +- Phase 6 `unittests_terrainlib`: 23,997 assertions in 425 test cases passed. +- Phase 6 `unittests_dagbuilder`: 439 assertions in 73 test cases passed. +- Phase 6 `unittests_sfmerger`: 121 assertions in 8 test cases passed. +- Phase 6 SF builder finalization: 15 assertions in 2 test cases passed. +- Phase 6 fixture-reference search, fixture-file inspection, and + `git diff --check`: passed. + +## Commit and tag log + +- `refactor_before` — annotated baseline tag at `d1309f5`. +- `75277f8` — Phase 0 compatibility tests, indexes, DAG payloads, and status + log. +- Phase 0 fixture-payload follow-up — force-adds the intentionally committed + `.terrain` golden payloads ignored by the repository's general artifact + rule. +- `refactor_phase_0` — annotated Phase 0 completion tag. +- `4c171c8` / `refactor_phase_1` — shared topology extraction and Phase 1 + completion tag. +- `fbc02d0` / `refactor_phase_2` — runtime layouts/codecs and Phase 2 + completion tag. +- `61acb74` — Phase 3 shared runtime-codec storage core. +- `4ffcdea` / `refactor_phase_3` — Phase 3 application migration and + completion tag. +- `3086336` / `refactor_phase_4` — hardened node reuse, SF topology + enforcement, and Phase 4 completion tag. +- `bca0b5c` / `refactor_phase_5` — removed migration shims, documented final + public boundaries, and tagged Phase 5 completion. +- Phase 6 completion commit / `refactor_phase_6` — removed migration-only + fixtures while retaining current-data coverage. diff --git a/docs/raster-store/rf_builder.md b/docs/raster-store/rf_builder.md new file mode 100644 index 00000000..7006b981 --- /dev/null +++ b/docs/raster-store/rf_builder.md @@ -0,0 +1,202 @@ +# DRAFT — `rf_builder` + +Status: **draft archive of removed ideas**. + +This is not a current implementation plan, accepted format specification, or +statement that the choices below are correct. It is a lightly reformatted +archive of RF-builder and persistent-2D material removed from +[refactor-plan.md](refactor-plan.md). The details are retained so they are not +lost; they require a separate design pass after the shared-store refactor. + +## Proposed names and source boundary + +The removed proposal placed the 2D implementation in +`src/terrainlib/raster_store` and used the `raster_store` namespace: + +```text +src/terrainlib/raster_store/ +├── StoreTraits.h +├── IndexFile.h +├── Storage.h +├── codec/ +│ ├── Amort.h +│ └── Debug.h +└── store_layout/ + └── ZoomXYGoogle.h +``` + +The proposed boundary was: + +- `store` contains dimension- and payload-neutral mechanisms; +- `raster_store` contains the 2D format, key adapters, and raster codecs; and +- subdirectory names match their namespaces where a subnamespace is used. + +The proposed concrete index type was: + +```cpp +store::Index +``` + +## `raster_store::StoreTraits` + +The removed proposal adapted `radix::tile::Id` through +`raster_store::StoreTraits` and specified that it: + +- treats zoom zero as the only root; +- never calls `radix::tile::Id::parent()` at zoom zero, where it underflows; +- rejects coordinates outside `[0, 2^zoom)`; +- accepts zoom levels 0 through + `std::numeric_limits::digits`, inclusive; +- treats that maximum zoom as terminal because a child cannot be represented + by the `uint32_t` x/y coordinates; +- validates the maximum zoom without evaluating an overflowing + `uint32_t{1} << 32`; +- uses `radix::tile::Id::Hasher`; and +- uses the Google/XYZ convention, with the origin at the north-west, at the + persistent boundary. + +The shared code was expected to obtain roots, parents, children, validation, +and hashing through the traits without specializing on the key type. + +## Node paths and layout + +The removed example mapped a raster node to an extensionless `store::NodePath`: + +```text +raster-store ZXY 12/2200/1400 +``` + +The proposed lookup functions were: + +```cpp +raster_store::store_layout::zoom_x_y_google() +raster_store::store_layout::from_id(id) +``` + +The stable layout ID was `zoom/x/y_google`. The default mapping produced +`//` directly below the snapshot root; there was no fixed +`chunks/` directory. A codec, rather than the layout, added `.amort` or debug +file endings. + +## Payload and codec sketches + +The removed proposal used the shared runtime codec interface and placed +raster codecs in `raster_store::codec`: + +```cpp +template +struct raster_store::codec::Amort + : store::Codec> { .. }; + +template +struct raster_store::codec::Debug + : store::Codec> { .. }; +``` + +`Amort` was proposed as readable and writable. `Debug` was proposed as +write-only, with runtime options for data format, attribution format, JPEG +quality, or similar debugging choices. It was not to be template-composed +from separate image codec types. + +The multi-file debug example was: + +```text +Debug raster codec configured for JPEG data and PNG attribution + 12/2200/1400 + -> 12/2200/1400.data.jpg + -> 12/2200/1400.attribution.png +``` + +The proposed final tile path was `//.amort`. The `.amort` payload +and source-attribution-table serialization were explicitly not defined beyond +the separate storage-format notes. + +## Index and format-adapter sketch + +The removed proposal used a separately versioned `raster_store::v1` DTO stored +as `raster_store.index`. + +The proposed version-1 contents were: + +- a layout ID; +- sparse key/status entries; +- serialized `Leaf`, `Inner`, and `Virtual` values; +- `Missing` represented by absence; +- no derived aggregate metadata until a concrete query requires it; +- fixed-width `uint32_t` zoom/x/y fields instead of platform `unsigned`; +- entries ordered lexicographically by `(zoom, x, y)`; and +- duplicate keys rejected while reading. + +The proposed serialization envelope contained: + +- a fixed, file-type-specific 64-bit magic value generated during + implementation; +- a 32-bit version; +- zlib CRC-32 stored as `uint32_t` and computed over the compressed payload; +- a compression enum; and +- a zstd-compressed payload using zstd's best-compression setting. + +The proposal imported zstd through the project's CMake dependency facility and +used the existing `ZLIB::ZLIB` dependency for CRC-32. + +Index serialization was not a responsibility of `store::Index`. The proposed +2D format adapter supplied the index filename, index conversion, mapping +lookup, and default mapping. + +## Snapshot publication sketch + +The removed proposal required explicit finalization/publication for the 2D +snapshot API; a destructor was not to make an incomplete snapshot +authoritative. + +The publication details are now retained in +[architecture.md](architecture.md#publication). The removed proposal used a +sibling `.part` directory, wrote the index last, validated and +closed all files, and atomically renamed it to `` on the same +filesystem. The destination could not already exist. + +The removed text explicitly did not promise durability or safe recovery after +a power failure, operating-system crash, or storage failure, and did not add +`fsync()`, `fdatasync()`, `FlushFileBuffers()`, or equivalent synchronization. + +## Removed implementation and verification ideas + +The removed 2D-adapter phase contained these items: + +- add the checked persistent-key conversion around `radix::tile::Id`; +- add the `//` mapping with stable ID `zoom/x/y_google`; +- define the versioned `raster_store.index` DTO; +- implement the magic/version/checksum/compression envelope; +- add the 2D format adapter and storage aliases under `raster_store`; +- add `raster_store::codec::Amort` when final `.amort` + serialization is available; +- add the output-only `raster_store::codec::Debug`; +- use a test codec instead of making `.amort` claims if final serialization is + unavailable; and +- keep raster-specific processing outside the shared store. + +The removed verification list contained: + +- invalid and boundary tile IDs, including maximum zoom and rejected children; +- index serialization and validation; +- `Leaf`/`Inner` coexistence; +- sparse traversal and ancestor lookup; +- path round trips; +- snapshot hard-link reuse; +- AMORT-to-debug output conversion; +- explicit cross-filesystem/preflight failure; +- publication from `.part` to ``; and +- publication rejection when the destination already exists. + +## Removed risks and controls + +| Removed risk | Removed control | +|---|---| +| `radix::tile::Id` root underflows | Traits intercept root parent lookup | +| Invalid 2D coordinates become persistent | Validate on every disk/API boundary | +| Linked snapshots are modified in place | Immutable snapshot API and overwrite-disabled output | +| Hard-link failure appears late | 2D operation preflight and explicit errors | +| Generic index dictates both disk formats | Separate 3D and 2D format adapters | + +All material in this document remains provisional despite the concrete names +preserved above. diff --git a/docs/raster-store/rf_merger.md b/docs/raster-store/rf_merger.md new file mode 100644 index 00000000..3fc97d0a --- /dev/null +++ b/docs/raster-store/rf_merger.md @@ -0,0 +1,231 @@ +# DRAFT — `rf_merger` + +Status: **draft archive of removed ideas**. + +This is not a current implementation plan or accepted RF merge specification. +It is a lightly reformatted archive of subtree-copy and paired-walker material +removed from [refactor-plan.md](refactor-plan.md). The details are retained so +they are not lost; later discussion already established that RF semantics must +be designed before choosing these abstractions. + +## Proposed names and source boundary + +The removed proposal added these dimension-neutral files: + +```text +src/terrainlib/store/ +├── copy_subtree.h +└── merge/ + ├── Action.h + └── walk.h +``` + +The proposed migration map was: + +| Existing code | Removed target | +|---|---| +| `sf_merger::NodeWriter` subtree loop | `store/copy_subtree.h` | +| `sf_merger::Merger` recursion | `store/merge/walk.h` | + +The proposal intended shared mechanisms to contain no mesh or raster merge +policy. + +## One-node copy proposal + +The proposed one-node API added: + +```cpp +struct CopyOptions { + bool force_reencode = false; +}; +``` + +For one key, the proposed `Storage::copy_from()` behaviour was: + +1. Call input and output `Codec::paths()` with the same fixed dummy + `NodePath`, proposed as `__codec_probe__/node`. +2. Compare path lists exactly, including count, order, and filename endings. +3. When the lists match and `force_reencode` is false, call both codecs with + the actual source and target `NodePath` and hard-link every corresponding + file. +4. When lists differ or re-encoding is forced, read with the input codec and + write with the output codec. +5. Update the target index only after all links or the write complete. + +If a multi-file link failed partway through, the proposal removed target links +created by that call before returning `CopyError`. `CopyError` retained any +underlying `CodecError`. There was no silent file-copy fallback. + +Codec settings that did not change `paths()`, such as compression level or +JPEG quality, did not force re-encoding by default. Callers could pass +`force_reencode = true`. + +The removed hard-link rules were: + +- never modify an existing linked payload in place; +- matching codec path lists hard-link every file; +- different path lists decode with the input codec and encode with the output + codec; +- `force_reencode` selects decode/encode; +- hard-link failure is explicit; +- 2D snapshot tools preflight hard-link support before a long operation; and +- no silent file-copy fallback. + +## Shared subtree-copy proposal + +The removed API sketch was: + +```cpp +std::expected +store::copy_subtree( + const IndexedStorage& source, + Storage& target, + const Key& root, + CopyOptions options = {}); +``` + +The proposed operation: + +1. traversed an indexed source subtree; +2. skipped `Virtual` nodes; +3. called `copy_from()` for physical payloads in `Leaf` and `Inner` states; +4. continued traversal below `Inner`; and +5. returned copy failures rather than asserting or terminating. + +The operation was intended to be payload-neutral and know nothing about +meshes, rasters, masks, attribution, or their encodings. + +The removed target call chain was: + +```text +merge policy decides to keep a source subtree unchanged + -> store::copy_subtree() + -> store::Storage::copy_from() + -> hard-link every codec path, or decode/encode +``` + +## Removed `Inner` copying rationale + +The removed proposal used this topology model: + +| Status | Physical payload | Indexed descendants | +|---|---:|---:| +| `Leaf` | yes | no | +| `Inner` | yes | yes | +| `Virtual` | no | yes | + +It used this RF example: + +```text +zoom 10 physical tile -> Inner +└── zoom 11 physical tile -> Leaf +``` + +The proposal said subtree reuse must preserve both payloads: copy the parent +payload, continue below it, and allow insertion of the descendant to promote +the copied parent from `Leaf` to `Inner` through normal index transitions. + +The proposed status handling was: + +```cpp +switch (status) { +case NodeStatus::Virtual: + break; +case NodeStatus::Leaf: +case NodeStatus::Inner: + target.copy_from(id, source, options); + break; +} +``` + +This described unchanged-subtree copying only. It did not define how two RF +trees should be merged. + +## Paired-tree walker proposal + +The removed proposal extracted dimension-neutral recursion from +`sf_merger::Merger` and introduced these policy results: + +```cpp +store::merge::Recurse +store::merge::Ignore +store::merge::KeepLeft +store::merge::KeepRight +store::merge::Write +``` + +The walker owned recursion and unchanged-subtree reuse. The policy owned +selection and payload combination. + +The removed proposal required all 16 combinations of `Missing`, `Leaf`, +`Inner`, and `Virtual` to be handled or rejected with a typed error rather than +falling into `UNREACHABLE()`. + +It kept these concerns outside the shared walker: + +- `NodeLoader` ancestor mesh reconstruction; +- ECEF node bounds; +- mesh masks and clipping; +- mesh combination and texture atlas generation; and +- mesh validation and auxiliary texture writes. + +The proposal suggested that a future 2D merger could supply a raster policy. +Later discussion identified that the mutually exclusive action list cannot +express both an action for an `Inner` payload and recursion into descendants. +Ideas mentioned after that were a combined `WriteAndRecurse` result or +independent current-node and descendant decisions. None is selected. + +## Removed implementation and verification ideas + +The removed subtree/walker phase contained: + +- compare input and output codec path lists for a fixed dummy `NodePath`; +- hard-link all files when lists match, including partial-failure cleanup; +- decode with the input codec and encode with the output codec when lists + differ; +- add `CopyOptions::force_reencode`; +- add the shared unchanged-subtree copier; +- add the paired hierarchy walker and typed actions; +- cover all 16 status pairs with table-driven tests; +- adapt the 3D merger while keeping mesh policy in `sf_merger`; +- remove generic recursion and copy logic from `sf_merger::Merger` and + `NodeWriter`; and +- add a 3D integration test for an unchanged hard-linked subtree and a newly + written changed boundary node. + +The removed focused tests included: + +- one-file hard linking; +- multi-file hard linking; +- different path counts and endings; +- forced re-encoding with otherwise equal paths; +- conversion between terrain and glTF; +- conversion into a write-only codec; +- runtime failure for an unsupported codec operation; +- copies containing `Leaf`, `Virtual`, and `Inner`; and +- all 16 paired status combinations. + +The proposed generic test file was: + +```text +unittests/terrainlib/store_merge_walk.cpp +``` + +## Removed error and risk notes + +The removed error-propagation proposal passed copy failures through +`copy_subtree()` and the merge call chain to the application boundary with the +affected key and path. Codec, filesystem, unsupported-conversion, +malformed-dataset, and overwrite failures used `std::expected`; operational +failures were not assertions or intentional exceptions. + +| Removed risk | Removed control | +|---|---| +| `Inner` payloads are lost during subtree reuse | Copy every physical status and test mixed-depth fixtures | +| Multi-file hard linking fails partway through | Remove links created by the failed `copy_from()` before returning | +| Incompatible codecs return the same path list | Treat path-list equality as a codec contract and test every concrete codec pairing | +| Output-only codec is selected for required input | Return a clear `UnsupportedOperation` error | +| Shared code accumulates mesh/raster policy | Dependency tests/review against the source boundary | + +All material in this document remains provisional despite the concrete names +preserved above. diff --git a/docs/raster-store/sampling-and-generation.md b/docs/raster-store/sampling-and-generation.md new file mode 100644 index 00000000..83309521 --- /dev/null +++ b/docs/raster-store/sampling-and-generation.md @@ -0,0 +1,249 @@ +# Sampling and pyramid generation + +This document defines generator-facing sampling terminology and invariants. +It deliberately separates output sampling from how an original source raster +was measured or produced. + +## Terminology + +### Vertex pixel + +A vertex pixel is a generated value located on a grid vertex. For a tile with +`N` intervals per side, vertex positions are: + +```text +x(i) = left + i × tile_width / N, i = 0 … N +``` + +The output has `N+1` pixels per side: + +```text +tile boundary tile boundary +●---------●---------●---------●---------● +``` + +Height maps used to form mesh vertices require this placement. Adjacent +rendering tiles contain overlapping copies of their shared edge and corner +vertex pixels. + +### Area pixel + +An area pixel is a generated value associated with one raster cell. For a tile +with `N` cells per side, effective cell-centre positions are: + +```text +x(i) = left + (i + 1/2) × tile_width / N, i = 0 … N-1 +``` + +The output has `N` pixels per side: + +```text +tile boundary tile boundary +│ × × × × │ +``` + +The term describes placement and support in the generated grid. It does not +assert that the input value was a physical area integral. + +## Source semantics versus output placement + +An input height raster may have been produced from LiDAR points through +gridding, interpolation, fitting, or averaging. An orthophoto may already have +passed through sensor integration, reconstruction, reprojection, and +resampling. Those histories do not decide where a delivery format requires +its output values. + +Generation is modelled as: + +```text +stored discrete raster + ↓ reconstruct its implied field +continuous or evaluable field + ↓ low-pass for target resolution +filtered field + ↓ evaluate on requested output grid +vertex pixels or area pixels +``` + +The source interpretation and reconstruction rule are layer/generator policy. +Vertex-pixel and area-pixel placement are output requirements. + +## Reduction by two + +### Vertex pixels + +At fine spacing `Δ`, fine vertex positions are `nΔ`. Coarse positions are +`2mΔ`, coinciding with every second fine location: + +```text +fine: ●---●---●---●---● +coarse: ●-------●-------● +``` + +Copying every second value would be unfiltered decimation and is unacceptable +because frequencies above the new Nyquist limit would alias. The generator +must low-pass first, using a kernel centred on each retained vertex position: + +```text +coarse[m] = Σ h[k] × fine[2m - k] +``` + +The spatial centre stays in place; the value generally changes because it is +sampled from the filtered signal. + +### Area pixels + +Fine area-pixel centres are `(n+1/2)Δ`. A coarse cell spans two fine cells and +has its centre at `(2m+1)Δ`, halfway between two fine centres: + +```text +fine cells: |---- × ----|---- × ----| +coarse cell: |---------- × ----------| +``` + +The simplest reduction is the average of each 2x2 fine block. If fine values +are exact equal-area averages, that produces the exact average over the union +of the four cells. A box filter is not an ideal anti-aliasing filter, however, +and may be insufficient for visual imagery or other signals. + +A higher-quality area-pixel reduction applies a low-pass filter with the +correct half-sample phase, centred on the coarse cell centre. Its support may +extend beyond the four cells geometrically covered by the coarse cell. + +## No duplicated height borders in the store + +The authoritative store does not persist overlapping rendering borders. +Pyramid generation constructs a vertex-pixel output only after reconstruction +and filtering. + +The implementation may obtain a requested `(N+1) × (N+1)` output window by +reading non-overlapping store chunks plus the filter halo required on every +side. The generated shared vertices must be computed from the same global +coordinates and source data for both neighbouring output tiles. + +The generator must not independently clamp its filter at each tile edge. +Clamping would make an internal tile boundary behave like a data boundary and +could produce seams. + +Two implementation strategies can satisfy the invariant: + +1. Evaluate shared global vertex coordinates deterministically from a common + window reader; or +2. Generate a metatile, filter it once, and split it into overlapping output + tiles. + +The first gives execution-order independence. The second may reduce repeated +I/O. They can coexist if tests establish identical results. + +## Filter halos and chunk boundaries + +Any nontrivial low-pass filter needs samples outside the exact output bounds. +The required halo is determined by the reconstruction and reduction filters, +not by a fixed one-pixel border flag. + +The store reader should expose a logical raster window over the quadtree. It +resolves: + +- physical chunks selected for the requested accuracy; +- ancestor fallback where finer data is absent; +- chunk and source-map decoding; and +- neighbouring data needed by the window. + +The generator determines the requested halo and applies boundary conditions +only at true dataset/world boundaries or NoData boundaries. + +## Mixed sources + +The raster store contains exactly one payload and one source ID for each +stored pixel. A generator filter may span pixels attributed to several +sources: + +```text +store pixels: A A A B B +filter support: [-------] +output value: blend of A and B payloads +``` + +This is allowed. A generated output pixel does not retain a source ID. The +generated tile records tile-level provenance, at minimum the set of source IDs +whose payload values contributed nonzero filter weight to any output pixel. + +Source IDs are categorical and are never averaged. Payload filtering and +provenance collection are parallel operations: + +```text +numeric payload samples → weighted filtered value +source IDs → contributing-source set +``` + +The exact handling of invalid/NoData samples requires a policy. A common +continuous-raster rule is to normalize by the total weight of valid samples, +but that must not be applied automatically to categorical data. + +## Layer-specific filtering + +Sampling placement alone does not determine a correct filter: + +| Semantic kind | Relevant considerations | +|---|---| +| Height | low-pass before decimation; terrain error and peak loss | +| Orthophoto | linear-light filtering; alpha premultiplication | +| Categorical | mode, coverage, or another categorical policy | +| Probability/coverage | conservative area averaging may be appropriate | +| Vector/normal | component filtering followed by normalization where needed | +| Mask/NoData | validity-aware weights and explicit coverage rules | + +The current `radix::raster::generate_mipmap` performs a component-wise 2x2 +box average. It may be a reference for simple area-pixel aggregation, but it +does not implement these policies or vertex-pixel filtering. + +## Coherent coarse-source selection + +The generator need not always filter the deepest available descendants. If a +physical chunk at the requested scale is sufficiently accurate, using that +single coherent source may be preferable to composing several finer sources. + +The selection process is conceptually: + +```text +choose physical representation(s) for the requested output and quality policy + ↓ +read a continuous window with fallback and required halo + ↓ +filter for the target resolution + ↓ +evaluate vertex pixels or area pixels +``` + +Source-selection/refinement policy precedes filtering. Filtering does not +change the authoritative hierarchy. + +## World and dataset boundaries + +The generator needs explicit rules for: + +- horizontal wrapping at the Web Mercator antimeridian; +- north/south limits of the Web Mercator world; +- areas with no physical ancestor or descendant; +- NoData holes inside otherwise covered chunks; and +- filters whose support crosses a layer's coverage boundary. + +These rules are not yet decided. Tests must distinguish true boundaries from +ordinary internal chunk and delivery-tile boundaries. + +## Required golden tests + +Before production filtering is implemented, synthetic fixtures should prove: + +1. A constant raster remains constant across chunks and pyramid levels. +2. An impulse or frequency sweep demonstrates the chosen anti-alias response. +3. Two adjacent area-pixel tiles match a single equivalent metatile result. +4. Two adjacent vertex-pixel tiles produce bit-identical shared edges. +5. Filtering is unchanged when a store window is split into different chunks. +6. A source boundary blends payloads but reports both tile-level sources. +7. A NoData boundary follows the configured validity rule. +8. A coherent physical parent can be chosen instead of finer descendants. +9. TMS and Slippy input IDs normalize to the same canonical spatial tile. + +The filter coefficients and acceptable numeric tolerances remain open design +decisions. The tests should lock them only after representative evaluation. diff --git a/docs/raster-store/status-quo.md b/docs/raster-store/status-quo.md new file mode 100644 index 00000000..461faf46 --- /dev/null +++ b/docs/raster-store/status-quo.md @@ -0,0 +1,287 @@ +# Status quo and reuse assessment + +> **Historical assessment:** this document records the repository before the +> shared-store refactor and is intentionally not rewritten as current code +> documentation. The completed implementation and verification record is in +> [refactor-status.md](refactor-status.md); the resulting public boundary is +> summarized in [architecture.md](architecture.md#implementation-status). + +The refactor chose the generalization option discussed below. Sparse topology, +traversal, layouts, runtime codecs, storage, index lifecycle, and cache +interfaces now live under `store` and are parameterized by hierarchy traits. +The legacy 3D format remains an adapter under `octree`, while +`raster_store::StoreTraits` proves the shared in-memory mechanisms with +`radix::tile::Id`. Persistent RF formats and RF tools remain future work. + +This document evaluates the current repository after commit `9cf9065` +(`Consolidate raster handling and use std::expected`). It distinguishes +reusable mechanisms from interfaces that encode assumptions unsuitable for +the raster store. + +## Summary + +The project already contains most low-level ingredients: + +- `radix::Raster` for contiguous typed raster memory; +- `radix::tile::Id` for quadtree addressing; +- Web Mercator grid calculations and GDAL reprojection in `tile_builder`; +- sparse topology, indexed traversal, codecs, layouts, and hard-link reuse in + the octree storage code; and +- the SF merger's snapshot-like reuse of unchanged subtrees. + +There is no existing component that should become the raster store unchanged. +The best path is to compose the Radix raster and tile primitives with a new +2D storage layer, while extracting or adapting selected octree-storage ideas. + +## Reuse matrix + +| Component | Assessment | Intended use | +|---|---|---| +| `radix::Raster` | Reuse directly | In-memory payloads and source maps | +| `radix::RasterMask` | Reuse directly | Temporary validity/selection masks | +| `radix::raster::transform` | Reuse directly | Typed pixel transformations | +| `radix::raster::generate_mipmap` | Do not use as general generator | Only a 2x2 component-wise box average | +| `radix::tile::Id` | Reuse after hardening or through an adapter | Persistent quadtree keys | +| `radix::quad_tree::Node` | Do not reuse for disk index | Dense in-memory ownership model | +| CTB `GlobalMercator` and grid bounds | Reuse initially | Tile bounds and Web Mercator resolution | +| `Dataset` and GDAL setup | Reuse/adapt | Input discovery and reprojection | +| `DatasetReader` | Adapt substantially | Windowed, typed, multi-band ingestion | +| `Tiler` / `ParallelTiler` | Reuse calculations, replace orchestration | Candidate chunk enumeration | +| `ParallelTileGenerator` | Do not use as store builder | Small-file writer with incompatible lifecycle | +| Octree `IndexMap` algorithm | Reuse design; generalize or port | Sparse physical/virtual topology | +| Octree `Storage_` / `RawStorage_` | Reuse design and selected code | Codec boundary, indexing, hard-link copy | +| Octree disk layouts | Do not use as-is | They encode `octree::Id` and 3D paths | +| SF merger visitors and geometry code | Do not reuse | Mesh- and ECEF-specific semantics | +| SF merger unchanged-subtree copy | Reuse design | Snapshot construction and hard links | +| `zpp_bits` serialization helpers | Reuse cautiously | Versioned metadata/index serialization | + +## Radix raster + +### What can be reused + +`extern/radix/src/radix/raster.h` now provides a shared, value-typed raster: + +- rectangular `glm::uvec2` dimensions; +- contiguous row-major `std::vector` storage; +- element and byte spans; +- typed pixel access; +- move construction from an existing vector; +- a contiguous byte-valued `RasterMask`; +- dimension-checked concatenation; and +- masked and unmasked transforms. + +These properties fit both store arrays: + +```cpp +radix::Raster payload; +radix::Raster source_map; +``` + +The class correctly remains independent of Web Mercator, tile IDs, source +metadata, compression, and file I/O. Those belong to higher layers. + +### What needs adaptation around it + +Large chunks and filtered generation will benefit from operations not +currently supplied by `Raster`: + +- non-owning raster views and subwindows; +- explicit row stride where external codecs require it; +- halo/window assembly across neighbouring chunks; +- checked construction that reports allocation/dimension errors without + relying on assertions; and +- streaming or block processing when a full set of input chunks would exceed + the memory budget. + +These should be introduced only when required. They are not reasons to embed +store concepts into `Raster`. + +### What cannot be reused for final filtering + +`radix::raster::generate_mipmap` requires a square, power-of-two raster and +reduces each 2x2 group by component-wise averaging. It does not provide: + +- a selectable reconstruction or low-pass filter; +- the phase difference between vertex pixels and area pixels; +- halo samples across tile boundaries; +- linear-light colour and premultiplied-alpha handling; +- NoData-aware normalization; +- categorical reduction; or +- source-contribution tracking. + +It is therefore a useful simple raster utility, not the raster-store pyramid +generator. + +## Radix tile addressing + +### What can be reused + +`radix::tile::Id` already contains the required 2D identity: + +- zoom level; +- `x/y` coordinates; +- parent and four-child relationships; +- TMS/Slippy conversion; and +- hashing and ordering support. + +### Required hardening + +Persistent storage needs stronger invariants than the current convenience +type supplies: + +- Calling `parent()` at zoom zero currently underflows. +- Construction does not reject coordinates outside `[0, 2^z)`. +- Shifting `1u << zoom_level` limits valid conversion at high zooms. +- The scheme participates in identity, so the same spatial tile in TMS and + Slippy form becomes two keys. +- There is no persistent serialization contract or format version. + +The store should choose one canonical scheme and normalize all IDs at its API +boundary. Whether the hardening belongs in Radix or in a checked store adapter +is open. + +`radix::quad_tree::Node` is not a replacement for the index. It owns a +fully allocated group of four children whenever refined, represents no +missing child within such a group, and has no persistence or physical/virtual +status. The store requires a sparse map keyed by tile ID. + +## Tile builder and GDAL path + +### Reusable foundations + +The current tile builder already demonstrates: + +- opening GDAL raster datasets; +- reading dataset bounds; +- selecting Web Mercator or geodetic CTB grids; +- transforming arbitrary source SRS data during reads; +- calculating tile bounds and resolutions; +- enumerating intersecting `radix::tile::Id` values; and +- parallel per-tile processing. + +`ctb::GlobalMercator`, `Tiler::tile_for`, and `ParallelTiler` are useful +references and may be reused initially for grid math. + +### Required changes + +`DatasetReader` currently reads one band into a float raster, uses cubic GDAL +warping, and constructs a warped VRT for a requested output rectangle. The +store needs typed and multi-band reads, explicit alpha/NoData handling, +controlled resampling, source metadata, and deterministic alignment with the +store chunk grid. + +`Tiler` models delivery-oriented dimensions through `Border::Yes/No` and a +south/east extra pixel. Store chunks have no rendering overlap, and generated +vertex pixels require an explicit global sampling/filtering model. The border +boolean should not define store geometry. + +`ParallelTileGenerator` writes individual image files for explicitly +enumerated tiles. It is not suitable as the snapshot builder because it lacks: + +- source-map generation; +- old-snapshot reuse; +- atomic chunk containers; +- index transactions and publication; +- resumability validation; and +- bounded enumeration/streaming for very large tile sets. + +Its parallel work pattern and progress reporting may still inform the new +builder. + +## Octree index and storage + +### Reusable design + +The octree index captures the required topology semantics: + +- `Leaf`: physical payload without indexed descendants; +- `Inner`: physical payload with indexed descendants; +- `Virtual`: no physical payload, but indexed descendants; and +- absence from the map: missing node and subtree. + +Adding a physical node creates virtual ancestors and promotes a physical +ancestor from leaf to inner. Removing nodes collapses unused virtual chains. +Traversal follows only present index entries. + +The storage stack also has useful separation between: + +- logical indexed storage; +- raw path-based storage; +- disk layout; +- payload codec; and +- optional cache. + +`RawStorage_::copy_from` demonstrates hard-link reuse when source and target +extensions match. The SF merger demonstrates copying unchanged physical +subtrees and writing a new output index. + +### Why it cannot be reused unchanged + +The entire stack uses concrete `octree::Id` types. `IndexMap`, cache APIs, +layouts, filesystem path parsing, traversal, storage, serialization, and +formatting all embed this type. The coordinate-directory layout is +`level/x/y/z`, and the default codec is a mesh codec. + +The existing index file also records no tree kind. Reinterpreting its +serialized `(level, index)` octree IDs as web tiles would be unsafe. + +A 2D implementation can either: + +1. generalize the hierarchy/storage stack over an ID and layout policy; or +2. create a raster-store-specific 2D implementation using the same algorithms. + +Generalization avoids duplicate infrastructure but has a larger blast radius +in mature octree code. A separate implementation is initially safer but risks +long-term duplication. This requires an explicit decision before coding. + +### Behaviours that should not be copied blindly + +- Hard-link failure currently has no copy fallback. +- Existing indexes are trusted rather than reconciled against filesystem + contents when opened. +- An unindexed output discovers files through a final recursive directory + scan. +- Some error paths use assertions or process termination. +- The binary index lacks an explicit magic/version/topology header suitable + for a new durable format. + +The raster store should retain the useful topology and codec boundaries while +specifying stronger snapshot, validation, and error-handling rules. + +## SF builder and merger + +The SF builder's ECEF octree placement, mesh construction, texture atlases, +mask clipping, and mesh visitors are not reusable for a Web Mercator raster +store. + +The reusable ideas are architectural: + +- transform source data into a canonical spatial system during build; +- preserve physical ancestors as fallback representations; +- use an index to avoid per-node filesystem probing; +- stop refinement when a coherent representation is sufficient; and +- hard-link unchanged chunks into a new output dataset. + +Those ideas should be reimplemented against Radix tile IDs and raster payloads +rather than adapted through mesh abstractions. + +## Recommended component boundary + +The current components suggest the following dependency direction: + +```text +Radix + Raster, RasterMask, tile::Id, geometry + ↓ +Terrain library + GDAL dataset access, Web Mercator grid math, filtering primitives + ↓ +Raster store + source catalog, chunk container, sparse index, snapshots + ↓ +Builders and generators + ingestion, source selection, texture pyramids, height/geometry pyramids +``` + +The raster store should consume `radix::Raster`; Radix should not depend on +the store's source catalog, file format, or snapshot lifecycle. diff --git a/docs/raster-store/storage-format.md b/docs/raster-store/storage-format.md new file mode 100644 index 00000000..18497b2d --- /dev/null +++ b/docs/raster-store/storage-format.md @@ -0,0 +1,111 @@ +# Storage format + +> **Status:** future raster-store design. The completed 2D/3D store refactor +> did not implement or finalize the RF/TB formats described below. These +> requirements were not acceptance criteria for that refactor. See +> [refactor-status.md](refactor-status.md) for the implemented shared boundary. + +The 3D octree stores now use the versioned envelope formats described in +[envelope-migration-plan.md](envelope-migration-plan.md). The shared store +still supplies only an in-memory `raster_store::StoreTraits` adapter for +`radix::tile::Id`; there is currently no persistent RF index, `.amort` codec, +snapshot publisher, or RF opening API. Names and formats below remain +proposals until their own implementation plans approve them. + +This document specifies the logical format and required invariants for the following data stores: +- raster-fundamentalis (our authoritative raster store) +- tile-base (the tile-pyramid for our tile-server) +- the source-attribution table (for correct copyright attribution and data selection) + +## Source-attribution table +The source attribution table is used in raster-fundamentalis, tile-base, and in abbreviated format in the delivered tiles. +- stores a vector of structs (called Table), each struct describing one source +- the struct (called Entity) contains: + - the spatial resolution in pixel-width at the equator (EPSG:3857) + - the date of data acquisition + - the date of ingestion + - a copyright string + - a copyright link string + - a license string +- the index 0 is reserved for "no-data" +- the index must be checked to be smaller than 2^16-1, and we throw unsupported if it becomes larger (this is because of the storage format of tiles). +- the source attribution table is implemented in src/terrainlib/source_attribution.h, in the namespace source_attribution::* +- We need a `struct` declaration and a `using Table = .. ` (both versioned) +- the source-attributino table is stored in a file named source_attribution_table.ard (.alpine raster data) +- there is one per directory tree (it's valid for all tiles stored within the same directory tree, all siblings and children). +- given a tile (either rf or tb), the lookup of the source attribution table is first in the same directory and then in all parrent dirs, until a source_attribution_table.ard is found (or a failure is thrown). + +## alpine maps raster store format +this is the binary format used to store rf and tb tiles. both share the same basic tile format, but use it in different ways. +- The hierarchy is a Web Mercator (EPSG:3857) quadtree keyed by tile IDs (radix::tile::ID, https://docs.maptiler.com/google-maps-coordinates-tile-bounds-projection/). +- stored tiles have a resolution of 4096x4096 pixels +- Every stored pixel has one data value (can be vector type) and one source attribution index. +- the source attribution index is stored as uint16, and indexes into a global source attribution table (see above) +- data is stored in radix::Raster objects (one for source attribution index, one for the actual data), i.e. template struct raster_store::Tile { radix::Raster data; radix::Raster source_attribution; }; +- the file ending is .amort (AlpineMapsOrg raster tile), it is serialised used the principles outlined below. +- each snapshot stores its index as `raster_store.index`. +- the default `zoom/x/y_google` layout stores a tile as + `//.amort`, directly below the snapshot root. + +## raster-fundamentalis (rf) format +raster-fundamentalis is our authoritative raster-store, containing only the data and no overviews / downsampled version. +- unlike a tile pyramid, not every level is occupied (there is no downsampled versions of the data). +- Coarse physical tiles (e.g. zoom level 10) may coexist with more accurate descendants (e.g. zoom level 15). +- the raster-fundamentalis builder is implemented in src/rf-builder/*, it consumes raw gdal data. + +## Tile-base Format (tb) +tile-base is a hierarchy build from raster-fundamentalis, containing all data and its overviews / downsampled versions. it is used directly by the tile-server to generate tiles at the requested resolution and format. +- every level is occupied, and every level selects an adequate data source +- the tile-base builder is implemented in src/tb-builder, it consumes rf + +### tile-server +- should generate tiles of requested resolution and pixel type (vertex|area) on the fly +- requests by url, e.g.: layer/vertex|area/resolution/z/x/y.ending +- live in src/tile-server/* +- the delivery tile format is not yet defined + +## serialization / deserialization envelope and versioning +- `zpp::bits` serialises C++ objects to byte streams and deserialises them again. +- The envelope is generic and is not specific to the raster store. +- We serialise objects with `zpp::bits` in two levels. + - The first level is an aggregate with the following fields, in this order: + - `uint64 magic`, always `F5FBD3EF919428CA`, identifying this envelope format; + - `string class_name`, identifying the payload type; + - `uint32 class_version`, identifying the versioned payload type; + - `ChecksumAlgorithm checksum_algorithm`, default `HandledByCompressionLib`, alternatively `Crc32c` or `None`; + - `string checksum`, empty when no external checksum is used, otherwise the CRC-32C value; + - `CompressionAlgorithm compression_algorithm`, default `ZstdDefaultCompressionWithChecksum`, alternatively `ZstdBestCompressionWithChecksum` or `None`; + - `uint64 uncompressed_size`, the exact size of the uncompressed second level; + - `Bytes compressed_data`, containing the second level. + - The second level is a compressed byte vector. It is deserialised directly into the selected versioned payload class. +- The magic is shared by all payload types. `class_name` distinguishes payload types. An incompatible future envelope layout requires a new magic. +- data structs are stored in versioned namespaces, e.g.: + `raster_store::v1::Tile` +- outside the versioned namespace, there is a using declaration for the newest version +- outside the versioned namespace, there is a serialization wrapper function taking only the newest version +- outside the versioned namespace, there is a deserialization function, taking a byte stream, and returning the newest version (convert to the newest version, if the payload encodes an older version) +- Newer versions provide a static `from_previous` function, e.g. `v2::Tile::from_previous(v1::Tile)`. Static conversion functions keep the payload types aggregates, allowing `zpp::bits` to serialise them without per-type serialisation declarations. A conversion trail upgrades v1 to v2 and then v3. +- `Version` pairs version numbers with payload types. `PayloadSchema` defines the class name, supported versions, latest type, and conversion trail. +- The generic serialisation function receives the schema and version as template parameters. Deserialisation reads `class_version`, deserialises exactly that payload type, and follows the conversion trail to the latest type. It does not speculatively try other payload versions. +- we have clear fails if + - the classname or magic is wrong, or the version is unsupported + - if the checksum check fails + - if the compression algorithm is missing or unsupported. + - deserialization fails +- we fail by returning an unexpected in these cases +- Compression uses libzstd at its default compression level. Libzstd is imported through the project's CMake install facility from https://github.com/AlpineMapsOrgDependencies/zstd. Both zstd modes write an embedded frame checksum, which libzstd verifies while decompressing; `ZstdBestCompressionWithChecksum` remains available for explicit callers. +- `compress_with_checksum` accepts a `vector` and returns the compressed bytes plus the external checksum string. `checked_decompress` accepts both and returns the decompressed bytes. Both use `std::expected` and dispatch with a switch. +- `Crc32c` is an external CRC-32C (Castagnoli) checksum of the complete uncompressed serialised payload. It uses the reflected polynomial `82F63B78`, an initial value of `FFFFFFFF`, and a final XOR of `FFFFFFFF`. The checksum string contains exactly eight lowercase hexadecimal characters. The empty payload has checksum `00000000`; the ASCII test vector `123456789` has checksum `e3069283`. CRC-32C detects accidental corruption but is not cryptographic. +- `checked_decompress` accepts a maximum decompressed size, defaulting to 1 GiB. It uses a size reported by the compression format when available, otherwise it uses the maximum as its allocation bound and shrinks the result to the produced size. +- Envelope deserialisation passes `uncompressed_size` as that maximum and requires the produced size to match it exactly. A mismatch is a decompression failure. A declared size above the caller's limit or the hard 1 GiB limit is a size-limit failure. +- `None` compression may be paired with `None` or `Crc32c`. Either zstd mode may be paired with `HandledByCompressionLib` or `Crc32c`; with `Crc32c`, both the embedded zstd checksum and the external CRC-32C are checked. `HandledByCompressionLib` is invalid with `None` compression, and `None` checksum is invalid with a zstd mode. +- `None` and `HandledByCompressionLib` require an empty checksum string. `Crc32c` requires its canonical eight-character checksum. Deserialisation verifies it after bounded decompression and before parsing the payload with `zpp::bits`; envelope deserialisation also requires the resulting size to exactly match `uncompressed_size`. +- Decompression never allocates or produces output larger than 1 GiB. + + +## to be defined +- semantic layer kind (height scalars, linear colour, gamma-encoded colour, categorical values), should be used for filtering +- human readable description? +- enumeration of layers etc? + +For now, these things will be defined in code, we will have one terrain-elevation, one surface-elevation and one ortho-photo store. later probably also a percentage store (for the snow layer) diff --git a/docs/raster-store/terminology.md b/docs/raster-store/terminology.md new file mode 100644 index 00000000..eb98976b --- /dev/null +++ b/docs/raster-store/terminology.md @@ -0,0 +1,32 @@ +# Terminology +** attribution raster ** +: A square data matrix (image), containing indices into the source-attribution table + +** source-attribution table ** +: A table of data sources, including meta data like resolution, dates etc. + +**raster-fundamentalis (rf)** +: The authoritative raster store + +**tile-base (tb)** +: basically rf with overviews, used to generate derived tiles + +**Vertex pixel** +: A generated value located on a grid vertex. Height tiles for mesh generation + require vertex pixels, including shared boundary positions. + +**Area pixel** +: A generated value associated with a raster cell. Ordinary texture outputs + use area pixels whose cell boundaries align with tile boundaries. + +The terms vertex pixel and area pixel describe generator outputs. They do not +assert how an original sensor or source raster produced its values. + + +**tile** +: A chunk of raster data with an tile ID. Rf is a store for tiles, tb is a store for tiles, and we generate derived tiles / output tiles for the client. + +**Derived / output tile** +: A filtered and encoded output tile generated from the tile base store. +It may have different dimensions, sampling placement, encoding, and +provenance granularity from a store chunk. diff --git a/docs/raster-store/todo.md b/docs/raster-store/todo.md new file mode 100644 index 00000000..fe9bf14a --- /dev/null +++ b/docs/raster-store/todo.md @@ -0,0 +1,5 @@ +# Raster store TODO + +- Consider extending `sf::validate_index()` beyond rejecting `Inner` after + additional SF invariants and their required error reporting are defined. + This is not part of the shared-store refactor. diff --git a/docs/tile-downloader/download.sh b/docs/tile-downloader/download.sh index b375ad75..dcd168f8 100755 --- a/docs/tile-downloader/download.sh +++ b/docs/tile-downloader/download.sh @@ -3,7 +3,7 @@ build_path="/home/madam/Documents/work/tuw/alpinemaps/build-terrain-builder-Desktop_Qt_6_2_3_GCC_64bit-Release/src" while read p; do - read zoom row col <<<${p//[^0-9]/ } - echo -e "nice -10 \$build_path/tile-downloader --provider basemap --zoom ${zoom} --row ${row} --col ${col} --verbosity 0&" -# nice -10 $build_path/tile-downloader --provider basemap --zoom ${zoom} --row ${row} --col ${col} --verbosity 0& + read zoom x y <<<${p//[^0-9]/ } + echo -e "nice -10 \$build_path/tile-downloader --provider basemap --zoom ${zoom} --x ${x} --y ${y} --verbosity 0&" +# nice -10 $build_path/tile-downloader --provider basemap --zoom ${zoom} --x ${x} --y ${y} --verbosity 0& done load_and_clusterize_mesh( - const octree::MeshStorage &storage, + const mesh::storage::Storage &storage, const octree::Id &id) { const auto result = storage.load(id); if (!result) { - if (result.error() != mesh::io::LoadMeshErrorKind::FileNotFound) { - LOG_ERROR("Failed to read node {}: {}", id, result.error()); + const auto *codec_error = std::get_if(&result.error()); + if (codec_error == nullptr + || codec_error->category != store::CodecErrorCategory::FileNotFound) { + LOG_ERROR( + "Failed to read node {}: {}", + id, + store::describe_error(result.error())); } return std::nullopt; } @@ -120,7 +125,7 @@ LodResult build_lod( // Load input meshes, clusterize, and filter them to the target region. std::vector load_input_clusters( const std::span input_ids, - const octree::MeshStorage &input_storage, + const mesh::storage::Storage &input_storage, const RegionFilter& filter) { std::vector result; @@ -147,8 +152,8 @@ std::vector load_input_clusters( // Dependencies shared across the whole dag building pipeline. struct BuildContext { - const octree::IndexedMeshStorage &input_storage; - ThreadSafeStorage output_storage; + const mesh::storage::IndexedStorage &input_storage; + ThreadSafeStorage output_storage; const BuildOptions &options; const octree::Space &space; const octree::OddLevelShifted &shifted_space; @@ -448,17 +453,17 @@ LevelWorkplan build_level_workplan( // Pre-filter input nodes to only those that intersect the target root bounds. std::vector> gather_relevant_input_leaves( - const octree::IndexMap &index, + const store::Index &index, const octree::Space &space, const radix::geometry::Aabb3d &root_bounds) { const auto start = space.find_smallest_node_encompassing_bounds(root_bounds) .value_or(octree::Id::root()); std::vector> result(octree::Id::max_level() + 1); - octree::traverse( + const auto traversal = store::traverse( index, - [&](const octree::Id &id, const octree::NodeStatus status) { - if (status == octree::NodeStatus::Leaf && radix::geometry::intersect(root_bounds, space.get_node_bounds(id))) { + [&](const octree::Id &id, const store::NodeStatus status) { + if (status == store::NodeStatus::Leaf && radix::geometry::intersect(root_bounds, space.get_node_bounds(id))) { result[id.level()].push_back(id); } }, @@ -466,6 +471,7 @@ std::vector> gather_relevant_input_leaves( return radix::geometry::intersect(root_bounds, space.get_node_bounds(id)); }, start); + DEBUG_ASSERT(traversal.has_value()); return result; } @@ -500,7 +506,7 @@ std::unordered_set build_level( std::unordered_set already_built; if (ctx.options.continuation_mode != ContinuationMode::Overwrite) { for (const octree::Id &target : targets) { - if (ctx.output_storage.has(target)) { + if (DEBUG_ASSERT_VAL(ctx.output_storage.has(target)).value()) { already_built.insert(target); } } @@ -511,12 +517,19 @@ std::unordered_set build_level( } // Initialize debug storage if requested (contains .glb meshes) - std::optional debug_storage; + std::optional debug_storage; if (ctx.options.write_debug_meshes) { - debug_storage = octree::open_folder( + octree::OpenOptions options; + options.preferred_extension = ".glb"; + auto debug_result = octree::open_folder( ctx.output_storage.base_path().string() + "-debug", - false, - octree::OpenOptions{.preferred_extension_with_dot = ".glb"}); + std::move(options)); + if (!debug_result.has_value()) { + LOG_ERROR_AND_EXIT( + "Failed to open debug mesh storage: {}", + store::describe_error(debug_result.error())); + } + debug_storage = std::move(debug_result.value()); } tbb::concurrent_vector saved_ids; @@ -538,11 +551,20 @@ std::unordered_set build_level( const auto save_result = ctx.output_storage.save(target, *result); if (save_result) { if (debug_storage) { - debug_storage->save(target, clustering_to_mesh(result->clustering)); + const auto debug_save_result = debug_storage->save(target, clustering_to_mesh(result->clustering)); + if (!debug_save_result.has_value()) { + LOG_ERROR_AND_EXIT( + "Failed to save debug mesh for node {}: {}", + target, + store::describe_error(debug_save_result.error())); + } } saved_ids.push_back(target); } else { - LOG_ERROR("Failed to save node {}: {}", target, save_result.error()); + LOG_ERROR( + "Failed to save node {}: {}", + target, + store::describe_error(save_result.error())); } } progress.task_finished(); @@ -561,11 +583,15 @@ std::unordered_set build_level( // Builds the DAG from input_storage into output_storage, restricted to levels within level_range. // Iterates octree levels from finest to coarsest, simplifying and re-clustering geometry at each // level from its children. -void build_levels( - const octree::IndexedMeshStorage &input_storage, - octree::IndexedDagStorage &output_storage, +std::expected build_levels( + const mesh::storage::IndexedStorage &input_storage, + dag::storage::IndexedStorage &output_storage, const BuildOptions &options, const AnyRange &level_range) { + const auto validation = sf::validate_index(input_storage.index()); + if (!validation.has_value()) { + return std::unexpected(validation.error()); + } const octree::OddLevelShifted shifted_space = octree::OddLevelShifted::earth(); const octree::Space space = octree::Space::earth(); const octree::Id root_node = options.root_node; @@ -576,7 +602,7 @@ void build_levels( auto max_input_level_opt = find_max_input_level(input_by_level); if (!max_input_level_opt.has_value()) { LOG_WARN("No input nodes found for root {}", root_node); - return; + return {}; } const uint32_t max_input_level = max_input_level_opt.value(); @@ -584,14 +610,14 @@ void build_levels( const Range range = valid_range.intersect(level_range).to_range(max_input_level + 1); if (range.is_empty()) { LOG_WARN("Requested level range does not overlap with buildable levels {}-{}", root_node.level(), max_input_level); - return; + return {}; } // Seed prev_level_built with any already-built nodes one level finer than the first level. // TODO: use hierachical lookup here based on root_bounds and IdRect std::unordered_set prev_level_built; for (const auto &[id, status] : output_storage.index()) { - if (id.level() == range.end && status != octree::NodeStatus::Virtual) { + if (id.level() == range.end && status != store::NodeStatus::Virtual) { prev_level_built.insert(id); } } @@ -607,19 +633,23 @@ void build_levels( // Persist per level so a finished level can be read back before the whole run completes if (const auto result = ctx.output_storage.save_or_create_index(); !result.has_value()) { - LOG_WARN("Could not save index after level {}: {}", level, result.error()); + LOG_WARN( + "Could not save index after level {}: {}", + level, + store::describe_error(result.error())); } } output_storage = std::move(ctx.output_storage).release(); + return {}; } // Builds the complete DAG from input_storage into output_storage. -void build_full( - const octree::IndexedMeshStorage &input_storage, - octree::IndexedDagStorage &output_storage, +std::expected build_full( + const mesh::storage::IndexedStorage &input_storage, + dag::storage::IndexedStorage &output_storage, const BuildOptions &options) { - build_levels(input_storage, output_storage, options, RangeFull{}); + return build_levels(input_storage, output_storage, options, RangeFull{}); } } // namespace dag diff --git a/src/dag_builder/build.h b/src/dag_builder/build.h index d39491d6..a2b774a4 100644 --- a/src/dag_builder/build.h +++ b/src/dag_builder/build.h @@ -2,13 +2,15 @@ #include #include +#include #include "ContinuationMode.h" #include "Range.h" #include "build_config.h" #include "texturing.h" -#include "octree/storage/MeshStorage.h" +#include "mesh/storage.h" #include "storage.h" +#include "sf/InvalidTopology.h" namespace dag { @@ -30,14 +32,14 @@ struct BuildOptions { ContinuationMode continuation_mode = ContinuationMode::Error; }; -void build_full( - const octree::IndexedMeshStorage &input_storage, - octree::IndexedDagStorage &output_storage, +std::expected build_full( + const mesh::storage::IndexedStorage &input_storage, + dag::storage::IndexedStorage &output_storage, const BuildOptions &options); -void build_levels( - const octree::IndexedMeshStorage &input_storage, - octree::IndexedDagStorage &output_storage, +std::expected build_levels( + const mesh::storage::IndexedStorage &input_storage, + dag::storage::IndexedStorage &output_storage, const BuildOptions &options, const AnyRange &level_range); diff --git a/src/dag_builder/codec/Dag.h b/src/dag_builder/codec/Dag.h new file mode 100644 index 00000000..3209f4bb --- /dev/null +++ b/src/dag_builder/codec/Dag.h @@ -0,0 +1,129 @@ +#pragma once + +#include +#include + +#include "codec/DagFormat.h" +#include "io/envelope_file.h" +#include "store/Codec.h" +#include "store/codec/Path.h" + +namespace dag::codec { + +inline store::CodecError file_error( + const store::CodecOperation operation, + const io::envelope::FileError &error) +{ + const bool missing = std::holds_alternative(error) + && std::get(error) == io::Error(io::Error::OpenFile); + return { + operation, + missing ? store::CodecErrorCategory::FileNotFound + : operation == store::CodecOperation::Read + ? store::CodecErrorCategory::InvalidData + : store::CodecErrorCategory::Io, + io::envelope::describe_error(error), + }; +} + +class Dag final : public store::Codec { +public: + std::vector paths( + const store::NodePath &node_path) const override + { + return { + store::codec::append_extension(node_path, ".dag"), + store::codec::append_extension(node_path, ".dagmeta"), + }; + } + + std::expected read( + const store::NodePath &node_path) const override + { + const auto node_paths = paths(node_path); + auto clustering_payload = + io::envelope::read_from_path(node_paths[0]); + if (!clustering_payload) { + return std::unexpected(file_error( + store::CodecOperation::Read, + clustering_payload.error())); + } + auto metadata_payload = + io::envelope::read_from_path(node_paths[1]); + if (!metadata_payload) { + return std::unexpected(file_error( + store::CodecOperation::Read, + metadata_payload.error())); + } + auto clustering = dag::format::decode_clustering(std::move(*clustering_payload)); + if (!clustering) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + store::CodecErrorCategory::InvalidData, + clustering.error(), + }); + } + auto metadata = dag::format::decode_metadata(std::move(*metadata_payload)); + if (!metadata) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + store::CodecErrorCategory::InvalidData, + metadata.error(), + }); + } + return dag::ClusterBatch{ + .metadata = std::move(*metadata), + .clustering = std::move(*clustering), + }; + } + + std::expected write( + const store::NodePath &node_path, + const dag::ClusterBatch &batch) const override + { + auto clustering = dag::format::encode_clustering(batch.clustering); + if (!clustering) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Domain, + clustering.error(), + }); + } + const dag::format::NodeMetadata metadata = + dag::format::encode_metadata(batch.metadata); + if (auto valid = dag::format::validate(*clustering); !valid) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Domain, + valid.error(), + }); + } + if (auto valid = dag::format::validate(metadata); !valid) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Domain, + valid.error(), + }); + } + const auto node_paths = paths(node_path); + auto data_written = io::envelope::write_to_path( + *clustering, + node_paths[0]); + if (!data_written) { + return std::unexpected(file_error( + store::CodecOperation::Write, + data_written.error())); + } + auto metadata_written = io::envelope::write_to_path( + metadata, + node_paths[1]); + if (!metadata_written) { + return std::unexpected(file_error( + store::CodecOperation::Write, + metadata_written.error())); + } + return {}; + } +}; + +} // namespace dag::codec diff --git a/src/dag_builder/codec/DagFormat.h b/src/dag_builder/codec/DagFormat.h new file mode 100644 index 00000000..41bb19e8 --- /dev/null +++ b/src/dag_builder/codec/DagFormat.h @@ -0,0 +1,391 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "cluster.h" +#include "dag_node.h" +#include "io/envelope.h" +#include "mesh/io/texture.h" +#include "meshopt.h" + +namespace dag::format { + +namespace v1 { + +struct OctreeId { + std::uint8_t level; + std::uint64_t index; +}; + +struct Id { + OctreeId source_batch; + std::uint32_t cluster_index; +}; + +struct Vec3d { + double x; + double y; + double z; +}; + +struct Bounds { + Vec3d min; + Vec3d max; +}; + +struct Group { + std::vector children; + double error; + Bounds bounds; +}; + +struct NodeMetadata { + std::vector group_assignment; + std::vector groups; +}; + +struct Texture { + std::vector jpeg; +}; + +struct Cluster { + std::uint64_t triangle_count; + std::uint64_t vertex_count; + std::uint64_t uv_count; + std::vector encoded_triangles; + std::vector encoded_vertex_indices; + std::vector encoded_uvs; + std::uint32_t id; + std::uint32_t texture_id; + double absolute_error; +}; + +struct Clustering { + std::uint64_t vertex_count; + std::vector encoded_positions; + std::vector clusters; + std::vector textures; +}; + +} // namespace v1 + +using MetadataSchema = io::envelope::PayloadSchema< + "dag.NodeMetadata", + io::envelope::Version<1, v1::NodeMetadata>>; +using ClusteringSchema = io::envelope::PayloadSchema< + "dag.Clustering", + io::envelope::Version<1, v1::Clustering>>; +using NodeMetadata = MetadataSchema::latest_type; +using Clustering = ClusteringSchema::latest_type; + +inline v1::OctreeId encode_id(const octree::Id &id) +{ + return {.level = id.level(), .index = id.index_on_level()}; +} + +inline std::expected decode_id(const v1::OctreeId &id) +{ + auto result = octree::Id::try_make( + static_cast(id.level), + static_cast(id.index)); + if (!result) { + return std::unexpected("DAG payload contains an invalid octree ID"); + } + return *result; +} + +inline NodeMetadata encode_metadata(const dag::NodeMetadata &metadata) +{ + NodeMetadata result; + result.group_assignment = metadata.group_assignment; + result.groups.reserve(metadata.groups.size()); + for (const dag::Group &group : metadata.groups) { + v1::Group encoded_group{ + .children = {}, + .error = group.error, + .bounds = { + .min = {group.bounds.min.x, group.bounds.min.y, group.bounds.min.z}, + .max = {group.bounds.max.x, group.bounds.max.y, group.bounds.max.z}, + }, + }; + encoded_group.children.reserve(group.children.size()); + for (const dag::Id &child : group.children) { + encoded_group.children.push_back({ + .source_batch = encode_id(child.source_batch), + .cluster_index = child.cluster_index, + }); + } + result.groups.push_back(std::move(encoded_group)); + } + return result; +} + +inline std::expected validate(const NodeMetadata &metadata) +{ + for (const std::uint32_t group : metadata.group_assignment) { + if (group >= metadata.groups.size()) { + return std::unexpected("DAG metadata contains an invalid group assignment"); + } + } + for (const v1::Group &group : metadata.groups) { + if (!std::isfinite(group.error)) { + return std::unexpected("DAG metadata contains a non-finite error"); + } + for (const v1::Id &child : group.children) { + if (!decode_id(child.source_batch)) { + return std::unexpected("DAG metadata contains an invalid child ID"); + } + } + } + return {}; +} + +inline std::expected decode_metadata(NodeMetadata metadata) +{ + if (auto valid = validate(metadata); !valid) { + return std::unexpected(valid.error()); + } + dag::NodeMetadata result; + result.group_assignment = std::move(metadata.group_assignment); + result.groups.reserve(metadata.groups.size()); + for (v1::Group &group : metadata.groups) { + dag::Group decoded_group{ + .children = {}, + .error = group.error, + .bounds = { + {group.bounds.min.x, group.bounds.min.y, group.bounds.min.z}, + {group.bounds.max.x, group.bounds.max.y, group.bounds.max.z}, + }, + }; + decoded_group.children.reserve(group.children.size()); + for (const v1::Id &child : group.children) { + auto source_batch = decode_id(child.source_batch); + if (!source_batch) { + return std::unexpected(source_batch.error()); + } + decoded_group.children.push_back({*source_batch, child.cluster_index}); + } + result.groups.push_back(std::move(decoded_group)); + } + return result; +} + +inline std::expected encode_clustering( + const ::Clustering &clustering) +{ + Clustering result{ + .vertex_count = clustering.positions.size(), + .encoded_positions = meshopt::encode_vertex_buffer(clustering.positions), + .clusters = {}, + .textures = {}, + }; + result.clusters.reserve(clustering.clusters.size()); + for (const ::Cluster &cluster : clustering.clusters) { + if (cluster.texture_id + == (std::numeric_limits::max)()) { + return std::unexpected("DAG texture ID collides with the on-disk null sentinel"); + } + result.clusters.push_back({ + .triangle_count = cluster.local_triangles.size(), + .vertex_count = cluster.vertex_indices.size(), + .uv_count = cluster.uvs.size(), + .encoded_triangles = meshopt::encode_index_buffer(cluster.local_triangles), + .encoded_vertex_indices = meshopt::encode_vertex_buffer(cluster.vertex_indices), + .encoded_uvs = meshopt::encode_vertex_buffer(cluster.uvs), + .id = cluster.id, + .texture_id = cluster.texture_id.value_or((std::numeric_limits::max)()), + .absolute_error = cluster.absolute_error, + }); + } + result.textures.reserve(clustering.textures.size()); + try { + for (const cv::Mat &texture : clustering.textures) { + auto jpeg = mesh::io::write_texture_to_encoded_buffer(texture, ".jpeg"); + if (!texture.empty() && jpeg.empty()) { + return std::unexpected("could not encode DAG texture"); + } + result.textures.push_back({std::move(jpeg)}); + } + } catch (const cv::Exception &) { + return std::unexpected("could not encode DAG texture"); + } + return result; +} + +inline bool count_fits(const std::uint64_t count, const std::size_t element_size) +{ + return count <= std::numeric_limits::max() / element_size + && count * element_size <= io::envelope::default_max_decompressed_size; +} + +inline std::expected validate(const Clustering &clustering) +{ + if (!count_fits(clustering.vertex_count, sizeof(glm::dvec3))) { + return std::unexpected("DAG position count exceeds the allocation limit"); + } + if (clustering.vertex_count != 0 && clustering.encoded_positions.empty()) { + return std::unexpected("DAG position count and encoded data disagree"); + } + std::size_t decoded_size = + static_cast(clustering.vertex_count) * sizeof(glm::dvec3); + for (const v1::Cluster &cluster : clustering.clusters) { + if (!count_fits(cluster.triangle_count, sizeof(glm::uvec3)) + || !count_fits(cluster.vertex_count, sizeof(std::uint32_t)) + || !count_fits(cluster.uv_count, sizeof(glm::dvec2))) { + return std::unexpected("DAG cluster count exceeds the allocation limit"); + } + const std::size_t cluster_size = + static_cast(cluster.triangle_count) * sizeof(glm::uvec3) + + static_cast(cluster.vertex_count) * sizeof(std::uint32_t) + + static_cast(cluster.uv_count) * sizeof(glm::dvec2); + if (cluster_size > io::envelope::default_max_decompressed_size - decoded_size) { + return std::unexpected("DAG decoded data exceeds the allocation limit"); + } + decoded_size += cluster_size; + if (cluster.uv_count != 0 && cluster.uv_count != cluster.vertex_count) { + return std::unexpected("DAG cluster UV count does not match its vertex count"); + } + if ((cluster.triangle_count != 0 && cluster.encoded_triangles.empty()) + || (cluster.vertex_count != 0 && cluster.encoded_vertex_indices.empty()) + || (cluster.uv_count != 0 && cluster.encoded_uvs.empty())) { + return std::unexpected("DAG cluster count and encoded data disagree"); + } + if (cluster.texture_id != (std::numeric_limits::max)() + && cluster.texture_id >= clustering.textures.size()) { + return std::unexpected("DAG cluster contains an invalid texture reference"); + } + if (!std::isfinite(cluster.absolute_error)) { + return std::unexpected("DAG cluster contains a non-finite error"); + } + } + return {}; +} + +template +inline std::expected, std::string> decode_vertex_buffer( + const std::uint64_t count, + const std::vector &encoded, + const std::string_view label) +{ + if (!count_fits(count, sizeof(T))) { + return std::unexpected(std::string(label) + " count exceeds the allocation limit"); + } + std::vector result(static_cast(count)); + if (meshopt_decodeVertexBuffer( + result.data(), + result.size(), + sizeof(T), + encoded.data(), + encoded.size()) + != 0) { + return std::unexpected("could not decode " + std::string(label)); + } + return result; +} + +inline std::expected, std::string> decode_triangles( + const std::uint64_t count, + const std::vector &encoded) +{ + if (!count_fits(count, sizeof(glm::uvec3))) { + return std::unexpected("DAG triangle count exceeds the allocation limit"); + } + std::vector result(static_cast(count)); + if (meshopt_decodeIndexBuffer( + result.data(), + result.size() * 3, + sizeof(std::uint32_t), + encoded.data(), + encoded.size()) + != 0) { + return std::unexpected("could not decode DAG triangles"); + } + return result; +} + +inline std::expected<::Clustering, std::string> decode_clustering(Clustering clustering) +{ + if (auto valid = validate(clustering); !valid) { + return std::unexpected(valid.error()); + } + auto positions = decode_vertex_buffer( + clustering.vertex_count, + clustering.encoded_positions, + "DAG positions"); + if (!positions) { + return std::unexpected(positions.error()); + } + + ::Clustering result; + result.positions = std::move(*positions); + result.clusters.reserve(clustering.clusters.size()); + for (const v1::Cluster &encoded : clustering.clusters) { + auto triangles = decode_triangles(encoded.triangle_count, encoded.encoded_triangles); + auto vertex_indices = decode_vertex_buffer( + encoded.vertex_count, + encoded.encoded_vertex_indices, + "DAG vertex indices"); + auto uvs = decode_vertex_buffer( + encoded.uv_count, + encoded.encoded_uvs, + "DAG UVs"); + if (!triangles) { + return std::unexpected(triangles.error()); + } + if (!vertex_indices) { + return std::unexpected(vertex_indices.error()); + } + if (!uvs) { + return std::unexpected(uvs.error()); + } + for (const std::uint32_t vertex : *vertex_indices) { + if (vertex >= result.positions.size()) { + return std::unexpected("DAG cluster contains an invalid global vertex index"); + } + } + for (const glm::uvec3 triangle : *triangles) { + if (glm::any(glm::greaterThanEqual( + triangle, + glm::uvec3(static_cast(vertex_indices->size()))))) { + return std::unexpected("DAG cluster contains an invalid local triangle index"); + } + } + result.clusters.push_back({ + .id = encoded.id, + .vertex_indices = std::move(*vertex_indices), + .local_triangles = std::move(*triangles), + .uvs = std::move(*uvs), + .texture_id = encoded.texture_id == (std::numeric_limits::max)() + ? std::nullopt + : std::optional{encoded.texture_id}, + .absolute_error = encoded.absolute_error, + }); + } + + std::vector textures; + textures.reserve(clustering.textures.size()); + try { + for (const v1::Texture &texture : clustering.textures) { + cv::Mat image = mesh::io::read_texture_from_encoded_bytes(texture.jpeg); + if (image.empty()) { + return std::unexpected("could not decode DAG texture"); + } + textures.push_back(std::move(image)); + } + } catch (const cv::Exception &) { + return std::unexpected("could not decode DAG texture"); + } + result.textures = TextureSet(std::move(textures)); + return result; +} + +} // namespace dag::format diff --git a/src/dag_builder/codec/MetadataView.h b/src/dag_builder/codec/MetadataView.h new file mode 100644 index 00000000..c27dfa62 --- /dev/null +++ b/src/dag_builder/codec/MetadataView.h @@ -0,0 +1,42 @@ +#pragma once + +#include +#include + +#include "codec/Dag.h" +#include "codec/DagFormat.h" +#include "io/envelope_file.h" +#include "store/Codec.h" +#include "store/codec/Path.h" + +namespace dag::codec { + +class MetadataView final : public store::Codec { +public: + std::vector paths( + const store::NodePath &node_path) const override + { + return {store::codec::append_extension(node_path, ".dagmeta")}; + } + + std::expected read( + const store::NodePath &node_path) const override + { + auto payload = io::envelope::read_from_path( + paths(node_path).front()); + if (!payload) { + return std::unexpected(file_error(store::CodecOperation::Read, payload.error())); + } + auto metadata = dag::format::decode_metadata(std::move(*payload)); + if (!metadata) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + store::CodecErrorCategory::InvalidData, + metadata.error(), + }); + } + return std::move(*metadata); + } +}; + +} // namespace dag::codec diff --git a/src/dag_builder/codec/from_extension.h b/src/dag_builder/codec/from_extension.h new file mode 100644 index 00000000..53d558d1 --- /dev/null +++ b/src/dag_builder/codec/from_extension.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include +#include + +#include + +#include "codec/MetadataView.h" +#include "codec/Dag.h" +#include "dag_node.h" + +namespace dag::codec { + +inline std::expected>, store::CodecError> +from_extension(const std::string_view extension) { + if (extension == ".dag") { + return std::make_unique(); + } + return std::unexpected(store::CodecError{ + store::CodecOperation::Resolve, + store::CodecErrorCategory::UnsupportedCodec, + "unsupported DAG codec selector: " + std::string(extension), + }); +} + +inline std::expected>, store::CodecError> +metadata_from_extension(const std::string_view extension) { + if (extension == ".dag") { + return std::make_unique(); + } + return std::unexpected(store::CodecError{ + store::CodecOperation::Resolve, + store::CodecErrorCategory::UnsupportedCodec, + "unsupported DAG metadata codec selector: " + std::string(extension), + }); +} + +} // namespace dag::codec diff --git a/src/dag_builder/dag_id.h b/src/dag_builder/dag_id.h index 232f66fd..3370d1bf 100644 --- a/src/dag_builder/dag_id.h +++ b/src/dag_builder/dag_id.h @@ -2,8 +2,6 @@ #include -#include - #include "octree/Id.h" #include "hash_utils.h" @@ -33,17 +31,3 @@ struct fmt::formatter { return fmt::format_to(ctx.out(), "{}:{}", id.source_batch, id.cluster_index); } }; - -namespace zpp::bits { - -template -auto serialize(Archive &archive, const dag::Id &id) { - return archive(id.source_batch, id.cluster_index); -} - -template -auto serialize(Archive &archive, dag::Id &id) { - return archive(id.source_batch, id.cluster_index); -} - -} diff --git a/src/dag_builder/dag_node.h b/src/dag_builder/dag_node.h index c276a27a..2b46e6ec 100644 --- a/src/dag_builder/dag_node.h +++ b/src/dag_builder/dag_node.h @@ -18,17 +18,3 @@ inline ClusterBatch make_leaf_batch(Clustering clustering) { } } - -namespace dag { - -template -auto serialize(Archive &archive, const dag::ClusterBatch &node) { - return archive(node.metadata, node.clustering); -} - -template -auto serialize(Archive &archive, dag::ClusterBatch &node) { - return archive(node.metadata, node.clustering); -} - -} // namespace dag diff --git a/src/dag_builder/encoded.h b/src/dag_builder/encoded.h deleted file mode 100644 index 769e567f..00000000 --- a/src/dag_builder/encoded.h +++ /dev/null @@ -1,178 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "TextureSet.h" -#include "cluster.h" -#include "io/bytes.h" -#include "io/serialize.h" -#include "mesh/io/texture.h" -#include "meshopt.h" - -template -auto serialize(Archive &archive, const TextureSet &textures) { - size_t size = textures.size(); - auto result = archive(size); - using Result = std::remove_cvref_t; - if constexpr (!std::is_same_v && !std::is_same_v) { - return result; - } else { - if (zpp::bits::failure(result)) { - return result; - } - - for (const auto &texture : textures) { - const mesh::io::ImageAndExt item{texture, ".jpeg"}; - result = archive(item); - if (zpp::bits::failure(result)) { - return result; - } - } - - return result; - } -} -template -auto serialize(Archive &archive, TextureSet &textures) { - size_t size; - auto result = archive(size); - using Result = std::remove_cvref_t; - if constexpr (!std::is_same_v && !std::is_same_v) { - return result; - } else { - if (zpp::bits::failure(result)) { - return result; - } - - std::vector images; - images.reserve(size); - for (size_t i = 0; i < size; i++) { - mesh::io::ImageAndExt item; - result = archive(item); - if (zpp::bits::failure(result)) { - return result; - } - - images.push_back(item.image); - } - textures = TextureSet(images); - - return result; - } -} - -template -auto serialize(Archive &archive, const Cluster &cluster) { - const size_t triangle_count = cluster.triangle_count(); - const size_t vertex_count = cluster.vertex_count(); - const size_t uv_count = cluster.uvs.size(); - - auto encoded_triangles = meshopt::encode_index_buffer(cluster.local_triangles); - auto encoded_vertex_indices = meshopt::encode_vertex_buffer(cluster.vertex_indices); - auto encoded_uvs = meshopt::encode_vertex_buffer(cluster.uvs); - - return archive( - triangle_count, - vertex_count, - uv_count, - encoded_triangles, - encoded_vertex_indices, - encoded_uvs, - cluster.id, - cluster.texture_id, - cluster.absolute_error); -} -template -auto serialize(Archive &archive, Cluster &cluster) { - size_t triangle_count; - size_t vertex_count; - size_t uv_count; - - std::vector encoded_triangles; - std::vector encoded_vertex_indices; - std::vector encoded_uvs; - - auto result = archive( - triangle_count, - vertex_count, - uv_count, - encoded_triangles, - encoded_vertex_indices, - encoded_uvs, - cluster.id, - cluster.texture_id, - cluster.absolute_error); - - using Result = std::remove_cvref_t; - if constexpr (!std::is_same_v && !std::is_same_v) { - return result; - } else { - if (zpp::bits::failure(result)) { - return result; - } - - meshopt::decode_index_buffer(cluster.local_triangles, triangle_count, encoded_triangles); - meshopt::decode_vertex_buffer(cluster.vertex_indices, vertex_count, encoded_vertex_indices); - meshopt::decode_vertex_buffer(cluster.uvs, uv_count, encoded_uvs); - - return result; - } -} - -template -auto serialize(Archive &archive, const Clustering &clustering) { - const size_t vertex_count = clustering.vertex_count(); - auto encoded_positions = meshopt::encode_vertex_buffer(clustering.positions); - - return archive( - vertex_count, - encoded_positions, - clustering.clusters, - clustering.textures); -} - -template -auto serialize(Archive &archive, Clustering &clustering) { - size_t vertex_count; - std::vector encoded_positions; - - auto result = archive( - vertex_count, - encoded_positions, - clustering.clusters, - clustering.textures); - - using Result = std::remove_cvref_t; - if constexpr (!std::is_same_v && !std::is_same_v) { - return result; - } else { - if (zpp::bits::failure(result)) { - return result; - } - - meshopt::decode_vertex_buffer(clustering.positions, vertex_count, encoded_positions); - - return result; - } -} - -inline tl::expected -save_clustering(const Clustering &clustering, - const std::filesystem::path &path, - const bool make_dirs = true) { - return ::io::write_to_path(clustering, path, make_dirs); -} - -inline tl::expected -load_clustering(const std::filesystem::path &path) { - return ::io::read_from_path(path); -} diff --git a/src/dag_builder/main.cpp b/src/dag_builder/main.cpp index d640d37f..b733c3c6 100644 --- a/src/dag_builder/main.cpp +++ b/src/dag_builder/main.cpp @@ -5,10 +5,10 @@ #include "cli.h" #include "build.h" #include "log.h" -#include "octree/storage/Storage.h" -#include "octree/storage/MeshStorage.h" #include "octree/storage/open.h" #include "storage.h" +#include "store/describe_error.h" +#include "sf/Error.h" #include "ContinuationMode.h" int main(int argc, char **argv) { @@ -17,8 +17,24 @@ int main(int argc, char **argv) { Log::init(args.log_level); try { - const octree::IndexedMeshStorage input_storage = octree::open_folder_indexed(args.input_path); - octree::IndexedDagStorage output_storage = octree::open_folder_indexed(args.output_path); + auto input_result = octree::open_folder_indexed(args.input_path); + if (!input_result.has_value()) { + LOG_ERROR( + "Failed to open input dataset {}: {}", + args.input_path, + store::describe_error(input_result.error())); + return EXIT_FAILURE; + } + auto output_result = dag::storage::open_folder_indexed(args.output_path); + if (!output_result.has_value()) { + LOG_ERROR( + "Failed to open output dataset {}: {}", + args.output_path, + store::describe_error(output_result.error())); + return EXIT_FAILURE; + } + const mesh::storage::IndexedStorage input_storage = std::move(input_result.value()); + dag::storage::IndexedStorage output_storage = std::move(output_result.value()); output_storage.settings().allow_overwrite = args.continuation_mode == ContinuationMode::Overwrite; dag::BuildOptions options{ @@ -43,12 +59,27 @@ int main(int argc, char **argv) { options.relative_target_error = 0.001f; } - dag::build_levels(input_storage, output_storage, options, args.level_range); - output_storage.save_index(); + const auto build_result = dag::build_levels( + input_storage, + output_storage, + options, + args.level_range); + if (!build_result.has_value()) { + LOG_ERROR("Invalid Structura Fundamentalis input: {}", sf::describe_error(build_result.error())); + return EXIT_FAILURE; + } + const auto index_result = output_storage.save_index(); + if (!index_result.has_value()) { + LOG_ERROR( + "Failed to save output index in {}: {}", + args.output_path, + store::describe_error(index_result.error())); + return EXIT_FAILURE; + } return EXIT_SUCCESS; } catch (const std::exception &e) { LOG_ERROR("{}", e.what()); return EXIT_FAILURE; } -} \ No newline at end of file +} diff --git a/src/dag_builder/metadata.h b/src/dag_builder/metadata.h index 5527fe3a..e5c5d35e 100644 --- a/src/dag_builder/metadata.h +++ b/src/dag_builder/metadata.h @@ -8,10 +8,6 @@ #include "cluster.h" #include "dag_id.h" #include "utils.h" -#include "zpp_bits_glm.h" -#include "octree/storage/defaults.h" -#include "octree/storage/codec/ZppBitsCodec.h" -#include "octree/storage/codec/ReadOnlyCodec.h" namespace dag { @@ -72,42 +68,3 @@ inline const radix::geometry::Aabb3d &get_group_bounds(const NodeMetadata &metad } } // namespace dag - -// Set default codec for metadata. -namespace octree { -template <> -struct DefaultCodec { - using type = ReadOnlyCodec>; -}; -} - -namespace zpp::bits { - -template -auto serialize(Archive &archive, const radix::geometry::Aabb3d &bounds) { - return archive(bounds.min, bounds.max); -} -template -auto serialize(Archive &archive, radix::geometry::Aabb3d &bounds) { - return archive(bounds.min, bounds.max); -} - -template -auto serialize(Archive &archive, const dag::Group &group) { - return archive(group.children, group.error, group.bounds); -} -template -auto serialize(Archive &archive, dag::Group &group) { - return archive(group.children, group.error, group.bounds); -} - -template -auto serialize(Archive &archive, const dag::NodeMetadata &metadata) { - return archive(metadata.group_assignment, metadata.groups); -} -template -auto serialize(Archive &archive, dag::NodeMetadata &metadata) { - return archive(metadata.group_assignment, metadata.groups); -} - -} // namespace zpp::bits diff --git a/src/dag_builder/simplify.h b/src/dag_builder/simplify.h index 1ab64f74..e641994b 100644 --- a/src/dag_builder/simplify.h +++ b/src/dag_builder/simplify.h @@ -264,7 +264,7 @@ inline Clustering simplify( uvs[new_index] = original_cluster.uvs[original_index]; } } - + // Make error absolute and combine with the input cluster's error const double absolute_error = result.relative_error * max_extents; const double combined_error = detail::combine_error(options.error_mode, original_cluster.absolute_error, absolute_error); diff --git a/src/dag_builder/storage.h b/src/dag_builder/storage.h index 8c520529..77250463 100644 --- a/src/dag_builder/storage.h +++ b/src/dag_builder/storage.h @@ -1,15 +1,66 @@ #pragma once -#include "dag_node.h" -#include "metadata.h" -#include "octree/storage/Storage.h" -#include "octree/storage/IndexedStorage.h" -namespace octree { +#include +#include -using DagStorage = Storage_; -using IndexedDagStorage = IndexedStorage_; +#include "codec/from_extension.h" +#include "octree/StoreTraits.h" +#include "octree/storage/IndexFile.h" +#include "octree/storage/open_runtime.h" +#include "store/IndexedStorage.h" -using DagMetaStorage = Storage_; -using IndexedDagMetaStorage = IndexedStorage_; +namespace dag::storage { +inline constexpr std::string_view payload_class = "dag.ClusterBatch"; + +using OpenOptions = octree::storage::OpenOptions; +using Storage = store::Storage; +using IndexedStorage = store::IndexedStorage; +using MetadataStorage = store::Storage; +using IndexedMetadataStorage = + store::IndexedStorage; + +inline std::expected> open_index( + const std::filesystem::path &path) { + return store::open_index( + path, + octree::storage::index_format(), + payload_class, + dag::codec::from_extension); } + +inline std::expected> open_folder( + const std::filesystem::path &path, + OpenOptions options = {}) { + return octree::storage::open_folder( + path, + std::string(payload_class), + ".dag", + dag::codec::from_extension, + std::move(options)); +} + +inline std::expected> +open_folder_indexed(const std::filesystem::path &path, OpenOptions options = {}) { + return octree::storage::open_folder_indexed( + path, + std::string(payload_class), + ".dag", + dag::codec::from_extension, + std::move(options)); +} + +inline std::expected> +open_metadata_indexed(const std::filesystem::path &path) { + const std::filesystem::path index_path = + path.filename() == octree::storage::index_file_name + ? path + : path / octree::storage::index_file_name; + return store::open_index( + index_path, + octree::storage::index_format(), + payload_class, + dag::codec::metadata_from_extension); +} + +} // namespace dag::storage diff --git a/src/dag_builder/thread_safe_storage.h b/src/dag_builder/thread_safe_storage.h index 66424881..9a4c483d 100644 --- a/src/dag_builder/thread_safe_storage.h +++ b/src/dag_builder/thread_safe_storage.h @@ -4,20 +4,19 @@ #include #include -#include - -#include "octree/Id.h" +#include namespace dag { -// A mutex-based thread-safe wrapper around a Storage_ instance. -// TODO: move this locking into Storage_ itself and remove this wrapper once that lands. +// A mutex-based thread-safe wrapper around a Storage instance. +// TODO: move this locking into Storage itself and remove this wrapper once that lands. template class ThreadSafeStorage { static_assert(!std::is_const_v, "ThreadSafeStorage requires a non-const Storage"); public: using value_type = typename Storage::value_type; + using key_type = typename Storage::key_type; using load_error = typename Storage::load_error; using save_error = typename Storage::save_error; @@ -33,12 +32,12 @@ class ThreadSafeStorage { return std::move(this->_storage); } - tl::expected load(const octree::Id &id) const { + std::expected load(const key_type &id) const { std::shared_lock lock(this->_mutex); return this->_storage.load(id); } - bool has(const octree::Id &id) const { + auto has(const key_type &id) const { std::shared_lock lock(this->_mutex); return this->_storage.has(id); } @@ -47,7 +46,7 @@ class ThreadSafeStorage { return this->_storage.base_path(); } - tl::expected save(const octree::Id &id, const value_type &value) const { + std::expected save(const key_type &id, const value_type &value) const { std::unique_lock lock(this->_mutex); return this->_storage.save(id, value); } diff --git a/src/dag_builder/utils.h b/src/dag_builder/utils.h index 1a01504c..81ca5ec7 100644 --- a/src/dag_builder/utils.h +++ b/src/dag_builder/utils.h @@ -147,7 +147,7 @@ inline mesh::Simple manifold_clustering_to_mesh(const Clustering &clustering, co mesh.triangles.push_back(local_triangle + base_vertex); } } - + DEBUG_ASSERT(mesh::is_manifold(mesh)); if (debug_texture) { diff --git a/src/dag_builder/zpp_bits_glm.h b/src/dag_builder/zpp_bits_glm.h deleted file mode 100644 index 0dade30d..00000000 --- a/src/dag_builder/zpp_bits_glm.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include -#include - -#include "glm_utils.h" - -namespace zpp::bits { - -template -auto serialize(Archive &archive, const glm::vec &v) { - return archive(as_span(v)); -} - -template -auto serialize(Archive &archive, glm::vec &v) { - return archive(as_span(v)); -} - -} diff --git a/src/dag_convert_debug/main.cpp b/src/dag_convert_debug/main.cpp index 88f776fa..6f21e90a 100644 --- a/src/dag_convert_debug/main.cpp +++ b/src/dag_convert_debug/main.cpp @@ -2,23 +2,30 @@ #include #include "cli.h" +#include "codec/Dag.h" #include "dag_node.h" -#include "encoded.h" #include "log.h" #include "mesh/io.h" -#include "octree/storage/MeshStorage.h" -#include "octree/storage/codec/ZppBitsCodec.h" +#include "mesh/storage.h" #include "octree/storage/open.h" #include "ProgressIndicator.h" #include "storage.h" #include "utils.h" +#include "store/describe_error.h" namespace { void export_node(const cli::Args &args) { - const auto load_result = octree::ZppBitsCodec::load_from_path(args.input_path); + if (args.input_path.extension() != ".dag") { + LOG_ERROR("Expected a .dag input file, got {}", args.input_path); + return; + } + std::filesystem::path node_path = args.input_path; + node_path.replace_extension(); + const dag::codec::Dag codec; + const auto load_result = codec.read(store::NodePath(node_path)); if (!load_result.has_value()) { - LOG_ERROR("Failed to load node from {}: {}", args.input_path, load_result.error()); + LOG_ERROR("Failed to load node from {}: {}", args.input_path, load_result.error().message); return; } @@ -31,12 +38,29 @@ void export_node(const cli::Args &args) { } void export_storage(const cli::Args &args) { - const octree::IndexedDagStorage input_storage = octree::open_folder_indexed(args.input_path); + auto input_result = dag::storage::open_folder_indexed(args.input_path); + if (!input_result.has_value()) { + LOG_ERROR( + "Failed to open input storage {}: {}", + args.input_path, + store::describe_error(input_result.error())); + return; + } + const dag::storage::IndexedStorage input_storage = std::move(input_result.value()); - octree::MeshStorage output_storage = octree::open_folder( + octree::OpenOptions options; + options.preferred_extension = ".glb"; + auto output_result = octree::open_folder( args.output_path, - false, - octree::OpenOptions{.preferred_extension_with_dot = ".glb"}); + std::move(options)); + if (!output_result.has_value()) { + LOG_ERROR( + "Failed to open output storage {}: {}", + args.output_path, + store::describe_error(output_result.error())); + return; + } + mesh::storage::Storage output_storage = std::move(output_result.value()); output_storage.settings().allow_overwrite = true; size_t exported_count = 0; @@ -45,14 +69,17 @@ void export_storage(const cli::Args &args) { auto progress_thread = progress.start_monitoring(); for (const auto &[id, status] : input_storage.index()) { - if (status == octree::NodeStatus::Virtual) { + if (status == store::NodeStatus::Virtual) { progress.task_finished(); continue; } const auto load_result = input_storage.load(id); if (!load_result.has_value()) { - LOG_ERROR("Failed to load node {}: {}", id, load_result.error()); + LOG_ERROR( + "Failed to load node {}: {}", + id, + store::describe_error(load_result.error())); progress.task_finished(); continue; } @@ -61,7 +88,10 @@ void export_storage(const cli::Args &args) { const auto save_result = output_storage.save(id, mesh); if (!save_result.has_value()) { - LOG_ERROR("Failed to save mesh for node {}: {}", id, save_result.error().description()); + LOG_ERROR( + "Failed to save mesh for node {}: {}", + id, + store::describe_error(save_result.error())); progress.task_finished(); continue; } @@ -74,7 +104,10 @@ void export_storage(const cli::Args &args) { const auto index_result = output_storage.save_or_create_index(); if (!index_result.has_value()) { - LOG_ERROR("Failed to save index for {}: {}", args.output_path, index_result.error()); + LOG_ERROR( + "Failed to save index for {}: {}", + args.output_path, + store::describe_error(index_result.error())); } LOG_INFO("Exported {} debug meshes to {}", exported_count, args.output_path); diff --git a/src/mesh_convert/main.cpp b/src/mesh_convert/main.cpp index ef258319..a396ce50 100644 --- a/src/mesh_convert/main.cpp +++ b/src/mesh_convert/main.cpp @@ -1,7 +1,7 @@ #include #include -#include +#include #include "mesh/SimpleMesh.h" #include "mesh/io.h" @@ -11,7 +11,7 @@ void run(const cli::Args& args) { LOG_INFO("Loading input mesh..."); - const tl::expected load_result = mesh::io::load_from_path(args.input_path); + const std::expected load_result = mesh::io::load_from_path(args.input_path); if (!load_result.has_value()) { LOG_ERROR("Failed to load mesh: {}", load_result.error().description()); return; @@ -30,7 +30,7 @@ void run(const cli::Args& args) { } LOG_INFO("Writing output mesh..."); - const tl::expected save_result = mesh::io::save_to_path(mesh, args.output_path); + const std::expected save_result = mesh::io::save_to_path(mesh, args.output_path); if (!save_result.has_value()) { LOG_ERROR("Failed to save mesh: {}", save_result.error().description()); return; diff --git a/src/sf_builder/CMakeLists.txt b/src/sf_builder/CMakeLists.txt index c50cae1c..cfdabb70 100644 --- a/src/sf_builder/CMakeLists.txt +++ b/src/sf_builder/CMakeLists.txt @@ -3,7 +3,7 @@ add_library(sfbuilderlib mesh_builder.cpp ) target_include_directories(sfbuilderlib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(sfbuilderlib PUBLIC terrainlib spdlog tl_expected) +target_link_libraries(sfbuilderlib PUBLIC terrainlib spdlog) add_executable(sf-builder main.cpp) target_link_libraries(sf-builder PRIVATE sfbuilderlib CLI11::CLI11) diff --git a/src/sf_builder/main.cpp b/src/sf_builder/main.cpp index b285134d..e95c3464 100644 --- a/src/sf_builder/main.cpp +++ b/src/sf_builder/main.cpp @@ -18,6 +18,7 @@ #include "terrainbuilder.h" #include "octree/Space.h" #include "tile_provider.h" +#include "sf/Error.h" #include "ctb/GlobalGeodetic.hpp" #include "ctb/GlobalMercator.hpp" @@ -70,7 +71,6 @@ radix::geometry::Aabb3d parse_bounds_from_values(const std::vector &data radix::geometry::Aabb3d parse_bounds_from_tile( const std::vector &data, - radix::tile::Scheme scheme, const OGRSpatialReference &srs) { // Determine the correct Grid type based on SRS @@ -87,7 +87,7 @@ radix::geometry::Aabb3d parse_bounds_from_tile( const uint32_t zoom_level = data[0]; const glm::uvec2 tile_coords(data[1], data[2]); - const radix::tile::Id target_tile(zoom_level, tile_coords, scheme); + const radix::tile::Id target_tile(zoom_level, tile_coords); return extend_bounds_to_3d(grid->srsBounds(target_tile, false)); } @@ -118,7 +118,6 @@ radix::geometry::Aabb3d parse_target_bounds( const std::vector &bounds_data, const std::vector &node_data, const std::vector &tile_data, - radix::tile::Scheme tile_scheme, OGRSpatialReference &srs) { if (!bounds_data.empty()) { @@ -126,7 +125,7 @@ radix::geometry::Aabb3d parse_target_bounds( } if (!tile_data.empty()) { - return parse_bounds_from_tile(tile_data, tile_scheme, srs); + return parse_bounds_from_tile(tile_data, srs); } if (!node_data.empty()) { @@ -223,7 +222,6 @@ int run(std::span args) { std::vector target_tile_data; std::vector target_node_data; std::string target_srs_input; - radix::tile::Scheme target_tile_scheme; auto *target = single->add_option_group("target"); target->add_option("--bounds", target_bounds_data, "Target bounds for the reference mesh as \"{xmin} {width} {ymin} {height} [{zmin} {depth}]\"") @@ -234,18 +232,9 @@ int run(std::span args) { ->expected(2, 4); target->require_option(1); - single->add_option("--output", output_path, "Output path were the mesh is written to (.terrain, .gltf or .glb)") + single->add_option("--output", output_path, "Output path were the mesh is written to (.sfmesh, .gltf or .glb)") ->required(); - std::map scheme_str_map{ - {"slippymap", radix::tile::Scheme::SlippyMap}, - {"google", radix::tile::Scheme::SlippyMap}, - {"tms", radix::tile::Scheme::Tms}}; - single->add_option("--scheme", target_tile_scheme, "Tile scheme") - ->default_val(radix::tile::Scheme::SlippyMap) - ->needs("--tile") - ->transform(CLI::CheckedTransformer(scheme_str_map, CLI::ignore_case)); - single->add_option("--srs", target_srs_input, "EPSG code of the srs of the target bounds or id"); single->callback([&]() { if (target_srs_input.empty()) { @@ -271,8 +260,8 @@ int run(std::span args) { ->required(); std::string output_format; batch->add_option("--format", output_format, "Output mesh format") - ->check(CLI::IsMember({".glb", ".gltf", ".terrain"})) - ->default_val(".terrain"); + ->check(CLI::IsMember({".glb", ".gltf", ".sfmesh"})) + ->default_val(".sfmesh"); uint32_t num_threads = 0; batch->add_option("--threads", num_threads, "Number of threads to use") ->check(CLI::PositiveNumber); @@ -290,14 +279,14 @@ int run(std::span args) { std::unique_ptr tile_provider; if (texture_base_path.has_value()) { - BasemapSchemeTilePathProvider basemap_provider(texture_base_path.value()); + GoogleMapboxTilePathProvider basemap_provider(texture_base_path.value()); if (min_texture_level.has_value() || max_texture_level.has_value()) { - tile_provider = std::make_unique>( + tile_provider = std::make_unique>( std::move(basemap_provider), min_texture_level, max_texture_level); } else { - tile_provider = std::make_unique(std::move(basemap_provider)); + tile_provider = std::make_unique(std::move(basemap_provider)); } } @@ -307,7 +296,6 @@ int run(std::span args) { target_bounds_data, target_node_data, target_tile_data, - target_tile_scheme, target_srs); terrainbuilder::build_and_save_patch( @@ -330,7 +318,7 @@ int run(std::span args) { const auto maybe_s = actual_num_threads == 1 ? "" : "s"; LOG_INFO("Using {} thread{} for batch processing.", actual_num_threads, maybe_s); - terrainbuilder::build_all_patches( + const auto build_result = terrainbuilder::build_all_patches( dataset, target_level, texture_srs, @@ -339,6 +327,12 @@ int run(std::span args) { output_base_path, output_format, overwrite_existing); + if (!build_result.has_value()) { + LOG_ERROR( + "Failed to finalize Structura Fundamentalis output: {}", + sf::describe_error(build_result.error())); + return 1; + } } return 0; diff --git a/src/sf_builder/mesh_builder.cpp b/src/sf_builder/mesh_builder.cpp index dc2b19b2..be440a94 100644 --- a/src/sf_builder/mesh_builder.cpp +++ b/src/sf_builder/mesh_builder.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -15,7 +16,7 @@ #include "mesh/SimpleMesh.h" #include "mesh/cleanup.h" #include "mesh_builder.h" -#include "raster.h" +#include #include "raw_dataset_reader.h" #include "srs.h" #include "mesh/clip.h" @@ -51,7 +52,7 @@ glm::dvec2 apply_transform(std::array transform, const glm::tvec2 return result; } -glm::dvec3 convert_pixel_to_vertex(const float height, const raster::Coords pixel_coords, const RawDatasetReader& reader, const PixelBounds& pixel_bounds) { +glm::dvec3 convert_pixel_to_vertex(const float height, const glm::uvec2 pixel_coords, const RawDatasetReader& reader, const PixelBounds& pixel_bounds) { const glm::dvec2 point_offset_in_raster(0.5); // Convert pixel coordinates into a point in the dataset's srs. const glm::dvec2 coords_raster_relative = glm::dvec2(pixel_coords) + point_offset_in_raster; const glm::dvec2 coords_raster_absolute = coords_raster_relative + glm::dvec2(pixel_bounds.min); @@ -59,9 +60,9 @@ glm::dvec3 convert_pixel_to_vertex(const float height, const raster::Coords pixe return coords_source; } -SimpleMesh meshify(const raster::Raster& source_points, const raster::Mask& mask) { +SimpleMesh meshify(const radix::Raster& source_points, const radix::RasterMask& mask) { // Compact the vertex grid into a list of valid ones. - const size_t valid_vertex_count = std::reduce(mask.begin(), mask.end(), 0); + const size_t valid_vertex_count = std::reduce(mask.begin(), mask.end(), size_t(0)); // Check if we even have any valid vertices. Can happen if all of the region is padding. if (valid_vertex_count == 0) { return SimpleMesh(); @@ -70,11 +71,15 @@ SimpleMesh meshify(const raster::Raster& source_points, const raster std::vector positions; positions.reserve(valid_vertex_count); - const raster::Raster vertex_index_map = raster::transform(source_points, mask, [&](const glm::dvec3 &point) -> size_t { + auto vertex_index_map_result = radix::raster::transform(source_points, mask, [&](const glm::dvec3& point) -> size_t { const size_t index = positions.size(); positions.push_back(point); return index; }); + DEBUG_ASSERT(vertex_index_map_result.has_value()); + if (!vertex_index_map_result.has_value()) + return {}; + const auto vertex_index_map = std::move(*vertex_index_map_result); DEBUG_ASSERT(positions.size() == valid_vertex_count); // Allocate triangle vector @@ -82,13 +87,13 @@ SimpleMesh meshify(const raster::Raster& source_points, const raster std::vector triangles; triangles.reserve(max_triangle_count); - for (size_t y = 0; y < source_points.height() - 1; y++) { - for (size_t x = 0; x < source_points.width() - 1; x++) { - const std::array quad { - raster::Coords{x, y}, - raster::Coords{x + 1, y}, - raster::Coords{x + 1, y + 1}, - raster::Coords{x, y + 1}}; + for (unsigned y = 0; y < source_points.height() - 1; y++) { + for (unsigned x = 0; x < source_points.width() - 1; x++) { + const std::array quad { + glm::uvec2{x, y}, + glm::uvec2{x + 1, y}, + glm::uvec2{x + 1, y + 1}, + glm::uvec2{x, y + 1}}; for (uint32_t i = 0; i < 4; i++) { const auto& v0 = quad[i]; @@ -143,7 +148,7 @@ radix::geometry::Aabb3d extend_bounds_to_3d(radix::geometry::Aabb2d bounds2d) { } } -tl::expected build_reference_mesh_tile( +std::expected build_reference_mesh_tile( Dataset &dataset, const OGRSpatialReference &mesh_srs, const OGRSpatialReference &tile_srs, const radix::tile::SrsBounds &tile_bounds, @@ -151,7 +156,7 @@ tl::expected build_reference_mesh_tile( return build_reference_mesh_patch(dataset, mesh_srs, tile_srs, extend_bounds_to_3d(tile_bounds), texture_srs, texture_bounds); } -tl::expected build_reference_mesh_patch( +std::expected build_reference_mesh_patch( Dataset &dataset, const OGRSpatialReference &mesh_srs, const OGRSpatialReference &clip_srs, const radix::geometry::Aabb3d &clip_bounds, @@ -172,35 +177,39 @@ tl::expected build_reference_mesh_patch( radix::geometry::Aabb2i pixel_bounds = reader.transform_srs_bounds_to_pixel_bounds(target_bounds_in_source_srs); add_border_to_aabb(pixel_bounds, Border(1)); LOG_TRACE("Reading pixels [({}, {})-({}, {})] from dataset", pixel_bounds.min.x, pixel_bounds.min.y, pixel_bounds.max.x, pixel_bounds.max.y); - const std::optional read_result = reader.read_data_in_pixel_bounds_clamped(pixel_bounds); - if (!read_result.has_value() || read_result->size() == 0) { - return tl::unexpected(BuildMeshError::OutOfBounds); + auto read_result = reader.read_data_in_pixel_bounds_clamped(pixel_bounds); + if (!read_result.has_value() || read_result->buffer().empty()) { + return std::unexpected(BuildMeshError::OutOfBounds); } - const raster::HeightMap height_map = read_result.value(); + const radix::Raster height_map = std::move(*read_result); LOG_TRACE("Finding valid pixels"); const float no_data_value = reader.get_no_data_value(); - const raster::Mask valid_mask = raster::transform(height_map, [=](const float height) { + const radix::RasterMask valid_mask = radix::raster::transform(height_map, [=](const float height) { return height != no_data_value; }); LOG_TRACE("Transforming pixels to vertices"); - const raster::Raster source_points = raster::transform(height_map, valid_mask, [&](const float height, const raster::Coords& coords) { + auto source_points_result = radix::raster::transform(height_map, valid_mask, [&](const float height, const glm::uvec2& coords) { return convert_pixel_to_vertex(height, coords, reader, pixel_bounds); }); + DEBUG_ASSERT(source_points_result.has_value()); + if (!source_points_result.has_value()) + return std::unexpected(BuildMeshError::EmptyRegion); + const auto source_points = std::move(*source_points_result); LOG_TRACE("Generating triangles"); SimpleMesh mesh_in_source_srs = meshify(source_points, valid_mask); // Check if we even have any valid vertices. Can happen if all of the region is padding. if (mesh_in_source_srs.vertex_count() == 0 || mesh_in_source_srs.face_count() == 0) { - return tl::unexpected(BuildMeshError::EmptyRegion); + return std::unexpected(BuildMeshError::EmptyRegion); } // Fast check if all vertices will be clipped const radix::geometry::Aabb3d actual_source_bounds = calculate_bounds(mesh_in_source_srs); const radix::geometry::Aabb3d approx_clip_bounds = srs::encompassing_bounds_transfer(source_srs, clip_srs, actual_source_bounds); if (!radix::geometry::intersect(approx_clip_bounds, clip_bounds)) { - return tl::unexpected(BuildMeshError::EmptyRegion); + return std::unexpected(BuildMeshError::EmptyRegion); } LOG_TRACE("Clipping mesh based on target bounds"); @@ -208,7 +217,7 @@ tl::expected build_reference_mesh_patch( SimpleMesh clipped_mesh = mesh::clip_on_bounds(mesh_in_clip_srs, clip_bounds); // Check if there are any vertices left if (clipped_mesh.vertex_count() == 0 || clipped_mesh.face_count() == 0) { - return tl::unexpected(BuildMeshError::EmptyRegion); + return std::unexpected(BuildMeshError::EmptyRegion); } // TODO: move this to another function? diff --git a/src/sf_builder/mesh_builder.h b/src/sf_builder/mesh_builder.h index acf3e4b8..2979042d 100644 --- a/src/sf_builder/mesh_builder.h +++ b/src/sf_builder/mesh_builder.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include "Dataset.h" #include "srs.h" @@ -17,7 +17,7 @@ enum class BuildMeshError { std::ostream &operator<<(std::ostream &os, BuildMeshError error); /// Builds a mesh from the given height dataset. -tl::expected build_reference_mesh_patch( +std::expected build_reference_mesh_patch( Dataset &dataset, const OGRSpatialReference &mesh_srs, const OGRSpatialReference &clip_srs, const radix::geometry::Aabb3d &clip_bounds, diff --git a/src/sf_builder/raster.h b/src/sf_builder/raster.h deleted file mode 100644 index 5b30794e..00000000 --- a/src/sf_builder/raster.h +++ /dev/null @@ -1,148 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -#include -#include - -namespace raster { - -using Index = size_t; -using Coords = glm::vec<2, size_t>; - -template -class Raster { -public: - Raster() = default; - Raster(Index width, Index height) - : _width(width), _height(height), _data(width * height) { - } - - [[nodiscard]] Index width() const { - return this->_width; - } - [[nodiscard]] Index height() const { - return this->_height; - } - [[nodiscard]] decltype(auto) pixel(const Coords &coords) { - DEBUG_ASSERT(coords.x < this->_width); - DEBUG_ASSERT(coords.y < this->_height); - const Index pixel_index = this->index(coords); - if constexpr (std::is_same_v) { - return static_cast(this->_data[pixel_index]); - } else { - return static_cast(this->_data[pixel_index]); - } - } - [[nodiscard]] decltype(auto) pixel(const Coords &coords) const { - DEBUG_ASSERT(coords.x < this->_width); - DEBUG_ASSERT(coords.y < this->_height); - const Index pixel_index = this->index(coords); - if constexpr (std::is_same_v) { - return static_cast(this->_data[pixel_index]); - } else { - return static_cast(this->_data[pixel_index]); - } - } - - [[nodiscard]] Index index(const Coords& coords) const { - return coords.y * this->_width + coords.x; - } - - [[nodiscard]] T *data() { - return this->_data.data(); - } - [[nodiscard]] const T *data() const { - return this->_data.data(); - } - - [[nodiscard]] Index size() const { - return this->_data.size(); - } - [[nodiscard]] auto begin() { - return this->_data.begin(); - } - [[nodiscard]] auto end() { - return this->_data.end(); - } - [[nodiscard]] auto begin() const { - return this->_data.begin(); - } - [[nodiscard]] auto end() const { - return this->_data.end(); - } - -private: - unsigned _width = 0; - unsigned _height = 0; - std::vector _data; -}; - -using Mask = Raster; -using HeightMap = Raster; - -template -concept TransformFn = requires(F f, In in) { - { f(in) }; -}; - -template -concept TransformFnWithCoords = requires(F f, In in, Coords coord) { - { f(in, coord) }; -}; - -template F> -[[nodiscard]] auto transform(const Raster &input, F &&f) { - using Out = decltype(f(input.pixel(Coords(0, 0)))); - Raster output(input.width(), input.height()); - std::transform(input.begin(), input.end(), output.begin(), std::forward(f)); - return output; -} - -template F> -[[nodiscard]] auto transform(const Raster &input, F &&f) { - using Out = decltype(f(input.pixel(Coords(0, 0)), Coords(0, 0))); - Raster output(input.width(), input.height()); - for (Index y = 0; y < input.height(); ++y) { - for (Index x = 0; x < input.width(); ++x) { - Coords coords(x, y); - output.pixel(coords) = f(input.pixel(coords), coords); - } - } - return output; -} - -template F> -[[nodiscard]] auto transform(const Raster &input, const Mask &mask, F &&f) { - using Out = decltype(f(input.pixel(Coords(0, 0)))); - Raster output(input.width(), input.height()); - for (Index y = 0; y < input.height(); ++y) { - for (Index x = 0; x < input.width(); ++x) { - Coords coords(x, y); - if (mask.pixel(coords)) { - output.pixel(coords) = f(input.pixel(coords)); - } - } - } - return output; -} - -template F> -[[nodiscard]] auto transform(const Raster &input, const Mask &mask, F &&f) { - using Out = decltype(f(input.pixel(Coords(0, 0)), Coords(0, 0))); - Raster output(input.width(), input.height()); - for (Index y = 0; y < input.height(); ++y) { - for (Index x = 0; x < input.width(); ++x) { - Coords coords(x, y); - if (mask.pixel(coords)) { - output.pixel(coords) = f(input.pixel(coords), coords); - } - } - } - return output; -} -} diff --git a/src/sf_builder/raw_dataset_reader.h b/src/sf_builder/raw_dataset_reader.h index be91a44d..096526be 100644 --- a/src/sf_builder/raw_dataset_reader.h +++ b/src/sf_builder/raw_dataset_reader.h @@ -14,7 +14,7 @@ #include #include "log.h" -#include "raster.h" +#include namespace terrainbuilder { @@ -53,7 +53,7 @@ class RawDatasetReader { } // TODO: support reading other data types - std::optional read_data_in_pixel_bounds(const radix::geometry::Aabb2i& bounds) { + std::optional> read_data_in_pixel_bounds(const radix::geometry::Aabb2i& bounds) { DEBUG_ASSERT(glm::all(glm::greaterThanEqual(bounds.min, glm::ivec2(0)))); DEBUG_ASSERT(glm::all(glm::greaterThanEqual(bounds.max, glm::ivec2(0)))); DEBUG_ASSERT(glm::all(glm::lessThan(bounds.min, glm::ivec2(this->dataset_size())))); @@ -63,7 +63,7 @@ class RawDatasetReader { GDALRasterBand *height_band = this->dataset->GetRasterBand(1); // non-owning pointer // Initialize the HeightData for reading - raster::HeightMap height_data(bounds.width(), bounds.height()); + radix::Raster height_data(glm::uvec2(bounds.width(), bounds.height())); if (bounds.width() == 0 || bounds.height() == 0) { LOG_WARN("Target dataset bounds are empty"); return height_data; @@ -72,7 +72,7 @@ class RawDatasetReader { // Read data from the heights band into heights_data const int32_t read_result = height_band->RasterIO( GF_Read, bounds.min.x, bounds.min.y, bounds.width(), bounds.height(), - static_cast(height_data.data()), bounds.width(), bounds.height(), GDT_Float32, 0, 0); + static_cast(height_data.buffer().data()), bounds.width(), bounds.height(), GDT_Float32, 0, 0); if (read_result != CE_None) { const char * message = CPLGetLastErrorMsg(); @@ -82,7 +82,7 @@ class RawDatasetReader { return height_data; } - std::optional read_data_in_pixel_bounds_clamped(radix::geometry::Aabb2i &bounds) { + std::optional> read_data_in_pixel_bounds_clamped(radix::geometry::Aabb2i &bounds) { const auto original_bounds = bounds; const glm::ivec2 max_in_bounds = glm::ivec2(this->dataset_size()) - glm::ivec2(1); @@ -97,13 +97,13 @@ class RawDatasetReader { if (bounds.width() == 0 || bounds.height() == 0) { LOG_WARN("Target dataset bounds are empty (clamped)"); - return raster::HeightMap(0, 0); + return radix::Raster(); } return this->read_data_in_pixel_bounds(bounds); } - std::optional read_data_in_srs_bounds(const radix::tile::SrsBounds &bounds) { + std::optional> read_data_in_srs_bounds(const radix::tile::SrsBounds &bounds) { // Transform the SrsBounds to pixel space radix::geometry::Aabb2i pixel_bounds = this->transform_srs_bounds_to_pixel_bounds(bounds); diff --git a/src/sf_builder/terrainbuilder.cpp b/src/sf_builder/terrainbuilder.cpp index ddeb01e6..22a4d392 100644 --- a/src/sf_builder/terrainbuilder.cpp +++ b/src/sf_builder/terrainbuilder.cpp @@ -18,7 +18,6 @@ #include #include #include -#include #include #include @@ -42,9 +41,10 @@ #include "octree/Id.h" #include "octree/Space.h" -#include "octree/Storage.h" -#include "octree/disk/layout/strategy/LevelAndCoordinateDirectories.h" +#include "octree/storage/open.h" #include "octree/utils.h" +#include "store/describe_error.h" +#include "sf/finalize_storage.h" namespace terrainbuilder { @@ -71,7 +71,7 @@ std::optional build_patch( std::chrono::high_resolution_clock::time_point start; start = std::chrono::high_resolution_clock::now(); LOG_INFO("Building mesh..."); - tl::expected mesh_result = build_reference_mesh_patch( + std::expected mesh_result = build_reference_mesh_patch( dataset, mesh_srs, target_bounds_srs, target_bounds, @@ -173,7 +173,11 @@ T expect(const std::optional &opt, const std::string &msg) { } } -void build_all_patches( +std::expected finalize_storage(mesh::storage::Storage &storage) { + return sf::finalize_storage(storage); +} + +std::expected build_all_patches( Dataset &dataset, const octree::Id::Level target_level, const OGRSpatialReference &texture_srs, @@ -190,13 +194,18 @@ void build_all_patches( LOG_ERROR_AND_EXIT("Output base path {} exists but is not a directory", output_base_path); } - octree::Storage storage = octree::open_folder( + octree::OpenOptions open_options; + open_options.preferred_extension = output_format; + auto storage_result = octree::open_folder( output_base_path, - false, - octree::OpenOptions { - .preferred_extension_with_dot = output_format - } - ); + std::move(open_options)); + if (!storage_result.has_value()) { + LOG_ERROR_AND_EXIT( + "Failed to open output dataset {}: {}", + output_base_path, + store::describe_error(storage_result.error())); + } + mesh::storage::Storage storage = std::move(storage_result.value()); storage.settings().allow_overwrite = overwrite_existing; const auto dataset_srs = dataset.srs(); @@ -279,9 +288,20 @@ void build_all_patches( logger->set_level(new_level); } + tbb::task_group_context context; tbb::parallel_for(size_t(0), target_nodes.size(), [&](size_t i) { const auto &node = target_nodes[i]; - if (!overwrite_existing && storage.has(node)) { + const auto already_exists = storage.has(node); + if (!already_exists.has_value()) { + LOG_ERROR( + "Failed to inspect node {}: {}", + node, + store::describe_error(already_exists.error())); + progress.task_finished(); + context.cancel_group_execution(); + return; + } + if (!overwrite_existing && already_exists.value()) { progress.task_finished(); // TODO: correctly handle virtual nodes return; } @@ -303,16 +323,33 @@ void build_all_patches( if (mesh_result.has_value()) { const auto mesh = std::move(mesh_result.value()); mesh::validate(mesh); - storage.save(node, mesh); + const auto save_result = storage.save(node, mesh); + if (!save_result.has_value()) { + LOG_ERROR( + "Failed to save mesh for node {}: {}", + node, + store::describe_error(save_result.error())); + progress.task_finished(); + context.cancel_group_execution(); + return; + } } progress.task_finished(); - }); + }, context); // Restore original level logger->set_level(original_level); + if (context.is_group_execution_cancelled()) { + progress_thread.request_stop(); + } progress_thread.join(); - storage.save_or_create_index(); + + if (context.is_group_execution_cancelled()) { + LOG_ERROR_AND_EXIT("Failed to build all terrain patches"); + } + + return finalize_storage(storage); } } diff --git a/src/sf_builder/terrainbuilder.h b/src/sf_builder/terrainbuilder.h index b4c0f581..8ddbeba0 100644 --- a/src/sf_builder/terrainbuilder.h +++ b/src/sf_builder/terrainbuilder.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include @@ -8,6 +9,8 @@ #include "octree/Id.h" #include "mesh/SimpleMesh.h" #include "tile_provider.h" +#include "mesh/storage.h" +#include "sf/Error.h" namespace terrainbuilder { @@ -28,7 +31,9 @@ std::optional build_patch( const TileProvider *tile_provider, const OGRSpatialReference &mesh_srs); -void build_all_patches( +std::expected finalize_storage(mesh::storage::Storage &storage); + +std::expected build_all_patches( Dataset &dataset, const octree::Id::Level target_level, const OGRSpatialReference &texture_srs, diff --git a/src/sf_builder/texture_assembler.h b/src/sf_builder/texture_assembler.h index bff6c018..8de959f1 100644 --- a/src/sf_builder/texture_assembler.h +++ b/src/sf_builder/texture_assembler.h @@ -196,7 +196,6 @@ inline TargetImageRegion calculate_target_image_region( const radix::tile::Id root_tile, const glm::uvec2 tile_image_pixel_size, const uint32_t max_zoom_level) { - tile = tile.to(radix::tile::Scheme::SlippyMap); const size_t relative_zoom_level = tile.zoom_level - root_tile.zoom_level; const glm::uvec2 tile_size_factor = glm::uvec2(std::pow(2, max_zoom_level - tile.zoom_level)); const glm::uvec2 tile_size = tile_image_pixel_size * tile_size_factor; @@ -469,7 +468,7 @@ inline std::optional assemble_texture_from_tiles( // Start by transforming the input bounds into the srs the tiles are in. const radix::tile::SrsBounds encompassing_bounds = srs::encompassing_bounds_transfer(target_srs, grid.getSRS(), target_bounds); // Then we find the smallest tile (id) that encompasses these bounds. - radix::tile::Id smallest_encompassing_tile = grid.findSmallestEncompassingTile(encompassing_bounds).value().to(radix::tile::Scheme::SlippyMap); + radix::tile::Id smallest_encompassing_tile = grid.findSmallestEncompassingTile(encompassing_bounds).value(); LOG_TRACE("Smallest encompassing tile for texture bounds is {}", radix::tile::to_string(smallest_encompassing_tile)); if (max_zoom.has_value() && smallest_encompassing_tile.zoom_level > max_zoom.value()) { diff --git a/src/sf_builder/tile_provider.h b/src/sf_builder/tile_provider.h index da90c0e2..c709536d 100644 --- a/src/sf_builder/tile_provider.h +++ b/src/sf_builder/tile_provider.h @@ -6,6 +6,8 @@ #include #include +#include "tile_path.h" + class TileProvider { public: virtual ~TileProvider() = default; @@ -52,16 +54,13 @@ class StaticTileProvider : public TileProvider { public: std::unordered_map tiles; - StaticTileProvider(const std::unordered_map& tiles) { - // TODO: remove this once the == operator of tile::Id is updated. - for (const auto& tile : tiles) { - const radix::tile::Id tile_id = tile.first.to(radix::tile::Scheme::SlippyMap); - this->tiles[tile_id] = tile.second; - } + StaticTileProvider(const std::unordered_map& tiles) + : tiles(tiles) + { } virtual std::optional get_tile(const radix::tile::Id tile_id) const override { - const auto tile = this->tiles.find(tile_id.to(radix::tile::Scheme::SlippyMap)); + const auto tile = this->tiles.find(tile_id); if (tile != this->tiles.end()) { return tile->second; } else { @@ -70,7 +69,7 @@ class StaticTileProvider : public TileProvider { } virtual bool has_tile(const radix::tile::Id tile_id) const override { - return this->tiles.find(tile_id.to(radix::tile::Scheme::SlippyMap)) != this->tiles.cend(); + return this->tiles.find(tile_id) != this->tiles.cend(); } }; @@ -139,13 +138,13 @@ class ZoomRangeTileProvider final : public TileProvider { uint32_t _max_zoom; }; -class BasemapSchemeTilePathProvider : public TilePathProvider { +class GoogleMapboxTilePathProvider : public TilePathProvider { public: - BasemapSchemeTilePathProvider(std::filesystem::path base_path) + GoogleMapboxTilePathProvider(std::filesystem::path base_path) : base_path(base_path) {} std::optional get_tile_path(const radix::tile::Id tile_id) const override { - return base_path / std::to_string(tile_id.zoom_level) / std::to_string(tile_id.coords.y) / (std::to_string(tile_id.coords.x) + ".jpeg"); + return google_tile_path(base_path, tile_id, ".jpeg"); } private: diff --git a/src/sf_index_browser/main.cpp b/src/sf_index_browser/main.cpp index 74c8f304..f1008ff3 100644 --- a/src/sf_index_browser/main.cpp +++ b/src/sf_index_browser/main.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -12,16 +13,16 @@ #include "log.h" #include "octree/Id.h" -#include "octree/NodeStatus.h" -#include "octree/NodeStatusOrMissing.h" -#include "octree/Storage.h" -#include "octree/traverse.h" +#include "mesh/storage.h" +#include "octree/storage/open.h" #include "cli.h" +#include "store/describe_error.h" +#include "store/NodeStatusOrMissing.h" namespace { struct IndexNode { octree::Id id; - octree::NodeStatusOrMissing status; + store::NodeStatusOrMissing status; bool expanded = false; }; struct MeshNode { @@ -41,8 +42,9 @@ class TreeView { using iterator = std::list::iterator; using const_iterator = std::list::const_iterator; - explicit TreeView(const octree::IndexedStorage &storage, const octree::Id root) : root(root), _storage(storage) { - const octree::NodeStatusOrMissing status = this->_storage.index().get(root); + explicit TreeView(const mesh::storage::IndexedStorage &storage, const octree::Id root) : root(root), _storage(storage) { + const store::NodeStatusOrMissing status( + DEBUG_ASSERT_VAL(this->_storage.index().get(root)).value()); this->_view.emplace_back(IndexNode{root, status, false}); } @@ -78,14 +80,14 @@ class TreeView { } parent.expanded = true; - if (parent.status == octree::NodeStatusOrMissing::Leaf) { + if (parent.status == store::NodeStatusOrMissing::Leaf) { auto result = this->_storage.load(parent.id); DisplayEntry entry; if (result.has_value()) { const auto &mesh = result.value(); entry = MeshNode{parent.id, mesh.vertex_count(), mesh.face_count()}; } else { - entry = ErrorNode{parent.id, result.error().description()}; + entry = ErrorNode{parent.id, store::describe_error(result.error())}; } it++; it = this->_view.emplace(it, entry); @@ -97,11 +99,12 @@ class TreeView { } const auto children = parent.id.children().value(); for (const auto &child_id : children) { - auto status_opt = this->_storage.index().get(child_id); - if (!status_opt.has_value()) { + auto status_result = this->_storage.index().get(child_id); + DEBUG_ASSERT(status_result.has_value()); + if (!status_result->has_value()) { continue; } - const octree::NodeStatus status = status_opt.value(); + const store::NodeStatus status = status_result->value(); IndexNode child_node{child_id, status, false}; it++; it = this->_view.emplace(it, child_node); @@ -123,7 +126,7 @@ class TreeView { } parent.expanded = false; - if (parent.status == octree::NodeStatusOrMissing::Leaf) { + if (parent.status == store::NodeStatusOrMissing::Leaf) { it++; it = this->_view.erase(it); return; @@ -145,14 +148,16 @@ class TreeView { private: std::list _view; - const octree::IndexedStorage& _storage; + const mesh::storage::IndexedStorage& _storage; }; -const octree::Id find_deepest_root(const octree::IndexMap &index, const octree::Id &root = octree::Id::root()) { - switch (octree::NodeStatusOrMissing(index.get(root))) { - case octree::NodeStatusOrMissing::Missing: +const octree::Id find_deepest_root( + const store::Index &index, + const octree::Id &root = octree::Id::root()) { + switch (store::NodeStatusOrMissing(DEBUG_ASSERT_VAL(index.get(root)).value())) { + case store::NodeStatusOrMissing::Missing: return octree::Id::root(); - case octree::NodeStatusOrMissing::Leaf: + case store::NodeStatusOrMissing::Leaf: return root; default: break; @@ -163,7 +168,8 @@ const octree::Id find_deepest_root(const octree::IndexMap &index, const octree:: const auto children = current.children().value(); std::optional next; for (const octree::Id &child_id : children) { - if (!index.get(child_id).has_value()) { + const auto status = DEBUG_ASSERT_VAL(index.get(child_id)).value(); + if (!status.has_value()) { continue; } if (next.has_value()) { @@ -183,12 +189,17 @@ const octree::Id find_deepest_root(const octree::IndexMap &index, const octree:: } // namespace -octree::IndexedStorage open_path_indexed(const std::filesystem::path& path) { - if (std::filesystem::is_directory(path)) { - return octree::open_folder_indexed(path); - } else { - return octree::open_index(path).value(); +mesh::storage::IndexedStorage open_path_indexed(const std::filesystem::path& path) { + auto result = std::filesystem::is_directory(path) + ? octree::open_folder_indexed(path) + : octree::open_index(path); + if (!result.has_value()) { + LOG_ERROR_AND_EXIT( + "Failed to open dataset {}: {}", + path, + store::describe_error(result.error())); } + return std::move(result.value()); } std::string render_line_content(const DisplayEntry &entry) { @@ -219,8 +230,8 @@ ftxui::Element render_line(const DisplayEntry &entry, size_t base_level, bool is } int run(const cli::Args &args) { - const octree::IndexedStorage storage = open_path_indexed(args.dataset_path); - const octree::IndexMap &index = storage.index(); + const mesh::storage::IndexedStorage storage = open_path_indexed(args.dataset_path); + const store::Index &index = storage.index(); const octree::Id root_id = args.full_view ? octree::Id::root() : find_deepest_root(index); TreeView tree_view(storage, root_id); diff --git a/src/sf_merger/NodeLoader.h b/src/sf_merger/NodeLoader.h index c6ac4027..23a2cbd1 100644 --- a/src/sf_merger/NodeLoader.h +++ b/src/sf_merger/NodeLoader.h @@ -9,23 +9,22 @@ #include "mesh/clip.h" #include "mesh/texture_trim.h" #include "octree/Id.h" -#include "octree/NodeStatusOrMissing.h" #include "octree/Space.h" -#include "octree/storage/IndexedStorage.h" -#include "octree/storage/cache/Dummy.h" -#include "octree/storage/cache/ICache.h" +#include "mesh/storage.h" +#include "store/NodeStatusOrMissing.h" +#include "store/cache/Interface.h" class NodeLoader { public: - NodeLoader(const octree::IndexedStorage &storage, octree::cache::ICache &cache, octree::Space space) - : _storage(storage), _cache(cache), _space(space) { - if (storage.cache().has_value() && dynamic_cast *>(&cache) != nullptr) { - LOG_WARN("Backing storage for NodeLoader instance is cached, but another cache was provided leading to double caching."); - } - } + NodeLoader( + const mesh::storage::IndexedStorage &storage, + store::cache::Interface &cache, + octree::Space space) + : _storage(storage), _cache(cache), _space(space) {} - octree::NodeStatusOrMissing get_status(const octree::Id &id) const noexcept { - return this->_storage.index().get(id); + store::NodeStatusOrMissing get_status(const octree::Id &id) const noexcept { + return store::NodeStatusOrMissing( + DEBUG_ASSERT_VAL(this->_storage.index().get(id)).value()); } // Try to retrieve or reconstruct node mesh by ID @@ -69,12 +68,12 @@ class NodeLoader { return std::nullopt; } - const octree::IndexedStorage &storage() const { + const mesh::storage::IndexedStorage &storage() const { return this->_storage; } private: - const octree::IndexedStorage &_storage; - octree::cache::ICache &_cache; + const mesh::storage::IndexedStorage &_storage; + store::cache::Interface &_cache; const octree::Space _space; }; diff --git a/src/sf_merger/NodeWriter.h b/src/sf_merger/NodeWriter.h index f160a8ad..6a29a556 100644 --- a/src/sf_merger/NodeWriter.h +++ b/src/sf_merger/NodeWriter.h @@ -1,49 +1,78 @@ #pragma once +#include +#include + #include #include "NodeLoader.h" #include "mesh/SimpleMesh.h" #include "octree/Id.h" -#include "octree/NodeStatus.h" -#include "octree/storage/Storage.h" -#include "octree/traverse.h" +#include "mesh/storage.h" +#include "store/traverse.h" // TODO: make thread safe class NodeWriter { public: - NodeWriter(octree::Storage &storage) : _storage(storage) {} + NodeWriter(mesh::storage::Storage &storage) : _storage(storage) {} - bool has_node(const octree::Id &id) { + std::expected> has_node( + const octree::Id &id) { return this->_storage.has(id); } - void write_node(const octree::Id &id, const SimpleMesh &mesh) { + std::expected> write_node( + const octree::Id &id, + const SimpleMesh &mesh) { mesh::validate(mesh); - DEBUG_ASSERT_VAL(this->_storage.save(id, mesh)); - auto p = this->_storage.path_for(id); + const auto save_result = this->_storage.save(id, mesh); + if (!save_result.has_value()) { + return save_result; + } + const auto path_result = this->_storage.path_for(id); + if (!path_result.has_value()) { + return std::unexpected(store::SaveError(path_result.error())); + } + auto p = path_result.value(); // change extension to .png p.replace_extension(".png"); - cv::imwrite(p, mesh.texture.value_or(cv::Mat())); + if (mesh.texture.has_value()) { + cv::imwrite(p, mesh.texture.value()); + } + return {}; } - void copy_subtree_to_output( + std::expected> copy_subtree_to_output( const octree::Id &id, const NodeLoader &loader) { - octree::traverse( + std::optional> error; + const auto traversal = store::traverse( loader.storage().index(), - [&](const octree::Id &child_id, const octree::NodeStatus &status) { - if (status == octree::NodeStatus::Virtual) { + [&](const octree::Id &child_id, const store::NodeStatus &status) { + if (error.has_value()) { return; } - DEBUG_ASSERT(status == octree::NodeStatus::Leaf); + if (status == store::NodeStatus::Virtual) { + return; + } + DEBUG_ASSERT(status == store::NodeStatus::Leaf); - DEBUG_ASSERT_VAL(this->_storage.copy_from(child_id, loader.storage())); + const auto result = this->_storage.copy_from(child_id, loader.storage()); + if (!result.has_value()) { + error = result.error(); + } }, - octree::always_refine, + [&](const octree::Id &) { return !error.has_value(); }, id); + if (!traversal.has_value()) { + return std::unexpected(store::CopyError(traversal.error())); + } + if (error.has_value()) { + return std::unexpected(std::move(error.value())); + } + return {}; } private: - octree::Storage &_storage; + mesh::storage::Storage &_storage; }; diff --git a/src/sf_merger/cut.h b/src/sf_merger/cut.h index 6f8e4b40..9a8d56a1 100644 --- a/src/sf_merger/cut.h +++ b/src/sf_merger/cut.h @@ -1,37 +1,42 @@ #pragma once +#include + #include #include "mesh/SimpleMesh.h" #include "log.h" #include "mask.h" #include "octree/Id.h" -#include "octree/NodeStatus.h" -#include "octree/storage/IndexedStorage.h" +#include "mesh/storage.h" #include "octree/storage/open.h" #include "octree/Space.h" -#include "octree/traverse.h" #include "utils.h" #include "containers/Cow.h" +#include "store/describe_error.h" +#include "sf/Error.h" +#include "sf/finalize_storage.h" +#include "sf/validate_index.h" struct Context { - const octree::IndexedStorage& input; - octree::Storage& output; + const mesh::storage::IndexedStorage& input; + mesh::storage::Storage& output; const octree::Space space; const bool keep_inside; }; -void cut_node( +std::expected cut_node( Context& ctx, const octree::Id& id, const MeshMask& mask); -inline void cut_leaf_node( +inline std::expected cut_leaf_node( Context& ctx, const octree::Id& id, const MeshMask& mask ) { - DEBUG_ASSERT(ctx.input.index().is(octree::NodeStatus::Leaf, id)); + DEBUG_ASSERT(DEBUG_ASSERT_VAL( + ctx.input.index().is(store::NodeStatus::Leaf, id)).value()); const SimpleMesh mesh = DEBUG_ASSERT_VAL(ctx.input.load(id)).value(); LOG_TRACE("Cutting mesh at {} using mask with {} vertices and {} triangles", @@ -39,25 +44,33 @@ inline void cut_leaf_node( const Cow clipped = clip_on_mask(mesh, mask, ctx.keep_inside); if (clipped.is_ref()) { LOG_TRACE("Mesh was fully inside the mask"); - DEBUG_ASSERT_VAL(ctx.output.copy_from(id, ctx.input)); + const auto copy_result = ctx.output.copy_from(id, ctx.input); + if (!copy_result.has_value()) { + return std::unexpected(sf::ProcessingError(copy_result.error())); + } } else { const SimpleMesh &clipped_mesh = clipped; if (!clipped_mesh.is_empty()) { LOG_TRACE("Mesh was clipped from {} vertices and {} triangles to {} vertices and {} triangles", mesh.vertex_count(), mesh.face_count(), clipped_mesh.vertex_count(), clipped_mesh.face_count()); - DEBUG_ASSERT_VAL(ctx.output.save(id, clipped_mesh)); + const auto save_result = ctx.output.save(id, clipped_mesh); + if (!save_result.has_value()) { + return std::unexpected(sf::ProcessingError(save_result.error())); + } } else { LOG_TRACE("Mesh was fully outside the mask"); } } + return {}; } -inline void cut_virtual_node( +inline std::expected cut_virtual_node( Context& ctx, const octree::Id& id, const MeshMask& mask ) { - DEBUG_ASSERT(ctx.input.index().is(octree::NodeStatus::Virtual, id)); + DEBUG_ASSERT(DEBUG_ASSERT_VAL( + ctx.input.index().is(store::NodeStatus::Virtual, id)).value()); DEBUG_ASSERT(id.has_children()); const auto children = id.children().value(); @@ -66,18 +79,22 @@ inline void cut_virtual_node( const auto child_bounds = geometry::pad_bounds_relative(ctx.space.get_node_bounds(child_id), 0.125); LOG_TRACE("Clipping mask for {}", child_id); const MeshMask child_mask = clip_mask_on_bounds(mask, child_bounds); - cut_node(ctx, child_id, child_mask); + const auto child_result = cut_node(ctx, child_id, child_mask); + if (!child_result.has_value()) { + return child_result; + } } + return {}; } -inline void cut_node( +inline std::expected cut_node( Context& ctx, const octree::Id& id, const MeshMask& mask ) { if (ctx.keep_inside && mask.is_empty()) { LOG_TRACE("Mesh was fully outside the mask"); - return; + return {}; } /* @@ -88,47 +105,68 @@ inline void cut_node( mesh::io::save_to_path(mask.mesh, new_path); */ - const auto status_opt = ctx.input.index().get(id); - if (!status_opt.has_value()) { - return; + const auto status_result = ctx.input.index().get(id); + DEBUG_ASSERT(status_result.has_value()); + if (!status_result->has_value()) { + return {}; } - const octree::NodeStatus status = status_opt.value(); + const store::NodeStatus status = status_result->value(); switch (status) { - case octree::NodeStatus::Virtual: - cut_virtual_node(ctx, id, mask); - break; - case octree::NodeStatus::Leaf: - cut_leaf_node(ctx, id, mask); - break; + case store::NodeStatus::Virtual: + return cut_virtual_node(ctx, id, mask); + case store::NodeStatus::Leaf: + return cut_leaf_node(ctx, id, mask); default: UNREACHABLE(); - break; + return {}; } + return {}; } -inline void cut_dataset( - const octree::IndexedStorage &input, +inline std::expected cut_dataset( + const mesh::storage::IndexedStorage &input, const MeshMask& mask, - octree::Storage &output, + mesh::storage::Storage &output, const bool keep_inside) { + const auto validation = sf::validate_index(input.index()); + if (!validation.has_value()) { + return std::unexpected(sf::ProcessingError(validation.error())); + } Context ctx(input, output, octree::Space::earth(), keep_inside); - cut_node(ctx, octree::Id::root(), mask); - output.save_or_create_index(); + const auto cut_result = cut_node(ctx, octree::Id::root(), mask); + if (!cut_result.has_value()) { + return cut_result; + } + const auto finalization = sf::finalize_storage(output); + if (!finalization.has_value()) { + return std::unexpected(std::visit( + [](const auto &error) -> sf::ProcessingError { return error; }, + finalization.error())); + } + return {}; } -inline void cut_dataset( - const octree::IndexedStorage &input_dataset, +inline std::expected cut_dataset( + const mesh::storage::IndexedStorage &input_dataset, const MeshMask& mask, const std::filesystem::path &output_path, const bool keep_inside) { LOG_TRACE("Creating output dataset at {}", output_path); std::filesystem::create_directories(output_path); - octree::IndexedStorage output_dataset = octree::open_folder_indexed(output_path, octree::OpenOptions(octree::disk::layout::strategy::make_default(), ".glb")); + octree::OpenOptions options; + options.default_mapping = octree::store_layout::level_and_coordinate_directories(); + options.preferred_extension = ".glb"; + auto output_result = octree::open_folder_indexed(output_path, std::move(options)); + if (!output_result.has_value()) { + return std::unexpected(sf::ProcessingError(output_result.error())); + } + mesh::storage::IndexedStorage output_dataset = std::move(output_result.value()); if (!output_dataset.index().empty()) { - LOG_ERROR_AND_EXIT("Output dataset has to be empty."); + return std::unexpected(sf::ProcessingError(store::SaveError( + store::AlreadyExists{output_path}))); } - cut_dataset(input_dataset, mask, output_dataset, keep_inside); + return cut_dataset(input_dataset, mask, output_dataset, keep_inside); } diff --git a/src/sf_merger/main.cpp b/src/sf_merger/main.cpp index 37e3f3dc..9fa72497 100644 --- a/src/sf_merger/main.cpp +++ b/src/sf_merger/main.cpp @@ -8,9 +8,10 @@ #include "log.h" #include "mask.h" #include "merge.h" -#include "octree/Storage.h" #include "optional_utils.h" #include "earth.h" +#include "store/describe_error.h" +#include "sf/Error.h" std::optional load_mask_from_path(const std::filesystem::path& path) { if (std::filesystem::exists(path)) { @@ -34,25 +35,70 @@ std::optional load_mask_from_path(const std::filesystem::path& path) { void run(const cli::MergeArgs& args) { LOG_TRACE("Loading base dataset from {}", args.base_path); - octree::IndexedStorage base_dataset = octree::open_folder_indexed(args.base_path); + auto base_result = octree::open_folder_indexed(args.base_path); + if (!base_result.has_value()) { + LOG_ERROR_AND_EXIT( + "Failed to open base dataset {}: {}", + args.base_path, + store::describe_error(base_result.error())); + } + mesh::storage::IndexedStorage base_dataset = std::move(base_result.value()); LOG_TRACE("Loading new dataset from {}", args.new_path); - octree::IndexedStorage new_dataset = octree::open_folder_indexed(args.new_path); + auto new_result = octree::open_folder_indexed(args.new_path); + if (!new_result.has_value()) { + LOG_ERROR_AND_EXIT( + "Failed to open new dataset {}: {}", + args.new_path, + store::describe_error(new_result.error())); + } + mesh::storage::IndexedStorage new_dataset = std::move(new_result.value()); LOG_TRACE("Creating output dataset at {}", args.output_path); std::filesystem::create_directories(args.output_path); - octree::Storage output_dataset = octree::open_folder(args.output_path, false, octree::OpenOptions{.preferred_extension_with_dot = std::string(base_dataset.layout().extension_with_dot())}); + octree::OpenOptions options; + options.default_mapping = base_dataset.layout().mapping(); + options.preferred_extension = std::string(base_dataset.codec_selector().value_or(".sfmesh")); + auto output_result = octree::open_folder(args.output_path, std::move(options)); + if (!output_result.has_value()) { + LOG_ERROR_AND_EXIT( + "Failed to open output dataset {}: {}", + args.output_path, + store::describe_error(output_result.error())); + } + mesh::storage::Storage output_dataset = std::move(output_result.value()); std::optional mask = flatten(map(args.mask_path, load_mask_from_path)); - return merge_datasets(base_dataset, new_dataset, output_dataset, mask); + const auto merge_result = merge_datasets( + base_dataset, + new_dataset, + output_dataset, + mask); + if (!merge_result.has_value()) { + LOG_ERROR_AND_EXIT("Failed to merge datasets: {}", sf::describe_error(merge_result.error())); + } } void run(const cli::CutArgs& args) { LOG_TRACE("Loading input dataset from {}", args.input_path); - const octree::IndexedStorage input_dataset = octree::open_folder_indexed(args.input_path); + auto input_result = octree::open_folder_indexed(args.input_path); + if (!input_result.has_value()) { + LOG_ERROR_AND_EXIT( + "Failed to open input dataset {}: {}", + args.input_path, + store::describe_error(input_result.error())); + } + const mesh::storage::IndexedStorage input_dataset = std::move(input_result.value()); const MeshMask mask = DEBUG_ASSERT_VAL(load_mask_from_path(args.mask_path)).value(); - cut_dataset(input_dataset, mask, args.output_path, args.keep_inside); + const auto cut_result = cut_dataset( + input_dataset, + mask, + args.output_path, + args.keep_inside); + if (!cut_result.has_value()) { + LOG_ERROR_AND_EXIT("Failed to cut dataset: {}", sf::describe_error(cut_result.error())); + } } void run(const cli::Args &args) { diff --git a/src/sf_merger/mask.h b/src/sf_merger/mask.h index fa6cd8af..8ceb8900 100644 --- a/src/sf_merger/mask.h +++ b/src/sf_merger/mask.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -91,7 +92,9 @@ namespace { enum class LoadErrorKind { UnsupportedFormat, FileNotFound, - EmptySource + EmptySource, + InvalidGeometry, + UnsupportedSpatialReference }; class LoadError { @@ -118,6 +121,10 @@ class LoadError { return "file not found"; case LoadErrorKind::EmptySource: return "empty input source"; + case LoadErrorKind::InvalidGeometry: + return "invalid geometry"; + case LoadErrorKind::UnsupportedSpatialReference: + return "unsupported spatial reference"; default: return "unknown error"; } @@ -132,6 +139,58 @@ class LoadError { }; namespace { +constexpr double simplification_tolerance_metres = 0.1; + +std::optional simplification_tolerance(const OGRSpatialReference &srs) { + if (srs.IsProjected()) { + const double metres_per_unit = srs.GetLinearUnits(); + if (metres_per_unit > 0) { + return simplification_tolerance_metres / metres_per_unit; + } + } + + if (srs.IsGeographic()) { + OGRErr error = OGRERR_NONE; + const double semi_major_axis_metres = srs.GetSemiMajor(&error); + const double radians_per_unit = srs.GetAngularUnits(); + if (error == OGRERR_NONE && semi_major_axis_metres > 0 && radians_per_unit > 0) { + return simplification_tolerance_metres / semi_major_axis_metres / radians_per_unit; + } + } + + return std::nullopt; +} + +uint64_t point_count(const OGRGeometry &geometry) { + const OGRwkbGeometryType geometry_type = wkbFlatten(geometry.getGeometryType()); + if (geometry_type == wkbPolygon) { + const OGRPolygon *polygon = geometry.toPolygon(); + uint64_t count = polygon->getExteriorRing()->getNumPoints(); + for (int i = 0; i < polygon->getNumInteriorRings(); ++i) { + count += polygon->getInteriorRing(i)->getNumPoints(); + } + return count; + } + + if (geometry_type == wkbMultiPolygon || geometry_type == wkbGeometryCollection) { + const OGRGeometryCollection *collection = geometry.toGeometryCollection(); + uint64_t count = 0; + for (int i = 0; i < collection->getNumGeometries(); ++i) { + count += point_count(*collection->getGeometryRef(i)); + } + return count; + } + + return 0; +} + +std::unique_ptr simplify_geometry( + const OGRGeometry &geometry, + const double tolerance +) { + return std::unique_ptr(geometry.SimplifyPreserveTopology(tolerance)); +} + std::optional convert_ring(const OGRLinearRing &ring, bool is_outer) { uint32_t num_points = ring.getNumPoints(); if (ring.get_IsClosed()) { @@ -272,21 +331,9 @@ glm::dvec2 calculate_radius_range(const std::span meshes) { } // namespace -inline tl::expected load_referenced_from_dataset(Dataset& mask_dataset) { +inline std::expected load_referenced_from_dataset(Dataset& mask_dataset) { GDALDataset *dataset = mask_dataset.gdalDataset(); - MultipolygonWithHoles2 polygons; - for (auto &&feature_layer_pair : dataset->GetFeatures()) { - OGRGeometry *geometry = feature_layer_pair.feature->GetGeometryRef(); - DEBUG_ASSERT(geometry); - process_geometry(*geometry, polygons); - } - - if (polygons.is_empty()) { - LOG_ERROR("No valid polygons found in mask dataset '{}'", mask_dataset.name()); - return tl::unexpected(LoadErrorKind::EmptySource); - } - OGRSpatialReference srs; // TODO: remove this try catch try { @@ -297,6 +344,35 @@ inline tl::expected load_referenced_from_datas // srs.SetAxisMappingStrategy(OAMS_AUTHORITY_COMPLIANT); } + const std::optional tolerance = simplification_tolerance(srs); + if (!tolerance) { + LOG_ERROR("Cannot express the {} m mask simplification tolerance in the source SRS", + simplification_tolerance_metres); + return std::unexpected(LoadErrorKind::UnsupportedSpatialReference); + } + + MultipolygonWithHoles2 polygons; + for (auto &&feature_layer_pair : dataset->GetFeatures()) { + OGRGeometry *geometry = feature_layer_pair.feature->GetGeometryRef(); + DEBUG_ASSERT(geometry); + + const uint64_t original_point_count = point_count(*geometry); + std::unique_ptr simplified = simplify_geometry(*geometry, *tolerance); + if (!simplified || simplified->IsEmpty() || !simplified->IsValid()) { + LOG_ERROR("Failed to simplify mask geometry while preserving its topology"); + return std::unexpected(LoadErrorKind::InvalidGeometry); + } + + LOG_DEBUG("Simplified mask with {} m tolerance from {} to {} points", + simplification_tolerance_metres, original_point_count, point_count(*simplified)); + process_geometry(*simplified, polygons); + } + + if (polygons.is_empty()) { + LOG_ERROR("No valid polygons found in mask dataset '{}'", mask_dataset.name()); + return std::unexpected(LoadErrorKind::EmptySource); + } + return ReferencedPolygonMask{.polygons = std::move(polygons), .srs = std::move(srs)}; } @@ -472,25 +548,25 @@ inline MeshMask extrude( return extrude(mask, padded_radius_range); } -inline tl::expected load_referenced_from_path(const std::filesystem::path &path) { +inline std::expected load_referenced_from_path(const std::filesystem::path &path) { if (!std::filesystem::exists(path)) { LOG_ERROR("Mask file does not exist: {}", path); - return tl::unexpected(LoadErrorKind::FileNotFound); + return std::unexpected(LoadErrorKind::FileNotFound); } auto ds_opt = Dataset::open_vector(path); if (!ds_opt.has_value()) { LOG_ERROR("Failed to load mask datset: {}", path); - return tl::unexpected(LoadErrorKind::FileNotFound); + return std::unexpected(LoadErrorKind::FileNotFound); } Dataset dataset = std::move(ds_opt.value()); return load_referenced_from_dataset(dataset); } -inline tl::expected load_from_path(const std::filesystem::path &path, const glm::dvec2& radius_range) { +inline std::expected load_from_path(const std::filesystem::path &path, const glm::dvec2& radius_range) { auto ref_mask_res = load_referenced_from_path(path); if (!ref_mask_res.has_value()) { - return tl::unexpected(ref_mask_res.error()); + return std::unexpected(ref_mask_res.error()); } ReferencedPolygonMask ref_polygon_mask = std::move(ref_mask_res.value()); SpherePolygonMask sphere_polygon_mask = project_onto_sphere(std::move(ref_polygon_mask), radius_range.x); diff --git a/src/sf_merger/merge.h b/src/sf_merger/merge.h index ea3a1b18..bef384ac 100644 --- a/src/sf_merger/merge.h +++ b/src/sf_merger/merge.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -12,10 +13,15 @@ #include "merge/visitor/Simple.h" #include "merge/visitor/Visitor.h" #include "octree/Id.h" -#include "octree/NodeStatusOrMissing.h" -#include "octree/storage/cache/Dummy.h" - -inline std::string get_dataset_name(const octree::Storage &storage) { +#include "mesh/storage.h" +#include "store/NodeStatusOrMissing.h" +#include "store/cache/Dummy.h" +#include "store/describe_error.h" +#include "sf/Error.h" +#include "sf/finalize_storage.h" +#include "sf/validate_index.h" + +inline std::string get_dataset_name(const mesh::storage::Storage &storage) { return storage.layout().base_path().filename().string(); } @@ -29,9 +35,10 @@ using smallest_uint_t = template class Merger { public: - using Status = octree::NodeStatusOrMissing; + using Status = store::NodeStatusOrMissing; using Context = Visitor::Context; using Result = merge::Result; + using Expected = std::expected; Merger( Visitor& visitor, @@ -40,62 +47,86 @@ class Merger { NodeWriter output) : _visitor(visitor), _left(left), _right(right), _output(output) { } - void merge_root() { + Expected merge_root() { const octree::Id id = octree::Id::root(); const Context ctx = this->_visitor.make_root_context(); - merge_node(id, ctx); + return merge_node(id, ctx); } - void merge_node(const octree::Id &id, const Context& ctx) { + Expected merge_node(const octree::Id &id, const Context& ctx) { const Status left_status = this->_left.get_status(id); const Status right_status = this->_right.get_status(id); return this->merge_node(id, left_status, right_status, ctx); } - void merge_node( + Expected merge_node( const octree::Id &id, const Status left_status, const Status right_status, const Context& ctx ) { LOG_DEBUG("[{}] Start merging (left = {}, right = {})", id, left_status, right_status); - if (this->_output.has_node(id)) { + const auto has_result = this->_output.has_node(id); + if (!has_result.has_value()) { + return std::unexpected(sf::ProcessingError(has_result.error())); + } + if (has_result.value()) { LOG_DEBUG("[{}] Already merged, skipping...", id); - return; // Already merged previously + return {}; // Already merged previously } const auto merge_result = this->call_merge(id, left_status, right_status, ctx); - std::visit([&](const auto &result) { + return std::visit([&](const auto &result) -> Expected { using Result = std::decay_t; if constexpr (std::is_same_v>) { LOG_DEBUG("[{}] needs recursion", id); DEBUG_ASSERT(id.has_children()); const auto children = id.children().value(); for (const auto &child_id : children) { - this->merge_node(child_id, result.context); + const auto child_result = this->merge_node(child_id, result.context); + if (!child_result.has_value()) { + return child_result; + } } + return {}; } else if constexpr (std::is_same_v) { LOG_DEBUG("[{}] remains unchanged (same as {})", id, (result.source == merge::Source::Left ? "left" : "right")); // If the node should remain unchanged, but its not present on disk we cannot copy it. if (result.source == merge::Source::Left && left_status == Status::Missing) { auto mesh_opt = this->_left.load_node(id); if (mesh_opt.has_value()) { - this->_output.write_node(id, mesh_opt.value()); + const auto write_result = this->_output.write_node(id, mesh_opt.value()); + if (!write_result.has_value()) { + return std::unexpected(sf::ProcessingError(write_result.error())); + } } } else if (result.source == merge::Source::Right && right_status == Status::Missing) { auto mesh_opt = this->_right.load_node(id); if (mesh_opt.has_value()) { - this->_output.write_node(id, mesh_opt.value()); + const auto write_result = this->_output.write_node(id, mesh_opt.value()); + if (!write_result.has_value()) { + return std::unexpected(sf::ProcessingError(write_result.error())); + } } } else { - this->_output.copy_subtree_to_output(id, result.source == merge::Source::Left ? this->_left : this->_right); + const auto copy_result = this->_output.copy_subtree_to_output( + id, + result.source == merge::Source::Left ? this->_left : this->_right); + if (!copy_result.has_value()) { + return std::unexpected(sf::ProcessingError(copy_result.error())); + } } + return {}; } else if constexpr (std::is_same_v) { LOG_DEBUG("[{}] was merged", id); - this->_output.write_node(id, result.mesh); + const auto write_result = this->_output.write_node(id, result.mesh); + if (!write_result.has_value()) { + return std::unexpected(sf::ProcessingError(write_result.error())); + } + return {}; } else if constexpr (std::is_same_v) { LOG_DEBUG("[{}] was ignored", id); - // do nothing + return {}; } }, merge_result); } @@ -151,31 +182,50 @@ class Merger { NodeWriter _output; }; -inline void merge_datasets( - const octree::IndexedStorage &left_dataset, - const octree::IndexedStorage &right_dataset, - octree::Storage &output_dataset, +inline std::expected merge_datasets( + const mesh::storage::IndexedStorage &left_dataset, + const mesh::storage::IndexedStorage &right_dataset, + mesh::storage::Storage &output_dataset, const std::optional mask = std::nullopt) { + for (const mesh::storage::IndexedStorage *input : {&left_dataset, &right_dataset}) { + const auto validation = sf::validate_index(input->index()); + if (!validation.has_value()) { + return std::unexpected(sf::ProcessingError(validation.error())); + } + } + LOG_TRACE("Merging {} and {} into {}", get_dataset_name(left_dataset), get_dataset_name(right_dataset), get_dataset_name(output_dataset)); octree::Space space = octree::Space::earth(); - octree::cache::Dummy left_cache; + store::cache::Dummy left_cache; NodeLoader left(left_dataset, left_cache, space); - octree::cache::Dummy right_cache; + store::cache::Dummy right_cache; NodeLoader right(right_dataset, right_cache, space); NodeWriter output(output_dataset); if (mask.has_value()) { merge::visitor::Masked visitor {mask.value(), space}; Merger merger(visitor, left, right, output); - merger.merge_root(); + const auto result = merger.merge_root(); + if (!result.has_value()) { + return result; + } } else { merge::visitor::Simple visitor; Merger merger(visitor, left, right, output); - merger.merge_root(); + const auto result = merger.merge_root(); + if (!result.has_value()) { + return result; + } } - output_dataset.save_or_create_index(); + const auto finalization = sf::finalize_storage(output_dataset); + if (!finalization.has_value()) { + return std::unexpected(std::visit( + [](const auto &error) -> sf::ProcessingError { return error; }, + finalization.error())); + } + return {}; } diff --git a/src/sf_merger/merge/NodeData.h b/src/sf_merger/merge/NodeData.h index c3c9de13..2bdd72f1 100644 --- a/src/sf_merger/merge/NodeData.h +++ b/src/sf_merger/merge/NodeData.h @@ -6,18 +6,17 @@ #include "log.h" #include "mesh/SimpleMesh.h" #include "octree/Id.h" -#include "octree/NodeStatusOrMissing.h" -#include "octree/Storage.h" #include "optional_utils.h" +#include "store/NodeStatusOrMissing.h" namespace merge { -template +template class NodeData { public: constexpr explicit NodeData(octree::Id id, const NodeLoader &loader) : _id(id), _loader(loader) {} - static constexpr octree::NodeStatusOrMissing status() { + static constexpr store::NodeStatusOrMissing status() { return Status; } @@ -26,8 +25,8 @@ class NodeData { } // Status::Leaf or Status::Inner guarantuess a node is present - template - std::enable_if_t + template + std::enable_if_t mesh() const { auto mesh = this->load_mesh(); if (mesh.has_value()) { @@ -38,8 +37,8 @@ class NodeData { } // Status::Missing and Status::Virtual do not exist on disk, but may be the child of a leaf or inner node - template - std::enable_if_t>> + template + std::enable_if_t>> mesh() const { return this->load_mesh(); } diff --git a/src/sf_merger/merge/visitor/Masked.h b/src/sf_merger/merge/visitor/Masked.h index 28fc8508..0a78b37b 100644 --- a/src/sf_merger/merge/visitor/Masked.h +++ b/src/sf_merger/merge/visitor/Masked.h @@ -11,7 +11,7 @@ #include "mesh/merging/helpers.h" #include "mesh/merging/mapping.h" #include "octree/Id.h" -#include "octree/NodeStatusOrMissing.h" +#include "store/NodeStatusOrMissing.h" #include "polygon/Polygon.h" #include "polygon/triangulate.h" #include "spatial_lookup/Hashmap.h" @@ -26,7 +26,7 @@ namespace merge::visitor { class Masked { public: - using Status = octree::NodeStatusOrMissing; + using Status = store::NodeStatusOrMissing; struct Context { MeshMask mask; bool has_left_parent; diff --git a/src/sf_merger/merge/visitor/Simple.h b/src/sf_merger/merge/visitor/Simple.h index 531cd686..9305ce7b 100644 --- a/src/sf_merger/merge/visitor/Simple.h +++ b/src/sf_merger/merge/visitor/Simple.h @@ -5,13 +5,13 @@ #include "merge/visitor/Visitor.h" #include "mesh/merge.h" #include "octree/Id.h" -#include "octree/NodeStatusOrMissing.h" +#include "store/NodeStatusOrMissing.h" namespace merge::visitor { class Simple { public: - using Status = octree::NodeStatusOrMissing; + using Status = store::NodeStatusOrMissing; struct Context{}; using Result = merge::Result; diff --git a/src/sf_merger/merge/visitor/Visitor.h b/src/sf_merger/merge/visitor/Visitor.h index 35206fe4..6607e222 100644 --- a/src/sf_merger/merge/visitor/Visitor.h +++ b/src/sf_merger/merge/visitor/Visitor.h @@ -3,7 +3,7 @@ #include "merge/NodeData.h" #include "merge/Result.h" #include "octree/Id.h" -#include "octree/NodeStatusOrMissing.h" +#include "store/NodeStatusOrMissing.h" namespace merge { template @@ -15,10 +15,10 @@ concept Visitor = requires(T t, const octree::Id &id) { { t.make_root_context() } -> std::same_as; { - t.template visit( + t.template visit( id, - std::declval &>(), - std::declval &>(), + std::declval &>(), + std::declval &>(), std::declval()) } -> std::same_as>; }; diff --git a/src/terrainlib/CMakeLists.txt b/src/terrainlib/CMakeLists.txt index fcfa9b3a..773d223a 100644 --- a/src/terrainlib/CMakeLists.txt +++ b/src/terrainlib/CMakeLists.txt @@ -7,6 +7,8 @@ add_library(terrainlib ctb/types.hpp io/bytes.cpp + io/compression.cpp + io/conversion.h io/utils.cpp mesh/connectivity/adjacency.cpp @@ -18,7 +20,6 @@ add_library(terrainlib mesh/igl/igl.cpp mesh/io/gltf.cpp - mesh/io/terrain.cpp mesh/io/texture.cpp mesh/io/utils.cpp @@ -35,8 +36,6 @@ add_library(terrainlib mesh/texture_trim.cpp mesh/validate.cpp - octree/storage/helpers.cpp - octree/IndexMap.cpp octree/Space.cpp octree/OddLevelShifted.cpp @@ -53,6 +52,7 @@ add_library(terrainlib init.cpp log.cpp ProgressIndicator.cpp + tile_path.h ) target_compile_features(terrainlib PUBLIC cxx_std_23) @@ -74,9 +74,9 @@ target_link_libraries(terrainlib PUBLIC spdlog fmt zpp_bits + zstd::libzstd_static cgltf TBB::tbb - tl_expected opencv_core opencv_imgproc opencv_imgcodecs diff --git a/src/terrainlib/ProgressIndicator.cpp b/src/terrainlib/ProgressIndicator.cpp index 565f7ee8..a9f49272 100644 --- a/src/terrainlib/ProgressIndicator.cpp +++ b/src/terrainlib/ProgressIndicator.cpp @@ -39,6 +39,7 @@ void ProgressIndicator::task_finished() { if (m_step > m_n_steps) { throw std::runtime_error("Too many steps reported."); } + m_monitor_condition.notify_all(); } std::jthread ProgressIndicator::start_monitoring() const { @@ -50,19 +51,30 @@ std::jthread ProgressIndicator::start_monitoring() const { }; const auto t0 = std::chrono::steady_clock::now(); - std::jthread thread([=, this]() { - const auto delta_t = 500ms; - auto v_t_minus_1 = this->m_step.load(); - do { - auto v_t = this->m_step.load(); - const auto delta_v = v_t - v_t_minus_1; - v_t_minus_1 = v_t; - print(delta_v, delta_t); - std::this_thread::sleep_for(delta_t); - } while (this->m_step < this->m_n_steps); - const auto t1 = std::chrono::steady_clock::now(); - print(this->m_n_steps, std::chrono::duration_cast(t1 - t0)); - std::cout << std::endl; + std::jthread thread([=, this](std::stop_token stop_token) noexcept { + try { + const auto delta_t = 500ms; + auto v_t_minus_1 = this->m_step.load(); + while (!stop_token.stop_requested() && this->m_step < this->m_n_steps) { + const auto v_t = this->m_step.load(); + const auto delta_v = v_t - v_t_minus_1; + v_t_minus_1 = v_t; + print(delta_v, delta_t); + + std::unique_lock lock(this->m_monitor_mutex); + this->m_monitor_condition.wait_for( + lock, stop_token, delta_t, [this]() { return this->m_step >= this->m_n_steps; }); + } + + if (!stop_token.stop_requested()) { + const auto t1 = std::chrono::steady_clock::now(); + print(this->m_n_steps, std::chrono::duration_cast(t1 - t0)); + std::cout << std::endl; + } + } catch (...) { + // Console output and formatting must not terminate the process from + // inside the monitoring thread. + } }); return thread; } diff --git a/src/terrainlib/ProgressIndicator.h b/src/terrainlib/ProgressIndicator.h index fcf34eb0..ca16e36e 100644 --- a/src/terrainlib/ProgressIndicator.h +++ b/src/terrainlib/ProgressIndicator.h @@ -21,12 +21,16 @@ #include #include +#include +#include #include #include class ProgressIndicator { const size_t m_n_steps; std::atomic m_step = 0; + mutable std::condition_variable_any m_monitor_condition; + mutable std::mutex m_monitor_mutex; public: ProgressIndicator(size_t n_steps); diff --git a/src/terrainlib/ctb/GlobalGeodetic.hpp b/src/terrainlib/ctb/GlobalGeodetic.hpp index 5ddeecc5..917690b9 100644 --- a/src/terrainlib/ctb/GlobalGeodetic.hpp +++ b/src/terrainlib/ctb/GlobalGeodetic.hpp @@ -28,10 +28,11 @@ namespace ctb { /** - * @brief An implementation of the TMS Global Geodetic Profile + * @brief An implementation of the global Geodetic grid profile * * This class models the [Tile Mapping Service Global Geodetic * Profile](http://wiki.osgeo.org/wiki/Tile_Map_Service_Specification#global-geodetic). + * Its public tile identifiers use Google/Mapbox/XYZ coordinates. */ class GlobalGeodetic : public Grid { public: @@ -42,7 +43,7 @@ class GlobalGeodetic : public Grid { cSRS, 4326, // global geodetic has 2 root tiles: https://wiki.osgeo.org/wiki/Tile_Map_Service_Specification#global-geodetic - std::vector{radix::tile::Id{0, {0, 0}, radix::tile::Scheme::Tms}, radix::tile::Id{0, {1, 0}, radix::tile::Scheme::Tms}}, + std::vector{radix::tile::Id{0, {0, 0}}, radix::tile::Id{0, {1, 0}}}, 2) { } diff --git a/src/terrainlib/ctb/GlobalMercator.hpp b/src/terrainlib/ctb/GlobalMercator.hpp index 2da275a4..9de4af42 100644 --- a/src/terrainlib/ctb/GlobalMercator.hpp +++ b/src/terrainlib/ctb/GlobalMercator.hpp @@ -29,10 +29,11 @@ class GlobalMercator; } /** - * @brief An implementation of the TMS Global Mercator Profile + * @brief An implementation of the global Mercator grid profile * * This class models the [Tile Mapping Service Global Mercator * Profile](http://wiki.osgeo.org/wiki/Tile_Map_Service_Specification#global-mercator). + * Its public tile identifiers use Google/Mapbox/XYZ coordinates. */ class ctb::GlobalMercator : public Grid { public: @@ -41,7 +42,7 @@ class ctb::GlobalMercator : public Grid { radix::tile::SrsBounds{{-cOriginShift, -cOriginShift}, {cOriginShift, cOriginShift}}, cSRS, 3857, - std::vector{radix::tile::Id{0, {0, 0}, radix::tile::Scheme::Tms}}, + std::vector{radix::tile::Id{0, {0, 0}}}, 2) { } diff --git a/src/terrainlib/ctb/Grid.hpp b/src/terrainlib/ctb/Grid.hpp index a4338843..8d81488c 100644 --- a/src/terrainlib/ctb/Grid.hpp +++ b/src/terrainlib/ctb/Grid.hpp @@ -51,15 +51,8 @@ class Grid; * The code here generalises the logic in the `gdal2tiles.py` script available * with the GDAL library. * - * Warning: The y directino is dangerous. Sometimes the positve y axis points north, sometimes not. - * - GlobalMercator has y positive pointing north - * - GlobalGeodetic as well. - * - https://www.maptiler.com/google-maps-coordinates-tile-bounds-projection/#1/175.75/56.27 - * - google webmercator has tile coordinates where y=0 is the northern most tile. - * - tms webmercator has tile coordinates with y=0 being southern most. - * - * Effectively, ctb::Grid is always positive pointing north. Support for google webmercator / - * slippyMap is done in Tile.h + * Tile identifiers use Google/Mapbox/XYZ coordinates: the origin is north-west and y grows south. + * CRS and pixel coordinates inside Grid retain their conventional positive-north orientation. */ class ctb::Grid { public: @@ -143,23 +136,25 @@ class ctb::Grid { /// Get the tile coordinate in which a location falls at a specific zoom level [[nodiscard]] inline radix::tile::Id crsToTile(const CRSPoint &coord, i_zoom zoom) const { const PixelPoint pixel = crsToPixels(coord, zoom); - TilePoint tile = pixelsToTile(pixel); + const TilePoint tile = pixelsToTile(pixel); + const auto tile_count = i_tile(1u << zoom); - return {zoom, tile, radix::tile::Scheme::Tms}; + return {zoom, {tile.x, tile_count - tile.y - 1}}; } /// Get the CRS bounds of a particular tile /// border_se should be true if a border should be included on the south eastern corner /// e.g., for the cesium raster terrain format (https://github.com/CesiumGS/cesium/wiki/heightmap-1%2E0) [[nodiscard]] inline radix::tile::SrsBounds srsBounds(const radix::tile::Id &tile_id, bool border_se) const { - const auto tms_tile_id = tile_id.to(radix::tile::Scheme::Tms); + const auto tile_count = i_tile(1u << tile_id.zoom_level); + const auto grid_y = tile_count - tile_id.coords.y - 1; // get the pixels coordinates representing the tile bounds - const PixelPoint pxMinLeft(tms_tile_id.coords.x * mGridSize, tms_tile_id.coords.y * mGridSize); - const PixelPoint pxMaxRight((tms_tile_id.coords.x + 1) * mGridSize + border_se, (tms_tile_id.coords.y + 1) * mGridSize + border_se); + const PixelPoint pxMinLeft(tile_id.coords.x * mGridSize, grid_y * mGridSize); + const PixelPoint pxMaxRight((tile_id.coords.x + 1) * mGridSize + border_se, (grid_y + 1) * mGridSize + border_se); // convert pixels to native coordinates - const CRSPoint minLeft = pixelsToCrs(pxMinLeft, tms_tile_id.zoom_level); - const CRSPoint maxRight = pixelsToCrs(pxMaxRight, tms_tile_id.zoom_level); + const CRSPoint minLeft = pixelsToCrs(pxMinLeft, tile_id.zoom_level); + const CRSPoint maxRight = pixelsToCrs(pxMaxRight, tile_id.zoom_level); return { minLeft, maxRight }; } diff --git a/src/terrainlib/io/Error.h b/src/terrainlib/io/Error.h index 831c9842..b4e7aece 100644 --- a/src/terrainlib/io/Error.h +++ b/src/terrainlib/io/Error.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include "log.h" @@ -14,8 +13,6 @@ class Error { WriteBytes, DetermineSize, ReadBytes, - Serialize, - Deserialize, OutOfMemory }; @@ -37,10 +34,6 @@ class Error { private: Value _value; - -public: - using serialize = zpp::bits::members<1>; - friend zpp::bits::access; }; } // namespace io @@ -70,12 +63,6 @@ struct fmt::formatter { case io::Error::Value::ReadBytes: name = "ReadBytes"; break; - case io::Error::Value::Serialize: - name = "Serialize"; - break; - case io::Error::Value::Deserialize: - name = "Deserialize"; - break; case io::Error::Value::OutOfMemory: name = "OutOfMemory"; break; diff --git a/src/terrainlib/io/bytes.cpp b/src/terrainlib/io/bytes.cpp index 499b0ee9..1412ce33 100644 --- a/src/terrainlib/io/bytes.cpp +++ b/src/terrainlib/io/bytes.cpp @@ -6,7 +6,7 @@ namespace io { -tl::expected write_bytes_to_path(const std::span bytes, const std::filesystem::path &path, bool make_dirs) { +std::expected write_bytes_to_path(const std::span bytes, const std::filesystem::path &path, bool make_dirs) { LOG_TRACE("Writing bytes to path {}", path); if (make_dirs) { @@ -16,31 +16,31 @@ tl::expected write_bytes_to_path(const std::span byt std::ofstream file(path, std::ios::binary); if (!file.is_open()) { LOG_DEBUG("Failed to open file for writing {}", path); - return tl::unexpected(Error::OpenFile); + return std::unexpected(Error::OpenFile); } file.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); if (!file.good()) { LOG_ERROR("Failed to write bytes to file {}", path); - return tl::unexpected(Error::WriteBytes); + return std::unexpected(Error::WriteBytes); } return {}; } -tl::expected, Error> read_bytes_from_path(const std::filesystem::path& path) { +std::expected, Error> read_bytes_from_path(const std::filesystem::path& path) { LOG_TRACE("Reading bytes from path {}", path); std::ifstream file(path, std::ios::binary | std::ios::ate); if (!file.is_open()) { LOG_DEBUG("Failed to open file for reading {}", path); - return tl::unexpected(Error::OpenFile); + return std::unexpected(Error::OpenFile); } const std::streamsize size = file.tellg(); if (size < 0) { LOG_ERROR("Failed to determine size for file {}", path); - return tl::unexpected(Error::DetermineSize); + return std::unexpected(Error::DetermineSize); } std::vector buffer(static_cast(size)); @@ -49,7 +49,7 @@ tl::expected, Error> read_bytes_from_path(const std::filesy if (!file.good()) { LOG_ERROR("Failed to read bytes from file {}", path); - return tl::unexpected(Error::ReadBytes); + return std::unexpected(Error::ReadBytes); } return buffer; diff --git a/src/terrainlib/io/bytes.h b/src/terrainlib/io/bytes.h index 5c156cf4..be3852dc 100644 --- a/src/terrainlib/io/bytes.h +++ b/src/terrainlib/io/bytes.h @@ -4,13 +4,13 @@ #include #include -#include +#include #include "io/Error.h" namespace io { -tl::expected write_bytes_to_path(const std::span bytes, const std::filesystem::path &path, bool make_dirs = true); -tl::expected, Error> read_bytes_from_path(const std::filesystem::path &path); +std::expected write_bytes_to_path(const std::span bytes, const std::filesystem::path &path, bool make_dirs = true); +std::expected, Error> read_bytes_from_path(const std::filesystem::path &path); } diff --git a/src/terrainlib/io/compression.cpp b/src/terrainlib/io/compression.cpp new file mode 100644 index 00000000..b3ee1e5f --- /dev/null +++ b/src/terrainlib/io/compression.cpp @@ -0,0 +1,230 @@ +#define ZSTD_STATIC_LINKING_ONLY +#include +#include + +#include "io/compression.h" + +#include +#include +#include +#include +#include + +namespace io::envelope { +namespace { + +using CompressionContext = std::unique_ptr; + +std::string crc32c_checksum(const Bytes &data) +{ + static constexpr auto table = [] { + constexpr std::uint32_t polynomial = 0x82f63b78u; + + std::array values{}; + for (std::size_t index = 0; index < values.size(); ++index) { + auto crc = static_cast(index); + for (int bit = 0; bit < 8; ++bit) { + crc = (crc >> 1u) ^ ((crc & 1u) != 0u ? polynomial : 0u); + } + values[index] = crc; + } + return values; + }(); + + std::uint32_t crc = 0xffffffffu; + for (const auto value : data) { + const auto index = (crc ^ std::to_integer(value)) & 0xffu; + crc = table[index] ^ (crc >> 8u); + } + crc = ~crc; + + constexpr char hex_digits[] = "0123456789abcdef"; + std::string checksum(8, '0'); + for (std::size_t index = 0; index < checksum.size(); ++index) { + const auto shift = static_cast((checksum.size() - index - 1) * 4); + checksum[index] = hex_digits[(crc >> shift) & 0x0fu]; + } + return checksum; +} + +std::expected validate_algorithms( + const CompressionAlgorithm compression_algorithm, + const ChecksumAlgorithm checksum_algorithm) +{ + switch (checksum_algorithm) { + case ChecksumAlgorithm::None: + case ChecksumAlgorithm::HandledByCompressionLib: + case ChecksumAlgorithm::Crc32c: + break; + default: + return std::unexpected(Error{ErrorCode::UnsupportedChecksumAlgorithm}); + } + + switch (compression_algorithm) { + case CompressionAlgorithm::None: + case CompressionAlgorithm::ZstdBestCompressionWithChecksum: + case CompressionAlgorithm::ZstdDefaultCompressionWithChecksum: + break; + default: + return std::unexpected(Error{ErrorCode::UnsupportedCompressionAlgorithm}); + } + + const bool no_compression = compression_algorithm == CompressionAlgorithm::None + && (checksum_algorithm == ChecksumAlgorithm::None + || checksum_algorithm == ChecksumAlgorithm::Crc32c); + const bool zstd_with_checksum = + (compression_algorithm == CompressionAlgorithm::ZstdBestCompressionWithChecksum + || compression_algorithm == CompressionAlgorithm::ZstdDefaultCompressionWithChecksum) + && (checksum_algorithm == ChecksumAlgorithm::HandledByCompressionLib + || checksum_algorithm == ChecksumAlgorithm::Crc32c); + if (!no_compression && !zstd_with_checksum) { + return std::unexpected(Error{ErrorCode::InvalidAlgorithmCombination}); + } + + return {}; +} + +} // namespace + +std::expected compress_with_checksum( + const Bytes &uncompressed_data, + const CompressionAlgorithm compression_algorithm, + const ChecksumAlgorithm checksum_algorithm) +{ + if (const auto validation = validate_algorithms(compression_algorithm, checksum_algorithm); !validation) { + return std::unexpected(validation.error()); + } + if (uncompressed_data.size() > default_max_decompressed_size) { + return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); + } + + const std::string checksum = checksum_algorithm == ChecksumAlgorithm::Crc32c + ? crc32c_checksum(uncompressed_data) + : std::string{}; + + if (compression_algorithm == CompressionAlgorithm::None) { + return CompressedData{uncompressed_data, checksum}; + } + + CompressionContext context{ZSTD_createCCtx(), &ZSTD_freeCCtx}; + if (!context) { + return std::unexpected(Error{ErrorCode::CompressionFailed}); + } + + const int compression_level = + compression_algorithm == CompressionAlgorithm::ZstdBestCompressionWithChecksum + ? ZSTD_maxCLevel() + : 0; + if (ZSTD_isError(ZSTD_CCtx_setParameter( + context.get(), + ZSTD_c_compressionLevel, + compression_level)) + || ZSTD_isError(ZSTD_CCtx_setParameter(context.get(), ZSTD_c_checksumFlag, 1))) { + return std::unexpected(Error{ErrorCode::CompressionFailed}); + } + + const std::size_t capacity = ZSTD_compressBound(uncompressed_data.size()); + Bytes compressed_data(capacity); + const std::size_t compressed_size = ZSTD_compress2( + context.get(), + compressed_data.data(), + compressed_data.size(), + uncompressed_data.data(), + uncompressed_data.size()); + if (ZSTD_isError(compressed_size)) { + return std::unexpected(Error{ErrorCode::CompressionFailed}); + } + + compressed_data.resize(compressed_size); + return CompressedData{std::move(compressed_data), checksum}; +} + +std::expected checked_decompress( + const Bytes &compressed_data, + const CompressionAlgorithm compression_algorithm, + const ChecksumAlgorithm checksum_algorithm, + const std::string_view checksum, + const std::size_t max_decompressed_size) +{ + if (const auto validation = validate_algorithms(compression_algorithm, checksum_algorithm); !validation) { + return std::unexpected(validation.error()); + } + if (checksum_algorithm != ChecksumAlgorithm::Crc32c && !checksum.empty()) { + return std::unexpected(Error{ErrorCode::InvalidAlgorithmCombination}); + } + if (checksum_algorithm == ChecksumAlgorithm::Crc32c + && (checksum.size() != 8 + || !std::all_of(checksum.begin(), checksum.end(), [](const char character) { + return (character >= '0' && character <= '9') + || (character >= 'a' && character <= 'f'); + }))) { + return std::unexpected(Error{ErrorCode::ChecksumMismatch}); + } + + const std::size_t effective_max_decompressed_size = + std::min(max_decompressed_size, default_max_decompressed_size); + + Bytes uncompressed_data; + if (compression_algorithm == CompressionAlgorithm::None) { + if (compressed_data.size() > effective_max_decompressed_size) { + return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); + } + uncompressed_data = compressed_data; + } else { + ZSTD_frameHeader frame_header{}; + const std::size_t frame_header_result = ZSTD_getFrameHeader( + &frame_header, compressed_data.data(), compressed_data.size()); + if (ZSTD_isError(frame_header_result) || frame_header_result != 0) { + return std::unexpected(Error{ErrorCode::DecompressionFailed}); + } + if (frame_header.checksumFlag == 0) { + return std::unexpected(Error{ErrorCode::ChecksumMismatch}); + } + + const unsigned long long frame_content_size = + ZSTD_getFrameContentSize(compressed_data.data(), compressed_data.size()); + if (frame_content_size == ZSTD_CONTENTSIZE_ERROR + || (frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN + && frame_content_size > std::numeric_limits::max())) { + return std::unexpected(Error{ErrorCode::DecompressionFailed}); + } + + const bool content_size_is_known = frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN; + if (content_size_is_known && frame_content_size > effective_max_decompressed_size) { + return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); + } + + const std::size_t allocation_size = content_size_is_known + ? static_cast(frame_content_size) + : effective_max_decompressed_size; + uncompressed_data.resize(allocation_size); + const std::size_t decompressed_size = ZSTD_decompress( + uncompressed_data.data(), + uncompressed_data.size(), + compressed_data.data(), + compressed_data.size()); + if (ZSTD_isError(decompressed_size)) { + if (ZSTD_getErrorCode(decompressed_size) == ZSTD_error_checksum_wrong) { + return std::unexpected(Error{ErrorCode::ChecksumMismatch}); + } + if (!content_size_is_known + && ZSTD_getErrorCode(decompressed_size) == ZSTD_error_dstSize_tooSmall) { + return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); + } + return std::unexpected(Error{ErrorCode::DecompressionFailed}); + } + if (content_size_is_known && decompressed_size != uncompressed_data.size()) { + return std::unexpected(Error{ErrorCode::DecompressionFailed}); + } + + uncompressed_data.resize(decompressed_size); + } + + if (checksum_algorithm == ChecksumAlgorithm::Crc32c + && crc32c_checksum(uncompressed_data) != checksum) { + return std::unexpected(Error{ErrorCode::ChecksumMismatch}); + } + return uncompressed_data; +} + +} // namespace io::envelope diff --git a/src/terrainlib/io/compression.h b/src/terrainlib/io/compression.h new file mode 100644 index 00000000..536fe694 --- /dev/null +++ b/src/terrainlib/io/compression.h @@ -0,0 +1,69 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +namespace io::envelope { + +using Bytes = std::vector; + +inline constexpr std::size_t default_max_decompressed_size = std::size_t{1} << 30; + +enum class ChecksumAlgorithm : std::uint8_t { + None, + HandledByCompressionLib, + Crc32c, +}; + +enum class CompressionAlgorithm : std::uint8_t { + None, + ZstdBestCompressionWithChecksum, + ZstdDefaultCompressionWithChecksum, +}; + +enum class ErrorCode : std::uint8_t { + SerializationFailed, + DeserializationFailed, + InvalidMagic, + WrongClassName, + UnsupportedClassVersion, + UnsupportedChecksumAlgorithm, + UnsupportedCompressionAlgorithm, + InvalidAlgorithmCombination, + ChecksumMismatch, + CompressionFailed, + DecompressionFailed, + SizeLimitExceeded, +}; + +struct Error { + ErrorCode code; + std::errc serialization_error{}; + + constexpr bool operator==(const Error &) const = default; +}; + +struct CompressedData { + Bytes compressed_data; + std::string checksum; +}; + +std::expected compress_with_checksum( + const Bytes &uncompressed_data, + CompressionAlgorithm compression_algorithm, + ChecksumAlgorithm checksum_algorithm); + +std::expected checked_decompress( + const Bytes &compressed_data, + CompressionAlgorithm compression_algorithm, + ChecksumAlgorithm checksum_algorithm, + std::string_view checksum, + std::size_t max_decompressed_size = default_max_decompressed_size); + +} // namespace io::envelope diff --git a/src/terrainlib/io/conversion.h b/src/terrainlib/io/conversion.h new file mode 100644 index 00000000..246e87b0 --- /dev/null +++ b/src/terrainlib/io/conversion.h @@ -0,0 +1,239 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace io::conversion { + +enum class ErrorCode { + EmptyInput, + UnsupportedDimensions, + UnsupportedPixelType, + PixelTypeMismatch, + DimensionsOutOfRange, + AllocationFailed, + OpenCvFailure, +}; + +struct Error { + ErrorCode code; + int expected_cv_type{-1}; + int actual_cv_type{-1}; + + constexpr bool operator==(const Error &) const = default; +}; + +namespace detail { + +template +inline constexpr int cv_depth = -1; + +template <> +inline constexpr int cv_depth = CV_8U; +template <> +inline constexpr int cv_depth = CV_8S; +template <> +inline constexpr int cv_depth = CV_16U; +template <> +inline constexpr int cv_depth = CV_16S; +template <> +inline constexpr int cv_depth = CV_32S; +template <> +inline constexpr int cv_depth = CV_32F; +template <> +inline constexpr int cv_depth = CV_64F; + +template +struct PixelTraits { + using channel_type = std::remove_cv_t; + static constexpr int channels = 1; + static constexpr bool supported = cv_depth >= 0; + + static channel_type &channel(Pixel &pixel, const int) + { + return pixel; + } + + static const channel_type &channel(const Pixel &pixel, const int) + { + return pixel; + } +}; + +template +struct PixelTraits> { + using channel_type = Channel; + static constexpr int channels = static_cast(Length); + static constexpr bool supported = channels >= 2 && channels <= 4 + && cv_depth >= 0; + + static channel_type &channel(glm::vec &pixel, const int index) + { + return pixel[static_cast(index)]; + } + + static const channel_type &channel( + const glm::vec &pixel, + const int index) + { + return pixel[static_cast(index)]; + } +}; + +template +using Traits = PixelTraits>; + +template +inline constexpr int cv_type = Traits::supported + ? CV_MAKETYPE(cv_depth::channel_type>, Traits::channels) + : -1; + +template +inline constexpr bool has_packed_layout = Traits::supported + && std::is_trivially_copyable_v + && sizeof(Pixel) == sizeof(typename Traits::channel_type) * Traits::channels; + +template +void copy_mat_to_raster(const cv::Mat &source, radix::Raster &destination) +{ + using PixelInfo = Traits; + using Channel = typename PixelInfo::channel_type; + + const auto row_bytes = static_cast(source.cols) * sizeof(Pixel); + if constexpr (has_packed_layout) { + for (int row = 0; row < source.rows; ++row) { + std::memcpy( + destination.data() + static_cast(row) * source.cols, + source.ptr(row), + row_bytes); + } + } else { + for (int row = 0; row < source.rows; ++row) { + const Channel *source_row = source.ptr(row); + for (int column = 0; column < source.cols; ++column) { + Pixel &pixel = destination.pixel({ + static_cast(column), + static_cast(row), + }); + for (int channel = 0; channel < PixelInfo::channels; ++channel) { + PixelInfo::channel(pixel, channel) = + source_row[column * PixelInfo::channels + channel]; + } + } + } + } +} + +template +void copy_raster_to_mat(const radix::Raster &source, cv::Mat &destination) +{ + using PixelInfo = Traits; + using Channel = typename PixelInfo::channel_type; + + const auto row_bytes = static_cast(destination.cols) * sizeof(Pixel); + if constexpr (has_packed_layout) { + for (int row = 0; row < destination.rows; ++row) { + std::memcpy( + destination.ptr(row), + source.data() + static_cast(row) * destination.cols, + row_bytes); + } + } else { + for (int row = 0; row < destination.rows; ++row) { + Channel *destination_row = destination.ptr(row); + for (int column = 0; column < destination.cols; ++column) { + const Pixel &pixel = source.pixel({ + static_cast(column), + static_cast(row), + }); + for (int channel = 0; channel < PixelInfo::channels; ++channel) { + destination_row[column * PixelInfo::channels + channel] = + PixelInfo::channel(pixel, channel); + } + } + } + } +} + +} // namespace detail + +template +std::expected, Error> to_raster(const cv::Mat &source) +{ + if constexpr (!detail::Traits::supported) { + return std::unexpected(Error{ErrorCode::UnsupportedPixelType}); + } else { + if (source.empty()) { + return std::unexpected(Error{ErrorCode::EmptyInput}); + } + if (source.dims != 2) { + return std::unexpected(Error{ErrorCode::UnsupportedDimensions}); + } + if (source.type() != detail::cv_type) { + return std::unexpected(Error{ + ErrorCode::PixelTypeMismatch, + detail::cv_type, + source.type(), + }); + } + if (source.cols < 0 || source.rows < 0 + || static_cast(source.cols) + > std::numeric_limits::max() + || static_cast(source.rows) + > std::numeric_limits::max()) { + return std::unexpected(Error{ErrorCode::DimensionsOutOfRange}); + } + + try { + radix::Raster destination({ + static_cast(source.cols), + static_cast(source.rows), + }); + detail::copy_mat_to_raster(source, destination); + return destination; + } catch (const std::bad_alloc &) { + return std::unexpected(Error{ErrorCode::AllocationFailed}); + } catch (const cv::Exception &) { + return std::unexpected(Error{ErrorCode::OpenCvFailure}); + } + } +} + +template +std::expected to_mat(const radix::Raster &source) +{ + if constexpr (!detail::Traits::supported) { + return std::unexpected(Error{ErrorCode::UnsupportedPixelType}); + } else { + if (source.width() == 0 || source.height() == 0) { + return std::unexpected(Error{ErrorCode::EmptyInput}); + } + if (source.width() > static_cast(std::numeric_limits::max()) + || source.height() > static_cast(std::numeric_limits::max())) { + return std::unexpected(Error{ErrorCode::DimensionsOutOfRange}); + } + + try { + cv::Mat destination( + static_cast(source.height()), + static_cast(source.width()), + detail::cv_type); + detail::copy_raster_to_mat(source, destination); + return destination; + } catch (const std::bad_alloc &) { + return std::unexpected(Error{ErrorCode::AllocationFailed}); + } catch (const cv::Exception &) { + return std::unexpected(Error{ErrorCode::OpenCvFailure}); + } + } +} + +} // namespace io::conversion diff --git a/src/terrainlib/io/envelope.h b/src/terrainlib/io/envelope.h new file mode 100644 index 00000000..d643e383 --- /dev/null +++ b/src/terrainlib/io/envelope.h @@ -0,0 +1,147 @@ +#pragma once + +#include "io/compression.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace io::envelope { + +inline constexpr std::uint64_t magic = 0xF5FBD3EF919428CAULL; + +struct Envelope { + std::uint64_t magic; + std::string class_name; + std::uint32_t class_version; + ChecksumAlgorithm checksum_algorithm; + std::string checksum; + CompressionAlgorithm compression_algorithm; + std::uint64_t uncompressed_size; + Bytes compressed_data; +}; + +template +struct FixedString { + char value[Size]; + + constexpr FixedString(const char (&text)[Size]) + { + std::copy_n(text, Size, value); + } + + constexpr auto operator<=>(const FixedString &) const = default; +}; + +template +struct Version { + static constexpr std::uint32_t number = Number; + using payload_type = VersionedPayloadType; +}; + +namespace detail { + +template +struct FindVersion; + +template +struct FindVersion { + using type = std::conditional_t::type>; +}; + +template +struct FindVersion { + using type = void; +}; + +template +consteval bool versions_are_strictly_increasing() +{ + constexpr std::uint32_t numbers[] = {Versions::number...}; + for (std::size_t index = 1; index < sizeof...(Versions); ++index) { + if (numbers[index - 1] >= numbers[index]) { + return false; + } + } + return true; +} + +template +concept ConvertsFromPrevious = requires(From previous) { + { To::from_previous(std::move(previous)) } -> std::same_as; +}; + +template +consteval bool conversions_are_valid(std::index_sequence) +{ + return (ConvertsFromPrevious< + typename std::tuple_element_t::payload_type, + typename std::tuple_element_t::payload_type> + && ...); +} + +} // namespace detail + +template +struct PayloadSchema { + static_assert(sizeof...(Versions) > 0, "a payload schema requires at least one version"); + static_assert(detail::versions_are_strictly_increasing(), + "payload versions must be strictly increasing"); + + using version_tuple = std::tuple; + static constexpr std::size_t version_count = sizeof...(Versions); + + template + using version_at = std::tuple_element_t; + + using latest_version_descriptor = version_at; + using latest_type = typename latest_version_descriptor::payload_type; + + static constexpr std::string_view class_name{ClassName.value, sizeof(ClassName.value) - 1}; + static constexpr std::uint32_t latest_version = latest_version_descriptor::number; + + template + static constexpr bool supports_version = + !std::is_void_v::type>; + + static constexpr bool supports_version_number(const std::uint32_t number) + { + return ((number == Versions::number) || ...); + } + + template + using payload_type = typename detail::FindVersion::type::payload_type; + + static_assert(version_count == 1 + || detail::conversions_are_valid( + std::make_index_sequence{}), + "each payload version must provide from_previous for the preceding version"); +}; + +template +std::expected serialize( + const typename Schema::template payload_type &payload, + CompressionAlgorithm compression_algorithm = CompressionAlgorithm::ZstdDefaultCompressionWithChecksum, + ChecksumAlgorithm checksum_algorithm = ChecksumAlgorithm::HandledByCompressionLib); + +template +std::expected serialize( + const typename Schema::latest_type &payload, + CompressionAlgorithm compression_algorithm = CompressionAlgorithm::ZstdDefaultCompressionWithChecksum, + ChecksumAlgorithm checksum_algorithm = ChecksumAlgorithm::HandledByCompressionLib); + +template +std::expected deserialize( + std::span bytes, + std::size_t max_decompressed_size = default_max_decompressed_size); + +} // namespace io::envelope + +#include "io/envelope.inl" diff --git a/src/terrainlib/io/envelope.inl b/src/terrainlib/io/envelope.inl new file mode 100644 index 00000000..4fc57314 --- /dev/null +++ b/src/terrainlib/io/envelope.inl @@ -0,0 +1,160 @@ +#pragma once + +#include + +#include + +namespace io::envelope { +namespace detail { + +template +std::expected serialize_to_bytes(const Value &value) +{ + Bytes bytes; + zpp::bits::out output(bytes, zpp::bits::alloc_limit()); + const zpp::bits::errc result = output(value); + if (zpp::bits::failure(result)) { + return std::unexpected(Error{ErrorCode::SerializationFailed, result.code}); + } + return bytes; +} + +template +std::expected deserialize_from_bytes(const std::span bytes) +{ + Value value{}; + zpp::bits::in input(bytes, zpp::bits::alloc_limit()); + const zpp::bits::errc result = input(value); + if (zpp::bits::failure(result) || input.position() != bytes.size()) { + return std::unexpected(Error{ + ErrorCode::DeserializationFailed, + zpp::bits::failure(result) ? result.code : std::errc::bad_message, + }); + } + return value; +} + +template +typename Schema::latest_type convert_to_latest(Current current) +{ + if constexpr (Index + 1 == Schema::version_count) { + return current; + } else { + using Next = typename Schema::template version_at::payload_type; + return convert_to_latest(Next::from_previous(std::move(current))); + } +} + +template +std::expected deserialize_version( + const std::uint32_t class_version, + const std::span payload_bytes) +{ + using CurrentVersion = typename Schema::template version_at; + if (class_version == CurrentVersion::number) { + auto payload = deserialize_from_bytes(payload_bytes); + if (!payload) { + return std::unexpected(payload.error()); + } + return convert_to_latest(std::move(*payload)); + } + + if constexpr (Index + 1 < Schema::version_count) { + return deserialize_version(class_version, payload_bytes); + } else { + return std::unexpected(Error{ErrorCode::UnsupportedClassVersion}); + } +} + +} // namespace detail + +template +std::expected serialize( + const typename Schema::template payload_type &payload, + const CompressionAlgorithm compression_algorithm, + const ChecksumAlgorithm checksum_algorithm) +{ + static_assert(Schema::template supports_version, + "the requested payload version is not part of the schema"); + + auto payload_bytes = detail::serialize_to_bytes(payload); + if (!payload_bytes) { + return std::unexpected(payload_bytes.error()); + } + + auto compressed = compress_with_checksum(*payload_bytes, compression_algorithm, checksum_algorithm); + if (!compressed) { + return std::unexpected(compressed.error()); + } + + const Envelope envelope{ + .magic = magic, + .class_name = std::string{Schema::class_name}, + .class_version = VersionNumber, + .checksum_algorithm = checksum_algorithm, + .checksum = std::move(compressed->checksum), + .compression_algorithm = compression_algorithm, + .uncompressed_size = payload_bytes->size(), + .compressed_data = std::move(compressed->compressed_data), + }; + return detail::serialize_to_bytes(envelope); +} + +template +std::expected serialize( + const typename Schema::latest_type &payload, + const CompressionAlgorithm compression_algorithm, + const ChecksumAlgorithm checksum_algorithm) +{ + return serialize( + payload, + compression_algorithm, + checksum_algorithm); +} + +template +std::expected deserialize( + const std::span bytes, + const std::size_t max_decompressed_size) +{ + auto envelope = detail::deserialize_from_bytes(bytes); + if (!envelope) { + return std::unexpected(envelope.error()); + } + if (envelope->magic != magic) { + return std::unexpected(Error{ErrorCode::InvalidMagic}); + } + if (envelope->class_name != Schema::class_name) { + return std::unexpected(Error{ErrorCode::WrongClassName}); + } + if (!Schema::supports_version_number(envelope->class_version)) { + return std::unexpected(Error{ErrorCode::UnsupportedClassVersion}); + } + + const std::size_t effective_max_decompressed_size = + std::min(max_decompressed_size, default_max_decompressed_size); + if (envelope->uncompressed_size > std::numeric_limits::max() + || envelope->uncompressed_size > effective_max_decompressed_size) { + return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); + } + + auto payload_bytes = checked_decompress( + envelope->compressed_data, + envelope->compression_algorithm, + envelope->checksum_algorithm, + envelope->checksum, + static_cast(envelope->uncompressed_size)); + if (!payload_bytes) { + if (payload_bytes.error().code == ErrorCode::SizeLimitExceeded) { + return std::unexpected(Error{ErrorCode::DecompressionFailed}); + } + return std::unexpected(payload_bytes.error()); + } + if (payload_bytes->size() != envelope->uncompressed_size) { + return std::unexpected(Error{ErrorCode::DecompressionFailed}); + } + + return detail::deserialize_version(envelope->class_version, *payload_bytes); +} + +} // namespace io::envelope diff --git a/src/terrainlib/io/envelope_file.h b/src/terrainlib/io/envelope_file.h new file mode 100644 index 00000000..f5e3cd6a --- /dev/null +++ b/src/terrainlib/io/envelope_file.h @@ -0,0 +1,80 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "io/bytes.h" +#include "io/envelope.h" + +namespace io::envelope { + +using FileError = std::variant<::io::Error, Error>; + +inline std::string describe_error(const Error &error) +{ + return "envelope error " + std::to_string(static_cast(error.code)); +} + +inline std::string describe_error(const FileError &error) +{ + return std::visit( + [](const auto &value) -> std::string { + using Value = std::remove_cvref_t; + if constexpr (std::same_as) { + return value.to_string(); + } else { + return describe_error(value); + } + }, + error); +} + +template +std::expected read_from_path( + const std::filesystem::path &path, + const std::size_t max_decompressed_size = default_max_decompressed_size) +{ + auto file_bytes = ::io::read_bytes_from_path(path); + if (!file_bytes) { + return std::unexpected(FileError{file_bytes.error()}); + } + const auto bytes = std::as_bytes(std::span{*file_bytes}); + auto result = deserialize(bytes, max_decompressed_size); + if (!result) { + return std::unexpected(FileError{result.error()}); + } + return std::move(*result); +} + +template +std::expected write_to_path( + const typename Schema::latest_type &payload, + const std::filesystem::path &path, + const bool make_dirs = true, + const CompressionAlgorithm compression_algorithm = + CompressionAlgorithm::ZstdDefaultCompressionWithChecksum, + const ChecksumAlgorithm checksum_algorithm = ChecksumAlgorithm::HandledByCompressionLib) +{ + auto serialized = serialize(payload, compression_algorithm, checksum_algorithm); + if (!serialized) { + return std::unexpected(FileError{serialized.error()}); + } + const auto bytes = std::span{ + reinterpret_cast(serialized->data()), + serialized->size(), + }; + auto result = ::io::write_bytes_to_path(bytes, path, make_dirs); + if (!result) { + return std::unexpected(FileError{result.error()}); + } + return {}; +} + +} // namespace io::envelope diff --git a/src/terrainlib/io/serialize.h b/src/terrainlib/io/serialize.h deleted file mode 100644 index 55919e23..00000000 --- a/src/terrainlib/io/serialize.h +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include "io/Error.h" - -namespace io { - -template -tl::expected, Error> write_to_bytes(const T &value); -template -tl::expected read_from_bytes(const std::span bytes); - -template -tl::expected write_to_path(const T &value, const std::filesystem::path &path, bool make_dirs = true); -template -tl::expected read_from_path(const std::filesystem::path &path); - -} - -#include "io/serialize.inl" diff --git a/src/terrainlib/io/serialize.inl b/src/terrainlib/io/serialize.inl deleted file mode 100644 index 90d7080b..00000000 --- a/src/terrainlib/io/serialize.inl +++ /dev/null @@ -1,88 +0,0 @@ -#pragma once - -#include - -#include "io/bytes.h" -#include "io/utils.h" -#include "type_utils.h" - -namespace io { - -template -tl::expected, Error> write_to_bytes(const T &value) { - std::vector data; - zpp::bits::out out(data); - const auto result = out(value); - if (zpp::bits::failure(result)) { - std::error_code error_code = std::make_error_code(result); - LOG_ERROR("error while writing {}: {}", type_name(), error_code.message()); - - switch (result) { - case std::errc::no_buffer_space: // growing buffer would grow beyond the allocation limits or overflow. - case std::errc::message_size: // message size is beyond the user defined allocation limits. - case std::errc::result_out_of_range: // attempting to write or read from a too short buffer. - return tl::unexpected(Error::OutOfMemory); - case std::errc::value_too_large: // varint (variable length integer) encoding is beyond the representation limits. - case std::errc::bad_message: // attempt to read a variant of unrecognized type. - case std::errc::invalid_argument: // attempting to serialize null pointer or a value-less variant. - return tl::unexpected(Error::Serialize); - case std::errc::protocol_error: // attempt to deserialize an invalid protocol message. - case std::errc::not_supported: // attempt to call an RPC that is not listed as supported. - UNREACHABLE(); - default: - UNREACHABLE(); - } - } - return data; -} - -template -tl::expected read_from_bytes(const std::span bytes) { - zpp::bits::in in(bytes); - T value; - const auto result = in(value); - if (zpp::bits::failure(result)) { - std::error_code error_code = std::make_error_code(result); - LOG_ERROR("error while reading {}: {}", type_name(), error_code.message()); - - switch (result) { - case std::errc::no_buffer_space: // growing buffer would grow beyond the allocation limits or overflow. - case std::errc::message_size: // message size is beyond the user defined allocation limits. - case std::errc::result_out_of_range: // attempting to write or read from a too short buffer. - return tl::unexpected(Error::OutOfMemory); - case std::errc::value_too_large: // varint (variable length integer) encoding is beyond the representation limits. - case std::errc::bad_message: // attempt to read a variant of unrecognized type. - case std::errc::invalid_argument: // attempting to serialize null pointer or a value-less variant. - return tl::unexpected(Error::Deserialize); - case std::errc::protocol_error: // attempt to deserialize an invalid protocol message. - case std::errc::not_supported: // attempt to call an RPC that is not listed as supported. - UNREACHABLE(); - default: - UNREACHABLE(); - } - } - return value; -} - -template -tl::expected read_from_path(const std::filesystem::path &path) { - const auto result = read_bytes_from_path(path); - if (!result.has_value()) { - return tl::unexpected(result.error()); - } - const std::vector bytes = result.value(); - return read_from_bytes(bytes); -} - -template -tl::expected write_to_path(const T &value, const std::filesystem::path &path, bool make_dirs) { - const auto result = write_to_bytes(value); - if (!result.has_value()) { - return tl::unexpected(result.error()); - } - - const std::vector bytes = result.value(); - return write_bytes_to_path(bytes, path, make_dirs); -} - -} // namespace io diff --git a/src/terrainlib/mesh/EncodedMesh.h b/src/terrainlib/mesh/EncodedMesh.h index 4e9e4610..fd0927a6 100644 --- a/src/terrainlib/mesh/EncodedMesh.h +++ b/src/terrainlib/mesh/EncodedMesh.h @@ -3,8 +3,6 @@ #include #include -#include - namespace mesh { class Encoded { @@ -20,7 +18,6 @@ class Encoded { bool operator!=(const Header &) const = default; }; - using serialize = zpp::bits::members<5>; Header header; std::vector triangles; std::vector positions; diff --git a/src/terrainlib/mesh/codec/Gltf.h b/src/terrainlib/mesh/codec/Gltf.h new file mode 100644 index 00000000..f2ea337a --- /dev/null +++ b/src/terrainlib/mesh/codec/Gltf.h @@ -0,0 +1,81 @@ +#pragma once + +#include +#include +#include +#include + +#include "mesh/SimpleMesh.h" +#include "mesh/io/gltf.h" +#include "store/Codec.h" +#include "store/codec/Path.h" + +namespace mesh::codec { + +enum class GltfContainer { + Binary, + Json, +}; + +class Gltf final : public store::Codec { +public: + explicit Gltf(const GltfContainer container) : _container(container) {} + + std::vector paths(const store::NodePath &node_path) const override { + return {store::codec::append_extension( + node_path, + _container == GltfContainer::Binary ? ".glb" : ".gltf")}; + } + + std::expected read( + const store::NodePath &node_path) const override { + const auto result = mesh::io::gltf::load_from_path(paths(node_path).front()); + if (!result.has_value()) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + result.error() == mesh::io::LoadMeshErrorKind::FileNotFound + ? store::CodecErrorCategory::FileNotFound + : store::CodecErrorCategory::InvalidData, + result.error().description(), + }); + } + return result.value(); + } + + std::expected write( + const store::NodePath &node_path, + const mesh::Simple &mesh) const override { + try { + const auto result = mesh::io::gltf::save_to_path(mesh, paths(node_path).front()); + if (!result.has_value()) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Domain, + result.error().description(), + }); + } + return {}; + } catch (const std::exception &error) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Domain, + error.what(), + }); + } catch (...) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Domain, + "unknown glTF writer exception", + }); + } + } + + GltfContainer container() const { + return _container; + } + +private: + GltfContainer _container; +}; + +} // namespace mesh::codec diff --git a/src/terrainlib/mesh/codec/SfMesh.h b/src/terrainlib/mesh/codec/SfMesh.h new file mode 100644 index 00000000..33fb2c80 --- /dev/null +++ b/src/terrainlib/mesh/codec/SfMesh.h @@ -0,0 +1,96 @@ +#pragma once + +#include +#include + +#include "io/envelope_file.h" +#include "mesh/SimpleMesh.h" +#include "mesh/codec/SfMeshFormat.h" +#include "store/Codec.h" +#include "store/codec/Path.h" + +namespace mesh::codec { + +class SfMesh final : public store::Codec { +public: + std::vector paths( + const store::NodePath &node_path) const override + { + return {store::codec::append_extension(node_path, ".sfmesh")}; + } + + std::expected read( + const store::NodePath &node_path) const override + { + auto payload = ::io::envelope::read_from_path( + paths(node_path).front()); + if (!payload) { + const bool missing = std::holds_alternative<::io::Error>(payload.error()) + && std::get<::io::Error>(payload.error()) == ::io::Error(::io::Error::OpenFile); + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + missing ? store::CodecErrorCategory::FileNotFound + : store::CodecErrorCategory::InvalidData, + ::io::envelope::describe_error(payload.error()), + }); + } + if (auto valid = mesh::sf::validate(*payload); !valid) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + store::CodecErrorCategory::InvalidData, + valid.error(), + }); + } + auto decoded = mesh::sf::decode_payload(std::move(*payload)); + if (!decoded) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + store::CodecErrorCategory::InvalidData, + "could not decode SF mesh payload", + }); + } + return std::move(*decoded); + } + + std::expected write( + const store::NodePath &node_path, + const mesh::Simple &mesh) const override + { + return write(node_path, mesh, {}); + } + + std::expected write( + const store::NodePath &node_path, + const mesh::Simple &mesh, + const mesh::EncodeOptions options) const + { + auto payload = mesh::sf::encode_payload(mesh, options); + if (!payload) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Domain, + "could not encode SF mesh payload", + }); + } + if (auto valid = mesh::sf::validate(*payload); !valid) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Domain, + valid.error(), + }); + } + auto result = ::io::envelope::write_to_path( + *payload, + paths(node_path).front()); + if (!result) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Io, + ::io::envelope::describe_error(result.error()), + }); + } + return {}; + } +}; + +} // namespace mesh::codec diff --git a/src/terrainlib/mesh/codec/SfMeshFormat.h b/src/terrainlib/mesh/codec/SfMeshFormat.h new file mode 100644 index 00000000..b1461505 --- /dev/null +++ b/src/terrainlib/mesh/codec/SfMeshFormat.h @@ -0,0 +1,96 @@ +#pragma once + +#include +#include +#include + +#include + +#include "io/envelope.h" +#include "mesh/EncodedMesh.h" +#include "mesh/SimpleMesh.h" +#include "mesh/encode.h" + +namespace mesh::sf { + +namespace v1 { + +struct Payload { + std::uint32_t vertex_count; + std::uint32_t face_count; + std::vector triangles; + std::vector positions; + std::vector uvs; + std::vector texture; +}; + +} // namespace v1 + +using Schema = ::io::envelope::PayloadSchema< + "mesh.SfMesh", + ::io::envelope::Version<1, v1::Payload>>; +using Payload = Schema::latest_type; + +inline std::expected validate(const Payload &payload) +{ + if (payload.vertex_count + > ::io::envelope::default_max_decompressed_size / sizeof(mesh::Simple::Position)) { + return std::unexpected("SF mesh vertex count exceeds the allocation limit"); + } + if (payload.face_count + > ::io::envelope::default_max_decompressed_size / sizeof(mesh::Simple::Triangle)) { + return std::unexpected("SF mesh face count exceeds the allocation limit"); + } + if (payload.vertex_count == 0 + && (!payload.positions.empty() || !payload.uvs.empty())) { + return std::unexpected("empty mesh contains vertex data"); + } + if (payload.vertex_count != 0 && payload.positions.empty()) { + return std::unexpected("non-empty mesh contains no position data"); + } + if (payload.face_count == 0 && !payload.triangles.empty()) { + return std::unexpected("empty mesh contains triangle data"); + } + if (payload.face_count != 0 && payload.triangles.empty()) { + return std::unexpected("non-empty mesh contains no triangle data"); + } + return {}; +} + +inline std::expected encode_payload( + const mesh::Simple &mesh, + const mesh::EncodeOptions options = {}) +{ + auto encoded = mesh::encode(mesh, options); + if (!encoded) { + return std::unexpected(encoded.error()); + } + return Payload{ + .vertex_count = encoded->header.vertex_count, + .face_count = encoded->header.face_count, + .triangles = std::move(encoded->triangles), + .positions = std::move(encoded->positions), + .uvs = std::move(encoded->uvs), + .texture = std::move(encoded->texture), + }; +} + +inline std::expected decode_payload(Payload payload) +{ + const mesh::Encoded encoded{ + .header = { + .version = 1, + .n_dims = mesh::Simple::n_dims, + .component_type = mesh::component_type_id(), + .vertex_count = payload.vertex_count, + .face_count = payload.face_count, + }, + .triangles = std::move(payload.triangles), + .positions = std::move(payload.positions), + .uvs = std::move(payload.uvs), + .texture = std::move(payload.texture), + }; + return mesh::decode(encoded); +} + +} // namespace mesh::sf diff --git a/src/terrainlib/mesh/codec/from_extension.h b/src/terrainlib/mesh/codec/from_extension.h new file mode 100644 index 00000000..21209dbc --- /dev/null +++ b/src/terrainlib/mesh/codec/from_extension.h @@ -0,0 +1,33 @@ +#pragma once + +#include +#include +#include + +#include + +#include "mesh/codec/Gltf.h" +#include "mesh/codec/SfMesh.h" +#include "store/Codec.h" + +namespace mesh::codec { + +inline std::expected>, store::CodecError> +from_extension(const std::string_view extension) { + if (extension == ".sfmesh") { + return std::make_unique(); + } + if (extension == ".glb") { + return std::make_unique(GltfContainer::Binary); + } + if (extension == ".gltf") { + return std::make_unique(GltfContainer::Json); + } + return std::unexpected(store::CodecError{ + store::CodecOperation::Resolve, + store::CodecErrorCategory::UnsupportedCodec, + "unsupported mesh codec selector: " + std::string(extension), + }); +} + +} // namespace mesh::codec diff --git a/src/terrainlib/mesh/encode.h b/src/terrainlib/mesh/encode.h index 302fd016..eca66508 100644 --- a/src/terrainlib/mesh/encode.h +++ b/src/terrainlib/mesh/encode.h @@ -4,7 +4,7 @@ #include #include -#include +#include #include "log.h" #include "mesh/EncodedMesh.h" @@ -81,7 +81,7 @@ inline std::ostream& operator<<(std::ostream& os, const EncodeError& err) { } template -tl::expected encode(const Simple_& mesh, const EncodeOptions options = {}) { +std::expected encode(const Simple_& mesh, const EncodeOptions options = {}) { using Mesh = Simple_; const size_t vertex_count = mesh.vertex_count(); @@ -96,7 +96,7 @@ tl::expected encode(const Simple_& mesh, const position_buf.resize(meshopt_encodeVertexBufferBound(vertex_count, position_size)); const size_t pos_written = meshopt_encodeVertexBuffer(position_buf.data(), position_buf.size(), mesh.positions.data(), vertex_count, position_size); if (pos_written == 0) { - return tl::unexpected(EncodeError::PositionEncode); + return std::unexpected(EncodeError::PositionEncode); } position_buf.resize(pos_written); } @@ -107,7 +107,7 @@ tl::expected encode(const Simple_& mesh, const uv_buf.resize(meshopt_encodeVertexBufferBound(vertex_count, uv_size)); const size_t uv_written = meshopt_encodeVertexBuffer(uv_buf.data(), uv_buf.size(), mesh.uvs.data(), vertex_count, uv_size); if (uv_written == 0) { - return tl::unexpected(EncodeError::UvEncode); + return std::unexpected(EncodeError::UvEncode); } uv_buf.resize(uv_written); } @@ -121,7 +121,7 @@ tl::expected encode(const Simple_& mesh, const reinterpret_cast(mesh.triangles.data()), index_count); if (index_written == 0) { - return tl::unexpected(EncodeError::TriangleEncode); + return std::unexpected(EncodeError::TriangleEncode); } index_buf.resize(index_written); } @@ -133,7 +133,7 @@ tl::expected encode(const Simple_& mesh, const texture_buf = mesh::io::write_texture_to_encoded_buffer(mesh.texture.value(), options.texture_format); } catch (const cv::Exception &e) { LOG_ERROR("Failed while encoding texture {}", e.what()); - return tl::unexpected(EncodeError::TextureEncode); + return std::unexpected(EncodeError::TextureEncode); } } @@ -189,13 +189,13 @@ inline std::ostream &operator<<(std::ostream &os, const DecodeError &err) { template -tl::expected, DecodeError> decode(const Encoded &encoded, const DecodeOptions = {}) { +std::expected, DecodeError> decode(const Encoded &encoded, const DecodeOptions = {}) { const uint32_t expected_component_type = component_type_id(); const Encoded::Header& header = encoded.header; if (header.version != 1 || header.n_dims != n_dims || header.component_type != expected_component_type) { - return tl::unexpected(DecodeError::IncompatibleData); + return std::unexpected(DecodeError::IncompatibleData); } using Mesh = Simple_; @@ -208,7 +208,7 @@ tl::expected, DecodeError> decode(const Encoded &encoded, con mesh.positions.resize(vertex_count); result = meshopt_decodeVertexBuffer(mesh.positions.data(), vertex_count, position_size, encoded.positions.data(), encoded.positions.size()); if (result != 0) { - return tl::unexpected(DecodeError::PositionDecode); + return std::unexpected(DecodeError::PositionDecode); } // Decode uvs @@ -217,7 +217,7 @@ tl::expected, DecodeError> decode(const Encoded &encoded, con mesh.uvs.resize(vertex_count); result = meshopt_decodeVertexBuffer(mesh.uvs.data(), vertex_count, uv_size, encoded.uvs.data(), encoded.uvs.size()); if (result != 0) { - return tl::unexpected(DecodeError::UvDecode); + return std::unexpected(DecodeError::UvDecode); } } @@ -228,7 +228,7 @@ tl::expected, DecodeError> decode(const Encoded &encoded, con mesh.triangles.resize(face_count); result = meshopt_decodeIndexBuffer(mesh.triangles.data(), index_count, index_size, encoded.triangles.data(), encoded.triangles.size()); if (result != 0) { - return tl::unexpected(DecodeError::TriangleDecode); + return std::unexpected(DecodeError::TriangleDecode); } // Decode texture @@ -237,7 +237,7 @@ tl::expected, DecodeError> decode(const Encoded &encoded, con mesh.texture = mesh::io::read_texture_from_encoded_bytes(encoded.texture); } catch (const cv::Exception &e) { LOG_ERROR("Failed while decoding texture {}", e.what()); - return tl::unexpected(DecodeError::TextureDecode); + return std::unexpected(DecodeError::TextureDecode); } } diff --git a/src/terrainlib/mesh/io.cpp b/src/terrainlib/mesh/io.cpp index 6d56c2c1..d1a9be1d 100644 --- a/src/terrainlib/mesh/io.cpp +++ b/src/terrainlib/mesh/io.cpp @@ -1,25 +1,32 @@ #include "mesh/io.h" #include "log.h" #include "mesh/io/gltf.h" -#include "mesh/io/terrain.h" +#include "mesh/codec/SfMesh.h" #include "mesh/validate.h" namespace mesh::io { -tl::expected load_from_path( +std::expected load_from_path( const std::filesystem::path &path, const LoadOptions& options) { const std::filesystem::path extension = path.extension(); if (extension == ".glb" || extension == ".gltf") { return gltf::load_from_path(path, options); - } else if (extension == ".terrain") { - return terrain::load_from_path(path, options); + } else if (extension == ".sfmesh") { + std::filesystem::path node_path = path; + node_path.replace_extension(); + const mesh::codec::SfMesh codec; + auto result = codec.read(store::NodePath(node_path)); + if (!result) { + return std::unexpected(LoadMeshErrorKind::InvalidFormat); + } + return std::move(*result); } else { - return tl::unexpected(LoadMeshErrorKind::UnsupportedFormat); + return std::unexpected(LoadMeshErrorKind::UnsupportedFormat); } } -tl::expected save_to_path( +std::expected save_to_path( const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions &options) { @@ -30,10 +37,20 @@ tl::expected save_to_path( const std::filesystem::path extension = path.extension(); if (extension == ".glb" || extension == ".gltf") { return gltf::save_to_path(mesh, path, options); - } else if (extension == ".terrain") { - return terrain::save_to_path(mesh, path, options); + } else if (extension == ".sfmesh") { + std::filesystem::path node_path = path; + node_path.replace_extension(); + const mesh::codec::SfMesh codec; + auto result = codec.write( + store::NodePath(node_path), + mesh, + mesh::EncodeOptions{.texture_format = options.texture_format}); + if (!result) { + return std::unexpected(SaveMeshErrorKind::WriteFile); + } + return {}; } else { - return tl::unexpected(SaveMeshErrorKind::UnsupportedFormat); + return std::unexpected(SaveMeshErrorKind::UnsupportedFormat); } } diff --git a/src/terrainlib/mesh/io.h b/src/terrainlib/mesh/io.h index 81fb076e..ea474c73 100644 --- a/src/terrainlib/mesh/io.h +++ b/src/terrainlib/mesh/io.h @@ -2,7 +2,7 @@ #include -#include +#include #include "mesh/SimpleMesh.h" #include "mesh/io/options.h" @@ -10,11 +10,11 @@ namespace mesh::io { -tl::expected load_from_path( +std::expected load_from_path( const std::filesystem::path &path, const LoadOptions& options = {}); -tl::expected save_to_path( +std::expected save_to_path( const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions& options = {}); diff --git a/src/terrainlib/mesh/io/gltf.cpp b/src/terrainlib/mesh/io/gltf.cpp index 36160963..318716a6 100644 --- a/src/terrainlib/mesh/io/gltf.cpp +++ b/src/terrainlib/mesh/io/gltf.cpp @@ -200,7 +200,7 @@ std::optional load_texture_from_material(const cgltf_material &material #define GET_OR_INVALID_FORMAT(var, opt) \ do { \ if (!(opt).has_value()) { \ - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); \ + return std::unexpected(LoadMeshErrorKind::InvalidFormat); \ } else { \ var = opt.value(); \ } \ @@ -265,56 +265,56 @@ static std::string image_ext_to_mime(std::string_view extension) { } } -tl::expected load_raw_from_path(const std::filesystem::path &path) { +std::expected load_raw_from_path(const std::filesystem::path &path) { cgltf_options options = {}; cgltf_data *data = NULL; const std::string path_str = path.string(); const char *path_ptr = path_str.c_str(); cgltf_result result = cgltf_parse_file(&options, path_ptr, &data); if (result != cgltf_result::cgltf_result_success) { - return tl::unexpected(result); + return std::unexpected(result); } result = cgltf_load_buffers(&options, data, path_ptr); if (result != cgltf_result::cgltf_result_success) { cgltf_free(data); - return tl::unexpected(result); + return std::unexpected(result); } result = cgltf_validate(data); if (result != cgltf_result_success) { cgltf_free(data); - return tl::unexpected(result); + return std::unexpected(result); } return RawMesh(data, cgltf_free); } -tl::expected load_from_raw(const RawMesh &raw, const LoadOptions& /* options */) { +std::expected load_from_raw(const RawMesh &raw, const LoadOptions& /* options */) { LOG_TRACE("Loading mesh from gltf data"); const cgltf_data &data = *raw; const auto mesh_opt = get_single_element("mesh", data.meshes_count, data.meshes); if (!mesh_opt.has_value()) { - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); + return std::unexpected(LoadMeshErrorKind::InvalidFormat); } const cgltf_mesh &mesh = mesh_opt.value(); const auto mesh_primitive_opt = get_single_element("mesh primitive", mesh.primitives_count, mesh.primitives); if (!mesh_primitive_opt.has_value()) { - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); + return std::unexpected(LoadMeshErrorKind::InvalidFormat); } const cgltf_primitive &mesh_primitive = mesh_primitive_opt.value(); if (mesh_primitive.type != cgltf_primitive_type::cgltf_primitive_type_triangles) { LOG_ERROR("mesh has invalid primitive type"); - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); + return std::unexpected(LoadMeshErrorKind::InvalidFormat); } // indices if (mesh_primitive.indices == nullptr) { LOG_ERROR("mesh primitive has no indices"); - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); + return std::unexpected(LoadMeshErrorKind::InvalidFormat); } cgltf_accessor &index_accessor = *mesh_primitive.indices; std::vector indices; @@ -330,13 +330,13 @@ tl::expected load_from_raw(const RawMesh &raw, const cgltf_attribute *position_attr = find_attribute_with_type(mesh_primitive.attributes, mesh_primitive.attributes_count, cgltf_attribute_type_position); if (position_attr == nullptr) { LOG_ERROR("mesh has no position attribute"); - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); + return std::unexpected(LoadMeshErrorKind::InvalidFormat); } cgltf_accessor &position_accessor = *position_attr->data; if (position_accessor.type != cgltf_type_vec3) { LOG_WARN("mesh positions are not vec3"); - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); + return std::unexpected(LoadMeshErrorKind::InvalidFormat); } std::vector positions; positions.resize(position_accessor.count); @@ -351,7 +351,7 @@ tl::expected load_from_raw(const RawMesh &raw, const cgltf_accessor &uv_accessor = *uv_attr->data; if (uv_accessor.type != cgltf_type_vec2) { LOG_WARN("mesh uvss are not vec2"); - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); + return std::unexpected(LoadMeshErrorKind::InvalidFormat); } uvs.resize(uv_accessor.count); cgltf_accessor_unpack_floats(&uv_accessor, reinterpret_cast(uvs.data()), uvs.size() * 2); @@ -381,7 +381,7 @@ tl::expected load_from_raw(const RawMesh &raw, const } /// Saves the mesh as a .gltf or .glb file at the given path. -tl::expected save_to_path( +std::expected save_to_path( const SimpleMesh &terrain_mesh, const std::filesystem::path &path, const SaveOptions& options) { @@ -719,10 +719,10 @@ tl::expected save_to_path( return {}; } -tl::expected load_from_path(const std::filesystem::path &path, const LoadOptions &options) { - tl::expected raw_mesh = load_raw_from_path(path); +std::expected load_from_path(const std::filesystem::path &path, const LoadOptions &options) { + std::expected raw_mesh = load_raw_from_path(path); if (!raw_mesh) { - return tl::unexpected(map_cgltf_error(raw_mesh.error())); + return std::unexpected(map_cgltf_error(raw_mesh.error())); } return load_from_raw(*raw_mesh, options); } diff --git a/src/terrainlib/mesh/io/gltf.h b/src/terrainlib/mesh/io/gltf.h index 60757382..20333e93 100644 --- a/src/terrainlib/mesh/io/gltf.h +++ b/src/terrainlib/mesh/io/gltf.h @@ -5,7 +5,7 @@ #include #include -#include +#include #include "mesh/SimpleMesh.h" #include "mesh/io/error.h" @@ -15,19 +15,19 @@ namespace mesh::io::gltf { using RawMesh = std::unique_ptr; -tl::expected load_from_path( +std::expected load_from_path( const std::filesystem::path &path, const LoadOptions &options = {}); -tl::expected load_from_raw( +std::expected load_from_raw( const RawMesh &mesh, const LoadOptions &options = {}); -tl::expected save_to_path( +std::expected save_to_path( const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions &options = {}); -// tl::expected save_to_raw(const RawMesh &mesh, const SaveOptions &options = {}); +// std::expected save_to_raw(const RawMesh &mesh, const SaveOptions &options = {}); -tl::expected load_raw_from_path(const std::filesystem::path &path); +std::expected load_raw_from_path(const std::filesystem::path &path); } // namespace mesh::io::gltf diff --git a/src/terrainlib/mesh/io/options.h b/src/terrainlib/mesh/io/options.h index e7c2ec27..332ade5e 100644 --- a/src/terrainlib/mesh/io/options.h +++ b/src/terrainlib/mesh/io/options.h @@ -11,7 +11,7 @@ struct LoadOptions { struct SaveOptions { std::string texture_format = ".jpeg"; - std::string name = "Terrain"; + std::string name = "Octree"; std::unordered_map metadata = {}; }; diff --git a/src/terrainlib/mesh/io/terrain.cpp b/src/terrainlib/mesh/io/terrain.cpp deleted file mode 100644 index 1068a991..00000000 --- a/src/terrainlib/mesh/io/terrain.cpp +++ /dev/null @@ -1,172 +0,0 @@ -#include - -#include - -#include "io/bytes.h" -#include "log.h" -#include "mesh/io/terrain.h" -#include "mesh/EncodedMesh.h" -#include "mesh/encode.h" - -namespace mesh::io::terrain { - -namespace { -LoadMeshError load_error_from_io_error(::io::Error error) { - switch (error) { - case ::io::Error::Value::OpenFile: - return LoadMeshErrorKind::FileNotFound; - case ::io::Error::Value::DetermineSize: - case ::io::Error::Value::ReadBytes: - return LoadMeshErrorKind::InvalidFormat; - default: - return LoadMeshErrorKind::InvalidFormat; - } -} - -SaveMeshError save_error_from_io_error(::io::Error error) { - switch (error) { - case ::io::Error::Value::OpenFile: - return SaveMeshErrorKind::OpenFile; - case ::io::Error::Value::WriteBytes: - return SaveMeshErrorKind::WriteFile; - default: - return SaveMeshErrorKind::WriteFile; - } -} - -tl::expected write_bytes_to_path( - const std::span bytes, const std::filesystem::path &path) { - const auto result = ::io::write_bytes_to_path(bytes, path); - if (!result.has_value()) { - return tl::unexpected(save_error_from_io_error(result.error())); - } - return {}; -} - -tl::expected, LoadMeshError> read_bytes_from_path(const std::filesystem::path &path) { - const auto result = ::io::read_bytes_from_path(path); - if (!result.has_value()) { - return tl::unexpected(load_error_from_io_error(result.error())); - } - return result.value(); -} - -tl::expected, SaveMeshError> save_encoded_to_buffer(const mesh::Encoded &mesh) { - LOG_TRACE("Serializing mesh to buffer"); - - // TODO: this ignores the texture format in SaveOptions - std::vector data; - zpp::bits::out out(data); - auto result = out(mesh); - if (zpp::bits::failure(result)) { - std::error_code error_code = std::make_error_code(result); - LOG_ERROR("Error while writing mesh: {}", error_code.message()); - - switch (result) { - case std::errc::no_buffer_space: - case std::errc::message_size: - case std::errc::result_out_of_range: - return tl::unexpected(SaveMeshErrorKind::OutOfMemory); - break; - default: - UNREACHABLE(); - break; - } - } - - return data; -} - -tl::expected load_encoded_from_buffer(const std::span bytes) { - LOG_TRACE("Deserializing mesh from buffer"); - - zpp::bits::in in(bytes); - mesh::Encoded mesh; - auto result = in(mesh); - if (zpp::bits::failure(result)) { - std::error_code error_code = std::make_error_code(result); - LOG_ERROR("error while reading mesh: {}", error_code.message()); - - switch (result) { - case std::errc::no_buffer_space: - case std::errc::message_size: - return tl::unexpected(LoadMeshErrorKind::OutOfMemory); - case std::errc::value_too_large: - case std::errc::bad_message: - case std::errc::protocol_error: - case std::errc::result_out_of_range: - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); - case std::errc::not_supported: - case std::errc::invalid_argument: - UNREACHABLE(); - break; - default: - throw std::runtime_error("unexpected error"); - } - } - - return mesh; -} -} - -tl::expected, SaveMeshError> save_to_buffer(const SimpleMesh &mesh, const SaveOptions& options) { - const auto encode_result = mesh::encode(mesh, mesh::EncodeOptions{ - .texture_format = options.texture_format}); - if (!encode_result.has_value()) { - return tl::unexpected(SaveMeshErrorKind::UnsupportedFormat); - } - const mesh::Encoded encoded = encode_result.value(); - - const auto deser_result = save_encoded_to_buffer(encoded); - if (!deser_result.has_value()) { - return tl::unexpected(deser_result.error()); - } - const std::vector buffer = deser_result.value(); - - return buffer; -} - -tl::expected load_from_buffer(const std::span bytes, const LoadOptions & /* options */) { - const auto deser_result = load_encoded_from_buffer(bytes); - if (!deser_result.has_value()) { - return tl::unexpected(deser_result.error()); - } - const mesh::Encoded encoded = deser_result.value(); - - const auto decode_result = mesh::decode(encoded); - if (!decode_result.has_value()) { - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); - } - const mesh::Simple mesh = decode_result.value(); - - return mesh; -} - -tl::expected load_from_path(const std::filesystem::path &path, const LoadOptions & /* options */) { - const auto bytes_result = read_bytes_from_path(path); - if (!bytes_result.has_value()) { - return tl::unexpected(bytes_result.error()); - } - const std::vector bytes = bytes_result.value(); - - return load_from_buffer(bytes); -} - -tl::expected save_to_path(const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions& options) { - LOG_TRACE("Saving mesh as high precision terrain"); - - const auto result = save_to_buffer(mesh, options); - if (!result.has_value()) { - return tl::unexpected(result.error()); - } - const std::vector bytes = result.value(); - - const auto write_result = write_bytes_to_path(bytes, path); - if (!write_result.has_value()) { - return tl::unexpected(write_result.error()); - } - - return {}; -} - -} // namespace mesh::io::terrain diff --git a/src/terrainlib/mesh/io/terrain.h b/src/terrainlib/mesh/io/terrain.h deleted file mode 100644 index 9c505c14..00000000 --- a/src/terrainlib/mesh/io/terrain.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include -#include - -#include - -#include "mesh/SimpleMesh.h" -#include "mesh/io/error.h" -#include "mesh/io/options.h" - -namespace mesh::io::terrain { - -tl::expected load_from_path(const std::filesystem::path &path, const LoadOptions &options = {}); -tl::expected load_from_buffer(const std::span buffer, const LoadOptions &options = {}); - -tl::expected save_to_path(const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions &options = {}); -tl::expected, SaveMeshError> save_to_buffer(const SimpleMesh &mesh, const SaveOptions &options = {}); - -} // namespace mesh::io::terrain diff --git a/src/terrainlib/mesh/io/texture.h b/src/terrainlib/mesh/io/texture.h index c6849014..e110878e 100644 --- a/src/terrainlib/mesh/io/texture.h +++ b/src/terrainlib/mesh/io/texture.h @@ -3,7 +3,6 @@ #include #include #include -#include #include @@ -21,33 +20,3 @@ std::vector write_texture_to_encoded_buffer(const cv::Mat &image, const std::vector write_texture_to_encoded_buffer(const ImageAndExt &item); } - -#include - -namespace mesh::io { - -template -auto serialize(Archive &archive, const mesh::io::ImageAndExt &item) { - const std::vector encoded = mesh::io::write_texture_to_encoded_buffer(item.image, item.ext); - return archive(item.ext, encoded); -} - -template -auto serialize(Archive &archive, mesh::io::ImageAndExt &item) { - std::vector encoded; - - auto result = archive(item.ext, encoded); - using Result = std::remove_cvref_t; - if constexpr (!std::is_same_v && !std::is_same_v) { - return result; - } else { - if (zpp::bits::failure(result)) { - return result; - } - - item.image = mesh::io::read_texture_from_encoded_bytes(encoded); - return result; - } -} - -} // namespace mesh::io diff --git a/src/terrainlib/mesh/storage.h b/src/terrainlib/mesh/storage.h new file mode 100644 index 00000000..a26ac905 --- /dev/null +++ b/src/terrainlib/mesh/storage.h @@ -0,0 +1,52 @@ +#pragma once + +#include +#include + +#include "mesh/SimpleMesh.h" +#include "mesh/codec/from_extension.h" +#include "octree/StoreTraits.h" +#include "octree/storage/IndexFile.h" +#include "octree/storage/open_runtime.h" +#include "store/IndexedStorage.h" + +namespace mesh::storage { + +inline constexpr std::string_view payload_class = "mesh.Simple3d"; + +using Storage = store::Storage; +using IndexedStorage = store::IndexedStorage; +using OpenOptions = octree::storage::OpenOptions; + +inline std::expected> open_index( + const std::filesystem::path &path) { + return store::open_index( + path, + octree::storage::index_format(), + payload_class, + mesh::codec::from_extension); +} + +inline std::expected> open_folder( + const std::filesystem::path &path, + OpenOptions options = {}) { + return octree::storage::open_folder( + path, + std::string(payload_class), + ".sfmesh", + mesh::codec::from_extension, + std::move(options)); +} + +inline std::expected> open_folder_indexed( + const std::filesystem::path &path, + OpenOptions options = {}) { + return octree::storage::open_folder_indexed( + path, + std::string(payload_class), + ".sfmesh", + mesh::codec::from_extension, + std::move(options)); +} + +} // namespace mesh::storage diff --git a/src/terrainlib/octree/Id.h b/src/terrainlib/octree/Id.h index 44518c71..f1619cb3 100644 --- a/src/terrainlib/octree/Id.h +++ b/src/terrainlib/octree/Id.h @@ -98,7 +98,7 @@ class Id_ { return Coords(max_coord_on_level(level)); } - constexpr Id_() = default; // zpp::bits requires a default constructor + constexpr Id_() = default; template requires(sizeof...(Coordinates) == Dimensions && (std::convertible_to && ...)) constexpr Id_(const Level level, Coordinates... coordinates) : Id_(level, Coords{coordinates...}) {} @@ -399,51 +399,3 @@ struct hash> { } }; } // namespace std - -#include -namespace octree { -namespace { -constexpr zpp::bits::errc success() { - return zpp::bits::errc(std::errc()); -} -} // namespace - -template < - size_t Dimensions, - size_t MaxLevel, - typename LevelT, - typename IndexT, - typename CoordT> -constexpr auto serialize(auto &archive, octree::Id_ &id) { - using Id = octree::Id_; - typename Id::Level level; - typename Id::Index index; - auto result = archive(level, index); - using Result = std::remove_cvref_t; - if constexpr (!std::is_same_v && !std::is_same_v) { - return result; - } else { - if (zpp::bits::failure(result)) { - return result; - } - - auto maybe_id = Id::try_make(level, index); - if (!maybe_id) { - return zpp::bits::errc(std::errc::bad_message); - } - - id = *maybe_id; - return success(); - } -} - -template < - size_t Dimensions, - size_t MaxLevel, - typename LevelT, - typename IndexT, - typename CoordT> -constexpr auto serialize(auto &archive, const octree::Id_ &id) { - return archive(id.level(), id.index_on_level()); -} -} // namespace octree diff --git a/src/terrainlib/octree/IndexMap.cpp b/src/terrainlib/octree/IndexMap.cpp deleted file mode 100644 index 8bb7e53c..00000000 --- a/src/terrainlib/octree/IndexMap.cpp +++ /dev/null @@ -1,174 +0,0 @@ -#include - -#include "octree/IndexMap.h" -#include "log.h" - -namespace octree { - -std::optional IndexMap::get(Id id) const { - const NodeStatus* status = this->get_raw(id); - if (status != nullptr) { - return *status; - } else { - return std::nullopt; - } -} - -bool IndexMap::add(Id id) { - NodeStatus* status = this->get_raw(id); - if (status != nullptr) { - switch (*status) { - case NodeStatus::Inner: - case NodeStatus::Leaf: - // Nothing to do - return false; - case NodeStatus::Virtual: - this->set_raw(id, NodeStatus::Inner); - return true; - } - } - - const auto parent_opt = id.parent(); - if (!parent_opt.has_value()) { - DEBUG_ASSERT(id.is_root()); - if (this->empty()) { - this->set_raw(id, NodeStatus::Leaf); - return true; - } else { - UNREACHABLE(); - return true; - } - } - - const auto parent = parent_opt.value(); - const NodeStatus* parent_status = this->get_raw(parent); - if (parent_status == nullptr) { - this->add(parent); - this->set_raw(parent, NodeStatus::Virtual); - this->set_raw(id, NodeStatus::Leaf); - return true; - } - - switch (*parent_status) { - case NodeStatus::Leaf: - this->set_raw(parent, NodeStatus::Inner); - this->set_raw(id, NodeStatus::Leaf); - break; - case NodeStatus::Inner: - case NodeStatus::Virtual: - this->set_raw(id, NodeStatus::Leaf); - break; - } - - return true; -} - -bool IndexMap::remove(Id id) { - NodeStatus* status = this->get_raw(id); - if (status == nullptr) { - return false; - } - - switch (*status) { - case NodeStatus::Inner: - *status = NodeStatus::Virtual; - return true; - case NodeStatus::Virtual: - // Nothing to do - return false; - case NodeStatus::Leaf: - this->remove_raw(id); - this->update_parent_after_remove(id); - return true; - } - - UNREACHABLE(); -} - -bool IndexMap::is_present(Id id) const { - auto it = this->_index.find(id); - return it != this->_index.end(); -} -bool IndexMap::is_absent(Id id) const { - return !this->is_present(id); -} -bool IndexMap::is(NodeStatus status, Id id) const { - return this->get(id) == status; -} - -void IndexMap::clear() { - this->_index.clear(); -} -bool IndexMap::empty() const { - return this->_index.empty(); -} -size_t IndexMap::size() const { - return this->_index.size(); -} - -IndexMap::const_iterator IndexMap::begin() const { - return this->_index.begin(); -} -IndexMap::const_iterator IndexMap::end() const { - return this->_index.end(); -} -IndexMap::const_iterator IndexMap::cbegin() const { - return this->_index.cbegin(); -} -IndexMap::const_iterator IndexMap::cend() const { - return this->_index.cend(); -} - -NodeStatus* IndexMap::get_raw(Id id) { - if (auto it = this->_index.find(id); it != this->_index.end()) { - return &it->second; - } - return nullptr; -} -const NodeStatus* IndexMap::get_raw(Id id) const { - if (auto it = this->_index.find(id); it != this->_index.end()) { - return &it->second; - } - return nullptr; -} -void IndexMap::set_raw(Id id, NodeStatus status) { - this->_index[id] = status; -} -void IndexMap::remove_raw(Id id) { - this->_index.erase(id); -} - -void IndexMap::update_parent_after_remove(Id id) { - const auto parent_opt = id.parent(); - if (!parent_opt.has_value()) { - return; - } - - const auto parent = parent_opt.value(); - NodeStatus* parent_status = this->get_raw(parent); - DEBUG_ASSERT(parent_status != nullptr); - - // We know there are children since we just removed one - DEBUG_ASSERT(parent.has_children()); - const auto siblings = parent.children().value(); - for (const auto& sibling : siblings) { - if (this->is_present(sibling) && sibling != id) { - return; - } - } - - switch (*parent_status) { - case NodeStatus::Inner: - *parent_status = NodeStatus::Leaf; - break; - case NodeStatus::Virtual: - this->remove_raw(parent); - this->update_parent_after_remove(parent); - break; - case NodeStatus::Leaf: - UNREACHABLE(); - break; - } -} - -} diff --git a/src/terrainlib/octree/IndexMap.h b/src/terrainlib/octree/IndexMap.h deleted file mode 100644 index c5dbf535..00000000 --- a/src/terrainlib/octree/IndexMap.h +++ /dev/null @@ -1,49 +0,0 @@ -#pragma once - -#include -#include - -#include - -#include "octree/Id.h" -#include "octree/NodeStatus.h" - -namespace octree { - -class IndexMap { -public: - using Container = std::unordered_map; - using iterator = Container::iterator; - using const_iterator = Container::const_iterator; - - std::optional get(Id id) const; - bool add(Id id); - bool remove(Id id); - - bool is_present(Id id) const; - bool is_absent(Id id) const; - bool is(NodeStatus status, Id id) const; - - void clear(); - bool empty() const; - size_t size() const; - - const_iterator begin() const; - const_iterator end() const; - const_iterator cbegin() const; - const_iterator cend() const; - - NodeStatus* get_raw(Id id); - const NodeStatus* get_raw(Id id) const; - void set_raw(Id id, NodeStatus status); - void remove_raw(Id id); - - using serialize = zpp::bits::members<1>; - friend zpp::bits::access; - -private: - Container _index; - void update_parent_after_remove(Id id); -}; - -} // namespace octree diff --git a/src/terrainlib/octree/NodeStatus.h b/src/terrainlib/octree/NodeStatus.h deleted file mode 100644 index 9e00d97c..00000000 --- a/src/terrainlib/octree/NodeStatus.h +++ /dev/null @@ -1,81 +0,0 @@ -#pragma once - -#include - -#include "log.h" - -namespace octree { - -class NodeStatus { -public: - using Underlying = uint8_t; - enum Value : Underlying { - Leaf = 0, - Inner = 1, - Virtual = 2 // not present on disk but has children - }; - - constexpr NodeStatus() = default; - constexpr NodeStatus(Value value) : _value(value) {} - - constexpr operator Value() const { - return this->_value; - } - explicit operator bool() const = delete; - constexpr bool operator==(NodeStatus other) const { - return this->_value == other._value; - } - constexpr bool operator!=(NodeStatus other) const { - return !(*this == other); - } - constexpr bool operator==(Value other_value) const { - return this->_value == other_value; - } - constexpr bool operator!=(Value other_value) const { - return this->_value != other_value; - } - - std::string to_string() const; - -private: - Value _value; - -public: - using serialize = zpp::bits::members<1>; - friend zpp::bits::access; -}; - -} - -#include -template <> -struct fmt::formatter { - template - constexpr auto parse(ParseContext &ctx) { - return ctx.begin(); - } - - template - auto format(const octree::NodeStatus &status, FormatContext &ctx) const { - switch (status) { - case octree::NodeStatus::Leaf: - return fmt::format_to(ctx.out(), "Leaf"); - case octree::NodeStatus::Inner: - return fmt::format_to(ctx.out(), "Inner"); - case octree::NodeStatus::Virtual: - return fmt::format_to(ctx.out(), "Virtual"); - default: - UNREACHABLE(); - } - } -}; - -#include -#include -inline std::string octree::NodeStatus::to_string() const { - return fmt::format("{}", *this); -} -inline std::ostream &operator<<(std::ostream &os, const octree::NodeStatus &status) { - fmt::print(os, "{}", status); - return os; -} diff --git a/src/terrainlib/octree/NodeStatusOrMissing.h b/src/terrainlib/octree/NodeStatusOrMissing.h deleted file mode 100644 index 0868cb93..00000000 --- a/src/terrainlib/octree/NodeStatusOrMissing.h +++ /dev/null @@ -1,138 +0,0 @@ -#pragma once - -#include - -#include - -#include "octree/NodeStatus.h" - -namespace octree { - -class NodeStatusOrMissing { -public: - using Underlying = NodeStatus::Underlying; - enum Value : Underlying { - Leaf = 0, - Inner = 1, - Virtual = 2, // not present on disk but has children - Missing = 3 - }; - - constexpr NodeStatusOrMissing() = default; - constexpr NodeStatusOrMissing(Value value) : _value(value) {} - NodeStatusOrMissing(NodeStatus opt) : _value(from_status(opt)) {} - NodeStatusOrMissing(std::optional opt) : _value(from_optional_status(opt)) {} - - constexpr operator Value() const { - return this->_value; - } - constexpr operator std::optional() const { - return this->to_optional_status(); - } - explicit operator bool() const = delete; - constexpr bool operator==(NodeStatusOrMissing other) const { - return this->_value == other._value; - } - constexpr bool operator!=(NodeStatusOrMissing other) const { - return !(*this == other); - } - constexpr bool operator==(Value other_value) const { - return this->_value == other_value; - } - constexpr bool operator!=(Value other_value) const { - return this->_value != other_value; - } - - std::string to_string() const; - -private: - constexpr static NodeStatusOrMissing from_status(NodeStatus status) noexcept { - return static_cast(static_cast(status)); - } - constexpr static NodeStatusOrMissing from_optional_status(std::optional opt) noexcept { - if (opt.has_value()) { - return from_status(opt.value()); - } else { - return NodeStatusOrMissing::Missing; - } - } - constexpr std::optional to_optional_status() const noexcept { - if (this->_value == NodeStatusOrMissing::Missing) { - return std::nullopt; - } else { - return static_cast(this->_value); - } - } - -public: - Value _value; -}; - -} - -#include -template <> -struct fmt::formatter { - template - constexpr auto parse(ParseContext &ctx) { - return ctx.begin(); - } - - template - auto format(const octree::NodeStatusOrMissing &status, FormatContext &ctx) const { - switch (status) { - case octree::NodeStatusOrMissing::Leaf: - return fmt::format_to(ctx.out(), "Leaf"); - case octree::NodeStatusOrMissing::Inner: - return fmt::format_to(ctx.out(), "Inner"); - case octree::NodeStatusOrMissing::Virtual: - return fmt::format_to(ctx.out(), "Virtual"); - case octree::NodeStatusOrMissing::Missing: - return fmt::format_to(ctx.out(), "Missing"); - default: - UNREACHABLE(); - } - } -}; - -#include -#include -inline std::string octree::NodeStatusOrMissing::to_string() const { - return fmt::format("{}", *this); -} -inline std::ostream &operator<<(std::ostream &os, const octree::NodeStatusOrMissing &status) { - fmt::print(os, "{}", status); - return os; -} - -namespace octree { -namespace { -template -consteval bool _verify_enum_values(std::index_sequence) { - constexpr auto ns_values = magic_enum::enum_values(); - constexpr auto nsom_values = magic_enum::enum_values(); - - return ((magic_enum::enum_underlying(ns_values[Is]) == - magic_enum::enum_underlying(nsom_values[Is])) && - ...); -} - -consteval bool _verify_enum() { - static_assert( - std::is_same_v, - magic_enum::underlying_type_t>, - "Mismatched underlying type"); - - constexpr auto ns_values = magic_enum::enum_values(); - constexpr auto nsom_values = magic_enum::enum_values(); - - static_assert(nsom_values.size() == ns_values.size() + 1, "Mismatched enum counts"); - - static_assert(_verify_enum_values(std::make_index_sequence()), - "Enum value mismatch"); - - return true; -} -static_assert(_verify_enum()); -} -} diff --git a/src/terrainlib/octree/Storage.h b/src/terrainlib/octree/Storage.h deleted file mode 100644 index e6776555..00000000 --- a/src/terrainlib/octree/Storage.h +++ /dev/null @@ -1,14 +0,0 @@ -#pragma once - -#include "octree/storage/IndexedStorage.h" -#include "octree/storage/MeshStorage.h" -#include "octree/storage/RawStorage.h" -#include "octree/storage/Storage.h" -#include "octree/storage/open.h" - -namespace octree { - -using Storage = MeshStorage; -using IndexedStorage = IndexedMeshStorage; - -} diff --git a/src/terrainlib/octree/StoreTraits.h b/src/terrainlib/octree/StoreTraits.h new file mode 100644 index 00000000..655656a4 --- /dev/null +++ b/src/terrainlib/octree/StoreTraits.h @@ -0,0 +1,28 @@ +#pragma once + +#include + +#include "octree/Id.h" + +namespace octree { + +struct StoreTraits { + using Key = Id; + using Hasher = std::hash; + + static constexpr Key root() { + return Key::root(); + } + static constexpr auto parent(const Key &key) { + return key.parent(); + } + static constexpr auto children(const Key &key) { + return key.children(); + } + static constexpr bool is_valid(const Key &key) { + return key.level() <= Key::max_level() + && key.index_on_level() <= Key::max_index_on_level(key.level()); + } +}; + +} // namespace octree diff --git a/src/terrainlib/octree/disk/IndexFile.h b/src/terrainlib/octree/disk/IndexFile.h deleted file mode 100644 index 478be69f..00000000 --- a/src/terrainlib/octree/disk/IndexFile.h +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once - -#include - -#include - -#include "octree/IndexMap.h" - -namespace octree::disk::v1 { - -inline constexpr std::string_view index_file_name() { - return "terrain.index"; -} -inline constexpr std::string_view index_extension() { - return ".index"; -} - -struct IndexFile { - using serialize = zpp::bits::members<3>; - - std::string layout_strategy_id; - std::string preferred_extension; - IndexMap map; -}; -} diff --git a/src/terrainlib/octree/disk/Layout.h b/src/terrainlib/octree/disk/Layout.h deleted file mode 100644 index ba56f40b..00000000 --- a/src/terrainlib/octree/disk/Layout.h +++ /dev/null @@ -1,49 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -#include - -#include "octree/Id.h" -#include "octree/disk/layout/Strategy.h" - -namespace octree::disk { - -class Layout { -public: - Layout(std::filesystem::path base_path, std::unique_ptr strategy, std::string extension_with_dot) - : _base_path(std::move(base_path)), _strategy(std::move(strategy)), _extension_with_dot(std::move(extension_with_dot)) { - DEBUG_ASSERT(!_extension_with_dot.empty() && _extension_with_dot[0] == '.'); - } - - std::filesystem::path get_node_path(Id id) const { - return this->_base_path / this->_strategy->get_relative_node_path(id, this->_extension_with_dot); - } - std::optional get_id_from_node_path(const std::filesystem::path& path) const { - const std::filesystem::path relative_path = std::filesystem::relative(path, _base_path); - return this->_strategy->get_id_from_relative_node_path(relative_path); - } - - const std::filesystem::path& base_path() const { - return this->_base_path; - } - const layout::Strategy &strategy() const { - return *this->_strategy; - } - std::string_view extension_with_dot() const { - return this->_extension_with_dot; - } - std::string_view extension() const { - return this->extension_with_dot().substr(1); - } - -private: - std::filesystem::path _base_path; - std::unique_ptr _strategy; - std::string _extension_with_dot; -}; - -} diff --git a/src/terrainlib/octree/disk/layout/Strategy.h b/src/terrainlib/octree/disk/layout/Strategy.h deleted file mode 100644 index a1d5fb70..00000000 --- a/src/terrainlib/octree/disk/layout/Strategy.h +++ /dev/null @@ -1,19 +0,0 @@ -#pragma once - -#include -#include -#include - -#include "octree/Id.h" - -namespace octree::disk::layout { - -class Strategy { -public: - virtual ~Strategy() = default; - - virtual std::filesystem::path get_relative_node_path(Id id, std::string_view extension_with_dot) const = 0; - virtual std::optional get_id_from_relative_node_path(const std::filesystem::path &relative_path) const = 0; -}; - -} diff --git a/src/terrainlib/octree/disk/layout/StrategyRegister.h b/src/terrainlib/octree/disk/layout/StrategyRegister.h deleted file mode 100644 index 026a44f7..00000000 --- a/src/terrainlib/octree/disk/layout/StrategyRegister.h +++ /dev/null @@ -1,84 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -#include "octree/disk/layout/Strategy.h" -#include "log.h" - -namespace octree::disk::layout { - -class StrategyRegister { -public: - using Factory = std::function()>; - - static StrategyRegister &instance() { - static StrategyRegister inst; - return inst; - } - - template - void register_strategy(const std::string &id) { - this->_factories[id] = [] { return std::make_unique(); }; - this->_type_to_id[std::type_index(typeid(T))] = id; - } - - std::unique_ptr create(const std::string &id) const { - auto it = this->_factories.find(id); - if (it != this->_factories.end()) { - return (it->second)(); - } - return nullptr; - } - - const std::unordered_map &factories() const { - return this->_factories; - } - - template - std::string get_id() const { - return this->_get_id(typeid(T)); - } - - std::string get_id(const Strategy &s) const { - return this->_get_id(typeid(s)); - } - -private: - std::unordered_map _factories; - std::unordered_map _type_to_id; - - // Private ctor/dtor to enforce singleton - StrategyRegister() = default; - ~StrategyRegister() = default; - StrategyRegister(const StrategyRegister &) = delete; - StrategyRegister &operator=(const StrategyRegister &) = delete; - - std::string _get_id(std::type_index type_index) const { - auto it = this->_type_to_id.find(type_index); - if (it != this->_type_to_id.end()) { - return it->second; - } - LOG_ERROR_AND_EXIT("Encountered octree layout strategy not registered"); - } -}; - -} // namespace octree::disk::layout - -#include "octree/disk/layout/strategy/Flat.h" -#include "octree/disk/layout/strategy/LevelAndCoordinateDirectories.h" - -namespace { -template -struct Registrar { - Registrar(const std::string &id) { - octree::disk::layout::StrategyRegister::instance().register_strategy(id); - } -}; - -const Registrar reg_flat("flat"); -const Registrar reg_lacd("level_and_coordinate_directories"); -} diff --git a/src/terrainlib/octree/disk/layout/strategy/Default.h b/src/terrainlib/octree/disk/layout/strategy/Default.h deleted file mode 100644 index 45c7842d..00000000 --- a/src/terrainlib/octree/disk/layout/strategy/Default.h +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once - -#include - -#include "octree/disk/layout/strategy/LevelAndCoordinateDirectories.h" - -namespace octree::disk::layout::strategy { - -using Default = LevelAndCoordinateDirectories; - -inline std::unique_ptr make_default() { - return std::make_unique(); -} - -} // namespace octree::disk::layout::strategy diff --git a/src/terrainlib/octree/disk/layout/strategy/Flat.h b/src/terrainlib/octree/disk/layout/strategy/Flat.h deleted file mode 100644 index 0a11a941..00000000 --- a/src/terrainlib/octree/disk/layout/strategy/Flat.h +++ /dev/null @@ -1,50 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include "octree/Id.h" -#include "octree/disk/layout/Strategy.h" -#include "string_utils.h" - -namespace octree::disk::layout::strategy { - -class Flat : public Strategy { -public: - std::filesystem::path get_relative_node_path(Id id, std::string_view extension_with_dot) const override { - return fmt::format("{}-{}{}", id.level(), id.index_on_level(), extension_with_dot); - } - - std::optional get_id_from_relative_node_path(const std::filesystem::path &relative_path) const override { - DEBUG_ASSERT(relative_path.is_relative()); - - const std::string _filestem = relative_path.stem().string(); - const std::string_view filestem = _filestem; - - const size_t dash_pos = filestem.find('-'); - if (dash_pos == std::string::npos) { - return std::nullopt; - } - const std::string_view level_str = filestem.substr(0, dash_pos); - const std::string_view index_str = filestem.substr(dash_pos + 1); - - const auto level_opt = from_chars(level_str); - if (!level_opt) { - return std::nullopt; - } - - const auto index_opt = from_chars(index_str); - if (!index_opt) { - return std::nullopt; - } - - const Id::Level level = *level_opt; - const Id::Index index = *index_opt; - return Id::try_make(level, index); - } -}; - -} // namespace octree::disk::layout::strategy diff --git a/src/terrainlib/octree/disk/layout/strategy/LevelAndCoordinateDirectories.h b/src/terrainlib/octree/disk/layout/strategy/LevelAndCoordinateDirectories.h deleted file mode 100644 index 6a5ffa68..00000000 --- a/src/terrainlib/octree/disk/layout/strategy/LevelAndCoordinateDirectories.h +++ /dev/null @@ -1,60 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include "octree/Id.h" -#include "octree/disk/layout/Strategy.h" -#include "string_utils.h" - -namespace octree::disk::layout::strategy { - -class LevelAndCoordinateDirectories : public Strategy { -public: - std::filesystem::path get_relative_node_path(Id id, std::string_view extension_with_dot) const override { - const Id::Coords coords = id.coords(); - return fmt::format("{}/{}/{}/{}{}", id.level(), coords.x, coords.y, coords.z, extension_with_dot); - } - std::optional get_id_from_relative_node_path(const std::filesystem::path &relative_path) const override { - DEBUG_ASSERT(relative_path.is_relative()); - - auto it = relative_path.begin(); - if (std::distance(it, relative_path.end()) < 4) { - return std::nullopt; - } - - const auto level_opt = from_chars(it->native()); - if (!level_opt) { - return std::nullopt; - } - const Id::Level level = *level_opt; - ++it; - - const auto x_opt = from_chars(it->native()); - if (!x_opt) { - return std::nullopt; - } - const Id::Coord x = *x_opt; - ++it; - - const auto y_opt = from_chars(it->native()); - if (!y_opt) { - return std::nullopt; - } - const Id::Coord y = *y_opt; - ++it; - - const auto z_opt = from_chars(it->stem().native()); - if (!z_opt) { - return std::nullopt; - } - const Id::Coord z = *z_opt; - - return Id::try_make(level, {x, y, z}); - } -}; - -} // namespace octree::disk::layout::strategy diff --git a/src/terrainlib/octree/storage/CopyError.h b/src/terrainlib/octree/storage/CopyError.h deleted file mode 100644 index 7bd6c278..00000000 --- a/src/terrainlib/octree/storage/CopyError.h +++ /dev/null @@ -1,59 +0,0 @@ -#pragma once - -#include -#include - -namespace octree { - -enum class CopyErrorKind { - FileNotFound, - CreateLink, - CreateDirectories, - RemoveOld, - Read, - Write -}; - -class CopyError { -public: - constexpr CopyError(CopyErrorKind kind) - : kind(kind) {} - - operator CopyErrorKind() const { - return this->kind; - } - constexpr bool operator==(CopyError other) const { - return this->kind == other.kind; - } - constexpr bool operator!=(CopyError other) const { - return this->kind != other.kind; - } - - std::string description() const { - switch (kind) { - case CopyErrorKind::FileNotFound: - return "file not found"; - case CopyErrorKind::CreateLink: - return "cannot create hard link"; - case CopyErrorKind::CreateDirectories: - return "cannot create directories"; - case CopyErrorKind::RemoveOld: - return "cannot remove current file"; - case CopyErrorKind::Read: - return "cannot read file"; - case CopyErrorKind::Write: - return "cannot write file"; - default: - return "undefined error"; - } - } - - friend std::ostream &operator<<(std::ostream &os, const CopyError &err) { - return os << err.description(); - } - -private: - CopyErrorKind kind; -}; - -} // namespace octree diff --git a/src/terrainlib/octree/storage/Format.h b/src/terrainlib/octree/storage/Format.h new file mode 100644 index 00000000..e7e48121 --- /dev/null +++ b/src/terrainlib/octree/storage/Format.h @@ -0,0 +1,138 @@ +#pragma once + +#include +#include +#include +#include + +#include + +#include "io/envelope.h" +#include "octree/StoreTraits.h" +#include "store/Index.h" + +namespace octree::storage { + +inline constexpr std::string_view metadata_file_name = "octree.storemeta"; +inline constexpr std::string_view index_file_name = "octree.storeindex"; + +namespace v1 { + +struct StoreMetadata { + std::string layout_id; + std::string payload_class; + std::string codec_selector; +}; + +struct IndexEntry { + std::uint8_t level; + std::uint64_t index; + std::uint8_t status; +}; + +struct StoreIndex { + std::vector entries; +}; + +} // namespace v1 + +using StoreMetadataSchema = io::envelope::PayloadSchema< + "octree.StoreMetadata", + io::envelope::Version<1, v1::StoreMetadata>>; +using StoreIndexSchema = io::envelope::PayloadSchema< + "octree.StoreIndex", + io::envelope::Version<1, v1::StoreIndex>>; + +using StoreMetadata = StoreMetadataSchema::latest_type; +using StoreIndex = StoreIndexSchema::latest_type; + +inline std::expected validate(const StoreMetadata &metadata) +{ + if (metadata.layout_id.empty()) { + return std::unexpected("layout ID is empty"); + } + if (metadata.payload_class.empty()) { + return std::unexpected("payload class is empty"); + } + if (metadata.codec_selector.empty()) { + return std::unexpected("codec selector is empty"); + } + return {}; +} + +inline StoreIndex encode_index(const store::Index &index) +{ + StoreIndex result; + result.entries.reserve(index.size()); + for (const auto &[id, status] : index) { + result.entries.push_back({ + .level = id.level(), + .index = id.index_on_level(), + .status = static_cast(static_cast(status)), + }); + } + return result; +} + +inline std::expected, std::string> decode_index( + const StoreIndex &encoded) +{ + store::Index result; + std::unordered_set seen; + for (const v1::IndexEntry &entry : encoded.entries) { + const auto id = Id::try_make( + static_cast(entry.level), + static_cast(entry.index)); + if (!id) { + return std::unexpected("index contains an invalid octree ID"); + } + if (!seen.insert(*id).second) { + return std::unexpected("index contains a duplicate octree ID"); + } + if (entry.status > static_cast(store::NodeStatus::Virtual)) { + return std::unexpected("index contains an invalid node status"); + } + const auto set = result.set_raw( + *id, + store::NodeStatus{static_cast(entry.status)}); + if (!set) { + return std::unexpected("index contains an invalid octree ID"); + } + } + + for (const auto &[id, status] : result) { + const auto children = StoreTraits::children(id); + bool has_child = false; + if (children) { + for (const Id &child : *children) { + const auto child_status = result.get(child); + if (!child_status) { + return std::unexpected("index child lookup failed"); + } + has_child = has_child || child_status->has_value(); + } + } + if ((status == store::NodeStatus::Leaf && has_child) + || ((status == store::NodeStatus::Inner + || status == store::NodeStatus::Virtual) + && !has_child)) { + return std::unexpected("index contains inconsistent node topology"); + } + + const auto parent = StoreTraits::parent(id); + if (!parent) { + if (id != StoreTraits::root()) { + return std::unexpected("index contains a non-root node without a parent"); + } + continue; + } + const auto parent_status = result.get(*parent); + if (!parent_status || !parent_status->has_value() + || parent_status->value() == store::NodeStatus::Leaf) { + return std::unexpected("index contains a node without a valid indexed parent"); + } + } + return result; +} + +} // namespace octree::storage diff --git a/src/terrainlib/octree/storage/IndexFile.h b/src/terrainlib/octree/storage/IndexFile.h new file mode 100644 index 00000000..1ef8f9b5 --- /dev/null +++ b/src/terrainlib/octree/storage/IndexFile.h @@ -0,0 +1,129 @@ +#pragma once + +#include + +#include "io/envelope_file.h" +#include "octree/StoreTraits.h" +#include "octree/storage/Format.h" +#include "octree/store_layout/Mappings.h" +#include "store/IndexFormat.h" + +namespace octree::storage { + +inline store::IndexFormatError read_error( + const std::filesystem::path &path, + const io::envelope::FileError &error) +{ + const bool missing = std::holds_alternative(error) + && std::get(error) == io::Error(io::Error::OpenFile); + return { + missing ? store::IndexFormatErrorCategory::Open + : store::IndexFormatErrorCategory::Malformed, + path, + io::envelope::describe_error(error), + }; +} + +inline std::expected read_store_metadata( + const std::filesystem::path &base_path) +{ + const std::filesystem::path path = base_path / metadata_file_name; + auto metadata = io::envelope::read_from_path(path); + if (!metadata) { + return std::unexpected(read_error(path, metadata.error())); + } + if (auto valid = validate(*metadata); !valid) { + return std::unexpected(store::IndexFormatError{ + store::IndexFormatErrorCategory::Malformed, + path, + valid.error(), + }); + } + return std::move(*metadata); +} + +inline std::expected, store::IndexFormatError> +read_index_file(const std::filesystem::path &path) +{ + auto metadata = read_store_metadata(path.parent_path()); + if (!metadata) { + return std::unexpected(metadata.error()); + } + + auto encoded_index = io::envelope::read_from_path(path); + if (!encoded_index) { + return std::unexpected(read_error(path, encoded_index.error())); + } + auto index = decode_index(*encoded_index); + if (!index) { + return std::unexpected(store::IndexFormatError{ + store::IndexFormatErrorCategory::Malformed, + path, + index.error(), + }); + } + return store::IndexMetadata{ + std::move(*index), + std::move(metadata->layout_id), + std::move(metadata->payload_class), + std::move(metadata->codec_selector), + }; +} + +inline std::expected write_index_file( + const std::filesystem::path &path, + const store::IndexMetadata &metadata) +{ + const StoreMetadata store_metadata{ + .layout_id = metadata.layout_id, + .payload_class = metadata.payload_class, + .codec_selector = metadata.codec_selector, + }; + if (auto valid = validate(store_metadata); !valid) { + return std::unexpected(store::IndexFormatError{ + store::IndexFormatErrorCategory::Write, + path.parent_path() / metadata_file_name, + valid.error(), + }); + } + const std::filesystem::path metadata_path = path.parent_path() / metadata_file_name; + if (auto written = io::envelope::write_to_path( + store_metadata, + metadata_path); + !written) { + return std::unexpected(store::IndexFormatError{ + store::IndexFormatErrorCategory::Write, + metadata_path, + io::envelope::describe_error(written.error()), + }); + } + + const StoreIndex encoded_index = encode_index(metadata.index); + if (auto written = io::envelope::write_to_path(encoded_index, path); + !written) { + return std::unexpected(store::IndexFormatError{ + store::IndexFormatErrorCategory::Write, + path, + io::envelope::describe_error(written.error()), + }); + } + return {}; +} + +inline store::PathMapping default_mapping() +{ + return store_layout::level_and_coordinate_directories(); +} + +inline store::IndexFormat index_format() +{ + return { + index_file_name, + read_index_file, + write_index_file, + store_layout::from_id, + default_mapping, + }; +} + +} // namespace octree::storage diff --git a/src/terrainlib/octree/storage/IndexedStorage.h b/src/terrainlib/octree/storage/IndexedStorage.h deleted file mode 100644 index 70076cd3..00000000 --- a/src/terrainlib/octree/storage/IndexedStorage.h +++ /dev/null @@ -1,67 +0,0 @@ -#pragma once - -#include - -#include - -#include "octree/Id.h" -#include "octree/IndexMap.h" -#include "octree/disk/Layout.h" -#include "octree/storage/Storage.h" -#include "octree/storage/codec/Codec.h" -#include "octree/storage/defaults.h" -#include "octree/storage/helpers.h" - -namespace octree { - -template Codec = DefaultCodecFor> -class IndexedStorage_ : public Storage_ { -public: - explicit IndexedStorage_(Storage_ inner) noexcept - : Storage_(std::move(inner)) { - this->ensure_indexed(); - DEBUG_ASSERT(this->is_indexed()); - } - explicit IndexedStorage_(RawStorage_ inner, IndexMap map) noexcept - : Storage_(std::move(inner), std::move(map)) { - DEBUG_ASSERT(this->is_indexed()); - } - - IndexedStorage_ &operator=(const IndexedStorage_ &) = delete; - IndexedStorage_(const IndexedStorage_ &) = delete; - IndexedStorage_(IndexedStorage_ &&) = default; - IndexedStorage_ &operator=(IndexedStorage_ &&) = default; - - ~IndexedStorage_() override { - if (this->is_index_dirty()) { - LOG_WARN("Index was not saved upon IndexedStorage_ destruction, use save_index()."); - } - } - - const IndexMap& index() const noexcept { - DEBUG_ASSERT(this->is_indexed()); - return Storage_::index().value(); - } - void update_index() noexcept { - Storage_::update_index(); - } - tl::expected save_index() const noexcept { - if (!this->is_index_dirty()) { - return {}; - } - - auto result = helpers::save_index_map(this->index(), this->layout()); - if (result.has_value()) { - this->set_index_dirty(false); - } - return result; - } - -private: - IndexMap& index_mut() noexcept { - DEBUG_ASSERT(this->is_indexed()); - return Storage_::index_mut().value(); - } -}; - -} // namespace octree diff --git a/src/terrainlib/octree/storage/MeshStorage.h b/src/terrainlib/octree/storage/MeshStorage.h deleted file mode 100644 index 46b69813..00000000 --- a/src/terrainlib/octree/storage/MeshStorage.h +++ /dev/null @@ -1,12 +0,0 @@ -#pragma once - -#include "mesh/SimpleMesh.h" -#include "octree/storage/Storage.h" -#include "octree/storage/IndexedStorage.h" - -namespace octree { - -using MeshStorage = Storage_; -using IndexedMeshStorage = IndexedStorage_; - -} diff --git a/src/terrainlib/octree/storage/RawStorage.h b/src/terrainlib/octree/storage/RawStorage.h deleted file mode 100644 index 679efb38..00000000 --- a/src/terrainlib/octree/storage/RawStorage.h +++ /dev/null @@ -1,118 +0,0 @@ -#pragma once - -#include -#include - -#include - -#include "octree/Id.h" -#include "octree/disk/Layout.h" -#include "octree/storage/codec/Codec.h" -#include "octree/storage/CopyError.h" -#include "octree/storage/defaults.h" - -namespace octree { - -template Codec = DefaultCodecFor> -class RawStorage_ { -public: - using value_type = T; - using codec_type = Codec; - using load_error = typename Codec::load_error; - using save_error = typename Codec::save_error; - - explicit RawStorage_(disk::Layout layout) noexcept : _layout(std::move(layout)) {} - - ~RawStorage_() = default; - RawStorage_ &operator=(const RawStorage_ &) = delete; - RawStorage_(const RawStorage_ &) = delete; - RawStorage_(RawStorage_ &&) = default; - RawStorage_ &operator=(RawStorage_ &&) = default; - - tl::expected load(const Id &id) const noexcept { - const auto path = this->path_for(id); - return Codec::load_from_path(path); - } - - tl::expected save(const Id &id, const T &node) const noexcept { - const auto path = this->path_for(id); - return Codec::save_to_path(node, path); - } - - tl::expected copy_to(const Id &id, RawStorage_ &target) const noexcept { - return target.copy_from(id, *this); - } - - tl::expected copy_from(const Id &id, const RawStorage_ &source) noexcept { - if (!source.has(id)) { - return tl::unexpected(CopyErrorKind::FileNotFound); - } - - const auto source_path = source.path_for(id); - const auto target_path = this->path_for(id); - if (source_path.extension() != target_path.extension()) { - // TODO: should this error instead? - const auto load_result = source.load(id); - if (!load_result.has_value()) { - return tl::unexpected(CopyErrorKind::Read); - } - const value_type node = load_result.value(); - - const auto save_result = this->save(id, node); - if (!save_result.has_value()) { - return tl::unexpected(CopyErrorKind::Write); - } - return {}; - } - - std::error_code ec; - if (std::filesystem::remove(target_path, ec)) { - if (ec) { - return tl::unexpected(CopyErrorKind::RemoveOld); - } - } - - std::filesystem::create_directories(target_path.parent_path(), ec); - if (ec) { - return tl::unexpected(CopyErrorKind::CreateDirectories); - } - - std::filesystem::create_hard_link(source_path, target_path, ec); - if (ec) { - return tl::unexpected(CopyErrorKind::CreateLink); - } - - return {}; - } - - bool remove(const Id &id) const noexcept { - const auto path = this->path_for(id); - std::error_code ec; - const auto removed = std::filesystem::remove(path, ec); - return !ec && removed; - } - - bool has(const Id &id) const noexcept { - const auto path = this->path_for(id); - std::error_code ec; - const auto exists = std::filesystem::exists(path, ec); - return !ec && exists; - } - - std::filesystem::path path_for(const Id &id) const noexcept { - return this->_layout.get_node_path(id); - } - - std::filesystem::path base_path() const noexcept { - return this->_layout.base_path(); - } - - const disk::Layout &layout() const noexcept { - return this->_layout; - } - -private: - disk::Layout _layout; -}; - -} // namespace octree diff --git a/src/terrainlib/octree/storage/Storage.h b/src/terrainlib/octree/storage/Storage.h deleted file mode 100644 index 7fbfc9eb..00000000 --- a/src/terrainlib/octree/storage/Storage.h +++ /dev/null @@ -1,312 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include - -#include - -#include "mesh/io.h" -#include "octree/Id.h" -#include "octree/IndexMap.h" -#include "octree/disk/Layout.h" -#include "octree/disk/layout/strategy/Default.h" -#include "octree/NodeStatusOrMissing.h" -#include "octree/storage/RawStorage.h" -#include "octree/storage/cache/ICache.h" -#include "octree/storage/codec/Codec.h" -#include "octree/storage/defaults.h" -#include "octree/storage/helpers.h" - -namespace octree { - -namespace detail { -struct MaybeIndex { - std::optional map; - mutable bool dirty = false; - - explicit MaybeIndex() : map(std::nullopt) {} - explicit MaybeIndex(IndexMap map) : map(std::move(map)) {} - - MaybeIndex(const MaybeIndex &) = default; - MaybeIndex &operator=(const MaybeIndex &) = default; - - MaybeIndex(MaybeIndex &&other) noexcept - : map(std::move(other.map)), dirty(std::exchange(other.dirty, false)) { - other.map.reset(); - } - - MaybeIndex &operator=(MaybeIndex &&other) noexcept { - this->map = std::move(other.map); - this->dirty = std::exchange(other.dirty, false); - other.map.reset(); - return *this; - } - - bool add(const Id &id) noexcept { - if (!map.has_value()) { - return false; - } - - const bool changed = map->add(id); - dirty = dirty || changed; - return changed; - } - - bool remove(const Id &id) noexcept { - if (!map.has_value()) { - return false; - } - - const bool changed = map->remove(id); - dirty = dirty || changed; - return changed; - } - - bool contains(const Id &id, const bool def = false) const noexcept { - if (this->map.has_value()) { - const octree::NodeStatusOrMissing status = this->map->get(id); - return status == octree::NodeStatusOrMissing::Leaf || status == NodeStatusOrMissing::Inner; - } - return def; - } -}; - -template -struct MaybeCache : public cache::ICache { - std::optional>> cache; - - explicit MaybeCache() : cache(std::nullopt) {} - explicit MaybeCache(std::unique_ptr> cache) : cache(std::move(cache)) {} - - std::optional get(const Id& id) noexcept override { - if (this->cache.has_value()) { - return this->cache->get()->get(id); - } - return std::nullopt; - } - - bool put(const Id &id, const T &value) noexcept override { - if (this->cache.has_value()) { - return this->cache->get()->put(id, value); - } - return false; - } - - bool remove(const Id &id) noexcept override { - if (this->cache.has_value()) { - return this->cache->get()->remove(id); - } - return false; - } - - bool contains(const Id &id) const noexcept override { - if (this->cache.has_value()) { - return this->cache->get()->contains(id); - } - return false; - } -}; -} - -struct StorageSettings { - bool allow_overwrite = false; -}; - -template Codec = DefaultCodecFor> -class Storage_ { -public: - using value_type = T; - using codec_type = Codec; - using load_error = typename Codec::load_error; - using save_error = typename Codec::save_error; - - explicit Storage_(RawStorage_ inner) - : _inner(std::move(inner)) {} - explicit Storage_(RawStorage_ inner, IndexMap index) - : _inner(std::move(inner)), _index(detail::MaybeIndex(std::move(index))) {} - - Storage_ &operator=(const Storage_ &) = delete; - Storage_(const Storage_ &) = delete; - Storage_(Storage_ &&) = default; - Storage_ &operator=(Storage_ &&) = default; - - virtual ~Storage_() { - if (this->_index.map.has_value() && this->_index.dirty) { - auto result = helpers::save_index_map(this->_index.map.value(), this->_inner.layout()); - if (!result.has_value()) { - LOG_ERROR("Failed to automatically save index when closing storage: {}", result.error()); - } - } - } - - tl::expected load(const Id &id) const noexcept { - if (const auto value_opt = this->_cache.get(id)) { - return value_opt.value(); - } - - if (!this->_index.contains(id, true)) { - return tl::unexpected(Codec::file_not_found()); - } - - const auto result = this->_inner.load(id); - if (result.has_value()) { - this->_cache.put(id, result.value()); - } - return result; - } - - tl::expected save(const Id &id, const value_type &value) noexcept { - if (this->check_overwrite(id)) { - LOG_ERROR_AND_EXIT("tried to overwrite value when not allowed"); - } - - const auto result = this->_inner.save(id, value); - if (result.has_value()) { - this->_cache.put(id, value); - this->_index.add(id); - } - return result; - } - - tl::expected copy_from(const Id &id, const Storage_ &source) noexcept { - if (!source._index.contains(id, true)) { - return tl::unexpected(CopyErrorKind::FileNotFound); - } - - if (this->check_overwrite(id)) { - // TODO: proper error - LOG_ERROR_AND_EXIT("tried to overwrite value when not allowed"); - } - - const auto result = this->_inner.copy_from(id, source._inner); - if (result.has_value()) { - this->_index.add(id); - } - return result; - } - - tl::expected copy_to(const Id &id, Storage_ &target) const noexcept { - return target.copy_from(id, *this); - } - - bool remove(const Id &id) noexcept { - this->_cache.remove(id); - this->_index.remove(id); - return this->_inner.remove(id); - } - - bool has(const Id &id) const noexcept { - if (this->is_indexed()) { - return this->_index.contains(id, false); - } else { - return this->_inner.has(id); - } - } - - std::filesystem::path path_for(const Id &id) const noexcept { - return this->_inner.path_for(id); - } - - std::filesystem::path base_path() const noexcept { - return this->_inner.base_path(); - } - - bool is_indexed() const noexcept { - return this->_index.map.has_value(); - } - - void ensure_indexed() noexcept { - if (this->is_indexed()) { - return; - } - - LOG_TRACE("Index not present, creating empty index"); - IndexMap map; - helpers::update_index_map(map, this->_inner.layout()); - LOG_TRACE("Index created with {} entries", map.size()); - this->_index.map = std::move(map); - } - - std::optional> index() const noexcept { - // I miss Option::as_ref - if (this->_index.map.has_value()) { - return this->_index.map.value(); - } else { - return std::nullopt; - } - } - - std::optional>> &cache() noexcept { - return this->_cache.cache; - } - - std::optional>> cache() const noexcept { - if (this->_cache.cache.has_value()) { - return *this->_cache.cache.value(); - } else { - return std::nullopt; - } - } - - tl::expected save_or_create_index() noexcept { - if (this->is_indexed() && !this->_index.dirty) { - return {}; - } - - this->ensure_indexed(); - - auto result = helpers::save_index_map(this->index().value(), this->_inner.layout()); - if (result.has_value()) { - this->_index.dirty = false; - } - return result; - } - - const octree::disk::Layout &layout() const noexcept { - return this->_inner.layout(); - } - - const StorageSettings& settings() const noexcept { - return this->_settings; - } - StorageSettings& settings() noexcept { - return this->_settings; - } - -protected: - bool check_overwrite(const Id &id) noexcept { - return !this->_settings.allow_overwrite && this->has(id); - } - - std::optional &index_mut() noexcept { - return this->_index.map; - } - - bool is_index_dirty() const noexcept { - return this->_index.dirty; - } - void set_index_dirty(const bool val = true) const noexcept { - this->_index.dirty = val; - } - - void update_index() noexcept { - if (!this->is_indexed()) { - return; - } - - helpers::update_index_map(this->index_mut().value(), this->_inner.layout()); - this->_index.dirty = false; - } - -private: - RawStorage_ _inner; - detail::MaybeIndex _index = detail::MaybeIndex(); - mutable detail::MaybeCache _cache = detail::MaybeCache(); - StorageSettings _settings = {}; -}; - -} // namespace octree diff --git a/src/terrainlib/octree/storage/cache/Dummy.h b/src/terrainlib/octree/storage/cache/Dummy.h deleted file mode 100644 index ecf6cff8..00000000 --- a/src/terrainlib/octree/storage/cache/Dummy.h +++ /dev/null @@ -1,27 +0,0 @@ -#pragma once - -#include - -#include "octree/Id.h" -#include "octree/storage/cache/ICache.h" - -namespace octree::cache { - - template -class Dummy : public ICache { -public: - std::optional get(const Id &) noexcept override { - return std::nullopt; - } - bool put(const Id &, const T &) noexcept override { - return false; - } - bool remove(const Id &) noexcept override { - return false; - } - bool contains(const Id &) const noexcept override { - return false; - } -}; - -} diff --git a/src/terrainlib/octree/storage/cache/ICache.h b/src/terrainlib/octree/storage/cache/ICache.h deleted file mode 100644 index bd0258bc..00000000 --- a/src/terrainlib/octree/storage/cache/ICache.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include - -#include "octree/Id.h" - -namespace octree::cache { - -template -class ICache { -public: - virtual ~ICache() = default; - - virtual std::optional get(const Id& id) noexcept = 0; - virtual bool put(const Id &id, const T &value) noexcept = 0; - virtual bool remove(const Id &id) noexcept = 0; - virtual bool contains(const Id &id) const noexcept = 0; -}; - -} diff --git a/src/terrainlib/octree/storage/cache/LruCache.h b/src/terrainlib/octree/storage/cache/LruCache.h deleted file mode 100644 index fcf4f20f..00000000 --- a/src/terrainlib/octree/storage/cache/LruCache.h +++ /dev/null @@ -1,68 +0,0 @@ -#pragma once - -#include -#include -#include - -#include "octree/Id.h" -#include "octree/RawStorage.h" -#include "octree/cache/ICache.h" - -namespace octree::cache { - -// TODO: UNTESTED -template -class Lru_ : public ICache { -public: - explicit Lru_(const size_t capacity) : _capacity(capacity) {} - - std::optional get(const Key& key) { - auto it = this->_map.find(key); - if (it == this->_map.end()) return std::nullopt; - - // Move to front (most recently used) - this->_usage.splice(this->_usage.begin(), this->_usage, it->second.second); - return it->second.first; - } - - void put(const Key& key, const Value& value) { - auto it = this->_map.find(key); - if (it != this->_map.end()) { - // Update value and move to front - it->second.first = value; - this->_usage.splice(this->_usage.begin(), this->_usage, it->second.second); - } else { - // Insert new entry - if (this->_map.size() == _capacity) { - // Evict least recently used - const Key& lru_key = this->_usage.back(); - this->_map.erase(lru_key); - this->_usage.pop_back(); - } - - this->_usage.push_front(key); - this->_map[key] = { value, this->_usage.begin() }; - } - } - - void remove(const Key& key) { - auto it = this->_map.find(key); - if (it != this->_map.end()) { - this->_usage.erase(it->second.second); - this->_map.erase(it); - } - } - - bool contains(const Key& key) const { - return this->_map.contains(key); - } - -private: - size_t _capacity; - std::list _usage; - std::unordered_map::iterator>> _map; -}; - -using Lru = Lru; - -} diff --git a/src/terrainlib/octree/storage/codec/Codec.h b/src/terrainlib/octree/storage/codec/Codec.h deleted file mode 100644 index b4c3bc47..00000000 --- a/src/terrainlib/octree/storage/codec/Codec.h +++ /dev/null @@ -1,32 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -template -concept CodecFor = - requires(const std::filesystem::path &path, const T &value) { - typename Codec::value_type; - typename Codec::load_error; - typename Codec::save_error; - - requires std::same_as; - - { Codec::load_from_path(path) } noexcept - -> std::same_as>; - - { Codec::save_to_path(value, path) } noexcept - -> std::same_as>; - - { Codec::file_not_found() } noexcept - -> std::same_as; - - { Codec::supported_extensions() } -> std::ranges::range; - - requires std::convertible_to< - std::ranges::range_value_t, - std::string_view>; - }; diff --git a/src/terrainlib/octree/storage/codec/MeshCodec.h b/src/terrainlib/octree/storage/codec/MeshCodec.h deleted file mode 100644 index 2dc65fb6..00000000 --- a/src/terrainlib/octree/storage/codec/MeshCodec.h +++ /dev/null @@ -1,35 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include "mesh/io.h" - -namespace octree { - -struct MeshCodec { - using value_type = mesh::Simple; - using load_error = mesh::io::LoadMeshError; - using save_error = mesh::io::SaveMeshError; - - static tl::expected load_from_path(const std::filesystem::path& path) noexcept { - return mesh::io::load_from_path(path); - } - - static tl::expected save_to_path(const value_type& value, const std::filesystem::path& path) noexcept { - return mesh::io::save_to_path(value, path); - } - - static load_error file_not_found() noexcept { - return mesh::io::LoadMeshErrorKind::FileNotFound; - } - - static std::vector supported_extensions() { - return {".terrain", ".glb", ".gltf"}; - } -}; - -} // namespace octree diff --git a/src/terrainlib/octree/storage/codec/ReadOnlyCodec.h b/src/terrainlib/octree/storage/codec/ReadOnlyCodec.h deleted file mode 100644 index 2d397cb2..00000000 --- a/src/terrainlib/octree/storage/codec/ReadOnlyCodec.h +++ /dev/null @@ -1,21 +0,0 @@ -#pragma once - -#include - -#include - -#include "log.h" - -namespace octree { - -// Wraps a codec so saving always fails loudly instead of writing, for storages that must stay -// read-only (e.g. a view that aliases another storage's files). -template -struct ReadOnlyCodec : Codec { - static tl::expected save_to_path( - const typename Codec::value_type &, const std::filesystem::path &) noexcept { - LOG_ERROR_AND_EXIT("Attempted to write through a read-only codec"); - } -}; - -} // namespace octree diff --git a/src/terrainlib/octree/storage/codec/ZppBitsCodec.h b/src/terrainlib/octree/storage/codec/ZppBitsCodec.h deleted file mode 100644 index dd70a27b..00000000 --- a/src/terrainlib/octree/storage/codec/ZppBitsCodec.h +++ /dev/null @@ -1,38 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include "Codec.h" -#include "io/serialize.h" -#include "io/Error.h" - -namespace octree { - -template -struct ZppBitsCodec { - using value_type = T; - using load_error = io::Error; - using save_error = io::Error; - - static tl::expected load_from_path(const std::filesystem::path& path) noexcept { - return io::read_from_path(path); - } - - static tl::expected save_to_path(const value_type& value, const std::filesystem::path& path) noexcept { - return io::write_to_path(value, path); - } - - static load_error file_not_found() noexcept { - return io::Error::Value::OpenFile; - } - - static std::vector supported_extensions() { - return {".bin"}; - } -}; - -} // namespace octree diff --git a/src/terrainlib/octree/storage/defaults.h b/src/terrainlib/octree/storage/defaults.h deleted file mode 100644 index fb4981c4..00000000 --- a/src/terrainlib/octree/storage/defaults.h +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once - -#include "mesh/SimpleMesh.h" -#include "codec/Codec.h" -#include "codec/MeshCodec.h" -#include "codec/ZppBitsCodec.h" - -namespace octree { - -using DefaultT = mesh::Simple; - -template -struct DefaultCodec { - using type = ZppBitsCodec; -}; - -template <> -struct DefaultCodec { - using type = MeshCodec; -}; - -template -using DefaultCodecFor = typename DefaultCodec>::type; - -} // namespace octree \ No newline at end of file diff --git a/src/terrainlib/octree/storage/helpers.cpp b/src/terrainlib/octree/storage/helpers.cpp deleted file mode 100644 index 7366be55..00000000 --- a/src/terrainlib/octree/storage/helpers.cpp +++ /dev/null @@ -1,136 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include - -#include "octree/storage/helpers.h" -#include "io/serialize.h" -#include "octree/NodeStatus.h" -#include "octree/IndexMap.h" -#include "octree/disk/IndexFile.h" -#include "octree/disk/Layout.h" -#include "octree/disk/layout/Strategy.h" -#include "octree/disk/layout/StrategyRegister.h" - -namespace octree::helpers { -namespace { -template -std::optional find_max_key(const std::unordered_map& map) { - if (map.empty()) { - return std::nullopt; - } - - auto max_it = map.begin(); - for (auto it = std::next(max_it); it != map.end(); it++) { - if (it->second > max_it->second) { - max_it = it; - } - } - - return max_it->first; -} -} - -std::optional guess_layout_strategy( - const std::filesystem::path &base_path, - size_t max_files_to_check) { - - if (!std::filesystem::is_directory(base_path)) { - return std::nullopt; - } - - std::vector candidate_paths; - std::unordered_map extension_counters; - for (const auto &entry : std::filesystem::recursive_directory_iterator(base_path)) { - if (max_files_to_check == 0) { - break; - } - if (!entry.is_regular_file() && !entry.is_symlink()) { - continue; - } - - const auto ext = entry.path().extension(); - if (ext == disk::v1::index_extension()) { - continue; - } - extension_counters[ext]++; - - candidate_paths.emplace_back(std::filesystem::relative(entry.path(), base_path)); - max_files_to_check--; - } - std::optional most_common_ext = find_max_key(extension_counters); - - std::unique_ptr best_strategy; - size_t best_match_count = 0; - const auto &factories = disk::layout::StrategyRegister::instance().factories(); - for (const auto &[_, make_strategy] : factories) { - auto strategy = make_strategy(); // assume returns unique_ptr - - size_t match_count = 0; - for (const auto &rel_path : candidate_paths) { - if (strategy->get_id_from_relative_node_path(rel_path).has_value()) { - match_count++; - } - } - - if (match_count > best_match_count) { - best_match_count = match_count; - best_strategy = std::move(strategy); - } - } - - if (best_strategy && most_common_ext.has_value()) { - return LayoutWithoutBase(std::move(best_strategy), most_common_ext.value()); - } - - return std::nullopt; -} - -tl::expected save_index_map(const IndexMap& index, const disk::Layout& layout) { - const auto index_path = layout.base_path() / disk::v1::index_file_name(); - LOG_TRACE("Saving octree storage index to {}", index_path); - - disk::v1::IndexFile index_file; - index_file.map = index; - index_file.preferred_extension = layout.extension_with_dot(); - index_file.layout_strategy_id = disk::layout::StrategyRegister::instance().get_id(layout.strategy()); - - const auto result = io::write_to_path(index_file, index_path); - if (!result.has_value()) { - LOG_ERROR("Failed to save octree storage index to {}", index_path); - return tl::unexpected(result.error()); - } - return {}; -} - -void update_index_map(IndexMap &index, const disk::Layout &layout) { - index.clear(); - const auto &base_path = layout.base_path(); - - std::filesystem::create_directories(base_path); - - for (const auto &entry : std::filesystem::recursive_directory_iterator(base_path)) { - if (!entry.is_regular_file() && !entry.is_symlink()) { - continue; - } - - const auto ext = entry.path().extension(); - if (ext == disk::v1::index_extension()) { - continue; - } - - auto id_opt = layout.get_id_from_node_path(entry.path()); - if (!id_opt) { - continue; - } - - const Id id = *id_opt; - index.add(id); - } -} - -} diff --git a/src/terrainlib/octree/storage/helpers.h b/src/terrainlib/octree/storage/helpers.h deleted file mode 100644 index a7dff810..00000000 --- a/src/terrainlib/octree/storage/helpers.h +++ /dev/null @@ -1,28 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include "io/Error.h" -#include "octree/IndexMap.h" -#include "octree/disk/IndexFile.h" -#include "octree/disk/Layout.h" -#include "octree/disk/layout/Strategy.h" - -namespace octree::helpers { - -struct LayoutWithoutBase { - std::unique_ptr strategy; - std::string extension_with_dot; -}; - -std::optional guess_layout_strategy( - const std::filesystem::path &base_path, - size_t max_files_to_check = 100); -tl::expected save_index_map(const IndexMap &index, const disk::Layout &layout); -void update_index_map(IndexMap &index, const disk::Layout &layout); - -} diff --git a/src/terrainlib/octree/storage/open.h b/src/terrainlib/octree/storage/open.h index ec212e88..cfd10db0 100644 --- a/src/terrainlib/octree/storage/open.h +++ b/src/terrainlib/octree/storage/open.h @@ -1,37 +1,29 @@ #pragma once #include -#include -#include +#include -#include - -#include "io/Error.h" -#include "octree/disk/layout/strategy/Default.h" -#include "octree/storage/IndexedStorage.h" -#include "octree/storage/Storage.h" -#include "octree/storage/codec/Codec.h" -#include "octree/storage/defaults.h" +#include "mesh/storage.h" namespace octree { -struct OpenOptions { - std::unique_ptr default_layout_strategy = {}; - std::optional preferred_extension_with_dot = {}; -}; - -template Codec = DefaultCodecFor> -tl::expected, io::Error> open_index(const std::filesystem::path &index_path); -template Codec = DefaultCodecFor> -Storage_ open_folder( - const std::filesystem::path &base_path, - const bool create_index = false, - OpenOptions options = {}); -template Codec = DefaultCodecFor> -IndexedStorage_ open_folder_indexed( - const std::filesystem::path &base_path, - OpenOptions options = {}); +using OpenOptions = mesh::storage::OpenOptions; + +inline std::expected> open_index( + const std::filesystem::path &path) { + return mesh::storage::open_index(path); } -#include "open.inl" +inline std::expected> open_folder( + const std::filesystem::path &path, + OpenOptions options = {}) { + return mesh::storage::open_folder(path, std::move(options)); +} + +inline std::expected> open_folder_indexed( + const std::filesystem::path &path, + OpenOptions options = {}) { + return mesh::storage::open_folder_indexed(path, std::move(options)); +} +} // namespace octree diff --git a/src/terrainlib/octree/storage/open.inl b/src/terrainlib/octree/storage/open.inl deleted file mode 100644 index e0c9b0d1..00000000 --- a/src/terrainlib/octree/storage/open.inl +++ /dev/null @@ -1,108 +0,0 @@ -#include -#include -#include -#include -#include - -#include - -#include "io/serialize.h" -#include "log.h" -#include "io/Error.h" -#include "octree/disk/IndexFile.h" -#include "octree/disk/layout/StrategyRegister.h" -#include "octree/storage/IndexedStorage.h" -#include "octree/storage/helpers.h" -#include "octree/storage/defaults.h" -#include "range_utils.h" - -namespace octree { - -template Codec> -tl::expected, io::Error> open_index(const std::filesystem::path &index_path) { - LOG_TRACE("Opening storage index {}", index_path); - - const auto result = io::read_from_path(index_path); - if (!result.has_value()) { - LOG_TRACE("Failed to open storage index due to {}", result.error()); - return tl::unexpected(result.error()); - } - auto index_file = result.value(); - LOG_TRACE("Successfully read storage index with {} entries.", index_file.map.size()); - auto index_map = std::move(index_file.map); - const auto base_path = index_path.parent_path(); - auto layout_strategy = disk::layout::StrategyRegister::instance().create(index_file.layout_strategy_id); - disk::Layout layout(base_path, std::move(layout_strategy), index_file.preferred_extension); - RawStorage_ raw_storage(std::move(layout)); - return IndexedStorage_(std::move(raw_storage), std::move(index_map)); -} - -template Codec> -Storage_ open_folder( - const std::filesystem::path &base_path, - bool create_index, - OpenOptions options) { - LOG_TRACE("Opening storage folder {}", base_path); - - if (!std::filesystem::is_directory(base_path)) { - if (std::filesystem::exists(base_path)) { - LOG_ERROR_AND_EXIT("Base path {} exists but is not a directory", base_path); - } - - LOG_TRACE("Base path {} does not exist, creating it", base_path); - std::filesystem::create_directories(base_path); - } - - const std::filesystem::path index_path = base_path / disk::v1::index_file_name(); - auto storage_opt = open_index(index_path); - if (storage_opt.has_value()) { - LOG_TRACE("Loaded existing index"); - return Storage_(std::move(storage_opt.value())); - } - - auto layout_info_opt = helpers::guess_layout_strategy(base_path); - if (layout_info_opt.has_value()) { - const octree::helpers::LayoutWithoutBase &layout_info = layout_info_opt.value(); - LOG_TRACE("Guessed layout strategy of dataset as {} and extension as {}", - disk::layout::StrategyRegister::instance().get_id(*layout_info.strategy), - layout_info.extension_with_dot); - } else { - auto default_layout_strategy = std::move(options.default_layout_strategy); - if (!default_layout_strategy) { - default_layout_strategy = disk::layout::strategy::make_default(); - } - std::string default_extension_with_dot; - const std::vector supported_extensions = Codec::supported_extensions(); - if (options.preferred_extension_with_dot.has_value()) { - ASSERT(contains(supported_extensions, options.preferred_extension_with_dot.value())); - default_extension_with_dot = options.preferred_extension_with_dot.value(); - } else { - default_extension_with_dot = supported_extensions[0]; - } - LOG_WARN("Unable to determine layout of dataset, using layout strategy {} and extension {}", - disk::layout::StrategyRegister::instance().get_id(*default_layout_strategy), - default_extension_with_dot); - layout_info_opt = octree::helpers::LayoutWithoutBase(std::move(default_layout_strategy), default_extension_with_dot); - } - - disk::Layout layout(base_path, std::move(layout_info_opt->strategy), layout_info_opt->extension_with_dot); - if (!create_index) { - return Storage_(RawStorage_(std::move(layout))); - } - - IndexMap map; - helpers::update_index_map(map, layout); - if (!map.empty()) { - helpers::save_index_map(map, layout); - } - return Storage_(RawStorage_(std::move(layout)), std::move(map)); -} - -template Codec> -IndexedStorage_ open_folder_indexed( - const std::filesystem::path &base_path, - OpenOptions options) { - return IndexedStorage_(open_folder(base_path, true, std::move(options))); -} - -} // namespace octree diff --git a/src/terrainlib/octree/storage/open_runtime.h b/src/terrainlib/octree/storage/open_runtime.h new file mode 100644 index 00000000..1994bfd9 --- /dev/null +++ b/src/terrainlib/octree/storage/open_runtime.h @@ -0,0 +1,134 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include "octree/StoreTraits.h" +#include "octree/storage/IndexFile.h" +#include "store/open.h" + +namespace octree::storage { + +struct OpenOptions { + std::optional> default_mapping; + std::optional preferred_extension; +}; + +template +std::expected, store::OpenError> open_folder( + const std::filesystem::path &base_path, + std::string payload_class, + std::string default_codec_selector, + CodecResolver resolve_codec, + OpenOptions options = {}) +{ + std::error_code error; + const bool base_exists = std::filesystem::exists(base_path, error); + if (error) { + return std::unexpected(store::OpenError(store::FilesystemError{ + base_path, + "exists", + error, + })); + } + if (base_exists && !std::filesystem::is_directory(base_path, error)) { + return std::unexpected(store::OpenError(store::FilesystemError{ + base_path, + "is_directory", + error ? error : std::make_error_code(std::errc::not_a_directory), + })); + } + if (!base_exists) { + std::filesystem::create_directories(base_path, error); + if (error) { + return std::unexpected(store::OpenError(store::FilesystemError{ + base_path, + "create_directories", + error, + })); + } + } + + const auto format = index_format(); + const std::filesystem::path index_path = base_path / format.index_filename; + const std::filesystem::path metadata_path = base_path / metadata_file_name; + const bool index_exists = std::filesystem::exists(index_path, error); + if (error) { + return std::unexpected(store::OpenError(store::FilesystemError{ + index_path, + "exists", + error, + })); + } + const bool metadata_exists = std::filesystem::exists(metadata_path, error); + if (error) { + return std::unexpected(store::OpenError(store::FilesystemError{ + metadata_path, + "exists", + error, + })); + } + if (index_exists != metadata_exists) { + return std::unexpected(store::OpenError(store::IndexFormatError{ + store::IndexFormatErrorCategory::Open, + index_exists ? metadata_path : index_path, + "octree dataset metadata and index must either both exist or both be absent", + })); + } + if (index_exists) { + auto indexed = store::open_index( + index_path, + format, + payload_class, + resolve_codec); + if (!indexed) { + return std::unexpected(indexed.error()); + } + return store::Storage(std::move(*indexed)); + } + + const store::PathMapping mapping = + options.default_mapping.value_or(format.default_mapping()); + std::string codec_selector = + options.preferred_extension.value_or(std::move(default_codec_selector)); + auto codec = resolve_codec(codec_selector); + if (!codec) { + return std::unexpected(store::OpenError(codec.error())); + } + + return store::make_storage( + base_path, + format, + mapping, + std::move(payload_class), + std::move(codec_selector), + std::move(*codec)); +} + +template +std::expected, store::OpenError> +open_folder_indexed( + const std::filesystem::path &base_path, + std::string payload_class, + std::string default_codec_selector, + CodecResolver resolve_codec, + OpenOptions options = {}) +{ + auto result = open_folder( + base_path, + std::move(payload_class), + std::move(default_codec_selector), + std::move(resolve_codec), + std::move(options)); + if (!result) { + return std::unexpected(result.error()); + } + return store::IndexedStorage(std::move(*result)); +} + +} // namespace octree::storage diff --git a/src/terrainlib/octree/store_layout/Flat.h b/src/terrainlib/octree/store_layout/Flat.h new file mode 100644 index 00000000..c7f58a7f --- /dev/null +++ b/src/terrainlib/octree/store_layout/Flat.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include +#include +#include + +#include + +#include "octree/Id.h" +#include "store/NodePath.h" +#include "string_utils.h" + +namespace octree::store_layout { + +inline store::NodePath flat_key_to_node_path(const Id &id) { + return store::NodePath(fmt::format("{}-{}", id.level(), id.index_on_level())); +} + +inline std::optional flat_node_path_to_key(const store::NodePath &node_path) { + const std::filesystem::path &path = node_path.path(); + if (path.empty() || path.is_absolute() || path.has_parent_path() || path.has_extension()) { + return std::nullopt; + } + + const std::string text = path.string(); + const size_t separator = text.find('-'); + if (separator == std::string::npos || separator != text.rfind('-')) { + return std::nullopt; + } + const auto level = from_chars(std::string_view(text).substr(0, separator)); + const auto index = from_chars(std::string_view(text).substr(separator + 1)); + if (!level.has_value() || !index.has_value()) { + return std::nullopt; + } + return Id::try_make(level.value(), index.value()); +} + +} // namespace octree::store_layout diff --git a/src/terrainlib/octree/store_layout/LevelAndCoordinateDirectories.h b/src/terrainlib/octree/store_layout/LevelAndCoordinateDirectories.h new file mode 100644 index 00000000..7036443a --- /dev/null +++ b/src/terrainlib/octree/store_layout/LevelAndCoordinateDirectories.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include +#include + +#include + +#include "octree/Id.h" +#include "store/NodePath.h" +#include "string_utils.h" + +namespace octree::store_layout { + +inline store::NodePath level_and_coordinate_key_to_node_path(const Id &id) { + const Id::Coords coordinates = id.coords(); + return store::NodePath(fmt::format( + "{}/{}/{}/{}", + id.level(), + coordinates.x, + coordinates.y, + coordinates.z)); +} + +inline std::optional level_and_coordinate_node_path_to_key( + const store::NodePath &node_path) { + const std::filesystem::path &path = node_path.path(); + if (path.empty() || path.is_absolute() || std::distance(path.begin(), path.end()) != 4) { + return std::nullopt; + } + + auto part = path.begin(); + const auto level = from_chars(part->string()); + ++part; + const auto x = from_chars(part->string()); + ++part; + const auto y = from_chars(part->string()); + ++part; + if (part->has_extension()) { + return std::nullopt; + } + const auto z = from_chars(part->string()); + if (!level.has_value() || !x.has_value() || !y.has_value() || !z.has_value()) { + return std::nullopt; + } + return Id::try_make(level.value(), {x.value(), y.value(), z.value()}); +} + +} // namespace octree::store_layout diff --git a/src/terrainlib/octree/store_layout/Mappings.h b/src/terrainlib/octree/store_layout/Mappings.h new file mode 100644 index 00000000..1b46bed8 --- /dev/null +++ b/src/terrainlib/octree/store_layout/Mappings.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include +#include + +#include "octree/store_layout/Flat.h" +#include "octree/store_layout/LevelAndCoordinateDirectories.h" +#include "store/PathMapping.h" + +namespace octree::store_layout { + +inline store::PathMapping flat() { + return {"flat", flat_key_to_node_path, flat_node_path_to_key}; +} + +inline store::PathMapping level_and_coordinate_directories() { + return { + "level_and_coordinate_directories", + level_and_coordinate_key_to_node_path, + level_and_coordinate_node_path_to_key, + }; +} + +inline std::array, 2> all() { + return {flat(), level_and_coordinate_directories()}; +} + +inline std::optional> from_id(const std::string_view id) { + for (const auto mapping : all()) { + if (mapping.id == id) { + return mapping; + } + } + return std::nullopt; +} + +} // namespace octree::store_layout diff --git a/src/terrainlib/octree/traverse.h b/src/terrainlib/octree/traverse.h deleted file mode 100644 index b912a0fa..00000000 --- a/src/terrainlib/octree/traverse.h +++ /dev/null @@ -1,81 +0,0 @@ -#pragma once - -#include -#include - -#include "octree/Id.h" -#include "octree/Storage.h" -#include "log.h" - -namespace octree { - -enum class TraversalOrder { - DepthFirst, - BreadthFirst -}; - -constexpr bool always_refine(const Id &) { - return true; -} - -template < - typename VisitFn, - typename RefineFn = std::function> -void traverse( - const IndexMap &index, - VisitFn &&visit_fn, - RefineFn &&refine_fn = always_refine, - const Id &root = Id::root(), - TraversalOrder order = TraversalOrder::DepthFirst) { - if (!index.is_present(root)) { - return; - } - - if (order == TraversalOrder::DepthFirst) { - std::function dfs; - dfs = [&](const Id& current) { - auto current_status_opt = index.get(current); - if (!current_status_opt) { - return; - } - const auto current_status = current_status_opt.value(); - - visit_fn(current, current_status); - - if (current.has_children() && refine_fn(current)) { - const auto children = current.children().value(); - for (const auto& child : children) { - dfs(child); - } - } - }; - dfs(root); - } else if (order == TraversalOrder::BreadthFirst) { - std::queue queue; - queue.push(root); - - while (!queue.empty()) { - Id current = queue.front(); - queue.pop(); - - auto current_status_opt = index.get(current); - if (!current_status_opt) { - continue; - } - const auto current_status = current_status_opt.value(); - - visit_fn(current, current_status); - - if (current.has_children() && refine_fn(current)) { - const auto children = current.children().value(); - for (const auto& child : children) { - queue.push(child); - } - } - } - } else { - UNREACHABLE(); - } -} - -} // namespace octree diff --git a/src/terrainlib/pch.h b/src/terrainlib/pch.h index dce9cf57..a936589c 100644 --- a/src/terrainlib/pch.h +++ b/src/terrainlib/pch.h @@ -34,8 +34,7 @@ #include #include #include -#include -#include +#include // Internal headers #include "log.h" diff --git a/src/terrainlib/raster_store/StoreTraits.h b/src/terrainlib/raster_store/StoreTraits.h new file mode 100644 index 00000000..a6ba4a8b --- /dev/null +++ b/src/terrainlib/raster_store/StoreTraits.h @@ -0,0 +1,45 @@ +#pragma once + +#include +#include +#include +#include + +#include + +namespace raster_store { + +struct StoreTraits { + using Key = radix::tile::Id; + using Hasher = Key::Hasher; + + static constexpr unsigned max_zoom_level = std::numeric_limits::digits; + + static Key root() { + return {0, {0, 0}}; + } + static std::optional parent(const Key &key) { + if (key.zoom_level == 0) { + return std::nullopt; + } + return key.parent(); + } + static std::optional> children(const Key &key) { + if (key.zoom_level >= max_zoom_level) { + return std::nullopt; + } + return key.children(); + } + static bool is_valid(const Key &key) { + if (key.zoom_level > max_zoom_level) { + return false; + } + if (key.zoom_level == max_zoom_level) { + return true; + } + const uint32_t extent = uint32_t{1} << key.zoom_level; + return key.coords.x < extent && key.coords.y < extent; + } +}; + +} // namespace raster_store diff --git a/src/terrainlib/sf/Error.h b/src/terrainlib/sf/Error.h new file mode 100644 index 00000000..a8274c72 --- /dev/null +++ b/src/terrainlib/sf/Error.h @@ -0,0 +1,42 @@ +#pragma once + +#include +#include +#include + +#include "octree/Id.h" +#include "sf/InvalidTopology.h" +#include "store/OpenError.h" +#include "store/describe_error.h" + +namespace sf { + +using FinalizeError = std::variant; + +using ProcessingError = std::variant< + InvalidTopology, + store::OpenError, + store::FileOperationError, + store::SaveError, + store::CopyError, + store::IndexFormatError>; + +inline std::string describe_error(const InvalidTopology &error) { + return "Structura Fundamentalis topology contains Inner node " + + error.key.to_string(); +} + +template +std::string describe_error(const std::variant &error) { + return std::visit( + [](const auto &value) -> std::string { + if constexpr (std::is_same_v, InvalidTopology>) { + return describe_error(value); + } else { + return store::describe_error(value); + } + }, + error); +} + +} // namespace sf diff --git a/src/terrainlib/sf/InvalidTopology.h b/src/terrainlib/sf/InvalidTopology.h new file mode 100644 index 00000000..643d769d --- /dev/null +++ b/src/terrainlib/sf/InvalidTopology.h @@ -0,0 +1,13 @@ +#pragma once + +#include "octree/Id.h" + +namespace sf { + +struct InvalidTopology { + octree::Id key; + + bool operator==(const InvalidTopology &) const = default; +}; + +} // namespace sf diff --git a/src/terrainlib/sf/finalize_storage.h b/src/terrainlib/sf/finalize_storage.h new file mode 100644 index 00000000..157d9852 --- /dev/null +++ b/src/terrainlib/sf/finalize_storage.h @@ -0,0 +1,28 @@ +#pragma once + +#include + +#include + +#include "mesh/storage.h" +#include "sf/Error.h" +#include "sf/validate_index.h" + +namespace sf { + +inline std::expected finalize_storage(mesh::storage::Storage &storage) { + const auto save_result = storage.save_or_create_index(); + if (!save_result.has_value()) { + return std::unexpected(FinalizeError(save_result.error())); + } + + const auto index = storage.index(); + DEBUG_ASSERT(index.has_value()); + const auto validation = validate_index(index->get()); + if (!validation.has_value()) { + return std::unexpected(FinalizeError(validation.error())); + } + return {}; +} + +} // namespace sf diff --git a/src/terrainlib/sf/validate_index.h b/src/terrainlib/sf/validate_index.h new file mode 100644 index 00000000..1cf33719 --- /dev/null +++ b/src/terrainlib/sf/validate_index.h @@ -0,0 +1,21 @@ +#pragma once + +#include + +#include "octree/StoreTraits.h" +#include "sf/InvalidTopology.h" +#include "store/Index.h" + +namespace sf { + +inline std::expected validate_index( + const store::Index &index) { + for (const auto &[key, status] : index) { + if (status == store::NodeStatus::Inner) { + return std::unexpected(InvalidTopology{key}); + } + } + return {}; +} + +} // namespace sf diff --git a/src/terrainlib/srs.h b/src/terrainlib/srs.h index af8308f5..a5e5ce5c 100644 --- a/src/terrainlib/srs.h +++ b/src/terrainlib/srs.h @@ -32,7 +32,7 @@ #include #include #include -#include +#include #include namespace srs { @@ -352,19 +352,19 @@ inline radix::geometry::Aabb3d encompassing_bounds_transfer( return encompassing_bounds_transfer(transform.get(), source_bounds, intermediate_points_edges, intermediate_points_faces); } -inline tl::expected from_epsg(const uint32_t epsg) { +inline std::expected from_epsg(const uint32_t epsg) { OGRSpatialReference srs; if (srs.importFromEPSG(epsg) != OGRERR_NONE) { - return tl::unexpected(fmt::format("Failed to import spatial reference from EPSG code: {}", epsg)); + return std::unexpected(fmt::format("Failed to import spatial reference from EPSG code: {}", epsg)); } srs.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); return srs; } -inline tl::expected from_user_input(const std::string &user_input) { +inline std::expected from_user_input(const std::string &user_input) { OGRSpatialReference srs; if (srs.SetFromUserInput(user_input.c_str()) != OGRERR_NONE) { - return tl::unexpected(fmt::format("Failed to set spatial reference from user input: {}", user_input)); + return std::unexpected(fmt::format("Failed to set spatial reference from user input: {}", user_input)); } srs.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); return srs; diff --git a/src/terrainlib/store/Codec.h b/src/terrainlib/store/Codec.h new file mode 100644 index 00000000..c9611589 --- /dev/null +++ b/src/terrainlib/store/Codec.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include + +#include + +#include "store/CodecError.h" +#include "store/NodePath.h" + +namespace store { + +template +class Codec { +public: + virtual ~Codec() = default; + + virtual std::vector paths(const NodePath &node_path) const = 0; + + virtual std::expected read(const NodePath &) const { + return std::unexpected( + CodecError::unsupported_operation(CodecOperation::Read, "read")); + } + + virtual std::expected write(const NodePath &, const NodeData &) const { + return std::unexpected( + CodecError::unsupported_operation(CodecOperation::Write, "write")); + } +}; + +} // namespace store diff --git a/src/terrainlib/store/CodecError.h b/src/terrainlib/store/CodecError.h new file mode 100644 index 00000000..ba4adeb8 --- /dev/null +++ b/src/terrainlib/store/CodecError.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include +#include + +namespace store { + +enum class CodecOperation { + Read, + Write, + Resolve, +}; + +enum class CodecErrorCategory { + UnsupportedOperation, + UnsupportedCodec, + FileNotFound, + InvalidData, + Io, + Domain, +}; + +struct CodecError { + CodecOperation operation; + CodecErrorCategory category; + std::string message; + + static CodecError unsupported_operation( + const CodecOperation operation, + const std::string_view name) { + return {operation, CodecErrorCategory::UnsupportedOperation, std::string(name)}; + } + + bool operator==(const CodecError &) const = default; +}; + +} // namespace store diff --git a/src/terrainlib/store/Index.h b/src/terrainlib/store/Index.h new file mode 100644 index 00000000..024e3b2b --- /dev/null +++ b/src/terrainlib/store/Index.h @@ -0,0 +1,234 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include "log.h" +#include "store/InvalidKey.h" +#include "store/NodeStatus.h" +#include "store/Traits.h" + +namespace store { + +template +class Index { +public: + using Key = typename Traits::Key; + using Error = InvalidKey; + using Container = std::unordered_map; + using iterator = typename Container::iterator; + using const_iterator = typename Container::const_iterator; + + std::expected, Error> get(const Key &key) const { + if (!Traits::is_valid(key)) { + return std::unexpected(Error{key}); + } + const NodeStatus *status = get_raw_unchecked(key); + if (status == nullptr) { + return std::nullopt; + } + return *status; + } + + std::expected add(const Key &key) { + if (!Traits::is_valid(key)) { + return std::unexpected(Error{key}); + } + + NodeStatus *status = get_raw_unchecked(key); + if (status != nullptr) { + switch (*status) { + case NodeStatus::Inner: + case NodeStatus::Leaf: + return false; + case NodeStatus::Virtual: + set_raw_unchecked(key, NodeStatus::Inner); + return true; + } + } + + const auto parent = Traits::parent(key); + if (!parent.has_value()) { + DEBUG_ASSERT(key == Traits::root()); + if (empty()) { + set_raw_unchecked(key, NodeStatus::Leaf); + return true; + } + UNREACHABLE(); + } + + if (!Traits::is_valid(parent.value())) { + return std::unexpected(Error{parent.value()}); + } + NodeStatus *parent_status = get_raw_unchecked(parent.value()); + if (parent_status == nullptr) { + const auto parent_result = add(parent.value()); + if (!parent_result.has_value()) { + return std::unexpected(parent_result.error()); + } + set_raw_unchecked(parent.value(), NodeStatus::Virtual); + set_raw_unchecked(key, NodeStatus::Leaf); + return true; + } + + switch (*parent_status) { + case NodeStatus::Leaf: + set_raw_unchecked(parent.value(), NodeStatus::Inner); + set_raw_unchecked(key, NodeStatus::Leaf); + break; + case NodeStatus::Inner: + case NodeStatus::Virtual: + set_raw_unchecked(key, NodeStatus::Leaf); + break; + } + return true; + } + + std::expected remove(const Key &key) { + if (!Traits::is_valid(key)) { + return std::unexpected(Error{key}); + } + + NodeStatus *status = get_raw_unchecked(key); + if (status == nullptr) { + return false; + } + switch (*status) { + case NodeStatus::Inner: + *status = NodeStatus::Virtual; + return true; + case NodeStatus::Virtual: + return false; + case NodeStatus::Leaf: + remove_raw_unchecked(key); + return update_parent_after_remove(key); + } + UNREACHABLE(); + } + + std::expected is_present(const Key &key) const { + if (!Traits::is_valid(key)) { + return std::unexpected(Error{key}); + } + return _index.contains(key); + } + std::expected is_absent(const Key &key) const { + const auto present = is_present(key); + if (!present.has_value()) { + return std::unexpected(present.error()); + } + return !present.value(); + } + std::expected is(const NodeStatus status, const Key &key) const { + const auto result = get(key); + if (!result.has_value()) { + return std::unexpected(result.error()); + } + return result.value() == status; + } + + std::expected get_raw(const Key &key) { + if (!Traits::is_valid(key)) { + return std::unexpected(Error{key}); + } + return get_raw_unchecked(key); + } + std::expected get_raw(const Key &key) const { + if (!Traits::is_valid(key)) { + return std::unexpected(Error{key}); + } + return get_raw_unchecked(key); + } + std::expected set_raw(const Key &key, const NodeStatus status) { + if (!Traits::is_valid(key)) { + return std::unexpected(Error{key}); + } + set_raw_unchecked(key, status); + return {}; + } + std::expected remove_raw(const Key &key) { + if (!Traits::is_valid(key)) { + return std::unexpected(Error{key}); + } + remove_raw_unchecked(key); + return {}; + } + + void clear() { + _index.clear(); + } + bool empty() const { + return _index.empty(); + } + size_t size() const { + return _index.size(); + } + + const_iterator begin() const { + return _index.begin(); + } + const_iterator end() const { + return _index.end(); + } + const_iterator cbegin() const { + return _index.cbegin(); + } + const_iterator cend() const { + return _index.cend(); + } + +private: + NodeStatus *get_raw_unchecked(const Key &key) { + const auto iterator = _index.find(key); + return iterator == _index.end() ? nullptr : &iterator->second; + } + const NodeStatus *get_raw_unchecked(const Key &key) const { + const auto iterator = _index.find(key); + return iterator == _index.end() ? nullptr : &iterator->second; + } + void set_raw_unchecked(const Key &key, const NodeStatus status) { + _index[key] = status; + } + void remove_raw_unchecked(const Key &key) { + _index.erase(key); + } + + std::expected update_parent_after_remove(const Key &key) { + const auto parent = Traits::parent(key); + if (!parent.has_value()) { + return true; + } + if (!Traits::is_valid(parent.value())) { + return std::unexpected(Error{parent.value()}); + } + + NodeStatus *parent_status = get_raw_unchecked(parent.value()); + DEBUG_ASSERT(parent_status != nullptr); + const auto siblings = Traits::children(parent.value()); + DEBUG_ASSERT(siblings.has_value()); + for (const Key &sibling : siblings.value()) { + if (sibling != key && get_raw_unchecked(sibling) != nullptr) { + return true; + } + } + + switch (*parent_status) { + case NodeStatus::Inner: + *parent_status = NodeStatus::Leaf; + break; + case NodeStatus::Virtual: + remove_raw_unchecked(parent.value()); + return update_parent_after_remove(parent.value()); + case NodeStatus::Leaf: + UNREACHABLE(); + } + return true; + } + + Container _index; +}; + +} // namespace store diff --git a/src/terrainlib/store/IndexFormat.h b/src/terrainlib/store/IndexFormat.h new file mode 100644 index 00000000..d307f14e --- /dev/null +++ b/src/terrainlib/store/IndexFormat.h @@ -0,0 +1,50 @@ +#pragma once + +#include +#include +#include +#include + +#include + +#include "store/Index.h" +#include "store/PathMapping.h" + +namespace store { + +enum class IndexFormatErrorCategory { + Open, + Write, + Malformed, +}; + +struct IndexFormatError { + IndexFormatErrorCategory category; + std::filesystem::path path; + std::string message; + + bool operator==(const IndexFormatError &) const = default; +}; + +template +struct IndexMetadata { + Index index; + std::string layout_id; + std::string payload_class; + std::string codec_selector; +}; + +template +struct IndexFormat { + std::string_view index_filename; + + std::expected, IndexFormatError> (*read)( + const std::filesystem::path &index_path); + std::expected (*write)( + const std::filesystem::path &index_path, + const IndexMetadata &metadata); + std::optional> (*mapping_from_id)(std::string_view id); + PathMapping (*default_mapping)(); +}; + +} // namespace store diff --git a/src/terrainlib/store/IndexedStorage.h b/src/terrainlib/store/IndexedStorage.h new file mode 100644 index 00000000..9a963a10 --- /dev/null +++ b/src/terrainlib/store/IndexedStorage.h @@ -0,0 +1,36 @@ +#pragma once + +#include + +#include "store/Storage.h" + +namespace store { + +template +class IndexedStorage : public Storage { +public: + using Base = Storage; + using typename Base::Persistence; + + explicit IndexedStorage(Storage storage) + : Base(std::move(storage)) { + this->ensure_indexed(); + } + + IndexedStorage( + RawStorage raw, + Index index, + Persistence persistence) + : Base(std::move(raw), std::move(index), std::move(persistence)) {} + + IndexedStorage(const IndexedStorage &) = delete; + IndexedStorage &operator=(const IndexedStorage &) = delete; + IndexedStorage(IndexedStorage &&) noexcept = default; + IndexedStorage &operator=(IndexedStorage &&) noexcept = default; + + const Index &index() const { + return this->index_ref(); + } +}; + +} // namespace store diff --git a/src/terrainlib/store/InvalidKey.h b/src/terrainlib/store/InvalidKey.h new file mode 100644 index 00000000..d8caad27 --- /dev/null +++ b/src/terrainlib/store/InvalidKey.h @@ -0,0 +1,12 @@ +#pragma once + +namespace store { + +template +struct InvalidKey { + Key key; + + bool operator==(const InvalidKey &) const = default; +}; + +} // namespace store diff --git a/src/terrainlib/store/Layout.h b/src/terrainlib/store/Layout.h new file mode 100644 index 00000000..b987c535 --- /dev/null +++ b/src/terrainlib/store/Layout.h @@ -0,0 +1,44 @@ +#pragma once + +#include +#include +#include + +#include "store/NodePath.h" +#include "store/PathMapping.h" + +namespace store { + +template +class Layout { +public: + Layout(std::filesystem::path base_path, PathMapping mapping) + : _base_path(std::move(base_path)), _mapping(mapping) {} + + NodePath node_path(const Key &key) const { + return NodePath(_base_path / _mapping.key_to_node_path(key).path()); + } + + std::optional key_from_node_path(const NodePath &node_path) const { + std::error_code error; + const std::filesystem::path relative = + std::filesystem::relative(node_path.path(), _base_path, error); + if (error || relative.empty() || relative.is_absolute()) { + return std::nullopt; + } + return _mapping.node_path_to_key(NodePath(relative)); + } + + const std::filesystem::path &base_path() const { + return _base_path; + } + PathMapping mapping() const { + return _mapping; + } + +private: + std::filesystem::path _base_path; + PathMapping _mapping; +}; + +} // namespace store diff --git a/src/terrainlib/store/NodePath.h b/src/terrainlib/store/NodePath.h new file mode 100644 index 00000000..31673862 --- /dev/null +++ b/src/terrainlib/store/NodePath.h @@ -0,0 +1,23 @@ +#pragma once + +#include +#include + +namespace store { + +class NodePath { +public: + NodePath() = default; + explicit NodePath(std::filesystem::path path) : _path(std::move(path)) {} + + const std::filesystem::path &path() const { + return _path; + } + + bool operator==(const NodePath &) const = default; + +private: + std::filesystem::path _path; +}; + +} // namespace store diff --git a/src/terrainlib/store/NodeStatus.h b/src/terrainlib/store/NodeStatus.h new file mode 100644 index 00000000..5a012f3f --- /dev/null +++ b/src/terrainlib/store/NodeStatus.h @@ -0,0 +1,79 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include "log.h" + +namespace store { + +class NodeStatus { +public: + using Underlying = uint8_t; + enum Value : Underlying { + Leaf = 0, + Inner = 1, + Virtual = 2, + }; + + constexpr NodeStatus() = default; + constexpr NodeStatus(const Value value) : _value(value) {} + + constexpr operator Value() const { + return _value; + } + explicit operator bool() const = delete; + constexpr bool operator==(const NodeStatus other) const { + return _value == other._value; + } + constexpr bool operator!=(const NodeStatus other) const { + return !(*this == other); + } + constexpr bool operator==(const Value other) const { + return _value == other; + } + constexpr bool operator!=(const Value other) const { + return _value != other; + } + + std::string to_string() const; + +private: + Value _value; +}; + +} // namespace store + +template<> +struct fmt::formatter { + template + constexpr auto parse(ParseContext &context) { + return context.begin(); + } + + template + auto format(const store::NodeStatus &status, FormatContext &context) const { + switch (status) { + case store::NodeStatus::Leaf: + return fmt::format_to(context.out(), "Leaf"); + case store::NodeStatus::Inner: + return fmt::format_to(context.out(), "Inner"); + case store::NodeStatus::Virtual: + return fmt::format_to(context.out(), "Virtual"); + default: + UNREACHABLE(); + } + } +}; + +inline std::string store::NodeStatus::to_string() const { + return fmt::format("{}", *this); +} + +inline std::ostream &operator<<(std::ostream &stream, const store::NodeStatus &status) { + fmt::print(stream, "{}", status); + return stream; +} diff --git a/src/terrainlib/store/NodeStatusOrMissing.h b/src/terrainlib/store/NodeStatusOrMissing.h new file mode 100644 index 00000000..4157b57d --- /dev/null +++ b/src/terrainlib/store/NodeStatusOrMissing.h @@ -0,0 +1,134 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "log.h" +#include "store/NodeStatus.h" + +namespace store { + +class NodeStatusOrMissing { +public: + using Underlying = NodeStatus::Underlying; + enum Value : Underlying { + Leaf = 0, + Inner = 1, + Virtual = 2, + Missing = 3, + }; + + constexpr NodeStatusOrMissing() = default; + constexpr NodeStatusOrMissing(const Value value) : _value(value) {} + NodeStatusOrMissing(const NodeStatus status) : _value(from_status(status)) {} + NodeStatusOrMissing(const std::optional status) : _value(from_optional_status(status)) {} + + constexpr operator Value() const { + return _value; + } + constexpr operator std::optional() const { + return to_optional_status(); + } + explicit operator bool() const = delete; + constexpr bool operator==(const NodeStatusOrMissing other) const { + return _value == other._value; + } + constexpr bool operator!=(const NodeStatusOrMissing other) const { + return !(*this == other); + } + constexpr bool operator==(const Value other) const { + return _value == other; + } + constexpr bool operator!=(const Value other) const { + return _value != other; + } + + std::string to_string() const; + +private: + static constexpr NodeStatusOrMissing from_status(const NodeStatus status) noexcept { + return static_cast(static_cast(status)); + } + static constexpr NodeStatusOrMissing from_optional_status(const std::optional status) noexcept { + return status.has_value() + ? from_status(status.value()) + : NodeStatusOrMissing(NodeStatusOrMissing::Missing); + } + constexpr std::optional to_optional_status() const noexcept { + if (_value == Missing) { + return std::nullopt; + } + return static_cast(_value); + } + +public: + Value _value; +}; + +namespace detail { +template +consteval bool verify_node_status_values(std::index_sequence) { + constexpr auto status_values = magic_enum::enum_values(); + constexpr auto optional_values = magic_enum::enum_values(); + return ((magic_enum::enum_underlying(status_values[Indices]) + == magic_enum::enum_underlying(optional_values[Indices])) + && ...); +} + +consteval bool verify_node_status_enums() { + static_assert( + std::is_same_v< + magic_enum::underlying_type_t, + magic_enum::underlying_type_t>); + constexpr auto status_values = magic_enum::enum_values(); + constexpr auto optional_values = magic_enum::enum_values(); + static_assert(optional_values.size() == status_values.size() + 1); + static_assert(verify_node_status_values(std::make_index_sequence())); + return true; +} + +static_assert(verify_node_status_enums()); +} // namespace detail + +} // namespace store + +template<> +struct fmt::formatter { + template + constexpr auto parse(ParseContext &context) { + return context.begin(); + } + + template + auto format(const store::NodeStatusOrMissing &status, FormatContext &context) const { + switch (status) { + case store::NodeStatusOrMissing::Leaf: + return fmt::format_to(context.out(), "Leaf"); + case store::NodeStatusOrMissing::Inner: + return fmt::format_to(context.out(), "Inner"); + case store::NodeStatusOrMissing::Virtual: + return fmt::format_to(context.out(), "Virtual"); + case store::NodeStatusOrMissing::Missing: + return fmt::format_to(context.out(), "Missing"); + default: + UNREACHABLE(); + } + } +}; + +inline std::string store::NodeStatusOrMissing::to_string() const { + return fmt::format("{}", *this); +} + +inline std::ostream &operator<<(std::ostream &stream, const store::NodeStatusOrMissing &status) { + fmt::print(stream, "{}", status); + return stream; +} diff --git a/src/terrainlib/store/OpenError.h b/src/terrainlib/store/OpenError.h new file mode 100644 index 00000000..2622b8a5 --- /dev/null +++ b/src/terrainlib/store/OpenError.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + +#include "store/IndexFormat.h" +#include "store/StorageError.h" + +namespace store { + +struct UnknownLayout { + std::string id; + bool operator==(const UnknownLayout &) const = default; +}; + +template +using OpenError = std::variant< + IndexFormatError, + FilesystemError, + UnknownLayout, + CodecError, + InvalidKey>; + +} // namespace store diff --git a/src/terrainlib/store/PathMapping.h b/src/terrainlib/store/PathMapping.h new file mode 100644 index 00000000..0a2fe725 --- /dev/null +++ b/src/terrainlib/store/PathMapping.h @@ -0,0 +1,19 @@ +#pragma once + +#include +#include + +#include "store/NodePath.h" + +namespace store { + +template +struct PathMapping { + std::string_view id; + NodePath (*key_to_node_path)(const Key &); + std::optional (*node_path_to_key)(const NodePath &); + + bool operator==(const PathMapping &) const = default; +}; + +} // namespace store diff --git a/src/terrainlib/store/RawStorage.h b/src/terrainlib/store/RawStorage.h new file mode 100644 index 00000000..9245c0dc --- /dev/null +++ b/src/terrainlib/store/RawStorage.h @@ -0,0 +1,206 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +#include "store/Codec.h" +#include "store/Layout.h" +#include "store/StorageError.h" +#include "store/Traits.h" + +namespace store { + +template +class RawStorage { +public: + using Key = typename Traits::Key; + using value_type = NodeData; + + RawStorage(Layout layout, std::unique_ptr> codec) + : _layout(std::move(layout)), _codec(std::move(codec)) {} + + RawStorage(const RawStorage &) = delete; + RawStorage &operator=(const RawStorage &) = delete; + RawStorage(RawStorage &&) noexcept = default; + RawStorage &operator=(RawStorage &&) noexcept = default; + + std::expected> load(const Key &key) const { + if (!Traits::is_valid(key)) { + return std::unexpected(LoadError(InvalidKey{key})); + } + const auto result = _codec->read(_layout.node_path(key)); + if (!result.has_value()) { + return std::unexpected(LoadError(result.error())); + } + return std::move(*result); + } + + std::expected> save(const Key &key, const NodeData &data) const { + if (!Traits::is_valid(key)) { + return std::unexpected(SaveError(InvalidKey{key})); + } + const auto result = _codec->write(_layout.node_path(key), data); + if (!result.has_value()) { + return std::unexpected(SaveError(result.error())); + } + return {}; + } + + std::expected, InvalidKey> paths( + const Key &key) const { + if (!Traits::is_valid(key)) { + return std::unexpected(InvalidKey{key}); + } + return _codec->paths(_layout.node_path(key)); + } + + std::expected> has(const Key &key) const { + const auto node_paths = paths(key); + if (!node_paths.has_value()) { + return std::unexpected(FileOperationError(node_paths.error())); + } + if (node_paths->empty()) { + return false; + } + for (const auto &path : node_paths.value()) { + std::error_code error; + const bool exists = std::filesystem::exists(path, error); + if (error) { + return std::unexpected(FileOperationError(FilesystemError{ + path, + "exists", + error, + })); + } + if (!exists) { + return false; + } + } + return true; + } + + std::expected> remove(const Key &key) const { + const auto node_paths = paths(key); + if (!node_paths.has_value()) { + return std::unexpected(FileOperationError(node_paths.error())); + } + bool removed = false; + for (const auto &path : node_paths.value()) { + std::error_code error; + removed = std::filesystem::remove(path, error) || removed; + if (error) { + return std::unexpected(FileOperationError(FilesystemError{ + path, + "remove", + error, + })); + } + } + return removed; + } + + std::expected> copy_from( + const Key &key, + const RawStorage &source) const { + return copy_from(key, source, []() -> std::expected> { + return {}; + }); + } + + template + std::expected> copy_from( + const Key &key, + const RawStorage &source, + BeforeModify &&before_modify) const { + if (!Traits::is_valid(key)) { + return std::unexpected(CopyError(InvalidKey{key})); + } + const auto source_exists = source.has(key); + if (!source_exists.has_value()) { + return std::unexpected(std::visit( + [](const auto &error) -> CopyError { return error; }, + source_exists.error())); + } + if (!source_exists.value()) { + return std::unexpected(CopyError(MissingSource{key})); + } + + const NodePath probe("__codec_probe__/node"); + if (_codec->paths(probe) != source._codec->paths(probe)) { + const auto loaded = source._codec->read(source._layout.node_path(key)); + if (!loaded.has_value()) { + return std::unexpected(CopyError(loaded.error())); + } + const auto prepared = before_modify(); + if (!prepared.has_value()) { + return prepared; + } + const auto written = _codec->write(_layout.node_path(key), loaded.value()); + if (!written.has_value()) { + return std::unexpected(CopyError(written.error())); + } + return {}; + } + + const auto source_paths = source._codec->paths(source._layout.node_path(key)); + const auto target_paths = _codec->paths(_layout.node_path(key)); + if (source_paths.size() != target_paths.size()) { + return std::unexpected(CopyError(CodecError{ + CodecOperation::Write, + CodecErrorCategory::Domain, + "matching codec probes produced different actual path counts", + })); + } + const auto prepared = before_modify(); + if (!prepared.has_value()) { + return prepared; + } + for (size_t index = 0; index < source_paths.size(); ++index) { + std::error_code error; + std::filesystem::remove(target_paths[index], error); + if (error) { + return std::unexpected(CopyError(FilesystemError{ + target_paths[index], + "remove", + error, + })); + } + std::filesystem::create_directories(target_paths[index].parent_path(), error); + if (error) { + return std::unexpected(CopyError(FilesystemError{ + target_paths[index].parent_path(), + "create_directories", + error, + })); + } + std::filesystem::create_hard_link(source_paths[index], target_paths[index], error); + if (error) { + return std::unexpected(CopyError(FilesystemError{ + target_paths[index], + "create_hard_link", + error, + })); + } + } + return {}; + } + + const Layout &layout() const { + return _layout; + } + const Codec &codec() const { + return *_codec; + } + +private: + Layout _layout; + std::unique_ptr> _codec; +}; + +} // namespace store diff --git a/src/terrainlib/store/Storage.h b/src/terrainlib/store/Storage.h new file mode 100644 index 00000000..e043d9a5 --- /dev/null +++ b/src/terrainlib/store/Storage.h @@ -0,0 +1,329 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include "log.h" +#include "store/IndexFormat.h" +#include "store/RawStorage.h" +#include "store/StorageSettings.h" + +namespace store { + +template +class Storage { +public: + using Key = typename Traits::Key; + using key_type = Key; + using value_type = NodeData; + using load_error = LoadError; + using save_error = SaveError; + + struct Persistence { + IndexFormat format; + std::filesystem::path index_path; + std::string layout_id; + std::string payload_class; + std::string codec_selector; + }; + + explicit Storage(RawStorage raw) : _raw(std::move(raw)) {} + + Storage( + RawStorage raw, + Index index, + Persistence persistence, + const bool dirty = false) + : _raw(std::move(raw)), + _index(std::move(index)), + _persistence(std::move(persistence)), + _dirty(dirty) {} + + virtual ~Storage() { + finalize_displaced_state(); + } + + Storage(const Storage &) = delete; + Storage &operator=(const Storage &) = delete; + + Storage(Storage &&other) noexcept + : _raw(std::move(other._raw)), + _index(std::move(other._index)), + _persistence(std::move(other._persistence)), + _settings(other._settings), + _dirty(std::exchange(other._dirty, false)) { + other._index.reset(); + other._persistence.reset(); + } + + Storage &operator=(Storage &&other) noexcept { + if (this == &other) { + return *this; + } + finalize_displaced_state(); + _raw = std::move(other._raw); + _index = std::move(other._index); + _persistence = std::move(other._persistence); + _settings = other._settings; + _dirty = std::exchange(other._dirty, false); + other._index.reset(); + other._persistence.reset(); + return *this; + } + + std::expected> load(const Key &key) const { + if (!Traits::is_valid(key)) { + return std::unexpected(LoadError(InvalidKey{key})); + } + if (_index.has_value()) { + const auto status = _index->get(key); + if (!status.has_value()) { + return std::unexpected(LoadError(status.error())); + } + if (!status->has_value() + || status->value() == NodeStatus::Virtual) { + return std::unexpected(LoadError(CodecError{ + CodecOperation::Read, + CodecErrorCategory::FileNotFound, + "node is not physically present in the index", + })); + } + } + return _raw.load(key); + } + + std::expected> save(const Key &key, const NodeData &data) { + if (!Traits::is_valid(key)) { + return std::unexpected(SaveError(InvalidKey{key})); + } + const auto exists = has(key); + if (!exists.has_value()) { + return std::unexpected(std::visit( + [](const auto &error) -> SaveError { return error; }, + exists.error())); + } + if (exists.value() && !_settings.allow_overwrite) { + const auto node_paths = _raw.paths(key).value(); + return std::unexpected(SaveError(AlreadyExists{ + node_paths.empty() ? _raw.layout().node_path(key).path() : node_paths.front(), + })); + } + + const auto result = _raw.save(key, data); + if (!result.has_value()) { + return result; + } + if (_index.has_value()) { + const auto added = _index->add(key); + if (!added.has_value()) { + return std::unexpected(SaveError(added.error())); + } + _dirty = _dirty || added.value(); + } + return {}; + } + + std::expected> copy_from( + const Key &key, + const Storage &source) { + if (!Traits::is_valid(key)) { + return std::unexpected(CopyError(InvalidKey{key})); + } + const auto exists = has(key); + if (!exists.has_value()) { + return std::unexpected(std::visit( + [](const auto &error) -> CopyError { return error; }, + exists.error())); + } + if (exists.value() && !_settings.allow_overwrite) { + const auto node_paths = _raw.paths(key).value(); + return std::unexpected(CopyError(AlreadyExists{ + node_paths.empty() ? _raw.layout().node_path(key).path() : node_paths.front(), + })); + } + const auto prepare_target = [&]() -> std::expected> { + if (!exists.value() || !_index.has_value()) { + return {}; + } + const auto removed = _index->remove(key); + if (!removed.has_value()) { + return std::unexpected(CopyError(removed.error())); + } + _dirty = _dirty || removed.value(); + return {}; + }; + + const auto result = _raw.copy_from(key, source._raw, prepare_target); + if (!result.has_value()) { + return result; + } + if (_index.has_value()) { + const auto added = _index->add(key); + if (!added.has_value()) { + return std::unexpected(CopyError(added.error())); + } + _dirty = _dirty || added.value(); + } + return {}; + } + + std::expected> remove(const Key &key) { + const auto removed = _raw.remove(key); + if (!removed.has_value()) { + return removed; + } + if (_index.has_value()) { + const auto index_removed = _index->remove(key); + if (!index_removed.has_value()) { + return std::unexpected(FileOperationError(index_removed.error())); + } + _dirty = _dirty || index_removed.value(); + } + return removed.value(); + } + + std::expected> has(const Key &key) const { + if (!Traits::is_valid(key)) { + return std::unexpected(FileOperationError(InvalidKey{key})); + } + if (!_index.has_value()) { + return _raw.has(key); + } + const auto status = _index->get(key); + if (!status.has_value()) { + return std::unexpected(FileOperationError(status.error())); + } + return status->has_value() && status->value() != NodeStatus::Virtual; + } + + std::expected, InvalidKey> paths( + const Key &key) const { + return _raw.paths(key); + } + + std::expected> path_for( + const Key &key) const { + const auto node_paths = paths(key); + if (!node_paths.has_value()) { + return std::unexpected(node_paths.error()); + } + return node_paths->empty() + ? _raw.layout().node_path(key).path() + : node_paths->front(); + } + + const std::filesystem::path &base_path() const { + return _raw.layout().base_path(); + } + const Layout &layout() const { + return _raw.layout(); + } + const Codec &codec() const { + return _raw.codec(); + } + std::optional codec_selector() const { + if (!_persistence.has_value()) { + return std::nullopt; + } + return _persistence->codec_selector; + } + + bool is_indexed() const { + return _index.has_value(); + } + void ensure_indexed() { + if (!_index.has_value()) { + _index.emplace(); + _dirty = true; + } + } + + std::optional>> index() const { + if (!_index.has_value()) { + return std::nullopt; + } + return std::cref(_index.value()); + } + + std::expected save_index() const { + if (!_dirty) { + return {}; + } + if (!_index.has_value() || !_persistence.has_value()) { + return std::unexpected(IndexFormatError{ + IndexFormatErrorCategory::Write, + {}, + "indexed storage has no persistence configuration", + }); + } + const IndexMetadata metadata{ + _index.value(), + _persistence->layout_id, + _persistence->payload_class, + _persistence->codec_selector, + }; + const auto result = _persistence->format.write( + _persistence->index_path, + metadata); + if (result.has_value()) { + _dirty = false; + } + return result; + } + + std::expected save_or_create_index() { + if (!_index.has_value()) { + _index.emplace(); + _dirty = true; + } + return save_index(); + } + + const StorageSettings &settings() const { + return _settings; + } + StorageSettings &settings() { + return _settings; + } + +protected: + Index &index_mut() { + return _index.value(); + } + const Index &index_ref() const { + return _index.value(); + } + bool is_index_dirty() const { + return _dirty; + } + void set_index_dirty(const bool dirty = true) const { + _dirty = dirty; + } + +private: + void finalize_displaced_state() noexcept { + if (!_dirty || !_index.has_value()) { + return; + } + const auto result = save_index(); + if (!result.has_value()) { + LOG_ERROR( + "Failed to automatically save index {}: {}", + result.error().path, + result.error().message); + } + } + + RawStorage _raw; + std::optional> _index; + std::optional _persistence; + StorageSettings _settings; + mutable bool _dirty = false; +}; + +} // namespace store diff --git a/src/terrainlib/store/StorageError.h b/src/terrainlib/store/StorageError.h new file mode 100644 index 00000000..ffd6d69b --- /dev/null +++ b/src/terrainlib/store/StorageError.h @@ -0,0 +1,52 @@ +#pragma once + +#include +#include +#include +#include + +#include "store/CodecError.h" +#include "store/InvalidKey.h" + +namespace store { + +struct AlreadyExists { + std::filesystem::path path; + bool operator==(const AlreadyExists &) const = default; +}; + +struct FilesystemError { + std::filesystem::path path; + std::string operation; + std::error_code error; + bool operator==(const FilesystemError &) const = default; +}; + +template +struct MissingSource { + Key key; + bool operator==(const MissingSource &) const = default; +}; + +template +using LoadError = std::variant, CodecError>; + +template +using SaveError = std::variant< + InvalidKey, + CodecError, + AlreadyExists, + FilesystemError>; + +template +using FileOperationError = std::variant, FilesystemError>; + +template +using CopyError = std::variant< + InvalidKey, + MissingSource, + AlreadyExists, + FilesystemError, + CodecError>; + +} // namespace store diff --git a/src/terrainlib/store/StorageSettings.h b/src/terrainlib/store/StorageSettings.h new file mode 100644 index 00000000..2a883ee1 --- /dev/null +++ b/src/terrainlib/store/StorageSettings.h @@ -0,0 +1,9 @@ +#pragma once + +namespace store { + +struct StorageSettings { + bool allow_overwrite = false; +}; + +} // namespace store diff --git a/src/terrainlib/store/Traits.h b/src/terrainlib/store/Traits.h new file mode 100644 index 00000000..80fe88a7 --- /dev/null +++ b/src/terrainlib/store/Traits.h @@ -0,0 +1,17 @@ +#pragma once + +#include + +namespace store { + +template +concept HierarchyTraits = requires(typename Traits::Key key) { + typename Traits::Key; + typename Traits::Hasher; + { Traits::root() } -> std::same_as; + Traits::parent(key); + Traits::children(key); + { Traits::is_valid(key) } -> std::same_as; +}; + +} // namespace store diff --git a/src/terrainlib/store/cache/Dummy.h b/src/terrainlib/store/cache/Dummy.h new file mode 100644 index 00000000..bcb7c7e7 --- /dev/null +++ b/src/terrainlib/store/cache/Dummy.h @@ -0,0 +1,25 @@ +#pragma once + +#include "store/cache/Interface.h" + +namespace store::cache { + +template +class Dummy final : public Interface { +public: + using Key = typename Traits::Key; + std::optional get(const Key &) noexcept override { + return std::nullopt; + } + bool put(const Key &, const NodeData &) noexcept override { + return false; + } + bool remove(const Key &) noexcept override { + return false; + } + bool contains(const Key &) const noexcept override { + return false; + } +}; + +} // namespace store::cache diff --git a/src/terrainlib/store/cache/Interface.h b/src/terrainlib/store/cache/Interface.h new file mode 100644 index 00000000..4ffd070d --- /dev/null +++ b/src/terrainlib/store/cache/Interface.h @@ -0,0 +1,21 @@ +#pragma once + +#include + +#include "store/Traits.h" + +namespace store::cache { + +template +class Interface { +public: + using Key = typename Traits::Key; + virtual ~Interface() = default; + + virtual std::optional get(const Key &key) noexcept = 0; + virtual bool put(const Key &key, const NodeData &value) noexcept = 0; + virtual bool remove(const Key &key) noexcept = 0; + virtual bool contains(const Key &key) const noexcept = 0; +}; + +} // namespace store::cache diff --git a/src/terrainlib/store/cache/Lru.h b/src/terrainlib/store/cache/Lru.h new file mode 100644 index 00000000..0e8d8abf --- /dev/null +++ b/src/terrainlib/store/cache/Lru.h @@ -0,0 +1,73 @@ +#pragma once + +#include +#include +#include + +#include "store/cache/Interface.h" + +namespace store::cache { + +template +class Lru final : public Interface { +public: + using Key = typename Traits::Key; + + explicit Lru(const size_t capacity) : _capacity(capacity) {} + + std::optional get(const Key &key) noexcept override { + const auto iterator = _values.find(key); + if (iterator == _values.end()) { + return std::nullopt; + } + _usage.splice(_usage.begin(), _usage, iterator->second.second); + return iterator->second.first; + } + + bool put(const Key &key, const NodeData &value) noexcept override { + const auto iterator = _values.find(key); + if (iterator != _values.end()) { + iterator->second.first = value; + _usage.splice(_usage.begin(), _usage, iterator->second.second); + return false; + } + if (_capacity == 0) { + return false; + } + if (_values.size() == _capacity) { + _values.erase(_usage.back()); + _usage.pop_back(); + } + _usage.push_front(key); + _values.emplace(key, std::pair::iterator>{ + value, + _usage.begin(), + }); + return true; + } + + bool remove(const Key &key) noexcept override { + const auto iterator = _values.find(key); + if (iterator == _values.end()) { + return false; + } + _usage.erase(iterator->second.second); + _values.erase(iterator); + return true; + } + + bool contains(const Key &key) const noexcept override { + return _values.contains(key); + } + +private: + size_t _capacity; + std::list _usage; + std::unordered_map< + Key, + std::pair::iterator>, + typename Traits::Hasher> + _values; +}; + +} // namespace store::cache diff --git a/src/terrainlib/store/codec/Path.h b/src/terrainlib/store/codec/Path.h new file mode 100644 index 00000000..4715291b --- /dev/null +++ b/src/terrainlib/store/codec/Path.h @@ -0,0 +1,19 @@ +#pragma once + +#include +#include + +#include "store/NodePath.h" + +namespace store::codec { + +inline std::filesystem::path append_extension( + const NodePath &node_path, + const std::string_view extension) +{ + std::filesystem::path result = node_path.path(); + result += extension; + return result; +} + +} // namespace store::codec diff --git a/src/terrainlib/store/describe_error.h b/src/terrainlib/store/describe_error.h new file mode 100644 index 00000000..aa61d2a1 --- /dev/null +++ b/src/terrainlib/store/describe_error.h @@ -0,0 +1,50 @@ +#pragma once + +#include +#include + +#include "store/IndexFormat.h" +#include "store/OpenError.h" + +namespace store { + +inline std::string describe_error(const CodecError &error) { + return error.message; +} + +inline std::string describe_error(const IndexFormatError &error) { + return error.path.empty() + ? error.message + : error.path.string() + ": " + error.message; +} + +inline std::string describe_error(const FilesystemError &error) { + return error.operation + " " + error.path.string() + ": " + error.error.message(); +} + +inline std::string describe_error(const UnknownLayout &error) { + return "unknown layout: " + error.id; +} + +inline std::string describe_error(const AlreadyExists &error) { + return "path already exists: " + error.path.string(); +} + +template +std::string describe_error(const InvalidKey &) { + return "invalid hierarchy key"; +} + +template +std::string describe_error(const MissingSource &) { + return "source node is missing"; +} + +template +std::string describe_error(const std::variant &error) { + return std::visit( + [](const auto &value) { return describe_error(value); }, + error); +} + +} // namespace store diff --git a/src/terrainlib/store/open.h b/src/terrainlib/store/open.h new file mode 100644 index 00000000..5bf36352 --- /dev/null +++ b/src/terrainlib/store/open.h @@ -0,0 +1,101 @@ +#pragma once + +#include +#include +#include + +#include + +#include "store/IndexedStorage.h" +#include "store/OpenError.h" + +namespace store { + +template +std::expected, OpenError> open_index( + const std::filesystem::path &index_path, + const IndexFormat format, + const std::string_view expected_payload_class, + CodecResolver &&resolve_codec) { + using Key = typename Traits::Key; + const auto metadata_result = format.read(index_path); + if (!metadata_result.has_value()) { + return std::unexpected(OpenError(metadata_result.error())); + } + IndexMetadata metadata = std::move(metadata_result.value()); + if (metadata.payload_class != expected_payload_class) { + return std::unexpected(OpenError(CodecError{ + CodecOperation::Resolve, + CodecErrorCategory::UnsupportedCodec, + "unexpected payload class: " + metadata.payload_class, + })); + } + for (const auto &[key, status] : metadata.index) { + static_cast(status); + if (!Traits::is_valid(key)) { + return std::unexpected(OpenError(InvalidKey{key})); + } + } + + const auto mapping = format.mapping_from_id(metadata.layout_id); + if (!mapping.has_value()) { + return std::unexpected(OpenError(UnknownLayout{metadata.layout_id})); + } + auto codec = resolve_codec(metadata.codec_selector); + if (!codec.has_value()) { + return std::unexpected(OpenError(codec.error())); + } + + const std::filesystem::path base_path = index_path.parent_path(); + using Storage = store::Storage; + return IndexedStorage( + RawStorage( + Layout(base_path, mapping.value()), + std::move(codec.value())), + std::move(metadata.index), + typename Storage::Persistence{ + format, + index_path, + std::move(metadata.layout_id), + std::move(metadata.payload_class), + std::move(metadata.codec_selector), + }); +} + +template +std::expected, OpenError> make_storage( + const std::filesystem::path &base_path, + const IndexFormat format, + const PathMapping mapping, + std::string payload_class, + std::string codec_selector, + std::unique_ptr> codec) { + using Key = typename Traits::Key; + std::error_code error; + std::filesystem::create_directories(base_path, error); + if (error) { + return std::unexpected(OpenError(FilesystemError{ + base_path, + "create_directories", + error, + })); + } + + typename Storage::Persistence persistence{ + format, + base_path / format.index_filename, + std::string(mapping.id), + std::move(payload_class), + std::move(codec_selector), + }; + RawStorage raw( + Layout(base_path, mapping), + std::move(codec)); + return Storage( + std::move(raw), + Index{}, + std::move(persistence), + true); +} + +} // namespace store diff --git a/src/terrainlib/store/traverse.h b/src/terrainlib/store/traverse.h new file mode 100644 index 00000000..8dbfe948 --- /dev/null +++ b/src/terrainlib/store/traverse.h @@ -0,0 +1,99 @@ +#pragma once + +#include +#include +#include + +#include "store/Index.h" + +namespace store { + +enum class TraversalOrder { + DepthFirst, + BreadthFirst, +}; + +struct AlwaysRefine { + template + constexpr bool operator()(const Key &) const { + return true; + } +}; + +template +std::expected> traverse( + const Index &index, + VisitFn &&visit, + RefineFn &&refine = {}, + const typename Traits::Key &root = Traits::root(), + const TraversalOrder order = TraversalOrder::DepthFirst) { + using Key = typename Traits::Key; + using Error = InvalidKey; + + if (!Traits::is_valid(root)) { + return std::unexpected(Error{root}); + } + const auto root_status = index.get(root); + if (!root_status.has_value()) { + return std::unexpected(root_status.error()); + } + if (!root_status.value().has_value()) { + return {}; + } + + if (order == TraversalOrder::DepthFirst) { + std::function(const Key &)> depth_first; + depth_first = [&](const Key ¤t) -> std::expected { + const auto status = index.get(current); + if (!status.has_value()) { + return std::unexpected(status.error()); + } + if (!status.value().has_value()) { + return {}; + } + visit(current, status.value().value()); + + if (refine(current)) { + const auto children = Traits::children(current); + if (children.has_value()) { + for (const Key &child : children.value()) { + const auto result = depth_first(child); + if (!result.has_value()) { + return result; + } + } + } + } + return {}; + }; + return depth_first(root); + } + + std::queue queue; + queue.push(root); + while (!queue.empty()) { + const Key current = queue.front(); + queue.pop(); + + const auto status = index.get(current); + if (!status.has_value()) { + return std::unexpected(status.error()); + } + if (!status.value().has_value()) { + continue; + } + visit(current, status.value().value()); + + if (refine(current)) { + const auto children = Traits::children(current); + if (children.has_value()) { + for (const Key &child : children.value()) { + queue.push(child); + } + } + } + } + return {}; +} + +} // namespace store diff --git a/src/terrainlib/tile_path.h b/src/terrainlib/tile_path.h new file mode 100644 index 00000000..8f866d80 --- /dev/null +++ b/src/terrainlib/tile_path.h @@ -0,0 +1,14 @@ +#pragma once + +#include +#include + +#include + +[[nodiscard]] inline std::filesystem::path google_tile_path( + const std::filesystem::path& base_path, + const radix::tile::Id& tile_id, + const std::string& extension) +{ + return base_path / std::to_string(tile_id.zoom_level) / std::to_string(tile_id.coords.x) / (std::to_string(tile_id.coords.y) + extension); +} diff --git a/src/terrainlib/uv/unwrap.cpp b/src/terrainlib/uv/unwrap.cpp index 1802ae91..9186c5de 100644 --- a/src/terrainlib/uv/unwrap.cpp +++ b/src/terrainlib/uv/unwrap.cpp @@ -130,7 +130,7 @@ struct CgalMap { double aspect = 1.0; }; -tl::expected parameterize_mesh(cgal::Mesh &mesh, Algorithm algorithm, Border border) { +std::expected parameterize_mesh(cgal::Mesh &mesh, Algorithm algorithm, Border border) { const cgal::HalfedgeDescriptor bhd = CGAL::Polygon_mesh_processing::longest_border(mesh).first; DEBUG_ASSERT(bhd != boost::graph_traits::null_halfedge()); @@ -179,7 +179,7 @@ tl::expected parameterize_mesh(cgal::Mesh &mesh, Algorithm } if (result != CGAL::Surface_mesh_parameterization::OK) { - return tl::unexpected(UnwrapError(result)); + return std::unexpected(UnwrapError(result)); } const auto vertex_count = CGAL::num_vertices(mesh); @@ -208,7 +208,7 @@ std::vector decode_uv_map(const UvMap &map, size_t vertex_count) { } } -tl::expected unwrap( +std::expected unwrap( const std::span triangles, const std::span positions, Algorithm algorithm, @@ -220,7 +220,7 @@ tl::expected unwrap( cgal::Mesh cgal_mesh = convert::to_cgal_mesh(mesh); auto result = parameterize_mesh(cgal_mesh, algorithm, border); if (!result) { - return tl::unexpected(result.error()); + return std::unexpected(result.error()); } const CgalMap &cgal_map = result.value(); Uvs uvs = decode_uv_map(cgal_map.uvs, mesh.vertex_count()); diff --git a/src/terrainlib/uv/unwrap.h b/src/terrainlib/uv/unwrap.h index f47967fd..98786483 100644 --- a/src/terrainlib/uv/unwrap.h +++ b/src/terrainlib/uv/unwrap.h @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include @@ -57,13 +57,13 @@ struct Map { inline constexpr Algorithm DEFAULT_ALGORITHM = Algorithm::TutteBarycentricMapping; inline constexpr Border DEFAULT_BORDER = Border::Circle; -tl::expected unwrap( +std::expected unwrap( const std::span triangles, const std::span positions, Algorithm algorithm = DEFAULT_ALGORITHM, Border border = DEFAULT_BORDER); -inline tl::expected unwrap( +inline std::expected unwrap( const std::vector& triangles, const std::vector& positions, Algorithm algorithm = DEFAULT_ALGORITHM, @@ -75,7 +75,7 @@ inline tl::expected unwrap( border); } -inline tl::expected unwrap( +inline std::expected unwrap( const mesh::View &mesh, Algorithm algorithm = DEFAULT_ALGORITHM, Border border = DEFAULT_BORDER) { @@ -86,7 +86,7 @@ inline tl::expected unwrap( border); } -inline tl::expected unwrap( +inline std::expected unwrap( const mesh::Simple &mesh, Algorithm algorithm = DEFAULT_ALGORITHM, Border border = DEFAULT_BORDER) { diff --git a/src/tile_builder/CMakeLists.txt b/src/tile_builder/CMakeLists.txt index d4464342..576df41c 100644 --- a/src/tile_builder/CMakeLists.txt +++ b/src/tile_builder/CMakeLists.txt @@ -1,7 +1,7 @@ add_library(tilebuilderlib alpine_raster.cpp DatasetReader.cpp - Image.cpp + image_writer.cpp ParallelTileGenerator.cpp ParallelTiler.cpp TileHeightsGenerator.cpp diff --git a/src/tile_builder/DatasetReader.cpp b/src/tile_builder/DatasetReader.cpp index bad96340..a472ae54 100644 --- a/src/tile_builder/DatasetReader.cpp +++ b/src/tile_builder/DatasetReader.cpp @@ -29,7 +29,7 @@ #include "Dataset.h" #include "Exception.h" -#include "Image.h" +#include #include "ctb/types.hpp" #include "log.h" @@ -185,12 +185,12 @@ DatasetReader::DatasetReader(const std::shared_ptr& dataset, const OGRS throw Exception(fmt::format("Dataset does not contain band number {} (there are {} bands).", band, dataset->n_bands())); } -HeightData DatasetReader::read(const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const +radix::Raster DatasetReader::read(const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const { return readFrom(m_dataset, bounds, width, height); } -HeightData DatasetReader::readWithOverviews(const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const +radix::Raster DatasetReader::readWithOverviews(const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const { #ifdef ALP_ENABLE_OVERVIEW_READING auto transformer_args = make_image_transform_args(*this, m_dataset.get(), bounds, width, height); @@ -202,7 +202,7 @@ HeightData DatasetReader::readWithOverviews(const radix::tile::SrsBounds& bounds #endif } -HeightData DatasetReader::readFrom(const std::shared_ptr& source_dataset, const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const +radix::Raster DatasetReader::readFrom(const std::shared_ptr& source_dataset, const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const { // if we have performance problems with the warping, it'd still be possible to approximate the warping operation with a linear transform (mostly when zoomed in / on higher zoom levels). // CTB does this in GDALTiler.cpp around line 375 ("// Decide if we are doing an approximate or exact transformation"). @@ -212,9 +212,9 @@ HeightData DatasetReader::readFrom(const std::shared_ptr& source_datase auto warped_dataset = Dataset(static_cast(GDALCreateWarpedVRT(source_dataset->gdalDataset(), int(width), int(height), adfGeoTransform.data(), warp_options.first.get()))); auto* heights_band = warped_dataset.gdalDataset()->GetRasterBand(1); // non-owning pointer - auto heights_data = HeightData(width, height); + auto heights_data = radix::Raster({ width, height }); if (heights_band->RasterIO(GF_Read, 0, 0, int(width), int(height), - static_cast(heights_data.data()), int(width), int(height), GDT_Float32, 0, 0) + static_cast(heights_data.buffer().data()), int(width), int(height), GDT_Float32, 0, 0) != CE_None) throw Exception("couldn't read data"); diff --git a/src/tile_builder/DatasetReader.h b/src/tile_builder/DatasetReader.h index e75b1ff8..a60400c6 100644 --- a/src/tile_builder/DatasetReader.h +++ b/src/tile_builder/DatasetReader.h @@ -23,7 +23,7 @@ #include #include -#include "Image.h" +#include #include class Dataset; @@ -33,8 +33,8 @@ class DatasetReader { public: DatasetReader(const std::shared_ptr& dataset, const OGRSpatialReference& targetSRS, unsigned band, bool warn_on_missing_overviews = true); - HeightData read(const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const; - HeightData readWithOverviews(const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const; + radix::Raster read(const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const; + radix::Raster readWithOverviews(const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const; unsigned dataset_band() const { return m_band; } bool isReprojecting() const { return m_requires_reprojection; } @@ -42,7 +42,7 @@ class DatasetReader { std::string target_srs_wkt() const { return m_target_srs_wkt; } protected: - HeightData readFrom(const std::shared_ptr& dataset, const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const; + radix::Raster readFrom(const std::shared_ptr& dataset, const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const; private: std::shared_ptr m_dataset; diff --git a/src/tile_builder/Image.cpp b/src/tile_builder/Image.cpp deleted file mode 100644 index 3b33465b..00000000 --- a/src/tile_builder/Image.cpp +++ /dev/null @@ -1,20 +0,0 @@ -#include -#include "Image.h" -#include - -void image::saveImageAsPng(const Image& inputImage, const std::string& path) -{ - const int width = static_cast(inputImage.width()); - const int height = static_cast(inputImage.height()); - - cv::Mat image(height, width, CV_8UC3); - - for (int row = 0; row < height; ++row) { - for (int col = 0; col < width; ++col) { - const glm::u8vec3& pixel = inputImage.pixel(height - row - 1, col); // Flip vertically - image.at(row, col) = cv::Vec3b(pixel.z, pixel.y, pixel.x); // RGB to BGR - } - } - - cv::imwrite(path, image); -} diff --git a/src/tile_builder/Image.h b/src/tile_builder/Image.h deleted file mode 100644 index 5785329d..00000000 --- a/src/tile_builder/Image.h +++ /dev/null @@ -1,105 +0,0 @@ -/***************************************************************************** - * Alpine Terrain Builder - * Copyright (C) 2022 alpinemaps.org - * Copyright (C) 2022 Adam Celarek - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *****************************************************************************/ - -#ifndef IMAGE_H -#define IMAGE_H - -#include -#include -#include -#include -#include -#include - -#include - -namespace tntn { -template -class Raster; -} - -template -class Image { -public: - Image() = default; - Image(unsigned width, unsigned height) - : m_width(width) - , m_height(height) - , m_data(size_t(m_width * m_height)) - { - } - - [[nodiscard]] unsigned width() const { return m_width; } - [[nodiscard]] unsigned height() const { return m_height; } - [[nodiscard]] T pixel(unsigned row, unsigned column) const - { - assert(column < m_width); - assert(row < m_height); - assert(m_data.size() == size_t(m_width * m_height)); - return m_data[row * m_width + column]; - } - - [[nodiscard]] float* data() { return m_data.data(); } - - [[nodiscard]] auto size() const { return m_data.size(); } - [[nodiscard]] auto begin() { return m_data.begin(); } - [[nodiscard]] auto end() { return m_data.end(); } - [[nodiscard]] auto begin() const { return m_data.begin(); } - [[nodiscard]] auto end() const { return m_data.end(); } - -private: - unsigned m_width = 0; - unsigned m_height = 0; - std::vector m_data; - - friend class tntn::Raster; -}; - -using HeightData = Image; -using RgbImage = Image; -using uchar = unsigned char; - -namespace image { -void saveImageAsPng(const Image& image, const std::string& path); - -template -[[nodiscard]] auto transformImage(const Image& i, Fun conversion_fun) -> Image -{ - using T2 = decltype(conversion_fun(*i.begin())); - Image i2(i.width(), i.height()); - std::transform(i.begin(), i.end(), i2.begin(), conversion_fun); - return i2; -} - -template -void debugOut(const Image& image, const std::string& path) -{ - auto [min, max] = std::ranges::minmax(image); - - // [min=min, ..] is required for cpp correctness. min/max from the capture are not variables, we need to copy them: - // https://stackoverflow.com/questions/50799719/reference-to-local-binding-declared-in-enclosing-function?noredirect=1&lq=1 - saveImageAsPng(transformImage(image, [min = min, max = max](auto v) { - const auto c = uchar(255.F * (float(v) - float(min)) / float(max - min)); - return glm::u8vec3(c, c, c); - }), - path); -} -} - -#endif // HEIGHTDATA_H diff --git a/src/tile_builder/ParallelTileGenerator.cpp b/src/tile_builder/ParallelTileGenerator.cpp index e0d7e731..6a52a497 100644 --- a/src/tile_builder/ParallelTileGenerator.cpp +++ b/src/tile_builder/ParallelTileGenerator.cpp @@ -41,7 +41,7 @@ ParallelTileGenerator::ParallelTileGenerator(const std::string& input_data_path, } ParallelTileGenerator ParallelTileGenerator::make(const std::string& input_data_path, - ctb::Grid::Srs srs, radix::tile::Scheme tiling_scheme, + ctb::Grid::Srs srs, std::unique_ptr tile_writer, const std::string& output_data_path, unsigned grid_resolution) @@ -51,7 +51,7 @@ ParallelTileGenerator ParallelTileGenerator::make(const std::string& input_data_ if (srs == ctb::Grid::Srs::SphericalMercator) grid = ctb::GlobalMercator(grid_resolution); const auto border = tile_writer->formatRequiresBorder(); - return { input_data_path, grid, ParallelTiler(grid, dataset->bounds(grid.getSRS()), border, tiling_scheme), std::move(tile_writer), output_data_path }; + return { input_data_path, grid, ParallelTiler(grid, dataset->bounds(grid.getSRS()), border), std::move(tile_writer), output_data_path }; } const ParallelTiler& ParallelTileGenerator::tiler() const @@ -64,7 +64,7 @@ const ctb::Grid& ParallelTileGenerator::grid() const return m_grid; } -void ParallelTileGenerator::write(const radix::tile::Descriptor& tile, const HeightData& heights) const +void ParallelTileGenerator::write(const radix::tile::Descriptor& tile, const radix::Raster& heights) const { const auto dir_path = fmt::format("{}/{}/{}", m_output_data_path, tile.id.zoom_level, tile.id.coords.x); const auto file_path = fmt::format("{}/{}.{}", dir_path, tile.id.coords.y, m_tile_writer->formatFileEnding()); diff --git a/src/tile_builder/ParallelTileGenerator.h b/src/tile_builder/ParallelTileGenerator.h index f95a412f..cd6f6240 100644 --- a/src/tile_builder/ParallelTileGenerator.h +++ b/src/tile_builder/ParallelTileGenerator.h @@ -22,7 +22,7 @@ #include #include -#include "Image.h" +#include #include "ParallelTiler.h" #include "ctb/Grid.hpp" #include "ctb/types.hpp" @@ -41,7 +41,7 @@ class ParallelTileGenerator { public: ParallelTileGenerator(const std::string& input_data_path, const ctb::Grid& grid, const ParallelTiler& tiler, std::unique_ptr tile_writer, const std::string& output_data_path); [[nodiscard]] static ParallelTileGenerator make(const std::string& input_data_path, - ctb::Grid::Srs srs, radix::tile::Scheme tiling_scheme, + ctb::Grid::Srs srs, std::unique_ptr tile_writer, const std::string& output_data_path, unsigned grid_resolution = 256); @@ -49,7 +49,7 @@ class ParallelTileGenerator { void setWarnOnMissingOverviews(bool flag) { m_warn_on_missing_overviews = flag; } [[nodiscard]] const ParallelTiler& tiler() const; [[nodiscard]] const ctb::Grid& grid() const; - void write(const radix::tile::Descriptor& tile, const HeightData& heights) const; + void write(const radix::tile::Descriptor& tile, const radix::Raster& heights) const; void process(const std::pair& zoom_range, bool progress_bar_on_console = false, bool generate_world_wide_tiles = false) const; }; @@ -68,7 +68,7 @@ class ParallelTileWriterInterface { virtual ~ParallelTileWriterInterface() = default; ParallelTileWriterInterface& operator=(const ParallelTileWriterInterface&) = default; ParallelTileWriterInterface& operator=(ParallelTileWriterInterface&&) = default; - virtual void write(const std::string& file_path, const radix::tile::Descriptor& tile, const HeightData& heights) const = 0; + virtual void write(const std::string& file_path, const radix::tile::Descriptor& tile, const radix::Raster& heights) const = 0; [[nodiscard]] radix::tile::Border formatRequiresBorder() const; [[nodiscard]] const std::string& formatFileEnding() const; }; diff --git a/src/tile_builder/ParallelTiler.cpp b/src/tile_builder/ParallelTiler.cpp index 3f1c2958..fe6b2330 100644 --- a/src/tile_builder/ParallelTiler.cpp +++ b/src/tile_builder/ParallelTiler.cpp @@ -22,32 +22,31 @@ #include "Exception.h" #include -ParallelTiler::ParallelTiler(const ctb::Grid& grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border, radix::tile::Scheme scheme) : Tiler(grid, bounds, border, scheme) +ParallelTiler::ParallelTiler(const ctb::Grid& grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border) : Tiler(grid, bounds, border) { } radix::tile::Id ParallelTiler::southWestTile(unsigned zoom_level) const { - return grid().crsToTile(bounds().min, zoom_level).to(scheme()); + return grid().crsToTile(bounds().min, zoom_level); } radix::tile::Id ParallelTiler::northEastTile(unsigned zoom_level) const { const auto epsilon = grid().resolution(zoom_level) / 100; - return grid().crsToTile(bounds().max - epsilon, zoom_level).to(scheme()); + return grid().crsToTile(bounds().max - epsilon, zoom_level); } std::vector ParallelTiler::generateTiles(unsigned zoom_level) const { - // in the tms scheme south west corresponds to the smaller numbers. hence we can iterate from sw to ne - const auto sw = southWestTile(zoom_level).to(radix::tile::Scheme::Tms).coords; - const auto ne = northEastTile(zoom_level).to(radix::tile::Scheme::Tms).coords; + const auto sw = southWestTile(zoom_level).coords; + const auto ne = northEastTile(zoom_level).coords; std::vector tiles; - tiles.reserve((ne.y - sw.y + 1) * (ne.x - sw.x + 1)); - for (auto ty = sw.y; ty <= ne.y; ++ty) { + tiles.reserve((sw.y - ne.y + 1) * (ne.x - sw.x + 1)); + for (auto ty = ne.y; ty <= sw.y; ++ty) { for (auto tx = sw.x; tx <= ne.x; ++tx) { - const auto tile_id = radix::tile::Id { zoom_level, { tx, ty }, radix::tile::Scheme::Tms }.to(scheme()); + const auto tile_id = radix::tile::Id { zoom_level, { tx, ty } }; tiles.emplace_back(tile_for(tile_id)); if (tiles.size() >= 1'000'000'000) // think about creating an on the fly tile generator. storing so many tiles takes a lot of memory. diff --git a/src/tile_builder/ParallelTiler.h b/src/tile_builder/ParallelTiler.h index 7d01f0de..62cd07ac 100644 --- a/src/tile_builder/ParallelTiler.h +++ b/src/tile_builder/ParallelTiler.h @@ -23,7 +23,7 @@ class ParallelTiler : public Tiler { public: - ParallelTiler(const ctb::Grid& grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border, radix::tile::Scheme scheme); + ParallelTiler(const ctb::Grid& grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border); [[nodiscard]] std::vector generateTiles(unsigned zoom_level) const; [[nodiscard]] std::vector generateTiles(const std::pair& zoom_range) const; diff --git a/src/tile_builder/TileHeightsGenerator.cpp b/src/tile_builder/TileHeightsGenerator.cpp index 6c816dde..0e535fab 100644 --- a/src/tile_builder/TileHeightsGenerator.cpp +++ b/src/tile_builder/TileHeightsGenerator.cpp @@ -28,10 +28,9 @@ #include "depth_first_tile_traverser.h" #include -TileHeightsGenerator::TileHeightsGenerator(std::string input_data_path, ctb::Grid::Srs srs, radix::tile::Scheme scheme, radix::tile::Border border, std::filesystem::path output_path) +TileHeightsGenerator::TileHeightsGenerator(std::string input_data_path, ctb::Grid::Srs srs, radix::tile::Border border, std::filesystem::path output_path) : m_input_data_path(std::move(input_data_path)) , m_srs(srs) - , m_scheme(scheme) , m_border(border) , m_output_path(std::move(output_path)) { @@ -51,7 +50,7 @@ void TileHeightsGenerator::run(unsigned max_zoom_level) const grid = ctb::GlobalMercator(64); const auto bounds = dataset->bounds(grid.getSRS()); const auto tile_reader = DatasetReader(dataset, grid.getSRS(), 1, false); - const auto tiler = TopDownTiler(grid, bounds, m_border, m_scheme); + const auto tiler = TopDownTiler(grid, bounds, m_border); auto tile_heights = radix::TileHeights(); const auto read_function = [&](const radix::tile::Descriptor& tile) -> MinMaxData { @@ -74,10 +73,10 @@ void TileHeightsGenerator::run(unsigned max_zoom_level) const }; - traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, { 0, { 0, 0 }, m_scheme }, max_zoom_level); + traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, { 0, { 0, 0 } }, max_zoom_level); if (m_srs == ctb::Grid::Srs::WGS84) { // two root tiles - traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, { 0, { 1, 0 }, m_scheme }, max_zoom_level); + traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, { 0, { 1, 0 } }, max_zoom_level); } tile_heights.write_to(m_output_path); diff --git a/src/tile_builder/TileHeightsGenerator.h b/src/tile_builder/TileHeightsGenerator.h index 0d64af56..703a1227 100644 --- a/src/tile_builder/TileHeightsGenerator.h +++ b/src/tile_builder/TileHeightsGenerator.h @@ -29,11 +29,9 @@ class TileHeightsGenerator { std::string m_input_data_path; ctb::Grid::Srs m_srs; - radix::tile::Scheme m_scheme; radix::tile::Border m_border; std::filesystem::path m_output_path; public: - TileHeightsGenerator(std::string input_data_path, ctb::Grid::Srs srs, radix::tile::Scheme scheme, radix::tile::Border border, std::filesystem::path output_path); + TileHeightsGenerator(std::string input_data_path, ctb::Grid::Srs srs, radix::tile::Border border, std::filesystem::path output_path); void run(unsigned max_zoom_level) const; }; - diff --git a/src/tile_builder/Tiler.cpp b/src/tile_builder/Tiler.cpp index 00153c6c..fafa3ba7 100644 --- a/src/tile_builder/Tiler.cpp +++ b/src/tile_builder/Tiler.cpp @@ -20,11 +20,10 @@ #include -Tiler::Tiler(ctb::Grid grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border, radix::tile::Scheme scheme) +Tiler::Tiler(ctb::Grid grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border) : m_grid(std::move(grid)) , m_bounds(bounds) , m_border_south_east(border) - , m_scheme(scheme) { } @@ -55,11 +54,6 @@ radix::tile::Descriptor Tiler::tile_for(const radix::tile::Id& tile_id) const return {tile_id, srs_bounds, grid().getEpsgCode(), grid_size(), tile_size()}; } -radix::tile::Scheme Tiler::scheme() const -{ - return m_scheme; -} - const radix::tile::SrsBounds& Tiler::bounds() const { return m_bounds; @@ -69,4 +63,3 @@ void Tiler::setBounds(const radix::tile::SrsBounds& newBounds) { m_bounds = newBounds; } - diff --git a/src/tile_builder/Tiler.h b/src/tile_builder/Tiler.h index d477ce3b..70b5049e 100644 --- a/src/tile_builder/Tiler.h +++ b/src/tile_builder/Tiler.h @@ -25,9 +25,8 @@ class Tiler { public: - Tiler(ctb::Grid grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border, radix::tile::Scheme scheme); + Tiler(ctb::Grid grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border); - [[nodiscard]] radix::tile::Scheme scheme() const; [[nodiscard]] const radix::tile::SrsBounds& bounds() const; void setBounds(const radix::tile::SrsBounds& newBounds); [[nodiscard]] radix::tile::Descriptor tile_for(const radix::tile::Id& tile_id) const; @@ -43,6 +42,4 @@ class Tiler const ctb::Grid m_grid; radix::tile::SrsBounds m_bounds; const radix::tile::Border m_border_south_east; - const radix::tile::Scheme m_scheme; }; - diff --git a/src/tile_builder/TopDownTiler.cpp b/src/tile_builder/TopDownTiler.cpp index 9962962e..ca5ead02 100644 --- a/src/tile_builder/TopDownTiler.cpp +++ b/src/tile_builder/TopDownTiler.cpp @@ -18,15 +18,14 @@ #include "TopDownTiler.h" -TopDownTiler::TopDownTiler(const ctb::Grid& grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border, radix::tile::Scheme scheme) - : Tiler(grid, bounds, border, scheme) +TopDownTiler::TopDownTiler(const ctb::Grid& grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border) + : Tiler(grid, bounds, border) { } std::vector TopDownTiler::generateTiles(const radix::tile::Id& parent_id) const { - assert(parent_id.scheme == scheme()); - const auto tile_ids = parent_id.to(scheme()).children(); + const auto tile_ids = parent_id.children(); std::vector tiles; for (const auto& tile_id : tile_ids) { radix::tile::Descriptor t = tile_for(tile_id); diff --git a/src/tile_builder/TopDownTiler.h b/src/tile_builder/TopDownTiler.h index a2e065d1..f078ca89 100644 --- a/src/tile_builder/TopDownTiler.h +++ b/src/tile_builder/TopDownTiler.h @@ -24,7 +24,7 @@ class TopDownTiler : public Tiler { public: - TopDownTiler(const ctb::Grid& grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border, radix::tile::Scheme scheme); + TopDownTiler(const ctb::Grid& grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border); [[nodiscard]] std::vector generateTiles(const radix::tile::Id& parent_id) const; }; diff --git a/src/tile_builder/algorithms/primitives.h b/src/tile_builder/algorithms/primitives.h deleted file mode 100644 index a6101364..00000000 --- a/src/tile_builder/algorithms/primitives.h +++ /dev/null @@ -1,103 +0,0 @@ -#include -#include -#include - -#ifndef ALGORITHMS_PRIMITIVES_H -#define ALGORITHMS_PRIMITIVES_H - -namespace primitives { - -// two times the area of the ccw triangle with vertices a, b, and c. -// negative, if the triangle is cw -template -inline T triAreaX2(const glm::tvec2& a, const glm::tvec2& b, const glm::tvec2& c) -{ - // doesn't work for unsigned types. if you need that, come up with something :) - static_assert(std::is_signed_v); - return (b.x - a.x) * (c.y - a.y) - (b.y - a.y) * (c.x - a.x); -} - -enum class Winding : char { - CW = -1, - Undefined = 0, - CCW = 1 -}; - -template -inline Winding winding(const glm::tvec2& a, const glm::tvec2& b, const glm::tvec2& c) -{ - constexpr bool is_integral = std::is_integral_v; - if constexpr (is_integral) { - using Signed = typename std::conditional::type, T>::type; - auto v = (Signed(b.x) - Signed(a.x)) * (Signed(c.y) - Signed(a.y)) - (Signed(b.y) - Signed(a.y)) * (Signed(c.x) - Signed(a.x)); - if (v == 0) - return Winding::Undefined; - if (v < 0) - return Winding::CW; - return Winding::CCW; - } else { - auto v = (b.x - a.x) * (c.y - a.y) - (b.y - a.y) * (c.x - a.x); - if (std::abs(v) < 0.0000000000000001) - return Winding::Undefined; - if (v < 0) - return Winding::CW; - return Winding::CCW; - } -} - -template -inline bool ccw(const glm::tvec2& a, const glm::tvec2& b, const glm::tvec2& c) -{ - constexpr bool is_integral = std::is_integral_v; - if constexpr (is_integral) { - using Signed = typename std::conditional::type, T>::type; - if constexpr (include_border) - return (Signed(b.x) - Signed(a.x)) * (Signed(c.y) - Signed(a.y)) >= (Signed(b.y) - Signed(a.y)) * (Signed(c.x) - Signed(a.x)); - return (Signed(b.x) - Signed(a.x)) * (Signed(c.y) - Signed(a.y)) > (Signed(b.y) - Signed(a.y)) * (Signed(c.x) - Signed(a.x)); - } - if constexpr (include_border) - return (b.x - a.x) * (c.y - a.y) >= (b.y - a.y) * (c.x - a.x); - return (b.x - a.x) * (c.y - a.y) > (b.y - a.y) * (c.x - a.x); -} - -template -inline bool rightOf(const glm::tvec2& x, const glm::tvec2& org, const glm::tvec2& dest) -{ - return ccw(x, dest, org); -} - -template -inline bool leftOf(const glm::tvec2& x, const glm::tvec2& org, const glm::tvec2& dest) -{ - return ccw(x, org, dest); -} - -// https://www.scratchapixel.com/lessons/3d-basic-rendering/rasterization-practical-implementation/rasterization-stage -// the bottom edge of the raster is not included! -template -inline bool inside(const glm::tvec2& x, const glm::tvec2& a, const glm::tvec2& b, const glm::tvec2& c) -{ - constexpr bool is_integral = std::is_integral_v; - using Signed = typename std::conditional::type, T>::type; - using sVec = glm::tvec2; - // return leftOf(x, a, b) && leftOf(x, b, c) && leftOf(x, c, a); - const auto ccw = [](Winding w) { return w == Winding::CCW; }; - const auto undef = [](Winding w) { return w == Winding::Undefined; }; - const auto topleftedge = [](const sVec& edge) { return (edge.y == 0 && edge.x < 0) || edge.y < 0; }; - - const auto w_ab = winding(x, a, b); - const auto w_bc = winding(x, b, c); - const auto w_ca = winding(x, c, a); - const auto e_ab = sVec(b) - sVec(a); - const auto e_bc = sVec(c) - sVec(b); - const auto e_ca = sVec(a) - sVec(c); - - bool overlap = true; - overlap &= ccw(w_ab) || (undef(w_ab) && topleftedge(e_ab)); - overlap &= ccw(w_bc) || (undef(w_bc) && topleftedge(e_bc)); - overlap &= ccw(w_ca) || (undef(w_ca) && topleftedge(e_ca)); - return overlap; -} -} - -#endif diff --git a/src/tile_builder/algorithms/raster_triangle_scanline.h b/src/tile_builder/algorithms/raster_triangle_scanline.h deleted file mode 100644 index 7e22fdca..00000000 --- a/src/tile_builder/algorithms/raster_triangle_scanline.h +++ /dev/null @@ -1,101 +0,0 @@ -/***************************************************************************** - * Alpine Terrain Builder - * Copyright (C) 2022 alpinemaps.org - * Copyright (C) 2022 Adam Celarek - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *****************************************************************************/ - -#ifndef ALGORITHMS_RASTER_TRIANGLE_SCANLINE_H -#define ALGORITHMS_RASTER_TRIANGLE_SCANLINE_H - -#include "primitives.h" -#include "tntn/Raster.h" -#include -#include -#include - -namespace raster { - -template -void triangle_scanline(const tntn::Raster& raster, const glm::uvec2& a, const glm::uvec2& b, const glm::uvec2& c, const Lambda& fun) -{ - assert(primitives::winding(a, b, c) == primitives::Winding::CCW); - - const auto min = glm::min(glm::min(a, b), c); - const auto max = glm::max(glm::max(a, b), c); - for (auto y = min.y; y <= max.y; ++y) { - for (auto x = min.x; x <= max.x; ++x) { - const auto coord = glm::uvec2(x, y); - if (primitives::inside(coord, a, b, c)) - fun(coord, raster.value(coord.y, coord.x)); // raster is row / column - } - } - if (min.y == 0) { - // bottom row of raster is not included in primitives::inside, so check for it and make an extrawurscht. - const auto walk_bottom = [&](const glm::uvec2& a, const glm::uvec2& b) { - if ((b - a).y == 0 && a.y == 0) { - const auto end_x = std::max(a.x, b.x); - for (auto x = std::min(a.x, b.x); x < end_x; ++x) { - const auto coord = glm::uvec2(x, 0); - fun(coord, raster.value(coord.y, coord.x)); // raster is row / column - } - } - }; - walk_bottom(a, b); - walk_bottom(b, c); - walk_bottom(c, a); - } - const auto last_x = raster.get_width() - 1; - if (max.x == last_x) { - // similar with the rightmost column - const auto walk_right = [&](const glm::uvec2& a, const glm::uvec2& b) { - if ((b - a).x == 0 && a.x == last_x) { - const auto end_y = std::max(a.y, b.y); - for (auto y = std::min(a.y, b.y); y < end_y; ++y) { - const auto coord = glm::uvec2(last_x, y); - fun(coord, raster.value(coord.y, coord.x)); // raster is row / column - } - } - }; - walk_right(a, b); - walk_right(b, c); - walk_right(c, a); - } - // if (max.x == last_x && min.y == 0) { - // const auto check_br = [&](const glm::uvec2& a, const glm::uvec2& b) { - // if ((b - a).y == 0 && a.y == 0 && b.x == last_x) { - // const auto coord = glm::uvec2(last_x, 0); - // fun(coord, raster.value(coord.y, coord.x)); // raster is row / column - // } - // }; - // check_br(a, b); - // check_br(b, c); - // check_br(c, a); - // } - const auto last_y = raster.get_height() - 1; - if (max.x == last_x && max.y == last_y) { - const auto check_tr = [&](const glm::uvec2& a, const glm::uvec2& b) { - if ((b - a).x == 0 && b.y == last_y && b.x == last_x) { - const auto coord = glm::uvec2(last_x, last_y); - fun(coord, raster.value(coord.y, coord.x)); // raster is row / column - } - }; - check_tr(a, b); - check_tr(b, c); - check_tr(c, a); - } -} -} -#endif diff --git a/src/tile_builder/alpine_raster.cpp b/src/tile_builder/alpine_raster.cpp index b9ce5353..c5d8a6b0 100644 --- a/src/tile_builder/alpine_raster.cpp +++ b/src/tile_builder/alpine_raster.cpp @@ -25,18 +25,18 @@ #include #include -#include "Image.h" #include "ParallelTileGenerator.h" #include "ctb/Grid.hpp" +#include "image_writer.h" +#include #include -ParallelTileGenerator alpine_raster::make_generator(const std::string& input_data_path, const std::string& output_data_path, ctb::Grid::Srs srs, radix::tile::Scheme tiling_scheme, radix::tile::Border border, unsigned grid_resolution) +ParallelTileGenerator alpine_raster::make_generator(const std::string& input_data_path, const std::string& output_data_path, ctb::Grid::Srs srs, radix::tile::Border border, unsigned grid_resolution) { - return ParallelTileGenerator::make(input_data_path, srs, tiling_scheme, std::make_unique(border), output_data_path, grid_resolution); + return ParallelTileGenerator::make(input_data_path, srs, std::make_unique(border), output_data_path, grid_resolution); } -void alpine_raster::TileWriter::write(const std::string& file_path, const radix::tile::Descriptor&, const HeightData& heights) const +void alpine_raster::TileWriter::write(const std::string& file_path, const radix::tile::Descriptor&, const radix::Raster& heights) const { - image::saveImageAsPng(image::transformImage(heights, radix::height_encoding::to_rgb), - file_path); + image::saveImageAsPng(radix::raster::transform(heights, radix::height_encoding::to_rgb), file_path); } diff --git a/src/tile_builder/alpine_raster.h b/src/tile_builder/alpine_raster.h index 966d55fb..d82f05a6 100644 --- a/src/tile_builder/alpine_raster.h +++ b/src/tile_builder/alpine_raster.h @@ -25,7 +25,7 @@ #include #include -#include "Image.h" +#include #include "ParallelTileGenerator.h" #include #include "ctb/Grid.hpp" @@ -38,13 +38,12 @@ class TileWriter : public ParallelTileWriterInterface { : ParallelTileWriterInterface(border, "png") { } - void write(const std::string& base_path, const radix::tile::Descriptor& tile, const HeightData& heights) const override; + void write(const std::string& base_path, const radix::tile::Descriptor& tile, const radix::Raster& heights) const override; }; [[nodiscard]] ParallelTileGenerator make_generator( const std::string& input_data_path, const std::string& output_data_path, ctb::Grid::Srs srs, - radix::tile::Scheme tiling_scheme, radix::tile::Border border, unsigned grid_resolution = 256); }; diff --git a/src/tile_builder/image_writer.cpp b/src/tile_builder/image_writer.cpp new file mode 100644 index 00000000..b3135f90 --- /dev/null +++ b/src/tile_builder/image_writer.cpp @@ -0,0 +1,46 @@ +/***************************************************************************** + * Alpine Terrain Builder + * Copyright (C) 2022 alpinemaps.org + * Copyright (C) 2022 Adam Celarek + * Copyright (C) 2025 Martin Braunsperger + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *****************************************************************************/ + +#include "image_writer.h" + +#include "io/conversion.h" + +#include +#include + +void image::saveImageAsPng(const radix::Raster& input_image, const std::string& path) +{ + auto converted = io::conversion::to_mat(input_image); + if (!converted) { + throw std::runtime_error("Failed to convert raster to OpenCV image"); + } + + cv::Mat flipped; + cv::flip(*converted, flipped, 0); + cv::Mat image; + cv::cvtColor(flipped, image, cv::COLOR_RGB2BGR); + + try { + if (!cv::imwrite(path, image)) + throw std::runtime_error("Failed to write PNG image to " + path); + } catch (const cv::Exception& error) { + throw std::runtime_error("Failed to write PNG image to " + path + ": " + error.what()); + } +} diff --git a/src/tile_builder/image_writer.h b/src/tile_builder/image_writer.h new file mode 100644 index 00000000..db8ff8f9 --- /dev/null +++ b/src/tile_builder/image_writer.h @@ -0,0 +1,51 @@ +/***************************************************************************** + * Alpine Terrain Builder + * Copyright (C) 2022 alpinemaps.org + * Copyright (C) 2022 Adam Celarek + * Copyright (C) 2025 Martin Braunsperger + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *****************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include + +#include +#include + +namespace image { + +void saveImageAsPng(const radix::Raster& image, const std::string& path); + +template +void debugOut(const radix::Raster& image, const std::string& path) +{ + if (image.buffer().empty()) + throw std::invalid_argument("Can't write an empty raster to " + path); + + const auto [min, max] = std::ranges::minmax(image); + const auto range = float(max) - float(min); + saveImageAsPng(radix::raster::transform(image, [min, range](const auto value) { + const auto intensity = range == 0.F ? std::uint8_t(0) : std::uint8_t(255.F * (float(value) - float(min)) / range); + return glm::u8vec3(intensity); + }), + path); +} + +} // namespace image diff --git a/src/tile_builder/main.cpp b/src/tile_builder/main.cpp index c2f0fe05..06ec1c93 100644 --- a/src/tile_builder/main.cpp +++ b/src/tile_builder/main.cpp @@ -12,28 +12,26 @@ int main() { // const std::string input_raster = "/home/madam/rajaton/raw/Oe_2020/OeRect_01m_gs_31287.img"; // const std::string output_path = "/home/madam/rajaton/tiles/atb_terrain/"; - //// const auto generator = alpine_raster::make_generator("./test_tiles/", "/home/madam/valtava/raw/Oe_2020/OeRect_01m_gs_31287.img", ctb::Grid::Srs::SphericalMercator, Tiler::Scheme::SlippyMap, Tiler::Border::No); + //// const auto generator = alpine_raster::make_generator("./test_tiles/", "/home/madam/valtava/raw/Oe_2020/OeRect_01m_gs_31287.img", ctb::Grid::Srs::SphericalMercator, radix::tile::Border::No); //// generator.process({16, 16}); - // const auto generator = cesium_tin_terra::make_generator(input_raster, output_path, ctb::Grid::Srs::WGS84, Tiler::Scheme::Tms, Tiler::Border::Yes); // generator.process({0, 5}, true, true); // generator.process({6, 16}, true, false); const std::string input_raster = "/home/madam/valtava/raw/Oe_2020/OeRect_01m_gs_31287.img"; // const std::string input_raster = "/home/madam/valtava/raw/vienna/innenstadt_gs_1m_mgi.tif"; const std::string output_path = "/home/madam/valtava/tiles/alpine_png2"; - // const auto generator = alpine_raster::make_generator("./test_tiles/", "/home/madam/valtava/raw/Oe_2020/OeRect_01m_gs_31287.img", ctb::Grid::Srs::SphericalMercator, Tiler::Scheme::SlippyMap, Tiler::Border::No); + // const auto generator = alpine_raster::make_generator("./test_tiles/", "/home/madam/valtava/raw/Oe_2020/OeRect_01m_gs_31287.img", ctb::Grid::Srs::SphericalMercator, radix::tile::Border::No); // generator.process({16, 16}); - const auto generator = alpine_raster::make_generator(input_raster, output_path, ctb::Grid::Srs::SphericalMercator, radix::tile::Scheme::Tms, radix::tile::Border::Yes, 64); + const auto generator = alpine_raster::make_generator(input_raster, output_path, ctb::Grid::Srs::SphericalMercator, radix::tile::Border::Yes, 64); // generator.process({0, 5}, true, true); generator.process({ 15, 16 }, true, false); - // const auto metadata = MetaDataGenerator::make(input_raster, ctb::Grid::Srs::WGS84, Tiler::Scheme::Tms); // const auto json = layer_json_writer::process(metadata); //// generate height data (min and max) for tiles up to level 13 // const auto base_path = std::filesystem::path(output_path); // constexpr auto file_name = "height_data.atb"; -// const auto generator = TileHeightsGenerator(input_raster, ctb::Grid::Srs::SphericalMercator, radix::tile::Scheme::Tms, radix::tile::Border::Yes, base_path / file_name); +// const auto generator = TileHeightsGenerator(input_raster, ctb::Grid::Srs::SphericalMercator, radix::tile::Border::Yes, base_path / file_name); // generator.run(13); return 0; diff --git a/src/tile_downloader/HttpClient.h b/src/tile_downloader/HttpClient.h index 4db7e230..d5ef1ca4 100644 --- a/src/tile_downloader/HttpClient.h +++ b/src/tile_downloader/HttpClient.h @@ -1,8 +1,10 @@ #pragma once +#include #include #include #include +#include #include #include @@ -18,7 +20,10 @@ using ProgressFn = std::function; class HttpClient { public: - HttpClient() : _curl(curl_easy_init()) { + using WriteFn = std::function &, const char *, size_t)>; + + explicit HttpClient(WriteFn write = {}) + : _curl(curl_easy_init()), _write(std::move(write)) { if (!_curl) { throw std::runtime_error("failed to init cURL"); } @@ -35,10 +40,12 @@ class HttpClient { HttpResponse get(const std::string &url, const ProgressFn &on_progress = {}) const { HttpResponse response; + WriteContext write_context{&response.body, &this->_write, {}}; + ProgressContext progress_context{&on_progress, {}}; curl_easy_setopt(this->_curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(this->_curl, CURLOPT_WRITEFUNCTION, write_cb); - curl_easy_setopt(this->_curl, CURLOPT_WRITEDATA, &response.body); + curl_easy_setopt(this->_curl, CURLOPT_WRITEDATA, &write_context); curl_easy_setopt(this->_curl, CURLOPT_TIMEOUT, 5L); curl_easy_setopt(this->_curl, CURLOPT_CONNECTTIMEOUT, 5L); curl_easy_setopt(this->_curl, CURLOPT_FAILONERROR, 1L); @@ -46,12 +53,18 @@ class HttpClient { if (on_progress) { curl_easy_setopt(this->_curl, CURLOPT_NOPROGRESS, 0L); curl_easy_setopt(this->_curl, CURLOPT_XFERINFOFUNCTION, progress_cb); - curl_easy_setopt(this->_curl, CURLOPT_XFERINFODATA, &on_progress); + curl_easy_setopt(this->_curl, CURLOPT_XFERINFODATA, &progress_context); } else { curl_easy_setopt(this->_curl, CURLOPT_NOPROGRESS, 1L); } response.curl_code = curl_easy_perform(this->_curl); + if (write_context.exception) { + std::rethrow_exception(write_context.exception); + } + if (progress_context.exception) { + std::rethrow_exception(progress_context.exception); + } char *ct = nullptr; if (curl_easy_getinfo(this->_curl, CURLINFO_CONTENT_TYPE, &ct) == CURLE_OK && ct) { @@ -68,23 +81,50 @@ class HttpClient { } private: - CURL *_curl; + struct WriteContext { + std::vector *buffer; + const WriteFn *write; + std::exception_ptr exception; + }; - static size_t write_cb(void *ptr, size_t size, size_t nmemb, void *userdata) { - auto &buf = *static_cast *>(userdata); - size_t total = size * nmemb; - buf.insert(buf.end(), static_cast(ptr), static_cast(ptr) + total); - return total; + struct ProgressContext { + const ProgressFn *progress; + std::exception_ptr exception; + }; + + CURL *_curl; + WriteFn _write; + + static size_t write_cb(void *ptr, size_t size, size_t nmemb, void *userdata) noexcept { + auto &context = *static_cast(userdata); + const size_t total = size * nmemb; + try { + const auto *data = static_cast(ptr); + if (*context.write) { + (*context.write)(*context.buffer, data, total); + } else { + context.buffer->insert(context.buffer->end(), data, data + total); + } + return total; + } catch (...) { + context.exception = std::current_exception(); + return 0; + } } static int progress_cb(void *clientp, curl_off_t dltotal, curl_off_t dlnow, - curl_off_t /*ultotal*/, curl_off_t /*ulnow*/) { - const auto &fn = *static_cast(clientp); - if (dltotal > 0) { - fn(double(dlnow) / double(dltotal)); - } else { - fn(-1.0); + curl_off_t /*ultotal*/, curl_off_t /*ulnow*/) noexcept { + auto &context = *static_cast(clientp); + try { + if (dltotal > 0) { + (*context.progress)(double(dlnow) / double(dltotal)); + } else { + (*context.progress)(-1.0); + } + return 0; + } catch (...) { + context.exception = std::current_exception(); + return 1; } - return 0; } }; diff --git a/src/tile_downloader/TileDownloader.h b/src/tile_downloader/TileDownloader.h index 4678376c..e1f49bf4 100644 --- a/src/tile_downloader/TileDownloader.h +++ b/src/tile_downloader/TileDownloader.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include #include @@ -12,85 +11,81 @@ #include "HttpClient.h" #include "TileLogger.h" #include "TileUrlBuilder.h" +#include "tile_path.h" +#include "write_file.h" class TileDownloader { public: - TileDownloader(const TileUrlBuilder &url_builder, std::string output_pattern, - bool early_skip, std::optional max_zoom_level, unsigned root_zoom_level) + TileDownloader(const TileUrlBuilder &url_builder, std::filesystem::path output_directory, + std::optional max_zoom_level, unsigned root_zoom_level) : _url_builder(url_builder), - _output_pattern(std::move(output_pattern)), + _output_directory(std::move(output_directory)), _logger(root_zoom_level), - _early_skip(early_skip), _max_zoom_level(max_zoom_level) {} - void download_recursive(const radix::tile::Id &root_id) { - this->_logger.start(); - this->download_recursive_core(root_id); - this->_logger.finish(); + [[nodiscard]] bool download_recursive(const radix::tile::Id &root_id) { + auto progress_session = this->_logger.start(); + const bool complete = this->download_recursive_core(root_id); + progress_session.finish(); + return complete; } private: const TileUrlBuilder &_url_builder; - std::string _output_pattern; + std::filesystem::path _output_directory; HttpClient _http; TileLogger _logger; - bool _early_skip; std::optional _max_zoom_level; - void download_recursive_core(const radix::tile::Id &root_id) { + [[nodiscard]] bool download_recursive_core(const radix::tile::Id &root_id) { auto result = this->download_tile(root_id); this->_logger.report_error(root_id, result); + if (std::holds_alternative(result)) { + this->_logger.skipped(root_id); + return true; + } + + if (std::holds_alternative(result)) { + this->_logger.missing(root_id); + return true; + } + if (is_failure(result)) { this->_logger.missing(root_id); - return; + return false; } const auto children = root_id.children(); + bool children_complete = true; for (size_t i = 0; i < children.size(); i++) { - if (this->_early_skip && i + 1 < children.size()) { - if (this->tile_exists(children[i + 1])) { - this->_logger.skipped(children[i]); - continue; - } - } - if (this->_max_zoom_level.has_value() && children[i].zoom_level > *this->_max_zoom_level) { this->_logger.skipped(children[i]); continue; } - this->download_recursive_core(children[i]); + if (!this->download_recursive_core(children[i])) { + children_complete = false; + } + } + + if (!children_complete) { + return false; } + + mark_tile_children_complete(this->tile_path(root_id)); + return true; } static bool is_failure(const TileResult::Status &result) { - return std::holds_alternative(result) - || std::holds_alternative(result) + return std::holds_alternative(result) || std::holds_alternative(result) || std::holds_alternative(result) || std::holds_alternative(result); } - bool tile_exists(const radix::tile::Id &tile) const { - return std::filesystem::exists(this->tile_path(tile)); - } - - std::string tile_path(const radix::tile::Id &tile) const { - std::string path = this->_output_pattern; - replace_all(path, "{zoom}", std::to_string(tile.zoom_level)); - replace_all(path, "{x}", std::to_string(tile.coords.x)); - replace_all(path, "{y}", std::to_string(tile.coords.y)); - replace_all(path, "{ext}", "jpeg"); - return path; - } - - static void replace_all(std::string &s, std::string_view find, std::string_view replace) { - size_t pos = 0; - while ((pos = s.find(find, pos)) != std::string::npos) { - s.replace(pos, find.length(), replace); - pos += replace.length(); - } + std::filesystem::path tile_path(const radix::tile::Id &tile) const { + return google_tile_path(_output_directory, tile, ".jpeg"); } static void ensure_parent_dirs(const std::filesystem::path &path) { @@ -101,20 +96,15 @@ class TileDownloader { } } - static void write_file(const std::filesystem::path &path, const std::vector &data) { - std::ofstream out(path, std::ios::binary); - if (!out) { - throw std::runtime_error(fmt::format("failed to open \"{}\" for writing", path.string())); - } - out.write(data.data(), data.size()); - } - TileResult::Status download_tile(const radix::tile::Id &tile) { const auto path = std::filesystem::absolute(this->tile_path(tile)); if (std::filesystem::exists(path)) { return TileResult::Skipped{}; } + if (std::filesystem::exists(children_pending_tile_path(path))) { + return TileResult::ChildrenPending{}; + } ensure_parent_dirs(path); @@ -124,7 +114,7 @@ class TileDownloader { HttpResponse response = this->_http.get(url); if (response.curl_code == CURLE_OK && this->_http.is_image(response)) { - write_file(path, response.body); + write_file_children_pending(path, response.body); return TileResult::Downloaded{}; } diff --git a/src/tile_downloader/TileLogger.h b/src/tile_downloader/TileLogger.h index 3919e74c..c9d6c81b 100644 --- a/src/tile_downloader/TileLogger.h +++ b/src/tile_downloader/TileLogger.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -19,6 +20,7 @@ struct TileResult { struct Downloaded {}; + struct ChildrenPending {}; struct Skipped {}; struct Absent {}; struct HttpError { long status_code; }; @@ -26,18 +28,17 @@ struct TileResult { struct CurlError { CURLcode code; }; struct TimedOut {}; - using Status = std::variant; + using Status = std::variant; }; class TileLogger { public: + class Session; + explicit TileLogger(unsigned root_zoom_level) : _root_zoom_level(root_zoom_level), _progress(PROGRESS_RESOLUTION) {} - // Starts the progress display; pair with a call to finish(). - void start() { - this->_progress_thread = this->_progress.start_monitoring(); - } + [[nodiscard]] Session start(); // The tile turned out not to exist (or errored out), so its branch stops here. void missing(const radix::tile::Id &tile) { @@ -64,9 +65,20 @@ class TileLogger { }, status); } - // Fills in any steps still missing due to floating point rounding, so the - // progress indicator reaches its total, then joins the thread started by start(). - void finish() { +private: + static constexpr size_t PROGRESS_RESOLUTION = 10'000; + + unsigned _root_zoom_level; + ProgressIndicator _progress; + std::jthread _progress_thread; + size_t _steps_done = 0; + std::map _level_counts; + + void start_monitoring() { + this->_progress_thread = this->_progress.start_monitoring(); + } + + void finish_monitoring() { while (this->_steps_done < PROGRESS_RESOLUTION) { this->_progress.task_finished(); this->_steps_done++; @@ -76,14 +88,18 @@ class TileLogger { } } -private: - static constexpr size_t PROGRESS_RESOLUTION = 10'000; + void cancel_monitoring() noexcept { + if (!this->_progress_thread.joinable()) { + return; + } - unsigned _root_zoom_level; - ProgressIndicator _progress; - std::jthread _progress_thread; - size_t _steps_done = 0; - std::map _level_counts; + this->_progress_thread.request_stop(); + try { + this->_progress_thread.join(); + } catch (...) { + // Session cleanup must not replace the active exception. + } + } static std::string format(const radix::tile::Id &tile) { return fmt::format("Tile[Zoom={}, X={}, Y={}]", tile.zoom_level, tile.coords.x, tile.coords.y); @@ -114,3 +130,37 @@ class TileLogger { } } }; + +class TileLogger::Session { +public: + explicit Session(TileLogger &logger) + : _logger(&logger) { + this->_logger->start_monitoring(); + } + + Session(const Session &) = delete; + Session &operator=(const Session &) = delete; + + Session(Session &&other) noexcept + : _logger(std::exchange(other._logger, nullptr)) {} + + Session &operator=(Session &&) = delete; + + ~Session() { + if (this->_logger) { + this->_logger->cancel_monitoring(); + } + } + + void finish() { + this->_logger->finish_monitoring(); + this->_logger = nullptr; + } + +private: + TileLogger *_logger; +}; + +inline TileLogger::Session TileLogger::start() { + return Session(*this); +} diff --git a/src/tile_downloader/TileUrlBuilder.h b/src/tile_downloader/TileUrlBuilder.h index 328cf110..e28f3002 100644 --- a/src/tile_downloader/TileUrlBuilder.h +++ b/src/tile_downloader/TileUrlBuilder.h @@ -1,39 +1,85 @@ #pragma once +#include +#include #include +#include +#include -#include #include -class TileUrlBuilder { -public: - virtual ~TileUrlBuilder() = default; - virtual std::string build_url(const radix::tile::Id &tile_id) const = 0; +enum class TileDownloadProvider { + Basemap, + Gataki }; -class BasemapTileUrlBuilder : public TileUrlBuilder { -public: - BasemapTileUrlBuilder(std::string layer, std::string style) - : _layer(std::move(layer)), _style(std::move(style)) {} - - std::string build_url(const radix::tile::Id &tile_id) const override { - const auto t = tile_id.to(radix::tile::Scheme::SlippyMap); - return fmt::format( - "https://mapsneu.wien.gv.at/basemap/{}/{}/google3857/{}/{}/{}.jpeg", - this->_layer, this->_style, t.zoom_level, t.coords.y, t.coords.x); - } +enum class TileYDirection { + Down, + Up +}; -private: - std::string _layer; - std::string _style; +struct TileProviderConfig { + std::string url_pattern; + TileYDirection y_direction = TileYDirection::Down; }; -class GatakiTileUrlBuilder : public TileUrlBuilder { +[[nodiscard]] inline TileProviderConfig tile_provider_config(TileDownloadProvider provider) +{ + switch (provider) { + case TileDownloadProvider::Basemap: + return { + "https://mapsneu.wien.gv.at/basemap/bmaporthofoto30cm/normal/google3857/{zoom}/{y}/{x}.jpeg", + TileYDirection::Down + }; + case TileDownloadProvider::Gataki: + return { + "https://gataki.cg.tuwien.ac.at/raw/basemap/tiles/{zoom}/{y}/{x}.jpeg", + TileYDirection::Down + }; + } + + throw std::invalid_argument("unknown tile download provider"); +} + +class TileUrlBuilder { public: - std::string build_url(const radix::tile::Id &tile_id) const override { - const auto t = tile_id.to(radix::tile::Scheme::SlippyMap); - return fmt::format( - "https://gataki.cg.tuwien.ac.at/raw/basemap/tiles/{}/{}/{}.jpeg", - t.zoom_level, t.coords.y, t.coords.x); + explicit TileUrlBuilder(TileProviderConfig config) + : _url_pattern(std::move(config.url_pattern)) + , _y_direction(config.y_direction) + { + if (_url_pattern.find("{zoom}") == std::string::npos + || _url_pattern.find("{x}") == std::string::npos + || _url_pattern.find("{y}") == std::string::npos) { + throw std::invalid_argument("tile URL pattern must contain {zoom}, {x}, and {y}"); + } + } + + [[nodiscard]] std::string build_url(const radix::tile::Id& tile_id) const + { + auto y = tile_id.coords.y; + if (_y_direction == TileYDirection::Up) { + if (tile_id.zoom_level >= std::numeric_limits::digits) + throw std::invalid_argument("tile zoom level is too large for legacy TMS coordinates"); + y = (1u << tile_id.zoom_level) - y - 1; + } + + auto url = _url_pattern; + replace_all(url, "{zoom}", std::to_string(tile_id.zoom_level)); + replace_all(url, "{x}", std::to_string(tile_id.coords.x)); + replace_all(url, "{y}", std::to_string(y)); + return url; } + +private: + static void replace_all(std::string& value, std::string_view placeholder, const std::string& replacement) + { + size_t position = 0; + while ((position = value.find(placeholder, position)) != std::string::npos) { + value.replace(position, placeholder.size(), replacement); + position += replacement.size(); + } + } + + std::string _url_pattern; + TileYDirection _y_direction; }; diff --git a/src/tile_downloader/cli.cpp b/src/tile_downloader/cli.cpp index b83d4c36..bd1ca8af 100644 --- a/src/tile_downloader/cli.cpp +++ b/src/tile_downloader/cli.cpp @@ -13,29 +13,33 @@ Args parse(int argc, const char *const *argv) { Args args; - app.add_option("--provider", args.provider, "Tile provider (basemap or gataki)") - ->required() - ->check(CLI::IsMember({"basemap", "gataki"}, CLI::ignore_case)); + const std::map provider_map{ + {"basemap", TileDownloadProvider::Basemap}, + {"gataki", TileDownloadProvider::Gataki}}; + auto* source_group = app.add_option_group("Tile source"); + source_group->add_option("--provider", args.provider, "Configured tile provider (basemap or gataki)") + ->transform(CLI::CheckedTransformer(provider_map, CLI::ignore_case)); + auto* url_option = source_group->add_option("--url", args.url_pattern, "Custom tile URL pattern containing {zoom}, {x}, and {y}"); + source_group->require_option(1); app.add_option("--zoom", args.zoom, "Root tile zoom level")->required(); - app.add_option("--x,--row", args.x, "Root tile x coordinate")->required(); - app.add_option("--y,--col", args.y, "Root tile y coordinate")->required(); - - const std::map scheme_map{ - {"slippymap", radix::tile::Scheme::SlippyMap}, - {"google", radix::tile::Scheme::SlippyMap}, - {"xyz", radix::tile::Scheme::SlippyMap}, - {"tms", radix::tile::Scheme::Tms}}; - args.scheme = radix::tile::Scheme::SlippyMap; - app.add_option("--scheme", args.scheme, "Tile scheme") - ->default_val(radix::tile::Scheme::SlippyMap) - ->transform(CLI::CheckedTransformer(scheme_map, CLI::ignore_case)); + app.add_option("--x,--col", args.x, "Root tile x/column in Google/Mapbox coordinates")->required(); + app.add_option("--y,--row", args.y, "Root tile y/row in Google/Mapbox coordinates")->required(); + + const std::map y_direction_map{ + {"down", TileYDirection::Down}, + {"up", TileYDirection::Up}}; + args.url_y_direction = TileYDirection::Down; + app.add_option("--url-y-direction", args.url_y_direction, "Custom URL y direction: down (Google/Mapbox) or up (legacy TMS)") + ->default_str("down") + ->needs(url_option) + ->transform(CLI::CheckedTransformer(y_direction_map, CLI::ignore_case)); args.srs = 3857; app.add_option("--srs", args.srs, "Spatial reference system EPSG code")->default_val(3857); - args.output = "tiles/{zoom}/{y}/{x}.{ext}"; - app.add_option("--output", args.output, "Output path template")->default_val(args.output); + args.output = "tiles"; + app.add_option("--output", args.output, "Output directory; files use Google/Mapbox zoom/x/y.jpeg layout")->default_val(args.output.string()); const std::map log_level_names{ {"off", spdlog::level::off}, @@ -50,18 +54,8 @@ Args parse(int argc, const char *const *argv) { ->transform(CLI::CheckedTransformer(log_level_names, CLI::ignore_case)) ->default_val(spdlog::level::info); - args.early_skip = true; - app.add_option("--early-skip", args.early_skip, "Resume optimization: skip completed subtrees") - ->default_val(true); - app.add_option("--max-zoom-level", args.max_zoom_level, "Maximum zoom level to descend to"); - args.layer = "bmaporthofoto30cm"; - app.add_option("--layer", args.layer, "Basemap layer name")->default_val(args.layer); - - args.style = "normal"; - app.add_option("--style", args.style, "Basemap style")->default_val(args.style); - try { app.parse(argc, argv); } catch (const CLI::ParseError &e) { diff --git a/src/tile_downloader/cli.h b/src/tile_downloader/cli.h index 03edd30f..8e2380f8 100644 --- a/src/tile_downloader/cli.h +++ b/src/tile_downloader/cli.h @@ -4,24 +4,23 @@ #include #include -#include #include +#include "TileUrlBuilder.h" + namespace cli { struct Args { - std::string provider; + std::optional provider; + std::optional url_pattern; unsigned int zoom; unsigned int x; unsigned int y; - radix::tile::Scheme scheme; + TileYDirection url_y_direction; unsigned int srs; - std::string output; + std::filesystem::path output; spdlog::level::level_enum log_level; - bool early_skip; std::optional max_zoom_level; - std::string layer; - std::string style; }; Args parse(int argc, const char *const *argv); diff --git a/src/tile_downloader/main.cpp b/src/tile_downloader/main.cpp index 09c69d64..17c25314 100644 --- a/src/tile_downloader/main.cpp +++ b/src/tile_downloader/main.cpp @@ -1,7 +1,3 @@ -#include -#include -#include - #include "TileDownloader.h" #include "TileUrlBuilder.h" #include "cli.h" @@ -15,20 +11,13 @@ int main(int argc, char *argv[]) { LOG_ERROR_AND_EXIT("unsupported srs EPSG \"{}\"", args.srs); } - std::unique_ptr url_builder; - std::string provider = args.provider; - std::transform(provider.begin(), provider.end(), provider.begin(), - [](unsigned char c) { return std::tolower(c); }); - if (provider == "basemap") { - url_builder = std::make_unique(args.layer, args.style); - } else { - url_builder = std::make_unique(); - } - - const radix::tile::Id root_id = {args.zoom, {args.x, args.y}, args.scheme}; + const auto provider_config = args.provider.has_value() + ? tile_provider_config(*args.provider) + : TileProviderConfig { *args.url_pattern, args.url_y_direction }; + const TileUrlBuilder url_builder(provider_config); - TileDownloader downloader(*url_builder, args.output, args.early_skip, args.max_zoom_level, root_id.zoom_level); - downloader.download_recursive(root_id); + const radix::tile::Id root_id = {args.zoom, {args.x, args.y}}; - return 0; + TileDownloader downloader(url_builder, args.output, args.max_zoom_level, root_id.zoom_level); + return downloader.download_recursive(root_id) ? 0 : 1; } diff --git a/src/tile_downloader/write_file.h b/src/tile_downloader/write_file.h new file mode 100644 index 00000000..ab2d07f2 --- /dev/null +++ b/src/tile_downloader/write_file.h @@ -0,0 +1,65 @@ +#pragma once + +#include +#include +#include +#include + +#include + +namespace tile_downloader_detail { + +inline void write_file_checked_direct(const std::filesystem::path &path, const std::vector &data) +{ + std::ofstream output(path, std::ios::binary); + if (!output) { + throw std::runtime_error(fmt::format("failed to open \"{}\" for writing", path.string())); + } + + output.write(data.data(), static_cast(data.size())); + if (!output) { + throw std::runtime_error(fmt::format("failed to write \"{}\"", path.string())); + } + + output.close(); + if (!output) { + throw std::runtime_error(fmt::format("failed to finish writing \"{}\"", path.string())); + } +} + +} + +[[nodiscard]] inline std::filesystem::path partial_tile_path(const std::filesystem::path &path) +{ + auto partial_path = path; + partial_path += ".part"; + return partial_path; +} + +[[nodiscard]] inline std::filesystem::path children_pending_tile_path(const std::filesystem::path &path) +{ + auto pending_path = path; + pending_path += ".children-pending"; + return pending_path; +} + +inline void write_file_children_pending(const std::filesystem::path &path, const std::vector &data) +{ + const auto partial_path = partial_tile_path(path); + const auto pending_path = children_pending_tile_path(path); + + try { + tile_downloader_detail::write_file_checked_direct(partial_path, data); + std::filesystem::rename(partial_path, pending_path); + } catch (...) { + std::error_code cleanup_error; + std::filesystem::remove(partial_path, cleanup_error); + throw; + } +} + +inline void mark_tile_children_complete(const std::filesystem::path &path) +{ + const auto pending_path = children_pending_tile_path(path); + std::filesystem::rename(pending_path, path); +} diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 5b750e29..d3338950 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -20,17 +20,24 @@ add_executable(unittests_terrainlib terrainlib/cow.cpp terrainlib/dataset.cpp terrainlib/enumerate.cpp + terrainlib/envelope.cpp terrainlib/fixed_vector.cpp terrainlib/geometry_frames.cpp terrainlib/grid.cpp terrainlib/hash_utils.cpp terrainlib/hybrid_index_pair_map.cpp terrainlib/hybrid_vector.cpp + terrainlib/io_conversion.cpp terrainlib/offset_table.cpp terrainlib/segmented_buffer.cpp + terrainlib/sf_validate_index.cpp terrainlib/stamp_set.cpp - terrainlib/index_map.cpp - terrainlib/index_traverse.cpp + terrainlib/store_compatibility.cpp + terrainlib/store_codec.cpp + terrainlib/store_index.cpp + terrainlib/store_layout.cpp + terrainlib/store_storage.cpp + terrainlib/store_traverse.cpp terrainlib/mesh_boundary.cpp terrainlib/mesh_bounds.cpp terrainlib/mesh_cleanup.cpp @@ -72,7 +79,7 @@ if(TARGET tilebuilderlib) tilebuilder/alpine_raster_format.cpp tilebuilder/dataset_reading.cpp tilebuilder/depth_first_tile_traverser.cpp - tilebuilder/image.cpp + tilebuilder/image_writer.cpp tilebuilder/parallel_tile_generator.cpp tilebuilder/parallel_tiler.cpp tilebuilder/tile_heights_generator.cpp @@ -90,6 +97,29 @@ if(TARGET sfbuilderlib) ) target_link_libraries(unittests_sfbuilder PRIVATE sfbuilderlib Catch2::Catch2WithMain) atb_configure_test(unittests_sfbuilder) + + add_executable(unittests_sfbuilder_finalization + catch2_helpers.h + sf_builder/storage_finalization.cpp + ) + target_link_libraries( + unittests_sfbuilder_finalization + PRIVATE sfbuilderlib Catch2::Catch2WithMain) + atb_configure_test(unittests_sfbuilder_finalization) +endif() + +if(TARGET tile-downloader) + find_package(CURL REQUIRED) + add_executable(unittests_tile_downloader + tile_downloader/downloader.cpp + tile_downloader/http_client.cpp + tile_downloader/logger.cpp + tile_downloader/url_builder.cpp + tile_downloader/write_file.cpp + ) + target_include_directories(unittests_tile_downloader PRIVATE ${CMAKE_SOURCE_DIR}/src/tile_downloader) + target_link_libraries(unittests_tile_downloader PRIVATE terrainlib Catch2::Catch2WithMain CURL::libcurl) + atb_configure_test(unittests_tile_downloader) endif() if(TARGET dagbuilderlib) @@ -106,6 +136,7 @@ if(TARGET dagbuilderlib) dag_builder/fixed_point.cpp dag_builder/texture_sizing.cpp dag_builder/texture_reuse.cpp + dag_builder/storage_compatibility.cpp ) target_link_libraries(unittests_dagbuilder PUBLIC dagbuilderlib Catch2::Catch2WithMain) atb_configure_test(unittests_dagbuilder) @@ -116,6 +147,7 @@ if(TARGET sfmergerlib) catch2_helpers.h sf_merger/mask.cpp sf_merger/sphere_projector.cpp + sf_merger/storage_boundaries.cpp ) target_link_libraries(unittests_sfmerger PRIVATE sfmergerlib Catch2::Catch2WithMain) if(TARGET sf-builder AND TARGET sf-merger) diff --git a/unittests/dag_builder/storage_compatibility.cpp b/unittests/dag_builder/storage_compatibility.cpp new file mode 100644 index 00000000..22d6a704 --- /dev/null +++ b/unittests/dag_builder/storage_compatibility.cpp @@ -0,0 +1,212 @@ +#include +#include +#include +#include +#include + +#include + +#include "build.h" +#include "codec/Dag.h" +#include "dag_node.h" +#include "mesh/SimpleMesh.h" +#include "octree/storage/open.h" +#include "storage.h" +#include "thread_safe_storage.h" + +namespace { + +class TemporaryDirectory { +public: + TemporaryDirectory() { + static std::atomic_uint64_t counter = 0; + const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); + _path = std::filesystem::temp_directory_path() + / ("atb-dag-codec-" + std::to_string(timestamp) + "-" + + std::to_string(counter++)); + REQUIRE(std::filesystem::create_directories(_path)); + } + ~TemporaryDirectory() { + std::error_code error; + std::filesystem::remove_all(_path, error); + } + const std::filesystem::path &path() const { + return _path; + } + +private: + std::filesystem::path _path; +}; + +dag::ClusterBatch sample_batch() { + dag::ClusterBatch batch; + batch.metadata.group_assignment = {0}; + batch.metadata.groups = {{ + .children = {{octree::Id::root().child(4).value(), 3}}, + .error = 12.5, + .bounds = {}, + }}; + batch.clustering.positions = { + {0.0, 0.0, 0.0}, + {1.0, 0.0, 0.0}, + {0.0, 1.0, 0.0}, + }; + batch.clustering.clusters = {{ + .id = 17, + .vertex_indices = {0, 1, 2}, + .local_triangles = {{0, 1, 2}}, + .uvs = {}, + .texture_id = std::nullopt, + .absolute_error = 0.0, + }}; + return batch; +} + +mesh::Simple sample_mesh() { + return mesh::Simple( + {{0, 1, 2}}, + {{0.0, 0.0, 0.0}, {1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}}); +} + +} // namespace + +TEST_CASE("runtime DAG envelope codec is reentrant") { + const dag::ClusterBatch batch = sample_batch(); + + TemporaryDirectory output; + dag::codec::Dag codec; + std::vector> operations; + for (int index = 0; index < 8; ++index) { + operations.push_back(std::async(std::launch::async, [&codec, &batch, &output, index] { + const store::NodePath path(output.path() / std::to_string(index) / "node"); + if (!codec.write(path, batch).has_value()) { + return false; + } + const auto loaded = codec.read(path); + return loaded.has_value() + && loaded->metadata.group_assignment == batch.metadata.group_assignment; + })); + } + for (auto &operation : operations) { + REQUIRE(operation.get()); + } +} + +TEST_CASE("versioned DAG payload validation is free-standing") { + dag::format::NodeMetadata invalid_metadata{ + .group_assignment = {0}, + .groups = {}, + }; + CHECK_FALSE(dag::format::validate(invalid_metadata).has_value()); + + dag::format::Clustering oversized_clustering{ + .vertex_count = std::numeric_limits::max(), + .encoded_positions = {}, + .clusters = {}, + .textures = {}, + }; + CHECK_FALSE(dag::format::validate(oversized_clustering).has_value()); +} + +TEST_CASE("DAG resolvers expose writable batches and read-only metadata", "[store][open]") { + const auto batch_codec = dag::codec::from_extension(".dag"); + REQUIRE(batch_codec.has_value()); + CHECK(batch_codec.value()->paths(store::NodePath("node")) + == std::vector{"node.dag", "node.dagmeta"}); + + const auto metadata_codec = dag::codec::metadata_from_extension(".dag"); + REQUIRE(metadata_codec.has_value()); + CHECK(metadata_codec.value()->paths(store::NodePath("node")) + == std::vector{"node.dagmeta"}); + const auto write_result = metadata_codec.value()->write( + store::NodePath("node"), + dag::NodeMetadata{}); + REQUIRE_FALSE(write_result.has_value()); + CHECK(write_result.error().operation == store::CodecOperation::Write); + CHECK(write_result.error().category == store::CodecErrorCategory::UnsupportedOperation); + + const auto unknown = dag::codec::from_extension(".unknown"); + REQUIRE_FALSE(unknown.has_value()); + CHECK(unknown.error().category == store::CodecErrorCategory::UnsupportedCodec); +} + +TEST_CASE("DAG indexed storage survives ThreadSafeStorage move and release", "[store][storage]") { + const dag::ClusterBatch batch = sample_batch(); + + TemporaryDirectory output; + auto storage_result = dag::storage::open_folder_indexed(output.path()); + REQUIRE(storage_result.has_value()); + dag::ThreadSafeStorage synchronized(std::move(storage_result.value())); + REQUIRE(synchronized.save(octree::Id::root(), batch).has_value()); + + auto released = std::move(synchronized).release(); + REQUIRE(released.save_index().has_value()); + CHECK(std::filesystem::is_regular_file(output.path() / "octree.storeindex")); + CHECK(std::filesystem::is_regular_file(output.path() / "octree.storemeta")); + + auto reopened_result = dag::storage::open_folder_indexed(output.path()); + REQUIRE(reopened_result.has_value()); + CHECK(reopened_result->has(octree::Id::root()).value()); + CHECK(reopened_result->load(octree::Id::root()).has_value()); + const auto full_paths = reopened_result->paths(octree::Id::root()); + REQUIRE(full_paths.has_value()); + REQUIRE(full_paths->size() == 2); + + auto metadata_result = dag::storage::open_metadata_indexed(output.path()); + REQUIRE(metadata_result.has_value()); + REQUIRE(std::filesystem::remove(full_paths->front())); + const auto metadata = metadata_result->load(octree::Id::root()); + REQUIRE(metadata.has_value()); + CHECK(metadata->group_assignment == batch.metadata.group_assignment); + CHECK(metadata->groups.front().error == batch.metadata.groups.front().error); +} + +TEST_CASE("DAG storage hard-links both files for the same format", "[store][copy]") { + TemporaryDirectory source_directory; + TemporaryDirectory target_directory; + auto source_result = dag::storage::open_folder_indexed(source_directory.path()); + auto target_result = dag::storage::open_folder_indexed(target_directory.path()); + REQUIRE(source_result.has_value()); + REQUIRE(target_result.has_value()); + auto source = std::move(*source_result); + auto target = std::move(*target_result); + const octree::Id root = octree::Id::root(); + REQUIRE(source.save(root, sample_batch()).has_value()); + REQUIRE(target.copy_from(root, source).has_value()); + + const auto source_paths = source.paths(root); + const auto target_paths = target.paths(root); + REQUIRE(source_paths.has_value()); + REQUIRE(target_paths.has_value()); + REQUIRE(source_paths->size() == 2); + REQUIRE(target_paths->size() == 2); + CHECK(std::filesystem::equivalent((*source_paths)[0], (*target_paths)[0])); + CHECK(std::filesystem::equivalent((*source_paths)[1], (*target_paths)[1])); +} + +TEST_CASE("DAG builder rejects invalid SF topology before processing", "[dag][sf][validation]") { + TemporaryDirectory input_directory; + TemporaryDirectory output_directory; + auto input_result = octree::open_folder_indexed(input_directory.path()); + REQUIRE(input_result.has_value()); + auto input = std::move(input_result.value()); + const octree::Id root = octree::Id::root(); + const mesh::Simple mesh = sample_mesh(); + REQUIRE(input.save(root, mesh).has_value()); + REQUIRE(input.save(root.child(0).value(), mesh).has_value()); + REQUIRE(input.index().is(store::NodeStatus::Inner, root).value()); + + auto output_result = dag::storage::open_folder_indexed(output_directory.path()); + REQUIRE(output_result.has_value()); + auto output = std::move(output_result.value()); + const dag::BuildOptions options{ + .clusters_per_partition = 1, + .target_ratio = 1.0f, + .relative_target_error = std::nullopt, + }; + + const auto built = dag::build_full(input, output, options); + REQUIRE_FALSE(built.has_value()); + CHECK(built.error().key == root); + CHECK(output.index().empty()); +} diff --git a/unittests/sf_builder/storage_finalization.cpp b/unittests/sf_builder/storage_finalization.cpp new file mode 100644 index 00000000..d864949a --- /dev/null +++ b/unittests/sf_builder/storage_finalization.cpp @@ -0,0 +1,77 @@ +#include +#include +#include + +#include + +#include "octree/storage/open.h" +#include "terrainbuilder.h" + +namespace { + +class TemporaryDirectory { +public: + TemporaryDirectory() { + static std::atomic_uint64_t counter = 0; + const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); + _path = std::filesystem::temp_directory_path() + / ("atb-sfbuilder-finalize-" + std::to_string(timestamp) + "-" + + std::to_string(counter++)); + REQUIRE(std::filesystem::create_directories(_path)); + } + + ~TemporaryDirectory() { + std::error_code error; + std::filesystem::remove_all(_path, error); + } + + const std::filesystem::path &path() const { + return _path; + } + +private: + std::filesystem::path _path; +}; + +mesh::Simple sample_mesh() { + return mesh::Simple( + {{0, 1, 2}}, + {{0.0, 0.0, 0.0}, {1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}}); +} + +} // namespace + +TEST_CASE("SF builder finalization writes and validates a valid index", "[sfbuilder][sf]") { + TemporaryDirectory directory; + auto storage_result = octree::open_folder(directory.path()); + REQUIRE(storage_result.has_value()); + auto storage = std::move(storage_result.value()); + REQUIRE(storage.save(octree::Id::root(), sample_mesh()).has_value()); + + CHECK(terrainbuilder::finalize_storage(storage).has_value()); + CHECK(std::filesystem::is_regular_file(directory.path() / "octree.storeindex")); + CHECK(std::filesystem::is_regular_file(directory.path() / "octree.storemeta")); +} + +TEST_CASE( + "SF builder finalization retains an invalid written index for diagnosis", + "[sfbuilder][sf]") { + TemporaryDirectory directory; + auto storage_result = octree::open_folder(directory.path()); + REQUIRE(storage_result.has_value()); + auto storage = std::move(storage_result.value()); + const octree::Id root = octree::Id::root(); + const mesh::Simple mesh = sample_mesh(); + REQUIRE(storage.save(root, mesh).has_value()); + REQUIRE(storage.save(root.child(0).value(), mesh).has_value()); + + const auto finalized = terrainbuilder::finalize_storage(storage); + REQUIRE_FALSE(finalized.has_value()); + REQUIRE(std::holds_alternative(finalized.error())); + CHECK(std::get(finalized.error()).key == root); + CHECK(std::filesystem::is_regular_file(directory.path() / "octree.storeindex")); + + auto reopened = octree::open_index(directory.path() / "octree.storeindex"); + REQUIRE(reopened.has_value()); + CHECK(reopened->index().is(store::NodeStatus::Inner, root).value()); +} diff --git a/unittests/sf_builder/texture.cpp b/unittests/sf_builder/texture.cpp index f8b4ffdf..3da052b4 100644 --- a/unittests/sf_builder/texture.cpp +++ b/unittests/sf_builder/texture.cpp @@ -34,7 +34,7 @@ TEST_CASE("estimate_zoom_level", "[terrainbuilder]") { const ctb::Grid grid = ctb::GlobalMercator(); - const radix::tile::Id tile(20, glm::uvec2(0, 1), radix::tile::Scheme::SlippyMap); + const radix::tile::Id tile(20, glm::uvec2(0, 1)); const radix::tile::SrsBounds tile_bounds = grid.srsBounds(tile, false); const radix::tile::SrsBounds shifted_bounds(tile_bounds.min + glm::dvec2(-100, 420), tile_bounds.max + glm::dvec2(-100, 420)); @@ -67,11 +67,11 @@ class AlwaysEmptyTileProvider : public TileProvider { }; TEST_CASE("texture assembler takes root tile if only available ", "[terrainbuilder]") { - const radix::tile::Id root_tile(3, glm::uvec2(5, 4), radix::tile::Scheme::SlippyMap); + const radix::tile::Id root_tile(3, glm::uvec2(5, 4)); const std::set available_tiles = { - {3, {5, 4}, radix::tile::Scheme::SlippyMap}}; + {3, {5, 4}}}; const std::set expected_tiles = { - {3, {5, 4}, radix::tile::Scheme::SlippyMap}}; + {3, {5, 4}}}; const AvailabilityListEmptyTileProvider tile_provider(available_tiles); @@ -88,18 +88,18 @@ TEST_CASE("texture assembler takes root tile if only available ", "[terrainbuild } TEST_CASE("texture assembler ignores parent if all children are present", "[terrainbuilder]") { - const radix::tile::Id root_tile(3, glm::uvec2(5, 4), radix::tile::Scheme::SlippyMap); + const radix::tile::Id root_tile(3, glm::uvec2(5, 4)); const std::set available_tiles = { - {3, {5, 4}, radix::tile::Scheme::SlippyMap}, - {4, {10, 8}, radix::tile::Scheme::SlippyMap}, - {4, {11, 8}, radix::tile::Scheme::SlippyMap}, - {4, {10, 9}, radix::tile::Scheme::SlippyMap}, - {4, {11, 9}, radix::tile::Scheme::SlippyMap}}; + {3, {5, 4}}, + {4, {10, 8}}, + {4, {11, 8}}, + {4, {10, 9}}, + {4, {11, 9}}}; const std::set expected_tiles = { - {4, {10, 8}, radix::tile::Scheme::SlippyMap}, - {4, {11, 8}, radix::tile::Scheme::SlippyMap}, - {4, {10, 9}, radix::tile::Scheme::SlippyMap}, - {4, {11, 9}, radix::tile::Scheme::SlippyMap}}; + {4, {10, 8}}, + {4, {11, 8}}, + {4, {10, 9}}, + {4, {11, 9}}}; const AvailabilityListEmptyTileProvider tile_provider(available_tiles); @@ -116,15 +116,15 @@ TEST_CASE("texture assembler ignores parent if all children are present", "[terr } TEST_CASE("texture assembler considers max zoom level", "[terrainbuilder]") { - const radix::tile::Id root_tile(3, glm::uvec2(5, 4), radix::tile::Scheme::SlippyMap); + const radix::tile::Id root_tile(3, glm::uvec2(5, 4)); const std::set available_tiles = { - {3, {5, 4}, radix::tile::Scheme::SlippyMap}, - {4, {10, 8}, radix::tile::Scheme::SlippyMap}, - {4, {11, 8}, radix::tile::Scheme::SlippyMap}, - {4, {10, 9}, radix::tile::Scheme::SlippyMap}, - {4, {11, 9}, radix::tile::Scheme::SlippyMap}}; + {3, {5, 4}}, + {4, {10, 8}}, + {4, {11, 8}}, + {4, {10, 9}}, + {4, {11, 9}}}; const std::set expected_tiles = { - {3, {5, 4}, radix::tile::Scheme::SlippyMap}}; + {3, {5, 4}}}; const AvailabilityListEmptyTileProvider tile_provider(available_tiles); @@ -143,29 +143,29 @@ TEST_CASE("texture assembler considers max zoom level", "[terrainbuilder]") { TEST_CASE("texture assembler works for arbitrary bounds", "[terrainbuilder]") { const std::set available_tiles = { - // {21, {1048576, 1048576}, radix::tile::Scheme::Tms}, - {21, {1048577, 1048576}, radix::tile::Scheme::Tms}, - {21, {1048578, 1048576}, radix::tile::Scheme::Tms}, - {21, {1048579, 1048576}, radix::tile::Scheme::Tms}, - {21, {1048580, 1048576}, radix::tile::Scheme::Tms}, - // {21, {1048581, 1048576}, radix::tile::Scheme::Tms}, - {20, {524288, 524288}, radix::tile::Scheme::Tms}, - {20, {524289, 524288}, radix::tile::Scheme::Tms}, - {20, {524290, 524288}, radix::tile::Scheme::Tms}, - // {19, {262144, 262144}, radix::tile::Scheme::Tms}, - {19, {262145, 262144}, radix::tile::Scheme::Tms}}; + // {21, {1048576, 1048575}}, + {21, {1048577, 1048575}}, + {21, {1048578, 1048575}}, + {21, {1048579, 1048575}}, + {21, {1048580, 1048575}}, + // {21, {1048581, 1048575}}, + {20, {524288, 524287}}, + {20, {524289, 524287}}, + {20, {524290, 524287}}, + // {19, {262144, 262143}}, + {19, {262145, 262143}}}; const std::set expected_tiles = { - // {21, {1048576, 1048576}, radix::tile::Scheme::Tms}, - {21, {1048577, 1048576}, radix::tile::Scheme::Tms}, - {21, {1048578, 1048576}, radix::tile::Scheme::Tms}, - {21, {1048579, 1048576}, radix::tile::Scheme::Tms}, - {21, {1048580, 1048576}, radix::tile::Scheme::Tms}, - // {21, {1048581, 1048576}, radix::tile::Scheme::Tms}, - {20, {524288, 524288}, radix::tile::Scheme::Tms}, - // {20, {524289, 524288}, radix::tile::Scheme::Tms}, - {20, {524290, 524288}, radix::tile::Scheme::Tms}, - // {19, {262144, 262144}, radix::tile::Scheme::Tms}, - // {19, {262145, 262144}, radix::tile::Scheme::Tms} + // {21, {1048576, 1048575}}, + {21, {1048577, 1048575}}, + {21, {1048578, 1048575}}, + {21, {1048579, 1048575}}, + {21, {1048580, 1048575}}, + // {21, {1048581, 1048575}}, + {20, {524288, 524287}}, + // {20, {524289, 524287}}, + {20, {524290, 524287}}, + // {19, {262144, 262143}}, + // {19, {262145, 262143}} }; const AvailabilityListEmptyTileProvider tile_provider(available_tiles); @@ -208,7 +208,7 @@ TEST_CASE("texture assembler does not fail if there are not tiles", "[terrainbui TEST_CASE("texture assembler assembles single tile", "[terrainbuilder]") { const std::unordered_map tiles_to_texture = { - {radix::tile::Id(0, {0, 0}, radix::tile::Scheme::SlippyMap), cv::Mat(1, 1, CV_8UC3, cv::Vec3b(0, 0, 255))}, + {radix::tile::Id(0, {0, 0}), cv::Mat(1, 1, CV_8UC3, cv::Vec3b(0, 0, 255))}, }; std::vector tiles_to_splatter; std::transform(tiles_to_texture.begin(), tiles_to_texture.end(), std::back_inserter(tiles_to_splatter), @@ -217,7 +217,7 @@ TEST_CASE("texture assembler assembles single tile", "[terrainbuilder]") { const StaticTileProvider tile_provider(tiles_to_texture); const ctb::Grid grid = ctb::GlobalMercator(); - const radix::tile::Id root_tile(0, {0, 0}, radix::tile::Scheme::SlippyMap); + const radix::tile::Id root_tile(0, {0, 0}); cv::Mat assembled_texture = terrainbuilder::splatter_tiles_to_texture( root_tile, grid, @@ -234,8 +234,8 @@ TEST_CASE("texture assembler assembles single tile", "[terrainbuilder]") { TEST_CASE("texture assembler assembles two tiles", "[terrainbuilder]") { const std::unordered_map tiles_to_texture = { - {radix::tile::Id(1, {0, 0}, radix::tile::Scheme::SlippyMap), cv::Mat(1, 1, CV_8UC3, cv::Vec3b(0, 0, 255))}, - {radix::tile::Id(1, {0, 1}, radix::tile::Scheme::SlippyMap), cv::Mat(1, 1, CV_8UC3, cv::Vec3b(0, 255, 0))}, + {radix::tile::Id(1, {0, 0}), cv::Mat(1, 1, CV_8UC3, cv::Vec3b(0, 0, 255))}, + {radix::tile::Id(1, {0, 1}), cv::Mat(1, 1, CV_8UC3, cv::Vec3b(0, 255, 0))}, }; std::vector tiles_to_splatter; std::transform(tiles_to_texture.begin(), tiles_to_texture.end(), std::back_inserter(tiles_to_splatter), @@ -244,7 +244,7 @@ TEST_CASE("texture assembler assembles two tiles", "[terrainbuilder]") { const StaticTileProvider tile_provider(tiles_to_texture); const ctb::Grid grid = ctb::GlobalMercator(); - const radix::tile::Id root_tile(0, {0, 0}, radix::tile::Scheme::SlippyMap); + const radix::tile::Id root_tile(0, {0, 0}); cv::Mat assembled_texture = terrainbuilder::splatter_tiles_to_texture( root_tile, grid, @@ -262,9 +262,9 @@ TEST_CASE("texture assembler assembles two tiles", "[terrainbuilder]") { TEST_CASE("texture assembler correct order of texture writes", "[terrainbuilder]") { const std::unordered_map tiles_to_texture = { - {radix::tile::Id(0, {0, 0}, radix::tile::Scheme::SlippyMap), cv::Mat(1, 1, CV_8UC1, uint8_t(1))}, - {radix::tile::Id(1, {0, 1}, radix::tile::Scheme::Tms), cv::Mat(1, 1, CV_8UC1, uint8_t(2))}, - {radix::tile::Id(1, {0, 1}, radix::tile::Scheme::SlippyMap), cv::Mat(1, 1, CV_8UC1, uint8_t(3))}, + {radix::tile::Id(0, {0, 0}), cv::Mat(1, 1, CV_8UC1, uint8_t(1))}, + {radix::tile::Id(1, {0, 0}), cv::Mat(1, 1, CV_8UC1, uint8_t(2))}, + {radix::tile::Id(1, {0, 1}), cv::Mat(1, 1, CV_8UC1, uint8_t(3))}, }; std::vector tiles_to_splatter; std::transform(tiles_to_texture.begin(), tiles_to_texture.end(), std::back_inserter(tiles_to_splatter), @@ -275,7 +275,7 @@ TEST_CASE("texture assembler correct order of texture writes", "[terrainbuilder] const StaticTileProvider tile_provider(tiles_to_texture); const ctb::Grid grid = ctb::GlobalMercator(); - const radix::tile::Id root_tile(0, {0, 0}, radix::tile::Scheme::SlippyMap); + const radix::tile::Id root_tile(0, {0, 0}); cv::Mat assembled_texture = terrainbuilder::splatter_tiles_to_texture( root_tile, grid, @@ -351,7 +351,7 @@ TEST_CASE("texture assembler reports the content region", "[terrainbuilder]") { tile_image.row(row).setTo(uint8_t(row)); } - const radix::tile::Id root_tile(0, {0, 0}, radix::tile::Scheme::SlippyMap); + const radix::tile::Id root_tile(0, {0, 0}); const StaticTileProvider tile_provider({{root_tile, tile_image}}); const std::vector tiles_to_splatter = {root_tile}; diff --git a/unittests/sf_merger/integration.cpp b/unittests/sf_merger/integration.cpp index 608d4d91..faa25e0d 100644 --- a/unittests/sf_merger/integration.cpp +++ b/unittests/sf_merger/integration.cpp @@ -58,7 +58,7 @@ int build_sf( " --min-texture-level 12 --max-texture-level 17" " --mesh-srs EPSG:4978 --verbosity warn" " batch --target-level 15 --output " + shell_quote(output) + - " --format .terrain --threads 1"; + " --format .sfmesh --threads 1"; return std::system(command.c_str()); } @@ -90,8 +90,8 @@ TEST_CASE("SF builders preserve geometry at regular and point-touching mask bord " --verbosity warn"; REQUIRE(std::system(merge_command.c_str()) == 0); - const std::filesystem::path working_relative = "15/26291/18610/27235.terrain"; - const std::filesystem::path regression_relative = "15/26290/18610/27235.terrain"; + const std::filesystem::path working_relative = "15/26291/18610/27235.sfmesh"; + const std::filesystem::path regression_relative = "15/26290/18610/27235.sfmesh"; const std::filesystem::path working_path = merged_output / working_relative; const std::filesystem::path regression_path = merged_output / regression_relative; diff --git a/unittests/sf_merger/mask.cpp b/unittests/sf_merger/mask.cpp index 9006484a..4689fd80 100644 --- a/unittests/sf_merger/mask.cpp +++ b/unittests/sf_merger/mask.cpp @@ -1,3 +1,5 @@ +#include "../catch2_helpers.h" + #include #include @@ -17,6 +19,58 @@ #include "octree/Space.h" #include "utils.h" +TEST_CASE("Mask simplification uses a fixed tenth of a metre") { + OGRSpatialReference projected; + REQUIRE(projected.importFromEPSG(31287) == OGRERR_NONE); + + const std::optional projected_tolerance = mask::simplification_tolerance(projected); + REQUIRE(projected_tolerance); + CHECK(*projected_tolerance == Catch::Approx(0.1)); + + OGRSpatialReference geographic; + REQUIRE(geographic.SetWellKnownGeogCS("WGS84") == OGRERR_NONE); + + const std::optional geographic_tolerance = mask::simplification_tolerance(geographic); + REQUIRE(geographic_tolerance); + CHECK(*geographic_tolerance == Catch::Approx( + 0.1 / geographic.GetSemiMajor() / geographic.GetAngularUnits())); +} + +TEST_CASE("Mask simplification preserves polygon topology") { + OGRLinearRing outer; + outer.addPoint(0.0, 0.0); + outer.addPoint(1.0, 0.01); + outer.addPoint(2.0, 0.0); + outer.addPoint(2.0, 2.0); + outer.addPoint(0.0, 2.0); + outer.addPoint(0.0, 0.0); + + OGRLinearRing hole; + hole.addPoint(0.5, 0.5); + hole.addPoint(1.0, 0.51); + hole.addPoint(1.5, 0.5); + hole.addPoint(1.5, 1.5); + hole.addPoint(0.5, 1.5); + hole.addPoint(0.5, 0.5); + + OGRPolygon polygon; + REQUIRE(polygon.addRing(&outer) == OGRERR_NONE); + REQUIRE(polygon.addRing(&hole) == OGRERR_NONE); + REQUIRE(polygon.IsValid()); + + const uint64_t original_point_count = mask::point_count(polygon); + std::unique_ptr simplified = mask::simplify_geometry(polygon, 0.1); + + REQUIRE(simplified); + REQUIRE_FALSE(simplified->IsEmpty()); + REQUIRE(simplified->IsValid()); + CHECK(mask::point_count(*simplified) < original_point_count); + + const OGRPolygon *simplified_polygon = simplified->toPolygon(); + REQUIRE(simplified_polygon); + CHECK(simplified_polygon->getNumInteriorRings() == 1); +} + namespace { Polygon2 make_square(const glm::dvec2 min, const glm::dvec2 max) { diff --git a/unittests/sf_merger/storage_boundaries.cpp b/unittests/sf_merger/storage_boundaries.cpp new file mode 100644 index 00000000..05de6686 --- /dev/null +++ b/unittests/sf_merger/storage_boundaries.cpp @@ -0,0 +1,196 @@ +#include +#include +#include + +#include + +#include "cut.h" +#include "merge.h" +#include "octree/storage/open.h" + +namespace { + +class TemporaryDirectory { +public: + explicit TemporaryDirectory(const std::string_view label) { + static std::atomic_uint64_t counter = 0; + const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); + _path = std::filesystem::temp_directory_path() + / ("atb-sfmerger-" + std::string(label) + "-" + + std::to_string(timestamp) + "-" + std::to_string(counter++)); + REQUIRE(std::filesystem::create_directories(_path)); + } + + ~TemporaryDirectory() { + std::error_code error; + std::filesystem::remove_all(_path, error); + } + + const std::filesystem::path &path() const { + return _path; + } + +private: + std::filesystem::path _path; +}; + +mesh::Simple triangle(const double x_offset = 0.0) { + return mesh::Simple( + {{0, 1, 2}}, + { + {-1.0 + x_offset, 0.0, 0.0}, + {1.0 + x_offset, 0.0, 0.0}, + {x_offset, 2.0, 0.0}, + }); +} + +mesh::Simple clipping_box() { + return mesh::Simple( + { + {0, 1, 2}, {0, 2, 3}, + {1, 5, 6}, {1, 6, 2}, + {5, 4, 7}, {5, 7, 6}, + {4, 0, 3}, {4, 3, 7}, + {3, 2, 6}, {3, 6, 7}, + {4, 5, 1}, {4, 1, 0}, + }, + { + {0.0, -1.0, -1.0}, + {2.0, -1.0, -1.0}, + {2.0, 3.0, -1.0}, + {0.0, 3.0, -1.0}, + {0.0, -1.0, 1.0}, + {2.0, -1.0, 1.0}, + {2.0, 3.0, 1.0}, + {0.0, 3.0, 1.0}, + }); +} + +mesh::storage::IndexedStorage indexed_storage(const std::filesystem::path &path) { + auto result = octree::open_folder_indexed(path); + REQUIRE(result.has_value()); + return std::move(result.value()); +} + +mesh::storage::Storage unindexed_storage(const std::filesystem::path &path) { + auto result = octree::open_folder(path); + REQUIRE(result.has_value()); + return std::move(result.value()); +} + +} // namespace + +TEST_CASE("SF merge rejects Inner input before dispatch", "[sfmerger][sf][validation]") { + TemporaryDirectory left_directory("invalid-left"); + TemporaryDirectory right_directory("invalid-right"); + TemporaryDirectory output_directory("invalid-output"); + auto left = indexed_storage(left_directory.path()); + auto right = indexed_storage(right_directory.path()); + auto output = unindexed_storage(output_directory.path()); + const octree::Id root = octree::Id::root(); + REQUIRE(left.save(root, triangle()).has_value()); + REQUIRE(left.save(root.child(0).value(), triangle()).has_value()); + + const auto merged = merge_datasets(left, right, output); + REQUIRE_FALSE(merged.has_value()); + REQUIRE(std::holds_alternative(merged.error())); + CHECK(std::get(merged.error()).key == root); + CHECK_FALSE(std::filesystem::exists(output_directory.path() / "octree.storeindex")); +} + +TEST_CASE( + "SF merge retains an invalid output index for diagnosis", + "[sfmerger][sf][validation]") { + TemporaryDirectory left_directory("valid-left"); + TemporaryDirectory right_directory("valid-right"); + TemporaryDirectory output_directory("bad-final-output"); + auto left = indexed_storage(left_directory.path()); + auto right = indexed_storage(right_directory.path()); + auto output = unindexed_storage(output_directory.path()); + const octree::Id root = octree::Id::root(); + REQUIRE(output.save(root, triangle()).has_value()); + REQUIRE(output.save(root.child(0).value(), triangle()).has_value()); + + const auto merged = merge_datasets(left, right, output); + REQUIRE_FALSE(merged.has_value()); + REQUIRE(std::holds_alternative(merged.error())); + CHECK(std::get(merged.error()).key == root); + CHECK(std::filesystem::is_regular_file(output_directory.path() / "octree.storeindex")); +} + +TEST_CASE("SF merge hard-links unchanged nodes and writes changed nodes", "[sfmerger][sf][copy]") { + const octree::Id root = octree::Id::root(); + + SECTION("unchanged") { + TemporaryDirectory left_directory("unchanged-left"); + TemporaryDirectory right_directory("unchanged-right"); + TemporaryDirectory output_directory("unchanged-output"); + auto left = indexed_storage(left_directory.path()); + auto right = indexed_storage(right_directory.path()); + auto output = unindexed_storage(output_directory.path()); + REQUIRE(left.save(root, triangle()).has_value()); + + REQUIRE(merge_datasets(left, right, output).has_value()); + REQUIRE(output.has(root).value()); + CHECK(std::filesystem::equivalent( + left.paths(root)->front(), + output.paths(root)->front())); + } + + SECTION("changed") { + TemporaryDirectory left_directory("changed-left"); + TemporaryDirectory right_directory("changed-right"); + TemporaryDirectory output_directory("changed-output"); + auto left = indexed_storage(left_directory.path()); + auto right = indexed_storage(right_directory.path()); + auto output = unindexed_storage(output_directory.path()); + REQUIRE(left.save(root, triangle()).has_value()); + REQUIRE(right.save(root, triangle(10.0)).has_value()); + + REQUIRE(merge_datasets(left, right, output).has_value()); + REQUIRE(output.has(root).value()); + CHECK_FALSE(std::filesystem::equivalent( + left.paths(root)->front(), + output.paths(root)->front())); + CHECK_FALSE(std::filesystem::equivalent( + right.paths(root)->front(), + output.paths(root)->front())); + REQUIRE(output.load(root).has_value()); + CHECK(output.load(root)->face_count() == 2); + } +} + +TEST_CASE("SF cut hard-links unchanged leaves and writes clipped leaves", "[sfmerger][sf][copy]") { + const octree::Id root = octree::Id::root(); + + SECTION("unchanged") { + TemporaryDirectory input_directory("cut-unchanged-input"); + TemporaryDirectory output_directory("cut-unchanged-output"); + auto input = indexed_storage(input_directory.path()); + auto output = unindexed_storage(output_directory.path()); + REQUIRE(input.save(root, triangle()).has_value()); + + REQUIRE(cut_dataset(input, MeshMask{}, output, false).has_value()); + REQUIRE(output.has(root).value()); + CHECK(std::filesystem::equivalent( + input.paths(root)->front(), + output.paths(root)->front())); + } + + SECTION("clipped") { + TemporaryDirectory input_directory("cut-clipped-input"); + TemporaryDirectory output_directory("cut-clipped-output"); + auto input = indexed_storage(input_directory.path()); + auto output = unindexed_storage(output_directory.path()); + REQUIRE(input.save(root, triangle()).has_value()); + + REQUIRE(cut_dataset(input, MeshMask{.components = {clipping_box()}}, output, true).has_value()); + REQUIRE(output.has(root).value()); + CHECK_FALSE(std::filesystem::equivalent( + input.paths(root)->front(), + output.paths(root)->front())); + const auto clipped = output.load(root); + REQUIRE(clipped.has_value()); + CHECK_FALSE(clipped->is_empty()); + } +} diff --git a/unittests/terrainlib/envelope.cpp b/unittests/terrainlib/envelope.cpp new file mode 100644 index 00000000..ab461b7a --- /dev/null +++ b/unittests/terrainlib/envelope.cpp @@ -0,0 +1,651 @@ +#include "../catch2_helpers.h" + +#include "io/envelope.h" + +#include +#include + +#include +#include +#include +#include + +namespace { + +namespace v1 { + +struct Payload { + std::uint32_t id; + std::string name; + + bool operator==(const Payload &) const = default; +}; + +} // namespace v1 + +namespace v2 { + +struct Payload { + std::uint64_t id; + std::string name; + bool enabled; + + static Payload from_previous(v1::Payload previous) + { + return { + .id = previous.id, + .name = std::move(previous.name), + .enabled = true, + }; + } + + bool operator==(const Payload &) const = default; +}; + +} // namespace v2 + +namespace v3 { + +struct Payload { + std::uint64_t id; + std::string label; + bool enabled; + std::vector samples; + + static Payload from_previous(v2::Payload previous) + { + return { + .id = previous.id, + .label = std::move(previous.name), + .enabled = previous.enabled, + .samples = {}, + }; + } + + bool operator==(const Payload &) const = default; +}; + +} // namespace v3 + +using Schema = io::envelope::PayloadSchema< + "test.Payload", + io::envelope::Version<1, v1::Payload>, + io::envelope::Version<2, v2::Payload>, + io::envelope::Version<3, v3::Payload>>; + +constexpr std::array zstd_compression_algorithms{ + io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, + io::envelope::CompressionAlgorithm::ZstdDefaultCompressionWithChecksum, +}; + +static_assert(std::is_aggregate_v); +static_assert(std::is_aggregate_v); +static_assert(std::is_aggregate_v); +static_assert(Schema::class_name == "test.Payload"); +static_assert(Schema::latest_version == 3); +static_assert(std::same_as); +static_assert(std::same_as, v1::Payload>); + +template +io::envelope::Bytes encode_value(const Value &value) +{ + io::envelope::Bytes bytes; + zpp::bits::out output(bytes); + output(value).or_throw(); + return bytes; +} + +io::envelope::Bytes encode_envelope(const io::envelope::Envelope &envelope) +{ + return encode_value(envelope); +} + +io::envelope::Envelope decode_envelope(const io::envelope::Bytes &bytes) +{ + io::envelope::Envelope envelope{}; + zpp::bits::in input(bytes); + input(envelope).or_throw(); + return envelope; +} + +io::envelope::Bytes compress_without_content_size(const io::envelope::Bytes &uncompressed_data) +{ + const std::unique_ptr context{ + ZSTD_createCCtx(), &ZSTD_freeCCtx}; + if (!context + || ZSTD_isError(ZSTD_CCtx_setParameter(context.get(), ZSTD_c_contentSizeFlag, 0)) + || ZSTD_isError(ZSTD_CCtx_setParameter(context.get(), ZSTD_c_checksumFlag, 1))) { + throw std::runtime_error{"could not configure zstd test context"}; + } + + io::envelope::Bytes compressed_data(ZSTD_compressBound(uncompressed_data.size())); + const std::size_t compressed_size = ZSTD_compress2( + context.get(), + compressed_data.data(), + compressed_data.size(), + uncompressed_data.data(), + uncompressed_data.size()); + if (ZSTD_isError(compressed_size)) { + throw std::runtime_error{"could not create zstd test data"}; + } + compressed_data.resize(compressed_size); + return compressed_data; +} + +template +void check_error(const Result &result, const io::envelope::ErrorCode expected) +{ + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code == expected); +} + +io::envelope::Bytes bytes_from_string(const std::string_view text) +{ + io::envelope::Bytes bytes; + bytes.reserve(text.size()); + for (const char character : text) { + bytes.push_back(static_cast(static_cast(character))); + } + return bytes; +} + +} // namespace + +TEST_CASE("Envelope round trips the latest payload version") +{ + const v3::Payload expected{ + .id = 42, + .label = "latest", + .enabled = false, + .samples = {1, 2, 3}, + }; + + const auto bytes = io::envelope::serialize(expected); + REQUIRE(bytes.has_value()); + + const auto envelope = decode_envelope(*bytes); + CHECK(envelope.magic == io::envelope::magic); + CHECK(envelope.class_name == Schema::class_name); + CHECK(envelope.class_version == 3); + CHECK(envelope.checksum_algorithm == io::envelope::ChecksumAlgorithm::HandledByCompressionLib); + CHECK(envelope.checksum.empty()); + CHECK(envelope.compression_algorithm + == io::envelope::CompressionAlgorithm::ZstdDefaultCompressionWithChecksum); + CHECK(envelope.uncompressed_size == encode_value(expected).size()); + + const auto result = io::envelope::deserialize(*bytes); + REQUIRE(result.has_value()); + CHECK(*result == expected); +} + +TEST_CASE("Envelope upgrades older payload versions") +{ + SECTION("version 1 is upgraded through every subsequent version") + { + const v1::Payload original{.id = 7, .name = "version one"}; + const auto bytes = io::envelope::serialize(original); + REQUIRE(bytes.has_value()); + + const auto result = io::envelope::deserialize(*bytes); + REQUIRE(result.has_value()); + CHECK(*result == v3::Payload{ + .id = 7, + .label = "version one", + .enabled = true, + .samples = {}, + }); + } + + SECTION("version 2 is upgraded to the latest version") + { + const v2::Payload original{.id = 9, .name = "version two", .enabled = false}; + const auto bytes = io::envelope::serialize(original); + REQUIRE(bytes.has_value()); + + const auto result = io::envelope::deserialize(*bytes); + REQUIRE(result.has_value()); + CHECK(*result == v3::Payload{ + .id = 9, + .label = "version two", + .enabled = false, + .samples = {}, + }); + } +} + +TEST_CASE("Envelope supports uncompressed data without a checksum") +{ + const v3::Payload expected{ + .id = 11, + .label = "plain", + .enabled = true, + .samples = {5, 8}, + }; + const auto bytes = io::envelope::serialize( + expected, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::None); + REQUIRE(bytes.has_value()); + + const auto envelope = decode_envelope(*bytes); + CHECK(envelope.compression_algorithm == io::envelope::CompressionAlgorithm::None); + CHECK(envelope.checksum_algorithm == io::envelope::ChecksumAlgorithm::None); + CHECK(envelope.checksum.empty()); + CHECK(envelope.uncompressed_size == envelope.compressed_data.size()); + + const auto result = io::envelope::deserialize(*bytes); + REQUIRE(result.has_value()); + CHECK(*result == expected); +} + +TEST_CASE("CRC-32C uses its canonical hexadecimal representation") +{ + SECTION("standard test vector") + { + const auto compressed = io::envelope::compress_with_checksum( + bytes_from_string("123456789"), + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::Crc32c); + REQUIRE(compressed.has_value()); + CHECK(compressed->checksum == "e3069283"); + } + + SECTION("empty input") + { + const auto compressed = io::envelope::compress_with_checksum( + {}, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::Crc32c); + REQUIRE(compressed.has_value()); + CHECK(compressed->checksum == "00000000"); + } +} + +TEST_CASE("CRC-32C protects independently compressed data") +{ + const auto original = bytes_from_string("payload protected by CRC-32C"); + + SECTION("without compression") + { + const auto compressed = io::envelope::compress_with_checksum( + original, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::Crc32c); + REQUIRE(compressed.has_value()); + + const auto result = io::envelope::checked_decompress( + compressed->compressed_data, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::Crc32c, + compressed->checksum); + REQUIRE(result.has_value()); + CHECK(*result == original); + + auto corrupted = compressed->compressed_data; + corrupted.front() ^= std::byte{0x01}; + check_error( + io::envelope::checked_decompress( + corrupted, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::Crc32c, + compressed->checksum), + io::envelope::ErrorCode::ChecksumMismatch); + } + + SECTION("with zstd compression") + { + for (const auto compression_algorithm : zstd_compression_algorithms) { + CAPTURE(compression_algorithm); + const auto compressed = io::envelope::compress_with_checksum( + original, + compression_algorithm, + io::envelope::ChecksumAlgorithm::Crc32c); + REQUIRE(compressed.has_value()); + REQUIRE(compressed->checksum.size() == 8); + + const auto result = io::envelope::checked_decompress( + compressed->compressed_data, + compression_algorithm, + io::envelope::ChecksumAlgorithm::Crc32c, + compressed->checksum); + REQUIRE(result.has_value()); + CHECK(*result == original); + + auto wrong_checksum = compressed->checksum; + wrong_checksum.front() = wrong_checksum.front() == '0' ? '1' : '0'; + check_error( + io::envelope::checked_decompress( + compressed->compressed_data, + compression_algorithm, + io::envelope::ChecksumAlgorithm::Crc32c, + wrong_checksum), + io::envelope::ErrorCode::ChecksumMismatch); + } + } + + SECTION("malformed checksum") + { + check_error( + io::envelope::checked_decompress( + original, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::Crc32c, + "E3069283"), + io::envelope::ErrorCode::ChecksumMismatch); + check_error( + io::envelope::checked_decompress( + original, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::Crc32c, + {}), + io::envelope::ErrorCode::ChecksumMismatch); + } +} + +TEST_CASE("Envelope round trips and upgrades payloads protected by CRC-32C") +{ + SECTION("latest version") + { + const v3::Payload expected{ + .id = 12, + .label = "crc", + .enabled = true, + .samples = {3, 5, 8}, + }; + const auto bytes = io::envelope::serialize( + expected, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::Crc32c); + REQUIRE(bytes.has_value()); + + const auto envelope = decode_envelope(*bytes); + CHECK(envelope.checksum_algorithm == io::envelope::ChecksumAlgorithm::Crc32c); + CHECK(envelope.checksum.size() == 8); + CHECK(envelope.compression_algorithm == io::envelope::CompressionAlgorithm::None); + + const auto result = io::envelope::deserialize(*bytes); + REQUIRE(result.has_value()); + CHECK(*result == expected); + + auto corrupted = envelope; + corrupted.compressed_data.front() ^= std::byte{0x01}; + check_error(io::envelope::deserialize(encode_envelope(corrupted)), + io::envelope::ErrorCode::ChecksumMismatch); + } + + SECTION("older version") + { + const v1::Payload original{.id = 13, .name = "crc version one"}; + const auto bytes = io::envelope::serialize( + original, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::Crc32c); + REQUIRE(bytes.has_value()); + + const auto result = io::envelope::deserialize(*bytes); + REQUIRE(result.has_value()); + CHECK(*result == v3::Payload{ + .id = 13, + .label = "crc version one", + .enabled = true, + .samples = {}, + }); + } +} + +TEST_CASE("Checked compression round trips and validates its checksum") +{ + const io::envelope::Bytes original(4096, std::byte{0x2a}); + for (const auto compression_algorithm : zstd_compression_algorithms) { + CAPTURE(compression_algorithm); + const auto compressed = io::envelope::compress_with_checksum( + original, + compression_algorithm, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib); + REQUIRE(compressed.has_value()); + CHECK(compressed->compressed_data.size() < original.size()); + CHECK(compressed->checksum.empty()); + + const auto result = io::envelope::checked_decompress( + compressed->compressed_data, + compression_algorithm, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib, + compressed->checksum); + REQUIRE(result.has_value()); + CHECK(*result == original); + + const auto empty_compressed = io::envelope::compress_with_checksum( + {}, + compression_algorithm, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib); + REQUIRE(empty_compressed.has_value()); + const auto empty_result = io::envelope::checked_decompress( + empty_compressed->compressed_data, + compression_algorithm, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib, + empty_compressed->checksum); + REQUIRE(empty_result.has_value()); + CHECK(empty_result->empty()); + + auto corrupted = compressed->compressed_data; + corrupted.back() ^= std::byte{0x01}; + check_error( + io::envelope::checked_decompress( + corrupted, + compression_algorithm, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib, + {}), + io::envelope::ErrorCode::ChecksumMismatch); + + check_error( + io::envelope::checked_decompress( + compressed->compressed_data, + compression_algorithm, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib, + {}, + original.size() - 1), + io::envelope::ErrorCode::SizeLimitExceeded); + } +} + +TEST_CASE("Checked decompression uses its maximum when the format omits the content size") +{ + const io::envelope::Bytes original(4096, std::byte{0x37}); + const auto compressed_data = compress_without_content_size(original); + + for (const auto compression_algorithm : zstd_compression_algorithms) { + CAPTURE(compression_algorithm); + const auto result = io::envelope::checked_decompress( + compressed_data, + compression_algorithm, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib, + {}, + original.size()); + REQUIRE(result.has_value()); + CHECK(*result == original); + + check_error( + io::envelope::checked_decompress( + compressed_data, + compression_algorithm, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib, + {}, + original.size() - 1), + io::envelope::ErrorCode::SizeLimitExceeded); + } +} + +TEST_CASE("Envelope rejects incompatible metadata") +{ + const v3::Payload payload{.id = 1, .label = "metadata", .enabled = true, .samples = {}}; + const auto serialized = io::envelope::serialize(payload); + REQUIRE(serialized.has_value()); + + SECTION("magic") + { + auto envelope = decode_envelope(*serialized); + ++envelope.magic; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::InvalidMagic); + } + + SECTION("class name") + { + auto envelope = decode_envelope(*serialized); + envelope.class_name = "other.Payload"; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::WrongClassName); + } + + SECTION("class version") + { + auto envelope = decode_envelope(*serialized); + envelope.class_version = 99; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::UnsupportedClassVersion); + } + + SECTION("checksum algorithm") + { + auto envelope = decode_envelope(*serialized); + envelope.checksum_algorithm = static_cast(99); + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::UnsupportedChecksumAlgorithm); + } + + SECTION("compression algorithm") + { + auto envelope = decode_envelope(*serialized); + envelope.compression_algorithm = static_cast(99); + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::UnsupportedCompressionAlgorithm); + } + + SECTION("algorithm combination") + { + auto envelope = decode_envelope(*serialized); + envelope.checksum_algorithm = io::envelope::ChecksumAlgorithm::None; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::InvalidAlgorithmCombination); + } + + SECTION("external checksum") + { + auto envelope = decode_envelope(*serialized); + envelope.checksum = "not used by zstd"; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::InvalidAlgorithmCombination); + } + + SECTION("malformed CRC-32C checksum") + { + auto envelope = decode_envelope(*serialized); + envelope.checksum_algorithm = io::envelope::ChecksumAlgorithm::Crc32c; + envelope.checksum = "1234"; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::ChecksumMismatch); + } + + SECTION("incorrect CRC-32C checksum") + { + auto envelope = decode_envelope(*serialized); + envelope.checksum_algorithm = io::envelope::ChecksumAlgorithm::Crc32c; + envelope.checksum = "00000000"; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::ChecksumMismatch); + } + + SECTION("uncompressed size is smaller than the payload") + { + auto envelope = decode_envelope(*serialized); + --envelope.uncompressed_size; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::DecompressionFailed); + } + + SECTION("uncompressed size is larger than the payload") + { + auto envelope = decode_envelope(*serialized); + ++envelope.uncompressed_size; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::DecompressionFailed); + } + + SECTION("zero uncompressed size does not mean unspecified") + { + auto envelope = decode_envelope(*serialized); + envelope.uncompressed_size = 0; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::DecompressionFailed); + } + + SECTION("uncompressed size exceeds the hard limit") + { + auto envelope = decode_envelope(*serialized); + envelope.uncompressed_size = io::envelope::default_max_decompressed_size + 1; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::SizeLimitExceeded); + } + + SECTION("uncompressed size exceeds a caller limit") + { + auto envelope = decode_envelope(*serialized); + check_error( + io::envelope::deserialize( + encode_envelope(envelope), + static_cast(envelope.uncompressed_size - 1)), + io::envelope::ErrorCode::SizeLimitExceeded); + } +} + +TEST_CASE("Envelope validates the size of uncompressed data") +{ + const v3::Payload payload{.id = 2, .label = "plain", .enabled = true, .samples = {1}}; + const auto serialized = io::envelope::serialize( + payload, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::None); + REQUIRE(serialized.has_value()); + + SECTION("declared size is smaller") + { + auto envelope = decode_envelope(*serialized); + --envelope.uncompressed_size; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::DecompressionFailed); + } + + SECTION("declared size is larger") + { + auto envelope = decode_envelope(*serialized); + ++envelope.uncompressed_size; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::DecompressionFailed); + } +} + +TEST_CASE("Envelope reports malformed serialized data") +{ + SECTION("envelope") + { + const io::envelope::Bytes malformed{std::byte{0x01}, std::byte{0x02}}; + check_error(io::envelope::deserialize(malformed), + io::envelope::ErrorCode::DeserializationFailed); + } + + SECTION("payload") + { + const io::envelope::Envelope envelope{ + .magic = io::envelope::magic, + .class_name = std::string{Schema::class_name}, + .class_version = 3, + .checksum_algorithm = io::envelope::ChecksumAlgorithm::None, + .checksum = {}, + .compression_algorithm = io::envelope::CompressionAlgorithm::None, + .uncompressed_size = 1, + .compressed_data = {std::byte{0x01}}, + }; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::DeserializationFailed); + } +} diff --git a/unittests/terrainlib/index_map.cpp b/unittests/terrainlib/index_map.cpp deleted file mode 100644 index 5d595c23..00000000 --- a/unittests/terrainlib/index_map.cpp +++ /dev/null @@ -1,100 +0,0 @@ -#include "../catch2_helpers.h" - -#include "octree/IndexMap.h" - -using namespace octree; - -TEST_CASE("IndexMap basic operations") { - IndexMap index; - - SECTION("Add root node") { - Id root = Id::root(); - index.add(root); - CHECK(index.is_present(root)); - CHECK(index.is(NodeStatus::Leaf, root)); - CHECK(index.get(root) == std::optional(NodeStatus::Leaf)); - } - - SECTION("Add child node creates virtual parent") { - Id root = Id::root(); - Id child = root.child(3).value(); - index.add(child); - CHECK(index.is_present(root)); - CHECK(index.is(NodeStatus::Virtual, root)); - CHECK(index.is_present(child)); - CHECK(index.is(NodeStatus::Leaf, child)); - } - - SECTION("Add child note promotes parent to inner") { - Id root = Id::root(); - Id child1 = root.child(0).value(); - index.add(child1); - index.add(root); - CHECK(index.is(NodeStatus::Leaf, child1)); - CHECK(index.is(NodeStatus::Inner, root)); - } - - SECTION("Removing a leaf node cleans up correctly") { - Id root = Id::root(); - Id child = root.child(2).value(); - index.add(child); - CHECK(index.is_present(child)); - index.remove(child); - CHECK(index.is_absent(child)); - CHECK(index.is_absent(root)); - } - - SECTION("Removing one of two children of an inner keeps it inner") { - Id root = Id::root(); - Id child1 = root.child(0).value(); - Id child2 = root.child(1).value(); - index.add(root); - index.add(child1); - index.add(child2); - index.remove(child1); - - CHECK(index.is_absent(child1)); - CHECK(index.is(NodeStatus::Leaf, child2)); - CHECK(index.is(NodeStatus::Inner, root)); - } - - SECTION("Removing last child of inner converts it to leaf") { - Id root = Id::root(); - Id child1 = root.child(0).value(); - index.add(child1); - index.add(root); - - index.remove(child1); - - CHECK(index.is_absent(child1)); - CHECK(index.is(NodeStatus::Leaf, root)); - } - - SECTION("Clear works") { - Id a = Id::root().child(1).value(); - Id b = Id::root().child(2).value(); - index.add(a); - index.add(b); - CHECK(index.size() == 3); // root + a + b - index.clear(); - CHECK(index.empty()); - } - - SECTION("Serialization round-trip") { - IndexMap original; - original.add(Id::root().child(3).value()); - - const std::errc success {}; - - std::vector buffer; - zpp::bits::out out(buffer); - CHECK(out(original).code == success); - - IndexMap loaded; - zpp::bits::in in(buffer); - CHECK(in(loaded).code == success); - - CHECK(original.size() == loaded.size()); - CHECK(loaded.is_present(Id::root().child(3).value())); - } -} diff --git a/unittests/terrainlib/index_traverse.cpp b/unittests/terrainlib/index_traverse.cpp deleted file mode 100644 index 5ada7575..00000000 --- a/unittests/terrainlib/index_traverse.cpp +++ /dev/null @@ -1,132 +0,0 @@ -#include -#include -#include -#include - -#include "octree/Id.h" -#include "octree/IndexMap.h" -#include "octree/traverse.h" - -using namespace octree; - -TEST_CASE("octree::traverse basic") { - IndexMap index; - - const Id root = Id::root(); - index.add(root); - - const auto children = root.children().value(); - for (const auto& child : children) { - index.add(child); - } - - SECTION("traversal visits root then children") { - std::vector visited; - traverse( - index, - [&](const Id &id, const NodeStatus &) { - visited.push_back(id); - }, - [](const Id &) { return true; }, // refine always - root, - TraversalOrder::DepthFirst); - - CHECK(visited.front() == root); - CHECK(visited.size() == 9); // root + 8 children - std::vector expected(children.begin(), children.end()); - visited.erase(visited.begin()); - CHECK(visited == expected); - } - - SECTION("Refine function disables recursion") { - std::vector visited; - traverse( - index, - [&](const Id &id, const NodeStatus &) { - visited.push_back(id); - }, - [](const Id &) { return false; }, // don't refine - root, - TraversalOrder::DepthFirst); - - CHECK(visited.size() == 1); - CHECK(visited.front() == root); - } - - SECTION("Traversal skips when root is missing") { - IndexMap empty; - bool visited = false; - - traverse( - empty, - [&](const Id&, const NodeStatus&) { - visited = true; - }, - [](const Id&) { return true; }, - root, - TraversalOrder::DepthFirst - ); - - CHECK_FALSE(visited); - } -} - -TEST_CASE("octree::traverse with depth") { - IndexMap index; - - const Id root = Id::root(); - index.add(root); - - const auto children = root.children().value(); - const Id c0 = children[0]; - const Id c1 = children[1]; - const Id c2 = children[2]; - - index.add(c0); - index.add(c1); - index.add(c2); - - // Add grandchildren - const Id gc0 = c0.child(0).value(); // grandchild of c0 - const Id gc1 = c2.child(1).value(); // grandchild of c2 - - index.add(gc0); - index.add(gc1); - - SECTION("DFS visits deeply before siblings") { - std::vector visited; - traverse( - index, - [&](const Id& id, const NodeStatus&) { - visited.push_back(id); - }, - [](const Id&) { return true; }, - root, - TraversalOrder::DepthFirst - ); - - CHECK(visited.size() == 6); - - // DFS order should be: root, c0, gc0, c1, c2, gc1 - std::vector expected{root, c0, gc0, c1, c2, gc1}; - CHECK(visited == expected); - } - - SECTION("BFS visits all nodes level by level") { - std::vector visited; - traverse( - index, - [&](const Id &id, const NodeStatus &) { - visited.push_back(id); - }, - [](const Id &) { return true; }, - root, - TraversalOrder::BreadthFirst); - - CHECK(visited.size() == 6); - - // BFS order should be: root, c0, c1, c2, gc0, gc1 - std::vector expected{root, c0, c1, c2, gc0, gc1}; - CHECK(visited == expected); - } -} diff --git a/unittests/terrainlib/io_conversion.cpp b/unittests/terrainlib/io_conversion.cpp new file mode 100644 index 00000000..2dd7c7e8 --- /dev/null +++ b/unittests/terrainlib/io_conversion.cpp @@ -0,0 +1,125 @@ +#include "../catch2_helpers.h" + +#include "io/conversion.h" + +#include +#include + +namespace { + +template +void check_round_trip(const int cv_type) +{ + cv::Mat source(2, 3, cv_type); + auto bytes = std::span(source.ptr(), source.total() * source.elemSize()); + for (std::size_t index = 0; index < bytes.size(); ++index) { + bytes[index] = static_cast(index + 1); + } + + const auto raster = io::conversion::to_raster(source); + REQUIRE(raster.has_value()); + CHECK(raster->size() == glm::uvec2(3, 2)); + + const auto result = io::conversion::to_mat(*raster); + REQUIRE(result.has_value()); + CHECK(result->type() == source.type()); + CHECK(result->rows == source.rows); + CHECK(result->cols == source.cols); + CHECK(std::equal(bytes.begin(), bytes.end(), result->template ptr())); +} + +} // namespace + +TEST_CASE("io::conversion round trips supported scalar and vector pixels") +{ + check_round_trip(CV_8UC1); + check_round_trip(CV_8SC1); + check_round_trip(CV_16UC1); + check_round_trip(CV_16SC1); + check_round_trip(CV_32SC1); + check_round_trip(CV_32FC1); + check_round_trip(CV_64FC1); + check_round_trip(CV_8UC2); + check_round_trip(CV_8UC3); + check_round_trip(CV_8UC4); + check_round_trip(CV_32FC3); + check_round_trip(CV_64FC4); +} + +TEST_CASE("io::conversion copies non-contiguous matrices row by row") +{ + cv::Mat backing(4, 6, CV_8UC3); + for (int row = 0; row < backing.rows; ++row) { + for (int column = 0; column < backing.cols; ++column) { + backing.at(row, column) = cv::Vec3b( + static_cast(row), + static_cast(column), + static_cast(row * backing.cols + column)); + } + } + const cv::Mat source = backing(cv::Rect(1, 1, 3, 2)); + REQUIRE_FALSE(source.isContinuous()); + + auto raster = io::conversion::to_raster(source); + REQUIRE(raster.has_value()); + CHECK(raster->pixel({0, 0}) == glm::u8vec3(1, 1, 7)); + CHECK(raster->pixel({2, 1}) == glm::u8vec3(2, 3, 15)); + + backing.setTo(cv::Scalar::all(0)); + CHECK(raster->pixel({0, 0}) == glm::u8vec3(1, 1, 7)); + + const auto round_trip = io::conversion::to_mat(*raster); + REQUIRE(round_trip.has_value()); + raster->fill(glm::u8vec3(0)); + CHECK(round_trip->at(0, 0) == cv::Vec3b(1, 1, 7)); +} + +TEST_CASE("io::conversion reports invalid inputs") +{ + SECTION("empty matrix") + { + const auto result = io::conversion::to_raster(cv::Mat{}); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code == io::conversion::ErrorCode::EmptyInput); + } + + SECTION("multidimensional matrix") + { + const std::array sizes{2, 2, 2}; + const cv::Mat source(static_cast(sizes.size()), sizes.data(), CV_8UC1); + const auto result = io::conversion::to_raster(source); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code == io::conversion::ErrorCode::UnsupportedDimensions); + } + + SECTION("pixel type mismatch") + { + const cv::Mat source(2, 2, CV_8UC3); + const auto result = io::conversion::to_raster(source); + REQUIRE_FALSE(result.has_value()); + const io::conversion::Error expected{ + io::conversion::ErrorCode::PixelTypeMismatch, + CV_8UC1, + CV_8UC3, + }; + CHECK(result.error() == expected); + } + + SECTION("unsupported pixel") + { + struct Pixel { + std::uint64_t value; + }; + const cv::Mat source(2, 2, CV_8UC1); + const auto result = io::conversion::to_raster(source); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code == io::conversion::ErrorCode::UnsupportedPixelType); + } + + SECTION("empty raster") + { + const auto result = io::conversion::to_mat(radix::Raster{}); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code == io::conversion::ErrorCode::EmptyInput); + } +} diff --git a/unittests/terrainlib/mesh_clip.cpp b/unittests/terrainlib/mesh_clip.cpp index f3b2d369..a5646c0e 100644 --- a/unittests/terrainlib/mesh_clip.cpp +++ b/unittests/terrainlib/mesh_clip.cpp @@ -354,37 +354,3 @@ TEST_CASE("clip_on_bounds produces manifold mesh") { CHECK(mesh::is_manifold(clipped)); CHECK(clipped.positions.size() == 6); } - -#ifdef NDEBUG -TEST_CASE("mesh::clip_on_bounds benchmark") { - BENCHMARK_ADVANCED("clip based on octree")(Catch::Benchmark::Chronometer meter) { - const std::filesystem::path mesh_path = ALP_TEST_DATA_DIR "/meshes/6857.terrain"; - auto mesh_result = mesh::io::load_from_path(mesh_path); - REQUIRE(mesh_result.has_value()); - SimpleMesh &mesh = mesh_result.value(); - mesh.uvs.clear(); - mesh.texture = std::nullopt; - - const octree::Space space = octree::Space::earth(); - const auto mesh_bounds = calculate_bounds(mesh); - const octree::Id id = space.find_smallest_node_encompassing_bounds(mesh_bounds).value(); - const auto child_ids = id.children().value(); - std::vector child_bounds; - child_bounds.reserve(child_ids.size()); - for (const auto &child_id : child_ids) { - const auto bounds = space.get_node_bounds(child_id); - child_bounds.push_back(bounds); - } - - meter.measure([mesh, child_bounds] { - std::vector child_meshes; - child_meshes.reserve(child_bounds.size()); - for (const auto &bounds : child_bounds) { - const auto clipped_mesh = mesh::clip_on_bounds(mesh, bounds); - child_meshes.push_back(clipped_mesh); - } - return child_meshes; - }); - }; -} -#endif diff --git a/unittests/terrainlib/mesh_io.cpp b/unittests/terrainlib/mesh_io.cpp index bac6225b..3100bbd3 100644 --- a/unittests/terrainlib/mesh_io.cpp +++ b/unittests/terrainlib/mesh_io.cpp @@ -44,14 +44,14 @@ TEST_CASE("transcode roundtrip") { mesh.texture = cv::Mat3b(100, 100); cv::randu(*mesh.texture, cv::Scalar(0, 0, 0), cv::Scalar(256, 256, 256)); - const tl::expected encode_result = + const std::expected encode_result = mesh::encode(mesh, mesh::EncodeOptions{.texture_format = ".png"}); if (!encode_result.has_value()) { FAIL(encode_result.error()); } const mesh::Encoded encoded = encode_result.value(); - const tl::expected decode_result = + const std::expected decode_result = mesh::decode(encoded, mesh::DecodeOptions{}); if (!decode_result.has_value()) { FAIL(decode_result.error()); @@ -90,10 +90,10 @@ TEST_CASE("io roundtrip") { std::filesystem::remove(mesh_path); CHECK(!std::filesystem::exists(mesh_path)); - mesh::io::save_to_path(mesh, mesh_path, mesh::io::SaveOptions{.texture_format = ".png"}); + REQUIRE(mesh::io::save_to_path(mesh, mesh_path, mesh::io::SaveOptions{.texture_format = ".png"}).has_value()); CHECK(std::filesystem::exists(mesh_path)); - const tl::expected result = mesh::io::load_from_path(mesh_path); + const std::expected result = mesh::io::load_from_path(mesh_path); if (!result.has_value()) { FAIL(result.error().description()); } @@ -109,7 +109,7 @@ TEST_CASE("io roundtrip") { } TEST_CASE("io roundtrip high precision") { - for (const auto& format : {"terrain"}) { + for (const auto& format : {"sfmesh"}) { DYNAMIC_SECTION(format) { SimpleMesh mesh; @@ -133,10 +133,10 @@ TEST_CASE("io roundtrip high precision") { std::filesystem::remove(mesh_path); CHECK(!std::filesystem::exists(mesh_path)); - mesh::io::save_to_path(mesh, mesh_path, mesh::io::SaveOptions{.texture_format = ".png"}); + REQUIRE(mesh::io::save_to_path(mesh, mesh_path, mesh::io::SaveOptions{.texture_format = ".png"}).has_value()); CHECK(std::filesystem::exists(mesh_path)); - const tl::expected result = mesh::io::load_from_path(mesh_path); + const std::expected result = mesh::io::load_from_path(mesh_path); if (!result.has_value()) { FAIL(result.error().description()); } @@ -152,7 +152,7 @@ TEST_CASE("io roundtrip high precision") { } TEST_CASE("io roundtrip no texture") { - for (const auto &format : {"gltf", "glb", "terrain"}) { + for (const auto &format : {"gltf", "glb", "sfmesh"}) { DYNAMIC_SECTION(format) { SimpleMesh mesh; @@ -176,10 +176,10 @@ TEST_CASE("io roundtrip no texture") { std::filesystem::remove(mesh_path); CHECK(!std::filesystem::exists(mesh_path)); - mesh::io::save_to_path(mesh, mesh_path); + REQUIRE(mesh::io::save_to_path(mesh, mesh_path).has_value()); CHECK(std::filesystem::exists(mesh_path)); - const tl::expected result = mesh::io::load_from_path(mesh_path); + const std::expected result = mesh::io::load_from_path(mesh_path); if (!result.has_value()) { FAIL(result.error().description()); } @@ -194,7 +194,7 @@ TEST_CASE("io roundtrip no texture") { } TEST_CASE("io roundtrip no texture and uvs") { - for (const auto &format : {"gltf", "glb", "terrain"}) { + for (const auto &format : {"gltf", "glb", "sfmesh"}) { DYNAMIC_SECTION(format) { SimpleMesh mesh; @@ -213,10 +213,10 @@ TEST_CASE("io roundtrip no texture and uvs") { std::filesystem::remove(mesh_path); CHECK(!std::filesystem::exists(mesh_path)); - mesh::io::save_to_path(mesh, mesh_path); + REQUIRE(mesh::io::save_to_path(mesh, mesh_path).has_value()); CHECK(std::filesystem::exists(mesh_path)); - const tl::expected result = mesh::io::load_from_path(mesh_path); + const std::expected result = mesh::io::load_from_path(mesh_path); if (!result.has_value()) { FAIL(result.error().description()); } diff --git a/unittests/terrainlib/progress_indicator_test.cpp b/unittests/terrainlib/progress_indicator_test.cpp index b994ddb7..826c6e8d 100644 --- a/unittests/terrainlib/progress_indicator_test.cpp +++ b/unittests/terrainlib/progress_indicator_test.cpp @@ -19,7 +19,10 @@ #include "ProgressIndicator.h" #include +#include +#include #include +#include #include #include @@ -100,4 +103,28 @@ TEST_CASE("progress indicator") monitoring_thread.join(); CHECK_THROWS(pi.task_finished()); } + + SECTION("monitoring can be stopped before all tasks finish") + { + ProgressIndicator pi(1); + auto monitoring_thread = pi.start_monitoring(); + + std::condition_variable_any fallback_condition; + std::mutex fallback_mutex; + std::jthread fallback([&](std::stop_token stop_token) { + std::unique_lock lock(fallback_mutex); + fallback_condition.wait_for(lock, stop_token, 2s, []() { return false; }); + if (!stop_token.stop_requested()) { + pi.task_finished(); + } + }); + + const auto before_stop = std::chrono::steady_clock::now(); + monitoring_thread.request_stop(); + monitoring_thread.join(); + const auto stop_duration = std::chrono::steady_clock::now() - before_stop; + fallback.request_stop(); + + CHECK(stop_duration < 250ms); + } } diff --git a/unittests/terrainlib/sf_validate_index.cpp b/unittests/terrainlib/sf_validate_index.cpp new file mode 100644 index 00000000..b8f7e2ec --- /dev/null +++ b/unittests/terrainlib/sf_validate_index.cpp @@ -0,0 +1,24 @@ +#include + +#include "octree/StoreTraits.h" +#include "sf/validate_index.h" + +TEST_CASE("Structura Fundamentalis validation accepts Leaf and Virtual nodes", "[sf][store]") { + store::Index index; + const octree::Id leaf = octree::Id::root().child(2).value().child(5).value(); + REQUIRE(index.add(leaf).has_value()); + + CHECK(sf::validate_index(index).has_value()); +} + +TEST_CASE("Structura Fundamentalis validation reports the first Inner node", "[sf][store]") { + store::Index index; + const octree::Id root = octree::Id::root(); + const octree::Id child = root.child(3).value(); + REQUIRE(index.add(root).has_value()); + REQUIRE(index.add(child).has_value()); + + const auto result = sf::validate_index(index); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().key == root); +} diff --git a/unittests/terrainlib/store_codec.cpp b/unittests/terrainlib/store_codec.cpp new file mode 100644 index 00000000..47dfdb46 --- /dev/null +++ b/unittests/terrainlib/store_codec.cpp @@ -0,0 +1,160 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include "io/bytes.h" +#include "mesh/codec/Gltf.h" +#include "mesh/codec/SfMesh.h" +#include "store/Codec.h" + +namespace { + +class TemporaryDirectory { +public: + explicit TemporaryDirectory(const std::string_view label) { + static std::atomic_uint64_t counter = 0; + const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); + _path = std::filesystem::temp_directory_path() + / ("atb-store-codec-" + std::string(label) + "-" + + std::to_string(timestamp) + "-" + std::to_string(counter++)); + REQUIRE(std::filesystem::create_directories(_path)); + } + ~TemporaryDirectory() { + std::error_code error; + std::filesystem::remove_all(_path, error); + } + const std::filesystem::path &path() const { + return _path; + } + +private: + std::filesystem::path _path; +}; + +class MultiFileCodec final : public store::Codec { +public: + std::vector paths(const store::NodePath &node_path) const override { + std::filesystem::path data = node_path.path(); + std::filesystem::path metadata = node_path.path(); + data += ".data"; + metadata += ".metadata"; + return {data, metadata}; + } +}; + +class WriteOnlyCodec final : public store::Codec { +public: + std::vector paths(const store::NodePath &node_path) const override { + std::filesystem::path path = node_path.path(); + path += ".out"; + return {path}; + } + std::expected write( + const store::NodePath &, + const int &) const override { + ++writes; + return {}; + } + + mutable std::atomic_uint32_t writes = 0; +}; + +mesh::Simple triangle_mesh(const double offset) { + return mesh::Simple( + {{0, 1, 2}}, + {{offset, 0.0, 0.0}, {offset + 1.0, 0.0, 0.0}, {offset, 1.0, 0.0}}); +} + +template +void check_mesh_codec_reentrancy(const Codec &codec, const std::filesystem::path &base) { + std::vector> writes; + for (int index = 0; index < 4; ++index) { + writes.push_back(std::async(std::launch::async, [&codec, base, index] { + return codec.write( + store::NodePath(base / std::to_string(index) / "node"), + triangle_mesh(index)) + .has_value(); + })); + } + for (auto &write : writes) { + REQUIRE(write.get()); + } + + std::vector> reads; + for (int index = 0; index < 4; ++index) { + reads.push_back(std::async(std::launch::async, [&codec, base, index] { + const auto result = codec.read( + store::NodePath(base / std::to_string(index) / "node")); + return result.has_value() && result->face_count() == 1; + })); + } + for (auto &read : reads) { + REQUIRE(read.get()); + } +} + +} // namespace + +TEST_CASE("runtime codec defaults report unsupported operations", "[store][codec]") { + MultiFileCodec codec; + const store::NodePath path("12/34/56/78"); + CHECK(codec.paths(path) + == std::vector{ + "12/34/56/78.data", + "12/34/56/78.metadata", + }); + + const auto read = codec.read(path); + REQUIRE_FALSE(read.has_value()); + CHECK(read.error().operation == store::CodecOperation::Read); + CHECK(read.error().category == store::CodecErrorCategory::UnsupportedOperation); + + const auto write = codec.write(path, 42); + REQUIRE_FALSE(write.has_value()); + CHECK(write.error().operation == store::CodecOperation::Write); + CHECK(write.error().category == store::CodecErrorCategory::UnsupportedOperation); +} + +TEST_CASE("runtime write-only codec remains explicitly unreadable", "[store][codec]") { + WriteOnlyCodec codec; + REQUIRE(codec.write(store::NodePath("node"), 42).has_value()); + CHECK(codec.writes == 1); + const auto read = codec.read(store::NodePath("node")); + REQUIRE_FALSE(read.has_value()); + CHECK(read.error().category == store::CodecErrorCategory::UnsupportedOperation); +} + +TEST_CASE("runtime mesh codecs are reentrant", "[store][codec]") { + TemporaryDirectory directory("mesh-reentrant"); + + SECTION("SF mesh") { + check_mesh_codec_reentrancy(mesh::codec::SfMesh{}, directory.path() / "sfmesh"); + } + SECTION("binary glTF") { + check_mesh_codec_reentrancy( + mesh::codec::Gltf(mesh::codec::GltfContainer::Binary), + directory.path() / "glb"); + } + SECTION("JSON glTF") { + check_mesh_codec_reentrancy( + mesh::codec::Gltf(mesh::codec::GltfContainer::Json), + directory.path() / "gltf"); + } +} + +TEST_CASE("runtime glTF codec converts writer exceptions", "[store][codec]") { + TemporaryDirectory directory("gltf-error"); + mesh::codec::Gltf codec(mesh::codec::GltfContainer::Binary); + const store::NodePath node_path(directory.path() / "blocked"); + REQUIRE(std::filesystem::create_directory(codec.paths(node_path).front())); + + const auto result = codec.write(node_path, triangle_mesh(0.0)); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().operation == store::CodecOperation::Write); + CHECK(result.error().category == store::CodecErrorCategory::Domain); +} diff --git a/unittests/terrainlib/store_compatibility.cpp b/unittests/terrainlib/store_compatibility.cpp new file mode 100644 index 00000000..9b5c8fad --- /dev/null +++ b/unittests/terrainlib/store_compatibility.cpp @@ -0,0 +1,283 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "io/bytes.h" +#include "mesh/SimpleMesh.h" +#include "mesh/codec/SfMeshFormat.h" +#include "mesh/codec/from_extension.h" +#include "mesh/storage.h" +#include "octree/store_layout/Mappings.h" +#include "octree/storage/IndexFile.h" +#include "octree/storage/open.h" + +namespace { + +class TemporaryDirectory { +public: + explicit TemporaryDirectory(const std::string_view label) { + static std::atomic_uint64_t counter = 0; + const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); + _path = std::filesystem::temp_directory_path() + / ("atb-raster-store-" + std::string(label) + "-" + + std::to_string(timestamp) + "-" + std::to_string(counter++)); + REQUIRE(std::filesystem::create_directories(_path)); + } + + ~TemporaryDirectory() { + std::error_code error; + std::filesystem::remove_all(_path, error); + } + + const std::filesystem::path &path() const { + return _path; + } + +private: + std::filesystem::path _path; +}; + +mesh::Simple triangle_mesh(const double offset) { + return mesh::Simple( + {{0, 1, 2}}, + {{offset, 0.0, 0.0}, {offset + 1.0, 0.0, 0.0}, {offset, 1.0, 0.0}}); +} + +mesh::storage::IndexedStorage make_storage( + const std::filesystem::path &path, + const store::PathMapping mapping, + const std::string &extension) { + auto codec = mesh::codec::from_extension(extension); + REQUIRE(codec.has_value()); + return mesh::storage::IndexedStorage( + store::RawStorage( + store::Layout(path, mapping), + std::move(codec.value())), + store::Index{}, + mesh::storage::Storage::Persistence{ + octree::storage::index_format(), + path / "octree.storeindex", + std::string(mapping.id), + std::string(mesh::storage::payload_class), + extension, + }); +} + +} // namespace + +TEST_CASE("octree layouts round-trip boundary IDs") { + const std::vector ids{ + octree::Id::root(), + octree::Id(octree::Id::max_level(), octree::Id::Index{0}), + octree::Id(octree::Id::max_level(), octree::Id::max_index_on_level(octree::Id::max_level())), + }; + + const auto flat = octree::store_layout::flat(); + const auto coordinates = octree::store_layout::level_and_coordinate_directories(); + for (const octree::Id id : ids) { + const auto flat_path = flat.key_to_node_path(id); + CHECK(flat.node_path_to_key(flat_path) == id); + + const auto coordinate_path = coordinates.key_to_node_path(id); + CHECK(coordinates.node_path_to_key(coordinate_path) == id); + } +} + +TEST_CASE("versioned octree and SF payload validation is free-standing") { + const octree::storage::StoreIndex invalid_index{{{ + .level = std::numeric_limits::max(), + .index = 0, + .status = static_cast(store::NodeStatus::Leaf), + }}}; + CHECK_FALSE(octree::storage::decode_index(invalid_index).has_value()); + + const mesh::sf::Payload oversized_mesh{ + .vertex_count = std::numeric_limits::max(), + .face_count = 0, + .triangles = {}, + .positions = {0}, + .uvs = {}, + .texture = {}, + }; + CHECK_FALSE(mesh::sf::validate(oversized_mesh).has_value()); +} + +TEST_CASE("storage hard-links matching payload formats") { + TemporaryDirectory source_directory("hard-link-source"); + TemporaryDirectory target_directory("hard-link-target"); + auto source = make_storage( + source_directory.path(), + octree::store_layout::flat(), + ".sfmesh"); + auto target = make_storage( + target_directory.path(), + octree::store_layout::flat(), + ".sfmesh"); + + const octree::Id root = octree::Id::root(); + REQUIRE(source.save(root, triangle_mesh(0.0)).has_value()); + REQUIRE(target.copy_from(root, source).has_value()); + CHECK(std::filesystem::equivalent(source.path_for(root).value(), target.path_for(root).value())); + REQUIRE(source.save_index().has_value()); + REQUIRE(target.save_index().has_value()); +} + +TEST_CASE("storage re-encodes differing payload formats") { + TemporaryDirectory source_directory("reencode-source"); + TemporaryDirectory target_directory("reencode-target"); + auto source = make_storage( + source_directory.path(), + octree::store_layout::flat(), + ".sfmesh"); + auto target = make_storage( + target_directory.path(), + octree::store_layout::flat(), + ".glb"); + + const octree::Id root = octree::Id::root(); + REQUIRE(source.save(root, triangle_mesh(4.0)).has_value()); + REQUIRE(target.copy_from(root, source).has_value()); + CHECK_FALSE(std::filesystem::equivalent( + source.path_for(root).value(), + target.path_for(root).value())); + const auto loaded = target.load(root); + REQUIRE(loaded.has_value()); + CHECK(loaded->face_count() == 1); + REQUIRE(source.save_index().has_value()); + REQUIRE(target.save_index().has_value()); +} + +TEST_CASE("storage supports enabled overwrites") { + TemporaryDirectory directory("overwrite"); + auto storage = make_storage( + directory.path(), + octree::store_layout::flat(), + ".sfmesh"); + storage.settings().allow_overwrite = true; + + const octree::Id root = octree::Id::root(); + REQUIRE(storage.save(root, triangle_mesh(1.0)).has_value()); + REQUIRE(storage.save(root, triangle_mesh(9.0)).has_value()); + const auto loaded = storage.load(root); + REQUIRE(loaded.has_value()); + REQUIRE_FALSE(loaded->positions.empty()); + CHECK(loaded->positions.front().x == 9.0); + REQUIRE(storage.save_index().has_value()); +} + +TEST_CASE("folder opening persists and reopens metadata-driven storage") { + TemporaryDirectory directory("folder-open"); + octree::OpenOptions options; + options.default_mapping = octree::store_layout::flat(); + options.preferred_extension = ".sfmesh"; + + { + auto storage_result = octree::open_folder(directory.path(), std::move(options)); + REQUIRE(storage_result.has_value()); + auto storage = std::move(storage_result.value()); + CHECK(storage.is_indexed()); + REQUIRE(storage.save(octree::Id::root(), triangle_mesh(2.0)).has_value()); + REQUIRE(storage.save_index().has_value()); + CHECK(std::filesystem::is_regular_file(directory.path() / "octree.storeindex")); + CHECK(std::filesystem::is_regular_file(directory.path() / "octree.storemeta")); + } + + { + auto storage_result = octree::open_folder_indexed(directory.path()); + REQUIRE(storage_result.has_value()); + const auto storage = std::move(storage_result.value()); + CHECK(storage.is_indexed()); + CHECK(storage.has(octree::Id::root()).value()); + CHECK(std::filesystem::is_regular_file(directory.path() / "octree.storeindex")); + } + + const auto reopened = octree::open_index(directory.path() / "octree.storeindex"); + REQUIRE(reopened.has_value()); + CHECK(reopened->has(octree::Id::root()).value()); + + const auto metadata = octree::storage::read_store_metadata(directory.path()); + REQUIRE(metadata.has_value()); + CHECK(metadata->layout_id == "flat"); + CHECK(metadata->payload_class == mesh::storage::payload_class); + CHECK(metadata->codec_selector == ".sfmesh"); +} + +TEST_CASE("mesh codec resolver dispatches every supported selector", "[store][open]") { + for (const std::string extension : {".sfmesh", ".glb", ".gltf"}) { + const auto codec = mesh::codec::from_extension(extension); + REQUIRE(codec.has_value()); + const auto paths = codec.value()->paths(store::NodePath("node")); + REQUIRE_FALSE(paths.empty()); + CHECK(paths.front().extension() == extension); + } + + const auto unknown = mesh::codec::from_extension(".unknown"); + REQUIRE_FALSE(unknown.has_value()); + CHECK(unknown.error().category == store::CodecErrorCategory::UnsupportedCodec); + CHECK(unknown.error().operation == store::CodecOperation::Resolve); +} + +TEST_CASE("octree opening rejects a mismatched metadata payload class", "[store][open]") { + TemporaryDirectory directory("payload-class"); + const std::filesystem::path index_path = directory.path() / "octree.storeindex"; + const store::IndexMetadata index_file{ + .index = {}, + .layout_id = "flat", + .payload_class = "dag.ClusterBatch", + .codec_selector = ".sfmesh", + }; + REQUIRE(octree::storage::write_index_file(index_path, index_file).has_value()); + + const auto result = octree::open_index(index_path); + REQUIRE_FALSE(result.has_value()); + REQUIRE(std::holds_alternative(result.error())); + CHECK(std::get(result.error()).category + == store::CodecErrorCategory::UnsupportedCodec); +} + +TEST_CASE("octree opening retains index failures without directory fallback", "[store][open]") { + TemporaryDirectory directory("open-errors"); + const std::filesystem::path index_path = directory.path() / "octree.storeindex"; + + store::IndexMetadata index_file{ + .index = {}, + .layout_id = "unknown-layout", + .payload_class = std::string(mesh::storage::payload_class), + .codec_selector = ".sfmesh", + }; + REQUIRE(octree::storage::write_index_file(index_path, index_file).has_value()); + + const auto unknown_layout = octree::open_folder_indexed(directory.path()); + REQUIRE_FALSE(unknown_layout.has_value()); + REQUIRE(std::holds_alternative(unknown_layout.error())); + CHECK(std::get(unknown_layout.error()).id == "unknown-layout"); + + index_file.layout_id = "flat"; + index_file.codec_selector = ".unknown"; + REQUIRE(octree::storage::write_index_file(index_path, index_file).has_value()); + const auto unknown_codec = octree::open_folder_indexed(directory.path()); + REQUIRE_FALSE(unknown_codec.has_value()); + REQUIRE(std::holds_alternative(unknown_codec.error())); + CHECK(std::get(unknown_codec.error()).category + == store::CodecErrorCategory::UnsupportedCodec); + + const std::array malformed_bytes{0xff, 0x00, 0x01}; + REQUIRE(io::write_bytes_to_path(malformed_bytes, index_path).has_value()); + const auto malformed = octree::open_folder_indexed(directory.path()); + REQUIRE_FALSE(malformed.has_value()); + REQUIRE(std::holds_alternative(malformed.error())); + CHECK(std::get(malformed.error()).category + == store::IndexFormatErrorCategory::Malformed); + + const auto independent_metadata = + octree::storage::read_store_metadata(directory.path()); + REQUIRE(independent_metadata.has_value()); + CHECK(independent_metadata->codec_selector == ".unknown"); +} diff --git a/unittests/terrainlib/store_index.cpp b/unittests/terrainlib/store_index.cpp new file mode 100644 index 00000000..454d7948 --- /dev/null +++ b/unittests/terrainlib/store_index.cpp @@ -0,0 +1,81 @@ +#include +#include + +#include +#include + +#include "octree/StoreTraits.h" +#include "raster_store/StoreTraits.h" +#include "store/Index.h" + +static_assert(store::HierarchyTraits); +static_assert(store::HierarchyTraits); + +TEMPLATE_TEST_CASE( + "shared hierarchy index transitions", + "[store][index]", + octree::StoreTraits, + raster_store::StoreTraits) { + using Traits = TestType; + using Index = store::Index; + + Index index; + const auto root = Traits::root(); + const auto children = Traits::children(root).value(); + const auto first = children[0]; + const auto second = children[1]; + + REQUIRE(index.add(first).value()); + CHECK(index.is(store::NodeStatus::Virtual, root).value()); + CHECK(index.is(store::NodeStatus::Leaf, first).value()); + + REQUIRE(index.add(root).value()); + CHECK(index.is(store::NodeStatus::Inner, root).value()); + CHECK_FALSE(index.add(root).value()); + + REQUIRE(index.add(second).value()); + REQUIRE(index.remove(first).value()); + CHECK(index.is(store::NodeStatus::Inner, root).value()); + REQUIRE(index.remove(second).value()); + CHECK(index.is(store::NodeStatus::Leaf, root).value()); + REQUIRE(index.remove(root).value()); + CHECK(index.empty()); +} + +TEST_CASE("raster store traits validate tile boundaries", "[store][index][raster]") { + using Traits = raster_store::StoreTraits; + using Key = Traits::Key; + + CHECK(Traits::is_valid(Key{0, {0, 0}})); + CHECK_FALSE(Traits::is_valid(Key{0, {1, 0}})); + CHECK(Traits::is_valid(Key{31, {uint32_t{0x7fffffff}, uint32_t{0x7fffffff}}})); + CHECK_FALSE(Traits::is_valid(Key{31, {uint32_t{0x80000000}, 0}})); + CHECK(Traits::is_valid(Key{32, {0, 0}})); + CHECK(Traits::is_valid(Key{32, {UINT32_MAX, UINT32_MAX}})); + CHECK_FALSE(Traits::is_valid(Key{33, {0, 0}})); + + CHECK_FALSE(Traits::parent(Traits::root()).has_value()); + CHECK_FALSE(Traits::children(Key{32, {UINT32_MAX, UINT32_MAX}}).has_value()); + + const Key level_31{31, {uint32_t{0x7fffffff}, uint32_t{0x7fffffff}}}; + const auto children = Traits::children(level_31); + REQUIRE(children.has_value()); + CHECK(children->at(0) == Key{32, {UINT32_MAX - 1, UINT32_MAX - 1}}); + CHECK(children->at(1) == Key{32, {UINT32_MAX, UINT32_MAX - 1}}); + CHECK(children->at(2) == Key{32, {UINT32_MAX - 1, UINT32_MAX}}); + CHECK(children->at(3) == Key{32, {UINT32_MAX, UINT32_MAX}}); +} + +TEST_CASE("shared index rejects invalid keys", "[store][index][raster]") { + using Traits = raster_store::StoreTraits; + using Key = Traits::Key; + store::Index index; + const Key invalid{4, {16, 0}}; + + CHECK(index.get(invalid) == std::unexpected(store::InvalidKey{invalid})); + CHECK(index.add(invalid) == std::unexpected(store::InvalidKey{invalid})); + CHECK(index.remove(invalid) == std::unexpected(store::InvalidKey{invalid})); + CHECK(index.is_present(invalid) == std::unexpected(store::InvalidKey{invalid})); + CHECK(index.is_absent(invalid) == std::unexpected(store::InvalidKey{invalid})); + CHECK(index.is(store::NodeStatus::Leaf, invalid) == std::unexpected(store::InvalidKey{invalid})); +} diff --git a/unittests/terrainlib/store_layout.cpp b/unittests/terrainlib/store_layout.cpp new file mode 100644 index 00000000..4d0db771 --- /dev/null +++ b/unittests/terrainlib/store_layout.cpp @@ -0,0 +1,69 @@ +#include + +#include + +#include "mesh/codec/Gltf.h" +#include "mesh/codec/SfMesh.h" +#include "octree/store_layout/Mappings.h" +#include "store/Layout.h" + +TEST_CASE("extensionless octree mappings preserve physical paths", "[store][layout]") { + const std::filesystem::path dataset = "dataset"; + mesh::codec::SfMesh sfmesh; + + SECTION("flat") { + const store::Layout layout(dataset / "sf-flat", octree::store_layout::flat()); + const store::NodePath node_path = layout.node_path(octree::Id::root()); + CHECK(node_path.path() == dataset / "sf-flat/0-0"); + CHECK(sfmesh.paths(node_path) == std::vector{dataset / "sf-flat/0-0.sfmesh"}); + CHECK(layout.key_from_node_path(node_path) == octree::Id::root()); + } + + SECTION("level and coordinate directories") { + const store::Layout layout( + dataset / "sf-coordinates", + octree::store_layout::level_and_coordinate_directories()); + const octree::Id child = octree::Id::root().child(2).value(); + const octree::Id deep = octree::Id::root().child(5).value().child(7).value(); + CHECK(sfmesh.paths(layout.node_path(child)) + == std::vector{dataset / "sf-coordinates/1/0/1/0.sfmesh"}); + CHECK(sfmesh.paths(layout.node_path(deep)) + == std::vector{dataset / "sf-coordinates/2/3/1/3.sfmesh"}); + CHECK(layout.key_from_node_path(layout.node_path(child)) == child); + CHECK(layout.key_from_node_path(layout.node_path(deep)) == deep); + } +} + +TEST_CASE("octree mapping lookup is explicit", "[store][layout]") { + REQUIRE(octree::store_layout::from_id("flat").has_value()); + CHECK(octree::store_layout::from_id("flat")->id == "flat"); + REQUIRE(octree::store_layout::from_id("level_and_coordinate_directories").has_value()); + CHECK(octree::store_layout::from_id("level_and_coordinate_directories")->id + == "level_and_coordinate_directories"); + CHECK_FALSE(octree::store_layout::from_id("unknown").has_value()); + CHECK(octree::store_layout::all().size() == 2); +} + +TEST_CASE("extensionless octree mappings validate complete paths", "[store][layout]") { + const auto flat = octree::store_layout::flat(); + CHECK_FALSE(flat.node_path_to_key(store::NodePath("1-2.sfmesh")).has_value()); + CHECK_FALSE(flat.node_path_to_key(store::NodePath("nested/1-2")).has_value()); + CHECK_FALSE(flat.node_path_to_key(store::NodePath("1-2-extra")).has_value()); + + const auto coordinates = octree::store_layout::level_and_coordinate_directories(); + CHECK_FALSE(coordinates.node_path_to_key(store::NodePath("1/0/0/0.sfmesh")).has_value()); + CHECK_FALSE(coordinates.node_path_to_key(store::NodePath("1/0/0")).has_value()); + CHECK_FALSE(coordinates.node_path_to_key(store::NodePath("1/0/0/0/extra")).has_value()); + CHECK_FALSE(coordinates.node_path_to_key(store::NodePath("22/0/0/0")).has_value()); +} + +TEST_CASE("runtime mesh codecs own their filename endings", "[store][layout][codec]") { + const store::NodePath path("12/34/56/78"); + mesh::codec::SfMesh sfmesh; + mesh::codec::Gltf binary(mesh::codec::GltfContainer::Binary); + mesh::codec::Gltf json(mesh::codec::GltfContainer::Json); + + CHECK(sfmesh.paths(path) == std::vector{"12/34/56/78.sfmesh"}); + CHECK(binary.paths(path) == std::vector{"12/34/56/78.glb"}); + CHECK(json.paths(path) == std::vector{"12/34/56/78.gltf"}); +} diff --git a/unittests/terrainlib/store_storage.cpp b/unittests/terrainlib/store_storage.cpp new file mode 100644 index 00000000..6a06b075 --- /dev/null +++ b/unittests/terrainlib/store_storage.cpp @@ -0,0 +1,568 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "io/bytes.h" +#include "octree/StoreTraits.h" +#include "octree/store_layout/Mappings.h" +#include "raster_store/StoreTraits.h" +#include "store/IndexedStorage.h" +#include "string_utils.h" + +namespace { + +class TemporaryDirectory { +public: + explicit TemporaryDirectory(const std::string_view label) { + static std::atomic_uint64_t counter = 0; + const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); + _path = std::filesystem::temp_directory_path() + / ("atb-store-storage-" + std::string(label) + "-" + + std::to_string(timestamp) + "-" + std::to_string(counter++)); + REQUIRE(std::filesystem::create_directories(_path)); + } + ~TemporaryDirectory() { + std::error_code error; + std::filesystem::remove_all(_path, error); + } + const std::filesystem::path &path() const { + return _path; + } + +private: + std::filesystem::path _path; +}; + +store::NodePath tile_to_path(const radix::tile::Id &key) { + return store::NodePath( + std::to_string(key.zoom_level) + "/" + std::to_string(key.coords.x) + + "/" + std::to_string(key.coords.y)); +} + +std::optional path_to_tile(const store::NodePath &node_path) { + const std::filesystem::path &path = node_path.path(); + if (path.is_absolute() || std::distance(path.begin(), path.end()) != 3) { + return std::nullopt; + } + auto part = path.begin(); + const auto zoom = from_chars(part->string()); + ++part; + const auto x = from_chars(part->string()); + ++part; + const auto y = from_chars(part->string()); + if (!zoom.has_value() || !x.has_value() || !y.has_value()) { + return std::nullopt; + } + const radix::tile::Id key{zoom.value(), {x.value(), y.value()}}; + return raster_store::StoreTraits::is_valid(key) + ? std::optional(key) + : std::nullopt; +} + +template +store::PathMapping test_mapping(); + +template<> +store::PathMapping test_mapping() { + return octree::store_layout::flat(); +} + +template<> +store::PathMapping test_mapping() { + return {"test_tiles", tile_to_path, path_to_tile}; +} + +std::expected read_int(const std::filesystem::path &path) { + auto bytes = io::read_bytes_from_path(path); + if (!bytes || bytes->size() != sizeof(int)) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + store::CodecErrorCategory::Io, + "test integer read failed", + }); + } + int value; + std::memcpy(&value, bytes->data(), sizeof(value)); + return value; +} + +std::expected write_int( + const int value, + const std::filesystem::path &path) +{ + const auto bytes = std::span( + reinterpret_cast(&value), + sizeof(value)); + if (!io::write_bytes_to_path(bytes, path)) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Io, + "test integer write failed", + }); + } + return {}; +} + +class IntCodec final : public store::Codec { +public: + std::vector paths( + const store::NodePath &node_path) const override + { + std::filesystem::path path = node_path.path(); + path += ".data"; + return {path}; + } + + std::expected read( + const store::NodePath &node_path) const override + { + return read_int(paths(node_path).front()); + } + + std::expected write( + const store::NodePath &node_path, + const int &value) const override + { + return write_int(value, paths(node_path).front()); + } +}; + +template +store::Storage make_storage(const std::filesystem::path &path) { + return store::Storage(store::RawStorage( + store::Layout(path, test_mapping()), + std::make_unique())); +} + +class MultiFileCodec final : public store::Codec { +public: + std::vector paths(const store::NodePath &node_path) const override { + std::filesystem::path first = node_path.path(); + std::filesystem::path second = node_path.path(); + first += ".data"; + second += ".metadata"; + return {first, second}; + } + + std::expected read( + const store::NodePath &node_path) const override { + const auto value = read_int(paths(node_path).front()); + if (!value) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + store::CodecErrorCategory::Io, + "multi-file read failed", + }); + } + return value.value(); + } + + std::expected write( + const store::NodePath &node_path, + const int &value) const override { + const auto node_paths = paths(node_path); + if (!write_int(value, node_paths[0]) + || !write_int(value + 1, node_paths[1])) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Io, + "multi-file write failed", + }); + } + return {}; + } +}; + +class ConfigurableCodec final : public store::Codec { +public: + explicit ConfigurableCodec(std::string extension) + : extension(std::move(extension)) {} + + std::vector paths( + const store::NodePath &node_path) const override { + std::filesystem::path path = node_path.path(); + path += extension; + return {path}; + } + + std::expected read( + const store::NodePath &node_path) const override { + if (fail_read) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + store::CodecErrorCategory::Domain, + "injected decode failure", + }); + } + const auto value = read_int(paths(node_path).front()); + if (!value) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + store::CodecErrorCategory::Io, + "configurable read failed", + }); + } + return value.value(); + } + + std::expected write( + const store::NodePath &node_path, + const int &value) const override { + if (fail_write) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Domain, + "injected encode failure", + }); + } + if (!write_int(value, paths(node_path).front())) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Io, + "configurable write failed", + }); + } + return {}; + } + + std::string extension; + bool fail_read = false; + bool fail_write = false; +}; + +std::atomic_uint32_t index_writes = 0; + +std::expected, store::IndexFormatError> +unused_index_read(const std::filesystem::path &path) { + return std::unexpected(store::IndexFormatError{ + store::IndexFormatErrorCategory::Open, + path, + "not used", + }); +} + +std::expected count_index_write( + const std::filesystem::path &, + const store::IndexMetadata &) { + ++index_writes; + return {}; +} + +std::optional> fake_mapping_from_id( + const std::string_view id) { + return id == "flat" + ? std::optional>(octree::store_layout::flat()) + : std::nullopt; +} + +store::PathMapping fake_default_mapping() { + return octree::store_layout::flat(); +} + +store::IndexFormat counting_index_format() { + return { + "octree.storeindex", + unused_index_read, + count_index_write, + fake_mapping_from_id, + fake_default_mapping, + }; +} + +store::IndexedStorage make_indexed_storage( + const std::filesystem::path &path, + std::unique_ptr> codec = + std::make_unique()) { + using Storage = store::Storage; + return store::IndexedStorage( + store::RawStorage( + store::Layout(path, octree::store_layout::flat()), + std::move(codec)), + store::Index{}, + Storage::Persistence{ + counting_index_format(), + path / "octree.storeindex", + "flat", + "test.Int", + ".data", + }); +} + +} // namespace + +TEMPLATE_TEST_CASE( + "shared raw storage has no hidden octree dependency", + "[store][storage]", + octree::StoreTraits, + raster_store::StoreTraits) { + using Traits = TestType; + TemporaryDirectory directory("traits"); + auto storage = make_storage(directory.path()); + const auto root = Traits::root(); + + CHECK_FALSE(storage.has(root).value()); + REQUIRE(storage.save(root, 42).has_value()); + CHECK(storage.has(root).value()); + REQUIRE(storage.load(root).has_value()); + CHECK(storage.load(root).value() == 42); + REQUIRE(storage.remove(root).has_value()); + CHECK_FALSE(storage.has(root).value()); +} + +TEST_CASE("shared storage rejects invalid 2D keys", "[store][storage]") { + TemporaryDirectory directory("invalid-key"); + auto storage = make_storage(directory.path()); + const radix::tile::Id invalid{3, {8, 0}}; + + CHECK(std::holds_alternative>( + storage.load(invalid).error())); + CHECK(std::holds_alternative>( + storage.save(invalid, 1).error())); + CHECK(std::holds_alternative>( + storage.has(invalid).error())); + CHECK(std::holds_alternative>( + storage.remove(invalid).error())); +} + +TEST_CASE("shared storage preserves overwrite settings", "[store][storage]") { + TemporaryDirectory directory("overwrite"); + auto storage = make_storage(directory.path()); + const octree::Id root = octree::Id::root(); + + REQUIRE(storage.save(root, 1).has_value()); + const auto rejected = storage.save(root, 2); + REQUIRE_FALSE(rejected.has_value()); + CHECK(std::holds_alternative(rejected.error())); + CHECK(storage.load(root).value() == 1); + + storage.settings().allow_overwrite = true; + REQUIRE(storage.save(root, 2).has_value()); + CHECK(storage.load(root).value() == 2); +} + +TEST_CASE("raw storage requires and removes every codec path", "[store][storage]") { + TemporaryDirectory directory("multi-file"); + store::RawStorage storage( + store::Layout(directory.path(), octree::store_layout::flat()), + std::make_unique()); + const octree::Id root = octree::Id::root(); + const auto paths = storage.paths(root).value(); + REQUIRE(paths.size() == 2); + + REQUIRE(write_int(1, paths[0]).has_value()); + CHECK_FALSE(storage.has(root).value()); + REQUIRE(write_int(2, paths[1]).has_value()); + CHECK(storage.has(root).value()); + REQUIRE(storage.remove(root).value()); + CHECK_FALSE(std::filesystem::exists(paths[0])); + CHECK_FALSE(std::filesystem::exists(paths[1])); +} + +TEST_CASE("storage moves transfer and finalize dirty index state", "[store][storage]") { + index_writes = 0; + TemporaryDirectory first_directory("move-first"); + TemporaryDirectory second_directory("move-second"); + + { + auto original = make_indexed_storage(first_directory.path()); + REQUIRE(original.save(octree::Id::root(), 1).has_value()); + auto moved = std::move(original); + CHECK(index_writes == 0); + REQUIRE(moved.save_index().has_value()); + CHECK(index_writes == 1); + } + CHECK(index_writes == 1); + + { + auto destination = make_indexed_storage(first_directory.path()); + destination.settings().allow_overwrite = true; + REQUIRE(destination.save(octree::Id::root(), 2).has_value()); + auto source = make_indexed_storage(second_directory.path()); + REQUIRE(source.save(octree::Id::root(), 3).has_value()); + destination = std::move(source); + CHECK(index_writes == 2); + } + CHECK(index_writes == 3); +} + +TEST_CASE("copy_from hard-links every matching codec file", "[store][storage][copy]") { + const octree::Id root = octree::Id::root(); + + SECTION("one file") { + TemporaryDirectory source_directory("copy-one-source"); + TemporaryDirectory target_directory("copy-one-target"); + auto source = make_indexed_storage(source_directory.path()); + auto target = make_indexed_storage(target_directory.path()); + REQUIRE(source.save(root, 41).has_value()); + + REQUIRE(target.copy_from(root, source).has_value()); + REQUIRE(target.has(root).value()); + CHECK(target.load(root).value() == 41); + CHECK(std::filesystem::equivalent( + source.paths(root)->front(), + target.paths(root)->front())); + } + + SECTION("multiple files") { + TemporaryDirectory source_directory("copy-many-source"); + TemporaryDirectory target_directory("copy-many-target"); + auto source = make_indexed_storage( + source_directory.path(), std::make_unique()); + auto target = make_indexed_storage( + target_directory.path(), std::make_unique()); + REQUIRE(source.save(root, 42).has_value()); + + REQUIRE(target.copy_from(root, source).has_value()); + const auto source_paths = source.paths(root).value(); + const auto target_paths = target.paths(root).value(); + REQUIRE(source_paths.size() == 2); + REQUIRE(target_paths.size() == 2); + CHECK(std::filesystem::equivalent(source_paths[0], target_paths[0])); + CHECK(std::filesystem::equivalent(source_paths[1], target_paths[1])); + } +} + +TEST_CASE("copy_from re-encodes differing path lists", "[store][storage][copy]") { + const octree::Id root = octree::Id::root(); + TemporaryDirectory source_directory("copy-reencode-source"); + TemporaryDirectory target_directory("copy-reencode-target"); + auto source = make_indexed_storage( + source_directory.path(), std::make_unique()); + auto target = make_indexed_storage(target_directory.path()); + REQUIRE(source.save(root, 43).has_value()); + + REQUIRE(target.copy_from(root, source).has_value()); + CHECK(target.load(root).value() == 43); + CHECK(target.paths(root)->size() == 1); + CHECK(source.paths(root)->size() == 2); + CHECK_FALSE(std::filesystem::equivalent( + source.paths(root)->front(), + target.paths(root)->front())); +} + +TEST_CASE("copy_from enforces overwrite settings", "[store][storage][copy]") { + const octree::Id root = octree::Id::root(); + TemporaryDirectory source_directory("copy-overwrite-source"); + TemporaryDirectory target_directory("copy-overwrite-target"); + auto source = make_indexed_storage(source_directory.path()); + auto target = make_indexed_storage(target_directory.path()); + REQUIRE(source.save(root, 44).has_value()); + REQUIRE(target.save(root, 1).has_value()); + + const auto rejected = target.copy_from(root, source); + REQUIRE_FALSE(rejected.has_value()); + CHECK(std::holds_alternative(rejected.error())); + CHECK(target.load(root).value() == 1); + + target.settings().allow_overwrite = true; + REQUIRE(target.copy_from(root, source).has_value()); + CHECK(target.load(root).value() == 44); + CHECK(std::filesystem::equivalent( + source.paths(root)->front(), + target.paths(root)->front())); +} + +TEST_CASE( + "failed multi-file overwrite leaves the target node unindexed", + "[store][storage][copy]") { + const octree::Id root = octree::Id::root(); + TemporaryDirectory source_directory("copy-partial-source"); + TemporaryDirectory target_directory("copy-partial-target"); + auto source = make_indexed_storage( + source_directory.path(), std::make_unique()); + auto target = make_indexed_storage( + target_directory.path(), std::make_unique()); + REQUIRE(source.save(root, 45).has_value()); + REQUIRE(target.save(root, 1).has_value()); + target.settings().allow_overwrite = true; + + const auto target_paths = target.paths(root).value(); + REQUIRE(std::filesystem::remove(target_paths[1])); + REQUIRE(std::filesystem::create_directory(target_paths[1])); + REQUIRE(write_int(9, target_paths[1] / "blocker.data").has_value()); + + const auto copied = target.copy_from(root, source); + REQUIRE_FALSE(copied.has_value()); + CHECK(std::holds_alternative(copied.error())); + CHECK_FALSE(target.has(root).value()); + CHECK_FALSE(target.index().get(root).value().has_value()); + CHECK(std::filesystem::equivalent( + source.paths(root)->front(), + target_paths.front())); + CHECK(std::filesystem::is_directory(target_paths[1])); +} + +TEST_CASE("copy_from propagates source and codec failures", "[store][storage][copy]") { + const octree::Id root = octree::Id::root(); + + SECTION("missing source") { + TemporaryDirectory source_directory("copy-missing-source"); + TemporaryDirectory target_directory("copy-missing-target"); + auto source = make_indexed_storage(source_directory.path()); + auto target = make_indexed_storage(target_directory.path()); + REQUIRE(target.save(root, 7).has_value()); + target.settings().allow_overwrite = true; + + const auto copied = target.copy_from(root, source); + REQUIRE_FALSE(copied.has_value()); + CHECK(std::holds_alternative>( + copied.error())); + CHECK(target.has(root).value()); + CHECK(target.load(root).value() == 7); + } + + SECTION("decode failure") { + TemporaryDirectory source_directory("copy-decode-source"); + TemporaryDirectory target_directory("copy-decode-target"); + auto source_codec = std::make_unique(".source"); + ConfigurableCodec *source_codec_ptr = source_codec.get(); + auto source = make_indexed_storage( + source_directory.path(), std::move(source_codec)); + auto target = make_indexed_storage( + target_directory.path(), + std::make_unique(".target")); + REQUIRE(source.save(root, 46).has_value()); + REQUIRE(target.save(root, 7).has_value()); + target.settings().allow_overwrite = true; + source_codec_ptr->fail_read = true; + + const auto copied = target.copy_from(root, source); + REQUIRE_FALSE(copied.has_value()); + REQUIRE(std::holds_alternative(copied.error())); + CHECK(std::get(copied.error()).operation + == store::CodecOperation::Read); + CHECK(target.has(root).value()); + CHECK(target.load(root).value() == 7); + } + + SECTION("encode failure") { + TemporaryDirectory source_directory("copy-encode-source"); + TemporaryDirectory target_directory("copy-encode-target"); + auto source = make_indexed_storage( + source_directory.path(), + std::make_unique(".source")); + auto target_codec = std::make_unique(".target"); + ConfigurableCodec *target_codec_ptr = target_codec.get(); + auto target = make_indexed_storage( + target_directory.path(), std::move(target_codec)); + REQUIRE(source.save(root, 47).has_value()); + REQUIRE(target.save(root, 7).has_value()); + target.settings().allow_overwrite = true; + target_codec_ptr->fail_write = true; + + const auto copied = target.copy_from(root, source); + REQUIRE_FALSE(copied.has_value()); + REQUIRE(std::holds_alternative(copied.error())); + CHECK(std::get(copied.error()).operation + == store::CodecOperation::Write); + CHECK_FALSE(target.has(root).value()); + } +} diff --git a/unittests/terrainlib/store_traverse.cpp b/unittests/terrainlib/store_traverse.cpp new file mode 100644 index 00000000..43992ed1 --- /dev/null +++ b/unittests/terrainlib/store_traverse.cpp @@ -0,0 +1,68 @@ +#include + +#include +#include + +#include "octree/StoreTraits.h" +#include "raster_store/StoreTraits.h" +#include "store/traverse.h" + +TEMPLATE_TEST_CASE( + "shared hierarchy traversal preserves trait child order", + "[store][traverse]", + octree::StoreTraits, + raster_store::StoreTraits) { + using Traits = TestType; + using Key = typename Traits::Key; + + store::Index index; + const Key root = Traits::root(); + const auto children = Traits::children(root).value(); + const Key first = children[0]; + const Key second = children[1]; + const Key grandchild = Traits::children(first).value()[0]; + REQUIRE(index.add(root).has_value()); + REQUIRE(index.add(first).has_value()); + REQUIRE(index.add(second).has_value()); + REQUIRE(index.add(grandchild).has_value()); + + std::vector depth_first; + REQUIRE(store::traverse( + index, + [&](const Key &key, const store::NodeStatus) { depth_first.push_back(key); }) + .has_value()); + CHECK(depth_first == std::vector{root, first, grandchild, second}); + + std::vector breadth_first; + REQUIRE(store::traverse( + index, + [&](const Key &key, const store::NodeStatus) { breadth_first.push_back(key); }, + store::AlwaysRefine{}, + root, + store::TraversalOrder::BreadthFirst) + .has_value()); + CHECK(breadth_first == std::vector{root, first, second, grandchild}); + + std::vector explicit_root; + REQUIRE(store::traverse( + index, + [&](const Key &key, const store::NodeStatus) { explicit_root.push_back(key); }, + store::AlwaysRefine{}, + first) + .has_value()); + CHECK(explicit_root == std::vector{first, grandchild}); +} + +TEST_CASE("shared traversal rejects an invalid explicit root", "[store][traverse][raster]") { + using Traits = raster_store::StoreTraits; + using Key = Traits::Key; + const Key invalid{1, {2, 0}}; + store::Index index; + + const auto result = store::traverse( + index, + [](const Key &, const store::NodeStatus) {}, + store::AlwaysRefine{}, + invalid); + CHECK(result == std::unexpected(store::InvalidKey{invalid})); +} diff --git a/unittests/terrainlib/string_utils.cpp b/unittests/terrainlib/string_utils.cpp index c54490a4..9ef3699a 100644 --- a/unittests/terrainlib/string_utils.cpp +++ b/unittests/terrainlib/string_utils.cpp @@ -21,7 +21,7 @@ TEST_CASE("from_chars rejects non-numeric input", "[string_utils]") { } TEST_CASE("from_chars rejects trailing garbage after a valid number", "[string_utils]") { - CHECK(from_chars(std::string_view("123.terrain")) == std::nullopt); + CHECK(from_chars(std::string_view("123.sfmesh")) == std::nullopt); CHECK(from_chars(std::string_view("123abc")) == std::nullopt); } diff --git a/unittests/tile_downloader/downloader.cpp b/unittests/tile_downloader/downloader.cpp new file mode 100644 index 00000000..083ecd49 --- /dev/null +++ b/unittests/tile_downloader/downloader.cpp @@ -0,0 +1,102 @@ +#include "TileDownloader.h" + +#include +#include +#include + +#include + +namespace { + +class TemporaryPyramid { +public: + explicit TemporaryPyramid(std::string_view name) + : _path(std::filesystem::temp_directory_path() / name) { + std::error_code error; + std::filesystem::remove_all(_path, error); + std::filesystem::create_directories(_path); + } + + ~TemporaryPyramid() { + std::error_code error; + std::filesystem::remove_all(_path, error); + } + + [[nodiscard]] const std::filesystem::path &path() const { + return _path; + } + + [[nodiscard]] std::filesystem::path tile_path(const radix::tile::Id &tile) const { + return google_tile_path(_path, tile, ".jpeg"); + } + + void create_pending(const radix::tile::Id &tile) const { + const auto path = tile_path(tile); + std::filesystem::create_directories(path.parent_path()); + write_file_children_pending(path, std::vector{'t', 'i', 'l', 'e'}); + } + + void create_complete(const radix::tile::Id &tile) const { + create_pending(tile); + mark_tile_children_complete(tile_path(tile)); + } + +private: + std::filesystem::path _path; +}; + +const TileUrlBuilder missing_file_url({ + "file:///definitely-missing-atb-tile/{zoom}/{x}/{y}.jpeg", + TileYDirection::Down +}); + +} + +TEST_CASE("tile downloader promotes parents after completed children") +{ + const TemporaryPyramid pyramid("atb-downloader-complete-pyramid"); + const radix::tile::Id root{0, {0, 0}}; + const auto children = root.children(); + + pyramid.create_pending(root); + for (const auto &child : children) { + pyramid.create_pending(child); + } + + TileDownloader downloader(missing_file_url, pyramid.path(), 1u, root.zoom_level); + REQUIRE(downloader.download_recursive(root)); + + REQUIRE(std::filesystem::exists(pyramid.tile_path(root))); + CHECK_FALSE(std::filesystem::exists(children_pending_tile_path(pyramid.tile_path(root)))); + + for (const auto &child : children) { + CHECK(std::filesystem::exists(pyramid.tile_path(child))); + CHECK_FALSE(std::filesystem::exists(children_pending_tile_path(pyramid.tile_path(child)))); + } + + REQUIRE(std::filesystem::remove(pyramid.tile_path(children.front()))); + TileDownloader resumed_downloader(missing_file_url, pyramid.path(), 1u, root.zoom_level); + CHECK(resumed_downloader.download_recursive(root)); + CHECK_FALSE(std::filesystem::exists(pyramid.tile_path(children.front()))); +} + +TEST_CASE("tile downloader leaves ancestors pending after a child failure") +{ + const TemporaryPyramid pyramid("atb-downloader-failed-pyramid"); + const radix::tile::Id root{0, {0, 0}}; + const auto children = root.children(); + + pyramid.create_pending(root); + for (size_t i = 1; i < children.size(); ++i) { + pyramid.create_complete(children[i]); + } + + TileDownloader downloader(missing_file_url, pyramid.path(), 1u, root.zoom_level); + CHECK_FALSE(downloader.download_recursive(root)); + + CHECK_FALSE(std::filesystem::exists(pyramid.tile_path(root))); + CHECK(std::filesystem::exists(children_pending_tile_path(pyramid.tile_path(root)))); + for (size_t i = 1; i < children.size(); ++i) { + CHECK(std::filesystem::exists(pyramid.tile_path(children[i]))); + } +} diff --git a/unittests/tile_downloader/http_client.cpp b/unittests/tile_downloader/http_client.cpp new file mode 100644 index 00000000..61311a8c --- /dev/null +++ b/unittests/tile_downloader/http_client.cpp @@ -0,0 +1,62 @@ +#include "HttpClient.h" + +#include +#include +#include +#include + +#include + +namespace { + +class TemporaryFile { +public: + TemporaryFile() + : _path(std::filesystem::temp_directory_path() / "atb-http-client-test.txt") { + std::ofstream output(_path, std::ios::binary); + output << "response body"; + REQUIRE(output); + } + + ~TemporaryFile() { + std::error_code error; + std::filesystem::remove(_path, error); + } + + [[nodiscard]] std::string url() const { + return "file://" + _path.string(); + } + +private: + std::filesystem::path _path; +}; + +class CallbackError : public std::runtime_error { +public: + CallbackError() + : std::runtime_error("callback failed") {} +}; + +} + +TEST_CASE("http client propagates response writer exceptions") +{ + const TemporaryFile source; + HttpClient client([](std::vector &, const char *, size_t) { + throw CallbackError(); + }); + + CHECK_THROWS_AS(client.get(source.url()), CallbackError); +} + +TEST_CASE("http client propagates progress callback exceptions") +{ + const TemporaryFile source; + HttpClient client; + + CHECK_THROWS_AS( + client.get(source.url(), [](double) { + throw CallbackError(); + }), + CallbackError); +} diff --git a/unittests/tile_downloader/logger.cpp b/unittests/tile_downloader/logger.cpp new file mode 100644 index 00000000..c41ecb4c --- /dev/null +++ b/unittests/tile_downloader/logger.cpp @@ -0,0 +1,33 @@ +#include "TileLogger.h" + +#include +#include + +#include + +using namespace std::literals; + +TEST_CASE("tile logger session stops monitoring during exceptional unwinding") +{ + TileLogger logger(0); + + const auto before_throw = std::chrono::steady_clock::now(); + CHECK_THROWS_AS( + [&]() { + auto session = logger.start(); + throw std::runtime_error("download failed"); + }(), + std::runtime_error); + const auto unwind_duration = std::chrono::steady_clock::now() - before_throw; + + CHECK(unwind_duration < 250ms); +} + +TEST_CASE("tile logger session can finish normally") +{ + TileLogger logger(0); + auto session = logger.start(); + + logger.skipped(radix::tile::Id{0, {0, 0}}); + session.finish(); +} diff --git a/unittests/tile_downloader/url_builder.cpp b/unittests/tile_downloader/url_builder.cpp new file mode 100644 index 00000000..e6bdaccf --- /dev/null +++ b/unittests/tile_downloader/url_builder.cpp @@ -0,0 +1,54 @@ +#include + +#include "TileUrlBuilder.h" +#include "tile_path.h" + +namespace { +constexpr radix::tile::Id tile { 3, { 1, 2 } }; +} + +TEST_CASE("configured tile provider URLs") +{ + { + const TileUrlBuilder builder(tile_provider_config(TileDownloadProvider::Basemap)); + CHECK(builder.build_url(tile) == "https://mapsneu.wien.gv.at/basemap/bmaporthofoto30cm/normal/google3857/3/2/1.jpeg"); + } + + { + const TileUrlBuilder builder(tile_provider_config(TileDownloadProvider::Gataki)); + CHECK(builder.build_url(tile) == "https://gataki.cg.tuwien.ac.at/raw/basemap/tiles/3/2/1.jpeg"); + } +} + +TEST_CASE("custom tile URL patterns") +{ + SECTION("zoom/x/y with downward y") + { + const TileUrlBuilder builder({ "https://example.test/{zoom}/{x}/{y}.png", TileYDirection::Down }); + CHECK(builder.build_url(tile) == "https://example.test/3/1/2.png"); + } + + SECTION("zoom/y/x with downward y") + { + const TileUrlBuilder builder({ "https://example.test/{zoom}/{y}/{x}.png", TileYDirection::Down }); + CHECK(builder.build_url(tile) == "https://example.test/3/2/1.png"); + } + + SECTION("upward legacy TMS y") + { + const TileUrlBuilder builder({ "https://example.test/{zoom}/{x}/{y}.png", TileYDirection::Up }); + CHECK(builder.build_url(tile) == "https://example.test/3/1/5.png"); + } +} + +TEST_CASE("tile URL patterns require all coordinate placeholders") +{ + CHECK_THROWS_AS(TileUrlBuilder({ "https://example.test/{x}/{y}.png", TileYDirection::Down }), std::invalid_argument); + CHECK_THROWS_AS(TileUrlBuilder({ "https://example.test/{zoom}/{y}.png", TileYDirection::Down }), std::invalid_argument); + CHECK_THROWS_AS(TileUrlBuilder({ "https://example.test/{zoom}/{x}.png", TileYDirection::Down }), std::invalid_argument); +} + +TEST_CASE("downloaded tile path uses Google and Mapbox layout") +{ + CHECK(google_tile_path("tiles", tile, ".jpeg") == std::filesystem::path("tiles/3/1/2.jpeg")); +} diff --git a/unittests/tile_downloader/write_file.cpp b/unittests/tile_downloader/write_file.cpp new file mode 100644 index 00000000..2773c4ed --- /dev/null +++ b/unittests/tile_downloader/write_file.cpp @@ -0,0 +1,94 @@ +#include "write_file.h" + +#include +#include +#include +#include +#include +#include + +#include + +namespace { + +class TemporaryOutput { +public: + explicit TemporaryOutput(std::string_view name) + : _path(std::filesystem::temp_directory_path() / name) { + std::error_code error; + std::filesystem::remove_all(_path, error); + std::filesystem::remove(partial_path(), error); + std::filesystem::remove(pending_path(), error); + } + + ~TemporaryOutput() { + std::error_code error; + std::filesystem::remove_all(_path, error); + std::filesystem::remove(partial_path(), error); + std::filesystem::remove(pending_path(), error); + } + + [[nodiscard]] const std::filesystem::path &path() const { + return _path; + } + + [[nodiscard]] std::filesystem::path partial_path() const { + return partial_tile_path(_path); + } + + [[nodiscard]] std::filesystem::path pending_path() const { + return children_pending_tile_path(_path); + } + +private: + std::filesystem::path _path; +}; + +} + +TEST_CASE("checked file writer persists the complete response") +{ + const TemporaryOutput output("atb-write-file-success.bin"); + const std::vector expected{'t', 'i', 'l', 'e'}; + + write_file_children_pending(output.path(), expected); + + CHECK_FALSE(std::filesystem::exists(output.path())); + CHECK_FALSE(std::filesystem::exists(output.partial_path())); + REQUIRE(std::filesystem::exists(output.pending_path())); + + std::ifstream input(output.pending_path(), std::ios::binary); + const auto begin = std::istreambuf_iterator(input); + const std::vector actual(begin, std::istreambuf_iterator{}); + CHECK(actual == expected); + + mark_tile_children_complete(output.path()); + CHECK(std::filesystem::exists(output.path())); + CHECK_FALSE(std::filesystem::exists(output.pending_path())); +} + +TEST_CASE("checked file writer preserves the final path when promotion fails") +{ + const TemporaryOutput output("atb-write-file-promotion-failure"); + REQUIRE(std::filesystem::create_directory(output.path())); + write_file_children_pending(output.path(), std::vector{'t', 'i', 'l', 'e'}); + + CHECK_THROWS_AS( + mark_tile_children_complete(output.path()), + std::filesystem::filesystem_error); + + CHECK(std::filesystem::is_directory(output.path())); + CHECK(std::filesystem::exists(output.pending_path())); +} + +#if defined(__linux__) +TEST_CASE("checked file writer reports persistence failures") +{ + const std::vector data(64 * 1024, 'x'); + + CHECK_THROWS_AS(write_file_children_pending("/dev/full", data), std::runtime_error); + CHECK(std::filesystem::is_character_file("/dev/full")); + CHECK_FALSE(std::filesystem::exists("/dev/full.part")); + CHECK_FALSE(std::filesystem::exists("/dev/full.children-pending")); +} +#endif diff --git a/unittests/tilebuilder/alpine_raster_format.cpp b/unittests/tilebuilder/alpine_raster_format.cpp index 00ec9bf4..a0d98223 100644 --- a/unittests/tilebuilder/alpine_raster_format.cpp +++ b/unittests/tilebuilder/alpine_raster_format.cpp @@ -24,7 +24,7 @@ #include #include -#include "Image.h" +#include #include "alpine_raster.h" #include "ctb/Grid.hpp" @@ -41,15 +41,15 @@ TEMPLATE_TEST_CASE("alpine raster format, border ", "", std::true_type, std::fal SECTION("raste write") { - const auto generator = alpine_raster::make_generator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", "./unittest_tiles/", ctb::Grid::Srs::SphericalMercator, radix::tile::Scheme::Tms, radix::tile::Border::Yes); - generator.write(radix::tile::Descriptor { {0, glm::uvec2(0, 0)}, {}, int(ctb::Grid::Srs::SphericalMercator), 256, 257 }, HeightData(257, 257)); + const auto generator = alpine_raster::make_generator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", "./unittest_tiles/", ctb::Grid::Srs::SphericalMercator, radix::tile::Border::Yes); + generator.write(radix::tile::Descriptor { {0, glm::uvec2(0, 0)}, {}, int(ctb::Grid::Srs::SphericalMercator), 256, 257 }, radix::Raster({ 257, 257 })); CHECK(std::filesystem::exists("./unittest_tiles/0/0/0.png")); - generator.write(radix::tile::Descriptor { {1, glm::uvec2(2, 3)}, {}, int(ctb::Grid::Srs::SphericalMercator), 256, 257 }, HeightData(257, 257)); + generator.write(radix::tile::Descriptor { {1, glm::uvec2(2, 3)}, {}, int(ctb::Grid::Srs::SphericalMercator), 256, 257 }, radix::Raster({ 257, 257 })); CHECK(std::filesystem::exists("./unittest_tiles/1/2/3.png")); // check that a second write doesn't crash - generator.write(radix::tile::Descriptor { {1, glm::uvec2(2, 3)}, {}, int(ctb::Grid::Srs::SphericalMercator), 256, 257 }, HeightData(257, 257)); + generator.write(radix::tile::Descriptor { {1, glm::uvec2(2, 3)}, {}, int(ctb::Grid::Srs::SphericalMercator), 256, 257 }, radix::Raster({ 257, 257 })); CHECK(std::filesystem::exists("./unittest_tiles/1/2/3.png")); // in the best case, we would read back the data and check it. but that's too much work for now. @@ -58,7 +58,7 @@ TEMPLATE_TEST_CASE("alpine raster format, border ", "", std::true_type, std::fal SECTION("process all tiles") { - auto generator = alpine_raster::make_generator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", "./unittest_tiles/", ctb::Grid::Srs::SphericalMercator, radix::tile::Scheme::Tms, testTypeValue2Border(TestType::value)); + auto generator = alpine_raster::make_generator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", "./unittest_tiles/", ctb::Grid::Srs::SphericalMercator, testTypeValue2Border(TestType::value)); generator.setWarnOnMissingOverviews(false); generator.process({ 0, 7 }); const auto tiles = generator.tiler().generateTiles({ 0, 7 }); @@ -71,7 +71,7 @@ TEMPLATE_TEST_CASE("alpine raster format, border ", "", std::true_type, std::fal #if defined(ALP_UNITTESTS_EXTENDED) && ALP_UNITTESTS_EXTENDED SECTION("process all tiles with max zoom") { - auto generator = alpine_raster::make_generator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", "./unittest_tiles/", ctb::Grid::Srs::SphericalMercator, radix::tile::Scheme::Tms, testTypeValue2Border(TestType::value)); + auto generator = alpine_raster::make_generator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", "./unittest_tiles/", ctb::Grid::Srs::SphericalMercator, testTypeValue2Border(TestType::value)); generator.setWarnOnMissingOverviews(false); generator.process({ 4, 8 }); const auto tiles = generator.tiler().generateTiles({ 4, 8 }); diff --git a/unittests/tilebuilder/dataset_reading.cpp b/unittests/tilebuilder/dataset_reading.cpp index 17c714e9..ba2804db 100644 --- a/unittests/tilebuilder/dataset_reading.cpp +++ b/unittests/tilebuilder/dataset_reading.cpp @@ -27,6 +27,7 @@ #include "Dataset.h" #include "DatasetReader.h" #include "ctb/types.hpp" +#include "image_writer.h" #include "srs.h" using namespace radix; @@ -184,7 +185,7 @@ TEST_CASE("reading") if (ALP_UNITTESTS_DEBUG_IMAGES) { image::debugOut(ref_heights, fmt::format("./heights_{}_{}.png", test_name, dataset_name.substr(s, l))); - auto height_diffs = HeightData(render_width, render_height); + auto height_diffs = radix::Raster({ render_width, render_height }); std::transform(ref_heights.begin(), ref_heights.end(), heights.begin(), height_diffs.begin(), [](auto a, auto b) { return std::abs(a - b); }); const auto path = fmt::format("./diffs_{}_{}.png", test_name, dataset_name.substr(s, l)); image::debugOut(height_diffs, path); @@ -194,7 +195,7 @@ TEST_CASE("reading") const auto t = std::abs(double(a) - double(b)); largest_abs_diff = std::max(t, largest_abs_diff); return t * t; - }) / double(ref_heights.size()); + }) / double(ref_heights.buffer_length()); // fmt::print("{} | {}; mse: {}, largest_abs_diff: {}\n", test_name, dataset_name.substr(s, l), mse, largest_abs_diff); CHECK(largest_abs_diff < double(max_abs_diff)); CHECK(mse < max_mse); @@ -248,7 +249,7 @@ TEST_CASE("reading") image::debugOut(low_res_heights, fmt::format("./low_res_heights.png")); image::debugOut(high_res_heights, fmt::format("./high_res_heights.png")); - auto height_diffs = HeightData(render_width, render_height); + auto height_diffs = radix::Raster({ render_width, render_height }); std::transform(low_res_heights.begin(), low_res_heights.end(), high_res_heights.begin(), height_diffs.begin(), [](auto a, auto b) { return std::abs(a - b); }); image::debugOut(height_diffs, "./diff_low_res_high_res.png"); } @@ -257,7 +258,7 @@ TEST_CASE("reading") const auto t = std::abs(double(a) - double(b)); largest_abs_diff = std::max(t, largest_abs_diff); return t * t; - }) / double(low_res_heights.size()); + }) / double(low_res_heights.buffer_length()); // fmt::print("mse: {}, largest_abs_diff: {}\n", mse, largest_abs_diff); CHECK(largest_abs_diff < double(max_abs_diff)); CHECK(mse < max_mse); @@ -290,7 +291,7 @@ TEST_CASE("reading") image::debugOut(low_res_heights, fmt::format("./ov_with_warping_low_res_heights.png")); image::debugOut(high_res_heights, fmt::format("./ov_with_warping_high_res_heights.png")); - auto height_diffs = HeightData(render_width, render_height); + auto height_diffs = radix::Raster({ render_width, render_height }); std::transform(low_res_heights.begin(), low_res_heights.end(), high_res_heights.begin(), height_diffs.begin(), [](auto a, auto b) { return std::abs(a - b); }); image::debugOut(height_diffs, "./ov_with_warping_diff_low_res_high_res.png"); } @@ -299,7 +300,7 @@ TEST_CASE("reading") const auto t = std::abs(double(a) - double(b)); largest_abs_diff = std::max(t, largest_abs_diff); return t * t; - }) / double(low_res_heights.size()); + }) / double(low_res_heights.buffer_length()); // fmt::print("mse: {}, largest_abs_diff: {}\n", mse, largest_abs_diff); CHECK(largest_abs_diff < double(max_abs_diff)); CHECK(mse < max_mse); @@ -332,7 +333,7 @@ TEST_CASE("reading") image::debugOut(low_res_heights, fmt::format("./lowres_ov_with_warping_low_res_heights.png")); image::debugOut(high_res_heights, fmt::format("./lowres_ov_with_warping_high_res_heights.png")); - auto height_diffs = HeightData(render_width, render_height); + auto height_diffs = radix::Raster({ render_width, render_height }); std::transform(low_res_heights.begin(), low_res_heights.end(), high_res_heights.begin(), height_diffs.begin(), [](auto a, auto b) { return std::abs(a - b); }); image::debugOut(height_diffs, "./lowres_ov_with_warping_diff_low_res_high_res.png"); } @@ -341,7 +342,7 @@ TEST_CASE("reading") const auto t = std::abs(double(a) - double(b)); largest_abs_diff = std::max(t, largest_abs_diff); return t * t; - }) / double(low_res_heights.size()); + }) / double(low_res_heights.buffer_length()); // fmt::print("render w/h: {}/{}, mse: {}, largest_abs_diff: {}\n", render_width, render_height, mse, largest_abs_diff); CHECK(largest_abs_diff < double(max_abs_diff)); CHECK(mse < max_mse); diff --git a/unittests/tilebuilder/depth_first_tile_traverser.cpp b/unittests/tilebuilder/depth_first_tile_traverser.cpp index ef753a3a..5cf7fb5e 100644 --- a/unittests/tilebuilder/depth_first_tile_traverser.cpp +++ b/unittests/tilebuilder/depth_first_tile_traverser.cpp @@ -37,8 +37,8 @@ TEST_CASE("depth_first_tile_traverser interface") const auto aggregate_function = [](std::vector) { return ReadType {}; }; const auto grid = ctb::GlobalMercator(); - const auto tiler = TopDownTiler(grid, grid.getExtent(), radix::tile::Border::No, radix::tile::Scheme::Tms); - const radix::tile::Id root_id = { 0, { 0, 0 }, tiler.scheme() }; + const auto tiler = TopDownTiler(grid, grid.getExtent(), radix::tile::Border::No); + const radix::tile::Id root_id = { 0, { 0, 0 } }; const unsigned max_zoom_level = 3; traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, root_id, max_zoom_level); @@ -68,15 +68,15 @@ TEST_CASE("depth_first_tile_traverser basics") }; const auto grid = ctb::GlobalMercator(); - const auto tiler = TopDownTiler(grid, grid.getExtent(), radix::tile::Border::No, radix::tile::Scheme::Tms); - const radix::tile::Id root_id = { 0, { 0, 0 }, tiler.scheme() }; + const auto tiler = TopDownTiler(grid, grid.getExtent(), radix::tile::Border::No); + const radix::tile::Id root_id = { 0, { 0, 0 } }; SECTION("reads root tile #1") { const auto result = traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, root_id, 0); REQUIRE(read_tiles.size() == 1); CHECK(aggregate_calls.size() == 0); - CHECK(read_tiles.contains(radix::tile::Id{0, {0, 0}, tiler.scheme()})); + CHECK(read_tiles.contains(radix::tile::Id{0, {0, 0}})); CHECK(result.d == glm::uvec2 { 0, 0 }); } @@ -84,7 +84,7 @@ TEST_CASE("depth_first_tile_traverser basics") { const auto result = traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, { 2, { 1, 3 } }, 2); REQUIRE(read_tiles.size() == 1); - CHECK(read_tiles.contains(radix::tile::Id{2, {1, 3}, tiler.scheme()})); + CHECK(read_tiles.contains(radix::tile::Id{2, {1, 3}})); CHECK(result.d == glm::uvec2 { 1, 3 }); } @@ -92,10 +92,10 @@ TEST_CASE("depth_first_tile_traverser basics") { traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, { 0, { 0, 0 } }, 1); REQUIRE(read_tiles.size() == 4); - CHECK(read_tiles.contains(radix::tile::Id{1, {0, 0}, tiler.scheme()})); - CHECK(read_tiles.contains(radix::tile::Id{1, {0, 1}, tiler.scheme()})); - CHECK(read_tiles.contains(radix::tile::Id{1, {1, 0}, tiler.scheme()})); - CHECK(read_tiles.contains(radix::tile::Id{1, {1, 1}, tiler.scheme()})); + CHECK(read_tiles.contains(radix::tile::Id{1, {0, 0}})); + CHECK(read_tiles.contains(radix::tile::Id{1, {0, 1}})); + CHECK(read_tiles.contains(radix::tile::Id{1, {1, 0}})); + CHECK(read_tiles.contains(radix::tile::Id{1, {1, 1}})); } SECTION("aggregate is called correctly") @@ -119,7 +119,7 @@ TEST_CASE("depth_first_tile_traverser austrian heights") const auto dataset = Dataset::open_shared_raster(ALP_TEST_DATA_DIR "/austria/at_100m_mgi.tif").value(); // const auto dataset = Dataset::open_shared_raster(ALP_TEST_DATA_DIR "/austria/at_mgi.tif"); const auto bounds = dataset->bounds(grid.getSRS()); - const auto tiler = TopDownTiler(grid, bounds, radix::tile::Border::No, radix::tile::Scheme::Tms); + const auto tiler = TopDownTiler(grid, bounds, radix::tile::Border::No); const auto tile_reader = DatasetReader(dataset, grid.getSRS(), 1, false); // const auto dataset_reader = DatasetReader() std::set read_tiles; @@ -142,14 +142,14 @@ TEST_CASE("depth_first_tile_traverser austrian heights") return aggr; }; - const radix::tile::Id root_id = { 0, { 0, 0 }, tiler.scheme() }; + const radix::tile::Id root_id = { 0, { 0, 0 } }; SECTION("reads root tile") { const auto result = traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, root_id, 0); REQUIRE(read_tiles.size() == 1); CHECK(aggregate_calls.size() == 0); - CHECK(read_tiles.contains(radix::tile::Id{0, {0, 0}, tiler.scheme()})); + CHECK(read_tiles.contains(radix::tile::Id{0, {0, 0}})); CHECK(result.first >= 0); CHECK(result.first <= 4000); CHECK(result.second >= 0); @@ -161,14 +161,14 @@ TEST_CASE("depth_first_tile_traverser austrian heights") const auto result = traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, root_id, 6); CHECK(read_tiles.size() == 6); CHECK(aggregate_calls.size() == 9); - CHECK(read_tiles.contains(radix::tile::Id{6, {33, 41}, tiler.scheme()})); - CHECK(read_tiles.contains(radix::tile::Id{6, {33, 42}, tiler.scheme()})); + CHECK(read_tiles.contains(radix::tile::Id{6, {33, 22}})); + CHECK(read_tiles.contains(radix::tile::Id{6, {33, 21}})); - CHECK(read_tiles.contains(radix::tile::Id{6, {34, 41}, tiler.scheme()})); - CHECK(read_tiles.contains(radix::tile::Id{6, {34, 42}, tiler.scheme()})); + CHECK(read_tiles.contains(radix::tile::Id{6, {34, 22}})); + CHECK(read_tiles.contains(radix::tile::Id{6, {34, 21}})); - CHECK(read_tiles.contains(radix::tile::Id{6, {35, 41}, tiler.scheme()})); - CHECK(read_tiles.contains(radix::tile::Id{6, {35, 42}, tiler.scheme()})); + CHECK(read_tiles.contains(radix::tile::Id{6, {35, 22}})); + CHECK(read_tiles.contains(radix::tile::Id{6, {35, 21}})); CHECK(result.first >= 0); CHECK(result.first <= 500); CHECK(result.second >= 2000); @@ -187,8 +187,8 @@ TEST_CASE("depth_first_tile_traverser aggregate is not called with an empty vect // parent tile is produced, because its border overlaps the extents // child tiles have smaller pixels -> their border does not overlap the extents any more. bounds.min.x = (bounds.width() / 256) / 4; - const auto tiler = TopDownTiler(grid, bounds, radix::tile::Border::Yes, radix::tile::Scheme::Tms); - const radix::tile::Id root_id = { 0, { 0, 0 }, tiler.scheme() }; + const auto tiler = TopDownTiler(grid, bounds, radix::tile::Border::Yes); + const radix::tile::Id root_id = { 0, { 0, 0 } }; const auto read_function = [&](const radix::tile::Descriptor&) -> int { return 0; diff --git a/unittests/tilebuilder/image.cpp b/unittests/tilebuilder/image.cpp deleted file mode 100644 index 448aef27..00000000 --- a/unittests/tilebuilder/image.cpp +++ /dev/null @@ -1,66 +0,0 @@ -/***************************************************************************** - * Alpine Terrain Builder - * Copyright (C) 2022 alpinemaps.org - * Copyright (C) 2022 Adam Celarek - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *****************************************************************************/ - -#include -#include -#include -#include "Image.h" - -using Catch::Approx; - -TEST_CASE("image") -{ - SECTION("iteration") - { - HeightData d { 40, 60 }; - REQUIRE(d.size() == 40 * 60); - REQUIRE(d.height() == 60); - REQUIRE(d.width() == 40); - int v = 0; - std::ranges::for_each(d, [&](auto& d) { d = float(v++); }); - - v = 0; - for (unsigned r = 0; r < d.height(); ++r) { - for (unsigned c = 0; c < d.width(); ++c) { - REQUIRE(d.pixel(r, c) == Approx(float(v++))); - } - } - } - - SECTION("conversion") - { - HeightData d { 40, 60 }; - int v_init = 0; - std::ranges::for_each(d, [&](auto& d) { d = float(v_init++); }); - - Image image = image::transformImage(d, [&](auto v) { const auto b = uchar(255.F * v / float(v_init)); return glm::u8vec3(b, b, b); }); - const auto max = float(v_init); - v_init = 0; - for (unsigned r = 0; r < d.height(); ++r) { - for (unsigned c = 0; c < d.width(); ++c) { - const auto t = uchar(float(v_init) * 255.F / max); - REQUIRE(image.pixel(r, c).x == t); - REQUIRE(image.pixel(r, c).y == t); - REQUIRE(image.pixel(r, c).z == t); - v_init++; - } - } -// image::saveImageAsPng(image, "/home/madam/Documents/work/tuw/alpinemaps/tmp/test.png"); - } -} diff --git a/unittests/tilebuilder/image_writer.cpp b/unittests/tilebuilder/image_writer.cpp new file mode 100644 index 00000000..aeddf2f2 --- /dev/null +++ b/unittests/tilebuilder/image_writer.cpp @@ -0,0 +1,86 @@ +/***************************************************************************** + * Alpine Terrain Builder + * Copyright (C) 2022 alpinemaps.org + * Copyright (C) 2022 Adam Celarek + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *****************************************************************************/ + +#include +#include +#include + +#include +#include +#include + +#include "image_writer.h" + +TEST_CASE("tile builder writes radix rasters as PNG images") +{ + const auto output_path = std::filesystem::temp_directory_path() / "alpine_terrain_builder_image_writer_test.png"; + std::filesystem::remove(output_path); + + radix::Raster raster({ 2, 2 }); + raster.pixel({ 0, 0 }) = { 255, 0, 0 }; + raster.pixel({ 1, 0 }) = { 0, 255, 0 }; + raster.pixel({ 0, 1 }) = { 0, 0, 255 }; + raster.pixel({ 1, 1 }) = { 255, 255, 255 }; + + image::saveImageAsPng(raster, output_path.string()); + + const auto decoded = cv::imread(output_path.string(), cv::IMREAD_COLOR); + REQUIRE(decoded.rows == 2); + REQUIRE(decoded.cols == 2); + CHECK(decoded.at(0, 0) == cv::Vec3b(255, 0, 0)); + CHECK(decoded.at(0, 1) == cv::Vec3b(255, 255, 255)); + CHECK(decoded.at(1, 0) == cv::Vec3b(0, 0, 255)); + CHECK(decoded.at(1, 1) == cv::Vec3b(0, 255, 0)); + + std::filesystem::remove(output_path); +} + +TEST_CASE("tile builder handles image writer edge cases") +{ + SECTION("empty debug raster is rejected") + { + CHECK_THROWS_AS(image::debugOut(radix::Raster {}, "empty.png"), std::invalid_argument); + } + + SECTION("constant debug raster is written as black") + { + const auto output_path = std::filesystem::temp_directory_path() / "alpine_terrain_builder_constant_raster_test.png"; + std::filesystem::remove(output_path); + + radix::Raster raster({ 2, 2 }); + std::ranges::fill(raster, 42.F); + image::debugOut(raster, output_path.string()); + + const auto decoded = cv::imread(output_path.string(), cv::IMREAD_COLOR); + REQUIRE(decoded.rows == 2); + REQUIRE(decoded.cols == 2); + CHECK(cv::countNonZero(decoded.reshape(1)) == 0); + + std::filesystem::remove(output_path); + } + + SECTION("PNG write failure is reported") + { + const auto missing_directory = std::filesystem::temp_directory_path() / "alpine_terrain_builder_missing_directory"; + std::filesystem::remove_all(missing_directory); + const auto output_path = missing_directory / "image.png"; + + CHECK_THROWS_AS(image::saveImageAsPng(radix::Raster({ 1, 1 }), output_path.string()), std::runtime_error); + } +} diff --git a/unittests/tilebuilder/parallel_tile_generator.cpp b/unittests/tilebuilder/parallel_tile_generator.cpp index ee57262d..f9326334 100644 --- a/unittests/tilebuilder/parallel_tile_generator.cpp +++ b/unittests/tilebuilder/parallel_tile_generator.cpp @@ -43,7 +43,7 @@ TEST_CASE("parallel tile generator") , m_validation_error_counter(validation_error_counter) { } - void write(const std::string& file_path, const radix::tile::Descriptor& tile, const HeightData& heights) const override + void write(const std::string& file_path, const radix::tile::Descriptor& tile, const radix::Raster& heights) const override { if (file_path.empty() || tile.gridSize != 256 || heights.width() != 256 || heights.height() != 256) (*m_validation_error_counter)++; @@ -55,7 +55,7 @@ TEST_CASE("parallel tile generator") }; std::filesystem::path base_path = "./unittest_tiles/"; - auto generator = ParallelTileGenerator::make(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", ctb::Grid::Srs::SphericalMercator, radix::tile::Scheme::Tms, std::make_unique(&tile_counter, &validation_error_counter), base_path); + auto generator = ParallelTileGenerator::make(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", ctb::Grid::Srs::SphericalMercator, std::make_unique(&tile_counter, &validation_error_counter), base_path); generator.setWarnOnMissingOverviews(false); SECTION("dataset tiles only") { @@ -63,12 +63,12 @@ TEST_CASE("parallel tile generator") CHECK(validation_error_counter == 0); CHECK(tile_counter == 27); CHECK(std::filesystem::exists(base_path / "0" / "0" / "0.empty")); - CHECK(std::filesystem::exists(base_path / "1" / "1" / "1.empty")); - CHECK(std::filesystem::exists(base_path / "2" / "2" / "2.empty")); - CHECK(std::filesystem::exists(base_path / "3" / "4" / "5.empty")); - CHECK(std::filesystem::exists(base_path / "4" / "8" / "10.empty")); - CHECK(std::filesystem::exists(base_path / "5" / "16" / "20.empty")); - CHECK(std::filesystem::exists(base_path / "7" / "70" / "84.empty")); + CHECK(std::filesystem::exists(base_path / "1" / "1" / "0.empty")); + CHECK(std::filesystem::exists(base_path / "2" / "2" / "1.empty")); + CHECK(std::filesystem::exists(base_path / "3" / "4" / "2.empty")); + CHECK(std::filesystem::exists(base_path / "4" / "8" / "5.empty")); + CHECK(std::filesystem::exists(base_path / "5" / "16" / "11.empty")); + CHECK(std::filesystem::exists(base_path / "7" / "70" / "43.empty")); } SECTION("world wide tiles") { diff --git a/unittests/tilebuilder/parallel_tiler.cpp b/unittests/tilebuilder/parallel_tiler.cpp index 815b48ba..b18124cf 100644 --- a/unittests/tilebuilder/parallel_tiler.cpp +++ b/unittests/tilebuilder/parallel_tiler.cpp @@ -23,23 +23,21 @@ #include "ctb/GlobalGeodetic.hpp" #include "ctb/GlobalMercator.hpp" #include -#include #include #include #include #include -#include using Catch::Approx; using namespace radix; -TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::false_type) +TEST_CASE("ParallelTiler") { // const auto bounds = radix::tile::SrsBounds(1'000'000, 6'000'000, 2'000'000, 6'700'000); // in m SECTION("mercator / level 0") { const auto grid = ctb::GlobalMercator(); - const auto tiler = ParallelTiler(grid, grid.getExtent(), radix::tile::Border::No, TestType::value ? radix::tile::Scheme::Tms : radix::tile::Scheme::SlippyMap); + const auto tiler = ParallelTiler(grid, grid.getExtent(), radix::tile::Border::No); CHECK(tiler.northEastTile(0).coords == glm::uvec2(0, 0)); CHECK(tiler.southWestTile(0).coords == glm::uvec2(0, 0)); @@ -59,26 +57,25 @@ TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::f CHECK(t.srsBounds.max.x == Approx(grid.getExtent().max.x)); } - SECTION("mercator tms / level 1 and 2") + SECTION("mercator / level 1 and 2") { const auto grid = ctb::GlobalMercator(); auto dataset = Dataset::open_shared_raster(ALP_TEST_DATA_DIR "/austria/at_mgi.tif").value(); - const auto tiler = ParallelTiler(grid, dataset->bounds(grid.getSRS()), radix::tile::Border::No, TestType::value ? radix::tile::Scheme::Tms : radix::tile::Scheme::SlippyMap); + const auto tiler = ParallelTiler(grid, dataset->bounds(grid.getSRS()), radix::tile::Border::No); - CHECK(tiler.northEastTile(1).coords == glm::uvec2(1, TestType::value ? 1 : 0)); - CHECK(tiler.southWestTile(1).coords == glm::uvec2(1, TestType::value ? 1 : 0)); + CHECK(tiler.northEastTile(1).coords == glm::uvec2(1, 0)); + CHECK(tiler.southWestTile(1).coords == glm::uvec2(1, 0)); - CHECK(tiler.northEastTile(2).coords == glm::uvec2(2, TestType::value ? 2 : 1)); - CHECK(tiler.southWestTile(2).coords == glm::uvec2(2, TestType::value ? 2 : 1)); + CHECK(tiler.northEastTile(2).coords == glm::uvec2(2, 1)); + CHECK(tiler.southWestTile(2).coords == glm::uvec2(2, 1)); { // https://www.maptiler.com/google-maps-coordinates-tile-bounds-projection/#1/-5.80/62.29 - // this code is for TMS mapping, i.e., tile y = 0 is south const auto l1_tiles = tiler.generateTiles(1); REQUIRE(l1_tiles.size() == 1); const auto t = l1_tiles.front(); CHECK(t.id.zoom_level == 1); - CHECK(t.id.coords == glm::uvec2(1, TestType::value ? 1 : 0)); + CHECK(t.id.coords == glm::uvec2(1, 0)); CHECK(t.gridSize == 256); CHECK(t.tileSize == 256); @@ -92,7 +89,7 @@ TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::f REQUIRE(l2_tiles.size() == 1); const auto t = l2_tiles.front(); CHECK(t.id.zoom_level == 2); - CHECK(t.id.coords == glm::uvec2(2, TestType::value ? 2 : 1)); + CHECK(t.id.coords == glm::uvec2(2, 1)); CHECK(t.gridSize == 256); CHECK(t.tileSize == 256); @@ -103,10 +100,10 @@ TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::f } } - SECTION("geodetic tms / level 0") + SECTION("geodetic / level 0") { const auto grid = ctb::GlobalGeodetic(64); - const auto tiler = ParallelTiler(grid, grid.getExtent(), radix::tile::Border::Yes, TestType::value ? radix::tile::Scheme::Tms : radix::tile::Scheme::SlippyMap); + const auto tiler = ParallelTiler(grid, grid.getExtent(), radix::tile::Border::Yes); CHECK(tiler.northEastTile(0).coords == glm::uvec2(1, 0)); CHECK(tiler.southWestTile(0).coords == glm::uvec2(0, 0)); @@ -140,24 +137,24 @@ TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::f CHECK(t1.srsBounds.max.x == Approx(grid.getExtent().max.x + grid.resolution(0))); } - SECTION("geodetic tms / level 1 and 2") + SECTION("geodetic / level 1 and 2") { const auto grid = ctb::GlobalGeodetic(64); auto dataset = Dataset::open_shared_raster(ALP_TEST_DATA_DIR "/austria/at_mgi.tif").value(); - const auto tiler = ParallelTiler(grid, dataset->bounds(grid.getSRS()), radix::tile::Border::Yes, TestType::value ? radix::tile::Scheme::Tms : radix::tile::Scheme::SlippyMap); + const auto tiler = ParallelTiler(grid, dataset->bounds(grid.getSRS()), radix::tile::Border::Yes); - CHECK(tiler.northEastTile(1).coords == glm::uvec2(2, TestType::value ? 1 : 0)); - CHECK(tiler.southWestTile(1).coords == glm::uvec2(2, TestType::value ? 1 : 0)); + CHECK(tiler.northEastTile(1).coords == glm::uvec2(2, 0)); + CHECK(tiler.southWestTile(1).coords == glm::uvec2(2, 0)); - CHECK(tiler.northEastTile(2).coords == glm::uvec2(4, TestType::value ? 3 : 0)); - CHECK(tiler.southWestTile(2).coords == glm::uvec2(4, TestType::value ? 3 : 0)); + CHECK(tiler.northEastTile(2).coords == glm::uvec2(4, 0)); + CHECK(tiler.southWestTile(2).coords == glm::uvec2(4, 0)); { const auto l1_tiles = tiler.generateTiles(1); REQUIRE(l1_tiles.size() == 1); const auto t = l1_tiles.front(); CHECK(t.id.zoom_level == 1); - CHECK(t.id.coords == glm::uvec2(2, TestType::value ? 1 : 0)); + CHECK(t.id.coords == glm::uvec2(2, 0)); CHECK(t.gridSize == 64); CHECK(t.tileSize == 65); @@ -173,7 +170,7 @@ TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::f REQUIRE(l2_tiles.size() == 1); const auto t = l2_tiles.front(); CHECK(t.id.zoom_level == 2); - CHECK(t.id.coords == glm::uvec2(4, TestType::value ? 3 : 0)); + CHECK(t.id.coords == glm::uvec2(4, 0)); CHECK(t.gridSize == 64); CHECK(t.tileSize == 65); @@ -186,26 +183,25 @@ TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::f } } - SECTION("mercator tms / level 1 and 2 (test with cape horn, on southern and western hemisphere)") + SECTION("mercator / level 1 and 2 (test with cape horn, on southern and western hemisphere)") { const auto grid = ctb::GlobalMercator(); auto dataset = Dataset::open_shared_raster(ALP_TEST_DATA_DIR "/capehorn/small.tif").value(); - const auto tiler = ParallelTiler(grid, dataset->bounds(grid.getSRS()), radix::tile::Border::No, TestType::value ? radix::tile::Scheme::Tms : radix::tile::Scheme::SlippyMap); + const auto tiler = ParallelTiler(grid, dataset->bounds(grid.getSRS()), radix::tile::Border::No); - CHECK(tiler.northEastTile(1).coords == glm::uvec2(0, TestType::value ? 0 : 1)); - CHECK(tiler.southWestTile(1).coords == glm::uvec2(0, TestType::value ? 0 : 1)); + CHECK(tiler.northEastTile(1).coords == glm::uvec2(0, 1)); + CHECK(tiler.southWestTile(1).coords == glm::uvec2(0, 1)); - CHECK(tiler.northEastTile(2).coords == glm::uvec2(1, TestType::value ? 1 : 2)); - CHECK(tiler.southWestTile(2).coords == glm::uvec2(1, TestType::value ? 1 : 2)); + CHECK(tiler.northEastTile(2).coords == glm::uvec2(1, 2)); + CHECK(tiler.southWestTile(2).coords == glm::uvec2(1, 2)); { // https://www.maptiler.com/google-maps-coordinates-tile-bounds-projection/#1/-5.80/62.29 - // this code is for TMS mapping, i.e., tile y = 0 is south const auto l1_tiles = tiler.generateTiles(1); REQUIRE(l1_tiles.size() == 1); const auto t = l1_tiles.front(); CHECK(t.id.zoom_level == 1); - CHECK(t.id.coords == glm::uvec2(0, TestType::value ? 0 : 1)); + CHECK(t.id.coords == glm::uvec2(0, 1)); CHECK(t.gridSize == 256); CHECK(t.tileSize == 256); @@ -219,7 +215,7 @@ TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::f REQUIRE(l2_tiles.size() == 1); const auto t = l2_tiles.front(); CHECK(t.id.zoom_level == 2); - CHECK(t.id.coords == glm::uvec2(1, TestType::value ? 1 : 2)); + CHECK(t.id.coords == glm::uvec2(1, 2)); CHECK(t.gridSize == 256); CHECK(t.tileSize == 256); @@ -230,24 +226,24 @@ TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::f } } - SECTION("geodetic tms / level 1 and 2 (test with cape horn, on southern and western hemisphere)") + SECTION("geodetic / level 1 and 2 (test with cape horn, on southern and western hemisphere)") { const auto grid = ctb::GlobalGeodetic(64); auto dataset = Dataset::open_shared_raster(ALP_TEST_DATA_DIR "/capehorn/small.tif").value(); - const auto tiler = ParallelTiler(grid, dataset->bounds(grid.getSRS()), radix::tile::Border::Yes, TestType::value ? radix::tile::Scheme::Tms : radix::tile::Scheme::SlippyMap); + const auto tiler = ParallelTiler(grid, dataset->bounds(grid.getSRS()), radix::tile::Border::Yes); - CHECK(tiler.northEastTile(1).coords == glm::uvec2(1, TestType::value ? 0 : 1)); - CHECK(tiler.southWestTile(1).coords == glm::uvec2(1, TestType::value ? 0 : 1)); + CHECK(tiler.northEastTile(1).coords == glm::uvec2(1, 1)); + CHECK(tiler.southWestTile(1).coords == glm::uvec2(1, 1)); - CHECK(tiler.northEastTile(2).coords == glm::uvec2(2, TestType::value ? 0 : 3)); - CHECK(tiler.southWestTile(2).coords == glm::uvec2(2, TestType::value ? 0 : 3)); + CHECK(tiler.northEastTile(2).coords == glm::uvec2(2, 3)); + CHECK(tiler.southWestTile(2).coords == glm::uvec2(2, 3)); { const auto l1_tiles = tiler.generateTiles(1); REQUIRE(l1_tiles.size() == 1); const auto t = l1_tiles.front(); CHECK(t.id.zoom_level == 1); - CHECK(t.id.coords == glm::uvec2(1, TestType::value ? 0 : 1)); + CHECK(t.id.coords == glm::uvec2(1, 1)); CHECK(t.gridSize == 64); CHECK(t.tileSize == 65); @@ -263,7 +259,7 @@ TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::f REQUIRE(l2_tiles.size() == 1); const auto t = l2_tiles.front(); CHECK(t.id.zoom_level == 2); - CHECK(t.id.coords == glm::uvec2(2, TestType::value ? 0 : 3)); + CHECK(t.id.coords == glm::uvec2(2, 3)); CHECK(t.gridSize == 64); CHECK(t.tileSize == 65); @@ -281,7 +277,7 @@ TEST_CASE("ParallelTiler returns tiles for several zoom levels") { const auto dataset = Dataset(ALP_TEST_DATA_DIR "/austria/at_mgi.tif"); const auto grid = ctb::GlobalMercator(256); - const auto tiler = ParallelTiler(grid, dataset.bounds(grid.getSRS()), radix::tile::Border::Yes, radix::tile::Scheme::Tms); + const auto tiler = ParallelTiler(grid, dataset.bounds(grid.getSRS()), radix::tile::Border::Yes); SECTION("generate from 0 to 7") { diff --git a/unittests/tilebuilder/tile_heights_generator.cpp b/unittests/tilebuilder/tile_heights_generator.cpp index 43f74237..89c532ae 100644 --- a/unittests/tilebuilder/tile_heights_generator.cpp +++ b/unittests/tilebuilder/tile_heights_generator.cpp @@ -30,7 +30,7 @@ TEST_CASE("TileHeightsGenerator") constexpr auto file_name = "height_data.atb"; SECTION("mercator") { - const auto generator = TileHeightsGenerator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", ctb::Grid::Srs::SphericalMercator, radix::tile::Scheme::Tms, radix::tile::Border::Yes, base_path / file_name); + const auto generator = TileHeightsGenerator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", ctb::Grid::Srs::SphericalMercator, radix::tile::Border::Yes, base_path / file_name); generator.run(8); const auto heights = TileHeights::read_from(base_path / file_name); @@ -41,7 +41,7 @@ TEST_CASE("TileHeightsGenerator") } { - auto [min, max] = heights.query({ 8, { 138, 166 } }); // part of styria, lower and upper austria (https://www.maptiler.com/google-maps-coordinates-tile-bounds-projection/#8/15.69/47.75) + auto [min, max] = heights.query({ 8, { 138, 89 } }); // part of styria, lower and upper austria (https://www.maptiler.com/google-maps-coordinates-tile-bounds-projection/#8/15.69/47.75) CHECK(min > 300); CHECK(min < 400); CHECK(max > 1500); @@ -50,7 +50,7 @@ TEST_CASE("TileHeightsGenerator") } SECTION("geodetic") { - const auto generator = TileHeightsGenerator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", ctb::Grid::Srs::WGS84, radix::tile::Scheme::Tms, radix::tile::Border::Yes, base_path / file_name); + const auto generator = TileHeightsGenerator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", ctb::Grid::Srs::WGS84, radix::tile::Border::Yes, base_path / file_name); generator.run(8); const auto heights = TileHeights::read_from(base_path / file_name); @@ -61,7 +61,7 @@ TEST_CASE("TileHeightsGenerator") } { - auto [min, max] = heights.query({ 8, { 270, 194 } }); // can't check the address easily, because there is no web service showing geodetic tile names. + auto [min, max] = heights.query({ 8, { 270, 61 } }); // can't check the address easily, because there is no web service showing geodetic tile names. CHECK(min > 500); CHECK(min < 700); CHECK(max > 3600); diff --git a/unittests/tilebuilder/top_down_tiler.cpp b/unittests/tilebuilder/top_down_tiler.cpp index cd1e33b5..61645361 100644 --- a/unittests/tilebuilder/top_down_tiler.cpp +++ b/unittests/tilebuilder/top_down_tiler.cpp @@ -21,7 +21,7 @@ #include "TopDownTiler.h" #include "ctb/GlobalGeodetic.hpp" #include "ctb/GlobalMercator.hpp" -#include +#include #include using namespace radix; @@ -49,18 +49,16 @@ void compare_tile_lists(const std::vector& a_tiles, std } -TEMPLATE_TEST_CASE("BottomUpTiler, using tms scheme", "", std::true_type, std::false_type) +TEST_CASE("TopDownTiler") { - const auto scheme = TestType::value ? radix::tile::Scheme::Tms : radix::tile::Scheme::SlippyMap; - SECTION("mercator / level 0 all") { const auto grid = ctb::GlobalMercator(); - const auto tiler = TopDownTiler(grid, grid.getExtent(), radix::tile::Border::No, scheme); + const auto tiler = TopDownTiler(grid, grid.getExtent(), radix::tile::Border::No); - const auto tiles = tiler.generateTiles({0, { 0, 0 }, scheme}); + const auto tiles = tiler.generateTiles({0, { 0, 0 }}); REQUIRE(tiles.size() == 4); - const auto parallel_tiler = ParallelTiler(grid, grid.getExtent(), radix::tile::Border::No, scheme); + const auto parallel_tiler = ParallelTiler(grid, grid.getExtent(), radix::tile::Border::No); compare_tile_lists(tiles, parallel_tiler.generateTiles(1)); } @@ -69,22 +67,22 @@ TEMPLATE_TEST_CASE("BottomUpTiler, using tms scheme", "", std::true_type, std::f const auto grid = ctb::GlobalMercator(); auto dataset = Dataset::open_shared_raster(ALP_TEST_DATA_DIR "/austria/at_mgi.tif").value(); const auto bounds = dataset->bounds(grid.getSRS()); - const auto tiler = TopDownTiler(grid, bounds, radix::tile::Border::No, scheme); + const auto tiler = TopDownTiler(grid, bounds, radix::tile::Border::No); - const auto tiles = tiler.generateTiles({0, { 0, 0 }, scheme}); + const auto tiles = tiler.generateTiles({0, { 0, 0 }}); REQUIRE(tiles.size() == 1); - const auto parallel_tiler = ParallelTiler(grid, bounds, radix::tile::Border::No, scheme); + const auto parallel_tiler = ParallelTiler(grid, bounds, radix::tile::Border::No); compare_tile_lists(tiles, parallel_tiler.generateTiles(1)); } SECTION("geodetic / level 0 east half") { const auto grid = ctb::GlobalGeodetic(); - const auto tiler = TopDownTiler(grid, grid.getExtent(), radix::tile::Border::No, scheme); + const auto tiler = TopDownTiler(grid, grid.getExtent(), radix::tile::Border::No); - const auto tiles = tiler.generateTiles({0, { 1, 0 }, scheme}); + const auto tiles = tiler.generateTiles({0, { 1, 0 }}); REQUIRE(tiles.size() == 4); - const auto parallel_tiler = ParallelTiler(grid, {{0, -90}, {180, 90}}, radix::tile::Border::No, scheme); + const auto parallel_tiler = ParallelTiler(grid, {{0, -90}, {180, 90}}, radix::tile::Border::No); compare_tile_lists(tiles, parallel_tiler.generateTiles(1)); } @@ -93,11 +91,11 @@ TEMPLATE_TEST_CASE("BottomUpTiler, using tms scheme", "", std::true_type, std::f const auto grid = ctb::GlobalGeodetic(); auto dataset = Dataset::open_shared_raster(ALP_TEST_DATA_DIR "/austria/at_mgi.tif").value(); const auto bounds = dataset->bounds(grid.getSRS()); - const auto tiler = TopDownTiler(grid, bounds, radix::tile::Border::No, scheme); + const auto tiler = TopDownTiler(grid, bounds, radix::tile::Border::No); - const auto tiles = tiler.generateTiles({0, { 1, 0 }, scheme}); + const auto tiles = tiler.generateTiles({0, { 1, 0 }}); REQUIRE(tiles.size() == 1); - const auto parallel_tiler = ParallelTiler(grid, bounds, radix::tile::Border::No, scheme); + const auto parallel_tiler = ParallelTiler(grid, bounds, radix::tile::Border::No); compare_tile_lists(tiles, parallel_tiler.generateTiles(1)); } }