From 7498b6ae025fdb7389632b2e64d1d6a7e8448910 Mon Sep 17 00:00:00 2001 From: kshcherban Date: Sat, 5 Sep 2026 15:50:06 +0200 Subject: [PATCH 1/4] Improve s3 storage, update sdk and tokio, fix thread crashes --- Cargo.lock | 615 ++-- Dockerfile | 2 +- .../20260904120000_s3_region_strings.down.sql | 44 + .../20260904120000_s3_region_strings.up.sql | 44 + crates/macros/src/dyn_repository_handler.rs | 6 +- crates/storage/src/dyn_storage.rs | 9 +- crates/storage/src/s3/mod.rs | 1933 +++++++++---- crates/storage/src/s3/regions.rs | 147 +- crates/storage/src/s3/tags.rs | 2 - crates/storage/src/s3/tests.rs | 2500 +++++++++++++++-- docs/docs/sysAdmin/s3.md | 58 +- pkgly/src/app/api/storage/s3.rs | 65 +- pkgly/src/main.rs | 4 + pkgly/src/repository/docker/proxy.rs | 16 +- pkgly/src/repository/docker/proxy/tests.rs | 46 + .../nr/storage/s3/S3StorageConfig.vue | 32 +- .../s3/__tests__/S3StorageConfig.spec.ts | 89 +- tests/docker/config/pkgly.test.toml | 4 +- tests/docker/docker-compose.test.yml | 34 + tests/docker/seed-data.sql | 31 + tests/integration/test_s3.sh | 175 ++ tests/run_integration_tests.sh | 80 +- 22 files changed, 4721 insertions(+), 1215 deletions(-) create mode 100644 crates/core/migrations/20260904120000_s3_region_strings.down.sql create mode 100644 crates/core/migrations/20260904120000_s3_region_strings.up.sql delete mode 100644 crates/storage/src/s3/tags.rs create mode 100755 tests/integration/test_s3.sh diff --git a/Cargo.lock b/Cargo.lock index d871408..4bddccf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -132,6 +132,15 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] + [[package]] name = "argon2" version = "0.5.3" @@ -163,7 +172,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -198,7 +207,7 @@ checksum = "ffdcb70bdbc4d478427380519163274ac86e52916e10f0a8889adf0f96d3fee7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -209,9 +218,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-config" -version = "1.8.10" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1856b1b48b65f71a4dd940b1c0931f9a7b646d4a924b9828ffefc1454714668a" +checksum = "b8d7b388a9fc3a6db15a5ec778c38b354eff1364882c94d08e0252f7a47dcaa4" dependencies = [ "aws-credential-types", "aws-runtime", @@ -221,6 +230,7 @@ dependencies = [ "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "aws-types", "bytes", @@ -234,9 +244,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.9" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86590e57ea40121d47d3f2e131bfd873dea15d78dc2f4604f4734537ad9e56c4" +checksum = "e93964ffdaf57857f544be3666a5f57570bb699e934700f11b49708f61bb556e" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -270,9 +280,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.5.14" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fe0fd441565b0b318c76e7206c8d1d0b0166b3e986cf30e890b61feb6192045" +checksum = "ef47857a1d4488b528f4a5d5715fa7c3300820897824152234d3fa22b1426657" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -284,9 +294,12 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", + "bytes-utils", "fastrand", "http 0.2.12", + "http 1.3.1", "http-body 0.4.6", + "http-body 1.0.1", "percent-encoding", "pin-project-lite", "tracing", @@ -295,10 +308,11 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.112.0" +version = "1.145.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eee73a27721035c46da0572b390a69fbdb333d0177c24f3d8f7ff952eeb96690" +checksum = "f0e6320417a37c8a62f78b443d0b4cf628b57cd340a09b0eb56173d47cc94e93" dependencies = [ + "arc-swap", "aws-credential-types", "aws-runtime", "aws-sigv4", @@ -307,54 +321,60 @@ dependencies = [ "aws-smithy-eventstream", "aws-smithy-http", "aws-smithy-json", + "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "aws-smithy-xml", "aws-types", "bytes", "fastrand", "hex", - "hmac", + "hmac 0.13.0", "http 0.2.12", "http 1.3.1", - "http-body 0.4.6", - "lru 0.12.5", + "http-body 1.0.1", + "lru 0.18.4", "percent-encoding", "regex-lite", - "sha2 0.10.9", + "sha2 0.11.0", "tracing", "url", ] [[package]] name = "aws-sdk-sts" -version = "1.92.0" +version = "1.114.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0c7808adcff8333eaa76a849e6de926c6ac1a1268b9fd6afe32de9c29ef29d2" +checksum = "ef45745026107ec30c4ef86bd8ae4b002e7e5f6a86e4225240bdf6b06a0b944a" dependencies = [ + "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", "aws-smithy-json", + "aws-smithy-observability", "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "aws-smithy-xml", "aws-types", "fastrand", "http 0.2.12", + "http 1.3.1", "regex-lite", "tracing", ] [[package]] name = "aws-sigv4" -version = "1.3.6" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c35452ec3f001e1f2f6db107b6373f1f48f05ec63ba2c5c9fa91f07dad32af11" +checksum = "723c2234ad7511ceef63eab016b7ba6ff7c55590fefb96fa8467af014a07309f" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -362,16 +382,15 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "bytes", - "crypto-bigint 0.5.5", + "crypto-bigint", "form_urlencoded", "hex", - "hmac", + "hmac 0.13.0", "http 0.2.12", "http 1.3.1", "p256", "percent-encoding", - "ring", - "sha2 0.10.9", + "sha2 0.11.0", "subtle", "time", "tracing", @@ -380,9 +399,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.6" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "127fcfad33b7dfc531141fda7e1c402ac65f88aca5511a4d31e2e3d2cd01ce9c" +checksum = "f02e407fb3b54891734224b9ffac8a71fdd35f542500fa1af95754a6b2beb316" dependencies = [ "futures-util", "pin-project-lite", @@ -391,29 +410,30 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.63.11" +version = "0.65.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95bd108f7b3563598e4dc7b62e1388c9982324a2abd622442167012690184591" +checksum = "b67ecd999972b58e67cab052f5129906c08c25883bd0788ceefc55ef97d61307" dependencies = [ "aws-smithy-http", "aws-smithy-types", "bytes", "crc-fast", "hex", - "http 0.2.12", - "http-body 0.4.6", - "md-5", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "md-5 0.11.0", "pin-project-lite", - "sha1", - "sha2 0.10.9", + "sha1 0.11.0", + "sha2 0.11.0", "tracing", ] [[package]] name = "aws-smithy-eventstream" -version = "0.60.13" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e29a304f8319781a39808847efb39561351b1bb76e933da7aa90232673638658" +checksum = "6de526c7b567420a31bc283657a7921b45c4cafe0827fdf2490713dcc770c28f" dependencies = [ "aws-smithy-types", "bytes", @@ -422,9 +442,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.62.5" +version = "0.64.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445d5d720c99eed0b4aa674ed00d835d9b1427dd73e04adaf2f94c6b2d6f9fca" +checksum = "37843d9add67c3aff5856f409c6dc315d3cdff60f9c0cb5b670dab1e9920306d" dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", @@ -433,9 +453,9 @@ dependencies = [ "bytes-utils", "futures-core", "futures-util", - "http 0.2.12", "http 1.3.1", - "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", "percent-encoding", "pin-project-lite", "pin-utils", @@ -444,9 +464,9 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.1.4" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "623254723e8dfd535f566ee7b2381645f8981da086b5c4aa26c0c41582bb1d2c" +checksum = "ebfd138fac0337cee7516c352757ea73b9f2266e57d0bcb5bc70e9547e45aef1" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -464,7 +484,7 @@ dependencies = [ "pin-project-lite", "rustls 0.21.12", "rustls 0.23.35", - "rustls-native-certs 0.8.2", + "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls 0.26.4", @@ -474,43 +494,49 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.61.7" +version = "0.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2db31f727935fc63c6eeae8b37b438847639ec330a9161ece694efba257e0c54" +checksum = "3dc65a121adb4b33729919fcfa14fa36fb33c1555a8f06bb0e2188dbfdc1d9ef" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", ] [[package]] name = "aws-smithy-observability" -version = "0.1.4" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d1881b1ea6d313f9890710d65c158bdab6fb08c91ea825f74c1c8c357baf4cc" +checksum = "8e86338c869539a581bf161247762a6e87f92c5c075060057b5ed6d06632ed0c" dependencies = [ "aws-smithy-runtime-api", ] [[package]] name = "aws-smithy-query" -version = "0.60.8" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d28a63441360c477465f80c7abac3b9c4d075ca638f982e605b7dc2a2c7156c9" +checksum = "512346c7212ab7436df2d77a16d976a468ae44a418835511d2a69269810aaf62" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", + "aws-smithy-xml", "urlencoding", ] [[package]] name = "aws-smithy-runtime" -version = "1.9.4" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bbe9d018d646b96c7be063dd07987849862b0e6d07c778aad7d93d1be6c1ef0" +checksum = "b82e438d30e02a825d363bd639a9efaed68a8089d86101054b0081e7e0d3e606" dependencies = [ "aws-smithy-async", "aws-smithy-http", "aws-smithy-http-client", "aws-smithy-observability", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "bytes", "fastrand", @@ -518,6 +544,7 @@ dependencies = [ "http 1.3.1", "http-body 0.4.6", "http-body 1.0.1", + "http-body-util", "pin-project-lite", "pin-utils", "tokio", @@ -526,11 +553,12 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.9.2" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec7204f9fd94749a7c53b26da1b961b4ac36bf070ef1e0b94bb09f79d4f6c193" +checksum = "9c054752dd9e4dc73d0b75748c99ac2d0feafbf2f25c7b0516f03a3534161223" dependencies = [ "aws-smithy-async", + "aws-smithy-runtime-api-macros", "aws-smithy-types", "bytes", "http 0.2.12", @@ -541,11 +569,33 @@ dependencies = [ "zeroize", ] +[[package]] +name = "aws-smithy-runtime-api-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "aws-smithy-schema" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56e0a4e53127a632224e43633b0fe045fa9e1e3cfc68b9830f1115e103f910" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 1.3.1", +] + [[package]] name = "aws-smithy-types" -version = "1.3.4" +version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25f535879a207fce0db74b679cfc3e91a3159c8144d717d55f5832aea9eef46e" +checksum = "8f94d16e797ec62cd999fc9d5942b48fa7050c3093ddadff48e4d7528d16fcb9" dependencies = [ "base64-simd", "bytes", @@ -569,22 +619,26 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.12" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab77cdd036b11056d2a30a7af7b775789fb024bf216acc13884c6c97752ae56" +checksum = "ce84f71c72fee2cbbadde6e7d082f5fb466e3a84733855295fa7aafd1b31b7d8" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", "xmlparser", ] [[package]] name = "aws-types" -version = "1.3.10" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d79fb68e3d7fe5d4833ea34dc87d2e97d26d3086cb3da660bb6b1f76d98680b6" +checksum = "209f3a6d82a6e9e5f94abbed94c7a26e1c052341002bf57a5fb5481f625896fc" dependencies = [ "aws-credential-types", "aws-smithy-async", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "rustc_version", "tracing", @@ -617,7 +671,7 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sha1", + "sha1 0.10.6", "sync_wrapper", "tokio", "tokio-tungstenite", @@ -680,14 +734,14 @@ checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] name = "base16ct" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" [[package]] name = "base64" @@ -695,12 +749,6 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - [[package]] name = "base64" version = "0.22.1" @@ -732,7 +780,7 @@ dependencies = [ "bitflags 2.10.0", "cexpr", "clang-sys", - "itertools 0.13.0", + "itertools 0.10.5", "log", "prettyplease", "proc-macro2", @@ -740,7 +788,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -820,9 +868,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "bytes-utils" @@ -1014,7 +1062,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -1032,6 +1080,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "cobs" version = "0.3.0" @@ -1160,15 +1214,12 @@ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crc-fast" -version = "1.6.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ddc2d09feefeee8bd78101665bd8645637828fa9317f9f292496dbbd8c65ff3" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" dependencies = [ - "crc", "digest 0.10.7", - "rand 0.9.2", - "regex", - "rustversion", + "spin 0.10.1", ] [[package]] @@ -1307,26 +1358,16 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" -[[package]] -name = "crypto-bigint" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" -dependencies = [ - "generic-array", - "rand_core 0.6.4", - "subtle", - "zeroize", -] - [[package]] name = "crypto-bigint" version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ + "generic-array", "rand_core 0.6.4", "subtle", + "zeroize", ] [[package]] @@ -1348,6 +1389,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "current_semver" version = "0.1.1" @@ -1356,7 +1406,7 @@ checksum = "61c60526ae35a7665328ec3de4f4bef07ef9630f824d73092d6f863c3ab049fe" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -1404,7 +1454,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -1426,7 +1476,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -1448,16 +1498,6 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" -[[package]] -name = "der" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" -dependencies = [ - "const-oid 0.9.6", - "zeroize", -] - [[package]] name = "der" version = "0.7.10" @@ -1486,7 +1526,7 @@ checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -1507,7 +1547,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -1517,7 +1557,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -1537,7 +1577,7 @@ checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", "unicode-xid", ] @@ -1568,6 +1608,7 @@ dependencies = [ "block-buffer 0.12.0", "const-oid 0.10.2", "crypto-common 0.2.1", + "ctutils", ] [[package]] @@ -1590,7 +1631,7 @@ source = "git+https://github.com/kshcherban/digestible.git#3a32cbac4ddbb25315da7 dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -1611,7 +1652,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -1622,7 +1663,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -1651,14 +1692,16 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "ecdsa" -version = "0.14.8" +version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ - "der 0.6.1", + "der", + "digest 0.10.7", "elliptic-curve", "rfc6979", - "signature 1.6.4", + "signature", + "spki", ] [[package]] @@ -1685,18 +1728,18 @@ dependencies = [ [[package]] name = "elliptic-curve" -version = "0.12.3" +version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ "base16ct", - "crypto-bigint 0.4.9", - "der 0.6.1", + "crypto-bigint", "digest 0.10.7", "ff", "generic-array", "group", - "pkcs8 0.9.0", + "pem-rfc7468", + "pkcs8", "rand_core 0.6.4", "sec1", "subtle", @@ -1753,7 +1796,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -1796,9 +1839,9 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "ff" -version = "0.12.1" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" dependencies = [ "rand_core 0.6.4", "subtle", @@ -1967,7 +2010,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -2026,6 +2069,7 @@ checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", + "zeroize", ] [[package]] @@ -2076,9 +2120,9 @@ dependencies = [ [[package]] name = "group" -version = "0.12.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ "ff", "rand_core 0.6.4", @@ -2198,6 +2242,17 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + [[package]] name = "hashlink" version = "0.9.1" @@ -2228,7 +2283,7 @@ dependencies = [ "http 1.3.1", "httpdate", "mime", - "sha1", + "sha1 0.10.6", ] [[package]] @@ -2278,7 +2333,7 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "hmac", + "hmac 0.12.1", ] [[package]] @@ -2290,6 +2345,15 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.2", +] + [[package]] name = "home" version = "0.5.12" @@ -2434,7 +2498,6 @@ dependencies = [ "hyper 0.14.32", "log", "rustls 0.21.12", - "rustls-native-certs 0.6.3", "tokio", "tokio-rustls 0.24.1", ] @@ -2449,7 +2512,7 @@ dependencies = [ "hyper 1.8.1", "hyper-util", "rustls 0.23.35", - "rustls-native-certs 0.8.2", + "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls 0.26.4", @@ -2700,7 +2763,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2718,15 +2781,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.14.0" @@ -2738,9 +2792,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.15" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jobserver" @@ -2775,7 +2829,7 @@ dependencies = [ "pem", "serde", "serde_json", - "signature 2.2.0", + "signature", "simple_asn1", ] @@ -2906,20 +2960,20 @@ checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" [[package]] name = "lru" -version = "0.12.5" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +checksum = "96051b46fc183dc9cd4a223960ef37b9af631b55191852a8274bfef064cda20f" dependencies = [ - "hashbrown 0.15.5", + "hashbrown 0.16.0", ] [[package]] name = "lru" -version = "0.16.2" +version = "0.18.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96051b46fc183dc9cd4a223960ef37b9af631b55191852a8274bfef064cda20f" +checksum = "ff9840bcc50b71349309900da0ce7279aa336ae71d73250b07998932c7d97c25" dependencies = [ - "hashbrown 0.16.0", + "hashbrown 0.17.1", ] [[package]] @@ -2981,6 +3035,16 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.2", +] + [[package]] name = "memchr" version = "2.7.6" @@ -3075,7 +3139,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -3220,7 +3284,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -3250,7 +3314,7 @@ dependencies = [ "hyper-util", "ipnet", "lru 0.16.2", - "md-5", + "md-5 0.10.6", "mime", "mime_guess", "nr-core", @@ -3259,7 +3323,7 @@ dependencies = [ "postcard", "serde", "serde_json", - "sha1", + "sha1 0.10.6", "sha2 0.10.9", "sha3", "strum", @@ -3294,7 +3358,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3533,12 +3597,13 @@ checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "p256" -version = "0.11.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" dependencies = [ "ecdsa", "elliptic-curve", + "primeorder", "sha2 0.10.9", ] @@ -3637,7 +3702,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -3683,7 +3748,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -3703,7 +3768,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -3724,19 +3789,9 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der 0.7.10", - "pkcs8 0.10.2", - "spki 0.7.3", -] - -[[package]] -name = "pkcs8" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" -dependencies = [ - "der 0.6.1", - "spki 0.6.0", + "der", + "pkcs8", + "spki", ] [[package]] @@ -3745,8 +3800,8 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der 0.7.10", - "spki 0.7.3", + "der", + "spki", ] [[package]] @@ -3796,7 +3851,7 @@ dependencies = [ "jsonwebtoken", "lettre", "maven-rs", - "md-5", + "md-5 0.10.6", "mime", "mime_guess", "mockall", @@ -3820,7 +3875,7 @@ dependencies = [ "rsa", "rust-embed", "rustls 0.23.35", - "rustls-pemfile 2.2.0", + "rustls-pemfile", "schemars", "semver", "serde", @@ -3829,7 +3884,7 @@ dependencies = [ "serde_path_to_error", "serde_urlencoded", "serde_yaml", - "sha1", + "sha1 0.10.6", "sha2 0.10.9", "sha2 0.11.0", "sha3", @@ -3994,14 +4049,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.110", + "syn 2.0.119", +] + +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", ] [[package]] name = "proc-macro2" -version = "1.0.103" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -4026,7 +4090,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -4101,14 +4165,14 @@ dependencies = [ "once_cell", "socket2 0.6.1", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] name = "quote" -version = "1.0.42" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -4250,7 +4314,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -4333,13 +4397,12 @@ dependencies = [ [[package]] name = "rfc6979" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" dependencies = [ - "crypto-bigint 0.4.9", - "hmac", - "zeroize", + "hmac 0.12.1", + "subtle", ] [[package]] @@ -4369,7 +4432,7 @@ checksum = "d4322a2a4e8cf30771dd9f27f7f37ca9ac8fe812dddd811096a98483080dabe6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -4398,10 +4461,10 @@ dependencies = [ "num-integer", "num-traits", "pkcs1", - "pkcs8 0.10.2", + "pkcs8", "rand_core 0.6.4", - "signature 2.2.0", - "spki 0.7.3", + "signature", + "spki", "subtle", "zeroize", ] @@ -4427,7 +4490,7 @@ dependencies = [ "quote", "rust-embed-utils", "shellexpand", - "syn 2.0.110", + "syn 2.0.119", "walkdir", ] @@ -4467,7 +4530,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4498,18 +4561,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "rustls-native-certs" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" -dependencies = [ - "openssl-probe", - "rustls-pemfile 1.0.4", - "schannel", - "security-framework 2.11.1", -] - [[package]] name = "rustls-native-certs" version = "0.8.2" @@ -4519,16 +4570,7 @@ dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework 3.5.1", -] - -[[package]] -name = "rustls-pemfile" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" -dependencies = [ - "base64 0.21.7", + "security-framework", ] [[package]] @@ -4580,9 +4622,9 @@ checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "ryu" -version = "1.0.20" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "same-file" @@ -4622,7 +4664,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -4643,31 +4685,18 @@ dependencies = [ [[package]] name = "sec1" -version = "0.3.0" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct", - "der 0.6.1", + "der", "generic-array", - "pkcs8 0.9.0", + "pkcs8", "subtle", "zeroize", ] -[[package]] -name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags 2.10.0", - "core-foundation 0.9.4", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - [[package]] name = "security-framework" version = "3.5.1" @@ -4738,7 +4767,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -4749,7 +4778,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -4843,6 +4872,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.2", +] + [[package]] name = "sha1_smol" version = "1.0.1" @@ -4935,16 +4975,6 @@ dependencies = [ "libc", ] -[[package]] -name = "signature" -version = "1.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" -dependencies = [ - "digest 0.10.7", - "rand_core 0.6.4", -] - [[package]] name = "signature" version = "2.2.0" @@ -5036,14 +5066,10 @@ dependencies = [ ] [[package]] -name = "spki" -version = "0.6.0" +name = "spin" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" -dependencies = [ - "base64ct", - "der 0.6.1", -] +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" [[package]] name = "spki" @@ -5052,7 +5078,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der 0.7.10", + "der", ] [[package]] @@ -5126,7 +5152,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -5149,7 +5175,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 2.0.110", + "syn 2.0.119", "tokio", "url", ] @@ -5177,17 +5203,17 @@ dependencies = [ "generic-array", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "itoa", "log", - "md-5", + "md-5 0.10.6", "memchr", "once_cell", "percent-encoding", "rand 0.8.5", "rsa", "serde", - "sha1", + "sha1 0.10.6", "sha2 0.10.9", "smallvec", "sqlx-core", @@ -5217,11 +5243,11 @@ dependencies = [ "futures-util", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "home", "itoa", "log", - "md-5", + "md-5 0.10.6", "memchr", "once_cell", "rand 0.8.5", @@ -5329,7 +5355,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -5351,9 +5377,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.110" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a99801b5bd34ede4cf3fc688c5919368fea4e4814a4664359503e6015b280aea" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" dependencies = [ "proc-macro2", "quote", @@ -5377,7 +5403,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -5442,7 +5468,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5460,7 +5486,7 @@ dependencies = [ "bollard-stubs", "futures", "hex", - "hmac", + "hmac 0.12.1", "log", "rand 0.8.5", "serde", @@ -5503,7 +5529,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -5514,7 +5540,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -5616,9 +5642,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.48.0" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" dependencies = [ "bytes", "libc", @@ -5638,7 +5664,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -5686,13 +5712,14 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.17" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", "futures-sink", + "libc", "pin-project-lite", "tokio", ] @@ -5813,9 +5840,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.41" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ "log", "pin-project-lite", @@ -5837,20 +5864,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] name = "tracing-core" -version = "0.1.34" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", "valuable", @@ -5934,7 +5961,7 @@ dependencies = [ "httparse", "log", "rand 0.9.2", - "sha1", + "sha1 0.10.6", "thiserror 2.0.17", "utf-8", ] @@ -6096,7 +6123,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.110", + "syn 2.0.119", "url", "uuid", ] @@ -6235,7 +6262,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", "wasm-bindgen-shared", ] @@ -6269,7 +6296,7 @@ checksum = "085b2df989e1e6f9620c1311df6c996e83fe16f57792b272ce1e024ac16a90f1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -6355,7 +6382,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -6404,7 +6431,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -6415,7 +6442,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -6764,7 +6791,7 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", "synstructure", ] @@ -6785,7 +6812,7 @@ checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -6805,7 +6832,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", "synstructure", ] @@ -6845,7 +6872,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] diff --git a/Dockerfile b/Dockerfile index 3c823b8..0118794 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ RUN --mount=type=cache,target=/root/.npm npm run build ############################ # Rust build stage ############################ -FROM rust:1.95.0 AS rust-base +FROM rust:1.98.0 AS rust-base WORKDIR /app RUN apt-get update \ diff --git a/crates/core/migrations/20260904120000_s3_region_strings.down.sql b/crates/core/migrations/20260904120000_s3_region_strings.down.sql new file mode 100644 index 0000000..c24e4d2 --- /dev/null +++ b/crates/core/migrations/20260904120000_s3_region_strings.down.sql @@ -0,0 +1,44 @@ +-- ABOUTME: Reverses known S3 raw region identifiers for legacy deployments. +-- ABOUTME: Leaves newer identifiers unchanged because enum-only binaries cannot represent them. +UPDATE storages +SET config = jsonb_set( + config, + '{settings,region}', + to_jsonb(CASE config #>> '{settings,region}' + WHEN 'us-east-1' THEN 'UsEast1' + WHEN 'us-east-2' THEN 'UsEast2' + WHEN 'us-west-1' THEN 'UsWest1' + WHEN 'us-west-2' THEN 'UsWest2' + WHEN 'ca-central-1' THEN 'CaCentral1' + WHEN 'af-south-1' THEN 'AfSouth1' + WHEN 'ap-east-1' THEN 'ApEast1' + WHEN 'ap-south-1' THEN 'ApSouth1' + WHEN 'ap-northeast-1' THEN 'ApNortheast1' + WHEN 'ap-northeast-2' THEN 'ApNortheast2' + WHEN 'ap-northeast-3' THEN 'ApNortheast3' + WHEN 'ap-southeast-1' THEN 'ApSoutheast1' + WHEN 'ap-southeast-2' THEN 'ApSoutheast2' + WHEN 'cn-north-1' THEN 'CnNorth1' + WHEN 'cn-northwest-1' THEN 'CnNorthwest1' + WHEN 'eu-north-1' THEN 'EuNorth1' + WHEN 'eu-central-1' THEN 'EuCentral1' + WHEN 'eu-central-2' THEN 'EuCentral2' + WHEN 'eu-west-1' THEN 'EuWest1' + WHEN 'eu-west-2' THEN 'EuWest2' + WHEN 'eu-west-3' THEN 'EuWest3' + WHEN 'il-central-1' THEN 'IlCentral1' + WHEN 'me-south-1' THEN 'MeSouth1' + WHEN 'sa-east-1' THEN 'SaEast1' + ELSE config #>> '{settings,region}' + END), + true +) +WHERE lower(storage_type) = 's3' + AND config #>> '{type}' = 'S3' + AND config #>> '{settings,region}' IN ( + 'us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'ca-central-1', 'af-south-1', + 'ap-east-1', 'ap-south-1', 'ap-northeast-1', 'ap-northeast-2', 'ap-northeast-3', + 'ap-southeast-1', 'ap-southeast-2', 'cn-north-1', 'cn-northwest-1', 'eu-north-1', + 'eu-central-1', 'eu-central-2', 'eu-west-1', 'eu-west-2', 'eu-west-3', 'il-central-1', + 'me-south-1', 'sa-east-1' + ); diff --git a/crates/core/migrations/20260904120000_s3_region_strings.up.sql b/crates/core/migrations/20260904120000_s3_region_strings.up.sql new file mode 100644 index 0000000..0c10f3b --- /dev/null +++ b/crates/core/migrations/20260904120000_s3_region_strings.up.sql @@ -0,0 +1,44 @@ +-- ABOUTME: Migrates legacy S3 enum region tokens to canonical raw identifiers. +-- ABOUTME: Keeps persisted S3 configuration compatible with string-based regions. +UPDATE storages +SET config = jsonb_set( + config, + '{settings,region}', + to_jsonb(CASE config #>> '{settings,region}' + WHEN 'UsEast1' THEN 'us-east-1' + WHEN 'UsEast2' THEN 'us-east-2' + WHEN 'UsWest1' THEN 'us-west-1' + WHEN 'UsWest2' THEN 'us-west-2' + WHEN 'CaCentral1' THEN 'ca-central-1' + WHEN 'AfSouth1' THEN 'af-south-1' + WHEN 'ApEast1' THEN 'ap-east-1' + WHEN 'ApSouth1' THEN 'ap-south-1' + WHEN 'ApNortheast1' THEN 'ap-northeast-1' + WHEN 'ApNortheast2' THEN 'ap-northeast-2' + WHEN 'ApNortheast3' THEN 'ap-northeast-3' + WHEN 'ApSoutheast1' THEN 'ap-southeast-1' + WHEN 'ApSoutheast2' THEN 'ap-southeast-2' + WHEN 'CnNorth1' THEN 'cn-north-1' + WHEN 'CnNorthwest1' THEN 'cn-northwest-1' + WHEN 'EuNorth1' THEN 'eu-north-1' + WHEN 'EuCentral1' THEN 'eu-central-1' + WHEN 'EuCentral2' THEN 'eu-central-2' + WHEN 'EuWest1' THEN 'eu-west-1' + WHEN 'EuWest2' THEN 'eu-west-2' + WHEN 'EuWest3' THEN 'eu-west-3' + WHEN 'IlCentral1' THEN 'il-central-1' + WHEN 'MeSouth1' THEN 'me-south-1' + WHEN 'SaEast1' THEN 'sa-east-1' + ELSE config #>> '{settings,region}' + END), + true +) +WHERE lower(storage_type) = 's3' + AND config #>> '{type}' = 'S3' + AND config #>> '{settings,region}' IN ( + 'UsEast1', 'UsEast2', 'UsWest1', 'UsWest2', 'CaCentral1', 'AfSouth1', + 'ApEast1', 'ApSouth1', 'ApNortheast1', 'ApNortheast2', 'ApNortheast3', + 'ApSoutheast1', 'ApSoutheast2', 'CnNorth1', 'CnNorthwest1', 'EuNorth1', + 'EuCentral1', 'EuCentral2', 'EuWest1', 'EuWest2', 'EuWest3', 'IlCentral1', + 'MeSouth1', 'SaEast1' + ); diff --git a/crates/macros/src/dyn_repository_handler.rs b/crates/macros/src/dyn_repository_handler.rs index 6b7189c..e2dc059 100644 --- a/crates/macros/src/dyn_repository_handler.rs +++ b/crates/macros/src/dyn_repository_handler.rs @@ -1,3 +1,5 @@ +// ABOUTME: Generates repository dispatch implementations for repository enums. +// ABOUTME: Bounds read-request future sizes by allocating backend handlers on the heap. use proc_macro2::TokenStream; use quote::quote; use syn::{ @@ -169,7 +171,7 @@ pub(crate) fn expand(derive_input: DeriveInput) -> Result { ) -> Result { match self { #( - #ident::#variants(variant) => variant.handle_get(request).await.map_err(Self::Error::from), + #ident::#variants(variant) => Box::pin(variant.handle_get(request)).await.map_err(Self::Error::from), )* } } @@ -220,7 +222,7 @@ pub(crate) fn expand(derive_input: DeriveInput) -> Result { ) -> Result { match self { #( - #ident::#variants(variant) => variant.handle_head(request).await.map_err(Self::Error::from), + #ident::#variants(variant) => Box::pin(variant.handle_head(request)).await.map_err(Self::Error::from), )* } } diff --git a/crates/storage/src/dyn_storage.rs b/crates/storage/src/dyn_storage.rs index 19ec025..930864d 100644 --- a/crates/storage/src/dyn_storage.rs +++ b/crates/storage/src/dyn_storage.rs @@ -1,3 +1,5 @@ +// ABOUTME: Dispatches repository storage operations to local and S3 backends. +// ABOUTME: Keeps backend-specific operations behind the shared storage interface. use nr_core::storage::StoragePath; use uuid::Uuid; @@ -47,8 +49,7 @@ impl Storage for DynStorage { .save_file(repository, file, location) .await .map_err(Into::into), - DynStorage::S3(storage) => storage - .save_file(repository, file, location) + DynStorage::S3(storage) => Box::pin(storage.save_file(repository, file, location)) .await .map_err(Into::into), } @@ -134,8 +135,8 @@ impl Storage for DynStorage { .open_file(repository, location) .await .map_err(Into::into), - DynStorage::S3(storage) => storage - .open_file(repository, location) + // Keep the S3 read state machine out of every caller's future, including local reads. + DynStorage::S3(storage) => Box::pin(storage.open_file(repository, location)) .await .map_err(Into::into), } diff --git a/crates/storage/src/s3/mod.rs b/crates/storage/src/s3/mod.rs index 55e5684..b778a8e 100644 --- a/crates/storage/src/s3/mod.rs +++ b/crates/storage/src/s3/mod.rs @@ -1,8 +1,9 @@ -#![allow(dead_code)] +// ABOUTME: Implements S3-backed repository storage and object metadata handling. +// ABOUTME: Provides guarded object mutation, listing, streaming, and local caching. use std::{ - borrow::Cow, collections::VecDeque, env, + future::Future, io::ErrorKind, net::IpAddr, num::NonZeroUsize, @@ -11,27 +12,26 @@ use std::{ pin::Pin, str::FromStr, sync::{Arc, OnceLock}, + time::{SystemTime, UNIX_EPOCH}, }; use aws_config::BehaviorVersion; use aws_config::sts::AssumeRoleProvider; use aws_credential_types::{Credentials as AwsCredentials, provider::SharedCredentialsProvider}; -use aws_sdk_s3::{ - Client as AwsS3Client, - types::{CommonPrefix, Tag}, -}; +use aws_sdk_s3::{Client as AwsS3Client, types::CommonPrefix}; use aws_smithy_runtime_api::client::dns::{DnsFuture, ResolveDns, ResolveDnsError}; -use aws_smithy_runtime_api::client::result::SdkError; +use aws_smithy_runtime_api::client::{orchestrator::HttpResponse, result::SdkError}; use aws_smithy_types::byte_stream::ByteStream; +use aws_smithy_types::error::metadata::ProvideErrorMetadata; use aws_types::{SdkConfig, region::Region}; -use bytes::Bytes; -use chrono::{FixedOffset, Local}; +use bytes::{Bytes, BytesMut}; +use chrono::{DateTime as ChronoDateTime, FixedOffset, Local, Utc}; use futures::future::BoxFuture; use hex::encode; use lru::LruCache; use mime::Mime; use nr_core::storage::{FileHashes, FileTypeCheck, SerdeMime, StoragePath}; -use regions::{CustomRegion, S3StorageRegion}; +use regions::CustomRegion; use sha2::{Digest, Sha256}; use sysinfo::System; use tokio::{ @@ -44,20 +44,19 @@ use tokio::{ use url::{Host, Url}; pub mod regions; -use serde::{Deserialize, Serialize}; -use tracing::{debug, error, info, instrument, warn}; -use utoipa::ToSchema; -pub mod tags; use ahash::HashSet; use ipnet::IpNet; -use parking_lot::RwLock; +use parking_lot::{Mutex as ParkingMutex, RwLock}; +use serde::{Deserialize, Deserializer, Serialize}; +use tracing::{debug, info, instrument, warn}; +use utoipa::ToSchema; use uuid::Uuid; #[derive(Debug, thiserror::Error)] pub enum S3StorageError { #[error("No Region Provided")] NoRegionSpecified, - #[error("AWS SDK error: {0}")] - AwsSdkError(String), + #[error("AWS SDK error ({kind:?}): {message}")] + AwsSdkError { kind: S3ErrorKind, message: String }, #[error("Bucket Does Not Exist {0}")] BucketDoesNotExist(String), #[error("IO Error: {0}")] @@ -67,21 +66,168 @@ pub enum S3StorageError { #[error(transparent)] InvalidConfigType(#[from] InvalidConfigType), - #[error("Missing Tag: {0}")] - MissingTag(Cow<'static, str>), - #[error(transparent)] PathCollision(#[from] PathCollisionError), #[error("S3 endpoint is blocked by egress policy")] BlockedEndpoint, } + +/// Broadly classifies S3 failures so callers can choose safe retry or conflict behavior. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum S3ErrorKind { + NotFound, + AccessDenied, + Throttled, + Conflict, + Network, + Other, +} + impl S3StorageError { - pub fn static_missing_tag(tag: &'static str) -> Self { - S3StorageError::MissingTag(tag.into()) + /// Returns the S3-specific classification when this is an AWS SDK failure. + pub fn kind(&self) -> Option { + match self { + Self::AwsSdkError { kind, .. } => Some(*kind), + _ => None, + } + } + + /// Returns whether the failure means the requested object or bucket was not found. + pub fn is_not_found(&self) -> bool { + self.kind() == Some(S3ErrorKind::NotFound) + } + + /// Returns whether a conditional mutation failed because the object changed. + pub fn is_conflict(&self) -> bool { + self.kind() == Some(S3ErrorKind::Conflict) + } + + /// Returns whether retrying may succeed without changing the request semantics. + pub fn is_retryable(&self) -> bool { + matches!( + self.kind(), + Some(S3ErrorKind::Network | S3ErrorKind::Throttled) + ) + } + + /// Converts an SDK error into a classified failure while retaining its full display context. + pub fn from_sdk_error(err: SdkError) -> Self + where + E: std::error::Error + std::fmt::Display + ProvideErrorMetadata + 'static, + { + let kind = match &err { + SdkError::TimeoutError(_) | SdkError::DispatchFailure(_) => S3ErrorKind::Network, + SdkError::ServiceError(context) => { + classify_service_error(context.err().code(), context.raw().status().as_u16()) + } + SdkError::ConstructionFailure(_) | SdkError::ResponseError(_) => S3ErrorKind::Other, + _ => S3ErrorKind::Other, + }; + let message = format!( + "{}", + aws_smithy_types::error::display::DisplayErrorContext(&err) + ); + Self::AwsSdkError { kind, message } + } + + fn aws_message(message: impl Into) -> Self { + let message = message.into(); + Self::AwsSdkError { + kind: classify_error_message(&message), + message, + } + } +} + +fn classify_service_error(code: Option<&str>, status: u16) -> S3ErrorKind { + let code = code.unwrap_or_default().to_ascii_lowercase(); + if status == 409 + || status == 412 + || matches!( + code.as_str(), + "preconditionfailed" | "conditionalrequestconflict" + ) + { + return S3ErrorKind::Conflict; + } + if status == 404 || matches!(code.as_str(), "nosuchkey" | "nosuchbucket" | "notfound") { + return S3ErrorKind::NotFound; } - pub fn from_sdk_error(err: impl std::fmt::Display) -> Self { - S3StorageError::AwsSdkError(err.to_string()) + if status == 401 || status == 403 || code == "accessdenied" { + return S3ErrorKind::AccessDenied; } + if status == 429 + || status == 503 + || code.contains("throttl") + || matches!( + code.as_str(), + "slowdown" | "requestlimitexceeded" | "toomanyrequests" | "serviceunavailable" + ) + { + return S3ErrorKind::Throttled; + } + S3ErrorKind::Other +} + +fn classify_error_message(message: &str) -> S3ErrorKind { + let lower = message.to_ascii_lowercase(); + if lower.contains("preconditionfailed") + || lower.contains("precondition failed") + || lower.contains("conditionalrequestconflict") + || contains_status_code(&lower, 409) + || contains_status_code(&lower, 412) + { + return S3ErrorKind::Conflict; + } + if lower.contains("nosuchkey") + || lower.contains("no such key") + || lower.contains("nosuchbucket") + || lower.contains("no such bucket") + || lower.contains("notfound") + || lower.contains("not found") + || contains_status_code(&lower, 404) + { + return S3ErrorKind::NotFound; + } + if lower.contains("accessdenied") + || lower.contains("access denied") + || contains_status_code(&lower, 401) + || contains_status_code(&lower, 403) + { + return S3ErrorKind::AccessDenied; + } + if lower.contains("slowdown") + || lower.contains("throttl") + || lower.contains("requestlimitexceeded") + || lower.contains("toomanyrequests") + || lower.contains("serviceunavailable") + || contains_status_code(&lower, 429) + || contains_status_code(&lower, 503) + { + return S3ErrorKind::Throttled; + } + if lower.contains("timeout") + || lower.contains("timed out") + || lower.contains("deadline") + || lower.contains("dispatch") + || lower.contains("connection") + || lower.contains("network") + { + return S3ErrorKind::Network; + } + S3ErrorKind::Other +} + +fn contains_status_code(message: &str, code: u16) -> bool { + [ + format!("status code: {code}"), + format!("status: {code}"), + format!("status={code}"), + format!("http {code}"), + format!("http status {code}"), + ] + .iter() + .any(|pattern| message.contains(pattern)) } #[derive(Debug, Clone, Default)] @@ -313,23 +459,72 @@ impl AdaptiveBufferConfig { } } +#[derive(Debug, Clone, Copy)] struct MemorySnapshot { total_bytes: u64, available_bytes: u64, } +#[derive(Debug, Default)] +struct MemorySnapshotCache { + captured_at: Option, + snapshot: Option, +} + +impl MemorySnapshotCache { + fn get_or_capture(&mut self, now: Instant, capture: F) -> Option + where + F: FnOnce() -> Option, + { + if let Some(captured_at) = self.captured_at + && now + .checked_duration_since(captured_at) + .is_some_and(|elapsed| elapsed < MEMORY_SNAPSHOT_TTL) + { + return self.snapshot; + } + + let snapshot = capture(); + self.captured_at = Some(now); + self.snapshot = snapshot; + snapshot + } +} + +static MEMORY_SNAPSHOT_CACHE: OnceLock> = OnceLock::new(); + impl MemorySnapshot { + #[cfg(test)] + fn from_values(total_bytes: u64, available_bytes: u64) -> Self { + Self { + total_bytes, + available_bytes: available_bytes.min(total_bytes), + } + } + fn capture() -> Option { + let cache = + MEMORY_SNAPSHOT_CACHE.get_or_init(|| ParkingMutex::new(MemorySnapshotCache::default())); + cache + .lock() + .get_or_capture(Instant::now(), Self::capture_uncached) + } + + fn capture_uncached() -> Option { let mut system = System::new(); system.refresh_memory(); - let total = system.total_memory(); + let host_total = system.total_memory(); + let host_available = system.available_memory(); + let cgroup = system + .cgroup_limits() + .map(|limits| (limits.total_memory, limits.free_memory)); + let (total, available) = effective_memory_limits(host_total, host_available, cgroup); if total == 0 { return None; } - let available = system.available_memory(); Some(Self { - total_bytes: total.saturating_mul(1024), - available_bytes: available.saturating_mul(1024), + total_bytes: total, + available_bytes: available, }) } @@ -342,6 +537,19 @@ impl MemorySnapshot { } } +fn effective_memory_limits( + host_total: u64, + host_available: u64, + cgroup: Option<(u64, u64)>, +) -> (u64, u64) { + match cgroup { + Some((total, available)) if total > 0 && total < host_total => { + (total, available.min(total)) + } + _ => (host_total, host_available.min(host_total)), + } +} + impl Default for S3CacheConfig { fn default() -> Self { Self { @@ -364,7 +572,8 @@ fn default_cache_entry_limit() -> usize { #[derive(Clone, Serialize, Deserialize, PartialEq, ToSchema)] pub struct S3Config { pub bucket_name: String, - pub region: Option, + #[serde(default, deserialize_with = "deserialize_region")] + pub region: Option, /// Custom region takes precedence over the region field #[serde(flatten)] pub custom_region: Option, @@ -378,6 +587,45 @@ pub struct S3Config { pub adaptive_buffer: AdaptiveBufferConfig, } +fn deserialize_region<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let value = Option::::deserialize(deserializer)?; + Ok(value.map(|region| legacy_region_id(®ion).unwrap_or(region))) +} + +fn legacy_region_id(value: &str) -> Option { + let region = match value { + "UsEast1" => "us-east-1", + "UsEast2" => "us-east-2", + "UsWest1" => "us-west-1", + "UsWest2" => "us-west-2", + "CaCentral1" => "ca-central-1", + "AfSouth1" => "af-south-1", + "ApEast1" => "ap-east-1", + "ApSouth1" => "ap-south-1", + "ApNortheast1" => "ap-northeast-1", + "ApNortheast2" => "ap-northeast-2", + "ApNortheast3" => "ap-northeast-3", + "ApSoutheast1" => "ap-southeast-1", + "ApSoutheast2" => "ap-southeast-2", + "CnNorth1" => "cn-north-1", + "CnNorthwest1" => "cn-northwest-1", + "EuNorth1" => "eu-north-1", + "EuCentral1" => "eu-central-1", + "EuCentral2" => "eu-central-2", + "EuWest1" => "eu-west-1", + "EuWest2" => "eu-west-2", + "EuWest3" => "eu-west-3", + "IlCentral1" => "il-central-1", + "MeSouth1" => "me-south-1", + "SaEast1" => "sa-east-1", + _ => return None, + }; + Some(region.to_owned()) +} + impl std::fmt::Debug for S3Config { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("S3Config") @@ -409,8 +657,13 @@ impl S3Config { .unwrap_or_else(|| "custom-endpoint".into()); return Ok(Region::new(name)); } - if let Some(region) = &self.region { - return Ok((*region).into()); + if let Some(region) = self + .region + .as_deref() + .map(str::trim) + .filter(|region| !region.is_empty()) + { + return Ok(Region::new(region.to_owned())); } Err(S3StorageError::NoRegionSpecified) } @@ -423,18 +676,12 @@ impl S3Config { self.cache.enabled && self.cache.max_bytes > 0 } } -#[derive(Debug, Clone)] -pub struct S3MetaTags { - pub name: String, - pub mime_type: Option, - pub is_directory: bool, -} - #[derive(Debug)] pub(super) struct S3DiskCache { dir: PathBuf, max_bytes: u64, state: Mutex, + publish_lock: Mutex<()>, } #[derive(Debug)] @@ -448,13 +695,28 @@ struct CacheState { struct CacheEntry { relative_path: PathBuf, size: u64, + digest: String, content_type: Option, + last_modified: Option>, } #[derive(Debug, Clone)] struct CachedObject { bytes: Bytes, content_type: Option, + last_modified: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct CacheMetadata { + version: u8, + key: String, + relative_path: PathBuf, + size: u64, + digest: String, + content_type: Option, + last_modified: Option>, + cached_at_ms: u128, } #[derive(Debug, Clone)] @@ -518,25 +780,25 @@ impl CacheState { impl S3DiskCache { async fn new(config: &S3CacheConfig, storage_name: &str) -> Result { if config.max_bytes == 0 { - return Err(S3StorageError::AwsSdkError( - "cache max_bytes must be greater than zero".into(), + return Err(S3StorageError::aws_message( + "cache max_bytes must be greater than zero", )); } - let dir = config - .path - .clone() - .unwrap_or_else(|| default_cache_dir(storage_name)); + let dir = resolve_cache_dir(config, storage_name); fs::create_dir_all(&dir).await?; let capacity = NonZeroUsize::new(config.max_entries.max(1)).unwrap_or(NonZeroUsize::MIN); + let entries = Self::recover_entries(&dir, capacity, config.max_bytes).await?; + let current_bytes = entries.iter().map(|(_, entry)| entry.size).sum(); let state = CacheState { - entries: LruCache::new(capacity), - current_bytes: 0, + entries, + current_bytes, failed_deletions: VecDeque::new(), }; Ok(Self { dir, max_bytes: config.max_bytes, state: Mutex::new(state), + publish_lock: Mutex::new(()), }) } @@ -547,65 +809,381 @@ impl S3DiskCache { PathBuf::from(prefix).join(rest) } + fn metadata_filename(relative: &std::path::Path) -> PathBuf { + let Some(file_name) = relative.file_name().and_then(|name| name.to_str()) else { + return relative.with_extension("meta.json"); + }; + let base_name = file_name + .split_once(".gen-") + .map_or(file_name, |(base, _)| base); + relative + .parent() + .map(|parent| parent.join(format!("{base_name}.meta.json"))) + .unwrap_or_else(|| PathBuf::from(format!("{base_name}.meta.json"))) + } + + fn generated_filename(key: &str) -> PathBuf { + let base = Self::hashed_filename(key); + let Some(file_name) = base.file_name().and_then(|name| name.to_str()) else { + return base; + }; + base.parent() + .map(|parent| parent.join(format!("{file_name}.gen-{}", Uuid::new_v4().simple()))) + .unwrap_or_else(|| { + PathBuf::from(format!("{file_name}.gen-{}", Uuid::new_v4().simple())) + }) + } + + fn is_owned_generation(name: &str) -> bool { + let Some((base, generation)) = name.split_once(".gen-") else { + return false; + }; + base.len() == 62 + && base.chars().all(|ch| ch.is_ascii_hexdigit()) + && !generation.is_empty() + && generation.chars().all(|ch| ch.is_ascii_hexdigit()) + } + + fn is_owned_legacy_content(name: &str) -> bool { + name.len() == 62 && name.chars().all(|ch| ch.is_ascii_hexdigit()) + } + + fn is_owned_metadata(name: &str) -> bool { + name.strip_suffix(".meta.json") + .is_some_and(Self::is_owned_legacy_content) + } + + fn is_owned_temp(name: &str) -> bool { + let Some((base, suffix)) = name.split_once(".tmp-") else { + return false; + }; + if suffix.is_empty() || !suffix.chars().all(|ch| ch.is_ascii_hexdigit()) { + return false; + } + Self::is_owned_legacy_content(base) + || base + .strip_suffix(".meta") + .is_some_and(Self::is_owned_legacy_content) + } + + fn is_safe_relative_path(path: &std::path::Path) -> bool { + !path.is_absolute() + && path + .components() + .all(|component| matches!(component, std::path::Component::Normal(_))) + } + + async fn recover_entries( + dir: &std::path::Path, + capacity: NonZeroUsize, + max_bytes: u64, + ) -> Result, S3StorageError> { + let mut recovered = Vec::new(); + let mut owned_prefixes = Vec::new(); + let mut valid_content = HashSet::default(); + let mut valid_metadata = HashSet::default(); + let mut prefixes = fs::read_dir(dir).await?; + while let Some(prefix_entry) = prefixes.next_entry().await? { + let prefix_path = prefix_entry.path(); + let prefix_name = prefix_entry.file_name(); + let prefix_name = prefix_name.to_string_lossy(); + if !prefix_entry.file_type().await?.is_dir() + || prefix_name.len() != 2 + || !prefix_name.chars().all(|ch| ch.is_ascii_hexdigit()) + { + continue; + } + owned_prefixes.push(prefix_path.clone()); + + let mut files = fs::read_dir(&prefix_path).await?; + while let Some(file_entry) = files.next_entry().await? { + let file_name = file_entry.file_name().to_string_lossy().into_owned(); + if !Self::is_owned_metadata(&file_name) { + continue; + } + let metadata_path = file_entry.path(); + if !file_entry.file_type().await?.is_file() { + let _ = fs::remove_file(&metadata_path).await; + continue; + } + let relative_metadata = metadata_path + .strip_prefix(dir) + .map(PathBuf::from) + .map_err(|_| std::io::Error::other("cache metadata escaped root"))?; + let metadata = fs::read(&metadata_path) + .await + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()); + let Some(metadata) = metadata else { + let _ = fs::remove_file(&metadata_path).await; + continue; + }; + let relative_content = metadata.relative_path.clone(); + let expected_content = Self::hashed_filename(&metadata.key); + let expected_metadata = Self::metadata_filename(&expected_content); + let valid_layout = Self::is_safe_relative_path(&relative_content) + && relative_content.parent() == expected_content.parent() + && relative_content + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| { + name.starts_with( + expected_content + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or_default(), + ) && Self::is_owned_generation(name) + }); + if metadata.version != CACHE_FORMAT_VERSION + || relative_metadata != expected_metadata + || !valid_layout + || metadata.size > max_bytes + || metadata.digest.len() != 64 + { + let _ = fs::remove_file(&metadata_path).await; + if valid_layout { + let _ = fs::remove_file(dir.join(&relative_content)).await; + } + continue; + } + + let content_path = dir.join(&relative_content); + let valid = if fs::symlink_metadata(&content_path) + .await + .map(|metadata| metadata.file_type().is_file()) + .unwrap_or(false) + { + match fs::read(&content_path).await { + Ok(bytes) => { + bytes.len() as u64 == metadata.size + && hex::encode(Sha256::digest(&bytes)) == metadata.digest + } + Err(_) => false, + } + } else { + false + }; + if !valid { + let _ = fs::remove_file(&metadata_path).await; + let _ = fs::remove_file(content_path).await; + continue; + } + + valid_content.insert(relative_content.clone()); + valid_metadata.insert(relative_metadata); + recovered.push(( + metadata.cached_at_ms, + metadata.key, + CacheEntry { + relative_path: relative_content, + size: metadata.size, + digest: metadata.digest, + content_type: metadata.content_type, + last_modified: metadata.last_modified, + }, + )); + } + } + + // Remove only artifacts from the recognized cache layout. User files in the configured + // directory, including files inside owned-looking prefixes, remain untouched. + for prefix_path in owned_prefixes { + let mut files = fs::read_dir(&prefix_path).await?; + while let Some(file_entry) = files.next_entry().await? { + let name = file_entry.file_name().to_string_lossy().into_owned(); + let relative = file_entry + .path() + .strip_prefix(dir) + .map(PathBuf::from) + .map_err(|_| std::io::Error::other("cache artifact escaped root"))?; + let recognized = if Self::is_owned_metadata(&name) { + !valid_metadata.contains(&relative) + } else if Self::is_owned_legacy_content(&name) + || Self::is_owned_generation(&name) + || Self::is_owned_temp(&name) + { + !valid_content.contains(&relative) + } else { + false + }; + if recognized { + let _ = fs::remove_file(file_entry.path()).await; + } + } + } + + recovered.sort_by_key(|(cached_at, _, _)| *cached_at); + let mut entries: LruCache = LruCache::new(capacity); + let mut current_bytes = 0u64; + let mut evicted: Vec = Vec::new(); + for (_, key, entry) in recovered { + if let Some(old) = entries.pop(&key) { + current_bytes = current_bytes.saturating_sub(old.size); + evicted.push(old.relative_path); + } + if entries.len() >= capacity.get() + && let Some((_, old)) = entries.pop_lru() + { + current_bytes = current_bytes.saturating_sub(old.size); + evicted.push(old.relative_path); + } + entries.put(key, entry.clone()); + current_bytes = current_bytes.saturating_add(entry.size); + while current_bytes > max_bytes { + if let Some((_, old)) = entries.pop_lru() { + current_bytes = current_bytes.saturating_sub(old.size); + evicted.push(old.relative_path); + } else { + break; + } + } + } + for relative in evicted { + Self::remove_owned_files(dir, &relative).await; + } + Ok(entries) + } + + async fn remove_owned_files(dir: &std::path::Path, relative: &std::path::Path) { + let _ = fs::remove_file(dir.join(relative)).await; + let _ = fs::remove_file(dir.join(Self::metadata_filename(relative))).await; + } + async fn get(&self, key: &str) -> Result, S3StorageError> { self.retry_failed_deletions().await; - let (relative_path, content_type) = { + let entry = { let mut state = self.state.lock().await; match state.entries.get(key) { - Some(entry) => (entry.relative_path.clone(), entry.content_type.clone()), + Some(entry) => entry.clone(), None => return Ok(None), } }; - let path = self.dir.join(relative_path); + let path = self.dir.join(&entry.relative_path); + let is_regular_file = fs::symlink_metadata(&path) + .await + .map(|metadata| metadata.file_type().is_file()) + .unwrap_or(false); + if !is_regular_file { + self.remove_if_matches(key, &entry).await?; + return Ok(None); + } match fs::read(&path).await { - Ok(data) => Ok(Some(CachedObject { - bytes: Bytes::from(data), - content_type, - })), - Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None), + Ok(data) + if data.len() as u64 == entry.size + && hex::encode(Sha256::digest(&data)) == entry.digest => + { + Ok(Some(CachedObject { + bytes: Bytes::from(data), + content_type: entry.content_type, + last_modified: entry.last_modified, + })) + } + Ok(_) => { + self.remove_if_matches(key, &entry).await?; + Ok(None) + } + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + self.remove_if_matches(key, &entry).await?; + Ok(None) + } Err(err) => Err(err.into()), } } + #[cfg(test)] async fn put( &self, key: &str, data: Bytes, content_type: Option<&str>, + ) -> Result<(), S3StorageError> { + self.put_with_metadata(key, data, content_type, None).await + } + + async fn put_with_metadata( + &self, + key: &str, + data: Bytes, + content_type: Option<&str>, + last_modified: Option>, ) -> Result<(), S3StorageError> { self.retry_failed_deletions().await; - let relative = Self::hashed_filename(key); + if data.len() as u64 > self.max_bytes { + self.remove(key).await?; + return Ok(()); + } + let relative = Self::generated_filename(key); let path = self.dir.join(&relative); if let Some(parent) = path.parent() { fs::create_dir_all(parent).await?; } - fs::write(&path, data.as_ref()).await?; - let mut removed = Vec::new(); + let temp_path = path.with_extension(format!("tmp-{}", Uuid::new_v4().simple())); + fs::write(&temp_path, data.as_ref()).await?; + let digest = hex::encode(Sha256::digest(data.as_ref())); + let cached_at_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis(); + let metadata = CacheMetadata { + version: CACHE_FORMAT_VERSION, + key: key.to_owned(), + relative_path: relative.clone(), + size: data.len() as u64, + digest: digest.clone(), + content_type: content_type.map(str::to_owned), + last_modified, + cached_at_ms, + }; + let metadata_relative = Self::metadata_filename(&relative); + let metadata_path = self.dir.join(&metadata_relative); + let metadata_temp = + metadata_path.with_extension(format!("tmp-{}", Uuid::new_v4().simple())); + let metadata_bytes = serde_json::to_vec(&metadata) + .map_err(|error| std::io::Error::other(error.to_string()))?; + fs::write(&metadata_temp, metadata_bytes).await?; + let mut removed_entries = Vec::new(); + let mut superseded_generations = Vec::new(); + let _publication = self.publish_lock.lock().await; + fs::rename(&temp_path, &path).await?; + fs::rename(&metadata_temp, &metadata_path).await?; { let mut state = self.state.lock().await; if let Some(old) = state.entries.pop(key) { state.current_bytes = state.current_bytes.saturating_sub(old.size); - removed.push(old.relative_path); + if old.relative_path != relative { + superseded_generations.push(old.relative_path); + } + } + if state.entries.len() >= state.entries.cap().get() + && let Some((_, evicted)) = state.entries.pop_lru() + { + state.current_bytes = state.current_bytes.saturating_sub(evicted.size); + removed_entries.push(evicted.relative_path); } state.entries.put( key.to_string(), CacheEntry { - relative_path: relative, + relative_path: relative.clone(), size: data.len() as u64, - content_type: content_type.map(|c| c.to_string()), + digest, + content_type: content_type.map(str::to_owned), + last_modified, }, ); state.current_bytes = state.current_bytes.saturating_add(data.len() as u64); while state.current_bytes > self.max_bytes { if let Some((_, evicted)) = state.entries.pop_lru() { state.current_bytes = state.current_bytes.saturating_sub(evicted.size); - removed.push(evicted.relative_path); + removed_entries.push(evicted.relative_path); } else { break; } } } - for rel in removed { + for rel in superseded_generations { + self.delete_content_path(rel, false).await; + } + for rel in removed_entries { self.delete_relative_path(rel).await; } Ok(()) @@ -626,8 +1204,39 @@ impl S3DiskCache { Ok(()) } + async fn remove_if_matches( + &self, + key: &str, + expected: &CacheEntry, + ) -> Result<(), S3StorageError> { + self.retry_failed_deletions().await; + let removed = { + let mut state = self.state.lock().await; + let matches = state.entries.peek(key).is_some_and(|entry| { + entry.relative_path == expected.relative_path && entry.digest == expected.digest + }); + if matches { + state.entries.pop(key).map(|entry| { + state.current_bytes = state.current_bytes.saturating_sub(entry.size); + entry.relative_path + }) + } else { + None + } + }; + if let Some(relative) = removed { + self.delete_relative_path(relative).await; + } + Ok(()) + } + async fn delete_relative_path(&self, relative: PathBuf) { + self.delete_content_path(relative, true).await; + } + + async fn delete_content_path(&self, relative: PathBuf, remove_metadata: bool) { let path = self.dir.join(&relative); + let metadata_path = self.dir.join(Self::metadata_filename(&relative)); match fs::remove_file(&path).await { Ok(_) => { debug!(path = %relative.display(), "Removed cache entry"); @@ -644,6 +1253,9 @@ impl S3DiskCache { self.enqueue_failed_deletion(relative).await; } } + if remove_metadata { + let _ = fs::remove_file(metadata_path).await; + } } async fn enqueue_failed_deletion(&self, relative_path: PathBuf) { @@ -708,13 +1320,185 @@ fn default_cache_dir(storage_name: &str) -> PathBuf { .join("s3-cache") .join(sanitized) } + +fn resolve_cache_dir(config: &S3CacheConfig, storage_name: &str) -> PathBuf { + config + .path + .as_ref() + .filter(|path| !path.as_os_str().is_empty()) + .cloned() + .unwrap_or_else(|| default_cache_dir(storage_name)) +} + #[derive(Debug)] pub struct S3StorageInner { pub config: S3Config, pub storage_config: StorageConfigInner, pub client: AwsS3Client, cache: Option>, + manifest_cache: ParkingMutex, + manifest_load_lock: Mutex<()>, +} + +#[derive(Debug)] +struct ManifestCache { + entries: LruCache, + generation: u64, +} + +#[derive(Debug, Clone)] +struct CachedManifestList { + items: Vec, + expires_at: Instant, +} + +impl ManifestCache { + fn new() -> Self { + Self { + entries: LruCache::new(NonZeroUsize::new(256).unwrap_or(NonZeroUsize::MIN)), + generation: 0, + } + } + + fn get(&mut self, repository: Uuid, now: Instant) -> Option> { + let cached = self.entries.get(&repository)?; + if cached.expires_at <= now { + self.entries.pop(&repository); + return None; + } + Some(cached.items.clone()) + } + + fn insert(&mut self, repository: Uuid, items: Vec, now: Instant) { + self.entries.put( + repository, + CachedManifestList { + items, + expires_at: now + MANIFEST_CACHE_TTL, + }, + ); + } + + fn insert_if_generation( + &mut self, + repository: Uuid, + items: Vec, + now: Instant, + generation: u64, + ) { + if self.generation == generation { + self.insert(repository, items, now); + } + } + + fn generation(&self) -> u64 { + self.generation + } + + fn invalidate(&mut self, repository: Uuid) { + self.generation = self.generation.wrapping_add(1); + self.entries.pop(&repository); + } +} + +const S3_CONNECT_TIMEOUT: Duration = Duration::from_secs(5); +const S3_CONTROL_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(30); +const S3_CONTROL_TIMEOUT: Duration = Duration::from_secs(90); +const S3_COPY_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(5 * 60); +const S3_COPY_TIMEOUT: Duration = Duration::from_secs(20 * 60); +const MULTIPART_COPY_THRESHOLD: u64 = 5 * 1024 * 1024 * 1024; +const MULTIPART_COPY_PART_SIZE: u64 = 64 * 1024 * 1024; + +fn control_timeout_config() -> aws_smithy_types::timeout::TimeoutConfig { + aws_smithy_types::timeout::TimeoutConfig::builder() + .connect_timeout(S3_CONNECT_TIMEOUT) + .read_timeout(S3_CONTROL_ATTEMPT_TIMEOUT) + .operation_attempt_timeout(S3_CONTROL_ATTEMPT_TIMEOUT) + .operation_timeout(S3_CONTROL_TIMEOUT) + .build() +} + +fn copy_timeout_config() -> aws_smithy_types::timeout::TimeoutConfig { + aws_smithy_types::timeout::TimeoutConfig::builder() + .connect_timeout(S3_CONNECT_TIMEOUT) + .read_timeout(S3_CONTROL_ATTEMPT_TIMEOUT) + .operation_attempt_timeout(S3_COPY_ATTEMPT_TIMEOUT) + .operation_timeout(S3_COPY_TIMEOUT) + .build() +} + +fn streaming_timeout_config() -> aws_smithy_types::timeout::TimeoutConfig { + aws_smithy_types::timeout::TimeoutConfig::builder() + .connect_timeout(S3_CONNECT_TIMEOUT) + .read_timeout(S3_CONTROL_ATTEMPT_TIMEOUT) + .disable_operation_attempt_timeout() + .disable_operation_timeout() + .build() +} + +fn timeout_override( + timeout_config: aws_smithy_types::timeout::TimeoutConfig, +) -> aws_sdk_s3::config::Builder { + aws_sdk_s3::config::Builder::new().timeout_config(timeout_config) +} + +async fn with_timeout(timeout_duration: Duration, future: F) -> Result +where + F: Future>>, + E: std::error::Error + std::fmt::Display + ProvideErrorMetadata + 'static, +{ + match tokio::time::timeout(timeout_duration, future).await { + Ok(result) => result.map_err(S3StorageError::from_sdk_error), + Err(_) => Err(S3StorageError::aws_message(format!( + "S3 request timed out after {} seconds", + timeout_duration.as_secs() + ))), + } +} + +async fn next_control_page( + deadline: Instant, + future: F, +) -> Result, S3StorageError> +where + F: Future>>>, + E: std::error::Error + std::fmt::Display + ProvideErrorMetadata + 'static, +{ + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(S3StorageError::aws_message( + "S3 control operation exceeded its deadline", + )); + } + let result = tokio::time::timeout(remaining, future) + .await + .map_err(|_| S3StorageError::aws_message("S3 LIST request timed out"))?; + result + .map(|page| page.map_err(S3StorageError::from_sdk_error)) + .transpose() +} + +fn encode_copy_source_component(value: &str) -> String { + let mut encoded = String::with_capacity(value.len()); + for byte in value.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { + encoded.push(byte as char); + } else { + encoded.push('%'); + encoded.push_str(&format!("{byte:02X}")); + } + } + encoded +} + +fn copy_source(bucket: &str, key: &str) -> String { + format!( + "/{}/{}", + encode_copy_source_component(bucket), + encode_copy_source_component(key) + ) } + impl S3StorageInner { fn bucket(&self) -> &str { &self.config.bucket_name @@ -730,6 +1514,7 @@ impl S3StorageInner { let mut builder = aws_sdk_s3::config::Builder::from(&base_config).force_path_style(config.path_style); + builder = builder.timeout_config(control_timeout_config()); let http_client = aws_smithy_http_client::Builder::new() .tls_provider(aws_smithy_http_client::tls::Provider::rustls( @@ -772,17 +1557,17 @@ impl S3StorageInner { } let client = AwsS3Client::from_conf(builder.build()); - match client - .head_bucket() - .bucket(&config.bucket_name) - .send() - .await + match with_timeout( + S3_CONTROL_TIMEOUT, + client.head_bucket().bucket(&config.bucket_name).send(), + ) + .await { Ok(_) => Ok(client), - Err(SdkError::ServiceError(err)) if err.err().is_not_found() => Err( - S3StorageError::BucketDoesNotExist(config.bucket_name.clone()), - ), - Err(err) => Err(S3StorageError::from_sdk_error(err)), + Err(error) if error.is_not_found() => Err(S3StorageError::BucketDoesNotExist( + config.bucket_name.clone(), + )), + Err(error) => Err(error), } } @@ -834,6 +1619,42 @@ impl S3StorageInner { key.ends_with(".nr-meta") || key.split('/').any(|part| part == ".nr-meta") } + async fn list_directory_entries( + &self, + prefix: &str, + ) -> Result<(Vec, Vec), S3StorageError> { + let mut paginator = self + .aws_client() + .list_objects_v2() + .bucket(self.bucket()) + .prefix(prefix) + .delimiter("/") + .max_keys(1000) + .into_paginator() + .send(); + let deadline = Instant::now() + S3_CONTROL_TIMEOUT; + let mut objects = Vec::new(); + let mut prefixes = Vec::new(); + while let Some(page) = next_control_page(deadline, paginator.next()).await? { + objects.extend(page.contents().iter().filter_map(|object| { + let key = object.key()?.to_owned(); + let size = object.size().unwrap_or_default().max(0) as u64; + Some(DirectoryObject { + key, + size, + last_modified: s3_last_modified(object.last_modified()), + }) + })); + prefixes.extend( + page.common_prefixes() + .iter() + .filter_map(CommonPrefix::prefix) + .map(str::to_owned), + ); + } + Ok((objects, prefixes)) + } + async fn cache_get( &self, repository: &Uuid, @@ -855,6 +1676,7 @@ impl S3StorageInner { location: &StoragePath, data: Bytes, content_type: Option, + last_modified: Option>, ) -> Result<(), S3StorageError> { if !self.should_cache(location) { return Ok(()); @@ -862,7 +1684,9 @@ impl S3StorageInner { if let Some(cache) = &self.cache { let key = self.cache_key(repository, location); let data_len = data.len(); - cache.put(&key, data, content_type.as_deref()).await?; + cache + .put_with_metadata(&key, data, content_type.as_deref(), last_modified) + .await?; debug!( repository = %repository, path = %location, @@ -887,6 +1711,156 @@ impl S3StorageInner { } Ok(()) } + + async fn multipart_copy( + &self, + destination: &str, + source: &str, + source_etag: &str, + object_size: u64, + head: &aws_sdk_s3::operation::head_object::HeadObjectOutput, + ) -> Result<(), S3StorageError> { + let mut create = self + .aws_client() + .create_multipart_upload() + .bucket(self.bucket()) + .key(destination); + if let Some(cache_control) = head.cache_control() { + create = create.cache_control(cache_control); + } + if let Some(content_disposition) = head.content_disposition() { + create = create.content_disposition(content_disposition); + } + if let Some(content_encoding) = head.content_encoding() { + create = create.content_encoding(content_encoding); + } + if let Some(content_language) = head.content_language() { + create = create.content_language(content_language); + } + if let Some(content_type) = head.content_type() { + create = create.content_type(content_type); + } + if let Some(expires) = head.expires_string().and_then(|value| { + aws_smithy_types::DateTime::from_str( + value, + aws_smithy_types::date_time::Format::HttpDate, + ) + .ok() + }) { + create = create.expires(expires); + } + if let Some(redirect) = head.website_redirect_location() { + create = create.website_redirect_location(redirect); + } + if let Some(metadata) = head.metadata() { + for (key, value) in metadata { + create = create.metadata(key, value); + } + } + let copy_deadline = Instant::now() + S3_COPY_TIMEOUT; + let create_timeout = copy_deadline + .saturating_duration_since(Instant::now()) + .min(S3_CONTROL_TIMEOUT); + let created = with_timeout(create_timeout, create.send()).await?; + let Some(upload_id) = created.upload_id() else { + return Err(S3StorageError::aws_message( + "S3 multipart copy did not return an upload ID", + )); + }; + let upload_id = upload_id.to_owned(); + let part_size = MULTIPART_COPY_PART_SIZE.max(object_size.div_ceil(10_000)); + let part_count = object_size.div_ceil(part_size); + let mut completed_parts = Vec::with_capacity(part_count as usize); + + for part in 0..part_count { + let start = part * part_size; + let end = (start + part_size).min(object_size) - 1; + let remaining = copy_deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + self.abort_multipart_copy(destination, &upload_id).await; + return Err(S3StorageError::aws_message( + "S3 multipart copy exceeded its operation deadline", + )); + } + let result = with_timeout( + remaining, + self.aws_client() + .upload_part_copy() + .bucket(self.bucket()) + .key(destination) + .upload_id(&upload_id) + .part_number((part + 1) as i32) + .copy_source(source) + .copy_source_range(format!("bytes={start}-{end}")) + .copy_source_if_match(source_etag) + .customize() + .config_override(timeout_override(copy_timeout_config())) + .send(), + ) + .await; + let result = match result { + Ok(result) => result, + Err(error) => { + self.abort_multipart_copy(destination, &upload_id).await; + return Err(error); + } + }; + let Some(etag) = result.copy_part_result().and_then(|part| part.e_tag()) else { + self.abort_multipart_copy(destination, &upload_id).await; + return Err(S3StorageError::aws_message( + "S3 multipart copy part did not return an ETag", + )); + }; + completed_parts.push( + aws_sdk_s3::types::CompletedPart::builder() + .part_number((part + 1) as i32) + .e_tag(etag) + .build(), + ); + } + + let complete = self + .aws_client() + .complete_multipart_upload() + .bucket(self.bucket()) + .key(destination) + .upload_id(upload_id.clone()) + .multipart_upload( + aws_sdk_s3::types::CompletedMultipartUpload::builder() + .set_parts(Some(completed_parts)) + .build(), + ); + let remaining = copy_deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + self.abort_multipart_copy(destination, &upload_id).await; + return Err(S3StorageError::aws_message( + "S3 multipart copy exceeded its operation deadline", + )); + } + if let Err(error) = with_timeout(remaining.min(S3_CONTROL_TIMEOUT), complete.send()).await { + self.abort_multipart_copy(destination, &upload_id).await; + return Err(error); + } + Ok(()) + } + + async fn abort_multipart_copy(&self, destination: &str, upload_id: &str) { + let _ = with_timeout( + S3_CONTROL_TIMEOUT, + self.aws_client() + .abort_multipart_upload() + .bucket(self.bucket()) + .key(destination) + .upload_id(upload_id) + .send(), + ) + .await; + } + + fn invalidate_manifest_cache(&self, repository: Uuid) { + self.manifest_cache.lock().invalidate(repository); + } + pub async fn get_path_for_creation( &self, repository: Uuid, @@ -902,9 +1876,7 @@ impl S3StorageInner { conflicting_path.push_mut(part.as_ref()); let is_last = iter.peek().is_none(); - let exists_as_object = self.does_path_exist(&path).await?; - - if exists_as_object && !is_last { + if !is_last && self.does_path_exist(&path).await? { // A parent segment is a concrete object, so we cannot place a child under it. return Err(PathCollisionError { path: location.clone(), @@ -912,20 +1884,22 @@ impl S3StorageInner { } .into()); } - // If this is the last segment, overwriting an existing object is allowed. } Ok(path) } #[instrument] async fn does_path_exist(&self, path: &str) -> Result { - let result = self - .aws_client() - .head_object() - .bucket(self.bucket()) - .key(path) - .send() - .await; + let result = tokio::time::timeout( + S3_CONTROL_TIMEOUT, + self.aws_client() + .head_object() + .bucket(self.bucket()) + .key(path) + .send(), + ) + .await + .map_err(|_| S3StorageError::aws_message("S3 HEAD request timed out"))?; match result { Ok(_) => Ok(true), @@ -933,121 +1907,23 @@ impl S3StorageInner { Err(err) => Err(S3StorageError::from_sdk_error(err)), } } - #[instrument] - fn is_directory_from_result( - &self, - result: &aws_sdk_s3::operation::list_objects_v2::ListObjectsV2Output, - path: &str, - ) -> (bool, Option) { - let contents = result.contents(); - let prefixes = result.common_prefixes(); - let is_contents_empty = contents.is_empty(); - let has_prefixes = !prefixes.is_empty(); - - if is_contents_empty && !has_prefixes { - return (true, None); - } - if path.ends_with('/') && !is_contents_empty { - return (true, None); - } - - let path_with_slash = format!("{}/", path); - if let Some(match_prefix) = prefixes - .iter() - .filter_map(CommonPrefix::prefix) - .find(|prefix| *prefix == path_with_slash) - { - return (true, Some(match_prefix.to_string())); - } - - (false, None) - } - - #[instrument] - async fn is_directory(&self, path: &str) -> Result { - let list = self - .aws_client() - .list_objects_v2() - .bucket(self.bucket()) - .prefix(path.to_owned()) - .delimiter("/") - .send() - .await - .map_err(S3StorageError::from_sdk_error)?; - - Ok(self.is_directory_from_result(&list, path).0) - } - async fn get_directory_meta( &self, path: &str, + modified: Option>, ) -> Result>, S3StorageError> { let file_file = FileType::Directory(DirectoryFileType { file_count: 0 }); let name = path.split_once('/').map(|(_, rest)| rest).unwrap_or(path); - let meta = StorageFileMeta { - name: name.to_owned(), - file_type: file_file, - modified: Local::now().fixed_offset(), - created: Local::now().fixed_offset(), - }; - - Ok(Some(meta)) - } - #[instrument] - async fn get_object_tagging(&self, path: &str) -> Result>, S3StorageError> { - let response = self - .aws_client() - .get_object_tagging() - .bucket(self.bucket()) - .key(path) - .send() - .await; - - match response { - Ok(output) => Ok(Some(output.tag_set().to_vec())), - Err(SdkError::ServiceError(err)) - if err - .err() - .meta() - .code() - .is_some_and(|code| code == "NoSuchKey") => - { - Ok(None) - } - Err(err) => Err(S3StorageError::from_sdk_error(err)), - } - } - - async fn get_meta_tags(&self, path: &str) -> Result, S3StorageError> { - let Some(tags) = self.get_object_tagging(path).await? else { - return Ok(None); - }; - - let name = tags - .iter() - .find(|tag| tag.key() == tags::NAME) - .map(|tag| tag.value().to_string()) - .ok_or_else(|| S3StorageError::static_missing_tag(tags::NAME))?; - - let mime_type = tags - .iter() - .find(|tag| tag.key() == tags::MIME_TYPE) - .map(|tag| Mime::from_str(tag.value())) - .transpose(); - let mime_type = match mime_type { - Ok(ok) => ok, - Err(e) => { - error!(?e, ?path, "Failed to parse mime type"); - None - } + let modified = modified.unwrap_or_else(|| Local::now().fixed_offset()); + let meta = StorageFileMeta { + name: name.to_owned(), + file_type: file_file, + modified, + created: modified, }; - Ok(Some(S3MetaTags { - name, - mime_type, - is_directory: false, - })) + Ok(Some(meta)) } } @@ -1091,35 +1967,33 @@ fn default_session_name() -> String { format!("pkgly-{}", Uuid::new_v4().simple()) } -fn bytes_to_stream(bytes: FileContentBytes) -> (ByteStream, usize) { - match bytes { - FileContentBytes::Content(content) => { - let len = content.len(); - (ByteStream::from(content), len) - } - FileContentBytes::Bytes(bytes) => { - let len = bytes.len(); - (ByteStream::from(bytes.to_vec()), len) - } - } +fn bytes_to_stream(bytes: Bytes) -> (ByteStream, usize) { + let len = bytes.len(); + (ByteStream::from(bytes), len) } -async fn file_into_bytes(file: FileContent) -> Result { +async fn file_into_bytes(file: FileContent) -> Result { let bytes = task::spawn_blocking(move || FileContentBytes::try_from(file)).await??; - Ok(bytes) + Ok(match bytes { + FileContentBytes::Content(content) => Bytes::from(content), + FileContentBytes::Bytes(bytes) => bytes, + }) } async fn collect_body(stream: ByteStream) -> Result { let aggregated = stream .collect() .await - .map_err(|err| S3StorageError::AwsSdkError(err.to_string()))?; + .map_err(|err| S3StorageError::aws_message(err.to_string()))?; Ok(aggregated.into_bytes()) } const DEFAULT_MIN_BUFFERED_OBJECT_BYTES: u64 = 1024 * 1024; // 1 MiB const DEFAULT_MAX_BUFFERED_OBJECT_BYTES: u64 = 8 * 1024 * 1024; // 8 MiB const DEFAULT_MEMORY_PRESSURE_THRESHOLD: f64 = 0.75; +const MEMORY_SNAPSHOT_TTL: Duration = Duration::from_secs(5); +const CACHE_FORMAT_VERSION: u8 = 1; +const MANIFEST_CACHE_TTL: Duration = Duration::from_secs(30); const FAILED_DELETION_QUEUE_LIMIT: usize = 1024; const FAILED_DELETION_MAX_RETRIES_PER_TICK: usize = 64; const FAILED_DELETION_BASE_DELAY_MS: u64 = 100; @@ -1160,8 +2034,23 @@ pub struct S3ListedObject { /// Key relative to the repository root (e.g. `packages/pkg/file.tgz`). pub key: String, pub size: u64, + /// Provider object timestamp, when the S3 response supplied a usable value. pub last_modified: Option>, } + +fn s3_last_modified( + value: Option<&aws_smithy_types::DateTime>, +) -> Option> { + let system_time: SystemTime = (*value?).try_into().ok()?; + Some(ChronoDateTime::::from(system_time).fixed_offset()) +} + +#[derive(Debug, Clone)] +struct DirectoryObject { + key: String, + size: u64, + last_modified: Option>, +} #[derive(Debug, Clone)] pub struct S3Storage(Arc); new_type_arc_type!(S3Storage(S3StorageInner)); @@ -1205,25 +2094,35 @@ impl Storage for S3Storage { "application/octet-stream" }; let file_as_bytes = file_into_bytes(file).await?; - let cache_buffer = file_as_bytes.clone_into_bytes(); - let (body, size) = bytes_to_stream(file_as_bytes); + let size = file_as_bytes.len(); + let cache_buffer = (self.should_cache(location) + && size as u64 <= self.cache.as_ref().map_or(0, |cache| cache.max_bytes)) + .then(|| file_as_bytes.clone()); + let (body, _) = bytes_to_stream(file_as_bytes); self.aws_client() .put_object() .bucket(self.bucket()) .key(&path) .body(body) .content_type(content_type) + .customize() + .config_override(timeout_override(streaming_timeout_config())) .send() .await .map_err(S3StorageError::from_sdk_error)?; debug!(path = %path, "File saved to S3"); - self.cache_put( - &repository, - location, - cache_buffer, - Some(content_type.to_string()), - ) - .await?; + self.invalidate_manifest_cache(repository); + let modified = Local::now().fixed_offset(); + if let Some(cache_buffer) = cache_buffer { + self.cache_put( + &repository, + location, + cache_buffer, + Some(content_type.to_string()), + Some(modified), + ) + .await?; + } Ok((size, !already_exists)) } #[instrument(name = "Storage::append_file", fields(storage_type = "s3"))] @@ -1238,48 +2137,78 @@ impl Storage for S3Storage { // This is still O(n) for S3 since network I/O dominates let path = self.get_path_for_creation(repository, location).await?; - let mut combined_buffer = if self.does_path_exist(&path).await? { - let response = self - .aws_client() - .get_object() - .bucket(self.bucket()) - .key(&path) - .send() - .await - .map_err(S3StorageError::from_sdk_error)?; - collect_body(response.body).await?.to_vec() - } else { - Vec::new() + let (current_bytes, current_etag) = match self + .aws_client() + .get_object() + .bucket(self.bucket()) + .key(&path) + .customize() + .config_override(timeout_override(streaming_timeout_config())) + .send() + .await + { + Ok(response) => { + let Some(etag) = response.e_tag().map(str::to_owned) else { + return Err(S3StorageError::aws_message( + "S3 existing object did not return an ETag; refusing an unguarded append", + )); + }; + (collect_body(response.body).await?, Some(etag)) + } + Err(error) => { + let error = S3StorageError::from_sdk_error(error); + if error.is_not_found() { + (Bytes::new(), None) + } else { + return Err(error); + } + } }; let appended = file_into_bytes(file).await?; - combined_buffer.extend_from_slice(appended.as_ref()); - - let combined_bytes = Bytes::from(combined_buffer); + let mut combined_buffer = BytesMut::with_capacity(current_bytes.len() + appended.len()); + combined_buffer.extend_from_slice(¤t_bytes); + combined_buffer.extend_from_slice(&appended); + let combined_bytes = combined_buffer.freeze(); let content_type = if location.is_directory() { "application/x-directory" } else { "application/octet-stream" }; - let size = combined_bytes.len(); - self.aws_client() + let mut request = self + .aws_client() .put_object() .bucket(self.bucket()) .key(&path) .content_type(content_type) - .body(ByteStream::from(combined_bytes.clone())) + .body(ByteStream::from(combined_bytes.clone())); + request = match current_etag { + Some(etag) => request.if_match(etag), + None => request.if_none_match("*"), + }; + request + .customize() + .config_override(timeout_override(streaming_timeout_config())) .send() .await .map_err(S3StorageError::from_sdk_error)?; - self.cache_put( - &repository, - location, - combined_bytes, - Some(content_type.to_string()), - ) - .await?; - Ok(size) + let appended_size = appended.len(); + self.invalidate_manifest_cache(repository); + let modified = Local::now().fixed_offset(); + if self.should_cache(location) + && combined_bytes.len() as u64 <= self.cache.as_ref().map_or(0, |cache| cache.max_bytes) + { + self.cache_put( + &repository, + location, + combined_bytes, + Some(content_type.to_string()), + Some(modified), + ) + .await?; + } + Ok(appended_size) } #[instrument(name = "Storage::put_repository_meta", fields(storage_type = "s3"))] async fn put_repository_meta( @@ -1307,10 +2236,8 @@ impl Storage for S3Storage { .list_objects_v2() .bucket(self.bucket()) .prefix(prefix) - .max_keys(1) - .send() - .await - .map_err(S3StorageError::from_sdk_error)?; + .max_keys(1); + let probe = with_timeout(S3_CONTROL_TIMEOUT, probe.send()).await?; if probe.key_count().unwrap_or(0) == 0 { return Err(S3StorageError::IOError(std::io::Error::new( std::io::ErrorKind::NotFound, @@ -1325,17 +2252,20 @@ impl Storage for S3Storage { .map(ByteStream::from) .map_err(|err| S3StorageError::IOError(std::io::Error::other(err)))?; - self.aws_client() - .put_object() - .bucket(self.bucket()) - .key(meta_path) - .content_type("application/json") - .body(body) - .send() - .await - .map_err(S3StorageError::from_sdk_error)?; + with_timeout( + S3_CONTROL_TIMEOUT, + self.aws_client() + .put_object() + .bucket(self.bucket()) + .key(meta_path) + .content_type("application/json") + .body(body) + .send(), + ) + .await?; // Repository meta is small; we intentionally do not cache it to avoid polluting the blob cache. + self.invalidate_manifest_cache(repository); Ok(()) } #[instrument(name = "Storage::get_repository_meta", fields(storage_type = "s3"))] @@ -1347,17 +2277,19 @@ impl Storage for S3Storage { let meta_location = S3StorageInner::meta_storage_path(location); let meta_path = self.s3_path(&repository, &meta_location); - let response = match self - .aws_client() - .get_object() - .bucket(self.bucket()) - .key(&meta_path) - .send() - .await + let response = match with_timeout( + S3_CONTROL_TIMEOUT, + self.aws_client() + .get_object() + .bucket(self.bucket()) + .key(&meta_path) + .send(), + ) + .await { Ok(resp) => resp, - Err(SdkError::ServiceError(err)) if err.err().is_no_such_key() => return Ok(None), - Err(err) => return Err(S3StorageError::from_sdk_error(err)), + Err(error) if error.is_not_found() => return Ok(None), + Err(error) => return Err(error), }; let body = collect_body(response.body).await?; @@ -1380,14 +2312,17 @@ impl Storage for S3Storage { if !exists { return Ok(false); } - self.aws_client() - .delete_object() - .bucket(self.bucket()) - .key(&path) - .send() - .await - .map_err(S3StorageError::from_sdk_error)?; + with_timeout( + S3_CONTROL_TIMEOUT, + self.aws_client() + .delete_object() + .bucket(self.bucket()) + .key(&path) + .send(), + ) + .await?; self.cache_remove(&repository, location).await?; + self.invalidate_manifest_cache(repository); Ok(true) } #[instrument( @@ -1404,54 +2339,72 @@ impl Storage for S3Storage { let from_path = self.s3_path(&repository, from); let to_path = self.s3_path(&repository, to); - // Check if source exists - if !self.does_path_exist(&from_path).await? { - return Ok(false); - } - - // For S3, we need to copy and then delete since there's no native rename - // Read the object - let response = self - .aws_client() - .get_object() - .bucket(self.bucket()) - .key(&from_path) - .send() - .await - .map_err(S3StorageError::from_sdk_error)?; - let bytes = collect_body(response.body).await?; - - // Get content type from original object metadata (if available) - let content_type = if to.is_directory() { - "application/x-directory" - } else { - "application/octet-stream" + let head = match with_timeout( + S3_CONTROL_TIMEOUT, + self.aws_client() + .head_object() + .bucket(self.bucket()) + .key(&from_path) + .send(), + ) + .await + { + Ok(head) => head, + Err(error) if error.is_not_found() => { + return Ok(false); + } + Err(error) => return Err(error), }; + let Some(source_etag) = head.e_tag().map(str::to_owned) else { + return Err(S3StorageError::aws_message( + "S3 source object did not return an ETag; refusing an unguarded move", + )); + }; + let object_size = head.content_length().unwrap_or_default().max(0) as u64; + let source = copy_source(self.bucket(), &from_path); + + if object_size <= MULTIPART_COPY_THRESHOLD { + with_timeout( + S3_COPY_TIMEOUT, + self.aws_client() + .copy_object() + .bucket(self.bucket()) + .key(&to_path) + .copy_source(source.clone()) + .copy_source_if_match(source_etag.clone()) + .customize() + .config_override(timeout_override(copy_timeout_config())) + .send(), + ) + .await?; + } else { + self.multipart_copy(&to_path, &source, &source_etag, object_size, &head) + .await?; + } - // Write to new location - self.aws_client() - .put_object() - .bucket(self.bucket()) - .key(&to_path) - .content_type(content_type) - .body(ByteStream::from(bytes.clone())) - .send() - .await - .map_err(S3StorageError::from_sdk_error)?; - - // Delete original - self.aws_client() - .delete_object() - .bucket(self.bucket()) - .key(&from_path) - .send() - .await - .map_err(S3StorageError::from_sdk_error)?; + // The destination changed as soon as the copy completed. Drop any stale destination + // cache before attempting the guarded source deletion. + self.cache_remove(&repository, to).await?; + let delete_result = with_timeout( + S3_CONTROL_TIMEOUT, + self.aws_client() + .delete_object() + .bucket(self.bucket()) + .key(&from_path) + .if_match(source_etag) + .send(), + ) + .await; + if let Err(error) = delete_result { + // A failed conditional delete may mean the source changed or that the request was + // accepted before the connection failed; both cache entries are unsafe to retain. + self.cache_remove(&repository, from).await?; + self.invalidate_manifest_cache(repository); + return Err(error); + } self.cache_remove(&repository, from).await?; - self.cache_put(&repository, to, bytes, Some(content_type.to_string())) - .await?; - + self.invalidate_manifest_cache(repository); Ok(true) } #[instrument( @@ -1471,6 +2424,9 @@ impl Storage for S3Storage { .and_then(|ct| Mime::from_str(ct).ok()) .map(SerdeMime); let size = cached.bytes.len() as u64; + let modified = cached + .last_modified + .unwrap_or_else(|| Local::now().fixed_offset()); return Ok(Some(StorageFileMeta:: { name: location.to_string(), file_type: FileType::File(FileFileType { @@ -1478,78 +2434,71 @@ impl Storage for S3Storage { mime_type, file_hash: FileHashes::default(), }), - modified: Local::now().fixed_offset(), - created: Local::now().fixed_offset(), + modified, + created: modified, })); } let path = self.s3_path(&repository, location); - let head = match self - .aws_client() - .head_object() - .bucket(self.bucket()) - .key(&path) - .send() - .await + let head = match with_timeout( + S3_CONTROL_TIMEOUT, + self.aws_client() + .head_object() + .bucket(self.bucket()) + .key(&path) + .send(), + ) + .await { Ok(head) => head, - Err(SdkError::ServiceError(err)) if err.err().is_not_found() => { + Err(error) if error.is_not_found() => { // Maybe this is a directory prefix without a placeholder object. let prefix = if path.ends_with('/') { path.clone() } else { format!("{}/", path) }; - let list = self - .aws_client() - .list_objects_v2() - .bucket(self.bucket()) - .prefix(prefix.clone()) - .delimiter("/") - .send() - .await - .map_err(S3StorageError::from_sdk_error)?; - - if list.key_count().unwrap_or(0) == 0 { + let (objects, prefixes) = self.list_directory_entries(&prefix).await?; + if objects.is_empty() && prefixes.is_empty() { return Ok(None); } - let mut count: u64 = 0; - for obj in list.contents() { - if let Some(key) = obj.key() { - if key == prefix || S3StorageInner::is_hidden_file(key) { - continue; - } - count += 1; - } - } - for pref in list - .common_prefixes() + let count = objects .iter() - .filter_map(CommonPrefix::prefix) - { - if S3StorageInner::is_hidden_file(pref) { - continue; - } - count += 1; - } + .filter(|object| { + object.key != prefix && !S3StorageInner::is_hidden_file(&object.key) + }) + .count() + + prefixes + .iter() + .filter(|prefix| !S3StorageInner::is_hidden_file(prefix)) + .count(); + let modified = objects + .iter() + .find(|object| object.key == prefix) + .and_then(|object| object.last_modified) + .unwrap_or_else(|| Local::now().fixed_offset()); let dir_meta = StorageFileMeta:: { name: location.to_string(), - file_type: FileType::Directory(DirectoryFileType { file_count: count }), - modified: Local::now().fixed_offset(), - created: Local::now().fixed_offset(), + file_type: FileType::Directory(DirectoryFileType { + file_count: count as u64, + }), + modified, + created: modified, }; return Ok(Some(dir_meta)); } - Err(err) => return Err(S3StorageError::from_sdk_error(err)), + Err(error) => return Err(error), }; let content_type = head.content_type().map(|ct| ct.to_string()); if content_type .as_deref() .is_some_and(|ct| ct == "application/x-directory") - && let Some(meta) = self.get_directory_meta(&path).await? + && let Some(meta) = self + .get_directory_meta(&path, s3_last_modified(head.last_modified())) + .await? { return Ok(Some(meta)); } @@ -1566,7 +2515,8 @@ impl Storage for S3Storage { .unwrap_or_default() .map(SerdeMime); - let modified = Local::now().fixed_offset(); + let modified = + s3_last_modified(head.last_modified()).unwrap_or_else(|| Local::now().fixed_offset()); let meta = StorageFileMeta:: { name: location.to_string(), @@ -1596,6 +2546,9 @@ impl Storage for S3Storage { .and_then(|ct| Mime::from_str(ct).ok()) .map(SerdeMime); let size = cached.bytes.len() as u64; + let modified = cached + .last_modified + .unwrap_or_else(|| Local::now().fixed_offset()); let meta = StorageFileMeta:: { name: location.to_string(), file_type: FileFileType { @@ -1603,8 +2556,8 @@ impl Storage for S3Storage { mime_type, file_hash: FileHashes::default(), }, - modified: Local::now().fixed_offset(), - created: Local::now().fixed_offset(), + modified, + created: modified, }; let result = StorageFile::File { meta, @@ -1618,23 +2571,28 @@ impl Storage for S3Storage { .get_object() .bucket(self.bucket()) .key(&path) + .customize() + .config_override(timeout_override(streaming_timeout_config())) .send() .await { Ok(resp) => resp, Err(SdkError::ServiceError(err)) if err.err().is_no_such_key() => { - return self.collect_directory(repository, location).await; + return self.collect_directory(repository, location, None).await; } Err(err) => return Err(S3StorageError::from_sdk_error(err)), }; let response_content_type = response.content_type().map(|ct| ct.to_string()); + let response_last_modified = s3_last_modified(response.last_modified()); if response_content_type .as_deref() .map(|ct| ct == "application/x-directory") .unwrap_or(false) { - return self.collect_directory(repository, location).await; + return self + .collect_directory(repository, location, response_last_modified) + .await; } let response_length_opt = response .content_length() @@ -1665,6 +2623,7 @@ impl Storage for S3Storage { ); } + let modified = response_last_modified.unwrap_or_else(|| Local::now().fixed_offset()); let meta = StorageFileMeta:: { name: location.to_string(), file_type: FileFileType { @@ -1677,8 +2636,8 @@ impl Storage for S3Storage { .map(SerdeMime), file_hash: FileHashes::default(), }, - modified: Local::now().fixed_offset(), - created: Local::now().fixed_offset(), + modified, + created: modified, }; let content = match strategy { BodyRetrievalStrategy::BufferAndCache => { @@ -1689,6 +2648,7 @@ impl Storage for S3Storage { location, body.clone(), response_content_type.clone(), + Some(modified), ) .await?; } @@ -1732,6 +2692,7 @@ impl Storage for S3Storage { )] async fn delete_repository(&self, repository: uuid::Uuid) -> Result<(), S3StorageError> { let prefix = format!("{repository}/"); + let deadline = Instant::now() + S3_CONTROL_TIMEOUT; let mut continuation: Option = None; loop { @@ -1746,10 +2707,13 @@ impl Storage for S3Storage { request = request.continuation_token(token); } - let response = request - .send() - .await - .map_err(S3StorageError::from_sdk_error)?; + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(S3StorageError::aws_message( + "S3 repository deletion exceeded its control deadline", + )); + } + let response = with_timeout(remaining, request.send()).await?; let paths: Vec = response .contents() @@ -1776,6 +2740,7 @@ impl Storage for S3Storage { break; } + self.invalidate_manifest_cache(repository); Ok(()) } @@ -1818,30 +2783,23 @@ impl Storage for S3Storage { format!("{base_prefix}/") }; - let list = self - .aws_client() - .list_objects_v2() - .bucket(self.bucket()) - .prefix(prefix.clone()) - .delimiter("/") - .send() - .await - .map_err(S3StorageError::from_sdk_error)?; + let (objects, prefixes) = self.list_directory_entries(&prefix).await?; + let observed_at = Local::now().fixed_offset(); // Convert objects to StorageFileMeta entries using names relative to the requested // directory (not repository root) so callers can safely append child segments. let mut entries: Vec> = Vec::new(); - for obj in list.contents() { - let Some(key) = obj.key() else { continue }; + for obj in objects { + let key = obj.key; if key == prefix { // Directory placeholder object continue; } - if S3StorageInner::is_hidden_file(key) { + if S3StorageInner::is_hidden_file(&key) { continue; } - let full_name = S3StorageInner::strip_repository_prefix(&repository, key); + let full_name = S3StorageInner::strip_repository_prefix(&repository, &key); let Some(relative_name) = full_name.strip_prefix(&base_prefix) else { continue; }; @@ -1849,26 +2807,21 @@ impl Storage for S3Storage { continue; } - let size: u64 = obj.size().unwrap_or(0i64).max(0) as u64; let meta = StorageFileMeta:: { name: relative_name.to_string(), file_type: FileType::File(FileFileType { - file_size: size, + file_size: obj.size, mime_type: None, file_hash: FileHashes::default(), }), - modified: Local::now().fixed_offset(), - created: Local::now().fixed_offset(), + modified: obj.last_modified.unwrap_or(observed_at), + created: obj.last_modified.unwrap_or(observed_at), }; entries.push(meta); } - for prefix_entry in list - .common_prefixes() - .iter() - .filter_map(CommonPrefix::prefix) - { - let full_name = S3StorageInner::strip_repository_prefix(&repository, prefix_entry); + for prefix_entry in prefixes { + let full_name = S3StorageInner::strip_repository_prefix(&repository, &prefix_entry); let Some(relative_name) = full_name.strip_prefix(&base_prefix) else { continue; }; @@ -1884,8 +2837,8 @@ impl Storage for S3Storage { let meta = StorageFileMeta:: { name: cleaned.to_string(), file_type: FileType::Directory(DirectoryFileType { file_count: 0 }), - modified: Local::now().fixed_offset(), - created: Local::now().fixed_offset(), + modified: observed_at, + created: observed_at, }; entries.push(meta); } @@ -1934,12 +2887,11 @@ impl S3Storage { .max_keys(1000) .into_paginator() .send(); + let deadline = Instant::now() + S3_CONTROL_TIMEOUT; let mut objects = Vec::new(); - while let Some(page) = paginator.next().await { - let page = page.map_err(S3StorageError::from_sdk_error)?; - + while let Some(page) = next_control_page(deadline, paginator.next()).await? { for obj in page.contents() { let Some(key) = obj.key() else { continue }; if S3StorageInner::is_hidden_file(key) { @@ -1954,7 +2906,7 @@ impl S3Storage { } let size = obj.size().unwrap_or(0i64).max(0) as u64; - let last_modified = None; + let last_modified = s3_last_modified(obj.last_modified()); objects.push(S3ListedObject { key: repo_relative.to_string(), @@ -1975,11 +2927,12 @@ impl S3Storage { fields(storage_type = "s3", ?repository), skip(self) )] - pub async fn list_docker_manifests( + async fn load_docker_manifests( &self, repository: Uuid, ) -> Result, S3StorageError> { let mut manifests = Vec::new(); + let deadline = Instant::now() + S3_CONTROL_TIMEOUT; let mut queue = VecDeque::new(); queue.push_back(format!("{}/v2/", repository)); @@ -1995,9 +2948,7 @@ impl S3Storage { .into_paginator() .send(); - while let Some(page) = paginator.next().await { - let page = page.map_err(S3StorageError::from_sdk_error)?; - + while let Some(page) = next_control_page(deadline, paginator.next()).await? { for p in page .common_prefixes() .iter() @@ -2019,8 +2970,9 @@ impl S3Storage { .into_paginator() .send(); - while let Some(mpage) = manifest_pages.next().await { - let mpage = mpage.map_err(S3StorageError::from_sdk_error)?; + while let Some(mpage) = + next_control_page(deadline, manifest_pages.next()).await? + { for obj in mpage.contents() { let Some(key) = obj.key() else { continue }; if S3StorageInner::is_hidden_file(key) { @@ -2036,7 +2988,7 @@ impl S3Storage { manifests.push(S3ListedObject { key: repo_relative.to_string(), size, - last_modified: None, + last_modified: s3_last_modified(obj.last_modified()), }); } } @@ -2051,6 +3003,33 @@ impl S3Storage { Ok(manifests) } + /// List Docker manifests using a short-lived per-repository index. + pub async fn list_docker_manifests( + &self, + repository: Uuid, + ) -> Result, S3StorageError> { + let cached = self.manifest_cache.lock().get(repository, Instant::now()); + if let Some(cached) = cached { + return Ok(cached); + } + let _load_lock = self.manifest_load_lock.lock().await; + let generation = { + let mut cache = self.manifest_cache.lock(); + if let Some(cached) = cache.get(repository, Instant::now()) { + return Ok(cached); + } + cache.generation() + }; + let manifests = self.load_docker_manifests(repository).await?; + self.manifest_cache.lock().insert_if_generation( + repository, + manifests.clone(), + Instant::now(), + generation, + ); + Ok(manifests) + } + /// Paginate Docker manifest objects without loading the entire repository into memory. /// /// Returns the requested page of manifest objects (ordered lexicographically by key) @@ -2066,87 +3045,9 @@ impl S3Storage { start: usize, limit: usize, ) -> Result<(Vec, usize), S3StorageError> { - let mut items = Vec::with_capacity(limit); - let mut total = 0usize; - - // Breadth-first traversal that skips heavy prefixes (`blobs`, uploads) while preserving - // lexicographic order of manifest keys. - let mut queue = VecDeque::new(); - queue.push_back(format!("{}/v2/", repository)); - - while let Some(prefix) = queue.pop_front() { - let mut paginator = self - .aws_client() - .list_objects_v2() - .bucket(self.bucket()) - .prefix(prefix.clone()) - .delimiter("/") - .max_keys(1000) - .into_paginator() - .send(); - - while let Some(page) = paginator.next().await { - let page = page.map_err(S3StorageError::from_sdk_error)?; - - // Descend into sub-prefixes (directories) - for p in page - .common_prefixes() - .iter() - .filter_map(CommonPrefix::prefix) - { - if p.ends_with("blobs/") || p.ends_with("uploads/") || p.ends_with("_uploads/") - { - continue; - } - - if p.ends_with("manifests/") { - // List manifest objects directly under this prefix (no delimiter) - let mut manifest_pages = self - .aws_client() - .list_objects_v2() - .bucket(self.bucket()) - .prefix(p) - .max_keys(1000) - .into_paginator() - .send(); - - while let Some(mpage) = manifest_pages.next().await { - let mpage = mpage.map_err(S3StorageError::from_sdk_error)?; - for obj in mpage.contents() { - let Some(key) = obj.key() else { continue }; - if S3StorageInner::is_hidden_file(key) { - continue; - } - - total += 1; - if total <= start { - continue; - } - if items.len() >= limit { - continue; - } - - let repo_relative = - S3StorageInner::strip_repository_prefix(&repository, key); - if repo_relative.is_empty() { - continue; - } - - let size = obj.size().unwrap_or(0i64).max(0) as u64; - items.push(S3ListedObject { - key: repo_relative.to_string(), - size, - last_modified: None, - }); - } - } - } else { - queue.push_back(p.to_string()); - } - } - } - } - + let manifests = self.list_docker_manifests(repository).await?; + let total = manifests.len(); + let items = manifests.into_iter().skip(start).take(limit).collect(); Ok((items, total)) } @@ -2154,6 +3055,7 @@ impl S3Storage { &self, repository: Uuid, location: &StoragePath, + modified: Option>, ) -> Result, S3StorageError> { let Some(stream) = self.stream_directory(repository, location).await? else { return Ok(None); @@ -2162,15 +3064,16 @@ impl S3Storage { let file_count = stream.number_of_files(); let files = collect_directory_stream(stream) .await - .map_err(|err| S3StorageError::AwsSdkError(err.to_string()))?; + .map_err(|err| S3StorageError::aws_message(err.to_string()))?; + let observed_at = modified.unwrap_or_else(|| Local::now().fixed_offset()); let meta = StorageFileMeta:: { name: location.to_string(), file_type: DirectoryFileType { file_count: file_count.max(files.len() as u64), }, - modified: Local::now().fixed_offset(), - created: Local::now().fixed_offset(), + modified: observed_at, + created: observed_at, }; Ok(Some(StorageFile::Directory { meta, files })) @@ -2209,7 +3112,7 @@ impl S3Storage { let obj_id = ObjectIdentifier::builder() .key(key) .build() - .map_err(|err| S3StorageError::AwsSdkError(err.to_string()))?; + .map_err(|err| S3StorageError::aws_message(err.to_string()))?; object_ids.push(obj_id); } @@ -2217,23 +3120,28 @@ impl S3Storage { continue; } - let response = self - .aws_client() - .delete_objects() - .bucket(self.bucket()) - .delete( - aws_sdk_s3::types::Delete::builder() - .set_objects(Some(object_ids)) - .quiet(true) // Don't return deleted objects in response - .build() - .map_err(|err| S3StorageError::AwsSdkError(err.to_string()))?, - ) - .send() - .await - .map_err(S3StorageError::from_sdk_error)?; + let response = with_timeout( + S3_CONTROL_TIMEOUT, + self.aws_client() + .delete_objects() + .bucket(self.bucket()) + .delete( + aws_sdk_s3::types::Delete::builder() + .set_objects(Some(object_ids)) + .quiet(true) // Don't return deleted objects in response + .build() + .map_err(|err| S3StorageError::aws_message(err.to_string()))?, + ) + .send(), + ) + .await?; // Count successful deletions (errors() returns objects that failed) let failed = response.errors(); + for path in chunk { + self.cache_remove(&repository, path).await?; + } + self.invalidate_manifest_cache(repository); if !failed.is_empty() { let (code, message, key) = failed .first() @@ -2255,17 +3163,12 @@ impl S3Storage { keys_sample = ?keys_for_log.get(0..5).map(|v| v.to_vec()), "S3 delete_objects reported errors" ); - return Err(S3StorageError::AwsSdkError(format!( + return Err(S3StorageError::aws_message(format!( "delete_objects failed for key {key}: {code} - {message}" ))); } deleted_count += chunk.len(); - - // Remove from cache - for path in chunk { - self.cache_remove(&repository, path).await?; - } } debug!( @@ -2282,6 +3185,7 @@ impl S3Storage { /// Skips internal Pkgly metadata objects. pub async fn repository_size_bytes(&self, repository: Uuid) -> Result { let prefix = format!("{repository}/"); + let deadline = Instant::now() + S3_CONTROL_TIMEOUT; let mut continuation: Option = None; let mut total: u64 = 0; @@ -2296,10 +3200,13 @@ impl S3Storage { request = request.continuation_token(token); } - let response = request - .send() - .await - .map_err(S3StorageError::from_sdk_error)?; + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(S3StorageError::aws_message( + "S3 repository size calculation exceeded its control deadline", + )); + } + let response = with_timeout(remaining, request.send()).await?; for obj in response.contents() { if let Some(key) = obj.key() { @@ -2355,6 +3262,8 @@ impl StaticStorageFactory for S3StorageFactory { storage_config: inner, client, cache, + manifest_cache: ParkingMutex::new(ManifestCache::new()), + manifest_load_lock: Mutex::new(()), }; let storage = S3Storage::from(inner); Ok(storage) @@ -2393,6 +3302,8 @@ impl StorageFactory for S3StorageFactory { storage_config, client, cache, + manifest_cache: ParkingMutex::new(ManifestCache::new()), + manifest_load_lock: Mutex::new(()), }; let storage = S3Storage::from(inner); Ok(DynStorage::S3(storage)) diff --git a/crates/storage/src/s3/regions.rs b/crates/storage/src/s3/regions.rs index c96352a..0d261b2 100644 --- a/crates/storage/src/s3/regions.rs +++ b/crates/storage/src/s3/regions.rs @@ -1,98 +1,65 @@ -use aws_types::region::Region; +// ABOUTME: Defines S3 region suggestions and custom endpoint configuration. +// ABOUTME: Region values remain raw strings so new providers need no code release. use serde::{Deserialize, Serialize}; -use std::fmt::{Display, Formatter}; -use strum::EnumIter; use url::Url; use utoipa::ToSchema; -#[derive(Clone, Debug, Eq, Copy, PartialEq, Serialize, Deserialize, ToSchema, EnumIter)] -pub enum S3StorageRegion { - /// us-east-1 - UsEast1, - /// us-east-2 - UsEast2, - /// us-west-1 - UsWest1, - /// us-west-2 - UsWest2, - /// ca-central-1 - CaCentral1, - /// af-south-1 - AfSouth1, - /// ap-east-1 - ApEast1, - /// ap-south-1 - ApSouth1, - /// ap-northeast-1 - ApNortheast1, - /// ap-northeast-2 - ApNortheast2, - /// ap-northeast-3 - ApNortheast3, - /// ap-southeast-1 - ApSoutheast1, - /// ap-southeast-2 - ApSoutheast2, - /// cn-north-1 - CnNorth1, - /// cn-northwest-1 - CnNorthwest1, - /// eu-north-1 - EuNorth1, - /// eu-central-1 - EuCentral1, - /// eu-central-2 - EuCentral2, - /// eu-west-1 - EuWest1, - /// eu-west-2 - EuWest2, - /// eu-west-3 - EuWest3, - /// il-central-1 - IlCentral1, - /// me-south-1 - MeSouth1, - /// sa-east-1 - SaEast1, -} -impl Display for S3StorageRegion { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let value = match self { - S3StorageRegion::UsEast1 => "us-east-1", - S3StorageRegion::UsEast2 => "us-east-2", - S3StorageRegion::UsWest1 => "us-west-1", - S3StorageRegion::UsWest2 => "us-west-2", - S3StorageRegion::CaCentral1 => "ca-central-1", - S3StorageRegion::AfSouth1 => "af-south-1", - S3StorageRegion::ApEast1 => "ap-east-1", - S3StorageRegion::ApSouth1 => "ap-south-1", - S3StorageRegion::ApNortheast1 => "ap-northeast-1", - S3StorageRegion::ApNortheast2 => "ap-northeast-2", - S3StorageRegion::ApNortheast3 => "ap-northeast-3", - S3StorageRegion::ApSoutheast1 => "ap-southeast-1", - S3StorageRegion::ApSoutheast2 => "ap-southeast-2", - S3StorageRegion::CnNorth1 => "cn-north-1", - S3StorageRegion::CnNorthwest1 => "cn-northwest-1", - S3StorageRegion::EuNorth1 => "eu-north-1", - S3StorageRegion::EuCentral1 => "eu-central-1", - S3StorageRegion::EuCentral2 => "eu-central-2", - S3StorageRegion::EuWest1 => "eu-west-1", - S3StorageRegion::EuWest2 => "eu-west-2", - S3StorageRegion::EuWest3 => "eu-west-3", - S3StorageRegion::IlCentral1 => "il-central-1", - S3StorageRegion::MeSouth1 => "me-south-1", - S3StorageRegion::SaEast1 => "sa-east-1", - }; - f.write_str(value) - } -} +/// Canonical region identifiers offered as UI suggestions. +/// +/// The S3 configuration accepts any non-empty region string; this list is only a convenience for +/// commonly used AWS partitions and does not constrain configuration or deserialization. +pub const KNOWN_S3_REGIONS: &[&str] = &[ + "af-south-1", + "ap-east-1", + "ap-east-2", + "ap-northeast-1", + "ap-northeast-2", + "ap-northeast-3", + "ap-south-1", + "ap-south-2", + "ap-southeast-1", + "ap-southeast-2", + "ap-southeast-3", + "ap-southeast-4", + "ap-southeast-5", + "ap-southeast-6", + "ap-southeast-7", + "ca-central-1", + "ca-west-1", + "cn-north-1", + "cn-northwest-1", + "eu-central-1", + "eu-central-2", + "eu-central-3", + "eu-north-1", + "eu-south-1", + "eu-south-2", + "eu-west-1", + "eu-west-2", + "eu-west-3", + "eusc-de-east-1", + "eusc-de-east-2", + "il-central-1", + "me-central-1", + "me-south-1", + "mx-central-1", + "sa-east-1", + "us-east-1", + "us-east-2", + "us-gov-east-1", + "us-gov-west-1", + "us-iso-east-1", + "us-iso-west-1", + "us-isob-east-1", + "us-isob-west-1", + "us-isof-east-1", + "us-isof-south-1", + "us-isof-south-2", + "us-west-1", + "us-west-2", +]; -impl From for Region { - fn from(value: S3StorageRegion) -> Self { - Region::new(value.to_string()) - } -} +/// An optional custom endpoint and the region label used to sign requests to it. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, ToSchema)] pub struct CustomRegion { pub custom_region: Option, diff --git a/crates/storage/src/s3/tags.rs b/crates/storage/src/s3/tags.rs deleted file mode 100644 index d150c95..0000000 --- a/crates/storage/src/s3/tags.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub static NAME: &str = "NR-NAME"; -pub static MIME_TYPE: &str = "NR-MIME-TYPE"; diff --git a/crates/storage/src/s3/tests.rs b/crates/storage/src/s3/tests.rs index afab828..6be9e1c 100644 --- a/crates/storage/src/s3/tests.rs +++ b/crates/storage/src/s3/tests.rs @@ -1,12 +1,28 @@ +// ABOUTME: Exercises S3 configuration, caching, and storage behavior. +// ABOUTME: Provides a deterministic HTTP S3 test service for protocol-level tests. #![allow(clippy::expect_used, clippy::panic, clippy::todo, clippy::unwrap_used)] use super::{ AdaptiveBufferConfig, BodyRetrievalStrategy, CustomRegion, DEFAULT_MAX_BUFFERED_OBJECT_BYTES, - S3CacheConfig, S3Config, S3Credentials, S3DiskCache, S3StorageRegion, + MemorySnapshot, MemorySnapshotCache, S3CacheConfig, S3Config, S3Credentials, S3DiskCache, + default_cache_dir, resolve_cache_dir, +}; +use aws_smithy_runtime_api::{ + client::{ + orchestrator::HttpResponse as SmithyHttpResponse, result::SdkError as SmithySdkError, + }, + http::StatusCode as SmithyStatusCode, +}; +use aws_smithy_types::{ + body::SdkBody, + error::{ErrorMetadata, metadata::ProvideErrorMetadata}, }; use bytes::Bytes; +use chrono::{FixedOffset, TimeZone}; use tempfile::tempdir; use tokio::{ fs, + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpStream, time::{Duration, sleep}, }; @@ -49,7 +65,7 @@ fn role_detection_prefers_non_empty_strings() { fn custom_region_returns_endpoint_and_name() { let config = S3Config { bucket_name: "pkgly".into(), - region: Some(S3StorageRegion::UsEast1), + region: Some("us-east-1".into()), custom_region: Some(CustomRegion { custom_region: Some("minio".into()), endpoint: "https://minio.local".parse().unwrap(), @@ -67,6 +83,205 @@ fn custom_region_returns_endpoint_and_name() { assert!(config.custom_endpoint().is_some()); } +#[test] +fn raw_region_values_are_passed_through() { + let config = S3Config { + bucket_name: "pkgly".into(), + region: Some("eu-central-99".into()), + custom_region: None, + credentials: S3Credentials::default(), + path_style: true, + cache: S3CacheConfig::default(), + adaptive_buffer: AdaptiveBufferConfig::default(), + }; + + assert_eq!( + config.resolved_region().expect("region").as_ref(), + "eu-central-99" + ); +} + +#[test] +fn copy_source_encodes_bucket_and_key() { + assert_eq!( + super::copy_source("bucket", "folder/file name+one"), + "/bucket/folder%2Ffile%20name%2Bone" + ); +} + +#[test] +fn sdk_error_messages_keep_actionable_kinds() { + assert_eq!( + super::S3StorageError::aws_message("PreconditionFailed: status code: 412").kind(), + Some(super::S3ErrorKind::Conflict) + ); + assert_eq!( + super::S3StorageError::aws_message("AccessDenied: status code: 403").kind(), + Some(super::S3ErrorKind::AccessDenied) + ); + assert_eq!( + super::S3StorageError::aws_message("NoSuchBucket").kind(), + Some(super::S3ErrorKind::NotFound) + ); + assert_eq!( + super::S3StorageError::aws_message("HTTP status 404").kind(), + Some(super::S3ErrorKind::NotFound) + ); + assert!(super::S3StorageError::aws_message("SlowDown: status code: 429").is_retryable()); + assert!(super::S3StorageError::aws_message("status code: 503").is_retryable()); + assert!(super::S3StorageError::aws_message("dispatch failure").is_retryable()); + assert!(super::S3StorageError::aws_message("operation deadline exceeded").is_retryable()); +} + +#[derive(Debug)] +struct MetadataOnlyError(ErrorMetadata); + +impl std::fmt::Display for MetadataOnlyError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("opaque service failure") + } +} + +impl std::error::Error for MetadataOnlyError {} + +impl ProvideErrorMetadata for MetadataOnlyError { + fn meta(&self) -> &ErrorMetadata { + &self.0 + } +} + +fn smithy_response(status: u16) -> SmithyHttpResponse { + SmithyHttpResponse::new( + SmithyStatusCode::try_from(status).expect("valid status"), + SdkBody::empty(), + ) +} + +#[test] +fn sdk_error_classification_uses_smithy_metadata_and_status() { + let denied = SmithySdkError::service_error( + MetadataOnlyError(ErrorMetadata::builder().code("AccessDenied").build()), + smithy_response(500), + ); + assert_eq!( + super::S3StorageError::from_sdk_error(denied).kind(), + Some(super::S3ErrorKind::AccessDenied) + ); + + let conflict = SmithySdkError::service_error( + MetadataOnlyError(ErrorMetadata::builder().code("OpaqueCode").build()), + smithy_response(412), + ); + assert_eq!( + super::S3StorageError::from_sdk_error(conflict).kind(), + Some(super::S3ErrorKind::Conflict) + ); +} + +#[test] +fn timeout_profiles_define_attempt_and_operation_deadlines() { + let control = super::control_timeout_config(); + assert_eq!( + control.operation_attempt_timeout(), + Some(Duration::from_secs(30)) + ); + assert_eq!(control.operation_timeout(), Some(Duration::from_secs(90))); + + let copy = super::copy_timeout_config(); + assert_eq!( + copy.operation_attempt_timeout(), + Some(Duration::from_secs(5 * 60)) + ); + assert_eq!(copy.operation_timeout(), Some(Duration::from_secs(20 * 60))); + + let streaming = super::streaming_timeout_config(); + assert_eq!(streaming.operation_attempt_timeout(), None); + assert_eq!(streaming.operation_timeout(), None); + assert_eq!(streaming.connect_timeout(), Some(Duration::from_secs(5))); + assert_eq!(streaming.read_timeout(), Some(Duration::from_secs(30))); +} + +#[test] +fn legacy_region_enum_values_deserialize_to_raw_ids() { + let config: S3Config = serde_json::from_value(serde_json::json!({ + "bucket_name": "pkgly", + "region": "UsEast1", + "credentials": {}, + })) + .expect("legacy region should remain readable"); + + assert_eq!(config.region.as_deref(), Some("us-east-1")); +} + +#[test] +fn every_legacy_region_token_is_migrated_by_deserializer() { + let mappings = [ + ("UsEast1", "us-east-1"), + ("UsEast2", "us-east-2"), + ("UsWest1", "us-west-1"), + ("UsWest2", "us-west-2"), + ("CaCentral1", "ca-central-1"), + ("AfSouth1", "af-south-1"), + ("ApEast1", "ap-east-1"), + ("ApSouth1", "ap-south-1"), + ("ApNortheast1", "ap-northeast-1"), + ("ApNortheast2", "ap-northeast-2"), + ("ApNortheast3", "ap-northeast-3"), + ("ApSoutheast1", "ap-southeast-1"), + ("ApSoutheast2", "ap-southeast-2"), + ("CnNorth1", "cn-north-1"), + ("CnNorthwest1", "cn-northwest-1"), + ("EuNorth1", "eu-north-1"), + ("EuCentral1", "eu-central-1"), + ("EuCentral2", "eu-central-2"), + ("EuWest1", "eu-west-1"), + ("EuWest2", "eu-west-2"), + ("EuWest3", "eu-west-3"), + ("IlCentral1", "il-central-1"), + ("MeSouth1", "me-south-1"), + ("SaEast1", "sa-east-1"), + ]; + for (legacy, expected) in mappings { + let config: S3Config = serde_json::from_value(serde_json::json!({ + "bucket_name": "pkgly", + "region": legacy, + "credentials": {}, + })) + .expect("legacy region should deserialize"); + assert_eq!(config.region.as_deref(), Some(expected)); + } +} + +#[test] +fn unknown_region_round_trips_as_raw_string() { + let config: S3Config = serde_json::from_value(serde_json::json!({ + "bucket_name": "pkgly", + "region": "provider-special-1", + "credentials": {}, + })) + .expect("raw region should deserialize"); + let serialized = serde_json::to_value(config).expect("config should serialize"); + assert_eq!(serialized["region"], "provider-special-1"); +} + +#[test] +fn blank_region_is_rejected_without_custom_endpoint() { + let config = S3Config { + bucket_name: "pkgly".into(), + region: Some(" ".into()), + custom_region: None, + credentials: S3Credentials::default(), + path_style: true, + cache: S3CacheConfig::default(), + adaptive_buffer: AdaptiveBufferConfig::default(), + }; + + assert!(matches!( + config.resolved_region(), + Err(super::S3StorageError::NoRegionSpecified) + )); +} + #[test] fn body_strategy_caches_small_objects() { let limit = DEFAULT_MAX_BUFFERED_OBJECT_BYTES; @@ -103,6 +318,21 @@ fn cache_config_with_dir(dir: &std::path::Path) -> S3CacheConfig { } } +#[test] +fn empty_cache_path_uses_storage_default() { + let config = S3CacheConfig { + enabled: true, + path: Some(std::path::PathBuf::new()), + max_bytes: 8, + max_entries: 4, + }; + + assert_eq!( + resolve_cache_dir(&config, "blank-cache-path"), + default_cache_dir("blank-cache-path") + ); +} + #[tokio::test] async fn disk_cache_retries_failed_deletions_on_next_put() { let temp_dir = tempdir().expect("tempdir"); @@ -115,7 +345,15 @@ async fn disk_cache_retries_failed_deletions_on_next_put() { .await .expect("initial write"); - let relative = S3DiskCache::hashed_filename("first"); + let relative = { + let state = cache.state.lock().await; + state + .entries + .peek("first") + .expect("first cache entry") + .relative_path + .clone() + }; let disk_path = cache.dir.join(&relative); fs::remove_file(&disk_path) .await @@ -150,122 +388,631 @@ async fn disk_cache_retries_failed_deletions_on_next_put() { assert!(!exists, "failed deletions get retried before new puts"); } -#[test] -fn adaptive_buffer_respects_pressure_threshold() { - let config = AdaptiveBufferConfig { - min_buffer_bytes: 1024 * 1024, - max_buffer_bytes: 16 * 1024 * 1024, - memory_pressure_threshold: 0.5, +#[tokio::test] +async fn disk_cache_recovers_entries_and_preserves_unrelated_files() { + let temp_dir = tempdir().expect("tempdir"); + let sentinel = temp_dir.path().join("sentinel.txt"); + fs::write(&sentinel, b"leave me alone") + .await + .expect("sentinel"); + + let cache_config = S3CacheConfig { + max_bytes: 64, + ..cache_config_with_dir(temp_dir.path()) }; + let cache = S3DiskCache::new(&cache_config, "test-cache") + .await + .expect("cache"); + cache + .put( + "recover", + Bytes::from_static(b"persisted"), + Some("text/plain"), + ) + .await + .expect("write cache entry"); + drop(cache); - assert_eq!(config.limit_for_pressure(0.0), 16 * 1024 * 1024); - let mid = config.limit_for_pressure(0.25); - assert!(mid < 16 * 1024 * 1024 && mid > 1024 * 1024); - assert_eq!(config.limit_for_pressure(0.5), 1024 * 1024); - assert_eq!(config.limit_for_pressure(0.9), 1024 * 1024); + let recovered = S3DiskCache::new(&cache_config, "test-cache") + .await + .expect("recovered cache"); + let object = recovered + .get("recover") + .await + .expect("cache read") + .expect("recovered object"); + assert_eq!(object.bytes, Bytes::from_static(b"persisted")); + assert_eq!(object.content_type.as_deref(), Some("text/plain")); + assert_eq!( + fs::read(&sentinel).await.expect("sentinel read"), + b"leave me alone" + ); } -// --------------------------------------------------------------------------- -// delete_repository integration tests against a lightweight mock S3 endpoint. -// --------------------------------------------------------------------------- -use super::{S3Storage, S3StorageInner}; -use crate::{Storage, StorageConfigInner}; -use aws_config::BehaviorVersion; -use aws_credential_types::{Credentials as AwsCredentials, provider::SharedCredentialsProvider}; -use aws_sdk_s3::config::Builder as S3ConfigBuilder; -use aws_types::region::Region; -use bytes::Buf; -use http_body_util::{BodyExt, Full}; -use hyper::{ - Request, Response, StatusCode, body::Incoming, server::conn::http1, service::service_fn, -}; -use hyper_util::rt::TokioIo; -use parking_lot::Mutex; -use std::{collections::VecDeque, convert::Infallible, net::SocketAddr, sync::Arc}; -use tokio::net::TcpListener; -use uuid::Uuid; - -type RecordedBody = bytes::Bytes; -type Responder = - Box) -> Response> + Send + 'static>; +#[tokio::test] +async fn disk_cache_recovers_latest_same_key_publication() { + let temp_dir = tempdir().expect("tempdir"); + let cache_config = S3CacheConfig { + max_bytes: 64, + ..cache_config_with_dir(temp_dir.path()) + }; + let cache = S3DiskCache::new(&cache_config, "test-cache") + .await + .expect("cache"); + cache + .put("replaced", Bytes::from_static(b"old"), Some("text/plain")) + .await + .expect("initial cache entry"); + cache + .put( + "replaced", + Bytes::from_static(b"latest"), + Some("text/plain"), + ) + .await + .expect("replacement cache entry"); + drop(cache); -#[derive(Clone, Debug)] -struct RecordedRequest { - method: hyper::Method, - uri: hyper::Uri, - body: RecordedBody, + let recovered = S3DiskCache::new(&cache_config, "test-cache") + .await + .expect("recovered cache"); + let object = recovered + .get("replaced") + .await + .expect("cache read") + .expect("latest entry should recover"); + assert_eq!(object.bytes, Bytes::from_static(b"latest")); } -struct MockS3Server { - address: SocketAddr, - requests: Arc>>, - shutdown: tokio::sync::oneshot::Sender<()>, - task: tokio::task::JoinHandle<()>, -} +#[tokio::test] +async fn disk_cache_recovery_preserves_object_timestamp() { + let temp_dir = tempdir().expect("tempdir"); + let cache_config = S3CacheConfig { + max_bytes: 64, + ..cache_config_with_dir(temp_dir.path()) + }; + let cache = S3DiskCache::new(&cache_config, "test-cache") + .await + .expect("cache"); + let timestamp = FixedOffset::east_opt(0) + .unwrap() + .with_ymd_and_hms(2025, 1, 1, 0, 0, 0) + .unwrap(); + cache + .put_with_metadata( + "timestamp", + Bytes::from_static(b"payload"), + Some("text/plain"), + Some(timestamp), + ) + .await + .expect("write cache entry"); + drop(cache); -impl MockS3Server { - async fn start(responders: Vec) -> Self { - let listener = TcpListener::bind(("127.0.0.1", 0)) + let recovered = S3DiskCache::new(&cache_config, "test-cache") + .await + .expect("recovered cache"); + assert_eq!( + recovered + .get("timestamp") .await - .expect("bind mock s3"); - let address = listener.local_addr().expect("address"); - let requests = Arc::new(Mutex::new(Vec::new())); - let responder_queue = Arc::new(Mutex::new(VecDeque::from(responders))); - - let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel::<()>(); - let requests_clone = Arc::clone(&requests); - let responders_clone = Arc::clone(&responder_queue); - - let task = tokio::spawn(async move { - loop { - tokio::select! { - _ = &mut shutdown_rx => break, - accept = listener.accept() => { - let (stream, _) = match accept { - Ok(ok) => ok, - Err(err) => { - eprintln!("mock s3 accept error: {err}"); - continue; - } - }; - let requests = Arc::clone(&requests_clone); - let responders = Arc::clone(&responders_clone); - tokio::spawn(async move { - let service = service_fn(move |req: Request| { - handle_request(req, Arc::clone(&requests), Arc::clone(&responders)) - }); - if let Err(err) = http1::Builder::new() - .serve_connection(TokioIo::new(stream), service) - .await - { - eprintln!("mock s3 connection error: {err}"); - } - }); - } - } - } - }); + .expect("cache read") + .expect("entry") + .last_modified, + Some(timestamp) + ); +} - Self { - address, - requests, - shutdown: shutdown_tx, - task, - } - } +#[tokio::test] +async fn disk_cache_discards_missing_or_malformed_sidecars() { + let temp_dir = tempdir().expect("tempdir"); + let cache_config = S3CacheConfig { + max_bytes: 64, + ..cache_config_with_dir(temp_dir.path()) + }; + let cache = S3DiskCache::new(&cache_config, "test-cache") + .await + .expect("cache"); + cache + .put("malformed", Bytes::from_static(b"payload"), None) + .await + .expect("malformed entry"); + cache + .put("missing-sidecar", Bytes::from_static(b"payload"), None) + .await + .expect("missing sidecar entry"); + let (malformed_meta, missing_meta) = { + let state = cache.state.lock().await; + let malformed = state.entries.peek("malformed").expect("malformed entry"); + let missing = state + .entries + .peek("missing-sidecar") + .expect("missing entry"); + ( + cache + .dir + .join(S3DiskCache::metadata_filename(&malformed.relative_path)), + cache + .dir + .join(S3DiskCache::metadata_filename(&missing.relative_path)), + ) + }; + fs::write(&malformed_meta, b"not json") + .await + .expect("malformed sidecar"); + fs::remove_file(&missing_meta) + .await + .expect("remove sidecar"); + drop(cache); - fn endpoint(&self) -> String { - format!("http://{}", self.address) - } + let recovered = S3DiskCache::new(&cache_config, "test-cache") + .await + .expect("recovered cache"); + assert!( + recovered + .get("malformed") + .await + .expect("malformed read") + .is_none() + ); + assert!( + recovered + .get("missing-sidecar") + .await + .expect("missing read") + .is_none() + ); +} - fn take_requests(&self) -> Vec { - self.requests.lock().clone() - } +#[tokio::test] +async fn disk_cache_treats_corrupt_content_as_a_miss() { + let temp_dir = tempdir().expect("tempdir"); + let cache_config = cache_config_with_dir(temp_dir.path()); + let cache = S3DiskCache::new(&cache_config, "test-cache") + .await + .expect("cache"); + cache + .put("corrupt", Bytes::from_static(b"original"), None) + .await + .expect("write cache entry"); + + let relative = { + let state = cache.state.lock().await; + state + .entries + .peek("corrupt") + .expect("cache entry") + .relative_path + .clone() + }; + fs::write(cache.dir.join(relative), b"tampered") + .await + .expect("corrupt cache entry"); - async fn shutdown(self) { - let _ = self.shutdown.send(()); - let _ = self.task.await; - } + assert!(cache.get("corrupt").await.expect("cache read").is_none()); +} + +#[tokio::test] +async fn disk_cache_recovers_generation_and_removes_owned_orphans() { + let temp_dir = tempdir().expect("tempdir"); + let cache_config = S3CacheConfig { + max_bytes: 64, + ..cache_config_with_dir(temp_dir.path()) + }; + let cache = S3DiskCache::new(&cache_config, "test-cache") + .await + .expect("cache"); + cache + .put("kept", Bytes::from_static(b"kept"), None) + .await + .expect("write cache entry"); + let content_path = { + let state = cache.state.lock().await; + state + .entries + .peek("kept") + .expect("entry") + .relative_path + .clone() + }; + let parent = temp_dir.path().join(content_path.parent().expect("parent")); + let base = S3DiskCache::hashed_filename("orphan"); + let orphan_generation = temp_dir + .path() + .join(base.parent().expect("base parent")) + .join(format!( + "{}.gen-abcdef", + base.file_name().expect("base file").to_string_lossy() + )); + fs::create_dir_all(orphan_generation.parent().unwrap()) + .await + .expect("orphan parent"); + fs::write(&orphan_generation, b"orphan") + .await + .expect("orphan generation"); + let legacy = temp_dir.path().join("aa").join("0".repeat(62)); + fs::create_dir_all(legacy.parent().unwrap()) + .await + .expect("legacy parent"); + fs::write(&legacy, b"legacy") + .await + .expect("legacy artifact"); + let unrelated = parent.join("notes.txt"); + fs::write(&unrelated, b"keep this") + .await + .expect("unrelated"); + let unrelated_temp = parent.join("notes.tmp-abcdef"); + fs::write(&unrelated_temp, b"keep this too") + .await + .expect("unrelated temp"); + let unrelated_metadata = parent.join("notes.meta.json"); + fs::write(&unrelated_metadata, b"keep this metadata too") + .await + .expect("unrelated metadata"); + drop(cache); + + let recovered = S3DiskCache::new(&cache_config, "test-cache") + .await + .expect("reopen cache"); + assert!(recovered.get("kept").await.expect("cache read").is_some()); + assert!( + !fs::try_exists(&orphan_generation) + .await + .expect("orphan exists") + ); + assert!(!fs::try_exists(&legacy).await.expect("legacy exists")); + assert!(fs::try_exists(&unrelated).await.expect("unrelated exists")); + assert!( + fs::try_exists(&unrelated_temp) + .await + .expect("unrelated temp exists") + ); + assert!( + fs::try_exists(&unrelated_metadata) + .await + .expect("unrelated metadata exists") + ); +} + +#[cfg(unix)] +#[tokio::test] +async fn disk_cache_rejects_symlinked_content() { + use std::os::unix::fs::symlink; + + let temp_dir = tempdir().expect("tempdir"); + let outside = tempdir().expect("outside tempdir"); + let cache_config = S3CacheConfig { + max_bytes: 64, + ..cache_config_with_dir(temp_dir.path()) + }; + let cache = S3DiskCache::new(&cache_config, "test-cache") + .await + .expect("cache"); + cache + .put("symlink", Bytes::from_static(b"payload"), None) + .await + .expect("write cache entry"); + let relative = { + let state = cache.state.lock().await; + state + .entries + .peek("symlink") + .expect("entry") + .relative_path + .clone() + }; + let content_path = cache.dir.join(relative); + let target = outside.path().join("target"); + fs::write(&target, b"secret").await.expect("target"); + fs::remove_file(&content_path) + .await + .expect("remove content"); + symlink(&target, &content_path).expect("symlink"); + drop(cache); + + let recovered = S3DiskCache::new(&cache_config, "test-cache") + .await + .expect("reopen cache"); + assert!( + recovered + .get("symlink") + .await + .expect("cache read") + .is_none() + ); +} + +#[tokio::test] +async fn disk_cache_recovery_enforces_entry_capacity() { + let temp_dir = tempdir().expect("tempdir"); + let write_config = S3CacheConfig { + max_entries: 4, + max_bytes: 64, + ..cache_config_with_dir(temp_dir.path()) + }; + let cache = S3DiskCache::new(&write_config, "test-cache") + .await + .expect("cache"); + cache + .put("old", Bytes::from_static(b"old"), None) + .await + .expect("old entry"); + sleep(Duration::from_millis(2)).await; + cache + .put("new", Bytes::from_static(b"new"), None) + .await + .expect("new entry"); + drop(cache); + + let read_config = S3CacheConfig { + max_entries: 1, + ..write_config + }; + let recovered = S3DiskCache::new(&read_config, "test-cache") + .await + .expect("reopen cache"); + assert!(recovered.get("old").await.expect("old read").is_none()); + assert!(recovered.get("new").await.expect("new read").is_some()); +} + +#[tokio::test] +async fn disk_cache_does_not_insert_oversized_objects() { + let temp_dir = tempdir().expect("tempdir"); + let config = S3CacheConfig { + max_bytes: 3, + ..cache_config_with_dir(temp_dir.path()) + }; + let cache = S3DiskCache::new(&config, "test-cache") + .await + .expect("cache"); + cache + .put("too-large", Bytes::from_static(b"four"), None) + .await + .expect("oversized writes are ignored"); + assert!(cache.get("too-large").await.expect("cache read").is_none()); +} + +#[tokio::test] +async fn disk_cache_concurrent_publications_leave_a_valid_entry() { + let temp_dir = tempdir().expect("tempdir"); + let config = S3CacheConfig { + max_bytes: 128, + ..cache_config_with_dir(temp_dir.path()) + }; + let cache = Arc::new( + S3DiskCache::new(&config, "test-cache") + .await + .expect("cache"), + ); + let mut tasks = Vec::new(); + for index in 0..8u8 { + let cache = Arc::clone(&cache); + tasks.push(tokio::spawn(async move { + cache + .put("same-key", Bytes::from(vec![index; 8]), None) + .await + .expect("concurrent publication"); + })); + } + for task in tasks { + task.await.expect("publication task"); + } + let object = cache + .get("same-key") + .await + .expect("cache read") + .expect("entry remains"); + assert_eq!(object.bytes.len(), 8); +} + +#[test] +fn adaptive_buffer_respects_pressure_threshold() { + let config = AdaptiveBufferConfig { + min_buffer_bytes: 1024 * 1024, + max_buffer_bytes: 16 * 1024 * 1024, + memory_pressure_threshold: 0.5, + }; + + assert_eq!(config.limit_for_pressure(0.0), 16 * 1024 * 1024); + let mid = config.limit_for_pressure(0.25); + assert!(mid < 16 * 1024 * 1024 && mid > 1024 * 1024); + assert_eq!(config.limit_for_pressure(0.5), 1024 * 1024); + assert_eq!(config.limit_for_pressure(0.9), 1024 * 1024); +} + +#[test] +fn memory_snapshot_values_are_already_bytes() { + let snapshot = MemorySnapshot::from_values(100, 25); + assert_eq!(snapshot.total_bytes, 100); + assert_eq!(snapshot.available_bytes, 25); + assert!((snapshot.pressure() - 0.75).abs() < f64::EPSILON); +} + +#[test] +fn memory_snapshot_cache_reuses_values_until_expiry() { + let mut cache = MemorySnapshotCache::default(); + let start = tokio::time::Instant::now(); + let mut captures = 0; + + let first = cache.get_or_capture(start, || { + captures += 1; + Some(MemorySnapshot::from_values(100, 50)) + }); + let second = cache.get_or_capture(start + Duration::from_secs(1), || { + captures += 1; + Some(MemorySnapshot::from_values(200, 100)) + }); + let third = cache.get_or_capture(start + Duration::from_secs(6), || { + captures += 1; + Some(MemorySnapshot::from_values(200, 100)) + }); + + assert_eq!(captures, 2); + assert_eq!(first.expect("first").total_bytes, 100); + assert_eq!(second.expect("second").total_bytes, 100); + assert_eq!(third.expect("third").total_bytes, 200); +} + +#[test] +fn memory_snapshot_cache_reuses_zero_memory_result_until_expiry() { + let mut cache = MemorySnapshotCache::default(); + let start = tokio::time::Instant::now(); + let mut captures = 0; + + let first = cache.get_or_capture(start, || { + captures += 1; + None + }); + let second = cache.get_or_capture(start + Duration::from_secs(1), || { + captures += 1; + Some(MemorySnapshot::from_values(100, 50)) + }); + + assert!(first.is_none()); + assert!(second.is_none()); + assert_eq!(captures, 1); +} + +#[test] +fn memory_limits_prefer_valid_lower_cgroup_limit() { + assert_eq!( + super::effective_memory_limits(1_000, 900, Some((400, 500))), + (400, 400) + ); + assert_eq!( + super::effective_memory_limits(1_000, 900, Some((2_000, 100))), + (1_000, 900) + ); + assert_eq!(super::effective_memory_limits(0, 0, Some((0, 0))), (0, 0)); +} + +#[test] +fn zero_total_memory_pressure_is_safely_saturated() { + let snapshot = MemorySnapshot::from_values(0, 0); + assert_eq!(snapshot.pressure(), 1.0); +} + +// --------------------------------------------------------------------------- +// delete_repository integration tests against a lightweight mock S3 endpoint. +// --------------------------------------------------------------------------- +use super::{S3Storage, S3StorageInner}; +use crate::{FileContent, Storage, StorageConfigInner}; +use aws_config::BehaviorVersion; +use aws_credential_types::{Credentials as AwsCredentials, provider::SharedCredentialsProvider}; +use aws_sdk_s3::config::Builder as S3ConfigBuilder; +use aws_smithy_types::retry::RetryConfig; +use aws_types::region::Region; +use bytes::Buf; +use http_body_util::{BodyExt, Full}; +use hyper::{ + Request, Response, StatusCode, body::Incoming, header, server::conn::http1, service::service_fn, +}; +use hyper_util::rt::TokioIo; +use nr_core::storage::StoragePath; +use parking_lot::Mutex; +use std::{collections::VecDeque, convert::Infallible, net::SocketAddr, sync::Arc}; +use tokio::net::TcpListener; +use uuid::Uuid; + +type RecordedBody = bytes::Bytes; +type Responder = Box) -> ResponsePlan + Send + 'static>; + +struct ResponsePlan { + delay: Duration, + response: Response>, +} + +impl ResponsePlan { + fn immediate(response: Response>) -> Self { + Self { + delay: Duration::ZERO, + response, + } + } + + fn delayed(delay: Duration, response: Response>) -> Self { + Self { delay, response } + } +} + +#[derive(Clone, Debug)] +struct RecordedRequest { + method: hyper::Method, + uri: hyper::Uri, + body: RecordedBody, + headers: hyper::HeaderMap, +} + +struct MockS3Server { + address: SocketAddr, + requests: Arc>>, + shutdown: tokio::sync::oneshot::Sender<()>, + task: tokio::task::JoinHandle<()>, +} + +impl MockS3Server { + async fn start(responders: Vec) -> Self { + let listener = TcpListener::bind(("127.0.0.1", 0)) + .await + .expect("bind mock s3"); + let address = listener.local_addr().expect("address"); + let requests = Arc::new(Mutex::new(Vec::new())); + let responder_queue = Arc::new(Mutex::new(VecDeque::from(responders))); + + let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel::<()>(); + let requests_clone = Arc::clone(&requests); + let responders_clone = Arc::clone(&responder_queue); + + let task = tokio::spawn(async move { + loop { + tokio::select! { + _ = &mut shutdown_rx => break, + accept = listener.accept() => { + let (stream, _) = match accept { + Ok(ok) => ok, + Err(err) => { + eprintln!("mock s3 accept error: {err}"); + continue; + } + }; + let requests = Arc::clone(&requests_clone); + let responders = Arc::clone(&responders_clone); + tokio::spawn(async move { + let service = service_fn(move |req: Request| { + handle_request(req, Arc::clone(&requests), Arc::clone(&responders)) + }); + if let Err(err) = http1::Builder::new() + .serve_connection(TokioIo::new(stream), service) + .await + { + eprintln!("mock s3 connection error: {err}"); + } + }); + } + } + } + }); + + Self { + address, + requests, + shutdown: shutdown_tx, + task, + } + } + + fn endpoint(&self) -> String { + format!("http://{}", self.address) + } + + fn take_requests(&self) -> Vec { + self.requests.lock().clone() + } + + async fn shutdown(self) { + let _ = self.shutdown.send(()); + let _ = self.task.await; + } } async fn handle_request( @@ -281,22 +1028,31 @@ async fn handle_request( method: req_with_body.method().clone(), uri: req_with_body.uri().clone(), body: body_bytes.clone(), + headers: req_with_body.headers().clone(), }); - let mut queue = responders.lock(); - let resp = if let Some(responder) = queue.front_mut() { - responder(req_with_body) - } else { - Response::builder() - .status(StatusCode::INTERNAL_SERVER_ERROR) - .body(Full::new(bytes::Bytes::from_static( - b"no responder for request", - ))) - .unwrap() + let plan = { + let mut queue = responders.lock(); + let plan = if let Some(responder) = queue.front_mut() { + responder(req_with_body) + } else { + ResponsePlan::immediate( + Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body(Full::new(bytes::Bytes::from_static( + b"no responder for request", + ))) + .unwrap(), + ) + }; + // pop only after a successful response to preserve strict ordering on errors + queue.pop_front(); + plan }; - // pop only after a successful response to preserve strict ordering on errors - queue.pop_front(); - Ok(resp) + if !plan.delay.is_zero() { + sleep(plan.delay).await; + } + Ok(plan.response) } fn build_s3_storage(endpoint: &str, bucket: &str) -> S3Storage { @@ -308,6 +1064,7 @@ fn build_s3_storage(endpoint: &str, bucket: &str) -> S3Storage { .behavior_version(BehaviorVersion::latest()) .force_path_style(true) .endpoint_url(endpoint) + .retry_config(RetryConfig::standard().with_max_attempts(1)) .credentials_provider(SharedCredentialsProvider::new(AwsCredentials::new( "AKIA", "SECRET", None, None, "mock", ))) @@ -316,7 +1073,7 @@ fn build_s3_storage(endpoint: &str, bucket: &str) -> S3Storage { let client = aws_sdk_s3::Client::from_conf(client_config); let config = S3Config { bucket_name: bucket.into(), - region: Some(S3StorageRegion::UsEast1), + region: Some("us-east-1".into()), custom_region: Some(CustomRegion { custom_region: Some("us-east-1".into()), endpoint: endpoint.parse().unwrap(), @@ -332,36 +1089,122 @@ fn build_s3_storage(endpoint: &str, bucket: &str) -> S3Storage { storage_config, client, cache: None, + manifest_cache: parking_lot::Mutex::new(super::ManifestCache::new()), + manifest_load_lock: tokio::sync::Mutex::new(()), }; S3Storage::from(inner) } -fn list_response_body(prefix: &str, keys: &[&str], truncated: bool, token: Option<&str>) -> String { - let contents = keys - .iter() - .map(|k| format!("{}", k)) - .collect::>() - .join(""); - let token_xml = token - .map(|t| format!("{t}")) - .unwrap_or_default(); - format!( - r#" - mock-bucket - {prefix} - {} - {} - {contents} - {token_xml} -"#, - keys.len(), - if truncated { "true" } else { "false" } - ) -} +fn build_s3_storage_with_cache(endpoint: &str, bucket: &str, cache: Arc) -> S3Storage { + let mut storage_config = StorageConfigInner::test_config(); + storage_config.storage_type = "s3".into(); -fn delete_ok_response() -> Response> { - Response::builder() + let client_config = S3ConfigBuilder::new() + .region(Region::new("us-east-1")) + .behavior_version(BehaviorVersion::latest()) + .force_path_style(true) + .endpoint_url(endpoint) + .retry_config(RetryConfig::standard().with_max_attempts(1)) + .credentials_provider(SharedCredentialsProvider::new(AwsCredentials::new( + "AKIA", "SECRET", None, None, "mock", + ))) + .build(); + + let client = aws_sdk_s3::Client::from_conf(client_config); + let config = S3Config { + bucket_name: bucket.into(), + region: Some("us-east-1".into()), + custom_region: Some(CustomRegion { + custom_region: Some("us-east-1".into()), + endpoint: endpoint.parse().unwrap(), + }), + credentials: S3Credentials::new_access_key("AKIA", "SECRET"), + path_style: true, + cache: S3CacheConfig { + enabled: true, + path: Some(cache.dir.clone()), + max_bytes: cache.max_bytes, + max_entries: 4, + }, + adaptive_buffer: AdaptiveBufferConfig::default(), + }; + + S3Storage::from(S3StorageInner { + config, + storage_config, + client, + cache: Some(cache), + manifest_cache: parking_lot::Mutex::new(super::ManifestCache::new()), + manifest_load_lock: tokio::sync::Mutex::new(()), + }) +} + +fn list_response_body(prefix: &str, keys: &[&str], truncated: bool, token: Option<&str>) -> String { + let contents = keys + .iter() + .map(|k| format!("{}", k)) + .collect::>() + .join(""); + let token_xml = token + .map(|t| format!("{t}")) + .unwrap_or_default(); + format!( + r#" + mock-bucket + {prefix} + {} + {} + {contents} + {token_xml} +"#, + keys.len(), + if truncated { "true" } else { "false" } + ) +} + +fn list_response_body_with_metadata( + prefix: &str, + objects: &[(&str, u64, Option<&str>)], + common_prefixes: &[&str], + truncated: bool, + token: Option<&str>, +) -> String { + let contents = objects + .iter() + .map(|(key, size, last_modified)| { + let last_modified = last_modified + .map(|value| format!("{value}")) + .unwrap_or_default(); + format!("{key}{last_modified}{size}") + }) + .collect::>() + .join(""); + let prefixes = common_prefixes + .iter() + .map(|prefix| format!("{prefix}")) + .collect::>() + .join(""); + let token_xml = token + .map(|value| format!("{value}")) + .unwrap_or_default(); + format!( + r#" + mock-bucket + {prefix} + {} + {} + {contents} + {prefixes} + {token_xml} +"#, + objects.len() + common_prefixes.len(), + if truncated { "true" } else { "false" } + ) +} + +fn delete_ok_response() -> Response> { + Response::builder() .status(StatusCode::OK) .body(Full::new(bytes::Bytes::from_static( br#""#, @@ -371,10 +1214,12 @@ fn delete_ok_response() -> Response> { fn respond_list(body: String) -> Responder { Box::new(move |_| { - Response::builder() - .status(StatusCode::OK) - .body(Full::new(bytes::Bytes::from(body.clone()))) - .unwrap() + ResponsePlan::immediate( + Response::builder() + .status(StatusCode::OK) + .body(Full::new(bytes::Bytes::from(body.clone()))) + .unwrap(), + ) }) } @@ -388,169 +1233,1338 @@ fn respond_delete(assert_keys: Vec) -> Responder { "delete payload should contain key {key}, payload was {body_text}" ); } - delete_ok_response() + ResponsePlan::immediate(delete_ok_response()) }) } -fn parse_deleted_keys(body: &RecordedBody) -> Vec { - let text = String::from_utf8_lossy(body); - text.split("") - .skip(1) - .filter_map(|part| part.split("").next()) - .map(|s| s.to_string()) - .collect() +fn respond_response(response: Response>) -> Responder { + Box::new(move |_| ResponsePlan::immediate(response.clone())) +} + +fn respond_delayed(delay: Duration, response: Response>) -> Responder { + Box::new(move |_| ResponsePlan::delayed(delay, response.clone())) +} + +fn response_with_body( + status: StatusCode, + body: impl Into, +) -> Response> { + Response::builder() + .status(status) + .body(Full::new(body.into())) + .unwrap() +} + +fn not_found_response(code: &str) -> Response> { + response_with_body( + StatusCode::NOT_FOUND, + bytes::Bytes::from(format!( + r#"{code}missing"# + )), + ) +} + +fn conditional_error_response(status: StatusCode, code: &str) -> Response> { + response_with_body( + status, + bytes::Bytes::from(format!( + r#"{code}conditional write failed"# + )), + ) +} + +fn head_response( + status: StatusCode, + etag: Option<&str>, + size: Option, + content_type: Option<&str>, + last_modified: Option<&str>, +) -> Response> { + let mut builder = Response::builder().status(status); + if let Some(etag) = etag { + builder = builder.header(header::ETAG, etag); + } + if let Some(size) = size { + builder = builder.header(header::CONTENT_LENGTH, size.to_string()); + } + if let Some(content_type) = content_type { + builder = builder.header(header::CONTENT_TYPE, content_type); + } + if let Some(last_modified) = last_modified { + builder = builder.header(header::LAST_MODIFIED, last_modified); + } + builder.body(Full::new(bytes::Bytes::new())).unwrap() +} + +fn get_response( + body: &'static [u8], + etag: Option<&str>, + content_type: Option<&str>, + last_modified: Option<&str>, +) -> Response> { + let mut builder = Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_LENGTH, body.len().to_string()); + if let Some(etag) = etag { + builder = builder.header(header::ETAG, etag); + } + if let Some(content_type) = content_type { + builder = builder.header(header::CONTENT_TYPE, content_type); + } + if let Some(last_modified) = last_modified { + builder = builder.header(header::LAST_MODIFIED, last_modified); + } + builder + .body(Full::new(bytes::Bytes::from_static(body))) + .unwrap() +} + +fn copy_response(etag: &str) -> Response> { + response_with_body( + StatusCode::OK, + bytes::Bytes::from(format!( + r#"{etag}2025-01-01T00:00:00.000Z"# + )), + ) +} + +fn multipart_create_response(upload_id: &str) -> Response> { + response_with_body( + StatusCode::OK, + bytes::Bytes::from(format!( + r#"{upload_id}"# + )), + ) +} + +fn multipart_part_response(etag: &str) -> Response> { + response_with_body( + StatusCode::OK, + bytes::Bytes::from(format!( + r#"{etag}2025-01-01T00:00:00.000Z"# + )), + ) +} + +fn multipart_complete_response() -> Response> { + response_with_body( + StatusCode::OK, + bytes::Bytes::from_static( + br#"\"complete\""#, + ), + ) +} + +fn parse_deleted_keys(body: &RecordedBody) -> Vec { + let text = String::from_utf8_lossy(body); + text.split("") + .skip(1) + .filter_map(|part| part.split("").next()) + .map(|s| s.to_string()) + .collect() +} + +#[tokio::test] +async fn delayed_responder_can_be_queued_for_timeout_tests() { + let server = MockS3Server::start(vec![respond_delayed( + Duration::from_millis(1), + response_with_body(StatusCode::OK, bytes::Bytes::new()), + )]) + .await; + let mut stream = TcpStream::connect(server.address) + .await + .expect("connect delayed responder"); + stream + .write_all(b"GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n") + .await + .expect("write request"); + let mut response = Vec::new(); + stream + .read_to_end(&mut response) + .await + .expect("read response"); + assert!(response.starts_with(b"HTTP/1.1 200 OK")); + server.shutdown().await; +} + +#[tokio::test] +async fn delete_repository_removes_all_files() { + let repository = Uuid::new_v4(); + let key_one = format!("{repository}/packages/a.bin"); + let key_two = format!("{repository}/packages/nested/b.bin"); + + let server = MockS3Server::start(vec![ + respond_list(list_response_body( + &format!("{repository}/"), + &[&key_one, &key_two], + false, + None, + )), + respond_delete(vec![key_one.clone(), key_two.clone()]), + ]) + .await; + + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + storage + .delete_repository(repository) + .await + .expect("delete_repository should succeed"); + + let requests = server.take_requests(); + assert_eq!(requests.len(), 2, "one list and one delete call expected"); + let delete_keys = parse_deleted_keys(&requests[1].body); + assert_eq!(delete_keys.len(), 2); + assert!(delete_keys.contains(&key_one)); + assert!(delete_keys.contains(&key_two)); + + server.shutdown().await; +} + +#[tokio::test] +async fn delete_repository_handles_pagination() { + let repository = Uuid::new_v4(); + let first_page_keys = vec![ + format!("{repository}/page1/one"), + format!("{repository}/page1/two"), + ]; + let second_page_keys = vec![format!("{repository}/page2/three")]; + + let server = MockS3Server::start(vec![ + respond_list(list_response_body( + &format!("{repository}/"), + &first_page_keys + .iter() + .map(|s| s.as_str()) + .collect::>(), + true, + Some("token-1"), + )), + respond_delete(first_page_keys.clone()), + respond_list(list_response_body( + &format!("{repository}/"), + &second_page_keys + .iter() + .map(|s| s.as_str()) + .collect::>(), + false, + None, + )), + respond_delete(second_page_keys.clone()), + ]) + .await; + + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + storage + .delete_repository(repository) + .await + .expect("delete_repository should succeed with pagination"); + + let requests = server.take_requests(); + assert_eq!( + requests.len(), + 4, + "list/delete/list/delete sequence expected" + ); + assert!( + requests[2] + .uri + .query() + .unwrap_or_default() + .contains("continuation-token=token-1"), + "second list should carry continuation token" + ); + + server.shutdown().await; +} + +#[tokio::test] +async fn delete_repository_is_idempotent_on_empty_prefix() { + let repository = Uuid::new_v4(); + let server = MockS3Server::start(vec![respond_list(list_response_body( + &format!("{repository}/"), + &[], + false, + None, + ))]) + .await; + + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + storage + .delete_repository(repository) + .await + .expect("empty prefixes should be handled gracefully"); + + let requests = server.take_requests(); + assert_eq!(requests.len(), 1, "only a list request is expected"); + + server.shutdown().await; +} + +#[tokio::test] +async fn delete_repository_preserves_other_repositories() { + let repository = Uuid::new_v4(); + let other_repo = Uuid::new_v4(); + + let keys = [ + format!("{repository}/packages/a.bin"), + format!("{other_repo}/packages/should-not-delete"), + ]; + + let server = MockS3Server::start(vec![ + respond_list(list_response_body( + &format!("{repository}/"), + &keys.iter().map(|s| s.as_str()).collect::>(), + false, + None, + )), + respond_delete(vec![format!("{repository}/packages/a.bin")]), + ]) + .await; + + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + storage + .delete_repository(repository) + .await + .expect("delete_repository should ignore other repo keys"); + + let requests = server.take_requests(); + assert_eq!(requests.len(), 2); + let deleted = parse_deleted_keys(&requests[1].body); + assert_eq!(deleted, vec![format!("{repository}/packages/a.bin")]); + assert!( + !deleted + .iter() + .any(|key| key.contains(&other_repo.to_string())), + "keys from other repositories must not be deleted" + ); + + server.shutdown().await; +} + +#[tokio::test] +async fn append_uses_if_match_and_returns_appended_bytes() { + let repository = Uuid::new_v4(); + let server = MockS3Server::start(vec![ + respond_response(get_response( + b"old", + Some("\"etag-old\""), + Some("application/octet-stream"), + None, + )), + respond_response(response_with_body(StatusCode::OK, bytes::Bytes::new())), + ]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + let appended = storage + .append_file( + repository, + FileContent::Bytes(Bytes::from_static(b"new")), + &StoragePath::from("file.bin"), + ) + .await + .expect("append should succeed"); + + assert_eq!(appended, 3); + let requests = server.take_requests(); + assert_eq!(requests.len(), 2); + assert_eq!(requests[0].method, hyper::Method::GET); + assert_eq!(requests[1].method, hyper::Method::PUT); + assert_eq!(requests[1].body, Bytes::from_static(b"oldnew")); + assert_eq!( + requests[1].headers.get(header::IF_MATCH).unwrap(), + "\"etag-old\"" + ); + assert!(requests[1].headers.get(header::IF_NONE_MATCH).is_none()); + server.shutdown().await; +} + +#[tokio::test] +async fn append_uses_if_none_match_for_missing_object() { + let repository = Uuid::new_v4(); + let server = MockS3Server::start(vec![ + respond_response(not_found_response("NoSuchKey")), + respond_response(response_with_body(StatusCode::OK, bytes::Bytes::new())), + ]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + let appended = storage + .append_file( + repository, + FileContent::Content(b"new".to_vec()), + &StoragePath::from("file.bin"), + ) + .await + .expect("append should create missing object"); + + assert_eq!(appended, 3); + let requests = server.take_requests(); + assert_eq!(requests[1].headers.get(header::IF_NONE_MATCH).unwrap(), "*"); + assert!(requests[1].headers.get(header::IF_MATCH).is_none()); + assert_eq!(requests[1].body, Bytes::from_static(b"new")); + server.shutdown().await; +} + +#[tokio::test] +async fn append_conflict_preserves_cache_and_returns_conflict() { + let repository = Uuid::new_v4(); + let server = MockS3Server::start(vec![ + respond_response(get_response(b"old", Some("\"etag-old\""), None, None)), + respond_response(conditional_error_response( + StatusCode::PRECONDITION_FAILED, + "PreconditionFailed", + )), + ]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + let error = storage + .append_file( + repository, + FileContent::Bytes(Bytes::from_static(b"new")), + &StoragePath::from("file.bin"), + ) + .await + .expect_err("stale ETag should produce a conflict"); + + assert!(error.is_conflict()); + assert_eq!(server.take_requests().len(), 2); + server.shutdown().await; +} + +#[tokio::test] +async fn append_rejects_existing_object_without_etag() { + let repository = Uuid::new_v4(); + let server = MockS3Server::start(vec![respond_response(get_response( + b"old", None, None, None, + ))]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + let error = storage + .append_file( + repository, + FileContent::Content(b"new".to_vec()), + &StoragePath::from("file.bin"), + ) + .await + .expect_err("append must not overwrite without an ETag"); + + assert!(matches!(error.kind(), Some(super::S3ErrorKind::Other))); + assert_eq!(server.take_requests().len(), 1); + server.shutdown().await; +} + +#[tokio::test] +async fn append_checks_concrete_ancestors_without_rechecking_final_target() { + let repository = Uuid::new_v4(); + let server = MockS3Server::start(vec![ + respond_response(not_found_response("NotFound")), + respond_response(get_response(b"old", Some("\"etag-old\""), None, None)), + respond_response(response_with_body(StatusCode::OK, bytes::Bytes::new())), + ]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + storage + .append_file( + repository, + FileContent::Content(b"new".to_vec()), + &StoragePath::from("parent/file.bin"), + ) + .await + .expect("append should succeed"); + let requests = server.take_requests(); + assert_eq!(requests.len(), 3); + assert_eq!(requests[0].method, hyper::Method::HEAD); + assert_eq!(requests[1].method, hyper::Method::GET); + assert_eq!(requests[2].method, hyper::Method::PUT); + server.shutdown().await; +} + +#[tokio::test] +async fn save_rejects_concrete_object_ancestor() { + let repository = Uuid::new_v4(); + let server = MockS3Server::start(vec![respond_response(head_response( + StatusCode::OK, + Some("\"parent\""), + Some(1), + None, + None, + ))]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + let error = storage + .save_file( + repository, + FileContent::Content(b"new".to_vec()), + &StoragePath::from("parent/file.bin"), + ) + .await + .expect_err("concrete parent objects must collide"); + assert!(matches!(error, super::S3StorageError::PathCollision(_))); + assert_eq!(server.take_requests().len(), 1); + server.shutdown().await; +} + +#[tokio::test] +async fn get_file_information_paginates_virtual_directory_and_counts_all_entries() { + let repository = Uuid::new_v4(); + let prefix = format!("{repository}/dir/"); + let server = MockS3Server::start(vec![ + respond_response(not_found_response("NotFound")), + respond_list(list_response_body_with_metadata( + &prefix, + &[( + &format!("{prefix}a.bin"), + 3, + Some("2025-01-01T00:00:00.000Z"), + )], + &[&format!("{prefix}nested/")], + true, + Some("page-2"), + )), + respond_list(list_response_body_with_metadata( + &prefix, + &[( + &format!("{prefix}b.bin"), + 4, + Some("2025-01-02T00:00:00.000Z"), + )], + &[], + false, + None, + )), + ]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + let meta = storage + .get_file_information(repository, &StoragePath::from("dir")) + .await + .expect("directory lookup should succeed") + .expect("directory should exist"); + match meta.file_type { + super::FileType::Directory(directory) => assert_eq!(directory.file_count, 3), + other => panic!("expected directory metadata, got {other:?}"), + } + let requests = server.take_requests(); + assert_eq!(requests.len(), 3); + assert!( + requests[2] + .uri + .query() + .unwrap_or_default() + .contains("continuation-token=page-2") + ); + server.shutdown().await; +} + +#[tokio::test] +async fn virtual_directory_uses_placeholder_timestamp_when_available() { + let repository = Uuid::new_v4(); + let prefix = format!("{repository}/dir/"); + let server = MockS3Server::start(vec![ + respond_response(not_found_response("NoSuchKey")), + respond_list(list_response_body_with_metadata( + &prefix, + &[(&prefix, 0, Some("2025-01-04T00:00:00.000Z"))], + &[], + false, + None, + )), + ]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + let meta = storage + .get_file_information(repository, &StoragePath::from("dir")) + .await + .expect("directory lookup") + .expect("directory") + .modified; + assert_eq!(meta.to_rfc3339(), "2025-01-04T00:00:00+00:00"); + server.shutdown().await; +} + +#[tokio::test] +async fn object_timestamps_are_propagated_from_head_and_list() { + let repository = Uuid::new_v4(); + let timestamp = "Wed, 01 Jan 2025 00:00:00 GMT"; + let server = MockS3Server::start(vec![respond_response(head_response( + StatusCode::OK, + Some("\"etag\""), + Some(7), + Some("text/plain"), + Some(timestamp), + ))]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + let meta = storage + .get_file_information(repository, &StoragePath::from("file.txt")) + .await + .expect("head should succeed") + .expect("file should exist"); + assert_eq!(meta.modified.to_rfc3339(), "2025-01-01T00:00:00+00:00"); + assert_eq!(meta.created, meta.modified); + server.shutdown().await; +} + +#[tokio::test] +async fn open_file_propagates_get_last_modified() { + let repository = Uuid::new_v4(); + let server = MockS3Server::start(vec![respond_response(get_response( + b"payload", + Some("\"etag\""), + Some("text/plain"), + Some("Wed, 01 Jan 2025 00:00:00 GMT"), + ))]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + let file = storage + .open_file(repository, &StoragePath::from("file.txt")) + .await + .expect("get should succeed") + .expect("file should exist"); + let (_, meta) = file.file().expect("file result"); + assert_eq!(meta.modified.to_rfc3339(), "2025-01-01T00:00:00+00:00"); + assert_eq!(meta.created, meta.modified); + server.shutdown().await; +} + +#[tokio::test] +async fn open_file_persists_request_timestamp_when_get_omits_last_modified() { + let repository = Uuid::new_v4(); + let temp_dir = tempdir().expect("cache directory"); + let cache = Arc::new( + S3DiskCache::new( + &S3CacheConfig { + max_bytes: 64, + ..cache_config_with_dir(temp_dir.path()) + }, + "test-cache", + ) + .await + .expect("cache"), + ); + let server = MockS3Server::start(vec![respond_response(get_response( + b"payload", + Some("\"etag\""), + Some("text/plain"), + None, + ))]) + .await; + let storage = build_s3_storage_with_cache(&server.endpoint(), "mock-bucket", cache); + + let first = storage + .open_file(repository, &StoragePath::from("file.txt")) + .await + .expect("first get") + .expect("file") + .file() + .expect("file result") + .1 + .modified; + let second = storage + .open_file(repository, &StoragePath::from("file.txt")) + .await + .expect("cache get") + .expect("cached file") + .file() + .expect("cached file result") + .1 + .modified; + + assert_eq!(first, second); + assert_eq!(server.take_requests().len(), 1); + server.shutdown().await; +} + +#[tokio::test] +async fn list_repository_objects_preserves_last_modified() { + let repository = Uuid::new_v4(); + let prefix = format!("{repository}/"); + let server = MockS3Server::start(vec![respond_list(list_response_body_with_metadata( + &prefix, + &[( + &format!("{prefix}manifest.json"), + 9, + Some("2025-01-03T00:00:00.000Z"), + )], + &[], + false, + None, + ))]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + let objects = storage + .list_repository_objects(repository, None) + .await + .expect("list should succeed"); + assert_eq!(objects.len(), 1); + assert_eq!(objects[0].key, "manifest.json"); + assert_eq!(objects[0].size, 9); + assert_eq!( + objects[0].last_modified.unwrap().to_rfc3339(), + "2025-01-03T00:00:00+00:00" + ); + server.shutdown().await; +} + +#[tokio::test] +async fn control_timeout_is_classified_as_network() { + let error = super::with_timeout(Duration::from_millis(5), async { + tokio::time::sleep(Duration::from_millis(50)).await; + Ok::<(), SmithySdkError>(()) + }) + .await + .expect_err("delayed control call should time out"); + assert!(matches!(error.kind(), Some(super::S3ErrorKind::Network))); +} + +#[tokio::test] +async fn copy_move_uses_guarded_server_side_copy_without_get() { + let repository = Uuid::new_v4(); + let source = StoragePath::from("folder/file name.bin"); + let destination = StoragePath::from("folder/moved.bin"); + let server = MockS3Server::start(vec![ + respond_response(head_response( + StatusCode::OK, + Some("\"source-etag\""), + Some(12), + Some("application/octet-stream"), + None, + )), + respond_response(copy_response("\"destination-etag\"")), + respond_response(response_with_body( + StatusCode::NO_CONTENT, + bytes::Bytes::new(), + )), + ]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock bucket"); + + assert!( + storage + .move_file(repository, &source, &destination) + .await + .expect("move should succeed") + ); + let requests = server.take_requests(); + assert_eq!(requests.len(), 3); + assert_eq!(requests[0].method, hyper::Method::HEAD); + assert_eq!(requests[1].method, hyper::Method::PUT); + assert_eq!(requests[2].method, hyper::Method::DELETE); + assert!( + requests + .iter() + .all(|request| request.method != hyper::Method::GET) + ); + assert_eq!( + requests[1] + .headers + .get("x-amz-copy-source") + .unwrap() + .to_str() + .unwrap(), + format!("/mock%20bucket/{repository}%2Ffolder%2Ffile%20name.bin") + ); + assert_eq!( + requests[1] + .headers + .get("x-amz-copy-source-if-match") + .unwrap(), + "\"source-etag\"" + ); + assert_eq!( + requests[2].headers.get(header::IF_MATCH).unwrap(), + "\"source-etag\"" + ); + server.shutdown().await; +} + +#[tokio::test] +async fn exactly_five_gib_uses_single_copy_object() { + let repository = Uuid::new_v4(); + let server = MockS3Server::start(vec![ + respond_response(head_response( + StatusCode::OK, + Some("\"source-etag\""), + Some(super::MULTIPART_COPY_THRESHOLD), + None, + None, + )), + respond_response(copy_response("\"destination-etag\"")), + respond_response(response_with_body( + StatusCode::NO_CONTENT, + bytes::Bytes::new(), + )), + ]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + assert!( + storage + .move_file( + repository, + &StoragePath::from("source.bin"), + &StoragePath::from("destination.bin"), + ) + .await + .expect("boundary move should succeed") + ); + let requests = server.take_requests(); + assert_eq!(requests.len(), 3); + assert_eq!(requests[1].method, hyper::Method::PUT); + assert!( + requests[1].uri.query().is_none() || !requests[1].uri.query().unwrap().contains("uploadId") + ); + server.shutdown().await; +} + +#[tokio::test] +async fn move_returns_false_for_missing_source() { + let repository = Uuid::new_v4(); + let server = MockS3Server::start(vec![respond_response(not_found_response("NoSuchKey"))]).await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + assert!( + !storage + .move_file( + repository, + &StoragePath::from("source.bin"), + &StoragePath::from("destination.bin"), + ) + .await + .expect("missing move should be idempotent") + ); + assert_eq!(server.take_requests().len(), 1); + server.shutdown().await; } #[tokio::test] -async fn delete_repository_removes_all_files() { +async fn move_copy_conflict_does_not_delete_source() { let repository = Uuid::new_v4(); - let key_one = format!("{repository}/packages/a.bin"); - let key_two = format!("{repository}/packages/nested/b.bin"); - let server = MockS3Server::start(vec![ - respond_list(list_response_body( - &format!("{repository}/"), - &[&key_one, &key_two], - false, + respond_response(head_response( + StatusCode::OK, + Some("\"source-etag\""), + Some(1), + None, None, )), - respond_delete(vec![key_one.clone(), key_two.clone()]), + respond_response(conditional_error_response( + StatusCode::PRECONDITION_FAILED, + "PreconditionFailed", + )), ]) .await; - let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); - storage - .delete_repository(repository) + let error = storage + .move_file( + repository, + &StoragePath::from("source.bin"), + &StoragePath::from("destination.bin"), + ) .await - .expect("delete_repository should succeed"); - + .expect_err("copy conflict should be returned"); + assert!(error.is_conflict()); let requests = server.take_requests(); - assert_eq!(requests.len(), 2, "one list and one delete call expected"); - let delete_keys = parse_deleted_keys(&requests[1].body); - assert_eq!(delete_keys.len(), 2); - assert!(delete_keys.contains(&key_one)); - assert!(delete_keys.contains(&key_two)); - + assert_eq!(requests.len(), 2); + assert!( + requests + .iter() + .all(|request| request.method != hyper::Method::DELETE) + ); server.shutdown().await; } #[tokio::test] -async fn delete_repository_handles_pagination() { +async fn move_delete_conflict_leaves_both_objects() { let repository = Uuid::new_v4(); - let first_page_keys = vec![ - format!("{repository}/page1/one"), - format!("{repository}/page1/two"), - ]; - let second_page_keys = vec![format!("{repository}/page2/three")]; - let server = MockS3Server::start(vec![ - respond_list(list_response_body( - &format!("{repository}/"), - &first_page_keys - .iter() - .map(|s| s.as_str()) - .collect::>(), - true, - Some("token-1"), - )), - respond_delete(first_page_keys.clone()), - respond_list(list_response_body( - &format!("{repository}/"), - &second_page_keys - .iter() - .map(|s| s.as_str()) - .collect::>(), - false, + respond_response(head_response( + StatusCode::OK, + Some("\"source-etag\""), + Some(1), + None, None, )), - respond_delete(second_page_keys.clone()), + respond_response(copy_response("\"destination-etag\"")), + respond_response(conditional_error_response( + StatusCode::PRECONDITION_FAILED, + "PreconditionFailed", + )), ]) .await; - let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); - storage - .delete_repository(repository) + let error = storage + .move_file( + repository, + &StoragePath::from("source.bin"), + &StoragePath::from("destination.bin"), + ) .await - .expect("delete_repository should succeed with pagination"); - + .expect_err("delete conflict should be returned"); + assert!(error.is_conflict()); let requests = server.take_requests(); + assert_eq!(requests.len(), 3); assert_eq!( - requests.len(), - 4, - "list/delete/list/delete sequence expected" + requests[2].headers.get(header::IF_MATCH).unwrap(), + "\"source-etag\"" ); + server.shutdown().await; +} + +#[tokio::test] +async fn multipart_copy_aborts_after_failed_part() { + let repository = Uuid::new_v4(); + let size = super::MULTIPART_COPY_THRESHOLD + 1; + let server = MockS3Server::start(vec![ + respond_response(head_response( + StatusCode::OK, + Some("\"source-etag\""), + Some(size), + None, + None, + )), + respond_response(multipart_create_response("upload-1")), + respond_response(response_with_body( + StatusCode::INTERNAL_SERVER_ERROR, + bytes::Bytes::from_static(b"copy part failed"), + )), + respond_response(response_with_body( + StatusCode::NO_CONTENT, + bytes::Bytes::new(), + )), + ]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + let error = storage + .move_file( + repository, + &StoragePath::from("source.bin"), + &StoragePath::from("destination.bin"), + ) + .await + .expect_err("part failure should abort multipart copy"); + assert!(!error.is_conflict()); + let requests = server.take_requests(); + assert_eq!(requests.len(), 4); + assert_eq!(requests[3].method, hyper::Method::DELETE); assert!( - requests[2] + requests[3] .uri .query() .unwrap_or_default() - .contains("continuation-token=token-1"), - "second list should carry continuation token" + .contains("uploadId=upload-1") + ); + server.shutdown().await; +} + +#[tokio::test] +async fn large_move_uses_multipart_copy_ranges_and_completes() { + let repository = Uuid::new_v4(); + let size = super::MULTIPART_COPY_THRESHOLD + 1; + let part_size = super::MULTIPART_COPY_PART_SIZE.max(size.div_ceil(10_000)); + let part_count = size.div_ceil(part_size); + let mut source_head = head_response( + StatusCode::OK, + Some("\"source-etag\""), + Some(size), + Some("application/octet-stream"), + None, + ); + source_head + .headers_mut() + .insert(header::CACHE_CONTROL, "max-age=3600".parse().unwrap()); + source_head.headers_mut().insert( + header::CONTENT_DISPOSITION, + "attachment; filename=artifact.bin".parse().unwrap(), + ); + source_head + .headers_mut() + .insert(header::CONTENT_ENCODING, "gzip".parse().unwrap()); + source_head + .headers_mut() + .insert(header::CONTENT_LANGUAGE, "en".parse().unwrap()); + source_head.headers_mut().insert( + header::EXPIRES, + "Wed, 21 Oct 2037 07:28:00 GMT".parse().unwrap(), + ); + source_head.headers_mut().insert( + header::HeaderName::from_static("x-amz-website-redirect-location"), + "/downloads/artifact.bin".parse().unwrap(), + ); + source_head.headers_mut().insert( + header::HeaderName::from_static("x-amz-meta-owner"), + "pkgly".parse().unwrap(), + ); + let mut responders = vec![ + respond_response(source_head), + respond_response(multipart_create_response("upload-1")), + ]; + for part in 0..part_count { + responders.push(respond_response(multipart_part_response(&format!( + "\"part-{part}\"" + )))); + } + responders.push(respond_response(multipart_complete_response())); + responders.push(respond_response(response_with_body( + StatusCode::NO_CONTENT, + bytes::Bytes::new(), + ))); + let server = MockS3Server::start(responders).await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + assert!( + storage + .move_file( + repository, + &StoragePath::from("source.bin"), + &StoragePath::from("destination.bin"), + ) + .await + .expect("multipart move should succeed") + ); + + let requests = server.take_requests(); + assert_eq!(requests.len(), 2 + part_count as usize + 2); + let create_headers = &requests[1].headers; + assert_eq!( + create_headers.get(header::CACHE_CONTROL).unwrap(), + "max-age=3600" + ); + assert_eq!( + create_headers.get(header::CONTENT_DISPOSITION).unwrap(), + "attachment; filename=artifact.bin" + ); + assert_eq!( + create_headers.get(header::CONTENT_ENCODING).unwrap(), + "gzip" + ); + assert_eq!(create_headers.get(header::CONTENT_LANGUAGE).unwrap(), "en"); + assert_eq!( + create_headers.get(header::EXPIRES).unwrap(), + "Wed, 21 Oct 2037 07:28:00 GMT" + ); + assert_eq!( + create_headers + .get("x-amz-website-redirect-location") + .unwrap(), + "/downloads/artifact.bin" + ); + assert_eq!(create_headers.get("x-amz-meta-owner").unwrap(), "pkgly"); + let part_requests = &requests[2..2 + part_count as usize]; + assert_eq!( + part_requests[0] + .headers + .get("x-amz-copy-source-range") + .unwrap(), + "bytes=0-67108863" + ); + assert!( + part_requests + .last() + .unwrap() + .headers + .get("x-amz-copy-source-range") + .is_some() ); + assert_eq!( + requests[2 + part_count as usize].method, + hyper::Method::POST + ); + assert_eq!(requests.last().unwrap().method, hyper::Method::DELETE); + server.shutdown().await; +} + +#[tokio::test] +async fn manifest_pages_share_one_cached_traversal() { + let repository = Uuid::new_v4(); + let server = MockS3Server::start(vec![respond_list(list_response_body_with_metadata( + &format!("{repository}/v2/"), + &[], + &[], + false, + None, + ))]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + let first = storage + .list_docker_manifests_paginated(repository, 0, 10) + .await + .expect("first page"); + let second = storage + .list_docker_manifests_paginated(repository, 10, 10) + .await + .expect("second page"); + assert!(first.0.is_empty()); + assert_eq!(first.1, 0); + assert!(second.0.is_empty()); + assert_eq!(server.take_requests().len(), 1); server.shutdown().await; } #[tokio::test] -async fn delete_repository_is_idempotent_on_empty_prefix() { +async fn concurrent_manifest_requests_share_one_traversal() { let repository = Uuid::new_v4(); - let server = MockS3Server::start(vec![respond_list(list_response_body( - &format!("{repository}/"), + let server = MockS3Server::start(vec![respond_list(list_response_body_with_metadata( + &format!("{repository}/v2/"), + &[], &[], false, None, ))]) .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + let first_storage = storage.clone(); + let first = tokio::spawn(async move { + first_storage + .list_docker_manifests(repository) + .await + .expect("first manifest request") + }); + let second_storage = storage.clone(); + let second = tokio::spawn(async move { + second_storage + .list_docker_manifests(repository) + .await + .expect("second manifest request") + }); + assert!(first.await.expect("first task").is_empty()); + assert!(second.await.expect("second task").is_empty()); + assert_eq!(server.take_requests().len(), 1); + server.shutdown().await; +} +#[tokio::test] +async fn mutation_during_manifest_load_prevents_stale_cache_publication() { + let repository = Uuid::new_v4(); + let root = format!("{repository}/v2/"); + let server = MockS3Server::start(vec![ + respond_delayed( + Duration::from_millis(75), + response_with_body( + StatusCode::OK, + bytes::Bytes::from(list_response_body_with_metadata( + &root, + &[], + &[], + false, + None, + )), + ), + ), + respond_response(not_found_response("NotFound")), + respond_response(response_with_body(StatusCode::OK, bytes::Bytes::new())), + respond_list(list_response_body_with_metadata( + &root, + &[], + &[], + false, + None, + )), + ]) + .await; let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + let loading_storage = storage.clone(); + let loading = tokio::spawn(async move { + loading_storage + .list_docker_manifests(repository) + .await + .expect("manifest traversal") + }); + for _ in 0..100 { + if !server.take_requests().is_empty() { + break; + } + sleep(Duration::from_millis(1)).await; + } + assert_eq!( + server.take_requests().len(), + 1, + "manifest load should start" + ); + storage - .delete_repository(repository) + .save_file( + repository, + FileContent::Content(b"manifest".to_vec()), + &StoragePath::from("manifest.json"), + ) .await - .expect("empty prefixes should be handled gracefully"); - - let requests = server.take_requests(); - assert_eq!(requests.len(), 1, "only a list request is expected"); + .expect("save during traversal"); + assert!(loading.await.expect("manifest task").is_empty()); + storage + .list_docker_manifests(repository) + .await + .expect("next read must traverse again"); + assert_eq!(server.take_requests().len(), 4); server.shutdown().await; } #[tokio::test] -async fn delete_repository_preserves_other_repositories() { +async fn manifest_loader_sorts_nested_results_before_pagination() { let repository = Uuid::new_v4(); - let other_repo = Uuid::new_v4(); + let root = format!("{repository}/v2/"); + let image = format!("{repository}/v2/library/"); + let manifests = format!("{repository}/v2/library/image/manifests/"); + let server = MockS3Server::start(vec![ + respond_list(list_response_body_with_metadata( + &root, + &[], + &[&image], + false, + None, + )), + respond_list(list_response_body_with_metadata( + &image, + &[], + &[&manifests], + false, + None, + )), + respond_list(list_response_body_with_metadata( + &manifests, + &[ + ( + &format!("{manifests}z"), + 1, + Some("2025-01-01T00:00:00.000Z"), + ), + ( + &format!("{manifests}a"), + 2, + Some("2025-01-01T00:00:00.000Z"), + ), + ], + &[], + false, + None, + )), + ]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); - let keys = [ - format!("{repository}/packages/a.bin"), - format!("{other_repo}/packages/should-not-delete"), - ]; + let (first, total) = storage + .list_docker_manifests_paginated(repository, 0, 1) + .await + .expect("first manifest page"); + let (second, second_total) = storage + .list_docker_manifests_paginated(repository, 1, 1) + .await + .expect("second manifest page"); + assert_eq!(total, 2); + assert_eq!(second_total, 2); + assert_eq!(first[0].key, "v2/library/image/manifests/a"); + assert_eq!(second[0].key, "v2/library/image/manifests/z"); + assert_eq!(server.take_requests().len(), 3); + server.shutdown().await; +} + +#[test] +fn manifest_cache_expires_and_invalidates_entries() { + let repository = Uuid::new_v4(); + let now = tokio::time::Instant::now(); + let mut cache = super::ManifestCache::new(); + cache.insert(repository, Vec::new(), now); + assert!( + cache + .get(repository, now + Duration::from_secs(29)) + .is_some() + ); + assert!( + cache + .get(repository, now + Duration::from_secs(30)) + .is_none() + ); + cache.insert(repository, Vec::new(), now); + cache.invalidate(repository); + assert!(cache.get(repository, now).is_none()); +} +#[tokio::test] +async fn failed_manifest_traversal_is_not_cached() { + let repository = Uuid::new_v4(); + let root = format!("{repository}/v2/"); let server = MockS3Server::start(vec![ - respond_list(list_response_body( - &format!("{repository}/"), - &keys.iter().map(|s| s.as_str()).collect::>(), + respond_response(response_with_body( + StatusCode::INTERNAL_SERVER_ERROR, + bytes::Bytes::from_static(b"temporary failure"), + )), + respond_list(list_response_body_with_metadata( + &root, + &[], + &[], false, None, )), - respond_delete(vec![format!("{repository}/packages/a.bin")]), ]) .await; - let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); - storage - .delete_repository(repository) - .await - .expect("delete_repository should ignore other repo keys"); - let requests = server.take_requests(); - assert_eq!(requests.len(), 2); - let deleted = parse_deleted_keys(&requests[1].body); - assert_eq!(deleted, vec![format!("{repository}/packages/a.bin")]); + assert!(storage.list_docker_manifests(repository).await.is_err()); assert!( - !deleted - .iter() - .any(|key| key.contains(&other_repo.to_string())), - "keys from other repositories must not be deleted" + storage + .list_docker_manifests(repository) + .await + .expect("retry should load") + .is_empty() ); + assert_eq!(server.take_requests().len(), 2); + server.shutdown().await; +} + +#[tokio::test] +async fn successful_save_invalidates_manifest_cache() { + let repository = Uuid::new_v4(); + let root = format!("{repository}/v2/"); + let server = MockS3Server::start(vec![ + respond_list(list_response_body_with_metadata( + &root, + &[], + &[], + false, + None, + )), + respond_response(not_found_response("NotFound")), + respond_response(response_with_body(StatusCode::OK, bytes::Bytes::new())), + respond_list(list_response_body_with_metadata( + &root, + &[], + &[], + false, + None, + )), + ]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + storage + .list_docker_manifests(repository) + .await + .expect("initial traversal"); + storage + .save_file( + repository, + FileContent::Content(b"manifest".to_vec()), + &StoragePath::from("manifest.json"), + ) + .await + .expect("save should succeed"); + storage + .list_docker_manifests(repository) + .await + .expect("traversal after invalidation"); + assert_eq!(server.take_requests().len(), 4); server.shutdown().await; } diff --git a/docs/docs/sysAdmin/s3.md b/docs/docs/sysAdmin/s3.md index 91eae15..71bb55c 100644 --- a/docs/docs/sysAdmin/s3.md +++ b/docs/docs/sysAdmin/s3.md @@ -8,8 +8,9 @@ Use this guide to back Pkgly repositories with an Amazon S3 bucket or any S3-com - You run Pkgly in Kubernetes or another stateless platform and do not want to manage block volumes. - You already store artifacts in an S3-compatible service and want Pkgly to sit in front of that bucket. -**Limitations (as of this commit):** -- `move_file` is implemented as `GetObject` + `PutObject` + `DeleteObject`. Renaming large blobs consumes bandwidth proportional to object size. +**Operational behavior:** +- Moves use an ETag-guarded server-side `CopyObject`; objects larger than 5 GiB use multipart copy. The source is deleted only after a successful copy. If the guarded delete conflicts, both source and destination remain so no changed source data is lost. +- Appends use a read/modify/write sequence guarded by `If-Match` (or `If-None-Match: *` for creation). A conditional conflict is returned to the caller, which should retry with a fresh object version. ## Prerequisites @@ -21,16 +22,9 @@ Use this guide to back Pkgly repositories with an Amazon S3 bucket or any S3-com - `s3:GetObject` - `s3:PutObject` - `s3:DeleteObject` - - `s3:GetObjectTagging` - `s3:HeadObject` - **IAM role to assume** (`role_arn`, optional `role_session_name` / `external_id`). Pkgly calls `AssumeRole` through the AWS SDK and uses those temporary credentials for all S3 traffic. - **Instance/container profile** – leave every credential field blank and Pkgly will fall back to the AWS SDK default chain (environment variables, shared config/credentials files, ECS/EKS credentials, IMDS, etc.). - - `s3:ListBucket` - - `s3:GetObject` - - `s3:PutObject` - - `s3:DeleteObject` - - `s3:GetObjectTagging` - - `s3:HeadObject` 4. **Administrative login** that carries the `StorageManager` capability (system manager or admin) so you can call `/api/storage/**`. ## Bucket layout expectations @@ -48,7 +42,7 @@ curl -s -H "Authorization: Bearer $PKGLY_TOKEN" \ https://pkgly.example.com/api/storage/s3/regions | jq ``` -Pick one of the values (e.g., `UsEast1`). For S3-compatible endpoints that report a custom name (MinIO, Ceph), skip this list and plan to provide a custom endpoint instead. +Pick one of the canonical raw identifiers (for example, `us-east-1`). The list is only a convenience: Pkgly accepts any non-empty provider region string, including identifiers not yet listed. For S3-compatible endpoints that report a custom name (MinIO, Ceph), use a custom endpoint and optional signing-region label. ## Step 2 – Decide between path-style and virtual-hosted-style URLs @@ -68,7 +62,7 @@ curl -X POST https://pkgly.example.com/api/storage/new/s3 \ "type": "S3", "settings": { "bucket_name": "pkgly-prod-artifacts", - "region": "UsEast1", + "region": "us-east-1", "credentials": { "access_key": "AKIA...", "secret_key": "••••••••", @@ -96,6 +90,15 @@ Notes: - Credentials are stored encrypted at rest inside the `storages` table. Rotate them the same way you created them: create a new storage (or future update endpoint once available) and reassign repositories to it. - During creation Pkgly invokes `S3StorageFactory::test_storage_config()`. If anything fails (missing region, auth error, bucket not found) the API responds with `400 Invalid Storage Config` and the precise driver error appears in the response body and server logs. +### Region migration and rollback + +The region field is stored as a raw string. The migration translates all legacy enum tokens (such +as `UsEast1`) to canonical identifiers such as `us-east-1`; newly serialized configurations always +use the raw form. Unknown provider identifiers are accepted and preserved unchanged. Rolling the +migration back converts only the original 24 identifiers and deliberately leaves newer or custom +values untouched. An enum-only binary cannot read those untouched values, so change them manually +before downgrading to such a binary. + ### Custom endpoint example (MinIO, DigitalOcean Spaces, etc.) The S3 driver flattens the optional `CustomRegion` struct, so provide `custom_region` (name label, optional) and `endpoint` (full URL) directly inside `settings`. When `endpoint` is present it takes precedence over `region`. @@ -130,7 +133,7 @@ To keep hot artifacts on the node and avoid repeated S3 downloads, enable the ca "type": "S3", "settings": { "bucket_name": "pkgly-artifacts", - "region": "UsEast1", + "region": "us-east-1", "credentials": { "role_arn": "arn:aws:iam::123:role/pkgly" }, "path_style": true, "cache": { @@ -144,7 +147,22 @@ To keep hot artifacts on the node and avoid repeated S3 downloads, enable the ca } ``` -Leave `path` blank to fall back to `$(TMPDIR)/pkgly/s3-cache/`. The cache runs fully asynchronously—blocking file reads/writes are offloaded to Tokio’s blocking pool—so enabling it will not starve the HTTP runtime. +Leave `path` blank to fall back to `$(TMPDIR)/pkgly/s3-cache/`. Prefer a directory dedicated to one Pkgly S3 storage. The cache writes versioned content sidecars atomically, verifies size and SHA-256 on every hit, and rebuilds its LRU index after restart while preserving unrelated files in the configured directory. + +Cache hits retain the S3 object `Last-Modified` timestamp. Files without a usable provider timestamp +fall back to the request time; virtual directories and common prefixes are always assigned one +request-time timestamp because S3 does not expose a directory creation time. Docker manifest indexes +are kept in a per-storage LRU cache for 30 seconds and are refreshed after that window or after a +successful mutation. + +The driver uses fixed internal deadlines for control traffic: 30 seconds per attempt and 90 seconds +for the complete control operation. Copy and multipart-copy requests allow five minutes per attempt +and 20 minutes for the operation. Streamed GET and PUT operations intentionally have no wall-clock +deadline so legitimate large transfers can finish; the SDK still stops stalled connections. + +The configured cache directory should be dedicated to this storage. Pkgly only removes recognized +cache sidecars, generation files, and legacy hash-layout artifacts, so unrelated files are retained, +but sharing a directory between multiple Pkgly caches can still cause avoidable churn. ### IAM role / default chain example @@ -157,7 +175,7 @@ To avoid long-lived keys entirely, leave the key fields blank and provide only t "type": "S3", "settings": { "bucket_name": "pkgly-artifacts", - "region": "UsEast1", + "region": "us-east-1", "credentials": { "role_arn": "arn:aws:iam::123456789012:role/pkgly-deploy", "role_session_name": "pkgly-ci", @@ -187,14 +205,18 @@ To avoid long-lived keys entirely, leave the key fields blank and provide only t | Symptom | Likely cause | Fix | | --- | --- | --- | -| `Invalid Storage Config: NoRegionSpecified` | Neither `region` nor `endpoint` provided | Supply one of the supported regions or a custom endpoint | -| `AWS SDK error: InvalidAccessKeyId` | Wrong credentials or user lacks permission | Regenerate the key pair or fix the role policy so it has the required bucket-scoped IAM actions | +| `Invalid Storage Config: NoRegionSpecified` | Neither `region` nor `endpoint` provided | Supply a non-empty provider region identifier or a custom endpoint | +| `AWS SDK error (AccessDenied): InvalidAccessKeyId` | Wrong credentials or user lacks permission | Regenerate the key pair or fix the role policy so it has the required bucket-scoped IAM actions | | Cached file never expires | Cache size too small or directory not writable | Increase `cache.max_bytes` / `max_entries` or ensure Pkgly can write to the cache path | | `Bucket Does Not Exist` | Typo in `bucket_name` or Pkgly lacks access to the bucket | Verify the bucket name and the IAM policy’s `Resource` list | -| Uploads stall or time out | Pkgly copying large files via `move_file` or `append_file` | Avoid mass renames; delete + re-upload is faster until the driver adopts server-side copy | +| Uploads stall or time out | The endpoint is unreachable or a control request exceeded its deadline | Check endpoint reachability and logs; control operations use 30-second attempt and 90-second total limits, while streamed transfers have stalled-read protection without a wall-clock cap | ## Operational tips +- Repository GET/HEAD dispatch, Docker manifest refresh/download, and S3 read/write operations use heap-allocated futures to limit nested request stack frames. This also protects local-storage Docker proxies, whose shared storage interface includes S3 operations, in debug builds with the runtime's default worker stack size. - Keep the bucket lifecycle rules aligned with Pkgly retention policies. Pkgly never deletes repositories automatically, so lifecycle rules that expire objects will surface as 404s to clients. -- Monitor object count and size to detect runaway storages; Pkgly’s S3 driver currently lacks the optimized directory streaming used by the local backend, so list-heavy operations will cost extra API calls. +- Monitor object count and size to detect runaway storages. Directory and manifest listings consume paginated S3 LIST calls; Docker manifest indexes are cached per repository for up to 30 seconds and are invalidated after successful mutations. - Back up the `storages` table whenever you rotate credentials—losing it means Pkgly forgets how to talk to the bucket even though the data still exists. + +Direct uploads larger than 5 GiB, distributed cache invalidation across Pkgly nodes, automatic +egress-policy discovery, and cache metrics endpoints are outside this driver’s scope. diff --git a/pkgly/src/app/api/storage/s3.rs b/pkgly/src/app/api/storage/s3.rs index 57ab7e9..0ae9fc9 100644 --- a/pkgly/src/app/api/storage/s3.rs +++ b/pkgly/src/app/api/storage/s3.rs @@ -1,10 +1,11 @@ +// ABOUTME: Exposes S3 storage administration endpoints. +// ABOUTME: Returns raw region identifiers for editable provider configuration. use axum::{ response::{IntoResponse, Response}, routing::get, }; use nr_core::user::permissions::HasPermissions; -use nr_storage::s3::regions::S3StorageRegion; -use strum::IntoEnumIterator; +use nr_storage::s3::regions::KNOWN_S3_REGIONS; use tracing::instrument; use utoipa::OpenApi; @@ -15,7 +16,7 @@ use crate::{ }; #[derive(OpenApi)] -#[openapi(paths(region_list), components(schemas(S3StorageRegion)))] +#[openapi(paths(region_list))] pub struct S3StorageAPI; pub fn s3_storage_api() -> axum::Router { axum::Router::new().route("/regions", get(region_list)) @@ -25,7 +26,7 @@ pub fn s3_storage_api() -> axum::Router { get, path = "/regions", responses( - (status = 200, description = "A list of available regions for the S3 storage", body = Vec) + (status = 200, description = "A list of suggested raw region identifiers for S3 storage", body = Vec) ) )] #[instrument(skip(auth), fields(user = %auth.id))] @@ -33,6 +34,60 @@ pub async fn region_list(auth: Authentication) -> Result = S3StorageRegion::iter().collect(); + let regions: Vec = KNOWN_S3_REGIONS + .iter() + .map(|region| (*region).to_owned()) + .collect(); Ok(ResponseBuilder::ok().json(®ions)) } + +#[cfg(test)] +mod tests { + use super::*; + use axum::response::Response; + use chrono::DateTime; + use http_body_util::BodyExt; + use nr_core::{ + database::entities::user::{UserSafeData, auth_token::AuthToken}, + user::{Email, Username}, + }; + + fn admin_auth() -> Authentication { + let timestamp = DateTime::parse_from_rfc3339("2024-01-01T00:00:00+00:00").unwrap(); + let user = UserSafeData { + id: 1, + name: "Admin".into(), + username: Username::new("admin".into()).unwrap(), + email: Some(Email::new("admin@example.com".into()).unwrap()), + require_password_change: false, + active: true, + admin: true, + user_manager: true, + system_manager: true, + default_repository_actions: vec![], + updated_at: timestamp, + created_at: timestamp, + }; + let token = AuthToken { + id: 1, + user_id: 1, + name: Some("test".into()), + description: None, + token: "token".into(), + active: true, + source: "test".into(), + expires_at: None, + created_at: timestamp, + }; + Authentication::AuthToken(token, user) + } + + #[tokio::test] + async fn region_list_returns_raw_identifiers() { + let response: Response = region_list(admin_auth()).await.unwrap(); + let body = response.into_body().collect().await.unwrap().to_bytes(); + let regions: Vec = serde_json::from_slice(&body).unwrap(); + assert!(regions.iter().any(|region| region == "us-east-1")); + assert!(!regions.iter().any(|region| region == "UsEast1")); + } +} diff --git a/pkgly/src/main.rs b/pkgly/src/main.rs index 53f8d6c..b22b86f 100644 --- a/pkgly/src/main.rs +++ b/pkgly/src/main.rs @@ -1,3 +1,4 @@ +#![recursion_limit = "256"] #![allow( elided_lifetimes_in_paths, clippy::all, @@ -217,6 +218,9 @@ fn web_start(config_path: Option) -> anyhow::Result<()> { let tokio = tokio::runtime::Builder::new_multi_thread() .worker_threads(worker_threads) .thread_name_fn(thread_name) + // Request chains through S3-backed Docker repositories nest the axum handler stack + // inside the AWS SDK state machine; std::thread defaults (2 MiB) overflow there. + .thread_stack_size(8 * 1024 * 1024) .enable_all() .build()?; tokio.block_on(app::web::start_with_config(config))?; diff --git a/pkgly/src/repository/docker/proxy.rs b/pkgly/src/repository/docker/proxy.rs index 783924d..279d2d5 100644 --- a/pkgly/src/repository/docker/proxy.rs +++ b/pkgly/src/repository/docker/proxy.rs @@ -1,3 +1,5 @@ +// ABOUTME: Implements Docker pull-through caching and upstream authentication. +// ABOUTME: Serves manifests and blobs through the shared repository storage interface. //! Docker proxy (pull-through cache) support. //! //! This module implements a read-only proxy repository that forwards @@ -1206,6 +1208,8 @@ pub(crate) async fn fetch_and_cache_manifest( ); let manifest_path = StoragePath::from(format!("v2/{}/manifests/{}", repository_name, reference)); + // Refresh/download futures contain nested SDK operations. Keep them heap-allocated so + // their state does not accumulate in the surrounding HTTP request's stack frames. if let Some(cached) = load_cached_manifest(storage, repository_id, &manifest_path, reference).await? { @@ -1226,7 +1230,7 @@ pub(crate) async fn fetch_and_cache_manifest( return Err(DockerError::ManifestNotFound(reference.to_string())); } } else { - match revalidate_manifest_tag( + match Box::pin(revalidate_manifest_tag( upstream, storage, repository_id, @@ -1235,7 +1239,7 @@ pub(crate) async fn fetch_and_cache_manifest( &manifest_path, cached, indexer, - ) + )) .await? { RevalidationOutcome::Unchanged(cached) => { @@ -1254,7 +1258,7 @@ pub(crate) async fn fetch_and_cache_manifest( } else { accept }; - match download_manifest_from_upstream( + match Box::pin(download_manifest_from_upstream( upstream, storage, repository_id, @@ -1263,7 +1267,7 @@ pub(crate) async fn fetch_and_cache_manifest( override_accept, &manifest_path, indexer, - ) + )) .await { Ok(manifest) => return Ok(manifest), @@ -1279,7 +1283,7 @@ pub(crate) async fn fetch_and_cache_manifest( } info!("fetch_and_cache_manifest: cache miss or refresh failed, downloading"); - download_manifest_from_upstream( + Box::pin(download_manifest_from_upstream( upstream, storage, repository_id, @@ -1288,7 +1292,7 @@ pub(crate) async fn fetch_and_cache_manifest( Some(MODERN_UPSTREAM_ACCEPT), &manifest_path, indexer, - ) + )) .await } diff --git a/pkgly/src/repository/docker/proxy/tests.rs b/pkgly/src/repository/docker/proxy/tests.rs index 6ed265c..0b7dca5 100644 --- a/pkgly/src/repository/docker/proxy/tests.rs +++ b/pkgly/src/repository/docker/proxy/tests.rs @@ -1,3 +1,5 @@ +// ABOUTME: Exercises Docker proxy caching, upstream requests, and manifest validation. +// ABOUTME: Checks request resource bounds and cache behavior across registry operations. #![allow(clippy::expect_used, clippy::panic, clippy::todo, clippy::unwrap_used)] use super::*; use crate::repository::{ @@ -295,6 +297,50 @@ impl ProxyIndexing for RecordingIndexer { } } +#[tokio::test] +async fn manifest_future_fits_worker_stack() -> anyhow::Result<()> { + let storage = test_storage().await; + let upstream = ProxyUpstream::new(&DockerProxyConfig { + upstream_url: "https://registry-1.docker.io".into(), + upstream_auth: None, + revalidation_ttl_seconds: default_revalidation_ttl(), + skip_tag_revalidation: false, + })?; + let future = fetch_and_cache_manifest( + &upstream, + &storage, + Uuid::new_v4(), + "library/nginx", + "alpine", + None, + None, + ); + let size = std::mem::size_of_val(&future); + assert!( + size < 16 * 1024, + "manifest future occupies {size} bytes on the request stack" + ); + Ok(()) +} + +#[test] +fn repository_read_futures_fit_worker_stack() { + fn size( + _: impl Fn(&'static crate::repository::DynRepository, RepositoryRequest) -> F, + ) -> usize { + std::mem::size_of::() + } + for (method, size) in [ + ("GET", size(|repo, request| repo.handle_get(request))), + ("HEAD", size(|repo, request| repo.handle_head(request))), + ] { + assert!( + size < 16 * 1024, + "repository {method} future occupies {size} bytes" + ); + } +} + #[tokio::test] async fn fetch_manifest_caches_locally() -> anyhow::Result<()> { let manifest = br#"{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.image.config.v1+json","size":7023,"digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000"},"layers":[]}"#; diff --git a/site/src/components/nr/storage/s3/S3StorageConfig.vue b/site/src/components/nr/storage/s3/S3StorageConfig.vue index 8d89d65..5ac8008 100644 --- a/site/src/components/nr/storage/s3/S3StorageConfig.vue +++ b/site/src/components/nr/storage/s3/S3StorageConfig.vue @@ -1,3 +1,5 @@ + +