From acf52ca81096d596ceb0e8fb1fb5a44ed1fa7d9b Mon Sep 17 00:00:00 2001 From: kangmaup Date: Thu, 24 Sep 2026 12:36:15 +0700 Subject: [PATCH 1/5] refactor(storage): move S3DiskConfig into its own module and share env helpers Moves `S3DiskConfig` verbatim from `facade.rs` into a new `rustasea_storage::s3` module, mirroring `sftp/config.rs`, so the S3 fix that follows has a home of its own. It stays re-exported from `facade` and the crate root, so existing paths keep working. `env_non_empty` moves from `sftp/config.rs` to `facade.rs`, and the `with_env` test helper (with its env lock) moves from `sftp/tests.rs` to a shared `test_support` module, ready for the S3 overlay. No behaviour change. --- crates/rustasea-storage/src/facade.rs | 33 +++++++-------------- crates/rustasea-storage/src/lib.rs | 4 +++ crates/rustasea-storage/src/s3.rs | 31 +++++++++++++++++++ crates/rustasea-storage/src/sftp/config.rs | 9 +----- crates/rustasea-storage/src/sftp/tests.rs | 25 +--------------- crates/rustasea-storage/src/test_support.rs | 25 ++++++++++++++++ 6 files changed, 73 insertions(+), 54 deletions(-) create mode 100644 crates/rustasea-storage/src/s3.rs create mode 100644 crates/rustasea-storage/src/test_support.rs diff --git a/crates/rustasea-storage/src/facade.rs b/crates/rustasea-storage/src/facade.rs index adbaa7f..88944a4 100644 --- a/crates/rustasea-storage/src/facade.rs +++ b/crates/rustasea-storage/src/facade.rs @@ -17,6 +17,8 @@ use crate::disk::LocalDisk; use crate::error::{Result, StorageError}; use crate::manager::{ManagedDisk, StorageConfig, StorageManager}; +pub use crate::s3::S3DiskConfig; + /// Top-level `[storage]` configuration document. #[derive(Debug, Clone, Deserialize, PartialEq, Eq, Default)] pub struct StorageFacadeConfig { @@ -143,28 +145,6 @@ pub struct LocalDiskConfig { pub settings: DiskSettings, } -/// Configuration for an S3 disk. -#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] -pub struct S3DiskConfig { - /// Bucket name. - pub bucket: String, - /// AWS region (falls back to the SDK default when absent). - #[serde(default)] - pub region: Option, - /// Explicit access key id (prefer environment/secret manager). - #[serde(default)] - pub access_key_id: Option, - /// Explicit secret access key (prefer environment/secret manager). - #[serde(default)] - pub secret_access_key: Option, - /// Custom endpoint for S3-compatible services (MinIO, R2). - #[serde(default)] - pub endpoint: Option, - /// Shared Laravel-parity disk settings. - #[serde(flatten, default)] - pub settings: DiskSettings, -} - /// Configuration for a Google Cloud Storage disk. #[derive(Debug, Clone, Deserialize, PartialEq, Eq)] pub struct GcsDiskConfig { @@ -346,3 +326,12 @@ fn build_sftp(_config: &crate::sftp::SftpDiskConfig, name: &str) -> Result Option { + std::env::var(key) + .ok() + .filter(|value| !value.trim().is_empty()) +} diff --git a/crates/rustasea-storage/src/lib.rs b/crates/rustasea-storage/src/lib.rs index eb98d05..ea1e26c 100644 --- a/crates/rustasea-storage/src/lib.rs +++ b/crates/rustasea-storage/src/lib.rs @@ -9,9 +9,13 @@ pub mod error; pub mod facade; pub mod manager; pub mod path; +pub mod s3; pub mod sftp; pub mod storage; +#[cfg(test)] +mod test_support; + pub use crate::disk::{DiskKind, LocalDisk, ReadThrough, ReadThroughDisk}; pub use crate::error::{PathError, Result, StorageError}; pub use crate::facade::{ diff --git a/crates/rustasea-storage/src/s3.rs b/crates/rustasea-storage/src/s3.rs new file mode 100644 index 0000000..22d954b --- /dev/null +++ b/crates/rustasea-storage/src/s3.rs @@ -0,0 +1,31 @@ +//! S3 disk configuration (`[storage.disks.*]` with `driver = "s3"`). +//! +//! The type is compiled unconditionally (the facade must parse an `s3` disk +//! even when the `aws` feature is off); only the facade's `object_store` +//! builder is feature-gated. + +use serde::Deserialize; + +use crate::facade::DiskSettings; + +/// Configuration for an S3 disk. +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +pub struct S3DiskConfig { + /// Bucket name. + pub bucket: String, + /// AWS region (falls back to the SDK default when absent). + #[serde(default)] + pub region: Option, + /// Explicit access key id (prefer environment/secret manager). + #[serde(default)] + pub access_key_id: Option, + /// Explicit secret access key (prefer environment/secret manager). + #[serde(default)] + pub secret_access_key: Option, + /// Custom endpoint for S3-compatible services (MinIO, R2). + #[serde(default)] + pub endpoint: Option, + /// Shared Laravel-parity disk settings. + #[serde(flatten, default)] + pub settings: DiskSettings, +} diff --git a/crates/rustasea-storage/src/sftp/config.rs b/crates/rustasea-storage/src/sftp/config.rs index d18094a..665c92c 100644 --- a/crates/rustasea-storage/src/sftp/config.rs +++ b/crates/rustasea-storage/src/sftp/config.rs @@ -9,7 +9,7 @@ use serde::Deserialize; use crate::error::{Result, StorageError}; -use crate::facade::DiskSettings; +use crate::facade::{env_non_empty, DiskSettings}; /// Default SSH port when `port` is omitted. pub const DEFAULT_SFTP_PORT: u16 = 22; @@ -212,10 +212,3 @@ impl SftpDiskConfig { prefix.trim_matches('/').to_string() } } - -/// Read an environment variable, treating unset or blank values as absent. -fn env_non_empty(key: &str) -> Option { - std::env::var(key) - .ok() - .filter(|value| !value.trim().is_empty()) -} diff --git a/crates/rustasea-storage/src/sftp/tests.rs b/crates/rustasea-storage/src/sftp/tests.rs index 173d3af..06195fe 100644 --- a/crates/rustasea-storage/src/sftp/tests.rs +++ b/crates/rustasea-storage/src/sftp/tests.rs @@ -1,33 +1,10 @@ //! Hermetic unit tests for the SFTP disk configuration (no network). -use std::sync::Mutex; - use super::config::{ SftpDiskConfig, DEFAULT_SFTP_PORT, DEFAULT_SFTP_ROOT, DEFAULT_SFTP_TIMEOUT_SECS, }; use crate::error::StorageError; - -/// Serializes environment-mutating tests (precedent: broadcast `manager/tests.rs`). -static ENV_LOCK: Mutex<()> = Mutex::new(()); - -/// Run `body` with the given env vars set, restoring prior values afterwards. -fn with_env(vars: &[(&str, &str)], body: F) { - let _guard = ENV_LOCK.lock().unwrap_or_else(|poison| poison.into_inner()); - let prior: Vec<(String, Option)> = vars - .iter() - .map(|(key, _)| ((*key).to_string(), std::env::var(key).ok())) - .collect(); - for (key, value) in vars { - std::env::set_var(key, value); - } - body(); - for (key, value) in prior { - match value { - Some(value) => std::env::set_var(&key, value), - None => std::env::remove_var(&key), - } - } -} +use crate::test_support::with_env; #[test] fn deserializes_from_toml_with_driver_tag() { diff --git a/crates/rustasea-storage/src/test_support.rs b/crates/rustasea-storage/src/test_support.rs new file mode 100644 index 0000000..1d61364 --- /dev/null +++ b/crates/rustasea-storage/src/test_support.rs @@ -0,0 +1,25 @@ +//! Test-only helpers shared by the crate's unit tests. + +use std::sync::Mutex; + +/// Serializes environment-mutating tests (precedent: broadcast `manager/tests.rs`). +static ENV_LOCK: Mutex<()> = Mutex::new(()); + +/// Run `body` with the given env vars set, restoring prior values afterwards. +pub(crate) fn with_env(vars: &[(&str, &str)], body: F) { + let _guard = ENV_LOCK.lock().unwrap_or_else(|poison| poison.into_inner()); + let prior: Vec<(String, Option)> = vars + .iter() + .map(|(key, _)| ((*key).to_string(), std::env::var(key).ok())) + .collect(); + for (key, value) in vars { + std::env::set_var(key, value); + } + body(); + for (key, value) in prior { + match value { + Some(value) => std::env::set_var(&key, value), + None => std::env::remove_var(&key), + } + } +} From c956953f7388d96f6454821ec5d3e7f1c2459431 Mon Sep 17 00:00:00 2001 From: kangmaup Date: Thu, 24 Sep 2026 12:37:19 +0700 Subject: [PATCH 2/5] fix(storage): support S3-compatible http endpoints and AWS_* env for s3 disks `s3` disks could not talk to an S3-compatible service in local development: - `object_store` only allows HTTPS unless `allow_http` is set, and `build_s3` never set it, so the documented `endpoint = "http://localhost:9000"` example failed every request with a reqwest builder error. An explicit `http://` endpoint (surrounding whitespace ignored) now enables `allow_http`; every other endpoint, and AWS itself, stays HTTPS-only. - `.env.example` documents that the `s3` disk reads `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_DEFAULT_REGION`, `AWS_BUCKET`, and `AWS_ENDPOINT` (the scaffolded docker-compose passes `AWS_ENDPOINT` to the app), but `StorageManager::from_toml_file` parses the file as-is and the builder used `AmazonS3Builder::new()`, so none were read and a disk without inline credentials fell back to EC2 instance metadata. `S3DiskConfig::apply_env` now overlays them onto every `s3` disk, mirroring the `SFTP_*` overlay. Like `SftpDiskConfig`, the config now redacts `secret_access_key` from `Debug` output and is validated after the overlay: `bucket` may be omitted when `AWS_BUCKET` supplies it, and a blank bucket is a `Config` error instead of a store with a malformed URL. A Docker-backed RustFS suite (`tests/rustfs_container.rs`, `#[ignore]`d like the SFTP one) reproduces both failures on the unfixed code and passes with the fix. --- config/storage.toml | 11 +- crates/rustasea-storage/Cargo.toml | 5 +- crates/rustasea-storage/src/facade.rs | 26 +-- crates/rustasea-storage/src/s3.rs | 136 ++++++++++- crates/rustasea-storage/src/s3/tests.rs | 169 ++++++++++++++ crates/rustasea-storage/src/sftp/mod.rs | 4 +- .../tests/rustfs_container.rs | 211 ++++++++++++++++++ 7 files changed, 531 insertions(+), 31 deletions(-) create mode 100644 crates/rustasea-storage/src/s3/tests.rs create mode 100644 crates/rustasea-storage/tests/rustfs_container.rs diff --git a/config/storage.toml b/config/storage.toml index 5d1dbbe..87e805b 100644 --- a/config/storage.toml +++ b/config/storage.toml @@ -60,10 +60,17 @@ report = false # visibility = "public" # throw = false # report = false -# # Prefer the environment / a secret manager over inline credentials. +# # Prefer the environment over inline credentials: AWS_ACCESS_KEY_ID, +# # AWS_SECRET_ACCESS_KEY, AWS_DEFAULT_REGION, AWS_BUCKET, and AWS_ENDPOINT +# # override the matching keys on every `s3` disk (blank values are ignored), +# # and `bucket` may be left out when AWS_BUCKET is set. `.env.example` sets +# # AWS_DEFAULT_REGION, so change the region there. With more than one `s3` +# # disk, leave AWS_BUCKET and AWS_ENDPOINT blank and set them per disk here. # # access_key_id = "..." # # secret_access_key = "..." -# # endpoint = "http://localhost:9000" # S3-compatible (MinIO, R2) +# # S3-compatible services (RustFS, MinIO, R2). An `http://` endpoint enables +# # plain HTTP for local development; any other endpoint stays HTTPS-only. +# # endpoint = "http://localhost:9000" # # [storage.disks.gcs] # driver = "gcs" diff --git a/crates/rustasea-storage/Cargo.toml b/crates/rustasea-storage/Cargo.toml index a4c4bd4..371cce8 100644 --- a/crates/rustasea-storage/Cargo.toml +++ b/crates/rustasea-storage/Cargo.toml @@ -33,6 +33,7 @@ azure = ["object_store/azure"] sftp = ["dep:russh", "dep:russh-sftp"] [dev-dependencies] -# Docker-backed SFTP integration tests (`tests/sftp_container.rs`), run only -# with `--features sftp -- --ignored`. +# Docker-backed integration tests, run only with `-- --ignored`: SFTP +# (`tests/sftp_container.rs`, `--features sftp`) and RustFS-backed S3 +# (`tests/rustfs_container.rs`, `--features aws`). testcontainers = { workspace = true } diff --git a/crates/rustasea-storage/src/facade.rs b/crates/rustasea-storage/src/facade.rs index 88944a4..3a1b216 100644 --- a/crates/rustasea-storage/src/facade.rs +++ b/crates/rustasea-storage/src/facade.rs @@ -220,23 +220,18 @@ impl StorageManager { } /// Build an S3-backed disk. +/// +/// The `AWS_*` environment overlay is applied and the config validated before +/// the `object_store` builder is configured (see [`S3DiskConfig::apply_env`]). #[cfg(feature = "aws")] fn build_s3(config: &S3DiskConfig, name: &str) -> Result> { - let mut builder = - object_store::aws::AmazonS3Builder::new().with_bucket_name(config.bucket.clone()); - if let Some(region) = &config.region { - builder = builder.with_region(region.clone()); - } - if let Some(key_id) = &config.access_key_id { - builder = builder.with_access_key_id(key_id.clone()); - } - if let Some(secret) = &config.secret_access_key { - builder = builder.with_secret_access_key(secret.clone()); - } - if let Some(endpoint) = &config.endpoint { - builder = builder.with_endpoint(endpoint.clone()); + let mut config = config.clone(); + config.apply_env(); + if let Err(error) = config.validate() { + return Err(StorageError::Config(format!("disk {name}: {error}"))); } - let store = builder + let store = config + .builder() .build() .map_err(|e| StorageError::StoreUnavailable(format!("disk {name}: {e}")))?; Ok(Arc::new(crate::ObjectDisk::new( @@ -329,7 +324,8 @@ fn build_sftp(_config: &crate::sftp::SftpDiskConfig, name: &str) -> Result Option { std::env::var(key) .ok() diff --git a/crates/rustasea-storage/src/s3.rs b/crates/rustasea-storage/src/s3.rs index 22d954b..923a7be 100644 --- a/crates/rustasea-storage/src/s3.rs +++ b/crates/rustasea-storage/src/s3.rs @@ -1,31 +1,147 @@ //! S3 disk configuration (`[storage.disks.*]` with `driver = "s3"`). //! -//! The type is compiled unconditionally (the facade must parse an `s3` disk -//! even when the `aws` feature is off); only the facade's `object_store` -//! builder is feature-gated. +//! Mirrors Laravel's `s3` disk keys (`bucket`, `region`, `key`/`secret` as +//! `access_key_id`/`secret_access_key`, `endpoint`). The type is compiled +//! unconditionally (the facade must parse an `s3` disk even when the `aws` +//! feature is off); only the `object_store` builder is feature-gated. +//! +//! # Environment bridge +//! +//! [`S3DiskConfig::apply_env`] overlays the Laravel-style `AWS_*` variables +//! that `.env.example` documents: `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, +//! `AWS_DEFAULT_REGION`, `AWS_BUCKET`, and `AWS_ENDPOINT`. +//! +//! # Plain-HTTP endpoints +//! +//! `object_store` rejects non-TLS URLs unless `allow_http` is set. An explicit +//! `http://` endpoint (RustFS, MinIO, or another S3-compatible service in +//! local development) opts in to plain HTTP; any other endpoint, and AWS +//! itself, stays HTTPS-only. use serde::Deserialize; -use crate::facade::DiskSettings; +use crate::error::{Result, StorageError}; +use crate::facade::{env_non_empty, DiskSettings}; -/// Configuration for an S3 disk. -#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +/// Typed `[storage.disks.*]` configuration for the `s3` driver. +/// +/// `secret_access_key` is redacted from [`Debug`] output so a debug dump or +/// log line can never leak the credential. +#[derive(Clone, Deserialize, PartialEq, Eq)] pub struct S3DiskConfig { - /// Bucket name. + /// Bucket name (may be left out when `AWS_BUCKET` supplies it). + #[serde(default)] pub bucket: String, /// AWS region (falls back to the SDK default when absent). #[serde(default)] pub region: Option, - /// Explicit access key id (prefer environment/secret manager). + /// Explicit access key id (prefer `AWS_ACCESS_KEY_ID` or a secret manager). #[serde(default)] pub access_key_id: Option, - /// Explicit secret access key (prefer environment/secret manager). + /// Explicit secret access key (prefer `AWS_SECRET_ACCESS_KEY` or a secret + /// manager). #[serde(default)] pub secret_access_key: Option, - /// Custom endpoint for S3-compatible services (MinIO, R2). + /// Custom endpoint for S3-compatible services (RustFS, MinIO, R2). #[serde(default)] pub endpoint: Option, /// Shared Laravel-parity disk settings. #[serde(flatten, default)] pub settings: DiskSettings, } + +impl std::fmt::Debug for S3DiskConfig { + /// Render the config with the `secret_access_key` masked. + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("S3DiskConfig") + .field("bucket", &self.bucket) + .field("region", &self.region) + .field("access_key_id", &self.access_key_id) + .field( + "secret_access_key", + &self.secret_access_key.as_ref().map(|_| "[REDACTED]"), + ) + .field("endpoint", &self.endpoint) + .field("settings", &self.settings) + .finish() + } +} + +impl S3DiskConfig { + /// Overlay the documented `AWS_*` environment variables. + /// + /// The Laravel `s3` disk names (`AWS_ACCESS_KEY_ID`, + /// `AWS_SECRET_ACCESS_KEY`, `AWS_DEFAULT_REGION`, `AWS_BUCKET`, + /// `AWS_ENDPOINT`) override the matching file keys; a blank value is + /// ignored. Like the `SFTP_*` overlay for `sftp` disks, it applies to every + /// `s3` disk. + pub fn apply_env(&mut self) { + if let Some(value) = env_non_empty("AWS_ACCESS_KEY_ID") { + self.access_key_id = Some(value); + } + if let Some(value) = env_non_empty("AWS_SECRET_ACCESS_KEY") { + self.secret_access_key = Some(value); + } + if let Some(value) = env_non_empty("AWS_DEFAULT_REGION") { + self.region = Some(value); + } + if let Some(value) = env_non_empty("AWS_BUCKET") { + self.bucket = value; + } + if let Some(value) = env_non_empty("AWS_ENDPOINT") { + self.endpoint = Some(value); + } + } + + /// Validate that the mandatory keys are present (after the env overlay). + /// + /// # Errors + /// + /// [`StorageError::Config`] when `bucket` is blank, i.e. neither the file + /// nor `AWS_BUCKET` supplied one. + pub fn validate(&self) -> Result<()> { + if self.bucket.trim().is_empty() { + return Err(StorageError::Config( + "s3 disk requires a non-empty `bucket` (or `AWS_BUCKET`)".into(), + )); + } + Ok(()) + } + + /// Translate the config into an `object_store` S3 builder. + /// + /// An `http://` endpoint enables `allow_http` (see the module docs). + #[cfg(feature = "aws")] + pub(crate) fn builder(&self) -> object_store::aws::AmazonS3Builder { + let mut builder = + object_store::aws::AmazonS3Builder::new().with_bucket_name(self.bucket.clone()); + if let Some(region) = &self.region { + builder = builder.with_region(region.clone()); + } + if let Some(key_id) = &self.access_key_id { + builder = builder.with_access_key_id(key_id.clone()); + } + if let Some(secret) = &self.secret_access_key { + builder = builder.with_secret_access_key(secret.clone()); + } + if let Some(endpoint) = &self.endpoint { + // URL parsing ignores surrounding whitespace, so the scheme check must too. + let endpoint = endpoint.trim(); + builder = builder + .with_endpoint(endpoint) + .with_allow_http(is_plain_http(endpoint)); + } + builder + } +} + +/// Whether `endpoint` uses the plain-HTTP scheme (case-insensitive). +#[cfg(feature = "aws")] +fn is_plain_http(endpoint: &str) -> bool { + endpoint + .get(..7) + .is_some_and(|scheme| scheme.eq_ignore_ascii_case("http://")) +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustasea-storage/src/s3/tests.rs b/crates/rustasea-storage/src/s3/tests.rs new file mode 100644 index 0000000..b47d55f --- /dev/null +++ b/crates/rustasea-storage/src/s3/tests.rs @@ -0,0 +1,169 @@ +//! Hermetic unit tests for the S3 disk configuration (no network). + +use super::S3DiskConfig; +use crate::test_support::with_env; + +/// A file-sourced config for the environment overlay to act on. +fn file_config() -> S3DiskConfig { + S3DiskConfig { + bucket: "file-bucket".into(), + region: Some("ap-southeast-1".into()), + access_key_id: Some("file-key".into()), + secret_access_key: Some("file-secret".into()), + endpoint: Some("https://file.example.com".into()), + settings: Default::default(), + } +} + +#[test] +fn apply_env_overlays_documented_variables() { + let mut config = file_config(); + with_env( + &[ + ("AWS_ACCESS_KEY_ID", "env-key"), + ("AWS_SECRET_ACCESS_KEY", "env-secret"), + ("AWS_DEFAULT_REGION", "eu-west-1"), + ("AWS_BUCKET", "env-bucket"), + ("AWS_ENDPOINT", "http://env.example.com:9000"), + ], + || config.apply_env(), + ); + assert_eq!(config.access_key_id.as_deref(), Some("env-key")); + assert_eq!(config.secret_access_key.as_deref(), Some("env-secret")); + assert_eq!(config.region.as_deref(), Some("eu-west-1")); + assert_eq!(config.bucket, "env-bucket"); + assert_eq!( + config.endpoint.as_deref(), + Some("http://env.example.com:9000") + ); +} + +#[test] +fn apply_env_ignores_blank_values() { + let mut config = file_config(); + with_env( + &[ + ("AWS_ACCESS_KEY_ID", ""), + ("AWS_SECRET_ACCESS_KEY", " "), + ("AWS_DEFAULT_REGION", ""), + ("AWS_BUCKET", " "), + ("AWS_ENDPOINT", ""), + ], + || config.apply_env(), + ); + assert_eq!(config, file_config()); +} + +#[test] +fn debug_output_redacts_secret_access_key() { + let config = S3DiskConfig { + secret_access_key: Some("super-secret-value".into()), + ..file_config() + }; + let rendered = format!("{config:?}"); + assert!(!rendered.contains("super-secret-value")); + assert!(rendered.contains("[REDACTED]")); + assert!(rendered.contains("file-bucket")); +} + +#[cfg(feature = "aws")] +mod builder { + use object_store::aws::AmazonS3ConfigKey; + use object_store::ClientConfigKey; + + use super::file_config; + use crate::s3::S3DiskConfig; + use crate::test_support::with_env; + + /// The `allow_http` value the builder hands to `object_store`. + fn allow_http(endpoint: Option<&str>) -> Option { + let config = S3DiskConfig { + endpoint: endpoint.map(str::to_string), + ..file_config() + }; + config + .builder() + .get_config_value(&AmazonS3ConfigKey::Client(ClientConfigKey::AllowHttp)) + } + + #[test] + fn plain_http_is_allowed_only_for_http_endpoints() { + let cases = [ + (Some("http://127.0.0.1:9000"), "true"), + (Some("HTTP://rustfs:9000"), "true"), + (Some(" http://127.0.0.1:9000 "), "true"), + (Some("https://s3.example.com"), "false"), + (None, "false"), + ]; + for (endpoint, want) in cases { + assert_eq!( + allow_http(endpoint).as_deref(), + Some(want), + "endpoint {endpoint:?}" + ); + } + } + + #[test] + fn built_disk_applies_the_environment_overlay() { + let toml = r#" +[storage] +default = "s3" + +[storage.disks.s3] +driver = "s3" +bucket = "file-bucket" +access_key_id = "file-key" +secret_access_key = "file-secret" +"#; + let mut label = String::new(); + with_env(&[("AWS_BUCKET", "env-bucket")], || { + let manager = crate::StorageManager::from_toml(toml).expect("s3 disk must build"); + label = manager + .disk("s3") + .expect("s3 disk must be registered") + .label(); + }); + assert_eq!(label, "s3://env-bucket"); + } + + #[test] + fn bucket_can_come_from_aws_bucket_alone() { + let toml = r#" +[storage] +default = "s3" + +[storage.disks.s3] +driver = "s3" +"#; + let mut label = String::new(); + with_env(&[("AWS_BUCKET", "env-bucket")], || { + let manager = crate::StorageManager::from_toml(toml).expect("s3 disk must build"); + label = manager + .disk("s3") + .expect("s3 disk must be registered") + .label(); + }); + assert_eq!(label, "s3://env-bucket"); + } + + #[test] + fn blank_bucket_is_rejected() { + let toml = r#" +[storage] +default = "s3" + +[storage.disks.s3] +driver = "s3" +bucket = " " +"#; + let mut outcome = None; + with_env(&[("AWS_BUCKET", "")], || { + outcome = Some(crate::StorageManager::from_toml(toml).map(|_| ())); + }); + assert!( + matches!(outcome, Some(Err(crate::StorageError::Config(_)))), + "got {outcome:?}" + ); + } +} diff --git a/crates/rustasea-storage/src/sftp/mod.rs b/crates/rustasea-storage/src/sftp/mod.rs index bd62ee4..2d06287 100644 --- a/crates/rustasea-storage/src/sftp/mod.rs +++ b/crates/rustasea-storage/src/sftp/mod.rs @@ -6,8 +6,8 @@ //! //! # Environment bridge //! -//! Unlike the cloud drivers, SFTP has no layered `ConfigLoader` bridge in this -//! crate, so a single-underscore `SFTP_*` overlay is applied by +//! SFTP has no layered `ConfigLoader` bridge in this crate, so (like the `s3` +//! driver's `AWS_*` overlay) a single-underscore `SFTP_*` overlay is applied by //! [`SftpDiskConfig::apply_env`]: `SFTP_HOST`, `SFTP_PORT`, `SFTP_USERNAME`, //! `SFTP_PASSWORD`, `SFTP_PRIVATE_KEY` (→ `private_key_path`), `SFTP_ROOT`, and //! `SFTP_TIMEOUT`. diff --git a/crates/rustasea-storage/tests/rustfs_container.rs b/crates/rustasea-storage/tests/rustfs_container.rs new file mode 100644 index 0000000..28c8c11 --- /dev/null +++ b/crates/rustasea-storage/tests/rustfs_container.rs @@ -0,0 +1,211 @@ +//! Docker-backed S3 integration tests against RustFS. +//! +//! These exercise the real `s3` disk, built through [`StorageManager`] exactly +//! as `config/storage.toml` is, against a `rustfs/rustfs` container on a +//! plain-HTTP endpoint (the local-development setup documented by +//! `config/storage.toml` and `.env.example`). They are `#[ignore]` because +//! they require a Docker daemon; run them explicitly: +//! +//! ```text +//! cargo test -p rustasea-storage --features aws --test rustfs_container -- --ignored --nocapture +//! ``` + +#![cfg(feature = "aws")] + +use std::sync::Mutex; +use std::time::Duration; + +use object_store::aws::{AwsAuthorizer, AwsCredential}; +use object_store::client::{HttpConnector, HttpRequest, HttpRequestBody, ReqwestConnector}; +use object_store::ClientOptions; +use rustasea_storage::{ManagedDisk, StorageManager}; +use testcontainers::core::ContainerPort; +use testcontainers::runners::AsyncRunner; +use testcontainers::{ContainerAsync, GenericImage, ImageExt}; + +/// Root credentials the container is started with. +const ACCESS_KEY: &str = "rustasea-test"; +/// Root secret the container is started with. +const SECRET_KEY: &str = "rustasea-test-secret"; +/// Bucket created before each test (RustFS starts empty). +const BUCKET: &str = "rustasea-test"; +/// Signing region (the SDK default, which RustFS accepts). +const REGION: &str = "us-east-1"; +/// How long to retry `CreateBucket` while RustFS starts before giving up. +const READY_TIMEOUT: Duration = Duration::from_secs(30); + +/// Every `AWS_*` variable the `s3` disk overlays onto its config. +const AWS_VARS: [&str; 5] = [ + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_DEFAULT_REGION", + "AWS_BUCKET", + "AWS_ENDPOINT", +]; + +/// Serializes the process-global `AWS_*` environment across tests. +static ENV_LOCK: Mutex<()> = Mutex::new(()); + +/// Start a RustFS container and create [`BUCKET`] once its S3 API is ready. +/// +/// Returns the container (dropping it stops RustFS) and its `http://` endpoint. +async fn start_rustfs() -> (ContainerAsync, String) { + let container = GenericImage::new("rustfs/rustfs", "1.0.0") + .with_exposed_port(ContainerPort::Tcp(9000)) + .with_env_var("RUSTFS_ACCESS_KEY", ACCESS_KEY) + .with_env_var("RUSTFS_SECRET_KEY", SECRET_KEY) + .start() + .await + .expect("rustfs container must start"); + let port = container + .get_host_port_ipv4(9000u16) + .await + .expect("S3 port must be mapped"); + let endpoint = format!("http://127.0.0.1:{port}"); + create_bucket(&endpoint).await; + (container, endpoint) +} + +/// Create [`BUCKET`] with a SigV4-signed `CreateBucket` request, retrying +/// while RustFS starts. +/// +/// `object_store` has no bucket-management API, so the request is signed with +/// its public [`AwsAuthorizer`] and sent through its HTTP client. RustFS +/// accepts connections (and answers `/health`) before its object layer is +/// ready and replies `503 Service Unavailable` meanwhile, so transport errors +/// (such as a refused connection) and 503s are retried until +/// [`READY_TIMEOUT`]; any other response is fatal. +async fn create_bucket(endpoint: &str) { + let credential = AwsCredential { + key_id: ACCESS_KEY.to_string(), + secret_key: SECRET_KEY.to_string(), + token: None, + }; + let client = ReqwestConnector {} + .connect(&ClientOptions::new().with_allow_http(true)) + .expect("HTTP client must build"); + let deadline = tokio::time::Instant::now() + READY_TIMEOUT; + loop { + let mut request = HttpRequest::new(HttpRequestBody::empty()); + *request.method_mut() = "PUT".parse().expect("PUT is a valid method"); + *request.uri_mut() = format!("{endpoint}/{BUCKET}") + .parse() + .expect("bucket URL must parse"); + AwsAuthorizer::new(&credential, "s3", REGION) + .try_authorize(&mut request, None) + .expect("CreateBucket request must sign"); + let last_error = match client.execute(request).await { + Ok(response) if response.status().is_success() => return, + Ok(response) if response.status().as_u16() == 503 => response.status().to_string(), + Ok(response) => { + let status = response.status(); + let body = response.into_body().bytes().await.unwrap_or_default(); + panic!( + "CreateBucket failed with {status}: {}", + String::from_utf8_lossy(&body) + ) + } + Err(error) => error.to_string(), + }; + assert!( + tokio::time::Instant::now() < deadline, + "rustfs did not become ready at {endpoint}: {last_error}" + ); + tokio::time::sleep(Duration::from_millis(200)).await; + } +} + +/// Run `body` with the `AWS_*` overlay variables set to `values`, restoring +/// the prior environment afterwards. +/// +/// Variables not listed are set blank (which the overlay ignores), so an +/// ambient AWS configuration on the developer machine cannot leak in. +fn with_aws_env(values: &[(&str, &str)], body: impl FnOnce() -> T) -> T { + let _guard = ENV_LOCK.lock().unwrap_or_else(|poison| poison.into_inner()); + let prior: Vec<(&str, Option)> = AWS_VARS + .iter() + .map(|key| (*key, std::env::var(key).ok())) + .collect(); + for key in AWS_VARS { + let value = values + .iter() + .find(|(name, _)| *name == key) + .map_or("", |(_, value)| *value); + std::env::set_var(key, value); + } + let result = body(); + for (key, value) in prior { + match value { + Some(value) => std::env::set_var(key, value), + None => std::env::remove_var(key), + } + } + result +} + +/// Put, read back, probe, and delete one object through `disk`. +async fn assert_round_trip(disk: &dyn ManagedDisk) { + let key = "reports/2026/hello.txt"; + disk.put(key, b"hello rustfs") + .await + .expect("put must succeed"); + assert!(disk.exists(key).await.expect("exists must succeed")); + assert_eq!( + disk.get(key).await.expect("get must succeed"), + b"hello rustfs" + ); + disk.delete(key).await.expect("delete must succeed"); + assert!(!disk.exists(key).await.expect("exists must succeed")); +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn s3_disk_round_trips_over_plain_http_endpoint() { + let (_container, endpoint) = start_rustfs().await; + let toml = format!( + r#" +[storage] +default = "s3" + +[storage.disks.s3] +driver = "s3" +bucket = "{BUCKET}" +region = "{REGION}" +access_key_id = "{ACCESS_KEY}" +secret_access_key = "{SECRET_KEY}" +endpoint = "{endpoint}" +"# + ); + let manager = + with_aws_env(&[], || StorageManager::from_toml(&toml)).expect("s3 disk must build"); + let disk = manager.disk("s3").expect("s3 disk must be registered"); + assert_round_trip(disk.as_ref()).await; +} + +#[tokio::test] +#[ignore = "requires docker"] +async fn s3_disk_reads_documented_aws_environment() { + let (_container, endpoint) = start_rustfs().await; + // `.env.example` documents that the credentials, region, bucket, and + // endpoint come from `AWS_*`, so the file names only the driver. + let toml = r#" +[storage] +default = "s3" + +[storage.disks.s3] +driver = "s3" +"#; + let manager = with_aws_env( + &[ + ("AWS_ACCESS_KEY_ID", ACCESS_KEY), + ("AWS_SECRET_ACCESS_KEY", SECRET_KEY), + ("AWS_DEFAULT_REGION", REGION), + ("AWS_BUCKET", BUCKET), + ("AWS_ENDPOINT", &endpoint), + ], + || StorageManager::from_toml(toml), + ) + .expect("s3 disk must build"); + let disk = manager.disk("s3").expect("s3 disk must be registered"); + assert_round_trip(disk.as_ref()).await; +} From 702e3c83551b5183b764a0ce0547983163016907 Mon Sep 17 00:00:00 2001 From: kangmaup Date: Thu, 24 Sep 2026 12:37:26 +0700 Subject: [PATCH 3/5] feat(rustasea): add storage-s3 umbrella feature Forwards to `rustasea-storage/aws`, next to `storage-sftp`, so apps built on the facade crate can enable `s3` disks without depending on `rustasea-storage` directly. Documents the feature in the README feature table, the storage crate README, and `config/storage.toml`, including that the `aws` feature needs rustc 1.89+: `object_store`'s AWS backend pulls in `crc-fast` 1.10, whose MSRV is above the workspace's 1.88. --- README.md | 1 + config/storage.toml | 7 ++++--- crates/rustasea-storage/README.md | 18 ++++++++++++++++++ crates/rustasea/Cargo.toml | 4 ++++ 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 379a41a..15f1f6d 100644 --- a/README.md +++ b/README.md @@ -416,6 +416,7 @@ The `rustasea` umbrella crate re-exports the whole framework, but several crates | `action` | `dep:rustasea-action` | Action pattern adapters for HTTP/queue/CLI/events, `make:action` (ADOPT-028) | | `google` | `dep:rustasea-google` | Service-account auth with cached OAuth2 access tokens (ADOPT-026) | | `storage-sftp` | `rustasea-storage/sftp` | Pure-Rust `russh`/`russh-sftp` SFTP disk (ADOPT-025) | +| `storage-s3` | `rustasea-storage/aws` | S3 disk for AWS or S3-compatible services (RustFS, MinIO, R2), with the `AWS_*` env overlay and plain-HTTP local endpoints (needs rustc 1.89+, via `crc-fast`) | | `excel` | `dep:rustasea-excel` | Excel/CSV import-export with queued jobs and signed links (ADOPT-023) | | `image` | `dep:rustasea-image` | Image transform pipeline with EXIF auto-orient (ADOPT-024) | | `debugbar` | `dep:rustasea-debugbar` | Dev request profiler / debug toolbar (ADOPT-009) | diff --git a/config/storage.toml b/config/storage.toml index 87e805b..33562f4 100644 --- a/config/storage.toml +++ b/config/storage.toml @@ -6,9 +6,10 @@ # contract (read-through with optional copy-back to the primary). # # Cloud drivers are opt-in: enable the matching crate feature -# (`rustasea-storage/{aws,gcp,azure}`) before referencing `s3`, `gcs`, or -# `azure` disks. The `sftp` driver is likewise opt-in -# (`rustasea-storage/sftp`, or `storage-sftp` on the `rustasea` facade crate). +# (`rustasea-storage/{aws,gcp,azure}`, or `storage-s3` on the `rustasea` facade +# crate for `s3`) before referencing `s3`, `gcs`, or `azure` disks. The `sftp` +# driver is likewise opt-in (`rustasea-storage/sftp`, or `storage-sftp` on the +# `rustasea` facade crate). [storage] default = "local" diff --git a/crates/rustasea-storage/README.md b/crates/rustasea-storage/README.md index 1194c7d..a9d374b 100644 --- a/crates/rustasea-storage/README.md +++ b/crates/rustasea-storage/README.md @@ -11,6 +11,24 @@ Part of the [RustaSea framework](https://github.com/rustasea/framework) - a Lara rustasea-storage = "0.1" ``` +## S3 and S3-compatible disks + +Enable the `aws` feature (or `storage-s3` on the `rustasea` facade crate) to +build `driver = "s3"` disks from `config/storage.toml`. `AWS_ACCESS_KEY_ID`, +`AWS_SECRET_ACCESS_KEY`, `AWS_DEFAULT_REGION`, `AWS_BUCKET`, and `AWS_ENDPOINT` +override the matching keys on every `s3` disk (blank values are ignored). An +`http://` endpoint enables plain HTTP for local S3-compatible services such as +RustFS or MinIO; any other endpoint stays HTTPS-only. + +The `aws` feature needs rustc 1.89 or newer: `object_store`'s AWS backend +depends on `crc-fast` 1.10, which raised its MSRV above the workspace's 1.88. + +The Docker-backed RustFS suite runs with: + +```text +cargo test -p rustasea-storage --features aws --test rustfs_container -- --ignored +``` + ## License MIT - see [LICENSE-MIT](https://github.com/rustasea/framework/blob/master/LICENSE-MIT). diff --git a/crates/rustasea/Cargo.toml b/crates/rustasea/Cargo.toml index fe55c0b..2d36dd5 100644 --- a/crates/rustasea/Cargo.toml +++ b/crates/rustasea/Cargo.toml @@ -181,6 +181,10 @@ broadcast-pusher = ["rustasea-broadcast/pusher"] # so `rustasea::storage` exposes the pure-Rust `russh`-backed SFTP disk; off by # default (pay-for-what-you-use, NFR-Sca-02). storage-sftp = ["rustasea-storage/sftp"] +# S3 storage disk. Forwards to `rustasea-storage`'s `aws` feature so `s3` disks +# (AWS or an S3-compatible service such as RustFS, MinIO, or R2) can be built +# from `config/storage.toml`; off by default (pay-for-what-you-use, NFR-Sca-02). +storage-s3 = ["rustasea-storage/aws"] # Locale-aware fake-data generation (ADOPT-012). Forwards to the testing crate's # opt-in `faker` feature so `rustasea::testing::faker` is available; off by # default (pay-for-what-you-use, NFR-Sca-02). From d8e0649dd50d4d6a86769c6a5c7e36ec71c8c72b Mon Sep 17 00:00:00 2001 From: kangmaup Date: Thu, 24 Sep 2026 12:37:26 +0700 Subject: [PATCH 4/5] ci: lint and test rustasea-storage with the aws and sftp features The `s3` and `sftp` disks sit behind `rustasea-storage`'s opt-in `aws` and `sftp` features, which the default workspace build never compiles, so a broken driver could not fail CI. `cargo xtask ci` now runs clippy on `rustasea-storage` with `--features aws,sftp`, and the test job runs its tests with both features. The Docker-backed RustFS and SFTP suites stay `#[ignore]`d. CONTRIBUTING and the README describe the new steps and the storage suites. --- .github/workflows/ci.yml | 5 +++++ CONTRIBUTING.md | 12 ++++++++++-- README.md | 3 ++- xtask/src/main.rs | 20 +++++++++++++++++++- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d16291..e1c9291 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,6 +62,11 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: cargo test run: cargo test --workspace + # The `s3` and `sftp` disks sit behind `rustasea-storage`'s opt-in `aws` + # and `sftp` features, so the default workspace run never compiles them. + # Their Docker-backed suites (RustFS, SFTP) stay `#[ignore]`d. + - name: cargo test (rustasea-storage, aws + sftp) + run: cargo test -p rustasea-storage --features aws,sftp # Supply-chain gate: license allow-list, advisory database, banned crates and # source provenance (configuration in deny.toml). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2e73f59..9ee87dd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -92,6 +92,14 @@ The integration suite requires a running Docker daemon and is opt-in: cargo test -p rustasea --features integration -- --ignored ``` +The storage drivers have their own Docker-backed suites (RustFS for the `s3` +disk, an SFTP server for the `sftp` disk): + +```bash +cargo test -p rustasea-storage --features aws --test rustfs_container -- --ignored +cargo test -p rustasea-storage --features sftp --test sftp_container -- --ignored +``` + ## Branch and commit conventions - **Branch from `master`.** Use a short, descriptive branch name such as @@ -125,8 +133,8 @@ cargo test -p rustasea --features integration -- --ignored | Job | What it runs | |---|---| - | `quality` | `cargo xtask ci` (fmt, clippy with `-D warnings`, `deps:check`, `lines:check`, cycle check) | - | `test` | `cargo test --workspace` | + | `quality` | `cargo xtask ci` (fmt, clippy with `-D warnings` on the workspace and on `rustasea-storage --features aws,sftp`, `deps:check`, `lines:check`, cycle check) | + | `test` | `cargo test --workspace`, then `cargo test -p rustasea-storage --features aws,sftp` | | `deny` | `cargo deny check` | | `audit` | `cargo audit` | | `msrv` | `cargo check --workspace` on Rust 1.88.0 | diff --git a/README.md b/README.md index 15f1f6d..61b836c 100644 --- a/README.md +++ b/README.md @@ -760,7 +760,8 @@ cargo xtask migrate # run migrations ``` CI runs the same gate — see [`.github/workflows/ci.yml`](.github/workflows/ci.yml): -`quality` (`cargo xtask ci`), `test` (`cargo test --workspace`), `deny` +`quality` (`cargo xtask ci`), `test` (`cargo test --workspace`, plus +`rustasea-storage` with its opt-in `aws`/`sftp` drivers), `deny` (`cargo deny check`), `audit` (`cargo audit`), and an `msrv` job that checks the workspace builds on the 1.88 floor (ADR-0001). **Formatting violations fail the build**: run `cargo fmt --all` before pushing, or `cargo xtask fmt` to check. diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 0a8d0b0..e84ad9d 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -58,7 +58,8 @@ fn main() { std::process::exit(code); } -/// Run the full CI gate: fmt → clippy → deps:check → cycle check. +/// Run the full CI gate: fmt → clippy (workspace, then the opt-in storage +/// drivers) → deps:check → lines:check → cycle check. fn run_ci() -> i32 { let steps: &[(&str, &[&str])] = &[ ("fmt", &["fmt", "--all", "--", "--check"]), @@ -73,6 +74,23 @@ fn run_ci() -> i32 { "warnings", ], ), + // The `s3` and `sftp` disks live behind `rustasea-storage`'s opt-in + // `aws` and `sftp` features, which the default workspace build never + // compiles. + ( + "clippy (rustasea-storage --features aws,sftp)", + &[ + "clippy", + "-p", + "rustasea-storage", + "--all-targets", + "--features", + "aws,sftp", + "--", + "-D", + "warnings", + ], + ), ]; for (label, args) in steps { println!("xtask ci: running {label}…"); From 3d90393f30b474e8c29e1fbafd09ac68dadabefc Mon Sep 17 00:00:00 2001 From: kangmaup Date: Thu, 24 Sep 2026 12:37:26 +0700 Subject: [PATCH 5/5] docs: record s3 storage changes in CHANGELOG --- CHANGELOG.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f897e72..b77a1b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,15 @@ Per the same section, each tag is expected to pass `cargo xtask ci` and ### Added +- 2026-09-24 - `storage-s3` umbrella feature forwarding to + `rustasea-storage/aws` (the `aws` feature needs rustc 1.89+ through + `object_store`'s `crc-fast` dependency), and a Docker-backed RustFS suite + (`crates/rustasea-storage/tests/rustfs_container.rs`, run with + `--features aws -- --ignored`) that round-trips an `s3` disk over a + plain-HTTP endpoint. `cargo xtask ci` and the CI test job now also cover + `rustasea-storage` with its opt-in `aws` and `sftp` drivers (M6; + `feat(rustasea): add storage-s3 umbrella feature`; `ci: lint and test + rustasea-storage with the aws and sftp features`). - 2026-09-17 - Svelte starter-kit variant with full auth/settings page parity: the `svelte` variant scaffolds the same 11 auth/settings pages as the react/vue kits on the shared Inertia contract @@ -120,6 +129,21 @@ Per the same section, each tag is expected to pass `cargo xtask ci` and ### Changed +- 2026-09-24 - `s3` disks now work with S3-compatible services such as RustFS + and MinIO: an explicit `http://` endpoint enables `object_store`'s + `allow_http` (previously every request failed with a reqwest builder error, + including the documented `endpoint = "http://localhost:9000"` example). + The `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_DEFAULT_REGION`, + `AWS_BUCKET`, and `AWS_ENDPOINT` variables documented in `.env.example`, + previously never read, now override the matching keys of every `s3` disk + in `config/storage.toml`; note that the stock `.env.example` sets + `AWS_DEFAULT_REGION=us-east-1`. `bucket` may be omitted when `AWS_BUCKET` + is set, and a blank bucket is rejected with `StorageError::Config`. + `S3DiskConfig` moved to `rustasea_storage::s3` (still re-exported from the + crate root and `facade`) and redacts `secret_access_key` from `Debug` + output (M6; `refactor(storage): move S3DiskConfig into its own module and + share env helpers`; `fix(storage): support S3-compatible http endpoints + and AWS_* env for s3 disks`). - 2026-09-17 - `cargo install rustasea` now installs the application scaffolder: the `cargo-rustasea` binary moved into the `rustasea` facade package (ships behind the `scaffold` feature, enabled by default) so `cargo rustasea new`