From 77536f69045c3272a19e23bc1c61392c9c6d7783 Mon Sep 17 00:00:00 2001 From: Guflly <145608489+Guflly@users.noreply.github.com> Date: Sun, 2 Aug 2026 17:39:14 -0700 Subject: [PATCH 1/2] fix: expose s3-specific endpoint config --- obstore/python/obstore/_store/_aws.pyi | 14 ++++++++++++-- pyo3-object_store/src/aws/store.rs | 19 +++++++++++++------ tests/store/test_s3.py | 24 ++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/obstore/python/obstore/_store/_aws.pyi b/obstore/python/obstore/_store/_aws.pyi index 365b6d00..2f0ae897 100644 --- a/obstore/python/obstore/_store/_aws.pyi +++ b/obstore/python/obstore/_store/_aws.pyi @@ -242,8 +242,7 @@ class S3Config(TypedDict, total=False): **Environment variables**: - `AWS_ENDPOINT` - - `AWS_ENDPOINT_URL` - - `AWS_ENDPOINT_URL_S3`. + - `AWS_ENDPOINT_URL`. """ imdsv1_fallback: bool @@ -310,6 +309,17 @@ class S3Config(TypedDict, total=False): **Environment variable**: `AWS_S3_EXPRESS`. """ + s3_endpoint: str + """The service-specific endpoint for communicating with AWS S3. + + This takes precedence over `endpoint` when both are set. + + By default, only HTTPS schemes are enabled. To connect to an HTTP endpoint, enable + `allow_http` in the client options. + + **Environment variable**: `AWS_ENDPOINT_URL_S3`. + """ + secret_access_key: str """Secret Access Key. diff --git a/pyo3-object_store/src/aws/store.rs b/pyo3-object_store/src/aws/store.rs index f8041c4a..d7b05926 100644 --- a/pyo3-object_store/src/aws/store.rs +++ b/pyo3-object_store/src/aws/store.rs @@ -230,7 +230,11 @@ impl<'py> FromPyObject<'_, 'py> for PyAmazonS3ConfigKey { fn extract(obj: Borrowed<'_, 'py, pyo3::PyAny>) -> PyResult { let s = obj.extract::()?.to_lowercase(); - let key = s.parse().map_err(PyObjectStoreError::ObjectStoreError)?; + let key = if s == "s3_endpoint" { + AmazonS3ConfigKey::S3Endpoint + } else { + s.parse().map_err(PyObjectStoreError::ObjectStoreError)? + }; Ok(Self(key)) } } @@ -247,11 +251,14 @@ impl<'py> IntoPyObject<'py> for &PyAmazonS3ConfigKey { type Error = PyErr; fn into_pyobject(self, py: Python<'py>) -> Result { - let s = self - .0 - .as_ref() - .strip_prefix("aws_") - .expect("Expected config prefix to start with aws_"); + let s = if self.0 == AmazonS3ConfigKey::S3Endpoint { + "s3_endpoint" + } else { + self.0 + .as_ref() + .strip_prefix("aws_") + .expect("Expected config prefix to start with aws_") + }; Ok(PyString::new(py, s)) } } diff --git a/tests/store/test_s3.py b/tests/store/test_s3.py index dd9061f2..3fc47eef 100644 --- a/tests/store/test_s3.py +++ b/tests/store/test_s3.py @@ -1,13 +1,18 @@ # ruff: noqa: PGH003 +from __future__ import annotations import pickle from datetime import datetime, timezone +from typing import TYPE_CHECKING import pytest from obstore.exceptions import BaseError, UnauthenticatedError from obstore.store import S3Store, from_url +if TYPE_CHECKING: + from obstore.store import ClientConfig, S3Config + @pytest.mark.asyncio async def test_list_async(minio_store: S3Store): @@ -103,6 +108,25 @@ def test_config_round_trip(): assert store.retry_config == new_store.retry_config +def test_s3_endpoint_overrides_environment( + monkeypatch: pytest.MonkeyPatch, + minio_bucket: tuple[S3Config, ClientConfig], +): + config, client_options = minio_bucket + monkeypatch.setenv("AWS_ENDPOINT_URL_S3", "http://localhost:1") + endpoint = config.get("endpoint") + assert endpoint is not None + store = S3Store( + config=config, + client_options=client_options, + s3_endpoint=endpoint, + ) + + assert store.list().collect() == [] + assert store.config.get("s3_endpoint") == endpoint + assert pickle.loads(pickle.dumps(store)) == store + + def test_invalid_credential_provider(): """Test that passing an invalid synchronous credential provider raises an error. From ca538590ce0a36b932f17b7f2da4617f3566f47e Mon Sep 17 00:00:00 2001 From: Guflly <145608489+Guflly@users.noreply.github.com> Date: Wed, 5 Aug 2026 16:35:53 -0700 Subject: [PATCH 2/2] fix: use upstream s3 endpoint key --- obstore/python/obstore/_store/_aws.pyi | 21 ++++++++++----------- pyo3-object_store/src/aws/store.rs | 19 ++++++------------- tests/store/test_s3.py | 24 ------------------------ 3 files changed, 16 insertions(+), 48 deletions(-) diff --git a/obstore/python/obstore/_store/_aws.pyi b/obstore/python/obstore/_store/_aws.pyi index 2f0ae897..948aa613 100644 --- a/obstore/python/obstore/_store/_aws.pyi +++ b/obstore/python/obstore/_store/_aws.pyi @@ -92,6 +92,16 @@ class S3Config(TypedDict, total=False): **Environment variable**: `AWS_ACCESS_KEY_ID`. """ + aws_endpoint_url_s3: str + """The service-specific endpoint for communicating with AWS S3. + + This takes precedence over `endpoint` when both are set. + + By default, only HTTPS schemes are enabled. To connect to an HTTP endpoint, enable + `allow_http` in the client options. + + **Environment variable**: `AWS_ENDPOINT_URL_S3`. + """ bucket: str """Bucket name (required). @@ -309,17 +319,6 @@ class S3Config(TypedDict, total=False): **Environment variable**: `AWS_S3_EXPRESS`. """ - s3_endpoint: str - """The service-specific endpoint for communicating with AWS S3. - - This takes precedence over `endpoint` when both are set. - - By default, only HTTPS schemes are enabled. To connect to an HTTP endpoint, enable - `allow_http` in the client options. - - **Environment variable**: `AWS_ENDPOINT_URL_S3`. - """ - secret_access_key: str """Secret Access Key. diff --git a/pyo3-object_store/src/aws/store.rs b/pyo3-object_store/src/aws/store.rs index d7b05926..f8041c4a 100644 --- a/pyo3-object_store/src/aws/store.rs +++ b/pyo3-object_store/src/aws/store.rs @@ -230,11 +230,7 @@ impl<'py> FromPyObject<'_, 'py> for PyAmazonS3ConfigKey { fn extract(obj: Borrowed<'_, 'py, pyo3::PyAny>) -> PyResult { let s = obj.extract::()?.to_lowercase(); - let key = if s == "s3_endpoint" { - AmazonS3ConfigKey::S3Endpoint - } else { - s.parse().map_err(PyObjectStoreError::ObjectStoreError)? - }; + let key = s.parse().map_err(PyObjectStoreError::ObjectStoreError)?; Ok(Self(key)) } } @@ -251,14 +247,11 @@ impl<'py> IntoPyObject<'py> for &PyAmazonS3ConfigKey { type Error = PyErr; fn into_pyobject(self, py: Python<'py>) -> Result { - let s = if self.0 == AmazonS3ConfigKey::S3Endpoint { - "s3_endpoint" - } else { - self.0 - .as_ref() - .strip_prefix("aws_") - .expect("Expected config prefix to start with aws_") - }; + let s = self + .0 + .as_ref() + .strip_prefix("aws_") + .expect("Expected config prefix to start with aws_"); Ok(PyString::new(py, s)) } } diff --git a/tests/store/test_s3.py b/tests/store/test_s3.py index 3fc47eef..dd9061f2 100644 --- a/tests/store/test_s3.py +++ b/tests/store/test_s3.py @@ -1,18 +1,13 @@ # ruff: noqa: PGH003 -from __future__ import annotations import pickle from datetime import datetime, timezone -from typing import TYPE_CHECKING import pytest from obstore.exceptions import BaseError, UnauthenticatedError from obstore.store import S3Store, from_url -if TYPE_CHECKING: - from obstore.store import ClientConfig, S3Config - @pytest.mark.asyncio async def test_list_async(minio_store: S3Store): @@ -108,25 +103,6 @@ def test_config_round_trip(): assert store.retry_config == new_store.retry_config -def test_s3_endpoint_overrides_environment( - monkeypatch: pytest.MonkeyPatch, - minio_bucket: tuple[S3Config, ClientConfig], -): - config, client_options = minio_bucket - monkeypatch.setenv("AWS_ENDPOINT_URL_S3", "http://localhost:1") - endpoint = config.get("endpoint") - assert endpoint is not None - store = S3Store( - config=config, - client_options=client_options, - s3_endpoint=endpoint, - ) - - assert store.list().collect() == [] - assert store.config.get("s3_endpoint") == endpoint - assert pickle.loads(pickle.dumps(store)) == store - - def test_invalid_credential_provider(): """Test that passing an invalid synchronous credential provider raises an error.