diff --git a/bin/clone-dependencies.sh b/bin/clone-dependencies.sh index 10260a8..1949ecb 100755 --- a/bin/clone-dependencies.sh +++ b/bin/clone-dependencies.sh @@ -15,7 +15,7 @@ set -u BDE_TOOLS_TAG=4.38.0.0 BDE_TAG=4.38.0.0 NTF_CORE_TAG=2.6.12 -BLAZINGMQ_TAG=v0.95.14 +BLAZINGMQ_TAG=v0.95.20 if [ ! -d "${DIR_THIRDPARTY}/bde-tools" ]; then diff --git a/docs/api_reference.rst b/docs/api_reference.rst index b380ab8..438ff85 100644 --- a/docs/api_reference.rst +++ b/docs/api_reference.rst @@ -187,6 +187,8 @@ Testing Utilities Helper Types ============ +.. autoclass:: blazingmq.AuthnCredentialProvider + .. autoclass:: blazingmq.PropertyTypeDict .. autoclass:: blazingmq.PropertyValueDict diff --git a/src/blazingmq/__init__.py b/src/blazingmq/__init__.py index 910d1ad..e630190 100644 --- a/src/blazingmq/__init__.py +++ b/src/blazingmq/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2019-2023 Bloomberg Finance L.P. +# Copyright 2019-2026 Bloomberg Finance L.P. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -27,6 +27,7 @@ from ._session import Session from ._session import SessionOptions from ._timeouts import Timeouts +from ._typing import AuthnCredentialProvider from ._typing import PropertyTypeDict from ._typing import PropertyValueDict from .exceptions import Error @@ -34,6 +35,7 @@ __all__ = [ "Ack", "AckStatus", + "AuthnCredentialProvider", "BasicHealthMonitor", "CompressionAlgorithmType", "Error", diff --git a/src/blazingmq/_ext.pyi b/src/blazingmq/_ext.pyi index 14d2478..1637d33 100644 --- a/src/blazingmq/_ext.pyi +++ b/src/blazingmq/_ext.pyi @@ -1,4 +1,4 @@ -# Copyright 2019-2023 Bloomberg Finance L.P. +# Copyright 2019-2026 Bloomberg Finance L.P. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -37,6 +37,9 @@ class FakeHostHealthMonitor: def set_healthy(self) -> None: ... def set_unhealthy(self) -> None: ... +class AuthnCredentialCbAdapter: + def __init__(self, callback: Callable[[], Optional[tuple[str, bytes]]]) -> None: ... + class Session: def __init__( self, @@ -53,6 +56,7 @@ class Session: timeouts: Timeouts = Timeouts(), monitor_host_health: bool = False, fake_host_health_monitor: Optional[FakeHostHealthMonitor] = None, + authn_credential_cb: Optional[AuthnCredentialCbAdapter] = None, ) -> None: ... def stop(self) -> None: ... def open_queue_sync( diff --git a/src/blazingmq/_ext.pyx b/src/blazingmq/_ext.pyx index e8ecb9d..30a3b76 100644 --- a/src/blazingmq/_ext.pyx +++ b/src/blazingmq/_ext.pyx @@ -1,4 +1,4 @@ -# Copyright 2019-2023 Bloomberg Finance L.P. +# Copyright 2019-2026 Bloomberg Finance L.P. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -37,6 +37,7 @@ from bmq.bmqt cimport k_DEFAULT_MAX_UNCONFIRMED_MESSAGES from bmq.bmqt cimport k_DEFAULT_SUSPENDS_ON_BAD_HOST_HEALTH from pybmq cimport BallUtil from pybmq cimport Session as NativeSession +from pybmq cimport SessionConfig from typing import Optional @@ -153,6 +154,37 @@ cdef class FakeHostHealthMonitor: self._monitor.get().setState(HostHealthState.e_UNHEALTHY) +cdef class AuthnCredentialCbAdapter: + cdef object _callback + + def __cinit__(self, callback): + self._callback = callback + + def get_credential_data(self): + """Call the provider and marshal its result for the C++ session. + + Called by ``pybmq::AuthnCredentialCbFunctor``. Returns the mechanism + and data as a tuple of ``bytes``, or `None` if credentials could not + be obtained, in which case authentication fails. + """ + try: + result = self._callback() + if result is None: + return None + + mechanism, data = result + if not isinstance(mechanism, str) or not isinstance(data, bytes): + raise TypeError( + "authn_credential_provider must return (str, bytes) or None" + ) + + return mechanism.encode('utf-8'), data + + except Exception: + LOGGER.exception("Error in authentication credential callback") + return None + + cdef class Session: cdef object __weakref__ cdef NativeSession* _session @@ -174,6 +206,7 @@ cdef class Session: timeouts: _timeouts.Timeouts = _timeouts.Timeouts(), monitor_host_health: bool = False, fake_host_health_monitor: FakeHostHealthMonitor = None, + authn_credential_cb: AuthnCredentialCbAdapter = None, _mock: Optional[object] = None, ) -> None: cdef shared_ptr[ManualHostHealthMonitor] fake_host_health_monitor_sp @@ -221,24 +254,30 @@ cdef class Session: cdef char *c_broker_uri = broker script_name = _script_name.get_script_name() cdef char *c_script_name = script_name + + cdef SessionConfig config + config.broker_uri = c_broker_uri + config.script_name = c_script_name + config.message_compression_type = ( + COMPRESSION_ALGO_FROM_PY_MAPPING[message_compression_algorithm]) + config.num_processing_threads = c_num_processing_threads + config.blob_buffer_size = c_blob_buffer_size + config.channel_high_watermark = c_channel_high_watermark + config.event_queue_watermarks = c_event_queue_watermarks + config.stats_dump_interval = c_stats_dump_interval + config.connect_timeout = c_connect_timeout + config.disconnect_timeout = c_disconnect_timeout + config.open_queue_timeout = c_open_queue_timeout + config.configure_queue_timeout = c_configure_queue_timeout + config.close_queue_timeout = c_close_queue_timeout + config.monitor_host_health = monitor_host_health + self._session = new NativeSession( session_cb, message_cb, ack_cb, - c_broker_uri, - c_script_name, - COMPRESSION_ALGO_FROM_PY_MAPPING[message_compression_algorithm], - c_num_processing_threads, - c_blob_buffer_size, - c_channel_high_watermark, - c_event_queue_watermarks, - c_stats_dump_interval, - c_connect_timeout, - c_disconnect_timeout, - c_open_queue_timeout, - c_configure_queue_timeout, - c_close_queue_timeout, - monitor_host_health, + authn_credential_cb, + config, fake_host_health_monitor_sp, Error, BrokerTimeoutError, diff --git a/src/blazingmq/_session.py b/src/blazingmq/_session.py index e0c09e3..ecba670 100644 --- a/src/blazingmq/_session.py +++ b/src/blazingmq/_session.py @@ -1,4 +1,4 @@ -# Copyright 2019-2023 Bloomberg Finance L.P. +# Copyright 2019-2026 Bloomberg Finance L.P. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,6 +25,7 @@ from . import _six as six from ._enums import CompressionAlgorithmType from ._enums import PropertyType +from ._ext import AuthnCredentialCbAdapter from ._ext import DEFAULT_CONSUMER_PRIORITY from ._ext import DEFAULT_MAX_UNCONFIRMED_BYTES from ._ext import DEFAULT_MAX_UNCONFIRMED_MESSAGES @@ -36,6 +37,7 @@ from ._messages import MessageHandle from ._monitors import BasicHealthMonitor from ._timeouts import Timeouts +from ._typing import AuthnCredentialProvider from ._typing import PropertyTypeDict from ._typing import PropertyValueDict from ._typing import PropertyValueType @@ -52,6 +54,10 @@ def DefaultMonitor() -> Union[BasicHealthMonitor, None]: return None +def DefaultAuthnCredentialProvider() -> Optional[AuthnCredentialProvider]: + return None + + DEFAULT_TIMEOUT = DefaultTimeoutType() KNOWN_MONITORS = ("blazingmq.BasicHealthMonitor",) @@ -288,6 +294,15 @@ class SessionOptions: 0, disable the recurring dump of stats (final stats are always dumped at the end of the session). The default is 5min; the value must be a multiple of 30s, in the range ``[0s - 60min]``. + authn_credential_provider (Optional[`~blazingmq.AuthnCredentialProvider`]): + An optional callable that returns authentication credentials as a + ``(mechanism, data)`` tuple of ``(str, bytes)``. It is called + each time the session authenticates with the broker, including on + reauthentication. If it returns ``None`` or raises, the + connection is closed: starting a session fails, while an + already-started session sees `.ConnectionLost` and then + reconnects, calling this callable again. If not provided, no + authentication credentials are sent to the broker. """ def __init__( @@ -300,6 +315,9 @@ def __init__( channel_high_watermark: Optional[int] = None, event_queue_watermarks: Optional[tuple[int, int]] = None, stats_dump_interval: Optional[float] = None, + authn_credential_provider: Optional[AuthnCredentialProvider] = ( + DefaultAuthnCredentialProvider() + ), ) -> None: self.message_compression_algorithm = message_compression_algorithm self.timeouts = timeouts @@ -309,6 +327,7 @@ def __init__( self.channel_high_watermark = channel_high_watermark self.event_queue_watermarks = event_queue_watermarks self.stats_dump_interval = stats_dump_interval + self.authn_credential_provider = authn_credential_provider def __eq__(self, other: object) -> bool: if not isinstance(other, SessionOptions): @@ -322,6 +341,7 @@ def __eq__(self, other: object) -> bool: and self.channel_high_watermark == other.channel_high_watermark and self.event_queue_watermarks == other.event_queue_watermarks and self.stats_dump_interval == other.stats_dump_interval + and self.authn_credential_provider == other.authn_credential_provider ) def __ne__(self, other: object) -> bool: @@ -337,6 +357,7 @@ def __repr__(self) -> str: "channel_high_watermark", "event_queue_watermarks", "stats_dump_interval", + "authn_credential_provider", ) params = [] @@ -399,6 +420,15 @@ class Session: stats are always dumped at the end of the session). The default is 5min; the value must be a multiple of 30s, in the range ``[0s - 60min]``. + authn_credential_provider (Optional[`~blazingmq.AuthnCredentialProvider`]): + an optional callable that returns authentication credentials as a + ``(mechanism, data)`` tuple of ``(str, bytes)``. It is called + each time the session authenticates with the broker, including on + reauthentication. If it returns ``None`` or raises, the + connection is closed: starting a session fails, while an + already-started session sees `.ConnectionLost` and then + reconnects, calling this callable again. If not provided, no + authentication credentials are sent to the broker. Raises: `~blazingmq.Error`: If the session start request was not successful. @@ -423,6 +453,9 @@ def __init__( channel_high_watermark: Optional[int] = None, event_queue_watermarks: Optional[tuple[int, int]] = None, stats_dump_interval: Optional[float] = None, + authn_credential_provider: Optional[AuthnCredentialProvider] = ( + DefaultAuthnCredentialProvider() + ), ) -> None: if host_health_monitor is not None: if not isinstance(host_health_monitor, BasicHealthMonitor): @@ -433,6 +466,11 @@ def __init__( monitor_host_health = host_health_monitor is not None fake_host_health_monitor = getattr(host_health_monitor, "_monitor", None) + authn_credential_cb = ( + AuthnCredentialCbAdapter(authn_credential_provider) + if authn_credential_provider is not None + else None + ) self._has_no_on_message = on_message is None @@ -459,6 +497,7 @@ def __init__( timeouts=_validate_timeouts(timeout), monitor_host_health=monitor_host_health, fake_host_health_monitor=fake_host_health_monitor, + authn_credential_cb=authn_credential_cb, ) self._ext.set_owned_by_session() @@ -503,34 +542,24 @@ def with_options( if message_compression_algorithm is None: message_compression_algorithm = CompressionAlgorithmType.NONE - if session_options.timeouts is None: - return cls( - on_session_event, - on_message, - broker, - message_compression_algorithm, - DEFAULT_TIMEOUT, - session_options.host_health_monitor, - session_options.num_processing_threads, - session_options.blob_buffer_size, - session_options.channel_high_watermark, - session_options.event_queue_watermarks, - session_options.stats_dump_interval, - ) - else: - return cls( - on_session_event, - on_message, - broker, - message_compression_algorithm, - session_options.timeouts, - session_options.host_health_monitor, - session_options.num_processing_threads, - session_options.blob_buffer_size, - session_options.channel_high_watermark, - session_options.event_queue_watermarks, - session_options.stats_dump_interval, - ) + timeout: Union[Timeouts, float] = DEFAULT_TIMEOUT + if session_options.timeouts is not None: + timeout = session_options.timeouts + + return cls( + on_session_event, + on_message=on_message, + broker=broker, + message_compression_algorithm=message_compression_algorithm, + timeout=timeout, + host_health_monitor=session_options.host_health_monitor, + num_processing_threads=session_options.num_processing_threads, + blob_buffer_size=session_options.blob_buffer_size, + channel_high_watermark=session_options.channel_high_watermark, + event_queue_watermarks=session_options.event_queue_watermarks, + stats_dump_interval=session_options.stats_dump_interval, + authn_credential_provider=session_options.authn_credential_provider, + ) def open_queue( self, diff --git a/src/blazingmq/_typing.py b/src/blazingmq/_typing.py index 39cf633..c6fe00a 100644 --- a/src/blazingmq/_typing.py +++ b/src/blazingmq/_typing.py @@ -1,4 +1,4 @@ -# Copyright 2019-2023 Bloomberg Finance L.P. +# Copyright 2019-2026 Bloomberg Finance L.P. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,7 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Callable from typing import Mapping +from typing import Optional from typing import Union from ._enums import PropertyType @@ -23,3 +25,10 @@ PropertyValueDict = Mapping[str, PropertyValueType] PropertyTypeDict = Mapping[str, PropertyType] + +AuthnCredentialProvider = Callable[[], Optional[tuple[str, bytes]]] +"""A callable that returns authentication credentials as a ``(mechanism, +data)`` tuple of ``(str, bytes)``, or ``None`` if an error occurs while +obtaining them, in which case authentication fails and the connection is +closed. +""" diff --git a/src/cpp/pybmq_pyref.h b/src/cpp/pybmq_pyref.h new file mode 100644 index 0000000..b817032 --- /dev/null +++ b/src/cpp/pybmq_pyref.h @@ -0,0 +1,114 @@ +// Copyright 2019-2026 Bloomberg Finance L.P. +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_PYBMQ_PYREF +#define INCLUDED_PYBMQ_PYREF + +#define PY_SSIZE_T_CLEAN +#include + +#include + +namespace BloombergLP { +namespace pybmq { + +// =========== +// class PyRef +// =========== + +/// A value-semantic owning reference to a Python object. +/// +/// Copying or destroying a `PyRef` adjusts the referent's reference count and +/// acquires the GIL to do so, so instances may be copied and destroyed on +/// threads that do not hold it. This is what the BlazingMQ SDK does to +/// callbacks it has been handed, on its own IO threads. +class PyRef +{ + private: + // DATA + PyObject* d_object_p; + + public: + // CREATORS + PyRef(); + + /// Create a reference to the specified `object`, which may be 0. + explicit PyRef(PyObject* object); + + PyRef(const PyRef& other); + + ~PyRef(); + + // MANIPULATORS + PyRef& operator=(const PyRef& rhs); + + // ACCESSORS + + /// Return the referent, or 0 if this reference is empty. The reference + /// count is not adjusted; the caller borrows the returned pointer. + PyObject* get() const; +}; + +// =========================================================================== +// INLINE DEFINITIONS +// =========================================================================== + +inline PyRef::PyRef() +: d_object_p(0) +{ +} + +inline PyRef::PyRef(PyObject* object) +: d_object_p(object) +{ + GilAcquireGuard guard; + Py_XINCREF(d_object_p); +} + +inline PyRef::PyRef(const PyRef& other) +: d_object_p(other.d_object_p) +{ + GilAcquireGuard guard; + Py_XINCREF(d_object_p); +} + +inline PyRef::~PyRef() +{ + GilAcquireGuard guard; + Py_XDECREF(d_object_p); +} + +inline PyRef& +PyRef::operator=(const PyRef& rhs) +{ + if (this != &rhs) { + GilAcquireGuard guard; + Py_XINCREF(rhs.d_object_p); + Py_XDECREF(d_object_p); + d_object_p = rhs.d_object_p; + } + return *this; +} + +inline PyObject* +PyRef::get() const +{ + return d_object_p; +} + +} // namespace pybmq +} // namespace BloombergLP + +#endif diff --git a/src/cpp/pybmq_session.cpp b/src/cpp/pybmq_session.cpp index acab799..d88c3f5 100644 --- a/src/cpp/pybmq_session.cpp +++ b/src/cpp/pybmq_session.cpp @@ -1,4 +1,4 @@ -// Copyright 2019-2023 Bloomberg Finance L.P. +// Copyright 2019-2026 Bloomberg Finance L.P. // SPDX-License-Identifier: Apache-2.0 // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,6 +15,7 @@ #include +#include #include #include #include @@ -25,12 +26,16 @@ #include #include #include +#include +#include #include #include +#include #include #include #include +#include #include #include #include @@ -71,26 +76,81 @@ class BrokerTimeoutError : public bsl::runtime_error } }; +// Invoke `get_credential_data()` on a Python object and convert the result to +// a `bmqt::AuthnCredential`. The object is held, not owned: the `Session` +// holds the reference and drops it only after destroying the `bmqa::Session` +// that owns every copy of this functor. +class AuthnCredentialCbFunctor +{ + private: + // DATA + PyObject* d_callback_p; + + public: + // CREATORS + explicit AuthnCredentialCbFunctor(PyObject* callback); + + // ACCESSORS + bsl::optional operator()() const; +}; + +AuthnCredentialCbFunctor::AuthnCredentialCbFunctor(PyObject* callback) +: d_callback_p(callback) +{ +} + +bsl::optional +AuthnCredentialCbFunctor::operator()() const +{ + pybmq::GilAcquireGuard guard; + + // The adapter validates the provider's result and reports any problem to + // the Python logger, so a failure here is just an empty credential. It + // hands back the mechanism and data already marshalled to `bytes`. + bslma::ManagedPtr result = RefUtils::toManagedPtr( + PyObject_CallMethod(d_callback_p, "get_credential_data", NULL)); + + if (!result) { + PyErr_WriteUnraisable(d_callback_p); + return bsl::optional(); + } + + if (result.get() == Py_None) { + return bsl::optional(); + } + + const char* mechanism_p; + Py_ssize_t mechanism_len; + const char* data_p; + Py_ssize_t data_len; + if (!PyArg_ParseTuple( + result.get(), + "y#y#", + &mechanism_p, + &mechanism_len, + &data_p, + &data_len)) + { + // The adapter broke its contract with us. + PyErr_WriteUnraisable(d_callback_p); + return bsl::optional(); + } + + bmqt::AuthnCredential credential( + bsl::string_view(mechanism_p, mechanism_len), + bsl::vector(data_p, data_p + data_len)); + return bsl::optional( + bslmf::MovableRefUtil::move(credential)); +} + } // namespace Session::Session( PyObject* py_session_event_callback, PyObject* py_message_event_callback, PyObject* py_ack_event_callback, - const char* broker_uri, - const char* script_name, - bmqt::CompressionAlgorithmType::Enum message_compression_type, - bsl::optional num_processing_threads, - bsl::optional blob_buffer_size, - bsl::optional channel_high_watermark, - bsl::optional > event_queue_watermarks, - const bsls::TimeInterval& stats_dump_interval, - const bsls::TimeInterval& connect_timeout, - const bsls::TimeInterval& disconnect_timeout, - const bsls::TimeInterval& open_queue_timeout, - const bsls::TimeInterval& configure_queue_timeout, - const bsls::TimeInterval& close_queue_timeout, - bool monitor_host_health, + PyObject* authn_credential_cb, + const SessionConfig& config, bsl::shared_ptr fake_host_health_monitor_sp, PyObject* error, PyObject* broker_timeout_error, @@ -100,72 +160,85 @@ Session::Session( , d_message_compression_type(bmqt::CompressionAlgorithmType::e_NONE) , d_error(error) , d_broker_timeout_error(broker_timeout_error) +, d_authn_credential_cb(NULL) , d_session_mp() { bsl::shared_ptr host_health_monitor_sp; if (fake_host_health_monitor_sp) { host_health_monitor_sp = fake_host_health_monitor_sp; - } else if (monitor_host_health) { + } else if (config.monitor_host_health) { } - if (message_compression_type + if (config.message_compression_type < bmqt::CompressionAlgorithmType::k_LOWEST_SUPPORTED_TYPE - || message_compression_type + || config.message_compression_type > bmqt::CompressionAlgorithmType::k_HIGHEST_SUPPORTED_TYPE) { PyErr_SetString(PyExc_ValueError, "Invalid message compression type"); throw bsl::runtime_error("propagating Python error"); } - d_message_compression_type = message_compression_type; + d_message_compression_type = config.message_compression_type; + + bmqt::SessionOptions::AuthnCredentialCb cpp_callback; + + if (authn_credential_cb != NULL && authn_credential_cb != Py_None) { + d_authn_credential_cb = authn_credential_cb; + cpp_callback = AuthnCredentialCbFunctor(d_authn_credential_cb); + } + { pybmq::GilReleaseGuard guard; bmqt::SessionOptions options; - options.setBrokerUri(broker_uri) - .setProcessNameOverride(script_name) + options.setBrokerUri(config.broker_uri) + .setProcessNameOverride(config.script_name) .setHostHealthMonitor(host_health_monitor_sp); - if (num_processing_threads.has_value()) { - options.setNumProcessingThreads(num_processing_threads.value()); + if (config.num_processing_threads.has_value()) { + options.setNumProcessingThreads(config.num_processing_threads.value()); } - if (blob_buffer_size.has_value()) { - options.setBlobBufferSize(blob_buffer_size.value()); + if (config.blob_buffer_size.has_value()) { + options.setBlobBufferSize(config.blob_buffer_size.value()); } - if (channel_high_watermark.has_value()) { - options.setChannelHighWatermark(channel_high_watermark.value()); + if (config.channel_high_watermark.has_value()) { + options.setChannelHighWatermark(config.channel_high_watermark.value()); } - if (event_queue_watermarks.has_value()) { + if (config.event_queue_watermarks.has_value()) { options.configureEventQueue( - event_queue_watermarks.value().first, - event_queue_watermarks.value().second); + config.event_queue_watermarks.value().first, + config.event_queue_watermarks.value().second); } - if (stats_dump_interval != bsls::TimeInterval()) { - options.setStatsDumpInterval(stats_dump_interval); + if (cpp_callback) { + options.setAuthnCredentialCb(cpp_callback); } - if (connect_timeout != bsls::TimeInterval()) { - options.setConnectTimeout(connect_timeout); + if (config.stats_dump_interval != bsls::TimeInterval()) { + options.setStatsDumpInterval(config.stats_dump_interval); } - if (disconnect_timeout != bsls::TimeInterval()) { - options.setDisconnectTimeout(disconnect_timeout); + if (config.connect_timeout != bsls::TimeInterval()) { + options.setConnectTimeout(config.connect_timeout); } - if (open_queue_timeout != bsls::TimeInterval()) { - options.setOpenQueueTimeout(open_queue_timeout); + if (config.disconnect_timeout != bsls::TimeInterval()) { + options.setDisconnectTimeout(config.disconnect_timeout); } - if (configure_queue_timeout != bsls::TimeInterval()) { - options.setConfigureQueueTimeout(configure_queue_timeout); + if (config.open_queue_timeout != bsls::TimeInterval()) { + options.setOpenQueueTimeout(config.open_queue_timeout); } - if (close_queue_timeout != bsls::TimeInterval()) { - options.setCloseQueueTimeout(close_queue_timeout); + if (config.configure_queue_timeout != bsls::TimeInterval()) { + options.setConfigureQueueTimeout(config.configure_queue_timeout); + } + + if (config.close_queue_timeout != bsls::TimeInterval()) { + options.setCloseQueueTimeout(config.close_queue_timeout); } bslma::ManagedPtr handler( @@ -183,15 +256,22 @@ Session::Session( } Py_INCREF(d_error); Py_INCREF(d_broker_timeout_error); + Py_XINCREF(d_authn_credential_cb); } Session::~Session() { + BSLS_ASSERT(!d_started); + { + // Destroy the session first: it owns the copies of + // `AuthnCredentialCbFunctor`, which borrow `d_authn_credential_cb`. + pybmq::GilReleaseGuard gil_release_guard; + d_session_mp.reset(); + } + + Py_XDECREF(d_authn_credential_cb); Py_DECREF(d_broker_timeout_error); Py_DECREF(d_error); - BSLS_ASSERT(!d_started); - pybmq::GilReleaseGuard gil_release_guard; - d_session_mp.reset(); } PyObject* @@ -529,8 +609,8 @@ Session::post( oss << "Failed to post message to " << queue_uri << " queue: " << post_rc; throw GenericError(oss.str()); } - // We have a successful post and the SDK now owns the `on_ack` callback object - // so release our reference without a DECREF. + // We have a successful post and the SDK now owns the `on_ack` callback + // object so release our reference without a DECREF. managed_on_ack.release(); } catch (const GenericError& exc) { PyErr_SetString(d_error, exc.what()); diff --git a/src/cpp/pybmq_session.h b/src/cpp/pybmq_session.h index f37a407..51fc1bd 100644 --- a/src/cpp/pybmq_session.h +++ b/src/cpp/pybmq_session.h @@ -1,4 +1,4 @@ -// Copyright 2019-2023 Bloomberg Finance L.P. +// Copyright 2019-2026 Bloomberg Finance L.P. // SPDX-License-Identifier: Apache-2.0 // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,6 +19,8 @@ #define PY_SSIZE_T_CLEAN #include +#include + #include #include #include @@ -41,6 +43,7 @@ class Session bmqt::CompressionAlgorithmType::Enum d_message_compression_type; PyObject* d_error; PyObject* d_broker_timeout_error; + PyObject* d_authn_credential_cb; bslma::ManagedPtr d_session_mp; // NOT IMPLEMENTED @@ -51,20 +54,8 @@ class Session Session(PyObject* py_session_event_callback, PyObject* py_message_event_callback, PyObject* py_ack_event_callback, - const char* broker_uri, - const char* script_name, - bmqt::CompressionAlgorithmType::Enum message_compression_type, - bsl::optional num_processing_threads, - bsl::optional blob_buffer_size, - bsl::optional channel_high_watermark, - bsl::optional > event_queue_watermarks, - const bsls::TimeInterval& stats_dump_interval, - const bsls::TimeInterval& connect_timeout, - const bsls::TimeInterval& disconnect_timeout, - const bsls::TimeInterval& open_queue_timeout, - const bsls::TimeInterval& configure_queue_timeout, - const bsls::TimeInterval& close_queue_timeout, - bool monitor_host_health, + PyObject* authn_credential_cb, + const SessionConfig& config, bsl::shared_ptr fake_host_health_monitor, PyObject* d_error, PyObject* d_broker_timeout_error, diff --git a/src/cpp/pybmq_sessionconfig.h b/src/cpp/pybmq_sessionconfig.h new file mode 100644 index 0000000..2f936c5 --- /dev/null +++ b/src/cpp/pybmq_sessionconfig.h @@ -0,0 +1,89 @@ +// Copyright 2019-2026 Bloomberg Finance L.P. +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_PYBMQ_SESSIONCONFIG +#define INCLUDED_PYBMQ_SESSIONCONFIG + +#include + +#include +#include +#include + +namespace BloombergLP { +namespace pybmq { + +// =================== +// struct SessionConfig +// =================== + +/// The options a `pybmq::Session` is constructed with, other than its Python +/// callbacks and host health monitor. +/// +/// A default-constructed `SessionConfig` means "no option set": empty +/// optionals, and default-constructed `bsls::TimeInterval`s, which +/// `Session` treats as unset. A new option can therefore be added as a +/// field without changing any existing call site. +/// +/// `broker_uri` and `script_name` are held, not owned, and must outlive the +/// `Session` constructor call. +struct SessionConfig +{ + // PUBLIC DATA + const char* broker_uri; + const char* script_name; + bmqt::CompressionAlgorithmType::Enum message_compression_type; + bsl::optional num_processing_threads; + bsl::optional blob_buffer_size; + bsl::optional channel_high_watermark; + bsl::optional > event_queue_watermarks; + bsls::TimeInterval stats_dump_interval; + bsls::TimeInterval connect_timeout; + bsls::TimeInterval disconnect_timeout; + bsls::TimeInterval open_queue_timeout; + bsls::TimeInterval configure_queue_timeout; + bsls::TimeInterval close_queue_timeout; + bool monitor_host_health; + + // CREATORS + SessionConfig(); +}; + +// =========================================================================== +// INLINE DEFINITIONS +// =========================================================================== + +inline SessionConfig::SessionConfig() +: broker_uri(0) +, script_name(0) +, message_compression_type(bmqt::CompressionAlgorithmType::e_NONE) +, num_processing_threads() +, blob_buffer_size() +, channel_high_watermark() +, event_queue_watermarks() +, stats_dump_interval() +, connect_timeout() +, disconnect_timeout() +, open_queue_timeout() +, configure_queue_timeout() +, close_queue_timeout() +, monitor_host_health(false) +{ +} + +} // namespace pybmq +} // namespace BloombergLP + +#endif diff --git a/src/cpp/pybmq_sessioneventhandler.cpp b/src/cpp/pybmq_sessioneventhandler.cpp index b5d270d..90d0668 100644 --- a/src/cpp/pybmq_sessioneventhandler.cpp +++ b/src/cpp/pybmq_sessioneventhandler.cpp @@ -39,18 +39,6 @@ SessionEventHandler::SessionEventHandler( , d_py_message_event_callback(py_message_event_callback) , d_py_ack_event_callback(py_ack_event_callback) { - GilAcquireGuard guard; - Py_INCREF(d_py_session_event_callback); - Py_INCREF(d_py_message_event_callback); - Py_INCREF(d_py_ack_event_callback); -} - -SessionEventHandler::~SessionEventHandler() -{ - GilAcquireGuard guard; - Py_DECREF(d_py_ack_event_callback); - Py_DECREF(d_py_message_event_callback); - Py_DECREF(d_py_session_event_callback); } void @@ -67,7 +55,7 @@ SessionEventHandler::onSessionEvent(const bmqa::SessionEvent& event) } bslma::ManagedPtr rv = RefUtils::toManagedPtr(PyObject_CallFunction( - d_py_session_event_callback, + d_py_session_event_callback.get(), "(N (i N i N s#))", PyBytes_FromStringAndSize( event.errorDescription().c_str(), @@ -92,16 +80,16 @@ SessionEventHandler::onMessageEvent(const bmqa::MessageEvent& event) PyObject* py_event; if (event.type() == bmqt::MessageEventType::e_PUSH) { - callback = d_py_message_event_callback; - py_event = MessageUtils::get_messages(event, d_py_session_event_callback); + callback = d_py_message_event_callback.get(); + py_event = MessageUtils::get_messages(event, d_py_session_event_callback.get()); } else if (event.type() == bmqt::MessageEventType::e_ACK) { - callback = d_py_ack_event_callback; + callback = d_py_ack_event_callback.get(); py_event = MessageUtils::get_acks(event); } else { bsl::ostringstream oss; oss << "Received an unexpected message event of type " << (int)event.type() << " (" << event.type() << ")"; - callback = d_py_session_event_callback; + callback = d_py_session_event_callback.get(); py_event = PyBytes_FromString(oss.str().c_str()); } bslma::ManagedPtr rv = diff --git a/src/cpp/pybmq_sessioneventhandler.h b/src/cpp/pybmq_sessioneventhandler.h index 3acb3b3..dd0c6f1 100644 --- a/src/cpp/pybmq_sessioneventhandler.h +++ b/src/cpp/pybmq_sessioneventhandler.h @@ -22,6 +22,8 @@ #include #include +#include + #include namespace BloombergLP { @@ -30,9 +32,9 @@ namespace pybmq { class SessionEventHandler : public bmqa::SessionEventHandler { private: - PyObject* d_py_session_event_callback; - PyObject* d_py_message_event_callback; - PyObject* d_py_ack_event_callback; + PyRef d_py_session_event_callback; + PyRef d_py_message_event_callback; + PyRef d_py_ack_event_callback; public: SessionEventHandler( @@ -40,8 +42,6 @@ class SessionEventHandler : public bmqa::SessionEventHandler PyObject* py_message_event_callback, PyObject* py_ack_event_callback); - ~SessionEventHandler(); - void onSessionEvent(const bmqa::SessionEvent& event) BSLS_KEYWORD_OVERRIDE; void onMessageEvent(const bmqa::MessageEvent& event) BSLS_KEYWORD_OVERRIDE; }; diff --git a/src/declarations/pybmq.pxd b/src/declarations/pybmq.pxd index 9ba293e..699c542 100644 --- a/src/declarations/pybmq.pxd +++ b/src/declarations/pybmq.pxd @@ -1,4 +1,4 @@ -# Copyright 2019-2023 Bloomberg Finance L.P. +# Copyright 2019-2026 Bloomberg Finance L.P. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -34,25 +34,32 @@ cdef extern from 'pybmq_ballutil.h' namespace 'BloombergLP::pybmq': @staticmethod object shutDownBallSingleton() except + +cdef extern from "pybmq_sessionconfig.h" namespace "BloombergLP::pybmq" nogil: + cdef cppclass SessionConfig: + SessionConfig() except+ + + const char* broker_uri + const char* script_name + CompressionAlgorithmType message_compression_type + optional[int] num_processing_threads + optional[int] blob_buffer_size + optional[int] channel_high_watermark + optional[pair[int, int]] event_queue_watermarks + TimeInterval stats_dump_interval + TimeInterval connect_timeout + TimeInterval disconnect_timeout + TimeInterval open_queue_timeout + TimeInterval configure_queue_timeout + TimeInterval close_queue_timeout + bint monitor_host_health + cdef extern from "pybmq_session.h" namespace "BloombergLP::pybmq" nogil: cdef cppclass Session: Session(object on_session_event, object on_message_event, object on_ack_event, - const char* broker_uri, - const char* script_name, - CompressionAlgorithmType message_compression_algorithm, - optional[int] num_processing_threads, - optional[int] blob_buffer_size, - optional[int] channel_high_watermark, - optional[pair[int, int]] event_queue_watermarks, - TimeInterval stats_dump_interval, - TimeInterval connect_timeout, - TimeInterval disconnect_timeout, - TimeInterval open_queue_timeout, - TimeInterval configure_queue_timeout, - TimeInterval close_queue_timeout, - bint monitor_host_health, + object authn_credential_cb, + const SessionConfig& config, shared_ptr[ManualHostHealthMonitor] fake_host_health_monitor_sp, object error, object broker_timeout_error, diff --git a/tests/unit/test_authn_credential_cb_adapter.py b/tests/unit/test_authn_credential_cb_adapter.py new file mode 100644 index 0000000..54b2d11 --- /dev/null +++ b/tests/unit/test_authn_credential_cb_adapter.py @@ -0,0 +1,156 @@ +# Copyright 2026 Bloomberg Finance L.P. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from blazingmq._ext import AuthnCredentialCbAdapter + + +def test_valid_return(): + # GIVEN + def provider(): + return ("mechanism", b"data") + + adapter = AuthnCredentialCbAdapter(provider) + + # WHEN + result = adapter.get_credential_data() + + # THEN + assert result == (b"mechanism", b"data") + + +def test_mechanism_is_encoded_as_utf8(): + # GIVEN + def provider(): + return ("mécanisme", b"data") + + adapter = AuthnCredentialCbAdapter(provider) + + # WHEN + result = adapter.get_credential_data() + + # THEN + assert result == ("mécanisme".encode("utf-8"), b"data") + + +def test_mechanism_not_encodable(): + # GIVEN a mechanism holding a lone surrogate, which has no UTF-8 encoding + def provider(): + return ("\ud800", b"data") + + adapter = AuthnCredentialCbAdapter(provider) + + # WHEN + result = adapter.get_credential_data() + + # THEN + assert result is None + + +def test_mechanism_with_embedded_nul_is_not_truncated(): + # GIVEN + def provider(): + return ("PLAIN\x00extra", b"data") + + adapter = AuthnCredentialCbAdapter(provider) + + # WHEN + result = adapter.get_credential_data() + + # THEN + assert result == (b"PLAIN\x00extra", b"data") + + +def test_none_return(): + # GIVEN + def provider(): + return None + + adapter = AuthnCredentialCbAdapter(provider) + + # WHEN + result = adapter.get_credential_data() + + # THEN + assert result is None + + +def test_not_a_tuple(): + # GIVEN + def provider(): + return "not a tuple" + + adapter = AuthnCredentialCbAdapter(provider) + + # WHEN + result = adapter.get_credential_data() + + # THEN + assert result is None + + +def test_tuple_wrong_length(): + # GIVEN + def provider(): + return ("mechanism", b"data", "extra") + + adapter = AuthnCredentialCbAdapter(provider) + + # WHEN + result = adapter.get_credential_data() + + # THEN + assert result is None + + +def test_mechanism_not_str(): + # GIVEN + def provider(): + return (123, b"data") + + adapter = AuthnCredentialCbAdapter(provider) + + # WHEN + result = adapter.get_credential_data() + + # THEN + assert result is None + + +def test_data_not_bytes(): + # GIVEN + def provider(): + return ("mechanism", "not bytes") + + adapter = AuthnCredentialCbAdapter(provider) + + # WHEN + result = adapter.get_credential_data() + + # THEN + assert result is None + + +def test_callback_raises(): + # GIVEN + def provider(): + raise RuntimeError("broken") + + adapter = AuthnCredentialCbAdapter(provider) + + # WHEN + result = adapter.get_credential_data() + + # THEN + assert result is None diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index 78fba1d..3858369 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -1,4 +1,4 @@ -# Copyright 2019-2023 Bloomberg Finance L.P. +# Copyright 2019-2026 Bloomberg Finance L.P. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -78,9 +78,60 @@ def dummy2(): ), monitor_host_health=False, fake_host_health_monitor=None, + authn_credential_cb=None, ) +@mock.patch("blazingmq._session.ExtSession") +def test_session_constructed_with_authn_credential_provider(ext_cls): + # GIVEN + ext_cls.mock_add_spec([]) + + def dummy1(): + pass + + def dummy2(): + pass + + def my_provider(): + return ("mechanism", b"data") + + # WHEN + Session( + dummy1, + on_message=dummy2, + broker="some_uri", + timeout=60.0, + host_health_monitor=None, + authn_credential_provider=my_provider, + ) + + # THEN + ext_cls.assert_called_once_with( + dummy1, + on_message=dummy2, + broker=b"some_uri", + message_compression_algorithm=CompressionAlgorithmType.NONE, + num_processing_threads=None, + blob_buffer_size=None, + channel_high_watermark=None, + event_queue_watermarks=None, + stats_dump_interval=None, + timeouts=Timeouts( + connect_timeout=None, + disconnect_timeout=None, + open_queue_timeout=60.0, + configure_queue_timeout=60.0, + close_queue_timeout=60.0, + ), + monitor_host_health=False, + fake_host_health_monitor=None, + authn_credential_cb=mock.ANY, + ) + call_kwargs = ext_cls.call_args[1] + assert call_kwargs["authn_credential_cb"] is not None + + @mock.patch("blazingmq._session.ExtSession") def test_session_constructed_with_timeouts(ext_cls): # GIVEN @@ -128,6 +179,7 @@ def dummy2(): timeouts=timeouts, monitor_host_health=False, fake_host_health_monitor=None, + authn_credential_cb=None, ) @@ -172,6 +224,7 @@ def dummy2(): timeouts=timeouts, monitor_host_health=False, fake_host_health_monitor=None, + authn_credential_cb=None, ) @@ -207,6 +260,7 @@ def dummy2(): timeouts=Timeouts(), monitor_host_health=False, fake_host_health_monitor=None, + authn_credential_cb=None, ) @@ -259,7 +313,101 @@ def dummy2(): timeouts=timeouts, monitor_host_health=False, fake_host_health_monitor=None, + authn_credential_cb=None, + ) + + +@mock.patch("blazingmq._session.ExtSession") +def test_session_default_with_options_authn_credential_provider(ext_cls): + # GIVEN + ext_cls.mock_add_spec([]) + + def dummy1(): + pass + + def dummy2(): + pass + + def my_provider(): + return ("mechanism", b"data") + + session_options = SessionOptions(authn_credential_provider=my_provider) + + # WHEN + Session.with_options( + dummy1, on_message=dummy2, broker="some_uri", session_options=session_options + ) + + # THEN + ext_cls.assert_called_once_with( + dummy1, + on_message=dummy2, + broker=b"some_uri", + message_compression_algorithm=CompressionAlgorithmType.NONE, + num_processing_threads=None, + blob_buffer_size=None, + channel_high_watermark=None, + event_queue_watermarks=None, + stats_dump_interval=None, + timeouts=Timeouts(), + monitor_host_health=False, + fake_host_health_monitor=None, + authn_credential_cb=mock.ANY, + ) + call_kwargs = ext_cls.call_args[1] + assert call_kwargs["authn_credential_cb"] is not None + + +@mock.patch("blazingmq._session.ExtSession") +def test_session_with_options_authn_credential_provider(ext_cls): + # GIVEN + ext_cls.mock_add_spec([]) + + def dummy1(): + pass + + def dummy2(): + pass + + def my_provider(): + return ("mechanism", b"data") + + timeouts = Timeouts( + connect_timeout=60.0, + disconnect_timeout=70.0, + open_queue_timeout=80.0, + configure_queue_timeout=90.0, + close_queue_timeout=100.0, + ) + + session_options = SessionOptions( + timeouts=timeouts, + authn_credential_provider=my_provider, + ) + + # WHEN + Session.with_options( + dummy1, on_message=dummy2, broker="some_uri", session_options=session_options + ) + + # THEN + ext_cls.assert_called_once_with( + dummy1, + on_message=dummy2, + broker=b"some_uri", + message_compression_algorithm=CompressionAlgorithmType.NONE, + num_processing_threads=None, + blob_buffer_size=None, + channel_high_watermark=None, + event_queue_watermarks=None, + stats_dump_interval=None, + timeouts=timeouts, + monitor_host_health=False, + fake_host_health_monitor=None, + authn_credential_cb=mock.ANY, ) + call_kwargs = ext_cls.call_args[1] + assert call_kwargs["authn_credential_cb"] is not None @mock.patch("blazingmq._session.ExtSession") @@ -304,6 +452,7 @@ def dummy2(): ), monitor_host_health=True, fake_host_health_monitor=monitor._monitor, + authn_credential_cb=None, ) @@ -335,6 +484,7 @@ def dummy2(): timeouts=Timeouts(), monitor_host_health=False, fake_host_health_monitor=None, + authn_credential_cb=None, ) diff --git a/tests/unit/test_session_options.py b/tests/unit/test_session_options.py index 6adc505..ec575b7 100644 --- a/tests/unit/test_session_options.py +++ b/tests/unit/test_session_options.py @@ -1,4 +1,4 @@ -# Copyright 2019-2023 Bloomberg Finance L.P. +# Copyright 2019-2026 Bloomberg Finance L.P. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -58,6 +58,7 @@ def test_session_options_default_to_none(): assert options.message_compression_algorithm is None assert options.timeouts is None assert options.host_health_monitor is None + assert options.authn_credential_provider is None assert options.num_processing_threads is None assert options.blob_buffer_size is None assert options.channel_high_watermark is None @@ -92,6 +93,7 @@ def test_session_options_equality(): blazingmq.SessionOptions(channel_high_watermark=8000000), blazingmq.SessionOptions(event_queue_watermarks=(6000000, 7000000)), blazingmq.SessionOptions(stats_dump_interval=30.0), + blazingmq.SessionOptions(authn_credential_provider=lambda: None), ], ) def test_queue_options_other_inequality(right): @@ -100,3 +102,15 @@ def test_queue_options_other_inequality(right): # THEN assert not left == right + + +def test_session_options_repr_with_authn_credential_provider(): + # GIVEN + def my_provider(): + return ("mechanism", b"data") + + # WHEN + options = blazingmq.SessionOptions(authn_credential_provider=my_provider) + + # THEN + assert "authn_credential_provider=" in repr(options)