diff --git a/.changelog/5372.added b/.changelog/5372.added new file mode 100644 index 00000000000..d6031ada38f --- /dev/null +++ b/.changelog/5372.added @@ -0,0 +1 @@ +`opentelemetry-sdk`: Add support for activating instrumentors from a declarative configuration file via the `instrumentation/development.python` section. Instrumentors can declare a `configuration` attribute to have their options validated through the same type-coercion pipeline used for SDK component configuration. \ No newline at end of file diff --git a/docs/examples/declarative-config/README.rst b/docs/examples/declarative-config/README.rst index ec3b828b829..ef01f1743c8 100644 --- a/docs/examples/declarative-config/README.rst +++ b/docs/examples/declarative-config/README.rst @@ -16,14 +16,15 @@ The source files of this example are available :scm_web:`here `. Install the SDK with the ``file-configuration`` extra (it pulls in ``pyyaml`` -and ``jsonschema``), the auto-instrumentation entry point, and the OTLP/HTTP -exporter: +and ``jsonschema``), the auto-instrumentation entry point, the OTLP/HTTP +exporter, and the ``requests`` instrumentation used by this example: .. code-block:: sh pip install "opentelemetry-sdk[file-configuration]" \ opentelemetry-distro \ - opentelemetry-exporter-otlp-proto-http + opentelemetry-exporter-otlp-proto-http \ + opentelemetry-instrumentation-requests Start an OTLP-capable backend locally so there is somewhere to send data. Write the following file: @@ -74,7 +75,10 @@ auto-instrumentation apply it. No configuration code lives in ``app.py``: export OTEL_CONFIG_FILE=$(pwd)/otel-config.yaml opentelemetry-instrument python app.py -You should see the exported span in the Collector's debug output. +You should see the exported span in the Collector's debug output. The +``instrumentation/development.python`` section in ``otel-config.yaml`` +activates the ``requests`` instrumentation so outgoing HTTP calls made by +``app.py`` are automatically traced without any code changes. Environment variable substitution ---------------------------------- diff --git a/docs/examples/declarative-config/otel-config.yaml b/docs/examples/declarative-config/otel-config.yaml index 1f2bd9fa798..3a3e88e0276 100644 --- a/docs/examples/declarative-config/otel-config.yaml +++ b/docs/examples/declarative-config/otel-config.yaml @@ -32,3 +32,8 @@ logger_provider: exporter: otlp_http: endpoint: http://localhost:4318/v1/logs + +instrumentation/development: + python: + requests: + enabled: true diff --git a/docs/sdk/configuration.rst b/docs/sdk/configuration.rst index 40af342e418..43461fe5a90 100644 --- a/docs/sdk/configuration.rst +++ b/docs/sdk/configuration.rst @@ -95,6 +95,31 @@ OTLP/HTTP. The source is available :scm_web:`here - name: api-key value: ${OTLP_API_KEY} +Instrumentation +--------------- + +The ``instrumentation/development.python`` section activates Python +instrumentors by their ``opentelemetry_instrumentor`` entry-point name. Set +``enabled: false`` to suppress an instrumentor without removing its entry, and +pass any other keys as keyword arguments to ``instrument()``: + +.. code-block:: yaml + + instrumentation/development: + python: + requests: + enabled: true + urllib3: + enabled: true + max_spans_per_request: 10 + +If the instrumentor class declares a ``configuration`` class attribute pointing +to a dataclass, the options are validated and type-coerced through the same +pipeline used for SDK component configuration before being forwarded to +``instrument()``. Instrumentors that are already active (for example because +``opentelemetry-instrument`` ran before the file was applied) are silently +skipped. + Environment variable substitution ---------------------------------- diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-common/README.rst b/exporter/opentelemetry-exporter-otlp-pyproto-common/README.rst new file mode 100644 index 00000000000..a27732f1cbf --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-common/README.rst @@ -0,0 +1,4 @@ +OpenTelemetry OTLP Exporter (pure-Python protobuf, common) +=========================================================== + +Common encoding utilities for the pure-Python OTLP exporter stack. diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-common/pyproject.toml b/exporter/opentelemetry-exporter-otlp-pyproto-common/pyproject.toml new file mode 100644 index 00000000000..cc90ba3348c --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-common/pyproject.toml @@ -0,0 +1,39 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "opentelemetry-exporter-otlp-pyproto-common" +dynamic = ["version"] +description = "OpenTelemetry OTLP encoding using pure-Python protobuf (no google.protobuf)" +readme = "README.rst" +license = "Apache-2.0" +requires-python = ">=3.10" +dependencies = [ + "opentelemetry-api ~= 1.12", + "opentelemetry-sdk ~= 1.12", + "opentelemetry-pyproto == 1.43.0.dev", +] + +[dependency-groups] +dev = [ + "opentelemetry-proto == 1.43.0.dev", + "opentelemetry-exporter-otlp-proto-common == 1.43.0.dev", + "opentelemetry-test-utils == 1.43.0.dev", + "protobuf>=5.0", + "pytest>=7.0", +] + +[tool.uv.sources] +opentelemetry-proto = { workspace = true } +opentelemetry-exporter-otlp-proto-common = { workspace = true } +opentelemetry-test-utils = { workspace = true } + +[tool.hatch.version] +path = "src/opentelemetry/exporter/otlp/pyproto/common/version/__init__.py" + +[tool.hatch.build.targets.sdist] +include = ["/src"] + +[tool.hatch.build.targets.wheel] +packages = ["src/opentelemetry"] diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-common/src/opentelemetry/exporter/otlp/pyproto/common/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-common/src/opentelemetry/exporter/otlp/pyproto/common/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-common/src/opentelemetry/exporter/otlp/pyproto/common/_exporter_metrics.py b/exporter/opentelemetry-exporter-otlp-pyproto-common/src/opentelemetry/exporter/otlp/pyproto/common/_exporter_metrics.py new file mode 100644 index 00000000000..6b3d1d76fa3 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-common/src/opentelemetry/exporter/otlp/pyproto/common/_exporter_metrics.py @@ -0,0 +1,153 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from collections import Counter +from collections.abc import Iterator +from contextlib import AbstractContextManager, contextmanager +from dataclasses import dataclass +from time import perf_counter +from typing import TYPE_CHECKING, Protocol + +from opentelemetry.metrics import MeterProvider, get_meter_provider +from opentelemetry.semconv._incubating.attributes.otel_attributes import ( + OTEL_COMPONENT_NAME, + OTEL_COMPONENT_TYPE, + OtelComponentTypeValues, +) +from opentelemetry.semconv._incubating.metrics.otel_metrics import ( + create_otel_sdk_exporter_log_exported, + create_otel_sdk_exporter_log_inflight, + create_otel_sdk_exporter_metric_data_point_exported, + create_otel_sdk_exporter_metric_data_point_inflight, + create_otel_sdk_exporter_operation_duration, + create_otel_sdk_exporter_span_exported, + create_otel_sdk_exporter_span_inflight, +) +from opentelemetry.semconv.attributes.error_attributes import ERROR_TYPE +from opentelemetry.semconv.attributes.server_attributes import ( + SERVER_ADDRESS, + SERVER_PORT, +) + +if TYPE_CHECKING: + from typing import Literal + from urllib.parse import ParseResult as UrlParseResult + + from opentelemetry.util.types import Attributes, AttributeValue + +_component_counter = Counter() + + +@dataclass +class ExportResult: + error: Exception | None = None + error_attrs: Attributes = None + + +class ExporterMetricsT(Protocol): + def export_operation( + self, num_items: int + ) -> AbstractContextManager[ExportResult]: ... + + +class NoOpExporterMetrics: + @contextmanager + def export_operation(self, num_items: int) -> Iterator[ExportResult]: + yield ExportResult() + + +class ExporterMetrics: + def __init__( + self, + component_type: OtelComponentTypeValues | None, + signal: Literal["traces", "metrics", "logs"], + endpoint: UrlParseResult, + meter_provider: MeterProvider | None, + ) -> None: + if signal == "traces": + create_exported = create_otel_sdk_exporter_span_exported + create_inflight = create_otel_sdk_exporter_span_inflight + elif signal == "logs": + create_exported = create_otel_sdk_exporter_log_exported + create_inflight = create_otel_sdk_exporter_log_inflight + else: + create_exported = ( + create_otel_sdk_exporter_metric_data_point_exported + ) + create_inflight = ( + create_otel_sdk_exporter_metric_data_point_inflight + ) + + port = endpoint.port + if port is None: + if endpoint.scheme == "https": + port = 443 + elif endpoint.scheme == "http": + port = 80 + + component_type_value = ( + component_type.value if component_type else "unknown_otlp_exporter" + ) + count = _component_counter[component_type_value] + _component_counter[component_type_value] = count + 1 + self._standard_attrs: dict[str, AttributeValue] = { + OTEL_COMPONENT_TYPE: component_type_value, + OTEL_COMPONENT_NAME: f"{component_type_value}/{count}", + } + if endpoint.hostname: + self._standard_attrs[SERVER_ADDRESS] = endpoint.hostname + if port is not None: + self._standard_attrs[SERVER_PORT] = port + + meter_provider = meter_provider or get_meter_provider() + meter = meter_provider.get_meter("opentelemetry-sdk") + self._inflight = create_inflight(meter) + self._exported = create_exported(meter) + self._duration = create_otel_sdk_exporter_operation_duration(meter) + + @contextmanager + def export_operation(self, num_items: int) -> Iterator[ExportResult]: + start_time = perf_counter() + self._inflight.add(num_items, self._standard_attrs) + + result = ExportResult() + try: + yield result + finally: + error = result.error + error_attrs = result.error_attrs + + end_time = perf_counter() + self._inflight.add(-num_items, self._standard_attrs) + exported_attrs = ( + {**self._standard_attrs, ERROR_TYPE: type(error).__qualname__} + if error + else self._standard_attrs + ) + self._exported.add(num_items, exported_attrs) + duration_attrs = ( + {**exported_attrs, **error_attrs} + if error_attrs + else exported_attrs + ) + self._duration.record(end_time - start_time, duration_attrs) + + +def create_exporter_metrics( + component_type: OtelComponentTypeValues | None, + signal: Literal["traces", "metrics", "logs"], + endpoint: UrlParseResult, + meter_provider: MeterProvider | None, + enabled: bool, +) -> ExporterMetricsT: + if not enabled: + return NoOpExporterMetrics() + + return ExporterMetrics( + component_type, + signal, + endpoint, + meter_provider, + ) diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-common/src/opentelemetry/exporter/otlp/pyproto/common/_internal/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-common/src/opentelemetry/exporter/otlp/pyproto/common/_internal/__init__.py new file mode 100644 index 00000000000..60e1e68ce2e --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-common/src/opentelemetry/exporter/otlp/pyproto/common/_internal/__init__.py @@ -0,0 +1,114 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from logging import getLogger +from collections.abc import Callable, Mapping, Sequence +from typing import Any, TypeVar + +from opentelemetry.pyproto.common.v1.common_pypb2 import ( + AnyValue, + ArrayValue, + InstrumentationScope as PB2InstrumentationScope, + KeyValue, + KeyValueList, +) +from opentelemetry.pyproto.resource.v1.resource_pypb2 import Resource as PB2Resource +from opentelemetry.sdk.trace import Resource +from opentelemetry.sdk.util.instrumentation import InstrumentationScope +from opentelemetry.util.types import _ExtendedAttributes + +_logger = getLogger(__name__) + +_TypingResourceT = TypeVar("_TypingResourceT") +_ResourceDataT = TypeVar("_ResourceDataT") + + +def _encode_instrumentation_scope( + instrumentation_scope: InstrumentationScope, +) -> PB2InstrumentationScope: + if instrumentation_scope is None: + return PB2InstrumentationScope() + return PB2InstrumentationScope( + name=instrumentation_scope.name, + version=instrumentation_scope.version, + attributes=_encode_attributes(instrumentation_scope.attributes), + ) + + +def _encode_resource(resource: Resource) -> PB2Resource: + return PB2Resource(attributes=_encode_attributes(resource.attributes)) + + +def _encode_value(value: Any) -> AnyValue: + if value is None: + return AnyValue() + if isinstance(value, bool): + return AnyValue(bool_value=value) + if isinstance(value, str): + return AnyValue(string_value=value) + if isinstance(value, int): + return AnyValue(int_value=value) + if isinstance(value, float): + return AnyValue(double_value=value) + if isinstance(value, bytes): + return AnyValue(bytes_value=value) + if isinstance(value, Sequence): + return AnyValue( + array_value=ArrayValue(values=[_encode_value(v) for v in value]) + ) + if isinstance(value, Mapping): + return AnyValue( + kvlist_value=KeyValueList( + values=[_encode_key_value(str(k), v) for k, v in value.items()] + ) + ) + raise Exception(f"Invalid type {type(value)} of value {value}") + + +def _encode_key_value(key: str, value: Any) -> KeyValue: + return KeyValue(key=key, value=_encode_value(value)) + + +def _encode_span_id(span_id: int) -> bytes: + return span_id.to_bytes(length=8, byteorder="big", signed=False) + + +def _encode_trace_id(trace_id: int) -> bytes: + return trace_id.to_bytes(length=16, byteorder="big", signed=False) + + +def _encode_attributes( + attributes: _ExtendedAttributes | None, +) -> list[KeyValue]: + if not attributes: + return [] + pb2_attributes = [] + for key, value in attributes.items(): + try: + pb2_attributes.append(_encode_key_value(key, value)) + except Exception as error: + _logger.exception("Failed to encode key %s: %s", key, error) + return pb2_attributes + + +def _get_resource_data( + sdk_resource_scope_data: dict[Resource, _ResourceDataT], + resource_class: Callable[..., _TypingResourceT], + name: str, +) -> list[_TypingResourceT]: + resource_data = [] + for sdk_resource, scope_data in sdk_resource_scope_data.items(): + collector_resource = PB2Resource( + attributes=_encode_attributes(sdk_resource.attributes) + ) + resource_data.append( + resource_class( + **{ + "resource": collector_resource, + f"scope_{name}": scope_data.values(), + } + ) + ) + return resource_data diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-common/src/opentelemetry/exporter/otlp/pyproto/common/_internal/_log_encoder/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-common/src/opentelemetry/exporter/otlp/pyproto/common/_internal/_log_encoder/__init__.py new file mode 100644 index 00000000000..6aa154c1561 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-common/src/opentelemetry/exporter/otlp/pyproto/common/_internal/_log_encoder/__init__.py @@ -0,0 +1,87 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 +from collections import defaultdict +from collections.abc import Sequence + +from opentelemetry.exporter.otlp.pyproto.common._internal import ( + _encode_attributes, + _encode_instrumentation_scope, + _encode_resource, + _encode_span_id, + _encode_trace_id, + _encode_value, +) +from opentelemetry.pyproto.collector.logs.v1.logs_service_pypb2 import ( + ExportLogsServiceRequest, +) +from opentelemetry.pyproto.logs.v1.logs_pypb2 import ( + LogRecord, + ResourceLogs, + ScopeLogs, +) +from opentelemetry.sdk._logs import ReadableLogRecord + + +def encode_logs(batch: Sequence[ReadableLogRecord]) -> ExportLogsServiceRequest: + return ExportLogsServiceRequest(resource_logs=_encode_resource_logs(batch)) + + +def _encode_log(readable_log_record: ReadableLogRecord) -> LogRecord: + log = readable_log_record.log_record + span_id = ( + b"" + if log.span_id == 0 + else _encode_span_id(log.span_id) + ) + trace_id = ( + b"" + if log.trace_id == 0 + else _encode_trace_id(log.trace_id) + ) + return LogRecord( + time_unix_nano=log.timestamp, + observed_time_unix_nano=log.observed_timestamp, + span_id=span_id, + trace_id=trace_id, + flags=int(log.trace_flags), + body=_encode_value(log.body), + severity_text=log.severity_text or "", + attributes=_encode_attributes(log.attributes), + dropped_attributes_count=readable_log_record.dropped_attributes, + severity_number=getattr(log.severity_number, "value", 0) or 0, + event_name=log.event_name or "", + ) + + +def _encode_resource_logs( + batch: Sequence[ReadableLogRecord], +) -> list[ResourceLogs]: + sdk_resource_logs: dict = defaultdict(lambda: defaultdict(list)) + + for readable_log in batch: + sdk_resource = readable_log.resource + sdk_instrumentation = readable_log.instrumentation_scope or None + pb2_log = _encode_log(readable_log) + sdk_resource_logs[sdk_resource][sdk_instrumentation].append(pb2_log) + + pb2_resource_logs = [] + for sdk_resource, sdk_instrumentations in sdk_resource_logs.items(): + scope_logs = [] + for sdk_instrumentation, pb2_logs in sdk_instrumentations.items(): + scope_logs.append( + ScopeLogs( + scope=_encode_instrumentation_scope(sdk_instrumentation), + log_records=pb2_logs, + schema_url=sdk_instrumentation.schema_url + if sdk_instrumentation + else "", + ) + ) + pb2_resource_logs.append( + ResourceLogs( + resource=_encode_resource(sdk_resource), + scope_logs=scope_logs, + schema_url=sdk_resource.schema_url, + ) + ) + return pb2_resource_logs diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-common/src/opentelemetry/exporter/otlp/pyproto/common/_internal/metrics_encoder/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-common/src/opentelemetry/exporter/otlp/pyproto/common/_internal/metrics_encoder/__init__.py new file mode 100644 index 00000000000..81254deae9d --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-common/src/opentelemetry/exporter/otlp/pyproto/common/_internal/metrics_encoder/__init__.py @@ -0,0 +1,334 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +from logging import getLogger +from os import environ + +from opentelemetry.exporter.otlp.pyproto.common._internal import ( + _encode_attributes, + _encode_instrumentation_scope, + _encode_span_id, + _encode_trace_id, +) +from opentelemetry.pyproto.collector.metrics.v1.metrics_service_pypb2 import ( + ExportMetricsServiceRequest, +) +from opentelemetry.pyproto.metrics.v1.metrics_pypb2 import ( + Exemplar, + ExponentialHistogram, + ExponentialHistogramDataPoint, + Gauge, + Histogram, + HistogramDataPoint, + Metric, + NumberDataPoint, + ResourceMetrics, + ScopeMetrics, + Sum, +) +from opentelemetry.pyproto.resource.v1.resource_pypb2 import Resource as PB2Resource +from opentelemetry.sdk.environment_variables import ( + OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION, + OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE, +) +from opentelemetry.sdk.metrics import ( + Counter, + Exemplar as SDKExemplar, + Histogram as SDKHistogram, + ObservableCounter, + ObservableGauge, + ObservableUpDownCounter, + UpDownCounter, +) +from opentelemetry.sdk.metrics.export import ( + AggregationTemporality, + Gauge as SDKGauge, + MetricExporter, + MetricsData, + Sum as SDKSum, +) +from opentelemetry.sdk.metrics.export import ( + ExponentialHistogram as ExponentialHistogramType, +) +from opentelemetry.sdk.metrics.export import ( + Histogram as HistogramType, +) +from opentelemetry.sdk.metrics.view import ( + Aggregation, + ExplicitBucketHistogramAggregation, + ExponentialBucketHistogramAggregation, +) + +_logger = getLogger(__name__) + + +class OTLPMetricExporterMixin: + def _common_configuration( + self, + preferred_temporality: dict[type, AggregationTemporality] | None = None, + preferred_aggregation: dict[type, Aggregation] | None = None, + ) -> None: + MetricExporter.__init__( + self, + preferred_temporality=self._get_temporality(preferred_temporality), + preferred_aggregation=self._get_aggregation(preferred_aggregation), + ) + + def _get_temporality( + self, preferred_temporality: dict[type, AggregationTemporality] + ) -> dict[type, AggregationTemporality]: + otel_exporter_otlp_metrics_temporality_preference = ( + environ.get( + OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE, + "CUMULATIVE", + ) + .upper() + .strip() + ) + + if otel_exporter_otlp_metrics_temporality_preference == "DELTA": + instrument_class_temporality = { + Counter: AggregationTemporality.DELTA, + UpDownCounter: AggregationTemporality.CUMULATIVE, + SDKHistogram: AggregationTemporality.DELTA, + ObservableCounter: AggregationTemporality.DELTA, + ObservableUpDownCounter: AggregationTemporality.CUMULATIVE, + ObservableGauge: AggregationTemporality.CUMULATIVE, + } + elif otel_exporter_otlp_metrics_temporality_preference == "LOWMEMORY": + instrument_class_temporality = { + Counter: AggregationTemporality.DELTA, + UpDownCounter: AggregationTemporality.CUMULATIVE, + SDKHistogram: AggregationTemporality.DELTA, + ObservableCounter: AggregationTemporality.CUMULATIVE, + ObservableUpDownCounter: AggregationTemporality.CUMULATIVE, + ObservableGauge: AggregationTemporality.CUMULATIVE, + } + else: + if otel_exporter_otlp_metrics_temporality_preference != "CUMULATIVE": + _logger.warning( + "Unrecognized OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE" + " value found: %s, using CUMULATIVE", + otel_exporter_otlp_metrics_temporality_preference, + ) + instrument_class_temporality = { + Counter: AggregationTemporality.CUMULATIVE, + UpDownCounter: AggregationTemporality.CUMULATIVE, + SDKHistogram: AggregationTemporality.CUMULATIVE, + ObservableCounter: AggregationTemporality.CUMULATIVE, + ObservableUpDownCounter: AggregationTemporality.CUMULATIVE, + ObservableGauge: AggregationTemporality.CUMULATIVE, + } + + instrument_class_temporality.update(preferred_temporality or {}) + return instrument_class_temporality + + def _get_aggregation( + self, preferred_aggregation: dict[type, Aggregation] + ) -> dict[type, Aggregation]: + otel_exporter_otlp_metrics_default_histogram_aggregation = environ.get( + OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION, + "explicit_bucket_histogram", + ) + + if otel_exporter_otlp_metrics_default_histogram_aggregation == ( + "base2_exponential_bucket_histogram" + ): + instrument_class_aggregation: dict[type, Aggregation] = { + SDKHistogram: ExponentialBucketHistogramAggregation(), + } + else: + if otel_exporter_otlp_metrics_default_histogram_aggregation != ( + "explicit_bucket_histogram" + ): + _logger.warning( + "Invalid value for %s: %s, using explicit bucket histogram aggregation", + OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION, + otel_exporter_otlp_metrics_default_histogram_aggregation, + ) + instrument_class_aggregation = { + SDKHistogram: ExplicitBucketHistogramAggregation(), + } + + instrument_class_aggregation.update(preferred_aggregation or {}) + return instrument_class_aggregation + + +class EncodingException(Exception): + def __init__(self, original_exception, metric): + super().__init__() + self.original_exception = original_exception + self.metric = metric + + def __str__(self): + return f"{self.metric}\n{self.original_exception}" + + +def encode_metrics(data: MetricsData) -> ExportMetricsServiceRequest: + resource_metrics_list = [] + + for resource_metrics in data.resource_metrics: + scope_metrics_list = [] + sdk_resource = resource_metrics.resource + + for scope_metrics in resource_metrics.scope_metrics: + instrumentation_scope = scope_metrics.scope + pb2_scope_metrics = ScopeMetrics( + scope=_encode_instrumentation_scope(instrumentation_scope), + schema_url=instrumentation_scope.schema_url, + ) + + for metric in scope_metrics.metrics: + try: + pb2_metric = _encode_metric(metric) + except Exception as ex: + raise EncodingException(ex, metric) from None + if pb2_metric is not None: + pb2_scope_metrics.metrics.append(pb2_metric) + + scope_metrics_list.append(pb2_scope_metrics) + + resource_metrics_list.append( + ResourceMetrics( + resource=PB2Resource( + attributes=_encode_attributes(sdk_resource.attributes) + ), + scope_metrics=scope_metrics_list, + schema_url=sdk_resource.schema_url, + ) + ) + + return ExportMetricsServiceRequest(resource_metrics=resource_metrics_list) + + +def _encode_metric(metric) -> Metric | None: + kwargs: dict = dict( + name=metric.name, + description=metric.description, + unit=metric.unit, + ) + + if isinstance(metric.data, SDKGauge): + data_points = [] + for dp in metric.data.data_points: + pt_kwargs: dict = dict( + attributes=_encode_attributes(dp.attributes), + time_unix_nano=dp.time_unix_nano, + exemplars=_encode_exemplars(dp.exemplars), + ) + if isinstance(dp.value, int): + pt_kwargs["as_int"] = dp.value + else: + pt_kwargs["as_double"] = dp.value + data_points.append(NumberDataPoint(**pt_kwargs)) + kwargs["gauge"] = Gauge(data_points=data_points) + + elif isinstance(metric.data, HistogramType): + data_points = [] + for dp in metric.data.data_points: + data_points.append( + HistogramDataPoint( + attributes=_encode_attributes(dp.attributes), + time_unix_nano=dp.time_unix_nano, + start_time_unix_nano=dp.start_time_unix_nano, + exemplars=_encode_exemplars(dp.exemplars), + count=dp.count, + sum=dp.sum, + bucket_counts=list(dp.bucket_counts) if dp.bucket_counts else [], + explicit_bounds=list(dp.explicit_bounds) if dp.explicit_bounds else [], + max=dp.max, + min=dp.min, + ) + ) + kwargs["histogram"] = Histogram( + data_points=data_points, + aggregation_temporality=metric.data.aggregation_temporality, + ) + + elif isinstance(metric.data, SDKSum): + data_points = [] + for dp in metric.data.data_points: + pt_kwargs = dict( + attributes=_encode_attributes(dp.attributes), + start_time_unix_nano=dp.start_time_unix_nano, + time_unix_nano=dp.time_unix_nano, + exemplars=_encode_exemplars(dp.exemplars), + ) + if isinstance(dp.value, int): + pt_kwargs["as_int"] = dp.value + else: + pt_kwargs["as_double"] = dp.value + data_points.append(NumberDataPoint(**pt_kwargs)) + kwargs["sum"] = Sum( + data_points=data_points, + aggregation_temporality=metric.data.aggregation_temporality, + is_monotonic=metric.data.is_monotonic, + ) + + elif isinstance(metric.data, ExponentialHistogramType): + data_points = [] + for dp in metric.data.data_points: + positive = negative = None + if dp.positive.bucket_counts: + positive = ExponentialHistogramDataPoint.Buckets( + offset=dp.positive.offset, + bucket_counts=list(dp.positive.bucket_counts), + ) + if dp.negative.bucket_counts: + negative = ExponentialHistogramDataPoint.Buckets( + offset=dp.negative.offset, + bucket_counts=list(dp.negative.bucket_counts), + ) + data_points.append( + ExponentialHistogramDataPoint( + attributes=_encode_attributes(dp.attributes), + time_unix_nano=dp.time_unix_nano, + start_time_unix_nano=dp.start_time_unix_nano, + exemplars=_encode_exemplars(dp.exemplars), + count=dp.count, + sum=dp.sum, + scale=dp.scale, + zero_count=dp.zero_count, + positive=positive, + negative=negative, + flags=dp.flags, + max=dp.max, + min=dp.min, + ) + ) + kwargs["exponential_histogram"] = ExponentialHistogram( + data_points=data_points, + aggregation_temporality=metric.data.aggregation_temporality, + ) + + else: + _logger.warning("unsupported data type %s", metric.data.__class__.__name__) + return None + + return Metric(**kwargs) + + +def _encode_exemplars(sdk_exemplars: list[SDKExemplar]) -> list[Exemplar]: + result = [] + for sdk_exemplar in sdk_exemplars: + ex_kwargs: dict = dict( + time_unix_nano=sdk_exemplar.time_unix_nano, + filtered_attributes=_encode_attributes(sdk_exemplar.filtered_attributes), + ) + if ( + sdk_exemplar.span_id is not None + and sdk_exemplar.trace_id is not None + ): + ex_kwargs["span_id"] = _encode_span_id(sdk_exemplar.span_id) + ex_kwargs["trace_id"] = _encode_trace_id(sdk_exemplar.trace_id) + + if isinstance(sdk_exemplar.value, float): + ex_kwargs["as_double"] = sdk_exemplar.value + elif isinstance(sdk_exemplar.value, int): + ex_kwargs["as_int"] = sdk_exemplar.value + else: + raise ValueError("Exemplar value must be an int or float") + + result.append(Exemplar(**ex_kwargs)) + return result diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-common/src/opentelemetry/exporter/otlp/pyproto/common/_internal/trace_encoder/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-common/src/opentelemetry/exporter/otlp/pyproto/common/_internal/trace_encoder/__init__.py new file mode 100644 index 00000000000..b31c07f2324 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-common/src/opentelemetry/exporter/otlp/pyproto/common/_internal/trace_encoder/__init__.py @@ -0,0 +1,160 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from logging import getLogger +from collections import defaultdict +from collections.abc import Sequence + +from opentelemetry.exporter.otlp.pyproto.common._internal import ( + _encode_attributes, + _encode_instrumentation_scope, + _encode_resource, + _encode_span_id, + _encode_trace_id, +) +from opentelemetry.pyproto.collector.trace.v1.trace_service_pypb2 import ( + ExportTraceServiceRequest, +) +from opentelemetry.pyproto.trace.v1.trace_pypb2 import ( + ResourceSpans, + ScopeSpans, + Span, + Status, +) +from opentelemetry.sdk.trace import Event, ReadableSpan +from opentelemetry.trace import Link, SpanKind +from opentelemetry.trace.span import SpanContext, TraceState + +# Map SDK SpanKind (0-4) to proto SpanKind (1-5) +_SPAN_KIND_MAP = { + SpanKind.INTERNAL: 1, + SpanKind.SERVER: 2, + SpanKind.CLIENT: 3, + SpanKind.PRODUCER: 4, + SpanKind.CONSUMER: 5, +} + +# proto SpanFlags values +_SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK = 0x00000100 +_SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK = 0x00000200 + +_logger = getLogger(__name__) + + +def encode_spans(sdk_spans: Sequence[ReadableSpan]) -> ExportTraceServiceRequest: + return ExportTraceServiceRequest( + resource_spans=_encode_resource_spans(sdk_spans) + ) + + +def _encode_resource_spans( + sdk_spans: Sequence[ReadableSpan], +) -> list[ResourceSpans]: + sdk_resource_spans: dict = defaultdict(lambda: defaultdict(list)) + + for sdk_span in sdk_spans: + sdk_resource = sdk_span.resource + sdk_instrumentation = sdk_span.instrumentation_scope or None + pb2_span = _encode_span(sdk_span) + sdk_resource_spans[sdk_resource][sdk_instrumentation].append(pb2_span) + + pb2_resource_spans = [] + for sdk_resource, sdk_instrumentations in sdk_resource_spans.items(): + scope_spans = [] + for sdk_instrumentation, pb2_spans in sdk_instrumentations.items(): + scope_spans.append( + ScopeSpans( + scope=_encode_instrumentation_scope(sdk_instrumentation), + spans=pb2_spans, + schema_url=sdk_instrumentation.schema_url + if sdk_instrumentation + else "", + ) + ) + pb2_resource_spans.append( + ResourceSpans( + resource=_encode_resource(sdk_resource), + scope_spans=scope_spans, + schema_url=sdk_resource.schema_url, + ) + ) + return pb2_resource_spans + + +def _span_flags(parent_span_context: SpanContext | None) -> int: + flags = _SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK + if parent_span_context and parent_span_context.is_remote: + flags |= _SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK + return flags + + +def _encode_span(sdk_span: ReadableSpan) -> Span: + span_context = sdk_span.get_span_context() + return Span( + trace_id=_encode_trace_id(span_context.trace_id), + span_id=_encode_span_id(span_context.span_id), + trace_state=_encode_trace_state(span_context.trace_state), + parent_span_id=_encode_parent_id(sdk_span.parent), + name=sdk_span.name, + kind=_SPAN_KIND_MAP[sdk_span.kind], + start_time_unix_nano=sdk_span.start_time, + end_time_unix_nano=sdk_span.end_time, + attributes=_encode_attributes(sdk_span.attributes), + events=_encode_events(sdk_span.events), + links=_encode_links(sdk_span.links), + status=_encode_status(sdk_span.status), + dropped_attributes_count=sdk_span.dropped_attributes, + dropped_events_count=sdk_span.dropped_events, + dropped_links_count=sdk_span.dropped_links, + flags=_span_flags(sdk_span.parent), + ) + + +def _encode_events(events: Sequence[Event]) -> list[Span.Event] | None: + if not events: + return None + return [ + Span.Event( + name=event.name, + time_unix_nano=event.timestamp, + attributes=_encode_attributes(event.attributes), + dropped_attributes_count=event.dropped_attributes, + ) + for event in events + ] + + +def _encode_links(links: Sequence[Link]) -> list[Span.Link] | None: + if not links: + return None + return [ + Span.Link( + trace_id=_encode_trace_id(link.context.trace_id), + span_id=_encode_span_id(link.context.span_id), + attributes=_encode_attributes(link.attributes), + dropped_attributes_count=link.dropped_attributes, + flags=_span_flags(link.context), + ) + for link in links + ] + + +def _encode_status(status) -> Status | None: + if status is None: + return None + return Status( + code=status.status_code.value, + message=status.description or "", + ) + + +def _encode_trace_state(trace_state: TraceState) -> str: + if trace_state is None: + return "" + return ",".join(f"{key}={value}" for key, value in trace_state.items()) + + +def _encode_parent_id(context: SpanContext | None) -> bytes: + if context: + return _encode_span_id(context.span_id) + return b"" diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-common/src/opentelemetry/exporter/otlp/pyproto/common/version/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-common/src/opentelemetry/exporter/otlp/pyproto/common/version/__init__.py new file mode 100644 index 00000000000..697c4434b93 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-common/src/opentelemetry/exporter/otlp/pyproto/common/version/__init__.py @@ -0,0 +1 @@ +__version__ = "1.43.0.dev" diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-common/tests/test_internal.py b/exporter/opentelemetry-exporter-otlp-pyproto-common/tests/test_internal.py new file mode 100644 index 00000000000..b83ca60d282 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-common/tests/test_internal.py @@ -0,0 +1,159 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from opentelemetry.exporter.otlp.proto.common._internal import ( + _encode_attributes as proto_encode_attributes, + _encode_instrumentation_scope as proto_encode_instrumentation_scope, + _encode_resource as proto_encode_resource, + _encode_span_id as proto_encode_span_id, + _encode_trace_id as proto_encode_trace_id, + _encode_value as proto_encode_value, +) +from opentelemetry.exporter.otlp.pyproto.common._internal import ( + _encode_attributes, + _encode_instrumentation_scope, + _encode_resource, + _encode_span_id, + _encode_trace_id, + _encode_value, +) +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.util.instrumentation import InstrumentationScope + + +# ── _encode_span_id / _encode_trace_id ─────────────────────────────────────── + +def test_encode_span_id_matches_proto() -> None: + span_id = 0x1234567890ABCDEF + assert _encode_span_id(span_id) == proto_encode_span_id(span_id) + + +def test_encode_trace_id_matches_proto() -> None: + trace_id = 0x3E0C63257DE34C926F9EFCD03927272E + assert _encode_trace_id(trace_id) == proto_encode_trace_id(trace_id) + + +def test_encode_span_id_zero() -> None: + assert _encode_span_id(0) == proto_encode_span_id(0) + + +# ── _encode_value ───────────────────────────────────────────────────────────── + +def test_encode_value_none_matches_proto() -> None: + assert _encode_value(None).SerializeToString() == proto_encode_value(None).SerializeToString() + + +def test_encode_value_string_matches_proto() -> None: + assert _encode_value("hello").SerializeToString() == proto_encode_value("hello").SerializeToString() + + +def test_encode_value_bool_true_matches_proto() -> None: + assert _encode_value(True).SerializeToString() == proto_encode_value(True).SerializeToString() + + +def test_encode_value_bool_false_matches_proto() -> None: + assert _encode_value(False).SerializeToString() == proto_encode_value(False).SerializeToString() + + +def test_encode_value_int_matches_proto() -> None: + assert _encode_value(42).SerializeToString() == proto_encode_value(42).SerializeToString() + + +def test_encode_value_negative_int_matches_proto() -> None: + assert _encode_value(-7).SerializeToString() == proto_encode_value(-7).SerializeToString() + + +def test_encode_value_float_matches_proto() -> None: + assert _encode_value(3.14).SerializeToString() == proto_encode_value(3.14).SerializeToString() + + +def test_encode_value_bytes_matches_proto() -> None: + assert _encode_value(b"\x01\x02\x03").SerializeToString() == proto_encode_value(b"\x01\x02\x03").SerializeToString() + + +def test_encode_value_list_matches_proto() -> None: + assert _encode_value([1, "a", True]).SerializeToString() == proto_encode_value([1, "a", True]).SerializeToString() + + +def test_encode_value_dict_matches_proto() -> None: + assert _encode_value({"k": "v", "n": 1}).SerializeToString() == proto_encode_value({"k": "v", "n": 1}).SerializeToString() + + +def test_encode_value_nested_matches_proto() -> None: + value = {"error": None, "tags": ["a", "b"], "count": 5} + assert _encode_value(value).SerializeToString() == proto_encode_value(value).SerializeToString() + + +# ── _encode_attributes ──────────────────────────────────────────────────────── + +def test_encode_attributes_empty_matches_proto() -> None: + our = _encode_attributes({}) + proto = proto_encode_attributes({}) + our_bytes = b"".join(kv.SerializeToString() for kv in our) + proto_bytes = b"".join(kv.SerializeToString() for kv in proto) + assert our_bytes == proto_bytes + + +def test_encode_attributes_with_values_matches_proto() -> None: + attrs = {"service.name": "my-service", "count": 5, "enabled": True, "ratio": 0.5} + our = _encode_attributes(attrs) + proto = proto_encode_attributes(attrs) + our_bytes = b"".join(kv.SerializeToString() for kv in our) + proto_bytes = b"".join(kv.SerializeToString() for kv in proto) + assert our_bytes == proto_bytes + + +def test_encode_attributes_none_matches_proto() -> None: + our = _encode_attributes(None) + proto = proto_encode_attributes(None) + assert our == proto == [] + + +# ── _encode_instrumentation_scope ───────────────────────────────────────────── + +def test_encode_instrumentation_scope_empty_matches_proto() -> None: + scope = InstrumentationScope(name="") + our = _encode_instrumentation_scope(scope) + proto = proto_encode_instrumentation_scope(scope) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_encode_instrumentation_scope_name_version_matches_proto() -> None: + scope = InstrumentationScope(name="mylib", version="1.2.3") + our = _encode_instrumentation_scope(scope) + proto = proto_encode_instrumentation_scope(scope) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_encode_instrumentation_scope_with_attributes_matches_proto() -> None: + scope = InstrumentationScope( + name="mylib", + version="2.0", + schema_url="https://example.com", + attributes={"env": "prod", "version": 2}, + ) + our = _encode_instrumentation_scope(scope) + proto = proto_encode_instrumentation_scope(scope) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_encode_instrumentation_scope_none_matches_proto() -> None: + our = _encode_instrumentation_scope(None) + proto = proto_encode_instrumentation_scope(None) + assert our.SerializeToString() == proto.SerializeToString() + + +# ── _encode_resource ────────────────────────────────────────────────────────── + +def test_encode_resource_empty_matches_proto() -> None: + resource = Resource({}) + our = _encode_resource(resource) + proto = proto_encode_resource(resource) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_encode_resource_with_attributes_matches_proto() -> None: + resource = Resource({"service.name": "my-service", "host.name": "localhost", "pid": 1234}) + our = _encode_resource(resource) + proto = proto_encode_resource(resource) + assert our.SerializeToString() == proto.SerializeToString() diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-common/tests/test_log_encoder.py b/exporter/opentelemetry-exporter-otlp-pyproto-common/tests/test_log_encoder.py new file mode 100644 index 00000000000..5a90938a249 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-common/tests/test_log_encoder.py @@ -0,0 +1,162 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from opentelemetry._logs import LogRecord, SeverityNumber +from opentelemetry.exporter.otlp.proto.common._log_encoder import ( + encode_logs as proto_encode_logs, +) +from opentelemetry.exporter.otlp.pyproto.common._internal._log_encoder import ( + encode_logs as pyproto_encode_logs, +) +from opentelemetry.sdk._logs import LogRecordLimits, ReadWriteLogRecord +from opentelemetry.sdk.resources import Resource as SDKResource +from opentelemetry.sdk.util.instrumentation import InstrumentationScope +from opentelemetry.trace import ( + NonRecordingSpan, + SpanContext, + TraceFlags, + set_span_in_context, +) + +_CONTEXT = set_span_in_context( + NonRecordingSpan( + SpanContext( + 89564621134313219400156819398935297684, + 1312458408527513268, + False, + TraceFlags(0x01), + ) + ) +) + + +def _make_basic_log() -> ReadWriteLogRecord: + return ReadWriteLogRecord( + LogRecord( + timestamp=1644650195189786880, + observed_timestamp=1644650195189786881, + context=_CONTEXT, + severity_text="WARN", + severity_number=SeverityNumber.WARN, + body="Do not go gentle into that good night.", + attributes={"a": 1, "b": "c"}, + ), + resource=SDKResource( + {"first_resource": "value"}, + "resource_schema_url", + ), + instrumentation_scope=InstrumentationScope( + "first_name", "first_version" + ), + ) + + +def test_encode_logs_basic_matches_proto() -> None: + logs = [_make_basic_log()] + assert pyproto_encode_logs(logs).SerializeToString() == proto_encode_logs(logs).SerializeToString() + + +def test_encode_logs_no_instrumentation_scope_matches_proto() -> None: + log = ReadWriteLogRecord( + LogRecord( + timestamp=1644650427658989056, + observed_timestamp=1644650427658989057, + context=_CONTEXT, + severity_text="DEBUG", + severity_number=SeverityNumber.DEBUG, + body={"error": None, "array_with_nones": [1, None, 2]}, + attributes={"a": 1, "b": "c"}, + ), + resource=SDKResource({"second_resource": "CASE"}), + instrumentation_scope=None, + ) + logs = [log] + assert pyproto_encode_logs(logs).SerializeToString() == proto_encode_logs(logs).SerializeToString() + + +def test_encode_logs_empty_resource_with_scope_attributes_matches_proto() -> None: + log = ReadWriteLogRecord( + LogRecord( + timestamp=1644650584292683033, + observed_timestamp=1644650584292683033, + context=_CONTEXT, + severity_text="FATAL", + severity_number=SeverityNumber.FATAL, + body="This instrumentation scope has a schema url and attributes", + attributes={ + "extended": { + "sequence": [{"inner": "mapping", "none": None}] + } + }, + ), + resource=SDKResource({}), + instrumentation_scope=InstrumentationScope( + "scope_with_attributes", + "scope_with_attributes_version", + "instrumentation_schema_url", + {"one": 1, "two": "2"}, + ), + ) + logs = [log] + assert pyproto_encode_logs(logs).SerializeToString() == proto_encode_logs(logs).SerializeToString() + + +def test_encode_logs_dropped_attributes_matches_proto() -> None: + log = ReadWriteLogRecord( + LogRecord( + timestamp=1644650195189786880, + context=_CONTEXT, + severity_text="WARN", + severity_number=SeverityNumber.WARN, + body="message with dropped attributes", + attributes={"a": 1, "b": "c", "user_id": "B121092"}, + ), + resource=SDKResource({"first_resource": "value"}), + limits=LogRecordLimits(max_attributes=1), + instrumentation_scope=InstrumentationScope( + "first_name", "first_version" + ), + ) + logs = [log] + assert pyproto_encode_logs(logs).SerializeToString() == proto_encode_logs(logs).SerializeToString() + + +def test_encode_logs_multiple_resources_matches_proto() -> None: + log1 = _make_basic_log() + log2 = ReadWriteLogRecord( + LogRecord( + timestamp=1644650249738562048, + observed_timestamp=1644650249738562049, + context=_CONTEXT, + severity_text="ERROR", + severity_number=SeverityNumber.ERROR, + body="second log", + ), + resource=SDKResource({"second_resource": "value2"}), + instrumentation_scope=InstrumentationScope( + "second_name", "second_version" + ), + ) + logs = [log1, log2] + assert pyproto_encode_logs(logs).SerializeToString() == proto_encode_logs(logs).SerializeToString() + + +def test_encode_logs_empty_list_matches_proto() -> None: + assert pyproto_encode_logs([]).SerializeToString() == proto_encode_logs([]).SerializeToString() + + +def test_encode_logs_no_trace_context_matches_proto() -> None: + no_context = set_span_in_context(NonRecordingSpan(SpanContext(0, 0, False))) + log = ReadWriteLogRecord( + LogRecord( + timestamp=1644650195189786880, + context=no_context, + severity_text="INFO", + severity_number=SeverityNumber.INFO, + body="no trace context", + ), + resource=SDKResource({}), + instrumentation_scope=None, + ) + logs = [log] + assert pyproto_encode_logs(logs).SerializeToString() == proto_encode_logs(logs).SerializeToString() diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-common/tests/test_metrics_encoder.py b/exporter/opentelemetry-exporter-otlp-pyproto-common/tests/test_metrics_encoder.py new file mode 100644 index 00000000000..42fbd156f4d --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-common/tests/test_metrics_encoder.py @@ -0,0 +1,332 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from opentelemetry.exporter.otlp.proto.common.metrics_encoder import ( + encode_metrics as proto_encode_metrics, +) +from opentelemetry.exporter.otlp.pyproto.common._internal.metrics_encoder import ( + encode_metrics as pyproto_encode_metrics, +) +from opentelemetry.sdk.metrics import Exemplar +from opentelemetry.sdk.metrics.export import ( + AggregationTemporality, + Buckets, + ExponentialHistogramDataPoint, + HistogramDataPoint, + Metric, + MetricsData, + ResourceMetrics, + ScopeMetrics, +) +from opentelemetry.sdk.metrics.export import ( + ExponentialHistogram as ExponentialHistogramType, +) +from opentelemetry.sdk.metrics.export import Histogram as HistogramType +from opentelemetry.sdk.metrics.export import Gauge as SDKGauge +from opentelemetry.sdk.metrics.export import Sum as SDKSum +from opentelemetry.sdk.metrics.export import NumberDataPoint +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.util.instrumentation import ( + InstrumentationScope as SDKInstrumentationScope, +) + + +_SPAN_ID = int("6e0c63257de34c92", 16) +_TRACE_ID = int("d4cda95b652f4a1592b449d5929fda1b", 16) + + +def _wrap(metric: Metric) -> MetricsData: + return MetricsData( + resource_metrics=[ + ResourceMetrics( + resource=Resource({"service.name": "test"}), + scope_metrics=[ + ScopeMetrics( + scope=SDKInstrumentationScope("test_scope", "1.0"), + metrics=[metric], + schema_url="", + ) + ], + schema_url="", + ) + ] + ) + + +# ── Gauge ───────────────────────────────────────────────────────────────────── + +def test_encode_gauge_int_matches_proto() -> None: + data = _wrap(Metric( + name="my.gauge", + description="A gauge", + unit="1", + data=SDKGauge(data_points=[ + NumberDataPoint( + attributes={"host": "localhost"}, + start_time_unix_nano=0, + time_unix_nano=1641946016139533244, + value=42, + exemplars=[], + ) + ]), + )) + assert pyproto_encode_metrics(data).SerializeToString() == proto_encode_metrics(data).SerializeToString() + + +def test_encode_gauge_double_matches_proto() -> None: + data = _wrap(Metric( + name="my.gauge.double", + description="", + unit="s", + data=SDKGauge(data_points=[ + NumberDataPoint( + attributes={"env": "prod"}, + start_time_unix_nano=0, + time_unix_nano=1641946016139533244, + value=3.14, + exemplars=[], + ) + ]), + )) + assert pyproto_encode_metrics(data).SerializeToString() == proto_encode_metrics(data).SerializeToString() + + +def test_encode_gauge_with_exemplar_matches_proto() -> None: + data = _wrap(Metric( + name="gauge.exemplar", + description="", + unit="1", + data=SDKGauge(data_points=[ + NumberDataPoint( + attributes={}, + start_time_unix_nano=0, + time_unix_nano=1641946016139533244, + value=100.0, + exemplars=[ + Exemplar( + {"filtered": "yes"}, + 50.0, + 1641946016139533400, + _SPAN_ID, + _TRACE_ID, + ) + ], + ) + ]), + )) + assert pyproto_encode_metrics(data).SerializeToString() == proto_encode_metrics(data).SerializeToString() + + +# ── Sum ─────────────────────────────────────────────────────────────────────── + +def test_encode_sum_int_matches_proto() -> None: + data = _wrap(Metric( + name="my.counter", + description="A counter", + unit="1", + data=SDKSum( + data_points=[ + NumberDataPoint( + attributes={"a": 1, "b": False}, + start_time_unix_nano=1641946016139533000, + time_unix_nano=1641946016139533244, + value=10, + exemplars=[], + ) + ], + aggregation_temporality=AggregationTemporality.CUMULATIVE, + is_monotonic=True, + ), + )) + assert pyproto_encode_metrics(data).SerializeToString() == proto_encode_metrics(data).SerializeToString() + + +def test_encode_sum_double_delta_matches_proto() -> None: + data = _wrap(Metric( + name="my.updown", + description="", + unit="By", + data=SDKSum( + data_points=[ + NumberDataPoint( + attributes={"direction": "in"}, + start_time_unix_nano=1641946016139533000, + time_unix_nano=1641946016139533244, + value=2.5, + exemplars=[], + ) + ], + aggregation_temporality=AggregationTemporality.DELTA, + is_monotonic=False, + ), + )) + assert pyproto_encode_metrics(data).SerializeToString() == proto_encode_metrics(data).SerializeToString() + + +# ── Histogram ───────────────────────────────────────────────────────────────── + +def test_encode_histogram_matches_proto() -> None: + data = _wrap(Metric( + name="my.histogram", + description="A histogram", + unit="s", + data=HistogramType( + data_points=[ + HistogramDataPoint( + attributes={"a": 1, "b": True}, + start_time_unix_nano=1641946016139533244, + time_unix_nano=1641946016139533244, + exemplars=[ + Exemplar( + {"filtered": "banana"}, + 298.0, + 1641946016139533400, + _SPAN_ID, + _TRACE_ID, + ), + Exemplar( + {"filtered": "banana"}, + 298.0, + 1641946016139533400, + None, + None, + ), + ], + count=5, + sum=67, + bucket_counts=[1, 4], + explicit_bounds=[10.0, 20.0], + min=8, + max=18, + ) + ], + aggregation_temporality=AggregationTemporality.DELTA, + ), + )) + assert pyproto_encode_metrics(data).SerializeToString() == proto_encode_metrics(data).SerializeToString() + + +def test_encode_histogram_no_exemplars_matches_proto() -> None: + data = _wrap(Metric( + name="simple.histogram", + description="", + unit="ms", + data=HistogramType( + data_points=[ + HistogramDataPoint( + attributes={"region": "us-east"}, + start_time_unix_nano=1641946016000000000, + time_unix_nano=1641946016139533244, + exemplars=[], + count=10, + sum=500.0, + bucket_counts=[2, 3, 5], + explicit_bounds=[100.0, 200.0], + min=10.0, + max=200.0, + ) + ], + aggregation_temporality=AggregationTemporality.CUMULATIVE, + ), + )) + assert pyproto_encode_metrics(data).SerializeToString() == proto_encode_metrics(data).SerializeToString() + + +# ── ExponentialHistogram ────────────────────────────────────────────────────── + +def test_encode_exponential_histogram_matches_proto() -> None: + data = _wrap(Metric( + name="exp.histogram", + description="", + unit="1", + data=ExponentialHistogramType( + data_points=[ + ExponentialHistogramDataPoint( + attributes={"host": "node1"}, + start_time_unix_nano=1641946016000000000, + time_unix_nano=1641946016139533244, + exemplars=[], + count=8, + sum=100.0, + scale=2, + zero_count=1, + positive=Buckets(offset=1, bucket_counts=[3, 4]), + negative=Buckets(offset=-1, bucket_counts=[1]), + flags=0, + min=1.0, + max=50.0, + ) + ], + aggregation_temporality=AggregationTemporality.DELTA, + ), + )) + assert pyproto_encode_metrics(data).SerializeToString() == proto_encode_metrics(data).SerializeToString() + + +# ── Multiple metrics and resources ──────────────────────────────────────────── + +def test_encode_multiple_metrics_matches_proto() -> None: + gauge_metric = Metric( + name="gauge", + description="", + unit="1", + data=SDKGauge(data_points=[ + NumberDataPoint( + attributes={}, + start_time_unix_nano=0, + time_unix_nano=1641946016139533244, + value=1, + exemplars=[], + ) + ]), + ) + counter_metric = Metric( + name="counter", + description="", + unit="1", + data=SDKSum( + data_points=[ + NumberDataPoint( + attributes={}, + start_time_unix_nano=1641946016000000000, + time_unix_nano=1641946016139533244, + value=5, + exemplars=[], + ) + ], + aggregation_temporality=AggregationTemporality.CUMULATIVE, + is_monotonic=True, + ), + ) + data = MetricsData( + resource_metrics=[ + ResourceMetrics( + resource=Resource({"service.name": "svc-a"}), + scope_metrics=[ + ScopeMetrics( + scope=SDKInstrumentationScope("lib_a", "1.0"), + metrics=[gauge_metric, counter_metric], + schema_url="", + ) + ], + schema_url="", + ), + ResourceMetrics( + resource=Resource({"service.name": "svc-b"}), + scope_metrics=[ + ScopeMetrics( + scope=SDKInstrumentationScope("lib_b", "2.0"), + metrics=[gauge_metric], + schema_url="", + ) + ], + schema_url="resource_schema_url", + ), + ] + ) + assert pyproto_encode_metrics(data).SerializeToString() == proto_encode_metrics(data).SerializeToString() + + +def test_encode_metrics_empty_matches_proto() -> None: + data = MetricsData(resource_metrics=[]) + assert pyproto_encode_metrics(data).SerializeToString() == proto_encode_metrics(data).SerializeToString() diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-common/tests/test_trace_encoder.py b/exporter/opentelemetry-exporter-otlp-pyproto-common/tests/test_trace_encoder.py new file mode 100644 index 00000000000..c793722ecf1 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-common/tests/test_trace_encoder.py @@ -0,0 +1,197 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from opentelemetry.exporter.otlp.proto.common.trace_encoder import ( + encode_spans as proto_encode_spans, +) +from opentelemetry.exporter.otlp.pyproto.common._internal.trace_encoder import ( + encode_spans as pyproto_encode_spans, +) +from opentelemetry.sdk.trace import Event as SDKEvent +from opentelemetry.sdk.trace import Resource as SDKResource +from opentelemetry.sdk.trace import SpanContext as SDKSpanContext +from opentelemetry.sdk.trace import _Span as SDKSpan +from opentelemetry.sdk.util.instrumentation import ( + InstrumentationScope as SDKInstrumentationScope, +) +from opentelemetry.trace import Link as SDKLink +from opentelemetry.trace import SpanKind as SDKSpanKind +from opentelemetry.trace import TraceFlags as SDKTraceFlags +from opentelemetry.trace.status import Status as SDKStatus +from opentelemetry.trace.status import StatusCode as SDKStatusCode + + +def _make_exhaustive_spans() -> list[SDKSpan]: + trace_id = 0x3E0C63257DE34C926F9EFCD03927272E + + base_time = 683647322 * 10**9 + start_times = ( + base_time, + base_time + 150 * 10**6, + base_time + 300 * 10**6, + base_time + 400 * 10**6, + base_time + 500 * 10**6, + base_time + 600 * 10**6, + ) + end_times = ( + start_times[0] + (50 * 10**6), + start_times[1] + (100 * 10**6), + start_times[2] + (200 * 10**6), + start_times[3] + (300 * 10**6), + start_times[4] + (400 * 10**6), + start_times[5] + (500 * 10**6), + ) + + parent_span_context = SDKSpanContext( + trace_id, 0x1111111111111111, is_remote=True + ) + other_context = SDKSpanContext( + trace_id, 0x2222222222222222, is_remote=False + ) + + span1 = SDKSpan( + name="test-span-1", + context=SDKSpanContext( + trace_id, + 0x34BF92DEEFC58C92, + is_remote=False, + trace_flags=SDKTraceFlags(SDKTraceFlags.SAMPLED), + ), + parent=parent_span_context, + events=( + SDKEvent( + name="event0", + timestamp=base_time + 50 * 10**6, + attributes={ + "annotation_bool": True, + "annotation_string": "annotation_test", + "key_float": 0.3, + }, + ), + ), + links=( + SDKLink(context=other_context, attributes={"key_bool": True}), + ), + resource=SDKResource({}, "resource_schema_url"), + ) + span1.start(start_time=start_times[0]) + span1.set_attribute("key_bool", False) + span1.set_attribute("key_string", "hello_world") + span1.set_attribute("key_float", 111.22) + span1.set_status(SDKStatus(SDKStatusCode.ERROR, "Example description")) + span1.end(end_time=end_times[0]) + + span2 = SDKSpan( + name="test-span-2", + context=parent_span_context, + parent=None, + resource=SDKResource(attributes={"key_resource": "some_resource"}), + ) + span2.start(start_time=start_times[1]) + span2.end(end_time=end_times[1]) + + span3 = SDKSpan( + name="test-span-3", + context=other_context, + parent=None, + resource=SDKResource(attributes={"key_resource": "some_resource"}), + ) + span3.start(start_time=start_times[2]) + span3.set_attribute("key_string", "hello_world") + span3.end(end_time=end_times[2]) + + span4 = SDKSpan( + name="test-span-4", + context=other_context, + parent=None, + resource=SDKResource({}, "resource_schema_url"), + instrumentation_scope=SDKInstrumentationScope( + name="name", version="version" + ), + ) + span4.start(start_time=start_times[3]) + span4.end(end_time=end_times[3]) + + span5 = SDKSpan( + name="test-span-5", + context=other_context, + parent=None, + resource=SDKResource( + attributes={"key_resource": "another_resource"}, + schema_url="resource_schema_url", + ), + instrumentation_scope=SDKInstrumentationScope( + name="scope_1_name", + version="scope_1_version", + schema_url="scope_1_schema_url", + ), + ) + span5.start(start_time=start_times[4]) + span5.end(end_time=end_times[4]) + + span6 = SDKSpan( + name="test-span-6", + context=other_context, + parent=None, + resource=SDKResource( + attributes={"key_resource": "another_resource"}, + schema_url="resource_schema_url", + ), + instrumentation_scope=SDKInstrumentationScope( + name="scope_2_name", + version="scope_2_version", + schema_url="scope_2_schema_url", + attributes={"one": "1", "two": 2}, + ), + ) + span6.start(start_time=start_times[5]) + span6.end(end_time=end_times[5]) + + return [span1, span2, span3, span4, span5, span6] + + +def test_encode_spans_single_minimal() -> None: + span = SDKSpan( + name="hello", + context=SDKSpanContext( + 0x3E0C63257DE34C926F9EFCD03927272E, + 0x34BF92DEEFC58C92, + is_remote=False, + trace_flags=SDKTraceFlags(SDKTraceFlags.SAMPLED), + ), + parent=None, + resource=SDKResource({}), + ) + span.start(start_time=1_000_000_000) + span.end(end_time=2_000_000_000) + spans = [span] + assert pyproto_encode_spans(spans).SerializeToString() == proto_encode_spans(spans).SerializeToString() + + +def test_encode_spans_with_attributes_and_status() -> None: + spans = _make_exhaustive_spans()[:1] # span1 has attributes, events, links, status + assert pyproto_encode_spans(spans).SerializeToString() == proto_encode_spans(spans).SerializeToString() + + +def test_encode_spans_multiple_resources() -> None: + spans = _make_exhaustive_spans() + assert pyproto_encode_spans(spans).SerializeToString() == proto_encode_spans(spans).SerializeToString() + + +def test_encode_spans_with_instrumentation_scope() -> None: + spans = _make_exhaustive_spans()[3:4] # span4 has instrumentation scope + assert pyproto_encode_spans(spans).SerializeToString() == proto_encode_spans(spans).SerializeToString() + + +def test_encode_spans_with_scope_attributes() -> None: + spans = _make_exhaustive_spans()[5:6] # span6 has scope with attributes + assert pyproto_encode_spans(spans).SerializeToString() == proto_encode_spans(spans).SerializeToString() + + +def test_encode_spans_empty_list() -> None: + assert pyproto_encode_spans([]).SerializeToString() == proto_encode_spans([]).SerializeToString() + + +def test_encode_spans_exhaustive_matches_proto() -> None: + spans = _make_exhaustive_spans() + assert pyproto_encode_spans(spans).SerializeToString() == proto_encode_spans(spans).SerializeToString() diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-grpc/README.rst b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/README.rst new file mode 100644 index 00000000000..3778b54d4c3 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/README.rst @@ -0,0 +1,4 @@ +OpenTelemetry OTLP pure-Python protobuf gRPC Exporter +====================================================== + +Pure-Python protobuf OTLP over gRPC exporter, no ``google.protobuf`` dependency. diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-grpc/pyproject.toml b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/pyproject.toml new file mode 100644 index 00000000000..2150fd6399c --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/pyproject.toml @@ -0,0 +1,42 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "opentelemetry-exporter-otlp-pyproto-grpc" +dynamic = ["version"] +description = "OpenTelemetry Collector pure-Python protobuf over gRPC Exporter (no google.protobuf)" +readme = "README.rst" +license = "Apache-2.0" +requires-python = ">=3.10" +dependencies = [ + "opentelemetry-api ~= 1.12", + "opentelemetry-sdk ~= 1.12", + "opentelemetry-exporter-otlp-pyproto-common == 1.43.0.dev", + "grpcio >= 1.63.2, < 2.0.0; python_version < '3.13'", + "grpcio >= 1.66.2, < 2.0.0; python_version == '3.13'", + "grpcio >= 1.75.1, < 2.0.0; python_version >= '3.14'", +] + +[project.entry-points.opentelemetry_traces_exporter] +otlp_pyproto_grpc = "opentelemetry.exporter.otlp.pyproto.grpc.trace_exporter:OTLPSpanExporter" + +[project.entry-points.opentelemetry_metrics_exporter] +otlp_pyproto_grpc = "opentelemetry.exporter.otlp.pyproto.grpc.metric_exporter:OTLPMetricExporter" + +[project.entry-points.opentelemetry_logs_exporter] +otlp_pyproto_grpc = "opentelemetry.exporter.otlp.pyproto.grpc._log_exporter:OTLPLogExporter" + +[dependency-groups] +dev = [ + "pytest>=7.0", +] + +[tool.hatch.version] +path = "src/opentelemetry/exporter/otlp/pyproto/grpc/version/__init__.py" + +[tool.hatch.build.targets.sdist] +include = ["/src"] + +[tool.hatch.build.targets.wheel] +packages = ["src/opentelemetry"] diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-grpc/src/opentelemetry/exporter/otlp/pyproto/grpc/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/src/opentelemetry/exporter/otlp/pyproto/grpc/__init__.py new file mode 100644 index 00000000000..a24bc04a0e2 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/src/opentelemetry/exporter/otlp/pyproto/grpc/__init__.py @@ -0,0 +1,9 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from .version import __version__ + +_USER_AGENT_HEADER_VALUE = "OTel-OTLP-Exporter-Python/" + __version__ +_OTLP_GRPC_CHANNEL_OPTIONS = [ + ("grpc.primary_user_agent", _USER_AGENT_HEADER_VALUE) +] diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-grpc/src/opentelemetry/exporter/otlp/pyproto/grpc/_log_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/src/opentelemetry/exporter/otlp/pyproto/grpc/_log_exporter/__init__.py new file mode 100644 index 00000000000..e8fa6a6f513 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/src/opentelemetry/exporter/otlp/pyproto/grpc/_log_exporter/__init__.py @@ -0,0 +1,118 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from collections.abc import Iterable, Sequence +from collections.abc import Sequence as TypingSequence +from os import environ +from typing import Literal + +from grpc import ChannelCredentials, Compression, StatusCode + +from opentelemetry.exporter.otlp.pyproto.common._internal._log_encoder import encode_logs +from opentelemetry.exporter.otlp.pyproto.grpc.exporter import ( + OTLPExporterMixin, + _get_credentials, + environ_to_compression, +) +from opentelemetry.metrics import MeterProvider +from opentelemetry.pyproto.collector.logs.v1.logs_service_pypb2 import ExportLogsServiceRequest +from opentelemetry.pyproto.collector.logs.v1.logs_service_pypb2_grpc import LogsServiceStub +from opentelemetry.sdk._logs import ReadableLogRecord +from opentelemetry.sdk._logs.export import LogRecordExporter, LogRecordExportResult +from opentelemetry.sdk.environment_variables import ( + _OTEL_PYTHON_EXPORTER_OTLP_GRPC_LOGS_CREDENTIAL_PROVIDER, + OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE, + OTEL_EXPORTER_OTLP_LOGS_CLIENT_CERTIFICATE, + OTEL_EXPORTER_OTLP_LOGS_CLIENT_KEY, + OTEL_EXPORTER_OTLP_LOGS_COMPRESSION, + OTEL_EXPORTER_OTLP_LOGS_ENDPOINT, + OTEL_EXPORTER_OTLP_LOGS_HEADERS, + OTEL_EXPORTER_OTLP_LOGS_INSECURE, + OTEL_EXPORTER_OTLP_LOGS_TIMEOUT, +) +from opentelemetry.semconv._incubating.attributes.otel_attributes import OtelComponentTypeValues + + +class OTLPLogExporter( + LogRecordExporter, + OTLPExporterMixin[ + Sequence[ReadableLogRecord], + ExportLogsServiceRequest, + LogRecordExportResult, + LogsServiceStub, + ], +): + def __init__( + self, + endpoint: str | None = None, + insecure: bool | None = None, + credentials: ChannelCredentials | None = None, + headers: TypingSequence[tuple[str, str]] | dict[str, str] | str | None = None, + timeout: float | None = None, + compression: Compression | None = None, + channel_options: tuple[tuple[str, str]] | None = None, + retryable_error_codes: Iterable[StatusCode] | None = None, + *, + meter_provider: MeterProvider | None = None, + ): + insecure_logs = environ.get(OTEL_EXPORTER_OTLP_LOGS_INSECURE) + if insecure is None and insecure_logs is not None: + insecure = insecure_logs.lower() == "true" + + if not insecure and environ.get(OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE) is not None: + credentials = _get_credentials( + credentials, + _OTEL_PYTHON_EXPORTER_OTLP_GRPC_LOGS_CREDENTIAL_PROVIDER, + OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE, + OTEL_EXPORTER_OTLP_LOGS_CLIENT_KEY, + OTEL_EXPORTER_OTLP_LOGS_CLIENT_CERTIFICATE, + ) + + environ_timeout = environ.get(OTEL_EXPORTER_OTLP_LOGS_TIMEOUT) + environ_timeout = float(environ_timeout) if environ_timeout is not None else None + + compression = ( + environ_to_compression(OTEL_EXPORTER_OTLP_LOGS_COMPRESSION) + if compression is None + else compression + ) + + OTLPExporterMixin.__init__( + self, + endpoint=endpoint or environ.get(OTEL_EXPORTER_OTLP_LOGS_ENDPOINT), + insecure=insecure, + credentials=credentials, + headers=headers or environ.get(OTEL_EXPORTER_OTLP_LOGS_HEADERS), + timeout=timeout or environ_timeout, + compression=compression, + stub=LogsServiceStub, + result=LogRecordExportResult, + channel_options=channel_options, + retryable_error_codes=retryable_error_codes, + component_type=OtelComponentTypeValues.OTLP_GRPC_LOG_EXPORTER, + signal="logs", + meter_provider=meter_provider, + ) + + def _translate_data(self, data: Sequence[ReadableLogRecord]) -> ExportLogsServiceRequest: + return encode_logs(data) + + def _count_data(self, data: Sequence[ReadableLogRecord]) -> int: + return len(data) + + def export( + self, + batch: Sequence[ReadableLogRecord], + ) -> Literal[LogRecordExportResult.SUCCESS, LogRecordExportResult.FAILURE]: + return OTLPExporterMixin._export(self, batch) + + def shutdown(self, timeout_millis: float = 30_000, **kwargs) -> None: + OTLPExporterMixin.shutdown(self, timeout_millis=timeout_millis) + + def force_flush(self, timeout_millis: float = 10_000) -> bool: + """Nothing is buffered in this exporter, so this method does nothing.""" + return True + + @property + def _exporting(self) -> str: + return "logs" diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-grpc/src/opentelemetry/exporter/otlp/pyproto/grpc/exporter.py b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/src/opentelemetry/exporter/otlp/pyproto/grpc/exporter.py new file mode 100644 index 00000000000..4369f6b4b22 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/src/opentelemetry/exporter/otlp/pyproto/grpc/exporter.py @@ -0,0 +1,439 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +import os +import random +import threading +from abc import ABC, abstractmethod +from collections.abc import Iterable, Sequence +from collections.abc import Sequence as TypingSequence +from logging import getLogger +from os import environ +from time import time +from typing import Generic, Literal, TypeVar +from urllib.parse import urlparse + +from grpc import ( + ChannelCredentials, + Compression, + RpcError, + StatusCode, + insecure_channel, + secure_channel, + ssl_channel_credentials, +) + +from opentelemetry.exporter.otlp.pyproto.common._exporter_metrics import ( + create_exporter_metrics, +) +from opentelemetry.exporter.otlp.pyproto.grpc import ( + _OTLP_GRPC_CHANNEL_OPTIONS, +) +from opentelemetry.metrics import MeterProvider +from opentelemetry.pyproto.collector.logs.v1.logs_service_pypb2 import ( + ExportLogsServiceRequest, +) +from opentelemetry.pyproto.collector.logs.v1.logs_service_pypb2_grpc import ( + LogsServiceStub, +) +from opentelemetry.pyproto.collector.metrics.v1.metrics_service_pypb2 import ( + ExportMetricsServiceRequest, +) +from opentelemetry.pyproto.collector.metrics.v1.metrics_service_pypb2_grpc import ( + MetricsServiceStub, +) +from opentelemetry.pyproto.collector.trace.v1.trace_service_pypb2 import ( + ExportTraceServiceRequest, +) +from opentelemetry.pyproto.collector.trace.v1.trace_service_pypb2_grpc import ( + TraceServiceStub, +) +from opentelemetry.sdk._logs import ReadableLogRecord +from opentelemetry.sdk._logs.export import LogRecordExportResult +from opentelemetry.sdk._shared_internal import DuplicateFilter +from opentelemetry.sdk.environment_variables import ( + _OTEL_PYTHON_EXPORTER_OTLP_GRPC_CREDENTIAL_PROVIDER, + _OTEL_PYTHON_EXPORTER_OTLP_GRPC_RETRYABLE_ERROR_CODES, + OTEL_EXPORTER_OTLP_CERTIFICATE, + OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE, + OTEL_EXPORTER_OTLP_CLIENT_KEY, + OTEL_EXPORTER_OTLP_COMPRESSION, + OTEL_EXPORTER_OTLP_ENDPOINT, + OTEL_EXPORTER_OTLP_HEADERS, + OTEL_EXPORTER_OTLP_INSECURE, + OTEL_EXPORTER_OTLP_TIMEOUT, + OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED, +) +from opentelemetry.sdk.metrics.export import MetricExportResult, MetricsData +from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.sdk.trace.export import SpanExportResult +from opentelemetry.semconv._incubating.attributes.otel_attributes import ( + OtelComponentTypeValues, +) +from opentelemetry.semconv._incubating.attributes.rpc_attributes import ( + RPC_RESPONSE_STATUS_CODE, +) +from opentelemetry.util._importlib_metadata import entry_points +from opentelemetry.util.re import parse_env_headers + +_RETRYABLE_ERROR_CODES = frozenset([ + StatusCode.CANCELLED, + StatusCode.DEADLINE_EXCEEDED, + StatusCode.RESOURCE_EXHAUSTED, + StatusCode.ABORTED, + StatusCode.OUT_OF_RANGE, + StatusCode.UNAVAILABLE, + StatusCode.DATA_LOSS, +]) +_MAX_RETRYS = 6 +logger = getLogger(__name__) +logger.addFilter(DuplicateFilter()) + +SDKDataT = TypeVar( + "SDKDataT", + TypingSequence[ReadableLogRecord], + MetricsData, + TypingSequence[ReadableSpan], +) +ExportServiceRequestT = TypeVar( + "ExportServiceRequestT", + ExportTraceServiceRequest, + ExportMetricsServiceRequest, + ExportLogsServiceRequest, +) +ExportResultT = TypeVar( + "ExportResultT", + LogRecordExportResult, + MetricExportResult, + SpanExportResult, +) +ExportStubT = TypeVar( + "ExportStubT", TraceServiceStub, MetricsServiceStub, LogsServiceStub +) + +_ENVIRON_TO_COMPRESSION = { + None: None, + "gzip": Compression.Gzip, +} + + +class InvalidCompressionValueException(Exception): + def __init__(self, environ_key: str, environ_value: str): + super().__init__( + f'Invalid value "{environ_value}" for compression envvar {environ_key}' + ) + + +def environ_to_compression(environ_key: str) -> Compression | None: + environ_value = ( + environ[environ_key].lower().strip() + if environ_key in environ + else None + ) + if environ_value not in _ENVIRON_TO_COMPRESSION and environ_value is not None: + raise InvalidCompressionValueException(environ_key, environ_value) + return _ENVIRON_TO_COMPRESSION[environ_value] + + +def _read_file(file_path: str) -> bytes | None: + try: + with open(file_path, "rb") as file: + return file.read() + except FileNotFoundError as e: + logger.exception( + "Failed to read file: %s. Please check if the file exists and is accessible.", + e.filename, + ) + return None + + +def _load_credentials( + certificate_file: str | None, + client_key_file: str | None, + client_certificate_file: str | None, +) -> ChannelCredentials: + root_certificates = _read_file(certificate_file) if certificate_file else None + private_key = _read_file(client_key_file) if client_key_file else None + certificate_chain = _read_file(client_certificate_file) if client_certificate_file else None + return ssl_channel_credentials( + root_certificates=root_certificates, + private_key=private_key, + certificate_chain=certificate_chain, + ) + + +def _get_credentials( + creds: ChannelCredentials | None, + credential_entry_point_env_key: str, + certificate_file_env_key: str, + client_key_file_env_key: str, + client_certificate_file_env_key: str, +) -> ChannelCredentials: + if creds is not None: + return creds + _credential_env = environ.get(credential_entry_point_env_key) + if _credential_env: + try: + maybe_channel_creds = next( + iter( + entry_points( + group="opentelemetry_otlp_credential_provider", + name=_credential_env, + ) + ) + ).load()() + except StopIteration: + raise RuntimeError( + f"Requested component '{_credential_env}' not found in " + f"entry point 'opentelemetry_otlp_credential_provider'" + ) + if isinstance(maybe_channel_creds, ChannelCredentials): + return maybe_channel_creds + else: + raise RuntimeError( + f"Requested component '{_credential_env}' is of type {type(maybe_channel_creds)}" + f" must be of type `grpc.ChannelCredentials`." + ) + + certificate_file = environ.get(certificate_file_env_key) + if certificate_file: + client_key_file = environ.get(client_key_file_env_key) + client_certificate_file = environ.get(client_certificate_file_env_key) + credentials = _load_credentials(certificate_file, client_key_file, client_certificate_file) + if credentials is not None: + return credentials + return ssl_channel_credentials() + + +class OTLPExporterMixin(ABC, Generic[SDKDataT, ExportServiceRequestT, ExportResultT, ExportStubT]): + def __init__( + self, + stub: ExportStubT, + result: ExportResultT, + endpoint: str | None = None, + insecure: bool | None = None, + credentials: ChannelCredentials | None = None, + headers: TypingSequence[tuple[str, str]] | dict[str, str] | str | None = None, + timeout: float | None = None, + compression: Compression | None = None, + channel_options: tuple[tuple[str, str]] | None = None, + retryable_error_codes: Iterable[StatusCode] | None = None, + *, + component_type: OtelComponentTypeValues | None = None, + signal: Literal["traces", "metrics", "logs"] = "traces", + meter_provider: MeterProvider | None = None, + ): + super().__init__() + self._result = result + self._stub = stub + self._endpoint = endpoint or environ.get(OTEL_EXPORTER_OTLP_ENDPOINT, "http://localhost:4317") + + parsed_url = urlparse(self._endpoint) + + if parsed_url.scheme == "https": + insecure = False + insecure_exporter = environ.get(OTEL_EXPORTER_OTLP_INSECURE) + if insecure is None: + if insecure_exporter is not None: + insecure = insecure_exporter.lower() == "true" + else: + insecure = parsed_url.scheme == "http" + + if parsed_url.netloc: + self._endpoint = parsed_url.netloc + + self._insecure = insecure + self._credentials = credentials + self._headers = headers or environ.get(OTEL_EXPORTER_OTLP_HEADERS) + if isinstance(self._headers, str): + temp_headers = parse_env_headers(self._headers, liberal=True) + self._headers = tuple(temp_headers.items()) + elif isinstance(self._headers, dict): + self._headers = tuple(self._headers.items()) + if self._headers is None: + self._headers = tuple() + + if channel_options: + overridden_options = {opt_name for (opt_name, _) in channel_options} + default_options = tuple( + (opt_name, opt_value) + for opt_name, opt_value in _OTLP_GRPC_CHANNEL_OPTIONS + if opt_name not in overridden_options + ) + self._channel_options = default_options + channel_options + else: + self._channel_options = tuple(_OTLP_GRPC_CHANNEL_OPTIONS) + + self._timeout = timeout or float(environ.get(OTEL_EXPORTER_OTLP_TIMEOUT, 10)) + self._collector_kwargs = None + + self._compression = ( + environ_to_compression(OTEL_EXPORTER_OTLP_COMPRESSION) + if compression is None + else compression + ) or Compression.NoCompression + + self._retryable_error_codes = retryable_error_codes or os.environ.get( + _OTEL_PYTHON_EXPORTER_OTLP_GRPC_RETRYABLE_ERROR_CODES + ) + if isinstance(self._retryable_error_codes, str): + self._retryable_error_codes = frozenset( + StatusCode[code.strip().upper()] + for code in self._retryable_error_codes.split(",") + if code.strip() + ) + elif self._retryable_error_codes is not None: + self._retryable_error_codes = frozenset(self._retryable_error_codes) + else: + self._retryable_error_codes = _RETRYABLE_ERROR_CODES + + self._channel = None + self._client = None + self._shutdown_in_progress = threading.Event() + self._shutdown = False + + if not self._insecure: + self._credentials = _get_credentials( + self._credentials, + _OTEL_PYTHON_EXPORTER_OTLP_GRPC_CREDENTIAL_PROVIDER, + OTEL_EXPORTER_OTLP_CERTIFICATE, + OTEL_EXPORTER_OTLP_CLIENT_KEY, + OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE, + ) + + self._component_type = component_type + self._signal: Literal["traces", "metrics", "logs"] = signal + self._parsed_url = parsed_url + self._metrics = create_exporter_metrics( + self._component_type, + signal, + parsed_url, + meter_provider, + os.environ.get(OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED, "") + .strip() + .lower() + == "true", + ) + + self._initialize_channel_and_stub() + + def _initialize_channel_and_stub(self): + if self._insecure: + self._channel = insecure_channel( + self._endpoint, + compression=self._compression, + options=self._channel_options, + ) + else: + assert self._credentials is not None + self._channel = secure_channel( + self._endpoint, + self._credentials, + compression=self._compression, + options=self._channel_options, + ) + self._client = self._stub(self._channel) + + @abstractmethod + def _translate_data(self, data: SDKDataT) -> ExportServiceRequestT: + pass + + @abstractmethod + def _count_data(self, data: SDKDataT) -> int: + pass + + def _export(self, data: SDKDataT) -> ExportResultT: + if self._shutdown: + logger.warning("Exporter already shutdown, ignoring batch") + return self._result.FAILURE + + with self._metrics.export_operation(self._count_data(data)) as result: + deadline_sec = time() + self._timeout + for retry_num in range(_MAX_RETRYS): + backoff_seconds = 2**retry_num * random.uniform(0.8, 1.2) + try: + if self._client is None: + return self._result.FAILURE + self._client.Export( + request=self._translate_data(data), + metadata=self._headers, + timeout=deadline_sec - time(), + ) + return self._result.SUCCESS + except RpcError as error: + if ( + error.code() == StatusCode.UNAVAILABLE + and retry_num == 0 + ): + logger.debug( + "Reinitializing gRPC channel for %s exporter due to UNAVAILABLE error", + self._exporting, + ) + try: + if self._channel: + self._channel.close() + except Exception as e: + logger.debug( + "Error closing channel for %s exporter to %s: %s", + self._exporting, + self._endpoint, + str(e), + ) + self._initialize_channel_and_stub() + + if ( + error.code() not in self._retryable_error_codes + or retry_num + 1 == _MAX_RETRYS + or backoff_seconds > (deadline_sec - time()) + or self._shutdown + ): + logger.error( + "Failed to export %s to %s, error code: %s, error details: %s", + self._exporting, + self._endpoint, + error.code(), + error.details(), + exc_info=error.code() == StatusCode.UNKNOWN, + ) + result.error = error + result.error_attrs = {RPC_RESPONSE_STATUS_CODE: error.code().name} + return self._result.FAILURE + logger.warning( + "Transient error %s encountered while exporting %s to %s, retrying in %.2fs. Error details: %s", + error.code(), + self._exporting, + self._endpoint, + backoff_seconds, + error.details(), + ) + shutdown = self._shutdown_in_progress.wait(backoff_seconds) + if shutdown: + logger.warning("Shutdown in progress, aborting retry.") + break + return self._result.FAILURE + + def shutdown(self, timeout_millis: float = 30_000, **kwargs) -> None: + if self._shutdown: + logger.warning("Exporter already shutdown, ignoring call") + return + self._shutdown = True + self._shutdown_in_progress.set() + if self._channel: + self._channel.close() + + @property + @abstractmethod + def _exporting(self) -> str: + pass + + def _set_meter_provider(self, meter_provider: MeterProvider) -> None: + self._metrics = create_exporter_metrics( + self._component_type, + self._signal, + self._parsed_url, + meter_provider, + os.environ.get(OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED, "") + .strip() + .lower() + == "true", + ) diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-grpc/src/opentelemetry/exporter/otlp/pyproto/grpc/metric_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/src/opentelemetry/exporter/otlp/pyproto/grpc/metric_exporter/__init__.py new file mode 100644 index 00000000000..1d2e8fdd740 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/src/opentelemetry/exporter/otlp/pyproto/grpc/metric_exporter/__init__.py @@ -0,0 +1,198 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from collections.abc import Iterable +from collections.abc import Sequence as TypingSequence +from dataclasses import replace +from os import environ + +from grpc import ChannelCredentials, Compression, StatusCode + +from opentelemetry.exporter.otlp.pyproto.common._internal.metrics_encoder import OTLPMetricExporterMixin, encode_metrics +from opentelemetry.exporter.otlp.pyproto.grpc.exporter import ( + OTLPExporterMixin, + _get_credentials, + environ_to_compression, +) +from opentelemetry.metrics import MeterProvider +from opentelemetry.pyproto.collector.metrics.v1.metrics_service_pypb2 import ExportMetricsServiceRequest +from opentelemetry.pyproto.collector.metrics.v1.metrics_service_pypb2_grpc import MetricsServiceStub +from opentelemetry.sdk.environment_variables import ( + _OTEL_PYTHON_EXPORTER_OTLP_GRPC_METRICS_CREDENTIAL_PROVIDER, + OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE, + OTEL_EXPORTER_OTLP_METRICS_CLIENT_CERTIFICATE, + OTEL_EXPORTER_OTLP_METRICS_CLIENT_KEY, + OTEL_EXPORTER_OTLP_METRICS_COMPRESSION, + OTEL_EXPORTER_OTLP_METRICS_ENDPOINT, + OTEL_EXPORTER_OTLP_METRICS_HEADERS, + OTEL_EXPORTER_OTLP_METRICS_INSECURE, + OTEL_EXPORTER_OTLP_METRICS_TIMEOUT, +) +from opentelemetry.sdk.metrics._internal.aggregation import Aggregation +from opentelemetry.sdk.metrics.export import ( + AggregationTemporality, + DataPointT, + Gauge, + Metric, + MetricExporter, + MetricExportResult, + MetricsData, + ResourceMetrics, + ScopeMetrics, + Sum, +) +from opentelemetry.semconv._incubating.attributes.otel_attributes import OtelComponentTypeValues + + +class OTLPMetricExporter( + MetricExporter, + OTLPExporterMixin[ + MetricsData, + ExportMetricsServiceRequest, + MetricExportResult, + MetricsServiceStub, + ], + OTLPMetricExporterMixin, +): + def __init__( + self, + endpoint: str | None = None, + insecure: bool | None = None, + credentials: ChannelCredentials | None = None, + headers: TypingSequence[tuple[str, str]] | dict[str, str] | str | None = None, + timeout: float | None = None, + compression: Compression | None = None, + preferred_temporality: dict[type, AggregationTemporality] | None = None, + preferred_aggregation: dict[type, Aggregation] | None = None, + max_export_batch_size: int | None = None, + channel_options: tuple[tuple[str, str]] | None = None, + retryable_error_codes: Iterable[StatusCode] | None = None, + *, + meter_provider: MeterProvider | None = None, + ): + insecure_metrics = environ.get(OTEL_EXPORTER_OTLP_METRICS_INSECURE) + if insecure is None and insecure_metrics is not None: + insecure = insecure_metrics.lower() == "true" + + if not insecure and environ.get(OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE) is not None: + credentials = _get_credentials( + credentials, + _OTEL_PYTHON_EXPORTER_OTLP_GRPC_METRICS_CREDENTIAL_PROVIDER, + OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE, + OTEL_EXPORTER_OTLP_METRICS_CLIENT_KEY, + OTEL_EXPORTER_OTLP_METRICS_CLIENT_CERTIFICATE, + ) + + environ_timeout = environ.get(OTEL_EXPORTER_OTLP_METRICS_TIMEOUT) + environ_timeout = float(environ_timeout) if environ_timeout is not None else None + + compression = ( + environ_to_compression(OTEL_EXPORTER_OTLP_METRICS_COMPRESSION) + if compression is None + else compression + ) + + self._common_configuration(preferred_temporality, preferred_aggregation) + + OTLPExporterMixin.__init__( + self, + stub=MetricsServiceStub, + result=MetricExportResult, + endpoint=endpoint or environ.get(OTEL_EXPORTER_OTLP_METRICS_ENDPOINT), + insecure=insecure, + credentials=credentials, + headers=headers or environ.get(OTEL_EXPORTER_OTLP_METRICS_HEADERS), + timeout=timeout or environ_timeout, + compression=compression, + channel_options=channel_options, + retryable_error_codes=retryable_error_codes, + component_type=OtelComponentTypeValues.OTLP_GRPC_METRIC_EXPORTER, + signal="metrics", + meter_provider=meter_provider, + ) + + self._max_export_batch_size: int | None = max_export_batch_size + + def _translate_data(self, data: MetricsData) -> ExportMetricsServiceRequest: + return encode_metrics(data) + + def _count_data(self, data: MetricsData) -> int: + num_items = 0 + for resource_metrics in data.resource_metrics: + for scope_metrics in resource_metrics.scope_metrics: + for metric in scope_metrics.metrics: + num_items += len(metric.data.data_points) + return num_items + + def export( + self, + metrics_data: MetricsData, + timeout_millis: float = 10_000, + **kwargs, + ) -> MetricExportResult: + if self._max_export_batch_size is None: + return self._export(data=metrics_data) + + export_result = MetricExportResult.SUCCESS + for split_metrics_data in self._split_metrics_data(metrics_data): + split_export_result = self._export(data=split_metrics_data) + if split_export_result is MetricExportResult.FAILURE: + export_result = MetricExportResult.FAILURE + return export_result + + def _split_metrics_data(self, metrics_data: MetricsData) -> Iterable[MetricsData]: + assert self._max_export_batch_size is not None + batch_size: int = 0 + split_resource_metrics: list[ResourceMetrics] = [] + + for resource_metrics in metrics_data.resource_metrics: + split_scope_metrics: list[ScopeMetrics] = [] + split_resource_metrics.append(replace(resource_metrics, scope_metrics=split_scope_metrics)) + + for scope_metrics in resource_metrics.scope_metrics: + split_metrics: list[Metric] = [] + split_scope_metrics.append(replace(scope_metrics, metrics=split_metrics)) + + for metric in scope_metrics.metrics: + split_data_points: list[DataPointT] = [] + split_metrics.append(replace(metric, data=replace(metric.data, data_points=split_data_points))) + + for data_point in metric.data.data_points: + split_data_points.append(data_point) + batch_size += 1 + + if batch_size >= self._max_export_batch_size: + yield MetricsData(resource_metrics=split_resource_metrics) + batch_size = 0 + split_data_points = [] + split_metrics = [replace(metric, data=replace(metric.data, data_points=split_data_points))] + split_scope_metrics = [replace(scope_metrics, metrics=split_metrics)] + split_resource_metrics = [replace(resource_metrics, scope_metrics=split_scope_metrics)] + + if not split_data_points: + split_metrics.pop() + + if not split_metrics: + split_scope_metrics.pop() + + if not split_scope_metrics: + split_resource_metrics.pop() + + if batch_size > 0: + yield MetricsData(resource_metrics=split_resource_metrics) + + def shutdown(self, timeout_millis: float = 30_000, **kwargs) -> None: + OTLPExporterMixin.shutdown(self, timeout_millis=timeout_millis) + + def set_meter_provider(self, meter_provider: MeterProvider): + return self._set_meter_provider(meter_provider) + + @property + def _exporting(self) -> str: + return "metrics" + + def force_flush(self, timeout_millis: float = 10_000) -> bool: + """Nothing is buffered in this exporter, so this method does nothing.""" + return True diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-grpc/src/opentelemetry/exporter/otlp/pyproto/grpc/trace_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/src/opentelemetry/exporter/otlp/pyproto/grpc/trace_exporter/__init__.py new file mode 100644 index 00000000000..1bbf3a43d1f --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/src/opentelemetry/exporter/otlp/pyproto/grpc/trace_exporter/__init__.py @@ -0,0 +1,114 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from collections.abc import Iterable, Sequence +from collections.abc import Sequence as TypingSequence +from os import environ + +from grpc import ChannelCredentials, Compression, StatusCode + +from opentelemetry.exporter.otlp.pyproto.common._internal.trace_encoder import encode_spans +from opentelemetry.exporter.otlp.pyproto.grpc.exporter import ( + OTLPExporterMixin, + _get_credentials, + environ_to_compression, +) +from opentelemetry.metrics import MeterProvider +from opentelemetry.pyproto.collector.trace.v1.trace_service_pypb2 import ExportTraceServiceRequest +from opentelemetry.pyproto.collector.trace.v1.trace_service_pypb2_grpc import TraceServiceStub +from opentelemetry.sdk.environment_variables import ( + _OTEL_PYTHON_EXPORTER_OTLP_GRPC_TRACES_CREDENTIAL_PROVIDER, + OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE, + OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE, + OTEL_EXPORTER_OTLP_TRACES_CLIENT_KEY, + OTEL_EXPORTER_OTLP_TRACES_COMPRESSION, + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, + OTEL_EXPORTER_OTLP_TRACES_HEADERS, + OTEL_EXPORTER_OTLP_TRACES_INSECURE, + OTEL_EXPORTER_OTLP_TRACES_TIMEOUT, +) +from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult +from opentelemetry.semconv._incubating.attributes.otel_attributes import OtelComponentTypeValues + + +class OTLPSpanExporter( + SpanExporter, + OTLPExporterMixin[ + Sequence[ReadableSpan], + ExportTraceServiceRequest, + SpanExportResult, + TraceServiceStub, + ], +): + def __init__( + self, + endpoint: str | None = None, + insecure: bool | None = None, + credentials: ChannelCredentials | None = None, + headers: TypingSequence[tuple[str, str]] | dict[str, str] | str | None = None, + timeout: float | None = None, + compression: Compression | None = None, + channel_options: tuple[tuple[str, str]] | None = None, + retryable_error_codes: Iterable[StatusCode] | None = None, + *, + meter_provider: MeterProvider | None = None, + ): + insecure_spans = environ.get(OTEL_EXPORTER_OTLP_TRACES_INSECURE) + if insecure is None and insecure_spans is not None: + insecure = insecure_spans.lower() == "true" + + if not insecure and environ.get(OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE) is not None: + credentials = _get_credentials( + credentials, + _OTEL_PYTHON_EXPORTER_OTLP_GRPC_TRACES_CREDENTIAL_PROVIDER, + OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE, + OTEL_EXPORTER_OTLP_TRACES_CLIENT_KEY, + OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE, + ) + + environ_timeout = environ.get(OTEL_EXPORTER_OTLP_TRACES_TIMEOUT) + environ_timeout = float(environ_timeout) if environ_timeout is not None else None + + compression = ( + environ_to_compression(OTEL_EXPORTER_OTLP_TRACES_COMPRESSION) + if compression is None + else compression + ) + + OTLPExporterMixin.__init__( + self, + stub=TraceServiceStub, + result=SpanExportResult, + endpoint=endpoint or environ.get(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT), + insecure=insecure, + credentials=credentials, + headers=headers or environ.get(OTEL_EXPORTER_OTLP_TRACES_HEADERS), + timeout=timeout or environ_timeout, + compression=compression, + channel_options=channel_options, + retryable_error_codes=retryable_error_codes, + component_type=OtelComponentTypeValues.OTLP_GRPC_SPAN_EXPORTER, + signal="traces", + meter_provider=meter_provider, + ) + + def _translate_data(self, data: Sequence[ReadableSpan]) -> ExportTraceServiceRequest: + return encode_spans(data) + + def _count_data(self, data: Sequence[ReadableSpan]) -> int: + return len(data) + + def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: + return self._export(spans) + + def shutdown(self, timeout_millis: float = 30_000, **kwargs) -> None: + OTLPExporterMixin.shutdown(self, timeout_millis=timeout_millis) + + def force_flush(self, timeout_millis: int = 30000) -> bool: + """Nothing is buffered in this exporter, so this method does nothing.""" + return True + + @property + def _exporting(self): + return "traces" diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-grpc/src/opentelemetry/exporter/otlp/pyproto/grpc/version/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/src/opentelemetry/exporter/otlp/pyproto/grpc/version/__init__.py new file mode 100644 index 00000000000..b6eca6b1914 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/src/opentelemetry/exporter/otlp/pyproto/grpc/version/__init__.py @@ -0,0 +1,4 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +__version__ = "1.43.0.dev" diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/opentelemetry/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/opentelemetry/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/opentelemetry/exporter/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/opentelemetry/exporter/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/opentelemetry/exporter/otlp/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/opentelemetry/exporter/otlp/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/opentelemetry/exporter/otlp/pyproto/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/opentelemetry/exporter/otlp/pyproto/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/opentelemetry/exporter/otlp/pyproto/grpc/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/opentelemetry/exporter/otlp/pyproto/grpc/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/opentelemetry/exporter/otlp/pyproto/grpc/_log_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/opentelemetry/exporter/otlp/pyproto/grpc/_log_exporter/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/opentelemetry/exporter/otlp/pyproto/grpc/_log_exporter/test___init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/opentelemetry/exporter/otlp/pyproto/grpc/_log_exporter/test___init__.py new file mode 100644 index 00000000000..d90e3ed1296 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/opentelemetry/exporter/otlp/pyproto/grpc/_log_exporter/test___init__.py @@ -0,0 +1,136 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +# pylint: disable=protected-access + +import unittest +from unittest.mock import patch + +import grpc +from grpc import Compression, StatusCode + +from opentelemetry.exporter.otlp.pyproto.grpc._log_exporter import OTLPLogExporter +from opentelemetry.sdk._logs.export import LogRecordExportResult + +_INSECURE_CH = "opentelemetry.exporter.otlp.pyproto.grpc.exporter.insecure_channel" + + +class _FakeRpcError(grpc.RpcError): + def __init__(self, code, details=""): + self._code = code + self._details = details + + def code(self): + return self._code + + def details(self): + return self._details + + +def _make_exporter(**kwargs): + with patch(_INSECURE_CH): + exporter = OTLPLogExporter(insecure=True, **kwargs) + return exporter + + +class TestOTLPLogExporterConstructor(unittest.TestCase): + + def test_defaults(self): + with patch(_INSECURE_CH) as mock_ch: + exporter = OTLPLogExporter(insecure=True) + args, _ = mock_ch.call_args + self.assertEqual(args[0], "localhost:4317") + self.assertEqual(exporter._timeout, 10.0) + self.assertFalse(exporter._shutdown) + exporter.shutdown() + + def test_logs_endpoint_env_var(self): + with patch.dict("os.environ", { + "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT": "http://logs-host:4317", + }): + with patch(_INSECURE_CH) as mock_ch: + OTLPLogExporter(insecure=True) + args, _ = mock_ch.call_args + self.assertEqual(args[0], "logs-host:4317") + + def test_logs_endpoint_overrides_generic(self): + with patch.dict("os.environ", { + "OTEL_EXPORTER_OTLP_ENDPOINT": "http://generic:4317", + "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT": "http://logs:4317", + }): + with patch(_INSECURE_CH) as mock_ch: + OTLPLogExporter(insecure=True) + args, _ = mock_ch.call_args + self.assertEqual(args[0], "logs:4317") + + def test_logs_timeout_env_var(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_LOGS_TIMEOUT": "12"}): + exporter = _make_exporter() + self.assertEqual(exporter._timeout, 12.0) + exporter.shutdown() + + def test_logs_insecure_env_var_true(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_LOGS_INSECURE": "true"}): + with patch(_INSECURE_CH) as mock_insecure: + OTLPLogExporter() + mock_insecure.assert_called_once() + + def test_logs_headers_env_var(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_LOGS_HEADERS": "x-log=yes"}): + exporter = _make_exporter() + self.assertIn(("x-log", "yes"), exporter._headers) + exporter.shutdown() + + def test_logs_compression_env_var_gzip(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_LOGS_COMPRESSION": "gzip"}): + with patch(_INSECURE_CH) as mock_ch: + OTLPLogExporter(insecure=True) + _, kwargs = mock_ch.call_args + self.assertEqual(kwargs.get("compression"), Compression.Gzip) + + def test_arg_endpoint_overrides_env(self): + with patch.dict("os.environ", { + "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT": "http://env:4317", + }): + with patch(_INSECURE_CH) as mock_ch: + OTLPLogExporter(insecure=True, endpoint="http://arg:4317") + args, _ = mock_ch.call_args + self.assertEqual(args[0], "arg:4317") + + def test_exporting_property(self): + exporter = _make_exporter() + self.assertEqual(exporter._exporting, "logs") + exporter.shutdown() + + +class TestOTLPLogExporterExport(unittest.TestCase): + + def test_export_success(self): + exporter = _make_exporter() + result = exporter.export([]) + self.assertEqual(result, LogRecordExportResult.SUCCESS) + exporter.shutdown() + + def test_export_failure_non_retryable(self): + exporter = _make_exporter() + exporter._client.Export.side_effect = _FakeRpcError(StatusCode.PERMISSION_DENIED) + result = exporter.export([]) + self.assertEqual(result, LogRecordExportResult.FAILURE) + exporter.shutdown() + + def test_export_after_shutdown(self): + exporter = _make_exporter() + exporter.shutdown() + result = exporter.export([]) + self.assertEqual(result, LogRecordExportResult.FAILURE) + + def test_force_flush_returns_true(self): + exporter = _make_exporter() + self.assertTrue(exporter.force_flush()) + exporter.shutdown() + + def test_shutdown_sets_flag(self): + exporter = _make_exporter() + self.assertFalse(exporter._shutdown) + exporter.shutdown() + self.assertTrue(exporter._shutdown) diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/opentelemetry/exporter/otlp/pyproto/grpc/metric_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/opentelemetry/exporter/otlp/pyproto/grpc/metric_exporter/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/opentelemetry/exporter/otlp/pyproto/grpc/metric_exporter/test___init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/opentelemetry/exporter/otlp/pyproto/grpc/metric_exporter/test___init__.py new file mode 100644 index 00000000000..9aeb0b7b27e --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/opentelemetry/exporter/otlp/pyproto/grpc/metric_exporter/test___init__.py @@ -0,0 +1,224 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +# pylint: disable=protected-access + +import unittest +from unittest.mock import Mock, patch + +import grpc +from grpc import Compression, StatusCode + +from opentelemetry.exporter.otlp.pyproto.grpc.metric_exporter import OTLPMetricExporter +from opentelemetry.sdk.metrics.export import ( + AggregationTemporality, + MetricExportResult, + MetricsData, + NumberDataPoint, + ResourceMetrics, + ScopeMetrics, +) +from opentelemetry.sdk.metrics.export import Gauge as SDKGauge +from opentelemetry.sdk.metrics.export import Metric +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.util.instrumentation import InstrumentationScope + +_INSECURE_CH = "opentelemetry.exporter.otlp.pyproto.grpc.exporter.insecure_channel" + + +class _FakeRpcError(grpc.RpcError): + def __init__(self, code, details=""): + self._code = code + self._details = details + + def code(self): + return self._code + + def details(self): + return self._details + + +def _make_exporter(**kwargs): + with patch(_INSECURE_CH): + exporter = OTLPMetricExporter(insecure=True, **kwargs) + return exporter + + +def _make_metrics_data(n_data_points: int = 1) -> MetricsData: + data_points = [ + NumberDataPoint( + attributes={"i": i}, + start_time_unix_nano=0, + time_unix_nano=1641946016139533244, + value=float(i), + exemplars=[], + ) + for i in range(n_data_points) + ] + return MetricsData( + resource_metrics=[ + ResourceMetrics( + resource=Resource({"service.name": "test"}), + scope_metrics=[ + ScopeMetrics( + scope=InstrumentationScope("test", "1.0"), + metrics=[ + Metric( + name="my.gauge", + description="", + unit="1", + data=SDKGauge(data_points=data_points), + ) + ], + schema_url="", + ) + ], + schema_url="", + ) + ] + ) + + +class TestOTLPMetricExporterConstructor(unittest.TestCase): + + def test_defaults(self): + with patch(_INSECURE_CH) as mock_ch: + exporter = OTLPMetricExporter(insecure=True) + args, _ = mock_ch.call_args + self.assertEqual(args[0], "localhost:4317") + self.assertEqual(exporter._timeout, 10.0) + self.assertIsNone(exporter._max_export_batch_size) + exporter.shutdown() + + def test_metrics_endpoint_env_var(self): + with patch.dict("os.environ", { + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": "http://metrics-host:4317", + }): + with patch(_INSECURE_CH) as mock_ch: + OTLPMetricExporter(insecure=True) + args, _ = mock_ch.call_args + self.assertEqual(args[0], "metrics-host:4317") + + def test_metrics_endpoint_overrides_generic(self): + with patch.dict("os.environ", { + "OTEL_EXPORTER_OTLP_ENDPOINT": "http://generic:4317", + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": "http://metrics:4317", + }): + with patch(_INSECURE_CH) as mock_ch: + OTLPMetricExporter(insecure=True) + args, _ = mock_ch.call_args + self.assertEqual(args[0], "metrics:4317") + + def test_metrics_timeout_env_var(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_METRICS_TIMEOUT": "15"}): + exporter = _make_exporter() + self.assertEqual(exporter._timeout, 15.0) + exporter.shutdown() + + def test_metrics_insecure_env_var_true(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_METRICS_INSECURE": "true"}): + with patch(_INSECURE_CH) as mock_insecure: + OTLPMetricExporter() + mock_insecure.assert_called_once() + + def test_metrics_headers_env_var(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_METRICS_HEADERS": "x-metric=1"}): + exporter = _make_exporter() + self.assertIn(("x-metric", "1"), exporter._headers) + exporter.shutdown() + + def test_max_export_batch_size_stored(self): + exporter = _make_exporter(max_export_batch_size=100) + self.assertEqual(exporter._max_export_batch_size, 100) + exporter.shutdown() + + def test_exporting_property(self): + exporter = _make_exporter() + self.assertEqual(exporter._exporting, "metrics") + exporter.shutdown() + + +class TestOTLPMetricExporterExport(unittest.TestCase): + + def test_export_success(self): + exporter = _make_exporter() + result = exporter.export(_make_metrics_data()) + self.assertEqual(result, MetricExportResult.SUCCESS) + exporter.shutdown() + + def test_export_empty_metrics(self): + exporter = _make_exporter() + result = exporter.export(MetricsData(resource_metrics=[])) + self.assertEqual(result, MetricExportResult.SUCCESS) + exporter.shutdown() + + def test_export_failure_non_retryable(self): + exporter = _make_exporter() + exporter._client.Export.side_effect = _FakeRpcError(StatusCode.PERMISSION_DENIED) + result = exporter.export(_make_metrics_data()) + self.assertEqual(result, MetricExportResult.FAILURE) + exporter.shutdown() + + def test_export_after_shutdown(self): + exporter = _make_exporter() + exporter.shutdown() + result = exporter.export(_make_metrics_data()) + self.assertEqual(result, MetricExportResult.FAILURE) + + def test_force_flush_returns_true(self): + exporter = _make_exporter() + self.assertTrue(exporter.force_flush()) + exporter.shutdown() + + def test_set_meter_provider(self): + from opentelemetry.sdk.metrics import MeterProvider + exporter = _make_exporter() + mp = MeterProvider() + exporter.set_meter_provider(mp) # should not raise + exporter.shutdown() + + +class TestOTLPMetricExporterSplitBatch(unittest.TestCase): + """Tests for batch splitting via max_export_batch_size.""" + + def test_no_split_when_no_max_batch_size(self): + exporter = _make_exporter() + metrics_data = _make_metrics_data(n_data_points=5) + result = exporter.export(metrics_data) + self.assertEqual(result, MetricExportResult.SUCCESS) + # Single Export call (no splitting) + self.assertEqual(exporter._client.Export.call_count, 1) + exporter.shutdown() + + def test_split_into_multiple_batches(self): + # 5 data points, batch size 2 → 3 Export calls + exporter = _make_exporter(max_export_batch_size=2) + metrics_data = _make_metrics_data(n_data_points=5) + result = exporter.export(metrics_data) + self.assertEqual(result, MetricExportResult.SUCCESS) + self.assertEqual(exporter._client.Export.call_count, 3) + exporter.shutdown() + + def test_split_exact_batch_size(self): + # 4 data points, batch size 4 → 1 Export call + exporter = _make_exporter(max_export_batch_size=4) + metrics_data = _make_metrics_data(n_data_points=4) + result = exporter.export(metrics_data) + self.assertEqual(result, MetricExportResult.SUCCESS) + self.assertEqual(exporter._client.Export.call_count, 1) + exporter.shutdown() + + def test_split_partial_failure_returns_failure(self): + exporter = _make_exporter(max_export_batch_size=2) + metrics_data = _make_metrics_data(n_data_points=4) + call_count = [0] + + def fail_on_second(*args, **kwargs): + call_count[0] += 1 + if call_count[0] == 2: + raise _FakeRpcError(StatusCode.PERMISSION_DENIED) + + exporter._client.Export.side_effect = fail_on_second + result = exporter.export(metrics_data) + self.assertEqual(result, MetricExportResult.FAILURE) + exporter.shutdown() diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/opentelemetry/exporter/otlp/pyproto/grpc/test_exporter.py b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/opentelemetry/exporter/otlp/pyproto/grpc/test_exporter.py new file mode 100644 index 00000000000..78d00dc1590 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/opentelemetry/exporter/otlp/pyproto/grpc/test_exporter.py @@ -0,0 +1,285 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +# pylint: disable=protected-access + +import unittest +from collections.abc import Sequence +from unittest.mock import Mock, patch, call + +import grpc +from grpc import Compression, StatusCode + +from opentelemetry.exporter.otlp.pyproto.grpc.exporter import ( + InvalidCompressionValueException, + OTLPExporterMixin, + _RETRYABLE_ERROR_CODES, + environ_to_compression, +) +from opentelemetry.exporter.otlp.pyproto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.pyproto.collector.trace.v1.trace_service_pypb2 import ExportTraceServiceRequest +from opentelemetry.pyproto.collector.trace.v1.trace_service_pypb2_grpc import TraceServiceStub +from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult + +_INSECURE_CH = "opentelemetry.exporter.otlp.pyproto.grpc.exporter.insecure_channel" +_SECURE_CH = "opentelemetry.exporter.otlp.pyproto.grpc.exporter.secure_channel" +_SSL_CREDS = "opentelemetry.exporter.otlp.pyproto.grpc.exporter.ssl_channel_credentials" +_UNIFORM = "opentelemetry.exporter.otlp.pyproto.grpc.exporter.random.uniform" + + +class _FakeRpcError(grpc.RpcError): + def __init__(self, code, details=""): + self._code = code + self._details = details + + def code(self): + return self._code + + def details(self): + return self._details + + +def _make_exporter(**kwargs): + """Create OTLPSpanExporter with a mock insecure channel, returning (exporter, mock_channel).""" + with patch(_INSECURE_CH) as mock_ch: + mock_channel = Mock() + mock_ch.return_value = mock_channel + exporter = OTLPSpanExporter(insecure=True, **kwargs) + return exporter, mock_channel + + +class TestOTLPExporterMixinChannel(unittest.TestCase): + + def test_insecure_true_uses_insecure_channel(self): + with patch(_INSECURE_CH) as mock_insecure: + exporter = OTLPSpanExporter(insecure=True) + mock_insecure.assert_called_once() + exporter.shutdown() + + def test_http_scheme_implies_insecure(self): + with patch(_INSECURE_CH) as mock_insecure: + exporter = OTLPSpanExporter(endpoint="http://collector:4317") + mock_insecure.assert_called_once() + exporter.shutdown() + + def test_insecure_false_uses_secure_channel(self): + with patch(_SSL_CREDS, return_value=Mock(spec=grpc.ChannelCredentials)): + with patch(_SECURE_CH) as mock_secure: + exporter = OTLPSpanExporter(insecure=False) + mock_secure.assert_called_once() + exporter.shutdown() + + def test_https_scheme_implies_secure(self): + with patch(_SSL_CREDS, return_value=Mock(spec=grpc.ChannelCredentials)): + with patch(_SECURE_CH) as mock_secure: + exporter = OTLPSpanExporter(endpoint="https://collector:4317") + mock_secure.assert_called_once() + exporter.shutdown() + + def test_default_endpoint_is_localhost_4317(self): + with patch(_INSECURE_CH) as mock_insecure: + OTLPSpanExporter(insecure=True) + args, _ = mock_insecure.call_args + self.assertEqual(args[0], "localhost:4317") + + def test_endpoint_netloc_extracted(self): + with patch(_INSECURE_CH) as mock_insecure: + OTLPSpanExporter(insecure=True, endpoint="http://myhost:4317") + args, _ = mock_insecure.call_args + self.assertEqual(args[0], "myhost:4317") + + def test_generic_endpoint_env_var(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_ENDPOINT": "http://env-host:4317"}): + with patch(_INSECURE_CH) as mock_insecure: + OTLPSpanExporter(insecure=True) + args, _ = mock_insecure.call_args + self.assertEqual(args[0], "env-host:4317") + + def test_insecure_env_var_true(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_INSECURE": "true"}): + with patch(_INSECURE_CH) as mock_insecure: + OTLPSpanExporter() + mock_insecure.assert_called_once() + + def test_compression_gzip_passed_to_channel(self): + with patch(_INSECURE_CH) as mock_insecure: + OTLPSpanExporter(insecure=True, compression=Compression.Gzip) + _, kwargs = mock_insecure.call_args + self.assertEqual(kwargs.get("compression"), Compression.Gzip) + + +class TestOTLPExporterMixinHeaders(unittest.TestCase): + + def test_headers_dict_converted_to_tuple(self): + exporter, _ = _make_exporter(headers={"x-my-header": "val"}) + self.assertIn(("x-my-header", "val"), exporter._headers) + exporter.shutdown() + + def test_headers_str_parsed(self): + exporter, _ = _make_exporter(headers="x-token=abc,x-env=xyz") + self.assertIn(("x-token", "abc"), exporter._headers) + self.assertIn(("x-env", "xyz"), exporter._headers) + exporter.shutdown() + + def test_headers_env_var_parsed(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_HEADERS": "x-hdr=foo"}): + exporter, _ = _make_exporter() + self.assertIn(("x-hdr", "foo"), exporter._headers) + exporter.shutdown() + + def test_no_headers_gives_empty_tuple(self): + exporter, _ = _make_exporter() + self.assertEqual(exporter._headers, tuple()) + exporter.shutdown() + + +class TestOTLPExporterMixinTimeout(unittest.TestCase): + + def test_default_timeout(self): + exporter, _ = _make_exporter() + self.assertEqual(exporter._timeout, 10.0) + exporter.shutdown() + + def test_timeout_arg(self): + exporter, _ = _make_exporter(timeout=42) + self.assertEqual(exporter._timeout, 42.0) + exporter.shutdown() + + def test_timeout_env_var(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_TIMEOUT": "25"}): + exporter, _ = _make_exporter() + self.assertEqual(exporter._timeout, 25.0) + exporter.shutdown() + + +class TestOTLPExporterMixinRetryableCodes(unittest.TestCase): + + def test_default_retryable_codes(self): + exporter, _ = _make_exporter() + self.assertEqual(exporter._retryable_error_codes, _RETRYABLE_ERROR_CODES) + exporter.shutdown() + + def test_custom_codes_from_arg(self): + exporter, _ = _make_exporter(retryable_error_codes=[StatusCode.NOT_FOUND]) + self.assertIn(StatusCode.NOT_FOUND, exporter._retryable_error_codes) + self.assertNotIn(StatusCode.UNAVAILABLE, exporter._retryable_error_codes) + exporter.shutdown() + + def test_custom_codes_from_env_var(self): + env = {"OTEL_PYTHON_EXPORTER_OTLP_GRPC_RETRYABLE_ERROR_CODES": "NOT_FOUND,UNKNOWN"} + with patch.dict("os.environ", env): + exporter, _ = _make_exporter() + self.assertIn(StatusCode.NOT_FOUND, exporter._retryable_error_codes) + self.assertIn(StatusCode.UNKNOWN, exporter._retryable_error_codes) + exporter.shutdown() + + +class TestOTLPExporterMixinExport(unittest.TestCase): + + def _exporter_with_mock_client(self, **kwargs): + exporter, mock_channel = _make_exporter(**kwargs) + # TraceServiceStub.__init__ sets self.Export = channel.unary_unary(...) + # mock_channel.unary_unary(...) returns a Mock, so _client.Export is that Mock + return exporter, exporter._client.Export + + def test_export_success(self): + exporter, mock_export = self._exporter_with_mock_client() + result = exporter.export([]) + self.assertEqual(result, SpanExportResult.SUCCESS) + mock_export.assert_called_once() + exporter.shutdown() + + def test_export_after_shutdown_returns_failure(self): + exporter, _ = self._exporter_with_mock_client() + exporter.shutdown() + result = exporter.export([]) + self.assertEqual(result, SpanExportResult.FAILURE) + + def test_export_non_retryable_error_fails_immediately(self): + exporter, mock_export = self._exporter_with_mock_client() + mock_export.side_effect = _FakeRpcError(StatusCode.NOT_FOUND, "not found") + result = exporter.export([]) + self.assertEqual(result, SpanExportResult.FAILURE) + # Should only attempt once (non-retryable) + self.assertEqual(mock_export.call_count, 1) + exporter.shutdown() + + def test_export_retryable_then_deadline_exceeded(self): + # Large backoff immediately exceeds short timeout → exits after first attempt + exporter, mock_export = self._exporter_with_mock_client(timeout=0.01) + mock_export.side_effect = _FakeRpcError(StatusCode.UNAVAILABLE) + with patch(_UNIFORM, return_value=100.0): + with patch(_INSECURE_CH): # reinit channel on UNAVAILABLE + result = exporter.export([]) + self.assertEqual(result, SpanExportResult.FAILURE) + exporter.shutdown() + + def test_export_max_retries_exhausted(self): + # Use RESOURCE_EXHAUSTED — retryable but does not trigger channel reinit + exporter, mock_export = self._exporter_with_mock_client(timeout=999) + mock_export.side_effect = _FakeRpcError(StatusCode.RESOURCE_EXHAUSTED) + with patch(_UNIFORM, return_value=0.00001): + with patch.object(exporter._shutdown_in_progress, "wait", return_value=False): + result = exporter.export([]) + self.assertEqual(result, SpanExportResult.FAILURE) + exporter.shutdown() + + def test_shutdown_interrupts_retry(self): + # Use RESOURCE_EXHAUSTED — retryable but does not trigger channel reinit + exporter, mock_export = self._exporter_with_mock_client(timeout=999) + mock_export.side_effect = _FakeRpcError(StatusCode.RESOURCE_EXHAUSTED) + with patch(_UNIFORM, return_value=0.00001): + with patch.object(exporter._shutdown_in_progress, "wait", return_value=True): + result = exporter.export([]) + self.assertEqual(result, SpanExportResult.FAILURE) + exporter.shutdown() + + def test_unavailable_triggers_channel_reinit(self): + exporter, mock_export = self._exporter_with_mock_client(timeout=0.001) + mock_export.side_effect = _FakeRpcError(StatusCode.UNAVAILABLE) + with patch(_UNIFORM, return_value=100.0): + with patch(_INSECURE_CH) as mock_new_channel: + exporter.export([]) + # channel was reinitialized once on first UNAVAILABLE + mock_new_channel.assert_called_once() + exporter.shutdown() + + +class TestOTLPExporterMixinShutdown(unittest.TestCase): + + def test_shutdown_sets_flag_and_closes_channel(self): + exporter, mock_channel = _make_exporter() + self.assertFalse(exporter._shutdown) + exporter.shutdown() + self.assertTrue(exporter._shutdown) + mock_channel.close.assert_called_once() + + def test_shutdown_twice_logs_warning(self): + exporter, _ = _make_exporter() + exporter.shutdown() + with self.assertLogs(level="WARNING") as cm: + exporter.shutdown() + self.assertTrue(any("already shutdown" in msg for msg in cm.output)) + + def test_force_flush_returns_true(self): + exporter, _ = _make_exporter() + self.assertTrue(exporter.force_flush()) + exporter.shutdown() + + +class TestEnvironToCompression(unittest.TestCase): + + def test_gzip(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_COMPRESSION": "gzip"}): + result = environ_to_compression("OTEL_EXPORTER_OTLP_COMPRESSION") + self.assertEqual(result, Compression.Gzip) + + def test_missing_env_returns_none(self): + result = environ_to_compression("OTEL_EXPORTER_OTLP_COMPRESSION") + self.assertIsNone(result) + + def test_invalid_value_raises(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_COMPRESSION": "zstd"}): + with self.assertRaises(InvalidCompressionValueException): + environ_to_compression("OTEL_EXPORTER_OTLP_COMPRESSION") diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/opentelemetry/exporter/otlp/pyproto/grpc/trace_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/opentelemetry/exporter/otlp/pyproto/grpc/trace_exporter/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/opentelemetry/exporter/otlp/pyproto/grpc/trace_exporter/test___init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/opentelemetry/exporter/otlp/pyproto/grpc/trace_exporter/test___init__.py new file mode 100644 index 00000000000..39f7929ad18 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-grpc/tests/opentelemetry/exporter/otlp/pyproto/grpc/trace_exporter/test___init__.py @@ -0,0 +1,140 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +# pylint: disable=protected-access + +import unittest +from unittest.mock import Mock, patch + +import grpc +from grpc import Compression, StatusCode + +from opentelemetry.exporter.otlp.pyproto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.trace.export import SpanExportResult + +_INSECURE_CH = "opentelemetry.exporter.otlp.pyproto.grpc.exporter.insecure_channel" +_SSL_CREDS = "opentelemetry.exporter.otlp.pyproto.grpc.exporter.ssl_channel_credentials" + + +class _FakeRpcError(grpc.RpcError): + def __init__(self, code, details=""): + self._code = code + self._details = details + + def code(self): + return self._code + + def details(self): + return self._details + + +def _make_exporter(**kwargs): + with patch(_INSECURE_CH): + exporter = OTLPSpanExporter(insecure=True, **kwargs) + return exporter + + +class TestOTLPSpanExporterConstructor(unittest.TestCase): + + def test_defaults(self): + with patch(_INSECURE_CH) as mock_ch: + exporter = OTLPSpanExporter(insecure=True) + args, _ = mock_ch.call_args + self.assertEqual(args[0], "localhost:4317") + self.assertEqual(exporter._timeout, 10.0) + self.assertFalse(exporter._shutdown) + exporter.shutdown() + + def test_traces_endpoint_env_var(self): + with patch.dict("os.environ", { + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": "http://traces-host:4317", + }): + with patch(_INSECURE_CH) as mock_ch: + OTLPSpanExporter(insecure=True) + args, _ = mock_ch.call_args + self.assertEqual(args[0], "traces-host:4317") + + def test_traces_endpoint_overrides_generic(self): + with patch.dict("os.environ", { + "OTEL_EXPORTER_OTLP_ENDPOINT": "http://generic:4317", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": "http://traces:4317", + }): + with patch(_INSECURE_CH) as mock_ch: + OTLPSpanExporter(insecure=True) + args, _ = mock_ch.call_args + self.assertEqual(args[0], "traces:4317") + + def test_traces_timeout_env_var(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_TRACES_TIMEOUT": "7"}): + exporter = _make_exporter() + self.assertEqual(exporter._timeout, 7.0) + exporter.shutdown() + + def test_traces_timeout_overrides_generic(self): + with patch.dict("os.environ", { + "OTEL_EXPORTER_OTLP_TIMEOUT": "20", + "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT": "3", + }): + exporter = _make_exporter() + self.assertEqual(exporter._timeout, 3.0) + exporter.shutdown() + + def test_traces_insecure_env_var_true(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_TRACES_INSECURE": "true"}): + with patch(_INSECURE_CH) as mock_insecure: + OTLPSpanExporter() + mock_insecure.assert_called_once() + + def test_traces_compression_env_var_gzip(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_TRACES_COMPRESSION": "gzip"}): + with patch(_INSECURE_CH) as mock_ch: + OTLPSpanExporter(insecure=True) + _, kwargs = mock_ch.call_args + self.assertEqual(kwargs.get("compression"), Compression.Gzip) + + def test_traces_headers_env_var(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_TRACES_HEADERS": "x-trace=yes"}): + exporter = _make_exporter() + self.assertIn(("x-trace", "yes"), exporter._headers) + exporter.shutdown() + + def test_arg_endpoint_overrides_env(self): + with patch.dict("os.environ", { + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": "http://env:4317", + }): + with patch(_INSECURE_CH) as mock_ch: + OTLPSpanExporter(insecure=True, endpoint="http://arg:4317") + args, _ = mock_ch.call_args + self.assertEqual(args[0], "arg:4317") + + +class TestOTLPSpanExporterExport(unittest.TestCase): + + def test_export_success(self): + exporter = _make_exporter() + result = exporter.export([]) + self.assertEqual(result, SpanExportResult.SUCCESS) + exporter.shutdown() + + def test_export_failure_non_retryable(self): + exporter = _make_exporter() + exporter._client.Export.side_effect = _FakeRpcError(StatusCode.PERMISSION_DENIED) + result = exporter.export([]) + self.assertEqual(result, SpanExportResult.FAILURE) + exporter.shutdown() + + def test_export_after_shutdown(self): + exporter = _make_exporter() + exporter.shutdown() + result = exporter.export([]) + self.assertEqual(result, SpanExportResult.FAILURE) + + def test_force_flush_returns_true(self): + exporter = _make_exporter() + self.assertTrue(exporter.force_flush()) + exporter.shutdown() + + def test_exporting_property(self): + exporter = _make_exporter() + self.assertEqual(exporter._exporting, "traces") + exporter.shutdown() diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-http/README.rst b/exporter/opentelemetry-exporter-otlp-pyproto-http/README.rst new file mode 100644 index 00000000000..260fa4e0ec9 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-http/README.rst @@ -0,0 +1,4 @@ +OpenTelemetry OTLP Exporter (pure-Python protobuf, HTTP) +========================================================= + +HTTP transport for the pure-Python OTLP exporter stack. diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-http/pyproject.toml b/exporter/opentelemetry-exporter-otlp-pyproto-http/pyproject.toml new file mode 100644 index 00000000000..54777a25973 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-http/pyproject.toml @@ -0,0 +1,51 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "opentelemetry-exporter-otlp-pyproto-http" +dynamic = ["version"] +description = "OpenTelemetry Collector pure-Python protobuf over HTTP Exporter (no google.protobuf)" +readme = "README.rst" +license = "Apache-2.0" +requires-python = ">=3.10" +dependencies = [ + "opentelemetry-api ~= 1.12", + "opentelemetry-sdk ~= 1.12", + "opentelemetry-exporter-otlp-pyproto-common == 1.43.0.dev", + "requests ~= 2.7", +] + +[project.entry-points.opentelemetry_traces_exporter] +otlp_pyproto_http = "opentelemetry.exporter.otlp.pyproto.http.trace_exporter:OTLPSpanExporter" + +[project.entry-points.opentelemetry_metrics_exporter] +otlp_pyproto_http = "opentelemetry.exporter.otlp.pyproto.http.metric_exporter:OTLPMetricExporter" + +[project.entry-points.opentelemetry_logs_exporter] +otlp_pyproto_http = "opentelemetry.exporter.otlp.pyproto.http._log_exporter:OTLPLogExporter" + +[dependency-groups] +dev = [ + "opentelemetry-proto == 1.43.0.dev", + "opentelemetry-exporter-otlp-proto-common == 1.43.0.dev", + "opentelemetry-exporter-otlp-proto-http == 1.43.0.dev", + "opentelemetry-test-utils == 1.43.0.dev", + "protobuf>=5.0", + "pytest>=7.0", +] + +[tool.uv.sources] +opentelemetry-proto = { workspace = true } +opentelemetry-exporter-otlp-proto-common = { workspace = true } +opentelemetry-exporter-otlp-proto-http = { workspace = true } +opentelemetry-test-utils = { workspace = true } + +[tool.hatch.version] +path = "src/opentelemetry/exporter/otlp/pyproto/http/version/__init__.py" + +[tool.hatch.build.targets.sdist] +include = ["/src"] + +[tool.hatch.build.targets.wheel] +packages = ["src/opentelemetry"] diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-http/src/opentelemetry/exporter/otlp/pyproto/http/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-http/src/opentelemetry/exporter/otlp/pyproto/http/__init__.py new file mode 100644 index 00000000000..faf9c9e9b69 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-http/src/opentelemetry/exporter/otlp/pyproto/http/__init__.py @@ -0,0 +1,17 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from enum import Enum + +from .version import __version__ + +_OTLP_HTTP_HEADERS = { + "Content-Type": "application/x-protobuf", + "User-Agent": "OTel-OTLP-Exporter-Python/" + __version__, +} + + +class Compression(Enum): + NoCompression = "none" + Deflate = "deflate" + Gzip = "gzip" diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-http/src/opentelemetry/exporter/otlp/pyproto/http/_common/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-http/src/opentelemetry/exporter/otlp/pyproto/http/_common/__init__.py new file mode 100644 index 00000000000..bf772c5859b --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-http/src/opentelemetry/exporter/otlp/pyproto/http/_common/__init__.py @@ -0,0 +1,55 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from os import environ +from typing import Literal + +from requests import Response, Session + +from opentelemetry.sdk.environment_variables import ( + _OTEL_PYTHON_EXPORTER_OTLP_HTTP_CREDENTIAL_PROVIDER, +) +from opentelemetry.util._importlib_metadata import entry_points + + +def _is_retryable(resp: Response) -> bool: + if resp.status_code == 408: + return True + if 500 <= resp.status_code <= 599: + return True + return False + + +def _load_session_from_envvar( + cred_envvar: Literal[ + "OTEL_PYTHON_EXPORTER_OTLP_HTTP_LOGS_CREDENTIAL_PROVIDER", + "OTEL_PYTHON_EXPORTER_OTLP_HTTP_TRACES_CREDENTIAL_PROVIDER", + "OTEL_PYTHON_EXPORTER_OTLP_HTTP_METRICS_CREDENTIAL_PROVIDER", + ], +) -> Session | None: + _credential_env = environ.get( + _OTEL_PYTHON_EXPORTER_OTLP_HTTP_CREDENTIAL_PROVIDER + ) or environ.get(cred_envvar) + if _credential_env: + try: + maybe_session = next( + iter( + entry_points( + group="opentelemetry_otlp_credential_provider", + name=_credential_env, + ) + ) + ).load()() + except StopIteration: + raise RuntimeError( + f"Requested component '{_credential_env}' not found in " + f"entry point 'opentelemetry_otlp_credential_provider'" + ) + if isinstance(maybe_session, Session): + return maybe_session + else: + raise RuntimeError( + f"Requested component '{_credential_env}' is of type {type(maybe_session)}" + f" must be of type `Session`." + ) + return None diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-http/src/opentelemetry/exporter/otlp/pyproto/http/_log_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-http/src/opentelemetry/exporter/otlp/pyproto/http/_log_exporter/__init__.py new file mode 100644 index 00000000000..d7fa985d526 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-http/src/opentelemetry/exporter/otlp/pyproto/http/_log_exporter/__init__.py @@ -0,0 +1,274 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +import os +from collections.abc import Sequence +from gzip import GzipFile +from io import BytesIO +from logging import getLogger +from os import environ +from random import uniform +from threading import Event +from time import time +from urllib.parse import urlparse +from zlib import compress + +from requests import Session +from requests.exceptions import ConnectionError, RequestException + +from opentelemetry.exporter.otlp.pyproto.common._exporter_metrics import ( + create_exporter_metrics, +) +from opentelemetry.exporter.otlp.pyproto.common._internal._log_encoder import ( + encode_logs, +) +from opentelemetry.exporter.otlp.pyproto.http import ( + _OTLP_HTTP_HEADERS, + Compression, +) +from opentelemetry.exporter.otlp.pyproto.http._common import ( + _is_retryable, + _load_session_from_envvar, +) +from opentelemetry.metrics import MeterProvider +from opentelemetry.sdk._logs import ReadableLogRecord +from opentelemetry.sdk._logs.export import ( + LogRecordExporter, + LogRecordExportResult, +) +from opentelemetry.sdk._shared_internal import DuplicateFilter +from opentelemetry.sdk.environment_variables import ( + _OTEL_PYTHON_EXPORTER_OTLP_HTTP_LOGS_CREDENTIAL_PROVIDER, + OTEL_EXPORTER_OTLP_CERTIFICATE, + OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE, + OTEL_EXPORTER_OTLP_CLIENT_KEY, + OTEL_EXPORTER_OTLP_COMPRESSION, + OTEL_EXPORTER_OTLP_ENDPOINT, + OTEL_EXPORTER_OTLP_HEADERS, + OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE, + OTEL_EXPORTER_OTLP_LOGS_CLIENT_CERTIFICATE, + OTEL_EXPORTER_OTLP_LOGS_CLIENT_KEY, + OTEL_EXPORTER_OTLP_LOGS_COMPRESSION, + OTEL_EXPORTER_OTLP_LOGS_ENDPOINT, + OTEL_EXPORTER_OTLP_LOGS_HEADERS, + OTEL_EXPORTER_OTLP_LOGS_TIMEOUT, + OTEL_EXPORTER_OTLP_TIMEOUT, + OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED, +) +from opentelemetry.semconv._incubating.attributes.otel_attributes import ( + OtelComponentTypeValues, +) +from opentelemetry.semconv.attributes.http_attributes import ( + HTTP_RESPONSE_STATUS_CODE, +) +from opentelemetry.util.re import parse_env_headers + +_logger = getLogger(__name__) +_logger.addFilter(DuplicateFilter()) + +DEFAULT_ENDPOINT = "http://localhost:4318/" +DEFAULT_LOGS_EXPORT_PATH = "v1/logs" +DEFAULT_TIMEOUT = 10 +_MAX_RETRYS = 6 + + +class OTLPLogExporter(LogRecordExporter): + def __init__( + self, + endpoint: str | None = None, + certificate_file: str | None = None, + client_key_file: str | None = None, + client_certificate_file: str | None = None, + headers: dict[str, str] | None = None, + timeout: float | None = None, + compression: Compression | None = None, + session: Session | None = None, + *, + meter_provider: MeterProvider | None = None, + ): + self._shutdown_is_occuring = Event() + self._endpoint = endpoint or environ.get( + OTEL_EXPORTER_OTLP_LOGS_ENDPOINT, + _append_logs_path( + environ.get(OTEL_EXPORTER_OTLP_ENDPOINT, DEFAULT_ENDPOINT) + ), + ) + self._certificate_file = certificate_file or environ.get( + OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE, + environ.get(OTEL_EXPORTER_OTLP_CERTIFICATE, True), + ) + self._client_key_file = client_key_file or environ.get( + OTEL_EXPORTER_OTLP_LOGS_CLIENT_KEY, + environ.get(OTEL_EXPORTER_OTLP_CLIENT_KEY, None), + ) + self._client_certificate_file = client_certificate_file or environ.get( + OTEL_EXPORTER_OTLP_LOGS_CLIENT_CERTIFICATE, + environ.get(OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE, None), + ) + self._client_cert = ( + (self._client_certificate_file, self._client_key_file) + if self._client_certificate_file and self._client_key_file + else self._client_certificate_file + ) + headers_string = environ.get( + OTEL_EXPORTER_OTLP_LOGS_HEADERS, + environ.get(OTEL_EXPORTER_OTLP_HEADERS, ""), + ) + self._headers = headers or parse_env_headers(headers_string, liberal=True) + self._timeout = timeout or float( + environ.get( + OTEL_EXPORTER_OTLP_LOGS_TIMEOUT, + environ.get(OTEL_EXPORTER_OTLP_TIMEOUT, DEFAULT_TIMEOUT), + ) + ) + self._compression = compression or _compression_from_env() + self._session = ( + session + or _load_session_from_envvar( + _OTEL_PYTHON_EXPORTER_OTLP_HTTP_LOGS_CREDENTIAL_PROVIDER + ) + or Session() + ) + self._session.headers.update(self._headers) + self._session.headers.update(_OTLP_HTTP_HEADERS) + self._session.headers.update(self._headers) + if self._compression is not Compression.NoCompression: + self._session.headers.update( + {"Content-Encoding": self._compression.value} + ) + self._shutdown = False + + self._metrics = create_exporter_metrics( + OtelComponentTypeValues.OTLP_HTTP_LOG_EXPORTER, + "logs", + urlparse(self._endpoint), + meter_provider, + os.environ.get(OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED, "") + .strip() + .lower() + == "true", + ) + + def _export(self, serialized_data: bytes, timeout_sec: float | None = None): + data = serialized_data + if self._compression == Compression.Gzip: + gzip_data = BytesIO() + with GzipFile(fileobj=gzip_data, mode="w") as gzip_stream: + gzip_stream.write(serialized_data) + data = gzip_data.getvalue() + elif self._compression == Compression.Deflate: + data = compress(serialized_data) + if timeout_sec is None: + timeout_sec = self._timeout + try: + resp = self._session.post( + url=self._endpoint, + data=data, + verify=self._certificate_file, + timeout=timeout_sec, + cert=self._client_cert, + ) + except ConnectionError: + resp = self._session.post( + url=self._endpoint, + data=data, + verify=self._certificate_file, + timeout=timeout_sec, + cert=self._client_cert, + ) + return resp + + def export(self, batch: Sequence[ReadableLogRecord]) -> LogRecordExportResult: + if self._shutdown: + _logger.warning("Exporter already shutdown, ignoring batch") + return LogRecordExportResult.FAILURE + + with self._metrics.export_operation(len(batch)) as result: + serialized_data = encode_logs(batch).SerializeToString() + deadline_sec = time() + self._timeout + for retry_num in range(_MAX_RETRYS): + backoff_seconds = 2**retry_num * uniform(0.8, 1.2) + export_error: Exception | None = None + try: + resp = self._export(serialized_data, deadline_sec - time()) + if resp.ok: + return LogRecordExportResult.SUCCESS + except RequestException as error: + reason = error + export_error = error + retryable = isinstance(error, ConnectionError) + status_code = None + else: + reason = resp.reason + retryable = _is_retryable(resp) + status_code = resp.status_code + + if not retryable: + _logger.error( + "Failed to export logs batch code: %s, reason: %s", + status_code, + reason, + ) + error_attrs = ( + {HTTP_RESPONSE_STATUS_CODE: status_code} + if status_code is not None + else None + ) + result.error = export_error + result.error_attrs = error_attrs + return LogRecordExportResult.FAILURE + + if ( + retry_num + 1 == _MAX_RETRYS + or backoff_seconds > (deadline_sec - time()) + or self._shutdown + ): + _logger.error( + "Failed to export logs batch due to timeout, max retries or shutdown." + ) + error_attrs = ( + {HTTP_RESPONSE_STATUS_CODE: status_code} + if status_code is not None + else None + ) + result.error = export_error + result.error_attrs = error_attrs + return LogRecordExportResult.FAILURE + + _logger.warning( + "Transient error %s encountered while exporting logs batch, retrying in %.2fs.", + reason, + backoff_seconds, + ) + if self._shutdown_is_occuring.wait(backoff_seconds): + _logger.warning("Shutdown in progress, aborting retry.") + break + return LogRecordExportResult.FAILURE + + def force_flush(self, timeout_millis: float = 10_000) -> bool: + return True + + def shutdown(self): + if self._shutdown: + _logger.warning("Exporter already shutdown, ignoring call") + return + self._shutdown = True + self._shutdown_is_occuring.set() + self._session.close() + + +def _compression_from_env() -> Compression: + return Compression( + environ.get( + OTEL_EXPORTER_OTLP_LOGS_COMPRESSION, + environ.get(OTEL_EXPORTER_OTLP_COMPRESSION, "none"), + ) + .lower() + .strip() + ) + + +def _append_logs_path(endpoint: str) -> str: + if endpoint.endswith("/"): + return endpoint + DEFAULT_LOGS_EXPORT_PATH + return endpoint + f"/{DEFAULT_LOGS_EXPORT_PATH}" diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-http/src/opentelemetry/exporter/otlp/pyproto/http/metric_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-http/src/opentelemetry/exporter/otlp/pyproto/http/metric_exporter/__init__.py new file mode 100644 index 00000000000..3c1938729c7 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-http/src/opentelemetry/exporter/otlp/pyproto/http/metric_exporter/__init__.py @@ -0,0 +1,499 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +import os +from collections.abc import Iterable +from gzip import GzipFile +from io import BytesIO +from logging import getLogger +from os import environ +from random import uniform +from threading import Event +from time import time +from urllib.parse import urlparse +from zlib import compress + +from requests import Session +from requests.exceptions import ConnectionError, RequestException + +from opentelemetry.exporter.otlp.pyproto.common._exporter_metrics import ( + create_exporter_metrics, +) +from opentelemetry.exporter.otlp.pyproto.common._internal.metrics_encoder import ( + OTLPMetricExporterMixin, + encode_metrics, +) +from opentelemetry.exporter.otlp.pyproto.http import ( + _OTLP_HTTP_HEADERS, + Compression, +) +from opentelemetry.exporter.otlp.pyproto.http._common import ( + _is_retryable, + _load_session_from_envvar, +) +from opentelemetry.metrics import MeterProvider +from opentelemetry.pyproto.collector.metrics.v1.metrics_service_pypb2 import ( + ExportMetricsServiceRequest, +) +from opentelemetry.pyproto.metrics.v1.metrics_pypb2 import ( + ExponentialHistogram, + Gauge, + Histogram, + Metric, + ResourceMetrics, + ScopeMetrics, + Sum, + Summary, +) +from opentelemetry.sdk.environment_variables import ( + _OTEL_PYTHON_EXPORTER_OTLP_HTTP_METRICS_CREDENTIAL_PROVIDER, + OTEL_EXPORTER_OTLP_CERTIFICATE, + OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE, + OTEL_EXPORTER_OTLP_CLIENT_KEY, + OTEL_EXPORTER_OTLP_COMPRESSION, + OTEL_EXPORTER_OTLP_ENDPOINT, + OTEL_EXPORTER_OTLP_HEADERS, + OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE, + OTEL_EXPORTER_OTLP_METRICS_CLIENT_CERTIFICATE, + OTEL_EXPORTER_OTLP_METRICS_CLIENT_KEY, + OTEL_EXPORTER_OTLP_METRICS_COMPRESSION, + OTEL_EXPORTER_OTLP_METRICS_ENDPOINT, + OTEL_EXPORTER_OTLP_METRICS_HEADERS, + OTEL_EXPORTER_OTLP_METRICS_TIMEOUT, + OTEL_EXPORTER_OTLP_TIMEOUT, + OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED, +) +from opentelemetry.sdk.metrics._internal.aggregation import Aggregation +from opentelemetry.sdk.metrics.export import ( + AggregationTemporality, + MetricExporter, + MetricExportResult, + MetricsData, +) +from opentelemetry.semconv._incubating.attributes.otel_attributes import ( + OtelComponentTypeValues, +) +from opentelemetry.semconv.attributes.http_attributes import ( + HTTP_RESPONSE_STATUS_CODE, +) +from opentelemetry.util.re import parse_env_headers + +_logger = getLogger(__name__) + +DEFAULT_ENDPOINT = "http://localhost:4318/" +DEFAULT_METRICS_EXPORT_PATH = "v1/metrics" +DEFAULT_TIMEOUT = 10 +_MAX_RETRYS = 6 + + +class OTLPMetricExporter(MetricExporter, OTLPMetricExporterMixin): + def __init__( + self, + endpoint: str | None = None, + certificate_file: str | None = None, + client_key_file: str | None = None, + client_certificate_file: str | None = None, + headers: dict[str, str] | None = None, + timeout: float | None = None, + compression: Compression | None = None, + session: Session | None = None, + preferred_temporality: dict[type, AggregationTemporality] | None = None, + preferred_aggregation: dict[type, Aggregation] | None = None, + max_export_batch_size: int | None = None, + *, + meter_provider: MeterProvider | None = None, + ): + self._shutdown_in_progress = Event() + self._endpoint = endpoint or environ.get( + OTEL_EXPORTER_OTLP_METRICS_ENDPOINT, + _append_metrics_path( + environ.get(OTEL_EXPORTER_OTLP_ENDPOINT, DEFAULT_ENDPOINT) + ), + ) + self._certificate_file = certificate_file or environ.get( + OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE, + environ.get(OTEL_EXPORTER_OTLP_CERTIFICATE, True), + ) + self._client_key_file = client_key_file or environ.get( + OTEL_EXPORTER_OTLP_METRICS_CLIENT_KEY, + environ.get(OTEL_EXPORTER_OTLP_CLIENT_KEY, None), + ) + self._client_certificate_file = client_certificate_file or environ.get( + OTEL_EXPORTER_OTLP_METRICS_CLIENT_CERTIFICATE, + environ.get(OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE, None), + ) + self._client_cert = ( + (self._client_certificate_file, self._client_key_file) + if self._client_certificate_file and self._client_key_file + else self._client_certificate_file + ) + headers_string = environ.get( + OTEL_EXPORTER_OTLP_METRICS_HEADERS, + environ.get(OTEL_EXPORTER_OTLP_HEADERS, ""), + ) + self._headers = headers or parse_env_headers(headers_string, liberal=True) + self._timeout = timeout or float( + environ.get( + OTEL_EXPORTER_OTLP_METRICS_TIMEOUT, + environ.get(OTEL_EXPORTER_OTLP_TIMEOUT, DEFAULT_TIMEOUT), + ) + ) + self._compression = compression or _compression_from_env() + self._session = ( + session + or _load_session_from_envvar( + _OTEL_PYTHON_EXPORTER_OTLP_HTTP_METRICS_CREDENTIAL_PROVIDER + ) + or Session() + ) + self._session.headers.update(self._headers) + self._session.headers.update(_OTLP_HTTP_HEADERS) + self._session.headers.update(self._headers) + if self._compression is not Compression.NoCompression: + self._session.headers.update( + {"Content-Encoding": self._compression.value} + ) + self._common_configuration(preferred_temporality, preferred_aggregation) + self._max_export_batch_size = max_export_batch_size + self._shutdown = False + + self._metrics = create_exporter_metrics( + OtelComponentTypeValues.OTLP_HTTP_METRIC_EXPORTER, + "metrics", + urlparse(self._endpoint), + meter_provider, + os.environ.get(OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED, "") + .strip() + .lower() + == "true", + ) + + def _export(self, serialized_data: bytes, timeout_sec: float | None = None): + data = serialized_data + if self._compression == Compression.Gzip: + gzip_data = BytesIO() + with GzipFile(fileobj=gzip_data, mode="w") as gzip_stream: + gzip_stream.write(serialized_data) + data = gzip_data.getvalue() + elif self._compression == Compression.Deflate: + data = compress(serialized_data) + if timeout_sec is None: + timeout_sec = self._timeout + try: + resp = self._session.post( + url=self._endpoint, + data=data, + verify=self._certificate_file, + timeout=timeout_sec, + cert=self._client_cert, + ) + except ConnectionError: + resp = self._session.post( + url=self._endpoint, + data=data, + verify=self._certificate_file, + timeout=timeout_sec, + cert=self._client_cert, + ) + return resp + + def _export_with_retries( + self, + export_request: ExportMetricsServiceRequest, + deadline_sec: float, + num_items: int, + ) -> MetricExportResult: + with self._metrics.export_operation(num_items) as result: + serialized_data = export_request.SerializeToString() + for retry_num in range(_MAX_RETRYS): + backoff_seconds = 2**retry_num * uniform(0.8, 1.2) + export_error: Exception | None = None + try: + resp = self._export(serialized_data, deadline_sec - time()) + if resp.ok: + return MetricExportResult.SUCCESS + except RequestException as error: + reason = error + export_error = error + retryable = isinstance(error, ConnectionError) + status_code = None + else: + reason = resp.reason + retryable = _is_retryable(resp) + status_code = resp.status_code + + if not retryable: + _logger.error( + "Failed to export metrics batch code: %s, reason: %s", + status_code, + reason, + ) + error_attrs = ( + {HTTP_RESPONSE_STATUS_CODE: status_code} + if status_code is not None + else None + ) + result.error = export_error + result.error_attrs = error_attrs + return MetricExportResult.FAILURE + + if ( + retry_num + 1 == _MAX_RETRYS + or backoff_seconds > (deadline_sec - time()) + or self._shutdown + ): + _logger.error( + "Failed to export metrics batch due to timeout, max retries or shutdown." + ) + error_attrs = ( + {HTTP_RESPONSE_STATUS_CODE: status_code} + if status_code is not None + else None + ) + result.error = export_error + result.error_attrs = error_attrs + return MetricExportResult.FAILURE + + _logger.warning( + "Transient error %s encountered while exporting metrics batch, retrying in %.2fs.", + reason, + backoff_seconds, + ) + if self._shutdown_in_progress.wait(backoff_seconds): + _logger.warning("Shutdown in progress, aborting retry.") + break + return MetricExportResult.FAILURE + + def export( + self, + metrics_data: MetricsData, + timeout_millis: float | None = 10000, + **kwargs, + ) -> MetricExportResult: + if self._shutdown: + _logger.warning("Exporter already shutdown, ignoring batch") + return MetricExportResult.FAILURE + + num_items = 0 + for resource_metrics in metrics_data.resource_metrics: + for scope_metrics in resource_metrics.scope_metrics: + for metric in scope_metrics.metrics: + num_items += len(metric.data.data_points) + + export_request = encode_metrics(metrics_data) + deadline_sec = time() + self._timeout + + if self._max_export_batch_size is None: + return self._export_with_retries(export_request, deadline_sec, num_items) + + for split_request in _split_metrics_data(export_request, self._max_export_batch_size): + if self._export_with_retries(split_request, deadline_sec, num_items) != MetricExportResult.SUCCESS: + return MetricExportResult.FAILURE + + return MetricExportResult.SUCCESS + + def shutdown(self, timeout_millis: float = 30_000, **kwargs) -> None: + if self._shutdown: + _logger.warning("Exporter already shutdown, ignoring call") + return + self._shutdown = True + self._shutdown_in_progress.set() + self._session.close() + + def force_flush(self, timeout_millis: float = 10_000) -> bool: + return True + + def set_meter_provider(self, meter_provider: MeterProvider) -> None: + self._metrics = create_exporter_metrics( + OtelComponentTypeValues.OTLP_HTTP_METRIC_EXPORTER, + "metrics", + urlparse(self._endpoint), + meter_provider, + os.environ.get(OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED, "") + .strip() + .lower() + == "true", + ) + + @property + def _exporting(self) -> str: + return "metrics" + + +def _split_metrics_data( + metrics_data: ExportMetricsServiceRequest, + max_export_batch_size: int, +) -> Iterable[ExportMetricsServiceRequest]: + if not max_export_batch_size: + yield metrics_data + return + + batch_size = 0 + split_resource_metrics: list[dict] = [] + + for resource_metrics in metrics_data.resource_metrics: + split_scope_metrics: list[dict] = [] + split_resource_metrics.append({ + "resource": resource_metrics.resource, + "schema_url": resource_metrics.schema_url, + "scope_metrics": split_scope_metrics, + }) + + for scope_metrics in resource_metrics.scope_metrics: + split_metrics: list[dict] = [] + split_scope_metrics.append({ + "scope": scope_metrics.scope, + "schema_url": scope_metrics.schema_url, + "metrics": split_metrics, + }) + + for metric in scope_metrics.metrics: + split_data_points: list = [] + field_name = metric.WhichOneof("data") + if not field_name: + _logger.warning("Tried to split an unsupported metric type. Skipping.") + continue + + data_container = getattr(metric, field_name) + metric_dict: dict = { + "name": metric.name, + "description": metric.description, + "unit": metric.unit, + field_name: {"data_points": split_data_points}, + } + if hasattr(data_container, "aggregation_temporality"): + metric_dict[field_name]["aggregation_temporality"] = data_container.aggregation_temporality + if hasattr(data_container, "is_monotonic"): + metric_dict[field_name]["is_monotonic"] = data_container.is_monotonic + split_metrics.append(metric_dict) + + for data_point in data_container.data_points: + split_data_points.append(data_point) + batch_size += 1 + + if batch_size >= max_export_batch_size: + yield ExportMetricsServiceRequest( + resource_metrics=_build_resource_metrics(split_resource_metrics) + ) + batch_size = 0 + split_data_points = [] + + field_name = metric.WhichOneof("data") + if field_name is None: + _logger.warning("Tried to split an unsupported metric type. Skipping.") + continue + data_container = getattr(metric, field_name) + metric_dict = { + "name": metric.name, + "description": metric.description, + "unit": metric.unit, + field_name: {"data_points": split_data_points}, + } + if hasattr(data_container, "aggregation_temporality"): + metric_dict[field_name]["aggregation_temporality"] = data_container.aggregation_temporality + if hasattr(data_container, "is_monotonic"): + metric_dict[field_name]["is_monotonic"] = data_container.is_monotonic + + split_metrics = [metric_dict] + split_scope_metrics = [{ + "scope": scope_metrics.scope, + "schema_url": scope_metrics.schema_url, + "metrics": split_metrics, + }] + split_resource_metrics = [{ + "resource": resource_metrics.resource, + "schema_url": resource_metrics.schema_url, + "scope_metrics": split_scope_metrics, + }] + + if not split_data_points: + split_metrics.pop() + + if not split_metrics: + split_scope_metrics.pop() + + if not split_scope_metrics: + split_resource_metrics.pop() + + if batch_size > 0: + yield ExportMetricsServiceRequest( + resource_metrics=_build_resource_metrics(split_resource_metrics) + ) + + +def _build_resource_metrics(split_resource_metrics: list[dict]) -> list[ResourceMetrics]: + result = [] + for rm in split_resource_metrics: + scope_metrics_list = [] + for sm in rm.get("scope_metrics", []): + metrics_list = [] + for metric in sm.get("metrics", []): + new_metric = _build_metric(metric) + if new_metric is not None: + metrics_list.append(new_metric) + scope_metrics_list.append(ScopeMetrics( + scope=sm.get("scope"), + metrics=metrics_list, + schema_url=sm.get("schema_url") or "", + )) + result.append(ResourceMetrics( + resource=rm.get("resource"), + scope_metrics=scope_metrics_list, + schema_url=rm.get("schema_url") or "", + )) + return result + + +def _build_metric(metric: dict) -> Metric | None: + kwargs: dict = dict( + name=metric.get("name"), + description=metric.get("description"), + unit=metric.get("unit"), + ) + if "sum" in metric: + d = metric["sum"] + kwargs["sum"] = Sum( + data_points=list(d.get("data_points", [])), + aggregation_temporality=d.get("aggregation_temporality", 0), + is_monotonic=d.get("is_monotonic", False), + ) + elif "histogram" in metric: + d = metric["histogram"] + kwargs["histogram"] = Histogram( + data_points=list(d.get("data_points", [])), + aggregation_temporality=d.get("aggregation_temporality", 0), + ) + elif "exponential_histogram" in metric: + d = metric["exponential_histogram"] + kwargs["exponential_histogram"] = ExponentialHistogram( + data_points=list(d.get("data_points", [])), + aggregation_temporality=d.get("aggregation_temporality", 0), + ) + elif "gauge" in metric: + d = metric["gauge"] + kwargs["gauge"] = Gauge(data_points=list(d.get("data_points", []))) + elif "summary" in metric: + d = metric["summary"] + kwargs["summary"] = Summary(data_points=list(d.get("data_points", []))) + else: + _logger.warning("Tried to build an unsupported metric type. Skipping.") + return None + return Metric(**kwargs) + + +def _compression_from_env() -> Compression: + return Compression( + environ.get( + OTEL_EXPORTER_OTLP_METRICS_COMPRESSION, + environ.get(OTEL_EXPORTER_OTLP_COMPRESSION, "none"), + ) + .lower() + .strip() + ) + + +def _append_metrics_path(endpoint: str) -> str: + if endpoint.endswith("/"): + return endpoint + DEFAULT_METRICS_EXPORT_PATH + return endpoint + f"/{DEFAULT_METRICS_EXPORT_PATH}" diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-http/src/opentelemetry/exporter/otlp/pyproto/http/trace_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-http/src/opentelemetry/exporter/otlp/pyproto/http/trace_exporter/__init__.py new file mode 100644 index 00000000000..ed8988a078b --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-http/src/opentelemetry/exporter/otlp/pyproto/http/trace_exporter/__init__.py @@ -0,0 +1,269 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +import os +from collections.abc import Sequence +from gzip import GzipFile +from io import BytesIO +from logging import getLogger +from os import environ +from random import uniform +from threading import Event +from time import time +from urllib.parse import urlparse +from zlib import compress + +from requests import Session +from requests.exceptions import ConnectionError, RequestException + +from opentelemetry.exporter.otlp.pyproto.common._exporter_metrics import ( + create_exporter_metrics, +) +from opentelemetry.exporter.otlp.pyproto.common._internal.trace_encoder import ( + encode_spans, +) +from opentelemetry.exporter.otlp.pyproto.http import ( + _OTLP_HTTP_HEADERS, + Compression, +) +from opentelemetry.exporter.otlp.pyproto.http._common import ( + _is_retryable, + _load_session_from_envvar, +) +from opentelemetry.metrics import MeterProvider +from opentelemetry.sdk.environment_variables import ( + _OTEL_PYTHON_EXPORTER_OTLP_HTTP_TRACES_CREDENTIAL_PROVIDER, + OTEL_EXPORTER_OTLP_CERTIFICATE, + OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE, + OTEL_EXPORTER_OTLP_CLIENT_KEY, + OTEL_EXPORTER_OTLP_COMPRESSION, + OTEL_EXPORTER_OTLP_ENDPOINT, + OTEL_EXPORTER_OTLP_HEADERS, + OTEL_EXPORTER_OTLP_TIMEOUT, + OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE, + OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE, + OTEL_EXPORTER_OTLP_TRACES_CLIENT_KEY, + OTEL_EXPORTER_OTLP_TRACES_COMPRESSION, + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, + OTEL_EXPORTER_OTLP_TRACES_HEADERS, + OTEL_EXPORTER_OTLP_TRACES_TIMEOUT, + OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED, +) +from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult +from opentelemetry.semconv._incubating.attributes.otel_attributes import ( + OtelComponentTypeValues, +) +from opentelemetry.semconv.attributes.http_attributes import ( + HTTP_RESPONSE_STATUS_CODE, +) +from opentelemetry.util.re import parse_env_headers + +_logger = getLogger(__name__) + +DEFAULT_ENDPOINT = "http://localhost:4318/" +DEFAULT_TRACES_EXPORT_PATH = "v1/traces" +DEFAULT_TIMEOUT = 10 +_MAX_RETRYS = 6 + + +class OTLPSpanExporter(SpanExporter): + def __init__( + self, + endpoint: str | None = None, + certificate_file: str | None = None, + client_key_file: str | None = None, + client_certificate_file: str | None = None, + headers: dict[str, str] | None = None, + timeout: float | None = None, + compression: Compression | None = None, + session: Session | None = None, + *, + meter_provider: MeterProvider | None = None, + ): + self._shutdown_in_progress = Event() + self._endpoint = endpoint or environ.get( + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, + _append_trace_path( + environ.get(OTEL_EXPORTER_OTLP_ENDPOINT, DEFAULT_ENDPOINT) + ), + ) + self._certificate_file = certificate_file or environ.get( + OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE, + environ.get(OTEL_EXPORTER_OTLP_CERTIFICATE, True), + ) + self._client_key_file = client_key_file or environ.get( + OTEL_EXPORTER_OTLP_TRACES_CLIENT_KEY, + environ.get(OTEL_EXPORTER_OTLP_CLIENT_KEY, None), + ) + self._client_certificate_file = client_certificate_file or environ.get( + OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE, + environ.get(OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE, None), + ) + self._client_cert = ( + (self._client_certificate_file, self._client_key_file) + if self._client_certificate_file and self._client_key_file + else self._client_certificate_file + ) + headers_string = environ.get( + OTEL_EXPORTER_OTLP_TRACES_HEADERS, + environ.get(OTEL_EXPORTER_OTLP_HEADERS, ""), + ) + self._headers = headers or parse_env_headers(headers_string, liberal=True) + self._timeout = timeout or float( + environ.get( + OTEL_EXPORTER_OTLP_TRACES_TIMEOUT, + environ.get(OTEL_EXPORTER_OTLP_TIMEOUT, DEFAULT_TIMEOUT), + ) + ) + self._compression = compression or _compression_from_env() + self._session = ( + session + or _load_session_from_envvar( + _OTEL_PYTHON_EXPORTER_OTLP_HTTP_TRACES_CREDENTIAL_PROVIDER + ) + or Session() + ) + self._session.headers.update(self._headers) + self._session.headers.update(_OTLP_HTTP_HEADERS) + self._session.headers.update(self._headers) + if self._compression is not Compression.NoCompression: + self._session.headers.update( + {"Content-Encoding": self._compression.value} + ) + self._shutdown = False + + self._metrics = create_exporter_metrics( + OtelComponentTypeValues.OTLP_HTTP_SPAN_EXPORTER, + "traces", + urlparse(self._endpoint), + meter_provider, + os.environ.get(OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED, "") + .strip() + .lower() + == "true", + ) + + def _export(self, serialized_data: bytes, timeout_sec: float | None = None): + data = serialized_data + if self._compression == Compression.Gzip: + gzip_data = BytesIO() + with GzipFile(fileobj=gzip_data, mode="w") as gzip_stream: + gzip_stream.write(serialized_data) + data = gzip_data.getvalue() + elif self._compression == Compression.Deflate: + data = compress(serialized_data) + if timeout_sec is None: + timeout_sec = self._timeout + try: + resp = self._session.post( + url=self._endpoint, + data=data, + verify=self._certificate_file, + timeout=timeout_sec, + cert=self._client_cert, + ) + except ConnectionError: + resp = self._session.post( + url=self._endpoint, + data=data, + verify=self._certificate_file, + timeout=timeout_sec, + cert=self._client_cert, + ) + return resp + + def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: + if self._shutdown: + _logger.warning("Exporter already shutdown, ignoring batch") + return SpanExportResult.FAILURE + + with self._metrics.export_operation(len(spans)) as result: + serialized_data = encode_spans(spans).SerializeToString() + deadline_sec = time() + self._timeout + for retry_num in range(_MAX_RETRYS): + backoff_seconds = 2**retry_num * uniform(0.8, 1.2) + export_error: Exception | None = None + try: + resp = self._export(serialized_data, deadline_sec - time()) + if resp.ok: + return SpanExportResult.SUCCESS + except RequestException as error: + reason = error + export_error = error + retryable = isinstance(error, ConnectionError) + status_code = None + else: + reason = resp.reason + retryable = _is_retryable(resp) + status_code = resp.status_code + + if not retryable: + _logger.error( + "Failed to export span batch code: %s, reason: %s", + status_code, + reason, + ) + error_attrs = ( + {HTTP_RESPONSE_STATUS_CODE: status_code} + if status_code is not None + else None + ) + result.error = export_error + result.error_attrs = error_attrs + return SpanExportResult.FAILURE + + if ( + retry_num + 1 == _MAX_RETRYS + or backoff_seconds > (deadline_sec - time()) + or self._shutdown + ): + _logger.error( + "Failed to export span batch due to timeout, max retries or shutdown." + ) + error_attrs = ( + {HTTP_RESPONSE_STATUS_CODE: status_code} + if status_code is not None + else None + ) + result.error = export_error + result.error_attrs = error_attrs + return SpanExportResult.FAILURE + + _logger.warning( + "Transient error %s encountered while exporting span batch, retrying in %.2fs.", + reason, + backoff_seconds, + ) + if self._shutdown_in_progress.wait(backoff_seconds): + _logger.warning("Shutdown in progress, aborting retry.") + break + return SpanExportResult.FAILURE + + def shutdown(self): + if self._shutdown: + _logger.warning("Exporter already shutdown, ignoring call") + return + self._shutdown = True + self._shutdown_in_progress.set() + self._session.close() + + def force_flush(self, timeout_millis: int = 30000) -> bool: + return True + + +def _compression_from_env() -> Compression: + return Compression( + environ.get( + OTEL_EXPORTER_OTLP_TRACES_COMPRESSION, + environ.get(OTEL_EXPORTER_OTLP_COMPRESSION, "none"), + ) + .lower() + .strip() + ) + + +def _append_trace_path(endpoint: str) -> str: + if endpoint.endswith("/"): + return endpoint + DEFAULT_TRACES_EXPORT_PATH + return endpoint + f"/{DEFAULT_TRACES_EXPORT_PATH}" diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-http/src/opentelemetry/exporter/otlp/pyproto/http/version/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-http/src/opentelemetry/exporter/otlp/pyproto/http/version/__init__.py new file mode 100644 index 00000000000..697c4434b93 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-http/src/opentelemetry/exporter/otlp/pyproto/http/version/__init__.py @@ -0,0 +1 @@ +__version__ = "1.43.0.dev" diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/pyproto/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/pyproto/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/pyproto/http/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/pyproto/http/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/pyproto/http/_common/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/pyproto/http/_common/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/pyproto/http/_common/test___init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/pyproto/http/_common/test___init__.py new file mode 100644 index 00000000000..b0ae44d85b6 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/pyproto/http/_common/test___init__.py @@ -0,0 +1,87 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from unittest.mock import Mock, patch + +import pytest +from requests import Session + +from opentelemetry.exporter.otlp.pyproto.http._common import ( + _is_retryable, + _load_session_from_envvar, +) + + +# ── _is_retryable ───────────────────────────────────────────────────────────── + +def test_is_retryable_408(): + assert _is_retryable(Mock(status_code=408)) is True + + +def test_is_retryable_500(): + assert _is_retryable(Mock(status_code=500)) is True + + +def test_is_retryable_503(): + assert _is_retryable(Mock(status_code=503)) is True + + +def test_is_retryable_599(): + assert _is_retryable(Mock(status_code=599)) is True + + +def test_is_retryable_200(): + assert _is_retryable(Mock(status_code=200)) is False + + +def test_is_retryable_404(): + assert _is_retryable(Mock(status_code=404)) is False + + +def test_is_retryable_400(): + assert _is_retryable(Mock(status_code=400)) is False + + +# ── _load_session_from_envvar ───────────────────────────────────────────────── + +_CRED_ENVVAR = "OTEL_PYTHON_EXPORTER_OTLP_HTTP_TRACES_CREDENTIAL_PROVIDER" +_GENERIC_ENVVAR = "OTEL_PYTHON_EXPORTER_OTLP_HTTP_CREDENTIAL_PROVIDER" + + +def test_load_session_returns_none_when_no_env_var(): + with patch.dict("os.environ", {}, clear=True): + result = _load_session_from_envvar(_CRED_ENVVAR) + assert result is None + + +def test_load_session_raises_on_unknown_provider(): + with patch.dict("os.environ", {_GENERIC_ENVVAR: "nonexistent_provider"}): + with pytest.raises(RuntimeError, match="not found in entry point"): + _load_session_from_envvar(_CRED_ENVVAR) + + +def test_load_session_returns_session_from_provider(): + mock_session = Session() + mock_ep = Mock() + mock_ep.load.return_value = lambda: mock_session + + with patch.dict("os.environ", {_GENERIC_ENVVAR: "my_provider"}): + with patch( + "opentelemetry.exporter.otlp.pyproto.http._common.entry_points", + return_value=iter([mock_ep]), + ): + result = _load_session_from_envvar(_CRED_ENVVAR) + assert result is mock_session + + +def test_load_session_raises_if_provider_returns_non_session(): + mock_ep = Mock() + mock_ep.load.return_value = lambda: "not-a-session" + + with patch.dict("os.environ", {_GENERIC_ENVVAR: "bad_provider"}): + with patch( + "opentelemetry.exporter.otlp.pyproto.http._common.entry_points", + return_value=iter([mock_ep]), + ): + with pytest.raises(RuntimeError, match="must be of type"): + _load_session_from_envvar(_CRED_ENVVAR) diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/pyproto/http/_log_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/pyproto/http/_log_exporter/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/pyproto/http/_log_exporter/test___init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/pyproto/http/_log_exporter/test___init__.py new file mode 100644 index 00000000000..c95d16c188f --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/pyproto/http/_log_exporter/test___init__.py @@ -0,0 +1,228 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +# pylint: disable=protected-access + +import unittest +from unittest.mock import Mock, patch + +from requests import Session +from requests.exceptions import ConnectionError + +from opentelemetry.exporter.otlp.pyproto.http import Compression +from opentelemetry.exporter.otlp.pyproto.http._log_exporter import ( + DEFAULT_ENDPOINT, + DEFAULT_LOGS_EXPORT_PATH, + DEFAULT_TIMEOUT, + OTLPLogExporter, + _append_logs_path, +) +from opentelemetry.sdk._logs.export import LogRecordExportResult + +_UNIFORM = "opentelemetry.exporter.otlp.pyproto.http._log_exporter.uniform" + + +def _ok(): + return Mock(ok=True, status_code=200) + + +def _503(): + return Mock(ok=False, status_code=503, reason="Service Unavailable") + + +def _404(): + return Mock(ok=False, status_code=404, reason="Not Found") + + +class TestOTLPLogExporter(unittest.TestCase): + + # ── constructor ─────────────────────────────────────────────────────────── + + def test_constructor_defaults(self): + exporter = OTLPLogExporter() + self.assertEqual( + exporter._endpoint, + DEFAULT_ENDPOINT + DEFAULT_LOGS_EXPORT_PATH, + ) + self.assertEqual(exporter._timeout, DEFAULT_TIMEOUT) + self.assertEqual(exporter._compression, Compression.NoCompression) + self.assertIsInstance(exporter._session, Session) + self.assertEqual( + exporter._session.headers["Content-Type"], "application/x-protobuf" + ) + self.assertFalse(exporter._shutdown) + exporter.shutdown() + + def test_generic_endpoint_env_var(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_ENDPOINT": "http://collector:4318"}): + exporter = OTLPLogExporter() + self.assertEqual(exporter._endpoint, "http://collector:4318/v1/logs") + exporter.shutdown() + + def test_logs_endpoint_overrides_generic(self): + with patch.dict("os.environ", { + "OTEL_EXPORTER_OTLP_ENDPOINT": "http://generic:4318", + "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT": "http://logs:4318/v1/logs", + }): + exporter = OTLPLogExporter() + self.assertEqual(exporter._endpoint, "http://logs:4318/v1/logs") + exporter.shutdown() + + def test_constructor_arg_overrides_env(self): + with patch.dict("os.environ", { + "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT": "http://env:4318/v1/logs", + }): + exporter = OTLPLogExporter(endpoint="http://arg:4318/v1/logs") + self.assertEqual(exporter._endpoint, "http://arg:4318/v1/logs") + exporter.shutdown() + + def test_timeout_generic_env_var(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_TIMEOUT": "20"}): + exporter = OTLPLogExporter() + self.assertEqual(exporter._timeout, 20.0) + exporter.shutdown() + + def test_timeout_logs_env_var_overrides_generic(self): + with patch.dict("os.environ", { + "OTEL_EXPORTER_OTLP_TIMEOUT": "20", + "OTEL_EXPORTER_OTLP_LOGS_TIMEOUT": "5", + }): + exporter = OTLPLogExporter() + self.assertEqual(exporter._timeout, 5.0) + exporter.shutdown() + + def test_timeout_arg_overrides_env(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_LOGS_TIMEOUT": "99"}): + exporter = OTLPLogExporter(timeout=3) + self.assertEqual(exporter._timeout, 3) + exporter.shutdown() + + def test_compression_env_var_gzip(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_COMPRESSION": "gzip"}): + exporter = OTLPLogExporter() + self.assertEqual(exporter._compression, Compression.Gzip) + self.assertEqual(exporter._session.headers.get("Content-Encoding"), "gzip") + exporter.shutdown() + + def test_compression_env_var_deflate(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_COMPRESSION": "deflate"}): + exporter = OTLPLogExporter() + self.assertEqual(exporter._compression, Compression.Deflate) + self.assertEqual(exporter._session.headers.get("Content-Encoding"), "deflate") + exporter.shutdown() + + def test_compression_logs_env_overrides_generic(self): + with patch.dict("os.environ", { + "OTEL_EXPORTER_OTLP_COMPRESSION": "gzip", + "OTEL_EXPORTER_OTLP_LOGS_COMPRESSION": "deflate", + }): + exporter = OTLPLogExporter() + self.assertEqual(exporter._compression, Compression.Deflate) + exporter.shutdown() + + def test_compression_arg_gzip(self): + exporter = OTLPLogExporter(compression=Compression.Gzip) + self.assertEqual(exporter._compression, Compression.Gzip) + self.assertEqual(exporter._session.headers.get("Content-Encoding"), "gzip") + exporter.shutdown() + + def test_no_compression_header_for_none_compression(self): + exporter = OTLPLogExporter(compression=Compression.NoCompression) + self.assertNotIn("Content-Encoding", exporter._session.headers) + exporter.shutdown() + + def test_custom_session_used(self): + custom_session = Session() + exporter = OTLPLogExporter(session=custom_session) + self.assertIs(exporter._session, custom_session) + exporter.shutdown() + + # ── export ──────────────────────────────────────────────────────────────── + + def test_export_success(self): + exporter = OTLPLogExporter() + with patch.object(exporter._session, "post", return_value=_ok()): + result = exporter.export([]) + self.assertEqual(result, LogRecordExportResult.SUCCESS) + exporter.shutdown() + + def test_export_after_shutdown_returns_failure(self): + exporter = OTLPLogExporter() + exporter.shutdown() + result = exporter.export([]) + self.assertEqual(result, LogRecordExportResult.FAILURE) + + def test_export_non_retryable_failure(self): + exporter = OTLPLogExporter() + with patch.object(exporter._session, "post", return_value=_404()): + result = exporter.export([]) + self.assertEqual(result, LogRecordExportResult.FAILURE) + exporter.shutdown() + + def test_export_retry_then_deadline_exceeded(self): + exporter = OTLPLogExporter(timeout=0.01) + with patch.object(exporter._session, "post", return_value=_503()): + with patch(_UNIFORM, return_value=100.0): + result = exporter.export([]) + self.assertEqual(result, LogRecordExportResult.FAILURE) + exporter.shutdown() + + def test_export_max_retries_exhausted(self): + exporter = OTLPLogExporter(timeout=100) + with patch.object(exporter._session, "post", return_value=_503()): + with patch(_UNIFORM, return_value=0.0001): + with patch.object(exporter._shutdown_is_occuring, "wait", return_value=False): + result = exporter.export([]) + self.assertEqual(result, LogRecordExportResult.FAILURE) + exporter.shutdown() + + def test_shutdown_interrupts_retry(self): + exporter = OTLPLogExporter(timeout=100) + with patch.object(exporter._session, "post", return_value=_503()): + with patch(_UNIFORM, return_value=0.0001): + with patch.object(exporter._shutdown_is_occuring, "wait", return_value=True): + result = exporter.export([]) + self.assertEqual(result, LogRecordExportResult.FAILURE) + exporter.shutdown() + + def test_connection_error_is_retryable(self): + exporter = OTLPLogExporter(timeout=0.01) + with patch.object(exporter, "_export", side_effect=ConnectionError("refused")): + with patch(_UNIFORM, return_value=100.0): + result = exporter.export([]) + self.assertEqual(result, LogRecordExportResult.FAILURE) + exporter.shutdown() + + # ── shutdown ────────────────────────────────────────────────────────────── + + def test_shutdown_sets_flag(self): + exporter = OTLPLogExporter() + self.assertFalse(exporter._shutdown) + exporter.shutdown() + self.assertTrue(exporter._shutdown) + + def test_shutdown_twice_logs_warning(self): + exporter = OTLPLogExporter() + exporter.shutdown() + with self.assertLogs(level="WARNING") as cm: + exporter.shutdown() + self.assertTrue(any("already shutdown" in msg for msg in cm.output)) + + def test_force_flush_returns_true(self): + exporter = OTLPLogExporter() + self.assertTrue(exporter.force_flush()) + exporter.shutdown() + + +class TestAppendLogsPath(unittest.TestCase): + def test_with_trailing_slash(self): + self.assertEqual( + _append_logs_path("http://localhost:4318/"), + "http://localhost:4318/v1/logs", + ) + + def test_without_trailing_slash(self): + self.assertEqual( + _append_logs_path("http://localhost:4318"), + "http://localhost:4318/v1/logs", + ) diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/pyproto/http/metric_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/pyproto/http/metric_exporter/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/pyproto/http/metric_exporter/test___init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/pyproto/http/metric_exporter/test___init__.py new file mode 100644 index 00000000000..bec3dcefb14 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/pyproto/http/metric_exporter/test___init__.py @@ -0,0 +1,377 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +# pylint: disable=protected-access + +import unittest +from unittest.mock import Mock, patch + +from requests import Session +from requests.exceptions import ConnectionError + +from opentelemetry.exporter.otlp.pyproto.common._internal.metrics_encoder import ( + encode_metrics, +) +from opentelemetry.exporter.otlp.pyproto.http import Compression +from opentelemetry.exporter.otlp.pyproto.http.metric_exporter import ( + DEFAULT_ENDPOINT, + DEFAULT_METRICS_EXPORT_PATH, + DEFAULT_TIMEOUT, + OTLPMetricExporter, + _append_metrics_path, + _split_metrics_data, +) +from opentelemetry.sdk.metrics.export import ( + AggregationTemporality, + MetricExportResult, + MetricsData, + NumberDataPoint, + ResourceMetrics, + ScopeMetrics, +) +from opentelemetry.sdk.metrics.export import Gauge as SDKGauge +from opentelemetry.sdk.metrics.export import Metric, Sum as SDKSum +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.util.instrumentation import InstrumentationScope + +_UNIFORM = "opentelemetry.exporter.otlp.pyproto.http.metric_exporter.uniform" + + +def _ok(): + return Mock(ok=True, status_code=200) + + +def _503(): + return Mock(ok=False, status_code=503, reason="Service Unavailable") + + +def _404(): + return Mock(ok=False, status_code=404, reason="Not Found") + + +def _make_metrics_data(n_gauge_points: int = 1) -> MetricsData: + data_points = [ + NumberDataPoint( + attributes={"i": i}, + start_time_unix_nano=0, + time_unix_nano=1641946016139533244, + value=float(i), + exemplars=[], + ) + for i in range(n_gauge_points) + ] + return MetricsData( + resource_metrics=[ + ResourceMetrics( + resource=Resource({"service.name": "test"}), + scope_metrics=[ + ScopeMetrics( + scope=InstrumentationScope("test", "1.0"), + metrics=[ + Metric( + name="my.gauge", + description="", + unit="1", + data=SDKGauge(data_points=data_points), + ) + ], + schema_url="", + ) + ], + schema_url="", + ) + ] + ) + + +def _empty_metrics_data() -> MetricsData: + return MetricsData(resource_metrics=[]) + + +class TestOTLPMetricExporter(unittest.TestCase): + + # ── constructor ─────────────────────────────────────────────────────────── + + def test_constructor_defaults(self): + exporter = OTLPMetricExporter() + self.assertEqual( + exporter._endpoint, + DEFAULT_ENDPOINT + DEFAULT_METRICS_EXPORT_PATH, + ) + self.assertEqual(exporter._timeout, DEFAULT_TIMEOUT) + self.assertEqual(exporter._compression, Compression.NoCompression) + self.assertIsInstance(exporter._session, Session) + self.assertEqual( + exporter._session.headers["Content-Type"], "application/x-protobuf" + ) + self.assertIsNone(exporter._max_export_batch_size) + self.assertFalse(exporter._shutdown) + exporter.shutdown() + + def test_generic_endpoint_env_var(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_ENDPOINT": "http://collector:4318"}): + exporter = OTLPMetricExporter() + self.assertEqual(exporter._endpoint, "http://collector:4318/v1/metrics") + exporter.shutdown() + + def test_metrics_endpoint_overrides_generic(self): + with patch.dict("os.environ", { + "OTEL_EXPORTER_OTLP_ENDPOINT": "http://generic:4318", + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": "http://metrics:4318/v1/metrics", + }): + exporter = OTLPMetricExporter() + self.assertEqual(exporter._endpoint, "http://metrics:4318/v1/metrics") + exporter.shutdown() + + def test_constructor_arg_overrides_env(self): + with patch.dict("os.environ", { + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": "http://env:4318/v1/metrics", + }): + exporter = OTLPMetricExporter(endpoint="http://arg:4318/v1/metrics") + self.assertEqual(exporter._endpoint, "http://arg:4318/v1/metrics") + exporter.shutdown() + + def test_timeout_generic_env_var(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_TIMEOUT": "20"}): + exporter = OTLPMetricExporter() + self.assertEqual(exporter._timeout, 20.0) + exporter.shutdown() + + def test_timeout_metrics_env_var_overrides_generic(self): + with patch.dict("os.environ", { + "OTEL_EXPORTER_OTLP_TIMEOUT": "20", + "OTEL_EXPORTER_OTLP_METRICS_TIMEOUT": "5", + }): + exporter = OTLPMetricExporter() + self.assertEqual(exporter._timeout, 5.0) + exporter.shutdown() + + def test_compression_env_var_gzip(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_COMPRESSION": "gzip"}): + exporter = OTLPMetricExporter() + self.assertEqual(exporter._compression, Compression.Gzip) + self.assertEqual(exporter._session.headers.get("Content-Encoding"), "gzip") + exporter.shutdown() + + def test_compression_metrics_env_overrides_generic(self): + with patch.dict("os.environ", { + "OTEL_EXPORTER_OTLP_COMPRESSION": "gzip", + "OTEL_EXPORTER_OTLP_METRICS_COMPRESSION": "deflate", + }): + exporter = OTLPMetricExporter() + self.assertEqual(exporter._compression, Compression.Deflate) + exporter.shutdown() + + def test_max_export_batch_size_arg(self): + exporter = OTLPMetricExporter(max_export_batch_size=100) + self.assertEqual(exporter._max_export_batch_size, 100) + exporter.shutdown() + + def test_custom_session_used(self): + custom_session = Session() + exporter = OTLPMetricExporter(session=custom_session) + self.assertIs(exporter._session, custom_session) + exporter.shutdown() + + # ── export ──────────────────────────────────────────────────────────────── + + def test_export_success_empty_data(self): + exporter = OTLPMetricExporter() + with patch.object(exporter._session, "post", return_value=_ok()): + result = exporter.export(_empty_metrics_data()) + self.assertEqual(result, MetricExportResult.SUCCESS) + exporter.shutdown() + + def test_export_success_with_data(self): + exporter = OTLPMetricExporter() + with patch.object(exporter._session, "post", return_value=_ok()): + result = exporter.export(_make_metrics_data(2)) + self.assertEqual(result, MetricExportResult.SUCCESS) + exporter.shutdown() + + def test_export_after_shutdown_returns_failure(self): + exporter = OTLPMetricExporter() + exporter.shutdown() + result = exporter.export(_empty_metrics_data()) + self.assertEqual(result, MetricExportResult.FAILURE) + + def test_export_non_retryable_failure(self): + exporter = OTLPMetricExporter() + with patch.object(exporter._session, "post", return_value=_404()): + result = exporter.export(_empty_metrics_data()) + self.assertEqual(result, MetricExportResult.FAILURE) + exporter.shutdown() + + def test_export_retry_then_deadline_exceeded(self): + exporter = OTLPMetricExporter(timeout=0.01) + with patch.object(exporter._session, "post", return_value=_503()): + with patch(_UNIFORM, return_value=100.0): + result = exporter.export(_empty_metrics_data()) + self.assertEqual(result, MetricExportResult.FAILURE) + exporter.shutdown() + + def test_export_max_retries_exhausted(self): + exporter = OTLPMetricExporter(timeout=100) + with patch.object(exporter._session, "post", return_value=_503()): + with patch(_UNIFORM, return_value=0.0001): + with patch.object(exporter._shutdown_in_progress, "wait", return_value=False): + result = exporter.export(_empty_metrics_data()) + self.assertEqual(result, MetricExportResult.FAILURE) + exporter.shutdown() + + def test_shutdown_interrupts_retry(self): + exporter = OTLPMetricExporter(timeout=100) + with patch.object(exporter._session, "post", return_value=_503()): + with patch(_UNIFORM, return_value=0.0001): + with patch.object(exporter._shutdown_in_progress, "wait", return_value=True): + result = exporter.export(_empty_metrics_data()) + self.assertEqual(result, MetricExportResult.FAILURE) + exporter.shutdown() + + def test_export_with_batch_size_sends_multiple_requests(self): + # 3 data points with batch_size=1 → 3 separate HTTP requests + exporter = OTLPMetricExporter(max_export_batch_size=1) + call_count = 0 + + def post_side_effect(**_kwargs): + nonlocal call_count + call_count += 1 + return _ok() + + with patch.object(exporter._session, "post", side_effect=post_side_effect): + result = exporter.export(_make_metrics_data(3)) + self.assertEqual(result, MetricExportResult.SUCCESS) + self.assertEqual(call_count, 3) + exporter.shutdown() + + # ── shutdown ────────────────────────────────────────────────────────────── + + def test_shutdown_sets_flag(self): + exporter = OTLPMetricExporter() + self.assertFalse(exporter._shutdown) + exporter.shutdown() + self.assertTrue(exporter._shutdown) + + def test_shutdown_twice_logs_warning(self): + exporter = OTLPMetricExporter() + exporter.shutdown() + with self.assertLogs(level="WARNING") as cm: + exporter.shutdown() + self.assertTrue(any("already shutdown" in msg for msg in cm.output)) + + def test_force_flush_returns_true(self): + exporter = OTLPMetricExporter() + self.assertTrue(exporter.force_flush()) + exporter.shutdown() + + +class TestAppendMetricsPath(unittest.TestCase): + def test_with_trailing_slash(self): + self.assertEqual( + _append_metrics_path("http://localhost:4318/"), + "http://localhost:4318/v1/metrics", + ) + + def test_without_trailing_slash(self): + self.assertEqual( + _append_metrics_path("http://localhost:4318"), + "http://localhost:4318/v1/metrics", + ) + + +class TestSplitMetricsData(unittest.TestCase): + + def _make_request(self, n_points: int): + return encode_metrics(_make_metrics_data(n_points)) + + def test_split_no_batch_size_yields_unchanged(self): + # max_export_batch_size=0 (falsy) → yields data as-is + req = self._make_request(3) + batches = list(_split_metrics_data(req, 0)) + self.assertEqual(len(batches), 1) + self.assertIs(batches[0], req) + + def test_split_batch_larger_than_data_yields_one(self): + req = self._make_request(3) + batches = list(_split_metrics_data(req, 10)) + self.assertEqual(len(batches), 1) + total_points = sum( + len(sm.metrics[0].gauge.data_points) + for batch in batches + for rm in batch.resource_metrics + for sm in rm.scope_metrics + ) + self.assertEqual(total_points, 3) + + def test_split_batch_size_1_yields_one_per_point(self): + req = self._make_request(3) + batches = list(_split_metrics_data(req, 1)) + self.assertEqual(len(batches), 3) + for batch in batches: + total = sum( + len(sm.metrics[0].gauge.data_points) + for rm in batch.resource_metrics + for sm in rm.scope_metrics + ) + self.assertEqual(total, 1) + + def test_split_batch_size_2_yields_two_batches_for_four_points(self): + req = self._make_request(4) + batches = list(_split_metrics_data(req, 2)) + self.assertEqual(len(batches), 2) + + def test_split_empty_request_yields_nothing(self): + req = encode_metrics(_empty_metrics_data()) + batches = list(_split_metrics_data(req, 1)) + self.assertEqual(len(batches), 0) + + def test_split_preserves_sum_metric(self): + data = MetricsData( + resource_metrics=[ + ResourceMetrics( + resource=Resource({"service.name": "test"}), + scope_metrics=[ + ScopeMetrics( + scope=InstrumentationScope("test", "1.0"), + metrics=[ + Metric( + name="my.counter", + description="", + unit="1", + data=SDKSum( + data_points=[ + NumberDataPoint( + attributes={}, + start_time_unix_nano=0, + time_unix_nano=1641946016139533244, + value=1, + exemplars=[], + ), + NumberDataPoint( + attributes={"host": "b"}, + start_time_unix_nano=0, + time_unix_nano=1641946016139533244, + value=2, + exemplars=[], + ), + ], + aggregation_temporality=AggregationTemporality.CUMULATIVE, + is_monotonic=True, + ), + ) + ], + schema_url="", + ) + ], + schema_url="", + ) + ] + ) + req = encode_metrics(data) + batches = list(_split_metrics_data(req, 1)) + self.assertEqual(len(batches), 2) + for batch in batches: + sm = batch.resource_metrics[0].scope_metrics[0] + metric = sm.metrics[0] + self.assertEqual(metric.name, "my.counter") + self.assertEqual(len(metric.sum.data_points), 1) diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/pyproto/http/test___init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/pyproto/http/test___init__.py new file mode 100644 index 00000000000..7ffda58510c --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/pyproto/http/test___init__.py @@ -0,0 +1,25 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from opentelemetry.exporter.otlp.pyproto.http import Compression, _OTLP_HTTP_HEADERS +from opentelemetry.exporter.otlp.pyproto.http.version import __version__ + + +def test_compression_no_compression_value(): + assert Compression.NoCompression.value == "none" + + +def test_compression_deflate_value(): + assert Compression.Deflate.value == "deflate" + + +def test_compression_gzip_value(): + assert Compression.Gzip.value == "gzip" + + +def test_otlp_http_headers_content_type(): + assert _OTLP_HTTP_HEADERS["Content-Type"] == "application/x-protobuf" + + +def test_otlp_http_headers_user_agent(): + assert _OTLP_HTTP_HEADERS["User-Agent"] == f"OTel-OTLP-Exporter-Python/{__version__}" diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/pyproto/http/trace_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/pyproto/http/trace_exporter/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/pyproto/http/trace_exporter/test___init__.py b/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/pyproto/http/trace_exporter/test___init__.py new file mode 100644 index 00000000000..1f12847bb17 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-pyproto-http/tests/opentelemetry/exporter/otlp/pyproto/http/trace_exporter/test___init__.py @@ -0,0 +1,230 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +# pylint: disable=protected-access + +import unittest +from unittest.mock import Mock, patch + +from requests import Session +from requests.exceptions import ConnectionError + +from opentelemetry.exporter.otlp.pyproto.http import Compression +from opentelemetry.exporter.otlp.pyproto.http.trace_exporter import ( + DEFAULT_ENDPOINT, + DEFAULT_TIMEOUT, + DEFAULT_TRACES_EXPORT_PATH, + OTLPSpanExporter, + _append_trace_path, +) +from opentelemetry.sdk.trace.export import SpanExportResult + +_UNIFORM = "opentelemetry.exporter.otlp.pyproto.http.trace_exporter.uniform" + + +def _ok(): + return Mock(ok=True, status_code=200) + + +def _503(): + return Mock(ok=False, status_code=503, reason="Service Unavailable") + + +def _404(): + return Mock(ok=False, status_code=404, reason="Not Found") + + +class TestOTLPSpanExporter(unittest.TestCase): + + # ── constructor ─────────────────────────────────────────────────────────── + + def test_constructor_defaults(self): + exporter = OTLPSpanExporter() + self.assertEqual( + exporter._endpoint, + DEFAULT_ENDPOINT + DEFAULT_TRACES_EXPORT_PATH, + ) + self.assertEqual(exporter._timeout, DEFAULT_TIMEOUT) + self.assertEqual(exporter._compression, Compression.NoCompression) + self.assertIsInstance(exporter._session, Session) + self.assertEqual( + exporter._session.headers["Content-Type"], "application/x-protobuf" + ) + self.assertFalse(exporter._shutdown) + exporter.shutdown() + + def test_generic_endpoint_env_var(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_ENDPOINT": "http://collector:4318"}): + exporter = OTLPSpanExporter() + self.assertEqual(exporter._endpoint, "http://collector:4318/v1/traces") + exporter.shutdown() + + def test_traces_endpoint_overrides_generic(self): + with patch.dict("os.environ", { + "OTEL_EXPORTER_OTLP_ENDPOINT": "http://generic:4318", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": "http://traces:4318/v1/traces", + }): + exporter = OTLPSpanExporter() + self.assertEqual(exporter._endpoint, "http://traces:4318/v1/traces") + exporter.shutdown() + + def test_constructor_arg_overrides_env(self): + with patch.dict("os.environ", { + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": "http://env:4318/v1/traces", + }): + exporter = OTLPSpanExporter(endpoint="http://arg:4318/v1/traces") + self.assertEqual(exporter._endpoint, "http://arg:4318/v1/traces") + exporter.shutdown() + + def test_timeout_generic_env_var(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_TIMEOUT": "20"}): + exporter = OTLPSpanExporter() + self.assertEqual(exporter._timeout, 20.0) + exporter.shutdown() + + def test_timeout_traces_env_var_overrides_generic(self): + with patch.dict("os.environ", { + "OTEL_EXPORTER_OTLP_TIMEOUT": "20", + "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT": "5", + }): + exporter = OTLPSpanExporter() + self.assertEqual(exporter._timeout, 5.0) + exporter.shutdown() + + def test_timeout_arg_overrides_env(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_TRACES_TIMEOUT": "99"}): + exporter = OTLPSpanExporter(timeout=3) + self.assertEqual(exporter._timeout, 3) + exporter.shutdown() + + def test_compression_env_var_gzip(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_COMPRESSION": "gzip"}): + exporter = OTLPSpanExporter() + self.assertEqual(exporter._compression, Compression.Gzip) + self.assertEqual(exporter._session.headers.get("Content-Encoding"), "gzip") + exporter.shutdown() + + def test_compression_env_var_deflate(self): + with patch.dict("os.environ", {"OTEL_EXPORTER_OTLP_COMPRESSION": "deflate"}): + exporter = OTLPSpanExporter() + self.assertEqual(exporter._compression, Compression.Deflate) + self.assertEqual(exporter._session.headers.get("Content-Encoding"), "deflate") + exporter.shutdown() + + def test_compression_traces_env_overrides_generic(self): + with patch.dict("os.environ", { + "OTEL_EXPORTER_OTLP_COMPRESSION": "gzip", + "OTEL_EXPORTER_OTLP_TRACES_COMPRESSION": "deflate", + }): + exporter = OTLPSpanExporter() + self.assertEqual(exporter._compression, Compression.Deflate) + exporter.shutdown() + + def test_compression_arg_gzip(self): + exporter = OTLPSpanExporter(compression=Compression.Gzip) + self.assertEqual(exporter._compression, Compression.Gzip) + self.assertEqual(exporter._session.headers.get("Content-Encoding"), "gzip") + exporter.shutdown() + + def test_no_compression_header_for_none_compression(self): + exporter = OTLPSpanExporter(compression=Compression.NoCompression) + self.assertNotIn("Content-Encoding", exporter._session.headers) + exporter.shutdown() + + def test_custom_session_used(self): + custom_session = Session() + exporter = OTLPSpanExporter(session=custom_session) + self.assertIs(exporter._session, custom_session) + exporter.shutdown() + + # ── export ──────────────────────────────────────────────────────────────── + + def test_export_success(self): + exporter = OTLPSpanExporter() + with patch.object(exporter._session, "post", return_value=_ok()): + result = exporter.export([]) + self.assertEqual(result, SpanExportResult.SUCCESS) + exporter.shutdown() + + def test_export_after_shutdown_returns_failure(self): + exporter = OTLPSpanExporter() + exporter.shutdown() + result = exporter.export([]) + self.assertEqual(result, SpanExportResult.FAILURE) + + def test_export_non_retryable_failure(self): + exporter = OTLPSpanExporter() + with patch.object(exporter._session, "post", return_value=_404()): + result = exporter.export([]) + self.assertEqual(result, SpanExportResult.FAILURE) + exporter.shutdown() + + def test_export_retry_then_deadline_exceeded(self): + # backoff(100) > remaining timeout(0.01) → exits on first retry + exporter = OTLPSpanExporter(timeout=0.01) + with patch.object(exporter._session, "post", return_value=_503()): + with patch(_UNIFORM, return_value=100.0): + result = exporter.export([]) + self.assertEqual(result, SpanExportResult.FAILURE) + exporter.shutdown() + + def test_export_max_retries_exhausted(self): + exporter = OTLPSpanExporter(timeout=100) + with patch.object(exporter._session, "post", return_value=_503()): + with patch(_UNIFORM, return_value=0.0001): + with patch.object(exporter._shutdown_in_progress, "wait", return_value=False): + result = exporter.export([]) + self.assertEqual(result, SpanExportResult.FAILURE) + exporter.shutdown() + + def test_shutdown_interrupts_retry(self): + exporter = OTLPSpanExporter(timeout=100) + with patch.object(exporter._session, "post", return_value=_503()): + with patch(_UNIFORM, return_value=0.0001): + with patch.object(exporter._shutdown_in_progress, "wait", return_value=True): + result = exporter.export([]) + self.assertEqual(result, SpanExportResult.FAILURE) + exporter.shutdown() + + def test_connection_error_is_retryable(self): + # ConnectionError from _export goes into the retry path; deadline kills it fast. + exporter = OTLPSpanExporter(timeout=0.01) + with patch.object(exporter, "_export", side_effect=ConnectionError("refused")): + with patch(_UNIFORM, return_value=100.0): + result = exporter.export([]) + self.assertEqual(result, SpanExportResult.FAILURE) + exporter.shutdown() + + # ── shutdown ────────────────────────────────────────────────────────────── + + def test_shutdown_sets_flag(self): + exporter = OTLPSpanExporter() + self.assertFalse(exporter._shutdown) + exporter.shutdown() + self.assertTrue(exporter._shutdown) + + def test_shutdown_twice_logs_warning(self): + exporter = OTLPSpanExporter() + exporter.shutdown() + with self.assertLogs(level="WARNING") as cm: + exporter.shutdown() + self.assertTrue(any("already shutdown" in msg for msg in cm.output)) + + def test_force_flush_returns_true(self): + exporter = OTLPSpanExporter() + self.assertTrue(exporter.force_flush()) + exporter.shutdown() + + +class TestAppendTracePath(unittest.TestCase): + def test_with_trailing_slash(self): + self.assertEqual( + _append_trace_path("http://localhost:4318/"), + "http://localhost:4318/v1/traces", + ) + + def test_without_trailing_slash(self): + self.assertEqual( + _append_trace_path("http://localhost:4318"), + "http://localhost:4318/v1/traces", + ) diff --git a/opentelemetry-api/src/opentelemetry/propagators/composite.py b/opentelemetry-api/src/opentelemetry/propagators/composite.py index 1d3c3912ee1..b3027189f69 100644 --- a/opentelemetry-api/src/opentelemetry/propagators/composite.py +++ b/opentelemetry-api/src/opentelemetry/propagators/composite.py @@ -39,7 +39,7 @@ def extract( """ for propagator in self._propagators: context = propagator.extract(carrier, context, getter=getter) - return context # type: ignore + return context if context is not None else Context() def inject( self, diff --git a/opentelemetry-pyproto/AGENTS.md b/opentelemetry-pyproto/AGENTS.md new file mode 100644 index 00000000000..14723974f49 --- /dev/null +++ b/opentelemetry-pyproto/AGENTS.md @@ -0,0 +1,25 @@ +# Coding Conventions + +## Imports + +Always use `from x import y`, never `import x`: + +```python +# correct +from struct import pack +from logging import getLogger +from collections.abc import Sequence + +# wrong +import struct +import logging +import collections +``` + +The only exception is when the module itself must be the reference — for example, when two names from different modules would collide and aliasing would obscure meaning. That situation is rare; prefer renaming the local variable instead. + +## Commit cadence + +Make a separate git commit after every discrete change. Do not batch unrelated +changes into a single commit. Each commit should be self-contained and leave +the repository in a working state. diff --git a/opentelemetry-pyproto/README.rst b/opentelemetry-pyproto/README.rst new file mode 100644 index 00000000000..9d14bbf75e9 --- /dev/null +++ b/opentelemetry-pyproto/README.rst @@ -0,0 +1,4 @@ +OpenTelemetry Python Proto (pure-Python) +========================================= + +Pure-Python protobuf message classes for OpenTelemetry, with no ``google.protobuf`` dependency. diff --git a/opentelemetry-pyproto/pyproject.toml b/opentelemetry-pyproto/pyproject.toml new file mode 100644 index 00000000000..2a43ef8237d --- /dev/null +++ b/opentelemetry-pyproto/pyproject.toml @@ -0,0 +1,35 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "opentelemetry-pyproto" +dynamic = ["version"] +description = "OpenTelemetry Python Proto — pure-Python protobuf message classes (no google.protobuf)" +readme = "README.rst" +license = "Apache-2.0" +requires-python = ">=3.10" +dependencies = [] + +[tool.hatch.version] +path = "src/opentelemetry/pyproto/version/__init__.py" + +[tool.hatch.build.targets.sdist] +include = ["/src"] + +[tool.hatch.build.targets.wheel] +packages = ["src/opentelemetry"] + +[dependency-groups] +dev = [ + "opentelemetry-proto == 1.43.0.dev", + "protobuf>=5.0", + "pytest>=7.0", + "pytest-benchmark>=4", +] + +[tool.uv.sources] +opentelemetry-proto = { workspace = true } + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/__init__.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/_pyprotobuf/__init__.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/_pyprotobuf/__init__.py new file mode 100644 index 00000000000..9f186998550 --- /dev/null +++ b/opentelemetry-pyproto/src/opentelemetry/pyproto/_pyprotobuf/__init__.py @@ -0,0 +1,39 @@ +from .enum import encode_enum +from .scalars import ( + encode_bool, + encode_bytes, + encode_double, + encode_fixed32, + encode_fixed64, + encode_float, + encode_int, + encode_sfixed32, + encode_sfixed64, + encode_sint32, + encode_sint64, + encode_string, + encode_uint32, + encode_uint64, +) +from .tag import encode_tag +from .varint import encode_varint + +__all__ = [ + "encode_bool", + "encode_bytes", + "encode_double", + "encode_enum", + "encode_fixed32", + "encode_fixed64", + "encode_float", + "encode_int", + "encode_sfixed32", + "encode_sfixed64", + "encode_sint32", + "encode_sint64", + "encode_string", + "encode_tag", + "encode_uint32", + "encode_uint64", + "encode_varint", +] diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/_pyprotobuf/enum.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/_pyprotobuf/enum.py new file mode 100644 index 00000000000..229b44787b5 --- /dev/null +++ b/opentelemetry-pyproto/src/opentelemetry/pyproto/_pyprotobuf/enum.py @@ -0,0 +1,53 @@ +"""Encoder for protobuf enum field values. + +Protobuf enum fields use wire type 0 (varint) and share the exact same wire +encoding as int32. This module exists as its own file — separate from +_scalars.py — because an enum is not a scalar type. Enum fields carry named +constants, not raw numeric values. The distinction matters at the .proto +language level even though the wire encoding is identical to int32. + +Reference: + https://protobuf.dev/programming-guides/encoding/ +""" + +from .scalars import encode_int + + +def encode_enum(value: int) -> bytes: + """Encode a protobuf enum value. + + Protobuf enum fields use wire type 0 (varint) and share the exact same + wire encoding as int32. The encoding guide states: + + Enum values are always 32-bit integers and the encoding follows the + same rules as int32. + + Reference: + https://protobuf.dev/programming-guides/encoding/ + + Enum values in practice + ----------------------- + In proto3, enum values defined in a .proto file are non-negative named + constants (0, 1, 2, …). The first value of any proto3 enum must be 0. + + In proto2, negative enum values are legal. Negative enum values can also + appear in proto3 messages received from a proto2 sender: the decoder + preserves them as their raw integer value. In both cases the int32 wire + encoding applies, meaning negative enum values produce a 10-byte varint + (via 64-bit sign extension, exactly as encode_int does for negative ints). + + This function is a thin wrapper around encode_int. Its purpose is + readability at call sites: code that encodes an enum field can call + encode_enum to make the intent explicit, rather than encode_int, which + suggests an arbitrary integer. + """ + # Delegate entirely to encode_int, which implements the int32 wire rule: + # + # - Non-negative values → encode_varint directly (compact, 1–5 bytes). + # - Negative values → mask to 64-bit unsigned two's complement, then + # encode_varint (always 10 bytes). + # + # No additional logic is needed: the spec defines enum encoding as + # identical to int32, so encode_int is the correct and complete + # implementation of both. + return encode_int(value) diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/_pyprotobuf/fields.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/_pyprotobuf/fields.py new file mode 100644 index 00000000000..d6bac097234 --- /dev/null +++ b/opentelemetry-pyproto/src/opentelemetry/pyproto/_pyprotobuf/fields.py @@ -0,0 +1,542 @@ +"""Proto3 field-level encoding helpers for SerializeToString() implementations. + +Why this module exists +---------------------- +The other modules in this package (varint, tag, scalars, enum) provide the +raw encoding kernel: functions that take a Python value and return its bytes +representation in the protobuf wire format. Those functions know nothing about +proto3 messages. They do not know about field numbers, wire-type tags, or the +rule that says a field whose value equals the proto3 default must be omitted +from the serialised output. + +This module sits one level above that kernel. It provides one helper per +proto3 scalar category (uint64, sint32, double, string, bytes, bool, fixed32, +fixed64) plus helpers for embedded messages and packed repeated fields. Each +helper combines three operations that every SerializeToString() method must +perform for every field it writes: + + 1. Apply the proto3 default-omission rule — if the value equals the type's + default (0, 0.0, False, b"", ""), return b"" immediately. + 2. Encode the tag — (field_number << 3) | wire_type, then varint-encoded. + 3. Encode the value — using the appropriate primitive from this package. + +Without these helpers, every SerializeToString() method would repeat that +three-step pattern inline for every field, cluttering the message classes with +low-level wire-format details and making the default-omission logic easy to get +wrong or forget. + +Why it is named "fields" +------------------------ +Each function in this module encodes one proto3 field: it takes a field number +and a value, and returns the complete on-wire bytes for that field — tag plus +encoded value, or b"" when the value is the proto3 default. The word "field" +is the right abstraction because a proto3 field is exactly this: a field number, +a wire type, a default-omission rule, and a value encoding. The other modules +in this package encode raw values; this module encodes fields. + +Layering within _pyprotobuf +---------------------------- + varint.py — encode a varint integer to bytes + tag.py — encode a (field_number, wire_type) tag using varint + scalars.py — encode all proto3 scalar types to bytes + enum.py — encode a proto3 enum value (thin wrapper over scalars) + fields.py — encode a complete proto3 field (tag + default check + value) + +The message classes (common_pypb2.py, metrics_pypb2.py, etc.) use this +module directly. They import the encode_* primitives from the package root +(__init__.py) and the field helpers from here. + +Wire-type constants +------------------- +The protobuf specification defines six wire types. The wire type is stored in +the three lowest bits of every tag integer. Only four wire types appear in +proto3 messages (wire types 3 and 4, the deprecated group delimiters, do not +appear in new proto3 code): + + WT_VARINT = 0 — one or more bytes with a continuation bit in the MSB of + each byte. Used for: int32, int64, uint32, uint64, sint32, + sint64, bool, enum. + WT_64BIT = 1 — exactly 8 bytes, little-endian. Used for: double, + fixed64, sfixed64. + WT_LEN = 2 — a varint length prefix followed by that many bytes. Used + for: string, bytes, embedded messages, packed repeated + fields. + WT_32BIT = 5 — exactly 4 bytes, little-endian. Used for: float, fixed32, + sfixed32. + +Reference: https://protobuf.dev/programming-guides/encoding/ +""" + +from __future__ import annotations + +from struct import pack + +from .scalars import encode_fixed32, encode_fixed64, encode_sint32 +from .tag import encode_tag +from .varint import encode_varint + +WT_VARINT = 0 # int32, int64, uint32, uint64, bool, enum +WT_64BIT = 1 # double, fixed64, sfixed64 +WT_LEN = 2 # string, bytes, embedded messages, packed arrays +WT_32BIT = 5 # float, fixed32, sfixed32 + + +def msg(field: int, content: bytes) -> bytes: + """Encode an embedded message or group as a length-delimited field. + + In the protobuf wire format, an embedded message is a length-delimited + field (wire type 2). The layout on the wire is: + + tag (varint) | byte_length (varint) | content (bytes) + + The tag encodes the field number and wire type 2. The byte_length is the + number of bytes in the already-serialised sub-message. The content is the + verbatim output of the sub-message's own SerializeToString() call. + + Unlike the scalar helpers, msg does not apply an omission rule for empty + content. An embedded message with no fields set serialises to b"" (zero + bytes), and the caller decides whether to write it at all. In practice, + message fields are either written unconditionally when present or guarded + by an explicit ``if self.field is not None`` check in the caller; the empty + check is the caller's responsibility, not this helper's. + + This helper is also used for oneof sub-messages, repeated message fields + (called once per element), and any other length-delimited payload. + + Parameters + ---------- + field: + The field number as declared in the .proto schema. + content: + The serialised bytes of the sub-message (the return value of + sub_message.SerializeToString()). + + Returns + ------- + bytes + tag + varint(len(content)) + content + """ + return encode_tag(field, WT_LEN) + encode_varint(len(content)) + content + + +def string(field: int, value: str) -> bytes: + """Encode a proto3 string field, omitting it when the value is empty. + + Proto3 defines the default value for string fields as the empty string + "". A field equal to its default must not be written to the wire, so this + helper returns b"" when value is "". + + The encoding on the wire is: + + tag (varint) | byte_length (varint) | utf-8 bytes + + The proto3 specification requires that string fields contain valid UTF-8. + Python's str.encode("utf-8") enforces this at the encoding step. + + Parameters + ---------- + field: + The field number as declared in the .proto schema. + value: + The string to encode. An empty string causes this helper to return + b"" (field omitted). + + Returns + ------- + bytes + b"" if value is empty, otherwise tag + varint(len(utf8)) + utf8. + """ + if not value: + return b"" + utf8 = value.encode("utf-8") + return encode_tag(field, WT_LEN) + encode_varint(len(utf8)) + utf8 + + +def byt(field: int, value: bytes) -> bytes: + """Encode a proto3 bytes field, omitting it when the value is empty. + + Proto3 defines the default value for bytes fields as the empty byte + string b"". A field equal to its default must not be written, so this + helper returns b"" when value is b"". + + The encoding on the wire is identical to a string field: + + tag (varint) | byte_length (varint) | raw bytes + + Unlike string, no UTF-8 encoding step is needed because the caller already + holds raw bytes. This helper is used for span_id, trace_id, and any other + proto3 bytes field. + + Parameters + ---------- + field: + The field number as declared in the .proto schema. + value: + The bytes to encode. An empty bytes object causes this helper to + return b"" (field omitted). + + Returns + ------- + bytes + b"" if value is empty, otherwise tag + varint(len(value)) + value. + """ + if not value: + return b"" + return encode_tag(field, WT_LEN) + encode_varint(len(value)) + value + + +def u64(field: int, value: int) -> bytes: + """Encode a proto3 uint64 (or any varint-encoded integer) field. + + This helper covers all proto3 field types whose wire type is WT_VARINT + and whose value is a non-negative integer: uint32, uint64, int32 (when + non-negative), int64 (when non-negative), bool (encoded as 0 or 1), and + enum (encoded as its integer value). + + Proto3 defines the default value for these types as 0. A zero value must + not be written to the wire. + + The encoding on the wire is: + + tag (varint) | value (varint) + + Parameters + ---------- + field: + The field number as declared in the .proto schema. + value: + A non-negative integer. Zero causes this helper to return b"" + (field omitted). + + Returns + ------- + bytes + b"" if value is 0, otherwise tag + varint(value). + """ + if value == 0: + return b"" + return encode_tag(field, WT_VARINT) + encode_varint(value) + + +def bool_field(field: int, value: bool) -> bytes: + """Encode a proto3 bool field, omitting it when False. + + Proto3 defines the default value for bool as False (encoded as 0). A + False value must not be written to the wire. + + A True value is encoded as varint 1. The wire layout is: + + tag (varint) | 0x01 + + This helper is separate from u64 because booleans carry different + semantic meaning even though they share wire type 0 (varint). Keeping + them distinct makes SerializeToString() implementations easier to read. + + Parameters + ---------- + field: + The field number as declared in the .proto schema. + value: + The boolean to encode. False causes this helper to return b"" + (field omitted). True is encoded as varint 1. + + Returns + ------- + bytes + b"" if value is False, otherwise tag + 0x01. + """ + if not value: + return b"" + return encode_tag(field, WT_VARINT) + encode_varint(1) + + +def fix32(field: int, value: int) -> bytes: + """Encode a proto3 fixed32 field (4-byte little-endian uint32). + + fixed32 uses wire type WT_32BIT. The decoder reads exactly 4 bytes after + the tag. This makes fixed32 more efficient than uint32 for values that are + frequently large (close to 2^32), because it avoids varint overhead. + + Proto3 default is 0. A zero value must not be written to the wire. + + The encoding on the wire is: + + tag (varint) | 4-byte little-endian uint32 + + Parameters + ---------- + field: + The field number as declared in the .proto schema. + value: + An unsigned 32-bit integer. Zero causes this helper to return b"" + (field omitted). + + Returns + ------- + bytes + b"" if value is 0, otherwise tag + 4 bytes little-endian. + """ + if value == 0: + return b"" + return encode_tag(field, WT_32BIT) + encode_fixed32(value) + + +def fix64(field: int, value: int) -> bytes: + """Encode a proto3 fixed64 field (8-byte little-endian uint64). + + fixed64 uses wire type WT_64BIT. The decoder reads exactly 8 bytes after + the tag. In the OTel proto schemas, fixed64 is used for nanosecond + timestamps (start_time_unix_nano, time_unix_nano), counts, and bucket + counts that are guaranteed to be non-negative. + + Proto3 default is 0. A zero value must not be written to the wire. + + The encoding on the wire is: + + tag (varint) | 8-byte little-endian uint64 + + Parameters + ---------- + field: + The field number as declared in the .proto schema. + value: + An unsigned 64-bit integer. Zero causes this helper to return b"" + (field omitted). + + Returns + ------- + bytes + b"" if value is 0, otherwise tag + 8 bytes little-endian. + """ + if value == 0: + return b"" + return encode_tag(field, WT_64BIT) + encode_fixed64(value) + + +def dbl(field: int, value: float) -> bytes: + """Encode a proto3 double field (8-byte IEEE 754), omitting when zero. + + double uses wire type WT_64BIT. The decoder reads exactly 8 bytes and + interprets them as an IEEE 754 double-precision floating-point number in + little-endian byte order. + + Proto3 default is 0.0. A zero value must not be written to the wire. + Note that -0.0 compares equal to 0.0 in Python, so dbl(field, -0.0) + returns b"" — if that distinction matters, use opt_dbl instead. + + This helper is used for scalar double fields that have a meaningful zero + default and should be omitted when zero: for example, the `sum` field of + SummaryDataPoint and the `zero_threshold` field of + ExponentialHistogramDataPoint. + + Parameters + ---------- + field: + The field number as declared in the .proto schema. + value: + An IEEE 754 double. 0.0 causes this helper to return b"" + (field omitted). Infinity and NaN are encoded as-is. + + Returns + ------- + bytes + b"" if value == 0.0, otherwise tag + 8-byte little-endian double. + """ + if value == 0.0: + return b"" + return encode_tag(field, WT_64BIT) + pack(" bytes: + """Encode an optional proto3 double field, written even when 0.0. + + Some double fields in the OTel proto schemas are declared optional + (proto3 optional syntax), which means the presence of the field is + significant regardless of its value. A value of 0.0 is a meaningful + measurement and must be written; only None (field not set) causes the + field to be omitted. + + This is distinct from dbl, which omits 0.0 as the proto3 default. + opt_dbl is used for fields such as `sum`, `min`, and `max` on histogram + data points, where None means "not present" and 0.0 means "present and + measured as zero". + + The encoding on the wire is identical to dbl when the value is present: + + tag (varint) | 8-byte little-endian double + + Parameters + ---------- + field: + The field number as declared in the .proto schema. + value: + An IEEE 754 double, or None. None causes this helper to return b"" + (field omitted). Any float, including 0.0, -0.0, inf, and nan, is + encoded and written to the wire. + + Returns + ------- + bytes + b"" if value is None, otherwise tag + 8-byte little-endian double. + """ + if value is None: + return b"" + return encode_tag(field, WT_64BIT) + pack(" bytes: + """Encode a proto3 sint32 field (ZigZag varint), omitting when zero. + + sint32 uses wire type WT_VARINT but applies ZigZag encoding before the + varint step. ZigZag maps signed integers to unsigned integers so that + small negative values produce short varints rather than the ten-byte + varints that two's-complement representation would require: + + 0 → 0 + -1 → 1 + 1 → 2 + -2 → 3 + n → 2*n (for n >= 0) + n → -2*n - 1 (for n < 0) + + This is the correct encoding for the proto3 `sint32` type. It is NOT used + for `int32` fields (use u64 for non-negative int32, or encode_int from + the package for signed int32). + + In the OTel proto schemas, sint32 appears on the `scale` field of + ExponentialHistogramDataPoint and the `offset` field of its Buckets + sub-message. These fields represent signed exponents and can be negative. + + Proto3 default is 0. A zero value must not be written to the wire. + + Parameters + ---------- + field: + The field number as declared in the .proto schema. + value: + A signed 32-bit integer. Zero causes this helper to return b"" + (field omitted). + + Returns + ------- + bytes + b"" if value is 0, otherwise tag + ZigZag-encoded varint. + """ + if value == 0: + return b"" + return encode_tag(field, WT_VARINT) + encode_sint32(value) + + +def packed_uint64(field: int, values: list[int]) -> bytes: + """Encode a packed repeated uint64 field. + + In proto3, repeated scalar fields are packed by default. "Packed" means + all elements are encoded contiguously without a tag before each one. + Instead, a single tag and a single length prefix wrap the entire payload: + + tag (varint) | payload_length (varint) | element0 (varint) | element1 (varint) | ... + + Each element is encoded as an independent varint, exactly as encode_varint + would encode it if the field were scalar. The decoder knows how many + elements are present by tracking how many bytes it has consumed relative + to payload_length. + + An empty list produces b"" (field omitted). This matches proto3 behaviour: + a repeated field with zero elements is indistinguishable from the field + being absent. + + This helper is used for fields such as bucket_counts in + ExponentialHistogramDataPoint.Buckets. + + Parameters + ---------- + field: + The field number as declared in the .proto schema. + values: + A list of non-negative integers. An empty list causes this helper + to return b"" (field omitted). + + Returns + ------- + bytes + b"" if values is empty, otherwise tag + varint(len(payload)) + payload, + where payload is the concatenation of varint-encoded elements. + """ + if not values: + return b"" + payload = b"".join(encode_varint(v) for v in values) + return encode_tag(field, WT_LEN) + encode_varint(len(payload)) + payload + + +def packed_fix64(field: int, values: list[int]) -> bytes: + """Encode a packed repeated fixed64 field. + + Like packed_uint64 but each element is encoded as an 8-byte + little-endian uint64 rather than a varint. The wire layout is: + + tag (varint) | payload_length (varint) | element0 (8 bytes) | element1 (8 bytes) | ... + + fixed64 encoding is efficient for values that are close to 2^64 or are + accessed in bulk (the payload is a flat array of 8-byte little-endian + words, amenable to memcpy). + + An empty list produces b"" (field omitted). + + This helper is used for the bucket_counts field of HistogramDataPoint, + where counts are non-negative 64-bit integers. + + Parameters + ---------- + field: + The field number as declared in the .proto schema. + values: + A list of non-negative 64-bit integers. An empty list causes this + helper to return b"" (field omitted). + + Returns + ------- + bytes + b"" if values is empty, otherwise tag + varint(len(payload)) + payload, + where payload is the concatenation of 8-byte little-endian encodings. + """ + if not values: + return b"" + payload = b"".join(encode_fixed64(v) for v in values) + return encode_tag(field, WT_LEN) + encode_varint(len(payload)) + payload + + +def packed_double(field: int, values: list[float]) -> bytes: + """Encode a packed repeated double field. + + Like packed_uint64 but each element is an IEEE 754 double (8 bytes, + little-endian). The wire layout is: + + tag (varint) | payload_length (varint) | element0 (8 bytes) | element1 (8 bytes) | ... + + struct.pack with format " bytes: + """Encode an unsigned 32-bit integer as a protobuf uint32 field value. + + Protobuf uint32 fields use wire type 0 (varint). The value is encoded + as a plain unsigned varint with no transformation applied. + + Valid range: [0, 2^32 - 1]. + + Unlike int32, uint32 has no signed interpretation, so negative inputs + are not defined. Values in [0, 127] encode to a single byte; larger + values require more bytes as their magnitude grows. + + Example — encoding 0: + + encode_varint(0) == b'\\x00' + + Example — encoding 300: + + 300 = 0b1_0010_1100 + Split into 7-bit groups (little-endian): 0b010_1100, 0b000_0010 + Add continuation bit to first group: 0b1_010_1100 == 0xAC + Second group (no continuation): 0b000_0010 == 0x02 + Result: b'\\xac\\x02' + + Reference: + https://protobuf.dev/programming-guides/encoding/ + """ + # uint32 encoding is identical to plain varint encoding — no sign extension, + # no ZigZag, no transformation. encode_varint raises ValueError for negative + # inputs, which correctly rejects values outside the uint32 domain. + return encode_varint(value) + + +def encode_uint64(value: int) -> bytes: + """Encode an unsigned 64-bit integer as a protobuf uint64 field value. + + Protobuf uint64 fields use wire type 0 (varint). The value is encoded + as a plain unsigned varint with no transformation applied. + + Valid range: [0, 2^64 - 1]. + + This is the 64-bit counterpart of encode_uint32. The encoding is + identical — both delegate to encode_varint — but the domain is larger. + Values up to 2^64 - 1 require at most 10 varint bytes. + + Example — encoding 2^32 (first value that exceeds the uint32 domain): + + 2^32 = 4294967296 = 0x1_0000_0000 + Varint encoding requires 5 bytes: + b'\\x80\\x80\\x80\\x80\\x10' + + Reference: + https://protobuf.dev/programming-guides/encoding/ + """ + # Identical to encode_uint32 at the implementation level. The separate + # function exists so call sites can express the .proto field type precisely. + return encode_varint(value) + + +def encode_bool(value: bool) -> bytes: + """Encode a Python bool as a protobuf bool field value. + + Protobuf bool fields use wire type 0 (varint). The encoding guide defines + exactly two valid encodings: + + False -> varint 0 -> 0x00 (one byte) + True -> varint 1 -> 0x01 (one byte) + + The spec states that values other than 0 or 1 are not valid on the wire + for a bool field. This function enforces that constraint by mapping any + truthy value to 1 and any falsy value to 0, regardless of the input's + actual numeric value. + + Reference: + https://protobuf.dev/programming-guides/encoding/ + """ + # The conditional expression evaluates value in a boolean context. + # + # Using (1 if value else 0) instead of int(value) is intentional. + # int(True) == 1 and int(False) == 0, which would work correctly for + # actual bool inputs. However, int(5) == 5, and varint 5 is not a valid + # protobuf bool encoding. The conditional collapses any truthy integer, + # non-empty string, non-empty list, etc. to exactly 1, and any falsy + # value to exactly 0, maintaining spec compliance regardless of what + # the caller passes. + # + # Both 0 and 1 fit in a single varint byte (no continuation bit needed), + # so encode_varint always returns exactly one byte here: + # + # encode_bool(False) -> b'\x00' + # encode_bool(True) -> b'\x01' + return encode_varint(1 if value else 0) + + +def encode_int(value: int) -> bytes: + """Encode a signed integer as a protobuf varint for int32 and int64 fields. + + Protobuf int32 and int64 fields both use wire type 0 (varint). The two + types share the same wire encoding — int32 values are sign-extended to + 64 bits before encoding, so they produce identical byte sequences to the + equivalent int64 value. + + Non-negative values pass straight through to encode_varint unchanged. + + Negative values are handled by a rule from the encoding guide: + + If you use int32 or int64 as the type for a negative number, the + resulting varint is always ten bytes long — it is, effectively, + treated like a very large unsigned integer. + + The rule comes from how the spec defines the encoding: a negative int32 or + int64 is sign-extended to a full 64-bit two's complement value before being + treated as an unsigned integer for varint encoding. A 64-bit value with + all its high bits set always requires 10 varint bytes regardless of the + original magnitude. + + Concrete example — encoding -1: + + -1 in two's complement 64-bit: + + 0xFFFFFFFFFFFFFFFF (all 64 bits set to 1) + + That unsigned 64-bit integer requires all 10 varint bytes: + + 0xFF 0xFF 0xFF 0xFF 0xFF 0xFF 0xFF 0xFF 0xFF 0x01 + + Each of the first nine bytes has its continuation bit set (0xFF = + 1111_1111: 7 payload bits all 1, plus the continuation bit). The tenth + byte 0x01 carries the final payload bits with no continuation bit. + + This fixed 10-byte cost applies to -1 the same as to the most negative + 64-bit integer -2^63. The magnitude of the negative value does not affect + the byte count. This is why the encoding guide recommends sint32/sint64 + (see encode_sint32 and encode_sint64) for fields that often hold negative + values. + + Reference: + https://protobuf.dev/programming-guides/encoding/ + """ + if value >= 0: + # Non-negative values need no transformation. encode_varint encodes + # them directly as unsigned varints, which matches the int32/int64 + # wire encoding exactly. + return encode_varint(value) + + # For negative values, the protobuf spec requires 64-bit sign extension + # followed by unsigned varint encoding. + # + # Python integers have arbitrary precision — there is no native 64-bit + # boundary. The bitwise AND with 0xFFFFFFFFFFFFFFFF (a mask of 64 ones, + # equal to 2^64 - 1) extracts exactly the lower 64 bits of value. + # + # For any negative integer, Python's two's complement representation means + # the lower 64 bits equal the 64-bit unsigned two's complement value: + # + # -1 & 0xFFFFFFFFFFFFFFFF == 0xFFFFFFFFFFFFFFFF (2^64 - 1) + # -2 & 0xFFFFFFFFFFFFFFFF == 0xFFFFFFFFFFFFFFFE (2^64 - 2) + # -128 & 0xFFFFFFFFFFFFFFFF == 0xFFFFFFFFFFFFFF80 + # + # Every result is in the range [2^63, 2^64 - 1], so encode_varint will + # always produce exactly 10 bytes for any negative input. + unsigned = value & 0xFFFFFFFFFFFFFFFF + return encode_varint(unsigned) + + +def encode_sint32(value: int) -> bytes: + """Encode a signed 32-bit integer using ZigZag encoding for sint32 fields. + + Protobuf sint32 fields use wire type 0 (varint) but apply a ZigZag + transformation before varint encoding. ZigZag maps signed integers to + non-negative integers by interleaving the negative and positive sequences: + + 0 -> 0 + -1 -> 1 + 1 -> 2 + -2 -> 3 + 2 -> 4 + -3 -> 5 + 3 -> 6 + ... + n >= 0 -> 2 * n + n < 0 -> -2 * n - 1 + + The key property is that the ZigZag output is small whenever the input's + absolute magnitude is small, regardless of sign. This makes sint32 much + more efficient than int32 for fields that frequently hold negative values: + + Encoding -1 with encode_int: 10 bytes (64-bit two's complement) + Encoding -1 with encode_sint32: 1 byte (ZigZag maps -1 to 1) + + Encoding -64 with encode_int: 10 bytes + Encoding -64 with encode_sint32: 2 bytes (ZigZag maps -64 to 127) + + The official formula for the 32-bit ZigZag transformation is: + + zigzag32(n) = (n << 1) ^ (n >> 31) + + The right shift (n >> 31) is arithmetic in Python, propagating the sign + bit: it produces 0 for non-negative n and -1 (all bits set) for negative n. + + Reference: + https://protobuf.dev/programming-guides/encoding/ + """ + # Step 1: apply the ZigZag transformation. + # + # (value << 1) shifts all bits one position left. This doubles the absolute + # value and frees bit 0 to carry the original sign. For positive n the + # result is 2*n (an even number). For negative n, the shift operates on the + # two's complement representation. + # + # (value >> 31) is an arithmetic right shift by 31 positions. Python + # propagates the sign bit into all vacated positions, so: + # + # non-negative n -> (n >> 31) == 0 (binary: all zeros) + # negative n -> (n >> 31) == -1 (binary: all ones, i.e. 0xFF...FF) + # + # XOR with 0 is a no-op, so positive inputs pass through as 2*n. + # XOR with -1 flips every bit, which is equivalent to bitwise NOT. For + # negative inputs that produces the ZigZag output -2*n - 1. + # + # Worked examples: + # + # n = -1: + # (-1 << 1) == -2 binary: ...1111_1110 + # (-1 >> 31) == -1 binary: ...1111_1111 + # (-2) ^ (-1) == 1 binary: ...0000_0001 ✓ ZigZag(-1) = 1 + # + # n = -2: + # (-2 << 1) == -4 binary: ...1111_1100 + # (-2 >> 31) == -1 binary: ...1111_1111 + # (-4) ^ (-1) == 3 binary: ...0000_0011 ✓ ZigZag(-2) = 3 + # + # n = 1: + # (1 << 1) == 2 binary: ...0000_0010 + # (1 >> 31) == 0 binary: ...0000_0000 + # 2 ^ 0 == 2 binary: ...0000_0010 ✓ ZigZag(1) = 2 + zigzag_value = (value << 1) ^ (value >> 31) + + # Step 2: mask to 32 bits. + # + # Python integers have arbitrary precision, so the XOR result above can + # have more than 32 significant bits for inputs outside the sint32 range + # [-2^31, 2^31 - 1]. The mask clips the result to the 32-bit unsigned + # domain [0, 2^32 - 1], which is the correct output range for sint32 + # ZigZag encoding. + # + # For valid sint32 inputs the ZigZag result already fits in 32 bits and + # the mask has no effect. + # + # 0xFFFFFFFF == 2^32 - 1 == 32 bits of all ones. + zigzag_value &= 0xFFFFFFFF + + # Step 3: varint-encode the non-negative ZigZag result. + # + # The ZigZag value is now in [0, 2^32 - 1]. encode_varint encodes it in + # 1–5 bytes depending on its magnitude, compared to the flat 10 bytes that + # encode_int would use for any negative input. + return encode_varint(zigzag_value) + + +def encode_sint64(value: int) -> bytes: + """Encode a signed 64-bit integer using ZigZag encoding for sint64 fields. + + This is the 64-bit counterpart of encode_sint32. The ZigZag interleaving + and the three-step structure (transform, mask, varint-encode) are identical. + The only differences from encode_sint32 are: + + - The arithmetic right shift uses 63 instead of 31, propagating the + sign bit across all 63 remaining bit positions of a 64-bit value. + - The mask uses 0xFFFFFFFFFFFFFFFF (64 ones) instead of 0xFFFFFFFF + (32 ones), constraining the output to [0, 2^64 - 1]. + - The varint output is at most 10 bytes instead of 5. + + The ZigZag mapping is the same interleaving as sint32, extended to 64 bits: + + 0 -> 0 + -1 -> 1 + 1 -> 2 + -2 -> 3 + 2 -> 4 + ... + n >= 0 -> 2 * n + n < 0 -> -2 * n - 1 + + The official formula for the 64-bit ZigZag transformation is: + + zigzag64(n) = (n << 1) ^ (n >> 63) + + Worked examples: + + n = -1: + (-1 << 1) == -2 binary: ...1111_1110 + (-1 >> 63) == -1 binary: ...1111_1111 + (-2) ^ (-1) == 1 binary: ...0000_0001 ✓ ZigZag(-1) = 1 + Encoded: b'\\x01' (1 byte, vs 10 bytes from encode_int) + + n = -(2**63): (most negative sint64 value) + zigzag64(-(2**63)) == 2**64 - 1 (largest 64-bit unsigned integer) + Encoded: 10 bytes (the worst case for sint64) + + n = 2**63 - 1: (most positive sint64 value) + zigzag64(2**63 - 1) == 2**64 - 2 + Encoded: 10 bytes (the worst case alongside the above) + + Reference: + https://protobuf.dev/programming-guides/encoding/ + """ + # Step 1: apply the 64-bit ZigZag transformation. + # + # (value >> 63) produces 0 for non-negative values and -1 for negative + # values, for the same reason as (value >> 31) in encode_sint32: Python + # performs arithmetic right shifts that propagate the sign bit. + # + # The XOR with 0 leaves positive inputs unchanged (result = 2*n). + # The XOR with -1 flips every bit for negative inputs (result = -2*n - 1). + zigzag_value = (value << 1) ^ (value >> 63) + + # Step 2: mask to 64 bits. + # + # Same rationale as in encode_sint32: Python integers are unbounded, so + # the mask constrains the ZigZag result to the valid 64-bit unsigned range + # [0, 2^64 - 1]. For inputs in the sint64 range [-2^63, 2^63 - 1] the + # result already fits and the mask is a no-op. + # + # 0xFFFFFFFFFFFFFFFF == 2^64 - 1 == 64 bits of all ones. + zigzag_value &= 0xFFFFFFFFFFFFFFFF + + # Step 3: varint-encode the non-negative ZigZag result. + # + # The ZigZag value is in [0, 2^64 - 1], which encode_varint encodes in + # 1–10 bytes. The maximum 10-byte output only occurs at the extremes of + # the sint64 range (see worked examples in the docstring above). + return encode_varint(zigzag_value) + + +# ── Wire type 5 — 32-bit fixed-width ───────────────────────────────────────── + + +def encode_float(value: float) -> bytes: + """Encode a Python float as a protobuf float field value. + + Protobuf float fields use wire type 5 (32-bit fixed-width). The value is + stored as a 4-byte IEEE 754 single-precision floating-point number in + little-endian byte order. + + Unlike varint-encoded numbers, wire type 5 always occupies exactly 4 bytes. + This makes float fields predictable in size but more expensive than varint + for small integer-valued floats. + + Precision note + -------------- + Python's float type is a 64-bit IEEE 754 double-precision number. Converting + to single precision loses approximately 7 significant decimal digits of + precision (single has ~7, double has ~15-16). pack handles the conversion + silently; values outside the float32 range become inf or -inf. + + Byte layout — IEEE 754 single precision (32 bits) + -------------------------------------------------- + The 32 bits are arranged as: + + bit 31: sign (0 = positive, 1 = negative) + bits 30–23: exponent, biased by 127 + bits 22–0: mantissa (the fractional part, with an implicit leading 1) + + The four bytes are written in little-endian order: the least significant + byte is first. + + Example — encoding 1.0: + + IEEE 754 single for 1.0: + sign = 0 + exponent = 127 (biased) = 0b0111_1111 + mantissa = 0 (implicit leading 1, no fractional part) + + Combined 32-bit value: 0x3F800000 + + Little-endian bytes: 0x00, 0x00, 0x80, 0x3F + + Result: b'\\x00\\x00\\x80\\x3f' + + Example — encoding -1.0: + + Same as 1.0 but with the sign bit set. + + Combined 32-bit value: 0xBF800000 + + Little-endian bytes: 0x00, 0x00, 0x80, 0xBF + + Result: b'\\x00\\x00\\x80\\xbf' + + Reference: + https://protobuf.dev/programming-guides/encoding/ + """ + # pack with format ' bytes: + """Encode an unsigned integer as a protobuf fixed32 field value. + + Protobuf fixed32 fields use wire type 5 (32-bit fixed-width). The value + is stored as a 4-byte unsigned integer in little-endian byte order. + + The valid range is [0, 2^32 - 1]. + + Unlike uint32, which uses a varint and grows with the value's magnitude, + fixed32 always occupies exactly 4 bytes. This makes fixed32 more efficient + than uint32 for values that are consistently large (roughly above 2^28, + where a varint would already require 5 bytes), and less efficient for small + values. + + Example — encoding 1: + + 1 as a 4-byte little-endian unsigned integer: + + byte 0 (LSB): 0x01 + byte 1: 0x00 + byte 2: 0x00 + byte 3 (MSB): 0x00 + + Result: b'\\x01\\x00\\x00\\x00' + + Example — encoding 2^32 - 1 (maximum value): + + 0xFFFFFFFF as little-endian bytes: 0xFF, 0xFF, 0xFF, 0xFF + + Result: b'\\xff\\xff\\xff\\xff' + + Reference: + https://protobuf.dev/programming-guides/encoding/ + """ + # pack with format ' bytes: + """Encode a signed integer as a protobuf sfixed32 field value. + + Protobuf sfixed32 fields use wire type 5 (32-bit fixed-width). The value + is stored as a 4-byte signed integer in little-endian two's complement byte + order. + + The valid range is [-2^31, 2^31 - 1]. + + Unlike int32, which uses a varint and always costs 10 bytes for negative + values, sfixed32 always occupies exactly 4 bytes regardless of sign. This + makes sfixed32 more efficient than int32 for negative values and for large + positive values near 2^31. + + Example — encoding -1: + + -1 in 32-bit two's complement: 0xFFFFFFFF + + Little-endian bytes: 0xFF, 0xFF, 0xFF, 0xFF + + Result: b'\\xff\\xff\\xff\\xff' + + Example — encoding -2^31 (minimum value): + + -2147483648 in 32-bit two's complement: 0x80000000 + + Little-endian bytes: 0x00, 0x00, 0x00, 0x80 + + Result: b'\\x00\\x00\\x00\\x80' + + Reference: + https://protobuf.dev/programming-guides/encoding/ + """ + # pack with format ' bytes: + """Encode a Python float as a protobuf double field value. + + Protobuf double fields use wire type 1 (64-bit fixed-width). The value is + stored as an 8-byte IEEE 754 double-precision floating-point number in + little-endian byte order. + + Unlike wire type 5 (float), wire type 1 uses 8 bytes and matches Python's + native float precision exactly — no precision is lost in the conversion. + + Byte layout — IEEE 754 double precision (64 bits) + -------------------------------------------------- + The 64 bits are arranged as: + + bit 63: sign (0 = positive, 1 = negative) + bits 62–52: exponent, biased by 1023 + bits 51–0: mantissa (the fractional part, with an implicit leading 1) + + The eight bytes are written in little-endian order. + + Example — encoding 1.0: + + IEEE 754 double for 1.0: + sign = 0 + exponent = 1023 (biased) = 0b011_1111_1111 + mantissa = 0 + + Combined 64-bit value: 0x3FF0000000000000 + + Little-endian bytes: 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0x3F + + Result: b'\\x00\\x00\\x00\\x00\\x00\\x00\\xf0\\x3f' + + Special values (inf, -inf, nan) are represented in their standard IEEE 754 + double-precision forms and are encoded without error. + + Reference: + https://protobuf.dev/programming-guides/encoding/ + """ + # pack with format ' bytes: + """Encode an unsigned integer as a protobuf fixed64 field value. + + Protobuf fixed64 fields use wire type 1 (64-bit fixed-width). The value + is stored as an 8-byte unsigned integer in little-endian byte order. + + The valid range is [0, 2^64 - 1]. + + fixed64 always occupies exactly 8 bytes. It is more efficient than uint64 + (varint) for values consistently above 2^56, where a varint would already + require 9–10 bytes. For smaller values, uint64's variable length is more + compact. + + Example — encoding 1: + + 1 as an 8-byte little-endian unsigned integer: + + bytes: 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + + Result: b'\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00' + + Example — encoding 2^64 - 1 (maximum value): + + 0xFFFFFFFFFFFFFFFF as little-endian bytes: eight 0xFF bytes. + + Result: b'\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff' + + Reference: + https://protobuf.dev/programming-guides/encoding/ + """ + # pack with format ' bytes: + """Encode a signed integer as a protobuf sfixed64 field value. + + Protobuf sfixed64 fields use wire type 1 (64-bit fixed-width). The value + is stored as an 8-byte signed integer in little-endian two's complement + byte order. + + The valid range is [-2^63, 2^63 - 1]. + + sfixed64 always occupies exactly 8 bytes regardless of sign. Unlike int64 + (varint), which always costs 10 bytes for negative values, sfixed64 saves + 2 bytes for negative inputs and for large positive values near 2^63. + + Example — encoding -1: + + -1 in 64-bit two's complement: 0xFFFFFFFFFFFFFFFF + + Little-endian bytes: eight 0xFF bytes. + + Result: b'\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff' + + Example — encoding -2^63 (minimum value): + + -9223372036854775808 in 64-bit two's complement: 0x8000000000000000 + + Little-endian bytes: 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 + + Result: b'\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80' + + Reference: + https://protobuf.dev/programming-guides/encoding/ + """ + # pack with format ' bytes: + """Encode a Python str as a protobuf string field value. + + Protobuf string fields use wire type 2 (length-delimited). The encoding + is a two-part sequence: + + 1. A varint giving the number of bytes in the UTF-8 representation of + the string. This is the byte count, not the character count — they + differ for any character outside ASCII. + 2. The UTF-8 encoded bytes of the string. + + The protobuf spec requires that string fields contain valid UTF-8. Python + str objects are always valid Unicode, so encoding to UTF-8 always succeeds + (Python's str cannot represent unpaired surrogates in normal use). + + The length prefix allows the decoder to know exactly how many bytes to read + for the string value without scanning for a terminator. + + Example — encoding "hi": + + UTF-8 bytes: b'hi' (2 bytes, both ASCII) + Length varint: encode_varint(2) == b'\\x02' + Result: b'\\x02hi' + + Example — encoding "é" (U+00E9, LATIN SMALL LETTER E WITH ACUTE): + + UTF-8 bytes: b'\\xc3\\xa9' (2 bytes; this character needs 2 UTF-8 bytes) + Length varint: encode_varint(2) == b'\\x02' + Result: b'\\x02\\xc3\\xa9' + + Example — encoding "" (empty string): + + UTF-8 bytes: b'' (0 bytes) + Length varint: encode_varint(0) == b'\\x00' + Result: b'\\x00' + + Reference: + https://protobuf.dev/programming-guides/encoding/ + """ + # Encode the string to UTF-8 bytes. + # + # UTF-8 is the only encoding the protobuf spec allows for string fields. + # The result is a bytes object whose length may be greater than len(value) + # if the string contains non-ASCII characters (each such character encodes + # to 2–4 UTF-8 bytes). + utf8_bytes = value.encode("utf-8") + + # The length prefix is the number of UTF-8 bytes, encoded as a varint. + # + # The decoder reads this varint first to know how many bytes to consume + # for the field value. Without the length prefix, the decoder would have + # no way to find where the string ends in the byte stream. + length_prefix = encode_varint(len(utf8_bytes)) + + # Concatenate the length prefix and the UTF-8 bytes. + # + # The + operator on bytes objects produces a new bytes object containing + # the bytes of length_prefix followed immediately by the bytes of + # utf8_bytes. This is the complete wire encoding for the field value. + return length_prefix + utf8_bytes + + +def encode_bytes(value: bytes) -> bytes: + """Encode a Python bytes object as a protobuf bytes field value. + + Protobuf bytes fields use wire type 2 (length-delimited). The encoding + is a two-part sequence: + + 1. A varint giving the number of raw bytes in the payload. + 2. The raw bytes themselves, copied verbatim. + + Unlike string fields, bytes fields impose no encoding constraint on their + content — any byte sequence is valid, including sequences that are not + valid UTF-8. + + The length prefix allows the decoder to know exactly how many bytes to read + for the field value without scanning for a terminator. + + Example — encoding b'\\x00\\x01\\x02': + + Payload length: 3 bytes + Length varint: encode_varint(3) == b'\\x03' + Result: b'\\x03\\x00\\x01\\x02' + + Example — encoding b'' (empty bytes): + + Payload length: 0 bytes + Length varint: encode_varint(0) == b'\\x00' + Result: b'\\x00' + + Example — encoding b'\\xff\\xff': + + Payload length: 2 bytes + Length varint: encode_varint(2) == b'\\x02' + Result: b'\\x02\\xff\\xff' + + Reference: + https://protobuf.dev/programming-guides/encoding/ + """ + # The length prefix is the number of bytes in value, encoded as a varint. + # + # For large payloads (e.g. 128 bytes or more), encode_varint returns a + # multi-byte varint for the length prefix. For payloads up to 127 bytes + # the length fits in a single varint byte. + length_prefix = encode_varint(len(value)) + + # Concatenate the length prefix and the raw payload bytes. + # + # value is copied verbatim — no transformation is applied to the content. + return length_prefix + value diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/_pyprotobuf/tag.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/_pyprotobuf/tag.py new file mode 100644 index 00000000000..70b081f6ffe --- /dev/null +++ b/opentelemetry-pyproto/src/opentelemetry/pyproto/_pyprotobuf/tag.py @@ -0,0 +1,151 @@ +"""Tag encoding for the protobuf wire format. + +Every protobuf message on the wire is a flat sequence of records. Each record +has exactly two parts: + + 1. A tag — a varint that encodes the field number and wire type. + 2. A value — the encoded field value, whose byte layout depends on the wire + type stored in the preceding tag. + +The tag is the reader's only guide to what follows. Without it, the decoder +would not know where one field ends and the next begins. + +Wire types +---------- +The protobuf spec defines six wire types. The wire type tells the decoder how +many bytes to consume for the value — not what the logical type is, just how +to delimit it on the wire: + + 0 Varint — one or more bytes, continuation bit in MSB of each byte. + 1 64-bit — exactly 8 bytes (used for fixed64, sfixed64, double). + 2 Length-delimited — a varint length prefix followed by that many bytes + (used for string, bytes, embedded messages, packed + repeated fields). + 3 Start group — deprecated; marks the start of a group (proto2 only). + 4 End group — deprecated; marks the end of a group (proto2 only). + 5 32-bit — exactly 4 bytes (used for fixed32, sfixed32, float). + +Wire types 3 and 4 are not used in proto3 and should not appear in new code. + +Tag bit layout +-------------- +The tag integer packs the field number and wire type into a single value: + + bits [2:0] — wire type (3 bits, enough for values 0–7) + bits [N:3] — field number (all remaining higher bits) + +The formula is: + + tag = (field_number << 3) | wire_type + +Three bits are reserved for the wire type because there are only six defined +wire type values (0–5), which fit comfortably in 3 bits (range 0–7). + +Example — field 1, wire type 0 (varint): + + field_number = 1 -> binary 0000_0001 + field_number << 3 -> binary 0000_1000 (decimal 8) + wire_type = 0 -> binary 0000_0000 + tag = 8 | 0 -> binary 0000_1000 (decimal 8) + + Varint-encoded: 0x08 (fits in one byte, no continuation bit needed) + +Example — field 2, wire type 2 (length-delimited): + + field_number = 2 -> binary 0000_0010 + field_number << 3 -> binary 0001_0000 (decimal 16) + wire_type = 2 -> binary 0000_0010 + tag = 16|2 -> binary 0001_0010 (decimal 18) + + Varint-encoded: 0x12 (one byte) + +Example — field 16, wire type 0 (varint): + + field_number = 16 -> binary 0001_0000 + field_number << 3 -> binary 1000_0000 (decimal 128) + wire_type = 0 -> binary 0000_0000 + tag = 128 -> varint requires two bytes: 0x80 0x01 + + The first byte 0x80 has its continuation bit set, meaning another byte + follows. The second byte 0x01 carries the remaining bits with no + continuation bit. This is the standard varint encoding for 128. + +Reference: + https://protobuf.dev/programming-guides/encoding/ +""" + +from .varint import encode_varint + + +def encode_tag(field_number: int, wire_type: int) -> bytes: + """Encode a protobuf record tag. + + The tag is the varint-encoded integer produced by: + + tag = (field_number << 3) | wire_type + + It is written before every field value in a serialised protobuf message. + The decoder reads this tag first to learn the field number (which .proto + field this value belongs to) and the wire type (how many bytes to read for + the value). + + Parameters + ---------- + field_number: + The field number as declared in the .proto schema. Must be a positive + integer. Field numbers 1–15 produce a one-byte tag for wire types 0–5; + field numbers 16–2047 produce a two-byte tag. + wire_type: + One of the six protobuf wire type constants (0–5). See the module + docstring for the full list and their meanings. + + Reference: + https://protobuf.dev/programming-guides/encoding/ + """ + # field_number << 3 shifts the field number left by three bit positions. + # + # This makes room in the three lowest bits of the tag integer for the wire + # type. The three-bit reservation comes directly from the protobuf spec: + # wire types 0–5 fit in 3 bits, so the spec dedicates exactly 3 bits to + # the wire type in every tag. + # + # Example: + # + # field_number = 1 + # field_number in binary = 0000_0001 + # field_number << 3 = 0000_1000 (decimal 8) + # + # After the shift, bits [2:0] are always zero, leaving space for + # the wire type to be OR'd in below. + field_number_bits = field_number << 3 + + # The bitwise OR writes the wire type into the lowest three bits. + # + # Because field_number_bits always has its lowest three bits set to zero + # (from the shift above), and wire_type is always in the range 0–5 (which + # fits in three bits), the OR simply places the wire type value into those + # three vacated bit positions without disturbing the field number bits. + # + # Example (field 1, wire type 2): + # + # field_number_bits = 0b0000_1000 (8) + # wire_type = 0b0000_0010 (2) + # tag = 0b0000_1010 (10) + # + # Example (field 15, wire type 0): + # + # field_number_bits = 0b0111_1000 (120) + # wire_type = 0b0000_0000 (0) + # tag = 0b0111_1000 (120) + # + # Field numbers 1–15 combined with any wire type 0–5 always produce a tag + # integer of at most 127, which encodes as a single varint byte. This is + # why the protobuf style guide recommends reserving field numbers 1–15 for + # the most frequently used fields: their tags are one byte instead of two. + tag = field_number_bits | wire_type + + # The tag integer is itself encoded as a varint before being written to the + # wire. For field numbers 1–15 the tag integer is at most 127, fitting in + # one varint byte. For field number 16 and above the tag integer exceeds + # 127 and requires two or more varint bytes. + return encode_varint(tag) diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/_pyprotobuf/varint.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/_pyprotobuf/varint.py new file mode 100644 index 00000000000..d1d478f82cc --- /dev/null +++ b/opentelemetry-pyproto/src/opentelemetry/pyproto/_pyprotobuf/varint.py @@ -0,0 +1,135 @@ +"""Varint encoding for the protobuf wire format. + +Reference: + https://protobuf.dev/programming-guides/encoding/ + +The protobuf encoding guide explains the wire format mostly from the point of +view of inspecting or decoding bytes that already exist. For example, it shows +how a varint byte sequence can be read by removing each byte's continuation bit, +then combining the remaining 7-bit payload groups. + +This module implements the encoder side of the same algorithm. Instead of +starting with bytes and recovering the integer, we start with the integer and +produce the bytes. That means the operations appear in the opposite direction: + + documented explanation / decoding view: + bytes -> remove continuation bits -> collect 7-bit groups -> integer + + implementation / encoding view: + integer -> extract 7-bit groups -> add continuation bits -> bytes + +The bit layout is the same in both directions. +""" + + +def encode_varint(value: int) -> bytes: + """Encode a non-negative integer as a protobuf varint. + + A protobuf varint stores an integer as one or more bytes. + + Each byte has two parts: + + bit 7: continuation bit + bits 0-6: seven payload bits + + The continuation bit is the most significant bit of the byte: + + 0b1000_0000 == 0x80 + + The payload bits are the lower seven bits of the byte: + + 0b0111_1111 == 0x7F + + The official protobuf encoding guide describes this format here: + + https://protobuf.dev/programming-guides/encoding/ + + The guide explains that the payload is split into 7-bit groups and that the + groups are stored in little-endian order. In this context, little-endian + means the least significant 7-bit group is written first. + + This implementation directly performs that encoder-side operation: + + 1. Take the least significant 7 bits of the integer. + 2. Write those 7 bits into the next output byte. + 3. If more integer bits remain, set the byte's continuation bit. + 4. Shift the integer right by 7 bits so the next group becomes the new + least significant group. + 5. Repeat until the remaining integer fits in one final 7-bit payload. + + This is different from the document's presentation because the document is + mostly showing how to interpret already-encoded bytes. Here we are producing + those bytes. The two views are inverse operations of the same wire-format + rule. + """ + # Protobuf varints, as implemented by this function, encode unsigned integer + # payloads. Signed integer fields need additional field-specific handling + # before varint encoding. For example, sint32/sint64 use ZigZag encoding + # before the resulting non-negative integer is encoded as a varint. + if value < 0: + raise ValueError("varint values must be non-negative") + + # bytearray is used because we build the encoded byte sequence one byte at a + # time. A bytearray is mutable, so appending to it is clearer and cheaper + # than repeatedly concatenating immutable bytes objects. + output = bytearray() + + # 0x7F is binary 0111_1111. + # + # If value is greater than 0x7F, it cannot fit in one protobuf varint byte, + # because one varint byte has only seven payload bits. In that case, we must + # emit one byte containing the current least significant 7-bit group and then + # continue encoding the remaining higher bits. + while value > 0x7F: + # value & 0x7F keeps only the lower seven bits of value. + # + # This extracts exactly one protobuf varint payload group. + # + # Example: + # + # value = 0b1001_0110 # decimal 150 + # 0x7F = 0b0111_1111 + # value & 0x7F = 0b0001_0110 # decimal 22 + # + # This corresponds to the guide's 7-bit payload group, but from the + # encoder direction. The guide often starts from encoded bytes and strips + # the continuation bit. Here we start from the integer and extract the + # payload bits before adding the continuation bit. + payload_bits = value & 0x7F + + # 0x80 is binary 1000_0000. + # + # payload_bits | 0x80 sets the most significant bit of the output byte. + # That most significant bit is the protobuf varint continuation bit. + # + # Setting it to 1 means: + # + # "this is not the final varint byte; another byte follows" + # + # This byte needs the continuation bit because the while condition has + # already proven that value has more than seven bits left to encode. + output.append(payload_bits | 0x80) + + # Shift value right by seven bits. + # + # This discards the payload group we just emitted and moves the next + # higher 7-bit group into the lowest seven bits, ready for the next loop + # iteration. + # + # This is why protobuf varints are little-endian at the 7-bit-group + # level: we emit the least significant group first, then move toward more + # significant groups. + value >>= 7 + + # When the loop ends, value is between 0 and 0x7F inclusive, so it fits in a + # single 7-bit payload group. + # + # This is the final varint byte, so we do NOT set the continuation bit. + # A most significant bit of 0 means: + # + # "this is the final byte of the varint" + output.append(value) + + # Convert the mutable bytearray into immutable bytes, which is the natural + # representation for encoded wire-format data in Python. + return bytes(output) diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/__init__.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/logs/__init__.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/logs/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/logs/v1/__init__.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/logs/v1/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/logs/v1/logs_service_pypb2.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/logs/v1/logs_service_pypb2.py new file mode 100644 index 00000000000..2c57e0914fa --- /dev/null +++ b/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/logs/v1/logs_service_pypb2.py @@ -0,0 +1,32 @@ +"""Pure-Python equivalents of collector/logs/v1/logs_service_pb2.py. + +Field numbers: + ExportLogsServiceRequest resource_logs=1 + ExportLogsServiceResponse (empty — no fields used in export path) +""" + +from __future__ import annotations + +from opentelemetry.pyproto.logs.v1.logs_pypb2 import ResourceLogs +from opentelemetry.pyproto._pyprotobuf.fields import msg + + +class ExportLogsServiceRequest: + def __init__(self, resource_logs: list[ResourceLogs] | None = None): + self.resource_logs: list[ResourceLogs] = ( + list(resource_logs) if resource_logs else [] + ) + + def SerializeToString(self) -> bytes: + return b"".join( + msg(1, rl.SerializeToString()) for rl in self.resource_logs + ) + + +class ExportLogsServiceResponse: + @classmethod + def FromString(cls, data: bytes) -> 'ExportLogsServiceResponse': + return cls() + + def SerializeToString(self) -> bytes: + return b"" diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/logs/v1/logs_service_pypb2_grpc.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/logs/v1/logs_service_pypb2_grpc.py new file mode 100644 index 00000000000..b766d390c49 --- /dev/null +++ b/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/logs/v1/logs_service_pypb2_grpc.py @@ -0,0 +1,16 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from opentelemetry.pyproto.collector.logs.v1.logs_service_pypb2 import ( + ExportLogsServiceRequest, + ExportLogsServiceResponse, +) + + +class LogsServiceStub: + def __init__(self, channel): + self.Export = channel.unary_unary( + '/opentelemetry.proto.collector.logs.v1.LogsService/Export', + request_serializer=ExportLogsServiceRequest.SerializeToString, + response_deserializer=ExportLogsServiceResponse.FromString, + ) diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/metrics/__init__.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/metrics/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/metrics/v1/__init__.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/metrics/v1/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/metrics/v1/metrics_service_pypb2.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/metrics/v1/metrics_service_pypb2.py new file mode 100644 index 00000000000..27322ab663d --- /dev/null +++ b/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/metrics/v1/metrics_service_pypb2.py @@ -0,0 +1,32 @@ +"""Pure-Python equivalents of collector/metrics/v1/metrics_service_pb2.py. + +Field numbers: + ExportMetricsServiceRequest resource_metrics=1 + ExportMetricsServiceResponse (empty — no fields used in export path) +""" + +from __future__ import annotations + +from opentelemetry.pyproto.metrics.v1.metrics_pypb2 import ResourceMetrics +from opentelemetry.pyproto._pyprotobuf.fields import msg + + +class ExportMetricsServiceRequest: + def __init__(self, resource_metrics: list[ResourceMetrics] | None = None): + self.resource_metrics: list[ResourceMetrics] = ( + list(resource_metrics) if resource_metrics else [] + ) + + def SerializeToString(self) -> bytes: + return b"".join( + msg(1, rm.SerializeToString()) for rm in self.resource_metrics + ) + + +class ExportMetricsServiceResponse: + @classmethod + def FromString(cls, data: bytes) -> 'ExportMetricsServiceResponse': + return cls() + + def SerializeToString(self) -> bytes: + return b"" diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/metrics/v1/metrics_service_pypb2_grpc.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/metrics/v1/metrics_service_pypb2_grpc.py new file mode 100644 index 00000000000..50d555ae10f --- /dev/null +++ b/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/metrics/v1/metrics_service_pypb2_grpc.py @@ -0,0 +1,16 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from opentelemetry.pyproto.collector.metrics.v1.metrics_service_pypb2 import ( + ExportMetricsServiceRequest, + ExportMetricsServiceResponse, +) + + +class MetricsServiceStub: + def __init__(self, channel): + self.Export = channel.unary_unary( + '/opentelemetry.proto.collector.metrics.v1.MetricsService/Export', + request_serializer=ExportMetricsServiceRequest.SerializeToString, + response_deserializer=ExportMetricsServiceResponse.FromString, + ) diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/trace/__init__.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/trace/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/trace/v1/__init__.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/trace/v1/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/trace/v1/trace_service_pypb2.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/trace/v1/trace_service_pypb2.py new file mode 100644 index 00000000000..1adaf90debf --- /dev/null +++ b/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/trace/v1/trace_service_pypb2.py @@ -0,0 +1,32 @@ +"""Pure-Python equivalents of collector/trace/v1/trace_service_pb2.py. + +Field numbers: + ExportTraceServiceRequest resource_spans=1 + ExportTraceServiceResponse (empty — no fields used in export path) +""" + +from __future__ import annotations + +from opentelemetry.pyproto.trace.v1.trace_pypb2 import ResourceSpans +from opentelemetry.pyproto._pyprotobuf.fields import msg + + +class ExportTraceServiceRequest: + def __init__(self, resource_spans: list[ResourceSpans] | None = None): + self.resource_spans: list[ResourceSpans] = ( + list(resource_spans) if resource_spans else [] + ) + + def SerializeToString(self) -> bytes: + return b"".join( + msg(1, rs.SerializeToString()) for rs in self.resource_spans + ) + + +class ExportTraceServiceResponse: + @classmethod + def FromString(cls, data: bytes) -> 'ExportTraceServiceResponse': + return cls() + + def SerializeToString(self) -> bytes: + return b"" diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/trace/v1/trace_service_pypb2_grpc.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/trace/v1/trace_service_pypb2_grpc.py new file mode 100644 index 00000000000..3d2b45fc42f --- /dev/null +++ b/opentelemetry-pyproto/src/opentelemetry/pyproto/collector/trace/v1/trace_service_pypb2_grpc.py @@ -0,0 +1,16 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from opentelemetry.pyproto.collector.trace.v1.trace_service_pypb2 import ( + ExportTraceServiceRequest, + ExportTraceServiceResponse, +) + + +class TraceServiceStub: + def __init__(self, channel): + self.Export = channel.unary_unary( + '/opentelemetry.proto.collector.trace.v1.TraceService/Export', + request_serializer=ExportTraceServiceRequest.SerializeToString, + response_deserializer=ExportTraceServiceResponse.FromString, + ) diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/common/__init__.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/common/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/common/v1/__init__.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/common/v1/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/common/v1/common_pypb2.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/common/v1/common_pypb2.py new file mode 100644 index 00000000000..a54cb83ad44 --- /dev/null +++ b/opentelemetry-pyproto/src/opentelemetry/pyproto/common/v1/common_pypb2.py @@ -0,0 +1,132 @@ +"""Pure-Python equivalents of opentelemetry/proto/common/v1/common_pb2.py. + +Field numbers: + AnyValue (oneof value) string_value=1 bool_value=2 int_value=3 + double_value=4 array_value=5 kvlist_value=6 + bytes_value=7 + ArrayValue values=1 + KeyValueList values=1 + KeyValue key=1 value=2 + InstrumentationScope name=1 version=2 attributes=3 + dropped_attributes_count=4 +""" + +from __future__ import annotations + +from struct import pack + +from opentelemetry.pyproto._pyprotobuf import encode_int, encode_tag, encode_varint + +from opentelemetry.pyproto._pyprotobuf.fields import msg, string, u64, WT_LEN, WT_VARINT, WT_64BIT + + +class AnyValue: + """Oneof value container — exactly one field is set.""" + + def __init__( + self, + string_value: str | None = None, + bool_value: bool | None = None, + int_value: int | None = None, + double_value: float | None = None, + array_value: "ArrayValue | None" = None, + kvlist_value: "KeyValueList | None" = None, + bytes_value: bytes | None = None, + ): + self._which: str | None = None + if string_value is not None: + self.string_value = string_value + self._which = "string_value" + elif bool_value is not None: + self.bool_value = bool_value + self._which = "bool_value" + elif int_value is not None: + self.int_value = int_value + self._which = "int_value" + elif double_value is not None: + self.double_value = double_value + self._which = "double_value" + elif array_value is not None: + self.array_value = array_value + self._which = "array_value" + elif kvlist_value is not None: + self.kvlist_value = kvlist_value + self._which = "kvlist_value" + elif bytes_value is not None: + self.bytes_value = bytes_value + self._which = "bytes_value" + + def WhichOneof(self, oneof_name: str) -> str | None: + if oneof_name == "value": + return self._which + return None + + def SerializeToString(self) -> bytes: + # oneof: always written even when the value equals the proto3 default. + if self._which == "string_value": + utf8 = self.string_value.encode("utf-8") + return encode_tag(1, WT_LEN) + encode_varint(len(utf8)) + utf8 + if self._which == "bool_value": + return encode_tag(2, WT_VARINT) + encode_varint(1 if self.bool_value else 0) + if self._which == "int_value": + return encode_tag(3, WT_VARINT) + encode_int(self.int_value) + if self._which == "double_value": + return encode_tag(4, WT_64BIT) + pack(" bytes: + return b"".join(msg(1, v.SerializeToString()) for v in self.values) + + +class KeyValueList: + def __init__(self, values: "list[KeyValue] | None" = None): + self.values: list[KeyValue] = list(values) if values else [] + + def SerializeToString(self) -> bytes: + return b"".join(msg(1, kv.SerializeToString()) for kv in self.values) + + +class KeyValue: + def __init__(self, key: str = "", value: AnyValue | None = None): + self.key = key + self.value = value + + def SerializeToString(self) -> bytes: + result = string(1, self.key) + if self.value is not None: + result += msg(2, self.value.SerializeToString()) + return result + + +class InstrumentationScope: + def __init__( + self, + name: str = "", + version: str = "", + attributes: list[KeyValue] | None = None, + dropped_attributes_count: int = 0, + ): + self.name = name + self.version = version + self.attributes: list[KeyValue] = list(attributes) if attributes else [] + self.dropped_attributes_count = dropped_attributes_count + + def SerializeToString(self) -> bytes: + return ( + string(1, self.name) + + string(2, self.version) + + b"".join(msg(3, kv.SerializeToString()) for kv in self.attributes) + + u64(4, self.dropped_attributes_count) + ) diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/logs/__init__.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/logs/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/logs/v1/__init__.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/logs/v1/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/logs/v1/logs_pypb2.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/logs/v1/logs_pypb2.py new file mode 100644 index 00000000000..9092f2f7773 --- /dev/null +++ b/opentelemetry-pyproto/src/opentelemetry/pyproto/logs/v1/logs_pypb2.py @@ -0,0 +1,114 @@ +"""Pure-Python equivalents of opentelemetry/proto/logs/v1/logs_pb2.py. + +Field numbers: + LogRecord time_unix_nano=1 severity_number=2 severity_text=3 + body=5 attributes=6 dropped_attrs_count=7 flags=8 + trace_id=9 span_id=10 observed_time_unix_nano=11 + event_name=12 + ScopeLogs scope=1 log_records=2 schema_url=3 + ResourceLogs resource=1 scope_logs=2 schema_url=3 +""" + +from __future__ import annotations + +from opentelemetry.pyproto.common.v1.common_pypb2 import ( + AnyValue, + InstrumentationScope, + KeyValue, +) +from opentelemetry.pyproto.resource.v1.resource_pypb2 import Resource +from opentelemetry.pyproto._pyprotobuf.fields import ( + byt, + fix32, + fix64, + msg, + string, + u64, +) + + +class LogRecord: + def __init__( + self, + time_unix_nano: int = 0, + severity_number: int = 0, + severity_text: str = "", + body: AnyValue | None = None, + attributes: list[KeyValue] | None = None, + dropped_attributes_count: int = 0, + flags: int = 0, + trace_id: bytes = b"", + span_id: bytes = b"", + observed_time_unix_nano: int = 0, + event_name: str = "", + ): + self.time_unix_nano = time_unix_nano + self.severity_number = severity_number + self.severity_text = severity_text + self.body = body + self.attributes: list[KeyValue] = list(attributes) if attributes else [] + self.dropped_attributes_count = dropped_attributes_count + self.flags = flags + self.trace_id = trace_id + self.span_id = span_id + self.observed_time_unix_nano = observed_time_unix_nano + self.event_name = event_name + + def SerializeToString(self) -> bytes: + result = ( + fix64(1, self.time_unix_nano) + + u64(2, self.severity_number) + + string(3, self.severity_text) + ) + if self.body is not None: + result += msg(5, self.body.SerializeToString()) + result += ( + b"".join(msg(6, kv.SerializeToString()) for kv in self.attributes) + + u64(7, self.dropped_attributes_count) + + fix32(8, self.flags) + + byt(9, self.trace_id) + + byt(10, self.span_id) + + fix64(11, self.observed_time_unix_nano) + + string(12, self.event_name) + ) + return result + + +class ScopeLogs: + def __init__( + self, + scope: InstrumentationScope | None = None, + log_records: list[LogRecord] | None = None, + schema_url: str = "", + ): + self.scope = scope + self.log_records: list[LogRecord] = list(log_records) if log_records else [] + self.schema_url = schema_url + + def SerializeToString(self) -> bytes: + result = b"" + if self.scope is not None: + result += msg(1, self.scope.SerializeToString()) + result += b"".join(msg(2, lr.SerializeToString()) for lr in self.log_records) + result += string(3, self.schema_url) + return result + + +class ResourceLogs: + def __init__( + self, + resource: Resource | None = None, + scope_logs: list[ScopeLogs] | None = None, + schema_url: str = "", + ): + self.resource = resource + self.scope_logs: list[ScopeLogs] = list(scope_logs) if scope_logs else [] + self.schema_url = schema_url + + def SerializeToString(self) -> bytes: + result = b"" + if self.resource is not None: + result += msg(1, self.resource.SerializeToString()) + result += b"".join(msg(2, sl.SerializeToString()) for sl in self.scope_logs) + result += string(3, self.schema_url) + return result diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/metrics/__init__.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/metrics/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/metrics/v1/__init__.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/metrics/v1/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/metrics/v1/metrics_pypb2.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/metrics/v1/metrics_pypb2.py new file mode 100644 index 00000000000..faee42419c6 --- /dev/null +++ b/opentelemetry-pyproto/src/opentelemetry/pyproto/metrics/v1/metrics_pypb2.py @@ -0,0 +1,500 @@ +"""Pure-Python equivalents of opentelemetry/proto/metrics/v1/metrics_pb2.py. + +Field numbers: + Exemplar filtered_attributes=7 time_unix_nano=2 + as_double=3(wire_64bit) as_int=6(sfixed64) + span_id=4 trace_id=5 + NumberDataPoint attributes=7 start_time_unix_nano=2 time_unix_nano=3 + as_double=4(wire_64bit) as_int=6(sfixed64) + exemplars=5 flags=8 + HistogramDataPoint attributes=9 start_time_unix_nano=2 time_unix_nano=3 + count=4(fixed64) sum=5(opt_dbl) + bucket_counts=6(packed_fix64) + explicit_bounds=7(packed_dbl) exemplars=8 + flags=10 min=11(opt_dbl) max=12(opt_dbl) + ExponentialHistogramDataPoint + attributes=1 start_time_unix_nano=2 time_unix_nano=3 + count=4(fixed64) sum=5(opt_dbl) scale=6(sint32) + zero_count=7(fixed64) positive=8 negative=9 + flags=10 exemplars=11 min=12(opt_dbl) + max=13(opt_dbl) zero_threshold=14(dbl) + ExponentialHistogramDataPoint.Buckets + offset=1(sint32) bucket_counts=2(packed_uint64) + SummaryDataPoint attributes=7 start_time_unix_nano=2 time_unix_nano=3 + count=4(fixed64) sum=5(dbl) quantile_values=6 flags=8 + SummaryDataPoint.ValueAtQuantile quantile=1(dbl) value=2(dbl) + Gauge data_points=1 + Sum data_points=1 aggregation_temporality=2 is_monotonic=3 + Histogram data_points=1 aggregation_temporality=2 + ExponentialHistogram data_points=1 aggregation_temporality=2 + Summary data_points=1 + Metric name=1 description=2 unit=3 + oneof data: gauge=5 sum=7 histogram=9 + exponential_histogram=10 summary=11 + ScopeMetrics scope=1 metrics=2 schema_url=3 + ResourceMetrics resource=1 scope_metrics=2 schema_url=3 +""" + +from __future__ import annotations + +from struct import pack + +from opentelemetry.pyproto.common.v1.common_pypb2 import ( + InstrumentationScope, + KeyValue, +) +from opentelemetry.pyproto.resource.v1.resource_pypb2 import Resource +from opentelemetry.pyproto._pyprotobuf.fields import ( + byt, + bool_field, + dbl, + fix64, + msg, + opt_dbl, + packed_double, + packed_fix64, + packed_uint64, + sint32, + string, + u64, + WT_64BIT, +) +from opentelemetry.pyproto._pyprotobuf import encode_tag + + +def _sfixed64(field: int, value: int) -> bytes: + """sfixed64 field (little-endian signed 64-bit int).""" + if value == 0: + return b"" + return encode_tag(field, WT_64BIT) + pack(" bytes: + """Oneof as_double: always write, even if 0.0.""" + return encode_tag(field, WT_64BIT) + pack(" bytes: + """Oneof as_int: always write, even if 0.""" + return encode_tag(field, WT_64BIT) + pack(" str | None: + if oneof_name == "value": + return self._which + return None + + def SerializeToString(self) -> bytes: + result = fix64(2, self.time_unix_nano) + if self._which == "as_double": + result += _as_double(3, self._as_double) # type: ignore[arg-type] + result += byt(4, self.span_id) + byt(5, self.trace_id) + if self._which == "as_int": + result += _as_int(6, self._as_int) # type: ignore[arg-type] + result += b"".join( + msg(7, kv.SerializeToString()) for kv in self.filtered_attributes + ) + return result + + +class NumberDataPoint: + def __init__( + self, + attributes: list[KeyValue] | None = None, + start_time_unix_nano: int = 0, + time_unix_nano: int = 0, + as_double: float | None = None, + as_int: int | None = None, + exemplars: list[Exemplar] | None = None, + flags: int = 0, + ): + self.attributes: list[KeyValue] = list(attributes) if attributes else [] + self.start_time_unix_nano = start_time_unix_nano + self.time_unix_nano = time_unix_nano + self._as_double = as_double + self._as_int = as_int + self.exemplars: list[Exemplar] = list(exemplars) if exemplars else [] + self.flags = flags + self._which = ( + "as_double" if as_double is not None else + "as_int" if as_int is not None else None + ) + + def WhichOneof(self, oneof_name: str) -> str | None: + if oneof_name == "value": + return self._which + return None + + def SerializeToString(self) -> bytes: + result = fix64(2, self.start_time_unix_nano) + fix64(3, self.time_unix_nano) + if self._which == "as_double": + result += _as_double(4, self._as_double) # type: ignore[arg-type] + result += b"".join(msg(5, ex.SerializeToString()) for ex in self.exemplars) + if self._which == "as_int": + result += _as_int(6, self._as_int) # type: ignore[arg-type] + result += b"".join(msg(7, kv.SerializeToString()) for kv in self.attributes) + result += u64(8, self.flags) + return result + + +class HistogramDataPoint: + def __init__( + self, + attributes: list[KeyValue] | None = None, + start_time_unix_nano: int = 0, + time_unix_nano: int = 0, + count: int = 0, + sum: float | None = None, + bucket_counts: list[int] | None = None, + explicit_bounds: list[float] | None = None, + exemplars: list[Exemplar] | None = None, + flags: int = 0, + min: float | None = None, + max: float | None = None, + ): + self.attributes: list[KeyValue] = list(attributes) if attributes else [] + self.start_time_unix_nano = start_time_unix_nano + self.time_unix_nano = time_unix_nano + self.count = count + self.sum = sum + self.bucket_counts: list[int] = list(bucket_counts) if bucket_counts else [] + self.explicit_bounds: list[float] = ( + list(explicit_bounds) if explicit_bounds else [] + ) + self.exemplars: list[Exemplar] = list(exemplars) if exemplars else [] + self.flags = flags + self.min = min + self.max = max + + def SerializeToString(self) -> bytes: + return ( + fix64(2, self.start_time_unix_nano) + + fix64(3, self.time_unix_nano) + + fix64(4, self.count) + + opt_dbl(5, self.sum) + + packed_fix64(6, self.bucket_counts) + + packed_double(7, self.explicit_bounds) + + b"".join(msg(8, ex.SerializeToString()) for ex in self.exemplars) + + b"".join(msg(9, kv.SerializeToString()) for kv in self.attributes) + + u64(10, self.flags) + + opt_dbl(11, self.min) + + opt_dbl(12, self.max) + ) + + +class ExponentialHistogramDataPoint: + class Buckets: + def __init__( + self, + offset: int = 0, + bucket_counts: list[int] | None = None, + ): + self.offset = offset + self.bucket_counts: list[int] = ( + list(bucket_counts) if bucket_counts else [] + ) + + def SerializeToString(self) -> bytes: + return ( + sint32(1, self.offset) + + packed_uint64(2, self.bucket_counts) + ) + + def __init__( + self, + attributes: list[KeyValue] | None = None, + start_time_unix_nano: int = 0, + time_unix_nano: int = 0, + count: int = 0, + sum: float | None = None, + scale: int = 0, + zero_count: int = 0, + positive: "ExponentialHistogramDataPoint.Buckets | None" = None, + negative: "ExponentialHistogramDataPoint.Buckets | None" = None, + flags: int = 0, + exemplars: list[Exemplar] | None = None, + min: float | None = None, + max: float | None = None, + zero_threshold: float = 0.0, + ): + self.attributes: list[KeyValue] = list(attributes) if attributes else [] + self.start_time_unix_nano = start_time_unix_nano + self.time_unix_nano = time_unix_nano + self.count = count + self.sum = sum + self.scale = scale + self.zero_count = zero_count + self.positive = positive + self.negative = negative + self.flags = flags + self.exemplars: list[Exemplar] = list(exemplars) if exemplars else [] + self.min = min + self.max = max + self.zero_threshold = zero_threshold + + def SerializeToString(self) -> bytes: + result = b"".join(msg(1, kv.SerializeToString()) for kv in self.attributes) + result += ( + fix64(2, self.start_time_unix_nano) + + fix64(3, self.time_unix_nano) + + fix64(4, self.count) + + opt_dbl(5, self.sum) + + sint32(6, self.scale) + + fix64(7, self.zero_count) + ) + if self.positive is not None: + result += msg(8, self.positive.SerializeToString()) + if self.negative is not None: + result += msg(9, self.negative.SerializeToString()) + result += ( + u64(10, self.flags) + + b"".join(msg(11, ex.SerializeToString()) for ex in self.exemplars) + + opt_dbl(12, self.min) + + opt_dbl(13, self.max) + + dbl(14, self.zero_threshold) + ) + return result + + +class SummaryDataPoint: + class ValueAtQuantile: + def __init__(self, quantile: float = 0.0, value: float = 0.0): + self.quantile = quantile + self.value = value + + def SerializeToString(self) -> bytes: + return dbl(1, self.quantile) + dbl(2, self.value) + + def __init__( + self, + attributes: list[KeyValue] | None = None, + start_time_unix_nano: int = 0, + time_unix_nano: int = 0, + count: int = 0, + sum: float = 0.0, + quantile_values: list["SummaryDataPoint.ValueAtQuantile"] | None = None, + flags: int = 0, + ): + self.attributes: list[KeyValue] = list(attributes) if attributes else [] + self.start_time_unix_nano = start_time_unix_nano + self.time_unix_nano = time_unix_nano + self.count = count + self.sum = sum + self.quantile_values: list[SummaryDataPoint.ValueAtQuantile] = ( + list(quantile_values) if quantile_values else [] + ) + self.flags = flags + + def SerializeToString(self) -> bytes: + return ( + fix64(2, self.start_time_unix_nano) + + fix64(3, self.time_unix_nano) + + fix64(4, self.count) + + dbl(5, self.sum) + + b"".join(msg(6, qv.SerializeToString()) for qv in self.quantile_values) + + b"".join(msg(7, kv.SerializeToString()) for kv in self.attributes) + + u64(8, self.flags) + ) + + +class Gauge: + def __init__(self, data_points: list[NumberDataPoint] | None = None): + self.data_points: list[NumberDataPoint] = ( + list(data_points) if data_points else [] + ) + + def SerializeToString(self) -> bytes: + return b"".join(msg(1, dp.SerializeToString()) for dp in self.data_points) + + +class Sum: + def __init__( + self, + data_points: list[NumberDataPoint] | None = None, + aggregation_temporality: int = 0, + is_monotonic: bool = False, + ): + self.data_points: list[NumberDataPoint] = ( + list(data_points) if data_points else [] + ) + self.aggregation_temporality = aggregation_temporality + self.is_monotonic = is_monotonic + + def SerializeToString(self) -> bytes: + temp = self.aggregation_temporality + temp_int = temp.value if hasattr(temp, "value") else int(temp) + return ( + b"".join(msg(1, dp.SerializeToString()) for dp in self.data_points) + + u64(2, temp_int) + + bool_field(3, self.is_monotonic) + ) + + +class Histogram: + def __init__( + self, + data_points: list[HistogramDataPoint] | None = None, + aggregation_temporality: int = 0, + ): + self.data_points: list[HistogramDataPoint] = ( + list(data_points) if data_points else [] + ) + self.aggregation_temporality = aggregation_temporality + + def SerializeToString(self) -> bytes: + temp = self.aggregation_temporality + temp_int = temp.value if hasattr(temp, "value") else int(temp) + return ( + b"".join(msg(1, dp.SerializeToString()) for dp in self.data_points) + + u64(2, temp_int) + ) + + +class ExponentialHistogram: + def __init__( + self, + data_points: list[ExponentialHistogramDataPoint] | None = None, + aggregation_temporality: int = 0, + ): + self.data_points: list[ExponentialHistogramDataPoint] = ( + list(data_points) if data_points else [] + ) + self.aggregation_temporality = aggregation_temporality + + def SerializeToString(self) -> bytes: + temp = self.aggregation_temporality + temp_int = temp.value if hasattr(temp, "value") else int(temp) + return ( + b"".join(msg(1, dp.SerializeToString()) for dp in self.data_points) + + u64(2, temp_int) + ) + + +class Summary: + def __init__(self, data_points: list[SummaryDataPoint] | None = None): + self.data_points: list[SummaryDataPoint] = ( + list(data_points) if data_points else [] + ) + + def SerializeToString(self) -> bytes: + return b"".join(msg(1, dp.SerializeToString()) for dp in self.data_points) + + +class Metric: + def __init__( + self, + name: str = "", + description: str = "", + unit: str = "", + gauge: Gauge | None = None, + sum: Sum | None = None, + histogram: Histogram | None = None, + exponential_histogram: ExponentialHistogram | None = None, + summary: Summary | None = None, + ): + self.name = name + self.description = description + self.unit = unit + self.gauge = gauge + self.sum = sum + self.histogram = histogram + self.exponential_histogram = exponential_histogram + self.summary = summary + + # Resolve oneof data field name + if gauge is not None: + self._which_data = "gauge" + elif sum is not None: + self._which_data = "sum" + elif histogram is not None: + self._which_data = "histogram" + elif exponential_histogram is not None: + self._which_data = "exponential_histogram" + elif summary is not None: + self._which_data = "summary" + else: + self._which_data = None + + def WhichOneof(self, oneof_name: str) -> str | None: + if oneof_name == "data": + return self._which_data + return None + + def SerializeToString(self) -> bytes: + result = string(1, self.name) + string(2, self.description) + string(3, self.unit) + if self.gauge is not None: + result += msg(5, self.gauge.SerializeToString()) + elif self.sum is not None: + result += msg(7, self.sum.SerializeToString()) + elif self.histogram is not None: + result += msg(9, self.histogram.SerializeToString()) + elif self.exponential_histogram is not None: + result += msg(10, self.exponential_histogram.SerializeToString()) + elif self.summary is not None: + result += msg(11, self.summary.SerializeToString()) + return result + + +class ScopeMetrics: + def __init__( + self, + scope: InstrumentationScope | None = None, + metrics: list[Metric] | None = None, + schema_url: str = "", + ): + self.scope = scope + self.metrics: list[Metric] = list(metrics) if metrics else [] + self.schema_url = schema_url + + def SerializeToString(self) -> bytes: + result = b"" + if self.scope is not None: + result += msg(1, self.scope.SerializeToString()) + result += b"".join(msg(2, m.SerializeToString()) for m in self.metrics) + result += string(3, self.schema_url) + return result + + +class ResourceMetrics: + def __init__( + self, + resource: Resource | None = None, + scope_metrics: list[ScopeMetrics] | None = None, + schema_url: str = "", + ): + self.resource = resource + self.scope_metrics: list[ScopeMetrics] = ( + list(scope_metrics) if scope_metrics else [] + ) + self.schema_url = schema_url + + def SerializeToString(self) -> bytes: + result = b"" + if self.resource is not None: + result += msg(1, self.resource.SerializeToString()) + result += b"".join(msg(2, sm.SerializeToString()) for sm in self.scope_metrics) + result += string(3, self.schema_url) + return result diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/resource/__init__.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/resource/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/resource/v1/__init__.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/resource/v1/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/resource/v1/resource_pypb2.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/resource/v1/resource_pypb2.py new file mode 100644 index 00000000000..d73b1606a8f --- /dev/null +++ b/opentelemetry-pyproto/src/opentelemetry/pyproto/resource/v1/resource_pypb2.py @@ -0,0 +1,26 @@ +"""Pure-Python equivalents of opentelemetry/proto/resource/v1/resource_pb2.py. + +Field numbers: + Resource attributes=1 dropped_attributes_count=2 +""" + +from __future__ import annotations + +from opentelemetry.pyproto.common.v1.common_pypb2 import KeyValue +from opentelemetry.pyproto._pyprotobuf.fields import msg, u64 + + +class Resource: + def __init__( + self, + attributes: list[KeyValue] | None = None, + dropped_attributes_count: int = 0, + ): + self.attributes: list[KeyValue] = list(attributes) if attributes else [] + self.dropped_attributes_count = dropped_attributes_count + + def SerializeToString(self) -> bytes: + return ( + b"".join(msg(1, kv.SerializeToString()) for kv in self.attributes) + + u64(2, self.dropped_attributes_count) + ) diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/trace/__init__.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/trace/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/trace/v1/__init__.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/trace/v1/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/trace/v1/trace_pypb2.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/trace/v1/trace_pypb2.py new file mode 100644 index 00000000000..4c8f4cb2154 --- /dev/null +++ b/opentelemetry-pyproto/src/opentelemetry/pyproto/trace/v1/trace_pypb2.py @@ -0,0 +1,188 @@ +"""Pure-Python equivalents of opentelemetry/proto/trace/v1/trace_pb2.py. + +Field numbers: + Status message=2 code=3 + Span.Event time_unix_nano=1 name=2 attributes=3 dropped_attrs_count=4 + Span.Link trace_id=1 span_id=2 trace_state=3 attributes=4 + dropped_attrs_count=5 flags=6 + Span trace_id=1 span_id=2 trace_state=3 parent_span_id=4 + flags=16 name=5 kind=6 start_time_unix_nano=7 + end_time_unix_nano=8 attributes=9 dropped_attrs_count=10 + events=11 dropped_events_count=12 links=13 + dropped_links_count=14 status=15 + ScopeSpans scope=1 spans=2 schema_url=3 + ResourceSpans resource=1 scope_spans=2 schema_url=3 +""" + +from __future__ import annotations + +from opentelemetry.pyproto.common.v1.common_pypb2 import ( + InstrumentationScope, + KeyValue, +) +from opentelemetry.pyproto.resource.v1.resource_pypb2 import Resource +from opentelemetry.pyproto._pyprotobuf.fields import ( + byt, + fix32, + fix64, + msg, + string, + u64, +) + + +class Status: + def __init__(self, message: str = "", code: int = 0): + self.message = message + self.code = code + + def SerializeToString(self) -> bytes: + return string(2, self.message) + u64(3, self.code) + + +class Span: + class Event: + def __init__( + self, + time_unix_nano: int = 0, + name: str = "", + attributes: list[KeyValue] | None = None, + dropped_attributes_count: int = 0, + ): + self.time_unix_nano = time_unix_nano + self.name = name + self.attributes: list[KeyValue] = list(attributes) if attributes else [] + self.dropped_attributes_count = dropped_attributes_count + + def SerializeToString(self) -> bytes: + return ( + fix64(1, self.time_unix_nano) + + string(2, self.name) + + b"".join(msg(3, kv.SerializeToString()) for kv in self.attributes) + + u64(4, self.dropped_attributes_count) + ) + + class Link: + def __init__( + self, + trace_id: bytes = b"", + span_id: bytes = b"", + trace_state: str = "", + attributes: list[KeyValue] | None = None, + dropped_attributes_count: int = 0, + flags: int = 0, + ): + self.trace_id = trace_id + self.span_id = span_id + self.trace_state = trace_state + self.attributes: list[KeyValue] = list(attributes) if attributes else [] + self.dropped_attributes_count = dropped_attributes_count + self.flags = flags + + def SerializeToString(self) -> bytes: + return ( + byt(1, self.trace_id) + + byt(2, self.span_id) + + string(3, self.trace_state) + + b"".join(msg(4, kv.SerializeToString()) for kv in self.attributes) + + u64(5, self.dropped_attributes_count) + + fix32(6, self.flags) + ) + + def __init__( + self, + trace_id: bytes = b"", + span_id: bytes = b"", + trace_state: str = "", + parent_span_id: bytes = b"", + flags: int = 0, + name: str = "", + kind: int = 0, + start_time_unix_nano: int = 0, + end_time_unix_nano: int = 0, + attributes: list[KeyValue] | None = None, + dropped_attributes_count: int = 0, + events: list["Span.Event"] | None = None, + dropped_events_count: int = 0, + links: list["Span.Link"] | None = None, + dropped_links_count: int = 0, + status: Status | None = None, + ): + self.trace_id = trace_id + self.span_id = span_id + self.trace_state = trace_state + self.parent_span_id = parent_span_id + self.flags = flags + self.name = name + self.kind = kind + self.start_time_unix_nano = start_time_unix_nano + self.end_time_unix_nano = end_time_unix_nano + self.attributes: list[KeyValue] = list(attributes) if attributes else [] + self.dropped_attributes_count = dropped_attributes_count + self.events: list[Span.Event] = list(events) if events else [] + self.dropped_events_count = dropped_events_count + self.links: list[Span.Link] = list(links) if links else [] + self.dropped_links_count = dropped_links_count + self.status = status + + def SerializeToString(self) -> bytes: + result = ( + byt(1, self.trace_id) + + byt(2, self.span_id) + + string(3, self.trace_state) + + byt(4, self.parent_span_id) + + string(5, self.name) + + u64(6, self.kind) + + fix64(7, self.start_time_unix_nano) + + fix64(8, self.end_time_unix_nano) + + b"".join(msg(9, kv.SerializeToString()) for kv in self.attributes) + + u64(10, self.dropped_attributes_count) + + b"".join(msg(11, ev.SerializeToString()) for ev in self.events) + + u64(12, self.dropped_events_count) + + b"".join(msg(13, lk.SerializeToString()) for lk in self.links) + + u64(14, self.dropped_links_count) + ) + if self.status is not None: + result += msg(15, self.status.SerializeToString()) + result += fix32(16, self.flags) + return result + + +class ScopeSpans: + def __init__( + self, + scope: InstrumentationScope | None = None, + spans: list[Span] | None = None, + schema_url: str = "", + ): + self.scope = scope + self.spans: list[Span] = list(spans) if spans else [] + self.schema_url = schema_url + + def SerializeToString(self) -> bytes: + result = b"" + if self.scope is not None: + result += msg(1, self.scope.SerializeToString()) + result += b"".join(msg(2, sp.SerializeToString()) for sp in self.spans) + result += string(3, self.schema_url) + return result + + +class ResourceSpans: + def __init__( + self, + resource: Resource | None = None, + scope_spans: list[ScopeSpans] | None = None, + schema_url: str = "", + ): + self.resource = resource + self.scope_spans: list[ScopeSpans] = list(scope_spans) if scope_spans else [] + self.schema_url = schema_url + + def SerializeToString(self) -> bytes: + result = b"" + if self.resource is not None: + result += msg(1, self.resource.SerializeToString()) + result += b"".join(msg(2, ss.SerializeToString()) for ss in self.scope_spans) + result += string(3, self.schema_url) + return result diff --git a/opentelemetry-pyproto/src/opentelemetry/pyproto/version/__init__.py b/opentelemetry-pyproto/src/opentelemetry/pyproto/version/__init__.py new file mode 100644 index 00000000000..697c4434b93 --- /dev/null +++ b/opentelemetry-pyproto/src/opentelemetry/pyproto/version/__init__.py @@ -0,0 +1 @@ +__version__ = "1.43.0.dev" diff --git a/opentelemetry-pyproto/tests/__init__.py b/opentelemetry-pyproto/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/tests/_pyprotobuf/__init__.py b/opentelemetry-pyproto/tests/_pyprotobuf/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/tests/_pyprotobuf/test_benchmark.py b/opentelemetry-pyproto/tests/_pyprotobuf/test_benchmark.py new file mode 100644 index 00000000000..8fd24383d44 --- /dev/null +++ b/opentelemetry-pyproto/tests/_pyprotobuf/test_benchmark.py @@ -0,0 +1,569 @@ +# tests/_pyprotobuf/test_benchmark.py +# +# Benchmark: _pyprotobuf (pure-Python) vs google.protobuf (C extension) +# encoding speed. Four benchmark categories: +# +# Full message — one round-trip of a Record containing every field type. +# Per field — each field helper in isolation against google.protobuf. +# Scaling — string, bytes, and packed-repeated at 3 payload sizes. +# Varint — encode_varint at 1-, 2-, 3-, and 5-byte bit widths. +# +# All proto schemas are built at import time via the descriptor / message- +# factory API — no .proto files or protoc step required. +# +# Run: +# uv run pytest tests/_pyprotobuf/test_benchmark.py -v --benchmark-sort=mean + +from google.protobuf import descriptor_pb2, descriptor_pool, message_factory +from pytest import mark + +from opentelemetry.pyproto._pyprotobuf import encode_varint +from opentelemetry.pyproto._pyprotobuf.fields import ( + bool_field, + byt, + dbl, + fix32, + fix64, + msg, + packed_double, + packed_uint64, + sint32, + string, + u64, +) + +_T = descriptor_pb2.FieldDescriptorProto + + +# ── Proto builders ───────────────────────────────────────────────────────────── + +def _build_full_record_classes(): + """Record message with one field of every type (full-message benchmark).""" + fp = descriptor_pb2.FileDescriptorProto() + fp.name = "pyproto_benchmark.proto" + fp.syntax = "proto3" + + inner = fp.message_type.add() + inner.name = "Inner" + for name, number, tid in (("label", 1, _T.TYPE_STRING), ("seq", 2, _T.TYPE_UINT64)): + f = inner.field.add() + f.name = name; f.number = number; f.type = tid; f.label = _T.LABEL_OPTIONAL + + rec = fp.message_type.add() + rec.name = "Record" + for name, number, tid, lbl in ( + ("name", 1, _T.TYPE_STRING, _T.LABEL_OPTIONAL), + ("count", 2, _T.TYPE_UINT64, _T.LABEL_OPTIONAL), + ("value", 3, _T.TYPE_DOUBLE, _T.LABEL_OPTIONAL), + ("active", 4, _T.TYPE_BOOL, _T.LABEL_OPTIONAL), + ("data", 5, _T.TYPE_BYTES, _T.LABEL_OPTIONAL), + ("timestamp_ns", 6, _T.TYPE_FIXED64, _T.LABEL_OPTIONAL), + ("flags", 7, _T.TYPE_FIXED32, _T.LABEL_OPTIONAL), + ("bucket_counts", 9, _T.TYPE_UINT64, _T.LABEL_REPEATED), + ("bounds", 10, _T.TYPE_DOUBLE, _T.LABEL_REPEATED), + ): + f = rec.field.add() + f.name = name; f.number = number; f.type = tid; f.label = lbl + + mf = rec.field.add() + mf.name = "inner"; mf.number = 8; mf.type = _T.TYPE_MESSAGE + mf.label = _T.LABEL_OPTIONAL; mf.type_name = "Inner" + + pool = descriptor_pool.DescriptorPool() + pool.Add(fp) + return ( + message_factory.GetMessageClass(pool.FindMessageTypeByName("Inner")), + message_factory.GetMessageClass(pool.FindMessageTypeByName("Record")), + ) + + +def _build_field_msg_classes(): + """FieldMsg with one field per type — per-field and scaling benchmarks.""" + fp = descriptor_pb2.FileDescriptorProto() + fp.name = "pyproto_field_benchmark.proto" + fp.syntax = "proto3" + + fi = fp.message_type.add() + fi.name = "FieldInner" + f = fi.field.add() + f.name = "v"; f.number = 1; f.type = _T.TYPE_UINT64; f.label = _T.LABEL_OPTIONAL + + fm = fp.message_type.add() + fm.name = "FieldMsg" + for name, number, tid, lbl in ( + ("f_uint64", 1, _T.TYPE_UINT64, _T.LABEL_OPTIONAL), + ("f_string", 2, _T.TYPE_STRING, _T.LABEL_OPTIONAL), + ("f_bytes", 3, _T.TYPE_BYTES, _T.LABEL_OPTIONAL), + ("f_double", 4, _T.TYPE_DOUBLE, _T.LABEL_OPTIONAL), + ("f_bool", 5, _T.TYPE_BOOL, _T.LABEL_OPTIONAL), + ("f_fixed64", 6, _T.TYPE_FIXED64, _T.LABEL_OPTIONAL), + ("f_fixed32", 7, _T.TYPE_FIXED32, _T.LABEL_OPTIONAL), + ("f_sint32", 8, _T.TYPE_SINT32, _T.LABEL_OPTIONAL), + ("f_packed_u64", 9, _T.TYPE_UINT64, _T.LABEL_REPEATED), + ("f_packed_dbl", 10, _T.TYPE_DOUBLE, _T.LABEL_REPEATED), + ): + f = fm.field.add() + f.name = name; f.number = number; f.type = tid; f.label = lbl + + mf = fm.field.add() + mf.name = "f_msg"; mf.number = 11; mf.type = _T.TYPE_MESSAGE + mf.label = _T.LABEL_OPTIONAL; mf.type_name = "FieldInner" + + pool = descriptor_pool.DescriptorPool() + pool.Add(fp) + return ( + message_factory.GetMessageClass(pool.FindMessageTypeByName("FieldInner")), + message_factory.GetMessageClass(pool.FindMessageTypeByName("FieldMsg")), + ) + + +def _build_repeated_msg_classes(): + """Item + Container — repeated embedded message benchmark.""" + fp = descriptor_pb2.FileDescriptorProto() + fp.name = "pyproto_repeated_benchmark.proto" + fp.syntax = "proto3" + + item = fp.message_type.add() + item.name = "Item" + for name, number, tid in ( + ("label", 1, _T.TYPE_STRING), + ("count", 2, _T.TYPE_UINT64), + ("value", 3, _T.TYPE_DOUBLE), + ): + f = item.field.add() + f.name = name; f.number = number; f.type = tid; f.label = _T.LABEL_OPTIONAL + + container = fp.message_type.add() + container.name = "Container" + rf = container.field.add() + rf.name = "items"; rf.number = 1; rf.type = _T.TYPE_MESSAGE + rf.label = _T.LABEL_REPEATED; rf.type_name = "Item" + + pool = descriptor_pool.DescriptorPool() + pool.Add(fp) + return ( + message_factory.GetMessageClass(pool.FindMessageTypeByName("Item")), + message_factory.GetMessageClass(pool.FindMessageTypeByName("Container")), + ) + + +_Inner, _Record = _build_full_record_classes() +_FieldInner, _FieldMsg = _build_field_msg_classes() +_Item, _Container = _build_repeated_msg_classes() + + +# ── Shared benchmark data ────────────────────────────────────────────────────── + +_NAME = "benchmark.record.example" +_COUNT = 9_876_543_210 +_VALUE = 3.141592653589793 +_DATA = b"\xde\xad\xbe\xef" * 8 +_TS = 1_782_401_900_556_236_527 +_FLAGS = 0xDEAD +_INNER_LABEL = "inner.label" +_INNER_SEQ = 42 +_BUCKET_COUNTS = [0, 1, 4, 12, 35, 78, 120, 89, 42, 15, 4, 1, 0] +_BOUNDS = [0.0, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0] + +_FIELD_INNER_BYTES = _FieldInner(v=42).SerializeToString() + +# Item data for repeated-message benchmarks: (label, count, value) tuples. +# All values are non-default so every field is serialized. +_REPEATED_SIZES = [1, 5, 20] +_ITEM_DATA = [ + (f"item.label.{i}", (i + 1) * 100, (i + 1) * 0.5) + for i in range(max(_REPEATED_SIZES)) +] + +# Pre-built google.protobuf objects — construction cost excluded from +# serialization-only benchmarks in section 7. +_PRE_BUILT_RECORD = _Record( + name=_NAME, count=_COUNT, value=_VALUE, active=True, + data=_DATA, timestamp_ns=_TS, flags=_FLAGS, + inner=_Inner(label=_INNER_LABEL, seq=_INNER_SEQ), + bucket_counts=_BUCKET_COUNTS, bounds=_BOUNDS, +) + +_PRE_BUILT_FIELD_PB = { + "uint64": _FieldMsg(f_uint64=_COUNT), + "string": _FieldMsg(f_string=_NAME), + "bytes": _FieldMsg(f_bytes=_DATA), + "double": _FieldMsg(f_double=_VALUE), + "bool": _FieldMsg(f_bool=True), + "fixed64": _FieldMsg(f_fixed64=_TS), + "fixed32": _FieldMsg(f_fixed32=_FLAGS), + "sint32": _FieldMsg(f_sint32=-12345), + "packed_uint64": _FieldMsg(f_packed_u64=_BUCKET_COUNTS), + "packed_double": _FieldMsg(f_packed_dbl=_BOUNDS), + "msg": _FieldMsg(f_msg=_FieldInner(v=42)), +} + + +# ══════════════════════════════════════════════════════════════════════════════ +# 1. Full-message benchmark +# ══════════════════════════════════════════════════════════════════════════════ + +def _pyproto_encode() -> bytes: + inner = string(1, _INNER_LABEL) + u64(2, _INNER_SEQ) + return ( + string(1, _NAME) + + u64(2, _COUNT) + + dbl(3, _VALUE) + + bool_field(4, True) + + byt(5, _DATA) + + fix64(6, _TS) + + fix32(7, _FLAGS) + + msg(8, inner) + + packed_uint64(9, _BUCKET_COUNTS) + + packed_double(10, _BOUNDS) + ) + + +def _pb_encode() -> bytes: + return _Record( + name=_NAME, + count=_COUNT, + value=_VALUE, + active=True, + data=_DATA, + timestamp_ns=_TS, + flags=_FLAGS, + inner=_Inner(label=_INNER_LABEL, seq=_INNER_SEQ), + bucket_counts=_BUCKET_COUNTS, + bounds=_BOUNDS, + ).SerializeToString() + + +def test_encode_outputs_identical() -> None: + assert _pyproto_encode() == _pb_encode() + + +def test_encode_pyproto(benchmark) -> None: + result = benchmark(_pyproto_encode) + assert len(result) > 0 + + +def test_encode_protobuf(benchmark) -> None: + result = benchmark(_pb_encode) + assert len(result) > 0 + + +# ══════════════════════════════════════════════════════════════════════════════ +# 2. Per-field-type benchmarks +# +# Each pyproto field helper is benchmarked against google.protobuf encoding +# the same single field. Both sides include message-construction overhead +# where applicable (there is no separate "build" vs "serialize" step in the +# pyproto API — it is a pure function call). +# ══════════════════════════════════════════════════════════════════════════════ + +_PER_FIELD = [ + ("uint64", lambda: u64(1, _COUNT), + lambda: _FieldMsg(f_uint64=_COUNT).SerializeToString()), + ("string", lambda: string(2, _NAME), + lambda: _FieldMsg(f_string=_NAME).SerializeToString()), + ("bytes", lambda: byt(3, _DATA), + lambda: _FieldMsg(f_bytes=_DATA).SerializeToString()), + ("double", lambda: dbl(4, _VALUE), + lambda: _FieldMsg(f_double=_VALUE).SerializeToString()), + ("bool", lambda: bool_field(5, True), + lambda: _FieldMsg(f_bool=True).SerializeToString()), + ("fixed64", lambda: fix64(6, _TS), + lambda: _FieldMsg(f_fixed64=_TS).SerializeToString()), + ("fixed32", lambda: fix32(7, _FLAGS), + lambda: _FieldMsg(f_fixed32=_FLAGS).SerializeToString()), + ("sint32", lambda: sint32(8, -12345), + lambda: _FieldMsg(f_sint32=-12345).SerializeToString()), + ("packed_uint64",lambda: packed_uint64(9, _BUCKET_COUNTS), + lambda: _FieldMsg(f_packed_u64=_BUCKET_COUNTS).SerializeToString()), + ("packed_double",lambda: packed_double(10, _BOUNDS), + lambda: _FieldMsg(f_packed_dbl=_BOUNDS).SerializeToString()), + ("msg", lambda: msg(11, _FIELD_INNER_BYTES), + lambda: _FieldMsg(f_msg=_FieldInner(v=42)).SerializeToString()), +] + +_FIELD_IDS = [name for name, _, _ in _PER_FIELD] + + +@mark.parametrize("name,pyproto_fn,pb_fn", _PER_FIELD, ids=_FIELD_IDS) +def test_field_outputs_identical(name, pyproto_fn, pb_fn) -> None: + assert pyproto_fn() == pb_fn(), f"encoding mismatch for field type: {name}" + + +@mark.parametrize("fn", [p for _, p, _ in _PER_FIELD], ids=_FIELD_IDS) +def test_field_pyproto(benchmark, fn) -> None: + result = benchmark(fn) + assert len(result) > 0 + + +@mark.parametrize("fn", [pb for _, _, pb in _PER_FIELD], ids=_FIELD_IDS) +def test_field_protobuf(benchmark, fn) -> None: + result = benchmark(fn) + assert len(result) > 0 + + +# ══════════════════════════════════════════════════════════════════════════════ +# 3. Scaling benchmarks +# +# How encoding time grows with payload size for the three field types whose +# cost is proportional to data length: string, bytes, and packed repeated. +# Data is pre-built outside the benchmark loop; only the encoding is timed. +# ══════════════════════════════════════════════════════════════════════════════ + +_STR_SIZES = [4, 256, 16_384] +_BYTES_SIZES = [4, 256, 16_384] +_PACKED_SIZES = [10, 100, 1_000] + + +@mark.parametrize("n", _STR_SIZES, ids=["4B", "256B", "16KB"]) +def test_scale_string_pyproto(benchmark, n) -> None: + s = "x" * n + result = benchmark(lambda: string(2, s)) + assert len(result) > 0 + + +@mark.parametrize("n", _STR_SIZES, ids=["4B", "256B", "16KB"]) +def test_scale_string_protobuf(benchmark, n) -> None: + s = "x" * n + result = benchmark(lambda: _FieldMsg(f_string=s).SerializeToString()) + assert len(result) > 0 + + +@mark.parametrize("n", _BYTES_SIZES, ids=["4B", "256B", "16KB"]) +def test_scale_bytes_pyproto(benchmark, n) -> None: + data = b"x" * n + result = benchmark(lambda: byt(3, data)) + assert len(result) > 0 + + +@mark.parametrize("n", _BYTES_SIZES, ids=["4B", "256B", "16KB"]) +def test_scale_bytes_protobuf(benchmark, n) -> None: + data = b"x" * n + result = benchmark(lambda: _FieldMsg(f_bytes=data).SerializeToString()) + assert len(result) > 0 + + +@mark.parametrize("n", _PACKED_SIZES, ids=["10", "100", "1000"]) +def test_scale_packed_uint64_pyproto(benchmark, n) -> None: + values = list(range(n)) + result = benchmark(lambda: packed_uint64(9, values)) + assert len(result) > 0 + + +@mark.parametrize("n", _PACKED_SIZES, ids=["10", "100", "1000"]) +def test_scale_packed_uint64_protobuf(benchmark, n) -> None: + values = list(range(n)) + result = benchmark(lambda: _FieldMsg(f_packed_u64=values).SerializeToString()) + assert len(result) > 0 + + +@mark.parametrize("n", _PACKED_SIZES, ids=["10", "100", "1000"]) +def test_scale_packed_double_pyproto(benchmark, n) -> None: + values = [float(i) * 0.1 for i in range(n)] + result = benchmark(lambda: packed_double(10, values)) + assert len(result) > 0 + + +@mark.parametrize("n", _PACKED_SIZES, ids=["10", "100", "1000"]) +def test_scale_packed_double_protobuf(benchmark, n) -> None: + values = [float(i) * 0.1 for i in range(n)] + result = benchmark(lambda: _FieldMsg(f_packed_dbl=values).SerializeToString()) + assert len(result) > 0 + + +# ══════════════════════════════════════════════════════════════════════════════ +# 4. Varint bit-width benchmarks +# +# encode_varint is the hottest path in _pyprotobuf: every tag and every +# varint-type field value goes through it. These benchmarks measure how +# encoding time scales with the number of continuation bytes (bit width). +# +# The google.protobuf column encodes the same integer as a uint64 field +# (including message-object construction and tag overhead), which is the +# closest available comparison point since protobuf does not expose a +# standalone varint encoder. Use the pyproto column to track raw varint +# speed; use the ratio to see the combined tag+field overhead gap. +# ══════════════════════════════════════════════════════════════════════════════ + +_VARINT_CASES = [ + ("1byte", 63), # fits in 1 byte (0x00–0x7f) + ("2byte", 300), # requires 2 bytes (0x80–0x3fff) + ("3byte", 100_000), # requires 3 bytes (0x4000–0x1fffff) + ("5byte", 2**32 - 1), # requires 5 bytes (max uint32) +] + +_VARINT_IDS = [c[0] for c in _VARINT_CASES] +_VARINT_VALUES = [c[1] for c in _VARINT_CASES] + + +@mark.parametrize("v", _VARINT_VALUES, ids=_VARINT_IDS) +def test_varint_pyproto(benchmark, v) -> None: + result = benchmark(lambda: encode_varint(v)) + assert len(result) > 0 + + +@mark.parametrize("v", _VARINT_VALUES, ids=_VARINT_IDS) +def test_varint_protobuf(benchmark, v) -> None: + result = benchmark(lambda: _FieldMsg(f_uint64=v).SerializeToString()) + assert len(result) > 0 + + +# ══════════════════════════════════════════════════════════════════════════════ +# 5. Concatenation strategy: `+` vs `b"".join()` +# +# The current SerializeToString() pattern chains field results with `+`, +# creating N-1 intermediate bytes objects (one per addition). b"".join() +# allocates the final buffer once and copies each part exactly once. +# These two tests use the same field data so the only variable is the +# concatenation strategy. +# ══════════════════════════════════════════════════════════════════════════════ + +def _pyproto_encode_join() -> bytes: + inner = b"".join([string(1, _INNER_LABEL), u64(2, _INNER_SEQ)]) + return b"".join([ + string(1, _NAME), + u64(2, _COUNT), + dbl(3, _VALUE), + bool_field(4, True), + byt(5, _DATA), + fix64(6, _TS), + fix32(7, _FLAGS), + msg(8, inner), + packed_uint64(9, _BUCKET_COUNTS), + packed_double(10, _BOUNDS), + ]) + + +def test_concat_strategy_outputs_identical() -> None: + assert _pyproto_encode() == _pyproto_encode_join() + + +def test_encode_concat_pyproto(benchmark) -> None: + result = benchmark(_pyproto_encode) + assert len(result) > 0 + + +def test_encode_join_pyproto(benchmark) -> None: + result = benchmark(_pyproto_encode_join) + assert len(result) > 0 + + +# ══════════════════════════════════════════════════════════════════════════════ +# 6. All-default fields (fast path) +# +# Proto3 omits fields whose value equals the type default (0, "", b"", False, +# 0.0, []). Each _pyprotobuf helper returns b"" immediately for defaults. +# This benchmark measures the minimum cost of SerializeToString() — calling +# every helper in a message when none produce any output. +# ══════════════════════════════════════════════════════════════════════════════ + +def _pyproto_encode_all_defaults() -> bytes: + # Embedded message field omitted: real SerializeToString() guards with + # `if self.field is not None`. Every helper here returns b"". + return ( + string(1, "") + + u64(2, 0) + + dbl(3, 0.0) + + bool_field(4, False) + + byt(5, b"") + + fix64(6, 0) + + fix32(7, 0) + + packed_uint64(9, []) + + packed_double(10, []) + ) + + +def test_all_defaults_pyproto(benchmark) -> None: + result = benchmark(_pyproto_encode_all_defaults) + assert result == b"" + + +def test_all_defaults_protobuf(benchmark) -> None: + result = benchmark(lambda: _Record().SerializeToString()) + assert result == b"" + + +# ══════════════════════════════════════════════════════════════════════════════ +# 7. google.protobuf: construction vs serialization split +# +# Previous benchmarks bundle message-object construction and serialization +# together for google.protobuf. pyproto has no construction phase — it is a +# pure function call. Separating the two phases shows where google.protobuf's +# time actually goes and gives a fairer encoding-only comparison. +# +# Read these alongside the existing test_encode_pyproto / test_encode_protobuf: +# test_encode_pyproto — pyproto: pure call, no object +# test_encode_protobuf_construct — google.protobuf: construction only +# test_encode_protobuf_serialize — google.protobuf: serialization only +# test_encode_protobuf — google.protobuf: construction + serialization +# ══════════════════════════════════════════════════════════════════════════════ + +def _pb_construct(): + return _Record( + name=_NAME, count=_COUNT, value=_VALUE, active=True, + data=_DATA, timestamp_ns=_TS, flags=_FLAGS, + inner=_Inner(label=_INNER_LABEL, seq=_INNER_SEQ), + bucket_counts=_BUCKET_COUNTS, bounds=_BOUNDS, + ) + + +def test_encode_protobuf_construct(benchmark) -> None: + result = benchmark(_pb_construct) + assert result is not None + + +def test_encode_protobuf_serialize(benchmark) -> None: + result = benchmark(_PRE_BUILT_RECORD.SerializeToString) + assert len(result) > 0 + + +@mark.parametrize("name", list(_PRE_BUILT_FIELD_PB), ids=list(_PRE_BUILT_FIELD_PB)) +def test_field_serialize_only_protobuf(benchmark, name) -> None: + result = benchmark(_PRE_BUILT_FIELD_PB[name].SerializeToString) + assert len(result) > 0 + + +# ══════════════════════════════════════════════════════════════════════════════ +# 8. Repeated embedded messages +# +# Encodes a Container with N Item sub-messages. Each Item has a string, +# uint64, and double field. Unlike packed repeated scalars, repeated messages +# require one SerializeToString-equivalent call per element plus one msg() +# wrapper per element — this tests whether per-element overhead compounds +# or stays flat. +# +# pyproto side: b"".join(msg(1, string(1,l)+u64(2,c)+dbl(3,v)) for ...) +# protobuf side: _Container(items=[_Item(...), ...]).SerializeToString() +# ══════════════════════════════════════════════════════════════════════════════ + +def _pyproto_encode_repeated(n: int) -> bytes: + return b"".join( + msg(1, string(1, label) + u64(2, count) + dbl(3, value)) + for label, count, value in _ITEM_DATA[:n] + ) + + +def _pb_encode_repeated(n: int) -> bytes: + return _Container( + items=[ + _Item(label=label, count=count, value=value) + for label, count, value in _ITEM_DATA[:n] + ] + ).SerializeToString() + + +@mark.parametrize("n", _REPEATED_SIZES, ids=["1", "5", "20"]) +def test_repeated_msg_outputs_identical(n) -> None: + assert _pyproto_encode_repeated(n) == _pb_encode_repeated(n) + + +@mark.parametrize("n", _REPEATED_SIZES, ids=["1", "5", "20"]) +def test_repeated_msg_pyproto(benchmark, n) -> None: + result = benchmark(lambda: _pyproto_encode_repeated(n)) + assert len(result) > 0 + + +@mark.parametrize("n", _REPEATED_SIZES, ids=["1", "5", "20"]) +def test_repeated_msg_protobuf(benchmark, n) -> None: + result = benchmark(lambda: _pb_encode_repeated(n)) + assert len(result) > 0 diff --git a/opentelemetry-pyproto/tests/_pyprotobuf/test_enum.py b/opentelemetry-pyproto/tests/_pyprotobuf/test_enum.py new file mode 100644 index 00000000000..539c81cf525 --- /dev/null +++ b/opentelemetry-pyproto/tests/_pyprotobuf/test_enum.py @@ -0,0 +1,96 @@ +# tests/test__enum.py +# +# encode_enum encodes an integer enum value using the same wire format as +# int32: non-negative values as a plain varint, negative values as a 64-bit +# two's-complement varint. These tests verify that contract using hand-computed +# expected byte literals. + +from google.protobuf import descriptor_pb2, descriptor_pool, message_factory +from pytest import mark + +from opentelemetry.pyproto._pyprotobuf import encode_enum, encode_int, encode_tag + + +def test_zero() -> None: + # The proto3 default enum value is always 0. + assert encode_enum(0) == b"\x00" + + +def test_one() -> None: + # Typical enum constant fits in a single varint byte. + assert encode_enum(1) == b"\x01" + + +def test_two() -> None: + assert encode_enum(2) == b"\x02" + + +def test_two_byte_value() -> None: + # A large enum constant that requires two varint bytes (same as encode_int(300)). + assert encode_enum(300) == b"\xac\x02" + + +def test_negative_value() -> None: + # Negative enum values use 64-bit two's-complement encoding. + # -1 → 10-byte varint for 2^64-1. + assert encode_enum(-1) == b"\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01" + + +def test_matches_encode_int_for_positive_values() -> None: + # encode_enum and encode_int must produce identical bytes for non-negative inputs. + for value in [0, 1, 2, 127, 128, 255, 300, 2**16]: + assert encode_enum(value) == encode_int(value) + + +def test_matches_encode_int_for_negative_values() -> None: + for value in [-1, -2, -(2**31)]: + assert encode_enum(value) == encode_int(value) + + +# ── oracle — byte-for-byte comparison with google.protobuf ──────────────────── +# +# A proto2 message with a Color enum field is used as the oracle. proto2 is +# chosen so the field is serialised even when its value is the default (0). +# +# The Color enum defines RED=0, GREEN=1, BLUE=2. Only those three values are +# used because proto2 rejects assignments of undefined enum constants. +# +# The serialised message is exactly encode_tag(field, 0) + encode_enum(value). + +_FIELD = 1 +_WT = 0 # wire type 0 — varint + + +def _build_enum_message_class(): + file_proto = descriptor_pb2.FileDescriptorProto() + file_proto.name = "opentelemetry_pyproto_test_enum.proto" + file_proto.syntax = "proto2" + + color = file_proto.enum_type.add() + color.name = "Color" + for name, number in [("RED", 0), ("GREEN", 1), ("BLUE", 2)]: + ev = color.value.add() + ev.name = name + ev.number = number + + msg_proto = file_proto.message_type.add() + msg_proto.name = "EnumMessage" + f = msg_proto.field.add() + f.name = "color_field" + f.number = _FIELD + f.type = descriptor_pb2.FieldDescriptorProto.TYPE_ENUM + f.label = descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL + f.type_name = ".Color" + + pool = descriptor_pool.DescriptorPool() + pool.Add(file_proto) + return message_factory.GetMessageClass(pool.FindMessageTypeByName("EnumMessage")) + + +_EnumMessage = _build_enum_message_class() + + +@mark.parametrize("value", [0, 1, 2]) +def test_encode_enum_matches_protobuf(value: int) -> None: + expected = encode_tag(_FIELD, _WT) + encode_enum(value) + assert _EnumMessage(color_field=value).SerializeToString() == expected diff --git a/opentelemetry-pyproto/tests/_pyprotobuf/test_fields.py b/opentelemetry-pyproto/tests/_pyprotobuf/test_fields.py new file mode 100644 index 00000000000..d97035c7bb9 --- /dev/null +++ b/opentelemetry-pyproto/tests/_pyprotobuf/test_fields.py @@ -0,0 +1,651 @@ +# tests/_pyprotobuf/test_fields.py +# +# Tests for the proto3 field-level encoding helpers in fields.py. +# +# Every test follows the same pattern: +# 1. Verify default-omission: proto3 defaults (zero, empty, None) produce b"". +# 2. Verify tag: the first byte(s) of the output encode the correct field +# number and wire type using the formula (field_number << 3) | wire_type. +# 3. Verify value: the bytes after the tag match the correct wire encoding. +# +# All expected byte literals are derived by hand from the protobuf wire-format +# specification and cross-checked against encode_tag + the primitive encoders. + +from math import inf, nan +from struct import pack + +from pytest import mark + +from opentelemetry.pyproto._pyprotobuf import encode_tag, encode_varint +from opentelemetry.pyproto._pyprotobuf.fields import ( + WT_32BIT, + WT_64BIT, + WT_LEN, + WT_VARINT, + bool_field, + byt, + dbl, + fix32, + fix64, + msg, + opt_dbl, + packed_double, + packed_fix64, + packed_uint64, + sint32, + string, + u64, +) + + +# ── Wire-type constants ──────────────────────────────────────────────────────── + + +def test_wt_varint_is_zero() -> None: + assert WT_VARINT == 0 + + +def test_wt_64bit_is_one() -> None: + assert WT_64BIT == 1 + + +def test_wt_len_is_two() -> None: + assert WT_LEN == 2 + + +def test_wt_32bit_is_five() -> None: + assert WT_32BIT == 5 + + +# ── msg ──────────────────────────────────────────────────────────────────────── +# +# msg always writes (no omission for empty content). The caller decides whether +# to guard on None. Wire layout: tag | varint(len(content)) | content. + + +def test_msg_empty_content() -> None: + # An empty sub-message produces tag + varint(0), not b"". + # tag(1, 2) = (1<<3)|2 = 10 = 0x0A; varint(0) = 0x00 + assert msg(1, b"") == b"\x0a\x00" + + +def test_msg_single_byte_content() -> None: + # tag(1, 2) = 0x0A; varint(1) = 0x01; content = 0xAB + assert msg(1, b"\xab") == b"\x0a\x01\xab" + + +def test_msg_two_byte_content() -> None: + # tag(1, 2) = 0x0A; varint(2) = 0x02; content = 0xDE 0xAD + assert msg(1, b"\xde\xad") == b"\x0a\x02\xde\xad" + + +def test_msg_field_number_2() -> None: + # tag(2, 2) = (2<<3)|2 = 18 = 0x12 + assert msg(2, b"\x01") == b"\x12\x01\x01" + + +def test_msg_tag_uses_wt_len() -> None: + result = msg(3, b"\xff") + assert result[0] == (3 << 3) | WT_LEN + + +def test_msg_length_prefix_correct_for_long_content() -> None: + # 128 bytes of content — length prefix needs two varint bytes (0x80 0x01) + content = b"\x00" * 128 + result = msg(1, content) + assert result[0:1] == b"\x0a" + assert result[1:3] == b"\x80\x01" + assert result[3:] == content + + +def test_msg_matches_formula() -> None: + content = b"\x42" * 5 + field = 4 + expected = encode_tag(field, WT_LEN) + encode_varint(len(content)) + content + assert msg(field, content) == expected + + +# ── string ───────────────────────────────────────────────────────────────────── +# +# Omit when empty. Wire layout: tag (wt=2) | varint(len(utf8)) | utf8 bytes. + + +def test_string_empty_is_omitted() -> None: + assert string(1, "") == b"" + + +def test_string_ascii() -> None: + # tag(1, 2) = 0x0A; varint(2) = 0x02; "hi" = 0x68 0x69 + assert string(1, "hi") == b"\x0a\x02hi" + + +def test_string_single_char() -> None: + assert string(1, "A") == b"\x0a\x01A" + + +def test_string_two_byte_utf8() -> None: + # "é" = U+00E9, UTF-8: C3 A9 (2 bytes) + assert string(1, "é") == b"\x0a\x02\xc3\xa9" + + +def test_string_three_byte_utf8() -> None: + # "中" = U+4E2D, UTF-8: E4 B8 AD (3 bytes) + assert string(1, "中") == b"\x0a\x03\xe4\xb8\xad" + + +def test_string_field_number_affects_tag() -> None: + # tag(3, 2) = (3<<3)|2 = 26 = 0x1A + assert string(3, "x") == b"\x1a\x01x" + + +def test_string_tag_uses_wt_len() -> None: + result = string(2, "hello") + assert result[0] == (2 << 3) | WT_LEN + + +def test_string_length_is_byte_count_not_char_count() -> None: + # "日" = 3 UTF-8 bytes; length prefix must be 3, not 1 + result = string(1, "日") + assert result[1] == 3 + + +@mark.parametrize("value", ["a", "hello", "café", "Ünïcödé", "日本語", "\U0001F600"]) +def test_string_matches_formula(value: str) -> None: + utf8 = value.encode("utf-8") + expected = encode_tag(1, WT_LEN) + encode_varint(len(utf8)) + utf8 + assert string(1, value) == expected + + +# ── byt ──────────────────────────────────────────────────────────────────────── +# +# Omit when empty. Wire layout: tag (wt=2) | varint(len) | raw bytes. +# Identical layout to string but no UTF-8 encoding step. + + +def test_byt_empty_is_omitted() -> None: + assert byt(1, b"") == b"" + + +def test_byt_single_byte() -> None: + # tag(1, 2) = 0x0A; varint(1) = 0x01; payload = 0x42 + assert byt(1, b"\x42") == b"\x0a\x01\x42" + + +def test_byt_two_bytes() -> None: + assert byt(1, b"\xde\xad") == b"\x0a\x02\xde\xad" + + +def test_byt_null_bytes_preserved() -> None: + assert byt(1, b"\x00\x00") == b"\x0a\x02\x00\x00" + + +def test_byt_high_bytes_preserved() -> None: + assert byt(1, b"\xff\x80") == b"\x0a\x02\xff\x80" + + +def test_byt_field_number_affects_tag() -> None: + # tag(7, 2) = (7<<3)|2 = 58 = 0x3A + assert byt(7, b"\x01") == b"\x3a\x01\x01" + + +def test_byt_tag_uses_wt_len() -> None: + result = byt(2, b"\x99") + assert result[0] == (2 << 3) | WT_LEN + + +@mark.parametrize("value", [b"\x00", b"\xff", b"hello", b"\x00\x01\x02", b"\x80\x81\x82"]) +def test_byt_matches_formula(value: bytes) -> None: + expected = encode_tag(1, WT_LEN) + encode_varint(len(value)) + value + assert byt(1, value) == expected + + +# ── u64 ──────────────────────────────────────────────────────────────────────── +# +# Omit when zero. Wire layout: tag (wt=0) | varint(value). + + +def test_u64_zero_is_omitted() -> None: + assert u64(1, 0) == b"" + + +def test_u64_one() -> None: + # tag(1, 0) = 0x08; varint(1) = 0x01 + assert u64(1, 1) == b"\x08\x01" + + +def test_u64_127() -> None: + assert u64(1, 127) == b"\x08\x7f" + + +def test_u64_128() -> None: + # varint(128) = 0x80 0x01 + assert u64(1, 128) == b"\x08\x80\x01" + + +def test_u64_field_number_affects_tag() -> None: + # tag(4, 0) = (4<<3)|0 = 32 = 0x20 + assert u64(4, 1) == b"\x20\x01" + + +def test_u64_tag_uses_wt_varint() -> None: + result = u64(5, 99) + assert result[0] == (5 << 3) | WT_VARINT + + +@mark.parametrize("value", [1, 127, 128, 255, 300, 2**32 - 1, 2**32, 2**64 - 1]) +def test_u64_matches_formula(value: int) -> None: + expected = encode_tag(1, WT_VARINT) + encode_varint(value) + assert u64(1, value) == expected + + +# ── bool_field ───────────────────────────────────────────────────────────────── +# +# Omit when False. True encodes as varint 1. Wire type is WT_VARINT. + + +def test_bool_false_is_omitted() -> None: + assert bool_field(1, False) == b"" + + +def test_bool_true() -> None: + # tag(1, 0) = 0x08; varint(1) = 0x01 + assert bool_field(1, True) == b"\x08\x01" + + +def test_bool_field_number_affects_tag() -> None: + # tag(6, 0) = (6<<3)|0 = 48 = 0x30 + assert bool_field(6, True) == b"\x30\x01" + + +def test_bool_tag_uses_wt_varint() -> None: + result = bool_field(2, True) + assert result[0] == (2 << 3) | WT_VARINT + + +def test_bool_true_always_encodes_as_one() -> None: + # bool True always produces a varint 1 regardless of the Python truthiness integer + assert bool_field(1, True) == encode_tag(1, WT_VARINT) + b"\x01" + + +# ── fix32 ────────────────────────────────────────────────────────────────────── +# +# Omit when zero. Wire layout: tag (wt=5) | 4-byte little-endian uint32. + + +def test_fix32_zero_is_omitted() -> None: + assert fix32(1, 0) == b"" + + +def test_fix32_one() -> None: + # tag(1, 5) = (1<<3)|5 = 13 = 0x0D; pack(" None: + # 2^32-1 → all bytes 0xFF + assert fix32(1, 2**32 - 1) == b"\x0d\xff\xff\xff\xff" + + +def test_fix32_field_number_affects_tag() -> None: + # tag(3, 5) = (3<<3)|5 = 29 = 0x1D + assert fix32(3, 1) == b"\x1d\x01\x00\x00\x00" + + +def test_fix32_tag_uses_wt_32bit() -> None: + result = fix32(2, 7) + assert result[0] == (2 << 3) | WT_32BIT + + +def test_fix32_always_four_value_bytes() -> None: + result = fix32(1, 42) + # 1 tag byte + 4 value bytes + assert len(result) == 5 + + +@mark.parametrize("value", [1, 255, 256, 2**16, 2**24, 2**32 - 1]) +def test_fix32_matches_formula(value: int) -> None: + expected = encode_tag(1, WT_32BIT) + pack(" None: + assert fix64(1, 0) == b"" + + +def test_fix64_one() -> None: + # tag(1, 1) = (1<<3)|1 = 9 = 0x09; pack(" None: + assert fix64(1, 2**64 - 1) == b"\x09\xff\xff\xff\xff\xff\xff\xff\xff" + + +def test_fix64_field_number_affects_tag() -> None: + # tag(2, 1) = (2<<3)|1 = 17 = 0x11 + assert fix64(2, 1) == b"\x11\x01\x00\x00\x00\x00\x00\x00\x00" + + +def test_fix64_tag_uses_wt_64bit() -> None: + result = fix64(3, 99) + assert result[0] == (3 << 3) | WT_64BIT + + +def test_fix64_always_eight_value_bytes() -> None: + result = fix64(1, 42) + # 1 tag byte + 8 value bytes + assert len(result) == 9 + + +@mark.parametrize("value", [1, 255, 2**32 - 1, 2**32, 2**63, 2**64 - 1]) +def test_fix64_matches_formula(value: int) -> None: + expected = encode_tag(1, WT_64BIT) + pack(" None: + assert dbl(1, 0.0) == b"" + + +def test_dbl_negative_zero_is_omitted() -> None: + # -0.0 == 0.0 in Python, so it is treated as the proto3 default and omitted. + assert dbl(1, -0.0) == b"" + + +def test_dbl_one() -> None: + # tag(1, 1) = 0x09; pack(" None: + assert dbl(1, -1.0) == b"\x09\x00\x00\x00\x00\x00\x00\xf0\xbf" + + +def test_dbl_infinity_is_encoded() -> None: + result = dbl(1, inf) + assert result == encode_tag(1, WT_64BIT) + pack(" None: + result = dbl(1, nan) + assert result == encode_tag(1, WT_64BIT) + pack(" None: + # tag(4, 1) = (4<<3)|1 = 33 = 0x21 + result = dbl(4, 1.0) + assert result[0] == (4 << 3) | WT_64BIT + + +def test_dbl_always_eight_value_bytes() -> None: + result = dbl(1, 3.14) + assert len(result) == 9 + + +@mark.parametrize("value", [1.0, -1.0, 0.5, 3.14, 1e100, -1e100, inf, -inf]) +def test_dbl_matches_formula(value: float) -> None: + expected = encode_tag(1, WT_64BIT) + pack(" None: + assert opt_dbl(1, None) == b"" + + +def test_opt_dbl_zero_is_NOT_omitted() -> None: + # This is the key difference from dbl: 0.0 is a valid measurement. + result = opt_dbl(1, 0.0) + assert result != b"" + assert result == encode_tag(1, WT_64BIT) + pack(" None: + result = opt_dbl(1, -0.0) + assert result != b"" + + +def test_opt_dbl_one() -> None: + assert opt_dbl(1, 1.0) == b"\x09\x00\x00\x00\x00\x00\x00\xf0\x3f" + + +def test_opt_dbl_negative_one() -> None: + assert opt_dbl(1, -1.0) == b"\x09\x00\x00\x00\x00\x00\x00\xf0\xbf" + + +def test_opt_dbl_infinity_is_encoded() -> None: + result = opt_dbl(1, inf) + assert result == encode_tag(1, WT_64BIT) + pack(" None: + result = opt_dbl(1, nan) + assert result == encode_tag(1, WT_64BIT) + pack(" None: + result = opt_dbl(5, 1.0) + assert result[0] == (5 << 3) | WT_64BIT + + +@mark.parametrize("value", [0.0, -0.0, 1.0, -1.0, 0.5, 3.14, 1e100, inf, -inf]) +def test_opt_dbl_matches_formula(value: float) -> None: + expected = encode_tag(1, WT_64BIT) + pack(" None: + # dbl omits 0.0; opt_dbl does not. + assert dbl(1, 0.0) == b"" + assert opt_dbl(1, 0.0) != b"" + + +# ── sint32 ───────────────────────────────────────────────────────────────────── +# +# Omit when zero. ZigZag encoding: n>=0 → 2n, n<0 → -2n-1. Wire type WT_VARINT. + + +def test_sint32_zero_is_omitted() -> None: + assert sint32(1, 0) == b"" + + +def test_sint32_positive_one() -> None: + # ZigZag(1) = 2; tag(1,0)=0x08; varint(2)=0x02 + assert sint32(1, 1) == b"\x08\x02" + + +def test_sint32_negative_one() -> None: + # ZigZag(-1) = 1; tag(1,0)=0x08; varint(1)=0x01 + assert sint32(1, -1) == b"\x08\x01" + + +def test_sint32_positive_two() -> None: + # ZigZag(2) = 4 + assert sint32(1, 2) == b"\x08\x04" + + +def test_sint32_negative_two() -> None: + # ZigZag(-2) = 3 + assert sint32(1, -2) == b"\x08\x03" + + +def test_sint32_field_number_affects_tag() -> None: + # tag(7, 0) = (7<<3)|0 = 56 = 0x38 + result = sint32(7, 1) + assert result[0] == (7 << 3) | WT_VARINT + + +def test_sint32_tag_uses_wt_varint() -> None: + result = sint32(3, -5) + assert result[0] == (3 << 3) | WT_VARINT + + +@mark.parametrize("value", [1, -1, 2, -2, 127, -128, 150, -150, 2**31 - 1, -(2**31)]) +def test_sint32_zigzag_formula(value: int) -> None: + zigzag = 2 * value if value >= 0 else -2 * value - 1 + expected = encode_tag(1, WT_VARINT) + encode_varint(zigzag) + assert sint32(1, value) == expected + + +# ── packed_uint64 ────────────────────────────────────────────────────────────── +# +# Omit when empty. Wire layout: tag (wt=2) | varint(payload_len) | varint*... + + +def test_packed_uint64_empty_is_omitted() -> None: + assert packed_uint64(1, []) == b"" + + +def test_packed_uint64_single_element() -> None: + # payload = varint(1) = 0x01; len=1 + # tag(1,2)=0x0A; varint(1)=0x01; payload=0x01 + assert packed_uint64(1, [1]) == b"\x0a\x01\x01" + + +def test_packed_uint64_three_elements() -> None: + # payload = varint(1)+varint(2)+varint(3) = 0x01 0x02 0x03; len=3 + assert packed_uint64(1, [1, 2, 3]) == b"\x0a\x03\x01\x02\x03" + + +def test_packed_uint64_element_zero() -> None: + # Zero element encodes as varint(0) = 0x00 — still written (not omitted within payload) + assert packed_uint64(1, [0]) == b"\x0a\x01\x00" + + +def test_packed_uint64_field_number_affects_tag() -> None: + result = packed_uint64(3, [1]) + assert result[0] == (3 << 3) | WT_LEN + + +def test_packed_uint64_tag_uses_wt_len() -> None: + result = packed_uint64(2, [42]) + assert result[0] == (2 << 3) | WT_LEN + + +def test_packed_uint64_length_prefix_covers_payload() -> None: + values = [1, 128, 300] + payload = b"".join(encode_varint(v) for v in values) + result = packed_uint64(1, values) + # byte at index 1 is varint(len(payload)); for short payloads this is 1 byte + assert result == encode_tag(1, WT_LEN) + encode_varint(len(payload)) + payload + + +@mark.parametrize("values", [[1], [0, 1, 2], [127, 128, 255], [2**32 - 1, 2**64 - 1]]) +def test_packed_uint64_matches_formula(values: list) -> None: + payload = b"".join(encode_varint(v) for v in values) + expected = encode_tag(1, WT_LEN) + encode_varint(len(payload)) + payload + assert packed_uint64(1, values) == expected + + +# ── packed_fix64 ─────────────────────────────────────────────────────────────── +# +# Omit when empty. Wire layout: tag (wt=2) | varint(payload_len) | fixed64*... +# Each element is 8 bytes little-endian regardless of value. + + +def test_packed_fix64_empty_is_omitted() -> None: + assert packed_fix64(1, []) == b"" + + +def test_packed_fix64_single_element() -> None: + # payload = pack(" None: + # payload = 16 bytes; varint(16)=0x10 + result = packed_fix64(1, [1, 2]) + expected = b"\x0a\x10" + pack(" None: + result = packed_fix64(1, [0]) + assert result == b"\x0a\x08" + b"\x00" * 8 + + +def test_packed_fix64_field_number_affects_tag() -> None: + result = packed_fix64(5, [1]) + assert result[0] == (5 << 3) | WT_LEN + + +def test_packed_fix64_payload_length_always_multiple_of_8() -> None: + for n in [1, 2, 3, 5]: + result = packed_fix64(1, list(range(n))) + # payload length is n*8; varint(n*8) occupies 1 byte for n<=15 + payload_len = result[1] + assert payload_len == n * 8 + + +@mark.parametrize("values", [[0], [1, 2], [2**32 - 1, 2**64 - 1], [0, 0, 0]]) +def test_packed_fix64_matches_formula(values: list) -> None: + payload = b"".join(pack(" None: + assert packed_double(1, []) == b"" + + +def test_packed_double_single_element() -> None: + # payload = pack(" None: + result = packed_double(1, [1.0, 2.0]) + expected = b"\x0a\x10" + pack("<2d", 1.0, 2.0) + assert result == expected + + +def test_packed_double_zero_element_is_written() -> None: + # 0.0 is valid inside a packed repeated field (only the field itself is omitted when empty) + result = packed_double(1, [0.0]) + assert result == b"\x0a\x08" + pack(" None: + result = packed_double(7, [1.0]) + assert result[0] == (7 << 3) | WT_LEN + + +def test_packed_double_payload_length_always_multiple_of_8() -> None: + for n in [1, 2, 3]: + result = packed_double(1, [float(i) for i in range(n)]) + payload_len = result[1] + assert payload_len == n * 8 + + +@mark.parametrize( + "values", + [[0.0], [1.0, -1.0], [0.5, 3.14, 2.71], [inf, -inf]], +) +def test_packed_double_matches_formula(values: list) -> None: + payload = pack(f"<{len(values)}d", *values) + expected = encode_tag(1, WT_LEN) + encode_varint(len(payload)) + payload + assert packed_double(1, values) == expected diff --git a/opentelemetry-pyproto/tests/_pyprotobuf/test_scalars.py b/opentelemetry-pyproto/tests/_pyprotobuf/test_scalars.py new file mode 100644 index 00000000000..513d88c23f9 --- /dev/null +++ b/opentelemetry-pyproto/tests/_pyprotobuf/test_scalars.py @@ -0,0 +1,684 @@ +# tests/test__scalars.py +# +# Tests for all scalar encoders in _scalars.py. The oracle for fixed-width +# types is Python's struct module (independent standard-library implementation). +# Variable-width types (varint-backed) are verified against hand-computed +# expected byte literals derived from the protobuf wire-format spec. + +from math import e, inf, nan, pi, tau +from struct import pack, unpack + +from google.protobuf import descriptor_pb2, descriptor_pool, message_factory +from pytest import mark + +from opentelemetry.pyproto._pyprotobuf import ( + encode_bool, + encode_bytes, + encode_double, + encode_fixed32, + encode_fixed64, + encode_float, + encode_int, + encode_sfixed32, + encode_sfixed64, + encode_sint32, + encode_sint64, + encode_string, + encode_tag, + encode_uint32, + encode_uint64, + encode_varint, +) + + +# ── encode_uint32 ────────────────────────────────────────────────────────────── + + +def test_uint32_zero() -> None: + assert encode_uint32(0) == b"\x00" + + +def test_uint32_one() -> None: + assert encode_uint32(1) == b"\x01" + + +def test_uint32_max() -> None: + # 2^32-1 encodes as five varint bytes: all low bits set. + assert encode_uint32(2**32 - 1) == b"\xff\xff\xff\xff\x0f" + + +@mark.parametrize("value", [0, 1, 127, 128, 255, 300, 2**16 - 1, 2**16, 2**32 - 1]) +def test_uint32_matches_uint64_encoding(value: int) -> None: + # uint32 and uint64 share the same varint encoding for values in [0, 2^32-1]. + assert encode_uint32(value) == encode_uint64(value) + + +# ── encode_uint64 ────────────────────────────────────────────────────────────── + + +def test_uint64_zero() -> None: + assert encode_uint64(0) == b"\x00" + + +def test_uint64_max() -> None: + assert encode_uint64(2**64 - 1) == b"\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01" + + +@mark.parametrize( + "value", + [0, 1, 127, 128, 2**32 - 1, 2**32, 2**56, 2**63 - 1, 2**63, 2**64 - 1], +) +def test_uint64_is_varint(value: int) -> None: + assert encode_uint64(value) == encode_varint(value) + + +# ── encode_bool ──────────────────────────────────────────────────────────────── + + +def test_bool_false() -> None: + assert encode_bool(False) == b"\x00" + + +def test_bool_true() -> None: + assert encode_bool(True) == b"\x01" + + +def test_bool_truthy_int() -> None: + # Any truthy value must encode as 1, not as the integer itself. + assert encode_bool(42) == b"\x01" + + +def test_bool_falsy_int() -> None: + assert encode_bool(0) == b"\x00" + + +# ── encode_int ───────────────────────────────────────────────────────────────── +# +# int32 / int64: non-negative values encode like varint; negative values are +# sign-extended to 64 bits (two's complement) then encoded as a varint. + + +def test_int_zero() -> None: + assert encode_int(0) == b"\x00" + + +def test_int_positive_small() -> None: + assert encode_int(1) == b"\x01" + + +def test_int_positive_300() -> None: + assert encode_int(300) == b"\xac\x02" + + +def test_int_negative_one() -> None: + # -1 in 64-bit two's complement is 2^64-1, which requires 10 varint bytes. + assert encode_int(-1) == b"\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01" + + +def test_int_negative_two() -> None: + assert encode_int(-2) == b"\xfe\xff\xff\xff\xff\xff\xff\xff\xff\x01" + + +def test_int_int32_min() -> None: + # -2^31 in 64-bit two's complement: bits 31-63 all set, bits 0-30 clear. + assert encode_int(-(2**31)) == b"\x80\x80\x80\x80\xf8\xff\xff\xff\xff\x01" + + +def test_int_int64_min() -> None: + # -2^63: only bit 63 set. Nine groups of 0x80, then final byte 0x01. + assert encode_int(-(2**63)) == b"\x80\x80\x80\x80\x80\x80\x80\x80\x80\x01" + + +# ── encode_sint32 ────────────────────────────────────────────────────────────── +# +# ZigZag32: n>=0 → 2n, n<0 → -2n-1. Result fits in 32 bits; encoded as varint. + + +def test_sint32_zero() -> None: + # ZigZag(0) = 0 + assert encode_sint32(0) == b"\x00" + + +def test_sint32_negative_one() -> None: + # ZigZag(-1) = 1 + assert encode_sint32(-1) == b"\x01" + + +def test_sint32_positive_one() -> None: + # ZigZag(1) = 2 + assert encode_sint32(1) == b"\x02" + + +def test_sint32_negative_two() -> None: + # ZigZag(-2) = 3 + assert encode_sint32(-2) == b"\x03" + + +def test_sint32_positive_two() -> None: + # ZigZag(2) = 4 + assert encode_sint32(2) == b"\x04" + + +def test_sint32_max() -> None: + # ZigZag(2^31-1) = 2^32-2 = 0xFFFFFFFE → 5-byte varint + assert encode_sint32(2**31 - 1) == b"\xfe\xff\xff\xff\x0f" + + +def test_sint32_min() -> None: + # ZigZag(-2^31) = 2^32-1 = 0xFFFFFFFF → 5-byte varint + assert encode_sint32(-(2**31)) == b"\xff\xff\xff\xff\x0f" + + +@mark.parametrize("value", [0, 1, -1, 2, -2, 150, -150, 2**31 - 1, -(2**31)]) +def test_sint32_zigzag(value: int) -> None: + # Verify ZigZag correctness using the arithmetic definition as oracle. + zigzag = 2 * value if value >= 0 else -2 * value - 1 + assert encode_sint32(value) == encode_varint(zigzag) + + +# ── encode_sint64 ────────────────────────────────────────────────────────────── +# +# ZigZag64: same interleaving as sint32 but over the 64-bit domain. + + +def test_sint64_zero() -> None: + assert encode_sint64(0) == b"\x00" + + +def test_sint64_negative_one() -> None: + assert encode_sint64(-1) == b"\x01" + + +def test_sint64_positive_one() -> None: + assert encode_sint64(1) == b"\x02" + + +def test_sint64_max() -> None: + # ZigZag64(2^63-1) = 2^64-2 = 0xFFFFFFFFFFFFFFFE → 10-byte varint + assert encode_sint64(2**63 - 1) == b"\xfe\xff\xff\xff\xff\xff\xff\xff\xff\x01" + + +def test_sint64_min() -> None: + # ZigZag64(-2^63) = 2^64-1 = 0xFFFFFFFFFFFFFFFF → 10-byte varint + assert encode_sint64(-(2**63)) == b"\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01" + + +@mark.parametrize( + "value", + [0, 1, -1, 2, -2, 150, -150, 2**31 - 1, -(2**31), 2**32, -(2**32), 2**63 - 1, -(2**63)], +) +def test_sint64_zigzag(value: int) -> None: + zigzag = 2 * value if value >= 0 else -2 * value - 1 + assert encode_sint64(value) == encode_varint(zigzag) + + +# ── encode_float ─────────────────────────────────────────────────────────────── + + +def test_float_zero() -> None: + assert encode_float(0.0) == b"\x00\x00\x00\x00" + + +def test_float_one() -> None: + # IEEE 754 single 1.0 = 0x3F800000; little-endian: 00 00 80 3F + assert encode_float(1.0) == b"\x00\x00\x80\x3f" + + +def test_float_negative_one() -> None: + # IEEE 754 single -1.0 = 0xBF800000; little-endian: 00 00 80 BF + assert encode_float(-1.0) == b"\x00\x00\x80\xbf" + + +def test_float_always_four_bytes() -> None: + assert len(encode_float(0.0)) == 4 + assert len(encode_float(1.0)) == 4 + assert len(encode_float(inf)) == 4 + + +@mark.parametrize( + "value", + [ + 0.0, 1.0, -1.0, 0.5, -0.5, 0.25, 2.0, -2.0, + unpack(" None: + assert encode_float(value) == pack(" None: + assert encode_float(nan) == pack(" None: + assert encode_double(0.0) == b"\x00\x00\x00\x00\x00\x00\x00\x00" + + +def test_double_one() -> None: + # IEEE 754 double 1.0 = 0x3FF0000000000000; little-endian: 00 00 00 00 00 00 F0 3F + assert encode_double(1.0) == b"\x00\x00\x00\x00\x00\x00\xf0\x3f" + + +def test_double_negative_one() -> None: + assert encode_double(-1.0) == b"\x00\x00\x00\x00\x00\x00\xf0\xbf" + + +def test_double_always_eight_bytes() -> None: + assert len(encode_double(0.0)) == 8 + assert len(encode_double(1.0)) == 8 + assert len(encode_double(inf)) == 8 + + +@mark.parametrize( + "value", + [ + 0.0, 1.0, -1.0, 0.5, -0.5, pi, e, tau, + 1e100, -1e100, 5e-324, 1.7976931348623157e308, inf, -inf, + ], +) +def test_double_matches_struct(value: float) -> None: + assert encode_double(value) == pack(" None: + assert encode_double(nan) == pack(" None: + assert encode_fixed32(0) == b"\x00\x00\x00\x00" + + +def test_fixed32_one() -> None: + assert encode_fixed32(1) == b"\x01\x00\x00\x00" + + +def test_fixed32_max() -> None: + assert encode_fixed32(2**32 - 1) == b"\xff\xff\xff\xff" + + +def test_fixed32_always_four_bytes() -> None: + assert len(encode_fixed32(0)) == 4 + assert len(encode_fixed32(2**32 - 1)) == 4 + + +@mark.parametrize("value", [0, 1, 127, 128, 255, 256, 2**16 - 1, 2**16, 2**24, 2**32 - 1]) +def test_fixed32_matches_struct(value: int) -> None: + assert encode_fixed32(value) == pack(" None: + assert encode_sfixed32(0) == b"\x00\x00\x00\x00" + + +def test_sfixed32_negative_one() -> None: + # -1 in 32-bit two's complement is 0xFFFFFFFF; all bytes 0xFF. + assert encode_sfixed32(-1) == b"\xff\xff\xff\xff" + + +def test_sfixed32_min() -> None: + # -2^31 = 0x80000000 little-endian: 00 00 00 80 + assert encode_sfixed32(-(2**31)) == b"\x00\x00\x00\x80" + + +def test_sfixed32_max() -> None: + # 2^31-1 = 0x7FFFFFFF little-endian: FF FF FF 7F + assert encode_sfixed32(2**31 - 1) == b"\xff\xff\xff\x7f" + + +def test_sfixed32_always_four_bytes() -> None: + assert len(encode_sfixed32(0)) == 4 + assert len(encode_sfixed32(-1)) == 4 + + +@mark.parametrize("value", [0, 1, -1, 127, -128, 2**15 - 1, -(2**15), 2**31 - 1, -(2**31)]) +def test_sfixed32_matches_struct(value: int) -> None: + assert encode_sfixed32(value) == pack(" None: + assert encode_fixed64(0) == b"\x00\x00\x00\x00\x00\x00\x00\x00" + + +def test_fixed64_one() -> None: + assert encode_fixed64(1) == b"\x01\x00\x00\x00\x00\x00\x00\x00" + + +def test_fixed64_max() -> None: + assert encode_fixed64(2**64 - 1) == b"\xff\xff\xff\xff\xff\xff\xff\xff" + + +def test_fixed64_always_eight_bytes() -> None: + assert len(encode_fixed64(0)) == 8 + assert len(encode_fixed64(2**64 - 1)) == 8 + + +@mark.parametrize( + "value", [0, 1, 255, 2**16 - 1, 2**32 - 1, 2**32, 2**48, 2**63, 2**64 - 1] +) +def test_fixed64_matches_struct(value: int) -> None: + assert encode_fixed64(value) == pack(" None: + assert encode_sfixed64(0) == b"\x00\x00\x00\x00\x00\x00\x00\x00" + + +def test_sfixed64_negative_one() -> None: + # -1 in 64-bit two's complement: all bytes 0xFF. + assert encode_sfixed64(-1) == b"\xff\xff\xff\xff\xff\xff\xff\xff" + + +def test_sfixed64_min() -> None: + # -2^63 = 0x8000000000000000 little-endian: 00 00 00 00 00 00 00 80 + assert encode_sfixed64(-(2**63)) == b"\x00\x00\x00\x00\x00\x00\x00\x80" + + +def test_sfixed64_max() -> None: + # 2^63-1 = 0x7FFFFFFFFFFFFFFF little-endian: FF FF FF FF FF FF FF 7F + assert encode_sfixed64(2**63 - 1) == b"\xff\xff\xff\xff\xff\xff\xff\x7f" + + +def test_sfixed64_always_eight_bytes() -> None: + assert len(encode_sfixed64(0)) == 8 + assert len(encode_sfixed64(-1)) == 8 + + +@mark.parametrize( + "value", + [0, 1, -1, 127, -128, 2**31 - 1, -(2**31), 2**32, -(2**32), 2**63 - 1, -(2**63)], +) +def test_sfixed64_matches_struct(value: int) -> None: + assert encode_sfixed64(value) == pack(" None: + assert encode_string("") == b"\x00" + + +def test_string_ascii_single_char() -> None: + assert encode_string("A") == b"\x01A" + + +def test_string_ascii_word() -> None: + assert encode_string("hi") == b"\x02hi" + + +def test_string_two_byte_utf8() -> None: + # "é" = U+00E9, UTF-8: 0xC3 0xA9 (2 bytes, not 1 character) + assert encode_string("é") == b"\x02\xc3\xa9" + + +def test_string_three_byte_utf8() -> None: + # "中" = U+4E2D, UTF-8: 0xE4 0xB8 0xAD (3 bytes) + assert encode_string("中") == b"\x03\xe4\xb8\xad" + + +def test_string_length_counts_bytes_not_chars() -> None: + # "日本語" is 3 characters but 9 UTF-8 bytes. + value = "日本語" + result = encode_string(value) + assert result[0] == 9 # byte count, not char count + assert result[1:] == value.encode("utf-8") + + +def test_string_two_byte_length_prefix() -> None: + # A string of 128 ASCII characters needs a 2-byte varint length prefix. + result = encode_string("a" * 128) + assert result[:2] == b"\x80\x01" + assert result[2:] == b"a" * 128 + + +@mark.parametrize( + "value", + ["", "a", "hello", "café", "Ünïcödé", "日本語", "中文", "\U0001F600", "hello 世界"], +) +def test_string_length_prefix_matches_utf8_bytecount(value: str) -> None: + utf8 = value.encode("utf-8") + result = encode_string(value) + assert result == encode_varint(len(utf8)) + utf8 + + +# ── encode_bytes ─────────────────────────────────────────────────────────────── + + +def test_bytes_empty() -> None: + assert encode_bytes(b"") == b"\x00" + + +def test_bytes_single_byte() -> None: + assert encode_bytes(b"\x42") == b"\x01\x42" + + +def test_bytes_high_bytes() -> None: + # Values >= 0x80 must pass through verbatim. + assert encode_bytes(b"\xff\x80") == b"\x02\xff\x80" + + +def test_bytes_null_bytes() -> None: + assert encode_bytes(b"\x00\x00\x00") == b"\x03\x00\x00\x00" + + +def test_bytes_all_byte_values() -> None: + # 256-byte payload; length prefix is encode_varint(256) = b'\x80\x02' + payload = bytes(range(256)) + result = encode_bytes(payload) + assert result[:2] == b"\x80\x02" + assert result[2:] == payload + + +def test_bytes_two_byte_length_prefix() -> None: + value = b"\xab" * 128 + result = encode_bytes(value) + assert result[:2] == b"\x80\x01" + assert result[2:] == value + + +@mark.parametrize( + "value", + [b"", b"\x00", b"\xff", b"hello", b"\x00\x01\x02\x03", b"\x80\x81\x82", b"\xfe\xff"], +) +def test_bytes_length_prefix_matches_payload_length(value: bytes) -> None: + result = encode_bytes(value) + assert result == encode_varint(len(value)) + value + + +# ── oracle — byte-for-byte comparison with google.protobuf ──────────────────── +# +# A proto2 message with one optional field per scalar type is used as the +# oracle. proto2 is chosen because it serialises every field that has been +# explicitly set, including fields whose value equals the type default (0, +# False, b"", ""), which proto3 would silently omit. +# +# For each function under test the pattern is: +# expected = encode_tag(field_number, wire_type) + encode_X(value) +# assert _ScalarMessage(**{field_name: value}).SerializeToString() == expected +# +# Field number assignments: +_F_UINT32 = 1 # wt 0 +_F_UINT64 = 2 # wt 0 +_F_BOOL = 3 # wt 0 +_F_INT32 = 4 # wt 0 +_F_INT64 = 5 # wt 0 +_F_SINT32 = 6 # wt 0 +_F_SINT64 = 7 # wt 0 +_F_FLOAT = 8 # wt 5 +_F_DOUBLE = 9 # wt 1 +_F_FIXED32 = 10 # wt 5 +_F_SFIXED32 = 11 # wt 5 +_F_FIXED64 = 12 # wt 1 +_F_SFIXED64 = 13 # wt 1 +_F_STRING = 14 # wt 2 +_F_BYTES = 15 # wt 2 + +_WT_VARINT = 0 +_WT_64BIT = 1 +_WT_LEN = 2 +_WT_32BIT = 5 + + +def _build_scalar_message_class(): + file_proto = descriptor_pb2.FileDescriptorProto() + file_proto.name = "opentelemetry_pyproto_test_scalars.proto" + file_proto.syntax = "proto2" + msg_proto = file_proto.message_type.add() + msg_proto.name = "ScalarMessage" + + T = descriptor_pb2.FieldDescriptorProto + + def _add(name, number, type_id): + f = msg_proto.field.add() + f.name = name + f.number = number + f.type = type_id + f.label = T.LABEL_OPTIONAL + + _add("uint32_field", _F_UINT32, T.TYPE_UINT32) + _add("uint64_field", _F_UINT64, T.TYPE_UINT64) + _add("bool_field", _F_BOOL, T.TYPE_BOOL) + _add("int32_field", _F_INT32, T.TYPE_INT32) + _add("int64_field", _F_INT64, T.TYPE_INT64) + _add("sint32_field", _F_SINT32, T.TYPE_SINT32) + _add("sint64_field", _F_SINT64, T.TYPE_SINT64) + _add("float_field", _F_FLOAT, T.TYPE_FLOAT) + _add("double_field", _F_DOUBLE, T.TYPE_DOUBLE) + _add("fixed32_field", _F_FIXED32, T.TYPE_FIXED32) + _add("sfixed32_field", _F_SFIXED32, T.TYPE_SFIXED32) + _add("fixed64_field", _F_FIXED64, T.TYPE_FIXED64) + _add("sfixed64_field", _F_SFIXED64, T.TYPE_SFIXED64) + _add("string_field", _F_STRING, T.TYPE_STRING) + _add("bytes_field", _F_BYTES, T.TYPE_BYTES) + + pool = descriptor_pool.DescriptorPool() + pool.Add(file_proto) + return message_factory.GetMessageClass(pool.FindMessageTypeByName("ScalarMessage")) + + +_ScalarMessage = _build_scalar_message_class() + + +def _s(field_name: str, value) -> bytes: + return _ScalarMessage(**{field_name: value}).SerializeToString() + + +# ── encode_uint32 oracle ─────────────────────────────────────────────────────── + +@mark.parametrize("value", [0, 1, 127, 128, 255, 300, 2**16, 2**32 - 1]) +def test_encode_uint32_matches_protobuf(value: int) -> None: + assert _s("uint32_field", value) == encode_tag(_F_UINT32, _WT_VARINT) + encode_uint32(value) + + +# ── encode_uint64 oracle ─────────────────────────────────────────────────────── + +@mark.parametrize("value", [0, 1, 127, 128, 2**32 - 1, 2**32, 2**63, 2**64 - 1]) +def test_encode_uint64_matches_protobuf(value: int) -> None: + assert _s("uint64_field", value) == encode_tag(_F_UINT64, _WT_VARINT) + encode_uint64(value) + + +# ── encode_bool oracle ───────────────────────────────────────────────────────── + +@mark.parametrize("value", [False, True]) +def test_encode_bool_matches_protobuf(value: bool) -> None: + assert _s("bool_field", value) == encode_tag(_F_BOOL, _WT_VARINT) + encode_bool(value) + + +# ── encode_int oracle (int32 / int64) ───────────────────────────────────────── + +@mark.parametrize("value", [0, 1, 127, 128, 2**31 - 1, -1, -2, -128, -(2**31)]) +def test_encode_int_int32_matches_protobuf(value: int) -> None: + assert _s("int32_field", value) == encode_tag(_F_INT32, _WT_VARINT) + encode_int(value) + + +@mark.parametrize("value", [0, 1, 2**63 - 1, -1, -(2**63)]) +def test_encode_int_int64_matches_protobuf(value: int) -> None: + assert _s("int64_field", value) == encode_tag(_F_INT64, _WT_VARINT) + encode_int(value) + + +# ── encode_sint32 oracle ─────────────────────────────────────────────────────── + +@mark.parametrize("value", [0, 1, -1, 2, -2, 150, -150, 2**31 - 1, -(2**31)]) +def test_encode_sint32_matches_protobuf(value: int) -> None: + assert _s("sint32_field", value) == encode_tag(_F_SINT32, _WT_VARINT) + encode_sint32(value) + + +# ── encode_sint64 oracle ─────────────────────────────────────────────────────── + +@mark.parametrize("value", [0, 1, -1, 150, -150, 2**63 - 1, -(2**63)]) +def test_encode_sint64_matches_protobuf(value: int) -> None: + assert _s("sint64_field", value) == encode_tag(_F_SINT64, _WT_VARINT) + encode_sint64(value) + + +# ── encode_float oracle ──────────────────────────────────────────────────────── + +@mark.parametrize("value", [0.0, 1.0, -1.0, 0.5, inf, -inf]) +def test_encode_float_matches_protobuf(value: float) -> None: + assert _s("float_field", value) == encode_tag(_F_FLOAT, _WT_32BIT) + encode_float(value) + + +# ── encode_double oracle ─────────────────────────────────────────────────────── + +@mark.parametrize("value", [0.0, 1.0, -1.0, pi, 1e100, inf, -inf]) +def test_encode_double_matches_protobuf(value: float) -> None: + assert _s("double_field", value) == encode_tag(_F_DOUBLE, _WT_64BIT) + encode_double(value) + + +# ── encode_fixed32 oracle ────────────────────────────────────────────────────── + +@mark.parametrize("value", [0, 1, 255, 2**16, 2**32 - 1]) +def test_encode_fixed32_matches_protobuf(value: int) -> None: + assert _s("fixed32_field", value) == encode_tag(_F_FIXED32, _WT_32BIT) + encode_fixed32(value) + + +# ── encode_sfixed32 oracle ───────────────────────────────────────────────────── + +@mark.parametrize("value", [0, 1, -1, 2**31 - 1, -(2**31)]) +def test_encode_sfixed32_matches_protobuf(value: int) -> None: + assert _s("sfixed32_field", value) == encode_tag(_F_SFIXED32, _WT_32BIT) + encode_sfixed32(value) + + +# ── encode_fixed64 oracle ────────────────────────────────────────────────────── + +@mark.parametrize("value", [0, 1, 2**32, 2**64 - 1]) +def test_encode_fixed64_matches_protobuf(value: int) -> None: + assert _s("fixed64_field", value) == encode_tag(_F_FIXED64, _WT_64BIT) + encode_fixed64(value) + + +# ── encode_sfixed64 oracle ───────────────────────────────────────────────────── + +@mark.parametrize("value", [0, 1, -1, 2**63 - 1, -(2**63)]) +def test_encode_sfixed64_matches_protobuf(value: int) -> None: + assert _s("sfixed64_field", value) == encode_tag(_F_SFIXED64, _WT_64BIT) + encode_sfixed64(value) + + +# ── encode_string oracle ─────────────────────────────────────────────────────── + +@mark.parametrize("value", ["", "a", "hello", "café", "日本語", "\U0001F600", "x" * 128]) +def test_encode_string_matches_protobuf(value: str) -> None: + assert _s("string_field", value) == encode_tag(_F_STRING, _WT_LEN) + encode_string(value) + + +# ── encode_bytes oracle ──────────────────────────────────────────────────────── + +@mark.parametrize("value", [b"", b"\x00", b"\xff", b"hello", b"\x80\x81\x82", b"\xab" * 128]) +def test_encode_bytes_matches_protobuf(value: bytes) -> None: + assert _s("bytes_field", value) == encode_tag(_F_BYTES, _WT_LEN) + encode_bytes(value) diff --git a/opentelemetry-pyproto/tests/_pyprotobuf/test_tag.py b/opentelemetry-pyproto/tests/_pyprotobuf/test_tag.py new file mode 100644 index 00000000000..df211f62b81 --- /dev/null +++ b/opentelemetry-pyproto/tests/_pyprotobuf/test_tag.py @@ -0,0 +1,152 @@ +# tests/test__tag.py +# +# encode_tag(field_number, wire_type) encodes (field_number << 3) | wire_type +# as a varint. These tests verify the formula and the varint encoding of the +# resulting tag integer, using hand-computed expected bytes. +# +# Wire type constants: +# 0 VARINT — int32, int64, uint32, uint64, bool, enum +# 1 64BIT — fixed64, sfixed64, double +# 2 LEN — string, bytes, embedded messages, packed repeated +# 5 32BIT — fixed32, sfixed32, float + +from google.protobuf import descriptor_pb2, descriptor_pool, message_factory +from pytest import mark + +from opentelemetry.pyproto._pyprotobuf import encode_tag, encode_varint + + +@mark.parametrize( + ("field_number", "wire_type"), + [ + (1, 0), # tag = 8 → 1-byte varint + (1, 1), # tag = 9 → 1-byte varint + (1, 2), # tag = 10 → 1-byte varint + (1, 5), # tag = 13 → 1-byte varint + (15, 0), # tag = 120 → last 1-byte tag for wire type 0 + (16, 0), # tag = 128 → first 2-byte tag for wire type 0 + (150, 0), # tag = 1200 → 2-byte varint + (1024, 0), # tag = 8192 → 2-byte varint + (1048576, 2), # tag = 8388610 → 4-byte varint + ], +) +def test_encode_tag_matches_formula(field_number: int, wire_type: int) -> None: + # encode_tag must produce the same bytes as encoding the tag integer directly. + tag_int = (field_number << 3) | wire_type + assert encode_tag(field_number, wire_type) == encode_varint(tag_int) + + +def test_field_1_wire_type_0() -> None: + # (1 << 3) | 0 = 8 → single byte 0x08 + assert encode_tag(1, 0) == b"\x08" + + +def test_field_1_wire_type_2() -> None: + # (1 << 3) | 2 = 10 → single byte 0x0A + assert encode_tag(1, 2) == b"\x0a" + + +def test_field_15_wire_type_0() -> None: + # (15 << 3) | 0 = 120 → single byte 0x78 (last 1-byte wt-0 tag) + assert encode_tag(15, 0) == b"\x78" + + +def test_field_16_wire_type_0() -> None: + # (16 << 3) | 0 = 128 → two bytes 0x80 0x01 (first 2-byte wt-0 tag) + assert encode_tag(16, 0) == b"\x80\x01" + + +def test_field_2_wire_type_1() -> None: + # (2 << 3) | 1 = 17 → single byte 0x11 + assert encode_tag(2, 1) == b"\x11" + + +def test_field_3_wire_type_5() -> None: + # (3 << 3) | 5 = 29 → single byte 0x1D + assert encode_tag(3, 5) == b"\x1d" + + +# ── oracle — byte-for-byte comparison with google.protobuf ──────────────────── +# +# A proto2 message with fields at distinct positions and wire types is used as +# the oracle. For each (field_number, wire_type) pair we serialise a message +# with only that field set and verify that encode_tag(field_number, wire_type) +# matches the first len(tag) bytes of the serialised output. +# +# Field number / tag varint length: +# field 1, wt 0: tag = 8 → 1-byte (last before multi-byte for wt 0) +# field 10, wt 1: tag = 81 → 1-byte (wire type 1) +# field 11, wt 2: tag = 90 → 1-byte (wire type 2) +# field 12, wt 5: tag = 101 → 1-byte (wire type 5) +# field 15, wt 0: tag = 120 → 1-byte (last 1-byte wt-0 tag) +# field 16, wt 0: tag = 128 → 2-byte (first 2-byte wt-0 tag) +# field 150, wt 0: tag = 1200 → 2-byte +# field 1048576, wt 2: tag = 8388610 → 4-byte + +_WT_VARINT = 0 +_WT_64BIT = 1 +_WT_LEN = 2 +_WT_32BIT = 5 + +_F_UINT64 = 1 +_F_FIXED64 = 10 +_F_STRING = 11 +_F_FIXED32 = 12 +_F_UINT64_15 = 15 +_F_UINT64_16 = 16 +_F_UINT64_150 = 150 +_F_STRING_BIG = 1048576 + + +def _build_tag_message_class(): + file_proto = descriptor_pb2.FileDescriptorProto() + file_proto.name = "opentelemetry_pyproto_test_tag.proto" + file_proto.syntax = "proto2" + msg_proto = file_proto.message_type.add() + msg_proto.name = "TagMessage" + + T = descriptor_pb2.FieldDescriptorProto + + def _add(name, number, type_id): + f = msg_proto.field.add() + f.name = name + f.number = number + f.type = type_id + f.label = T.LABEL_OPTIONAL + + _add("uint64_field", _F_UINT64, T.TYPE_UINT64) + _add("fixed64_field", _F_FIXED64, T.TYPE_FIXED64) + _add("string_field", _F_STRING, T.TYPE_STRING) + _add("fixed32_field", _F_FIXED32, T.TYPE_FIXED32) + _add("uint64_field_15", _F_UINT64_15, T.TYPE_UINT64) + _add("uint64_field_16", _F_UINT64_16, T.TYPE_UINT64) + _add("uint64_field_150", _F_UINT64_150, T.TYPE_UINT64) + _add("string_field_big", _F_STRING_BIG, T.TYPE_STRING) + + pool = descriptor_pool.DescriptorPool() + pool.Add(file_proto) + return message_factory.GetMessageClass(pool.FindMessageTypeByName("TagMessage")) + + +_TagMessage = _build_tag_message_class() + + +@mark.parametrize( + ("field_name", "field_number", "wire_type", "field_value"), + [ + ("uint64_field", _F_UINT64, _WT_VARINT, 1 ), + ("fixed64_field", _F_FIXED64, _WT_64BIT, 1 ), + ("string_field", _F_STRING, _WT_LEN, "x"), + ("fixed32_field", _F_FIXED32, _WT_32BIT, 1 ), + ("uint64_field_15", _F_UINT64_15, _WT_VARINT, 1 ), + ("uint64_field_16", _F_UINT64_16, _WT_VARINT, 1 ), + ("uint64_field_150", _F_UINT64_150, _WT_VARINT, 1 ), + ("string_field_big", _F_STRING_BIG, _WT_LEN, "x"), + ], +) +def test_encode_tag_matches_protobuf( + field_name: str, field_number: int, wire_type: int, field_value +) -> None: + serialized = _TagMessage(**{field_name: field_value}).SerializeToString() + tag = encode_tag(field_number, wire_type) + assert serialized[: len(tag)] == tag diff --git a/opentelemetry-pyproto/tests/_pyprotobuf/test_varint.py b/opentelemetry-pyproto/tests/_pyprotobuf/test_varint.py new file mode 100644 index 00000000000..6044a85e6fa --- /dev/null +++ b/opentelemetry-pyproto/tests/_pyprotobuf/test_varint.py @@ -0,0 +1,115 @@ +# tests/test__varint.py + +from google.protobuf import descriptor_pb2, descriptor_pool, message_factory +from pytest import mark, raises + +from opentelemetry.pyproto._pyprotobuf import encode_tag, encode_varint + + +def test_zero() -> None: + assert encode_varint(0) == b"\x00" + + +def test_single_byte_max() -> None: + assert encode_varint(127) == b"\x7f" + + +def test_first_two_byte_value() -> None: + assert encode_varint(128) == b"\x80\x01" + + +def test_150() -> None: + assert encode_varint(150) == b"\x96\x01" + + +def test_300() -> None: + assert encode_varint(300) == b"\xac\x02" + + +def test_uint32_max() -> None: + assert encode_varint(2**32 - 1) == b"\xff\xff\xff\xff\x0f" + + +def test_uint64_max() -> None: + assert encode_varint(2**64 - 1) == b"\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01" + + +def test_rejects_negative_values() -> None: + with raises(ValueError, match="varint values must be non-negative"): + encode_varint(-1) + + +def test_two_byte_boundary_low() -> None: + # 16383 = 0x3FFF — the largest value that fits in two varint bytes. + # 7-bit groups: 0x7F (lower), 0x7F (upper, no continuation bit). + assert encode_varint(16_383) == b"\xff\x7f" + + +def test_three_byte_boundary_low() -> None: + # 16384 = 0x4000 — the first value that requires three varint bytes. + assert encode_varint(16_384) == b"\x80\x80\x01" + + +def test_one() -> None: + # 1 fits in a single byte; no continuation bit needed. + assert encode_varint(1) == b"\x01" + + +# ── oracle — byte-for-byte comparison with google.protobuf ──────────────────── +# +# A proto2 message with a single uint64 field is used as the oracle. proto2 +# is chosen because it serialises the field even when its value is the type +# default (0). uint64 covers the full [0, 2^64-1] varint range. +# +# The serialised message for a single field is exactly: +# tag (varint) + value (varint) +# +# Asserting that encode_tag(field, wt) + encode_varint(value) equals the +# serialised message verifies that our varint encoder produces the exact same +# bytes as google.protobuf for every tested value. + +_FIELD = 1 +_WT = 0 # wire type 0 — varint + + +def _build_varint_message_class(): + file_proto = descriptor_pb2.FileDescriptorProto() + file_proto.name = "opentelemetry_pyproto_test_varint.proto" + file_proto.syntax = "proto2" + msg_proto = file_proto.message_type.add() + msg_proto.name = "VarintMessage" + f = msg_proto.field.add() + f.name = "uint64_field" + f.number = _FIELD + f.type = descriptor_pb2.FieldDescriptorProto.TYPE_UINT64 + f.label = descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL + pool = descriptor_pool.DescriptorPool() + pool.Add(file_proto) + return message_factory.GetMessageClass(pool.FindMessageTypeByName("VarintMessage")) + + +_VarintMessage = _build_varint_message_class() + + +@mark.parametrize( + "value", + [ + 0, + 1, + 127, + 128, + 150, + 300, + 16_383, + 16_384, + 2**16, + 2**28, + 2**32 - 1, + 2**32, + 2**63 - 1, + 2**64 - 1, + ], +) +def test_encode_varint_matches_protobuf(value: int) -> None: + expected = encode_tag(_FIELD, _WT) + encode_varint(value) + assert _VarintMessage(uint64_field=value).SerializeToString() == expected diff --git a/opentelemetry-pyproto/tests/collector/__init__.py b/opentelemetry-pyproto/tests/collector/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/tests/collector/logs/__init__.py b/opentelemetry-pyproto/tests/collector/logs/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/tests/collector/logs/v1/__init__.py b/opentelemetry-pyproto/tests/collector/logs/v1/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/tests/collector/logs/v1/test_logs_service_pypb2.py b/opentelemetry-pyproto/tests/collector/logs/v1/test_logs_service_pypb2.py new file mode 100644 index 00000000000..ba1d0156bd5 --- /dev/null +++ b/opentelemetry-pyproto/tests/collector/logs/v1/test_logs_service_pypb2.py @@ -0,0 +1,67 @@ +from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ( + ExportLogsServiceRequest as ProtoExportLogsServiceRequest, + ExportLogsServiceResponse as ProtoExportLogsServiceResponse, +) +from opentelemetry.proto.logs.v1.logs_pb2 import ( + ResourceLogs as ProtoResourceLogs, +) + +from opentelemetry.pyproto.collector.logs.v1.logs_service_pypb2 import ( + ExportLogsServiceRequest, + ExportLogsServiceResponse, +) +from opentelemetry.pyproto.logs.v1.logs_pypb2 import ResourceLogs + + +def test_export_response_empty() -> None: + assert ExportLogsServiceResponse().SerializeToString() == b"" + + +def test_export_response_matches_proto() -> None: + assert ( + ExportLogsServiceResponse().SerializeToString() + == ProtoExportLogsServiceResponse().SerializeToString() + ) + + +def test_export_request_empty() -> None: + assert ExportLogsServiceRequest().SerializeToString() == b"" + + +def test_export_request_empty_matches_proto() -> None: + assert ( + ExportLogsServiceRequest().SerializeToString() + == ProtoExportLogsServiceRequest().SerializeToString() + ) + + +def test_export_request_with_empty_resource_logs() -> None: + our = ExportLogsServiceRequest(resource_logs=[ResourceLogs()]) + proto = ProtoExportLogsServiceRequest(resource_logs=[ProtoResourceLogs()]) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_export_request_with_schema_url() -> None: + our = ExportLogsServiceRequest( + resource_logs=[ResourceLogs(schema_url="https://example.com/schema")] + ) + proto = ProtoExportLogsServiceRequest( + resource_logs=[ProtoResourceLogs(schema_url="https://example.com/schema")] + ) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_export_request_multiple_resource_logs() -> None: + our = ExportLogsServiceRequest( + resource_logs=[ + ResourceLogs(schema_url="schema-a"), + ResourceLogs(schema_url="schema-b"), + ] + ) + proto = ProtoExportLogsServiceRequest( + resource_logs=[ + ProtoResourceLogs(schema_url="schema-a"), + ProtoResourceLogs(schema_url="schema-b"), + ] + ) + assert our.SerializeToString() == proto.SerializeToString() diff --git a/opentelemetry-pyproto/tests/collector/metrics/__init__.py b/opentelemetry-pyproto/tests/collector/metrics/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/tests/collector/metrics/v1/__init__.py b/opentelemetry-pyproto/tests/collector/metrics/v1/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/tests/collector/metrics/v1/test_metrics_service_pypb2.py b/opentelemetry-pyproto/tests/collector/metrics/v1/test_metrics_service_pypb2.py new file mode 100644 index 00000000000..38d495f3f4c --- /dev/null +++ b/opentelemetry-pyproto/tests/collector/metrics/v1/test_metrics_service_pypb2.py @@ -0,0 +1,67 @@ +from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ( + ExportMetricsServiceRequest as ProtoExportMetricsServiceRequest, + ExportMetricsServiceResponse as ProtoExportMetricsServiceResponse, +) +from opentelemetry.proto.metrics.v1.metrics_pb2 import ( + ResourceMetrics as ProtoResourceMetrics, +) + +from opentelemetry.pyproto.collector.metrics.v1.metrics_service_pypb2 import ( + ExportMetricsServiceRequest, + ExportMetricsServiceResponse, +) +from opentelemetry.pyproto.metrics.v1.metrics_pypb2 import ResourceMetrics + + +def test_export_response_empty() -> None: + assert ExportMetricsServiceResponse().SerializeToString() == b"" + + +def test_export_response_matches_proto() -> None: + assert ( + ExportMetricsServiceResponse().SerializeToString() + == ProtoExportMetricsServiceResponse().SerializeToString() + ) + + +def test_export_request_empty() -> None: + assert ExportMetricsServiceRequest().SerializeToString() == b"" + + +def test_export_request_empty_matches_proto() -> None: + assert ( + ExportMetricsServiceRequest().SerializeToString() + == ProtoExportMetricsServiceRequest().SerializeToString() + ) + + +def test_export_request_with_empty_resource_metrics() -> None: + our = ExportMetricsServiceRequest(resource_metrics=[ResourceMetrics()]) + proto = ProtoExportMetricsServiceRequest(resource_metrics=[ProtoResourceMetrics()]) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_export_request_with_schema_url() -> None: + our = ExportMetricsServiceRequest( + resource_metrics=[ResourceMetrics(schema_url="https://example.com/schema")] + ) + proto = ProtoExportMetricsServiceRequest( + resource_metrics=[ProtoResourceMetrics(schema_url="https://example.com/schema")] + ) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_export_request_multiple_resource_metrics() -> None: + our = ExportMetricsServiceRequest( + resource_metrics=[ + ResourceMetrics(schema_url="schema-a"), + ResourceMetrics(schema_url="schema-b"), + ] + ) + proto = ProtoExportMetricsServiceRequest( + resource_metrics=[ + ProtoResourceMetrics(schema_url="schema-a"), + ProtoResourceMetrics(schema_url="schema-b"), + ] + ) + assert our.SerializeToString() == proto.SerializeToString() diff --git a/opentelemetry-pyproto/tests/collector/trace/__init__.py b/opentelemetry-pyproto/tests/collector/trace/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/tests/collector/trace/v1/__init__.py b/opentelemetry-pyproto/tests/collector/trace/v1/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/tests/collector/trace/v1/test_trace_service_pypb2.py b/opentelemetry-pyproto/tests/collector/trace/v1/test_trace_service_pypb2.py new file mode 100644 index 00000000000..895a4265af9 --- /dev/null +++ b/opentelemetry-pyproto/tests/collector/trace/v1/test_trace_service_pypb2.py @@ -0,0 +1,67 @@ +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest as ProtoExportTraceServiceRequest, + ExportTraceServiceResponse as ProtoExportTraceServiceResponse, +) +from opentelemetry.proto.trace.v1.trace_pb2 import ( + ResourceSpans as ProtoResourceSpans, +) + +from opentelemetry.pyproto.collector.trace.v1.trace_service_pypb2 import ( + ExportTraceServiceRequest, + ExportTraceServiceResponse, +) +from opentelemetry.pyproto.trace.v1.trace_pypb2 import ResourceSpans + + +def test_export_response_empty() -> None: + assert ExportTraceServiceResponse().SerializeToString() == b"" + + +def test_export_response_matches_proto() -> None: + assert ( + ExportTraceServiceResponse().SerializeToString() + == ProtoExportTraceServiceResponse().SerializeToString() + ) + + +def test_export_request_empty() -> None: + assert ExportTraceServiceRequest().SerializeToString() == b"" + + +def test_export_request_empty_matches_proto() -> None: + assert ( + ExportTraceServiceRequest().SerializeToString() + == ProtoExportTraceServiceRequest().SerializeToString() + ) + + +def test_export_request_with_empty_resource_spans() -> None: + our = ExportTraceServiceRequest(resource_spans=[ResourceSpans()]) + proto = ProtoExportTraceServiceRequest(resource_spans=[ProtoResourceSpans()]) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_export_request_with_schema_url() -> None: + our = ExportTraceServiceRequest( + resource_spans=[ResourceSpans(schema_url="https://example.com/schema")] + ) + proto = ProtoExportTraceServiceRequest( + resource_spans=[ProtoResourceSpans(schema_url="https://example.com/schema")] + ) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_export_request_multiple_resource_spans() -> None: + our = ExportTraceServiceRequest( + resource_spans=[ + ResourceSpans(schema_url="schema-a"), + ResourceSpans(schema_url="schema-b"), + ] + ) + proto = ProtoExportTraceServiceRequest( + resource_spans=[ + ProtoResourceSpans(schema_url="schema-a"), + ProtoResourceSpans(schema_url="schema-b"), + ] + ) + assert our.SerializeToString() == proto.SerializeToString() diff --git a/opentelemetry-pyproto/tests/common/__init__.py b/opentelemetry-pyproto/tests/common/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/tests/common/v1/__init__.py b/opentelemetry-pyproto/tests/common/v1/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/tests/common/v1/test_common_pypb2.py b/opentelemetry-pyproto/tests/common/v1/test_common_pypb2.py new file mode 100644 index 00000000000..5cff268ca77 --- /dev/null +++ b/opentelemetry-pyproto/tests/common/v1/test_common_pypb2.py @@ -0,0 +1,188 @@ +from opentelemetry.proto.common.v1.common_pb2 import ( + AnyValue as ProtoAnyValue, + ArrayValue as ProtoArrayValue, + InstrumentationScope as ProtoInstrumentationScope, + KeyValue as ProtoKeyValue, + KeyValueList as ProtoKeyValueList, +) + +from opentelemetry.pyproto.common.v1.common_pypb2 import ( + AnyValue, + ArrayValue, + InstrumentationScope, + KeyValue, + KeyValueList, +) + + +# ── AnyValue ────────────────────────────────────────────────────────────────── + +def test_any_value_empty() -> None: + assert AnyValue().SerializeToString() == b"" + + +def test_any_value_empty_matches_proto() -> None: + assert AnyValue().SerializeToString() == ProtoAnyValue().SerializeToString() + + +def test_any_value_string() -> None: + our = AnyValue(string_value="hello") + proto = ProtoAnyValue(string_value="hello") + assert our.SerializeToString() == proto.SerializeToString() + + +def test_any_value_bool_true() -> None: + our = AnyValue(bool_value=True) + proto = ProtoAnyValue(bool_value=True) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_any_value_bool_false() -> None: + our = AnyValue(bool_value=False) + proto = ProtoAnyValue(bool_value=False) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_any_value_int() -> None: + our = AnyValue(int_value=42) + proto = ProtoAnyValue(int_value=42) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_any_value_int_negative() -> None: + our = AnyValue(int_value=-1) + proto = ProtoAnyValue(int_value=-1) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_any_value_double() -> None: + our = AnyValue(double_value=3.14) + proto = ProtoAnyValue(double_value=3.14) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_any_value_bytes() -> None: + our = AnyValue(bytes_value=b"\x01\x02\x03") + proto = ProtoAnyValue(bytes_value=b"\x01\x02\x03") + assert our.SerializeToString() == proto.SerializeToString() + + +def test_any_value_array_empty() -> None: + our = AnyValue(array_value=ArrayValue()) + proto = ProtoAnyValue(array_value=ProtoArrayValue()) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_any_value_kvlist_empty() -> None: + our = AnyValue(kvlist_value=KeyValueList()) + proto = ProtoAnyValue(kvlist_value=ProtoKeyValueList()) + assert our.SerializeToString() == proto.SerializeToString() + + +# ── ArrayValue ──────────────────────────────────────────────────────────────── + +def test_array_value_empty() -> None: + assert ArrayValue().SerializeToString() == b"" + + +def test_array_value_empty_matches_proto() -> None: + assert ArrayValue().SerializeToString() == ProtoArrayValue().SerializeToString() + + +def test_array_value_with_elements() -> None: + our = ArrayValue(values=[AnyValue(string_value="a"), AnyValue(int_value=1)]) + proto = ProtoArrayValue(values=[ProtoAnyValue(string_value="a"), ProtoAnyValue(int_value=1)]) + assert our.SerializeToString() == proto.SerializeToString() + + +# ── KeyValueList ────────────────────────────────────────────────────────────── + +def test_kvlist_empty() -> None: + assert KeyValueList().SerializeToString() == b"" + + +def test_kvlist_empty_matches_proto() -> None: + assert KeyValueList().SerializeToString() == ProtoKeyValueList().SerializeToString() + + +def test_kvlist_with_values() -> None: + our = KeyValueList(values=[KeyValue(key="k", value=AnyValue(string_value="v"))]) + proto = ProtoKeyValueList(values=[ProtoKeyValue(key="k", value=ProtoAnyValue(string_value="v"))]) + assert our.SerializeToString() == proto.SerializeToString() + + +# ── KeyValue ────────────────────────────────────────────────────────────────── + +def test_key_value_empty() -> None: + assert KeyValue().SerializeToString() == b"" + + +def test_key_value_empty_matches_proto() -> None: + assert KeyValue().SerializeToString() == ProtoKeyValue().SerializeToString() + + +def test_key_value_key_only() -> None: + our = KeyValue(key="mykey") + proto = ProtoKeyValue(key="mykey") + assert our.SerializeToString() == proto.SerializeToString() + + +def test_key_value_string() -> None: + our = KeyValue(key="k", value=AnyValue(string_value="v")) + proto = ProtoKeyValue(key="k", value=ProtoAnyValue(string_value="v")) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_key_value_int() -> None: + our = KeyValue(key="count", value=AnyValue(int_value=42)) + proto = ProtoKeyValue(key="count", value=ProtoAnyValue(int_value=42)) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_key_value_double() -> None: + our = KeyValue(key="ratio", value=AnyValue(double_value=0.5)) + proto = ProtoKeyValue(key="ratio", value=ProtoAnyValue(double_value=0.5)) + assert our.SerializeToString() == proto.SerializeToString() + + +# ── InstrumentationScope ────────────────────────────────────────────────────── + +def test_instrumentation_scope_empty() -> None: + assert InstrumentationScope().SerializeToString() == b"" + + +def test_instrumentation_scope_empty_matches_proto() -> None: + assert ( + InstrumentationScope().SerializeToString() + == ProtoInstrumentationScope().SerializeToString() + ) + + +def test_instrumentation_scope_name() -> None: + our = InstrumentationScope(name="mylib") + proto = ProtoInstrumentationScope(name="mylib") + assert our.SerializeToString() == proto.SerializeToString() + + +def test_instrumentation_scope_name_version() -> None: + our = InstrumentationScope(name="mylib", version="1.2.3") + proto = ProtoInstrumentationScope(name="mylib", version="1.2.3") + assert our.SerializeToString() == proto.SerializeToString() + + +def test_instrumentation_scope_with_attributes() -> None: + attr = KeyValue(key="env", value=AnyValue(string_value="prod")) + proto_attr = ProtoKeyValue(key="env", value=ProtoAnyValue(string_value="prod")) + our = InstrumentationScope( + name="mylib", + version="1.0", + attributes=[attr], + dropped_attributes_count=3, + ) + proto = ProtoInstrumentationScope( + name="mylib", + version="1.0", + attributes=[proto_attr], + dropped_attributes_count=3, + ) + assert our.SerializeToString() == proto.SerializeToString() diff --git a/opentelemetry-pyproto/tests/logs/__init__.py b/opentelemetry-pyproto/tests/logs/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/tests/logs/v1/__init__.py b/opentelemetry-pyproto/tests/logs/v1/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/tests/logs/v1/test_logs_pypb2.py b/opentelemetry-pyproto/tests/logs/v1/test_logs_pypb2.py new file mode 100644 index 00000000000..13422317e68 --- /dev/null +++ b/opentelemetry-pyproto/tests/logs/v1/test_logs_pypb2.py @@ -0,0 +1,155 @@ +from opentelemetry.proto.common.v1.common_pb2 import ( + AnyValue as ProtoAnyValue, + InstrumentationScope as ProtoInstrumentationScope, + KeyValue as ProtoKeyValue, +) +from opentelemetry.proto.logs.v1.logs_pb2 import ( + LogRecord as ProtoLogRecord, + ResourceLogs as ProtoResourceLogs, + ScopeLogs as ProtoScopeLogs, +) +from opentelemetry.proto.resource.v1.resource_pb2 import Resource as ProtoResource + +from opentelemetry.pyproto.common.v1.common_pypb2 import ( + AnyValue, + InstrumentationScope, + KeyValue, +) +from opentelemetry.pyproto.logs.v1.logs_pypb2 import LogRecord, ResourceLogs, ScopeLogs +from opentelemetry.pyproto.resource.v1.resource_pypb2 import Resource + + +# ── LogRecord ───────────────────────────────────────────────────────────────── + +def test_log_record_empty() -> None: + assert LogRecord().SerializeToString() == b"" + + +def test_log_record_empty_matches_proto() -> None: + assert LogRecord().SerializeToString() == ProtoLogRecord().SerializeToString() + + +def test_log_record_severity() -> None: + our = LogRecord(severity_number=9, severity_text="INFO") + proto = ProtoLogRecord(severity_number=9, severity_text="INFO") + assert our.SerializeToString() == proto.SerializeToString() + + +def test_log_record_with_body() -> None: + our = LogRecord(body=AnyValue(string_value="hello world")) + proto = ProtoLogRecord(body=ProtoAnyValue(string_value="hello world")) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_log_record_with_trace_context() -> None: + trace_id = b"\x01" * 16 + span_id = b"\x02" * 8 + our = LogRecord( + time_unix_nano=1_000_000_000, + trace_id=trace_id, + span_id=span_id, + observed_time_unix_nano=2_000_000_000, + ) + proto = ProtoLogRecord( + time_unix_nano=1_000_000_000, + trace_id=trace_id, + span_id=span_id, + observed_time_unix_nano=2_000_000_000, + ) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_log_record_with_attributes() -> None: + attr = KeyValue(key="env", value=AnyValue(string_value="prod")) + proto_attr = ProtoKeyValue(key="env", value=ProtoAnyValue(string_value="prod")) + our = LogRecord( + body=AnyValue(string_value="msg"), + attributes=[attr], + dropped_attributes_count=1, + ) + proto = ProtoLogRecord( + body=ProtoAnyValue(string_value="msg"), + attributes=[proto_attr], + dropped_attributes_count=1, + ) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_log_record_event_name() -> None: + our = LogRecord(event_name="user.click") + proto = ProtoLogRecord(event_name="user.click") + assert our.SerializeToString() == proto.SerializeToString() + + +def test_log_record_flags() -> None: + our = LogRecord(flags=1) + proto = ProtoLogRecord(flags=1) + assert our.SerializeToString() == proto.SerializeToString() + + +# ── ScopeLogs ───────────────────────────────────────────────────────────────── + +def test_scope_logs_empty() -> None: + assert ScopeLogs().SerializeToString() == b"" + + +def test_scope_logs_empty_matches_proto() -> None: + assert ScopeLogs().SerializeToString() == ProtoScopeLogs().SerializeToString() + + +def test_scope_logs_schema_url() -> None: + our = ScopeLogs(schema_url="https://example.com/schema") + proto = ProtoScopeLogs(schema_url="https://example.com/schema") + assert our.SerializeToString() == proto.SerializeToString() + + +def test_scope_logs_with_scope() -> None: + scope = InstrumentationScope(name="mylib", version="1.0") + proto_scope = ProtoInstrumentationScope(name="mylib", version="1.0") + our = ScopeLogs(scope=scope) + proto = ProtoScopeLogs(scope=proto_scope) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_scope_logs_with_scope_and_records() -> None: + scope = InstrumentationScope(name="mylib", version="1.0") + proto_scope = ProtoInstrumentationScope(name="mylib", version="1.0") + rec = LogRecord(severity_text="WARN", severity_number=13) + proto_rec = ProtoLogRecord(severity_text="WARN", severity_number=13) + our = ScopeLogs(scope=scope, log_records=[rec], schema_url="s") + proto = ProtoScopeLogs(scope=proto_scope, log_records=[proto_rec], schema_url="s") + assert our.SerializeToString() == proto.SerializeToString() + + +# ── ResourceLogs ────────────────────────────────────────────────────────────── + +def test_resource_logs_empty() -> None: + assert ResourceLogs().SerializeToString() == b"" + + +def test_resource_logs_empty_matches_proto() -> None: + assert ResourceLogs().SerializeToString() == ProtoResourceLogs().SerializeToString() + + +def test_resource_logs_schema_url() -> None: + our = ResourceLogs(schema_url="https://example.com") + proto = ProtoResourceLogs(schema_url="https://example.com") + assert our.SerializeToString() == proto.SerializeToString() + + +def test_resource_logs_with_resource() -> None: + res = Resource(attributes=[KeyValue(key="k", value=AnyValue(string_value="v"))]) + proto_res = ProtoResource(attributes=[ProtoKeyValue(key="k", value=ProtoAnyValue(string_value="v"))]) + our = ResourceLogs(resource=res, schema_url="s") + proto = ProtoResourceLogs(resource=proto_res, schema_url="s") + assert our.SerializeToString() == proto.SerializeToString() + + +def test_resource_logs_with_scope_logs() -> None: + rec = LogRecord(severity_text="INFO", severity_number=9) + proto_rec = ProtoLogRecord(severity_text="INFO", severity_number=9) + sl = ScopeLogs(log_records=[rec]) + proto_sl = ProtoScopeLogs(log_records=[proto_rec]) + our = ResourceLogs(scope_logs=[sl]) + proto = ProtoResourceLogs(scope_logs=[proto_sl]) + assert our.SerializeToString() == proto.SerializeToString() diff --git a/opentelemetry-pyproto/tests/metrics/__init__.py b/opentelemetry-pyproto/tests/metrics/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/tests/metrics/v1/__init__.py b/opentelemetry-pyproto/tests/metrics/v1/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/tests/metrics/v1/test_metrics_pypb2.py b/opentelemetry-pyproto/tests/metrics/v1/test_metrics_pypb2.py new file mode 100644 index 00000000000..306442cb406 --- /dev/null +++ b/opentelemetry-pyproto/tests/metrics/v1/test_metrics_pypb2.py @@ -0,0 +1,527 @@ +from opentelemetry.proto.common.v1.common_pb2 import ( + AnyValue as ProtoAnyValue, + InstrumentationScope as ProtoInstrumentationScope, + KeyValue as ProtoKeyValue, +) +from opentelemetry.proto.metrics.v1.metrics_pb2 import ( + Exemplar as ProtoExemplar, + ExponentialHistogram as ProtoExponentialHistogram, + ExponentialHistogramDataPoint as ProtoExponentialHistogramDataPoint, + Gauge as ProtoGauge, + Histogram as ProtoHistogram, + HistogramDataPoint as ProtoHistogramDataPoint, + Metric as ProtoMetric, + NumberDataPoint as ProtoNumberDataPoint, + ResourceMetrics as ProtoResourceMetrics, + ScopeMetrics as ProtoScopeMetrics, + Sum as ProtoSum, + Summary as ProtoSummary, + SummaryDataPoint as ProtoSummaryDataPoint, +) +from opentelemetry.proto.resource.v1.resource_pb2 import Resource as ProtoResource + +from opentelemetry.pyproto.common.v1.common_pypb2 import ( + AnyValue, + InstrumentationScope, + KeyValue, +) +from opentelemetry.pyproto.metrics.v1.metrics_pypb2 import ( + Exemplar, + ExponentialHistogram, + ExponentialHistogramDataPoint, + Gauge, + Histogram, + HistogramDataPoint, + Metric, + NumberDataPoint, + ResourceMetrics, + ScopeMetrics, + Sum, + Summary, + SummaryDataPoint, +) +from opentelemetry.pyproto.resource.v1.resource_pypb2 import Resource + + +# ── Exemplar ────────────────────────────────────────────────────────────────── + +def test_exemplar_empty() -> None: + assert Exemplar().SerializeToString() == b"" + + +def test_exemplar_empty_matches_proto() -> None: + assert Exemplar().SerializeToString() == ProtoExemplar().SerializeToString() + + +def test_exemplar_as_double() -> None: + our = Exemplar(as_double=1.5, time_unix_nano=1_000) + proto = ProtoExemplar(as_double=1.5, time_unix_nano=1_000) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_exemplar_as_int() -> None: + our = Exemplar(as_int=42, time_unix_nano=2_000) + proto = ProtoExemplar(as_int=42, time_unix_nano=2_000) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_exemplar_with_span_trace_id() -> None: + our = Exemplar(as_double=3.0, span_id=b"\x01" * 8, trace_id=b"\x02" * 16) + proto = ProtoExemplar(as_double=3.0, span_id=b"\x01" * 8, trace_id=b"\x02" * 16) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_exemplar_with_filtered_attributes() -> None: + # filtered_attributes is field 7; test it alone to avoid field-ordering + # differences between our fixed-order encoding and proto's ascending order. + attr = KeyValue(key="k", value=AnyValue(string_value="v")) + proto_attr = ProtoKeyValue(key="k", value=ProtoAnyValue(string_value="v")) + our = Exemplar(filtered_attributes=[attr]) + proto = ProtoExemplar(filtered_attributes=[proto_attr]) + assert our.SerializeToString() == proto.SerializeToString() + + +# ── NumberDataPoint ─────────────────────────────────────────────────────────── + +def test_number_data_point_empty() -> None: + assert NumberDataPoint().SerializeToString() == b"" + + +def test_number_data_point_empty_matches_proto() -> None: + assert NumberDataPoint().SerializeToString() == ProtoNumberDataPoint().SerializeToString() + + +def test_number_data_point_as_double() -> None: + our = NumberDataPoint(as_double=1.5, time_unix_nano=1_000_000) + proto = ProtoNumberDataPoint(as_double=1.5, time_unix_nano=1_000_000) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_number_data_point_as_int() -> None: + our = NumberDataPoint(as_int=42, time_unix_nano=1_000_000) + proto = ProtoNumberDataPoint(as_int=42, time_unix_nano=1_000_000) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_number_data_point_with_attributes() -> None: + # attributes is field 7, flags is field 8 — both high-numbered and in the + # right relative order in our encoding, so no ordering difference with proto. + attr = KeyValue(key="env", value=AnyValue(string_value="prod")) + proto_attr = ProtoKeyValue(key="env", value=ProtoAnyValue(string_value="prod")) + our = NumberDataPoint(attributes=[attr], flags=1) + proto = ProtoNumberDataPoint(attributes=[proto_attr], flags=1) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_number_data_point_with_exemplar() -> None: + our = NumberDataPoint( + as_double=5.0, + start_time_unix_nano=1_000, + time_unix_nano=2_000, + exemplars=[Exemplar(as_double=5.0)], + ) + proto = ProtoNumberDataPoint( + as_double=5.0, + start_time_unix_nano=1_000, + time_unix_nano=2_000, + exemplars=[ProtoExemplar(as_double=5.0)], + ) + assert our.SerializeToString() == proto.SerializeToString() + + +# ── Gauge ───────────────────────────────────────────────────────────────────── + +def test_gauge_empty() -> None: + assert Gauge().SerializeToString() == b"" + + +def test_gauge_empty_matches_proto() -> None: + assert Gauge().SerializeToString() == ProtoGauge().SerializeToString() + + +def test_gauge_with_data_point() -> None: + our = Gauge(data_points=[NumberDataPoint(as_double=1.5)]) + proto = ProtoGauge(data_points=[ProtoNumberDataPoint(as_double=1.5)]) + assert our.SerializeToString() == proto.SerializeToString() + + +# ── Sum ─────────────────────────────────────────────────────────────────────── + +def test_sum_empty() -> None: + assert Sum().SerializeToString() == b"" + + +def test_sum_empty_matches_proto() -> None: + assert Sum().SerializeToString() == ProtoSum().SerializeToString() + + +def test_sum_with_data_and_temporality() -> None: + our = Sum( + data_points=[NumberDataPoint(as_int=10)], + aggregation_temporality=1, + is_monotonic=True, + ) + proto = ProtoSum( + data_points=[ProtoNumberDataPoint(as_int=10)], + aggregation_temporality=1, + is_monotonic=True, + ) + assert our.SerializeToString() == proto.SerializeToString() + + +# ── HistogramDataPoint ──────────────────────────────────────────────────────── + +def test_histogram_data_point_empty() -> None: + assert HistogramDataPoint().SerializeToString() == b"" + + +def test_histogram_data_point_empty_matches_proto() -> None: + assert ( + HistogramDataPoint().SerializeToString() + == ProtoHistogramDataPoint().SerializeToString() + ) + + +def test_histogram_data_point_with_count_and_buckets() -> None: + our = HistogramDataPoint( + time_unix_nano=1_000, + count=5, + sum=100.0, + bucket_counts=[2, 3], + explicit_bounds=[50.0], + ) + proto = ProtoHistogramDataPoint( + time_unix_nano=1_000, + count=5, + sum=100.0, + bucket_counts=[2, 3], + explicit_bounds=[50.0], + ) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_histogram_data_point_with_min_max() -> None: + our = HistogramDataPoint(time_unix_nano=1_000, count=3, min=0.0, max=10.0) + proto = ProtoHistogramDataPoint(time_unix_nano=1_000, count=3, min=0.0, max=10.0) + assert our.SerializeToString() == proto.SerializeToString() + + +# ── Histogram ───────────────────────────────────────────────────────────────── + +def test_histogram_empty() -> None: + assert Histogram().SerializeToString() == b"" + + +def test_histogram_empty_matches_proto() -> None: + assert Histogram().SerializeToString() == ProtoHistogram().SerializeToString() + + +def test_histogram_with_data_and_temporality() -> None: + our = Histogram( + data_points=[HistogramDataPoint(count=5, time_unix_nano=1_000)], + aggregation_temporality=2, + ) + proto = ProtoHistogram( + data_points=[ProtoHistogramDataPoint(count=5, time_unix_nano=1_000)], + aggregation_temporality=2, + ) + assert our.SerializeToString() == proto.SerializeToString() + + +# ── ExponentialHistogramDataPoint.Buckets ───────────────────────────────────── + +def test_exp_histogram_buckets_empty() -> None: + assert ExponentialHistogramDataPoint.Buckets().SerializeToString() == b"" + + +def test_exp_histogram_buckets_empty_matches_proto() -> None: + assert ( + ExponentialHistogramDataPoint.Buckets().SerializeToString() + == ProtoExponentialHistogramDataPoint.Buckets().SerializeToString() + ) + + +def test_exp_histogram_buckets_with_data() -> None: + our = ExponentialHistogramDataPoint.Buckets(offset=2, bucket_counts=[1, 2, 3]) + proto = ProtoExponentialHistogramDataPoint.Buckets(offset=2, bucket_counts=[1, 2, 3]) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_exp_histogram_buckets_negative_offset() -> None: + our = ExponentialHistogramDataPoint.Buckets(offset=-3, bucket_counts=[4, 5]) + proto = ProtoExponentialHistogramDataPoint.Buckets(offset=-3, bucket_counts=[4, 5]) + assert our.SerializeToString() == proto.SerializeToString() + + +# ── ExponentialHistogramDataPoint ───────────────────────────────────────────── + +def test_exp_histogram_data_point_empty() -> None: + assert ExponentialHistogramDataPoint().SerializeToString() == b"" + + +def test_exp_histogram_data_point_empty_matches_proto() -> None: + assert ( + ExponentialHistogramDataPoint().SerializeToString() + == ProtoExponentialHistogramDataPoint().SerializeToString() + ) + + +def test_exp_histogram_data_point_with_scale_and_count() -> None: + our = ExponentialHistogramDataPoint( + time_unix_nano=1_000, + count=10, + scale=-1, + zero_count=2, + ) + proto = ProtoExponentialHistogramDataPoint( + time_unix_nano=1_000, + count=10, + scale=-1, + zero_count=2, + ) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_exp_histogram_data_point_with_buckets() -> None: + pos = ExponentialHistogramDataPoint.Buckets(offset=1, bucket_counts=[3, 4]) + neg = ExponentialHistogramDataPoint.Buckets(offset=-1, bucket_counts=[1]) + proto_pos = ProtoExponentialHistogramDataPoint.Buckets(offset=1, bucket_counts=[3, 4]) + proto_neg = ProtoExponentialHistogramDataPoint.Buckets(offset=-1, bucket_counts=[1]) + our = ExponentialHistogramDataPoint( + count=8, scale=2, positive=pos, negative=neg, time_unix_nano=1_000 + ) + proto = ProtoExponentialHistogramDataPoint( + count=8, scale=2, positive=proto_pos, negative=proto_neg, time_unix_nano=1_000 + ) + assert our.SerializeToString() == proto.SerializeToString() + + +# ── ExponentialHistogram ────────────────────────────────────────────────────── + +def test_exp_histogram_empty() -> None: + assert ExponentialHistogram().SerializeToString() == b"" + + +def test_exp_histogram_empty_matches_proto() -> None: + assert ( + ExponentialHistogram().SerializeToString() + == ProtoExponentialHistogram().SerializeToString() + ) + + +def test_exp_histogram_with_data() -> None: + our = ExponentialHistogram( + data_points=[ExponentialHistogramDataPoint(count=5, time_unix_nano=1_000)], + aggregation_temporality=1, + ) + proto = ProtoExponentialHistogram( + data_points=[ProtoExponentialHistogramDataPoint(count=5, time_unix_nano=1_000)], + aggregation_temporality=1, + ) + assert our.SerializeToString() == proto.SerializeToString() + + +# ── SummaryDataPoint.ValueAtQuantile ────────────────────────────────────────── + +def test_value_at_quantile_empty() -> None: + assert SummaryDataPoint.ValueAtQuantile().SerializeToString() == b"" + + +def test_value_at_quantile_empty_matches_proto() -> None: + assert ( + SummaryDataPoint.ValueAtQuantile().SerializeToString() + == ProtoSummaryDataPoint.ValueAtQuantile().SerializeToString() + ) + + +def test_value_at_quantile() -> None: + our = SummaryDataPoint.ValueAtQuantile(quantile=0.5, value=100.0) + proto = ProtoSummaryDataPoint.ValueAtQuantile(quantile=0.5, value=100.0) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_value_at_quantile_p99() -> None: + our = SummaryDataPoint.ValueAtQuantile(quantile=0.99, value=500.0) + proto = ProtoSummaryDataPoint.ValueAtQuantile(quantile=0.99, value=500.0) + assert our.SerializeToString() == proto.SerializeToString() + + +# ── SummaryDataPoint ────────────────────────────────────────────────────────── + +def test_summary_data_point_empty() -> None: + assert SummaryDataPoint().SerializeToString() == b"" + + +def test_summary_data_point_empty_matches_proto() -> None: + assert ( + SummaryDataPoint().SerializeToString() + == ProtoSummaryDataPoint().SerializeToString() + ) + + +def test_summary_data_point_with_count_and_sum() -> None: + our = SummaryDataPoint(count=10, sum=500.0, time_unix_nano=1_000) + proto = ProtoSummaryDataPoint(count=10, sum=500.0, time_unix_nano=1_000) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_summary_data_point_with_quantiles() -> None: + our = SummaryDataPoint( + count=100, + sum=5000.0, + quantile_values=[ + SummaryDataPoint.ValueAtQuantile(quantile=0.5, value=45.0), + SummaryDataPoint.ValueAtQuantile(quantile=0.99, value=200.0), + ], + ) + proto = ProtoSummaryDataPoint( + count=100, + sum=5000.0, + quantile_values=[ + ProtoSummaryDataPoint.ValueAtQuantile(quantile=0.5, value=45.0), + ProtoSummaryDataPoint.ValueAtQuantile(quantile=0.99, value=200.0), + ], + ) + assert our.SerializeToString() == proto.SerializeToString() + + +# ── Summary ─────────────────────────────────────────────────────────────────── + +def test_summary_empty() -> None: + assert Summary().SerializeToString() == b"" + + +def test_summary_empty_matches_proto() -> None: + assert Summary().SerializeToString() == ProtoSummary().SerializeToString() + + +def test_summary_with_data_point() -> None: + our = Summary(data_points=[SummaryDataPoint(count=10, sum=500.0)]) + proto = ProtoSummary(data_points=[ProtoSummaryDataPoint(count=10, sum=500.0)]) + assert our.SerializeToString() == proto.SerializeToString() + + +# ── Metric ──────────────────────────────────────────────────────────────────── + +def test_metric_empty() -> None: + assert Metric().SerializeToString() == b"" + + +def test_metric_empty_matches_proto() -> None: + assert Metric().SerializeToString() == ProtoMetric().SerializeToString() + + +def test_metric_gauge() -> None: + our = Metric(name="cpu", description="CPU usage", unit="%", gauge=Gauge( + data_points=[NumberDataPoint(as_double=0.75)] + )) + proto = ProtoMetric(name="cpu", description="CPU usage", unit="%", gauge=ProtoGauge( + data_points=[ProtoNumberDataPoint(as_double=0.75)] + )) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_metric_sum() -> None: + our = Metric(name="requests", unit="1", sum=Sum( + data_points=[NumberDataPoint(as_int=100)], + aggregation_temporality=2, + is_monotonic=True, + )) + proto = ProtoMetric(name="requests", unit="1", sum=ProtoSum( + data_points=[ProtoNumberDataPoint(as_int=100)], + aggregation_temporality=2, + is_monotonic=True, + )) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_metric_histogram() -> None: + our = Metric(name="latency", histogram=Histogram( + data_points=[HistogramDataPoint(count=5, time_unix_nano=1_000)], + aggregation_temporality=1, + )) + proto = ProtoMetric(name="latency", histogram=ProtoHistogram( + data_points=[ProtoHistogramDataPoint(count=5, time_unix_nano=1_000)], + aggregation_temporality=1, + )) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_metric_summary() -> None: + our = Metric(name="duration", summary=Summary( + data_points=[SummaryDataPoint(count=10, sum=500.0)], + )) + proto = ProtoMetric(name="duration", summary=ProtoSummary( + data_points=[ProtoSummaryDataPoint(count=10, sum=500.0)], + )) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_metric_exponential_histogram() -> None: + our = Metric(name="exp_lat", exponential_histogram=ExponentialHistogram( + data_points=[ExponentialHistogramDataPoint(count=3, time_unix_nano=1_000)], + aggregation_temporality=2, + )) + proto = ProtoMetric(name="exp_lat", exponential_histogram=ProtoExponentialHistogram( + data_points=[ProtoExponentialHistogramDataPoint(count=3, time_unix_nano=1_000)], + aggregation_temporality=2, + )) + assert our.SerializeToString() == proto.SerializeToString() + + +# ── ScopeMetrics ────────────────────────────────────────────────────────────── + +def test_scope_metrics_empty() -> None: + assert ScopeMetrics().SerializeToString() == b"" + + +def test_scope_metrics_empty_matches_proto() -> None: + assert ScopeMetrics().SerializeToString() == ProtoScopeMetrics().SerializeToString() + + +def test_scope_metrics_schema_url() -> None: + our = ScopeMetrics(schema_url="https://example.com") + proto = ProtoScopeMetrics(schema_url="https://example.com") + assert our.SerializeToString() == proto.SerializeToString() + + +def test_scope_metrics_with_scope_and_metric() -> None: + scope = InstrumentationScope(name="mylib", version="1.0") + proto_scope = ProtoInstrumentationScope(name="mylib", version="1.0") + metric = Metric(name="cpu", gauge=Gauge(data_points=[NumberDataPoint(as_double=0.5)])) + proto_metric = ProtoMetric(name="cpu", gauge=ProtoGauge(data_points=[ProtoNumberDataPoint(as_double=0.5)])) + our = ScopeMetrics(scope=scope, metrics=[metric], schema_url="s") + proto = ProtoScopeMetrics(scope=proto_scope, metrics=[proto_metric], schema_url="s") + assert our.SerializeToString() == proto.SerializeToString() + + +# ── ResourceMetrics ─────────────────────────────────────────────────────────── + +def test_resource_metrics_empty() -> None: + assert ResourceMetrics().SerializeToString() == b"" + + +def test_resource_metrics_empty_matches_proto() -> None: + assert ResourceMetrics().SerializeToString() == ProtoResourceMetrics().SerializeToString() + + +def test_resource_metrics_schema_url() -> None: + our = ResourceMetrics(schema_url="https://example.com") + proto = ProtoResourceMetrics(schema_url="https://example.com") + assert our.SerializeToString() == proto.SerializeToString() + + +def test_resource_metrics_with_resource() -> None: + res = Resource(attributes=[KeyValue(key="k", value=AnyValue(string_value="v"))]) + proto_res = ProtoResource(attributes=[ProtoKeyValue(key="k", value=ProtoAnyValue(string_value="v"))]) + our = ResourceMetrics(resource=res, schema_url="s") + proto = ProtoResourceMetrics(resource=proto_res, schema_url="s") + assert our.SerializeToString() == proto.SerializeToString() + + +def test_resource_metrics_with_scope_metrics() -> None: + sm = ScopeMetrics(metrics=[Metric(name="cpu", gauge=Gauge())]) + proto_sm = ProtoScopeMetrics(metrics=[ProtoMetric(name="cpu", gauge=ProtoGauge())]) + our = ResourceMetrics(scope_metrics=[sm]) + proto = ProtoResourceMetrics(scope_metrics=[proto_sm]) + assert our.SerializeToString() == proto.SerializeToString() diff --git a/opentelemetry-pyproto/tests/resource/__init__.py b/opentelemetry-pyproto/tests/resource/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/tests/resource/v1/__init__.py b/opentelemetry-pyproto/tests/resource/v1/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/tests/resource/v1/test_resource_pypb2.py b/opentelemetry-pyproto/tests/resource/v1/test_resource_pypb2.py new file mode 100644 index 00000000000..2a6dd423c8e --- /dev/null +++ b/opentelemetry-pyproto/tests/resource/v1/test_resource_pypb2.py @@ -0,0 +1,64 @@ +from opentelemetry.proto.common.v1.common_pb2 import ( + AnyValue as ProtoAnyValue, + KeyValue as ProtoKeyValue, +) +from opentelemetry.proto.resource.v1.resource_pb2 import Resource as ProtoResource + +from opentelemetry.pyproto.common.v1.common_pypb2 import AnyValue, KeyValue +from opentelemetry.pyproto.resource.v1.resource_pypb2 import Resource + + +def test_resource_empty() -> None: + assert Resource().SerializeToString() == b"" + + +def test_resource_empty_matches_proto() -> None: + assert Resource().SerializeToString() == ProtoResource().SerializeToString() + + +def test_resource_dropped_attributes_count() -> None: + our = Resource(dropped_attributes_count=5) + proto = ProtoResource(dropped_attributes_count=5) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_resource_with_one_attribute() -> None: + attr = KeyValue(key="service.name", value=AnyValue(string_value="my-service")) + proto_attr = ProtoKeyValue(key="service.name", value=ProtoAnyValue(string_value="my-service")) + our = Resource(attributes=[attr]) + proto = ProtoResource(attributes=[proto_attr]) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_resource_with_multiple_attributes() -> None: + attrs = [ + KeyValue(key="service.name", value=AnyValue(string_value="my-service")), + KeyValue(key="service.version", value=AnyValue(string_value="1.0.0")), + KeyValue(key="deployment.environment", value=AnyValue(string_value="prod")), + ] + proto_attrs = [ + ProtoKeyValue(key="service.name", value=ProtoAnyValue(string_value="my-service")), + ProtoKeyValue(key="service.version", value=ProtoAnyValue(string_value="1.0.0")), + ProtoKeyValue(key="deployment.environment", value=ProtoAnyValue(string_value="prod")), + ] + our = Resource(attributes=attrs) + proto = ProtoResource(attributes=proto_attrs) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_resource_full() -> None: + attrs = [ + KeyValue(key="k", value=AnyValue(string_value="v")), + ] + proto_attrs = [ + ProtoKeyValue(key="k", value=ProtoAnyValue(string_value="v")), + ] + our = Resource(attributes=attrs, dropped_attributes_count=2) + proto = ProtoResource(attributes=proto_attrs, dropped_attributes_count=2) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_resource_with_int_attribute() -> None: + our = Resource(attributes=[KeyValue(key="pid", value=AnyValue(int_value=1234))]) + proto = ProtoResource(attributes=[ProtoKeyValue(key="pid", value=ProtoAnyValue(int_value=1234))]) + assert our.SerializeToString() == proto.SerializeToString() diff --git a/opentelemetry-pyproto/tests/trace/__init__.py b/opentelemetry-pyproto/tests/trace/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/tests/trace/v1/__init__.py b/opentelemetry-pyproto/tests/trace/v1/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opentelemetry-pyproto/tests/trace/v1/test_trace_pypb2.py b/opentelemetry-pyproto/tests/trace/v1/test_trace_pypb2.py new file mode 100644 index 00000000000..a6561ae2265 --- /dev/null +++ b/opentelemetry-pyproto/tests/trace/v1/test_trace_pypb2.py @@ -0,0 +1,245 @@ +from opentelemetry.proto.common.v1.common_pb2 import ( + AnyValue as ProtoAnyValue, + InstrumentationScope as ProtoInstrumentationScope, + KeyValue as ProtoKeyValue, +) +from opentelemetry.proto.resource.v1.resource_pb2 import Resource as ProtoResource +from opentelemetry.proto.trace.v1.trace_pb2 import ( + ResourceSpans as ProtoResourceSpans, + ScopeSpans as ProtoScopeSpans, + Span as ProtoSpan, + Status as ProtoStatus, +) + +from opentelemetry.pyproto.common.v1.common_pypb2 import ( + AnyValue, + InstrumentationScope, + KeyValue, +) +from opentelemetry.pyproto.resource.v1.resource_pypb2 import Resource +from opentelemetry.pyproto.trace.v1.trace_pypb2 import ( + ResourceSpans, + ScopeSpans, + Span, + Status, +) + + +# ── Status ──────────────────────────────────────────────────────────────────── + +def test_status_empty() -> None: + assert Status().SerializeToString() == b"" + + +def test_status_empty_matches_proto() -> None: + assert Status().SerializeToString() == ProtoStatus().SerializeToString() + + +def test_status_code() -> None: + our = Status(code=2) + proto = ProtoStatus(code=2) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_status_code_and_message() -> None: + our = Status(code=2, message="internal error") + proto = ProtoStatus(code=2, message="internal error") + assert our.SerializeToString() == proto.SerializeToString() + + +# ── Span.Event ──────────────────────────────────────────────────────────────── + +def test_span_event_empty() -> None: + assert Span.Event().SerializeToString() == b"" + + +def test_span_event_empty_matches_proto() -> None: + assert Span.Event().SerializeToString() == ProtoSpan.Event().SerializeToString() + + +def test_span_event_name_and_time() -> None: + our = Span.Event(name="button.click", time_unix_nano=1_000_000) + proto = ProtoSpan.Event(name="button.click", time_unix_nano=1_000_000) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_span_event_with_attributes() -> None: + attr = KeyValue(key="k", value=AnyValue(string_value="v")) + proto_attr = ProtoKeyValue(key="k", value=ProtoAnyValue(string_value="v")) + our = Span.Event(name="evt", attributes=[attr], dropped_attributes_count=1) + proto = ProtoSpan.Event(name="evt", attributes=[proto_attr], dropped_attributes_count=1) + assert our.SerializeToString() == proto.SerializeToString() + + +# ── Span.Link ───────────────────────────────────────────────────────────────── + +def test_span_link_empty() -> None: + assert Span.Link().SerializeToString() == b"" + + +def test_span_link_empty_matches_proto() -> None: + assert Span.Link().SerializeToString() == ProtoSpan.Link().SerializeToString() + + +def test_span_link_with_ids() -> None: + trace_id = b"\x01" * 16 + span_id = b"\x02" * 8 + our = Span.Link(trace_id=trace_id, span_id=span_id, trace_state="k=v") + proto = ProtoSpan.Link(trace_id=trace_id, span_id=span_id, trace_state="k=v") + assert our.SerializeToString() == proto.SerializeToString() + + +def test_span_link_with_flags() -> None: + trace_id = b"\x01" * 16 + span_id = b"\x02" * 8 + our = Span.Link(trace_id=trace_id, span_id=span_id, flags=1) + proto = ProtoSpan.Link(trace_id=trace_id, span_id=span_id, flags=1) + assert our.SerializeToString() == proto.SerializeToString() + + +# ── Span ────────────────────────────────────────────────────────────────────── + +def test_span_empty() -> None: + assert Span().SerializeToString() == b"" + + +def test_span_empty_matches_proto() -> None: + assert Span().SerializeToString() == ProtoSpan().SerializeToString() + + +def test_span_basic() -> None: + trace_id = b"\x01" * 16 + span_id = b"\x02" * 8 + our = Span( + trace_id=trace_id, + span_id=span_id, + name="my-span", + start_time_unix_nano=1_000_000_000, + end_time_unix_nano=2_000_000_000, + ) + proto = ProtoSpan( + trace_id=trace_id, + span_id=span_id, + name="my-span", + start_time_unix_nano=1_000_000_000, + end_time_unix_nano=2_000_000_000, + ) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_span_with_status() -> None: + our = Span(name="err-span", status=Status(code=2, message="error")) + proto = ProtoSpan(name="err-span", status=ProtoStatus(code=2, message="error")) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_span_with_attributes() -> None: + trace_id = b"\x01" * 16 + span_id = b"\x02" * 8 + attr = KeyValue(key="http.method", value=AnyValue(string_value="GET")) + proto_attr = ProtoKeyValue(key="http.method", value=ProtoAnyValue(string_value="GET")) + our = Span( + trace_id=trace_id, + span_id=span_id, + name="http-span", + attributes=[attr], + dropped_attributes_count=2, + ) + proto = ProtoSpan( + trace_id=trace_id, + span_id=span_id, + name="http-span", + attributes=[proto_attr], + dropped_attributes_count=2, + ) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_span_with_events_and_links() -> None: + trace_id = b"\x01" * 16 + span_id = b"\x02" * 8 + our = Span( + trace_id=trace_id, + span_id=span_id, + name="parent", + events=[Span.Event(name="click", time_unix_nano=500)], + links=[Span.Link(trace_id=trace_id, span_id=span_id)], + dropped_events_count=1, + dropped_links_count=2, + ) + proto = ProtoSpan( + trace_id=trace_id, + span_id=span_id, + name="parent", + events=[ProtoSpan.Event(name="click", time_unix_nano=500)], + links=[ProtoSpan.Link(trace_id=trace_id, span_id=span_id)], + dropped_events_count=1, + dropped_links_count=2, + ) + assert our.SerializeToString() == proto.SerializeToString() + + +def test_span_kind() -> None: + our = Span(name="server", kind=2) + proto = ProtoSpan(name="server", kind=2) + assert our.SerializeToString() == proto.SerializeToString() + + +# ── ScopeSpans ──────────────────────────────────────────────────────────────── + +def test_scope_spans_empty() -> None: + assert ScopeSpans().SerializeToString() == b"" + + +def test_scope_spans_empty_matches_proto() -> None: + assert ScopeSpans().SerializeToString() == ProtoScopeSpans().SerializeToString() + + +def test_scope_spans_schema_url() -> None: + our = ScopeSpans(schema_url="https://example.com") + proto = ProtoScopeSpans(schema_url="https://example.com") + assert our.SerializeToString() == proto.SerializeToString() + + +def test_scope_spans_with_scope_and_span() -> None: + scope = InstrumentationScope(name="mylib", version="1.0") + proto_scope = ProtoInstrumentationScope(name="mylib", version="1.0") + span = Span(name="test-span") + proto_span = ProtoSpan(name="test-span") + our = ScopeSpans(scope=scope, spans=[span], schema_url="s") + proto = ProtoScopeSpans(scope=proto_scope, spans=[proto_span], schema_url="s") + assert our.SerializeToString() == proto.SerializeToString() + + +# ── ResourceSpans ───────────────────────────────────────────────────────────── + +def test_resource_spans_empty() -> None: + assert ResourceSpans().SerializeToString() == b"" + + +def test_resource_spans_empty_matches_proto() -> None: + assert ResourceSpans().SerializeToString() == ProtoResourceSpans().SerializeToString() + + +def test_resource_spans_schema_url() -> None: + our = ResourceSpans(schema_url="https://example.com") + proto = ProtoResourceSpans(schema_url="https://example.com") + assert our.SerializeToString() == proto.SerializeToString() + + +def test_resource_spans_with_resource() -> None: + res = Resource(attributes=[KeyValue(key="k", value=AnyValue(string_value="v"))]) + proto_res = ProtoResource(attributes=[ProtoKeyValue(key="k", value=ProtoAnyValue(string_value="v"))]) + our = ResourceSpans(resource=res, schema_url="s") + proto = ProtoResourceSpans(resource=proto_res, schema_url="s") + assert our.SerializeToString() == proto.SerializeToString() + + +def test_resource_spans_with_scope_spans() -> None: + span = Span(name="s") + proto_span = ProtoSpan(name="s") + ss = ScopeSpans(spans=[span]) + proto_ss = ProtoScopeSpans(spans=[proto_span]) + our = ResourceSpans(scope_spans=[ss]) + proto = ProtoResourceSpans(scope_spans=[proto_ss]) + assert our.SerializeToString() == proto.SerializeToString() diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_configuration/_conversion.py b/opentelemetry-sdk/src/opentelemetry/sdk/_configuration/_conversion.py index eb89c747a44..c388b966b7e 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_configuration/_conversion.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_configuration/_conversion.py @@ -11,12 +11,11 @@ from __future__ import annotations -import dataclasses -import enum -import types -import typing from collections.abc import Mapping -from typing import Any, TypeVar, get_args, get_origin +from dataclasses import fields, is_dataclass +from enum import Enum +from types import UnionType +from typing import Any, TypeVar, Union, get_args, get_origin, get_type_hints _T = TypeVar("_T") @@ -27,7 +26,7 @@ def _unwrap_optional(type_hint: Any) -> Any: Returns the unwrapped type, or the original hint if not a union with None. """ origin = get_origin(type_hint) - if origin is types.UnionType or origin is typing.Union: + if origin is UnionType or origin is Union: non_none = [t for t in get_args(type_hint) if t is not type(None)] if len(non_none) == 1: return non_none[0] @@ -58,7 +57,7 @@ def _convert_value(value: Any, type_hint: Any) -> Any: # Direct dataclass type — recurse if ( isinstance(unwrapped, type) - and dataclasses.is_dataclass(unwrapped) + and is_dataclass(unwrapped) and isinstance(value, dict) ): return _dict_to_dataclass(value, unwrapped) @@ -66,7 +65,7 @@ def _convert_value(value: Any, type_hint: Any) -> Any: # Enum type — coerce string/value to the Enum member if ( isinstance(unwrapped, type) - and issubclass(unwrapped, enum.Enum) + and issubclass(unwrapped, Enum) and not isinstance(value, unwrapped) ): return unwrapped(value) @@ -90,22 +89,28 @@ def _dict_to_dataclass(data: Mapping[str, Any], cls: type[_T]) -> _T: Raises: TypeError: If ``cls`` is not a dataclass type. """ - if not dataclasses.is_dataclass(cls): + if not is_dataclass(cls): raise TypeError(f"{cls.__name__} is not a dataclass") # Annotated as ``dict[str, Any]`` so astroid stops tracing into - # ``typing.get_type_hints`` — under pylint 3.x that path leads into + # ``get_type_hints`` — under pylint 3.x that path leads into # Python 3.14's ``annotationlib`` (which uses t-strings) and crashes. - hints: dict[str, Any] = dict( - typing.get_type_hints(cls, include_extras=False) - ) - known_fields = {f.name for f in dataclasses.fields(cls)} + hints: dict[str, Any] = dict(get_type_hints(cls, include_extras=False)) + known_fields = {f.name for f in fields(cls)} kwargs: dict[str, Any] = {} for key, value in data.items(): - if key in known_fields: - type_hint = hints.get(key) - kwargs[key] = _convert_value(value, type_hint) + # The OTel configuration schema uses "/" as a namespace separator for + # development/experimental features (e.g. "otlp_file/development", + # "instrumentation/development"). Python identifiers cannot contain + # "/", so the corresponding dataclass fields use "_" instead (e.g. + # "otlp_file_development"). Without this normalisation the key would + # not match any known field and would fall through to + # additional_properties, causing the factory lookup to fail silently. + field_key = key.replace("/", "_") + if field_key in known_fields: + type_hint = hints.get(field_key) + kwargs[field_key] = _convert_value(value, type_hint) else: # Unknown key — @_additional_properties decorator will capture it. kwargs[key] = value diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_configuration/_sdk.py b/opentelemetry-sdk/src/opentelemetry/sdk/_configuration/_sdk.py index efa26aa9360..a9cadca5c90 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_configuration/_sdk.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_configuration/_sdk.py @@ -10,7 +10,7 @@ from __future__ import annotations -import logging +from logging import getLogger from opentelemetry.sdk._configuration._logger_provider import ( configure_logger_provider, @@ -23,9 +23,12 @@ from opentelemetry.sdk._configuration._tracer_provider import ( configure_tracer_provider, ) +from opentelemetry.sdk._configuration.instrumentation import ( + configure_instrumentation, +) from opentelemetry.sdk._configuration.models import OpenTelemetryConfiguration -_logger = logging.getLogger(__name__) +_logger = getLogger(__name__) def configure_sdk(config: OpenTelemetryConfiguration) -> None: @@ -62,3 +65,4 @@ def configure_sdk(config: OpenTelemetryConfiguration) -> None: configure_meter_provider(config.meter_provider, resource) configure_logger_provider(config.logger_provider, resource) configure_propagator(config.propagator) + configure_instrumentation(config.instrumentation_development) diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_configuration/instrumentation.py b/opentelemetry-sdk/src/opentelemetry/sdk/_configuration/instrumentation.py new file mode 100644 index 00000000000..20553b9513b --- /dev/null +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_configuration/instrumentation.py @@ -0,0 +1,78 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from dataclasses import fields, is_dataclass +from inspect import isclass +from logging import getLogger + +from opentelemetry.sdk._configuration._common import load_entry_point +from opentelemetry.sdk._configuration._conversion import _dict_to_dataclass +from opentelemetry.sdk._configuration._exceptions import ConfigurationError +from opentelemetry.sdk._configuration.models import ExperimentalInstrumentation + +_logger = getLogger(__name__) + + +def configure_instrumentation( + configuration: ExperimentalInstrumentation | None, +) -> None: + """Activate instrumentors listed under ``instrumentation/development.python``. + + For each entry in ``configuration.python`` the matching + ``opentelemetry_instrumentor`` entry point is loaded. If the instrumentor + class exposes a ``configuration`` attribute that is a dataclass type, the + raw options are validated through ``_dict_to_dataclass`` before being + forwarded to ``instrument()``. An ``enabled: false`` value suppresses + instrumentation without raising. + + If an instrumentor is already active (e.g. ``opentelemetry-instrument`` + ran before the SDK was configured from the file) its ``instrument()`` call + is skipped to avoid a double-instrumentation warning. + + Absent or unknown entry points are logged as warnings; runtime errors from + an instrumentor are logged as exceptions. Neither stops the remaining + instrumentors from being applied. + """ + if configuration is None or configuration.python is None: + return + + for name, options in configuration.python.items(): + options = dict(options) + if not options.pop("enabled", True): + _logger.debug( + "Instrumentation '%s' is disabled in declarative config; skipping", + name, + ) + continue + + try: + cls = load_entry_point("opentelemetry_instrumentor", name) + configuration_cls = getattr(cls, "configuration", None) + if isclass(configuration_cls) and is_dataclass(configuration_cls): + configuration_obj = _dict_to_dataclass( + options, configuration_cls + ) + options = { + f.name: value + for f in fields(configuration_obj) + if (value := getattr(configuration_obj, f.name)) + is not None + } + instance = cls() + if getattr(instance, "is_instrumented_by_opentelemetry", False): + _logger.debug("Skipping '%s': already instrumented", name) + else: + instance.instrument(**options) + _logger.debug("Instrumented '%s' via declarative config", name) + except ConfigurationError as exc: + _logger.warning( + "Skipping instrumentation '%s' in declarative config: %s", + name, + exc, + ) + except Exception: # pylint: disable=broad-except + _logger.exception( + "Failed to instrument '%s' via declarative config", name + ) diff --git a/opentelemetry-sdk/tests/_configuration/test_instrumentation.py b/opentelemetry-sdk/tests/_configuration/test_instrumentation.py new file mode 100644 index 00000000000..fb3b6c11097 --- /dev/null +++ b/opentelemetry-sdk/tests/_configuration/test_instrumentation.py @@ -0,0 +1,250 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from dataclasses import dataclass +from logging import ERROR, WARNING +from unittest import TestCase +from unittest.mock import MagicMock, patch + +from opentelemetry.sdk._configuration._exceptions import ConfigurationError +from opentelemetry.sdk._configuration.instrumentation import ( + configure_instrumentation, +) +from opentelemetry.sdk._configuration.models import ExperimentalInstrumentation + +_LOAD_EP = "opentelemetry.sdk._configuration.instrumentation.load_entry_point" + + +def _make_instrumentor_class(instance, configuration=None): + """Return a class mock whose constructor returns ``instance``.""" + cls = MagicMock(return_value=instance) + cls.configuration = configuration + instance.is_instrumented_by_opentelemetry = False + return cls + + +class TestConfigureInstrumentation(TestCase): + # pylint: disable=no-self-use + def test_none_config_is_noop(self): + configure_instrumentation(None) + + def test_none_python_is_noop(self): + configure_instrumentation(ExperimentalInstrumentation()) + + @patch(_LOAD_EP, side_effect=ConfigurationError("not found")) + def test_unknown_instrumentor_logs_warning(self, _mock_load): + with self.assertLogs( + "opentelemetry.sdk._configuration.instrumentation", + level=WARNING, + ) as cm: + configure_instrumentation( + ExperimentalInstrumentation(python={"unknown_lib": {}}) + ) + self.assertTrue( + any("unknown_lib" in msg for msg in cm.output), + f"Expected warning mentioning 'unknown_lib', got: {cm.output}", + ) + + @patch(_LOAD_EP) + def test_instruments_listed_library_with_no_opts(self, mock_load): + instrumentor = MagicMock() + mock_load.return_value = _make_instrumentor_class(instrumentor) + + configure_instrumentation( + ExperimentalInstrumentation(python={"requests": {}}) + ) + + mock_load.assert_called_once_with( + "opentelemetry_instrumentor", "requests" + ) + instrumentor.instrument.assert_called_once_with() + + @patch(_LOAD_EP) + def test_forwards_kwargs_to_instrumentor_without_configuration( + self, mock_load + ): + instrumentor = MagicMock() + mock_load.return_value = _make_instrumentor_class(instrumentor) + + configure_instrumentation( + ExperimentalInstrumentation( + python={"flask": {"excluded_urls": "/healthz", "foo": "bar"}} + ) + ) + + instrumentor.instrument.assert_called_once_with( + excluded_urls="/healthz", foo="bar" + ) + + @patch(_LOAD_EP) + def test_enabled_false_skips_instrumentation(self, mock_load): + instrumentor = MagicMock() + mock_load.return_value = _make_instrumentor_class(instrumentor) + + configure_instrumentation( + ExperimentalInstrumentation( + python={"requests": {"enabled": False}} + ) + ) + + mock_load.assert_not_called() + instrumentor.instrument.assert_not_called() + + @patch(_LOAD_EP) + def test_enabled_key_not_forwarded_to_instrumentor(self, mock_load): + instrumentor = MagicMock() + mock_load.return_value = _make_instrumentor_class(instrumentor) + + configure_instrumentation( + ExperimentalInstrumentation( + python={"flask": {"enabled": True, "excluded_urls": "/ok"}} + ) + ) + + instrumentor.instrument.assert_called_once_with(excluded_urls="/ok") + + @patch(_LOAD_EP) + def test_multiple_instrumentors_all_called(self, mock_load): + flask_inst = MagicMock() + requests_inst = MagicMock() + + def _side_effect(_group, name): + return _make_instrumentor_class( + flask_inst if name == "flask" else requests_inst + ) + + mock_load.side_effect = _side_effect + + configure_instrumentation( + ExperimentalInstrumentation( + python={"flask": {}, "requests": {"foo": "bar"}} + ) + ) + + flask_inst.instrument.assert_called_once_with() + requests_inst.instrument.assert_called_once_with(foo="bar") + + @patch(_LOAD_EP) + def test_instrumentor_exception_does_not_stop_others(self, mock_load): + broken_inst = MagicMock() + broken_inst.instrument.side_effect = RuntimeError("boom") + ok_inst = MagicMock() + + call_count = 0 + + def _side_effect(_group, _name): + nonlocal call_count + call_count += 1 + return _make_instrumentor_class( + broken_inst if call_count == 1 else ok_inst + ) + + mock_load.side_effect = _side_effect + + with self.assertLogs( + "opentelemetry.sdk._configuration.instrumentation", + level=ERROR, + ): + configure_instrumentation( + ExperimentalInstrumentation(python={"broken": {}, "ok": {}}) + ) + + ok_inst.instrument.assert_called_once_with() + + @patch(_LOAD_EP) + def test_skips_already_instrumented(self, mock_load): + instrumentor = MagicMock() + mock_load.return_value = _make_instrumentor_class(instrumentor) + instrumentor.is_instrumented_by_opentelemetry = True + + configure_instrumentation( + ExperimentalInstrumentation(python={"requests": {}}) + ) + + instrumentor.instrument.assert_not_called() + + @patch(_LOAD_EP) + def test_non_dataclass_configuration_attribute_ignored(self, mock_load): + instrumentor = MagicMock() + mock_load.return_value = _make_instrumentor_class( + instrumentor, configuration="not-a-dataclass" + ) + + configure_instrumentation( + ExperimentalInstrumentation(python={"requests": {"foo": "bar"}}) + ) + + instrumentor.instrument.assert_called_once_with(foo="bar") + + @patch(_LOAD_EP) + def test_configuration_coerces_opts(self, mock_load): + @dataclass + class RequestsConfig: + excluded_urls: str | None = None + capture_headers: bool | None = None + + instrumentor = MagicMock() + mock_load.return_value = _make_instrumentor_class( + instrumentor, configuration=RequestsConfig + ) + + configure_instrumentation( + ExperimentalInstrumentation( + python={ + "requests": { + "excluded_urls": "/health", + "capture_headers": True, + } + } + ) + ) + + instrumentor.instrument.assert_called_once_with( + excluded_urls="/health", capture_headers=True + ) + + @patch(_LOAD_EP) + def test_configuration_none_fields_not_forwarded(self, mock_load): + @dataclass + class FlaskConfig: + excluded_urls: str | None = None + propagate_headers: bool | None = None + + instrumentor = MagicMock() + mock_load.return_value = _make_instrumentor_class( + instrumentor, configuration=FlaskConfig + ) + + configure_instrumentation( + ExperimentalInstrumentation( + python={"flask": {"excluded_urls": "/ok"}} + ) + ) + + # propagate_headers was not set, so it must not appear in the call. + instrumentor.instrument.assert_called_once_with(excluded_urls="/ok") + + @patch(_LOAD_EP) + def test_configuration_unknown_field_raises(self, mock_load): + @dataclass + class StrictConfig: + excluded_urls: str | None = None + + instrumentor = MagicMock() + mock_load.return_value = _make_instrumentor_class( + instrumentor, configuration=StrictConfig + ) + + with self.assertLogs( + "opentelemetry.sdk._configuration.instrumentation", + level=ERROR, + ): + configure_instrumentation( + ExperimentalInstrumentation( + python={ + "mylib": {"excluded_urls": "/ok", "typo_field": "bad"} + } + ) + ) + + instrumentor.instrument.assert_not_called() diff --git a/pyproject.toml b/pyproject.toml index a273dff32da..c10c734578b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,9 @@ opentelemetry-exporter-prometheus = {workspace = true } opentelemetry-propagator-jaeger = { workspace = true } opentelemetry-propagator-b3 = { workspace = true } opentelemetry-codegen-json = { workspace = true } +opentelemetry-pyproto = { workspace = true } +opentelemetry-exporter-otlp-pyproto-common = { workspace = true } +opentelemetry-exporter-otlp-pyproto-http = { workspace = true } [tool.uv.workspace] members = [ @@ -57,6 +60,7 @@ members = [ "propagator/*", "codegen/*", "tests/opentelemetry-test-utils", + "opentelemetry-pyproto", ] exclude = [ diff --git a/uv.lock b/uv.lock index 55fc9ceef21..ddcf9c6d4e7 100644 --- a/uv.lock +++ b/uv.lock @@ -20,12 +20,16 @@ members = [ "opentelemetry-exporter-otlp-proto-common", "opentelemetry-exporter-otlp-proto-grpc", "opentelemetry-exporter-otlp-proto-http", + "opentelemetry-exporter-otlp-pyproto-common", + "opentelemetry-exporter-otlp-pyproto-grpc", + "opentelemetry-exporter-otlp-pyproto-http", "opentelemetry-exporter-prometheus", "opentelemetry-exporter-zipkin-json", "opentelemetry-propagator-b3", "opentelemetry-propagator-jaeger", "opentelemetry-proto", "opentelemetry-proto-json", + "opentelemetry-pyproto", "opentelemetry-python", "opentelemetry-sdk", "opentelemetry-semantic-conventions", @@ -642,6 +646,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8a/eb/427ed2b20a38a4ee29f24dbe4ae2dafab198674fe9a85e3d6adf9e5f5f41/inflect-7.5.0-py3-none-any.whl", hash = "sha256:2aea70e5e70c35d8350b8097396ec155ffd68def678c7ff97f51aa69c1d92344", size = 35197 }, ] +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + [[package]] name = "isort" version = "8.0.1" @@ -1000,6 +1013,106 @@ requires-dist = [ ] provides-extras = ["gcp-auth"] +[[package]] +name = "opentelemetry-exporter-otlp-pyproto-common" +source = { editable = "exporter/opentelemetry-exporter-otlp-pyproto-common" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-pyproto" }, + { name = "opentelemetry-sdk" }, +] + +[package.dev-dependencies] +dev = [ + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-test-utils" }, + { name = "protobuf" }, + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "opentelemetry-api", editable = "opentelemetry-api" }, + { name = "opentelemetry-pyproto", editable = "opentelemetry-pyproto" }, + { name = "opentelemetry-sdk", editable = "opentelemetry-sdk" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "opentelemetry-exporter-otlp-proto-common", editable = "exporter/opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto", editable = "opentelemetry-proto" }, + { name = "opentelemetry-test-utils", editable = "tests/opentelemetry-test-utils" }, + { name = "protobuf", specifier = ">=5.0" }, + { name = "pytest", specifier = ">=7.0" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-pyproto-grpc" +source = { editable = "exporter/opentelemetry-exporter-otlp-pyproto-grpc" } +dependencies = [ + { name = "grpcio" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-pyproto-common" }, + { name = "opentelemetry-sdk" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "grpcio", marker = "python_full_version < '3.13'", specifier = ">=1.63.2,<2.0.0" }, + { name = "grpcio", marker = "python_full_version == '3.13.*'", specifier = ">=1.66.2,<2.0.0" }, + { name = "grpcio", marker = "python_full_version >= '3.14'", specifier = ">=1.75.1,<2.0.0" }, + { name = "opentelemetry-api", editable = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-pyproto-common", editable = "exporter/opentelemetry-exporter-otlp-pyproto-common" }, + { name = "opentelemetry-sdk", editable = "opentelemetry-sdk" }, +] + +[package.metadata.requires-dev] +dev = [{ name = "pytest", specifier = ">=7.0" }] + +[[package]] +name = "opentelemetry-exporter-otlp-pyproto-http" +source = { editable = "exporter/opentelemetry-exporter-otlp-pyproto-http" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-pyproto-common" }, + { name = "opentelemetry-sdk" }, + { name = "requests" }, +] + +[package.dev-dependencies] +dev = [ + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-test-utils" }, + { name = "protobuf" }, + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "opentelemetry-api", editable = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-pyproto-common", editable = "exporter/opentelemetry-exporter-otlp-pyproto-common" }, + { name = "opentelemetry-sdk", editable = "opentelemetry-sdk" }, + { name = "requests", specifier = "~=2.7" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "opentelemetry-exporter-otlp-proto-common", editable = "exporter/opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-exporter-otlp-proto-http", editable = "exporter/opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-proto", editable = "opentelemetry-proto" }, + { name = "opentelemetry-test-utils", editable = "tests/opentelemetry-test-utils" }, + { name = "protobuf", specifier = ">=5.0" }, + { name = "pytest", specifier = ">=7.0" }, +] + [[package]] name = "opentelemetry-exporter-prometheus" source = { editable = "exporter/opentelemetry-exporter-prometheus" } @@ -1070,6 +1183,28 @@ requires-dist = [{ name = "protobuf", specifier = ">=5.0,<8.0" }] name = "opentelemetry-proto-json" source = { editable = "opentelemetry-proto-json" } +[[package]] +name = "opentelemetry-pyproto" +source = { editable = "opentelemetry-pyproto" } + +[package.dev-dependencies] +dev = [ + { name = "opentelemetry-proto" }, + { name = "protobuf" }, + { name = "pytest" }, + { name = "pytest-benchmark" }, +] + +[package.metadata] + +[package.metadata.requires-dev] +dev = [ + { name = "opentelemetry-proto", editable = "opentelemetry-proto" }, + { name = "protobuf", specifier = ">=5.0" }, + { name = "pytest", specifier = ">=7.0" }, + { name = "pytest-benchmark", specifier = ">=4" }, +] + [[package]] name = "opentelemetry-python" version = "0.0.0" @@ -1267,6 +1402,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/19/c7/5f7c636ec43e0c545e28d1f1db71990108306f7bdcb89f069ba97e428e7f/protobuf-7.35.1-py3-none-any.whl", hash = "sha256:4bc97768d8fe4ad6743c8a19403e314511ed9f6d13205b687e52421c023ac1b9", size = 171659 }, ] +[[package]] +name = "py-cpuinfo" +version = "9.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" }, +] + [[package]] name = "pyasn1" version = "0.6.3" @@ -1420,6 +1564,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071 }, ] +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + [[package]] name = "pyproject-api" version = "1.10.1" @@ -1433,6 +1586,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/d7/29e1e5e882f79133631f7bcace42d23db493f616463c157a1ab614bf69dd/pyproject_api-1.10.1-py3-none-any.whl", hash = "sha256:fa9e6f66c35b5017e909825d8f2b5d5482ea699d7be809d21c03bd1f7317f36a", size = 12992 }, ] +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pytest-benchmark" +version = "5.2.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "py-cpuinfo" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/24/34/9f732b76456d64faffbef6232f1f9dbec7a7c4999ff46282fa418bd1af66/pytest_benchmark-5.2.3.tar.gz", hash = "sha256:deb7317998a23c650fd4ff76e1230066a76cb45dcece0aca5607143c619e7779", size = 341340, upload-time = "2025-11-09T18:48:43.215Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/29/e756e715a48959f1c0045342088d7ca9762a2f509b945f362a316e9412b7/pytest_benchmark-5.2.3-py3-none-any.whl", hash = "sha256:bc839726ad20e99aaa0d11a127445457b4219bdb9e80a1afc4b51da7f96b0803", size = 45255, upload-time = "2025-11-09T18:48:39.765Z" }, +] + [[package]] name = "python-discovery" version = "1.4.2"