Merge instrumentation declarative config with pyprotobuf exporter work

.changelog/5372.added

@@ -0,0 +1 @@
+`opentelemetry-sdk`: Add support for activating instrumentors from a declarative configuration file via the `instrumentation/development.python` section. Instrumentors can declare a `configuration` attribute to have their options validated through the same type-coercion pipeline used for SDK component configuration.

docs/examples/declarative-config/README.rst

@@ -16,14 +16,15 @@ The source files of this example are available :scm_web:`here
 <docs/examples/declarative-config/>`.
 
 Install the SDK with the ``file-configuration`` extra (it pulls in ``pyyaml``
-and ``jsonschema``), the auto-instrumentation entry point, and the OTLP/HTTP
-exporter:
+and ``jsonschema``), the auto-instrumentation entry point, the OTLP/HTTP
+exporter, and the ``requests`` instrumentation used by this example:
 
 .. code-block:: sh
 
     pip install "opentelemetry-sdk[file-configuration]" \
         opentelemetry-distro \
-        opentelemetry-exporter-otlp-proto-http
+        opentelemetry-exporter-otlp-proto-http \
+        opentelemetry-instrumentation-requests
 
 Start an OTLP-capable backend locally so there is somewhere to send data. Write
 the following file:
@@ -74,7 +75,10 @@ auto-instrumentation apply it. No configuration code lives in ``app.py``:
     export OTEL_CONFIG_FILE=$(pwd)/otel-config.yaml
     opentelemetry-instrument python app.py
 
-You should see the exported span in the Collector's debug output.
+You should see the exported span in the Collector's debug output. The
+``instrumentation/development.python`` section in ``otel-config.yaml``
+activates the ``requests`` instrumentation so outgoing HTTP calls made by
+``app.py`` are automatically traced without any code changes.
 
 Environment variable substitution
 ----------------------------------

docs/examples/declarative-config/otel-config.yaml

@@ -32,3 +32,8 @@ logger_provider:
         exporter:
           otlp_http:
             endpoint: http://localhost:4318/v1/logs
+
+instrumentation/development:
+  python:
+    requests:
+      enabled: true

docs/sdk/configuration.rst

@@ -95,6 +95,31 @@ OTLP/HTTP. The source is available :scm_web:`here
                   - name: api-key
                     value: ${OTLP_API_KEY}
 
+Instrumentation
+---------------
+
+The ``instrumentation/development.python`` section activates Python
+instrumentors by their ``opentelemetry_instrumentor`` entry-point name. Set
+``enabled: false`` to suppress an instrumentor without removing its entry, and
+pass any other keys as keyword arguments to ``instrument()``:
+
+.. code-block:: yaml
+
+    instrumentation/development:
+      python:
+        requests:
+          enabled: true
+        urllib3:
+          enabled: true
+          max_spans_per_request: 10
+
+If the instrumentor class declares a ``configuration`` class attribute pointing
+to a dataclass, the options are validated and type-coerced through the same
+pipeline used for SDK component configuration before being forwarded to
+``instrument()``. Instrumentors that are already active (for example because
+``opentelemetry-instrument`` ran before the file was applied) are silently
+skipped.
+
 Environment variable substitution
 ----------------------------------
 

exporter/opentelemetry-exporter-otlp-pyproto-common/README.rst

@@ -0,0 +1,4 @@
+OpenTelemetry OTLP Exporter (pure-Python protobuf, common)
+===========================================================
+
+Common encoding utilities for the pure-Python OTLP exporter stack.

exporter/opentelemetry-exporter-otlp-pyproto-common/pyproject.toml

@@ -0,0 +1,39 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "opentelemetry-exporter-otlp-pyproto-common"
+dynamic = ["version"]
+description = "OpenTelemetry OTLP encoding using pure-Python protobuf (no google.protobuf)"
+readme = "README.rst"
+license = "Apache-2.0"
+requires-python = ">=3.10"
+dependencies = [
+  "opentelemetry-api ~= 1.12",
+  "opentelemetry-sdk ~= 1.12",
+  "opentelemetry-pyproto == 1.43.0.dev",
+]
+
+[dependency-groups]
+dev = [
+    "opentelemetry-proto == 1.43.0.dev",
+    "opentelemetry-exporter-otlp-proto-common == 1.43.0.dev",
+    "opentelemetry-test-utils == 1.43.0.dev",
+    "protobuf>=5.0",
+    "pytest>=7.0",
+]
+
+[tool.uv.sources]
+opentelemetry-proto = { workspace = true }
+opentelemetry-exporter-otlp-proto-common = { workspace = true }
+opentelemetry-test-utils = { workspace = true }
+
+[tool.hatch.version]
+path = "src/opentelemetry/exporter/otlp/pyproto/common/version/__init__.py"
+
+[tool.hatch.build.targets.sdist]
+include = ["/src"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/opentelemetry"]

exporter/opentelemetry-exporter-otlp-pyproto-common/src/opentelemetry/exporter/otlp/pyproto/common/_exporter_metrics.py

@@ -0,0 +1,153 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+from collections import Counter
+from collections.abc import Iterator
+from contextlib import AbstractContextManager, contextmanager
+from dataclasses import dataclass
+from time import perf_counter
+from typing import TYPE_CHECKING, Protocol
+
+from opentelemetry.metrics import MeterProvider, get_meter_provider
+from opentelemetry.semconv._incubating.attributes.otel_attributes import (
+    OTEL_COMPONENT_NAME,
+    OTEL_COMPONENT_TYPE,
+    OtelComponentTypeValues,
+)
+from opentelemetry.semconv._incubating.metrics.otel_metrics import (
+    create_otel_sdk_exporter_log_exported,
+    create_otel_sdk_exporter_log_inflight,
+    create_otel_sdk_exporter_metric_data_point_exported,
+    create_otel_sdk_exporter_metric_data_point_inflight,
+    create_otel_sdk_exporter_operation_duration,
+    create_otel_sdk_exporter_span_exported,
+    create_otel_sdk_exporter_span_inflight,
+)
+from opentelemetry.semconv.attributes.error_attributes import ERROR_TYPE
+from opentelemetry.semconv.attributes.server_attributes import (
+    SERVER_ADDRESS,
+    SERVER_PORT,
+)
+
+if TYPE_CHECKING:
+    from typing import Literal
+    from urllib.parse import ParseResult as UrlParseResult
+
+    from opentelemetry.util.types import Attributes, AttributeValue
+
+_component_counter = Counter()
+
+
+@dataclass
+class ExportResult:
+    error: Exception | None = None
+    error_attrs: Attributes = None
+
+
+class ExporterMetricsT(Protocol):
+    def export_operation(
+        self, num_items: int
+    ) -> AbstractContextManager[ExportResult]: ...
+
+
+class NoOpExporterMetrics:
+    @contextmanager
+    def export_operation(self, num_items: int) -> Iterator[ExportResult]:
+        yield ExportResult()
+
+
+class ExporterMetrics:
+    def __init__(
+        self,
+        component_type: OtelComponentTypeValues | None,
+        signal: Literal["traces", "metrics", "logs"],
+        endpoint: UrlParseResult,
+        meter_provider: MeterProvider | None,
+    ) -> None:
+        if signal == "traces":
+            create_exported = create_otel_sdk_exporter_span_exported
+            create_inflight = create_otel_sdk_exporter_span_inflight
+        elif signal == "logs":
+            create_exported = create_otel_sdk_exporter_log_exported
+            create_inflight = create_otel_sdk_exporter_log_inflight
+        else:
+            create_exported = (
+                create_otel_sdk_exporter_metric_data_point_exported
+            )
+            create_inflight = (
+                create_otel_sdk_exporter_metric_data_point_inflight
+            )
+
+        port = endpoint.port
+        if port is None:
+            if endpoint.scheme == "https":
+                port = 443
+            elif endpoint.scheme == "http":
+                port = 80
+
+        component_type_value = (
+            component_type.value if component_type else "unknown_otlp_exporter"
+        )
+        count = _component_counter[component_type_value]
+        _component_counter[component_type_value] = count + 1
+        self._standard_attrs: dict[str, AttributeValue] = {
+            OTEL_COMPONENT_TYPE: component_type_value,
+            OTEL_COMPONENT_NAME: f"{component_type_value}/{count}",
+        }
+        if endpoint.hostname:
+            self._standard_attrs[SERVER_ADDRESS] = endpoint.hostname
+        if port is not None:
+            self._standard_attrs[SERVER_PORT] = port
+
+        meter_provider = meter_provider or get_meter_provider()
+        meter = meter_provider.get_meter("opentelemetry-sdk")
+        self._inflight = create_inflight(meter)
+        self._exported = create_exported(meter)
+        self._duration = create_otel_sdk_exporter_operation_duration(meter)
+
+    @contextmanager
+    def export_operation(self, num_items: int) -> Iterator[ExportResult]:
+        start_time = perf_counter()
+        self._inflight.add(num_items, self._standard_attrs)
+
+        result = ExportResult()
+        try:
+            yield result
+        finally:
+            error = result.error
+            error_attrs = result.error_attrs
+
+            end_time = perf_counter()
+            self._inflight.add(-num_items, self._standard_attrs)
+            exported_attrs = (
+                {**self._standard_attrs, ERROR_TYPE: type(error).__qualname__}
+                if error
+                else self._standard_attrs
+            )
+            self._exported.add(num_items, exported_attrs)
+            duration_attrs = (
+                {**exported_attrs, **error_attrs}
+                if error_attrs
+                else exported_attrs
+            )
+            self._duration.record(end_time - start_time, duration_attrs)
+
+
+def create_exporter_metrics(
+    component_type: OtelComponentTypeValues | None,
+    signal: Literal["traces", "metrics", "logs"],
+    endpoint: UrlParseResult,
+    meter_provider: MeterProvider | None,
+    enabled: bool,
+) -> ExporterMetricsT:
+    if not enabled:
+        return NoOpExporterMetrics()
+
+    return ExporterMetrics(
+        component_type,
+        signal,
+        endpoint,
+        meter_provider,
+    )

exporter/opentelemetry-exporter-otlp-pyproto-common/src/opentelemetry/exporter/otlp/pyproto/common/_internal/__init__.py

@@ -0,0 +1,114 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+from logging import getLogger
+from collections.abc import Callable, Mapping, Sequence
+from typing import Any, TypeVar
+
+from opentelemetry.pyproto.common.v1.common_pypb2 import (
+    AnyValue,
+    ArrayValue,
+    InstrumentationScope as PB2InstrumentationScope,
+    KeyValue,
+    KeyValueList,
+)
+from opentelemetry.pyproto.resource.v1.resource_pypb2 import Resource as PB2Resource
+from opentelemetry.sdk.trace import Resource
+from opentelemetry.sdk.util.instrumentation import InstrumentationScope
+from opentelemetry.util.types import _ExtendedAttributes
+
+_logger = getLogger(__name__)
+
+_TypingResourceT = TypeVar("_TypingResourceT")
+_ResourceDataT = TypeVar("_ResourceDataT")
+
+
+def _encode_instrumentation_scope(
+    instrumentation_scope: InstrumentationScope,
+) -> PB2InstrumentationScope:
+    if instrumentation_scope is None:
+        return PB2InstrumentationScope()
+    return PB2InstrumentationScope(
+        name=instrumentation_scope.name,
+        version=instrumentation_scope.version,
+        attributes=_encode_attributes(instrumentation_scope.attributes),
+    )
+
+
+def _encode_resource(resource: Resource) -> PB2Resource:
+    return PB2Resource(attributes=_encode_attributes(resource.attributes))
+
+
+def _encode_value(value: Any) -> AnyValue:
+    if value is None:
+        return AnyValue()
+    if isinstance(value, bool):
+        return AnyValue(bool_value=value)
+    if isinstance(value, str):
+        return AnyValue(string_value=value)
+    if isinstance(value, int):
+        return AnyValue(int_value=value)
+    if isinstance(value, float):
+        return AnyValue(double_value=value)
+    if isinstance(value, bytes):
+        return AnyValue(bytes_value=value)
+    if isinstance(value, Sequence):
+        return AnyValue(
+            array_value=ArrayValue(values=[_encode_value(v) for v in value])
+        )
+    if isinstance(value, Mapping):
+        return AnyValue(
+            kvlist_value=KeyValueList(
+                values=[_encode_key_value(str(k), v) for k, v in value.items()]
+            )
+        )
+    raise Exception(f"Invalid type {type(value)} of value {value}")
+
+
+def _encode_key_value(key: str, value: Any) -> KeyValue:
+    return KeyValue(key=key, value=_encode_value(value))
+
+
+def _encode_span_id(span_id: int) -> bytes:
+    return span_id.to_bytes(length=8, byteorder="big", signed=False)
+
+
+def _encode_trace_id(trace_id: int) -> bytes:
+    return trace_id.to_bytes(length=16, byteorder="big", signed=False)
+
+
+def _encode_attributes(
+    attributes: _ExtendedAttributes | None,
+) -> list[KeyValue]:
+    if not attributes:
+        return []
+    pb2_attributes = []
+    for key, value in attributes.items():
+        try:
+            pb2_attributes.append(_encode_key_value(key, value))
+        except Exception as error:
+            _logger.exception("Failed to encode key %s: %s", key, error)
+    return pb2_attributes
+
+
+def _get_resource_data(
+    sdk_resource_scope_data: dict[Resource, _ResourceDataT],
+    resource_class: Callable[..., _TypingResourceT],
+    name: str,
+) -> list[_TypingResourceT]:
+    resource_data = []
+    for sdk_resource, scope_data in sdk_resource_scope_data.items():
+        collector_resource = PB2Resource(
+            attributes=_encode_attributes(sdk_resource.attributes)
+        )
+        resource_data.append(
+            resource_class(
+                **{
+                    "resource": collector_resource,
+                    f"scope_{name}": scope_data.values(),
+                }
+            )
+        )
+    return resource_data