HDDS-15781. Adding Spankind information to ozone sh commands

hadoop-hdds/client/pom.xml

@@ -32,6 +32,10 @@
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
     </dependency>
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-api</artifactId>
+    </dependency>
     <dependency>
       <groupId>jakarta.annotation</groupId>
       <artifactId>jakarta.annotation-api</artifactId>

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java

@@ -20,6 +20,7 @@
 import static org.apache.hadoop.hdds.HddsUtils.processForDebug;
 
 import com.google.common.annotations.VisibleForTesting;
+import io.opentelemetry.api.trace.SpanKind;
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.io.UncheckedIOException;
@@ -421,7 +422,7 @@ private XceiverClientReply sendCommandWithTraceIDAndRetry(
 
     String spanName = "XceiverClientGrpc." + request.getCmdType().name();
 
-    return TracingUtil.executeInNewSpan(spanName,
+    return TracingUtil.executeInNewSpan(spanName, SpanKind.CLIENT,
         () -> {
           ContainerCommandRequestProto.Builder builder =
               ContainerCommandRequestProto.newBuilder(request)
@@ -676,7 +677,7 @@ public XceiverClientReply sendCommandAsync(
       throws IOException, ExecutionException, InterruptedException {
 
     try (TracingUtil.TraceCloseable ignored = TracingUtil.createActivatedSpan(
-        "XceiverClientGrpc." + request.getCmdType().name())) {
+        "XceiverClientGrpc." + request.getCmdType().name(), SpanKind.CLIENT)) {
 
       ContainerCommandRequestProto.Builder builder =
           ContainerCommandRequestProto.newBuilder(request)

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java

@@ -19,6 +19,7 @@
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
+import io.opentelemetry.api.trace.SpanKind;
 import java.io.IOException;
 import java.util.Collection;
 import java.util.Comparator;
@@ -253,7 +254,7 @@ private CompletableFuture<RaftClientReply> sendRequestAsync(
       }
     }
     return TracingUtil.executeInNewSpan(
-        "XceiverClientRatis." + request.getCmdType().name(),
+        "XceiverClientRatis." + request.getCmdType().name(), SpanKind.CLIENT,
         () -> {
           final ContainerCommandRequestMessage message
               = ContainerCommandRequestMessage.toMessage(

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/GrpcServerInterceptor.java

@@ -18,6 +18,7 @@
 package org.apache.hadoop.hdds.tracing;
 
 import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.SpanKind;
 import io.opentelemetry.context.Scope;
 import org.apache.ratis.thirdparty.io.grpc.ForwardingServerCallListener.SimpleForwardingServerCallListener;
 import org.apache.ratis.thirdparty.io.grpc.Metadata;
@@ -44,7 +45,8 @@ public void onMessage(ReqT message) {
         Span span = TracingUtil
             .importAndCreateSpan(
                 call.getMethodDescriptor().getFullMethodName(),
-                headers.get(GrpcClientInterceptor.TRACING_HEADER));
+                headers.get(GrpcClientInterceptor.TRACING_HEADER),
+                SpanKind.SERVER);
         try (Scope ignored = span.makeCurrent()) {
           super.onMessage(message);
         } finally {

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/InternalSpanKind.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.tracing;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation that marks a proxied client method as SpanKind.INTERNAL.
+ * Used for methods that do not send an outbound RPC.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.METHOD)
+public @interface InternalSpanKind {
+}
+

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TraceAllMethod.java

@@ -19,6 +19,7 @@
 
 import static java.util.Collections.emptyMap;
 
+import io.opentelemetry.api.trace.SpanKind;
 import java.lang.reflect.InvocationHandler;
 import java.lang.reflect.Method;
 import java.util.Arrays;
@@ -77,7 +78,13 @@ public Object invoke(Object proxy, Method method, Object[] args)
       }
     }
 
-    try (TracingUtil.TraceCloseable ignored = TracingUtil.createActivatedSpan(name + "." + method.getName())) {
+    // if a call is within a process, not an outbound RPC to other processes (OM/SCM/DN) , it is marked as INTERNAL.
+    SpanKind spanKind = delegateMethod.isAnnotationPresent(InternalSpanKind.class)
+        ? SpanKind.INTERNAL
+        : SpanKind.CLIENT;
+
+    try (TracingUtil.TraceCloseable ignored = TracingUtil.createActivatedSpan(
+        name + "." + method.getName(), spanKind)) {
       try {
         return delegateMethod.invoke(delegate, args);
       } catch (Exception ex) {

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java

@@ -21,6 +21,7 @@
 import io.opentelemetry.api.common.AttributeKey;
 import io.opentelemetry.api.common.Attributes;
 import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.SpanKind;
 import io.opentelemetry.api.trace.StatusCode;
 import io.opentelemetry.api.trace.Tracer;
 import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator;
@@ -121,9 +122,18 @@ public static <R, E extends Exception> R execute(
       String spanName,
       ConfigurationSource conf,
       CheckedSupplier<R, E> supplier) throws E {
+    return execute(serviceName, spanName, SpanKind.INTERNAL, conf, supplier);
+  }
+
+  public static <R, E extends Exception> R execute(
+      String serviceName,
+      String spanName,
+      SpanKind spanKind,
+      ConfigurationSource conf,
+      CheckedSupplier<R, E> supplier) throws E {
     initTracing(serviceName, conf);
     try {
-      return executeInNewSpan(spanName, supplier);
+      return executeInNewSpan(spanName, spanKind, supplier);
     } finally {
       flushTracing();
     }
@@ -217,14 +227,20 @@ public static String exportCurrentSpan() {
    * @return Tracing scope.
    */
   public static Span importAndCreateSpan(String name, String encodedParent) {
+    return importAndCreateSpan(name, encodedParent, SpanKind.INTERNAL);
+  }
+
+  public static Span importAndCreateSpan(String name, String encodedParent,
+                                         SpanKind spanKind) {
     if (encodedParent == null || encodedParent.isEmpty()) {
-      return tracer.spanBuilder(name).setNoParent().startSpan();
+      return tracer.spanBuilder(name).setNoParent().setSpanKind(spanKind).startSpan();
     }
 
     W3CTraceContextPropagator propagator = W3CTraceContextPropagator.getInstance();
     Context extract = propagator.extract(Context.current(), encodedParent, new TextExtractor());
     return tracer.spanBuilder(name)
         .setParent(extract)
+        .setSpanKind(spanKind)
         .startSpan();
   }
 
@@ -298,7 +314,12 @@ static Map<String, LoopSampler> parseSpanSamplingConfig(String configStr) {
    */
   public static <E extends Exception> void executeInNewSpan(String spanName,
       CheckedRunnable<E> runnable) throws E {
-    Span span = buildSpan(spanName);
+    executeInNewSpan(spanName, SpanKind.INTERNAL, runnable);
+  }
+
+  public static <E extends Exception> void executeInNewSpan(String spanName,
+      SpanKind spanKind, CheckedRunnable<E> runnable) throws E {
+    Span span = buildSpan(spanName, spanKind);
     executeInSpan(span, runnable);
   }
 
@@ -307,7 +328,12 @@ public static <E extends Exception> void executeInNewSpan(String spanName,
    */
   public static <R, E extends Exception> R executeInNewSpan(String spanName,
       CheckedSupplier<R, E> supplier) throws E {
-    Span span = buildSpan(spanName);
+    return executeInNewSpan(spanName, SpanKind.INTERNAL, supplier);
+  }
+
+  public static <R, E extends Exception> R executeInNewSpan(String spanName,
+      SpanKind spanKind, CheckedSupplier<R, E> supplier) throws E {
+    Span span = buildSpan(spanName, spanKind);
     return executeInSpan(span, supplier);
   }
 
@@ -361,7 +387,11 @@ public static <E extends Exception> void executeAsChildSpan(String spanName,
    * in case of Exceptions.
    */
   public static TraceCloseable createActivatedSpan(String spanName) {
-    Span span = buildSpan(spanName);
+    return createActivatedSpan(spanName, SpanKind.INTERNAL);
+  }
+
+  public static TraceCloseable createActivatedSpan(String spanName, SpanKind spanKind) {
+    Span span = buildSpan(spanName, spanKind);
     Scope scope = span.makeCurrent();
     return () -> {
       scope.close();
@@ -421,14 +451,14 @@ private void parse(String carrier) {
    * Creates a new span, using the current context as a parent if valid;
    * otherwise, creates a root span.
    */
-  private static Span buildSpan(String spanName) {
+  private static Span buildSpan(String spanName, SpanKind spanKind) {
     Context currentContext = Context.current();
     Span parentSpan = Span.fromContext(currentContext);
 
     if (parentSpan.getSpanContext().isValid()) {
-      return tracer.spanBuilder(spanName).setParent(currentContext).startSpan();
+      return tracer.spanBuilder(spanName).setParent(currentContext).setSpanKind(spanKind).startSpan();
     } else {
-      return tracer.spanBuilder(spanName).setNoParent().startSpan();
+      return tracer.spanBuilder(spanName).setNoParent().setSpanKind(spanKind).startSpan();
     }
   }
 
@@ -464,6 +494,7 @@ public static TraceCloseable createActivatedSpanFromW3cHttpHeaders(
 
     Span span = tracer.spanBuilder(spanName)
         .setParent(remote)
+        .setSpanKind(SpanKind.SERVER)
         .startSpan();
 
     Scope scope = span.makeCurrent();

hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTraceAllMethod.java

@@ -21,6 +21,8 @@
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
 import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.SpanKind;
+import io.opentelemetry.sdk.trace.ReadableSpan;
 import java.io.IOException;
 import org.junit.jupiter.api.Test;
 
@@ -54,6 +56,8 @@ default String defaultMethod() {
       return otherMethod("default");
     }
 
+    String localMethod();
+
     String skippedMethod();
 
     void throwingMethod() throws IOException;
@@ -69,6 +73,11 @@ default String defaultMethod() {
   public static class ServiceImpl implements Service {
 
     private boolean spanActive = false;
+    private SpanKind spanKind;
+
+    public SpanKind lastSpanKind() {
+      return spanKind;
+    }
 
     @Override
     public String otherMethod(String name) {
@@ -95,6 +104,17 @@ public String normalMethod() {
       return "normal";
     }
 
+    @Override
+    @InternalSpanKind
+    public String localMethod() {
+      Span span = Span.current();
+      this.spanActive = span.getSpanContext().isValid();
+      if (spanActive) {
+        this.spanKind = ((ReadableSpan) span).getKind();
+      }
+      return "local";
+    }
+
     public boolean wasSpanActive() {
       return spanActive;
     }

hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTracingAnnotations.java

@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.tracing;
+
+import static org.apache.hadoop.hdds.tracing.TracingUtil.createProxy;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import io.opentelemetry.api.trace.SpanKind;
+import java.io.IOException;
+import org.apache.hadoop.hdds.conf.InMemoryConfigurationForTesting;
+import org.apache.hadoop.hdds.conf.MutableConfigurationSource;
+import org.apache.hadoop.hdds.tracing.TestTraceAllMethod.Service;
+import org.apache.hadoop.hdds.tracing.TestTraceAllMethod.ServiceImpl;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Test for Tracing related annotations.
+ * {@link SkipTracing} , {@link InternalSpanKind}.
+ */
+public class TestTracingAnnotations {
+
+  private static MutableConfigurationSource tracingEnabled() {
+    MutableConfigurationSource config = new InMemoryConfigurationForTesting();
+    config.setBoolean("ozone.tracing.enabled", true);
+    return config;
+  }
+
+  @Test
+  public void testSkipTracingNoSpan() {
+    TracingUtil.initTracing("TestService", tracingEnabled());
+    ServiceImpl impl = new ServiceImpl();
+    Service serviceProxy = createProxy(impl, Service.class, tracingEnabled());
+
+    serviceProxy.skippedMethod();
+    assertFalse(impl.wasSpanActive(), "Span should NOT be created for @SkipTracing methods.");
+  }
+
+  @Test
+  public void testSkipTracingExceptionUnwrapped() {
+    TracingUtil.initTracing("TestService", tracingEnabled());
+    ServiceImpl impl = new ServiceImpl();
+    Service serviceProxy = createProxy(impl, Service.class, tracingEnabled());
+
+    IOException ex = assertThrows(IOException.class,
+        () -> serviceProxy.throwingMethod());
+    assertEquals("Original Exception", ex.getMessage());
+    assertFalse(impl.wasSpanActive(), "Span should NOT have been created for a @SkipTracing throwing method.");
+  }
+
+  @Test
+  public void testProxyNormalVsSkipped() {
+    TracingUtil.initTracing("TestService", tracingEnabled());
+    ServiceImpl impl = new ServiceImpl();
+    Service serviceProxy = createProxy(impl, Service.class, tracingEnabled());
+
+    serviceProxy.normalMethod();
+    assertTrue(impl.wasSpanActive(), "Normal method should have an active span.");
+  }
+
+  @Test
+  public void testInternalSpanKind() {
+    TracingUtil.initTracing("TestService", tracingEnabled());
+    ServiceImpl impl = new ServiceImpl();
+    Service serviceProxy = createProxy(impl, Service.class, tracingEnabled());
+    serviceProxy.localMethod();
+    assertTrue(impl.wasSpanActive(), "Local method should have an active span.");
+    assertEquals(SpanKind.INTERNAL, impl.lastSpanKind(), "@InternalSpanKind method should use INTERNAL.");
+  }
+
+}