gRPC Tracing — Distributed Tracing with OpenTelemetry for gRPC Services

package main

import (
    "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace"
    "go.opentelemetry.io/otel/sdk/trace"
    "google.golang.org/grpc"
)

func initTracer() {
    exporter, _ := otlptrace.New(context.Background())
    tp := trace.NewTracerProvider(
        trace.WithBatcher(exporter),
        trace.WithSampler(trace.ProbabilityBased(0.1)), // 10% sampling
    )
    otel.SetTracerProvider(tp)
}

func main() {
    initTracer()
    
    // Server-side instrumentation
    s := grpc.NewServer(
        grpc.StatsHandler(otelgrpc.NewServerHandler()),
    )
    
    // Client-side instrumentation
    conn, _ := grpc.Dial("localhost:50051",
        grpc.WithInsecure(),
        grpc.WithStatsHandler(otelgrpc.NewClientHandler()),
    )
}

// Add custom span attributes in resolvers
func (s *server) ReportThreat(ctx context.Context,
    req *pb.ThreatRequest) (*pb.ThreatResponse, error) {
    
    tracer := otel.Tracer("threat-service")
    ctx, span := tracer.Start(ctx, "ReportThreat")
    defer span.End()
    
    // Add custom attributes
    span.SetAttributes(
        attribute.String("threat.name", req.ThreatName),
        attribute.String("device.id", req.DeviceId),
        attribute.String("severity", req.Severity.String()),
    )
    
    result, err := s.processThreat(ctx, req)
    if err != nil {
        span.RecordError(err)
        span.SetStatus(codes.Error, err.Error())
        return nil, err
    }
    
    span.SetAttributes(
        attribute.String("result.status",
            result.Status.String()),
    )
    return result, nil
}

from opentelemetry import trace
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter \
    import OTLPSpanExporter
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.instrumentation.grpc import GrpcInstrumentor

# Initialize tracing
provider = TracerProvider()
processor = BatchSpanProcessor(OTLPSpanExporter())
provider.add_span_processor(processor)
trace.set_tracer_provider(provider)

# Instrument gRPC automatically
GrpcInstrumentor().instrument()

# Manual tracing in resolvers
tracer = trace.get_tracer(__name__)

class ThreatService(pb.ThreatServiceServicer):
    def ReportThreat(self, request, context):
        with tracer.start_as_current_span("ReportThreat") as span:
            span.set_attribute("threat.name", request.threat_name)
            span.set_attribute("device.id", request.device_id)
            
            # Trace downstream calls
            with tracer.start_as_current_span(
                "validate_device") as validate_span:
                device = self.device_service.get_device(
                    request.device_id)
                validate_span.set_attribute(
                    "device.found", str(device is not None))
            
            with tracer.start_as_current_span(
                "analyze_threat") as analyze_span:
                result = self.analyzer.analyze(
                    request.threat_name)
                analyze_span.set_attribute(
                    "severity", result.severity)
            
            return pb.ThreatResponse(status=result.status)

const grpc = require('@grpc/grpc-js');
const { diag, DiagConsoleLogger, DiagLogLevel } = require('@opentelemetry/api');
const { GrpcInstrumentation } = require('@opentelemetry/instrumentation-grpc');
const { NodeTracerProvider } = require('@opentelemetry/sdk-trace-node');
const { JaegerExporter } = require('@opentelemetry/exporter-jaeger');

// Setup tracing
const provider = new NodeTracerProvider();
const jaegerExporter = new JaegerExporter({
  endpoint: 'http://jaeger:14268/api/traces',
});

provider.addSpanProcessor(new BatchSpanProcessor(jaegerExporter));
provider.register();

// Instrument gRPC
const grpcInstrumentation = new GrpcInstrumentation({
  // Add custom attributes from metadata
  metadataToSpanAttributes: {
    client: {
      'authorization': 'auth.token',
      'x-correlation-id': 'correlation.id',
    },
    server: {
      'x-region': 'request.region',
    },
  },
});

grpcInstrumentation.setConfig({
  // Capture gRPC status code
  responseHook: (span, response) => {
    if (response.code) {
      span.setAttribute('grpc.status_code', response.code);
    }
  },
});

// Access the tracer in resolvers
const tracer = trace.getTracer('threat-service');

function reportThreat(call, callback) {
  const span = tracer.startSpan('process_threat', {
    attributes: {
      'threat.name': call.request.name,
    },
  });
  
  // Process...
  span.end();
  callback(null, response);
}