OpenTelemetry: unified observability

<?php
// src/Middleware/TracingMiddleware.php

declare(strict_types=1);

namespace App\Middleware;

use OpenTelemetry\API\Globals;
use OpenTelemetry\API\Trace\SpanKind;
use OpenTelemetry\API\Trace\StatusCode;
use OpenTelemetry\Context\Context;
use OpenTelemetry\SDK\Trace\TracerProviderInterface;
use Symfony\Component\HttpFoundation\Request;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\HttpKernel\Event\RequestEvent;
use Symfony\Component\HttpKernel\Event\ResponseEvent;
use Symfony\Component\HttpKernel\Event\ExceptionEvent;
use Symfony\Component\HttpKernel\KernelEvents;
use Symfony\Component\EventDispatcher\EventSubscriberInterface;

final class TracingMiddleware implements EventSubscriberInterface
{
    public static function getSubscribedEvents(): array
    {
        return [
            KernelEvents::REQUEST => ['onRequest', 255],
            KernelEvents::RESPONSE => ['onResponse', -255],
            KernelEvents::EXCEPTION => ['onException', 0],
        ];
    }

    public function onRequest(RequestEvent $event): void
    {
        if (!$event->isMainRequest()) {
            return;
        }

        $request = $event->getRequest();
        $tracer = Globals::tracerProvider()->getTracer('symfony-app');

        // Extract parent context from incoming headers (W3C Trace Context)
        $parentContext = Globals::propagator()->extract(
            $request->headers->all(),
        );

        $span = $tracer->spanBuilder($request->getMethod() . ' ' . $request->getPathInfo())
            ->setParent($parentContext)
            ->setSpanKind(SpanKind::KIND_SERVER)
            ->setAttribute('http.method', $request->getMethod())
            ->setAttribute('http.url', $request->getUri())
            ->setAttribute('http.route', $request->attributes->get('_route', 'unknown'))
            ->setAttribute('http.user_agent', $request->headers->get('User-Agent', ''))
            ->startSpan();

        // Store span in request attributes for later use
        $request->attributes->set('_otel_span', $span);
        $request->attributes->set('_otel_scope', $span->activate());
    }

    public function onResponse(ResponseEvent $event): void
    {
        if (!$event->isMainRequest()) {
            return;
        }

        $request = $event->getRequest();
        $span = $request->attributes->get('_otel_span');
        $scope = $request->attributes->get('_otel_scope');

        if ($span === null) {
            return;
        }

        $response = $event->getResponse();
        $span->setAttribute('http.status_code', $response->getStatusCode());

        if ($response->getStatusCode() >= 400) {
            $span->setStatus(StatusCode::STATUS_ERROR, 'HTTP ' . $response->getStatusCode());
        }

        $scope->detach();
        $span->end();
    }

    public function onException(ExceptionEvent $event): void
    {
        $span = $event->getRequest()->attributes->get('_otel_span');
        if ($span === null) {
            return;
        }

        $span->recordException($event->getThrowable());
        $span->setStatus(StatusCode::STATUS_ERROR, $event->getThrowable()->getMessage());
    }
}

// internal/telemetry/init.go
package telemetry

import (
	"context"
	"time"

	"go.opentelemetry.io/otel"
	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
	"go.opentelemetry.io/otel/propagation"
	"go.opentelemetry.io/otel/sdk/resource"
	sdktrace "go.opentelemetry.io/otel/sdk/trace"
	semconv "go.opentelemetry.io/otel/semconv/v1.24.0"
)

// Init sets up the OpenTelemetry TracerProvider and returns a shutdown function.
func Init(ctx context.Context, serviceName, collectorEndpoint string) (func(context.Context) error, error) {
	// Create OTLP exporter (sends to OTel Collector via gRPC)
	exporter, err := otlptracegrpc.New(ctx,
		otlptracegrpc.WithEndpoint(collectorEndpoint),
		otlptracegrpc.WithInsecure(), // for local dev; use TLS in production
	)
	if err != nil {
		return nil, err
	}

	// Define resource (service metadata)
	res, err := resource.Merge(
		resource.Default(),
		resource.NewWithAttributes(
			semconv.SchemaURL,
			semconv.ServiceName(serviceName),
			semconv.ServiceVersion("1.0.0"),
			semconv.DeploymentEnvironment("production"),
		),
	)
	if err != nil {
		return nil, err
	}

	// Create TracerProvider with batch span processor
	tp := sdktrace.NewTracerProvider(
		sdktrace.WithBatcher(exporter, sdktrace.WithBatchTimeout(5*time.Second)),
		sdktrace.WithResource(res),
		sdktrace.WithSampler(sdktrace.ParentBased(
			sdktrace.TraceIDRatioBased(0.1), // sample 10%
		)),
	)

	// Register globally
	otel.SetTracerProvider(tp)
	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
		propagation.TraceContext{}, // W3C Trace Context
		propagation.Baggage{},     // W3C Baggage
	))

	return tp.Shutdown, nil
}

// Program.cs — SDK bootstrap
using App.Telemetry;
using OpenTelemetry.Exporter;
using OpenTelemetry.Resources;
using OpenTelemetry.Trace;

var builder = WebApplication.CreateBuilder(args);

builder.Services.AddOpenTelemetry()
    .ConfigureResource(resource => resource
        .AddService(serviceName: "order-service", serviceVersion: "1.0.0")
        .AddAttributes([new KeyValuePair<string, object>("deployment.environment", "production")]))
    .WithTracing(tracing => tracing
        .AddSource(Telemetry.Source.Name)
        .AddAspNetCoreInstrumentation()   // extracts W3C traceparent from incoming requests
        .AddHttpClientInstrumentation()   // injects traceparent into outgoing requests
        .SetSampler(new ParentBasedSampler(new TraceIdRatioBasedSampler(0.1))) // sample 10%
        .AddOtlpExporter(options =>
        {
            options.Endpoint = new Uri("http://otel-collector:4317");
            options.Protocol = OtlpExportProtocol.Grpc;
        }));

var app = builder.Build();
app.UseMiddleware<TracingMiddleware>();
app.Run();

// src/Telemetry/Telemetry.cs
using System.Diagnostics;

namespace App.Telemetry;

public static class Telemetry
{
    // One ActivitySource per service, registered above via AddSource
    public static readonly ActivitySource Source = new("order-service", "1.0.0");
}

// src/Telemetry/TracingMiddleware.cs
using System.Diagnostics;

namespace App.Telemetry;

public sealed class TracingMiddleware(RequestDelegate next)
{
    public async Task InvokeAsync(HttpContext context)
    {
        var request = context.Request;

        // Parent context is already restored by AddAspNetCoreInstrumentation,
        // so StartActivity attaches this span to the incoming trace
        using var activity = Telemetry.Source.StartActivity(
            $"{request.Method} {request.Path}",
            ActivityKind.Server);

        activity?.SetTag("http.method", request.Method);
        activity?.SetTag("http.url", $"{request.Scheme}://{request.Host}{request.Path}{request.QueryString}");
        activity?.SetTag("http.route", context.GetEndpoint()?.DisplayName ?? "unknown");
        activity?.SetTag("http.user_agent", request.Headers.UserAgent.ToString());

        try
        {
            await next(context);
        }
        catch (Exception exception)
        {
            activity?.AddException(exception);
            activity?.SetStatus(ActivityStatusCode.Error, exception.Message);
            throw;
        }

        var statusCode = context.Response.StatusCode;
        activity?.SetTag("http.status_code", statusCode);

        if (statusCode >= 400)
        {
            activity?.SetStatus(ActivityStatusCode.Error, $"HTTP {statusCode}");
        }
    }
}

# app/telemetry/tracing.py
from collections.abc import Awaitable, Callable

from opentelemetry import trace
from opentelemetry.baggage.propagation import W3CBaggagePropagator
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.propagate import extract, set_global_textmap
from opentelemetry.propagators.composite import CompositePropagator
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
from opentelemetry.trace import SpanKind, Status, StatusCode
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.requests import Request
from starlette.responses import Response

def init_tracing(service_name: str, collector_endpoint: str) -> TracerProvider:
    """Set up the TracerProvider and register it globally. Returns it for shutdown."""
    resource = Resource.create(
        {
            "service.name": service_name,
            "service.version": "1.0.0",
            "deployment.environment": "production",
        }
    )

    provider = TracerProvider(
        resource=resource,
        sampler=ParentBased(TraceIdRatioBased(0.1)),  # sample 10%
    )

    # OTLP exporter sends spans to the OTel Collector via gRPC
    exporter = OTLPSpanExporter(endpoint=collector_endpoint, insecure=True)
    provider.add_span_processor(BatchSpanProcessor(exporter, schedule_delay_millis=5_000))

    trace.set_tracer_provider(provider)
    set_global_textmap(
        CompositePropagator([TraceContextTextMapPropagator(), W3CBaggagePropagator()])
    )

    return provider

class TracingMiddleware(BaseHTTPMiddleware):
    """Creates a SERVER span for every incoming request."""

    async def dispatch(
        self,
        request: Request,
        call_next: Callable[[Request], Awaitable[Response]],
    ) -> Response:
        tracer = trace.get_tracer("order-service")

        # Extract parent context from incoming headers (W3C Trace Context)
        parent_context = extract(dict(request.headers))

        with tracer.start_as_current_span(
            f"{request.method} {request.url.path}",
            context=parent_context,
            kind=SpanKind.SERVER,
        ) as span:
            span.set_attribute("http.method", request.method)
            span.set_attribute("http.url", str(request.url))
            span.set_attribute("http.route", request.scope.get("route_path", "unknown"))
            span.set_attribute("http.user_agent", request.headers.get("user-agent", ""))

            try:
                response = await call_next(request)
            except Exception as exc:
                span.record_exception(exc)
                span.set_status(Status(StatusCode.ERROR, str(exc)))
                raise

            span.set_attribute("http.status_code", response.status_code)

            if response.status_code >= 400:
                span.set_status(Status(StatusCode.ERROR, f"HTTP {response.status_code}"))

            return response

<?php
// src/Service/OrderService.php

declare(strict_types=1);

namespace App\Service;

use OpenTelemetry\API\Globals;
use OpenTelemetry\API\Trace\SpanKind;
use OpenTelemetry\API\Trace\StatusCode;

final readonly class OrderService
{
    public function __construct(
        private PaymentGateway $paymentGateway,
        private OrderRepository $orderRepository,
    ) {}

    public function processOrder(int $orderId): OrderResult
    {
        $tracer = Globals::tracerProvider()->getTracer('order-service');

        // Create a span for the entire order processing
        $span = $tracer->spanBuilder('OrderService.processOrder')
            ->setSpanKind(SpanKind::KIND_INTERNAL)
            ->setAttribute('order.id', $orderId)
            ->startSpan();

        $scope = $span->activate();

        try {
            // Child span: load order from DB
            $loadSpan = $tracer->spanBuilder('DB: load order')
                ->setAttribute('db.system', 'postgresql')
                ->setAttribute('db.operation', 'SELECT')
                ->startSpan();

            $order = $this->orderRepository->find($orderId);
            $loadSpan->setAttribute('order.total', $order->getTotal());
            $loadSpan->end();

            if ($order === null) {
                throw new \RuntimeException("Order {$orderId} not found");
            }

            // Child span: process payment (external API call)
            $paymentSpan = $tracer->spanBuilder('Payment: charge')
                ->setSpanKind(SpanKind::KIND_CLIENT)
                ->setAttribute('payment.amount', $order->getTotal())
                ->setAttribute('payment.currency', 'USD')
                ->startSpan();

            $paymentResult = $this->paymentGateway->charge($order);
            $paymentSpan->setAttribute('payment.transaction_id', $paymentResult->transactionId);
            $paymentSpan->end();

            $span->setAttribute('order.status', 'completed');

            return new OrderResult(success: true, transactionId: $paymentResult->transactionId);
        } catch (\Throwable $e) {
            $span->recordException($e);
            $span->setStatus(StatusCode::STATUS_ERROR, $e->getMessage());

            return new OrderResult(success: false, error: $e->getMessage());
        } finally {
            $scope->detach();
            $span->end();
        }
    }
}

// internal/server/server.go
package server

import (
	"context"
	"log/slog"
	"net/http"

	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
	"go.opentelemetry.io/otel"
	"go.opentelemetry.io/otel/attribute"
	"go.opentelemetry.io/otel/codes"
)

func NewRouter() http.Handler {
	mux := http.NewServeMux()

	mux.HandleFunc("GET /api/orders/{id}", getOrder)
	mux.HandleFunc("POST /api/orders", createOrder)

	// Wrap entire mux with OTel HTTP instrumentation
	// Automatically creates spans for every HTTP request
	return otelhttp.NewHandler(mux, "api-server",
		otelhttp.WithMessageEvents(otelhttp.ReadEvents, otelhttp.WriteEvents),
	)
}

func getOrder(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()
	tracer := otel.Tracer("order-service")

	// Manual child span for business logic
	ctx, span := tracer.Start(ctx, "OrderService.GetOrder")
	defer span.End()

	orderID := r.PathValue("id")
	span.SetAttributes(attribute.String("order.id", orderID))

	// Child span for database query
	order, err := fetchOrderFromDB(ctx, orderID)
	if err != nil {
		span.RecordError(err)
		span.SetStatus(codes.Error, err.Error())
		http.Error(w, "order not found", http.StatusNotFound)
		return
	}

	span.SetAttributes(
		attribute.Float64("order.total", order.Total),
		attribute.String("order.status", order.Status),
	)

	writeJSON(w, order)
}

func fetchOrderFromDB(ctx context.Context, id string) (*Order, error) {
	tracer := otel.Tracer("order-service")
	ctx, span := tracer.Start(ctx, "DB: SELECT order")
	defer span.End()

	span.SetAttributes(
		attribute.String("db.system", "postgresql"),
		attribute.String("db.operation", "SELECT"),
		attribute.String("db.sql.table", "orders"),
	)

	// actual database query here...
	_ = ctx // use ctx for query cancellation

	return &Order{ID: id, Total: 99.99, Status: "pending"}, nil
}

// src/Services/OrderService.cs
using System.Diagnostics;
using App.Telemetry;

namespace App.Services;

public sealed class OrderService(
    IPaymentGateway paymentGateway,
    IOrderRepository orderRepository)
{
    public async Task<OrderResult> ProcessOrderAsync(int orderId, CancellationToken cancellationToken)
    {
        // Span for the entire order processing
        using var activity = Telemetry.Source.StartActivity(
            "OrderService.processOrder",
            ActivityKind.Internal);

        activity?.SetTag("order.id", orderId);

        try
        {
            Order? order;

            // Child span: load order from DB
            using (var loadActivity = Telemetry.Source.StartActivity("DB: load order"))
            {
                loadActivity?.SetTag("db.system", "postgresql");
                loadActivity?.SetTag("db.operation", "SELECT");

                order = await orderRepository.FindAsync(orderId, cancellationToken);
                loadActivity?.SetTag("order.total", order?.Total);
            }

            if (order is null)
            {
                throw new InvalidOperationException($"Order {orderId} not found");
            }

            // Child span: process payment (external API call)
            PaymentResult paymentResult;

            using (var paymentActivity = Telemetry.Source.StartActivity("Payment: charge", ActivityKind.Client))
            {
                paymentActivity?.SetTag("payment.amount", order.Total);
                paymentActivity?.SetTag("payment.currency", "USD");

                paymentResult = await paymentGateway.ChargeAsync(order, cancellationToken);
                paymentActivity?.SetTag("payment.transaction_id", paymentResult.TransactionId);
            }

            activity?.SetTag("order.status", "completed");

            return new OrderResult(Success: true, TransactionId: paymentResult.TransactionId);
        }
        catch (Exception exception)
        {
            activity?.AddException(exception);
            activity?.SetStatus(ActivityStatusCode.Error, exception.Message);

            return new OrderResult(Success: false, Error: exception.Message);
        }
    }
}

# app/services/order_service.py
from dataclasses import dataclass

from opentelemetry import trace
from opentelemetry.trace import SpanKind, Status, StatusCode

from app.domain.order import Order
from app.gateways.payment import PaymentGateway
from app.repositories.order import OrderRepository

tracer = trace.get_tracer("order-service")

@dataclass(frozen=True, slots=True)
class OrderResult:
    success: bool
    transaction_id: str | None = None
    error: str | None = None

class OrderService:
    def __init__(
        self,
        payment_gateway: PaymentGateway,
        order_repository: OrderRepository,
    ) -> None:
        self._payment_gateway = payment_gateway
        self._order_repository = order_repository

    async def process_order(self, order_id: int) -> OrderResult:
        # Span for the entire order processing
        with tracer.start_as_current_span(
            "OrderService.processOrder",
            kind=SpanKind.INTERNAL,
        ) as span:
            span.set_attribute("order.id", order_id)

            try:
                # Child span: load order from DB
                with tracer.start_as_current_span("DB: load order") as load_span:
                    load_span.set_attribute("db.system", "postgresql")
                    load_span.set_attribute("db.operation", "SELECT")

                    order: Order | None = await self._order_repository.find(order_id)

                    if order is None:
                        raise LookupError(f"Order {order_id} not found")

                    load_span.set_attribute("order.total", order.total)

                # Child span: process payment (external API call)
                with tracer.start_as_current_span(
                    "Payment: charge",
                    kind=SpanKind.CLIENT,
                ) as payment_span:
                    payment_span.set_attribute("payment.amount", order.total)
                    payment_span.set_attribute("payment.currency", "USD")

                    payment_result = await self._payment_gateway.charge(order)
                    payment_span.set_attribute(
                        "payment.transaction_id", payment_result.transaction_id
                    )

                span.set_attribute("order.status", "completed")

                return OrderResult(
                    success=True, transaction_id=payment_result.transaction_id
                )
            except Exception as exc:
                span.record_exception(exc)
                span.set_status(Status(StatusCode.ERROR, str(exc)))

                return OrderResult(success=False, error=str(exc))

// internal/client/payment.go
package client

import (
	"context"
	"fmt"
	"net/http"

	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
)

// PaymentClient wraps HTTP client with OTel instrumentation.
// Automatically propagates trace context to downstream services.
type PaymentClient struct {
	httpClient *http.Client
	baseURL    string
}

func NewPaymentClient(baseURL string) *PaymentClient {
	return &PaymentClient{
		// otelhttp.NewTransport wraps http.DefaultTransport:
		// - creates client span for outgoing request
		// - injects W3C traceparent header
		// - records duration and status
		httpClient: &http.Client{
			Transport: otelhttp.NewTransport(http.DefaultTransport),
		},
		baseURL: baseURL,
	}
}

func (c *PaymentClient) Charge(ctx context.Context, amount float64) error {
	// Context carries the current span — otelhttp will create child span
	// and add traceparent header to the outgoing request
	req, err := http.NewRequestWithContext(ctx, http.MethodPost,
		fmt.Sprintf("%s/api/charge", c.baseURL), nil)
	if err != nil {
		return err
	}

	resp, err := c.httpClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("payment failed: status %d", resp.StatusCode)
	}

	return nil
}

// Program.cs — registration
// AddHttpClientInstrumentation() makes every HttpClient created by
// IHttpClientFactory inject the W3C traceparent header automatically
builder.Services.AddHttpClient<PaymentClient>(client =>
{
    client.BaseAddress = new Uri(builder.Configuration["Payment:BaseUrl"]!);
});

// src/Clients/PaymentClient.cs
using System.Diagnostics;
using System.Net.Http.Json;
using App.Telemetry;

namespace App.Clients;

// PaymentClient wraps HttpClient with OTel instrumentation.
// Automatically propagates trace context to downstream services.
public sealed class PaymentClient(HttpClient httpClient)
{
    public async Task ChargeAsync(decimal amount, CancellationToken cancellationToken)
    {
        // Activity.Current carries the server span — the client span below
        // becomes its child, and traceparent is added to the outgoing request
        using var activity = Telemetry.Source.StartActivity("Payment: charge", ActivityKind.Client);
        activity?.SetTag("payment.amount", amount);

        using var response = await httpClient.PostAsJsonAsync(
            "/api/charge",
            new { amount },
            cancellationToken);

        activity?.SetTag("http.status_code", (int)response.StatusCode);

        if (!response.IsSuccessStatusCode)
        {
            throw new HttpRequestException($"payment failed: status {(int)response.StatusCode}");
        }
    }
}

# app/clients/payment.py
import httpx
from opentelemetry import trace
from opentelemetry.propagate import inject
from opentelemetry.trace import SpanKind

tracer = trace.get_tracer("order-service")

class PaymentClient:
    """HTTP client that propagates trace context to downstream services."""

    def __init__(self, base_url: str, client: httpx.AsyncClient) -> None:
        self._base_url = base_url
        self._client = client

    async def charge(self, amount: float) -> None:
        # The active span is taken from the current context — inject() writes
        # the W3C traceparent header derived from it into the outgoing request
        with tracer.start_as_current_span(
            "Payment: charge",
            kind=SpanKind.CLIENT,
        ) as span:
            span.set_attribute("payment.amount", amount)

            headers: dict[str, str] = {}
            inject(headers)

            response = await self._client.post(
                f"{self._base_url}/api/charge",
                headers=headers,
            )
            span.set_attribute("http.status_code", response.status_code)

            if response.status_code != httpx.codes.OK:
                raise RuntimeError(f"payment failed: status {response.status_code}")

<?php
// src/Service/MetricsService.php

declare(strict_types=1);

namespace App\Service;

use OpenTelemetry\API\Globals;
use OpenTelemetry\API\Metrics\CounterInterface;
use OpenTelemetry\API\Metrics\HistogramInterface;
use OpenTelemetry\API\Metrics\UpDownCounterInterface;

final class MetricsService
{
    private CounterInterface $ordersCreated;
    private HistogramInterface $orderProcessingDuration;
    private UpDownCounterInterface $activeConnections;

    public function __construct()
    {
        $meter = Globals::meterProvider()->getMeter('order-service');

        // Counter: number of created orders
        $this->ordersCreated = $meter->createCounter(
            'orders_created_total',
            'orders',
            'Total number of created orders'
        );

        // Histogram: order processing duration
        $this->orderProcessingDuration = $meter->createHistogram(
            'order_processing_duration_seconds',
            's',
            'Time to process an order'
        );

        // UpDownCounter: active DB connections
        $this->activeConnections = $meter->createUpDownCounter(
            'db_active_connections',
            'connections',
            'Number of active database connections'
        );
    }

    public function recordOrderCreated(string $paymentMethod, string $region): void
    {
        $this->ordersCreated->add(1, [
            'payment_method' => $paymentMethod,
            'region' => $region,
        ]);
    }

    public function recordProcessingDuration(float $seconds, string $orderType): void
    {
        $this->orderProcessingDuration->record($seconds, [
            'order_type' => $orderType,
        ]);
    }

    public function connectionOpened(): void
    {
        $this->activeConnections->add(1);
    }

    public function connectionClosed(): void
    {
        $this->activeConnections->add(-1);
    }
}

// internal/telemetry/metrics.go
package telemetry

import (
	"context"

	"go.opentelemetry.io/otel"
	"go.opentelemetry.io/otel/attribute"
	"go.opentelemetry.io/otel/metric"
)

// Metrics holds all application-level OTel metrics instruments.
type Metrics struct {
	OrdersCreated           metric.Int64Counter
	OrderProcessingDuration metric.Float64Histogram
	ActiveConnections       metric.Int64UpDownCounter
	QueueSize               metric.Int64Gauge
}

// NewMetrics creates and registers application metrics.
func NewMetrics() (*Metrics, error) {
	meter := otel.Meter("order-service")

	ordersCreated, err := meter.Int64Counter("orders_created_total",
		metric.WithDescription("Total number of created orders"),
		metric.WithUnit("orders"),
	)
	if err != nil {
		return nil, err
	}

	duration, err := meter.Float64Histogram("order_processing_duration_seconds",
		metric.WithDescription("Time to process an order"),
		metric.WithUnit("s"),
		metric.WithExplicitBucketBoundaries(0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10),
	)
	if err != nil {
		return nil, err
	}

	activeConns, err := meter.Int64UpDownCounter("db_active_connections",
		metric.WithDescription("Active database connections"),
		metric.WithUnit("connections"),
	)
	if err != nil {
		return nil, err
	}

	queueSize, err := meter.Int64Gauge("message_queue_size",
		metric.WithDescription("Current number of messages in queue"),
		metric.WithUnit("messages"),
	)
	if err != nil {
		return nil, err
	}

	return &Metrics{
		OrdersCreated:           ordersCreated,
		OrderProcessingDuration: duration,
		ActiveConnections:       activeConns,
		QueueSize:               queueSize,
	}, nil
}

// RecordOrderCreated increments the orders counter with attributes.
func (m *Metrics) RecordOrderCreated(ctx context.Context, paymentMethod, region string) {
	m.OrdersCreated.Add(ctx, 1,
		metric.WithAttributes(
			attribute.String("payment_method", paymentMethod),
			attribute.String("region", region),
		),
	)
}

// Program.cs — register the meter and export metrics via OTLP
using App.Telemetry;
using OpenTelemetry.Metrics;

builder.Services.AddSingleton<OrderMetrics>();
builder.Services.AddOpenTelemetry()
    .WithMetrics(metrics => metrics
        .AddMeter(OrderMetrics.MeterName)
        .AddView(
            instrumentName: "order_processing_duration_seconds",
            new ExplicitBucketHistogramConfiguration
            {
                Boundaries = [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
            })
        .AddOtlpExporter());

// src/Telemetry/OrderMetrics.cs
using System.Diagnostics.Metrics;

namespace App.Telemetry;

public sealed class OrderMetrics : IDisposable
{
    public const string MeterName = "order-service";

    private readonly Meter _meter;
    private readonly Counter<long> _ordersCreated;
    private readonly Histogram<double> _orderProcessingDuration;
    private readonly UpDownCounter<long> _activeConnections;
    private readonly Gauge<long> _queueSize;

    public OrderMetrics(IMeterFactory meterFactory)
    {
        _meter = meterFactory.Create(MeterName, "1.0.0");

        // Counter: number of created orders
        _ordersCreated = _meter.CreateCounter<long>(
            "orders_created_total",
            unit: "orders",
            description: "Total number of created orders");

        // Histogram: order processing duration
        _orderProcessingDuration = _meter.CreateHistogram<double>(
            "order_processing_duration_seconds",
            unit: "s",
            description: "Time to process an order");

        // UpDownCounter: active DB connections
        _activeConnections = _meter.CreateUpDownCounter<long>(
            "db_active_connections",
            unit: "connections",
            description: "Number of active database connections");

        // Gauge: current queue depth
        _queueSize = _meter.CreateGauge<long>(
            "message_queue_size",
            unit: "messages",
            description: "Current number of messages in queue");
    }

    public void RecordOrderCreated(string paymentMethod, string region) =>
        _ordersCreated.Add(
            1,
            new KeyValuePair<string, object?>("payment_method", paymentMethod),
            new KeyValuePair<string, object?>("region", region));

    public void RecordProcessingDuration(double seconds, string orderType) =>
        _orderProcessingDuration.Record(
            seconds,
            new KeyValuePair<string, object?>("order_type", orderType));

    public void ConnectionOpened() => _activeConnections.Add(1);

    public void ConnectionClosed() => _activeConnections.Add(-1);

    public void RecordQueueSize(long size) => _queueSize.Record(size);

    public void Dispose() => _meter.Dispose();
}

# app/telemetry/metrics.py
from opentelemetry import metrics
from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
from opentelemetry.sdk.resources import Resource

def init_metrics(service_name: str, collector_endpoint: str) -> MeterProvider:
    """Set up the MeterProvider with an OTLP exporter and register it globally."""
    reader = PeriodicExportingMetricReader(
        OTLPMetricExporter(endpoint=collector_endpoint, insecure=True),
        export_interval_millis=15_000,
    )

    provider = MeterProvider(
        resource=Resource.create({"service.name": service_name}),
        metric_readers=[reader],
        views=[
            View(
                instrument_name="order_processing_duration_seconds",
                aggregation=ExplicitBucketHistogramAggregation(
                    boundaries=[0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]
                ),
            )
        ],
    )

    metrics.set_meter_provider(provider)

    return provider

class OrderMetrics:
    """Holds all application-level OTel metrics instruments."""

    def __init__(self) -> None:
        meter = metrics.get_meter("order-service")

        # Counter: number of created orders
        self._orders_created = meter.create_counter(
            "orders_created_total",
            unit="orders",
            description="Total number of created orders",
        )

        # Histogram: order processing duration
        self._order_processing_duration = meter.create_histogram(
            "order_processing_duration_seconds",
            unit="s",
            description="Time to process an order",
        )

        # UpDownCounter: active DB connections
        self._active_connections = meter.create_up_down_counter(
            "db_active_connections",
            unit="connections",
            description="Number of active database connections",
        )

        # Gauge: current queue depth
        self._queue_size = meter.create_gauge(
            "message_queue_size",
            unit="messages",
            description="Current number of messages in queue",
        )

    def record_order_created(self, payment_method: str, region: str) -> None:
        self._orders_created.add(
            1,
            {"payment_method": payment_method, "region": region},
        )

    def record_processing_duration(self, seconds: float, order_type: str) -> None:
        self._order_processing_duration.record(seconds, {"order_type": order_type})

    def connection_opened(self) -> None:
        self._active_connections.add(1)

    def connection_closed(self) -> None:
        self._active_connections.add(-1)

    def record_queue_size(self, size: int) -> None:
        self._queue_size.set(size)

<?php
// src/Logger/TraceContextProcessor.php

declare(strict_types=1);

namespace App\Logger;

use Monolog\LogRecord;
use Monolog\Processor\ProcessorInterface;
use OpenTelemetry\API\Trace\Span;
use OpenTelemetry\Context\Context;

/**
 * Adds OpenTelemetry trace_id and span_id to every log record.
 * Enables jumping from log entry to full trace in Grafana.
 */
final class TraceContextProcessor implements ProcessorInterface
{
    public function __invoke(LogRecord $record): LogRecord
    {
        $span = Span::fromContext(Context::getCurrent());
        $spanContext = $span->getContext();

        if (!$spanContext->isValid()) {
            return $record;
        }

        return $record->with(
            extra: array_merge($record->extra, [
                'trace_id' => $spanContext->getTraceId(),
                'span_id' => $spanContext->getSpanId(),
                'trace_flags' => $spanContext->getTraceFlags(),
            ]),
        );
    }
}

// internal/logger/otel.go
package logger

import (
	"context"
	"log/slog"

	"go.opentelemetry.io/otel/trace"
)

// TraceHandler wraps slog.Handler to inject trace context into every log.
type TraceHandler struct {
	inner slog.Handler
}

func NewTraceHandler(inner slog.Handler) *TraceHandler {
	return &TraceHandler{inner: inner}
}

func (h *TraceHandler) Enabled(ctx context.Context, level slog.Level) bool {
	return h.inner.Enabled(ctx, level)
}

func (h *TraceHandler) Handle(ctx context.Context, record slog.Record) error {
	// Extract span context from ctx
	spanCtx := trace.SpanContextFromContext(ctx)
	if spanCtx.IsValid() {
		record.AddAttrs(
			slog.String("trace_id", spanCtx.TraceID().String()),
			slog.String("span_id", spanCtx.SpanID().String()),
		)
	}

	return h.inner.Handle(ctx, record)
}

func (h *TraceHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
	return &TraceHandler{inner: h.inner.WithAttrs(attrs)}
}

func (h *TraceHandler) WithGroup(name string) slog.Handler {
	return &TraceHandler{inner: h.inner.WithGroup(name)}
}

// Program.cs — OTel logging pipeline
// TraceId and SpanId of the active Activity are attached to every LogRecord
builder.Logging.AddOpenTelemetry(logging =>
{
    logging.IncludeScopes = true;
    logging.IncludeFormattedMessage = true;
    logging.AddOtlpExporter();
});

app.UseMiddleware<TraceContextLoggingMiddleware>();

// src/Logging/TraceContextLoggingMiddleware.cs
using System.Diagnostics;

namespace App.Logging;

/// <summary>
/// Adds OpenTelemetry trace_id and span_id to every log record written
/// while handling the request. Enables jumping from a log entry to the
/// full trace in Grafana.
/// </summary>
public sealed class TraceContextLoggingMiddleware(
    RequestDelegate next,
    ILogger<TraceContextLoggingMiddleware> logger)
{
    public async Task InvokeAsync(HttpContext context)
    {
        var activity = Activity.Current;

        if (activity is null)
        {
            await next(context);
            return;
        }

        var traceContext = new Dictionary<string, object>
        {
            ["trace_id"] = activity.TraceId.ToString(),
            ["span_id"] = activity.SpanId.ToString(),
            ["trace_flags"] = (int)activity.ActivityTraceFlags,
        };

        using (logger.BeginScope(traceContext))
        {
            await next(context);
        }
    }
}

# app/telemetry/logging.py
import json
import logging
from typing import Any

from opentelemetry import trace

class TraceContextFilter(logging.Filter):
    """Adds OpenTelemetry trace_id and span_id to every log record."""

    def filter(self, record: logging.LogRecord) -> bool:
        span_context = trace.get_current_span().get_span_context()

        if span_context.is_valid:
            record.trace_id = trace.format_trace_id(span_context.trace_id)
            record.span_id = trace.format_span_id(span_context.span_id)
            record.trace_flags = int(span_context.trace_flags)

        return True

class JsonFormatter(logging.Formatter):
    """Renders log records as JSON so Loki can index trace_id as a label."""

    def format(self, record: logging.LogRecord) -> str:
        payload: dict[str, Any] = {
            "message": record.getMessage(),
            "level_name": record.levelname,
            "channel": record.name,
            "datetime": self.formatTime(record, "%Y-%m-%dT%H:%M:%S%z"),
        }

        extra: dict[str, Any] = {}
        for key in ("trace_id", "span_id", "trace_flags"):
            value = getattr(record, key, None)
            if value is not None:
                extra[key] = value

        if extra:
            payload["extra"] = extra

        return json.dumps(payload, ensure_ascii=False)

def configure_logging(level: int = logging.INFO) -> None:
    """Wire the JSON formatter and the trace context filter into the root logger."""
    handler = logging.StreamHandler()
    handler.setFormatter(JsonFormatter())
    handler.addFilter(TraceContextFilter())

    root = logging.getLogger()
    root.handlers = [handler]
    root.setLevel(level)