Observability (наблюдаемость) — способность понять внутреннее состояние системы по её внешним выходным данным. В отличие от мониторинга, который отвечает на известные вопросы, наблюдаемость позволяет отвечать на любые вопросы о поведении системы.
Столп
Что это
Для чего
Metrics
Числовые значения во времени
Алерты, тренды, SLI
Logs
Дискретные события
Детальная отладка, аудит
Traces
Путь запроса через систему
Анализ latency, зависимости
Метрики
Метрики — это числовые измерения, собираемые с определённым интервалом. Четыре типа метрик:
Тип
Описание
Пример
Counter
Только растёт
Количество запросов
Gauge
Растёт и уменьшается
CPU usage, memory
Histogram
Распределение значений
Latency buckets
Summary
Квантили
p50, p95, p99 latency
Логи
Логи — записи о дискретных событиях. Современный подход — structured logging (JSON вместо plain text).
# Plain text — плохо для парсинга
[2024-01-15 10:23:45] ERROR: Payment failed for user 12345
# Structured JSON — удобно для поиска и фильтрации
{"timestamp":"2024-01-15T10:23:45Z","level":"error","message":"Payment failed","user_id":12345,"amount":99.99,"currency":"USD","error_code":"INSUFFICIENT_FUNDS"}
Трейсы
Distributed tracing показывает путь запроса через все сервисы. Каждый трейс состоит из спанов (spans), образующих дерево вызовов.
Prometheus собирает метрики по HTTP (pull model). Приложение экспортирует метрики на эндпоинте /metrics.
Формат экспорта Prometheus:
# HELP http_requests_total Total HTTP requests
# TYPE http_requests_total counter
http_requests_total{method="GET",path="/api/orders",status="200"} 15234
http_requests_total{method="POST",path="/api/orders",status="201"} 892
http_requests_total{method="GET",path="/api/orders",status="500"} 12
# HELP http_request_duration_seconds HTTP request latency
# TYPE http_request_duration_seconds histogram
http_request_duration_seconds_bucket{le="0.05"} 9800
http_request_duration_seconds_bucket{le="0.1"} 12400
http_request_duration_seconds_bucket{le="0.25"} 14900
http_request_duration_seconds_bucket{le="0.5"} 15100
http_request_duration_seconds_bucket{le="+Inf"} 15234
PHP: Structured Logging
Monolog с JSON-форматированием
<?php
declare(strict_types=1);
namespace App\Logging;
use Psr\Log\LoggerInterface;
final readonly class StructuredLogger
{
public function __construct(
private LoggerInterface $logger,
) {}
/**
* Log an HTTP request with structured context.
*
* @param array<string, mixed> $extra Additional context
*/
public function logRequest(
string $method,
string $path,
int $statusCode,
float $durationMs,
array $extra = [],
): void {
$context = [
'http_method' => $method,
'http_path' => $path,
'http_status' => $statusCode,
'duration_ms' => round($durationMs, 2),
'trace_id' => $this->getTraceId(),
...$extra,
];
$level = match (true) {
$statusCode >= 500 => 'error',
$statusCode >= 400 => 'warning',
default => 'info',
};
$this->logger->log($level, 'HTTP request processed', $context);
}
/**
* Log a business event.
*/
public function logBusinessEvent(
string $event,
string $entityType,
string $entityId,
array $data = [],
): void {
$this->logger->info('Business event', [
'event' => $event,
'entity_type' => $entityType,
'entity_id' => $entityId,
'trace_id' => $this->getTraceId(),
...$data,
]);
}
private function getTraceId(): string
{
return $_SERVER['HTTP_X_TRACE_ID']
?? $_SERVER['HTTP_X_REQUEST_ID']
?? bin2hex(random_bytes(16));
}
}
package logging
import (
"log/slog"
"math"
"net/http"
"github.com/google/uuid"
)
// StructuredLogger provides structured logging with trace context.
type StructuredLogger struct {
logger *slog.Logger
}
// NewStructuredLogger creates a new StructuredLogger.
func NewStructuredLogger(logger *slog.Logger) *StructuredLogger {
return &StructuredLogger{logger: logger}
}
// LogRequest logs an HTTP request with structured context.
func (l *StructuredLogger) LogRequest(r *http.Request, statusCode int, durationMs float64, extra ...slog.Attr) {
attrs := []slog.Attr{
slog.String("http_method", r.Method),
slog.String("http_path", r.URL.Path),
slog.Int("http_status", statusCode),
slog.Float64("duration_ms", math.Round(durationMs*100)/100),
slog.String("trace_id", traceIDFromRequest(r)),
}
attrs = append(attrs, extra...)
level := slog.LevelInfo
switch {
case statusCode >= 500:
level = slog.LevelError
case statusCode >= 400:
level = slog.LevelWarn
}
l.logger.LogAttrs(r.Context(), level, "HTTP request processed", attrs...)
}
// LogBusinessEvent logs a business event.
func (l *StructuredLogger) LogBusinessEvent(r *http.Request, event, entityType, entityID string, extra ...slog.Attr) {
attrs := []slog.Attr{
slog.String("event", event),
slog.String("entity_type", entityType),
slog.String("entity_id", entityID),
slog.String("trace_id", traceIDFromRequest(r)),
}
attrs = append(attrs, extra...)
l.logger.LogAttrs(r.Context(), slog.LevelInfo, "Business event", attrs...)
}
func traceIDFromRequest(r *http.Request) string {
if id := r.Header.Get("X-Trace-ID"); id != "" {
return id
}
if id := r.Header.Get("X-Request-ID"); id != "" {
return id
}
return uuid.New().String()
}
namespace App.Logging;
using System.Diagnostics;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Logging;
// StructuredLogger emits JSON logs enriched with the current trace id.
public sealed class StructuredLogger(ILogger<StructuredLogger> logger, IHttpContextAccessor accessor)
{
// Log an HTTP request with structured context.
public void LogRequest(
string method,
string path,
int statusCode,
double durationMs,
IReadOnlyDictionary<string, object?>? extra = null)
{
var level = statusCode switch
{
>= 500 => LogLevel.Error,
>= 400 => LogLevel.Warning,
_ => LogLevel.Information,
};
var scope = new Dictionary<string, object?>
{
["http_method"] = method,
["http_path"] = path,
["http_status"] = statusCode,
["duration_ms"] = Math.Round(durationMs, 2),
["trace_id"] = GetTraceId(),
};
Merge(scope, extra);
using (logger.BeginScope(scope))
{
logger.Log(level, "HTTP request processed");
}
}
// Log a business event.
public void LogBusinessEvent(
string eventName,
string entityType,
string entityId,
IReadOnlyDictionary<string, object?>? data = null)
{
var scope = new Dictionary<string, object?>
{
["event"] = eventName,
["entity_type"] = entityType,
["entity_id"] = entityId,
["trace_id"] = GetTraceId(),
};
Merge(scope, data);
using (logger.BeginScope(scope))
{
logger.LogInformation("Business event");
}
}
private static void Merge(Dictionary<string, object?> target, IReadOnlyDictionary<string, object?>? source)
{
if (source is null)
{
return;
}
foreach (var (key, value) in source)
{
target[key] = value;
}
}
// Prefer the ambient Activity trace id; fall back to the incoming headers.
private string GetTraceId()
{
if (Activity.Current is { } activity)
{
return activity.TraceId.ToString();
}
var headers = accessor.HttpContext?.Request.Headers;
return headers?["X-Trace-Id"].FirstOrDefault()
?? headers?["X-Request-Id"].FirstOrDefault()
?? Guid.NewGuid().ToString("N");
}
}
import uuid
from typing import Any
import structlog
from opentelemetry import trace
logger = structlog.get_logger(__name__)
class StructuredLogger:
"""Emits JSON logs enriched with the current trace id."""
def __init__(self, log: structlog.BoundLogger | None = None) -> None:
self._log = log or logger
def log_request(
self,
method: str,
path: str,
status_code: int,
duration_ms: float,
headers: dict[str, str] | None = None,
**extra: Any,
) -> None:
"""Log an HTTP request with structured context."""
context = {
"http_method": method,
"http_path": path,
"http_status": status_code,
"duration_ms": round(duration_ms, 2),
"trace_id": self._trace_id(headers or {}),
**extra,
}
if status_code >= 500:
self._log.error("HTTP request processed", **context)
elif status_code >= 400:
self._log.warning("HTTP request processed", **context)
else:
self._log.info("HTTP request processed", **context)
def log_business_event(
self,
event: str,
entity_type: str,
entity_id: str,
headers: dict[str, str] | None = None,
**data: Any,
) -> None:
"""Log a business event."""
self._log.info(
"Business event",
event=event,
entity_type=entity_type,
entity_id=entity_id,
trace_id=self._trace_id(headers or {}),
**data,
)
@staticmethod
def _trace_id(headers: dict[str, str]) -> str:
# Prefer the active OpenTelemetry span; fall back to the incoming headers.
span_context = trace.get_current_span().get_span_context()
if span_context.is_valid:
return format(span_context.trace_id, "032x")
return headers.get("x-trace-id") or headers.get("x-request-id") or uuid.uuid4().hex
### Middleware для автоматического логирования
<?php
declare(strict_types=1);
namespace App\Middleware;
use App\Logging\StructuredLogger;
use Symfony\Component\HttpFoundation\Request;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\HttpKernel\Event\RequestEvent;
use Symfony\Component\HttpKernel\Event\ResponseEvent;
final class RequestLoggingMiddleware
{
private float $startTime;
public function __construct(
private readonly StructuredLogger $logger,
) {}
public function onKernelRequest(RequestEvent $event): void
{
if (!$event->isMainRequest()) {
return;
}
$this->startTime = microtime(true);
}
public function onKernelResponse(ResponseEvent $event): void
{
if (!$event->isMainRequest()) {
return;
}
$request = $event->getRequest();
$response = $event->getResponse();
$durationMs = (microtime(true) - $this->startTime) * 1000;
$this->logger->logRequest(
method: $request->getMethod(),
path: $request->getPathInfo(),
statusCode: $response->getStatusCode(),
durationMs: $durationMs,
extra: [
'ip' => $request->getClientIp(),
'user_agent' => $request->headers->get('User-Agent', 'unknown'),
'request_size' => $request->headers->get('Content-Length', '0'),
'response_size' => strlen($response->getContent() ?: ''),
],
);
}
}
package middleware
import (
"log/slog"
"net/http"
"time"
)
// RequestLoggingMiddleware logs every HTTP request with duration and metadata.
func RequestLoggingMiddleware(logger *slog.Logger) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
// Wrap ResponseWriter to capture status code and size
rw := &responseWriter{ResponseWriter: w, statusCode: http.StatusOK}
next.ServeHTTP(rw, r)
durationMs := float64(time.Since(start).Microseconds()) / 1000.0
level := slog.LevelInfo
switch {
case rw.statusCode >= 500:
level = slog.LevelError
case rw.statusCode >= 400:
level = slog.LevelWarn
}
logger.LogAttrs(r.Context(), level, "HTTP request processed",
slog.String("http_method", r.Method),
slog.String("http_path", r.URL.Path),
slog.Int("http_status", rw.statusCode),
slog.Float64("duration_ms", durationMs),
slog.String("ip", r.RemoteAddr),
slog.String("user_agent", r.UserAgent()),
slog.Int64("request_size", r.ContentLength),
slog.Int("response_size", rw.bytesWritten),
)
})
}
}
type responseWriter struct {
http.ResponseWriter
statusCode int
bytesWritten int
}
func (rw *responseWriter) WriteHeader(code int) {
rw.statusCode = code
rw.ResponseWriter.WriteHeader(code)
}
func (rw *responseWriter) Write(b []byte) (int, error) {
n, err := rw.ResponseWriter.Write(b)
rw.bytesWritten += n
return n, err
}
namespace App.Middleware;
using System.Diagnostics;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Logging;
// RequestLoggingMiddleware logs every HTTP request with duration and metadata.
public sealed class RequestLoggingMiddleware(RequestDelegate next, ILogger<RequestLoggingMiddleware> logger)
{
public async Task InvokeAsync(HttpContext context)
{
var startTimestamp = Stopwatch.GetTimestamp();
await next(context);
var durationMs = Stopwatch.GetElapsedTime(startTimestamp).TotalMilliseconds;
var statusCode = context.Response.StatusCode;
var level = statusCode switch
{
>= 500 => LogLevel.Error,
>= 400 => LogLevel.Warning,
_ => LogLevel.Information,
};
// Named placeholders are emitted as structured fields by the JSON console formatter
logger.Log(
level,
"HTTP request processed {http_method} {http_path} {http_status} {duration_ms} {ip} {user_agent} {request_size} {response_size}",
context.Request.Method,
context.Request.Path.Value ?? "/",
statusCode,
Math.Round(durationMs, 2),
context.Connection.RemoteIpAddress?.ToString() ?? "unknown",
context.Request.Headers.UserAgent.ToString(),
context.Request.ContentLength ?? 0,
context.Response.ContentLength ?? 0);
}
}
// Registration in Program.cs:
// app.UseMiddleware<RequestLoggingMiddleware>();
import time
import structlog
from starlette.types import ASGIApp, Message, Receive, Scope, Send
logger = structlog.get_logger(__name__)
class RequestLoggingMiddleware:
"""ASGI middleware that logs every HTTP request with duration and metadata."""
def __init__(self, app: ASGIApp) -> None:
self._app = app
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
if scope["type"] != "http":
await self._app(scope, receive, send)
return
start = time.perf_counter()
status_code = 500
response_size = 0
async def send_wrapper(message: Message) -> None:
# Capture the status code and response size as they stream out
nonlocal status_code, response_size
if message["type"] == "http.response.start":
status_code = message["status"]
elif message["type"] == "http.response.body":
response_size += len(message.get("body", b""))
await send(message)
await self._app(scope, receive, send_wrapper)
headers = {key.decode().lower(): value.decode() for key, value in scope.get("headers", [])}
context = {
"http_method": scope["method"],
"http_path": scope["path"],
"http_status": status_code,
"duration_ms": round((time.perf_counter() - start) * 1000, 2),
"ip": scope.get("client", ("unknown", 0))[0],
"user_agent": headers.get("user-agent", "unknown"),
"request_size": int(headers.get("content-length", 0)),
"response_size": response_size,
}
if status_code >= 500:
logger.error("HTTP request processed", **context)
elif status_code >= 400:
logger.warning("HTTP request processed", **context)
else:
logger.info("HTTP request processed", **context)
## Экспорт метрик для Prometheus
<?php
declare(strict_types=1);
namespace App\Metrics;
/**
* Simple Prometheus metrics collector.
* In production, use promphp/prometheus_client_php library.
*/
final class MetricsCollector
{
/** @var array<string, array{type: string, help: string, values: array}> */
private array $metrics = [];
public function registerCounter(string $name, string $help): void
{
$this->metrics[$name] = [
'type' => 'counter',
'help' => $help,
'values' => [],
];
}
public function registerHistogram(string $name, string $help, array $buckets): void
{
$this->metrics[$name] = [
'type' => 'histogram',
'help' => $help,
'buckets' => $buckets,
'values' => [],
];
}
/**
* Increment a counter.
*
* @param array<string, string> $labels
*/
public function incrementCounter(string $name, array $labels = [], float $value = 1.0): void
{
$key = $this->labelsToKey($labels);
$this->metrics[$name]['values'][$key] ??= 0;
$this->metrics[$name]['values'][$key] += $value;
}
/**
* Observe a value for a histogram.
*
* @param array<string, string> $labels
*/
public function observeHistogram(string $name, float $value, array $labels = []): void
{
$key = $this->labelsToKey($labels);
$this->metrics[$name]['values'][$key][] = $value;
}
/**
* Render all metrics in Prometheus exposition format.
*/
public function render(): string
{
$output = '';
foreach ($this->metrics as $name => $metric) {
$output .= "# HELP {$name} {$metric['help']}\n";
$output .= "# TYPE {$name} {$metric['type']}\n";
foreach ($metric['values'] as $labels => $value) {
$labelStr = $labels ? "{{$labels}}" : '';
$output .= "{$name}{$labelStr} {$value}\n";
}
$output .= "\n";
}
return $output;
}
private function labelsToKey(array $labels): string
{
$parts = [];
foreach ($labels as $key => $value) {
$parts[] = "{$key}=\"{$value}\"";
}
return implode(',', $parts);
}
}
package metrics
import (
"net/http"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
// Collector registers and exposes Prometheus metrics.
// In production, use github.com/prometheus/client_golang directly.
type Collector struct {
requestsTotal *prometheus.CounterVec
requestDuration *prometheus.HistogramVec
}
// NewCollector creates a Collector with standard HTTP metrics.
func NewCollector(reg prometheus.Registerer) *Collector {
c := &Collector{
requestsTotal: prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "http_requests_total",
Help: "Total HTTP requests",
},
[]string{"method", "path", "status"},
),
requestDuration: prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "http_request_duration_seconds",
Help: "HTTP request latency",
Buckets: []float64{0.05, 0.1, 0.25, 0.5, 1.0},
},
[]string{"method", "path"},
),
}
reg.MustRegister(c.requestsTotal, c.requestDuration)
return c
}
// IncRequestsTotal increments the request counter.
func (c *Collector) IncRequestsTotal(method, path, status string) {
c.requestsTotal.WithLabelValues(method, path, status).Inc()
}
// ObserveDuration records a request duration.
func (c *Collector) ObserveDuration(method, path string, seconds float64) {
c.requestDuration.WithLabelValues(method, path).Observe(seconds)
}
// Handler returns an HTTP handler that serves /metrics.
func Handler() http.Handler {
return promhttp.Handler()
}
namespace App.Metrics;
using Prometheus;
// MetricsCollector registers and exposes Prometheus metrics.
// In production, use the prometheus-net package directly.
public sealed class MetricsCollector
{
private readonly Counter _requestsTotal = Prometheus.Metrics.CreateCounter(
"http_requests_total",
"Total HTTP requests",
new CounterConfiguration { LabelNames = ["method", "path", "status"] });
private readonly Histogram _requestDuration = Prometheus.Metrics.CreateHistogram(
"http_request_duration_seconds",
"HTTP request latency",
new HistogramConfiguration
{
LabelNames = ["method", "path"],
Buckets = [0.05, 0.1, 0.25, 0.5, 1.0],
});
// Increment the request counter.
public void IncRequestsTotal(string method, string path, string status) =>
_requestsTotal.WithLabels(method, path, status).Inc();
// Record a request duration in seconds.
public void ObserveDuration(string method, string path, double seconds) =>
_requestDuration.WithLabels(method, path).Observe(seconds);
}
// The exposition format is rendered by the library, not by hand.
// Expose /metrics for Prometheus scraping in Program.cs:
// app.MapMetrics();
from prometheus_client import CONTENT_TYPE_LATEST, Counter, Histogram, generate_latest
# Collectors are module-level singletons: the default registry rejects duplicates.
# In production, use prometheus_client directly.
REQUESTS_TOTAL = Counter(
"http_requests_total",
"Total HTTP requests",
labelnames=("method", "path", "status"),
)
REQUEST_DURATION = Histogram(
"http_request_duration_seconds",
"HTTP request latency",
labelnames=("method", "path"),
buckets=(0.05, 0.1, 0.25, 0.5, 1.0),
)
class MetricsCollector:
"""Thin facade over the Prometheus collectors."""
@staticmethod
def inc_requests_total(method: str, path: str, status: str) -> None:
"""Increment the request counter."""
REQUESTS_TOTAL.labels(method=method, path=path, status=status).inc()
@staticmethod
def observe_duration(method: str, path: str, seconds: float) -> None:
"""Record a request duration in seconds."""
REQUEST_DURATION.labels(method=method, path=path).observe(seconds)
@staticmethod
def render() -> tuple[bytes, str]:
"""Render all metrics in the Prometheus exposition format."""
return generate_latest(), CONTENT_TYPE_LATEST
### Использование в Symfony Controller
<?php
declare(strict_types=1);
namespace App\Controller;
use App\Metrics\MetricsCollector;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\Routing\Attribute\Route;
final class MetricsController
{
public function __construct(
private readonly MetricsCollector $collector,
) {}
#[Route('/metrics', methods: ['GET'])]
public function __invoke(): Response
{
return new Response(
content: $this->collector->render(),
headers: ['Content-Type' => 'text/plain; charset=utf-8'],
);
}
}
using Prometheus;
var builder = WebApplication.CreateBuilder(args);
var app = builder.Build();
// Register the /metrics endpoint for Prometheus scraping
app.MapMetrics();
app.Run();
from fastapi import FastAPI, Response
from prometheus_client import CONTENT_TYPE_LATEST, generate_latest
app = FastAPI()
@app.get("/metrics")
def metrics() -> Response:
"""Expose the /metrics endpoint for Prometheus scraping."""
return Response(content=generate_latest(), media_type=CONTENT_TYPE_LATEST)
## Практики алертинга
Правила хороших алертов
Принцип
Описание
Actionable
Алерт требует конкретного действия
Symptom-based
Алертить на симптомы, не причины
Low noise
Минимизировать ложные срабатывания
Documented
Каждый алерт содержит runbook
Tiered
Severity определяет канал уведомления
Уровни severity
Severity
Описание
Реакция
P1 Critical
Сервис недоступен
Немедленно, on-call
P2 High
Деградация для >10% пользователей
В течение 30 минут
P3 Medium
Деградация для <10% пользователей
В рабочие часы
P4 Low
Потенциальная проблема
При планировании
Практика: Если алерт срабатывает и не требует действия — удалите его. Каждый алерт должен будить on-call инженера только если это действительно необходимо.
OpenTelemetry
OpenTelemetry — vendor-neutral стандарт для сбора телеметрии. Объединяет метрики, логи и трейсы в единый SDK.
<?php
declare(strict_types=1);
namespace App\Tracing;
/**
* Simplified tracing context propagation.
* In production use open-telemetry/opentelemetry-php SDK.
*/
final class TraceContext
{
private string $traceId;
private string $spanId;
private ?string $parentSpanId;
public function __construct(?string $traceId = null)
{
$this->traceId = $traceId ?? bin2hex(random_bytes(16));
$this->spanId = bin2hex(random_bytes(8));
$this->parentSpanId = null;
}
public function createChildSpan(string $operationName): SpanRecord
{
$childSpanId = bin2hex(random_bytes(8));
return new SpanRecord(
traceId: $this->traceId,
spanId: $childSpanId,
parentSpanId: $this->spanId,
operationName: $operationName,
startTime: microtime(true),
);
}
public function getTraceId(): string
{
return $this->traceId;
}
/**
* Extract trace context from HTTP headers (W3C Trace Context).
*/
public static function fromHeaders(array $headers): self
{
$traceparent = $headers['traceparent'] ?? null;
if ($traceparent !== null && preg_match('/^00-([a-f0-9]{32})-([a-f0-9]{16})-\d{2}$/', $traceparent, $m)) {
$context = new self($m[1]);
$context->parentSpanId = $m[2];
return $context;
}
return new self();
}
}
final readonly class SpanRecord
{
public function __construct(
public string $traceId,
public string $spanId,
public string $parentSpanId,
public string $operationName,
public float $startTime,
public ?float $endTime = null,
public array $attributes = [],
) {}
public function finish(): self
{
return new self(
traceId: $this->traceId,
spanId: $this->spanId,
parentSpanId: $this->parentSpanId,
operationName: $this->operationName,
startTime: $this->startTime,
endTime: microtime(true),
attributes: $this->attributes,
);
}
public function getDurationMs(): float
{
$end = $this->endTime ?? microtime(true);
return ($end - $this->startTime) * 1000;
}
}
package tracing
import (
"context"
"fmt"
"net/http"
"regexp"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
)
// In production, use go.opentelemetry.io/otel SDK.
// This example shows the standard OpenTelemetry Go usage.
var tracer = otel.Tracer("myapp")
// StartSpan creates a child span from the current context.
func StartSpan(ctx context.Context, operationName string, attrs ...attribute.KeyValue) (context.Context, trace.Span) {
ctx, span := tracer.Start(ctx, operationName,
trace.WithAttributes(attrs...),
)
return ctx, span
}
// TraceIDFromContext extracts the trace ID from the current span.
func TraceIDFromContext(ctx context.Context) string {
span := trace.SpanFromContext(ctx)
if span.SpanContext().HasTraceID() {
return span.SpanContext().TraceID().String()
}
return ""
}
// W3C Trace Context header regex for manual parsing.
var traceparentRe = regexp.MustCompile(`^00-([a-f0-9]{32})-([a-f0-9]{16})-\d{2}$`)
// ExtractTraceParent parses a W3C traceparent header manually.
func ExtractTraceParent(header string) (traceID, parentSpanID string, ok bool) {
m := traceparentRe.FindStringSubmatch(header)
if len(m) != 3 {
return "", "", false
}
return m[1], m[2], true
}
// TracingMiddleware adds OpenTelemetry spans to HTTP requests.
func TracingMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx, span := tracer.Start(r.Context(),
fmt.Sprintf("%s %s", r.Method, r.URL.Path),
trace.WithAttributes(
attribute.String("http.method", r.Method),
attribute.String("http.url", r.URL.String()),
),
)
defer span.End()
next.ServeHTTP(w, r.WithContext(ctx))
})
}
namespace App.Tracing;
using System.Diagnostics;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Http.Extensions;
// In production, use the OpenTelemetry .NET SDK.
// .NET has no separate span type: OpenTelemetry spans are System.Diagnostics.Activity.
public static class Tracing
{
public static readonly ActivitySource Source = new("myapp");
// Start a child span from the current ambient context.
public static Activity? StartSpan(string operationName, params (string Key, object? Value)[] attributes)
{
var activity = Source.StartActivity(operationName);
foreach (var (key, value) in attributes)
{
activity?.SetTag(key, value);
}
return activity;
}
// Extract the trace id from the current span.
public static string TraceIdFromContext() => Activity.Current?.TraceId.ToString() ?? string.Empty;
// Parse a W3C traceparent header. ActivityContext.TryParse implements the spec.
public static bool TryExtractTraceParent(string header, out string traceId, out string parentSpanId)
{
traceId = string.Empty;
parentSpanId = string.Empty;
if (!ActivityContext.TryParse(header, null, out var context))
{
return false;
}
traceId = context.TraceId.ToString();
parentSpanId = context.SpanId.ToString();
return true;
}
}
// TracingMiddleware adds OpenTelemetry spans to HTTP requests.
public sealed class TracingMiddleware(RequestDelegate next)
{
public async Task InvokeAsync(HttpContext context)
{
using var activity = Tracing.Source.StartActivity(
$"{context.Request.Method} {context.Request.Path}",
ActivityKind.Server);
activity?.SetTag("http.method", context.Request.Method);
activity?.SetTag("http.url", context.Request.GetDisplayUrl());
await next(context);
}
}
import re
from opentelemetry import trace
from opentelemetry.trace import SpanKind
from starlette.types import ASGIApp, Receive, Scope, Send
# In production, use the opentelemetry-sdk package.
tracer = trace.get_tracer("myapp")
# W3C Trace Context header pattern for manual parsing.
TRACEPARENT_RE = re.compile(r"^00-([a-f0-9]{32})-([a-f0-9]{16})-\d{2}$")
def trace_id_from_context() -> str:
"""Return the trace id of the currently active span, or an empty string."""
span_context = trace.get_current_span().get_span_context()
return format(span_context.trace_id, "032x") if span_context.is_valid else ""
def extract_traceparent(header: str) -> tuple[str, str] | None:
"""Parse a W3C traceparent header manually: (trace_id, parent_span_id)."""
match = TRACEPARENT_RE.match(header)
return (match.group(1), match.group(2)) if match else None
class TracingMiddleware:
"""ASGI middleware that opens a server span for every HTTP request."""
def __init__(self, app: ASGIApp) -> None:
self._app = app
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
if scope["type"] != "http":
await self._app(scope, receive, send)
return
name = f"{scope['method']} {scope['path']}"
with tracer.start_as_current_span(name, kind=SpanKind.SERVER) as span:
span.set_attribute("http.method", scope["method"])
span.set_attribute("http.url", scope["path"])
await self._app(scope, receive, send)
## Dashboards: что показывать
Уровни dashboard-ов
Уровень
Аудитория
Содержание
Business
Менеджмент
KPI, конверсии, revenue
Service
Разработчики
Latency, errors, throughput
Infrastructure
SRE/DevOps
CPU, RAM, disk, network
Debug
On-call
Детальные метрики для troubleshooting
Совет: Начинайте с USE/RED метрик для каждого сервиса. Добавляйте детали по мере необходимости. Dashboard, на который никто не смотрит — мёртвый dashboard.