Паттерны отказоустойчивости

Зачем нужна отказоустойчивость

В распределенной системе сбои -- это не исключение, а норма. Сеть ненадежна, сервисы перезапускаются, базы данных перегружаются. Отказоустойчивость (resilience) -- это способность системы продолжать работу при частичных сбоях.

Типы сбоев

Паттерн 1: Retry (Повторная попытка)

Простейший паттерн. При временном сбое повторяем запрос.

<?php

declare(strict_types=1);

/**
 * Retry pattern with exponential backoff and jitter
 *
 * Simple retry: can overwhelm a recovering service
 * Exponential backoff: increasing delay between retries
 * Jitter: randomized delay to prevent thundering herd
 */
final class RetryHandler
{
    public function __construct(
        private readonly int $maxRetries = 3,
        private readonly int $baseDelayMs = 100,
        private readonly int $maxDelayMs = 10000,
        private readonly LoggerInterface $logger = new NullLogger(),
    ) {}

    /**
     * Execute an operation with retry logic.
     *
     * @template T
     * @param callable(): T $operation
     * @param array<class-string<\Throwable>> $retryableExceptions
     * @return T
     */
    public function execute(callable $operation, array $retryableExceptions = [\Throwable::class]): mixed
    {
        $lastException = null;

        for ($attempt = 0; $attempt <= $this->maxRetries; $attempt++) {
            try {
                return $operation();
            } catch (\Throwable $e) {
                $lastException = $e;

                if (!$this->isRetryable($e, $retryableExceptions)) {
                    throw $e; // Not retryable: fail immediately
                }

                if ($attempt === $this->maxRetries) {
                    break; // Last attempt: no more retries
                }

                $delay = $this->calculateDelay($attempt);

                $this->logger->warning('Operation failed, retrying', [
                    'attempt' => $attempt + 1,
                    'max_retries' => $this->maxRetries,
                    'delay_ms' => $delay,
                    'error' => $e->getMessage(),
                ]);

                usleep($delay * 1000); // Convert ms to us
            }
        }

        throw $lastException;
    }

    /**
     * Exponential backoff with full jitter.
     * Delay = random(0, min(maxDelay, baseDelay * 2^attempt))
     */
    private function calculateDelay(int $attempt): int
    {
        $exponentialDelay = $this->baseDelayMs * (2 ** $attempt);
        $cappedDelay = min($exponentialDelay, $this->maxDelayMs);

        // Full jitter: random between 0 and capped delay
        return random_int(0, $cappedDelay);
    }

    private function isRetryable(\Throwable $e, array $retryableExceptions): bool
    {
        foreach ($retryableExceptions as $className) {
            if ($e instanceof $className) {
                return true;
            }
        }

        return false;
    }
}

// Usage:
// $retry = new RetryHandler(maxRetries: 3, baseDelayMs: 200);
// $result = $retry->execute(
//     fn() => $httpClient->get('https://api.external.com/data'),
//     [TimeoutException::class, ServiceUnavailableException::class],
// );

Паттерн 2: Circuit Breaker (Предохранитель)

Предотвращает каскадные сбои, отключая вызовы к неработающему сервису.

Состояния

         Success
    ┌──────────────┐
    │              │
    ▼              │
[CLOSED] ──Failure threshold──► [OPEN] ──Timer expires──► [HALF-OPEN]
    ▲                              ▲                          │
    │                              │                          │
    └──────────────────────────────┘                          │
    │              Success                    Failure         │
    └─────────────────────────────────────────────────────────┘

• CLOSED -- нормальная работа, считаем ошибки
• OPEN -- сервис сломан, сразу возвращаем ошибку (не вызываем)
• HALF-OPEN -- пробуем один запрос, чтобы проверить восстановление

<?php

declare(strict_types=1);

/**
 * Circuit Breaker implementation
 *
 * Prevents cascading failures by cutting off calls to failing services.
 * Instead of waiting for timeout on every call to a dead service,
 * fails fast and gives the service time to recover.
 */
enum CircuitState: string
{
    case Closed = 'closed';
    case Open = 'open';
    case HalfOpen = 'half_open';
}

final class CircuitBreaker
{
    private CircuitState $state = CircuitState::Closed;
    private int $failureCount = 0;
    private int $successCount = 0;
    private ?float $openedAt = null;
    private ?float $lastFailureAt = null;

    public function __construct(
        private readonly string $name,
        private readonly int $failureThreshold = 5,
        private readonly int $successThreshold = 3,
        private readonly int $openDurationSeconds = 30,
        private readonly LoggerInterface $logger = new NullLogger(),
        private readonly ?MetricsCollector $metrics = null,
    ) {}

    /**
     * Execute an operation through the circuit breaker.
     *
     * @template T
     * @param callable(): T $operation
     * @param callable(): T|null $fallback
     * @return T
     */
    public function execute(callable $operation, ?callable $fallback = null): mixed
    {
        if (!$this->canExecute()) {
            $this->recordMetric('circuit_breaker.rejected');
            $this->logger->warning('Circuit breaker is OPEN, rejecting call', [
                'circuit' => $this->name,
            ]);

            if ($fallback !== null) {
                return $fallback();
            }

            throw new CircuitOpenException(
                "Circuit breaker '{$this->name}' is open. Service is unavailable."
            );
        }

        try {
            $result = $operation();
            $this->onSuccess();
            return $result;
        } catch (\Throwable $e) {
            $this->onFailure();
            throw $e;
        }
    }

    private function canExecute(): bool
    {
        return match ($this->state) {
            CircuitState::Closed => true,
            CircuitState::HalfOpen => true, // Allow one test request
            CircuitState::Open => $this->shouldAttemptReset(),
        };
    }

    /**
     * After open duration, transition to half-open to test recovery.
     */
    private function shouldAttemptReset(): bool
    {
        if ($this->openedAt === null) {
            return false;
        }

        $elapsed = microtime(true) - $this->openedAt;

        if ($elapsed >= $this->openDurationSeconds) {
            $this->transitionTo(CircuitState::HalfOpen);
            return true;
        }

        return false;
    }

    private function onSuccess(): void
    {
        $this->recordMetric('circuit_breaker.success');

        match ($this->state) {
            CircuitState::HalfOpen => $this->handleHalfOpenSuccess(),
            default => $this->failureCount = 0,
        };
    }

    private function handleHalfOpenSuccess(): void
    {
        $this->successCount++;

        if ($this->successCount >= $this->successThreshold) {
            // Enough successes: service has recovered
            $this->transitionTo(CircuitState::Closed);
        }
    }

    private function onFailure(): void
    {
        $this->recordMetric('circuit_breaker.failure');
        $this->lastFailureAt = microtime(true);
        $this->failureCount++;

        match ($this->state) {
            CircuitState::Closed => $this->handleClosedFailure(),
            CircuitState::HalfOpen => $this->transitionTo(CircuitState::Open),
            default => null,
        };
    }

    private function handleClosedFailure(): void
    {
        if ($this->failureCount >= $this->failureThreshold) {
            $this->transitionTo(CircuitState::Open);
        }
    }

    private function transitionTo(CircuitState $newState): void
    {
        $previousState = $this->state;
        $this->state = $newState;

        $this->logger->info('Circuit breaker state changed', [
            'circuit' => $this->name,
            'from' => $previousState->value,
            'to' => $newState->value,
        ]);

        match ($newState) {
            CircuitState::Open => $this->openedAt = microtime(true),
            CircuitState::Closed => $this->reset(),
            CircuitState::HalfOpen => $this->successCount = 0,
        };
    }

    private function reset(): void
    {
        $this->failureCount = 0;
        $this->successCount = 0;
        $this->openedAt = null;
    }

    public function getState(): CircuitState
    {
        return $this->state;
    }

    private function recordMetric(string $name): void
    {
        $this->metrics?->increment($name, ['circuit' => $this->name]);
    }
}

Использование Circuit Breaker

<?php

declare(strict_types=1);

/**
 * Practical usage of Circuit Breaker in a service
 */
final class PaymentServiceClient
{
    private CircuitBreaker $circuitBreaker;

    public function __construct(
        private readonly HttpClientInterface $httpClient,
        private readonly LoggerInterface $logger,
    ) {
        $this->circuitBreaker = new CircuitBreaker(
            name: 'payment-service',
            failureThreshold: 5,
            successThreshold: 3,
            openDurationSeconds: 30,
            logger: $this->logger,
        );
    }

    public function processPayment(PaymentRequest $request): PaymentResult
    {
        return $this->circuitBreaker->execute(
            operation: fn() => $this->callPaymentApi($request),
            fallback: fn() => $this->handlePaymentUnavailable($request),
        );
    }

    private function callPaymentApi(PaymentRequest $request): PaymentResult
    {
        $response = $this->httpClient->post('/api/payments', [
            'json' => $request->toArray(),
            'timeout' => 5,
        ]);

        if ($response->getStatusCode() >= 500) {
            throw new ServiceUnavailableException('Payment service returned 5xx');
        }

        return PaymentResult::fromResponse($response);
    }

    private function handlePaymentUnavailable(PaymentRequest $request): PaymentResult
    {
        // Fallback: queue payment for later processing
        $this->logger->warning('Payment service unavailable, queuing for retry', [
            'order_id' => $request->orderId,
        ]);

        return PaymentResult::pending($request->orderId);
    }
}

Паттерн 3: Timeout

Ограничивает время ожидания ответа от внешнего сервиса.

<?php

declare(strict_types=1);

/**
 * Timeout pattern: don't wait forever
 *
 * Without timeout: one slow service blocks the entire request
 * With timeout: fail fast and handle the failure gracefully
 */
final class TimeoutWrapper
{
    /**
     * Execute with timeout.
     * Uses pcntl_alarm for process-level timeout (CLI only).
     * For web: use HTTP client timeout settings.
     *
     * @template T
     * @param callable(): T $operation
     * @return T
     */
    public function execute(callable $operation, int $timeoutSeconds): mixed
    {
        $timeoutOccurred = false;

        // Set up alarm signal handler
        $previousHandler = pcntl_signal(SIGALRM, function () use (&$timeoutOccurred) {
            $timeoutOccurred = true;
        });

        pcntl_alarm($timeoutSeconds);

        try {
            $result = $operation();

            if ($timeoutOccurred) {
                throw new TimeoutException("Operation timed out after {$timeoutSeconds}s");
            }

            return $result;
        } finally {
            pcntl_alarm(0); // Cancel alarm
            if ($previousHandler !== null) {
                pcntl_signal(SIGALRM, $previousHandler);
            }
        }
    }
}

// More practical: HTTP client with timeout
final class HttpServiceClient
{
    public function __construct(
        private readonly HttpClientInterface $client,
        private readonly float $connectTimeoutSec = 2.0,
        private readonly float $requestTimeoutSec = 5.0,
    ) {}

    public function get(string $url): array
    {
        $response = $this->client->request('GET', $url, [
            'connect_timeout' => $this->connectTimeoutSec,
            'timeout' => $this->requestTimeoutSec,
        ]);

        return json_decode($response->getBody()->getContents(), true);
    }
}

Паттерн 4: Bulkhead (Переборка)

Изолирует ресурсы между компонентами, чтобы сбой в одном не влиял на другие.

<?php

declare(strict_types=1);

/**
 * Bulkhead pattern: isolate failures
 *
 * Named after ship bulkheads that prevent flooding from spreading.
 * If one compartment floods, others stay dry.
 *
 * In software: separate thread pools, connection pools,
 * or rate limits per dependency.
 */
final class BulkheadManager
{
    /** @var array<string, Semaphore> */
    private array $bulkheads = [];

    /**
     * Create isolated resource pools for different dependencies.
     * Each dependency has its own concurrency limit.
     */
    public function register(string $name, int $maxConcurrency): void
    {
        $this->bulkheads[$name] = new Semaphore($maxConcurrency);
    }

    /**
     * Execute operation within a bulkhead.
     * If the bulkhead is full, reject immediately (fail fast).
     *
     * @template T
     * @param callable(): T $operation
     * @return T
     */
    public function execute(string $bulkheadName, callable $operation): mixed
    {
        $semaphore = $this->bulkheads[$bulkheadName]
            ?? throw new \RuntimeException("Bulkhead '$bulkheadName' not registered");

        if (!$semaphore->tryAcquire()) {
            throw new BulkheadFullException(
                "Bulkhead '$bulkheadName' is full. Max concurrent: {$semaphore->getMax()}"
            );
        }

        try {
            return $operation();
        } finally {
            $semaphore->release();
        }
    }
}

final class Semaphore
{
    private int $current = 0;

    public function __construct(
        private readonly int $max,
    ) {}

    public function tryAcquire(): bool
    {
        if ($this->current >= $this->max) {
            return false;
        }

        $this->current++;
        return true;
    }

    public function release(): void
    {
        if ($this->current > 0) {
            $this->current--;
        }
    }

    public function getMax(): int
    {
        return $this->max;
    }
}

// Usage: Isolate payment and inventory services
// $bulkhead = new BulkheadManager();
// $bulkhead->register('payment-service', maxConcurrency: 10);
// $bulkhead->register('inventory-service', maxConcurrency: 20);
// $bulkhead->register('notification-service', maxConcurrency: 5);
//
// // If payment service is slow and using all 10 slots,
// // inventory service still has its own 20 slots available

Паттерн 5: Fallback (Запасной вариант)

Предоставляет альтернативный ответ, когда основной сервис недоступен.

<?php

declare(strict_types=1);

/**
 * Fallback pattern: graceful degradation
 *
 * Instead of showing an error, provide a degraded but usable response.
 */
final class ProductCatalogService
{
    public function __construct(
        private readonly HttpClientInterface $catalogApi,
        private readonly \Redis $cache,
        private readonly LoggerInterface $logger,
    ) {}

    /**
     * Get product with multiple fallback levels.
     *
     * Level 1: Live API (freshest data)
     * Level 2: Cache (may be slightly stale)
     * Level 3: Default response (minimal information)
     */
    public function getProduct(string $productId): ProductResponse
    {
        // Level 1: Try live API
        try {
            $product = $this->fetchFromApi($productId);
            // Update cache on successful fetch
            $this->cache->setex("product:$productId", 3600, json_encode($product));
            return $product;
        } catch (\Throwable $e) {
            $this->logger->warning('Catalog API unavailable, trying cache', [
                'product_id' => $productId,
                'error' => $e->getMessage(),
            ]);
        }

        // Level 2: Try cache
        $cached = $this->cache->get("product:$productId");
        if ($cached !== false) {
            $this->logger->info('Serving product from cache (fallback)', [
                'product_id' => $productId,
            ]);

            $product = ProductResponse::fromJson($cached);
            $product->markAsStale(); // Let the client know data may be old

            return $product;
        }

        // Level 3: Default response
        $this->logger->error('No data available for product, returning default', [
            'product_id' => $productId,
        ]);

        return ProductResponse::unavailable($productId);
    }

    private function fetchFromApi(string $productId): ProductResponse
    {
        $response = $this->catalogApi->get("/api/products/$productId", [
            'timeout' => 3,
        ]);

        return ProductResponse::fromApiResponse($response);
    }
}

Комбинирование паттернов

В реальных системах паттерны комбинируются.

<?php

declare(strict_types=1);

/**
 * Combining patterns: Retry + Circuit Breaker + Timeout + Fallback
 *
 * Order of wrapping (outside to inside):
 * Fallback -> CircuitBreaker -> Retry -> Timeout -> Actual call
 */
final class ResilientServiceClient
{
    public function __construct(
        private readonly HttpClientInterface $client,
        private readonly CircuitBreaker $circuitBreaker,
        private readonly RetryHandler $retryHandler,
        private readonly \Redis $cache,
        private readonly LoggerInterface $logger,
    ) {}

    public function fetchData(string $endpoint): array
    {
        // Outer: Fallback
        try {
            // Middle: Circuit Breaker
            return $this->circuitBreaker->execute(
                operation: function () use ($endpoint) {
                    // Inner: Retry with timeout
                    return $this->retryHandler->execute(
                        operation: fn() => $this->client->get($endpoint, [
                            'timeout' => 5, // Timeout per attempt
                        ]),
                        retryableExceptions: [
                            TimeoutException::class,
                            ServiceUnavailableException::class,
                        ],
                    );
                },
            );
        } catch (\Throwable $e) {
            // Fallback: return cached or default data
            $this->logger->error('All resilience layers exhausted', [
                'endpoint' => $endpoint,
                'error' => $e->getMessage(),
            ]);

            $cached = $this->cache->get("fallback:$endpoint");
            if ($cached !== false) {
                return json_decode($cached, true);
            }

            return ['error' => 'Service temporarily unavailable', 'data' => []];
        }
    }
}

Мониторинг отказоустойчивости

<?php

declare(strict_types=1);

/**
 * Monitoring resilience patterns
 *
 * What to monitor:
 * - Circuit breaker state changes (alerts on OPEN)
 * - Retry rates (high rate = upstream problem)
 * - Fallback invocations (degraded user experience)
 * - Bulkhead rejection rates (resource contention)
 * - Timeout rates per dependency
 */
final class ResilienceMetrics
{
    public function __construct(
        private readonly MetricsCollector $metrics,
    ) {}

    public function recordCircuitStateChange(string $circuit, string $from, string $to): void
    {
        $this->metrics->increment('circuit_breaker.state_change', [
            'circuit' => $circuit,
            'from' => $from,
            'to' => $to,
        ]);
    }

    public function recordRetry(string $operation, int $attempt, bool $success): void
    {
        $this->metrics->increment('retry.attempt', [
            'operation' => $operation,
            'attempt' => (string) $attempt,
            'success' => $success ? 'true' : 'false',
        ]);
    }

    public function recordFallback(string $operation, string $level): void
    {
        $this->metrics->increment('fallback.invoked', [
            'operation' => $operation,
            'level' => $level,
        ]);
    }
}

Сравнение паттернов

Выводы

Отказоустойчивость -- это не один паттерн, а многоуровневая защита. Начните с таймаутов (самое простое), добавьте retries для transient ошибок, circuit breaker для защиты от каскадных сбоев и fallback для graceful degradation. Каждый слой защищает от определенного типа проблем. Всегда мониторьте срабатывание защитных механизмов -- они являются ранними индикаторами проблем в системе.