Round #351

Implement an async circuit breaker that tracks consecutive failures

# Async Circuit Breaker Implementation ## Overview A circuit breaker that prevents cascading failures by tracking consecutive failures and transitioning between **Closed**, **Open**, and **Half-Open** states. ``` Closed ──(failures >= threshold)──► Open ──(timeout elapsed)──► Half-Open ▲ │ └──────────────(success)──────────────────────────────────────────┘ Half-Open ──(failure)──► Open ``` ## Implementation ```csharp using System; using System.Threading; using System.Threading.Tasks; namespace CircuitBreaker { /// <summary> /// Represents the current state of the circuit breaker. /// </summary> public enum CircuitState { /// <summary>Normal operation — requests pass through.</summary> Closed, /// <summary>Failure threshold exceeded — requests are blocked.</summary> Open, /// <summary>Testing recovery — one request is allowed through.</summary> HalfOpen } /// <summary> /// Thrown when a request is made while the circuit is open. /// </summary> public sealed class CircuitBreakerOpenException : Exception { public CircuitState State { get; } public int ConsecutiveFailures { get; } public DateTime? OpenedAt { get; } public CircuitBreakerOpenException( int consecutiveFailures, DateTime openedAt, string circuitName) : base($"Circuit breaker '{circuitName}' is OPEN. " + $"Consecutive failures: {consecutiveFailures}. " + $"Opened at: {openedAt:O}.") { State = CircuitState.Open; ConsecutiveFailures = consecutiveFailures; OpenedAt = openedAt; } } /// <summary> /// Snapshot of the circuit breaker's current metrics. /// </summary> public sealed record CircuitBreakerSnapshot( string Name, CircuitState State, int ConsecutiveFailures, int FailureThreshold, DateTime? LastFailureTime, DateTime? OpenedAt, TimeSpan RecoveryTimeout, long TotalSuccesses, long TotalFailures, long TotalRejections); /// <summary> /// Configuration options for the circuit breaker. /// </summary> public sealed class CircuitBreakerOptions { /// <summary> /// Number of consecutive failures before the circuit opens. /// Default: 5 /// </summary> public int FailureThreshold { get; init; } = 5; /// <summary> /// How long the circuit stays open before transitioning to half-open. /// Default: 30 seconds /// </summary> public TimeSpan RecoveryTimeout { get; init; } = TimeSpan.FromSeconds(30); /// <summary> /// Optional predicate to determine if an exception should count as a failure. /// If null, all exceptions count as failures. /// </summary> public Func<Exception, bool>? ShouldHandleException { get; init; } /// <summary> /// Optional callback invoked when the circuit state changes. /// </summary> public Action<CircuitState, CircuitState, CircuitBreakerSnapshot>? OnStateChanged { get; init; } internal void Validate() { if (FailureThreshold < 1) throw new ArgumentOutOfRangeException(nameof(FailureThreshold), "Failure threshold must be at least 1."); if (RecoveryTimeout <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(RecoveryTimeout), "Recovery timeout must be positive."); } } /// <summary> /// Thread-safe async circuit breaker that tracks consecutive failures. /// </summary> public sealed class AsyncCircuitBreaker : IDisposable { // ── State ──────────────────────────────────────────────────────────── private CircuitState _state = CircuitState.Closed; private int _consecutiveFailures; private DateTime? _lastFailureTime; private DateTime? _openedAt; private long _totalSuccesses; private long _totalFailures; private long _totalRejections; // ── Synchronisation ────────────────────────────────────────────────── private readonly SemaphoreSlim _stateLock = new(1, 1); // ── Config ─────────────────────────────────────────────────────────── private readonly CircuitBreakerOptions _options; private readonly ISystemClock _clock; public string Name { get; } public CircuitState State => _state; // Volatile read is fine for monitoring // ── Constructor ────────────────────────────────────────────────────── public AsyncCircuitBreaker( string name, CircuitBreakerOptions? options = null, ISystemClock? clock = null) { if (string.IsNullOrWhiteSpace(name)) throw new ArgumentException("Circuit breaker name cannot be empty.", nameof(name)); Name = name; _options = options ?? new CircuitBreakerOptions(); _options.Validate(); _clock = clock ?? SystemClock.Instance; } // ── Public API ─────────────────────────────────────────────────────── /// <summary> /// Executes an async operation through the circuit breaker. /// </summary> public async Task ExecuteAsync( Func<CancellationToken, Task> operation, CancellationToken cancellationToken = default) { await ExecuteAsync<bool>(async ct => { await operation(ct).ConfigureAwait(false); return true; }, cancellationToken).ConfigureAwait(false); } /// <summary> /// Executes an async operation through the circuit breaker and returns a result. /// </summary> public async Task<T> ExecuteAsync<T>( Func<CancellationToken, Task<T>> operation, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(operation); await EnsureAllowedAsync(cancellationToken).ConfigureAwait(false); try { T result = await operation(cancellationToken).ConfigureAwait(false); await OnSuccessAsync(cancellationToken).ConfigureAwait(false); return result; } catch (Exception ex) when (ShouldCount(ex)) { await OnFailureAsync(cancellationToken).ConfigureAwait(false); throw; } } /// <summary> /// Returns a snapshot of the current circuit breaker metrics. /// </summary> public CircuitBreakerSnapshot GetSnapshot() => new( Name, _state, _consecutiveFailures, _options.FailureThreshold, _lastFailureTime, _openedAt, _options.RecoveryTimeout, Interlocked.Read(ref _totalSuccesses), Interlocked.Read(ref _totalFailures), Interlocked.Read(ref _totalRejections)); /// <summary> /// Manually resets the circuit breaker to the Closed state. /// </summary> public async Task ResetAsync(CancellationToken cancellationToken = default) { await _stateLock.WaitAsync(cancellationToken).ConfigureAwait(false); try { var previous = _state; TransitionTo(CircuitState.Closed); _consecutiveFailures = 0; _openedAt = null; if (previous != CircuitState.Closed) NotifyStateChange(previous, CircuitState.Closed); } finally { _stateLock.Release(); }

# Async Circuit Breaker with Consecutive Failure Tracking ## Implementation ### NuGet Dependencies ```xml <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" /> <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.0" /> <PackageReference Include="Polly" Version="8.4.0" /> <PackageReference Include="Polly.Extensions" Version="8.4.0" /> ``` --- ### `CircuitBreakerState.cs` ```csharp namespace Resilience.CircuitBreaker; public enum CircuitState { Closed, // Normal operation — requests flow through Open, // Failing — requests are rejected immediately HalfOpen // Probing — one trial request is allowed } ``` --- ### `CircuitBreakerMetrics.cs` ```csharp namespace Resilience.CircuitBreaker; public sealed record CircuitBreakerMetrics( CircuitState State, int ConsecutiveFailures, int FailureThreshold, DateTimeOffset? OpenedAt, DateTimeOffset? WillAttemptRetryAt, long TotalSuccesses, long TotalFailures, long TotalRejections); ``` --- ### `CircuitBreakerOptions.cs` ```csharp namespace Resilience.CircuitBreaker; public sealed class CircuitBreakerOptions { /// <summary>Number of consecutive failures before the circuit opens.</summary> public int FailureThreshold { get; init; } = 5; /// <summary>How long the circuit stays open before a half-open probe is allowed.</summary> public TimeSpan BreakDuration { get; init; } = TimeSpan.FromSeconds(30); /// <summary>How long to wait for an individual call before treating it as a failure.</summary> public TimeSpan CallTimeout { get; init; } = TimeSpan.FromSeconds(10); /// <summary>Optional callback invoked when the circuit transitions to Open.</summary> public Func<CircuitBreakerMetrics, Task>? OnCircuitOpened { get; init; } /// <summary>Optional callback invoked when the circuit transitions to Closed.</summary> public Func<CircuitBreakerMetrics, Task>? OnCircuitClosed { get; init; } /// <summary>Optional callback invoked when the circuit transitions to Half-Open.</summary> public Func<CircuitBreakerMetrics, Task>? OnCircuitHalfOpened { get; init; } /// <summary>Predicate that determines whether an exception counts as a failure.</summary> public Func<Exception, bool> ShouldHandleException { get; init; } = _ => true; public void Validate() { if (FailureThreshold < 1) throw new ArgumentOutOfRangeException(nameof(FailureThreshold), "Must be >= 1."); if (BreakDuration <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(BreakDuration), "Must be positive."); if (CallTimeout <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(CallTimeout), "Must be positive."); } } ``` --- ### `CircuitOpenException.cs` ```csharp namespace Resilience.CircuitBreaker; public sealed class CircuitOpenException : Exception { public CircuitBreakerMetrics Metrics { get; } public CircuitOpenException(CircuitBreakerMetrics metrics) : base($"Circuit breaker is OPEN. " + $"{metrics.ConsecutiveFailures} consecutive failures. " + $"Retry allowed after {metrics.WillAttemptRetryAt:O}.") { Metrics = metrics; } } ``` --- ### `AsyncCircuitBreaker.cs` ```csharp using Microsoft.Extensions.Logging; namespace Resilience.CircuitBreaker; /// <summary> /// Thread-safe async circuit breaker that tracks consecutive failures. /// Transitions: Closed → Open (on threshold) → HalfOpen (after break) → Closed/Open. /// </summary> public sealed class AsyncCircuitBreaker : IAsyncDisposable { // ── state ──────────────────────────────────────────────────────────────── private CircuitState _state = CircuitState.Closed; private int _consecutiveFailures; private long _totalSuccesses; private long _totalFailures; private long _totalRejections; private DateTimeOffset? _openedAt; // Held by the single half-open probe attempt; prevents concurrent probes. private int _halfOpenProbeInFlight; // 0 = no probe, 1 = probe in flight private readonly SemaphoreSlim _stateLock = new(1, 1); private readonly CircuitBreakerOptions _options; private readonly ILogger<AsyncCircuitBreaker> _logger; private readonly TimeProvider _timeProvider; // ── ctor ────────────────────────────────────────────────────────────────── public AsyncCircuitBreaker( CircuitBreakerOptions options, ILogger<AsyncCircuitBreaker> logger, TimeProvider? timeProvider = null) { options.Validate(); _options = options; _logger = logger; _timeProvider = timeProvider ?? TimeProvider.System; } // ── public surface ──────────────────────────────────────────────────────── public CircuitBreakerMetrics Metrics => BuildMetrics(); /// <summary> /// Executes <paramref name="action"/> through the circuit breaker. /// Throws <see cref="CircuitOpenException"/> when the circuit is Open. /// </summary> public async Task ExecuteAsync( Func<CancellationToken, Task> action, CancellationToken cancellationToken = default) { await ExecuteAsync<object?>( async ct => { await action(ct); return null; }, cancellationToken); } /// <summary> /// Executes <paramref name="action"/> through the circuit breaker and returns its result. /// Throws <see cref="CircuitOpenException"/> when the circuit is Open. /// </summary> public async Task<T> ExecuteAsync<T>( Func<CancellationToken, Task<T>> action, CancellationToken cancellationToken = default) { await ThrowIfOpenAsync(cancellationToken); using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(_options.CallTimeout); try { T result = await action(cts.Token).ConfigureAwait(false); await RecordSuccessAsync(cancellationToken).ConfigureAwait(false); return result; } catch (OperationCanceledException ex) when (!cancellationToken.IsCancellationRequested) { // The call-timeout fired, not the caller's token. var timeoutEx = new TimeoutException( $"Circuit breaker call timed out after {_options.CallTimeout}.", ex); await RecordFailureAsync(timeoutEx, cancellationToken).ConfigureAwait(false); throw timeoutEx; } catch (Exception ex) when (_options.ShouldHandleException(ex)) { await RecordFailureAsync(ex, cancellationToken).ConfigureAwait(false); throw; } finally { // Release the half-open probe slot regardless of outcome so // RecordSuccess/Failure can transition state cleanly. if (_state == CircuitState.HalfOpen) Interlocked.Exchange(ref _halfOpenProbeInFlight, 0); } } /// <summary>Manually forces the circuit into the Closed state (e.g. for admin