Resilience Patterns

Resilience patterns help systems handle failures gracefully.

Circuit Breaker Pattern

enum CircuitState {
  CLOSED = 'CLOSED',     // Normal operation
  OPEN = 'OPEN',         // Failing, reject requests
  HALF_OPEN = 'HALF_OPEN' // Testing if service recovered
}

class CircuitBreaker {
  private state: CircuitState = CircuitState.CLOSED;
  private failureCount: number = 0;
  private successCount: number = 0;
  private lastFailureTime: number = 0;

  private readonly failureThreshold: number;
  private readonly successThreshold: number;
  private readonly timeout: number; // ms to wait before trying again

  constructor(options: {
    failureThreshold?: number;
    successThreshold?: number;
    timeout?: number;
  } = {}) {
    this.failureThreshold = options.failureThreshold || 5;
    this.successThreshold = options.successThreshold || 3;
    this.timeout = options.timeout || 30000;
  }

  async execute(fn: () => Promise): Promise {
    if (this.state === CircuitState.OPEN) {
      if (Date.now() - this.lastFailureTime > this.timeout) {
        this.state = CircuitState.HALF_OPEN;
        this.successCount = 0;
      } else {
        throw new Error('Circuit breaker is OPEN');
      }
    }

    try {
      const result = await fn();
      this.onSuccess();
      return result;
    } catch (error) {
      this.onFailure();
      throw error;
    }
  }

  private onSuccess(): void {
    this.failureCount = 0;

    if (this.state === CircuitState.HALF_OPEN) {
      this.successCount++;
      if (this.successCount >= this.successThreshold) {
        this.state = CircuitState.CLOSED;
        console.log('Circuit breaker CLOSED');
      }
    }
  }

  private onFailure(): void {
    this.failureCount++;
    this.lastFailureTime = Date.now();

    if (this.state === CircuitState.HALF_OPEN) {
      this.state = CircuitState.OPEN;
      console.log('Circuit breaker OPEN (failed in half-open)');
    } else if (this.failureCount >= this.failureThreshold) {
      this.state = CircuitState.OPEN;
      console.log('Circuit breaker OPEN (threshold reached)');
    }
  }

  getState(): CircuitState {
    return this.state;
  }
}

// Usage
const breaker = new CircuitBreaker({ failureThreshold: 3, timeout: 10000 });

async function callExternalService() {
  return breaker.execute(async () => {
    const response = await fetch('https://api.example.com/data');
    if (!response.ok) throw new Error('Service error');
    return response.json();
  });
}

Retry Pattern with Exponential Backoff

interface RetryOptions {
  maxAttempts: number;
  baseDelay: number;      // ms
  maxDelay: number;       // ms
  exponentialBase?: number;
  jitter?: boolean;
}

class RetryHandler {
  private options: Required;

  constructor(options: RetryOptions) {
    this.options = {
      ...options,
      exponentialBase: options.exponentialBase || 2,
      jitter: options.jitter ?? true
    };
  }

  async execute(
    fn: () => Promise,
    shouldRetry: (error: Error) => boolean = () => true
  ): Promise {
    let lastError: Error;

    for (let attempt = 1; attempt <= this.options.maxAttempts; attempt++) {
      try {
        return await fn();
      } catch (error) {
        lastError = error as Error;

        if (attempt === this.options.maxAttempts || !shouldRetry(lastError)) {
          throw lastError;
        }

        const delay = this.calculateDelay(attempt);
        console.log(`Attempt ${attempt} failed, retrying in ${delay}ms...`);
        await this.sleep(delay);
      }
    }

    throw lastError!;
  }

  private calculateDelay(attempt: number): number {
    let delay = this.options.baseDelay *
      Math.pow(this.options.exponentialBase, attempt - 1);

    delay = Math.min(delay, this.options.maxDelay);

    if (this.options.jitter) {
      // Add random jitter to prevent thundering herd
      delay = delay * (0.5 + Math.random());
    }

    return Math.floor(delay);
  }

  private sleep(ms: number): Promise {
    return new Promise(resolve => setTimeout(resolve, ms));
  }
}

// Usage
const retry = new RetryHandler({
  maxAttempts: 5,
  baseDelay: 1000,
  maxDelay: 30000
});

const data = await retry.execute(
  () => fetchFromAPI(),
  (error) => error.message.includes('timeout') // Only retry timeouts
);

Bulkhead Pattern

class Bulkhead {
  private semaphore: number;
  private queue: Array<() => void> = [];
  private readonly maxConcurrent: number;
  private readonly maxQueue: number;

  constructor(maxConcurrent: number, maxQueue: number = 100) {
    this.maxConcurrent = maxConcurrent;
    this.maxQueue = maxQueue;
    this.semaphore = maxConcurrent;
  }

  async execute(fn: () => Promise): Promise {
    await this.acquire();

    try {
      return await fn();
    } finally {
      this.release();
    }
  }

  private async acquire(): Promise {
    if (this.semaphore > 0) {
      this.semaphore--;
      return;
    }

    if (this.queue.length >= this.maxQueue) {
      throw new Error('Bulkhead queue full - request rejected');
    }

    return new Promise((resolve) => {
      this.queue.push(resolve);
    });
  }

  private release(): void {
    const next = this.queue.shift();
    if (next) {
      next();
    } else {
      this.semaphore++;
    }
  }
}

// Isolate different services
const paymentBulkhead = new Bulkhead(10);
const inventoryBulkhead = new Bulkhead(20);
const notificationBulkhead = new Bulkhead(5);

// Payment service failure won't affect inventory
await paymentBulkhead.execute(() => processPayment(order));
await inventoryBulkhead.execute(() => reserveInventory(order));