Fix Gold API Downtime in Production - Build Resilient Fallback Systems

// Learned this after hammering Gold API during maintenance and making it worse
const axios = require('axios');

const GOLD_API_BASE = 'https://api.gold.io';
const MAX_RETRIES = 3;
const INITIAL_DELAY = 1000; // 1 second

async function callGoldAPIWithRetry(endpoint, options = {}) {
  let lastError;
  
  for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
    try {
      const response = await axios.get(
        `${GOLD_API_BASE}${endpoint}`,
        {
          timeout: 8000, // 8 second timeout, not 30
          ...options
        }
      );
      
      // Success on first try - return immediately
      return response.data;
      
    } catch (error) {
      lastError = error;
      
      // Watch out: Don't retry on client errors (401, 403, 404)
      if (error.response?.status >= 400 && error.response?.status < 500) {
        throw error; // Fail fast for bad requests
      }
      
      // Only retry for server errors or network issues
      if (attempt < MAX_RETRIES) {
        // Exponential backoff: 1s, 2s, 4s
        const delay = INITIAL_DELAY * Math.pow(2, attempt);
        console.log(`Retry attempt ${attempt + 1}/${MAX_RETRIES} in ${delay}ms`);
        
        // Add jitter to prevent thundering herd
        const jitter = Math.random() * 1000;
        await new Promise(resolve => setTimeout(resolve, delay + jitter));
      }
    }
  }
  
  // All retries exhausted
  throw lastError;
}

module.exports = { callGoldAPIWithRetry };

// Personal note: Caching saved us 23 minutes during the 2 AM downtime
const redis = require('redis');

const redisClient = redis.createClient({
  host: 'localhost',
  port: 6379
});

const CACHE_TTL = 300; // 5 minutes in seconds

async function callGoldAPIWithCache(endpoint, options = {}) {
  const cacheKey = `gold_api:${endpoint}:${JSON.stringify(options)}`;
  
  // Check cache first
  try {
    const cached = await redisClient.get(cacheKey);
    if (cached) {
      console.log(`Cache hit for ${endpoint}`);
      return JSON.parse(cached);
    }
  } catch (cacheError) {
    // If Redis fails, continue to API call
    console.warn('Cache miss (Redis unavailable):', cacheError.message);
  }
  
  try {
    // Try the API with retry logic
    const data = await callGoldAPIWithRetry(endpoint, options);
    
    // Cache the successful response
    try {
      await redisClient.setEx(cacheKey, CACHE_TTL, JSON.stringify(data));
    } catch (cacheError) {
      // Redis failure doesn't break the API call
      console.warn('Failed to cache response:', cacheError.message);
    }
    
    return data;
    
  } catch (error) {
    // Fallback: Return stale cache if available
    try {
      const stale = await redisClient.get(`${cacheKey}:stale`);
      if (stale) {
        console.warn(`API failed, returning stale cache for ${endpoint}`);
        return JSON.parse(stale);
      }
    } catch (err) {
      // Stale cache unavailable
    }
    
    throw error; // No cache available, throw original error
  }
}

module.exports = { callGoldAPIWithCache };

// Watch out: Without this, one failing endpoint can cascade failures across your entire system
class CircuitBreaker {
  constructor(endpoint, failureThreshold = 5, resetTimeout = 60000) {
    this.endpoint = endpoint;
    this.failureThreshold = failureThreshold;
    this.resetTimeout = resetTimeout;
    this.failureCount = 0;
    this.state = 'CLOSED'; // CLOSED = working, OPEN = broken, HALF_OPEN = testing
    this.lastFailureTime = null;
  }
  
  async execute(fn) {
    // If circuit is OPEN and timeout hasn't passed, fail immediately
    if (this.state === 'OPEN') {
      const timeSinceFailure = Date.now() - this.lastFailureTime;
      
      if (timeSinceFailure < this.resetTimeout) {
        throw new Error(
          `Circuit OPEN for ${this.endpoint}. Failing fast to prevent cascading failures.`
        );
      }
      
      // Timeout passed, try HALF_OPEN
      this.state = 'HALF_OPEN';
      console.log(`Circuit HALF_OPEN for ${this.endpoint}. Testing recovery...`);
    }
    
    try {
      const result = await fn();
      
      // Success - reset the circuit
      if (this.state === 'HALF_OPEN' || this.state === 'CLOSED') {
        this.failureCount = 0;
        this.state = 'CLOSED';
      }
      
      return result;
      
    } catch (error) {
      this.failureCount++;
      this.lastFailureTime = Date.now();
      
      console.error(
        `Failure ${this.failureCount}/${this.failureThreshold} for ${this.endpoint}`
      );
      
      if (this.failureCount >= this.failureThreshold) {
        this.state = 'OPEN';
        console.error(`Circuit OPEN for ${this.endpoint}. Stopping requests.`);
      }
      
      throw error;
    }
  }
}

// Usage
const breaker = new CircuitBreaker('/v1/spot-price', 5, 60000);

async function getGoldPrice() {
  return breaker.execute(async () => {
    return callGoldAPIWithCache('/v1/spot-price');
  });
}

module.exports = { CircuitBreaker };