Log Shipping — Reliable Log Transport from Services to Central Storage

// Reliable log shipper
class LogShipper {
  constructor(options = {}) {
    this.endpoint = options.endpoint;
    this.buffer = [];
    this.maxBufferSize = options.maxBufferSize || 1000;
    this.flushInterval = options.flushInterval || 5000;
    this.retryDelay = options.retryDelay || 1000;
    this.maxRetries = options.maxRetries || 5;
    this.batchSize = options.batchSize || 100;

    this.startPeriodicFlush();
  }

  async ship(logEntry) {
    this.buffer.push({
      entry: logEntry,
      retries: 0,
      timestamp: Date.now()
    });

    if (this.buffer.length >= this.maxBufferSize) {
      await this.flush();
    }
  }

  async flush() {
    if (this.buffer.length === 0) return;

    const batch = this.buffer.splice(0, this.batchSize);
    const entries = batch.map(b => b.entry);

    try {
      const response = await fetch(this.endpoint, {
        method: 'POST',
        headers: {
          'Content-Type': 'application/json',
          'X-API-Key': process.env.LOG_SHIP_KEY
        },
        body: JSON.stringify({ entries, count: entries.length })
      });

      if (!response.ok) throw new Error(`HTTP ${response.status}`);
    } catch (err) {
      // Re-queue failed entries with backoff
      for (const item of batch) {
        item.retries++;
        if (item.retries < this.maxRetries) {
          setTimeout(() => {
            this.buffer.push(item);
          }, this.retryDelay * Math.pow(2, item.retries));
        } else {
          // Fall back to local file
          fs.appendFileSync('failed-logs.ndjson', JSON.stringify(item.entry) + '\n');
          console.error('Log shipping failed after retries:', err.message);
        }
      }
    }
  }

  startPeriodicFlush() {
    setInterval(() => this.flush(), this.flushInterval);
  }

  // Graceful shutdown
  async shutdown() {
    await this.flush();
    const remaining = this.buffer.length;
    if (remaining > 0) {
      fs.writeFileSync('pending-logs.ndjson',
        this.buffer.map(b => JSON.stringify(b.entry)).join('\n'));
    }
  }
}