Log Sampling — Smart Log Sampling for High-Volume Systems

// Adaptive log sampler
class LogSampler {
  constructor(options = {}) {
    this.targetRate = options.targetRate || 0.1;  // 10% sample
    this.minRate = options.minRate || 0.01;
    this.maxRate = options.maxRate || 1.0;
    this.errorSampleRate = 1.0;  // Always sample errors
    this.sampleCounters = new Map();
    this.windowSize = options.windowSize || 60000;
  }

  shouldSample(logEntry) {
    // Always log errors
    if (logEntry.level === 'error' || logEntry.level === 'fatal') {
      return true;
    }

    // Always log security events
    if (logEntry.type === 'security' || logEntry.type === 'audit') {
      return true;
    }

    // Always log slow operations
    if (logEntry.duration && logEntry.duration > 5000) {
      return true;
    }

    // Probabilistic sampling for regular logs
    const key = `${logEntry.service}:${logEntry.level}`;
    const count = (this.sampleCounters.get(key) || 0) + 1;
    this.sampleCounters.set(key, count);

    // Adaptive rate based on volume
    const rate = this.calculateAdaptiveRate(key, count);

    return Math.random() < rate;
  }

  calculateAdaptiveRate(key, count) {
    // Reduce sampling rate as volume increases
    if (count > 10000) return Math.max(this.minRate, this.targetRate * 0.1);
    if (count > 1000) return Math.max(this.minRate, this.targetRate * 0.5);
    return this.targetRate;
  }

  resetCounters() {
    this.sampleCounters.clear();
    setTimeout(() => this.resetCounters(), this.windowSize);
  }
}

// Usage
const sampler = new LogSampler({ targetRate: 0.05 });
app.use((req, res, next) => {
  const logEntry = buildLogEntry(req);
  if (sampler.shouldSample(logEntry)) {
    logger.info(logEntry);
  }
  next();
});