Log Cost Optimization — Managing Log Storage and Ingestion Costs

// Cost-optimized log router
class CostOptimizedLogger {
  constructor(options = {}) {
    this.tiers = {
      hot: { retention: '7d', storage: 'ssd', costPerGB: 0.35 },
      warm: { retention: '30d', storage: 'hdd', costPerGB: 0.10 },
      cold: { retention: '365d', storage: 's3', costPerGB: 0.023 }
    };
    this.dailyBudget = options.dailyBudget || 100; // Daily log budget in $/GB
    this.volumeTracking = new Map();
  }

  classifyLog(entry) {
    // Critical errors -> hot storage
    if (entry.level === 'ERROR' || entry.level === 'FATAL') return 'hot';
    if (entry.level === 'WARN' && entry.type === 'security') return 'hot';

    // Business metrics -> warm storage
    if (['audit', 'transaction', 'scan.result'].includes(entry.type)) return 'warm';

    // Debug/trace -> sampled cold storage
    if (entry.level === 'DEBUG' || entry.level === 'TRACE') {
      if (Math.random() > 0.1) return 'drop'; // Sample 10%
      return 'cold';
    }

    // Default info logs -> warm
    return 'warm';
  }

  shouldDrop(entry) {
    const tier = this.classifyLog(entry);
    if (tier === 'drop') return true;

    // Check budget
    const today = new Date().toISOString().slice(0, 10);
    const todayVolume = this.volumeTracking.get(today) || 0;
    const estimatedSize = JSON.stringify(entry).length / (1024 * 1024 * 1024); // GB

    if (todayVolume + estimatedSize > this.dailyBudget && tier !== 'hot') {
      return true; // Drop non-critical logs when over budget
    }

    this.volumeTracking.set(today, todayVolume + estimatedSize);
    return false;
  }
}

// Retention management
async function applyRetentionPolicy(index, policy) {
  await elasticsearch.indices.putSettings({
    index,
    body: {
      index: {
        'translog.retention.size': '512mb',
        'routing.allocation.require.box_type': policy.storage
      }
    }
  });

  await elasticsearch.indices.putLifecyclePolicy({
    name: `${index}_policy`,
    body: {
      phases: {
        hot: { min_age: '0d', actions: { rollover: { max_size: '50gb', max_age: '1d' } } },
        warm: { min_age: '7d', actions: { allocate: { require: { box_type: 'warm' } }, forcemerge: { max_num_segments: 1 } } },
        cold: { min_age: '30d', actions: { freeze: {} } },
        delete: { min_age: '365d', actions: { delete: {} } }
      }
    }
  });
}