Centralized Logging Architecture — Designing Log Infrastructure for Scale

# Log pipeline architecture
# Service -> Filebeat -> Kafka -> Logstash -> Elasticsearch -> Kibana
#                    \-> S3 (cold storage) -> Athena (SQL queries)

# Kafka log topic configuration
log_topic:
  name: logs-scanapp
  partitions: 12
  replication_factor: 3
  retention_ms: 604800000  # 7 days
  config:
    cleanup.policy: delete
    compression.type: snappy

# Logstash pipeline with buffering
input {
  kafka {
    topics: ["logs-scanapp"]
    consumer_threads: 4
    max_poll_records: 500
    fetch_max_wait_ms: 500
  }
}
filter {
  if "_jsonparsefailure" in [tags] {
    drop {}
  }
  date {
    match => ["timestamp", "ISO8601"]
    target => "@timestamp"
  }
}
output {
  elasticsearch {
    hosts => ["${ELASTIC_HOSTS}"]
    index => "logs-scanapp-%{+YYYY.MM.dd}"
    ilm_enabled: true
    ilm_policy: "90-days-retention"
  }
  s3 {
    bucket => "scanapp-logs-archive"
    prefix => "logs/%{+YYYY/MM/dd}"
    rotation_strategy => "size"
    rotation_size => "5 GB"
  }
}