SRE for Microservices — Distributed Systems Reliability

class ServiceMeshConfig:
    def __init__(self, service_name):
        self.service = service_name
        self.rules = []

    def add_retry_rule(self, max_retries, timeout_ms):
        self.rules.append({
            "type": "retry",
            "max_retries": max_retries,
            "timeout_ms": timeout_ms
        })
        print(f"Added retry rule: {max_retries} retries, {timeout_ms}ms timeout")

    def add_circuit_breaker(self, consecutive_errors, cooldown_sec):
        self.rules.append({
            "type": "circuit_breaker",
            "consecutive_errors": consecutive_errors,
            "cooldown_sec": cooldown_sec
        })
        print(f"Added circuit breaker: {consecutive_errors} errors, {cooldown_sec}s cooldown")

    def add_timeout(self, timeout_ms):
        self.rules.append({
            "type": "timeout",
            "timeout_ms": timeout_ms
        })
        print(f"Added timeout: {timeout_ms}ms")

    def apply(self):
        print(f"\nApplying mesh config to {self.service}:")
        for rule in self.rules:
            print(f"  - {rule['type']}: {rule}")

mesh = ServiceMeshConfig("sync-service")
mesh.add_retry_rule(3, 200)
mesh.add_circuit_breaker(5, 30)
mesh.add_timeout(500)
mesh.apply()

import random
import time
import uuid

class DistributedTrace:
    def __init__(self, trace_id=None):
        self.trace_id = trace_id or str(uuid.uuid4())[:8]
        self.spans = []

    def start_span(self, service, operation):
        span = {
            "trace_id": self.trace_id,
            "span_id": str(uuid.uuid4())[:8],
            "service": service,
            "operation": operation,
            "start": time.time(),
            "end": None,
            "duration_ms": None
        }
        self.spans.append(span)
        print(f"[{self.trace_id}] START {service}/{operation}")
        return span

    def end_span(self, span):
        span["end"] = time.time()
        span["duration_ms"] = (span["end"] - span["start"]) * 1000
        print(f"[{self.trace_id}] END   {span['service']}/{span['operation']} ({span['duration_ms']:.1f}ms)")

    def trace_report(self):
        total_ms = sum(s["duration_ms"] for s in self.spans if s["duration_ms"])
        print(f"\nTrace {self.trace_id} — Total: {total_ms:.1f}ms")
        for s in self.spans:
            if s["duration_ms"]:
                pct = (s["duration_ms"] / total_ms) * 100 if total_ms > 0 else 0
                print(f"  {s['service']:20s} {s['operation']:30s} {s['duration_ms']:8.1f}ms ({pct:.0f}%)")

trace = DistributedTrace()
span1 = trace.start_span("api-gateway", "handle_request")
time.sleep(random.uniform(0.01, 0.05))
span2 = trace.start_span("sync-service", "sync_files")
time.sleep(random.uniform(0.02, 0.08))
span3 = trace.start_span("storage-service", "store_file")
time.sleep(random.uniform(0.05, 0.1))
trace.end_span(span3)
trace.end_span(span2)
trace.end_span(span1)
trace.trace_report()

class GracefulDegradation:
    def __init__(self, critical_deps, non_critical_deps):
        self.critical = critical_deps
        self.non_critical = non_critical_deps

    def handle_request(self, feature, deps_needed):
        available = []
        unavailable = []

        for dep in deps_needed:
            is_healthy = random.random() > 0.3
            if is_healthy:
                available.append(dep)
            else:
                unavailable.append(dep)

        print(f"Feature: {feature}")
        print(f"  Dependencies needed: {deps_needed}")
        print(f"  Available: {available}")
        print(f"  Unavailable: {unavailable}")

        can_degrade = all(d in self.non_critical for d in unavailable)
        if len(available) == len(deps_needed):
            print(f"  Result: FULL SERVICE")
        elif can_degrade:
            print(f"  Result: DEGRADED SERVICE (non-critical deps missing)")
        else:
            print(f"  Result: SERVICE UNAVAILABLE (critical deps missing)")

dg = GracefulDegradation(
    critical_deps=["auth", "database"],
    non_critical_deps=["notifications", "analytics", "recommendations"]
)

dg.handle_request("File Sync", ["auth", "database", "notifications"])
dg.handle_request("Recommendations", ["auth", "recommendations"])

class DependencyGraph:
    def __init__(self):
        self.services = {}
        self.dependencies = {}

    def add_service(self, name, critical=True):
        self.services[name] = {"critical": critical}
        self.dependencies[name] = []

    def add_dependency(self, service, depends_on):
        self.dependencies[service].append(depends_on)

    def impact_analysis(self, failed_service):
        print(f"Impact analysis: {failed_service} failed")
        affected = []
        for svc, deps in self.dependencies.items():
            if failed_service in deps:
                affected.append(svc)
        for svc in affected:
            critical = self.services[svc]["critical"]
            print(f"  {svc} affected ({'CRITICAL' if critical else 'NON-CRITICAL'})")
        if not affected:
            print("  No downstream services affected")

graph = DependencyGraph()
graph.add_service("api-gateway", critical=True)
graph.add_service("sync-service", critical=True)
graph.add_service("notification-service", critical=False)
graph.add_service("storage-service", critical=True)

graph.add_dependency("api-gateway", "sync-service")
graph.add_dependency("sync-service", "storage-service")
graph.add_dependency("sync-service", "notification-service")

graph.impact_analysis("notification-service")
print()
graph.impact_analysis("storage-service")

def validate_slo_chain(upstream_slo, downstream_slo):
    print(f"Upstream SLO (sync): P99 < {upstream_slo}ms")
    print(f"Downstream SLO (storage): P99 < {downstream_slo}ms")
    if downstream_slo > upstream_slo * 0.5:
        print("WARNING: Downstream latency consumes most of the upstream budget")
    elif downstream_slo > upstream_slo * 0.3:
        print("ADVISORY: Downstream latency is significant but manageable")
    else:
        print("OK: Downstream latency leaves reasonable headroom")

validate_slo_chain(200, 80)
validate_slo_chain(200, 150)