gRPC Deadline Propagation — Passing Timeouts Across Service Chains

package main

import (
    "context"
    "time"
    "google.golang.org/grpc"
    "google.golang.org/grpc/codes"
    "google.golang.org/grpc/status"
)

// Service A calls Service B with deadline propagation
func (s *server) AnalyzeThreat(ctx context.Context,
    req *pb.AnalyzeRequest) (*pb.AnalyzeResponse, error) {
    
    // Check if we have time remaining
    deadline, ok := ctx.Deadline()
    if !ok {
        // No deadline set - create one
        var cancel context.CancelFunc
        ctx, cancel = context.WithTimeout(ctx, 5*time.Second)
        defer cancel()
    } else {
        // Check if deadline has passed
        if time.Now().After(deadline) {
            return nil, status.Error(codes.DeadlineExceeded,
                "request already expired")
        }
    }
    
    // Call downstream with remaining deadline
    remaining := time.Until(deadline)
    buffer := 500 * time.Millisecond
    downstreamDeadline := remaining - buffer
    
    if downstreamDeadline < 100*time.Millisecond {
        return nil, status.Error(codes.DeadlineExceeded,
            "insufficient time for downstream call")
    }
    
    downstreamCtx, cancel := context.WithTimeout(
        ctx, downstreamDeadline)
    defer cancel()
    
    resp, err := s.scanner.ScanFile(downstreamCtx, req.FileId)
    if err != nil {
        if status.Code(err) == codes.DeadlineExceeded {
            // Log which service timed out
            log.Warn("scan service timed out",
                "remaining", remaining)
        }
        return nil, err
    }
    
    return resp, nil
}

import grpc
import time
from contextlib import contextmanager

class DeadlineTracker:
    """Track and log deadline propagation."""
    
    def __init__(self, service_name):
        self.service_name = service_name
    
    @contextmanager
    def track_deadline(self, context, downstream_name):
        """Track a downstream call with deadline info."""
        # Get remaining time
        deadline = context.time_remaining()
        if deadline is None:
            print(f"No deadline set for {downstream_name}")
            yield
            return
        
        start = time.time()
        remaining_before = deadline
        
        try:
            yield
        finally:
            elapsed = (time.time() - start) * 1000
            remaining_after = context.time_remaining()
            
            print(f"Call to {downstream_name}:")
            print(f"  Remaining before: {remaining_before:.0f}ms")
            print(f"  Elapsed: {elapsed:.0f}ms")
            print(f"  Remaining after: {remaining_after:.0f}ms")
            
            # Warn if we're using too much of the budget
            if remaining_before > 0:
                usage = elapsed / remaining_before * 100
                if usage > 50:
                    print(f"  WARNING: Used {usage:.0f}% of "
                          f"remaining deadline!")

# Usage in a service
class ThreatService(pb.ThreatServiceServicer):
    def __init__(self):
        self.tracker = DeadlineTracker("threat-service")
    
    def AnalyzeThreat(self, request, context):
        with self.tracker.track_deadline(
            context, "ml-service"):
            # Check time before calling downstream
            if context.time_remaining() < 1000:
                context.abort(
                    grpc.StatusCode.DEADLINE_EXCEEDED,
                    "Not enough time for analysis")
            
            # Call downstream with remaining deadline
            response = self.ml_stub.Analyze(
                request, timeout=context.time_remaining())
            
        return response

const grpc = require('@grpc/grpc-js');

class DeadlineAwareClient {
  constructor(address, options = {}) {
    this.client = new ThreatServiceClient(
      address,
      grpc.credentials.createInsecure(),
      {
        'grpc.timeout_ms': options.defaultTimeout || 5000,
        ...options,
      },
    );
  }
  
  async callWithDeadline(method, request, timeoutMs) {
    const deadline = new Date();
    deadline.setMilliseconds(
      deadline.getMilliseconds() + (timeoutMs || 5000)
    );
    
    return new Promise((resolve, reject) => {
      this.client[method](request, { deadline },
        (error, response) => {
          if (error) {
            if (error.code === grpc.status.DEADLINE_EXCEEDED) {
              console.error(
                `Deadline exceeded for ${method}`);
            }
            reject(error);
          } else {
            resolve(response);
          }
        }
      );
    });
  }
  
  // Chain call with automatic deadline adjustment
  async chainCall(requests) {
    const totalDeadline = Date.now() + 5000;
    const bufferPerHop = 200; // 200ms buffer per hop
    
    for (const { method, request } of requests) {
      const remaining = totalDeadline - Date.now();
      const adjustedTimeout = Math.max(
        remaining - bufferPerHop, 100);
      
      if (adjustedTimeout < 100) {
        throw new Error('Deadline would be exceeded');
      }
      
      console.log(`Calling ${method} with ${adjustedTimeout}ms`);
      
      await this.callWithDeadline(
        method, request, adjustedTimeout);
    }
  }
}

// Test deadline propagation
async function testDeadlinePropagation() {
  const client = new DeadlineAwareClient('localhost:50051');
  
  // Chain of 3 calls with 5s total limit
  const operations = [
    { method: 'ValidateDevice', request: { deviceId: 'dev-001' }},
    { method: 'AnalyzeThreat', request: { threatName: 'Malware' }},
    { method: 'StoreResult', request: { resultId: 'res-001' }},
  ];
  
  try {
    await client.chainCall(operations);
    console.log('All calls completed within deadline');
  } catch (error) {
    if (error.message === 'Deadline would be exceeded') {
      console.error('Not enough time left for remaining calls');
    }
  }
}