Job Retries and Backoff — Complete Guide

DodaTech Updated 2026-06-28 6 min read

In this tutorial, you will learn about Job Retries and Backoff. We cover key concepts, practical examples, and best practices to help you master this topic.

Implement job retries with exponential backoff, configure retry limits, handle transient vs permanent failures, and build robust retry strategies.

What You Learn

You will learn how to implement retry mechanisms, exponential backoff with jitter, distinguish between transient and permanent failures, and configure retry limits.

Why It Matters

Network failures, database timeouts, and service disruptions are inevitable. Without retries, these temporary failures permanently lose jobs. Proper retry strategies ensure jobs eventually succeed without overwhelming failing services.

Real-World Use

DodaTech's Webhook delivery system uses exponential backoff: 10s, 20s, 40s, 80s, 160s, then dead letter. This gives external services time to recover while not overwhelming them with retries.

Basic Retry Mechanism

import time
import random

class RetryHandler:
    def __init__(self, max_retries=3, base_delay=1):
        self.max_retries = max_retries
        self.base_delay = base_delay

    def execute(self, func, *args, **kwargs):
        last_exception = None

        for attempt in range(1, self.max_retries + 1):
            try:
                result = func(*args, **kwargs)
                print(f"Success on attempt {attempt}")
                return result
            except Exception as e:
                last_exception = e
                if attempt < self.max_retries:
                    delay = self.base_delay * (2 ** (attempt - 1))
                    print(f"Attempt {attempt} failed: {e}. Retrying in {delay}s")
                    time.sleep(delay)

        raise last_exception

def flaky_api_call(url):
    if random.random() < 0.6:
        raise ConnectionError("Network timeout")
    return f"Response from {url}"

retry = RetryHandler(max_retries=5, base_delay=1)
try:
    result = retry.execute(flaky_api_call, "https://api.example.com/data")
    print(f"Got: {result}")
except Exception as e:
    print(f"All retries exhausted: {e}")

Exponential Backoff with Jitter

import time
import random

def exponential_backoff(attempt, base=1, max_delay=300):
    delay = base * (2 ** (attempt - 1))
    jitter = random.uniform(0, delay * 0.5)
    return min(delay + jitter, max_delay)

def retry_with_backoff(func, max_retries=5, base_delay=1, max_delay=300):
    for attempt in range(1, max_retries + 1):
        try:
            return func()
        except Exception as e:
            if attempt == max_retries:
                raise
            delay = exponential_backoff(attempt, base_delay, max_delay)
            print(f"Attempt {attempt} failed, retrying in {delay:.1f}s")
            time.sleep(delay)

call_count = 0

def unstable_service():
    global call_count
    call_count += 1
    if call_count < 3:
        raise TimeoutError("Service busy")
    return "Service OK"

result = retry_with_backoff(unstable_service, max_retries=5, base_delay=1)
print(f"Result: {result}")

Expected output:

Attempt 1 failed, retrying in 1.3s
Attempt 2 failed, retrying in 2.7s
Result: Service OK

Transient vs Permanent Failures

import time
import random

class PermanentFailure(Exception):
    pass

def retry_with_classification(func, max_retries=3):
    for attempt in range(1, max_retries + 1):
        try:
            return func()
        except PermanentFailure as e:
            print(f"Permanent failure: {e}. Not retrying.")
            raise
        except Exception as e:
            if attempt == max_retries:
                raise
            delay = 2 ** attempt
            print(f"Transient failure (attempt {attempt}): {e}. Retrying in {delay}s")
            time.sleep(delay)

def process_payment(data):
    rand = random.random()
    if rand < 0.3:
        raise PermanentFailure("Invalid card number")
    if rand < 0.6:
        raise ConnectionError("Gateway timeout")
    return "Payment processed"

try:
    result = retry_with_classification(process_payment, max_retries=3)
    print(result)
except PermanentFailure as e:
    print(f"Payment permanently failed: {e}")
except Exception as e:
    print(f"Payment failed after retries: {e}")

Retry with Redis

import redis
import json
import time

r = redis.Redis()

class RetryQueue:
    def __init__(self, main_queue='jobs', retry_queue='retries'):
        self.main = main_queue
        self.retry_queue = retry_queue

    def enqueue(self, job, priority=5):
        job['retries'] = 0
        r.lpush(self.main, json.dumps(job))

    def retry(self, job, max_retries=3):
        job['retries'] = job.get('retries', 0) + 1
        if job['retries'] <= max_retries:
            delay = 10 * (2 ** (job['retries'] - 1))
            retry_at = time.time() + delay
            job['retry_at'] = retry_at
            r.zadd(self.retry_queue, {json.dumps(job): retry_at})
            print(f"Will retry in {delay}s (attempt {job['retries']}/{max_retries})")
        else:
            r.lpush('dead_letter', json.dumps(job))
            print(f"Dead letter after {max_retries} retries")

    def process_retries(self):
        now = time.time()
        jobs = r.zrangebyscore(self.retry_queue, 0, now)
        if jobs:
            r.zremrangebyscore(self.retry_queue, 0, now)
        for job_data in jobs:
            job = json.loads(job_data)
            r.lpush(self.main, json.dumps(job))
            print(f"Re-enqueued: {job.get('task')}")

rq = RetryQueue()
rq.enqueue({'task': 'process_payment', 'amount': 100})

# Simulate failure
job = {'task': 'process_payment', 'amount': 100, 'retries': 0}
rq.retry(job, max_retries=3)
time.sleep(12)
rq.process_retries()

Expected output:

Will retry in 10s (attempt 1/3)
Re-enqueued: process_payment

Retry in Bull (Node.js)

const Queue = require('bull');

const queue = new Queue('retry_demo', 'redis://127.0.0.1:6379');

queue.add(
  { task: 'process_payment', amount: 100 },
  {
    attempts: 5,
    backoff: {
      type: 'exponential',
      delay: 2000,
    },
  }
);

queue.process(async (job) => {
  console.log(`Attempt ${job.attemptsMade + 1}`);
  if (Math.random() < 0.5) {
    throw new Error('Transient error');
  }
  console.log('Payment processed');
  return { status: 'success' };
});

Retry with Dead Letter

import time
import json

class DeadLetterRetry:
    def __init__(self, max_retries=3):
        self.max_retries = max_retries
        self.dead_letter = []

    def execute(self, func, *args, **kwargs):
        for attempt in range(1, self.max_retries + 1):
            try:
                return func(*args, **kwargs)
            except Exception as e:
                if attempt == self.max_retries:
                    self.dead_letter.append({
                        'func': func.__name__,
                        'args': args,
                        'kwargs': kwargs,
                        'error': str(e),
                        'attempts': attempt,
                    })
                    print(f"Dead letter: {func.__name__} after {attempt} attempts")
                    return None
                time.sleep(2 ** attempt)

Common Mistakes

1. Retrying Permanent Errors

Retrying invalid data or permission errors wastes resources. Classify errors as transient or permanent before deciding to retry.

2. Constant Retry Delays

Retrying every 10 seconds creates thundering herd. Use exponential backoff so the system has time to recover.

3. Unlimited Retries

Without max_retries, retries continue forever, masking permanent failures. Always set a reasonable maximum.

4. No Dead Letter Queue

Jobs that exhaust retries disappear forever. Route them to a dead letter queue for inspection and manual reprocessing.

5. Not Adding Jitter

Without jitter, all retries happen simultaneously, overwhelming already-struggling services.

Practice Questions

1. What is exponential backoff?

Doubling the delay after each retry: 1s, 2s, 4s, 8s, 16s. This reduces load on failing services while eventually succeeding.

2. What is jitter in retry context?

Random variation added to the delay to prevent all clients from retrying simultaneously. Without jitter, retries synchronize and overwhelm services.

3. How do you distinguish transient from permanent errors?

Transient errors: network timeouts, connection resets, 503 Service Unavailable. Permanent: 400 Bad Request, 404 Not Found, validation errors.

4. What is a dead letter queue?

A queue for jobs that have exhausted their retries. Failed jobs are stored for manual inspection, debugging, and eventual reprocessing.

Challenge

Build a retry system for an external API client. Handle: 429 Too Many Requests (retry after Retry-After header, no max retries), 5xx errors (retry 5x with exponential backoff), 4xx client errors (no retry, permanent failure), network timeouts (retry 3x with jitter).

FAQ

What is the default retry delay?

Depends on the system. Celery defaults to 3 minutes. Bull requires explicit configuration. Sidekiq uses exponential backoff starting at a few seconds.

Can I have different retry strategies per job type?

Yes. Each job type can specify its own retry count, delay, and backoff strategy based on failure characteristics.

What happens to retried jobs if the worker crashes?

With acks_late, unacknowledged jobs are redelivered to another worker. The retry count is preserved.

How long should the maximum retry delay be?

5-10 minutes for most systems. Backoff algorithms should cap at this maximum to prevent excessively long delays.

Should retry counts reset after a successful job?

No. The retry count tracks consecutive failures. A success resets the count for the next job execution.

Mini Project: Retry System

import time
import random
import json

class RetrySystem:
    def __init__(self, max_retries=3, base_delay=1, max_delay=300):
        self.max_retries = max_retries
        self.base_delay = base_delay
        self.max_delay = max_delay
        self.dead_letter = []

    def execute(self, func, *args, **kwargs):
        for attempt in range(1, self.max_retries + 1):
            try:
                return func(*args, **kwargs)
            except PermanentError as e:
                self.dead_letter.append({
                    'func': func.__name__,
                    'error': str(e),
                    'type': 'permanent',
                })
                return None
            except TransientError as e:
                if attempt < self.max_retries:
                    delay = min(self.base_delay * (2 ** (attempt - 1)) + random.uniform(0, 1), self.max_delay)
                    print(f"  Retry {attempt}/{self.max_retries} in {delay:.1f}s: {e}")
                    time.sleep(delay)
                else:
                    self.dead_letter.append({
                        'func': func.__name__,
                        'args': args,
                        'error': str(e),
                        'type': 'exhausted',
                    })
                    return None

class PermanentError(Exception):
    pass

class TransientError(Exception):
    pass

retry_sys = RetrySystem(max_retries=3, base_delay=1)

def unreliable_task(data):
    roll = random.random()
    if roll < 0.2:
        raise PermanentError("Invalid data")
    if roll < 0.7:
        raise TransientError("Service timeout")
    return f"Processed: {data}"

for i in range(5):
    result = retry_sys.execute(unreliable_task, f"item_{i}")
    status = "OK" if result else "FAIL"
    print(f"  Item {i}: {status}")
    time.sleep(0.5)

print(f"\nDead letter: {len(retry_sys.dead_letter)} items")

What's Next

Now that you understand retries, explore job failure handling patterns, then learn about progress tracking for long-running jobs.

← Previous Job Priorities in Background Processing Next → Job Failure Handling Patterns — Complete Guide

Built by the developers of DodaTech

Doda Browser, DodaZIP & Durga Antivirus Pro

Home Browse Background Jobs