Common Background Job Use Cases
In this tutorial, you will learn about Common Background Job Use Cases. We cover key concepts, practical examples, and best practices to help you master this topic.
Explore real-world background job use cases including email delivery, report generation, image processing, Webhook dispatch, and data import/export.
What You Learn
You will learn the most common background job use cases with concrete code examples, understand why each benefits from async processing, and see implementation patterns.
Why It Matters
Knowing common use cases helps you recognize when to use Background Jobs in your own applications. Each use case has specific requirements for reliability, retry policies, and monitoring.
Real-World Use
DodaTech uses background jobs for five core operations: email notifications (Durga Antivirus Pro signups), image processing (Doda Browser thumbnails), report generation (weekly analytics), webhook delivery (scan results), and data import (malware signature updates).
Email Delivery
import time
import json
import redis
import smtplib
from email.mime.text import MIMEText
r = redis.Redis()
def enqueue_email(to, subject, body):
email_job = {
'to': to,
'subject': subject,
'body': body,
'priority': 'normal',
}
r.lpush('email_queue', json.dumps(email_job))
print(f"Email queued for {to}")
def process_email(job):
msg = MIMEText(job['body'])
msg['Subject'] = job['subject']
msg['To'] = job['to']
msg['From'] = 'noreply@dodatech.com'
print(f"Sending email to {job['to']}: {job['subject']}")
time.sleep(1)
print(f"Email sent to {job['to']}")
return True
# Enqueue happens in request handler
enqueue_email('user@example.com', 'Welcome', 'Thank you for joining')
enqueue_email('admin@dodatech.com', 'Alert', 'New user registered')
Expected output:
Email queued for user@example.com
Email queued for admin@dodatech.com
Image Processing
import time
import json
import redis
r = redis.Redis()
def enqueue_image_processing(image_path, operations):
job = {
'path': image_path,
'operations': operations, # ['resize', 'thumbnail', 'watermark']
}
r.lpush('image_queue', json.dumps(job))
print(f"Image processing queued: {image_path}")
def process_image(job):
path = job['path']
ops = job['operations']
print(f"Processing image: {path}")
for op in ops:
print(f" Running {op}...")
time.sleep(1)
print(f" {op} complete")
print(f"Image processing done: {path}")
return {'path': path, 'status': 'done'}
# Enqueue
enqueue_image_processing('/uploads/photo.jpg', ['resize', 'thumbnail', 'watermark'])
enqueue_image_processing('/uploads/banner.png', ['resize'])
Expected output:
Image processing queued: /uploads/photo.jpg
Image processing queued: /uploads/banner.png
Report Generation
import time
import json
import redis
r = redis.Redis()
def enqueue_report(report_type, params):
job = {
'type': report_type,
'params': params,
'created_at': time.time(),
}
r.lpush('report_queue', json.dumps(job))
print(f"Report queued: {report_type}")
def generate_report(job):
print(f"Generating {job['type']} report...")
print(f" Parameters: {job['params']}")
if job['type'] == 'daily':
time.sleep(5)
elif job['type'] == 'weekly':
time.sleep(15)
elif job['type'] == 'monthly':
time.sleep(30)
report_url = f"https://reports.dodatech.com/{job['type']}_{int(time.time())}.pdf"
print(f" Report generated: {report_url}")
return {'url': report_url, 'type': job['type']}
# Enqueue
enqueue_report('daily', {'date': '2026-06-28'})
enqueue_report('weekly', {'start': '2026-06-21', 'end': '2026-06-28'})
Expected output:
Report queued: daily
Report queued: weekly
Webhook Dispatch
import time
import json
import redis
import requests
r = redis.Redis()
def enqueue_webhook(url, payload, event_type):
job = {
'url': url,
'payload': payload,
'event': event_type,
'max_retries': 3,
}
r.lpush('webhook_queue', json.dumps(job))
print(f"Webhook queued: {event_type} -> {url}")
def deliver_webhook(job):
print(f"Delivering webhook: {job['event']} to {job['url']}")
for attempt in range(1, job['max_retries'] + 1):
try:
response = requests.post(
job['url'],
json=job['payload'],
timeout=10,
headers={'X-Event-Type': job['event']}
)
if response.status_code < 300:
print(f" Delivered (attempt {attempt})")
return {'status': 'delivered', 'attempts': attempt}
else:
print(f" HTTP {response.status_code} (attempt {attempt})")
except requests.RequestException as e:
print(f" Failed (attempt {attempt}): {e}")
if attempt < job['max_retries']:
time.sleep(2 ** attempt)
print(f" Failed after {job['max_retries']} attempts")
return {'status': 'failed', 'attempts': job['max_retries']}
# Enqueue
enqueue_webhook(
'https://hooks.example.com/scan-complete',
{'file': 'report.pdf', 'threats': 0},
'scan.completed'
)
Expected output:
Webhook queued: scan.completed -> https://hooks.example.com/scan-complete
Data Import/Export
import time
import json
import redis
import csv
import io
r = redis.Redis()
def enqueue_data_import(file_path, format_type):
job = {
'file': file_path,
'format': format_type,
'mode': 'import',
}
r.lpush('data_queue', json.dumps(job))
print(f"Data import queued: {file_path}")
def enqueue_data_export(query, format_type):
job = {
'query': query,
'format': format_type,
'mode': 'export',
}
r.lpush('data_queue', json.dumps(job))
print(f"Data export queued: {query}")
def process_data_job(job):
if job['mode'] == 'import':
print(f"Importing {job['file']} as {job['format']}")
time.sleep(3)
print(f" Imported 5000 records")
return {'records': 5000, 'status': 'imported'}
else:
print(f"Exporting {job['query']} as {job['format']}")
time.sleep(4)
print(f" Exported 10000 records")
return {'records': 10000, 'status': 'exported'}
# Enqueue
enqueue_data_import('/uploads/users.csv', 'csv')
enqueue_data_export('SELECT * FROM orders WHERE date > NOW() - 30', 'xlsx')
Expected output:
Data import queued: /uploads/users.csv
Data export queued: SELECT * FROM orders WHERE date > NOW() - 30
Common Mistakes
1. Treating All Emails as Background Jobs
Transactional emails (password reset) need fast delivery. Marketing emails can batch. Use different queues with different priorities.
2. Processing Images Without Time Limits
Image processing can hang on corrupted files. Always set time limits. A 10MB image should not take more than 30 seconds.
3. Generating Reports Synchronously for Admin UI
Admin users also expect fast responses. Move report generation to background, show a "Report will be emailed" message.
4. Not Handling Webhook Payload Size Limits
Large webhook payloads fail. Keep payloads under 1MB. For large data, include a reference URL instead of the full data.
5. Blocking Import Jobs on Validation
Validate data format and schema synchronously before queuing the import. A corrupt file should be rejected immediately, not discovered 5 minutes later.
Practice Questions
1. What is the most common background job use case?
Email delivery. SMTP connections are slow (1-5 seconds), and users do not need the email to be sent before the HTTP response returns.
2. Why should image processing be a background job?
Image processing is CPU-intensive and can take 0.5-10 seconds per image. Doing it synchronously blocks the request thread.
3. How do webhook deliveries typically handle failures?
With retry logic: exponential backoff (10s, 20s, 40s), configurable max retries, and a dead letter queue for permanently failed deliveries.
4. What is the risk of background job processing for data import?
The user gets an immediate success response but the import may fail minutes later. Implement status tracking and notification on completion.
Challenge
Design a background job system for a document management platform. Use cases: PDF generation (5-30s), OCR text extraction (10-60s), email notifications (1-3s), backup creation (60-600s), and file format conversion (5-20s). Define queues, worker counts, retry policies, and progress tracking for each.
FAQ
Mini Project: Multi-Use-Case Job System
import redis
import json
import time
import threading
r = redis.Redis()
class JobRouter:
def __init__(self):
self.handlers = {}
self.workers = {}
def register(self, queue, job_type):
def decorator(func):
if queue not in self.handlers:
self.handlers[queue] = {}
self.handlers[queue][job_type] = func
return func
return decorator
def enqueue(self, queue, job_type, data):
job = {'type': job_type, 'data': data, 'time': time.time()}
r.lpush(queue, json.dumps(job))
print(f"[{queue}] Queued: {job_type}")
def start_worker(self, queue, concurrency=2):
def worker_loop():
while True:
job_data = r.brpop(queue, timeout=5)
if job_data:
_, data = job_data
job = json.loads(data)
handler = self.handlers.get(queue, {}).get(job['type'])
if handler:
handler(job['data'])
for i in range(concurrency):
t = threading.Thread(target=worker_loop, daemon=True)
t.start()
self.workers[f"{queue}_{i}"] = t
router = JobRouter()
@router.register('email', 'welcome')
def send_welcome(data):
time.sleep(0.5)
print(f" Welcome email to {data['email']}")
@router.register('email', 'alert')
def send_alert(data):
time.sleep(0.3)
print(f" Alert to {data['email']}: {data['message']}")
@router.register('images', 'resize')
def resize_image(data):
time.sleep(1)
print(f" Resized {data['path']} to {data['width']}x{data['height']}")
@router.register('reports', 'daily')
def daily_report(data):
time.sleep(2)
print(f" Daily report generated for {data['date']}")
router.start_worker('email', 2)
router.start_worker('images', 1)
router.start_worker('reports', 1)
router.enqueue('email', 'welcome', {'email': 'user@example.com'})
router.enqueue('images', 'resize', {'path': 'photo.jpg', 'width': 800, 'height': 600})
router.enqueue('email', 'alert', {'email': 'admin@dodatech.com', 'message': 'High CPU'})
router.enqueue('reports', 'daily', {'date': '2026-06-28'})
time.sleep(4)
Expected output:
[email] Queued: welcome
[images] Queued: resize
[email] Queued: alert
[reports] Queued: daily
Welcome email to user@example.com
Resized photo.jpg to 800x600
Alert to admin@dodatech.com: High CPU
Daily report generated for 2026-06-28
What's Next
Now that you understand common use cases, explore job queue concepts in detail, then learn about worker processes for executing jobs.
Built by the developers of DodaTech
Doda Browser, DodaZIP & Durga Antivirus Pro