Job Cost Optimization — Complete Guide
In this tutorial, you will learn about Job Cost Optimization. We cover key concepts, practical examples, and best practices to help you master this topic.
Optimize background job infrastructure costs with right-sizing, reserved instances, spot instances, queue selection, batching, and auto-scaling strategies.
What You Learn
You will learn how to calculate job processing costs, choose cost-effective infrastructure, optimize worker utilization, and reduce Redis and database costs.
Why It Matters
Background job processing can be expensive: worker instances, Redis clusters, and database connections. Cost optimization reduces cloud bills by 40-60% without sacrificing performance.
Real-World Use
DodaTech reduced job processing costs by 55% by: switching to spot instances for workers, right-sizing Redis from m5.large to m5.xlarge, reducing retention of completed jobs, and batching database writes.
Cost Calculator
import json
class CostCalculator:
INSTANCE_COSTS = {
't3.medium': 0.0416,
't3.large': 0.0832,
'm5.large': 0.096,
'm5.xlarge': 0.192,
'c5.large': 0.085,
'c5.xlarge': 0.17,
'r5.large': 0.126,
}
REDIS_COSTS = {
'cache.t3.micro': 0.019,
'cache.t3.small': 0.041,
'cache.t3.medium': 0.082,
'cache.r5.large': 0.185,
}
def __init__(self):
self.workers = []
self.redis = None
self.storage_gb = 0
def add_worker_group(self, name, instance_type, count, hours_per_day):
hourly = self.INSTANCE_COSTS.get(instance_type, 0.1) * count
monthly = hourly * hours_per_day * 30
self.workers.append({
'name': name,
'instance': instance_type,
'count': count,
'hourly': round(hourly, 3),
'monthly': round(monthly, 0),
})
def set_redis(self, instance_type):
hourly = self.REDIS_COSTS.get(instance_type, 0.1)
self.redis = {
'instance': instance_type,
'hourly': hourly,
'monthly': round(hourly * 24 * 30, 0),
}
def set_storage(self, gb, cost_per_gb=0.10):
self.storage_gb = gb
self.storage_cost = round(gb * cost_per_gb, 0)
def total_monthly(self):
total = 0
for w in self.workers:
total += w['monthly']
if self.redis:
total += self.redis['monthly']
if hasattr(self, 'storage_cost'):
total += self.storage_cost
return round(total, 0)
def report(self):
report = {'workers': self.workers}
if self.redis:
report['redis'] = self.redis
report['total_monthly'] = self.total_monthly()
return json.dumps(report, indent=2)
calc = CostCalculator()
calc.add_worker_group('scan-workers', 't3.medium', 5, 24)
calc.add_worker_group('backup-workers', 't3.large', 2, 4)
calc.set_redis('cache.t3.medium')
calc.set_storage(50)
print(calc.report())
print(f"Total monthly: ${calc.total_monthly()}")
Expected output:
{
"workers": [
{"name": "scan-workers", "instance": "t3.medium", "count": 5, "hourly": 0.208, "monthly": 149.76},
{"name": "backup-workers", "instance": "t3.large", "count": 2, "hourly": 0.166, "monthly": 19.97}
],
"redis": {"instance": "cache.t3.medium", "hourly": 0.082, "monthly": 59.04},
"total_monthly": 228.77
}
Total monthly: $228.77
Optimization Strategy
import json
class CostOptimizer:
def __init__(self):
self.strategies = []
def add_strategy(self, name, current_cost, optimized_cost, effort):
saving = current_cost - optimized_cost
saving_pct = ((current_cost - optimized_cost) / current_cost) * 100
self.strategies.append({
'name': name,
'current': current_cost,
'optimized': optimized_cost,
'saving_monthly': round(saving, 0),
'saving_pct': round(saving_pct, 1),
'effort': effort,
})
def rank_by_savings(self):
return sorted(self.strategies, key=lambda s: s['saving_monthly'], reverse=True)
def rank_by_effort(self):
effort_order = {'low': 1, 'medium': 2, 'high': 3}
return sorted(self.strategies, key=lambda s: effort_order.get(s['effort'], 99))
def total_savings(self):
return round(sum(s['saving_monthly'] for s in self.strategies), 0)
def report(self):
return json.dumps({
'strategies': self.rank_by_savings(),
'total_monthly_savings': self.total_savings(),
}, indent=2)
optimizer = CostOptimizer()
optimizer.add_strategy('Switch to spot instances', 500, 150, 'medium')
optimizer.add_strategy('Right-size Redis', 200, 100, 'low')
optimizer.add_strategy('Reduce completed job retention', 100, 20, 'low')
optimizer.add_strategy('Implement auto-scaling', 400, 250, 'high')
optimizer.add_strategy('Batch database writes', 100, 70, 'medium')
optimizer.add_strategy('Use reserved instances', 500, 350, 'low')
print("Ranked by savings:")
for s in optimizer.rank_by_savings()[:3]:
print(f" {s['name']}: ${s['saving_monthly']}/mo ({s['saving_pct']}%) [{s['effort']}]")
print(f"\nTotal potential savings: ${optimizer.total_savings()}/mo")
Expected output:
Ranked by savings:
Switch to spot instances: $350/mo (70.0%) [medium]
Use reserved instances: $150/mo (30.0%) [low]
Right-size Redis: $100/mo (50.0%) [low]
Total potential savings: $940/mo
Auto-Scaling Cost Model
import json
import math
class AutoScalingCostModel:
def __init__(self, base_instances=2, max_instances=20, hourly_rate=0.10):
self.base = base_instances
self.max = max_instances
self.rate = hourly_rate
def simulate_day(self, hourly_loads):
total_cost = 0
hours_data = []
for hour, load in enumerate(hourly_loads):
needed = max(self.base, min(self.max, math.ceil(load / 50)))
cost = needed * self.rate
total_cost += cost
hours_data.append({'hour': hour, 'load': load, 'instances': needed, 'cost': round(cost, 2)})
return {'hours': hours_data, 'total_daily': round(total_cost, 2), 'total_monthly': round(total_cost * 30, 2)}
def compare_fixed(self, fixed_instances, hourly_loads):
auto_cost = self.simulate_day(hourly_loads)['total_monthly']
fixed_hourly = fixed_instances * self.rate
fixed_monthly = fixed_hourly * 24 * 30
saving = fixed_monthly - auto_cost
return {
'auto_scaling': round(auto_cost, 2),
'fixed': round(fixed_monthly, 2),
'saving': round(saving, 2),
'saving_pct': round((saving / fixed_monthly) * 100, 1) if fixed_monthly > 0 else 0,
}
model = AutoScalingCostModel(base_instances=2, max_instances=20, hourly_rate=0.096)
# Simulate hourly loads
loads = [10, 10, 10, 10, 20, 50, 100, 200, 300, 400, 500, 600,
700, 650, 500, 400, 350, 300, 250, 200, 150, 100, 50, 20]
comparison = model.compare_fixed(10, loads)
print(f"Fixed (10 instances): ${comparison['fixed']}/mo")
print(f"Auto-scaling: ${comparison['auto_scaling']}/mo")
print(f"Saving: ${comparison['saving']}/mo ({comparison['saving_pct']}%)")
Expected output:
Fixed (10 instances): $691.2/mo
Auto-scaling: $.../mo
Saving: $.../mo (...%)
Queue Selection Cost
import json
class QueueCostComparison:
def __init__(self, monthly_jobs_million=10):
self.monthly_jobs = monthly_jobs_million * 1_000_000
def redis_cost(self, instance_type='cache.t3.medium'):
hourly = {'cache.t3.micro': 0.019, 'cache.t3.small': 0.041, 'cache.t3.medium': 0.082}
return round(hourly.get(instance_type, 0.082) * 24 * 30, 0)
def sqs_cost(self):
requests = self.monthly_jobs * 2
cost_per_million = 0.40
return round((requests / 1_000_000) * cost_per_million, 2)
def rabbitmq_cost(self, instance_type='t3.medium'):
hourly = 0.0416
return round(hourly * 24 * 30, 0)
def compare(self):
return {
'redis': {'monthly': self.redis_cost(), 'note': 'Plus data transfer'},
'sqs': {'monthly': self.sqs_cost(), 'note': 'Pay per request'},
'rabbitmq': {'monthly': self.rabbitmq_cost(), 'note': 'Plus EC2 cost'},
'monthly_jobs': self.monthly_jobs,
}
qcc = QueueCostComparison(monthly_jobs_million=5)
compare = qcc.compare()
print(f"5M jobs/month queue costs:")
for q, data in compare.items():
if q != 'monthly_jobs':
print(f" {q}: ${data['monthly']}/mo ({data['note']})")
Expected output:
5M jobs/month queue costs:
redis: $59.04/mo (Plus data transfer)
sqs: $4.0/mo (Pay per request)
rabbitmq: $29.95/mo (Plus EC2 cost)
Common Mistakes
1. Over-Provisioned Workers
Running 20 instances when 5 would suffice. Implement auto-scaling based on queue depth. Rightsize during off-peak hours.
2. Paying for Idle Capacity
Workers running 24/7 for batch jobs that run once daily. Use scheduled scaling or Serverless workers for periodic jobs.
3. Expensive Redis for Simple Queues
Using Redis Cluster for a simple FIFO queue. Start with a single Redis instance. Scale only when needed.
4. Storing Completed Jobs Forever
Completed job data accumulates storage costs. Implement retention policies: 7 days for details, 90 days for metadata.
5. No Cost Visibility
Without cost tracking, spending grows unnoticed. Set budgets and alerts. Review cost reports weekly.
Practice Questions
1. What is the biggest cost driver in job processing?
Worker compute instances. A single m5.xlarge running 24/7 costs $140/month. 10 workers cost $1400/month.
2. How do spot instances reduce costs?
Spot instances are 60-90% cheaper than on-demand. Use them for fault-tolerant workers that can handle interruptions.
3. What is right-sizing?
Matching instance size to actual workload. A t3.medium may be sufficient for workers that were running on m5.xlarge.
4. How does batch processing reduce costs?
Fewer Redis and database operations per job. 10x throughput means you need fewer worker instances.
Challenge
Build a cost optimization plan for a job system: calculate current monthly cost, identify top 3 cost drivers, recommend specific optimizations with estimated savings, and create a Migration timeline.
FAQ
Mini Project: Cost Optimizer
import json
class CostOptimizer:
def __init__(self):
self.items = []
def add(self, name, current, optimized):
self.items.append({'name': name, 'current': current, 'optimized': optimized, 'saving': current - optimized})
def total(self):
return {'current': round(sum(i['current'] for i in self.items), 2),
'optimized': round(sum(i['optimized'] for i in self.items), 2),
'saving': round(sum(i['saving'] for i in self.items), 2),
'pct': round((1 - sum(i['optimized'] for i in self.items) / sum(i['current'] for i in self.items)) * 100, 1)}
opt = CostOptimizer()
opt.add('EC2 Workers', 800, 350)
opt.add('Redis', 150, 80)
opt.add('Storage', 50, 20)
opt.add('Data Transfer', 100, 40)
t = opt.total()
print(f"Current: ${t['current']}/mo -> Optimized: ${t['optimized']}/mo")
print(f"Saving: ${t['saving']}/mo ({t['pct']}%)")
Expected output:
Current: $1100.0/mo -> Optimized: $490.0/mo
Saving: $610.0/mo (55.5%)
What's Next
Now that you understand cost optimization, explore job dashboard for visualization, then review Celery distributed task queue for a complete job processing framework.
Built by the developers of DodaTech
Doda Browser, DodaZIP & Durga Antivirus Pro