Locking and Concurrency Control in Cron
In this tutorial, you will learn about Locking and Concurrency Control in Cron. We cover key concepts, practical examples, and best practices to help you master this topic.
Prevent overlapping cron executions using file locks, flock, PID files, Redis locks, and database-based locking to ensure only one instance of a cron job runs at a time.
What You Learn
You will learn how to prevent overlapping cron job executions using file-based locks with flock, PID files, Redis distributed locks, database advisory locks, and wrapper scripts for concurrency control.
Why It Matters
If a cron job takes longer than its schedule interval, a second instance starts before the first finishes. Two backup processes writing to the same file corrupt data. Two cache warming processes duplicate work. Locking prevents these problems.
Real-World Use
DodaTech's hourly cache warming job sometimes takes 70 minutes during peak load. Without locking, two instances run simultaneously, doubling database load. A Redis lock ensures only one instance runs at a time, queuing the second until the first completes.
Using flock for File Locking
# flock acquires a lock on a file descriptor
# If the lock cannot be acquired, the command fails
# Basic flock usage (exclusive lock, wait if locked)
0 * * * * /usr/bin/flock -w 0 /tmp/cache-warm.lock \
/usr/local/bin/cache-warm.sh
# Wait for lock (with timeout)
0 * * * * /usr/bin/flock -w 300 /tmp/backup.lock \
/usr/local/bin/backup.sh
# Non-blocking: fail immediately if locked
0 * * * * /usr/bin/flock -n /tmp/report.lock \
/usr/local/bin/report.sh || logger -t cron "Report skipped (already running)"
# flock inside a script
#!/bin/bash
# /usr/local/bin/locked-job.sh
LOCK_FILE="/tmp/${0##*/}.lock"
exec 200>"$LOCK_FILE"
if ! flock -n 200; then
echo "Another instance is running, exiting"
exit 1
fi
# Critical section
echo "Running exclusive job at $(date)"
sleep 60
PID File Locking
#!/usr/bin/env python3
"""Cron job with PID file locking."""
import os
import sys
import time
import signal
class PIDLock:
def __init__(self, pid_dir='/tmp/cron-pids'):
self.pid_dir = pid_dir
os.makedirs(pid_dir, exist_ok=True)
def acquire(self, job_name):
pid_file = os.path.join(self.pid_dir, f"{job_name}.pid")
if os.path.exists(pid_file):
with open(pid_file) as f:
try:
old_pid = int(f.read().strip())
except ValueError:
old_pid = None
if old_pid and self._is_running(old_pid):
print(f"Job {job_name} already running (PID {old_pid})")
return False
else:
print(f"Removing stale PID file for {job_name}")
os.remove(pid_file)
with open(pid_file, 'w') as f:
f.write(str(os.getpid()))
return True
def _is_running(self, pid):
"""Check if a process with this PID is still running."""
try:
os.kill(pid, 0)
return True
except OSError:
return False
def release(self, job_name):
pid_file = os.path.join(self.pid_dir, f"{job_name}.pid")
if os.path.exists(pid_file):
os.remove(pid_file)
lock = PIDLock()
job_name = 'hourly-cache-warm'
if not lock.acquire(job_name):
sys.exit(1)
try:
print(f"Job {job_name} starting (PID {os.getpid()})")
time.sleep(5)
print(f"Job {job_name} completed")
finally:
lock.release(job_name)
Redis Distributed Lock
#!/usr/bin/env python3
"""Distributed locking with Redis for cron jobs across multiple servers."""
import redis
import time
import uuid
import sys
class RedisLock:
def __init__(self, redis_client, lock_name, ttl=3600):
self.redis = redis_client
self.lock_key = f"cron_lock:{lock_name}"
self.ttl = ttl
self.lock_value = str(uuid.uuid4())
def acquire(self, blocking=False, timeout=10):
"""Acquire the lock. Returns True if successful."""
if blocking:
start = time.time()
while time.time() - start < timeout:
if self.redis.setnx(self.lock_key, self.lock_value):
self.redis.expire(self.lock_key, self.ttl)
return True
time.sleep(1)
return False
else:
acquired = self.redis.setnx(self.lock_key, self.lock_value)
if acquired:
self.redis.expire(self.lock_key, self.ttl)
return acquired
def release(self):
"""Release the lock only if we own it."""
current = self.redis.get(self.lock_key)
if current and current.decode() == self.lock_value:
self.redis.delete(self.lock_key)
return True
return False
def __enter__(self):
if not self.acquire():
raise RuntimeError(f"Could not acquire lock: {self.lock_key}")
return self
def __exit__(self, *args):
self.release()
r = redis.Redis()
try:
with RedisLock(r, 'daily-backup', ttl=7200):
print("Running daily backup (locked with Redis)")
time.sleep(2)
print("Backup completed, lock released")
except RuntimeError:
print("Backup already running on another server")
sys.exit(1)
Expected output:
Running daily backup (locked with Redis)
Backup completed, lock released
Database Advisory Locks
#!/usr/bin/env python3
"""PostgreSQL advisory lock for cron concurrency control."""
import psycopg2
import hashlib
class PostgresAdvisoryLock:
def __init__(self, conn_string):
self.conn_string = conn_string
self.conn = None
def acquire(self, lock_name, blocking=True):
lock_id = self._name_to_id(lock_name)
self.conn = psycopg2.connect(self.conn_string)
self.conn.autocommit = True
cur = self.conn.cursor()
if blocking:
cur.execute("SELECT pg_advisory_lock(%s)", (lock_id,))
else:
cur.execute("SELECT pg_try_advisory_lock(%s)", (lock_id,))
result = cur.fetchone()[0]
if not result:
self.conn.close()
self.conn = None
return False
return True
def release(self):
if self.conn:
cur = self.conn.cursor()
cur.execute("SELECT pg_advisory_unlock_all()")
self.conn.close()
self.conn = None
def _name_to_id(self, name):
return int(hashlib.md5(name.encode()).hexdigest()[:15], 16)
lock = PostgresAdvisoryLock("dbname=mydb user=app")
if lock.acquire("daily-report"):
try:
print("Generating report with advisory lock")
finally:
lock.release()
Lock Wrapper Script
#!/bin/bash
# /usr/local/bin/cron-lock-wrapper.sh
# Usage: cron-lock-wrapper.sh <lock-name> <command...>
LOCK_NAME="$1"
shift
LOCK_DIR="/var/lock/cron"
LOCK_FILE="${LOCK_DIR}/${LOCK_NAME}.lock"
TIMEOUT="${LOCK_TIMEOUT:-300}"
mkdir -p "$LOCK_DIR"
# Try to acquire lock with timeout
/usr/bin/flock -w "$TIMEOUT" "$LOCK_FILE" -c "$@"
EXIT_CODE=$?
if [ $EXIT_CODE -eq 1 ]; then
echo "[$(date)] Could not acquire lock for ${LOCK_NAME} within ${TIMEOUT}s"
logger -t cron-lock "Lock timeout: ${LOCK_NAME}"
fi
exit $EXIT_CODE
# Usage in crontab:
# 0 * * * * /usr/local/bin/cron-lock-wrapper.sh cache-warm /usr/local/bin/warm-cache.sh
# */30 * * * * /usr/local/bin/cron-lock-wrapper.sh health-check /usr/local/bin/health.sh
Common Mistakes
1. No Locking
Jobs run more frequently than their duration, causing overlapping executions. Always consider locking, even for short jobs.
2. Stale Lock Files
A crashed job leaves the lock file, preventing future executions. Use lock files with timeouts or store PID to detect stale locks.
3. Using the Same Lock for Different Jobs
Backup and cache warming use the same lock file, blocking each other unnecessarily. Use unique lock names per job.
4. Not Handling Lock Acquisition Failure
When a lock is not acquired, the job fails silently. Log lock failures so you know when jobs are being skipped.
5. Lock Without Timeout
Without timeout, a stuck lock blocks all future executions indefinitely. Always set a lock TTL.
Practice Questions
1. What problem does cron locking solve?
Prevents overlapping executions when a job takes longer than its schedule interval. Without locking, two instances of the same job run simultaneously.
2. How does flock work?
flock associates a file descriptor with a lock file. Other processes trying to lock the same file wait or fail based on the flags used.
3. What is a distributed lock?
A lock that works across multiple servers, typically using Redis or a database. It ensures only one server runs the job even when cron runs on every server.
4. How do you handle stale locks?
Store a PID or timestamp in the lock file. On startup, check if the owning Process is still alive. If not, clear the lock and acquire it.
Challenge
Build a distributed locking system for Cron Jobs running on 5 servers. Use Redis with automatic TTL, handle crash recovery (lock is released after TTL), handle lock contention (queue and retry), and provide a status endpoint to check which server holds which lock.
FAQ
Mini Project: Lock Manager
#!/usr/bin/env python3
"""Lock manager for cron jobs with monitoring."""
import os
import time
import json
import fcntl
import sys
from datetime import datetime
class LockManager:
def __init__(self, lock_dir='/var/lock/cron'):
self.lock_dir = lock_dir
os.makedirs(lock_dir, exist_ok=True)
def acquire(self, name, timeout=300, wait=True):
lock_path = os.path.join(self.lock_dir, f"{name}.lock")
fd = os.open(lock_path, os.O_CREAT | os.O_RDWR, 0o644)
try:
if wait:
fcntl.flock(fd, fcntl.LOCK_EX)
else:
try:
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError:
os.close(fd)
return False
os.write(fd, json.dumps({
'pid': os.getpid(),
'acquired_at': datetime.now().isoformat(),
'host': os.uname().nodename,
}).encode())
self.fd = fd
self.lock_path = lock_path
return True
except Exception:
os.close(fd)
return False
def release(self):
if hasattr(self, 'fd'):
fcntl.flock(self.fd, fcntl.LOCK_UN)
os.close(self.fd)
del self.fd
def status(self, name):
lock_path = os.path.join(self.lock_dir, f"{name}.lock")
if not os.path.exists(lock_path):
return {'status': 'free'}
try:
fd = os.open(lock_path, os.O_RDONLY)
try:
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
fcntl.flock(fd, fcntl.LOCK_UN)
os.close(fd)
return {'status': 'stale', 'detail': 'Lock file exists but no holder'}
except IOError:
data = os.read(fd, 4096)
os.close(fd)
return json.loads(data)
except FileNotFoundError:
return {'status': 'free'}
lm = LockManager()
status = lm.status('daily-backup')
print(f"Lock status: {status}")
What's Next
Now that you understand cron locking, explore distributed cron scheduling for running cron in multi-server environments, then learn about cron in Docker.
Built by the developers of DodaTech
Doda Browser, DodaZIP & Durga Antivirus Pro