Cron Health Checks and Proactive Alerts
In this tutorial, you will learn about Cron Health Checks and Proactive Alerts. We cover key concepts, practical examples, and best practices to help you master this topic.
Implement cron health checks with push-based pings, heartbeat monitoring, failure callbacks, dead man switches, and proactive alerting for missed cron job executions.
What You Learn
You will learn how to set up proactive health checks for Cron Jobs using push-based monitoring services, implement dead man switches for critical jobs, handle late and missed executions, and build self-healing cron jobs.
Why It Matters
Passive monitoring (checking logs) detects failures after they happen. Health checks detect failures proactively by verifying that jobs are running on schedule. A dead man switch alerts you when a job DID NOT run, preventing silent data loss.
Real-World Use
DodaTech's critical cron jobs (database backup, SSL renewal) use Healthchecks.io dead man switches. If the backup job does not ping within 26 hours (daily job + 2 hour grace), an SMS alert is sent to the on-call engineer.
Push-Based Health Check Client
#!/usr/bin/env python3
"""Health check client for cron jobs."""
import requests
import time
import sys
import os
class HealthCheckClient:
def __init__(self, ping_key, base_url="https://hc-ping.com"):
self.ping_url = f"{base_url}/{ping_key}"
self.session = requests.Session()
def start(self):
"""Signal that the job has started."""
try:
self.session.post(f"{self.ping_url}/start", timeout=10)
except requests.RequestException:
pass
def success(self, duration=None):
"""Signal successful completion."""
url = self.ping_url
if duration:
url = f"{self.ping_url}/{duration}"
try:
response = self.session.get(url, timeout=10)
return response.status_code == 200
except requests.RequestException:
return False
def failure(self):
"""Signal job failure."""
try:
response = self.session.post(f"{self.ping_url}/fail", timeout=10)
return response.status_code == 200
except requests.RequestException:
return False
def log(self, message):
"""Send a log line visible in the health check dashboard."""
try:
self.session.post(f"{self.ping_url}/log", data=message, timeout=10)
except requests.RequestException:
pass
def backup_database():
time.sleep(2)
return True
hc = HealthCheckClient("YOUR-PING-KEY")
hc.start()
try:
success = backup_database()
duration = 2.1
if success:
hc.success(duration)
print("Backup succeeded, health check sent")
else:
hc.failure()
sys.exit(1)
except Exception as e:
hc.failure()
hc.log(f"Exception: {e}")
sys.exit(1)
Dead Man Switch Implementation
#!/usr/bin/env python3
"""Dead man switch for critical cron jobs."""
import time
import redis
import requests
import threading
r = redis.Redis()
class DeadManSwitch:
def __init__(self, job_name, expected_interval, grace_period=3600):
self.job_name = job_name
self.expected_interval = expected_interval
self.grace_period = grace_period
self.watch_key = f"deadman:{job_name}"
self.alerts_sent = 0
def ping(self):
"""Called by the cron job to signal it is alive."""
r.setex(self.watch_key, self.expected_interval + self.grace_period, time.time())
def check(self):
"""Check if the job has pinged recently."""
last_ping = r.get(self.watch_key)
if not last_ping:
return {'status': 'dead', 'message': 'No ping ever received'}
elapsed = time.time() - float(last_ping)
if elapsed > self.expected_interval + self.grace_period:
return {
'status': 'dead',
'message': f'Last ping {elapsed:.0f}s ago (grace: {self.grace_period}s)',
}
return {'status': 'alive', 'message': f'Pinged {elapsed:.0f}s ago'}
def start_monitor(self, alert_callback, check_interval=60):
"""Start a background thread that periodically checks the switch."""
def monitor():
while True:
status = self.check()
if status['status'] == 'dead' and self.alerts_sent < 3:
alert_callback(self.job_name, status['message'])
self.alerts_sent += 1
elif status['status'] == 'alive':
self.alerts_sent = 0
time.sleep(check_interval)
thread = threading.Thread(target=monitor, daemon=True)
thread.start()
def send_alert(job_name, message):
print(f"DEAD MAN ALERT: {job_name} - {message}")
dms = DeadManSwitch('daily-backup', expected_interval=86400, grace_period=7200)
dms.start_monitor(send_alert, check_interval=10)
dms.ping()
print(f"Status: {dms.check()['status']}")
time.sleep(15)
print(f"After timeout: {dms.check()['status']}")
Expected output:
DEAD MAN ALERT: daily-backup - No ping ever received
Status: alive
After timeout: dead
Self-Healing Cron Jobs
#!/usr/bin/env python3
"""Self-healing cron job with automatic recovery."""
import time
import subprocess
import os
class SelfHealingCron:
def __init__(self, job_name, command, max_retries=3):
self.job_name = job_name
self.command = command
self.max_retries = max_retries
self.consecutive_failures = 0
def run(self):
for attempt in range(1, self.max_retries + 1):
try:
result = subprocess.run(
self.command,
shell=True,
capture_output=True,
text=True,
timeout=300,
)
if result.returncode == 0:
self.consecutive_failures = 0
self._report_healed()
return {'status': 'success', 'attempt': attempt}
self.consecutive_failures += 1
self._log_failure(attempt, result.stderr)
if attempt < self.max_retries:
self._heal()
time.sleep(30)
except subprocess.TimeoutExpired:
self._log_failure(attempt, "Timeout")
self._kill_stuck_process()
return {'status': 'failed', 'attempts': attempt}
def _heal(self):
actions_taken = []
if not self._check_disk_space():
self._clean_disk()
actions_taken.append('cleaned_disk')
if not self._check_database():
self._restart_database()
actions_taken.append('restarted_db')
if not self._check_network():
self._restart_network()
actions_taken.append('restarted_network')
print(f"Healing actions: {actions_taken}")
def _check_disk_space(self):
stat = os.statvfs('/')
free_percent = (stat.f_frsize * stat.f_bavail) / (stat.f_frsize * stat.f_blocks) * 100
return free_percent > 10
def _clean_disk(self):
subprocess.run('find /tmp -type f -atime +1 -delete', shell=True)
def _check_database(self):
return True
def _restart_database(self):
subprocess.run('systemctl restart postgresql', shell=True)
def _check_network(self):
return True
def _restart_network(self):
subprocess.run('systemctl restart networking', shell=True)
def _kill_stuck_process(self):
subprocess.run(f"pkill -f '{self.command}'", shell=True)
def _log_failure(self, attempt, error):
print(f"Attempt {attempt} failed: {error[:100]}")
def _report_healed(self):
if self.consecutive_failures == 0 and hasattr(self, '_was_failing'):
print("Job recovered after healing actions")
healer = SelfHealingCron('daily-backup', 'echo "Backup OK"')
result = healer.run()
print(f"Result: {result['status']} (attempt {result['attempt']})")
Multi-Channel Alerting
#!/usr/bin/env python3
"""Multi-channel alerting for cron health check failures."""
import smtplib
import requests
import json
import os
class CronAlertManager:
def __init__(self):
self.channels = []
def add_slack(self, webhook_url):
self.channels.append(('slack', webhook_url))
def add_email(self, smtp_server, from_addr, to_addrs):
self.channels.append(('email', (smtp_server, from_addr, to_addrs)))
def add_pagerduty(self, routing_key):
self.channels.append(('pagerduty', routing_key))
def add_webhook(self, url):
self.channels.append(('webhook', url))
def send_alert(self, job_name, message, severity='critical'):
for channel_type, config in self.channels:
try:
if channel_type == 'slack':
self._send_slack(config, job_name, message, severity)
elif channel_type == 'email':
self._send_email(config, job_name, message)
elif channel_type == 'pagerduty':
self._send_pagerduty(config, job_name, message)
elif channel_type == 'webhook':
self._send_webhook(config, job_name, message)
except Exception as e:
print(f"Failed to send via {channel_type}: {e}")
def _send_slack(self, webhook, job, message, severity):
color = 'danger' if severity == 'critical' else 'warning'
payload = {
'attachments': [{
'color': color,
'title': f'Cron Health: {job}',
'text': message,
'fields': [
{'title': 'Severity', 'value': severity, 'short': True},
{'title': 'Host', 'value': os.uname().nodename, 'short': True},
],
}]
}
requests.post(webhook, json=payload, timeout=5)
def _send_email(self, config, job, message):
server, from_addr, to_addrs = config
msg = f"Subject: CRON ALERT: {job}\n\n{message}"
with smtplib.SMTP(server) as smtp:
smtp.sendmail(from_addr, to_addrs, msg)
def _send_pagerduty(self, routing_key, job, message):
payload = {
'routing_key': routing_key,
'event_action': 'trigger',
'payload': {
'summary': f'Cron health: {job}',
'source': os.uname().nodename,
'severity': 'critical',
'custom_details': {'message': message},
},
}
requests.post('https://events.pagerduty.com/v2/enqueue', json=payload, timeout=5)
def _send_webhook(self, url, job, message):
payload = {'job': job, 'message': message, 'host': os.uname().nodename}
requests.post(url, json=payload, timeout=5)
alerts = CronAlertManager()
alerts.add_slack('https://hooks.slack.com/services/...')
alerts.add_pagerduty('YOUR_PD_KEY')
alerts.send_alert('daily-backup', 'Backup did not run within expected window', 'critical')
Common Mistakes
1. No Grace Period
Setting the expected interval exactly equal to the job frequency causes false alerts when the job is a few seconds late. Always add a grace period of 10-30% of the interval.
2. Health Check Without Start Signal
Without a start signal, you cannot distinguish between a job that failed and a job that never started. Always send both start and completion pings.
3. Only Checking Exit Code
A job may exit zero but produce no useful work. Add result validation: check file sizes, row counts, or API responses.
4. No Dead Man Switch for Critical Jobs
Some jobs are too important to fail silently. Use a dead man switch that alerts when a ping is NOT received within the expected window.
5. Alerting Without Escalation
If the on-call engineer does not respond to an alert, it should escalate. Configure escalation policies: notify after 5 minutes, page manager after 15 minutes.
Practice Questions
1. What is a dead man switch in cron monitoring?
A mechanism that alerts when a job does NOT send its expected health check ping. If the ping is missing, something is wrong and an alert fires.
2. Why should you send a start ping in addition to a completion ping?
The start ping detects if the job never started. The completion ping detects if it started but failed. Without start, you cannot distinguish the two cases.
3. What is a grace period in health checks?
Extra time beyond the expected interval to account for normal delays. A daily job with 24-hour interval might have a 2-hour grace period.
4. How do you implement self-healing cron jobs?
Detect common failure causes (disk full, database down, network issue), attempt to fix them automatically (clean disk, restart service), then retry the job.
Challenge
Build a health check system for critical cron jobs that: uses a dead man switch with 10-minute check interval, sends start and completion pings to Healthchecks.io, alerts via Slack and PagerDuty on failure, implements a 30-minute grace period for daily jobs, and self-heals by retrying after restarting failed dependencies.
FAQ
Mini Project: Health Check Integration
#!/usr/bin/env python3
"""Health check integration for multiple cron jobs."""
import time
import os
import json
import requests
import redis
r = redis.Redis()
class HealthCheckIntegrator:
def __init__(self, base_ping_url="https://hc-ping.com"):
self.base_url = base_ping_url
def register_job(self, job_name, ping_key, expected_interval, grace=3600):
config = {
'ping_key': ping_key,
'interval': expected_interval,
'grace': grace,
}
r.hset('healthcheck:jobs', job_name, json.dumps(config))
def ping_start(self, job_name, duration=None):
config = self._get_config(job_name)
if not config:
return False
url = f"{self.base_url}/{config['ping_key']}/start"
try:
requests.get(url, timeout=5)
return True
except requests.RequestException:
return False
def ping_success(self, job_name, duration=None):
config = self._get_config(job_name)
if not config:
return False
url = f"{self.base_url}/{config['ping_key']}"
if duration:
url = f"{url}/{duration}"
try:
requests.get(url, timeout=5)
return True
except requests.RequestException:
return False
def ping_failure(self, job_name):
config = self._get_config(job_name)
if not config:
return False
try:
requests.post(f"{self.base_url}/{config['ping_key']}/fail", timeout=5)
return True
except requests.RequestException:
return False
def _get_config(self, job_name):
data = r.hget('healthcheck:jobs', job_name)
if data:
return json.loads(data)
return None
hci = HealthCheckIntegrator()
hci.register_job('daily-backup', 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee', 86400)
hci.register_job('hourly-cache', 'ffffffff-gggg-hhhh-iiii-jjjjjjjjjjjj', 3600)
hci.ping_start('daily-backup')
time.sleep(1)
hci.ping_success('daily-backup', 1.2)
print("Health check pings sent")
What's Next
Now that you understand cron health checks, build the mini project: backup system to apply everything you have learned about cron patterns, then explore server-sent events for real-time backend communication.
Built by the developers of DodaTech
Doda Browser, DodaZIP & Durga Antivirus Pro