Health Check During Shutdown — Complete Implementation Guide
In this tutorial, you will learn about Health Check During Shutdown. We cover key concepts, practical examples, and best practices to help you master this topic.
Health check management during shutdown ensures load balancers and orchestrators stop routing traffic to a shutting-down instance before it starts draining connections and rejecting requests.
What You'll Learn
By the end of this tutorial, you will know how to mark health as unhealthy before draining, coordinate readiness probe timing, integrate with Kubernetes probes, and prevent traffic from reaching shutting-down instances.
Why It Matters
If the health check still reports healthy during shutdown, load balancers continue routing traffic to the instance. Those requests fail or are rejected, defeating the purpose of graceful shutdown.
Real-World Use
DodaTech's Kubernetes deployments use a preStop hook that sets the readiness probe to unhealthy, waits 5 seconds for the service endpoint to update, then starts the application's shutdown sequence.
Health Check During Shutdown Learning Path
flowchart LR
A[Closing Message Queues] --> B[Health Check During Shutdown]
B --> C[Marking Unhealthy]
B --> D[Probe Timing]
B --> E[Kubernetes]
B --> F{You Are Here}
style F fill:#f90,color:#fff
Marking the Health Endpoint Unhealthy
The first step of shutdown is to make the health check endpoint return unhealthy status.
const http = require("http");
class HealthAwareServer {
constructor() {
this.healthy = true;
this.healthServer = http.createServer((req, res) => {
if (req.url === "/healthz" || req.url === "/readyz") {
if (this.healthy) {
res.writeHead(200, { "Content-Type": "application/json" });
res.end(JSON.stringify({ status: "healthy" }));
} else {
res.writeHead(503, { "Content-Type": "application/json" });
res.end(JSON.stringify({ status: "unhealthy" }));
}
}
});
}
start(port = 8080) {
this.healthServer.listen(port);
console.log(`Health endpoint on :${port}/healthz`);
}
async markUnhealthy() {
this.healthy = false;
console.log("Health endpoint set to unhealthy (503)");
await new Promise(r => setTimeout(r, 2000));
console.log("Drain delay complete, load balancer should stop routing");
}
async shutdown() {
await this.markUnhealthy();
}
}
const healthServer = new HealthAwareServer();
healthServer.start();
console.log("Health-aware server running");
// Health endpoint on :8080/healthz
Readiness vs Liveness During Shutdown
During shutdown, readiness should return unhealthy while liveness should still return healthy (the Process is alive, just leaving).
class ProbeHandler {
constructor() {
this.startupTime = Date.now();
this.shuttingDown = false;
}
handleLiveness() {
return {
status: 200,
body: { status: "alive", uptime: Date.now() - this.startupTime }
};
}
handleReadiness() {
if (this.shuttingDown) {
return {
status: 503,
body: { status: "not-ready", reason: "shutdown in progress" }
};
}
return {
status: 200,
body: { status: "ready" }
};
}
handleStartup() {
if (Date.now() - this.startupTime < 5000) {
return { status: 503, body: { status: "starting" } };
}
return { status: 200, body: { status: "started" } };
}
startShutdown() {
this.shuttingDown = true;
console.log("Readiness -> unhealthy, Liveness -> healthy");
}
}
const probes = new ProbeHandler();
console.log("Readiness:", probes.handleReadiness());
probes.startShutdown();
console.log("Readiness during shutdown:", probes.handleReadiness());
console.log("Liveness during shutdown:", probes.handleLiveness());
// Readiness: { status: 200, body: { status: 'ready' } }
// Readiness during shutdown: { status: 503, body: { status: 'not-ready', reason: 'shutdown in progress' } }
// Liveness during shutdown: { status: 200, body: { status: 'alive', uptime: ... } }
Coordinating Shutdown with Kubernetes Probes
Kubernetes needs time to remove the pod from service endpoints after the readiness probe fails.
class KubernetesShutdownCoordinator {
constructor(options = {}) {
this.probeDrainDelay = options.probeDrainDelay || 5000;
this.shutdownSequence = [];
}
addStep(name, fn) {
this.shutdownSequence.push({ name, fn });
}
async executeShutdown() {
console.log("Phase 1: Mark readiness probe unhealthy");
await this.setReadinessUnhealthy();
console.log(`Phase 2: Wait ${this.probeDrainDelay}ms for endpoint propagation`);
await new Promise(r => setTimeout(r, this.probeDrainDelay));
console.log("Phase 3: Execute shutdown sequence");
for (const { name, fn } of this.shutdownSequence) {
console.log(` Step: ${name}`);
await fn();
}
console.log("Shutdown complete, exiting");
process.exit(0);
}
async setReadinessUnhealthy() {
// In real implementation, update a shared state
console.log(" Readiness endpoint now returns 503");
}
}
const coordinator = new KubernetesShutdownCoordinator({ probeDrainDelay: 3000 });
coordinator.addStep("Close HTTP server", () => Promise.resolve());
coordinator.addStep("Drain database pool", () => Promise.resolve());
coordinator.executeShutdown();
// Phase 1: Mark readiness probe unhealthy
// Readiness endpoint now returns 503
// Phase 2: Wait 3000ms for endpoint propagation
// Phase 3: Execute shutdown sequence
// Step: Close HTTP server
// Step: Drain database pool
// Shutdown complete, exiting
PreStop Hook Integration
Kubernetes preStop hooks run before the SIGTERM signal is sent to the main process.
# deployment.yaml
apiVersion: apps/v1
kind: Deployment
spec:
template:
spec:
containers:
- name: app
lifecycle:
preStop:
exec:
command:
- /bin/sh
- -c
- |
curl -X POST http://localhost:8080/shutdown-prepare
sleep 5
// Application code
let readinessHealthy = true;
app.post("/shutdown-prepare", (req, res) => {
console.log("PreStop hook triggered");
readinessHealthy = false;
res.json({ status: "preparing for shutdown" });
});
// Health check endpoint
app.get("/readyz", (req, res) => {
if (readinessHealthy) {
res.json({ status: "ready" });
} else {
res.status(503).json({ status: "not ready" });
}
});
Common Mistakes
Marking health as unhealthy and immediately exiting -- Kubernetes needs time to remove the pod from service endpoints. Wait at least 2-5 seconds after marking unhealthy before proceeding.
Making the liveness probe fail during shutdown -- If the liveness probe fails, Kubernetes restarts the container. The liveness probe should return healthy even during shutdown.
Not having a separate health check endpoint -- Using the application port for health checks means the health check fails when the server closes. Use a separate port for health endpoints.
Setting readiness and liveness to the same endpoint -- They serve different purposes. Readiness controls traffic routing, liveness controls restart behavior. Differentiate them.
Forgetting to handle the startup probe -- During initial startup, the readiness probe should return unhealthy until the application is fully initialized and ready to accept traffic.
Practice Questions
Why should you wait after marking the readiness probe unhealthy? To give the Kubernetes service controller and load balancer time to remove the pod from endpoint lists. Without the delay, traffic still arrives.
What is the difference between readiness and liveness probes during shutdown? Readiness should become unhealthy to stop traffic. Liveness should remain healthy to prevent the orchestrator from restarting the container during shutdown.
How does a Kubernetes preStop hook help with graceful shutdown? It runs commands before SIGTERM, allowing the pod to mark itself unhealthy and signal upstream services before the application starts draining.
Challenge: Implement a health coordinator that integrates readiness, liveness, and startup probes with a configurable drain delay.
class ProbeCoordinator {
constructor() {
this.state = "starting";
this.drainDelay = 5000;
}
handleStartup() {
return this.state === "starting" ? 503 : 200;
}
handleReadiness() {
if (this.state === "shutting-down") return 503;
if (this.state === "starting") return 503;
return 200;
}
handleLiveness() {
return 200; // Always alive unless crashed
}
start() {
setTimeout(() => {
this.state = "ready";
console.log("Startup complete, ready for traffic");
}, 3000);
}
shutdown() {
this.state = "shutting-down";
console.log("Readiness -> 503");
setTimeout(() => {
console.log("Drain delay complete, starting application shutdown");
}, this.drainDelay);
}
}
const coord = new ProbeCoordinator();
coord.start();
coord.shutdown();
FAQ
Mini Project
Build a health check server on a separate port that coordinates with the main application shutdown, implements readiness/liveness/startup probes, and includes a configurable drain delay.
const http = require("http");
class HealthServer {
constructor(mainApp, options = {}) {
this.mainApp = mainApp;
this.drainDelay = options.drainDelay || 3000;
this.state = "starting";
this.server = http.createServer((req, res) => this.handle(req, res));
}
handle(req, res) {
const body = JSON.stringify(this.getResponse(req.url));
res.writeHead(this.getStatus(req.url), { "Content-Type": "application/json" });
res.end(body);
}
getStatus(url) {
if (url === "/healthz") return 200;
if (url === "/readyz") return this.state === "ready" ? 200 : 503;
if (url === "/startupz") return this.state !== "starting" ? 200 : 503;
return 404;
}
getResponse(url) {
return {
status: this.getStatus(url) === 200 ? "ok" : "not-ok",
state: this.state,
timestamp: new Date().toISOString()
};
}
start(port = 8081) {
this.server.listen(port, () => {
this.state = "ready";
console.log(`Health server on :${port}`);
});
}
async shutdown() {
this.state = "shutting-down";
console.log(`Marked unhealthy, waiting ${this.drainDelay}ms`);
await new Promise(r => setTimeout(r, this.drainDelay));
this.server.close();
console.log("Health server closed");
}
}
const health = new HealthServer(null, { drainDelay: 3000 });
health.start();
What's Next
Now that you understand health checks during shutdown, learn about Kubernetes pod termination lifecycle. Then explore zero-downtime deployment strategies.
Built by the developers of DodaTech
Doda Browser, DodaZIP & Durga Antivirus Pro