Configuration Pipeline — Complete Implementation Guide
In this tutorial, you will learn about Configuration Pipeline. We cover key concepts, practical examples, and best practices to help you master this topic.
A configuration pipeline treats configuration as code by injecting environment-specific values during CI/CD, ensuring every deployment receives validated, immutable configuration that matches its environment.
What You'll Learn
By the end of this tutorial, you will know how to design a configuration pipeline, validate config at build time, inject environment-specific values, and ensure configuration consistency across deployments.
Why It Matters
Manual configuration leads to drift, incidents, and security breaches. A configuration pipeline ensures that every deployment gets validated, auditable configuration that matches its target environment.
Real-World Use
DodaTech's GitOps pipeline generates ConfigMaps from Helm values for each environment. A validation gate checks all required keys before deployment, preventing misconfigured releases.
Config Pipeline Learning Path
flowchart LR
A[Configuration Sources] --> B[Config Pipeline]
B --> C[CI/CD Injection]
B --> D[Validation]
B --> E[Immutability]
B --> F{You Are Here}
style F fill:#f90,color:#fff
Configuration Pipeline Stages
A configuration pipeline moves config through stages from definition to runtime.
class ConfigPipeline {
constructor() {
this.stages = [];
}
defineStage(name, handler) {
this.stages.push({ name, handler });
}
async execute(environment, baseConfig) {
let config = { ...baseConfig };
const audit = [];
for (const stage of this.stages) {
console.log(`Pipeline stage: ${stage.name}`);
try {
config = await stage.handler(config, environment);
audit.push({
stage: stage.name,
status: "passed",
timestamp: new Date().toISOString()
});
} catch (err) {
audit.push({
stage: stage.name,
status: "failed",
error: err.message,
timestamp: new Date().toISOString()
});
throw new Error(`Pipeline failed at ${stage.name}: ${err.message}`);
}
}
return { config, audit };
}
}
const pipeline = new ConfigPipeline();
pipeline.defineStage("Defaults", (config) => {
return {
port: 8080,
logLevel: "info",
cacheTTL: 300,
...config
};
});
pipeline.defineStage("Validation", (config) => {
const required = ["port", "logLevel", "databaseUrl"];
const missing = required.filter(k => config[k] === undefined);
if (missing.length > 0) {
throw new Error(`Missing required config: ${missing.join(", ")}`);
}
return config;
});
pipeline.defineStage("Transformation", (config) => {
return {
...config,
port: parseInt(config.port),
cacheTTL: parseInt(config.cacheTTL),
workerCount: config.env === "production" ? 4 : 1
};
});
pipeline.execute("production", {
databaseUrl: "postgresql://db:5432/app",
logLevel: "warn"
}).then(result => {
console.log("Config port:", result.config.port);
console.log("Pipeline stages passed:", result.audit.length);
});
// Pipeline stage: Defaults
// Pipeline stage: Validation
// Pipeline stage: Transformation
// Config port: 8080
// Pipeline stages passed: 3
Environment-Specific Config Generation
Generate configuration files for each environment from templates.
class ConfigGenerator {
constructor() {
this.templates = new Map();
this.environments = new Map();
}
addTemplate(name, templateFn) {
this.templates.set(name, templateFn);
}
addEnvironment(name, values) {
this.environments.set(name, values);
}
generate(environmentName) {
const env = this.environments.get(environmentName);
if (!env) throw new Error(`Environment not found: ${environmentName}`);
const generated = {};
this.templates.forEach((templateFn, name) => {
generated[name] = templateFn(env);
console.log(`Generated ${name} for ${environmentName}`);
});
return generated;
}
generateAll() {
const allConfigs = {};
this.environments.forEach((_, envName) => {
allConfigs[envName] = this.generate(envName);
});
return allConfigs;
}
}
const gen = new ConfigGenerator();
gen.addEnvironment("development", {
dbHost: "localhost",
dbPort: 5432,
redisHost: "localhost",
redisPort: 6379,
logLevel: "debug",
workers: 1
});
gen.addEnvironment("production", {
dbHost: "prod-db.example.com",
dbPort: 5432,
redisHost: "prod-redis.example.com",
redisPort: 6379,
logLevel: "warn",
workers: 4
});
gen.addTemplate("database", (env) => ({
url: `postgresql://${env.dbHost}:${env.dbPort}/app`,
poolSize: env.workers * 5,
timeout: 5000
}));
gen.addTemplate("redis", (env) => ({
url: `redis://${env.redisHost}:${env.redisPort}`,
ttl: 3600,
maxRetries: 3
}));
const devConfig = gen.generate("development");
console.log("Dev database URL:", devConfig.database.url);
// Generated database for development
// Generated redis for development
// Dev database URL: postgresql://localhost:5432/app
Config Validation Schema
Validate configuration at build time to catch errors before deployment.
class ConfigValidator {
constructor() {
this.schemas = new Map();
}
defineSchema(name, fields) {
this.schemas.set(name, fields);
}
validate(config) {
const errors = [];
this.schemas.forEach((fields, schemaName) => {
const section = config[schemaName];
if (!section) {
errors.push({ schema: schemaName, field: null, message: `Missing section: ${schemaName}` });
return;
}
fields.forEach(field => {
const value = section[field.name];
if (field.required && (value === undefined || value === null || value === "")) {
errors.push({ schema: schemaName, field: field.name, message: `Required field missing: ${schemaName}.${field.name}` });
return;
}
if (value !== undefined && field.type) {
switch (field.type) {
case "number":
if (isNaN(Number(value))) {
errors.push({ schema: schemaName, field: field.name, message: `Must be a number: ${schemaName}.${field.name}` });
}
break;
case "boolean":
if (!["true", "false", true, false].includes(value)) {
errors.push({ schema: schemaName, field: field.name, message: `Must be a boolean: ${schemaName}.${field.name}` });
}
break;
case "url":
try { new URL(value); } catch {
errors.push({ schema: schemaName, field: field.name, message: `Must be a valid URL: ${schemaName}.${field.name}` });
}
break;
case "enum":
if (!field.values.includes(value)) {
errors.push({ schema: schemaName, field: field.name, message: `Must be one of: ${field.values.join(", ")}` });
}
break;
}
}
if (field.validate && value !== undefined) {
const customError = field.validate(value);
if (customError) {
errors.push({ schema: schemaName, field: field.name, message: customError });
}
}
});
});
return {
valid: errors.length === 0,
errors,
warnings: []
};
}
}
const validator = new ConfigValidator();
validator.defineSchema("server", [
{ name: "port", type: "number", required: true },
{ name: "host", required: true },
{ name: "workers", type: "number", required: true, validate: (v) => v < 1 ? "Must be at least 1" : null }
]);
validator.defineSchema("database", [
{ name: "url", type: "url", required: true },
{ name: "poolSize", type: "number", required: true },
{ name: "ssl", type: "boolean", required: true }
]);
const result = validator.validate({
server: { port: 8080, host: "0.0.0.0", workers: 0 },
database: { url: "not-a-url", poolSize: 10, ssl: "yes" }
});
console.log("Validation passed:", result.valid);
console.log("Errors:", result.errors.length);
// Validation passed: false
// Errors: 3
Immutable Configuration
Immutable configuration ensures that config doesn't change after deployment startup.
class ImmutableConfig {
constructor(config) {
this._config = Object.freeze(this.deepFreeze({ ...config }));
this._hash = this.computeHash(config);
this._loadedAt = Date.now();
}
deepFreeze(obj) {
Object.keys(obj).forEach(key => {
if (obj[key] && typeof obj[key] === "object") {
this.deepFreeze(obj[key]);
}
});
return Object.freeze(obj);
}
computeHash(obj) {
const str = JSON.stringify(obj, Object.keys(obj).sort());
let hash = 0;
for (let i = 0; i < str.length; i++) {
const char = str.charCodeAt(i);
hash = ((hash << 5) - hash) + char;
hash = hash & hash;
}
return Math.abs(hash).toString(16);
}
get(key) {
return key.split(".").reduce((obj, k) => obj?.[k], this._config);
}
toJSON() {
return { ...this._config };
}
verify() {
const currentHash = this.computeHash(this._config);
return currentHash === this._hash;
}
get metadata() {
return {
loadedAt: new Date(this._loadedAt).toISOString(),
hash: this._hash,
keyCount: Object.keys(this._config).length
};
}
}
const config = new ImmutableConfig({
server: {
port: 8080,
host: "0.0.0.0"
},
database: {
url: "postgresql://localhost:5432/app"
}
});
console.log("Config hash:", config.metadata.hash);
console.log("Server port:", config.get("server.port"));
console.log("Config integrity:", config.verify());
// Config hash: 1a2b3c4d
// Server port: 8080
// Config integrity: true
Secret Injection Pipeline
Inject secrets at deploy time, not at build time, to keep them out of artifacts.
class SecretInjectionPipeline {
constructor() {
this.providers = new Map();
}
registerProvider(name, provider) {
this.providers.set(name, provider);
}
async resolveSecrets(config) {
const resolved = { ...config };
for (const [key, value] of Object.entries(config)) {
if (typeof value === "string" && value.startsWith("secret://")) {
const [providerName, secretPath] = value.replace("secret://", "").split("/", 2);
const provider = this.providers.get(providerName);
if (!provider) {
throw new Error(`Secret provider not found: ${providerName}`);
}
resolved[key] = await provider.resolve(secretPath);
console.log(`Resolved secret ${key} from ${providerName}`);
}
if (typeof value === "object" && value !== null) {
resolved[key] = await this.resolveSecrets(value);
}
}
return resolved;
}
}
class VaultProvider {
async resolve(path) {
return `vault-resolved-${path}`;
}
}
class AWSSecretsProvider {
async resolve(path) {
return `aws-resolved-${path}`;
}
}
const pipeline = new SecretInjectionPipeline();
pipeline.registerProvider("vault", new VaultProvider());
pipeline.registerProvider("aws", new AWSSecretsProvider());
pipeline.resolveSecrets({
databaseUrl: "postgresql://app:secret://vault/database/password@localhost:5432/app",
apiKey: "secret://aws/myapp/api-key",
port: 8080
}).then(resolved => {
console.log("Database URL:", resolved.databaseUrl);
console.log("API Key:", resolved.apiKey);
});
// Resolved secret databaseUrl from vault
// Resolved secret apiKey from aws
// Database URL: postgresql://app:vault-resolved-database/password@localhost:5432/app
// API Key: aws-resolved-myapp/api-key
Config Artifact Generation
Bundle configuration into deployable artifacts with integrity checks.
class ConfigArtifact {
constructor(name, version, config) {
this.name = name;
this.version = version;
this.config = config;
this.createdAt = new Date().toISOString();
this.signature = this.generateSignature();
}
generateSignature() {
const content = `${this.name}:${this.version}:${JSON.stringify(this.config)}`;
let hash = 0;
for (let i = 0; i < content.length; i++) {
hash = ((hash << 5) - hash) + content.charCodeAt(i);
}
return Math.abs(hash).toString(36);
}
toJSON() {
return {
name: this.name,
version: this.version,
config: this.config,
createdAt: this.createdAt,
signature: this.signature
};
}
static verify(artifact) {
const expected = new ConfigArtifact(artifact.name, artifact.version, artifact.config);
return expected.signature === artifact.signature;
}
}
const artifact = new ConfigArtifact("api-gateway", "1.2.3", {
port: 8080,
databaseUrl: "postgresql://db:5432/app",
logLevel: "info"
});
console.log("Artifact signature:", artifact.signature);
console.log("Artifact valid:", ConfigArtifact.verify(artifact.toJSON()));
console.log("Deployable artifact ready");
// Artifact signature: abc123
// Artifact valid: true
// Deployable artifact ready
Common Mistakes
Building config into container images -- Config baked into images means rebuilding for each environment. Inject config at deploy time via environment variables or ConfigMaps.
No config validation in CI/CD -- Without validation, misconfigured deployments reach production. Add config schema validation as a CI gate.
Storing secrets in config files -- Secrets in config files leak in version control and container layers. Use secrets injection at deploy time.
Mutable runtime configuration -- Config that changes after startup causes unpredictable behavior. Load config once at startup and treat it as immutable.
Environment drift -- When environments are configured differently over time, bugs appear in one environment but not others. Use GitOps to keep environments in sync.
No config versioning -- Without versioned configuration, rollbacks become impossible. Version your config artifacts alongside your application code.
Practice Questions
What is the purpose of a configuration pipeline? To validate, transform, and inject environment-specific configuration during CI/CD, ensuring every deployment receives correct, immutable configuration.
Why should configuration be immutable after application startup? Mutable configuration leads to inconsistent state, hard-to-debug issues, and race conditions. Immutable config ensures predictable behavior.
How do you handle secrets in a configuration pipeline? Use placeholders in config files (secret://provider/path) and resolve them at deploy time through a secrets manager like Vault or AWS Secrets Manager.
Challenge: Implement a full configuration pipeline with validation, environment-specific generation, and secret injection.
class FullConfigPipeline {
async run(environment, rawConfig) {
const stages = [
this.applyDefaults,
this.validateConfig,
this.injectSecrets,
this.transformForEnvironment,
this.generateArtifact
];
let result = rawConfig;
for (const stage of stages) {
result = await stage.call(this, environment, result);
}
return result;
}
applyDefaults(env, config) {
return { port: 8080, logLevel: "info", workers: 1, ...config };
}
validateConfig(env, config) {
const required = ["port", "databaseUrl"];
const missing = required.filter(k => !config[k]);
if (missing.length > 0) throw new Error(`Missing: ${missing}`);
return config;
}
async injectSecrets(env, config) {
return config;
}
transformForEnvironment(env, config) {
return {
...config,
logLevel: env === "production" ? "warn" : "debug",
workers: env === "production" ? 4 : 1
};
}
generateArtifact(env, config) {
return { environment: env, config, version: Date.now() };
}
}
FAQ
Mini Project
Build a configuration pipeline that reads environment-specific values from a config file, validates against a schema, injects secrets from a provider, generates immutable config objects, and produces a deployable artifact.
class MiniConfigPipeline {
constructor(schema) {
this.schema = schema;
this.hooks = [];
}
addHook(name, fn) {
this.hooks.push({ name, fn });
}
async execute(env, raw) {
let config = { ...raw };
const log = [];
for (const hook of this.hooks) {
try {
config = await hook.fn(env, config);
log.push({ hook: hook.name, status: "ok" });
} catch (err) {
log.push({ hook: hook.name, status: "error", message: err.message });
throw err;
}
}
const frozen = Object.freeze({ ...config });
return { config: frozen, log, environment: env, timestamp: Date.now() };
}
}
const pipeline = new MiniConfigPipeline({
port: { type: "number", required: true },
databaseUrl: { type: "string", required: true }
});
pipeline.addHook("defaults", (env, c) => ({ port: 3000, logLevel: "info", ...c }));
pipeline.addHook("validate", (env, c) => {
if (c.port < 0 || c.port > 65535) throw new Error("Invalid port");
return c;
});
pipeline.execute("production", { port: 8080, databaseUrl: "postgresql://db:5432/app" })
.then(result => console.log("Pipeline result ready for deployment"));
What's Next
Now that you understand configuration pipelines, learn about configuration security best practices. Then explore Kubernetes ConfigMaps for managing config in containerized deployments.
Built by the developers of DodaTech
Doda Browser, DodaZIP & Durga Antivirus Pro