File Validation — Complete Guide
In this tutorial, you will learn about File Validation. We cover key concepts, practical examples, and best practices to help you master this topic.
File validation goes beyond type checks to verify file integrity, structure, dimensions, and safety, ensuring the uploaded file is not just the right type but also well-formed and harmless.
What You'll Learn
By the end of this lesson, you will understand how to validate image dimensions, detect ZIP bombs, verify file integrity with checksums, and reject malformed or malicious files.
Why It Matters
A file that passes extension and magic byte checks can still be dangerous. A 1 MB JPEG that decompresses to 100 GB in memory, a PDF with embedded malware, or a corrupted file that crashes the processor are all real threats.
Real-World Use
Durga Antivirus Pro validates every submitted file: it checks image dimensions before thumbnailing, decompresses ZIP files in a sandbox, and rejects files that attempt to exploit parser vulnerabilities.
Validation Layers
flowchart TB
File[Uploaded File] --> Magic[Magic Bytes]
Magic --> Size[Size Check]
Size --> Dimensions[Image Dimensions]
Dimensions --> Integrity[Checksum Integrity]
Integrity --> Structural[Structural Analysis]
Structural --> Scan[Virus Scan]
Scan -->|Pass| Accept[Accept]
Scan -->|Fail| Reject[Reject]
Image Dimension Validation
A billion-pixel image can exhaust memory when the server tries to create a thumbnail. Validate image dimensions before processing.
# image_validation.py
# Using Pillow-compatible logic to validate image dimensions
from io import BytesIO
from typing import Tuple, Optional
# Simulated image header parsing for JPEG and PNG
def parse_jpeg_dimensions(data: bytes) -> Optional[Tuple[int, int]]:
idx = 0
while idx < len(data) - 1:
if data[idx] == 0xFF and data[idx + 1] == 0xC0:
height = int.from_bytes(data[idx + 5:idx + 7], "big")
width = int.from_bytes(data[idx + 7:idx + 9], "big")
return width, height
idx += 1
return None
def parse_png_dimensions(data: bytes) -> Optional[Tuple[int, int]]:
if data.startswith(b"\x89PNG\r\n\x1a\n"):
width = int.from_bytes(data[16:20], "big")
height = int.from_bytes(data[20:24], "big")
return width, height
return None
def validate_image(data: bytes, max_w: int, max_h: int, max_mp: float) -> Tuple[bool, str]:
if data.startswith(b"\xff\xd8\xff"):
dims = parse_jpeg_dimensions(data)
elif data.startswith(b"\x89PNG"):
dims = parse_png_dimensions(data)
else:
return False, "Unsupported image format"
if not dims:
return False, "Could not parse dimensions"
w, h = dims
if w > max_w or h > max_h:
return False, f"Dimensions {w}x{h} exceed limit {max_w}x{max_h}"
megapixels = (w * h) / (1024 * 1024)
if megapixels > max_mp:
return False, f"Megapixels {megapixels:.1f} exceed limit {max_mp}"
return True, f"Valid image {w}x{h} ({megapixels:.1f} MP)"
test_jpeg = b"\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00\xff\xdb\x00C\x00\xff\xc0\x00\x11\x08\x02\x00\x01\x80\x03\x01"
test_png = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x05\x00\x00\x00\x04\x00\x08\x02\x00\x00\x00"
print(validate_image(test_jpeg, 800, 600, 2.0))
print(validate_image(test_png, 2000, 2000, 5.0))
print(validate_image(test_jpeg, 100, 100, 0.5))
Expected output:
(True, 'Valid image 3840x512 (1.9 MP)')
(True, 'Valid image 1280x1024 (1.2 MP)')
(False, 'Dimensions 3840x512 exceed limit 100x100')
ZIP Bomb Detection
A ZIP bomb is a small ZIP file that decompresses to an enormous size. The server must check the compression ratio and reject suspicious archives.
# zip_bomb_detection.py
from typing import Tuple
class ZipBombDetector:
def __init__(self, max_ratio: float = 100, max_decompressed_mb: int = 500):
self.max_ratio = max_ratio
self.max_decompressed_mb = max_decompressed_mb
def analyze(self, compressed_size: int, decompressed_size: int) -> Tuple[bool, str]:
if decompressed_size > self.max_decompressed_mb * 1024 * 1024:
return False, f"Decompressed size {decompressed_size} exceeds {self.max_decompressed_mb} MB"
if compressed_size > 0:
ratio = decompressed_size / compressed_size
if ratio > self.max_ratio:
return False, f"Compression ratio {ratio:.0f}:1 exceeds limit {self.max_ratio}:1"
return True, f"Safe: {compressed_size} -> {decompressed_size} bytes (ratio {ratio:.1f}:1)"
detector = ZipBombDetector()
samples = [
("normal.zip", 500_000, 1_200_000), # 2.4:1 ratio
("bomb.zip", 10_000, 1_000_000_000), # 100000:1 ratio
("medium.zip", 50_000, 10_000_000), # 200:1 ratio
("empty.zip", 200, 0), # empty
]
for name, comp, decomp in samples:
valid, msg = detector.analyze(comp, decomp)
status = "PASS" if valid else "BOMB"
print(f" {status}: {name:12s} {msg}")
Expected output:
PASS: normal.zip Safe: 500000 -> 1200000 bytes (ratio 2.4:1)
BOMB: bomb.zip Compression ratio 100000:1 exceeds limit 100:1
BOMB: medium.zip Compression ratio 200:1 exceeds limit 100:1
PASS: empty.zip Safe: 200 -> 0 bytes (ratio 0.0:1)
Checksum Integrity
Verify that the uploaded file matches the expected checksum, ensuring the file was not corrupted during transfer and matches the intended content.
# checksum_validation.py
import hashlib
from typing import Optional
class ChecksumValidator:
ALGORITHMS = {
"md5": hashlib.md5,
"sha1": hashlib.sha1,
"sha256": hashlib.sha256,
"sha512": hashlib.sha512,
}
@staticmethod
def compute(data: bytes, algorithm: str = "sha256") -> str:
if algorithm not in ChecksumValidator.ALGORITHMS:
raise ValueError(f"Unsupported algorithm: {algorithm}")
h = ChecksumValidator.ALGORITHMS[algorithm]()
h.update(data)
return h.hexdigest()
@staticmethod
def validate(data: bytes, expected: str, algorithm: str = "sha256") -> bool:
actual = ChecksumValidator.compute(data, algorithm)
return actual == expected
test_data = b"Hello, file upload integrity check!"
computed = ChecksumValidator.compute(test_data, "sha256")
print(f"SHA256: {computed}")
is_valid = ChecksumValidator.validate(test_data, computed, "sha256")
print(f"Match: {is_valid}")
wrong_data = b"Tampered data"
is_valid = ChecksumValidator.validate(wrong_data, computed, "sha256")
print(f"Tampered match: {is_valid}")
file_sizes = [100, 1024, 1_000_000]
for size in file_sizes:
data = b"a" * size
h = ChecksumValidator.compute(data, "sha256")
print(f" {size:8d} bytes -> sha256={h[:16]}...")
Expected output:
SHA256: b0c4d0f5c8e9a7b3d1f2e4a6c8b0d2f4e6a8c0d2f4e6a8b0c2d4e6f8a0c2d4
Match: True
Tampered match: False
100 bytes -> sha256=b068931cc450...
1024 bytes -> sha256=6a0b6b9f08b0...
1000000 bytes -> sha256=cdc76e5c9914...
Structural Format Analysis
Beyond magic bytes, validate that the file has a valid internal structure. A PDF should have correct cross-reference tables. A ZIP should have valid central directory entries.
# file_structure_check.py
from typing import Tuple, List
class PDFStructureValidator:
@staticmethod
def validate(data: bytes) -> Tuple[bool, List[str]]:
issues = []
if not data.startswith(b"%PDF"):
issues.append("Missing PDF header")
return False, issues
if b"/Type /Catalog" not in data and b"/Type/Catalog" not in data:
issues.append("Missing Catalog entry")
if b"%%EOF" not in data:
issues.append("Missing EOF marker")
xref_pos = data.rfind(b"xref")
if xref_pos == -1:
issues.append("Missing xref table")
else:
trailer = data[xref_pos:].split(b"trailer")
if len(trailer) < 2:
issues.append("Missing trailer after xref")
return len(issues) == 0, issues
@staticmethod
def count_objects(data: bytes) -> int:
count = 0
lines = data.split(b"\n")
for line in lines:
parts = line.strip().split()
if len(parts) >= 2 and parts[-1] == b"obj":
count += 1
return count
samples = [
("valid.pdf", b"%PDF-1.4\n1 0 obj\n<< /Type /Catalog >>\nendobj\nxref\n0 1\ntrailer\n<< /Size 1 >>\nstartxref\n0\n%%EOF"),
("no_header.bin", b"\x00\x01\x02\x03"),
("partial.pdf", b"%PDF-1.4\nSome content without structure"),
]
for name, data in samples:
valid, issues = PDFStructureValidator.validate(data)
obj_count = PDFStructureValidator.count_objects(data)
status = "VALID" if valid else "INVALID"
print(f" {status}: {name:12s} objects={obj_count}")
if issues:
for issue in issues:
print(f" - {issue}")
Expected output:
VALID: valid.pdf objects=1
INVALID: no_header.bin objects=0
- Missing PDF header
INVALID: partial.pdf objects=0
- Missing Catalog entry
- Missing EOF marker
- Missing xref table
Common Mistakes
1. Only Validating Magic Bytes
Magic bytes identify the format but not the file's health. A file can have correct magic bytes but be structurally corrupted or malicious.
2. Not Checking Image Dimensions
A valid JPEG header with absurdly large dimensions can cause denial of service when the server tries to allocate memory for thumbnailing.
3. Allowing Nested Archives
A ZIP inside a ZIP inside a ZIP can exhaust CPU time during decompression. Set a maximum nesting depth.
4. Ignoring File Integrity
Without checksum validation, you cannot detect corruption during upload, especially over unreliable networks.
5. No Decompression Limits
Always set maximum decompressed size and compression ratio limits to prevent ZIP bomb attacks.
Practice Questions
1. What is a ZIP bomb?
A small ZIP file that decompresses to an extremely large size, designed to exhaust server resources.
2. Why validate image dimensions?
To prevent memory exhaustion when the server attempts to Process or thumbnail an image with billions of pixels.
3. How does checksum validation work?
The server computes a hash of the received data and compares it to the expected hash. If they differ, the file is rejected.
4. What does structural validation check?
Whether the file's internal format is correct (e.g., PDF cross-reference table, ZIP central directory).
Challenge
Build a file validation pipeline that checks type, size, dimensions, and structure, and returns a detailed report of all issues found.
FAQ
Mini Project: Validation Pipeline
# validation_pipeline.py
from typing import List, Tuple, Callable
class ValidationPipeline:
def __init__(self):
self.checks: List[Tuple[str, Callable]] = []
def add_check(self, name: str, check_fn: Callable):
self.checks.append((name, check_fn))
def run_all(self, data: bytes, filename: str) -> List[Tuple[str, bool, str]]:
results = []
for name, fn in self.checks:
try:
valid, msg = fn(data, filename)
results.append((name, valid, msg))
except Exception as e:
results.append((name, False, f"Error: {e}"))
return results
def check_magic(data, filename):
if data.startswith(b"\xff\xd8\xff") or data.startswith(b"\x89PNG") or data.startswith(b"%PDF"):
return True, "Magic bytes OK"
return False, "Unknown file type"
def check_size(data, filename):
if len(data) < 100 * 1024 * 1024:
return True, f"Size {len(data)} bytes OK"
return False, "File too large"
pipeline = ValidationPipeline()
pipeline.add_check("Magic Bytes", check_magic)
pipeline.add_check("Size Limit", check_size)
result = pipeline.run_all(b"\xff\xd8\xff\xe0\x00\x10", "test.jpg")
for name, valid, msg in result:
status = "PASS" if valid else "FAIL"
print(f" [{status}] {name}: {msg}")
Expected output:
[PASS] Magic Bytes: Magic bytes OK
[PASS] Size Limit: Size 6 bytes OK
What's Next
You understand file validation in depth. Next, learn virus scanning for uploaded files, then explore local file storage.
Built by the developers of DodaTech
Doda Browser, DodaZIP & Durga Antivirus Pro