Python Serialization — JSON, Pickle, MessagePack, and Data Exchange Formats

import json
from datetime import datetime
from pathlib import Path

# Basic serialization
data = {
    "name": "Alice",
    "age": 30,
    "skills": ["Python", "JavaScript"],
    "active": True,
    "score": None
}

json_string = json.dumps(data, indent=2)
print(json_string)

# Write to file
with open("data.json", "w") as f:
    json.dump(data, f, indent=2)

# Read from file
with open("data.json") as f:
    loaded = json.load(f)

print(loaded["name"])  # "Alice"

import json
from datetime import datetime, date
from decimal import Decimal
from pathlib import Path

class CustomEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, datetime):
            return obj.isoformat()
        if isinstance(obj, date):
            return obj.isoformat()
        if isinstance(obj, Decimal):
            return str(obj)
        if isinstance(obj, Path):
            return str(obj)
        if isinstance(obj, set):
            return list(obj)
        if isinstance(obj, bytes):
            return obj.hex()
        return super().default(obj)

# Usage
data = {
    "timestamp": datetime.now(),
    "birthday": date(1995, 6, 15),
    "price": Decimal("19.99"),
    "path": Path("/home/user/config.yaml"),
    "tags": {"python", "json"},
    "hash": b"\x00\x01\x02"
}

print(json.dumps(data, cls=CustomEncoder, indent=2))

# Decoding custom objects
class CustomDecoder(json.JSONDecoder):
    def __init__(self):
        super().__init__(object_hook=self.object_hook)

    def object_hook(self, dct):
        for key in ("timestamp", "birthday"):
            if key in dct:
                try:
                    dct[key] = datetime.fromisoformat(dct[key])
                except (ValueError, TypeError):
                    pass
        return dct

# One JSON object per line — ideal for streaming/logs
import json

def write_jsonl(filename, records):
    with open(filename, "w") as f:
        for record in records:
            f.write(json.dumps(record) + "\n")

def read_jsonl(filename):
    records = []
    with open(filename) as f:
        for line in f:
            line = line.strip()
            if line:
                records.append(json.loads(line))
    return records

import pickle
from datetime import datetime

# Pickle can serialize almost any Python object
data = {
    "name": "Alice",
    "timestamp": datetime.now(),
    "scores": [95, 88, 92],
    "metadata": {"version": 2, "checksum": None}
}

# Serialize to bytes
pickled = pickle.dumps(data)
print(f"Pickle size: {len(pickled)} bytes")

# Deserialize
unpickled = pickle.loads(pickled)
print(unpickled["name"])  # "Alice"

# Write to file
with open("data.pkl", "wb") as f:
    pickle.dump(data, f)

# Read from file
with open("data.pkl", "rb") as f:
    data_loaded = pickle.load(f)

# NEVER unpickle untrusted data!
# Pickle can execute arbitrary code:

class DangerousPickle:
    def __reduce__(self):
        import os
        return (os.system, ("rm -rf /",))

# If someone sends you this:
# pickle.loads(evil_pickle_data)  # Disaster!

# SAFE alternatives:
# - Use JSON for data exchange
# - Use itsdangerous if you need signed data
# - Use safepickle for controlled environments only

import msgpack
import json

data = {
    "name": "Alice",
    "age": 30,
    "scores": [95.5, 88.3, 92.1],
    "metadata": {"compressed": True, "format": "v2"}
}

# Serialize
packed = msgpack.packb(data)
json_data = json.dumps(data)

print(f"MessagePack: {len(packed)} bytes")
print(f"JSON: {len(json_data)} bytes")

# Deserialize
unpacked = msgpack.unpackb(packed)
print(unpacked["name"])  # "Alice"

// person.proto
syntax = "proto3";

message Person {
  string name = 1;
  int32 age = 2;
  repeated string skills = 3;
  Address address = 4;
}

message Address {
  string city = 1;
  string country = 2;
}

protoc --python_out=. person.proto

import person_pb2

# Create
person = person_pb2.Person()
person.name = "Alice"
person.age = 30
person.skills.extend(["Python", "Go"])

# Nested message
address = person_pb2.Address()
address.city = "NYC"
address.country = "USA"
person.address.CopyFrom(address)

# Serialize
serialized = person.SerializeToString()
print(f"Protobuf: {len(serialized)} bytes")

# Deserialize
person2 = person_pb2.Person()
person2.ParseFromString(serialized)
print(person2.name)

import yaml
from pathlib import Path

config = {
    "server": {
        "host": "localhost",
        "port": 8080,
        "debug": True
    },
    "database": {
        "url": "postgresql://localhost/mydb",
        "pool_size": 10
    },
    "features": {
        "rate_limiting": True,
        "caching": "redis"
    }
}

# Write
with open("config.yaml", "w") as f:
    yaml.dump(config, f, default_flow_style=False)

# Read
with open("config.yaml") as f:
    loaded = yaml.safe_load(f)  # Use safe_load, not load!

import time
import json, pickle, msgpack
import numpy as np

data = {
    "numbers": list(range(10000)),
    "text": "Hello World " * 1000,
    "nested": {"a": 1, "b": 2, "c": [1, 2, 3]}
}

formats = {
    "json": lambda: json.dumps(data),
    "pickle": lambda: pickle.dumps(data),
    "msgpack": lambda: msgpack.packb(data),
}

for name, func in formats.items():
    start = time.perf_counter()
    for _ in range(100):
        serialized = func()
    elapsed = time.perf_counter() - start
    print(f"{name:8s}: {elapsed:.3f}s, {len(serialized):6d} bytes")

# Typical results:
# json:    0.045s,  32145 bytes
# pickle:  0.021s,  28789 bytes
# msgpack: 0.031s,  20456 bytes

class DataExporter:
    def export_json(self, records, filepath): ...
    def export_jsonl(self, records, filepath): ...
    def export_msgpack(self, records, filepath): ...
    def export_csv(self, records, filepath): ...
    def export_protobuf(self, records, filepath): ...