Python Performance — Profiling, Optimization, C Extensions, and Benchmarking

import cProfile
import pstats

def slow_function():
    total = 0
    for i in range(1000000):
        total += i ** 2
    return total

# Profile
profiler = cProfile.Profile()
profiler.enable()
result = slow_function()
profiler.disable()

# Print stats sorted by cumulative time
stats = pstats.Stats(profiler)
stats.sort_stats("cumulative")
stats.print_stats(10)  # Top 10

# Save to file
profiler.dump_stats("profile.prof")

# View with:
# python -m snakeviz profile.prof  # Web-based viewer
# python -m pstats profile.prof     # Interactive CLI

import cProfile, pstats
from contextlib import contextmanager

@contextmanager
def profile(sort="cumulative", limit=20):
    profiler = cProfile.Profile()
    profiler.enable()
    try:
        yield
    finally:
        profiler.disable()
        stats = pstats.Stats(profiler)
        stats.sort_stats(sort)
        stats.print_stats(limit)

# Usage
with profile(limit=10):
    slow_function()

pip install line_profiler

@profile
def expensive_function(n):
    total = 0
    for i in range(n):
        for j in range(100):
            total += i * j
    return total

# Run: kernprof -l -v script.py

# Without __slots__: each instance has a __dict__ (overhead)
class Point:
    def __init__(self, x, y, z):
        self.x = x
        self.y = y
        self.z = z

# With __slots__: no __dict__, attributes stored in tuple
class PointOptimized:
    __slots__ = ("x", "y", "z")

    def __init__(self, x, y, z):
        self.x = x
        self.y = y
        self.z = z

# Compare memory usage
import sys

p1 = Point(1, 2, 3)
p2 = PointOptimized(1, 2, 3)

print(f"Regular: {sys.getsizeof(p1)} bytes")      # ~56 bytes + dict
print(f"Slots: {sys.getsizeof(p2)} bytes")         # ~56 bytes, no dict

# For millions of instances, __slots__ saves gigabytes

# math_ops.pyx
def sum_of_squares(int n):
    cdef int i
    cdef long total = 0
    for i in range(n):
        total += i * i
    return total

# setup.py
from setuptools import setup
from Cython.Build import cythonize

setup(
    ext_modules=cythonize("math_ops.pyx"),
)

# Pure Python version (for comparison)
def sum_of_squares_py(n):
    return sum(i * i for i in range(n))

# After building:
from math_ops import sum_of_squares
# Cython version is ~10-50x faster

from numba import njit
import numpy as np
import time

# Pure Python
def monte_carlo_pi_py(n):
    inside = 0
    for _ in range(n):
        x = np.random.random()
        y = np.random.random()
        if x * x + y * y <= 1:
            inside += 1
    return 4 * inside / n

# Numba JIT compiled
@njit
def monte_carlo_pi_jit(n):
    inside = 0
    for _ in range(n):
        x = np.random.random()
        y = np.random.random()
        if x * x + y * y <= 1:
            inside += 1
    return 4 * inside / n

# Compare
n = 10000000
start = time.time()
result_py = monte_carlo_pi_py(n)
print(f"Pure Python: {time.time() - start:.2f}s")

start = time.time()
result_jit = monte_carlo_pi_jit(n)
print(f"Numba JIT: {time.time() - start:.2f}s")
# Typically 50-200x faster

import multiprocessing as mp
import threading
import asyncio
import time

# CPU-bound: use multiprocessing
def cpu_intensive(n):
    total = 0
    for i in range(n):
        total += i ** 2
    return total

def parallel_cpu():
    with mp.Pool(processes=mp.cpu_count()) as pool:
        results = pool.map(cpu_intensive, [1000000] * 8)
    return results

# I/O-bound: use asyncio or threading
async def fetch_url_async(session, url):
    async with session.get(url) as response:
        return await response.text()

async def parallel_io():
    import aiohttp
    async with aiohttp.ClientSession() as session:
        tasks = [
            fetch_url_async(session, f"https://api.example.com/{i}")
            for i in range(100)
        ]
        return await asyncio.gather(*tasks)

# Benchmark rule of thumb:
# - CPU-bound: multiprocessing (bypasses GIL)
# - I/O-bound with many connections: asyncio (lightweight)
# - I/O-bound with blocking libraries: threading

import timeit

def list_comprehension():
    return [i ** 2 for i in range(1000)]

def manual_loop():
    result = []
    for i in range(1000):
        result.append(i ** 2)
    return result

# Run 10000 times
time_comp = timeit.timeit(list_comprehension, number=10000)
time_loop = timeit.timeit(manual_loop, number=10000)

print(f"List comprehension: {time_comp:.4f}s")
print(f"Manual loop: {time_loop:.4f}s")

# Using pytest-benchmark
# def test_performance(benchmark):
#     result = benchmark(list_comprehension)
#     assert len(result) == 1000

# BAD: O(n^2) — creates new string each iteration
def bad_concat(items):
    result = ""
    for item in items:
        result += item
    return result

# GOOD: O(n)
def good_concat(items):
    return "".join(items)

# BAD: looks up math.sqrt every iteration
def bad_loop(values):
    import math
    return [math.sqrt(v) for v in values]

# GOOD: local reference
def good_loop(values):
    import math
    sqrt = math.sqrt
    return [sqrt(v) for v in values]

# BAD: O(n) lookup
def bad_membership(items, target):
    return target in items  # items is a list

# GOOD: O(1) lookup
def good_membership(items, target):
    return target in items  # items is a set