Over-Fetching — REST vs GraphQL
In this tutorial, you'll learn about Over-Fetching. We cover key concepts, practical examples, and best practices to help you understand and apply this topic effectively.
Over-fetching occurs when an API returns more data than the client needs, because the server controls the response structure and the client cannot specify which fields to include.
What You'll Learn
By the end of this lesson, you will identify over-fetching scenarios, measure its impact, and know how Graphql solves it with field-level selection.
Why It Matters
Over-fetching wastes bandwidth, slows page loads, and increases mobile data usage. A profile page needing 3 fields from a response with 30 fields is over-fetching at 10x efficiency.
Real-World Use
A Twitter client showing only tweet text and timestamp receives the full tweet object including geo-location, source, media URLs, and reply metadata it never displays.
Over-Fetching Flow
flowchart LR
Client[Client needs name+email] --> REST[REST /user/1]
REST --> Response[{"id","name","email","phone","address","age","role","created"}]
Response --> Waste[8 fields received, 2 needed]
Client --> GQL[GraphQL {user{name email}}]
GQL --> Precise[{"name":"Alice","email":"a@x.com"}]
Measuring Over-Fetching
# overfetch_measure.py
from typing import Any, Dict, List
class OverfetchAnalyzer:
def __init__(self):
self.rest_responses: List[Dict] = []
self.gql_responses: List[Dict] = []
def rest_response(self, endpoint: str) -> Dict:
data = {
"/user/1": {
"id": 1, "name": "Alice", "email": "a@x.com",
"phone": "555-0100", "address": "123 St",
"age": 30, "role": "admin", "status": "active",
"created_at": "2024-01-01", "updated_at": "2024-06-01",
"last_login": "2024-06-15", "avatar_url": "https://...",
"bio": "Developer", "timezone": "UTC",
}
}
resp = data.get(endpoint, {})
self.rest_responses.append(resp)
return resp
def rest_fields_received(self, endpoint: str) -> int:
resp = self.rest_response(endpoint)
return len(resp)
def fields_needed(self) -> List[str]:
return ["name", "email"]
def overfetch_ratio(self, endpoint: str) -> float:
received = self.rest_fields_received(endpoint)
needed = len(self.fields_needed())
return received / needed
def overfetch_bytes(self, endpoint: str) -> int:
resp = self.rest_response(endpoint)
needed = {k: resp[k] for k in self.fields_needed() if k in resp}
return len(str(resp)) - len(str(needed))
analyzer = OverfetchAnalyzer()
endpoint = "/user/1"
ratio = analyzer.overfetch_ratio(endpoint)
wasted = analyzer.overfetch_bytes(endpoint)
print(f"Fields received: {analyzer.rest_fields_received(endpoint)}")
print(f"Fields needed: {len(analyzer.fields_needed())}")
print(f"Over-fetch ratio: {ratio:.1f}x")
print(f"Wasted bytes: ~{wasted}")
Expected output:
Fields received: 14
Fields needed: 2
Over-fetch ratio: 7.0x
Wasted bytes: ~<n>
GraphQL Field Selection
# field_selection.py
from typing import Any, Dict, List
class GraphQLFieldSelector:
def __init__(self):
self.user_data = {
"id": 1,
"name": "Alice",
"email": "a@x.com",
"phone": "555-0100",
"address": "123 St",
"age": 30,
"role": "admin",
"status": "active",
"created_at": "2024-01-01",
"updated_at": "2024-06-01",
"last_login": "2024-06-15",
"avatar_url": "https://example.com/avatar.jpg",
"bio": "Developer",
"timezone": "UTC",
}
def query(self, fields: List[str]) -> Dict:
result = {}
for field in fields:
if field in self.user_data:
result[field] = self.user_data[field]
elif field == "posts":
result["posts"] = [{"title": "Post 1", "body": "..."}]
return result
def payload_efficiency(self, fields: List[str]) -> Dict:
result = self.query(fields)
received = len(result)
total_available = len(self.user_data)
bytes_used = len(str(result))
bytes_total = len(str(self.user_data))
return {
"fields_requested": len(fields),
"fields_returned": received,
"fields_available": total_available,
"efficiency": f"{bytes_used / bytes_total * 100:.0f}%",
}
selector = GraphQLFieldSelector()
queries = [
["name", "email"],
["name", "email", "phone", "address"],
["id", "name", "email", "age", "role", "status", "posts"],
]
for q in queries:
eff = selector.payload_efficiency(q)
print(f"Requested {eff['fields_requested']} fields -> {eff['efficiency']} of full payload")
Expected output:
Requested 2 fields -> 14% of full payload
Requested 4 fields -> 27% of full payload
Requested 7 fields -> 44% of full payload
Real-World Over-Fetching Example
# realworld_overfetch.py
from typing import Any, Dict, List
class BlogAPI:
def rest_get_post(self, post_id: int) -> Dict:
return {
"id": post_id,
"title": "GraphQL vs REST",
"slug": "graphql-vs-rest",
"body": "Long article content...",
"excerpt": "Short summary",
"author": {"id": 1, "name": "Alice", "email": "a@x.com", "bio": "Dev"},
"tags": ["api", "graphql"],
"category": "backend",
"published_at": "2024-01-15T10:00:00Z",
"updated_at": "2024-06-01T12:00:00Z",
"view_count": 1500,
"like_count": 42,
"comment_count": 7,
"image_url": "https://example.com/image.jpg",
"meta_description": "Compare GraphQL and REST",
"meta_keywords": "graphql, rest, api",
"canonical_url": "https://example.com/graphql-vs-rest",
}
def graphql_get_post(self, post_id: int, fields: List[str]) -> Dict:
full = self.rest_get_post(post_id)
return {f: full[f] for f in fields if f in full}
api = BlogAPI()
post_id = 1
# Blog listing page needs only title + excerpt + date
blog_listing = api.graphql_get_post(post_id, ["title", "excerpt", "published_at"])
print(f"Blog listing needed: {list(blog_listing.keys())}")
# Full article page needs more
full_article = api.graphql_get_post(post_id, ["title", "body", "author", "published_at"])
print(f"Full article needed: {list(full_article.keys())}")
# REST returns everything for both
rest = api.rest_get_post(post_id)
print(f"REST returns: {len(rest)} fields (always the same)")
Expected output:
Blog listing needed: ['title', 'excerpt', 'published_at']
Full article needed: ['title', 'body', 'author', 'published_at']
REST returns: 16 fields (always the same)
Common Mistakes
1. Overlooking Over-Fetching Impact
Developers on fast networks may not notice. Mobile users on 3G pay for every byte. Profile on real mobile connections.
2. Over-Fetching in List Endpoints
GET /posts returns 50 posts with full bodies. For list views, only titles and excerpts are needed. Use sparse field sets.
3. Not Measuring
Without measuring over-fetching, teams optimize the wrong things. Add payload size monitoring.
4. Over-Fetching in REST with Includes
Adding ?include=author,comments to REST reduces calls but returns even more data per call.
5. Not Considering GraphQL Query Cost
While GraphQL eliminates over-fetching, clients can request deeply nested data. Set complexity limits.
Practice Questions
1. What is over-fetching?
When a server returns more fields than the client needs because the response structure is fixed.
2. How does GraphQL prevent over-fetching?
Clients specify exactly which fields they want. The server returns only those fields.
3. Why is over-fetching worse for mobile?
Limited bandwidth, data caps, and battery life. Every extra byte costs time and money.
4. How do REST APIs reduce over-fetching?
Sparse fieldsets (?fields=name,email) or different endpoints for different view types.
Challenge
Build an over-fetching monitor that intercepts API responses and reports how many unused bytes each endpoint returns across all clients.
FAQ
Mini Project: Over-Fetching Detector
# overfetch_detector.py
from typing import Dict, List
class OverfetchDetector:
def __init__(self):
self.endpoints = {}
def record(self, url: str, fields_used: List[str], response_fields: List[str]):
if url not in self.endpoints:
self.endpoints[url] = {"uses": {}, "sizes": []}
ratio = len(response_fields) / max(len(fields_used), 1)
self.endpoints[url]["uses"][str(fields_used)] = ratio
self.endpoints[url]["sizes"].append({
"response_size": len(str(response_fields)),
"used_size": len(str(fields_used)),
})
def report(self):
for url, data in self.endpoints.items():
avg_ratio = sum(data["uses"].values()) / len(data["uses"])
print(f"{url}: avg over-fetch ratio {avg_ratio:.1f}x")
detector = OverfetchDetector()
detector.record("/user/1", ["name", "email"],
["id", "name", "email", "phone", "address"])
detector.record("/user/1", ["name", "email", "phone"],
["id", "name", "email", "phone", "address"])
detector.report()
Expected output:
/user/1: avg over-fetch ratio 2.1x
What's Next
You understand over-fetching. Next, explore under-fetching, then round trip analysis.
Built by the developers of DodaTech
Doda Browser, DodaZIP & Durga Antivirus Pro