Versioning gRPC APIs — Protobuf Compatibility and Schema Evolution
In this tutorial, you'll learn about Versioning Grpc. We cover key concepts, practical examples, and best practices to help you understand and apply this topic effectively.
gRPC API Versioning relies on Protocol Buffers design for backward compatibility, using field numbers, wire format rules, and deprecation conventions instead of URL versioning.
What You'll Learn
By the end of this lesson, you will apply protobuf backward compatibility rules, manage field number assignments, deprecate fields safely, evolve gRPC services, and handle breaking changes.
Why It Matters
Protobufs wire format supports many changes without breaking existing clients, making gRPC naturally version-tolerant when you follow the compatibility rules.
Real-World Use
Durga Antivirus Pro evolved its gRPC ScanService from version 1 to version 2 by adding new fields and services while keeping old field numbers reserved, maintaining full backward compatibility.
Protobuf Compatibility Rules
flowchart TD
Change[Protobuf Change]-->Compat{Wire Compatible?}
Compat-->|Add Field|Yes[Safe - new field number]
Compat-->|Remove Field|No[Breaking - reserve number]
Compat-->|Rename Field|Yes[Wire format unchanged]
Compat-->|Change Type|No[Breaking - different wire type]
Compat-->|Change Field Number|No[Breaking - different number]
Compat-->|Add Service|Yes[New RPC, old unchanged]
Protobuf Compatibility Checker
Validate protobuf changes for backward compatibility.
from typing import Dict, List, Optional, Set, Tuple
from enum import Enum
class FieldRule:
def __init__(self, name: str, number: int,
field_type: str, label: str = "optional"):
self.name = name
self.number = number
self.field_type = field_type
self.label = label
class ProtoService:
def __init__(self, name: str):
self.name = name
self.rpcs: Dict[str, Dict] = {}
class ProtoCompatibilityChecker:
WIRE_TYPES = {
"int32": 0, "int64": 0, "uint32": 0, "uint64": 0,
"sint32": 0, "sint64": 0, "bool": 0, "enum": 0,
"fixed64": 1, "sfixed64": 1, "double": 1,
"string": 2, "bytes": 2, "message": 2,
"fixed32": 5, "sfixed32": 5, "float": 5,
}
def __init__(self, old: Dict[str, List[FieldRule]],
new: Dict[str, List[FieldRule]]):
self.old = old
self.new = new
self.issues: List[str] = []
def check_compatibility(self) -> bool:
for msg_name, old_fields in self.old.items():
new_fields = self.new.get(msg_name, [])
old_map = {f.number: f for f in old_fields}
new_map = {f.number: f for f in new_fields}
for number, old_field in old_map.items():
if number not in new_map:
self.issues.append(
f"Field {old_field.name} (number {number}) "
f"removed from {msg_name}. Use reserved."
)
else:
new_field = new_map[number]
old_wt = self.WIRE_TYPES.get(
old_field.field_type, -1
)
new_wt = self.WIRE_TYPES.get(
new_field.field_type, -1
)
if old_wt != new_wt:
self.issues.append(
f"Field {old_field.name} type changed"
f" from {old_field.field_type} to "
f"{new_field.field_type} "
f"(different wire type)"
)
for number, new_field in new_map.items():
if number not in old_map:
if new_field.label == "required":
self.issues.append(
f"New field {new_field.name} is required"
)
return len(self.issues) == 0
def get_report(self) -> Dict:
return {
"compatible": self.check_compatibility(),
"issues": self.issues,
}
old_fields = [
FieldRule("id", 1, "string"),
FieldRule("name", 2, "string"),
FieldRule("status", 3, "int32"),
]
new_fields = [
FieldRule("id", 1, "string"),
FieldRule("name", 2, "string"),
FieldRule("status", 3, "int32"),
FieldRule("email", 4, "string"),
]
checker = ProtoCompatibilityChecker(
{"User": old_fields},
{"User": new_fields}
)
report = checker.get_report()
print(f"Compatible: {report['compatible']}")
Field Number Management
Manage protobuf field number assignments for future compatibility.
from typing import Dict, List, Optional, Set
class FieldNumberRegistry:
def __init__(self, message_name: str):
self.message_name = message_name
self.used_numbers: Set[int] = set()
self.reserved_numbers: Set[int] = set()
self.fields: Dict[int, str] = {}
def add_field(self, number: int, name: str):
if number in self.used_numbers:
raise ValueError(
f"Field number {number} already used"
)
if number in self.reserved_numbers:
raise ValueError(
f"Field number {number} is reserved"
)
self.used_numbers.add(number)
self.fields[number] = name
def reserve_number(self, number: int):
self.reserved_numbers.add(number)
def reserve_range(self, start: int, end: int):
for n in range(start, end + 1):
self.reserved_numbers.add(n)
def get_next_available(self) -> int:
n = 1
while n in self.used_numbers or n in self.reserved_numbers:
n += 1
return n
def generate_proto(self) -> str:
lines = [f"message {self.message_name} {{"]
for num in sorted(self.reserved_numbers):
lines.append(f" reserved {num};")
for num in sorted(self.fields):
lines.append(f" {self.fields[num]} = {num};")
lines.append("}")
return "\n".join(lines)
def get_stats(self) -> Dict:
return {
"message": self.message_name,
"used": len(self.used_numbers),
"reserved": len(self.reserved_numbers),
"available": 19000 - len(self.used_numbers)
- len(self.reserved_numbers),
}
registry = FieldNumberRegistry("ScanRequest")
registry.add_field(1, "string file_path = 1")
registry.add_field(2, "string scan_type = 2")
registry.reserve_range(3, 10)
next_num = registry.get_next_available()
print(f"Next available field number: {next_num}")
gRPC Service Versioning
Version gRPC services through package naming and service suffixes.
from typing import Dict, Optional
class GRPCServiceVersion:
def __init__(self, service_name: str,
major_version: int = 1):
self.service_name = service_name
self.major_version = major_version
def get_package_name(self) -> str:
return f"{self.service_name.lower()}.v{self.major_version}"
def get_proto_package(self) -> str:
return f"package {self.get_package_name()};"
def get_service_definition(self,
rpcs: list) -> str:
lines = [
f'syntax = "proto3";',
f'{self.get_proto_package()}',
"",
f'service {self.service_name}V{self.major_version}',
" {"
]
for rpc in rpcs:
lines.append(
f" rpc {rpc['name']}({rpc['input']}) "
f"returns ({rpc['output']});"
)
lines.append("}")
return "\n".join(lines)
def get_grpc_endpoint(self) -> str:
return (f"{self.service_name.lower()}.v{self.major_version}."
f"{self.service_name}V{self.major_version}")
svc = GRPCServiceVersion("ScanService", 2)
proto = svc.get_service_definition([
{"name": "ScanFile", "input": "ScanRequest",
"output": "ScanResponse"},
{"name": "GetStatus", "input": "StatusRequest",
"output": "StatusResponse"},
])
print(proto[:300] + "...")
Common Mistakes
Mistake 1: Reusing Field Numbers
Once a field number is used for one type, it cannot be reused for a different type without breaking wire compatibility.
Mistake 2: Not Reserving Removed Numbers
Removed field numbers must be reserved to prevent accidental reuse with different types.
Mistake 3: Changing Field Types
Changing a field from int32 to string changes the wire type and breaks existing serialized data.
Mistake 4: Using Required Fields
Required fields break forward compatibility. New fields should always be optional.
Mistake 5: Ignoring Unknown Field Handling
Protobuf preserves unknown fields. Ensure your code handles them when re-serializing.
Practice Questions
- What protobuf changes are wire-compatible?
- Why must removed field numbers be reserved?
- How does protobuf handle unknown fields?
- What is the difference between forward and backward compatibility in protobuf?
- How do you version gRPC service names?
Challenge
Build a protobuf compatibility checker that validates field changes between proto file versions, detects wire-type changes, ensures removed field numbers are reserved, flags new required fields, and generates a compatibility report.
FAQ
Mini Project
Build a protobuf versioning system that manages field number assignments, validates backward compatibility of proto changes, detects breaking wire-type changes, generates proto definitions with reserved numbers, and produces a compatibility report for each change.
What's Next
Learn about Versioning REST APIs for REST API strategies, or explore Schema Evolution for broader database and API schema evolution patterns.
Built by the developers of DodaTech
Doda Browser, DodaZIP & Durga Antivirus Pro