-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathcommand_protocol.py
More file actions
360 lines (298 loc) · 13.3 KB
/
Copy pathcommand_protocol.py
File metadata and controls
360 lines (298 loc) · 13.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
from __future__ import annotations
import re
from collections.abc import Mapping
from dataclasses import dataclass
__all__ = [
"BOOLEAN",
"CommandCodec",
"CommandProtocolError",
"CommandSchemaRegistry",
"DEFAULT_SCHEMA_REGISTRY",
"FieldSpec",
"NULLABLE_STRING",
"RECORD_SCHEMAS",
"STRING",
"dumps_record",
"dumps_records",
"loads_records",
"register_record_schema",
]
PROTOCOL_HEADER = "COMMAND_PROTOCOL_V1"
MAX_RECORD_COUNT = 1_000_000
class CommandProtocolError(ValueError):
pass
@dataclass(frozen=True)
class FieldSpec:
value_type: str
nullable: bool = False
STRING = FieldSpec("string")
NULLABLE_STRING = FieldSpec("string", nullable=True)
BOOLEAN = FieldSpec("boolean")
RecordValue = str | bool | None
Record = Mapping[str, RecordValue]
class CommandSchemaRegistry:
"""Own an isolated set of command record schemas.
The module-level helpers below continue to use the default registry for
compatibility, while consumers that host more than one protocol boundary
can construct independent registries and codecs.
"""
def __init__(self) -> None:
self.schemas: dict[str, dict[str, FieldSpec]] = {}
def register(self, record_type: str, fields: Mapping[str, FieldSpec]) -> None:
_validate_and_store_schema(self.schemas, record_type, fields)
def schema(self, record_type: str) -> dict[str, FieldSpec]:
return _lookup_schema(self.schemas, record_type)
class CommandCodec:
"""Encode and decode records using one isolated schema registry."""
def __init__(self, registry: CommandSchemaRegistry | None = None) -> None:
self.registry = registry or CommandSchemaRegistry()
def register_schema(self, record_type: str, fields: Mapping[str, FieldSpec]) -> None:
self.registry.register(record_type, fields)
def dumps_record(
self,
record_type: str,
record: Record,
*,
protocol_header: str = PROTOCOL_HEADER,
) -> str:
return dumps_record(
record_type,
record,
protocol_header=protocol_header,
registry=self.registry,
)
def dumps_records(
self,
record_type: str,
records: tuple[Record, ...] | list[Record],
*,
protocol_header: str = PROTOCOL_HEADER,
) -> str:
return dumps_records(
record_type,
records,
protocol_header=protocol_header,
registry=self.registry,
)
def loads_records(
self,
payload: str,
expected_record_type: str | None = None,
*,
protocol_header: str = PROTOCOL_HEADER,
) -> tuple[str, tuple[dict[str, RecordValue], ...]]:
return loads_records(
payload,
expected_record_type,
protocol_header=protocol_header,
registry=self.registry,
)
DEFAULT_SCHEMA_REGISTRY = CommandSchemaRegistry()
# Preserve the existing mutable compatibility surface. New consumers should
# use CommandSchemaRegistry or CommandCodec instead of process-global state.
RECORD_SCHEMAS = DEFAULT_SCHEMA_REGISTRY.schemas
def register_record_schema(record_type: str, fields: Mapping[str, FieldSpec]) -> None:
"""Register an application-specific record schema for the wire protocol.
Schema names and field names are intentionally constrained to the same
framing-safe characters used by the built-in protocol. Registration is
additive and rejects replacement of an existing schema so that a process
cannot silently change the meaning of an established record type.
"""
DEFAULT_SCHEMA_REGISTRY.register(record_type, fields)
def dumps_record(
record_type: str,
record: Record,
*,
protocol_header: str = PROTOCOL_HEADER,
registry: CommandSchemaRegistry | None = None,
) -> str:
return dumps_records(
record_type,
(record,),
protocol_header=protocol_header,
registry=registry,
)
def dumps_records(
record_type: str,
records: tuple[Record, ...] | list[Record],
*,
protocol_header: str = PROTOCOL_HEADER,
registry: CommandSchemaRegistry | None = None,
) -> str:
active_registry = registry or DEFAULT_SCHEMA_REGISTRY
schema = active_registry.schema(record_type)
if len(records) > MAX_RECORD_COUNT:
raise CommandProtocolError(f"record_count exceeds protocol maximum of {MAX_RECORD_COUNT}")
lines = [
protocol_header,
f"record_type={record_type}",
f"record_count={len(records)}",
]
for index, record in enumerate(records):
_validate_record(schema, record_type, record)
lines.append(f"record={index}")
for field_name, spec in schema.items():
wire_type, payload = _encode_value(field_name, spec, record[field_name])
lines.append(f"field.{field_name}:{wire_type}={payload}")
lines.append(f"end_record={index}")
lines.append("end_protocol=")
return "\n".join(lines)
def loads_records(
payload: str,
expected_record_type: str | None = None,
*,
protocol_header: str = PROTOCOL_HEADER,
registry: CommandSchemaRegistry | None = None,
) -> tuple[str, tuple[dict[str, RecordValue], ...]]:
active_registry = registry or DEFAULT_SCHEMA_REGISTRY
# The wire framing is LF-delimited. `str.splitlines()` also accepts CR,
# vertical tab, form feed, and Unicode separators, which would make the
# Python decoder more permissive than the Bash and Zsh readers.
# A CLI `print()` adds one terminal LF; accept that conventional text-file
# terminator without accepting an extra blank line or other separators.
framed_payload = payload.removesuffix("\n")
lines = framed_payload.split("\n")
cursor = 0
def take(label: str) -> str:
nonlocal cursor
if cursor >= len(lines):
raise CommandProtocolError(f"missing {label}")
line = lines[cursor]
cursor += 1
return line
if take("protocol header") != protocol_header:
raise CommandProtocolError(f"unsupported protocol header; expected {protocol_header}")
record_type = _metadata_value(take("record_type"), "record_type")
schema = active_registry.schema(record_type)
if expected_record_type is not None and record_type != expected_record_type:
raise CommandProtocolError(f"expected record_type '{expected_record_type}', got '{record_type}'")
record_count_text = _metadata_value(take("record_count"), "record_count")
record_count = _parse_record_count(record_count_text)
records: list[dict[str, RecordValue]] = []
for index in range(record_count):
if take(f"record {index}") != f"record={index}":
raise CommandProtocolError(f"expected record={index}")
record: dict[str, RecordValue] = {}
for _field_index in range(len(schema)):
field_name, wire_type, encoded_value = _parse_field_line(take(f"field in record {index}"))
if field_name in record:
raise CommandProtocolError(f"record {index} duplicates field '{field_name}'")
try:
spec = schema[field_name]
except KeyError as exc:
raise CommandProtocolError(
f"record {index} has unknown field '{field_name}' for '{record_type}'"
) from exc
record[field_name] = _decode_value(field_name, spec, wire_type, encoded_value)
missing = sorted(set(schema) - set(record))
if missing:
raise CommandProtocolError(f"record {index} is missing fields: {', '.join(missing)}")
if take(f"end_record {index}") != f"end_record={index}":
raise CommandProtocolError(f"expected end_record={index}")
records.append(record)
if take("end_protocol") != "end_protocol=":
raise CommandProtocolError("expected end_protocol marker")
if cursor != len(lines):
raise CommandProtocolError("unexpected data after end_protocol marker")
return record_type, tuple(records)
def _lookup_schema(
schemas: Mapping[str, dict[str, FieldSpec]],
record_type: str,
) -> dict[str, FieldSpec]:
try:
return schemas[record_type]
except KeyError as exc:
supported = ", ".join(sorted(schemas))
raise CommandProtocolError(f"unsupported record_type '{record_type}'; expected one of: {supported}") from exc
def _validate_and_store_schema(
schemas: dict[str, dict[str, FieldSpec]],
record_type: str,
fields: Mapping[str, FieldSpec],
) -> None:
if not isinstance(record_type, str) or re.fullmatch(r"[A-Za-z][A-Za-z0-9-]*", record_type) is None:
raise CommandProtocolError(
"record_type must start with a letter and contain only letters, digits, and hyphens"
)
if record_type in schemas:
raise CommandProtocolError(f"record_type '{record_type}' is already registered")
if not isinstance(fields, Mapping) or not fields:
raise CommandProtocolError("record schema fields must be a non-empty mapping")
normalized: dict[str, FieldSpec] = {}
for field_name, spec in fields.items():
if not isinstance(field_name, str) or re.fullmatch(r"[A-Za-z][A-Za-z0-9_]*", field_name) is None:
raise CommandProtocolError(
f"field name '{field_name}' must start with a letter and contain only letters, digits, and underscores"
)
if not isinstance(spec, FieldSpec) or spec.value_type not in {"string", "boolean"}:
raise CommandProtocolError(
f"field '{field_name}' must use a FieldSpec with value_type 'string' or 'boolean'"
)
normalized[field_name] = spec
schemas[record_type] = normalized
def _validate_record(schema: Mapping[str, FieldSpec], record_type: str, record: Record) -> None:
missing = sorted(set(schema) - set(record))
unknown = sorted(set(record) - set(schema))
if missing:
raise CommandProtocolError(f"record for '{record_type}' is missing fields: {', '.join(missing)}")
if unknown:
raise CommandProtocolError(f"record for '{record_type}' has unknown fields: {', '.join(unknown)}")
def _encode_value(field_name: str, spec: FieldSpec, value: RecordValue) -> tuple[str, str]:
if value is None:
if not spec.nullable:
raise CommandProtocolError(f"field '{field_name}' cannot be null")
return "null", ""
if spec.value_type == "boolean":
if not isinstance(value, bool):
raise CommandProtocolError(f"field '{field_name}' must be a boolean")
return "boolean", "true" if value else "false"
if not isinstance(value, str):
raise CommandProtocolError(f"field '{field_name}' must be a string")
if "\0" in value:
raise CommandProtocolError(f"field '{field_name}' cannot contain NUL")
return "string", value.encode("utf-8").hex()
def _decode_value(field_name: str, spec: FieldSpec, wire_type: str, payload: str) -> RecordValue:
if wire_type == "null":
if not spec.nullable:
raise CommandProtocolError(f"field '{field_name}' cannot be null")
if payload:
raise CommandProtocolError(f"null field '{field_name}' must have an empty payload")
return None
if spec.value_type == "boolean":
if wire_type != "boolean" or payload not in ("true", "false"):
raise CommandProtocolError(f"field '{field_name}' must use boolean:true or boolean:false")
return payload == "true"
if wire_type != "string":
expected = "string or null" if spec.nullable else "string"
raise CommandProtocolError(f"field '{field_name}' must use {expected} encoding")
if len(payload) % 2 != 0 or re.fullmatch(r"[0-9a-f]*", payload) is None:
raise CommandProtocolError(f"field '{field_name}' has invalid lowercase hexadecimal data")
try:
value = bytes.fromhex(payload).decode("utf-8")
except UnicodeDecodeError as exc:
raise CommandProtocolError(f"field '{field_name}' has invalid UTF-8 data") from exc
if "\0" in value:
raise CommandProtocolError(f"field '{field_name}' cannot contain NUL")
return value
def _metadata_value(line: str, name: str) -> str:
prefix = f"{name}="
if not line.startswith(prefix):
raise CommandProtocolError(f"expected {name} metadata")
return line[len(prefix) :]
def _parse_record_count(record_count_text: str) -> int:
if re.fullmatch(r"0|[1-9][0-9]*", record_count_text) is None:
raise CommandProtocolError("record_count must be a canonical non-negative integer")
if len(record_count_text) > len(str(MAX_RECORD_COUNT)):
raise CommandProtocolError(f"record_count exceeds protocol maximum of {MAX_RECORD_COUNT}")
record_count = int(record_count_text)
if record_count > MAX_RECORD_COUNT:
raise CommandProtocolError(f"record_count exceeds protocol maximum of {MAX_RECORD_COUNT}")
return record_count
def _parse_field_line(line: str) -> tuple[str, str, str]:
key, separator, payload = line.partition("=")
if not separator or not key.startswith("field."):
raise CommandProtocolError("expected field.<name>:<type>=<payload>")
descriptor = key.removeprefix("field.")
field_name, separator, wire_type = descriptor.partition(":")
if not separator or not field_name or not wire_type:
raise CommandProtocolError("expected field.<name>:<type>=<payload>")
return field_name, wire_type, payload