diff --git a/docker/Dockerfile.server b/docker/Dockerfile.server index ffc1cf5..0ba5c65 100644 --- a/docker/Dockerfile.server +++ b/docker/Dockerfile.server @@ -35,6 +35,13 @@ LABEL org.opencontainers.image.source="https://github.com/CortexLM/hypertrain" \ io.cortex.challenge.slug="hypertrain" \ io.cortex.challenge.contract="1" RUN install -d -o 65532 -g 65532 -m 0750 /data +# Role dispatch (HYPERTRAIN_ROLE_DISPATCH) drives owned GPU hosts over ssh; OpenSSH refuses a uid +# without a passwd entry. No key or known_hosts enters the image: both live in the run directory. +RUN apt-get update \ + && apt-get install -y --no-install-recommends openssh-client \ + && rm -rf /var/lib/apt/lists/* \ + && echo 'challenge:x:65532:65532::/tmp:/usr/sbin/nologin' >> /etc/passwd \ + && echo 'challenge:x:65532:' >> /etc/group COPY --from=build /opt/venv /opt/venv ENV PATH=/opt/venv/bin:$PATH \ HOME=/tmp \ diff --git a/experiments/gpu_network_v2/orchestrate.py b/experiments/gpu_network_v2/orchestrate.py index b7f30fd..c1d52b1 100644 --- a/experiments/gpu_network_v2/orchestrate.py +++ b/experiments/gpu_network_v2/orchestrate.py @@ -3,18 +3,23 @@ from __future__ import annotations import argparse +import contextlib import importlib.util import io import json import math import os import shlex +import shutil +import signal +import struct import subprocess import sys import tarfile import threading import time import urllib.parse +from collections.abc import Iterator from concurrent.futures import ThreadPoolExecutor from decimal import Decimal from pathlib import Path @@ -32,6 +37,7 @@ sha256, ) from hypertrain.gpu_ops.launcher import Orchestrator, Reject +from hypertrain.gpu_ops.network_qualification import LEGACY_TIMING, lifecycle_timing from hypertrain.gpu_ops.provider import Provider, list_rows, loopback, read_api_key from hypertrain.gpu_ops.remote import Ssh @@ -96,6 +102,19 @@ def timeout(self, value: int) -> None: self._limit = value +def staged_match(ssh: Any, remote: str, local: Path, logs: Path) -> bool: + """True only for an exact size+sha256 read of the remote copy; any failure means upload.""" + q = shlex.quote(remote) + proc = ssh.run(f"stat -c %s {q} && sha256sum {q}", "staged-" + Path(remote).name, logs) + out = proc.stdout.split() + return ( + proc.returncode == 0 + and len(out) >= 2 + and out[0] == str(local.stat().st_size) + and out[1] == fsha(local) + ) + + def deadline_ssh(orch: Orchestrator, role: str, cutoff: float) -> DeadlineSsh: ssh = orch.ssh(role) bounded = DeadlineSsh(**vars(ssh)) @@ -157,25 +176,50 @@ def refresh_readonly(provider: Provider) -> Record: def artifact_budget(profile: dict[str, Any]) -> Record: """Shape/count bounds from the actual AdamW checkpoint and trace paths; no allocation.""" + from hypertrain.protocol.messages import ODSpec + from hypertrain.trainer.config import ModelConfig + from hypertrain.trainer.model import param_shapes + m, inn, lay, a = (profile[k] for k in ("model", "inner", "layout", "artifacts")) - d, e, ff, layers, vocab = (m[k] for k in ("d_model", "n_experts", "d_ff", "n_layers", "vocab")) - params = 2 * vocab * d + d + layers * (2 * d + 4 * d * d + d * e + 3 * e * d * ff) + layers = m["n_layers"] + # Exact tensor geometry of the profile's architecture (decoder MoE or od-encoder), not a + # per-arch closed form; the declared param_count must agree with it. + shapes = param_shapes( + ModelConfig( + **{k: m[k] for k in ("n_layers", "d_model", "n_heads", "n_kv_heads", "d_ff", "vocab")}, + seq_len=m["seq_len"], + n_experts=m["n_experts"], + top_k_experts=m["top_k_experts"], + arch=m["arch"], + od=ODSpec.model_validate(m["od"]) if m.get("od") is not None else None, + ) + ) + params = sum(math.prod(s) for s in shapes.values()) if ( params != m["param_count"] or inn["opt"] != "adamw" or inn["H"] % inn["J"] - or lay != {"pp": 1, "n_gpus": 2, "dp_size": 1, "ep_size": 2, "zero1": True} + or lay["pp"] != 1 + or lay["n_gpus"] < 1 + or lay["dp_size"] * lay["ep_size"] != lay["n_gpus"] or m["n_heads"] != m["n_kv_heads"] - or e != 4 ): raise Reject("unsupported_artifact_geometry") - tensors = 3 + 10 * layers + tensors = len(shapes) leaves = inn["H"] // inn["J"] + 1 header = a["safetensors_header_bytes_per_file"] state = 12 * params + header ef = 4 * params + header - # SparseLoCo framing plus ceil(.01*n) indices/codes/levels for each tensor. - delta = math.ceil(params * 0.01) * 5 + tensors * 256 + # Exact compress() framing per tensor (name/shape/length prefixes bounded by 256 bytes) + # for the codec the outer optimizer selects; dense-int8 for nesterov, sparseloco otherwise. + out = profile["outer"] + if out["opt"] == "sparseloco": + frac = struct.unpack(" Record: outputs = lay["n_gpus"] * ((leaves + 1) * state + ef + delta + trace) logs = max(a["logs_bytes_per_execution"], 2 * math.ceil(max(trace, state) / 1024) * 1024) execution = inputs + outputs + logs + a["metadata_bytes_per_execution"] - retained = execution * profile["schedule"]["executions_per_host"] + policy = a.get("remote_retention", "all") + if policy == "all": + resident = profile["schedule"]["executions_per_host"] + elif ( + policy == "prune_rescued" + and a["retain_all_checkpoints"] is True + and a["path_preserving_rescue"] is True + ): + # Both qualification rounds stay remote until promotion; afterwards only the current + # workload execution, since every finished one is pruned after verified local custody. + resident = 2 + else: + raise Reject("unsupported_remote_retention_policy") + retained = execution * resident # Tar record rounding + path headers + terminating records for every retained file. files_per_execution = lay["n_gpus"] * (leaves + 5) + 14 - tar_overhead = ( - files_per_execution * profile["schedule"]["executions_per_host"] * 2048 + 1048576 + 10240 - ) + tar_overhead = files_per_execution * resident * 2048 + 1048576 + 10240 rescue = retained + tar_overhead + per_host = profile["schedule"]["executions_per_host"] + if policy == "all": + remote_peak = retained + rescue # rescue.tar is written next to the outputs + local = profile["schedule"]["hosts"] * (per_host + 1) * (rescue + execution) + else: + # The archive streams over SSH straight into local custody: no remote copy. Locally the + # qualification tar stays, and each workload keeps its tar plus one extraction (the + # canonical copy hardlinks the verified extraction; cross-rank dedup is not counted). + remote_peak = retained + workload_tar = execution + files_per_execution * 2048 + 1048576 + 10240 + local = profile["schedule"]["hosts"] * ( + rescue + (per_host - 2) * (workload_tar + execution) + ) return { + "remote_retention": policy, + "local_custody_bytes_required": local, "param_count": params, "tensor_count": tensors, "state_bytes": state, @@ -204,7 +274,7 @@ def artifact_budget(profile: dict[str, Any]) -> Record: "execution_bytes": execution, "retained_bytes_per_host": retained, "rescue_tar_bytes_per_host": rescue, - "disk_artifacts_peak_bytes": retained + rescue, + "disk_artifacts_peak_bytes": remote_peak, "artifact_transfer_bytes_per_host": retained + rescue, "rescue_min_bytes_per_second": math.ceil(rescue / a["rescue_transfer_seconds"]), } @@ -236,7 +306,15 @@ def readiness_budget(profile: dict[str, Any], evidence: dict[str, Any]) -> Recor raise Reject("measured_artifact_transport_or_image_budget_missing") if bounds["disk_artifacts_peak_bytes"] + image + 2 * stage > a["disk_bytes"]: raise Reject("artifact_disk_budget_exceeded") - if a["rescue_transfer_seconds"] + a["delete_absence_seconds"] > 300: + if bounds["remote_retention"] == "prune_rescued" and ( + type(a.get("local_custody_bytes")) is not int + or a["local_custody_bytes"] < bounds["local_custody_bytes_required"] + ): + raise Reject("local_custody_disk_budget_exceeded") + if ( + a["rescue_transfer_seconds"] + a["delete_absence_seconds"] + > lifecycle_timing(profile)["teardown"] + ): raise Reject("rescue_delete_budget_exceeded") return { **bounds, @@ -286,15 +364,247 @@ def staged_command( ) +LEGACY_ADMISSION = { + "ceiling_usd": "50", + "ceiling": Decimal("50"), + "reserve": Decimal("5"), + "hardware_cap": Decimal("5"), + "ancillary": Decimal("1"), + "disk_gb": 80, + "gpu_name": "RTX 5090", + "gpus": 2, + "hourly": Decimal("2"), + "cpu": 8, + "ram_mib": 32768, +} + + +def admission_limits(profile: dict[str, Any]) -> Record: + """Legacy "all" keeps its exact USD50/80GB/2x5090 review. prune_rescued takes every limit from + the reviewed profile admission section; the signed authorization must carry the same ceiling. + Hardware is a whole-lifecycle total (price x hours + storage + egress), so its cap is the + ceiling less the unchanged cleanup reserve.""" + if profile.get("artifacts", {}).get("remote_retention", "all") == "all": + return dict(LEGACY_ADMISSION) + a = profile["admission"] + ceiling, reserve = budget.dec(a["ceiling_usd"]), budget.dec(a["cleanup_reserve_usd"]) + hourly, ancillary = budget.dec(a["max_host_hourly_usd"]), budget.dec(a["max_ancillary_usd"]) + ints = (a["gpus_per_host"], a["min_cpu_cores"], a["min_host_ram_gib"], a["min_disk_gib"]) + if ceiling is None or reserve is None or hourly is None or ancillary is None: + raise Reject("admission_limits_invalid") + if ( + type(a["ceiling_usd"]) is not str + or reserve != LEGACY_ADMISSION["reserve"] + or not ceiling > reserve + or not hourly > 0 + or any(type(v) is not int or v <= 0 for v in ints) + or a["gpus_per_host"] != profile["layout"]["n_gpus"] + # Requested minimum disk may not exceed the measured disk the artifact budget used. + or a["min_disk_gib"] * 2**30 > profile["artifacts"]["disk_bytes"] + ): + raise Reject("admission_limits_invalid") + return { + "ceiling_usd": a["ceiling_usd"], + "ceiling": ceiling, + "reserve": reserve, + "hardware_cap": ceiling - reserve, + "ancillary": ancillary, + # Provider disk units are GiB: task-4 hosts created with disk 400 report disk_space 400. + "disk_gb": a["min_disk_gib"], + "gpu_name": a["gpu_name"], + "gpus": a["gpus_per_host"], + "hourly": hourly, + "cpu": a["min_cpu_cores"], + "ram_mib": a["min_host_ram_gib"] * 1024, + } + + +def owned_instances(snapshot: Record, cfg: Record) -> list[Record]: + """Full fresh inventory stays in the signed snapshot; only rows on the signed config's + machines are ours. Foreign or protected instances are preserved and never acted on.""" + rows = snapshot.get("instances") if isinstance(snapshot, dict) else None + if not isinstance(rows, list) or not all(isinstance(i, dict) for i in rows): + raise Reject("adoption_ownership_mismatch") + machines = {h["machine_id"] for h in cfg["hosts"]} + return [i for i in rows if i.get("machine_id") in machines] + + +def staging_cap(profile: dict[str, Any]) -> int: + """Legacy keeps its exact 1GB declaration cap. prune_rescued declares the exact staged bytes + (checked against the real files per role), bounded by the measured host disk.""" + if lifecycle_timing(profile) == LEGACY_TIMING: + return 1_000_000_000 + return int(profile["artifacts"]["disk_bytes"]) + + +def file_bytes(paths: list[Path]) -> int: + return sum(p.stat().st_size for p in paths) + + +def registry_layers(registry: Record, platform: Path | None) -> int: + """Total image bytes. An OCI index (the digest the profile pins) resolves to its single + linux/amd64 manifest: that file's raw sha256 must be the digest the index lists for it.""" + if "manifests" in registry: + rows = [ + m + for m in registry["manifests"] + if isinstance(m, dict) and m.get("platform") == {"architecture": "amd64", "os": "linux"} + ] + if ( + type(registry.get("schemaVersion")) is not int + or registry["schemaVersion"] != 2 + or platform is None + or len(rows) != 1 + or rows[0].get("digest") != "sha256:" + fsha(platform) + ): + raise Reject("reviewed_registry_platform") + registry = json.loads(platform.read_bytes()) + layers = registry.get("layers") + config = registry.get("config") + if ( + type(registry.get("schemaVersion")) is not int + or registry["schemaVersion"] != 2 + or not isinstance(layers, list) + or not layers + or not all( + isinstance(d, dict) and type(d.get("size")) is int and d["size"] >= 0 for d in layers + ) + or not isinstance(config, dict) + or type(config.get("size")) is not int + or config["size"] < 0 + ): + raise Reject("reviewed_complete_image_layers_required") + total = sum(d["size"] for d in layers) + config["size"] + if total > 20_000_000_000: + raise Reject("reviewed_complete_image_layers_required") + return total + + +def qualification_start(j: Journal) -> float | None: + """Adopted hosts qualify from the fresh signed admission; created hosts from first create.""" + adopted = j.last("adopted") + if adopted is not None: + return float(adopted["admitted_unix"]) + intents = j.all("create_intent") + return min(float(r["unix"]) for r in intents) if intents else None + + +def adoption_receipt( + record: Record, + cfg: Record, + profile: dict[str, Any], + ownership: Record, + provider_rows: list[Record], + snapshot: Record, + now: float, +) -> Record: + """Retained task-4 hosts: import the provider journal's own create times, never a new create. + Exact two owned instances, signed fresh admission A, and A + Q inside the original horizon.""" + timing, limits = lifecycle_timing(profile), admission_limits(profile) + if timing == LEGACY_TIMING: + raise Reject("adoption_requires_reviewed_finite_profile") + api = ownership.get("api_lookup") if isinstance(ownership, dict) else None + if ( + not isinstance(api, dict) + or not isinstance(provider_rows, list) + or not all(isinstance(r, dict) for r in provider_rows) + or not all(isinstance(v, dict) for k, v in api.items() if k != "endpoint") + ): + raise Reject("adoption_ownership_mismatch") + if not isinstance(record, dict) or type(record.get("cutoff_unix")) is not int: + raise Reject("adoption_clock_invalid") + lookup = {k: v for k, v in api.items() if k != "endpoint"} + intents = {r.get("role"): r for r in provider_rows if r.get("event") == "create_intent"} + created = { + r.get("instance_id") + for r in provider_rows + if r.get("event") == "create_response" and r.get("success") is True + } + roles = {h["machine_id"]: h["role"] for h in cfg["hosts"]} + live = {i.get("id"): i for i in owned_instances(snapshot, cfg)} + adopted: Record = {} + for source_role, row in lookup.items(): + intent = intents.get(source_role) + role = roles.get(row.get("machine_id")) + current = live.get(row.get("instance_id")) + if ( + intent is None + or role is None + or role in adopted + or type(row.get("instance_id")) is not int + or current is None + or current.get("machine_id") != row["machine_id"] + or current.get("actual_status") != "running" + or current.get("num_gpus") != limits["gpus"] + or current.get("gpu_name") != limits["gpu_name"] + or not current.get("disk_space", 0) >= limits["disk_gb"] + or type(intent.get("unix")) is not float + or not intent.get("label") + ): + raise Reject("adoption_ownership_mismatch") + adopted[role] = { + "instance_id": row["instance_id"], + "machine_id": row["machine_id"], + "label": intent["label"], + "create_unix": intent["unix"], + } + ids = {r["instance_id"] for r in adopted.values()} + if len(adopted) != 2 or set(intents) != set(lookup) or created != ids or set(live) != ids: + raise Reject("adoption_ownership_mismatch") + first = min(r["create_unix"] for r in adopted.values()) + end = first + timing["horizon"] + admitted = record.get("admitted_unix") + if ( + type(admitted) is not int + or not first <= admitted <= now + or now - admitted > 300 + or admitted + timing["qualification"] + profile["schedule"]["hard_margin_seconds"] > end + or type(record["cutoff_unix"]) is not int + or record["cutoff_unix"] > end + or admitted + timing["qualification"] + profile["schedule"]["hard_margin_seconds"] + > record["cutoff_unix"] + or now + timing["teardown"] >= end + ): + raise Reject("adoption_clock_invalid") + return { + "first_billable_unix": first, + "admitted_unix": admitted, + "horizon_end_unix": end, + "cutoff_unix": record["cutoff_unix"], + "roles": adopted, + } + + +def execution_quota(profile: dict[str, Any]) -> Record: + """Execution ceilings from the reviewed profile schedule; disk fit is checked by the budget.""" + s = profile["schedule"] + hosts, per_role, total = s["hosts"], s["executions_per_host"], s["executions_total"] + if ( + any(type(v) is not int for v in (hosts, per_role, total)) + or hosts < 1 + or per_role <= 2 # two qualification rounds plus at least one workload execution + or total != hosts * per_role + ): + raise Reject("execution_reservation_mismatch") + qualification = 2 * hosts + return { + "total": total, + "per_role": per_role, + "qualification": qualification, + "remaining": total - qualification, + } + + def runtime_admission(profile: dict[str, Any], evidence: dict[str, Any]) -> int: """Exact-profile evidence required before CREATE, not a guessed toy throughput.""" from hypertrain.protocol.hashing import sha256_hex from hypertrain.protocol.jcs import canonicalize bound = profile["execution_and_transfer_bound_seconds"] + timing = lifecycle_timing(profile) if ( type(bound) is not int - or not 0 < bound <= 40 + or not 0 < bound <= timing["execution_cap"] or profile["runtime_evidence_sha256"] != sha256_hex(canonicalize(evidence)) ): raise Reject("missing_or_unqualified_exact_profile_runtime") @@ -322,7 +632,13 @@ def runtime_admission(profile: dict[str, Any], evidence: dict[str, Any]) -> int: ): raise Reject("real_cuda_staged_reference_evidence_missing") s = profile["schedule"] + quota = execution_quota(profile) if ( + s["shadow_executions"] + s["live_executions"] + s["probes"] + s["fault_trace_reserve"] + > quota["total"] + ): + raise Reject("execution_reservation_mismatch") + if profile["artifacts"].get("remote_retention", "all") == "all" and ( s["executions_total"] != 126 or s["executions_per_host"] != 63 or s["shadow_executions"] != 96 @@ -330,12 +646,18 @@ def runtime_admission(profile: dict[str, Any], evidence: dict[str, Any]) -> int: or s["probes"] != 2 or s["fault_trace_reserve"] != 12 ): - raise Reject("execution_reservation_mismatch") + raise Reject("execution_reservation_mismatch") # legacy decoder schedule stays exact total = ( - s["staging_seconds"] + 63 * bound + s["beacon_service_seconds"] + s["rescue_delete_seconds"] + s["staging_seconds"] + + quota["per_role"] * bound + + s["beacon_service_seconds"] + + s["rescue_delete_seconds"] ) - if total > 3300 or total + s["hard_margin_seconds"] > 3600: - raise Reject("one_hour_schedule_exceeded") + if timing == LEGACY_TIMING: + if total > 3300 or total + s["hard_margin_seconds"] > 3600: + raise Reject("one_hour_schedule_exceeded") + elif total + s["hard_margin_seconds"] > timing["horizon"]: + raise Reject("lifecycle_schedule_exceeded") return total @@ -347,14 +669,19 @@ def admission_plan( offers: list[dict[str, Any]], *, qualification: bool = False, + priced_seconds: int | None = None, ) -> Record: - """Consume authenticated snapshots/quotes recorded by gpu_ops; malformed is never absent.""" + """Consume authenticated snapshots/quotes recorded by gpu_ops; malformed is never absent. + + priced_seconds (adopt only): signed cutoff - fresh admission of already-owned hosts. + """ if qualification: q = profile["microqualification"] bounds = artifact_budget(profile) + timing = lifecycle_timing(profile) if ( q["max_full_rounds_per_host"] != 2 - or q["qualification_deadline_seconds"] != 900 + or q["qualification_deadline_seconds"] != timing["qualification"] or sum( q[k] for k in ( @@ -364,7 +691,7 @@ def admission_plan( "failure_margin_seconds", ) ) - > 900 + > timing["qualification"] or evidence.get("root_verified_bounded_lifecycle") is not True or evidence.get("candidate_image_digest") != profile["runtime"]["image_digest"] or not profile["runtime"]["image_digest"] @@ -380,13 +707,15 @@ def admission_plan( transfer_gb = q["candidate_transfer_gb_per_host"] else: runtime_admission(profile, evidence) + timing = lifecycle_timing(profile) transfer_gb = readiness_budget(profile, evidence)["priced_transfer_gb_per_host"] + limits = admission_limits(profile) if ( authorization.get("experiment") != "hypertrain-network-v2" or authorization.get("currency") != "USD" - or authorization.get("ceiling_usd") != "50" + or authorization.get("ceiling_usd") != limits["ceiling_usd"] or authorization.get("max_instances") != 2 - or authorization.get("max_seconds_per_instance") != 3600 + or authorization.get("max_seconds_per_instance") != timing["horizon"] or not authorization.get("source_message") or not authorization.get("date") ): @@ -405,13 +734,13 @@ def admission_plan( for o in offers: hourly = budget.dec(o.get("dph_total")) if ( - o.get("gpu_name") != "RTX 5090" - or o.get("num_gpus") != 2 + o.get("gpu_name") != limits["gpu_name"] + or o.get("num_gpus") != limits["gpus"] or hourly is None - or hourly > 2 - or o.get("cpu_cores_effective", 0) < 8 - or o.get("cpu_ram", 0) < 32768 - or o.get("disk_space", 0) < 80 + or hourly > limits["hourly"] + or o.get("cpu_cores_effective", 0) < limits["cpu"] + or o.get("cpu_ram", 0) < limits["ram_mib"] + or o.get("disk_space", 0) < limits["disk_gb"] or type(o["verified_unix"]) is not int or not 0 <= time.time() - o["verified_unix"] <= 30 ): @@ -421,25 +750,42 @@ def admission_plan( available = Decimal(snapshot["available_credit_usd"]) - liability if min(prior, liability, available) < 0: raise Reject("invalid_accounting") + seconds = timing["horizon"] + if priced_seconds is not None: + if ( + not qualification + or type(priced_seconds) is not int + or not timing["qualification"] + profile["schedule"]["hard_margin_seconds"] + <= priced_seconds + <= timing["horizon"] + ): + raise Reject("adoption_priced_window_invalid") + seconds = priced_seconds plan = budget.evaluate( snapshot0_credit=available + prior, current_credit=available, offers=offers, - hard_deadline_seconds=3600, - disk_gb=80, + hard_deadline_seconds=seconds, + disk_gb=limits["disk_gb"], egress_gb=transfer_gb, - phase_cap=Decimal("50"), - reserve=Decimal("5"), + phase_cap=limits["ceiling"], + reserve=limits["reserve"], + global_cap=limits["ceiling"], ) hardware = Decimal(plan["hardware_worst_case_usd"]) - hourly_total = sum(Decimal(str(o["dph_total"])) for o in offers) - if not plan["admit"] or hardware > 5 or hardware - hourly_total > 1: + hours = Decimal(seconds) / Decimal(3600) + hourly_total = sum(Decimal(str(o["dph_total"])) for o in offers) * hours + if ( + not plan["admit"] + or hardware > limits["hardware_cap"] + or hardware - hourly_total > limits["ancillary"] + ): raise Reject("network_v2_cost_ceiling_exceeded") return { **plan, "admission_mode": "qualification" if qualification else "workload", - "rental_deadline_seconds": 3600, - "qualification_deadline_seconds": 900, + "rental_deadline_seconds": seconds, + "qualification_deadline_seconds": timing["qualification"], "long_workload_allowed": not qualification, } @@ -482,17 +828,21 @@ def reserve(self, role: str, name: str, phase: str, executions: int) -> Record: intents = self.journal.all("network_execution_intent") q = sum(r["executions"] for r in intents if r["phase"] == "qualification") used_role = sum(r["executions"] for r in intents if r["role"] == role) + quota = execution_quota(self.profile) if ( - used_role + executions > 63 - or sum(r["executions"] for r in intents) + executions > 126 + used_role + executions > quota["per_role"] + or sum(r["executions"] for r in intents) + executions > quota["total"] ): raise Reject("execution_reservation_exhausted") if phase == "qualification": - if q + executions > 4 or any( + if q + executions > quota["qualification"] or any( r["role"] == role and r["phase"] == "qualification" for r in intents ): raise Reject("qualification_reservation_exhausted") - elif q != 4 or self.journal.last("network_workload_promoted") is None: + elif ( + q != quota["qualification"] + or self.journal.last("network_workload_promoted") is None + ): raise Reject("workload_not_promoted") return self.journal.append( "network_execution_intent", @@ -510,23 +860,26 @@ def promote(self, evidence: dict[str, Any], budget_plan: Record) -> Record: raise Reject("network_cleanup_already_started") intents = self.journal.all("network_execution_intent", phase="qualification") done = self.journal.all("network_qualification_done") + quota = execution_quota(self.profile) if ( - len(intents) != 2 - or sum(r["executions"] for r in intents) != 4 + len(intents) != len(self.lifecycle.roles) + or sum(r["executions"] for r in intents) != quota["qualification"] or {r["role"] for r in intents} != set(self.lifecycle.roles) - or len(done) != 2 + or len(done) != len(self.lifecycle.roles) or {r["role"] for r in done} != set(self.lifecycle.roles) or evidence.get("root_reviewed_two_host_full_round_transfer") is not True or budget_plan.get("admit") is not True or budget_plan.get("long_workload_allowed") is not True - or time.time() + 61 * self.profile["execution_and_transfer_bound_seconds"] + 600 + or time.time() + + (quota["per_role"] - 2) * self.profile["execution_and_transfer_bound_seconds"] + + 600 > self.lifecycle.deadline() ): raise Reject("qualification_promotion_gate_failed") return self.journal.append( "network_workload_promoted", - remaining_total=122, - remaining_per_host=61, + remaining_total=quota["remaining"], + remaining_per_host=quota["per_role"] - 2, hard_deadline_unix=self.lifecycle.deadline(), budget_plan_sha256=sha256( json.dumps(budget_plan, sort_keys=True, separators=(",", ":")).encode() @@ -573,7 +926,8 @@ def stage_context( cutoff = min( cutoff, self.lifecycle.deadline(), - self.lifecycle.cfg["cleanup_deadline_unix"] - 300, + self.lifecycle.cfg["cleanup_deadline_unix"] + - lifecycle_timing(self.profile)["teardown"], ) ssh = deadline_ssh(self.lifecycle, role, cutoff) ssh.cancel = cancel @@ -619,7 +973,8 @@ def operation( spec.cutoff, job.deadline, self.lifecycle.deadline(), - self.lifecycle.cfg["cleanup_deadline_unix"] - 300, + self.lifecycle.cfg["cleanup_deadline_unix"] + - lifecycle_timing(self.profile)["teardown"], ) if ( spec.cutoff != cutoff @@ -687,6 +1042,12 @@ def operation( assert trusted is not None if fsha(ssh.known_hosts) != trusted["known_hosts_sha256"]: raise Reject("operation_known_hosts_custody_changed") + prune = self.profile["artifacts"].get("remote_retention", "all") == "prune_rescued" + if prune: + self.prune_rescued(role) + bounds = artifact_budget(self.profile) + if shutil.disk_usage(self.lifecycle.dir).free < 2 * bounds["execution_bytes"]: + raise Reject("local_custody_disk_exhausted") self.reserve(role, name, "workload", 1) self.journal.append( "network_operation_started", @@ -801,6 +1162,7 @@ def stop() -> None: destination = directory / name if destination.exists(): raise Reject("operation_result_destination_exists") + seen: dict[str, Path] = {} with tarfile.open(archive) as source: for member in source.getmembers(): prefix = "out/" + name + "/" @@ -811,8 +1173,7 @@ def stop() -> None: raise Reject("operation_custody_path") stream = source.extractfile(member) assert stream is not None - path.parent.mkdir(parents=True, exist_ok=True) - durable_write(path, stream.read()) + place_verified(path, stream.read(), seen, prune) result = json.loads((destination / "operation-result.json").read_bytes()) if ( result.get("status") != "CAPTURED_NOT_ACCEPTED" @@ -893,7 +1254,8 @@ def completion_custody( spec.cutoff, spec.job.deadline, self.lifecycle.deadline(), - self.lifecycle.cfg["cleanup_deadline_unix"] - 300, + self.lifecycle.cfg["cleanup_deadline_unix"] + - lifecycle_timing(self.profile)["teardown"], ) if ( spec.operation not in ("reference", "probe") @@ -1437,10 +1799,16 @@ def launch( with tempfile.TemporaryDirectory(prefix="canonical-", dir=directory) as temporary: staged = Path(temporary) / "published" staged.mkdir() + link = ( + self.profile["artifacts"].get("remote_retention", "all") == "prune_rescued" + ) for relative, digest in files.items(): path = staged / relative path.parent.mkdir(parents=True, exist_ok=True) - durable_write(path, (original / relative).read_bytes()) + if link and fsha(original / relative) == digest: + os.link(original / relative, path) + else: + durable_write(path, (original / relative).read_bytes()) if fsha(path) != digest: raise Reject("operation_canonical_copy_changed") validate_artifacts(job, staged) @@ -1530,11 +1898,11 @@ def run_qualification( if cutoff is None else deadline_ssh(self.lifecycle, role, cutoff) ) - intents = self.journal.all("create_intent") - first_billable = min(float(r["unix"]) for r in intents) if intents else None + first_billable = qualification_start(self.journal) if first_billable is None: raise Reject("qualification_first_billable_receipt_missing") - ssh.timeout = int(min(30, first_billable + 900 - time.time() - 420)) + timing = lifecycle_timing(self.profile) + ssh.timeout = int(min(30, first_billable + timing["qualification"] - time.time() - 420)) if ssh.timeout <= 0: raise Reject("qualification_deadline_margin") config_file = self.lifecycle.dir / f"qualification-config-{role}.json" @@ -1569,9 +1937,9 @@ def run_qualification( ) ssh.timeout = int( min( - 180, + timing["qual_probe"], self.lifecycle.deadline() - time.time() - 420, - cfg.admitted_unix + 900 - time.time() - 420, + cfg.admitted_unix + timing["qualification"] - time.time() - 420, ) ) if ssh.timeout <= 0: @@ -1693,21 +2061,23 @@ def run_staged( ): raise Reject("lifecycle_receipts_not_ready") used = sum(r["executions"] for r in self.journal.all("network_execution_intent", role=role)) - if used + trace_equivalents > 63: + if used + trace_equivalents > execution_quota(self.profile)["per_role"]: raise Reject("execution_reservation_exhausted") bound = ( self.profile["microqualification"]["probe_and_transfer_seconds"] // 2 if qualification else self.profile["execution_and_transfer_bound_seconds"] ) - if type(bound) is not int or bound > (180 if qualification else 40): + timing = lifecycle_timing(self.profile) + cap = timing["qual_probe"] if qualification else timing["execution_cap"] + if type(bound) is not int or bound > cap: raise Reject("runtime_unqualified") if qualification and ( trace_equivalents != 1 or len(self.journal.all("network_qualification_started", role=role)) >= 2 ): raise Reject("microqualification_round_limit") - if qualification and time.time() + bound + 420 > admitted["unix"] + 900: + if qualification and time.time() + bound + 420 > admitted["unix"] + timing["qualification"]: raise Reject("microqualification_deadline_margin") remaining = self.lifecycle.deadline() - time.time() if remaining < trace_equivalents * bound + 600: @@ -1897,7 +2267,7 @@ def data(name: str) -> Record: cfg = engine.Qualification.model_validate_json(cfg_path.read_bytes()) if engine.check_inputs(cfg).manifest != manifest or cfg.sources != data("sources"): raise Reject("signed_d2_qualification_run_differs") - engine.compare(list(results)) + engine.compare(list(results), manifest) sources, bundle_bytes = effective_bundle( engine, Path(evidence["tree"]), files["source_profile"] ) @@ -2055,6 +2425,75 @@ def host(role: str) -> list[Record]: raise Reject("network_rescue_incomplete:" + ",".join(errors)) return result + def prune_rescued(self, role: str) -> list[Record]: + """Opt-in: free remote out/ only once its exact bytes are in verified local custody. + + Eligible: completed operations (network_operation_done, i.e. extracted, bound and + journal-published) and qualification rounds after promotion. Never seed, sources or an + operation without a done record; every rank must appear in the hashed rescue manifest. + """ + if self.profile["artifacts"].get("remote_retention", "all") != "prune_rescued": + raise Reject("remote_prune_policy_disabled") + pruned = {r["name"] for r in self.journal.all("network_remote_pruned", role=role)} + names = [r["name"] for r in self.journal.all("network_operation_done", role=role)] + if self.journal.last("network_workload_promoted") is not None: + names += [r["name"] for r in self.journal.all("network_qualification_done", role=role)] + rescued = self.journal.all("network_rescued", role=role) + ranks = range(self.profile["layout"]["n_gpus"]) + out = [] + for name in dict.fromkeys(names): + if name in pruned: + continue + if name == "seed" or not name.replace("-", "").isalnum(): + raise Reject("remote_prune_name_rejected") + done = self.journal.last("network_operation_done", role=role, name=name) + qual = self.journal.last("network_qualification_done", role=role, name=name) + + def bound(r: Record, done: Record | None = done, qual: Record | None = qual) -> bool: + if done is not None: + return bool(r["tar_sha256"] == done["tar_sha256"]) + return bool( + qual is not None + and sha256(json.dumps(r["files"], sort_keys=True).encode()) + == qual["result"]["artifact_manifest_sha256"] + ) + + custody = next( + ( + r + for r in reversed(rescued) + if bound(r) and any(f.startswith(name + "/") for f in r["files"]) + ), + None, + ) + if custody is None: + raise Reject("remote_prune_without_rescued_custody") + archive = Path(custody["tar"]) + if archive.is_symlink() or not archive.is_file(): + raise Reject("remote_prune_local_custody_missing") + if fsha(archive) != custody["tar_sha256"]: + raise Reject("remote_prune_local_custody_changed") + files = verify_export( + archive, artifact_budget(self.profile)["rescue_tar_bytes_per_host"] + ) + mine = [f for f in files if f.startswith(name + "/")] + if files != custody["files"] or any( + not any(f"rank-{r}" in Path(f).parts for f in mine) for r in ranks + ): + raise Reject("remote_prune_rank_custody_incomplete") + target = self.lifecycle.remote_root(role) + "/out/" + name + ssh = self.lifecycle.ssh(role) + if ssh.run( + f"rm -rf -- {shlex.quote(target)}", "prune-" + name, self.lifecycle.dir / "logs" + ).returncode: + raise Reject("remote_prune_failed") + out.append( + self.journal.append( + "network_remote_pruned", role=role, name=name, tar_sha256=custody["tar_sha256"] + ) + ) + return out + def rescue(self, role: str, fresh: bool = False) -> Record: """Complete path-preserving, byte/hash checked custody; failure never becomes PASS.""" generation = sha256( @@ -2097,25 +2536,50 @@ def remaining() -> None: remaining() python = self.lifecycle.cfg.get("remote_python", self.profile["runtime"]["remote_python"]) - cmd = ( - f"cd {shlex.quote(root)} && env PYTHONPATH=src {shlex.quote(python)} " - "experiments/gpu_network_v2/orchestrate.py export out rescue.tar " - "experiments/gpu_network_v2/profile.json" - ) - if ssh.run(cmd, "network-export", self.lifecycle.dir / "logs").returncode: - raise Reject("network_export_failed") - remaining() - expected = ssh.remote_sha256(f"{root}/rescue.tar", self.lifecycle.dir / "logs") destination = self.lifecycle.dir / "rescue" / role destination.mkdir(parents=True, exist_ok=True) archive = destination / f"network-rescue-{generation}-{time.time_ns()}.tar" - remaining() - actual = ssh.get(f"{root}/rescue.tar", archive) - if actual != expected: - raise Reject("network_rescue_sha_mismatch") - manifest = verify_export( - archive, artifact_budget(self.profile)["rescue_tar_bytes_per_host"] - ) + bound = artifact_budget(self.profile)["rescue_tar_bytes_per_host"] + if self.profile["artifacts"].get("remote_retention", "all") == "prune_rescued": + # Stream the tar over the pinned SSH channel into local custody; no remote archive. + cmd = ( + f"cd {shlex.quote(root)} && env PYTHONPATH=src {shlex.quote(python)} " + "experiments/gpu_network_v2/orchestrate.py export out - " + "experiments/gpu_network_v2/profile.json" + ) + log = self.lifecycle.dir / "logs" / f"{role}-network-export-stream.log" + try: + stream_bounded(ssh_argv(ssh, cmd), archive, log, bound, ssh.timeout) + remaining() + try: + manifest = verify_export(archive, bound) + except (tarfile.TarError, KeyError, ValueError) as exc: + raise Reject("network_rescue_archive_invalid") from exc + # Remote's own manifest digest (stderr) must equal the verified local manifest. + reported = log.read_text().split() + if not reported or reported[-1] != sha256( + json.dumps(manifest, sort_keys=True).encode() + ): + raise Reject("network_rescue_sha_mismatch") + except BaseException: + archive.unlink(missing_ok=True) # a rejected stream never occupies custody disk + raise + actual = fsha(archive) + else: + cmd = ( + f"cd {shlex.quote(root)} && env PYTHONPATH=src {shlex.quote(python)} " + "experiments/gpu_network_v2/orchestrate.py export out rescue.tar " + "experiments/gpu_network_v2/profile.json" + ) + if ssh.run(cmd, "network-export", self.lifecycle.dir / "logs").returncode: + raise Reject("network_export_failed") + remaining() + expected = ssh.remote_sha256(f"{root}/rescue.tar", self.lifecycle.dir / "logs") + remaining() + actual = ssh.get(f"{root}/rescue.tar", archive) + if actual != expected: + raise Reject("network_rescue_sha_mismatch") + manifest = verify_export(archive, bound) remaining() if generation != sha256( json.dumps( @@ -2164,12 +2628,21 @@ def export_artifacts(directory: Path, archive: Path, profile: dict[str, Any]) -> if sum(v["bytes"] for v in files.values()) > bounds["retained_bytes_per_host"]: raise Reject("actual_artifact_budget_exceeded") metadata = json.dumps(files, sort_keys=True).encode() - with tarfile.open(archive, "w") as out: + streaming = str(archive) == "-" + with ( + tarfile.open(fileobj=sys.stdout.buffer, mode="w|") + if streaming + else tarfile.open(archive, "w") + ) as out: for name in files: out.add(directory / name, arcname="out/" + name, recursive=False) info = tarfile.TarInfo("manifest.json") info.size, info.mode, info.mtime = len(metadata), 0o600, 0 out.addfile(info, io.BytesIO(metadata)) + if streaming: + # Nothing archived remotely; the caller re-reads this digest over a separate channel. + sys.stdout.buffer.flush() + return {"bytes": None, "manifest_sha256": sha256_hex(metadata)} if archive.stat().st_size > bounds["rescue_tar_bytes_per_host"]: raise Reject("actual_rescue_archive_budget_exceeded") with archive.open("rb") as file: @@ -2177,6 +2650,63 @@ def export_artifacts(directory: Path, archive: Path, profile: dict[str, Any]) -> return {"bytes": archive.stat().st_size, "manifest_sha256": sha256_hex(metadata)} +def ssh_argv(ssh: Any, command: str) -> list[str]: + """Same pinned options as Ssh.run, for a binary stream that must not be held in memory.""" + return ["ssh", *ssh._opts(), "-p", str(ssh.port), f"{ssh.user}@{ssh.host}", command] + + +def stream_bounded(argv: list[str], archive: Path, log: Path, bound: int, timeout: float) -> None: + """Copy the child's stdout into a new archive, killing its process group past `bound` bytes + or `timeout` seconds. stderr goes to a file, so it can never fill a pipe and block.""" + with archive.open("xb") as sink, log.open("wb") as err: + proc = subprocess.Popen( # noqa: S603 (argv list from the pinned Ssh) + argv, stdout=subprocess.PIPE, stderr=err, start_new_session=True + ) + + def kill() -> None: + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + + timer = threading.Timer(timeout, kill) + timer.start() + written, oversized, eof = 0, False, False + try: + assert proc.stdout is not None + while block := proc.stdout.read(1 << 20): + written += len(block) + if written > bound: + oversized = True + break + sink.write(block) + else: + eof = True + finally: + if not eof: # oversize or a local error (e.g. ENOSPC): never wait on a live child + kill() + proc.stdout.close() if proc.stdout else None + rc = proc.wait() + timer.cancel() # only after wait, so the deadline covers the whole stream + sink.flush() + os.fsync(sink.fileno()) + if oversized: + raise Reject("network_rescue_archive_budget_exceeded") + if rc: + raise Reject("network_export_failed") + + +def place_verified(path: Path, data: bytes, seen: dict[str, Path], dedup: bool) -> None: + """Write custody bytes; with dedup, an identical verified file is hardlinked instead.""" + digest = sha256(data) + path.parent.mkdir(parents=True, exist_ok=True) + if dedup and digest in seen and fsha(seen[digest]) == digest: + os.link(seen[digest], path) + else: + durable_write(path, data) + seen[digest] = path + + def verify_export(archive: Path, maximum_bytes: int) -> Record: import hashlib @@ -2211,6 +2741,98 @@ def verify_export(archive: Path, maximum_bytes: int) -> Record: return dict(files) +SEED_OBJECTS = ("start_state", "ef_in", "v0", "samples", "sample_proofs") +# Verified seed proofs of ONE launch invocation; None outside it (external CLI never reuses). +_SEED_GENESIS_SCOPE: set[tuple[str, ...]] | None = None + + +@contextlib.contextmanager +def seed_genesis_scope() -> Iterator[None]: + """One task-8 launch invocation; nothing survives it and nothing is persisted.""" + global _SEED_GENESIS_SCOPE + if _SEED_GENESIS_SCOPE is not None: + raise Reject("seed_genesis_scope_nested") + _SEED_GENESIS_SCOPE = set() + try: + yield + finally: + _SEED_GENESIS_SCOPE = None + + +def seed_genesis_key( + job_sha256: str, profile_sha256: str, hotkey: str, digests: dict[str, str] +) -> tuple[str, ...]: + return (job_sha256, profile_sha256, hotkey, *(digests[k] for k in SEED_OBJECTS)) + + +def seed_genesis( + job_path: Path, + job_sha256: str, + profile_path: Path, + profile_sha256: str, + hotkey: str, + objects: dict[str, Path], + digests: dict[str, str], +) -> None: + """Full carry genesis + public sample membership for one seed identity. + + Every input is rehashed from its bytes here before parsing; inside seed_genesis_scope() the + result is remembered only for these exact digests and this hotkey. + """ + job_bytes, profile_bytes = job_path.read_bytes(), profile_path.read_bytes() + blobs = {k: objects[k].read_bytes() for k in SEED_OBJECTS} + if ( + sha256(job_bytes) != job_sha256 + or sha256(profile_bytes) != profile_sha256 + or any(sha256(blobs[k]) != digests[k] for k in SEED_OBJECTS) + ): + raise Reject("reviewed_seed_hash") + from hypertrain.protocol.messages_v2 import IslandJobV1 + + job = IslandJobV1.model_validate_json(job_bytes) + seq_len = json.loads(profile_bytes)["model"]["seq_len"] + # Reuse complete deterministic carry genesis; never accept theta-only state. + from hypertrain.auditor.replay import AnchorCache, pack_state, unpack_state + + theta, state = unpack_state(blobs["start_state"]) + anchor = AnchorCache().genesis(job.manifest, hotkey, theta) + if ( + state is None + or pack_state(theta, state) != pack_state(anchor.theta, anchor.state) + or blobs["ef_in"] != pack_state(anchor.ef) + or blobs["v0"] != pack_state({}) + ): + raise Reject("reviewed_seed_full_genesis") + from hypertrain.protocol.hashing import MerkleTree + + samples = blobs["samples"] + proofs = json.loads(blobs["sample_proofs"]) + dataset = job.manifest.training.dataset + sample_bytes = { + "u16[seq_len+1] token ids": 2, + "u32[seq_len+1] token ids": 4, + }.get(dataset.sample_format) + width = (sample_bytes or 0) * (seq_len + 1) + if ( + sample_bytes is None + or len(samples) != width * len(job.sample_ids) + or len(proofs) != len(job.sample_ids) + or any( + not MerkleTree.verify( + samples[index * width : (index + 1) * width], + sample, + [bytes.fromhex(p) for p in proof], + bytes.fromhex(dataset.merkle_root), + dataset.n_samples, + ) + for index, (sample, proof) in enumerate(zip(job.sample_ids, proofs, strict=True)) + ) + ): + raise Reject("reviewed_public_sample_membership") + if _SEED_GENESIS_SCOPE is not None: + _SEED_GENESIS_SCOPE.add(seed_genesis_key(job_sha256, profile_sha256, hotkey, digests)) + + def reviewed_launch(record_path: Path, owner: str, beacon: int, action: str) -> Record: """Local preflight only. Existing signed Receipt binds every supplied artifact.""" from hypertrain.miner.island_launch import confined @@ -2279,8 +2901,8 @@ def data(name: str) -> Any: or {h["role"] for h in cfg["hosts"]} != set(roles) or len(cfg["hosts"]) != 2 or cfg["cleanup_contract"] != "network-v2" - or cfg["disk_gb"] != 80 - or cfg["hard_deadline_seconds"] != 3600 + or cfg["disk_gb"] != admission_limits(profile)["disk_gb"] + or cfg["hard_deadline_seconds"] != lifecycle_timing(profile)["horizon"] or cfg["hard_grace_seconds"] != 0 or type(cfg["boot_timeout_seconds"]) is not int or not 0 < cfg["boot_timeout_seconds"] <= 300 @@ -2295,8 +2917,10 @@ def data(name: str) -> Any: or type(record["parent_pid"]) is not int or record["parent_pid"] <= 0 or type(record["cutoff_unix"]) is not int - or not time.time() < record["cutoff_unix"] <= time.time() + 3600 - or record["quota"] != {"total": 126, "per_role": 63, "qualification": 4, "remaining": 122} + or not time.time() + < record["cutoff_unix"] + <= time.time() + lifecycle_timing(profile)["horizon"] + or record["quota"] != execution_quota(profile) ): raise Reject("reviewed_layout_clock_or_config") artifact_budget(profile) @@ -2304,22 +2928,14 @@ def data(name: str) -> Any: profile["runtime"]["backend"] != "cuda" or profile["runtime"]["remote_python"] != "/opt/hypertrain/venv/bin/python" or type(record["staging_bytes"]) is not int - or not 0 < record["staging_bytes"] <= 1_000_000_000 + or not 0 < record["staging_bytes"] <= staging_cap(profile) or record["public_dataset_reviewed"] is not True or record["full_genesis_reviewed"] is not True ): raise Reject("reviewed_source_seed_authority_missing") - registry = data("registry") if "sha256:" + fsha(artifacts["registry"]) != profile["runtime"]["image_digest"]: raise Reject("reviewed_registry_digest") - layers = registry.get("layers") - if ( - registry.get("schemaVersion") != 2 - or not isinstance(layers, list) - or not layers - or sum(d["size"] for d in layers) + registry["config"]["size"] > 20_000_000_000 - ): - raise Reject("reviewed_complete_image_layers_required") + registry_layers(data("registry"), artifacts.get("registry_platform")) tree = Path(record["tree"]) if not tree.is_absolute(): raise Reject("reviewed_source_tree") @@ -2419,47 +3035,28 @@ def data(name: str) -> Any: or path != artifacts[inputs["objects"][key]] ): raise Reject("reviewed_seed_object_custody") + # Every committed file was rehashed from its bytes at the top of this call. + digests = {k: files[n]["sha256"] for k, n in inputs["objects"].items()} for key in ("start_state", "ef_in", "v0"): - if fsha(artifacts[inputs["objects"][key]]) != getattr(job, key + "_sha256"): + if digests[key] != getattr(job, key + "_sha256"): raise Reject("reviewed_seed_hash") - # Reuse complete deterministic carry genesis; never accept theta-only state. - from hypertrain.auditor.replay import AnchorCache, pack_state, unpack_state - - theta, state = unpack_state(artifacts[inputs["objects"]["start_state"]].read_bytes()) - anchor = AnchorCache().genesis(job.manifest, inputs["hotkey"], theta) - if ( - state is None - or pack_state(theta, state) != pack_state(anchor.theta, anchor.state) - or artifacts[inputs["objects"]["ef_in"]].read_bytes() != pack_state(anchor.ef) - or artifacts[inputs["objects"]["v0"]].read_bytes() != pack_state({}) - ): - raise Reject("reviewed_seed_full_genesis") - from hypertrain.protocol.hashing import MerkleTree - - samples = artifacts[inputs["objects"]["samples"]].read_bytes() - proofs = json.loads(artifacts[inputs["objects"]["sample_proofs"]].read_bytes()) - dataset = job.manifest.training.dataset - sample_bytes = { - "u16[seq_len+1] token ids": 2, - "u32[seq_len+1] token ids": 4, - }.get(dataset.sample_format) - width = (sample_bytes or 0) * (profile["model"]["seq_len"] + 1) - if ( - sample_bytes is None - or len(samples) != width * len(job.sample_ids) - or len(proofs) != len(job.sample_ids) - or any( - not MerkleTree.verify( - samples[index * width : (index + 1) * width], - sample, - [bytes.fromhex(p) for p in proof], - bytes.fromhex(dataset.merkle_root), - dataset.n_samples, - ) - for index, (sample, proof) in enumerate(zip(job.sample_ids, proofs, strict=True)) + job_digest, profile_digest = files[inputs["job"]]["sha256"], files["profile"]["sha256"] + proof = seed_genesis_key(job_digest, profile_digest, inputs["hotkey"], digests) + if _SEED_GENESIS_SCOPE is None or proof not in _SEED_GENESIS_SCOPE: + seed_genesis( + artifacts[inputs["job"]], + job_digest, + artifacts["profile"], + profile_digest, + inputs["hotkey"], + {k: artifacts[n] for k, n in inputs["objects"].items()}, + digests, ) + if staging_cap(profile) != 1_000_000_000 and record["staging_bytes"] != file_bytes( + [artifacts["bundle"], artifacts[inputs["job"]]] + + [artifacts[name] for name in inputs["objects"].values()] ): - raise Reject("reviewed_public_sample_membership") + raise Reject("reviewed_staging_bytes_not_actual") snapshot, offers = data("financial"), data("offers") if any(type(snapshot[k]) is not int for k in ("verified_unix", "account_id")) or any( type(o[k]) is not int @@ -2490,9 +3087,9 @@ def fresh(row: Record) -> bool: if ( authorization.get("experiment") != "hypertrain-network-v2" or authorization.get("currency") != "USD" - or authorization.get("ceiling_usd") != "50" + or authorization.get("ceiling_usd") != admission_limits(profile)["ceiling_usd"] or authorization.get("max_instances") != 2 - or authorization.get("max_seconds_per_instance") != 3600 + or authorization.get("max_seconds_per_instance") != lifecycle_timing(profile)["horizon"] or not authorization.get("source_message") or not authorization.get("date") or snapshot.get("authenticated") is not True @@ -2500,7 +3097,11 @@ def fresh(row: Record) -> bool: or snapshot.get("volumes") != [] or len(offers) != 2 or len({o["machine_id"] for o in offers}) != 2 - or any(o["num_gpus"] != 2 or o["gpu_name"] != "RTX 5090" for o in offers) + or any( + o["num_gpus"] != admission_limits(profile)["gpus"] + or o["gpu_name"] != admission_limits(profile)["gpu_name"] + for o in offers + ) ): raise Reject("reviewed_authorization_or_two_host_financial") if action == "continue": @@ -2511,14 +3112,18 @@ def fresh(row: Record) -> bool: current_credit=Decimal(snapshot["available_credit_usd"]) - Decimal(snapshot["active_liabilities_usd"]), offers=offers, - hard_deadline_seconds=3600, - disk_gb=80, + hard_deadline_seconds=lifecycle_timing(profile)["horizon"], + disk_gb=admission_limits(profile)["disk_gb"], egress_gb=30, - phase_cap=Decimal("50"), - reserve=Decimal("5"), + phase_cap=admission_limits(profile)["ceiling"], + reserve=admission_limits(profile)["reserve"], + global_cap=admission_limits(profile)["ceiling"], ) plan = {**cost, "long_workload_allowed": True} - if not cost["admit"] or Decimal(cost["hardware_worst_case_usd"]) > 5: + if ( + not cost["admit"] + or Decimal(cost["hardware_worst_case_usd"]) > admission_limits(profile)["hardware_cap"] + ): raise Reject("reviewed_continuation_budget") if "graph" not in artifacts or "graph_driver" not in artifacts or "jobs" in artifacts: raise Reject("signed_continuation_graph_required") @@ -2536,13 +3141,25 @@ def fresh(row: Record) -> bool: sys.modules[graph_spec.name] = continuation_driver graph_spec.loader.exec_module(continuation_driver) continuation_driver.ContinuationGraph.model_validate_json(artifacts["graph"].read_bytes()) - driver.compare([artifacts[inputs["qualification_result"]] for inputs in roles.values()]) + # Every role's job hashes to env.run_id above, so any job carries the one manifest. + driver.compare( + [artifacts[inputs["qualification_result"]] for inputs in roles.values()], job.manifest + ) else: planning_snapshot = dict(snapshot) - if action == "qualification": - if len(snapshot["instances"]) != 2: + if action in ("qualification", "adopt"): + if len(owned_instances(snapshot, cfg)) != 2: raise Reject("qualification_owned_inventory_required") planning_snapshot["instances"] = [] + elif action == "admit" and "admitted_unix" in record: + # Fresh finite window signed now; foreign rows are priced in active liabilities. + if ( + owned_instances(snapshot, cfg) + or type(record["admitted_unix"]) is not int + or not 0 <= time.time() - record["admitted_unix"] <= 300 + ): + raise Reject("fresh_window_clock_or_inventory_invalid") + planning_snapshot["instances"] = [] plan = admission_plan( profile, data("evidence"), @@ -2550,10 +3167,30 @@ def fresh(row: Record) -> bool: planning_snapshot, offers, qualification=True, + priced_seconds=( + # Adopted runs are billed only to the adopted cutoff (network_hard_end). + record["cutoff_unix"] - record["admitted_unix"] + if "admitted_unix" in record + else None + ), ) if plan != data("plan"): raise Reject("reviewed_plan_differs") + adoption = None + if action == "adopt": + if not {"ownership", "provider_journal"} <= set(artifacts): + raise Reject("adoption_custody_files_missing") + adoption = adoption_receipt( + record, + cfg, + profile, + data("ownership"), + [json.loads(line) for line in artifacts["provider_journal"].read_bytes().splitlines()], + snapshot, + time.time(), + ) return { + "adoption": adoption, "record": record, "files": artifacts, "config": cfg, @@ -2587,7 +3224,9 @@ def launch_action( if not record["cutoff_unix"] > time.time(): raise Reject("reviewed_cutoff_expired") margin = ( - 61 * checked["profile"]["execution_and_transfer_bound_seconds"] + 600 + (execution_quota(checked["profile"])["per_role"] - 2) + * checked["profile"]["execution_and_transfer_bound_seconds"] + + 600 if action == "continue" else 900 if action == "admit" @@ -2608,8 +3247,8 @@ def launch_action( provider = Provider(cfg["base_url"], read_api_key(cfg["key_file"]), run_dir, j, live) orch = Orchestrator(cfg, j, provider, run_dir) runtime = NetworkRuntime(orch, checked["profile"]) - if action == "admit": - if j.last("transaction_open") or j.last("create_intent"): + if action in ("admit", "adopt"): + if j.last("transaction_open") or j.last("create_intent") or j.last("adopted"): raise Reject("admission_not_retryable") durable_write(run_dir / "config.json", files["config"].read_bytes()) j.append("transaction_open", pid=record["parent_pid"], live=live) @@ -2620,6 +3259,28 @@ def launch_action( config_sha256=fsha(files["config"]), sources_sha256=fsha(files["sources"]), ) + if action == "admit" and "admitted_unix" in record: + j.append( + "fresh_window", + admitted_unix=record["admitted_unix"], + cutoff_unix=record["cutoff_unix"], + ) + if action == "adopt": + adoption = checked["adoption"] + j.append( + "adopted", + **adoption, + ownership_sha256=fsha(files["ownership"]), + provider_journal_sha256=fsha(files["provider_journal"]), + ) + for role, row in adoption["roles"].items(): + j.append( + "receipt", + role=role, + instance_id=row["instance_id"], + source="adopted_task4_ownership", + label=row["label"], + ) quotes = { role: next( o for o in checked["offers"] if o["machine_id"] == orch.host[role]["machine_id"] @@ -2654,6 +3315,11 @@ def launch_action( raise Reject("supervisor_not_alive") finally: os.close(fd) + if action == "adopt": + return { + "status": "ADOPTED_NOT_QUALIFIED", + "receipts": [orch.receipt(r) for r in orch.roles], + } try: for role in orch.roles: if time.time() > checked["financial"]["verified_unix"] + 30: @@ -2668,6 +3334,15 @@ def launch_action( raise opened = j.all("transaction_open") authority = j.last("network_admission_authority") + adopted = j.last("adopted") or j.last("fresh_window") + if ("admitted_unix" in record) != (adopted is not None) or ( + adopted is not None + and ( + adopted["admitted_unix"] != record["admitted_unix"] + or adopted["cutoff_unix"] != record["cutoff_unix"] + ) + ): + raise Reject("adopted_window_mismatch") if ( authority is None or authority["owner"] != record["owner"] @@ -2678,7 +3353,7 @@ def launch_action( not opened or opened[0]["pid"] != record["parent_pid"] or orch.cleanup_started() - or len(j.all("create_intent")) != 2 + or len(j.all("create_intent")) != (0 if j.last("adopted") else 2) or not all(orch.receipt(r) for r in orch.roles) or not j.last("supervisor_ready") or time.time() >= orch.deadline() @@ -2709,7 +3384,7 @@ def launch_action( ): raise Reject("existing_lifecycle_config_changed") receipts = [orch.receipt(r) for r in orch.roles] - if {row["id"] for row in checked["financial"]["instances"]} != { + if {row["id"] for row in owned_instances(checked["financial"], cfg)} != { row["instance_id"] for row in receipts if row is not None }: raise Reject("continuation_owned_inventory_mismatch") @@ -2829,12 +3504,19 @@ def launch_action( def qualify_role(checked: Record, orch: Orchestrator, runtime: NetworkRuntime, role: str) -> Record: """Stage exact source/seed bytes, inspect actual import origin, then invoke frozen driver.""" inputs, files = checked["record"]["roles"][role], checked["files"] + gpus = checked["profile"]["layout"]["n_gpus"] + if type(gpus) is not int or not 1 <= gpus <= 8: + raise Reject("qualification_gpu_count_invalid") orch.boot(role) orch.attach(role) orch.trust(role) root = orch.remote_root(role) - first = min(r["unix"] for r in orch.j.all("create_intent")) - stage_cutoff = min(first + 300, orch.deadline() - 420, checked["record"]["cutoff_unix"] - 420) + first = qualification_start(orch.j) + assert first is not None + timing = lifecycle_timing(runtime.profile) + stage_cutoff = min( + first + timing["qual_stage"], orch.deadline() - 420, checked["record"]["cutoff_unix"] - 420 + ) ssh = deadline_ssh(orch, role, stage_cutoff) def remaining() -> None: @@ -2855,15 +3537,20 @@ def remaining() -> None: orch.dir / "logs", ).returncode: raise Reject("source_extract_failed") - job = checked["driver"].IslandJobV1.model_validate_json(files[inputs["job"]].read_bytes()) + from hypertrain.protocol.messages_v2 import IslandJobV1 + + job = IslandJobV1.model_validate_json(files[inputs["job"]].read_bytes()) uploads = { "job.json": files[inputs["job"]], **{job.object_paths[k]: files[v] for k, v in inputs["objects"].items()}, } for rel, path in uploads.items(): remaining() - ssh.put(path, root + "/out/seed/" + rel) - if ssh.remote_sha256(root + "/out/seed/" + rel, orch.dir / "logs") != fsha(path): + target = root + "/out/seed/" + rel + if staged_match(ssh, target, path, orch.dir / "logs"): + continue # exact committed bytes already staged on this host + ssh.put(path, target) + if ssh.remote_sha256(target, orch.dir / "logs") != fsha(path): raise Reject("seed_transport_hash") for name in ("profile", "registry"): remaining() @@ -2880,11 +3567,11 @@ def remaining() -> None: "import hypertrain.gpu_ops.launcher,hypertrain.trainer,torch; " "assert pathlib.Path(hypertrain.gpu_ops.launcher.__file__).resolve()==" "root/'src/hypertrain/gpu_ops/launcher.py'; " - "assert torch.cuda.is_available() and torch.cuda.device_count()==2; " + f"assert torch.cuda.is_available() and torch.cuda.device_count()=={gpus}; " "assert torch.__version__=='2.14.0+cu130' and torch.version.cuda=='13.0'; " "assert pathlib.Path(torch.__file__).resolve().is_relative_to('/venv/main'); " "assert all(torch.cuda.get_device_properties(i).multi_processor_count==170 " - "for i in range(2)); " + f"for i in range({gpus})); " "print('IMPORT_ATTESTED')" ) image = checked["profile"]["runtime"]["image_digest"] @@ -2965,7 +3652,11 @@ def remaining() -> None: role, inputs["name"], "qualification.json", - cutoff=min(first + 480, orch.deadline() - 420, checked["record"]["cutoff_unix"] - 420), + cutoff=min( + first + timing["qual_stage"] + timing["qual_probe"], + orch.deadline() - 420, + checked["record"]["cutoff_unix"] - 420, + ), ) runtime.rescue(role, fresh=True) return result @@ -2975,7 +3666,9 @@ def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( description="Reviewed D2 lifecycle; no implicit live permission" ) - parser.add_argument("action", choices=("plan", "admit", "qualification", "continue", "export")) + parser.add_argument( + "action", choices=("plan", "admit", "adopt", "qualification", "continue", "export") + ) parser.add_argument("input", type=Path) parser.add_argument("extra", nargs="*") parser.add_argument("--owner") @@ -2984,15 +3677,13 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--live", action="store_true") args = parser.parse_args(argv) if args.action == "export": - print( - json.dumps( - export_artifacts( - args.input, - Path(args.extra[0]), - json.loads(Path(args.extra[1]).read_bytes()), - ) - ) + exported = export_artifacts( + args.input, Path(args.extra[0]), json.loads(Path(args.extra[1]).read_bytes()) ) + if args.extra[0] == "-": + print(exported["manifest_sha256"], file=sys.stderr) + else: + print(json.dumps(exported)) elif args.action == "plan": profile = json.loads(args.input.read_bytes()) print(json.dumps({"paid_execution": False, "artifact_budget": artifact_budget(profile)})) diff --git a/scripts/network_gpu_operation.py b/scripts/network_gpu_operation.py index d349774..3bfd212 100644 --- a/scripts/network_gpu_operation.py +++ b/scripts/network_gpu_operation.py @@ -190,6 +190,11 @@ def execution_environment(op: Operation) -> dict: ): raise ValueError("custody CUDA driver differs") observed["drivers"] = [d.strip() for d in drivers] + from hypertrain.miner.island_launch import gpu_uuid + + observed["gpu_uuids"] = [ + gpu_uuid(torch.cuda.get_device_properties(i).uuid) for i in range(layout.n_gpus) + ] observed["sm_counts"] = [ torch.cuda.get_device_properties(i).multi_processor_count for i in range(layout.n_gpus) ] diff --git a/scripts/network_gpu_prepare.py b/scripts/network_gpu_prepare.py index d036b44..3b7244c 100644 --- a/scripts/network_gpu_prepare.py +++ b/scripts/network_gpu_prepare.py @@ -7,11 +7,26 @@ import importlib.util import io import json +import math import os import secrets import sys import tarfile from pathlib import Path +from types import ModuleType + +# ponytail: mission bandwidth floor (Mbit/s) lives here, not in the signed profile; move it into +# profile["admission"] when the profile schema is next re-signed. +MIN_INET_MBPS = 800 + + +def fast_link(value: object) -> bool: + return ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + and value >= MIN_INET_MBPS + ) def digest(path: Path) -> str: @@ -227,9 +242,158 @@ def candidate_freeze(original: Path, destination: Path, candidates: list[str]) - return destination / "freeze.json" +def prepare_qualification_base( + freeze: dict, + tree: Path, + maps: dict, + controller: ModuleType, + output: Path, + out: Path, + job_dir: Path, + ownership: Path, + registry: Path, + platform: Path, +) -> dict: + """prune_rescued qualification base only: reuses the generated V2 manifest/job/objects and the + task-4 hosts by reference. No synthetic roles, joins, genesis, relay, TLS or services: the real + uid247 join and seed context are signed at run time with the existing keys.""" + profile_path, job_path = out / "profile.json", job_dir / "job.json" + profile = json.loads(profile_path.read_bytes()) + job = json.loads(job_path.read_bytes()) + if profile["artifacts"].get("remote_retention") != "prune_rescued": + raise ValueError("qualification base requires the generated prune_rescued profile") + if "sha256:" + digest(registry) != profile["runtime"]["image_digest"]: + raise ValueError("registry index must be the pinned image digest") + controller.registry_layers(json.loads(registry.read_bytes()), platform) + objects = {key: job_dir / rel for key, rel in job["object_paths"].items()} + from hypertrain.gpu_ops import network_qualification + + # Exactly what reviewed_launch recomputes: raw tree map with the qualification profile + # overlaid at experiments/gpu_network_v2/profile.json (the remote check sees the same tree). + overlay, bundle_bytes = controller.effective_bundle(network_qualification, tree, profile_path) + source = json.loads(ownership.read_bytes()) + if isinstance(source, list): + # Fresh admit: the measured GET offer rows select two distinct exact-shape machines. + limits = controller.admission_limits(profile) + if len(source) != 2 or not all(isinstance(o, dict) for o in source): + raise ValueError("fresh offers must be two distinct exact-shape pinned-driver machines") + hosts = [ + {"role": r, "machine_id": o.get("machine_id")} + for r, o in zip(("h0", "h1"), source, strict=True) + ] + if ( + any(type(h["machine_id"]) is not int for h in hosts) + or len({h["machine_id"] for h in hosts}) != 2 + or any( + type(o.get("id")) is not int + or o.get("num_gpus") != limits["gpus"] + or o.get("gpu_name") != limits["gpu_name"] + or o.get("driver_version") not in profile["runtime"]["driver_allowlist"] + or not fast_link(o.get("inet_up")) + or not fast_link(o.get("inet_down")) + for o in source + ) + ): + raise ValueError("fresh offers must be two distinct exact-shape pinned-driver machines") + else: + hosts = [ + {"role": role, "machine_id": source["api_lookup"][s]["machine_id"]} + for role, s in (("h0", "hostA"), ("h1", "hostB")) + ] + if any(type(h["machine_id"]) is not int for h in hosts): + raise ValueError("task-4 ownership must name hostA/hostB machine ids") + output.mkdir(mode=0o700) + private, public = output / "private", output / "public" + save(public / "qualification-sources.json", overlay) + bundle = public / "qualification-source.tar" + save(bundle, bundle_bytes) + save( + private / "lifecycle-config.json", + { + "hosts": hosts, + "cleanup_contract": "network-v2", + "tree": str(tree), + "network_profile_file": str(profile_path), + "image": freeze["image_candidate"]["ref"], + "remote_python": profile["runtime"]["remote_python"], + "disk_gb": controller.admission_limits(profile)["disk_gb"], + "hard_deadline_seconds": controller.lifecycle_timing(profile)["horizon"], + "hard_grace_seconds": 0, + "base_url": "https://console.vast.ai", + "boot_timeout_seconds": 300, + "keyscan_attempts": 1, + "create_margin_seconds": 600, + # ponytail: fixed remote layout; prepare_record adds key_file + cleanup_deadline_unix. + "remote_root": "/data/hypertrain-{role}", + "evidence_path": str(output / "private/evidence-run.json"), + }, + True, + ) + save( + public / "lifecycle-hashes.json", + { + name: maps["qualification-sources.json"][name] + for name in ( + "src/hypertrain/gpu_ops/launcher.py", + "src/hypertrain/gpu_ops/supervisor.py", + ) + }, + ) + paths = { + "config": private / "lifecycle-config.json", + "profile": profile_path, + "sources": public / "qualification-sources.json", + "bundle": bundle, + "manifest": out / "manifest-envelope.json", + "lifecycle_hashes": public / "lifecycle-hashes.json", + "registry": registry, + "registry_platform": platform, + "ownership": ownership, + "job": job_path, + **{"object_" + key: path for key, path in objects.items()}, + } + staged = sum(p.stat().st_size for p in [bundle, job_path, *objects.values()]) + record = { + "tree": str(tree), + "files": {name: {"path": str(p), "sha256": digest(p)} for name, p in sorted(paths.items())}, + "quota": controller.execution_quota(profile), + "staging_bytes": staged, + "public_dataset_reviewed": True, + "full_genesis_reviewed": True, + "preparation_only": True, + # Per-role join/seed_context (one real uid247 identity on both hosts) and every fresh + # provider/authority file are added by .e2e/qualification/prepare_record.py at run time. + "role_inputs": { + "job": "job", + "objects": {key: "object_" + key for key in objects}, + }, + "remaining": ["roles", "action", "owner", "parent_pid", "cutoff_unix", "admitted_unix"], + } + save(private / "review-base.json", {"record": record}, True) + result = { + "status": "QUALIFICATION_BASE_NOT_LAUNCHABLE", + "run_id": job["run_id"], + "review_base": str(private / "review-base.json"), + "review_base_sha256": digest(private / "review-base.json"), + "staging_bytes": staged, + "quota": record["quota"], + "kernels_executed": 0, + "provider_calls": 0, + } + save(output / "prepared.json", result) + return result + + def prepare( - freeze_path: Path, output: Path, genesis: int, candidates: list[str] | None = None + freeze_path: Path, + output: Path, + genesis: int, + candidates: list[str] | None = None, + qualification: tuple[Path, Path, Path, Path, Path] | None = None, ) -> dict: + if qualification is not None: + freeze, tree, maps, _, _, controller = load_frozen(freeze_path) + return prepare_qualification_base(freeze, tree, maps, controller, output, *qualification) if output.exists() or genesis < 1: raise ValueError("new output and positive deterministic CPU genesis required") if candidates is not None and candidates != ["580.95.05", "580.159.03"]: @@ -627,6 +791,13 @@ def main() -> None: parser.add_argument("--output", type=Path, required=True) parser.add_argument("--cpu-genesis-unix", type=int, required=True) parser.add_argument("--candidate-driver", action="append") + parser.add_argument( + "--qualification-base", + nargs=5, + type=Path, + metavar=("OUT_DIR", "JOB_DIR", "OWNERSHIP", "REGISTRY_INDEX", "REGISTRY_PLATFORM"), + help="prune_rescued qualification-only base from generated V2 assets", + ) args = parser.parse_args() os.umask(0o077) print( @@ -636,6 +807,9 @@ def main() -> None: args.output.resolve(), args.cpu_genesis_unix, args.candidate_driver, + tuple(p.resolve() for p in args.qualification_base) + if args.qualification_base + else None, ), sort_keys=True, ) diff --git a/src/hypertrain/aggregator/rollback_v2.py b/src/hypertrain/aggregator/rollback_v2.py index a0f9685..1383112 100644 --- a/src/hypertrain/aggregator/rollback_v2.py +++ b/src/hypertrain/aggregator/rollback_v2.py @@ -36,7 +36,7 @@ optimizer_hash, pack_state, ) -from hypertrain.data.store import Store +from hypertrain.data.store import Store, get_large from hypertrain.protocol.envelope_v2 import load_json, parse_envelope, tape_signing_message from hypertrain.protocol.envelope_v2 import verify_envelope as verify_source from hypertrain.protocol.hashing import sha256_hex @@ -490,9 +490,11 @@ def _replay_history( or parse_envelope(job.challenge_envelope).signer != signer ): raise TapeError("predecessor audit authority/commit differs") - theta, supplied = unpack_state( - get_object(store, job.start_state.state_object_sha256) - ) + + def whole(key: str) -> bytes: # StartState/ArtifactRef keys may be part indexes + return get_large(key, lambda k: get_object(store, k)) + + theta, supplied = unpack_state(whole(job.start_state.state_object_sha256)) prior_state = load_state(store, arithmetic.prev_state) if any( not torch.equal(theta[name].detach().cpu(), torch.from_numpy(value.copy())) @@ -505,19 +507,18 @@ def _replay_history( or job.start_state.theta_hash != opening.body["theta_hash"] ): raise TapeError("predecessor original global theta differs") - _finite_state( - comm, theta, supplied, unpack_state(get_object(store, job.ef_in.sha256))[0] - ) + _finite_state(comm, theta, supplied, unpack_state(whole(job.ef_in.sha256))[0]) outcome, result = audit_island( job, comm, get_sample, - get_object(store, job.start_state.state_object_sha256), - get_object(store, job.ef_in.sha256), - get_object(store, job.v0.sha256), + whole(job.start_state.state_object_sha256), + whole(job.ef_in.sha256), + whole(job.v0.sha256), cache, device=backend, now_round=job.created_beacon, + objects=lambda k: get_object(store, k), ) _finite_state(comm, result.final_theta, result.final_state, result.ef_out) if outcome.result != "MATCH": diff --git a/src/hypertrain/auditor/__main__.py b/src/hypertrain/auditor/__main__.py index d3c69b7..4b88bd3 100644 --- a/src/hypertrain/auditor/__main__.py +++ b/src/hypertrain/auditor/__main__.py @@ -2,6 +2,16 @@ Leases audit jobs for ONE run, replays them on the local verified shards and posts verdicts. Jobs of any other run are handed back (retry) so they are never replayed on the wrong data. + +`--contract 2 --api URL --run-id HEX --keyfile F`: signed v2 Receipt lease, master-executed +replay (`/execute`), then the auditor signs exactly that ReplayVerdict (`/complete`). The +auditor never computes or chooses the verdict; jobs not coordinator-signed for this run and key +are refused unexecuted (the lease then expires server-side). + +Exit 0: settled (--once) or no lease granted (empty queue, or the global running-lease cap). +Exit 2: refused job, malformed lease, HTTP error/timeout, or the server's 409 "one live lease +per auditor" after a restart mid-job (that lease is left to expire server-side). The CLI never +sleeps or polls: the supervisor re-invokes it on its own audit-scheduling or exit events. """ from __future__ import annotations @@ -11,7 +21,10 @@ import json import logging import os +import secrets import sys +import time +from collections.abc import Callable from pathlib import Path from typing import Any @@ -21,7 +34,11 @@ from hypertrain.auditor.worker import Auditor, HttpApi from hypertrain.miner.cli import shard_sampler from hypertrain.miner.core import load_keyfile -from hypertrain.protocol.messages import ReplayEnv, RunManifest +from hypertrain.protocol import envelope_v2 +from hypertrain.protocol.envelope import body_digest +from hypertrain.protocol.keys import Keypair +from hypertrain.protocol.messages import Receipt, ReplayEnv, ReplayVerdict, RunManifest +from hypertrain.protocol.messages_v2 import AuditJobV2, RunManifestV2 class RunScopedApi(HttpApi): @@ -37,6 +54,86 @@ def lease(self) -> dict[str, Any] | None: return raw +class RefusedJob(RuntimeError): + pass + + +MARGIN_S = 60.0 + + +def audit_v2( + client: httpx.Client, + base: str, + run_id: str, + key: Keypair, + once: bool, + clock: Callable[[], float] = time.time, +) -> int: + root = f"{base}/v2/runs/{run_id}" + + def post(path: str, body: dict[str, Any], timeout: float) -> httpx.Response: + r = client.post(root + path, json=body, timeout=timeout) + r.raise_for_status() + return r + + def now() -> int: + r = client.get(root, timeout=60) + r.raise_for_status() + return int(r.json()["now_round"]) + + r = client.get(root, timeout=60) + r.raise_for_status() + manifest = RunManifestV2.model_validate(r.json()["manifest_envelope"]["body"]) + if r.json()["contract"] != 2 or manifest.run_id() != run_id: + raise RefusedJob("served v2 manifest hashes to another run_id") + while True: + t = now() + # Unique Receipt replay ID per lease request; w is not lease-bound server-side. + ask = Receipt(w=0, commit_hash=secrets.token_hex(32), received_round=t) + r = client.post( + root + "/worker/lease", + json=envelope_v2.seal(key, "Receipt", run_id, ask, t + 100), + timeout=60, + ) + # Empty queue: 204, or 503 from an admitted service run (store.lease_v2). + if r.status_code == 204 or (r.status_code == 503 and "no authenticated queued" in r.text): + return 0 + r.raise_for_status() + env = envelope_v2.parse_envelope(r.json()) + if ( + env.type != "AuditJobV2" + or env.run_id != run_id + or env.signer != manifest.training.coord_pubkey + or not envelope_v2.verify_envelope(r.json()) + ): + raise RefusedJob("lease is not a coordinator-signed job of this run") + job = AuditJobV2.model_validate(env.body) + if job.run_id != run_id or job.manifest != manifest or job.auditor_id != key.ss58: + raise RefusedJob("leased job binds another run, manifest or auditor") + w, exp = job.start_state.w, job.lease_expires + receipt = Receipt(w=w, commit_hash=job.lease_nonce, received_round=now()) + # Server LeaseGuard cancels replay at this wall deadline (store.execute_audit_v2). + beacon = manifest.training.beacon + deadline = beacon.genesis_time + (min(exp, job.absolute_deadline) - 1) * beacon.period + r = post( + f"/worker/jobs/{job.job_id}/execute", + envelope_v2.seal(key, "Receipt", run_id, receipt, exp), + max(deadline - clock(), 0.0) + MARGIN_S, + ) + verdict = ReplayVerdict.model_validate(r.json()["verdict"]) + challenge = envelope_v2.parse_envelope(job.challenge_envelope) + if verdict.challenge_hash != body_digest(challenge.body): + raise RefusedJob("executed verdict answers another challenge") + done = post( + f"/worker/jobs/{job.job_id}/complete", + envelope_v2.seal(key, "ReplayVerdict", run_id, verdict, exp), + MARGIN_S, + ).json() + print(json.dumps({"job": job.job_id, "result": done["result"]}), flush=True) + if once: + return 0 + + def replay_env(image_digest: str, device: str) -> ReplayEnv: import torch @@ -63,14 +160,24 @@ def main(argv: list[str] | None = None) -> int: p = argparse.ArgumentParser(prog="python -m hypertrain.auditor") p.add_argument("--api", required=True) p.add_argument("--run-id", required=True) - p.add_argument("--token-file", type=Path, required=True) + p.add_argument("--contract", type=int, choices=[1, 2], default=1, help="2: signed v2 routes") + p.add_argument("--token-file", type=Path, help="worker bearer token (contract 1)") p.add_argument("--keyfile", type=Path, required=True) - p.add_argument("--data-dir", type=Path, required=True) + p.add_argument("--data-dir", type=Path, help="verified shards (contract 1)") p.add_argument("--device", choices=["cuda", "cpu"], default="cuda") p.add_argument("--image-digest", default=os.environ.get("HYPERTRAIN_IMAGE_DIGEST", "")) p.add_argument("--once", action="store_true", help="settle at most one job, then exit") a = p.parse_args(argv) + if a.contract == 1 and (a.token_file is None or a.data_dir is None): + p.error("contract 1 requires --token-file and --data-dir") logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + if a.contract == 2: + with httpx.Client() as v2: + try: + return audit_v2(v2, a.api.rstrip("/"), a.run_id, load_keyfile(a.keyfile), a.once) + except (RefusedJob, ValueError, httpx.HTTPError) as error: + logging.error("%s", error) + return 2 if a.device == "cuda": import torch diff --git a/src/hypertrain/auditor/replay.py b/src/hypertrain/auditor/replay.py index 45dca8f..9753052 100644 --- a/src/hypertrain/auditor/replay.py +++ b/src/hypertrain/auditor/replay.py @@ -31,6 +31,7 @@ from safetensors.torch import save as st_save import hypertrain.trainer # noqa: F401 (determinism before torch users below) +from hypertrain.data.store import CorruptObjectError, parse_index from hypertrain.protocol.envelope import body_digest from hypertrain.protocol.hashing import MerkleTree, sha256_hex from hypertrain.protocol.messages import ( @@ -108,6 +109,32 @@ class ServedState: blob: bytes +def ref_binds( + ref: str, + blob: bytes, + objects: Callable[[str], bytes] | None, + ref_size: int | None = None, +) -> bool: + """True iff stored ref `ref` (of stored length `ref_size`, if given) names exactly `blob`. + + Plain object: ref is sha256(blob) and the stored length is len(blob). ht-parts-v1: ref + is the SHA of the stored index object (length `ref_size`), whose verified `sha256`/`size` + are the whole-file SHA and len(blob). Without `objects` only plain refs bind. + """ + if sha256_hex(blob) == ref: + return ref_size is None or ref_size == len(blob) + if objects is None: + return False + raw = objects(ref) + if sha256_hex(raw) != ref or (ref_size is not None and ref_size != len(raw)): + return False + try: + index = parse_index(raw) + except CorruptObjectError: + return False + return index is not None and (index.sha256, index.size) == (sha256_hex(blob), len(blob)) + + def challenge_hash(ch: AuditChallenge) -> str: return body_digest(ch.model_dump(mode="json")) @@ -375,6 +402,7 @@ def audit_island( hook: TraceHook | None = None, device: torch.device | str = "cpu", beacon_now: Callable[[], int] | None = None, + objects: Callable[[str], bytes] | None = None, ) -> tuple[Outcome, RoundResult]: """Full exact-layout replay from an independently checked carry lineage, never served prefix. @@ -394,11 +422,11 @@ def live() -> None: if job.anchor_age > 1 or not cfg.inner.H <= job.replay_step_budget <= 2 * cfg.inner.H: raise AuditInputError("ANCHOR_BUDGET_EXCEEDED") start = job.start_state - if sha256_hex(theta_blob) != start.state_object_sha256: + if not ref_binds(start.state_object_sha256, theta_blob, objects): raise AuditInputError("start object hash mismatch") - if sha256_hex(ef_blob) != job.ef_in.sha256 or len(ef_blob) != job.ef_in.size: + if not ref_binds(job.ef_in.sha256, ef_blob, objects, job.ef_in.size): raise AuditInputError("EF object hash/size mismatch") - if sha256_hex(v0_blob) != job.v0.sha256 or len(v0_blob) != job.v0.size: + if not ref_binds(job.v0.sha256, v0_blob, objects, job.v0.size): raise AuditInputError("v0 object hash/size mismatch") theta, supplied = unpack_state(theta_blob) ef, _ = unpack_state(ef_blob) @@ -501,6 +529,7 @@ def audit_island_chain( cancel: CancelHook | None = None, device: torch.device | str = "cpu", beacon_now: Callable[[], int] | None = None, + objects: Callable[[str], bytes] | None = None, ) -> tuple[Outcome, RoundResult]: """At most one missing predecessor; both accepted jobs consume current reserved 2H. @@ -534,6 +563,7 @@ def live() -> None: cancel=live, device=device, beacon_now=beacon_now, + objects=objects, ) if out.result != "MATCH": raise AuditInputError("predecessor replay did not MATCH") @@ -547,6 +577,7 @@ def live() -> None: cancel=live, device=device, beacon_now=beacon_now, + objects=objects, ) diff --git a/src/hypertrain/auditor/worker.py b/src/hypertrain/auditor/worker.py index d513d93..e77aea6 100644 --- a/src/hypertrain/auditor/worker.py +++ b/src/hypertrain/auditor/worker.py @@ -14,7 +14,7 @@ import hashlib import threading -from collections.abc import Mapping +from collections.abc import Callable, Mapping from dataclasses import dataclass from pathlib import Path from typing import Any, Literal, Protocol @@ -98,15 +98,18 @@ def execute_island_audit( publication_job: IslandJobV1 | None = None, publication_directory: Path | None = None, launch: IslandLaunch | None = None, + objects: Callable[[str], bytes] | None = None, ) -> tuple[Outcome, IslandArtifacts]: """Actual torchrun audit runtime; L0 stages authenticated objects and sample proofs. Subscribe guard.advance to verified beacon events before invoking; no heartbeat renewal. directory contains start_state, ef_in, v0, samples, sample_proofs. No HTTP route invented. + StartState/ArtifactRef SHAs and sizes describe stored objects (an ht-parts-v1 index above + 1 MiB) resolved via `objects`; the local IslandJobV1 binds whole-file SHAs. """ import json - from hypertrain.auditor.replay import VerifiedAnchor, optimizer_hash, pack_state + from hypertrain.auditor.replay import VerifiedAnchor, optimizer_hash, pack_state, ref_binds from hypertrain.gpu_ops.journal import durable_write from hypertrain.protocol.hashing import sha256_hex from hypertrain.trainer.compress import state_hash @@ -123,21 +126,19 @@ def execute_island_audit( theta, st = unpack_state(theta_blob) ef, _ = unpack_state(ef_blob) if ( - sha256_hex(theta_blob) != start.state_object_sha256 + not ref_binds(start.state_object_sha256, theta_blob, objects) or st is None or optimizer_hash(st) != optimizer_hash(prior.state) or state_hash(theta) != start.theta_hash or (job.manifest.training.inner.state_policy == "carry" and st.step != start.global_step0) - or sha256_hex(ef_blob) != job.ef_in.sha256 - or len(ef_blob) != job.ef_in.size - or sha256_hex(ef_blob) != start.ef_object_sha256 + or not ref_binds(job.ef_in.sha256, ef_blob, objects, job.ef_in.size) + or start.ef_object_sha256 != job.ef_in.sha256 or state_hash(ef) != start.ef_hash - or sha256_hex(v0_blob) != job.v0.sha256 - or len(v0_blob) != job.v0.size + or not ref_binds(job.v0.sha256, v0_blob, objects, job.v0.size) ): raise AuditInputError("untrusted audit start/state/EF") start_path = "start_state" - start_hash = start.state_object_sha256 + start_hash = sha256_hex(theta_blob) if job.manifest.training.inner.state_policy != "carry": # The authenticated anchor still includes m/v; reset/derived kernels take theta only. start_path = "audit-theta.safetensors" @@ -157,8 +158,8 @@ def execute_island_audit( sample_ids=job.sample_ids, global_step0=start.global_step0, start_state_sha256=start_hash, - ef_in_sha256=job.ef_in.sha256, - v0_sha256=job.v0.sha256, + ef_in_sha256=sha256_hex(ef_blob), + v0_sha256=sha256_hex(v0_blob), deadline=deadline_unix, object_paths={ "start_state": start_path, diff --git a/src/hypertrain/beacon/drand.py b/src/hypertrain/beacon/drand.py index 03fdd29..e83d9b1 100644 --- a/src/hypertrain/beacon/drand.py +++ b/src/hypertrain/beacon/drand.py @@ -2,6 +2,7 @@ from __future__ import annotations +import functools import hashlib from collections.abc import Mapping, Sequence from typing import Any @@ -43,6 +44,7 @@ def _hex(value: Any, n_bytes: int, field: str) -> bytes: raise BeaconVerificationError(f"{field}: not hex") from None +@functools.lru_cache(maxsize=64) def _pairing_ok(sig: bytes, round: int, pubkey: bytes) -> bool: # Unchained G1 scheme: e(sig, g2) == e(H(sha256(be64(round))), pk), checked as one product. sig_int = int.from_bytes(sig, "big") diff --git a/src/hypertrain/challenge/admission.py b/src/hypertrain/challenge/admission.py index 40b072e..77954c8 100644 --- a/src/hypertrain/challenge/admission.py +++ b/src/hypertrain/challenge/admission.py @@ -19,7 +19,14 @@ from hypertrain.beacon.core import BeaconRound from hypertrain.challenge.admission_store import AdmissionError, AdmissionRecord, AdmissionStore from hypertrain.challenge.trust_v2 import FundedStatus, nominal_q -from hypertrain.data.store import Store +from hypertrain.data.store import ( + Store, + StoreError, + _stream, + get_large, + parse_index, + put_large, +) from hypertrain.data.trial_assignment import trial_assignment_hash, trial_samples from hypertrain.gpu_ops.work_screen import IslandLaunch, screen_work from hypertrain.ledger.escrow_v2 import EscrowV2, digest @@ -29,7 +36,9 @@ from hypertrain.protocol.keys import Keypair from hypertrain.protocol.messages import Finalize from hypertrain.protocol.messages_v2 import ( + PART_BYTES, AdmissionPolicyV2, + ArtifactRef, EscrowLock, IslandJobV1, JoinChallenge, @@ -40,6 +49,45 @@ ) +def _logical(objects: Store, ref: ArtifactRef, code: str) -> tuple[str, int, bool]: + """(whole sha256, logical size, indexed) of `ref`; ref.size is the stored object length.""" + try: + raw = objects.get(ref.sha256) + index = parse_index(raw) + except StoreError: + raise AdmissionError(code) from None + if len(raw) != ref.size: + raise AdmissionError(code) + return (ref.sha256, ref.size, False) if index is None else (index.sha256, index.size, True) + + +def _verify_ref(objects: Store, ref: ArtifactRef, cap: int) -> tuple[str, int]: + """Caps bind each stored object (ref and every part); the logical size is bounded by + PartIndexV1 (<= MAX_PARTS * PART_BYTES). Parts stream through hashing, never assembled.""" + if ref.size > cap: + raise AdmissionError("PROOF_OBJECT_LIMIT") + whole, size, indexed = _logical(objects, ref, "PROOF_OBJECT_HASH") + if indexed: + if PART_BYTES > cap: + raise AdmissionError("PROOF_OBJECT_LIMIT") + try: + _stream(ref.sha256, objects.get, lambda _part: None) # part lengths + whole sha + except StoreError: + raise AdmissionError("PROOF_OBJECT_HASH") from None + return whole, size + + +def _put_capped(objects: Store, data: bytes, cap: int) -> str: + """put_large with the policy cap bound to every stored object (plain, part, index).""" + + def put(chunk: bytes) -> str: + if len(chunk) > cap: + raise AdmissionError("SHADOW_PUBLICATION_OBJECT_LIMIT") + return objects.put(chunk) + + return put_large(data, put) + + @dataclass(frozen=True, slots=True) class AdmissionStatus: record: AdmissionRecord @@ -289,7 +337,11 @@ def prepare_reference( strict=True, ): data = (directory / "published" / "rank-0" / name).read_bytes() - if self.objects.put(data) != ref.sha256: + logical = (sha256_hex(data), len(data)) + if ( + put_large(data, self.objects.put) != ref.sha256 + or logical != _logical(self.objects, ref, "REFERENCE_OBJECT_HASH")[:2] + ): raise AdmissionError("REFERENCE_OBJECT_HASH") with self.escrow.tx(): current = self.store.db.execute( @@ -404,8 +456,11 @@ def retain_runtime_custody( or reference.leaves_root != summaries[0]["commitments"]["leaves_root"] or reference.delta_hash != summaries[0]["commitments"]["delta_hash"] or any( - publication["files"]["rank-0/" + name]["sha256"] != ref.sha256 - or publication["files"]["rank-0/" + name]["bytes"] != ref.size + ( + publication["files"]["rank-0/" + name]["sha256"], + publication["files"]["rank-0/" + name]["bytes"], + ) + != _logical(self.objects, ref, "SHADOW_RUNTIME_ACCEPTED_REFERENCE")[:2] for name, ref in zip( ("state.safetensors", "ef.safetensors", "delta.bin", "leaves.json"), reference.artifact_refs, @@ -534,9 +589,9 @@ def retain_trial_publication( if path.is_file(): relative = str(path.relative_to(publication)) data = confined(publication, relative).read_bytes() - if len(data) > challenge.artifact_limits.max_object_bytes: - raise AdmissionError("SHADOW_PUBLICATION_OBJECT_LIMIT") - files[relative] = {"sha256": self.objects.put(data), "bytes": len(data)} + _put_capped(self.objects, data, challenge.artifact_limits.max_object_bytes) + # Custody `files` keep whole-file sha/bytes; the store holds plain or indexed refs. + files[relative] = {"sha256": sha256_hex(data), "bytes": len(data)} if len(canonicalize(files)) > challenge.artifact_limits.max_chunk_manifest_bytes: raise AdmissionError("SHADOW_PUBLICATION_MANIFEST_LIMIT") if any( @@ -556,7 +611,9 @@ def retain_trial_publication( strict=True, ): data = path.read_bytes() - if sha256_hex(data) != ref.sha256 or len(data) != ref.size: + if (sha256_hex(data), len(data)) != _logical( + self.objects, ref, "SHADOW_PUBLICATION_PROOF" + )[:2]: raise AdmissionError("SHADOW_PUBLICATION_PROOF") job_hash = self.objects.put(canonicalize(job.body())) publication_hash = self.objects.put(canonicalize(files)) @@ -684,11 +741,7 @@ def proof(self, raw: bytes, screen_raw: bytes, *, now: int) -> AdmissionRecord: ) return self.store.by_id(r.admission_id) for a in proof.artifact_refs: - if a.size > c.artifact_limits.max_object_bytes: - raise AdmissionError("PROOF_OBJECT_LIMIT") - data = self.objects.get(a.sha256) - if len(data) != a.size or sha256_hex(data) != a.sha256: - raise AdmissionError("PROOF_OBJECT_HASH") + _verify_ref(self.objects, a, c.artifact_limits.max_object_bytes) self.store.db.execute( "UPDATE admissions_v2 SET " "proof_digest=?,screen_json=?,screen_until=? WHERE admission_id=?", @@ -764,7 +817,7 @@ def trial_finality(self, admission_id: str, *, now: int) -> bytes | None: return None expected = WorkProof.model_validate_json(row["reference_json"]) challenge = JoinChallenge.model_validate_json(row["challenge"]) - theta, _ = unpack_state(self.objects.get(expected.artifact_refs[0].sha256)) + theta, _ = unpack_state(get_large(expected.artifact_refs[0].sha256, self.objects.get)) final = Finalize( w=row["trial_epoch"], final_theta_hash_w1=state_hash(theta), @@ -807,7 +860,7 @@ def finalize_trial(self, admission_id: str, raw: bytes, *, now: int) -> Admissio from hypertrain.auditor.replay import unpack_state from hypertrain.trainer.compress import state_hash - theta, _ = unpack_state(self.objects.get(state_ref.sha256)) + theta, _ = unpack_state(get_large(state_ref.sha256, self.objects.get)) if ( final.w != row["trial_epoch"] or final.included != [r.hotkey] diff --git a/src/hypertrain/challenge/app.py b/src/hypertrain/challenge/app.py index bfa05af..830f80f 100644 --- a/src/hypertrain/challenge/app.py +++ b/src/hypertrain/challenge/app.py @@ -31,6 +31,7 @@ from hypertrain.challenge.store import ChallengeError, ChallengeStore from hypertrain.data.store import LocalFSStore from hypertrain.data.store import Store as ObjectStore +from hypertrain.gpu_ops import role_dispatch from hypertrain.ledger import Params, vest_rounds_for_q from hypertrain.ledger.escrow_v2 import EscrowError from hypertrain.protocol.envelope import EnvelopeError @@ -135,6 +136,7 @@ class Config: owner_hotkey: str | None params: Params netuid: int = 100 + role_dispatch_file: Path | None = None @classmethod def from_env(cls) -> Config: @@ -160,6 +162,7 @@ def path(name: str) -> Path | None: vest_rounds=int(env("HYPERTRAIN_VEST_ROUNDS", str(vest_rounds_for_q("0.1")))), ), netuid=int(env("CHALLENGE_NETUID", "100")), + role_dispatch_file=path("HYPERTRAIN_ROLE_DISPATCH"), ) @@ -304,6 +307,11 @@ def create_app( config.owner_hotkey, verify_beacon, objects or LocalFSStore(config.state_dir / "objects"), + _role_launch=( + role_dispatch.load(config.role_dispatch_file) + if config.role_dispatch_file is not None + else None + ), ) ) store.clock = clock diff --git a/src/hypertrain/challenge/store.py b/src/hypertrain/challenge/store.py index e408082..4fc7ba2 100644 --- a/src/hypertrain/challenge/store.py +++ b/src/hypertrain/challenge/store.py @@ -33,12 +33,21 @@ from scripts.network_gpu_operation import Operation from hypertrain.gpu_ops.work_screen import IslandLaunch + from hypertrain.protocol.messages import ReplayEnv from hypertrain.beacon import BeaconError, BeaconRound from hypertrain.challenge.admission import Admission from hypertrain.challenge.disputes_v2 import Contest, DisputesV2 from hypertrain.data.assignment import assign_round -from hypertrain.data.store import ObjectNotFound, StoreError +from hypertrain.data.store import ( + ObjectNotFound, + StoreError, + _stream, + get_large, + parse_index, + put_file, + put_large, +) from hypertrain.data.store import Store as ObjectStore from hypertrain.data.trial_assignment import trial_assignment_hash from hypertrain.ledger import Ledger, LedgerError, Params @@ -87,6 +96,7 @@ f32val, ) from hypertrain.protocol.messages_v2 import ( + PART_BYTES, AdmissionPolicyV2, AggregationPolicyV2, AuditPolicyV2, @@ -1208,7 +1218,7 @@ def _served_root(self, blob_sha: str) -> str | None: """tensor_root = hypertrain.auditor.replay.tensor_root of the uploaded state blob.""" from hypertrain.auditor.replay import tensor_root, unpack_state - theta, st = unpack_state(self.get_object(blob_sha)) + theta, st = unpack_state(get_large(blob_sha, self.get_object)) return None if st is None else tensor_root(theta, st) def dispute(self, run_id: str, env: Any) -> dict[str, Any]: @@ -2293,7 +2303,7 @@ def _bootstrap_backend_v2( or not Path(result.environment.torch_path).is_relative_to("/venv/main") ): raise ChallengeError(403, "reviewed CUDA runtime/reference binding differs") - engine.compare(list(results)) # Rehash all rescued rank/round/checkpoint bytes. + engine.compare(list(results), manifest) # Rehash all rescued rank/round/checkpoint bytes. summaries = [ fsha(path.parent / f"round-{round_index}/published/rank-{rank}/summary.json") for path in results @@ -3024,11 +3034,9 @@ def _shadow_trial_v2( } if actual_files != publication: raise ChallengeError(409, "shadow original publication changed") - for metadata in publication.values(): + for relative, metadata in publication.items(): assert isinstance(metadata, dict) and isinstance(metadata["sha256"], str) - data = self.objects.get(metadata["sha256"]) - if sha256_hex(data) != metadata["sha256"] or len(data) != metadata["bytes"]: - raise ChallengeError(409, "shadow addressed publication changed") + self._addressed_file_v2(confined(directory, relative)) artifacts = validate_artifacts(job, directory) if tuple(retained_screen.rank_results) != artifacts.ranks: raise ChallengeError(409, "shadow original rank screen differs") @@ -3354,6 +3362,19 @@ def _shadow_settlement_v2(self, run_id: str, origin: str) -> Settlement: self._shadow_custody_v2(run_id, binding) return self._shadow_status_v2(run_id, binding) + def _addressed_file_v2(self, path: Path) -> None: + """The store holds `path` under its deterministic ht-parts-v1 ref, parts verified.""" + try: + ref = put_file(path, sha256_hex) # hash-only ref derivation, streamed + _stream(ref, self.objects.get, lambda _part: None) + except StoreError: + raise ChallengeError(409, "shadow addressed publication changed") from None + + def _whole_sha_v2(self, ref: str) -> str: + """StartState/ArtifactRef keys are stored refs; IslandJob binds whole-file sha256.""" + index = parse_index(self.objects.get(ref)) + return ref if index is None else index.sha256 + def _shadow_anchor_v2(self, job: IslandJobV1, hotkey: str, directory: Path) -> str: """Derive original trial genesis anchor from actual start/optimizer/EF bytes.""" from hypertrain.auditor.replay import AnchorCache, optimizer_hash, unpack_state @@ -3361,13 +3382,18 @@ def _shadow_anchor_v2(self, job: IslandJobV1, hotkey: str, directory: Path) -> s from hypertrain.protocol.messages_v2 import StartStateV2 from hypertrain.trainer.compress import state_hash - theta, carried = unpack_state( - confined(directory, job.object_paths["start_state"]).read_bytes() - ) + start_blob = confined(directory, job.object_paths["start_state"]).read_bytes() + ef_blob = confined(directory, job.object_paths["ef_in"]).read_bytes() + if (sha256_hex(start_blob), sha256_hex(ef_blob)) != ( + job.start_state_sha256, + job.ef_in_sha256, + ): + raise ChallengeError(409, "shadow original trial start bytes differ from job") + theta, carried = unpack_state(start_blob) genesis = AnchorCache().genesis(job.manifest, hotkey, theta) if carried is not None and optimizer_hash(carried) != optimizer_hash(genesis.state): raise ChallengeError(409, "shadow original trial optimizer differs from genesis") - ef, _ = unpack_state(confined(directory, job.object_paths["ef_in"]).read_bytes()) + ef, _ = unpack_state(ef_blob) if job.global_step0 != 0 or state_hash(ef) != state_hash(genesis.ef): raise ChallengeError(409, "shadow original trial EF/start differs from genesis") return StartStateV2( @@ -3375,9 +3401,9 @@ def _shadow_anchor_v2(self, job: IslandJobV1, hotkey: str, directory: Path) -> s w=job.w, hotkey=hotkey, theta_hash=state_hash(theta), - state_object_sha256=job.start_state_sha256, + state_object_sha256=put_large(start_blob, sha256_hex), # stored ref, not whole opt_state_hash=optimizer_hash(genesis.state), - ef_object_sha256=job.ef_in_sha256, + ef_object_sha256=put_large(ef_blob, sha256_hex), ef_hash=state_hash(ef), parent_anchor_hash=genesis.anchor_hash, global_step0=0, @@ -4900,9 +4926,9 @@ def open_round_v2(self, run_id: str, raw: bytes) -> dict[str, Any]: w=body.w, hotkey=entry.hotkey, theta_hash=state_hash(theta), - state_object_sha256=self.objects.put(state_blob), + state_object_sha256=put_large(state_blob, self.objects.put), opt_state_hash=optimizer_hash(anchor.state), - ef_object_sha256=self.objects.put(ef_blob), + ef_object_sha256=put_large(ef_blob, self.objects.put), ef_hash=state_hash(anchor.ef), parent_anchor_hash=anchor.anchor_hash, global_step0=anchor.state.step, @@ -6531,12 +6557,12 @@ def read(key: str) -> bytes: body = self._record_v2(run_id, "round", str(w))["body"] directory = self.state_dir / "jobs-v2" / str(w) / hotkey directory.mkdir(parents=True, exist_ok=True) - state = self.objects.get(start.state_object_sha256) + state = get_large(start.state_object_sha256, self.objects.get) if manifest.training.inner.state_policy != "carry": state = pack_state(unpack_state(state)[0]) inputs = { "start_state": state, - "ef_in": self.objects.get(start.ef_object_sha256), + "ef_in": get_large(start.ef_object_sha256, self.objects.get), "v0": pack_state({}), } dataset = self._record_v2(run_id, "dataset", "inputs") @@ -6552,7 +6578,7 @@ def read(key: str) -> bytes: hashes = {} for name, value in inputs.items(): durable_write(directory / name, value) - hashes[name] = self.objects.put(value) + hashes[name] = put_large(value, self.objects.put) # ht-parts-v1 above 1 MiB job = IslandJobV1( job_version=1, run_id=run_id, @@ -6560,9 +6586,9 @@ def read(key: str) -> bytes: manifest=manifest, sample_ids=list(samples), global_step0=start.global_step0, - start_state_sha256=hashes["start_state"], - ef_in_sha256=hashes["ef_in"], - v0_sha256=hashes["v0"], + start_state_sha256=sha256_hex(inputs["start_state"]), + ef_in_sha256=sha256_hex(inputs["ef_in"]), + v0_sha256=sha256_hex(inputs["v0"]), object_paths={k: k for k in inputs}, deadline=manifest.training.beacon.genesis_time + (int(body["d_commit"]) - 1) * 3, ) @@ -6659,6 +6685,15 @@ def read_object(key: str, limit: int = 65536) -> bytes: raise ChallengeError(422, "SERVICE_AUDIT_OBJECT_HASH") return data + def read_ref(key: str) -> tuple[bytes, int]: + """(whole bytes, stored length); every stored object (plain/part/index) + is read bounded at PART_BYTES and content-verified.""" + try: + stored = len(read_object(key, PART_BYTES)) + return get_large(key, lambda k: read_object(k, PART_BYTES)), stored + except StoreError: + raise ChallengeError(422, "SERVICE_AUDIT_OBJECT_HASH") from None + descriptor_bytes = read_object(str(accepted["descriptor_hash"]), 1 << 20) preflight_descriptor = envelope_v2.load_json(descriptor_bytes, max_bytes=1 << 20) target = self.state_dir / "audit-geometry-v2" / job_id / row["nonce"] @@ -6678,9 +6713,10 @@ def read_object(key: str, limit: int = 65536) -> bytes: or geometry.run_id != run_id or geometry.w != row["w"] or geometry.sample_ids != job.sample_ids - or geometry.start_state_sha256 != job.start_state.state_object_sha256 - or geometry.ef_in_sha256 != job.ef_in.sha256 - or geometry.v0_sha256 != job.v0.sha256 + or geometry.start_state_sha256 + != self._whole_sha_v2(job.start_state.state_object_sha256) + or geometry.ef_in_sha256 != self._whole_sha_v2(job.ef_in.sha256) + or geometry.v0_sha256 != self._whole_sha_v2(job.v0.sha256) or set(geometry.object_paths) != {"start_state", "ef_in", "v0", "samples", "sample_proofs"} or any( @@ -6716,17 +6752,20 @@ def read_object(key: str, limit: int = 65536) -> bytes: "ef_sha256": job.ef_in.sha256, }: raise ChallengeError(409, "SERVICE_AUDIT_ANCHOR_METADATA") - inputs = { - "start_state": read_object(job.start_state.state_object_sha256), - "ef_in": read_object(job.ef_in.sha256), - "v0": read_object(job.v0.sha256), + refs = { + "start_state": job.start_state.state_object_sha256, + "ef_in": job.ef_in.sha256, + "v0": job.v0.sha256, } + decoded = {name: read_ref(key) for name, key in refs.items()} + inputs = {name: data for name, (data, _) in decoded.items()} + # Anchor files hold whole bytes; their streamed ht-parts-v1 ref must be the key. if ( - bounded_bytes(path / "state", 65536) != inputs["start_state"] - or bounded_bytes(path / "ef", 65536) != inputs["ef_in"] + put_file(path / "state", sha256_hex) != refs["start_state"] + or put_file(path / "ef", sha256_hex) != refs["ef_in"] ): raise ChallengeError(422, "SERVICE_AUDIT_ANCHOR_BYTES") - if len(inputs["ef_in"]) != job.ef_in.size or len(inputs["v0"]) != job.v0.size: + if (decoded["ef_in"][1], decoded["v0"][1]) != (job.ef_in.size, job.v0.size): raise ChallengeError(422, "SERVICE_AUDIT_REF_SIZE") shapes = param_shapes(TrainConfig.from_manifest_v2(manifest).model) for name, data in inputs.items(): @@ -6735,7 +6774,7 @@ def read_object(key: str, limit: int = 65536) -> bytes: size = struct.unpack_from(" len(data) - 8: raise ChallengeError(422, "SERVICE_AUDIT_STATE_HEADER") - header = envelope_v2.load_json(data[8 : 8 + size], max_bytes=65536) + header = envelope_v2.load_json(data[8 : 8 + size], max_bytes=PART_BYTES) expected = { f"{part}/{tensor}": list(shape) for part in (("theta", "m", "v") if name == "start_state" else ("theta",)) @@ -6799,7 +6838,9 @@ def read_object(key: str, limit: int = 65536) -> bytes: ) miner = self.state_dir / "jobs-v2" / str(row["w"]) / row["hotkey"] for name, relative in geometry.object_paths.items(): - if bounded_bytes(miner / relative, 65536) != inputs[name]: + if put_file(miner / relative, sha256_hex) != put_large( + inputs[name], sha256_hex + ): raise ChallengeError(422, "SERVICE_AUDIT_GEOMETRY_BYTES") self._service_boundary_v2(run_id, row["w"], heavy=True) deadline = ( @@ -6814,9 +6855,9 @@ def read_object(key: str, limit: int = 65536) -> bytes: directory = self.state_dir / "audit-v2" / job_id / row["nonce"] directory.mkdir(parents=True, exist_ok=True) inputs = { - "start_state": self.objects.get(start.state_object_sha256), - "ef_in": self.objects.get(start.ef_object_sha256), - "v0": self.objects.get(job.v0.sha256), + "start_state": get_large(start.state_object_sha256, self.objects.get), + "ef_in": get_large(start.ef_object_sha256, self.objects.get), + "v0": get_large(job.v0.sha256, self.objects.get), } dataset = self._record_v2(run_id, "dataset", "inputs") samples_raw = self.objects.get(str(dataset["samples_hash"])) @@ -6897,6 +6938,7 @@ def read_object(key: str, limit: int = 65536) -> bytes: now_round=now, deadline_unix=deadline, backend=backend, + objects=self.objects.get, # resolves stored ht-parts-v1 refs publication_job=geometry_job, publication_directory=geometry_directory, launch=( @@ -6926,6 +6968,11 @@ def read_object(key: str, limit: int = 65536) -> bytes: if artifacts.directory != geometry_directory / "published": raise ChallengeError(409, "independent audit publication directory differs") IslandParty.published(row["hotkey"], geometry_job, artifacts.directory) + dispatched = ( + self._dispatch_env_v2(manifest, geometry_directory) + if backend == "cuda" + else None + ) guard.check() with self._tx(): current = self._db.execute( @@ -6949,13 +6996,20 @@ def read_object(key: str, limit: int = 65536) -> bytes: result=outcome.result, first_bad_leaf=outcome.first_bad_leaf, recomputed_leaves_root=outcome.recomputed_leaves_root, - replay_env=ReplayEnv( - image_digest=manifest.training.reference_spec.image_digest, - driver=manifest.training.reference_spec.driver_allowlist[0], - gpu_uuid_sha256="0" * 64, - sm_count=max(1, manifest.training.reference_spec.sm_count), + replay_env=( + dispatched[0] + if dispatched is not None + # CPU functional backend only: never a CUDA hardware claim. + else ReplayEnv( + image_digest=manifest.training.reference_spec.image_digest, + driver=manifest.training.reference_spec.driver_allowlist[0], + gpu_uuid_sha256="0" * 64, + sm_count=max(1, manifest.training.reference_spec.sm_count), + ) ), ) + if dispatched is not None: + self._put_record_v2(run_id, "audit-dispatch", job_id, dispatched[1]) # Store independently executed evidence. Caller signs this exact verdict separately. identity = f"{job.start_state.w}:{job.start_state.hotkey}" self._put_record_v2( @@ -6983,6 +7037,47 @@ def read_object(key: str, limit: int = 65536) -> bytes: finally: self._lease_guards_v2.pop(job_id, None) + def _dispatch_env_v2( + self, manifest: RunManifestV2, directory: Path + ) -> tuple[ReplayEnv, dict[str, JsonValue]]: + """Actual remote CUDA hardware bound to the dispatched publication; never a placeholder.""" + from hypertrain.miner.island_launch import IslandFailure, gpu_uuid + from hypertrain.protocol.messages import ReplayEnv + + ref = manifest.training.reference_spec + try: + raw = envelope_v2.load_json((directory / "role-dispatch.json").read_bytes()) + env = raw["environment"] + if not isinstance(env, dict): + raise TypeError("environment") + drivers, sms, found = env["drivers"], env["sm_counts"], env["gpu_uuids"] + if not ( + isinstance(drivers, list) and isinstance(sms, list) and isinstance(found, list) + ): + raise TypeError("environment lists") + uuids = [gpu_uuid(u) for u in found] + except (OSError, ValueError, KeyError, TypeError, IslandFailure) as error: + raise ChallengeError(503, "ROLE_DISPATCH_PROVENANCE_UNAVAILABLE") from error + n = ref.layout.n_gpus + if ( + env.get("backend") != "cuda" + or env.get("image_digest") != ref.image_digest + or len(drivers) != n + or len(set(drivers)) != 1 + or any(d not in ref.driver_allowlist for d in drivers) + or sms != [ref.sm_count] * n + or len(uuids) != n + or len(set(uuids)) != n + ): + raise ChallengeError(409, "ROLE_DISPATCH_PROVENANCE_MISMATCH") + replay = ReplayEnv( + image_digest=ref.image_digest, + driver=str(drivers[0]), + gpu_uuid_sha256=sha256_hex(",".join(uuids).encode()), + sm_count=ref.sm_count, + ) + return replay, raw + def complete_audit_v2(self, run_id: str, job_id: str, raw: bytes) -> dict[str, str]: from hypertrain.challenge.trust_v2 import ReplayEvidence from hypertrain.protocol.messages_v2 import AuditJobV2, CommitV2 @@ -7446,9 +7541,10 @@ def verified_inputs_v2(self, run_id: str, w: int) -> list[Any]: or geometry.run_id != run_id or geometry.w != w or geometry.sample_ids != audit_job.sample_ids - or geometry.start_state_sha256 != audit_job.start_state.state_object_sha256 - or geometry.ef_in_sha256 != audit_job.ef_in.sha256 - or geometry.v0_sha256 != audit_job.v0.sha256 + or geometry.start_state_sha256 + != self._whole_sha_v2(audit_job.start_state.state_object_sha256) + or geometry.ef_in_sha256 != self._whole_sha_v2(audit_job.ef_in.sha256) + or geometry.v0_sha256 != self._whole_sha_v2(audit_job.v0.sha256) ): raise ChallengeError(409, "SERVICE_INPUT_DESCRIPTOR_BINDING") raw_delta = bounded_bytes(self.objects._path(delta.delta_hash), 65536) @@ -8248,8 +8344,10 @@ def referee_v2(self, run_id: str, dispute_id: str) -> dict[str, Any]: original_start.run_id != run_id or original_start.w != turn.contest.w or original_start.hotkey != turn.contest.miner - or original_start.state_object_sha256 != preflight_job.start_state_sha256 - or original_start.ef_object_sha256 != preflight_job.ef_in_sha256 + or self._whole_sha_v2(original_start.state_object_sha256) + != preflight_job.start_state_sha256 + or self._whole_sha_v2(original_start.ef_object_sha256) + != preflight_job.ef_in_sha256 or anchor_record["hotkey"] != turn.contest.miner or anchor_record["w"] != turn.contest.w - 1 or anchor_record["anchor_hash"] != original_start.parent_anchor_hash diff --git a/src/hypertrain/data/store.py b/src/hypertrain/data/store.py index 8f69965..eac8032 100644 --- a/src/hypertrain/data/store.py +++ b/src/hypertrain/data/store.py @@ -14,12 +14,24 @@ import os import re import stat -from collections.abc import Mapping +import tempfile +from collections.abc import Callable, Mapping from pathlib import Path from typing import Protocol from urllib.parse import parse_qsl, quote, urlsplit import httpx +from pydantic import ValidationError + +from hypertrain.protocol.envelope import EnvelopeError +from hypertrain.protocol.envelope_v2 import load_json +from hypertrain.protocol.messages_v2 import ( + INDEX_MAGIC, + MAX_MANIFEST_BYTES, + MAX_PARTS, + PART_BYTES, + PartIndexV1, +) _HEX64 = re.compile(r"[0-9a-f]{64}") @@ -260,3 +272,113 @@ def presign(self, sha: str, method: str = "GET", expires: int = 3600) -> str: if method not in ("GET", "PUT"): raise ValueError("method must be GET or PUT") return sigv4_presign(method, self._url(sha), self._creds, self.region, _now(), expires) + + +# ---- ht-parts-v1: files above PART_BYTES as 1MiB parts + an index object published last ---- + +Put = Callable[[bytes], str] +Get = Callable[[str], bytes] + + +def _put(put: Put, data: bytes) -> str: + sha = hashlib.sha256(data).hexdigest() + if put(data) != sha: + raise StoreError("store returned a key that is not the content sha256") + return sha + + +def _get(get: Get, sha: str) -> bytes: + return _verified(_check_key(sha), get(sha)) + + +def parse_index(raw: bytes) -> PartIndexV1 | None: + """None if `raw` is a plain object; the index if it carries INDEX_MAGIC; else corrupt.""" + if not raw.startswith(INDEX_MAGIC): + return None + if len(raw) > MAX_MANIFEST_BYTES: + raise CorruptObjectError("part index exceeds manifest byte bound") + try: + index = PartIndexV1.model_validate(load_json(raw[len(INDEX_MAGIC) :])) + except (EnvelopeError, ValidationError) as e: + raise CorruptObjectError(f"malformed part index ({type(e).__name__})") from None + if index.encode() != raw: + raise CorruptObjectError("part index is not canonical") + return index + + +def _publish(chunks: Callable[[], bytes], put: Put) -> str: + buf, nxt = chunks(), chunks() + if not nxt: + if buf.startswith(INDEX_MAGIC): + raise StoreError("plain object must not begin with the part-index magic") + return _put(put, buf) + whole, size, parts = hashlib.sha256(), 0, list[str]() + while buf: + if len(parts) >= MAX_PARTS: + raise StoreError("object exceeds MAX_PARTS * PART_BYTES") + if nxt and len(buf) != PART_BYTES: + raise StoreError("short read before end of object") + whole.update(buf) + size += len(buf) + parts.append(_put(put, buf)) + buf, nxt = nxt, (chunks() if nxt else b"") + index = PartIndexV1(sha256=whole.hexdigest(), size=size, parts=parts) + return _put(put, index.encode()) # last: readers never see an index before its parts + + +def put_file(path: Path, put: Put) -> str: + """Store `path`; returns the key of the plain object (<= PART_BYTES) or of its part index. + The final put() call's bytes are the referenced object (len <= MAX_MANIFEST_BYTES).""" + with Path(path).open("rb") as f: + return _publish(lambda: f.read(PART_BYTES), put) + + +def put_large(data: bytes, put: Put) -> str: + view, off = memoryview(data), 0 + + def chunks() -> bytes: + nonlocal off + off += PART_BYTES + return bytes(view[off - PART_BYTES : off]) + + return _publish(chunks, put) + + +def _stream(sha: str, get: Get, sink: Callable[[bytes], object]) -> None: + raw = _get(get, sha) + index = parse_index(raw) + if index is None: + sink(raw) + return + whole, size = hashlib.sha256(), 0 + for i, part_sha in enumerate(index.parts): + part = _get(get, part_sha) + want = PART_BYTES if i < len(index.parts) - 1 else index.size - size + if len(part) != want: + raise CorruptObjectError(f"part {i} has length {len(part)}, expected {want}") + whole.update(part) + size += len(part) + sink(part) + if whole.hexdigest() != index.sha256: + raise CorruptObjectError("reassembled object does not match index sha256") + + +def get_large(sha: str, get: Get) -> bytes: + out = bytearray() + _stream(sha, get, out.extend) + return bytes(out) + + +def fetch_to(sha: str, get: Get, dest: Path) -> None: + """Verified reassembly into `dest` via same-directory temp + atomic replace.""" + dest = Path(dest) + fd, tmp = tempfile.mkstemp(dir=dest.parent, prefix=f".{dest.name}.", suffix=".part") + try: + with os.fdopen(fd, "wb") as f: + _stream(sha, get, f.write) + f.flush() + os.fsync(f.fileno()) + os.replace(tmp, dest) + except BaseException: + Path(tmp).unlink(missing_ok=True) + raise diff --git a/src/hypertrain/gpu_ops/budget.py b/src/hypertrain/gpu_ops/budget.py index 97f483f..3c86a94 100644 --- a/src/hypertrain/gpu_ops/budget.py +++ b/src/hypertrain/gpu_ops/budget.py @@ -57,9 +57,10 @@ def evaluate( egress_gb: int, phase_cap: Decimal = PHASE_A_CAP_USD, reserve: Decimal = CLEANUP_RESERVE_USD, + global_cap: Decimal = GLOBAL_CAP_USD, ) -> dict[str, Any]: hours = Decimal(hard_deadline_seconds) / Decimal(3600) - effective_cap = min(GLOBAL_CAP_USD, snapshot0_credit) + effective_cap = min(global_cap, snapshot0_credit) debits = max(Decimal(0), snapshot0_credit - current_credit) per_host = [host_worst_case(o, hours, disk_gb, egress_gb) for o in offers] hardware = sum(per_host, Decimal(0)) diff --git a/src/hypertrain/gpu_ops/launcher.py b/src/hypertrain/gpu_ops/launcher.py index a8da988..f111550 100644 --- a/src/hypertrain/gpu_ops/launcher.py +++ b/src/hypertrain/gpu_ops/launcher.py @@ -30,6 +30,9 @@ from hypertrain.gpu_ops.remote import Ssh, ensure_identity from hypertrain.gpu_ops.verdict import compute_verdict +# Exact Vast reply when this key is already on the instance; any other false reply rejects. +SSH_ALREADY_ASSOCIATED = {"success": False, "msg": "SSH key already associated with instance."} + EXIT = {"ok": 0, "admission_rejected": 2, "failed_cleaned": 3, "liability_open": 4, "refused": 5} MIN_PUT_SPACING = 5.0 PUT_ENDPOINT = "PUT /api/v0/asks/N/" @@ -45,6 +48,46 @@ def crash_hook(point: str, base: str) -> None: os._exit(137) +def network_first_billable(j: Journal) -> float | None: + """Earliest provider create; adopted hosts carry the imported, immutable original create.""" + intents = j.all("create_intent") + if intents: + return min(float(r["unix"]) for r in intents) + adopted = j.last("adopted") + return None if adopted is None else float(adopted["first_billable_unix"]) + + +def network_hard_end(j: Journal, cfg: dict[str, Any]) -> float | None: + """first + H, cut to the signed adoption cutoff when adopted hosts are billed to it.""" + first = network_first_billable(j) + if first is None: + return None + end = first + network_horizon(cfg) + adopted = j.last("adopted") + if adopted is not None and not j.all("create_intent"): + cutoff = adopted.get("cutoff_unix") + if type(cutoff) is not int or not first < cutoff <= end: + raise Reject("adoption_cutoff_invalid") + end = min(end, cutoff) + window = j.last("fresh_window") + if window is not None and j.all("create_intent"): + # Fresh rental admitted for a signed finite window: never later, never extended. + cutoff = window.get("cutoff_unix") + if type(cutoff) is not int or not first < cutoff <= end: + raise Reject("fresh_window_cutoff_invalid") + end = min(end, cutoff) + return end + + +def network_horizon(cfg: dict[str, Any]) -> int: + """Horizon is reviewed upstream (equal to the signed authorization and profile H); here it + is only bounded. Absent means the legacy one-hour lifecycle, exactly.""" + horizon = cfg.get("hard_deadline_seconds", 3600) + if type(horizon) is not int or not 0 < horizon <= 86400: + raise Reject("lifecycle_horizon_invalid") + return horizon + + class Orchestrator: def __init__( self, cfg: dict[str, Any], j: Journal, p: Provider, run_dir: Path, actor: str = "launcher" @@ -68,10 +111,9 @@ def admitted(self) -> Record | None: def deadline(self) -> float: if self.cfg.get("cleanup_contract") == "network-v2": - intents = self.j.all("create_intent") - if intents: - first = min(float(r["unix"]) for r in intents) - return first + 3600 + end = network_hard_end(self.j, self.cfg) + if end is not None: + return end rec = self.admitted() assert rec is not None return float(rec["deadline_unix"]) @@ -394,9 +436,17 @@ def attach(self, role: str) -> None: "attach-" + role, body={"ssh_key": pub}, ) - if not r.ok() or r.parsed.get("success") is not True: + already = r.ok() and r.parsed == SSH_ALREADY_ASSOCIATED + if not r.ok() or (r.parsed.get("success") is not True and not already): raise Reject(f"ssh_attach_failed:{role}:HTTP{r.status}") - self.j.append("ssh_attached", role=role, instance_id=rec["instance_id"]) + # The reply does not prove which key; trust() and the pinned SSH probe prove access. + self.j.append( + "ssh_attached", + role=role, + instance_id=rec["instance_id"], + key_sha256=sha256(pub.encode()), + already_associated=already, + ) def ssh(self, role: str) -> Ssh: st = self.j.last("instance_ready", role=role) diff --git a/src/hypertrain/gpu_ops/network_qualification.py b/src/hypertrain/gpu_ops/network_qualification.py index 7377afe..4de9a75 100644 --- a/src/hypertrain/gpu_ops/network_qualification.py +++ b/src/hypertrain/gpu_ops/network_qualification.py @@ -21,13 +21,77 @@ from pathlib import Path from typing import Annotated, Literal -from pydantic import BaseModel, ConfigDict, Field, StrictInt +from pydantic import BaseModel, ConfigDict, Field, StrictInt, model_validator from hypertrain.gpu_ops.journal import durable_write, fsha from hypertrain.gpu_ops.launcher import Reject from hypertrain.miner.island_launch import confined, launch_argv, validate_artifacts from hypertrain.protocol.messages import Hex64, ImageDigest -from hypertrain.protocol.messages_v2 import IslandJobV1 +from hypertrain.protocol.messages_v2 import IslandJobV1, RunManifestV2 + +LEGACY_TIMING = { + "horizon": 3600, + "execution_cap": 40, + "teardown": 300, + "qualification": 900, + "qual_stage": 300, + "qual_probe": 180, +} +# ponytail: finite ceiling for one prune_rescued lifecycle; raise with a reviewed profile change. +MAX_LIFECYCLE_SECONDS = 86400 + + +def lifecycle_timing(profile: dict) -> dict[str, int]: + """Legacy "all" keeps its exact one-hour constants. prune_rescued derives every window from + reviewed profile fields, each a positive int that must fit one finite lifecycle horizon.""" + a = profile.get("artifacts", {}) + if a.get("remote_retention", "all") == "all": + return dict(LEGACY_TIMING) + s, q = profile["schedule"], profile["microqualification"] + fields = [ + s["hard_deadline_seconds"], + s["staging_seconds"], + s["beacon_service_seconds"], + s["rescue_delete_seconds"], + s["hard_margin_seconds"], + s["executions_per_host"], + q["qualification_deadline_seconds"], + q["staging_seconds"], + q["probe_and_transfer_seconds"], + q["rescue_delete_seconds"], + q["failure_margin_seconds"], + a["rescue_transfer_seconds"], + a["delete_absence_seconds"], + ] + if any(type(v) is not int or v <= 0 for v in fields): + raise Reject("lifecycle_timing_invalid") + horizon, rescue_delete = s["hard_deadline_seconds"], s["rescue_delete_seconds"] + qualification = q["qualification_deadline_seconds"] + cap = (horizon - s["staging_seconds"] - s["beacon_service_seconds"] - rescue_delete) - s[ + "hard_margin_seconds" + ] + cap //= s["executions_per_host"] + if ( + horizon > MAX_LIFECYCLE_SECONDS + or a["rescue_transfer_seconds"] + a["delete_absence_seconds"] > rescue_delete + or q["rescue_delete_seconds"] < a["rescue_transfer_seconds"] + a["delete_absence_seconds"] + or q["staging_seconds"] + + q["probe_and_transfer_seconds"] + + q["rescue_delete_seconds"] + + q["failure_margin_seconds"] + > qualification + or qualification + s["hard_margin_seconds"] > horizon + or cap <= 0 + ): + raise Reject("lifecycle_timing_invalid") + return { + "horizon": horizon, + "execution_cap": cap, + "teardown": rescue_delete, + "qualification": qualification, + "qual_stage": q["staging_seconds"], + "qual_probe": q["probe_and_transfer_seconds"], + } class Boundary(BaseModel): @@ -61,10 +125,16 @@ class Environment(Boundary): cuda: Literal["13.0"] python: Literal["/opt/hypertrain/venv/bin/python"] torch_path: str - drivers: Annotated[list[str], Field(min_length=2, max_length=2)] - sm_counts: Annotated[list[Literal[170]], Field(min_length=2, max_length=2)] + drivers: Annotated[list[str], Field(min_length=2, max_length=8)] + sm_counts: Annotated[list[Literal[170]], Field(min_length=2, max_length=8)] image_digest: ImageDigest + @model_validator(mode="after") + def _one_driver_per_device(self) -> Environment: + if len(self.drivers) != len(self.sm_counts): + raise ValueError("driver/device count mismatch") + return self + class Result(Boundary): status: Literal["CUDA_ROUNDS_CAPTURED_NOT_WORKLOAD_PASS"] @@ -80,6 +150,24 @@ class Result(Boundary): workload_allowed: Literal[False] +def round_files(ranks: int, H: int, J: int) -> list[str]: # noqa: N803 - manifest names + """Every rank's final artifacts plus each J-th checkpoint of an H-step round.""" + if min(ranks, H, J) < 1 or H % J: + raise Reject("qualification_layout_invalid") + return [ + f"rank-{rank}/{rel}" + for rank in range(ranks) + for rel in ( + "state.safetensors", + "ef.safetensors", + "delta.bin", + "leaves.json", + "trace.json", + *(f"checkpoints/{t}.safetensors" for t in range(0, H + 1, J)), + ) + ] + + def select_inventory(text: str, allocated: int, drivers: list[str]) -> tuple[str, str]: """Validate physical CSV; select two full UUIDs, not CUDA qualification.""" rows = list(csv.reader(io.StringIO(text), skipinitialspace=True)) @@ -147,6 +235,7 @@ def check_inputs(cfg: Qualification) -> IslandJobV1: if fsha(cfg.lifecycle_receipts) != cfg.lifecycle_receipts_sha256: raise Reject("qualification_lifecycle_receipts_changed") receipts = json.loads(cfg.lifecycle_receipts.read_bytes()) + timing = lifecycle_timing(json.loads(cfg.profile.read_bytes())) if ( receipts["instance_id"] != cfg.instance_id or receipts["machine_id"] != cfg.machine_id @@ -156,7 +245,7 @@ def check_inputs(cfg: Qualification) -> IslandJobV1: or receipts["known_hosts_sha256"] != cfg.known_hosts_sha256 or receipts["lifecycle_contract_sha256"] != cfg.lifecycle_contract_sha256 or receipts["supervisor_pid"] <= 0 - or receipts["deadline_unix"] > cfg.admitted_unix + 3600 + or receipts["deadline_unix"] > cfg.admitted_unix + timing["horizon"] or receipts["cuda_observed"] is not True ): raise Reject("qualification_owned_lifecycle_receipt_missing") @@ -185,12 +274,12 @@ def check_inputs(cfg: Qualification) -> IslandJobV1: return job -def inspect_environment(cfg: Qualification) -> Environment: +def inspect_environment(cfg: Qualification, n_gpus: int) -> Environment: __import__("hypertrain.trainer") # Configure determinism before torch import. import torch - if not torch.cuda.is_available() or torch.cuda.device_count() != 2: - raise Reject("qualification_requires_actual_two_cuda_devices") + if not torch.cuda.is_available() or torch.cuda.device_count() != n_gpus: + raise Reject("qualification_requires_actual_layout_cuda_devices") if not Path(torch.__file__).resolve().is_relative_to("/venv/main"): raise Reject("qualification_torch_import_origin_mismatch") smi = shutil.which("nvidia-smi") @@ -203,7 +292,9 @@ def inspect_environment(cfg: Qualification) -> Environment: text=True, timeout=10, ).stdout.splitlines() - if len(drivers) != 2 or any(d.strip() not in cfg.driver_allowlist for d in drivers): + if len(drivers) != n_gpus: + raise Reject("qualification_requires_actual_layout_cuda_devices") + if any(d.strip() not in cfg.driver_allowlist for d in drivers): raise Reject("qualification_driver_mismatch") return Environment.model_validate( { @@ -214,7 +305,7 @@ def inspect_environment(cfg: Qualification) -> Environment: "torch_path": str(Path(torch.__file__).resolve()), "drivers": [d.strip() for d in drivers], "sm_counts": [ - torch.cuda.get_device_properties(i).multi_processor_count for i in range(2) + torch.cuda.get_device_properties(i).multi_processor_count for i in range(n_gpus) ], "image_digest": cfg.image_digest, } @@ -223,22 +314,13 @@ def inspect_environment(cfg: Qualification) -> Environment: def capture(directory: Path, job: IslandJobV1) -> dict[str, str]: validate_artifacts(job, directory) - fingerprints = {} - for rank in range(2): + n_gpus = job.manifest.training.reference_spec.layout.n_gpus + for rank in range(n_gpus): summary = json.loads((directory / f"rank-{rank}/summary.json").read_bytes()) if summary["backend"] != "cuda": raise Reject("qualification_cpu_artifacts_refused") - for rel in [ - "state.safetensors", - "ef.safetensors", - "delta.bin", - "leaves.json", - "trace.json", - ] + [f"checkpoints/{t}.safetensors" for t in range(0, 31, 5)]: - fingerprints[f"rank-{rank}/{rel}"] = fsha(directory / f"rank-{rank}/{rel}") - if not (directory / f"rank-{rank}/trace.json").is_file(): - raise Reject("qualification_trace_missing") - return fingerprints + inner = job.manifest.training.inner + return {rel: fsha(directory / rel) for rel in round_files(n_gpus, inner.H, inner.J)} def run(config_path: Path) -> Result: @@ -256,7 +338,8 @@ def run(config_path: Path) -> Result: os.environ[key] = value if os.environ.get("HT_IMAGE_DIGEST") != cfg.image_digest: raise Reject("qualification_actual_image_receipt_missing") - environment = inspect_environment(cfg) + environment = inspect_environment(cfg, job.manifest.training.reference_spec.layout.n_gpus) + inner = job.manifest.training.inner from hypertrain.auditor.replay import AnchorCache, pack_state, unpack_state theta, state = unpack_state( @@ -269,7 +352,9 @@ def run(config_path: Path) -> Result: anchor.ef ): raise Reject("qualification_requires_genesis_ef") - if time.time() + 600 > cfg.admitted_unix + 900: + timing = lifecycle_timing(json.loads(cfg.profile.read_bytes())) + # Legacy: 600 + 900 means staging ends by admitted + 300 (= qual_stage). + if time.time() > cfg.admitted_unix + timing["qual_stage"]: raise Reject("qualification_staging_deadline_exceeded") cfg.output.mkdir(mode=0o700, parents=True, exist_ok=False) durable_write(cfg.output / "config.json", config_path.read_bytes()) @@ -289,8 +374,11 @@ def run(config_path: Path) -> Result: for key, rel in [("start_state", "state.safetensors"), ("ef_in", "ef.safetensors")]: shutil.copyfile(previous / "rank-0" / rel, directory / job.object_paths[key]) body[key + "_sha256"] = fsha(directory / job.object_paths[key]) - body.update(w=job.w + 1, global_step0=30) - left = min(90, cfg.admitted_unix + 480 - time.time()) + body.update(w=job.w + 1, global_step0=job.global_step0 + inner.H) + left = min( + timing["qual_probe"] // 2, + cfg.admitted_unix + timing["qual_stage"] + timing["qual_probe"] - time.time(), + ) if left <= 0: raise Reject("qualification_probe_deadline_exceeded") body["deadline"] = min(job.deadline, int(time.time() + left)) @@ -340,10 +428,12 @@ def run(config_path: Path) -> Result: durable_write(cfg.output / "custody.json", json.dumps(inventory, sort_keys=True).encode()) -def compare(paths: list[Path]) -> dict[str, bool]: +def compare(paths: list[Path], manifest: RunManifestV2) -> dict[str, bool]: + """Ranks/H/J/image come from the caller's validated manifest, never from a Result.""" if len(paths) != 2: raise Reject("qualification_requires_two_host_results") a, b = [Result.model_validate_json(p.read_bytes()) for p in paths] + ref, inner = manifest.training.reference_spec, manifest.training.inner if ( a.machine_id == b.machine_id or a.instance_id == b.instance_id @@ -351,23 +441,17 @@ def compare(paths: list[Path]) -> dict[str, bool]: or a.profile_sha256 != b.profile_sha256 or a.sources != b.sources or a.environment.image_digest != b.environment.image_digest + or any( + r.environment.image_digest != ref.image_digest + or len(r.environment.sm_counts) != ref.layout.n_gpus + for r in (a, b) + ) or a.rounds != b.rounds ): raise Reject("qualification_two_host_binding_or_bitwise_mismatch") + expected = set(round_files(ref.layout.n_gpus, inner.H, inner.J)) for path, result in zip(paths, (a, b), strict=True): for index, files in enumerate(result.rounds): - expected = { - f"rank-{rank}/{rel}" - for rank in range(2) - for rel in [ - "state.safetensors", - "ef.safetensors", - "delta.bin", - "leaves.json", - "trace.json", - ] - + [f"checkpoints/{t}.safetensors" for t in range(0, 31, 5)] - } if set(files) != expected: raise Reject("qualification_full_round_coverage_missing") for rel, want in files.items(): @@ -388,9 +472,16 @@ def main() -> None: durable_write(Path(destination), bundle(Path(tree))) case ["run", config]: print(run(Path(config)).model_dump_json()) - case ["compare", a, b]: - print(json.dumps(compare([Path(a), Path(b)]))) + case ["compare", config_a, config_b, a, b]: + jobs = [] + for config, result in ((Path(config_a), Path(a)), (Path(config_b), Path(b))): + if Result.model_validate_json(result.read_bytes()).config_sha256 != fsha(config): + raise Reject("qualification_result_config_mismatch") + jobs.append(check_inputs(Qualification.model_validate_json(config.read_bytes()))) + if jobs[0].manifest != jobs[1].manifest: + raise Reject("qualification_two_host_manifest_mismatch") + print(json.dumps(compare([Path(a), Path(b)], jobs[0].manifest))) case _: raise SystemExit( - "usage: sources TREE | run CONFIG | compare A/result.json B/result.json" + "usage: sources TREE | run CONFIG | compare CFG_A CFG_B A/result.json B/result.json" ) diff --git a/src/hypertrain/gpu_ops/role_dispatch.py b/src/hypertrain/gpu_ops/role_dispatch.py new file mode 100644 index 0000000..5ab19d2 --- /dev/null +++ b/src/hypertrain/gpu_ops/role_dispatch.py @@ -0,0 +1,141 @@ +"""Production `_role_launch`: master replay dispatched to an owned, qualified remote CUDA island. + +Reuses scripts/network_service_proof.py `service_factory` (owner Receipt per operation) and the +experiments/gpu_network_v2 NetworkRuntime (debit, strict SSH, rescue custody, canonical +publication). Nothing runs locally; the config is parsed at startup, and the lifecycle journal +is read per call so a run qualified after startup needs no restart. +""" + +from __future__ import annotations + +import importlib.util +import json +import sys +import threading +import time +from collections.abc import Callable, Mapping +from pathlib import Path +from types import ModuleType +from typing import TYPE_CHECKING, Any, Literal + +from pydantic import BaseModel, ConfigDict + +from hypertrain.gpu_ops.journal import Journal, durable_write, fsha +from hypertrain.protocol.jcs import canonicalize + +if TYPE_CHECKING: + from hypertrain.gpu_ops.work_screen import IslandLaunch + from hypertrain.miner.island_launch import IslandArtifacts + from hypertrain.protocol.messages_v2 import IslandJobV1 + +RoleLaunch = Callable[[str, "IslandJobV1", Path, Mapping[str, Any]], "IslandLaunch"] + + +class DispatchConfig(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + tree: Path # reviewed source tree holding scripts/ and experiments/gpu_network_v2/ + run_dir: Path # existing admitted lifecycle journal + config.json (never created here) + owner_key_file: Path + roles: dict[str, str] # admitted hotkey -> owned host role + run_id: str # the one run this owner authorized; any other job is refused before any I/O + + +def _module(name: str, path: Path) -> ModuleType: + if name in sys.modules: + return sys.modules[name] + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"role dispatch source missing: {path}") + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +def provenance(journal: Journal, directory: Path) -> dict[str, Any]: + """Bind the rescued remote environment of THIS canonical publication into role-dispatch.json.""" + canonical = journal.last("network_canonical_done", directory=str(directory / "published")) + if canonical is None: + raise RuntimeError("role dispatch canonical publication absent") + done = journal.last("network_operation_done", role=canonical["role"], name=canonical["name"]) + result_path = directory / canonical["name"] / "operation-result.json" + if done is None or result_path.is_symlink() or fsha(result_path) != done["result_sha256"]: + raise RuntimeError("role dispatch operation result custody changed") + result = json.loads(result_path.read_bytes()) + value = { + "role": canonical["role"], + "name": canonical["name"], + "operation_result_sha256": done["result_sha256"], + "tar_sha256": done["tar_sha256"], + "environment": result["environment"], + } + path, data = directory / "role-dispatch.json", canonicalize(value) + if path.exists() or path.is_symlink(): + # Resume after a crash between this write and lease settlement: identical bytes only. + if path.is_symlink() or path.read_bytes() != data: + raise RuntimeError("role dispatch provenance already differs") + return value + durable_write(path, data) + return value + + +def load(path: Path) -> RoleLaunch: + config = DispatchConfig.model_validate_json(path.read_bytes()) + scripts = str(config.tree / "scripts") + if scripts not in sys.path: + sys.path.insert(0, scripts) + + def factory( + operation: str, job: IslandJobV1, directory: Path, context: Mapping[str, Any] + ) -> IslandLaunch: + from hypertrain.gpu_ops.launcher import Orchestrator + from hypertrain.gpu_ops.network_qualification import sources + from hypertrain.gpu_ops.provider import Provider, read_api_key + from hypertrain.miner.core import load_keyfile + + if job.run_id != config.run_id: + raise RuntimeError("role dispatch job is not the authorized run") + proof = _module("network_service_proof", config.tree / "scripts/network_service_proof.py") + orchestrate = _module( + "network_role_runtime", config.tree / "experiments/gpu_network_v2/orchestrate.py" + ) + cfg = json.loads((config.run_dir / "config.json").read_bytes()) + profile = json.loads(Path(cfg["network_profile_file"]).read_bytes()) + journal = Journal(config.run_dir) + # live=False: dispatch never creates/deletes rentals; only owned-host SSH operations. + provider = Provider( + cfg["base_url"], read_api_key(cfg["key_file"]), config.run_dir, journal, False + ) + runtime = orchestrate.NetworkRuntime( + Orchestrator(cfg, journal, provider, config.run_dir), profile + ) + pinned = sources(config.tree) + pinned["scripts/network_gpu_operation.py"] = fsha( + config.tree / "scripts/network_gpu_operation.py" + ) + beacon = job.manifest.training.beacon + launch = proof.service_factory( + proof, + runtime, + load_keyfile(config.owner_key_file), + config.run_id, + pinned, + {hotkey: {"role": role} for hotkey, role in config.roles.items()}, + lambda: int(time.time() - beacon.genesis_time) // beacon.period + 1, + )(operation, job, directory, context) + + def dispatched( + job: IslandJobV1, + directory: Path, + *, + backend: Literal["cpu", "cuda"], + cancel: threading.Event | None = None, + trace: bool = False, + ) -> IslandArtifacts: + artifacts = launch(job, directory, backend=backend, cancel=cancel, trace=trace) + provenance(journal, Path(directory)) + return artifacts + + return dispatched + + return factory diff --git a/src/hypertrain/gpu_ops/supervisor.py b/src/hypertrain/gpu_ops/supervisor.py index 74ac9a3..49dc7d5 100644 --- a/src/hypertrain/gpu_ops/supervisor.py +++ b/src/hypertrain/gpu_ops/supervisor.py @@ -17,7 +17,11 @@ from pathlib import Path from hypertrain.gpu_ops.journal import Journal -from hypertrain.gpu_ops.launcher import EXIT, Orchestrator +from hypertrain.gpu_ops.launcher import ( + EXIT, + Orchestrator, + network_hard_end, +) from hypertrain.gpu_ops.provider import Provider, read_api_key @@ -26,9 +30,9 @@ def supervise( ) -> int: tick = float(cfg.get("supervisor_check_seconds", 30)) # type: ignore[arg-type] if cfg.get("cleanup_contract") == "network-v2": - first = j.all("create_intent") - if first: - hard = min(float(r["unix"]) for r in first) + 3600 + end = network_hard_end(j, dict(cfg)) + if end is not None: + hard = end deadline = min(deadline, hard - 420) parent = j.last("transaction_open") if cfg.get("cleanup_contract") == "network-v2": @@ -47,9 +51,9 @@ def supervise( os.close(parent_fd) return 0 if cfg.get("cleanup_contract") == "network-v2": - first = j.all("create_intent") - if first: - hard = min(float(r["unix"]) for r in first) + 3600 + end = network_hard_end(j, dict(cfg)) + if end is not None: + hard = end deadline = min(deadline, hard - 420) timeout = min(tick, max(0.0, deadline - time.time())) if parent_fd is not None: @@ -108,9 +112,9 @@ def main(run_dir_arg: str) -> int: deadline = float(admitted["deadline_unix"]) hard = deadline + float(cfg.get("hard_grace_seconds", 1800)) if cfg.get("cleanup_contract") == "network-v2": - first = j.all("create_intent") - if first: - hard = min(float(r["unix"]) for r in first) + 3600 + end = network_hard_end(j, dict(cfg)) + if end is not None: + hard = end deadline = min(deadline, hard - 420) j.append("supervisor_ready", supervisor_pid=os.getpid(), deadline_unix=deadline) print("READY", flush=True) diff --git a/src/hypertrain/gpu_ops/work_screen.py b/src/hypertrain/gpu_ops/work_screen.py index 8e6c94a..430c133 100644 --- a/src/hypertrain/gpu_ops/work_screen.py +++ b/src/hypertrain/gpu_ops/work_screen.py @@ -2,12 +2,14 @@ from __future__ import annotations +import hashlib import json import threading import time from pathlib import Path from typing import Literal, Protocol +from hypertrain.data.store import Put, put_file from hypertrain.data.trial_assignment import trial_assignment_hash from hypertrain.miner.island_launch import ( IslandArtifacts, @@ -15,7 +17,7 @@ launch_island, validate_artifacts, ) -from hypertrain.protocol.hashing import MerkleTree, sha256_hex +from hypertrain.protocol.hashing import MerkleTree from hypertrain.protocol.messages import LeafPreimage from hypertrain.protocol.messages_v2 import ( ArtifactRef, @@ -46,12 +48,26 @@ def __call__( ) -> IslandArtifacts: ... +def artifact_ref(path: Path, put: Put | None = None) -> ArtifactRef: + """put_file key + length of the referenced object (plain bytes or ht-parts-v1 index). + + put=None hashes only (no upload), yielding the same ref the uploader publishes.""" + last = b"" + + def record(data: bytes) -> str: + nonlocal last + last = data + return put(data) if put is not None else hashlib.sha256(data).hexdigest() + + sha = put_file(path, record) + return ArtifactRef(sha256=sha, size=len(last)) + + def commitments(artifacts: IslandArtifacts) -> tuple[str, str]: leaves = [LeafPreimage.model_validate(v) for v in json.loads(artifacts.leaves.read_bytes())] - return ( - MerkleTree([bytes.fromhex(p.digest()) for p in leaves]).root.hex(), - sha256_hex(artifacts.delta.read_bytes()), - ) + with artifacts.delta.open("rb") as f: + delta = hashlib.file_digest(f, "sha256").hexdigest() + return MerkleTree([bytes.fromhex(p.digest()) for p in leaves]).root.hex(), delta def screen_work( @@ -102,10 +118,11 @@ def screen_work( raise WorkScreenError("CANCELLED") refs = [] for path in (artifacts.state, artifacts.ef, artifacts.delta, artifacts.leaves): - size = path.stat().st_size - if size > challenge.artifact_limits.max_object_bytes: + # Ref = stored object (plain or part index); every stored object is <= 1 MiB. + ref = artifact_ref(path) + if ref.size > challenge.artifact_limits.max_object_bytes: raise WorkScreenError("ARTIFACT_LIMIT") - refs.append(ArtifactRef(sha256=sha256_hex(path.read_bytes()), size=size)) + refs.append(ref) root, delta = commitments(artifacts) screen = WorkScreenV2( evidence_kind="work-proof-v1", diff --git a/src/hypertrain/miner/core.py b/src/hypertrain/miner/core.py index a74d9f6..2461aea 100644 --- a/src/hypertrain/miner/core.py +++ b/src/hypertrain/miner/core.py @@ -789,26 +789,100 @@ def traced_launch( self.kp, "WorkScreenV2", self.run_id, screen, challenge.deadline_beacon ), } - for filename in ("state.safetensors", "ef.safetensors", "delta.bin", "leaves.json"): + from hypertrain.gpu_ops.work_screen import artifact_ref + + put = self._bounded_put(job.deadline, cancel) + for ref, filename in zip( + proof.artifact_refs, + ("state.safetensors", "ef.safetensors", "delta.bin", "leaves.json"), + strict=True, + ): if time.time() >= job.deadline or cancel is not None and cancel.is_set(): raise MinerError("probe deadline/cancellation") path = job_path.parent / "published" / "rank-0" / filename - response = self.api.c.put( - self.api.base + self.url + "/objects/" + sha256_hex(path.read_bytes()), - content=path.read_bytes(), - headers={ - "X-Object-Signature": self.kp.sign( - b"hypertrain/object/2|" - + self.run_id.encode() - + b"|" - + sha256_hex(path.read_bytes()).encode() - ).hex(), - "X-Hotkey": self.kp.ss58, - }, - ) - response.raise_for_status() + if artifact_ref(path, put) != ref: + raise MinerError("published artifact differs from screened ref") + if time.time() >= job.deadline or cancel is not None and cancel.is_set(): + raise MinerError("probe deadline/cancellation") return result + def _bounded_put( + self, deadline: float, cancel: threading.Event | None + ) -> Callable[[bytes], str]: + """_put_object that refuses every further part/index PUT once deadline or cancel hits.""" + + def put(data: bytes) -> str: + if time.time() >= deadline or cancel is not None and cancel.is_set(): + raise MinerError("publication deadline/cancellation") + return self._put_object(data) + + return put + + def _put_object(self, data: bytes) -> str: + """One signed v2 object PUT (plain object, part or part index; each <= 1 MiB).""" + digest = sha256_hex(data) + response = self.api.c.put( + self.api.base + self.url + "/objects/" + digest, + content=iter([data]), # one-shot body: httpx keeps no request.content copy + headers={ + "X-Hotkey": self.kp.ss58, + "X-Object-Signature": self.kp.sign( + f"hypertrain/object/2|{self.run_id}|{digest}".encode() + ).hex(), + }, + ) + response.raise_for_status() + return digest + + def _fetch_inputs(self, job: IslandJobV1, objects: Mapping[str, str], directory: Path) -> None: + """objects: name -> stored ref (plain or ht-parts-v1 index); the job pins whole SHAs. + + Each file is verified and atomically replaced; a bad ref never leaves a file behind.""" + from hypertrain.data.store import StoreError, fetch_to + + whole = { + "start_state": job.start_state_sha256, + "ef_in": job.ef_in_sha256, + "v0": job.v0_sha256, + } + # Untrusted response: names must be exactly the job's validated (confined) object paths. + targets = {(directory / p).resolve() for p in job.object_paths.values()} + if set(objects) != set(job.object_paths) or len(targets) != len(job.object_paths): + raise MinerError("job object names differ from the job's object paths") + for name, digest in objects.items(): + dest = directory / job.object_paths[name] + try: + fetch_to(digest, self._get_object, dest) + except StoreError as e: + raise MinerError(f"job input {name} corrupt: {e}") from None + if name in whole: + with dest.open("rb") as f: + if hashlib.file_digest(f, "sha256").hexdigest() != whole[name]: + dest.unlink() + raise MinerError("job input hash mismatch") + + def _get_object(self, digest: str) -> bytes: + """Bounded streamed GET: ht-parts-v1 parts and indexes are <= 1 MiB each, so a larger + body is refused before it is buffered (legacy >1 MiB plain objects are not fetchable).""" + from hypertrain.protocol.messages_v2 import MAX_MANIFEST_BYTES, PART_BYTES + + limit = max(PART_BYTES, MAX_MANIFEST_BYTES) + with self.api.c.stream( + "GET", + self.api.base + self.url + "/objects/" + digest, + headers={"Accept-Encoding": "identity"}, + ) as response: + response.raise_for_status() + # httpx would decode gzip/br before the bound check; a hostile server ignores headers. + if response.headers.get("Content-Encoding", "identity").lower() != "identity": + raise MinerError("job input object has a non-identity content encoding") + body = bytearray() + for chunk in response.iter_bytes(chunk_size=64 << 10): + if len(body) + len(chunk) > limit: + raise MinerError("job input object exceeds the 1 MiB part bound") + body.extend(chunk) + return bytes(body) + def proof(self, document: Path) -> dict[str, Any]: from hypertrain.protocol.envelope_v2 import load_json @@ -850,7 +924,7 @@ def run_round(self, w: int) -> str: from hypertrain.miner.island_launch import launch_island from hypertrain.protocol import envelope_v2, relay_envelope - from hypertrain.protocol.messages_v2 import ArtifactRef, IslandJobV1, RoundOpenV2, WorkProof + from hypertrain.protocol.messages_v2 import IslandJobV1, RoundOpenV2, WorkProof from hypertrain.protocol.relay_messages import RelayRegistryV1 from hypertrain.relay.client import RelayClient, chunk_manifest @@ -911,12 +985,7 @@ def run_round(self, w: int) -> str: job = IslandJobV1.model_validate(staged["job"]) directory = self.directory / str(w) directory.mkdir(parents=True, exist_ok=True) - for name, digest in staged["objects"].items(): - response = self.api.c.get(self.api.base + self.url + "/objects/" + digest) - response.raise_for_status() - if sha256_hex(response.content) != digest: - raise MinerError("job input hash mismatch") - (directory / name).write_bytes(response.content) + self._fetch_inputs(job, staged["objects"], directory) artifacts = launch_island( job, directory, @@ -950,21 +1019,16 @@ def run_round(self, w: int) -> str: self.url + "/commit", json=envelope_v2.seal(self.kp, "CommitV2", self.run_id, commit, schedule.d_final), ) - refs = [] - for path in (artifacts.state, artifacts.ef, artifacts.delta, artifacts.leaves): - digest = sha256_hex(path.read_bytes()) - response = self.api.c.put( - self.api.base + self.url + "/objects/" + digest, - content=path.read_bytes(), - headers={ - "X-Hotkey": self.kp.ss58, - "X-Object-Signature": self.kp.sign( - f"hypertrain/object/2|{self.run_id}|{digest}".encode() - ).hex(), - }, - ) - response.raise_for_status() - refs.append(ArtifactRef(sha256=digest, size=path.stat().st_size)) + from hypertrain.gpu_ops.work_screen import artifact_ref + + # Same d_final -> wall clock mapping as the master (store.py: genesis + (d - 1) * 3). + put = self._bounded_put( + self.manifest.training.beacon.genesis_time + (schedule.d_final - 1) * 3, None + ) + refs = [ + artifact_ref(path, put) + for path in (artifacts.state, artifacts.ef, artifacts.delta, artifacts.leaves) + ] descriptor = WorkProof( admission_id=status["record"]["admission_id"], challenge_hash=sha256_hex(canonical(commit)), @@ -1003,12 +1067,11 @@ async def upload() -> RelayEnvelope: acceptance = self.api.call( "POST", "/v2/relay-receipts", content=canonical(receipt.model_dump(mode="json")) ) - payload = artifacts.delta.read_bytes() delta = { "w": w, "hotkey": self.kp.ss58, "delta_hash": commit["delta_hash"], - "size": len(payload), + "size": artifacts.delta.stat().st_size, "uri": "sha256:" + commit["delta_hash"], "format": "ht-sparse-v1" if self.manifest.training.outer.opt == "sparseloco" diff --git a/src/hypertrain/protocol/messages_v2.py b/src/hypertrain/protocol/messages_v2.py index f6758df..eb23577 100644 --- a/src/hypertrain/protocol/messages_v2.py +++ b/src/hypertrain/protocol/messages_v2.py @@ -52,6 +52,10 @@ MAX_OBJECT_BYTES = 2 << 30 MAX_BODY_BYTES = 64 << 10 MAX_MANIFEST_BYTES = 1 << 20 +# ht-parts-v1: objects above PART_BYTES are stored as parts plus a magic-prefixed JSON index. +INDEX_MAGIC = b"ht-parts-v1\n" +PART_BYTES = 1 << 20 +MAX_PARTS = 8192 class WireModel(BaseModel): @@ -362,6 +366,23 @@ class ArtifactRef(WireModel): size: Annotated[StrictInt, Field(ge=1, le=MAX_OBJECT_BYTES)] +class PartIndexV1(WireModel): + """Stored as INDEX_MAGIC + JCS(body); `sha256`/`size` describe the reassembled file.""" + + sha256: Hex64 + size: Annotated[StrictInt, Field(ge=PART_BYTES + 1, le=MAX_PARTS * PART_BYTES)] + parts: Annotated[list[Hex64], Field(min_length=2, max_length=MAX_PARTS)] + + @model_validator(mode="after") + def _count(self) -> PartIndexV1: + if len(self.parts) != -(-self.size // PART_BYTES): + raise ValueError("part count must equal ceil(size / PART_BYTES)") + return self + + def encode(self) -> bytes: + return INDEX_MAGIC + canonicalize(self.body(), allow_float=False) + + class AuditJobV2(WireModel): run_id: Hex64 job_id: Hex64 diff --git a/src/hypertrain/public_api/__init__.py b/src/hypertrain/public_api/__init__.py index f039998..fa055fd 100644 --- a/src/hypertrain/public_api/__init__.py +++ b/src/hypertrain/public_api/__init__.py @@ -55,27 +55,39 @@ def read(path: str) -> tuple[int, Any]: if hit and now - hit[0] < ttl: return hit[1], hit[2] try: - db = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) - db.row_factory = sqlite3.Row + dbs = [] try: - status, body = build(db, path) + # Root first, then each isolated run sub-store (ChallengeStore._create_child). + for p in (db_path, *sorted(db_path.parent.glob("runs/*/challenge.db"))): + db = sqlite3.connect(f"file:{p}?mode=ro", uri=True) + db.row_factory = sqlite3.Row + dbs.append(db) + status, body = build(dbs, path) finally: - db.close() + for db in dbs: + db.close() except sqlite3.Error: return 503, {"detail": "database unreadable"} cache[path] = (now, status, body) return status, body - def build(db: sqlite3.Connection, path: str) -> tuple[int, Any]: + def build(dbs: list[sqlite3.Connection], path: str) -> tuple[int, Any]: metrics = _metrics(metrics_path) if path == "/v1/runs": - return 200, views.list_runs(db) + return 200, views.list_runs(*dbs) if path == "/v1/snapshot": - snap = views.default_snapshot(db, metrics) + snap = views.default_snapshot(dbs[0], metrics, *dbs[1:]) return (200, snap) if snap else (404, {"detail": "no runs"}) run_id, _, tail = path.removeprefix("/v1/runs/").partition("/") - snap = views.run_snapshot( - db, run_id, [x for x in metrics if x.get("run_id", run_id) == run_id] + legacy = [x for x in metrics if x.get("run_id", run_id) == run_id] + tagged = [x for x in metrics if x.get("run_id") == run_id] # untagged rows are root's + db, snap = next( + ( + (d, s) + for i, d in enumerate(dbs) + if (s := views.run_snapshot(d, run_id, tagged if i else legacy)) is not None + ), + (dbs[0], None), ) if snap is None: return 404, {"detail": "unknown run"} diff --git a/src/hypertrain/public_api/views.py b/src/hypertrain/public_api/views.py index 981acdb..4b680ea 100644 --- a/src/hypertrain/public_api/views.py +++ b/src/hypertrain/public_api/views.py @@ -10,8 +10,9 @@ from hypertrain.protocol.messages import f32val -# Mirror of hypertrain.challenge.store.FAULTS (kept local: this package never imports the store). -FAULTS = frozenset({"MISMATCH", "WITHHELD", "BAD_PROOF", "ASSIGNMENT_VIOLATION"}) +# Mirror of hypertrain.challenge.store.FAULTS (kept local: this package never imports the store), +# plus EXCLUDED_V2: a v2 commit left out of its round's signed Finalize.included set. +FAULTS = frozenset({"MISMATCH", "WITHHELD", "BAD_PROOF", "ASSIGNMENT_VIOLATION", "EXCLUDED_V2"}) MAX_PARTICIPANTS = 500 _RANK = {"training": 0, "syncing": 1, "finished": 2, "offline": 3} @@ -45,8 +46,11 @@ class _Run: def __init__(self, row: sqlite3.Row) -> None: self.id: str = row["run_id"] self.status: str = row["status"] - self.m: dict[str, Any] = json.loads(row["manifest"]) - self.cfg: dict[str, Any] = json.loads(row["config"]) + manifest = json.loads(row["manifest"]) + # RunManifestV2 wraps the v1 training fields; v2 runs keep state in records_v2. + self.v2: bool = manifest.get("manifest_version") == 2 + self.m: dict[str, Any] = manifest["training"] if self.v2 else manifest + self.cfg: dict[str, Any] = json.loads(row["config"] or "{}") b = self.m["beacon"] self.genesis: int = b["genesis_time"] self.period: int = b["period"] @@ -61,18 +65,33 @@ def is_od(self) -> bool: def _runs(db: sqlite3.Connection) -> list[_Run]: out = [] - for row in db.execute("SELECT * FROM runs WHERE config IS NOT NULL ORDER BY run_id"): + for row in db.execute("SELECT * FROM runs ORDER BY run_id"): run = _Run(row) + if (row["config"] is None and not run.v2) or (run.v2 and not run.is_od): + continue # ponytail: v2 exposure is od-encoder only; widen when v2 decoders go public if run.m["outer"]["opt"] != "nesterov": continue - if db.execute("SELECT 1 FROM rounds WHERE run_id=?", (run.id,)).fetchone() is None: + if not _rounds(db, run): continue out.append(run) return out +def _records(db: sqlite3.Connection, run: _Run, kind: str) -> dict[str, Any]: + rows = db.execute("SELECT id, data FROM records_v2 WHERE run_id=? AND kind=?", (run.id, kind)) + return {r["id"]: json.loads(r["data"]) for r in rows} + + def _rounds(db: sqlite3.Connection, run: _Run) -> list[dict[str, Any]]: out = [] + if run.v2: + # Final only once the signed Finalize was replayed (never for shadow evidence). + fin = _records(db, run, "replayed-finality") + for w, env in _records(db, run, "round").items(): + f = fin.get(w) + final_at = run.chain(f["finalized_beacon"]) if f and f["shadow"] is False else None + out.append({"w": int(w), "body": env["body"], "final_at": final_at}) + return sorted(out, key=lambda r: r["w"]) for r in db.execute( "SELECT w, body, final_at FROM rounds WHERE run_id=? ORDER BY w", (run.id,) ): @@ -82,6 +101,32 @@ def _rounds(db: sqlite3.Connection, run: _Run) -> list[dict[str, Any]]: def _miners(db: sqlite3.Connection, run: _Run) -> list[dict[str, Any]]: out = [] + if run.v2: + rows: dict[tuple[int, str], dict[str, Any]] = {} + for w, env in _records(db, run, "round").items(): + for e in env["body"]["roster"]: + rows.setdefault((int(w), e["hotkey"]), {}) + for kind in ("accept", "commit"): + for key, env in _records(db, run, kind).items(): + ws, _, hk = key.partition(":") + rows.setdefault((int(ws), hk), {})[kind] = env["body"] + included = { + int(w): set(env["body"]["included"]) for w, env in _records(db, run, "finalize").items() + } + for (wi, hk), x in sorted(rows.items()): + status = "COMMITTED" if "commit" in x else "ASSIGNED" + if wi in included and hk not in included[wi]: + status = "EXCLUDED_V2" + out.append( + { + "w": wi, + "hotkey": hk, + "status": status, + "commit": x.get("commit"), + "accept": x.get("accept"), + } + ) + return out for r in db.execute( "SELECT w, hotkey, status, accept, commit_env FROM miners WHERE run_id=?", (run.id,) ): @@ -208,8 +253,9 @@ def _run_view( } -def list_runs(db: sqlite3.Connection) -> list[dict[str, Any]]: - views = [_run_view(db, r, _rounds(db, r), _miners(db, r)) for r in _runs(db)] +def list_runs(*dbs: sqlite3.Connection) -> list[dict[str, Any]]: + """Runs of the root DB plus any per-run child sub-store DBs passed after it.""" + views = [_run_view(db, r, _rounds(db, r), _miners(db, r)) for db in dbs for r in _runs(db)] views.sort(key=lambda v: v["id"]) views.sort(key=lambda v: v["startedAt"], reverse=True) return prune_none(views) @@ -226,13 +272,16 @@ def run_snapshot( def default_snapshot( - db: sqlite3.Connection, metrics: Sequence[dict[str, Any]] + db: sqlite3.Connection, metrics: Sequence[dict[str, Any]], *children: sqlite3.Connection ) -> dict[str, Any] | None: - runs = list_runs(db) + runs = list_runs(db, *children) if not runs: return None pick = next((r for r in runs if r["status"] == "running"), runs[0]) - return run_snapshot(db, pick["id"], [x for x in metrics if x.get("run_id") == pick["id"]]) + picked = [x for x in metrics if x.get("run_id") == pick["id"]] + return next( + s for d in (db, *children) if (s := run_snapshot(d, pick["id"], picked)) is not None + ) def _snapshot( @@ -286,6 +335,11 @@ def compute(w: int) -> int: (run.id,), ) ] + if run.v2: # v2 roster is the signed opening; no cluster/region is recorded + roster = [ + {"hotkey": hk, "cluster": None, "region": None} + for hk in sorted(e["hotkey"] for e in current["body"]["roster"]) + ] def last_good(hk: str) -> dict[str, Any] | None: rows = [x for x in mn if x["hotkey"] == hk and _counted(x, finals)] @@ -404,7 +458,7 @@ def pstatus(hk: str) -> str: def model_description(manifest: Mapping[str, Any]) -> dict[str, Any]: """Full architecture description from a signed manifest dict (B3 `/model`).""" run = _Run.__new__(_Run) - run.m = dict(manifest) + run.m = dict(manifest["training"] if manifest.get("manifest_version") == 2 else manifest) mod = run.m["model"] if not run.is_od: return _architecture(run) or {} diff --git a/tests/auditor/test_auditor_cli.py b/tests/auditor/test_auditor_cli.py index f57b6b9..25c1a89 100644 --- a/tests/auditor/test_auditor_cli.py +++ b/tests/auditor/test_auditor_cli.py @@ -1,12 +1,20 @@ from __future__ import annotations import json +from pathlib import Path from typing import Any import httpx +import pytest from aud_fixtures import manifest +from test_island_replay_v2 import fixture -from hypertrain.auditor.__main__ import RunScopedApi +from hypertrain.auditor import __main__ as cli +from hypertrain.auditor.__main__ import RefusedJob, RunScopedApi, audit_v2, main +from hypertrain.protocol import envelope_v2 +from hypertrain.protocol.envelope import body_digest +from hypertrain.protocol.keys import Keypair +from hypertrain.protocol.messages_v2 import RunManifestV2 def _api(run_id: str, calls: list[tuple[str, Any]]) -> RunScopedApi: @@ -36,3 +44,158 @@ def test_own_run_job_is_returned() -> None: calls: list[tuple[str, Any]] = [] raw = _api(manifest().run_id(), calls).lease() assert raw is not None and raw["id"] == "j1" and len(calls) == 1 + + +def test_contract_one_stays_default_and_requires_token_and_shards() -> None: + with pytest.raises(SystemExit) as exit_info: + main(["--api", "http://c", "--run-id", "ab" * 32, "--keyfile", "k"]) + assert exit_info.value.code == 2 + + +COORD = Keypair(bytes(range(32))) # example manifest coord_pubkey +AUDITOR = Keypair(bytes([45]) * 32) # auditor bound into the fixture job + + +@pytest.fixture(scope="module") +def job() -> Any: + return fixture()[0] + + +def _v2(job: Any, **fault: Any) -> tuple[httpx.Client, list[tuple[str, Any]]]: + run = job.run_id + verdict = { + "challenge_hash": fault.get("challenge", body_digest(job.challenge_envelope["body"])), + "first_bad_leaf": None, + "result": "MATCH", + "recomputed_leaves_root": "66" * 32, + "replay_env": { + "image_digest": job.manifest.training.reference_spec.image_digest, + "driver": "cpu", + "gpu_uuid_sha256": "77" * 32, + "sm_count": 1, + }, + } + lease = envelope_v2.seal(fault.get("signer", COORD), "AuditJobV2", run, job, 10_000) + calls: list[tuple[str, Any]] = [] + timeouts: list[float] = fault.setdefault("timeouts", []) + queue = [lease] * fault.get("jobs", 1) + + def handler(request: httpx.Request) -> httpx.Response: + path = request.url.path.removeprefix(f"/v2/runs/{run}") + body = json.loads(request.content or b"null") + calls.append((path, body)) + if path.endswith(("/execute", "/complete")): + timeouts.append(request.extensions["timeout"]["read"]) + if path == "": + served = fault.get("manifest", job.manifest) + return httpx.Response( + 200, + json={"contract": 2, "now_round": 3, "manifest_envelope": {"body": served.body()}}, + ) + if path == "/worker/lease": + if "lease" in fault: + return fault["lease"] + if queue: + return httpx.Response(200, json=queue.pop()) + empty = fault.get("empty", 204) + detail = {"detail": "SERVICE_RUNTIME_NOT_ENFORCED: no authenticated queued lease"} + return httpx.Response(empty, json=detail) + if path.endswith("/execute"): + if fault.get("stall"): + raise httpx.ReadTimeout("master stalled", request=request) + return httpx.Response(200, json={"verdict": verdict}) + assert path.endswith("/complete") and body["body"] == verdict # signs master's verdict + return httpx.Response(200, json={"result": verdict["result"]}) + + return httpx.Client(transport=httpx.MockTransport(handler)), calls + + +def _posts(calls: list[tuple[str, Any]]) -> list[str]: + return [p for p, b in calls if b is not None] + + +def test_v2_lease_execute_complete_signs_master_verdict(job: Any, capsys: Any) -> None: + client, calls = _v2(job) + assert audit_v2(client, "http://m", job.run_id, AUDITOR, once=True) == 0 + jid = job.job_id + assert _posts(calls) == [ + "/worker/lease", + f"/worker/jobs/{jid}/execute", + f"/worker/jobs/{jid}/complete", + ] + signed = [b for p, b in calls if b is not None] + assert all(envelope_v2.verify_envelope(b) and b["signer"] == AUDITOR.ss58 for b in signed) + assert [b["type"] for b in signed] == ["Receipt", "Receipt", "ReplayVerdict"] + assert signed[1]["body"]["commit_hash"] == job.lease_nonce + assert signed[0]["body"]["commit_hash"] != job.lease_nonce + assert json.loads(capsys.readouterr().out) == {"job": jid, "result": "MATCH"} + + +def test_v2_service_drains_queue_then_exits_and_once_settles_one(job: Any) -> None: + client, calls = _v2(job, jobs=2) + assert audit_v2(client, "http://m", job.run_id, AUDITOR, once=False) == 0 + assert _posts(calls).count("/worker/lease") == 3 and _posts(calls)[-1] == "/worker/lease" + client, calls = _v2(job, jobs=1, empty=503) + assert audit_v2(client, "http://m", job.run_id, AUDITOR, once=False) == 0 + client, calls = _v2(job, jobs=2) + assert audit_v2(client, "http://m", job.run_id, AUDITOR, once=True) == 0 + assert _posts(calls).count("/worker/lease") == 1 + + +@pytest.mark.parametrize( + "fault", + [ + {"signer": AUDITOR}, # lease not signed by coordinator + {"challenge": "88" * 32}, # executed verdict answers another challenge + ], +) +def test_v2_refuses_unauthenticated_job_or_foreign_verdict(job: Any, fault: Any) -> None: + client, calls = _v2(job, **fault) + with pytest.raises(RefusedJob): + audit_v2(client, "http://m", job.run_id, AUDITOR, once=True) + assert not any(p.endswith("/complete") for p in _posts(calls)) + + +def test_v2_refuses_other_auditor_and_wrong_run(job: Any) -> None: + client, calls = _v2(job) + with pytest.raises(RefusedJob): + audit_v2(client, "http://m", job.run_id, Keypair(bytes([46]) * 32), once=True) + assert _posts(calls) == ["/worker/lease"] + body = job.manifest.body() + body["training"]["init_state_hash"] = "99" * 32 + client, calls = _v2(job, manifest=RunManifestV2.model_validate(body)) + with pytest.raises(RefusedJob): + audit_v2(client, "http://m", job.run_id, AUDITOR, once=True) + assert _posts(calls) == [] + + +def test_v2_execute_bounded_by_lease_wall_deadline_complete_by_margin(job: Any) -> None: + beacon = job.manifest.training.beacon + deadline = beacon.genesis_time + (job.lease_expires - 1) * beacon.period + seen: list[float] = [] + client, _ = _v2(job, timeouts=seen) + audit_v2(client, "http://m", job.run_id, AUDITOR, once=True, clock=lambda: deadline - 30) + assert seen == [30 + cli.MARGIN_S, cli.MARGIN_S] + seen.clear() + client, _ = _v2(job, timeouts=seen) + audit_v2(client, "http://m", job.run_id, AUDITOR, once=True, clock=lambda: deadline + 999) + assert seen == [cli.MARGIN_S, cli.MARGIN_S] + + +@pytest.mark.parametrize( + "fault", + [ + {"stall": True}, + {"lease": httpx.Response(200, json={"type": "AuditJobV2", "body": "garbage"})}, + {"lease": httpx.Response(409, json={"detail": "one live lease per auditor"})}, + ], +) +def test_v2_cli_exits_2_without_complete_on_stall_malformed_or_live_lease( + job: Any, fault: Any, monkeypatch: Any, tmp_path: Path +) -> None: + client, calls = _v2(job, **fault) + monkeypatch.setattr(cli.httpx, "Client", lambda: client) + monkeypatch.setattr(cli, "load_keyfile", lambda _: AUDITOR) + argv = ["--api", "http://m", "--run-id", job.run_id, "--keyfile", str(tmp_path / "k")] + assert main([*argv, "--contract", "2"]) == 2 + assert not any(p.endswith("/complete") for p in _posts(calls)) diff --git a/tests/auditor/test_island_replay_v2.py b/tests/auditor/test_island_replay_v2.py index b049080..a2b255e 100644 --- a/tests/auditor/test_island_replay_v2.py +++ b/tests/auditor/test_island_replay_v2.py @@ -3,10 +3,13 @@ import importlib.util import threading import time +from collections.abc import Callable from pathlib import Path +from typing import Literal, NoReturn import numpy as np import pytest +import torch from pydantic import ValidationError import hypertrain.trainer # noqa: F401 @@ -20,13 +23,17 @@ replay_island_windows, ) from hypertrain.auditor.worker import LeaseGuard +from hypertrain.data.store import LocalFSStore, put_large from hypertrain.protocol.envelope_v2 import seal from hypertrain.protocol.hashing import sha256_hex from hypertrain.protocol.keys import Keypair +from hypertrain.protocol.messages import LeafPreimage, StageState from hypertrain.protocol.messages_v2 import ( + ArtifactRef, AuditChallengeV2, AuditJobV2, CommitV2, + IslandJobV1, RunManifestV2, StartStateV2, ) @@ -534,3 +541,181 @@ def test_real_parent_auditor_runtime(tmp_path: Path, state_policy: str) -> None: ) assert out.result == "MATCH" assert artifacts.delta.read_bytes() == res.delta_payload + + +class _Preflighted(Exception): + pass + + +def _indexed_ef_job( + store: LocalFSStore, +) -> tuple[AuditJobV2, AnchorCache, tuple[bytes, bytes, bytes]]: + """Untrained job whose signed StartState binds an indexed (>1 MiB) EF; lineage in cache.""" + from dataclasses import replace + + spec = importlib.util.spec_from_file_location( + "replay_l1_manifest", Path(__file__).parents[1] / "miner/test_island_launch_v2.py" + ) + assert spec and spec.loader + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + manifest = mod.tiny_manifest(2) + cfg = TrainConfig.from_manifest_v2(manifest) + key, auditor = Keypair(bytes([44]) * 32), Keypair(bytes([45]) * 32) + cache = AnchorCache() + theta = init_params(cfg.model) + genesis = cache.genesis(manifest, key.ss58, theta) + ef = {"ef": torch.arange(300_000, dtype=torch.float32)} # pack > PART_BYTES + entry = replace(genesis, ef=ef) + cache.entries[(manifest.run_id(), key.ss58, -1, entry.layout_hash)] = entry + blobs = (pack_state(theta, entry.state), pack_state(ef), pack_state({})) + refs = [put_large(b, store.put) for b in blobs] + assert refs[0] == sha256_hex(blobs[0]) and refs[1] != sha256_hex(blobs[1]) + start = StartStateV2( + run_id=manifest.run_id(), + w=0, + hotkey=key.ss58, + theta_hash=state_hash(theta), + state_object_sha256=refs[0], + opt_state_hash=optimizer_hash(entry.state), + ef_object_sha256=refs[1], + ef_hash=state_hash(ef), + parent_anchor_hash=entry.anchor_hash, + global_step0=0, + anchor_verdict_hash=entry.proof_hash, + ) + n = manifest.training.batch_samples() + commit = CommitV2( + w=0, + hotkey=key.ss58, + leaf_scheme="ht-leaf-v1", + n_leaves=cfg.inner.n_leaves, + leaves_root="11" * 32, + metrics_root="11" * 32, + final_theta_hash="11" * 32, + ef_in_hash=state_hash(ef), + ef_out_hash="11" * 32, + delta_hash="11" * 32, + delta_bytes=1, + tokens=n * cfg.model.seq_len, + ) + challenge = AuditChallengeV2( + w=0, + target=key.ss58, + beacon_round=1, + beacon_sig_sha256="22" * 32, + mode="full", + segments=[], + reasons=["final"], + serve_deadline=100, + anchor_hash=start.digest(), + audit_mode="anchored-full", + ) + leaf = LeafPreimage( + run_id=manifest.run_id(), + w=0, + t=0, + stages=[StageState(theta="11" * 32, m="11" * 32, v="11" * 32)], + batch_ids_sha256="11" * 32, + rng_ctr=0, + loss_f32="00000000", + norm_f32="00000000", + ) + job = AuditJobV2( + run_id=manifest.run_id(), + job_id="33" * 32, + auditor_id=auditor.ss58, + attempt=1, + lease_nonce="44" * 32, + lease_expires=50, + absolute_deadline=100, + reservation_id="55" * 32, + replay_step_budget=2 * cfg.inner.H, + anchor_age=0, + manifest=manifest, + challenge_envelope=seal(auditor, "AuditChallengeV2", manifest.run_id(), challenge, 100), + commit_envelope=seal(key, "CommitV2", manifest.run_id(), commit, 100), + sample_ids=list(range(n)), + start_state=start, + preimages=[leaf] * cfg.inner.n_leaves, + # ArtifactRef.size is the stored object's length: the index for indexed objects. + ef_in=ArtifactRef(sha256=refs[1], size=len(store.get(refs[1]))), + v0=ArtifactRef(sha256=refs[2], size=len(blobs[2])), + created_beacon=1, + ) + return job, cache, blobs + + +def test_indexed_ef_ref_binds_in_audit_preflight_without_training(tmp_path: Path) -> None: + from hypertrain.auditor.worker import execute_island_audit + + store = LocalFSStore(tmp_path / "objects") + job, cache, blobs = _indexed_ef_job(store) + + def preflight(j: AuditJobV2, objects: Callable[[str], bytes] | None = store.get) -> None: + def launch( + job: IslandJobV1, + directory: Path, + *, + backend: Literal["cpu", "cuda"], + cancel: threading.Event | None = None, + trace: bool = False, + ) -> NoReturn: + raise _Preflighted(job) + + directory = tmp_path / "in" + directory.mkdir(exist_ok=True) + for name, blob in zip(("start_state", "ef_in", "v0"), blobs, strict=True): + (directory / name).write_bytes(blob) + with LeaseGuard(50, 100, time.time() + 120) as guard: + execute_island_audit( + j, + directory, + cache, + guard, + now_round=2, + deadline_unix=int(time.time()) + 120, + backend="cpu", + launch=launch, + objects=objects, + ) + + with pytest.raises(_Preflighted) as accepted: + preflight(job) + local = accepted.value.args[0] + assert (local.ef_in_sha256, local.v0_sha256) == tuple(map(sha256_hex, blobs[1:])) + # Replay preflight accepts the same refs: it reaches the (emptied) anchor lookup. + theta, ef, v0 = blobs + + def replay( + j: AuditJobV2, c: AnchorCache, objects: Callable[[str], bytes] | None + ) -> None: # fails in preflight, before any collective or training step + emulate( + 2, + lambda comm: audit_island( + j, + comm, + lambda i: np.zeros(0, dtype=np.uint32), + theta, + ef, + v0, + c, + now_round=2, + objects=objects, + ), + ) + + with pytest.raises(AuditInputError, match="no independently verified predecessor"): + replay(job, AnchorCache(), store.get) + other_ref = put_large(pack_state({"ef": torch.ones(300_000)}), store.put) + cases: tuple[tuple[dict[str, object], Callable[[str], bytes] | None], ...] = ( + ({"size": len(blobs[1])}, store.get), # logical size where stored index size belongs + ({}, None), # index ref without resolving it + ({"sha256": other_ref}, store.get), # index of another file + ) + for bad, objects in cases: + wrong = job.model_copy(update={"ef_in": job.ef_in.model_copy(update=bad)}) + with pytest.raises(AuditInputError, match="untrusted audit start/state/EF"): + preflight(wrong, objects) + with pytest.raises(AuditInputError, match="EF object hash/size mismatch"): + replay(wrong, cache, objects) diff --git a/tests/challenge/test_indexed_transport_v2.py b/tests/challenge/test_indexed_transport_v2.py new file mode 100644 index 0000000..b40d3cc --- /dev/null +++ b/tests/challenge/test_indexed_transport_v2.py @@ -0,0 +1,286 @@ +"""ht-parts-v1 server transport: indexed refs on the wire, whole hashes in jobs/admission.""" + +from __future__ import annotations + +import contextlib +import hashlib +import importlib +import sys +from pathlib import Path + +import pytest + +from hypertrain.challenge.admission import _logical, _put_capped, _verify_ref +from hypertrain.challenge.admission_store import AdmissionError +from hypertrain.data.store import LocalFSStore, get_large, parse_index, put_large +from hypertrain.protocol.messages_v2 import ( + MAX_OBJECT_BYTES, + PART_BYTES, + ArtifactRef, + IslandJobV1, + PartIndexV1, + StartStateV2, +) + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "ledger")) +fixtures = importlib.import_module("test_escrow_v2") +BIG = bytes(range(256)) * ((PART_BYTES + 4096) // 256) # deterministic, > 1 MiB + + +def _sha(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def test_logical_ref_binds_stored_length_and_whole_hash(tmp_path: Path) -> None: + objects = LocalFSStore(tmp_path) + small = b"plain" + plain = ArtifactRef(sha256=objects.put(small), size=len(small)) + assert _logical(objects, plain, "X") == (_sha(small), len(small), False) + key = put_large(BIG, objects.put) + stored = len(objects.get(key)) + assert key != _sha(BIG) and stored < PART_BYTES + indexed = ArtifactRef(sha256=key, size=stored) + assert _logical(objects, indexed, "X") == (_sha(BIG), len(BIG), True) + for bad in ( + ArtifactRef(sha256=key, size=len(BIG)), # logical size is not the stored ref size + ArtifactRef(sha256=_sha(BIG), size=len(BIG)), # whole sha was never stored + ArtifactRef(sha256=plain.sha256, size=len(small) + 1), + ): + with pytest.raises(AdmissionError, match="X"): + _logical(objects, bad, "X") + + +def test_served_root_decodes_indexed_state(tmp_path: Path) -> None: + import torch + + from hypertrain.auditor.replay import pack_state, tensor_root + from hypertrain.challenge.store import ChallengeStore + from hypertrain.trainer.optim import OptState + + theta = {"w": torch.arange(300_000, dtype=torch.float32)} + st = OptState(m={"w": torch.ones(300_000)}, v={"w": torch.zeros(300_000)}, step=3) + store = ChallengeStore.__new__(ChallengeStore) + store.objects = LocalFSStore(tmp_path) + blob = put_large(pack_state(theta, st), store.objects.put) + assert parse_index(store.objects.get(blob)) is not None + assert store._served_root(blob) == tensor_root(theta, st) + + +def test_island_job_maps_indexed_refs_to_whole_hashes(tmp_path: Path, monkeypatch) -> None: + import torch + + from hypertrain.auditor.replay import pack_state, unpack_state + from hypertrain.challenge.store import ChallengeStore + from hypertrain.protocol.jcs import canonicalize + + manifest = fixtures.setup().manifest + run_id, hot = manifest.run_id(), fixtures.HOT.ss58 + objects = LocalFSStore(tmp_path / "objects") + theta = {"w": torch.arange(300_000, dtype=torch.float32)} + ef = {"w": torch.ones(300_000)} + start = StartStateV2( + run_id=run_id, + w=0, + hotkey=hot, + theta_hash="11" * 32, + state_object_sha256=put_large(pack_state(theta), objects.put), + opt_state_hash="22" * 32, + ef_object_sha256=put_large(pack_state(ef), objects.put), + ef_hash="33" * 32, + parent_anchor_hash="44" * 32, + global_step0=0, + anchor_verdict_hash="55" * 32, + ) + training = manifest.training + width = (training.model.seq_len + 1) * 4 + samples = bytes(range(256)) * (width * training.dataset.n_samples // 256 + 1) + samples = samples[: width * training.dataset.n_samples] + proofs = canonicalize([[i] for i in range(training.dataset.n_samples)]) + dataset = {"samples_hash": objects.put(samples), "proofs_hash": objects.put(proofs)} + records = { + "start": start.body(), + "round": {"body": {"d_commit": 10}}, + "dataset": dataset, + } + store = ChallengeStore.__new__(ChallengeStore) + store.objects, store.state_dir = objects, tmp_path / "state" + put: dict[str, object] = {} + monkeypatch.setattr(store, "_tx", contextlib.nullcontext, raising=False) + monkeypatch.setattr(store, "_run_v2", lambda *_: manifest, raising=False) + monkeypatch.setattr(store, "_service_boundary_v2", lambda *_a, **_k: None, raising=False) + monkeypatch.setattr(store, "_owner_v2", lambda *_: hot, raising=False) + monkeypatch.setattr(store, "_assignment_v2", lambda *_: (0,), raising=False) + monkeypatch.setattr(store, "_record_v2", lambda _r, kind, _i: records[kind], raising=False) + monkeypatch.setattr( + store, "_put_record_v2", lambda *a: put.__setitem__(a[1], a[3]), raising=False + ) + staged = store.island_job_v2(run_id, 0, hot) + job = IslandJobV1.model_validate(staged["job"]) + directory = store.state_dir / "jobs-v2" / "0" / hot + for name, field in (("start_state", "start_state_sha256"), ("ef_in", "ef_in_sha256")): + whole = (directory / name).read_bytes() + ref = staged["objects"][name] + assert getattr(job, field) == _sha(whole) != ref and len(whole) > PART_BYTES + assert parse_index(objects.get(ref)).sha256 == _sha(whole) + assert get_large(ref, objects.get) == whole + assert unpack_state((directory / "start_state").read_bytes())[0]["w"].equal(theta["w"]) + assert staged["objects"]["v0"] == job.v0_sha256 == _sha(pack_state({})) + assert put["island-job"] == job.body() + + +def _huge_index(size: int) -> tuple[bytes, list[str]]: + """Metadata-only index of `size` logical bytes; no part bytes exist or are allocated.""" + count = -(-size // PART_BYTES) + parts = [f"{i:064x}" for i in range(count)] + return PartIndexV1(sha256="ab" * 32, size=size, parts=parts).encode(), parts + + +def test_proof_cap_binds_stored_ref_not_logical_size() -> None: + raw, parts = _huge_index(MAX_OBJECT_BYTES + 3 * PART_BYTES) # logical > 2 GiB cap + key, fetched = _sha(raw), [] + + class Objects: + def get(self, sha: str) -> bytes: + if sha == key: + return raw + fetched.append(sha) + return b"not-the-part" # first part fails its content hash + + ref = ArtifactRef(sha256=key, size=len(raw)) + assert ref.size < PART_BYTES < MAX_OBJECT_BYTES + # Past the cap boundary into streamed part verification; stops at the first bad part. + with pytest.raises(AdmissionError, match="PROOF_OBJECT_HASH"): + _verify_ref(Objects(), ref, MAX_OBJECT_BYTES) + assert fetched == parts[:1] + # Every part is a stored object too: a cap below PART_BYTES rejects before any part read. + with pytest.raises(AdmissionError, match="PROOF_OBJECT_LIMIT"): + _verify_ref(Objects(), ref, PART_BYTES - 1) + with pytest.raises(AdmissionError, match="PROOF_OBJECT_LIMIT"): + _verify_ref(Objects(), ref, len(raw) - 1) + assert fetched == parts[:1] + + +def test_proof_ref_verifies_genuine_index_and_plain(tmp_path: Path) -> None: + objects = LocalFSStore(tmp_path) + key = put_large(BIG, objects.put) + ref = ArtifactRef(sha256=key, size=len(objects.get(key))) + assert _verify_ref(objects, ref, MAX_OBJECT_BYTES) == (_sha(BIG), len(BIG)) + small = ArtifactRef(sha256=objects.put(b"leaf"), size=4) + assert _verify_ref(objects, small, 4) == (_sha(b"leaf"), 4) + with pytest.raises(AdmissionError, match="PROOF_OBJECT_LIMIT"): + _verify_ref(objects, small, 3) + + +def test_whole_sha_maps_stored_ref_for_job_bindings(tmp_path: Path) -> None: + from hypertrain.challenge.store import ChallengeStore + + store = ChallengeStore.__new__(ChallengeStore) + store.objects = LocalFSStore(tmp_path) + key = put_large(BIG, store.objects.put) + plain = store.objects.put(b"v0") + assert store._whole_sha_v2(key) == _sha(BIG) != key + assert store._whole_sha_v2(plain) == plain + + +def test_shadow_anchor_binds_job_whole_bytes_and_start_refs(tmp_path: Path) -> None: + from hypertrain.auditor.replay import AnchorCache, optimizer_hash, pack_state + from hypertrain.challenge.store import ChallengeError, ChallengeStore + from hypertrain.trainer.compress import state_hash + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.model import init_params + + manifest = fixtures.setup().manifest + hot = fixtures.HOT.ss58 + theta = init_params(TrainConfig.from_manifest_v2(manifest).model) + genesis = AnchorCache().genesis(manifest, hot, theta) + start, ef = pack_state(theta, genesis.state), pack_state(genesis.ef) + (tmp_path / "start_state").write_bytes(start) + (tmp_path / "ef_in").write_bytes(ef) + job = IslandJobV1( + job_version=1, + run_id=manifest.run_id(), + w=0, + manifest=manifest, + sample_ids=list(range(manifest.training.batch_samples())), + global_step0=0, + start_state_sha256=_sha(start), + ef_in_sha256=_sha(ef), + v0_sha256=_sha(pack_state({})), + object_paths={"start_state": "start_state", "ef_in": "ef_in"}, + deadline=1, + ) + store = ChallengeStore.__new__(ChallengeStore) + expected = StartStateV2( + run_id=job.run_id, + w=0, + hotkey=hot, + theta_hash=state_hash(theta), + state_object_sha256=put_large(start, _sha), + opt_state_hash=optimizer_hash(genesis.state), + ef_object_sha256=put_large(ef, _sha), + ef_hash=state_hash(genesis.ef), + parent_anchor_hash=genesis.anchor_hash, + global_step0=0, + anchor_verdict_hash=genesis.proof_hash, + ).digest() + assert store._shadow_anchor_v2(job, hot, tmp_path) == expected + bad = job.model_copy(update={"ef_in_sha256": "00" * 32}) + with pytest.raises(ChallengeError, match="start bytes differ"): + store._shadow_anchor_v2(bad, hot, tmp_path) + + +def test_publication_cap_binds_stored_objects_not_file_size(tmp_path: Path) -> None: + from hypertrain.challenge.store import ChallengeError, ChallengeStore + + objects = LocalFSStore(tmp_path / "objects") + assert len(BIG) > PART_BYTES # logical file above the per-object cap below + key = _put_capped(objects, BIG, PART_BYTES) + assert get_large(key, objects.get) == BIG + with pytest.raises(AdmissionError, match="SHADOW_PUBLICATION_OBJECT_LIMIT"): + _put_capped(objects, BIG, PART_BYTES - 1) + with pytest.raises(AdmissionError, match="SHADOW_PUBLICATION_OBJECT_LIMIT"): + _put_capped(objects, b"abc", 2) + store = ChallengeStore.__new__(ChallengeStore) + store.objects = objects + published = tmp_path / "state.safetensors" + published.write_bytes(BIG) + store._addressed_file_v2(published) # whole file addressed through its index ref + published.write_bytes(BIG[:-1] + b"x") + with pytest.raises(ChallengeError, match="addressed publication changed"): + store._addressed_file_v2(published) + + +def test_service_audit_staging_refs_match_whole_files(tmp_path: Path) -> None: + from hypertrain.data.store import put_file + + objects = LocalFSStore(tmp_path / "objects") + key = put_large(BIG, objects.put) + anchor = tmp_path / "state" + anchor.write_bytes(BIG) + # Anchor/miner files compare by streamed ref; every stored object reads within PART_BYTES. + assert put_file(anchor, _sha) == key == put_large(BIG, _sha) + assert all( + len(objects.get(k)) <= PART_BYTES for k in [key, *parse_index(objects.get(key)).parts] + ) + anchor.write_bytes(BIG[:-1] + b"x") + assert put_file(anchor, _sha) != key + + +def test_service_audit_header_bound_admits_od_large_header() -> None: + import json + + from hypertrain.protocol.envelope_v2 import load_json + + # od-large state header size class: many tensor entries, ~104600 bytes total. + entry = {"dtype": "F32", "shape": [1024, 1024], "data_offsets": [0, 4194304]} + names = [ + f"{p}/blocks.{i}.attn.w{j}" + for p in ("theta", "m", "v") + for i in range(24) + for j in range(12) + ] + header = json.dumps({n: entry for n in names}, separators=(",", ":")).encode() + assert 65536 < len(header) <= PART_BYTES + assert len(load_json(header, max_bytes=PART_BYTES)) == len(names) + with pytest.raises(ValueError): + load_json(header, max_bytes=65536) diff --git a/tests/challenge/test_role_dispatch_consumers_v2.py b/tests/challenge/test_role_dispatch_consumers_v2.py index 2a895f4..1f6365b 100644 --- a/tests/challenge/test_role_dispatch_consumers_v2.py +++ b/tests/challenge/test_role_dispatch_consumers_v2.py @@ -440,3 +440,84 @@ def launch(local, target, *, backend, cancel, trace=False): network.store._db.execute("SELECT reference_commitment FROM admissions_v2").fetchone()[0] is None ) + + +def _dispatch_manifest(manifest, n_gpus): + ref = manifest.training.reference_spec + ref = ref.model_copy( + update={ + "layout": ref.layout.model_copy(update={"n_gpus": n_gpus, "dp_size": n_gpus}), + "driver_allowlist": ["595.71.05"], + "sm_count": 170, + } + ) + return manifest.model_copy( + update={"training": manifest.training.model_copy(update={"reference_spec": ref})} + ) + + +def _uuid(i): + return f"GPU-{i:08x}-0000-0000-0000-000000000000" + + +@pytest.mark.parametrize( + "fault", + [ + None, + "absent", + "malformed-uuid", + "zero-uuids", + "duplicate-uuid", + "trimmed-ranks", + "driver", + "mixed-driver", + "sm", + "image", + "cpu", + ], +) +def test_dispatch_replay_env_is_actual_remote_hardware(network, tmp_path, fault): + """Eight-rank manifest-derived ReplayEnv; never the zero placeholder or a reported shape.""" + from hypertrain.protocol.hashing import sha256_hex + + manifest = _dispatch_manifest(network.manifest, 8) + ref = manifest.training.reference_spec + env = { + "backend": "cuda", + "image_digest": ref.image_digest, + "drivers": ["595.71.05"] * 8, + "sm_counts": [170] * 8, + "gpu_uuids": [_uuid(i) for i in range(8)], + } + if fault == "malformed-uuid": + env["gpu_uuids"][3] = "0" * 64 + if fault == "zero-uuids": + env["gpu_uuids"] = [_uuid(0)] * 8 + if fault == "duplicate-uuid": + env["gpu_uuids"][7] = _uuid(0) + if fault == "trimmed-ranks": + env = {k: v[:2] if isinstance(v, list) else v for k, v in env.items()} + if fault == "driver": + env["drivers"] = ["580.00"] * 8 + if fault == "mixed-driver": + env["drivers"][1] = "595.71.06" + if fault == "sm": + env["sm_counts"][5] = 128 + if fault == "image": + env["image_digest"] = "sha256:" + "22" * 32 + if fault == "cpu": + env["backend"] = "cpu" + record = {"name": "op-x", "environment": env} + if fault != "absent": + (tmp_path / "role-dispatch.json").write_bytes(canonicalize(record)) + if fault is None: + replay, raw = network.store._dispatch_env_v2(manifest, tmp_path) + assert raw == record and replay.driver == "595.71.05" and replay.sm_count == 170 + assert replay.image_digest == ref.image_digest + assert replay.gpu_uuid_sha256 == sha256_hex(",".join(env["gpu_uuids"]).encode()) + assert replay.gpu_uuid_sha256 != "0" * 64 + return + status = 503 if fault in ("absent", "malformed-uuid") else 409 + with pytest.raises(ChallengeError) as error: + network.store._dispatch_env_v2(manifest, tmp_path) + assert error.value.status == status diff --git a/tests/challenge/test_subnet_gate_v2.py b/tests/challenge/test_subnet_gate_v2.py index 4380eab..b7b5476 100644 --- a/tests/challenge/test_subnet_gate_v2.py +++ b/tests/challenge/test_subnet_gate_v2.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import importlib.util import shutil import sys @@ -362,3 +363,56 @@ def test_weights_fail_closed_on_wrong_netuid(tmp_path, secrets_dir, clock): # An epoch with nothing payable has nothing to filter: 200 without the metagraph. empty = c.get("/internal/v1/get_weights?epoch=1", headers=headers) assert empty.status_code == 200 and empty.json()["weights"] == {} + + +def test_signed_object_route_over_real_http_chunked(gated, subnet_master): + """Real uvicorn socket + chunked transfer: signed v2 object PUT boundary and auth.""" + import socket + import threading + + import uvicorn + + n = gated + run_id = n.manifest.run_id() + subnet_master.registered.add(HOT[0].ss58) + assert n.join(0).status_code == 200 + refresh(n) + limit = n.setup.admission_policy.artifact_limits.max_object_bytes + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + sock.listen(16) # bound before serving: early connections wait in the backlog + server = uvicorn.Server(uvicorn.Config(n.client.app, lifespan="off", log_level="warning")) + thread = threading.Thread(target=server.run, kwargs={"sockets": [sock]}, daemon=True) + thread.start() + base = f"http://127.0.0.1:{sock.getsockname()[1]}{n.url}" + + def put(body: bytes, key=HOT[0], sha=None, signer=None, chunk=65536): + sha = sha or hashlib.sha256(body).hexdigest() + sig = (signer or key).sign(f"hypertrain/object/2|{run_id}|{sha}".encode()).hex() + parts = (body[i : i + chunk] for i in range(0, len(body), chunk)) + return httpx.put( + base + "/objects/" + sha, + content=parts, # generator body: Transfer-Encoding: chunked on the wire + headers={"x-hotkey": key.ss58, "x-object-signature": sig}, + timeout=30, + ) + + try: + exact = bytes(range(256)) * (limit // 256) + b"x" * (limit % 256) + response = put(exact) + assert response.status_code == 200, response.text + assert n.store.objects.get(response.json()["sha256"]) == exact + assert put(exact + b"y").status_code == 413 + assert put(b"tampered", sha=hashlib.sha256(b"other").hexdigest()).status_code == 422 + assert put(b"forged", signer=HOT[1]).status_code == 403 + assert put(b"unregistered", key=HOT[1]).status_code == 403 + subnet_master.registered.add(HOT[1].ss58) + refresh(n) + assert put(b"not-admitted", key=HOT[1]).status_code == 403 + sha = hashlib.sha256(b"x").hexdigest() + missing = httpx.put(base + "/objects/" + sha, content=b"x", timeout=30) + assert missing.status_code == 422 # absent x-hotkey/x-object-signature headers + finally: + server.should_exit = True + thread.join(timeout=30) + assert not thread.is_alive() diff --git a/tests/data/test_parts.py b/tests/data/test_parts.py new file mode 100644 index 0000000..a1d26ec --- /dev/null +++ b/tests/data/test_parts.py @@ -0,0 +1,213 @@ +import hashlib +import json +import os +from pathlib import Path + +import pytest + +from hypertrain.auditor.replay import pack_state +from hypertrain.data.store import ( + CorruptObjectError, + LocalFSStore, + ObjectNotFound, + StoreError, + _publish, + fetch_to, + get_large, + parse_index, + put_file, + put_large, +) +from hypertrain.protocol.messages_v2 import ( + INDEX_MAGIC, + MAX_MANIFEST_BYTES, + MAX_PARTS, + PART_BYTES, + ArtifactRef, + PartIndexV1, +) +from hypertrain.trainer.compress import CompressConfig, compress + + +class Recorder: + """LocalFSStore wrapper that records publish order.""" + + def __init__(self, root: Path) -> None: + self.store = LocalFSStore(root) + self.puts: list[bytes] = [] + + def put(self, data: bytes) -> str: + self.puts.append(data) + return self.store.put(data) + + +def _blob(n: int) -> bytes: + return os.urandom(n) + + +@pytest.mark.parametrize("n", [1, PART_BYTES - 1, PART_BYTES]) +def test_plain_unchanged(tmp_path: Path, n: int) -> None: + rec, data = Recorder(tmp_path / "s"), _blob(n) + assert put_large(data, rec.put) == hashlib.sha256(data).hexdigest() + assert rec.puts == [data] + src = tmp_path / "f" + src.write_bytes(data) + assert put_file(src, rec.put) == hashlib.sha256(data).hexdigest() + + +@pytest.mark.parametrize("n", [PART_BYTES + 1, 3 * PART_BYTES, 3 * PART_BYTES + 7]) +def test_roundtrip_index_last(tmp_path: Path, n: int) -> None: + rec, data = Recorder(tmp_path / "s"), _blob(n) + src, dest = tmp_path / "src", tmp_path / "out" / "dest" + src.write_bytes(data) + dest.parent.mkdir() + ref = put_file(src, rec.put) + index = parse_index(rec.puts[-1]) + assert index is not None and hashlib.sha256(rec.puts[-1]).hexdigest() == ref + assert index.sha256 == hashlib.sha256(data).hexdigest() and index.size == n + assert [hashlib.sha256(p).hexdigest() for p in rec.puts[:-1]] == index.parts + ArtifactRef(sha256=ref, size=len(rec.puts[-1])) + assert put_large(data, rec.put) == ref + fetch_to(ref, rec.store.get, dest) + assert dest.read_bytes() == data and get_large(ref, rec.store.get) == data + assert os.listdir(dest.parent) == ["dest"] + + +def test_real_serializers_never_look_like_index(tmp_path: Path) -> None: + import torch # after hypertrain imports: determinism setup must precede torch + + torch.manual_seed(0) + theta = {"w": torch.randn(300_000), "b": torch.randn(8)} + state = pack_state(theta) + delta, _ = compress(CompressConfig(), theta, {k: torch.zeros_like(v) for k, v in theta.items()}) + rec = Recorder(tmp_path / "s") + put_large(state, rec.put) + assert len(state) > PART_BYTES and parse_index(rec.puts[-1]) is not None + for blob in (state, delta, b"[]", b"{}", b'{"parts":[]}', INDEX_MAGIC[:-1]): + assert parse_index(blob) is None + assert get_large(put_large(blob, rec.put), rec.store.get) == blob + + +def test_plain_with_magic_prefix_rejected(tmp_path: Path) -> None: + rec = Recorder(tmp_path / "s") + with pytest.raises(StoreError): + put_large(INDEX_MAGIC + b'{"parts":[]}', rec.put) + assert rec.puts == [] + + +def _setup(tmp_path: Path, n: int = 3 * PART_BYTES + 5) -> tuple[Recorder, bytes, PartIndexV1]: + rec, data = Recorder(tmp_path / "s"), _blob(n) + put_large(data, rec.put) + index = parse_index(rec.puts[-1]) + assert index is not None + return rec, data, index + + +def _bad(rec: Recorder, index: PartIndexV1, **kw: object) -> str: + return rec.store.put(INDEX_MAGIC + json.dumps(index.body() | kw).encode()) + + +def _assert_rejected(tmp_path: Path, rec: Recorder, ref: str, exc: type[Exception]) -> None: + out = tmp_path / "out" + out.mkdir(exist_ok=True) + (out / "dest").write_bytes(b"old") + with pytest.raises(exc): + fetch_to(ref, rec.store.get, out / "dest") + assert os.listdir(out) == ["dest"] and (out / "dest").read_bytes() == b"old" + + +def test_tampered_indexes_rejected(tmp_path: Path) -> None: + rec, _, index = _setup(tmp_path) + p = index.parts + swapped = [p[1], p[0], *p[2:]] + junk = hashlib.sha256(b"x").hexdigest() + corrupt = [ + rec.store.put( + PartIndexV1.model_validate(index.body() | {"parts": swapped}).encode() + ), # reordered + rec.store.put( + PartIndexV1.model_validate(index.body() | {"size": index.size - 1}).encode() + ), # truncated size + rec.store.put( + PartIndexV1.model_validate(index.body() | {"sha256": junk}).encode() + ), # stale whole sha + rec.store.put(index.encode() + b" "), # non-canonical tail + rec.store.put(INDEX_MAGIC + b'{"parts":'), # truncated json + _bad(rec, index, extra=1), + _bad(rec, index, parts=p[:-1]), # count mismatch + _bad(rec, index, size=index.size + PART_BYTES), + rec.store.put(INDEX_MAGIC + b" " * MAX_MANIFEST_BYTES), # oversize + ] + for ref in corrupt: + _assert_rejected(tmp_path, rec, ref, CorruptObjectError) + + +def test_missing_or_dirty_part_rejected(tmp_path: Path) -> None: + rec, _, index = _setup(tmp_path) + path = rec.store._path(index.parts[1]) + ref = hashlib.sha256(index.encode()).hexdigest() + path.write_bytes(path.read_bytes()[:-1]) # dirty on-disk part + _assert_rejected(tmp_path, rec, ref, CorruptObjectError) + path.unlink() + _assert_rejected(tmp_path, rec, ref, ObjectNotFound) + + +def test_lying_get_and_put_rejected(tmp_path: Path) -> None: + rec, data, index = _setup(tmp_path) + ref = hashlib.sha256(index.encode()).hexdigest() + shorter = data[: 2 * PART_BYTES] # misleading store serving another object + with pytest.raises(CorruptObjectError): + get_large(ref, lambda sha: shorter if sha == index.parts[1] else rec.store.get(sha)) + with pytest.raises(StoreError): + put_large(data, lambda b: "0" * 64) + + +def test_cancel_mid_fetch_leaves_no_temp(tmp_path: Path) -> None: + rec, _, index = _setup(tmp_path) + ref = hashlib.sha256(index.encode()).hexdigest() + calls = 0 + + def get(sha: str) -> bytes: + nonlocal calls + calls += 1 + if calls == 3: + raise KeyboardInterrupt + return rec.store.get(sha) + + out = tmp_path / "out" + out.mkdir() + with pytest.raises(KeyboardInterrupt): + fetch_to(ref, get, out / "dest") + assert os.listdir(out) == [] + + +def test_bounds() -> None: + assert ( + len( + PartIndexV1( + sha256="0" * 64, size=MAX_PARTS * PART_BYTES, parts=["0" * 64] * MAX_PARTS + ).encode() + ) + <= MAX_MANIFEST_BYTES + ) + with pytest.raises(ValueError): + PartIndexV1(sha256="0" * 64, size=PART_BYTES, parts=["0" * 64]) + with pytest.raises(ValueError): + PartIndexV1( + sha256="0" * 64, size=MAX_PARTS * PART_BYTES + 1, parts=["0" * 64] * (MAX_PARTS + 1) + ) + + +def test_oversize_input_never_publishes_index(tmp_path: Path) -> None: + rec = Recorder(tmp_path / "s") + small = b"\0" * PART_BYTES + calls = 0 + + def chunks() -> bytes: + nonlocal calls + calls += 1 + return small if calls <= MAX_PARTS + 1 else b"" + + with pytest.raises(StoreError): + _publish(chunks, rec.put) + assert all(parse_index(b) is None for b in rec.puts) diff --git a/tests/e2e/test_network_gpu_prepare.py b/tests/e2e/test_network_gpu_prepare.py index 7b5134a..5b5728d 100644 --- a/tests/e2e/test_network_gpu_prepare.py +++ b/tests/e2e/test_network_gpu_prepare.py @@ -10,6 +10,8 @@ import subprocess import sys import tarfile +import time +import urllib.parse from pathlib import Path import pytest @@ -297,3 +299,276 @@ def test_unapproved_candidate_rejects_before_artifacts(tmp_path, freeze_path, un ) assert result.returncode != 0 and "exact approved ordered pair" in result.stderr assert not output.exists() + + +def test_prepare_qualification_base_reuses_v2_assets_without_synthetic_roles(tmp_path, freeze_path): + """prune_rescued qualification base: V2 job/profile by reference, no seeds/joins/genesis.""" + v2 = tmp_path / "v2" + (v2 / "out").mkdir(parents=True) + (v2 / "job").mkdir() + tree = Path(json.loads(freeze_path.read_bytes())["snapshot"]) / "tree" + profile = json.loads((tree / "experiments/gpu_network_v2/profile.json").read_bytes()) + profile["artifacts"] |= {"remote_retention": "prune_rescued", "disk_bytes": 429495787520} + profile["admission"] |= {"ceiling_usd": "500", "min_disk_gib": 399} + platform = tmp_path / "amd64.manifest.json" + platform.write_text( + json.dumps({"schemaVersion": 2, "config": {"size": 1}, "layers": [{"size": 2}]}) + ) + index = tmp_path / "index.manifest.json" + index.write_text( + json.dumps( + { + "schemaVersion": 2, + "manifests": [ + { + "digest": "sha256:" + hashlib.sha256(platform.read_bytes()).hexdigest(), + "platform": {"architecture": "amd64", "os": "linux"}, + } + ], + } + ) + ) + profile["runtime"]["image_digest"] = "sha256:" + hashlib.sha256(index.read_bytes()).hexdigest() + (v2 / "out/profile.json").write_text(json.dumps(profile)) + (v2 / "out/manifest-envelope.json").write_text("{}") + objects = {"start_state": "start_state", "ef_in": "ef_in", "v0": "v0"} + for rel in objects.values(): + (v2 / "job" / rel).write_bytes(rel.encode()) + (v2 / "job/job.json").write_text(json.dumps({"run_id": "ab" * 32, "object_paths": objects})) + ownership = tmp_path / "ownership-reconciled.json" + ownership.write_text( + json.dumps( + { + "api_lookup": { + "hostA": {"instance_id": 55618302, "machine_id": 147322}, + "hostB": {"instance_id": 55618304, "machine_id": 138496}, + } + } + ) + ) + output = tmp_path / "base" + result = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts/network_gpu_prepare.py"), + "--freeze", + str(freeze_path), + "--output", + str(output), + "--cpu-genesis-unix", + "1", + "--qualification-base", + str(v2 / "out"), + str(v2 / "job"), + str(ownership), + str(index), + str(platform), + ], + capture_output=True, + text=True, + timeout=120, + env=dict(os.environ, PYTHONDONTWRITEBYTECODE="1"), + ) + assert result.returncode == 0, result.stderr + prepared = json.loads(result.stdout) + assert prepared["status"] == "QUALIFICATION_BASE_NOT_LAUNCHABLE" + assert prepared["kernels_executed"] == prepared["provider_calls"] == 0 + record = json.loads((output / "private/review-base.json").read_bytes())["record"] + bundle = Path(record["files"]["bundle"]["path"]) + assert record["staging_bytes"] == bundle.stat().st_size + sum( + p.stat().st_size for p in (v2 / "job").iterdir() + ) + assert record["files"]["ownership"]["path"] == str(ownership) + assert record["files"]["registry_platform"]["path"] == str(platform) + assert record["role_inputs"]["objects"] == {k: "object_" + k for k in objects} + cfg = json.loads((output / "private/lifecycle-config.json").read_bytes()) + assert cfg["hosts"] == [ + {"role": "h0", "machine_id": 147322}, + {"role": "h1", "machine_id": 138496}, + ] + assert cfg["disk_gb"] == 399 + # Fresh admit: measured offer rows select the hosts; duplicates or wrong shape reject. + limits_gpus = profile["admission"]["gpus_per_host"] + driver = "595.71.05" + pinned = v2 / "out-fresh" + pinned.mkdir() + profile["runtime"]["driver_allowlist"] = [driver] + (pinned / "profile.json").write_text(json.dumps(profile)) + (pinned / "manifest-envelope.json").write_text("{}") + + def offer(i, machine, **change): + row = {"id": i, "machine_id": machine, "num_gpus": limits_gpus} + row |= {"gpu_name": "RTX 5090", "driver_version": driver} + return {**row, "inet_up": 800, "inet_down": 800.0, **change} + + slow = offer(2, 140854, inet_up=799.99) + no_down = {k: v for k, v in offer(2, 140854).items() if k != "inet_down"} + + for name, rows, ok in ( + ("fresh", [offer(1, 139775), offer(2, 140854)], True), + ("dup", [offer(1, 139775), offer(2, 139775)], False), + ("shape", [offer(1, 139775), offer(2, 140854, num_gpus=1)], False), + ("driver", [offer(1, 139775), offer(2, 140854, driver_version="0")], False), + ("one", [offer(1, 139775)], False), + ("not-dict", [offer(1, 139775), [140854]], False), + ("bool-id", [offer(1, 139775), offer(2, True)], False), + ("slow-up", [offer(1, 139775), slow], False), + ("missing-down", [offer(1, 139775), no_down], False), + ("nan-down", [offer(1, 139775), offer(2, 140854, inet_down=float("nan"))], False), + ("inf-up", [offer(1, 139775), offer(2, 140854, inet_up=float("inf"))], False), + ("bool-up", [offer(1, 139775), offer(2, 140854, inet_up=True)], False), + ("negative-down", [offer(1, 139775), offer(2, 140854, inet_down=-900)], False), + ("string-up", [offer(1, 139775), offer(2, 140854, inet_up="900")], False), + ): + offers = tmp_path / (name + "-offers.json") + offers.write_text(json.dumps(rows)) + argv = [sys.executable, str(ROOT / "scripts/network_gpu_prepare.py")] + argv += ["--freeze", str(freeze_path), "--output", str(tmp_path / ("base-" + name))] + argv += ["--cpu-genesis-unix", "1", "--qualification-base"] + argv += [str(pinned), str(v2 / "job"), str(offers), str(index), str(platform)] + run = subprocess.run(argv, capture_output=True, text=True, timeout=120) + assert (run.returncode == 0) is ok, run.stderr + if ok: + fresh = tmp_path / "base-fresh/private/lifecycle-config.json" + assert json.loads(fresh.read_bytes())["hosts"] == [ + {"role": "h0", "machine_id": 139775}, + {"role": "h1", "machine_id": 140854}, + ] + else: + assert "two distinct exact-shape" in run.stderr + assert {"base_url", "boot_timeout_seconds", "keyscan_attempts", "create_margin_seconds"} <= set( + cfg + ) + assert "key_file" not in cfg and "cleanup_deadline_unix" not in cfg # added at run time + # Qualification-only: no synthetic role seeds, joins, genesis, relay, TLS or service state. + assert sorted(p.name for p in (output / "private").iterdir()) == [ + "lifecycle-config.json", + "review-base.json", + ] + assert sorted(p.name for p in (output / "public").iterdir()) == [ + "lifecycle-hashes.json", + "qualification-source.tar", + "qualification-sources.json", + ] + overlay = json.loads((output / "public/qualification-sources.json").read_bytes()) + assert ( + overlay["experiments/gpu_network_v2/profile.json"] + == hashlib.sha256((v2 / "out/profile.json").read_bytes()).hexdigest() + ) + index.write_text(json.dumps({"schemaVersion": 2, "manifests": []})) + refused = subprocess.run( + [*result.args[:5], str(tmp_path / "refused"), *result.args[6:]], + capture_output=True, + text=True, + timeout=120, + env=dict(os.environ, PYTHONDONTWRITEBYTECODE="1"), + ) + assert ( + refused.returncode != 0 + and "registry index must be the pinned image digest" in refused.stderr + ) + + +def test_runtime_financial_offers_keep_every_budget_price_field(tmp_path, monkeypatch): + """GET-only financial step: offers keep the provider price fields budget.evaluate prices.""" + import importlib.util + + from hypertrain.gpu_ops import provider as provider_module + from hypertrain.gpu_ops.provider import Response + + spec = importlib.util.spec_from_file_location( + "prepare_runtime", ROOT / ".e2e/qualification/prepare_runtime.py" + ) + runtime = importlib.util.module_from_spec(spec) + spec.loader.exec_module(runtime) + profile = json.loads((ROOT / "experiments/gpu_network_v2/profile.json").read_bytes()) + profile_path = tmp_path / "profile.json" + profile_path.write_text(json.dumps(profile)) + config = tmp_path / "config.json" + hosts = [{"role": "h0", "machine_id": 11}, {"role": "h1", "machine_id": 12}] + config.write_text(json.dumps({"hosts": hosts, "network_profile_file": str(profile_path)})) + key = tmp_path / "key" + key.write_text("a" * 32) + now = int(time.time()) + row = { + "num_gpus": 2, + "gpu_name": "x", + "dph_total": 1.5, + "storage_cost": 0.4, + "inet_up_cost": 0.003, + "inet_down_cost": 0.002, + "disk_space": 400.0, + "cpu_cores_effective": 32.0, + "cpu_ram": 65536, + "driver_version": "1", + "actual_status": "running", + "start_date": now - 3600, + } + bodies = { + "/api/v0/users/current/": {"id": 7, "credit": 90.0}, + "/api/v1/instances/": { + "success": True, + "instances": [ + {**row, "id": 1, "machine_id": 11}, + {**row, "id": 2, "machine_id": 12}, + {**row, "id": 3, "machine_id": 99, "actual_status": "exited"}, + { + **row, + "id": 4, + "machine_id": 98, + "actual_status": "exited", + "cur_state": "stopped", + "storage_total_cost": 0.096, + }, + ], + }, + "/api/v0/volumes/": {"volumes": []}, + "/api/v0/charges/": {"results": [{"source": "instance-1", "amount": 2.0}]}, + } + + def call(self, method, path, tag, **kwargs): + assert method == "GET" + return Response(200, bodies[path.split("?")[0]], None, "", None) + + monkeypatch.setattr(provider_module.Provider, "call", call) + result = runtime.financial(ROOT, config, key, "test", tmp_path / "out") + offers = json.loads((tmp_path / "out/offers.json").read_bytes()) + assert [o["id"] for o in offers] == [1, 2] + for offer in offers: + for field in ("dph_total", "storage_cost", "inet_up_cost", "inet_down_cost"): + assert offer[field] == row[field] + snapshot = json.loads((tmp_path / "out/financial.json").read_bytes()) + assert snapshot["account_id"] == 7 and snapshot["available_credit_usd"] == "90.0" + horizon = sys.modules["runtime_controller"].lifecycle_timing(profile)["horizon"] + # Foreign only: unknown-state exited bills full dph; a stopped one bills storage only. + assert snapshot["active_liabilities_usd"] == f"{(1.5 + 0.096) * horizon / 3600:.6f}" + assert result["must_seal_before_unix"] == result["verified_unix"] + 30 + + # Fresh admit (no owned rows): the live ask create will use must carry up/down >= 800. + bodies["/api/v1/instances/"] = {"success": True, "instances": []} + for name, link, ok in ( + ("exact", {"inet_up": 800, "inet_down": 800.0}, True), + ("slow", {"inet_up": 800, "inet_down": 799.99}, False), + ("missing", {"inet_up": 800}, False), + ("bool", {"inet_up": True, "inet_down": 900}, False), + ): + + def fresh(self, method, path, tag, link=link, **kwargs): + assert method == "GET" + if path.startswith("/api/v0/bundles/"): + machine = json.loads(urllib.parse.parse_qs(path.split("?")[1])["q"][0]) + mid = machine["machine_id"]["eq"] + offer = {**row, "id": 100 + mid, "machine_id": mid, **link} + return Response(200, {"offers": [offer]}, None, "", None) + return Response(200, bodies[path.split("?")[0]], None, "", None) + + monkeypatch.setattr(provider_module.Provider, "call", fresh) + out = tmp_path / ("fresh-" + name) + if ok: + runtime.financial(ROOT, config, key, "test", out) + fresh_offers = json.loads((out / "offers.json").read_bytes()) + assert [o["id"] for o in fresh_offers] == [111, 112] + else: + with pytest.raises(SystemExit, match="below 800"): + runtime.financial(ROOT, config, key, "test", out) + assert not (out / "offers.json").exists() # rejected before any plan input diff --git a/tests/gpu_ops/test_network_admit_cli.py b/tests/gpu_ops/test_network_admit_cli.py index 1f98923..869dcb3 100644 --- a/tests/gpu_ops/test_network_admit_cli.py +++ b/tests/gpu_ops/test_network_admit_cli.py @@ -2,12 +2,15 @@ from __future__ import annotations +import ast import importlib.util import json import os import select +import shlex import subprocess import sys +import tarfile import time from pathlib import Path from typing import Literal @@ -1190,6 +1193,39 @@ def test_total50_budget_preserves_prior_liability(contract): cli.admission_plan(profile, evidence, authorization, snapshot, offers, qualification=True) +def test_fresh_finite_window_priced_and_bounds_supervisor_end(contract, tmp_path): + from hypertrain.gpu_ops.launcher import network_hard_end + + record, _, _ = contract + + def load(name): + return json.loads(Path(record["files"][name]["path"]).read_bytes()) + + profile, evidence, authorization = load("profile"), load("evidence"), load("authorization") + snapshot, offers = load("financial"), load("offers") + snapshot["available_credit_usd"] = "5.6" + with pytest.raises(Reject, match="cost_ceiling"): # full horizon does not fit + cli.admission_plan(profile, evidence, authorization, snapshot, offers, qualification=True) + timing = cli.lifecycle_timing(profile) + window = timing["qualification"] + profile["schedule"]["hard_margin_seconds"] + assert window < timing["horizon"] + plan = cli.admission_plan( + profile, evidence, authorization, snapshot, offers, qualification=True, priced_seconds=window + ) + assert plan["admit"] is True + j = Journal(tmp_path) + cfg = {"cleanup_contract": "network-v2", "hard_deadline_seconds": timing["horizon"]} + first = j.append("create_intent", role="h0")["unix"] + assert network_hard_end(j, cfg) == first + timing["horizon"] # no window: full H unchanged + j.append("fresh_window", admitted_unix=int(first), cutoff_unix=int(first) + window) + assert network_hard_end(j, cfg) == int(first) + window # supervisor ends at signed cutoff + j.append( + "fresh_window", admitted_unix=int(first), cutoff_unix=int(first) + timing["horizon"] + 1 + ) + with pytest.raises(Reject, match="fresh_window_cutoff_invalid"): # never extended past H + network_hard_end(j, cfg) + + @pytest.mark.parametrize( "fault", [ @@ -1560,7 +1596,7 @@ def test_qualification_failure_debits_four_and_cleans_both(contract, tmp_path, m record["action"] = "qualification" financial = Path(record["files"]["financial"]["path"]) snapshot = json.loads(financial.read_bytes()) - snapshot["instances"] = [{"id": 1}, {"id": 2}] + snapshot["instances"] = [{"id": 1, "machine_id": 5000}, {"id": 2, "machine_id": 5001}] financial.write_text(json.dumps(snapshot)) record["files"]["financial"]["sha256"] = fsha(financial) seal() @@ -1611,7 +1647,7 @@ def test_continue_dispatches_signed_graph_not_raw_jobs( record["action"] = "qualification" financial = Path(record["files"]["financial"]["path"]) snapshot = json.loads(financial.read_bytes()) - snapshot["instances"] = [{"id": 1}, {"id": 2}] + snapshot["instances"] = [{"id": 1, "machine_id": 5000}, {"id": 2, "machine_id": 5001}] financial.write_text(json.dumps(snapshot)) record["files"]["financial"]["sha256"] = fsha(financial) seal() @@ -1667,7 +1703,10 @@ def test_continue_dispatches_signed_graph_not_raw_jobs( qualification_result=f"result-{role}", qualification_receipt=f"receipt-{role}", ) - checked["financial"]["instances"] = [{"id": 1}, {"id": 2}] + checked["financial"]["instances"] = [ + {"id": 1, "machine_id": 5000}, + {"id": 2, "machine_id": 5001}, + ] graph_path = tmp_path / "graph.json" checked["run_id"] = json.loads(checked["files"]["job"].read_bytes())["run_id"] @@ -2190,7 +2229,7 @@ def put(name, value): offer["machine_id"] = i + 10 put("offers", offers) financial = json.loads(Path(files["financial"]["path"]).read_bytes()) - financial["instances"] = [{"id": 1}, {"id": 2}] + financial["instances"] = [{"id": 1, "machine_id": 10}, {"id": 2, "machine_id": 11}] put("financial", financial) cost = cli.budget.evaluate( snapshot0_credit=Decimal("20"), @@ -2660,3 +2699,458 @@ def replay(*args, **kwargs): assert built == settled == [0, 1] assert replays == ["0" * 64, f"{1:064x}"] assert lineage[1]["predecessor_tape_hash"] == lineage[0]["tape_hash"] + + +def test_task8_record_builder_seals_and_preflights(contract, tmp_path): + """Real build + owner seal + reviewed_launch on the prepared fixture; no provider calls.""" + spec = importlib.util.spec_from_file_location( + "task8_prepare_record", TREE / ".e2e/qualification/prepare_record.py" + ) + assert spec and spec.loader + builder = importlib.util.module_from_spec(spec) + spec.loader.exec_module(builder) + record, path, _ = contract + files = {k: Path(v["path"]) for k, v in record["files"].items()} + cfg = json.loads(files["config"].read_bytes()) + run_id = json.loads(files["job"].read_bytes())["run_id"] + out = tmp_path / "task8" + out.mkdir() + seed = tmp_path / "owner.seed" + seed.write_bytes(bytes(range(32))) + financial = tmp_path / "fresh-financial.json" + snapshot = json.loads(files["financial"].read_bytes()) + snapshot["instances"] = [ + {"id": 1, "machine_id": 5000}, + {"id": 2, "machine_id": 5001}, + {"id": 55620228, "machine_id": 146355}, # foreign, preserved in the signed file + ] + financial.write_text(json.dumps(snapshot)) + + def build(name, **change): + args = { + "base": record, + "files": {"financial": financial}, + "action": "qualification", + "parent_pid": os.getppid(), + "cutoff_unix": cfg["cleanup_deadline_unix"], + "admitted_unix": None, + "owner_seed": seed, + "run_id": run_id, + "beacon": 10, + "output": out / (name + ".json"), + } + return builder.build(**{**args, **change}) + + got = build("review") + assert got["action"] == "qualification" and got["adoption"] is None + assert got["review_sha256"] == fsha(out / "review.json") + sealed = json.loads((out / "review.json").read_bytes())["record"] + assert sealed["files"]["financial"] == {"path": str(financial), "sha256": fsha(financial)} + assert sealed["parent_pid"] == os.getppid() + assert sealed["quota"] == cli.execution_quota(json.loads(files["profile"].read_bytes())) + # The sealed file is the one the lifecycle CLI accepts, re-checked independently here. + assert cli.reviewed_launch(out / "review.json", OWNER.ss58, 10, "qualification") + with pytest.raises(Reject, match="reviewed_owner_receipt_invalid"): + cli.reviewed_launch(out / "review.json", OWNER.ss58, 10, "adopt") + + changed = tmp_path / "changed-config.json" + changed.write_text(json.dumps({**cfg, "disk_gb": 81})) + with pytest.raises(Reject, match="reviewed_layout_clock_or_config"): + build("bad-config", files={"financial": financial, "config": changed}) + with pytest.raises(Reject, match="persistent_controller_parent_required"): + build("bad-parent", parent_pid=os.getpid()) + with pytest.raises(SystemExit, match="cutoff"): + build("past", cutoff_unix=int(time.time()) - 1) + with pytest.raises(SystemExit, match="admitted-unix"): + build("admitted", action="continue", admitted_unix=int(time.time())) + # Adopted qualification is priced over the SAME signed window as its adopt record. + profile = json.loads(files["profile"].read_bytes()) + window = cli.lifecycle_timing(profile)["qualification"] + window += profile["schedule"]["hard_margin_seconds"] + admitted = cfg["cleanup_deadline_unix"] - window + adopted_qual = build("adopted-qual", admitted_unix=admitted) + assert adopted_qual["plan_sha256"] != got["plan_sha256"] + checked = cli.reviewed_launch(out / "adopted-qual.json", OWNER.ss58, 10, "qualification") + run = tmp_path / "run" + run.mkdir() + j = Journal(run) + j.append("transaction_open", pid=os.getppid(), live=False) + j.append( + "network_admission_authority", + owner=OWNER.ss58, + sources_sha256=fsha(checked["files"]["sources"]), + ) + with pytest.raises(Reject, match="adopted_window_mismatch"): # no adoption: no window + cli.launch_action("qualification", checked, run) + cutoff = cfg["cleanup_deadline_unix"] + for a, c in ((admitted + 1, cutoff), (admitted, cutoff + 1)): + j.append("adopted", admitted_unix=a, cutoff_unix=c) + with pytest.raises(Reject, match="adopted_window_mismatch"): + cli.launch_action("qualification", checked, run) + j.append("adopted", admitted_unix=admitted, cutoff_unix=cutoff) + plain = cli.reviewed_launch(out / "review.json", OWNER.ss58, 10, "qualification") + with pytest.raises(Reject, match="adopted_window_mismatch"): # adopted run needs its window + cli.launch_action("qualification", plain, run) + # Fresh rental: no owned rows, signed finite window required and priced. + foreign = tmp_path / "foreign-financial.json" + foreign.write_text(json.dumps({**snapshot, "instances": snapshot["instances"][2:]})) + with pytest.raises(SystemExit, match="admitted-unix"): + build("fresh-no-window", action="admit", files={"financial": foreign}) + with pytest.raises(SystemExit, match="exactly the two owned"): + build("fresh-owned", action="admit", admitted_unix=int(time.time())) + fresh = build( + "fresh", action="admit", files={"financial": foreign}, admitted_unix=int(time.time()) + ) + checked = cli.reviewed_launch(out / "fresh.json", OWNER.ss58, 10, "admit") + assert checked["record"]["admitted_unix"] == fresh["admitted_unix"] + extra = tmp_path / "extra-financial.json" + extra.write_text( + json.dumps( + {**snapshot, "instances": snapshot["instances"] + [{"id": 3, "machine_id": 5000}]} + ) + ) + with pytest.raises(SystemExit, match="exactly the two owned"): + build("extra", files={"financial": extra}) + with pytest.raises(SystemExit, match="invalid --file"): + builder.parse_files(["financial=relative.json"], "qualification") + with pytest.raises(SystemExit, match="invalid --file"): + builder.parse_files([f"secret={financial}"], "qualification") + with pytest.raises(SystemExit, match="adopt requires"): + builder.parse_files([f"financial={financial}"], "adopt") + with pytest.raises(SystemExit, match="long-lived"): + builder.check_parent(os.getpid()) + with pytest.raises(SystemExit, match="required together"): + builder.parse_files([f"join_h0={financial}"], "qualification") + + # Qualification base: no roles and no run-time config fields; both are completed here. + base_cfg = {k: v for k, v in cfg.items() if k not in ("key_file", "cleanup_deadline_unix")} + base_config = tmp_path / "base-config.json" + base_config.write_text(json.dumps(base_cfg)) + roles = record["roles"] + base = { + **{k: v for k, v in record.items() if k not in ("roles", "owner")}, + "files": { + **record["files"], + "config": {"path": str(base_config), "sha256": fsha(base_config)}, + }, + "role_inputs": {"job": roles["h0"]["job"], "objects": roles["h0"]["objects"]}, + } + role_files = { + kind + "_" + role: files[roles[role][kind]] + for kind in ("join", "seed_context") + for role in ("h0", "h1") + } + with pytest.raises(SystemExit, match="provider-key-file"): + build("no-key", base=base, files={"financial": financial, **role_files}) + got = build( + "from-base", + base=base, + files={"financial": financial, **role_files}, + provider_key=Path(cfg["key_file"]), + ) + sealed = json.loads((out / "from-base.json").read_bytes())["record"] + assert sealed["roles"]["h0"]["join"] == "join_h0" and "role_inputs" not in sealed + # The real qualification base carries no owner; the builder binds its signer. + assert sealed["owner"] == got["owner"] == record["owner"] + completed = json.loads(Path(sealed["files"]["config"]["path"]).read_bytes()) + assert completed == { + **base_cfg, + "key_file": cfg["key_file"], + "cleanup_deadline_unix": cfg["cleanup_deadline_unix"], + } + + +def test_task8_run_dir_warms_genesis_before_fresh_get(contract, tmp_path, monkeypatch): + """Measured root cause: genesis ran after the fresh GET (124s against a 30s window).""" + from hypertrain.auditor import replay + + spec = importlib.util.spec_from_file_location( + "task8_prepare_record", TREE / ".e2e/qualification/prepare_record.py" + ) + assert spec and spec.loader + builder = importlib.util.module_from_spec(spec) + spec.loader.exec_module(builder) + monkeypatch.setitem(sys.modules, "task8_controller", cli) + record, _, _ = contract + files = {k: Path(v["path"]) for k, v in record["files"].items()} + cfg = json.loads(files["config"].read_bytes()) + roles = record["roles"] + base = { + **{k: v for k, v in record.items() if k not in ("roles", "owner")}, + "role_inputs": {"job": roles["h0"]["job"], "objects": roles["h0"]["objects"]}, + } + base_path = tmp_path / "base.json" + base_path.write_text(json.dumps({"record": base})) + seed = tmp_path / "owner.seed" + seed.write_bytes(bytes(range(32))) + calls = [] + real = replay.AnchorCache.genesis + + def genesis(self, *args): + calls.append("genesis") + return real(self, *args) + + monkeypatch.setattr(replay.AnchorCache, "genesis", genesis) + snapshot = json.loads(files["financial"].read_bytes()) + snapshot["instances"] = [{"id": 1, "machine_id": 5000}, {"id": 2, "machine_id": 5001}] + age = {"seconds": 0} + + class Runtime: + @staticmethod + def financial(tree, config, key_file, message, destination): + calls.append("get") + now = int(time.time()) - age["seconds"] + stamps = dict.fromkeys(("requested_unix", "received_unix", "verified_unix"), now) + destination.mkdir() + fresh = destination / "fresh-financial.json" + fresh.write_text(json.dumps({**snapshot, **stamps})) + return {"files": [f"financial={fresh}"]} + + def launch(action, checked, run_dir, live=False): + calls.append("launch") + return {"ok": True} + + monkeypatch.setattr(builder, "load_runtime", lambda: Runtime) + monkeypatch.setattr(cli, "launch_action", launch) + + def main(name): + argv = ["prepare_record.py", "--base", str(base_path), "--action", "qualification"] + argv += ["--parent-pid", str(os.getppid())] + argv += ["--cutoff-unix", str(cfg["cleanup_deadline_unix"])] + argv += ["--owner-seed", str(seed), "--beacon", "10", "--source-message", "test"] + argv += ["--run-id", json.loads(files["job"].read_bytes())["run_id"]] + argv += ["--output", str(tmp_path / (name + ".json"))] + argv += ["--run-dir", str(tmp_path / name), "--provider-key-file", cfg["key_file"]] + for kind in ("join", "seed_context"): + for role in ("h0", "h1"): + argv += ["--file", f"{kind}_{role}={files[roles[role][kind]]}"] + monkeypatch.setattr(sys, "argv", argv) + builder.main() + + main("first") + assert calls == ["genesis", "get", "launch"] # shared seed once; nothing expensive after GET + assert cli._SEED_GENESIS_SCOPE is None # the approval died with the invocation + calls.clear() + # The external reviewed path stays fully validating: every role recomputes genesis. + owner = Keypair(bytes(range(32))).ss58 + cli.reviewed_launch(tmp_path / "first.json", owner, 10, "qualification") + assert calls == ["genesis", "genesis"] + calls.clear() + main("second") + assert calls == ["genesis", "get", "launch"] # a new invocation recomputes + + job, profile = files[roles["h0"]["job"]], files["profile"] + for target in (job, profile, files[roles["h0"]["objects"]["start_state"]]): + original = target.read_bytes() + target.write_bytes(original + b" ") + calls.clear() + with pytest.raises(Reject, match="reviewed_seed_hash"): + main("tampered-" + target.stem) + target.write_bytes(original) + assert calls == [] # rejected from bytes before any genesis and before the GET + + class Other: + __file__ = str(tmp_path / "orchestrate.py") + + monkeypatch.setitem(sys.modules, "task8_controller", Other) + with pytest.raises(SystemExit, match="not the prepared tree"): + main("other-tree") + monkeypatch.setitem(sys.modules, "task8_controller", cli) + + objects = roles["h0"]["objects"] + pinned = ( + job, + record["files"][roles["h0"]["job"]]["sha256"], + profile, + record["files"]["profile"]["sha256"], + ) + paths = {k: files[n] for k, n in objects.items()} + digests = {k: record["files"][n]["sha256"] for k, n in objects.items()} + calls.clear() + with cli.seed_genesis_scope(): + cli.seed_genesis(*pinned, roles["h0"]["hotkey"], paths, digests) + # Genesis may or may not depend on the hotkey; either way it must be recomputed. + try: + cli.seed_genesis(*pinned, Keypair(bytes(range(1, 33))).ss58, paths, digests) + except Reject as error: + assert str(error) == "reviewed_seed_full_genesis" + assert calls == ["genesis", "genesis"] # a second identity is recomputed, never reused + + age["seconds"] = 31 + calls.clear() + # The builder's own admission_plan applies the unchanged 30s rule first, before sealing. + with pytest.raises(Reject, match="inventory_liability_or_quote_unknown"): + main("stale") + assert calls == ["genesis", "get"] # the unchanged 30s rule rejects before launch + + +@pytest.mark.parametrize( + ("rc", "stdout", "expected"), + [ + (0, "match", True), + (0, "other-sha", False), # different bytes on the host + (0, "other-size", False), + (1, "", False), # missing file or read failure is never a match + (0, "", False), + ], +) +def test_qualification_staging_skips_only_exact_staged_copy(tmp_path, rc, stdout, expected): + import types + + local = tmp_path / "start_state" + local.write_bytes(b"x" * 10) + out = { + "match": f"10\n{fsha(local)} /r/start_state\n", + "other-sha": f"10\n{'0' * 64} /r/start_state\n", + "other-size": f"11\n{fsha(local)} /r/start_state\n", + }.get(stdout, stdout) + ssh = types.SimpleNamespace( + run=lambda command, tag, logs: subprocess.CompletedProcess(command, rc, out, "") + ) + assert cli.staged_match(ssh, "/r/start_state", local, tmp_path) is expected + + +@pytest.mark.parametrize( + ("profile_gpus", "host_gpus", "attested"), + [(8, 8, True), (8, 2, False), (2, 2, True), (2, 8, False)], +) +def test_qualification_probe_attests_profile_gpu_count( + contract, tmp_path, monkeypatch, profile_gpus, host_gpus, attested +): + """Capture the probe qualify_role really sends, then run it against a fake torch.""" + import hypertrain.gpu_ops.remote as remote + + record, path, _ = contract + checked = cli.reviewed_launch(path, OWNER.ss58, 10, "admit") + checked["profile"]["layout"]["n_gpus"] = profile_gpus + run = tmp_path / "probe-run" + (run / "logs").mkdir(parents=True) + j = Journal(run) + j.append("create_intent", role="h0", unix=time.time()) + orch = cli.Orchestrator( + checked["config"], j, cli.Provider("http://127.0.0.1:1", "a" * 32, run, j, False), run + ) + for method in ("boot", "attach", "trust"): + monkeypatch.setattr(orch, method, lambda role: None) + ssh = remote.Ssh( + host="127.0.0.1", + port=22, + user="root", + identity=run / "identity", + known_hosts=run / "known", + journal=j, + role="h0", + timeout=60, + ) + monkeypatch.setattr(orch, "ssh", lambda role: ssh) + puts = [] + monkeypatch.setattr( + remote.Ssh, "put", lambda self, local, remote_path: puts.append(remote_path) + ) + monkeypatch.setattr( + remote.Ssh, + "remote_sha256", + lambda self, remote_path, logs: fsha(Path(local_of[remote_path])), + ) + files = checked["files"] + inputs = checked["record"]["roles"]["h0"] + from hypertrain.protocol.messages_v2 import IslandJobV1 + + job = IslandJobV1.model_validate_json(files[inputs["job"]].read_bytes()) + root = orch.remote_root("h0") + local_of = { + root + "/source.tar": files["bundle"], + root + "/out/seed/job.json": files[inputs["job"]], + root + "/profile.json": files["profile"], + root + "/registry.json": files["registry"], + **{ + root + "/out/seed/" + job.object_paths[k]: files[v] + for k, v in inputs["objects"].items() + }, + } + probes = [] + + def run_remote(self, command, tag, logs): + if tag == "cli-import": + probes.append(shlex.split(command)[-1]) + return subprocess.CompletedProcess(command, 1, "", "") + if tag.startswith("staged-"): + target = shlex.split(command)[3] + staged = local_of[target] + if target.endswith("/" + job.object_paths["start_state"]): # exact copy staged + sized = f"{staged.stat().st_size}\n{fsha(staged)} {target}\n" + return subprocess.CompletedProcess(command, 0, sized, "") + if target.endswith("/" + job.object_paths["ef_in"]): # stale copy: re-upload + return subprocess.CompletedProcess(command, 0, f"1\n{'0' * 64} {target}\n", "") + return subprocess.CompletedProcess(command, 1, "", "no such file") + return subprocess.CompletedProcess(command, 0, "", "") + + monkeypatch.setattr(remote.Ssh, "run", run_remote) + with pytest.raises(Reject, match="actual_staged_import_attestation_failed"): + cli.qualify_role(checked, orch, cli.NetworkRuntime(orch, checked["profile"]), "h0") + [probe] = probes + seed_paths = {target for target in local_of if "/out/seed/" in target} + start = root + "/out/seed/" + job.object_paths["start_state"] + assert seed_paths - set(puts) == {start} # only the exact staged copy skipped its upload + calls = { + ast.unparse(node) + for node in ast.walk(ast.parse(probe)) + if isinstance(node, ast.Compare) and "device_count" in ast.unparse(node.left) + } + ranges = { + ast.unparse(node) + for node in ast.walk(ast.parse(probe)) + if isinstance(node, ast.Call) and ast.unparse(node.func) == "range" + } + assert calls == {f"torch.cuda.device_count() == {profile_gpus}"} + assert ranges == {f"range({profile_gpus})"} + # Execute the exact probe with a fake torch reporting host_gpus devices; nothing trains. + remote_root = tmp_path / "remote" + with tarfile.open(files["bundle"]) as archive: # the exact bytes qualify_role uploads + archive.extractall(remote_root, filter="data") + fake = tmp_path / "fake" + (fake / "torch").mkdir(parents=True) + venv = tmp_path / "venv/main" + venv.mkdir(parents=True) + (fake / "torch/__init__.py").write_text( + "import pathlib, types\n" + "__version__ = '2.14.0+cu130'\n" + "version = types.SimpleNamespace(cuda='13.0')\n" + f"cuda = types.SimpleNamespace(is_available=lambda: True, device_count=lambda: {host_gpus}," + " get_device_properties=lambda i: types.SimpleNamespace(multi_processor_count=170))\n" + # Only the determinism hooks `import hypertrain.trainer` sets; no tensor ops exist. + "_flags = {}\n" + "use_deterministic_algorithms = lambda on, **kw: _flags.update(det=on)\n" + "are_deterministic_algorithms_enabled = lambda: _flags.get('det', False)\n" + "is_deterministic_algorithms_warn_only_enabled = lambda: False\n" + "set_float32_matmul_precision = lambda p: _flags.update(precision=p)\n" + "set_num_threads = lambda n: _flags.update(threads=n)\n" + "get_num_threads = lambda: _flags.get('threads', 1)\n" + "backends = types.SimpleNamespace(cudnn=types.SimpleNamespace()," + " cuda=types.SimpleNamespace(matmul=types.SimpleNamespace()))\n" + ) + script = probe.replace(repr(str(Path(root).resolve())), repr(str(remote_root))).replace( + "'/venv/main'", repr(str(fake)) + ) + result = subprocess.run( + [sys.executable, "-c", script], + env={"PYTHONPATH": f"{fake}:{remote_root}/src", "PATH": os.environ["PATH"]}, + capture_output=True, + text=True, + timeout=60, + ) + assert (result.stdout.strip() == "IMPORT_ATTESTED") is attested, result.stderr[-400:] + + +@pytest.mark.parametrize("gpus", [0, 9, True, "8"]) +def test_qualification_rejects_malformed_gpu_count_before_ssh(contract, tmp_path, gpus): + record, path, _ = contract + checked = cli.reviewed_launch(path, OWNER.ss58, 10, "admit") + checked["profile"]["layout"]["n_gpus"] = gpus + + class NoContact: + def __getattr__(self, name): + raise AssertionError("host contacted: " + name) + + with pytest.raises(Reject, match="qualification_gpu_count_invalid"): + cli.qualify_role(checked, NoContact(), None, "h0") diff --git a/tests/gpu_ops/test_network_gpu_qualification.py b/tests/gpu_ops/test_network_gpu_qualification.py index 93b311b..c53a60b 100644 --- a/tests/gpu_ops/test_network_gpu_qualification.py +++ b/tests/gpu_ops/test_network_gpu_qualification.py @@ -4,6 +4,7 @@ import importlib.util import json +import math import os import subprocess import sys @@ -17,11 +18,13 @@ from hypertrain.gpu_ops.launcher import Reject from hypertrain.protocol.example import example_manifest from hypertrain.protocol.messages_v2 import IslandJobV1, RunManifestV2 +from hypertrain.trainer.config import TrainConfig +from hypertrain.trainer.model import param_shapes TREE = Path(__file__).parents[2] -def results(tmp_path: Path) -> list[Path]: +def results(tmp_path: Path, ranks: int = 2, H: int = 30, J: int = 5) -> list[Path]: # noqa: N803 """Two synthetic CUDA-labelled JSON sets exercise parsing, never execute GPUs.""" paths = [] for i in range(2): @@ -29,20 +32,12 @@ def results(tmp_path: Path) -> list[Path]: hashes = [] for index in range(2): root = directory / f"round-{index}/published" - root.mkdir(parents=True) files = {} - for rank in range(2): - for rel in [ - "state.safetensors", - "ef.safetensors", - "delta.bin", - "leaves.json", - "trace.json", - ] + [f"checkpoints/{t}.safetensors" for t in range(0, 31, 5)]: - path = root / f"rank-{rank}" / rel - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(bytes([index])) - files[str(path.relative_to(root))] = fsha(path) + for rel in driver.round_files(ranks, H, J): + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(bytes([index])) + files[rel] = fsha(path) hashes.append(files) result = driver.Result( status="CUDA_ROUNDS_CAPTURED_NOT_WORKLOAD_PASS", @@ -55,8 +50,8 @@ def results(tmp_path: Path) -> list[Path]: cuda="13.0", python="/opt/hypertrain/venv/bin/python", torch_path="/venv/torch/__init__.py", - drivers=["595.84", "595.84"], - sm_counts=[170, 170], + drivers=["595.84"] * ranks, + sm_counts=[170] * ranks, image_digest="sha256:" + "44" * 32, ), instance_id=i + 1, @@ -72,11 +67,40 @@ def results(tmp_path: Path) -> list[Path]: return paths +def manifest(ranks: int = 2, H: int = 30, J: int = 5) -> RunManifestV2: # noqa: N803 + """Caller-validated authority for compare(); results never supply their own shape.""" + body = example_manifest().body() + shapes = param_shapes(TrainConfig.from_manifest(example_manifest()).model) + body["model"]["param_count"] = sum(math.prod(s) for s in shapes.values()) + body["inner"].update(H=H, J=J) + body["reference_spec"].update(image_digest="sha256:" + "44" * 32) + body["reference_spec"]["layout"].update(n_gpus=ranks, dp_size=ranks, ep_size=1) + network = { + **{ + k: "11" * 32 + for k in ( + "admission_policy_hash", + "economics_policy_hash", + "aggregation_policy_hash", + "dispute_policy_hash", + "audit_policy_hash", + "relay_registry_hash", + ) + }, + "audit_mode": "anchored-full", + "full_anchor_version": 1, + "capabilities": ["island-replay", "all-level-disputes", "transport-receipts"], + } + return RunManifestV2.model_validate( + {"manifest_version": 2, "training": body, "network": network} + ) + + def test_bitwise_comparison_preserves_workload_gate(tmp_path: Path) -> None: # Given paths = results(tmp_path) # When - result = driver.compare(paths) + result = driver.compare(paths, manifest()) # Then assert result == {"two_host_bitwise_match": True, "workload_allowed": False} @@ -87,7 +111,7 @@ def test_tampered_artifact_rejects_claimed_matching_hashes(tmp_path: Path) -> No (paths[1].parent / "round-1/published/rank-0/state.safetensors").write_bytes(b"tampered") # When / Then with pytest.raises(Reject, match="custody_changed"): - driver.compare(paths) + driver.compare(paths, manifest()) def test_same_machine_rejects_two_host_claim(tmp_path: Path) -> None: @@ -98,7 +122,7 @@ def test_same_machine_rejects_two_host_claim(tmp_path: Path) -> None: paths[1].write_text(json.dumps(data)) # When / Then with pytest.raises(Reject, match="binding_or_bitwise"): - driver.compare(paths) + driver.compare(paths, manifest()) @pytest.mark.parametrize( @@ -157,12 +181,122 @@ def test_partial_round_coverage_rejects_even_identical_results(tmp_path: Path) - path.write_text(json.dumps(data)) # When / Then with pytest.raises(Reject, match="coverage_missing"): - driver.compare(paths) + driver.compare(paths, manifest()) + + +def test_eight_gpu_h4_j2_results_compare_with_full_rank_coverage(tmp_path: Path) -> None: + # Given + paths = results(tmp_path, ranks=8, H=4, J=2) + # When + result = driver.compare(paths, manifest(8, 4, 2)) + # Then: 8 ranks x (5 finals + checkpoints 0,2,4), workload gate unchanged. + rounds = json.loads(paths[0].read_bytes())["rounds"] + assert result == {"two_host_bitwise_match": True, "workload_allowed": False} + assert [len(r) for r in rounds] == [64, 64] and "rank-7/checkpoints/4.safetensors" in rounds[1] + + +@pytest.mark.parametrize("attack", ["missing_rank", "same_host", "tamper", "wrong_image"]) +def test_eight_gpu_results_reject_rank_host_custody_and_shape_attacks( + tmp_path: Path, attack: str +) -> None: + # Given + paths = results(tmp_path, ranks=8, H=4, J=2) + datas = [json.loads(p.read_bytes()) for p in paths] + if attack == "missing_rank": # Both hosts agree on a 7-rank capture. + for data in datas: + data["rounds"] = [ + {k: v for k, v in r.items() if not k.startswith("rank-7/")} for r in data["rounds"] + ] + elif attack == "same_host": + datas[1]["machine_id"] = datas[0]["machine_id"] + elif attack == "wrong_image": # Both hosts agree on an image the manifest never named. + for data in datas: + data["environment"]["image_digest"] = "sha256:" + "55" * 32 + for path, data in zip(paths, datas, strict=True): + path.write_text(json.dumps(data)) + if attack == "tamper": + (paths[1].parent / "round-1/published/rank-7/checkpoints/2.safetensors").write_bytes(b"x") + match = {"missing_rank": "coverage_missing", "tamper": "custody_changed"} + # When / Then + with pytest.raises(Reject, match=match.get(attack, "binding_or_bitwise")): + driver.compare(paths, manifest(8, 4, 2)) + + +@pytest.mark.parametrize( + "attack", ["two_ranks_j4", "eight_ranks_j4", "legacy_j15", "no_checkpoints"] +) +def test_coordinated_self_consistent_results_cannot_redefine_layout( + tmp_path: Path, attack: str +) -> None: + # Given: both hosts agree bitwise on a shape the 8-GPU H4/J2 seed manifest never named. + shape = {"two_ranks_j4": (2, 4, 4), "eight_ranks_j4": (8, 4, 4), "legacy_j15": (8, 30, 15)} + paths = results(tmp_path, *shape.get(attack, (8, 4, 2))) + if attack == "no_checkpoints": # What a J=-1 capture listed: finals only. + for path in paths: + data = json.loads(path.read_bytes()) + data["rounds"] = [ + {k: v for k, v in r.items() if "/checkpoints/" not in k} for r in data["rounds"] + ] + path.write_text(json.dumps(data)) + # When / Then: the manifest, not the results, defines ranks and checkpoints. + with pytest.raises(Reject, match="binding_or_bitwise|coverage_missing"): + driver.compare(paths, manifest(8, 4, 2)) + + +@pytest.mark.parametrize("shape", [(8, 30, 0), (8, 30, -1), (8, 30, 7), (0, 4, 2)]) +def test_round_files_refuses_non_positive_or_non_dividing_shape( + shape: tuple[int, int, int], +) -> None: + # When / Then + with pytest.raises(Reject, match="layout_invalid"): + driver.round_files(*shape) + + +def test_environment_rejects_driver_count_differing_from_devices(tmp_path: Path) -> None: + # Given + data = json.loads(results(tmp_path, ranks=8, H=4, J=2)[0].read_bytes())["environment"] + data["drivers"] = data["drivers"][:7] + # When / Then + with pytest.raises(ValidationError, match="driver/device count"): + driver.Environment.model_validate(data) + + +def test_eight_gpu_seed_manifest_drives_rank_and_checkpoint_coverage( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # Given: exact profile with an 8-GPU dp4 x ep2 layout and H4/J2 inner loop. + profile = json.loads((TREE / "experiments/gpu_network_v2/profile.json").read_bytes()) + profile["layout"].update(n_gpus=8, dp_size=4, ep_size=2) + profile["inner"].update(H=4, J=2) + profile_path = tmp_path / "profile-8.json" + profile_path.write_text(json.dumps(profile)) + job = driver.check_inputs(seed_config(tmp_path, profile_path)) + published = tmp_path / "published" + for rel in driver.round_files(8, 4, 2): + (published / rel).parent.mkdir(parents=True, exist_ok=True) + (published / rel).write_bytes(rel.encode()) + for rank in range(8): + (published / f"rank-{rank}/summary.json").write_text('{"backend": "cuda"}') + monkeypatch.setattr(driver, "validate_artifacts", lambda job, directory: None) + # When + captured = driver.capture(published, job) + # Then + assert len(job.sample_ids) == 32 and set(captured) == set(driver.round_files(8, 4, 2)) + (published / "rank-7/checkpoints/4.safetensors").unlink() + with pytest.raises(FileNotFoundError): + driver.capture(published, job) def test_exact_profile_accepts_wire_defaults_without_training(tmp_path: Path) -> None: # Given - profile_path = TREE / "experiments/gpu_network_v2/profile.json" + cfg = seed_config(tmp_path, TREE / "experiments/gpu_network_v2/profile.json") + # When + accepted = driver.check_inputs(cfg) + # Then + assert accepted.run_id == json.loads(cfg.seed_job.read_bytes())["run_id"] + + +def seed_config(tmp_path: Path, profile_path: Path) -> driver.Qualification: profile = json.loads(profile_path.read_bytes()) body = example_manifest().body() body["model"].update(profile["model"]) @@ -201,7 +335,7 @@ def test_exact_profile_accepts_wire_defaults_without_training(tmp_path: Path) -> run_id=wrapper.run_id(), w=0, manifest=wrapper, - sample_ids=list(range(60)), + sample_ids=list(range(wrapper.training.batch_samples())), global_step0=0, start_state_sha256="22" * 32, ef_in_sha256="33" * 32, @@ -234,7 +368,7 @@ def test_exact_profile_accepts_wire_defaults_without_training(tmp_path: Path) -> } ) ) - cfg = driver.Qualification( + return driver.Qualification( tree=TREE, profile=profile_path, profile_sha256=fsha(profile_path), @@ -254,10 +388,6 @@ def test_exact_profile_accepts_wire_defaults_without_training(tmp_path: Path) -> lifecycle_receipts=receipts, lifecycle_receipts_sha256=fsha(receipts), ) - # When - accepted = driver.check_inputs(cfg) - # Then - assert accepted.run_id == wrapper.run_id() def test_script_reexports_exact_packaged_engine() -> None: diff --git a/tests/gpu_ops/test_network_lifecycle.py b/tests/gpu_ops/test_network_lifecycle.py index 81d61b6..bfe2033 100644 --- a/tests/gpu_ops/test_network_lifecycle.py +++ b/tests/gpu_ops/test_network_lifecycle.py @@ -14,10 +14,11 @@ import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path +from types import SimpleNamespace import pytest -from hypertrain.gpu_ops.journal import Journal +from hypertrain.gpu_ops.journal import Journal, fsha from hypertrain.gpu_ops.launcher import Orchestrator, Reject from hypertrain.gpu_ops.provider import Provider from hypertrain.gpu_ops.remote import ensure_identity @@ -337,7 +338,6 @@ def test_exact_qualification_reservations_and_immutable_deadline(tmp_path: Path) def test_qualification_entrypoint_consumes_frozen_schema( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - from types import SimpleNamespace from hypertrain.gpu_ops.journal import fsha @@ -565,3 +565,1029 @@ def test_promotion_requires_real_qualification_and_remaining_original_clock( {"root_reviewed_two_host_full_round_transfer": True}, {"admit": True, "long_workload_allowed": True}, ) + + +def od_large_profile(per_host: int) -> dict: + """Decoder profile with the od-large/8-GPU geometry of the run manifest (task 6 shape).""" + profile = json.loads((TREE / "experiments/gpu_network_v2/profile.json").read_bytes()) + profile["model"] |= { + "arch": "od-encoder", + "n_layers": 24, + "d_model": 1024, + "n_heads": 16, + "n_kv_heads": 16, + "d_ff": 4096, + "n_experts": 1, + "top_k_experts": 1, + "vocab": 50432, + "seq_len": 1024, + "param_count": 353861632, + "od": { + "preset": "od-large", + "head_layers": 4, + "objective": "mlm", + "mask_ratio": "9a99993e", + "mask_seed": 7, + "tokenizer_offset": 3, + "warm_start": False, + "record": None, + "decision_rule": "log", + "rps_weight": "0000803f", + "distill_temp": "0000803f", + }, + } + profile["outer"] = {"opt": "nesterov", "topk_frac": "0000803f", "bits": 8} + profile["inner"] |= {"H": 4, "J": 2} + profile["layout"] = {"pp": 1, "n_gpus": 8, "dp_size": 8, "ep_size": 1, "zero1": False} + profile["schedule"] |= {"executions_per_host": per_host, "executions_total": 2 * per_host} + return profile + + +def test_eight_gpu_od_large_budget_is_geometry_derived() -> None: + mod = network_module() + decoder = json.loads((TREE / "experiments/gpu_network_v2/profile.json").read_bytes()) + assert mod.execution_quota(decoder) == { + "total": 126, + "per_role": 63, + "qualification": 4, + "remaining": 122, + } + bounds = mod.artifact_budget(od_large_profile(3)) + assert bounds["param_count"] == 353861632 and bounds["tensor_count"] == 339 + # Task 19 measured: state 4246444200, ef 1415481880, delta 359411534, + # eight-rank execution output 150085361712 bytes; bounds must cover all of them. + assert bounds["state_bytes"] >= 4246444200 - 65536 + assert bounds["execution_bytes"] >= 150085361712 + assert bounds["retained_bytes_per_host"] == 3 * bounds["execution_bytes"] + assert mod.execution_quota(od_large_profile(3))["remaining"] == 2 + for change in ( + {"param_count": 353861631}, + {"n_kv_heads": 8}, + ): + profile = od_large_profile(3) + profile["model"] |= change + with pytest.raises(Reject, match="unsupported_artifact_geometry"): + mod.artifact_budget(profile) + profile = od_large_profile(3) + profile["layout"]["dp_size"] = 4 + with pytest.raises(Reject, match="unsupported_artifact_geometry"): + mod.artifact_budget(profile) + for schedule in ( + {"executions_total": 7}, + {"executions_per_host": 2, "executions_total": 4}, + {"executions_per_host": True, "executions_total": 2}, + ): + profile = od_large_profile(3) + profile["schedule"] |= schedule + with pytest.raises(Reject, match="execution_reservation_mismatch"): + mod.execution_quota(profile) + + +def test_eight_gpu_reservations_follow_profile_quota(tmp_path: Path) -> None: + import time + + cfg = { + "base_url": "http://127.0.0.1:1", + "hosts": [{"role": "h0"}, {"role": "h1"}], + "cleanup_contract": "network-v2", + } + j = Journal(tmp_path) + j.append("admitted", deadline_unix=time.time() + 7200) + orch = Orchestrator(cfg, j, Provider(cfg["base_url"], "a" * 32, tmp_path, j, False), tmp_path) + runtime = network_module().NetworkRuntime(orch, od_large_profile(4)) + for role in ("h0", "h1"): + runtime.reserve(role, "qual", "qualification", 2) + with pytest.raises(Reject, match="exhausted"): + runtime.reserve("h0", "requal", "qualification", 2) + j.append("network_workload_promoted", remaining_total=4, remaining_per_host=2) + for role in ("h0", "h1"): + runtime.reserve(role, "work-0", "workload", 1) + runtime.reserve(role, "work-1", "workload", 1) + with pytest.raises(Reject, match="exhausted"): + runtime.reserve(role, "overflow", "workload", 1) + assert sum(r["executions"] for r in j.all("network_execution_intent")) == 8 + + +def test_prune_rescued_policy_budget_bounds_and_requires_local_custody() -> None: + mod = network_module() + # Legacy default retains every execution remotely plus a remote rescue.tar: > 400GB. + assert mod.artifact_budget(od_large_profile(4))["disk_artifacts_peak_bytes"] > 400_000_000_000 + pruned = od_large_profile(7) + pruned["artifacts"] |= {"remote_retention": "prune_rescued"} + bounds = mod.artifact_budget(pruned) + # Streamed rescue: only the two resident round directories remain remotely, < 400GB with + # image and staging; every finished execution is in local custody before it is pruned. + assert bounds["disk_artifacts_peak_bytes"] == 2 * bounds["execution_bytes"] + assert bounds["disk_artifacts_peak_bytes"] < 400_000_000_000 + # 7 per host = 2 qualification + 5 workload executions per host (10 total) within 4.7TB. + assert bounds["local_custody_bytes_required"] < 4_700_000_000_000 + larger = od_large_profile(8) + larger["artifacts"] |= {"remote_retention": "prune_rescued"} + assert mod.artifact_budget(larger)["local_custody_bytes_required"] > 4_700_000_000_000 + for change in ( + {"remote_retention": "delete"}, + {"remote_retention": "prune_rescued", "retain_all_checkpoints": False}, + {"remote_retention": "prune_rescued", "path_preserving_rescue": False}, + ): + bad = od_large_profile(7) + bad["artifacts"] |= change + with pytest.raises(Reject, match="unsupported_remote_retention_policy"): + mod.artifact_budget(bad) + artifacts = pruned["artifacts"] | prune_inputs() | {"disk_bytes": 400_000_000_000} + rate = bounds["rescue_min_bytes_per_second"] + artifacts["rescue_bytes_per_second"] = rate + evidence = { + "measured_rescue_bytes_per_second": rate, + "image_and_dependency_bytes": 1000, + "staging_bytes_per_host": 1000, + } + with pytest.raises(Reject, match="local_custody_disk_budget_exceeded"): + mod.readiness_budget(pruned | {"artifacts": artifacts}, evidence) + artifacts["local_custody_bytes"] = 4_700_000_000_000 + ready = mod.readiness_budget(pruned | {"artifacts": artifacts}, evidence) + assert ready["remote_retention"] == "prune_rescued" + # The image and staging the host still needs are part of the 400GB check, not waived. + artifacts |= {"image_and_dependency_bytes": 60_000_000_000} + evidence |= {"image_and_dependency_bytes": 60_000_000_000} + with pytest.raises(Reject, match="artifact_disk_budget_exceeded"): + mod.readiness_budget(pruned | {"artifacts": artifacts}, evidence) + + +def prune_inputs() -> dict: + return {"image_and_dependency_bytes": 1000, "staging_bytes_per_host": 1000} + + +class PruneSsh: + def __init__(self, calls: list[str], rc: int = 0) -> None: + self.calls, self.rc = calls, rc + + def run(self, command: str, tag: str, log_dir: Path) -> subprocess.CompletedProcess[str]: + self.calls.append(command) + return subprocess.CompletedProcess(command, self.rc, "", "") + + +def prune_runtime(tmp_path: Path, n_gpus: int = 2): + import tarfile + + mod = network_module() + profile = json.loads((TREE / "experiments/gpu_network_v2/profile.json").read_bytes()) + profile["artifacts"]["remote_retention"] = "prune_rescued" + (tmp_path / "run").mkdir() + j = Journal(tmp_path / "run") + calls: list[str] = [] + lifecycle = SimpleNamespace( + roles=("h0", "h1"), + dir=tmp_path / "run", + remote_root=lambda role: "/root/ht", + ssh=lambda role: PruneSsh(calls), + cleanup_started=lambda: False, + ) + runtime = mod.NetworkRuntime.__new__(mod.NetworkRuntime) + runtime.lifecycle, runtime.journal, runtime.profile = lifecycle, j, profile + + def rescued(names: list[str]) -> dict: + src = tmp_path / "remote" / str(len(j.records())) + files = {} + for name in names: + for rank in range(n_gpus): + rel = f"{name}/published/rank-{rank}/summary.json" + (src / rel).parent.mkdir(parents=True, exist_ok=True) + (src / rel).write_bytes(b"{}") + archive = tmp_path / f"rescue-{len(j.records())}.tar" + mod.export_artifacts(src, archive, profile) + with tarfile.open(archive) as t: + files = json.loads(t.extractfile("manifest.json").read()) + return j.append( + "network_rescued", role="h0", tar=str(archive), tar_sha256=fsha(archive), files=files + ) + + return mod, runtime, j, calls, rescued + + +def test_prune_rescued_deletes_only_verified_completed_remote_outputs(tmp_path: Path) -> None: + mod, runtime, j, calls, rescued = prune_runtime(tmp_path) + custody = rescued(["op-a"]) + # Started but not done (active/interrupted operation): never pruned. + j.append("network_job_started", role="h0", name="op-b") + assert runtime.prune_rescued("h0") == [] + j.append("network_operation_done", role="h0", name="op-a", tar_sha256=custody["tar_sha256"]) + j.append("network_qualification_done", role="h0", name="qual", result={}) + # Qualification is retained until promotion; op-a is verified, so only op-a goes. + assert [r["name"] for r in runtime.prune_rescued("h0")] == ["op-a"] + assert calls == ["rm -rf -- /root/ht/out/op-a"] + assert runtime.prune_rescued("h0") == [] # idempotent, never re-deletes + runtime.profile["artifacts"]["remote_retention"] = "all" + with pytest.raises(Reject, match="policy_disabled"): + runtime.prune_rescued("h0") + + +@pytest.mark.parametrize("fault", ["missing", "tampered", "symlink", "rank", "interrupted", "rm"]) +def test_prune_rescued_refuses_without_intact_local_custody(tmp_path: Path, fault: str) -> None: + # "rank": the rescued tar holds rank-0 only while the profile layout has two ranks. + mod, runtime, j, calls, rescued = prune_runtime(tmp_path, 1 if fault == "rank" else 2) + custody = rescued(["op-a"]) + archive = Path(custody["tar"]) + if fault == "missing": + archive.unlink() + elif fault == "tampered": + archive.write_bytes(archive.read_bytes()[:-1] + b"x") + elif fault == "symlink": + real = archive.with_suffix(".real") + archive.rename(real) + archive.symlink_to(real) + tar = custody["tar_sha256"] if fault != "interrupted" else "0" * 64 + j.append("network_operation_done", role="h0", name="op-a", tar_sha256=tar) + if fault == "rm": + runtime.lifecycle.ssh = lambda role: PruneSsh(calls, rc=1) + expected = { + "missing": "local_custody_missing", + "tampered": "local_custody_changed", + "symlink": "local_custody_missing", + "rank": "rank_custody_incomplete", + "interrupted": "without_rescued_custody", + "rm": "remote_prune_failed", + }[fault] + with pytest.raises(Reject, match=expected): + runtime.prune_rescued("h0") + assert calls == (["rm -rf -- /root/ht/out/op-a"] if fault == "rm" else []) + assert j.last("network_remote_pruned") is None + + +def test_prune_rescued_qualification_only_after_promotion_and_manifest_binding( + tmp_path: Path, +) -> None: + mod, runtime, j, calls, rescued = prune_runtime(tmp_path) + custody = rescued(["qual"]) + digest = mod.sha256(json.dumps(custody["files"], sort_keys=True).encode()) + j.append( + "network_qualification_done", + role="h0", + name="qual", + result={"artifact_manifest_sha256": "f" * 64}, + ) + assert runtime.prune_rescued("h0") == [] # not promoted: both rounds stay remote + j.append("network_workload_promoted", remaining_total=122, remaining_per_host=61) + with pytest.raises(Reject, match="without_rescued_custody"): + runtime.prune_rescued("h0") # recorded manifest differs from local custody + j.append( + "network_qualification_done", + role="h0", + name="qual", + result={"artifact_manifest_sha256": digest}, + ) + assert [r["name"] for r in runtime.prune_rescued("h0")] == ["qual"] + assert calls == ["rm -rf -- /root/ht/out/qual"] + + +@pytest.mark.parametrize("fault", [None, "rc", "digest", "truncated"]) +def test_streamed_rescue_writes_no_remote_archive_and_verifies_locally( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fault: str | None +) -> None: + mod = network_module() + remote = tmp_path / "remote" + (remote / "out/op-a/published/rank-0").mkdir(parents=True) + (remote / "out/op-a/published/rank-0/summary.json").write_bytes(b"{}") + (remote / "experiments/gpu_network_v2").mkdir(parents=True) + profile = json.loads((TREE / "experiments/gpu_network_v2/profile.json").read_bytes()) + profile["artifacts"]["remote_retention"] = "prune_rescued" + (remote / "experiments/gpu_network_v2/profile.json").write_text(json.dumps(profile)) + run = tmp_path / "run" + (run / "logs").mkdir(parents=True) + script = TREE / "experiments/gpu_network_v2/orchestrate.py" + tail = { + None: "", + "rc": "; exit 3", + "digest": "; echo deadbeef >&2", + "truncated": " | head -c 1024", + }[fault or None] + monkeypatch.setattr( + mod, + "ssh_argv", + lambda ssh, command: [ + "sh", + "-c", + f"cd {remote} && PYTHONPATH={TREE / 'src'} {sys.executable} {script} export out - " + f"experiments/gpu_network_v2/profile.json{tail}", + ], + ) + j = Journal(run) + runtime = mod.NetworkRuntime.__new__(mod.NetworkRuntime) + runtime.journal, runtime.profile = j, profile + runtime.lifecycle = SimpleNamespace( + dir=run, + cfg={"remote_python": sys.executable}, + remote_root=lambda role: str(remote), + ssh=lambda role: SimpleNamespace(timeout=60), + ) + if fault is None: + record = runtime.rescue("h0", fresh=True) + assert set(record["files"]) == {"op-a/published/rank-0/summary.json"} + assert fsha(Path(record["tar"])) == record["tar_sha256"] + assert not (remote / "rescue.tar").exists() # nothing archived remotely + else: + with pytest.raises(Reject): + runtime.rescue("h0", fresh=True) + assert j.last("network_rescued") is None + assert not list((run / "rescue/h0").iterdir()) # partial archive removed + + +def test_verified_dedup_hardlinks_identical_bytes_only(tmp_path: Path) -> None: + mod = network_module() + seen: dict[str, Path] = {} + a, b, c = tmp_path / "rank-0/x", tmp_path / "rank-1/x", tmp_path / "rank-1/y" + mod.place_verified(a, b"same", seen, True) + mod.place_verified(b, b"same", seen, True) + mod.place_verified(c, b"other", seen, True) + assert a.stat().st_ino == b.stat().st_ino and c.stat().st_ino != a.stat().st_ino + assert b.read_bytes() == b"same" and c.read_bytes() == b"other" + # A changed source is never linked: the copy is written fresh from the custody bytes. + a.unlink() + a.write_bytes(b"tampered") + seen[mod.sha256(b"same")] = a + d = tmp_path / "rank-2/x" + mod.place_verified(d, b"same", seen, True) + assert d.stat().st_ino != a.stat().st_ino and d.read_bytes() == b"same" + # Legacy (dedup off) never links. + e = tmp_path / "rank-3/x" + mod.place_verified(e, b"same", {mod.sha256(b"same"): d}, False) + assert e.stat().st_ino != d.stat().st_ino + + +def test_streamed_rescue_oversized_stream_aborts_and_leaves_nothing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + mod = network_module() + profile = json.loads((TREE / "experiments/gpu_network_v2/profile.json").read_bytes()) + profile["artifacts"]["remote_retention"] = "prune_rescued" + real = mod.artifact_budget + monkeypatch.setattr( + mod, "artifact_budget", lambda p: real(p) | {"rescue_tar_bytes_per_host": 1 << 20} + ) + # An endless stream: only the byte cap (not a timeout or EOF) can end this test. + monkeypatch.setattr(mod, "ssh_argv", lambda ssh, command: ["sh", "-c", "yes; echo x >&2"]) + run = tmp_path / "run" + (run / "logs").mkdir(parents=True) + j = Journal(run) + runtime = mod.NetworkRuntime.__new__(mod.NetworkRuntime) + runtime.journal, runtime.profile = j, profile + calls: list[str] = [] + runtime.lifecycle = SimpleNamespace( + dir=run, + cfg={"remote_python": sys.executable}, + remote_root=lambda role: "/root/ht", + ssh=lambda role: SimpleNamespace(timeout=600, run=lambda *a: calls.append(a[0])), + ) + with pytest.raises(Reject, match="network_rescue_archive_budget_exceeded"): + runtime.rescue("h0", fresh=True) + assert not list((run / "rescue/h0").iterdir()) + assert j.last("network_rescued") is None and j.last("network_remote_pruned") is None + assert calls == [] # no remote command, so no prune + + +def test_stream_bounded_local_write_error_kills_group_before_wait( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import errno + + mod = network_module() + started: list[subprocess.Popen[bytes]] = [] + real = subprocess.Popen + + def popen(*args, **kwargs): + started.append(real(*args, **kwargs)) + return started[-1] + + monkeypatch.setattr(mod.subprocess, "Popen", popen) + + class FullDisk: + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def write(self, data: bytes) -> int: + raise OSError(errno.ENOSPC, "No space left on device") + + # One process writes exactly one block and then blocks forever, so it gets no SIGPIPE. + # Only an explicit group kill ends it; the 600s timer cannot be what releases wait(). + child = "import os, signal; os.write(1, bytes(1 << 20)); signal.pause()" + argv = [sys.executable, "-c", child] + with pytest.raises(OSError) as raised: + mod.stream_bounded( + argv, SimpleNamespace(open=lambda mode: FullDisk()), tmp_path / "e.log", 1 << 30, 600 + ) + assert raised.value.errno == errno.ENOSPC + assert started[0].returncode == -signal.SIGKILL + with pytest.raises(ProcessLookupError): + os.killpg(started[0].pid, 0) # no orphan left in the process group + + +def test_legacy_runtime_schedule_stays_exact() -> None: + mod = network_module() + profile = json.loads((TREE / "experiments/gpu_network_v2/profile.json").read_bytes()) + profile["execution_and_transfer_bound_seconds"] = 1 + profile["schedule"]["live_executions"] = 15 # still fits the quota sum, not the legacy shape + with pytest.raises(Reject, match="execution_reservation_mismatch"): + schedule_only(mod, profile) + profile["schedule"]["live_executions"] = 16 + schedule_only(mod, profile) # the reviewed legacy schedule itself still admits + + +def schedule_only(mod, profile: dict) -> None: + """runtime_admission's schedule block, reached with every earlier evidence check satisfied.""" + from hypertrain.protocol.hashing import sha256_hex + from hypertrain.protocol.jcs import canonicalize + + pin = profile["runtime"] | {"qualified": True, "image_digest": "sha256:" + "a" * 64} + profile["runtime"] = pin + profile["artifacts"] |= {"image_and_dependency_bytes": 1000, "staging_bytes_per_host": 1000} + bounds = mod.artifact_budget(profile) + profile["artifacts"]["rescue_bytes_per_second"] = bounds["rescue_min_bytes_per_second"] + evidence = { + "profile_model": profile["model"], + "layout": profile["layout"], + "device": "cuda", + "torch": "2.14.0+cu130", + "full_round_transfer_bound_seconds": profile["execution_and_transfer_bound_seconds"], + "no_slower_hosts": True, + "image_digest": pin["image_digest"], + "driver_allowlist": pin["driver_allowlist"], + "python": pin["remote_python"], + "cuda": pin["cuda"], + "full_round_artifacts_sha256": "b" * 64, + "outer_tape_verified": True, + "measured_rescue_bytes_per_second": profile["artifacts"]["rescue_bytes_per_second"], + "image_and_dependency_bytes": 1000, + "staging_bytes_per_host": 1000, + } + profile["runtime_evidence_sha256"] = sha256_hex(canonicalize(evidence)) + mod.runtime_admission(profile, evidence) + + +def prune_timing_profile() -> dict: + profile = json.loads((TREE / "experiments/gpu_network_v2/profile.json").read_bytes()) + profile["artifacts"] |= { + "remote_retention": "prune_rescued", + "rescue_transfer_seconds": 3300, + "delete_absence_seconds": 300, + "local_custody_bytes": 10**15, + } + profile["schedule"] |= { + "hard_deadline_seconds": 43200, + "staging_seconds": 1800, + "beacon_service_seconds": 600, + "rescue_delete_seconds": 3600, + "hard_margin_seconds": 1800, + } + profile["microqualification"] |= { + "staging_seconds": 1800, + "probe_and_transfer_seconds": 3600, + "rescue_delete_seconds": 3600, + "failure_margin_seconds": 600, + "qualification_deadline_seconds": 9600, + } + return profile + + +def test_lifecycle_timing_legacy_exact_and_prune_bounded() -> None: + mod = network_module() + legacy = json.loads((TREE / "experiments/gpu_network_v2/profile.json").read_bytes()) + assert mod.lifecycle_timing(legacy) == { + "horizon": 3600, + "execution_cap": 40, + "teardown": 300, + "qualification": 900, + "qual_stage": 300, + "qual_probe": 180, + } + timing = mod.lifecycle_timing(prune_timing_profile()) + assert timing["horizon"] == 43200 and timing["qualification"] == 9600 + assert timing["execution_cap"] == (43200 - 1800 - 600 - 3600 - 1800) // 63 + for section, change in ( + ("schedule", {"hard_deadline_seconds": 86401}), # beyond the finite ceiling + ("schedule", {"hard_deadline_seconds": True}), # malformed + ("schedule", {"staging_seconds": 0}), + ("schedule", {"rescue_delete_seconds": "3600"}), + ("artifacts", {"rescue_transfer_seconds": 3400}), # rescue + delete > teardown + ("microqualification", {"failure_margin_seconds": 601}), # stages exceed Q + ("schedule", {"hard_deadline_seconds": 11000}), # Q + margin > H + ("schedule", {"hard_deadline_seconds": 7801}), # no execution time left + ): + profile = prune_timing_profile() + profile[section] |= change + with pytest.raises(Reject, match="lifecycle_timing_invalid"): + mod.lifecycle_timing(profile) + + +def test_prune_runtime_admission_uses_derived_cap_not_legacy_40() -> None: + mod = network_module() + profile = prune_timing_profile() + cap = mod.lifecycle_timing(profile)["execution_cap"] + profile["execution_and_transfer_bound_seconds"] = cap + 1 + with pytest.raises(Reject, match="missing_or_unqualified_exact_profile_runtime"): + schedule_only(mod, profile) + profile = prune_timing_profile() + profile["execution_and_transfer_bound_seconds"] = cap + assert cap > 40 + assert schedule_only_total(mod, profile) <= 43200 - 1800 + + +def schedule_only_total(mod, profile: dict) -> int: + seen: list[int] = [] + real = mod.runtime_admission + + def capture(p, e): + seen.append(real(p, e)) + return seen[-1] + + mod.runtime_admission = capture + try: + schedule_only(mod, profile) + finally: + mod.runtime_admission = real + return seen[0] + + +@pytest.mark.parametrize( + ("horizon", "expected"), + [(None, 3600), (3600, 3600), (43200, 43200), (86400, 86400)], +) +def test_network_lifecycle_deadline_is_reviewed_config_horizon( + tmp_path: Path, horizon: int | None, expected: int +) -> None: + cfg = { + "base_url": "http://127.0.0.1:1", + "hosts": [{"role": "h0"}], + "cleanup_contract": "network-v2", + } + if horizon is not None: + cfg["hard_deadline_seconds"] = horizon + j = Journal(tmp_path) + first = j.append("create_intent", role="h0") + orch = Orchestrator(cfg, j, Provider(cfg["base_url"], "a" * 32, tmp_path, j, False), tmp_path) + assert orch.deadline() == first["unix"] + expected + + +@pytest.mark.parametrize("horizon", [0, -1, 86401, True, "3600", 3600.0]) +def test_network_lifecycle_deadline_rejects_unbounded_or_malformed( + tmp_path: Path, horizon: object +) -> None: + cfg = { + "base_url": "http://127.0.0.1:1", + "hosts": [{"role": "h0"}], + "cleanup_contract": "network-v2", + "hard_deadline_seconds": horizon, + } + j = Journal(tmp_path) + j.append("create_intent", role="h0") + orch = Orchestrator(cfg, j, Provider(cfg["base_url"], "a" * 32, tmp_path, j, False), tmp_path) + with pytest.raises(Reject, match="lifecycle_horizon_invalid"): + orch.deadline() + + +def test_mission_seven_per_host_execution_cap_boundary() -> None: + mod = network_module() + profile = prune_timing_profile() + seven = { + "executions_per_host": 7, + "executions_total": 14, + "shadow_executions": 8, + "live_executions": 2, + "probes": 2, + "fault_trace_reserve": 2, + } + profile["schedule"] |= seven + cap = mod.lifecycle_timing(profile)["execution_cap"] + assert cap == (43200 - 1800 - 600 - 3600 - 1800) // 7 == 5057 + profile["execution_and_transfer_bound_seconds"] = cap + assert schedule_only_total(mod, profile) == 1800 + 7 * cap + 600 + 3600 + profile = prune_timing_profile() + profile["schedule"] |= seven + profile["execution_and_transfer_bound_seconds"] = cap + 1 + with pytest.raises(Reject, match="missing_or_unqualified_exact_profile_runtime"): + schedule_only(mod, profile) + + +def mission_admission_profile() -> dict: + profile = prune_timing_profile() + profile["layout"] = {"pp": 1, "n_gpus": 8, "dp_size": 8, "ep_size": 1, "zero1": False} + profile["artifacts"]["disk_bytes"] = 429495787520 # measured 400GiB host disk + profile["admission"] |= { + "ceiling_usd": "500", + "cleanup_reserve_usd": "5", + "max_host_hourly_usd": "5", + "max_ancillary_usd": "50", + "gpus_per_host": 8, + "min_disk_gib": 399, + } + return profile + + +def test_admission_limits_legacy_exact_and_mission_from_profile() -> None: + from decimal import Decimal + + from hypertrain.gpu_ops import budget + + mod = network_module() + legacy = json.loads((TREE / "experiments/gpu_network_v2/profile.json").read_bytes()) + limits = mod.admission_limits(legacy) + assert (limits["ceiling_usd"], limits["reserve"], limits["hardware_cap"]) == ( + "50", + Decimal("5"), + Decimal("5"), + ) + assert (limits["disk_gb"], limits["gpus"], limits["hourly"]) == (80, 2, Decimal("2")) + mission = mod.admission_limits(mission_admission_profile()) + assert mission["ceiling_usd"] == "500" and mission["reserve"] == Decimal("5") + assert mission["hardware_cap"] == Decimal("495") + assert mission["disk_gb"] == 399 # provider disk units are GiB; measured disk is 399.999GiB + assert mission["gpus"] == 8 + # Two 8-GPU hosts at 4.35/h for the 12h horizon, plus storage and egress, fit the ceiling. + offer = { + "dph_total": "4.35", + "storage_cost": "0.1", + "inet_up_cost": "0.01", + "inet_down_cost": "0.01", + } + plan = budget.evaluate( + snapshot0_credit=Decimal("1000"), + current_credit=Decimal("1000"), + offers=[offer, offer], + hard_deadline_seconds=43200, + disk_gb=mission["disk_gb"], + egress_gb=400, + phase_cap=mission["ceiling"], + reserve=mission["reserve"], + global_cap=mission["ceiling"], + ) + assert plan["admit"] and Decimal(plan["hardware_worst_case_usd"]) <= mission["hardware_cap"] + # The legacy cap still rejects the same lifecycle. + assert not budget.evaluate( + snapshot0_credit=Decimal("1000"), + current_credit=Decimal("1000"), + offers=[offer, offer], + hard_deadline_seconds=43200, + disk_gb=80, + egress_gb=400, + phase_cap=Decimal("50"), + )["admit"] + + +@pytest.mark.parametrize( + "change", + [ + {"ceiling_usd": 500}, # not the signed decimal string + {"ceiling_usd": "NaN"}, + {"ceiling_usd": "5"}, # no room above the reserve + {"cleanup_reserve_usd": "50"}, # reserve is fixed, never derived + {"max_host_hourly_usd": "0"}, + {"gpus_per_host": 2}, # disagrees with the 8-GPU layout + {"min_disk_gib": 401}, # more than the measured disk + {"min_disk_gib": True}, + ], +) +def test_admission_limits_reject_malformed_mission(change: dict) -> None: + mod = network_module() + profile = mission_admission_profile() + profile["admission"] |= change + with pytest.raises(Reject, match="admission_limits_invalid"): + mod.admission_limits(profile) + + +def adoption_inputs() -> tuple: + profile = mission_admission_profile() + first = 1791838912.29 + rows = [ + {"event": "create_intent", "role": "hostA", "label": "la", "unix": first}, + {"event": "create_response", "instance_id": 11, "success": True, "unix": first + 1}, + {"event": "create_intent", "role": "hostB", "label": "lb", "unix": 1791838912.902}, + {"event": "create_response", "instance_id": 22, "success": True, "unix": first + 1}, + ] + ownership = { + "api_lookup": { + "endpoint": "x", + "hostA": {"instance_id": 11, "machine_id": 101}, + "hostB": {"instance_id": 22, "machine_id": 202}, + } + } + live = {"actual_status": "running", "num_gpus": 8, "gpu_name": "RTX 5090", "disk_space": 400.0} + snapshot = { + "instances": [ + {"id": 11, "machine_id": 101, **live}, + {"id": 22, "machine_id": 202, **live}, + ] + } + cfg = {"hosts": [{"role": "h0", "machine_id": 101}, {"role": "h1", "machine_id": 202}]} + now = first + 8000 + record = {"admitted_unix": int(now) - 10, "cutoff_unix": int(first) + 43200} + return record, cfg, profile, ownership, rows, snapshot, now + + +def test_adoption_imports_original_create_and_fresh_admission() -> None: + mod = network_module() + record, cfg, profile, ownership, rows, snapshot, now = adoption_inputs() + got = mod.adoption_receipt(record, cfg, profile, ownership, rows, snapshot, now) + assert got["first_billable_unix"] == 1791838912.29 + assert got["horizon_end_unix"] == 1791838912.29 + 43200 + assert got["cutoff_unix"] == record["cutoff_unix"] + assert got["admitted_unix"] == record["admitted_unix"] + assert got["roles"] == { + "h0": {"instance_id": 11, "machine_id": 101, "label": "la", "create_unix": 1791838912.29}, + "h1": {"instance_id": 22, "machine_id": 202, "label": "lb", "create_unix": 1791838912.902}, + } + + +def _foreign_machine(i): + i[1]["hosts"][1]["machine_id"] = 999 + + +def _foreign_live(i): + i[5]["instances"][1]["machine_id"] = 999 + + +def _extra_live(i): + i[5]["instances"].append({"id": 33, "machine_id": 101}) # second instance on an owned machine + + +def _stopped(i): + i[5]["instances"][0]["actual_status"] = "exited" + + +def _small_disk(i): + i[5]["instances"][0]["disk_space"] = 80.0 + + +def _two_gpu(i): + i[5]["instances"][0]["num_gpus"] = 2 + + +def _missing_intent(i): + del i[4][2] + + +def _failed_create(i): + i[4][3]["success"] = False + + +@pytest.mark.parametrize( + "mutate", + [ + _foreign_machine, + _foreign_live, + _extra_live, + _stopped, + _small_disk, + _two_gpu, + _missing_intent, + _failed_create, + ], +) +def test_adoption_rejects_foreign_or_missing_custody(mutate) -> None: + mod = network_module() + inputs = list(adoption_inputs()) + mutate(inputs) + with pytest.raises(Reject, match="adoption_ownership_mismatch"): + mod.adoption_receipt(*inputs) + + +@pytest.mark.parametrize( + "change", + [ + {"admitted_unix": None}, + {"admitted_unix": 1791838912 + 8000 + 60}, # future + {"admitted_unix": 1791838912 + 8000 - 400}, # stale + {"admitted_unix": 1791838900}, # before the original create + {"cutoff_unix": 1791838912 + 43201}, # beyond the original horizon + {"cutoff_unix": 1791838912 + 8000 - 10 + 9600 + 1800 - 1}, # qual+margin past cutoff + ], +) +def test_adoption_rejects_invalid_clock(change: dict) -> None: + mod = network_module() + record, *rest = adoption_inputs() + with pytest.raises(Reject, match="adoption_clock_invalid"): + mod.adoption_receipt({**record, **change}, *rest) + + +def test_adoption_rejects_when_qualification_cannot_finish_or_horizon_expired() -> None: + mod = network_module() + record, cfg, profile, ownership, rows, snapshot, _ = adoption_inputs() + late = 1791838912.29 + 43200 - 9600 - 1800 + 5 + with pytest.raises(Reject, match="adoption_clock_invalid"): + mod.adoption_receipt( + {**record, "admitted_unix": int(late)}, cfg, profile, ownership, rows, snapshot, late + ) + legacy = json.loads((TREE / "experiments/gpu_network_v2/profile.json").read_bytes()) + with pytest.raises(Reject, match="adoption_requires_reviewed_finite_profile"): + mod.adoption_receipt(record, cfg, legacy, ownership, rows, snapshot, 1791838912.29 + 8000) + + +def test_adoption_preserves_and_ignores_foreign_inventory() -> None: + mod = network_module() + record, cfg, profile, ownership, rows, snapshot, now = adoption_inputs() + snapshot["instances"] += [ + {"id": 55598802, "machine_id": 140475, "actual_status": "exited"}, + {"id": 55620228, "machine_id": 146355, "actual_status": "running"}, + ] + got = mod.adoption_receipt(record, cfg, profile, ownership, rows, snapshot, now) + assert {r["instance_id"] for r in got["roles"].values()} == {11, 22} + assert len(snapshot["instances"]) == 4 + assert [i["id"] for i in mod.owned_instances(snapshot, cfg)] == [11, 22] + + +def _no_lookup(i): + del i[3]["api_lookup"] + + +def _lookup_row_not_dict(i): + i[3]["api_lookup"]["hostA"] = 55618302 + + +def _journal_line_not_dict(i): + i[4].append([1, 2]) + + +def _no_instances(i): + del i[5]["instances"] + + +def _instance_not_dict(i): + i[5]["instances"].append("55620228") + + +def _no_cutoff(i): + del i[0]["cutoff_unix"] + + +@pytest.mark.parametrize( + ("mutate", "reason"), + [ + (_no_lookup, "adoption_ownership_mismatch"), + (_lookup_row_not_dict, "adoption_ownership_mismatch"), + (_journal_line_not_dict, "adoption_ownership_mismatch"), + (_no_instances, "adoption_ownership_mismatch"), + (_instance_not_dict, "adoption_ownership_mismatch"), + (_no_cutoff, "adoption_clock_invalid"), + ], +) +def test_adoption_rejects_malformed_inputs(mutate, reason: str) -> None: + mod = network_module() + inputs = list(adoption_inputs()) + mutate(inputs) + with pytest.raises(Reject, match=reason): + mod.adoption_receipt(*inputs) + + +def test_adopted_lifecycle_deadline_uses_imported_first_billable(tmp_path: Path) -> None: + from hypertrain.gpu_ops.launcher import network_first_billable + + cfg = { + "base_url": "http://127.0.0.1:1", + "hosts": [{"role": "h0"}], + "cleanup_contract": "network-v2", + "hard_deadline_seconds": 43200, + } + j = Journal(tmp_path) + assert network_first_billable(j) is None + cutoff = 1791846900 + 14400 + j.append( + "adopted", first_billable_unix=1791838912.29, admitted_unix=1791846900, cutoff_unix=cutoff + ) + orch = Orchestrator(cfg, j, Provider(cfg["base_url"], "a" * 32, tmp_path, j, False), tmp_path) + # Billing ends at the signed adoption cutoff, never the 12h metadata horizon. + assert orch.deadline() == cutoff + assert network_module().qualification_start(j) == 1791846900.0 + + +@pytest.mark.parametrize("cutoff", [None, "1791861300", 1791838912, 1791838912 + 43201, True]) +def test_adopted_deadline_rejects_malformed_or_out_of_window_cutoff(tmp_path, cutoff) -> None: + cfg = {"base_url": "http://127.0.0.1:1", "hosts": [{"role": "h0"}]} + cfg.update(cleanup_contract="network-v2", hard_deadline_seconds=43200) + j = Journal(tmp_path) + j.append("adopted", first_billable_unix=1791838912.29, cutoff_unix=cutoff) + orch = Orchestrator(cfg, j, Provider(cfg["base_url"], "a" * 32, tmp_path, j, False), tmp_path) + with pytest.raises(Reject, match="adoption_cutoff_invalid"): + orch.deadline() + + +def test_supervisor_alarm_fires_at_adopted_cutoff_not_horizon(tmp_path, monkeypatch) -> None: + from hypertrain.gpu_ops import supervisor + + now = 1791850000.0 + cutoff = int(now) + 600 + cfg = {"base_url": "http://127.0.0.1:1", "hosts": [{"role": "h0"}], "key_file": "k"} + cfg.update(cleanup_contract="network-v2", hard_deadline_seconds=43200) + j = Journal(tmp_path) + j.append("adopted", first_billable_unix=1791838912.29, cutoff_unix=cutoff) + j.append("transaction_open", pid=os.getpid(), live=False) + clock = {"t": now} + monkeypatch.setattr(supervisor.time, "time", lambda: clock["t"]) + + def wait(fds, _w, _x, timeout): + clock["t"] += timeout # deterministic: advance the fake clock, no real sleep + return ([], [], []) + + monkeypatch.setattr(supervisor.select, "select", wait) + + class Stop(Exception): + pass + + def provider(*args, **kwargs): + raise Stop + + monkeypatch.setattr(supervisor, "Provider", provider) + monkeypatch.setattr(supervisor, "read_api_key", lambda _: "a" * 32) + with pytest.raises(Stop): + supervisor.supervise(tmp_path, cfg, j, now + 43200, now + 43200) + reached = j.last("supervisor_deadline_reached") + assert reached is not None and reached["deadline_unix"] == cutoff - 420 + assert clock["t"] == cutoff - 420 + + +def test_registry_layers_resolves_oci_index_to_exact_linux_amd64(tmp_path: Path) -> None: + import hashlib + + mod = network_module() + platform = tmp_path / "amd64.manifest.json" + platform.write_text( + json.dumps({"schemaVersion": 2, "config": {"size": 10}, "layers": [{"size": 32}]}) + ) + digest = "sha256:" + hashlib.sha256(platform.read_bytes()).hexdigest() + + def index(rows): + return {"schemaVersion": 2, "manifests": rows} + + amd64 = {"digest": digest, "platform": {"architecture": "amd64", "os": "linux"}} + other = {"digest": "sha256:" + "0" * 64, "platform": {"architecture": "arm64", "os": "linux"}} + assert mod.registry_layers(index([amd64, other]), platform) == 42 + legacy = {"schemaVersion": 2, "config": {"size": 10}, "layers": [{"size": 42}]} + assert mod.registry_layers(legacy, None) == 52 + for rows, file, reason in ( + ([amd64], None, "reviewed_registry_platform"), # index without the platform file + ([other], platform, "reviewed_registry_platform"), # no linux/amd64 entry + ([amd64, amd64], platform, "reviewed_registry_platform"), # ambiguous + ([{**amd64, "digest": "sha256:" + "1" * 64}], platform, "reviewed_registry_platform"), + ): + with pytest.raises(Reject, match=reason): + mod.registry_layers(index(rows), file) + platform.write_text(json.dumps({"schemaVersion": 2, "config": {"size": 1}, "layers": []})) + bad = {**amd64, "digest": "sha256:" + hashlib.sha256(platform.read_bytes()).hexdigest()} + with pytest.raises(Reject, match="reviewed_complete_image_layers_required"): + mod.registry_layers(index([bad]), platform) + for body in ( + {"schemaVersion": 2, "config": {"size": 10**12}, "layers": [{"size": -(10**12)}]}, + {"schemaVersion": 2, "config": {"size": -1}, "layers": [{"size": 2}]}, + {"schemaVersion": True, "config": {"size": 1}, "layers": [{"size": 2}]}, + {"schemaVersion": 2, "config": {"size": 1}, "layers": [{"size": True}]}, + ): + with pytest.raises(Reject, match="reviewed_complete_image_layers_required"): + mod.registry_layers(body, None) + + +def test_staging_cap_legacy_exact_and_prune_bounded_by_measured_disk() -> None: + mod = network_module() + legacy = json.loads((TREE / "experiments/gpu_network_v2/profile.json").read_bytes()) + assert mod.staging_cap(legacy) == 1_000_000_000 + assert mod.staging_cap(mission_admission_profile()) == 429495787520 + + +@pytest.mark.parametrize( + ("parsed", "accepted"), + [ + ({"success": True}, True), + ({"success": False, "msg": "SSH key already associated with instance."}, True), + ({"success": False, "msg": "SSH key already associated with instance"}, False), + ({"success": False, "msg": "Invalid ssh key."}, False), + ({"success": False}, False), + ({"success": False, "msg": "SSH key already associated with instance.", "x": 1}, False), + ], +) +def test_attach_accepts_only_exact_already_associated_reply( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, parsed: dict, accepted: bool +) -> None: + from hypertrain.gpu_ops.journal import sha256 + from hypertrain.gpu_ops.provider import Response + + cfg = {"base_url": "http://127.0.0.1:1", "hosts": [{"role": "h0"}, {"role": "h1"}]} + j = Journal(tmp_path) + j.append("receipt", role="h0", instance_id=7) + provider = Provider(cfg["base_url"], "a" * 32, tmp_path, j, False) + orch = Orchestrator(cfg, j, provider, tmp_path) + sent = [] + + def call(method, path, tag, body=None, timeout=30): + sent.append((method, path, body)) + return Response(200, parsed, None, "mock", None) + + monkeypatch.setattr(provider, "call", call) + pub = ensure_identity(tmp_path).with_suffix(".pub").read_text().strip() + if not accepted: + with pytest.raises(Reject, match="ssh_attach_failed:h0:HTTP200"): + orch.attach("h0") + assert j.last("ssh_attached", role="h0") is None + return + orch.attach("h0") + assert sent == [("POST", "/api/v0/instances/7/ssh/", {"ssh_key": pub})] + row = j.last("ssh_attached", role="h0") + assert row["key_sha256"] == sha256(pub.encode()) and row["instance_id"] == 7 + assert row["already_associated"] is (parsed["success"] is False) diff --git a/tests/gpu_ops/test_network_role_dispatch.py b/tests/gpu_ops/test_network_role_dispatch.py index 918c427..10788d4 100644 --- a/tests/gpu_ops/test_network_role_dispatch.py +++ b/tests/gpu_ops/test_network_role_dispatch.py @@ -1031,3 +1031,111 @@ def interrupt(source, destination, **kwargs): if fault == "failed-response": failed = json.loads((path.parent / "response-api-1.json").read_bytes()) assert failed["status"] == 409 and failed["response_hex"] == replies[1][1].hex() + + +def test_execution_environment_records_actual_rank_gpu_uuids(monkeypatch, tmp_path): + """Remote custody observation carries per-rank UUIDs; layout-derived, no two-GPU constant.""" + import torch + + n = 8 + job = SimpleNamespace( + manifest=SimpleNamespace( + training=SimpleNamespace( + reference_spec=SimpleNamespace( + layout=SimpleNamespace(n_gpus=n), driver_allowlist=["595.71.05"] + ) + ) + ) + ) + op = SimpleNamespace(backend="cuda", image_digest="sha256:" + "11" * 32, job=job) + raw = [f"{i:08x}-1111-2222-3333-444444444444" for i in range(n)] + monkeypatch.setattr(OP.shutil, "which", lambda name: "/usr/bin/nvidia-smi") + monkeypatch.setattr( + OP.subprocess, + "run", + lambda *a, **k: SimpleNamespace(stdout="595.71.05\n" * n), + ) + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(torch.cuda, "device_count", lambda: n) + monkeypatch.setattr( + torch.cuda, + "get_device_properties", + lambda i: SimpleNamespace(uuid=raw[i], multi_processor_count=170), + ) + observed = OP.execution_environment(op) + assert observed["gpu_uuids"] == ["GPU-" + u for u in raw] + assert observed["drivers"] == ["595.71.05"] * n and observed["sm_counts"] == [170] * n + monkeypatch.setattr(torch.cuda, "device_count", lambda: 2) + with pytest.raises(ValueError, match="layout differs"): + OP.execution_environment(op) + + +def test_role_dispatch_provenance_binds_rescued_operation_result(tmp_path): + from hypertrain.gpu_ops import role_dispatch + + (tmp_path / "run").mkdir() + journal = Journal(tmp_path / "run") + directory = tmp_path / "geometry" + (directory / "op-x").mkdir(parents=True) + result = directory / "op-x/operation-result.json" + result.write_bytes(json.dumps({"environment": {"gpu_uuids": ["GPU-a"]}}).encode()) + with pytest.raises(RuntimeError, match="canonical publication absent"): + role_dispatch.provenance(journal, directory) + journal.append( + "network_canonical_done", role="h1", name="op-x", directory=str(directory / "published") + ) + journal.append( + "network_operation_done", role="h1", name="op-x", result_sha256=fsha(result), tar_sha256="t" + ) + value = role_dispatch.provenance(journal, directory) + assert value["environment"] == {"gpu_uuids": ["GPU-a"]} and value["role"] == "h1" + assert json.loads((directory / "role-dispatch.json").read_bytes()) == value + assert role_dispatch.provenance(journal, directory) == value # crash-resume is idempotent + (directory / "role-dispatch.json").write_bytes(b"{}") + with pytest.raises(RuntimeError, match="already differs"): + role_dispatch.provenance(journal, directory) + (directory / "role-dispatch.json").unlink() + (directory / "role-dispatch.json").symlink_to(directory / "op-x/operation-result.json") + with pytest.raises(RuntimeError, match="already differs"): + role_dispatch.provenance(journal, directory) + result.write_bytes(b'{"environment": {"gpu_uuids": ["GPU-forged"]}}') + with pytest.raises(RuntimeError, match="custody changed"): + role_dispatch.provenance(journal, directory) + + +def test_app_role_dispatch_config_installs_constructor_hook(tmp_path, monkeypatch): + from pydantic import ValidationError + + from hypertrain.challenge import app as app_module + from hypertrain.gpu_ops import role_dispatch + + config = tmp_path / "dispatch.json" + config.write_text( + json.dumps( + { + "tree": str(TREE), + "run_dir": str(tmp_path / "lifecycle"), + "owner_key_file": str(tmp_path / "owner"), + "roles": {OWNER.ss58: "h1"}, + "run_id": "a" * 64, + } + ) + ) + monkeypatch.setenv("HYPERTRAIN_ROLE_DISPATCH", str(config)) + monkeypatch.setenv("CHALLENGE_STATE_DIR", str(tmp_path / "state")) + parsed = app_module.Config.from_env() + assert parsed.role_dispatch_file == config + store = app_module.create_app(parsed).state.store + assert store._role_launch is not None + assert store._open_child(tmp_path / "child")._role_launch is store._role_launch + # Foreign run: refused before reading the lifecycle (absent here), so before SSH or debit. + with pytest.raises(RuntimeError, match="not the authorized run"): + store._role_launch("audit", SimpleNamespace(run_id="b" * 64), tmp_path, {}) + # Authorized run, unadmitted lifecycle: rejects before any SSH, debit or rental. + with pytest.raises((FileNotFoundError, Reject)): + store._role_launch("audit", SimpleNamespace(run_id="a" * 64), tmp_path, {}) + config.write_text(json.dumps({"tree": str(TREE), "extra": 1})) + with pytest.raises(ValidationError): + role_dispatch.load(config) + monkeypatch.delenv("HYPERTRAIN_ROLE_DISPATCH") + assert app_module.create_app(app_module.Config.from_env()).state.store._role_launch is None diff --git a/tests/miner/test_client_network_v2.py b/tests/miner/test_client_network_v2.py index 35dcb0b..b88f70a 100644 --- a/tests/miner/test_client_network_v2.py +++ b/tests/miner/test_client_network_v2.py @@ -122,3 +122,207 @@ def run_round(self, w): assert cli.main(["run-v2", "--round", "7"]) == 0 assert calls == [7] assert json.loads(capsys.readouterr().out) == {"status": "UPLOADED"} + + +def _parts_miner(tmp_path: Path): + """NetworkMiner over a mock v2 object route backed by a real LocalFSStore.""" + import importlib.util + import sys + + from hypertrain.data.store import LocalFSStore + from hypertrain.protocol.keys import verify + + spec = importlib.util.spec_from_file_location( + "parts_fixture", Path(__file__).parents[1] / "ledger/test_escrow_v2.py" + ) + fixture = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = fixture + spec.loader.exec_module(fixture) + manifest = fixture.setup().manifest + keyfile = tmp_path / "key" + keyfile.write_bytes(b"c" * 32) + keyfile.chmod(0o600) + cfg = MinerConfig( + "https://service.test", + keyfile, + tmp_path / "work", + "unused", + manifest.training.reference_spec.image_digest, + run_id=manifest.run_id(), + owner_hotkey=fixture.COORD.ss58, + ) + store, puts = LocalFSStore(tmp_path / "objects"), [] + envelope = seal(fixture.COORD, "RunManifestV2", manifest.run_id(), manifest, 1000) + hotkey = Keypair(b"c" * 32) + + def route(request: httpx.Request) -> httpx.Response: + if "/objects/" not in request.url.path: + return httpx.Response(200, json={"manifest_envelope": envelope}) + sha = request.url.path.rsplit("/", 1)[1] + if request.method == "PUT": + message = f"hypertrain/object/2|{manifest.run_id()}|{sha}".encode() + signature = bytes.fromhex(request.headers["X-Object-Signature"]) + assert request.headers["X-Hotkey"] == hotkey.ss58 + assert verify(hotkey.public, message, signature) and len(request.content) <= 1 << 20 + puts.append(request.content) + return httpx.Response(200, json={"sha256": store.put(request.content)}) + path = store._path(sha) + return ( + httpx.Response(200, content=path.read_bytes()) + if path.is_file() + else httpx.Response(404) + ) + + return NetworkMiner(cfg, httpx.Client(transport=httpx.MockTransport(route))), store, puts + + +def test_parts_publish_and_fetch_when_artifact_exceeds_one_part(tmp_path: Path): + import hashlib + import os + from types import SimpleNamespace + + from hypertrain.data.store import parse_index + from hypertrain.gpu_ops.work_screen import artifact_ref + + # Given: a >1 MiB artifact and a small one. + miner, _, puts = _parts_miner(tmp_path) + big, small = os.urandom(3 * (1 << 20) + 5), b"small-object" + (tmp_path / "big").write_bytes(big) + (tmp_path / "small").write_bytes(small) + # When: signed per-part publication. + ref = artifact_ref(tmp_path / "big", miner._put_object) + plain = artifact_ref(tmp_path / "small", miner._put_object) + # Then: ref = index object (true SHA + stored length), index last, screen ref identical. + index = parse_index(puts[4]) + assert index is not None and index.sha256 == hashlib.sha256(big).hexdigest() + assert (ref.sha256, ref.size) == (hashlib.sha256(puts[4]).hexdigest(), len(puts[4])) + assert artifact_ref(tmp_path / "big") == ref and len(puts) == 6 + assert (plain.sha256, plain.size) == (hashlib.sha256(small).hexdigest(), len(small)) + # When: the job pins the whole SHA, objects map points at the index ref. + directory = tmp_path / "job" + directory.mkdir() + job = SimpleNamespace( + start_state_sha256=index.sha256, + ef_in_sha256="00" * 32, + v0_sha256="00" * 32, + object_paths={"start_state": "start_state"}, + ) + miner._fetch_inputs(job, {"start_state": ref.sha256}, directory) + # Then: reassembled file exists before any launch. + assert (directory / "start_state").read_bytes() == big + + +@pytest.mark.parametrize("fault", ["part", "index", "whole", "oversize", "name", "alias"]) +def test_parts_fetch_rejects_without_partial_file(tmp_path: Path, fault: str): + import hashlib + import os + from types import SimpleNamespace + + from hypertrain.data.store import parse_index + from hypertrain.gpu_ops.work_screen import artifact_ref + + # Given: honest publication, then one fault. + miner, store, puts = _parts_miner(tmp_path) + big = os.urandom(2 * (1 << 20) + 9) + (tmp_path / "big").write_bytes(big) + ref = artifact_ref(tmp_path / "big", miner._put_object).sha256 + whole = hashlib.sha256(big).hexdigest() + if fault == "part": # tampered part bytes under its key + part = store._path(parse_index(puts[-1]).parts[1]) + raw = part.read_bytes() + part.write_bytes(raw[:-1] + bytes([raw[-1] ^ 1])) + elif fault == "index": # objects map names an unknown ref + ref = "ab" * 32 + elif fault == "whole": # parts honest, but the job pins another whole SHA + whole = "cd" * 32 + elif fault == "oversize": # an oversized body on the wire + ref = store.put(os.urandom(3 << 20)) + objects = {"../start_state" if fault == "name" else "start_state": ref} + paths = {"start_state": "start_state"} + if fault == "alias": # two names, one confined file: later would overwrite earlier + objects["ef_in"], paths["ef_in"] = ref, "./start_state" + directory = tmp_path / "job" + directory.mkdir() + job = SimpleNamespace( + start_state_sha256=whole, + ef_in_sha256=whole, + v0_sha256=whole, + object_paths=paths, + ) + # When / Then: rejected, and nothing is published into the job directory. + with pytest.raises( + (MinerError, httpx.HTTPStatusError), match="corrupt|mismatch|404|part bound|object names" + ): + miner._fetch_inputs(job, objects, directory) + assert os.listdir(directory) == [] and not (tmp_path / "start_state").exists() + + +def test_object_get_stops_reading_when_stream_exceeds_part_bound(tmp_path: Path): + # Given: a server streaming an unbounded body in 64 KiB pieces, never prebuffered. + miner, _, _ = _parts_miner(tmp_path) + served = [] + + class Endless(httpx.SyncByteStream): + def __iter__(self): + while len(served) < 1024: # 64 MiB if fully consumed + served.append(64 << 10) + yield bytes(64 << 10) + + route = miner.api.c._transport.handler + miner.api.c._transport.handler = lambda request: ( + httpx.Response(200, stream=Endless()) if request.method == "GET" else route(request) + ) + # When / Then: refused at the 1 MiB bound; at most one piece read beyond it. + with pytest.raises(MinerError, match="part bound"): + miner._get_object("ab" * 32) + assert sum(served) <= (1 << 20) + (64 << 10) + + +def test_publication_stops_when_cancel_set_during_first_put(tmp_path: Path): + import threading + import time + + from hypertrain.data.store import parse_index + from hypertrain.gpu_ops.work_screen import artifact_ref + + # Given: a 4-part file; cancel is set while the first part PUT is in flight. + miner, _, puts = _parts_miner(tmp_path) + (tmp_path / "big").write_bytes(bytes(range(256)) * (3 * 4096 + 1)) + cancel, sent = threading.Event(), miner._put_object + + def first(data: bytes) -> str: + cancel.set() + return sent(data) + + miner._put_object = first + # When / Then: no further part and no index; the one part is harmless content-addressed. + with pytest.raises(MinerError, match="deadline/cancellation"): + artifact_ref(tmp_path / "big", miner._bounded_put(time.time() + 3600, cancel)) + assert len(puts) == 1 and parse_index(puts[0]) is None + # And: a passed deadline publishes nothing at all. + with pytest.raises(MinerError, match="deadline/cancellation"): + artifact_ref(tmp_path / "big", miner._bounded_put(time.time() - 1, None)) + assert len(puts) == 1 + + +def test_object_get_rejects_compressed_response_before_decoding(tmp_path: Path): + import gzip + + # Given: a hostile server ignoring Accept-Encoding: identity, gzip bomb body. + miner, _, _ = _parts_miner(tmp_path) + body, served, seen = gzip.compress(bytes(4 << 20)), [], [] + + class Bomb(httpx.SyncByteStream): + def __iter__(self): + served.append(len(body)) + yield body + + def gzip_route(request: httpx.Request) -> httpx.Response: + seen.append(request.headers["Accept-Encoding"]) + return httpx.Response(200, headers={"Content-Encoding": "gzip"}, stream=Bomb()) + + miner.api.c._transport.handler = gzip_route + # When / Then: refused from headers alone; nothing read or decoded. + with pytest.raises(MinerError, match="content encoding"): + miner._get_object("ab" * 32) + assert seen == ["identity"] and served == [] diff --git a/tests/miner/test_island_launch_v2.py b/tests/miner/test_island_launch_v2.py index 369230d..9bfe235 100644 --- a/tests/miner/test_island_launch_v2.py +++ b/tests/miner/test_island_launch_v2.py @@ -739,6 +739,18 @@ def test_qualification_cost_admits_without_measured_runtime_but_not_long_workloa assert plan["admit"] and not plan["long_workload_allowed"] assert float(plan["hard_deadline_hours"]) == 1 # failure still priced for whole liability assert float(plan["worst_case_total_usd"]) <= 15 + timing = mod.lifecycle_timing(profile) + low = timing["qualification"] + profile["schedule"]["hard_margin_seconds"] + short = mod.admission_plan( + profile, evidence, auth, snapshot, offers, qualification=True, priced_seconds=low + ) + assert short["rental_deadline_seconds"] == low + assert float(short["worst_case_total_usd"]) <= float(plan["worst_case_total_usd"]) + for bad in (low - 1, timing["horizon"] + 1, True): + with pytest.raises(mod.Reject, match="adoption_priced_window_invalid"): + mod.admission_plan( + profile, evidence, auth, snapshot, offers, qualification=True, priced_seconds=bad + ) with pytest.raises(mod.Reject, match="unqualified"): mod.admission_plan(profile, evidence, auth, snapshot, offers) with pytest.raises(mod.Reject, match="contract_missing"):