From 1cd08b749d891e2822c371af2314bfe70434ecd2 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:25:31 +0000 Subject: [PATCH 1/3] feat: publish verified network-v2 CUDA candidate --- .github/workflows/ci.yml | 115 +- .github/workflows/images.yml | 161 + .gitleaks.toml | 27 + .publishignore | 18 + .secretscan-allow | 15 + README.md | 16 +- deploy/k8s/base/configmap.yaml | 12 + deploy/k8s/base/deployment.yaml | 95 + deploy/k8s/base/hpa.yaml | 21 + deploy/k8s/base/kustomization.yaml | 14 + deploy/k8s/base/namespace.yaml | 26 + deploy/k8s/base/networkpolicy.yaml | 90 + deploy/k8s/base/pdb.yaml | 9 + deploy/k8s/base/service.yaml | 10 + deploy/k8s/local/fixtures.yaml | 281 + deploy/k8s/local/kind.yaml | 9 + deploy/k8s/overlays/apac/kustomization.yaml | 15 + deploy/k8s/overlays/eu/kustomization.yaml | 15 + deploy/k8s/overlays/local/kustomization.yaml | 16 + deploy/k8s/overlays/local/patch.yaml | 44 + deploy/k8s/overlays/us/kustomization.yaml | 15 + deploy/k8s/versions.json | 19 + docs/RESULTS.md | 18 + docs/network-v2.md | 437 + experiments/gpu_network_v2/orchestrate.py | 2711 ++++++ experiments/gpu_network_v2/profile.json | 80 + experiments/gpu_network_v2/run.py | 28 + scripts/canary.sh | 3 +- scripts/check.sh | 4 +- scripts/network_authority_snapshot.py | 647 ++ scripts/network_gpu_operation.py | 739 ++ scripts/network_gpu_prepare.py | 646 ++ scripts/network_gpu_qualification.py | 45 + scripts/network_gpu_seed.py | 523 ++ scripts/network_service_proof.py | 2488 ++++++ scripts/publish.sh | 110 +- scripts/relay_kind_smoke.py | 3936 ++++++++ scripts/verify_image.sh | 77 + src/hypertrain/aggregator/capacity_worker.py | 509 ++ src/hypertrain/aggregator/checkpoint.py | 526 +- src/hypertrain/aggregator/cli.py | 91 +- src/hypertrain/aggregator/core.py | 2 +- src/hypertrain/aggregator/rollback_v2.py | 1131 +++ src/hypertrain/aggregator/tape_v2.py | 383 + src/hypertrain/aggregator/weighted_v2.py | 226 + src/hypertrain/auditor/bisect.py | 2 +- src/hypertrain/auditor/honeypot.py | 76 +- src/hypertrain/auditor/island_bisect.py | 318 + src/hypertrain/auditor/replay.py | 482 +- src/hypertrain/auditor/worker.py | 177 +- src/hypertrain/beacon/freshness.py | 112 + src/hypertrain/challenge/admission.py | 987 +++ src/hypertrain/challenge/admission_store.py | 215 + src/hypertrain/challenge/app.py | 420 +- src/hypertrain/challenge/disputes_v2.py | 901 ++ src/hypertrain/challenge/finality_v2.py | 111 + src/hypertrain/challenge/store.py | 7884 ++++++++++++++++- src/hypertrain/challenge/trust_v2.py | 165 + src/hypertrain/data/stream_store.py | 251 + src/hypertrain/data/trial_assignment.py | 38 + src/hypertrain/gpu_ops/launcher.py | 176 +- .../gpu_ops/network_qualification.py | 396 + src/hypertrain/gpu_ops/supervisor.py | 53 +- src/hypertrain/gpu_ops/work_screen.py | 128 + src/hypertrain/ledger/escrow_v2.py | 1081 +++ src/hypertrain/miner/admission.py | 52 + src/hypertrain/miner/cli.py | 44 +- src/hypertrain/miner/core.py | 380 +- src/hypertrain/miner/dispute_watch.py | 292 + src/hypertrain/miner/island_launch.py | 685 ++ src/hypertrain/miner/island_worker.py | 216 + .../models/opendecision/__init__.py | 51 +- src/hypertrain/protocol/envelope_v2.py | 394 + src/hypertrain/protocol/messages_v2.py | 917 ++ src/hypertrain/protocol/relay_envelope.py | 75 + src/hypertrain/protocol/relay_messages.py | 508 ++ src/hypertrain/protocol/schema_v2.py | 40 + src/hypertrain/relay/__init__.py | 1 + src/hypertrain/relay/app.py | 293 + src/hypertrain/relay/client.py | 290 + src/hypertrain/relay/core.py | 967 ++ src/hypertrain/trainer/compress.py | 91 +- src/hypertrain/trainer/config.py | 11 + src/hypertrain/trainer/island.py | 173 +- src/hypertrain/trainer/model.py | 39 +- tests/aggregator/test_rollback_v2.py | 587 ++ .../aggregator/test_scalar_state_roundtrip.py | 205 + tests/aggregator/test_tape_v2.py | 48 + tests/aggregator/test_weighted_v2.py | 639 ++ tests/auditor/test_island_bisect_v2.py | 181 + tests/auditor/test_island_replay_v2.py | 536 ++ tests/beacon/test_freshness_v2.py | 105 + tests/challenge/authority_fixture_v2.py | 256 + tests/challenge/protected_opening_fixture.py | 1865 ++++ .../protected_opening_fixture_static.py | 214 + tests/challenge/test_admission_v2.py | 653 ++ tests/challenge/test_audit_deadline_v2.py | 412 + tests/challenge/test_authority_snapshot_v2.py | 223 + tests/challenge/test_backend_authority_v2.py | 877 ++ tests/challenge/test_capacity_outer_v2.py | 2558 ++++++ tests/challenge/test_carry_genesis_v2.py | 97 + tests/challenge/test_disputes_v2.py | 627 ++ .../test_role_dispatch_consumers_v2.py | 415 + tests/challenge/test_service_capacity_v2.py | 790 ++ tests/challenge/test_service_network_v2.py | 3706 ++++++++ tests/challenge/test_trust_v2.py | 253 + tests/e2e/test_network_gpu_prepare.py | 299 + tests/e2e/test_network_service_proof.py | 1612 ++++ tests/e2e/test_rollback_boundary_v2.py | 736 ++ tests/e2e/test_rollback_service_v2.py | 902 ++ tests/e2e/test_service_network_v2_e2e.py | 2150 +++++ tests/gpu_ops/test_network_admit_cli.py | 2553 ++++++ .../gpu_ops/test_network_gpu_qualification.py | 350 + tests/gpu_ops/test_network_gpu_seed.py | 107 + tests/gpu_ops/test_network_lifecycle.py | 545 ++ tests/gpu_ops/test_network_role_dispatch.py | 1022 +++ tests/gpu_ops/test_phase_b.py | 68 +- tests/gpu_ops/test_runtime_custody.py | 373 + tests/gpu_ops/test_work_screen_v2.py | 85 + tests/layout/test_island_network_v2.py | 218 + tests/layout/test_od_island_v2.py | 155 + tests/ledger/test_escrow_v2.py | 1343 +++ tests/miner/test_capacity_runtime_v2.py | 211 + tests/miner/test_client_network_v2.py | 124 + tests/miner/test_dispute_watch_v2.py | 468 + tests/miner/test_genesis_cpu_binding.py | 69 + tests/miner/test_island_launch_v2.py | 867 ++ .../fixtures/network_v1_baseline.json | 45 + tests/protocol/test_network_v2.py | 1078 +++ tests/relay/fixtures/kind_contract_docker.py | 106 + tests/relay/fixtures/podman-network-list.json | 251 + .../real-kind-hostconfig-normalization.json | 174 + tests/relay/test_lifecycle_v2.py | 676 ++ tests/relay/test_transport_v2.py | 2960 +++++++ tests/test_publish_exclusions.py | 512 ++ tests/trainer/test_arch_dispatch.py | 35 +- tests/trainer/test_codec_metadata.py | 215 + tests/trainer/test_config_v2.py | 30 + 138 files changed, 71368 insertions(+), 113 deletions(-) create mode 100644 .gitleaks.toml create mode 100644 .publishignore create mode 100644 .secretscan-allow create mode 100644 deploy/k8s/base/configmap.yaml create mode 100644 deploy/k8s/base/deployment.yaml create mode 100644 deploy/k8s/base/hpa.yaml create mode 100644 deploy/k8s/base/kustomization.yaml create mode 100644 deploy/k8s/base/namespace.yaml create mode 100644 deploy/k8s/base/networkpolicy.yaml create mode 100644 deploy/k8s/base/pdb.yaml create mode 100644 deploy/k8s/base/service.yaml create mode 100644 deploy/k8s/local/fixtures.yaml create mode 100644 deploy/k8s/local/kind.yaml create mode 100644 deploy/k8s/overlays/apac/kustomization.yaml create mode 100644 deploy/k8s/overlays/eu/kustomization.yaml create mode 100644 deploy/k8s/overlays/local/kustomization.yaml create mode 100644 deploy/k8s/overlays/local/patch.yaml create mode 100644 deploy/k8s/overlays/us/kustomization.yaml create mode 100644 deploy/k8s/versions.json create mode 100644 docs/network-v2.md create mode 100644 experiments/gpu_network_v2/orchestrate.py create mode 100644 experiments/gpu_network_v2/profile.json create mode 100644 experiments/gpu_network_v2/run.py create mode 100644 scripts/network_authority_snapshot.py create mode 100644 scripts/network_gpu_operation.py create mode 100644 scripts/network_gpu_prepare.py create mode 100644 scripts/network_gpu_qualification.py create mode 100644 scripts/network_gpu_seed.py create mode 100644 scripts/network_service_proof.py create mode 100644 scripts/relay_kind_smoke.py create mode 100644 src/hypertrain/aggregator/capacity_worker.py create mode 100644 src/hypertrain/aggregator/rollback_v2.py create mode 100644 src/hypertrain/aggregator/tape_v2.py create mode 100644 src/hypertrain/aggregator/weighted_v2.py create mode 100644 src/hypertrain/auditor/island_bisect.py create mode 100644 src/hypertrain/beacon/freshness.py create mode 100644 src/hypertrain/challenge/admission.py create mode 100644 src/hypertrain/challenge/admission_store.py create mode 100644 src/hypertrain/challenge/disputes_v2.py create mode 100644 src/hypertrain/challenge/finality_v2.py create mode 100644 src/hypertrain/challenge/trust_v2.py create mode 100644 src/hypertrain/data/stream_store.py create mode 100644 src/hypertrain/data/trial_assignment.py create mode 100644 src/hypertrain/gpu_ops/network_qualification.py create mode 100644 src/hypertrain/gpu_ops/work_screen.py create mode 100644 src/hypertrain/ledger/escrow_v2.py create mode 100644 src/hypertrain/miner/admission.py create mode 100644 src/hypertrain/miner/dispute_watch.py create mode 100644 src/hypertrain/miner/island_launch.py create mode 100644 src/hypertrain/miner/island_worker.py create mode 100644 src/hypertrain/protocol/envelope_v2.py create mode 100644 src/hypertrain/protocol/messages_v2.py create mode 100644 src/hypertrain/protocol/relay_envelope.py create mode 100644 src/hypertrain/protocol/relay_messages.py create mode 100644 src/hypertrain/protocol/schema_v2.py create mode 100644 src/hypertrain/relay/__init__.py create mode 100644 src/hypertrain/relay/app.py create mode 100644 src/hypertrain/relay/client.py create mode 100644 src/hypertrain/relay/core.py create mode 100644 tests/aggregator/test_rollback_v2.py create mode 100644 tests/aggregator/test_scalar_state_roundtrip.py create mode 100644 tests/aggregator/test_tape_v2.py create mode 100644 tests/aggregator/test_weighted_v2.py create mode 100644 tests/auditor/test_island_bisect_v2.py create mode 100644 tests/auditor/test_island_replay_v2.py create mode 100644 tests/beacon/test_freshness_v2.py create mode 100644 tests/challenge/authority_fixture_v2.py create mode 100644 tests/challenge/protected_opening_fixture.py create mode 100644 tests/challenge/protected_opening_fixture_static.py create mode 100644 tests/challenge/test_admission_v2.py create mode 100644 tests/challenge/test_audit_deadline_v2.py create mode 100644 tests/challenge/test_authority_snapshot_v2.py create mode 100644 tests/challenge/test_backend_authority_v2.py create mode 100644 tests/challenge/test_capacity_outer_v2.py create mode 100644 tests/challenge/test_carry_genesis_v2.py create mode 100644 tests/challenge/test_disputes_v2.py create mode 100644 tests/challenge/test_role_dispatch_consumers_v2.py create mode 100644 tests/challenge/test_service_capacity_v2.py create mode 100644 tests/challenge/test_service_network_v2.py create mode 100644 tests/challenge/test_trust_v2.py create mode 100644 tests/e2e/test_network_gpu_prepare.py create mode 100644 tests/e2e/test_network_service_proof.py create mode 100644 tests/e2e/test_rollback_boundary_v2.py create mode 100644 tests/e2e/test_rollback_service_v2.py create mode 100644 tests/e2e/test_service_network_v2_e2e.py create mode 100644 tests/gpu_ops/test_network_admit_cli.py create mode 100644 tests/gpu_ops/test_network_gpu_qualification.py create mode 100644 tests/gpu_ops/test_network_gpu_seed.py create mode 100644 tests/gpu_ops/test_network_lifecycle.py create mode 100644 tests/gpu_ops/test_network_role_dispatch.py create mode 100644 tests/gpu_ops/test_runtime_custody.py create mode 100644 tests/gpu_ops/test_work_screen_v2.py create mode 100644 tests/layout/test_island_network_v2.py create mode 100644 tests/layout/test_od_island_v2.py create mode 100644 tests/ledger/test_escrow_v2.py create mode 100644 tests/miner/test_capacity_runtime_v2.py create mode 100644 tests/miner/test_client_network_v2.py create mode 100644 tests/miner/test_dispute_watch_v2.py create mode 100644 tests/miner/test_genesis_cpu_binding.py create mode 100644 tests/miner/test_island_launch_v2.py create mode 100644 tests/protocol/fixtures/network_v1_baseline.json create mode 100644 tests/protocol/test_network_v2.py create mode 100644 tests/relay/fixtures/kind_contract_docker.py create mode 100644 tests/relay/fixtures/podman-network-list.json create mode 100644 tests/relay/fixtures/real-kind-hostconfig-normalization.json create mode 100644 tests/relay/test_lifecycle_v2.py create mode 100644 tests/relay/test_transport_v2.py create mode 100644 tests/test_publish_exclusions.py create mode 100644 tests/trainer/test_codec_metadata.py create mode 100644 tests/trainer/test_config_v2.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 407c704..75c9c45 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,120 @@ jobs: version: "0.12.17" python-version: "3.12" - run: uv sync --locked --all-extras --group dev - - run: bash scripts/check.sh --ignore=tests/sim --ignore=tests/e2e + - name: Native systemd regression (isolated root environment) + shell: bash + run: | + set -euo pipefail + test "$(ps -p 1 -o comm= | xargs)" = systemd + sudo -n systemctl show --property=Version + test -f /sys/fs/cgroup/cgroup.controllers + native="$(sudo -n mktemp -d /tmp/htn.XXXXXX)" + test "$(sudo -n stat -c '%a:%u' "$native")" = 700:0 + cleanup() { + status=$? + sudo -n python3 - "$native" "$status" <<'PY' + import json + import pathlib + import re + import shutil + import subprocess + import sys + + root = pathlib.Path(sys.argv[1]) + if sys.argv[2] != "0": + for name in ("capacity-stdout.log", "capacity-stderr.log"): + for log in (root / "tmp").rglob(name): + print(str(log), log.read_text(errors="replace"), flush=True) + units = set() + for record in (root / "tmp").rglob("capacity-attempt.json"): + body = json.loads(record.read_text()) + for field in ("unit", "timer", "timer_service"): + unit = body[field] + if not re.fullmatch(r"ht-cap-[0-9a-f]{32}(?:-deadline)?\.(?:service|timer)", unit): + raise ValueError("unexpected native unit name") + units.add(unit) + for unit in sorted(units): + observed = subprocess.run( + ["systemctl", "show", unit, "-p", "LoadState"], + check=True, capture_output=True, text=True, timeout=10, + ).stdout + if observed.strip() == "LoadState=not-found": + continue + subprocess.run(["systemctl", "stop", unit], check=True, timeout=30) + observed = subprocess.run( + ["systemctl", "show", unit, "-p", "ActiveState", "-p", "MainPID", "-p", "ControlPID"], + check=True, capture_output=True, text=True, timeout=10, + ).stdout + fields = dict(line.split("=", 1) for line in observed.splitlines()) + if fields["ActiveState"] not in ("inactive", "failed") or any( + int(fields.get(field, "0")) for field in ("MainPID", "ControlPID") + ): + raise RuntimeError("native child still active; retaining isolated environment") + shutil.rmtree(root) + PY + } + trap cleanup EXIT + sudo -n install -m 755 "$(command -v uv)" "$native/uv" + sudo -n mkdir -m 700 "$native/source" "$native/tmp" + git ls-files -z -- . ':!**/.git/**' ':!**/.venv/**' ':!**/*cache*/**' \ + ':!**/__pycache__/**' ':!**/secrets/**' ':!**/.env' ':!**/.env.*' | \ + tar --null -T - -cf - | sudo -n tar --no-same-owner -xf - -C "$native/source" + for file in uv.lock tests/challenge/test_capacity_outer_v2.py tests/challenge/test_service_capacity_v2.py tests/miner/test_capacity_runtime_v2.py tests/miner/test_island_launch_v2.py tests/relay/test_transport_v2.py; do + expected="$(sha256sum "$file" | cut -d ' ' -f 1)" + actual="$(sudo -n sha256sum "$native/source/$file" | cut -d ' ' -f 1)" + test "$expected" = "$actual" + printf 'native source %s %s\n' "$file" "$actual" + done + sudo -n env UV_CACHE_DIR="$native/cache" UV_PYTHON_INSTALL_DIR="$native/python" \ + UV_PYTHON_PREFERENCE=only-managed UV_PROJECT_ENVIRONMENT="$native/venv" \ + "$native/uv" sync --project "$native/source" --locked --all-extras --group dev --python 3.12 + sudo -n env -u PYTHONPATH HOME="$native" TMPDIR="$native/tmp" \ + PYTHONDONTWRITEBYTECODE=1 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 \ + bash -c ' + set -euo pipefail + cd "$1/source" + "$1/venv/bin/python" -c "import os, hypertrain; print(\"native affinity\", sorted(os.sched_getaffinity(0))); assert os.geteuid() == 0; assert hypertrain.__file__.startswith(os.getcwd() + \"/src/\")" + test -x "$1/venv/bin/hypertrain" + "$1/venv/bin/hypertrain" --help >/dev/null + /usr/bin/timeout --signal=TERM --kill-after=10s 299s \ + "$1/venv/bin/python" -m pytest -q -x --tb=short -p no:cacheprovider \ + --basetemp="$1/tmp" \ + tests/challenge/test_capacity_outer_v2.py::test_actual_bounded_outer_child_matches_original_signed_replay \ + tests/challenge/test_service_capacity_v2.py::test_real_capacity_launch_failure_keeps_debit_and_prevents_restart_rerun \ + "tests/miner/test_capacity_runtime_v2.py::test_actual_small_kernel_attempt_and_cleanup[success]" \ + "tests/miner/test_capacity_runtime_v2.py::test_actual_small_kernel_attempt_and_cleanup[oom]" \ + "tests/miner/test_capacity_runtime_v2.py::test_actual_small_kernel_attempt_and_cleanup[cancel]" \ + "tests/miner/test_capacity_runtime_v2.py::test_actual_small_kernel_attempt_and_cleanup[deadline]" \ + tests/miner/test_capacity_runtime_v2.py::test_real_runner_parent_death_retains_absolute_manager_timer \ + tests/miner/test_island_launch_v2.py::test_capacity_branch_uses_real_bounded_failure_before_publication \ + tests/challenge/test_capacity_outer_v2.py::test_actual_bounded_genesis_preparation_and_duplicate_debit \ + tests/relay/test_transport_v2.py::test_launch_safety_independent_systemd_units + ' bash "$native" 2>&1 | tee "$RUNNER_TEMP/hypertrain-native-stdout.log" + - name: Install pinned kind v0.31.0 + shell: bash + run: | + set -euo pipefail + tools="$(mktemp -d "$RUNNER_TEMP/kind.XXXXXX")" + curl --fail --silent --show-error --location \ + https://github.com/kubernetes-sigs/kind/releases/download/v0.31.0/kind-linux-amd64 \ + --output "$tools/kind" + printf '%s %s\n' eb244cbafcc157dff60cf68693c14c9a75c4e6e6fedaf9cd71c58117cb93e3fa "$tools/kind" | sha256sum --check --strict + chmod 755 "$tools/kind" + test "$("$tools/kind" version -q)" = 0.31.0 + printf '%s\n' "$tools" >> "$GITHUB_PATH" + - name: Remaining quality gate (normal runner user) + run: >- + bash scripts/check.sh --ignore=tests/sim --ignore=tests/e2e + --deselect=tests/challenge/test_capacity_outer_v2.py::test_actual_bounded_outer_child_matches_original_signed_replay + --deselect=tests/challenge/test_service_capacity_v2.py::test_real_capacity_launch_failure_keeps_debit_and_prevents_restart_rerun + "--deselect=tests/miner/test_capacity_runtime_v2.py::test_actual_small_kernel_attempt_and_cleanup[success]" + "--deselect=tests/miner/test_capacity_runtime_v2.py::test_actual_small_kernel_attempt_and_cleanup[oom]" + "--deselect=tests/miner/test_capacity_runtime_v2.py::test_actual_small_kernel_attempt_and_cleanup[cancel]" + "--deselect=tests/miner/test_capacity_runtime_v2.py::test_actual_small_kernel_attempt_and_cleanup[deadline]" + --deselect=tests/miner/test_capacity_runtime_v2.py::test_real_runner_parent_death_retains_absolute_manager_timer + --deselect=tests/miner/test_island_launch_v2.py::test_capacity_branch_uses_real_bounded_failure_before_publication + --deselect=tests/challenge/test_capacity_outer_v2.py::test_actual_bounded_genesis_preparation_and_duplicate_debit + --deselect=tests/relay/test_transport_v2.py::test_launch_safety_independent_systemd_units # The slow subprocess tests, in parallel with test. Together the two jobs run the whole suite. test-slow: diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml index 8dfc6ba..3acae19 100644 --- a/.github/workflows/images.yml +++ b/.github/workflows/images.yml @@ -20,6 +20,7 @@ jobs: uses: ./.github/workflows/ci.yml publish: + if: github.ref == 'refs/heads/main' needs: ci runs-on: ubuntu-24.04 timeout-minutes: 120 @@ -78,3 +79,163 @@ jobs: subject-name: ${{ matrix.image }} subject-digest: ${{ steps.build.outputs.digest }} push-to-registry: true + + candidate-gpu: + if: github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/hypertrain-cuda-qualification' + runs-on: ubuntu-24.04 + timeout-minutes: 120 + permissions: + contents: read + packages: write + id-token: write + attestations: write + env: + CANDIDATE_IMAGE: ghcr.io/cortexlm/hypertrain-gpu:candidate-${{ github.sha }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Free disk for the CUDA base image + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL + df -h / + - name: Freeze candidate source and build backend pins + id: meta + shell: bash + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$GITHUB_SHA" + python3 - "$RUNNER_TEMP" <<'PY' + import hashlib + import json + import pathlib + import tomllib + import os + + root = pathlib.Path.cwd() + output = pathlib.Path(os.environ['GITHUB_OUTPUT']) + temporary = pathlib.Path(os.environ['RUNNER_TEMP']) + files = sorted(p for p in (root / 'src/hypertrain').rglob('*') if p.is_file() and '__pycache__' not in p.parts and p.suffix != '.pyc') + assert files and all(not p.is_symlink() for p in files) + assert all(p.suffix == '.py' or p.name == 'py.typed' or p.relative_to(root).as_posix() == 'src/hypertrain/datasets/registry.json' for p in files) + source = {str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest() for p in files} + copied = {str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest() for p in files + [root / p for p in ('pyproject.toml', 'uv.lock', 'README.md', 'LICENSE')]} + lock = tomllib.loads((root / 'uv.lock').read_text()) + versions = {p['name']: p['version'] for p in lock['package'] if p['name'] in {'hatchling', 'packaging', 'pathspec', 'pluggy', 'trove-classifiers'}} + versions.update(hatchling='1.27.0', **{'trove-classifiers': '2026.6.1.19'}) + pins = { + 'hatchling': ('1.27.0', 'd3a2f3567c4f926ea39849cdf924c7e99e6686c9c8e288ae1037c8fa2a5d937b'), + 'pathspec': ('1.1.1', 'a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189'), + 'pluggy': ('1.6.0', 'e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746'), + 'trove-classifiers': ('2026.6.1.19', 'ab4c4ec93cc4a4e7815fa759906e05e6bb3f2fbd92ea0f897288c6a43efd15b3'), + } + assert all(versions[name] == version for name, (version, _) in pins.items()) + manifest = {'source_files': source, 'copy_files': copied, 'backend_versions': versions} + raw = json.dumps(manifest, sort_keys=True, separators=(',', ':')).encode() + (temporary / 'candidate-source.json').write_bytes(raw) + requirements = '\n'.join(f'{name}=={version} --hash=sha256:{digest}' for name, (version, digest) in pins.items()) + '\n' + backend = "RUN printf '%b' " + repr(requirements) + " > /tmp/candidate-build-requirements.txt && VIRTUAL_ENV=/opt/hypertrain/venv uv-pinned pip install --require-hashes --no-deps -r /tmp/candidate-build-requirements.txt && rm /tmp/candidate-build-requirements.txt\n" + # Only the candidate recipe persists the reviewed pinned backend packages. + (temporary / 'Dockerfile.candidate').write_text((root / 'docker/Dockerfile.gpu').read_text() + '\n' + backend) + with output.open('a') as stream: + print('source_hash=' + hashlib.sha256(raw).hexdigest(), file=stream) + print('version=' + tomllib.loads((root / 'pyproject.toml').read_text())['project']['version'], file=stream) + PY + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + - name: Build candidate locally before publication + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + file: ${{ runner.temp }}/Dockerfile.candidate + load: true + push: false + platforms: linux/amd64 + provenance: false + sbom: false + build-args: | + VERSION=${{ steps.meta.outputs.version }} + REVISION=${{ github.sha }} + labels: | + io.cortex.challenge.source-manifest=${{ steps.meta.outputs.source_hash }} + tags: ${{ env.CANDIDATE_IMAGE }} + - name: Verify installed candidate source before registry write + run: ENGINE=docker bash scripts/verify_image.sh --cuda-candidate "$CANDIDATE_IMAGE" "$RUNNER_TEMP/candidate-source.json" "$GITHUB_SHA" + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Refuse an existing or unverifiable candidate tag + env: + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GHCR_ACTOR: ${{ github.actor }} + shell: bash + run: | + set -euo pipefail + python3 - <<'PY' + import base64 + import http.client + import json + import os + import re + import urllib.parse + + repository = 'cortexlm/hypertrain-gpu' + tag = 'candidate-' + os.environ['GITHUB_SHA'] + assert re.fullmatch(r'candidate-[0-9a-f]{40}', tag) + assert os.environ['CANDIDATE_IMAGE'] == 'ghcr.io/' + repository + ':' + tag + credentials = base64.b64encode((os.environ['GHCR_ACTOR'] + ':' + os.environ['GHCR_TOKEN']).encode()).decode() + query = urllib.parse.urlencode({'service': 'ghcr.io', 'scope': 'repository:' + repository + ':pull'}) + try: + connection = http.client.HTTPSConnection('ghcr.io', timeout=30) + connection.request('GET', '/token?' + query, headers={'Authorization': 'Basic ' + credentials, 'Accept': 'application/json'}) + response = connection.getresponse() + assert response.status == 200 and response.getheader('Content-Type', '').split(';')[0] == 'application/json', 'registry authentication refused' + payload = response.read(1048577) + assert len(payload) <= 1048576, 'invalid registry token response' + bearer = json.loads(payload)['token'] + assert isinstance(bearer, str) and bearer and not any(c.isspace() for c in bearer), 'invalid registry token' + connection.close() + connection = http.client.HTTPSConnection('ghcr.io', timeout=30) + # Validate the granted repository pull scope by exercising it; GHCR tokens are opaque. + connection.request('GET', '/v2/' + repository + '/tags/list?n=1', headers={'Authorization': 'Bearer ' + bearer, 'Accept': 'application/json'}) + response = connection.getresponse() + assert response.status == 200 and response.getheader('Docker-Distribution-Api-Version') == 'registry/2.0', 'repository pull scope refused' + assert response.getheader('Content-Type', '').split(';')[0] == 'application/json', 'invalid repository response' + payload = response.read(1048577) + assert len(payload) <= 1048576, 'invalid repository response size' + scope = json.loads(payload) + assert scope.get('name') == repository and isinstance(scope.get('tags'), list) and all(isinstance(value, str) for value in scope['tags']), 'repository pull scope not verified' + connection.close() + connection = http.client.HTTPSConnection('ghcr.io', timeout=30) + connection.request('GET', '/v2/' + repository + '/manifests/' + tag, headers={'Authorization': 'Bearer ' + bearer, 'Accept': 'application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json'}) + response = connection.getresponse() + assert response.status == 404, 'candidate tag exists or registry lookup refused' + assert response.getheader('Docker-Distribution-Api-Version') == 'registry/2.0', 'invalid registry response' + assert response.getheader('Content-Type', '').split(';')[0] == 'application/json', 'invalid registry error media type' + payload = response.read(1048577) + assert len(payload) <= 1048576, 'invalid registry error response' + errors = json.loads(payload)['errors'] + assert isinstance(errors, list) and errors and all(isinstance(error, dict) and error.get('code') == 'MANIFEST_UNKNOWN' for error in errors), 'registry absence not authoritative' + connection.close() + except (OSError, http.client.HTTPException, ValueError, KeyError, TypeError, AttributeError, AssertionError): + raise SystemExit('Candidate tag guard refused publication') from None + print('Authenticated exact candidate tag absent; publication may proceed') + PY + - name: Publish only the verified candidate + id: publish + shell: bash + run: | + set -euo pipefail + docker push "$CANDIDATE_IMAGE" + reference="$(docker image inspect "$CANDIDATE_IMAGE" --format '{{index .RepoDigests 0}}')" + test "${reference%@*}" = ghcr.io/cortexlm/hypertrain-gpu + digest="${reference#*@}" + [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] + printf 'digest=%s\n' "$digest" >> "$GITHUB_OUTPUT" + - name: Attest actual candidate digest + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-name: ghcr.io/cortexlm/hypertrain-gpu + subject-digest: ${{ steps.publish.outputs.digest }} + push-to-registry: true diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 0000000..7e3f07a --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,27 @@ +# Only reviewed public values; different secrets on the same line still detect. +[extend] +useDefault = true + +[[rules]] +id = "generic-api-key" + +[[rules.allowlists]] +description = "Historical public source digest, not a signing key" +condition = "AND" +regexTarget = "secret" +paths = ['''(^|/)(scripts/publish\.sh|tests/protocol/fixtures/network_v1_baseline\.json)$'''] +regexes = ['''^d3030488eb18aaafc0feceb256666bcc63fc15c4e6ae0bb008980affd96976a2$'''] + +[[rules.allowlists]] +description = "Deliberate synthetic credential-shaped rejection input" +condition = "AND" +regexTarget = "secret" +paths = ['''(^|/)tests/gpu_ops/test_gpu_ops\.py$'''] +regexes = ['''^7875e0437f396a1d$'''] + +[[rules.allowlists]] +description = "Static fake S3 signing input for in-memory test transport" +condition = "AND" +regexTarget = "secret" +paths = ['''(^|/)tests/storage/fakes\.py$'''] +regexes = ['''^SECRETSECRETSECRETSECRET1234567890abcd$'''] diff --git a/.publishignore b/.publishignore new file mode 100644 index 0000000..7cf352f --- /dev/null +++ b/.publishignore @@ -0,0 +1,18 @@ +# patterns (grep -E on repo-relative paths) excluded from the published tree +(^|/)\.venv/ +(^|/)__pycache__/ +(^|/)\.(mypy|ruff|pytest)_cache/ +^\.git/ +^\.transport-admission\.json$ +^data/ +^experiments/results/.*(\.log|smoke|parity_state).* +^experiments/gpu_phase_[ab]/live-config\.json$ +\.env$ +known_hosts +id_(ed25519|rsa) +\.(pem|key)$ +# Local authority, custody and recovery material; public nested data stays publishable. +^\.(omo|herdr-web-ui)/ +(^|/)(private|secrets|roles|private-roles|role-keys|private-keys|authority|authority-state|portable-authority|portable-bundle|portable-bundles|snapshots|preserved-snapshots|evidence|service-export|integration-root|rescue)/ +\.(seed|priv|p12|pfx|pkcs8)$ +(^|/)(private[-_]key|admin\.token|drain\.token|challenge\.db(-wal|-shm)?|bundle\.json)$ diff --git a/.secretscan-allow b/.secretscan-allow new file mode 100644 index 0000000..b5239d2 --- /dev/null +++ b/.secretscan-allow @@ -0,0 +1,15 @@ +# literal substrings; a line containing one is exempt from the secret scan (documented public example values) +AKIAIOSFODNN7EXAMPLE +wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY +X-Amz-Signature +123456789ABCDEFGHJKLMNPQRSTUVWXYZ +5DP4qTec9XxffaALGWsEPhS1oWrDWMBjzhBmyzShREMJpymt +5HB7kpn92RS7uF9uWn8bXSvPVKFPg8kPUFDd5sbveGjX6Dbi +_TOKEN_FILE= +PYTORCH_CUDA_ALLOC_CONF +HYPERTRAIN_CHALLENGE_ +HYPERTRAIN_COORD_KEY_FILE=/run/secrets +AKIDEXAMPLE +wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY +R2_ACCOUNT_ID=your-cloudflare-account-id +ARG OPENDECISION_SHA= diff --git a/README.md b/README.md index bfe7638..db258b7 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ flowchart LR C --> X["Cortex"] ``` -Regions hold several islands each, and the diagram collapses them into one node. An island is one machine or a cluster of GPUs in one datacenter, and it trains on its own. Updates from nearby islands are combined by a regional relay first, so the long-distance links only carry a few merged updates. The challenge service keeps the round state and the ledger, auditors replay work, and Cortex reads the final weights from it each epoch. +Regions hold several islands each, and the diagram collapses them into one node. An island is one machine or a cluster of GPUs in one datacenter, and it trains on its own. Regional relays transport and cache each island's original update; the central deterministic aggregator merges them. The challenge service keeps the round state and the ledger, auditors replay work, and Cortex reads the final weights from it each epoch. ### 3. When rewards are paid @@ -102,6 +102,14 @@ Nothing is paid the moment work is submitted. A round is final only after its au ## Results so far +### Network v2: scoped evidence, three tasks still open + +Attempt008 FIRST4 passed same-driver two-host bitwise CUDA qualification: four rounds, eight rank workers, a 61,856-parameter decoder, both hosts on driver580.95.05. This does not qualify cross-driver execution, quality parity or funded116 continuation. Public miner CLI evidence is separate in-process CPU execution, not installed-CLI subprocess or CUDA proof. + +Local Kubernetes r2 passed import/source109, TLS/CNI, full upload to the fixture master, restart/replica replay, regional fallback, retention/GC and namespace denial, with verified owned cleanup. Its outer caller still exited1 on READY-before-ownership; an isolated caller phase correction subsequently passed two native-event regression cases, without rewriting that historical failure. The shipped gateway namespace AND pod TCP8443 policy passed one regression rendering all four Kustomize overlays. APAC runtime, L0 finality, full D3, funded116 and roster/service capacity remain unproven; the root ledger is20/23, not network completion. Three tasks remain: allocation80, CPU verification and final push/CI. Operator, consumer, wrapper and antiabuse contracts are complete within their verified CPU scope. Capacity controller safeguards were previously reviewed; current source requalification is pending after type/runtime changes. Genuine80 H2 execution remains unrun; fresh CPU approval is absent and root execution admission remains pending. + +Exact independent evidence and scope limits: [Network v2 current proof boundary](docs/network-v2.md#current-proof-boundary) and [current network results](docs/RESULTS.md#current-network-v2-update---2026-10-10). + ### GPU determinism, Phase A: PASS The open question behind the whole design was whether an RTX 5090 gives bit-identical BF16 training results on different machines. It does, for this trainer: @@ -228,10 +236,10 @@ What is enforced: What isn't done yet, honestly: -- **Single GPU miner client.** `hypertrain-miner` currently enforces one GPU per island. The multi-rank island layout exists in `trainer/island.py` and is tested on CPU, but the client doesn't launch it yet. +- **Multi-rank evidence is scoped.** Network v2 public CLI multi-rank execution is verified in-process on CPU, not as an installed-CLI subprocess or current CUDA admission. Historical FIRST4 CUDA008 qualification is separate. - **Joining is manual.** Contract v1 has no public join route. `hypertrain-miner join` prints a signed admission request (with a hardware self-check) that the operator adds to the roster. -- **Dispute bisection in the miner is step level only.** The auditor side bisects down to layer and operation; the miner client takes part at step level and doesn't poll open disputes on its own. -- **GPU determinism is proven for one setup.** Phase A covers single RTX 5090 hosts with a pinned image. Multi-GPU hosts (Phase B) passed on two hosts; see the Phase B note above. The hardware self-check has only run on CPU. +- **Dispute watch evidence is scoped.** Network v2 public CLI watch performs one operator-invoked bounded polling pass. CPU evidence covers STEP-to-LAYER signed reply/acknowledgment over actual FastAPI and verified TLS, with durable cursor/outbox retries. Published STEP/LAYER/OP traces are supported; installed-binary, current CUDA and full referee execution remain separate. +- **GPU qualification is scoped.** Historical Phase A/B and network-v2 FIRST4 attempt008 retain their separate model/image/driver scopes; see the results above. Public CLI CPU evidence does not establish current production CUDA admission. - **Quality parity is unproven.** No parity claim is made until the pending CPU and GPU work reports. - **One run per state directory** in the challenge service. - **The publish secret scan is a heuristic.** `scripts/publish.sh` skips digit-free values of 20+ characters under a credential parent. It does not guarantee a clean tree, so the out-of-band token approval and a manual review stay mandatory. diff --git a/deploy/k8s/base/configmap.yaml b/deploy/k8s/base/configmap.yaml new file mode 100644 index 0000000..57da789 --- /dev/null +++ b/deploy/k8s/base/configmap.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: relay-environment +data: + RELAY_CONFIG: /config/relay.json + RELAY_SECRETS: /run/relay-secrets + REGION: eu + OMP_NUM_THREADS: "1" + PYTHONDONTWRITEBYTECODE: "1" + # Operator creates relay-contract ConfigMap from signed run-specific relay.json. + # No fake public run/registry/master key is committed. diff --git a/deploy/k8s/base/deployment.yaml b/deploy/k8s/base/deployment.yaml new file mode 100644 index 0000000..d7d9266 --- /dev/null +++ b/deploy/k8s/base/deployment.yaml @@ -0,0 +1,95 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hypertrain-relay +spec: + replicas: 2 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: hypertrain-relay + strategy: + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + template: + metadata: + labels: + app: hypertrain-relay + hypertrain.network/role: relay + spec: + automountServiceAccountToken: false + terminationGracePeriodSeconds: 120 + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app: hypertrain-relay + containers: + - name: relay + image: ghcr.io/cortexlm/hypertrain + imagePullPolicy: IfNotPresent + command: [uvicorn, hypertrain.relay.app:app] + args: [--host, 0.0.0.0, --port, "8000", --workers, "1"] + envFrom: + - configMapRef: + name: relay-environment + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + ports: + - name: http + containerPort: 8000 + resources: + requests: + cpu: 250m + memory: 256Mi + ephemeral-storage: 2Gi + limits: + cpu: "1" + memory: 1Gi + ephemeral-storage: 3Gi + startupProbe: + httpGet: {path: /livez, port: http} + periodSeconds: 2 + failureThreshold: 30 + livenessProbe: + httpGet: {path: /livez, port: http} + periodSeconds: 10 + readinessProbe: + httpGet: {path: /readyz, port: http} + periodSeconds: 5 + timeoutSeconds: 20 + lifecycle: + preStop: + exec: + command: + - /opt/venv/bin/python + - -c + - "import httpx,pathlib; t=pathlib.Path('/run/relay-secrets/drain.token').read_text().strip(); r=httpx.post('http://127.0.0.1:8000/internal/drain',headers={'Authorization':'Bearer '+t},timeout=100); r.raise_for_status()" + volumeMounts: + - {name: config, mountPath: /config, readOnly: true} + - {name: secrets, mountPath: /run/relay-secrets, readOnly: true} + - {name: cache, mountPath: /tmp} + volumes: + - name: config + configMap: + name: relay-contract + - name: secrets + secret: + secretName: relay-runtime-v1 + defaultMode: 0440 + - name: cache + emptyDir: + sizeLimit: 2Gi diff --git a/deploy/k8s/base/hpa.yaml b/deploy/k8s/base/hpa.yaml new file mode 100644 index 0000000..7e88c19 --- /dev/null +++ b/deploy/k8s/base/hpa.yaml @@ -0,0 +1,21 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: hypertrain-relay +spec: + minReplicas: 2 + maxReplicas: 4 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: hypertrain-relay + behavior: + scaleDown: + stabilizationWindowSeconds: 300 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 diff --git a/deploy/k8s/base/kustomization.yaml b/deploy/k8s/base/kustomization.yaml new file mode 100644 index 0000000..fe11349 --- /dev/null +++ b/deploy/k8s/base/kustomization.yaml @@ -0,0 +1,14 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: hypertrain-relay +resources: + - namespace.yaml + - configmap.yaml + - deployment.yaml + - service.yaml + - networkpolicy.yaml + - pdb.yaml + - hpa.yaml +images: + - name: ghcr.io/cortexlm/hypertrain + digest: sha256:ebdb8bc97122488682c63200f55fae92a980e5a75b50cbe4d0ce3bde0fbf7fdb diff --git a/deploy/k8s/base/namespace.yaml b/deploy/k8s/base/namespace.yaml new file mode 100644 index 0000000..603b6dd --- /dev/null +++ b/deploy/k8s/base/namespace.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: hypertrain-relay + labels: + hypertrain.network/role: relay + pod-security.kubernetes.io/enforce: restricted + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/warn: restricted +--- +apiVersion: v1 +kind: ResourceQuota +metadata: + name: relay-budget +spec: + hard: + pods: "12" + requests.cpu: "2" + limits.cpu: "6" + requests.memory: 2Gi + limits.memory: 6Gi + requests.ephemeral-storage: 12Gi + limits.ephemeral-storage: 18Gi + persistentvolumeclaims: "0" + services.loadbalancers: "0" + services.nodeports: "0" diff --git a/deploy/k8s/base/networkpolicy.yaml b/deploy/k8s/base/networkpolicy.yaml new file mode 100644 index 0000000..15bf88c --- /dev/null +++ b/deploy/k8s/base/networkpolicy.yaml @@ -0,0 +1,90 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: relay-deny-default +spec: + podSelector: {} + policyTypes: [Ingress, Egress] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: relay-authorized-flows +spec: + podSelector: + matchLabels: + app: hypertrain-relay + policyTypes: [Ingress, Egress] + ingress: + - from: + - namespaceSelector: + matchLabels: + hypertrain.network/role: gateway + podSelector: + matchLabels: + hypertrain.network/role: gateway + - namespaceSelector: + matchLabels: + hypertrain.network/role: control + podSelector: + matchLabels: + hypertrain.network/role: master + ports: + - {protocol: TCP, port: 8000} + egress: + - to: + - namespaceSelector: + matchLabels: + hypertrain.network/role: gateway + podSelector: + matchLabels: + hypertrain.network/role: gateway + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - {protocol: UDP, port: 53} + - {protocol: TCP, port: 53} + - to: + - namespaceSelector: + matchLabels: + hypertrain.network/role: gateway + podSelector: + matchLabels: + hypertrain.network/role: gateway + - namespaceSelector: + matchLabels: + hypertrain.network/role: egress + podSelector: + matchLabels: + hypertrain.network/role: egress + - namespaceSelector: + matchLabels: + hypertrain.network/role: control + podSelector: + matchLabels: + hypertrain.network/role: master + - namespaceSelector: + matchLabels: + hypertrain.network/role: store + podSelector: + matchLabels: + hypertrain.network/role: store + ports: + - {protocol: TCP, port: 443} + - {protocol: TCP, port: 8000} + - {protocol: TCP, port: 9000} + - to: + - namespaceSelector: + matchLabels: + hypertrain.network/role: gateway + podSelector: + matchLabels: + hypertrain.network/role: gateway + ports: + - {protocol: TCP, port: 8443} + # External S3/drand/master require operator-pinned CIDRs or an approved egress + # gateway; NetworkPolicy cannot enforce hostnames. No 0.0.0.0/0 exemption. diff --git a/deploy/k8s/base/pdb.yaml b/deploy/k8s/base/pdb.yaml new file mode 100644 index 0000000..d4d96e6 --- /dev/null +++ b/deploy/k8s/base/pdb.yaml @@ -0,0 +1,9 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: hypertrain-relay +spec: + minAvailable: 1 + selector: + matchLabels: + app: hypertrain-relay diff --git a/deploy/k8s/base/service.yaml b/deploy/k8s/base/service.yaml new file mode 100644 index 0000000..5779a19 --- /dev/null +++ b/deploy/k8s/base/service.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: hypertrain-relay +spec: + type: ClusterIP + selector: + app: hypertrain-relay + ports: + - {name: http, port: 8000, targetPort: http} diff --git a/deploy/k8s/local/fixtures.yaml b/deploy/k8s/local/fixtures.yaml new file mode 100644 index 0000000..53567ba --- /dev/null +++ b/deploy/k8s/local/fixtures.yaml @@ -0,0 +1,281 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: hypertrain-proof + labels: + hypertrain.network/role: control + pod-security.kubernetes.io/enforce: restricted +--- +apiVersion: v1 +kind: Namespace +metadata: + name: hypertrain-proof-store + labels: + hypertrain.network/role: store +--- +apiVersion: v1 +kind: Namespace +metadata: + name: hypertrain-proof-gateway + labels: + hypertrain.network/role: gateway + pod-security.kubernetes.io/enforce: restricted +--- +apiVersion: v1 +kind: Namespace +metadata: + name: hypertrain-proof-denied + labels: + pod-security.kubernetes.io/enforce: restricted +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: hypertrain-proof-store + labels: + hypertrain.network/proof: hypertrain-v2-proof +spec: + capacity: {storage: 512Mi} + accessModes: [ReadWriteOnce] + storageClassName: "" + persistentVolumeReclaimPolicy: Retain + hostPath: + path: /var/local/hypertrain-proof-store + type: DirectoryOrCreate +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: objects + namespace: hypertrain-proof-store +spec: + accessModes: [ReadWriteOnce] + storageClassName: "" + volumeName: hypertrain-proof-store + resources: + requests: {storage: 512Mi} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: objects + namespace: hypertrain-proof-store +spec: + replicas: 1 + selector: + matchLabels: {app: objects} + template: + metadata: + labels: {app: objects, hypertrain.network/role: store} + spec: + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: {type: RuntimeDefault} + containers: + - name: objects + image: ghcr.io/cortexlm/hypertrain@sha256:ebdb8bc97122488682c63200f55fae92a980e5a75b50cbe4d0ce3bde0fbf7fdb + command: [/opt/venv/bin/python, /fixture/relay_kind_smoke.py, --fixture, store] + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: [ALL]} + resources: + requests: {cpu: 50m, memory: 64Mi} + limits: {cpu: 250m, memory: 192Mi} + readinessProbe: + httpGet: {path: /livez, port: 9000} + volumeMounts: + - {name: script, mountPath: /fixture, readOnly: true} + - {name: secrets, mountPath: /run/proof, readOnly: true} + - {name: objects, mountPath: /objects} + - {name: tmp, mountPath: /tmp} + volumes: + - name: script + configMap: {name: proof-script} + - name: secrets + secret: {secretName: proof-secrets, defaultMode: 0440} + - name: objects + persistentVolumeClaim: {claimName: objects} + - name: tmp + emptyDir: {sizeLimit: 64Mi} +--- +apiVersion: v1 +kind: Service +metadata: {name: objects, namespace: hypertrain-proof-store} +spec: + selector: {app: objects} + ports: [{port: 9000, targetPort: 9000}] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: {name: master, namespace: hypertrain-proof} +spec: + replicas: 1 + selector: + matchLabels: {app: master} + template: + metadata: + labels: {app: master, hypertrain.network/role: master} + spec: + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: {type: RuntimeDefault} + containers: + - name: master + image: ghcr.io/cortexlm/hypertrain@sha256:ebdb8bc97122488682c63200f55fae92a980e5a75b50cbe4d0ce3bde0fbf7fdb + command: [/opt/venv/bin/python, /fixture/relay_kind_smoke.py, --fixture, master] + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: [ALL]} + resources: + requests: {cpu: 50m, memory: 64Mi} + limits: {cpu: 250m, memory: 192Mi} + readinessProbe: + httpGet: {path: /livez, port: 8000} + volumeMounts: + - {name: script, mountPath: /fixture, readOnly: true} + - {name: secrets, mountPath: /run/proof, readOnly: true} + - {name: master, mountPath: /master} + - {name: tmp, mountPath: /tmp} + volumes: + - name: script + configMap: {name: proof-script} + - name: secrets + secret: {secretName: proof-secrets, defaultMode: 0440} + - name: master + persistentVolumeClaim: {claimName: master} + - name: tmp + emptyDir: {sizeLimit: 96Mi} +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: hypertrain-proof-master + labels: + hypertrain.network/proof: hypertrain-v2-proof +spec: + capacity: {storage: 32Mi} + accessModes: [ReadWriteOnce] + storageClassName: "" + persistentVolumeReclaimPolicy: Retain + hostPath: + path: /var/local/hypertrain-proof-master + type: Directory +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: {name: master, namespace: hypertrain-proof} +spec: + accessModes: [ReadWriteOnce] + storageClassName: "" + volumeName: hypertrain-proof-master + resources: + requests: {storage: 32Mi} +--- +apiVersion: v1 +kind: Service +metadata: {name: master, namespace: hypertrain-proof} +spec: + selector: {app: master} + ports: [{port: 8000, targetPort: 8000}] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: {name: gateway, namespace: hypertrain-proof-gateway} +spec: + replicas: 1 + selector: + matchLabels: {app: gateway} + template: + metadata: + labels: {app: gateway, hypertrain.network/role: gateway} + spec: + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: {type: RuntimeDefault} + containers: + - name: gateway + image: ghcr.io/cortexlm/hypertrain@sha256:ebdb8bc97122488682c63200f55fae92a980e5a75b50cbe4d0ce3bde0fbf7fdb + command: [/opt/venv/bin/python, /fixture/relay_kind_smoke.py, --fixture, gateway] + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: [ALL]} + resources: + requests: {cpu: 50m, memory: 64Mi} + limits: {cpu: 250m, memory: 192Mi} + readinessProbe: + tcpSocket: {port: 8443} + volumeMounts: + - {name: script, mountPath: /fixture, readOnly: true} + - {name: secrets, mountPath: /run/proof, readOnly: true} + - {name: tmp, mountPath: /tmp} + volumes: + - name: script + configMap: {name: proof-script} + - name: secrets + secret: {secretName: proof-secrets, defaultMode: 0440} + - name: tmp + emptyDir: {sizeLimit: 64Mi} +--- +apiVersion: v1 +kind: Service +metadata: {name: gateway, namespace: hypertrain-proof-gateway} +spec: + selector: {app: gateway} + ports: [{port: 443, targetPort: 8443}] +--- +apiVersion: batch/v1 +kind: Job +metadata: {name: mock-miner, namespace: hypertrain-proof} +spec: + backoffLimit: 0 + suspend: true + activeDeadlineSeconds: 180 + template: + metadata: + labels: {hypertrain.network/role: master} + spec: + restartPolicy: Never + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: {type: RuntimeDefault} + containers: + - name: miner + image: ghcr.io/cortexlm/hypertrain@sha256:ebdb8bc97122488682c63200f55fae92a980e5a75b50cbe4d0ce3bde0fbf7fdb + command: [/opt/venv/bin/python, /fixture/relay_kind_smoke.py, --fixture, miner] + resources: + requests: {cpu: 50m, memory: 64Mi} + limits: {cpu: 250m, memory: 192Mi} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: [ALL]} + volumeMounts: + - {name: script, mountPath: /fixture, readOnly: true} + - {name: secrets, mountPath: /run/proof, readOnly: true} + - {name: tmp, mountPath: /tmp} + volumes: + - name: script + configMap: {name: proof-script} + - name: secrets + secret: {secretName: proof-secrets, defaultMode: 0440} + - name: tmp + emptyDir: {sizeLimit: 96Mi} diff --git a/deploy/k8s/local/kind.yaml b/deploy/k8s/local/kind.yaml new file mode 100644 index 0000000..f382ef7 --- /dev/null +++ b/deploy/k8s/local/kind.yaml @@ -0,0 +1,9 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + disableDefaultCNI: true + podSubnet: 192.168.0.0/16 +nodes: + - role: control-plane + # The smoke script replaces/validates this against versions.json. + image: kindest/node:v1.35.0@sha256:452d707d4862f52530247495d180205e029056831160e22870e37e3f6c1ac31f diff --git a/deploy/k8s/overlays/apac/kustomization.yaml b/deploy/k8s/overlays/apac/kustomization.yaml new file mode 100644 index 0000000..55f50fc --- /dev/null +++ b/deploy/k8s/overlays/apac/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: hypertrain-relay-apac +resources: [../../base] +patches: + - target: {kind: Namespace, name: hypertrain-relay} + patch: |- + - op: replace + path: /metadata/name + value: hypertrain-relay-apac + - target: {kind: ConfigMap, name: relay-environment} + patch: |- + - op: replace + path: /data/REGION + value: apac diff --git a/deploy/k8s/overlays/eu/kustomization.yaml b/deploy/k8s/overlays/eu/kustomization.yaml new file mode 100644 index 0000000..d0ac0c3 --- /dev/null +++ b/deploy/k8s/overlays/eu/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: hypertrain-relay-eu +resources: [../../base] +patches: + - target: {kind: Namespace, name: hypertrain-relay} + patch: |- + - op: replace + path: /metadata/name + value: hypertrain-relay-eu + - target: {kind: ConfigMap, name: relay-environment} + patch: |- + - op: replace + path: /data/REGION + value: eu diff --git a/deploy/k8s/overlays/local/kustomization.yaml b/deploy/k8s/overlays/local/kustomization.yaml new file mode 100644 index 0000000..f30ce60 --- /dev/null +++ b/deploy/k8s/overlays/local/kustomization.yaml @@ -0,0 +1,16 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: hypertrain-relay-local +resources: [../../base] +patches: + - path: patch.yaml + - target: {kind: Namespace, name: hypertrain-relay} + patch: |- + - op: replace + path: /metadata/name + value: hypertrain-relay-local + - target: {kind: ConfigMap, name: relay-environment} + patch: |- + - op: replace + path: /data/REGION + value: local diff --git a/deploy/k8s/overlays/local/patch.yaml b/deploy/k8s/overlays/local/patch.yaml new file mode 100644 index 0000000..9855e31 --- /dev/null +++ b/deploy/k8s/overlays/local/patch.yaml @@ -0,0 +1,44 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: hypertrain-relay +$patch: delete +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hypertrain-relay +spec: + template: + spec: + containers: + - name: relay + command: [/opt/venv/bin/python, /fixture/relay_kind_smoke.py, --fixture, relay] + args: [] + lifecycle: + preStop: + exec: + command: + - /opt/venv/bin/python + - -c + - "import httpx,pathlib,socket; t=pathlib.Path('/run/proof/drain.token').read_text().strip(); h=socket.gethostbyname(socket.gethostname()); r=httpx.post('http://'+h+':8000/internal/drain',headers={'Authorization':'Bearer '+t},timeout=100); r.raise_for_status()" + volumeMounts: + - {name: script, mountPath: /fixture, readOnly: true} + - {name: fixture-secrets, mountPath: /run/proof, readOnly: true} + resources: + requests: + cpu: 50m + memory: 64Mi + ephemeral-storage: 32Mi + limits: + cpu: 250m + memory: 192Mi + ephemeral-storage: 128Mi + volumes: + - name: script + configMap: {name: proof-script} + - name: fixture-secrets + secret: {secretName: proof-secrets, defaultMode: 0440} + - name: cache + emptyDir: + sizeLimit: 96Mi diff --git a/deploy/k8s/overlays/us/kustomization.yaml b/deploy/k8s/overlays/us/kustomization.yaml new file mode 100644 index 0000000..7c7f851 --- /dev/null +++ b/deploy/k8s/overlays/us/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: hypertrain-relay-us +resources: [../../base] +patches: + - target: {kind: Namespace, name: hypertrain-relay} + patch: |- + - op: replace + path: /metadata/name + value: hypertrain-relay-us + - target: {kind: ConfigMap, name: relay-environment} + patch: |- + - op: replace + path: /data/REGION + value: us diff --git a/deploy/k8s/versions.json b/deploy/k8s/versions.json new file mode 100644 index 0000000..f27fc6a --- /dev/null +++ b/deploy/k8s/versions.json @@ -0,0 +1,19 @@ +{ + "server_image": "ghcr.io/cortexlm/hypertrain@sha256:ebdb8bc97122488682c63200f55fae92a980e5a75b50cbe4d0ce3bde0fbf7fdb", + "server_source_sha": "f92a3a9c1bcedbf968318c2845bb66c71f19e100", + "server_build_job": "https://github.com/CortexLM/hypertrain/actions/runs/37815568607/job/113454067723", + "kind_version": "v0.31.0", + "kubernetes_version": "v1.35.0", + "node_image": "kindest/node:v1.35.0@sha256:452d707d4862f52530247495d180205e029056831160e22870e37e3f6c1ac31f", + "calico_version": "v3.31.2", + "calico_manifest_url": "https://raw.githubusercontent.com/projectcalico/calico/dd5575465ed825f86a3bc10f17bb5f7dc106d0f0/manifests/calico.yaml", + "calico_manifest_sha256": "c74c5a073af478dc841c909de708bd93242d685a6543616638c8f81b5d8daa71", + "calico_images": { + "docker.io/calico/node:v3.31.2": "docker.io/calico/node@sha256:5255c2844abb4eea023cd15ceda5380d5bab3d5cf714fd113774597d2265f876", + "docker.io/calico/cni:v3.31.2": "docker.io/calico/cni@sha256:62f50ed3070fd5da4df907fc2d618faf53f2d69d2c64232f7f464a646806cb5b", + "docker.io/calico/kube-controllers:v3.31.2": "docker.io/calico/kube-controllers@sha256:7308b279c79816c9386a593644814a7f0f5caaf3b753982cbf226953782524ee" + }, + "harness": "kind-calico", + "max_node_cpus": 4, + "max_node_memory_bytes": 4294967296 +} diff --git a/docs/RESULTS.md b/docs/RESULTS.md index 66c94e9..7487ae4 100644 --- a/docs/RESULTS.md +++ b/docs/RESULTS.md @@ -2,6 +2,24 @@ Final results of the hypertrain challenge build, with the budget reconciliation and an honest qualification status. Numbers tagged `value{ev:path#key}` are checked by `scripts/check_doc_numbers.py` against [experiments/results/final_results.json](../experiments/results/final_results.json). Evidence paths below are in the workspace evidence directory `.omo/evidence/hypertrain-challenge/` (not part of the published tree). +## Current network v2 update - 2026-10-10 + +The original challenge results and budget below are historical, not current network-v2 completion or current account balance. The root network ledger is **20/23; allocation80, CPU verification and final push/CI remain open**. Later scoped evidence resides in workspace `.omo/evidence/hypertrain-network/`, outside the published tree: + +| Scope | Actual credit | Independent artifact | +| --- | --- | --- | +| FIRST4 CUDA008 | Four CUDA rounds/eight rank workers across two two-GPU hosts, both580.95.05; small61,856-parameter decoder, two-host bitwise PASS, workload_allowed=false | `qualification-usd50-20261009T181257Z/NEXT30-008-RESULT-VERIFY.md`; final authenticated instance/volume closure retained separately | +| Public miner CLI | In-process public run-v2 CPU interface PASS; not installed-CLI subprocess or CUDA attestation | `MINER-PUBLIC-CLI-VERIFY.md` | +| Local Kubernetes r2 | Import/source109, TLS/CNI, four authorized10MiB transfers, fixture-master acceptance, replica/regional failover, retention/GC and namespace denial PASS; owned cleanup verified | `K8S-TRANSPORT-R2-RESULT-VERIFY.md`; original outer caller exit1 remains historical failure | +| Isolated caller phase fix | READY without preownership ID stays pending; two native-event cases passed, malformed terminal fails closed | `K8S-CALLER-READY-PHASE-FIX-VERIFY.md`; no new cluster execution or retroactive caller exit0 | +| Product relay policy | Gateway namespace AND pod TCP8443 shared-base delta; one passing regression, actual local/EU/US/APAC renders | `K8S-RELAY-PRODUCT-INTEGRATION-VERIFY.md`; APAC render is not APAC runtime | +| Admission contracts | Operator, consumer, wrapper and antiabuse bounded CPU contracts complete; public CLI multi-rank evidence is in-process CPU, separate from historical FIRST4 CUDA008 | `PUBLIC-ADMISSION-CUSTODY-VERIFY.md`, `ANTIABUSE-TRUST-VERIFY.md`, `MINER-PUBLIC-CLI-VERIFY.md`; production116 unproven | +| Capacity | Owned accounting, collector and native CPU fallback safeguards previously reviewed; current source requalification pending after type/runtime changes invalidated the old112 pins. Genuine80 H2 execution unrun; CPU verification not PASS | `CAPACITY-CPU-FALLBACK-VERIFY.md` is historical source review; fresh CPU approval absent and root execution admission pending | + +Internal runtime-custody or `NOTSTOREACCEPTED` descriptors retain private evidence only. Accepted shadow-execution remains gated by the original validated execution context, actual backend qualification and economic authority; private retention does not grant eligibility or funded production116 acceptance. + +APAC runtime, L0 finality, full D3 integration, funded116 continuation and roster/service capacity remain unproven. No quality, profitability, current decoder/production or global network completion claim follows these gates. See [network-v2.md](network-v2.md#current-proof-boundary) for operator boundaries. No publication occurred during this docs-only update. + ## 1. Status **Equivalence not demonstrated at >=0.5B; GPU proxy ladder gated.** The determinism machinery works on real hardware. Training-quality parity with a centralized run is unproven at every scale that matters. diff --git a/docs/network-v2.md b/docs/network-v2.md new file mode 100644 index 0000000..ef07ea3 --- /dev/null +++ b/docs/network-v2.md @@ -0,0 +1,437 @@ +# Network v2 operator guide + +Network v2 uses owner-signed `RunManifestV2`, `ht/2` messages and `/v2` routes. +V1 runs and signing domains stay separate; existing state directories are not +upgraded. Use this guide for the multi-rank client, not the single-rank v1 +`join`/`run` workflow in [miner.md](miner.md). General layouts use +`N = DP * EP`, pipeline size 1; OpenDecision uses dense DP=N, EP=1, optional +ZeRO-1. Logical rank support is not hardware qualification. + +## Current proof boundary + +| Surface | Implemented/proven scope | Limit | +| --- | --- | --- | +| OpenDecision MLM, decision, distill | Each objective passed a CPU N2/DP2/EP1/ZeRO-1, H2/J1, four-client, two-round carry gate: independent audits, exact m/v/step/EF, capped tapes, test reward finality and checkpoint | Tiny functional evidence, not quality/calibration, throughput, profitability or CUDA. Original signed admission artifacts retained; final pytest production journals/checkpoints were removed by retention. Production outcomes are observed gates, not fresh artifact re-verification | +| Backend authority | One immutable owner-reviewed backend selection shared by admission, audit, trace geometry, referee and rollback; restart/cache contradictions reject. FIRST4 attempt008 passed bounded same-driver two-host CUDA bitwise qualification | Selection/dispatch and public CLI CPU execution are not CUDA attestation. Attempt008 qualifies only its frozen small decoder/source/image/profile; workload_allowed=false, current production client admission remains separate | +| Codec | One shared metadata parser validates complete payload before decode, exact expected shapes and raw-byte cap | Expected shapes must come from trusted manifest/state. Legacy `decompress(payload)` has no geometry cap; byte preflight does not bound an earlier storage fetch or legitimate tensor footprint | +| Scale | Allocator supports up to 1024 candidate metadata entries; legacy service path permits at most 16 without a capacity profile. Bounded capacity intake/opening fail-closed proof is APPROVE: authenticated profile opening rejects before cache, state loads, tensor allocation or publication | Allocator1024 is not service capacity. Tiny32 profile metadata/intake exist; heavy32 remains denied pending OS limits, consumer guards, bounded reads and execution charging | +| Rollback | CPU two-round exclusion, fresh replay, CAS/restart and checkpoint lineage; authenticated predecessor history reconstruction | History bound 16 rounds; larger history fails closed. No CUDA rollback proof | +| Relays | Local HTTP/ASGI lifecycle evidence; independent r2 local kind import/source109, TLS/CNI, upload/fixture-master acceptance, restart/replica/regional fallback, retention/GC and namespace denial PASS with owned cleanup. Product gateway namespace AND pod TCP8443 delta passed one four-overlay render regression | Original r2 outer caller exit1 retained; isolated READY phase repair passed two native-event cases, not another cluster run. APAC runtime, L0 finality and full D3/funded service integration unproven; no WAN or three-live-region independence claim | +| External services | Teacher-label and R2 client/config paths exist | Live OpenRouter/R2 checks deferred pending credentials. No live teacher or R2 success inferred from CPU fixtures | + +Root ledger **20/23; three tasks remain open: allocation80, CPU verification and final push/CI**. Workspace evidence sources +(not shipped runtime authority): `K8S-TRANSPORT-R2-RESULT-VERIFY.md`, +`K8S-CALLER-READY-PHASE-FIX-VERIFY.md`, +`K8S-RELAY-PRODUCT-INTEGRATION-VERIFY.md`, `MINER-PUBLIC-CLI-VERIFY.md` in +`.omo/evidence/hypertrain-network/`; FIRST4 is +`qualification-usd50-20261009T181257Z/NEXT30-008-RESULT-VERIFY.md` there. +Public CLI proof is in-process CPU run-v2, not installed-CLI subprocess or CUDA. +Operator, consumer, wrapper and antiabuse contracts are complete within their +verified CPU scope. Capacity controller safeguards were previously reviewed under +`CAPACITY-CPU-FALLBACK-VERIFY.md`; current source requalification is pending after +type/runtime changes. Genuine80 execution remains unrun; fresh CPU approval is +absent and root execution admission remains pending. Production116 gates remain unproven. +Internal runtime-custody or `NOTSTOREACCEPTED` descriptors are private retention, +not accepted shadow-execution, backend/economic qualification or eligibility. +Accepted shadow-execution requires the original validated execution context, +actual backend qualification and economic authority; private retention cannot +bypass those gates or establish funded production116 acceptance. + +These are scoped proofs, not an all-network release approval. Harness, CLI and +capacity work may change; publication must reconcile their final evidence before +removing a pending label. Ordinary checkpoint verification independently replays +outer aggregation; inner training/carry evidence comes from service audits and +anchor comparisons, not the checkpoint alone. + +## Install and pin the run + +Run from the repository root, Linux x86_64, Python 3.12. Commands below are +source-checked recipes; they were not executed during this documentation pass. + +```bash +uv sync --frozen --extra trainer +uv run --frozen hypertrain-miner --help +uv run --frozen hypertrain-miner run-v2 --help +uv run --frozen hypertrain-miner watch --help +``` + +For OpenDecision also install the pinned `od` extra. The checkout's trainer extra +uses CPU Torch; CUDA requires the exact reviewed image/runtime, not merely +changing `device`. Obtain the owner hotkey, full wrapper run ID, image digest, +signed policies and verified dataset from the operator. Protect both hotkey and +coldkey seed files: regular files, mode 0600, owned by the running user; 32 raw +bytes or 64 hex characters. Never put seeds in the TOML or publish them. + +Minimal `miner.toml` template; replace the capitalized values with actual pins: + +```toml +api = "https://MASTER/challenge/hypertrain" +keyfile = "miner.key" +workdir = "work" +state_source = "states" +image_digest = "sha256:IMAGE_DIGEST" +run_id = "RUN_ID" +owner_hotkey = "OWNER_SS58" +device = "cpu" +``` + +`api` is the service base, without `/v2/runs/...`. `NetworkMiner` requires pinned +`run_id` and `owner_hotkey`; it verifies the signed wrapper before work. +`state_source` and `image_digest` remain required shared config fields; v2 round +inputs are fetched through the job/object endpoints. Relative paths resolve +against the TOML directory. Environment `HYPERTRAIN_MINER_` overrides +matching TOML fields; unknown TOML keys reject. Other existing fields: +`data_dir`, `poll_seconds` (2.0), `chunk_bytes` (67108864), +`allow_file_upload` (false). Do not assume these v1-oriented tuning fields alter +the v2 server's deadlines or relay chunk policy. + +## Join, prove work, fund, graduate + +Set `REQUEST_ID` to a fresh 64-hex request ID, `EXPIRY_BEACON` to a future +verified beacon round. Public join needs both keys: + +```bash +uv run --frozen hypertrain-miner join-v2 --config miner.toml \ + --coldkey cold.key --request-id "$REQUEST_ID" --expiry "$EXPIRY_BEACON" +uv run --frozen hypertrain-miner status-v2 --config miner.toml +``` + +Applications/proofs are bound to the exact signed manifest, admission, nonce, +image, layout and epoch. A decoder/reset/other-objective admission cache cannot +authorize a carry run. Hardware hints are advisory; a fresh full-round work screen +and independent same-layout replay establish work, not physical GPU ownership. +Trial `WorkProof.challenge_hash` is the canonical `JoinChallenge` body digest, +which includes its nonce; `WorkScreenV2` is reserved by admission and nonce. +`WorkProof` reservations include admission and `challenge_hash`, not delta bytes +alone. Retry the exact stored signed operation: changing expiry or signature does +not create a new semantic attempt (`protocol/envelope_v2.py:replay_key`). + +For each challenge, obtain the coordinator-signed envelope from +`GET /v2/runs/{run_id}/join/{admission_id}/challenge`. The operator must supply the +matching staged `IslandJobV1` and all files in its `object_paths` beside the job. +The CLI does not download/stage an admission job automatically. Given those +existing files: + +```bash +uv run --frozen hypertrain-miner probe-v2 --config miner.toml \ + --job trial/job.json --challenge challenge.json > proof.json +uv run --frozen hypertrain-miner proof-v2 --config miner.toml --proof proof.json +``` + +`probe-v2` executes the screen and uploads original rank-0 state/EF/delta/leaves; +stdout contains the signed proof/screen pair. The operator independently prepares +and finalizes references through authenticated admin routes; miners cannot +self-finalize or turn a successful probe into ACTIVE status. + +Current admission policy fixes E=4 and requires 12 clean finalized participations +plus canary blocks 1,2,3. The first four are shadow-only. Later PROBATION influence +also requires funded eligibility; ACTIVE requires current screening, sufficient +matured units locked, no pending dispute/outbox and the conservative economic +bound. Actual audit floor is 100% anchored-full replay in this phase, despite +lower nominal probation/base rates. Calendar age or a legacy bond flag is not +graduation. Join quotas, one concurrent non-ACTIVE identity per coldkey, funded +locks and known-owner caps bound exposure; they do not reveal every Sybil owner. + +Prefix quotas use a stable coordinator-private HMAC namespace pinned atomically +in the store. Reopening the same store with the same coordinator does not reset +current prefix buckets. A populated legacy store without the namespace pin, or +a coordinator namespace mismatch, fails with HTTP 503 without resetting quotas. +No automatic migration is provided; separate operator resolution is required. + +Funding uses signed `EscrowLock` via `POST /v2/runs/{run_id}/escrow/lock`, not a +CLI `--bond` flag. Test genesis has pinned nonfinancial origins; production mode +has separate reward authority/issuance bounds and no test-genesis conversion. +Rewards are issued only through accepted finality, uniquely tracked by origin, +then mature after vesting/dispute conditions. Transfers preserve origin units; +fraud burns collectible exposure rather than redistributing it. Shadow work does +not influence global theta; authorized production bootstrap issuance is a +separate policy, not proof of market-backed collateral. + +## Train one live round and watch disputes + +Only run after the operator has opened an eligible, funded round: + +```bash +uv run --frozen hypertrain-miner run-v2 --config miner.toml --round 0 +``` + +This follows accept, authenticated inputs, exact-N launcher, one island commit, +original artifacts, regional custody, master acceptance and delta submission. +Tokens derive from assigned samples times sequence length, not self-reported GPU +count. `UPLOADED` is not audit success, settlement or payment. Unlike v1 `run`, +`run-v2` has no `--rounds`/`--start` option. Preserve the work directory; do not +claim the CLI is a complete unattended production scheduler. + +The decoder operation driver has an executable original four-message +AcceptV2/CommitV2/WorkProof/DeltaManifestV2 consumer. Capture records ordered +original requests, bounded responses and only the successful-response accepted +prefix. Canonical publication-path and same-Event probe-cancellation repairs are +review-approved; they are not current missing implementations. These scoped +`D2-DECODER-DRIVER-VERIFY.md`, `D2-CAPTURE-VERIFY.md` and +`SERVICE-ROLE-DISPATCH-VERIFY.md` approvals do not prove actual paid CUDA or +complete owner-accepted trial/audit/dispute/finality execution. + +Production `WorkProof.challenge_hash` is the accepted canonical `CommitV2` body +digest. The upload-grant `commit_hash` query uses that same body digest, not the +delta hash or signed envelope hash. Identical delta bytes across live rounds are +valid with distinct explicit commit/challenge subjects; an omitted selector is +accepted only for a unique exact match, otherwise ambiguity rejects. See +`miner/core.py:NetworkMiner.run_round` and +`protocol/envelope_v2.py:replay_key`; signatures/expiry changes cannot bypass +their semantic reservations. + +`run-v2 --job path/job.json` is a local staged-job launch returning artifact paths; +it does not submit a live round. Both forms use the same launcher. Failure, +deadline or rank disagreement prevents a partial validated publication. + +Persist the exact original job JSON beside its `published/` directory. Watch +independently of training: + +```bash +uv run --frozen hypertrain-miner watch --config miner.toml \ + --job work/round/job.json --timeout 30 +``` + +This command performs one bounded long-poll pass, then exits with a processed +count; repeat under an operator supervisor. Timeout range is 0..30 seconds. +STEP/LAYER/OP answers and requested `StateServe` proofs come from validated +published traces/checkpoints. The coordinator signature covers the event's +`Turn.contest.w`; the watcher requires that round to match the original validated +publication job, alongside run and signer checks. Wrong-round events raise +`WATCH_PUBLICATION_CONTEXT` before event-driven trace/state responses, signing, +outbox writes, acknowledgments, sends or cursor advancement; they are not skipped +and acknowledged. Pending events and fetched batches are checked before delivery. +Durable cursor/outbox retries identical signed +replies; one watch-directory lock prevents concurrent watchers. Do not share a +watch directory across identities or discard it on restart. The CLI has no +`--level` or dispute-opening subcommand: signed `DisputeV2`, `BisectV2`, +`StateServe`, `ResolutionV2` and recovery routes own those transitions. +Independent referees use their own accepted current execution deadline while +preserving historical publication descriptors. Losing-party checks, funded +contests and fixed absolute horizons prevent arbitrary transcript resets. + +## Weights, finality and custody + +Flat aggregation retains each miner's original delta: norm preclip, previous +outer-update CClip, then deterministic integer allocation. Q=16777216; final +weights sum Q, each miner <=4194304, total probation <=4194304, plus signed +known-owner group bounds. No post-cap renormalization. Four eligible ACTIVE +miners can carry exact quarter weights; infeasible cohorts reject. Similarity +alone is not fraud evidence. Audits replay each miner's own assignment. + +At most two speculative rounds may be applied; opening w+2 waits for w to settle +or be excluded and repaired. Retain original input objects, signed tapes, +accepted envelopes, carry anchors and full bounded predecessor history. Reward +`get_weights` is not a model-checkpoint download endpoint. + +Regional relays are transport/cache only, not regional arithmetic aggregators. +An upload opportunity acknowledgment is not custody. Completion failure requires +an exact complete receipt or full original chunk coverage, authenticated healthy +witnesses and original grant/assignment lineage. The master fetches/hash-checks +original bytes before acceptance. Rescue preserves custody without backdating +late acceptance or charging the miner for proven storage failure. +Retention release waits for finality, disputes and vesting; signed extensions, +overlapping old/new keys and drain preserve existing retrieval authority. + +Relay configuration is `RelayConfig` in `src/hypertrain/relay/app.py`: +`run_id`, `master_public`, `master_url`, signed `registry`/`network_manifest`, +`relay_id`, `region`, `active_key`, `signing_files`, `observer_public_keys`, plus +`s3_endpoint`/`s3_bucket`/`s3_prefix`/`s3_region` or `local_backing`. +`RELAY_CONFIG` selects its JSON; `RELAY_SECRETS` selects mounted signing files, +`drain.token` and S3 `object.json`. Do not mount coordinator/admin keys into relays. +Authoritative external backing is separate from disposable pod cache. + +Operator manifests are `deploy/k8s/base` and overlays `eu`, `us`, `apac`, `local`. +Render before considering deployment: + +```bash +kubectl kustomize deploy/k8s/overlays/eu +``` + +Deployment expects operator-created `relay-contract` ConfigMap and +`relay-runtime-v1` Secret; validate the pinned image against current source, +signed contracts, TLS, policy enforcement, backing credentials and cluster +resources first. The independent local r2 test imported the closed historical +source109 image and passed actual CNI/TLS, upload/fixture-master acceptance, +restart, replica/regional fallback, retention/GC and namespace denial; owned +cleanup and all native unit terminals are verified. Its outer caller exited1 +on early READY without ownership, despite runtime PASS and a legitimate VERIFIED +cleanup barrier. The isolated caller phase fix passed two native-event cases; +it does not rewrite that failed caller or certify current product caller bytes. +The product shared-base policy now permits only gateway namespace AND pod +TCP8443 (Service443 targetPort8443), retaining443/default deny; one actual +four-overlay render regression passed. APAC actual runtime, L0 finality and +full D3/restored funded service integration remain unproven. +Deployment is +operator-managed Kustomize, not a custom reconciliation controller. + +## Executable local D2 preparation + +The preparation tool exists; do not recreate keys/configs manually or substitute +live source for its frozen archive. From the repository root, supply an +operator-reviewed freeze and an unused output directory through `REVIEWED_FREEZE` +and `PREPARATION_OUTPUT`; set `CPU_GENESIS_UNIX` to the approved synthetic genesis +time. These variables are inputs, not shipped authority. The base recipe is: + +```bash +flock /tmp/hypertrain-network-cpu.lock nice -n 19 taskset -c 0-3 \ + env OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 PYTHONDONTWRITEBYTECODE=1 \ + uv run --frozen python scripts/network_gpu_prepare.py \ + --freeze "$REVIEWED_FREEZE" \ + --output "$PREPARATION_OUTPUT" \ + --cpu-genesis-unix "$CPU_GENESIS_UNIX" +``` + +This is a retained executable recipe, not permission to overwrite or rerun its +existing output. Preparation rejects an existing output, altered freeze or live +`hypertrain` modules loaded before frozen imports. New execution needs an unused +output and a freshly reviewed freeze. The archive binds 162 staged source files; +canonical qualification and operation-wrapper maps remain distinct from that +staging map. Installed dependencies are not claimed to be bundled in the archive. + +Output includes `prepared.json` with +`status="LOCAL_PREPARED_NOT_LAUNCHABLE"`; `public/` contains signed manifest/full +TestGenesis, dataset/tokenizer/proofs, full initialized state/optimizer/EF/v0, +CA/server certificate chains, public configs and hash index +`public/index.json`. `private/` directories are0700; key/seed/token/TOML +files0600, excluded from the public index. Synthetic deterministic identities are +not production credentials. `cpu-service/` is a conserved CPU genesis store; +`production-service/` is empty. Join requests exist as signed inputs, but no +join/trial is accepted and no ACTIVE history is created. CPU genesis initialization +is not a training trajectory, qualification or hidden extra H execution. + +Candidate preparation adds the approved input pair to the same tool. Set +`CANDIDATE_OUTPUT` to a distinct unused directory: + +```bash +flock /tmp/hypertrain-network-cpu.lock nice -n 19 taskset -c 0-3 \ + env OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 PYTHONDONTWRITEBYTECODE=1 \ + uv run --frozen python scripts/network_gpu_prepare.py \ + --freeze "$REVIEWED_FREEZE" \ + --output "$CANDIDATE_OUTPUT" \ + --cpu-genesis-unix "$CPU_GENESIS_UNIX" \ + --candidate-driver 580.95.05 --candidate-driver 580.159.03 +``` + +The recipe's `580.95.05`/`580.159.03` values are qualification **INPUT only**, +not observed drivers or paid GO. Actual FIRST4 attempt008 subsequently ran on +two hosts BOTH580.95.05 and passed same-driver bitwise qualification; this is +not evidence for the proposed cross-driver pair. Candidate changes only the +profile across copied frozen sources under new run/maps/source receipt; qualified +false and spend_authorized false remain explicit. Normal execution is +4 qualification +96 shadow +16 live =116,58 per host. Ten additional fault-trace +equivalents are contingent, giving a hard ceiling of126,63 per host, including +qualification. Unused fault capacity is not fault-proof completion or permission +for free retries. cu130/CUDA13.0/image contract does not change by supplying candidates. + +Prepared public remaining-authority fields are NOT_PRESENT for observed qualification, +accepted trials/admission, current production contexts, financial/rate admission, +runtime/service-controller admission and resolved launch contract. Original +validators reject `RUNTIME_AUTHORIZATION_REQUIRED`, +`SERVICE_CONTROLLER_NOT_ADMITTED` and `QUALIFICATION_SOURCE` in their respective +unadmitted scopes. Local synthetic source/index receipts do not satisfy those +runtime predicates. + +Attempt008's actual same-driver qualification and verified paid cleanup are +separate rescued evidence, not promotion of those prepared fields. The funded +116-operation continuation and live roster/service capacity remain unproven. + +`build_runtime`/`configure_dispatch` and the dynamic verified-beacon context +factory are implemented, but need exact current owner receipts, verified image, +observed role/host qualification, backend-before-run bootstrap, accepted subjects +and shared cancellation/clock installation. No manufactured context plan, source +qualification row or synthetic ACTIVE may bypass them. Four same-instance +qualification executions must precede the112 normal continuation operations; +up to ten further fault equivalents remain contingent within the126 ceiling, +admitted staging/runtime/rescue/cleanup budget and original deadline. + +Continuation accepts only the owner-signed reviewed record, with its rescued +same-instance qualification, exact source/profile/image subjects and all signer +identities validated before promotion. There is no raw-plan fallback. With the +operator-supplied `REVIEW_JSON`, `OWNER_SS58`, verified `BEACON` and unused +`RUN_DIRECTORY`, the existing entrypoint is: + +```bash +uv run --frozen python experiments/gpu_network_v2/orchestrate.py continue \ + "$REVIEW_JSON" --owner "$OWNER_SS58" --beacon "$BEACON" --run-dir "$RUN_DIRECTORY" +``` + +The joined signed CLI intake/flat-store qualification/identity preflight is +approved at metadata scope only. It does not establish production bootstrap, +live TLS, actual CUDA qualification, public ACTIVE12 or current decoder service +integration. This command is not a launch authorization. + +`D2-QUOTE-VERIFY.md` corrects historical evidence to two advertised595.84 offers, +54488349/machine153074 and54730345/machine24815. One-hour/80GB/30GB quoted +2.02+1.80 plus one5USD buffer is **8.82USD conditional**, not verified current +total cost: prior experiment debit/pending charges are unknown. That historical +pair does not price the580 candidates. Before execution, refresh authenticated +account, complete instances/volumes/charges and exact offers; attribute liabilities +and obtain explicit bounded GO. No such admission or provider action is credited +to preparation. + +Later relevant store/source changes invalidate that reviewed execution-source +closure. The publication inventory identifies eight stale D2 source entries; +the historical candidate freeze does not cover the new bytes. Regenerate/review +effective staged archive/map, canonical qualification +map, operation-wrapper map and owner receipt subjects before launch. Preserve +old hashes/artifacts as history; do not silently patch the frozen tree or infer a +new image build from package/runtime repairs. + +## Configuration and remaining activation gates + +Policies and wire fields: `src/hypertrain/protocol/messages_v2.py`. +CLI/config: `src/hypertrain/miner/{cli,core}.py`. Service setup: +[operator.md](operator.md). Teacher configuration: [teacher.md](teacher.md). +Dataset R2 credentials/publication: [r2.md](r2.md); relay object credentials are +the separate `S3Credentials` contract in `src/hypertrain/data/store.py`. + +The portable D3 capability is **restored-finalized-admission** for the original +OD MLM run +`587fc10ec7aeee92e235d39c9332c763e16cde05e5e13a8562204c700adce325`. +Its validator independently checks original owner manifest/TestGenesis, signed +trial challenges/proofs/screens/finalities, saved artifacts, conserved funding +and current eligibility. Original dual-signed `JoinRequest` and signed +`EscrowLock` envelope bytes are missing: this restores finalized admission +authority, not complete historical join/lock intake. It neither fabricates those +signatures nor includes later production checkpoints. Source: +[`network_authority_snapshot.py`](../scripts/network_authority_snapshot.py) +`Bundle`, `_authority`, `bootstrap`. Root must separately admit the exact bundle +and image hashes; portable authority APPROVE is not D3 execution PASS. + +Activation status at this documentation correction: + +| Gate | Current scope and remaining work | +| --- | --- | +| D2 GPU | FIRST4 attempt008 same-driver580.95.05 two-host bitwise CUDA qualification PASS, four rounds/eight workers; workload_allowed=false. Public CLI CPU evidence separate. Cross-driver qualification, funded116 continuation, current production admission and complete live GPU graph unproven | +| D3 Kubernetes | Independent local r2 import/source109/TLS/CNI/upload/fixture-master/failover/GC/deny runtime PASS and owned cleanup verified. Historical outer caller exit1 retained; isolated READY phase fix two native cases PASS. Product policy four-overlay rendered regression one PASS. APAC runtime, L0 finality and full D3/restored funded integration remain unproven | +| Capacity | Bounded opening/job/audit/referee/rollback/input/aggregate/final metadata guards, owned accounting, terminal collector and native CPU fallback were previously reviewed. Current source requalification is pending after type/runtime changes invalidated the old112 pins. Genuine80 H2 execution remains unrun; eligible MATCH/apply/CAS and CPU verification remain pending. Fresh CPU approval is absent and root execution admission remains pending; heavy32 DENIED | +| Remaining staging/runtime | Executable paths now exist; fresh admitted source, observed host, owner runtime/service receipts and production context subjects are still required. Metadata/local readiness is not execution admission | + +These statuses follow `NETWORK-COMPLETION-MAP.md`, +`D2-DECODER-DRIVER-VERIFY.md`, `D2-CAPTURE-VERIFY.md`, +`D2-LAUNCH-PREP-VERIFY.md`, `D2-CANDIDATE-INPUT-VERIFY.md`, `D2-QUOTE-VERIFY.md`, +`CAPACITY-FINAL-INTEGRITY-VERIFY.md`, `D3-AUTHORITY-VERIFY.md`, +`D3-BUILD-ADMISSION.json` and `D3-POSTSTART-REAL-VERIFY.md` in the network evidence +directory, superseded for local runtime scope by `K8S-TRANSPORT-R2-RESULT-VERIFY.md`, +`K8S-CALLER-READY-PHASE-FIX-VERIFY.md` and +`K8S-RELAY-PRODUCT-INTEGRATION-VERIFY.md`. FIRST4 scope follows +`qualification-usd50-20261009T181257Z/NEXT30-008-RESULT-VERIFY.md`; +`MINER-PUBLIC-CLI-VERIFY.md` supplies distinct CPU-only interface evidence. + +Keep heavy32 `SERVICE_RUNTIME_NOT_ENFORCED` denial until independent runtime +memory/CPU/deadline enforcement, bounded consumer reads, original-roster guards, +durable attempt charging and explicit source-bound requalification pass. Codec +approval and allocator limits cannot grant execution. FIRST4 CUDA008 and local +r2 Kubernetes transport now have scoped PASS evidence, not full network/service +integration. APAC runtime, L0 finality, full D3 and funded116 remain unproven; +live OpenRouter/R2 credentials remain deferred. Historical Phase A/B GPU results +retain only their original scope. Root's ledger is20/23 with allocation80, CPU +verification and final push/CI open; no80 execution or publication occurred in +this documentation update. Earlier D3 +HostConfig false/null rejections, NEEDS-FIX +reports, frozen local image and historical595.84 quote remain chronology, not +current missing kernel/path implementation or present availability claims. +Bounded opening denial approval is specifically +`challenge/store.py:_service_boundary_v2` / `open_round_v2`, as reviewed in +`CAPACITY-INTAKE-VERIFY.md`; it does not qualify heavy execution. diff --git a/experiments/gpu_network_v2/orchestrate.py b/experiments/gpu_network_v2/orchestrate.py new file mode 100644 index 0000000..383f101 --- /dev/null +++ b/experiments/gpu_network_v2/orchestrate.py @@ -0,0 +1,2711 @@ +"""Reviewed D2 CLI using the existing provider/lifecycle; no implicit spend authorization.""" + +from __future__ import annotations + +import argparse +import importlib.util +import io +import json +import math +import os +import shlex +import subprocess +import sys +import tarfile +import threading +import time +import urllib.parse +from concurrent.futures import ThreadPoolExecutor +from decimal import Decimal +from pathlib import Path +from typing import Any + +from hypertrain.gpu_ops import budget +from hypertrain.gpu_ops.journal import ( + Journal, + Record, + durable_write, + flock, + fsha, + sha256, +) +from hypertrain.gpu_ops.launcher import Orchestrator, Reject +from hypertrain.gpu_ops.provider import Provider, list_rows, loopback, read_api_key +from hypertrain.gpu_ops.remote import Ssh + + +class DeadlineSsh(Ssh): + """Existing SSH methods, one absolute clock checked at each subprocess boundary.""" + + cutoff: float + lifecycle: Orchestrator + _limit: float + cancel: threading.Event | None = None + + @property + def timeout(self) -> int: + remaining = int(min(self._limit, self.cutoff - time.time())) + if ( + remaining <= 0 + or self.lifecycle.cleanup_started() + or self.cancel is not None + and self.cancel.is_set() + ): + raise Reject("remote_absolute_deadline_exhausted") + return remaining + + @timeout.setter + def timeout(self, value: int) -> None: + self._limit = value + + +def deadline_ssh(orch: Orchestrator, role: str, cutoff: float) -> DeadlineSsh: + ssh = orch.ssh(role) + bounded = DeadlineSsh(**vars(ssh)) + bounded.cutoff, bounded.lifecycle = cutoff, orch + return bounded + + +def effective_bundle(driver: Any, tree: Path, profile: Path) -> tuple[Record, bytes]: + """Explicit signed profile overlay; all other approved bytes remain unchanged.""" + sources = driver.sources(tree) + sources["experiments/gpu_network_v2/profile.json"] = fsha(profile) + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w") as archive: + for rel in sorted(sources): + data = ( + profile.read_bytes() + if rel == "experiments/gpu_network_v2/profile.json" + else (tree / rel).read_bytes() + ) + info = tarfile.TarInfo(rel) + info.size, info.mode, info.mtime = len(data), 0o600, 0 + archive.addfile(info, io.BytesIO(data)) + return sources, buffer.getvalue() + + +def refresh_readonly(provider: Provider) -> Record: + """Fresh GET-only custody/account evidence; no credentials in return value.""" + account = provider.call("GET", "/api/v0/users/current/", "network-account") + inventory = provider.call("GET", "/api/v1/instances/", "network-instances") + volumes = provider.call("GET", "/api/v0/volumes/", "network-volumes") + now = int(time.time()) + filters = urllib.parse.quote(json.dumps({"day": {"gte": now - 2592000, "lte": now}})) + charges = provider.call( + "GET", f"/api/v0/charges/?select_filters={filters}&limit=500", "network-charges" + ) + credit = budget.dec(account.parsed.get("credit")) if account.ok() else None + live = list_rows(inventory) + if ( + credit is None + or live is None + or not volumes.ok() + or not charges.ok() + or volumes.parsed.get("next_token") + or charges.parsed.get("next_token") + or not isinstance(volumes.parsed.get("volumes"), list) + or not isinstance(charges.parsed.get("results"), list) + ): + raise Reject("fresh_account_inventory_or_charges_unknown") + return { + "authenticated": True, + "available_credit_usd": str(credit), + "instances": live, + "volumes": volumes.parsed["volumes"], + "charges": charges.parsed["results"], + "verified_unix": int(time.time()), + "methods": ["GET"] * 4, + } + + +def artifact_budget(profile: dict[str, Any]) -> Record: + """Shape/count bounds from the actual AdamW checkpoint and trace paths; no allocation.""" + m, inn, lay, a = (profile[k] for k in ("model", "inner", "layout", "artifacts")) + d, e, ff, layers, vocab = (m[k] for k in ("d_model", "n_experts", "d_ff", "n_layers", "vocab")) + params = 2 * vocab * d + d + layers * (2 * d + 4 * d * d + d * e + 3 * e * d * ff) + if ( + params != m["param_count"] + or inn["opt"] != "adamw" + or inn["H"] % inn["J"] + or lay != {"pp": 1, "n_gpus": 2, "dp_size": 1, "ep_size": 2, "zero1": True} + or m["n_heads"] != m["n_kv_heads"] + or e != 4 + ): + raise Reject("unsupported_artifact_geometry") + tensors = 3 + 10 * layers + leaves = inn["H"] // inn["J"] + 1 + header = a["safetensors_header_bytes_per_file"] + state = 12 * params + header + ef = 4 * params + header + # SparseLoCo framing plus ceil(.01*n) indices/codes/levels for each tensor. + delta = math.ceil(params * 0.01) * 5 + tensors * 256 + forward = inn["grad_accum"] * (5 + 6 * layers + 20 * layers) + # Conservative: every gradient replicated; every parameter owned for all 3 state hooks. + step_events = forward + tensors * 3 + tensors + tensors * 3 + 6 + tensors * 3 + trace_events = inn["H"] * step_events + (leaves + 1) * 9 * tensors + trace = trace_events * a["trace_bytes_per_event"] + sample_count = inn["H"] * inn["micro_batch"] * inn["grad_accum"] * lay["n_gpus"] + sample_bytes = sample_count * (m["seq_len"] + 1) * 2 + # Inputs remain at job root AND copied into published; proofs/job JSON in metadata budget. + inputs = 2 * (state + ef + header + sample_bytes) + outputs = lay["n_gpus"] * ((leaves + 1) * state + ef + delta + trace) + logs = max(a["logs_bytes_per_execution"], 2 * math.ceil(max(trace, state) / 1024) * 1024) + execution = inputs + outputs + logs + a["metadata_bytes_per_execution"] + retained = execution * profile["schedule"]["executions_per_host"] + # Tar record rounding + path headers + terminating records for every retained file. + files_per_execution = lay["n_gpus"] * (leaves + 5) + 14 + tar_overhead = ( + files_per_execution * profile["schedule"]["executions_per_host"] * 2048 + 1048576 + 10240 + ) + rescue = retained + tar_overhead + return { + "param_count": params, + "tensor_count": tensors, + "state_bytes": state, + "trace_events_per_rank": trace_events, + "trace_bytes_per_rank": trace, + "execution_bytes": execution, + "retained_bytes_per_host": retained, + "rescue_tar_bytes_per_host": rescue, + "disk_artifacts_peak_bytes": retained + rescue, + "artifact_transfer_bytes_per_host": retained + rescue, + "rescue_min_bytes_per_second": math.ceil(rescue / a["rescue_transfer_seconds"]), + } + + +def readiness_budget(profile: dict[str, Any], evidence: dict[str, Any]) -> Record: + """Measured transport/image staging mandatory; advertised Mbps never qualifies rescue.""" + bounds = artifact_budget(profile) + a = profile["artifacts"] + rate, image, stage = ( + a[k] + for k in ( + "rescue_bytes_per_second", + "image_and_dependency_bytes", + "staging_bytes_per_host", + ) + ) + if ( + type(rate) is not int + or rate < bounds["rescue_min_bytes_per_second"] + or type(image) is not int + or image < 1 + or type(stage) is not int + or stage < 1 + or evidence.get("measured_rescue_bytes_per_second") != rate + or evidence.get("image_and_dependency_bytes") != image + or evidence.get("staging_bytes_per_host") != stage + ): + raise Reject("measured_artifact_transport_or_image_budget_missing") + if bounds["disk_artifacts_peak_bytes"] + image + 2 * stage > a["disk_bytes"]: + raise Reject("artifact_disk_budget_exceeded") + if a["rescue_transfer_seconds"] + a["delete_absence_seconds"] > 300: + raise Reject("rescue_delete_budget_exceeded") + return { + **bounds, + "priced_transfer_gb_per_host": math.ceil( + (bounds["artifact_transfer_bytes_per_host"] + image + stage) / 1_000_000_000 + ), + } + + +def staged_command( + profile: dict[str, Any], + root: str, + name: str, + python: str, + image: str, + backend: str, + *, + qualification: bool = False, +) -> str: + """One real launcher command; CUDA token and interpreter are not inferred from CPU tests.""" + if backend not in ("cpu", "cuda") or not name.replace("-", "").isalnum(): + raise Reject("invalid_staged_job_or_backend") + pin = profile["runtime"] + if backend == "cuda": + if ( + python != pin["remote_python"] + or image != pin["image_digest"] + or not pin["driver_allowlist"] + or (not qualification and not pin["qualified"]) + ): + raise Reject("cuda_image_interpreter_profile_unqualified") + env = " ".join( + shlex.quote(f"{k}={pin[k]}") + for k in ( + "CUBLAS_WORKSPACE_CONFIG", + "CUDA_DISABLE_PTX_JIT", + "OMP_NUM_THREADS", + "MKL_NUM_THREADS", + "PYTHONHASHSEED", + "NVIDIA_TF32_OVERRIDE", + ) + ) + return ( + f"cd {shlex.quote(root)} && env {env} PYTHONPATH=src " + f"HT_IMAGE_DIGEST={shlex.quote(image)} {shlex.quote(python)} " + f"experiments/gpu_network_v2/run.py out/{name} {backend}" + ) + + +def runtime_admission(profile: dict[str, Any], evidence: dict[str, Any]) -> int: + """Exact-profile evidence required before CREATE, not a guessed toy throughput.""" + from hypertrain.protocol.hashing import sha256_hex + from hypertrain.protocol.jcs import canonicalize + + bound = profile["execution_and_transfer_bound_seconds"] + if ( + type(bound) is not int + or not 0 < bound <= 40 + or profile["runtime_evidence_sha256"] != sha256_hex(canonicalize(evidence)) + ): + raise Reject("missing_or_unqualified_exact_profile_runtime") + if ( + evidence.get("profile_model") != profile["model"] + or evidence.get("layout") != profile["layout"] + or evidence.get("device") != "cuda" + or evidence.get("torch") != "2.14.0+cu130" + or evidence.get("full_round_transfer_bound_seconds") != bound + or evidence.get("no_slower_hosts") is not True + ): + raise Reject("runtime_evidence_profile_mismatch") + readiness_budget(profile, evidence) + pin = profile["runtime"] + if ( + evidence.get("image_digest") != pin["image_digest"] + or evidence.get("driver_allowlist") != pin["driver_allowlist"] + or evidence.get("python") != pin["remote_python"] + or evidence.get("cuda") != pin["cuda"] + or pin["backend"] != "cuda" + or not pin["image_digest"] + or not pin["qualified"] + or not evidence.get("full_round_artifacts_sha256") + or evidence.get("outer_tape_verified") is not True + ): + raise Reject("real_cuda_staged_reference_evidence_missing") + s = profile["schedule"] + if ( + s["executions_total"] != 126 + or s["executions_per_host"] != 63 + or s["shadow_executions"] != 96 + or s["live_executions"] != 16 + or s["probes"] != 2 + or s["fault_trace_reserve"] != 12 + ): + raise Reject("execution_reservation_mismatch") + total = ( + s["staging_seconds"] + 63 * bound + s["beacon_service_seconds"] + s["rescue_delete_seconds"] + ) + if total > 3300 or total + s["hard_margin_seconds"] > 3600: + raise Reject("one_hour_schedule_exceeded") + return total + + +def admission_plan( + profile: dict[str, Any], + evidence: dict[str, Any], + authorization: dict[str, Any], + snapshot: dict[str, Any], + offers: list[dict[str, Any]], + *, + qualification: bool = False, +) -> Record: + """Consume authenticated snapshots/quotes recorded by gpu_ops; malformed is never absent.""" + if qualification: + q = profile["microqualification"] + bounds = artifact_budget(profile) + if ( + q["max_full_rounds_per_host"] != 2 + or q["qualification_deadline_seconds"] != 900 + or sum( + q[k] + for k in ( + "staging_seconds", + "probe_and_transfer_seconds", + "rescue_delete_seconds", + "failure_margin_seconds", + ) + ) + > 900 + or evidence.get("root_verified_bounded_lifecycle") is not True + or evidence.get("candidate_image_digest") != profile["runtime"]["image_digest"] + or not profile["runtime"]["image_digest"] + or evidence.get("registry_image_verified") is not True + or not evidence.get("staged_contract_sha256") + or not evidence.get("deadline_supervisor_contract_sha256") + or bounds["disk_artifacts_peak_bytes"] + + q["candidate_image_max_bytes"] + + 2 * q["candidate_stage_max_bytes"] + > profile["artifacts"]["disk_bytes"] + ): + raise Reject("bounded_microqualification_contract_missing") + transfer_gb = q["candidate_transfer_gb_per_host"] + else: + runtime_admission(profile, evidence) + transfer_gb = readiness_budget(profile, evidence)["priced_transfer_gb_per_host"] + if ( + authorization.get("experiment") != "hypertrain-network-v2" + or authorization.get("currency") != "USD" + or authorization.get("ceiling_usd") != "15" + or authorization.get("max_instances") != 2 + or authorization.get("max_seconds_per_instance") != 3600 + or not authorization.get("source_message") + or not authorization.get("date") + ): + raise Reject("fresh_authorization_missing") + if ( + not snapshot.get("authenticated") + or snapshot.get("instances") != [] + or snapshot.get("volumes") != [] + or snapshot.get("uncapped_charges") is not False + or type(snapshot["verified_unix"]) is not int + or not 0 <= time.time() - snapshot["verified_unix"] <= 30 + ): + raise Reject("inventory_liability_or_quote_unknown") + if len(offers) != 2 or len({o["machine_id"] for o in offers}) != 2: + raise Reject("two_distinct_hosts_required") + for o in offers: + hourly = budget.dec(o.get("dph_total")) + if ( + o.get("gpu_name") != "RTX 5090" + or o.get("num_gpus") != 2 + or hourly is None + or hourly > 2 + or o.get("cpu_cores_effective", 0) < 8 + or o.get("cpu_ram", 0) < 32768 + or o.get("disk_space", 0) < 80 + or type(o["verified_unix"]) is not int + or not 0 <= time.time() - o["verified_unix"] <= 30 + ): + raise Reject("exact_host_shape_or_fresh_quote_missing") + prior = Decimal(snapshot["prior_attempts_usd"]) + liability = Decimal(snapshot["active_liabilities_usd"]) + available = Decimal(snapshot["available_credit_usd"]) - liability + if min(prior, liability, available) < 0: + raise Reject("invalid_accounting") + plan = budget.evaluate( + snapshot0_credit=available + prior, + current_credit=available, + offers=offers, + hard_deadline_seconds=3600, + disk_gb=80, + egress_gb=transfer_gb, + phase_cap=Decimal("15"), + reserve=Decimal("5"), + ) + hardware = Decimal(plan["hardware_worst_case_usd"]) + hourly_total = sum(Decimal(str(o["dph_total"])) for o in offers) + if not plan["admit"] or hardware > 5 or hardware - hourly_total > 1: + raise Reject("network_v2_cost_ceiling_exceeded") + return { + **plan, + "admission_mode": "qualification" if qualification else "workload", + "rental_deadline_seconds": 3600, + "qualification_deadline_seconds": 900, + "long_workload_allowed": not qualification, + } + + +class NetworkRuntime: + """Reuse existing journal, trusted SSH, stage, rescue and independent supervisor receipts. + + A prepared job is concrete IslandJobV1 + objects in out/; no fake API service. + This runtime does not claim the L0 probation/ledger/relay integration already exists. + """ + + def __init__(self, lifecycle: Orchestrator, profile: dict[str, Any]) -> None: + self.lifecycle, self.profile = lifecycle, profile + self.journal: Journal = lifecycle.j + + def owned_host(self, role: str) -> Record: + """Current admitted subject, never an advertised or synthetic instance.""" + receipt = self.lifecycle.receipt(role) + if receipt is None or self.lifecycle.cleanup_started(): + raise Reject("owned-role actual owned role observation absent") + return { + "instance_id": receipt["instance_id"], + "machine_id": self.lifecycle.host[role]["machine_id"], + "image_digest": self.profile["runtime"]["image_digest"], + } + + def reserve(self, role: str, name: str, phase: str, executions: int) -> Record: + """Debit intent before execution. Failed/unfinished intents never refund or retry.""" + if ( + role not in self.lifecycle.roles + or phase not in ("qualification", "workload") + or (executions != 2 if phase == "qualification" else not 1 <= executions <= 12) + ): + raise Reject("invalid_execution_reservation") + with flock(self.lifecycle.dir / "network-reservation.lock"): + if self.lifecycle.cleanup_started(): + raise Reject("network_cleanup_already_started") + if self.journal.last("network_execution_intent", role=role, name=name): + raise Reject("execution_intent_not_retryable") + intents = self.journal.all("network_execution_intent") + q = sum(r["executions"] for r in intents if r["phase"] == "qualification") + used_role = sum(r["executions"] for r in intents if r["role"] == role) + if ( + used_role + executions > 63 + or sum(r["executions"] for r in intents) + executions > 126 + ): + raise Reject("execution_reservation_exhausted") + if phase == "qualification": + if q + executions > 4 or any( + r["role"] == role and r["phase"] == "qualification" for r in intents + ): + raise Reject("qualification_reservation_exhausted") + elif q != 4 or self.journal.last("network_workload_promoted") is None: + raise Reject("workload_not_promoted") + return self.journal.append( + "network_execution_intent", + role=role, + name=name, + phase=phase, + executions=executions, + ) + + def promote(self, evidence: dict[str, Any], budget_plan: Record) -> Record: + """Promotion needs root-reviewed actual qualification, measured runtime and cost gates.""" + runtime_admission(self.profile, evidence) + with flock(self.lifecycle.dir / "network-reservation.lock"): + if self.lifecycle.cleanup_started(): + raise Reject("network_cleanup_already_started") + intents = self.journal.all("network_execution_intent", phase="qualification") + done = self.journal.all("network_qualification_done") + if ( + len(intents) != 2 + or sum(r["executions"] for r in intents) != 4 + or {r["role"] for r in intents} != set(self.lifecycle.roles) + or len(done) != 2 + or {r["role"] for r in done} != set(self.lifecycle.roles) + or evidence.get("root_reviewed_two_host_full_round_transfer") is not True + or budget_plan.get("admit") is not True + or budget_plan.get("long_workload_allowed") is not True + or time.time() + 61 * self.profile["execution_and_transfer_bound_seconds"] + 600 + > self.lifecycle.deadline() + ): + raise Reject("qualification_promotion_gate_failed") + return self.journal.append( + "network_workload_promoted", + remaining_total=122, + remaining_per_host=61, + hard_deadline_unix=self.lifecycle.deadline(), + ) + + def record_qualification(self, role: str, name: str, result: Record) -> Record: + """Root verified driver custody result; reservation already debited TWO full rounds.""" + intent = self.journal.last("network_execution_intent", role=role, name=name) + if ( + intent is None + or intent["phase"] != "qualification" + or intent["executions"] != 2 + or result.get("passed") is not True + or result.get("root_verified") is not True + or not result.get("result_sha256") + or not result.get("artifact_manifest_sha256") + ): + raise Reject("qualification_result_not_verified") + with flock(self.lifecycle.dir / "network-reservation.lock"): + old = self.journal.last("network_qualification_done", role=role, name=name) + if old: + if old["result"] != result: + raise Reject("qualification_result_changed_on_resume") + return old + return self.journal.append( + "network_qualification_done", + role=role, + name=name, + executions=2, + result=dict(result), + ) + + def stage_context( + self, + role: str, + paths: dict[str, Path], + context: str, + name: str, + cutoff: float, + cancel: threading.Event | None, + ) -> None: + """Private staging shares workload absolute deadline and cancellation boundary.""" + cutoff = min( + cutoff, self.lifecycle.deadline(), self.lifecycle.cfg["cleanup_deadline_unix"] - 300 + ) + ssh = deadline_ssh(self.lifecycle, role, cutoff) + ssh.cancel = cancel + root = self.lifecycle.remote_root(role) + logs = self.lifecycle.dir / "logs" + if ssh.run( + f"mkdir -m 700 -p {shlex.quote(root + '/' + context)}", name + "-context", logs + ).returncode: + raise Reject("continuation_private_context_stage_failed") + for rel, path in paths.items(): + target = root + "/" + rel + ssh.put(path, target) + if ssh.remote_sha256(target, logs) != fsha(path): + raise Reject("continuation_private_context_transport_changed") + if ssh.run(f"chmod 600 {shlex.quote(target)}", name + "-mode", logs).returncode: + raise Reject("continuation_private_context_mode_failed") + + def operation( + self, + spec: Any, + directory: Path, + *, + cancel: threading.Event | None = None, + ) -> Any: + """Internal accepted descriptor; debit, strict SSH, custody then validation.""" + from hypertrain.miner.island_launch import validate_artifacts + from hypertrain.protocol.envelope_v2 import parse_envelope, verify_envelope + from hypertrain.protocol.keys import decode_hotkey + + script = Path(__file__).resolve().parents[2] / "scripts/network_gpu_operation.py" + engine_spec = importlib.util.spec_from_file_location("network_operation_contract", script) + assert engine_spec is not None and engine_spec.loader is not None + engine = importlib.util.module_from_spec(engine_spec) + sys.modules[engine_spec.name] = engine + engine_spec.loader.exec_module(engine) + spec = engine.Operation.model_validate(spec.model_dump(mode="json")) + # Remote script is an operator support file: explicitly add it to the approved source map. + decode_hotkey(spec.hotkey) + role, job = spec.role, spec.job + cutoff = min( + spec.cutoff, + job.deadline, + self.lifecycle.deadline(), + self.lifecycle.cfg["cleanup_deadline_unix"] - 300, + ) + if ( + spec.cutoff != cutoff + or cutoff <= time.time() + or cancel is not None + and cancel.is_set() + or self.lifecycle.cleanup_started() + or not self.profile["runtime"]["qualified"] + or spec.image_digest != self.profile["runtime"]["image_digest"] + or job.manifest.training.reference_spec.image_digest != spec.image_digest + or job.manifest.training.reference_spec.layout.model_dump() != self.profile["layout"] + or self.journal.last("supervisor_ready") is None + or self.lifecycle.receipt(role) is None + or self.journal.last("ssh_trusted", role=role) is None + ): + raise Reject("operation_context_not_qualified") + attested = self.journal.last("network_operation_source", role=role) + map_hash = sha256(json.dumps(spec.sources, sort_keys=True).encode()) + if attested is None or attested["sources_sha256"] != map_hash: + raise Reject("operation_source_authority_missing") + if attested["run_id"] != job.run_id or spec.sources.get( + "scripts/network_gpu_operation.py" + ) != fsha(script): + raise Reject("operation_source_or_run_changed") + from hypertrain.gpu_ops.network_qualification import sources as actual_sources + + tree = Path(__file__).resolve().parents[2] + actual = actual_sources(tree) + # Profile is a root-reviewed effective overlay; its hash stays explicitly in spec. + profile_key = "experiments/gpu_network_v2/profile.json" + actual["scripts/network_gpu_operation.py"] = fsha(script) + if set(actual) != set(spec.sources) or any( + sha != spec.sources[p] for p, sha in actual.items() if p != profile_key + ): + raise Reject("operation_actual_source_changed") + operation_authority = self.journal.last( + "network_operation_authority", + role=role, + binding=spec.binding, + operation=spec.operation, + hotkey=spec.hotkey, + w=job.w, + ) + if operation_authority is None or operation_authority["spec_sha256"] != sha256( + spec.model_dump_json().encode() + ): + raise Reject("operation_accepted_authority_missing") + if operation_authority["owner"] != spec.owner: + raise Reject("operation_owner_changed") + receipt = self.lifecycle.receipt(role) + assert receipt is not None + if ( + receipt["instance_id"] != spec.instance_id + or self.lifecycle.host[role]["machine_id"] != spec.machine_id + ): + raise Reject("operation_accepted_host_changed") + name = "op-" + sha256( + (job.run_id + spec.operation + spec.hotkey + spec.binding + job.digest()).encode() + ) + root = self.lifecycle.remote_root(role) + remote = root + "/out/" + name + ssh = deadline_ssh(self.lifecycle, role, cutoff) + ssh.cancel = cancel + trusted = self.journal.last("ssh_trusted", role=role) + assert trusted is not None + if fsha(ssh.known_hosts) != trusted["known_hosts_sha256"]: + raise Reject("operation_known_hosts_custody_changed") + self.reserve(role, name, "workload", 1) + self.journal.append( + "network_operation_started", + role=role, + name=name, + binding=spec.binding, + job_sha256=job.digest(), + operation=spec.operation, + hotkey=spec.hotkey, + sources_sha256=map_hash, + cutoff=cutoff, + ) + error: Exception | None = None + custody: Record | None = None + try: + if ssh.run( + f"mkdir -p {shlex.quote(remote)}", + name + "-mkdir", + self.lifecycle.dir / "logs", + ).returncode: + raise Reject("operation_stage_failed") + paths = {rel: directory / rel for rel in job.object_paths.values()} + local_spec = self.lifecycle.dir / (name + ".json") + durable_write(local_spec, spec.model_dump_json().encode()) + paths["operation.json"] = local_spec + for rel, path in paths.items(): + if Path(rel).is_absolute() or ".." in Path(rel).parts or path.is_symlink(): + raise Reject("operation_input_path") + target = remote + "/" + rel + if ssh.run( + f"mkdir -p {shlex.quote(str(Path(target).parent))}", + name + "-parents", + self.lifecycle.dir / "logs", + ).returncode: + raise Reject("operation_stage_failed") + ssh.put(path, target) + if ssh.remote_sha256(target, self.lifecycle.dir / "logs") != fsha(path): + raise Reject("operation_transport_changed") + prefix = staged_command( + self.profile, + root, + name, + self.lifecycle.cfg["remote_python"], + spec.image_digest, + "cuda", + ) + command = prefix.split(" experiments/gpu_network_v2/run.py", 1)[0] + ( + f" scripts/network_gpu_operation.py {shlex.quote(root)} {shlex.quote(remote)} " + f"{shlex.quote(remote + '/operation.json')}" + ) + done = threading.Event() + condition = vars(cancel)["_cond"] if cancel is not None else None + + def stop() -> None: + if condition is None: + return + with condition: + condition.wait_for( + lambda: done.is_set() or cancel is not None and cancel.is_set(), + timeout=max(0, cutoff - time.time()), + ) + if not done.is_set(): + # SSH disconnect alone cannot kill grandchildren; explicit same-path PID group. + killer = self.lifecycle.ssh(role) + killer.timeout = int(min(10, cutoff - time.time())) + if killer.timeout > 0: + killer.run( + f"touch {shlex.quote(remote)}/operation-cancelled; " + f"test ! -f {shlex.quote(remote)}/operation-pid || " + f"kill -TERM -- -$(cat {shlex.quote(remote)}/operation-pid)", + name + "-cancel", + self.lifecycle.dir / "logs", + ) + + watcher = threading.Thread(target=stop) + watcher.start() + try: + proc = ssh.run(command, name, self.lifecycle.dir / "logs") + finally: + done.set() + if condition is not None: + with condition: + condition.notify_all() + watcher.join() + if proc.returncode or cancel is not None and cancel.is_set(): + raise Reject("operation_failed_or_cancelled") + except (Reject, OSError, subprocess.SubprocessError, ValueError) as exc: + error = exc + self.journal.append("network_operation_failed", role=role, name=name, error=str(exc)) + finally: + try: + custody = self.rescue(role, fresh=True) + except ( + Reject, + OSError, + subprocess.SubprocessError, + ValueError, + ) as rescue_error: + self.journal.append( + "network_operation_censored", + role=role, + name=name, + error=str(rescue_error), + ) + raise + if error is not None: + raise error + assert custody is not None + archive = Path(custody["tar"]) + if fsha(archive) != custody["tar_sha256"]: + raise Reject("operation_custody_changed") + destination = directory / name + if destination.exists(): + raise Reject("operation_result_destination_exists") + with tarfile.open(archive) as source: + for member in source.getmembers(): + prefix = "out/" + name + "/" + if member.name.startswith(prefix) and member.isfile(): + rel = member.name.removeprefix(prefix) + path = destination / rel + if Path(rel).is_absolute() or ".." in Path(rel).parts: + raise Reject("operation_custody_path") + stream = source.extractfile(member) + assert stream is not None + path.parent.mkdir(parents=True, exist_ok=True) + durable_write(path, stream.read()) + result = json.loads((destination / "operation-result.json").read_bytes()) + if ( + result.get("status") != "CAPTURED_NOT_ACCEPTED" + or result.get("binding") != spec.binding + or result.get("job_sha256") != job.digest() + or result.get("hotkey") != spec.hotkey + or result.get("operation") != spec.operation + or result.get("role") != role + ): + raise Reject("operation_original_result_binding") + rel = Path(result["publication"]) + if rel.is_absolute() or ".." in rel.parts: + raise Reject("operation_publication_path") + publication = destination / rel + artifacts = validate_artifacts(job, publication) + summaries = [ + json.loads((publication / f"rank-{rank}/summary.json").read_bytes()) + for rank in range(job.manifest.training.reference_spec.layout.n_gpus) + ] + if any(s["backend"] != "cuda" for s in summaries): + raise Reject("CPU_publication_cannot_serve_as_CUDA") + if spec.trace and any( + not (publication / f"rank-{r}/trace.json").is_file() for r in range(len(summaries)) + ): + raise Reject("operation_trace_missing") + for raw in result.get("signed_api", []): + env = parse_envelope(raw) + if not verify_envelope(raw) or env.run_id != job.run_id or env.signer != spec.hotkey: + raise Reject("operation_signed_API_outcome_changed") + if spec.operation == "live": + signed = result.get("signed_api", []) + for row in result.get("accepted_api", []): + if row["request"] not in signed: + raise Reject("operation_API_receipt_without_original_signature") + if any( + raw["body"].get("w") != job.w + for raw in signed + if raw["type"] in ("AcceptV2", "CommitV2", "DeltaManifestV2") + ): + raise Reject("operation_live_API_round_changed") + if spec.operation == "live" and not { + "AcceptV2", + "CommitV2", + "DeltaManifestV2", + } <= {row["request"]["type"] for row in result.get("accepted_api", [])}: + raise Reject("operation_live_API_acceptance_missing") + if spec.operation == "probe" and not {"WorkScreenV2", "WorkProof"} <= { + row["type"] for row in result.get("signed_api", []) + }: + raise Reject("operation_probe_signed_outcomes_missing") + if time.time() >= cutoff or cancel is not None and cancel.is_set(): + raise Reject("operation_expired_before_acceptance") + self.journal.append( + "network_operation_done", + role=role, + name=name, + result_sha256=fsha(destination / "operation-result.json"), + tar_sha256=custody["tar_sha256"], + ) + return artifacts if spec.operation in ("reference", "audit", "referee") else result + + def completion_custody( + self, + spec: Any, + directory: Path, + *, + tree: Path, + cancel: threading.Event | None = None, + ) -> Record: + """Authenticate completed trial custody internally; never store/payment acceptance.""" + from hypertrain.gpu_ops.network_qualification import sources + from hypertrain.miner.island_launch import confined, validate_artifacts + from hypertrain.protocol.envelope_v2 import Intake, load_json, parse_envelope + from hypertrain.protocol.jcs import canonicalize + from hypertrain.protocol.messages_v2 import CommitV2, JoinChallenge, WorkProof + + cutoff = min( + spec.cutoff, + spec.job.deadline, + self.lifecycle.deadline(), + self.lifecycle.cfg["cleanup_deadline_unix"] - 300, + ) + if ( + spec.operation not in ("reference", "probe") + or spec.binding_kind != "trial" + or spec.backend != "cuda" + or spec.cutoff != cutoff + or time.time() >= cutoff + or cancel is not None + and cancel.is_set() + or self.lifecycle.cleanup_started() + or not self.profile["runtime"]["qualified"] + or self.journal.last("supervisor_ready") is None + or self.journal.last("ssh_trusted", role=spec.role) is None + ): + raise Reject("completion_context_or_deadline") + host = self.owned_host(spec.role) + if host != { + "instance_id": spec.instance_id, + "machine_id": spec.machine_id, + "image_digest": spec.image_digest, + }: + raise Reject("completion_owned_host_changed") + if ( + spec.image_digest != spec.job.manifest.training.reference_spec.image_digest + or spec.job.manifest.training.reference_spec.layout.model_dump() + != self.profile["layout"] + ): + raise Reject("completion_layout_or_image_changed") + actual = sources(tree) + actual["scripts/network_gpu_operation.py"] = fsha(tree / "scripts/network_gpu_operation.py") + actual["experiments/gpu_network_v2/orchestrate.py"] = fsha(Path(__file__)) + if actual != spec.sources: + raise Reject("completion_actual_source_changed") + name = "op-" + sha256( + ( + spec.job.run_id + spec.operation + spec.hotkey + spec.binding + spec.job.digest() + ).encode() + ) + authority = self.journal.last( + "network_operation_authority", + role=spec.role, + binding=spec.binding, + operation=spec.operation, + hotkey=spec.hotkey, + w=spec.job.w, + ) + if authority is None or authority["owner"] != spec.owner: + raise Reject("completion_owner_authority_missing") + # Original Receipt intake also checks admitted owner and exact immutable spec. + receipt = authority["receipt"] + self.accept_operation( + spec, canonicalize(receipt), owner=spec.owner, beacon=receipt["body"]["received_round"] + ) + attested = self.journal.last("network_operation_source", role=spec.role) + map_hash = sha256(json.dumps(spec.sources, sort_keys=True).encode()) + if attested is None or (attested["run_id"], attested["sources_sha256"]) != ( + spec.job.run_id, + map_hash, + ): + raise Reject("completion_source_authority_changed") + started = self.journal.last("network_operation_started", role=spec.role, name=name) + done = self.journal.last("network_operation_done", role=spec.role, name=name) + intent = self.journal.last("network_execution_intent", role=spec.role, name=name) + if ( + started is None + or done is None + or intent is None + or intent["phase"] != "workload" + or intent["executions"] != 1 + or any( + started[k] != v + for k, v in { + "binding": spec.binding, + "job_sha256": spec.job.digest(), + "operation": spec.operation, + "hotkey": spec.hotkey, + "sources_sha256": map_hash, + "cutoff": cutoff, + }.items() + ) + or not authority["unix"] <= started["unix"] <= done["unix"] <= cutoff + ): + raise Reject("completion_original_done_missing") + directory = Path(directory) + original = directory / name + if original.is_symlink() or original.resolve().parent != directory.resolve(): + raise Reject("completion_custody_path_changed") + result_path = confined(original, "operation-result.json") + if fsha(result_path) != done["result_sha256"]: + raise Reject("completion_result_changed") + result = load_json(result_path.read_bytes()) + publication_name = result["publication"] + if not isinstance(publication_name, str): + raise Reject("completion_publication_path_changed") + publication = original / publication_name + if ( + publication.is_symlink() + or not publication.resolve().is_relative_to(original.resolve()) + or not publication.is_dir() + ): + raise Reject("completion_publication_path_changed") + artifacts = validate_artifacts(spec.job, publication) + custody_path = confined(original, "publication-custody.json") + custody = load_json(custody_path.read_bytes()) + body = custody["body"] + if not isinstance(body, dict): + raise Reject("completion_custody_binding_changed") + required_environment = spec.job.manifest.training.reference_spec.env.model_dump(mode="json") + if ( + custody["status"] != "CAPTURED_NOT_ACCEPTED" + or custody["sha256"] != sha256(canonicalize(body)) + or result.get("status") != "CAPTURED_NOT_ACCEPTED" + or result.get("publication_custody_sha256") != fsha(custody_path) + or any( + body[k] != v + for k, v in { + "operation_sha256": sha256(spec.model_dump_json().encode()), + "operation": spec.operation, + "binding": spec.binding, + "hotkey": spec.hotkey, + "owner": spec.owner, + "role": spec.role, + "instance_id": spec.instance_id, + "machine_id": spec.machine_id, + "image_digest": spec.image_digest, + "backend": "cuda", + "run_id": spec.job.run_id, + "job": spec.job.body(), + "job_sha256": spec.job.digest(), + "sources": spec.sources, + "manifest_sha256": spec.job.manifest.digest(), + "sample_ids_sha256": sha256(canonicalize(spec.job.sample_ids)), + "required_environment": required_environment, + "layout": self.profile["layout"], + "publication": result["publication"], + }.items() + ) + or any( + result[k] != body[k] + for k in ("operation", "binding", "hotkey", "role", "job_sha256") + ) + ): + raise Reject("completion_custody_binding_changed") + files = {} + for path in publication.rglob("*"): + if path.is_symlink() or not (path.is_file() or path.is_dir()): + raise Reject("completion_publication_special_file") + if path.is_file(): + files[str(path.relative_to(publication))] = { + "sha256": fsha(path), + "bytes": path.stat().st_size, + } + if files != body["files"] or sha256(canonicalize(files)) != body["publication_sha256"]: + raise Reject("completion_publication_changed") + summaries = [ + load_json(confined(publication, f"rank-{r}/summary.json").read_bytes()) + for r in range(spec.job.manifest.training.reference_spec.layout.n_gpus) + ] + if summaries != body["rank_summaries"] or any(s["backend"] != "cuda" for s in summaries): + raise Reject("completion_rank_backend_changed") + if spec.trace and any(f"rank-{r}/trace.json" not in files for r in range(len(summaries))): + raise Reject("completion_trace_missing") + trial = body["trial_authority"] + if not isinstance(trial, dict): + raise Reject("completion_original_challenge_changed") + trial_envelope, seed_beacon = trial["envelope"], trial["seed_beacon"] + if not isinstance(trial_envelope, dict) or type(seed_beacon) is not int: + raise Reject("completion_original_challenge_changed") + challenge = Intake( + spec.job.run_id, {"JoinChallenge": spec.job.manifest.training.coord_pubkey.__eq__} + ).accept(trial_envelope, seed_beacon) + from hypertrain.auditor.replay import unpack_state + from hypertrain.data.trial_assignment import trial_assignment_hash + from hypertrain.trainer.compress import state_hash + + if ( + not isinstance(challenge, JoinChallenge) + or challenge.admission_id != spec.binding + or challenge.manifest_hash != spec.job.run_id + or challenge.layout != spec.job.manifest.training.reference_spec.layout + or challenge.assignment_hash + != trial_assignment_hash(spec.job.manifest, spec.job.w, tuple(spec.job.sample_ids)) + or challenge.theta_hash + != state_hash( + unpack_state( + confined(publication, spec.job.object_paths["start_state"]).read_bytes() + )[0] + ) + or trial["sha256"] != sha256(canonicalize(trial["envelope"])) + or trial["challenge_hash"] != challenge.digest() + or trial["nonce"] != challenge.nonce + or trial["seed_beacon"] != challenge.seed_beacon + or trial["assignment_hash"] != challenge.assignment_hash + or parse_envelope(trial_envelope).exp_drand < challenge.deadline_beacon + or spec.challenge is None + or spec.context_files.get(spec.challenge) != trial["sha256"] + ): + raise Reject("completion_original_challenge_changed") + signed_cutoff = ( + spec.job.manifest.training.beacon.genesis_time + (challenge.deadline_beacon - 1) * 3 + ) + if ( + spec.job.deadline != signed_cutoff + or cutoff > signed_cutoff + or not authority["unix"] <= started["unix"] <= done["unix"] <= signed_cutoff + ): + raise Reject("completion_original_signed_deadline_changed") + if spec.operation == "probe": + commit_env, proof_env = body["miner_commit"], body["miner_proof"] + if not isinstance(commit_env, dict) or not isinstance(proof_env, dict): + raise Reject("completion_original_miner_signature_changed") + commit_envelope, proof_envelope = commit_env["envelope"], proof_env["envelope"] + if not isinstance(commit_envelope, dict) or not isinstance(proof_envelope, dict): + raise Reject("completion_original_miner_signature_changed") + intake = Intake( + spec.job.run_id, {"CommitV2": spec.hotkey.__eq__, "WorkProof": spec.hotkey.__eq__} + ) + commit = intake.accept(commit_envelope, challenge.seed_beacon) + proof = intake.accept(proof_envelope, challenge.seed_beacon) + from hypertrain.protocol.hashing import MerkleTree + from hypertrain.protocol.messages import LeafPreimage + + leaves = [ + LeafPreimage.model_validate(p) for p in json.loads(artifacts.leaves.read_bytes()) + ] + summary = summaries[0]["commitments"] + if not isinstance(summary, dict): + raise Reject("completion_original_miner_work_changed") + if ( + not isinstance(commit, CommitV2) + or not isinstance(proof, WorkProof) + or (commit.w, commit.hotkey) != (spec.job.w, spec.hotkey) + or commit.n_leaves != len(leaves) + or commit.metrics_root + != MerkleTree( + [bytes.fromhex(p.loss_f32) + bytes.fromhex(p.norm_f32) for p in leaves] + ).root.hex() + or commit.ef_in_hash + != state_hash( + unpack_state( + confined(publication, spec.job.object_paths["ef_in"]).read_bytes() + )[0] + ) + or (proof.admission_id, proof.challenge_hash) != (spec.binding, challenge.digest()) + or any( + getattr(commit, k) != summary[k] + for k in ("leaves_root", "final_theta_hash", "ef_out_hash", "delta_hash") + ) + or proof.leaves_root != commit.leaves_root + or proof.delta_hash != fsha(artifacts.delta) + or commit.delta_bytes != artifacts.delta.stat().st_size + or result.get("miner_commit") != commit_env + ): + raise Reject("completion_original_miner_work_changed") + commit.validate_assignment(spec.job.manifest, len(spec.job.sample_ids)) + for signed, envelope in ((commit_env, commit_envelope), (proof_env, proof_envelope)): + if ( + signed["sha256"] != sha256(canonicalize(envelope)) + or parse_envelope(envelope).exp_drand < challenge.deadline_beacon + ): + raise Reject("completion_original_miner_signature_changed") + for ref, path in zip( + proof.artifact_refs, + (artifacts.state, artifacts.ef, artifacts.delta, artifacts.leaves), + strict=True, + ): + if (ref.sha256, ref.size) != (fsha(path), path.stat().st_size): + raise Reject("completion_original_proof_object_changed") + elif body["miner_commit"] is not None: + raise Reject("completion_reference_claims_miner") + execution_path = confined(original, "execution-custody.json") + execution = load_json(execution_path.read_bytes()) + environment = body["environment"] + if not isinstance(environment, dict): + raise Reject("completion_execution_custody_changed") + expected = { + k: body[k] + for k in ( + "operation_sha256", + "job_sha256", + "binding", + "operation", + "instance_id", + "machine_id", + ) + } + expected.update( + publication_custody_sha256=fsha(custody_path), + operation_result_sha256=fsha(result_path), + environment_sha256=sha256(canonicalize(result["environment"])), + status="CAPTURED_NOT_ACCEPTED", + ) + if ( + execution != expected + or result["environment"] != body["environment"] + or environment.get("backend") != "cuda" + or environment.get("image_digest") != spec.image_digest + ): + raise Reject("completion_execution_custody_changed") + rescue = next( + ( + r + for r in self.journal.all("network_rescued") + if r["role"] == spec.role and r["tar_sha256"] == done["tar_sha256"] + ), + None, + ) + if rescue is None or fsha(rescue["tar"]) != done["tar_sha256"]: + raise Reject("completion_original_rescue_changed") + prefix = "out/" + name + "/" + with tarfile.open(rescue["tar"]) as archive: + rescued = {} + for member in archive.getmembers(): + if member.name.startswith(prefix) and member.isfile(): + rel = member.name.removeprefix(prefix) + if rel in rescued: + raise Reject("completion_duplicate_rescue_member") + stream = archive.extractfile(member) + assert stream is not None + rescued[rel] = sha256(stream.read()) + expected_rescue = { + publication_name + "/" + rel: meta["sha256"] for rel, meta in files.items() + } + expected_rescue.update( + {p.name: fsha(p) for p in (result_path, custody_path, execution_path)} + ) + if any(rescued.get(rel) != digest for rel, digest in expected_rescue.items()): + raise Reject("completion_original_rescue_bytes_changed") + # Existing canonical reader handles confinement, rescue and duplicate publication. + if spec.operation == "reference": + artifacts = self.launch_adapter(spec)( + spec.job, directory, backend="cuda", cancel=cancel, trace=spec.trace + ) + if ( + time.time() >= cutoff + or cancel is not None + and cancel.is_set() + or self.lifecycle.cleanup_started() + ): + raise Reject("completion_expired_before_descriptor") + descriptor = { + "status": "NOTSTOREACCEPTED", + "production_backend_acceptance": False, + "economic_acceptance": False, + "spec": spec.model_dump(mode="json"), + "owner_receipt": receipt, + "done": done, + "directory": str(artifacts.directory), + "publication": body, + "result_sha256": fsha(result_path), + "execution_custody_sha256": fsha(execution_path), + "rescue_sha256": done["tar_sha256"], + } + return {"body": descriptor, "sha256": sha256(canonicalize(descriptor))} + + def accept_operation(self, spec: Any, raw: bytes, *, owner: str, beacon: int) -> None: + """Trusted root Receipt intake; exact operation, no public route.""" + from hypertrain.protocol.envelope_v2 import Intake, parse_envelope + from hypertrain.protocol.jcs import canonicalize + from hypertrain.protocol.messages import Receipt + + receipt = Intake(spec.job.run_id, {"Receipt": lambda h: h == owner}).accept(raw, beacon) + env = parse_envelope(raw) + context = { + "role": spec.role, + "binding": spec.binding, + "operation": spec.operation, + "hotkey": spec.hotkey, + "spec_sha256": sha256(spec.model_dump_json().encode()), + } + if not isinstance(receipt, Receipt) or receipt.commit_hash != sha256(canonicalize(context)): + raise Reject("operation_owner_receipt_changed") + current = self.journal.last("network_admission_authority") + if current is None or current["owner"] != owner or spec.owner != owner: + raise Reject("operation_owner_not_admitted") + old = self.journal.last( + "network_operation_authority", + role=spec.role, + binding=spec.binding, + operation=spec.operation, + hotkey=spec.hotkey, + w=spec.job.w, + ) + if old is not None: + if old["spec_sha256"] != context["spec_sha256"]: + raise Reject("operation_accepted_context_immutable") + return + self.journal.append( + "network_operation_source", + role=spec.role, + run_id=spec.job.run_id, + sources_sha256=sha256(json.dumps(spec.sources, sort_keys=True).encode()), + ) + self.journal.append( + "network_operation_authority", + **context, + w=spec.job.w, + owner=env.signer, + receipt=env.model_dump(mode="json"), + ) + + def launch_adapter(self, spec: Any) -> Any: + """Bind trusted accepted trial/lease context once; preserve canonical launch signature.""" + + def launch( + job: Any, + directory: Path, + *, + backend: str, + cancel: threading.Event | None = None, + trace: bool = False, + ) -> Any: + if ( + job != spec.job + or backend != "cuda" + or trace != spec.trace + or spec.operation not in ("reference", "audit", "referee") + ): + raise Reject("operation_accepted_descriptor_changed") + import tempfile + + from hypertrain.miner.island_launch import validate_artifacts + + name = "op-" + sha256( + (job.run_id + spec.operation + spec.hotkey + spec.binding + job.digest()).encode() + ) + directory = Path(directory) + with flock(directory / "canonical-publication.lock"): + if ( + time.time() >= min(spec.cutoff, job.deadline) + or cancel is not None + and cancel.is_set() + or self.lifecycle.cleanup_started() + ): + raise Reject("operation_expired_before_canonical_publication") + custody = directory / name + canonical_done = self.journal.last( + "network_canonical_done", role=spec.role, name=name + ) + if ( + self.journal.last("network_canonical_intent", role=spec.role, name=name) + is not None + and canonical_done is None + ): + raise Reject("operation_canonical_failed_not_retryable") + if canonical_done is None: + self.journal.append("network_canonical_intent", role=spec.role, name=name) + done = self.journal.last("network_operation_done", role=spec.role, name=name) + if done is None: + artifacts = self.operation(spec, directory, cancel=cancel) + else: + authority = self.journal.last( + "network_operation_authority", + role=spec.role, + binding=spec.binding, + operation=spec.operation, + hotkey=spec.hotkey, + w=job.w, + ) + result = custody / "operation-result.json" + if ( + authority is None + or authority["spec_sha256"] != sha256(spec.model_dump_json().encode()) + or result.is_symlink() + or fsha(result) != done["result_sha256"] + ): + raise Reject("operation_canonical_duplicate_authority_changed") + receipt = self.lifecycle.receipt(spec.role) + if ( + receipt is None + or receipt["instance_id"] != spec.instance_id + or self.lifecycle.host[spec.role]["machine_id"] != spec.machine_id + ): + raise Reject("operation_accepted_host_changed") + publication_relative = Path(json.loads(result.read_bytes())["publication"]) + if publication_relative.is_absolute() or ".." in publication_relative.parts: + raise Reject("operation_publication_path") + artifacts = validate_artifacts(job, custody / publication_relative) + original = artifacts.directory + if ( + custody.is_symlink() + or custody.resolve().parent != directory.resolve() + or original.is_symlink() + or not original.resolve().is_relative_to(custody.resolve()) + ): + raise Reject("operation_canonical_custody_path") + files = {} + for path in original.rglob("*"): + if path.is_symlink() or not (path.is_file() or path.is_dir()): + raise Reject("operation_canonical_symlink_or_special") + if path.is_file(): + files[str(path.relative_to(original))] = fsha(path) + if any( + json.loads((original / f"rank-{rank}/summary.json").read_bytes())["backend"] + != "cuda" + or spec.trace + and f"rank-{rank}/trace.json" not in files + for rank in range(job.manifest.training.reference_spec.layout.n_gpus) + ): + raise Reject("operation_canonical_rank_backend_or_trace_changed") + # Exact original rescued bytes also bind an idempotent local return. + done = self.journal.last("network_operation_done", role=spec.role, name=name) + assert done is not None + rescue = next( + ( + row + for row in self.journal.all("network_rescued") + if row["role"] == spec.role and row["tar_sha256"] == done["tar_sha256"] + ), + None, + ) + if rescue is None or fsha(Path(rescue["tar"])) != done["tar_sha256"]: + raise Reject("operation_canonical_rescue_changed") + prefix = "out/" + name + "/" + str(original.relative_to(custody)) + "/" + with tarfile.open(rescue["tar"]) as archive: + rescued = {} + for member in archive.getmembers(): + if member.name.startswith(prefix) and member.isfile(): + stream = archive.extractfile(member) + assert stream is not None + rescued[member.name.removeprefix(prefix)] = sha256(stream.read()) + if rescued != files: + raise Reject("operation_canonical_rescue_changed") + canonical = directory / "published" + with tempfile.TemporaryDirectory(prefix="canonical-", dir=directory) as temporary: + staged = Path(temporary) / "published" + staged.mkdir() + for relative, digest in files.items(): + path = staged / relative + path.parent.mkdir(parents=True, exist_ok=True) + durable_write(path, (original / relative).read_bytes()) + if fsha(path) != digest: + raise Reject("operation_canonical_copy_changed") + validate_artifacts(job, staged) + for path in sorted(staged.rglob("*"), key=lambda p: len(p.parts), reverse=True): + if path.is_dir(): + fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + fd = os.open(staged, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + if canonical.exists() or canonical.is_symlink(): + if canonical.is_symlink() or not canonical.is_dir(): + raise Reject("operation_canonical_collision") + existing = {} + for path in canonical.rglob("*"): + if path.is_symlink() or not (path.is_file() or path.is_dir()): + raise Reject("operation_canonical_collision") + if path.is_file(): + existing[str(path.relative_to(canonical))] = fsha(path) + if existing != files: + raise Reject("operation_canonical_collision") + if ( + time.time() >= min(spec.cutoff, job.deadline) + or cancel is not None + and cancel.is_set() + or self.lifecycle.cleanup_started() + ): + raise Reject("operation_expired_before_canonical_publication") + if not canonical.exists(): + os.rename(staged, canonical) + fd = os.open(directory, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + validated = validate_artifacts(job, canonical) + if ( + time.time() >= min(spec.cutoff, job.deadline) + or cancel is not None + and cancel.is_set() + ): + raise Reject("operation_expired_before_canonical_result") + if canonical_done is None: + self.journal.append( + "network_canonical_done", + role=spec.role, + name=name, + directory=str(canonical), + ) + return validated + + return launch + + def run_qualification( + self, role: str, name: str, config_path: str, *, cutoff: float | None = None + ) -> Record: + """Launch frozen driver once; TWO executions charged before any SSH command.""" + if ( + not config_path.replace("-", "") + .replace("_", "") + .replace("/", "") + .replace(".", "") + .isalnum() + ): + raise Reject("invalid_qualification_config_path") + if Path(config_path).is_absolute() or ".." in Path(config_path).parts: + raise Reject("qualification_config_outside_custody") + if ( + self.lifecycle.cfg.get("cleanup_contract") != "network-v2" + or self.journal.last("supervisor_ready") is None + or self.lifecycle.receipt(role) is None + or self.journal.last("staged", role=role) is None + or self.lifecycle.cleanup_started() + ): + raise Reject("qualification_lifecycle_not_ready") + root = self.lifecycle.remote_root(role) + python = self.lifecycle.cfg.get("remote_python", self.profile["runtime"]["remote_python"]) + image = self.lifecycle.cfg["image"].rsplit("@", 1)[-1] + self.reserve(role, name, "qualification", 2) + ssh = ( + self.lifecycle.ssh(role) + if cutoff is None + else deadline_ssh(self.lifecycle, role, cutoff) + ) + intents = self.journal.all("create_intent") + first_billable = min(float(r["unix"]) for r in intents) if intents else None + if first_billable is None: + raise Reject("qualification_first_billable_receipt_missing") + ssh.timeout = int(min(30, first_billable + 900 - time.time() - 420)) + if ssh.timeout <= 0: + raise Reject("qualification_deadline_margin") + config_file = self.lifecycle.dir / f"qualification-config-{role}.json" + ssh.get(f"{root}/{config_path}", config_file) + spec = importlib.util.spec_from_file_location( + "network_driver_contract", + Path(__file__).resolve().parents[2] / "scripts/network_gpu_qualification.py", + ) + if spec is None or spec.loader is None: + raise Reject("qualification_driver_missing") + driver = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = driver + spec.loader.exec_module(driver) + cfg = driver.Qualification.model_validate(json.loads(config_file.read_bytes())) + receipt = self.lifecycle.receipt(role) + assert receipt is not None + if ( + cfg.role != role + or cfg.instance_id != receipt["instance_id"] + or first_billable is None + or cfg.admitted_unix != int(first_billable) + or ".." in cfg.output.parts + or not cfg.output.is_relative_to(Path(root) / "out") + or cfg.tree != Path(root) + or cfg.image_digest != image + or cfg.known_hosts_sha256 != fsha(ssh.known_hosts) + ): + raise Reject("qualification_driver_custody_or_admission_mismatch") + prefix = staged_command(self.profile, root, name, python, image, "cuda", qualification=True) + command = prefix.rsplit("experiments/gpu_network_v2/run.py", 1)[0] + ( + f"scripts/network_gpu_qualification.py run {shlex.quote(config_path)}" + ) + ssh.timeout = int( + min( + 180, + self.lifecycle.deadline() - time.time() - 420, + cfg.admitted_unix + 900 - time.time() - 420, + ) + ) + if ssh.timeout <= 0: + raise Reject("qualification_deadline_margin") + proc = ssh.run(command, "qualification-" + name, self.lifecycle.dir / "logs") + if proc.returncode: + self.journal.append( + "network_qualification_failed", role=role, name=name, rc=proc.returncode + ) + raise Reject("qualification_driver_failed") + return self.journal.append( + "network_qualification_captured", + role=role, + name=name, + executions=2, + workload_allowed=False, + ) + + def stage(self, role: str, tree: Path, jobs: dict[str, Path]) -> None: + """Existing receipt-bound SSH transports source and concrete authenticated job inputs.""" + if not self.lifecycle.receipt(role) or not self.journal.last("ssh_trusted", role=role): + raise Reject("receipt_bound_ssh_required") + files = {str(p.relative_to(tree)): p for p in (tree / "src/hypertrain").rglob("*.py")} + for rel in ( + "experiments/gpu_network_v2/run.py", + "experiments/gpu_network_v2/profile.json", + "experiments/gpu_network_v2/orchestrate.py", + "scripts/network_gpu_qualification.py", + ): + files[rel] = tree / rel + operation_script = tree / "scripts/network_gpu_operation.py" + if operation_script.is_file(): + files["scripts/network_gpu_operation.py"] = operation_script + for name, directory in jobs.items(): + if not name.replace("-", "").isalnum(): + raise Reject("invalid_staged_job") + for path in directory.rglob("*"): + if path.is_file(): + if path.is_symlink(): + raise Reject("staged_symlink_rejected") + files[f"out/{name}/{path.relative_to(directory)}"] = path + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w") as archive: + for rel, path in sorted(files.items()): + data = ( + json.dumps(self.profile, sort_keys=True).encode() + if rel == "experiments/gpu_network_v2/profile.json" + else path.read_bytes() + ) + info = tarfile.TarInfo(rel) + info.size, info.mode, info.mtime = len(data), 0o600, 0 + archive.addfile(info, io.BytesIO(data)) + tar = self.lifecycle.dir / f"network-stage-{role}.tar" + from hypertrain.protocol.hashing import sha256_hex + + digest = sha256_hex(buffer.getvalue()) + staged = self.journal.last("staged", role=role) + if staged: + if staged["tar_sha256"] != digest: + raise Reject("staged_bundle_changed_on_resume") + return + if not tar.exists(): + durable_write(tar, buffer.getvalue()) + elif fsha(tar) != digest: + raise Reject("stage_bundle_changed_on_resume") + ssh = self.lifecycle.ssh(role) + root = self.lifecycle.remote_root(role) + if ssh.run( + f"mkdir -p {shlex.quote(root)}/out", + "network-mkdir", + self.lifecycle.dir / "logs", + ).returncode: + raise Reject("stage_mkdir_failed") + ssh.put(tar, f"{root}/stage.tar") + if ssh.remote_sha256(f"{root}/stage.tar", self.lifecycle.dir / "logs") != fsha(tar): + raise Reject("stage_bundle_hash_mismatch") + if ssh.run( + f"cd {shlex.quote(root)} && tar -xf stage.tar --no-same-owner", + "network-extract", + self.lifecycle.dir / "logs", + ).returncode: + raise Reject("stage_extract_failed") + self.journal.append("staged", role=role, tar_sha256=fsha(tar), contract="network-v2") + + def run_staged( + self, + role: str, + name: str, + trace_equivalents: int = 1, + backend: str = "cuda", + *, + qualification: bool = False, + ) -> Record: + if not name.replace("-", "").isalnum() or not 1 <= trace_equivalents <= 12: + raise Reject("invalid_staged_job") + if backend not in ("cuda", "cpu"): + raise Reject("invalid_backend") + old = self.journal.last("network_job_done", role=role, name=name) + if old: + return old + admitted = self.lifecycle.admitted() + supervisor = self.journal.last("supervisor_ready") + receipt = self.lifecycle.receipt(role) + if ( + not admitted + or not supervisor + or not receipt + or not self.journal.last("staged", role=role) + or self.lifecycle.cleanup_started() + ): + raise Reject("lifecycle_receipts_not_ready") + used = sum(r["executions"] for r in self.journal.all("network_execution_intent", role=role)) + if used + trace_equivalents > 63: + raise Reject("execution_reservation_exhausted") + bound = ( + self.profile["microqualification"]["probe_and_transfer_seconds"] // 2 + if qualification + else self.profile["execution_and_transfer_bound_seconds"] + ) + if type(bound) is not int or bound > (180 if qualification else 40): + raise Reject("runtime_unqualified") + if qualification and ( + trace_equivalents != 1 + or len(self.journal.all("network_qualification_started", role=role)) >= 2 + ): + raise Reject("microqualification_round_limit") + if qualification and time.time() + bound + 420 > admitted["unix"] + 900: + raise Reject("microqualification_deadline_margin") + remaining = self.lifecycle.deadline() - time.time() + if remaining < trace_equivalents * bound + 600: + raise Reject("deadline_teardown_margin") + if self.journal.last("network_job_started", role=role, name=name): + raise Reject("incomplete_job_requires_rescue_not_automatic_retry") + if qualification: + raise Reject("qualification_requires_driver_two_round_reservation") + if self.lifecycle.cfg.get("cleanup_contract") == "network-v2": + self.reserve(role, name, "workload", trace_equivalents) + self.journal.append( + "network_job_started", + role=role, + name=name, + equivalents=trace_equivalents, + instance_id=receipt["instance_id"], + ) + root = self.lifecycle.remote_root(role) + image = self.lifecycle.cfg["image"].rsplit("@", 1)[-1] + python = self.lifecycle.cfg.get("remote_python", self.profile["runtime"]["remote_python"]) + command = staged_command( + self.profile, + root, + name, + python, + image, + backend, + qualification=qualification, + ) + if qualification: + self.journal.append("network_qualification_started", role=role, name=name) + bounds = artifact_budget(self.profile) + file_cap = max( + bounds["trace_bytes_per_rank"], + bounds["state_bytes"], + self.profile["artifacts"]["logs_bytes_per_execution"], + ) + command = f"ulimit -f {math.ceil(file_cap / 1024)}; " + command + ssh = self.lifecycle.ssh(role) + ssh.timeout = trace_equivalents * bound + started = time.monotonic() + result = ssh.run(command, "network-" + name, self.lifecycle.dir / "logs") + if result.returncode: + raise Reject("network_rank_or_deadline_failure") + return self.journal.append( + "network_job_done", + role=role, + name=name, + instance_id=receipt["instance_id"], + ssh_rc=result.returncode, + elapsed_seconds=time.monotonic() - started, + ) + + def execute_plan(self, evidence: dict[str, Any], jobs: list[Record]) -> list[Record]: + """Integrate D2 staged real launches on existing owned hosts; never create a host. + + L0 supplies concrete ordinary lease/admission/trace jobs and verifies final ledger/ + tape. This method executes exactly the reservation, rescues in finally, no fake state. + """ + runtime_admission(self.profile, evidence) + if ( + not 1 <= len(jobs) <= 122 + or len(self.lifecycle.roles) != 2 + or any( + sum(j["equivalents"] for j in jobs if j["role"] == role) != 61 + for role in self.lifecycle.roles + ) + or len({(j["role"], j["name"]) for j in jobs}) != len(jobs) + or any(not 1 <= j["equivalents"] <= 12 for j in jobs) + or any(j["backend"] != "cuda" for j in jobs) + ): + raise Reject("complete_d2_execution_reservation_required") + + def host(role: str) -> list[Record]: + return [ + self.run_staged(j["role"], j["name"], j["equivalents"], "cuda") + for j in jobs + if j["role"] == role + ] + + try: + with ThreadPoolExecutor(max_workers=2) as pool: + futures = [pool.submit(host, role) for role in self.lifecycle.roles] + result = [row for future in futures for row in future.result()] + finally: + # Every role attempted even when a sibling fails. Exceptions retain liability. + errors = [] + with ThreadPoolExecutor(max_workers=2) as pool: + pending = {role: pool.submit(self.rescue, role) for role in self.lifecycle.roles} + for role, future in pending.items(): + try: + future.result() + except Exception as e: + self.journal.append("network_rescue_failed", role=role, error=repr(e)) + errors.append(role) + if errors: + raise Reject("network_rescue_incomplete:" + ",".join(errors)) + return result + + def rescue(self, role: str, fresh: bool = False) -> Record: + """Complete path-preserving, byte/hash checked custody; failure never becomes PASS.""" + generation = sha256( + json.dumps( + [ + r + for r in self.journal.records() + if r.get("role") == role + and r["kind"] + in ( + "network_execution_intent", + "network_job_started", + "network_job_done", + "network_qualification_captured", + "network_qualification_failed", + ) + ], + sort_keys=True, + ).encode() + ) + old = self.journal.last("network_rescued", role=role) + if old and old.get("generation") == generation and not fresh: + archive = Path(old["tar"]) + if not archive.is_file() or fsha(archive) != old["tar_sha256"]: + raise Reject("network_rescue_changed_on_resume") + verify_export(archive, artifact_budget(self.profile)["rescue_tar_bytes_per_host"]) + return old + root = self.lifecycle.remote_root(role) + ssh = self.lifecycle.ssh(role) + seconds = self.profile["artifacts"]["rescue_transfer_seconds"] + if self.lifecycle.cfg.get("cleanup_deadline_unix") is not None: + seconds = min(seconds, self.lifecycle.cfg["cleanup_deadline_unix"] - time.time() - 120) + started = time.monotonic() + + def remaining() -> None: + left = seconds - (time.monotonic() - started) + if left <= 0: + raise Reject("network_rescue_deadline_exceeded") + ssh.timeout = left + + remaining() + python = self.lifecycle.cfg.get("remote_python", self.profile["runtime"]["remote_python"]) + cmd = ( + f"cd {shlex.quote(root)} && env PYTHONPATH=src {shlex.quote(python)} " + "experiments/gpu_network_v2/orchestrate.py export out rescue.tar " + "experiments/gpu_network_v2/profile.json" + ) + if ssh.run(cmd, "network-export", self.lifecycle.dir / "logs").returncode: + raise Reject("network_export_failed") + remaining() + expected = ssh.remote_sha256(f"{root}/rescue.tar", self.lifecycle.dir / "logs") + destination = self.lifecycle.dir / "rescue" / role + destination.mkdir(parents=True, exist_ok=True) + archive = destination / f"network-rescue-{generation}-{time.time_ns()}.tar" + remaining() + actual = ssh.get(f"{root}/rescue.tar", archive) + if actual != expected: + raise Reject("network_rescue_sha_mismatch") + manifest = verify_export( + archive, artifact_budget(self.profile)["rescue_tar_bytes_per_host"] + ) + remaining() + if generation != sha256( + json.dumps( + [ + r + for r in self.journal.records() + if r.get("role") == role + and r["kind"] + in ( + "network_execution_intent", + "network_job_started", + "network_job_done", + "network_qualification_captured", + "network_qualification_failed", + ) + ], + sort_keys=True, + ).encode() + ): + raise Reject("network_rescue_generation_changed") + return self.journal.append( + "network_rescued", + role=role, + generation=generation, + tar=str(archive), + tar_sha256=actual, + files=manifest, + elapsed_seconds=time.monotonic() - started, + ) + + +def export_artifacts(directory: Path, archive: Path, profile: dict[str, Any]) -> Record: + """Archive full paths and every byte; enforce actual shape-derived byte budget.""" + from hypertrain.protocol.hashing import sha256_hex + + files: dict[str, dict[str, Any]] = {} + for path in sorted(directory.rglob("*")): + if path.is_symlink(): + raise Reject("export_symlink_rejected") + if path.is_file(): + files[str(path.relative_to(directory))] = { + "bytes": path.stat().st_size, + "sha256": fsha(path), + } + bounds = artifact_budget(profile) + if sum(v["bytes"] for v in files.values()) > bounds["retained_bytes_per_host"]: + raise Reject("actual_artifact_budget_exceeded") + metadata = json.dumps(files, sort_keys=True).encode() + with tarfile.open(archive, "w") as out: + for name in files: + out.add(directory / name, arcname="out/" + name, recursive=False) + info = tarfile.TarInfo("manifest.json") + info.size, info.mode, info.mtime = len(metadata), 0o600, 0 + out.addfile(info, io.BytesIO(metadata)) + if archive.stat().st_size > bounds["rescue_tar_bytes_per_host"]: + raise Reject("actual_rescue_archive_budget_exceeded") + with archive.open("rb") as file: + os.fsync(file.fileno()) + return {"bytes": archive.stat().st_size, "manifest_sha256": sha256_hex(metadata)} + + +def verify_export(archive: Path, maximum_bytes: int) -> Record: + import hashlib + + if archive.stat().st_size > maximum_bytes: + raise Reject("rescue_archive_budget_exceeded") + with tarfile.open(archive) as source: + entries = source.getmembers() + if len({m.name for m in entries}) != len(entries): + raise Reject("rescue_duplicate_path") + metadata = source.extractfile("manifest.json") + if metadata is None or source.getmember("manifest.json").size > 1048576: + raise Reject("rescue_manifest_missing_or_oversized") + files = json.loads(metadata.read()) + if {m.name for m in entries} != {"manifest.json", *("out/" + n for n in files)}: + raise Reject("rescue_manifest_coverage_mismatch") + for name, want in files.items(): + path = Path(name) + if ( + path.is_absolute() + or ".." in path.parts + or not source.getmember("out/" + name).isfile() + ): + raise Reject("rescue_unsafe_path") + stream = source.extractfile("out/" + name) + assert stream is not None + digest, size = hashlib.sha256(), 0 + while block := stream.read(65536): + digest.update(block) + size += len(block) + if digest.hexdigest() != want["sha256"] or size != want["bytes"]: + raise Reject("rescue_content_hash_mismatch") + return dict(files) + + +def reviewed_launch(record_path: Path, owner: str, beacon: int, action: str) -> Record: + """Local preflight only. Existing signed Receipt binds every supplied artifact.""" + from hypertrain.miner.island_launch import confined + from hypertrain.protocol.envelope_v2 import ( + load_json, + parse_envelope, + verify_envelope, + verify_join, + ) + from hypertrain.protocol.jcs import canonicalize + from hypertrain.protocol.keys import decode_hotkey + from hypertrain.protocol.messages_v2 import IslandJobV1, JoinRequest + + raw = load_json(record_path.read_bytes()) + if not isinstance(raw["record"], dict) or not isinstance(raw["receipt"], dict): + raise Reject("reviewed_launch_shape") + record: Record = json.loads(canonicalize(raw["record"])) + signed = raw["receipt"] + env = parse_envelope(signed) + from hypertrain.protocol.messages import Receipt + + receipt = Receipt.model_validate(env.body) + if ( + not verify_envelope(signed) + or env.type != "Receipt" + or env.signer != owner + or env.exp_drand < beacon + or receipt.received_round > beacon + or receipt.commit_hash != sha256(canonicalize(record)) + or record["action"] != action + or record["owner"] != owner + or type(beacon) is not int + or beacon < 1 + ): + raise Reject("reviewed_owner_receipt_invalid") + if record["parent_pid"] == os.getpid(): + raise Reject("persistent_controller_parent_required") + files = record["files"] + if not isinstance(files, dict) or not files: + raise Reject("reviewed_files_missing") + artifacts = {} + for name, item in files.items(): + if not isinstance(item, dict) or set(item) != {"path", "sha256"}: + raise Reject("reviewed_file_shape") + path = Path(item["path"]) + if not path.is_absolute() or path.is_symlink() or not path.is_file(): + raise Reject("reviewed_file_missing:" + name) + if fsha(path) != item["sha256"]: + raise Reject("reviewed_file_changed:" + name) + artifacts[name] = path + + def data(name: str) -> Any: + return json.loads(artifacts[name].read_bytes()) + + cfg, profile = data("config"), data("profile") + for required in ( + "src/hypertrain/gpu_ops/launcher.py", + "src/hypertrain/gpu_ops/supervisor.py", + ): + actual = Path(__file__).resolve().parents[2] / required + if data("lifecycle_hashes").get(required) != fsha(actual): + raise Reject("reviewed_lifecycle_source_changed") + roles = record["roles"] + if ( + set(roles) != {"h0", "h1"} + or {h["role"] for h in cfg["hosts"]} != set(roles) + or len(cfg["hosts"]) != 2 + or cfg["cleanup_contract"] != "network-v2" + or cfg["disk_gb"] != 80 + or cfg["hard_deadline_seconds"] != 3600 + or cfg["hard_grace_seconds"] != 0 + or type(cfg["boot_timeout_seconds"]) is not int + or not 0 < cfg["boot_timeout_seconds"] <= 300 + or cfg["keyscan_attempts"] != 1 + or cfg["cleanup_deadline_unix"] != record["cutoff_unix"] + or type(cfg["create_margin_seconds"]) is not int + or not 420 <= cfg["create_margin_seconds"] <= 900 + or cfg["remote_python"] != profile["runtime"]["remote_python"] + or cfg["network_profile_file"] != str(artifacts["profile"]) + or not cfg["image"].endswith("@" + profile["runtime"]["image_digest"]) + or not profile["runtime"]["driver_allowlist"] + or type(record["parent_pid"]) is not int + or record["parent_pid"] <= 0 + or type(record["cutoff_unix"]) is not int + or not time.time() < record["cutoff_unix"] <= time.time() + 3600 + or record["quota"] != {"total": 126, "per_role": 63, "qualification": 4, "remaining": 122} + ): + raise Reject("reviewed_layout_clock_or_config") + artifact_budget(profile) + if ( + profile["runtime"]["backend"] != "cuda" + or profile["runtime"]["remote_python"] != "/opt/hypertrain/venv/bin/python" + or type(record["staging_bytes"]) is not int + or not 0 < record["staging_bytes"] <= 1_000_000_000 + or record["public_dataset_reviewed"] is not True + or record["full_genesis_reviewed"] is not True + ): + raise Reject("reviewed_source_seed_authority_missing") + registry = data("registry") + if "sha256:" + fsha(artifacts["registry"]) != profile["runtime"]["image_digest"]: + raise Reject("reviewed_registry_digest") + layers = registry.get("layers") + if ( + registry.get("schemaVersion") != 2 + or not isinstance(layers, list) + or not layers + or sum(d["size"] for d in layers) + registry["config"]["size"] > 20_000_000_000 + ): + raise Reject("reviewed_complete_image_layers_required") + tree = Path(record["tree"]) + if not tree.is_absolute(): + raise Reject("reviewed_source_tree") + spec = importlib.util.spec_from_file_location( + "network_cli_driver", tree / "scripts/network_gpu_qualification.py" + ) + if spec is None or spec.loader is None: + raise Reject("reviewed_driver_missing") + # Check before executing the supplied engine; its bytes must also be in the signed source map. + sources = data("sources") + source_profile = artifacts["source_profile"] if action == "continue" else artifacts["profile"] + if action == "continue" and any( + data("source_profile")[k] != profile[k] for k in ("model", "inner", "outer", "layout") + ): + raise Reject("reviewed_source_profile_training_changed") + for rel, digest in sources.items(): + actual = ( + source_profile + if rel == "experiments/gpu_network_v2/profile.json" + else confined(tree, rel) + ) + if fsha(actual) != digest: + raise Reject("reviewed_source_changed:" + rel) + if sources.get("scripts/network_gpu_qualification.py") != fsha(Path(spec.origin or "")): + raise Reject("reviewed_driver_not_bound") + driver = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = driver + spec.loader.exec_module(driver) + expected_sources, expected_bundle = effective_bundle(driver, tree, source_profile) + if sources != expected_sources or sha256(expected_bundle) != fsha(artifacts["bundle"]): + raise Reject("reviewed_source_bundle_mismatch") + if Path(__file__).resolve() != tree / "experiments/gpu_network_v2/orchestrate.py": + raise Reject("actual_local_overlay_import_mismatch") + import hypertrain.gpu_ops.launcher as installed_launcher + + if Path(installed_launcher.__file__).resolve() != tree / "src/hypertrain/gpu_ops/launcher.py": + raise Reject("actual_local_overlay_import_mismatch") + for inputs in roles.values(): + if not inputs["name"].replace("-", "").isalnum(): + raise Reject("reviewed_job_name") + job = IslandJobV1.model_validate_json(artifacts[inputs["job"]].read_bytes()) + decode_hotkey(inputs["hotkey"]) + join = JoinRequest.model_validate_json(artifacts[inputs["join"]].read_bytes()) + seed_context = data(inputs["seed_context"]) + admission_id = sha256(f"join|{env.run_id}|{join.coldkey}|{join.request_id}".encode()) + if ( + not verify_join(join, env.run_id, beacon) + or join.hotkey != inputs["hotkey"] + or join.policy_hash != job.manifest.network.admission_policy_hash + or seed_context + != { + "run_id": env.run_id, + "owner": owner, + "hotkey": join.hotkey, + "coldkey": join.coldkey, + "request_id": join.request_id, + "policy_hash": join.policy_hash, + "job_sha256": fsha(artifacts[inputs["job"]]), + "admission_id": admission_id, + "state": "PROBATION", + "join_sha256": fsha(artifacts[inputs["join"]]), + } + ): + raise Reject("reviewed_seed_identity_authority") + if ( + job.run_id != env.run_id + or job.global_step0 != 0 + or job.deadline < record["cutoff_unix"] + ): + raise Reject("reviewed_seed_run_or_deadline") + training = job.manifest.training.model_dump(mode="json") + if any( + any( + training[k].get(key) != value + for key, value in profile[k].items() + if key != "lr_schedule" + ) + for k in ("model", "inner", "outer") + ) or any( + training["inner"]["lr_schedule"].get(key) != value + for key, value in profile["inner"]["lr_schedule"].items() + ): + raise Reject("reviewed_seed_profile") + ref = training["reference_spec"] + if ( + ref["layout"] != profile["layout"] + or ref["image_digest"] != profile["runtime"]["image_digest"] + or ref["driver_allowlist"] != profile["runtime"]["driver_allowlist"] + or job.manifest.training.dataset.n_samples < 4096 + ): + raise Reject("reviewed_seed_reference") + directory = artifacts[inputs["job"]].parent + for key, rel in job.object_paths.items(): + path = confined(directory, rel) + if ( + inputs["objects"].get(key) not in artifacts + or path != artifacts[inputs["objects"][key]] + ): + raise Reject("reviewed_seed_object_custody") + for key in ("start_state", "ef_in", "v0"): + if fsha(artifacts[inputs["objects"][key]]) != getattr(job, key + "_sha256"): + raise Reject("reviewed_seed_hash") + # Reuse complete deterministic carry genesis; never accept theta-only state. + from hypertrain.auditor.replay import AnchorCache, pack_state, unpack_state + + theta, state = unpack_state(artifacts[inputs["objects"]["start_state"]].read_bytes()) + anchor = AnchorCache().genesis(job.manifest, inputs["hotkey"], theta) + if ( + state is None + or pack_state(theta, state) != pack_state(anchor.theta, anchor.state) + or artifacts[inputs["objects"]["ef_in"]].read_bytes() != pack_state(anchor.ef) + or artifacts[inputs["objects"]["v0"]].read_bytes() != pack_state({}) + ): + raise Reject("reviewed_seed_full_genesis") + from hypertrain.protocol.hashing import MerkleTree + + samples = artifacts[inputs["objects"]["samples"]].read_bytes() + proofs = json.loads(artifacts[inputs["objects"]["sample_proofs"]].read_bytes()) + dataset = job.manifest.training.dataset + sample_bytes = { + "u16[seq_len+1] token ids": 2, + "u32[seq_len+1] token ids": 4, + }.get(dataset.sample_format) + width = (sample_bytes or 0) * (profile["model"]["seq_len"] + 1) + if ( + sample_bytes is None + or len(samples) != width * len(job.sample_ids) + or len(proofs) != len(job.sample_ids) + or any( + not MerkleTree.verify( + samples[index * width : (index + 1) * width], + sample, + [bytes.fromhex(p) for p in proof], + bytes.fromhex(dataset.merkle_root), + dataset.n_samples, + ) + for index, (sample, proof) in enumerate(zip(job.sample_ids, proofs, strict=True)) + ) + ): + raise Reject("reviewed_public_sample_membership") + snapshot, offers = data("financial"), data("offers") + if any(type(snapshot[k]) is not int for k in ("verified_unix", "account_id")) or any( + type(o[k]) is not int + for o in offers + for k in ("verified_unix", "id", "machine_id", "num_gpus") + ): + raise Reject("reviewed_financial_strict_integers") + observed = time.time() + + def fresh(row: Record) -> bool: + return ( + all( + type(row[k]) is int and observed - 30 <= row[k] <= observed + for k in ("verified_unix", "requested_unix", "received_unix") + ) + and row["requested_unix"] <= row["received_unix"] <= row["verified_unix"] + ) + + if ( + not fresh(snapshot) + or any(not fresh(o) for o in offers) + or not data("observations") + or not all(fresh(o) for o in data("observations")) + or {o["machine_id"] for o in offers} != {h["machine_id"] for h in cfg["hosts"]} + ): + raise Reject("reviewed_financial_freshness") + authorization = data("authorization") + if ( + authorization.get("experiment") != "hypertrain-network-v2" + or authorization.get("currency") != "USD" + or authorization.get("ceiling_usd") != "15" + or authorization.get("max_instances") != 2 + or authorization.get("max_seconds_per_instance") != 3600 + or not authorization.get("source_message") + or not authorization.get("date") + or snapshot.get("authenticated") is not True + or snapshot.get("uncapped_charges") is not False + or snapshot.get("volumes") != [] + or len(offers) != 2 + or len({o["machine_id"] for o in offers}) != 2 + or any(o["num_gpus"] != 2 or o["gpu_name"] != "RTX 5090" for o in offers) + ): + raise Reject("reviewed_authorization_or_two_host_financial") + if action == "continue": + runtime_admission(profile, data("evidence")) + cost = budget.evaluate( + snapshot0_credit=Decimal(snapshot["available_credit_usd"]) + + Decimal(snapshot["prior_attempts_usd"]), + current_credit=Decimal(snapshot["available_credit_usd"]) + - Decimal(snapshot["active_liabilities_usd"]), + offers=offers, + hard_deadline_seconds=3600, + disk_gb=80, + egress_gb=30, + phase_cap=Decimal("15"), + reserve=Decimal("5"), + ) + plan = {**cost, "long_workload_allowed": True} + if not cost["admit"] or Decimal(cost["hardware_worst_case_usd"]) > 5: + raise Reject("reviewed_continuation_budget") + if "graph" not in artifacts or "graph_driver" not in artifacts or "jobs" in artifacts: + raise Reject("signed_continuation_graph_required") + if artifacts["graph_driver"].resolve() != tree / "scripts/network_service_proof.py": + raise Reject("signed_continuation_driver_source") + graph = json.loads(artifacts["graph"].read_bytes()) + if graph.get("owner") != owner or graph.get("run_id") != env.run_id: + raise Reject("reviewed_graph_owner_run") + sys.path.insert(0, str(tree / "scripts")) + graph_spec = importlib.util.spec_from_file_location( + "signed_continuation_driver", artifacts["graph_driver"] + ) + assert graph_spec is not None and graph_spec.loader is not None + continuation_driver = importlib.util.module_from_spec(graph_spec) + sys.modules[graph_spec.name] = continuation_driver + graph_spec.loader.exec_module(continuation_driver) + continuation_driver.ContinuationGraph.model_validate_json(artifacts["graph"].read_bytes()) + driver.compare([artifacts[inputs["qualification_result"]] for inputs in roles.values()]) + else: + planning_snapshot = dict(snapshot) + if action == "qualification": + if len(snapshot["instances"]) != 2: + raise Reject("qualification_owned_inventory_required") + planning_snapshot["instances"] = [] + plan = admission_plan( + profile, + data("evidence"), + data("authorization"), + planning_snapshot, + offers, + qualification=True, + ) + if plan != data("plan"): + raise Reject("reviewed_plan_differs") + return { + "record": record, + "files": artifacts, + "config": cfg, + "profile": profile, + "driver": driver, + "plan": plan, + "offers": offers, + "financial": snapshot, + "run_id": env.run_id, + "continuation_driver": continuation_driver if action == "continue" else None, + } + + +def launch_action( + action: str, + checked: Record, + run_dir: Path, + *, + live: bool = False, + cancel: threading.Event | None = None, +) -> Record: + """Use the existing lifecycle. A short CLI never masquerades as the controller parent.""" + import select + + record, cfg, files = checked["record"], checked["config"], checked["files"] + cancel = cancel if cancel is not None else threading.Event() + if cancel.is_set(): + raise Reject("continuation_cancelled_before_action") + if not loopback(cfg["base_url"]) and not live: + raise Reject("live_create_requires_explicit_flag_and_reviewed_authorization") + if not record["cutoff_unix"] > time.time(): + raise Reject("reviewed_cutoff_expired") + margin = ( + 61 * checked["profile"]["execution_and_transfer_bound_seconds"] + 600 + if action == "continue" + else 900 + if action == "admit" + else 600 + ) + if time.time() + margin > record["cutoff_unix"]: + raise Reject("reviewed_cutoff_work_margin") + if not 0 <= time.time() - checked["financial"]["verified_unix"] <= 30: + raise Reject("financial_expired_before_action") + parent_fd = os.pidfd_open(record["parent_pid"]) + try: + if select.select([parent_fd], [], [], 0)[0]: + raise Reject("reviewed_parent_not_alive") + finally: + os.close(parent_fd) + run_dir.mkdir(mode=0o700, parents=True, exist_ok=True) + j = Journal(run_dir) + provider = Provider(cfg["base_url"], read_api_key(cfg["key_file"]), run_dir, j, live) + orch = Orchestrator(cfg, j, provider, run_dir) + runtime = NetworkRuntime(orch, checked["profile"]) + if action == "admit": + if j.last("transaction_open") or j.last("create_intent"): + raise Reject("admission_not_retryable") + durable_write(run_dir / "config.json", files["config"].read_bytes()) + j.append("transaction_open", pid=record["parent_pid"], live=live) + j.append( + "network_admission_authority", + digest=sha256(json.dumps(record, sort_keys=True).encode()), + owner=record["owner"], + config_sha256=fsha(files["config"]), + sources_sha256=fsha(files["sources"]), + ) + quotes = { + role: next( + o for o in checked["offers"] if o["machine_id"] == orch.host[role]["machine_id"] + ) + for role in orch.roles + } + j.append( + "admitted", + plan=checked["plan"], + quotes=quotes, + deadline_unix=record["cutoff_unix"], + baseline_ids=[], + account_id=checked["financial"]["account_id"], + ) + old_pythonpath = os.environ.get("PYTHONPATH") + os.environ["PYTHONPATH"] = str(Path(record["tree"]) / "src") + try: + ready = orch.ensure_supervisor() + finally: + if old_pythonpath is None: + del os.environ["PYTHONPATH"] + else: + os.environ["PYTHONPATH"] = old_pythonpath + if ( + ready["supervisor_pid"] in (os.getpid(), record["parent_pid"]) + or type(ready["supervisor_pid"]) is not int + ): + raise Reject("independent_supervisor_required") + fd = os.pidfd_open(ready["supervisor_pid"]) + try: + if select.select([fd], [], [], 0)[0]: + raise Reject("supervisor_not_alive") + finally: + os.close(fd) + try: + for role in orch.roles: + if time.time() > checked["financial"]["verified_unix"] + 30: + raise Reject("financial_expired_before_create") + orch.create(role) + return { + "status": "ADMITTED_NOT_QUALIFIED", + "receipts": [orch.receipt(r) for r in orch.roles], + } + except Exception: + orch.cleanup(force=True) + raise + opened = j.all("transaction_open") + authority = j.last("network_admission_authority") + if ( + authority is None + or authority["owner"] != record["owner"] + or authority["sources_sha256"] != fsha(files["sources"]) + ): + raise Reject("existing_admission_authority_mismatch") + if ( + not opened + or opened[0]["pid"] != record["parent_pid"] + or orch.cleanup_started() + or len(j.all("create_intent")) != 2 + or not all(orch.receipt(r) for r in orch.roles) + or not j.last("supervisor_ready") + or time.time() >= orch.deadline() + ): + raise Reject("existing_lifecycle_required") + original = json.loads((run_dir / "config.json").read_bytes()) + if record["cutoff_unix"] > original["cleanup_deadline_unix"]: + raise Reject("existing_cutoff_cannot_extend") + if ( + action == "qualification" + and cfg["network_profile_file"] != original["network_profile_file"] + ): + raise Reject("existing_profile_cannot_change_before_qualification") + if any( + cfg[key] != original[key] + for key in ( + "hosts", + "image", + "base_url", + "key_file", + "remote_root", + "remote_python", + "cleanup_contract", + "disk_gb", + "hard_deadline_seconds", + "hard_grace_seconds", + ) + ): + raise Reject("existing_lifecycle_config_changed") + receipts = [orch.receipt(r) for r in orch.roles] + if {row["id"] for row in checked["financial"]["instances"]} != { + row["instance_id"] for row in receipts if row is not None + }: + raise Reject("continuation_owned_inventory_mismatch") + try: + if action == "qualification": + with ThreadPoolExecutor(max_workers=2) as pool: + results = list( + pool.map( + lambda role: qualify_role(checked, orch, runtime, role), + orch.roles, + ) + ) + return {"status": "CAPTURED_PENDING_ROOT_REVIEW", "results": results} + evidence = json.loads(files["evidence"].read_bytes()) + original_profile = json.loads(Path(original["network_profile_file"]).read_bytes()) + if any( + original_profile[k] != checked["profile"][k] + for k in ("model", "inner", "outer", "layout") + ): + raise Reject("continuation_training_profile_changed") + for role in orch.roles: + expected = j.last("cli_import_attested", role=role) + if ( + expected is None + or expected["sources_sha256"] != fsha(files["sources"]) + or expected["bundle_sha256"] != fsha(files["bundle"]) + ): + raise Reject("existing_staged_import_authority_missing") + existing_ready = j.last("supervisor_ready") + assert existing_ready is not None + fd = os.pidfd_open(existing_ready["supervisor_pid"]) + try: + if select.select([fd], [], [], 0)[0]: + raise Reject("supervisor_not_alive") + finally: + os.close(fd) + if len(j.all("network_execution_intent")) != 2 or any( + r["phase"] != "qualification" for r in j.all("network_execution_intent") + ): + raise Reject("unfinished_continuation_not_retryable") + for role, inputs in record["roles"].items(): + result = json.loads(files[inputs["qualification_receipt"]].read_bytes()) + captured = j.last("network_qualification_captured", role=role, name=inputs["name"]) + actual = checked["driver"].Result.model_validate_json( + files[inputs["qualification_result"]].read_bytes() + ) + receipt = orch.receipt(role) + assert receipt is not None + if ( + captured is None + or actual.role != role + or actual.instance_id != receipt["instance_id"] + or actual.machine_id != orch.host[role]["machine_id"] + or actual.config_sha256 != fsha(run_dir / f"cli-qualification-{role}.json") + or actual.sources != json.loads(files["sources"].read_bytes()) + or actual.environment.image_digest != cfg["image"].rsplit("@", 1)[-1] + or result["result_sha256"] != fsha(files[inputs["qualification_result"]]) + ): + raise Reject("reviewed_qualification_result_binding") + custody = runtime.rescue(role) + if result["artifact_manifest_sha256"] != sha256( + json.dumps(custody["files"], sort_keys=True).encode() + ): + raise Reject("reviewed_qualification_custody_mismatch") + runtime.record_qualification(role, inputs["name"], result) + source = Path(record["tree"]) / "scripts/network_service_proof.py" + if "graph_driver" not in files or fsha(source) != fsha(files["graph_driver"]): + raise Reject("signed_continuation_driver_changed") + graph_driver = checked["continuation_driver"] + if Path(graph_driver.__file__).resolve() != source: + raise Reject("signed_continuation_driver_import") + from hypertrain.protocol.messages_v2 import IslandJobV1 + + manifest = IslandJobV1.model_validate_json( + files[record["roles"]["h0"]["job"]].read_bytes() + ).manifest + graph_driver.continuation_graph( + files["graph"], runtime, record["owner"], checked["run_id"], manifest + ) + graph_doc = json.loads(files["graph"].read_bytes()) + if graph_doc.get("operation_sources", {}).get( + "experiments/gpu_network_v2/profile.json" + ) != fsha(files["profile"]): + raise Reject("signed_continuation_effective_profile") + if cancel.is_set() or j.last("network_graph_started"): + raise Reject("signed_continuation_cancelled_or_not_retryable") + runtime.promote(evidence, checked["plan"]) + j.append( + "network_graph_started", + graph_sha256=fsha(files["graph"]), + run_id=checked["run_id"], + ) + results = graph_driver.decoder_continue(files["graph"], runtime, checked, cancel) + if ( + results.get("status") != "SIGNED_GRAPH_SETTLED_PENDING_RELEASE_REVIEW" + or results.get("run_id") != checked["run_id"] + or results.get("graph_sha256") != fsha(files["graph"]) + or results.get("fault_proof_complete") is not False + or len(results.get("trial_finalizations", [])) != 48 + or len(results.get("lineage", [])) != 2 + or results.get("normal_total") != 116 + or results.get("per_host") != 58 + ): + raise Reject("signed_continuation_not_settled") + cleanup = orch.cleanup(force=True) + return { + "status": "SIGNED_GRAPH_SETTLED_PENDING_RELEASE_REVIEW", + "results": results, + "cleanup": cleanup, + } + except Exception: + cancel.set() + orch.cleanup(force=True) + raise + + +def qualify_role(checked: Record, orch: Orchestrator, runtime: NetworkRuntime, role: str) -> Record: + """Stage exact source/seed bytes, inspect actual import origin, then invoke frozen driver.""" + inputs, files = checked["record"]["roles"][role], checked["files"] + orch.boot(role) + orch.attach(role) + orch.trust(role) + root = orch.remote_root(role) + first = min(r["unix"] for r in orch.j.all("create_intent")) + stage_cutoff = min(first + 300, orch.deadline() - 420, checked["record"]["cutoff_unix"] - 420) + ssh = deadline_ssh(orch, role, stage_cutoff) + + def remaining() -> None: + ssh.timeout = 60 + _ = ssh.timeout + + remaining() + if ssh.run(f"mkdir -p {shlex.quote(root)}/out/seed", "cli-mkdir", orch.dir / "logs").returncode: + raise Reject("source_staging_failed") + remaining() + ssh.put(files["bundle"], root + "/source.tar") + if ssh.remote_sha256(root + "/source.tar", orch.dir / "logs") != fsha(files["bundle"]): + raise Reject("source_transport_hash") + remaining() + if ssh.run( + f"cd {shlex.quote(root)} && tar -xf source.tar --no-same-owner", + "cli-extract", + orch.dir / "logs", + ).returncode: + raise Reject("source_extract_failed") + job = checked["driver"].IslandJobV1.model_validate_json(files[inputs["job"]].read_bytes()) + uploads = { + "job.json": files[inputs["job"]], + **{job.object_paths[k]: files[v] for k, v in inputs["objects"].items()}, + } + for rel, path in uploads.items(): + remaining() + ssh.put(path, root + "/out/seed/" + rel) + if ssh.remote_sha256(root + "/out/seed/" + rel, orch.dir / "logs") != fsha(path): + raise Reject("seed_transport_hash") + for name in ("profile", "registry"): + remaining() + ssh.put(files[name], root + "/" + name + ".json") + if ssh.remote_sha256(root + "/" + name + ".json", orch.dir / "logs") != fsha(files[name]): + raise Reject("qualification_metadata_transport_hash") + # Fixed inspector checks source map and module paths inside real remote Python. + sources = json.loads(files["sources"].read_bytes()) + probe = ( + "import hashlib,json,pathlib,sys,importlib.util; " + f"root=pathlib.Path({root!r}).resolve(); expected={sources!r}; " + "assert all(hashlib.sha256((root/p).read_bytes()).hexdigest()==h " + "for p,h in expected.items()); " + "import hypertrain.gpu_ops.launcher,hypertrain.trainer,torch; " + "assert pathlib.Path(hypertrain.gpu_ops.launcher.__file__).resolve()==" + "root/'src/hypertrain/gpu_ops/launcher.py'; " + "assert torch.cuda.is_available() and torch.cuda.device_count()==2; " + "assert torch.__version__=='2.14.0+cu130' and torch.version.cuda=='13.0'; " + "assert pathlib.Path(torch.__file__).resolve().is_relative_to('/venv/main'); " + "assert all(torch.cuda.get_device_properties(i).multi_processor_count==170 " + "for i in range(2)); " + "print('IMPORT_ATTESTED')" + ) + image = checked["profile"]["runtime"]["image_digest"] + command = staged_command( + checked["profile"], + root, + inputs["name"], + orch.cfg["remote_python"], + image, + "cuda", + qualification=True, + ) + command = ( + command.split(" experiments/gpu_network_v2/run.py", 1)[0] + " -c " + shlex.quote(probe) + ) + remaining() + inspected = ssh.run(command, "cli-import", orch.dir / "logs") + if inspected.returncode or inspected.stdout.strip() != "IMPORT_ATTESTED": + raise Reject("actual_staged_import_attestation_failed") + orch.j.append( + "cli_import_attested", + role=role, + sources_sha256=fsha(files["sources"]), + bundle_sha256=fsha(files["bundle"]), + image_digest=image, + ) + receipt = orch.receipt(role) + ready = orch.j.last("supervisor_ready") + assert receipt is not None and ready is not None + known_hash = fsha(ssh.known_hosts) + contract_hash = fsha(files["config"]) + custody = { + "instance_id": receipt["instance_id"], + "machine_id": orch.host[role]["machine_id"], + "role": role, + "admitted_unix": int(first), + "image_digest": image, + "known_hosts_sha256": known_hash, + "lifecycle_contract_sha256": contract_hash, + "supervisor_pid": ready["supervisor_pid"], + "deadline_unix": min(orch.deadline(), checked["record"]["cutoff_unix"]), + "cuda_observed": True, + } + local_receipt = orch.dir / f"cli-receipt-{role}.json" + durable_write(local_receipt, json.dumps(custody, sort_keys=True).encode()) + cfg = checked["driver"].Qualification( + tree=root, + profile=root + "/profile.json", + profile_sha256=fsha(files["profile"]), + sources=sources, + registry_manifest=root + "/registry.json", + image_digest=image, + driver_allowlist=checked["profile"]["runtime"]["driver_allowlist"], + seed_job=root + "/out/seed/job.json", + hotkey=inputs["hotkey"], + output=root + "/out/" + inputs["name"], + admitted_unix=int(first), + instance_id=receipt["instance_id"], + machine_id=custody["machine_id"], + role=role, + lifecycle_contract_sha256=contract_hash, + known_hosts_sha256=known_hash, + lifecycle_receipts=root + "/receipts.json", + lifecycle_receipts_sha256=fsha(local_receipt), + ) + local_config = orch.dir / f"cli-qualification-{role}.json" + durable_write(local_config, cfg.model_dump_json().encode()) + remaining() + ssh.put(local_receipt, root + "/receipts.json") + if ssh.remote_sha256(root + "/receipts.json", orch.dir / "logs") != fsha(local_receipt): + raise Reject("qualification_receipt_transport_hash") + remaining() + ssh.put(local_config, root + "/qualification.json") + if ssh.remote_sha256(root + "/qualification.json", orch.dir / "logs") != fsha(local_config): + raise Reject("qualification_config_transport_hash") + orch.j.append("staged", role=role, tar_sha256=fsha(files["bundle"]), contract="network-v2") + result = runtime.run_qualification( + role, + inputs["name"], + "qualification.json", + cutoff=min(first + 480, orch.deadline() - 420, checked["record"]["cutoff_unix"] - 420), + ) + runtime.rescue(role, fresh=True) + return result + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Reviewed D2 lifecycle; no implicit live permission" + ) + parser.add_argument("action", choices=("plan", "admit", "qualification", "continue", "export")) + parser.add_argument("input", type=Path) + parser.add_argument("extra", nargs="*") + parser.add_argument("--owner") + parser.add_argument("--beacon", type=int) + parser.add_argument("--run-dir", type=Path) + parser.add_argument("--live", action="store_true") + args = parser.parse_args(argv) + if args.action == "export": + print( + json.dumps( + export_artifacts( + args.input, + Path(args.extra[0]), + json.loads(Path(args.extra[1]).read_bytes()), + ) + ) + ) + elif args.action == "plan": + profile = json.loads(args.input.read_bytes()) + print(json.dumps({"paid_execution": False, "artifact_budget": artifact_budget(profile)})) + else: + if not args.owner or args.beacon is None or args.run_dir is None: + parser.error("lifecycle actions require --owner --beacon --run-dir") + checked = reviewed_launch(args.input, args.owner, args.beacon, args.action) + import signal + + cancel = threading.Event() + previous = {s: signal.getsignal(s) for s in (signal.SIGTERM, signal.SIGINT)} + try: + for s in previous: + signal.signal(s, lambda signum, frame: cancel.set()) + print( + json.dumps( + launch_action( + args.action, + checked, + args.run_dir, + live=args.live, + cancel=cancel, + ) + ) + ) + finally: + for s, handler in previous.items(): + signal.signal(s, handler) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/gpu_network_v2/profile.json b/experiments/gpu_network_v2/profile.json new file mode 100644 index 0000000..9da1808 --- /dev/null +++ b/experiments/gpu_network_v2/profile.json @@ -0,0 +1,80 @@ +{ + "profile_version": 2, + "qualification": "BOUNDED_RENTAL_QUALIFICATION_REQUIRED", + "microqualification": { + "max_full_rounds_per_host": 2, + "staging_seconds": 300, + "probe_and_transfer_seconds": 180, + "rescue_delete_seconds": 300, + "failure_margin_seconds": 120, + "qualification_deadline_seconds": 900, + "same_instances_only": true, + "counts_inside_126_execution_reservation": true, + "root_verified_lifecycle_required": true, + "image_registry_inspection_required": true, + "candidate_image_max_bytes": 20000000000, + "candidate_stage_max_bytes": 1000000000, + "candidate_transfer_gb_per_host": 30, + "claim": "CORRECTNESS_ONLY_NOT_PRODUCTION_SPEED_MEMORY_OR_QUALITY" + }, + "runtime_evidence_sha256": null, + "execution_and_transfer_bound_seconds": null, + "model": { + "arch": "decoder", "n_layers": 2, "d_model": 32, "n_heads": 4, + "n_kv_heads": 4, "d_ff": 64, "n_experts": 4, "top_k_experts": 2, + "vocab": 64, "seq_len": 16, "capacity_factor": "0000a03f", + "aux_loss_coef": "0ad7233c", "compute_dtype": "bf16", "master_dtype": "fp32", + "init_seed": 17, "init_std": "0ad7a33c", "param_count": 61856 + }, + "layout": {"pp": 1, "n_gpus": 2, "dp_size": 1, "ep_size": 2, "zero1": true}, + "inner": { + "opt": "adamw", "H": 30, "J": 5, "micro_batch": 1, "grad_accum": 1, + "betas": ["6666663f", "3333733f"], "eps": "77cc2b32", "wd": "cdcccc3d", + "grad_clip": "0000803f", "state_policy": "carry", "rewarmup_steps": 0, + "lr_schedule": {"peak_lr": "6f12833a", "warmup": 0, "stable": 100000, "decay": 0} + }, + "outer": {"opt": "sparseloco", "topk_frac": "0ad7233c", "bits": 2, "ef_beta": "0000803f"}, + "schedule": { + "hosts": 2, "logical_identities": 4, "shadow_participations_per_identity": 12, + "shadow_executions": 96, "live_executions": 16, "probes": 2, + "fault_trace_reserve": 12, "executions_total": 126, "executions_per_host": 63, + "staging_seconds": 300, "execution_transfer_seconds": 2520, + "beacon_service_seconds": 180, "rescue_delete_seconds": 300, + "hard_margin_seconds": 300, "hard_deadline_seconds": 3600 + }, + "admission": { + "ceiling_usd": "15", "cleanup_reserve_usd": "5", "max_host_hourly_usd": "2", + "max_ancillary_usd": "1", "gpu_name": "RTX 5090", "gpus_per_host": 2, + "min_cpu_cores": 8, "min_host_ram_gib": 32, "min_disk_gib": 80, + "required_torch": "2.14.0+cu130", "sm_count": 170 + }, + "artifacts": { + "safetensors_header_bytes_per_file": 65536, + "trace_bytes_per_event": 512, + "logs_bytes_per_execution": 2097152, + "metadata_bytes_per_execution": 1048576, + "image_and_dependency_bytes": null, + "staging_bytes_per_host": null, + "rescue_bytes_per_second": null, + "rescue_transfer_seconds": 180, + "delete_absence_seconds": 120, + "disk_bytes": 80000000000, + "retain_all_checkpoints": true, + "path_preserving_rescue": true + }, + "runtime": { + "backend": "cuda", + "remote_python": "/opt/hypertrain/venv/bin/python", + "torch": "2.14.0+cu130", + "cuda": "13.0", + "image_digest": null, + "driver_allowlist": [], + "CUBLAS_WORKSPACE_CONFIG": ":4096:8", + "CUDA_DISABLE_PTX_JIT": "1", + "OMP_NUM_THREADS": "1", + "MKL_NUM_THREADS": "1", + "PYTHONHASHSEED": "0", + "NVIDIA_TF32_OVERRIDE": "0", + "qualified": false + } +} diff --git a/experiments/gpu_network_v2/run.py b/experiments/gpu_network_v2/run.py new file mode 100644 index 0000000..c20178e --- /dev/null +++ b/experiments/gpu_network_v2/run.py @@ -0,0 +1,28 @@ +"""Run a staged, content-authenticated IslandJobV1; no provider actions. + +Usage: uv run --frozen python experiments/gpu_network_v2/run.py JOB_DIR [cpu|cuda] +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +from hypertrain.miner.island_launch import launch_island +from hypertrain.protocol.envelope_v2 import load_json +from hypertrain.protocol.messages_v2 import IslandJobV1 + + +def run(directory: Path, device: str = "cuda") -> Path: + job = IslandJobV1.model_validate(load_json((directory / "job.json").read_bytes())) + match device: + case "cpu": + return launch_island(job, directory, backend="cpu", trace=True).directory + case "cuda": + return launch_island(job, directory, backend="cuda", trace=True).directory + case _: + raise ValueError("device must be cpu or cuda") + + +if __name__ == "__main__": + print(run(Path(sys.argv[1]), sys.argv[2] if len(sys.argv) > 2 else "cuda")) diff --git a/scripts/canary.sh b/scripts/canary.sh index 192222e..164c115 100755 --- a/scripts/canary.sh +++ b/scripts/canary.sh @@ -29,8 +29,6 @@ trap cleanup EXIT pass() { echo "PASS: $*"; } die() { echo "FAIL: $*" >&2; exit 1; } -code() { curl -s -o /dev/null -w '%{http_code}' "$@"; } - # podman rejects the uid= tmpfs option; a world-writable /data is equivalent for the uid-65532 check. if [ "$engine" = docker ]; then data_opt=/data:uid=65532,gid=65532; else data_opt=/data:mode=1777; fi @@ -39,6 +37,7 @@ start() { # extra run args... "$engine" run -d --name "$name" --read-only --user 65532:65532 --cap-drop ALL \ --security-opt no-new-privileges --tmpfs /tmp --tmpfs "$data_opt" \ -p "127.0.0.1:$port:8000" "$@" "$image" >/dev/null + code() { curl -s -o /dev/null -w '%{http_code}' "$@"; } for _ in $(seq 120); do [ "$(code "$base/version")" = 200 ] && return 0 sleep 1 diff --git a/scripts/check.sh b/scripts/check.sh index 97c5768..5cbb662 100755 --- a/scripts/check.sh +++ b/scripts/check.sh @@ -4,6 +4,8 @@ set -euo pipefail cd "$(dirname "$0")/.." uv run --frozen ruff format --check uv run --frozen ruff check -uv run --frozen mypy src +MYPYPATH=src:scripts:. uv run --frozen mypy --explicit-package-bases \ + --package hypertrain --module experiments.gpu_network_v2.orchestrate \ + --module scripts.network_service_proof --module scripts.network_gpu_operation # Extra args go to pytest (CI splits the slow sim/e2e tests into their own job). uv run --frozen pytest -q --durations=25 "$@" diff --git a/scripts/network_authority_snapshot.py b/scripts/network_authority_snapshot.py new file mode 100644 index 0000000..8571357 --- /dev/null +++ b/scripts/network_authority_snapshot.py @@ -0,0 +1,647 @@ +"""Admitted OD MLM finalized-admission export; no reconstructed historical intake.""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import sqlite3 +import stat +import time +from collections.abc import Iterator, Mapping +from contextlib import closing, contextmanager +from pathlib import Path, PurePosixPath +from typing import Never + +from pydantic import BaseModel, ConfigDict, Field + +from hypertrain.beacon.core import BeaconRound, FixtureBeacon +from hypertrain.challenge.admission import Admission +from hypertrain.challenge.app import Config, create_app +from hypertrain.data.store import LocalFSStore +from hypertrain.data.trial_assignment import trial_assignment_hash, trial_samples +from hypertrain.ledger import Params +from hypertrain.ledger.escrow_v2 import EscrowV2, FinalityEvidence, authority_message +from hypertrain.miner.island_launch import validate_artifacts +from hypertrain.protocol import envelope_v2 +from hypertrain.protocol.hashing import sha256_hex +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.keys import Keypair, decode_hotkey, verify +from hypertrain.protocol.messages_v2 import ( + AdmissionPolicyV2, + DisputePolicyV2, + EconomicsPolicyV2, + IslandJobV1, + JoinChallenge, + RunManifestV2, + TestGenesis, + WorkProof, + WorkScreenV2, +) +from hypertrain.protocol.relay_messages import RelayRegistryV1 + +RUN = "587fc10ec7aeee92e235d39c9332c763e16cde05e5e13a8562204c700adce325" +MISSING = ("original dual-signed JoinRequest envelopes", "original signed EscrowLock envelopes") +MAX_FILES, MAX_BYTES, MAX_FILE = 8192, 1 << 30, 16 << 20 + + +class FileEntry(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + size: int = Field(strict=True, ge=0, le=MAX_FILE) + + +class Bundle(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + version: int = Field(default=1, strict=True, ge=1, le=1) + run_id: str = Field(default=RUN, pattern="^" + RUN + "$") + capability: str = Field( + default="restored-finalized-admission", pattern="^restored-finalized-admission$" + ) + missing_provenance: tuple[str, ...] = MISSING + files: dict[str, FileEntry] = Field(min_length=1, max_length=MAX_FILES) + public_roles: dict[str, str] + manifest_envelope_sha256: str + genesis_envelope_sha256: str + beacon_round: int = Field(strict=True, ge=1) + + +def _files(root: Path) -> dict[str, FileEntry]: + """Bound regular files; no symlinks, sockets, path escapes or SQLite sidecars.""" + if root.is_symlink(): + raise ValueError("snapshot root symlink") + entries: dict[str, FileEntry] = {} + total = 0 + for path in sorted(root.rglob("*")): + if path.is_symlink(): + raise ValueError("snapshot symlink") + if path.is_dir(): + continue + relative = path.relative_to(root).as_posix() + if relative == "bundle.json" or relative in ("challenge.db-wal", "challenge.db-shm"): + continue + if not stat.S_ISREG(path.stat().st_mode) or path.stat().st_size > MAX_FILE: + raise ValueError("snapshot file size/type") + total += path.stat().st_size + if len(entries) >= MAX_FILES or total > MAX_BYTES: + raise ValueError("snapshot total bound") + entries[relative] = FileEntry( + sha256=sha256_hex(path.read_bytes()), size=path.stat().st_size + ) + return entries + + +def _unavailable(*args: str | int | FinalityEvidence) -> Never: + raise ValueError("snapshot validation cannot authorize live settlement") + + +class _StatusCoordinator(Keypair): + """Public-only identity for status; historical validation has no signing capability.""" + + def __init__(self, public: str) -> None: + self.ss58 = public + self.public = decode_hotkey(public) + + def sign(self, message: bytes) -> Never: + raise ValueError("status-only coordinator cannot sign") + + +def _authority(root: Path) -> Bundle: + """Validate original signatures/assignments and saved rank artifacts, never train.""" + from hypertrain.auditor.replay import AnchorCache, pack_state, unpack_state + from hypertrain.protocol.hashing import MerkleTree + from hypertrain.trainer.compress import state_hash + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.model import init_params + + if not (root / "objects").is_dir() or not (root / "challenge.db").is_file(): + raise ValueError("critical snapshot storage missing") + objects = LocalFSStore(root / "objects") + with ( + closing(sqlite3.connect((root / "challenge.db").as_uri() + "?mode=ro", uri=True)) as source, + closing(sqlite3.connect(":memory:", isolation_level=None)) as db, + ): + source.backup(db) + db.row_factory = sqlite3.Row + runs = db.execute("SELECT * FROM runs").fetchall() + if len(runs) != 1: + raise ValueError("snapshot run count") + manifest = RunManifestV2.model_validate_json(runs[0]["manifest"]) + signed = envelope_v2.parse_envelope(runs[0]["envelope"]) + if ( + manifest.run_id() != RUN + or runs[0]["run_id"] != RUN + or signed.type != "RunManifestV2" + or signed.run_id != RUN + or signed.body != manifest.body() + or not envelope_v2.verify_envelope(runs[0]["envelope"]) + ): + raise ValueError("foreign original manifest") + m, layout = manifest.training.model, manifest.training.reference_spec.layout + if ( + m.arch != "od-encoder" + or m.od is None + or m.od.objective != "mlm" + or m.param_count != 116288 + or layout.n_gpus != 2 + or layout.dp_size != 2 + or layout.ep_size != 1 + or not layout.zero1 + ): + raise ValueError("OD MLM N2 profile mismatch") + policy = EconomicsPolicyV2.model_validate_json( + objects.get(manifest.network.economics_policy_hash) + ) + admission = AdmissionPolicyV2.model_validate_json( + objects.get(manifest.network.admission_policy_hash) + ) + dispute = DisputePolicyV2.model_validate_json( + objects.get(manifest.network.dispute_policy_hash) + ) + registry = RelayRegistryV1.model_validate_json( + objects.get(manifest.network.relay_registry_hash) + ) + for field in ("aggregation_policy_hash", "audit_policy_hash"): + objects.get(getattr(manifest.network, field)) + if ( + policy.ledger_mode != "test" + or db.execute("SELECT COUNT(*) FROM rounds").fetchone()[0] + or db.execute("SELECT COUNT(*) FROM disputes_v2").fetchone()[0] + or db.execute("SELECT COUNT(*) FROM audit_leases_v2").fetchone()[0] + ): + raise ValueError("snapshot is not admission-only test authority") + if db.execute( + "SELECT COUNT(*) FROM records_v2 WHERE kind NOT IN ('dataset','execution-backend')" + ).fetchone()[0]: + raise ValueError("snapshot contains production state") + beacons = db.execute("SELECT * FROM beacon ORDER BY round").fetchall() + for b in beacons: + fixture = FixtureBeacon(current=b["round"]).get(b["round"]) + if b["bls"] or (b["signature"], b["randomness"]) != ( + fixture.signature, + fixture.randomness, + ): + raise ValueError("fixture beacon mapping differs") + for pin in db.execute("SELECT * FROM beacon_pins_v2"): + b = db.execute("SELECT * FROM beacon WHERE round=?", (pin["pinned_round"],)).fetchone() + if ( + b is None + or pin["run_id"] != RUN + or pin["signature"] != b["signature"] + or pin["signature_hash"] != sha256_hex(bytes.fromhex(b["signature"])) + or pin["paused"] + ): + raise ValueError("accepted beacon pin differs") + genesis_rows = db.execute("SELECT envelope FROM accepted_v2").fetchall() + if len(genesis_rows) != 1: + raise ValueError("original genesis intake missing") + genesis_raw = genesis_rows[0][0] + env = envelope_v2.parse_envelope(genesis_raw) + genesis = TestGenesis.model_validate(env.body) + event = db.execute("SELECT request FROM escrow_events WHERE kind='TEST_GENESIS'").fetchone() + if ( + env.type != "TestGenesis" + or env.run_id != RUN + or env.signer != manifest.training.coord_pubkey + or not envelope_v2.verify_envelope(genesis_raw) + or not verify( + decode_hotkey(env.signer), + authority_message(genesis), + bytes.fromhex(genesis.authority_sig), + ) + or event is None + or genesis.model_dump(mode="json", exclude={"authority_sig"}) != json.loads(event[0]) + ): + raise ValueError("original signed genesis differs") + escrow = EscrowV2( + db, + manifest, + policy, + auditors=frozenset(manifest.training.auditors), + replay_finality=_unavailable, + settlement=_unavailable, + owner_of=_unavailable, + assignment_of=_unavailable, + ) + escrow.verify() + current_round = beacons[-1]["round"] + + def beacon(number: int) -> BeaconRound: + row = db.execute("SELECT * FROM beacon WHERE round=?", (number,)).fetchone() + if row is None: + raise ValueError("current verified beacon missing") + return BeaconRound(number, row["signature"], row["randomness"], False) + + def no_stage(c: JoinChallenge, samples: tuple[int, ...], epoch: int) -> Never: + raise ValueError("status validation cannot execute a new reference") + + current_admission = Admission( + escrow, + admission, + _StatusCoordinator(manifest.training.coord_pubkey), + beacon=beacon, + stage_reference=no_stage, + objects=objects, + ip_secret=b"", + qualified=lambda screen: False, + conservative_bound=lambda: policy.ledger_mode == "test", + backend="cpu", + ) + rows = db.execute("SELECT * FROM admissions_v2 ORDER BY hotkey").fetchall() + if len(rows) != 4 or len({r["coldkey"] for r in rows}) != 4: + raise ValueError("four original identities required") + roles = {"coord": manifest.training.coord_pubkey, "owner": signed.signer} + for prefix, keys in ( + ("auditor", manifest.training.auditors), + ("referee", dispute.referees), + ("relay", [k.pubkey for spec in registry.specs for k in spec.pubkeys]), + ): + roles.update({f"{prefix}-{i}": key for i, key in enumerate(keys)}) + dataset = json.loads( + db.execute( + "SELECT data FROM records_v2 WHERE kind='dataset' AND id='inputs'" + ).fetchone()[0] + ) + samples_raw = objects.get(dataset["samples_hash"]) + proofs = json.loads(objects.get(dataset["proofs_hash"])) + width = (m.seq_len + 1) * 2 + if ( + len(samples_raw) != manifest.training.dataset.n_samples * width + or len(proofs) != manifest.training.dataset.n_samples + ): + raise ValueError("signed dataset dimensions differ") + tree = MerkleTree([samples_raw[i : i + width] for i in range(0, len(samples_raw), width)]) + if tree.root.hex() != manifest.training.dataset.merkle_root: + raise ValueError("signed dataset root differs") + theta = init_params(TrainConfig.from_manifest_v2(manifest).model) + if state_hash(theta) != manifest.training.init_state_hash: + raise ValueError("signed genesis theta differs") + if ( + db.execute("SELECT COUNT(*) FROM admission_trials").fetchone()[0] != 48 + or db.execute("SELECT COUNT(*) FROM admission_trial_results").fetchone()[0] != 48 + ): + raise ValueError("unexpected trial authority rows") + for i, row in enumerate(rows): + roles.update({f"hot-{i}": row["hotkey"], f"cold-{i}": row["coldkey"]}) + status = current_admission.status(row["hotkey"], now=current_round) + history = db.execute( + "SELECT admission_id,coldkey FROM admission_history WHERE hotkey=?", + (row["hotkey"],), + ).fetchone() + if ( + not status.eligible + or status.shadow_only + or history is None + or tuple(history) != (row["admission_id"], row["coldkey"]) + ): + raise ValueError("current funded admission is ineligible at restored beacon") + if ( + row["run_id"] != RUN + or row["state"] != "ACTIVE" + or row["clean_count"] != 12 + or row["canary_blocks"] != "1,2,3" + or row["strikes"] + or row["pending_dispute"] + or row["policy_hash"] != admission.digest() + or escrow.locked(row["admission_id"], row["coldkey"])[0] < 1000 + ): + raise ValueError("original admission eligibility differs") + trials = db.execute( + "SELECT t.*,r.challenge,r.reference,r.proof,r.screen,r.outcome " + "FROM admission_trials t JOIN admission_trial_results r USING(epoch) " + "WHERE admission_id=?", + (row["admission_id"],), + ).fetchall() + if len(trials) != 12: + raise ValueError("twelve original trials required") + for trial in trials: + c = JoinChallenge.model_validate_json(trial["challenge"]) + ref = WorkProof.model_validate_json(trial["reference"]) + ch_raw = db.execute( + "SELECT receipt FROM admission_reservations WHERE reservation=?", + (f"challenge|{row['admission_id']}|{c.nonce}",), + ).fetchone()[0] + final_raw = db.execute( + "SELECT receipt FROM admission_reservations WHERE reservation=?", + (f"trial-final|{row['admission_id']}|{trial['epoch']}",), + ).fetchone()[0] + envs = [ + envelope_v2.parse_envelope(raw) + for raw in (ch_raw, trial["proof"], trial["screen"], final_raw) + ] + for raw, envelope, expected_type, signer in zip( + (ch_raw, trial["proof"], trial["screen"], final_raw), + envs, + ("JoinChallenge", "WorkProof", "WorkScreenV2", "Finalize"), + (roles["coord"], row["hotkey"], row["hotkey"], roles["coord"]), + strict=True, + ): + if ( + envelope.type != expected_type + or envelope.run_id != RUN + or envelope.signer != signer + or not envelope_v2.verify_envelope(raw) + or envelope.exp_drand < trial["finalized_beacon"] + ): + raise ValueError("original trial signature/expiry differs") + screen = WorkScreenV2.model_validate(envs[2].body) + b = db.execute("SELECT * FROM beacon WHERE round=?", (c.seed_beacon,)).fetchone() + ids = trial_samples( + manifest, + row["admission_id"], + c.nonce, + BeaconRound(b["round"], b["signature"], b["randomness"], False), + ) + if ( + trial["outcome"] != "MATCH" + or ref != WorkProof.model_validate(envs[1].body) + or c.body() != envs[0].body + or c.admission_id != row["admission_id"] + or c.nonce != trial["nonce"] + or c.manifest_hash != RUN + or c.policy_hash != admission.digest() + or c.assignment_hash != trial_assignment_hash(manifest, trial["epoch"], ids) + or ref.challenge_hash != c.digest() + or ref.digest() != trial["evidence_hash"] + or screen.challenge_hash != c.digest() + or screen.nonce != c.nonce + or screen.admission_id != row["admission_id"] + or screen.layout != layout + ): + raise ValueError("original reference/challenge binding differs") + directory = root / "trials-v2" / c.nonce + inputs = { + name: (directory / name).read_bytes() + for name in ("start_state", "ef_in", "v0", "samples", "sample_proofs") + } + genesis_anchor = AnchorCache().genesis(manifest, row["hotkey"], theta) + if ( + inputs["start_state"] != pack_state(genesis_anchor.theta, genesis_anchor.state) + or inputs["ef_in"] != pack_state(genesis_anchor.ef) + or inputs["v0"] != pack_state({}) + ): + raise ValueError("saved trial genesis anchor differs") + if ( + not trial["received_beacon"] <= trial["finalized_beacon"] <= c.deadline_beacon + or screen.image_digest != manifest.training.reference_spec.image_digest + or screen.artifact_hashes != [a.sha256 for a in ref.artifact_refs] + ): + raise ValueError("original trial deadline/screen differs") + if inputs["samples"] != b"".join( + samples_raw[x * width : (x + 1) * width] for x in ids + ) or json.loads(inputs["sample_proofs"]) != [proofs[x] for x in ids]: + raise ValueError("saved trial assignment inputs differ") + job = IslandJobV1( + job_version=1, + run_id=RUN, + w=trial["epoch"], + manifest=manifest, + sample_ids=list(ids), + global_step0=0, + start_state_sha256=sha256_hex(inputs["start_state"]), + ef_in_sha256=sha256_hex(inputs["ef_in"]), + v0_sha256=sha256_hex(inputs["v0"]), + object_paths={k: k for k in inputs}, + deadline=manifest.training.beacon.genesis_time + (c.deadline_beacon - 1) * 3, + ) + artifacts = validate_artifacts(job, directory / "published") + if list(artifacts.ranks) != screen.rank_results: + raise ValueError("original screen rank artifacts differ") + for path, artifact in zip( + (artifacts.state, artifacts.ef, artifacts.delta, artifacts.leaves), + ref.artifact_refs, + strict=True, + ): + if ( + path.read_bytes() != objects.get(artifact.sha256) + or path.stat().st_size != artifact.size + ): + raise ValueError("original artifact reference differs") + final = envs[3].body + if ( + final["w"] != trial["epoch"] + or final["included"] != [row["hotkey"]] + or final["entitlements_root"] != ref.digest() + or final["final_theta_hash_w1"] + != state_hash(unpack_state(artifacts.state.read_bytes())[0]) + ): + raise ValueError("original final reference differs") + manifest_raw = runs[0]["envelope"] + if isinstance(manifest_raw, str): + manifest_raw = manifest_raw.encode() + if isinstance(genesis_raw, str): + genesis_raw = genesis_raw.encode() + return Bundle( + files=_files(root), + public_roles=roles, + manifest_envelope_sha256=sha256_hex(manifest_raw), + genesis_envelope_sha256=sha256_hex(genesis_raw), + beacon_round=beacons[-1]["round"], + ) + + +def export(source: Path, destination: Path) -> str: + """Export exact snapshot, return root-admittable index digest; no secrets included.""" + source = source.resolve(strict=True) + before = _files(source) + _authority(source) + if destination.exists(): + raise ValueError("export destination exists") + destination.mkdir(mode=0o700, parents=True) + with ( + closing(sqlite3.connect((source / "challenge.db").as_uri() + "?mode=ro", uri=True)) as src, + closing(sqlite3.connect(destination / "challenge.db")) as dst, + ): + src.backup(dst) + original_database = sha256_hex(dst.serialize()) + (destination / "challenge.db").chmod(0o600) + for relative in before: + if relative != "challenge.db": + target = destination / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source / relative, target) + bundle = _authority(destination) + if _files(source) != before: + raise ValueError("source changed during export") + with ( + closing(sqlite3.connect((source / "challenge.db").as_uri() + "?mode=ro", uri=True)) as src, + closing(sqlite3.connect(":memory:")) as check, + ): + src.backup(check) + if sha256_hex(check.serialize()) != original_database: + raise ValueError("source database changed during export") + raw = canonicalize(bundle.model_dump(mode="json")) + (destination / "bundle.json").write_bytes(raw) + return sha256_hex(raw) + + +def validate(bundle_dir: Path, admitted_hash: str) -> Bundle: + """Fail before restore for missing, foreign or modified admitted package bytes.""" + bundle_dir = bundle_dir.resolve(strict=True) + if (bundle_dir / "bundle.json").is_symlink(): + raise ValueError("bundle index symlink") + raw = (bundle_dir / "bundle.json").read_bytes() + if len(raw) > 2 << 20 or sha256_hex(raw) != admitted_hash: + raise ValueError("admitted bundle hash differs") + bundle = Bundle.model_validate_json(raw) + for relative in bundle.files: + path = PurePosixPath(relative) + if path.is_absolute() or ".." in path.parts or "\\" in relative or str(path) != relative: + raise ValueError("unsafe indexed path") + if ( + bundle.missing_provenance != MISSING + or _files(bundle_dir) != bundle.files + or _authority(bundle_dir) != bundle + ): + raise ValueError("admitted authority/index differs") + return bundle + + +def verify_secrets(secrets: Path, bundle: Bundle) -> None: + """Require supplied original role identities; never mint or export replacement keys.""" + if ( + secrets.is_symlink() + or secrets.stat().st_mode & 0o077 + or secrets.stat().st_uid != os.getuid() + ): + raise ValueError("secret directory must be owned private") + for role, public in bundle.public_roles.items(): + path = secrets / (role + ".seed") + if ( + path.is_symlink() + or not stat.S_ISREG(path.stat().st_mode) + or path.stat().st_mode & 0o077 + or path.stat().st_uid != os.getuid() + or path.stat().st_size != 32 + ): + raise ValueError("role secret must be owned regular0600 raw32") + if Keypair(path.read_bytes()).ss58 != public: + raise ValueError("role public identity differs") + token = secrets / "admin.token" + if ( + token.is_symlink() + or not stat.S_ISREG(token.stat().st_mode) + or token.stat().st_mode & 0o077 + or token.stat().st_uid != os.getuid() + or not 1 <= token.stat().st_size <= 4096 + or not token.read_bytes().strip() + ): + raise ValueError("private admin token required") + + +@contextmanager +def bootstrap( + bundle_dir: Path, admitted_hash: str, state_dir: Path, secrets: Path, master_url: str +) -> Iterator[Config]: + """Configure existing actual service with original-role files and bounded fixture clock.""" + bundle = validate(bundle_dir, admitted_hash) + verify_secrets(secrets, bundle) + token = secrets / "admin.token" + if state_dir.exists(): + raise ValueError("bootstrap state already exists") + state_dir.mkdir(mode=0o700, parents=True) + for relative in bundle.files: + target = state_dir / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(bundle_dir / relative, target) + (state_dir / "challenge.db").chmod(0o600) + with closing(sqlite3.connect(state_dir / "challenge.db")) as db: + manifest = RunManifestV2.model_validate_json( + db.execute("SELECT manifest FROM runs").fetchone()[0] + ) + ledger = state_dir / "ledger/journal.jsonl" + params = ( + Params(**json.loads(ledger.read_text().splitlines()[0])["params"]) + if ledger.exists() + else Params( + "hypertrain", + manifest.training.beacon.genesis_time, + 4320, + manifest.training.budget.epochs_per_round, + manifest.training.verify.E_vest_rounds, + ) + ) + historical = manifest.training.beacon.genesis_time + (bundle.beacon_round - 1) * 3 + baseline = time.monotonic() + old_time = time.time + offset = old_time() - historical + clock_dir = state_dir / "fixture-clock" + clock_dir.mkdir() + (clock_dir / "sitecustomize.py").write_text( + "import os,time\n_real=time.time\n" + "_offset=float(os.environ['HT_FIXTURE_CLOCK_OFFSET'])\n" + "time.time=lambda:_real()-_offset\n" + ) + previous = {k: os.environ.get(k) for k in ("PYTHONPATH", "HT_FIXTURE_CLOCK_OFFSET")} + os.environ["PYTHONPATH"] = ( + str(clock_dir) + os.pathsep + previous["PYTHONPATH"] + if previous["PYTHONPATH"] + else str(clock_dir) + ) + os.environ["HT_FIXTURE_CLOCK_OFFSET"] = str(offset) + time.time = lambda: historical + time.monotonic() - baseline + try: + yield Config( + params.challenge_slug, + state_dir, + master_url, + None, + token, + None, + secrets / "coord.seed", + bundle.public_roles["owner"], + params, + ) + finally: + time.time = old_time + for key, value in previous.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + +def fixture_beacon(payload: Mapping[str, int | str]) -> BeaconRound: + """Explicit deterministic CPU test authority; never advertise BLS verification.""" + number = int(payload["round"]) + expected = FixtureBeacon(current=number).get(number) + if payload["signature"] != expected.signature or payload["randomness"] != expected.randomness: + raise ValueError("fixture beacon payload differs") + return expected + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("operation", choices=("export", "validate", "serve")) + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--destination", type=Path) + parser.add_argument("--admitted-hash") + parser.add_argument("--secrets", type=Path) + parser.add_argument("--master-url", default="https://master.test") + parser.add_argument("--port", type=int, default=8080) + parser.add_argument("--host", default="127.0.0.1") + args = parser.parse_args() + if args.operation == "export": + if args.destination is None: + parser.error("export requires --destination") + print(export(args.source, args.destination)) + elif args.operation == "validate": + if args.admitted_hash is None: + parser.error("validate requires --admitted-hash") + print(validate(args.source, args.admitted_hash).model_dump_json(exclude={"files"})) + else: + if args.admitted_hash is None or args.destination is None or args.secrets is None: + parser.error("serve requires --admitted-hash --destination --secrets") + import uvicorn + + with bootstrap( + args.source, args.admitted_hash, args.destination, args.secrets, args.master_url + ) as config: + uvicorn.run( + create_app(config, verify_beacon=fixture_beacon), host=args.host, port=args.port + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/network_gpu_operation.py b/scripts/network_gpu_operation.py new file mode 100644 index 0000000..d349774 --- /dev/null +++ b/scripts/network_gpu_operation.py @@ -0,0 +1,739 @@ +"""Trusted existing-host operation runner; no provider or public callback API.""" + +from __future__ import annotations + +import contextlib +import io +import json +import os +import shutil +import signal +import stat +import subprocess +import sys +import threading +import time +from pathlib import Path +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, JsonValue, StrictInt + +from hypertrain.gpu_ops.journal import durable_write, fsha +from hypertrain.miner.island_launch import IslandArtifacts, confined, validate_artifacts +from hypertrain.protocol.envelope_v2 import load_json, parse_envelope, seal, verify_envelope +from hypertrain.protocol.hashing import MerkleTree, sha256_hex +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.keys import Keypair, decode_hotkey +from hypertrain.protocol.messages import LeafPreimage +from hypertrain.protocol.messages_v2 import CommitV2, IslandJobV1, JoinChallenge, WorkProof + + +class Operation(BaseModel): + """Internal accepted descriptor from trusted service adapter, not wire authority.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + operation: Literal["reference", "audit", "referee", "probe", "live"] + binding: str = Field(pattern=r"^[0-9a-f]{64}$") + hotkey: str + owner: str + role: str = Field(pattern=r"^h[01]$") + instance_id: StrictInt = Field(gt=0) + machine_id: StrictInt = Field(gt=0) + job: IslandJobV1 + binding_kind: Literal["trial", "commit-lease"] + sources: dict[str, str] + image_digest: str + cutoff: StrictInt + backend: Literal["cuda"] = "cuda" + trace: bool = True + miner_config: str | None = None + ca: str | None = None + master: str | None = None + challenge: str | None = None + context_files: dict[str, str] = Field(default_factory=dict) + + +def check(op: Operation, tree: Path, directory: Path) -> None: + """Check exact materialized source, original objects and immutable public context.""" + decode_hotkey(op.hotkey) + decode_hotkey(op.owner) + if ( + time.time() >= op.cutoff + or op.cutoff > op.job.deadline + or (directory / "operation-cancelled").exists() + ): + raise ValueError("operation deadline") + if op.image_digest != op.job.manifest.training.reference_spec.image_digest: + raise ValueError("operation image") + if ( + op.operation in ("reference", "probe") + and op.binding_kind != "trial" + or op.operation in ("audit", "referee", "live") + and op.binding_kind != "commit-lease" + ): + raise ValueError("operation accepted binding kind") + from hypertrain.gpu_ops import network_qualification as engine + + observed = engine.sources(tree) + observed["scripts/network_gpu_operation.py"] = fsha(tree / "scripts/network_gpu_operation.py") + if op.sources != observed: + raise ValueError("operation source map") + for rel, digest in op.context_files.items(): + if fsha(confined(tree, rel)) != digest: + raise ValueError("operation context file hash") + required = ( + (op.miner_config, op.ca) + if op.operation == "live" + else (op.miner_config, op.ca, op.challenge) + if op.operation == "probe" + else () + ) + if any(path is None or path not in op.context_files for path in required): + raise ValueError("operation context file pins missing") + if op.operation == "reference" and op.challenge is not None: + if op.challenge not in op.context_files: + raise ValueError("reference challenge context pin missing") + for key in ("start_state", "ef_in", "v0"): + if fsha(confined(directory, op.job.object_paths[key])) != getattr(op.job, key + "_sha256"): + raise ValueError("operation input hash") + + +def trial_authority(op: Operation, raw: dict) -> JoinChallenge: + """Validate original coordinator authority, not a claimed nonce or result.""" + from hypertrain.data.trial_assignment import trial_assignment_hash + + env = parse_envelope(raw) + challenge = JoinChallenge.model_validate(env.body) + if ( + env.type != "JoinChallenge" + or env.signer != op.job.manifest.training.coord_pubkey + or env.run_id != op.job.run_id + or not verify_envelope(raw) + or env.exp_drand < challenge.deadline_beacon + or challenge.admission_id != op.binding + or challenge.manifest_hash != op.job.run_id + or challenge.layout != op.job.manifest.training.reference_spec.layout + or challenge.assignment_hash + != trial_assignment_hash(op.job.manifest, op.job.w, tuple(op.job.sample_ids)) + ): + raise ValueError("custody original trial authority differs") + return challenge + + +def probe_commit(op: Operation, artifacts: IslandArtifacts, key: Keypair, challenge: dict) -> dict: + """Actual miner signs original publication; no launcher or acceptance assertion.""" + from hypertrain.auditor.replay import unpack_state + from hypertrain.trainer.compress import state_hash + + bound = trial_authority(op, challenge) + if key.ss58 != op.hotkey or op.operation != "probe": + raise ValueError("custody miner signing identity") + artifacts = validate_artifacts(op.job, artifacts.directory) + summary = load_json((artifacts.directory / "rank-0/summary.json").read_bytes())["commitments"] + if not isinstance(summary, dict): + raise ValueError("custody publication commitments must be an object") + hashes: dict[str, str] = {} + for name in ("leaves_root", "final_theta_hash", "ef_out_hash", "delta_hash"): + value = summary[name] + if not isinstance(value, str): + raise ValueError("custody publication commitment must be a string") + hashes[name] = value + preimages = [LeafPreimage.model_validate(p) for p in json.loads(artifacts.leaves.read_bytes())] + ef, _ = unpack_state(confined(artifacts.directory, op.job.object_paths["ef_in"]).read_bytes()) + commit = CommitV2( + w=op.job.w, + hotkey=key.ss58, + leaf_scheme="ht-leaf-v1", + n_leaves=len(preimages), + metrics_root=MerkleTree( + [bytes.fromhex(p.loss_f32) + bytes.fromhex(p.norm_f32) for p in preimages] + ).root.hex(), + tokens=len(op.job.sample_ids) * op.job.manifest.training.model.seq_len, + delta_bytes=artifacts.delta.stat().st_size, + ef_in_hash=state_hash(ef), + **hashes, + ) + commit.validate_assignment(op.job.manifest, len(op.job.sample_ids)) + return seal(key, "CommitV2", op.job.run_id, commit, bound.deadline_beacon) + + +def execution_environment(op: Operation) -> dict: + """Observe this original child; no training or inferred CUDA qualification.""" + import torch + + observed: dict[str, JsonValue] = { + "backend": op.backend, + "python": sys.executable, + "torch": str(torch.__version__), + "torch_path": str(Path(torch.__file__).resolve()), + "cuda": torch.version.cuda, + "image_digest": op.image_digest, + } + if op.backend == "cuda": + smi = shutil.which("nvidia-smi") + if smi is None or not torch.cuda.is_available(): + raise ValueError("custody CUDA environment unavailable") + drivers = subprocess.run( + args=["/usr/bin/nvidia-smi", "--query-gpu=driver_version", "--format=csv,noheader"], + executable=smi, + check=True, + capture_output=True, + text=True, + timeout=10, + ).stdout.splitlines() + layout = op.job.manifest.training.reference_spec.layout + if torch.cuda.device_count() != layout.n_gpus or len(drivers) != layout.n_gpus: + raise ValueError("custody CUDA device layout differs") + if any( + d.strip() not in op.job.manifest.training.reference_spec.driver_allowlist + for d in drivers + ): + raise ValueError("custody CUDA driver differs") + observed["drivers"] = [d.strip() for d in drivers] + observed["sm_counts"] = [ + torch.cuda.get_device_properties(i).multi_processor_count for i in range(layout.n_gpus) + ] + return observed + + +def publication_custody( + op: Operation, + directory: Path, + artifacts: IslandArtifacts, + challenge: dict | None, + commit: dict | None = None, + proof: dict | None = None, +) -> dict: + """Hash original all-rank execution output without replay or accepted-status claim.""" + validated = validate_artifacts(op.job, artifacts.directory) + from hypertrain.auditor.replay import unpack_state + from hypertrain.trainer.compress import state_hash + + start, _ = unpack_state( + confined(artifacts.directory, op.job.object_paths["start_state"]).read_bytes() + ) + files: dict[str, JsonValue] = {} + for path in sorted(artifacts.directory.rglob("*")): + if path.is_symlink(): + raise ValueError("custody publication symlink") + if path.is_file(): + rel = str(path.relative_to(artifacts.directory)) + confined(artifacts.directory, rel) + files[rel] = {"sha256": fsha(path), "bytes": path.stat().st_size} + ranks: list[dict[str, JsonValue]] = [] + for rank in range(op.job.manifest.training.reference_spec.layout.n_gpus): + raw = load_json(confined(artifacts.directory, f"rank-{rank}/summary.json").read_bytes()) + if raw["backend"] != op.backend: + raise ValueError("custody execution backend differs") + if op.trace and f"rank-{rank}/trace.json" not in files: + raise ValueError("custody original trace absent") + ranks.append(raw) + body: dict[str, JsonValue] = { + "operation": op.operation, + "operation_sha256": sha256_hex(op.model_dump_json().encode()), + "binding": op.binding, + "hotkey": op.hotkey, + "owner": op.owner, + "role": op.role, + "instance_id": op.instance_id, + "machine_id": op.machine_id, + "run_id": op.job.run_id, + "job": op.job.body(), + "job_sha256": op.job.digest(), + "manifest_sha256": op.job.manifest.digest(), + "sample_ids_sha256": sha256_hex(canonicalize(op.job.sample_ids)), + "layout": op.job.manifest.training.reference_spec.layout.model_dump(mode="json"), + "sources": {name: pin for name, pin in op.sources.items()}, + "image_digest": op.image_digest, + "backend": op.backend, + "rank_summaries": [rank for rank in ranks], + "environment": execution_environment(op), + "required_environment": op.job.manifest.training.reference_spec.env.model_dump(mode="json"), + "publication": str(artifacts.directory.relative_to(directory)), + "files": files, + "publication_sha256": sha256_hex(canonicalize(files)), + "trial_authority": None, + "miner_commit": None, + } + if challenge is not None: + bound = trial_authority(op, challenge) + if bound.theta_hash != state_hash(start): + raise ValueError("custody original challenge start differs") + body["trial_authority"] = { + "envelope": challenge, + "sha256": sha256_hex(canonicalize(challenge)), + "challenge_hash": bound.digest(), + "nonce": bound.nonce, + "seed_beacon": bound.seed_beacon, + "assignment_hash": bound.assignment_hash, + } + if commit is not None: + env = parse_envelope(commit) + c = CommitV2.model_validate(env.body) + summary = ranks[0]["commitments"] + if not isinstance(summary, dict): + raise ValueError("custody publication commitments must be an object") + if ( + challenge is None + or env.type != "CommitV2" + or not verify_envelope(commit) + or env.signer != op.hotkey + or env.run_id != op.job.run_id + or (c.hotkey, c.w) != (op.hotkey, op.job.w) + or c.leaves_root != summary["leaves_root"] + or c.delta_hash != fsha(validated.delta) + or c.final_theta_hash != summary["final_theta_hash"] + or c.ef_out_hash != summary["ef_out_hash"] + or env.exp_drand < JoinChallenge.model_validate(challenge["body"]).deadline_beacon + or c.delta_bytes != validated.delta.stat().st_size + ): + raise ValueError("custody original miner commit differs") + c.validate_assignment(op.job.manifest, len(op.job.sample_ids)) + body["miner_commit"] = {"envelope": commit, "sha256": sha256_hex(canonicalize(commit))} + if proof is not None: + env = parse_envelope(proof) + p = WorkProof.model_validate(env.body) + summary = ranks[0]["commitments"] + if not isinstance(summary, dict): + raise ValueError("custody publication commitments must be an object") + if ( + challenge is None + or env.type != "WorkProof" + or not verify_envelope(proof) + or env.signer != op.hotkey + or env.run_id != op.job.run_id + or p.challenge_hash != JoinChallenge.model_validate(challenge["body"]).digest() + or p.admission_id != op.binding + or p.delta_hash != fsha(validated.delta) + or p.leaves_root != summary["leaves_root"] + ): + raise ValueError("custody original miner proof differs") + for ref, path in zip( + p.artifact_refs, + (validated.state, validated.ef, validated.delta, validated.leaves), + strict=True, + ): + if ref.sha256 != fsha(path) or ref.size != path.stat().st_size: + raise ValueError("custody original proof object differs") + body["miner_proof"] = {"envelope": proof, "sha256": sha256_hex(canonicalize(proof))} + return { + "body": body, + "sha256": sha256_hex(canonicalize(body)), + "status": "CAPTURED_NOT_ACCEPTED", + } + + +def publish_custody_result( + op: Operation, + directory: Path, + result: dict, + custody: dict, + cancel: threading.Event, +) -> None: + """Final finite-lifecycle gate after hashing, before any success publication.""" + if cancel.is_set() or time.time() >= op.cutoff: + raise ValueError("custody publication deadline/cancellation") + durable_write(directory / "publication-custody.json", canonicalize(custody)) + result["publication_custody_sha256"] = fsha(directory / "publication-custody.json") + result["miner_commit"] = custody["body"]["miner_commit"] + result["environment"] = custody["body"]["environment"] + if cancel.is_set() or time.time() >= op.cutoff: + raise ValueError("custody publication deadline/cancellation") + durable_write(directory / "operation-result.json", json.dumps(result, sort_keys=True).encode()) + durable_write( + directory / "execution-custody.json", + canonicalize( + { + "operation_sha256": custody["body"]["operation_sha256"], + "job_sha256": op.job.digest(), + "binding": op.binding, + "operation": op.operation, + "instance_id": op.instance_id, + "machine_id": op.machine_id, + "publication_custody_sha256": result["publication_custody_sha256"], + "operation_result_sha256": fsha(directory / "operation-result.json"), + "environment_sha256": sha256_hex(canonicalize(result["environment"])), + "status": "CAPTURED_NOT_ACCEPTED", + } + ), + ) + + +def _capture_miner_cli( + op: Operation, tree: Path, directory: Path, cancel: threading.Event +) -> tuple[dict, Path, dict | None, dict | None, dict | None]: + """Capture original miner CLI/API output only; no completion or acceptance claim.""" + if op.operation not in ("probe", "live"): + raise ValueError("miner capture operation kind") + result: dict = {} + original_challenge = None + miner_commit = None + miner_proof = None + from hypertrain.miner import cli + from hypertrain.miner.core import MinerConfig, NetworkMiner, load_keyfile + from hypertrain.protocol.envelope_v2 import parse_envelope, verify_envelope + + if op.miner_config is None or op.ca is None or op.master is None: + raise ValueError("miner operation context missing") + cfg = MinerConfig.load(confined(tree, op.miner_config), env={}) + key = load_keyfile(cfg.keyfile) + if ( + cfg.run_id != op.job.run_id + or cfg.owner_hotkey != op.owner + or key.ss58 != op.hotkey + or cfg.api != op.master + or cfg.device != "cuda" + or cfg.image_digest != op.image_digest + or cfg.workdir.resolve() != directory.resolve() / "miner" + ): + raise ValueError("miner operation identity/config") + os.environ["SSL_CERT_FILE"] = str(confined(tree, op.ca)) + for name in list(os.environ): + if name.startswith("HYPERTRAIN_MINER_"): + del os.environ[name] + # In this private CLI child, only observe original transport; never change request/response. + import httpx + + original = httpx.Client + import hypertrain.miner.island_launch as island + + original_launch = island.launch_island + operation_cancel = cancel + + def guarded_launch( + job: IslandJobV1, + directory: Path, + *, + backend: Literal["cpu", "cuda"], + cancel: threading.Event | None = cancel, + trace: bool = False, + ) -> IslandArtifacts: + nonlocal miner_commit + if job != op.job or backend != op.backend or trace != op.trace: + raise ValueError("miner launch accepted descriptor changed") + if cancel is not operation_cancel or time.time() >= op.cutoff or operation_cancel.is_set(): + raise ValueError("miner launch deadline/cancellation") + if op.operation == "probe": + publication = directory / "published" + publication.mkdir() + for relative in job.object_paths.values(): + target = publication / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(confined(directory, relative), target) + durable_write(publication / "job.json", job.model_dump_json().encode()) + from hypertrain.miner.island_launch import _launch_unbounded + + _launch_unbounded( + job, + publication, + backend, + dict(os.environ, HT_ISLAND_TRACE="1" if trace else "0"), + cancel, + ) + captured = validate_artifacts(job, publication) + assert op.challenge is not None + miner_commit = probe_commit( + op, captured, key, load_json(confined(tree, op.challenge).read_bytes()) + ) + return captured + return original_launch(job, directory, backend=backend, trace=trace, cancel=cancel) + + signed: list[dict] = [] + accepted: list[dict] = [] + from hypertrain.protocol.messages_v2 import MAX_MANIFEST_BYTES + + # One run_round emits Accept, Commit, WorkProof, Delta; no hidden capture retries. + capture_limit = 4 if op.operation == "live" else 0 + capture_bytes = MAX_MANIFEST_BYTES + 1024 + responses = 0 + + def publish_capture(filename: str, payload: bytes) -> None: + """Replace only this owned regular capture, relative to a no-follow directory fd.""" + fd = os.open(directory, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + temporary = "." + filename + ".next" + try: + if os.fstat(fd).st_uid != os.getuid(): + raise ValueError("capture directory owner") + + def existing() -> os.stat_result | None: + try: + info = os.stat(filename, dir_fd=fd, follow_symlinks=False) + except FileNotFoundError: + return None + if ( + not stat.S_ISREG(info.st_mode) + or info.st_uid != os.getuid() + or info.st_nlink != 1 + ): + raise ValueError("capture path not owned regular file") + return info + + prior = existing() + stream_fd = os.open( + temporary, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, + dir_fd=fd, + ) + try: + with os.fdopen(stream_fd, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + current = existing() + if (None if current is None else (current.st_dev, current.st_ino)) != ( + None if prior is None else (prior.st_dev, prior.st_ino) + ): + raise ValueError("capture destination changed") + os.replace(temporary, filename, src_dir_fd=fd, dst_dir_fd=fd) + os.fsync(fd) + finally: + try: + os.unlink(temporary, dir_fd=fd) + except FileNotFoundError: + pass + finally: + os.close(fd) + + def request_hook(request: httpx.Request) -> None: + if request.method == "POST" and request.content: + body = json.loads(request.content) + if isinstance(body, dict) and body.get("v") == "ht/2": + if len(signed) >= capture_limit or len(request.content) > capture_bytes: + raise ValueError("capture request count/byte bound") + if time.time() >= op.cutoff or cancel.is_set(): + raise ValueError("capture deadline/cancellation") + env = parse_envelope(body) + if ( + not verify_envelope(body) + or env.run_id != op.job.run_id + or env.signer != op.hotkey + ): + raise ValueError("miner original signature/run") + index = len(signed) + publish_capture(f"signed-api-{index}.bin", request.content) + request.extensions["operation_capture_index"] = index + signed.append(body) + publish_capture("signed-api.json", json.dumps(signed).encode()) + + def response_hook(response: httpx.Response) -> None: + nonlocal responses + if response.request.method == "POST" and response.request.content: + raw = json.loads(response.request.content) + if isinstance(raw, dict) and raw.get("v") == "ht/2": + index = response.request.extensions["operation_capture_index"] + if responses >= capture_limit or index != responses: + raise ValueError("capture response order/count") + content = bytearray() + for chunk in response.iter_bytes(): + if len(content) + len(chunk) > capture_bytes: + raise ValueError("capture response byte bound") + content.extend(chunk) + response._content = bytes(content) + provenance = { + "index": index, + "method": response.request.method, + "url": str(response.request.url), + "status": response.status_code, + "request_hex": response.request.content.hex(), + "response_hex": bytes(content).hex(), + } + publish_capture(f"response-api-{index}.json", json.dumps(provenance).encode()) + responses += 1 + response.raise_for_status() + accepted.append({"request": raw, "response": response.json(), **provenance}) + publish_capture("accepted-api.json", json.dumps(accepted).encode()) + if time.time() >= op.cutoff or cancel.is_set(): + raise ValueError("capture deadline/cancellation") + if response.request.url.path.endswith("/job/" + op.hotkey): + response.read() + actual = IslandJobV1.model_validate(response.json()["job"]) + if actual != op.job: + raise ValueError("accepted live job changed") + + class ObservedClient(httpx.Client): + def __init__(self, *, timeout: float, follow_redirects: bool) -> None: + super().__init__( + timeout=timeout, + follow_redirects=follow_redirects, + event_hooks={"request": [request_hook], "response": [response_hook]}, + ) + + argv = [ + "probe-v2" if op.operation == "probe" else "run-v2", + "--config", + str(tree / op.miner_config), + ] + if op.operation == "probe": + if op.challenge is None: + raise ValueError("probe challenge missing") + argv += [ + "--job", + str(directory / "job.json"), + "--challenge", + str(confined(tree, op.challenge)), + ] + from hypertrain.protocol.messages_v2 import JoinChallenge + + challenge_env = parse_envelope(confined(tree, op.challenge).read_bytes()) + original_challenge = challenge_env.model_dump(mode="json") + challenge = JoinChallenge.model_validate(challenge_env.body) + if ( + not verify_envelope(challenge_env.model_dump(mode="json")) + or challenge_env.signer != op.job.manifest.training.coord_pubkey + or challenge.admission_id != op.binding + ): + raise ValueError("probe accepted trial differs") + else: + argv += ["--round", str(op.job.w)] + output = io.StringIO() + try: + with contextlib.redirect_stdout(output): + if op.operation == "probe": + assert op.challenge is not None + with ObservedClient(timeout=120, follow_redirects=False) as client: + outcome = NetworkMiner(cfg, client).probe( + directory / "job.json", + confined(tree, op.challenge), + launch=guarded_launch, + cancel=cancel, + trace=op.trace, + ) + print(json.dumps(outcome, sort_keys=True)) + code = 0 + else: + island.__dict__["launch_island"] = guarded_launch + cli.httpx.__dict__["Client"] = ObservedClient + code = cli.main(argv) + finally: + if op.operation != "probe": + cli.httpx.__dict__["Client"] = original + island.__dict__["launch_island"] = original_launch + if code: + raise ValueError("product miner CLI failed") + outcome = json.loads(output.getvalue()) + if op.operation == "probe": + miner_proof = outcome["proof"] + signed.extend([outcome["proof"], outcome["screen"]]) + for raw in signed: + signed_env = parse_envelope(raw) + if ( + not verify_envelope(raw) + or signed_env.run_id != op.job.run_id + or signed_env.signer != op.hotkey + ): + raise ValueError("probe signed outcome") + published = directory / "published" + else: + if not {"AcceptV2", "CommitV2", "DeltaManifestV2"} <= {raw["type"] for raw in signed}: + raise ValueError("live signed outcomes missing") + published = cfg.workdir / op.job.run_id / op.hotkey / str(op.job.w) / "published" + result["signed_api"] = signed + result["accepted_api"] = accepted + result["cli_outcome"] = outcome + return result, published, original_challenge, miner_commit, miner_proof + + +def execute(op: Operation, tree: Path, directory: Path) -> None: + """Only real product launch/CLI; original signed outbound API bodies kept unchanged.""" + check(op, tree, directory) + cancel = threading.Event() + signal.signal(signal.SIGTERM, lambda signum, frame: cancel.set()) + signal.signal(signal.SIGINT, lambda signum, frame: cancel.set()) + durable_write(directory / "job.json", op.job.model_dump_json().encode()) + result: dict = { + "binding": op.binding, + "operation": op.operation, + "hotkey": op.hotkey, + "role": op.role, + "job_sha256": op.job.digest(), + "status": "FAILED", + } + original_challenge = None + miner_commit = None + miner_proof = None + if op.operation in ("reference", "audit", "referee"): + from hypertrain.miner.island_launch import _launch_unbounded + + publication = directory / "published" + publication.mkdir() + for rel in op.job.object_paths.values(): + target = publication / rel + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(confined(directory, rel), target) + durable_write(publication / "job.json", op.job.model_dump_json().encode()) + env = dict(os.environ, HT_ISLAND_TRACE="1" if op.trace else "0") + _launch_unbounded(op.job, publication, op.backend, env, cancel) + artifacts = validate_artifacts(op.job, publication) + if op.operation == "reference" and op.challenge is not None: + original_challenge = load_json(confined(tree, op.challenge).read_bytes()) + else: + captured, published, original_challenge, miner_commit, miner_proof = _capture_miner_cli( + op, tree, directory, cancel + ) + result.update(captured) + artifacts = validate_artifacts(op.job, published) + if time.time() >= op.cutoff or cancel.is_set(): + raise ValueError("operation expired before result") + result.update( + status="CAPTURED_NOT_ACCEPTED", publication=str(artifacts.directory.relative_to(directory)) + ) + custody = publication_custody( + op, directory, artifacts, original_challenge, miner_commit, miner_proof + ) + publish_custody_result(op, directory, result, custody, cancel) + + +def main() -> None: + tree, directory, spec = (Path(v).resolve() for v in sys.argv[1:4]) + op = Operation.model_validate(load_json(spec.read_bytes())) + os.environ["HT_IMAGE_DIGEST"] = op.image_digest + if len(sys.argv) == 5 and sys.argv[4] == "child": + execute(op, tree, directory) + return + check(op, tree, directory) + directory.mkdir(parents=True, exist_ok=True) + if (directory / "operation-result.json").exists(): + raise ValueError("operation not retryable") + durable_write(directory / "operation-result.json", b'{"status":"FAILED_OR_UNFINISHED"}') + with ( + (directory / "operation-stdout.log").open("xb") as out, + (directory / "operation-stderr.log").open("xb") as err, + ): + proc = subprocess.Popen( # noqa: S603 - fixed self/interpreter argv; descriptor already parsed + [sys.executable, __file__, str(tree), str(directory), str(spec), "child"], + stdout=out, + stderr=err, + start_new_session=True, + ) + durable_write(directory / "operation-pid", str(proc.pid).encode()) + if (directory / "operation-cancelled").exists(): + os.killpg(proc.pid, signal.SIGKILL) + + def terminate(signum: int, frame: object) -> None: + try: + os.killpg(proc.pid, signal.SIGTERM) + except ProcessLookupError: + pass + + signal.signal(signal.SIGTERM, terminate) + signal.signal(signal.SIGINT, terminate) + try: + code = proc.wait(timeout=max(0.0, op.cutoff - time.time() - 2)) + finally: + terminate(signal.SIGTERM, None) + try: + proc.wait(timeout=max(0.0, op.cutoff - time.time())) + except subprocess.TimeoutExpired: + pass + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + proc.wait() + if code: + raise ValueError("operation child failed; custody retained") + + +if __name__ == "__main__": + main() diff --git a/scripts/network_gpu_prepare.py b/scripts/network_gpu_prepare.py new file mode 100644 index 0000000..d036b44 --- /dev/null +++ b/scripts/network_gpu_prepare.py @@ -0,0 +1,646 @@ +"""Prepare local D2 artifacts from verified frozen bytes; never admit or launch GPUs.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import io +import json +import os +import secrets +import sys +import tarfile +from pathlib import Path + + +def digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def openssl(arguments: list[str]) -> None: + pid = os.posix_spawn( + "/usr/bin/openssl", + ["/usr/bin/openssl", *arguments], + os.environ, + file_actions=[(os.POSIX_SPAWN_OPEN, 2, os.devnull, os.O_WRONLY, 0o600)], + ) + _, status = os.waitpid(pid, 0) + if os.waitstatus_to_exitcode(status) != 0: + raise ValueError("experiment-local certificate generation failed") + + +def save(path: Path, value, private: bool = False) -> None: + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + data = value if isinstance(value, bytes) else json.dumps(value, sort_keys=True).encode() + with path.open("xb") as stream: + os.fchmod(stream.fileno(), 0o600 if private else 0o644) + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + + +def load_frozen(freeze_path: Path): + """Verify complete staging, qualification and operation maps before importing.""" + freeze = json.loads(freeze_path.read_bytes()) + tree = Path(freeze["snapshot"]) / "tree" + for item in ( + *freeze["maps"].values(), + freeze["archive"], + freeze["qualification_archive"], + ): + path = Path(item["path"]) + if ( + path.is_symlink() + or path.stat().st_size != item["bytes"] + or digest(path) != item["sha256"] + ): + raise ValueError("frozen archive/map changed") + maps = { + name: json.loads(Path(item["path"]).read_bytes()) for name, item in freeze["maps"].items() + } + for relative, expected in maps["sources.json"].items(): + path = tree / relative + if ( + Path(relative).is_absolute() + or ".." in Path(relative).parts + or path.is_symlink() + or digest(path) != expected + ): + raise ValueError("frozen source changed: " + relative) + if any(digest(tree / name) != expected for name, expected in freeze["approved_pins"].items()): + raise ValueError("approved frozen pin changed") + if any(name == "hypertrain" or name.startswith("hypertrain.") for name in sys.modules): + raise ValueError("preparation requires clean process without live package imports") + sys.path[:0] = [str(tree / "src"), str(tree / "scripts")] + + def module(name: str, relative: str): + spec = importlib.util.spec_from_file_location(name, tree / relative) + if spec is None or spec.loader is None: + raise ValueError("frozen callable absent: " + relative) + loaded = importlib.util.module_from_spec(spec) + sys.modules[name] = loaded + spec.loader.exec_module(loaded) + return loaded + + seed = module("prepared_seed", "scripts/network_gpu_seed.py") + driver = module("prepared_driver", "scripts/network_service_proof.py") + controller = module("prepared_controller", "experiments/gpu_network_v2/orchestrate.py") + from hypertrain.gpu_ops.network_qualification import sources + + if sources(tree) != maps["qualification-sources.json"]: + raise ValueError("canonical qualification map differs") + operation = dict(sources(tree)) + operation["scripts/network_gpu_operation.py"] = digest( + tree / "scripts/network_gpu_operation.py" + ) + if operation != maps["operation-sources.json"]: + raise ValueError("operation map differs") + return freeze, tree, maps, seed, driver, controller + + +def service_factory(driver, runtime, owner, run_id, sources, roles, current_beacon): + """Derive receipts only for actual accepted contexts, through frozen factory.""" + from network_service_proof import service_factory as accepted_factory + + return accepted_factory(driver, runtime, owner, run_id, sources, roles, current_beacon) + + +def validate_launch(controller, profile: dict, remaining: dict) -> None: + """Exercise original admission refusal; never infer observations from candidates.""" + from hypertrain.gpu_ops.launcher import Reject + + try: + controller.runtime_admission(profile, {}) + except Reject as error: + raise ValueError( + "launch refused: " + str(error) + "; missing=" + ",".join(remaining) + ) from error + raise ValueError("launch refused: required observed admission inputs absent") + + +def candidate_freeze(original: Path, destination: Path, candidates: list[str]) -> Path: + """Copy verified original bytes; change only the expressly approved profile input.""" + if candidates != ["580.95.05", "580.159.03"]: + raise ValueError("candidate drivers require exact approved ordered pair") + freeze = json.loads(original.read_bytes()) + old_tree = Path(freeze["snapshot"]) / "tree" + for item in ( + *freeze["maps"].values(), + freeze["archive"], + freeze["qualification_archive"], + ): + path = Path(item["path"]) + if ( + path.is_symlink() + or path.stat().st_size != item["bytes"] + or digest(path) != item["sha256"] + ): + raise ValueError("original frozen archive/map changed") + maps = { + name: json.loads(Path(item["path"]).read_bytes()) for name, item in freeze["maps"].items() + } + for relative, expected in maps["sources.json"].items(): + path = old_tree / relative + if ( + Path(relative).is_absolute() + or ".." in Path(relative).parts + or path.is_symlink() + or digest(path) != expected + ): + raise ValueError("original source changed: " + relative) + key = "experiments/gpu_network_v2/profile.json" + profile = json.loads((old_tree / key).read_bytes()) + profile["runtime"]["driver_allowlist"] = candidates + tree = destination / "tree" + for relative in maps["sources.json"]: + save( + tree / relative, + profile if relative == key else (old_tree / relative).read_bytes(), + ) + changed = digest(tree / key) + for name, values in maps.items(): + if name == "files.json": + for relative, entry in values.items(): + entry.update( + origin=str(tree / relative), + size=(tree / relative).stat().st_size, + sha256=digest(tree / relative), + ) + else: + values[key] = changed + save(destination / name, values) + freeze["maps"][name].update( + path=str(destination / name), + bytes=(destination / name).stat().st_size, + sha256=digest(destination / name), + ) + for field, map_name, filename in ( + ("archive", "sources.json", "effective-source.tar"), + ( + "qualification_archive", + "qualification-sources.json", + "qualification-source.tar", + ), + ): + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w") as archive: + for relative in sorted(maps[map_name]): + data = (tree / relative).read_bytes() + info = tarfile.TarInfo(relative) + info.size, info.mode, info.mtime = len(data), 0o600, 0 + archive.addfile(info, io.BytesIO(data)) + save(destination / filename, buffer.getvalue()) + freeze[field].update( + path=str(destination / filename), + bytes=(destination / filename).stat().st_size, + sha256=digest(destination / filename), + files=len(maps[map_name]), + ) + freeze.update( + snapshot=str(destination), + effective_profile_sha256=changed, + parent_freeze_sha256=digest(original), + candidate_input_scope={ + "drivers": candidates, + "qualification_executions": 4, + "observed_driver": "NOT_PRESENT", + "CUDA_qualified": False, + "spend_authorized": False, + }, + ) + for relative, entry in freeze["service_source_observed"].items(): + entry["origin"] = str(tree / relative) + freeze["gate"] = {"scope": "CANDIDATE_INPUT_REGENERATION_NOT_RUNTIME_QUALIFICATION"} + freeze["qualification_parser"]["package_origin"] = str( + tree / "src/hypertrain/gpu_ops/network_qualification.py" + ) + freeze["qualification_parser"]["wrapper_origin"] = str( + tree / "scripts/network_gpu_qualification.py" + ) + freeze["public_inputs"] = { + "claim": "SYNTHETIC_FUNCTIONAL_SEED_NOT_CUDA_QUALIFIED", + "historical_only": True, + "fresh_seed_generated": False, + } + save(destination / "freeze.json", freeze) + return destination / "freeze.json" + + +def prepare( + freeze_path: Path, output: Path, genesis: int, candidates: list[str] | None = None +) -> dict: + if output.exists() or genesis < 1: + raise ValueError("new output and positive deterministic CPU genesis required") + if candidates is not None and candidates != ["580.95.05", "580.159.03"]: + raise ValueError("candidate drivers require exact approved ordered pair") + if candidates is not None: + freeze_path = candidate_freeze(freeze_path, output / "source-freeze", candidates) + freeze, tree, maps, seed, driver, controller = load_frozen(freeze_path) + output.mkdir(parents=True, exist_ok=candidates is not None, mode=0o700) + private, public = output / "private", output / "public" + profile = json.loads((tree / "experiments/gpu_network_v2/profile.json").read_bytes()) + seed_hash = seed.generate( + public / "seed", + private / "roles", + profile["runtime"]["image_digest"], + profile["runtime"]["driver_allowlist"], + genesis, + genesis + 3600, + ) + job = seed.validate(public / "seed", seed_hash) + from hypertrain.beacon.core import FixtureBeacon + from hypertrain.miner.core import MinerConfig + from hypertrain.protocol.keys import Keypair + + owner = Keypair((private / "roles/owner.seed").read_bytes()) + for role in seed.ROLES: + save( + private / (role + ".hex"), + (private / ("roles/" + role + ".seed")).read_bytes().hex().encode(), + True, + ) + for name in ("admin", "worker", "internal", "drain"): + save(private / (name + ".token"), secrets.token_hex(32).encode(), True) + # Experiment-local trust only; never publish the CA private key. + openssl( + [ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-days", + "2", + "-subj", + "/CN=Hypertrain local experiment CA", + "-addext", + "basicConstraints=critical,CA:TRUE", + "-keyout", + str(private / "ca.key"), + "-out", + str(public / "ca.pem"), + ], + ) + (private / "ca.key").chmod(0o600) + for host in ("master.test", "relay.test"): + openssl( + [ + "req", + "-newkey", + "rsa:2048", + "-nodes", + "-subj", + "/CN=" + host, + "-keyout", + str(private / (host + ".key")), + "-out", + str(private / (host + ".csr")), + ], + ) + save( + private / (host + ".ext"), + ("subjectAltName=DNS:" + host + "\nextendedKeyUsage=serverAuth\n").encode(), + True, + ) + openssl( + [ + "x509", + "-req", + "-in", + str(private / (host + ".csr")), + "-CA", + str(public / "ca.pem"), + "-CAkey", + str(private / "ca.key"), + "-set_serial", + str(1 if host == "master.test" else 2), + "-days", + "2", + "-extfile", + str(private / (host + ".ext")), + "-out", + str(public / (host + ".crt")), + ], + ) + (private / (host + ".key")).chmod(0o600) + beacon = FixtureBeacon(current=1).get(1) + save( + public / "cpu-beacon.json", + { + "round": 1, + "signature": beacon.signature, + "randomness": beacon.randomness, + "bls_verified": False, + "purpose": "DETERMINISTIC_CPU_ONLY", + "genesis_unix": genesis, + }, + ) + joins = [] + for i in range(4): + hot, cold = ( + Keypair((private / ("roles/" + kind + f"-{i}.seed")).read_bytes()) + for kind in ("hot", "cold") + ) + join = driver.sign_join( + hot, + cold, + run_id=job.run_id, + request_id=f"{i + 1:064x}", + expires_beacon=10000, + policy_hash=job.manifest.network.admission_policy_hash, + hardware_hint=driver.HardwareHint( + device_name="advisory", device_count=2, driver="not-qualification" + ), + ) + save(public / f"join-{i}.json", driver.canonicalize(join.body())) + joins.append( + { + "hotkey": hot.ss58, + "coldkey": cold.ss58, + "role": "h0" if i < 2 else "h1", + "join": f"join-{i}.json", + } + ) + config = { + "api": "https://master.test", + "keyfile": str(private / f"hot-{i}.hex"), + "workdir": str(output / f"operation-hot-{i}/miner"), + "state_source": str(public / "seed/start_state"), + "image_digest": profile["runtime"]["image_digest"], + "run_id": job.run_id, + "owner_hotkey": owner.ss58, + "device": "cpu", + } + toml = "\n".join(key + " = " + json.dumps(value) for key, value in config.items()) + "\n" + path = private / f"miner-{i}.toml" + save(path, toml.encode(), True) + MinerConfig.load(path, env={}) + save(public / "membership.json", joins) + from hypertrain.data.store import LocalFSStore + from hypertrain.protocol.relay_messages import RelayNetworkManifest, RelayRegistryV1 + from hypertrain.relay.app import RelayConfig + + registry = RelayRegistryV1.model_validate_json( + LocalFSStore(public / "seed/objects").get(job.manifest.network.relay_registry_hash) + ) + coord = Keypair((private / "roles/coord.seed").read_bytes()) + network = RelayNetworkManifest( + run_id=job.run_id, + base_manifest_hash=job.manifest.digest(), + registry_hash=registry.digest(), + specs_hash=driver.sha256_hex(driver.canonicalize([s.body() for s in registry.specs])), + assignment_policy="master-observed-median3-v1", + ) + relay_cfg = RelayConfig( + run_id=job.run_id, + master_public=coord.ss58, + master_url="https://master.test", + registry=driver.relay_envelope.seal(coord, "RelayRegistryV1", job.run_id, registry, 10000), + network_manifest=driver.relay_envelope.seal( + coord, "RelayNetworkManifest", job.run_id, network, 10000 + ), + relay_id="local", + region="local", + active_key="k1", + signing_files={"k1": "relay.hex"}, + observer_public_keys=job.manifest.training.auditors, + local_backing=str(output / "relay-objects"), + ) + save(private / "relay.json", relay_cfg.model_dump(mode="json"), True) + save( + private / "tls.json", + { + "master": { + "cert": str(public / "master.test.crt"), + "key": str(private / "master.test.key"), + }, + "relay": { + "cert": str(public / "relay.test.crt"), + "key": str(private / "relay.test.key"), + }, + "ca": str(public / "ca.pem"), + }, + True, + ) + save( + private / "service.json", + { + "state_dir": str(output / "cpu-service"), + "production_state_dir": str(output / "production-service"), + "master_url": "https://master.test", + "relay_url": "https://relay.test", + "ca": str(public / "ca.pem"), + "coord_key_file": str(private / "coord.hex"), + "owner_hotkey": owner.ss58, + "admin_token_file": str(private / "admin.token"), + "genesis_unix": genesis, + "backend": "cpu", + "production_launchable": False, + }, + True, + ) + (output / "production-service").mkdir(mode=0o700) + # Genuine signed bootstrap only; qualification must precede creation in the OTHER state. + from hypertrain.challenge.store import ChallengeStore + from hypertrain.data.store import LocalFSStore + from hypertrain.ledger import Params + + store = ChallengeStore( + output / "cpu-service", + Params("hypertrain", genesis, 4320, 1, job.manifest.training.verify.E_vest_rounds), + Keypair((private / "roles/coord.seed").read_bytes()), + owner.ss58, + driver.authority.fixture_beacon, + LocalFSStore(public / "seed/objects"), + ) + store.clock = lambda: float(genesis) + try: + store.push_beacon( + {"round": 1, "signature": beacon.signature, "randomness": beacon.randomness} + ) + store.create_run_v2((public / "seed/manifest-envelope.json").read_bytes()) + store.genesis_v2(job.run_id, (public / "seed/genesis-envelope.json").read_bytes()) + escrow, _, _ = store._services(job.run_id) + escrow.verify() + if store._db.execute("SELECT COUNT(*) FROM admission_trials").fetchone()[0]: + raise ValueError("unexpected trial during preparation") + finally: + store.close_v2_notifications() + store._db.close() + count = driver.decoder_execution_count(profile) + save(public / "artifact-bounds.json", controller.artifact_budget(profile)) + remaining = { + name: "NOT_PRESENT" + for name in ( + "owned_instances_and_offers", + "CUDA_qualification", + "observed_driver_and_imports", + "fresh_financial_observations_and_spend_GO", + "verified_production_beacon", + "measured_staging_rescue_rates", + "current_trial_commit_lease_dispute_anchor_contexts", + "supervisor_and_custody_receipts", + ) + } + refusal = "" + try: + validate_launch(controller, profile, remaining) + except ValueError as error: + refusal = str(error) + if not refusal: + raise ValueError("preparation unexpectedly accepted launch") + for name in ( + "qualification-sources.json", + "operation-sources.json", + "sources.json", + ): + save(public / name, maps[name]) + save( + private / "lifecycle-config.json", + { + "hosts": [], + "cleanup_contract": "network-v2", + "tree": str(tree), + "network_profile_file": str(tree / "experiments/gpu_network_v2/profile.json"), + "image": freeze["image_candidate"]["ref"], + "remote_python": profile["runtime"]["remote_python"], + "disk_gb": 80, + "hard_deadline_seconds": 3600, + "hard_grace_seconds": 0, + }, + True, + ) + save( + public / "lifecycle-hashes.json", + { + name: maps["qualification-sources.json"][name] + for name in ( + "src/hypertrain/gpu_ops/launcher.py", + "src/hypertrain/gpu_ops/supervisor.py", + ) + }, + ) + review_files = { + name: {"path": str(path), "sha256": digest(path)} + for name, path in { + "config": private / "lifecycle-config.json", + "profile": tree / "experiments/gpu_network_v2/profile.json", + "sources": Path(freeze["maps"]["qualification-sources.json"]["path"]), + "bundle": Path(freeze["qualification_archive"]["path"]), + "manifest": public / "seed/manifest-envelope.json", + "genesis": public / "seed/genesis-envelope.json", + "lifecycle_hashes": public / "lifecycle-hashes.json", + }.items() + } + record = { + "action": "admit", + "owner": owner.ss58, + "tree": str(tree), + "parent_pid": os.getppid(), + "files": review_files, + "roles": { + "h0": [x["hotkey"] for x in joins[:2]], + "h1": [x["hotkey"] for x in joins[2:]], + }, + "quota": {"total": 126, "per_role": 63, "qualification": 4, "remaining": 122}, + "preparation_only": True, + "remaining": remaining, + } + signed_record = driver.envelope_v2.seal( + owner, + "Receipt", + job.run_id, + { + "w": 0, + "commit_hash": driver.sha256_hex(driver.canonicalize(record)), + "received_round": 1, + }, + 10000, + ) + save(private / "review.json", {"record": record, "receipt": signed_record}, True) + from hypertrain.gpu_ops.launcher import Reject + + try: + controller.reviewed_launch(private / "review.json", owner.ss58, 1, "admit") + except Reject as error: + controller_refusal = str(error) + else: + raise ValueError("incomplete preparation unexpectedly admitted by controller") + inventory = { + str(path.relative_to(public)): digest(path) + for path in sorted(public.rglob("*")) + if path.is_file() + } + save(public / "index.json", inventory) + receipts = { + "source": maps["operation-sources.json"], + "index": inventory, + "freeze": freeze, + } + for name, subject in receipts.items(): + save( + public / (name + "-receipt.json"), + driver.canonicalize( + driver.envelope_v2.seal( + owner, + "Receipt", + job.run_id, + { + "w": 0, + "commit_hash": driver.sha256_hex(driver.canonicalize(subject)), + "received_round": 1, + }, + 10000, + ) + ), + ) + result = { + "status": "LOCAL_PREPARED_NOT_LAUNCHABLE", + "run_id": job.run_id, + "seed_index_sha256": seed_hash, + "freeze_sha256": digest(freeze_path), + "tree": str(tree), + "public_index_sha256": digest(public / "index.json"), + "count": count, + "remaining": remaining, + "launch_refusal": refusal, + "controller_refusal": controller_refusal, + "operation_receipts": "CREATED_ONLY_AT_RUNTIME_FROM_ACCEPTED_CONTEXTS_BY_service_factory", + "keys": "ORIGINAL_DETERMINISTIC_SYNTHETIC_SEED_ROLES_NOT_PRODUCTION_CREDENTIALS", + "kernels_executed": 0, + "provider_calls": 0, + "candidate_input_scope": freeze.get( + "candidate_input_scope", {"drivers": profile["runtime"]["driver_allowlist"]} + ), + } + result["controller_review"] = str(private / "review.json") + result["production_state"] = "EMPTY_UNTIL_SAME_INSTANCE_CUDA_QUALIFICATION_BOOTSTRAP" + save(private / "controller.json", result, True) + save(output / "prepared.json", result) + return result + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--freeze", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--cpu-genesis-unix", type=int, required=True) + parser.add_argument("--candidate-driver", action="append") + args = parser.parse_args() + os.umask(0o077) + print( + json.dumps( + prepare( + args.freeze.resolve(), + args.output.resolve(), + args.cpu_genesis_unix, + args.candidate_driver, + ), + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/network_gpu_qualification.py b/scripts/network_gpu_qualification.py new file mode 100644 index 0000000..487aa28 --- /dev/null +++ b/scripts/network_gpu_qualification.py @@ -0,0 +1,45 @@ +"""Source-host CLI for the installed qualification engine; no provider actions. + +The package contains the implementation. Qualification also needs the explicitly +staged source tree/profile/jobs; this wrapper alone is not an image deployment. +""" + +from hypertrain.gpu_ops.network_qualification import ( + Boundary as Boundary, +) +from hypertrain.gpu_ops.network_qualification import ( + Environment as Environment, +) +from hypertrain.gpu_ops.network_qualification import ( + Qualification as Qualification, +) +from hypertrain.gpu_ops.network_qualification import ( + Result as Result, +) +from hypertrain.gpu_ops.network_qualification import ( + bundle as bundle, +) +from hypertrain.gpu_ops.network_qualification import ( + capture as capture, +) +from hypertrain.gpu_ops.network_qualification import ( + check_inputs as check_inputs, +) +from hypertrain.gpu_ops.network_qualification import ( + compare as compare, +) +from hypertrain.gpu_ops.network_qualification import ( + inspect_environment as inspect_environment, +) +from hypertrain.gpu_ops.network_qualification import ( + main as main, +) +from hypertrain.gpu_ops.network_qualification import ( + run as run, +) +from hypertrain.gpu_ops.network_qualification import ( + sources as sources, +) + +if __name__ == "__main__": + main() diff --git a/scripts/network_gpu_seed.py b/scripts/network_gpu_seed.py new file mode 100644 index 0000000..95f70fb --- /dev/null +++ b/scripts/network_gpu_seed.py @@ -0,0 +1,523 @@ +"""Synthetic functional D2 decoder seed; no public-quality or CUDA observation claim.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import numpy as np +from pydantic import BaseModel, ConfigDict, Field +from tokenizers import Tokenizer, models, pre_tokenizers + +import hypertrain.trainer # noqa: F401 +from hypertrain.auditor.replay import AnchorCache, optimizer_hash, pack_state +from hypertrain.data.store import LocalFSStore +from hypertrain.data.tokenizer import HFTokenizer +from hypertrain.ledger.escrow_v2 import ( + authority_message, + genesis_allocation_hash, + genesis_origin_id, +) +from hypertrain.protocol import envelope_v2 +from hypertrain.protocol.example import example_manifest +from hypertrain.protocol.hashing import MerkleTree, sha256_hex +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.keys import Keypair +from hypertrain.protocol.messages import Receipt +from hypertrain.protocol.messages_v2 import ( + AdmissionPolicyV2, + AggregationPolicyV2, + ArtifactLimits, + AuditPolicyV2, + DisputePolicyV2, + EconomicsPolicyV2, + IslandJobV1, + OriginAllocation, + RunManifestV2, + StartStateV2, + TestGenesis, +) +from hypertrain.protocol.relay_messages import RelayRegistryV1 +from hypertrain.trainer.compress import state_hash +from hypertrain.trainer.config import TrainConfig +from hypertrain.trainer.model import init_params + +TREE = Path(__file__).resolve().parents[1] +ROLES = ("owner", "coord", "auditor-0", "auditor-1", "referee", "relay") + tuple( + f"{role}-{i}" for role in ("hot", "cold") for i in range(4) +) + + +class SeedIndex(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + run_id: str = Field(pattern=r"^[0-9a-f]{64}$") + hotkey: str + roles: dict[str, str] + claim: str + files: dict[str, str] = Field(min_length=1, max_length=256) + + +def generate( + destination: Path, + secrets: Path, + image: str, + drivers: list[str], + genesis_time: int, + deadline: int, +) -> str: + """Create pinned real inputs from explicit candidate runtime fields, never qualify them.""" + if destination.exists() or secrets.exists() or destination.resolve() == secrets.resolve(): + raise ValueError("new disjoint output/secret directories required") + if secrets.resolve().is_relative_to( + destination.resolve() + ) or destination.resolve().is_relative_to(secrets.resolve()): + raise ValueError("secrets must be separate from public output") + if genesis_time < 1 or deadline <= genesis_time or not drivers: + raise ValueError("explicit runtime clock and driver candidates required") + profile_raw = (TREE / "experiments/gpu_network_v2/profile.json").read_bytes() + profile = json.loads(profile_raw) + keys = { + role: Keypair(bytes.fromhex(sha256_hex(b"ht-d2-functional-seed-v1|" + role.encode()))) + for role in ROLES + } + allocations = [(keys[f"cold-{i}"].ss58, 10000, 1) for i in range(4)] + admission = AdmissionPolicyV2( + q_base="0000803e", + E=4, + clean_finalizations=12, + work_screen_epoch_rounds=1000, + max_pending=32, + max_trial_replays=2, + max_trials_per_beacon=2, + join_per_beacon=2, + join_burst=4, + ip_prefix_per_beacon=32, + ip_prefix_burst=64, + challenge_rounds=100, + artifact_limits=ArtifactLimits( + max_object_bytes=8000000, max_body_bytes=65536, max_chunk_manifest_bytes=1048576 + ), + ) + economics = EconomicsPolicyV2( + ledger_mode="test", + G_max_units=100, + R_collectible_units=100, + S_min_units=1000, + beta_ppm=1000000, + gammaV_units=0, + s_lower_ppm=1000000, + q_floor=1000000, + genesis_allocation_hash=genesis_allocation_hash(allocations), + reward_authority=keys["coord"].ss58, + round_reward_units=1000000, + max_total_issuance=20000000, + shadow_bootstrap_rounds=12, + ) + aggregation = AggregationPolicyV2( + arithmetic="flat-cclip-cap-v1", + order="utf8", + center="prev_outer_update", + trust_mode="uniform-verified", + weight_quantum=16777216, + miner_cap_units=4194304, + probation_cap_units=4194304, + owner_group_caps={}, + preclip_norm=profile["outer"].get("preclip_norm", "0000803f"), + cclip_tau="0000803f", + cclip_iters=1, + ) + audit = AuditPolicyV2( + lease_rounds=100, + max_attempts=2, + max_concurrent_per_auditor=1, + max_running_jobs=2, + max_queued_jobs=32, + max_anchor_age_rounds=1, + max_steps_per_attempt=60, + ) + dispute = DisputePolicyV2( + referees=[keys["referee"].ss58], + max_open=32, + max_transcript_entries=256, + max_entry_bytes=65536, + fanout=2, + max_referee_reassignments=1, + max_referee_cost_units=100, + ) + registry = RelayRegistryV1.model_validate( + { + "registry_version": 1, + "epoch": 0, + "previous_registry_hash": None, + "specs": [ + { + "id": "local", + "region": "local", + "https_url": "https://relay.test", + "pubkeys": [ + { + "key_id": "k1", + "pubkey": keys["relay"].ss58, + "valid_from_round": 0, + "valid_until_round": 10000, + } + ], + "max_object_bytes": 8000000, + "max_inflight_bytes": 32000000, + "codecs": ["ht-sparse-v1", "ht-dense-int8-v1"], + "mode": "transport", + } + ], + } + ) + eos = "<|endoftext|>" + tokenizer = Tokenizer( + models.WordLevel({**{f"t{i}": i for i in range(63)}, eos: 63}, unk_token=eos) + ) + tokenizer.pre_tokenizer = pre_tokenizers.WhitespaceSplit() + tokenizer_raw = tokenizer.to_str().encode() + rows = np.asarray([[(i * 17 + j) % 63 for j in range(17)] for i in range(4096)], dtype=" IslandJobV1: + """Check pinned inputs and original signatures; never run CUDA or train.""" + raw = (directory / "seed.json").read_bytes() + if len(raw) > 1 << 20 or sha256_hex(raw) != admitted_hash: + raise ValueError("seed index digest differs") + index = SeedIndex.model_validate_json(raw) + for name, digest in index.files.items(): + path = directory / name + if ( + Path(name).is_absolute() + or ".." in Path(name).parts + or path.is_symlink() + or path.stat().st_size > 4 << 20 + or sha256_hex(path.read_bytes()) != digest + ): + raise ValueError("seed file binding differs") + job = IslandJobV1.model_validate_json((directory / "job.json").read_bytes()) + profile = json.loads((directory / "profile.json").read_bytes()) + training = job.manifest.training.model_dump(mode="json") + if ( + profile["profile_version"] != 2 + or job.global_step0 != 0 + or training["reference_spec"]["layout"] != profile["layout"] + or not all( + all( + training[section].get(k) == v + for k, v in profile[section].items() + if k != "lr_schedule" + ) + for section in ("model", "inner", "outer") + ) + or not all( + training["inner"]["lr_schedule"].get(k) == v + for k, v in profile["inner"]["lr_schedule"].items() + ) + ): + raise ValueError("exact decoder profile differs") + if ( + index.run_id != job.run_id + or RunManifestV2.model_validate_json((directory / "manifest.json").read_bytes()) + != job.manifest + ): + raise ValueError("manifest/job binding differs") + for name, digest in ( + ("start_state", job.start_state_sha256), + ("ef_in", job.ef_in_sha256), + ("v0", job.v0_sha256), + ): + if sha256_hex((directory / name).read_bytes()) != digest: + raise ValueError("job input digest differs") + tokenizer = HFTokenizer( + directory / "tokenizer.json", + job.manifest.training.tokenizer.sha256, + job.manifest.training.tokenizer.name, + ) + if tokenizer.vocab_size != 64 or tokenizer.encode("t0 t62") != [0, 62]: + raise ValueError("synthetic tokenizer differs") + theta = init_params(TrainConfig.from_manifest_v2(job.manifest).model) + anchor = AnchorCache().genesis(job.manifest, index.hotkey, theta) + start = StartStateV2.model_validate_json((directory / "start.json").read_bytes()) + if ( + start.run_id != job.run_id + or start.hotkey != index.hotkey + or start.w != job.w + or start.theta_hash != state_hash(theta) + or start.opt_state_hash != optimizer_hash(anchor.state) + or start.state_object_sha256 != job.start_state_sha256 + or start.ef_object_sha256 != job.ef_in_sha256 + or start.parent_anchor_hash != anchor.anchor_hash + or start.anchor_verdict_hash != anchor.proof_hash + or start.global_step0 != 0 + or start.ef_hash != state_hash(anchor.ef) + ): + raise ValueError("full genesis descriptor differs") + reference = json.loads((directory / "reference.json").read_bytes()) + if reference["profile_sha256"] != sha256_hex( + (directory / "profile.json").read_bytes() + ) or job.manifest.training.reference_spec.spec_doc_sha256 != sha256_hex( + (directory / "reference.json").read_bytes() + ): + raise ValueError("reference/profile pin differs") + expected = { + "start_state": pack_state(theta, anchor.state), + "ef_in": pack_state(anchor.ef), + "v0": pack_state({}), + } + for name, value in expected.items(): + if (directory / name).read_bytes() != value: + raise ValueError("genesis optimizer/EF/v0 differs") + for name, signer in ( + ("manifest-envelope.json", "owner"), + ("genesis-envelope.json", "coord"), + ("reference-envelope.json", "coord"), + ("job-envelope.json", "coord"), + ): + envelope = envelope_v2.parse_envelope((directory / name).read_bytes()) + if ( + envelope.run_id != job.run_id + or envelope.signer != index.roles[signer] + or not envelope_v2.verify_envelope(envelope.model_dump()) + ): + raise ValueError("seed authority signature differs") + if name == "manifest-envelope.json" and envelope.body != job.manifest.body(): + raise ValueError("signed manifest body differs") + if name in ("reference-envelope.json", "job-envelope.json"): + expected_digest = ( + sha256_hex((directory / "reference.json").read_bytes()) + if name.startswith("reference") + else job.digest() + ) + if envelope.type != "Receipt" or envelope.body["commit_hash"] != expected_digest: + raise ValueError("signed descriptor digest differs") + if name == "genesis-envelope.json": + genesis = TestGenesis.model_validate(envelope.body) + from hypertrain.protocol.keys import decode_hotkey, verify + + if not verify( + decode_hotkey(envelope.signer), + authority_message(genesis), + bytes.fromhex(genesis.authority_sig), + ): + raise ValueError("signed genesis authority differs") + proofs = json.loads((directory / "sample_proofs").read_bytes()) + all_rows = np.frombuffer((directory / "samples.u32").read_bytes(), "= 64) + ): + raise ValueError("full dataset root/range differs") + rows = np.frombuffer((directory / "samples").read_bytes(), " None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--destination", type=Path, required=True) + parser.add_argument("--secrets", type=Path, required=True) + parser.add_argument("--image-digest", required=True) + parser.add_argument("--driver", action="append", required=True) + parser.add_argument("--genesis-unix", type=int, required=True) + parser.add_argument("--deadline-unix", type=int, required=True) + args = parser.parse_args() + print( + generate( + args.destination, + args.secrets, + args.image_digest, + args.driver, + args.genesis_unix, + args.deadline_unix, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/network_service_proof.py b/scripts/network_service_proof.py new file mode 100644 index 0000000..cab8c28 --- /dev/null +++ b/scripts/network_service_proof.py @@ -0,0 +1,2488 @@ +"""Bounded D3 CPU service proof. Requires admitted authority and read-only service journal.""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import re +import shutil +import signal +import socket +import sqlite3 +import ssl +import subprocess +import sys +import threading +import time +from collections.abc import Callable, Collection, Mapping +from contextlib import closing +from pathlib import Path +from types import FrameType, ModuleType +from typing import TYPE_CHECKING, Literal +from urllib.parse import urlsplit + +import httpx +import uvicorn +from pydantic import BaseModel, ConfigDict, Field, JsonValue, StrictInt, field_validator + +import hypertrain.trainer # noqa: F401 +import network_authority_snapshot as authority +from hypertrain.aggregator.checkpoint import ( + network_inputs, + verify_network_checkpoint, + write_network_checkpoint, +) +from hypertrain.aggregator.core import load_state +from hypertrain.aggregator.tape_v2 import TapeV2, replay_tape +from hypertrain.auditor.replay import AnchorCache, optimizer_hash, pack_state +from hypertrain.data.store import LocalFSStore +from hypertrain.gpu_ops.journal import Journal, durable_write, fsha +from hypertrain.gpu_ops.work_screen import IslandLaunch +from hypertrain.miner.admission import sign_join +from hypertrain.miner.core import load_keyfile +from hypertrain.miner.island_launch import IslandArtifacts +from hypertrain.protocol import envelope_v2, relay_envelope +from hypertrain.protocol.hashing import sha256_hex +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.keys import Keypair +from hypertrain.protocol.messages import Finalize, f32hex +from hypertrain.protocol.messages_v2 import ( + AggregationPolicyV2, + CommitV2, + EscrowLock, + HardwareHint, + IslandJobV1, + JoinChallenge, + PolicyHashes, + RosterEntryV2, + RoundOpenV2, + RunManifestV2, + StartStateV2, + WorkProof, + WorkScreenV2, +) +from hypertrain.protocol.relay_messages import RelayRegistryV1 +from hypertrain.trainer.compress import state_hash +from hypertrain.trainer.config import TrainConfig +from hypertrain.trainer.model import init_params + +if TYPE_CHECKING: + from experiments.gpu_network_v2.orchestrate import NetworkRuntime + + from hypertrain.challenge.store import ChallengeStore + + +class DriverError(RuntimeError): + pass + + +class Settings(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + master_https: str + relay_https: str + ca: Path + snapshot: Path + snapshot_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + secrets: Path + service_state: Path + evidence: Path + verified_beacon: Path + fixture_clock_directory: Path + fixture_clock_offset: float = Field(allow_inf_nan=False) + rounds: int = Field(default=2, strict=True, ge=1, le=2) + child_timeout: int = Field(default=600, strict=True, ge=1, le=900) + successor_registry: Path | None = None + + @field_validator("master_https", "relay_https") + @classmethod + def https(cls, value: str) -> str: + url = urlsplit(value) + if url.scheme != "https" or not url.hostname or url.username or url.query or url.fragment: + raise ValueError("explicit HTTPS endpoint without credentials/query required") + return value.rstrip("/") + + +def record(state: Path, kind: str, identity: str, run_id: str = authority.RUN) -> dict: + """Read accepted service authority, never write admission/ledger rows.""" + with closing(sqlite3.connect((state / "challenge.db").as_uri() + "?mode=ro", uri=True)) as db: + row = db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind=? AND id=?", + (run_id, kind, identity), + ).fetchone() + if row is None: + raise DriverError(f"missing accepted {kind}:{identity}") + return json.loads(row[0]) + + +def durable_destination(path: Path) -> bool: + return not path.exists() and not any("pytest" in part for part in path.resolve().parts) + + +def run_child( + argv: list[str], + env: dict[str, str], + log: Path, + cancel: threading.Event, + timeout: int, +) -> None: + """Event subscription precedes trigger; cancellation kills/reaps entire child group.""" + if cancel.is_set(): + raise DriverError("cancelled before miner") + done = threading.Event() + condition = vars(cancel)["_cond"] + with log.open("xb") as output: + process = subprocess.Popen( # noqa: S603 - caller constructs fixed Python product CLI + argv, env=env, stdout=output, stderr=output, start_new_session=True + ) + + def kill() -> None: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + + def watch() -> None: + with condition: + condition.wait_for(lambda: done.is_set() or cancel.is_set(), timeout=timeout) + if not done.is_set(): + kill() + + watcher = threading.Thread(target=watch) + watcher.start() + try: + code = process.wait(timeout=timeout) + except subprocess.TimeoutExpired as error: + kill() + process.wait() + raise DriverError("miner hard deadline") from error + finally: + done.set() + with condition: + condition.notify_all() + watcher.join() + kill() + if cancel.is_set() or code: + raise DriverError(f"miner cancelled/exit {code}; see {log}") + + +def miner_command(w: int) -> list[str]: + """Use shipped actual CLI, never fixture training or an alternate miner.""" + return [sys.executable, "-m", "hypertrain.miner.cli", "run-v2", "--round", str(w)] + + +def bound_clock(settings: Settings, bundle: authority.Bundle, manifest: RunManifestV2) -> float: + """Consume bootstrap runtime, not an independently selected child clock origin.""" + runtime = json.loads((settings.service_state / "runtime.json").read_bytes()) + offset = runtime["fixture_clock_offset"] + if type(offset) not in (float, int) or not math.isfinite(offset): + raise DriverError("nonfinite bootstrap clock offset") + beacon = authority.fixture_beacon(runtime["beacon"]) + supplied = authority.fixture_beacon(json.loads(settings.verified_beacon.read_bytes())) + if beacon != supplied or beacon.round != bundle.beacon_round: + raise DriverError("bootstrap clock original beacon mismatch") + if offset != settings.fixture_clock_offset: + raise DriverError("child clock offset differs from bootstrap runtime") + current = time.time() - offset + historical = manifest.training.beacon.genesis_time + (beacon.round - 1) * 3 + if current < historical: + raise DriverError("bootstrap clock precedes original historical origin") + return current - time.monotonic() + + +def prepare(settings: Settings) -> tuple[authority.Bundle, RunManifestV2]: + """Validate admitted portable bytes and role identities before any live write.""" + bundle = authority.validate(settings.snapshot, settings.snapshot_digest) + if not durable_destination(settings.evidence): + raise DriverError("fresh durable evidence outside pytest retention required") + if not (settings.service_state / "challenge.db").is_file(): + raise DriverError("MISSING_ADAPTER: mount verified bootstrap service journal read-only") + restored = authority._authority(settings.service_state) + if ( + restored.public_roles != bundle.public_roles + or restored.manifest_envelope_sha256 != bundle.manifest_envelope_sha256 + or restored.genesis_envelope_sha256 != bundle.genesis_envelope_sha256 + ): + raise DriverError("restored original authority differs from admitted package") + if settings.secrets.is_symlink() or settings.secrets.stat().st_mode & 0o077: + raise DriverError("private role secret directory required") + for role, public in bundle.public_roles.items(): + if role == "coord" or role.startswith(("hot-", "auditor-")): + if load_keyfile(settings.secrets / f"{role}.seed").ss58 != public: + raise DriverError(f"wrong role key {role}") + token = settings.secrets / "admin.token" + if ( + token.is_symlink() + or not token.is_file() + or token.stat().st_mode & 0o077 + or token.stat().st_uid != os.getuid() + or not token.read_bytes().strip() + ): + raise DriverError("private admin token required") + with closing( + sqlite3.connect((settings.service_state / "challenge.db").as_uri() + "?mode=ro", uri=True) + ) as db: + manifest = RunManifestV2.model_validate_json( + db.execute("SELECT manifest FROM runs").fetchone()[0] + ) + if db.execute("SELECT COUNT(*) FROM records_v2 WHERE kind='round'").fetchone()[0]: + raise DriverError("fresh restored-admission service required; no implicit resume") + if manifest.run_id() != bundle.run_id: + raise DriverError("service snapshot run mismatch") + monotonic_origin = bound_clock(settings, bundle, manifest) + if not (settings.fixture_clock_directory / "sitecustomize.py").is_file(): + raise DriverError("MISSING_ADAPTER: explicit bootstrap child fixture clock required") + expected_clock = settings.service_state / "fixture-clock/sitecustomize.py" + if ( + settings.fixture_clock_directory / "sitecustomize.py" + ).resolve() != expected_clock.resolve(): + raise DriverError("child clock must be bootstrap-owned code") + ssl.create_default_context(cafile=str(settings.ca)) + settings.evidence.mkdir(parents=True, mode=0o700) + child_clock = settings.evidence / "child-clock" + child_clock.mkdir() + (child_clock / "sitecustomize.py").write_text( + f"import time\ntime.time=lambda:time.monotonic()+{monotonic_origin!r}\n" + ) + sources = settings.evidence / "source" + sources.mkdir() + for path in (Path(__file__), Path(authority.__file__)): + shutil.copyfile(path, sources / path.name) + with ( + closing( + sqlite3.connect( + (settings.service_state / "challenge.db").as_uri() + "?mode=ro", + uri=True, + ) + ) as src, + closing(sqlite3.connect(settings.evidence / "initial-service.db")) as dst, + ): + src.backup(dst) + return bundle, manifest + + +def proof(settings: Settings, cancel: threading.Event) -> None: + import torch + + bundle, manifest = prepare(settings) + coord = load_keyfile(settings.secrets / "coord.seed") + objects = LocalFSStore(settings.service_state / "objects") + root = f"/v2/runs/{bundle.run_id}" + admin = {"Authorization": "Bearer " + (settings.secrets / "admin.token").read_text().strip()} + cfg = TrainConfig.from_manifest_v2(manifest) + rounds: list[dict] = [] + grants: list[str] = [] + with httpx.Client( + base_url=settings.master_https, + verify=ssl.create_default_context(cafile=str(settings.ca)), + timeout=settings.child_timeout, + follow_redirects=False, + ) as client: + + def call( + method: str, path: str, body: dict | None = None, privileged: bool = False + ) -> dict: + if cancel.is_set(): + raise DriverError("proof cancelled") + response = client.request(method, path, json=body, headers=admin if privileged else {}) + response.raise_for_status() + value = response.json() + with (settings.evidence / "http.jsonl").open("ab") as log: + log.write(canonicalize({"method": method, "path": path, "response": value}) + b"\n") + log.flush() + os.fsync(log.fileno()) + return value + + status = call("GET", root) + signed_manifest = status["manifest_envelope"] + if not envelope_v2.verify_envelope(signed_manifest) or ( + signed_manifest["signer"] != bundle.public_roles["owner"] + or signed_manifest["body"] != manifest.body() + ): + raise DriverError("live manifest authority mismatch") + beacon = json.loads(settings.verified_beacon.read_bytes()) + current = authority.fixture_beacon(beacon) + if status["now_round"] != current.round or current.round != bundle.beacon_round: + raise DriverError("live/verified/restored beacon mismatch") + + def advance(number: int) -> None: + from hypertrain.beacon.core import FixtureBeacon + + b = FixtureBeacon(current=number).get(number) + payload: dict[str, int | str] = { + "round": number, + "signature": b.signature, + "randomness": b.randomness, + } + authority.fixture_beacon(payload) + call("POST", "/v1/admin/beacon", payload, True) + + registry = RelayRegistryV1.model_validate_json( + objects.get(manifest.network.relay_registry_hash) + ) + if settings.successor_registry is not None: + successor = json.loads(settings.successor_registry.read_bytes()) + env = relay_envelope.parse_envelope(successor) + if ( + env.signer != coord.ss58 + or env.run_id != bundle.run_id + or not relay_envelope.verify_envelope(successor) + ): + raise DriverError("successor registry authority mismatch") + call("POST", root + "/admin/relay-registry", successor, True) + accepted = record(settings.service_state, "registry", "current") + if accepted != successor: + raise DriverError("accepted successor differs") + registry = RelayRegistryV1.model_validate(accepted["body"]) + if settings.relay_https not in {spec.https_url.rstrip("/") for spec in registry.specs}: + raise DriverError("relay endpoint not pinned; accepted successor adapter required") + for w in range(settings.rounds): + now = call("GET", root)["now_round"] + theta = init_params(cfg.model) + predecessor = "0" * 64 + if w: + applied = record(settings.service_state, "applied", str(w - 1)) + theta = { + k: torch.from_numpy(v.copy()) + for k, v in load_state(objects, applied["out_state"]).theta.items() + } + predecessor = applied["tape_hash"] + starts, roster = [], [] + for i in range(4): + hot = bundle.public_roles[f"hot-{i}"] + admission = call("GET", root + "/admission/" + hot) + if not admission["eligible"] or admission["record"]["state"] != "ACTIVE": + raise DriverError("live admission no longer ACTIVE/funded") + cache = AnchorCache() + if w: + a = record(settings.service_state, "anchor", f"{w - 1}:{hot}") + anchor = cache.restore( + Path(a["path"]), + manifest, + a["anchor_hash"], + a["proof_hash"], + a["state_root"], + a["ef_hash"], + expected_hotkey=hot, + expected_round=w - 1, + expected_backend=a["backend"], + ) + else: + anchor = cache.genesis(manifest, hot, theta) + if anchor.state.step != w * cfg.inner.H: + raise DriverError("carried optimizer step mismatch") + starts.append( + StartStateV2( + run_id=bundle.run_id, + w=w, + hotkey=hot, + theta_hash=state_hash(theta), + state_object_sha256=sha256_hex(pack_state(theta, anchor.state)), + opt_state_hash=optimizer_hash(anchor.state), + ef_object_sha256=sha256_hex(pack_state(anchor.ef)), + ef_hash=state_hash(anchor.ef), + parent_anchor_hash=anchor.anchor_hash, + global_step0=anchor.state.step, + anchor_verdict_hash=anchor.proof_hash, + ) + ) + roster.append( + { + "hotkey": hot, + "slot": i, + "q_i": f32hex(1), + "admission_id": admission["record"]["admission_id"], + "coldkey_group": admission["record"]["coldkey"], + "state": "ACTIVE", + "eligible_weight": 4194304, + } + ) + opening = RoundOpenV2( + w=w, + prev_final_hash=sha256_hex(canonicalize(rounds[-1]["finalize"]["body"])) + if w + else "0" * 64, + theta_hash=state_hash(theta), + outer_state_hash="0" * 64, + center_hash="0" * 64, + roster_hash=sha256_hex(canonicalize(roster)), + honeypot_commit="0" * 64, + d_open=now + 1, + d_assign=now + 2, + d_commit=now + 100, + d_audit=now + 101, + d_upload=now + 280, + d_final=now + 300, + contract_version=2, + policy_hashes=PolicyHashes.model_validate( + {k: getattr(manifest.network, k) for k in PolicyHashes.model_fields} + ), + registry_epoch=registry.epoch, + start_state_index_hash=sha256_hex(canonicalize([s.body() for s in starts])), + audit_mode="anchored-full", + roster=[RosterEntryV2.model_validate(row) for row in roster], + ) + call( + "POST", + root + "/admin/rounds", + envelope_v2.seal(coord, "RoundOpenV2", bundle.run_id, opening, opening.d_final), + True, + ) + advance(opening.d_assign) + call("GET", root + f"/rounds/{w}") + for i in range(4): + child_env = { + k: v for k, v in os.environ.items() if not k.startswith("HYPERTRAIN_MINER_") + } + child_env.update( + SSL_CERT_FILE=str(settings.ca), + OMP_NUM_THREADS="1", + MKL_NUM_THREADS="1", + HT_FIXTURE_CLOCK_OFFSET=str(settings.fixture_clock_offset), + PYTHONPATH=str(settings.evidence / "child-clock") + + os.pathsep + + child_env.get("PYTHONPATH", ""), + HYPERTRAIN_MINER_API=settings.master_https, + HYPERTRAIN_MINER_KEYFILE=str(settings.secrets / f"hot-{i}.seed"), + HYPERTRAIN_MINER_WORKDIR=str(settings.evidence / "miners"), + HYPERTRAIN_MINER_STATE_SOURCE="cpu", + HYPERTRAIN_MINER_DEVICE="cpu", + HYPERTRAIN_MINER_IMAGE_DIGEST=manifest.training.reference_spec.image_digest, + HYPERTRAIN_MINER_RUN_ID=bundle.run_id, + HYPERTRAIN_MINER_OWNER_HOTKEY=bundle.public_roles["owner"], + ) + run_child( + miner_command(w), + child_env, + settings.evidence / f"miner-{w}-{i}.log", + cancel, + settings.child_timeout, + ) + delta = record( + settings.service_state, + "delta", + f"{w}:" + bundle.public_roles[f"hot-{i}"], + ) + grants.append(delta["body"]["grant_hash"]) + advance(opening.d_audit) + call("POST", root + f"/admin/rounds/{w}/audits", privileged=True) + for i in range(4): + auditor = load_keyfile(settings.secrets / f"auditor-{i % 2}.seed") + now = call("GET", root)["now_round"] + request = { + "w": w, + "commit_hash": f"{w * 4 + i + 1:064x}", + "received_round": now, + } + lease = call( + "POST", + root + "/worker/lease", + envelope_v2.seal(auditor, "Receipt", bundle.run_id, request, opening.d_final), + ) + if not envelope_v2.verify_envelope(lease) or lease["signer"] != coord.ss58: + raise DriverError("audit lease authority mismatch") + job = lease["body"] + request["commit_hash"] = job["lease_nonce"] + execution = call( + "POST", + root + "/worker/jobs/" + job["job_id"] + "/execute", + envelope_v2.seal(auditor, "Receipt", bundle.run_id, request, opening.d_final), + ) + if execution["verdict"]["result"] != "MATCH": + raise DriverError("independent inner audit did not MATCH") + call( + "POST", + root + "/worker/jobs/" + job["job_id"] + "/complete", + envelope_v2.seal( + auditor, + "ReplayVerdict", + bundle.run_id, + execution["verdict"], + opening.d_final, + ), + ) + applied = call("POST", root + f"/admin/rounds/{w}/aggregate", privileged=True) + tape = TapeV2.from_bytes(objects.get(applied["tape_hash"])) + if ( + len(tape.body.allocation.entries) != 4 + or sum(e.weight_units for e in tape.body.allocation.entries) != 16777216 + ): + raise DriverError("four identity weights do not sum Q") + if any(e.probation or e.weight_units != 4194304 for e in tape.body.allocation.entries): + raise DriverError("shadow/probation influence or unequal four identity weights") + if {e.hotkey for e in tape.body.allocation.entries} != {r["hotkey"] for r in roster}: + raise DriverError("allocation identities differ from actual four-miner roster") + inputs = record(settings.service_state, "tape-inputs", str(w))["inputs"] + if any( + x["commit"]["tokens"] != manifest.training.batch_samples() * cfg.model.seq_len + for x in inputs + ): + raise DriverError("N2 token entitlement differs") + replayed = replay_tape( + objects, + tape, + manifest, + AggregationPolicyV2.model_validate_json( + objects.get(manifest.network.aggregation_policy_hash) + ), + objects.get(manifest.network.economics_policy_hash), + signer=coord.ss58, + w=w, + prev_state=applied["prev_state"], + predecessor_tape_hash=predecessor, + inputs=network_inputs(inputs), + reference_reward_units=100, + ) + if replayed.to_bytes() != objects.get(applied["out_state"]): + raise DriverError("independent outer tape state differs") + final = Finalize( + w=w, + final_theta_hash_w1=applied["theta_hash"], + included=sorted(e["hotkey"] for e in roster), + entitlements_root="0" * 64, + ) + call( + "POST", + root + f"/admin/rounds/{w}/finalize", + envelope_v2.seal(coord, "Finalize", bundle.run_id, final, opening.d_final), + True, + ) + settled = record(settings.service_state, "finality", str(w)) + if ( + settled["disposition"] != "SETTLED" + or settled["unresolved_audits"] + or settled["unresolved_disputes"] + ): + raise DriverError("service finality not independently settled") + rounds.append( + { + "round_open": record(settings.service_state, "round", str(w)), + "finalize": record(settings.service_state, "finalize", str(w)), + "inputs": inputs, + "tape_hash": applied["tape_hash"], + "prev_state": applied["prev_state"], + "predecessor_tape_hash": predecessor, + "reference_reward_units": 100, + } + ) + write_network_checkpoint(settings.evidence / "checkpoint", coord, objects, manifest, rounds) + if verify_network_checkpoint(settings.evidence / "checkpoint", coord.ss58): + raise DriverError("independent outer checkpoint replay failed") + for grant in grants: + call("POST", root + f"/admin/relay/{grant}/extend", privileged=True) + horizons = [ + call("GET", root + "/admin/relay-settlement/" + g, privileged=True) for g in grants + ] + advance( + max( + max(h["vesting_beacon"], h["closed_beacon"], h["finality_beacon"]) + 100 + for h in horizons + ) + ) + for grant in grants: + released = call("POST", root + f"/admin/relay/{grant}/release", privileged=True) + if ( + released["type"] != "CustodyRelease" + or released["run_id"] != bundle.run_id + or released["signer"] != coord.ss58 + or not relay_envelope.verify_envelope(released) + ): + raise DriverError("accepted custody release signature differs") + + +def durable_json(path: Path, value: dict) -> None: + """Atomic, fsynced publication; never expose a partially written verdict.""" + temporary = path.with_suffix(".pending") + with temporary.open("wb") as stream: + stream.write(canonicalize(value)) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + directory = os.open(path.parent, os.O_DIRECTORY) + try: + os.fsync(directory) + finally: + os.close(directory) + + +def finalize_evidence(settings: Settings, completed: bool, failure: str | None) -> None: + """FAIL first, rescue independently, commit PASS only after verified durable index.""" + root = settings.evidence + errors = [] + result = { + "status": "FAIL", + "run_id": authority.RUN, + "rounds_requested": settings.rounds, + "inner_audits_expected": 4 * settings.rounds, + "outer_checkpoint_replay": completed, + "snapshot_digest": settings.snapshot_digest, + "missing_provenance": authority.MISSING, + "gpu_claim": False, + "workload_error": failure, + "export_complete": False, + } + durable_json(root / "result.json", result) + destination = root / "service-export" + try: + destination.mkdir(exist_ok=True) + with ( + closing( + sqlite3.connect( + (settings.service_state / "challenge.db").as_uri() + "?mode=ro", + uri=True, + ) + ) as src, + closing(sqlite3.connect(destination / "challenge.db")) as dst, + ): + src.backup(dst) + except (OSError, sqlite3.Error) as error: + errors.append(f"database export: {error}") + for name in ( + "objects", + "trials-v2", + "anchors-v2", + "jobs-v2", + "audit-v2", + "audit-geometry-v2", + "ledger", + ): + source = settings.service_state / name + try: + if source.exists(): + shutil.copytree(source, destination / name, dirs_exist_ok=True) + except OSError as error: + errors.append(f"{name} export: {error}") + files = {} + for path in sorted(root.rglob("*")): + if path in (root / "index.json", root / "result.json") or path.suffix == ".pending": + continue + try: + if path.is_file(): + data = path.read_bytes() + with path.open("rb") as stream: + os.fsync(stream.fileno()) + files[path.relative_to(root).as_posix()] = { + "sha256": sha256_hex(data), + "size": len(data), + } + except OSError as error: + errors.append(f"index {path.relative_to(root)}: {error}") + index = {"files": files, "errors": errors, "export_complete": not errors} + index_hash = None + try: + durable_json(root / "index.json", index) + raw = (root / "index.json").read_bytes() + if raw != canonicalize(index): + raise DriverError("published index differs") + index_hash = sha256_hex(raw) + except (OSError, DriverError) as error: + errors.append(f"index publication: {error}") + # A censored failure index is rescue evidence, never a complete custody claim. + try: + durable_json( + root / "index.json", + { + "files": files, + "errors": errors, + "export_complete": False, + }, + ) + index_hash = sha256_hex((root / "index.json").read_bytes()) + except OSError as rescue_error: + errors.append(f"censored index publication: {rescue_error}") + result.update( + status="PASS" if completed and not errors else "FAIL", + export_complete=not errors, + errors=errors, + index_sha256=index_hash, + ) + durable_json(root / "result.json", result) + if errors: + raise DriverError("incomplete evidence export: " + "; ".join(errors)) + + +def decoder_join_lock( + client: httpx.Client, + manifest: RunManifestV2, + hot: Keypair, + cold: Keypair, + origins: list[str], + units: int, + request_id: str, + operation_id: str, + expiry: int, +) -> str: + """Ordinary dual-signed join then conserved admission lock; no state writes.""" + root = f"/v2/runs/{manifest.run_id()}" + request = sign_join( + hot, + cold, + run_id=manifest.run_id(), + request_id=request_id, + expires_beacon=expiry, + policy_hash=manifest.network.admission_policy_hash, + hardware_hint=HardwareHint( + device_name="advisory", + device_count=manifest.training.reference_spec.layout.n_gpus, + driver="not-qualification", + ), + ) + response = client.post(root + "/join", content=canonicalize(request.body())) + response.raise_for_status() + admission_id = response.json()["admission_id"] + lock = EscrowLock( + operation_id=operation_id, + owner=cold.ss58, + units=units, + origin_ids=origins, + admission_id=admission_id, + dispute_id=None, + kind="LOCK_ADMISSION", + ) + response = client.post( + root + "/escrow/lock", + content=canonicalize(envelope_v2.seal(cold, "EscrowLock", manifest.run_id(), lock, expiry)), + ) + response.raise_for_status() + return admission_id + + +def decoder_operation( + client: httpx.Client, + runtime: NetworkRuntime, + operation: dict, + receipt: dict, + hot: Keypair, + pins: dict, + artifacts: Path, + cancel: threading.Event, + finalize: dict | None = None, +) -> dict: + """Trusted Runtime dispatch; only probe/live API producers, never raw island PASS.""" + if TYPE_CHECKING: + from scripts.network_gpu_operation import Operation + else: + from network_gpu_operation import Operation + + spec = Operation.model_validate(operation) + if cancel.is_set() or spec.hotkey != hot.ss58 or spec.operation not in ("probe", "live"): + raise DriverError("decoder operation identity/kind/cancellation differs") + if spec.owner != pins["owner"] or spec.job.run_id != pins["run_id"]: + raise DriverError("decoder trusted owner/run differs") + root = f"/v2/runs/{spec.job.run_id}" + if spec.operation == "probe": + challenge_response = client.get(root + "/join/" + spec.binding + "/challenge") + challenge_response.raise_for_status() + signed_challenge = envelope_v2.parse_envelope(challenge_response.json()) + if ( + signed_challenge.type != "JoinChallenge" + or signed_challenge.signer != spec.job.manifest.training.coord_pubkey + or not envelope_v2.verify_envelope(signed_challenge.model_dump()) + ): + raise DriverError("decoder signed trial authority differs") + challenge = JoinChallenge.model_validate(signed_challenge.body) + if spec.binding != challenge.admission_id or spec.job.run_id != challenge.manifest_hash: + raise DriverError("decoder trial operation binding differs") + else: + round_response = client.get(root + f"/rounds/{spec.job.w}") + round_response.raise_for_status() + assignments = round_response.json()["assignment"] + if not any(a["hotkey"] == hot.ss58 for a in assignments): + raise DriverError("decoder live assignment identity differs") + runtime.accept_operation( + spec, canonicalize(receipt), owner=pins["owner"], beacon=pins["beacon"] + ) + if spec.operation == "probe": + reference = client.post( + root + "/admin/join/" + challenge.admission_id + "/reference", + headers=pins["admin_headers"], + ) + # Service callback debits and executes the independent reference. + reference.raise_for_status() + result = runtime.operation(spec, artifacts, cancel=cancel) + if result.get("status") != "CAPTURED_NOT_ACCEPTED": + raise DriverError("API operation producer incomplete") + api = result["cli_outcome"] + if spec.operation == "live": + if api != {"status": "UPLOADED"}: + raise DriverError("actual API miner upload absent") + if any( + result.get(k) != v + for k, v in { + "operation": "live", + "binding": spec.binding, + "hotkey": hot.ss58, + "role": spec.role, + "job_sha256": spec.job.digest(), + }.items() + ): + raise DriverError("live original artifact binding differs") + signed = result["signed_api"] + kinds = ["AcceptV2", "CommitV2", "WorkProof", "DeltaManifestV2"] + if [x["type"] for x in signed] != kinds: + raise DriverError("actual signed live API submissions absent") + accepted = result["accepted_api"] + if [entry["request"] for entry in accepted] != signed: + raise DriverError("live original API acceptance absent") + commit_env = envelope_v2.parse_envelope(accepted[1]["request"]) + commit = CommitV2.model_validate(commit_env.body) + commit.validate_assignment(spec.job.manifest, len(spec.job.sample_ids)) + for raw in signed: + env = envelope_v2.parse_envelope(raw) + if ( + env.signer != hot.ss58 + or env.run_id != spec.job.run_id + or not envelope_v2.verify_envelope(raw) + ): + raise DriverError("live signed identity differs") + match env.type: + case "WorkProof": + live_proof = WorkProof.model_validate(env.body) + if ( + live_proof.challenge_hash != sha256_hex(canonicalize(commit_env.body)) + or live_proof.leaves_root != commit.leaves_root + or live_proof.delta_hash != commit.delta_hash + ): + raise DriverError("live WorkProof accepted commit differs") + case "AcceptV2" | "CommitV2" | "DeltaManifestV2": + if env.body["w"] != spec.job.w or env.body["hotkey"] != hot.ss58: + raise DriverError("live signed identity differs") + return result + proof = envelope_v2.parse_envelope(api["proof"]) + screen = envelope_v2.parse_envelope(api["screen"]) + if any( + e.signer != hot.ss58 + or e.run_id != spec.job.run_id + or not envelope_v2.verify_envelope(e.model_dump()) + for e in (proof, screen) + ): + raise DriverError("decoder probe signer differs") + if proof.type != "WorkProof" or screen.type != "WorkScreenV2": + raise DriverError("decoder probe message types differ") + p, s = ( + WorkProof.model_validate(proof.body), + WorkScreenV2.model_validate(screen.body), + ) + if p.challenge_hash != challenge.digest() or s.challenge_hash != challenge.digest(): + raise DriverError("decoder probe challenge differs") + response = client.post( + root + "/join/proof", json={"proof": api["proof"], "screen": api["screen"]} + ) + response.raise_for_status() + if finalize is None: + return response.json() # Proof verified, not finalized or graduated. + signed_final = envelope_v2.parse_envelope(finalize) + if ( + signed_final.type != "Finalize" + or signed_final.signer != spec.job.manifest.training.coord_pubkey + or signed_final.run_id != spec.job.run_id + or signed_final.body["w"] != spec.job.w + or signed_final.body["included"] != [hot.ss58] + or not envelope_v2.verify_envelope(finalize) + ): + raise DriverError("trial finality coordinator authority differs") + response = client.post( + root + "/admin/join/" + challenge.admission_id + "/finalize", + content=canonicalize(finalize), + headers=pins["admin_headers"], + ) + response.raise_for_status() + if response.json()["state"] not in ("PROBATION", "ACTIVE"): + raise DriverError("decoder trial not finalized") + return response.json() + + +def decoder_live_graph( + client: httpx.Client, + runtime: NetworkRuntime, + manifest: RunManifestV2, + rounds: list[dict | Callable[[], dict]], + coordinator: Keypair, + admin: dict[str, str], + advance: Callable[[int], None], + objects: LocalFSStore, + checkpoint: Path, + cancel: threading.Event, +) -> list[dict]: + """One/two ordinary live rounds from root-reviewed opening/operation contexts.""" + if not 1 <= len(rounds) <= 2: + raise DriverError("decoder graph requires one or two complete rounds") + run_id = manifest.run_id() + lineage: list[dict] = [] + root = f"/v2/runs/{run_id}" + + def call(method: str, path: str, body: dict | None = None, privileged: bool = False) -> dict: + if cancel.is_set(): + raise DriverError("decoder live graph cancelled") + response = client.request( + method, root + path, json=body, headers=admin if privileged else {} + ) + response.raise_for_status() + return response.json() + + for round_entry in rounds: + context = round_entry() if callable(round_entry) else round_entry + opening = envelope_v2.parse_envelope(context["round_open"]) + body = RoundOpenV2.model_validate(opening.body) + if ( + opening.run_id != run_id + or opening.signer != coordinator.ss58 + or coordinator.ss58 != manifest.training.coord_pubkey + or not envelope_v2.verify_envelope(context["round_open"]) + ): + raise DriverError("decoder opening coordinator/run authority differs") + if len(body.roster) != 4 or ( + not callable(context["miners"]) and len(context["miners"]) != 4 + ): + raise DriverError("decoder complete four-identity roster required") + call("POST", "/admin/rounds", context["round_open"], True) + advance(body.d_assign) + miners = context["miners"]() if callable(context["miners"]) else context["miners"] + if len(miners) != 4: + raise DriverError("decoder complete assigned miner roster required") + for miner in miners: + decoder_operation( + client, + runtime, + miner["operation"], + miner["receipt"], + miner["key"], + miner["pins"], + miner["artifacts"], + cancel, + ) + advance(body.d_audit) + call("POST", f"/admin/rounds/{body.w}/audits", privileged=True) + for i, auditor in enumerate(context["auditors"]): + request: dict[str, JsonValue] = { + "w": body.w, + "commit_hash": f"{body.w * 4 + i + 1:064x}", + "received_round": body.d_audit, + } + lease = call( + "POST", + "/worker/lease", + envelope_v2.seal(auditor, "Receipt", run_id, request, body.d_final), + ) + env = envelope_v2.parse_envelope(lease) + if ( + env.signer != coordinator.ss58 + or env.run_id != run_id + or not envelope_v2.verify_envelope(lease) + ): + raise DriverError("decoder audit lease coordinator differs") + request["commit_hash"] = env.body["lease_nonce"] + job_id = env.body["job_id"] + if not isinstance(job_id, str): + raise DriverError("decoder audit lease job differs") + base = "/worker/jobs/" + job_id + audit = call( + "POST", + base + "/execute", + envelope_v2.seal(auditor, "Receipt", run_id, request, body.d_final), + ) + if audit["verdict"]["result"] != "MATCH": + raise DriverError("decoder independent audit mismatch") + call( + "POST", + base + "/complete", + envelope_v2.seal(auditor, "ReplayVerdict", run_id, audit["verdict"], body.d_final), + ) + applied = call("POST", f"/admin/rounds/{body.w}/aggregate", privileged=True) + tape = TapeV2.from_bytes(objects.get(applied["tape_hash"])) + entries = tape.body.allocation.entries + if {e.hotkey for e in entries} != {r.hotkey for r in body.roster} or any( + e.probation or e.weight_units != 4194304 for e in entries + ): + raise DriverError("decoder weight identity/shadow/Q mismatch") + inputs = context["accepted_inputs"]() + predecessor = lineage[-1]["tape_hash"] if lineage else "0" * 64 + replayed = replay_tape( + objects, + tape, + manifest, + AggregationPolicyV2.model_validate_json( + objects.get(manifest.network.aggregation_policy_hash) + ), + objects.get(manifest.network.economics_policy_hash), + signer=coordinator.ss58, + w=body.w, + prev_state=applied["prev_state"], + predecessor_tape_hash=predecessor, + inputs=network_inputs(inputs), + reference_reward_units=100, + ) + if replayed.to_bytes() != objects.get(applied["out_state"]): + raise DriverError("decoder original outer replay differs") + final = Finalize( + w=body.w, + final_theta_hash_w1=applied["theta_hash"], + included=sorted(r.hotkey for r in body.roster), + entitlements_root="0" * 64, + ) + signed = envelope_v2.seal(coordinator, "Finalize", run_id, final, body.d_final) + call("POST", f"/admin/rounds/{body.w}/finalize", signed, True) + settled = context["accepted_finality"]() + if ( + settled["disposition"] != "SETTLED" + or settled["unresolved_audits"] + or settled["unresolved_disputes"] + ): + raise DriverError("decoder actual finality unsettled") + lineage.append( + { + "round_open": context["round_open"], + "finalize": signed, + "inputs": inputs, + "tape_hash": applied["tape_hash"], + "prev_state": applied["prev_state"], + "predecessor_tape_hash": predecessor, + "reference_reward_units": 100, + } + ) + write_network_checkpoint(checkpoint, coordinator, objects, manifest, lineage) + if verify_network_checkpoint(checkpoint, coordinator.ss58): + raise DriverError("decoder independent checkpoint replay differs") + return lineage + + +def decoder_service_factory( + runtime: NetworkRuntime, + plan: Callable[[str, IslandJobV1, Path, Mapping], tuple[dict, dict | None, dict]], +) -> Callable[[str, IslandJobV1, Path, Mapping], IslandLaunch]: + """Constructor-only trusted plan supplies admitted Operation and owner receipt.""" + + def factory( + operation: str, job: IslandJobV1, directory: Path, context: Mapping + ) -> IslandLaunch: + if TYPE_CHECKING: + from scripts.network_gpu_operation import Operation + else: + from network_gpu_operation import Operation + + raw, receipt, pins = plan(operation, job, directory, context) + spec = Operation.model_validate(raw) + if spec.operation != operation or spec.job != job or spec.owner != pins["owner"]: + raise DriverError("service role factory accepted descriptor differs") + runtime.accept_operation( + spec, canonicalize(receipt), owner=pins["owner"], beacon=pins["beacon"] + ) + if spec.operation == "reference": + if spec.challenge is None or set(spec.context_files) != {spec.challenge}: + raise DriverError("reference original challenge context missing") + challenge_path = spec.challenge + path = directory / "original-join-challenge.json" + if fsha(path) != spec.context_files[challenge_path]: + raise DriverError("reference original challenge bytes changed") + original_launch = runtime.launch_adapter(spec) + + def reference_launch( + job: IslandJobV1, + directory: Path, + *, + backend: Literal["cpu", "cuda"], + cancel: threading.Event | None = None, + trace: bool = False, + ) -> IslandArtifacts: + if job != spec.job or trace and not spec.trace: + raise DriverError("reference original launch binding differs") + if cancel is not None and cancel.is_set() or time.time() >= spec.cutoff: + raise DriverError("reference context deadline/cancellation") + if fsha(path) != spec.context_files[challenge_path]: + raise DriverError("reference original challenge bytes changed") + runtime.stage_context( + spec.role, + {challenge_path: path}, + str(Path(challenge_path).parent), + "reference-" + spec.job.digest(), + spec.cutoff, + cancel, + ) + if cancel is not None and cancel.is_set() or time.time() >= spec.cutoff: + raise DriverError("reference context deadline/cancellation") + return original_launch( + job, + directory, + backend=backend, + cancel=cancel, + trace=spec.trace, + ) + + return reference_launch + return runtime.launch_adapter(spec) + + return factory + + +def service_factory( + driver: ModuleType, + runtime: NetworkRuntime, + owner: Keypair, + run_id: str, + sources: dict[str, str], + roles: dict, + current_beacon: Callable[[], int], +) -> Callable[[str, IslandJobV1, Path, Mapping], IslandLaunch]: + """Original accepted service context authorizes exactly one guarded operation.""" + + def factory( + operation: str, job: IslandJobV1, directory: Path, context: Mapping + ) -> IslandLaunch: + beacon = current_beacon() + key = operation + ":" + job.digest() + receipts: dict[str, dict | None] = {key: None} + plan = driver.decoder_context_plan( + runtime, owner.ss58, run_id, sources, roles, receipts, beacon + ) + raw, _, _ = plan(operation, job, directory, context) + if TYPE_CHECKING: + from scripts.network_gpu_operation import Operation + else: + from network_gpu_operation import Operation + + spec = Operation.model_validate(raw) + subject = { + "role": spec.role, + "binding": spec.binding, + "operation": spec.operation, + "hotkey": spec.hotkey, + "spec_sha256": sha256_hex(spec.model_dump_json().encode()), + } + receipts[key] = envelope_v2.seal( + owner, + "Receipt", + run_id, + { + "w": job.w, + "commit_hash": sha256_hex(canonicalize(subject)), + "received_round": beacon, + }, + beacon + 1, + ) + return driver.decoder_service_factory(runtime, plan)(operation, job, directory, context) + + return factory + + +class ContinuationIdentity(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + hotkey: str + coldkey: str + role: str = Field(pattern=r"^h[01]$") + hot_key_file: Path + cold_key_file: Path + origin_ids: list[str] = Field(min_length=1) + admission_units: StrictInt = Field(gt=0) + + +class ContinuationGraph(BaseModel): + """Hash-bound private master-local inputs; never a serialized callback or job grid.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + run_id: str = Field(pattern=r"^[0-9a-f]{64}$") + owner: str + owner_key_file: Path + coord_key_file: Path + auditor_key_files: list[Path] = Field(min_length=4, max_length=4) + identities: list[ContinuationIdentity] = Field(min_length=4, max_length=4) + hosts: dict[str, dict[str, StrictInt | str]] + state_dir: Path + evidence: Path + master_https: str + relay_https: str + metagraph_https: str + listen_host: str + listen_port: StrictInt = Field(gt=0, lt=65536) + ca: Path + cert: Path + tls_key: Path + admin_token_file: Path + qualification_receipt: Path + economic_authority: Path | None = None + operation_sources: dict[str, str] + private_files: dict[str, str] + + @field_validator("master_https", "relay_https", "metagraph_https") + @classmethod + def https(cls, value: str) -> str: + return Settings.https(value) + + +def continuation_graph( + path: Path, runtime: NetworkRuntime, owner: str, run_id: str, manifest: RunManifestV2 +) -> ContinuationGraph: + """Validate immutable authority/host/file bindings before any signed graph work.""" + graph = ContinuationGraph.model_validate_json(path.read_bytes()) + if graph.economic_authority is None: + raise DriverError("current continuation economic authority required") + if graph.owner != owner or graph.run_id != run_id: + raise DriverError("continuation graph owner/run differs") + if ( + set(graph.hosts) != set(runtime.lifecycle.roles) + or len({i.hotkey for i in graph.identities}) != 4 + or len({i.coldkey for i in graph.identities}) != 4 + or any(sum(i.role == r for i in graph.identities) != 2 for r in graph.hosts) + ): + raise DriverError("continuation complete role roster differs") + for role, observed in graph.hosts.items(): + frozen = runtime.owned_host(role) + if observed != {k: frozen[k] for k in ("instance_id", "machine_id", "image_digest")}: + raise DriverError("continuation graph owned instance differs") + paths = [ + graph.owner_key_file, + graph.coord_key_file, + *graph.auditor_key_files, + graph.ca, + graph.cert, + graph.tls_key, + graph.admin_token_file, + graph.qualification_receipt, + graph.economic_authority, + *(i.hot_key_file for i in graph.identities), + *(i.cold_key_file for i in graph.identities), + ] + for p in paths: + if ( + not p.is_absolute() + or p.is_symlink() + or graph.private_files.get(str(p)) != sha256_hex(p.read_bytes()) + ): + raise DriverError("continuation private input hash differs") + secret_paths = [ + graph.owner_key_file, + graph.coord_key_file, + *graph.auditor_key_files, + graph.tls_key, + graph.admin_token_file, + *(i.hot_key_file for i in graph.identities), + *(i.cold_key_file for i in graph.identities), + ] + if any(p.stat().st_mode & 0o077 for p in secret_paths): + raise DriverError("continuation private signing/TLS inputs not restrictive") + if load_keyfile(graph.owner_key_file).ss58 != owner: + raise DriverError("continuation owner signing identity differs") + if manifest.run_id() != run_id or ( + load_keyfile(graph.coord_key_file).ss58 != manifest.training.coord_pubkey + ): + raise DriverError("continuation coordinator signing identity differs") + for identity in graph.identities: + if ( + load_keyfile(identity.hot_key_file).ss58, + load_keyfile(identity.cold_key_file).ss58, + ) != (identity.hotkey, identity.coldkey): + raise DriverError("continuation miner signing identity differs") + if any(load_keyfile(p).ss58 not in manifest.training.auditors for p in graph.auditor_key_files): + raise DriverError("continuation auditor signing identity differs") + if graph.evidence.exists() or graph.evidence.is_symlink(): + raise DriverError("continuation evidence must be a new owned directory") + if ( + not graph.state_dir.is_absolute() + or graph.state_dir.is_symlink() + or not graph.evidence.is_absolute() + ): + raise DriverError("continuation state/evidence roots must be private absolute paths") + from hypertrain.gpu_ops.network_qualification import sources + + expected = sources(Path(__file__).resolve().parents[1]) + expected["scripts/network_gpu_operation.py"] = sha256_hex( + (Path(__file__).parent / "network_gpu_operation.py").read_bytes() + ) + profile_key = "experiments/gpu_network_v2/profile.json" + if set(expected) != set(graph.operation_sources) or any( + digest != graph.operation_sources[p] for p, digest in expected.items() if p != profile_key + ): + raise DriverError("continuation operation source map differs") + if urlsplit(graph.master_https).port != graph.listen_port: + raise DriverError("continuation listener differs from pinned master") + return graph + + +def continuation_graph_hash(graph: ContinuationGraph) -> str: + body = graph.model_dump(mode="json", exclude={"economic_authority"}) + if graph.economic_authority is not None: + body["private_files"].pop(str(graph.economic_authority), None) + return sha256_hex(canonicalize(body)) + + +def continuation_allocation_authority( + runtime: NetworkRuntime, + checked: dict, + owner: str, + run_id: str, + beacon: int, + role: str, + host: dict, +) -> str: + """Authenticate shipped source closure before parsing real physical CSV.""" + from hypertrain.gpu_ops.network_qualification import select_inventory, sources + + worker_path = checked["files"]["allocation_worker_receipt"] + source_path = checked["files"]["source_receipt"] + worker = json.loads(worker_path.read_bytes()) + source = json.loads(source_path.read_bytes()) + for payload in (worker, source): + env = envelope_v2.parse_envelope(payload["receipt"]) + received_round = env.body["received_round"] + if type(received_round) is not int: + raise DriverError("original signed allocation/worker authority differs") + if ( + env.type != "Receipt" + or env.signer != owner + or env.run_id != run_id + or env.exp_drand < beacon + or received_round > beacon + or not envelope_v2.verify_envelope(payload["receipt"]) + or env.body["commit_hash"] != sha256_hex(canonicalize(payload["record"])) + ): + raise DriverError("original signed allocation/worker authority differs") + record = worker["record"] + if ( + record.get("allocation_contract") != "canonical-shipped-source-v1" + or record["run_id"] != run_id + or record["training_ranks"] != 2 + or record["physical_allocation"] != ">=2" + or record["original_worker_sha256"] != graph_worker_hash(checked) + ): + raise DriverError("original allocation worker/run/layout differs") + tree = Path(checked["record"]["tree"]) + expected = sources(tree) + loader = tree / "scripts/network_service_proof.py" + if loader.is_symlink(): + raise DriverError("canonical allocation loader source symlink") + expected["scripts/network_service_proof.py"] = fsha(loader) + profile_hash = sha256_hex(canonicalize(runtime.profile)) + if ( + record["files"] != expected + or source["record"].get("files") != expected + or record.get("profile_sha256") != profile_hash + or source["record"].get("profile_sha256") != profile_hash + or fsha(Path(__file__)) != expected["scripts/network_service_proof.py"] + or fsha(Path(sys.modules[select_inventory.__module__].__file__ or "")) + != expected["src/hypertrain/gpu_ops/network_qualification.py"] + ): + raise DriverError("signed canonical allocation source/profile closure differs") + allocation = runtime.journal.last("allocated_gpu_inventory", role=role) + if ( + allocation is None + or type(host["quote"]["num_gpus"]) is not int + or allocation["allocated_count"] != host["quote"]["num_gpus"] + ): + raise DriverError("original actual allocated physical count differs") + try: + selected = select_inventory( + allocation["inventory"], + allocation["allocated_count"], + runtime.profile["runtime"]["driver_allowlist"], + ) + except ValueError as exc: + raise DriverError("original allocated physical CSV differs") from exc + if tuple(allocation["selected_uuids"]) != selected: + raise DriverError("original actual selected full UUID order differs") + return sha256_hex( + canonicalize( + { + "allocation": allocation, + "worker_receipt": worker, + "source_receipt": source, + } + ) + ) + + +def graph_worker_hash(checked: dict) -> str: + source = Path(checked["record"]["tree"]) / "src/hypertrain/miner/island_worker.py" + if source.is_symlink(): + raise DriverError("original worker source symlink") + return fsha(source) + + +def continuation_operator_record( + graph: ContinuationGraph, + runtime: NetworkRuntime, + checked: dict, + backend: dict, + beacon: int, + expires: int, + launch_path: Path, +) -> dict: + """Exact finite subjects from the existing verified launch; no accounting oracle.""" + from hypertrain.protocol.messages_v2 import IslandJobV1 + + manifest = IslandJobV1.model_validate_json( + checked["files"][checked["record"]["roles"]["h0"]["job"]].read_bytes() + ).manifest + authorization = json.loads(checked["files"]["authorization"].read_bytes()) + decoder_execution_count(runtime.profile) + if continuation_graph_hash( + ContinuationGraph.model_validate_json(checked["files"]["graph"].read_bytes()) + ) != continuation_graph_hash(graph): + raise DriverError("original signed graph semantic binding differs") + from decimal import Decimal + + accrual = json.loads(checked["files"]["accrual_receipt"].read_bytes()) + accrued = accrual["record"] + accrual_env = envelope_v2.parse_envelope(accrual["receipt"]) + if ( + accrual_env.signer != graph.owner + or accrual_env.type != "Receipt" + or not envelope_v2.verify_envelope(accrual["receipt"]) + or accrual_env.body["commit_hash"] != sha256_hex(canonicalize(accrued)) + or accrual_env.exp_drand < beacon + or accrued.get("one_cleanup_usd") != "5" + or fsha(Path(accrued["evidence_path"])) != accrued["evidence_sha256"] + ): + raise DriverError("original signed accrued liability authority differs") + for root, closure in accrued["closed_predecessors"].items(): + if closure.get("outcome") not in ("CENSORED_RESCUE", "CENSORED", "PASS"): + raise DriverError("original closure unknown/live") + for name in ("journal.jsonl", "lifecycle-evidence.json"): + if fsha(Path(root) / name) != closure[name + "_sha256"]: + raise DriverError("original closure custody differs") + closed = json.loads((Path(root) / "lifecycle-evidence.json").read_bytes()) + if ( + closed.get("inventory", {}).get("parsed") is not True + or closed["inventory"].get("owned_instances") != 0 + ): + raise DriverError("original predecessor closure unknown/live") + finance = checked["financial"] + opening, credit = ( + Decimal(accrued["opening_credit_usd"]), + Decimal(finance["available_credit_usd"]), + ) + debit = max(Decimal(0), opening - credit) + total_accrued = Decimal(accrued["accrual_usd"]) + if ( + Decimal(finance["prior_attempts_usd"]) != max(debit, total_accrued) + or Decimal(finance["active_liabilities_usd"]) != max(Decimal(0), total_accrued - debit) + or Decimal(checked["plan"]["snapshot0_credit_usd"]) != opening + ): + raise DriverError("original accrued/debit-once admission arithmetic differs") + if any( + type(value) is not str or not Decimal(value).is_finite() or Decimal(value) < 0 + for key in ("opening_credit_usd", "accrual_usd", "one_cleanup_usd") + for value in [accrued[key]] + ): + raise DriverError("original decimal USD values invalid") + if ( + checked["record"]["action"] != "continue" + or authorization.get("scope") != "FULL116_CONTINUATION" + or checked["plan"].get("long_workload_allowed") is not True + or checked["plan"].get("admit") is not True + or checked["plan"].get("reserve_usd") != "5" + ): + raise DriverError("full116 explicit economic authorization required") + hosts = { + role: { + **runtime.owned_host(role), + "allocation": next(h for h in checked["config"]["hosts"] if h["role"] == role), + "quote": next( + o + for o in checked["offers"] + if o["machine_id"] == runtime.owned_host(role)["machine_id"] + ), + } + for role in ("h0", "h1") + } + for role, host in hosts.items(): + host["uuid_authority_hash"] = continuation_allocation_authority( + runtime, checked, graph.owner, graph.run_id, beacon, role, host + ) + return { + "domain": "shadow-operator-admission-v1", + "run_id": graph.run_id, + "owner": graph.owner, + "admission_hash": sha256_hex(canonicalize(checked["record"])), + "manifest_hash": manifest.run_id(), + "economics_policy_hash": manifest.network.economics_policy_hash, + "graph_hash": continuation_graph_hash(graph), + "workload": { + "action": "continue", + "schedule_hash": sha256_hex(canonicalize(runtime.profile["schedule"])), + "identities": [ + {"hotkey": i.hotkey, "coldkey": i.coldkey, "role": i.role} for i in graph.identities + ], + "shadow_ordinals": list(range(12)), + "live_rounds": [0, 1], + "qualification": 4, + "normal": 116, + "ceiling": 126, + "per_host": 63, + "fault_allowance": 10, + }, + "backend_qualification_authority_hash": backend["authority_hash"], + "source_map_hash": sha256_hex(canonicalize(graph.operation_sources)), + "profile_hash": fsha(checked["files"]["profile"]), + "hosts": hosts, + "budget": { + "scope": "FULL116_CONTINUATION", + "launch_path": str(launch_path), + "launch_sha256": fsha(launch_path), + "files": { + name: fsha(path) + for name, path in checked["files"].items() + if name not in ("graph",) + }, + "plan": checked["plan"], + "financial": checked["financial"], + "full_quotes": checked["offers"], + "disk_gb": 80, + "egress_gb": 30, + }, + "accepted_beacon": beacon, + "cutoff_unix": int(min(runtime.lifecycle.deadline(), checked["record"]["cutoff_unix"])) + - 300, + "expires_beacon": expires, + } + + +def continuation_operator_intake( + store: ChallengeStore, graph_path: Path, runtime: NetworkRuntime +) -> str: + """Owner record is private detached authority; intake remains a store transaction.""" + return store.accept_shadow_operator_v2(graph_path, runtime) + + +def continuation_qualification( + store: ChallengeStore, + manifest: RunManifestV2, + graph: ContinuationGraph, + runtime: NetworkRuntime, + checked: dict, +) -> None: + """Consume original bootstrap object evidence and its flat accepted record.""" + from hypertrain.gpu_ops.journal import fsha + from hypertrain.gpu_ops.network_qualification import Qualification + + if store._run_v2(graph.run_id) != manifest or store._backend_v2(manifest) != "cuda": + raise DriverError("accepted production backend differs") + with store._lock: + backend = store._record_v2( + graph.run_id, "qualification", manifest.training.reference_spec.image_digest + ) + evidence = envelope_v2.load_json(store.objects.get(backend["authority_hash"])) + if canonicalize(backend["receipt"]) != canonicalize( + json.loads(graph.qualification_receipt.read_bytes()) + ) or evidence["results"] != [ + fsha(checked["files"][checked["record"]["roles"][r]["qualification_result"]]) + for r in ("h0", "h1") + ]: + raise DriverError("accepted production qualification receipt differs") + configs = evidence["configs"] + if not isinstance(configs, list): + raise DriverError("accepted production qualification receipt differs") + for role, config_hash in zip(("h0", "h1"), configs, strict=True): + config_path = runtime.lifecycle.dir / f"cli-qualification-{role}.json" + qualification = Qualification.model_validate_json(config_path.read_bytes()) + host = runtime.owned_host(role) + if fsha(config_path) != config_hash or ( + qualification.instance_id, + qualification.machine_id, + qualification.role, + qualification.image_digest, + ) != (host["instance_id"], host["machine_id"], role, host["image_digest"]): + raise DriverError("accepted qualification owned instance differs") + + +def continuation_results(journal: Journal, roles: Collection[str]) -> list[dict]: + """Require actual accepted done/result custody for every physical workload intent.""" + started = journal.all("network_operation_started") + completed = journal.all("network_operation_done") + work = journal.all("network_execution_intent", phase="workload") + + def keys(rows: list[dict]) -> set[tuple[str, str]]: + return {(r["role"], r["name"]) for r in rows} + + if ( + len(completed) != 112 + or len(keys(completed)) != 112 + or keys(completed) != keys(started) + or keys(completed) != keys(work) + or len(started) != 112 + or len(work) != 112 + ): + raise DriverError("continuation accepted operation coverage differs") + result = [] + for done in completed: + start = next(r for r in started if (r["role"], r["name"]) == (done["role"], done["name"])) + if any( + not re.fullmatch(r"[0-9a-f]{64}", done.get(k, "")) + for k in ("result_sha256", "tar_sha256") + ): + raise DriverError("continuation accepted result/custody hash absent") + result.append({**start, **done}) + for role in roles: + if { + kind: sum(r["role"] == role and r["operation"] == kind for r in result) + for kind in ("reference", "probe", "live", "audit") + } != { + "reference": 24, + "probe": 24, + "live": 4, + "audit": 4, + }: + raise DriverError("continuation accepted role/kind coverage differs") + return result + + +def continuation_close( + store: ChallengeStore, + server: uvicorn.Server | None, + thread: threading.Thread | None, + watcher: threading.Thread | None, + cancel: threading.Event, + done: threading.Event, +) -> None: + """Always close local resources; retain a pending graph exception plus teardown notes.""" + primary = sys.exception() + errors: list[Exception] = [] + try: + cancel.set() + for guard in tuple(store._lease_guards_v2.values()): + guard.cancelled.set() + done.set() + with vars(cancel)["_cond"]: + vars(cancel)["_cond"].notify_all() + if server is not None: + server.should_exit = True + if watcher is not None: + watcher.join(timeout=5) + if watcher.is_alive(): + raise DriverError("continuation cancellation watcher failed to stop") + if thread is not None: + thread.join(timeout=10) + if thread.is_alive(): + if server is not None: + server.force_exit = True + raise DriverError("continuation HTTPS failed to stop") + except (DriverError, RuntimeError, OSError) as exc: + errors.append(exc) + finally: + try: + store.close_v2_notifications() + except (RuntimeError, OSError, sqlite3.Error) as exc: + errors.append(exc) + finally: + try: + store._db.close() + except sqlite3.Error as exc: + errors.append(exc) + if primary is not None: + for error in errors: + primary.add_note(f"continuation teardown: {error}") + elif errors: + for error in errors[1:]: + errors[0].add_note(f"continuation teardown: {error}") + raise errors[0] + + +def decoder_continue( + graph_path: Path, runtime: NetworkRuntime, checked: dict, cancel: threading.Event +) -> dict: + """Run the signed admission/live graph, never raw island jobs or fixture authority.""" + import uvicorn + + from hypertrain.beacon import parse_round + from hypertrain.challenge.admission import trial_samples + from hypertrain.challenge.app import Config, create_app + from hypertrain.challenge.store import ChallengeStore + from hypertrain.gpu_ops.journal import durable_write, fsha + from hypertrain.ledger import Params + from hypertrain.protocol.messages_v2 import EconomicsPolicyV2, IslandJobV1 + + if TYPE_CHECKING: + from scripts.network_gpu_operation import Operation + else: + from network_gpu_operation import Operation + + if cancel.is_set(): + raise DriverError("continuation cancelled before graph intake") + seed = checked["files"][checked["record"]["roles"]["h0"]["job"]] + manifest = IslandJobV1.model_validate_json(seed.read_bytes()).manifest + graph = continuation_graph( + graph_path, runtime, checked["record"]["owner"], checked["run_id"], manifest + ) + cutoff = int(min(runtime.lifecycle.deadline(), checked["record"]["cutoff_unix"])) - 300 + owner, coord = ( + load_keyfile(graph.owner_key_file), + load_keyfile(graph.coord_key_file), + ) + if manifest.run_id() != graph.run_id or coord.ss58 != manifest.training.coord_pubkey: + raise DriverError("continuation manifest/coordinator differs") + if not (graph.state_dir / "challenge.db").is_file(): + raise DriverError("accepted production store absent") + objects = LocalFSStore(graph.state_dir / "objects") + policy = EconomicsPolicyV2.model_validate_json( + objects.get(manifest.network.economics_policy_hash) + ) + if policy.ledger_mode == "test": + raise DriverError("CPU/test genesis cannot authorize signed continuation") + store = ChallengeStore( + graph.state_dir, + Params( + "hypertrain", + manifest.training.beacon.genesis_time, + 4320, + 1, + manifest.training.verify.E_vest_rounds, + ), + coord, + owner.ss58, + parse_round, + objects, + ) + server = thread = watcher = None + done = threading.Event() + try: + continuation_qualification(store, manifest, graph, runtime, checked) + + def now() -> int: + with store._lock: + current = store._fresh_v2(manifest, "signed-continuation") + if not store._beacon_v2(current).bls_verified: + raise DriverError("production beacon not BLS verified") + return current + + def advance(number: int) -> None: + with store.beacon_arrived: + ready = store.beacon_arrived.wait_for( + lambda: cancel.is_set() or store._now(store._db) >= number, + timeout=max(0, cutoff - time.time()), + ) + if not ready or cancel.is_set(): + raise DriverError("continuation beacon/cancellation deadline") + now() + + now() + store.accept_shadow_operator_v2(graph_path, runtime) + roles = {i.hotkey: {"role": i.role} for i in graph.identities} + store._role_launch = service_factory( + sys.modules[__name__], + runtime, + owner, + graph.run_id, + graph.operation_sources, + roles, + now, + ) + app_config = Config( + "hypertrain", + graph.state_dir, + graph.metagraph_https, + None, + graph.admin_token_file, + None, + graph.coord_key_file, + owner.ss58, + store.params, + ) + app = create_app(app_config, objects=objects, _store=store) + started = threading.Event() + + class ContinuationServer(uvicorn.Server): + async def startup(self, sockets: list[socket.socket] | None = None) -> None: + await super().startup(sockets) + started.set() + + server = ContinuationServer( + uvicorn.Config( + app, + host=graph.listen_host, + port=graph.listen_port, + ssl_certfile=str(graph.cert), + ssl_keyfile=str(graph.tls_key), + log_level="error", + timeout_graceful_shutdown=5, + ) + ) + + def stop() -> None: + with vars(cancel)["_cond"]: + vars(cancel)["_cond"].wait_for(lambda: done.is_set() or cancel.is_set()) + if cancel.is_set(): + for guard in tuple(store._lease_guards_v2.values()): + guard.cancelled.set() + with store.beacon_arrived: + store.beacon_arrived.notify_all() + server.should_exit = True + + watcher = threading.Thread(target=stop, name="continuation-cancel", daemon=True) + watcher.start() + thread = threading.Thread(target=server.run, name="continuation-https", daemon=True) + thread.start() + if not started.wait(timeout=min(15, max(0, runtime.lifecycle.deadline() - time.time()))): + raise DriverError("continuation HTTPS startup deadline") + if not server.started or cancel.is_set(): + raise DriverError("continuation HTTPS failed/cancelled") + graph.evidence.mkdir(mode=0o700, parents=True, exist_ok=False) + admin = {"Authorization": "Bearer " + graph.admin_token_file.read_text().strip()} + with httpx.Client( + base_url=graph.master_https, + verify=ssl.create_default_context(cafile=str(graph.ca)), + timeout=60, + follow_redirects=False, + ) as client: + registry = store._registry_v2(manifest) + if graph.relay_https not in {s.https_url.rstrip("/") for s in registry.specs}: + raise DriverError("continuation relay not accepted registry") + + def materialize( + identity: ContinuationIdentity, + job: IslandJobV1, + directory: Path, + kind: Literal["reference", "audit", "referee", "probe", "live"], + binding: str, + challenge: dict | None = None, + ) -> dict: + if cancel.is_set() or time.time() >= min(job.deadline, cutoff): + raise DriverError("continuation context cancelled/expired") + frozen = runtime.owned_host(identity.role) + remote_root = runtime.lifecycle.remote_root(identity.role) + name = "op-" + sha256_hex( + (job.run_id + kind + identity.hotkey + binding + job.digest()).encode() + ) + context = "context/" + name + key_rel, cfg_rel, ca_rel = ( + context + "/miner.key", + context + "/miner.toml", + context + "/ca.pem", + ) + config = ( + f"api={json.dumps(graph.master_https)}\n" + f"keyfile={json.dumps(remote_root + '/' + key_rel)}\n" + f"workdir={json.dumps(remote_root + '/out/' + name + '/miner')}\n" + 'state_source="network"\ndevice="cuda"\n' + f"image_digest={json.dumps(frozen['image_digest'])}\n" + f"run_id={json.dumps(graph.run_id)}\nowner_hotkey={json.dumps(owner.ss58)}\n" + ).encode() + local_config = directory / "miner.toml" + durable_write(local_config, config) + paths = { + key_rel: identity.hot_key_file, + cfg_rel: local_config, + ca_rel: graph.ca, + } + challenge_rel = None + if challenge is not None: + challenge_rel = context + "/challenge.json" + path = directory / "challenge.json" + durable_write(path, canonicalize(challenge)) + paths[challenge_rel] = path + runtime.stage_context( + identity.role, paths, context, name, min(job.deadline, cutoff), cancel + ) + spec = Operation( + operation=kind, + binding=binding, + hotkey=identity.hotkey, + owner=owner.ss58, + role=identity.role, + instance_id=frozen["instance_id"], + machine_id=frozen["machine_id"], + job=job, + binding_kind="trial" if kind == "probe" else "commit-lease", + sources=graph.operation_sources, + image_digest=frozen["image_digest"], + cutoff=min(job.deadline, cutoff), + trace=kind in ("probe", "live"), + miner_config=cfg_rel, + ca=ca_rel, + master=graph.master_https, + challenge=challenge_rel, + context_files={rel: fsha(path) for rel, path in paths.items()}, + ) + subject = { + "role": spec.role, + "binding": spec.binding, + "operation": spec.operation, + "hotkey": spec.hotkey, + "spec_sha256": sha256_hex(spec.model_dump_json().encode()), + } + beacon = now() + receipt = envelope_v2.seal( + owner, + "Receipt", + job.run_id, + { + "w": job.w, + "commit_hash": sha256_hex(canonicalize(subject)), + "received_round": beacon, + }, + beacon + 1, + ) + return { + "operation": spec.model_dump(mode="json"), + "receipt": receipt, + "key": load_keyfile(identity.hot_key_file), + "pins": { + "owner": owner.ss58, + "run_id": job.run_id, + "beacon": beacon, + "admin_headers": admin, + }, + "artifacts": directory, + } + + identities, trials = [], [] + for identity in graph.identities: + if cancel.is_set(): + raise DriverError("continuation cancelled before admission") + hot, cold = ( + load_keyfile(identity.hot_key_file), + load_keyfile(identity.cold_key_file), + ) + if (hot.ss58, cold.ss58) != (identity.hotkey, identity.coldkey): + raise DriverError("continuation identity signing keys differ") + request_id = sha256_hex((graph.run_id + identity.hotkey + "join").encode()) + with store._lock: + accepted = store._db.execute( + "SELECT admission_id,coldkey,clean_count,pending_dispute " + "FROM admissions_v2 WHERE hotkey=?", + (hot.ss58,), + ).fetchone() + if accepted is not None: + if ( + accepted["coldkey"] != cold.ss58 + or accepted["clean_count"] != 0 + or accepted["pending_dispute"] + ): + raise DriverError("continuation prior admission differs") + admission = accepted["admission_id"] + else: + admission = decoder_join_lock( + client, + manifest, + hot, + cold, + identity.origin_ids, + identity.admission_units, + request_id, + sha256_hex((request_id + "lock").encode()), + now() + 400, + ) + identities.append((identity, admission)) + for _ in range(12): + with store._lock: + row = store._db.execute( + "SELECT seed_beacon FROM admissions_v2 WHERE admission_id=?", + (admission,), + ).fetchone() + advance(row[0]) + response = client.get(f"/v2/runs/{graph.run_id}/join/{admission}/challenge") + response.raise_for_status() + signed = response.json() + challenge = JoinChallenge.model_validate( + envelope_v2.parse_envelope(signed).body + ) + with store._lock: + epoch = store._db.execute( + "SELECT trial_epoch FROM admissions_v2 WHERE admission_id=?", + (admission,), + ).fetchone()[0] + job, directory = store.stage_trial_v2( + graph.run_id, + challenge, + trial_samples( + manifest, + admission, + challenge.nonce, + store._beacon_v2(challenge.seed_beacon), + ), + epoch, + ) + op = materialize(identity, job, directory, "probe", admission, signed) + decoder_operation( + client, + runtime, + op["operation"], + op["receipt"], + hot, + op["pins"], + directory, + cancel, + ) + with store._lock: + reference = WorkProof.model_validate_json( + store._db.execute( + "SELECT reference_json FROM admissions_v2 WHERE admission_id=?", + (admission,), + ).fetchone()[0] + ) + from hypertrain.auditor.replay import unpack_state + + theta, _ = unpack_state(objects.get(reference.artifact_refs[0].sha256)) + final = envelope_v2.seal( + coord, + "Finalize", + graph.run_id, + Finalize( + w=job.w, + final_theta_hash_w1=state_hash(theta), + included=[hot.ss58], + entitlements_root=sha256_hex(canonicalize(reference.body())), + ), + challenge.deadline_beacon, + ) + response = client.post( + f"/v2/runs/{graph.run_id}/admin/join/{admission}/finalize", + json=final, + headers=admin, + ) + response.raise_for_status() + trials.append(response.json()) + + def round_context(w: int) -> Callable[[], dict]: + def build() -> dict: + import torch + + from hypertrain.aggregator.capacity_worker import ( + GenesisRequest, + collect_genesis, + ) + from hypertrain.auditor.replay import unpack_state + + cfg = TrainConfig.from_manifest_v2(manifest) + roster = [] + for slot, (identity, admission) in enumerate(identities): + status = store.admission_v2( + graph.run_id, "status", identity=identity.hotkey + ) + if not status["eligible"] or status["record"]["state"] != "ACTIVE": + raise DriverError("continuation identity lacks actual graduation") + roster.append( + { + "hotkey": identity.hotkey, + "slot": slot, + "q_i": f32hex(1), + "admission_id": admission, + "coldkey_group": identity.coldkey, + "state": "ACTIVE", + "eligible_weight": 4194304, + } + ) + if not w: + beacon = now() + intent = RoundOpenV2( + w=0, + prev_final_hash="0" * 64, + theta_hash="0" * 64, + outer_state_hash="0" * 64, + center_hash="0" * 64, + roster_hash=sha256_hex(canonicalize(roster)), + honeypot_commit="0" * 64, + d_open=beacon + 1, + d_assign=beacon + 2, + d_commit=beacon + 100, + d_audit=beacon + 101, + d_upload=beacon + 180, + d_final=beacon + 200, + contract_version=2, + policy_hashes=PolicyHashes.model_validate( + {k: getattr(manifest.network, k) for k in PolicyHashes.model_fields} + ), + registry_epoch=registry.epoch, + start_state_index_hash="0" * 64, + audit_mode="anchored-full", + roster=[RosterEntryV2.model_validate(row) for row in roster], + ) + store._prepare_capacity_genesis_v2( + graph.run_id, + canonicalize( + envelope_v2.seal( + coord, "RoundOpenV2", graph.run_id, intent, intent.d_final + ) + ), + cancel=cancel, + ) + genesis_dir = graph.state_dir / "capacity-genesis" / graph.run_id / "1" + genesis_request = GenesisRequest.model_validate_json( + (genesis_dir / "request.json").read_bytes() + ) + genesis, blobs = collect_genesis(genesis_dir, genesis_request) + if genesis_request.manifest != manifest: + raise DriverError("continuation accepted genesis manifest differs") + theta, _ = unpack_state(blobs[genesis.starts[0].state_object_sha256]) + for digest, payload in blobs.items(): + if objects.get(digest) != payload: + raise DriverError("accepted genesis object custody differs") + state_hashes = genesis.outer_hashes + previous_final = "0" * 64 + if w: + applied = record(graph.state_dir, "applied", str(w - 1), graph.run_id) + theta = { + k: torch.from_numpy(v.copy()) + for k, v in load_state(objects, applied["out_state"]).theta.items() + } + previous_final = sha256_hex( + canonicalize( + record(graph.state_dir, "finalize", str(w - 1), graph.run_id)[ + "body" + ] + ) + ) + previous_tape = TapeV2.from_bytes(objects.get(applied["tape_hash"])) + state_hashes = previous_tape.body.out_hashes + starts = [] + for identity, _ in identities: + status = store.admission_v2( + graph.run_id, "status", identity=identity.hotkey + ) + if not status["eligible"] or status["record"]["state"] != "ACTIVE": + raise DriverError("continuation identity lacks actual graduation") + cache = AnchorCache() + anchor = ( + store._restore_anchor_v2(graph.run_id, identity.hotkey, w - 1, cache) + if w + else cache.genesis(manifest, identity.hotkey, theta) + ) + if anchor.state.step != w * cfg.inner.H: + raise DriverError("continuation carry step differs") + starts.append( + StartStateV2( + run_id=graph.run_id, + w=w, + hotkey=identity.hotkey, + theta_hash=state_hash(theta), + state_object_sha256=sha256_hex(pack_state(theta, anchor.state)), + opt_state_hash=optimizer_hash(anchor.state), + ef_object_sha256=sha256_hex(pack_state(anchor.ef)), + ef_hash=state_hash(anchor.ef), + parent_anchor_hash=anchor.anchor_hash, + global_step0=anchor.state.step, + anchor_verdict_hash=anchor.proof_hash, + ) + ) + beacon = now() + opening = RoundOpenV2( + w=w, + prev_final_hash=previous_final, + theta_hash=state_hash(theta), + outer_state_hash=state_hashes["outer_state_hash"], + center_hash=state_hashes["center_hash"], + roster_hash=sha256_hex(canonicalize(roster)), + honeypot_commit="0" * 64, + d_open=beacon + 1, + d_assign=beacon + 2, + d_commit=beacon + 100, + d_audit=beacon + 101, + d_upload=beacon + 180, + d_final=beacon + 200, + contract_version=2, + policy_hashes=PolicyHashes.model_validate( + {k: getattr(manifest.network, k) for k in PolicyHashes.model_fields} + ), + registry_epoch=registry.epoch, + start_state_index_hash=sha256_hex(canonicalize([s.body() for s in starts])), + audit_mode="anchored-full", + roster=[RosterEntryV2.model_validate(row) for row in roster], + ) + if not w: + intent = RoundOpenV2.model_validate(genesis_request.opening["body"]) + if [r.body() for r in intent.roster] != roster: + raise DriverError("continuation prepared roster differs") + opening = intent.model_copy( + update={ + "theta_hash": genesis.outer_hashes["theta_hash"], + "outer_state_hash": genesis.outer_hashes["outer_state_hash"], + "center_hash": genesis.outer_hashes["center_hash"], + "start_state_index_hash": sha256_hex( + canonicalize([s.body() for s in genesis.starts]) + ), + } + ) + + def miners() -> list[dict]: + entries = [] + for identity, _ in identities: + value = store.island_job_v2(graph.run_id, w, identity.hotkey) + job = IslandJobV1.model_validate(value["job"]) + directory = graph.evidence / f"live-{w}-{identity.hotkey}" + directory.mkdir(mode=0o700) + for rel, digest in value["objects"].items(): + if Path(rel).is_absolute() or ".." in Path(rel).parts: + raise DriverError("continuation accepted object path") + path = directory / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(objects.get(digest)) + binding = sha256_hex( + canonicalize( + { + "run_id": graph.run_id, + "w": w, + "hotkey": identity.hotkey, + "assignment": store._record_v2( + graph.run_id, + "assignment", + f"{w}:{identity.hotkey}", + ), + } + ) + ) + entries.append(materialize(identity, job, directory, "live", binding)) + return entries + + return { + "round_open": envelope_v2.seal( + coord, "RoundOpenV2", graph.run_id, opening, opening.d_final + ), + "miners": miners, + "auditors": [load_keyfile(p) for p in graph.auditor_key_files], + "accepted_inputs": lambda: record( + graph.state_dir, "tape-inputs", str(w), graph.run_id + )["inputs"], + "accepted_finality": lambda: record( + graph.state_dir, "finality", str(w), graph.run_id + ), + } + + return build + + lineage = decoder_live_graph( + client, + runtime, + manifest, + [round_context(0), round_context(1)], + coord, + admin, + advance, + objects, + graph.evidence / "checkpoint", + cancel, + ) + intents = runtime.journal.all("network_execution_intent") + if any( + sum(r["executions"] for r in intents if r["role"] == role) != 58 for role in graph.hosts + ): + raise DriverError("continuation normal debit coverage differs") + if cancel.is_set() or len(trials) != 48 or len(lineage) != 2: + raise DriverError("continuation cancelled/incomplete accepted graph") + operation_results = continuation_results(runtime.journal, graph.hosts) + result = { + "status": "SIGNED_GRAPH_SETTLED_PENDING_RELEASE_REVIEW", + "run_id": graph.run_id, + "graph_sha256": fsha(graph_path), + "trial_finalizations": trials, + "lineage": lineage, + "normal_total": 116, + "per_host": 58, + "ceiling": 126, + "per_host_ceiling": 63, + "unused_faulttrace": 10, + "fault_proof_complete": False, + "review_sha256": sha256_hex(canonicalize(checked["record"])), + "profile_sha256": fsha(checked["files"]["profile"]), + "source_map_sha256": sha256_hex(canonicalize(graph.operation_sources)), + "qualification_receipt_sha256": fsha(graph.qualification_receipt), + "hosts": graph.hosts, + "operation_receipts": runtime.journal.all("network_operation_authority"), + "operation_results": operation_results, + "checkpoint_manifest_sha256": fsha(graph.evidence / "checkpoint" / "MANIFEST.json"), + } + durable_json(graph.evidence / "result.json", result) + return result + finally: + continuation_close(store, server, thread, watcher, cancel, done) + + +def decoder_context_plan( + runtime: NetworkRuntime, + owner: str, + run_id: str, + sources: dict, + roles: dict, + receipts: dict, + beacon: int, +) -> Callable[[str, IslandJobV1, Path, dict], tuple[dict, dict | None, dict]]: + """Inline trusted bootstrap: original service context plus preaccepted root receipts.""" + + def plan( + operation: str, job: IslandJobV1, directory: Path, context: Mapping + ) -> tuple[dict, dict | None, dict]: + if operation not in ("reference", "audit", "referee"): + raise DriverError("service factory operation not reference/audit/referee") + if TYPE_CHECKING: + from scripts.network_gpu_operation import Operation + else: + from network_gpu_operation import Operation + + if job.run_id != run_id or context["run_id"] != run_id: + raise DriverError("service context run differs from admitted bootstrap") + hotkey = ( + context["hotkey"] + if operation == "reference" + else ( + context["audit_job"]["start_state"]["hotkey"] + if operation == "audit" + else context["turn"]["contest"]["miner"] + ) + ) + host = roles[hotkey] + frozen = runtime.owned_host(host["role"]) + if operation == "reference": + signed = context["challenge_envelope"] + env = envelope_v2.parse_envelope(signed) + if ( + env.type != "JoinChallenge" + or env.run_id != run_id + or env.signer != job.manifest.training.coord_pubkey + or not envelope_v2.verify_envelope(signed) + ): + raise DriverError("reference original challenge authority differs") + challenge = JoinChallenge.model_validate(env.body) + binding, kind, trace = challenge.admission_id, "trial", True + if ( + binding != context["admission_id"] + or challenge.body() != context["challenge"] + or challenge.digest() != context["challenge_hash"] + or job.w != context["epoch"] + or challenge.layout != job.manifest.training.reference_spec.layout + ): + raise DriverError("reference accepted challenge differs") + from hypertrain.data.trial_assignment import trial_assignment_hash + + if ( + challenge.assignment_hash + != trial_assignment_hash(job.manifest, job.w, tuple(job.sample_ids)) + or env.exp_drand < challenge.deadline_beacon + ): + raise DriverError("reference original assignment/deadline differs") + # Preserve original envelope bytes as canonical wire JSON; never re-sign it. + wire = canonicalize(signed) + path = directory / "original-join-challenge.json" + if path.exists(): + if path.read_bytes() != wire: + raise DriverError("reference original challenge file conflicts") + else: + durable_write(path, wire) + remote_challenge = "reference-context-" + job.digest() + "/original-join-challenge.json" + challenge_files = {remote_challenge: sha256_hex(wire)} + elif operation in ("audit", "referee"): + descriptor = context["descriptor"] + digest = sha256_hex(canonicalize(descriptor)) + envelope = context["descriptor_receipt"] + if operation == "audit" and "receipt" in envelope: + envelope = envelope["receipt"] + signed = envelope_v2.parse_envelope(envelope) + if ( + digest != context["descriptor_hash"] + or signed.signer != job.manifest.training.coord_pubkey + or signed.run_id != run_id + or signed.type != "Receipt" + or signed.body["commit_hash"] != digest + or not envelope_v2.verify_envelope(envelope) + ): + raise DriverError("original accepted trace descriptor authority differs") + if descriptor["job"] != job.body(): + raise DriverError("trace accepted job differs") + binding, kind, trace = digest, "commit-lease", True + else: + raise DriverError("service factory operation not reference/audit/referee") + spec = Operation.model_validate( + { + "operation": operation, + "binding": binding, + "hotkey": hotkey, + "owner": owner, + "role": host["role"], + "instance_id": frozen["instance_id"], + "machine_id": frozen["machine_id"], + "job": job, + "binding_kind": kind, + "sources": sources, + "image_digest": frozen["image_digest"], + "cutoff": job.deadline, + "trace": trace, + "challenge": remote_challenge if operation == "reference" else None, + "context_files": challenge_files if operation == "reference" else {}, + } + ) + # Never sign an owner authorization here; root must supply exact reviewed context receipt. + receipt = receipts[operation + ":" + job.digest()] + return spec.model_dump(mode="json"), receipt, {"owner": owner, "beacon": beacon} + + return plan + + +def decoder_execution_count(profile: dict) -> dict: + """No-kernel reconciliation; independent audit publishes its own trace once.""" + schedule = profile["schedule"] + if ( + profile["model"]["arch"] != "decoder" + or profile["inner"]["H"] != 30 + or profile["inner"]["J"] != 5 + or profile["layout"]["n_gpus"] != 2 + or schedule["logical_identities"] != 4 + or schedule["shadow_participations_per_identity"] != 12 + or schedule["executions_total"] != 126 + or schedule["executions_per_host"] != 63 + or schedule["shadow_executions"] != 96 + or schedule["live_executions"] != 16 + or schedule["probes"] != 2 + or schedule["fault_trace_reserve"] != 12 + ): + raise DriverError("decoder original execution contract differs") + shadow = 4 * 12 * 2 # Trusted reference + miner screen; staging has no training kernel. + live = 4 * 2 * 2 # Miner + independently replayed and traced audit; no geometry rerun. + qualification = 4 + faults_remaining = schedule["fault_trace_reserve"] - 2 + required = qualification + shadow + live + faults_remaining + return { + "status": "COUNT_FITS", + "reason": "D2_EXECUTION_COUNT_ONLY", + "qualification": qualification, + "shadow": shadow, + "live": live, + "faulttrace_remaining": faults_remaining, + "required_total": required, + "required_per_host": required // 2, + "ceiling": 126, + "per_host_ceiling": 63, + "excess": required - 126, + "hot_roles": {"h0": [0, 1], "h1": [2, 3]}, + "backend_authority": "NONE", + "kernels_executed": 0, + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + inputs = parser.add_mutually_exclusive_group(required=True) + inputs.add_argument("--config", type=Path) + inputs.add_argument("--decoder-count-profile", type=Path) + args = parser.parse_args(argv) + if args.decoder_count_profile is not None: + print( + json.dumps(decoder_execution_count(json.loads(args.decoder_count_profile.read_bytes()))) + ) + return 0 + settings = Settings.model_validate_json(args.config.read_bytes()) + cancel = threading.Event() + + def stop(signum: int, frame: FrameType | None) -> None: + cancel.set() + + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + owned = not settings.evidence.exists() + completed = False + try: + proof(settings, cancel) + completed = True + finally: + if owned and settings.evidence.is_dir(): + workload_error = sys.exception() + try: + finalize_evidence( + settings, completed, str(workload_error) if workload_error else None + ) + except (OSError, sqlite3.Error, DriverError) as rescue_error: + if workload_error is None: + raise + workload_error.add_note(f"evidence rescue error: {rescue_error}") + print(f"evidence rescue error: {rescue_error}", file=sys.stderr) + print((settings.evidence / "result.json").read_text()) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/publish.sh b/scripts/publish.sh index 2b88bc2..1a382b1 100755 --- a/scripts/publish.sh +++ b/scripts/publish.sh @@ -18,9 +18,54 @@ while [ $# -gt 0 ]; do case "$1" in list_files() { # allowlist = everything not matching .publishignore (cd "$ROOT" && find . -type f | sed 's|^\./||' | grep -Ev -f <(grep -Ev '^(#|$)' .publishignore) | LC_ALL=C sort) } +copy_files() { # stdin: selected relative paths; no source/destination symlink resolution + python3 -c ' +import os,shutil,stat,sys +from contextlib import ExitStack + +dirs=os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW +def directory(stack, parent, name, create=False): + if create: + try: os.mkdir(name, dir_fd=parent) + except FileExistsError: pass + fd=os.open(name, dirs, dir_fd=parent) + stack.callback(os.close, fd) + return fd + +with ExitStack() as roots: + source=directory(roots, None, sys.argv[1]) + target=directory(roots, None, sys.argv[2]) + for line in sys.stdin: + parts=line.rstrip("\n").split("/") + if any(p in ("", ".", "..") for p in parts): + raise ValueError("unsafe selected relative path") + with ExitStack() as stack: + src,dst=source,target + for part in parts[:-1]: + src=directory(stack, src, part) + dst=directory(stack, dst, part, create=True) + fd=os.open(parts[-1], os.O_RDONLY|os.O_NOFOLLOW|os.O_NONBLOCK, dir_fd=src) + stack.callback(os.close, fd) + before=os.fstat(fd) + named=os.stat(parts[-1], dir_fd=src, follow_symlinks=False) + if not stat.S_ISREG(before.st_mode) or (before.st_dev,before.st_ino) != (named.st_dev,named.st_ino): + raise ValueError("selected source is not the same regular file") + out=os.open(parts[-1], os.O_WRONLY|os.O_CREAT|os.O_NOFOLLOW|os.O_NONBLOCK, 0o600, dir_fd=dst) + stack.callback(os.close, out) + if not stat.S_ISREG(os.fstat(out).st_mode): + raise ValueError("copy destination is not a regular file") + os.ftruncate(out, 0) + with os.fdopen(os.dup(fd), "rb") as reader, os.fdopen(os.dup(out), "wb") as writer: + shutil.copyfileobj(reader, writer) + after=os.fstat(fd) + if (before.st_size,before.st_mtime_ns,before.st_ctime_ns) != (after.st_size,after.st_mtime_ns,after.st_ctime_ns): + raise ValueError("selected source changed during copy") + os.fchmod(out, stat.S_IMODE(before.st_mode) & 0o777) +' "$ROOT" "$1" +} scan() { # regex + entropy secret scan over the file list (single python pass); allowlist = literal substrings in .secretscan-allow list_files | (cd "$ROOT" && python3 -c ' -import re,sys,math +import re,sys,math,json,hashlib allow=[l.strip() for l in open(".secretscan-allow") if l.strip() and not l.startswith("#")] pats=[r"AKIA[0-9A-Z]{16}",r"-----BEGIN [A-Z ]*PRIVATE KEY-----",r"gh[pousr]_[A-Za-z0-9]{30,}",r"xox[abp]-[A-Za-z0-9-]{10,}", r"sk-[A-Za-z0-9]{32,}",r"FAL_KEY *[=:] *[\"\x27]?[A-Za-z0-9-]{20,}",r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}:[0-9a-f]{32,}", @@ -29,10 +74,56 @@ pats=[r"AKIA[0-9A-Z]{16}",r"-----BEGIN [A-Z ]*PRIVATE KEY-----",r"gh[pousr]_[A-Z CW=r"token(?!iz)|secret|key|password|passwd|auth|credential|apikey" CRED=re.compile(r"([\w.-]*(?:"+CW+r")[\w.-]*)[\"\x27]?\s*[:=]\s*[\"\x27]?(?:@?sha(?:256|512):)?[0-9a-fA-F]{32,}(?![0-9a-fA-F])",re.I) bad=0 +public_json={ + "deploy/k8s/versions.json": ((),"server_build_job","https://github.com/"+"CortexLM/hypertrain/actions/runs/"+"37815568607/job/113454067723"), + "tests/protocol/fixtures/network_v1_baseline.json": (("source_sha256",),"protocol/keys.py","d3030488eb18aaafc0feceb256666bcc63fc15c4e6ae0bb008980affd96976a2"), +} +public_inline={ + "docs/network-v2.md": ("--freeze ","/root/distributed-decision-training/"+".omo/evidence/hypertrain-network/"+"D2-FINAL-SOURCE-FREEZE.json"), + "scripts/network_gpu_prepare.py": ("\"operation_receipts\": ","CREATED_ONLY_AT_RUNTIME_FROM_ACCEPTED_CONTEXTS_BY_"+"service_factory"), + "tests/miner/test_island_launch_v2.py": ("\"experiment\": ","hypertrain-network-v2"), +} +public_entropy={ + "docs/RESULTS.md": {"f360080129ca48b776b8f50b61bfe90a3a7e5ee84eb8e379fefcf33201c67528"}, + "docs/network-v2.md": {"f360080129ca48b776b8f50b61bfe90a3a7e5ee84eb8e379fefcf33201c67528","3bf8752c74172542a19ec8f387d79c16821bd42cba7335fda8b5bb365211019f"}, + "scripts/relay_kind_smoke.py": {"9af042788b3249f7b1d329581bfe135d1a625af872d71ea22e487b3491f3839b"}, + "tests/gpu_ops/test_network_admit_cli.py": {"451abbd0ff08a09966c08c027ca7a49c24182fbc1c5e7f5ad33ab1ef72247158"}, + "scripts/verify_image.sh": {"22b0248ee25b21e6cd0485679f4adee029af9680d61eaed99e439d766222cf11"}, +} +public_token_grammar=re.compile( + r"(?:[A-Za-z0-9_-]+/)+[A-Za-z0-9_-]+|" + r"--setenv=KIND_EXPERIMENTAL_PROVIDER=" + "docker" +) for f in sys.stdin.read().split("\n"): if not f or f.endswith((".u32",".png",".jpg",".lock")) or f==".secretscan-allow": continue + content=open(f,errors="ignore").read() + public=None + if f in public_json: + parents,key,value=public_json[f] + try: + obj=json.loads(content) + for parent in parents: obj=obj[parent] + if obj[key]==value: public=(key,value) + except (ValueError,KeyError,TypeError): pass stack=[] # (indent, key) ancestors, indentation-aware - for n,line in enumerate(open(f,errors="ignore"),1): + for n,line in enumerate(content.splitlines(),1): + heuristic=line + if public: + key,value=public + declaration=json.dumps(key)+": "+json.dumps(value) + indent=len(line)-len(line.lstrip()) + if line.strip().rstrip(",")==declaration and tuple(k for i,k in stack if i4.5 and not s.startswith(("sha","http")): print(f"ENTROPY {f}:{n} {s[:12]}... H={e:.2f}"); bad=1 + if e>4.5 and not s.startswith(("sha","http")): print(f"ENTROPY {f}:{n} H={e:.2f}"); bad=1 sys.exit(bad)') } # raw tree (before .publishignore), only venv/caches skipped; exact-path allowlist FORBID_ALLOW (add path + comment why) @@ -82,7 +176,7 @@ for f in docs/assets/hypertrain-hero.png README.md LICENSE NOTICE CHANGELOG.md; echo "== secret scan ==" B="$(tracked_bad)"; if [ -n "$B" ]; then echo "forbidden files (raw tree):" >&2; echo "$B" >&2; [ -n "$OUT" ] && echo "$B" > "$OUT/scan.txt"; exit 1; fi -if command -v gitleaks >/dev/null; then gitleaks detect --no-git --source "$ROOT" -q || { echo "gitleaks hit" >&2; exit 1; }; else echo "gitleaks not installed (skipped)"; fi +if command -v gitleaks >/dev/null; then gitleaks detect --no-git --source "$ROOT" --config "$ROOT/.gitleaks.toml" --log-level error --redact=100 || { echo "gitleaks hit" >&2; exit 1; }; else echo "gitleaks not installed (skipped)"; fi S="$(scan 2>&1 || true)"; if [ -n "$OUT" ]; then printf '%s\n' "${S:-clean}" > "$OUT/scan.txt"; fi if [ -n "$S" ]; then echo "$S" >&2; echo "secret scan FAILED" >&2; exit 1; fi echo "secret scan clean" @@ -108,7 +202,7 @@ else echo "refused: cannot determine emptiness of $REPO: $R" >&2; exit 6; fi if [ "$N" != 0 ] && [ "$TAG" = 0 ]; then echo "refused: $REPO is not empty" >&2; exit 4; fi TMP="$(mktemp -d /tmp/hypertrain-publish.XXXXXX)"; trap 'rm -rf "$TMP"' EXIT case "$TMP" in "$ROOT"*) echo "temp inside workspace" >&2; exit 5;; esac -list_files | while IFS= read -r f; do mkdir -p "$TMP/$(dirname "$f")"; cp -p "$ROOT/$f" "$TMP/$f"; done +list_files | copy_files "$TMP" cd "$TMP" if [ "$TAG" = 0 ]; then git init -q -b main; git add -A; git -c user.name="echobt" -c user.email="154886644+echobt@users.noreply.github.com" commit -qm "Hypertrain v0.1.0" diff --git a/scripts/relay_kind_smoke.py b/scripts/relay_kind_smoke.py new file mode 100644 index 0000000..1562b6d --- /dev/null +++ b/scripts/relay_kind_smoke.py @@ -0,0 +1,3936 @@ +# /// script +# requires-python = "==3.12.*" +# dependencies = [] +# /// +# Run inside the locked repository environment: +# env OMP_NUM_THREADS=1 nice -n 19 taskset -c 0-3 flock /tmp/hypertrain-network-cpu.lock \ +# uv run --frozen python scripts/relay_kind_smoke.py --create \ +# --cluster hypertrain-v2-proof --cpus 0-3 --cpu-quota 4 --cleanup +"""Pinned, policy-enforcing single-node proof. Never executed by lane implementation. + +Fixture roles implement actual bounded storage/TLS/master/miner requests, not a +success simulator. Integrated image must contain relay code; historical image +alone fails preflight. Full cross-lane integration proof needs L0 separately. +""" + +from __future__ import annotations + +import argparse +import base64 +import ctypes +import fcntl +import hashlib +import hmac +import importlib +import ipaddress +import json +import math +import os +import re +import secrets +import select +import shutil +import signal +import socket +import sqlite3 +import stat +import subprocess +import sys +import tarfile +import tempfile +import time +from collections.abc import AsyncIterator, Callable +from contextlib import asynccontextmanager, closing, contextmanager +from pathlib import Path +from typing import TYPE_CHECKING, Literal + +import anyio +import httpx +from anyio.abc import ByteReceiveStream +from fastapi import FastAPI, HTTPException, Request +from fastapi.responses import JSONResponse, StreamingResponse +from pydantic import BaseModel, ConfigDict +from starlette.responses import Response + +from hypertrain.data.store import LocalFSStore, S3Credentials, S3Store, sigv4_headers +from hypertrain.data.stream_store import BLOCK, StreamStore, blocks, spool +from hypertrain.protocol import envelope_v2, relay_envelope +from hypertrain.protocol.keys import Keypair +from hypertrain.protocol.messages_v2 import ( + RoundOpenV2, + RunManifestV2, +) +from hypertrain.protocol.relay_envelope import RelayEnvelope +from hypertrain.protocol.relay_messages import ( + CustodyRelease, + MasterAcceptanceV2, + RelayAssignment, + RelayKey, + RelayReceipt, + RelayRegistryV1, + RelaySpec, + RetentionExtension, + RetrievalRequest, + RetrievalResponse, + UploadGrant, +) +from hypertrain.relay.app import create_app +from hypertrain.relay.client import ( + RelayClient, + chunk_manifest, + forward_receipt, + receipt_verified, +) +from hypertrain.relay.core import Relay, Settlement + +if TYPE_CHECKING: + from network_authority_snapshot import Bundle + +ROOT = Path(__file__).resolve().parents[1] +FIXTURE = Path("/run/proof") +OBJECT_ROOT = Path("/objects") +MASTER_ROOT = Path("/master") +RUN = hashlib.sha256(b"hypertrain-kind-transport-proof-v2").hexdigest() +GATEWAY = "https://gateway.hypertrain-proof-gateway.svc" + + +def inspect_integration_snapshot(source: Path, admitted_hash: str) -> Bundle: + """Use the L0-owned original-byte validator, not a second authority gate.""" + import network_authority_snapshot as authority + + return authority.validate(source, admitted_hash) + + +class Versions(BaseModel): + model_config = ConfigDict(extra="forbid") + server_image: str + server_source_sha: str + server_build_job: str + kind_version: str + kubernetes_version: str + node_image: str + calico_version: str + calico_manifest_url: str + calico_manifest_sha256: str + calico_images: dict[str, str] + harness: str + max_node_cpus: int + max_node_memory_bytes: int + + +class IntegrationServiceConfig(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + snapshot: Path + admitted_hash: str + state: Path + secrets: Path + master_url: str + host: str + port: int = 8080 + + +def integration_role(role: str, config_path: Path) -> None: + """Use actual owner bootstrap/driver; never construct fixture funding.""" + import network_authority_snapshot as authority + + if role == "service": + import uvicorn + + from hypertrain.challenge.app import create_app as challenge_app + + config = IntegrationServiceConfig.model_validate_json(config_path.read_bytes()) + with authority.bootstrap( + config.snapshot, + config.admitted_hash, + config.state, + config.secrets, + config.master_url, + ) as actual: + from hypertrain.beacon.core import FixtureBeacon + + bundle = authority.validate(config.snapshot, config.admitted_hash) + beacon = FixtureBeacon(current=bundle.beacon_round).get(bundle.beacon_round) + ready = config.state / "runtime.json" + ready.write_text( + json.dumps( + { + "fixture_clock_offset": float(os.environ["HT_FIXTURE_CLOCK_OFFSET"]), + "beacon": { + "round": beacon.round, + "signature": beacon.signature, + "randomness": beacon.randomness, + }, + } + ) + ) + uvicorn.run( + challenge_app(actual, verify_beacon=authority.fixture_beacon), + host=config.host, + port=config.port, + ) + elif role == "driver": + driver = importlib.import_module("network_service_proof") + settings = json.loads(config_path.read_bytes()) + runtime = json.loads((Path(settings["service_state"]) / "runtime.json").read_bytes()) + settings["fixture_clock_offset"] = runtime["fixture_clock_offset"] + beacon_path = Path(settings["evidence"]).parent / "verified-beacon.json" + beacon_path.write_text(json.dumps(runtime["beacon"])) + settings["verified_beacon"] = str(beacon_path) + prepared = Path(settings["evidence"]).parent / "driver.json" + prepared.write_text(json.dumps(settings)) + driver.main(["--config", str(prepared)]) + elif role == "relay": + import uvicorn + + relay_config = IntegrationRelayConfig.model_validate_json(config_path.read_bytes()) + uvicorn.run( + integration_relay(relay_config), + host=relay_config.host, + port=relay_config.port, + ) + elif role == "gateway": + import uvicorn + + uvicorn.run( + integration_gateway(), + host=os.environ["D3_BIND_HOST"], + port=8443, + ssl_certfile="/proof/tls/server.crt", + ssl_keyfile="/proof/tls/server.key", + ) + elif role == "wrong-ca": + with httpx.Client(timeout=10, trust_env=False) as client: + try: + client.get("https://relay.test/health") + except httpx.ConnectError as error: + if "CERTIFICATE_VERIFY_FAILED" not in str(error): + raise + print("TLS_WRONG_CA_REJECTED") + else: + raise ValueError("wrong CA unexpectedly accepted") + else: + raise ValueError("unknown integration role") + + +class IntegrationRelayConfig(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + state: Path + backing: Path + ca: Path + master_url: str + relay_seed: Path + admin_token: Path + drain_token: Path + host: str + port: int = 8000 + + +def integration_relay(config: IntegrationRelayConfig) -> FastAPI: + """Actual original-registry relay; service supplies clock and accepted settlement.""" + from dataclasses import fields + + from hypertrain.miner.core import load_keyfile + from hypertrain.protocol.jcs import canonicalize + from hypertrain.protocol.relay_messages import RelayNetworkManifest + + def authority_state() -> tuple[RunManifestV2, RelayRegistryV1, int]: + with closing( + sqlite3.connect((config.state / "challenge.db").as_uri() + "?mode=ro", uri=True) + ) as db: + run_id, raw = db.execute("SELECT run_id,manifest FROM runs").fetchone() + manifest = RunManifestV2.model_validate_json(raw) + if run_id != manifest.run_id(): + raise ValueError("relay actual-service run mismatch") + current = db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='registry' AND id='current'", + (run_id,), + ).fetchone() + pinned = RelayRegistryV1.model_validate_json( + json.dumps(json.loads(current[0])["body"]) + if current + else LocalFSStore(config.state / "objects").get( + manifest.network.relay_registry_hash + ) + ) + beacon = db.execute("SELECT MAX(round) FROM beacon").fetchone()[0] + return manifest, pinned, beacon + + @asynccontextmanager + async def lifespan(app: FastAPI) -> AsyncIterator[None]: + import ssl + + manifest, pinned, _ = authority_state() + original = pinned.specs[0] + if ( + len(pinned.specs) != 1 + or original.id != "local" + or original.https_url != "https://relay.test" + ): + raise ValueError("integration relay requires original local endpoint registry") + network = RelayNetworkManifest( + run_id=manifest.run_id(), + base_manifest_hash=manifest.training.training_hash(), + registry_hash=pinned.digest(), + specs_hash=hashlib.sha256(canonicalize([s.body() for s in pinned.specs])).hexdigest(), + assignment_policy="master-observed-median3-v1", + ) + async with httpx.AsyncClient( + verify=ssl.create_default_context(cafile=str(config.ca)), + timeout=60, + follow_redirects=False, + ) as client: + + async def now() -> int: + return (await anyio.to_thread.run_sync(authority_state))[2] + + async def forward(raw: RelayEnvelope) -> RelayEnvelope: + return await forward_receipt(client, config.master_url, raw) + + async def settlement(grant: str) -> Settlement: + response = await client.get( + config.master_url + + f"/v2/runs/{manifest.run_id()}/admin/relay-settlement/{grant}", + headers={"Authorization": "Bearer " + config.admin_token.read_text().strip()}, + ) + response.raise_for_status() + data = response.json() + if set(data) != {f.name for f in fields(Settlement)}: + raise ValueError("actual service settlement schema mismatch") + return Settlement(**data) + + relay = Relay( + run_id=manifest.run_id(), + master=manifest.training.coord_pubkey, + registry=pinned, + network_manifest_hash=network.digest(), + observers=set(manifest.training.auditors), + relay_id=original.id, + region=original.region, + keys={original.pubkeys[0].key_id: load_keyfile(config.relay_seed)}, + active_key=original.pubkeys[0].key_id, + backing=StreamStore(LocalFSStore(config.backing), client), + now=now, + forward=forward, + settlement=settlement, + ) + app.mount("/", create_app(relay, config.drain_token.read_text().strip())) + await relay.recover() + yield + + return FastAPI(lifespan=lifespan) + + +def integration_gateway() -> FastAPI: + """TLS host routing retains original master.test/relay.test identities.""" + app = FastAPI() + + @app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "DELETE"]) + async def proxy(path: str, request: Request) -> Response: + hostname = request.url.hostname + if hostname not in {"master.test", "relay.test"}: + raise HTTPException(421, "unrecognized original authority hostname") + target = "http://service:8080" if hostname == "master.test" else "http://relay:8000" + body = bytearray() + async for part in request.stream(): + if len(body) + len(part) > 1 << 20: + raise HTTPException(413, "original relay proof request exceeds 1MiB") + body.extend(part) + async with httpx.AsyncClient(timeout=900, follow_redirects=False) as client: + response = await client.request( + request.method, + target + "/" + path, + params=request.query_params, + headers={ + k: v + for k, v in request.headers.items() + if k.lower() + not in {"host", "content-length", "transfer-encoding", "connection"} + }, + content=bytes(body), + ) + return Response( + response.content, + response.status_code, + headers={ + k: v + for k, v in response.headers.items() + if k.lower() + not in { + "content-length", + "transfer-encoding", + "connection", + "content-encoding", + } + }, + ) + + return app + + +def integration_manifests(image: str, *, local_loaded: bool = False) -> dict: + """Render real role jobs against original DNS/registry; no kubectl/image/build calls. + + /proof is the root-admitted isolated kind mount, not ambient production state. + Config files/authority/TLS/secrets must exist there before root launch. + """ + if re.fullmatch(r"[^\s]+@sha256:[0-9a-f]{64}", image) is None and not ( + local_loaded and re.fullmatch(r"hypertrain-relay-proof:[0-9a-f]{64}", image) + ): + raise ValueError("integration render requires exact image digest") + namespace = "hypertrain-d3" + items = [{"apiVersion": "v1", "kind": "Namespace", "metadata": {"name": namespace}}] + for role, cpu, memory in ( + ("service", "1000m", "1Gi"), + ("relay", "250m", "256Mi"), + ("driver", "2000m", "2Gi"), + ): + mounts = [ + {"name": "bundle", "mountPath": "/proof/bundle", "readOnly": True}, + {"name": "configs", "mountPath": "/proof/configs", "readOnly": True}, + {"name": "tls", "mountPath": "/proof/tls", "readOnly": True}, + {"name": "secrets", "mountPath": "/proof/secrets", "readOnly": True}, + { + "name": "state", + "mountPath": "/proof/state", + "readOnly": role != "service", + }, + {"name": "work", "mountPath": "/proof/work"}, + {"name": "scripts", "mountPath": "/scripts", "readOnly": True}, + ] + volumes = [ + {"name": name, "hostPath": {"path": "/proof/" + path, "type": "Directory"}} + for name, path in ( + ("bundle", "bundle"), + ("configs", "configs"), + ("tls", "tls"), + ("secrets", "secrets/" + role), + ("state", "state"), + ("work", "work/" + role), + ) + ] + volumes.append({"name": "scripts", "configMap": {"name": "actual-scripts"}}) + container = { + "name": role, + "image": image, + "imagePullPolicy": "Never" if local_loaded else "IfNotPresent", + "command": [ + "/usr/bin/nice", + "-n", + "19", + "/opt/venv/bin/python", + "/scripts/relay_kind_smoke.py", + "--mode", + "integration", + "--integration-role", + role, + "--integration-config", + "/proof/configs/" + role + ".json", + ], + "env": [{"name": name, "value": "1"} for name in ("OMP_NUM_THREADS", "MKL_NUM_THREADS")] + + [{"name": "SSL_CERT_FILE", "value": "/proof/tls/ca.crt"}], + "resources": { + "requests": {"cpu": cpu, "memory": memory}, + "limits": {"cpu": cpu, "memory": memory}, + }, + "volumeMounts": mounts, + "securityContext": { + "allowPrivilegeEscalation": False, + "capabilities": {"drop": ["ALL"]}, + }, + } + spec = { + "automountServiceAccountToken": False, + "securityContext": { + "runAsNonRoot": True, + "runAsUser": 65532, + "seccompProfile": {"type": "RuntimeDefault"}, + }, + "hostAliases": [{"ip": "10.96.0.30", "hostnames": ["master.test", "relay.test"]}], + "containers": [container], + "volumes": volumes, + "restartPolicy": "Never", + } + if role != "driver": + container["readinessProbe"] = { + "httpGet": { + "path": "/health", + "port": 8080 if role == "service" else 8000, + }, + "periodSeconds": 1, + "timeoutSeconds": 1, + "failureThreshold": 900, + } + if role == "driver": + items.append( + { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": {"name": role, "namespace": namespace}, + "spec": { + "backoffLimit": 0, + "activeDeadlineSeconds": 9600, + "template": { + "metadata": {"labels": {"role": role}}, + "spec": spec, + }, + }, + } + ) + else: + spec["activeDeadlineSeconds"] = 10800 + items.append( + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "name": role, + "namespace": namespace, + "labels": {"role": role}, + }, + "spec": spec, + } + ) + items.append( + { + "apiVersion": "v1", + "kind": "Service", + "metadata": {"name": role, "namespace": namespace}, + "spec": { + "selector": {"role": role}, + "ports": [{"port": 8080 if role == "service" else 8000}], + }, + } + ) + items.append( + { + "apiVersion": "v1", + "kind": "Service", + "metadata": {"name": "gateway", "namespace": namespace}, + "spec": { + "clusterIP": "10.96.0.30", + "selector": {"role": "gateway"}, + "ports": [{"port": 443, "targetPort": 8443}], + }, + } + ) + items.append( + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "name": "gateway", + "namespace": namespace, + "labels": {"role": "gateway"}, + }, + "spec": { + "automountServiceAccountToken": False, + "activeDeadlineSeconds": 10800, + "containers": [ + { + "name": "gateway", + "image": image, + "imagePullPolicy": "Never" if local_loaded else "IfNotPresent", + "command": [ + "/usr/bin/nice", + "-n", + "19", + "/opt/venv/bin/python", + "/scripts/relay_kind_smoke.py", + "--mode", + "integration", + "--integration-role", + "gateway", + "--integration-config", + "/proof/tls/server.crt", + ], + "ports": [{"containerPort": 8443}], + "readinessProbe": { + "tcpSocket": {"port": 8443}, + "periodSeconds": 1, + }, + "env": [ + { + "name": "D3_BIND_HOST", + "value": str(ipaddress.IPv4Address(0)), + } + ], + "resources": {"limits": {"cpu": "100m", "memory": "128Mi"}}, + "volumeMounts": [ + { + "name": "tls", + "mountPath": "/proof/tls", + "readOnly": True, + }, + { + "name": "scripts", + "mountPath": "/scripts", + "readOnly": True, + }, + ], + } + ], + "volumes": [ + { + "name": "tls", + "hostPath": {"path": "/proof/tls", "type": "Directory"}, + }, + {"name": "scripts", "configMap": {"name": "actual-scripts"}}, + ], + }, + } + ) + return {"apiVersion": "v1", "kind": "List", "items": items} + + +def command( + *args: str, + input_text: str | None = None, + timeout: float = 180, + environment: dict[str, str] | None = None, +) -> str: + executable = shutil.which(args[0]) + if executable is None or Path(executable).name not in { + "docker", + "podman", + "kubectl", + "kind", + "openssl", + "systemd-run", + "systemctl", + }: + raise ValueError("unapproved/missing proof executable") + if Path(executable).name == "kind": + if ( + os.environ.get("KIND_EXPERIMENTAL_PROVIDER", "docker") != "docker" + or (environment or {}).get("KIND_EXPERIMENTAL_PROVIDER", "docker") != "docker" + ): + raise ValueError("kind provider must be explicitly Docker; fallback forbidden") + environment = {**os.environ, **(environment or {}), "KIND_EXPERIMENTAL_PROVIDER": "docker"} + + async def execute() -> str: + with anyio.fail_after(timeout): + async with await anyio.open_process( + [executable, *args[1:]], + stdin=subprocess.PIPE if input_text is not None else subprocess.DEVNULL, + env=environment, + ) as process: + captured = [bytearray(), bytearray()] + + async def capture(stream: ByteReceiveStream | None, target: bytearray) -> None: + if stream is None: + return + async for part in stream: + if len(target) + len(part) > 4 << 20: + process.kill() + raise ValueError("proof diagnostic output exceeds 4MiB") + target.extend(part) + + async def send() -> None: + if process.stdin is not None: + await process.stdin.send((input_text or "").encode()) + await process.stdin.aclose() + + async with anyio.create_task_group() as tasks: + tasks.start_soon(capture, process.stdout, captured[0]) + tasks.start_soon(capture, process.stderr, captured[1]) + tasks.start_soon(send) + exit_code = await process.wait() + if exit_code: + raise RuntimeError( + f"{args[0]} exit{exit_code}: " + captured[1].decode(errors="replace") + ) + return captured[0].decode() + + return anyio.run(execute) + + +def key(name: str) -> Keypair: + return Keypair(bytes.fromhex((FIXTURE / f"{name}.seed").read_text().strip())) + + +def signed(model: BaseModel, signer: Keypair | None = None) -> RelayEnvelope: + return relay_envelope.parse_envelope( + relay_envelope.seal(signer or key("master"), type(model).__name__, RUN, model, 10000) + ) + + +def registry() -> RelayRegistryV1: + return RelayRegistryV1.model_validate_json((FIXTURE / "registry.json").read_bytes()) + + +def credentials() -> S3Credentials: + # Kubernetes projected Secrets are root-owned; copy to an owned restrictive file. + with tempfile.NamedTemporaryFile() as file: + file.write((FIXTURE / "object.json").read_bytes()) + file.flush() + os.fchmod(file.fileno(), 0o600) + return S3Credentials.from_secret_file(Path(file.name)) + + +def backing(client: httpx.AsyncClient, region: str) -> StreamStore: + return StreamStore( + S3Store( + "http://objects.hypertrain-proof-store.svc:9000", + "proof", + credentials(), + region="local", + prefix=f"relay/{region}/", + ), + client, + ) + + +def fixture_store() -> FastAPI: + """Tiny durable S3 subset, validates real SigV4 and conditional writes.""" + app = FastAPI() + directory = OBJECT_ROOT + directory.mkdir(exist_ok=True) + lock = anyio.Lock() + + @app.get("/livez") + async def live() -> dict[str, bool]: + return {"live": True} + + @app.api_route("/proof/relay/{region}/{name}", methods=["PUT", "GET", "DELETE"]) + async def object_request(region: Literal["eu", "us"], name: str, request: Request) -> Response: + if not re.fullmatch(r"[0-9a-f]{64}(\.journal)?", name): + raise HTTPException(400, "bad key") + sha = request.headers.get("x-amz-content-sha256", "") + stamp = request.headers.get("x-amz-date", "") + extra = { + k: request.headers[k] for k in ("if-match", "if-none-match") if k in request.headers + } + expected = sigv4_headers( + request.method, + str(request.url), + credentials(), + "local", + stamp, + sha, + extra=extra, + ) + if not hmac.compare_digest( + expected["Authorization"], request.headers.get("authorization", "") + ): + raise HTTPException(403, "SigV4 denied") + regional = directory / region + regional.mkdir(exist_ok=True) + path = regional / name + + def sync_directory() -> None: + fd = os.open(regional, os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + + if request.method == "DELETE": + async with lock: + path.unlink(missing_ok=True) + sync_directory() + return JSONResponse({}) + if request.method == "GET": + # Bind ETag/body to the same immutable inode through atomic journal replacement. + async with lock: + if not path.exists(): + raise HTTPException(404, "missing") + file = path.open("rb") + size = os.fstat(file.fileno()).st_size + etag = hashlib.file_digest(file, "sha256").hexdigest() + file.seek(0) + + async def content() -> AsyncIterator[bytes]: + try: + async for part in blocks(file): + yield part + finally: + file.close() + + return StreamingResponse( + content(), + headers={"ETag": '"' + etag + '"', "Content-Length": str(size)}, + ) + with spool() as upload_file: + digest, count = hashlib.sha256(), 0 + async for part in request.stream(): + count += len(part) + if count > 16 << 20: + raise HTTPException(413, "fixture object quota") + upload_file.write(part) + digest.update(part) + if digest.hexdigest() != sha: + raise HTTPException(400, "payload hash mismatch") + async with lock: + current = None + if path.exists(): + with path.open("rb") as current_file: + current = ( + '"' + hashlib.file_digest(current_file, "sha256").hexdigest() + '"' + ) + if (extra.get("if-none-match") == "*" and current is not None) or ( + "if-match" in extra and extra["if-match"] != current + ): + raise HTTPException(412, "conditional mismatch") + upload_file.seek(0) + tmp = directory / (name + ".tmp") + with tmp.open("wb") as output: + while part := upload_file.read(BLOCK): + output.write(part) + output.flush() + os.fsync(output.fileno()) + tmp.replace(path) + sync_directory() + return JSONResponse({}, headers={"ETag": '"' + sha + '"'}) + + return app + + +def fixture_master() -> FastAPI: + """Explicit transport test master; independent reads before durable acceptance.""" + app = FastAPI() + MASTER_ROOT.mkdir(exist_ok=True) + database = MASTER_ROOT / "transport.sqlite" + with sqlite3.connect(database) as db: + db.execute( + "CREATE TABLE IF NOT EXISTS grants (hash TEXT PRIMARY KEY, envelope TEXT NOT NULL)" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS receipts (grant_hash TEXT PRIMARY KEY, " + "receipt TEXT NOT NULL, acceptance TEXT NOT NULL)" + ) + + @app.get("/livez") + async def live() -> dict[str, bool]: + return {"live": True} + + @app.post("/v2/relay-receipts") + async def accept(request: Request) -> JSONResponse: + raw = relay_envelope.parse_envelope(await request.body()) + receipt = RelayReceipt.model_validate(raw.body) + with sqlite3.connect(database) as db: + row = db.execute( + "SELECT envelope FROM grants WHERE hash=?", (receipt.grant_hash,) + ).fetchone() + if row is None: + raise HTTPException(400, "unregistered grant") + grant = UploadGrant.model_validate(relay_envelope.parse_envelope(row[0]).body) + receipt_verified( + raw, + registry=registry(), + relay_id=grant.relay_id, + run_id=RUN, + grant=grant, + received_before=50, + ) + async with httpx.AsyncClient(timeout=30, follow_redirects=False) as client: + with spool() as file: + await backing(client, grant.relay_id.split("-", 1)[0]).get( + grant.delta_hash, grant.size, file + ) + accepted = signed( + MasterAcceptanceV2( + w=grant.w, + hotkey=grant.hotkey, + delta_hash=grant.delta_hash, + receipt_hash=receipt.digest(), + received_drand=20, + ) + ) + with sqlite3.connect(database) as db: + db.execute("BEGIN IMMEDIATE") + prior = db.execute( + "SELECT receipt, acceptance FROM receipts WHERE grant_hash=?", + (receipt.grant_hash,), + ).fetchone() + if prior is not None: + previous = RelayReceipt.model_validate(relay_envelope.parse_envelope(prior[0]).body) + if previous != receipt: + raise HTTPException(409, "conflicting receipt") + accepted = relay_envelope.parse_envelope(prior[1]) + else: + db.execute( + "INSERT INTO receipts VALUES (?,?,?)", + ( + receipt.grant_hash, + raw.model_dump_json(), + accepted.model_dump_json(), + ), + ) + return JSONResponse(accepted.model_dump(mode="json")) + + @app.post("/register") + async def register(request: Request) -> JSONResponse: + raw = relay_envelope.parse_envelope(await request.body()) + if ( + raw.signer != key("master").ss58 + or raw.type != "UploadGrant" + or raw.run_id != RUN + or raw.exp_drand < 20 + or not relay_envelope.verify_envelope(raw.model_dump()) + ): + raise HTTPException(403, "grant denied") + grant = UploadGrant.model_validate(raw.body) + if grant.exp_drand < 20: + raise HTTPException(400, "expired grant") + with sqlite3.connect(database) as db: + db.execute( + "INSERT OR IGNORE INTO grants VALUES (?,?)", + (grant.digest(), raw.model_dump_json()), + ) + return JSONResponse({"registered": grant.digest()}) + + return app + + +def fixture_gateway() -> FastAPI: + @asynccontextmanager + async def lifespan(app: FastAPI) -> AsyncIterator[None]: + async with httpx.AsyncClient( + timeout=60, follow_redirects=False, limits=httpx.Limits(max_connections=8) + ) as client: + app.state.client = client + yield + + app = FastAPI(lifespan=lifespan) + + @app.api_route("/{region}/{path:path}", methods=["GET", "POST", "PUT"]) + async def proxy(region: Literal["eu", "us", "master"], path: str, request: Request) -> Response: + target = ( + "http://master.hypertrain-proof.svc:8000" + if region == "master" + else f"http://hypertrain-relay.hypertrain-relay-{region}.svc:8000" + ) + client: httpx.AsyncClient = app.state.client + headers = { + k: v + for k, v in request.headers.items() + if k.lower() not in {"host", "connection", "transfer-encoding"} + } + upstream = client.build_request( + request.method, + target + "/" + path, + params=request.query_params, + headers=headers, + content=request.stream(), + ) + response = await client.send(upstream, stream=True) + + async def content() -> AsyncIterator[bytes]: + try: + async for part in response.aiter_raw(BLOCK): + yield part + finally: + await response.aclose() + + return StreamingResponse( + content(), + status_code=response.status_code, + headers={ + k: v + for k, v in response.headers.items() + if k.lower() not in {"connection", "transfer-encoding"} + }, + ) + + return app + + +def fixture_relay() -> FastAPI: + @asynccontextmanager + async def lifespan(app: FastAPI) -> AsyncIterator[None]: + region = os.environ["REGION"] + async with httpx.AsyncClient( + verify=str(FIXTURE / "ca.crt"), + timeout=30, + follow_redirects=False, + limits=httpx.Limits(max_connections=8), + ) as client: + + async def now() -> int: + return 20 # deterministic proof beacon, no production time claim + + async def forward(raw: RelayEnvelope) -> RelayEnvelope: + return await forward_receipt(client, GATEWAY + "/master", raw) + + relay = Relay( + run_id=RUN, + master=(FIXTURE / "master.public").read_text().strip(), + registry=registry(), + network_manifest_hash=registry().digest(), + observers={(FIXTURE / "master.public").read_text().strip()}, + relay_id=f"{region}-1", + region=region, + keys={"k1": key(region)}, + active_key="k1", + backing=backing(client, region), + now=now, + forward=forward, + ) + app.mount("/", create_app(relay, (FIXTURE / "drain.token").read_text().strip())) + await relay.recover() + yield + + return FastAPI(lifespan=lifespan) + + +async def fixture_miner() -> None: + payload = bytes(range(256)) * (10 << 12) # exactly 10MiB + digest = hashlib.sha256(payload).hexdigest() + path = Path(tempfile.mkdtemp(prefix="miner-")) / "delta" + await anyio.to_thread.run_sync(path.write_bytes, payload) + manifest = chunk_manifest(path) + reg = registry() + async with httpx.AsyncClient( + verify=str(FIXTURE / "ca.crt"), timeout=90, follow_redirects=False + ) as client: + relay_client = RelayClient(reg, client) + originals: list[str] = [] + for identity in range(4): + region = "eu" if identity < 2 else "us" + fallback = os.environ.get("PROOF_STAGE") == "fallback" + if fallback and identity != 0: + continue + if fallback: + region = "us" + round_index = 4 if fallback else identity + miner = Keypair(hashlib.sha256(f"proof-miner-{identity}".encode()).digest()) + grant = UploadGrant( + w=round_index, + hotkey=miner.ss58, + relay_id=f"{region}-1", + assignment_epoch=0, + delta_hash=digest, + size=len(payload), + chunk_manifest_hash=manifest.chunk_manifest_hash(), + retain_until=500, + nonce=hashlib.sha256(f"proof-grant-{round_index}".encode()).hexdigest(), + exp_drand=50, + ) + raw = signed(grant) + registered = await client.post( + GATEWAY + "/master/register", content=raw.model_dump_json() + ) + registered.raise_for_status() + assignment = RelayAssignment( + w=round_index, + hotkey=miner.ss58, + primary_id="eu-1", + fallback_ids=["us-1"], + assignment_epoch=0, + manifest_hash=reg.digest(), + exp_drand=50, + ) + h = "12" * 32 + opened = RoundOpenV2.model_validate( + { + "w": round_index, + **{ + name: h + for name in ( + "prev_final_hash", + "theta_hash", + "outer_state_hash", + "center_hash", + "roster_hash", + "honeypot_commit", + "start_state_index_hash", + ) + }, + "d_open": 1, + "d_assign": 2, + "d_commit": 10, + "d_audit": 15, + "d_upload": 50, + "d_final": 60, + "contract_version": 2, + "policy_hashes": { + name: h + for name in ( + "admission_policy_hash", + "economics_policy_hash", + "aggregation_policy_hash", + "dispute_policy_hash", + "audit_policy_hash", + ) + }, + "registry_epoch": 0, + "audit_mode": "anchored-full", + "roster": [], + } + ) + round_raw = envelope_v2.seal(key("master"), "RoundOpenV2", RUN, opened, 100) + if os.environ.get("PROOF_STAGE") == "prefix": + url = reg.specs[0].https_url + accepted = await client.post( + url + f"/v1/uploads/{grant.digest()}/accept", + json={ + "grant": raw.model_dump(mode="json"), + "manifest": manifest.body(), + "assignment": signed(assignment).model_dump(mode="json"), + "round_open": round_raw, + }, + ) + accepted.raise_for_status() + capability = base64.b64encode(raw.model_dump_json().encode()).decode() + uploaded = await client.put( + url + f"/v1/uploads/{grant.digest()}/chunks/0", + content=payload[: 4 << 20], + headers={"X-Upload-Grant": capability}, + ) + uploaded.raise_for_status() + print( + json.dumps({"event": "PREFIX_DURABLE", "grant": grant.digest()}), + flush=True, + ) + return + receipt_raw = await relay_client.upload( + path, grant_raw=raw, assignment=signed(assignment), round_open=round_raw + ) + receipt = RelayReceipt.model_validate(receipt_raw.body) + request = RetrievalRequest( + request_id=hashlib.sha256(f"proof-request-{round_index}".encode()).hexdigest(), + nonce=hashlib.sha256(f"proof-nonce-{round_index}".encode()).hexdigest(), + relay_id=f"{region}-1", + key_id="k1", + assignment_hash=assignment.digest(), + grant_hash=grant.digest(), + custody_ack_hashes=[], + receipt_hash=receipt.digest(), + retention_hash=receipt.digest(), + object_or_chunk_hash=digest, + size=len(payload), + requested_beacon=20, + deadline_beacon=30, + ) + with spool() as file: + response = await relay_client.retrieval( + signed(request), file, relay_public=receipt_raw.signer + ) + assert RetrievalResponse.model_validate(response.body).status == "SERVED" + assert hashlib.sha256(file.read()).hexdigest() == digest + originals.append(receipt.digest()) + print( + json.dumps( + { + "event": "FOUR_ORIGINAL_OBJECTS_VERIFIED", + "sha256": digest, + "receipts": originals, + "bytes_per_object": len(payload), + } + ), + flush=True, + ) + + +async def fixture_gc() -> None: + """Transport-only signed retention controls; never evidence of L0 finality.""" + clock = 490 + + async def now() -> int: + return clock + + async def settlement(grant_hash: str) -> Settlement: + # Explicit fixture settlement, used only by internal transport GC test. + return Settlement("12" * 32, 100, "34" * 32, 120, 110) + + async with httpx.AsyncClient(timeout=30) as client: + eu = Relay( + run_id=RUN, + master=key("master").ss58, + registry=registry(), + network_manifest_hash=registry().digest(), + observers={key("master").ss58}, + relay_id="eu-1", + region="eu", + keys={"k1": key("eu")}, + active_key="k1", + backing=backing(client, "eu"), + now=now, + settlement=settlement, + ) + # Separate privileged transport verifier; extensions still go to the actual pod. + state = await eu.state() + if not state.uploads or any(u.receipt is None for u in state.uploads.values()): + raise ValueError("GC proof requires actual complete relay custody") + extensions: dict[str, list[str]] = {} + async with httpx.AsyncClient(verify=str(FIXTURE / "ca.crt"), timeout=30) as tls: + for grant_hash, record in state.uploads.items(): + extensions[grant_hash] = [] + for digest in record.pins: + extension = RetentionExtension( + grant_hash=grant_hash, + custody_hashes=[digest], + seq=1, + previous_retention_hash=digest, + dispute_ids=["56" * 32], + issued_beacon=20, + retain_until=600, + ) + response = await tls.post( + GATEWAY + f"/eu/v1/uploads/{grant_hash}/retention", + content=signed(extension).model_dump_json(), + ) + response.raise_for_status() + extensions[grant_hash].append(extension.digest()) + clock = 550 + assert not await eu.collect(), "unreleased extended custody was deleted" + digest = next(iter(state.uploads.values())).manifest.chunks[0].chunk_sha256 + with spool() as file: + await backing(client, "us").get(digest, 4 << 20, file) + clock = 1001 + for grant_hash, record in (await eu.state()).uploads.items(): + for custody_hash, pin in record.pins.items(): + await eu.release( + signed( + CustodyRelease( + grant_hash=grant_hash, + custody_hashes=[custody_hash], + retention_hash=pin.retention_hash, + finality_hash="12" * 32, + vesting_beacon=110, + closed_dispute_root="34" * 32, + release_beacon=1000, + ) + ) + ) + deleted = await eu.collect() + assert deleted, "released expired EU custody did not collect" + with spool() as file: + await backing(client, "us").get(digest, 4 << 20, file) + print( + json.dumps( + { + "event": "TRANSPORT_RETENTION_GC_PREFIX_ISOLATION", + "deleted": deleted, + "L0_finality": "NOT_TESTED", + } + ), + flush=True, + ) + + +def fixture(role: str) -> None: + import uvicorn + + host = socket.gethostbyname(socket.gethostname()) + match role: + case "store": + uvicorn.run(fixture_store(), host=host, port=9000) + case "master": + uvicorn.run(fixture_master(), host=host, port=8000) + case "gateway": + uvicorn.run( + fixture_gateway(), + host=host, + port=8443, + ssl_keyfile=str(FIXTURE / "tls.key"), + ssl_certfile=str(FIXTURE / "tls.crt"), + ) + case "relay": + uvicorn.run(fixture_relay(), host=host, port=8000) + case "miner": + anyio.run(fixture_miner) + case "gc": + anyio.run(fixture_gc) + case _: + raise ValueError("unknown fixture role") + + +def apply(kubectl: list[str], resource: object) -> None: + command(*kubectl, "apply", "-f", "-", input_text=json.dumps(resource)) + + +class CleanupJournal: + """Atomic, fsynced evidence for an external lifecycle supervisor.""" + + def __init__(self, path: Path) -> None: + self.path = path + + def write_text(self, data: str) -> None: + incoming = json.loads(data) + if self.path.exists(): + previous = json.loads(self.path.read_bytes()) + for key in ( + "cluster", + "experiment", + "proof_dir", + "kubeconfig", + "deadline_monotonic", + "cleanup_cutoff_monotonic", + "cleanup_budget_cutoff_monotonic", + "boot_id", + "containers", + "volumes", + "networks", + "provider_baselines", + ): + if key in previous and key in incoming and previous[key] != incoming[key]: + raise ValueError("immutable journal field changed: " + key) + incoming = {**previous, **incoming} + data = json.dumps(incoming) + with tempfile.NamedTemporaryFile(dir=self.path.parent, delete=False) as file: + file.write(data.encode()) + file.flush() + os.fsync(file.fileno()) + temporary = file.name + os.replace(temporary, self.path) + fd = os.open(self.path.parent, os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + + +@contextmanager +def transport_archive(args): + """Admitted native archive, pinned by no-follow descriptor and Linux read lease.""" + from hypertrain.aggregator.capacity_worker import _directory_fd, _read_at + + if args.mode != "transport" or args.local_image_id or args.image: + raise ValueError("archive requires transport-only exclusive image route") + if not all( + ( + args.archive_sha256, + args.archive_image_id, + args.archive_reference, + args.source_manifest, + args.source_manifest_sha256, + ) + ): + raise ValueError("archive requires admitted hash/image/reference/source") + if ( + not re.fullmatch(r"[0-9a-f]{64}", args.archive_sha256) + or not re.fullmatch(r"sha256:[0-9a-f]{64}", args.archive_image_id) + or not re.fullmatch(r"[a-z0-9][a-z0-9._/-]*:[a-z0-9][a-z0-9._-]*", args.archive_reference) + ): + raise ValueError("archive image/hash/reference invalid") + source = Path(args.source_manifest) + with _directory_fd(source.parent) as parent: + source_data = _read_at(parent, Path(source.name), 1 << 20) + if hashlib.sha256(source_data).hexdigest() != args.source_manifest_sha256: + raise ValueError("frozen source manifest digest mismatch") + source_map = json.loads(source_data) + if set(source_map) != {"files"} or not isinstance(source_map["files"], dict): + raise ValueError("source manifest requires exact files map") + files = source_map["files"] + if len(files) != 109 or any( + not key.startswith("src/hypertrain/") + or ".." in Path(key).parts + or not re.fullmatch(r"[0-9a-f]{64}", value) + for key, value in files.items() + ): + raise ValueError("archive requires frozen source109") + archive = Path(args.image_archive) + with _directory_fd(archive.parent) as parent: + fd = os.open(archive.name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=parent) + previous = signal.getsignal(signal.SIGIO) + leased = False + try: + identity = os.fstat(fd) + if ( + not stat.S_ISREG(identity.st_mode) + or identity.st_uid != os.geteuid() + or identity.st_mode & 0o222 + or identity.st_nlink != 1 + or not 0 < identity.st_size <= 2 << 30 + ): + raise ValueError("archive must be owned readonly single-link bounded file") + + def reject_write(*_): + raise ValueError("archive write/lease break refused") + + signal.signal(signal.SIGIO, reject_write) + fcntl.fcntl(fd, fcntl.F_SETLEASE, fcntl.F_RDLCK) + leased = True + with os.fdopen(os.dup(fd), "rb") as stream: + if hashlib.file_digest(stream, "sha256").hexdigest() != args.archive_sha256: + raise ValueError("archive SHA256 differs from admission") + stream.seek(0) + with tarfile.open(fileobj=stream, mode="r:") as tar: + + def document(name): + member = tar.getmember(name) + if not member.isfile() or not 0 < member.size <= 1 << 20: + raise ValueError("archive metadata must be bounded regular file") + content = tar.extractfile(member) + assert content is not None + return content.read() + + names = tar.getnames() + if len(names) != len(set(names)): + raise ValueError("duplicate archive members") + manifest = json.loads(document("manifest.json")) + config_name = "blobs/sha256/" + args.archive_image_id.split(":", 1)[1] + if ( + len(manifest) != 1 + or manifest[0]["Config"] != config_name + or manifest[0]["RepoTags"] != [args.archive_reference] + ): + raise ValueError("archive stored reference/config mismatch") + config_data = document(config_name) + if "sha256:" + hashlib.sha256(config_data).hexdigest() != args.archive_image_id: + raise ValueError("archive config digest mismatch") + config = json.loads(config_data) + if ( + config.get("os"), + config.get("architecture"), + config.get("config", {}).get("User"), + ) != ("linux", "amd64", "65532:65532"): + raise ValueError("archive requires linux/amd64 nonroot65532") + index = json.loads(document("index.json")) + descriptors = index["manifests"] + if len(descriptors) != 1 or descriptors[0]["annotations"].get( + "io.containerd.image.name" + ) != ("docker.io/library/" + args.archive_reference): + raise ValueError("archive OCI reference mismatch") + descriptor = descriptors[0] + if not re.fullmatch(r"sha256:[0-9a-f]{64}", descriptor["digest"]): + raise ValueError("archive manifest descriptor invalid") + manifest_data = document( + "blobs/sha256/" + descriptor["digest"].split(":", 1)[1] + ) + if ( + len(manifest_data) != descriptor["size"] + or "sha256:" + hashlib.sha256(manifest_data).hexdigest() + != descriptor["digest"] + ): + raise ValueError("archive OCI manifest mismatch") + oci = json.loads(manifest_data) + if oci["config"]["digest"] != args.archive_image_id or oci["config"][ + "size" + ] != len(config_data): + raise ValueError("archive OCI config mismatch") + if not oci["layers"] or any( + not re.fullmatch(r"sha256:[0-9a-f]{64}", d["digest"]) for d in oci["layers"] + ): + raise ValueError("archive layer descriptors invalid") + layers = ["blobs/sha256/" + d["digest"].split(":", 1)[1] for d in oci["layers"]] + if layers != manifest[0]["Layers"] or any( + not tar.getmember(name).isfile() or tar.getmember(name).size != d["size"] + for name, d in zip(layers, oci["layers"], strict=True) + ): + raise ValueError("archive layer closure mismatch") + # A path rename cannot substitute the descriptor passed to kind. + yield f"/proc/{os.getpid()}/fd/{fd}", args.archive_reference, files + finally: + if leased: + fcntl.fcntl(fd, fcntl.F_SETLEASE, fcntl.F_UNLCK) + os.close(fd) + signal.signal(signal.SIGIO, previous) + + +def load_transport_image(cluster, image, archive=None): + command( + "kind", + "load", + "image-archive" if archive else "docker-image", + archive[0] if archive else image, + "--name", + cluster, + timeout=240, + ) + + +def verify_archive_node(node, reference, image_id, files, kubectl): + info = json.loads( + command("docker", "exec", node, "crictl", "inspecti", "-o", "json", reference) + ) + if ( + info["status"]["id"] != image_id + or "docker.io/library/" + reference not in info["status"]["repoTags"] + ): + raise ValueError("owned node imported image identity differs") + check = """import hashlib,json,pathlib,os,hypertrain +assert os.getuid()==65532 +expected=json.loads(os.environ['SOURCE109']) +root=pathlib.Path(hypertrain.__file__).parent +actual={} +for p in root.rglob('*'): + if p.is_file() and '__pycache__' not in p.parts and p.suffix!='.pyc': + key='src/hypertrain/'+p.relative_to(root).as_posix() + actual[key]=hashlib.sha256(p.read_bytes()).hexdigest() +assert actual==expected +import hypertrain.relay.app +print('ARCHIVE_NODE_SOURCE109_RELAY_IMPORT_VERIFIED',flush=True) +""" + apply( + kubectl, + { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": {"name": "archive-source-check"}, + "spec": { + "backoffLimit": 0, + "activeDeadlineSeconds": 60, + "template": { + "spec": { + "restartPolicy": "Never", + "nodeName": node, + "securityContext": { + "runAsUser": 65532, + "runAsGroup": 65532, + "runAsNonRoot": True, + "fsGroup": 65532, + }, + "volumes": [ + {"name": "data", "emptyDir": {"medium": "Memory", "sizeLimit": "16Mi"}} + ], + "containers": [ + { + "name": "check", + "image": reference, + "imagePullPolicy": "Never", + "command": ["/opt/venv/bin/python", "-c", check], + "volumeMounts": [{"name": "data", "mountPath": "/data"}], + "env": [ + {"name": "SOURCE109", "value": json.dumps(files)}, + {"name": "OMP_NUM_THREADS", "value": "1"}, + ], + "resources": { + "requests": {"cpu": "50m", "memory": "64Mi"}, + "limits": {"cpu": "500m", "memory": "256Mi"}, + }, + } + ], + } + }, + }, + }, + ) + command( + *kubectl, "wait", "--for=condition=complete", "job/archive-source-check", "--timeout=90s" + ) + log = command(*kubectl, "logs", "job/archive-source-check") + if log.strip() != "ARCHIVE_NODE_SOURCE109_RELAY_IMPORT_VERIFIED": + raise ValueError("owned node source109/import verification failed") + return log + + +def verify_local_image(image_id: str, manifest: Path, digest: str) -> str: + """Local byte evidence, not an attestation or repository manifest digest.""" + if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id): + raise ValueError("local image requires actual Docker image ID") + data = manifest.read_bytes() + if hashlib.sha256(data).hexdigest() != digest: + raise ValueError("frozen source manifest digest mismatch") + parsed = json.loads(data) + if set(parsed) != {"files"} or not isinstance(parsed["files"], dict): + raise ValueError("manifest requires exact files hash map") + files = parsed["files"] + if len(data) > 1 << 20 or len(files) > 1024: + raise ValueError("source manifest exceeds bounded scope") + required = { + "src/hypertrain/relay/core.py", + "src/hypertrain/relay/app.py", + "src/hypertrain/data/stream_store.py", + } + if not required <= files.keys() or any( + not isinstance(name, str) + or not isinstance(value, str) + or not name.startswith("src/hypertrain/") + or ".." in Path(name).parts + or not re.fullmatch(r"[0-9a-f]{64}", value) + for name, value in files.items() + ): + raise ValueError("invalid/missing frozen source hashes") + inspected = json.loads(command("docker", "image", "inspect", image_id)) + if len(inspected) != 1 or inspected[0]["Id"] != image_id: + raise ValueError("Docker image inspection differs from admitted image ID") + check = ( + "import hashlib,json,pathlib,sys,hypertrain; f=json.loads(sys.stdin.read()); " + "root=pathlib.Path(hypertrain.__file__).parent; " + "actual={'src/hypertrain/'+p.relative_to(root).as_posix() for p in root.rglob('*') " + "if p.is_file() and '__pycache__' not in p.parts and p.suffix!='.pyc'}; " + "assert actual==set(f),(actual-set(f),set(f)-actual); " + "bad=[n for n,h in f.items() if hashlib.sha256(" + "(root/n.removeprefix('src/hypertrain/')).read_bytes()).hexdigest()!=h]; " + "assert not bad,bad; print('LOCAL_SOURCE_BYTES_VERIFIED')" + ) + result = command( + "docker", + "run", + "--rm", + "-i", + "--tmpfs", + "/data:rw,size=16m,uid=65532,gid=65532", + "--cpuset-cpus=0-3", + "--cpus=0.5", + "--memory=256m", + "--memory-swap=256m", + "--entrypoint", + "/opt/venv/bin/python", + image_id, + "-c", + check, + input_text=json.dumps(files), + ) + if result.strip() != "LOCAL_SOURCE_BYTES_VERIFIED": + raise ValueError("image source verification did not finish") + tag = "hypertrain-relay-proof:" + image_id.split(":", 1)[1] + command("docker", "tag", image_id, tag) + return tag + + +def validate_node_limits(limits: dict) -> None: + quota = limits.get("NanoCpus", 0) / 1e9 + if not quota and limits.get("CpuPeriod", 0) > 0: + quota = limits.get("CpuQuota", 0) / limits["CpuPeriod"] + if ( + limits.get("CpusetCpus") != "0-3" + or not 0 < quota <= 3.5 + or not 0 < limits.get("Memory", 0) <= 4 << 30 + or limits.get("MemorySwap") != limits["Memory"] + ): + raise ValueError("node lacks admitted positive CPU/memory/swap constraints") + + +def runner_admission(*, archive: bool = False) -> None: + """Archive reserves caller0.05/128MiB within original aggregate4CPU/4.5GiB.""" + group = next( + line.split(":", 2)[2] + for line in Path("/proc/self/cgroup").read_text().splitlines() + if line.startswith("0::") + ) + path = Path("/sys/fs/cgroup") / group.lstrip("/") + quota, period = (path / "cpu.max").read_text().split() + memory = (path / "memory.max").read_text().strip() + if ( + quota == "max" + or not 0 < int(quota) / int(period) <= (0.40 if archive else 0.45) + or memory == "max" + or not 0 < int(memory) <= (256 if archive else 384) << 20 + or (path / "memory.swap.max").read_text().strip() != "0" + or set(os.sched_getaffinity(0)) - {0, 1, 2, 3} + ): + raise ValueError("runner exceeds admitted CPU/memory/swap/cpuset budget") + + +def integration_workload( + args: argparse.Namespace, image: str, kubectl: list[str], node: str +) -> dict: + """Root-only live execution, real authority/driver; no fixture credit or manifest mutation.""" + prepared = Path(args.integration_root).resolve(strict=True) + if (prepared / "rescue").exists() or (prepared / "state/service").exists(): + raise ValueError("fresh integration state/rescue required") + config = IntegrationServiceConfig.model_validate_json( + (prepared / "configs/service.json").read_bytes() + ) + inspect_integration_snapshot(prepared / "bundle", config.admitted_hash) + for name in ( + "relay_kind_smoke.py", + "network_authority_snapshot.py", + "network_service_proof.py", + ): + digest = hashlib.sha256((ROOT / "scripts" / name).read_bytes()).hexdigest() + if digest != json.loads(Path(args.integration_scripts).read_bytes())[name]: + raise ValueError("integration script differs from root-admitted source freeze") + command("docker", "exec", node, "mkdir", "-p", "/proof") + command("docker", "cp", str(prepared) + "/.", node + ":/proof") + command("docker", "exec", node, "chown", "-R", "65532:65532", "/proof") + apply( + kubectl, + { + "apiVersion": "v1", + "kind": "Namespace", + "metadata": {"name": "hypertrain-d3"}, + }, + ) + apply( + kubectl, + { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": {"name": "actual-scripts", "namespace": "hypertrain-d3"}, + "data": { + name: (ROOT / "scripts" / name).read_text() + for name in ( + "relay_kind_smoke.py", + "network_authority_snapshot.py", + "network_service_proof.py", + ) + }, + }, + ) + rendered = integration_manifests(image, local_loaded=bool(args.local_image_id)) + job = next(i for i in rendered["items"] if i["kind"] == "Job") + for item in rendered["items"]: + if item["kind"] not in {"Job", "Namespace"} and not ( + item["kind"] == "Pod" and item["metadata"]["name"] != "service" + ): + apply(kubectl, item) + for role in ("service", "relay", "gateway"): + if role != "service": + apply( + kubectl, + next( + i + for i in rendered["items"] + if i["kind"] == "Pod" and i["metadata"]["name"] == role + ), + ) + # Runtime readiness is an exact kube event; gateway TLS is tested by driver. + command( + *kubectl, + "wait", + "--for=condition=Ready", + "pod/" + role, + "-n", + "hypertrain-d3", + "--timeout=900s", + timeout=920, + ) + relay_pod = next( + i for i in rendered["items"] if i["kind"] == "Pod" and i["metadata"]["name"] == "relay" + ) + before = json.loads(command(*kubectl, "get", "pod/relay", "-n", "hypertrain-d3", "-o", "json"))[ + "metadata" + ]["uid"] + command( + *kubectl, + "delete", + "pod/relay", + "-n", + "hypertrain-d3", + "--wait=true", + "--timeout=60s", + ) + apply(kubectl, relay_pod) + command( + *kubectl, + "wait", + "--for=condition=Ready", + "pod/relay", + "-n", + "hypertrain-d3", + "--timeout=60s", + ) + after = json.loads(command(*kubectl, "get", "pod/relay", "-n", "hypertrain-d3", "-o", "json"))[ + "metadata" + ]["uid"] + if before == after: + raise ValueError("pod fault did not replace real relay") + negative = json.loads(json.dumps(job)) + negative["metadata"]["name"] = "wrong-ca" + negative["spec"]["activeDeadlineSeconds"] = 60 + neg_container = negative["spec"]["template"]["spec"]["containers"][0] + neg_container["command"][neg_container["command"].index("driver")] = "wrong-ca" + neg_container["resources"] = { + "requests": {"cpu": "100m", "memory": "128Mi"}, + "limits": {"cpu": "100m", "memory": "128Mi"}, + } + neg_container["env"] = [ + {"name": name, "value": "1"} for name in ("OMP_NUM_THREADS", "MKL_NUM_THREADS") + ] + apply(kubectl, negative) + command( + *kubectl, + "wait", + "--for=condition=complete", + "job/wrong-ca", + "-n", + "hypertrain-d3", + "--timeout=60s", + ) + if "TLS_WRONG_CA_REJECTED" not in command( + *kubectl, "logs", "job/wrong-ca", "-n", "hypertrain-d3" + ): + raise ValueError("real wrong-CA negative lacks TLS refusal evidence") + apply(kubectl, job) + try: + command( + *kubectl, + "wait", + "--for=condition=complete", + "job/driver", + "-n", + "hypertrain-d3", + "--timeout=9600s", + timeout=9620, + ) + finally: + command("docker", "cp", node + ":/proof/.", str(prepared / "rescue"), timeout=300) + evidence = prepared / "rescue/work/driver/evidence" + result = json.loads((evidence / "result.json").read_bytes()) + if result["status"] != "PASS": + raise ValueError("actual driver failed; rescued evidence preserved") + return { + "full_D3_integration": "ACTUAL_DRIVER_PASS", + "driver": result, + "relay_replaced_uids": [before, after], + "wrong_ca": "TLS_WRONG_CA_REJECTED", + "registry": "original-local", + "rescue": str(evidence), + } + + +def prepare_integration_root(destination: Path, bundle_dir: Path, digest: str) -> None: + """Write exact public role/config topology; original private secrets/TLS supplied by root.""" + import network_authority_snapshot as authority + + authority.validate(bundle_dir, digest) + destination.mkdir(mode=0o700, parents=True, exist_ok=False) + shutil.copytree(bundle_dir, destination / "bundle") + for name in ( + "configs", + "state", + "tls", + "secrets/service", + "secrets/relay", + "secrets/driver", + "work/service", + "work/relay", + "work/driver", + ): + (destination / name).mkdir(mode=0o700, parents=True, exist_ok=True) + configs = { + "service": { + "snapshot": "/proof/bundle", + "admitted_hash": digest, + "state": "/proof/state/service", + "secrets": "/proof/secrets", + "master_url": "https://master.test", + "host": str(ipaddress.IPv4Address(0)), + "port": 8080, + }, + "relay": { + "state": "/proof/state/service", + "backing": "/proof/work/relay", + "ca": "/proof/tls/ca.crt", + "master_url": "https://master.test", + "relay_seed": "/proof/secrets/relay-0.seed", + "admin_token": "/proof/secrets/admin.token", + "drain_token": "/proof/secrets/drain.token", + "host": str(ipaddress.IPv4Address(0)), + "port": 8000, + }, + "driver": { + "master_https": "https://master.test", + "relay_https": "https://relay.test", + "ca": "/proof/tls/ca.crt", + "snapshot": "/proof/bundle", + "snapshot_digest": digest, + "secrets": "/proof/secrets", + "service_state": "/proof/state/service", + "evidence": "/proof/work/evidence", + "verified_beacon": "/proof/work/verified-beacon.json", + "fixture_clock_directory": "/proof/state/service/fixture-clock", + "fixture_clock_offset": 0.0, + "rounds": 2, + "child_timeout": 900, + }, + } + for role, data in configs.items(): + (destination / "configs" / (role + ".json")).write_text(json.dumps(data, indent=2)) + sources = { + name: hashlib.sha256((ROOT / "scripts" / name).read_bytes()).hexdigest() + for name in ( + "relay_kind_smoke.py", + "network_authority_snapshot.py", + "network_service_proof.py", + ) + } + (destination / "scripts.json").write_text(json.dumps(sources, sort_keys=True)) + + +def provision_original_fixture_secrets(root: Path, bundle_dir: Path, digest: str) -> None: + """Materialize original test identities separately; never sign historical operations.""" + import network_authority_snapshot as authority + + bundle = authority.validate(bundle_dir, digest) + original = { + "coord": bytes(range(32)), + "owner": bytes([113]) * 32, + "auditor-0": bytes(31) + b"\x01", + "auditor-1": bytes([114]) * 32, + "referee-0": bytes([115]) * 32, + "relay-0": bytes([116]) * 32, + **{f"hot-{i}": bytes([80 + i]) * 32 for i in range(4)}, + **{f"cold-{i}": bytes([90 + i]) * 32 for i in range(4)}, + } + by_public = {Keypair(raw).ss58: raw for raw in original.values()} + if set(by_public) != set(bundle.public_roles.values()): + raise ValueError("known original fixture identities differ from admitted bundle") + original = {role: by_public[public] for role, public in bundle.public_roles.items()} + for role, names in { + "service": set(original), + "driver": {"coord", "auditor-0", "auditor-1", *(f"hot-{i}" for i in range(4))}, + "relay": {"relay-0"}, + }.items(): + directory = root / "secrets" / role + for name in names: + path = directory / (name + ".seed") + with path.open("xb") as stream: + stream.write(original[name]) + path.chmod(0o600) + with (directory / "admin.token").open("x") as stream: + stream.write("admin-test-token") + (directory / "admin.token").chmod(0o600) + if role == "relay": + with (directory / "drain.token").open("x") as stream: + stream.write(secrets.token_hex(32)) + (directory / "drain.token").chmod(0o600) + + +def provider_inventory( + provider: str, execute: Callable[..., str] = command +) -> dict[str, list[str]]: + """Successful parsed snapshots only; malformed output is never absence.""" + result = {} + for resource, argv in ( + ("containers", ["ps", "-aq", "--no-trunc"]), + ("volumes", ["volume", "ls", "-q"]), + ("networks", ["network", "ls", "-q", "--no-trunc"]), + ): + if provider == "podman" and resource == "networks": + try: + rows = json.loads(execute(provider, "network", "ls", "--format", "json")) + except json.JSONDecodeError as error: + raise ValueError("malformed podman networks inventory") from error + if not isinstance(rows, list) or any( + not isinstance(row, dict) + or not isinstance(row.get("id"), str) + or not isinstance(row.get("name"), str) + or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", row["name"]) + for row in rows + ): + raise ValueError("malformed podman networks inventory") + values = [row["id"] for row in rows] + names = [row["name"] for row in rows] + if len(names) != len(set(names)): + raise ValueError("duplicate podman network names") + result["network_names"] = sorted(names) + else: + values = execute(provider, *argv).splitlines() + pattern = r"[0-9a-f]{64}" if resource != "volumes" else r"[A-Za-z0-9][A-Za-z0-9_.-]*" + if len(values) != len(set(values)) or any(not re.fullmatch(pattern, v) for v in values): + raise ValueError("malformed " + provider + " " + resource + " inventory") + result[resource] = sorted(values) + return result + + +def admitted_host_config_matches(actual: dict, expected: dict) -> bool: + """Keep admitted resources/security immutable; Docker default false/null is equivalent.""" + fields = ( + "Memory", + "MemorySwap", + "NanoCpus", + "CpuQuota", + "CpuPeriod", + "CpusetCpus", + "Privileged", + "SecurityOpt", + "CgroupnsMode", + "Init", + "Binds", + "Tmpfs", + ) + for name in fields: + if ( + (name in actual) != (name in expected) + or type(actual.get(name)) is not type(expected.get(name)) + or actual.get(name) != expected.get(name) + ): + return False + before, after = expected.get("OomKillDisable"), actual.get("OomKillDisable") + if "OomKillDisable" in expected and "OomKillDisable" not in actual: + return False + if before is None or before is False: + return after is None or after is False + return before is True and after is True + + +def require_capped_node(journal_path: Path, node: str) -> dict: + journal = json.loads(journal_path.read_bytes()) + info = json.loads(command("docker", "inspect", node))[0] + if ( + not journal.get("node_capped_before_start") + or info["Id"] not in journal.get("owned_containers", []) + or not admitted_host_config_matches(info["HostConfig"], journal["create_host_config"]) + ): + raise ValueError("kind returned without journal-owned pre-start caps") + validate_node_limits(info["HostConfig"]) + if info["Config"]["Labels"].get("io.hypertrain.proof") != journal["experiment"]: + raise ValueError("created node proof label mismatch") + return info + + +def cleanup_owned(journal_path: Path) -> None: + initial = json.loads(journal_path.read_bytes()) + cutoff = initial.get( + "cleanup_budget_cutoff_monotonic", + min( + initial["cleanup_cutoff_monotonic"], + time.monotonic() + 1200, + ), + ) + + def bounded(*args: str, timeout: float = 180) -> str: + remaining = cutoff - time.monotonic() + if remaining <= 0: + state = json.loads(journal_path.read_bytes()) + CleanupJournal(journal_path).write_text( + json.dumps( + { + **state, + "state": "CLEANUP_CUTOFF", + "proof_status": "CENSORED", + "absence_verified": False, + } + ) + ) + raise TimeoutError("aggregate cleanup cutoff exhausted") + try: + return command(*args, timeout=min(timeout, remaining)) + except Exception: + state = json.loads(journal_path.read_bytes()) + CleanupJournal(journal_path).write_text( + json.dumps( + { + **state, + "proof_status": "CENSORED", + "absence_verified": False, + } + ) + ) + raise + + """Serialize rescue/cleanup; deletion never touches a baseline resource.""" + with journal_path.with_suffix(".lock").open("a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + journal = json.loads(journal_path.read_bytes()) + if journal.get("absence_verified"): + journal.setdefault("absence_history", []).append( + { + "state": journal["state"], + "provider_after": journal.get("provider_after"), + "absence_verified": True, + } + ) + journal["absence_verified"] = False + journal["provider_after"] = None + journal["cleanup_budget_cutoff_monotonic"] = cutoff + CleanupJournal(journal_path).write_text(json.dumps(journal)) + try: + cluster = journal["cluster"] + if not re.fullmatch(r"hypertrain-v2-[a-z0-9-]{1,40}", cluster): + raise ValueError("deadline journal cluster not owned") + if journal["proof_dir"] != str(journal_path.parent.resolve()): + raise ValueError("journal proof directory mismatch") + labelled = bounded( + "docker", + "ps", + "-aq", + "--no-trunc", + "--filter", + "label=io.x-k8s.kind.cluster=" + cluster, + ).splitlines() + errors = [] + for provider, baseline_provider in journal["provider_baselines"].items(): + if provider == "docker": + continue + current = provider_inventory(provider, bounded) + for identity in set(current["containers"]) - set(baseline_provider["containers"]): + info = json.loads(bounded(provider, "inspect", identity))[0] + if info["Config"]["Labels"].get("io.x-k8s.kind.cluster") != cluster: + continue + if ( + info["Id"] != identity + or info["Config"]["Labels"].get("io.hypertrain.proof") + != journal["experiment"] + ): + raise ValueError("unexpected provider node lacks verified proof ownership") + rescue_other = journal_path.parent / "rescue" / provider + rescue_other.mkdir(parents=True, exist_ok=True) + (rescue_other / (identity + "-inspect.json")).write_text(json.dumps(info)) + if info["State"]["Running"]: + bounded(provider, "stop", "--time", "3", identity, timeout=15) + try: + (rescue_other / (identity + "-logs.txt")).write_text( + bounded(provider, "logs", "--tail", "100", identity, timeout=30) + ) + except Exception as error: + errors.append(provider + ": " + str(error)) + bounded(provider, "rm", identity) + for mount in info["Mounts"]: + if ( + mount["Type"] != "volume" + or mount["Name"] in baseline_provider["volumes"] + ): + continue + volume = json.loads(bounded(provider, "volume", "inspect", mount["Name"]))[ + 0 + ] + if volume.get("Labels", {}).get(cluster + "-control-plane") == "true": + refs = bounded( + provider, "ps", "-aq", "--filter", "volume=" + mount["Name"] + ) + if not refs.strip(): + bounded(provider, "volume", "rm", mount["Name"]) + for container in labelled: + info = json.loads(bounded("docker", "inspect", container))[0] + if ( + info["Id"] in journal["containers"] + or info["Config"]["Labels"].get("io.hypertrain.proof") != journal["experiment"] + ): + raise ValueError("cluster ownership conflicts with journal") + journal.setdefault("owned_containers", []).append(info["Id"]) + for mount in info["Mounts"]: + if mount["Type"] == "volume" and mount["Name"] not in journal["volumes"]: + journal.setdefault("owned_volumes", []).append(mount["Name"]) + for network in info["NetworkSettings"]["Networks"].values(): + if network["NetworkID"] not in journal["networks"]: + journal.setdefault("owned_networks", []).append(network["NetworkID"]) + rescue = journal_path.parent / "rescue" + rescue.mkdir(exist_ok=True) + journal["state"] = "RESCUE_INTENT" + CleanupJournal(journal_path).write_text(json.dumps(journal)) + if labelled: + kube = [ + "kubectl", + "--kubeconfig", + journal["kubeconfig"], + "--context", + "kind-" + cluster, + ] + captures = [ + ( + ["kind", "export", "logs", str(rescue / "logs"), "--name", cluster], + "kind-logs.txt", + ), + ( + kube + + [ + "get", + "pods,jobs,services,configmaps,events", + "-A", + "-o", + "json", + ], + "objects.json", + ), + ] + for argv, name in captures: + try: + (rescue / name).write_text(bounded(*argv, timeout=60)) + except Exception as error: + errors.append(name + ": " + str(error)) + paths = [ + "/var/local/hypertrain-proof-store", + "/var/local/hypertrain-proof-master", + ] + if journal.get("integration_root"): + paths = ["/proof/work", "/proof/state"] + for path in paths: + try: + bounded( + "docker", + "cp", + cluster + "-control-plane:" + path, + str(rescue / Path(path).name), + timeout=120, + ) + except Exception as error: + errors.append(path + ": " + str(error)) + journal["rescue_errors"] = errors + journal["proof_status"] = ( + "CENSORED" if errors else journal.get("proof_status", "INTERRUPTED") + ) + journal["state"] = "DELETE_INTENT" + CleanupJournal(journal_path).write_text(json.dumps(journal)) + try: + if labelled: + bounded("kind", "delete", "cluster", "--name", cluster, timeout=120) + except Exception as error: + journal["delete_error"] = str(error) + journal["proof_status"] = "CENSORED" + finally: + remaining = set(bounded("docker", "ps", "-aq", "--no-trunc").splitlines()) + for identity in set(journal.get("owned_containers", [])) & remaining: + bounded("docker", "rm", "-f", identity) + for resource, field in ( + ("volume", "owned_volumes"), + ("network", "owned_networks"), + ): + argv = ["docker", resource, "ls", "-q"] + if resource == "network": + argv.append("--no-trunc") + remaining = set(bounded(*argv).splitlines()) + for identity in set(journal.get(field, [])) & remaining: + bounded("docker", resource, "rm", identity) + after = { + provider: provider_inventory(provider, bounded) + for provider in journal["provider_baselines"] + } + if any(after[p] != journal["provider_baselines"][p] for p in after): + journal["state"] = "CLEANUP_FAILED" + journal["proof_status"] = "CENSORED" + CleanupJournal(journal_path).write_text(json.dumps(journal)) + raise ValueError("cleanup inventory differs from admitted baseline") + journal["state"] = "CLEANUP_VERIFIED" + journal["absence_verified"] = True + journal["provider_after"] = after + CleanupJournal(journal_path).write_text(json.dumps(journal)) + + except Exception: + current = json.loads(journal_path.read_bytes()) + current["absence_verified"] = False + current["provider_after"] = None + current["proof_status"] = "CENSORED" + if current["state"] != "CLEANUP_CUTOFF": + current["state"] = "CLEANUP_FAILED" + CleanupJournal(journal_path).write_text(json.dumps(current)) + raise + + +def deadline_supervisor( + journal_path: Path, parent_pid: int, ready_socket: Path | None = None +) -> None: + """READY before CREATE; pidfd and immutable absolute monotonic deadline.""" + journal = json.loads(journal_path.read_bytes()) + if journal["boot_id"] != Path("/proc/sys/kernel/random/boot_id").read_text().strip(): + raise ValueError("supervisor clock boot mismatch") + try: + fd = os.pidfd_open(parent_pid) + except ProcessLookupError: + fd = None + if ready_socket is not None: + group = Path("/proc/self/cgroup").read_text().strip() + cgroup = Path("/sys/fs/cgroup") / group.split("::")[1].lstrip("/") + controls = { + name: (cgroup / name).read_text().strip() + for name in ("cpu.max", "memory.max", "memory.swap.max", "cpuset.cpus.effective") + } + quota, period = controls["cpu.max"].split() + if ( + quota == "max" + or int(quota) / int(period) > 0.05 + or int(controls["memory.max"]) > 128 << 20 + or controls["memory.swap.max"] != "0" + or controls["cpuset.cpus.effective"] != "0-3" + ): + raise ValueError("supervisor unit exceeds admitted reserve") + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as connection: + connection.settimeout(30) + connection.connect(str(ready_socket)) + connection.sendall( + json.dumps( + { + "state": "READY", + "unit": journal["cluster"] + "-supervisor", + "cgroup": group, + "controls": controls, + } + ).encode() + ) + if fd is not None: + try: + poll = select.poll() + poll.register(fd, select.POLLIN) + remaining = max(0, journal["deadline_monotonic"] - time.monotonic()) + if not poll.poll(math.ceil(remaining * 1000)): + signal.pidfd_send_signal(fd, signal.SIGKILL) + poll.poll(30000) + finally: + os.close(fd) + cleanup_owned(journal_path) + print("SUPERVISOR_CLEANUP_DONE", flush=True) + + +def start_supervisor(journal_path: Path) -> None: + journal = json.loads(journal_path.read_bytes()) + unit = journal["cluster"] + "-supervisor" + if ( + command("systemctl", "show", unit + ".service", "--property=LoadState", "--value").strip() + != "not-found" + ): + raise ValueError("pre-existing supervisor unit ownership conflict") + remaining = journal["cleanup_cutoff_monotonic"] - time.monotonic() + if remaining <= 0: + raise ValueError("supervisor cutoff already expired") + listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + ready_directory = tempfile.TemporaryDirectory(prefix="ht-ready-") + ready_path = Path(ready_directory.name) / "ready.sock" + listener.bind(str(ready_path)) + listener.listen(1) + listener.settimeout(30) + created = False + try: + command( + "systemd-run", + "--unit=" + unit, + "--property=CPUQuota=5%", + "--property=MemoryMax=128M", + "--property=MemorySwapMax=0", + "--property=AllowedCPUs=0-3", + "--property=Nice=19", + "--property=RuntimeMaxSec=" + str(remaining), + "--property=KillMode=control-group", + "--setenv=OMP_NUM_THREADS=1", + "--setenv=MKL_NUM_THREADS=1", + "--setenv=PYTHONDONTWRITEBYTECODE=1", + "--setenv=PATH=" + os.environ["PATH"], + "--property=StandardOutput=append:" + str(journal_path.parent / "supervisor.log"), + "--property=StandardError=append:" + str(journal_path.parent / "supervisor.log"), + sys.executable, + str(Path(__file__).resolve()), + "--supervise", + str(journal_path), + "--parent-pid", + str(os.getpid()), + "--ready-socket", + str(ready_path), + ) + created = True + with listener.accept()[0] as connection: + connection.settimeout(30) + ready = json.loads(connection.recv(2048)) + own_group = Path("/proc/self/cgroup").read_text().strip() + if ready["state"] != "READY" or ready["unit"] != unit or ready["cgroup"] == own_group: + raise ValueError("supervisor unit independence not confirmed") + CleanupJournal(journal_path).write_text(json.dumps({"supervisor_ready": ready})) + except Exception: + if created: + command("systemctl", "stop", unit) + raise + finally: + listener.close() + ready_path.unlink(missing_ok=True) + ready_directory.cleanup() + + +def docker_create_hook(journal_path: Path, docker: str, args: list[str]) -> None: + """Kind CLI local provider: create immutable limits, inspect before ANY start.""" + + def exit_status(status: int) -> None: + if status < 0: + if -status not in {signal.SIGKILL, signal.SIGSTOP}: + signal.signal(-status, signal.SIG_DFL) + os.kill(os.getpid(), -status) + sys.exit(status) + + def forward(argv: list[str]) -> None: + pid = os.posix_spawn(docker, [docker, *argv], os.environ) + exit_status(os.waitstatus_to_exitcode(os.waitpid(pid, 0)[1])) + + def budget(cleanup: bool = False) -> float: + current = json.loads(journal_path.read_bytes()) + cutoff = min( + current["cleanup_cutoff_monotonic"], + current.get("cleanup_budget_cutoff_monotonic", current["cleanup_cutoff_monotonic"]), + ) + if not cleanup: + cutoff = min(cutoff, current["deadline_monotonic"]) + remaining = cutoff - time.monotonic() + if remaining <= 0: + raise TimeoutError("node gate absolute budget exhausted") + return remaining + + def read_control(path: Path) -> str: + budget() + value = path.read_text().strip() + budget() + return value + + def save_receipt(receipt: dict, *, cleanup: bool = False) -> None: + budget(cleanup) + CleanupJournal(journal_path).write_text(json.dumps({"node_kernel_receipt": receipt})) + budget(cleanup) + + def start_owned(identity: str) -> None: + receipt: dict = { + "identity": identity, + "verified": False, + "controls": {}, + "cgroup_read_reached": False, + } + try: + remaining = budget() + + async def start() -> int: + with anyio.fail_after(min(60, remaining)): + async with await anyio.open_process( + [docker, "start", identity], stdin=None, stdout=None, stderr=None + ) as process: + return await process.wait() + + status = anyio.run(start) + if status: + exit_status(status) + info = json.loads(command(docker, "inspect", identity, timeout=min(10, budget())))[0] + pid = info["State"]["Pid"] + receipt["pid"] = pid + resource_fields = ( + "NanoCpus", + "CpuQuota", + "CpuPeriod", + "Memory", + "MemorySwap", + "CpusetCpus", + "Privileged", + "SecurityOpt", + "CgroupnsMode", + "Init", + "Binds", + "Tmpfs", + "OomKillDisable", + ) + receipt["observed"] = { + "container_id": str(info["Id"])[:256], + "pid": pid, + "running": info["State"]["Running"], + "proof_label": str(info["Config"]["Labels"].get("io.hypertrain.proof"))[:256], + "cluster_label": str(info["Config"]["Labels"].get("io.x-k8s.kind.cluster"))[:256], + "resources": {name: info["HostConfig"].get(name) for name in resource_fields}, + } + receipt["expected"] = { + "container_id": identity, + "pid": "positive strict integer", + "running": True, + "proof_label": journal["cluster"], + "cluster_label": journal["cluster"], + "resources": { + name: journal["create_host_config"].get(name) for name in resource_fields + }, + } + receipt["host_config_changed_fields"] = sorted( + name + for name in info["HostConfig"].keys() | journal["create_host_config"].keys() + if info["HostConfig"].get(name) != journal["create_host_config"].get(name) + or (name in info["HostConfig"]) != (name in journal["create_host_config"]) + )[:128] + receipt["rejection_reasons"] = [ + reason + for reason, failed in ( + ("container_id", info["Id"] != identity), + ("running", not info["State"]["Running"]), + ("pid", type(pid) is not int or pid <= 0), + ( + "host_config", + not admitted_host_config_matches( + info["HostConfig"], journal["create_host_config"] + ), + ), + ( + "proof_label", + info["Config"]["Labels"].get("io.hypertrain.proof") != journal["cluster"], + ), + ) + if failed + ] + if receipt["rejection_reasons"]: + raise ValueError("started node identity/PID mismatch") + if not re.fullmatch(r"[0-9a-f]{64}", identity): + raise ValueError("node requires full owned container ID") + receipt["cgroup_read_reached"] = True + group = read_control(Path(f"/proc/{pid}/cgroup")) + receipt["cgroup"] = group + if not group.startswith("0::/") or "\n" in group: + raise ValueError("node requires unified cgroup") + relative = group.removeprefix("0::/") + if not relative or ".." in Path(relative).parts: + raise ValueError("invalid node cgroup path") + parts = Path(relative).parts + authority = next( + ( + i + for i, part in enumerate(parts) + if ( + i == 1 and parts[0] == "system.slice" and part == f"docker-{identity}.scope" + ) + or (i == 1 and parts[0] == "docker" and part == identity) + ), + None, + ) + if authority is None: + raise ValueError("node cgroup does not match exact owned container ID") + parent = Path("/sys/fs/cgroup").joinpath(*parts[: authority + 1]) + receipt["authority_cgroup"] = str(parent) + for name in ("memory.max", "cpu.max", "memory.swap.max", "cpuset.cpus.effective"): + receipt["controls"][name] = read_control(parent / name) + quota, period = map(int, receipt["controls"]["cpu.max"].split()) + if ( + quota <= 0 + or period <= 0 + or quota * 2 != period * 7 + or receipt["controls"]["memory.max"] != str(4 << 30) + or receipt["controls"]["memory.swap.max"] != "0" + or receipt["controls"]["cpuset.cpus.effective"] != "0-3" + ): + raise ValueError("node kernel limits mismatch") + receipt["descendants"] = {} + for depth in range(authority + 2, len(parts) + 1): + descendant = Path("/sys/fs/cgroup").joinpath(*parts[:depth]) + controls: dict[str, str] = {} + receipt["descendants"][str(descendant)] = controls + for name in ("memory.max", "cpu.max", "memory.swap.max", "cpuset.cpus.effective"): + controls[name] = read_control(descendant / name) + # Ancestor limits apply to every descendant, including leaf "max". + for name in ("memory.max", "memory.swap.max"): + if controls[name] != "max" and int(controls[name]) < 0: + raise ValueError("invalid descendant kernel limit") + leaf_quota, leaf_period = controls["cpu.max"].split() + if int(leaf_period) <= 0 or (leaf_quota != "max" and int(leaf_quota) <= 0): + raise ValueError("invalid descendant CPU limit") + if controls["cpuset.cpus.effective"] != "0-3": + raise ValueError("descendant effective CPU set mismatch") + receipt["effective_bound"] = {"cpu": 3.5, "memory": 4 << 30, "swap": 0} + current = json.loads(command(docker, "inspect", identity, timeout=min(10, budget())))[0] + if current["Id"] != identity or current["State"] != info["State"]: + raise ValueError("node identity/PID changed during kernel inspection") + if read_control(Path(f"/proc/{pid}/cgroup")) != group: + raise ValueError("node cgroup changed during kernel inspection") + receipt["verified"] = True + save_receipt(receipt) + except Exception as error: + receipt["verified"] = False + receipt["error"] = str(error) + try: + save_receipt(receipt, cleanup=True) + except Exception as diagnostic_error: + receipt["diagnostic_error"] = str(diagnostic_error) + try: + command(docker, "stop", "--time=0", identity, timeout=min(30, budget(True))) + receipt["stopped"] = True + except Exception as stop_error: + receipt["stop_error"] = str(stop_error) + finally: + try: + save_receipt(receipt, cleanup=True) + except Exception as diagnostic_error: + receipt["diagnostic_error"] = str(diagnostic_error) + raise + + journal = json.loads(journal_path.read_bytes()) + if not args: + raise ValueError("empty Docker operation") + operation = args[0] + if args in (["-v"], ["--version"]): + forward(args) + if operation == "pull": + versions = Versions.model_validate_json((ROOT / "deploy/k8s/versions.json").read_bytes()) + if args != ["pull", versions.node_image]: + raise ValueError("pull requires exact trusted pinned kind node image") + forward(args) + if operation in {"create", "run"}: + cluster = journal["cluster"] + if any( + a.startswith( + ("--cpus", "--cpuset", "--cpu-", "--memory", "--label=io.hypertrain.proof") + ) + for a in args[1:] + ): + raise ValueError("kind cannot override admitted node resource policy") + identity = command( + docker, + "create", + "--cpuset-cpus=0-3", + "--cpus=3.5", + "--memory=4g", + "--memory-swap=4g", + "--label=io.hypertrain.proof=" + cluster, + *[a for a in args[1:] if a not in {"-d", "--detach"} and not a.startswith("--detach=")], + timeout=min(180, budget()), + ).strip() + info = json.loads(command(docker, "inspect", identity, timeout=min(10, budget())))[0] + validate_node_limits(info["HostConfig"]) + if info["State"]["Running"] or info["Id"] in journal["containers"]: + raise ValueError("node create must be stopped and new") + labels = info["Config"]["Labels"] + if ( + labels.get("io.hypertrain.proof") != cluster + or labels.get("io.x-k8s.kind.cluster") != cluster + ): + raise ValueError("node create ownership label mismatch") + budget() + CleanupJournal(journal_path).write_text( + json.dumps( + { + "owned_containers": [info["Id"]], + "create_host_config": info["HostConfig"], + "node_capped_before_start": True, + } + ) + ) + budget() + if operation == "run": + journal = json.loads(journal_path.read_bytes()) + start_owned(info["Id"]) + else: + print(identity) + elif operation == "start": + if len(args) != 2: + raise ValueError("start requires one exact owned node") + for identity in args[1:]: + if identity.startswith("-"): + raise ValueError("unexpected kind start option") + info = json.loads(command(docker, "inspect", identity, timeout=min(10, budget())))[0] + validate_node_limits(info["HostConfig"]) + if info["Id"] not in journal.get( + "owned_containers", [] + ) or not admitted_host_config_matches( + info["HostConfig"], journal["create_host_config"] + ): + raise ValueError("start requires journal-owned capped node") + start_owned(info["Id"]) + elif operation in {"update", "container"}: + raise ValueError("Docker operation bypasses immutable create policy") + elif operation in { + "inspect", + "info", + "version", + "ps", + "images", + "image", + "load", + "save", + "exec", + "cp", + "network", + "rm", + "logs", + "wait", + "stop", + "kill", + }: + forward(args) + else: + raise ValueError("unsupported local kind Docker operation: " + operation) + + +def smoke(args: argparse.Namespace) -> None: + if args.image_archive: + archive_cutoffs(args) + with transport_archive(args) as archive: + _smoke(args, archive) + else: + if any((args.archive_sha256, args.archive_image_id, args.archive_reference)): + raise ValueError("archive admission fields require image-archive") + _smoke(args) + + +class CleanupEvents: + """File/process subscriptions armed before dispatch; no timed polling.""" + + def __init__(self, proof: Path) -> None: + self.proof = proof + self.libc = ctypes.CDLL(None, use_errno=True) + self.fd = self.libc.inotify_init1(os.O_NONBLOCK | os.O_CLOEXEC) + if self.fd < 0: + raise OSError(ctypes.get_errno(), "inotify_init1") + self.processes: dict[int, int] = {} + self.watched = False + if self.libc.inotify_add_watch(self.fd, os.fsencode(proof.parent), 0x100 | 0x80) < 0: + os.close(self.fd) + raise OSError(ctypes.get_errno(), "inotify parent watch") + + def wait(self, cutoff: float, supervisor_pid: int) -> None: + if not self.watched and self.proof.exists(): + if self.libc.inotify_add_watch(self.fd, os.fsencode(self.proof), 0x8 | 0x80) < 0: + raise OSError(ctypes.get_errno(), "inotify proof watch") + self.watched = True + return # Re-read after arming: closes the receipt-before-watch race. + if supervisor_pid > 0 and supervisor_pid not in self.processes: + try: + self.processes[supervisor_pid] = os.pidfd_open(supervisor_pid) + except ProcessLookupError: + return # Terminal state can now be read without a polling delay. + remaining = cutoff - time.monotonic() + if remaining <= 0: + raise TimeoutError("caller original cleanup cutoff exhausted") + ready, _, _ = select.select([self.fd, *self.processes.values()], [], [], remaining) + if not ready: + raise TimeoutError("caller original cleanup cutoff exhausted") + if self.fd in ready: + os.read(self.fd, 65536) + for pid, fd in list(self.processes.items()): + if fd in ready: + os.close(fd) + del self.processes[pid] + + def close(self) -> None: + for fd in self.processes.values(): + os.close(fd) + os.close(self.fd) + + +def caller_resources() -> float: + """Require the finite 5%/128MiB caller service within the inclusive resource budget.""" + group = Path("/proc/self/cgroup").read_text().strip().removeprefix("0::") + root = Path("/sys/fs/cgroup") / group.lstrip("/") + quota, period = (root / "cpu.max").read_text().split() + if ( + quota == "max" + or not 0 < int(quota) / int(period) <= 0.05 + or not 0 < int((root / "memory.max").read_text()) <= 128 << 20 + or (root / "memory.swap.max").read_text().strip() != "0" + or (root / "cpuset.cpus.effective").read_text().strip() != "0-3" + ): + raise ValueError("caller requires 5% CPU/128MiB/swap0/cores0-3") + unit = Path(group).name + properties = dict( + line.split("=", 1) + for line in command( + "systemctl", + "show", + unit, + "--property=ActiveEnterTimestampMonotonic", + "--property=RuntimeMaxUSec", + ).splitlines() + ) + if properties["RuntimeMaxUSec"] != "3h 22min": + raise ValueError("caller requires finite 12120s outer cap") + return int(properties["ActiveEnterTimestampMonotonic"]) / 1e6 + 12120 + + +def caller_launch(args: argparse.Namespace, runner_argv: list[str]) -> None: + """Keep flock through supervisor terminal/verified absence, otherwise persist INCOMPLETE.""" + if not re.fullmatch(r"[a-z0-9][a-z0-9-]{1,80}", args.caller_unit or ""): + raise ValueError("caller requires unique outer unit") + if args.mode != "transport" or not args.image_archive or args.deadline_seconds != 10800: + raise ValueError("caller requires admitted archive transport 10800s work") + proof = Path(args.proof_dir).resolve() + if not proof.parent.is_dir() or proof.exists(): + raise ValueError("caller requires existing owned parent and unique proof path") + caller_cap = caller_resources() + lock_path = Path(args.caller_lock) + barrier = CleanupJournal(ROOT / ".transport-admission.json") + with os.fdopen(os.open(lock_path, os.O_RDWR | os.O_NOFOLLOW), "a") as lock: + info = os.fstat(lock.fileno()) + if not stat.S_ISREG(info.st_mode) or info.st_uid != os.geteuid() or info.st_mode & 0o022: + raise ValueError("caller lock must be existing owned regular no-follow file") + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + if barrier.path.exists() and json.loads(barrier.path.read_bytes())["state"] != "VERIFIED": + raise ValueError("prior caller admission INCOMPLETE; no dispatch allowed") + started = time.monotonic() + work_end = started + 10800 + cutoff = work_end + 1200 + if cutoff >= caller_cap: + raise ValueError("admitted cleanup endpoint exceeds finite caller cap") + events = CleanupEvents(proof) + cancelled = False + + def cancel(*_): + nonlocal cancelled + cancelled = True # Ownership survives cancellation; supervisor owns cleanup. + + previous = {sig: signal.signal(sig, cancel) for sig in (signal.SIGINT, signal.SIGTERM)} + journal_path = proof / "cleanup.json" + unit = args.caller_unit + ".service" + supervisor = args.cluster + "-supervisor.service" + boot_id = Path("/proc/sys/kernel/random/boot_id").read_text().strip() + expected_baseline = { + p: provider_inventory(p, lambda *argv: command(*argv, timeout=10)) + for p in ("docker", "podman") + } + primary_error = None + try: + # A killed caller leaves ACTIVE, which also blocks subsequent admission. + barrier.write_text( + json.dumps( + { + "state": "ACTIVE", + "caller_proof": str(proof), + "caller_work_end": work_end, + "caller_cleanup_end": cutoff, + "caller_boot_id": boot_id, + } + ) + ) + try: + command( + "systemd-run", + "--unit=" + args.caller_unit, + "--property=CPUQuota=40%", + "--property=MemoryMax=256M", + "--property=MemorySwapMax=0", + "--property=AllowedCPUs=0-3", + "--property=Nice=19", + "--property=RuntimeMaxSec=12000", + "--property=KillMode=control-group", + "--setenv=OMP_NUM_THREADS=1", + "--setenv=MKL_NUM_THREADS=1", + "--setenv=PYTHONDONTWRITEBYTECODE=1", + "--setenv=KIND_EXPERIMENTAL_PROVIDER=docker", + "--setenv=PATH=" + os.environ["PATH"], + "--setenv=TMPDIR=" + os.environ["TMPDIR"], + "--property=WorkingDirectory=" + str(ROOT), + sys.executable, + str(Path(__file__).resolve()), + *runner_argv, + "--launch-unit", + unit, + "--work-end", + repr(work_end), + "--cleanup-end", + repr(cutoff), + "--clock-boot-id", + boot_id, + "--caller-cap", + repr(caller_cap), + timeout=30, + ) + except (OSError, RuntimeError, TimeoutError, ValueError) as error: + primary_error = error # Dispatch may have succeeded; custody cannot unwind. + + def bounded(*argv, timeout=10): + remaining = cutoff - time.monotonic() + if remaining <= 0: + raise TimeoutError("caller original cleanup cutoff exhausted") + return command(*argv, timeout=min(timeout, remaining)) + + while True: # Iterations resume only on native file/process notifications. + pid = 0 + try: + state = json.loads(journal_path.read_bytes()) if journal_path.exists() else None + if state and state.get("supervisor_ready"): + ready = state["supervisor_ready"] + if ( + state["cluster"] != args.cluster + or state["experiment"] != args.cluster + or state["proof_dir"] != str(proof) + or state["boot_id"] != boot_id + or ready["state"] != "READY" + or ready["unit"] != supervisor.removesuffix(".service") + or ready["cgroup"].strip() != "0::/system.slice/" + supervisor + or state["provider_baselines"] != expected_baseline + or len(state["owned_containers"]) != 1 + or any( + not re.fullmatch(r"[0-9a-f]{64}", identity) + or identity in expected_baseline["docker"]["containers"] + for identity in state["owned_containers"] + ) + ): + raise ValueError( + "caller cleanup receipt subject differs from admitted ownership" + ) + if ( + state["cleanup_cutoff_monotonic"] != cutoff + or state["deadline_monotonic"] != work_end + ): + raise ValueError("caller and journal cutoff differ") + status = dict( + line.split("=", 1) + for line in bounded( + "systemctl", + "show", + supervisor, + "--property=MainPID", + "--property=ActiveState", + ).splitlines() + ) + pid = int(status["MainPID"]) + log = proof / "supervisor.log" + done = ( + log.exists() + and "SUPERVISOR_CLEANUP_DONE" in log.read_text().splitlines() + ) + if done and pid == 0 and status["ActiveState"] in {"inactive", "failed"}: + if ( + state["state"] != "CLEANUP_VERIFIED" + or not state.get("absence_verified") + or set(state["provider_baselines"]) != {"docker", "podman"} + ): + raise ValueError( + "supervisor terminal without exact cleanup receipt" + ) + fresh = { + p: provider_inventory(p, bounded) for p in ("docker", "podman") + } + if ( + fresh != state["provider_baselines"] + or fresh != state["provider_after"] + ): + raise ValueError( + "fresh caller inventories differ from verified cleanup" + ) + barrier.write_text(json.dumps({"state": "VERIFIED"})) + break + except (OSError, RuntimeError, TimeoutError, ValueError, KeyError) as error: + if primary_error is None: + primary_error = error + end = cutoff + try: + events.wait(end, pid) + except TimeoutError as error: + if primary_error is not None: + raise primary_error from error + raise + except OSError as error: + if primary_error is None: + primary_error = error + # Broken subscription cannot justify early custody release. + select.select([], [], [], max(0, end - time.monotonic())) + raise primary_error from error + finally: + if json.loads(barrier.path.read_bytes()).get("state") != "VERIFIED": + barrier.write_text(json.dumps({"state": "INCOMPLETE"})) + if journal_path.exists(): + try: + CleanupJournal(journal_path).write_text( + json.dumps({"caller_status": "INCOMPLETE"}) + ) + except OSError: + if primary_error is None: + raise + events.close() + for sig, handler in previous.items(): + signal.signal(sig, handler) + if primary_error is not None: + raise primary_error + if cancelled: + raise KeyboardInterrupt("caller cancelled; verified cleanup completed before release") + + +def archive_cutoffs(args: argparse.Namespace) -> None: + """Exact caller-admitted monotonic endpoints, validated before archive admission.""" + if not args.launch_unit or not re.fullmatch( + r"[a-z0-9][a-z0-9-]{1,80}\.service", args.launch_unit + ): + raise ValueError("archive launch requires exact outer service unit") + if Path("/proc/self/cgroup").read_text().strip() != "0::/system.slice/" + args.launch_unit: + raise ValueError("archive runner cgroup differs from admitted launch unit") + now = time.monotonic() + if ( + args.clock_boot_id != Path("/proc/sys/kernel/random/boot_id").read_text().strip() + or args.deadline_seconds != 10800 + or not all( + value is not None and math.isfinite(value) + for value in (args.work_end, args.cleanup_end, args.caller_cap) + ) + or not now < args.work_end <= now + 10800 + or args.cleanup_end - args.work_end != 1200 + or not args.cleanup_end < args.caller_cap <= now + 12120 + ): + raise ValueError("archive boot/deadline endpoints invalid or beyond caller cap") + + +def archive_workload(text: str, image: str) -> str: + """Known fixture/rendered YAML image lines: no remote pull fallback.""" + return re.sub( + r"(?m)^([ \t]*)image: " + re.escape(image) + r"\n(?:[ \t]*imagePullPolicy: [^\n]+\n)?", + lambda match: f"{match[1]}image: {image}\n{match[1]}imagePullPolicy: Never\n", + text, + ) + + +def _smoke(args: argparse.Namespace, archive=None) -> None: + """Only --create enters cluster lifecycle; --render performs no cluster calls.""" + if archive and time.monotonic() >= args.work_end: + raise TimeoutError("archive admission exhausted original work deadline") + barrier = ROOT / ".transport-admission.json" + if args.create and barrier.exists(): + caller_state = json.loads(barrier.read_bytes()) + if caller_state["state"] == "INCOMPLETE" or ( + caller_state["state"] == "ACTIVE" + and caller_state["caller_proof"] != str(Path(args.proof_dir).resolve()) + ): + raise ValueError("prior caller admission INCOMPLETE; CREATE refused") + versions = Versions.model_validate_json((ROOT / "deploy/k8s/versions.json").read_bytes()) + if args.render: + for region in ("eu", "us", "apac", "local"): + rendered = command("kubectl", "kustomize", str(ROOT / "deploy/k8s/overlays" / region)) + assert "@sha256:" in rendered and "automountServiceAccountToken: false" in rendered + print("RENDERED", region) + return + if ( + not args.create + or not re.fullmatch(r"hypertrain-v2-[a-z0-9-]{1,40}", args.cluster) + or not 1 <= args.deadline_seconds <= 10800 + or args.cpus != "0-3" + or args.cpu_quota != 4 + ): + raise ValueError( + "proof requires experiment-owned --cluster hypertrain-v2- --cpus 0-3 --cpu-quota 4" + ) + if not args.cleanup: + raise ValueError("isolated proof requires --cleanup") + if not args.proof_dir or (args.image and args.local_image_id): + raise ValueError("unique --proof-dir and exactly one image route required") + runner_admission(archive=bool(archive)) + if os.environ.get("KIND_EXPERIMENTAL_PROVIDER", "docker") != "docker": + raise ValueError("preflight refuses provider fallback environment") + if args.mode == "integration" and ( + not args.integration_root + or not args.source_manifest + or not args.integration_scripts + or not args.integration_scripts_sha256 + ): + raise ValueError( + "integration requires root-admitted private integration root and source freeze" + ) + if ( + args.mode == "integration" + and hashlib.sha256(Path(args.integration_scripts).read_bytes()).hexdigest() + != args.integration_scripts_sha256 + ): + raise ValueError("integration scripts digest mismatch") + if args.mode == "integration" and not args.local_image_id: + raise ValueError( + "integration requires verified locally loaded package image; " + "remote image route not frozen" + ) + if versions.kind_version not in command("kind", "version"): + raise ValueError("kind executable differs from pinned version") + baseline = set(command("kind", "get", "clusters").splitlines()) + if args.cluster in baseline: + raise ValueError("refusing to touch pre-existing proof cluster") + if command("docker", "ps", "-aq", "--filter", "label=io.x-k8s.kind.cluster=" + args.cluster): + raise ValueError("pre-existing cluster-labelled container conflicts with ownership") + containers = set(command("docker", "ps", "-aq", "--no-trunc").splitlines()) + volumes = set(command("docker", "volume", "ls", "-q").splitlines()) + networks = set(command("docker", "network", "ls", "-q", "--no-trunc").splitlines()) + provider_baselines = { + provider: provider_inventory(provider) for provider in ("docker", "podman") + } + for provider in provider_baselines: + if command( + provider, "ps", "-aq", "--filter", "label=io.x-k8s.kind.cluster=" + args.cluster + ): + raise ValueError("pre-existing experiment label in " + provider) + # Existing published digest is not claimed to contain unmerged relay changes. + if archive: + _, image, _ = archive + elif args.local_image_id: + if not args.source_manifest or not args.source_manifest_sha256: + raise ValueError("local image requires frozen source manifest and admitted digest") + image = verify_local_image( + args.local_image_id, Path(args.source_manifest), args.source_manifest_sha256 + ) + else: + image = args.image or versions.server_image + if not re.fullmatch(r"[^\s]+@sha256:[0-9a-f]{64}", image): + raise ValueError("remote integration image must be digest-pinned") + if not archive: + command( + "docker", + "run", + "--rm", + "--cpuset-cpus=0-3", + "--cpus=0.5", + "--memory=256m", + "--memory-swap=256m", + "--tmpfs", + "/data:rw,size=16m,uid=65532,gid=65532", + "--entrypoint", + "/opt/venv/bin/python", + image, + "-c", + "import hypertrain.relay.app", + ) + events: list[str] = [] + proof_dir = Path(args.proof_dir).resolve() + proof_dir.mkdir(parents=True, exist_ok=False) + journal = CleanupJournal(proof_dir / "cleanup.json") + journal.write_text( + json.dumps( + { + "cluster": args.cluster, + "containers": sorted(containers), + "volumes": sorted(volumes), + "networks": sorted(networks), + "experiment": args.cluster, + "provider_baselines": provider_baselines, + "proof_dir": str(proof_dir), + "kubeconfig": str(proof_dir / "kubeconfig"), + "deadline_monotonic": args.work_end + if archive + else time.monotonic() + args.deadline_seconds, + "cleanup_cutoff_monotonic": args.cleanup_end + if archive + else time.monotonic() + args.deadline_seconds + 1200, + "boot_id": args.clock_boot_id + if archive + else Path("/proc/sys/kernel/random/boot_id").read_text().strip(), + "integration_root": ( + str(args.integration_root) if args.mode == "integration" else None + ), + "state": "ADMITTED_NOT_CREATED", + } + ) + ) + result: dict = {} + with tempfile.TemporaryDirectory(prefix="hypertrain-kind-proof-") as temporary: + tmp = Path(temporary) + kubeconfig = proof_dir / "kubeconfig" + kubectl = [ + "kubectl", + "--kubeconfig", + str(kubeconfig), + "--context", + "kind-" + args.cluster, + ] + try: + # CPU/memory constrain Docker node during creation, not merely after it. + # kind invokes `docker run`; wrapper inserts daemon-side constraints. + real_docker = shutil.which("docker") + if real_docker is None: + raise ValueError("Docker missing") + wrapper = tmp / "docker" + import shlex + + wrapper.write_text( + "#!/bin/sh\nexec " + + shlex.join( + [ + sys.executable, + str(Path(__file__).resolve()), + "--docker-hook", + str(journal.path), + "--real-docker", + real_docker, + "--", + ] + ) + + ' "$@"\n' + ) + wrapper.chmod(0o700) + environment = {**os.environ, "PATH": str(tmp) + ":" + os.environ["PATH"]} + journal.write_text( + json.dumps( + { + "cluster": args.cluster, + "state": "CREATE_INTENT", + "integration_root": str(args.integration_root) + if args.mode == "integration" + else None, + "containers": sorted(containers), + "volumes": sorted(volumes), + "networks": sorted(networks), + "kubeconfig": str(kubeconfig), + } + ) + ) + start_supervisor(journal.path) + if archive and time.monotonic() >= args.work_end: + raise TimeoutError("archive preflight exhausted original work deadline") + command( + "kind", + "create", + "cluster", + "--name", + args.cluster, + "--config", + str(ROOT / "deploy/k8s/local/kind.yaml"), + "--kubeconfig", + str(kubeconfig), + "--wait", + "0", + environment=environment, + timeout=240, + ) + journal.write_text( + json.dumps( + { + "cluster": args.cluster, + "state": "CREATED", + "integration_root": str(args.integration_root) + if args.mode == "integration" + else None, + "kubeconfig": str(kubeconfig), + "containers": sorted(containers), + "volumes": sorted(volumes), + "networks": sorted(networks), + } + ) + ) + node = args.cluster + "-control-plane" + require_capped_node(journal.path, node) + command( + "docker", + "exec", + node, + "mkdir", + "-p", + "/var/local/hypertrain-proof-store", + ) + command( + "docker", + "exec", + node, + "mkdir", + "-p", + "/var/local/hypertrain-proof-master", + ) + command( + "docker", + "exec", + node, + "chown", + "65532:65532", + "/var/local/hypertrain-proof-store", + ) + command( + "docker", + "exec", + node, + "chown", + "65532:65532", + "/var/local/hypertrain-proof-master", + ) + node_info = json.loads(command("docker", "inspect", node))[0] + inspect = node_info["HostConfig"] + owned_volumes = [ + mount["Name"] + for mount in node_info["Mounts"] + if mount["Type"] == "volume" and mount["Name"] not in volumes + ] + owned_networks = [ + info["NetworkID"] + for info in node_info["NetworkSettings"]["Networks"].values() + if info["NetworkID"] not in networks + ] + journal.write_text( + json.dumps( + { + "cluster": args.cluster, + "state": "OWNERS_RECORDED", + "integration_root": str(args.integration_root) + if args.mode == "integration" + else None, + "containers": sorted(containers), + "volumes": sorted(volumes), + "networks": sorted(networks), + "owned_volumes": owned_volumes, + "owned_networks": owned_networks, + } + ) + ) + validate_node_limits(inspect) + added = set(command("docker", "ps", "-aq", "--no-trunc").splitlines()) - containers + if added != {node_info["Id"]}: + raise ValueError("unadmitted additional daemon containers during kind creation") + if archive: + load_transport_image(args.cluster, image, archive) + elif args.local_image_id: + load_transport_image(args.cluster, image) + with httpx.Client(timeout=30, follow_redirects=False) as client: + cni = client.get(versions.calico_manifest_url) + cni.raise_for_status() + assert hashlib.sha256(cni.content).hexdigest() == versions.calico_manifest_sha256 + manifest = cni.text + for tag, pin in versions.calico_images.items(): + manifest = manifest.replace(tag.replace("docker.io/", "quay.io/"), pin) + assert not re.search(r"image:.*calico/.*:v", manifest) + command(*kubectl, "apply", "-f", "-", input_text=manifest) + command( + *kubectl, + "rollout", + "status", + "deployment/calico-kube-controllers", + "-n", + "kube-system", + "--timeout=180s", + ) + command( + *kubectl, + "rollout", + "status", + "daemonset/calico-node", + "-n", + "kube-system", + "--timeout=180s", + ) + command( + *kubectl, + "wait", + "--for=condition=Ready", + "node/" + node, + "--timeout=180s", + ) + events.append("POLICY_CNI_READY") + if archive: + log = verify_archive_node(node, image, args.archive_image_id, archive[2], kubectl) + result["archive_node_verification"] = log.strip() + result["transport_cluster_smoke"] = "PENDING" + events.append("ARCHIVE_NODE_SOURCE109_RELAY_IMPORT_VERIFIED") + if args.mode == "integration": + supervisor_state = json.loads(journal.path.read_bytes()) + supervisor_state["integration_root"] = str(args.integration_root) + journal.write_text(json.dumps(supervisor_state)) + result = integration_workload(args, image, kubectl, node) + else: + # Generate ephemeral runtime secrets/test CA; never committed. + for name in ("master", "eu", "us"): + (tmp / f"{name}.seed").write_text(secrets.token_hex(32)) + local_registry = RelayRegistryV1( + registry_version=1, + epoch=0, + previous_registry_hash=None, + specs=[ + RelaySpec( + id=f"{region}-1", + region=region, + https_url=f"{GATEWAY}/{region}", + pubkeys=[ + RelayKey( + key_id="k1", + pubkey=Keypair( + bytes.fromhex((tmp / f"{region}.seed").read_text()) + ).ss58, + valid_from_round=1, + valid_until_round=10000, + ) + ], + max_object_bytes=16 << 20, + max_inflight_bytes=64 << 20, + codecs=["ht-sparse-v1"], + mode="transport", + ) + for region in ("eu", "us") + ], + ) + (tmp / "registry.json").write_text(local_registry.model_dump_json()) + (tmp / "master.public").write_text( + Keypair(bytes.fromhex((tmp / "master.seed").read_text())).ss58 + ) + (tmp / "drain.token").write_text(secrets.token_hex(32)) + (tmp / "object.json").write_text( + json.dumps( + { + "access_key_id": "proof-local", + "secret_access_key": secrets.token_hex(32), + } + ) + ) + command( + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-days", + "1", + "-subj", + "/CN=hypertrain-proof", + "-addext", + "subjectAltName=DNS:gateway.hypertrain-proof-gateway.svc", + "-keyout", + str(tmp / "tls.key"), + "-out", + str(tmp / "tls.crt"), + ) + (tmp / "ca.crt").write_bytes((tmp / "tls.crt").read_bytes()) + # Namespaces first; install secrets before relay workloads. + fixture_text = ( + (ROOT / "deploy/k8s/local/fixtures.yaml") + .read_text() + .replace(versions.server_image, image) + ) + if archive: + fixture_text = archive_workload(fixture_text, image) + command(*kubectl, "apply", "-f", "-", input_text=fixture_text) + for namespace in ( + "hypertrain-proof", + "hypertrain-proof-store", + "hypertrain-proof-gateway", + "hypertrain-relay-eu", + "hypertrain-relay-us", + ): + if namespace.startswith("hypertrain-relay-"): + apply( + kubectl, + { + "apiVersion": "v1", + "kind": "Namespace", + "metadata": { + "name": namespace, + "labels": {"hypertrain.network/role": "relay"}, + }, + }, + ) + # Relays/gateway/store never receive master signing seed. + names = ["registry.json", "ca.crt", "master.public"] + if namespace == "hypertrain-proof": + names += ["master.seed", "object.json"] + elif namespace == "hypertrain-proof-store": + names += ["object.json"] + elif namespace == "hypertrain-proof-gateway": + names += ["tls.key", "tls.crt"] + else: + names += [ + namespace.rsplit("-", 1)[1] + ".seed", + "object.json", + "drain.token", + ] + data = { + name: base64.b64encode((tmp / name).read_bytes()).decode() for name in names + } + apply( + kubectl, + { + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "proof-secrets", + "namespace": namespace, + }, + "type": "Opaque", + "data": data, + }, + ) + apply( + kubectl, + { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": "proof-script", + "namespace": namespace, + }, + "data": {"relay_kind_smoke.py": Path(__file__).read_text()}, + }, + ) + for region in ("eu", "us"): + rendered = command( + "kubectl", "kustomize", str(ROOT / "deploy/k8s/overlays/local") + ) + rendered = rendered.replace( + "hypertrain-relay-local", "hypertrain-relay-" + region + ) + rendered = rendered.replace("REGION: local", "REGION: " + region) + rendered = rendered.replace(versions.server_image, image) + if archive: + rendered = archive_workload(rendered, image) + # Fixture relay reads proof registry, never production authority keys. + rendered = rendered.replace( + "secretName: relay-runtime-v1", "secretName: proof-secrets" + ) + apply( + kubectl, + { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": "relay-contract", + "namespace": "hypertrain-relay-" + region, + }, + "data": {"relay.json": local_registry.model_dump_json()}, + }, + ) + command(*kubectl, "apply", "-f", "-", input_text=rendered) + command( + *kubectl, + "rollout", + "status", + "deployment/hypertrain-relay", + "-n", + "hypertrain-relay-" + region, + "--timeout=180s", + ) + for namespace, deployment in ( + ("hypertrain-proof-store", "objects"), + ("hypertrain-proof", "master"), + ("hypertrain-proof-gateway", "gateway"), + ): + command( + *kubectl, + "rollout", + "status", + "deployment/" + deployment, + "-n", + namespace, + "--timeout=180s", + ) + prefix_job = json.loads( + command( + *kubectl, + "get", + "job/mock-miner", + "-n", + "hypertrain-proof", + "-o", + "json", + ) + )["spec"] + prefix_job.pop("selector", None) + prefix_job["template"]["metadata"] = { + "labels": {"hypertrain.network/role": "master"} + } + prefix_job["suspend"] = False + prefix_job["template"]["spec"]["containers"][0]["env"] = [ + {"name": "PROOF_STAGE", "value": "prefix"} + ] + apply( + kubectl, + { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "name": "prefix-upload", + "namespace": "hypertrain-proof", + }, + "spec": prefix_job, + }, + ) + command( + *kubectl, + "wait", + "--for=condition=complete", + "job/prefix-upload", + "-n", + "hypertrain-proof", + "--timeout=180s", + ) + prefix_log = command( + *kubectl, "logs", "job/prefix-upload", "-n", "hypertrain-proof" + ) + assert '"event": "PREFIX_DURABLE"' in prefix_log + command( + *kubectl, + "delete", + "pods", + "-n", + "hypertrain-relay-eu", + "-l", + "app=hypertrain-relay", + "--grace-period=0", + "--force", + "--wait=true", + "--timeout=30s", + ) + command( + *kubectl, + "rollout", + "status", + "deployment/hypertrain-relay", + "-n", + "hypertrain-relay-eu", + "--timeout=180s", + ) + events.append("DURABLE_PREFIX_RESTARTED_BEFORE_FULL_RESUME") + command( + *kubectl, + "patch", + "job/mock-miner", + "-n", + "hypertrain-proof", + "--type=merge", + "-p", + '{"spec":{"suspend":false}}', + ) + command( + *kubectl, + "wait", + "--for=condition=complete", + "job/mock-miner", + "-n", + "hypertrain-proof", + "--timeout=180s", + ) + log = command(*kubectl, "logs", "job/mock-miner", "-n", "hypertrain-proof") + assert '"event": "FOUR_ORIGINAL_OBJECTS_VERIFIED"' in log + events.append(log.strip()) + # Event-backed actual pod kill and replay through the surviving shared-CAS replica. + pods = json.loads( + command( + *kubectl, + "get", + "pods", + "-n", + "hypertrain-relay-eu", + "-l", + "app=hypertrain-relay", + "-o", + "json", + ) + )["items"] + if len(pods) < 2: + raise ValueError("replica-switch proof requires two ready relay pods") + killed = pods[0]["metadata"]["name"] + command( + *kubectl, + "delete", + "pod", + killed, + "-n", + "hypertrain-relay-eu", + "--grace-period=0", + "--force", + "--wait=true", + "--timeout=30s", + ) + command( + *kubectl, + "rollout", + "status", + "deployment/hypertrain-relay", + "-n", + "hypertrain-relay-eu", + "--timeout=180s", + ) + events.append("RELAY_POD_KILLED_REPLACEMENT_READY") + job = json.loads( + command( + *kubectl, + "get", + "job/mock-miner", + "-n", + "hypertrain-proof", + "-o", + "json", + ) + ) + job_spec = job["spec"] + job_spec.pop("selector", None) + job_spec["template"]["metadata"] = {"labels": {"hypertrain.network/role": "master"}} + job_spec["suspend"] = False + job_spec["template"]["spec"]["containers"][0]["env"] = [ + {"name": "PROOF_STAGE", "value": "resume"} + ] + apply( + kubectl, + { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "name": "replica-resume", + "namespace": "hypertrain-proof", + }, + "spec": job_spec, + }, + ) + command( + *kubectl, + "wait", + "--for=condition=complete", + "job/replica-resume", + "-n", + "hypertrain-proof", + "--timeout=180s", + ) + resumed = command(*kubectl, "logs", "job/replica-resume", "-n", "hypertrain-proof") + first_receipts = json.loads(log.strip().splitlines()[-1])["receipts"] + assert json.loads(resumed.strip().splitlines()[-1])["receipts"] == first_receipts + events.append("REPLICA_RESUME_ORIGINAL_RECEIPTS_VERIFIED") + command( + *kubectl, + "scale", + "deployment/hypertrain-relay", + "-n", + "hypertrain-relay-eu", + "--replicas=0", + ) + command( + *kubectl, + "wait", + "--for=delete", + "pods", + "-n", + "hypertrain-relay-eu", + "-l", + "app=hypertrain-relay", + "--timeout=60s", + ) + job_spec["template"]["spec"]["containers"][0]["env"] = [ + {"name": "PROOF_STAGE", "value": "fallback"} + ] + apply( + kubectl, + { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "name": "regional-fallback", + "namespace": "hypertrain-proof", + }, + "spec": job_spec, + }, + ) + command( + *kubectl, + "wait", + "--for=condition=complete", + "job/regional-fallback", + "-n", + "hypertrain-proof", + "--timeout=180s", + ) + fallback_log = command( + *kubectl, "logs", "job/regional-fallback", "-n", "hypertrain-proof" + ) + assert len(json.loads(fallback_log.strip().splitlines()[-1])["receipts"]) == 1 + events.append("PRIMARY_REGION_DOWN_AUTHORIZED_FALLBACK_BYTES_VERIFIED") + command( + *kubectl, + "scale", + "deployment/hypertrain-relay", + "-n", + "hypertrain-relay-eu", + "--replicas=2", + ) + command( + *kubectl, + "rollout", + "status", + "deployment/hypertrain-relay", + "-n", + "hypertrain-relay-eu", + "--timeout=180s", + ) + gc_data = { + name: base64.b64encode((tmp / name).read_bytes()).decode() + for name in ( + "registry.json", + "ca.crt", + "master.public", + "master.seed", + "eu.seed", + "object.json", + ) + } + apply( + kubectl, + { + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "proof-gc-only", + "namespace": "hypertrain-proof", + }, + "data": gc_data, + }, + ) + job_spec["template"]["spec"]["containers"][0]["command"] = [ + "/opt/venv/bin/python", + "/fixture/relay_kind_smoke.py", + "--fixture", + "gc", + ] + for volume in job_spec["template"]["spec"]["volumes"]: + if volume["name"] == "secrets": + volume["secret"]["secretName"] = "proof-gc-only" + apply( + kubectl, + { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "name": "transport-gc", + "namespace": "hypertrain-proof", + }, + "spec": job_spec, + }, + ) + command( + *kubectl, + "wait", + "--for=condition=complete", + "job/transport-gc", + "-n", + "hypertrain-proof", + "--timeout=180s", + ) + gc_log = command(*kubectl, "logs", "job/transport-gc", "-n", "hypertrain-proof") + assert '"event": "TRANSPORT_RETENTION_GC_PREFIX_ISOLATION"' in gc_log + events.append(gc_log.strip()) + # Actual deny control, not presence of YAML. + code = ( + "import socket; s=socket.socket(); s.settimeout(3); " + "s.connect(('hypertrain-relay.hypertrain-relay-eu.svc',8000))" + ) + denied = command( + *kubectl, + "run", + "unauthorized", + "-n", + "hypertrain-proof-denied", + "--image=" + image, + "--restart=Never", + "--overrides=" + + json.dumps( + { + "spec": { + "automountServiceAccountToken": False, + "securityContext": { + "runAsNonRoot": True, + "runAsUser": 65532, + "seccompProfile": {"type": "RuntimeDefault"}, + }, + "containers": [ + { + "name": "unauthorized", + "image": image, + **({"imagePullPolicy": "Never"} if archive else {}), + "command": ["/opt/venv/bin/python", "-c", code], + "securityContext": { + "allowPrivilegeEscalation": False, + "capabilities": {"drop": ["ALL"]}, + }, + "resources": {"limits": {"cpu": "100m", "memory": "64Mi"}}, + } + ], + } + } + ), + "-o", + "json", + ) + pod = json.loads(denied)["metadata"]["name"] + command( + *kubectl, + "wait", + "--for=jsonpath={.status.phase}=Failed", + "pod/" + pod, + "-n", + "hypertrain-proof-denied", + "--timeout=30s", + ) + denied_logs = command(*kubectl, "logs", pod, "-n", "hypertrain-proof-denied") + assert "TimeoutError" in denied_logs + events.append("UNAUTHORIZED_NAMESPACE_DENIED") + result = { + **result, + "events": events, + "cpu_quota": 3.5, + "memory_bytes": inspect["Memory"], + "transport_cluster_smoke": "PASS", + "full_D3_integration": "NOT_RUN", + } + finally: + completed = result.get("transport_cluster_smoke") == "PASS" + if result: + if archive: + result["transport_cluster_smoke"] = "PENDING" + (proof_dir / "proof-result.json").write_text(json.dumps(result)) + state = json.loads(journal.path.read_bytes()) + journal.write_text( + json.dumps({**state, "proof_status": "INTERRUPTED" if archive else "PASS"}) + ) + cleanup_owned(journal.path) + cleanup_state = json.loads(journal.path.read_bytes()) + if cleanup_state["proof_status"] == "CENSORED": + raise ValueError("rescue incomplete; proof CENSORED, cleanup verified") + if archive and completed: + result["transport_cluster_smoke"] = "PASS" + (proof_dir / "proof-result.json").write_text(json.dumps(result)) + journal.write_text(json.dumps({**cleanup_state, "proof_status": "PASS"})) + print(json.dumps(result)) # PASS only after exact cleanup verification. + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--fixture", choices=("store", "master", "gateway", "relay", "miner", "gc")) + parser.add_argument("--render", action="store_true") + parser.add_argument("--create", action="store_true") + parser.add_argument("--cluster", default="hypertrain-v2-" + secrets.token_hex(8)) + parser.add_argument("--deadline-seconds", type=int, default=10800) + parser.add_argument( + "--launch-unit", help="Exact outer archive service identity; secondary runtime ceiling" + ) + parser.add_argument("--work-end", type=float, help="Admitted absolute monotonic work endpoint") + parser.add_argument( + "--cleanup-end", type=float, help="Admitted absolute monotonic cleanup endpoint" + ) + parser.add_argument("--clock-boot-id", help="Boot identity of admitted monotonic endpoints") + parser.add_argument("--caller-cap", type=float, help="Finite absolute caller resource cutoff") + parser.add_argument( + "--caller-unit", help="Own flock through independent cleanup before release" + ) + parser.add_argument("--caller-lock", type=Path, help="Existing owned shared flock file") + parser.add_argument("--cpus", default="0-3") + parser.add_argument("--cpu-quota", type=int, default=4) + parser.add_argument("--cleanup", action="store_true") + parser.add_argument("--image", help="Integrated GHCR server digest containing relay code") + parser.add_argument("--local-image-id", help="Actual locally built Docker sha256 image ID") + parser.add_argument( + "--image-archive", type=Path, help="Owned immutable native archive, transport only" + ) + parser.add_argument("--archive-sha256", help="Root-admitted complete archive SHA256") + parser.add_argument("--archive-image-id", help="Root-admitted image config sha256 ID") + parser.add_argument("--archive-reference", help="Exact stored archive tag; Never pull") + parser.add_argument("--source-manifest") + parser.add_argument("--source-manifest-sha256") + parser.add_argument("--proof-dir", required=False) + parser.add_argument("--mode", choices=("transport", "integration"), default="transport") + parser.add_argument("--prepare-integration", action="store_true") + parser.add_argument("--snapshot", type=Path) + parser.add_argument("--package-sha256") + parser.add_argument( + "--integration-role", + choices=("service", "relay", "driver", "gateway", "wrong-ca"), + ) + parser.add_argument("--integration-config", type=Path) + parser.add_argument("--render-integration", action="store_true") + parser.add_argument("--integration-root", type=Path) + parser.add_argument("--integration-scripts", type=Path) + parser.add_argument("--integration-scripts-sha256") + parser.add_argument("--supervise", type=Path) + parser.add_argument("--parent-pid", type=int) + parser.add_argument("--ready-socket", type=Path) + parser.add_argument("--docker-hook", type=Path) + parser.add_argument("--real-docker") + parser.add_argument("docker_args", nargs=argparse.REMAINDER) + parser.add_argument("--prepare-root", type=Path) + parser.add_argument("--original-fixture-secrets", action="store_true") + args = parser.parse_args() + if args.caller_unit: + if args.caller_lock is None: + parser.error("caller requires existing shared --caller-lock") + index = sys.argv.index("--caller-unit") + runner_argv = sys.argv[1:index] + sys.argv[index + 2 :] + caller_launch(args, runner_argv) + elif args.docker_hook: + if not args.real_docker: + parser.error("Docker hook requires absolute real Docker executable") + docker_create_hook( + args.docker_hook, + args.real_docker, + args.docker_args[1:] if args.docker_args[:1] == ["--"] else args.docker_args, + ) + elif args.supervise: + if args.parent_pid is None: + parser.error("supervisor requires parent pid") + deadline_supervisor(args.supervise, args.parent_pid, args.ready_socket) + elif args.prepare_root: + if args.snapshot is None or args.package_sha256 is None or args.create: + parser.error("prepare root requires admitted adapter bundle/hash; no create") + prepare_integration_root(args.prepare_root, args.snapshot, args.package_sha256) + if args.original_fixture_secrets: + provision_original_fixture_secrets( + args.prepare_root, args.snapshot, args.package_sha256 + ) + elif args.original_fixture_secrets: + if args.integration_root is None or args.snapshot is None or args.package_sha256 is None: + parser.error("original private roles require prepared root and admitted bundle/hash") + provision_original_fixture_secrets( + args.integration_root, args.snapshot, args.package_sha256 + ) + elif args.render_integration: + if args.mode != "integration" or args.create or not args.image: + parser.error("integration render requires integration mode and exact image; no create") + print(json.dumps(integration_manifests(args.image))) + elif args.integration_role: + if ( + args.mode != "integration" + or args.prepare_integration + or args.create + or args.render + or args.fixture + or args.integration_config is None + ): + parser.error("integration role requires integration mode and config; no fixture/launch") + integration_role(args.integration_role, args.integration_config) + elif args.prepare_integration: + if ( + args.mode != "integration" + or args.create + or args.render + or args.fixture + or not args.snapshot + or not args.package_sha256 + ): + parser.error( + "preparation requires integration mode, snapshot, " + "root-admitted package hash; no launch" + ) + prepared = inspect_integration_snapshot(args.snapshot, args.package_sha256) + print(prepared.model_dump_json(exclude={"files"})) + elif args.fixture: + fixture(args.fixture) + else: + smoke(args) + + +if __name__ == "__main__": + main() diff --git a/scripts/verify_image.sh b/scripts/verify_image.sh index de6dee1..73e7967 100755 --- a/scripts/verify_image.sh +++ b/scripts/verify_image.sh @@ -3,6 +3,83 @@ # the contract-v1 user/port rules. Exit 0 = conformant; 1 = refused (reasons on stderr). # scripts/verify_image.sh IMAGE [SLUG] [SOURCE] ENGINE=docker|podman (default podman) set -euo pipefail +if [ "${1:-}" = --cuda-candidate ]; then + image=${2:?candidate image required} + manifest=${3:?source manifest required} + revision=${4:?Git revision required} + engine=${ENGINE:-podman} + python3 - "$manifest" <<'PY' +import hashlib +import json +import pathlib +import sys + +expected = json.loads(pathlib.Path(sys.argv[1]).read_text()) +root = pathlib.Path.cwd() +actual = { + str(path.relative_to(root)): hashlib.sha256(path.read_bytes()).hexdigest() + for path in (root / 'src/hypertrain').rglob('*') + if path.is_file() and '__pycache__' not in path.parts and path.suffix != '.pyc' +} +assert actual == expected['source_files'], 'source changed after candidate freeze' +assert all(hashlib.sha256((root / name).read_bytes()).hexdigest() == digest for name, digest in expected['copy_files'].items()) +PY + config=$("$engine" image inspect "$image" --format '{{json .Config}}') + test "$(jq -r '.Labels["org.opencontainers.image.revision"]' <<<"$config")" = "$revision" + test "$(jq -r '.Labels["io.cortex.challenge.source-manifest"]' <<<"$config")" = "$(sha256sum "$manifest" | cut -d ' ' -f 1)" + test "$(jq -c '.Entrypoint' <<<"$config")" = '["hypertrain-miner"]' + "$engine" run --rm -i --network=none --read-only \ + -e PYTHONDONTWRITEBYTECODE=1 -e OMP_NUM_THREADS=1 -e MKL_NUM_THREADS=1 \ + -v "$(realpath "$manifest"):/candidate-source.json:ro" \ + --entrypoint /opt/hypertrain/venv/bin/python "$image" - <<'PY' +import hashlib +import importlib.metadata +import json +import pathlib +import subprocess +import tomllib + +import hypertrain +import torch + +expected = json.loads(pathlib.Path('/candidate-source.json').read_text()) +package = pathlib.Path(hypertrain.__file__).resolve().parent +assert package == pathlib.Path('/opt/hypertrain/venv/lib/python3.12/site-packages/hypertrain') +actual = { + 'src/hypertrain/' + str(path.relative_to(package)): hashlib.sha256(path.read_bytes()).hexdigest() + for path in package.rglob('*') + if path.is_file() and '__pycache__' not in path.parts and path.suffix != '.pyc' +} +assert actual == expected['source_files'], 'installed package differs from frozen source' +packages = {entry['name']: entry for entry in tomllib.loads(pathlib.Path('/opt/hypertrain/src/uv.lock').read_text())['package']} +pending = [entry['name'] for entry in packages['hypertrain']['dependencies']] +versions = {} +while pending: + name = pending.pop() + if name in versions: + continue + entry = packages[name] + versions[name] = entry['version'] + pending.extend(dependency['name'] for dependency in entry.get('dependencies', [])) +versions.update(expected['backend_versions']) +assert len(versions) == 39, 'review required for changed dependency set' +assert {name: importlib.metadata.version(name) for name in versions} == versions +assert torch.__version__ == '2.14.0+cu130' +od = json.loads(importlib.metadata.distribution('opendecision').read_text('direct_url.json')) +assert od['url'] == 'https://github.com/CortexLM/opendecision/archive/e125ff756d57b726fbce44c430e3d2207e7bbda8.tar.gz' +scripts = {entry.name: entry.value for entry in importlib.metadata.distribution('hypertrain').entry_points if entry.group == 'console_scripts'} +assert scripts == {'hypertrain': 'hypertrain.aggregator.cli:main', 'hypertrain-miner': 'hypertrain.miner.cli:main'} +for name in scripts: + executable = pathlib.Path('/opt/hypertrain/venv/bin') / name + assert executable.read_text().splitlines()[0] == '#!/opt/hypertrain/venv/bin/python' + subprocess.run([str(executable), '--help'], check=True, timeout=30) +import hypertrain.trainer +import hypertrain.auditor.__main__ +import hypertrain.models.opendecision +print(json.dumps({'source_files': actual, 'versions': versions, 'package': str(package), 'torch': torch.__version__, 'entrypoints': scripts}, sort_keys=True)) +PY + exit 0 +fi image=${1:?usage: verify_image.sh IMAGE [SLUG] [SOURCE]} slug=${2:-hypertrain} source=${3:-https://github.com/CortexLM/hypertrain} diff --git a/src/hypertrain/aggregator/capacity_worker.py b/src/hypertrain/aggregator/capacity_worker.py new file mode 100644 index 0000000..88ee3b8 --- /dev/null +++ b/src/hypertrain/aggregator/capacity_worker.py @@ -0,0 +1,509 @@ +"""One bounded outer arithmetic file contract; no wallet, SQL or job scheduler.""" + +from __future__ import annotations + +import os +import stat +import sys +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import Annotated + +from pydantic import Field, JsonValue + +from hypertrain.aggregator.tape_v2 import TapeBodyV2, VerifiedInput, compute_body +from hypertrain.challenge.trust_v2 import FundedStatus, ReplayEvidence, SettlementStatus +from hypertrain.data.store import LocalFSStore +from hypertrain.protocol.envelope_v2 import load_json +from hypertrain.protocol.hashing import sha256_hex +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.messages import Hex64 +from hypertrain.protocol.messages_v2 import ( + AggregationPolicyV2, + CommitV2, + DeltaManifestV2, + RosterEntryV2, + RunManifestV2, + StartStateV2, + WireModel, +) + + +class GenesisRequest(WireModel): + operation: Annotated[str, Field(pattern="^genesis$")] = "genesis" + manifest: RunManifestV2 + profile: dict[str, JsonValue] + manifest_envelope: dict[str, JsonValue] + receipt: dict[str, JsonValue] + opening: dict[str, JsonValue] + accepted: Annotated[int, Field(strict=True, ge=1)] + + def validate_authority(self) -> list[RosterEntryV2]: + from hypertrain.challenge.store import ChallengeStore + from hypertrain.protocol import envelope_v2 + from hypertrain.protocol.messages import Receipt + from hypertrain.protocol.messages_v2 import RoundOpenV2 + + _geometry(self.manifest) + run = self.manifest.run_id() + manifest_env = envelope_v2.parse_envelope(canonicalize(self.manifest_envelope)) + if ( + manifest_env.run_id != run + or manifest_env.type != "RunManifestV2" + or manifest_env.body != self.manifest.body() + or not envelope_v2.verify_envelope(self.manifest_envelope) + ): + raise ValueError("capacity genesis manifest authority differs") + receipt = envelope_v2.Intake(run, {"Receipt": manifest_env.signer.__eq__}).accept( + canonicalize(self.receipt), self.accepted + ) + opening = envelope_v2.Intake( + run, {"RoundOpenV2": self.manifest.training.coord_pubkey.__eq__} + ).accept(canonicalize(self.opening), self.accepted) + assert isinstance(receipt, Receipt) and isinstance(opening, RoundOpenV2) + paths = ( + "aggregator/core.py", + "aggregator/tape_v2.py", + "aggregator/rollback_v2.py", + "trainer/compress.py", + "trainer/config.py", + "trainer/model.py", + "miner/island_launch.py", + "aggregator/capacity_worker.py", + "auditor/replay.py", + ) + source = ChallengeStore._service_implementation_v2( + Path(__file__).resolve().parents[1], paths + ) + if ( + self.profile.get("run_id") != run + or self.profile.get("implementation_hash") != source + or receipt.commit_hash != sha256_hex(canonicalize(self.profile)) + or receipt.w != 0 + or receipt.received_round != self.accepted + or opening.w != 0 + or opening.d_final <= self.accepted + or opening.policy_hashes.body() + != {k: getattr(self.manifest.network, k) for k in opening.policy_hashes.body()} + or len({x.hotkey for x in opening.roster}) != len(opening.roster) + or opening.roster_hash != sha256_hex(canonicalize([x.body() for x in opening.roster])) + ): + raise ValueError("capacity genesis authority differs") + return opening.roster + + +class GenesisResult(WireModel): + request_hash: Hex64 + starts: Annotated[list[StartStateV2], Field(min_length=4, max_length=32)] + state_root: Hex64 + outer_state: Hex64 + outer_hashes: dict[str, Hex64] + + +def collect_genesis( + directory: Path, request: GenesisRequest +) -> tuple[GenesisResult, dict[str, bytes]]: + roster = request.validate_authority() + with _directory_fd(directory) as root: + raw = _read_at(root, Path("result.json"), 1 << 20) + result = GenesisResult.model_validate(load_json(raw, max_bytes=1 << 20)) + if ( + canonicalize(result.body()) != raw + or result.request_hash != sha256_hex(canonicalize(request.body())) + or [s.hotkey for s in result.starts] != [r.hotkey for r in roster] + ): + raise ValueError("capacity genesis result binding differs") + first = result.starts[0] + for start in result.starts: + proof = sha256_hex( + bytes.fromhex(request.manifest.run_id() + start.opt_state_hash + start.ef_hash) + ) + if ( + start.run_id != request.manifest.run_id() + or start.w != 0 + or start.global_step0 != 0 + or start.parent_anchor_hash != proof + or start.anchor_verdict_hash != proof + or start.model_dump(exclude={"hotkey"}) != first.model_dump(exclude={"hotkey"}) + ): + raise ValueError("capacity genesis start authority differs") + if ( + set(result.outer_hashes) != {"theta_hash", "outer_state_hash", "center_hash"} + or result.outer_hashes["theta_hash"] != first.theta_hash + ): + raise ValueError("capacity genesis outer metadata differs") + objects = {} + for key in {result.outer_state, first.state_object_sha256, first.ef_object_sha256}: + data = _read_at(root, Path("objects") / key[:2] / key, 65536) + if sha256_hex(data) != key: + raise ValueError("capacity genesis object hash differs") + objects[key] = data + return result, objects + + +def _genesis(root: int, request: GenesisRequest) -> None: + from hypertrain.aggregator.core import OuterState + from hypertrain.auditor.replay import AnchorCache, optimizer_hash, pack_state, tensor_root + from hypertrain.trainer.compress import state_hash + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.model import init_params + + roster = request.validate_authority() + theta = init_params(TrainConfig.from_manifest_v2(request.manifest).model) + cache = AnchorCache() + starts = [] + + def put(data: bytes) -> str: + if len(data) > 65536: + raise ValueError("capacity genesis object too large") + key = sha256_hex(data) + _write_at(root, Path("objects") / key[:2] / key, data) + return key + + for entry in roster: + anchor = cache.genesis(request.manifest, entry.hotkey, theta) + starts.append( + StartStateV2( + run_id=request.manifest.run_id(), + w=0, + hotkey=entry.hotkey, + theta_hash=state_hash(theta), + state_object_sha256=put(pack_state(theta, anchor.state)), + opt_state_hash=optimizer_hash(anchor.state), + ef_object_sha256=put(pack_state(anchor.ef)), + ef_hash=state_hash(anchor.ef), + parent_anchor_hash=anchor.anchor_hash, + global_step0=anchor.state.step, + anchor_verdict_hash=anchor.proof_hash, + ) + ) + outer = OuterState.init({k: v.detach().cpu().numpy() for k, v in theta.items()}) + result = GenesisResult( + request_hash=sha256_hex(canonicalize(request.body())), + starts=starts, + state_root=tensor_root(theta, anchor.state), + outer_state=put(outer.to_bytes()), + outer_hashes=outer.hashes(), + ) + _write_at(root, Path("result.json"), canonicalize(result.body())) + print("CAPACITY_GENESIS_COMPLETE", flush=True) + + +def _geometry(manifest: RunManifestV2) -> None: + model, inner, layout = ( + manifest.training.model, + manifest.training.inner, + manifest.training.reference_spec.layout, + ) + if ( + ( + model.arch, + model.n_layers, + model.d_model, + model.n_heads, + model.n_kv_heads, + model.d_ff, + model.vocab, + model.seq_len, + model.param_count, + model.compute_dtype, + ) + != ("decoder", 1, 8, 2, 2, 8, 16, 2, 728, "fp32") + or (inner.H, inner.J, inner.micro_batch, inner.grad_accum, inner.opt, inner.state_policy) + != (2, 1, 1, 1, "adamw", "carry") + or (layout.pp, layout.n_gpus, layout.dp_size, layout.ep_size, layout.zero1) + != (1, 1, 1, 1, False) + or manifest.training.outer.opt != "nesterov" + ): + raise ValueError("capacity outer geometry is unqualified") + + +class OuterInput(WireModel): + roster: RosterEntryV2 + commit: CommitV2 + delta_manifest: DeltaManifestV2 + replay: ReplayEvidence + funding: FundedStatus + settlement: SettlementStatus + assignment_hash: Hex64 + clean_finalizations: Annotated[int, Field(strict=True, ge=0)] + + def work(self) -> VerifiedInput: + return VerifiedInput(**{name: getattr(self, name) for name in type(self).model_fields}) + + +class OuterRequest(WireModel): + manifest: RunManifestV2 + w: Annotated[int, Field(strict=True, ge=0)] + original_roster: Annotated[list[RosterEntryV2], Field(min_length=4, max_length=32)] + inputs: Annotated[list[OuterInput], Field(min_length=4, max_length=32)] + prev_state: Hex64 | None + predecessor_tape_hash: Hex64 + reference_reward_units: Annotated[int, Field(strict=True, ge=0)] + objects: Annotated[list[Hex64], Field(max_length=100)] + + @classmethod + def parse(cls, raw: bytes) -> OuterRequest: + request = cls.model_validate(load_json(raw, max_bytes=1 << 20)) + _geometry(request.manifest) + if canonicalize(request.body()) != raw: + raise ValueError("noncanonical capacity request") + original = {entry.hotkey: entry for entry in request.original_roster} + if len(original) != len(request.original_roster) or any( + original.get(work.roster.hotkey) != work.roster for work in request.inputs + ): + raise ValueError("capacity original roster differs") + if len({work.roster.hotkey for work in request.inputs}) != len(request.inputs): + raise ValueError("duplicate capacity input") + if len(set(request.objects)) != len(request.objects): + raise ValueError("duplicate capacity object") + if request.w != 0 and request.prev_state is None: + raise ValueError("capacity predecessor missing") + return request + + +class OuterResult(WireModel): + request_hash: Hex64 + tape_body: TapeBodyV2 + objects: Annotated[list[Hex64], Field(min_length=1, max_length=2)] + replayed: Annotated[bool, Field(strict=True)] + + +@contextmanager +def _directory_fd(path: Path, *, root_fd: int | None = None, create: bool = False) -> Iterator[int]: + """Walk directories without links and pin the resulting inode.""" + parts = path.parts if root_fd is not None else path.absolute().parts[1:] + if any(part in ("..", "/") for part in parts): + raise ValueError("capacity relative path escapes directory") + fd = os.dup(root_fd) if root_fd is not None else os.open("/", os.O_RDONLY | os.O_DIRECTORY) + try: + for part in parts: + if part == ".": + continue + if create: + try: + os.mkdir(part, mode=0o700, dir_fd=fd) + except FileExistsError: + pass + child = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd) + os.close(fd) + fd = child + yield fd + except OSError as exc: + raise ValueError("capacity directory confinement failed") from exc + finally: + os.close(fd) + + +def _read_at(root_fd: int, relative: Path, limit: int) -> bytes: + with _directory_fd(relative.parent, root_fd=root_fd) as parent: + try: + fd = os.open(relative.name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=parent) + except OSError as exc: + raise ValueError("capacity object confinement failed") from exc + with os.fdopen(fd, "rb") as stream: + if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode): + raise ValueError("capacity object is not a regular file") + raw = stream.read(limit + 1) + if len(raw) > limit: + raise ValueError("capacity bytes exceed limit") + return raw + + +def _write_at(root_fd: int, relative: Path, data: bytes) -> None: + """Exclusive descriptor-relative create; an existing hash object must match.""" + with _directory_fd(relative.parent, root_fd=root_fd, create=True) as parent: + try: + fd = os.open( + relative.name, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, + dir_fd=parent, + ) + except FileExistsError: + if _read_at(parent, Path(relative.name), len(data)) != data: + raise ValueError("capacity existing output differs") from None + return + with os.fdopen(fd, "wb") as stream: + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + os.fsync(parent) + + +def bounded_bytes(path: Path, limit: int) -> bytes: + with _directory_fd(path.parent) as parent: + return _read_at(parent, Path(path.name), limit) + + +def collect_result(directory: Path, request: OuterRequest) -> tuple[TapeBodyV2, dict[str, bytes]]: + """Parent consumes bounded metadata/bytes only; no tensor parser or arithmetic.""" + with _directory_fd(directory) as root_fd: + return _collect_result(root_fd, request) + + +def _collect_result(root_fd: int, request: OuterRequest) -> tuple[TapeBodyV2, dict[str, bytes]]: + raw = _read_at(root_fd, Path("result.json"), 1 << 20) + result = OuterResult.model_validate(load_json(raw, max_bytes=1 << 20)) + if ( + canonicalize(result.body()) != raw + or not result.replayed + or result.request_hash != sha256_hex(canonicalize(request.body())) + ): + raise ValueError("capacity result binding differs") + body = result.tape_body + manifest = request.manifest + expected = sorted(request.inputs, key=lambda x: x.roster.hotkey.encode()) + from hypertrain.aggregator.tape_v2 import _input + + records = [_input(work.work(), manifest, request.w) for work in expected] + from hypertrain.aggregator.weighted_v2 import Candidate, allocate_weights + + policy_hash = manifest.network.aggregation_policy_hash + policy_raw = _read_at(root_fd, Path("objects") / policy_hash[:2] / policy_hash, 65536) + if sha256_hex(policy_raw) != manifest.network.aggregation_policy_hash: + raise ValueError("capacity policy hash differs") + policy = AggregationPolicyV2.model_validate_json(policy_raw) + allocation = allocate_weights( + [Candidate(r.roster, r.commit_hash, r.delta_manifest_hash) for r in records], policy + ) + if ( + body.run_id != manifest.run_id() + or body.w != request.w + or body.inputs != records + or body.excluded + or body.input_root != sha256_hex(canonicalize([r.body() for r in records])) + or body.predecessor_tape_hash != request.predecessor_tape_hash + or request.prev_state is not None + and body.prev_state != request.prev_state + or body.policy_hash != manifest.network.aggregation_policy_hash + or body.allocation != allocation + or body.policy_hashes.body() + != {k: getattr(manifest.network, k) for k in body.policy_hashes.body()} + or set(result.objects) + != ({body.out_state, body.prev_state} if request.prev_state is None else {body.out_state}) + ): + raise ValueError("capacity output authority differs") + objects = {} + for key in result.objects: + data = _read_at(root_fd, Path("objects") / key[:2] / key, 65536) + if sha256_hex(data) != key: + raise ValueError("capacity output hash differs") + objects[key] = data + return body, objects + + +def execute(directory: Path) -> None: + """Child owns init/load/decode, original-object verification and two arithmetic passes.""" + with _directory_fd(directory) as root_fd: + _execute(root_fd) + + +def _execute(root_fd: int) -> None: + import hypertrain.trainer # noqa: F401 + from hypertrain.aggregator.core import OuterState, load_state + from hypertrain.trainer.compress import validate_payload + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.model import init_params, param_shapes + + raw = _read_at(root_fd, Path("request.json"), 1 << 20) + parsed = load_json(raw, max_bytes=1 << 20) + if isinstance(parsed, dict) and parsed.get("operation") == "genesis": + request_genesis = GenesisRequest.model_validate(parsed) + if canonicalize(request_genesis.body()) != raw: + raise ValueError("noncanonical genesis request") + _genesis(root_fd, request_genesis) + return + request = OuterRequest.parse(raw) + config = TrainConfig.from_manifest_v2(request.manifest) + shapes = param_shapes(config.model) + deltas = {x.commit.delta_hash for x in request.inputs} + + class Objects(LocalFSStore): + def __init__(self) -> None: + pass + + def get(self, key: str) -> bytes: + from hypertrain.data.store import _check_key + + _check_key(key) + data = _read_at(root_fd, Path("objects") / key[:2] / key, 65536) + if sha256_hex(data) != key: + raise ValueError("capacity original object hash differs") + if ( + key in deltas + and validate_payload(data, shapes, max_payload_bytes=65536) != config.compress.codec + ): + raise ValueError("capacity delta codec differs") + return data + + def put(self, data: bytes) -> str: + if len(data) > 65536: + raise ValueError("capacity output bytes exceed limit") + key = sha256_hex(data) + _write_at(root_fd, Path("objects") / key[:2] / key, data) + return key + + store = Objects() + for key in request.objects: + store.get(key) + manifest = request.manifest + policy = AggregationPolicyV2.model_validate_json( + store.get(manifest.network.aggregation_policy_hash) + ) + economics = store.get(manifest.network.economics_policy_hash) + previous = request.prev_state + output_objects = [] + if previous is None: + theta = init_params(config.model) + previous = store.put( + OuterState.init({k: v.detach().cpu().numpy() for k, v in theta.items()}).to_bytes() + ) + output_objects.append(previous) + previous_state = load_state(store, previous) + if {name: tuple(value.shape) for name, value in previous_state.theta.items()} != shapes: + raise ValueError("capacity predecessor geometry differs from manifest") + works = [x.work() for x in request.inputs] + body, state = compute_body( + store, + manifest, + policy, + economics, + w=request.w, + prev_state=previous, + predecessor_tape_hash=request.predecessor_tape_hash, + inputs=works, + reference_reward_units=request.reference_reward_units, + ) + state_bytes = state.to_bytes() + if store.put(state_bytes) != body.out_state: + raise ValueError("capacity output object differs") + replayed_body, replayed_state = compute_body( + store, + manifest, + policy, + economics, + w=request.w, + prev_state=previous, + predecessor_tape_hash=request.predecessor_tape_hash, + inputs=works, + reference_reward_units=request.reference_reward_units, + ) + if replayed_body != body or replayed_state.to_bytes() != state_bytes: + raise ValueError("capacity independent replay differs") + output_objects.append(body.out_state) + result = OuterResult( + request_hash=sha256_hex(raw), + tape_body=body, + objects=list(dict.fromkeys(output_objects)), + replayed=True, + ) + _write_at(root_fd, Path("result.json"), canonicalize(result.body())) + print("CAPACITY_OUTER_REPLAY_COMPLETE", flush=True) + + +if __name__ == "__main__": + if len(sys.argv) != 2: + raise SystemExit("capacity worker requires one private attempt directory") + execute(Path(sys.argv[1])) diff --git a/src/hypertrain/aggregator/checkpoint.py b/src/hypertrain/aggregator/checkpoint.py index 3ec3152..c57a1e3 100644 --- a/src/hypertrain/aggregator/checkpoint.py +++ b/src/hypertrain/aggregator/checkpoint.py @@ -74,7 +74,9 @@ def write_checkpoint( if not tapes: raise CheckpointError("a checkpoint needs at least one global tape") d.mkdir(parents=True, exist_ok=False) - model = st_save({n: np.ascontiguousarray(theta[n], dtype=" None: + """Guard the complete accepted roster before every v2 compute/make/replay call.""" + if len(roster) > 16: + raise CheckpointError("ROSTER_LIMIT_16: complete roster requires runtime qualification") + + +def network_inputs(data: Sequence[Mapping[str, Any]]) -> list[Any]: + from hypertrain.aggregator.tape_v2 import VerifiedInput + from hypertrain.challenge.trust_v2 import FundedStatus, ReplayEvidence, SettlementStatus + from hypertrain.protocol.messages_v2 import CommitV2, DeltaManifestV2, RosterEntryV2 + + return [ + VerifiedInput( + RosterEntryV2.model_validate(x["roster"]), + CommitV2.model_validate(x["commit"]), + DeltaManifestV2.model_validate(x["delta_manifest"]), + ReplayEvidence(**x["replay"]), + FundedStatus(**x["funding"]), + SettlementStatus(**x["settlement"]), + x["assignment_hash"], + x["clean_finalizations"], + ) + for x in data + ] + + +def _network_subjects( + item: Mapping[str, Any], + tape: Any, + manifest: Any, + signer: str, + previous: Mapping[str, Any] | None, +) -> None: + from hypertrain.protocol import envelope_v2 + from hypertrain.protocol.messages import Finalize + from hypertrain.protocol.messages_v2 import RoundOpenV2 + + if "repair_context" in item: + _repair_subjects(item, tape, manifest, signer, previous) + return + run_id = manifest.run_id() + for field, expected in (("round_open", "RoundOpenV2"), ("finalize", "Finalize")): + try: + env = envelope_v2.parse_envelope(item[field]) + except ValueError as error: + raise CheckpointError(f"v2 {field} invalid envelope: {error}") from error + if ( + env.type != expected + or env.run_id != run_id + or env.signer != signer + or (not envelope_v2.verify_envelope(env.model_dump())) + ): + raise CheckpointError(f"v2 {field} authority/subject mismatch") + opening = RoundOpenV2.model_validate(item["round_open"]["body"]) + final = Finalize.model_validate(item["finalize"]["body"]) + contributors = [work["roster"]["hotkey"] for work in item["inputs"]] + roster = {entry.hotkey: entry.body() for entry in opening.roster} + if len(contributors) != len(set(contributors)) or any( + work["roster"] != roster.get(work["roster"]["hotkey"]) for work in item["inputs"] + ): + raise CheckpointError("v2 contributors differ from accepted round roster") + expected_included = sorted(contributors, key=lambda hot: hot.encode("utf-8")) + if ( + final.w != opening.w + or tape.body.w != opening.w + or tape.body.run_id != run_id + or ( + final.included != expected_included + or final.final_theta_hash_w1 != tape.body.out_hashes["theta_hash"] + ) + ): + raise CheckpointError("v2 finality round/contributor/output subject mismatch") + if opening.theta_hash != tape.body.prev_hashes["theta_hash"] or ( + tape.body.prev_state != item["prev_state"] + or tape.body.predecessor_tape_hash != item["predecessor_tape_hash"] + ): + raise CheckpointError("v2 round/tape state lineage mismatch") + if previous is not None: + if ( + opening.w != previous["w"] + 1 + or item["prev_state"] != previous["out_state"] + or ( + item["predecessor_tape_hash"] != previous["tape_hash"] + or opening.prev_final_hash != previous["final_hash"] + ) + ): + raise CheckpointError("v2 checkpoint rounds/predecessors are not contiguous") + elif opening.w == 0 and item["predecessor_tape_hash"] != "0" * 64: + raise CheckpointError("v2 genesis predecessor mismatch") + + +def _repair_subjects( + item: Mapping[str, Any], + tape: Any, + manifest: Any, + signer: str, + previous: Mapping[str, Any] | None, +) -> None: + from hypertrain.protocol import envelope_v2 + from hypertrain.protocol.messages import Rollback + from hypertrain.protocol.messages_v2 import RoundOpenV2 + + opening = envelope_v2.parse_envelope(item["round_open"]) + rollback = envelope_v2.parse_envelope(item["rollback"]) + if any( + not envelope_v2.verify_envelope(e.model_dump()) + or e.signer != signer + or e.run_id != manifest.run_id() + for e in (opening, rollback) + ) or (opening.type != "RoundOpenV2" or rollback.type != "Rollback"): + raise CheckpointError("repair checkpoint subject authority differs") + original_open = RoundOpenV2.model_validate(opening.body) + request = Rollback.model_validate(rollback.body) + context = item["repair_context"]["body"] + roster = {r.hotkey: r.body() for r in original_open.roster} + inputs = [s["work"] for s in context["sources"]] + if any(work["roster"]["hotkey"] in request.excluded for work in item["inputs"]): + raise CheckpointError("repair excluded identity still included") + if context["w"] == request.w + 1 and previous is None: + raise CheckpointError("repair checkpoint requires both repaired rounds") + if set(request.excluded) != {e["hotkey"] for e in context["excluded"]} or ( + set(request.cause_hashes) != {e["evidence_hash"] for e in context["excluded"]} + ): + raise CheckpointError("repair checkpoint exact exclusion/cause set differs") + if any(x["roster"] != roster.get(x["roster"]["hotkey"]) for x in inputs) or ( + context["w"] != original_open.w + or tape.body.w != original_open.w + or original_open.w not in (request.w, request.w + 1) + or context["prev_state"] != tape.body.prev_state + or context["predecessor_tape_hash"] != tape.body.predecessor_tape_hash + or item["prev_state"] != tape.body.prev_state + or item["predecessor_tape_hash"] != tape.body.predecessor_tape_hash + or request.recomputed != ["agg_w", "step_w", "agg_w1", "step_w1"] + or inputs != item["inputs"] + ): + raise CheckpointError("repair checkpoint original roster/round/lineage differs") + if original_open.w == request.w + 1 and ( + request.new_theta_hash_w2 != tape.body.out_hashes["theta_hash"] + or request.new_outer_state_hash != tape.body.out_state + ): + raise CheckpointError("repair checkpoint terminal output differs") + if previous is not None and ( + original_open.w != previous["w"] + 1 + or item["prev_state"] != previous["out_state"] + or item["predecessor_tape_hash"] != previous["tape_hash"] + ): + raise CheckpointError("repair checkpoint predecessor differs") + if previous is None and original_open.w == 0 and item["predecessor_tape_hash"] != "0" * 64: + raise CheckpointError("repair genesis predecessor differs") + + +def _network_tape(store: Any, item: Mapping[str, Any]) -> Any: + from types import SimpleNamespace + + from hypertrain.aggregator.rollback_v2 import RepairTape + from hypertrain.aggregator.tape_v2 import TapeV2 + + if "repair_context" in item: + repair = RepairTape.from_bytes(store.get(item["tape_hash"])) + return SimpleNamespace(body=repair.body.arithmetic, repair=repair) + return TapeV2.from_bytes(store.get(item["tape_hash"])) + + +def _network_replay( + store: Any, + item: Mapping[str, Any], + tape: Any, + manifest: Any, + policy: Any, + economics: bytes, + signer: str, + prior_anchors: Any, +) -> tuple[Any, Any]: + from hypertrain.aggregator.rollback_v2 import execute_repair_context + from hypertrain.aggregator.tape_v2 import replay_tape + + if "repair_context" not in item: + state = replay_tape( + store, + tape, + manifest, + policy, + economics, + signer=signer, + w=tape.body.w, + prev_state=item["prev_state"], + predecessor_tape_hash=item["predecessor_tape_hash"], + inputs=network_inputs(item["inputs"]), + reference_reward_units=item["reference_reward_units"], + ) + return state, None + from hypertrain.auditor.replay import pack_state + + context = item["repair_context"] + if tape.body.w > 0 and prior_anchors is None: + from hypertrain.aggregator.rollback_v2 import replay_history_context + + if not context["body"].get("history") or len(context["body"]["history"]) != tape.body.w: + raise CheckpointError( + "first non-genesis repair requires complete authenticated predecessor history" + ) + last = _network_tape(store, context["body"]["history"][-1]) + if (last.body.out_state, context["body"]["history"][-1]["tape_hash"]) != ( + item["prev_state"], + item["predecessor_tape_hash"], + ): + raise CheckpointError("authenticated predecessor history global endpoint differs") + prior_anchors = replay_history_context(store, manifest, context, signer) + from hypertrain.aggregator.tape_v2 import TapeV2 + + original_tape = TapeV2.from_bytes(store.get(context["body"]["source_tape_hash"])) + if tape.body.w == item["rollback"]["body"]["w"] and ( + original_tape.body.prev_state != item["prev_state"] + or original_tape.body.predecessor_tape_hash != item["predecessor_tape_hash"] + or original_tape.body.prev_hashes["theta_hash"] != item["round_open"]["body"]["theta_hash"] + ): + raise CheckpointError("repair original round predecessor differs") + if tape.body.w == item["rollback"]["body"]["w"] + 1 and ( + original_tape.body.out_hashes["theta_hash"] != item["rollback"]["body"]["old_theta_hash_w2"] + ): + raise CheckpointError("repair original terminal theta differs") + if prior_anchors is not None: + anchors = {a.hotkey: a for a in prior_anchors} + for source in context["body"]["sources"]: + a = source["anchor"] + expected = anchors.get(a["hotkey"]) + if expected is None or ( + a["anchor_hash"] != expected.anchor_hash + or a["proof_hash"] != expected.proof_hash + or a["state_object"] != sha256_hex(pack_state(expected.theta, expected.state)) + or a["ef_object"] != sha256_hex(pack_state(expected.ef)) + ): + raise CheckpointError( + "repair checkpoint carried anchor differs from independent replay" + ) + result = execute_repair_context(store, manifest, context, signer, tape=tape.repair) + return result.state, result.anchors + + +def write_network_checkpoint( + directory: Path, key: Keypair, store: Any, manifest: Any, rounds: Sequence[Mapping[str, Any]] +) -> dict[str, Any]: + """Replay each full accepted roster and tape before emitting a signed v2 checkpoint.""" + from hypertrain.aggregator.tape_v2 import TapeV2 + from hypertrain.protocol.messages_v2 import AggregationPolicyV2, RoundOpenV2 + + for item in rounds: + require_network_roster(RoundOpenV2.model_validate(item["round_open"]["body"]).roster) + if not rounds: + raise CheckpointError("checkpoint requires accepted finalized rounds") + policy = AggregationPolicyV2.model_validate_json( + store.get(manifest.network.aggregation_policy_hash) + ) + economics = store.get(manifest.network.economics_policy_hash) + state = None + previous = None + prior_anchors = None + pending_repair = None + for item in rounds: + from hypertrain.protocol import envelope_v2 + + opening = item["round_open"] + if ( + opening["signer"] != key.ss58 + or opening["run_id"] != manifest.run_id() + or not envelope_v2.verify_envelope(opening) + ): + raise CheckpointError("round opening authority mismatch") + tape = _network_tape(store, item) + if "repair_context" in item: + request_body = item["rollback"]["body"] + if pending_repair is None: + if tape.body.w != request_body["w"]: + raise CheckpointError("checkpoint repair pair must start at excluded round") + pending_repair = request_body + else: + if request_body != pending_repair or tape.body.w != pending_repair["w"] + 1: + raise CheckpointError("checkpoint repair pair authority differs") + pending_repair = None + elif pending_repair is not None: + raise CheckpointError("checkpoint repair pair interrupted") + _network_subjects(item, tape, manifest, key.ss58, previous) + require_network_roster(item["round_open"]["body"]["roster"]) + state, prior_anchors = _network_replay( + store, + item, + tape, + manifest, + policy, + economics, + key.ss58, + prior_anchors, + ) + final = item.get("rollback", item.get("finalize")) + from hypertrain.protocol import envelope_v2 + + if ( + not envelope_v2.verify_envelope(final) + or final["signer"] != key.ss58 + or ( + "repair_context" not in item + and final["body"]["final_theta_hash_w1"] != tape.body.out_hashes["theta_hash"] + ) + ): + raise CheckpointError("unaccepted finality linkage") + previous = { + "w": tape.body.w, + "out_state": tape.body.out_state, + "tape_hash": item["tape_hash"], + "final_hash": sha256_hex(canonicalize(final["body"])), + } + if pending_repair is not None: + raise CheckpointError("checkpoint incomplete repair pair") + assert state is not None + directory.mkdir(parents=True, exist_ok=False) + model = st_save( + {n: np.ascontiguousarray(a, dtype=" list[str]: + from hypertrain.data.store import LocalFSStore + from hypertrain.protocol.messages_v2 import AggregationPolicyV2, RoundOpenV2, RunManifestV2 + + doc = read_manifest(directory) + body = doc["body"] + message = b"hypertrain/checkpoint/2|" + sha256_hex(canonicalize(body)).encode() + if doc["signer"] != signer or not verify( + decode_hotkey(signer), message, bytes.fromhex(doc["sig"]) + ): + return ["v2 checkpoint authority/signature mismatch"] + lineage = json.loads((directory / LINEAGE).read_bytes()) + # COMPLETE rosters, before object reads or replay cost. + for item in lineage["rounds"]: + require_network_roster(RoundOpenV2.model_validate(item["round_open"]["body"]).roster) + for name, digest in body["files"].items(): + if sha256_hex((directory / name).read_bytes()) != digest: + return [f"{name}: hash mismatch"] + manifest = RunManifestV2.model_validate(lineage["manifest"]) + if body["run_id"] != manifest.run_id(): + return ["v2 checkpoint run mismatch"] + if not lineage["rounds"] or body["v"] != "ht-ckpt-v2" or body["status"] != "final": + return ["v2 checkpoint version/status/empty lineage mismatch"] + expected_range = [ + lineage["rounds"][0]["round_open"]["body"]["w"], + lineage["rounds"][-1]["round_open"]["body"]["w"], + ] + if body["rounds"] != expected_range or body["included"] != sorted( + work["roster"]["hotkey"] for work in lineage["rounds"][-1]["inputs"] + ): + return ["v2 checkpoint manifest round/contributor subject mismatch"] + store = LocalFSStore(directory / "objects") + policy = AggregationPolicyV2.model_validate_json( + store.get(manifest.network.aggregation_policy_hash) + ) + econ = store.get(manifest.network.economics_policy_hash) + previous = None + prior_anchors = None + pending_repair = None + for item in lineage["rounds"]: + from hypertrain.protocol import envelope_v2 + + if ( + not envelope_v2.verify_envelope(item["round_open"]) + or item["round_open"]["signer"] != signer + ): + return ["v2 round opening authority mismatch"] + tape = _network_tape(store, item) + if "repair_context" in item: + request_body = item["rollback"]["body"] + if pending_repair is None: + if tape.body.w != request_body["w"]: + return ["checkpoint repair pair must start at excluded round"] + pending_repair = request_body + else: + if request_body != pending_repair or tape.body.w != pending_repair["w"] + 1: + return ["checkpoint repair pair authority differs"] + pending_repair = None + elif pending_repair is not None: + return ["checkpoint repair pair interrupted"] + try: + _network_subjects(item, tape, manifest, signer, previous) + except CheckpointError as error: + return [str(error)] + require_network_roster(item["round_open"]["body"]["roster"]) + try: + state, prior_anchors = _network_replay( + store, + item, + tape, + manifest, + policy, + econ, + signer, + prior_anchors, + ) + except (ValueError, CheckpointError) as error: + return [str(error)] + final = item.get("rollback", item.get("finalize")) + if ( + not envelope_v2.verify_envelope(final) + or final["signer"] != signer + or final["run_id"] != manifest.run_id() + or ( + "repair_context" not in item + and final["body"]["final_theta_hash_w1"] != tape.body.out_hashes["theta_hash"] + ) + ): + return ["v2 checkpoint finality mismatch"] + previous = { + "w": tape.body.w, + "out_state": tape.body.out_state, + "tape_hash": item["tape_hash"], + "final_hash": sha256_hex(canonicalize(final["body"])), + } + if pending_repair is not None: + return ["checkpoint incomplete repair pair"] + if th(st_load((directory / MODEL).read_bytes())) != th(state.theta) or body["theta_hash"] != th( + state.theta + ): + return ["checkpoint model differs from independently replayed tape"] + return [] diff --git a/src/hypertrain/aggregator/cli.py b/src/hypertrain/aggregator/cli.py index 34a7748..5bd91dc 100644 --- a/src/hypertrain/aggregator/cli.py +++ b/src/hypertrain/aggregator/cli.py @@ -8,7 +8,14 @@ from collections.abc import Sequence from pathlib import Path -from hypertrain.aggregator.checkpoint import read_manifest, verify_checkpoint +from hypertrain.aggregator.checkpoint import ( + network_inputs, + read_manifest, + require_network_roster, + verify_checkpoint, + verify_network_checkpoint, + write_network_checkpoint, +) def main(argv: Sequence[str] | None = None) -> int: @@ -17,18 +24,96 @@ def main(argv: Sequence[str] | None = None) -> int: vc = sub.add_parser("verify-checkpoint", help="verify a final checkpoint directory") vc.add_argument("dir", type=Path) vc.add_argument("--signer", help="expected coordinator ss58 (default: manifest signer)") + vn = sub.add_parser("verify-checkpoint-v2") + vn.add_argument("dir", type=Path) + vn.add_argument("--signer", required=True) + nc = sub.add_parser("checkpoint-v2") + nc.add_argument("bundle", type=Path) + nc.add_argument("dir", type=Path) + nc.add_argument("--objects", type=Path, required=True) + nc.add_argument("--keyfile", type=Path, required=True) + ar = sub.add_parser("aggregate-v2") + ar.add_argument("bundle", type=Path) + ar.add_argument("--objects", type=Path, required=True) + ar.add_argument("--keyfile", type=Path, required=True) + rs = sub.add_parser("relay-serve") + rs.add_argument("--host", default="127.0.0.1") + rs.add_argument("--port", type=int, default=8080) args = ap.parse_args(argv) + if args.cmd == "relay-serve": + import uvicorn + + uvicorn.run("hypertrain.relay.app:app", host=args.host, port=args.port) + return 0 + if args.cmd in ("checkpoint-v2", "aggregate-v2"): + from hypertrain.aggregator.tape_v2 import make_tape, replay_tape + from hypertrain.data.store import LocalFSStore + from hypertrain.miner.core import load_keyfile + from hypertrain.protocol.messages_v2 import AggregationPolicyV2, RoundOpenV2, RunManifestV2 + + bundle = json.loads(args.bundle.read_bytes()) + rounds = bundle["rounds"] + for item in rounds: + require_network_roster(RoundOpenV2.model_validate(item["round_open"]["body"]).roster) + manifest = RunManifestV2.model_validate(bundle["manifest"]) + store = LocalFSStore(args.objects) + key = load_keyfile(args.keyfile) + if args.cmd == "checkpoint-v2": + result = write_network_checkpoint(args.dir, key, store, manifest, rounds) + else: + item = rounds[-1] + policy = AggregationPolicyV2.model_validate_json( + store.get(manifest.network.aggregation_policy_hash) + ) + economics = store.get(manifest.network.economics_policy_hash) + require_network_roster(item["round_open"]["body"]["roster"]) + inputs = network_inputs(item["inputs"]) + tape = make_tape( + store, + manifest, + policy, + economics, + key, + w=item["round_open"]["body"]["w"], + prev_state=item["prev_state"], + predecessor_tape_hash=item["predecessor_tape_hash"], + inputs=inputs, + reference_reward_units=item["reference_reward_units"], + ) + require_network_roster(item["round_open"]["body"]["roster"]) + replay_tape( + store, + tape, + manifest, + policy, + economics, + signer=key.ss58, + w=tape.body.w, + prev_state=item["prev_state"], + predecessor_tape_hash=item["predecessor_tape_hash"], + inputs=inputs, + reference_reward_units=item["reference_reward_units"], + ) + result = {"tape_hash": store.put(tape.to_bytes())} + print(json.dumps(result, sort_keys=True)) + return 0 if not args.dir.is_dir(): print(f"FAIL: {args.dir} is not a directory", file=sys.stderr) return 2 - errors = verify_checkpoint(args.dir, signer=args.signer) + errors = ( + verify_network_checkpoint(args.dir, args.signer) + if args.cmd == "verify-checkpoint-v2" + else verify_checkpoint(args.dir, signer=args.signer) + ) if errors: for e in errors: print(f"FAIL: {e}", file=sys.stderr) return 1 b = read_manifest(args.dir)["body"] summary = { - k: b[k] for k in ("run_id", "rounds", "theta_hash", "included", "license", "dataset") + k: b[k] + for k in ("run_id", "rounds", "theta_hash", "included", "license", "dataset") + if k in b } print(json.dumps({"result": "VERIFIED", **summary}, sort_keys=True)) return 0 diff --git a/src/hypertrain/aggregator/core.py b/src/hypertrain/aggregator/core.py index aa1d61e..49a058b 100644 --- a/src/hypertrain/aggregator/core.py +++ b/src/hypertrain/aggregator/core.py @@ -147,7 +147,7 @@ def to_bytes(self) -> bytes: flat: dict[str, Arr] = {} for part in _PARTS: for n, a in getattr(self, part).items(): - flat[f"{part}/{n}"] = np.ascontiguousarray(a, dtype=" RepairResult: + """Fresh qualified torchrun computes/replays a coordinator-authenticated repair context.""" + from hypertrain.protocol.jcs import canonicalize + + raw = canonicalize(context) + message = b"hypertrain/rollback/context/1|" + sha256_hex(canonicalize(context["body"])).encode() + if context["signer"] != signer or not verify( + decode_hotkey(signer), message, bytes.fromhex(context["sig"]) + ): + raise TapeError("repair context authority mismatch") + body = context["body"] + if body["run_id"] != manifest.run_id() or body["backend"] not in ("cpu", "cuda"): + raise TapeError("repair context runtime/run mismatch") + if body["qualification_hash"] != sha256_hex(canonicalize(body["qualification"])): + raise TapeError("repair context qualification object differs") + _qualification_body(manifest, body) + if body["manifest"] != manifest.body(): + raise TapeError("repair context embedded manifest differs") + if body["w"] > 0 and body.get("history"): + last = body["history"][-1] + prior_raw = get_object(store, last["tape_hash"]) + prior = ( + RepairTape.from_bytes(prior_raw).body.arithmetic + if "repair_context" in last + else TapeV2.from_bytes(prior_raw).body + ) + if len(body["history"]) != body["w"] or ( + prior.w != body["w"] - 1 + or prior.out_state != body["prev_state"] + or last["tape_hash"] != body["predecessor_tape_hash"] + ): + raise TapeError("repair authenticated predecessor history endpoint differs") + if body["timeout_seconds"] < 1 or body["timeout_seconds"] > 600: + raise TapeError("repair context timeout budget differs") + from hypertrain.protocol.messages_v2 import DisputePolicyV2 + + dispute_policy = DisputePolicyV2.model_validate_json( + get_object(store, manifest.network.dispute_policy_hash) + ) + proof_ids = set() + for proof in body["exclusion_proofs"]: + resolution = proof["resolution"] + evidence = proof["referee_evidence"] + if ( + not verify_source(resolution) + or resolution["type"] != "ResolutionV2" + or resolution["run_id"] != manifest.run_id() + or resolution["signer"] not in dispute_policy.referees + or resolution["body"]["reason"] not in ("FRAUD", "WITHHELD", "PARTY_TIMEOUT") + or resolution["body"]["evidence_hash"] != sha256_hex(canonicalize(evidence)) + or resolution["body"]["loser"] != evidence["loser"] + or resolution["body"]["reason"] != evidence["reason"] + or resolution["body"]["dispute_id"] != evidence["dispute_id"] + or resolution["body"]["transcript_hash"] != evidence["transcript_hash"] + ): + raise TapeError("repair exclusion independent authority differs") + proof_ids.add((resolution["body"]["loser"], sha256_hex(canonicalize(resolution["body"])))) + if not {(e["hotkey"], e["evidence_hash"]) for e in body["excluded"]} <= proof_ids: + raise TapeError("repair exclusions lack exact independent authority") + for source in body["sources"]: + for receipt, envelope_hash in zip( + source["receipts"], + ( + source["commit_envelope_hash"], + source["delta_envelope_hash"], + ), + strict=True, + ): + original = parse_envelope(get_object(store, envelope_hash)) + if ( + not verify_source(receipt) + or receipt["signer"] != signer + or ( + receipt["type"] != "Receipt" + or receipt["run_id"] != manifest.run_id() + or receipt["body"]["w"] != body["w"] + or receipt["body"]["commit_hash"] != sha256_hex(canonicalize(original.body)) + ) + ): + raise TapeError("repair historical intake receipt differs") + if source["acceptance_hash"] != sha256_hex(canonicalize(source["receipts"])): + raise TapeError("repair acceptance receipt hash differs") + if tape is not None and ( + tape.signer != signer + or not verify(decode_hotkey(signer), repair_message(tape.body), bytes.fromhex(tape.sig)) + ): + raise TapeError("repair candidate signature mismatch") + context_hash = store.put(raw) + with tempfile.TemporaryDirectory(prefix="hypertrain-repair-") as tmp: + env = os.environ.copy() + env["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" + env["OMP_NUM_THREADS"] = "1" + argv = [ + sys.executable, + "-m", + "torch.distributed.run", + "--standalone", + f"--nproc-per-node={manifest.training.reference_spec.layout.n_gpus}", + "--max-restarts=0", + "-m", + "hypertrain.aggregator.rollback_v2", + str(store.root), + context_hash, + tmp, + signer, + ] + try: + subprocess.run( # noqa: S603 - interpreter-owned torchrun, signed context only + argv, + env=env, + check=True, + capture_output=True, + timeout=body["timeout_seconds"], + ) + except subprocess.CalledProcessError as error: + raise TapeError( + "qualified repair worker failed: " + error.stderr.decode()[-6000:] + ) from error + computed = RepairBody.model_validate_json((Path(tmp) / "body.json").read_bytes()) + state = load_state(store, computed.arithmetic.out_state) + if tape is not None: + if computed != tape.body: + raise TapeError("fresh independent repair context replay differs") + else: + if key is None or key.ss58 != signer: + raise TapeError("repair context signing authority unavailable") + tape = RepairTape( + body=computed, signer=signer, sig=key.sign(repair_message(computed)).hex() + ) + return RepairResult(tape, state, context_anchors(store, computed)) + + +def _qualification_body(manifest: RunManifestV2, body: dict[str, Any]) -> None: + selection = body["qualification"]["execution_backend"] + if selection["backend"] != body["backend"]: + raise TapeError("repair frozen backend selection differs") + if body["backend"] == "cpu": + if selection["authority_hash"] is not None: + raise TapeError("CPU repair cannot substitute CUDA qualification") + else: + reviewed = body["qualification"]["reviewed_qualification"] + ref = manifest.training.reference_spec + if sha256_hex(canonicalize(reviewed)) != selection["authority_hash"] or ( + reviewed["run_id"] != manifest.run_id() + or reviewed["backend"] != "cuda" + or reviewed["reference_hash"] != body["reference_hash"] + or reviewed["layout_hash"] != ref.layout.model_dump_json() + ): + raise TapeError("repair exact reviewed CUDA qualification differs") + + +def context_anchors(store: Store, body: RepairBody) -> tuple[VerifiedAnchor, ...]: + from hypertrain.auditor.replay import unpack_state + + anchors = [] + for e in body.recomputed: + theta, state = unpack_state(get_object(store, e.final_state_object)) + ef, _ = unpack_state(get_object(store, e.final_ef_object)) + if state is None: + raise TapeError("repair output optimizer absent") + if any( + not bool(torch.isfinite(x).all()) + for values in (theta, state.m, state.v, ef) + for x in values.values() + ): + raise TapeError("nonfinite repair output artifact") + if ( + optimizer_hash(state) != e.final_optimizer_hash + or state_hash(theta) != e.final_theta_hash + or state_hash(ef) != e.ef_out_hash + ): + raise TapeError("repair output artifact commitments differ") + proof = sha256_hex( + canonicalize( + [ + "ht-rollback-anchor/1", + body.arithmetic.run_id, + body.arithmetic.w, + body.arithmetic.predecessor_tape_hash, + body.arithmetic.prev_state, + body.source_tape_hash, + body.qualification_hash, + body.reference_hash, + body.layout_hash, + body.backend, + e.body(), + ] + ) + ) + anchors.append( + VerifiedAnchor( + body.arithmetic.run_id, + e.hotkey, + body.arithmetic.w, + body.layout_hash, + state, + ef, + proof, + sha256_hex(b"hypertrain/rollback/anchor/1|" + bytes.fromhex(proof)), + theta, + body.backend, + ) + ) + return tuple(anchors) + + +def replay_history_context( + store: Any, + manifest: RunManifestV2, + context: dict[str, Any], + signer: str, +) -> tuple[VerifiedAnchor, ...]: + """Authenticated ordinary/repair history, independently carried from deterministic genesis.""" + message = b"hypertrain/rollback/context/1|" + sha256_hex(canonicalize(context["body"])).encode() + if ( + context["signer"] != signer + or not verify(decode_hotkey(signer), message, bytes.fromhex(context["sig"])) + or context["body"]["manifest"] != manifest.body() + ): + raise TapeError("predecessor history authority differs") + context_hash = store.put(canonicalize(context)) + with tempfile.TemporaryDirectory(prefix="hypertrain-history-") as tmp: + env = os.environ.copy() + env["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" + env["OMP_NUM_THREADS"] = "1" + process = subprocess.run( # noqa: S603 - fixed interpreter, authenticated context + [ + sys.executable, + "-m", + "torch.distributed.run", + "--standalone", + f"--nproc-per-node={manifest.training.reference_spec.layout.n_gpus}", + "--max-restarts=0", + "-m", + "hypertrain.aggregator.rollback_v2", + str(store.root), + context_hash, + tmp, + signer, + "history", + ], + env=env, + capture_output=True, + timeout=context["body"]["timeout_seconds"], + ) + if process.returncode: + raise TapeError("predecessor history replay failed: " + process.stderr.decode()[-6000:]) + records = json.loads((Path(tmp) / "anchors.json").read_bytes()) + from hypertrain.auditor.replay import unpack_state + + anchors = [] + for record in records: + theta, state = unpack_state(get_object(store, record["state_object"])) + ef, _ = unpack_state(get_object(store, record["ef_object"])) + if state is None: + raise TapeError("predecessor history optimizer absent") + anchors.append( + VerifiedAnchor( + manifest.run_id(), + record["hotkey"], + record["w"], + record["layout_hash"], + state, + ef, + record["proof_hash"], + record["anchor_hash"], + theta, + record["backend"], + ) + ) + return tuple(anchors) + + +def _replay_history( + store: Store, + manifest: RunManifestV2, + entries: Sequence[dict[str, Any]], + signer: str, + comm: Comm, + get_sample: SampleFn, + backend: Literal["cpu", "cuda"], +) -> tuple[VerifiedAnchor, ...]: + """No hash assertion can substitute for actual inner work and outer predecessor replay.""" + from hypertrain.aggregator.checkpoint import network_inputs + from hypertrain.aggregator.tape_v2 import replay_tape + from hypertrain.auditor.replay import audit_island, unpack_state + from hypertrain.protocol.messages_v2 import AuditJobV2, RoundOpenV2 + from hypertrain.trainer.model import init_params + + if not entries or len(entries) > 16: + raise TapeError("bounded complete predecessor history required") + cache = AnchorCache() + previous = None + previous_hash = "0" * 64 + policy = AggregationPolicyV2.model_validate_json( + get_object(store, manifest.network.aggregation_policy_hash) + ) + economics = get_object(store, manifest.network.economics_policy_hash) + for w, item in enumerate(entries): + tape_hash = item["tape_hash"] + raw = get_object(store, tape_hash) + if "repair_context" in item: + candidate = RepairTape.from_bytes(raw) + c = item["repair_context"] + message = ( + b"hypertrain/rollback/context/1|" + sha256_hex(canonicalize(c["body"])).encode() + ) + if c["signer"] != signer or not verify( + decode_hotkey(signer), message, bytes.fromhex(c["sig"]) + ): + raise TapeError("predecessor repair context authority differs") + if candidate.signer != signer or not verify( + decode_hotkey(signer), repair_message(candidate.body), bytes.fromhex(candidate.sig) + ): + raise TapeError("predecessor repair signature differs") + b = c["body"] + sources = [] + for record in b["sources"]: + a = record["anchor"] + if w == 0: + anchor = cache.genesis( + manifest, + a["hotkey"], + init_params(TrainConfig.from_manifest_v2(manifest).model), + ) + else: + anchor = cache.entries[ + (manifest.run_id(), a["hotkey"], w - 1, AnchorCache.layout_hash(manifest)) + ] + if (a["anchor_hash"], a["proof_hash"], a["state_object"], a["ef_object"]) != ( + anchor.anchor_hash, + anchor.proof_hash, + sha256_hex(pack_state(anchor.theta, anchor.state)), + sha256_hex(pack_state(anchor.ef)), + ): + raise TapeError("predecessor repair carry differs from replayed history") + sources.append( + RepairSource( + network_inputs([record["work"]])[0], + get_object(store, record["commit_envelope_hash"]), + get_object(store, record["delta_envelope_hash"]), + record["acceptance_hash"], + tuple(record["samples"]), + anchor, + ) + ) + body, state, anchors = _compute_repair( + store, + manifest, + policy, + economics, + RepairAuthority( + signer, + b["source_tape_hash"], + b["qualification_hash"], + b["reference_hash"], + b["layout_hash"], + backend, + ), + comm, + get_sample, + w=w, + prev_state=b["prev_state"], + predecessor_tape_hash=previous_hash, + sources=sources, + excluded=[Exclusion.model_validate(e) for e in b["excluded"]], + reference_reward_units=b["reference_reward_units"], + ) + if candidate.body != body: + raise TapeError("predecessor repair replay differs") + for anchor in anchors: + cache.entries[(manifest.run_id(), anchor.hotkey, w, anchor.layout_hash)] = anchor + arithmetic = body.arithmetic + else: + tape = TapeV2.from_bytes(raw) + arithmetic = tape.body + opening = parse_envelope(item["round_open"]) + if ( + not verify_source(item["round_open"]) + or opening.signer != signer + or opening.type != "RoundOpenV2" + or opening.run_id != manifest.run_id() + or opening.body["w"] != w + ): + raise TapeError("predecessor original round authority differs") + inputs = network_inputs(item["inputs"]) + opening_model = RoundOpenV2.model_validate(opening.body) + if sorted((x.roster for x in inputs), key=lambda r: r.hotkey) != sorted( + opening_model.roster, key=lambda r: r.hotkey + ): + raise TapeError("predecessor original full roster differs") + replay_tape( + store, + tape, + manifest, + policy, + economics, + signer=signer, + w=w, + prev_state=arithmetic.prev_state, + predecessor_tape_hash=previous_hash, + inputs=inputs, + reference_reward_units=item["reference_reward_units"], + ) + jobs = { + job["start_state"]["hotkey"]: AuditJobV2.model_validate(job) + for job in item["audit_jobs"] + } + if set(jobs) != {x.roster.hotkey for x in inputs}: + raise TapeError("predecessor audit history incomplete") + for work in inputs: + job = jobs[work.roster.hotkey] + if w == 0: + cache.genesis( + manifest, + work.roster.hotkey, + init_params(TrainConfig.from_manifest_v2(manifest).model), + ) + if ( + job.manifest != manifest + or job.start_state.w != w + or job.commit_envelope["body"] != work.commit.model_dump(mode="json") + or parse_envelope(job.challenge_envelope).signer != signer + ): + raise TapeError("predecessor audit authority/commit differs") + theta, supplied = unpack_state( + get_object(store, job.start_state.state_object_sha256) + ) + prior_state = load_state(store, arithmetic.prev_state) + if any( + not torch.equal(theta[name].detach().cpu(), torch.from_numpy(value.copy())) + for name, value in prior_state.theta.items() + ) or set(theta) != set(prior_state.theta): + raise TapeError("predecessor exact original global theta bytes differ") + if ( + supplied is None + or state_hash(theta) != arithmetic.prev_hashes["theta_hash"] + or job.start_state.theta_hash != opening.body["theta_hash"] + ): + raise TapeError("predecessor original global theta differs") + _finite_state( + comm, theta, supplied, unpack_state(get_object(store, job.ef_in.sha256))[0] + ) + outcome, result = audit_island( + job, + comm, + get_sample, + get_object(store, job.start_state.state_object_sha256), + get_object(store, job.ef_in.sha256), + get_object(store, job.v0.sha256), + cache, + device=backend, + now_round=job.created_beacon, + ) + _finite_state(comm, result.final_theta, result.final_state, result.ef_out) + if outcome.result != "MATCH": + raise TapeError("predecessor actual carried audit differs") + if ( + arithmetic.w != w + or arithmetic.predecessor_tape_hash != previous_hash + or (previous is not None and arithmetic.prev_state != previous) + ): + raise TapeError("predecessor history outer lineage differs") + if w == 0: + genesis = init_params(TrainConfig.from_manifest_v2(manifest).model) + initial = OuterState.init({n: x.detach().cpu().numpy() for n, x in genesis.items()}) + if ( + state_hash(genesis) != arithmetic.prev_hashes["theta_hash"] + or get_object(store, arithmetic.prev_state) != initial.to_bytes() + ): + raise TapeError("predecessor history global genesis differs") + previous, previous_hash = arithmetic.out_state, tape_hash + return tuple(a for a in cache.entries.values() if a.w == len(entries) - 1) + + +def _context_worker( + store_path: str, context_hash: str, output: str, signer: str, mode: str = "repair" +) -> None: + from datetime import timedelta + + import numpy as np + import torch.distributed as dist + + from hypertrain.aggregator.checkpoint import network_inputs + from hypertrain.auditor.replay import unpack_state + from hypertrain.data.store import LocalFSStore + from hypertrain.protocol.hashing import MerkleTree + from hypertrain.trainer.determinism import apply_reference_env + from hypertrain.trainer.island import DistComm + + store = LocalFSStore(Path(store_path)) + context: dict[str, Any] = json.loads(get_object(store, context_hash)) + message = b"hypertrain/rollback/context/1|" + sha256_hex(canonicalize(context["body"])).encode() + if context["signer"] != signer or not verify( + decode_hotkey(signer), message, bytes.fromhex(context["sig"]) + ): + raise TapeError("worker repair context authority mismatch") + body = context["body"] + manifest = RunManifestV2.model_validate(body["manifest"]) + _qualification_body(manifest, body) + backend = body["backend"] + ref = manifest.training.reference_spec + if backend == "cuda": + rank = int(os.environ["LOCAL_RANK"]) + torch.cuda.set_device(rank) + torch.set_default_device(torch.device("cuda", rank)) + if os.environ.get("HT_IMAGE_DIGEST") != ref.image_digest: + raise TapeError("repair runtime image mismatch") + if torch.__version__ != "2.14.0+cu130": + raise TapeError("repair unqualified CUDA runtime") + smi = shutil.which("nvidia-smi") + if smi is None: + raise TapeError("repair GPU driver inspector absent") + drivers = subprocess.run( # noqa: S603 - PATH-resolved fixed GPU inspection command + [smi, "--query-gpu=driver_version", "--format=csv,noheader"], + check=True, + capture_output=True, + text=True, + timeout=10, + ).stdout.splitlines() + if len(drivers) != ref.layout.n_gpus or drivers[rank].strip() not in ref.driver_allowlist: + raise TapeError("repair unqualified CUDA driver") + if torch.cuda.get_device_properties(rank).multi_processor_count != ref.sm_count: + raise TapeError("repair unqualified CUDA SM profile") + apply_reference_env(ref.env.model_dump()) + dist.init_process_group("gloo" if backend == "cpu" else "nccl", timeout=timedelta(minutes=5)) + try: + sources = [] + for item in body["sources"]: + a = item["anchor"] + theta, state = unpack_state(get_object(store, a["state_object"])) + ef, _ = unpack_state(get_object(store, a["ef_object"])) + if state is None: + raise TapeError("repair source optimizer absent") + anchor = VerifiedAnchor( + manifest.run_id(), + a["hotkey"], + a["w"], + a["layout_hash"], + state, + ef, + a["proof_hash"], + a["anchor_hash"], + theta, + a["backend"], + ) + if a["w"] == -1: + from hypertrain.trainer.model import init_params + + expected = AnchorCache().genesis( + manifest, + a["hotkey"], + init_params(TrainConfig.from_manifest_v2(manifest).model), + ) + if ( + anchor.anchor_hash != expected.anchor_hash + or anchor.proof_hash != expected.proof_hash + or pack_state(anchor.theta, anchor.state) + != pack_state(expected.theta, expected.state) + or pack_state(anchor.ef) != pack_state(expected.ef) + ): + raise TapeError("repair independently reconstructed genesis differs") + sources.append( + RepairSource( + network_inputs([item["work"]])[0], + get_object(store, item["commit_envelope_hash"]), + get_object(store, item["delta_envelope_hash"]), + item["acceptance_hash"], + tuple(item["samples"]), + anchor, + ) + ) + dataset = manifest.training.dataset + dtype = " np.ndarray: + row = rows[i] + if not MerkleTree.verify( + row.tobytes(), + i, + [bytes.fromhex(p) for p in proofs[i]], + bytes.fromhex(dataset.merkle_root), + dataset.n_samples, + ): + raise TapeError("repair dataset proof differs") + return row.astype(np.uint32) + + if mode == "history": + anchors = _replay_history( + store, manifest, body["history"], signer, DistComm(), get_sample, backend + ) + records = [ + { + "hotkey": a.hotkey, + "w": a.w, + "layout_hash": a.layout_hash, + "proof_hash": a.proof_hash, + "anchor_hash": a.anchor_hash, + "backend": a.backend, + "state_object": store.put(pack_state(a.theta, a.state)), + "ef_object": store.put(pack_state(a.ef)), + } + for a in anchors + ] + if dist.get_rank() == 0: + (Path(output) / "anchors.json").write_bytes(canonicalize(records)) + return + + if body["w"] > 0 and body.get("history"): + verified = { + a.hotkey: a + for a in _replay_history( + store, + manifest, + body["history"], + signer, + DistComm(), + get_sample, + backend, + ) + } + for source in sources: + expected = verified[source.work.roster.hotkey] + if ( + source.anchor.anchor_hash, + source.anchor.proof_hash, + pack_state(source.anchor.theta, source.anchor.state), + pack_state(source.anchor.ef), + ) != ( + expected.anchor_hash, + expected.proof_hash, + pack_state(expected.theta, expected.state), + pack_state(expected.ef), + ): + raise TapeError( + "first repair carry differs from authenticated predecessor history" + ) + + authority = RepairAuthority( + signer, + body["source_tape_hash"], + body["qualification_hash"], + body["reference_hash"], + body["layout_hash"], + backend, + ) + computed, outer_state, _ = _compute_repair( + store, + manifest, + AggregationPolicyV2.model_validate_json( + get_object(store, manifest.network.aggregation_policy_hash) + ), + get_object(store, manifest.network.economics_policy_hash), + authority, + DistComm(), + get_sample, + w=body["w"], + prev_state=body["prev_state"], + predecessor_tape_hash=body["predecessor_tape_hash"], + sources=sources, + excluded=[Exclusion.model_validate(e) for e in body["excluded"]], + reference_reward_units=body["reference_reward_units"], + ) + store.put(outer_state.to_bytes()) + if dist.get_rank() == 0: + (Path(output) / "body.json").write_bytes(canonicalize(computed.body())) + finally: + dist.destroy_process_group() + + +@dataclass(frozen=True, slots=True) +class RepairSource: + """L0 accepted journal facts; receipt hash must come from durable intake, not a request.""" + + work: VerifiedInput + commit_envelope: bytes + delta_envelope: bytes + acceptance_hash: str + sample_ids: tuple[int, ...] + anchor: VerifiedAnchor + + +@dataclass(frozen=True, slots=True) +class RepairAuthority: + """Independent L0 pins, resolved from run authority and qualified runtime registry.""" + + coordinator: str + source_tape_hash: str + qualification_hash: str + reference_hash: str + layout_hash: str + backend: Literal["cpu", "cuda"] + + +class RecomputedWork(WireModel): + hotkey: SS58 + source_commit_envelope: Hex64 + source_delta_envelope: Hex64 + acceptance_hash: Hex64 + source_anchor_hash: Hex64 + source_anchor_proof: Hex64 + start_theta_hash: Hex64 + start_optimizer_hash: Hex64 + start_ef_hash: Hex64 + global_step0: U64 + sample_ids: Annotated[list[U64], Field(min_length=1, max_length=1_000_000)] + delta_hash: Hex64 + leaves_root: Hex64 + final_theta_hash: Hex64 + final_state_object: Hex64 + final_ef_object: Hex64 + final_optimizer_hash: Hex64 + ef_out_hash: Hex64 + + +class RepairBody(WireModel): + v: Literal["ht-rollback-replay/1"] + purpose: Literal["model-repair-no-reward"] + source_tape_hash: Hex64 + qualification_hash: Hex64 + reference_hash: Hex64 + layout_hash: Hex64 + backend: Literal["cpu", "cuda"] + arithmetic: TapeBodyV2 + recomputed: Annotated[list[RecomputedWork], Field(min_length=4, max_length=1024)] + + +def repair_message(body: RepairBody) -> bytes: + return ( + b"hypertrain/rollback/replay/1|" + + body.digest().encode() + + b"|" + + body.arithmetic.run_id.encode() + ) + + +class RepairTape(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + body: RepairBody + signer: SS58 + sig: Annotated[str, Field(pattern=r"^[0-9a-f]{128}$")] + + def to_bytes(self) -> bytes: + return canonicalize(self.model_dump(mode="json"), allow_float=False) + + @classmethod + def from_bytes(cls, raw: bytes) -> RepairTape: + tape = cls.model_validate(load_json(raw, max_bytes=1 << 20)) + if tape.to_bytes() != raw: + raise TapeError("noncanonical repair tape") + return tape + + +@dataclass(frozen=True, slots=True) +class RepairResult: + tape: RepairTape + state: OuterState + anchors: tuple[VerifiedAnchor, ...] + + +def _finite_state( + comm: Comm, + theta: dict[str, torch.Tensor], + state: OptState, + ef: dict[str, torch.Tensor], +) -> None: + """Every exact-layout rank rejects invalid decoded tensors before publication.""" + finite = all( + bool(torch.isfinite(x).all()) + for values in (theta, state.m, state.v, ef) + for x in values.values() + ) + flag = torch.tensor([int(finite)], dtype=torch.int64, device=next(iter(theta.values())).device) + if any(int(x[0]) != 1 for x in comm.all_gather(flag)): + raise TapeError("nonfinite repair theta/optimizer/EF") + + +def _compute_repair( + store: Store, + manifest: RunManifestV2, + policy: AggregationPolicyV2, + economics: bytes, + authority: RepairAuthority, + comm: Comm, + get_sample: SampleFn, + *, + w: int, + prev_state: str, + predecessor_tape_hash: str, + sources: Sequence[RepairSource], + excluded: Sequence[Exclusion], + reference_reward_units: int, +) -> tuple[RepairBody, OuterState, tuple[VerifiedAnchor, ...]]: + """Every rank executes real H-step carried training; caller owns bounded reservation.""" + cfg = TrainConfig.from_manifest_v2(manifest) + reference = manifest.training.reference_spec + econ = EconomicsPolicyV2.model_validate(load_json(economics)) + if ( + policy.digest() != manifest.network.aggregation_policy_hash + or econ.digest() != manifest.network.economics_policy_hash + or authority.reference_hash != sha256_hex(canonicalize(reference.model_dump(mode="json"))) + or authority.layout_hash != AnchorCache.layout_hash(manifest) + or cfg.inner.state_policy != "carry" + or comm.world != reference.layout.n_gpus + or (authority.backend == "cpu" and econ.ledger_mode != "test") + ): + raise TapeError("repair policy/layout/runtime not qualified") + source_tape = TapeV2.from_bytes(get_object(store, authority.source_tape_hash)) + if ( + source_tape.body.run_id != manifest.run_id() + or source_tape.body.w != w + or source_tape.signer != authority.coordinator + or not verify( + decode_hotkey(authority.coordinator), + tape_signing_message(source_tape.body.body(), manifest.run_id()), + bytes.fromhex(source_tape.sig), + ) + ): + raise TapeError("source tape authority/run/round mismatch") + ordered = sorted(sources, key=lambda s: s.work.roster.hotkey.encode()) + records = [_input(s.work, manifest, w) for s in ordered] + exclusions = sorted(excluded, key=lambda e: e.hotkey.encode()) + ids = [r.roster.hotkey for r in records] + omitted = [e.hotkey for e in exclusions] + original_ids = {r.roster.hotkey for r in source_tape.body.inputs} | { + e.hotkey for e in source_tape.body.excluded + } + if ( + len(set(ids)) != len(ids) + or len(set(omitted)) != len(omitted) + or set(ids) & set(omitted) + or set(ids) | set(omitted) != original_ids + or any(r not in source_tape.body.inputs for r in records) + or any(e not in exclusions for e in source_tape.body.excluded) + ): + raise TapeError("repair source/exclusion set differs from accepted tape") + previous = load_state(store, prev_state) + theta = {n: torch.from_numpy(x.copy()).to(authority.backend) for n, x in previous.theta.items()} + for source in ordered: + anchor = source.anchor + _finite_state(comm, theta, anchor.state, anchor.ef) + _finite_state(comm, anchor.theta, anchor.state, anchor.ef) + evidence = [] + anchors = [] + deltas = {} + for source, record in zip(ordered, records, strict=True): + work, anchor = source.work, source.anchor + _verify_original(store, manifest, econ, work, record, reference_reward_units) + for raw, kind, original in ( + (source.commit_envelope, "CommitV2", work.commit), + (source.delta_envelope, "DeltaManifestV2", work.delta_manifest), + ): + env = parse_envelope(raw) + if ( + not verify_source(raw) + or env.type != kind + or env.run_id != manifest.run_id() + or env.signer != work.roster.hotkey + or env.body != original.model_dump(mode="json") + ): + raise TapeError("original miner source signature/authority differs") + if ( + (anchor.run_id, anchor.hotkey, anchor.w, anchor.layout_hash) + != (manifest.run_id(), work.roster.hotkey, w - 1, authority.layout_hash) + or anchor.backend not in ("genesis", authority.backend) + or anchor.state.step != w * cfg.inner.H + or len(source.sample_ids) != manifest.training.batch_samples() + ): + raise TapeError("repair anchor/layout/step/assignment mismatch") + # Same byte encoding as challenge.store.assignment_hash; no dependency on service. + assignment_hash = sha256_hex( + b"ht-assignment-v1" + + bytes.fromhex(manifest.run_id()) + + struct.pack(">QQ", w, work.roster.slot) + + b"".join(struct.pack(" RepairResult: + if key.ss58 != authority.coordinator: + raise TapeError("repair signer is not pinned coordinator") + body, state, anchors = _compute_repair( + store, + manifest, + policy, + economics, + authority, + comm, + get_sample, + w=w, + prev_state=prev_state, + predecessor_tape_hash=predecessor_tape_hash, + sources=sources, + excluded=excluded, + reference_reward_units=reference_reward_units, + ) + store.put(state.to_bytes()) + tape = RepairTape(body=body, signer=key.ss58, sig=key.sign(repair_message(body)).hex()) + return RepairResult(tape, state, anchors) + + +def replay_repair_tape( + store: Store, + tape: RepairTape, + manifest: RunManifestV2, + policy: AggregationPolicyV2, + economics: bytes, + authority: RepairAuthority, + comm: Comm, + get_sample: SampleFn, + *, + w: int, + prev_state: str, + predecessor_tape_hash: str, + sources: Sequence[RepairSource], + excluded: Sequence[Exclusion], + reference_reward_units: int, +) -> RepairResult: + """Accept only independently recomputed bytes under caller-pinned source authorities.""" + if tape.signer != authority.coordinator or not verify( + decode_hotkey(authority.coordinator), repair_message(tape.body), bytes.fromhex(tape.sig) + ): + raise TapeError("repair signature/authority differs") + body, state, anchors = _compute_repair( + store, + manifest, + policy, + economics, + authority, + comm, + get_sample, + w=w, + prev_state=prev_state, + predecessor_tape_hash=predecessor_tape_hash, + sources=sources, + excluded=excluded, + reference_reward_units=reference_reward_units, + ) + if body != tape.body or get_object(store, body.arithmetic.out_state) != state.to_bytes(): + raise TapeError("independent repair replay differs") + return RepairResult(tape, state, anchors) + + +if __name__ == "__main__": + _context_worker(*sys.argv[1:]) diff --git a/src/hypertrain/aggregator/tape_v2.py b/src/hypertrain/aggregator/tape_v2.py new file mode 100644 index 0000000..33a8d16 --- /dev/null +++ b/src/hypertrain/aggregator/tape_v2.py @@ -0,0 +1,383 @@ +"""Signed versioned flat tapes replay original objects and trusted input facts.""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from typing import Annotated, Literal + +import numpy as np +from pydantic import BaseModel, ConfigDict, Field, JsonValue + +from hypertrain.aggregator.core import ( + OuterParams, + OuterState, + TapeError, + get_object, + load_delta, + load_state, +) +from hypertrain.aggregator.weighted_v2 import Candidate, aggregate_weighted, allocate_weights +from hypertrain.challenge.trust_v2 import ( + FundedStatus, + ReplayEvidence, + SettlementStatus, + audit_floor, +) +from hypertrain.data.store import Store +from hypertrain.protocol.envelope_v2 import load_json, tape_signing_message +from hypertrain.protocol.hashing import sha256_hex +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.keys import Keypair, decode_hotkey, verify +from hypertrain.protocol.messages import SS58, Hex64 +from hypertrain.protocol.messages_v2 import ( + U64, + AggregationPolicyV2, + CommitV2, + DeltaManifestV2, + EconomicsPolicyV2, + PolicyHashes, + RosterEntryV2, + RunManifestV2, + WeightAllocationV2, + WireModel, +) + + +@dataclass(frozen=True, slots=True) +class VerifiedInput: + """Only authorized L0 intake may assemble these from L1/L2/L5 records.""" + + roster: RosterEntryV2 + commit: CommitV2 + delta_manifest: DeltaManifestV2 + replay: ReplayEvidence + funding: FundedStatus + settlement: SettlementStatus + assignment_hash: str + clean_finalizations: int + + +class TapeInput(WireModel): + roster: RosterEntryV2 + commit_hash: Hex64 + delta_manifest_hash: Hex64 + assignment_hash: Hex64 + anchor_hash: Hex64 + verdict_hash: Hex64 + lock_receipt_hash: Hex64 + settlement_hash: Hex64 + + +class Exclusion(WireModel): + hotkey: SS58 + reason: Literal["SHADOW", "UNVERIFIED", "DISPUTE", "UNFUNDED", "INFRASTRUCTURE", "FRAUD"] + evidence_hash: Hex64 + + +class TapeBodyV2(WireModel): + v: Literal["ht-tape-v2"] + arithmetic: Literal["flat-cclip-cap-v1"] + run_id: Hex64 + w: U64 + policy_hash: Hex64 + policy_hashes: PolicyHashes + predecessor_tape_hash: Hex64 + prev_state: Hex64 + prev_hashes: dict[str, Hex64] + inputs: Annotated[list[TapeInput], Field(min_length=4, max_length=1024)] + input_root: Hex64 + excluded: Annotated[list[Exclusion], Field(max_length=1024)] + allocation: WeightAllocationV2 + preclipped: list[SS58] + copy_suspicion: list[dict[str, str]] + out_state: Hex64 + out_hashes: dict[str, Hex64] + + +class TapeV2(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + body: TapeBodyV2 + signer: SS58 + sig: Annotated[str, Field(pattern=r"^[0-9a-f]{128}$")] + + def to_bytes(self) -> bytes: + return canonicalize(self.body_json(), allow_float=False) + + def body_json(self) -> dict[str, JsonValue]: + return {"body": self.body.body(), "signer": self.signer, "sig": self.sig} + + @classmethod + def from_bytes(cls, raw: bytes) -> TapeV2: + data = load_json(raw, max_bytes=1 << 20) + tape = cls.model_validate(data) + if canonicalize(data, allow_float=False) != tape.to_bytes(): + raise TapeError("noncanonical tape body") + return tape + + +def _input(work: VerifiedInput, manifest: RunManifestV2, w: int) -> TapeInput: + c, d, r, s = work.commit, work.delta_manifest, work.replay, work.settlement + if ( + (c.w, d.w, r.w, s.w) != (w, w, w, w) + or (c.hotkey, d.hotkey, r.hotkey, s.hotkey) != (work.roster.hotkey,) * 4 + or (r.run_id, s.run_id) != (manifest.run_id(),) * 2 + ): + raise TapeError("input identity/run/round mismatch") + c.validate_assignment(manifest, manifest.training.batch_samples()) + if (d.delta_hash, d.size) != (c.delta_hash, c.delta_bytes): + raise TapeError("delta manifest differs from commit") + if ( + r.result != "MATCH" + or r.audit_mode != "anchored-full" + or ( + r.assignment_hash, + r.leaves_root, + r.delta_hash, + r.final_theta_hash, + r.ef_in_hash, + r.ef_out_hash, + ) + != ( + work.assignment_hash, + c.leaves_root, + c.delta_hash, + c.final_theta_hash, + c.ef_in_hash, + c.ef_out_hash, + ) + ): + raise TapeError("own-assignment full replay not VERIFIED") + if s.unresolved_dispute or s.excluded: + raise TapeError("input unsettled/excluded") + return TapeInput( + roster=work.roster, + commit_hash=sha256_hex(canonicalize(c.model_dump(mode="json"))), + delta_manifest_hash=sha256_hex(canonicalize(d.model_dump(mode="json"))), + assignment_hash=work.assignment_hash, + anchor_hash=r.anchor_hash, + verdict_hash=r.verdict_hash, + lock_receipt_hash=work.funding.lock_receipt_hash, + settlement_hash=s.status_hash, + ) + + +def compute_body( + store: Store, + manifest: RunManifestV2, + policy: AggregationPolicyV2, + economics: bytes, + *, + w: int, + prev_state: str, + predecessor_tape_hash: str, + inputs: Sequence[VerifiedInput], + excluded: Sequence[Exclusion] = (), + reference_reward_units: int, +) -> tuple[TapeBodyV2, OuterState]: + """Fetch every original object, allocate exact weights, emit reproducible arithmetic.""" + from hypertrain.protocol.messages_v2 import EconomicsPolicyV2 + + if policy.digest() != manifest.network.aggregation_policy_hash: + raise TapeError("policy not pinned by run") + econ = EconomicsPolicyV2.model_validate(load_json(economics)) + if econ.digest() != manifest.network.economics_policy_hash: + raise TapeError("economics not pinned by run") + ordered = sorted(inputs, key=lambda x: x.roster.hotkey.encode("utf-8")) + records = [_input(x, manifest, w) for x in ordered] + if len({x.roster.hotkey for x in ordered}) != len(ordered): + raise TapeError("duplicate input") + exclusions = sorted(excluded, key=lambda x: x.hotkey.encode("utf-8")) + if len({x.hotkey for x in exclusions}) != len(exclusions) or ( + {x.hotkey for x in exclusions} & {x.roster.hotkey for x in ordered} + ): + raise TapeError("conflicting exclusion set") + for x, record in zip(ordered, records, strict=True): + _verify_original(store, manifest, econ, x, record, reference_reward_units) + previous = load_state(store, prev_state) + return _aggregate_body( + manifest, + policy, + previous, + w=w, + prev_state=prev_state, + predecessor_tape_hash=predecessor_tape_hash, + records=records, + exclusions=exclusions, + deltas={ + x.roster.hotkey: load_delta(store, x.commit.delta_hash, previous.theta) for x in ordered + }, + ) + + +def _verify_original( + store: Store, + manifest: RunManifestV2, + econ: EconomicsPolicyV2, + work: VerifiedInput, + record: TapeInput, + reference_reward_units: int, +) -> None: + floor = audit_floor( + econ, + manifest.training.verify, + work.roster, + work.funding, + run_id=manifest.run_id(), + clean_finalizations=work.clean_finalizations, + reference_reward_units=reference_reward_units, + ) + if not floor.influence_allowed: + raise TapeError("funded influence gate failed") + for key, original_body in ( + (record.commit_hash, work.commit.model_dump(mode="json")), + (record.delta_manifest_hash, work.delta_manifest.model_dump(mode="json")), + ): + if get_object(store, key) != canonicalize(original_body, allow_float=False): + raise TapeError("original input body differs") + payload = get_object(store, work.commit.delta_hash) + if len(payload) != work.delta_manifest.size or any( + sha256_hex(payload[c.off : c.off + c.len]) != c.sha256 for c in work.delta_manifest.chunks + ): + raise TapeError("original delta/chunks differ") + + +def _aggregate_body( + manifest: RunManifestV2, + policy: AggregationPolicyV2, + previous: OuterState, + *, + w: int, + prev_state: str, + predecessor_tape_hash: str, + records: list[TapeInput], + exclusions: list[Exclusion], + deltas: dict[str, dict[str, np.ndarray]], +) -> tuple[TapeBodyV2, OuterState]: + """Shared arithmetic; provenance is checked by each version's input boundary.""" + allocation = allocate_weights( + [Candidate(r.roster, r.commit_hash, r.delta_manifest_hash) for r in records], policy + ) + o = manifest.training.outer + params = OuterParams( + o.opt, o.lr, o.momentum, policy.preclip_norm, policy.cclip_tau, policy.cclip_iters + ) + result = aggregate_weighted( + previous, + deltas, + allocation, + policy, + params, + ) + hashes = PolicyHashes.model_validate( + { + name: getattr(manifest.network, name) + for name in ( + "admission_policy_hash", + "economics_policy_hash", + "aggregation_policy_hash", + "dispute_policy_hash", + "audit_policy_hash", + ) + } + ) + body = TapeBodyV2( + v="ht-tape-v2", + arithmetic=policy.arithmetic, + run_id=manifest.run_id(), + w=w, + policy_hash=policy.digest(), + policy_hashes=hashes, + predecessor_tape_hash=predecessor_tape_hash, + prev_state=prev_state, + prev_hashes=previous.hashes(), + inputs=records, + input_root=sha256_hex(canonicalize([r.body() for r in records])), + excluded=exclusions, + allocation=allocation, + preclipped=list(result.preclipped), + copy_suspicion=list(result.copy_suspicion), + out_state=sha256_hex(result.state.to_bytes()), + out_hashes=result.state.hashes(), + ) + return body, result.state + + +def make_tape( + store: Store, + manifest: RunManifestV2, + policy: AggregationPolicyV2, + economics: bytes, + key: Keypair, + *, + w: int, + prev_state: str, + predecessor_tape_hash: str, + inputs: Sequence[VerifiedInput], + excluded: Sequence[Exclusion] = (), + reference_reward_units: int, +) -> TapeV2: + body, state = compute_body( + store, + manifest, + policy, + economics, + w=w, + prev_state=prev_state, + predecessor_tape_hash=predecessor_tape_hash, + inputs=inputs, + excluded=excluded, + reference_reward_units=reference_reward_units, + ) + store.put(state.to_bytes()) + return TapeV2( + body=body, + signer=key.ss58, + sig=key.sign(tape_signing_message(body.body(), manifest.run_id())).hex(), + ) + + +def replay_tape( + store: Store, + tape: TapeV2, + manifest: RunManifestV2, + policy: AggregationPolicyV2, + economics: bytes, + *, + signer: str, + w: int, + prev_state: str, + predecessor_tape_hash: str, + inputs: Sequence[VerifiedInput], + excluded: Sequence[Exclusion] = (), + reference_reward_units: int, +) -> OuterState: + """Caller pins expected predecessor/input set; a signer cannot omit accepted work.""" + if ( + tape.signer != signer + or tape.body.run_id != manifest.run_id() + or not verify( + decode_hotkey(signer), + tape_signing_message(tape.body.body(), manifest.run_id()), + bytes.fromhex(tape.sig), + ) + ): + raise TapeError("tape signature/run/signer mismatch") + body, state = compute_body( + store, + manifest, + policy, + economics, + w=w, + prev_state=prev_state, + predecessor_tape_hash=predecessor_tape_hash, + inputs=inputs, + excluded=excluded, + reference_reward_units=reference_reward_units, + ) + if body.body() != tape.body.body(): + raise TapeError("tape replay differs from original inputs/weights/policy/predecessor") + if get_object(store, body.out_state) != state.to_bytes(): + raise TapeError("output object differs") + return state diff --git a/src/hypertrain/aggregator/weighted_v2.py b/src/hypertrain/aggregator/weighted_v2.py new file mode 100644 index 0000000..2299ddc --- /dev/null +++ b/src/hypertrain/aggregator/weighted_v2.py @@ -0,0 +1,226 @@ +"""Exact max-min allocation with intersecting probation and known-owner bounds.""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from typing import Final + +import numpy as np + +from hypertrain.aggregator.core import ( + OuterParams, + OuterState, + Params, + centered_clip, + copy_flags, + names, + outer_step, + preclip, +) +from hypertrain.protocol.messages import f32val +from hypertrain.protocol.messages_v2 import ( + WEIGHT_QUANTUM, + AggregationPolicyV2, + RosterEntryV2, + WeightAllocationV2, + WeightedEntryV2, +) + +Q: Final = WEIGHT_QUANTUM + + +class InsufficientEligibleWeight(ValueError): + """INSUFFICIENT_ELIGIBLE_WEIGHT: caps cannot carry one complete update.""" + + +@dataclass(frozen=True, slots=True) +class Candidate: + roster: RosterEntryV2 + commit_hash: str + delta_manifest_hash: str + + +def _completion_capacity( + active: Sequence[int], + probation: Sequence[int], + owner: Sequence[int], + cohort_cap: int, +) -> int: + """Two-cohort min-cut: all-owner capacity versus active capacity plus probation cap.""" + return min( + sum(min(r, a + p) for a, p, r in zip(active, probation, owner, strict=True)), + sum(min(r, a) for a, r in zip(active, owner, strict=True)) + cohort_cap, + ) + + +def allocate_weights( + candidates: Sequence[Candidate], policy: AggregationPolicyV2 +) -> WeightAllocationV2: + """Lexicographic max-min integer filling, UTF-8 ties, no renormalization. + + Closed-form completion probes are O(n); incremental tie checks are O(1). + At most n nonfinal filling iterations, O(n^2 log Q) work, O(n) scratch. + """ + if len(candidates) > 1024: + raise InsufficientEligibleWeight("oversized candidate set") + ordered = sorted(candidates, key=lambda c: c.roster.hotkey.encode("utf-8")) + ids = [c.roster.hotkey for c in ordered] + if len(ids) != len(set(ids)): + raise InsufficientEligibleWeight("duplicate or oversized candidate set") + if any(c.roster.state not in ("ACTIVE", "PROBATION") for c in ordered): + raise InsufficientEligibleWeight("shadow/suspended identity supplied for live allocation") + groups = sorted({c.roster.coldkey_group for c in ordered}) + group_index = {g: i for i, g in enumerate(groups)} + owner_ids = [group_index[c.roster.coldkey_group] for c in ordered] + probation_ids = [c.roster.state == "PROBATION" for c in ordered] + caps = [min(Q, policy.owner_group_caps.get(g, Q)) for g in groups] + ceilings = [min(policy.miner_cap_units, c.roster.eligible_weight) for c in ordered] + weights = [0] * len(ordered) + free = set(range(len(ordered))) + + def completion(lower: list[int]) -> tuple[list[int], list[int], list[int], int, int] | None: + residual, active, probation = caps.copy(), [0] * len(groups), [0] * len(groups) + total = 0 + cohort: int = policy.probation_cap_units + for i, w in enumerate(lower): + if w > ceilings[i]: + return None + g = owner_ids[i] + residual[g] -= w + total += w + if probation_ids[i]: + cohort -= w + if i in free: + target = probation if probation_ids[i] else active + target[g] += ceilings[i] - w + if total > Q or cohort < 0 or min(residual, default=0) < 0: + return None + if total + _completion_capacity(active, probation, residual, cohort) < Q: + return None + return residual, active, probation, cohort, total + + if completion(weights) is None: + raise InsufficientEligibleWeight("INSUFFICIENT_ELIGIBLE_WEIGHT") + while sum(weights) < Q: + lo, hi = 0, (Q - sum(weights)) // len(free) + # Local upper bounds are exact necessary constraints, not relaxed feasibility. + counts = [0] * len(groups) + residual = caps.copy() + cohort, probation_count = int(policy.probation_cap_units), 0 + for i, w in enumerate(weights): + residual[owner_ids[i]] -= w + if probation_ids[i]: + cohort -= w + if i in free: + hi = min(hi, ceilings[i] - w) + counts[owner_ids[i]] += 1 + probation_count += probation_ids[i] + hi = min(hi, *(r // count for r, count in zip(residual, counts, strict=True) if count)) + if probation_count: + hi = min(hi, cohort // probation_count) + while lo < hi: + mid = (lo + hi + 1) // 2 + trial = [w + mid if i in free else w for i, w in enumerate(weights)] + if completion(trial) is not None: + lo = mid + else: + hi = mid - 1 + weights = [w + lo if i in free else w for i, w in enumerate(weights)] + state = completion(weights) + if state is None: + raise InsufficientEligibleWeight("INSUFFICIENT_ELIGIBLE_WEIGHT") + residual, active, probation, cohort, total = state + all_capacity = sum( + min(r, a + p) for r, a, p in zip(residual, active, probation, strict=True) + ) + active_capacity = sum(min(r, a) for r, a in zip(residual, active, strict=True)) + for i in sorted(free): + if total == Q: + break + g = owner_ids[i] + r, a, p = residual[g], active[g], probation[g] + a1, p1 = a - (not probation_ids[i]), p - probation_ids[i] + cohort1 = cohort - probation_ids[i] + all1 = all_capacity - min(r, a + p) + min(r - 1, a1 + p1) + active1 = active_capacity - min(r, a) + min(r - 1, a1) + if ( + weights[i] < ceilings[i] + and r > 0 + and cohort1 >= 0 + and total + 1 + min(all1, active1 + cohort1) >= Q + ): + weights[i] += 1 + total += 1 + residual[g], active[g], probation[g] = r - 1, a1, p1 + cohort, all_capacity, active_capacity = cohort1, all1, active1 + else: + free.remove(i) + remaining = ceilings[i] - weights[i] + if probation_ids[i]: + probation[g] -= remaining + else: + active[g] -= remaining + all_capacity += min(r, active[g] + probation[g]) - min(r, a + p) + active_capacity += min(r, active[g]) - min(r, a) + if not free and sum(weights) != Q: + raise InsufficientEligibleWeight("INSUFFICIENT_ELIGIBLE_WEIGHT") + return WeightAllocationV2( + policy_hash=policy.digest(), + entries=[ + WeightedEntryV2( + hotkey=c.roster.hotkey, + admission_id=c.roster.admission_id, + probation=c.roster.state == "PROBATION", + weight_units=w, + commit_hash=c.commit_hash, + delta_manifest_hash=c.delta_manifest_hash, + ) + for c, w in zip(ordered, weights, strict=True) + ], + ) + + +@dataclass(frozen=True, slots=True) +class WeightedResult: + state: OuterState + preclipped: tuple[str, ...] + copy_suspicion: tuple[dict[str, str], ...] + + +def aggregate_weighted( + previous: OuterState, + deltas: dict[str, Params], + allocation: WeightAllocationV2, + policy: AggregationPolicyV2, + params: OuterParams, +) -> WeightedResult: + """Reuse qualified preclip/CClip/outer update with exact final dyadic weights.""" + if allocation.policy_hash != policy.digest() or ( + params.preclip_norm, + params.cclip_tau, + params.cclip_iters, + ) != (policy.preclip_norm, policy.cclip_tau, policy.cclip_iters): + raise InsufficientEligibleWeight("aggregation policy/parameters differ") + if set(deltas) != {e.hotkey for e in allocation.entries}: + raise InsufficientEligibleWeight("input set differs from allocation") + clipped, hits = [], [] + for entry in allocation.entries: + delta = deltas[entry.hotkey] + if names(delta) != names(previous.theta) or any( + a.dtype != np.float32 + or a.shape != previous.theta[n].shape + or not bool(np.isfinite(a).all()) + for n, a in delta.items() + ): + raise InsufficientEligibleWeight("malformed delta") + value, hit = preclip(delta, f32val(policy.preclip_norm)) + clipped.append(value) + if hit: + hits.append(entry.hotkey) + weights = [np.float32(e.weight_units / Q) for e in allocation.entries] + g = centered_clip( + clipped, weights, previous.center, f32val(policy.cclip_tau), policy.cclip_iters + ) + flags = copy_flags([(e.hotkey, d) for e, d in zip(allocation.entries, clipped, strict=True)]) + return WeightedResult(outer_step(previous, g, params), tuple(hits), tuple(flags)) diff --git a/src/hypertrain/auditor/bisect.py b/src/hypertrain/auditor/bisect.py index 25fb839..892d202 100644 --- a/src/hypertrain/auditor/bisect.py +++ b/src/hypertrain/auditor/bisect.py @@ -66,7 +66,7 @@ def points(a: int, b: int, n: int) -> list[int]: def tensor_digest(x: Tensor) -> bytes: - y = x.detach().contiguous() + y = x.detach().cpu().contiguous() h = hashlib.sha256(f"{y.dtype}|{tuple(y.shape)}|".encode()) h.update(y.reshape(-1).view(torch.uint8).numpy().tobytes()) return h.digest() diff --git a/src/hypertrain/auditor/honeypot.py b/src/hypertrain/auditor/honeypot.py index b6c64d5..079d330 100644 --- a/src/hypertrain/auditor/honeypot.py +++ b/src/hypertrain/auditor/honeypot.py @@ -10,7 +10,7 @@ import hashlib import hmac -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from typing import Literal @@ -19,8 +19,82 @@ import hypertrain.trainer # noqa: F401 from hypertrain.protocol.hashing import MerkleTree from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.keys import Keypair +from hypertrain.protocol.messages_v2 import RunManifestV2 from hypertrain.trainer.config import TrainConfig +from hypertrain.trainer.island import CancelHook, Comm, TraceHook, train_island from hypertrain.trainer.loop import Assignment, Params, RoundResult, SampleFn, train_round +from hypertrain.trainer.optim import OptState + + +def honeypot_island( + pot: Honeypot, + wrapper: RunManifestV2, + comm: Comm, + theta: Params, + a: Assignment, + get: SampleFn, + *, + carry: OptState | None = None, + ef: Params | None = None, + v0: Params | None = None, + hook: TraceHook | None = None, + cancel: CancelHook | None = None, +) -> RoundResult: + """Ordinary island path, private mode only; no pot metadata in job/trace artifacts.""" + cfg = TrainConfig.from_manifest_v2(wrapper) + lay = wrapper.training.reference_spec.layout + start = {n: x * 1.01 for n, x in theta.items()} if pot.mode == "fabricate" else theta + + def fault(t: int, local: Params) -> None: + with torch.no_grad(): + match pot.mode: + case "last_step": + if t == cfg.inner.H: + local[sorted(local)[0]].view(-1)[0] += 1e-3 + case "noise": + if t == max(1, cfg.inner.H // 2): + for x in local.values(): + x.add_(1e-6) + case "honest" | "fabricate": + return + + return train_island( + cfg, + lay, + comm, + start, + a, + get, + carry=carry, + ef_in=ef, + v0=v0, + hook=hook, + cancel=cancel, + after_step=fault, + ) + + +def fresh_honeypots( + previous: Sequence[Honeypot], + modes: Sequence[Mode], + retain_key: Callable[[Keypair], None] | None = None, +) -> list[Honeypot]: + """Never reuse revealed hotkeys. Caller retains private keys outside public job metadata.""" + import os + + from hypertrain.protocol.keys import Keypair + + old = {p.hotkey for p in previous} + keys = [Keypair(os.urandom(32)) for _ in modes] + result = [Honeypot(key.ss58, mode) for key, mode in zip(keys, modes, strict=True)] + if retain_key is not None: + for key in keys: + retain_key(key) + if len({p.hotkey for p in result}) != len(result) or any(p.hotkey in old for p in result): + raise RuntimeError("honeypot identity collision") + return result + Mode = Literal["honest", "fabricate", "last_step", "noise"] FAULT_RESULTS = {"MISMATCH", "WITHHELD", "BAD_PROOF", "ASSIGNMENT_VIOLATION"} diff --git a/src/hypertrain/auditor/island_bisect.py b/src/hypertrain/auditor/island_bisect.py new file mode 100644 index 0000000..be8cb46 --- /dev/null +++ b/src/hypertrain/auditor/island_bisect.py @@ -0,0 +1,318 @@ +"""Layout-preserving disputes over hooks emitted by the real island trainer. + +STEP indexes committed windows, not fabricated intermediate full states. LAYER +indexes contiguous context spans; OP indexes rank-ordered real hook boundaries. +Every window starts with its full-state root and ends with its next state root. +SIZE_OK: all-level point indexing and its referee share one trace coordinate +system; the exact seven-path scope precludes a second indexing module. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Literal, assert_never + +import torch +from pydantic import Field, StrictInt, TypeAdapter +from torch import Tensor + +import hypertrain.trainer # noqa: F401 +from hypertrain.auditor.bisect import Level, RefereeError, points +from hypertrain.auditor.replay import pack_state, tensor_root, unpack_state +from hypertrain.miner.island_launch import confined, validate_artifacts +from hypertrain.protocol.hashing import sha256_hex +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.messages import SS58, Hex64, Layout +from hypertrain.protocol.messages_v2 import IslandJobV1, WireModel +from hypertrain.trainer.config import TrainConfig +from hypertrain.trainer.island import CancelHook, Comm, TraceContext, TraceHook, train_island +from hypertrain.trainer.loop import Assignment, Params, RoundResult, SampleFn +from hypertrain.trainer.optim import OptState + + +class TraceEntry(WireModel): + rank: StrictInt = Field(ge=0) + step: StrictInt = Field(ge=0) + microbatch: StrictInt = Field(ge=-1) + layer: StrictInt = Field(ge=-1) + op: str = Field(min_length=1, max_length=256) + shape: list[StrictInt] + dtype: str + sha256: Hex64 + + +class IslandCapture: + """Mutable rank-local collector; identity return preserves actual training.""" + + def __init__(self, fault: TraceHook | None = None) -> None: + self.entries: list[TraceEntry] = [] + self.states: dict[int, bytes] = {} + self.fault = fault + + def hook(self, ctx: TraceContext, tensor: Tensor) -> Tensor: + out = tensor if self.fault is None else self.fault(ctx, tensor) + raw = out.detach().cpu().contiguous().reshape(-1).view(torch.uint8) + self.entries.append( + TraceEntry( + rank=ctx.rank, + step=ctx.step, + microbatch=ctx.microbatch, + layer=ctx.layer, + op=ctx.op, + shape=list(out.shape), + dtype=str(out.dtype), + sha256=sha256_hex(raw.numpy().tobytes()), + ) + ) + return out + + def checkpoint(self, t: int, theta: Params, state: OptState) -> None: + self.states[t] = pack_state(theta, state) + + +@dataclass(frozen=True, slots=True) +class IslandExecution: + cfg: TrainConfig + layout: Layout + assignment: Assignment + theta: Params + get_sample: SampleFn + carry: OptState | None = None + ef_in: Params | None = None + v0: Params | None = None + + def trace( + self, + comm: Comm, + *, + fault: TraceHook | None = None, + cancel: CancelHook | None = None, + ) -> tuple[RoundResult, IslandCapture]: + capture = IslandCapture(fault) + result = train_island( + self.cfg, + self.layout, + comm, + self.theta, + self.assignment, + self.get_sample, + carry=self.carry, + ef_in=self.ef_in, + v0=self.v0, + hook=capture.hook, + checkpoint=capture.checkpoint, + cancel=cancel, + ) + return result, capture + + +class IslandParty: + """Read-only, rank-major transcript from authenticated execution artifacts.""" + + def __init__( + self, + hotkey: str, + captures: Sequence[IslandCapture], + *, + J: int, + ) -> None: + self.hotkey, self.J = hotkey, J + if not captures or J < 1: + raise RefereeError("missing island captures or invalid checkpoint stride") + self.states = dict(captures[0].states) + if not self.states or any(c.states != self.states for c in captures): + raise RefereeError("rank checkpoint disagreement") + self.entries = tuple( + entry + for rank, capture in enumerate(captures) + for entry in capture.entries + if self._rank(entry, rank) + ) + self.H = max(self.states) + if set(self.states) != set(range(0, self.H + 1, J)): + raise RefereeError("incomplete committed window checkpoints") + + @staticmethod + def _rank(entry: TraceEntry, rank: int) -> bool: + if entry.rank != rank: + raise RefereeError("trace rank binding mismatch") + return True + + @classmethod + def published(cls, hotkey: str, job: IslandJobV1, directory: Path) -> IslandParty: + validate_artifacts(job, directory) + captures: list[IslandCapture] = [] + for rank in range(job.manifest.training.reference_spec.layout.n_gpus): + capture = IslandCapture() + raw = confined(directory, f"rank-{rank}/trace.json").read_bytes() + if len(raw) > 64 * 1024 * 1024: + raise RefereeError("local trace exceeds artifact byte budget") + # Local hooks contain -1 collective contexts, unlike unsigned wire envelopes. + capture.entries = TypeAdapter(list[TraceEntry]).validate_json(raw) + for t in range(0, job.manifest.training.inner.H + 1, job.manifest.training.inner.J): + capture.states[t] = confined( + directory, + f"rank-{rank}/checkpoints/{t}.safetensors", + ).read_bytes() + captures.append(capture) + return cls(hotkey, captures, J=job.manifest.training.inner.J) + + def state_blob(self, leaf: int) -> bytes: + try: + return self.states[leaf * self.J] + except KeyError as error: + raise RefereeError("unknown checkpoint") from error + + def state_root(self, leaf: int) -> str: + theta, state = unpack_state(self.state_blob(leaf)) + if state is None: + raise RefereeError("checkpoint omits optimizer") + return tensor_root(theta, state) + + def window(self, leaf: int) -> tuple[tuple[TraceEntry, ...], tuple[int, ...], list[str]]: + if not 1 <= leaf <= self.H // self.J: + raise RefereeError("window outside committed trajectory") + entries = tuple(e for e in self.entries if (leaf - 1) * self.J < e.step <= leaf * self.J) + if not entries: + raise RefereeError("missing actual island hooks") + chain = [self.state_root(leaf - 1)] + ends = [] + for i, entry in enumerate(entries): + chain.append(sha256_hex(bytes.fromhex(chain[-1]) + canonicalize(entry.body()))) + if i + 1 == len(entries) or (entry.rank, entry.step, entry.layer, entry.microbatch) != ( + entries[i + 1].rank, + entries[i + 1].step, + entries[i + 1].layer, + entries[i + 1].microbatch, + ): + ends.append(i + 1) + # The full-state boundary is a real authenticated checkpoint, not a hook surrogate. + chain.append(self.state_root(leaf)) + ends.append(len(chain) - 1) + return entries, tuple(ends), chain + + def span(self, level: Level, ctx: tuple[int, ...]) -> int: + match level: + case "step": + if ctx: + raise RefereeError("STEP context must be empty") + return self.H // self.J + case "layer": + if len(ctx) != 1: + raise RefereeError("LAYER context requires window") + return len(self.window(ctx[0])[1]) + case "op": + if len(ctx) != 2: + raise RefereeError("OP context requires window/span") + _, ends, _ = self.window(ctx[0]) + if not 0 <= ctx[1] < len(ends): + raise RefereeError("unknown layer span") + return ends[ctx[1]] - (ends[ctx[1] - 1] if ctx[1] else 0) + case other: + assert_never(other) + + def hashes(self, level: Level, ctx: tuple[int, ...], at: Sequence[int]) -> list[str]: + span = self.span(level, ctx) + if any(not 0 <= i <= span for i in at): + raise RefereeError("point outside dispute span") + match level: + case "step": + return [self.state_root(i) for i in at] + case "layer": + _, ends, chain = self.window(ctx[0]) + return [chain[(0, *ends)[i]] for i in at] + case "op": + _, ends, chain = self.window(ctx[0]) + start = ends[ctx[1] - 1] if ctx[1] else 0 + return [chain[start + i] for i in at] + case other: + assert_never(other) + + def op_name(self, ctx: tuple[int, ...], end: int) -> str: + entries, ends, _ = self.window(ctx[0]) + start = ends[ctx[1] - 1] if ctx[1] else 0 + index = start + end - 1 + if index == len(entries): + return "full_state.checkpoint" + e = entries[index] + return f"rank={e.rank};step={e.step};microbatch={e.microbatch};layer={e.layer};op={e.op}" + + +class RefereeEvidence(WireModel): + dispute_id: Hex64 + transcript_hash: Hex64 + reason: Literal["MATCH", "FRAUD", "INFRASTRUCTURE"] + loser: SS58 | None + predecessor_hash: Hex64 + inputs_hash: Hex64 + output_hash: Hex64 + op_spec: str + + +def adjudicate( + record_id: tuple[str, str], + parties: tuple[IslandParty, IslandParty], + referee: IslandParty, +) -> RefereeEvidence: + """Referee must originate in independently anchored same-layout execution. + + Comparing only a miner-supplied checkpoint is insufficient. All agreed starts + below must also match the independent referee's anchored trajectory. + """ + if referee.hotkey in {p.hotkey for p in parties}: + raise RefereeError("referee must be independent") + level: Level = "step" + ctx: tuple[int, ...] = () + a, b = 0, referee.span(level, ctx) + predecessor = referee.state_root(0) + while True: + at = points(a, b, 2) if b - a > 1 else [a, b] + left, right, truth = [p.hashes(level, ctx, at) for p in (*parties, referee)] + if left[0] != right[0] or left[0] != truth[0]: + raise RefereeError("independent replay cannot reproduce agreed input") + different = next((i for i in range(1, len(at)) if left[i] != right[i]), None) + if different is None: + if left != truth: + raise RefereeError("both claimants disagree with independent replay") + return RefereeEvidence( + dispute_id=record_id[0], + transcript_hash=record_id[1], + reason="MATCH", + loser=None, + predecessor_hash=predecessor, + inputs_hash=truth[0], + output_hash=truth[-1], + op_spec="anchored-full-match", + ) + a, b = at[different - 1], at[different] + if left[different - 1] != truth[different - 1]: + raise RefereeError("independent replay cannot reproduce agreed op input") + if b - a > 1: + continue + match level: + case "step": + predecessor = referee.state_root(a) + ctx, level = (b,), "layer" + case "layer": + ctx, level = (ctx[0], a), "op" + case "op": + claims = [p.hashes(level, ctx, [b])[0] for p in parties] + output = referee.hashes(level, ctx, [b])[0] + wrong = [p.hotkey for p, h in zip(parties, claims, strict=True) if h != output] + if len(wrong) != 1: + raise RefereeError("independent output matches neither claimant") + return RefereeEvidence( + dispute_id=record_id[0], + transcript_hash=record_id[1], + reason="FRAUD", + loser=wrong[0], + predecessor_hash=predecessor, + inputs_hash=referee.hashes(level, ctx, [a])[0], + output_hash=output, + op_spec=referee.op_name(ctx, b), + ) + case other: + assert_never(other) + a, b = 0, referee.span(level, ctx) diff --git a/src/hypertrain/auditor/replay.py b/src/hypertrain/auditor/replay.py index 25ba9ce..45dca8f 100644 --- a/src/hypertrain/auditor/replay.py +++ b/src/hypertrain/auditor/replay.py @@ -21,8 +21,9 @@ import hashlib import struct -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, replace +from pathlib import Path from typing import Literal import torch @@ -42,11 +43,21 @@ StateServe, f32val, ) +from hypertrain.protocol.messages_v2 import AuditJobV2, CommitV2, RunManifestV2, StartStateV2 from hypertrain.trainer.compress import compress, payload_hash, state_hash from hypertrain.trainer.config import TrainConfig +from hypertrain.trainer.island import ( + CancelHook, + Comm, + IslandAssignment, + TraceContext, + TraceHook, + train_island, +) from hypertrain.trainer.loop import ( Assignment, Params, + RoundResult, SampleFn, StepHook, batch_hash, @@ -54,7 +65,8 @@ stage_states, train_round, ) -from hypertrain.trainer.optim import OptState +from hypertrain.trainer.model import init_params +from hypertrain.trainer.optim import OptState, init_state from hypertrain.trainer.rng import rng_ctr Result = Literal["MATCH", "MISMATCH", "WITHHELD", "BAD_PROOF", "ASSIGNMENT_VIOLATION"] @@ -101,11 +113,11 @@ def challenge_hash(ch: AuditChallenge) -> str: def pack_state(theta: Params, st: OptState | None = None) -> bytes: - out = {f"theta/{n}": x.contiguous() for n, x in theta.items()} + out = {f"theta/{n}": x.detach().cpu().contiguous() for n, x in theta.items()} if st is not None: - out |= {f"m/{n}": x.contiguous() for n, x in st.m.items()} - out |= {f"v/{n}": x.contiguous() for n, x in st.v.items()} - out["step"] = torch.tensor(st.step, dtype=torch.int64) + out |= {f"m/{n}": x.detach().cpu().contiguous() for n, x in st.m.items()} + out |= {f"v/{n}": x.detach().cpu().contiguous() for n, x in st.v.items()} + out["step"] = torch.tensor(st.step, dtype=torch.int64, device="cpu") return bytes(st_save(out)) @@ -135,6 +147,464 @@ def tensor_root(theta: Params, st: OptState) -> str: ) +def optimizer_hash(st: OptState) -> str: + return sha256_hex( + bytes.fromhex(state_hash(st.m) + state_hash(st.v)) + struct.pack(" None: + self.entries: dict[tuple[str, str, int, str], VerifiedAnchor] = {} + + @staticmethod + def layout_hash(wrapper: RunManifestV2) -> str: + from hypertrain.protocol.jcs import canonicalize + + return sha256_hex(canonicalize(wrapper.training.reference_spec.layout.model_dump())) + + def genesis(self, wrapper: RunManifestV2, hotkey: str, theta: Params) -> VerifiedAnchor: + cfg = TrainConfig.from_manifest_v2(wrapper) + if cfg.model.od is not None and cfg.model.od.warm_start: + raise AuditInputError("warm start requires independently verified checkpoint lineage") + if state_hash(theta) != state_hash(init_params(cfg.model)): + raise AuditInputError("genesis theta differs from deterministic initialization") + reset = replace(cfg.inner, state_policy="reset") + st = init_state(reset, theta) + ef = {n: torch.zeros_like(x) for n, x in theta.items()} + proof = sha256_hex(bytes.fromhex(wrapper.run_id() + optimizer_hash(st) + state_hash(ef))) + entry = VerifiedAnchor( + wrapper.run_id(), + hotkey, + -1, + self.layout_hash(wrapper), + st, + ef, + proof, + proof, + {n: x.clone() for n, x in theta.items()}, + "genesis", + ) + self.entries[(entry.run_id, hotkey, -1, entry.layout_hash)] = entry + return entry + + def warm_start( + self, + wrapper: RunManifestV2, + hotkey: str, + theta: Params, + source: VerifiedAnchor, + ) -> VerifiedAnchor: + """New OD run from a cached independently replayed checkpoint; heads only may extend.""" + from hypertrain.models.opendecision import extend_params + + cfg = TrainConfig.from_manifest_v2(wrapper) + key = (source.run_id, source.hotkey, source.w, source.layout_hash) + if self.entries.get(key) is not source or source.w < 0: + raise AuditInputError("warm-start source is not independently replayed") + if cfg.model.od is None or not cfg.model.od.warm_start: + raise AuditInputError("manifest does not permit OD warm start") + if state_hash(theta) != state_hash(extend_params(source.theta, cfg.model)): + raise AuditInputError("warm-start checkpoint tensor mismatch") + st = init_state(replace(cfg.inner, state_policy="reset"), theta) + ef = {n: torch.zeros_like(x) for n, x in theta.items()} + proof = sha256_hex(bytes.fromhex(wrapper.run_id() + source.proof_hash + state_hash(theta))) + entry = VerifiedAnchor( + wrapper.run_id(), + hotkey, + -1, + self.layout_hash(wrapper), + st, + ef, + proof, + proof, + {n: x.clone() for n, x in theta.items()}, + source.backend, + ) + self.entries[(entry.run_id, hotkey, -1, entry.layout_hash)] = entry + return entry + + def persist(self, directory: Path, entry: VerifiedAnchor) -> Path: + """Durable cache file, atomically published after MATCH; caller owns dispute pins.""" + import json + import os + import tempfile + + from hypertrain.gpu_ops.journal import durable_write + + key = (entry.run_id, entry.hotkey, entry.w, entry.layout_hash) + if self.entries.get(key) is not entry: + raise AuditInputError("cannot persist unverified anchor") + directory.mkdir(parents=True, exist_ok=True) + target = directory / entry.anchor_hash + if target.exists(): + return target + with tempfile.TemporaryDirectory(dir=directory, prefix="anchor-") as tmp: + path = Path(tmp) + state = pack_state(entry.theta, entry.state) + ef = pack_state(entry.ef) + durable_write(path / "state", state) + durable_write(path / "ef", ef) + durable_write( + path / "metadata", + json.dumps( + { + "run_id": entry.run_id, + "hotkey": entry.hotkey, + "w": entry.w, + "layout_hash": entry.layout_hash, + "proof_hash": entry.proof_hash, + "anchor_hash": entry.anchor_hash, + "backend": entry.backend, + "state_sha256": sha256_hex(state), + "ef_sha256": sha256_hex(ef), + }, + sort_keys=True, + ).encode(), + ) + os.rename(path, target) + return target + + def restore( + self, + path: Path, + wrapper: RunManifestV2, + expected_anchor: str, + expected_proof: str, + expected_state_root: str, + expected_ef_hash: str, + *, + expected_hotkey: str, + expected_round: int, + expected_backend: str, + ) -> VerifiedAnchor: + """L0 supplies roots AND identity/round/backend from its authorized replay journal. + + Metadata beside the tensors is never authority for execution provenance. + """ + import json + + key = (wrapper.run_id(), expected_hotkey, expected_round, self.layout_hash(wrapper)) + # An unsuccessful reload must not leave a previously inserted anchor usable. + self.entries.pop(key, None) + data = json.loads((path / "metadata").read_bytes()) + state_blob, ef_blob = (path / "state").read_bytes(), (path / "ef").read_bytes() + if ( + data["run_id"] != wrapper.run_id() + or data["layout_hash"] != self.layout_hash(wrapper) + or data["anchor_hash"] != expected_anchor + or data["proof_hash"] != expected_proof + or data["hotkey"] != expected_hotkey + or type(data["w"]) is not int + or data["w"] != expected_round + or data["backend"] != expected_backend + or sha256_hex(state_blob) != data["state_sha256"] + or sha256_hex(ef_blob) != data["ef_sha256"] + ): + raise AuditInputError("cached anchor authentication mismatch") + theta, st = unpack_state(state_blob) + ef, _ = unpack_state(ef_blob) + if st is None: + raise AuditInputError("cached anchor has no optimizer") + if tensor_root(theta, st) != expected_state_root or state_hash(ef) != expected_ef_hash: + raise AuditInputError("cached anchor does not match authenticated replay roots") + entry = VerifiedAnchor( + data["run_id"], + data["hotkey"], + data["w"], + data["layout_hash"], + st, + ef, + data["proof_hash"], + data["anchor_hash"], + theta, + data["backend"], + ) + self.entries[(entry.run_id, entry.hotkey, entry.w, entry.layout_hash)] = entry + return entry + + def prior(self, wrapper: RunManifestV2, start: StartStateV2) -> VerifiedAnchor: + key = (wrapper.run_id(), start.hotkey, start.w - 1, self.layout_hash(wrapper)) + try: + entry = self.entries[key] + except KeyError as e: + raise AuditInputError( + "ANCHOR_BUDGET_EXCEEDED: no independently verified predecessor" + ) from e + if ( + start.parent_anchor_hash != entry.anchor_hash + or start.anchor_verdict_hash != entry.proof_hash + or start.opt_state_hash != optimizer_hash(entry.state) + or start.ef_hash != state_hash(entry.ef) + ): + raise AuditInputError("carried anchor lineage mismatch") + return entry + + +def audit_island( + job: AuditJobV2, + comm: Comm, + get_sample: SampleFn, + theta_blob: bytes, + ef_blob: bytes, + v0_blob: bytes, + cache: AnchorCache, + *, + now_round: int, + cancel: CancelHook | None = None, + hook: TraceHook | None = None, + device: torch.device | str = "cpu", + beacon_now: Callable[[], int] | None = None, +) -> tuple[Outcome, RoundResult]: + """Full exact-layout replay from an independently checked carry lineage, never served prefix. + + L0 authenticates job issuer/roles/reservation before calling. Live lease rechecked here. + If one predecessor is missing L0 first executes it under the same reserved 2H budget; + ordinary worker never silently performs an unreserved historical rebuild. + """ + job.validate_embedded(now_round) + + def live() -> None: + if beacon_now is not None: + job.validate_embedded(beacon_now()) + if cancel is not None: + cancel() + + cfg = TrainConfig.from_manifest_v2(job.manifest) + if job.anchor_age > 1 or not cfg.inner.H <= job.replay_step_budget <= 2 * cfg.inner.H: + raise AuditInputError("ANCHOR_BUDGET_EXCEEDED") + start = job.start_state + if sha256_hex(theta_blob) != start.state_object_sha256: + raise AuditInputError("start object hash mismatch") + if sha256_hex(ef_blob) != job.ef_in.sha256 or len(ef_blob) != job.ef_in.size: + raise AuditInputError("EF object hash/size mismatch") + if sha256_hex(v0_blob) != job.v0.sha256 or len(v0_blob) != job.v0.size: + raise AuditInputError("v0 object hash/size mismatch") + theta, supplied = unpack_state(theta_blob) + ef, _ = unpack_state(ef_blob) + v0, _ = unpack_state(v0_blob) + if ( + state_hash(theta) != start.theta_hash + or state_hash(ef) != start.ef_hash + or start.ef_object_sha256 != job.ef_in.sha256 + ): + raise AuditInputError("start theta/EF binding mismatch") + entry = cache.prior(job.manifest, start) + backend = torch.device(device).type + if entry.backend not in ("genesis", backend): + raise AuditInputError("CPU anchor cannot serve as CUDA replay oracle") + if supplied is None or optimizer_hash(supplied) != optimizer_hash(entry.state): + raise AuditInputError("fabricated carried prefix") + if cfg.inner.state_policy == "carry" and start.global_step0 != entry.state.step: + raise AuditInputError("global optimizer step mismatch") + theta = {n: x.to(device) for n, x in theta.items()} + ef = {n: x.to(device) for n, x in ef.items()} + v0 = {n: x.to(device) for n, x in v0.items()} + carry = OptState( + {n: x.to(device) for n, x in entry.state.m.items()}, + {n: x.to(device) for n, x in entry.state.v.items()}, + entry.state.step, + ) + commit = CommitV2.model_validate(job.commit_envelope["body"]) + a = IslandAssignment( + job.run_id, + commit.w, + tuple(job.sample_ids), + start.global_step0, + job.manifest.training.reference_spec.layout.n_gpus, + ) + if any( + p.batch_ids_sha256 != _expected_batch_hash(cfg, a, i * cfg.inner.J) + for i, p in enumerate(job.preimages) + ): + raise AuditInputError("ASSIGNMENT_VIOLATION") + result = train_island( + cfg, + job.manifest.training.reference_spec.layout, + comm, + theta, + a, + get_sample, + ef_in=ef, + carry=carry if cfg.inner.state_policy == "carry" else None, + v0=v0 if cfg.inner.state_policy == "derived" else None, + cancel=live, + hook=hook, + ) + live() + first = next( + ( + i + for i, (p, r) in enumerate(zip(job.preimages, result.leaves, strict=True)) + if p.digest() != r.preimage.digest() + ), + None, + ) + match = ( + first is None + and result.leaves_root == commit.leaves_root + and result.delta_hash == commit.delta_hash + and result.final_theta_hash == commit.final_theta_hash + and result.ef_in_hash == commit.ef_in_hash + and result.ef_out_hash == commit.ef_out_hash + ) + out = Outcome("MATCH" if match else "MISMATCH", first, result.leaves_root) + if match: + proof = sha256_hex(bytes.fromhex(job.job_id + result.leaves_root + result.delta_hash)) + anchor_hash = sha256_hex(bytes.fromhex(start.digest() + proof)) + layout = cache.layout_hash(job.manifest) + cache.entries[(job.run_id, commit.hotkey, commit.w, layout)] = VerifiedAnchor( + job.run_id, + commit.hotkey, + commit.w, + layout, + result.final_state.clone(), + {n: x.clone() for n, x in result.ef_out.items()}, + proof, + anchor_hash, + {n: x.clone() for n, x in result.final_theta.items()}, + backend, + ) + return out, result + + +def audit_island_chain( + current: AuditJobV2, + predecessor: AuditJobV2 | None, + comm: Comm, + get: SampleFn, + current_blobs: tuple[bytes, bytes, bytes], + predecessor_blobs: tuple[bytes, bytes, bytes] | None, + cache: AnchorCache, + *, + now_round: int, + cancel: CancelHook | None = None, + device: torch.device | str = "cpu", + beacon_now: Callable[[], int] | None = None, +) -> tuple[Outcome, RoundResult]: + """At most one missing predecessor; both accepted jobs consume current reserved 2H. + + L0 supplies authenticated live predecessor job/reservation, never an unsigned prefix. + """ + current.validate_embedded(beacon_now() if beacon_now is not None else now_round) + + def live() -> None: + current.validate_embedded(beacon_now() if beacon_now is not None else now_round) + if cancel is not None: + cancel() + + h = current.manifest.training.inner.H + if predecessor is not None: + if ( + current.anchor_age != 1 + or current.replay_step_budget < 2 * h + or predecessor.start_state.w + 1 != current.start_state.w + or predecessor.run_id != current.run_id + or predecessor.start_state.hotkey != current.start_state.hotkey + or predecessor_blobs is None + ): + raise AuditInputError("ANCHOR_BUDGET_EXCEEDED: predecessor reservation mismatch") + out, _ = audit_island( + predecessor, + comm, + get, + *predecessor_blobs, + cache, + now_round=now_round, + cancel=live, + device=device, + beacon_now=beacon_now, + ) + if out.result != "MATCH": + raise AuditInputError("predecessor replay did not MATCH") + return audit_island( + current, + comm, + get, + *current_blobs, + cache, + now_round=now_round, + cancel=live, + device=device, + beacon_now=beacon_now, + ) + + +def replay_island_windows( + cfg: TrainConfig, + wrapper: RunManifestV2, + comm: Comm, + theta: Params, + st: OptState, + a: Assignment, + start_leaf: int, + end_leaf: int, + get_sample: SampleFn, + *, + cancel: CancelHook | None = None, + hook: TraceHook | None = None, +) -> tuple[list[LeafPreimage], Params, OptState]: + """Same-layout windows for L5 AFTER anchored prefix verification; no independent verdict.""" + if cfg.inner.state_policy != "carry" or cfg.inner.rewarmup_steps: + raise AuditInputError("island windows require carry without rewarmup") + if not 0 <= start_leaf < end_leaf <= cfg.inner.H // cfg.inner.J: + raise AuditInputError("invalid island window") + per = ( + cfg.inner.micro_batch * cfg.inner.grad_accum * wrapper.training.reference_spec.layout.n_gpus + ) + wcfg = replace(cfg, inner=replace(cfg.inner, H=cfg.inner.J)) + pres = [] + for i in range(start_leaf, end_leaf): + s = i * cfg.inner.J + wa = Assignment( + a.run_id, a.w, a.sample_ids[s * per : (s + cfg.inner.J) * per], a.global_step0 + s + ) + + def window_hook(ctx: TraceContext, x: torch.Tensor, offset: int = s) -> torch.Tensor: + assert hook is not None + return hook(replace(ctx, step=ctx.step + offset), x) + + result = train_island( + wcfg, + wrapper.training.reference_spec.layout, + comm, + theta, + wa, + get_sample, + carry=st, + cancel=cancel, + hook=window_hook if hook is not None else None, + ) + t = s + cfg.inner.J + pres.append( + result.leaves[-1].preimage.model_copy( + update={"t": t, "rng_ctr": rng_ctr(a.run_id, a.w, t, -1)} + ) + ) + theta, st = result.final_theta, result.final_state + return pres, theta, st + + def _uniform(seed: bytes, n: int, ctr: list[int]) -> int: """Unbiased integer in [0, n) by rejection over sha256(seed | ctr).""" limit = (1 << 256) - (1 << 256) % n diff --git a/src/hypertrain/auditor/worker.py b/src/hypertrain/auditor/worker.py index 41ea610..d513d93 100644 --- a/src/hypertrain/auditor/worker.py +++ b/src/hypertrain/auditor/worker.py @@ -16,12 +16,14 @@ import threading from collections.abc import Mapping from dataclasses import dataclass -from typing import Any, Protocol +from pathlib import Path +from typing import Any, Literal, Protocol import httpx import hypertrain.trainer # noqa: F401 from hypertrain.auditor.replay import ( + AnchorCache, AuditInputError, AuditInputs, NotReady, @@ -34,6 +36,8 @@ select_segments, unpack_state, ) +from hypertrain.gpu_ops.work_screen import IslandLaunch +from hypertrain.miner.island_launch import IslandArtifacts, confined, launch_island from hypertrain.protocol.envelope import seal from hypertrain.protocol.keys import Keypair from hypertrain.protocol.messages import ( @@ -44,9 +48,180 @@ RunManifest, StateServe, ) +from hypertrain.protocol.messages_v2 import AuditJobV2, CommitV2, IslandJobV1 from hypertrain.trainer.config import TrainConfig from hypertrain.trainer.loop import Assignment, Params, SampleFn + +class LeaseGuard: + """L0 verified-beacon subscriber calls advance/revoke; trainer checks every boundary. + + A local hard timer also cancels torchrun while blocked inside a collective. No renewals. + """ + + def __init__(self, lease_expires: int, absolute_deadline: int, deadline_unix: float) -> None: + self.expires = min(lease_expires, absolute_deadline) + self.cancelled = threading.Event() + import time + + self.timer = threading.Timer(max(0.0, deadline_unix - time.time()), self.cancelled.set) + self.timer.daemon = True + + def __enter__(self) -> LeaseGuard: + self.timer.start() + return self + + def __exit__(self, *args: Any) -> None: + self.timer.cancel() + + def advance(self, verified_beacon: int) -> None: + if verified_beacon >= self.expires: + self.cancelled.set() + + def revoke(self) -> None: + self.cancelled.set() + + def check(self) -> None: + if self.cancelled.is_set(): + raise AuditInputError("audit lease expired or revoked") + + +def execute_island_audit( + job: AuditJobV2, + directory: Path, + cache: AnchorCache, + guard: LeaseGuard, + *, + now_round: int, + deadline_unix: int, + backend: Literal["cpu", "cuda"] = "cuda", + publication_job: IslandJobV1 | None = None, + publication_directory: Path | None = None, + launch: IslandLaunch | None = None, +) -> tuple[Outcome, IslandArtifacts]: + """Actual torchrun audit runtime; L0 stages authenticated objects and sample proofs. + + Subscribe guard.advance to verified beacon events before invoking; no heartbeat renewal. + directory contains start_state, ef_in, v0, samples, sample_proofs. No HTTP route invented. + """ + import json + + from hypertrain.auditor.replay import VerifiedAnchor, optimizer_hash, pack_state + from hypertrain.gpu_ops.journal import durable_write + from hypertrain.protocol.hashing import sha256_hex + from hypertrain.trainer.compress import state_hash + + job.validate_embedded(now_round) + guard.check() + start = job.start_state + prior = cache.prior(job.manifest, start) + if prior.backend not in ("genesis", backend): + raise AuditInputError("CPU anchor cannot serve as CUDA replay oracle") + theta_blob = confined(directory, "start_state").read_bytes() + ef_blob = confined(directory, "ef_in").read_bytes() + v0_blob = confined(directory, "v0").read_bytes() + theta, st = unpack_state(theta_blob) + ef, _ = unpack_state(ef_blob) + if ( + sha256_hex(theta_blob) != start.state_object_sha256 + or st is None + or optimizer_hash(st) != optimizer_hash(prior.state) + or state_hash(theta) != start.theta_hash + or (job.manifest.training.inner.state_policy == "carry" and st.step != start.global_step0) + or sha256_hex(ef_blob) != job.ef_in.sha256 + or len(ef_blob) != job.ef_in.size + or sha256_hex(ef_blob) != start.ef_object_sha256 + or state_hash(ef) != start.ef_hash + or sha256_hex(v0_blob) != job.v0.sha256 + or len(v0_blob) != job.v0.size + ): + raise AuditInputError("untrusted audit start/state/EF") + start_path = "start_state" + start_hash = start.state_object_sha256 + if job.manifest.training.inner.state_policy != "carry": + # The authenticated anchor still includes m/v; reset/derived kernels take theta only. + start_path = "audit-theta.safetensors" + packed = pack_state(theta) + start_hash = sha256_hex(packed) + path = directory / start_path + if path.exists(): + if confined(directory, start_path).read_bytes() != packed: + raise AuditInputError("conflicting prepared audit theta") + else: + durable_write(path, packed) + local = IslandJobV1( + job_version=1, + run_id=job.run_id, + w=start.w, + manifest=job.manifest, + sample_ids=job.sample_ids, + global_step0=start.global_step0, + start_state_sha256=start_hash, + ef_in_sha256=job.ef_in.sha256, + v0_sha256=job.v0.sha256, + deadline=deadline_unix, + object_paths={ + "start_state": start_path, + **{k: k for k in ("ef_in", "v0", "samples", "sample_proofs")}, + }, + ) + if publication_job is not None: + if publication_directory is None or publication_job != local.model_copy( + update={"object_paths": publication_job.object_paths} + ): + raise AuditInputError("accepted audit publication job differs from independent replay") + for name, relative in publication_job.object_paths.items(): + expected = ( + pack_state(theta) + if name == "start_state" and start_path != "start_state" + else confined(directory, name).read_bytes() + ) + if confined(publication_directory, relative).read_bytes() != expected: + raise AuditInputError("accepted audit publication input differs") + local, directory = publication_job, publication_directory + elif publication_directory is not None: + raise AuditInputError("publication directory lacks accepted descriptor") + artifacts = (launch or launch_island)( + local, directory, backend=backend, cancel=guard.cancelled, trace=True + ) + if launch is not None: + from hypertrain.miner.island_launch import validate_artifacts + + artifacts = validate_artifacts(local, artifacts.directory) + from hypertrain.auditor.island_bisect import IslandParty + + IslandParty.published(start.hotkey, local, artifacts.directory) + guard.check() + commit = CommitV2.model_validate(job.commit_envelope["body"]) + summary = json.loads((artifacts.directory / "rank-0/summary.json").read_bytes())["commitments"] + leaves = [LeafPreimage.model_validate(x) for x in json.loads(artifacts.leaves.read_bytes())] + first = next( + (i for i, (a, b) in enumerate(zip(job.preimages, leaves, strict=True)) if a != b), None + ) + matches = ( + first is None + and summary["leaves_root"] == commit.leaves_root + and summary["delta_hash"] == commit.delta_hash + and summary["final_theta_hash"] == commit.final_theta_hash + and summary["ef_out_hash"] == commit.ef_out_hash + and state_hash(ef) == commit.ef_in_hash + ) + outcome = Outcome("MATCH" if matches else "MISMATCH", first, summary["leaves_root"]) + if matches: + theta, st = unpack_state(artifacts.state.read_bytes()) + ef, _ = unpack_state(artifacts.ef.read_bytes()) + assert st is not None + proof = sha256_hex( + bytes.fromhex(job.job_id + summary["leaves_root"] + summary["delta_hash"]) + ) + anchor_hash = sha256_hex(bytes.fromhex(start.digest() + proof)) + layout = cache.layout_hash(job.manifest) + cache.entries[(job.run_id, commit.hotkey, commit.w, layout)] = VerifiedAnchor( + job.run_id, commit.hotkey, commit.w, layout, st, ef, proof, anchor_hash, theta, backend + ) + return outcome, artifacts + + HEARTBEAT_SECONDS = 300.0 VERDICT_TTL_ROUNDS = 1200 diff --git a/src/hypertrain/beacon/freshness.py b/src/hypertrain/beacon/freshness.py new file mode 100644 index 0000000..e3cde94 --- /dev/null +++ b/src/hypertrain/beacon/freshness.py @@ -0,0 +1,112 @@ +"""Durable exact-beacon pins: pause infrastructure, never replace a draw seed.""" + +from __future__ import annotations + +import hashlib +import sqlite3 +from dataclasses import dataclass + +from hypertrain.beacon.core import ( + Beacon, + BeaconRound, + BeaconUnavailable, + BeaconVerificationError, +) +from hypertrain.beacon.drand import verify_quicknet + + +@dataclass(frozen=True, slots=True) +class FreshnessStatus: + run_id: str + operation_id: str + pinned_round: int + signature_hash: str | None + paused: bool + reason: str | None + + +class FreshnessGate: + """Use L0's SQLite connection; caller commits pin/pause before returning. + + Calling refresh after an arrival event resumes the *same* round. This class + does not poll, move deadlines, or grant eligibility during an outage. + """ + + def __init__(self, db: sqlite3.Connection, run_id: str) -> None: + self.db, self.run_id = db, run_id + db.execute("""CREATE TABLE IF NOT EXISTS beacon_pins_v2 ( + run_id TEXT NOT NULL, operation_id TEXT NOT NULL, pinned_round INTEGER NOT NULL, + signature TEXT, signature_hash TEXT, paused INTEGER NOT NULL, + reason TEXT, PRIMARY KEY(run_id, operation_id))""") + + def pin(self, operation_id: str, round_number: int) -> FreshnessStatus: + if type(round_number) is not int or round_number < 1: + raise BeaconVerificationError("invalid pinned round") + self.db.execute( + """INSERT OR IGNORE INTO beacon_pins_v2 + VALUES (?, ?, ?, NULL, NULL, 1, 'AWAITING_PINNED_BEACON')""", + (self.run_id, operation_id, round_number), + ) + status = self.status(operation_id) + if status.pinned_round != round_number: + raise BeaconVerificationError("replacement seed forbidden") + return status + + def status(self, operation_id: str) -> FreshnessStatus: + row = self.db.execute( + """SELECT pinned_round, signature_hash, paused, reason + FROM beacon_pins_v2 WHERE run_id=? AND operation_id=?""", + (self.run_id, operation_id), + ).fetchone() + if row is None: + raise BeaconUnavailable("draw not pinned") + return FreshnessStatus(self.run_id, operation_id, row[0], row[1], bool(row[2]), row[3]) + + def refresh( + self, + operation_id: str, + beacon: Beacon, + *, + now_seconds: int, + genesis_seconds: int, + period_seconds: int, + max_lag_rounds: int, + ) -> BeaconRound: + """Verify current freshness separately; historical draw stays exactly pinned.""" + status = self.status(operation_id) + if period_seconds < 1 or max_lag_rounds < 0 or now_seconds < genesis_seconds: + raise BeaconVerificationError("invalid freshness clock") + emitted = (now_seconds - genesis_seconds) // period_seconds + 1 + health_round = max(1, emitted - max_lag_rounds) + try: + draw = beacon.get(status.pinned_round) + health = draw if health_round == status.pinned_round else beacon.get(health_round) + for br, expected in ((draw, status.pinned_round), (health, health_round)): + if ( + br.round != expected + or not verify_quicknet(br.round, br.signature) + or (hashlib.sha256(bytes.fromhex(br.signature)).hexdigest() != br.randomness) + ): + raise BeaconVerificationError("exact beacon signature/randomness mismatch") + digest = hashlib.sha256(bytes.fromhex(draw.signature)).hexdigest() + if status.signature_hash is not None and status.signature_hash != digest: + raise BeaconVerificationError("pinned signature changed") + except (BeaconUnavailable, BeaconVerificationError, ValueError) as exc: + self.db.execute( + """UPDATE beacon_pins_v2 SET paused=1, reason=? + WHERE run_id=? AND operation_id=?""", + (type(exc).__name__, self.run_id, operation_id), + ) + raise BeaconUnavailable("infrastructure pause; exact seed retained") from exc + self.db.execute( + """UPDATE beacon_pins_v2 SET signature=?, signature_hash=?, + paused=0, reason=NULL WHERE run_id=? AND operation_id=?""", + (draw.signature, digest, self.run_id, operation_id), + ) + return draw + + def require_fresh(self, operation_id: str) -> FreshnessStatus: + status = self.status(operation_id) + if status.paused or status.signature_hash is None: + raise BeaconUnavailable("eligibility/draw/deadline advancement frozen") + return status diff --git a/src/hypertrain/challenge/admission.py b/src/hypertrain/challenge/admission.py new file mode 100644 index 0000000..999d13f --- /dev/null +++ b/src/hypertrain/challenge/admission.py @@ -0,0 +1,987 @@ +"""Bounded, persisted, shadow admission backed by actual full-round references.""" + +from __future__ import annotations + +import hmac +import secrets +import threading +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Literal + +from pydantic import JsonValue + +if TYPE_CHECKING: + from experiments.gpu_network_v2.orchestrate import NetworkRuntime + from scripts.network_gpu_operation import Operation + +from hypertrain.beacon.core import BeaconRound +from hypertrain.challenge.admission_store import AdmissionError, AdmissionRecord, AdmissionStore +from hypertrain.challenge.trust_v2 import FundedStatus, nominal_q +from hypertrain.data.store import Store +from hypertrain.data.trial_assignment import trial_assignment_hash, trial_samples +from hypertrain.gpu_ops.work_screen import IslandLaunch, screen_work +from hypertrain.ledger.escrow_v2 import EscrowV2, digest +from hypertrain.protocol.envelope import body_digest +from hypertrain.protocol.envelope_v2 import Intake, load_json, parse_envelope, seal, verify_join +from hypertrain.protocol.hashing import sha256_hex +from hypertrain.protocol.keys import Keypair +from hypertrain.protocol.messages import Finalize +from hypertrain.protocol.messages_v2 import ( + AdmissionPolicyV2, + EscrowLock, + IslandJobV1, + JoinChallenge, + JoinRequest, + RotateRequest, + WorkProof, + WorkScreenV2, +) + + +@dataclass(frozen=True, slots=True) +class AdmissionStatus: + record: AdmissionRecord + funding: FundedStatus + nominal_q_ppm: int + actual_q_ppm: int + eligible: bool + shadow_only: bool + + +class Admission: + """Trusted service API, not routes. L0 supplies current verified beacons and jobs. + + Job staging authenticates genesis/carry starts and exact-image qualification. + Default CUDA references; explicit CPU test mode never qualifies production. + """ + + def __init__( + self, + escrow: EscrowV2, + policy: AdmissionPolicyV2, + coordinator: Keypair, + *, + beacon: Callable[[int], BeaconRound], + stage_reference: Callable[[JoinChallenge, tuple[int, ...], int], tuple[IslandJobV1, Path]], + objects: Store, + ip_secret: bytes, + qualified: Callable[[WorkScreenV2], bool], + conservative_bound: Callable[[], bool], + backend: Literal["cpu", "cuda"] = "cuda", + ) -> None: + if digest(policy) != escrow.manifest.network.admission_policy_hash: + raise AdmissionError("ADMISSION_POLICY_HASH") + if coordinator.ss58 != escrow.manifest.training.coord_pubkey: + raise AdmissionError("COORDINATOR_AUTHORITY") + if backend == "cpu" and escrow.policy.ledger_mode != "test": + raise AdmissionError("CPU_NOT_PRODUCTION_QUALIFICATION") + self.escrow, self.policy, self.coord = escrow, policy, coordinator + self.store = AdmissionStore(escrow) + self.beacon, self.stage_reference, self.objects = beacon, stage_reference, objects + self.ip_secret, self.qualified = ip_secret, qualified + self.conservative_bound, self.backend = conservative_bound, backend + + def _beacon(self, now: int) -> BeaconRound: + b = self.beacon(now) + if b.round != now or ( + self.escrow.policy.ledger_mode == "production" and not b.bls_verified + ): + raise AdmissionError("UNVERIFIED_BEACON") + return b + + def join(self, raw: bytes, *, now: int, ip_prefix: str) -> AdmissionRecord: + self._beacon(now) + request = JoinRequest.model_validate(load_json(raw, max_bytes=65536)) + if not verify_join(request, self.escrow.run_id, now): + raise AdmissionError("DUAL_SIGNATURE_RUN_OR_EXPIRY") + if request.policy_hash != digest(self.policy): + raise AdmissionError("POLICY_CHANGED") + key = f"join|{self.escrow.run_id}|{request.coldkey}|{request.request_id}" + request_hash = body_digest(request.model_dump(mode="json", exclude={"hot_sig", "cold_sig"})) + with self.escrow.tx(): + old = self.store.db.execute( + "SELECT * FROM admission_reservations WHERE reservation=?", (key,) + ).fetchone() + if old: + self.store.reserve(key, request_hash, old["receipt"]) + return self.store.by_id(old["receipt"]) + if self.store.db.execute( + "SELECT 1 FROM admission_history WHERE hotkey=?", (request.hotkey,) + ).fetchone(): + raise AdmissionError("HOTKEY_HISTORY_EXISTS") + if self.store.db.execute( + "SELECT 1 FROM admissions_v2 WHERE coldkey=? AND state!='ACTIVE'", + (request.coldkey,), + ).fetchone(): + raise AdmissionError("COLDKEY_PROBATION_EXISTS") + count = self.store.db.execute( + "SELECT COUNT(*) FROM admissions_v2 WHERE state IN ('APPLIED','PROBATION')" + ).fetchone()[0] + if count >= self.policy.max_pending: + raise AdmissionError("PENDING_CAPACITY") + for identity in (request.hotkey, request.coldkey): + self.store.quota(identity, now, self.policy.join_per_beacon, self.policy.join_burst) + epoch = now // self.policy.work_screen_epoch_rounds + prefix = hmac.digest(self.ip_secret, f"{epoch}|{ip_prefix}".encode(), "sha256").hex() + self.store.db.execute( + "DELETE FROM admission_quota WHERE identity LIKE 'ip:%' AND beacon dict[str, JsonValue]: + self._beacon(now) + with self.escrow.tx(): + r = self.store.by_id(admission_id) + row = self.store.db.execute( + "SELECT * FROM admissions_v2 WHERE admission_id=?", (admission_id,) + ).fetchone() + if r.state not in ("APPLIED", "PROBATION", "ACTIVE") or r.pending_dispute: + raise AdmissionError("CHALLENGE_STATE") + if row["challenge"]: + c = JoinChallenge.model_validate_json(row["challenge"]) + trial = self.store.db.execute( + "SELECT outcome FROM admission_trial_results WHERE epoch=?", + (row["trial_epoch"],), + ).fetchone() + if trial and trial["outcome"] == "OPEN": + if now > c.deadline_beacon: + self.store.transition(admission_id, "SUSPENDED", "CHALLENGE_EXPIRED", now) + self.store.db.execute( + "UPDATE admission_trial_results SET outcome='EXPIRED' WHERE epoch=?", + (row["trial_epoch"],), + ) + reservation = self.store.db.execute( + "SELECT receipt FROM admission_reservations WHERE reservation=?", + (f"challenge|{admission_id}|{c.nonce}",), + ).fetchone() + if reservation is None: + raise AdmissionError("MISSING_CHALLENGE_RECEIPT") + return load_json(reservation["receipt"]) + seed = row["seed_beacon"] + if now < seed: + raise AdmissionError("SEED_NOT_READY") + if now > seed + self.policy.challenge_rounds: + raise AdmissionError("CHALLENGE_EXPIRED") + b = self._beacon(seed) + running = self.store.db.execute( + "SELECT COUNT(*) FROM admission_trial_results WHERE outcome='OPEN'" + ).fetchone()[0] + if running >= self.policy.max_trial_replays: + raise AdmissionError("TRIAL_CAPACITY") + trial_epoch = self.store.db.execute( + "SELECT COALESCE(MAX(epoch),-1)+1 FROM admission_trials" + ).fetchone()[0] + nonce = secrets.token_hex(32) + samples = trial_samples(self.escrow.manifest, admission_id, nonce, b) + c = JoinChallenge( + admission_id=admission_id, + nonce=nonce, + seed_beacon=seed, + deadline_beacon=seed + 100, + manifest_hash=self.escrow.run_id, + theta_hash=self.escrow.manifest.training.init_state_hash, + assignment_hash=trial_assignment_hash(self.escrow.manifest, trial_epoch, samples), + layout=self.escrow.manifest.training.reference_spec.layout, + artifact_limits=self.policy.artifact_limits, + policy_hash=digest(self.policy), + ) + self.store.db.execute( + "INSERT INTO " + "admission_trials(epoch,admission_id,nonce,received_beacon) " + "VALUES(?,?,?,?)", + (trial_epoch, admission_id, nonce, now), + ) + self.store.db.execute( + "INSERT INTO admission_trial_results(epoch,challenge) VALUES(?,?)", + (trial_epoch, c.model_dump_json()), + ) + self.store.db.execute( + "UPDATE admissions_v2 SET " + "challenge=?,trial_epoch=?,proof_digest=NULL," + "reference_commitment=NULL,reference_json=NULL " + "WHERE admission_id=?", + (c.model_dump_json(), trial_epoch, admission_id), + ) + envelope = seal(self.coord, "JoinChallenge", self.escrow.run_id, c, c.deadline_beacon) + from hypertrain.protocol.jcs import canonicalize + + self.store.reserve( + f"challenge|{admission_id}|{c.nonce}", c.digest(), canonicalize(envelope).decode() + ) + return envelope + + def prepare_reference( + self, + admission_id: str, + *, + now: int, + trusted_launch: Callable[[IslandJobV1, Path, JoinChallenge, int], IslandLaunch] + | None = None, + cancel: threading.Event | None = None, + retain_shadow_custody: bool = False, + ) -> str: + """Execute independently from trusted start; commit before accepting newcomer proof.""" + self._beacon(now) + row = self.store.db.execute( + "SELECT * FROM admissions_v2 WHERE admission_id=?", (admission_id,) + ).fetchone() + if row is None or row["challenge"] is None: + raise AdmissionError("NO_CHALLENGE") + c = JoinChallenge.model_validate_json(row["challenge"]) + trial = self.store.db.execute( + "SELECT outcome FROM admission_trial_results WHERE epoch=?", (row["trial_epoch"],) + ).fetchone() + if trial is None or trial["outcome"] != "OPEN": + raise AdmissionError("REFERENCE_TRIAL_CLOSED") + if row["reference_commitment"] is not None: + return str(row["reference_commitment"]) + samples = trial_samples( + self.escrow.manifest, admission_id, c.nonce, self._beacon(c.seed_beacon) + ) + job, directory = self.stage_reference(c, samples, row["trial_epoch"]) + if tuple(job.sample_ids) != samples or job.w != row["trial_epoch"]: + raise AdmissionError("REFERENCE_ASSIGNMENT") + if trusted_launch is None: + screen, proof = screen_work(job, directory, c, now_beacon=now, backend=self.backend) + else: + screen, proof = screen_work( + job, + directory, + c, + now_beacon=now, + backend=self.backend, + launch=trusted_launch(job, directory, c, row["trial_epoch"]), + cancel=cancel, + ) + if not self.qualified(screen): + raise AdmissionError("UNQUALIFIED_REFERENCE") + for name, ref in zip( + ("state.safetensors", "ef.safetensors", "delta.bin", "leaves.json"), + proof.artifact_refs, + strict=True, + ): + data = (directory / "published" / "rank-0" / name).read_bytes() + if self.objects.put(data) != ref.sha256: + raise AdmissionError("REFERENCE_OBJECT_HASH") + with self.escrow.tx(): + current = self.store.db.execute( + "SELECT * FROM admissions_v2 WHERE admission_id=?", (admission_id,) + ).fetchone() + if current["challenge"] != c.model_dump_json() or current["proof_digest"] is not None: + raise AdmissionError("REFERENCE_STATE_CHANGED") + commitment = digest(proof) + self.store.db.execute( + "UPDATE admissions_v2 SET " + "reference_commitment=?,reference_json=?,reference_beacon=? WHERE " + "admission_id=?", + (commitment, proof.model_dump_json(), now, admission_id), + ) + self.store.db.execute( + "UPDATE admission_trial_results SET reference=? WHERE epoch=?", + (proof.model_dump_json(), current["trial_epoch"]), + ) + if retain_shadow_custody: + self.retain_trial_publication("reference", job, directory, screen, proof, now) + return commitment + + def retain_runtime_custody( + self, + runtime: NetworkRuntime, + spec: Operation, + directory: Path, + *, + tree: Path, + now: int, + cancel: threading.Event | None = None, + ) -> dict[str, str]: + """Retain authenticated internal completion only; never production acceptance.""" + from experiments.gpu_network_v2.orchestrate import NetworkRuntime + + from hypertrain.protocol.jcs import canonicalize + + if type(runtime) is not NetworkRuntime: + raise AdmissionError("SHADOW_TRUSTED_RUNTIME_REQUIRED") + self._beacon(now) + with self.escrow.tx(): + descriptor = NetworkRuntime.completion_custody( + runtime, spec, directory, tree=tree, cancel=cancel + ) + body = descriptor["body"] + if descriptor["sha256"] != sha256_hex(canonicalize(body)): + raise AdmissionError("SHADOW_RUNTIME_DESCRIPTOR_HASH") + if spec.job.manifest != self.escrow.manifest: + raise AdmissionError("SHADOW_RUNTIME_MANIFEST") + row = self.store.db.execute( + "SELECT * FROM admissions_v2 WHERE admission_id=? AND run_id=?", + (spec.binding, self.escrow.run_id), + ).fetchone() + if row is None or row["challenge"] is None or row["reference_json"] is None: + raise AdmissionError("SHADOW_RUNTIME_REFERENCE_NOT_ACCEPTED") + publication = body["publication"] + challenge = JoinChallenge.model_validate_json(row["challenge"]) + signed_challenge = self.store.db.execute( + "SELECT receipt FROM admission_reservations WHERE reservation=?", + (f"challenge|{spec.binding}|{challenge.nonce}",), + ).fetchone() + if ( + row["hotkey"] != spec.hotkey + or row["trial_epoch"] != spec.job.w + or publication["trial_authority"]["challenge_hash"] != challenge.digest() + or signed_challenge is None + or Intake(self.escrow.run_id, {"JoinChallenge": self.coord.ss58.__eq__}).accept( + signed_challenge["receipt"], challenge.seed_beacon + ) + != challenge + or not challenge.seed_beacon <= now <= challenge.deadline_beacon + or row["reference_commitment"] + != digest(WorkProof.model_validate_json(row["reference_json"])) + ): + raise AdmissionError("SHADOW_RUNTIME_ACCEPTED_CONTEXT") + accepted = self.store.db.execute( + "SELECT * FROM admission_trial_results WHERE epoch=?", (spec.job.w,) + ).fetchone() + reference = WorkProof.model_validate_json(row["reference_json"]) + if accepted is None or accepted["reference"] != row["reference_json"]: + raise AdmissionError("SHADOW_RUNTIME_REFERENCE_NOT_ACCEPTED") + if spec.operation == "probe": + if accepted["proof"] is None or accepted["screen"] is None: + raise AdmissionError("SHADOW_RUNTIME_PROOF_NOT_ACCEPTED") + proof = Intake(self.escrow.run_id, {"WorkProof": spec.hotkey.__eq__}).accept( + accepted["proof"], now + ) + screen = Intake(self.escrow.run_id, {"WorkScreenV2": spec.hotkey.__eq__}).accept( + accepted["screen"], now + ) + assert isinstance(proof, WorkProof) and isinstance(screen, WorkScreenV2) + if ( + proof != reference + or proof + != WorkProof.model_validate(publication["miner_proof"]["envelope"]["body"]) + or screen.challenge_hash != challenge.digest() + or screen.nonce != challenge.nonce + or screen.admission_id != spec.binding + or screen.artifact_hashes != [ref.sha256 for ref in proof.artifact_refs] + or screen.layout != challenge.layout + or screen.policy_hash != challenge.policy_hash + or screen.image_digest != spec.image_digest + or [rank.model_dump(mode="json") for rank in screen.rank_results] + != [summary["work"] for summary in publication["rank_summaries"]] + or not self.qualified(screen) + ): + raise AdmissionError("SHADOW_RUNTIME_ACCEPTED_PROOF") + else: + summaries = publication["rank_summaries"] + if ( + reference.challenge_hash != challenge.digest() + or reference.leaves_root != summaries[0]["commitments"]["leaves_root"] + or reference.delta_hash != summaries[0]["commitments"]["delta_hash"] + or any( + publication["files"]["rank-0/" + name]["sha256"] != ref.sha256 + or publication["files"]["rank-0/" + name]["bytes"] != ref.size + for name, ref in zip( + ("state.safetensors", "ef.safetensors", "delta.bin", "leaves.json"), + reference.artifact_refs, + strict=True, + ) + ) + ): + raise AdmissionError("SHADOW_RUNTIME_ACCEPTED_REFERENCE") + # Internal custody is not accepted backend/economic authority or shadow-execution. + key = f"{spec.binding}|{spec.job.w}|{spec.operation}" + raw = canonicalize(descriptor).decode() + old = self.store.db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='runtime-custody' AND id=?", + (self.escrow.run_id, key), + ).fetchone() + if old is not None and old["data"] != raw: + raise AdmissionError("SHADOW_RUNTIME_CUSTODY_CONFLICT") + if self.objects.put(canonicalize(body)) != descriptor["sha256"]: + raise AdmissionError("SHADOW_RUNTIME_OBJECT_HASH") + self.store.db.execute( + "INSERT OR IGNORE INTO records_v2 VALUES(?,?,?,?)", + (self.escrow.run_id, "runtime-custody", key, raw), + ) + return {"custody_hash": descriptor["sha256"], "status": "NOTSTOREACCEPTED"} + + def retain_trial_publication( + self, + operation: Literal["reference", "probe"], + job: IslandJobV1, + directory: Path, + screen: WorkScreenV2, + proof: WorkProof, + now: int, + *, + commit_raw: bytes | None = None, + ) -> dict[str, str]: + """Internal completion seam only; uploaded objects never create this record.""" + from hypertrain.miner.island_launch import confined, validate_artifacts + from hypertrain.protocol.jcs import canonicalize + from hypertrain.protocol.messages_v2 import CommitV2 + + self._beacon(now) + if self.backend != "cpu" or self.escrow.policy.ledger_mode != "test": + raise AdmissionError("SHADOW_ACCEPTED_RUNTIME_CUSTODY_UNAVAILABLE") + row = self.store.db.execute( + "SELECT * FROM admissions_v2 WHERE admission_id=?", (proof.admission_id,) + ).fetchone() + if row is None or row["challenge"] is None: + raise AdmissionError("SHADOW_EXECUTION_CHALLENGE") + challenge = JoinChallenge.model_validate_json(row["challenge"]) + if ( + row["run_id"] != job.run_id + or row["trial_epoch"] != job.w + or proof.challenge_hash != challenge.digest() + or screen.challenge_hash != challenge.digest() + or screen.nonce != challenge.nonce + or screen.admission_id != proof.admission_id + or row["reference_json"] is None + or WorkProof.model_validate_json(row["reference_json"]) != proof + or challenge.assignment_hash + != trial_assignment_hash(job.manifest, job.w, tuple(job.sample_ids)) + or job.manifest != self.escrow.manifest + or not challenge.seed_beacon <= now <= challenge.deadline_beacon + ): + raise AdmissionError("SHADOW_EXECUTION_BINDING") + commit: CommitV2 | None = None + if operation == "probe": + accepted = self.store.db.execute( + "SELECT proof,screen FROM admission_trial_results WHERE epoch=?", (job.w,) + ).fetchone() + if ( + accepted is None + or accepted["proof"] is None + or accepted["screen"] is None + or WorkProof.model_validate(parse_envelope(accepted["proof"]).body) != proof + or WorkScreenV2.model_validate(parse_envelope(accepted["screen"]).body) != screen + ): + raise AdmissionError("SHADOW_MINER_PROOF_NOT_ACCEPTED") + if commit_raw is None: + raise AdmissionError("SHADOW_MINER_COMMIT_MISSING") + accepted_commit = Intake(self.escrow.run_id, {"CommitV2": row["hotkey"].__eq__}).accept( + commit_raw, now + ) + if not isinstance(accepted_commit, CommitV2) or accepted_commit.w != job.w: + raise AdmissionError("SHADOW_MINER_COMMIT_BINDING") + commit = accepted_commit + commit.validate_assignment(job.manifest, len(job.sample_ids)) + publication = directory / "published" + artifacts = validate_artifacts(job, publication) + if tuple(screen.rank_results) != artifacts.ranks: + raise AdmissionError("SHADOW_PUBLICATION_RANK_SCREEN") + if operation == "probe": + assert commit is not None + from hypertrain.auditor.replay import unpack_state + from hypertrain.protocol.hashing import MerkleTree + from hypertrain.protocol.messages import LeafPreimage + from hypertrain.trainer.compress import state_hash + + summary = load_json((publication / "rank-0/summary.json").read_bytes())["commitments"] + assert isinstance(summary, dict) + import json + + leaves = [ + LeafPreimage.model_validate(p) for p in json.loads(artifacts.leaves.read_bytes()) + ] + if ( + commit.leaves_root != summary["leaves_root"] + or commit.final_theta_hash != summary["final_theta_hash"] + or commit.ef_out_hash != summary["ef_out_hash"] + or commit.delta_hash != sha256_hex(artifacts.delta.read_bytes()) + or commit.delta_bytes != artifacts.delta.stat().st_size + or commit.metrics_root + != MerkleTree( + [bytes.fromhex(p.loss_f32) + bytes.fromhex(p.norm_f32) for p in leaves] + ).root.hex() + or commit.ef_in_hash + != state_hash( + unpack_state(confined(publication, job.object_paths["ef_in"]).read_bytes())[0] + ) + ): + raise AdmissionError("SHADOW_MINER_COMMIT_PUBLICATION") + files = {} + for path in sorted(publication.rglob("*")): + if path.is_symlink() or not (path.is_file() or path.is_dir()): + raise AdmissionError("SHADOW_PUBLICATION_SPECIAL_FILE") + if path.is_file(): + relative = str(path.relative_to(publication)) + data = confined(publication, relative).read_bytes() + if len(data) > challenge.artifact_limits.max_object_bytes: + raise AdmissionError("SHADOW_PUBLICATION_OBJECT_LIMIT") + files[relative] = {"sha256": self.objects.put(data), "bytes": len(data)} + if len(canonicalize(files)) > challenge.artifact_limits.max_chunk_manifest_bytes: + raise AdmissionError("SHADOW_PUBLICATION_MANIFEST_LIMIT") + if any( + load_json(confined(publication, f"rank-{rank}/summary.json").read_bytes())["backend"] + != self.backend + for rank in range(len(artifacts.ranks)) + ) or not self.qualified(screen): + raise AdmissionError("SHADOW_PUBLICATION_BACKEND") + for ref, path in zip( + proof.artifact_refs, + ( + artifacts.state, + artifacts.ef, + artifacts.delta, + artifacts.leaves, + ), + strict=True, + ): + data = path.read_bytes() + if sha256_hex(data) != ref.sha256 or len(data) != ref.size: + raise AdmissionError("SHADOW_PUBLICATION_PROOF") + job_hash = self.objects.put(canonicalize(job.body())) + publication_hash = self.objects.put(canonicalize(files)) + result = { + "job_hash": job_hash, + "publication_hash": publication_hash, + "proof": proof.body(), + "screen": screen.body(), + } + if operation == "probe": + assert commit is not None and commit_raw is not None + result["commit_hash"] = digest(commit) + result["commit_envelope"] = parse_envelope(commit_raw).model_dump(mode="json") + result_hash = self.objects.put(canonicalize(result)) + body = { + "operation": operation, + "admission_id": proof.admission_id, + "challenge_hash": proof.challenge_hash, + "run_id": job.run_id, + "epoch": job.w, + "job_hash": job_hash, + "publication_hash": publication_hash, + "result_hash": result_hash, + "backend": self.backend, + "accepted_beacon": now, + } + operation_hash = self.objects.put(canonicalize(body)) + record = {**body, "directory": str(publication.resolve())} + old = self.store.db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='shadow-execution' AND id=?", + (job.run_id, operation_hash), + ).fetchone() + if old is not None and old["data"] != canonicalize(record).decode(): + raise AdmissionError("SHADOW_EXECUTION_CONFLICT") + self.store.db.execute( + "INSERT OR IGNORE INTO records_v2 VALUES(?,?,?,?)", + (job.run_id, "shadow-execution", operation_hash, canonicalize(record).decode()), + ) + return { + "job_hash": job_hash, + "publication_hash": publication_hash, + "result_hash": result_hash, + "operation_hash": operation_hash, + } + + def proof(self, raw: bytes, screen_raw: bytes, *, now: int) -> AdmissionRecord: + self._beacon(now) + env = parse_envelope(raw) + screen_env = parse_envelope(screen_raw) + with self.escrow.tx(): + if env.type != "WorkProof" or screen_env.type != "WorkScreenV2": + raise AdmissionError("PROOF_TYPE") + proof = WorkProof.model_validate(env.body) + r = self.store.by_id(proof.admission_id) + intake = Intake( + self.escrow.run_id, + {"WorkProof": lambda s: s == r.hotkey, "WorkScreenV2": lambda s: s == r.hotkey}, + ) + intake.accept(raw, now) + intake.accept(screen_raw, now) + screen = WorkScreenV2.model_validate(screen_env.body) + row = self.store.db.execute( + "SELECT * FROM admissions_v2 WHERE admission_id=?", (r.admission_id,) + ).fetchone() + if row["challenge"] is None or row["reference_json"] is None: + raise AdmissionError("REFERENCE_NOT_COMMITTED") + c = JoinChallenge.model_validate_json(row["challenge"]) + if ( + r.state not in ("APPLIED", "PROBATION", "ACTIVE") + or r.pending_dispute + or now > c.deadline_beacon + ): + raise AdmissionError("PROOF_STATE_OR_DEADLINE") + if ( + ( + proof.challenge_hash, + screen.challenge_hash, + screen.admission_id, + screen.nonce, + screen.policy_hash, + screen.image_digest, + screen.layout, + ) + != ( + c.digest(), + c.digest(), + c.admission_id, + c.nonce, + c.policy_hash, + self.escrow.manifest.training.reference_spec.image_digest, + c.layout, + ) + or screen.artifact_hashes != [a.sha256 for a in proof.artifact_refs] + or not self.qualified(screen) + ): + raise AdmissionError("WORK_SCREEN_BINDING") + if row["proof_digest"] is not None: + if ( + row["proof_digest"] != digest(proof) + or row["screen_json"] != screen.model_dump_json() + ): + raise AdmissionError("PROOF_REPLAY_CONFLICT") + return r + if now < row["reference_beacon"]: + raise AdmissionError("PROOF_BEFORE_REFERENCE") + expected = WorkProof.model_validate_json(row["reference_json"]) + if expected != proof: + evidence_hash = sha256_hex( + f"ht-admission-mismatch-v2|{c.digest()}|{digest(expected)}|{digest(proof)}".encode() + ) + dispute_id = sha256_hex( + f"ht-admission-dispute-v2|{self.escrow.run_id}|{r.admission_id}|{evidence_hash}".encode() + ) + self.store.bind_pending(r, dispute_id, evidence_hash) + self.store.transition(r.admission_id, "SUSPENDED", "PENDING_REPLAY_MISMATCH", now) + self.store.db.execute( + "UPDATE admissions_v2 SET pending_dispute=1,strikes=strikes+1 " + "WHERE admission_id=?", + (r.admission_id,), + ) + self.store.db.execute( + "UPDATE admission_trial_results SET proof=?,screen=?,outcome='MISMATCH' " + "WHERE epoch=?", + (raw.decode(), screen_raw.decode(), row["trial_epoch"]), + ) + return self.store.by_id(r.admission_id) + for a in proof.artifact_refs: + if a.size > c.artifact_limits.max_object_bytes: + raise AdmissionError("PROOF_OBJECT_LIMIT") + data = self.objects.get(a.sha256) + if len(data) != a.size or sha256_hex(data) != a.sha256: + raise AdmissionError("PROOF_OBJECT_HASH") + self.store.db.execute( + "UPDATE admissions_v2 SET " + "proof_digest=?,screen_json=?,screen_until=? WHERE admission_id=?", + ( + digest(proof), + screen.model_dump_json(), + now + self.policy.work_screen_epoch_rounds, + r.admission_id, + ), + ) + self.store.db.execute( + "UPDATE admission_trial_results SET proof=?,screen=? WHERE epoch=?", + (raw.decode(), screen_raw.decode(), row["trial_epoch"]), + ) + if r.state == "APPLIED": + self.store.transition(r.admission_id, "PROBATION", "FULL_REFERENCE_MATCH", now) + return self.store.by_id(r.admission_id) + + def finalize_trial(self, admission_id: str, raw: bytes, *, now: int) -> AdmissionStatus: + """One authenticated finalization per actually replayed shadow participation.""" + self._beacon(now) + env = parse_envelope(raw) + if env.type != "Finalize": + raise AdmissionError("TRIAL_FINALITY_TYPE") + with self.escrow.tx(): + r = self.store.by_id(admission_id) + final = Intake(self.escrow.run_id, {"Finalize": lambda s: s == self.coord.ss58}).accept( + raw, now + ) + assert isinstance(final, Finalize) + row = self.store.db.execute( + "SELECT * FROM admissions_v2 WHERE admission_id=?", (admission_id,) + ).fetchone() + trial = self.store.db.execute( + "SELECT * FROM admission_trials WHERE epoch=?", (row["trial_epoch"],) + ).fetchone() + if trial is None or row["proof_digest"] is None: + raise AdmissionError("UNVERIFIED_TRIAL") + expected = WorkProof.model_validate_json(row["reference_json"]) + state_ref = expected.artifact_refs[0] + import hypertrain.trainer # noqa: F401 + from hypertrain.auditor.replay import unpack_state + from hypertrain.trainer.compress import state_hash + + theta, _ = unpack_state(self.objects.get(state_ref.sha256)) + if ( + final.w != row["trial_epoch"] + or final.included != [r.hotkey] + or final.entitlements_root != digest(expected) + or final.final_theta_hash_w1 != state_hash(theta) + ): + raise AdmissionError("TRIAL_FINALITY_BINDING") + key = f"trial-final|{admission_id}|{final.w}" + self.store.reserve(key, body_digest(env.body), raw.decode()) + if trial["finalized_beacon"] is not None: + return self.status(r.hotkey, now=now) + if r.state not in ("PROBATION", "ACTIVE") or r.pending_dispute: + raise AdmissionError("UNSETTLED_TRIAL") + count = self.store.db.execute( + "SELECT COUNT(*) FROM admission_trials WHERE finalized_beacon=?", (now,) + ).fetchone()[0] + if count >= self.policy.max_trials_per_beacon: + raise AdmissionError("TRIAL_FINALIZATION_CAPACITY") + blocks = set(r.canary_blocks) + blocks.add(min(r.clean_count // self.policy.E, 2) + 1) + self.store.db.execute( + "UPDATE admission_trials SET finalized_beacon=?,evidence_hash=? WHERE epoch=?", + (now, digest(expected), final.w), + ) + self.store.db.execute( + "UPDATE admission_trial_results SET outcome='MATCH' WHERE epoch=?", (final.w,) + ) + self.store.db.execute( + "UPDATE admissions_v2 SET " + "clean_count=clean_count+1,canary_blocks=?,seed_beacon=? WHERE " + "admission_id=?", + (",".join(map(str, sorted(blocks))), now + 1, admission_id), + ) + return self.graduate(r.hotkey, now=now) + + def lock(self, raw: bytes, *, now: int) -> str: + self._beacon(now) + env = parse_envelope(raw) + if env.type != "EscrowLock": + raise AdmissionError("LOCK_TYPE") + op = EscrowLock.model_validate(env.body) + with self.escrow.tx(): + r = self.store.by_id(op.admission_id or "") + Intake(self.escrow.run_id, {"EscrowLock": lambda s: s == r.coldkey}).accept(raw, now) + if op.owner != r.coldkey or op.kind != "LOCK_ADMISSION": + raise AdmissionError("LOCK_OWNER_OR_KIND") + self.store.enqueue_lock(op, env.signer) + return self.store.reconcile_lock(op.operation_id).event_hash + + def status(self, hotkey: str, *, now: int) -> AdmissionStatus: + self._beacon(now) + r = self.store.record(hotkey) + units, receipt = self.escrow.locked(r.admission_id, r.coldkey) + p = self.escrow.policy + denominator = p.s_lower_ppm * p.beta_ppm * 1_000_000 + required = max( + p.S_min_units, + self.escrow.manifest.training.verify.s_min_units(p.R_collectible_units), + (p.G_max_units * 10**18 + denominator - 1) // denominator + if denominator + else p.G_max_units + units + 1, + ) + funding = FundedStatus( + self.escrow.run_id, + hotkey, + r.admission_id, + p.ledger_mode, + units, + 0, + receipt, + self.conservative_bound(), + ) + live_phase = r.state == "ACTIVE" or (r.state == "PROBATION" and r.clean_count >= 4) + eligible = ( + live_phase + and not r.pending_dispute + and now <= r.screen_until + and units >= required + and funding.conservative_bound + and p.s_lower_ppm > 0 + and not self.store.pending_outbox(r.admission_id) + ) + return AdmissionStatus( + r, funding, nominal_q(r.state, r.clean_count), p.q_floor, eligible, not eligible + ) + + def graduate(self, hotkey: str, *, now: int) -> AdmissionStatus: + self._beacon(now) + with self.escrow.tx(): + s = self.status(hotkey, now=now) + r = s.record + if ( + r.state == "PROBATION" + and r.clean_count >= self.policy.clean_finalizations + and r.canary_blocks == (1, 2, 3) + and s.eligible + ): + self.store.transition(r.admission_id, "ACTIVE", "GRADUATED_FULL_REPLAY", now) + return self.status(hotkey, now=now) + + def rotate(self, raw: bytes, *, now: int) -> AdmissionRecord: + self._beacon(now) + env = parse_envelope(raw) + if env.type != "RotateRequest": + raise AdmissionError("ROTATION_TYPE") + op = RotateRequest.model_validate(env.body) + with self.escrow.tx(): + history = self.store.db.execute( + "SELECT * FROM admission_history WHERE hotkey=?", (op.hotkey,) + ).fetchone() + if history is None: + raise AdmissionError("UNKNOWN_ROTATION_HISTORY") + Intake(self.escrow.run_id, {"RotateRequest": lambda s: s == history["coldkey"]}).accept( + raw, now + ) + key = f"rotate|{op.operation_id}" + old = self.store.db.execute( + "SELECT * FROM admission_reservations WHERE reservation=?", (key,) + ).fetchone() + if old: + self.store.reserve(key, body_digest(env.body), old["receipt"]) + return self.store.by_id(history["admission_id"]) + r = self.store.record(op.hotkey) + if self.store.db.execute( + "SELECT 1 FROM admission_history WHERE hotkey=?", (op.new_hotkey,) + ).fetchone(): + raise AdmissionError("NEW_HOTKEY_HISTORY") + self.store.db.execute( + "UPDATE admissions_v2 SET hotkey=? WHERE admission_id=? AND hotkey=?", + (op.new_hotkey, r.admission_id, op.hotkey), + ) + self.store.db.execute( + "INSERT INTO admission_history VALUES(?,?,?)", + (op.new_hotkey, r.admission_id, r.coldkey), + ) + self.store.reserve(key, body_digest(env.body), r.admission_id) + return self.store.by_id(r.admission_id) + + def suspend( + self, + hotkey: str, + *, + reason: str, + now: int, + pending_dispute: bool = False, + dispute_id: str | None = None, + evidence_hash: str | None = None, + ) -> None: + """Internal authenticated admin/dispute hook; no accounting release.""" + self._beacon(now) + with self.escrow.tx(): + r = self.store.record(hotkey) + if r.state == "BANNED": + raise AdmissionError("TERMINAL_BANNED") + if pending_dispute and (dispute_id is not None or evidence_hash is not None): + if dispute_id is None or evidence_hash is None: + raise AdmissionError("INCOMPLETE_PENDING_REFERENCE") + self.store.bind_pending(r, dispute_id, evidence_hash) + self.store.transition(r.admission_id, "SUSPENDED", reason, now) + if pending_dispute: + self.store.db.execute( + "UPDATE admissions_v2 SET pending_dispute=1 WHERE admission_id=?", + (r.admission_id,), + ) + + def resume(self, hotkey: str, *, now: int) -> None: + self._beacon(now) + with self.escrow.tx(): + r = self.store.record(hotkey) + if r.state != "SUSPENDED" or r.pending_dispute: + raise AdmissionError("RESUME_STATE") + self.store.transition(r.admission_id, "PROBATION", "RESUME_FRESH_PROBATION", now) + self.store.db.execute( + "UPDATE admission_trial_results SET outcome='INFRASTRUCTURE' " + "WHERE epoch=(SELECT trial_epoch FROM admissions_v2 WHERE admission_id=?) " + "AND outcome='OPEN'", + (r.admission_id,), + ) + self.store.db.execute( + "UPDATE admissions_v2 SET challenge=NULL,proof_digest=NULL,reference_json=NULL," + "reference_commitment=NULL,screen_json=NULL," + "clean_count=0,canary_blocks='',screen_until=0,seed_beacon=? WHERE " + "admission_id=?", + (now + 1, r.admission_id), + ) + + def resolve_pending(self, hotkey: str, reference: str, *, now: int) -> None: + """Resume only from L5's authenticated nonfraud settlement, preserving strikes.""" + self._beacon(now) + with self.escrow.tx(): + r = self.store.record(hotkey) + pending = self.store.db.execute( + "SELECT * FROM admission_pending WHERE admission_id=? AND resolved=0", + (r.admission_id,), + ).fetchone() + if pending is None or pending["dispute_id"] != reference: + raise AdmissionError("PENDING_SETTLEMENT_REFERENCE") + result = self.escrow.settlement(reference) + if ( + r.state != "SUSPENDED" + or not r.pending_dispute + or result.unresolved + or result.outcome == "FRAUD" + or now < result.release_beacon + or ( + result.run_id, + result.admission_id, + result.coldkey, + result.dispute_id, + result.evidence_hash, + ) + != ( + self.escrow.run_id, + r.admission_id, + r.coldkey, + pending["dispute_id"], + pending["evidence_hash"], + ) + ): + raise AdmissionError("PENDING_SETTLEMENT") + self.store.db.execute( + "UPDATE admissions_v2 SET pending_dispute=0 WHERE admission_id=?", (r.admission_id,) + ) + self.store.db.execute( + "UPDATE admission_pending SET resolved=1 WHERE dispute_id=? AND resolved=0", + (reference,), + ) + self.resume(hotkey, now=now) + + def settle_fraud(self, hotkey: str, lock_id: str, *, now: int) -> None: + """Only an authenticated, finalized proven training-fraud settlement bans.""" + self._beacon(now) + with self.escrow.tx(): + r = self.store.record(hotkey) + status = self.escrow.settlement(lock_id) + lock_row = self.store.db.execute( + "SELECT request FROM escrow_events WHERE operation_id=? AND kind='LOCK_ADMISSION'", + (lock_id,), + ).fetchone() + if lock_row is None: + raise AdmissionError("UNKNOWN_FRAUD_LOCK") + op = EscrowLock.model_validate_json(lock_row["request"]) + if ( + op.admission_id != r.admission_id + or op.owner != r.coldkey + or status.unresolved + or status.outcome != "FRAUD" + or now < status.release_beacon + ): + raise AdmissionError("UNPROVEN_TRAINING_FRAUD") + self.store.transition(r.admission_id, "BANNED", "FINALIZED_TRAINING_FRAUD", now) diff --git a/src/hypertrain/challenge/admission_store.py b/src/hypertrain/challenge/admission_store.py new file mode 100644 index 0000000..fa58d25 --- /dev/null +++ b/src/hypertrain/challenge/admission_store.py @@ -0,0 +1,215 @@ +"""Shared-connection admission persistence, durable reservations and escrow outbox.""" + +from __future__ import annotations + +import sqlite3 +from dataclasses import dataclass + +from pydantic import TypeAdapter + +from hypertrain.ledger.escrow_v2 import EscrowError, EscrowV2 +from hypertrain.protocol.messages_v2 import AdmissionState, EscrowLock, EscrowReceipt, Hex64 + + +class AdmissionError(ValueError): + def __init__(self, code: str) -> None: + self.code = code + super().__init__(code) + + +@dataclass(frozen=True, slots=True) +class AdmissionRecord: + admission_id: str + hotkey: str + coldkey: str + state: AdmissionState + clean_count: int + canary_blocks: tuple[int, ...] + strikes: int + pending_dispute: bool + reason: str + policy_hash: str + screen_until: int + challenge_json: str | None + + +class AdmissionStore: + """Use the same SQLite connection as EscrowV2/L0, never independent eligibility copies.""" + + def __init__(self, escrow: EscrowV2) -> None: + self.escrow, self.db = escrow, escrow.db + self.db.executescript(""" + CREATE TABLE IF NOT EXISTS admissions_v2( + admission_id TEXT PRIMARY KEY, run_id TEXT, hotkey TEXT UNIQUE, coldkey TEXT, + state TEXT, clean_count INTEGER DEFAULT 0, canary_blocks TEXT DEFAULT '', + strikes INTEGER DEFAULT 0, pending_dispute INTEGER DEFAULT 0, reason TEXT, + policy_hash TEXT, received_beacon INTEGER, seed_beacon INTEGER, + screen_until INTEGER DEFAULT 0, challenge TEXT, proof_digest TEXT, + reference_commitment TEXT, reference_json TEXT, reference_beacon INTEGER, + trial_epoch INTEGER, screen_json TEXT); + CREATE TABLE IF NOT EXISTS admission_reservations( + reservation TEXT PRIMARY KEY, digest TEXT NOT NULL, receipt TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS admission_quota( + identity TEXT PRIMARY KEY, beacon INTEGER, tokens INTEGER); + CREATE TABLE IF NOT EXISTS admission_history( + hotkey TEXT PRIMARY KEY, admission_id TEXT, coldkey TEXT); + CREATE TABLE IF NOT EXISTS admission_trials( + epoch INTEGER PRIMARY KEY, admission_id TEXT, nonce TEXT, received_beacon INTEGER, + finalized_beacon INTEGER, evidence_hash TEXT); + CREATE TABLE IF NOT EXISTS admission_trial_results( + epoch INTEGER PRIMARY KEY, challenge TEXT NOT NULL, reference TEXT, + proof TEXT, screen TEXT, outcome TEXT NOT NULL DEFAULT 'OPEN'); + CREATE TABLE IF NOT EXISTS admission_outbox( + operation_id TEXT PRIMARY KEY, request TEXT NOT NULL, signer TEXT NOT NULL, + receipt TEXT); + CREATE TABLE IF NOT EXISTS admission_transitions( + seq INTEGER PRIMARY KEY, admission_id TEXT, state TEXT, reason TEXT, beacon INTEGER); + CREATE TABLE IF NOT EXISTS admission_pending( + dispute_id TEXT PRIMARY KEY, admission_id TEXT NOT NULL, evidence_hash TEXT NOT NULL, + coldkey TEXT NOT NULL, resolved INTEGER NOT NULL DEFAULT 0); + """) + + def record(self, hotkey: str) -> AdmissionRecord: + row = self.db.execute("SELECT * FROM admissions_v2 WHERE hotkey=?", (hotkey,)).fetchone() + if row is None: + raise AdmissionError("UNKNOWN_HOTKEY") + return self._record(row) + + def by_id(self, admission_id: str) -> AdmissionRecord: + row = self.db.execute( + "SELECT * FROM admissions_v2 WHERE admission_id=?", (admission_id,) + ).fetchone() + if row is None: + raise AdmissionError("UNKNOWN_ADMISSION") + return self._record(row) + + @staticmethod + def _record(row: sqlite3.Row) -> AdmissionRecord: + state: AdmissionState + match row["state"]: + case "APPLIED": + state = "APPLIED" + case "PROBATION": + state = "PROBATION" + case "ACTIVE": + state = "ACTIVE" + case "SUSPENDED": + state = "SUSPENDED" + case "BANNED": + state = "BANNED" + case _: + raise AdmissionError("CORRUPT_STATE") + return AdmissionRecord( + row["admission_id"], + row["hotkey"], + row["coldkey"], + state, + row["clean_count"], + tuple(int(i) for i in row["canary_blocks"].split(",") if i), + row["strikes"], + bool(row["pending_dispute"]), + row["reason"], + row["policy_hash"], + row["screen_until"], + row["challenge"], + ) + + def reserve(self, key: str, digest: str, receipt: str) -> str: + row = self.db.execute( + "SELECT * FROM admission_reservations WHERE reservation=?", (key,) + ).fetchone() + if row: + if row["digest"] != digest: + raise AdmissionError("REPLAY_CONFLICT") + return str(row["receipt"]) + self.db.execute("INSERT INTO admission_reservations VALUES(?,?,?)", (key, digest, receipt)) + return receipt + + def quota(self, identity: str, now: int, rate: int, burst: int) -> None: + row = self.db.execute( + "SELECT * FROM admission_quota WHERE identity=?", (identity,) + ).fetchone() + if row is not None and now < row["beacon"]: + raise AdmissionError("BEACON_REGRESSION") + tokens = min(burst, row["tokens"] + (now - row["beacon"]) * rate) if row else burst + if not tokens: + raise AdmissionError("JOIN_QUOTA") + self.db.execute( + "INSERT INTO admission_quota VALUES(?,?,?) ON CONFLICT(identity) DO " + "UPDATE SET beacon=excluded.beacon,tokens=excluded.tokens", + (identity, now, tokens - 1), + ) + + def transition( + self, admission_id: str, state: AdmissionState, reason: str, beacon: int + ) -> None: + self.db.execute( + "UPDATE admissions_v2 SET state=?,reason=? WHERE admission_id=?", + (state, reason, admission_id), + ) + self.db.execute( + "INSERT INTO admission_transitions(admission_id,state,reason,beacon) VALUES(?,?,?,?)", + (admission_id, state, reason, beacon), + ) + + def bind_pending(self, record: AdmissionRecord, dispute_id: str, evidence_hash: str) -> None: + """Bind current evidence; settled IDs cannot authorize later incidents.""" + TypeAdapter(Hex64).validate_python(dispute_id) + TypeAdapter(Hex64).validate_python(evidence_hash) + existing = self.db.execute( + "SELECT * FROM admission_pending WHERE dispute_id=?", (dispute_id,) + ).fetchone() + if existing is not None: + if ( + existing["admission_id"], + existing["coldkey"], + existing["evidence_hash"], + existing["resolved"], + ) != (record.admission_id, record.coldkey, evidence_hash, 0): + raise AdmissionError("PENDING_REFERENCE_CONFLICT") + return + current = self.db.execute( + "SELECT 1 FROM admission_pending WHERE admission_id=? AND resolved=0", + (record.admission_id,), + ).fetchone() + if current is not None: + raise AdmissionError("CURRENT_PENDING_REFERENCE_EXISTS") + self.db.execute( + "INSERT INTO admission_pending VALUES(?,?,?,?,0)", + (dispute_id, record.admission_id, evidence_hash, record.coldkey), + ) + + def enqueue_lock(self, operation: EscrowLock, signer: str) -> None: + with self.escrow.tx(): + row = self.db.execute( + "SELECT * FROM admission_outbox WHERE operation_id=?", (operation.operation_id,) + ).fetchone() + if row and (row["request"] != operation.model_dump_json() or row["signer"] != signer): + raise EscrowError("OUTBOX_CONFLICT") + self.db.execute( + "INSERT OR IGNORE INTO admission_outbox(operation_id,request,signer) VALUES(?,?,?)", + (operation.operation_id, operation.model_dump_json(), signer), + ) + + def reconcile_lock(self, operation_id: str) -> EscrowReceipt: + """Crash after ledger commit/before receipt acknowledgment is exactly-once.""" + row = self.db.execute( + "SELECT * FROM admission_outbox WHERE operation_id=?", (operation_id,) + ).fetchone() + if row is None: + raise AdmissionError("UNKNOWN_OUTBOX") + receipt = self.escrow.lock( + EscrowLock.model_validate_json(row["request"]), signer=row["signer"] + ) + with self.escrow.tx(): + self.db.execute( + "UPDATE admission_outbox SET receipt=? WHERE operation_id=?", + (receipt.model_dump_json(), operation_id), + ) + return receipt + + def pending_outbox(self, admission_id: str) -> bool: + for row in self.db.execute("SELECT request FROM admission_outbox WHERE receipt IS NULL"): + if EscrowLock.model_validate_json(row["request"]).admission_id == admission_id: + return True + return False diff --git a/src/hypertrain/challenge/app.py b/src/hypertrain/challenge/app.py index a7898cb..fe55eca 100644 --- a/src/hypertrain/challenge/app.py +++ b/src/hypertrain/challenge/app.py @@ -16,13 +16,26 @@ import httpx from fastapi import FastAPI, Header, Query, Request from fastapi.responses import JSONResponse, Response -from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictInt, ValidationError +from pydantic import ( + BaseModel, + ConfigDict, + Field, + StrictBool, + StrictInt, + ValidationError, +) from hypertrain.beacon import BeaconRound, parse_round +from hypertrain.challenge.admission_store import AdmissionError +from hypertrain.challenge.disputes_v2 import DisputeError, EventAck from hypertrain.challenge.store import ChallengeError, ChallengeStore from hypertrain.data.store import LocalFSStore from hypertrain.data.store import Store as ObjectStore from hypertrain.ledger import Params, vest_rounds_for_q +from hypertrain.ledger.escrow_v2 import EscrowError +from hypertrain.protocol.envelope import EnvelopeError +from hypertrain.protocol.envelope_v2 import load_json +from hypertrain.protocol.jcs import canonicalize from hypertrain.protocol.keys import KeyError_, Keypair, decode_hotkey from hypertrain.protocol.messages import QUICKNET_GENESIS from hypertrain.public_api import create_public_app @@ -167,7 +180,8 @@ def _require(path: Path | None, authorization: str | None) -> None: raise ChallengeError(401, "unauthorized") presented = authorization.removeprefix("Bearer ").strip() if not hmac.compare_digest( - hashlib.sha256(presented.encode()).digest(), hashlib.sha256(expected.encode()).digest() + hashlib.sha256(presented.encode()).digest(), + hashlib.sha256(expected.encode()).digest(), ): raise ChallengeError(401, "unauthorized") @@ -257,15 +271,33 @@ def create_app( transport: httpx.AsyncBaseTransport | None = None, verify_beacon: Callable[[Mapping[str, Any]], BeaconRound] = parse_round, objects: ObjectStore | None = None, + _store: ChallengeStore | None = None, ) -> FastAPI: - store = ChallengeStore( - config.state_dir, - config.params, - load_coord_key(config.coord_key_file), - config.owner_hotkey, - verify_beacon, - objects or LocalFSStore(config.state_dir / "objects"), + if _store is not None and ( + _store.state_dir.resolve() != config.state_dir.resolve() + or _store.owner_hotkey != config.owner_hotkey + or _store.params != config.params + or _store.coord is None + or config.coord_key_file is None + or (configured_coord := load_coord_key(config.coord_key_file)) is None + or _store.coord.ss58 != configured_coord.ss58 + or objects is not None + and _store.objects is not objects + ): + raise ValueError("trusted continuation store/config differs") + store = ( + _store + if _store is not None + else ChallengeStore( + config.state_dir, + config.params, + load_coord_key(config.coord_key_file), + config.owner_hotkey, + verify_beacon, + objects or LocalFSStore(config.state_dir / "objects"), + ) ) + store.clock = clock client = httpx.AsyncClient(transport=transport) metagraph = Metagraph(config.master_url, client) app = FastAPI( @@ -277,6 +309,373 @@ def create_app( ) app.state.store = store + @app.on_event("shutdown") + def close_notifications() -> None: + store.close_v2_notifications() + + async def v2_raw(request: Request) -> bytes: + data = bytearray() + async for chunk in request.stream(): + data.extend(chunk) + if len(data) > BODY_MAX + 1024: + raise ChallengeError(413, "v2 body exceeds limit") + load_json(bytes(data), max_bytes=BODY_MAX + 1024) + return bytes(data) + + async def v2_error(_: Request, error: Exception) -> JSONResponse: + return JSONResponse({"detail": str(error)}, status_code=409) + + for cls in ( + AdmissionError, + DisputeError, + EscrowError, + EnvelopeError, + ValidationError, + ): + app.add_exception_handler(cls, v2_error) + + @app.post("/v2/admin/runs", status_code=201) + async def create_v2(request: Request, authorization: Auth = None) -> Any: + admin(authorization) + return await run(store.create_run_v2, await v2_raw(request)) + + @app.get("/v2/runs/{run_id}") + async def status_v2(run_id: str) -> Any: + return await run(store.run_status_v2, run_id) + + @app.post("/v2/runs/{run_id}/join") + async def join_v2(run_id: str, request: Request) -> Any: + import ipaddress + + address = request.client.host if request.client else "127.0.0.1" + try: + ip = ipaddress.ip_address(address) + prefix = str( + ipaddress.ip_network(f"{ip}/{24 if ip.version == 4 else 64}", strict=False) + ) + except ValueError: + prefix = "local-transport" + return await run(store.admission_v2, run_id, "join", await v2_raw(request), "", prefix) + + @app.get("/v2/runs/{run_id}/join/{admission_id}/challenge") + async def challenge_v2(run_id: str, admission_id: str) -> Any: + return await run(store.admission_v2, run_id, "challenge", b"", admission_id) + + @app.get("/v2/runs/{run_id}/admission/{hotkey}") + async def admission_status_v2(run_id: str, hotkey: str) -> Any: + return await run(store.admission_v2, run_id, "status", b"", hotkey) + + @app.post("/v2/runs/{run_id}/join/proof") + async def proof_v2(run_id: str, request: Request) -> Any: + data = load_json(await v2_raw(request)) + if not isinstance(data, dict) or set(data) != {"proof", "screen"}: + raise ChallengeError(422, "expected original signed proof and screen") + return await run( + store.trial_proof_v2, + run_id, + canonicalize(data["proof"]), + canonicalize(data["screen"]), + ) + + @app.post("/v2/runs/{run_id}/admin/join/{admission_id}/reference") + async def reference_v2(run_id: str, admission_id: str, authorization: Auth = None) -> Any: + admin(authorization) + return await run(store.trial_reference_v2, run_id, admission_id) + + @app.post("/v2/runs/{run_id}/admin/join/{admission_id}/finalize") + async def trial_final_v2( + run_id: str, admission_id: str, request: Request, authorization: Auth = None + ) -> Any: + admin(authorization) + return await run(store.trial_finalize_v2, run_id, admission_id, await v2_raw(request)) + + @app.post("/v2/runs/{run_id}/admin/join/{admission_id}/shadow-reservation") + async def shadow_reservation_v2( + run_id: str, admission_id: str, request: Request, authorization: Auth = None + ) -> Any: + admin(authorization) + return await run(store.shadow_reservation_v2, run_id, admission_id, await v2_raw(request)) + + @app.post("/v2/runs/{run_id}/admin/join/{admission_id}/shadow-reward") + async def shadow_reward_v2( + run_id: str, admission_id: str, request: Request, authorization: Auth = None + ) -> Any: + admin(authorization) + return await run(store.shadow_reward_v2, run_id, admission_id, await v2_raw(request)) + + @app.post("/v2/runs/{run_id}/admission/{hotkey}/rotate") + async def rotate_v2(run_id: str, hotkey: str, request: Request) -> Any: + raw = await v2_raw(request) + from hypertrain.protocol.envelope_v2 import parse_envelope + + if parse_envelope(raw).body.get("hotkey") != hotkey: + raise ChallengeError(422, "rotation path and body differ") + return await run(store.admission_v2, run_id, "rotate", raw) + + def v2_operation(path: str, action: str) -> None: + async def handler(run_id: str, request: Request) -> Any: + return await run(store.admission_v2, run_id, action, await v2_raw(request)) + + app.post("/v2/runs/{run_id}/" + path, name="v2_" + action)(handler) + + for path, action in ( + ("escrow/lock", "lock"), + ("escrow/transfer", "transfer"), + ("escrow/release", "release"), + ("dispute", "dispute"), + ("bisect", "bisect"), + ("resolution", "resolution"), + ("state-serve", "state-serve"), + ("admission/recover", "recover"), + ): + v2_operation(path, action) + + @app.put("/v2/runs/{run_id}/admin/dataset") + async def dataset_v2(run_id: str, request: Request, authorization: Auth = None) -> Any: + admin(authorization) + data = load_json(await v2_raw(request)) + if not isinstance(data, dict): + raise ChallengeError(422, "expected dataset object linkage") + return await run(store.configure_inputs_v2, run_id, data) + + @app.post("/v2/runs/{run_id}/admin/genesis") + async def genesis_v2(run_id: str, request: Request, authorization: Auth = None) -> Any: + admin(authorization) + return await run(store.genesis_v2, run_id, await v2_raw(request)) + + @app.post("/v2/runs/{run_id}/admin/rounds") + async def open_v2(run_id: str, request: Request, authorization: Auth = None) -> Any: + admin(authorization) + return await run(store.open_round_v2, run_id, await v2_raw(request)) + + @app.put("/v2/runs/{run_id}/objects/{sha256}") + async def put_object_v2( + run_id: str, + sha256: str, + request: Request, + x_hotkey: Annotated[str, Header()], + x_object_signature: Annotated[str, Header()], + ) -> Any: + from hypertrain.protocol.keys import verify + + with store._lock: + store._run_v2(run_id) + store._owner_v2(run_id, x_hotkey) + try: + valid = verify( + decode_hotkey(x_hotkey), + f"hypertrain/object/2|{run_id}|{sha256}".encode(), + bytes.fromhex(x_object_signature), + ) + except (ValueError, KeyError_): + valid = False + if not valid: + raise ChallengeError(403, "object signature mismatch") + # Admission objects are separately bounded by the pinned policy. + raw = bytearray() + limit = store._services(run_id)[1].policy.artifact_limits.max_object_bytes + async for chunk in request.stream(): + raw.extend(chunk) + if len(raw) > limit: + raise ChallengeError(413, "object exceeds pinned admission limit") + if hashlib.sha256(raw).hexdigest() != sha256: + raise ChallengeError(422, "object hash differs") + key = await run(store.objects.put, bytes(raw)) + return {"sha256": key} + + @app.get("/v2/runs/{run_id}/rounds/{w}") + async def round_v2(run_id: str, w: int) -> Any: + return await run(store.round_view_v2, run_id, w) + + @app.get("/v2/runs/{run_id}/rounds/{w}/job/{hotkey}") + async def job_v2(run_id: str, w: int, hotkey: str) -> Any: + return await run(store.island_job_v2, run_id, w, hotkey) + + @app.get("/v2/runs/{run_id}/objects/{sha256}") + async def object_v2(run_id: str, sha256: str) -> Response: + store._run_v2(run_id) + # Published run datasets/state are public, content identity still verifies. + return Response(await run(store.objects.get, sha256), media_type="application/octet-stream") + + for action in ("accept", "commit", "delta"): + + def add_training(action: str) -> None: + async def handler(run_id: str, request: Request) -> Any: + return await run(store.training_v2, run_id, action, await v2_raw(request)) + + app.post("/v2/runs/{run_id}/" + action, name="v2_" + action)(handler) + + add_training(action) + + @app.post("/v2/runs/{run_id}/leaves") + async def leaves_v2(run_id: str, request: Request) -> Any: + return await run(store.leaves_v2, run_id, await v2_raw(request)) + + @app.post("/v2/runs/{run_id}/admin/rounds/{w}/audits") + async def audits_v2(run_id: str, w: int, authorization: Auth = None) -> Any: + admin(authorization) + return {"jobs": await run(store.schedule_audits_v2, run_id, w)} + + @app.post("/v2/runs/{run_id}/worker/lease") + async def lease_v2(run_id: str, request: Request) -> Any: + job = await run(store.lease_v2, run_id, await v2_raw(request)) + return Response(status_code=204) if job is None else job + + @app.post("/v2/runs/{run_id}/worker/jobs/{job_id}/execute") + async def execute_v2(run_id: str, job_id: str, request: Request) -> Any: + return await run(store.execute_audit_v2, run_id, job_id, await v2_raw(request)) + + @app.post("/v2/runs/{run_id}/worker/jobs/{job_id}/complete") + async def complete_v2(run_id: str, job_id: str, request: Request) -> Any: + return await run(store.complete_audit_v2, run_id, job_id, await v2_raw(request)) + + @app.post("/v2/runs/{run_id}/admin/rounds/{w}/aggregate") + async def aggregate_v2(run_id: str, w: int, authorization: Auth = None) -> Any: + admin(authorization) + return await run(store.aggregate_v2, run_id, w) + + @app.post("/v2/runs/{run_id}/admin/rounds/{w}/finalize") + async def final_v2(run_id: str, w: int, request: Request, authorization: Auth = None) -> Any: + admin(authorization) + return await run(store.finalize_v2, run_id, w, await v2_raw(request)) + + @app.get("/v2/runs/{run_id}/rounds/{w}/relay/{hotkey}") + async def assignment_v2(run_id: str, w: int, hotkey: str) -> Any: + return await run(store.relay_assignment_v2, run_id, w, hotkey) + + @app.post("/v2/runs/{run_id}/upload-grant") + async def grant_v2(run_id: str, request: Request, commit_hash: str | None = None) -> Any: + return await run(store.upload_grant_v2, run_id, await v2_raw(request), commit_hash) + + @app.post("/v2/relay-receipts") + async def receipt_v2(request: Request) -> Any: + async with httpx.AsyncClient(follow_redirects=False, timeout=30) as transport: + return await store.relay_receipt_v2(await v2_raw(request), transport) + + @app.get("/v2/runs/{run_id}/admin/relay-settlement/{grant_hash}") + async def relay_settlement_v2(run_id: str, grant_hash: str, authorization: Auth = None) -> Any: + admin(authorization) + return await run(store.relay_settlement_v2, run_id, grant_hash) + + @app.post("/v2/runs/{run_id}/admin/disputes/{dispute_id}/state/{checkpoint}") + async def dispute_state_v2( + run_id: str, dispute_id: str, checkpoint: int, authorization: Auth = None + ) -> Any: + admin(authorization) + return await run(store.dispute_state_v2, run_id, dispute_id, checkpoint) + + @app.post("/v2/runs/{run_id}/admin/rollback") + async def rollback_v2(run_id: str, request: Request, authorization: Auth = None) -> Any: + from hypertrain.aggregator.core import TapeError + + admin(authorization) + try: + return await run(store.rollback_v2, run_id, await v2_raw(request)) + except TapeError as error: + raise ChallengeError(409, str(error)) from error + + @app.post("/v2/runs/{run_id}/admin/rollback-preview") + async def rollback_preview_v2(run_id: str, request: Request, authorization: Auth = None) -> Any: + from hypertrain.aggregator.core import TapeError + + admin(authorization) + try: + return await run(store.rollback_preview_v2, run_id, await v2_raw(request)) + except TapeError as error: + raise ChallengeError(409, str(error)) from error + + @app.post("/v2/runs/{run_id}/admin/disputes/{dispute_id}/referee") + async def referee_v2(run_id: str, dispute_id: str, authorization: Auth = None) -> Any: + admin(authorization) + return await run(store.referee_v2, run_id, dispute_id) + + @app.post("/v2/runs/{run_id}/admin/disputes/{dispute_id}/timeout") + async def timeout_v2( + run_id: str, dispute_id: str, request: Request, authorization: Auth = None + ) -> Any: + admin(authorization) + return await run(store.dispute_timeout_v2, run_id, dispute_id, await v2_raw(request)) + + @app.post("/v2/runs/{run_id}/admin/relay-registry") + async def registry_v2(run_id: str, request: Request, authorization: Auth = None) -> Any: + admin(authorization) + return await run(store.rotate_registry_v2, run_id, await v2_raw(request)) + + @app.post("/v2/runs/{run_id}/admin/relay/{grant_hash}/{action}") + async def relay_control_v2( + run_id: str, grant_hash: str, action: str, authorization: Auth = None + ) -> Any: + admin(authorization) + async with httpx.AsyncClient(follow_redirects=False, timeout=30) as client: + return await store.relay_control_v2(run_id, grant_hash, action, client) + + @app.post("/v2/runs/{run_id}/admin/relay-failure") + async def failure_v2(run_id: str, request: Request, authorization: Auth = None) -> Any: + admin(authorization) + return await run(store.relay_failure_v2, run_id, await v2_raw(request)) + + @app.get("/v2/runs/{run_id}/relay-observers/{grant_hash}/{observer}") + async def observer_challenge_v2(run_id: str, grant_hash: str, observer: str) -> Any: + return await run(store.relay_observer_v2, run_id, grant_hash, observer) + + @app.post("/v2/runs/{run_id}/relay-observers/{grant_hash}/{observer}") + async def observer_response_v2( + run_id: str, grant_hash: str, observer: str, request: Request + ) -> Any: + return await run( + store.relay_observer_v2, run_id, grant_hash, observer, await v2_raw(request) + ) + + @app.post("/v2/runs/{run_id}/admin/relay-fallback/{failure_hash}") + async def fallback_v2(run_id: str, failure_hash: str, authorization: Auth = None) -> Any: + admin(authorization) + async with httpx.AsyncClient(follow_redirects=False, timeout=30) as client: + return await store.relay_fallback_v2(run_id, failure_hash, client) + + @app.post("/v2/runs/{run_id}/admin/relay-drain/{relay_id}") + async def drain_v2(run_id: str, relay_id: str, authorization: Auth = None) -> Any: + admin(authorization) + return await run(store.relay_drain_v2, run_id, relay_id) + + @app.get("/v2/runs/{run_id}/disputes") + async def events_v2( + run_id: str, + party: str, + cursor: int = 0, + wait_seconds: Annotated[float, Query(alias="timeout")] = 0, + x_dispute_signature: Annotated[str | None, Header()] = None, + ) -> Any: + from hypertrain.protocol.keys import verify + + if cursor < 0 or not 0 <= wait_seconds <= 30: + raise ChallengeError(422, "invalid event cursor or wait") + try: + valid = verify( + decode_hotkey(party), + f"hypertrain/watch/2|{run_id}|{party}".encode(), + bytes.fromhex(x_dispute_signature or ""), + ) + except (ValueError, KeyError_): + valid = False + if not valid: + raise ChallengeError(403, "party event access requires signature") + _, _, disputes = await run(store._services, run_id) + events = await run(lambda: disputes.events(party, cursor, timeout=wait_seconds)) + return [e.model_dump(mode="json") for e in events] + + @app.post("/v2/runs/{run_id}/disputes/ack") + async def ack_v2(run_id: str, request: Request) -> Any: + ack = EventAck.model_validate(load_json(await v2_raw(request))) + _, _, disputes = await run(store._services, run_id) + + def accepted_ack() -> None: + with store._tx(): + store._snapshot_current_v2(store._run_v2(run_id)) + disputes.acknowledge(ack, now=store._now(store._db)) + + await run(accepted_ack) + return {"accepted": ack.cursor} + def run(function: Callable[..., Any], *args: Any) -> Awaitable[Any]: return asyncio.to_thread(function, *args) @@ -520,7 +919,8 @@ async def handler(run_id: str, w: int, request: Request, authorization: Auth = N app.mount( "/public", create_public_app( - config.state_dir / "challenge.db", config.state_dir / "public" / "metrics.jsonl" + config.state_dir / "challenge.db", + config.state_dir / "public" / "metrics.jsonl", ), ) return app diff --git a/src/hypertrain/challenge/disputes_v2.py b/src/hypertrain/challenge/disputes_v2.py new file mode 100644 index 0000000..2dbf964 --- /dev/null +++ b/src/hypertrain/challenge/disputes_v2.py @@ -0,0 +1,901 @@ +"""Durable funded STEP/LAYER/OP progression on L2's shared transaction boundary. + +SIZE_OK: one indivisible dispute state machine; transaction, event and evidence +updates must commit together within the seven-path lane ownership boundary. +""" + +from __future__ import annotations + +import math +import threading +from collections.abc import Callable +from typing import Literal + +from pydantic import Field, JsonValue, StrictBool, StrictInt + +from hypertrain.auditor.island_bisect import RefereeEvidence +from hypertrain.ledger.escrow_v2 import EscrowV2, Settlement +from hypertrain.protocol.envelope_v2 import Intake, parse_envelope +from hypertrain.protocol.hashing import sha256_hex +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.keys import Keypair, decode_hotkey, verify +from hypertrain.protocol.messages import SS58, Hex64, StateServe +from hypertrain.protocol.messages_v2 import ( + BisectV2, + DisputePolicyV2, + DisputeV2, + EscrowLock, + EscrowRelease, + ResolutionV2, + WireModel, +) + + +class DisputeError(ValueError): + def __init__(self, code: str) -> None: + self.code = code + super().__init__(code) + + +class Contest(WireModel): + """L0 derives this from an authorized accepted verdict, never an HTTP body.""" + + verdict_hash: Hex64 + challenge_hash: Hex64 + w: StrictInt = Field(ge=0) + miner: SS58 + auditor: SS58 + referee: SS58 + step_span: StrictInt = Field(ge=1) + layer_span: StrictInt = Field(ge=1) + op_span: StrictInt = Field(ge=1) + admission_id: Hex64 | None = None + coldkey: SS58 | None = None + evidence_hash: Hex64 | None = None + auditor_coldkey: SS58 | None = None + + +class DisputeSettlement(Settlement): + """Internal accepted-resolution authority, scoped to one actual funded lock. + + outcome describes this lock's disposition, not guilt of another identity. + Unknown auditor ownership remains None; no synthetic penalty account. + """ + + loser: SS58 | None + loser_coldkey: SS58 | None + referee: SS58 + lock_id: Hex64 + lock_owner: SS58 + resolution_hash: Hex64 | None + + +class Turn(WireModel): + run_id: Hex64 + dispute_id: Hex64 + contest: Contest + seq: StrictInt = Field(ge=0, le=256) + level: Literal["step", "layer", "op"] + ctx: list[StrictInt] + interval: tuple[StrictInt, StrictInt] + expected_party: SS58 + transcript_hash: Hex64 + turn_deadline: StrictInt + absolute_deadline: StrictInt + lock_id: Hex64 + paused: StrictBool = False + reassignments: StrictInt = 0 + resolution: ResolutionV2 | None = None + resolved_beacon: StrictInt | None = None + + +class WatchEvent(WireModel): + run_id: Hex64 + cursor: StrictInt = Field(ge=1) + party: SS58 + kind: Literal["turn", "state"] + turn: Turn + checkpoint: StrictInt | None + issued_beacon: StrictInt = Field(ge=1) + signer: SS58 + sig: str + + +class EventAck(WireModel): + run_id: Hex64 + cursor: StrictInt = Field(ge=1) + event_hash: Hex64 + party: SS58 + received_beacon: StrictInt = Field(ge=1) + sig: str + + +class Availability(WireModel): + run_id: Hex64 + dispute_id: Hex64 + seq: StrictInt + transcript_hash: Hex64 + observed_beacon: StrictInt + service_healthy: StrictBool + beacon_healthy: StrictBool + reference_healthy: StrictBool + signer: SS58 + sig: str + + +def signed_message(model: WatchEvent | EventAck | Availability) -> bytes: + return ( + b"hypertrain/dispute/2|" + + type(model).__name__.encode() + + b"|" + + canonicalize( + model.model_dump(mode="json", exclude={"sig"}), + ) + ) + + +def authentic(model: WatchEvent | EventAck | Availability) -> bool: + signer = model.party if isinstance(model, EventAck) else model.signer + try: + signature = bytes.fromhex(model.sig) + except ValueError: + return False + return verify(decode_hotkey(signer), signed_message(model), signature) + + +class DisputesV2: + """Mutable service; shared SQLite plus condition subscription, no training dependency. + + L0 supplies authoritative verdict/identity lookup, qualified span bounds and + independent local replay evidence. Cross-process route notification calls + notify() after the shared transaction commits; the durable event table is truth. + """ + + def __init__( + self, + escrow: EscrowV2, + policy: DisputePolicyV2, + coordinator: Keypair, + *, + contest_of: Callable[[str], Contest], + owner_of: Callable[[str], str], + span: Callable[[Contest, str, tuple[int, ...]], int], + ) -> None: + if coordinator.ss58 != escrow.manifest.training.coord_pubkey: + raise DisputeError("COORDINATOR_AUTHORITY") + if policy.digest() != escrow.manifest.network.dispute_policy_hash: + raise DisputeError("DISPUTE_POLICY_HASH") + self.escrow, self.policy, self.key = escrow, policy, coordinator + self.run_id, self.db = escrow.run_id, escrow.db + self.contest_of, self.owner_of, self.span = contest_of, owner_of, span + self.condition = threading.Condition() + with escrow.tx(): + for sql in ( + "CREATE TABLE IF NOT EXISTS disputes_v2(id TEXT PRIMARY KEY,verdict TEXT UNIQUE," + "miner TEXT,turn TEXT,request TEXT)", + "CREATE TABLE IF NOT EXISTS dispute_entries_v2(id TEXT,seq INTEGER," + "body TEXT,envelope BLOB,PRIMARY KEY(id,seq))", + "CREATE TABLE IF NOT EXISTS dispute_events_v2(cursor INTEGER PRIMARY KEY " + "AUTOINCREMENT,party TEXT,event TEXT)", + "CREATE TABLE IF NOT EXISTS dispute_acks_v2(cursor INTEGER,party TEXT," + "ack TEXT,PRIMARY KEY(cursor,party))", + "CREATE TABLE IF NOT EXISTS dispute_evidence_v2(hash TEXT PRIMARY KEY,body TEXT)", + "CREATE TABLE IF NOT EXISTS dispute_states_v2(challenge TEXT,t INTEGER," + "party TEXT,body TEXT,envelope BLOB,PRIMARY KEY(challenge,t,party))", + "CREATE TABLE IF NOT EXISTS dispute_receipts_v2(id TEXT,seq INTEGER,turn TEXT," + "PRIMARY KEY(id,seq))", + ): + self.db.execute(sql) + + def notify(self) -> None: + with self.condition: + self.condition.notify_all() + + def get(self, dispute_id: str) -> Turn: + row = self.db.execute("SELECT turn FROM disputes_v2 WHERE id=?", (dispute_id,)).fetchone() + if row is None: + raise DisputeError("UNKNOWN_DISPUTE") + return Turn.model_validate_json(row[0]) + + def _save(self, turn: Turn) -> None: + self.db.execute( + "UPDATE disputes_v2 SET turn=? WHERE id=?", + ( + turn.model_dump_json(), + turn.dispute_id, + ), + ) + + def _event(self, turn: Turn, now: int, checkpoint: int | None = None) -> WatchEvent: + cursor = self.db.execute( + "INSERT INTO dispute_events_v2(party,event) VALUES(?,?)", + ( + turn.expected_party, + "", + ), + ).lastrowid + assert cursor is not None + event = WatchEvent( + run_id=self.run_id, + cursor=cursor, + party=turn.expected_party, + kind="turn" if checkpoint is None else "state", + turn=turn, + checkpoint=checkpoint, + issued_beacon=now, + signer=self.key.ss58, + sig="", + ) + event = event.model_copy(update={"sig": self.key.sign(signed_message(event)).hex()}) + self.db.execute( + "UPDATE dispute_events_v2 SET event=? WHERE cursor=?", + ( + event.model_dump_json(), + cursor, + ), + ) + return event + + def open(self, raw: bytes, *, now: int) -> Turn: + env = parse_envelope(raw) + with self.escrow.tx(): + model = Intake(self.run_id, {"DisputeV2": lambda h: h == env.body["hotkey"]}).accept( + raw, + now, + ) + if not isinstance(model, DisputeV2) or model.action != "contest": + raise DisputeError("FUNDED_CONTEST_REQUIRED") + contest = self.contest_of(model.verdict_hash) + recovery = (contest.admission_id, contest.coldkey, contest.evidence_hash) + if any(value is not None for value in recovery) and not all( + value is not None for value in recovery + ): + raise DisputeError("INCOMPLETE_ACCEPTED_RECOVERY_LINKAGE") + if contest.coldkey is not None and contest.coldkey != self.owner_of(contest.miner): + raise DisputeError("ACCEPTED_RECOVERY_OWNER") + if contest.auditor_coldkey is not None and contest.auditor_coldkey != self.owner_of( + contest.auditor, + ): + raise DisputeError("ACCEPTED_AUDITOR_OWNER") + if ( + contest.step_span + != self.escrow.manifest.training.inner.H // self.escrow.manifest.training.inner.J + ): + raise DisputeError("CONTEST_WINDOW_GEOMETRY") + if contest.miner != env.signer or contest.auditor not in self.escrow.auditors: + raise DisputeError("CONTEST_IDENTITY") + if contest.referee not in self.policy.referees or contest.referee in ( + contest.miner, + contest.auditor, + self.key.ss58, + ): + raise DisputeError("INDEPENDENT_REFEREE_REQUIRED") + dispute_id = sha256_hex( + canonicalize([self.run_id, contest.verdict_hash, contest.miner]) + ) + old = self.db.execute( + "SELECT request FROM disputes_v2 WHERE id=?", (dispute_id,) + ).fetchone() + if old is not None: + if old[0] != model.model_dump_json(): + raise DisputeError("CONTEST_REPLAY_CONFLICT") + return self.get(dispute_id) + active = [ + Turn.model_validate_json(r[0]) + for r in self.db.execute( + "SELECT turn FROM disputes_v2", + ) + ] + if sum(t.resolution is None for t in active) >= self.policy.max_open or any( + t.resolution is None and t.contest.miner == contest.miner for t in active + ): + raise DisputeError("QUEUE_CAPACITY_EXCLUDE_WITHOUT_FRAUD") + lockrow = self.db.execute( + "SELECT kind,request FROM escrow_events WHERE operation_id=?", (model.bond_lock,) + ).fetchone() + if lockrow is None or lockrow[0] != "LOCK_CONTEST": + raise DisputeError("UNFUNDED_CONTEST") + lock = EscrowLock.model_validate_json(lockrow[1]) + amount = self.db.execute( + "SELECT COALESCE(SUM(units),0) FROM escrow_units WHERE " + "owner=? AND bucket='dispute_locked' AND ref=?", + (lock.owner, lock.operation_id), + ).fetchone()[0] + if (lock.owner, lock.dispute_id) != (self.owner_of(contest.miner), dispute_id) or ( + amount != lock.units or amount < self.policy.max_referee_cost_units + ): + raise DisputeError("CONTEST_LOCK_BINDING_OR_COST") + timeout = self.escrow.manifest.training.verify.T.dispute_per_level + budget = ( + 2 + * timeout + * ( + sum( + math.ceil(math.log2(n)) + for n in ( + contest.step_span, + contest.layer_span, + contest.op_span, + ) + ) + + 3 + ) + ) + turn = Turn( + run_id=self.run_id, + dispute_id=dispute_id, + contest=contest, + seq=0, + level="step", + ctx=[], + interval=(0, contest.step_span), + expected_party=contest.miner, + transcript_hash=model.digest(), + turn_deadline=now + timeout, + absolute_deadline=now + budget, + lock_id=lock.operation_id, + ) + self.db.execute( + "INSERT INTO disputes_v2 VALUES(?,?,?,?,?)", + ( + dispute_id, + contest.verdict_hash, + contest.miner, + turn.model_dump_json(), + model.model_dump_json(), + ), + ) + self._event(turn, now) + self.notify() + return turn + + def bisect(self, raw: bytes, *, now: int) -> Turn: + env = parse_envelope(raw) + with self.escrow.tx(): + model = Intake(self.run_id, {"BisectV2": lambda h: h == env.body["party"]}).accept( + raw, now + ) + if not isinstance(model, BisectV2): + raise DisputeError("BISECT_TYPE") + turn = self.get(model.dispute_id) + old = self.db.execute( + "SELECT body FROM dispute_entries_v2 WHERE id=? AND seq=?", + ( + model.dispute_id, + model.seq, + ), + ).fetchone() + if old is not None: + if old[0] != model.model_dump_json(): + raise DisputeError("SEQUENCE_REPLAY_CONFLICT") + receipt = self.db.execute( + "SELECT turn FROM dispute_receipts_v2 WHERE id=? AND seq=?", + (model.dispute_id, model.seq), + ).fetchone() + return Turn.model_validate_json(receipt[0]) + if ( + turn.resolution is not None + or turn.paused + or now + > min( + turn.turn_deadline, + turn.absolute_deadline, + ) + ): + raise DisputeError("CLOSED_PAUSED_OR_EXPIRED") + if ( + model.seq, + model.level, + model.ctx, + model.interval, + model.party, + model.previous_transcript_hash, + ) != ( + turn.seq, + turn.level, + turn.ctx, + turn.interval, + turn.expected_party, + turn.transcript_hash, + ): + raise DisputeError("TURN_SEQUENCE_CONTEXT_PARTY_OR_HASH") + at = [model.interval[0], -(-(sum(model.interval)) // 2), model.interval[1]] + if any( + a == b and left != right + for a, b, left, right in zip( + at[:-1], + at[1:], + model.hashes[:-1], + model.hashes[1:], + strict=True, + ) + ): + raise DisputeError("CONTRADICTORY_REPEATED_POINT") + transcript = sha256_hex( + bytes.fromhex(turn.transcript_hash) + canonicalize(model.body()) + ) + self.db.execute( + "INSERT INTO dispute_entries_v2 VALUES(?,?,?,?)", + ( + turn.dispute_id, + model.seq, + model.model_dump_json(), + raw, + ), + ) + update: dict[str, JsonValue] = dict(seq=turn.seq + 1, transcript_hash=transcript) + if model.party == turn.contest.miner: + update["expected_party"] = turn.contest.auditor + else: + first = BisectV2.model_validate_json( + self.db.execute( + "SELECT body FROM dispute_entries_v2 WHERE id=? AND seq=?", + (turn.dispute_id, model.seq - 1), + ).fetchone()[0] + ) + if first.hashes[0] != model.hashes[0]: + update["paused"] = True + else: + at = [turn.interval[0], -(-(sum(turn.interval)) // 2), turn.interval[1]] + i = next((i for i in (1, 2) if first.hashes[i] != model.hashes[i]), None) + if i is None: + update["paused"] = True # referee distinguishes MATCH from both-wrong + else: + lo, hi = at[i - 1], at[i] + if hi - lo == 1 and turn.level != "op": + level = "layer" if turn.level == "step" else "op" + ctx = [hi] if level == "layer" else [turn.ctx[0], lo] + update.update( + level=level, + ctx=[int(n) for n in ctx], + interval=[ + 0, + self.span( + turn.contest, + level, + tuple(ctx), + ), + ], + ) + else: + update["interval"] = [lo, hi] + update["paused"] = hi - lo == 1 + update["expected_party"] = turn.contest.miner + update["turn_deadline"] = min( + now + self.escrow.manifest.training.verify.T.dispute_per_level, + turn.absolute_deadline, + ) + next_turn = Turn.model_validate({**turn.body(), **update}) + if next_turn.interval[0] >= next_turn.interval[1]: + raise DisputeError("NONEMPTY_INTERVAL_REQUIRED") + if next_turn.seq >= self.policy.max_transcript_entries: + next_turn = next_turn.model_copy(update={"paused": True}) + self._save(next_turn) + self.db.execute( + "INSERT INTO dispute_receipts_v2 VALUES(?,?,?)", + ( + turn.dispute_id, + model.seq, + next_turn.model_dump_json(), + ), + ) + if not next_turn.paused and next_turn.seq < self.policy.max_transcript_entries: + self._event(next_turn, now) + self.notify() + return next_turn + + def events(self, party: str, cursor: int, *, timeout: float = 0) -> list[WatchEvent]: + if cursor < 0 or not 0 <= timeout <= 30: + raise DisputeError("CURSOR_OR_LONG_POLL_BOUND") + + def available() -> list[WatchEvent]: + with self.escrow.mutex: + return [ + WatchEvent.model_validate_json(r[0]) + for r in self.db.execute( + "SELECT event FROM dispute_events_v2 WHERE party=? AND cursor>? " + "ORDER BY cursor LIMIT 64", + (party, cursor), + ) + ] + + with self.condition: + self.condition.wait_for(lambda: bool(available()), timeout=timeout) + return available() + + def acknowledge(self, ack: EventAck, *, now: int) -> None: + if not authentic(ack) or ack.run_id != self.run_id or ack.received_beacon > now: + raise DisputeError("ACK_AUTHORITY_OR_TIME") + with self.escrow.tx(): + row = self.db.execute( + "SELECT event FROM dispute_events_v2 WHERE cursor=? AND party=?", + ( + ack.cursor, + ack.party, + ), + ).fetchone() + if row is None: + raise DisputeError("ACK_UNKNOWN_EVENT") + event = WatchEvent.model_validate_json(row[0]) + if ( + ack.event_hash != event.digest() + or not event.issued_beacon + <= ack.received_beacon + <= min( + event.turn.turn_deadline, + event.turn.absolute_deadline, + ) + ): + raise DisputeError("ACK_EVENT_OR_DEADLINE") + old = self.db.execute( + "SELECT ack FROM dispute_acks_v2 WHERE cursor=? AND party=?", + ( + ack.cursor, + ack.party, + ), + ).fetchone() + if ( + old is not None + and EventAck.model_validate_json(old[0]).event_hash != ack.event_hash + ): + raise DisputeError("ACK_CONFLICT") + self.db.execute( + "INSERT OR IGNORE INTO dispute_acks_v2 VALUES(?,?,?)", + ( + ack.cursor, + ack.party, + ack.model_dump_json(), + ), + ) + + def state_request(self, dispute_id: str, checkpoint: int, *, now: int) -> WatchEvent: + with self.escrow.tx(): + turn = self.get(dispute_id) + if ( + turn.resolution is not None + or now > turn.absolute_deadline + or not (0 <= checkpoint <= turn.contest.step_span) + ): + raise DisputeError("STATE_REQUEST_CONTEXT") + event = self._event( + turn.model_copy(update={"expected_party": turn.contest.miner}), now, checkpoint + ) + self.notify() + return event + + def reassign(self, dispute_id: str, referee: str, *, now: int) -> Turn: + """One persisted infrastructure retry, never extends the absolute budget.""" + with self.escrow.tx(): + turn = self.get(dispute_id) + if ( + not turn.paused + or turn.resolution is not None + or now >= turn.absolute_deadline + or turn.reassignments >= self.policy.max_referee_reassignments + or referee not in self.policy.referees + or referee + in ( + turn.contest.miner, + turn.contest.auditor, + turn.contest.referee, + self.key.ss58, + ) + ): + raise DisputeError("REFEREE_REASSIGNMENT_BOUND") + contest = turn.contest.model_copy(update={"referee": referee}) + turn = turn.model_copy(update={"contest": contest, "reassignments": 1}) + self._save(turn) + return turn + + def pause(self, dispute_id: str) -> Turn: + """Internal beacon/reference infrastructure freeze; immutable absolute horizon.""" + with self.escrow.tx(): + turn = self.get(dispute_id) + if turn.resolution is not None: + raise DisputeError("CLOSED_DISPUTE") + turn = turn.model_copy(update={"paused": True}) + self._save(turn) + return turn + + def register_evidence(self, evidence: RefereeEvidence) -> str: + """Internal independently anchored executor capability; NOT signed claimant intake.""" + with self.escrow.tx(): + turn = self.get(evidence.dispute_id) + if evidence.transcript_hash != turn.transcript_hash: + raise DisputeError("REFEREE_STALE_EVIDENCE") + self.db.execute( + "INSERT OR IGNORE INTO dispute_evidence_v2 VALUES(?,?)", + ( + evidence.digest(), + evidence.model_dump_json(), + ), + ) + return evidence.digest() + + def state_serve(self, raw: bytes, *, now: int) -> StateServe: + """Persist signed claims; independent replay verifies the served bytes.""" + env = parse_envelope(raw) + with self.escrow.tx(): + model = Intake( + self.run_id, + { + "StateServe": lambda h: h == env.body["hotkey"], + }, + ).accept(raw, now) + if not isinstance(model, StateServe): + raise DisputeError("STATE_SERVE_TYPE") + requests = [ + WatchEvent.model_validate_json(r[0]) + for r in self.db.execute( + "SELECT event FROM dispute_events_v2 WHERE party=?", + (env.signer,), + ) + ] + stride = self.escrow.manifest.training.inner.J + if not any( + e.kind == "state" + and e.checkpoint is not None + and ( + e.turn.contest.challenge_hash, + e.checkpoint * stride, + ) + == (model.challenge_hash, model.t) + and now <= e.turn.absolute_deadline + for e in requests + ): + raise DisputeError("UNREQUESTED_STATE_SERVE") + old = self.db.execute( + "SELECT body FROM dispute_states_v2 WHERE challenge=? AND t=? AND party=?", + (model.challenge_hash, model.t, env.signer), + ).fetchone() + if old is not None and old[0] != model.model_dump_json(): + raise DisputeError("STATE_SERVE_REPLAY_CONFLICT") + self.db.execute( + "INSERT OR IGNORE INTO dispute_states_v2 VALUES(?,?,?,?,?)", + ( + model.challenge_hash, + model.t, + env.signer, + model.model_dump_json(), + raw, + ), + ) + return model + + def resolution(self, raw: bytes, *, now: int) -> Turn: + env = parse_envelope(raw) + with self.escrow.tx(): + turn = self.get(str(env.body["dispute_id"])) + model = Intake( + self.run_id, {"ResolutionV2": lambda h: h == turn.contest.referee} + ).accept( + raw, + now, + ) + if not isinstance(model, ResolutionV2) or model.transcript_hash != turn.transcript_hash: + raise DisputeError("RESOLUTION_TYPE_OR_TRANSCRIPT") + if turn.resolution is not None: + if model != turn.resolution: + raise DisputeError("RESOLUTION_CONFLICT") + return turn + if now > turn.absolute_deadline: + raise DisputeError("RESOLUTION_ABSOLUTE_DEADLINE") + row = self.db.execute( + "SELECT body FROM dispute_evidence_v2 WHERE hash=?", (model.evidence_hash,) + ).fetchone() + if row is None: + raise DisputeError("UNVERIFIED_REFEREE_EVIDENCE") + evidence = RefereeEvidence.model_validate_json(row[0]) + if (evidence.dispute_id, evidence.transcript_hash, evidence.reason, evidence.loser) != ( + turn.dispute_id, + turn.transcript_hash, + model.reason, + model.loser, + ): + raise DisputeError("REFEREE_EVIDENCE_BINDING") + if model.loser is not None and model.loser not in ( + turn.contest.miner, + turn.contest.auditor, + ): + raise DisputeError("UNKNOWN_LOSER") + turn = turn.model_copy(update={"resolution": model, "resolved_beacon": now}) + self._save(turn) + self.notify() + return turn + + def timeout( + self, dispute_id: str, proofs: tuple[Availability, Availability], *, now: int + ) -> Turn: + """Signed delivery and healthy distinct observers; silence is not computation fraud.""" + with self.escrow.tx(): + turn = self.get(dispute_id) + if turn.resolution is not None or now <= min( + turn.turn_deadline, turn.absolute_deadline + ): + raise DisputeError("TIMEOUT_NOT_DUE") + valid = all( + authentic(p) + and ( + p.run_id, + p.dispute_id, + p.seq, + p.transcript_hash, + p.observed_beacon, + p.service_healthy, + p.beacon_healthy, + p.reference_healthy, + ) + == (self.run_id, dispute_id, turn.seq, turn.transcript_hash, now, True, True, True) + for p in proofs + ) + authorities = {p.signer for p in proofs} + allowed = set(self.policy.referees) | {self.key.ss58} + if ( + not valid + or len(authorities) != 2 + or not authorities <= allowed + or authorities + & { + turn.contest.miner, + turn.contest.auditor, + } + ): + raise DisputeError("UNCONFIRMED_AVAILABILITY") + delivered = False + for row in self.db.execute( + "SELECT e.event FROM dispute_events_v2 e JOIN dispute_acks_v2 a " + "ON e.cursor=a.cursor AND e.party=a.party WHERE e.party=?", + (turn.expected_party,), + ): + event = WatchEvent.model_validate_json(row[0]) + delivered |= ( + event.kind == "turn" + and event.turn.dispute_id == dispute_id + and (event.turn.seq, event.turn.transcript_hash) + == (turn.seq, turn.transcript_hash) + ) + if not delivered: + raise DisputeError("NO_AUTHENTICATED_TURN_DELIVERY") + reason: Literal["AUDITOR_FAULT", "PARTY_TIMEOUT"] = ( + "AUDITOR_FAULT" if turn.expected_party == turn.contest.auditor else "PARTY_TIMEOUT" + ) + evidence_hash = sha256_hex(canonicalize([p.body() for p in proofs])) + self.db.execute( + "INSERT OR IGNORE INTO dispute_evidence_v2 VALUES(?,?)", + ( + evidence_hash, + canonicalize([p.body() for p in proofs]).decode(), + ), + ) + resolution = ResolutionV2( + dispute_id=dispute_id, + transcript_hash=turn.transcript_hash, + reason=reason, + loser=turn.expected_party, + evidence_hash=evidence_hash, + ) + turn = turn.model_copy(update={"resolution": resolution, "resolved_beacon": now}) + self._save(turn) + self.notify() + return turn + + def exclude(self, dispute_id: str, *, now: int) -> Turn: + """Absolute exhaustion or infrastructure exclusion; no claimant loses collateral.""" + with self.escrow.tx(): + turn = self.get(dispute_id) + if turn.resolution is not None: + return turn + if not turn.paused and now < turn.absolute_deadline: + raise DisputeError("EXCLUSION_NOT_DUE") + evidence = RefereeEvidence( + dispute_id=dispute_id, + transcript_hash=turn.transcript_hash, + reason="INFRASTRUCTURE", + loser=None, + predecessor_hash="0" * 64, + inputs_hash="0" * 64, + output_hash="0" * 64, + op_spec="bounded-infrastructure-exclusion", + ) + self.register_evidence(evidence) + turn = turn.model_copy( + update={ + "resolved_beacon": now, + "resolution": ResolutionV2( + dispute_id=dispute_id, + transcript_hash=turn.transcript_hash, + reason="INFRASTRUCTURE", + loser=None, + evidence_hash=evidence.digest(), + ), + } + ) + self._save(turn) + self.notify() + return turn + + def settlement(self, lock_id: str) -> DisputeSettlement: + row = self.db.execute( + "SELECT turn FROM disputes_v2 WHERE id=? OR json_extract(turn,'$.lock_id')=?", + (lock_id, lock_id), + ).fetchone() + if row is None: + raise DisputeError("UNKNOWN_CONTEST_LOCK") + turn = Turn.model_validate_json(row[0]) + lockrow = self.db.execute( + "SELECT request,kind FROM escrow_events WHERE operation_id=?", (turn.lock_id,) + ).fetchone() + if lockrow is None or lockrow[1] != "LOCK_CONTEST": + raise DisputeError("UNKNOWN_CONTEST_LOCK") + lock = EscrowLock.model_validate_json(lockrow[0]) + if lock.dispute_id != turn.dispute_id: + raise DisputeError("SETTLEMENT_LOCK_REFERENCE") + resolution = turn.resolution + losing = ( + resolution is not None + and resolution.reason == "FRAUD" + and (resolution.loser == turn.contest.miner) + ) + loser = resolution.loser if resolution else None + return DisputeSettlement( + finality_hash=resolution.digest() if resolution else "0" * 64, + closed_dispute_root=turn.transcript_hash, + release_beacon=turn.resolved_beacon + if turn.resolved_beacon is not None + else turn.absolute_deadline, + unresolved=resolution is None, + outcome="FRAUD" if losing else "MATCH", + run_id=turn.run_id, + admission_id=turn.contest.admission_id, + coldkey=turn.contest.coldkey, + dispute_id=turn.dispute_id, + evidence_hash=turn.contest.evidence_hash, + loser=loser, + loser_coldkey=lock.owner + if loser == turn.contest.miner + else (turn.contest.auditor_coldkey if loser == turn.contest.auditor else None), + referee=turn.contest.referee, + lock_id=lock.operation_id, + lock_owner=lock.owner, + resolution_hash=resolution.digest() if resolution else None, + ) + + def settle_lock(self, dispute_id: str, *, now: int, referee_cost: int = 0) -> None: + with self.escrow.tx(): + turn = self.get(dispute_id) + status = self.settlement(turn.lock_id) + if status.unresolved: + raise DisputeError("UNRESOLVED_CONTEST") + row = self.db.execute( + "SELECT request FROM escrow_events WHERE operation_id=?", (turn.lock_id,) + ).fetchone() + lock = EscrowLock.model_validate_json(row[0]) + if (status.lock_id, status.lock_owner, lock.dispute_id) != ( + lock.operation_id, + lock.owner, + turn.dispute_id, + ): + raise DisputeError("SETTLEMENT_LOCK_OWNERSHIP") + release = EscrowRelease( + **lock.model_dump(exclude={"kind", "operation_id"}), + operation_id=sha256_hex(canonicalize([turn.dispute_id, "settle"])), + lock_id=turn.lock_id, + finality_hash=status.finality_hash, + closed_dispute_root=status.closed_dispute_root, + ) + if ( + turn.resolution is not None + and turn.resolution.reason == "FRAUD" + and status.loser == turn.contest.miner + and status.loser_coldkey == lock.owner + ): + self.escrow.slash( + release, + authority=self.key.ss58, + now_beacon=now, + referee=self.owner_of(turn.contest.referee) if referee_cost else None, + referee_cost=referee_cost, + max_referee_cost=self.policy.max_referee_cost_units, + ) + else: + self.escrow.release(release, signer=lock.owner, now_beacon=now) diff --git a/src/hypertrain/challenge/finality_v2.py b/src/hypertrain/challenge/finality_v2.py new file mode 100644 index 0000000..5d6a2c4 --- /dev/null +++ b/src/hypertrain/challenge/finality_v2.py @@ -0,0 +1,111 @@ +"""Transaction-local predicates for the two-round speculative window. + +L0 persists these records with its opening/apply/settlement CAS. No timers or +elapsed-time assumptions settle an audit. Late finalized fraud is money-only. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass, replace +from typing import Literal + +from hypertrain.aggregator.core import FinalityError +from hypertrain.protocol.messages import Finalize, Rollback + + +@dataclass(frozen=True, slots=True) +class RoundStatus: + run_id: str + w: int + applied: bool + disposition: Literal["PENDING", "SETTLED", "EXCLUDED"] + unresolved_disputes: int + unresolved_audits: int + rollback_complete: bool + state_hash: str + resolution_hash: str | None + + @property + def closed(self) -> bool: + return ( + self.unresolved_disputes == 0 + and self.unresolved_audits == 0 + and ( + self.disposition == "SETTLED" + or (self.disposition == "EXCLUDED" and self.rollback_complete) + ) + ) + + +def require_open(run_id: str, w: int, history: Sequence[RoundStatus]) -> None: + """Block w+2 opening while w is unresolved, even if it was never applied.""" + if ( + type(w) is not int + or w < 0 + or any( + r.run_id != run_id or r.w < 0 or min(r.unresolved_audits, r.unresolved_disputes) < 0 + for r in history + ) + or len({r.w for r in history}) != len(history) + ): + raise FinalityError("invalid finality history") + ordered = sorted(history, key=lambda r: r.w) + if [r.w for r in ordered] != list(range(w)): + raise FinalityError("opening requires complete sequential history") + if any(not r.closed for r in ordered if r.w <= w - 2): + raise FinalityError("w+2 blocked until settlement or exclusion plus rollback") + + +def require_apply(run_id: str, w: int, history: Sequence[RoundStatus]) -> None: + require_open(run_id, w, history) + if w and not next(r for r in history if r.w == w - 1).applied: + raise FinalityError("predecessor not applied") + if sum(r.applied and not r.closed for r in history) >= 2: + raise FinalityError("two speculative applied rounds already exist") + + +def settle(status: RoundStatus, final: Finalize, finality_hash: str) -> RoundStatus: + """L0 calls only after authorized finality and all audit/dispute records close.""" + if (status.w, status.state_hash) != (final.w, final.final_theta_hash_w1) or ( + not status.applied + or status.unresolved_audits + or status.unresolved_disputes + or status.disposition == "EXCLUDED" + ): + raise FinalityError("round cannot settle") + if status.resolution_hash is not None and status.resolution_hash != finality_hash: + raise FinalityError("conflicting finality") + return replace(status, disposition="SETTLED", resolution_hash=finality_hash) + + +def exclude(status: RoundStatus, evidence_hash: str) -> RoundStatus: + if status.disposition == "SETTLED": + raise FinalityError("finalized late fraud cannot rewrite model history") + if status.resolution_hash is not None and status.resolution_hash != evidence_hash: + raise FinalityError("conflicting exclusion") + return replace( + status, + disposition="EXCLUDED", + resolution_hash=evidence_hash, + rollback_complete=not status.applied, + ) + + +def finish_rollback( + status: RoundStatus, + rollback: Rollback, + replayed_theta_hash: str, + replayed_outer_hash: str, +) -> RoundStatus: + """Require replayed two-round repair, not merely an exclusion flag.""" + if status.disposition != "EXCLUDED" or status.w != rollback.w or not status.applied: + raise FinalityError("rollback has no pending applied exclusion") + if ( + rollback.recomputed != ["agg_w", "step_w", "agg_w1", "step_w1"] + or (rollback.new_theta_hash_w2, rollback.new_outer_state_hash) + != (replayed_theta_hash, replayed_outer_hash) + or (status.resolution_hash not in rollback.cause_hashes) + ): + raise FinalityError("rollback lacks complete authenticated replay") + return replace(status, rollback_complete=True) diff --git a/src/hypertrain/challenge/store.py b/src/hypertrain/challenge/store.py index 3dde3a2..c260a0d 100644 --- a/src/hypertrain/challenge/store.py +++ b/src/hypertrain/challenge/store.py @@ -7,25 +7,50 @@ from __future__ import annotations +import ast import hashlib +import hmac import json +import math import secrets +import socket import sqlite3 import struct +import tempfile import threading +import time from collections.abc import Callable, Iterator, Mapping from contextlib import contextmanager +from dataclasses import asdict from fractions import Fraction from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any, Literal -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict, JsonValue, StrictInt + +if TYPE_CHECKING: + from experiments.gpu_network_v2.orchestrate import NetworkRuntime + from scripts.network_gpu_operation import Operation + + from hypertrain.gpu_ops.work_screen import IslandLaunch from hypertrain.beacon import BeaconError, BeaconRound +from hypertrain.challenge.admission import Admission +from hypertrain.challenge.disputes_v2 import Contest, DisputesV2 from hypertrain.data.assignment import assign_round from hypertrain.data.store import ObjectNotFound, StoreError from hypertrain.data.store import Store as ObjectStore +from hypertrain.data.trial_assignment import trial_assignment_hash from hypertrain.ledger import Ledger, LedgerError, Params +from hypertrain.ledger.escrow_v2 import ( + EscrowError, + EscrowV2, + FinalityEvidence, + RewardWork, + Settlement, + ShadowReservationEvidence, +) +from hypertrain.protocol import envelope_v2 from hypertrain.protocol.envelope import ( EnvelopeError, ExpiredError, @@ -43,6 +68,7 @@ from hypertrain.protocol.jcs import canonicalize from hypertrain.protocol.keys import KeyError_, Keypair, decode_hotkey, verify from hypertrain.protocol.messages import ( + SS58, Accept, AuditChallenge, Bisect, @@ -60,6 +86,23 @@ f32hex, f32val, ) +from hypertrain.protocol.messages_v2 import ( + AdmissionPolicyV2, + AggregationPolicyV2, + AuditPolicyV2, + DisputePolicyV2, + EconomicsPolicyV2, + Hex64, + IslandJobV1, + JoinChallenge, + RunManifestV2, + ShadowReplayReceiptV1, + ShadowReservationRequestV1, + ShadowReservationV1, + ShadowRewardFinalizeV1, + WireModel, + WorkScreenV2, +) NEVER = 2**53 - 1 # exp_drand of container-signed records LEASE_ROUNDS = 600 # 30 min of 3 s drand rounds @@ -69,6 +112,92 @@ MINER_TYPES = frozenset({"Accept", "Commit", "DeltaManifest", "StateServe", "Dispute", "Bisect"}) COORD_TYPES = ("RoundOpen", "Rollback", "Finalize") + +class ShadowCustodyIntake(WireModel): + job_hash: Hex64 + reference_publication_hash: Hex64 + miner_publication_hash: Hex64 + reference_result_hash: Hex64 + miner_result_hash: Hex64 + reference_operation_hash: Hex64 + miner_operation_hash: Hex64 + + +class ShadowReservationIntake(WireModel): + finalize: envelope_v2.EnvelopeV2 + commit: envelope_v2.EnvelopeV2 + custody: ShadowCustodyIntake + + +class ShadowAcceptanceIntake(WireModel): + reservation_hash: Hex64 + audit_challenge: envelope_v2.EnvelopeV2 + verdict: envelope_v2.EnvelopeV2 + auditor_receipt: envelope_v2.EnvelopeV2 + reward: ShadowRewardFinalizeV1 + + +class ShadowOperatorRecord(BaseModel): + """Private strict addressed authority, not a public protocol or qualification.""" + + model_config = ConfigDict(extra="forbid", strict=True, frozen=True) + domain: Literal["shadow-operator-admission-v1"] + run_id: Hex64 + owner: SS58 + admission_hash: Hex64 + manifest_hash: Hex64 + economics_policy_hash: Hex64 + graph_hash: Hex64 + workload: dict[str, JsonValue] + backend_qualification_authority_hash: Hex64 + source_map_hash: Hex64 + profile_hash: Hex64 + hosts: dict[str, JsonValue] + budget: dict[str, JsonValue] + accepted_beacon: StrictInt + cutoff_unix: StrictInt + expires_beacon: StrictInt + + +def shadow_operator_receipt( + raw: bytes, owner: str, run_id: str, accepted_at: int +) -> tuple[ShadowOperatorRecord, str, dict[str, JsonValue]]: + """Authenticate ORIGINAL receipt beacon on intake/recovery, never current expiry renewal.""" + from hypertrain.protocol.messages import Receipt + + payload = envelope_v2.load_json(raw) + if set(payload) != {"record", "receipt"}: + raise ChallengeError(403, "operator payload fields differ") + record = ShadowOperatorRecord.model_validate(payload["record"]) + receipt = envelope_v2.Intake(run_id, {"Receipt": owner.__eq__}).accept( + canonicalize(payload["receipt"]), accepted_at + ) + if type(accepted_at) is not int or accepted_at < 1: + raise ChallengeError(403, "operator retained verified beacon invalid") + env = envelope_v2.parse_envelope(canonicalize(payload["receipt"])) + digest = sha256_hex(canonicalize(record.model_dump(mode="json"))) + if ( + not isinstance(receipt, Receipt) + or record.owner != owner + or record.run_id != run_id + or receipt.commit_hash != digest + or not receipt.received_round == record.accepted_beacon == accepted_at + or env.exp_drand != record.expires_beacon + or accepted_at > record.expires_beacon + ): + raise ChallengeError(403, "operator original receipt beacon/binding differs") + return record, digest, payload + + +def shadow_operator_expected(record: ShadowOperatorRecord, expected: dict, clock: float) -> None: + if ( + canonicalize(record.model_dump(mode="json")) != canonicalize(expected) + or clock >= record.cutoff_unix + or record.budget.get("scope") != "FULL116_CONTINUATION" + ): + raise ChallengeError(403, "operator graph/workload/budget/source/cutoff differs") + + _SCHEMA = """ CREATE TABLE IF NOT EXISTS meta(key TEXT PRIMARY KEY, value TEXT NOT NULL); CREATE TABLE IF NOT EXISTS beacon(round INTEGER PRIMARY KEY, signature TEXT NOT NULL, @@ -106,6 +235,25 @@ """ +class _ServiceCapacityV1(BaseModel): + """Internal owner admission, not a public wire type or a CPU runtime benchmark.""" + + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + version: StrictInt + profile_id: Literal["tiny-cpu-service-32-v1"] + run_id: str + backend_binding_hash: str + implementation_hash: str + max_complete_roster: StrictInt + memory_reservation_bytes: StrictInt + max_outer_work_units: StrictInt + max_audit_step_units: StrictInt + max_repair_step_units: StrictInt + max_object_bytes: StrictInt + max_tape_bytes: StrictInt + + class ChallengeError(Exception): def __init__(self, status: int, detail: str) -> None: super().__init__(detail) @@ -163,6 +311,10 @@ def __init__( owner_hotkey: str | None, verify_beacon: Callable[[Mapping[str, Any]], BeaconRound], objects: ObjectStore, + *, + _role_launch: ( + Callable[[str, IslandJobV1, Path, Mapping[str, Any]], IslandLaunch] | None + ) = None, ) -> None: state_dir.mkdir(parents=True, exist_ok=True) self.params = params @@ -170,6 +322,7 @@ def __init__( self.owner_hotkey = owner_hotkey self.verify_beacon = verify_beacon self.objects = objects + self._role_launch = _role_launch self._lock = threading.RLock() self._db = sqlite3.connect( state_dir / "challenge.db", check_same_thread=False, isolation_level=None @@ -180,9 +333,31 @@ def __init__( self._db.executescript(_SCHEMA) self.ledger = Ledger(state_dir / "ledger", params) self._intakes: dict[str, Intake] = {} + self.state_dir = state_dir + self.clock: Callable[[], float] = time.time + self._services_v2: dict[str, tuple[Any, Any, Any]] = {} + self.beacon_arrived = threading.Condition() + self._lease_guards_v2: dict[str, Any] = {} + self._ipc_v2: socket.socket | None = None + self._ipc_path_v2: Path | None = None + self._ipc_thread_v2: threading.Thread | None = None + self._db.executescript(""" + CREATE TABLE IF NOT EXISTS accepted_v2( + key TEXT PRIMARY KEY, digest TEXT NOT NULL, envelope BLOB NOT NULL, + accepted_beacon INTEGER NOT NULL, receipt TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS records_v2( + run_id TEXT, kind TEXT, id TEXT, data TEXT NOT NULL, + PRIMARY KEY(run_id,kind,id)); + CREATE TABLE IF NOT EXISTS audit_leases_v2( + id TEXT PRIMARY KEY, run_id TEXT, w INTEGER, hotkey TEXT, + challenge TEXT, created INTEGER, absolute INTEGER, state TEXT, + attempts INTEGER NOT NULL DEFAULT 0, auditor TEXT, nonce TEXT, + expires INTEGER, tried TEXT NOT NULL DEFAULT '[]', reservation TEXT, + step_budget INTEGER, result TEXT); + """) @contextmanager - def _tx(self) -> Iterator[sqlite3.Connection]: + def _tx(self, *, notify: bool = False) -> Iterator[sqlite3.Connection]: with self._lock: self._db.execute("BEGIN IMMEDIATE") try: @@ -191,6 +366,8 @@ def _tx(self) -> Iterator[sqlite3.Connection]: self._db.execute("ROLLBACK") raise self._db.execute("COMMIT") + if notify: + self._notify_processes_v2() def healthy(self) -> bool: try: @@ -200,6 +377,16 @@ def healthy(self) -> bool: return False return True + def close_v2_notifications(self) -> None: + if self._ipc_v2 is not None: + self._ipc_v2.sendto(b"stop", str(self._ipc_path_v2)) + if self._ipc_thread_v2 is not None: + self._ipc_thread_v2.join(timeout=5) + self._ipc_v2.close() + assert self._ipc_path_v2 is not None + self._ipc_path_v2.unlink(missing_ok=True) + self._ipc_v2 = None + def _meta(self, db: sqlite3.Connection, key: str, default: int) -> int: row = db.execute("SELECT value FROM meta WHERE key=?", (key,)).fetchone() return int(row["value"]) if row else default @@ -230,6 +417,8 @@ def _run(self, db: sqlite3.Connection, run_id: str) -> tuple[sqlite3.Row, RunMan row = db.execute("SELECT * FROM runs WHERE run_id=?", (run_id,)).fetchone() if row is None: raise ChallengeError(404, "unknown run") + if json.loads(row["manifest"]).get("manifest_version") == 2: + raise ChallengeError(409, "route and stored contract version differ") return row, RunManifest.model_validate_json(row["manifest"]) def _round(self, db: sqlite3.Connection, run_id: str, w: int) -> tuple[sqlite3.Row, RoundOpen]: @@ -320,7 +509,38 @@ def push_beacon(self, payload: Mapping[str, Any]) -> dict[str, Any]: (br.round, br.signature, br.randomness, int(br.bls_verified)), ) self._advance(db) - return {"round": br.round, "latest": self._now(db)} + result = {"round": br.round, "latest": self._now(db)} + with self.beacon_arrived: + self.beacon_arrived.notify_all() + for guard in self._lease_guards_v2.values(): + guard.advance(br.round) + for run_id, (escrow, _, _) in self._services_v2.items(): + with self._tx(): + origins = self._db.execute( + "SELECT o.origin FROM escrow_origins o JOIN escrow_units u " + "ON o.origin=u.origin WHERE o.mature_at<=? " + "AND u.bucket='reward_pending' AND u.units>0", + (br.round,), + ).fetchall() + for origin in origins: + try: + escrow.mature( + sha256_hex(f"ht-mature/2|{run_id}|{origin[0]}".encode()), + (origin[0],), + br.round, + ) + except EscrowError as error: + if error.code not in ( + "DISPUTED_OR_UNVESTED", + "IMMATURE_OR_UNKNOWN_ORIGIN", + "ALREADY_MATURED", + ): + raise + self._expire_leases_v2(run_id, br.round) + for _, _, disputes in self._services_v2.values(): + disputes.notify() + self._notify_processes_v2() + return result def latest_beacon(self) -> dict[str, Any]: with self._lock: @@ -1603,3 +1823,7659 @@ def weights(self, epoch: int, epoch_at: int | None, computed_at: int) -> bytes: last = self._meta(db, "last_answer_at", -1) self._set_meta(db, "last_answer_at", max(last, epoch_at)) return answer + + def _run_v2(self, run_id: str) -> RunManifestV2: + row = self._db.execute("SELECT manifest FROM runs WHERE run_id=?", (run_id,)).fetchone() + if row is None: + raise ChallengeError(404, "unknown run") + data = envelope_v2.load_json(row[0]) + if not isinstance(data, dict) or data.get("manifest_version") != 2: + raise ChallengeError(409, "route and stored contract version differ") + return RunManifestV2.model_validate(data) + + def _record_v2(self, run_id: str, kind: str, identity: str) -> dict[str, Any]: + row = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind=? AND id=?", + (run_id, kind, identity), + ).fetchone() + if row is None: + raise ChallengeError(409, f"missing accepted {kind}: {identity}") + value = json.loads(row[0]) + if not isinstance(value, dict): + raise ChallengeError(409, "invalid persisted record") + return value + + def _put_record_v2( + self, run_id: str, kind: str, identity: str, data: Mapping[str, JsonValue] + ) -> None: + self._db.execute( + "INSERT OR REPLACE INTO records_v2 VALUES(?,?,?,?)", + (run_id, kind, identity, canonicalize(dict(data), allow_float=False).decode()), + ) + + def _beacon_v2(self, rnd: int) -> BeaconRound: + from hypertrain.beacon import BeaconUnavailable + + row = self._db.execute("SELECT * FROM beacon WHERE round=?", (rnd,)).fetchone() + if row is None: + raise BeaconUnavailable(f"round {rnd} not pushed") + return BeaconRound(row[0], row[1], row[2], bool(row[3])) + + def _snapshot_current_v2(self, manifest: RunManifestV2) -> None: + """Read freshness gate for every eligibility/lease deadline boundary.""" + now = self._now(self._db) + emitted = ( + int(self.clock()) - manifest.training.beacon.genesis_time + ) // manifest.training.beacon.period + 1 + if now < 1 or now < emitted - 2: + raise ChallengeError(503, "beacon infrastructure paused") + + def _fresh_v2(self, manifest: RunManifestV2, operation: str, rnd: int | None = None) -> int: + from hypertrain.beacon.core import BeaconUnavailable + from hypertrain.beacon.freshness import FreshnessGate + + now = self._now(self._db) + pin = now if rnd is None else rnd + if pin < 1: + raise ChallengeError(503, "no verified beacon") + gate = FreshnessGate(self._db, manifest.run_id()) + gate.pin(operation, pin) + if manifest.network.economics_policy_hash == "0" * 64: + raise ChallengeError(409, "economics policy missing") + policy = EconomicsPolicyV2.model_validate_json( + self.objects.get(manifest.network.economics_policy_hash) + ) + if policy.ledger_mode == "test": + # Test-only functional CPU evidence; never a production BLS qualification. + b = self._beacon_v2(pin) + emitted = (int(self.clock()) - manifest.training.beacon.genesis_time) // 3 + 1 + if now < emitted - 2: + self._db.execute( + "UPDATE beacon_pins_v2 SET paused=1,reason='STALE' " + "WHERE run_id=? AND operation_id=?", + (manifest.run_id(), operation), + ) + raise ChallengeError(503, "beacon infrastructure paused") + self._db.execute( + "UPDATE beacon_pins_v2 SET signature=?,signature_hash=?,paused=0,reason=NULL " + "WHERE run_id=? AND operation_id=?", + (b.signature, sha256_hex(bytes.fromhex(b.signature)), manifest.run_id(), operation), + ) + else: + store = self + + class AcceptedBeacon: + def get(self, round: int) -> BeaconRound: + return store._beacon_v2(round) + + def round_at_or_after(self, t: int) -> int: + from hypertrain.beacon.core import round_at_or_after + + return round_at_or_after(t, manifest.training.beacon.genesis_time, 3) + + try: + gate.refresh( + operation, + AcceptedBeacon(), + now_seconds=int(self.clock()), + genesis_seconds=manifest.training.beacon.genesis_time, + period_seconds=manifest.training.beacon.period, + max_lag_rounds=2, + ) + except BeaconUnavailable: + raise ChallengeError( + 503, "beacon infrastructure paused; pinned seed retained" + ) from None + return now + + def create_run_v2(self, raw: bytes) -> dict[str, JsonValue]: + if self.owner_hotkey is None: + raise ChallengeError(503, "owner hotkey is not configured") + env = envelope_v2.parse_envelope(raw) + if env.type != "RunManifestV2": + raise ChallengeError(400, "expected RunManifestV2") + with self._tx(): + model = envelope_v2.Intake( + env.run_id, {"RunManifestV2": self.owner_hotkey.__eq__} + ).accept(raw, self._now(self._db)) + assert isinstance(model, RunManifestV2) + if model.training.coord_pubkey != self._coord().ss58: + raise ChallengeError(403, "wrong coordinator") + if self._db.execute("SELECT 1 FROM runs").fetchone(): + raise ChallengeError(409, "one run per state directory; no version upgrade") + for name, cls in ( + ("admission_policy_hash", AdmissionPolicyV2), + ("economics_policy_hash", EconomicsPolicyV2), + ("aggregation_policy_hash", AggregationPolicyV2), + ("audit_policy_hash", AuditPolicyV2), + ("dispute_policy_hash", DisputePolicyV2), + ): + policy = cls.model_validate_json(self.objects.get(getattr(model.network, name))) + if name == "audit_policy_hash": + assert isinstance(policy, AuditPolicyV2) + policy.validate_training(model.training) + from hypertrain.protocol.relay_messages import RelayRegistryV1 + + RelayRegistryV1.model_validate_json(self.objects.get(model.network.relay_registry_hash)) + if len(set(model.training.auditors)) < 2: + raise ChallengeError(422, "two distinct audit reference identities required") + if model.training.inner.state_policy == "derived": + raise ChallengeError( + 422, "v2 derived-state influence requires authenticated v0 integration" + ) + self._db.execute( + "INSERT INTO runs VALUES(?,?,?,'created',NULL)", + (model.run_id(), model.model_dump_json(), raw.decode()), + ) + self._services(run_id=model.run_id()) + return {"run_id": model.run_id(), "status": "created", "contract": 2} + + def _owner_v2(self, run_id: str, hotkey: str) -> str: + row = self._db.execute( + "SELECT coldkey FROM admissions_v2 WHERE run_id=? AND hotkey=?", (run_id, hotkey) + ).fetchone() + if row is None: + raise ChallengeError(403, "hotkey is not current accepted identity") + return str(row[0]) + + def _assignment_v2(self, run_id: str, w: int, hotkey: str) -> tuple[int, ...]: + self.require_roster_v2(run_id, w) + value = self._record_v2(run_id, "assignment", f"{w}:{hotkey}") + samples = value["samples"] + assert isinstance(samples, list) and all(type(i) is int for i in samples) + return tuple(int(str(i)) for i in samples) + + def _settlement_v2(self, run_id: str, reference: str) -> Settlement: + disputes = self._services_v2.get(run_id) + if disputes is not None: + row = self._db.execute( + "SELECT turn FROM disputes_v2 WHERE id=? OR json_extract(turn,'$.lock_id')=?", + (reference, reference), + ).fetchone() + if row: + return disputes[2].settlement(reference) + row = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='settlement' AND id=?", + (run_id, reference), + ).fetchone() + if row is None: + raise EscrowError("UNACCEPTED_SETTLEMENT") + return Settlement.model_validate_json(row[0]) + + def _finality_replay_v2(self, run_id: str, evidence: FinalityEvidence) -> str: + if evidence.shadow: + return self._shadow_replay_v2(run_id, evidence) + record = self._record_v2(run_id, "replayed-finality", str(evidence.finalize.body["w"])) + if ( + record["finalize_hash"] != body_digest(evidence.finalize.body) + or record["tape_hash"] != evidence.tape_hash + or record["finalized_beacon"] != evidence.finalized_beacon + or record["shadow"] != evidence.shadow + ): + raise EscrowError("FINALITY_ACCEPTANCE_LINKAGE") + from hypertrain.aggregator.checkpoint import network_inputs + from hypertrain.aggregator.tape_v2 import TapeV2, replay_tape + + manifest = self._run_v2(run_id) + w = int(str(evidence.finalize.body["w"])) + self.require_roster_v2(run_id, w) + applied = self._record_v2(run_id, "applied", str(w)) + inputs = network_inputs(self._record_v2(run_id, "tape-inputs", str(w))["inputs"]) + tape = TapeV2.from_bytes(self.objects.get(evidence.tape_hash)) + replayed = replay_tape( + self.objects, + tape, + manifest, + AggregationPolicyV2.model_validate_json( + self.objects.get(manifest.network.aggregation_policy_hash) + ), + self.objects.get(manifest.network.economics_policy_hash), + signer=self._coord().ss58, + w=w, + prev_state=str(applied["prev_state"]), + predecessor_tape_hash=str(applied["predecessor_tape_hash"]), + inputs=inputs, + reference_reward_units=self._services_v2[run_id][0].policy.R_collectible_units, + ) + from hypertrain.aggregator.core import th + + if th(replayed.theta) != record["theta_hash"]: + raise EscrowError("FINALITY_TAPE_REPLAY_MISMATCH") + return str(record["theta_hash"]) + + def _bootstrap_backend_v2( + self, + manifest: RunManifestV2, + configs: tuple[Path, Path], + results: tuple[Path, Path], + raw: bytes, + ) -> None: + """Internal owner-reviewed CUDA evidence; install before run/service creation.""" + from hypertrain.gpu_ops import network_qualification as engine + from hypertrain.gpu_ops.journal import fsha + from hypertrain.protocol.messages import Receipt + + ref = manifest.training.reference_spec + config_bytes = tuple(p.read_bytes() for p in configs) + result_bytes = tuple(p.read_bytes() for p in results) + evidence: dict[str, JsonValue] = { + "run_id": manifest.run_id(), + "backend": "cuda", + "reference_hash": sha256_hex(canonicalize(ref.model_dump(mode="json"))), + "layout_hash": ref.layout.model_dump_json(), + "configs": [sha256_hex(data) for data in config_bytes], + "results": [sha256_hex(data) for data in result_bytes], + "summaries": [ + fsha(path.parent / f"round-{round_index}/published/rank-{rank}/summary.json") + for path in results + for round_index in range(2) + for rank in range(ref.layout.n_gpus) + ], + } + digest = sha256_hex(canonicalize(evidence)) + receipt = envelope_v2.Intake( + manifest.run_id(), {"Receipt": lambda h: h == self.owner_hotkey} + ).accept(raw, self._now(self._db)) + env = envelope_v2.parse_envelope(raw) + if ( + env.type != "Receipt" + or not isinstance(receipt, Receipt) + or (receipt.commit_hash != digest or receipt.received_round != self._now(self._db)) + ): + raise ChallengeError(403, "reviewed backend receipt differs from exact evidence") + for config_path, result_path in zip(configs, results, strict=True): + cfg = engine.Qualification.model_validate_json(config_path.read_bytes()) + job = engine.check_inputs(cfg) + result = engine.Result.model_validate_json(result_path.read_bytes()) + for round_index in range(2): + for rank in range(ref.layout.n_gpus): + summary = json.loads( + ( + result_path.parent + / f"round-{round_index}/published" + / f"rank-{rank}/summary.json" + ).read_bytes() + ) + if summary.get("backend") != "cuda": + raise ChallengeError(403, "CPU artifacts cannot qualify CUDA execution") + if ( + job.manifest != manifest + or result.config_sha256 != fsha(config_path) + or result.profile_sha256 != cfg.profile_sha256 + or result.sources != cfg.sources + or (result.instance_id, result.machine_id, result.role) + != (cfg.instance_id, cfg.machine_id, cfg.role) + or result.environment.image_digest != ref.image_digest + or any(d not in ref.driver_allowlist for d in result.environment.drivers) + or any(s != ref.sm_count for s in result.environment.sm_counts) + or not Path(result.environment.torch_path).is_relative_to("/venv/main") + ): + raise ChallengeError(403, "reviewed CUDA runtime/reference binding differs") + engine.compare(list(results)) # Rehash all rescued rank/round/checkpoint bytes. + summaries = [ + fsha(path.parent / f"round-{round_index}/published/rank-{rank}/summary.json") + for path in results + for round_index in range(2) + for rank in range(ref.layout.n_gpus) + ] + if evidence["summaries"] != summaries: + raise ChallengeError(409, "qualification summary custody changed during bootstrap") + with self._tx(): + row = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='qualification' AND id=?", + (manifest.run_id(), ref.image_digest), + ).fetchone() + if row is not None: + if json.loads(row[0])["authority_hash"] != digest: + raise ChallengeError(409, "immutable backend qualification conflicts") + return + if ( + manifest.run_id() in self._services_v2 + or self._db.execute( + "SELECT 1 FROM runs WHERE run_id=?", (manifest.run_id(),) + ).fetchone() + or self._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='execution-backend'", + (manifest.run_id(),), + ).fetchone() + ): + raise ChallengeError(409, "backend qualification must precede run services") + for path, data in zip( + (*configs, *results), (*config_bytes, *result_bytes), strict=True + ): + if path.read_bytes() != data: + raise ChallengeError(409, "qualification custody changed during bootstrap") + self.objects.put(data) + authority = self.objects.put(canonicalize(evidence)) + record = { + **evidence, + "authority_hash": authority, + "receipt": env.model_dump(mode="json"), + } + self._put_record_v2(manifest.run_id(), "qualification", ref.image_digest, record) + + def _backend_v2(self, manifest: RunManifestV2) -> Literal["cpu", "cuda"]: + """Resolve frozen run execution authority independently of monetary policy.""" + with self._lock: + ref = manifest.training.reference_spec + row = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='qualification' AND id=?", + (manifest.run_id(), ref.image_digest), + ).fetchone() + authority = None + if row is not None: + record = json.loads(row[0]) + authority = str(record["authority_hash"]) + evidence = envelope_v2.load_json(self.objects.get(authority)) + env = envelope_v2.parse_envelope(record["receipt"]) + if ( + sha256_hex(canonicalize(evidence)) != authority + or any(record.get(k) != v for k, v in evidence.items()) + or evidence.get("run_id") != manifest.run_id() + or evidence.get("backend") != "cuda" + or evidence.get("reference_hash") + != sha256_hex(canonicalize(ref.model_dump(mode="json"))) + or evidence.get("layout_hash") != ref.layout.model_dump_json() + or env.type != "Receipt" + or env.run_id != manifest.run_id() + or env.signer != self.owner_hotkey + or env.body["commit_hash"] != authority + or not envelope_v2.verify_envelope(record["receipt"]) + ): + raise ChallengeError(403, "backend qualification authority differs") + backend: Literal["cpu", "cuda"] = "cuda" + else: + econ = EconomicsPolicyV2.model_validate_json( + self.objects.get(manifest.network.economics_policy_hash) + ) + if econ.ledger_mode != "test": + raise ChallengeError(409, "production CUDA qualification required") + backend = "cpu" + selection: dict[str, JsonValue] = {"backend": backend, "authority_hash": authority} + frozen = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? " + "AND kind='execution-backend' AND id='run'", + (manifest.run_id(),), + ).fetchone() + if frozen is not None and json.loads(frozen[0]) != selection: + raise ChallengeError(409, "frozen execution backend cannot change") + if frozen is None: + self._put_record_v2(manifest.run_id(), "execution-backend", "run", selection) + return backend + + def _prefix_secret_v2(self) -> bytes: + secret = hmac.digest(self._coord()._secret, b"hypertrain/ip-prefix/2", "sha256") + namespace = sha256_hex(secret) + with self._tx() as db: + row = db.execute( + "SELECT value FROM meta WHERE key='admission-prefix-namespace-v2'" + ).fetchone() + if row is not None: + if row["value"] != namespace: + raise ChallengeError(503, "admission prefix namespace differs") + else: + quota = db.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='admission_quota'" + ).fetchone() + if ( + quota is not None + and db.execute( + "SELECT 1 FROM admission_quota WHERE identity LIKE 'ip:%' LIMIT 1" + ).fetchone() + ): + raise ChallengeError(503, "legacy admission prefix namespace unavailable") + db.execute( + "INSERT INTO meta VALUES('admission-prefix-namespace-v2', ?)", (namespace,) + ) + return secret + + def _services(self, run_id: str) -> tuple[EscrowV2, Admission, DisputesV2]: + with self._lock: + existing = self._services_v2.get(run_id) + if existing is not None: + self._backend_v2(self._run_v2(run_id)) + return existing + manifest = self._run_v2(run_id) + backend = self._backend_v2(manifest) + n = manifest.network + econ = EconomicsPolicyV2.model_validate_json(self.objects.get(n.economics_policy_hash)) + escrow = EscrowV2( + self._db, + manifest, + econ, + auditors=frozenset(manifest.training.auditors), + replay_finality=lambda e: self._finality_replay_v2(run_id, e), + settlement=lambda ref: self._settlement_v2(run_id, ref), + owner_of=lambda hot: self._owner_v2(run_id, hot), + assignment_of=lambda w, hot: self._assignment_v2(run_id, w, hot), + shadow_assignment_of=lambda binding: self._shadow_assignment_v2(run_id, binding), + shadow_settlement=lambda origin: self._shadow_settlement_v2(run_id, origin), + ) + escrow.mutex = self._lock # One SQLite authority lock, including event acknowledgments. + + def qualify(screen: WorkScreenV2) -> bool: + if screen.layout != manifest.training.reference_spec.layout or ( + screen.image_digest != manifest.training.reference_spec.image_digest + ): + return False + if backend == "cpu": + return True + try: + record = self._record_v2(run_id, "qualification", screen.image_digest) + except ChallengeError: + return False + return record.get("layout_hash") == screen.layout.model_dump_json() and ( + record.get("reference_hash") + == sha256_hex( + canonicalize(manifest.training.reference_spec.model_dump(mode="json")) + ) + ) + + policy = AdmissionPolicyV2.model_validate_json( + self.objects.get(n.admission_policy_hash) + ) + admission = Admission( + escrow, + policy, + self._coord(), + beacon=self._beacon_v2, + stage_reference=lambda c, samples, epoch: self.stage_trial_v2( + run_id, c, samples, epoch + ), + objects=self.objects, + ip_secret=self._prefix_secret_v2(), + qualified=qualify, + conservative_bound=lambda: ( + econ.ledger_mode == "test" + or ( + self._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='economic-bound'", + (run_id,), + ).fetchone() + is not None + ) + ), + backend=backend, + ) + dispute_policy = DisputePolicyV2.model_validate_json( + self.objects.get(n.dispute_policy_hash) + ) + disputes = DisputesV2( + escrow, + dispute_policy, + self._coord(), + contest_of=lambda h: Contest.model_validate(self._record_v2(run_id, "contest", h)), + owner_of=lambda hot: self._owner_v2(run_id, hot), + span=lambda contest, level, ctx: self._dispute_span_v2(run_id, contest, level, ctx), + ) + self._services_v2[run_id] = (escrow, admission, disputes) + self._start_notifications_v2() + return escrow, admission, disputes + + def _start_notifications_v2(self) -> None: + """Explicit datagram IPC; durable SQLite events are truth, no event polling.""" + import os + + if self._ipc_v2 is not None: + return + directory = Path(tempfile.gettempdir()) / ( + "ht-notify-" + sha256_hex(str(self.state_dir.resolve()).encode())[:20] + ) + directory.mkdir(mode=0o700, exist_ok=True) + if directory.stat().st_uid != os.getuid() or directory.stat().st_mode & 0o077: + raise ChallengeError(503, "unsafe notification directory") + path = directory / f"{os.getpid()}-{secrets.token_hex(4)}.sock" + ipc = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + ipc.bind(str(path)) + self._ipc_v2, self._ipc_path_v2 = ipc, path + + def receive() -> None: + while True: + message = ipc.recv(64) + if message == b"stop": + return + with self._lock: + now = self._now(self._db) + for guard in tuple(self._lease_guards_v2.values()): + guard.advance(now) + disputes = [s[2] for s in self._services_v2.values()] + for dispute in disputes: + dispute.notify() + with self.beacon_arrived: + self.beacon_arrived.notify_all() + + self._ipc_thread_v2 = threading.Thread(target=receive, daemon=True, name="ht-v2-notify") + self._ipc_thread_v2.start() + + def _notify_processes_v2(self) -> None: + if self._ipc_v2 is None or self._ipc_path_v2 is None: + return + for path in self._ipc_path_v2.parent.glob("*.sock"): + if path == self._ipc_path_v2: + continue + try: + self._ipc_v2.sendto(b"committed", str(path)) + except (FileNotFoundError, ConnectionRefusedError): + path.unlink(missing_ok=True) + + def stage_trial_v2( + self, run_id: str, challenge: JoinChallenge, samples: tuple[int, ...], epoch: int + ) -> tuple[IslandJobV1, Path]: + import torch + + import hypertrain.trainer # noqa: F401 + from hypertrain.auditor.replay import AnchorCache, pack_state + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.model import init_params + + manifest = self._run_v2(run_id) + cfg = TrainConfig.from_manifest_v2(manifest) + theta = init_params(cfg.model) + start_state = pack_state(theta) + ef_in = pack_state({k: torch.zeros_like(v) for k, v in theta.items()}) + if cfg.inner.state_policy == "carry": + admission = self._services(run_id)[1].store.by_id(challenge.admission_id) + genesis = AnchorCache().genesis(manifest, admission.hotkey, theta) + start_state = pack_state(genesis.theta, genesis.state) + ef_in = pack_state(genesis.ef) + directory = self.state_dir / "trials-v2" / challenge.nonce + directory.mkdir(parents=True, exist_ok=True) + dataset = self._record_v2(run_id, "dataset", "inputs") + raw = self.objects.get(str(dataset["samples_hash"])) + proofs = json.loads(self.objects.get(str(dataset["proofs_hash"]))) + assert isinstance(proofs, list) + width = (cfg.model.seq_len + 1) * ( + 2 if manifest.training.dataset.sample_format.startswith("u16") else 4 + ) + if ( + len(raw) != manifest.training.dataset.n_samples * width + or len(proofs) != manifest.training.dataset.n_samples + ): + raise ChallengeError(422, "dataset object dimensions differ from signed manifest") + inputs = { + "start_state": start_state, + "ef_in": ef_in, + "v0": pack_state({}), + "samples": b"".join(raw[i * width : (i + 1) * width] for i in samples), + "sample_proofs": canonicalize([proofs[i] for i in samples]), + } + for name, value in inputs.items(): + path = directory / name + if path.exists() and path.read_bytes() != value: + raise ChallengeError(409, "immutable trial inputs changed") + path.write_bytes(value) + deadline = manifest.training.beacon.genesis_time + (challenge.deadline_beacon - 1) * 3 + return IslandJobV1( + job_version=1, + run_id=run_id, + w=epoch, + manifest=manifest, + sample_ids=list(samples), + global_step0=0, + start_state_sha256=sha256_hex(inputs["start_state"]), + ef_in_sha256=sha256_hex(inputs["ef_in"]), + v0_sha256=sha256_hex(inputs["v0"]), + object_paths={name: name for name in inputs}, + deadline=deadline, + ), directory + + def _dispute_span_v2( + self, run_id: str, contest: Contest, level: Any, ctx: tuple[int, ...] + ) -> int: + from hypertrain.auditor.island_bisect import IslandParty + from hypertrain.protocol.messages_v2 import IslandJobV1 + + geometry = self._record_v2(run_id, "trace-geometry", contest.verdict_hash) + job = IslandJobV1.model_validate(geometry["job"]) + if job.run_id != run_id or job.w != contest.w: + raise ChallengeError(409, "accepted trace geometry cross-run/round") + party = IslandParty.published(contest.miner, job, Path(str(geometry["directory"]))) + return party.span(level, ctx) + + def admission_v2( + self, run_id: str, action: str, raw: bytes = b"", identity: str = "", ip_prefix: str = "" + ) -> dict[str, Any]: + escrow, admission, disputes = self._services(run_id) + failure: ChallengeError | None = None + result: dict[str, Any] = {} + with self._tx(): + manifest = self._run_v2(run_id) + try: + now = self._fresh_v2(manifest, f"admission:{action}:{self._now(self._db)}") + except ChallengeError as error: + failure = error # Persist infrastructure pause, do not roll it back. + if failure is None: + match action: + case "join": + result = asdict(admission.join(raw, now=now, ip_prefix=ip_prefix)) + case "challenge": + result = admission.challenge(identity, now=now) + case "status": + result = asdict(admission.status(identity, now=now)) + screen = self._db.execute( + "SELECT screen_json FROM admissions_v2 WHERE hotkey=?", (identity,) + ).fetchone() + result["work_screen_hash"] = ( + WorkScreenV2.model_validate_json(screen[0]).digest() + if screen and screen[0] + else None + ) + case "rotate": + result = asdict(admission.rotate(raw, now=now)) + case "recover": + env = envelope_v2.parse_envelope(raw) + from hypertrain.protocol.messages import Receipt + + auth = self._authenticated_v2(run_id, raw, "Receipt", env.signer) + assert isinstance(auth, Receipt) + row = self._db.execute( + "SELECT p.dispute_id FROM admission_pending p JOIN admissions_v2 a " + "ON a.admission_id=p.admission_id WHERE a.hotkey=? AND p.resolved=0", + (env.signer,), + ).fetchone() + if row is None or row[0] != auth.commit_hash: + raise ChallengeError( + 403, "recovery reference differs from exact current pending dispute" + ) + admission.resolve_pending(env.signer, auth.commit_hash, now=now) + result = asdict(admission.status(env.signer, now=now)) + case "lock": + env = envelope_v2.parse_envelope(raw) + from hypertrain.protocol.messages_v2 import EscrowLock + + op = EscrowLock.model_validate(env.body) + if op.kind == "LOCK_ADMISSION": + result = {"event_hash": admission.lock(raw, now=now)} + else: + self._authenticated_v2(run_id, raw, "EscrowLock", op.owner) + result = escrow.lock(op, signer=env.signer).body() + case "transfer" | "release": + from hypertrain.protocol.messages_v2 import EscrowRelease, EscrowTransfer + + env = envelope_v2.parse_envelope(raw) + expected = "EscrowTransfer" if action == "transfer" else "EscrowRelease" + model = self._authenticated_v2(run_id, raw, expected, env.signer) + if action == "transfer": + assert isinstance(model, EscrowTransfer) + result = escrow.transfer(model, signer=env.signer).body() + else: + assert isinstance(model, EscrowRelease) + result = escrow.release(model, signer=env.signer, now_beacon=now).body() + case "dispute" | "bisect" | "resolution" | "state-serve": + methods = { + "dispute": disputes.open, + "bisect": disputes.bisect, + "resolution": disputes.resolution, + "state-serve": disputes.state_serve, + } + result = methods[action](raw, now=now).model_dump(mode="json") + if action == "dispute": + admission.suspend( + result["contest"]["miner"], + now=now, + reason="PENDING_DISPUTE", + pending_dispute=True, + dispute_id=result["dispute_id"], + evidence_hash=result["contest"]["evidence_hash"], + ) + if action in ("dispute", "resolution"): + w = result["contest"]["w"] + finality = self._record_v2(run_id, "finality", str(w)) + pending = self._db.execute( + "SELECT COUNT(*) FROM disputes_v2 " + "WHERE json_extract(turn,'$.contest.w')=? " + "AND json_extract(turn,'$.resolution') IS NULL", + (w,), + ).fetchone()[0] + finality["unresolved_disputes"] = pending + self._put_record_v2(run_id, "finality", str(w), finality) + if action == "resolution": + disputes.settle_lock(result["dispute_id"], now=now) + self._put_record_v2( + run_id, + "resolution", + result["dispute_id"], + envelope_v2.parse_envelope(raw).model_dump(mode="json"), + ) + case _: + raise ChallengeError(404, "unknown v2 operation") + if failure is not None: + raise failure + disputes.notify() # Only after outer acceptance transaction commits. + self._notify_processes_v2() + return result + + def _authenticated_v2(self, run_id: str, raw: bytes, expected: str, signer: str) -> BaseModel: + env = envelope_v2.parse_envelope(raw) + if env.type != expected: + raise ChallengeError(400, f"expected {expected}") + model = envelope_v2.Intake(run_id, {expected: signer.__eq__}).accept( + raw, self._now(self._db) + ) + key = "auth:" + _dumps(list(envelope_v2.replay_key(expected, run_id, env.signer, model))) + digest = body_digest(env.body) + known = self._db.execute("SELECT digest FROM accepted_v2 WHERE key=?", (key,)).fetchone() + if known is not None and known[0] != digest: + raise ChallengeError(409, "conflicting signed operation nonce") + if known is None: + self._db.execute( + "INSERT INTO accepted_v2 VALUES(?,?,?,?,?)", + (key, digest, raw, self._now(self._db), "{}"), + ) + return model + + def trial_proof_v2(self, run_id: str, proof: bytes, screen: bytes) -> dict[str, Any]: + with self._tx(): + _, admission, _ = self._services(run_id) + now = self._fresh_v2(self._run_v2(run_id), f"proof:{self._now(self._db)}") + return asdict(admission.proof(proof, screen, now=now)) + + def retain_runtime_custody_v2( + self, + run_id: str, + runtime: NetworkRuntime, + spec: Operation, + directory: Path, + *, + tree: Path, + cancel: threading.Event | None = None, + ) -> dict[str, str]: + """Internal trusted-runtime intake; no HTTP route or production authority.""" + with self._tx(): + _, admission, _ = self._services(run_id) + return admission.retain_runtime_custody( + runtime, spec, directory, tree=tree, now=self._now(self._db), cancel=cancel + ) + + def trial_reference_v2( + self, run_id: str, admission_id: str, *, retain_shadow_custody: bool = False + ) -> dict[str, str]: + from hypertrain.auditor.worker import LeaseGuard + + _, admission, _ = self._services(run_id) + with self._lock: + now = self._fresh_v2( + self._run_v2(run_id), f"reference:{admission_id}:{self._now(self._db)}" + ) + if self._role_launch is None: + value = admission.prepare_reference( + admission_id, now=now, retain_shadow_custody=retain_shadow_custody + ) + else: + current = admission.store.db.execute( + "SELECT * FROM admissions_v2 WHERE admission_id=?", (admission_id,) + ).fetchone() + if current is None or current["challenge"] is None: + raise ChallengeError(409, "reference lacks accepted challenge") + challenge = JoinChallenge.model_validate_json(current["challenge"]) + cutoff = ( + self._run_v2(run_id).training.beacon.genesis_time + + (challenge.deadline_beacon - 1) * 3 + ) + guard = LeaseGuard(challenge.deadline_beacon, challenge.deadline_beacon, cutoff) + key = f"reference:{admission_id}:{challenge.nonce}" + + def trusted( + job: IslandJobV1, directory: Path, c: JoinChallenge, epoch: int + ) -> IslandLaunch: + assert self._role_launch is not None + if c != challenge or job.deadline != cutoff: + raise ChallengeError(409, "reference launch subject changed") + return self._role_launch( + "reference", + job, + directory, + { + "run_id": run_id, + "admission_id": admission_id, + "epoch": epoch, + "challenge": c.body(), + "challenge_hash": c.digest(), + "challenge_envelope": json.loads( + admission.store.db.execute( + "SELECT receipt FROM admission_reservations " + "WHERE reservation=?", + (f"challenge|{admission_id}|{c.nonce}",), + ).fetchone()[0] + ), + "hotkey": current["hotkey"], + }, + ) + + self._lease_guards_v2[key] = guard + try: + with guard: + guard.advance(now) + guard.check() + value = admission.prepare_reference( + admission_id, + now=now, + trusted_launch=trusted, + cancel=guard.cancelled, + retain_shadow_custody=retain_shadow_custody, + ) + guard.check() + finally: + self._lease_guards_v2.pop(key, None) + return {"reference_commitment": value} + + def trial_finalize_v2(self, run_id: str, admission_id: str, raw: bytes) -> dict[str, Any]: + with self._tx(): + _, admission, _ = self._services(run_id) + now = self._fresh_v2(self._run_v2(run_id), f"trial-final:{self._now(self._db)}") + return asdict(admission.finalize_trial(admission_id, raw, now=now)) + + def _shadow_trial_v2( + self, run_id: str, admission_id: str, raw: bytes, *, accepted_at: int | None = None + ) -> tuple: + """Revalidate original accepted custody without reserving or issuing.""" + from hypertrain.auditor.replay import unpack_state + from hypertrain.miner.island_launch import confined, validate_artifacts + from hypertrain.protocol.messages_v2 import CommitV2, WorkProof + from hypertrain.trainer.compress import state_hash + + request = ShadowReservationIntake.model_validate(envelope_v2.load_json(raw)) + escrow, admission, _ = self._services(run_id) + with self._lock: + now = self._now(self._db) if accepted_at is None else accepted_at + row = self._db.execute( + "SELECT * FROM admissions_v2 WHERE admission_id=? AND run_id=?", + (admission_id, run_id), + ).fetchone() + if row is None: + raise ChallengeError(409, "shadow admission missing") + final = envelope_v2.Intake(run_id, {"Finalize": self._coord().ss58.__eq__}).accept( + request.finalize.model_dump(mode="json"), now + ) + commit = envelope_v2.Intake(run_id, {"CommitV2": row["hotkey"].__eq__}).accept( + request.commit.model_dump(mode="json"), now + ) + if not isinstance(final, Finalize) or not isinstance(commit, CommitV2): + raise ChallengeError(422, "expected original Finalize and CommitV2") + trial = self._db.execute( + "SELECT * FROM admission_trials WHERE epoch=? AND admission_id=?", + (final.w, admission_id), + ).fetchone() + result = self._db.execute( + "SELECT * FROM admission_trial_results WHERE epoch=?", (final.w,) + ).fetchone() + accepted_final = self._db.execute( + "SELECT digest FROM admission_reservations WHERE reservation=?", + (f"trial-final|{admission_id}|{final.w}",), + ).fetchone() + if ( + trial is None + or trial["finalized_beacon"] is None + or result is None + or accepted_final is None + or accepted_final[0] != body_digest(final.model_dump(mode="json")) + or result["reference"] is None + or result["proof"] is None + or result["screen"] is None + ): + raise ChallengeError(409, "shadow original accepted trial missing") + challenge = JoinChallenge.model_validate_json(result["challenge"]) + challenge_receipt = self._db.execute( + "SELECT receipt FROM admission_reservations WHERE reservation=?", + (f"challenge|{admission_id}|{challenge.nonce}",), + ).fetchone() + if ( + challenge_receipt is None + or envelope_v2.Intake(run_id, {"JoinChallenge": self._coord().ss58.__eq__}).accept( + challenge_receipt[0], trial["received_beacon"] + ) + != challenge + ): + raise ChallengeError(409, "shadow original signed challenge differs") + reference = WorkProof.model_validate_json(result["reference"]) + proof_env = envelope_v2.parse_envelope(result["proof"]) + proof = envelope_v2.Intake(run_id, {"WorkProof": row["hotkey"].__eq__}).accept( + proof_env.model_dump(mode="json"), trial["finalized_beacon"] + ) + screen = WorkScreenV2.model_validate( + envelope_v2.Intake(run_id, {"WorkScreenV2": row["hotkey"].__eq__}) + .accept(result["screen"], trial["finalized_beacon"]) + .model_dump(mode="json") + ) + job = IslandJobV1.model_validate_json(self.objects.get(request.custody.job_hash)) + if ( + job.manifest != escrow.manifest + or job.run_id != run_id + or job.w != final.w + or challenge.admission_id != admission_id + or challenge.nonce != trial["nonce"] + or challenge.manifest_hash != run_id + or proof != reference + or job.deadline + != escrow.manifest.training.beacon.genesis_time + + (challenge.deadline_beacon - 1) * 3 + or job.manifest.training.reference_spec.layout != challenge.layout + or trial["evidence_hash"] != reference.digest() + or reference.challenge_hash != challenge.digest() + or screen.challenge_hash != challenge.digest() + or screen.nonce != challenge.nonce + or not admission.qualified(screen) + or challenge.assignment_hash + != trial_assignment_hash(job.manifest, job.w, tuple(job.sample_ids)) + or commit.hotkey != row["hotkey"] + or commit.w != final.w + or final.included != [row["hotkey"]] + or final.entitlements_root != reference.digest() + or commit.leaves_root != reference.leaves_root + or commit.delta_hash != reference.delta_hash + ): + raise ChallengeError(409, "shadow original work binding differs") + from hypertrain.data.trial_assignment import trial_samples + + if tuple(job.sample_ids) != trial_samples( + job.manifest, + admission_id, + challenge.nonce, + self._beacon_v2(challenge.seed_beacon), + ): + raise ChallengeError(409, "shadow original ordered samples differ") + commit.validate_assignment(job.manifest, len(job.sample_ids)) + records = [] + for prefix, operation in (("reference", "reference"), ("miner", "probe")): + operation_hash = getattr(request.custody, prefix + "_operation_hash") + accepted = self._record_v2(run_id, "shadow-execution", operation_hash) + operation_body = envelope_v2.load_json(self.objects.get(operation_hash)) + if operation_body != {k: v for k, v in accepted.items() if k != "directory"}: + raise ChallengeError(409, "shadow accepted operation object differs") + publication_hash = getattr(request.custody, prefix + "_publication_hash") + result_hash = getattr(request.custody, prefix + "_result_hash") + if ( + accepted["operation"] != operation + or accepted["run_id"] != run_id + or accepted["admission_id"] != admission_id + or accepted["epoch"] != job.w + or accepted["challenge_hash"] != challenge.digest() + or accepted["job_hash"] != request.custody.job_hash + or accepted["publication_hash"] != publication_hash + or accepted["result_hash"] != result_hash + or accepted["backend"] != admission.backend + or accepted["accepted_beacon"] > trial["finalized_beacon"] + ): + raise ChallengeError(409, "shadow accepted execution binding differs") + publication = envelope_v2.load_json(self.objects.get(publication_hash)) + original_result = envelope_v2.load_json(self.objects.get(result_hash)) + if ( + original_result["job_hash"] != request.custody.job_hash + or original_result["publication_hash"] != publication_hash + or WorkProof.model_validate(original_result["proof"]) != reference + or operation == "probe" + and original_result.get("commit_hash") + != body_digest(commit.model_dump(mode="json")) + ): + raise ChallengeError(409, "shadow original result differs") + retained_screen = WorkScreenV2.model_validate(original_result["screen"]) + if not admission.qualified(retained_screen) or ( + retained_screen.challenge_hash != challenge.digest() + or retained_screen.admission_id != admission_id + or retained_screen.nonce != challenge.nonce + or retained_screen.artifact_hashes + != [a.sha256 for a in reference.artifact_refs] + ): + raise ChallengeError(409, "shadow retained qualified screen differs") + if operation == "probe": + original_commit = envelope_v2.Intake( + run_id, {"CommitV2": row["hotkey"].__eq__} + ).accept( + canonicalize(original_result["commit_envelope"]), trial["finalized_beacon"] + ) + if original_commit != commit: + raise ChallengeError(409, "shadow original accepted miner commit differs") + directory = Path(accepted["directory"]) + if directory.is_symlink() or not directory.resolve().is_relative_to( + self.state_dir.resolve() + ): + raise ChallengeError(409, "shadow accepted publication outside store custody") + actual_files = {} + for path in directory.rglob("*"): + if path.is_symlink() or not (path.is_file() or path.is_dir()): + raise ChallengeError(409, "shadow publication special file") + if path.is_file(): + data = confined(directory, str(path.relative_to(directory))).read_bytes() + actual_files[str(path.relative_to(directory))] = { + "sha256": sha256_hex(data), + "bytes": len(data), + } + if actual_files != publication: + raise ChallengeError(409, "shadow original publication changed") + for metadata in publication.values(): + assert isinstance(metadata, dict) and isinstance(metadata["sha256"], str) + data = self.objects.get(metadata["sha256"]) + if sha256_hex(data) != metadata["sha256"] or len(data) != metadata["bytes"]: + raise ChallengeError(409, "shadow addressed publication changed") + artifacts = validate_artifacts(job, directory) + if tuple(retained_screen.rank_results) != artifacts.ranks: + raise ChallengeError(409, "shadow original rank screen differs") + if any( + envelope_v2.load_json( + confined(directory, f"rank-{rank}/summary.json").read_bytes() + )["backend"] + != admission.backend + for rank in range(len(artifacts.ranks)) + ): + raise ChallengeError(409, "shadow original rank backend differs") + summary = envelope_v2.load_json((directory / "rank-0/summary.json").read_bytes()) + commitments = summary["commitments"] + assert isinstance(commitments, dict) + if ( + summary["backend"] != admission.backend + or commitments["final_theta_hash"] != commit.final_theta_hash + or commitments["ef_out_hash"] != commit.ef_out_hash + or artifacts.delta.stat().st_size != commit.delta_bytes + or state_hash(unpack_state(artifacts.state.read_bytes())[0]) + != final.final_theta_hash_w1 + or state_hash( + unpack_state( + confined(directory, job.object_paths["start_state"]).read_bytes() + )[0] + ) + != challenge.theta_hash + or state_hash( + unpack_state(confined(directory, job.object_paths["ef_in"]).read_bytes())[0] + ) + != commit.ef_in_hash + ): + raise ChallengeError(409, "shadow final commitment differs") + records.append(accepted) + if ( + request.custody.reference_operation_hash == request.custody.miner_operation_hash + or records[0]["directory"] == records[1]["directory"] + or records[0]["accepted_beacon"] > records[1]["accepted_beacon"] + ): + raise ChallengeError(409, "shadow independent execution missing") + graph_hash = body_digest({"manifest": escrow.manifest.body()}) + if escrow.policy.ledger_mode == "test": + economic_hash = escrow.policy.digest() + else: + authority = self._shadow_operator_v2(run_id, historical=accepted_at is not None) + identities = authority["workload"]["identities"] + identity = next( + ( + i + for i in identities + if (i["hotkey"], i["coldkey"]) == (row["hotkey"], row["coldkey"]) + ), + None, + ) + if identity is None or trial["finalized_beacon"] > authority["expires_beacon"]: + raise ChallengeError(403, "operator reservation roster/time differs") + for accepted in records: + if ( + accepted.get("operator_authority_hash") != authority["hash"] + or accepted.get("graph_hash") != authority["graph_hash"] + or accepted.get("source_map_hash") != authority["source_map_hash"] + or accepted.get("host") != authority["hosts"][identity["role"]] + or type(accepted.get("completed_unix")) is not int + or accepted["completed_unix"] > authority["cutoff_unix"] + ): + raise ChallengeError(403, "operator original execution custody differs") + graph_hash, economic_hash = authority["graph_hash"], authority["hash"] + binding = ShadowReservationRequestV1( + run_id=run_id, + admission_id=admission_id, + hotkey=row["hotkey"], + coldkey=row["coldkey"], + trial_epoch=final.w, + commit_hash=body_digest(commit.model_dump(mode="json")), + finalize_hash=body_digest(final.model_dump(mode="json")), + custody_hash=request.custody.digest(), + graph_hash=graph_hash, + economic_admission_hash=economic_hash, + nonce=challenge.nonce, + assignment_hash=challenge.assignment_hash, + finalized_beacon=trial["finalized_beacon"], + mature_at=trial["finalized_beacon"] + escrow.manifest.training.verify.E_vest_rounds, + ) + return binding, request, job, dict(trial), dict(result), records, final + + def shadow_reservation_v2(self, run_id: str, admission_id: str, raw: bytes) -> dict[str, Any]: + """Authenticate original accepted work; reserve only, never issue units.""" + binding, request, job, trial, result, records, final = self._shadow_trial_v2( + run_id, admission_id, raw + ) + escrow, admission, _ = self._services(run_id) + with self._lock: + now = self._now(self._db) + row = admission.store.by_id(admission_id) + semantic_key = f"shadow-reserve|{run_id}|{admission_id}|{final.w}" + snapshots = [ + ("SELECT * FROM admission_trials WHERE epoch=?", final.w, dict(trial)), + ("SELECT * FROM admission_trial_results WHERE epoch=?", final.w, dict(result)), + ] + store = self + + class ReservationEscrow(EscrowV2): + _reserving_shadow: bool = False + + def reserve_shadow( + self, + request: ShadowReservationRequestV1, + evidence: ShadowReservationEvidence, + ) -> ShadowReservationV1: + # This disposable instance alone enables the guard for this one call. + self._reserving_shadow = True + try: + return super().reserve_shadow(request, evidence) + finally: + self._reserving_shadow = False + + @contextmanager + def tx(self) -> Iterator[sqlite3.Connection]: + # Constructor/verification use original tx semantics, without freshness writes. + with super().tx() as db: + if not self._reserving_shadow: + yield db + return + for query, value, expected in snapshots: + current_row = db.execute(query, (value,)).fetchone() + if current_row is None or dict(current_row) != expected: + raise ChallengeError( + 409, "shadow accepted trial changed before reservation" + ) + for accepted in records: + operation_hash = body_digest( + {k: v for k, v in accepted.items() if k != "directory"} + ) + if ( + store._record_v2(run_id, "shadow-execution", operation_hash) + != accepted + ): + raise ChallengeError( + 409, "shadow execution changed before reservation" + ) + old = db.execute( + "SELECT digest FROM admission_reservations WHERE reservation=?", + (semantic_key,), + ).fetchone() + if old is None: + if result["outcome"] != "MATCH": + raise ChallengeError( + 409, "shadow new reservation requires original MATCH" + ) + store._fresh_v2( + self.manifest, f"shadow-reservation:{admission_id}:{now}" + ) + current = admission.store.by_id(admission_id) + first = db.execute( + "SELECT MIN(epoch) FROM admission_trials WHERE admission_id=? " + "AND finalized_beacon IS NOT NULL", + (admission_id,), + ).fetchone()[0] + if ( + current.clean_count != 1 + or first != final.w + or current.pending_dispute + or db.execute( + "SELECT 1 FROM admission_pending WHERE admission_id=? " + "AND resolved=0", + (admission_id,), + ).fetchone() + is not None + ): + raise ChallengeError( + 409, "shadow new reservation requires settled first trial" + ) + yield db + if old is None: + store._put_record_v2( + run_id, + "shadow-custody", + semantic_key, + { + "request": request.model_dump(mode="json"), + "accepted_at": now, + }, + ) + + intake = ReservationEscrow( + self._db, + escrow.manifest, + escrow.policy, + auditors=escrow.auditors, + replay_finality=escrow.replay_finality, + settlement=escrow.settlement, + owner_of=escrow.owner_of, + assignment_of=escrow.assignment_of, + ) + intake.mutex = self._lock + reserved = intake.reserve_shadow( + binding, + ShadowReservationEvidence( + finalize=request.finalize, + commit=request.commit, + owner=row.coldkey, + sample_ids=tuple(job.sample_ids), + custody_hash=request.custody.digest(), + reference_final_state_hash=final.final_theta_hash_w1, + ), + ) + return { + "reservation": reserved.body(), + "reservation_hash": reserved.digest(), + "shadow_ordinal": reserved.shadow_ordinal, + } + + def _shadow_binding_v2(self, run_id: str, reservation_hash: str) -> ShadowReservationV1: + rows = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='shadow-reservation'", + (run_id,), + ).fetchall() + for row in rows: + binding = ShadowReservationV1.model_validate(json.loads(row[0])["binding"]) + if binding.digest() == reservation_hash: + semantic = self._db.execute( + "SELECT receipt FROM admission_reservations WHERE reservation=?", + (f"shadow-reserve|{run_id}|{binding.admission_id}|{binding.trial_epoch}",), + ).fetchone() + if ( + semantic is None + or ShadowReservationV1.model_validate_json(semantic[0]) != binding + ): + raise ChallengeError(409, "shadow semantic reservation differs") + return binding + raise ChallengeError(409, "shadow reservation missing") + + def _shadow_custody_v2(self, run_id: str, binding: ShadowReservationV1) -> tuple: + record = self._record_v2( + run_id, + "shadow-custody", + f"shadow-reserve|{run_id}|{binding.admission_id}|{binding.trial_epoch}", + ) + validated = self._shadow_trial_v2( + run_id, + binding.admission_id, + canonicalize(record["request"]), + accepted_at=record["accepted_at"], + ) + if validated[0].body() != binding.model_dump(mode="json", exclude={"shadow_ordinal"}): + raise ChallengeError(409, "shadow original reservation custody differs") + return validated + + def _shadow_assignment_v2(self, run_id: str, binding: ShadowReservationV1) -> tuple[int, ...]: + job = self._shadow_custody_v2(run_id, binding)[2] + return tuple(job.sample_ids) + + def _shadow_status_v2(self, run_id: str, binding: ShadowReservationV1) -> Settlement: + """Resolve current disputes from accepted store records, never supplied flags.""" + admission = self._services(run_id)[1].store.by_id(binding.admission_id) + trial = self._db.execute( + "SELECT outcome FROM admission_trial_results WHERE epoch=?", (binding.trial_epoch,) + ).fetchone() + if trial is None: + raise EscrowError("SHADOW_TRIAL_MISSING") + unresolved = admission.pending_dispute or trial[0] != "MATCH" + fraud = admission.state == "BANNED" or trial[0] == "FRAUD" + release = binding.mature_at + closed = [] + seen = set() + for row in self._db.execute( + "SELECT * FROM admission_pending WHERE admission_id=?", (binding.admission_id,) + ).fetchall(): + if not row["resolved"]: + unresolved = True + continue + status = self._settlement_v2(run_id, row["dispute_id"]) + if ( + status.run_id, + status.admission_id, + status.coldkey, + status.dispute_id, + status.evidence_hash, + ) != ( + run_id, + binding.admission_id, + binding.coldkey, + row["dispute_id"], + row["evidence_hash"], + ): + raise EscrowError("SHADOW_DISPUTE_LINKAGE") + unresolved = unresolved or status.unresolved + fraud = fraud or status.outcome == "FRAUD" + release = max(release, status.release_beacon) + closed.append([row["dispute_id"], status.closed_dispute_root]) + seen.add(row["dispute_id"]) + for row in self._db.execute( + "SELECT id FROM disputes_v2 WHERE json_extract(turn,'$.contest.admission_id')=? " + "OR (json_extract(turn,'$.contest.miner')=? AND json_extract(turn,'$.contest.w')=?)", + (binding.admission_id, binding.hotkey, binding.trial_epoch), + ).fetchall(): + if row[0] in seen: + continue + status = self._services(run_id)[2].settlement(row[0]) + unresolved = unresolved or status.unresolved + fraud = fraud or status.outcome == "FRAUD" + release = max(release, status.release_beacon) + closed.append([row[0], status.closed_dispute_root]) + return Settlement( + finality_hash=binding.finalize_hash, + closed_dispute_root=sha256_hex(canonicalize(sorted(closed))), + release_beacon=release, + unresolved=unresolved, + outcome="FRAUD" if fraud else "MATCH", + run_id=run_id, + admission_id=binding.admission_id, + coldkey=binding.coldkey, + ) + + def _shadow_settlement_v2(self, run_id: str, origin: str) -> Settlement: + row = self._db.execute( + "SELECT reservation_hash FROM escrow_shadow_finalized WHERE origin=?", (origin,) + ).fetchone() + if row is None: + raise EscrowError("SHADOW_FINALITY_MISSING") + binding = self._shadow_binding_v2(run_id, row[0]) + self._record_v2(run_id, "shadow-evidence", binding.digest()) + self._shadow_custody_v2(run_id, binding) + return self._shadow_status_v2(run_id, binding) + + def _shadow_anchor_v2(self, job: IslandJobV1, hotkey: str, directory: Path) -> str: + """Derive original trial genesis anchor from actual start/optimizer/EF bytes.""" + from hypertrain.auditor.replay import AnchorCache, optimizer_hash, unpack_state + from hypertrain.miner.island_launch import confined + from hypertrain.protocol.messages_v2 import StartStateV2 + from hypertrain.trainer.compress import state_hash + + theta, carried = unpack_state( + confined(directory, job.object_paths["start_state"]).read_bytes() + ) + genesis = AnchorCache().genesis(job.manifest, hotkey, theta) + if carried is not None and optimizer_hash(carried) != optimizer_hash(genesis.state): + raise ChallengeError(409, "shadow original trial optimizer differs from genesis") + ef, _ = unpack_state(confined(directory, job.object_paths["ef_in"]).read_bytes()) + if job.global_step0 != 0 or state_hash(ef) != state_hash(genesis.ef): + raise ChallengeError(409, "shadow original trial EF/start differs from genesis") + return StartStateV2( + run_id=job.run_id, + w=job.w, + hotkey=hotkey, + theta_hash=state_hash(theta), + state_object_sha256=job.start_state_sha256, + opt_state_hash=optimizer_hash(genesis.state), + ef_object_sha256=job.ef_in_sha256, + ef_hash=state_hash(ef), + parent_anchor_hash=genesis.anchor_hash, + global_step0=0, + anchor_verdict_hash=genesis.proof_hash, + ).digest() + + def _shadow_replay_v2(self, run_id: str, evidence: FinalityEvidence) -> str: + """Read accepted signed receipt and revalidate the original independent publication.""" + signed = envelope_v2.parse_envelope(self.objects.get(evidence.tape_hash)) + receipt = ShadowReplayReceiptV1.model_validate(signed.body) + binding = self._shadow_binding_v2(run_id, receipt.reservation_hash) + record = self._record_v2(run_id, "shadow-evidence", binding.digest()) + if ( + record["auditor_receipt_hash"] != evidence.tape_hash + or not evidence.shadow + or evidence.finalized_beacon != binding.finalized_beacon + or body_digest(evidence.finalize.body) != binding.finalize_hash + ): + raise EscrowError("SHADOW_REPLAY_LINKAGE") + request = ShadowAcceptanceIntake.model_validate( + { + "reservation_hash": binding.digest(), + "audit_challenge": envelope_v2.load_json( + self.objects.get(record["audit_challenge_hash"]) + ), + "verdict": envelope_v2.load_json(self.objects.get(record["verdict_hash"])), + "auditor_receipt": signed.model_dump(mode="json"), + "reward": record["reward"], + } + ) + validated, _, _, _ = self._validate_shadow_acceptance_v2( + run_id, binding, request, record["accepted_at"], historical=True + ) + if ( + len(evidence.works) != 1 + or body_digest(evidence.works[0].commit.body) != binding.commit_hash + or evidence.works[0].owner != binding.coldkey + or evidence.works[0].sample_ids != tuple(validated[2].sample_ids) + or body_digest(evidence.works[0].verdict.body) != body_digest(request.verdict.body) + or body_digest(evidence.works[0].challenge.body) + != body_digest(request.audit_challenge.body) + ): + raise EscrowError("SHADOW_REPLAY_WORK_LINKAGE") + status = self._shadow_status_v2(run_id, binding) + if status.unresolved or status.outcome == "FRAUD": + raise EscrowError("SHADOW_UNSETTLED") + # State comes from the independently executed reference, not a signed MATCH scalar. + from hypertrain.auditor.replay import unpack_state + from hypertrain.trainer.compress import state_hash + + reference = Path(validated[5][0]["directory"]) / "rank-0/state.safetensors" + return state_hash(unpack_state(reference.read_bytes())[0]) + + def _validate_shadow_acceptance_v2( + self, + run_id: str, + binding: ShadowReservationV1, + request: ShadowAcceptanceIntake, + now: int, + *, + historical: bool = False, + ) -> tuple: + from hypertrain.protocol.messages_v2 import AuditChallengeV2, WorkProof + + manifest = self._run_v2(run_id) + auditor = request.auditor_receipt.signer + if auditor not in manifest.training.auditors or auditor == binding.hotkey: + raise ChallengeError(403, "shadow distinct pinned auditor required") + audit = envelope_v2.Intake(run_id, {"AuditChallengeV2": self._coord().ss58.__eq__}).accept( + request.audit_challenge.model_dump(mode="json"), now + ) + verdict = envelope_v2.Intake(run_id, {"ReplayVerdict": auditor.__eq__}).accept( + request.verdict.model_dump(mode="json"), now + ) + receipt = envelope_v2.Intake(run_id, {"ShadowReplayReceiptV1": auditor.__eq__}).accept( + request.auditor_receipt.model_dump(mode="json"), now + ) + if ( + not isinstance(audit, AuditChallengeV2) + or not isinstance(verdict, ReplayVerdict) + or not isinstance(receipt, ShadowReplayReceiptV1) + ): + raise ChallengeError(422, "shadow evidence types differ") + self._services(run_id)[0]._authority(request.reward) + validated = self._shadow_custody_v2(run_id, binding) + _, original, job, trial, result, operations, final = validated + challenge = JoinChallenge.model_validate_json(result["challenge"]) + signed_challenge = self._db.execute( + "SELECT receipt FROM admission_reservations WHERE reservation=?", + (f"challenge|{binding.admission_id}|{challenge.nonce}",), + ).fetchone()[0] + reference = WorkProof.model_validate_json(result["reference"]) + miner = WorkProof.model_validate(envelope_v2.parse_envelope(result["proof"]).body) + expected = { + "run_id": run_id, + "admission_id": binding.admission_id, + "hotkey": binding.hotkey, + "coldkey": binding.coldkey, + "trial_epoch": binding.trial_epoch, + "shadow_ordinal": binding.shadow_ordinal, + "reservation_hash": binding.digest(), + "challenge_hash": challenge.digest(), + "signed_challenge_hash": sha256_hex( + canonicalize(envelope_v2.load_json(signed_challenge)) + ), + "assignment_hash": binding.assignment_hash, + "sample_ids_hash": sha256_hex(canonicalize(job.sample_ids)), + **original.custody.body(), + "reference_work_proof_hash": reference.digest(), + "miner_work_proof_hash": miner.digest(), + "commit_hash": binding.commit_hash, + "audit_challenge_hash": body_digest(audit.model_dump(mode="json")), + "verdict_hash": body_digest(verdict.model_dump(mode="json")), + "trial_finalize_hash": binding.finalize_hash, + "final_state_hash": final.final_theta_hash_w1, + "backend_qualification_authority_hash": self._shadow_backend_authority_v2( + run_id, historical=historical + ), + "completed_beacon": receipt.completed_beacon, + } + if receipt.body() != expected or ( + not binding.finalized_beacon <= receipt.completed_beacon <= now + or (audit.w, audit.target) != (binding.trial_epoch, binding.hotkey) + or audit.anchor_hash + != self._shadow_anchor_v2(job, binding.hotkey, Path(operations[0]["directory"])) + or audit.mode != "full" + or audit.segments + or audit.audit_mode != "anchored-full" + or audit.beacon_round < challenge.seed_beacon + or audit.beacon_round > receipt.completed_beacon + or audit.beacon_sig_sha256 + != sha256_hex(bytes.fromhex(self._beacon_v2(audit.beacon_round).signature)) + or not receipt.completed_beacon <= audit.serve_deadline <= challenge.deadline_beacon + or verdict.challenge_hash != body_digest(audit.model_dump(mode="json")) + or verdict.result != "MATCH" + or verdict.first_bad_leaf is not None + or verdict.recomputed_leaves_root != reference.leaves_root + or verdict.replay_env.image_digest != manifest.training.reference_spec.image_digest + or verdict.replay_env.driver not in manifest.training.reference_spec.driver_allowlist + or verdict.replay_env.sm_count != manifest.training.reference_spec.sm_count + or request.reward.reservation_hash != binding.digest() + or request.reward.shadow_ordinal != binding.shadow_ordinal + or request.reward.w != binding.trial_epoch + or request.reward.finalize_hash != binding.finalize_hash + or request.reward.mature_at != binding.mature_at + ): + raise ChallengeError(409, "shadow authenticated evidence binding differs") + return validated, audit, verdict, receipt + + def accept_shadow_operator_v2(self, graph_path: Path, runtime: NetworkRuntime) -> str: + """Private genuine-runtime intake, atomic and immutable. No provider or kernels.""" + from experiments.gpu_network_v2.orchestrate import NetworkRuntime, reviewed_launch + from scripts.network_service_proof import ( + ContinuationGraph, + continuation_graph, + continuation_graph_hash, + continuation_operator_record, + continuation_qualification, + ) + + if type(runtime) is not NetworkRuntime: + raise ChallengeError(403, "operator genuine runtime required") + untrusted = envelope_v2.load_json(graph_path.read_bytes()) + run_id = untrusted["run_id"] + assert isinstance(run_id, str) + owner = self.owner_hotkey + assert owner is not None + with self._tx(): + manifest = self._run_v2(run_id) + now = self._now(self._db) + if not self._beacon_v2(now).bls_verified: + raise ChallengeError(403, "operator verified beacon required") + graph = ContinuationGraph.model_validate(untrusted) + assert graph.economic_authority is not None + raw = graph.economic_authority.read_bytes() + existing = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? " + "AND kind='shadow-operator' AND id='run'", + (run_id,), + ).fetchone() + if existing is not None: + stored = json.loads(existing[0]) + record, digest, payload = shadow_operator_receipt( + raw, owner, run_id, stored["accepted_at"] + ) + if stored["payload"] != payload or stored["hash"] != digest: + raise ChallengeError(409, "operator lost-response authority changed") + if record.graph_hash != continuation_graph_hash(graph): + raise ChallengeError(409, "operator lost-response graph changed") + for role, frozen in record.hosts.items(): + assert isinstance(frozen, dict) + if runtime.owned_host(role) != { + key: frozen[key] for key in ("instance_id", "machine_id", "image_digest") + }: + raise ChallengeError(409, "operator lost-response host changed") + self._shadow_operator_v2(run_id, historical=True) + return digest + graph = continuation_graph(graph_path, runtime, owner, run_id, manifest) + record, digest, payload = shadow_operator_receipt(raw, owner, run_id, now) + policy = EconomicsPolicyV2.model_validate_json( + self.objects.get(manifest.network.economics_policy_hash) + ) + if policy.ledger_mode != "production" or self._backend_v2(manifest) != "cuda": + raise ChallengeError(409, "operator accepted production CUDA required") + if record.budget.get("scope") != "FULL116_CONTINUATION": + raise ChallengeError(403, "operator FIRST4 is not full116 authorization") + launch_path = Path(str(record.budget.get("launch_path", ""))) + if ( + not launch_path.is_absolute() + or launch_path.is_symlink() + or (sha256_hex(launch_path.read_bytes()) != record.budget.get("launch_sha256")) + ): + raise ChallengeError(403, "operator original signed launch custody differs") + checked = reviewed_launch(launch_path, owner, now, "continue") + continuation_qualification(self, manifest, graph, runtime, checked) + backend = self._record_v2( + run_id, "qualification", manifest.training.reference_spec.image_digest + ) + expected = continuation_operator_record( + graph, runtime, checked, backend, now, record.expires_beacon, launch_path + ) + shadow_operator_expected(record, expected, time.time()) + self._retain_shadow_operator_v2(run_id, payload, digest, now, graph_path) + return digest + + def _retain_shadow_operator_v2( + self, + run_id: str, + payload: dict[str, JsonValue], + digest: str, + accepted_at: int, + graph_path: Path, + ) -> None: + """Persistence only; caller must complete genuine admission predicates first.""" + with self._lock: + self._db.execute("SAVEPOINT shadow_operator_intake") + try: + self._retain_shadow_operator_record( + run_id, payload, digest, accepted_at, graph_path + ) + except BaseException: + self._db.execute("ROLLBACK TO shadow_operator_intake") + raise + finally: + self._db.execute("RELEASE shadow_operator_intake") + + def _retain_shadow_operator_record( + self, + run_id: str, + payload: dict[str, JsonValue], + digest: str, + accepted_at: int, + graph_path: Path, + ) -> None: + with self._lock: + old = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? " + "AND kind='shadow-operator' AND id='run'", + (run_id,), + ).fetchone() + body: dict[str, JsonValue] = { + "payload": payload, + "hash": digest, + "accepted_at": accepted_at, + "payload_hash": sha256_hex(canonicalize(payload)), + "graph_path": str(graph_path), + } + if old is not None and json.loads(old[0]) != body: + raise ChallengeError(409, "operator immutable retained authority conflicts") + if old is not None: + return + if self.objects.put(canonicalize(payload)) != body["payload_hash"]: + raise ChallengeError(403, "operator addressed payload hash differs") + self._put_record_v2( + run_id, + "shadow-operator", + "run", + body, + ) + + def _shadow_operator_v2(self, run_id: str, *, historical: bool = False) -> dict: + stored = self._record_v2(run_id, "shadow-operator", "run") + owner = self.owner_hotkey + assert owner is not None + record, digest, _ = shadow_operator_receipt( + canonicalize(stored["payload"]), owner, run_id, stored["accepted_at"] + ) + if self.objects.get(stored["payload_hash"]) != canonicalize(stored["payload"]): + raise ChallengeError(403, "operator addressed retained payload differs") + from scripts.network_service_proof import ContinuationGraph, continuation_graph_hash + + graph = ContinuationGraph.model_validate_json(Path(stored["graph_path"]).read_bytes()) + if continuation_graph_hash(graph) != record.graph_hash: + raise ChallengeError(403, "operator retained graph differs") + for name, pin in graph.private_files.items(): + path = Path(name) + if path.is_symlink() or sha256_hex(path.read_bytes()) != pin: + raise ChallengeError(403, "operator retained private input differs") + launch_path = record.budget["launch_path"] + assert isinstance(launch_path, str) + launch = envelope_v2.load_json(Path(launch_path).read_bytes()) + launch_record = launch["record"] + assert isinstance(launch_record, dict) + if ( + sha256_hex(canonicalize(launch_record)) != record.admission_hash + or sha256_hex(Path(launch_path).read_bytes()) != record.budget["launch_sha256"] + ): + raise ChallengeError(403, "operator retained launch differs") + budget_files, launch_files = record.budget["files"], launch_record["files"] + assert isinstance(budget_files, dict) and isinstance(launch_files, dict) + for name, digest_pin in budget_files.items(): + subject = launch_files[name] + assert isinstance(subject, dict) and isinstance(subject["path"], str) + path = Path(subject["path"]) + if path.is_symlink() or sha256_hex(path.read_bytes()) != digest_pin: + raise ChallengeError(403, "operator retained input binding differs") + assert isinstance(launch_record["tree"], str) + tree = Path(launch_record["tree"]) + source_profile = launch_files["source_profile"] + assert isinstance(source_profile, dict) and isinstance(source_profile["path"], str) + for name, pin in graph.operation_sources.items(): + source = ( + Path(source_profile["path"]) + if name == "experiments/gpu_network_v2/profile.json" + else tree / name + ) + if source.is_symlink() or sha256_hex(source.read_bytes()) != pin: + raise ChallengeError(403, "operator current operation source differs") + manifest = self._run_v2(run_id) + backend = self._record_v2( + run_id, "qualification", manifest.training.reference_spec.image_digest + ) + if ( + digest != stored["hash"] + or record.manifest_hash != manifest.run_id() + or record.economics_policy_hash != manifest.network.economics_policy_hash + or self._backend_v2(manifest) != "cuda" + or record.backend_qualification_authority_hash != backend["authority_hash"] + or not historical + and (self._now(self._db) > record.expires_beacon or time.time() >= record.cutoff_unix) + ): + raise ChallengeError(403, "operator retained qualification/cutoff differs") + return {**record.model_dump(mode="json"), "hash": digest} + + def _shadow_backend_authority_v2(self, run_id: str, *, historical: bool = False) -> str: + manifest = self._run_v2(run_id) + if ( + self._backend_v2(manifest) != "cpu" + or self._services(run_id)[0].policy.ledger_mode != "test" + ): + return self._shadow_operator_v2(run_id, historical=historical)[ + "backend_qualification_authority_hash" + ] + return body_digest( + { + "domain": "cpu-test-only", + "reference": manifest.training.reference_spec.model_dump(mode="json"), + } + ) + + def shadow_reward_v2(self, run_id: str, admission_id: str, raw: bytes) -> dict[str, Any]: + """Accept original evidence and conserved issuance in one shared SQLite transaction.""" + request = ShadowAcceptanceIntake.model_validate(envelope_v2.load_json(raw)) + escrow, admission, _ = self._services(run_id) + with self._tx(): + binding = self._shadow_binding_v2(run_id, request.reservation_hash) + if binding.admission_id != admission_id: + raise ChallengeError(409, "shadow acceptance admission differs") + now = self._now(self._db) + semantic = body_digest( + { + "reservation_hash": binding.digest(), + "audit_challenge": request.audit_challenge.body, + "verdict": request.verdict.body, + "auditor_receipt": request.auditor_receipt.body, + "reward": request.reward.model_dump(mode="json", exclude={"authority_sig"}), + } + ) + previous = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='shadow-evidence' AND id=?", + (run_id, binding.digest()), + ).fetchone() + if previous is not None: + retained = json.loads(previous[0]) + if retained["semantic_digest"] != semantic or "response" not in retained: + raise ChallengeError(409, "shadow acceptance semantic conflict") + try: + stored = ShadowAcceptanceIntake.model_validate( + { + "reservation_hash": binding.digest(), + "audit_challenge": envelope_v2.load_json( + self.objects.get(retained["audit_challenge_hash"]) + ), + "verdict": envelope_v2.load_json( + self.objects.get(retained["verdict_hash"]) + ), + "auditor_receipt": envelope_v2.load_json( + self.objects.get(retained["auditor_receipt_hash"]) + ), + "reward": retained["reward"], + } + ) + except StoreError as error: + raise ChallengeError( + 409, "shadow accepted evidence object unavailable" + ) from error + validated, _, _, _ = self._validate_shadow_acceptance_v2( + run_id, binding, stored, retained["accepted_at"], historical=True + ) + self._validate_shadow_acceptance_v2( + run_id, binding, request, retained["accepted_at"], historical=True + ) + stored_semantic = body_digest( + { + "reservation_hash": binding.digest(), + "audit_challenge": stored.audit_challenge.body, + "verdict": stored.verdict.body, + "auditor_receipt": stored.auditor_receipt.body, + "reward": stored.reward.model_dump(mode="json", exclude={"authority_sig"}), + } + ) + if ( + stored_semantic != semantic + or stored.reward.tape_hash != retained["auditor_receipt_hash"] + or retained["response"]["authority_hash"] != retained["auditor_receipt_hash"] + or retained["response"]["reservation_hash"] != binding.digest() + or retained["response"]["shadow_ordinal"] != binding.shadow_ordinal + ): + raise ChallengeError(409, "shadow accepted evidence semantic differs") + finalized = self._db.execute( + "SELECT reservation_hash FROM escrow_shadow_finalized WHERE ordinal=?", + (binding.shadow_ordinal,), + ).fetchone() + if finalized is None or finalized[0] != binding.digest(): + raise ChallengeError(409, "shadow accepted issuance missing") + original = validated[1] + recovered = escrow.reward_finalize( + request.reward, + FinalityEvidence( + original.finalize, + ( + RewardWork( + original.commit, + stored.verdict, + stored.audit_challenge, + binding.coldkey, + tuple(validated[2].sample_ids), + ), + ), + retained["auditor_receipt_hash"], + True, + binding.finalized_beacon, + binding.mature_at, + ), + ) + if recovered.body() != retained["response"]["escrow_receipt"]: + raise ChallengeError(409, "shadow accepted ledger receipt differs") + return retained["response"] + validated, _, _, _ = self._validate_shadow_acceptance_v2(run_id, binding, request, now) + current = admission.store.by_id(admission_id) + status = self._shadow_status_v2(run_id, binding) + if ( + validated[4]["outcome"] != "MATCH" + or current.clean_count != 1 + or status.unresolved + or status.outcome == "FRAUD" + ): + raise ChallengeError(409, "shadow acceptance requires settled first trial") + self._fresh_v2(escrow.manifest, f"shadow-acceptance:{binding.digest()}:{now}") + receipt_hash = self.objects.put( + canonicalize(request.auditor_receipt.model_dump(mode="json")) + ) + if request.reward.tape_hash != receipt_hash: + raise ChallengeError(409, "shadow reward original signed receipt differs") + record: dict[str, JsonValue] = { + "semantic_digest": semantic, + "accepted_at": now, + "audit_challenge_hash": self.objects.put( + canonicalize(request.audit_challenge.model_dump(mode="json")) + ), + "verdict_hash": self.objects.put( + canonicalize(request.verdict.model_dump(mode="json")) + ), + "auditor_receipt_hash": receipt_hash, + "reward": request.reward.body(), + } + self._put_record_v2(run_id, "shadow-evidence", binding.digest(), record) + original = validated[1] + evidence = FinalityEvidence( + original.finalize, + ( + RewardWork( + original.commit, + request.verdict, + request.audit_challenge, + binding.coldkey, + tuple(validated[2].sample_ids), + ), + ), + receipt_hash, + True, + binding.finalized_beacon, + binding.mature_at, + ) + ledger_receipt = escrow.reward_finalize(request.reward, evidence) + response: dict[str, JsonValue] = { + "reservation_hash": binding.digest(), + "shadow_ordinal": binding.shadow_ordinal, + "authority_hash": receipt_hash, + "escrow_receipt": ledger_receipt.body(), + "origin_ids": [origin for origin in request.reward.origin_ids], + "mature_at": binding.mature_at, + } + self._put_record_v2(run_id, "shadow-settlement", binding.digest(), status.body()) + self._put_record_v2( + run_id, "shadow-evidence", binding.digest(), {**record, "response": response} + ) + return response + + def run_status_v2(self, run_id: str) -> dict[str, Any]: + with self._lock: + self._run_v2(run_id) + row = self._db.execute("SELECT * FROM runs WHERE run_id=?", (run_id,)).fetchone() + return { + "run_id": run_id, + "contract": 2, + "status": row["status"], + "manifest_envelope": json.loads(row["envelope"]), + "now_round": self._now(self._db), + } + + def configure_inputs_v2(self, run_id: str, data: Mapping[str, JsonValue]) -> dict[str, str]: + """Administrative object linkage; samples still verify against the pinned dataset root.""" + with self._tx(): + self._run_v2(run_id) + if set(data) != {"samples_hash", "proofs_hash"}: + raise ChallengeError(422, "expected dataset object hashes") + for value in data.values(): + self.objects.get(str(value)) + self._put_record_v2(run_id, "dataset", "inputs", data) + return {"status": "configured"} + + def genesis_v2(self, run_id: str, raw: bytes) -> dict[str, JsonValue]: + from hypertrain.protocol.messages_v2 import TestGenesis + + escrow, _, _ = self._services(run_id) + with self._tx(): + model = self._authenticated_v2(run_id, raw, "TestGenesis", self._coord().ss58) + assert isinstance(model, TestGenesis) + receipt = escrow.test_genesis(model) + escrow.mature( + sha256_hex(b"ht-genesis-mature|" + run_id.encode()), + tuple(o.origin_id for o in model.origins), + self._now(self._db), + ) + return receipt.body() + + def _service_profile_v2(self, run_id: str) -> _ServiceCapacityV1: + """Derive the sole admitted tiny shape; never allocate tensors.""" + manifest = self._run_v2(run_id) + m, inner = manifest.training.model, manifest.training.inner + layout = manifest.training.reference_spec.layout + if ( + ( + m.arch, + m.compute_dtype, + m.n_layers, + m.d_model, + m.n_heads, + m.n_kv_heads, + m.d_ff, + m.vocab, + m.seq_len, + m.param_count, + ) + != ("decoder", "fp32", 1, 8, 2, 2, 8, 16, 2, 728) + or (layout.pp, layout.n_gpus, layout.dp_size, layout.ep_size, layout.zero1) + != (1, 1, 1, 1, False) + or ( + inner.H, + inner.J, + inner.micro_batch, + inner.grad_accum, + inner.opt, + inner.state_policy, + ) + != (2, 1, 1, 1, "adamw", "carry") + or (manifest.training.outer.opt != "nesterov") + ): + raise ChallengeError(409, "SERVICE_PROFILE_GEOMETRY") + if self._backend_v2(manifest) != "cpu": + raise ChallengeError(409, "SERVICE_PROFILE_BACKEND") + from hypertrain.aggregator.core import get_object + from hypertrain.protocol.messages_v2 import AdmissionPolicyV2, AggregationPolicyV2 + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.model import param_shapes + + policy = AggregationPolicyV2.model_validate_json( + get_object(self.objects, manifest.network.aggregation_policy_hash) + ) + if policy.cclip_iters != 1: + raise ChallengeError(409, "SERVICE_PROFILE_CCLIP") + admission_policy = AdmissionPolicyV2.model_validate_json( + get_object(self.objects, manifest.network.admission_policy_hash) + ) + if admission_policy.artifact_limits.max_object_bytes < 65536: + raise ChallengeError(409, "SERVICE_PROFILE_OBJECT_LIMIT") + shapes = param_shapes(TrainConfig.from_manifest_v2(manifest).model) + if sum(math.prod(shape) for shape in shapes.values()) != 728 or len(shapes) != 12: + raise ChallengeError(409, "SERVICE_PROFILE_SHAPES") + paths = ( + "aggregator/core.py", + "aggregator/tape_v2.py", + "aggregator/rollback_v2.py", + "trainer/compress.py", + "trainer/config.py", + "trainer/model.py", + "miner/island_launch.py", + "aggregator/capacity_worker.py", + "auditor/replay.py", + ) + source = Path(__file__).resolve().parents[1] + implementation = self._service_implementation_v2(source, paths) + backend = self._record_v2(run_id, "execution-backend", "run") + return _ServiceCapacityV1( + version=1, + profile_id="tiny-cpu-service-32-v1", + run_id=run_id, + backend_binding_hash=sha256_hex(canonicalize(backend)), + implementation_hash=implementation, + max_complete_roster=32, + memory_reservation_bytes=1 << 30, + max_outer_work_units=50_000_000, + max_audit_step_units=512, + max_repair_step_units=1024, + max_object_bytes=65536, + max_tape_bytes=1 << 20, + ) + + @staticmethod + def _service_implementation_v2(source: Path, paths: tuple[str, ...]) -> str: + """Bind capacity semantics, not unrelated shared-store methods or line numbers.""" + tree = ast.parse((source / "challenge/store.py").read_text()) + owned = { + "_ServiceCapacityV1", + "_service_profile_v2", + "_service_implementation_v2", + "bootstrap_service_capacity_v2", + "_service_admission_v2", + "require_roster_v2", + "_service_boundary_v2", + "_service_bytes_v2", + "_service_charge_v2", + "_backend_v2", + "open_round_v2", + "_assignment_v2", + "round_view_v2", + "training_v2", + "verified_inputs_v2", + "aggregate_v2", + "_capacity_outer_v2", + "_prepare_capacity_genesis_v2", + "island_job_v2", + "execute_audit_v2", + "referee_v2", + "rollback_v2", + "rollback_preview_v2", + "_rollback_context_v2", + "schedule_audits_v2", + "lease_v2", + "complete_audit_v2", + "finalize_v2", + } + nodes = [ + node + for node in ast.walk(tree) + if isinstance(node, (ast.ClassDef, ast.FunctionDef)) and node.name in owned + ] + code = {node.name: ast.dump(node, include_attributes=False) for node in nodes} + if set(code) != owned: + raise ChallengeError(409, "SERVICE_CAPACITY_IMPLEMENTATION") + code.update({path: sha256_hex((source / path).read_bytes()) for path in paths}) + return sha256_hex(canonicalize(code)) + + def _service_boundary_v2( + self, + run_id: str, + w: int | None = None, + *, + heavy: bool = False, + roster: list[Any] | None = None, + ) -> dict[str, Any] | None: + """Original complete roster first; accounting cannot grant heavy execution.""" + estimate = self.require_roster_v2(run_id, w, roster=roster) + if heavy and estimate is not None: + raise ChallengeError( + 503, "SERVICE_RUNTIME_NOT_ENFORCED: OS and codec preflight pending" + ) + return estimate + + def _service_bytes_v2( + self, run_id: str, w: int, raw: bytes, *, kind: Literal["object", "tape", "context"] + ) -> None: + """Actual bytes, never a declared length or decoded tensor estimate.""" + if self._service_boundary_v2(run_id, w) is None: + return + profile, _ = self._service_admission_v2(run_id) + limit = profile.max_object_bytes if kind == "object" else profile.max_tape_bytes + if len(raw) > limit: + raise ChallengeError(413, f"SERVICE_{kind.upper()}_BYTES") + + def _service_charge_v2( + self, + run_id: str, + w: int, + operation: str, + attempt: int, + *, + kind: Literal["audit", "outer", "repair"], + units: int, + ) -> bool: + """Committed debit: True grants attempt, False never grants duplicate execution.""" + if ( + not operation + or len(operation.encode()) > 256 + or type(attempt) is not int + or attempt not in (1, 2) + or type(units) is not int + or units < 1 + ): + raise ChallengeError(422, "SERVICE_CHARGE_BOUNDS") + if self._db.in_transaction: + raise ChallengeError(409, "SERVICE_CHARGE_REQUIRES_COMMITTED_TRANSACTION") + with self._tx(): + preparation = operation == "genesis" and w == -1 + reservation = ( + self._record_v2(run_id, "genesis-resource", "run") if preparation else None + ) + if ( + self._service_boundary_v2( + run_id, + None if preparation else w, + roster=reservation["roster"] if reservation else None, + ) + is None + ): + return True + if reservation is None: + reservation = self._record_v2(run_id, "round-resource", str(w)) + if self._now(self._db) > reservation["deadline"]: + raise ChallengeError(409, "SERVICE_RESOURCE_DEADLINE") + profile, digest = self._service_admission_v2(run_id) + charge: dict[str, JsonValue] = { + "w": w, + "operation": operation, + "attempt": attempt, + "kind": kind, + "units": units, + "profile_hash": digest, + } + identity = sha256_hex( + canonicalize({"w": w, "operation": operation, "attempt": attempt}) + ) + row = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='service-charge' AND id=?", + (run_id, identity), + ).fetchone() + if row is not None: + if json.loads(row[0]) != charge: + raise ChallengeError(409, "SERVICE_CHARGE_CONFLICT") + return False + limit = { + "audit": profile.max_audit_step_units, + "outer": profile.max_outer_work_units, + "repair": profile.max_repair_step_units, + }[kind] + spent = 0 + for row in self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='service-charge'", (run_id,) + ).fetchall(): + prior = json.loads(row[0]) + if prior["kind"] == kind: + spent += prior["units"] + if spent + units > limit: + raise ChallengeError(409, "SERVICE_WORK_BUDGET_EXHAUSTED") + self._put_record_v2(run_id, "service-charge", identity, charge) + return True + + def bootstrap_service_capacity_v2(self, run_id: str, body: bytes, raw: bytes) -> str: + """Trusted operator bootstrap: owner Receipt signs exact immutable profile.""" + from hypertrain.protocol.messages import Receipt + + with self._tx(): + try: + profile = _ServiceCapacityV1.model_validate( + envelope_v2.load_json(body, max_bytes=16384) + ) + except ValueError as exc: + raise ChallengeError(422, "SERVICE_PROFILE_BODY") from exc + if ( + profile != self._service_profile_v2(run_id) + or canonicalize(profile.model_dump(mode="json")) != body + ): + raise ChallengeError(409, "SERVICE_PROFILE_BINDING") + digest = sha256_hex(body) + env = envelope_v2.parse_envelope(raw) + if (env.run_id, env.type, env.signer) != (run_id, "Receipt", self.owner_hotkey): + raise ChallengeError(403, "SERVICE_PROFILE_ROLE") + existing = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? " + "AND kind='service-admission' AND id='run'", + (run_id,), + ).fetchone() + if existing is not None: + stored = json.loads(existing[0]) + if stored["hash"] != digest or stored["receipt"] != env.model_dump(mode="json"): + raise ChallengeError(409, "SERVICE_PROFILE_IMMUTABLE") + self._service_admission_v2(run_id) + return digest + if self.owner_hotkey is None: + raise ChallengeError(409, "SERVICE_PROFILE_OWNER") + model = self._authenticated_v2(run_id, raw, "Receipt", self.owner_hotkey) + if not isinstance(model, Receipt) or ( + model.w, + model.received_round, + model.commit_hash, + ) != (0, self._now(self._db), digest): + raise ChallengeError(409, "SERVICE_PROFILE_RECEIPT") + if self._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='round' LIMIT 1", (run_id,) + ).fetchone(): + raise ChallengeError(409, "SERVICE_PROFILE_BEFORE_OPEN") + self.objects.put(body) + self._put_record_v2( + run_id, + "service-admission", + "run", + { + "hash": digest, + "body": profile.model_dump(mode="json"), + "receipt": env.model_dump(mode="json"), + "accepted": model.received_round, + }, + ) + return digest + + def _prepare_capacity_genesis_v2( + self, run_id: str, raw: bytes, *, cancel: threading.Event | None = None + ) -> dict[str, Any]: + """Owner bootstrap admission plus signed original opening intent; never opens a round.""" + import os + import sys + + from hypertrain.aggregator.capacity_worker import ( + GenesisRequest, + _directory_fd, + _write_at, + collect_genesis, + ) + from hypertrain.data.store import LocalFSStore + from hypertrain.miner.island_launch import CapacityAttempt, run_capacity_argv + from hypertrain.protocol.messages_v2 import RoundOpenV2 + + with self._tx(): + manifest = self._run_v2(run_id) + profile, digest = self._service_admission_v2(run_id) + opening = self._authenticated_v2( + run_id, raw, "RoundOpenV2", manifest.training.coord_pubkey + ) + assert isinstance(opening, RoundOpenV2) + now = self._now(self._db) + if ( + opening.w != 0 + or opening.d_final <= now + or not isinstance(self.objects, LocalFSStore) + ): + raise ChallengeError(409, "SERVICE_GENESIS_PRECONDITION") + if self._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='round'", (run_id,) + ).fetchone(): + raise ChallengeError(409, "SERVICE_PROFILE_BEFORE_OPEN") + estimate = self._service_boundary_v2(run_id, roster=opening.roster) + assert estimate is not None + admission = self._record_v2(run_id, "service-admission", "run") + envelope = json.loads( + self._db.execute("SELECT envelope FROM runs WHERE run_id=?", (run_id,)).fetchone()[ + 0 + ] + ) + request = GenesisRequest( + manifest=manifest, + profile=profile.model_dump(mode="json"), + manifest_envelope=envelope, + receipt=admission["receipt"], + opening=envelope_v2.parse_envelope(raw).model_dump(mode="json"), + accepted=admission["accepted"], + ) + request.validate_authority() + deadline = manifest.training.beacon.genesis_time + (opening.d_final - 1) * 3 + if time.time() >= deadline or cancel is not None and cancel.is_set(): + raise ChallengeError(409, "SERVICE_GENESIS_DEADLINE_CANCEL") + directory = self.state_dir / "capacity-genesis" / run_id / "1" + with _directory_fd(directory, create=True) as root: + _write_at(root, Path("request.json"), canonicalize(request.body())) + if self._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='genesis-resource'", (run_id,) + ).fetchone(): + raise ChallengeError(409, "SERVICE_GENESIS_ATTEMPT_RECORDED") + self._put_record_v2( + run_id, + "genesis-resource", + "run", + { + **estimate, + "roster": opening.body()["roster"], + "deadline": opening.d_final, + "request_hash": request.digest(), + }, + ) + identity = sha256_hex( + canonicalize({"run_id": run_id, "w": -1, "operation": "genesis", "attempt": 1}) + ) + capacity = CapacityAttempt( + identity, + digest, + self.state_dir / "capacity-runtime.lock", + lambda: self._service_charge_v2( + run_id, -1, "genesis", 1, kind="outer", units=int(estimate["outer_work_units"]) + ), + cpu_quota="50%", + ) + run_capacity_argv( + [ + sys.executable, + "-m", + "hypertrain.aggregator.capacity_worker", + str(directory.absolute()), + ], + directory / "runtime", + deadline, + capacity, + env=dict( + PATH=os.defpath, + OMP_NUM_THREADS="1", + MKL_NUM_THREADS="1", + CUBLAS_WORKSPACE_CONFIG=":4096:8", + ), + cancel=cancel, + ) + result, objects = collect_genesis(directory, request) + with self._tx(): + self._service_admission_v2(run_id) + if ( + self._now(self._db) >= opening.d_final + or cancel is not None + and cancel.is_set() + or self._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='round'", (run_id,) + ).fetchone() + or self._record_v2(run_id, "genesis-resource", "run")["request_hash"] + != request.digest() + ): + raise ChallengeError(409, "SERVICE_GENESIS_STALE") + for key, value in objects.items(): + if self.objects.put(value) != key: + raise ChallengeError(422, "SERVICE_GENESIS_OBJECT_HASH") + record: dict[str, JsonValue] = { + "profile_hash": digest, + "request_hash": request.digest(), + "result": result.body(), + } + self._put_record_v2(run_id, "genesis-prepared", "run", record) + return record + + def _service_admission_v2(self, run_id: str) -> tuple[_ServiceCapacityV1, str]: + from hypertrain.aggregator.core import get_object + + record = self._record_v2(run_id, "service-admission", "run") + raw = get_object(self.objects, record["hash"]) + profile = _ServiceCapacityV1.model_validate_json(raw) + env = envelope_v2.parse_envelope(record["receipt"]) + if ( + profile != self._service_profile_v2(run_id) + or canonicalize(profile.model_dump(mode="json")) != raw + or record["body"] != profile.model_dump(mode="json") + or not envelope_v2.verify_envelope(record["receipt"]) + or (env.run_id, env.type, env.signer) != (run_id, "Receipt", self.owner_hotkey) + or (env.body["w"], env.body["commit_hash"], env.body["received_round"]) + != (0, record["hash"], record["accepted"]) + or record["accepted"] > env.exp_drand + ): + raise ChallengeError(409, "SERVICE_PROFILE_AUTHORITY") + return profile, record["hash"] + + def require_roster_v2( + self, run_id: str, w: int | None = None, roster: list[Any] | None = None + ) -> dict[str, Any] | None: + """Complete roster guard before storage, assignment or synchronous arithmetic.""" + if roster is None: + assert w is not None + record = self._record_v2(run_id, "round", str(w)) + roster = record["body"]["roster"] + n = len(roster) + if n > 32: + raise ChallengeError(409, "ROSTER_LIMIT_32") + admitted = self._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='service-admission' AND id='run'", + (run_id,), + ).fetchone() + if n <= 16 and not admitted: + return None + if not admitted: + raise ChallengeError(409, "ROSTER_LIMIT_16: pause complete roster until qualification") + profile, digest = self._service_admission_v2(run_id) + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.model import param_shapes + + manifest = self._run_v2(run_id) + shapes = param_shapes(TrainConfig.from_manifest_v2(manifest).model) + p, t = manifest.training.model.param_count, len(shapes) + delta_bytes = 20 + sum( + 32 + + len(name.encode()) + + 8 * len(shape) + + 4 * ((math.prod(shape) + 255) // 256) + + math.prod(shape) + for name, shape in shapes.items() + ) + from hypertrain.aggregator.rollback_v2 import RecomputedWork + from hypertrain.aggregator.tape_v2 import TapeInput + from hypertrain.protocol.messages_v2 import PolicyHashes + + entries = [e.model_dump(mode="json") if isinstance(e, BaseModel) else e for e in roster] + h, key = "f" * 64, "x" * 50 + flag = {"a": key, "b": key, "overlap": "f" * 8, "cosine": "f" * 8} + tape_shell = { + "body": { + "v": "ht-tape-v2", + "arithmetic": "flat-cclip-cap-v1", + "run_id": h, + "w": 2**53 - 1, + "policy_hash": h, + "policy_hashes": {name: h for name in PolicyHashes.model_fields}, + "predecessor_tape_hash": h, + "prev_state": h, + "prev_hashes": {name: h for name in ("theta", "u", "center")}, + "inputs": [ + {name: (entry if name == "roster" else h) for name in TapeInput.model_fields} + for entry in entries + ], + "input_root": h, + "excluded": [{"hotkey": key, "reason": "INFRASTRUCTURE", "evidence_hash": h}] * n, + "allocation": { + "policy_hash": h, + "entries": [ + { + "hotkey": key, + "admission_id": h, + "probation": False, + "weight_units": 1 << 22, + "commit_hash": h, + "delta_manifest_hash": h, + } + ] + * n, + }, + "preclipped": [key] * n, + "copy_suspicion": [flag] * (n * (n - 1) // 2), + "out_state": h, + "out_hashes": {name: h for name in ("theta", "u", "center")}, + }, + "signer": key, + "sig": "f" * 128, + } + repair_shell = { + "body": { + "v": "ht-rollback-replay/1", + "purpose": "model-repair-no-reward", + **{ + name: h + for name in ( + "source_tape_hash", + "qualification_hash", + "reference_hash", + "layout_hash", + ) + }, + "backend": "cuda", + "arithmetic": tape_shell["body"], + "recomputed": [ + { + name: ( + key + if name == "hotkey" + else ( + [2**53 - 1] * 2 + if name == "sample_ids" + else (2**53 - 1 if name == "global_step0" else h) + ) + ) + for name in RecomputedWork.model_fields + } + ] + * n, + }, + "signer": key, + "sig": "f" * 128, + } + tape_bytes = max(len(canonicalize(tape_shell)), len(canonicalize(repair_shell))) + work = 2 * (n * (n - 1) // 2 * p + n * p * (p - 1).bit_length() + 2 * n * p) + memory_estimate = (68 << 20) + (32 * n + 256) * p + 4096 * n * t + if ( + delta_bytes > min(profile.max_object_bytes, 2 << 30) + or 16 * p + 16384 > profile.max_object_bytes + or tape_bytes > profile.max_tape_bytes + or work > profile.max_outer_work_units + or memory_estimate > profile.memory_reservation_bytes + ): + raise ChallengeError(409, "SERVICE_RESOURCE_ESTIMATE") + estimate = { + "profile_hash": digest, + "roster_hash": sha256_hex(canonicalize(entries)), + "count": n, + "delta_bytes": delta_bytes, + "tape_bytes_bound": tape_bytes, + "outer_work_units": work, + "memory_estimate_bytes": memory_estimate, + "audit_steps": 8 * n, + "repair_steps": 16 * n, + } + if w is not None: + reservation = self._record_v2(run_id, "round-resource", str(w)) + if any(reservation.get(name) != value for name, value in estimate.items()): + raise ChallengeError(409, "SERVICE_ROUND_RESOURCE_BINDING") + return estimate + + def open_round_v2(self, run_id: str, raw: bytes) -> dict[str, Any]: + from hypertrain.protocol.messages_v2 import RoundOpenV2, StartStateV2 + + env = envelope_v2.parse_envelope(raw) + body = RoundOpenV2.model_validate(env.body) + resource = self._service_boundary_v2(run_id, roster=body.roster) + if resource is not None: + if not self._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='genesis-prepared'", (run_id,) + ).fetchone(): + self._service_boundary_v2(run_id, roster=body.roster, heavy=True) + from hypertrain.aggregator.capacity_worker import ( + GenesisRequest, + _directory_fd, + _write_at, + bounded_bytes, + collect_genesis, + ) + from hypertrain.challenge.finality_v2 import require_open + from hypertrain.data.store import LocalFSStore + + with self._tx(): + manifest = self._run_v2(run_id) + signed = envelope_v2.Intake( + run_id, {"RoundOpenV2": manifest.training.coord_pubkey.__eq__} + ).accept(raw, self._now(self._db)) + assert isinstance(signed, RoundOpenV2) + require_open(run_id, signed.w, self._history_v2(run_id)) + self._snapshot_current_v2(manifest) + now = self._now(self._db) + if ( + signed.w != 0 + or signed.d_open <= now + or signed.d_final - signed.d_audit < 2 + or signed.d_final > signed.d_audit + 200 + or len(signed.roster) * manifest.training.batch_samples() + > manifest.training.dataset.n_samples + or signed.registry_epoch != self._registry_v2(manifest).epoch + or signed.policy_hashes.body() + != {k: getattr(manifest.network, k) for k in signed.policy_hashes.body()} + or signed.roster_hash != resource["roster_hash"] + ): + raise ChallengeError(422, "SERVICE_PREPARED_OPEN_PRECONDITION") + reservation = self._record_v2(run_id, "genesis-resource", "run") + prepared = self._record_v2(run_id, "genesis-prepared", "run") + if ( + any(reservation.get(k) != v for k, v in resource.items()) + or reservation["roster"] != signed.body()["roster"] + or reservation["deadline"] < signed.d_final + or prepared["profile_hash"] != resource["profile_hash"] + or prepared["request_hash"] != reservation["request_hash"] + ): + raise ChallengeError(409, "SERVICE_PREPARED_OPEN_BINDING") + identity = sha256_hex(canonicalize({"w": -1, "operation": "genesis", "attempt": 1})) + charge = self._record_v2(run_id, "service-charge", identity) + if charge != { + "w": -1, + "operation": "genesis", + "attempt": 1, + "kind": "outer", + "units": resource["outer_work_units"], + "profile_hash": resource["profile_hash"], + }: + raise ChallengeError(409, "SERVICE_PREPARED_OPEN_CHARGE") + directory = self.state_dir / "capacity-genesis" / run_id / "1" + request = GenesisRequest.model_validate_json( + bounded_bytes(directory / "request.json", 1 << 20) + ) + if request.digest() != prepared["request_hash"] or request.manifest != manifest: + raise ChallengeError(409, "SERVICE_PREPARED_OPEN_REQUEST") + original_intent = RoundOpenV2.model_validate(request.opening["body"]) + if original_intent.model_dump( + exclude={ + "theta_hash", + "outer_state_hash", + "center_hash", + "start_state_index_hash", + } + ) != signed.model_dump( + exclude={ + "theta_hash", + "outer_state_hash", + "center_hash", + "start_state_index_hash", + } + ): + raise ChallengeError(409, "SERVICE_PROTECTED_OPEN_INTENT") + result, objects = collect_genesis(directory, request) + if ( + result.body() != prepared["result"] + or signed.theta_hash != result.starts[0].theta_hash + or signed.outer_state_hash != result.outer_hashes["outer_state_hash"] + or signed.center_hash != result.outer_hashes["center_hash"] + or signed.start_state_index_hash + != sha256_hex(canonicalize([s.body() for s in result.starts])) + ): + raise ChallengeError(422, "SERVICE_PREPARED_OPEN_STATE") + if not isinstance(self.objects, LocalFSStore): + raise ChallengeError(503, "SERVICE_PREPARED_OPEN_LOCAL_STORE") + for digest, payload in objects.items(): + if bounded_bytes(self.objects._path(digest), 65536) != payload: + raise ChallengeError(422, "SERVICE_PREPARED_OPEN_OBJECT") + runtime = directory / "runtime" + if not (runtime / "capacity-cleaned.json").exists(): + self._service_boundary_v2(run_id, roster=signed.roster, heavy=True) + # Existing manager custody grants this exact preparation only, never heavy work. + import os + import sys + + from hypertrain.aggregator.weighted_v2 import Candidate, allocate_weights + from hypertrain.auditor.replay import AnchorCache + + attempt = envelope_v2.load_json( + bounded_bytes(runtime / "capacity-attempt.json", 65536) + ) + kernel = envelope_v2.load_json( + bounded_bytes(runtime / "capacity-observed.json", 65536) + ) + completed = envelope_v2.load_json( + bounded_bytes(runtime / "capacity-result.json", 65536) + ) + cleaned = envelope_v2.load_json( + bounded_bytes(runtime / "capacity-cleaned.json", 65536) + ) + execution = envelope_v2.load_json( + bounded_bytes(runtime / "capacity-exec.json", 65536) + ) + if type(attempt["deadline"]) is not int: + raise ChallengeError(409, "SERVICE_PROTECTED_OPEN_RUNTIME") + attempt_identity = sha256_hex( + canonicalize({"run_id": run_id, "w": -1, "operation": "genesis", "attempt": 1}) + ) + if ( + attempt["identity"] != attempt_identity + or attempt["profile_hash"] != prepared["profile_hash"] + or attempt["deadline"] + != manifest.training.beacon.genesis_time + (reservation["deadline"] - 1) * 3 + or execution["deadline"] != attempt["deadline"] + or attempt.get("cpu_quota") != "50%" + or execution.get("cpu_quota") != attempt["cpu_quota"] + or execution["argv"] + != [ + sys.executable, + "-m", + "hypertrain.aggregator.capacity_worker", + str(directory.absolute()), + ] + or execution["env"] + != { + "PATH": os.defpath, + "OMP_NUM_THREADS": "1", + "MKL_NUM_THREADS": "1", + "CUBLAS_WORKSPACE_CONFIG": ":4096:8", + } + or kernel["memory.max"] != str(1 << 30) + or kernel["memory.swap.max"] != "0" + or kernel["cpu.max"] != "50000 100000" + or kernel["memory.oom.group"] != "1" + or kernel["affinity"] != [0, 1, 2, 3] + or kernel["nice"] != 19 + or not str(kernel["cgroup"]).endswith("/" + str(attempt["unit"])) + or completed["observed"] != kernel + or not isinstance(completed["status"], str) + or not {"Result=success", "ExecMainStatus=0", "oom 0", "oom_kill 0"} + <= set(completed["status"].splitlines()) + or cleaned + != {"units": [attempt["unit"], attempt["timer"], attempt["timer_service"]]} + or b"CAPACITY_GENESIS_COMPLETE" + not in bounded_bytes(runtime / "capacity-stdout.log", 65536) + or time.time() >= attempt["deadline"] + ): + raise ChallengeError(409, "SERVICE_PROTECTED_OPEN_RUNTIME") + if len(signed.roster) != 4 or len(result.starts) != 4: + self._service_boundary_v2(run_id, roster=signed.roster, heavy=True) + escrow, admission, _ = self._services(run_id) + for entry in signed.roster: + status = admission.status(entry.hotkey, now=now) + record = status.record + units, receipt = escrow.locked(record.admission_id, record.coldkey) + if ( + not status.eligible + or (entry.admission_id, entry.coldkey_group, entry.state, entry.q_i) + != (record.admission_id, record.coldkey, record.state, f32hex(1.0)) + or entry.eligible_weight != 4194304 + or units <= 0 + or (status.funding.locked_units, status.funding.lock_receipt_hash) + != (units, receipt) + ): + raise ChallengeError(403, "SERVICE_PROTECTED_OPEN_ELIGIBILITY") + policy_bytes = bounded_bytes( + self.objects._path(manifest.network.aggregation_policy_hash), 65536 + ) + if sha256_hex(policy_bytes) != manifest.network.aggregation_policy_hash: + raise ChallengeError(422, "SERVICE_PROTECTED_OPEN_POLICY") + allocate_weights( + [Candidate(r, "0" * 64, "0" * 64) for r in signed.roster], + AggregationPolicyV2.model_validate_json(policy_bytes), + ) + if ( + self._record_v2(run_id, "genesis-prepared", "run") != prepared + or self._record_v2(run_id, "genesis-resource", "run") != reservation + or self._service_admission_v2(run_id)[1] != prepared["profile_hash"] + or self._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='round'", (run_id,) + ).fetchone() + ): + raise ChallengeError(409, "SERVICE_PROTECTED_OPEN_CAS") + self._fresh_v2(manifest, f"open-prepared:{signed.w}") + for start in result.starts: + anchor_directory = ( + self.state_dir / "anchors-v2" / start.hotkey / start.parent_anchor_hash + ) + with _directory_fd(anchor_directory, create=True) as anchor_fd: + _write_at(anchor_fd, Path("state"), objects[start.state_object_sha256]) + _write_at(anchor_fd, Path("ef"), objects[start.ef_object_sha256]) + _write_at( + anchor_fd, + Path("metadata"), + canonicalize( + { + "run_id": run_id, + "hotkey": start.hotkey, + "w": -1, + "layout_hash": AnchorCache.layout_hash(manifest), + "proof_hash": start.anchor_verdict_hash, + "anchor_hash": start.parent_anchor_hash, + "backend": "genesis", + "state_sha256": start.state_object_sha256, + "ef_sha256": start.ef_object_sha256, + } + ), + ) + self._put_record_v2(run_id, "round", str(signed.w), env.model_dump(mode="json")) + self._put_record_v2(run_id, "round-meta", str(signed.w), {"base": 0}) + self._put_record_v2( + run_id, + "round-resource", + str(signed.w), + { + **resource, + "deadline": signed.d_final, + "max_attempts": 2, + "max_running_jobs": 2, + "memory_reservation_bytes": 1 << 30, + }, + ) + for start in result.starts: + self._put_record_v2(run_id, "start", f"{signed.w}:{start.hotkey}", start.body()) + self._put_record_v2( + run_id, + "anchor", + f"-1:{start.hotkey}", + { + "path": str( + self.state_dir + / "anchors-v2" + / start.hotkey + / start.parent_anchor_hash + ), + "anchor_hash": start.parent_anchor_hash, + "proof_hash": start.anchor_verdict_hash, + "state_root": result.state_root, + "ef_hash": start.ef_hash, + "hotkey": start.hotkey, + "w": -1, + "backend": "genesis", + }, + ) + self._put_record_v2( + run_id, + "finality", + str(signed.w), + { + "run_id": run_id, + "w": signed.w, + "applied": False, + "disposition": "PENDING", + "unresolved_disputes": 0, + "unresolved_audits": len(signed.roster), + "rollback_complete": False, + "state_hash": signed.theta_hash, + "resolution_hash": None, + }, + ) + self._db.execute("UPDATE runs SET status='running' WHERE run_id=?", (run_id,)) + return env.model_dump(mode="json") + self._service_boundary_v2(run_id, roster=body.roster, heavy=True) + import hypertrain.trainer # noqa: F401 + from hypertrain.auditor.replay import AnchorCache, optimizer_hash, pack_state + from hypertrain.challenge.finality_v2 import require_open + from hypertrain.trainer.compress import state_hash + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.model import init_params + + escrow, admission, _ = self._services(run_id) + with self._tx(): + manifest = self._run_v2(run_id) + self._authenticated_v2(run_id, raw, "RoundOpenV2", manifest.training.coord_pubkey) + prior = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='round' AND id=?", + (run_id, str(body.w)), + ).fetchone() + if prior: + if body_digest(json.loads(prior[0])["body"]) != body.digest(): + raise ChallengeError(409, "conflicting round opening") + return json.loads(prior[0]) + history = self._history_v2(run_id) + require_open(run_id, body.w, history) + resource = self.require_roster_v2(run_id, roster=body.roster) + if resource is not None: + if ( + len(body.roster) * manifest.training.batch_samples() + > manifest.training.dataset.n_samples + ): + raise ChallengeError(422, "service roster assignment capacity") + active_resources = [] + for row in self._db.execute( + "SELECT id,data FROM records_v2 WHERE run_id=? AND kind='round-resource'", + (run_id,), + ).fetchall(): + if not self._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='finalize' AND id=?", + (run_id, row[0]), + ).fetchone(): + active_resources.append(json.loads(row[1])) + if ( + sum(r["count"] for r in active_resources) + resource["count"] > 32 + or sum(r["audit_steps"] for r in active_resources) + resource["audit_steps"] + > 512 + or sum(r["repair_steps"] for r in active_resources) + resource["repair_steps"] + > 1024 + ): + raise ChallengeError(409, "SERVICE_RESOURCE_POOL_EXHAUSTED") + if body.d_final > body.d_audit + 200: + raise ChallengeError(422, "service resource absolute horizon") + resource["deadline"] = body.d_final + resource["max_attempts"] = 2 + resource["max_running_jobs"] = 2 + resource["memory_reservation_bytes"] = 1 << 30 + self._put_record_v2(run_id, "round-resource", str(body.w), resource) + now = self._fresh_v2(manifest, f"open:{body.w}") + if body.d_open <= now or body.d_final - body.d_audit < 2: + raise ChallengeError(422, "round schedule has no future replay budget") + if body.policy_hashes.body() != { + k: getattr(manifest.network, k) for k in body.policy_hashes.body() + }: + raise ChallengeError(422, "round policies differ from pinned wrapper") + registry = self._registry_v2(manifest) + if body.registry_epoch != registry.epoch: + raise ChallengeError(422, "registry epoch differs") + starts = [] + cache = AnchorCache() + cfg = TrainConfig.from_manifest_v2(manifest) + theta = init_params(cfg.model) + if body.w: + previous = self._record_v2(run_id, "applied", str(body.w - 1)) + from hypertrain.aggregator.core import load_state + + theta_np = load_state(self.objects, str(previous["out_state"])).theta + import torch + + theta = {k: torch.from_numpy(v.copy()) for k, v in theta_np.items()} + if body.theta_hash != state_hash(theta): + raise ChallengeError(422, "public theta does not match authenticated predecessor") + for entry in body.roster: + status = admission.status(entry.hotkey, now=now) + r = status.record + if not status.eligible or ( + entry.admission_id, + entry.coldkey_group, + entry.state, + entry.q_i, + ) != (r.admission_id, r.coldkey, r.state, f32hex(1.0)): + raise ChallengeError(403, "roster lacks current funded admission authority") + if entry.eligible_weight != 4194304: + raise ChallengeError(422, "live eligible ceiling must be Q/4") + if body.w == 0: + anchor = cache.genesis(manifest, entry.hotkey, theta) + else: + anchor = self._restore_anchor_v2(run_id, entry.hotkey, body.w - 1, cache) + state_blob = pack_state(theta, anchor.state) + ef_blob = pack_state(anchor.ef) + start = StartStateV2( + run_id=run_id, + w=body.w, + hotkey=entry.hotkey, + theta_hash=state_hash(theta), + state_object_sha256=self.objects.put(state_blob), + opt_state_hash=optimizer_hash(anchor.state), + ef_object_sha256=self.objects.put(ef_blob), + ef_hash=state_hash(anchor.ef), + parent_anchor_hash=anchor.anchor_hash, + global_step0=anchor.state.step, + anchor_verdict_hash=anchor.proof_hash, + ) + starts.append(start) + if body.start_state_index_hash != sha256_hex(canonicalize([s.body() for s in starts])): + raise ChallengeError(422, "start-state index differs from authenticated anchors") + # Feasibility stays on authenticated opening, never join/status. + from hypertrain.aggregator.weighted_v2 import Candidate, allocate_weights + + policy = AggregationPolicyV2.model_validate_json( + self.objects.get(manifest.network.aggregation_policy_hash) + ) + allocate_weights([Candidate(r, "0" * 64, "0" * 64) for r in body.roster], policy) + base = 0 + if body.w: + prev_round = self._record_v2(run_id, "round", str(body.w - 1)) + prev_base = int(str(self._record_v2(run_id, "round-meta", str(body.w - 1))["base"])) + base = ( + prev_base + + len(prev_round["body"]["roster"]) * manifest.training.batch_samples() + ) + n = manifest.training.dataset.n_samples + if ( + base + len(body.roster) * manifest.training.batch_samples() - 1 + ) // n != base // n: + base = (base // n + 1) * n + if ( + len(body.roster) * manifest.training.batch_samples() + > manifest.training.dataset.n_samples + ): + raise ChallengeError(422, "complete roster exceeds dataset") + self._put_record_v2(run_id, "round", str(body.w), env.model_dump(mode="json")) + self._put_record_v2(run_id, "round-meta", str(body.w), {"base": base}) + for start in starts: + self._put_record_v2(run_id, "start", f"{body.w}:{start.hotkey}", start.body()) + if body.w == 0: + anchor = cache.entries[(run_id, start.hotkey, -1, cache.layout_hash(manifest))] + self._persist_anchor_v2(manifest, anchor, cache) + self._put_record_v2( + run_id, + "finality", + str(body.w), + { + "run_id": run_id, + "w": body.w, + "applied": False, + "disposition": "PENDING", + "unresolved_disputes": 0, + "unresolved_audits": len(body.roster), + "rollback_complete": False, + "state_hash": body.theta_hash, + "resolution_hash": None, + }, + ) + self._db.execute("UPDATE runs SET status='running' WHERE run_id=?", (run_id,)) + return env.model_dump(mode="json") + + def _registry_v2(self, manifest: RunManifestV2) -> Any: + from hypertrain.protocol.relay_messages import RelayRegistryV1 + + current = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='registry' AND id='current'", + (manifest.run_id(),), + ).fetchone() + if current is not None: + return RelayRegistryV1.model_validate(json.loads(current[0])["body"]) + return RelayRegistryV1.model_validate_json( + self.objects.get(manifest.network.relay_registry_hash) + ) + + def rotate_registry_v2(self, run_id: str, raw: bytes) -> dict[str, Any]: + from hypertrain.protocol import relay_envelope + from hypertrain.protocol.relay_messages import RelayRegistryV1 + + with self._tx(): + manifest = self._run_v2(run_id) + old = self._registry_v2(manifest) + candidate = relay_envelope.Intake( + run_id, {"RelayRegistryV1": self._coord().ss58.__eq__} + ).accept(raw, self._now(self._db)) + assert isinstance(candidate, RelayRegistryV1) + if candidate.digest() == old.digest(): + return {"registry_hash": old.digest(), "epoch": old.epoch} + if candidate.previous_registry_hash != old.digest() or candidate.epoch != old.epoch + 1: + raise ChallengeError(409, "successor registry must preserve exact accepted lineage") + now = self._now(self._db) + for row in self._db.execute( + "SELECT id,data FROM records_v2 WHERE run_id=? AND kind='grant'", (run_id,) + ): + grant = json.loads(row["data"])["body"] + released = self._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='relay-release' AND id=?", + (run_id, row["id"]), + ).fetchone() + if released is not None: + continue + prior = next(s for s in old.specs if s.id == grant["relay_id"]) + successor = next((s for s in candidate.specs if s.id == prior.id), None) + if successor is None or successor.https_url != prior.https_url: + raise ChallengeError(409, "unreleased custody requires original relay endpoint") + for key in prior.pubkeys: + if key.valid_from_round <= now: + replacement = next( + (k for k in successor.pubkeys if k.key_id == key.key_id), None + ) + if ( + replacement is None + or replacement.pubkey != key.pubkey + or replacement.valid_from_round != key.valid_from_round + or replacement.valid_until_round < key.valid_until_round + ): + raise ChallengeError(409, "successor dropped an unreleased custody key") + self.objects.put(canonicalize(candidate.body())) + self._put_record_v2(run_id, "registry", old.digest(), old.body()) + self._put_record_v2( + run_id, + "registry", + "current", + relay_envelope.parse_envelope(raw).model_dump(mode="json"), + ) + result = {"registry_hash": candidate.digest(), "epoch": candidate.epoch} + self._notify_processes_v2() + return result + + def relay_observer_v2( + self, run_id: str, grant_hash: str, observer: str, raw: bytes | None = None + ) -> dict[str, Any]: + """Fresh signed challenge/response plus accepted reference bytes, not health flags.""" + from hypertrain.protocol.messages import Receipt + from hypertrain.protocol.messages_v2 import WorkProof + + with self._tx(): + manifest = self._run_v2(run_id) + now = self._fresh_v2(manifest, f"relay-observer:{grant_hash}:{observer}") + grant = self._record_v2(run_id, "grant", grant_hash)["body"] + if observer not in manifest.training.auditors or observer in { + grant["hotkey"], + self._coord().ss58, + }: + raise ChallengeError(403, "independent pinned observer required") + if self._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='auditor-disabled' AND id=?", + (run_id, observer), + ).fetchone(): + raise ChallengeError(403, "disabled observer cannot establish health") + reference = self._db.execute( + "SELECT reference FROM admission_trial_results WHERE outcome='MATCH' " + "AND reference IS NOT NULL ORDER BY epoch DESC LIMIT 1" + ).fetchone() + if reference is None: + raise ChallengeError(409, "independent accepted reference unavailable") + proof = WorkProof.model_validate_json(reference[0]) + for artifact in proof.artifact_refs: + content = self.objects.get(artifact.sha256) + if len(content) != artifact.size or sha256_hex(content) != artifact.sha256: + raise ChallengeError(409, "accepted reference bytes unhealthy") + identity = f"{grant_hash}:{observer}" + if raw is None: + challenge = Receipt( + w=grant["w"], commit_hash=secrets.token_hex(32), received_round=now + ) + signed = envelope_v2.seal(self._coord(), "Receipt", run_id, challenge, now + 10) + self._put_record_v2(run_id, "relay-observer-challenge", identity, signed) + result = signed + else: + accepted_challenge = self._record_v2(run_id, "relay-observer-challenge", identity) + model = self._authenticated_v2(run_id, raw, "Receipt", observer) + if ( + model.model_dump(mode="json") != accepted_challenge["body"] + or now >= accepted_challenge["exp_drand"] + ): + raise ChallengeError(409, "observer response differs from fresh challenge") + result = { + "beacon": now, + "expires": now + 10, + "reference_hash": proof.digest(), + "response": envelope_v2.parse_envelope(raw).model_dump(mode="json"), + } + self._put_record_v2(run_id, "relay-observer-health", identity, result) + self._notify_processes_v2() + return result + + async def relay_control_v2( + self, run_id: str, grant_hash: str, action: str, client: Any + ) -> dict[str, Any]: + from hypertrain.protocol import relay_envelope + from hypertrain.protocol.relay_messages import ( + ChunkCustodyAck, + CustodyRelease, + RelayReceipt, + RetentionExtension, + RetentionExtensionAck, + UploadGrant, + extend_retention, + ) + + with self._tx(): + manifest = self._run_v2(run_id) + self._snapshot_current_v2(manifest) + grant = UploadGrant.model_validate(self._record_v2(run_id, "grant", grant_hash)["body"]) + accepted = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='relay-receipt' " + "AND json_extract(data,'$.body.grant_hash')=?", + (run_id, grant_hash), + ).fetchone() + if accepted is None: + raise ChallengeError(409, "no accepted durable receipt custody to extend/release") + original = relay_envelope.parse_envelope(json.loads(accepted[0])) + receipt = RelayReceipt.model_validate(original.body) + registry = self._registry_v2(manifest) + spec = next(s for s in registry.specs if s.id == grant.relay_id) + current = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='retention' AND id=?", + (run_id, grant_hash), + ).fetchone() + lineage: dict[str, Any] = ( + json.loads(current[0]) + if current + else {"hash": receipt.digest(), "seq": 0, "horizon": receipt.retain_until} + ) + message: RetentionExtension | CustodyRelease + now = self._now(self._db) + if action == "extend" and now >= lineage["horizon"]: + raise ChallengeError(409, "custody horizon already expired; rescue required") + if action == "extend": + unresolved = [ + r[0] + for r in self._db.execute( + "SELECT id FROM disputes_v2 " + "WHERE json_extract(turn,'$.contest.w')=? " + "AND json_extract(turn,'$.resolution') IS NULL", + (grant.w,), + ) + ] + horizon = max( + lineage["horizon"] + 100, now + manifest.training.verify.E_vest_rounds + 1440 + ) + message = RetentionExtension( + grant_hash=grant_hash, + custody_hashes=[receipt.digest()], + seq=lineage["seq"] + 1, + previous_retention_hash=lineage["hash"], + dispute_ids=unresolved, + issued_beacon=now, + retain_until=horizon, + ) + route, type_name = f"/v1/uploads/{grant_hash}/retention", "RetentionExtension" + elif action == "release": + settled = self.relay_settlement_v2(run_id, grant_hash) + if now < max(settled["vesting_beacon"], settled["closed_beacon"]): + raise ChallengeError(409, "vesting/dispute horizon not closed") + grace = ( + max( + settled["finality_beacon"], + settled["vesting_beacon"], + settled["closed_beacon"], + ) + + 100 + ) + if now < grace: + raise ChallengeError(409, "relay settlement epoch grace has not elapsed") + message = CustodyRelease( + grant_hash=grant_hash, + custody_hashes=[receipt.digest()], + retention_hash=lineage["hash"], + finality_hash=settled["finality_hash"], + closed_dispute_root=settled["closed_dispute_root"], + vesting_beacon=settled["vesting_beacon"], + release_beacon=now, + ) + route, type_name = f"/v1/uploads/{grant_hash}/release", "CustodyRelease" + else: + raise ChallengeError(404, "unknown relay lifecycle operation") + signed = relay_envelope.seal( + self._coord(), type_name, run_id, message, max(now + 10, lineage["horizon"]) + ) + self._put_record_v2(run_id, "relay-control-request", message.digest(), signed) + response = await client.post(spec.https_url + route, content=canonicalize(signed)) + response.raise_for_status() + data = response.json() + chunk_releases: list[JsonValue] = [] + if action == "release": + accepted_custody = self._record_v2(run_id, "relay-custody", grant_hash) + acknowledgments = [ + relay_envelope.parse_envelope(a) for a in accepted_custody["artifacts"] + ] + for ack_env in acknowledgments: + if ( + ack_env.run_id != run_id + or ack_env.signer != original.signer + or not relay_envelope.verify_envelope(ack_env.model_dump()) + ): + raise ChallengeError(403, "release chunk custody authority mismatch") + chunks = [ + ChunkCustodyAck.model_validate(a.body) + for a in acknowledgments + if a.type == "ChunkCustodyAck" + ] + from hypertrain.protocol.relay_messages import UploadChunkManifest + + expected = UploadChunkManifest.model_validate( + self._record_v2(run_id, "chunk-manifest", grant_hash) + ) + if len(chunks) != len(expected.chunks) or any( + (c.index, c.off, c.len, c.chunk_sha256) != (e.index, e.off, e.len, e.chunk_sha256) + or c.grant_hash != grant_hash + or c.key_id != receipt.key_id + or c.retain_until < grant.retain_until + for c, e in zip(sorted(chunks, key=lambda c: c.index), expected.chunks, strict=True) + ): + raise ChallengeError(409, "release requires exact original full chunk custody") + assert isinstance(message, CustodyRelease) + for chunk in chunks: + release = message.model_copy( + update={ + "custody_hashes": [chunk.digest()], + "retention_hash": chunk.digest(), + } + ) + signed_chunk = relay_envelope.seal( + self._coord(), + "CustodyRelease", + run_id, + release, + max(now + 10, chunk.retain_until), + ) + reply = await client.post( + spec.https_url + route, content=canonicalize(signed_chunk) + ) + reply.raise_for_status() + parsed = relay_envelope.parse_envelope(reply.json()) + if parsed.body != release.body() or parsed.signer != self._coord().ss58: + raise ChallengeError( + 403, "chunk release response differs from signed authority" + ) + if not relay_envelope.verify_envelope(parsed.model_dump()): + raise ChallengeError(403, "chunk release signature invalid") + chunk_releases.append({"request": signed_chunk, "response": reply.json()}) + with self._tx(): + latest = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='retention' AND id=?", + (run_id, grant_hash), + ).fetchone() + if latest is not None and json.loads(latest[0])["hash"] != lineage["hash"]: + raise ChallengeError(409, "retention changed during lifecycle request") + self._snapshot_current_v2(manifest) + if action == "extend": + ack_env = relay_envelope.parse_envelope(data) + if ( + ack_env.type != "RetentionExtensionAck" + or ack_env.signer != original.signer + or ack_env.run_id != run_id + or not relay_envelope.verify_envelope(data) + ): + raise ChallengeError(403, "retention acknowledgment authority mismatch") + ack = RetentionExtensionAck.model_validate(ack_env.body) + assert isinstance(message, RetentionExtension) + effective = extend_retention( + message, + ack, + grant_hash=grant_hash, + custody_hashes=[receipt.digest()], + key_id=receipt.key_id, + previous_hash=lineage["hash"], + previous_seq=lineage["seq"], + previous_horizon=lineage["horizon"], + ) + self._put_record_v2( + run_id, + "retention", + grant_hash, + { + "hash": message.digest(), + "seq": message.seq, + "horizon": effective, + "request": signed, + "ack": data, + }, + ) + else: + release_env = relay_envelope.parse_envelope(data) + if ( + release_env.signer != self._coord().ss58 + or release_env.run_id != run_id + or release_env.type != "CustodyRelease" + or release_env.body != message.body() + or not relay_envelope.verify_envelope(data) + ): + raise ChallengeError(403, "release response differs from signed authority") + if self.relay_settlement_v2(run_id, grant_hash) != settled: + raise ChallengeError(409, "settlement changed during release") + self._put_record_v2( + run_id, + "relay-release", + grant_hash, + {"request": signed, "response": data, "chunks": chunk_releases}, + ) + self._notify_processes_v2() + return data + + def relay_failure_v2(self, run_id: str, raw: bytes) -> dict[str, Any]: + """Master checks original custody and two pinned observations; no miner slash.""" + from hypertrain.protocol import relay_envelope + from hypertrain.protocol.relay_messages import ( + AcceptedUploadAck, + ChunkCustodyAck, + RelayFailureEvidence, + RelayReceipt, + UploadChunkManifest, + UploadGrant, + has_full_custody, + ) + + data = envelope_v2.load_json(raw) + if set(data) != {"evidence", "artifacts", "observations"}: + raise ChallengeError(422, "original signed failure artifacts and observers required") + if ( + not isinstance(data["evidence"], dict) + or not isinstance(data["artifacts"], list) + or not isinstance(data["observations"], list) + ): + raise ChallengeError(422, "invalid failure envelope arrays") + if any(not isinstance(a, dict) for a in [*data["artifacts"], *data["observations"]]): + raise ChallengeError(422, "failure artifacts must be signed envelope objects") + env = relay_envelope.parse_envelope(data["evidence"]) + artifacts = [relay_envelope.parse_envelope(canonicalize(a)) for a in data["artifacts"]] + observers = [relay_envelope.parse_envelope(canonicalize(o)) for o in data["observations"]] + with self._tx(notify=True): + manifest = self._run_v2(run_id) + self._snapshot_current_v2(manifest) + evidence = RelayFailureEvidence.model_validate(env.body) + grant = UploadGrant.model_validate( + self._record_v2(run_id, "grant", evidence.grant_hash)["body"] + ) + registry = self._registry_v2(manifest) + spec = next(s for s in registry.specs if s.id == grant.relay_id) + key = next((k for k in spec.pubkeys if k.key_id == evidence.key_id), None) + if ( + key is None + or env.type != "RelayFailureEvidence" + or env.run_id != run_id + or evidence.run_id != run_id + or evidence.relay_id != grant.relay_id + or env.signer != self._coord().ss58 + or ( + not relay_envelope.verify_envelope(env.model_dump()) + or self._now(self._db) >= env.exp_drand + ) + ): + raise ChallengeError(403, "failure authority/domain mismatch") + if any( + a.run_id != run_id or not relay_envelope.verify_envelope(a.model_dump()) + for a in artifacts + ): + raise ChallengeError(403, "unsigned or cross-run custody artifact") + if evidence.evidence_hash != sha256_hex( + canonicalize([a.model_dump(mode="json") for a in artifacts]) + ): + raise ChallengeError(422, "failure artifact hash differs") + authorized = { + o.signer + for o in observers + if o.signer in manifest.training.auditors + and o.run_id == run_id + and o.type == "RelayFailureEvidence" + and o.body == env.body + and relay_envelope.verify_envelope(o.model_dump()) + } + if evidence.result == "UNCONFIRMED_AVAILABILITY": + self._put_record_v2( + run_id, "relay-failure", evidence.digest(), env.model_dump(mode="json") + ) + self._put_record_v2( + run_id, + "relay-failure-artifacts", + evidence.digest(), + {"artifacts": data["artifacts"], "observations": data["observations"]}, + ) + return {"result": evidence.result, "disabled": False} + if authorized != set(evidence.observer_ids) or len(authorized) != 2: + raise ChallengeError(403, "failure requires two distinct pinned healthy observers") + for observer, observed in zip( + evidence.observer_ids, evidence.observed_beacons, strict=True + ): + health = self._record_v2( + run_id, "relay-observer-health", f"{grant.digest()}:{observer}" + ) + if health["beacon"] != observed or self._now(self._db) >= health["expires"]: + raise ChallengeError( + 403, "observer has no current authenticated health evidence" + ) + if any( + o.signer == observer and o.exp_drand <= self._now(self._db) for o in observers + ): + raise ChallengeError(403, "observer signature expired") + assert key is not None + if key.pubkey in authorized: + raise ChallengeError(403, "relay signing key cannot observe its own failure") + custody = [a for a in artifacts if a.signer == key.pubkey] + opportunity = next( + ( + AcceptedUploadAck.model_validate(a.body) + for a in custody + if a.type == "AcceptedUploadAck" + and AcceptedUploadAck.model_validate(a.body).digest() + == evidence.upload_ack_hash + ), + None, + ) + if opportunity is None: + raise ChallengeError(422, "exact signed custody opportunity missing") + if not key.valid_from_round <= opportunity.accepted_beacon < key.valid_until_round: + raise ChallengeError(403, "custody signed outside original key validity") + if opportunity.key_id != evidence.key_id: + raise ChallengeError(422, "failure key differs from original signed custody") + chunks = [ + ChunkCustodyAck.model_validate(a.body) + for a in custody + if a.type == "ChunkCustodyAck" + and ChunkCustodyAck.model_validate(a.body).digest() in evidence.chunk_ack_hashes + ] + receipt = next( + ( + RelayReceipt.model_validate(a.body) + for a in custody + if a.type == "RelayReceipt" + and RelayReceipt.model_validate(a.body).digest() == evidence.receipt_hash + ), + None, + ) + chunk_manifest = UploadChunkManifest.model_validate( + self._record_v2(run_id, "chunk-manifest", grant.digest()) + ) + assignment = self._record_v2(run_id, "relay-assignment", f"{grant.w}:{grant.hotkey}")[ + "body" + ] + if ( + evidence.assignment_hash != body_digest(assignment) + or evidence.chunk_manifest_hash != grant.chunk_manifest_hash + ): + raise ChallengeError(422, "failure differs from accepted assignment/chunk scope") + if evidence.result != "CONTRACTUAL_SERVICE_FAILURE" or evidence.request is not None: + raise ChallengeError( + 409, "failure mode requires independently staged retrieval payload evidence" + ) + if not has_full_custody(grant, chunk_manifest, opportunity, chunks, receipt): + raise ChallengeError( + 422, "prefix/opportunity cannot establish full custody failure" + ) + if set(evidence.chunk_ack_hashes) != {c.digest() for c in chunks}: + raise ChallengeError(422, "failure includes unproved chunk custody") + if min(evidence.observed_beacons) <= opportunity.service_deadline or max( + evidence.observed_beacons + ) > self._now(self._db): + raise ChallengeError(422, "failure observed before signed service cutoff or future") + horizon = ( + receipt.retain_until if receipt is not None else min(c.retain_until for c in chunks) + ) + lineage = ( + receipt.digest() + if receipt is not None + else sha256_hex(canonicalize(sorted(c.digest() for c in chunks))) + ) + record = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='retention' AND id=?", + (run_id, grant.digest()), + ).fetchone() + if record is not None: + lineage, horizon = json.loads(record[0])["hash"], json.loads(record[0])["horizon"] + if evidence.retention_hash != lineage or max(evidence.observed_beacons) > horizon: + raise ChallengeError(422, "failure outside acknowledged exact retention horizon") + accepted = self._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='master-acceptance' " + "AND json_extract(data,'$.body.w')=? AND json_extract(data,'$.body.hotkey')=?", + (run_id, grant.w, grant.hotkey), + ).fetchone() + if accepted: + raise ChallengeError( + 409, "completed master acceptance contradicts completion failure" + ) + self._put_record_v2( + run_id, "relay-failure", evidence.digest(), env.model_dump(mode="json") + ) + self._put_record_v2( + run_id, + "relay-failure-artifacts", + evidence.digest(), + {"artifacts": data["artifacts"], "observations": data["observations"]}, + ) + self._put_record_v2( + run_id, + "relay-disabled", + f"{grant.relay_id}:{grant.assignment_epoch}:{evidence.key_id}", + {"key_id": evidence.key_id, "evidence_hash": evidence.digest()}, + ) + result = {"result": evidence.result, "disabled": True, "miner_penalty": 0} + self._notify_processes_v2() + return result + + async def relay_fallback_v2( + self, run_id: str, failure_hash: str, client: Any + ) -> dict[str, Any]: + """Rescue exact signed bytes; fallback grants never renew the original cutoff.""" + from hypertrain.data.stream_store import spool + from hypertrain.protocol import relay_envelope + from hypertrain.protocol.relay_messages import ( + AcceptedUploadAck, + ChunkCustodyAck, + RelayAssignment, + RelayFailureEvidence, + RelayReceipt, + RetrievalRequest, + RetrievalResponse, + UploadChunkManifest, + UploadGrant, + has_full_custody, + ) + from hypertrain.relay.client import RelayClient + + with self._tx(notify=True): + manifest = self._run_v2(run_id) + self._snapshot_current_v2(manifest) + failure = RelayFailureEvidence.model_validate( + self._record_v2(run_id, "relay-failure", failure_hash)["body"] + ) + grant_env = self._record_v2(run_id, "grant", failure.grant_hash) + grant = UploadGrant.model_validate(grant_env["body"]) + assignment_env = self._record_v2( + run_id, "relay-assignment", f"{grant.w}:{grant.hotkey}" + ) + assignment = RelayAssignment.model_validate(assignment_env["body"]) + registry = self._registry_v2(manifest) + now = self._now(self._db) + choices = [ + s + for s in registry.specs + if s.id in assignment.fallback_ids + and any(k.valid_from_round <= now < k.valid_until_round for k in s.pubkeys) + and self._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='relay-draining' AND id=?", + (run_id, s.id), + ).fetchone() + is None + and self._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? " + "AND kind='relay-disabled' AND id LIKE ?", + (run_id, s.id + ":%"), + ).fetchone() + is None + ] + if now < grant.exp_drand and choices: + successor = UploadGrant.model_validate( + { + **grant.body(), + "relay_id": choices[0].id, + "nonce": sha256_hex( + canonicalize([grant.digest(), failure_hash, choices[0].id]) + ), + } + ) + signed = relay_envelope.seal( + self._coord(), "UploadGrant", run_id, successor, successor.exp_drand + ) + self._put_record_v2(run_id, "grant", successor.digest(), signed) + self._put_record_v2( + run_id, + "chunk-manifest", + successor.digest(), + self._record_v2(run_id, "chunk-manifest", grant.digest()), + ) + result: dict[str, Any] = { + "grant": signed, + "assignment": assignment_env, + "rescued": False, + } + self._put_record_v2(run_id, "relay-fallback", failure_hash, result) + return result + if failure.result != "CONTRACTUAL_SERVICE_FAILURE": + raise ChallengeError(409, "no timely fallback or authenticated full custody rescue") + artifacts = self._record_v2(run_id, "relay-failure-artifacts", failure_hash)[ + "artifacts" + ] + original = [relay_envelope.parse_envelope(a) for a in artifacts] + chunk_manifest = UploadChunkManifest.model_validate( + self._record_v2(run_id, "chunk-manifest", grant.digest()) + ) + opportunity = next( + AcceptedUploadAck.model_validate(a.body) + for a in original + if a.type == "AcceptedUploadAck" and body_digest(a.body) == failure.upload_ack_hash + ) + scopes: list[tuple[str, int, list[str], str | None, str]] = [] + chunks = sorted( + [ + (ChunkCustodyAck.model_validate(a.body), a.signer) + for a in original + if a.type == "ChunkCustodyAck" + and ChunkCustodyAck.model_validate(a.body).digest() in failure.chunk_ack_hashes + ], + key=lambda item: item[0].index, + ) + if len({c.index for c, _ in chunks}) != len(chunks) or any( + c.index >= len(chunk_manifest.chunks) + or (c.off, c.len, c.chunk_sha256) + != ( + chunk_manifest.chunks[c.index].off, + chunk_manifest.chunks[c.index].len, + chunk_manifest.chunks[c.index].chunk_sha256, + ) + or c.grant_hash != grant.digest() + or c.upload_ack_hash != opportunity.digest() + or c.key_id != failure.key_id + for c, _ in chunks + ): + raise ChallengeError(422, "contradictory original chunk rescue custody") + if failure.receipt_hash is not None: + receipts = [ + a + for a in original + if a.type == "RelayReceipt" and body_digest(a.body) == failure.receipt_hash + ] + if len(receipts) != 1: + raise ChallengeError(422, "exact original complete rescue receipt required") + receipt_env = receipts[0] + receipt = RelayReceipt.model_validate(receipt_env.body) + if not has_full_custody(grant, chunk_manifest, opportunity, [], receipt): + raise ChallengeError(422, "partial receipt cannot authorize complete rescue") + scopes = [(grant.delta_hash, grant.size, [], receipt.digest(), receipt_env.signer)] + else: + if not has_full_custody(grant, chunk_manifest, opportunity, [c for c, _ in chunks]): + raise ChallengeError(422, "complete original chunk rescue coverage required") + scopes = [ + (c.chunk_sha256, c.len, [c.digest()], None, signer) for c, signer in chunks + ] + retention = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='retention' AND id=?", + (run_id, grant.digest()), + ).fetchone() + requests = [] + for sha, size, chunk_hashes, receipt_hash, signer in scopes: + lineage = chunk_hashes[0] if chunk_hashes else str(receipt_hash) + horizon = grant.retain_until + if retention: + record = json.loads(retention[0]) + lineage, horizon = record["hash"], record["horizon"] + if now >= horizon: + raise ChallengeError(409, "original rescue custody horizon expired") + request = RetrievalRequest( + request_id=secrets.token_hex(32), + nonce=secrets.token_hex(32), + relay_id=grant.relay_id, + key_id=failure.key_id, + assignment_hash=assignment.digest(), + grant_hash=grant.digest(), + custody_ack_hashes=chunk_hashes, + receipt_hash=receipt_hash, + retention_hash=lineage, + object_or_chunk_hash=sha, + size=size, + requested_beacon=now, + deadline_beacon=min(now + 10, horizon), + ) + signed_request = relay_envelope.parse_envelope( + relay_envelope.seal( + self._coord(), "RetrievalRequest", run_id, request, request.deadline_beacon + ) + ) + self._put_record_v2( + run_id, "retrieval", request.digest(), signed_request.model_dump(mode="json") + ) + requests.append((signed_request, signer)) + relay = RelayClient(registry, client) + recovered = bytearray() + responses = [] + for request_raw, signer in requests: + with spool() as payload: + response = await relay.retrieval(request_raw, payload, relay_public=signer) + if RetrievalResponse.model_validate(response.body).status != "SERVED": + raise ChallengeError(409, "original acknowledged bytes unavailable for rescue") + payload.seek(0) + recovered.extend(payload.read()) + responses.append((request_raw, response)) + with self._tx(): + self._snapshot_current_v2(manifest) + if len(recovered) != grant.size or sha256_hex(bytes(recovered)) != grant.delta_hash: + raise ChallengeError(422, "rescued original bytes differ from signed commit") + self.objects.put(bytes(recovered)) + for request_raw, response in responses: + self._put_record_v2( + run_id, + "retrieval-response", + body_digest(request_raw.body), + response.model_dump(mode="json"), + ) + result = { + "rescued": True, + "delta_hash": grant.delta_hash, + "grant": grant_env, + "assignment": assignment_env, + "master_acceptance": None, + } + self._put_record_v2(run_id, "relay-fallback", failure_hash, result) + self._notify_processes_v2() + return result + + def relay_drain_v2(self, run_id: str, relay_id: str) -> dict[str, Any]: + """Remove from new assignments, retain original custody/endpoint authority.""" + with self._tx(): + manifest = self._run_v2(run_id) + self._snapshot_current_v2(manifest) + if not any(s.id == relay_id for s in self._registry_v2(manifest).specs): + raise ChallengeError(404, "relay absent from accepted registry") + result: dict[str, Any] = {"relay_id": relay_id, "draining_beacon": self._now(self._db)} + self._put_record_v2(run_id, "relay-draining", relay_id, result) + self._notify_processes_v2() + return result + + def _history_v2(self, run_id: str) -> list[Any]: + from hypertrain.challenge.finality_v2 import RoundStatus + + return [ + RoundStatus(**json.loads(r[0])) + for r in self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='finality' " + "ORDER BY CAST(id AS INTEGER)", + (run_id,), + ) + ] + + def _persist_anchor_v2(self, manifest: RunManifestV2, anchor: Any, cache: Any) -> None: + from hypertrain.auditor.replay import tensor_root + from hypertrain.trainer.compress import state_hash + + if anchor.backend not in ("genesis", self._backend_v2(manifest)): + raise ChallengeError(409, "anchor backend differs from frozen run execution") + path = cache.persist(self.state_dir / "anchors-v2" / anchor.hotkey, anchor) + self._put_record_v2( + manifest.run_id(), + "anchor", + f"{anchor.w}:{anchor.hotkey}", + { + "path": str(path), + "anchor_hash": anchor.anchor_hash, + "proof_hash": anchor.proof_hash, + "state_root": tensor_root(anchor.theta, anchor.state), + "ef_hash": state_hash(anchor.ef), + "hotkey": anchor.hotkey, + "w": anchor.w, + "backend": anchor.backend, + }, + ) + + def _restore_anchor_v2(self, run_id: str, hotkey: str, w: int, cache: Any) -> Any: + record = self._record_v2(run_id, "anchor", f"{w}:{hotkey}") + if record["backend"] not in ("genesis", self._backend_v2(self._run_v2(run_id))): + raise ChallengeError(409, "anchor backend differs from frozen run execution") + return cache.restore( + Path(str(record["path"])), + self._run_v2(run_id), + str(record["anchor_hash"]), + str(record["proof_hash"]), + str(record["state_root"]), + str(record["ef_hash"]), + expected_hotkey=hotkey, + expected_round=w, + expected_backend=str(record["backend"]), + ) + + def round_view_v2(self, run_id: str, w: int) -> dict[str, Any]: + self._service_boundary_v2(run_id, w) + from hypertrain.protocol.messages_v2 import RoundOpenV2 + + with self._tx(): + manifest = self._run_v2(run_id) + env = self._record_v2(run_id, "round", str(w)) + body = RoundOpenV2.model_validate(env["body"]) + self.require_roster_v2(run_id, roster=body.roster) + self._snapshot_current_v2(manifest) + if self._now(self._db) >= body.d_assign: + self._fresh_v2(manifest, f"assignment:{w}", body.d_assign) + assignment = assign_round( + bytes.fromhex(run_id), + w, + self._sig(self._db, body.d_assign), + n_samples=manifest.training.dataset.n_samples, + n_slots=len(body.roster), + batch=manifest.training.batch_samples(), + base_w=int(str(self._record_v2(run_id, "round-meta", str(w))["base"])), + unit=manifest.training.dataset.assign_unit or 1, + ) + for e in body.roster: + samples = assignment.slices[e.slot] + self._put_record_v2( + run_id, + "assignment", + f"{w}:{e.hotkey}", + { + "hotkey": e.hotkey, + "slot": e.slot, + "samples": list(samples), + "assignment_hash": assignment_hash(run_id, w, e.slot, samples), + }, + ) + assignments = [ + json.loads(r[0]) + for r in self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? " + "AND kind='assignment' AND id LIKE ?", + (run_id, f"{w}:%"), + ) + ] + return { + "round_open": env, + "assignment": assignments, + "now_round": self._now(self._db), + "starts": [ + self._record_v2(run_id, "start", f"{w}:{e.hotkey}") for e in body.roster + ], + } + + def training_v2(self, run_id: str, action: str, raw: bytes) -> dict[str, Any]: + from hypertrain.protocol.messages_v2 import AcceptV2, CommitV2, DeltaManifestV2, RoundOpenV2 + + env = envelope_v2.parse_envelope(raw) + expected = {"accept": "AcceptV2", "commit": "CommitV2", "delta": "DeltaManifestV2"}[action] + with self._tx(): + manifest = self._run_v2(run_id) + w = int(str(env.body["w"])) + self.require_roster_v2(run_id, w) + self._service_bytes_v2(run_id, w, raw, kind="object") + self._owner_v2(run_id, env.signer) + model = self._authenticated_v2(run_id, raw, expected, env.signer) + body = RoundOpenV2.model_validate(self._record_v2(run_id, "round", str(w))["body"]) + assignment = self._record_v2(run_id, "assignment", f"{w}:{env.signer}") + now = self._fresh_v2(manifest, f"{action}:{w}:{self._now(self._db)}") + reservation = _dumps(list(envelope_v2.replay_key(expected, run_id, env.signer, model))) + old = self._db.execute( + "SELECT * FROM accepted_v2 WHERE key=?", (reservation,) + ).fetchone() + digest = body_digest(env.body) + if old: + if old["digest"] != digest: + raise ChallengeError(409, "conflicting semantic replay") + return json.loads(old["receipt"]) + if action in ("accept", "commit") and not body.d_assign <= now < body.d_commit: + raise ChallengeError(409, "commit window closed") + if action == "delta" and now >= body.d_upload: + raise ChallengeError(409, "upload window closed") + if action == "accept": + assert isinstance(model, AcceptV2) + model.validate_manifest(manifest) + if model.assignment_hash != assignment["assignment_hash"]: + raise ChallengeError(422, "assignment differs") + _, admission, _ = self._services(run_id) + status = admission.status(env.signer, now=now) + row = self._db.execute( + "SELECT screen_json FROM admissions_v2 WHERE hotkey=?", (env.signer,) + ).fetchone() + if ( + not status.eligible + or model.work_screen_hash != WorkScreenV2.model_validate_json(row[0]).digest() + ): + raise ChallengeError(403, "screen or funded admission stale") + elif action == "commit": + assert isinstance(model, CommitV2) + self._record_v2(run_id, "accept", f"{w}:{env.signer}") + model.validate_assignment(manifest, len(self._assignment_v2(run_id, w, env.signer))) + start = self._record_v2(run_id, "start", f"{w}:{env.signer}") + if model.ef_in_hash != start["ef_hash"]: + raise ChallengeError(422, "commit EF differs from authenticated anchor") + else: + assert isinstance(model, DeltaManifestV2) + commit = CommitV2.model_validate( + self._record_v2(run_id, "commit", f"{w}:{env.signer}")["body"] + ) + acceptance = self._record_v2( + run_id, "master-acceptance", model.master_acceptance_hash + ) + if (model.delta_hash, model.size, acceptance["body"]["delta_hash"]) != ( + commit.delta_hash, + commit.delta_bytes, + commit.delta_hash, + ): + raise ChallengeError(422, "delta/master acceptance differs from commit") + resource = self._service_boundary_v2(run_id, w) + if resource is not None: + from hypertrain.data.store import LocalFSStore + + profile, _ = self._service_admission_v2(run_id) + if model.size > profile.max_object_bytes: + raise ChallengeError(413, "SERVICE_OBJECT_BYTES") + if not isinstance(self.objects, LocalFSStore): + raise ChallengeError(503, "SERVICE_BOUNDED_GET_REQUIRED") + with self.objects._path(model.delta_hash).open("rb") as source: + payload = source.read(model.size + 1) + else: + payload = self.objects.get(model.delta_hash) + self._service_bytes_v2(run_id, w, payload, kind="object") + if ( + len(payload) != model.size + or (resource is not None and sha256_hex(payload) != model.delta_hash) + or any( + sha256_hex(payload[c.off : c.off + c.len]) != c.sha256 for c in model.chunks + ) + ): + raise ChallengeError(422, "original payload chunks mismatch") + if resource is not None: + from hypertrain.trainer.compress import MAGIC, validate_payload + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.model import param_shapes + + config = TrainConfig.from_manifest_v2(manifest) + try: + codec = validate_payload( + payload, + param_shapes(config.model), + max_payload_bytes=profile.max_object_bytes, + ) + except ValueError as exc: + raise ChallengeError(422, "SERVICE_DELTA_PAYLOAD") from exc + if codec != config.compress.codec or model.format != MAGIC[codec].decode(): + raise ChallengeError(422, "SERVICE_DELTA_CODEC") + self.objects.put(canonicalize(model.model_dump(mode="json"))) + receipt = envelope_v2.seal( + self._coord(), + "Receipt", + run_id, + {"w": w, "commit_hash": digest, "received_round": now}, + NEVER, + ) + self._put_record_v2(run_id, action, f"{w}:{env.signer}", env.model_dump(mode="json")) + self.objects.put(canonicalize(env.body)) + self._db.execute( + "INSERT INTO accepted_v2 VALUES(?,?,?,?,?)", + (reservation, digest, raw, now, _dumps(receipt)), + ) + return receipt + + def leaves_v2(self, run_id: str, raw: bytes) -> dict[str, str]: + """Signed immutable leaf object descriptor; leaves themselves stay content-addressed.""" + env = envelope_v2.parse_envelope(raw) + from hypertrain.protocol.messages_v2 import CommitV2, WorkProof + + with self._tx(): + model = self._authenticated_v2(run_id, raw, "WorkProof", env.signer) + assert isinstance(model, WorkProof) + matches = self._db.execute( + "SELECT id,data FROM records_v2 WHERE run_id=? AND kind='commit' AND id LIKE ?", + (run_id, "%:" + env.signer), + ).fetchall() + found = [ + r + for r in matches + if body_digest(json.loads(r["data"])["body"]) == model.challenge_hash + ] + if len(found) != 1: + raise ChallengeError(422, "leaf publication must bind accepted commit digest") + commit = CommitV2.model_validate(json.loads(found[0]["data"])["body"]) + if (model.leaves_root, model.delta_hash) != (commit.leaves_root, commit.delta_hash): + raise ChallengeError(422, "leaf root differs from commit") + leaves = json.loads(self.objects.get(model.artifact_refs[-1].sha256)) + assert isinstance(leaves, list) + parsed = [LeafPreimage.model_validate(p) for p in leaves] + if ( + len(parsed) != commit.n_leaves + or any(p.run_id != run_id or p.w != commit.w for p in parsed) + or ( + MerkleTree([bytes.fromhex(p.digest()) for p in parsed]).root.hex() + != commit.leaves_root + ) + ): + raise ChallengeError(422, "leaf object does not match accepted commit") + identity = f"{commit.w}:{env.signer}" + self._put_record_v2( + run_id, "leaves", identity, {"object": model.artifact_refs[-1].sha256} + ) + return {"leaves_root": commit.leaves_root} + + def schedule_audits_v2(self, run_id: str, w: int) -> list[str]: + from hypertrain.protocol.messages_v2 import AuditChallengeV2, CommitV2, RoundOpenV2 + + with self._tx(): + manifest = self._run_v2(run_id) + resource = self._service_boundary_v2(run_id, w) + if resource is not None: + from hypertrain.aggregator.capacity_worker import bounded_bytes + from hypertrain.data.store import LocalFSStore + from hypertrain.protocol.messages_v2 import StartStateV2 + + now = self._now(self._db) + opening = envelope_v2.Intake( + run_id, {"RoundOpenV2": manifest.training.coord_pubkey.__eq__} + ).accept(canonicalize(self._record_v2(run_id, "round", str(w))), now) + assert isinstance(opening, RoundOpenV2) + if ( + opening.w != w + or opening.roster_hash != resource["roster_hash"] + or opening.policy_hashes.body() + != {k: getattr(manifest.network, k) for k in opening.policy_hashes.body()} + ): + raise ChallengeError(409, "SERVICE_SCHEDULE_ROUND_AUTHORITY") + for entry in opening.roster: + identity = f"{w}:{entry.hotkey}" + signed_commit = self._record_v2(run_id, "commit", identity) + commit_model = envelope_v2.Intake( + run_id, {"CommitV2": entry.hotkey.__eq__} + ).accept(canonicalize(signed_commit), now) + assert isinstance(commit_model, CommitV2) + commit_model.validate_assignment(manifest, manifest.training.batch_samples()) + start_model = StartStateV2.model_validate( + self._record_v2(run_id, "start", identity) + ) + if ( + commit_model.w != w + or commit_model.hotkey != entry.hotkey + or (start_model.run_id, start_model.w, start_model.hotkey) + != (run_id, w, entry.hotkey) + ): + raise ChallengeError(409, "SERVICE_SCHEDULE_INPUT_AUTHORITY") + key = _dumps( + list(envelope_v2.replay_key("CommitV2", run_id, entry.hotkey, commit_model)) + ) + accepted_commit = self._db.execute( + "SELECT digest,envelope,receipt FROM accepted_v2 WHERE key=?", (key,) + ).fetchone() + if ( + accepted_commit is None + or accepted_commit["digest"] != body_digest(signed_commit["body"]) + or envelope_v2.parse_envelope(accepted_commit["envelope"]).model_dump( + mode="json" + ) + != signed_commit + or not envelope_v2.verify_envelope(json.loads(accepted_commit["receipt"])) + ): + raise ChallengeError(409, "SERVICE_SCHEDULE_ACCEPTED_COMMIT") + intake_receipt = envelope_v2.parse_envelope( + json.loads(accepted_commit["receipt"]) + ) + if ( + intake_receipt.type != "Receipt" + or intake_receipt.run_id != run_id + or intake_receipt.signer != manifest.training.coord_pubkey + or intake_receipt.body["w"] != w + or intake_receipt.body["commit_hash"] != accepted_commit["digest"] + ): + raise ChallengeError(409, "SERVICE_SCHEDULE_ACCEPTANCE_RECEIPT") + if not isinstance(self.objects, LocalFSStore): + raise ChallengeError(503, "SERVICE_SCHEDULE_LOCAL_STORE") + policy_bytes = bounded_bytes( + self.objects._path(manifest.network.audit_policy_hash), 65536 + ) + if sha256_hex(policy_bytes) != manifest.network.audit_policy_hash: + raise ChallengeError(422, "SERVICE_SCHEDULE_POLICY_HASH") + audit_policy = AuditPolicyV2.model_validate_json(policy_bytes) + audit_policy.validate_training(manifest.training) + self._service_boundary_v2(run_id, w, heavy=True) + body = RoundOpenV2.model_validate(self._record_v2(run_id, "round", str(w))["body"]) + self.require_roster_v2(run_id, roster=body.roster) + now = self._fresh_v2(manifest, f"audit:{w}", body.d_audit) + if now < body.d_audit: + raise ChallengeError(409, "audit beacon not ready") + policy = AuditPolicyV2.model_validate_json( + self.objects.get(manifest.network.audit_policy_hash) + ) + active = self._db.execute( + "SELECT COUNT(*) FROM audit_leases_v2 WHERE state IN ('queued','running')" + ).fetchone()[0] + missing = [] + for entry in body.roster: + identity = f"{w}:{entry.hotkey}" + commit_env = self._record_v2(run_id, "commit", identity) + commit = CommitV2.model_validate(commit_env["body"]) + job_id = sha256_hex(f"ht-audit-job/2|{run_id}|{w}|{entry.hotkey}".encode()) + if not self._db.execute( + "SELECT 1 FROM audit_leases_v2 WHERE id=?", (job_id,) + ).fetchone(): + missing.append((entry, identity, job_id, commit)) + if active + len(missing) > policy.max_queued_jobs: + raise ChallengeError(503, "audit capacity unavailable; no new reservation") + absolute = min(body.d_final, body.d_audit + 200) + if absolute <= now + 1: + raise ChallengeError(503, "audit absolute horizon exhausted") + result = [] + for entry, identity, job_id, _commit in missing: + start = self._record_v2(run_id, "start", identity) + challenge = AuditChallengeV2( + w=w, + target=entry.hotkey, + beacon_round=body.d_audit, + beacon_sig_sha256=sha256_hex(self._sig(self._db, body.d_audit)), + mode="full", + segments=[], + reasons=["random", "final"], + serve_deadline=absolute, + anchor_hash=sha256_hex(canonicalize(start)), + audit_mode="anchored-full", + ) + signed = envelope_v2.seal( + self._coord(), "AuditChallengeV2", run_id, challenge, absolute + ) + reservation = sha256_hex(f"ht-audit-reserve/2|{job_id}".encode()) + self._db.execute( + "INSERT INTO audit_leases_v2(id,run_id,w,hotkey,challenge,created,absolute," + "state,reservation,step_budget) " + "VALUES(?,?,?,?,?,?,?,'queued',?,?)", + ( + job_id, + run_id, + w, + entry.hotkey, + _dumps(signed), + body.d_audit, + absolute, + reservation, + policy.max_steps_per_attempt, + ), + ) + self._put_record_v2( + run_id, + "audit-reservation", + reservation, + { + "attempts": 2, + "steps_per_attempt": 2 * manifest.training.inner.H, + "absolute": absolute, + "auditors": list(manifest.training.auditors[:2]), + }, + ) + result.append(job_id) + return result + + def lease_v2(self, run_id: str, raw: bytes) -> dict[str, Any] | None: + from hypertrain.protocol.messages import Receipt + from hypertrain.protocol.messages_v2 import ArtifactRef, AuditJobV2, StartStateV2 + + env = envelope_v2.parse_envelope(raw) + with self._tx(): + manifest = self._run_v2(run_id) + if env.signer not in manifest.training.auditors: + raise ChallengeError(403, "auditor not allowlisted") + admitted = self._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='service-admission' AND id='run'", + (run_id,), + ).fetchone() + if admitted: + from hypertrain.aggregator.capacity_worker import bounded_bytes + from hypertrain.data.store import LocalFSStore + from hypertrain.protocol.messages_v2 import AuditChallengeV2, RoundOpenV2 + + self._service_admission_v2(run_id) + now = self._now(self._db) + envelope_v2.Intake(run_id, {"Receipt": env.signer.__eq__}).accept(raw, now) + queued = self._db.execute( + "SELECT * FROM audit_leases_v2 WHERE run_id=? " + "AND state='queued' ORDER BY created,id LIMIT 1", + (run_id,), + ).fetchone() + if queued is None: + raise ChallengeError( + 503, "SERVICE_RUNTIME_NOT_ENFORCED: no authenticated queued lease" + ) + resource = self._service_boundary_v2(run_id, queued["w"]) + assert resource is not None + opening = envelope_v2.Intake( + run_id, {"RoundOpenV2": manifest.training.coord_pubkey.__eq__} + ).accept(canonicalize(self._record_v2(run_id, "round", str(queued["w"]))), now) + assert isinstance(opening, RoundOpenV2) + challenge = envelope_v2.Intake( + run_id, {"AuditChallengeV2": manifest.training.coord_pubkey.__eq__} + ).accept(queued["challenge"].encode(), now) + assert isinstance(challenge, AuditChallengeV2) + original_start = StartStateV2.model_validate( + self._record_v2(run_id, "start", f"{queued['w']}:{queued['hotkey']}") + ) + if ( + opening.w != queued["w"] + or opening.roster_hash != resource["roster_hash"] + or queued["hotkey"] not in {r.hotkey for r in opening.roster} + or (challenge.w, challenge.target) != (queued["w"], queued["hotkey"]) + or challenge.anchor_hash != original_start.digest() + or challenge.serve_deadline != queued["absolute"] + or now >= queued["absolute"] + or queued["attempts"] >= 2 + or queued["step_budget"] != 2 * manifest.training.inner.H + ): + raise ChallengeError(409, "SERVICE_LEASE_AUTHORITY") + signed_commit = self._record_v2( + run_id, "commit", f"{queued['w']}:{queued['hotkey']}" + ) + from hypertrain.protocol.messages_v2 import CommitV2 + + commit_model = envelope_v2.Intake( + run_id, {"CommitV2": queued["hotkey"].__eq__} + ).accept(canonicalize(signed_commit), now) + assert isinstance(commit_model, CommitV2) + commit_model.validate_assignment(manifest, manifest.training.batch_samples()) + if ( + commit_model.w != queued["w"] + or commit_model.hotkey != queued["hotkey"] + or original_start.run_id != run_id + or original_start.w != queued["w"] + or original_start.hotkey != queued["hotkey"] + ): + raise ChallengeError(409, "SERVICE_LEASE_COMMIT_AUTHORITY") + if not isinstance(self.objects, LocalFSStore): + raise ChallengeError(503, "SERVICE_LEASE_LOCAL_STORE") + leaves_key = str( + self._record_v2(run_id, "leaves", f"{queued['w']}:{queued['hotkey']}")["object"] + ) + leaves_bytes = bounded_bytes(self.objects._path(leaves_key), 65536) + if sha256_hex(leaves_bytes) != leaves_key: + raise ChallengeError(422, "SERVICE_LEASE_LEAVES_HASH") + from hypertrain.protocol.messages import LeafPreimage + + leaves_json = json.loads(leaves_bytes) + if ( + not isinstance(leaves_json, list) + or len(leaves_json) + != manifest.training.inner.H // manifest.training.inner.J + 1 + ): + raise ChallengeError(422, "SERVICE_LEASE_LEAVES_SIZE") + for leaf in leaves_json: + p = LeafPreimage.model_validate(leaf) + if p.run_id != run_id or p.w != queued["w"]: + raise ChallengeError(422, "SERVICE_LEASE_LEAVES_AUTHORITY") + hashes = [ + bytes.fromhex(LeafPreimage.model_validate(p).digest()) for p in leaves_json + ] + if MerkleTree(hashes).root.hex() != commit_model.leaves_root or [ + p["t"] for p in leaves_json + ] != list(range(0, manifest.training.inner.H + 1, manifest.training.inner.J)): + raise ChallengeError(422, "SERVICE_LEASE_LEAVES_COMMITMENT") + self._service_boundary_v2(run_id, queued["w"], heavy=True) + model = self._authenticated_v2(run_id, raw, "Receipt", env.signer) + assert isinstance(model, Receipt) + now = self._fresh_v2(manifest, f"lease:{self._now(self._db)}") + self._expire_leases_v2(run_id, now) + if self._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='auditor-disabled' AND id=?", + (run_id, env.signer), + ).fetchone(): + raise ChallengeError(403, "silent auditor disabled pending review") + if self._db.execute( + "SELECT 1 FROM audit_leases_v2 WHERE auditor=? AND state='running'", (env.signer,) + ).fetchone(): + raise ChallengeError(409, "one live lease per auditor") + if ( + self._db.execute( + "SELECT COUNT(*) FROM audit_leases_v2 WHERE state='running'" + ).fetchone()[0] + >= 2 + ): + return None + rows = self._db.execute( + "SELECT * FROM audit_leases_v2 WHERE run_id=? " + "AND state='queued' ORDER BY created,id", + (run_id,), + ).fetchall() + row = next( + ( + r + for r in rows + if env.signer not in json.loads(r["tried"]) and env.signer != r["hotkey"] + ), + None, + ) + if row is None: + return None + nonce = secrets.token_hex(32) + expires = min(now + 100, row["absolute"]) + identity = f"{row['w']}:{row['hotkey']}" + start = StartStateV2.model_validate(self._record_v2(run_id, "start", identity)) + commit_env = self._record_v2(run_id, "commit", identity) + leaves_record = self._record_v2(run_id, "leaves", identity) + leaves = json.loads(self.objects.get(str(leaves_record["object"]))) + ef = self.objects.get(start.ef_object_sha256) + from hypertrain.auditor.replay import pack_state + + v0_hash = self.objects.put(pack_state({})) + job = AuditJobV2( + run_id=run_id, + job_id=row["id"], + auditor_id=env.signer, + attempt=row["attempts"] + 1, + lease_nonce=nonce, + lease_expires=expires, + absolute_deadline=row["absolute"], + reservation_id=row["reservation"], + replay_step_budget=row["step_budget"], + anchor_age=0, + manifest=manifest, + challenge_envelope=json.loads(row["challenge"]), + commit_envelope=commit_env, + sample_ids=list(self._assignment_v2(run_id, row["w"], row["hotkey"])), + start_state=start, + preimages=leaves, + ef_in=ArtifactRef(sha256=start.ef_object_sha256, size=len(ef)), + v0=ArtifactRef(sha256=v0_hash, size=len(self.objects.get(v0_hash))), + created_beacon=row["created"], + ) + tried = json.loads(row["tried"]) + [env.signer] + self._db.execute( + "UPDATE audit_leases_v2 SET state='running',attempts=attempts+1," + "auditor=?,nonce=?,expires=?,tried=? WHERE id=?", + (env.signer, nonce, expires, _dumps(tried), row["id"]), + ) + self._put_record_v2(run_id, "audit-job", row["id"], job.body()) + return envelope_v2.seal(self._coord(), "AuditJobV2", run_id, job, expires) + + def _expire_leases_v2(self, run_id: str, now: int) -> None: + rows = self._db.execute( + "SELECT * FROM audit_leases_v2 WHERE run_id=? AND state IN ('running','queued')", + (run_id,), + ).fetchall() + for row in rows: + if row["state"] == "exhausted": + continue + if row["state"] == "running" and now >= row["expires"]: + self._put_record_v2( + run_id, + "auditor-disabled", + row["auditor"], + {"job_id": row["id"], "reason": "LEASE_EXPIRED"}, + ) + state = "exhausted" if row["attempts"] >= 2 or now >= row["absolute"] else "queued" + self._db.execute( + "UPDATE audit_leases_v2 SET state=?,result='AUDIT_INFRASTRUCTURE' WHERE id=?", + (state, row["id"]), + ) + if row["id"] in self._lease_guards_v2: + self._lease_guards_v2[row["id"]].revoke() + elif now >= row["absolute"]: + self._db.execute( + "UPDATE audit_leases_v2 SET state='exhausted'," + "result='AUDIT_INFRASTRUCTURE' WHERE id=?", + (row["id"],), + ) + current = self._db.execute( + "SELECT state FROM audit_leases_v2 WHERE id=?", (row["id"],) + ).fetchone()[0] + if current == "exhausted": + self._put_record_v2( + run_id, + "excluded", + f"{row['w']}:{row['hotkey']}", + {"reason": "INFRASTRUCTURE", "evidence_hash": sha256_hex(row["id"].encode())}, + ) + finality = self._record_v2(run_id, "finality", str(row["w"])) + finality["disposition"] = "EXCLUDED" + finality["resolution_hash"] = sha256_hex(row["id"].encode()) + finality["rollback_complete"] = not finality["applied"] + finality["unresolved_audits"] = max(0, int(str(finality["unresolved_audits"])) - 1) + self._put_record_v2(run_id, "finality", str(row["w"]), finality) + + def island_job_v2(self, run_id: str, w: int, hotkey: str) -> dict[str, Any]: + from hypertrain.auditor.replay import pack_state, unpack_state + from hypertrain.gpu_ops.journal import durable_write + from hypertrain.protocol.messages_v2 import IslandJobV1, StartStateV2 + + with self._tx(): + manifest = self._run_v2(run_id) + resource = self._service_boundary_v2(run_id, w) + if resource is not None: + from hypertrain.aggregator.capacity_worker import ( + GenesisRequest, + bounded_bytes, + collect_genesis, + ) + from hypertrain.data.store import LocalFSStore + from hypertrain.protocol.messages_v2 import RoundOpenV2 + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.model import param_shapes + + if not isinstance(self.objects, LocalFSStore): + raise ChallengeError(503, "SERVICE_JOB_LOCAL_STORE") + opening = envelope_v2.Intake( + run_id, {"RoundOpenV2": manifest.training.coord_pubkey.__eq__} + ).accept( + canonicalize(self._record_v2(run_id, "round", str(w))), self._now(self._db) + ) + assert isinstance(opening, RoundOpenV2) + if ( + opening.w != w + or w != 0 + or opening.roster_hash != resource["roster_hash"] + or hotkey not in {r.hotkey for r in opening.roster} + or opening.policy_hashes.body() + != {k: getattr(manifest.network, k) for k in opening.policy_hashes.body()} + ): + raise ChallengeError(409, "SERVICE_JOB_ROSTER_AUTHORITY") + self._owner_v2(run_id, hotkey) + start = StartStateV2.model_validate( + self._record_v2(run_id, "start", f"{w}:{hotkey}") + ) + if (start.run_id, start.w, start.hotkey) != (run_id, w, hotkey): + raise ChallengeError(409, "SERVICE_JOB_START_AUTHORITY") + prepared = self._record_v2(run_id, "genesis-prepared", "run") + reservation = self._record_v2(run_id, "genesis-resource", "run") + directory = self.state_dir / "capacity-genesis" / run_id / "1" + request = GenesisRequest.model_validate_json( + bounded_bytes(directory / "request.json", 1 << 20) + ) + if ( + prepared["profile_hash"] != resource["profile_hash"] + or request.manifest != manifest + or request.digest() != prepared["request_hash"] + or reservation["request_hash"] != request.digest() + or reservation["roster"] != opening.body()["roster"] + ): + raise ChallengeError(409, "SERVICE_JOB_PREPARED_BINDING") + if ( + any(reservation.get(k) != v for k, v in resource.items()) + or reservation["deadline"] < opening.d_final + ): + raise ChallengeError(409, "SERVICE_JOB_RESOURCE_BINDING") + charge_id = sha256_hex( + canonicalize({"w": -1, "operation": "genesis", "attempt": 1}) + ) + charge = self._record_v2(run_id, "service-charge", charge_id) + if charge != { + "w": -1, + "operation": "genesis", + "attempt": 1, + "kind": "outer", + "units": resource["outer_work_units"], + "profile_hash": resource["profile_hash"], + }: + raise ChallengeError(409, "SERVICE_JOB_CHARGE_BINDING") + result, prepared_objects = collect_genesis(directory, request) + if ( + result.body() != prepared["result"] + or start not in result.starts + or opening.theta_hash != start.theta_hash + or opening.outer_state_hash != result.outer_hashes["outer_state_hash"] + or opening.center_hash != result.outer_hashes["center_hash"] + or opening.start_state_index_hash + != sha256_hex(canonicalize([s.body() for s in result.starts])) + ): + raise ChallengeError(409, "SERVICE_JOB_PREPARED_START") + + def read(key: str) -> bytes: + assert isinstance(self.objects, LocalFSStore) + raw = bounded_bytes(self.objects._path(key), 65536) + if sha256_hex(raw) != key: + raise ChallengeError(422, "SERVICE_JOB_OBJECT_HASH") + return raw + + state, ef = read(start.state_object_sha256), read(start.ef_object_sha256) + if ( + state != prepared_objects[start.state_object_sha256] + or ef != prepared_objects[start.ef_object_sha256] + ): + raise ChallengeError(422, "SERVICE_JOB_PREPARED_OBJECT") + shapes = param_shapes(TrainConfig.from_manifest_v2(manifest).model) + # Header-only safetensors preflight: no tensor allocation or alternate decoder. + for raw, optimizer in ((state, True), (ef, False)): + if len(raw) < 8: + raise ChallengeError(422, "SERVICE_JOB_STATE_HEADER") + header_size = struct.unpack_from(" len(raw) - 8: + raise ChallengeError(422, "SERVICE_JOB_STATE_HEADER") + header = envelope_v2.load_json(raw[8 : 8 + header_size], max_bytes=65536) + expected = { + f"{part}/{name}": list(shape) + for part in (("theta", "m", "v") if optimizer else ("theta",)) + for name, shape in shapes.items() + } + if optimizer: + expected["step"] = [] + if not isinstance(header, dict) or set(header) != set(expected): + raise ChallengeError(422, "SERVICE_JOB_STATE_NAMES") + offsets: list[tuple[int, int]] = [] + for name, shape in expected.items(): + info = header[name] + if ( + not isinstance(info, dict) + or set(info) != {"dtype", "shape", "data_offsets"} + or info["shape"] != shape + or not isinstance(info["shape"], list) + or any(type(dim) is not int for dim in info["shape"]) + or info["dtype"] != ("I64" if name == "step" else "F32") + ): + raise ChallengeError(422, "SERVICE_JOB_STATE_SHAPE") + span = info["data_offsets"] + if ( + not isinstance(span, list) + or len(span) != 2 + or type(span[0]) is not int + or type(span[1]) is not int + or not 0 <= span[0] <= span[1] <= len(raw) - 8 - header_size + or span[1] - span[0] != (8 if name == "step" else 4 * math.prod(shape)) + ): + raise ChallengeError(422, "SERVICE_JOB_STATE_OFFSETS") + offsets.append((span[0], span[1])) + cursor = 0 + for begin, end in sorted(offsets): + if begin != cursor: + raise ChallengeError(422, "SERVICE_JOB_STATE_OFFSETS") + cursor = end + if cursor != len(raw) - 8 - header_size: + raise ChallengeError(422, "SERVICE_JOB_STATE_OFFSETS") + samples = self._assignment_v2(run_id, w, hotkey) + if len(samples) != manifest.training.batch_samples() or any( + not 0 <= i < manifest.training.dataset.n_samples for i in samples + ): + raise ChallengeError(422, "SERVICE_JOB_ASSIGNMENT") + dataset = self._record_v2(run_id, "dataset", "inputs") + data = read(str(dataset["samples_hash"])) + proofs = json.loads(read(str(dataset["proofs_hash"]))) + width = (manifest.training.model.seq_len + 1) * ( + 2 if manifest.training.dataset.sample_format.startswith("u16") else 4 + ) + if ( + len(data) != width * manifest.training.dataset.n_samples + or not isinstance(proofs, list) + or len(proofs) != manifest.training.dataset.n_samples + ): + raise ChallengeError(422, "SERVICE_JOB_DATASET") + # Read/preflight only until all consumers qualify: never stage or publish a job. + self._service_boundary_v2(run_id, w, heavy=True) + self._owner_v2(run_id, hotkey) + samples = self._assignment_v2(run_id, w, hotkey) + start = StartStateV2.model_validate(self._record_v2(run_id, "start", f"{w}:{hotkey}")) + body = self._record_v2(run_id, "round", str(w))["body"] + directory = self.state_dir / "jobs-v2" / str(w) / hotkey + directory.mkdir(parents=True, exist_ok=True) + state = self.objects.get(start.state_object_sha256) + if manifest.training.inner.state_policy != "carry": + state = pack_state(unpack_state(state)[0]) + inputs = { + "start_state": state, + "ef_in": self.objects.get(start.ef_object_sha256), + "v0": pack_state({}), + } + dataset = self._record_v2(run_id, "dataset", "inputs") + raw = self.objects.get(str(dataset["samples_hash"])) + proofs = json.loads(self.objects.get(str(dataset["proofs_hash"]))) + width = (manifest.training.model.seq_len + 1) * ( + 2 if manifest.training.dataset.sample_format.startswith("u16") else 4 + ) + inputs.update( + samples=b"".join(raw[i * width : (i + 1) * width] for i in samples), + sample_proofs=canonicalize([proofs[i] for i in samples]), + ) + hashes = {} + for name, value in inputs.items(): + durable_write(directory / name, value) + hashes[name] = self.objects.put(value) + job = IslandJobV1( + job_version=1, + run_id=run_id, + w=w, + manifest=manifest, + sample_ids=list(samples), + global_step0=start.global_step0, + start_state_sha256=hashes["start_state"], + ef_in_sha256=hashes["ef_in"], + v0_sha256=hashes["v0"], + object_paths={k: k for k in inputs}, + deadline=manifest.training.beacon.genesis_time + (int(body["d_commit"]) - 1) * 3, + ) + self._put_record_v2(run_id, "island-job", f"{w}:{hotkey}", job.body()) + return {"job": job.body(), "objects": hashes} + + def execute_audit_v2(self, run_id: str, job_id: str, raw: bytes) -> dict[str, Any]: + """Independent service execution; auditor signature cannot manufacture MATCH.""" + import hypertrain.trainer # noqa: F401 + from hypertrain.auditor.replay import AnchorCache + from hypertrain.auditor.worker import LeaseGuard, execute_island_audit + from hypertrain.gpu_ops.journal import durable_write + from hypertrain.protocol.messages import ReplayEnv + from hypertrain.protocol.messages_v2 import AuditJobV2 + + env = envelope_v2.parse_envelope(raw) + with self._tx(): + manifest = self._run_v2(run_id) + backend = self._backend_v2(manifest) + row = self._db.execute( + "SELECT * FROM audit_leases_v2 WHERE id=? AND run_id=?", (job_id, run_id) + ).fetchone() + if row is None or row["state"] != "running" or row["auditor"] != env.signer: + raise ChallengeError(403, "audit request lacks owned live lease") + request = self._authenticated_v2(run_id, raw, "Receipt", row["auditor"]) + assert hasattr(request, "commit_hash") + if request.commit_hash != row["nonce"]: + raise ChallengeError(403, "audit lease nonce differs") + now = self._fresh_v2(manifest, f"audit-execute:{job_id}:{self._now(self._db)}") + job = AuditJobV2.model_validate(self._record_v2(run_id, "audit-job", job_id)) + job.validate_embedded(now) + resource = self._service_boundary_v2(run_id, row["w"]) + if resource is not None: + from hypertrain.aggregator.capacity_worker import bounded_bytes + from hypertrain.data.store import LocalFSStore + from hypertrain.protocol.messages import Receipt + from hypertrain.protocol.messages_v2 import IslandJobV1, RoundOpenV2 + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.model import param_shapes + + if not isinstance(self.objects, LocalFSStore): + raise ChallengeError(503, "SERVICE_AUDIT_LOCAL_STORE") + opening = envelope_v2.Intake( + run_id, {"RoundOpenV2": manifest.training.coord_pubkey.__eq__} + ).accept(canonicalize(self._record_v2(run_id, "round", str(row["w"]))), now) + assert isinstance(opening, RoundOpenV2) + if ( + job.run_id != run_id + or job.manifest != manifest + or job.job_id != job_id + or job.auditor_id != row["auditor"] + or job.lease_nonce != row["nonce"] + or job.lease_expires != row["expires"] + or job.absolute_deadline != row["absolute"] + or job.attempt != row["attempts"] + or job.reservation_id != row["reservation"] + or job.replay_step_budget > row["step_budget"] + or job.start_state.w != row["w"] + or job.start_state.hotkey != row["hotkey"] + or now >= min(row["expires"], row["absolute"]) + or opening.w != row["w"] + or opening.roster_hash != resource["roster_hash"] + or row["hotkey"] not in {entry.hotkey for entry in opening.roster} + or env.signer not in manifest.training.auditors + or opening.policy_hashes.body() + != {k: getattr(manifest.network, k) for k in opening.policy_hashes.body()} + or job.start_state.body() + != self._record_v2(run_id, "start", f"{row['w']}:{row['hotkey']}") + ): + raise ChallengeError(409, "SERVICE_AUDIT_LEASE_BINDING") + geometry = IslandJobV1.model_validate( + self._record_v2(run_id, "island-job", f"{row['w']}:{row['hotkey']}") + ) + deadline = ( + manifest.training.beacon.genesis_time + + (min(row["expires"], row["absolute"]) - 1) * 3 + ) + accepted = self._record_v2(run_id, "audit-geometry", f"{job_id}:{row['nonce']}") + receipt = envelope_v2.Intake( + run_id, {"Receipt": manifest.training.coord_pubkey.__eq__} + ).accept(canonicalize(accepted["receipt"]), now) + assert isinstance(receipt, Receipt) + if ( + receipt.w != row["w"] + or receipt.commit_hash != accepted["descriptor_hash"] + or receipt.received_round > now + ): + raise ChallengeError(409, "SERVICE_AUDIT_DESCRIPTOR_RECEIPT") + + def read_object(key: str, limit: int = 65536) -> bytes: + assert isinstance(self.objects, LocalFSStore) + data = bounded_bytes(self.objects._path(key), limit) + if sha256_hex(data) != key: + raise ChallengeError(422, "SERVICE_AUDIT_OBJECT_HASH") + return data + + descriptor_bytes = read_object(str(accepted["descriptor_hash"]), 1 << 20) + preflight_descriptor = envelope_v2.load_json(descriptor_bytes, max_bytes=1 << 20) + target = self.state_dir / "audit-geometry-v2" / job_id / row["nonce"] + if ( + not isinstance(preflight_descriptor, dict) + or canonicalize(preflight_descriptor) != descriptor_bytes + or preflight_descriptor + != { + "job": geometry.model_copy(update={"deadline": deadline}).body(), + "audit_job_hash": job.digest(), + "lease_nonce": row["nonce"], + "expires": row["expires"], + "absolute": row["absolute"], + "directory": str(target), + } + or geometry.manifest != manifest + or geometry.run_id != run_id + or geometry.w != row["w"] + or geometry.sample_ids != job.sample_ids + or geometry.start_state_sha256 != job.start_state.state_object_sha256 + or geometry.ef_in_sha256 != job.ef_in.sha256 + or geometry.v0_sha256 != job.v0.sha256 + or set(geometry.object_paths) + != {"start_state", "ef_in", "v0", "samples", "sample_proofs"} + or any( + Path(p).is_absolute() or ".." in Path(p).parts + for p in geometry.object_paths.values() + ) + ): + raise ChallengeError(409, "SERVICE_AUDIT_DESCRIPTOR_BINDING") + anchor_record = self._record_v2(run_id, "anchor", f"{row['w'] - 1}:{row['hotkey']}") + if ( + anchor_record["hotkey"] != row["hotkey"] + or anchor_record["w"] != row["w"] - 1 + or anchor_record["anchor_hash"] != job.start_state.parent_anchor_hash + or anchor_record["proof_hash"] != job.start_state.anchor_verdict_hash + or anchor_record["ef_hash"] != job.start_state.ef_hash + or anchor_record["backend"] not in ("genesis", backend) + ): + raise ChallengeError(409, "SERVICE_AUDIT_ANCHOR_BINDING") + anchor_directory = self.state_dir / "anchors-v2" / row["hotkey"] + path = Path(str(anchor_record["path"])) + if not path.is_relative_to(anchor_directory): + raise ChallengeError(409, "SERVICE_AUDIT_ANCHOR_PATH") + anchor_metadata = json.loads(bounded_bytes(path / "metadata", 65536)) + if not isinstance(anchor_metadata, dict) or anchor_metadata != { + "run_id": run_id, + "hotkey": row["hotkey"], + "w": row["w"] - 1, + "layout_hash": AnchorCache.layout_hash(manifest), + "proof_hash": anchor_record["proof_hash"], + "anchor_hash": anchor_record["anchor_hash"], + "backend": anchor_record["backend"], + "state_sha256": job.start_state.state_object_sha256, + "ef_sha256": job.ef_in.sha256, + }: + raise ChallengeError(409, "SERVICE_AUDIT_ANCHOR_METADATA") + inputs = { + "start_state": read_object(job.start_state.state_object_sha256), + "ef_in": read_object(job.ef_in.sha256), + "v0": read_object(job.v0.sha256), + } + if ( + bounded_bytes(path / "state", 65536) != inputs["start_state"] + or bounded_bytes(path / "ef", 65536) != inputs["ef_in"] + ): + raise ChallengeError(422, "SERVICE_AUDIT_ANCHOR_BYTES") + if len(inputs["ef_in"]) != job.ef_in.size or len(inputs["v0"]) != job.v0.size: + raise ChallengeError(422, "SERVICE_AUDIT_REF_SIZE") + shapes = param_shapes(TrainConfig.from_manifest_v2(manifest).model) + for name, data in inputs.items(): + if len(data) < 8: + raise ChallengeError(422, "SERVICE_AUDIT_STATE_HEADER") + size = struct.unpack_from(" len(data) - 8: + raise ChallengeError(422, "SERVICE_AUDIT_STATE_HEADER") + header = envelope_v2.load_json(data[8 : 8 + size], max_bytes=65536) + expected = { + f"{part}/{tensor}": list(shape) + for part in (("theta", "m", "v") if name == "start_state" else ("theta",)) + for tensor, shape in shapes.items() + } + if name == "start_state": + expected["step"] = [] + if name == "v0" and header == {}: + expected = {} + if not isinstance(header, dict) or set(header) != set(expected): + raise ChallengeError(422, "SERVICE_AUDIT_STATE_NAMES") + spans: list[tuple[int, int]] = [] + for tensor, shape in expected.items(): + info = header[tensor] + if ( + not isinstance(info, dict) + or set(info) != {"shape", "dtype", "data_offsets"} + or info["shape"] != shape + or not isinstance(info["shape"], list) + or any(type(dim) is not int for dim in info["shape"]) + or info["dtype"] != ("I64" if tensor == "step" else "F32") + ): + raise ChallengeError(422, "SERVICE_AUDIT_STATE_SHAPE") + span = info["data_offsets"] + if ( + not isinstance(span, list) + or len(span) != 2 + or type(span[0]) is not int + or type(span[1]) is not int + or not 0 <= span[0] <= span[1] <= len(data) - 8 - size + or span[1] - span[0] + != (8 if tensor == "step" else 4 * math.prod(shape)) + ): + raise ChallengeError(422, "SERVICE_AUDIT_STATE_OFFSETS") + spans.append((span[0], span[1])) + cursor = 0 + for begin, end in sorted(spans): + if begin != cursor: + raise ChallengeError(422, "SERVICE_AUDIT_STATE_OFFSETS") + cursor = end + if cursor != len(data) - 8 - size: + raise ChallengeError(422, "SERVICE_AUDIT_STATE_OFFSETS") + dataset = self._record_v2(run_id, "dataset", "inputs") + samples_data = read_object(str(dataset["samples_hash"])) + proofs = json.loads(read_object(str(dataset["proofs_hash"]))) + width = (manifest.training.model.seq_len + 1) * ( + 2 if manifest.training.dataset.sample_format.startswith("u16") else 4 + ) + if ( + len(samples_data) != width * manifest.training.dataset.n_samples + or not isinstance(proofs, list) + or len(proofs) != manifest.training.dataset.n_samples + or any(not 0 <= i < manifest.training.dataset.n_samples for i in job.sample_ids) + ): + raise ChallengeError(422, "SERVICE_AUDIT_DATASET") + inputs.update( + samples=b"".join( + samples_data[i * width : (i + 1) * width] for i in job.sample_ids + ), + sample_proofs=canonicalize([proofs[i] for i in job.sample_ids]), + ) + miner = self.state_dir / "jobs-v2" / str(row["w"]) / row["hotkey"] + for name, relative in geometry.object_paths.items(): + if bounded_bytes(miner / relative, 65536) != inputs[name]: + raise ChallengeError(422, "SERVICE_AUDIT_GEOMETRY_BYTES") + self._service_boundary_v2(run_id, row["w"], heavy=True) + deadline = ( + manifest.training.beacon.genesis_time + + (min(row["expires"], row["absolute"]) - 1) * 3 + ) + guard = LeaseGuard(row["expires"], row["absolute"], deadline) + self._lease_guards_v2[job_id] = guard # Subscribe before triggering execution. + cache = AnchorCache() + self._restore_anchor_v2(run_id, row["hotkey"], row["w"] - 1, cache) + start = job.start_state + directory = self.state_dir / "audit-v2" / job_id / row["nonce"] + directory.mkdir(parents=True, exist_ok=True) + inputs = { + "start_state": self.objects.get(start.state_object_sha256), + "ef_in": self.objects.get(start.ef_object_sha256), + "v0": self.objects.get(job.v0.sha256), + } + dataset = self._record_v2(run_id, "dataset", "inputs") + samples_raw = self.objects.get(str(dataset["samples_hash"])) + proofs = json.loads(self.objects.get(str(dataset["proofs_hash"]))) + assert isinstance(proofs, list) + width = (manifest.training.model.seq_len + 1) * ( + 2 if manifest.training.dataset.sample_format.startswith("u16") else 4 + ) + inputs["samples"] = b"".join( + samples_raw[i * width : (i + 1) * width] for i in job.sample_ids + ) + inputs["sample_proofs"] = canonicalize([proofs[i] for i in job.sample_ids]) + for name, value in inputs.items(): + durable_write(directory / name, value) + try: + with guard: + from hypertrain.protocol.messages_v2 import IslandJobV1 + + with self._lock: + geometry_job = IslandJobV1.model_validate( + self._record_v2( + run_id, "island-job", f"{job.start_state.w}:{job.start_state.hotkey}" + ) + ).model_copy(update={"deadline": deadline}) + miner_directory = ( + self.state_dir / "jobs-v2" / str(job.start_state.w) / job.start_state.hotkey + ) + geometry_directory = self.state_dir / "audit-geometry-v2" / job_id / row["nonce"] + geometry_directory.mkdir(parents=True, exist_ok=True) + for relative in geometry_job.object_paths.values(): + durable_write( + geometry_directory / relative, (miner_directory / relative).read_bytes() + ) + descriptor: dict[str, JsonValue] = { + "job": geometry_job.body(), + "audit_job_hash": job.digest(), + "lease_nonce": row["nonce"], + "expires": row["expires"], + "absolute": row["absolute"], + "directory": str(geometry_directory), + } + descriptor_hash = self.objects.put(canonicalize(descriptor)) + with self._tx(): + key = f"{job_id}:{row['nonce']}" + old = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? " + "AND kind='audit-geometry' AND id=?", + (run_id, key), + ).fetchone() + if old is not None and json.loads(old[0])["descriptor_hash"] != descriptor_hash: + raise ChallengeError(409, "immutable audit geometry descriptor conflicts") + if old is None: + self._put_record_v2( + run_id, + "audit-geometry", + key, + { + "descriptor_hash": descriptor_hash, + "receipt": envelope_v2.seal( + self._coord(), + "Receipt", + run_id, + { + "w": row["w"], + "commit_hash": descriptor_hash, + "received_round": now, + }, + NEVER, + ), + }, + ) + guard.check() + outcome, artifacts = execute_island_audit( + job, + directory, + cache, + guard, + now_round=now, + deadline_unix=deadline, + backend=backend, + publication_job=geometry_job, + publication_directory=geometry_directory, + launch=( + self._role_launch( + "audit", + geometry_job, + geometry_directory, + { + "run_id": run_id, + "job_id": job_id, + "audit_job": job.body(), + "descriptor_hash": descriptor_hash, + "descriptor": descriptor, + "descriptor_receipt": self._record_v2( + run_id, "audit-geometry", f"{job_id}:{row['nonce']}" + )["receipt"], + "hotkey": row["hotkey"], + "auditor": row["auditor"], + }, + ) + if self._role_launch is not None + else None + ), + ) + from hypertrain.auditor.island_bisect import IslandParty + + if artifacts.directory != geometry_directory / "published": + raise ChallengeError(409, "independent audit publication directory differs") + IslandParty.published(row["hotkey"], geometry_job, artifacts.directory) + guard.check() + with self._tx(): + current = self._db.execute( + "SELECT * FROM audit_leases_v2 WHERE id=?", (job_id,) + ).fetchone() + fresh = self._fresh_v2(manifest, f"audit-result:{job_id}:{self._now(self._db)}") + if ( + current["nonce"] != job.lease_nonce + or current["auditor"] != env.signer + or ( + current["state"] != "running" + or fresh >= current["expires"] + or fresh >= current["absolute"] + ) + ): + raise ChallengeError(409, "late or revoked audit result cannot settle lease") + verdict = ReplayVerdict( + challenge_hash=body_digest( + envelope_v2.parse_envelope(job.challenge_envelope).body + ), + result=outcome.result, + first_bad_leaf=outcome.first_bad_leaf, + recomputed_leaves_root=outcome.recomputed_leaves_root, + replay_env=ReplayEnv( + image_digest=manifest.training.reference_spec.image_digest, + driver=manifest.training.reference_spec.driver_allowlist[0], + gpu_uuid_sha256="0" * 64, + sm_count=max(1, manifest.training.reference_spec.sm_count), + ), + ) + # Store independently executed evidence. Caller signs this exact verdict separately. + identity = f"{job.start_state.w}:{job.start_state.hotkey}" + self._put_record_v2( + run_id, "executed-verdict", job_id, verdict.model_dump(mode="json") + ) + self._put_record_v2( + run_id, "audit-artifacts", job_id, {"directory": str(artifacts.directory)} + ) + if outcome.result == "MATCH": + anchor = cache.entries[ + ( + run_id, + job.start_state.hotkey, + job.start_state.w, + cache.layout_hash(manifest), + ) + ] + self._persist_anchor_v2(manifest, anchor, cache) + return { + "verdict": verdict.model_dump(mode="json"), + "job_id": job_id, + "lease_nonce": job.lease_nonce, + "identity": identity, + } + finally: + self._lease_guards_v2.pop(job_id, None) + + def complete_audit_v2(self, run_id: str, job_id: str, raw: bytes) -> dict[str, str]: + from hypertrain.challenge.trust_v2 import ReplayEvidence + from hypertrain.protocol.messages_v2 import AuditJobV2, CommitV2 + + with self._tx(): + row = self._db.execute( + "SELECT * FROM audit_leases_v2 WHERE id=? AND run_id=?", (job_id, run_id) + ).fetchone() + if row is None: + raise ChallengeError(404, "unknown audit lease") + manifest = self._run_v2(run_id) + resource = self._service_boundary_v2(run_id, row["w"]) + if resource is not None: + from hypertrain.aggregator.capacity_worker import bounded_bytes + from hypertrain.data.store import LocalFSStore + from hypertrain.protocol.messages_v2 import IslandJobV1, RoundOpenV2 + + now = self._now(self._db) + signed_result = envelope_v2.Intake( + run_id, {"ReplayVerdict": row["auditor"].__eq__} + ).accept(raw, now) + assert isinstance(signed_result, ReplayVerdict) + opening = envelope_v2.Intake( + run_id, {"RoundOpenV2": manifest.training.coord_pubkey.__eq__} + ).accept(canonicalize(self._record_v2(run_id, "round", str(row["w"]))), now) + assert isinstance(opening, RoundOpenV2) + original_job = AuditJobV2.model_validate( + self._record_v2(run_id, "audit-job", job_id) + ) + original_job.validate_embedded(now) + original_challenge = envelope_v2.parse_envelope(original_job.challenge_envelope) + original_verdict = self._record_v2(run_id, "executed-verdict", job_id) + if ( + opening.w != row["w"] + or opening.roster_hash != resource["roster_hash"] + or row["hotkey"] not in {r.hotkey for r in opening.roster} + or original_job.manifest != manifest + or original_job.run_id != run_id + or original_job.job_id != job_id + or original_job.auditor_id != row["auditor"] + or original_job.lease_nonce != row["nonce"] + or original_job.lease_expires != row["expires"] + or original_job.absolute_deadline != row["absolute"] + or original_job.start_state.hotkey != row["hotkey"] + or original_job.start_state.w != row["w"] + or row["state"] != "running" + or now >= min(row["expires"], row["absolute"]) + or signed_result.model_dump(mode="json") != original_verdict + or signed_result.challenge_hash != body_digest(original_challenge.body) + ): + raise ChallengeError(409, "SERVICE_COMPLETE_LEASE_RESULT") + original_geometry = IslandJobV1.model_validate( + self._record_v2(run_id, "island-job", f"{row['w']}:{row['hotkey']}") + ) + accepted_descriptor = self._record_v2( + run_id, "audit-geometry", f"{job_id}:{row['nonce']}" + ) + descriptor_receipt = envelope_v2.parse_envelope(accepted_descriptor["receipt"]) + if ( + descriptor_receipt.type != "Receipt" + or descriptor_receipt.run_id != run_id + or descriptor_receipt.signer != manifest.training.coord_pubkey + or descriptor_receipt.body["w"] != row["w"] + or descriptor_receipt.body["commit_hash"] + != accepted_descriptor["descriptor_hash"] + or not envelope_v2.verify_envelope(accepted_descriptor["receipt"]) + ): + raise ChallengeError(409, "SERVICE_COMPLETE_DESCRIPTOR_SIGNATURE") + if not isinstance(self.objects, LocalFSStore): + raise ChallengeError(503, "SERVICE_COMPLETE_LOCAL_STORE") + descriptor_data = bounded_bytes( + self.objects._path(str(accepted_descriptor["descriptor_hash"])), 1 << 20 + ) + output_directory = self.state_dir / "audit-geometry-v2" / job_id / row["nonce"] + job_deadline = ( + manifest.training.beacon.genesis_time + + (min(row["expires"], row["absolute"]) - 1) * 3 + ) + if sha256_hex(descriptor_data) != accepted_descriptor[ + "descriptor_hash" + ] or envelope_v2.load_json(descriptor_data, max_bytes=1 << 20) != { + "job": original_geometry.model_copy(update={"deadline": job_deadline}).body(), + "audit_job_hash": original_job.digest(), + "lease_nonce": row["nonce"], + "expires": row["expires"], + "absolute": row["absolute"], + "directory": str(output_directory), + }: + raise ChallengeError(409, "SERVICE_COMPLETE_DESCRIPTOR_BINDING") + # Metadata read only; IslandParty/tensors and settlement stay denied. + bounded_bytes(output_directory / "published/rank-0/trace.json", 65536) + self._service_boundary_v2(run_id, row["w"], heavy=True) + verdict = self._authenticated_v2(run_id, raw, "ReplayVerdict", row["auditor"]) + assert isinstance(verdict, ReplayVerdict) + expected = self._record_v2(run_id, "executed-verdict", job_id) + if verdict.model_dump(mode="json") != expected: + raise ChallengeError(422, "signed verdict differs from independent real replay") + now = self._fresh_v2(self._run_v2(run_id), f"complete:{job_id}:{self._now(self._db)}") + if row["state"] == "completed": + if row["result"] != verdict.result: + raise ChallengeError(409, "audit replay conflicts") + return {"result": row["result"]} + if row["state"] != "running" or now >= row["expires"] or now >= row["absolute"]: + raise ChallengeError(409, "audit result outside owned immutable lease") + job = AuditJobV2.model_validate(self._record_v2(run_id, "audit-job", job_id)) + commit = CommitV2.model_validate(job.commit_envelope["body"]) + identity = f"{row['w']}:{row['hotkey']}" + assignment = self._record_v2(run_id, "assignment", identity) + evidence = ReplayEvidence( + run_id, + row["w"], + row["hotkey"], + str(assignment["assignment_hash"]), + job.start_state.digest(), + commit.leaves_root, + commit.delta_hash, + commit.final_theta_hash, + commit.ef_in_hash, + commit.ef_out_hash, + verdict.digest() if hasattr(verdict, "digest") else body_digest(expected), + verdict.result, + "anchored-full", + ) + self._put_record_v2(run_id, "replay", identity, asdict(evidence)) + self._put_record_v2( + run_id, "verdict", identity, envelope_v2.parse_envelope(raw).model_dump(mode="json") + ) + self._db.execute( + "UPDATE audit_leases_v2 SET state='completed',result=? WHERE id=?", + (verdict.result, job_id), + ) + # Derived from independently executed published trace, never client geometry. + from hypertrain.auditor.island_bisect import IslandParty + from hypertrain.protocol.messages_v2 import IslandJobV1 + + geometry_job = IslandJobV1.model_validate( + self._record_v2(run_id, "island-job", identity) + ) + accepted_geometry = self._record_v2( + run_id, "audit-geometry", f"{job_id}:{row['nonce']}" + ) + descriptor_hash = str(accepted_geometry["descriptor_hash"]) + descriptor_bytes = self.objects.get(descriptor_hash) + descriptor = envelope_v2.load_json(descriptor_bytes) + receipt = envelope_v2.parse_envelope(accepted_geometry["receipt"]) + deadline = ( + job.manifest.training.beacon.genesis_time + + (min(row["expires"], row["absolute"]) - 1) * 3 + ) + geometry_directory = self.state_dir / "audit-geometry-v2" / job_id / row["nonce"] + if ( + sha256_hex(descriptor_bytes) != descriptor_hash + or receipt.type != "Receipt" + or receipt.run_id != run_id + or receipt.signer != self._coord().ss58 + or not envelope_v2.verify_envelope(accepted_geometry["receipt"]) + or receipt.body["commit_hash"] != descriptor_hash + or receipt.body["w"] != row["w"] + or descriptor + != { + "job": geometry_job.model_copy(update={"deadline": deadline}).body(), + "audit_job_hash": job.digest(), + "lease_nonce": row["nonce"], + "expires": row["expires"], + "absolute": row["absolute"], + "directory": str(geometry_directory), + } + ): + raise ChallengeError(409, "accepted audit geometry authority differs") + geometry_job = IslandJobV1.model_validate(descriptor["job"]) + geometry_directory = geometry_directory / "published" + if not (geometry_directory / "rank-0/trace.json").is_file(): + raise ChallengeError( + 503, "qualified immutable same-layout trace geometry unavailable" + ) + party = IslandParty.published(row["hotkey"], geometry_job, geometry_directory) + layer_span = max( + party.span("layer", (window,)) for window in range(1, party.H // party.J + 1) + ) + op_span = max( + party.span("op", (window, layer)) + for window in range(1, party.H // party.J + 1) + for layer in range(party.span("layer", (window,))) + ) + policy = DisputePolicyV2.model_validate_json( + self.objects.get(job.manifest.network.dispute_policy_hash) + ) + referee = next( + ( + h + for h in policy.referees + if h not in (row["hotkey"], row["auditor"], self._coord().ss58) + ), + None, + ) + if referee is None: + raise ChallengeError(503, "independent referee unavailable") + admission = self._services(run_id)[1].store.record(row["hotkey"]) + contest = Contest( + verdict_hash=body_digest(expected), + challenge_hash=verdict.challenge_hash, + w=row["w"], + miner=row["hotkey"], + auditor=row["auditor"], + referee=referee, + step_span=party.H // party.J, + layer_span=layer_span, + op_span=op_span, + admission_id=admission.admission_id, + coldkey=admission.coldkey, + evidence_hash=body_digest(expected), + ) + self._put_record_v2(run_id, "contest", contest.verdict_hash, contest.body()) + self._put_record_v2( + run_id, + "trace-geometry", + contest.verdict_hash, + {"job": geometry_job.body(), "directory": str(geometry_directory)}, + ) + finality = self._record_v2(run_id, "finality", str(row["w"])) + finality["unresolved_audits"] = max(0, int(str(finality["unresolved_audits"])) - 1) + if verdict.result != "MATCH": + self._put_record_v2( + run_id, + "excluded", + identity, + {"reason": "UNVERIFIED", "evidence_hash": body_digest(expected)}, + ) + self._put_record_v2(run_id, "finality", str(row["w"]), finality) + return {"result": verdict.result} + + def verified_inputs_v2(self, run_id: str, w: int) -> list[Any]: + resource = self._service_boundary_v2(run_id, w) + if resource is not None: + from hypertrain.aggregator.capacity_worker import bounded_bytes + from hypertrain.challenge.trust_v2 import ReplayEvidence as AcceptedReplay + from hypertrain.data.store import LocalFSStore + from hypertrain.protocol.messages_v2 import ( + AuditJobV2, + CommitV2, + DeltaManifestV2, + EscrowLock, + IslandJobV1, + RoundOpenV2, + ) + + manifest = self._run_v2(run_id) + now = self._now(self._db) + opening = envelope_v2.Intake( + run_id, {"RoundOpenV2": manifest.training.coord_pubkey.__eq__} + ).accept(canonicalize(self._record_v2(run_id, "round", str(w))), now) + assert isinstance(opening, RoundOpenV2) + if ( + opening.w != w + or opening.roster_hash != resource["roster_hash"] + or opening.policy_hashes.body() + != {k: getattr(manifest.network, k) for k in opening.policy_hashes.body()} + ): + raise ChallengeError(409, "SERVICE_INPUT_ROUND_AUTHORITY") + if not isinstance(self.objects, LocalFSStore): + raise ChallengeError(503, "SERVICE_INPUT_LOCAL_STORE") + payloads = [] + for entry in opening.roster: + identity = f"{w}:{entry.hotkey}" + # No exclusions silently shorten the complete original roster on this path. + commit_envelope = self._record_v2(run_id, "commit", identity) + delta_envelope = self._record_v2(run_id, "delta", identity) + accepted_commit = envelope_v2.Intake( + run_id, {"CommitV2": entry.hotkey.__eq__} + ).accept(canonicalize(commit_envelope), now) + accepted_delta = envelope_v2.Intake( + run_id, {"DeltaManifestV2": entry.hotkey.__eq__} + ).accept(canonicalize(delta_envelope), now) + assert isinstance(accepted_commit, CommitV2) and isinstance( + accepted_delta, DeltaManifestV2 + ) + accepted_commit.validate_assignment(manifest, manifest.training.batch_samples()) + if ( + accepted_commit.w != w + or accepted_commit.hotkey != entry.hotkey + or accepted_delta.w != w + or accepted_delta.hotkey != entry.hotkey + or (accepted_delta.delta_hash, accepted_delta.size) + != (accepted_commit.delta_hash, accepted_commit.delta_bytes) + ): + raise ChallengeError(409, "SERVICE_INPUT_COMMIT_AUTHORITY") + for kind, wire, typed in ( + ("CommitV2", commit_envelope, accepted_commit), + ("DeltaManifestV2", delta_envelope, accepted_delta), + ): + intake_key = _dumps( + list(envelope_v2.replay_key(kind, run_id, entry.hotkey, typed)) + ) + intake = self._db.execute( + "SELECT * FROM accepted_v2 WHERE key=?", (intake_key,) + ).fetchone() + if intake is None: + raise ChallengeError(409, "SERVICE_INPUT_ORIGINAL_INTAKE") + receipt = envelope_v2.parse_envelope(json.loads(intake["receipt"])) + if ( + intake["digest"] != body_digest(wire["body"]) + or envelope_v2.parse_envelope(intake["envelope"]).model_dump(mode="json") + != wire + or receipt.type != "Receipt" + or receipt.run_id != run_id + or receipt.signer != manifest.training.coord_pubkey + or receipt.body["w"] != w + or receipt.body["commit_hash"] != intake["digest"] + or receipt.body["received_round"] != intake["accepted_beacon"] + or not envelope_v2.verify_envelope(json.loads(intake["receipt"])) + ): + raise ChallengeError(409, "SERVICE_INPUT_INTAKE_AUTHORITY") + replay = AcceptedReplay(**self._record_v2(run_id, "replay", identity)) + signed_verdict = self._record_v2(run_id, "verdict", identity) + matched = envelope_v2.Intake( + run_id, {"ReplayVerdict": lambda key: key in manifest.training.auditors} + ).accept(canonicalize(signed_verdict), now) + assert isinstance(matched, ReplayVerdict) + lease = self._db.execute( + "SELECT * FROM audit_leases_v2 WHERE run_id=? AND w=? AND hotkey=?", + (run_id, w, entry.hotkey), + ).fetchone() + if lease is None: + raise ChallengeError(409, "SERVICE_INPUT_MATCH_LEASE") + audit_job = AuditJobV2.model_validate( + self._record_v2(run_id, "audit-job", lease["id"]) + ) + if ( + matched.result != "MATCH" + or matched.first_bad_leaf is not None + or lease["state"] != "completed" + or lease["result"] != "MATCH" + or lease["auditor"] != signed_verdict["signer"] + or audit_job.auditor_id != lease["auditor"] + or audit_job.run_id != run_id + or audit_job.manifest != manifest + or audit_job.job_id != lease["id"] + or audit_job.lease_nonce != lease["nonce"] + or audit_job.lease_expires != lease["expires"] + or audit_job.absolute_deadline != lease["absolute"] + or audit_job.start_state.hotkey != entry.hotkey + or audit_job.start_state.w != w + or audit_job.commit_envelope != commit_envelope + or matched.challenge_hash + != body_digest(envelope_v2.parse_envelope(audit_job.challenge_envelope).body) + or matched.recomputed_leaves_root != accepted_commit.leaves_root + or self._record_v2(run_id, "executed-verdict", lease["id"]) + != matched.model_dump(mode="json") + or (replay.run_id, replay.w, replay.hotkey, replay.result, replay.audit_mode) + != (run_id, w, entry.hotkey, "MATCH", "anchored-full") + or replay.verdict_hash != body_digest(signed_verdict["body"]) + or replay.anchor_hash != audit_job.start_state.digest() + or ( + replay.leaves_root, + replay.delta_hash, + replay.final_theta_hash, + replay.ef_in_hash, + replay.ef_out_hash, + ) + != ( + accepted_commit.leaves_root, + accepted_commit.delta_hash, + accepted_commit.final_theta_hash, + accepted_commit.ef_in_hash, + accepted_commit.ef_out_hash, + ) + or replay.assignment_hash + != self._record_v2(run_id, "assignment", identity)["assignment_hash"] + ): + raise ChallengeError(409, "SERVICE_INPUT_MATCH_AUTHORITY") + descriptor_record = self._record_v2( + run_id, "audit-geometry", f"{lease['id']}:{lease['nonce']}" + ) + descriptor_receipt = envelope_v2.parse_envelope(descriptor_record["receipt"]) + if ( + descriptor_receipt.type != "Receipt" + or descriptor_receipt.run_id != run_id + or descriptor_receipt.signer != manifest.training.coord_pubkey + or descriptor_receipt.body["w"] != w + or descriptor_receipt.body["commit_hash"] + != descriptor_record["descriptor_hash"] + or not envelope_v2.verify_envelope(descriptor_record["receipt"]) + ): + raise ChallengeError(409, "SERVICE_INPUT_DESCRIPTOR_SIGNATURE") + # Existing ledger/admission predicates own origin-backed influence. + escrow, admission, _ = self._services(run_id) + funding_status = admission.status(entry.hotkey, now=now) + owner = self._owner_v2(run_id, entry.hotkey) + locked, lock_receipt = escrow.locked(entry.admission_id, owner) + if ( + not funding_status.eligible + or funding_status.record.admission_id != entry.admission_id + or funding_status.record.coldkey != owner + or ( + funding_status.funding.run_id, + funding_status.funding.hotkey, + funding_status.funding.admission_id, + ) + != (run_id, entry.hotkey, entry.admission_id) + or funding_status.funding.locked_units != locked + or funding_status.funding.lock_receipt_hash != lock_receipt + or locked <= 0 + or not funding_status.funding.conservative_bound + ): + raise ChallengeError(403, "SERVICE_INPUT_FUNDING_AUTHORITY") + origins = self._db.execute( + "SELECT u.origin,u.units,o.origin AS issued_origin,e.request " + "FROM escrow_units u " + "LEFT JOIN escrow_origins o ON o.origin=u.origin " + "JOIN escrow_events e ON e.operation_id=u.ref AND e.kind='LOCK_ADMISSION' " + "WHERE u.owner=? AND u.bucket='admission_locked' AND u.units>0 " + "AND json_extract(e.request,'$.admission_id')=?", + (owner, entry.admission_id), + ).fetchall() + if ( + not origins + or sum(row["units"] for row in origins) != locked + or any(row["issued_origin"] is None for row in origins) + ): + raise ChallengeError(403, "SERVICE_INPUT_FUNDING_ORIGINS") + for origin in origins: + lock = EscrowLock.model_validate_json(origin["request"]) + if ( + lock.owner != owner + or lock.admission_id != entry.admission_id + or origin["origin"] not in lock.origin_ids + ): + raise ChallengeError(403, "SERVICE_INPUT_FUNDING_ORIGINS") + payloads.append((accepted_delta, descriptor_record, audit_job)) + for delta, descriptor_record, audit_job in payloads: + descriptor_bytes = bounded_bytes( + self.objects._path(str(descriptor_record["descriptor_hash"])), 1 << 20 + ) + descriptor = envelope_v2.load_json(descriptor_bytes, max_bytes=1 << 20) + geometry = IslandJobV1.model_validate( + self._record_v2( + run_id, + "island-job", + f"{audit_job.start_state.w}:{audit_job.start_state.hotkey}", + ) + ) + deadline = ( + manifest.training.beacon.genesis_time + + (min(audit_job.lease_expires, audit_job.absolute_deadline) - 1) * 3 + ) + directory = ( + self.state_dir / "audit-geometry-v2" / audit_job.job_id / audit_job.lease_nonce + ) + if ( + sha256_hex(descriptor_bytes) != descriptor_record["descriptor_hash"] + or descriptor + != { + "job": geometry.model_copy(update={"deadline": deadline}).body(), + "audit_job_hash": audit_job.digest(), + "lease_nonce": audit_job.lease_nonce, + "expires": audit_job.lease_expires, + "absolute": audit_job.absolute_deadline, + "directory": str(directory), + } + or geometry.manifest != manifest + or geometry.run_id != run_id + or geometry.w != w + or geometry.sample_ids != audit_job.sample_ids + or geometry.start_state_sha256 != audit_job.start_state.state_object_sha256 + or geometry.ef_in_sha256 != audit_job.ef_in.sha256 + or geometry.v0_sha256 != audit_job.v0.sha256 + ): + raise ChallengeError(409, "SERVICE_INPUT_DESCRIPTOR_BINDING") + raw_delta = bounded_bytes(self.objects._path(delta.delta_hash), 65536) + if sha256_hex(raw_delta) != delta.delta_hash or len(raw_delta) != delta.size: + raise ChallengeError(422, "SERVICE_INPUT_DELTA_HASH") + from hypertrain.trainer.compress import validate_payload + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.model import param_shapes + + config = TrainConfig.from_manifest_v2(manifest) + if ( + validate_payload(raw_delta, param_shapes(config.model), max_payload_bytes=65536) + != config.compress.codec + ): + raise ChallengeError(422, "SERVICE_INPUT_DELTA_CODEC") + self._service_boundary_v2(run_id, w, heavy=True) + self._service_boundary_v2(run_id, w) + from hypertrain.aggregator.tape_v2 import VerifiedInput + from hypertrain.challenge.trust_v2 import ReplayEvidence, SettlementStatus + from hypertrain.protocol.messages_v2 import CommitV2, DeltaManifestV2, RoundOpenV2 + + self.require_roster_v2(run_id, w) + _, admission, disputes = self._services(run_id) + body = RoundOpenV2.model_validate(self._record_v2(run_id, "round", str(w))["body"]) + now = self._now(self._db) + works = [] + for entry in body.roster: + identity = f"{w}:{entry.hotkey}" + excluded = self._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='excluded' AND id=?", + (run_id, identity), + ).fetchone() + if excluded: + continue + status = admission.status(entry.hotkey, now=now) + if not status.eligible or status.record.admission_id != entry.admission_id: + raise ChallengeError(403, "current admission funding/identity no longer eligible") + pending = ( + self._db.execute( + "SELECT 1 FROM disputes_v2 WHERE miner=? " + "AND json_extract(turn,'$.resolution') IS NULL", + (entry.hotkey,), + ).fetchone() + is not None + ) + settlement = SettlementStatus( + run_id, + w, + entry.hotkey, + pending, + False, + sha256_hex(canonicalize([run_id, w, entry.hotkey, status.record.hotkey, pending])), + ) + works.append( + VerifiedInput( + entry, + CommitV2.model_validate(self._record_v2(run_id, "commit", identity)["body"]), + DeltaManifestV2.model_validate( + self._record_v2(run_id, "delta", identity)["body"] + ), + ReplayEvidence(**self._record_v2(run_id, "replay", identity)), + status.funding, + settlement, + str(self._record_v2(run_id, "assignment", identity)["assignment_hash"]), + status.record.clean_count, + ) + ) + return works + + def _capacity_outer_v2(self, run_id: str, w: int) -> dict[str, Any]: + """Internal bounded adapter; public heavy gate remains independently closed.""" + import os + import sys + + from hypertrain.aggregator.capacity_worker import ( + OuterInput, + OuterRequest, + bounded_bytes, + collect_result, + ) + from hypertrain.aggregator.tape_v2 import TapeV2 + from hypertrain.challenge.finality_v2 import require_apply + from hypertrain.data.store import LocalFSStore + from hypertrain.gpu_ops.journal import durable_write + from hypertrain.miner.island_launch import CapacityAttempt, run_capacity_argv + from hypertrain.protocol.envelope_v2 import tape_signing_message + + with self._tx(): + manifest = self._run_v2(run_id) + resource = self._service_boundary_v2(run_id, w) + if resource is None or not isinstance(self.objects, LocalFSStore): + raise ChallengeError(503, "SERVICE_CAPACITY_OUTER_AUTHORITY") + _, profile_hash = self._service_admission_v2(run_id) + self._fresh_v2(manifest, f"aggregate:{w}:{self._now(self._db)}") + require_apply(run_id, w, [s for s in self._history_v2(run_id) if s.w < w]) + if self._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='applied' AND id=?", + (run_id, str(w)), + ).fetchone(): + raise ChallengeError(409, "SERVICE_CAPACITY_OUTER_ALREADY_APPLIED") + opening = self._record_v2(run_id, "round", str(w)) + works = self.verified_inputs_v2(run_id, w) + inputs = [OuterInput.model_validate(self._serialize_input_v2(x)) for x in works] + prior = self._record_v2(run_id, "applied", str(w - 1)) if w else None + prev_state = str(prior["out_state"]) if prior else None + predecessor = str(prior["tape_hash"]) if prior else "0" * 64 + objects = { + manifest.network.aggregation_policy_hash, + manifest.network.economics_policy_hash, + *(x.commit.delta_hash for x in inputs), + *(sha256_hex(canonicalize(x.commit.model_dump(mode="json"))) for x in inputs), + *( + sha256_hex(canonicalize(x.delta_manifest.model_dump(mode="json"))) + for x in inputs + ), + } + if prev_state: + objects.add(prev_state) + request = OuterRequest( + manifest=manifest, + w=w, + original_roster=opening["body"]["roster"], + inputs=inputs, + prev_state=prev_state, + predecessor_tape_hash=predecessor, + reference_reward_units=self._services(run_id)[0].policy.R_collectible_units, + objects=sorted(objects), + ) + snapshot = canonicalize(request.body()) + OuterRequest.parse(snapshot) + directory = self.state_dir / "capacity-outer" / run_id / str(w) / "1" + directory.mkdir(parents=True, exist_ok=True) + if (directory / "request.json").exists(): + raise ChallengeError(409, "SERVICE_CAPACITY_OUTER_ATTEMPT_RECORDED") + staged = LocalFSStore(directory / "objects") + for key in request.objects: + payload = bounded_bytes(self.objects._path(key), 65536) + if sha256_hex(payload) != key: + raise ChallengeError(422, "SERVICE_CAPACITY_OUTER_OBJECT_HASH") + staged.put(payload) + durable_write(directory / "request.json", snapshot) + reservation = self._record_v2(run_id, "round-resource", str(w)) + deadline = ( + manifest.training.beacon.genesis_time + (int(reservation["deadline"]) - 1) * 3 + ) + identity = sha256_hex( + canonicalize({"run_id": run_id, "w": w, "operation": "outer", "attempt": 1}) + ) + capacity = CapacityAttempt( + identity, + profile_hash, + self.state_dir / "capacity-runtime.lock", + lambda: self._service_charge_v2( + run_id, w, "outer", 1, kind="outer", units=int(resource["outer_work_units"]) + ), + ) + env = dict( + os.environ, OMP_NUM_THREADS="1", MKL_NUM_THREADS="1", CUBLAS_WORKSPACE_CONFIG=":4096:8" + ) + run_capacity_argv( + [ + sys.executable, + "-m", + "hypertrain.aggregator.capacity_worker", + str(directory.resolve()), + ], + directory / "runtime", + deadline, + capacity, + env=env, + ) + body, output_objects = collect_result(directory, request) + with self._tx(): + self._service_boundary_v2(run_id, w) + self._fresh_v2(manifest, f"aggregate-result:{w}:{self._now(self._db)}") + require_apply(run_id, w, [s for s in self._history_v2(run_id) if s.w < w]) + current = [ + OuterInput.model_validate(self._serialize_input_v2(x)) + for x in self.verified_inputs_v2(run_id, w) + ] + if ( + current != inputs + or self._record_v2(run_id, "round", str(w)) != opening + or (w and self._record_v2(run_id, "applied", str(w - 1)) != prior) + or self._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='applied' AND id=?", + (run_id, str(w)), + ).fetchone() + ): + raise ChallengeError(409, "SERVICE_CAPACITY_OUTER_STALE") + signing_key = self._coord() + tape = TapeV2( + body=body, + signer=signing_key.ss58, + sig=signing_key.sign(tape_signing_message(body.body(), run_id)).hex(), + ) + for digest, payload in output_objects.items(): + if self.objects.put(payload) != digest: + raise ChallengeError(422, "SERVICE_CAPACITY_OUTER_OUTPUT_HASH") + tape_hash = self.objects.put(tape.to_bytes()) + applied = { + "tape_hash": tape_hash, + "out_state": body.out_state, + "theta_hash": body.out_hashes["theta_hash"], + "prev_state": body.prev_state, + "predecessor_tape_hash": predecessor, + } + self._put_record_v2(run_id, "applied", str(w), applied) + self._put_record_v2( + run_id, "tape-inputs", str(w), {"inputs": [x.body() for x in inputs]} + ) + finality = self._record_v2(run_id, "finality", str(w)) + finality["applied"], finality["state_hash"] = True, applied["theta_hash"] + self._put_record_v2(run_id, "finality", str(w), finality) + return {"tape": tape.body_json(), **applied} + + def aggregate_v2(self, run_id: str, w: int) -> dict[str, Any]: + preflight_resource = self._service_boundary_v2(run_id, w) + if preflight_resource is not None: + from hypertrain.aggregator.capacity_worker import bounded_bytes + from hypertrain.aggregator.tape_v2 import TapeV2 + from hypertrain.data.store import LocalFSStore + from hypertrain.protocol.envelope_v2 import tape_signing_message + from hypertrain.protocol.messages_v2 import RoundOpenV2 + + manifest = self._run_v2(run_id) + if not isinstance(self.objects, LocalFSStore): + raise ChallengeError(503, "SERVICE_AGGREGATE_LOCAL_STORE") + opening = envelope_v2.Intake( + run_id, {"RoundOpenV2": manifest.training.coord_pubkey.__eq__} + ).accept(canonicalize(self._record_v2(run_id, "round", str(w))), self._now(self._db)) + assert isinstance(opening, RoundOpenV2) + if opening.w != w or opening.roster_hash != preflight_resource["roster_hash"]: + raise ChallengeError(409, "SERVICE_AGGREGATE_ROUND_AUTHORITY") + if opening.policy_hashes.body() != { + k: getattr(manifest.network, k) for k in opening.policy_hashes.body() + }: + raise ChallengeError(409, "SERVICE_INPUT_ROUND_AUTHORITY") + prior_tape = None + for round_w in range(max(0, w - 1), w + 1): + applied_row = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='applied' AND id=?", + (run_id, str(round_w)), + ).fetchone() + if applied_row is None: + if round_w < w: + raise ChallengeError(409, "SERVICE_AGGREGATE_PREDECESSOR_MISSING") + continue + applied_metadata = json.loads(applied_row[0]) + tape_bytes = bounded_bytes( + self.objects._path(str(applied_metadata["tape_hash"])), 1 << 20 + ) + if sha256_hex(tape_bytes) != applied_metadata["tape_hash"]: + raise ChallengeError(422, "SERVICE_AGGREGATE_TAPE_HASH") + signed_tape = TapeV2.from_bytes(tape_bytes) + tape_body = signed_tape.body + tape_opening = envelope_v2.Intake( + run_id, {"RoundOpenV2": manifest.training.coord_pubkey.__eq__} + ).accept( + canonicalize(self._record_v2(run_id, "round", str(round_w))), + self._now(self._db), + ) + assert isinstance(tape_opening, RoundOpenV2) + if ( + signed_tape.signer != manifest.training.coord_pubkey + or tape_body.run_id != run_id + or tape_body.w != round_w + or not verify( + decode_hotkey(signed_tape.signer), + tape_signing_message(tape_body.body(), run_id), + bytes.fromhex(signed_tape.sig), + ) + or tape_body.prev_state != applied_metadata["prev_state"] + or tape_body.out_state != applied_metadata["out_state"] + or tape_body.predecessor_tape_hash != applied_metadata["predecessor_tape_hash"] + or tape_body.out_hashes["theta_hash"] != applied_metadata["theta_hash"] + or tape_body.policy_hash != manifest.network.aggregation_policy_hash + or tape_body.policy_hashes.body() != tape_opening.policy_hashes.body() + or {entry.roster.hotkey for entry in tape_body.inputs} + | {entry.hotkey for entry in tape_body.excluded} + != {entry.hotkey for entry in tape_opening.roster} + or any(entry.roster not in tape_opening.roster for entry in tape_body.inputs) + or (round_w == 0 and tape_body.predecessor_tape_hash != "0" * 64) + ): + raise ChallengeError(409, "SERVICE_AGGREGATE_TAPE_AUTHORITY") + if round_w == w - 1: + prior_tape = (applied_metadata["tape_hash"], tape_body.out_state) + elif w and (tape_body.predecessor_tape_hash, tape_body.prev_state) != prior_tape: + raise ChallengeError(409, "SERVICE_AGGREGATE_TAPE_LINEAGE") + self._service_boundary_v2(run_id, w, heavy=True) + self.verified_inputs_v2(run_id, w) + resource = self._service_boundary_v2(run_id, w, heavy=True) + if resource is not None: + return self._capacity_outer_v2(run_id, w) + import hypertrain.trainer # noqa: F401 + from hypertrain.aggregator.core import OuterState + from hypertrain.aggregator.tape_v2 import make_tape, replay_tape + from hypertrain.challenge.finality_v2 import require_apply + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.model import init_params + + with self._tx(): + manifest = self._run_v2(run_id) + self.require_roster_v2(run_id, w) + self._fresh_v2(manifest, f"aggregate:{w}:{self._now(self._db)}") + existing = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='applied' AND id=?", + (run_id, str(w)), + ).fetchone() + if existing is not None: + from hypertrain.aggregator.tape_v2 import TapeV2 + + saved = json.loads(existing[0]) + if saved.get("kind") == "repair": + from hypertrain.aggregator.rollback_v2 import RepairTape + + repair = RepairTape.from_bytes(self.objects.get(saved["tape_hash"])) + return {"repair": repair.model_dump(mode="json"), **saved} + tape = TapeV2.from_bytes(self.objects.get(saved["tape_hash"])) + return {"tape": tape.body_json(), **saved} + history = [s for s in self._history_v2(run_id) if s.w < w] + require_apply(run_id, w, history) + inputs = self.verified_inputs_v2(run_id, w) + policy = AggregationPolicyV2.model_validate_json( + self.objects.get(manifest.network.aggregation_policy_hash) + ) + economics = self.objects.get(manifest.network.economics_policy_hash) + if w: + previous = self._record_v2(run_id, "applied", str(w - 1)) + prev_state, predecessor = str(previous["out_state"]), str(previous["tape_hash"]) + else: + theta = init_params(TrainConfig.from_manifest_v2(manifest).model) + prev_state = self.objects.put( + OuterState.init( + {k: v.detach().cpu().numpy() for k, v in theta.items()} + ).to_bytes() + ) + predecessor = "0" * 64 + tape = make_tape( + self.objects, + manifest, + policy, + economics, + self._coord(), + w=w, + prev_state=prev_state, + predecessor_tape_hash=predecessor, + inputs=inputs, + reference_reward_units=self._services(run_id)[0].policy.R_collectible_units, + ) + self.require_roster_v2(run_id, w) + replay_tape( + self.objects, + tape, + manifest, + policy, + economics, + signer=self._coord().ss58, + w=w, + prev_state=prev_state, + predecessor_tape_hash=predecessor, + inputs=inputs, + reference_reward_units=self._services(run_id)[0].policy.R_collectible_units, + ) + tape_hash = self.objects.put(tape.to_bytes()) + applied = { + "tape_hash": tape_hash, + "out_state": tape.body.out_state, + "theta_hash": tape.body.out_hashes["theta_hash"], + "prev_state": prev_state, + "predecessor_tape_hash": predecessor, + } + self._put_record_v2(run_id, "applied", str(w), applied) + # Persist historical authority facts, never replace with future released locks. + self._put_record_v2( + run_id, + "tape-inputs", + str(w), + {"inputs": [self._serialize_input_v2(x) for x in inputs]}, + ) + finality = self._record_v2(run_id, "finality", str(w)) + finality["applied"], finality["state_hash"] = True, applied["theta_hash"] + self._put_record_v2(run_id, "finality", str(w), finality) + return {"tape": tape.body_json(), **applied} + + @staticmethod + def _serialize_input_v2(work: Any) -> dict[str, Any]: + return { + "roster": work.roster.body(), + "commit": work.commit.model_dump(mode="json"), + "delta_manifest": work.delta_manifest.model_dump(mode="json"), + "replay": asdict(work.replay), + "funding": asdict(work.funding), + "settlement": asdict(work.settlement), + "assignment_hash": work.assignment_hash, + "clean_finalizations": work.clean_finalizations, + } + + def finalize_v2(self, run_id: str, w: int, raw: bytes) -> dict[str, Any]: + from hypertrain.challenge.finality_v2 import RoundStatus, settle + from hypertrain.ledger.escrow_v2 import RewardWork, authority_message, reward_origin_id + from hypertrain.protocol.messages_v2 import RewardFinalize + + with self._tx(): + manifest = self._run_v2(run_id) + if self._service_boundary_v2(run_id, w) is not None: + final_request = envelope_v2.Intake( + run_id, {"Finalize": manifest.training.coord_pubkey.__eq__} + ).accept(raw, self._now(self._db)) + assert isinstance(final_request, Finalize) + original_applied = self._record_v2(run_id, "applied", str(w)) + from hypertrain.protocol.messages_v2 import RoundOpenV2 + + final_opening = envelope_v2.Intake( + run_id, {"RoundOpenV2": manifest.training.coord_pubkey.__eq__} + ).accept( + canonicalize(self._record_v2(run_id, "round", str(w))), self._now(self._db) + ) + assert isinstance(final_opening, RoundOpenV2) + if ( + final_request.w != w + or final_request.final_theta_hash_w1 != original_applied["theta_hash"] + or original_applied.get("kind") == "repair" + or final_request.included + != sorted(entry.hotkey for entry in final_opening.roster) + ): + raise ChallengeError(409, "SERVICE_FINALIZE_AUTHORITY") + self.aggregate_v2(run_id, w) + self._service_boundary_v2(run_id, w, heavy=True) + model = self._authenticated_v2(run_id, raw, "Finalize", manifest.training.coord_pubkey) + assert isinstance(model, Finalize) + previous_final = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='finalize' AND id=?", + (run_id, str(w)), + ).fetchone() + if previous_final is not None: + if json.loads(previous_final[0])["body"] != model.model_dump(mode="json"): + raise ChallengeError(409, "conflicting finalized origin authority") + return self._record_v2(run_id, "finalize-result", str(w)) + self.require_roster_v2(run_id, w) + applied = self._record_v2(run_id, "applied", str(w)) + if applied.get("kind") == "repair": + raise ChallengeError( + 409, "repair evidence is model-only; cannot mint miner rewards" + ) + status = RoundStatus(**self._record_v2(run_id, "finality", str(w))) + included = sorted(x.roster.hotkey for x in self.verified_inputs_v2(run_id, w)) + if model.w != w or model.included != included: + raise ChallengeError( + 422, "finalized included set differs from full accepted authority" + ) + now = self._fresh_v2(manifest, f"finalize:{w}:{self._now(self._db)}") + final_hash = body_digest(model.model_dump(mode="json")) + settled = settle(status, model, final_hash) + self._put_record_v2(run_id, "finality", str(w), asdict(settled)) + self._put_record_v2( + run_id, "finalize", str(w), envelope_v2.parse_envelope(raw).model_dump(mode="json") + ) + self._put_record_v2( + run_id, + "replayed-finality", + str(w), + { + "finalize_hash": final_hash, + "tape_hash": str(applied["tape_hash"]), + "finalized_beacon": now, + "shadow": False, + "theta_hash": str(applied["theta_hash"]), + }, + ) + works = [] + for hotkey in included: + identity = f"{w}:{hotkey}" + row = self._db.execute( + "SELECT challenge FROM audit_leases_v2 WHERE run_id=? AND w=? AND hotkey=?", + (run_id, w, hotkey), + ).fetchone() + works.append( + RewardWork( + envelope_v2.parse_envelope(self._record_v2(run_id, "commit", identity)), + envelope_v2.parse_envelope(self._record_v2(run_id, "verdict", identity)), + envelope_v2.parse_envelope(json.loads(row[0])), + self._owner_v2(run_id, hotkey), + self._assignment_v2(run_id, w, hotkey), + ) + ) + escrow = self._services(run_id)[0] + vesting = now + manifest.training.verify.E_vest_rounds + evidence = FinalityEvidence( + envelope_v2.parse_envelope(raw), + tuple(works), + str(applied["tape_hash"]), + False, + now, + vesting, + ) + total = sum(int(str(item.commit.body["tokens"])) for item in works) + units = { + item.commit.signer: escrow.policy.round_reward_units + * int(str(item.commit.body["tokens"])) + // total + for item in works + } + remainder = escrow.policy.round_reward_units - sum(units.values()) + for hot in included[:remainder]: + units[hot] += 1 + origins = [ + { + "origin_id": reward_origin_id(run_id, w, h, final_hash), + "owner": self._owner_v2(run_id, h), + "units": units[h], + "mature_at": vesting, + } + for h in included + if units[h] + ] + reward = RewardFinalize( + run_id=run_id, + w=w, + finalize_hash=final_hash, + tape_hash=str(applied["tape_hash"]), + verdict_root=sha256_hex( + canonicalize([body_digest(item.verdict.body) for item in works]) + ), + allocation_hash=sha256_hex(canonicalize(origins)), + budget_units=escrow.policy.round_reward_units, + origin_ids=[str(o["origin_id"]) for o in origins], + mature_at=vesting, + authority_sig="0" * 128, + ) + reward = reward.model_copy( + update={"authority_sig": self._coord().sign(authority_message(reward)).hex()} + ) + receipt = escrow.reward_finalize(reward, evidence) + for origin in origins: + self._put_record_v2( + run_id, + "settlement", + str(origin["origin_id"]), + Settlement( + finality_hash=final_hash, + closed_dispute_root=sha256_hex(canonicalize([])), + release_beacon=vesting, + unresolved=False, + outcome="MATCH", + ).body(), + ) + answer: dict[str, JsonValue] = { + "status": "SETTLED", + "reward_receipt": receipt.body(), + "theta_hash": model.final_theta_hash_w1, + } + self._put_record_v2(run_id, "finalize-result", str(w), answer) + return answer + + def relay_assignment_v2(self, run_id: str, w: int, hotkey: str) -> dict[str, Any]: + from hypertrain.protocol import relay_envelope + from hypertrain.protocol.relay_messages import RelayAssignment, RelayNetworkManifest + + with self._tx(): + manifest = self._run_v2(run_id) + self.require_roster_v2(run_id, w) + self._owner_v2(run_id, hotkey) + self._record_v2(run_id, "assignment", f"{w}:{hotkey}") + registry = self._registry_v2(manifest) + network = RelayNetworkManifest( + run_id=run_id, + base_manifest_hash=manifest.training.training_hash(), + registry_hash=registry.digest(), + specs_hash=sha256_hex(canonicalize([s.body() for s in registry.specs])), + assignment_policy="master-observed-median3-v1", + ) + prior = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='relay-assignment' AND id=?", + (run_id, f"{w}:{hotkey}"), + ).fetchone() + if prior is not None: + return { + "assignment": json.loads(prior[0]), + "registry": registry.body(), + "network_manifest": self._record_v2(run_id, "relay-network", "current"), + } + choices = sorted(registry.specs, key=lambda s: (s.region, s.id)) + disabled = { + r[0] + for r in self._db.execute( + "SELECT id FROM records_v2 WHERE run_id=? AND kind='relay-disabled'", (run_id,) + ) + } + now = self._now(self._db) + draining = { + r[0] + for r in self._db.execute( + "SELECT id FROM records_v2 WHERE run_id=? AND kind='relay-draining'", (run_id,) + ) + } + choices = [ + s + for s in choices + if s.id not in draining + and any( + k.valid_from_round <= now < k.valid_until_round + and f"{s.id}:{registry.epoch}:{k.key_id}" not in disabled + for k in s.pubkeys + ) + ] + if not choices: + raise ChallengeError(503, "no approved healthy relay; direct rescue required") + body = self._record_v2(run_id, "round", str(w))["body"] + assignment = RelayAssignment( + w=w, + hotkey=hotkey, + primary_id=choices[0].id, + fallback_ids=[s.id for s in choices[1:]], + assignment_epoch=registry.epoch, + manifest_hash=network.digest(), + exp_drand=int(str(body["d_upload"])), + ) + env = relay_envelope.seal( + self._coord(), "RelayAssignment", run_id, assignment, assignment.exp_drand + ) + self._put_record_v2(run_id, "relay-assignment", f"{w}:{hotkey}", env) + self._put_record_v2( + run_id, + "relay-network", + "current", + relay_envelope.seal(self._coord(), "RelayNetworkManifest", run_id, network, NEVER), + ) + return { + "assignment": env, + "registry": registry.body(), + "network_manifest": self._record_v2(run_id, "relay-network", "current"), + } + + def relay_settlement_v2(self, run_id: str, grant_hash: str) -> dict[str, Any]: + """Authenticated control-plane adapter for L4 retention release, not client facts.""" + from hypertrain.relay.core import Settlement as RelaySettlement + + with self._lock: + grant = self._record_v2(run_id, "grant", grant_hash)["body"] + w = int(grant["w"]) + status = self._record_v2(run_id, "finality", str(w)) + if ( + status["disposition"] != "SETTLED" + or status["unresolved_audits"] + or status["unresolved_disputes"] + ): + raise ChallengeError(409, "relay custody remains pinned by unresolved finality") + accepted = self._record_v2(run_id, "replayed-finality", str(w)) + turns = [ + json.loads(r[0]) + for r in self._db.execute( + "SELECT turn FROM disputes_v2 WHERE json_extract(turn,'$.contest.w')=?", (w,) + ) + ] + if any(t["resolution"] is None or t["resolved_beacon"] is None for t in turns): + raise ChallengeError(409, "relay custody remains dispute pinned") + closed = [t["resolution"] for t in turns] + settled = RelaySettlement( + str(accepted["finalize_hash"]), + int(accepted["finalized_beacon"]), + sha256_hex(canonicalize(closed)), + max([int(accepted["finalized_beacon"]), *[t["resolved_beacon"] for t in turns]]), + int(accepted["finalized_beacon"]) + + self._run_v2(run_id).training.verify.E_vest_rounds, + ) + return asdict(settled) + + def dispute_state_v2(self, run_id: str, dispute_id: str, checkpoint: int) -> dict[str, Any]: + _, _, disputes = self._services(run_id) + with self._tx(): + self._snapshot_current_v2(self._run_v2(run_id)) + event = disputes.state_request(dispute_id, checkpoint, now=self._now(self._db)) + disputes.notify() + self._notify_processes_v2() + return event.model_dump(mode="json") + + def referee_v2(self, run_id: str, dispute_id: str) -> dict[str, Any]: + """Internal full anchored referee execution; HTTP never supplies replay truth.""" + import shutil + + from hypertrain.auditor.island_bisect import IslandParty, adjudicate + from hypertrain.auditor.replay import AnchorCache + from hypertrain.auditor.worker import LeaseGuard + from hypertrain.miner.island_launch import launch_island + from hypertrain.protocol.messages_v2 import IslandJobV1 + + _, _, disputes = self._services(run_id) + with self._tx(): + manifest = self._run_v2(run_id) + backend = self._backend_v2(manifest) + self._snapshot_current_v2(manifest) + turn = disputes.get(dispute_id) + if turn.resolution is not None or self._now(self._db) >= turn.absolute_deadline: + raise ChallengeError(409, "referee outside unresolved absolute horizon") + resource = self._service_boundary_v2(run_id, turn.contest.w) + if resource is not None: + from hypertrain.aggregator.capacity_worker import bounded_bytes + from hypertrain.challenge.disputes_v2 import WatchEvent, authentic + from hypertrain.data.store import LocalFSStore + from hypertrain.protocol.messages import Receipt, ReplayVerdict + from hypertrain.protocol.messages_v2 import RoundOpenV2 + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.model import param_shapes + + now = self._now(self._db) + opening = envelope_v2.Intake( + run_id, {"RoundOpenV2": manifest.training.coord_pubkey.__eq__} + ).accept(canonicalize(self._record_v2(run_id, "round", str(turn.contest.w))), now) + assert isinstance(opening, RoundOpenV2) + event_row = self._db.execute( + "SELECT event FROM dispute_events_v2 " + "WHERE json_extract(event,'$.turn.dispute_id')=? ORDER BY cursor DESC LIMIT 1", + (dispute_id,), + ).fetchone() + event = WatchEvent.model_validate_json(event_row[0]) if event_row else None + if ( + turn.run_id != run_id + or turn.dispute_id != dispute_id + or event is None + or not authentic(event) + or event.signer != manifest.training.coord_pubkey + or event.turn != turn + or event.issued_beacon > now + or opening.w != turn.contest.w + or opening.roster_hash != resource["roster_hash"] + or turn.contest.miner not in {r.hotkey for r in opening.roster} + or turn.contest.auditor not in manifest.training.auditors + ): + raise ChallengeError(409, "SERVICE_REFEREE_TURN_AUTHORITY") + claim = self._record_v2(run_id, "verdict", f"{turn.contest.w}:{turn.contest.miner}") + verdict = envelope_v2.Intake( + run_id, {"ReplayVerdict": turn.contest.auditor.__eq__} + ).accept(canonicalize(claim), now) + assert isinstance(verdict, ReplayVerdict) + if ( + body_digest(claim["body"]) != turn.contest.verdict_hash + or verdict.challenge_hash != turn.contest.challenge_hash + ): + raise ChallengeError(409, "SERVICE_REFEREE_EVIDENCE_AUTHORITY") + metadata = self._record_v2(run_id, "trace-geometry", turn.contest.verdict_hash) + preflight_job = IslandJobV1.model_validate(metadata["job"]) + directory = self.state_dir / "referee-v2" / dispute_id / turn.transcript_hash + deadline = manifest.training.beacon.genesis_time + (turn.absolute_deadline - 1) * 3 + expected_descriptor = { + "job": preflight_job.model_copy(update={"deadline": deadline}).body(), + "historical_geometry_hash": body_digest(metadata), + "dispute_id": dispute_id, + "transcript_hash": turn.transcript_hash, + "seq": turn.seq, + "referee": turn.contest.referee, + "absolute_deadline": turn.absolute_deadline, + "backend": backend, + } + current_descriptor = self._record_v2( + run_id, "referee-job", f"{dispute_id}:{turn.transcript_hash}" + ) + receipt = envelope_v2.Intake( + run_id, {"Receipt": manifest.training.coord_pubkey.__eq__} + ).accept(canonicalize(current_descriptor["receipt"]), now) + assert isinstance(receipt, Receipt) + if ( + receipt.w != turn.contest.w + or receipt.commit_hash != current_descriptor["descriptor_hash"] + or receipt.received_round > now + or preflight_job.manifest != manifest + or preflight_job.run_id != run_id + or preflight_job.w != turn.contest.w + or set(preflight_job.object_paths) + != {"start_state", "ef_in", "v0", "samples", "sample_proofs"} + or any( + Path(p).is_absolute() or ".." in Path(p).parts + for p in preflight_job.object_paths.values() + ) + ): + raise ChallengeError(409, "SERVICE_REFEREE_DESCRIPTOR_AUTHORITY") + if not isinstance(self.objects, LocalFSStore): + raise ChallengeError(503, "SERVICE_REFEREE_LOCAL_STORE") + descriptor_bytes = bounded_bytes( + self.objects._path(str(current_descriptor["descriptor_hash"])), 1 << 20 + ) + if ( + sha256_hex(descriptor_bytes) != current_descriptor["descriptor_hash"] + or envelope_v2.load_json(descriptor_bytes, max_bytes=1 << 20) + != expected_descriptor + ): + raise ChallengeError(409, "SERVICE_REFEREE_DESCRIPTOR_BINDING") + from hypertrain.protocol.messages_v2 import StartStateV2 + + original_start = StartStateV2.model_validate( + self._record_v2(run_id, "start", f"{turn.contest.w}:{turn.contest.miner}") + ) + anchor_record = self._record_v2( + run_id, "anchor", f"{turn.contest.w - 1}:{turn.contest.miner}" + ) + if ( + original_start.run_id != run_id + or original_start.w != turn.contest.w + or original_start.hotkey != turn.contest.miner + or original_start.state_object_sha256 != preflight_job.start_state_sha256 + or original_start.ef_object_sha256 != preflight_job.ef_in_sha256 + or anchor_record["hotkey"] != turn.contest.miner + or anchor_record["w"] != turn.contest.w - 1 + or anchor_record["anchor_hash"] != original_start.parent_anchor_hash + or anchor_record["proof_hash"] != original_start.anchor_verdict_hash + or anchor_record["backend"] not in ("genesis", backend) + ): + raise ChallengeError(409, "SERVICE_REFEREE_ANCHOR_AUTHORITY") + published = Path(str(metadata["directory"])) + root = self.state_dir / "audit-geometry-v2" + if ( + not published.is_relative_to(root) + or ".." in published.parts + or published.name != "published" + ): + raise ChallengeError(409, "SERVICE_REFEREE_PATH") + shapes = param_shapes(TrainConfig.from_manifest_v2(manifest).model) + for name, key in ( + ("start_state", preflight_job.start_state_sha256), + ("ef_in", preflight_job.ef_in_sha256), + ("v0", preflight_job.v0_sha256), + ): + data = bounded_bytes(published.parent / preflight_job.object_paths[name], 65536) + if sha256_hex(data) != key: + raise ChallengeError(422, "SERVICE_REFEREE_OBJECT_HASH") + if len(data) < 8 or struct.unpack_from(" len(data) - 8: + raise ChallengeError(422, "SERVICE_REFEREE_STATE_HEADER") + size = struct.unpack_from(" 4096: + raise ChallengeError(422, "SERVICE_REFEREE_TRACE_SIZE") + for entry in trace_data: + trace_entry = TraceEntry.model_validate(entry) + if ( + trace_entry.rank != rank + or not 0 <= trace_entry.step <= manifest.training.inner.H + or any( + dim < 0 + or dim + > max( + manifest.training.model.seq_len, + manifest.training.model.d_model, + manifest.training.model.vocab, + ) + * 4 + for dim in trace_entry.shape + ) + or math.prod(trace_entry.shape) * 4 > 65536 + ): + raise ChallengeError(422, "SERVICE_REFEREE_TRACE_SHAPE") + # Trace decoding, anchor restore and replay stay beyond the profile guard. + self._service_boundary_v2(run_id, turn.contest.w, heavy=True) + geometry = self._record_v2(run_id, "trace-geometry", turn.contest.verdict_hash) + job = IslandJobV1.model_validate(geometry["job"]) + if job.run_id != run_id or job.w != turn.contest.w or job.manifest != manifest: + raise ChallengeError(409, "historical referee geometry subject differs") + self._restore_anchor_v2(run_id, turn.contest.miner, turn.contest.w - 1, AnchorCache()) + miner = IslandParty.published(turn.contest.miner, job, Path(str(geometry["directory"]))) + original = Path(str(geometry["directory"])).parent + # Published claim must match the independently accepted ordinary replay. + accepted = self._record_v2(run_id, "verdict", f"{turn.contest.w}:{turn.contest.miner}")[ + "body" + ] + directory = self.state_dir / "referee-v2" / dispute_id / turn.transcript_hash + directory.mkdir(parents=True, exist_ok=True) + for relative in job.object_paths.values(): + source = original / relative + target = directory / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, target) + snapshot = turn.transcript_hash + deadline = manifest.training.beacon.genesis_time + (turn.absolute_deadline - 1) * 3 + replay_job = job.model_copy(update={"deadline": deadline}) + descriptor: dict[str, JsonValue] = { + "job": replay_job.body(), + "historical_geometry_hash": body_digest(geometry), + "dispute_id": dispute_id, + "transcript_hash": snapshot, + "seq": turn.seq, + "referee": turn.contest.referee, + "absolute_deadline": turn.absolute_deadline, + "backend": backend, + } + descriptor_hash = self.objects.put(canonicalize(descriptor)) + key = f"{dispute_id}:{snapshot}" + old = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='referee-job' AND id=?", + (run_id, key), + ).fetchone() + if old is not None and json.loads(old[0])["descriptor_hash"] != descriptor_hash: + raise ChallengeError(409, "immutable referee descriptor conflicts") + if old is not None: + receipt_raw = json.loads(old[0])["receipt"] + receipt = envelope_v2.parse_envelope(receipt_raw) + if ( + receipt.type != "Receipt" + or receipt.run_id != run_id + or receipt.signer != self._coord().ss58 + or receipt.body["commit_hash"] != descriptor_hash + or receipt.body["w"] != turn.contest.w + or not envelope_v2.verify_envelope(receipt_raw) + ): + raise ChallengeError(409, "accepted referee descriptor signature differs") + if old is None: + self._put_record_v2( + run_id, + "referee-job", + key, + { + "descriptor_hash": descriptor_hash, + "receipt": envelope_v2.seal( + self._coord(), + "Receipt", + run_id, + { + "w": turn.contest.w, + "commit_hash": descriptor_hash, + "received_round": self._now(self._db), + }, + NEVER, + ), + }, + ) + guard = LeaseGuard(turn.absolute_deadline, turn.absolute_deadline, deadline) + self._lease_guards_v2[key] = guard + try: + with guard: + guard.check() + launch = ( + self._role_launch( + "referee", + replay_job, + directory, + { + "run_id": run_id, + "dispute_id": dispute_id, + "turn": turn.model_dump(mode="json"), + "descriptor_hash": descriptor_hash, + "descriptor": descriptor, + "descriptor_receipt": self._record_v2(run_id, "referee-job", key)[ + "receipt" + ], + "historical_geometry": geometry, + }, + ) + if self._role_launch is not None + else launch_island + ) + artifacts = launch( + replay_job, directory, backend=backend, trace=True, cancel=guard.cancelled + ) + guard.check() + finally: + self._lease_guards_v2.pop(key, None) + referee = IslandParty.published(turn.contest.referee, replay_job, artifacts.directory) + # MATCH ordinary audit already authenticated these full trajectory commitments. + auditor = IslandParty.published(turn.contest.auditor, replay_job, artifacts.directory) + if turn.level == "op" and turn.paused and turn.interval[1] - turn.interval[0] == 1: + from hypertrain.auditor.island_bisect import RefereeEvidence + from hypertrain.protocol.messages_v2 import BisectV2 + + with self._lock: + answers = [ + BisectV2.model_validate_json(r[0]) + for r in self._db.execute( + "SELECT body FROM dispute_entries_v2 WHERE id=? ORDER BY seq DESC LIMIT 2", + (dispute_id,), + ) + ] + if ( + len(answers) != 2 + or {a.party for a in answers} != {turn.contest.miner, turn.contest.auditor} + or any( + a.level != "op" or a.ctx != turn.ctx or a.interval != turn.interval + for a in answers + ) + ): + raise ChallengeError(409, "final op lacks both exact signed claimant outputs") + lo, hi = turn.interval + at = [lo, -(-(lo + hi) // 2), hi] + truth = referee.hashes("op", tuple(turn.ctx), at) + if any(a.hashes[0] != truth[0] for a in answers): + raise ChallengeError( + 409, "independent full replay does not reproduce agreed op input" + ) + wrong = [a.party for a in answers if a.hashes[-1] != truth[-1]] + if len(wrong) != 1: + raise ChallengeError(409, "independent op cannot identify one wrong claimant") + evidence = RefereeEvidence( + dispute_id=dispute_id, + transcript_hash=snapshot, + reason="FRAUD", + loser=wrong[0], + predecessor_hash=referee.state_root(turn.ctx[0] - 1), + inputs_hash=truth[0], + output_hash=truth[-1], + op_spec=referee.op_name(tuple(turn.ctx), hi), + ) + else: + if accepted["result"] != "MATCH": + raise ChallengeError( + 409, "mismatch contest must reach exact signed final-op outputs" + ) + evidence = adjudicate((dispute_id, snapshot), (miner, auditor), referee) + with self._tx(): + self._snapshot_current_v2(manifest) + current = disputes.get(dispute_id) + if ( + current.transcript_hash != snapshot + or self._now(self._db) >= current.absolute_deadline + ): + raise ChallengeError(409, "referee transcript changed during independent execution") + digest = disputes.register_evidence(evidence) + self._put_record_v2( + run_id, + "referee-artifacts", + digest, + { + "descriptor_hash": descriptor_hash, + "directory": str(artifacts.directory), + "historical_geometry_hash": body_digest(geometry), + }, + ) + return {"evidence_hash": digest, "evidence": evidence.body()} + + def dispute_timeout_v2(self, run_id: str, dispute_id: str, raw: bytes) -> dict[str, Any]: + from hypertrain.challenge.disputes_v2 import Availability + + data = envelope_v2.load_json(raw) + if ( + set(data) != {"witnesses"} + or not isinstance(data["witnesses"], list) + or len(data["witnesses"]) != 2 + ): + raise ChallengeError(422, "two original signed permitted witness observations required") + proofs = tuple(Availability.model_validate(x) for x in data["witnesses"]) + _, _, disputes = self._services(run_id) + with self._tx(): + self._snapshot_current_v2(self._run_v2(run_id)) + result = disputes.timeout(dispute_id, (proofs[0], proofs[1]), now=self._now(self._db)) + if result.resolution is not None: + disputes.settle_lock(dispute_id, now=self._now(self._db)) + w = result.contest.w + status = self._record_v2(run_id, "finality", str(w)) + status["unresolved_disputes"] = self._db.execute( + "SELECT COUNT(*) FROM disputes_v2 " + "WHERE json_extract(turn,'$.contest.w')=? " + "AND json_extract(turn,'$.resolution') IS NULL", + (w,), + ).fetchone()[0] + self._put_record_v2(run_id, "finality", str(w), status) + disputes.notify() + self._notify_processes_v2() + return result.body() + + def rollback_v2(self, run_id: str, raw: bytes) -> dict[str, Any]: + """Bounded real inner recomputation, fresh replay, then one atomic repair commit.""" + from dataclasses import replace + + from hypertrain.aggregator.rollback_v2 import execute_repair_context + from hypertrain.auditor.replay import AnchorCache + from hypertrain.challenge.finality_v2 import RoundStatus, exclude, finish_rollback + from hypertrain.protocol.messages import Rollback + + with self._tx(): + manifest = self._run_v2(run_id) + model = self._authenticated_v2(run_id, raw, "Rollback", self._coord().ss58) + assert isinstance(model, Rollback) + if self._service_boundary_v2(run_id, model.w) is not None: + preflight_first = self._record_v2(run_id, "applied", str(model.w)) + self._rollback_context_v2( + run_id, + model.w, + preflight_first["prev_state"], + preflight_first["predecessor_tape_hash"], + [], + body_digest(model.model_dump(mode="json")), + profile_request=raw, + ) + existing = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='rollback' AND id=?", + (run_id, str(model.w)), + ).fetchone() + if existing: + saved = json.loads(existing[0]) + if saved["request"]["body"] != model.model_dump(mode="json"): + raise ChallengeError(409, "conflicting rollback") + return saved["result"] + self._snapshot_current_v2(manifest) + statuses = [] + for w in (model.w, model.w + 1): + self.require_roster_v2(run_id, w) + status = RoundStatus(**self._record_v2(run_id, "finality", str(w))) + statuses.append(status) + if status.disposition == "SETTLED": + raise ChallengeError(409, "finalized late fraud is money-only") + if not status.applied or status.unresolved_audits or status.unresolved_disputes: + raise ChallengeError(409, "rollback requires applied closed audit/dispute work") + if model.recomputed != ["agg_w", "step_w", "agg_w1", "step_w1"]: + raise ChallengeError(422, "rollback requires complete inner and outer work") + old = self._record_v2(run_id, "applied", str(model.w + 1)) + if old.get("kind") == "repair": + raise ChallengeError( + 409, "already repaired history requires exact original rollback" + ) + if model.old_theta_hash_w2 != old["theta_hash"]: + raise ChallengeError(409, "rollback original theta subject differs") + if self._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='round' " + "AND CAST(id AS INTEGER)>?", + (run_id, model.w + 1), + ).fetchone(): + raise ChallengeError( + 409, "rollback cannot rewrite beyond speculative two-round window" + ) + cause = self._rollback_exclusions_v2(run_id, model) + snapshot_before_reservation = self._rollback_snapshot_v2(run_id) + operation = body_digest(model.model_dump(mode="json")) + reservation = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='repair-reservation' AND id=?", + (run_id, operation), + ).fetchone() + if reservation is None: + active = self._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='repair-reservation' " + "AND json_extract(data,'$.state')='RUNNING'", + (run_id,), + ).fetchone() + if active: + raise ChallengeError(409, "repair reservation already occupied") + now = self._now(self._db) + self._put_record_v2( + run_id, + "repair-reservation", + operation, + { + "state": "RUNNING", + "created": now, + "absolute": now + 200, + "max_attempts": 2, + "source_revision": snapshot_before_reservation, + "attempts": 1, + "steps": 2 + * manifest.training.inner.H + * sum( + len(self._record_v2(run_id, "tape-inputs", str(round_w))["inputs"]) + - len(model.excluded) + for round_w in (model.w, model.w + 1) + ), + }, + ) + else: + reserved = json.loads(reservation[0]) + if reserved["source_revision"] != snapshot_before_reservation: + raise ChallengeError(409, "rollback reserved source revision changed") + if self._now(self._db) >= reserved["absolute"] or reserved["attempts"] >= 2: + raise ChallengeError(409, "repair reservation exhausted") + reserved["attempts"] += 1 + self._put_record_v2(run_id, "repair-reservation", operation, reserved) + snapshot = self._rollback_snapshot_v2(run_id) + first_applied = self._record_v2(run_id, "applied", str(model.w)) + first_context = self._rollback_context_v2( + run_id, + model.w, + first_applied["prev_state"], + first_applied["predecessor_tape_hash"], + cause, + operation, + ) + first = execute_repair_context( + self.objects, + manifest, + first_context, + self._coord().ss58, + key=self._coord(), + ) + checked_first = execute_repair_context( + self.objects, + manifest, + first_context, + self._coord().ss58, + tape=first.tape, + ) + first_hash = self.objects.put(first.tape.to_bytes()) + with self._tx(): + if self._rollback_snapshot_v2(run_id) != snapshot: + raise ChallengeError(409, "rollback authority revision changed") + second_context = self._rollback_context_v2( + run_id, + model.w + 1, + first.tape.body.arithmetic.out_state, + first_hash, + cause, + operation, + anchors=checked_first.anchors, + ) + second = execute_repair_context( + self.objects, + manifest, + second_context, + self._coord().ss58, + key=self._coord(), + ) + checked_second = execute_repair_context( + self.objects, + manifest, + second_context, + self._coord().ss58, + tape=second.tape, + ) + second_hash = self.objects.put(second.tape.to_bytes()) + if (model.new_theta_hash_w2, model.new_outer_state_hash) != ( + checked_second.tape.body.arithmetic.out_hashes["theta_hash"], + checked_second.tape.body.arithmetic.out_state, + ): + raise ChallengeError(422, "signed rollback outputs differ from actual repaired work") + with self._tx(): + self._snapshot_current_v2(manifest) + if self._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='rollback' AND id=?", + (run_id, str(model.w)), + ).fetchone(): + return self._record_v2(run_id, "rollback", str(model.w))["result"] + reserved = self._record_v2(run_id, "repair-reservation", operation) + if self._now(self._db) >= reserved["absolute"]: + raise ChallengeError(409, "repair absolute deadline expired") + if self._rollback_snapshot_v2(run_id) != snapshot: + raise ChallengeError(409, "rollback authority revision changed") + for result, context, tape_hash, status in ( + (checked_first, first_context, first_hash, statuses[0]), + (checked_second, second_context, second_hash, statuses[1]), + ): + body = result.tape.body.arithmetic + original = self._record_v2(run_id, "applied", str(body.w)) + self._put_record_v2(run_id, "original-applied", str(body.w), original) + self._put_record_v2(run_id, "repair-context", str(body.w), context) + for e in cause: + self._put_record_v2(run_id, "excluded", f"{body.w}:{e['hotkey']}", e) + applied = { + "tape_hash": tape_hash, + "out_state": body.out_state, + "theta_hash": body.out_hashes["theta_hash"], + "prev_state": body.prev_state, + "predecessor_tape_hash": body.predecessor_tape_hash, + "kind": "repair", + } + self._put_record_v2(run_id, "applied", str(body.w), applied) + cache = AnchorCache() + for anchor in result.anchors: + cache.entries[(run_id, anchor.hotkey, body.w, anchor.layout_hash)] = anchor + old_anchor = self._record_v2(run_id, "anchor", f"{body.w}:{anchor.hotkey}") + self._put_record_v2( + run_id, "original-anchor", f"{body.w}:{anchor.hotkey}", old_anchor + ) + self._persist_anchor_v2(manifest, anchor, cache) + if status.w == model.w: + excluded_status = exclude(status, model.cause_hashes[0]) + repaired = finish_rollback( + excluded_status, + model, + model.new_theta_hash_w2, + model.new_outer_state_hash, + ) + repaired = replace(repaired, state_hash=body.out_hashes["theta_hash"]) + else: + repaired = replace( + status, + disposition="EXCLUDED", + rollback_complete=True, + resolution_hash=model.cause_hashes[0], + state_hash=body.out_hashes["theta_hash"], + ) + self._put_record_v2(run_id, "finality", str(body.w), asdict(repaired)) + result_data: dict[str, Any] = { + "status": "REPAIRED", + "tape_hashes": [first_hash, second_hash], + "theta_hash": model.new_theta_hash_w2, + "outer_state_hash": model.new_outer_state_hash, + "reward_units": 0, + } + self._put_record_v2( + run_id, + "rollback", + str(model.w), + { + "request": envelope_v2.parse_envelope(raw).model_dump(mode="json"), + "result": result_data, + }, + ) + reserved["state"] = "COMPLETE" + self._put_record_v2(run_id, "repair-reservation", operation, reserved) + return result_data + + def _rollback_snapshot_v2(self, run_id: str) -> str: + """Revision digest excludes only reservation bookkeeping, not live source authorities.""" + records = self._db.execute( + "SELECT kind,id,data FROM records_v2 WHERE run_id=? AND kind!='repair-reservation' " + "ORDER BY kind,id", + (run_id,), + ).fetchall() + admissions = self._db.execute("SELECT * FROM admissions_v2 ORDER BY hotkey").fetchall() + disputes = self._db.execute("SELECT * FROM disputes_v2 ORDER BY id").fetchall() + intake = self._db.execute( + "SELECT key,digest,envelope,accepted_beacon,receipt FROM accepted_v2 ORDER BY key" + ).fetchall() + origins = self._db.execute( + "SELECT * FROM escrow_units ORDER BY origin,owner,bucket,ref" + ).fetchall() + return sha256_hex( + canonicalize( + [ + [ + [ + {"bytes": value.hex()} if isinstance(value, bytes) else value + for value in row + ] + for row in rows + ] + for rows in (records, admissions, disputes, intake, origins) + ] + ) + ) + + def _rollback_exclusions_v2(self, run_id: str, model: Any) -> list[dict[str, Any]]: + """Only accepted independent dispute resolutions justify removing applied contributions.""" + evidence = [] + for hotkey in model.excluded: + rows = self._db.execute( + "SELECT turn FROM disputes_v2 WHERE miner=? AND json_extract(turn,'$.contest.w')=?", + (hotkey, model.w), + ).fetchall() + matched = [] + for row in rows: + turn = json.loads(row[0]) + resolution = turn["resolution"] + if resolution is None: + continue + digest = body_digest(resolution) + if digest not in model.cause_hashes: + continue + if ( + turn["contest"]["miner"] != hotkey + or resolution["reason"] + not in ( + "FRAUD", + "WITHHELD", + "PARTY_TIMEOUT", + ) + or resolution["loser"] != hotkey + ): + raise ChallengeError(409, "rollback cause does not exclude this miner") + signed = self._record_v2(run_id, "resolution", turn["dispute_id"]) + policy = DisputePolicyV2.model_validate_json( + self.objects.get(self._run_v2(run_id).network.dispute_policy_hash) + ) + if ( + not envelope_v2.verify_envelope(signed) + or signed["body"] != resolution + or ( + signed["signer"] != turn["contest"]["referee"] + or signed["signer"] not in policy.referees + or signed["type"] != "ResolutionV2" + or signed["run_id"] != run_id + or turn["run_id"] != run_id + ) + ): + raise ChallengeError(409, "rollback cause lacks original referee signature") + independent = self._db.execute( + "SELECT body FROM dispute_evidence_v2 WHERE hash=?", + (resolution["evidence_hash"],), + ).fetchone() + if independent is None: + raise ChallengeError(409, "rollback cause lacks independent referee work") + matched.append(digest) + if len(matched) != 1: + raise ChallengeError(409, "rollback exclusion lacks exact accepted referee cause") + evidence.append({"hotkey": hotkey, "reason": "FRAUD", "evidence_hash": matched[0]}) + if {e["evidence_hash"] for e in evidence} != set(model.cause_hashes): + raise ChallengeError(409, "rollback cause set differs") + return evidence + + def rollback_preview_v2(self, run_id: str, raw: bytes) -> dict[str, Any]: + """Compute a bounded repair proposal; cannot change applied/finality/reward records.""" + from hypertrain.aggregator.rollback_v2 import execute_repair_context + from hypertrain.protocol.messages import Rollback + + with self._tx(): + manifest = self._run_v2(run_id) + model = envelope_v2.Intake( + run_id, + {"Rollback": self._coord().ss58.__eq__}, + ).accept(raw, self._now(self._db)) + assert isinstance(model, Rollback) + if self._service_boundary_v2(run_id, model.w) is not None: + first = self._record_v2(run_id, "applied", str(model.w)) + self._rollback_context_v2( + run_id, + model.w, + first["prev_state"], + first["predecessor_tape_hash"], + [], + body_digest(model.model_dump(mode="json")), + profile_request=raw, + ) + causes = self._rollback_exclusions_v2(run_id, model) + operation = sha256_hex( + b"repair-preview|" + bytes.fromhex(body_digest(model.model_dump(mode="json"))) + ) + now = self._now(self._db) + previous_preview = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='repair-preview' AND id=?", + (run_id, operation), + ).fetchone() + if previous_preview: + return json.loads(previous_preview[0]) + reserved = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='repair-reservation' AND id=?", + (run_id, operation), + ).fetchone() + if reserved: + raise ChallengeError(409, "repair preview reservation consumed") + self._put_record_v2( + run_id, + "repair-reservation", + operation, + { + "state": "PREVIEW", + "created": now, + "absolute": now + 200, + "max_attempts": 1, + "attempts": 1, + }, + ) + revision = self._rollback_snapshot_v2(run_id) + first = self._record_v2(run_id, "applied", str(model.w)) + context = self._rollback_context_v2( + run_id, + model.w, + first["prev_state"], + first["predecessor_tape_hash"], + causes, + operation, + ) + computed = execute_repair_context( + self.objects, manifest, context, self._coord().ss58, key=self._coord() + ) + first_hash = self.objects.put(computed.tape.to_bytes()) + with self._tx(): + if self._rollback_snapshot_v2(run_id) != revision: + raise ChallengeError(409, "repair preview source revision changed") + context = self._rollback_context_v2( + run_id, + model.w + 1, + computed.tape.body.arithmetic.out_state, + first_hash, + causes, + operation, + anchors=computed.anchors, + ) + result = execute_repair_context( + self.objects, manifest, context, self._coord().ss58, key=self._coord() + ) + answer = { + "new_theta_hash_w2": result.tape.body.arithmetic.out_hashes["theta_hash"], + "new_outer_state_hash": result.tape.body.arithmetic.out_state, + } + with self._tx(): + if self._rollback_snapshot_v2(run_id) != revision: + raise ChallengeError(409, "repair preview source revision changed") + self._put_record_v2(run_id, "repair-preview", operation, answer) + return answer + + def _rollback_context_v2( + self, + run_id: str, + w: int, + prev_state: str, + predecessor: str, + excluded: list[dict[str, Any]], + operation: str, + *, + anchors: Any = None, + profile_request: bytes | None = None, + ) -> dict[str, Any]: + from hypertrain.aggregator.checkpoint import network_inputs + from hypertrain.aggregator.tape_v2 import TapeV2 + from hypertrain.auditor.replay import AnchorCache, pack_state, tensor_root + from hypertrain.protocol.messages_v2 import AuditJobV2 + from hypertrain.trainer.compress import state_hash + + manifest = self._run_v2(run_id) + resource = self._service_boundary_v2(run_id, w) + if resource is not None: + from hypertrain.aggregator.capacity_worker import bounded_bytes + from hypertrain.auditor.island_bisect import RefereeEvidence + from hypertrain.challenge.disputes_v2 import Turn + from hypertrain.data.store import LocalFSStore + from hypertrain.protocol.envelope_v2 import tape_signing_message + from hypertrain.protocol.messages import Rollback + from hypertrain.protocol.messages_v2 import RoundOpenV2 + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.model import param_shapes + + if not isinstance(self.objects, LocalFSStore): + raise ChallengeError(503, "SERVICE_ROLLBACK_LOCAL_STORE") + now = self._now(self._db) + if profile_request is None: + raise ChallengeError(409, "SERVICE_ROLLBACK_REQUEST_REQUIRED") + rollback_request = envelope_v2.Intake( + run_id, {"Rollback": manifest.training.coord_pubkey.__eq__} + ).accept(profile_request, now) + assert isinstance(rollback_request, Rollback) + if ( + rollback_request.w != w + or body_digest(rollback_request.model_dump(mode="json")) != operation + or rollback_request.recomputed != ["agg_w", "step_w", "agg_w1", "step_w1"] + ): + raise ChallengeError(409, "SERVICE_ROLLBACK_REQUEST_AUTHORITY") + policy_raw = bounded_bytes( + self.objects._path(manifest.network.dispute_policy_hash), 65536 + ) + if sha256_hex(policy_raw) != manifest.network.dispute_policy_hash: + raise ChallengeError(422, "SERVICE_ROLLBACK_POLICY_HASH") + policy = DisputePolicyV2.model_validate_json(policy_raw) + causes = [] + for hotkey in rollback_request.excluded: + rows = self._db.execute( + "SELECT turn FROM disputes_v2 WHERE miner=? " + "AND json_extract(turn,'$.contest.w')=?", + (hotkey, w), + ).fetchall() + matches = [] + for row in rows: + accepted_turn = Turn.model_validate_json(row[0]) + resolution = accepted_turn.resolution + if ( + resolution is None + or resolution.digest() not in rollback_request.cause_hashes + ): + continue + signed_resolution = self._record_v2( + run_id, "resolution", accepted_turn.dispute_id + ) + resolved = envelope_v2.Intake( + run_id, {"ResolutionV2": accepted_turn.contest.referee.__eq__} + ).accept(canonicalize(signed_resolution), now) + evidence_row = self._db.execute( + "SELECT body FROM dispute_evidence_v2 WHERE hash=?", + (resolution.evidence_hash,), + ).fetchone() + if evidence_row is None: + raise ChallengeError(409, "SERVICE_ROLLBACK_EVIDENCE_MISSING") + evidence = RefereeEvidence.model_validate_json(evidence_row[0]) + if ( + accepted_turn.run_id != run_id + or accepted_turn.contest.miner != hotkey + or accepted_turn.contest.referee not in policy.referees + or resolved != resolution + or resolution.dispute_id != accepted_turn.dispute_id + or resolution.transcript_hash != accepted_turn.transcript_hash + or resolution.reason not in ("FRAUD", "WITHHELD", "PARTY_TIMEOUT") + or resolution.loser != hotkey + or evidence.digest() != resolution.evidence_hash + or evidence.dispute_id != accepted_turn.dispute_id + or evidence.transcript_hash != resolution.transcript_hash + or evidence.reason != resolution.reason + or evidence.loser != hotkey + ): + raise ChallengeError(409, "SERVICE_ROLLBACK_EVIDENCE_AUTHORITY") + artifact = self._record_v2( + run_id, "referee-artifacts", resolution.evidence_hash + ) + current_descriptor = self._record_v2( + run_id, + "referee-job", + f"{accepted_turn.dispute_id}:{accepted_turn.transcript_hash}", + ) + signed_receipt = envelope_v2.parse_envelope(current_descriptor["receipt"]) + if ( + artifact["descriptor_hash"] != current_descriptor["descriptor_hash"] + or signed_receipt.type != "Receipt" + or signed_receipt.run_id != run_id + or signed_receipt.signer != manifest.training.coord_pubkey + or signed_receipt.body["commit_hash"] + != current_descriptor["descriptor_hash"] + or not envelope_v2.verify_envelope(current_descriptor["receipt"]) + ): + raise ChallengeError(409, "SERVICE_ROLLBACK_DESCRIPTOR_AUTHORITY") + descriptor_raw = bounded_bytes( + self.objects._path(str(current_descriptor["descriptor_hash"])), 1 << 20 + ) + descriptor = envelope_v2.load_json(descriptor_raw, max_bytes=1 << 20) + if ( + sha256_hex(descriptor_raw) != current_descriptor["descriptor_hash"] + or descriptor["dispute_id"] != accepted_turn.dispute_id + or descriptor["transcript_hash"] != accepted_turn.transcript_hash + or descriptor["referee"] != accepted_turn.contest.referee + ): + raise ChallengeError(409, "SERVICE_ROLLBACK_DESCRIPTOR_BINDING") + matches.append(resolution.digest()) + if len(matches) != 1: + raise ChallengeError(409, "SERVICE_ROLLBACK_CAUSE_AUTHORITY") + causes.extend(matches) + if set(causes) != set(rollback_request.cause_hashes): + raise ChallengeError(409, "SERVICE_ROLLBACK_CAUSE_SET") + previous_tape = None + for round_w in (w, w + 1): + estimate = self._service_boundary_v2(run_id, round_w) + assert estimate is not None + opening = envelope_v2.Intake( + run_id, {"RoundOpenV2": manifest.training.coord_pubkey.__eq__} + ).accept(canonicalize(self._record_v2(run_id, "round", str(round_w))), now) + assert isinstance(opening, RoundOpenV2) + if opening.w != round_w or opening.roster_hash != estimate["roster_hash"]: + raise ChallengeError(409, "SERVICE_ROLLBACK_ROSTER") + original = self._record_v2(run_id, "applied", str(round_w)) + if ( + round_w == w + 1 + and original["theta_hash"] != rollback_request.old_theta_hash_w2 + ): + raise ChallengeError(409, "SERVICE_ROLLBACK_OLD_THETA") + tape_raw = bounded_bytes(self.objects._path(str(original["tape_hash"])), 1 << 20) + if sha256_hex(tape_raw) != original["tape_hash"]: + raise ChallengeError(422, "SERVICE_ROLLBACK_TAPE_HASH") + signed_tape = TapeV2.from_bytes(tape_raw) + tape_body = signed_tape.body + if ( + signed_tape.signer != manifest.training.coord_pubkey + or tape_body.run_id != run_id + or tape_body.w != round_w + or not verify( + decode_hotkey(signed_tape.signer), + tape_signing_message(tape_body.body(), run_id), + bytes.fromhex(signed_tape.sig), + ) + or tape_body.policy_hash != manifest.network.aggregation_policy_hash + or tape_body.policy_hashes.body() != opening.policy_hashes.body() + or tape_body.prev_state != original["prev_state"] + or tape_body.predecessor_tape_hash != original["predecessor_tape_hash"] + or tape_body.out_state != original["out_state"] + or tape_body.out_hashes["theta_hash"] != original["theta_hash"] + or {x.roster.hotkey for x in tape_body.inputs} + | {x.hotkey for x in tape_body.excluded} + != {x.hotkey for x in opening.roster} + or any(x.roster not in opening.roster for x in tape_body.inputs) + or ( + previous_tape is None + and ( + tape_body.prev_state != prev_state + or tape_body.predecessor_tape_hash != predecessor + ) + ) + or ( + previous_tape is not None + and ( + tape_body.prev_state != previous_tape[1] + or tape_body.predecessor_tape_hash != previous_tape[0] + ) + ) + ): + raise ChallengeError(409, "SERVICE_ROLLBACK_TAPE_AUTHORITY") + previous_tape = (original["tape_hash"], tape_body.out_state) + data = bounded_bytes(self.objects._path(tape_body.prev_state), 65536) + if sha256_hex(data) != tape_body.prev_state: + raise ChallengeError(422, "SERVICE_ROLLBACK_STATE_HASH") + if len(data) < 8 or struct.unpack_from(" len(data) - 8: + raise ChallengeError(422, "SERVICE_ROLLBACK_STATE_HEADER") + size = struct.unpack_from(" 0 and anchors is None: + if w > 16: + raise ChallengeError( + 409, "bounded authenticated predecessor replay exceeds16rounds" + ) + for prior_w in range(w): + applied = self._record_v2(run_id, "applied", str(prior_w)) + entry = { + **applied, + "round_open": self._record_v2(run_id, "round", str(prior_w)), + "inputs": self._record_v2(run_id, "tape-inputs", str(prior_w))["inputs"], + "reference_reward_units": self._services(run_id)[0].policy.R_collectible_units, + } + if applied.get("kind") == "repair": + context = self._record_v2(run_id, "repair-context", str(prior_w)) + entry["repair_context"] = context + entry["inputs"] = [s["work"] for s in context["body"]["sources"]] + else: + jobs = [] + for work in entry["inputs"]: + identity = f"{prior_w}:{work['roster']['hotkey']}" + replay = self._record_v2(run_id, "replay", identity) + rows = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='audit-job' " + "AND json_extract(data,'$.start_state.w')=? " + "AND json_extract(data,'$.start_state.hotkey')=?", + (run_id, prior_w, work["roster"]["hotkey"]), + ).fetchall() + matching = [ + json.loads(r[0]) + for r in rows + if AuditJobV2.model_validate(json.loads(r[0])).start_state.digest() + == replay["anchor_hash"] + ] + if len(matching) != 1: + raise ChallengeError(409, "authenticated predecessor audit job missing") + jobs.extend(matching) + entry["audit_jobs"] = jobs + history.append(entry) + source = self._record_v2(run_id, "applied", str(w)) + tape = TapeV2.from_bytes(self.objects.get(source["tape_hash"])) + historical = self._record_v2(run_id, "tape-inputs", str(w))["inputs"] + excluded_ids = {e["hotkey"] for e in excluded} + if len(excluded_ids) != len(excluded) or not excluded_ids <= { + x["roster"]["hotkey"] for x in historical + }: + raise ChallengeError(409, "rollback exclusion set not in original accepted inputs") + retained = [x for x in historical if x["roster"]["hotkey"] not in excluded_ids] + # Validate feasibility before launching any work; never silently truncate. + from hypertrain.aggregator.weighted_v2 import Candidate, allocate_weights + + aggregation = AggregationPolicyV2.model_validate_json( + self.objects.get(manifest.network.aggregation_policy_hash) + ) + works = network_inputs(retained) + allocate_weights([Candidate(x.roster, "0" * 64, "0" * 64) for x in works], aggregation) + supplied = {a.hotkey: a for a in anchors} if anchors is not None else {} + items = [] + backend = self._backend_v2(manifest) + for work, data in zip(works, retained, strict=True): + hotkey = work.roster.hotkey + current = self._services(run_id)[1].status(hotkey, now=self._now(self._db)) + if not current.eligible or current.record.admission_id != work.roster.admission_id: + raise ChallengeError(409, "repair retained contributor no longer funded/eligible") + anchor = supplied.get(hotkey) + if anchor is None: + anchor = self._restore_anchor_v2(run_id, hotkey, w - 1, AnchorCache()) + start = self._record_v2(run_id, "start", f"{w}:{hotkey}") + if (start["parent_anchor_hash"], start["anchor_verdict_hash"]) != ( + anchor.anchor_hash, + anchor.proof_hash, + ): + raise ChallengeError(409, "repair original carried anchor authority differs") + import torch + + if any( + not bool(torch.isfinite(x).all()) + for values in ( + anchor.theta, + anchor.state.m, + anchor.state.v, + anchor.ef, + ) + for x in values.values() + ): + raise ChallengeError(409, "nonfinite authenticated repair anchor") + envelopes, receipts = {}, [] + for kind, obj in (("commit", work.commit), ("delta", work.delta_manifest)): + envelope = self._record_v2(run_id, kind, f"{w}:{hotkey}") + msg = "CommitV2" if kind == "commit" else "DeltaManifestV2" + reservation = _dumps(list(envelope_v2.replay_key(msg, run_id, hotkey, obj))) + row = self._db.execute( + "SELECT * FROM accepted_v2 WHERE key=?", (reservation,) + ).fetchone() + if row is None or row["digest"] != body_digest(envelope["body"]): + raise ChallengeError(409, "repair source lacks original durable intake") + receipt = json.loads(row["receipt"]) + if ( + not envelope_v2.verify_envelope(receipt) + or receipt["signer"] != self._coord().ss58 + or ( + receipt["type"] != "Receipt" + or receipt["run_id"] != run_id + or receipt["body"]["commit_hash"] != row["digest"] + or receipt["body"]["w"] != w + or receipt["body"]["received_round"] != row["accepted_beacon"] + ) + ): + raise ChallengeError(409, "repair source intake receipt authority differs") + original = envelope_v2.parse_envelope(row["envelope"]).model_dump(mode="json") + if original != envelope: + raise ChallengeError(409, "repair source original envelope changed") + envelopes[kind] = self.objects.put(canonicalize(original)) + receipts.append(receipt) + items.append( + { + "work": data, + "commit_envelope_hash": envelopes["commit"], + "delta_envelope_hash": envelopes["delta"], + "acceptance_hash": sha256_hex(canonicalize(receipts)), + "samples": list(self._assignment_v2(run_id, w, hotkey)), + "anchor": { + "hotkey": hotkey, + "w": w - 1, + "layout_hash": anchor.layout_hash, + "proof_hash": anchor.proof_hash, + "anchor_hash": anchor.anchor_hash, + "backend": anchor.backend, + "state_object": self.objects.put(pack_state(anchor.theta, anchor.state)), + "ef_object": self.objects.put(pack_state(anchor.ef)), + "state_root": tensor_root(anchor.theta, anchor.state), + "ef_hash": state_hash(anchor.ef), + }, + "receipts": receipts, + } + ) + ref = manifest.training.reference_spec + reference_hash = sha256_hex(canonicalize(ref.model_dump(mode="json"))) + selection = self._record_v2(run_id, "execution-backend", "run") + if selection["backend"] != backend: + raise ChallengeError(409, "repair immutable backend authority differs") + qualification: dict[str, Any] = {"execution_backend": selection} + if backend == "cuda": + reviewed = json.loads(self.objects.get(selection["authority_hash"])) + if ( + reviewed.get("reference_hash") != reference_hash + or reviewed.get("layout_hash") != ref.layout.model_dump_json() + ): + raise ChallengeError(409, "repair reviewed qualification subject differs") + qualification["reviewed_qualification"] = reviewed + repair_reservation = self._record_v2(run_id, "repair-reservation", operation) + dataset = self._record_v2(run_id, "dataset", "inputs") + exclusion_proofs = [] + for exclusion in excluded: + row = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='resolution' " + "AND json_extract(data,'$.body.loser')=?", + (run_id, exclusion["hotkey"]), + ).fetchall() + signed = next( + json.loads(r[0]) + for r in row + if body_digest(json.loads(r[0])["body"]) == exclusion["evidence_hash"] + ) + independent = self._db.execute( + "SELECT body FROM dispute_evidence_v2 WHERE hash=?", + (signed["body"]["evidence_hash"],), + ).fetchone() + exclusion_proofs.append( + {"resolution": signed, "referee_evidence": json.loads(independent[0])} + ) + body = { + "v": "ht-rollback-context/1", + "run_id": run_id, + "w": w, + "manifest": manifest.body(), + "history": history, + "source_tape_hash": source["tape_hash"], + "prev_state": prev_state, + "predecessor_tape_hash": predecessor, + "sources": items, + "excluded": excluded + [e.body() for e in tape.body.excluded], + "exclusion_proofs": exclusion_proofs, + "qualification_hash": sha256_hex(canonicalize(qualification)), + "qualification": qualification, + "reference_hash": reference_hash, + "layout_hash": AnchorCache.layout_hash(manifest), + "backend": backend, + "reference_reward_units": self._services(run_id)[0].policy.R_collectible_units, + "samples_hash": dataset["samples_hash"], + "proofs_hash": dataset["proofs_hash"], + "reservation": repair_reservation, + "timeout_seconds": max( + 1, min(600, (repair_reservation["absolute"] - self._now(self._db)) * 3) + ), + } + message = b"hypertrain/rollback/context/1|" + sha256_hex(canonicalize(body)).encode() + return { + "body": body, + "signer": self._coord().ss58, + "sig": self._coord().sign(message).hex(), + } + + def upload_grant_v2( + self, run_id: str, raw: bytes, commit_hash: str | None = None + ) -> dict[str, Any]: + from hypertrain.protocol import relay_envelope + from hypertrain.protocol.messages_v2 import CommitV2 + from hypertrain.protocol.relay_messages import ( + CHUNK_BYTES, + RelayAssignment, + UploadChunk, + UploadChunkManifest, + UploadGrant, + ) + + # UploadChunkManifest is miner-signed in its relay domain, not a trust flag. + env = relay_envelope.parse_envelope(raw) + if commit_hash is not None and ( + len(commit_hash) != 64 or any(c not in "0123456789abcdef" for c in commit_hash) + ): + raise ChallengeError(422, "commit subject must be canonical body digest") + with self._tx(): + manifest = self._run_v2(run_id) + self._owner_v2(run_id, env.signer) + chunk = relay_envelope.Intake( + run_id, {"UploadChunkManifest": env.signer.__eq__} + ).accept(raw, self._now(self._db)) + assert isinstance(chunk, UploadChunkManifest) + assignments = self._db.execute( + "SELECT id,data FROM records_v2 WHERE run_id=? " + "AND kind='relay-assignment' AND id LIKE ?", + (run_id, "%:" + env.signer), + ).fetchall() + eligible = [] + for row in assignments: + assignment_env = relay_envelope.parse_envelope(json.loads(row["data"])) + a = RelayAssignment.model_validate(assignment_env.body) + if ( + assignment_env.run_id != run_id + or assignment_env.type != "RelayAssignment" + or assignment_env.signer != self._coord().ss58 + or not relay_envelope.verify_envelope(assignment_env.model_dump()) + or a.hotkey != env.signer + or row["id"] != f"{a.w}:{env.signer}" + or assignment_env.exp_drand != a.exp_drand + ): + raise ChallengeError(409, "relay assignment differs from accepted work subject") + if self._now(self._db) >= a.exp_drand: + continue + committed = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='commit' AND id=?", + (run_id, f"{a.w}:{env.signer}"), + ).fetchone() + if committed is None: + continue + commit_env = envelope_v2.parse_envelope(json.loads(committed[0])) + if commit_hash is not None and body_digest(commit_env.body) != commit_hash: + continue + committed_work = CommitV2.model_validate(commit_env.body) + if ( + commit_env.run_id != run_id + or commit_env.signer != env.signer + or commit_env.type != "CommitV2" + or not envelope_v2.verify_envelope(commit_env.model_dump()) + or committed_work.w != a.w + or committed_work.hotkey != env.signer + ): + raise ChallengeError(409, "commit differs from signed relay assignment subject") + if chunk.size != committed_work.delta_bytes: + continue + try: + original = self.objects.get(committed_work.delta_hash) + except ObjectNotFound: + continue + if len(original) != chunk.size or sha256_hex(original) != committed_work.delta_hash: + raise ChallengeError(409, "accepted commit original delta corrupt") + expected = UploadChunkManifest( + size=len(original), + chunks=[ + UploadChunk( + index=i, + off=off, + len=len(original[off : off + CHUNK_BYTES]), + chunk_sha256=sha256_hex(original[off : off + CHUNK_BYTES]), + ) + for i, off in enumerate(range(0, len(original), CHUNK_BYTES)) + ], + ) + if expected == chunk: + eligible.append(a) + if len(eligible) != 1: + raise ChallengeError(409, "one current signed relay assignment required") + assignment = eligible[0] + registry = self._registry_v2(manifest) + spec = next(s for s in registry.specs if s.id == assignment.primary_id) + now = self._now(self._db) + if self._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='relay-draining' AND id=?", + (run_id, spec.id), + ).fetchone() is not None or not any( + k.valid_from_round <= now < k.valid_until_round + and self._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='relay-disabled' AND id=?", + (run_id, f"{spec.id}:{assignment.assignment_epoch}:{k.key_id}"), + ).fetchone() + is None + for k in spec.pubkeys + ): + raise ChallengeError( + 409, "original relay drained or key disabled; use signed fallback" + ) + commit = self._record_v2(run_id, "commit", f"{assignment.w}:{env.signer}")["body"] + if chunk.size != commit["delta_bytes"]: + raise ChallengeError(422, "grant size differs from accepted commit") + grant = UploadGrant( + w=assignment.w, + hotkey=env.signer, + relay_id=assignment.primary_id, + assignment_epoch=assignment.assignment_epoch, + delta_hash=str(commit["delta_hash"]), + size=chunk.size, + chunk_manifest_hash=chunk.chunk_manifest_hash(), + retain_until=assignment.exp_drand + + manifest.training.verify.E_vest_rounds + + manifest.training.verify.T.dispute_per_level * 20 + + 100, + nonce=sha256_hex(canonicalize([run_id, assignment.body(), chunk.body()])), + exp_drand=assignment.exp_drand, + ) + signed = relay_envelope.seal( + self._coord(), "UploadGrant", run_id, grant, grant.exp_drand + ) + self._put_record_v2(run_id, "grant", grant.digest(), signed) + self._put_record_v2(run_id, "chunk-manifest", grant.digest(), chunk.body()) + return signed + + async def relay_receipt_v2(self, raw: bytes, client: Any) -> dict[str, Any]: + import base64 + + from hypertrain.data.stream_store import spool + from hypertrain.protocol import relay_envelope + from hypertrain.protocol.relay_messages import RelayReceipt, RetrievalRequest, UploadGrant + from hypertrain.relay.client import RelayClient, receipt_verified + + env = relay_envelope.parse_envelope(raw) + run_id = env.run_id + receipt = RelayReceipt.model_validate(env.body) + with self._tx(): + manifest = self._run_v2(run_id) + grant = UploadGrant.model_validate( + self._record_v2(run_id, "grant", receipt.grant_hash)["body"] + ) + registry = self._registry_v2(manifest) + body = self._record_v2(run_id, "round", str(grant.w))["body"] + now = self._fresh_v2( + manifest, f"relay-receipt:{receipt.digest()}:{self._now(self._db)}" + ) + deadline = int(str(body["d_upload"])) + receipt_verified( + env, + registry=registry, + relay_id=grant.relay_id, + run_id=run_id, + grant=grant, + received_before=deadline, + ) + assignment = self._record_v2(run_id, "relay-assignment", f"{grant.w}:{grant.hotkey}")[ + "body" + ] + request = RetrievalRequest( + request_id=secrets.token_hex(32), + nonce=secrets.token_hex(32), + relay_id=grant.relay_id, + key_id=receipt.key_id, + assignment_hash=body_digest(assignment), + grant_hash=grant.digest(), + custody_ack_hashes=[], + receipt_hash=receipt.digest(), + retention_hash=receipt.digest(), + object_or_chunk_hash=grant.delta_hash, + size=grant.size, + requested_beacon=now, + deadline_beacon=min(now + 10, receipt.retain_until), + ) + request_raw = relay_envelope.parse_envelope( + relay_envelope.seal( + self._coord(), "RetrievalRequest", run_id, request, request.deadline_beacon + ) + ) + self._put_record_v2( + run_id, "retrieval", request.digest(), request_raw.model_dump(mode="json") + ) + spec = next(s for s in registry.specs if s.id == grant.relay_id) + capability = base64.b64encode( + canonicalize(self._record_v2(run_id, "grant", grant.digest())) + ).decode() + custody_response = await client.get( + spec.https_url + f"/v1/uploads/{grant.digest()}/acks", + headers={"X-Upload-Grant": capability}, + ) + custody_response.raise_for_status() + custody = custody_response.json() + relay = RelayClient(registry, client) + with spool() as payload: + response = await relay.retrieval(request_raw, payload, relay_public=env.signer) + payload.seek(0) + original = payload.read() + with self._tx(): + current = self._fresh_v2( + manifest, f"relay-accept:{receipt.digest()}:{self._now(self._db)}" + ) + if ( + current >= deadline + or sha256_hex(original) != grant.delta_hash + or len(original) != grant.size + ): + raise ChallengeError( + 422, "master independently retrieved mismatched or late payload" + ) + self.objects.put(original) + from hypertrain.protocol.relay_messages import MasterAcceptanceV2 + + acceptance = MasterAcceptanceV2( + w=grant.w, + hotkey=grant.hotkey, + delta_hash=grant.delta_hash, + receipt_hash=receipt.digest(), + received_drand=current, + ) + signed = relay_envelope.seal( + self._coord(), "MasterAcceptanceV2", run_id, acceptance, grant.retain_until + ) + self._put_record_v2( + run_id, "relay-receipt", receipt.digest(), env.model_dump(mode="json") + ) + self._put_record_v2(run_id, "relay-custody", grant.digest(), {"artifacts": custody}) + self._put_record_v2( + run_id, "retrieval-response", request.digest(), response.model_dump(mode="json") + ) + self._put_record_v2(run_id, "master-acceptance", acceptance.digest(), signed) + return signed diff --git a/src/hypertrain/challenge/trust_v2.py b/src/hypertrain/challenge/trust_v2.py new file mode 100644 index 0000000..9cbb179 --- /dev/null +++ b/src/hypertrain/challenge/trust_v2.py @@ -0,0 +1,165 @@ +"""Audit floors and influence gates consume authenticated accounting/replay facts.""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from typing import Literal, assert_never + +from hypertrain.protocol.messages import VerifySpec +from hypertrain.protocol.messages_v2 import ( + AdmissionState, + EconomicsPolicyV2, + RosterEntryV2, +) + + +class TrustError(ValueError): + """Influence lacks funded, settled, independently verified work.""" + + +@dataclass(frozen=True, slots=True) +class FundedStatus: + """L2 supplies collectible origin-backed locks, never available/unmatured promises.""" + + run_id: str + hotkey: str + admission_id: str + ledger_mode: Literal["test", "production"] + locked_units: int + collectible_reward_units: int + lock_receipt_hash: str + conservative_bound: bool + + +@dataclass(frozen=True, slots=True) +class SettlementStatus: + """L5/L0 authenticated status at the transaction's current state revision.""" + + run_id: str + w: int + hotkey: str + unresolved_dispute: bool + excluded: bool + status_hash: str + + +@dataclass(frozen=True, slots=True) +class ReplayEvidence: + """L1 authenticated full anchored replay outputs; not a norm-screen attestation. + + These are independently recomputed values, not hashes copied from the commit. + L0 obtains them only from authorized replay acceptance, binding own assignment. + """ + + run_id: str + w: int + hotkey: str + assignment_hash: str + anchor_hash: str + leaves_root: str + delta_hash: str + final_theta_hash: str + ef_in_hash: str + ef_out_hash: str + verdict_hash: str + result: Literal["MATCH", "MISMATCH", "WITHHELD", "BAD_PROOF", "ASSIGNMENT_VIOLATION"] + audit_mode: Literal["anchored-full"] + + +@dataclass(frozen=True, slots=True) +class AuditFloor: + q_ppm: int + required_locked_units: int + influence_allowed: bool + + +def nominal_q(state: AdmissionState, clean_finalizations: int) -> int: + if clean_finalizations < 0: + raise TrustError("negative clean count") + match state: + case "PROBATION": + return ( + 1_000_000 + if clean_finalizations < 4 + else (500_000 if clean_finalizations < 8 else 250_000) + ) + case "ACTIVE": + return 250_000 + case "APPLIED" | "SUSPENDED" | "BANNED": + return 1_000_000 + case unreachable: + assert_never(unreachable) + + +def audit_floor( + policy: EconomicsPolicyV2, + verify: VerifySpec, + roster: RosterEntryV2, + funding: FundedStatus, + *, + run_id: str, + clean_finalizations: int, + reference_reward_units: int, + anomaly_ppm: int = 0, +) -> AuditFloor: + """Ceil integer economics: q*s*beta*(R+S)>=G; phase q remains one.""" + if ( + (funding.run_id, funding.hotkey, funding.admission_id, funding.ledger_mode) + != (run_id, roster.hotkey, roster.admission_id, policy.ledger_mode) + or min(funding.locked_units, funding.collectible_reward_units, reference_reward_units) < 0 + or not 0 <= anomaly_ppm <= 1_000_000 + ): + raise TrustError("accounting status/bounds differ") + reward = min(funding.collectible_reward_units, policy.R_collectible_units) + exposure = reward + funding.locked_units + denominator = policy.s_lower_ppm * policy.beta_ppm * exposure + economic = ( + (policy.G_max_units * 10**18 + denominator - 1) // denominator + if denominator + else (1_000_001 if policy.G_max_units else 0) + ) + q = max(nominal_q(roster.state, clean_finalizations), anomaly_ppm, policy.q_floor, economic) + penalty_den = min(q, 1_000_000) * policy.s_lower_ppm * policy.beta_ppm + required_exposure = ( + (policy.G_max_units * 10**18 + penalty_den - 1) // penalty_den + if penalty_den + else (policy.G_max_units + exposure + 1) + ) + required = max( + policy.S_min_units, + verify.s_min_units(reference_reward_units), + required_exposure - reward, + 0, + ) + live = (roster.state == "PROBATION" and clean_finalizations >= 4) or ( + roster.state == "ACTIVE" and clean_finalizations >= 12 + ) + allowed = ( + live + and funding.conservative_bound + and q <= 1_000_000 + and policy.s_lower_ppm > 0 + and funding.locked_units >= required + ) + return AuditFloor(min(q, 1_000_000), required, allowed) + + +@dataclass(frozen=True, slots=True) +class AuditSelection: + selected: bool + mode: Literal["anchored-full"] | None + final_transition: bool + compression_and_ef: bool + + +def select_audit( + run_id: str, w: int, signature_hash: str, hotkey: str, q_ppm: int +) -> AuditSelection: + """Exact integer draw from the pinned beacon; q=0 is not a final-only audit.""" + if type(w) is not int or w < 0 or type(q_ppm) is not int or not 0 <= q_ppm <= 1_000_000: + raise TrustError("invalid audit draw bounds") + seed = f"ht-audit-v2|{run_id}|{w}|{signature_hash}|{hotkey}".encode() + x = int.from_bytes(hashlib.sha256(seed).digest(), "big") + selected = x * 1_000_000 < q_ppm * 2**256 + return AuditSelection(selected, "anchored-full" if selected else None, selected, selected) diff --git a/src/hypertrain/data/stream_store.py b/src/hypertrain/data/stream_store.py new file mode 100644 index 0000000..953e097 --- /dev/null +++ b/src/hypertrain/data/stream_store.py @@ -0,0 +1,251 @@ +"""Bounded transfers over existing LocalFS/S3 stores, shared conditional metadata. + +S3 requires strong read-after-write consistency and conditional PutObject support. +LocalFS metadata uses flock on shared POSIX storage, not a pod's emptyDir. +""" + +from __future__ import annotations + +import datetime as dt +import fcntl +import hashlib +import os +import tempfile +from collections.abc import AsyncIterator, Callable, Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import BinaryIO + +import anyio +import httpx + +from hypertrain.data.store import ( + CorruptObjectError, + LocalFSStore, + ObjectNotFound, + S3Store, + StoreError, + _check_key, + sigv4_headers, +) + +BLOCK = 64 << 10 + + +class MetadataConflict(StoreError): + """Another replica committed the same metadata generation first.""" + + +@contextmanager +def spool() -> Iterator[BinaryIO]: + """Disk-only temporary transfer; never materialize a whole object in RAM.""" + with tempfile.TemporaryFile() as file: + yield file + + +async def receive(source: AsyncIterator[bytes], file: BinaryIO, size: int, sha: str) -> None: + digest, received = hashlib.sha256(), 0 + with anyio.fail_after(90): + async for data in source: + received += len(data) + if received > size: + raise CorruptObjectError("stream exceeds declared size") + digest.update(data) + for off in range(0, len(data), BLOCK): + file.write(data[off : off + BLOCK]) + if received != size or digest.hexdigest() != _check_key(sha): + raise CorruptObjectError("stream hash/size mismatch") + file.seek(0) + + +async def blocks(file: BinaryIO) -> AsyncIterator[bytes]: + while data := file.read(BLOCK): + yield data + await anyio.lowlevel.checkpoint() + + +class StreamStore: + """Content objects immutable; one bounded CAS journal shared by all replicas.""" + + def __init__(self, store: LocalFSStore | S3Store, client: httpx.AsyncClient) -> None: + self.store, self.client = store, client + + def _headers( + self, method: str, url: str, digest: str, extra: dict[str, str] | None = None + ) -> dict[str, str]: + if not isinstance(self.store, S3Store): + raise StoreError("HTTP backing requires S3") + return { + **sigv4_headers( + method, + url, + self.store._creds, + self.store.region, + dt.datetime.now(dt.UTC).strftime("%Y%m%dT%H%M%SZ"), + digest, + extra=extra, + ), + **(extra or {}), + } + + async def put(self, file: BinaryIO, sha: str, size: int) -> str: + """Caller supplies a verified spool; backing bytes rechecked before publish.""" + _check_key(sha) + file.seek(0) + digest, count = hashlib.sha256(), 0 + while data := file.read(BLOCK): + digest.update(data) + count += len(data) + if count != size or digest.hexdigest() != sha: + raise CorruptObjectError("unverified durable upload") + file.seek(0) + if isinstance(self.store, LocalFSStore): + target = self.store._path(sha) + target.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=target.parent) as temp: + while data := file.read(BLOCK): + temp.write(data) + temp.flush() + os.fsync(temp.fileno()) + try: + os.link(temp.name, target) + except FileExistsError: + with spool() as check: + await self.get(sha, size, check) + with _directory(target.parent): + pass + else: + url = self.store._url(sha) + headers = self._headers("PUT", url, sha, {"If-None-Match": "*"}) + headers["Content-Length"] = str(size) + response = await self.client.put(url, content=blocks(file), headers=headers) + if response.status_code == 412: + with spool() as check: + await self.get(sha, size, check) + else: + response.raise_for_status() + return sha + + async def get(self, sha: str, size: int, file: BinaryIO) -> None: + _check_key(sha) + if isinstance(self.store, LocalFSStore): + path = self.store._path(sha) + if not path.is_file(): + raise ObjectNotFound(sha) + with path.open("rb") as source: + await receive(blocks(source), file, size, sha) + else: + url = self.store._url(sha) + async with self.client.stream( + "GET", url, headers=self._headers("GET", url, hashlib.sha256(b"").hexdigest()) + ) as r: + if r.status_code == 404: + raise ObjectNotFound(sha) + r.raise_for_status() + await receive(r.aiter_raw(BLOCK), file, size, sha) + + async def delete(self, sha: str) -> None: + _check_key(sha) + if isinstance(self.store, LocalFSStore): + path = self.store._path(sha) + path.unlink(missing_ok=True) + if path.parent.exists(): + with _directory(path.parent): + pass + else: + url = self.store._url(sha) + response = await self.client.delete( + url, headers=self._headers("DELETE", url, hashlib.sha256(b"").hexdigest()) + ) + response.raise_for_status() + + async def journal(self, name: str) -> tuple[bytes, str | None]: + """The mutable journal key is not a content object; bounded to 16MiB.""" + _check_key(name) + if isinstance(self.store, LocalFSStore): + path = self.store.root / "metadata" / name + if not path.exists(): + return b"", None + if path.stat().st_size > 16 << 20: + raise StoreError("journal exceeds metadata quota") + data = path.read_bytes() + return data, hashlib.sha256(data).hexdigest() + url = self.store._url(name) + ".journal" + async with self.client.stream( + "GET", url, headers=self._headers("GET", url, hashlib.sha256(b"").hexdigest()) + ) as response: + if response.status_code == 404: + return b"", None + response.raise_for_status() + collected = bytearray() + async for part in response.aiter_bytes(BLOCK): + collected.extend(part) + if len(collected) > 16 << 20: + raise StoreError("journal exceeds metadata quota") + return bytes(collected), response.headers["etag"] + + async def compare_swap(self, name: str, data: bytes, etag: str | None) -> None: + _check_key(name) + if len(data) > 16 << 20: + raise StoreError("journal exceeds metadata quota") + if isinstance(self.store, LocalFSStore): + directory = self.store.root / "metadata" + directory.mkdir(exist_ok=True) + path = directory / _check_key(name) + with (directory / (name + ".lock")).open("ab") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise MetadataConflict(name) from None + current = hashlib.sha256(path.read_bytes()).hexdigest() if path.exists() else None + if current != etag: + raise MetadataConflict(name) + with tempfile.NamedTemporaryFile(dir=directory, delete=False) as temp: + temp.write(data) + temp.flush() + os.fsync(temp.fileno()) + temporary = Path(temp.name) + os.replace(temporary, path) + with _directory(directory): + pass + else: + url = self.store._url(name) + ".journal" + condition = {"If-Match": etag} if etag is not None else {"If-None-Match": "*"} + response = await self.client.put( + url, + content=data, + headers=self._headers("PUT", url, hashlib.sha256(data).hexdigest(), condition), + ) + if response.status_code == 412: + raise MetadataConflict(name) + response.raise_for_status() + + async def mutate[T]( + self, + name: str, + parse: Callable[[bytes], T], + serialize: Callable[[T], bytes], + change: Callable[[T], None], + ) -> T: + """Bounded optimistic CAS: concurrent replicas never replace accepted receipts.""" + for _ in range(8): + data, etag = await self.journal(name) + state = parse(data) + change(state) + try: + await self.compare_swap(name, serialize(state), etag) + return state + except MetadataConflict: + await anyio.lowlevel.checkpoint() + continue + raise MetadataConflict("metadata contention exceeds bounded attempts") + + +@contextmanager +def _directory(path: Path) -> Iterator[None]: + fd = os.open(path, os.O_DIRECTORY) + try: + os.fsync(fd) + yield + finally: + os.close(fd) diff --git a/src/hypertrain/data/trial_assignment.py b/src/hypertrain/data/trial_assignment.py new file mode 100644 index 0000000..108142d --- /dev/null +++ b/src/hypertrain/data/trial_assignment.py @@ -0,0 +1,38 @@ +"""Separate trial domain; no production exposure cursor is read or advanced.""" + +from __future__ import annotations + +import struct + +from hypertrain.beacon.core import BeaconRound +from hypertrain.data.assignment import assign_round +from hypertrain.protocol.hashing import sha256_hex +from hypertrain.protocol.messages_v2 import RunManifestV2 + + +def trial_samples( + manifest: RunManifestV2, admission_id: str, nonce: str, beacon: BeaconRound +) -> tuple[int, ...]: + domain = sha256_hex( + f"ht-trial-v2|{manifest.run_id()}|{admission_id}|{nonce}|{beacon.round}".encode() + ) + ds = manifest.training.dataset + return assign_round( + bytes.fromhex(domain), + beacon.round, + bytes.fromhex(beacon.signature), + n_samples=ds.n_samples, + n_slots=1, + batch=manifest.training.batch_samples(), + base_w=0, + unit=ds.assign_unit or 1, + ).slices[0] + + +def trial_assignment_hash(manifest: RunManifestV2, epoch: int, samples: tuple[int, ...]) -> str: + return sha256_hex( + b"ht-assignment-v1" + + bytes.fromhex(manifest.run_id()) + + struct.pack(">QQ", epoch, 0) + + b"".join(struct.pack(" Record | None: return self.j.last("admitted") def deadline(self) -> float: + if self.cfg.get("cleanup_contract") == "network-v2": + intents = self.j.all("create_intent") + if intents: + first = min(float(r["unix"]) for r in intents) + return first + 3600 rec = self.admitted() assert rec is not None return float(rec["deadline_unix"]) @@ -76,8 +83,44 @@ def cleanup_started(self) -> bool: return self.j.last("supervisor_cleanup_started") is not None def sleep(self, seconds: float) -> None: + cutoff = self.cfg.get("cleanup_deadline_unix") + if cutoff is not None: + seconds = min(seconds, max(0.0, cutoff - time.time())) + if seconds <= 0: + return self.p.sleep(seconds) + def cleanup_call( + self, method: str, path: str, tag: str, body: Any = None, timeout: float = 30 + ) -> Response: + """Budget provider pacing plus request against the same immutable cutoff.""" + cutoff = self.cfg.get("cleanup_deadline_unix") + if cutoff is None: + return self.p.call(method, path, tag, body=body, timeout=timeout) + endpoint = self.p.endpoint(method, path) + interval = self.p.pace.get(endpoint) + gap = max( + 0.0, + self.p.pace.get(endpoint, self.p.pace.get("*", 0.0)) + - (self.p.monotonic() - self.p.last.get(endpoint, -1e18)), + ) + if cutoff - time.time() <= gap: + return Response(None, None, None, "", "cleanup_deadline_exhausted") + if gap: + self.j.append("pace_wait", endpoint=endpoint, seconds=gap) + self.sleep(gap) + timeout = min(timeout, cutoff - time.time()) + if timeout <= 0: + return Response(None, None, None, "", "cleanup_deadline_exhausted") + self.p.pace[endpoint] = 0.0 + try: + return self.p.call(method, path, tag, body=body, timeout=timeout) + finally: + if interval is None: + del self.p.pace[endpoint] + else: + self.p.pace[endpoint] = interval + def offer(self, role: str, tag: str) -> dict[str, Any]: h = self.host[role] q = { @@ -114,7 +157,7 @@ def inventory(self, tag: str) -> list[dict[str, Any]] | None: q = urllib.parse.urlencode( {"limit": 25, "select_cols": json.dumps(["id", "label", "machine_id"])} ) - return list_rows(self.p.call("GET", "/api/v1/instances/?" + q, tag)) + return list_rows(self.cleanup_call("GET", "/api/v1/instances/?" + q, tag)) def admit(self) -> None: cfg = self.cfg @@ -207,8 +250,10 @@ def recover(self, intent: Record, attempts: int = 6) -> tuple[str, int | None]: "select_filters": json.dumps({"label": {"eq": intent["label"]}}), } ) + if self.cfg.get("cleanup_deadline_unix") is not None: + attempts = 1 for _ in range(attempts): - r = self.p.call("GET", "/api/v1/instances/?" + q, "recover-" + intent["role"]) + r = self.cleanup_call("GET", "/api/v1/instances/?" + q, "recover-" + intent["role"]) rows = list_rows(r) if rows is not None: hits = [ @@ -220,7 +265,8 @@ def recover(self, intent: Record, attempts: int = 6) -> tuple[str, int | None]: raise Reject("ownership_ambiguous:" + intent["role"]) return ("found", hits[0]) if hits else ("absent", None) self.j.append("recover_unknown", role=intent["role"], http_status=r.status) - self.sleep(r.retry_after if r.retry_after is not None else self.poll) + if self.cfg.get("cleanup_deadline_unix") is None: + self.sleep(r.retry_after if r.retry_after is not None else self.poll) return ("unknown", None) def create(self, role: str) -> Record: @@ -292,7 +338,10 @@ def create(self, role: str) -> Record: raise Reject(f"create_failed_no_retry:{role}:HTTP{r.status}:{state}") def show(self, iid: int, tag: str) -> tuple[Response, Any]: - r = self.p.call("GET", f"/api/v0/instances/{iid}/?owner=me", tag) + timeout = 30.0 + if self.cfg.get("cleanup_deadline_unix") is not None: + timeout = min(timeout, self.cfg["cleanup_deadline_unix"] - time.time()) + r = self.cleanup_call("GET", f"/api/v0/instances/{iid}/?owner=me", tag, timeout=timeout) return r, (r.parsed.get("instances", ...) if r.ok() else ...) def boot(self, role: str) -> Record: @@ -511,8 +560,18 @@ def delete(self, role: str) -> bool: rec = self.receipt(role) assert rec is not None for attempt in range(int(self.cfg.get("delete_attempts", 6))): - r = self.p.call( - "DELETE", f"/api/v0/instances/{rec['instance_id']}/", "delete-" + role, body={} + timeout = 30.0 + if self.cfg.get("cleanup_deadline_unix") is not None: + left = self.cfg["cleanup_deadline_unix"] - time.time() + if left <= 0: + return False + timeout = min(timeout, left) + r = self.cleanup_call( + "DELETE", + f"/api/v0/instances/{rec['instance_id']}/", + "delete-" + role, + body={}, + timeout=timeout, ) ok = r.ok() and r.parsed.get("success") is True self.j.append( @@ -525,12 +584,24 @@ def delete(self, role: str) -> bool: ) if ok: return True + if self.cfg.get("cleanup_deadline_unix") is not None: + return False self.sleep(r.retry_after if r.retry_after is not None else self.poll) return False def absent_once(self, iid: int, tag: str) -> str: q = urllib.parse.urlencode({"limit": 25, "select_cols": json.dumps(["id", "label"])}) - rows = list_rows(self.p.call("GET", "/api/v1/instances/?" + q, "absence-list-" + tag)) + timeout = 30.0 + if self.cfg.get("cleanup_deadline_unix") is not None: + left = self.cfg["cleanup_deadline_unix"] - time.time() + if left <= 0: + return "unknown" + timeout = min(timeout, left) + rows = list_rows( + self.cleanup_call( + "GET", "/api/v1/instances/?" + q, "absence-list-" + tag, timeout=timeout + ) + ) if rows is None: return "unknown" listed = any(i["id"] == iid for i in rows) @@ -550,6 +621,8 @@ def confirm_absence(self, role: str) -> bool: if state == "absent": self.j.append("absence_confirmed", role=role, instance_id=rec["instance_id"]) return True + if self.cfg.get("cleanup_deadline_unix") is not None: + return False self.sleep(self.poll) return False @@ -578,35 +651,98 @@ def owned(self) -> tuple[list[str], list[str]]: def cleanup(self, force: bool) -> dict[str, Any]: with flock(self.dir / "cleanup.lock"): - if self.actor == "supervisor": + if self.actor == "supervisor" or self.cfg.get("cleanup_contract") == "network-v2": with flock(self.dir / "create.lock"): - self.j.append("supervisor_cleanup_started") + if not self.cleanup_started(): + self.j.append("supervisor_cleanup_started") + if self.cfg.get("cleanup_contract") == "network-v2": + start = self.j.last("supervisor_cleanup_started") + assert start is not None + self.cfg["cleanup_deadline_unix"] = min( + self.deadline(), + start["unix"] + 420, + self.cfg.get("cleanup_deadline_unix", float("inf")), + ) owned, unresolved = self.owned() live = [r for r in owned if not self.j.last("absence_confirmed", role=r)] - rescue = {r: self.rescue(r) for r in live} + network = self.cfg.get("cleanup_contract") == "network-v2" + if network: + module_path = ( + Path(__file__).resolve().parents[3] + / "experiments/gpu_network_v2/orchestrate.py" + ) + spec = importlib.util.spec_from_file_location( + "network_cleanup_runtime", module_path + ) + if spec is None or spec.loader is None: + raise Reject("network_cleanup_runtime_missing") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + profile = json.loads(Path(self.cfg["network_profile_file"]).read_bytes()) + runtime = module.NetworkRuntime(self, profile) + + def custody(role: str) -> str: + try: + runtime.rescue(role, fresh=True) + self.j.append( + "rescued", role=role, status="verified", contract="network-v2" + ) + return "verified" + except Exception as e: + self.j.append("network_rescue_failed", role=role, error=repr(e)) + return "failed" + + with ThreadPoolExecutor(max_workers=2) as pool: + statuses = list(pool.map(custody, live)) + rescue = dict(zip(live, statuses, strict=True)) + else: + rescue = {r: self.rescue(r) for r in live} ok_rescue = all( s == "verified" or s.startswith("not_applicable") for s in rescue.values() ) + if network: + ok_rescue = ok_rescue and all( + self.j.last("network_rescued", role=r) is not None for r in owned + ) + for role in owned: + if role not in live and self.j.last("network_rescued", role=role) is not None: + try: + runtime.rescue(role) + except Exception as e: + self.j.append("network_rescue_failed", role=role, error=repr(e)) + ok_rescue = False if not ok_rescue and not force: self.j.append("delete_withheld_rescue_pending", rescue=rescue) return {"all_absent": False, "rescue": rescue, "unresolved": unresolved} if not ok_rescue: self.j.append("rescue_incomplete_forced_delete", rescue=rescue) for r in live: + if network and self.cfg.get("cleanup_deadline_unix") is not None: + self.cfg["delete_attempts"] = 1 if not any(a.get("success") for a in self.j.all("delete_ack", role=r)): self.delete(r) - absent = all(self.confirm_absence(r) for r in owned) + absent_results = [self.confirm_absence(r) for r in owned] + absent = all(absent_results) ok = absent and not unresolved + custody_ok = ok and ok_rescue + if network and not custody_ok: + self.j.append("network_cleanup_censored", all_absent=ok, rescue=rescue) if not ok: self.j.append("liability_open", owned=owned, unresolved=unresolved) - return {"all_absent": ok, "rescue": rescue, "unresolved": unresolved} + return { + "all_absent": ok, + "rescue": rescue, + "unresolved": unresolved, + "custody_complete": custody_ok, + "censored": network and not custody_ok, + } def close(self, code: int, outcome: str) -> Record: with flock(self.dir / "close.lock"): done = self.j.last("transaction_closed") if done: return done - acct = self.p.call("GET", "/api/v0/users/current/", "final-account") + acct = self.cleanup_call("GET", "/api/v0/users/current/", "final-account") credit = acct.parsed.get("credit") if acct.ok() else None rows = self.inventory("final-inventory") ours = {r["instance_id"] for r in self.j.all("receipt")} @@ -632,6 +768,20 @@ def close(self, code: int, outcome: str) -> Record: ) def evidence(self, outcome: str, credit: Any, inv: dict[str, Any]) -> dict[str, Any]: + if self.cfg.get("cleanup_contract") == "network-v2": + return { + "schema_version": 1, + "contract": "network-v2", + "outcome": outcome, + "verdict": "CENSORED", + "reason": "lifecycle_cleanup_not_cuda_d2_verdict", + "inventory": inv, + "credit": credit, + "network_rescued": self.j.all("network_rescued"), + "network_rescue_failed": self.j.all("network_rescue_failed"), + "execution_intents": self.j.all("network_execution_intent"), + "deadline_unix": self.deadline(), + } snap = json.loads(Path(self.cfg["budget_snapshot"]).read_text()) runs = [] for role, name, _neg in self.jobs(): diff --git a/src/hypertrain/gpu_ops/network_qualification.py b/src/hypertrain/gpu_ops/network_qualification.py new file mode 100644 index 0000000..7377afe --- /dev/null +++ b/src/hypertrain/gpu_ops/network_qualification.py @@ -0,0 +1,396 @@ +"""Existing-host CUDA carry qualification; no provider actions or workload promotion. + +Run with the existing project environment: python scripts/network_gpu_qualification.py +sources TREE | run CONFIG.json | compare HOST_A/result.json HOST_B/result.json. +""" + +from __future__ import annotations + +import csv +import hashlib +import io +import json +import os +import re +import shutil +import signal +import subprocess +import sys +import tarfile +import time +from pathlib import Path +from typing import Annotated, Literal + +from pydantic import BaseModel, ConfigDict, Field, StrictInt + +from hypertrain.gpu_ops.journal import durable_write, fsha +from hypertrain.gpu_ops.launcher import Reject +from hypertrain.miner.island_launch import confined, launch_argv, validate_artifacts +from hypertrain.protocol.messages import Hex64, ImageDigest +from hypertrain.protocol.messages_v2 import IslandJobV1 + + +class Boundary(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + +class Qualification(Boundary): + tree: Path + profile: Path + profile_sha256: Hex64 + sources: dict[str, Hex64] + registry_manifest: Path + image_digest: ImageDigest + driver_allowlist: Annotated[list[str], Field(min_length=1)] + seed_job: Path + hotkey: str + output: Path + admitted_unix: StrictInt + instance_id: StrictInt + machine_id: StrictInt + role: str + lifecycle_contract_sha256: Hex64 + known_hosts_sha256: Hex64 + lifecycle_receipts: Path + lifecycle_receipts_sha256: Hex64 + + +class Environment(Boundary): + backend: Literal["cuda"] + torch: Literal["2.14.0+cu130"] + cuda: Literal["13.0"] + python: Literal["/opt/hypertrain/venv/bin/python"] + torch_path: str + drivers: Annotated[list[str], Field(min_length=2, max_length=2)] + sm_counts: Annotated[list[Literal[170]], Field(min_length=2, max_length=2)] + image_digest: ImageDigest + + +class Result(Boundary): + status: Literal["CUDA_ROUNDS_CAPTURED_NOT_WORKLOAD_PASS"] + config_sha256: Hex64 + profile_sha256: Hex64 + sources: dict[str, Hex64] + environment: Environment + instance_id: StrictInt + machine_id: StrictInt + role: str + rounds: Annotated[list[dict[str, Hex64]], Field(min_length=2, max_length=2)] + elapsed_seconds: float + workload_allowed: Literal[False] + + +def select_inventory(text: str, allocated: int, drivers: list[str]) -> tuple[str, str]: + """Validate physical CSV; select two full UUIDs, not CUDA qualification.""" + rows = list(csv.reader(io.StringIO(text), skipinitialspace=True)) + if type(allocated) is not int or allocated < 2 or len(rows) != allocated: + raise ValueError("actual allocated GPU count mismatch") + uuids = [] + for row in rows: + if ( + len(row) != 3 + or not re.fullmatch(r"GPU-[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", row[0]) + or row[1].strip() not in ("NVIDIA GeForce RTX 5090", "GeForce RTX 5090") + or row[2].strip() not in drivers + ): + raise ValueError("allocated GPU UUID/name/driver mismatch") + uuids.append("GPU-" + row[0][4:].lower()) + if len(set(uuids)) != allocated: + raise ValueError("duplicate allocated GPU UUID") + selected = sorted(uuids) + return selected[0], selected[1] + + +def sources(tree: Path) -> dict[str, str]: + paths = sorted((tree / "src/hypertrain").rglob("*.py")) + [ + tree / p + for p in ( + "experiments/gpu_network_v2/run.py", + "experiments/gpu_network_v2/orchestrate.py", + "experiments/gpu_network_v2/profile.json", + "scripts/network_gpu_qualification.py", + "docker/Dockerfile.gpu", + "pyproject.toml", + "uv.lock", + ) + ] + if any(p.is_symlink() for p in paths): + raise Reject("qualification_source_symlink") + return {str(p.relative_to(tree)): fsha(p) for p in paths} + + +def bundle(tree: Path) -> bytes: + """Deterministic source-only archive; jobs/receipts stay separately hash-bound.""" + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w") as archive: + for rel in sorted(sources(tree)): + data = (tree / rel).read_bytes() + info = tarfile.TarInfo(rel) + info.size, info.mode, info.mtime = len(data), 0o600, 0 + archive.addfile(info, io.BytesIO(data)) + return buffer.getvalue() + + +def check_inputs(cfg: Qualification) -> IslandJobV1: + if sources(cfg.tree) != cfg.sources or fsha(cfg.profile) != cfg.profile_sha256: + raise Reject("qualification_source_or_profile_changed") + if "sha256:" + fsha(cfg.registry_manifest) != cfg.image_digest: + raise Reject("qualification_registry_digest_mismatch") + registry = json.loads(cfg.registry_manifest.read_bytes()) + descriptors = registry.get("layers", registry.get("manifests", [])) + if ( + registry.get("schemaVersion") != 2 + or not descriptors + or sum(d["size"] for d in descriptors) > 20_000_000_000 + ): + raise Reject("qualification_registry_descriptors_missing") + if fsha(cfg.lifecycle_receipts) != cfg.lifecycle_receipts_sha256: + raise Reject("qualification_lifecycle_receipts_changed") + receipts = json.loads(cfg.lifecycle_receipts.read_bytes()) + if ( + receipts["instance_id"] != cfg.instance_id + or receipts["machine_id"] != cfg.machine_id + or receipts["role"] != cfg.role + or receipts["admitted_unix"] != cfg.admitted_unix + or receipts["image_digest"] != cfg.image_digest + or receipts["known_hosts_sha256"] != cfg.known_hosts_sha256 + or receipts["lifecycle_contract_sha256"] != cfg.lifecycle_contract_sha256 + or receipts["supervisor_pid"] <= 0 + or receipts["deadline_unix"] > cfg.admitted_unix + 3600 + or receipts["cuda_observed"] is not True + ): + raise Reject("qualification_owned_lifecycle_receipt_missing") + p = json.loads(cfg.profile.read_bytes()) + job = IslandJobV1.model_validate_json(cfg.seed_job.read_bytes()) + training = job.manifest.training.model_dump(mode="json") + exact = all( + all(training[section].get(k) == v for k, v in p[section].items() if k != "lr_schedule") + for section in ("model", "inner", "outer") + ) + exact = exact and all( + training["inner"]["lr_schedule"].get(k) == v for k, v in p["inner"]["lr_schedule"].items() + ) + if ( + p["profile_version"] != 2 + or not exact + or training["reference_spec"]["layout"] != p["layout"] + or training["reference_spec"]["image_digest"] != cfg.image_digest + or training["reference_spec"]["driver_allowlist"] != cfg.driver_allowlist + or job.global_step0 != 0 + or job.manifest.training.dataset.n_samples < 4096 + or cfg.instance_id <= 0 + or cfg.machine_id <= 0 + ): + raise Reject("qualification_exact_profile_or_receipt_mismatch") + return job + + +def inspect_environment(cfg: Qualification) -> Environment: + __import__("hypertrain.trainer") # Configure determinism before torch import. + import torch + + if not torch.cuda.is_available() or torch.cuda.device_count() != 2: + raise Reject("qualification_requires_actual_two_cuda_devices") + if not Path(torch.__file__).resolve().is_relative_to("/venv/main"): + raise Reject("qualification_torch_import_origin_mismatch") + smi = shutil.which("nvidia-smi") + if smi is None: + raise Reject("qualification_driver_inspector_missing") + drivers = subprocess.run( # noqa: S603 - fixed GPU query, resolved executable + [smi, "--query-gpu=driver_version", "--format=csv,noheader"], + check=True, + capture_output=True, + text=True, + timeout=10, + ).stdout.splitlines() + if len(drivers) != 2 or any(d.strip() not in cfg.driver_allowlist for d in drivers): + raise Reject("qualification_driver_mismatch") + return Environment.model_validate( + { + "backend": "cuda", + "torch": str(torch.__version__), + "cuda": torch.version.cuda, + "python": sys.executable, + "torch_path": str(Path(torch.__file__).resolve()), + "drivers": [d.strip() for d in drivers], + "sm_counts": [ + torch.cuda.get_device_properties(i).multi_processor_count for i in range(2) + ], + "image_digest": cfg.image_digest, + } + ) + + +def capture(directory: Path, job: IslandJobV1) -> dict[str, str]: + validate_artifacts(job, directory) + fingerprints = {} + for rank in range(2): + summary = json.loads((directory / f"rank-{rank}/summary.json").read_bytes()) + if summary["backend"] != "cuda": + raise Reject("qualification_cpu_artifacts_refused") + for rel in [ + "state.safetensors", + "ef.safetensors", + "delta.bin", + "leaves.json", + "trace.json", + ] + [f"checkpoints/{t}.safetensors" for t in range(0, 31, 5)]: + fingerprints[f"rank-{rank}/{rel}"] = fsha(directory / f"rank-{rank}/{rel}") + if not (directory / f"rank-{rank}/trace.json").is_file(): + raise Reject("qualification_trace_missing") + return fingerprints + + +def run(config_path: Path) -> Result: + cfg = Qualification.model_validate_json(config_path.read_bytes()) + job = check_inputs(cfg) + for key, value in { + "CUBLAS_WORKSPACE_CONFIG": ":4096:8", + "CUDA_DISABLE_PTX_JIT": "1", + "OMP_NUM_THREADS": "1", + "MKL_NUM_THREADS": "1", + "PYTHONHASHSEED": "0", + "NVIDIA_TF32_OVERRIDE": "0", + "HT_ISLAND_TRACE": "1", + }.items(): + os.environ[key] = value + if os.environ.get("HT_IMAGE_DIGEST") != cfg.image_digest: + raise Reject("qualification_actual_image_receipt_missing") + environment = inspect_environment(cfg) + from hypertrain.auditor.replay import AnchorCache, pack_state, unpack_state + + theta, state = unpack_state( + confined(cfg.seed_job.parent, job.object_paths["start_state"]).read_bytes() + ) + anchor = AnchorCache().genesis(job.manifest, cfg.hotkey, theta) + if state is None or pack_state(theta, state) != pack_state(anchor.theta, anchor.state): + raise Reject("qualification_requires_authenticated_genesis_carry") + if confined(cfg.seed_job.parent, job.object_paths["ef_in"]).read_bytes() != pack_state( + anchor.ef + ): + raise Reject("qualification_requires_genesis_ef") + if time.time() + 600 > cfg.admitted_unix + 900: + raise Reject("qualification_staging_deadline_exceeded") + cfg.output.mkdir(mode=0o700, parents=True, exist_ok=False) + durable_write(cfg.output / "config.json", config_path.read_bytes()) + started = time.monotonic() + rounds = [] + try: + previous = None + for index in range(2): + directory = cfg.output / f"round-{index}/published" + directory.mkdir(mode=0o700, parents=True) + for relative in job.object_paths.values(): + target = directory / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(confined(cfg.seed_job.parent, relative), target) + body = job.model_dump(mode="json") + if previous is not None: + for key, rel in [("start_state", "state.safetensors"), ("ef_in", "ef.safetensors")]: + shutil.copyfile(previous / "rank-0" / rel, directory / job.object_paths[key]) + body[key + "_sha256"] = fsha(directory / job.object_paths[key]) + body.update(w=job.w + 1, global_step0=30) + left = min(90, cfg.admitted_unix + 480 - time.time()) + if left <= 0: + raise Reject("qualification_probe_deadline_exceeded") + body["deadline"] = min(job.deadline, int(time.time() + left)) + current = IslandJobV1.model_validate(body) + durable_write(directory / "job.json", current.model_dump_json().encode()) + with ( + (directory / "driver.stdout").open("xb") as out, + (directory / "driver.stderr").open("xb") as err, + ): + with subprocess.Popen( # noqa: S603 - shared fixed torchrun argv, typed job + launch_argv(current, directory, "cuda"), + cwd=cfg.tree, + stdout=out, + stderr=err, + start_new_session=True, + env={**os.environ, "PYTHONPATH": str(cfg.tree / "src")}, + ) as proc: + try: + code = proc.wait(timeout=left) + finally: + if proc.poll() is None: + os.killpg(proc.pid, signal.SIGKILL) + if code: + raise Reject("qualification_cuda_runner_failed") + previous = directory + rounds.append(capture(previous, current)) + result = Result( + status="CUDA_ROUNDS_CAPTURED_NOT_WORKLOAD_PASS", + config_sha256=fsha(config_path), + profile_sha256=cfg.profile_sha256, + sources=cfg.sources, + environment=environment, + instance_id=cfg.instance_id, + machine_id=cfg.machine_id, + role=cfg.role, + rounds=rounds, + elapsed_seconds=time.monotonic() - started, + workload_allowed=False, + ) + durable_write(cfg.output / "result.json", result.model_dump_json().encode()) + return result + finally: + # ponytail: custody only; provider teardown remains existing root-owned gpu_ops. + inventory = { + str(p.relative_to(cfg.output)): fsha(p) for p in cfg.output.rglob("*") if p.is_file() + } + durable_write(cfg.output / "custody.json", json.dumps(inventory, sort_keys=True).encode()) + + +def compare(paths: list[Path]) -> dict[str, bool]: + if len(paths) != 2: + raise Reject("qualification_requires_two_host_results") + a, b = [Result.model_validate_json(p.read_bytes()) for p in paths] + if ( + a.machine_id == b.machine_id + or a.instance_id == b.instance_id + or a.role == b.role + or a.profile_sha256 != b.profile_sha256 + or a.sources != b.sources + or a.environment.image_digest != b.environment.image_digest + or a.rounds != b.rounds + ): + raise Reject("qualification_two_host_binding_or_bitwise_mismatch") + for path, result in zip(paths, (a, b), strict=True): + for index, files in enumerate(result.rounds): + expected = { + f"rank-{rank}/{rel}" + for rank in range(2) + for rel in [ + "state.safetensors", + "ef.safetensors", + "delta.bin", + "leaves.json", + "trace.json", + ] + + [f"checkpoints/{t}.safetensors" for t in range(0, 31, 5)] + } + if set(files) != expected: + raise Reject("qualification_full_round_coverage_missing") + for rel, want in files.items(): + if fsha(confined(path.parent / f"round-{index}/published", rel)) != want: + raise Reject("qualification_artifact_custody_changed") + return {"two_host_bitwise_match": True, "workload_allowed": False} + + +def main() -> None: + """Existing source-host qualification CLI; no provider or workload promotion.""" + match sys.argv[1:]: + case ["sources", tree]: + print(json.dumps(sources(Path(tree)), sort_keys=True)) + case ["bundle-hash", tree]: + data = bundle(Path(tree)) + print(json.dumps({"bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()})) + case ["bundle", tree, destination]: + durable_write(Path(destination), bundle(Path(tree))) + case ["run", config]: + print(run(Path(config)).model_dump_json()) + case ["compare", a, b]: + print(json.dumps(compare([Path(a), Path(b)]))) + case _: + raise SystemExit( + "usage: sources TREE | run CONFIG | compare A/result.json B/result.json" + ) diff --git a/src/hypertrain/gpu_ops/supervisor.py b/src/hypertrain/gpu_ops/supervisor.py index a4b2d1b..74ac9a3 100644 --- a/src/hypertrain/gpu_ops/supervisor.py +++ b/src/hypertrain/gpu_ops/supervisor.py @@ -10,6 +10,7 @@ import fcntl import json import os +import select import sys import time import traceback @@ -24,10 +25,41 @@ def supervise( run_dir: Path, cfg: dict[str, object], j: Journal, deadline: float, hard: float ) -> int: tick = float(cfg.get("supervisor_check_seconds", 30)) # type: ignore[arg-type] + if cfg.get("cleanup_contract") == "network-v2": + first = j.all("create_intent") + if first: + hard = min(float(r["unix"]) for r in first) + 3600 + deadline = min(deadline, hard - 420) + parent = j.last("transaction_open") + if cfg.get("cleanup_contract") == "network-v2": + opens = j.all("transaction_open") + parent = opens[0] if opens else None + parent_fd = None + if cfg.get("cleanup_contract") == "network-v2" and parent is not None: + try: + parent_fd = os.pidfd_open(int(parent["pid"])) + except ProcessLookupError: + j.append("supervisor_parent_exit") + deadline = time.time() while time.time() < deadline: if j.last("transaction_closed"): + if parent_fd is not None: + os.close(parent_fd) return 0 - time.sleep(min(tick, max(0.0, deadline - time.time()))) + if cfg.get("cleanup_contract") == "network-v2": + first = j.all("create_intent") + if first: + hard = min(float(r["unix"]) for r in first) + 3600 + deadline = min(deadline, hard - 420) + timeout = min(tick, max(0.0, deadline - time.time())) + if parent_fd is not None: + if select.select([parent_fd], [], [], timeout)[0]: + j.append("supervisor_parent_exit") + break + else: + time.sleep(timeout) + if parent_fd is not None: + os.close(parent_fd) if j.last("transaction_closed"): return 0 j.append("supervisor_deadline_reached", deadline_unix=deadline) @@ -38,13 +70,24 @@ def supervise( ) orch = Orchestrator(dict(cfg), j, p, run_dir, actor="supervisor") while True: + if cfg.get("cleanup_contract") == "network-v2": + starts = j.all("supervisor_cleanup_started") + start = min(float(r["unix"]) for r in starts) if starts else time.time() + orch.cfg["cleanup_deadline_unix"] = min(hard, start + 420) result = orch.cleanup(force=True) if result["all_absent"]: + if cfg.get("cleanup_contract") == "network-v2" and not result.get("custody_complete"): + j.append("supervisor_censored", cleanup=result) + orch.close(EXIT["failed_cleaned"], "CENSORED_RESCUE") + return 4 orch.close(EXIT["failed_cleaned"], "supervisor_deadline_cleanup") return 0 if time.time() >= hard: j.append("supervisor_liability_open", cleanup=result) return 4 + if cfg.get("cleanup_contract") == "network-v2": + j.append("supervisor_liability_open", cleanup=result) + return 4 time.sleep(float(cfg.get("poll_seconds", 15))) # type: ignore[arg-type] @@ -64,6 +107,11 @@ def main(run_dir_arg: str) -> int: return 5 deadline = float(admitted["deadline_unix"]) hard = deadline + float(cfg.get("hard_grace_seconds", 1800)) + if cfg.get("cleanup_contract") == "network-v2": + first = j.all("create_intent") + if first: + hard = min(float(r["unix"]) for r in first) + 3600 + deadline = min(deadline, hard - 420) j.append("supervisor_ready", supervisor_pid=os.getpid(), deadline_unix=deadline) print("READY", flush=True) logf = open(run_dir / "supervisor.log", "a") @@ -77,6 +125,9 @@ def main(run_dir_arg: str) -> int: logf.write(traceback.format_exc()) logf.flush() j.append("supervisor_exception", error=f"{type(e).__name__}:{e}"[:300]) + if cfg.get("cleanup_contract") == "network-v2": + j.append("supervisor_liability_open", cleanup=None) + return 4 if time.time() >= hard: j.append("supervisor_liability_open", cleanup=None) return 4 diff --git a/src/hypertrain/gpu_ops/work_screen.py b/src/hypertrain/gpu_ops/work_screen.py new file mode 100644 index 0000000..8e6c94a --- /dev/null +++ b/src/hypertrain/gpu_ops/work_screen.py @@ -0,0 +1,128 @@ +"""Full-round observed work, never physical-device attestation or a payment multiplier.""" + +from __future__ import annotations + +import json +import threading +import time +from pathlib import Path +from typing import Literal, Protocol + +from hypertrain.data.trial_assignment import trial_assignment_hash +from hypertrain.miner.island_launch import ( + IslandArtifacts, + confined, + launch_island, + validate_artifacts, +) +from hypertrain.protocol.hashing import MerkleTree, sha256_hex +from hypertrain.protocol.messages import LeafPreimage +from hypertrain.protocol.messages_v2 import ( + ArtifactRef, + IslandJobV1, + JoinChallenge, + WorkProof, + WorkScreenV2, +) + + +class WorkScreenError(ValueError): + def __init__(self, code: str) -> None: + self.code = code + super().__init__(code) + + +class IslandLaunch(Protocol): + """Internal synchronous launch seam; never a wire/public callback.""" + + def __call__( + self, + job: IslandJobV1, + directory: Path, + *, + backend: Literal["cpu", "cuda"], + cancel: threading.Event | None = None, + trace: bool = False, + ) -> IslandArtifacts: ... + + +def commitments(artifacts: IslandArtifacts) -> tuple[str, str]: + leaves = [LeafPreimage.model_validate(v) for v in json.loads(artifacts.leaves.read_bytes())] + return ( + MerkleTree([bytes.fromhex(p.digest()) for p in leaves]).root.hex(), + sha256_hex(artifacts.delta.read_bytes()), + ) + + +def screen_work( + job: IslandJobV1, + directory: Path, + challenge: JoinChallenge, + *, + now_beacon: int, + backend: Literal["cpu", "cuda"] = "cuda", + launch: IslandLaunch | None = None, + cancel: threading.Event | None = None, +) -> tuple[WorkScreenV2, WorkProof]: + """Launch the actual island API and consume its validated, all-rank publication. + + CPU backend is local functional evidence only; admission controls qualification. + Caller rechecks verified beacon deadline when accepting the resulting proof. + """ + if cancel is not None and cancel.is_set(): + raise WorkScreenError("CANCELLED") + if time.time() >= job.deadline: + raise WorkScreenError("DEADLINE") + if ( + not challenge.seed_beacon <= now_beacon <= challenge.deadline_beacon + or challenge.manifest_hash != job.run_id + or job.manifest.run_id() != job.run_id + or challenge.layout != job.manifest.training.reference_spec.layout + or challenge.assignment_hash + != trial_assignment_hash(job.manifest, job.w, tuple(job.sample_ids)) + ): + raise WorkScreenError("CHALLENGE_BINDING") + import hypertrain.trainer # noqa: F401 + from hypertrain.auditor.replay import unpack_state + from hypertrain.trainer.compress import state_hash + + theta, _ = unpack_state(confined(directory, job.object_paths["start_state"]).read_bytes()) + if state_hash(theta) != challenge.theta_hash: + raise WorkScreenError("CHALLENGE_START") + if cancel is not None and cancel.is_set(): + raise WorkScreenError("CANCELLED") + if time.time() >= job.deadline: + raise WorkScreenError("DEADLINE") + if launch is None and cancel is None: + artifacts = launch_island(job, directory, backend=backend) + else: + artifacts = (launch or launch_island)(job, directory, backend=backend, cancel=cancel) + artifacts = validate_artifacts(job, artifacts.directory) + if cancel is not None and cancel.is_set(): + raise WorkScreenError("CANCELLED") + refs = [] + for path in (artifacts.state, artifacts.ef, artifacts.delta, artifacts.leaves): + size = path.stat().st_size + if size > challenge.artifact_limits.max_object_bytes: + raise WorkScreenError("ARTIFACT_LIMIT") + refs.append(ArtifactRef(sha256=sha256_hex(path.read_bytes()), size=size)) + root, delta = commitments(artifacts) + screen = WorkScreenV2( + evidence_kind="work-proof-v1", + admission_id=challenge.admission_id, + nonce=challenge.nonce, + policy_hash=challenge.policy_hash, + image_digest=job.manifest.training.reference_spec.image_digest, + layout=challenge.layout, + challenge_hash=challenge.digest(), + rank_results=list(artifacts.ranks), + artifact_hashes=[r.sha256 for r in refs], + ) + proof = WorkProof( + admission_id=challenge.admission_id, + challenge_hash=challenge.digest(), + leaves_root=root, + delta_hash=delta, + artifact_refs=refs, + ) + return screen, proof diff --git a/src/hypertrain/ledger/escrow_v2.py b/src/hypertrain/ledger/escrow_v2.py new file mode 100644 index 0000000..03d7cfd --- /dev/null +++ b/src/hypertrain/ledger/escrow_v2.py @@ -0,0 +1,1081 @@ +"""Conserved origin units. SQLite event journal and receipts commit with bucket changes. + +No legacy entitlement import, external deposit, client mint or client slash exists. +The service supplies authenticated finality/replay and settlement from its own records. +""" + +from __future__ import annotations + +import json +import sqlite3 +import threading +from collections.abc import Callable, Iterator, Sequence +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Literal, assert_never + +from pydantic import BaseModel + +from hypertrain.protocol.envelope import body_digest +from hypertrain.protocol.envelope_v2 import EnvelopeV2, Intake +from hypertrain.protocol.hashing import sha256_hex +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.keys import decode_hotkey, verify +from hypertrain.protocol.messages import SS58, Finalize, ReplayVerdict +from hypertrain.protocol.messages_v2 import ( + U64, + AuditChallengeV2, + CommitV2, + EconomicsPolicyV2, + EscrowLock, + EscrowOperation, + EscrowReceipt, + EscrowRelease, + EscrowTransfer, + Hex64, + OriginAllocation, + RewardFinalize, + RunManifestV2, + ShadowReservationRequestV1, + ShadowReservationV1, + ShadowRewardFinalizeV1, + TestGenesis, + WireModel, +) + + +class EscrowError(ValueError): + def __init__(self, code: str) -> None: + self.code = code + super().__init__(code) + + +def digest(model: BaseModel) -> str: + return body_digest(model.model_dump(mode="json")) + + +def authority_message(model: TestGenesis | RewardFinalize | ShadowRewardFinalizeV1) -> bytes: + """Origin authority signature is independent of the enclosing envelope signer.""" + return ( + b"hypertrain/origin/2|" + + type(model).__name__.encode() + + b"|" + + model.run_id.encode() + + b"|" + + body_digest(model.model_dump(mode="json", exclude={"authority_sig"})).encode() + ) + + +def genesis_allocation_hash(allocations: Sequence[tuple[str, int, int]]) -> str: + """Hash owner/units/maturity first; origin IDs bind this hash without recursion.""" + return sha256_hex(canonicalize([list(a) for a in allocations], allow_float=False)) + + +def genesis_origin_id(run_id: str, allocation_hash: str, index: int) -> str: + return sha256_hex(f"ht-origin-genesis-v2|{run_id}|{allocation_hash}|{index}".encode()) + + +def reward_origin_id(run_id: str, w: int, hotkey: str, finalize_hash: str) -> str: + return sha256_hex(f"ht-origin-reward-v2|{run_id}|{w}|{hotkey}|{finalize_hash}".encode()) + + +def shadow_origin_id(reservation: ShadowReservationV1) -> str: + return sha256_hex( + ( + f"ht-origin-shadow-v1|{reservation.run_id}|{reservation.shadow_ordinal}|" + f"{reservation.trial_epoch}|{reservation.hotkey}|{reservation.finalize_hash}" + ).encode() + ) + + +@dataclass(frozen=True, slots=True) +class RewardWork: + """Accepted signed work, with ownership/assignment fetched from service state.""" + + commit: EnvelopeV2 + verdict: EnvelopeV2 + challenge: EnvelopeV2 + owner: str + sample_ids: tuple[int, ...] + + +@dataclass(frozen=True, slots=True) +class FinalityEvidence: + finalize: EnvelopeV2 + works: tuple[RewardWork, ...] + tape_hash: str + shadow: bool + finalized_beacon: int + vesting_beacon: int + + +@dataclass(frozen=True, slots=True) +class ShadowReservationEvidence: + """Original trial custody validated by trusted intake before auditor signing. + + The store resolves custody_hash from accepted original operation records. + This input contains no acceptance receipt, auditor verdict or replay result. + """ + + finalize: EnvelopeV2 + commit: EnvelopeV2 + owner: str + sample_ids: tuple[int, ...] + custody_hash: str + reference_final_state_hash: str + + +class Settlement(WireModel): + """Internal L0/L5 acceptance record, never a client supplied unlock flag.""" + + finality_hash: Hex64 + closed_dispute_root: Hex64 + release_beacon: U64 + unresolved: bool + outcome: Literal["MATCH", "FRAUD", "INFRASTRUCTURE"] + run_id: Hex64 | None = None + admission_id: Hex64 | None = None + coldkey: SS58 | None = None + dispute_id: Hex64 | None = None + evidence_hash: Hex64 | None = None + + +@dataclass(frozen=True, slots=True) +class Balances: + issued: int + available: int + reward_pending: int + admission_locked: int + dispute_locked: int + paid: int + burned: int + + def conserved(self) -> bool: + return self.issued == sum( + ( + self.available, + self.reward_pending, + self.admission_locked, + self.dispute_locked, + self.paid, + self.burned, + ) + ) + + +class EscrowV2: + """Mutable accounting service; BEGIN IMMEDIATE serializes independent processes. + + Pass the service SQLite connection to share eligibility/rotation transactions. + Independent replay and settlement callbacks are mandatory authority boundaries. + """ + + def __init__( + self, + database: Path | sqlite3.Connection, + manifest: RunManifestV2, + policy: EconomicsPolicyV2, + *, + auditors: frozenset[str], + replay_finality: Callable[[FinalityEvidence], str], + settlement: Callable[[str], Settlement], + owner_of: Callable[[str], str], + assignment_of: Callable[[int, str], tuple[int, ...]], + shadow_assignment_of: Callable[[ShadowReservationV1], tuple[int, ...]] | None = None, + shadow_settlement: Callable[[str], Settlement] | None = None, + ) -> None: + if digest(policy) != manifest.network.economics_policy_hash: + raise EscrowError("ECONOMICS_POLICY_HASH") + if policy.round_reward_units != manifest.training.budget.epochs_per_round * 1_000_000: + raise EscrowError("MANIFEST_REWARD_BUDGET") + self.manifest, self.policy, self.run_id = manifest, policy, manifest.run_id() + if policy.reward_authority != manifest.training.coord_pubkey or auditors != frozenset( + manifest.training.auditors + ): + raise EscrowError("PINNED_AUTHORITIES") + self.auditors, self.replay_finality = auditors, replay_finality + self.settlement, self.owner_of, self.assignment_of = settlement, owner_of, assignment_of + self.shadow_assignment_of, self.shadow_settlement = shadow_assignment_of, shadow_settlement + self.mutex = threading.RLock() + match database: + case Path(): + self.db = sqlite3.connect(database, isolation_level=None, check_same_thread=False) + case sqlite3.Connection(): + self.db = database + self.db.row_factory = sqlite3.Row + self.db.execute("PRAGMA journal_mode=WAL") + self.db.execute("PRAGMA synchronous=FULL") + self.db.executescript(""" + CREATE TABLE IF NOT EXISTS escrow_config(run_id TEXT PRIMARY KEY, policy TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS escrow_origins( + origin TEXT PRIMARY KEY, issued INTEGER NOT NULL, mature_at INTEGER NOT NULL, + reward_round INTEGER, hotkey TEXT); + CREATE TABLE IF NOT EXISTS escrow_units( + origin TEXT, owner TEXT, bucket TEXT, ref TEXT, units INTEGER NOT NULL CHECK(units>=0), + PRIMARY KEY(origin,owner,bucket,ref)); + CREATE TABLE IF NOT EXISTS escrow_events( + seq INTEGER PRIMARY KEY, operation_id TEXT UNIQUE, kind TEXT NOT NULL, + request TEXT NOT NULL, prev_hash TEXT NOT NULL, event_hash TEXT NOT NULL, + receipt TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS escrow_finalized(w INTEGER PRIMARY KEY, finalize_hash TEXT); + CREATE TABLE IF NOT EXISTS escrow_shadow_finalized( + ordinal INTEGER PRIMARY KEY, trial_epoch INTEGER, finalize_hash TEXT, + reservation_hash TEXT UNIQUE, origin TEXT UNIQUE); + CREATE TABLE IF NOT EXISTS admission_reservations( + reservation TEXT PRIMARY KEY, digest TEXT NOT NULL, receipt TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS records_v2( + run_id TEXT, kind TEXT, id TEXT, data TEXT NOT NULL, + PRIMARY KEY(run_id,kind,id)); + CREATE TABLE IF NOT EXISTS escrow_snapshots( + operation_id TEXT PRIMARY KEY, state TEXT NOT NULL); + """) + with self.tx(): + rows = self.db.execute("SELECT * FROM escrow_config").fetchall() + if rows and ( + len(rows) != 1 + or rows[0]["run_id"] != self.run_id + or rows[0]["policy"] != policy.model_dump_json() + ): + raise EscrowError("LEDGER_MODE_OR_RUN_CHANGED") + self.db.execute( + "INSERT OR IGNORE INTO escrow_config VALUES(?,?)", + (self.run_id, policy.model_dump_json()), + ) + self.verify() + + @contextmanager + def tx(self) -> Iterator[sqlite3.Connection]: + with self.mutex: + nested = self.db.in_transaction + self.db.execute("SAVEPOINT escrow" if nested else "BEGIN IMMEDIATE") + try: + yield self.db + finally: + # Commit only on normal exit; exception state is captured by contextlib. + import sys + + failed = sys.exc_info()[0] is not None + if nested: + if failed: + self.db.execute("ROLLBACK TO escrow") + self.db.execute("RELEASE escrow") + else: + self.db.execute("ROLLBACK" if failed else "COMMIT") + + def _existing(self, operation_id: str, request: str) -> EscrowReceipt | None: + row = self.db.execute( + "SELECT request,receipt FROM escrow_events WHERE operation_id=?", (operation_id,) + ).fetchone() + if row is None: + return None + if row["request"] != request: + raise EscrowError("OPERATION_CONFLICT") + return EscrowReceipt.model_validate_json(row["receipt"]) + + def _event(self, operation_id: str, kind: str, request: str, owner: str) -> EscrowReceipt: + self._conservation() + last = self.db.execute( + "SELECT seq,event_hash FROM escrow_events ORDER BY seq DESC LIMIT 1" + ).fetchone() + seq, prev = (last["seq"] + 1, last["event_hash"]) if last else (0, "0" * 64) + state = self._snapshot() + event_hash = sha256_hex( + canonicalize([self.run_id, seq, prev, operation_id, kind, request, state]) + ) + b = self.balances(owner) + receipt = EscrowReceipt( + operation_id=operation_id, + event_hash=event_hash, + ledger_mode=self.policy.ledger_mode, + available_units=b.available, + admission_locked_units=b.admission_locked, + dispute_locked_units=b.dispute_locked, + ) + self.db.execute( + "INSERT INTO escrow_events VALUES(?,?,?,?,?,?,?)", + (seq, operation_id, kind, request, prev, event_hash, receipt.model_dump_json()), + ) + self.db.execute("INSERT INTO escrow_snapshots VALUES(?,?)", (operation_id, state)) + return receipt + + def _authority(self, model: TestGenesis | RewardFinalize | ShadowRewardFinalizeV1) -> None: + if model.run_id != self.run_id or not verify( + decode_hotkey(self.policy.reward_authority), + authority_message(model), + bytes.fromhex(model.authority_sig), + ): + raise EscrowError("ORIGIN_AUTHORITY") + + def _issue(self, origin: OriginAllocation, w: int | None, hotkey: str | None) -> None: + if self.db.execute( + "SELECT 1 FROM escrow_origins WHERE origin=?", (origin.origin_id,) + ).fetchone(): + raise EscrowError("DUPLICATE_ORIGIN") + self.db.execute( + "INSERT INTO escrow_origins VALUES(?,?,?,?,?)", + (origin.origin_id, origin.units, origin.mature_at, w, hotkey), + ) + self._credit(origin.origin_id, origin.owner, "reward_pending", "", origin.units) + + def test_genesis(self, genesis: TestGenesis) -> EscrowReceipt: + self._authority(genesis) + with self.tx(): + request = genesis.model_dump_json(exclude={"authority_sig"}) + operation = sha256_hex(f"ht-genesis-v2|{self.run_id}".encode()) + old = self._existing(operation, request) + if old is not None: + return old + if ( + self.policy.ledger_mode != "test" + or self.db.execute("SELECT 1 FROM escrow_events LIMIT 1").fetchone() + ): + raise EscrowError("TEST_GENESIS_FORBIDDEN") + alloc = [(o.owner, o.units, o.mature_at) for o in genesis.origins] + if ( + genesis.allocation_hash != self.policy.genesis_allocation_hash + or (genesis_allocation_hash(alloc) != genesis.allocation_hash) + or genesis.total_units > self.policy.max_total_issuance + ): + raise EscrowError("GENESIS_ALLOCATION") + for i, o in enumerate(genesis.origins): + if o.origin_id != genesis_origin_id(self.run_id, genesis.allocation_hash, i): + raise EscrowError("GENESIS_ORIGIN_DOMAIN") + self._issue(o, None, None) + return self._event(operation, "TEST_GENESIS", request, genesis.origins[0].owner) + + def reward_allocations( + self, reward: RewardFinalize | ShadowRewardFinalizeV1, evidence: FinalityEvidence + ) -> tuple[OriginAllocation, ...]: + """Revalidate actual signed finality/work. Never mint from a canary PASS flag.""" + self._authority(reward) + if ( + evidence.finalize.type != "Finalize" + or evidence.finalize.signer != self.policy.reward_authority + ): + raise EscrowError("FINALITY_AUTHORITY") + final = Intake( + self.run_id, {"Finalize": lambda s: s == self.policy.reward_authority} + ).accept(evidence.finalize.model_dump(mode="json"), evidence.finalized_beacon) + assert isinstance(final, Finalize) + if (reward.finalize_hash, reward.w, reward.tape_hash) != ( + body_digest(evidence.finalize.body), + final.w, + evidence.tape_hash, + ) or reward.mature_at < max(evidence.vesting_beacon, evidence.finalized_beacon): + raise EscrowError("FINALITY_BINDING_OR_VESTING") + reservation = None + match reward: + case ShadowRewardFinalizeV1(): + if not evidence.shadow: + raise EscrowError("REWARD_DOMAIN") + reservation = self._shadow_reservation(reward) + if ( + evidence.finalized_beacon != reservation.finalized_beacon + or reward.w != reservation.trial_epoch + or reward.mature_at != reservation.mature_at + or final.included != [reservation.hotkey] + ): + raise EscrowError("SHADOW_FINALITY_BINDING") + if self.shadow_assignment_of is None or self.shadow_settlement is None: + raise EscrowError("SHADOW_AUTHORITY_UNAVAILABLE") + case RewardFinalize(): + if evidence.shadow: + raise EscrowError("REWARD_DOMAIN") + case unreachable: + assert_never(unreachable) + if self.replay_finality(evidence) != final.final_theta_hash_w1: + raise EscrowError("FINALITY_REPLAY") + work: dict[str, tuple[str, int]] = {} + verdicts = [] + for item in evidence.works: + + def commit_authority(signer: str, expected: str = item.commit.signer) -> bool: + return signer == expected + + c = Intake(self.run_id, {"CommitV2": commit_authority}).accept( + item.commit.model_dump(mode="json"), evidence.finalized_beacon + ) + v = Intake(self.run_id, {"ReplayVerdict": lambda s: s in self.auditors}).accept( + item.verdict.model_dump(mode="json"), evidence.finalized_beacon + ) + challenge = Intake( + self.run_id, {"AuditChallengeV2": lambda s: s == self.policy.reward_authority} + ).accept(item.challenge.model_dump(mode="json"), evidence.finalized_beacon) + if not isinstance(c, CommitV2) or not isinstance(v, ReplayVerdict): + raise EscrowError("REWARD_WORK_TYPE") + if ( + not isinstance(challenge, AuditChallengeV2) + or (challenge.w, challenge.target) != (c.w, c.hotkey) + or v.challenge_hash != digest(challenge) + ): + raise EscrowError("REWARD_AUDIT_BINDING") + c.validate_assignment(self.manifest, len(item.sample_ids)) + if ( + c.w != final.w + or c.hotkey in work + or c.hotkey not in final.included + or item.sample_ids + != ( + self.shadow_assignment_of(reservation) + if reservation is not None and self.shadow_assignment_of is not None + else self.assignment_of(final.w, c.hotkey) + ) + or item.owner != self.owner_of(c.hotkey) + or item.verdict.signer == c.hotkey + or v.result != "MATCH" + or v.first_bad_leaf is not None + or v.recomputed_leaves_root != c.leaves_root + or v.replay_env.image_digest != self.manifest.training.reference_spec.image_digest + or v.replay_env.driver not in self.manifest.training.reference_spec.driver_allowlist + ): + raise EscrowError("UNVERIFIED_REWARD_WORK") + work[c.hotkey] = item.owner, c.tokens + if reservation is not None and ( + digest(c) != reservation.commit_hash or item.owner != reservation.coldkey + ): + raise EscrowError("SHADOW_WORK_BINDING") + verdicts.append(digest(v)) + if ( + not work + or set(work) != set(final.included) + or len(set(final.included)) != len(final.included) + ): + raise EscrowError("NO_VERIFIED_WORK_OR_EXCLUSIONS") + if sha256_hex(canonicalize(verdicts)) != reward.verdict_root: + raise EscrowError("VERDICT_ROOT") + if reward.budget_units != self.policy.round_reward_units: + raise EscrowError("ROUND_BUDGET") + ordered = sorted(work, key=lambda h: h.encode()) + total = sum(work[h][1] for h in ordered) + units = {h: reward.budget_units * work[h][1] // total for h in ordered} + remainder = reward.budget_units - sum(units.values()) + for h in ordered[:remainder]: + units[h] += 1 + origins = tuple( + OriginAllocation( + origin_id=( + shadow_origin_id(reservation) + if reservation is not None + else reward_origin_id(self.run_id, reward.w, h, reward.finalize_hash) + ), + owner=work[h][0], + units=units[h], + mature_at=reward.mature_at, + ) + for h in ordered + if units[h] + ) + if reward.origin_ids != [o.origin_id for o in origins] or reward.allocation_hash != ( + sha256_hex(canonicalize([o.model_dump(mode="json") for o in origins])) + ): + raise EscrowError("REWARD_ALLOCATION") + return origins + + def reserve_shadow( + self, request: ShadowReservationRequestV1, evidence: ShadowReservationEvidence + ) -> ShadowReservationV1: + """Trusted intake reserves one immutable ordinal before external reward signing.""" + key = f"shadow-reserve|{self.run_id}|{request.admission_id}|{request.trial_epoch}" + if self.db.in_transaction: + raise EscrowError("SHADOW_RESERVATION_REQUIRES_COMMIT") + with self.tx(): + if request.run_id != self.run_id: + raise EscrowError("SHADOW_RESERVATION_DOMAIN") + final = Intake(self.run_id, {"Finalize": self.policy.reward_authority.__eq__}).accept( + evidence.finalize.model_dump(mode="json"), request.finalized_beacon + ) + if not isinstance(final, Finalize) or ( + evidence.finalize.type != "Finalize" + or final.w != request.trial_epoch + or digest(final) != request.finalize_hash + or final.included != [request.hotkey] + or request.mature_at + != (request.finalized_beacon + self.manifest.training.verify.E_vest_rounds) + or evidence.custody_hash != request.custody_hash + ): + raise EscrowError("SHADOW_RESERVATION_BINDING") + commit = Intake(self.run_id, {"CommitV2": request.hotkey.__eq__}).accept( + evidence.commit.model_dump(mode="json"), request.finalized_beacon + ) + if not isinstance(commit, CommitV2) or ( + evidence.commit.type != "CommitV2" + or commit.w != final.w + or commit.hotkey != request.hotkey + or digest(commit) != request.commit_hash + or evidence.owner != request.coldkey + ): + raise EscrowError("SHADOW_RESERVATION_WORK") + commit.validate_assignment(self.manifest, len(evidence.sample_ids)) + semantic = digest(request) + old = self.db.execute( + "SELECT digest,receipt FROM admission_reservations WHERE reservation=?", (key,) + ).fetchone() + if old is not None: + if old["digest"] != semantic: + raise EscrowError("SHADOW_RESERVATION_CONFLICT") + return ShadowReservationV1.model_validate_json(old["receipt"]) + if ( + self.owner_of(commit.hotkey) != request.coldkey + or evidence.reference_final_state_hash != final.final_theta_hash_w1 + or commit.final_theta_hash != final.final_theta_hash_w1 + ): + raise EscrowError("SHADOW_RESERVATION_WORK") + pending = self.db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='shadow-reservation' " + "AND json_extract(data,'$.accepted') IS NULL", + (self.run_id,), + ).fetchone() + if pending is not None: + raise EscrowError("SHADOW_RESERVATION_PENDING") + last = self.db.execute("SELECT MAX(ordinal) FROM escrow_shadow_finalized").fetchone()[0] + ordinal = last + 1 if last is not None else 0 + if ordinal >= self.policy.shadow_bootstrap_rounds: + raise EscrowError("BOOTSTRAP_FINISHED") + existing = self.db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='shadow-reservation' AND id=?", + (self.run_id, str(ordinal)), + ).fetchone() + if existing is not None: + raise EscrowError("SHADOW_RESERVATION_CONFLICT") + if ( + self.balances().issued + self.policy.round_reward_units + > self.policy.max_total_issuance + ): + raise EscrowError("TOTAL_ISSUANCE_BUDGET") + binding = ShadowReservationV1.model_validate( + {**request.body(), "shadow_ordinal": ordinal} + ) + self.db.execute( + "INSERT INTO admission_reservations VALUES(?,?,?)", + (key, semantic, binding.model_dump_json()), + ) + self.db.execute( + "INSERT INTO records_v2 VALUES(?,?,?,?)", + ( + self.run_id, + "shadow-reservation", + str(ordinal), + canonicalize({"binding": binding.body(), "accepted": None}).decode(), + ), + ) + return binding + + def _shadow_reservation(self, reward: ShadowRewardFinalizeV1) -> ShadowReservationV1: + row = self.db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='shadow-reservation' AND id=?", + (self.run_id, str(reward.shadow_ordinal)), + ).fetchone() + if row is None: + raise EscrowError("SHADOW_RESERVATION_REQUIRED") + binding = ShadowReservationV1.model_validate(json.loads(row["data"])["binding"]) + if (digest(binding), binding.trial_epoch, binding.finalize_hash, binding.mature_at) != ( + reward.reservation_hash, + reward.w, + reward.finalize_hash, + reward.mature_at, + ): + raise EscrowError("SHADOW_RESERVATION_CONFLICT") + return binding + + def reward_finalize( + self, reward: RewardFinalize | ShadowRewardFinalizeV1, evidence: FinalityEvidence + ) -> EscrowReceipt: + with self.tx(): + # L0 callbacks read this same BEGIN IMMEDIATE snapshot; no authority + # read may race ownership/finality revocation before bucket mutation. + match reward: + case ShadowRewardFinalizeV1(): + operation = sha256_hex( + ( + f"ht-shadow-finalize-v1|{self.run_id}|{reward.shadow_ordinal}|" + f"{reward.reservation_hash}" + ).encode() + ) + self._authority(reward) + self._shadow_reservation(reward) + if not evidence.shadow: + raise EscrowError("REWARD_DOMAIN") + case RewardFinalize(): + operation = sha256_hex( + f"ht-reward-finalize-v2|{self.run_id}|{reward.w}".encode() + ) + case unreachable: + assert_never(unreachable) + request = reward.model_dump_json(exclude={"authority_sig"}) + # Shadow replay authenticates the immutable operation before new authority predicates. + origins = None + match reward: + case ShadowRewardFinalizeV1(): + pass + case RewardFinalize(): + origins = self.reward_allocations(reward, evidence) + case unreachable: + assert_never(unreachable) + old = self._existing(operation, request) + if old is not None: + return old + if origins is None: + origins = self.reward_allocations(reward, evidence) + match reward: + case ShadowRewardFinalizeV1(): + last = self.db.execute( + "SELECT MAX(ordinal) FROM escrow_shadow_finalized" + ).fetchone()[0] + number = reward.shadow_ordinal + case RewardFinalize(): + last = self.db.execute("SELECT MAX(w) FROM escrow_finalized").fetchone()[0] + number = reward.w + case unreachable: + assert_never(unreachable) + if number != (last + 1 if last is not None else 0): + raise EscrowError("FINALIZE_ORDER") + pending_units = 0 + match reward: + case ShadowRewardFinalizeV1(): + pass + case RewardFinalize(): + pending_units = ( + self.policy.round_reward_units + * self.db.execute( + "SELECT COUNT(*) FROM records_v2 WHERE run_id=? " + "AND kind='shadow-reservation' " + "AND json_extract(data,'$.accepted') IS NULL", + (self.run_id,), + ).fetchone()[0] + ) + case unreachable: + assert_never(unreachable) + if ( + self.balances().issued + pending_units + reward.budget_units + > self.policy.max_total_issuance + ): + raise EscrowError("TOTAL_ISSUANCE_BUDGET") + match reward: + case ShadowRewardFinalizeV1(): + binding = self._shadow_reservation(reward) + by_origin = {shadow_origin_id(binding): binding.hotkey} + case RewardFinalize(): + by_origin = { + reward_origin_id( + self.run_id, + reward.w, + CommitV2.model_validate(item.commit.body).hotkey, + reward.finalize_hash, + ): CommitV2.model_validate(item.commit.body).hotkey + for item in evidence.works + } + case unreachable: + assert_never(unreachable) + for o in origins: + self._issue(o, reward.w, by_origin[o.origin_id]) + match reward: + case ShadowRewardFinalizeV1(): + self.db.execute( + "INSERT INTO escrow_shadow_finalized VALUES(?,?,?,?,?)", + ( + reward.shadow_ordinal, + reward.w, + reward.finalize_hash, + reward.reservation_hash, + origins[0].origin_id, + ), + ) + receipt = self._event( + operation, "SHADOW_REWARD_FINALIZE", request, origins[0].owner + ) + binding = self._shadow_reservation(reward) + self.db.execute( + "UPDATE records_v2 SET data=? WHERE run_id=? " + "AND kind='shadow-reservation' AND id=?", + ( + canonicalize( + {"binding": binding.body(), "accepted": receipt.body()} + ).decode(), + self.run_id, + str(reward.shadow_ordinal), + ), + ) + return receipt + case RewardFinalize(): + self.db.execute( + "INSERT INTO escrow_finalized VALUES(?,?)", (reward.w, reward.finalize_hash) + ) + return self._event(operation, "REWARD_FINALIZE", request, origins[0].owner) + case unreachable: + assert_never(unreachable) + + def _credit(self, origin: str, owner: str, bucket: str, ref: str, units: int) -> None: + self.db.execute( + """INSERT INTO escrow_units VALUES(?,?,?,?,?) ON CONFLICT(origin,owner,bucket,ref) + DO UPDATE SET units=units+excluded.units""", + (origin, owner, bucket, ref, units), + ) + + def _move( + self, + op: EscrowOperation, + source: str, + destination: str, + *, + source_ref: str = "", + destination_ref: str = "", + recipient: str | None = None, + ) -> None: + rows = self.db.execute( + "SELECT * FROM escrow_units WHERE owner=? AND bucket=? AND ref=?", + (op.owner, source, source_ref), + ).fetchall() + selected = sorted( + (r for r in rows if r["origin"] in op.origin_ids), key=lambda r: r["origin"] + ) + if {r["origin"] for r in selected} != set(op.origin_ids) or sum( + r["units"] for r in selected + ) < op.units: + raise EscrowError("UNFUNDED_OR_UNKNOWN_ORIGIN") + left = op.units + for row in selected: + take = min(left, row["units"]) + self.db.execute( + "UPDATE escrow_units SET units=units-? WHERE origin=? AND owner=? " + "AND bucket=? AND ref=?", + (take, row["origin"], op.owner, source, source_ref), + ) + self._credit(row["origin"], recipient or op.owner, destination, destination_ref, take) + left -= take + + def _owner_operation(self, op: EscrowOperation, signer: str) -> EscrowReceipt | None: + if signer != op.owner: + raise EscrowError("OWNER_AUTHORITY") + return self._existing(op.operation_id, op.model_dump_json()) + + def lock(self, op: EscrowLock, *, signer: str) -> EscrowReceipt: + with self.tx(): + old = self._owner_operation(op, signer) + if old is not None: + return old + bucket = "admission_locked" if op.kind == "LOCK_ADMISSION" else "dispute_locked" + self._move(op, "available", bucket, destination_ref=op.operation_id) + return self._event(op.operation_id, op.kind, op.model_dump_json(), op.owner) + + def transfer(self, op: EscrowTransfer, *, signer: str) -> EscrowReceipt: + with self.tx(): + old = self._owner_operation(op, signer) + if old is not None: + return old + self._move(op, "available", "available", recipient=op.recipient) + return self._event(op.operation_id, "TRANSFER", op.model_dump_json(), op.owner) + + def mature( + self, operation_id: str, origin_ids: tuple[str, ...], now_beacon: int + ) -> EscrowReceipt: + """Server-only vesting; disputed reward origins remain pending.""" + if not origin_ids or len(set(origin_ids)) != len(origin_ids) or type(now_beacon) is not int: + raise EscrowError("MATURITY_INPUT") + request = canonicalize(list(origin_ids)).decode() + with self.tx(): + old = self._existing(operation_id, request) + if old is not None: + return old + owner = "" + for origin in origin_ids: + record = self.db.execute( + "SELECT * FROM escrow_origins WHERE origin=?", (origin,) + ).fetchone() + if record is None or now_beacon < record["mature_at"]: + raise EscrowError("IMMATURE_OR_UNKNOWN_ORIGIN") + if record["reward_round"] is not None: + shadow = self.db.execute( + "SELECT finalize_hash FROM escrow_shadow_finalized WHERE origin=?", + (origin,), + ).fetchone() + if shadow is not None: + if self.shadow_settlement is None: + raise EscrowError("SHADOW_AUTHORITY_UNAVAILABLE") + status, finality = self.shadow_settlement(origin), shadow + else: + status = self.settlement(origin) + finality = self.db.execute( + "SELECT finalize_hash FROM escrow_finalized WHERE w=?", + (record["reward_round"],), + ).fetchone() + if ( + status.unresolved + or status.outcome == "FRAUD" + or now_beacon < status.release_beacon + or finality is None + or status.finality_hash != finality["finalize_hash"] + ): + raise EscrowError("DISPUTED_OR_UNVESTED") + rows = self.db.execute( + "SELECT * FROM escrow_units WHERE origin=? AND " + "bucket='reward_pending' AND units>0", + (origin,), + ).fetchall() + if not rows: + raise EscrowError("ALREADY_MATURED") + for row in rows: + owner = row["owner"] + self.db.execute( + "UPDATE escrow_units SET units=0 WHERE origin=? AND owner=? " + "AND bucket='reward_pending'", + (origin, owner), + ) + self._credit(origin, owner, "available", "", row["units"]) + return self._event(operation_id, "MATURE", request, owner) + + def release(self, op: EscrowRelease, *, signer: str, now_beacon: int) -> EscrowReceipt: + with self.tx(): + old = self._owner_operation(op, signer) + if old is not None: + return old + status = self.settlement(op.lock_id) + if ( + status.unresolved + or status.outcome == "FRAUD" + or now_beacon < status.release_beacon + or (op.finality_hash, op.closed_dispute_root) + != (status.finality_hash, status.closed_dispute_root) + ): + raise EscrowError("DISPUTED_OR_UNVESTED") + lock = self.db.execute( + "SELECT request,kind FROM escrow_events WHERE operation_id=?", (op.lock_id,) + ).fetchone() + if lock is None or lock["kind"] not in ("LOCK_ADMISSION", "LOCK_CONTEST"): + raise EscrowError("UNKNOWN_LOCK") + original = EscrowLock.model_validate_json(lock["request"]) + if (op.admission_id, op.dispute_id, op.owner) != ( + original.admission_id, + original.dispute_id, + original.owner, + ): + raise EscrowError("LOCK_REFERENCE") + source = "admission_locked" if original.kind == "LOCK_ADMISSION" else "dispute_locked" + self._move(op, source, "available", source_ref=op.lock_id) + return self._event(op.operation_id, "RELEASE", op.model_dump_json(), op.owner) + + def slash( + self, + op: EscrowRelease, + *, + authority: str, + now_beacon: int, + referee: str | None = None, + referee_cost: int = 0, + max_referee_cost: int = 0, + ) -> EscrowReceipt: + """Internal proven-fraud settlement, no client slash route; funded cost then burn.""" + if ( + authority != self.policy.reward_authority + or min(referee_cost, max_referee_cost) < 0 + or referee_cost > min(max_referee_cost, op.units) + ): + raise EscrowError("SLASH_AUTHORITY_OR_COST") + with self.tx(): + request = canonicalize( + [op.model_dump(mode="json"), referee, referee_cost, max_referee_cost] + ).decode() + old = self._existing(op.operation_id, request) + if old is not None: + return old + status = self.settlement(op.lock_id) + if ( + status.unresolved + or status.outcome != "FRAUD" + or now_beacon < status.release_beacon + or (op.finality_hash, op.closed_dispute_root) + != (status.finality_hash, status.closed_dispute_root) + ): + raise EscrowError("UNSETTLED_SLASH") + row = self.db.execute( + "SELECT request,kind FROM escrow_events WHERE operation_id=?", (op.lock_id,) + ).fetchone() + if row is None or row["kind"] not in ("LOCK_ADMISSION", "LOCK_CONTEST"): + raise EscrowError("UNKNOWN_LOCK") + original = EscrowLock.model_validate_json(row["request"]) + if (op.admission_id, op.dispute_id, op.owner) != ( + original.admission_id, + original.dispute_id, + original.owner, + ): + raise EscrowError("LOCK_REFERENCE") + source = "admission_locked" if original.kind == "LOCK_ADMISSION" else "dispute_locked" + if referee_cost: + if ( + original.kind != "LOCK_CONTEST" + or referee is None + or not self.balances(referee).issued + ): + raise EscrowError("UNFUNDED_REFEREE") + cost = EscrowOperation.model_validate( + { + **op.model_dump(mode="json", include=set(EscrowOperation.model_fields)), + "units": referee_cost, + } + ) + self._move(cost, source, "available", source_ref=op.lock_id, recipient=referee) + if op.units > referee_cost: + burn = EscrowOperation.model_validate( + { + **op.model_dump(mode="json", include=set(EscrowOperation.model_fields)), + "units": op.units - referee_cost, + } + ) + self._move(burn, source, "burned", source_ref=op.lock_id) + if original.kind == "LOCK_ADMISSION": + for row in self.db.execute( + "SELECT * FROM escrow_units WHERE owner=? AND " + "bucket='reward_pending' AND units>0", + (op.owner,), + ).fetchall(): + self.db.execute( + "UPDATE escrow_units SET units=0 WHERE origin=? AND owner=? " + "AND bucket='reward_pending'", + (row["origin"], op.owner), + ) + self._credit(row["origin"], op.owner, "burned", "", row["units"]) + return self._event(op.operation_id, "SLASH", request, op.owner) + + def pay(self, op: EscrowOperation, *, signer: str) -> EscrowReceipt: + """Terminal unit sink, not a cash payout or redemption adapter.""" + with self.tx(): + old = self._owner_operation(op, signer) + if old is not None: + return old + self._move(op, "available", "paid") + return self._event(op.operation_id, "PAY", op.model_dump_json(), op.owner) + + def balances(self, owner: str | None = None) -> Balances: + query = "SELECT bucket,SUM(units) total FROM escrow_units" + rows = self.db.execute( + query + (" WHERE owner=?" if owner else "") + " GROUP BY bucket", + (owner,) if owner else (), + ).fetchall() + buckets = {r["bucket"]: r["total"] for r in rows} + issued = ( + self.db.execute("SELECT COALESCE(SUM(issued),0) FROM escrow_origins").fetchone()[0] + if owner is None + else sum(buckets.values()) + ) + return Balances( + issued, + *( + buckets.get(b, 0) + for b in ( + "available", + "reward_pending", + "admission_locked", + "dispute_locked", + "paid", + "burned", + ) + ), + ) + + def locked(self, admission_id: str, owner: str) -> tuple[int, str]: + units, hashes = 0, [] + for r in self.db.execute("SELECT * FROM escrow_events WHERE kind='LOCK_ADMISSION'"): + op = EscrowLock.model_validate_json(r["request"]) + if op.admission_id == admission_id and op.owner == owner: + amount = self.db.execute( + "SELECT COALESCE(SUM(units),0) FROM escrow_units WHERE " + "bucket='admission_locked' AND ref=? AND owner=?", + (op.operation_id, owner), + ).fetchone()[0] + if amount: + units += amount + hashes.append(r["event_hash"]) + return units, sha256_hex(canonicalize(hashes)) if hashes else "0" * 64 + + def _conservation(self) -> None: + for row in self.db.execute( + "SELECT o.origin,o.issued,COALESCE(SUM(u.units),0) held FROM escrow_origins o " + "LEFT JOIN escrow_units u ON o.origin=u.origin GROUP BY o.origin" + ): + if row["issued"] != row["held"]: + raise EscrowError("ORIGIN_CONSERVATION") + if not self.balances().conserved(): + raise EscrowError("CONSERVATION") + + def verify(self) -> None: + """Check canonical chain/receipts and origin conservation after restart.""" + for expected, row in enumerate( + self.db.execute("SELECT * FROM escrow_shadow_finalized ORDER BY ordinal") + ): + if row["ordinal"] != expected: + raise EscrowError("SHADOW_FINALIZE_ORDER") + record = self.db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='shadow-reservation' AND id=?", + (self.run_id, str(expected)), + ).fetchone() + if record is None: + raise EscrowError("SHADOW_RESERVATION_REQUIRED") + data = json.loads(record["data"]) + binding = ShadowReservationV1.model_validate(data["binding"]) + if ( + digest(binding) != row["reservation_hash"] + or binding.finalize_hash != row["finalize_hash"] + or binding.trial_epoch != row["trial_epoch"] + or shadow_origin_id(binding) != row["origin"] + or data["accepted"] is None + ): + raise EscrowError("SHADOW_PERSISTED_BINDING") + operation = sha256_hex( + f"ht-shadow-finalize-v1|{self.run_id}|{expected}|{digest(binding)}".encode() + ) + event = self.db.execute( + "SELECT receipt FROM escrow_events WHERE operation_id=?", (operation,) + ).fetchone() + if event is None or json.loads(event["receipt"]) != data["accepted"]: + raise EscrowError("SHADOW_PERSISTED_RECEIPT") + prev = "0" * 64 + last_state = None + for i, row in enumerate(self.db.execute("SELECT * FROM escrow_events ORDER BY seq")): + snap = self.db.execute( + "SELECT state FROM escrow_snapshots WHERE operation_id=?", (row["operation_id"],) + ).fetchone() + if snap is None: + raise EscrowError("MISSING_EVENT_SNAPSHOT") + last_state = snap["state"] + actual = sha256_hex( + canonicalize( + [ + self.run_id, + i, + prev, + row["operation_id"], + row["kind"], + row["request"], + last_state, + ] + ) + ) + receipt = EscrowReceipt.model_validate_json(row["receipt"]) + if ( + row["seq"] != i + or row["prev_hash"] != prev + or row["event_hash"] != actual + or receipt.event_hash != actual + or receipt.operation_id != row["operation_id"] + or receipt.ledger_mode != self.policy.ledger_mode + ): + raise EscrowError("EVENT_CHAIN") + prev = actual + if last_state is not None and last_state != self._snapshot(): + raise EscrowError("PERSISTED_STATE_MISMATCH") + self._conservation() + + def _snapshot(self) -> str: + origins = [list(r) for r in self.db.execute("SELECT * FROM escrow_origins ORDER BY origin")] + units = [ + list(r) + for r in self.db.execute("SELECT * FROM escrow_units ORDER BY origin,owner,bucket,ref") + ] + finals = [list(r) for r in self.db.execute("SELECT * FROM escrow_finalized ORDER BY w")] + shadow = [ + list(r) + for r in self.db.execute("SELECT * FROM escrow_shadow_finalized ORDER BY ordinal") + ] + if shadow: + return canonicalize([origins, units, finals, shadow]).decode() + return canonicalize([origins, units, finals]).decode() diff --git a/src/hypertrain/miner/admission.py b/src/hypertrain/miner/admission.py new file mode 100644 index 0000000..49ed0f5 --- /dev/null +++ b/src/hypertrain/miner/admission.py @@ -0,0 +1,52 @@ +"""Client-side signatures only; the service decides funding and eligibility.""" + +from __future__ import annotations + +from pydantic import JsonValue + +from hypertrain.protocol.envelope_v2 import join_signing_message, seal +from hypertrain.protocol.keys import Keypair +from hypertrain.protocol.messages_v2 import HardwareHint, JoinRequest, RotateRequest, WorkProof + + +def sign_join( + hotkey: Keypair, + coldkey: Keypair, + *, + run_id: str, + request_id: str, + expires_beacon: int, + policy_hash: str, + hardware_hint: HardwareHint, +) -> JoinRequest: + request = JoinRequest( + run_id=run_id, + request_id=request_id, + hotkey=hotkey.ss58, + coldkey=coldkey.ss58, + expires_beacon=expires_beacon, + policy_hash=policy_hash, + hardware_hint=hardware_hint, + hot_sig="0" * 128, + cold_sig="0" * 128, + ) + message = join_signing_message(request) + return JoinRequest.model_validate( + { + **request.body(), + "hot_sig": hotkey.sign(message).hex(), + "cold_sig": coldkey.sign(message).hex(), + } + ) + + +def sign_proof( + hotkey: Keypair, run_id: str, proof: WorkProof, expires_beacon: int +) -> dict[str, JsonValue]: + return seal(hotkey, "WorkProof", run_id, proof, expires_beacon) + + +def sign_rotation( + coldkey: Keypair, run_id: str, request: RotateRequest, expires_beacon: int +) -> dict[str, JsonValue]: + return seal(coldkey, "RotateRequest", run_id, request, expires_beacon) diff --git a/src/hypertrain/miner/cli.py b/src/hypertrain/miner/cli.py index b4cc4ed..8a91e3a 100644 --- a/src/hypertrain/miner/cli.py +++ b/src/hypertrain/miner/cli.py @@ -15,7 +15,7 @@ from hypertrain.data.shards import ShardSamples, ShardSetManifest, verify_shards from hypertrain.datasets.shards16 import ShardSet16Manifest, U16ShardSamples, verify_shards16 -from hypertrain.miner.core import Miner, MinerConfig, MinerError, canonical +from hypertrain.miner.core import Miner, MinerConfig, MinerError, NetworkMiner, canonical from hypertrain.protocol.messages import RunManifest from hypertrain.trainer.loop import SampleFn @@ -89,10 +89,52 @@ def main(argv: list[str] | None = None) -> int: if name == "run": s.add_argument("--rounds", type=int, default=1) s.add_argument("--start", type=int) + for name in ("join-v2", "status-v2", "probe-v2", "proof-v2", "run-v2", "watch"): + s = sub.add_parser(name) + s.add_argument("--config", type=Path) + if name == "join-v2": + s.add_argument("--coldkey", type=Path, required=True) + s.add_argument("--request-id", required=True) + s.add_argument("--expiry", type=int, required=True) + if name in ("probe-v2", "run-v2", "watch"): + s.add_argument("--job", type=Path, required=name != "run-v2") + if name == "run-v2": + s.add_argument("--round", type=int) + if name == "probe-v2": + s.add_argument("--challenge", type=Path, required=True) + if name == "proof-v2": + s.add_argument("--proof", type=Path, required=True) + if name == "watch": + s.add_argument("--timeout", type=float, default=30) args = p.parse_args(argv) logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") try: cfg = MinerConfig.load(args.config) + if args.command not in ("join", "run"): + with httpx.Client(timeout=120, follow_redirects=False) as client: + network = NetworkMiner(cfg, client) + match args.command: + case "join-v2": + result = network.join(args.coldkey, args.request_id, args.expiry) + case "status-v2": + result = network.status() + case "probe-v2": + result = network.probe(args.job, args.challenge) + case "proof-v2": + result = network.proof(args.proof) + case "run-v2": + if args.job is not None: + result = network.launch(args.job) + elif args.round is not None: + result = {"status": network.run_round(args.round)} + else: + raise MinerError("run-v2 requires --round or --job") + case "watch": + result = {"processed": network.watch(args.job, args.timeout)} + case _: + raise MinerError("unknown v2 command") + print(json.dumps(result, sort_keys=True)) + return 0 miner = _miner(cfg) if args.command == "join": # ponytail: admission is the operator's PUT /roster (no public join route in diff --git a/src/hypertrain/miner/core.py b/src/hypertrain/miner/core.py index 778ac1f..1bb3c54 100644 --- a/src/hypertrain/miner/core.py +++ b/src/hypertrain/miner/core.py @@ -15,16 +15,22 @@ import os import shutil import stat +import threading import time import tomllib from collections.abc import Callable, Mapping from dataclasses import dataclass, fields, replace from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any, Literal from urllib.parse import urlsplit import httpx +if TYPE_CHECKING: + from hypertrain.gpu_ops.work_screen import IslandLaunch + from hypertrain.miner.island_launch import IslandArtifacts + from hypertrain.protocol.messages_v2 import IslandJobV1 + import hypertrain.trainer # noqa: F401 (determinism setup before torch) from hypertrain.auditor.bisect import Executor from hypertrain.auditor.replay import challenge_hash, pack_state, tensor_root, unpack_state @@ -627,3 +633,375 @@ def run(self, rounds: int, start: int | None = None) -> list[tuple[int, str]]: def canonical(obj: Any) -> bytes: return canonicalize(obj, allow_float=False) + + +class NetworkMiner: + """Explicit ht/2 client; shared API, keyfile, launcher, admission and watch primitives.""" + + def __init__(self, cfg: MinerConfig, client: httpx.Client) -> None: + from hypertrain.protocol import envelope_v2 + from hypertrain.protocol.messages_v2 import RunManifestV2 + + if cfg.run_id is None or cfg.owner_hotkey is None: + raise MinerError("v2 requires pinned run_id and owner_hotkey") + self.cfg, self.api, self.kp = cfg, Api(client, cfg.api), load_keyfile(cfg.keyfile) + self.run_id, self.url = cfg.run_id, f"/v2/runs/{cfg.run_id}" + status = self.api.call("GET", self.url) + env = envelope_v2.parse_envelope(status["manifest_envelope"]) + if ( + env.signer != cfg.owner_hotkey + or not envelope_v2.verify_envelope(env.model_dump()) + or env.type != "RunManifestV2" + ): + raise MinerError("v2 manifest authority/signature mismatch") + self.manifest = RunManifestV2.model_validate(env.body) + self.manifest.validate_run_id(self.run_id) + self.directory = cfg.workdir / self.run_id / self.kp.ss58 + self.directory.mkdir(parents=True, exist_ok=True) + + def status(self) -> dict[str, Any]: + return self.api.call("GET", self.url + "/admission/" + self.kp.ss58) + + def join(self, coldfile: Path, request_id: str, expiry: int) -> dict[str, Any]: + from hypertrain.miner.admission import sign_join + from hypertrain.protocol.messages_v2 import HardwareHint + + request = sign_join( + self.kp, + load_keyfile(coldfile), + run_id=self.run_id, + request_id=request_id, + expires_beacon=expiry, + policy_hash=self.manifest.network.admission_policy_hash, + hardware_hint=HardwareHint( + device_name=self.cfg.device, + device_count=self.manifest.training.reference_spec.layout.n_gpus, + driver="advisory", + ), + ) + return self.api.call("POST", self.url + "/join", content=canonical(request.body())) + + def probe( + self, + job_path: Path, + challenge_path: Path, + *, + launch: IslandLaunch | None = None, + cancel: threading.Event | None = None, + trace: bool = False, + ) -> dict[str, Any]: + from hypertrain.gpu_ops.work_screen import screen_work + from hypertrain.protocol import envelope_v2 + from hypertrain.protocol.messages_v2 import IslandJobV1, JoinChallenge + + job = IslandJobV1.model_validate_json(job_path.read_bytes()) + if time.time() >= job.deadline or cancel is not None and cancel.is_set(): + raise MinerError("probe deadline/cancellation") + env = envelope_v2.parse_envelope(challenge_path.read_bytes()) + if ( + job.run_id != self.run_id + or env.signer != self.manifest.training.coord_pubkey + or (env.run_id != self.run_id or not envelope_v2.verify_envelope(env.model_dump())) + ): + raise MinerError("job/challenge authority mismatch") + now = self.api.call("GET", self.url)["now_round"] + challenge = JoinChallenge.model_validate(env.body) + requested_launch = launch + if trace: + from hypertrain.miner.island_launch import confined, launch_island + + def traced_launch( + job: IslandJobV1, + directory: Path, + *, + backend: Literal["cpu", "cuda"], + cancel: threading.Event | None = None, + trace: bool = False, + ) -> IslandArtifacts: + artifacts = (launch or launch_island)( + job, directory, backend=backend, cancel=cancel, trace=True + ) + for rank in range(job.manifest.training.reference_spec.layout.n_gpus): + relative = f"rank-{rank}/trace.json" + if not (artifacts.directory / relative).is_file(): + raise MinerError("probe required trace absent: " + relative) + confined(artifacts.directory, relative) + return artifacts + + requested_launch = traced_launch + screen, proof = screen_work( + job, + job_path.parent, + challenge, + now_beacon=now, + backend="cpu" if self.cfg.device == "cpu" else "cuda", + launch=requested_launch, + cancel=cancel, + ) + if time.time() >= job.deadline or cancel is not None and cancel.is_set(): + raise MinerError("probe deadline/cancellation") + result = { + "proof": envelope_v2.seal( + self.kp, "WorkProof", self.run_id, proof, challenge.deadline_beacon + ), + "screen": envelope_v2.seal( + self.kp, "WorkScreenV2", self.run_id, screen, challenge.deadline_beacon + ), + } + for filename in ("state.safetensors", "ef.safetensors", "delta.bin", "leaves.json"): + if time.time() >= job.deadline or cancel is not None and cancel.is_set(): + raise MinerError("probe deadline/cancellation") + path = job_path.parent / "published" / "rank-0" / filename + response = self.api.c.put( + self.api.base + self.url + "/objects/" + sha256_hex(path.read_bytes()), + content=path.read_bytes(), + headers={ + "X-Object-Signature": self.kp.sign( + b"hypertrain/object/2|" + + self.run_id.encode() + + b"|" + + sha256_hex(path.read_bytes()).encode() + ).hex(), + "X-Hotkey": self.kp.ss58, + }, + ) + response.raise_for_status() + return result + + def proof(self, document: Path) -> dict[str, Any]: + from hypertrain.protocol.envelope_v2 import load_json + + return self.api.call( + "POST", self.url + "/join/proof", json=load_json(document.read_bytes()) + ) + + def launch(self, job_path: Path, trace: bool = True) -> dict[str, str]: + from hypertrain.miner.island_launch import launch_island + from hypertrain.protocol.messages_v2 import IslandJobV1 + + job = IslandJobV1.model_validate_json(job_path.read_bytes()) + if job.run_id != self.run_id: + raise MinerError("cross-run island job") + artifacts = launch_island( + job, job_path.parent, backend="cpu" if self.cfg.device == "cpu" else "cuda", trace=trace + ) + return { + "state": str(artifacts.state), + "delta": str(artifacts.delta), + "leaves": str(artifacts.leaves), + } + + def run_round(self, w: int) -> str: + import anyio + + from hypertrain.miner.island_launch import launch_island + from hypertrain.protocol import envelope_v2, relay_envelope + from hypertrain.protocol.messages_v2 import ArtifactRef, IslandJobV1, RoundOpenV2, WorkProof + from hypertrain.protocol.relay_messages import RelayRegistryV1 + from hypertrain.relay.client import RelayClient, chunk_manifest + + view = self.api.call("GET", self.url + f"/rounds/{w}") + opening = envelope_v2.parse_envelope(view["round_open"]) + if opening.signer != self.manifest.training.coord_pubkey or not envelope_v2.verify_envelope( + opening.model_dump() + ): + raise MinerError("round authority mismatch") + schedule = RoundOpenV2.model_validate(opening.body) + assignment = next((a for a in view["assignment"] if a["hotkey"] == self.kp.ss58), None) + if assignment is None: + raise MinerError("authenticated assignment not ready") + status = self.status() + screen_hash = status["work_screen_hash"] + driver = "cpu" + if self.cfg.device != "cpu": + # Parent launch validates exact logical count/SM/runtime; this is observed driver only. + version = Path("/proc/driver/nvidia/version") + parts = version.read_text().split() + driver = next((p for p in parts if p[:1].isdigit() and "." in p), "unknown") + if ( + self.cfg.device != "cpu" + and driver not in self.manifest.training.reference_spec.driver_allowlist + ): + raise HardwareMismatch("observed v2 driver not in manifest allowlist") + if self.cfg.device == "cpu": + from hypertrain.protocol.messages_v2 import EconomicsPolicyV2 + + policy_response = self.api.c.get( + self.api.base + self.url + "/objects/" + self.manifest.network.economics_policy_hash + ) + policy_response.raise_for_status() + if sha256_hex( + policy_response.content + ) != self.manifest.network.economics_policy_hash or ( + EconomicsPolicyV2.model_validate_json(policy_response.content).ledger_mode != "test" + ): + raise HardwareMismatch( + "CPU client cannot assert production CUDA reference identity" + ) + driver = self.manifest.training.reference_spec.driver_allowlist[0] + accept = { + "w": w, + "hotkey": self.kp.ss58, + "assignment_hash": assignment["assignment_hash"], + "work_screen_hash": screen_hash, + "image_digest": self.manifest.training.reference_spec.image_digest, + "driver_version": driver, + "n_gpus": self.manifest.training.reference_spec.layout.n_gpus, + } + self.api.call( + "POST", + self.url + "/accept", + json=envelope_v2.seal(self.kp, "AcceptV2", self.run_id, accept, schedule.d_final), + ) + staged = self.api.call("GET", self.url + f"/rounds/{w}/job/{self.kp.ss58}") + job = IslandJobV1.model_validate(staged["job"]) + directory = self.directory / str(w) + directory.mkdir(parents=True, exist_ok=True) + for name, digest in staged["objects"].items(): + response = self.api.c.get(self.api.base + self.url + "/objects/" + digest) + response.raise_for_status() + if sha256_hex(response.content) != digest: + raise MinerError("job input hash mismatch") + (directory / name).write_bytes(response.content) + artifacts = launch_island( + job, directory, backend="cpu" if self.cfg.device == "cpu" else "cuda", trace=True + ) + preimages = json.loads(artifacts.leaves.read_bytes()) + summary = json.loads((artifacts.directory / "rank-0/summary.json").read_bytes())[ + "commitments" + ] + start = next(s for s in view["starts"] if s["hotkey"] == self.kp.ss58) + commit = { + "w": w, + "hotkey": self.kp.ss58, + "leaf_scheme": "ht-leaf-v1", + "n_leaves": len(preimages), + "metrics_root": MerkleTree( + [bytes.fromhex(p["loss_f32"] + p["norm_f32"]) for p in preimages] + ).root.hex(), + "tokens": len(job.sample_ids) * self.manifest.training.model.seq_len, + "delta_bytes": artifacts.delta.stat().st_size, + "ef_in_hash": start["ef_hash"], + **{ + k: summary[k] + for k in ("leaves_root", "final_theta_hash", "ef_out_hash", "delta_hash") + }, + } + self.api.call( + "POST", + self.url + "/commit", + json=envelope_v2.seal(self.kp, "CommitV2", self.run_id, commit, schedule.d_final), + ) + refs = [] + for path in (artifacts.state, artifacts.ef, artifacts.delta, artifacts.leaves): + digest = sha256_hex(path.read_bytes()) + response = self.api.c.put( + self.api.base + self.url + "/objects/" + digest, + content=path.read_bytes(), + headers={ + "X-Hotkey": self.kp.ss58, + "X-Object-Signature": self.kp.sign( + f"hypertrain/object/2|{self.run_id}|{digest}".encode() + ).hex(), + }, + ) + response.raise_for_status() + refs.append(ArtifactRef(sha256=digest, size=path.stat().st_size)) + descriptor = WorkProof( + admission_id=status["record"]["admission_id"], + challenge_hash=sha256_hex(canonical(commit)), + leaves_root=commit["leaves_root"], + delta_hash=commit["delta_hash"], + artifact_refs=refs, + ) + self.api.call( + "POST", + self.url + "/leaves", + json=envelope_v2.seal(self.kp, "WorkProof", self.run_id, descriptor, schedule.d_final), + ) + assigned = self.api.call("GET", self.url + f"/rounds/{w}/relay/{self.kp.ss58}") + chunks = chunk_manifest(artifacts.delta) + grant = self.api.call( + "POST", + self.url + "/upload-grant?commit_hash=" + sha256_hex(canonical(commit)), + json=relay_envelope.seal( + self.kp, "UploadChunkManifest", self.run_id, chunks, schedule.d_upload + ), + ) + from hypertrain.protocol.relay_envelope import RelayEnvelope + + async def upload() -> RelayEnvelope: + async with httpx.AsyncClient(follow_redirects=False, timeout=60) as client: + return await RelayClient( + RelayRegistryV1.model_validate(assigned["registry"]), client + ).upload( + artifacts.delta, + grant_raw=relay_envelope.parse_envelope(grant), + assignment=relay_envelope.parse_envelope(assigned["assignment"]), + round_open=view["round_open"], + ) + + receipt = anyio.run(upload) + acceptance = self.api.call( + "POST", "/v2/relay-receipts", content=canonical(receipt.model_dump(mode="json")) + ) + payload = artifacts.delta.read_bytes() + delta = { + "w": w, + "hotkey": self.kp.ss58, + "delta_hash": commit["delta_hash"], + "size": len(payload), + "uri": "sha256:" + commit["delta_hash"], + "format": "ht-sparse-v1" + if self.manifest.training.outer.opt == "sparseloco" + else "ht-dense-int8-v1", + "chunks": [ + {"off": c.off, "len": c.len, "sha256": c.chunk_sha256} for c in chunks.chunks + ], + "grant_hash": sha256_hex(canonical(grant["body"])), + "master_acceptance_hash": sha256_hex(canonical(acceptance["body"])), + } + self.api.call( + "POST", + self.url + "/delta", + json=envelope_v2.seal( + self.kp, "DeltaManifestV2", self.run_id, delta, schedule.d_upload + ), + ) + return "UPLOADED" + + def watch(self, job_path: Path, timeout: float = 30) -> int: + from hypertrain.auditor.island_bisect import IslandParty + from hypertrain.data.store import LocalFSStore + from hypertrain.miner.dispute_watch import ( + DisputeWatch, + HttpDisputeTransport, + PublishedStates, + ) + from hypertrain.protocol.messages_v2 import IslandJobV1 + + job = IslandJobV1.model_validate_json(job_path.read_bytes()) + if job.run_id != self.run_id: + raise MinerError("cross-run watch job") + published = job_path.parent / "published" + states = PublishedStates(job, published, LocalFSStore(self.directory / "state-objects")) + party = IslandParty.published(self.kp.ss58, job, published) + client = self.api.c + client.headers["X-Dispute-Signature"] = self.kp.sign( + f"hypertrain/watch/2|{self.run_id}|{self.kp.ss58}".encode() + ).hex() + watcher = DisputeWatch( + self.directory / "watch", + self.kp, + HttpDisputeTransport(client, self.api.base + self.url), + run_id=self.run_id, + coordinator=self.manifest.training.coord_pubkey, + party=lambda _: party, + state_serve=states.serve, + beacon=lambda: self.api.call("GET", self.url)["now_round"], + job=job, + ) + try: + return watcher.run_once(timeout=timeout) + finally: + watcher.close() diff --git a/src/hypertrain/miner/dispute_watch.py b/src/hypertrain/miner/dispute_watch.py new file mode 100644 index 0000000..9ee52d3 --- /dev/null +++ b/src/hypertrain/miner/dispute_watch.py @@ -0,0 +1,292 @@ +"""Training-independent signed long-poll watch with durable cursor/outbox. + +Crash after delivery reuses the identical signed reply. The service's semantic +reservation makes its effect exactly once; networks cannot promise one send. +""" + +from __future__ import annotations + +import sqlite3 +from collections.abc import Callable +from pathlib import Path +from typing import Protocol + +import httpx +from pydantic import TypeAdapter + +from hypertrain.auditor.island_bisect import IslandParty +from hypertrain.auditor.replay import tensor_root, unpack_state +from hypertrain.challenge.disputes_v2 import ( + DisputeError, + EventAck, + WatchEvent, + authentic, + signed_message, +) +from hypertrain.data.store import Store +from hypertrain.gpu_ops.journal import flock +from hypertrain.miner.island_launch import confined, validate_artifacts +from hypertrain.protocol.envelope_v2 import load_json, seal +from hypertrain.protocol.hashing import MerkleTree +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.keys import Keypair +from hypertrain.protocol.messages import LeafPreimage, StateServe +from hypertrain.protocol.messages_v2 import BisectV2, IslandJobV1 + + +class DisputeTransport(Protocol): + def events(self, party: str, cursor: int, *, timeout: float) -> list[WatchEvent]: ... + def acknowledge(self, ack: EventAck) -> None: ... + def submit(self, envelope: bytes) -> None: ... + + +class HttpDisputeTransport: + """L0 routes own role authorization. Client bounds response bytes before parsing.""" + + def __init__(self, client: httpx.Client, run_url: str) -> None: + self.client, self.url = client, run_url.rstrip("/") + + def events(self, party: str, cursor: int, *, timeout: float) -> list[WatchEvent]: + if not 0 <= timeout <= 30: + raise DisputeError("LONG_POLL_BOUND") + with self.client.stream( + "GET", + self.url + "/disputes", + params={ + "party": party, + "cursor": cursor, + "timeout": timeout, + }, + timeout=timeout + 5, + ) as response: + response.raise_for_status() + data = bytearray() + for chunk in response.iter_bytes(): + data.extend(chunk) + if len(data) > 1_048_576: + raise DisputeError("EVENT_RESPONSE_BOUND") + parsed = load_json(b'{"items":' + bytes(data) + b"}") + events = TypeAdapter(list[WatchEvent]).validate_python(parsed["items"]) + if len(events) > 64: + raise DisputeError("EVENT_COUNT_BOUND") + return events + + def acknowledge(self, ack: EventAck) -> None: + response = self.client.post( + self.url + "/disputes/ack", content=canonicalize(ack.body()), timeout=5 + ) + response.raise_for_status() + + def submit(self, envelope: bytes) -> None: + raw = load_json(envelope) + if not isinstance(raw, dict): + raise DisputeError("ENVELOPE_TYPE") + route = {"BisectV2": "bisect", "StateServe": "state-serve"}.get(str(raw["type"])) + if route is None: + raise DisputeError("WATCH_REPLY_TYPE") + response = self.client.post(self.url + "/" + route, content=envelope, timeout=5) + response.raise_for_status() + + +class PublishedStates: + """Serve validated L1 leaf checkpoints, original Merkle preimages and existing Store.""" + + def __init__(self, job: IslandJobV1, directory: Path, objects: Store) -> None: + artifacts = validate_artifacts(job, directory) + self.job, self.directory, self.objects = job, directory, objects + raw = load_json(b'{"items":' + artifacts.leaves.read_bytes() + b"}")["items"] + if not isinstance(raw, list): + raise DisputeError("LEAF_ARTIFACT_TYPE") + self.leaves: list[LeafPreimage] = [LeafPreimage.model_validate(x) for x in raw] + self.tree: MerkleTree = MerkleTree([bytes.fromhex(leaf.digest()) for leaf in self.leaves]) + + def serve(self, event: WatchEvent) -> StateServe: + leaf = event.checkpoint + if ( + leaf is None + or not 0 <= leaf < len(self.leaves) + or (event.turn.contest.w != self.job.w or event.run_id != self.job.run_id) + ): + raise DisputeError("STATE_CHECKPOINT_BINDING") + t = self.leaves[leaf].t + blob = confined(self.directory, f"rank-0/checkpoints/{t}.safetensors").read_bytes() + theta, state = unpack_state(blob) + if state is None: + raise DisputeError("CHECKPOINT_OPTIMIZER_MISSING") + key = self.objects.put(blob) + return StateServe( + hotkey=event.party, + challenge_hash=event.turn.contest.challenge_hash, + t=t, + uri=self.objects.presign(key), + tensor_root=tensor_root(theta, state), + merkle_proof_leaf_in_leaves_root=[h.hex() for h in self.tree.proof(leaf)], + ) + + +class DisputeWatch: + """Mutable local delivery journal; one process owns a watch directory via flock.""" + + def __init__( + self, + directory: Path, + key: Keypair, + transport: DisputeTransport, + *, + run_id: str, + coordinator: str, + party: Callable[[str], IslandParty], + state_serve: Callable[[WatchEvent], StateServe], + beacon: Callable[[], int], + job: IslandJobV1 | None = None, + ) -> None: + publication = getattr(state_serve, "__self__", None) + if job is None and isinstance(publication, PublishedStates): + job = publication.job + if job is None or job.run_id != run_id: + raise DisputeError("WATCH_PUBLICATION_CONTEXT") + self.job = IslandJobV1.model_validate(job.model_dump(mode="json")) + directory.mkdir(parents=True, exist_ok=True) + self.directory, self.key, self.transport = directory, key, transport + self.run_id, self.coordinator = run_id, coordinator + self.party, self.state_serve, self.beacon = party, state_serve, beacon + self.db = sqlite3.connect(directory / "watch.db", isolation_level=None) + self.db.execute("PRAGMA journal_mode=WAL") + self.db.execute("PRAGMA synchronous=FULL") + self.db.executescript(""" + CREATE TABLE IF NOT EXISTS watch_identity(run TEXT,party TEXT,coordinator TEXT); + CREATE TABLE IF NOT EXISTS watch_outbox(cursor INTEGER PRIMARY KEY,event TEXT, + ack TEXT,envelope BLOB,done INTEGER NOT NULL DEFAULT 0); + """) + identity = self.db.execute("SELECT * FROM watch_identity").fetchone() + expected = (run_id, key.ss58, coordinator) + if identity is not None and identity != expected: + raise DisputeError("WATCH_IDENTITY_CHANGED") + if identity is None: + self.db.execute("INSERT INTO watch_identity VALUES(?,?,?)", expected) + + @property + def cursor(self) -> int: + return int( + self.db.execute( + "SELECT COALESCE(MAX(cursor),0) FROM watch_outbox WHERE done=1", + ).fetchone()[0] + ) + + def _validate_event(self, event: WatchEvent) -> None: + if not authentic(event) or (event.run_id, event.party, event.signer, event.turn.run_id) != ( + self.run_id, + self.key.ss58, + self.coordinator, + self.run_id, + ): + raise DisputeError("WATCH_EVENT_AUTHORITY") + if event.turn.contest.w != self.job.w: + raise DisputeError("WATCH_PUBLICATION_CONTEXT") + + def _prepare(self, event: WatchEvent) -> None: + self._validate_event(event) + old = self.db.execute( + "SELECT event FROM watch_outbox WHERE cursor=?", (event.cursor,) + ).fetchone() + if old is not None: + if WatchEvent.model_validate_json(old[0]) != event: + raise DisputeError("WATCH_CURSOR_CONFLICT") + return + now = self.beacon() + if ( + not event.issued_beacon + <= now + <= min( + event.turn.turn_deadline, + event.turn.absolute_deadline, + ) + ): + raise DisputeError("WATCH_EVENT_EXPIRED") + match event.kind: + case "turn": + turn = event.turn + executor = self.party(turn.dispute_id) + if executor.hotkey != self.key.ss58 or turn.expected_party != self.key.ss58: + raise DisputeError("WATCH_PARTY_BINDING") + lo, hi = turn.interval + at = [lo, -(-(lo + hi) // 2), hi] # N2 remains three hashes for a unit span + body = BisectV2( + dispute_id=turn.dispute_id, + seq=turn.seq, + level=turn.level, + ctx=turn.ctx, + interval=turn.interval, + N=2, + hashes=executor.hashes(turn.level, tuple(turn.ctx), at), + party=self.key.ss58, + previous_transcript_hash=turn.transcript_hash, + ) + envelope = seal(self.key, "BisectV2", self.run_id, body, turn.absolute_deadline) + case "state": + serve = self.state_serve(event) + if ( + serve.hotkey != self.key.ss58 + or serve.challenge_hash != event.turn.contest.challenge_hash + ): + raise DisputeError("STATE_SERVE_IDENTITY") + envelope = seal( + self.key, "StateServe", self.run_id, serve, event.turn.absolute_deadline + ) + ack = EventAck( + run_id=self.run_id, + cursor=event.cursor, + event_hash=event.digest(), + party=self.key.ss58, + received_beacon=now, + sig="", + ) + ack = ack.model_copy(update={"sig": self.key.sign(signed_message(ack)).hex()}) + self.db.execute( + "INSERT INTO watch_outbox(cursor,event,ack,envelope) VALUES(?,?,?,?)", + ( + event.cursor, + event.model_dump_json(), + ack.model_dump_json(), + canonicalize(envelope), + ), + ) + + def run_once(self, *, timeout: float = 30) -> int: + """Validate publication context before any durable pending or fresh sends.""" + with flock(self.directory / "watch.lock", blocking=False): + for row in self.db.execute("SELECT event FROM watch_outbox WHERE done=0"): + self._validate_event(WatchEvent.model_validate_json(row[0])) + pending = self.db.execute( + "SELECT cursor,ack,envelope FROM watch_outbox WHERE done=0 ORDER BY cursor", + ).fetchall() + requested_cursor = max([self.cursor, *(row[0] for row in pending)]) + events = self.transport.events(self.key.ss58, requested_cursor, timeout=timeout) + for event in events: + self._validate_event(event) + sent = 0 + for cursor, ack, envelope in pending: + self.transport.acknowledge(EventAck.model_validate_json(ack)) + self.transport.submit(envelope) + self.db.execute("UPDATE watch_outbox SET done=1 WHERE cursor=?", (cursor,)) + sent += 1 + if len(events) > 64 or any( + e.cursor <= self.cursor or (i and e.cursor <= events[i - 1].cursor) + for i, e in enumerate(events) + ): + raise DisputeError("WATCH_EVENT_ORDER_OR_BOUND") + for event in events: + self._validate_event(event) + for event in events: + self._prepare(event) + row = self.db.execute( + "SELECT ack,envelope FROM watch_outbox WHERE cursor=?", (event.cursor,) + ).fetchone() + self.transport.acknowledge(EventAck.model_validate_json(row[0])) + self.transport.submit(row[1]) + self.db.execute("UPDATE watch_outbox SET done=1 WHERE cursor=?", (event.cursor,)) + sent += 1 + return sent + + def close(self) -> None: + self.db.close() diff --git a/src/hypertrain/miner/island_launch.py b/src/hypertrain/miner/island_launch.py new file mode 100644 index 0000000..2005ce6 --- /dev/null +++ b/src/hypertrain/miner/island_launch.py @@ -0,0 +1,685 @@ +"""Manifest-driven torchrun parent. Publish only complete, independently checked rank artifacts.""" + +from __future__ import annotations + +import hashlib +import json +import os +import shlex +import shutil +import signal +import socket +import subprocess +import sys +import tempfile +import threading +import time +import uuid +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +from hypertrain.gpu_ops.journal import durable_write, flock +from hypertrain.protocol.envelope_v2 import load_json +from hypertrain.protocol.hashing import MerkleTree +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.messages import LeafPreimage +from hypertrain.protocol.messages_v2 import IslandJobV1, RankWorkResult + + +class IslandFailure(RuntimeError): + """No artifact may be committed after this failure.""" + + +@dataclass(frozen=True, slots=True) +class IslandArtifacts: + directory: Path + state: Path + ef: Path + delta: Path + leaves: Path + ranks: tuple[RankWorkResult, ...] + + +@dataclass(frozen=True, slots=True) +class CapacityAttempt: + """Internal owner-validated identity, shared lock and committed ledger debit seam.""" + + identity: str + profile_hash: str + lock_path: Path + charge: Callable[[], bool] + cpu_quota: Literal["50%", "400%"] | None = None + + +_CAPACITY_MEMORY = 1 << 30 +_CAPACITY_CPU_PERCENT = 400 +_CAPACITY_EXEC = """import json,os,socket,sys,time +c=json.load(open(sys.argv[1])) +g=open('/proc/self/cgroup').read().strip().split('::',1)[1] +p='/sys/fs/cgroup'+g +names=('memory.max','memory.swap.max','cpu.max','memory.oom.group') +v={n:open(p+'/'+n).read().strip() for n in names} +v.update(cgroup=g,affinity=sorted(os.sched_getaffinity(0)),nice=os.getpriority(os.PRIO_PROCESS,0),pid=os.getpid()) +s=socket.socket(socket.AF_UNIX);s.connect(c['socket']);s.sendall(json.dumps(v).encode()+b'\\n') +assert s.recv(1)==b'G' +s.close() +assert time.time() str: + try: + result = subprocess.run( # noqa: S603 - fixed manager argv and owned unit names + argv, capture_output=True, text=True, timeout=15, check=False + ) + except OSError as exc: + if check: + raise IslandFailure("capacity manager unavailable") from exc + return "manager-unavailable" + if check and result.returncode: + raise IslandFailure(f"capacity manager failure: {result.stderr[-2000:]}") + return result.stdout.strip() + + +def run_capacity_argv( + argv: list[str], + directory: Path, + deadline: int, + capacity: CapacityAttempt, + *, + env: Mapping[str, str], + cancel: threading.Event | None = None, +) -> None: + """One real manager-owned attempt. Never reset debit/deadline or retry on restart.""" + cpu_quota = capacity.cpu_quota or f"{_CAPACITY_CPU_PERCENT}%" + cpu_max = f"{int(cpu_quota.removesuffix('%')) * 1000} 100000" + if len(capacity.identity) != 64 or len(capacity.profile_hash) != 64: + raise IslandFailure("capacity identity binding") + try: + bytes.fromhex(capacity.identity + capacity.profile_hash) + except ValueError as exc: + raise IslandFailure("capacity identity binding") from exc + directory.mkdir(parents=True, exist_ok=True) + with flock(capacity.lock_path): + if time.time() >= deadline or cancel is not None and cancel.is_set(): + raise IslandFailure("capacity deadline/cancellation") + record = directory / "capacity-attempt.json" + if record.exists(): + saved = load_json(record.read_bytes()) + if not isinstance(saved, dict) or ( + saved.get("identity") != capacity.identity + or saved.get("profile_hash") != capacity.profile_hash + or saved.get("cpu_quota") != cpu_quota + ): + raise IslandFailure("capacity recovery identity differs") + unit = str(saved["unit"]) + base = unit.removesuffix(".service") + if not base.startswith("ht-cap-") or len(base.removeprefix("ht-cap-")) != 32: + raise IslandFailure("capacity recovery unit differs") + bytes.fromhex(base.removeprefix("ht-cap-")) + if ( + saved["timer"] != base + "-deadline.timer" + or saved["timer_service"] != base + "-deadline.service" + ): + raise IslandFailure("capacity recovery timer differs") + names = (unit, base + "-deadline.timer", base + "-deadline.service") + if not (directory / "capacity-cleaned.json").exists(): + status = _manager( + "systemctl", "show", unit, "-p", "Result", "-p", "ExecMainStatus", check=False + ) + durable_write( + directory / "capacity-recovery.json", + canonicalize({"status": status, "unit": unit}), + ) + for name in names: + _manager("systemctl", "stop", name, check=False) + _manager("systemctl", "reset-failed", *names, check=False) + for name in names: + state = _manager("systemctl", "show", name, "-p", "ActiveState", check=False) + if "ActiveState=active" in state or "ActiveState=activating" in state: + raise IslandFailure("capacity recovery cleanup incomplete") + durable_write( + directory / "capacity-cleaned.json", canonicalize({"units": list(names)}) + ) + raise IslandFailure("capacity attempt already recorded; recovery required") + if not capacity.charge(): + raise IslandFailure("capacity attempt already charged; recovery required") + unit = "ht-cap-" + uuid.uuid4().hex + ".service" + timer_service = unit.removesuffix(".service") + "-deadline.service" + timer = timer_service.removesuffix(".service") + ".timer" + durable_write( + record, + canonicalize( + { + "identity": capacity.identity, + "profile_hash": capacity.profile_hash, + "cpu_quota": cpu_quota, + "deadline": deadline, + "unit": unit, + "timer": timer, + "timer_service": timer_service, + } + ), + ) + observed: dict[str, str | int | list[int]] = {} + status = "" + proc: subprocess.Popen[bytes] | None = None + memory_events: int | None = None + try: + with tempfile.TemporaryDirectory(prefix="ht-cap-ipc-") as tmp: + address = str(Path(tmp) / "ready.sock") + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as listener: + listener.bind(address) + listener.listen(1) + listener.settimeout(min(15, max(0.001, deadline - time.time()))) + config = directory / "capacity-exec.json" + durable_write( + config, + canonicalize( + { + "socket": address, + "argv": argv, + "env": dict(env), + "deadline": deadline, + "cpu_quota": cpu_quota, + } + ), + ) + _manager( + "systemd-run", + f"--unit={timer_service}", + f"--on-calendar=@{deadline}", + "--timer-property=AccuracySec=1us", + "--timer-property=RandomizedDelaySec=0", + "--property=RemainAfterExit=yes", + "--property=Nice=19", + "/usr/bin/systemctl", + "kill", + "--kill-whom=all", + "--signal=SIGKILL", + unit, + ) + properties = [ + "Type=exec", + f"MemoryMax={_CAPACITY_MEMORY}", + "MemorySwapMax=0", + f"CPUQuota={cpu_quota}", + "CPUQuotaPeriodSec=100ms", + "CPUAffinity=0 1 2 3", + "Nice=19", + "OOMPolicy=kill", + "KillMode=control-group", + "KillSignal=SIGKILL", + "Restart=no", + "TimeoutStopSec=1s", + "NoNewPrivileges=yes", + "CapabilityBoundingSet=", + "ProtectControlGroups=yes", + "RuntimeMaxSec=600s", + "ExecStopPost=" + + shlex.join( + [ + sys.executable, + "-c", + _CAPACITY_STOP, + str((directory / "capacity-memory-events.txt").resolve()), + ] + ), + ] + with ( + open(directory / "capacity-stdout.log", "xb") as stdout, + open(directory / "capacity-stderr.log", "xb") as stderr, + ): + executable = shutil.which("systemd-run") + if executable is None: + raise IslandFailure("capacity manager unavailable") + proc = subprocess.Popen( # noqa: S603 - fixed manager/package wrapper + [ + executable, + f"--unit={unit}", + "--wait", + "--pipe", + "--expand-environment=no", + *["--property=" + p for p in properties], + sys.executable, + "-c", + _CAPACITY_EXEC, + str(config.resolve()), + ], + stdout=stdout, + stderr=stderr, + ) + connection, _ = listener.accept() + with connection: + connection.settimeout(5) + raw = bytearray() + while not raw.endswith(b"\n"): + chunk = connection.recv(4096) + if not chunk or len(raw) + len(chunk) > 8192: + raise IslandFailure("capacity readiness malformed") + raw.extend(chunk) + observed = json.loads(raw) + if ( + observed["memory.max"] != str(_CAPACITY_MEMORY) + or observed["memory.swap.max"] != "0" + or observed["cpu.max"] != cpu_max + or observed["memory.oom.group"] != "1" + or observed["affinity"] != [0, 1, 2, 3] + or observed["nice"] != 19 + or not str(observed["cgroup"]).endswith("/" + unit) + ): + raise IslandFailure("capacity kernel settings differ") + group = Path("/sys/fs/cgroup") / str(observed["cgroup"]).lstrip("/") + memory_events = os.open(group / "memory.events", os.O_RDONLY) + durable_write( + directory / "capacity-observed.json", canonicalize(observed) + ) + if time.time() >= deadline or cancel is not None and cancel.is_set(): + raise IslandFailure("capacity deadline/cancellation") + connection.sendall(b"G") + done = threading.Event() + condition = vars(cancel)["_cond"] if cancel is not None else None + + def cancelled() -> None: + assert cancel is not None and condition is not None + with condition: + condition.wait_for( + lambda: done.is_set() or cancel.is_set(), + timeout=max(0, deadline - time.time()), + ) + if not done.is_set(): + _manager( + "systemctl", + "kill", + "--kill-whom=all", + "--signal=SIGKILL", + unit, + check=False, + ) + + watcher = None + if cancel is not None: + watcher = threading.Thread(target=cancelled, name="capacity-cancel") + watcher.start() + try: + proc.wait(timeout=max(0.001, deadline - time.time()) + 5) + finally: + done.set() + if condition is not None: + with condition: + condition.notify_all() + if watcher is not None: + watcher.join() + status = _manager( + "systemctl", + "show", + unit, + "-p", + "Result", + "-p", + "ExecMainStatus", + "-p", + "MemoryPeak", + check=False, + ) + if ( + proc.returncode != 0 + or cancel is not None + and cancel.is_set() + or time.time() >= deadline + ): + raise IslandFailure("capacity execution failed: " + status) + except (OSError, subprocess.SubprocessError, ValueError, KeyError) as exc: + raise IslandFailure("capacity launch failed closed: " + str(exc)) from exc + finally: + if not status: + status = _manager( + "systemctl", "show", unit, "-p", "Result", "-p", "ExecMainStatus", check=False + ) + if memory_events is not None: + try: + os.lseek(memory_events, 0, os.SEEK_SET) + status += "\n" + os.read(memory_events, 8192).decode() + except OSError as exc: + status += "\nmemory events unavailable: " + str(exc) + finally: + os.close(memory_events) + counters_path = directory / "capacity-memory-events.txt" + if counters_path.exists(): + status += "\n" + counters_path.read_text() + durable_write( + directory / "capacity-result.json", + canonicalize({"observed": observed, "status": status}), + ) + for name in (unit, timer, timer_service): + _manager("systemctl", "stop", name, check=False) + _manager("systemctl", "reset-failed", unit, timer, timer_service, check=False) + if proc is not None and proc.poll() is None: + proc.wait(timeout=15) + for name in (unit, timer, timer_service): + result = _manager("systemctl", "show", name, "-p", "ActiveState", check=False) + if ( + "manager-unavailable" in result + or "ActiveState=active" in result + or "ActiveState=activating" in result + ): + raise IslandFailure("capacity cleanup incomplete: " + name) + group = Path("/sys/fs/cgroup") / str(observed.get("cgroup", "nonexistent")).lstrip("/") + if group.exists() and (group / "cgroup.procs").read_text().strip(): + raise IslandFailure("capacity descendants survived") + durable_write( + directory / "capacity-cleaned.json", + canonicalize({"units": [unit, timer, timer_service]}), + ) + + +def confined(directory: Path, relative: str) -> Path: + path = (directory / relative).resolve(strict=True) + if not path.is_relative_to(directory.resolve()) or not path.is_file(): + raise IslandFailure(f"object escapes attempt directory: {relative}") + return path + + +def launch_argv(job: IslandJobV1, attempt: Path, backend: Literal["cpu", "cuda"]) -> list[str]: + """No fixed logical-rank ceiling; parent interpreter is also the worker interpreter.""" + return [ + sys.executable, + "-m", + "torch.distributed.run", + "--standalone", + f"--nproc-per-node={job.manifest.training.reference_spec.layout.n_gpus}", + "--max-restarts=0", + "-m", + "hypertrain.miner.island_worker", + str(attempt), + backend, + ] + + +def validate_artifacts(job: IslandJobV1, directory: Path) -> IslandArtifacts: + """Recompute state, leaf, EF and byte commitments, not just rank-reported summaries.""" + import hypertrain.trainer # noqa: F401 + from hypertrain.auditor.replay import tensor_root, unpack_state + from hypertrain.trainer.compress import compress, state_hash + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.island import IslandAssignment + from hypertrain.trainer.loop import _make_leaf, batch_hash + from hypertrain.trainer.model import param_shapes + from hypertrain.trainer.optim import uses_muon + from hypertrain.trainer.rng import rng_ctr + + cfg = TrainConfig.from_manifest_v2(job.manifest) + n = job.manifest.training.reference_spec.layout.n_gpus + identity = hashlib.sha256(canonicalize(job.model_dump(mode="json"))).hexdigest() + baseline = None + backend = None + ranks = [] + for rank in range(n): + d = confined(directory, f"rank-{rank}/summary.json").parent + raw = load_json((d / "summary.json").read_bytes()) + if not isinstance(raw, dict) or raw.get("job_hash") != identity: + raise IslandFailure("rank job binding mismatch") + if raw.get("backend") not in ("cpu", "cuda"): + raise IslandFailure("rank backend missing") + if backend is not None and backend != raw["backend"]: + raise IslandFailure("rank backend disagreement") + backend = raw["backend"] + work = RankWorkResult.model_validate(raw["work"]) + if work.rank != rank: + raise IslandFailure("rank identity mismatch") + ranks.append(work) + th, st = unpack_state(confined(directory, f"rank-{rank}/state.safetensors").read_bytes()) + ef, ef_st = unpack_state(confined(directory, f"rank-{rank}/ef.safetensors").read_bytes()) + delta = confined(directory, f"rank-{rank}/delta.bin").read_bytes() + shapes = param_shapes(cfg.model) + if st is None or ef_st is not None or set(th) != set(shapes) or set(ef) != set(shapes): + raise IslandFailure("incomplete final state") + if any(tuple(th[k].shape) != s or tuple(ef[k].shape) != s for k, s in shapes.items()): + raise IslandFailure("final state shape mismatch") + adam = {k for k in th if not uses_muon(cfg.inner, k)} + expected_step = (job.global_step0 if cfg.inner.state_policy == "carry" else 0) + cfg.inner.H + if ( + set(st.m) != set(th) + or set(st.v) != adam + or st.step != expected_step + or st.step != raw["optimizer_step"] + ): + raise IslandFailure("incomplete optimizer state") + if any(tuple(x.shape) != shapes[k] for state in (st.m, st.v) for k, x in state.items()): + raise IslandFailure("optimizer state shape mismatch") + pres = [ + LeafPreimage.model_validate(x) + for x in json.loads(confined(directory, f"rank-{rank}/leaves.json").read_bytes()) + ] + if len(pres) != cfg.inner.n_leaves: + raise IslandFailure("leaf count mismatch") + a = IslandAssignment(job.run_id, job.w, tuple(job.sample_ids), job.global_step0, n) + for i, p in enumerate(pres): + if p.run_id != job.run_id or p.w != job.w or p.t != i * cfg.inner.J: + raise IslandFailure("leaf identity mismatch") + ids = ( + tuple(s for t in range(p.t - cfg.inner.J + 1, p.t + 1) for s in a.batch_ids(cfg, t)) + if p.t + else () + ) + if p.batch_ids_sha256 != batch_hash(ids) or p.rng_ctr != rng_ctr( + job.run_id, job.w, p.t, -1 + ): + raise IslandFailure("leaf assignment/rng mismatch") + chk, opt = unpack_state( + confined(directory, f"rank-{rank}/checkpoints/{p.t}.safetensors").read_bytes() + ) + from hypertrain.trainer.loop import stage_states + + expected_checkpoint_step = ( + job.global_step0 if cfg.inner.state_policy == "carry" else 0 + ) + p.t + if ( + opt is None + or opt.step != expected_checkpoint_step + or stage_states(cfg, chk, opt) != p.stages + ): + raise IslandFailure("checkpoint leaf state mismatch") + last = pres[-1] + from hypertrain.protocol.messages import f32val + + if ( + _make_leaf( + cfg, a, last.t, th, st, f32val(last.loss_f32), f32val(last.norm_f32) + ).preimage + != last + ): + raise IslandFailure("final leaf state mismatch") + actual = { + "leaves_root": MerkleTree([bytes.fromhex(p.digest()) for p in pres]).root.hex(), + "final_theta_hash": state_hash(th), + "state_root": tensor_root(th, st), + "ef_out_hash": state_hash(ef), + "delta_hash": hashlib.sha256(delta).hexdigest(), + "leaves": [p.digest() for p in pres], + } + start_blob = confined(directory, job.object_paths["start_state"]).read_bytes() + ef_blob = confined(directory, job.object_paths["ef_in"]).read_bytes() + v0_blob = confined(directory, job.object_paths["v0"]).read_bytes() + if ( + hashlib.sha256(start_blob).hexdigest() != job.start_state_sha256 + or hashlib.sha256(ef_blob).hexdigest() != job.ef_in_sha256 + or hashlib.sha256(v0_blob).hexdigest() != job.v0_sha256 + ): + raise IslandFailure("published input hash mismatch") + from hypertrain.trainer.optim import init_state + + theta0, carry0 = unpack_state(start_blob) + v0, _ = unpack_state(v0_blob) + initial = init_state( + cfg.inner, + theta0, + carry0 if cfg.inner.state_policy == "carry" else None, + v0 if cfg.inner.state_policy == "derived" else None, + ) + if _make_leaf(cfg, a, 0, theta0, initial, 0.0, 0.0).preimage != pres[0]: + raise IslandFailure("initial leaf differs from authenticated start") + ef0, _ = unpack_state(ef_blob) + payload, ef_expected = compress(cfg.compress, {k: theta0[k] - th[k] for k in th}, ef0) + if payload != delta or state_hash(ef_expected) != state_hash(ef): + raise IslandFailure("terminal compression/EF mismatch") + if raw["commitments"] != actual or baseline is not None and baseline != actual: + raise IslandFailure("rank artifact disagreement or corruption") + baseline = actual + d = directory / "rank-0" + return IslandArtifacts( + directory, + d / "state.safetensors", + d / "ef.safetensors", + d / "delta.bin", + d / "leaves.json", + tuple(ranks), + ) + + +def launch_island( + job: IslandJobV1, + directory: Path, + *, + backend: Literal["cpu", "cuda"] = "cuda", + cancel: threading.Event | None = None, + trace: bool = False, + capacity: CapacityAttempt | None = None, +) -> IslandArtifacts: + """Inputs in directory; deadline is Unix seconds. Restart returns the validated publication.""" + job = IslandJobV1.model_validate(job.model_dump(mode="json")) + directory = directory.resolve() + directory.mkdir(parents=True, exist_ok=True) + with flock(directory / "launch.lock"): + if time.time() >= job.deadline or cancel is not None and cancel.is_set(): + raise IslandFailure("job deadline/cancellation") + published = directory / "published" + if published.exists(): + summary = load_json(confined(published, "rank-0/summary.json").read_bytes()) + if not isinstance(summary, dict) or summary.get("backend") != backend: + raise IslandFailure("CPU publication cannot serve as CUDA oracle") + if trace: + for rank in range(job.manifest.training.reference_spec.layout.n_gpus): + if not (published / f"rank-{rank}/trace.json").is_file(): + raise IslandFailure("publication has no requested actual-path trace") + confined(published, f"rank-{rank}/trace.json") + artifacts = validate_artifacts(job, published) + if time.time() >= job.deadline or cancel is not None and cancel.is_set(): + raise IslandFailure("job expired during publication validation") + return artifacts + with tempfile.TemporaryDirectory(prefix="attempt-", dir=directory) as tmp: + attempt = Path(tmp) + for relative in job.object_paths.values(): + source = confined(directory, relative) + target = attempt / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, target) + durable_write(attempt / "job.json", canonicalize(job.model_dump(mode="json"))) + ref = job.manifest.training.reference_spec + env = dict( + os.environ, + CUBLAS_WORKSPACE_CONFIG=ref.env.CUBLAS_WORKSPACE_CONFIG, + OMP_NUM_THREADS="1", + CUDA_DISABLE_PTX_JIT=ref.env.CUDA_DISABLE_PTX_JIT, + HT_ISLAND_TRACE="1" if trace else "0", + ) + if capacity is not None: + run_capacity_argv( + launch_argv(job, attempt, backend), + directory / "capacity-runtime", + job.deadline, + capacity, + env=env, + cancel=cancel, + ) + # Parent-side tensor artifact validation remains unqualified for heavy32. + else: + _launch_unbounded(job, attempt, backend, env, cancel) + validate_artifacts(job, attempt) + if time.time() >= job.deadline or cancel is not None and cancel.is_set(): + raise IslandFailure("job expired before atomic publication") + os.rename(attempt, published) + fd = os.open(directory, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + return validate_artifacts(job, published) + + +def _launch_unbounded( + job: IslandJobV1, + attempt: Path, + backend: Literal["cpu", "cuda"], + env: dict[str, str], + cancel: threading.Event | None, +) -> None: + """Original ordinary launcher lifecycle; no new resource policy for legacy callers.""" + with ( + open(attempt / "stdout.log", "xb", buffering=0) as stdout, + open(attempt / "stderr.log", "xb+", buffering=0) as stderr, + ): + proc = subprocess.Popen( # noqa: S603 - fixed interpreter/package, typed layout + launch_argv(job, attempt, backend), + env=env, + stdout=stdout, + stderr=stderr, + start_new_session=True, + ) + done = threading.Event() + + def terminate() -> None: + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + return + + # Python 3.12 Event has no public wait-any subscription. Share its condition + # to wake on cancellation OR completion without altering the caller's event. + condition: threading.Condition | None = ( + vars(cancel)["_cond"] if cancel is not None else None + ) + + def cancelled() -> None: + assert cancel is not None and condition is not None + with condition: + condition.wait_for( + lambda: done.is_set() or cancel.is_set(), + timeout=max(0.0, job.deadline - time.time()), + ) + if not done.is_set(): + terminate() + + watcher = None + if cancel is not None: + watcher = threading.Thread(target=cancelled, name="island-cancellation") + watcher.start() + try: + proc.wait(timeout=max(0.001, job.deadline - time.time())) + except subprocess.TimeoutExpired as e: + terminate() + proc.wait() + raise IslandFailure("island deadline expired") from e + finally: + done.set() + if condition is not None: + with condition: + condition.notify_all() + if watcher is not None: + watcher.join() + terminate() + os.fsync(stdout.fileno()) + os.fsync(stderr.fileno()) + if proc.returncode != 0 or cancel is not None and cancel.is_set(): + stderr.seek(max(0, stderr.seek(0, os.SEEK_END) - 4000)) + tail = stderr.read(4000).decode(errors="replace") + raise IslandFailure(f"rank failure {proc.returncode}: {tail}") diff --git a/src/hypertrain/miner/island_worker.py b/src/hypertrain/miner/island_worker.py new file mode 100644 index 0000000..6eb209d --- /dev/null +++ b/src/hypertrain/miner/island_worker.py @@ -0,0 +1,216 @@ +"""Torchrun worker. Import determinism before torch; no pickle or implicit CPU CUDA oracle.""" + +from __future__ import annotations + +import hypertrain.trainer # noqa: F401 + +# isort: split +import hashlib +import json +import os +import shutil +import subprocess +import sys +import time +from datetime import timedelta +from pathlib import Path + +import numpy as np +import numpy.typing as npt +import torch +import torch.distributed as dist + +from hypertrain.auditor.replay import pack_state, tensor_root, unpack_state +from hypertrain.gpu_ops.journal import durable_write +from hypertrain.miner.island_launch import IslandFailure, confined +from hypertrain.protocol.envelope_v2 import load_json +from hypertrain.protocol.hashing import MerkleTree +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.messages_v2 import IslandJobV1, RankWorkResult +from hypertrain.trainer.compress import state_hash +from hypertrain.trainer.determinism import apply_reference_env +from hypertrain.trainer.island import DistComm, TraceContext, island_from_manifest, train_island +from hypertrain.trainer.loop import Assignment +from hypertrain.trainer.optim import OptState + + +def execute(directory: Path, backend: str) -> None: + job = IslandJobV1.model_validate(load_json((directory / "job.json").read_bytes())) + cfg, lay, run_id = island_from_manifest(job.manifest.body()) + local_rank = int(os.environ["LOCAL_RANK"]) + match backend: + case "cuda": + if not torch.cuda.is_available() or torch.cuda.device_count() != lay.n_gpus: + raise IslandFailure("exact CUDA device count unavailable") + torch.cuda.set_device(local_rank) + ref = job.manifest.training.reference_spec + if torch.__version__ != "2.14.0+cu130": + raise IslandFailure("unqualified CUDA torch version") + if os.environ.get("HT_IMAGE_DIGEST") != ref.image_digest: + raise IslandFailure("runtime image differs from manifest") + smi = shutil.which("nvidia-smi") + if smi is None: + raise IslandFailure("nvidia-smi unavailable") + driver = subprocess.run( # noqa: S603 - PATH-resolved nvidia-smi, fixed query + [smi, "--query-gpu=driver_version", "--format=csv,noheader"], + check=True, + capture_output=True, + text=True, + timeout=10, + ).stdout.splitlines() + if len(driver) != lay.n_gpus or driver[local_rank].strip() not in ref.driver_allowlist: + raise IslandFailure("unqualified CUDA driver") + if torch.cuda.get_device_properties(local_rank).multi_processor_count != ref.sm_count: + raise IslandFailure("unqualified CUDA SM profile") + device = torch.device("cuda", local_rank) + torch.set_default_device(device) + torch.cuda.reset_peak_memory_stats(device) + engine = "nccl" + case "cpu": + device = torch.device("cpu") + engine = "gloo" + case _: + raise IslandFailure("unsupported execution backend") + apply_reference_env(job.manifest.training.reference_spec.env.model_dump()) + dist.init_process_group(engine, timeout=timedelta(minutes=5)) + try: + + def blob(name: str, expected: str) -> bytes: + raw = confined(directory, job.object_paths[name]).read_bytes() + if hashlib.sha256(raw).hexdigest() != expected: + raise IslandFailure(f"input hash mismatch: {name}") + return raw + + theta, carry = unpack_state(blob("start_state", job.start_state_sha256)) + ef, _ = unpack_state(blob("ef_in", job.ef_in_sha256)) + v0, _ = unpack_state(blob("v0", job.v0_sha256)) + + def move(p: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: + return {n: x.to(device) for n, x in p.items()} + + theta, ef, v0 = move(theta), move(ef), move(v0) + if cfg.inner.state_policy != "carry": + if carry is not None: + raise IslandFailure("unexpected carried optimizer") + elif carry is None: + raise IslandFailure("missing carried optimizer") + else: + carry.m, carry.v = move(carry.m), move(carry.v) + raw = confined(directory, job.object_paths["samples"]).read_bytes() + proofs = json.loads(confined(directory, job.object_paths["sample_proofs"]).read_bytes()) + dataset = job.manifest.training.dataset + width = cfg.model.seq_len + 1 + dtype = " npt.NDArray[np.uint32]: + return samples[i] + + started = time.monotonic_ns() + out = directory / f"rank-{dist.get_rank()}" + out.mkdir() + (out / "checkpoints").mkdir() + + def checkpoint(t: int, theta: dict[str, torch.Tensor], state: OptState) -> None: + durable_write(out / "checkpoints" / f"{t}.safetensors", pack_state(theta, state)) + + traces = [] + + def trace(ctx: TraceContext, x: torch.Tensor) -> torch.Tensor: + raw = x.detach().cpu().contiguous().reshape(-1).view(torch.uint8).numpy().tobytes() + traces.append( + { + "rank": ctx.rank, + "step": ctx.step, + "microbatch": ctx.microbatch, + "layer": ctx.layer, + "op": ctx.op, + "shape": list(x.shape), + "dtype": str(x.dtype), + "sha256": hashlib.sha256(raw).hexdigest(), + } + ) + return x + + def deadline() -> None: + if time.time() >= job.deadline: + raise IslandFailure("job deadline expired") + + result = train_island( + cfg, + lay, + DistComm(), + theta, + Assignment(run_id, job.w, tuple(job.sample_ids), job.global_step0), + get, + ef_in=ef, + carry=carry, + v0=v0 if cfg.inner.state_policy == "derived" else None, + hook=trace if os.environ.get("HT_ISLAND_TRACE") == "1" else None, + cancel=deadline, + checkpoint=checkpoint, + ) + allocated = reserved = 0 + total = 1 + if backend == "cuda": + torch.cuda.synchronize(device) + allocated = torch.cuda.max_memory_allocated(device) + reserved = torch.cuda.max_memory_reserved(device) + total = torch.cuda.get_device_properties(device).total_memory + work = RankWorkResult( + rank=dist.get_rank(), + fingerprint=hashlib.sha256( + f"{backend}:{local_rank}:{torch.__version__}".encode() + ).hexdigest(), + elapsed_ns=time.monotonic_ns() - started, + allocated_bytes=allocated, + reserved_bytes=reserved, + total_bytes=total, + ) + if os.environ.get("HT_ISLAND_TRACE") == "1": + durable_write(out / "trace.json", canonicalize(traces)) + durable_write(out / "state.safetensors", pack_state(result.final_theta, result.final_state)) + durable_write(out / "ef.safetensors", pack_state(result.ef_out)) + durable_write(out / "delta.bin", result.delta_payload) + durable_write( + out / "leaves.json", + canonicalize([x.preimage.model_dump(mode="json") for x in result.leaves]), + ) + durable_write( + out / "summary.json", + canonicalize( + { + "job_hash": hashlib.sha256( + canonicalize(job.model_dump(mode="json")) + ).hexdigest(), + "backend": backend, + "work": work.model_dump(mode="json"), + "optimizer_step": result.final_state.step, + "commitments": { + "leaves_root": result.leaves_root, + "final_theta_hash": result.final_theta_hash, + "state_root": tensor_root(result.final_theta, result.final_state), + "ef_out_hash": state_hash(result.ef_out), + "delta_hash": result.delta_hash, + "leaves": [x.digest.hex() for x in result.leaves], + }, + } + ), + ) + finally: + dist.destroy_process_group() + + +if __name__ == "__main__": + execute(Path(sys.argv[1]), sys.argv[2]) diff --git a/src/hypertrain/models/opendecision/__init__.py b/src/hypertrain/models/opendecision/__init__.py index 2bf0069..10ca07b 100644 --- a/src/hypertrain/models/opendecision/__init__.py +++ b/src/hypertrain/models/opendecision/__init__.py @@ -11,6 +11,7 @@ # isort: split import math +import threading from collections.abc import Callable, Sequence from functools import lru_cache from typing import Any @@ -22,6 +23,7 @@ from torch.nn.attention import SDPBackend, sdpa_kernel from hypertrain.protocol.messages import ODSpec, RunManifest, f32val +from hypertrain.protocol.messages_v2 import RunManifestV2 from hypertrain.trainer.config import ModelConfig, TrainConfig from hypertrain.trainer.loop import SampleFn from hypertrain.trainer.model import Params, embed, permute_rows @@ -41,6 +43,7 @@ Hook = Callable[[int, str, Tensor], Tensor] PROFILE_DTYPE = {"od-bf16-det-eager-v1": "bf16", "od-fp32-ref-v1": "fp32"} _NORM_WEIGHTS = (".qn.weight", ".kn.weight", ".n1.weight", ".n2.weight", ".n3.weight") +_SDPA_LOCK = threading.RLock() def _identity(layer: int, op: str, x: Tensor) -> Tensor: @@ -69,12 +72,18 @@ def od_config(cfg: ModelConfig) -> Any: ) -def check_manifest(m: RunManifest) -> None: +def check_manifest(m: RunManifest | RunManifestV2) -> None: """Adapter-side rules on top of the L2 RunManifest validator (which already ran on ``m``).""" - ms, od = m.model, m.model.od + match m: + case RunManifestV2(): + cfg = TrainConfig.from_manifest_v2(m).model + ms = m.training.model + case RunManifest(): + cfg = TrainConfig.from_manifest(m.body()).model + ms = m.model + od = ms.od if ms.arch != "od-encoder" or od is None: raise ValueError("not an OpenDecision manifest") - cfg = TrainConfig.from_manifest(m.body()).model want = sum(math.prod(s) for s in param_shapes(cfg).values()) if ms.param_count != want: raise ValueError(f"param_count {ms.param_count} != {want} trainable for {od.objective}") @@ -141,9 +150,9 @@ def take_rows(flat: Tensor, rows: Tensor) -> Tensor: backward is ``_Permute`` (index_select), never index_put/scatter_add (review defect 7).""" n = flat.shape[0] r = rows.detach().to("cpu", torch.int64).reshape(-1) - rest = torch.ones(n, dtype=torch.bool) + rest = torch.ones(n, dtype=torch.bool, device="cpu") rest[r] = False - perm = torch.cat([r, torch.arange(n)[rest]]) + perm = torch.cat([r, torch.arange(n, device="cpu")[rest]]) if perm.numel() != n: raise ValueError("rows must be distinct") return permute_rows(flat, perm.to(flat.device))[: r.numel()] @@ -158,7 +167,8 @@ def _det_gather(h: Tensor, positions: Tensor) -> Tensor: @lru_cache(maxsize=16) def _rope_cpu(n: int, head_dim: int, theta: float) -> tuple[Tensor, Tensor]: - cos, sin = rope_table(n, head_dim, theta) + with torch.device("cpu"): + cos, sin = rope_table(n, head_dim, theta) return cos, sin @@ -170,7 +180,7 @@ def _blocks(m: ModelConfig, p: Params, x: Tensor, mask: Tensor | None, hook: Hoo """Op-for-op copy of opendecision ``Block.forward`` over the pinned names, with op hooks.""" b, n, d = x.shape h, hd = m.n_heads, m.d_model // m.n_heads - cos, sin = _rope_cpu(n, hd, float(m.rope_theta)) + cos, sin = (v.to(x.device) for v in _rope_cpu(n, hd, float(m.rope_theta))) am = None if mask is None else mask[:, None, None, :] for i in range(m.n_layers): pre = f"encoder.blocks.{i}" @@ -217,10 +227,19 @@ def forward(self, *args: Tensor) -> Any: return self.m.decide(*args) -@lru_cache(maxsize=8) +_DECIDERS = threading.local() + + def _decider(cfg: ModelConfig) -> _Decide: + # functional_call temporarily swaps module tensors: never share its module across ranks. + if not hasattr(_DECIDERS, "models"): + _DECIDERS.models = {} + if cfg in _DECIDERS.models: + return _DECIDERS.models[cfg] with torch.device("meta"): - return _Decide(OpenDecisionModel(od_config(cfg), embed_fn=embed)) + module = _Decide(OpenDecisionModel(od_config(cfg), embed_fn=embed)) + _DECIDERS.models[cfg] = module + return module def _decision(m: ModelConfig, od: ODSpec, p: Params, tokens: Tensor, hook: Hook) -> Tensor: @@ -232,7 +251,10 @@ def _decision(m: ModelConfig, od: ODSpec, p: Params, tokens: Tensor, hook: Hook) z = hook(L, "final_norm", _ln(x, p, "encoder.norm")) args = (z, d["mask"], d["opt_ids"], d["opt_mask"], d["instr_ids"], d["qtype"]) cos, sin = _rope_cpu(PRESETS[od.preset].max_len, m.d_model // m.n_heads, float(m.rope_theta)) - tensors = {f"m.{k}": v for k, v in p.items()} | {"m.rope_cos": cos, "m.rope_sin": sin} + tensors = {f"m.{k}": v for k, v in p.items()} | { + "m.rope_cos": cos.to(tokens.device), + "m.rope_sin": sin.to(tokens.device), + } logits, ext = torch.func.functional_call(_decider(m), tensors, args) logits = hook(L, "head", logits) if od.objective == "distill": @@ -257,6 +279,7 @@ def _run(m: ModelConfig, p: Params, tokens: Tensor, hook: Hook) -> tuple[Tensor, # FLASH only on the unmasked stage A path of the bf16 profile; masked B/C and fp32-ref: MATH. backend = SDPBackend.FLASH_ATTENTION if mlm and dt == "bf16" else SDPBackend.MATH with ( + _SDPA_LOCK, sdpa_kernel([backend]), torch.autocast(tokens.device.type, dtype=torch.bfloat16, enabled=dt == "bf16"), ): @@ -265,12 +288,16 @@ def _run(m: ModelConfig, p: Params, tokens: Tensor, hook: Hook) -> tuple[Tensor, def forward( - cfg: ModelConfig, params: Params, tokens: Tensor, moe: object | None = None + cfg: ModelConfig, + params: Params, + tokens: Tensor, + moe: object | None = None, + hook: Hook | None = None, ) -> tuple[Tensor, Tensor]: """Same signature as ``trainer.model.forward``; OD is dense, so ``moe`` must be None.""" if moe is not None: raise ValueError("OpenDecision is dense: moe must be None") - return _run(cfg, params, tokens, _identity) + return _run(cfg, params, tokens, hook or _identity) def traced_forward( diff --git a/src/hypertrain/protocol/envelope_v2.py b/src/hypertrain/protocol/envelope_v2.py new file mode 100644 index 0000000..0a201de --- /dev/null +++ b/src/hypertrain/protocol/envelope_v2.py @@ -0,0 +1,394 @@ +"""Bounded canonical v2 signing and intake; durable replay receipts belong to L0 storage.""" + +from __future__ import annotations + +import json +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, JsonValue, ValidationError + +from hypertrain.protocol.envelope import ( + ExpiredError, + MalformedEnvelope, + ReplayError, + RunMismatchError, + SignatureError, + UnauthorizedSigner, + body_digest, +) +from hypertrain.protocol.jcs import CanonicalizationError, canonicalize +from hypertrain.protocol.keys import KeyError_, Keypair, decode_hotkey, verify +from hypertrain.protocol.messages import SS58, Hex64 +from hypertrain.protocol.messages_v2 import ( + MAX_BODY_BYTES, + MAX_MANIFEST_BYTES, + MESSAGE_TYPES_V2, + AuditJobV2, + Identifier, + JoinRequest, + Positive, + RunManifestV2, +) + +VERSION = "ht/2" +DOMAIN = b"hypertrain/2|" +MAX_DEPTH = 32 +MAX_ARRAY = 1_000_000 +SIGNER_BOUND_TYPES = {"AcceptV2", "CommitV2", "DeltaManifestV2", "StateServe", "DisputeV2"} + + +class SignedEnvelope(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + v: str + type: Identifier + run_id: Hex64 + body: dict[str, JsonValue] + signer: SS58 + exp_drand: Positive + sig: str = Field(pattern=r"^[0-9a-f]{128}$") + + +class EnvelopeV2(SignedEnvelope): + v: Literal["ht/2"] + + +def _pairs(pairs: list[tuple[str, JsonValue]]) -> dict[str, JsonValue]: + result: dict[str, JsonValue] = {} + for key, value in pairs: + if key in result: + raise MalformedEnvelope("duplicate JSON field") + result[key] = value + return result + + +def _reject_number(value: str) -> JsonValue: + raise MalformedEnvelope(f"raw float/nonfinite JSON number: {value}") + + +def _bounded(value: JsonValue, depth: int = 0) -> None: + if depth > MAX_DEPTH: + raise MalformedEnvelope("message nesting exceeds 32") + match value: + case dict(): + if len(value) > 4096: + raise MalformedEnvelope("object exceeds field bound") + for key, child in value.items(): + if len(key) > 4096: + raise MalformedEnvelope("field name exceeds bound") + _bounded(child, depth + 1) + case list(): + if len(value) > MAX_ARRAY: + raise MalformedEnvelope("array exceeds bound") + for child in value: + _bounded(child, depth + 1) + case str(): + if len(value) > 4096: + raise MalformedEnvelope("string exceeds bound") + case float(): + raise MalformedEnvelope("raw floats forbidden; use f32 hex") + case int(): + if not isinstance(value, bool) and not 0 <= value <= 2**53 - 1: + raise MalformedEnvelope("integer exceeds canonical unsigned bound") + case None: + return + + +def load_json(raw: bytes | str, *, max_bytes: int = MAX_MANIFEST_BYTES) -> dict[str, JsonValue]: + """Reject duplicates and floats before schema dispatch, with bounded parser depth.""" + data = raw.encode("utf-8") if isinstance(raw, str) else raw + if len(data) > max_bytes: + raise MalformedEnvelope("message exceeds byte bound") + # Bound nesting before json.loads can recurse; ignore brackets inside strings. + depth, quoted, escaped = 0, False, False + for byte in data: + if quoted: + if escaped: + escaped = False + elif byte == 92: + escaped = True + elif byte == 34: + quoted = False + elif byte == 34: + quoted = True + elif byte in (91, 123): + depth += 1 + if depth > MAX_DEPTH: + raise MalformedEnvelope("message nesting exceeds 32") + elif byte in (93, 125): + depth -= 1 + try: + value: JsonValue = json.loads( + data, + object_pairs_hook=_pairs, + parse_float=_reject_number, + parse_constant=_reject_number, + ) + except (json.JSONDecodeError, UnicodeDecodeError) as error: + raise MalformedEnvelope("invalid UTF-8 JSON") from error + if not isinstance(value, dict): + raise MalformedEnvelope("message must be an object") + _bounded(value) + return value + + +def _validate_body( + msg_type: str, + data: dict[str, JsonValue], + models: Mapping[str, type[BaseModel]], +) -> BaseModel: + model_type = models.get(msg_type) + if model_type is None: + raise MalformedEnvelope("unsupported message type") + _bounded(data) + limit = ( + MAX_MANIFEST_BYTES + if msg_type in {"RunManifestV2", "DeltaManifestV2", "UploadChunkManifest", "AuditJobV2"} + else MAX_BODY_BYTES + ) + if len(canonicalize(data, allow_float=False)) > limit: + raise MalformedEnvelope("body exceeds message byte bound") + try: + model = model_type.model_validate(data) + except ValidationError as error: + raise MalformedEnvelope(f"invalid {msg_type} body") from error + if canonicalize(model.model_dump(mode="json"), allow_float=False) != canonicalize( + data, allow_float=False + ): + raise MalformedEnvelope("body differs from canonical validated model") + return model + + +def _signing_message( + domain: bytes, + msg_type: str, + body: Mapping[str, JsonValue], + exp_drand: int, + run_id: str, +) -> bytes: + if type(exp_drand) is not int or not 1 <= exp_drand <= 2**53 - 1: + raise MalformedEnvelope("expiration must be a strict canonical positive integer") + return ( + domain + + msg_type.encode("ascii") + + b"|" + + body_digest(body).encode("ascii") + + b"|" + + str(exp_drand).encode("ascii") + + b"|" + + run_id.encode("ascii") + ) + + +def signing_message( + msg_type: str, + body: Mapping[str, JsonValue], + exp_drand: int, + run_id: str, +) -> bytes: + return _signing_message(DOMAIN, msg_type, body, exp_drand, run_id) + + +def _seal( + keypair: Keypair, + msg_type: str, + run_id: str, + body: BaseModel | dict[str, JsonValue], + exp: int, + *, + version: str, + domain: bytes, + models: Mapping[str, type[BaseModel]], +) -> dict[str, JsonValue]: + data = body.model_dump(mode="json") if isinstance(body, BaseModel) else body + _validate_body(msg_type, data, models) + env: dict[str, JsonValue] = { + "v": version, + "type": msg_type, + "run_id": run_id, + "body": data, + "signer": keypair.ss58, + "exp_drand": exp, + "sig": keypair.sign(_signing_message(domain, msg_type, data, exp, run_id)).hex(), + } + return env + + +def seal( + keypair: Keypair, + msg_type: str, + run_id: str, + body: BaseModel | dict[str, JsonValue], + exp: int, +) -> dict[str, JsonValue]: + env = _seal( + keypair, + msg_type, + run_id, + body, + exp, + version=VERSION, + domain=DOMAIN, + models=MESSAGE_TYPES_V2, + ) + _parse(env, EnvelopeV2, MESSAGE_TYPES_V2) + return env + + +def _parse( + raw: bytes | str | dict[str, JsonValue], + envelope_type: type[SignedEnvelope], + models: Mapping[str, type[BaseModel]], +) -> tuple[SignedEnvelope, BaseModel]: + data = ( + load_json(raw, max_bytes=MAX_MANIFEST_BYTES + 1024) if isinstance(raw, bytes | str) else raw + ) + _bounded(data) + if len(canonicalize(data, allow_float=False)) > MAX_MANIFEST_BYTES + 1024: + raise MalformedEnvelope("envelope exceeds bound") + try: + env = envelope_type.model_validate(data) + except ValidationError as error: + raise MalformedEnvelope("invalid versioned envelope") from error + return env, _validate_body(env.type, env.body, models) + + +def parse_envelope(raw: bytes | str | dict[str, JsonValue]) -> EnvelopeV2: + env, _ = _parse(raw, EnvelopeV2, MESSAGE_TYPES_V2) + assert isinstance(env, EnvelopeV2) + return env + + +def _verify(env: SignedEnvelope, domain: bytes) -> bool: + return verify( + decode_hotkey(env.signer), + _signing_message(domain, env.type, env.body, env.exp_drand, env.run_id), + bytes.fromhex(env.sig), + ) + + +def verify_envelope(raw: bytes | str | dict[str, JsonValue]) -> bool: + try: + return _verify(parse_envelope(raw), DOMAIN) + except (MalformedEnvelope, KeyError_, CanonicalizationError): + return False + + +def replay_key(msg_type: str, run_id: str, signer: str, model: BaseModel) -> tuple[str, ...]: + """Semantic reservation ID, independent of signature randomness and expiration.""" + fields: dict[str, tuple[str, ...]] = { + "DisputeV2": ("verdict_hash",), + "BisectV2": ("dispute_id", "seq", "party"), + "ResolutionV2": ("dispute_id",), + "JoinRequest": ("coldkey", "request_id"), + "WorkProof": ("admission_id", "challenge_hash"), + "WorkScreenV2": ("admission_id", "nonce"), + "JoinChallenge": ("admission_id", "nonce"), + "StateServe": ("challenge_hash", "t"), + "EscrowLock": ("operation_id",), + "EscrowRelease": ("operation_id",), + "EscrowTransfer": ("operation_id",), + "EscrowReceipt": ("operation_id",), + "RotateRequest": ("operation_id",), + "TestGenesis": ("run_id",), + "RewardFinalize": ("w",), + "AuditJobV2": ("job_id", "attempt"), + "AuditChallengeV2": ("w", "target"), + "ReplayVerdict": ("challenge_hash",), + "Receipt": ("w", "commit_hash"), + "RelayAssignment": ("w", "hotkey", "assignment_epoch"), + "UploadGrant": ("w", "hotkey", "assignment_epoch"), + "AcceptedUploadAck": ("grant_hash",), + "ChunkCustodyAck": ("grant_hash", "index"), + "RelayReceipt": ("grant_hash",), + "MasterAcceptanceV2": ("w", "hotkey", "delta_hash"), + "RetentionExtension": ("grant_hash", "seq"), + "RetentionExtensionAck": ("extension_hash",), + "CustodyRelease": ("grant_hash",), + "RetrievalRequest": ("request_id",), + "RetrievalResponse": ("request_hash",), + "RelayFailureEvidence": ("evidence_hash",), + "RelayRegistryV1": ("epoch",), + } + names = fields.get(msg_type) + if names is None: + names = tuple(name for name in ("w", "hotkey", "target") if hasattr(model, name)) + subject = tuple(str(getattr(model, name)) for name in names) if names else ("immutable",) + return (run_id, msg_type, signer, *subject) + + +@dataclass +class Intake: + """Mutable in-memory reservations; caller restores/persists them transactionally.""" + + run_id: str + allowed_signers: Mapping[str, Callable[[str], bool]] = field(default_factory=dict) + reservations: dict[tuple[str, ...], str] = field(default_factory=dict) + + def accept(self, raw: bytes | str | dict[str, JsonValue], now_round: int) -> BaseModel: + return self._accept(raw, now_round, EnvelopeV2, MESSAGE_TYPES_V2, DOMAIN) + + def _accept( + self, + raw: bytes | str | dict[str, JsonValue], + now_round: int, + envelope_type: type[SignedEnvelope], + models: Mapping[str, type[BaseModel]], + domain: bytes, + ) -> BaseModel: + env, model = _parse(raw, envelope_type, models) + if env.run_id != self.run_id: + raise RunMismatchError("envelope belongs to another run") + if type(now_round) is not int or not 1 <= now_round <= 2**53 - 1: + raise MalformedEnvelope("now_round must be strict positive beacon round") + if not _verify(env, domain): + raise SignatureError("sr25519 signature does not verify") + if now_round > env.exp_drand: + raise ExpiredError("envelope expired") + check = self.allowed_signers.get(env.type) + if check is not None and not check(env.signer): + raise UnauthorizedSigner("signer is not authorized for message role") + model_body = model.model_dump(mode="json") + if env.type in SIGNER_BOUND_TYPES and model_body["hotkey"] != env.signer: + raise UnauthorizedSigner("hotkey differs from signer") + if env.type == "BisectV2" and model_body["party"] != env.signer: + raise UnauthorizedSigner("bisection party differs from signer") + body_run = model_body.get("run_id", env.run_id) + if body_run != env.run_id: + raise RunMismatchError("embedded run ID differs from envelope") + if isinstance(model, RunManifestV2) and model.run_id() != env.run_id: + raise RunMismatchError("manifest wrapper hash differs from envelope") + if isinstance(model, AuditJobV2): + model.validate_embedded(now_round) + key = replay_key(env.type, env.run_id, env.signer, model) + digest = body_digest(env.body) + existing = self.reservations.get(key) + if existing is not None and existing != digest: + raise ReplayError("conflicting body for reserved replay ID") + self.reservations[key] = digest + return model + + +def join_signing_message(request: JoinRequest) -> bytes: + data = request.model_dump(mode="json", exclude={"hot_sig", "cold_sig"}) + return _signing_message( + b"hypertrain/join/2|", "JoinRequest", data, request.expires_beacon, request.run_id + ) + + +def verify_join(request: JoinRequest, run_id: str, now_round: int) -> bool: + if request.run_id != run_id or not 1 <= now_round <= request.expires_beacon: + return False + try: + msg = join_signing_message(request) + return verify( + decode_hotkey(request.hotkey), msg, bytes.fromhex(request.hot_sig) + ) and verify(decode_hotkey(request.coldkey), msg, bytes.fromhex(request.cold_sig)) + except (KeyError_, CanonicalizationError): + return False + + +def tape_signing_message(body: Mapping[str, JsonValue], run_id: str) -> bytes: + """L3 owns tape schemas/arithmetic; this is the exact approved signing preimage.""" + return b"hypertrain/tape/2|" + body_digest(body).encode("ascii") + b"|" + run_id.encode("ascii") diff --git a/src/hypertrain/protocol/messages_v2.py b/src/hypertrain/protocol/messages_v2.py new file mode 100644 index 0000000..f6758df --- /dev/null +++ b/src/hypertrain/protocol/messages_v2.py @@ -0,0 +1,917 @@ +"""Network-v2 bodies. Unchanged training field definitions remain owned by v1.""" + +from __future__ import annotations + +import math +from typing import Annotated, Literal, get_args, get_origin + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + JsonValue, + StrictBool, + StrictInt, + StringConstraints, + model_validator, +) + +from hypertrain.protocol.hashing import sha256_hex +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.messages import ( + F32, + OD_PRESETS, + SS58, + Accept, + AuditChallenge, + Chunk, + Commit, + DeltaManifest, + Finalize, + Hex64, + ImageDigest, + Layout, + LeafPreimage, + Receipt, + ReplayVerdict, + Rollback, + RunManifest, + StateServe, + f32val, +) + +# I-JSON is narrower than u64: reuse the frozen canonicalizer's exact integer ceiling. +U64 = Annotated[StrictInt, Field(ge=0, le=2**53 - 1)] +Positive = Annotated[StrictInt, Field(ge=1, le=2**53 - 1)] +PPM = Annotated[StrictInt, Field(ge=0, le=1_000_000)] +Text = Annotated[str, StringConstraints(min_length=1, max_length=4096)] +Identifier = Annotated[str, StringConstraints(pattern=r"^[A-Za-z0-9_.:-]{1,128}$")] +Hashes = Annotated[list[Hex64], Field(max_length=512)] +AdmissionState = Literal["APPLIED", "PROBATION", "ACTIVE", "SUSPENDED", "BANNED"] +WEIGHT_QUANTUM = 1 << 24 +MAX_OBJECT_BYTES = 2 << 30 +MAX_BODY_BYTES = 64 << 10 +MAX_MANIFEST_BYTES = 1 << 20 + + +class WireModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + @model_validator(mode="before") + @classmethod + def _strict_literals(cls, data: JsonValue) -> JsonValue: + if isinstance(data, dict): + for name, info in cls.model_fields.items(): + args = get_args(info.annotation) + if ( + get_origin(info.annotation) is Literal + and args + and type(args[0]) is int + and name in data + and type(data[name]) is not int + ): + raise ValueError("integer literal fields require strict integers") + return data + + def body(self) -> dict[str, JsonValue]: + return self.model_dump(mode="json") + + def digest(self) -> str: + return sha256_hex(canonicalize(self.body(), allow_float=False)) + + +class TrainingManifestV2(RunManifest): + """Replace only the version-dependent validator; never invoke the v1 OD layout gate.""" + + @model_validator(mode="after") + def _assign_unit_and_od(self) -> TrainingManifestV2: + ds, inn, ms, ref = self.dataset, self.inner, self.model, self.reference_spec + lay = ref.layout + if lay.pp != 1: + raise ValueError("v2 supports pp1 only") + if ds.assign_unit is not None: + if self.batch_samples() % ds.assign_unit or ds.n_samples % ds.assign_unit: + raise ValueError("assignment unit must divide island batch and dataset") + if ds.unit_sha256_root is None: + raise ValueError("assign_unit requires unit_sha256_root") + elif ds.unit_sha256_root is not None: + raise ValueError("unit_sha256_root requires assign_unit") + if self.batch_samples() > ds.n_samples: + raise ValueError("dataset cannot supply one island assignment") + if ms.d_model % ms.n_heads or (ms.d_model // ms.n_heads) % 2: + raise ValueError("model must split into even-sized attention heads") + if ms.n_heads % ms.n_kv_heads or ms.top_k_experts > ms.n_experts: + raise ValueError("invalid attention or expert geometry") + if ms.n_experts == 1 and lay.ep_size != 1: + raise ValueError("dense models require EP1") + if ms.n_experts % lay.ep_size: + raise ValueError("EP must divide expert count") + match ms.arch: + case "decoder": + if ref.profile is not None: + raise ValueError("profile is only valid for OD") + d, kv = ms.d_model, ms.n_kv_heads * (ms.d_model // ms.n_heads) + ff = 3 * d * ms.d_ff * ms.n_experts + router = d * ms.n_experts if ms.n_experts > 1 else 0 + count = ( + 2 * ms.vocab * d + + d + + ms.n_layers * (2 * d + 2 * d * d + 2 * d * kv + ff + router) + ) + case "od-encoder": + od = ms.od + assert od is not None # ModelSpec already enforces arch iff OD. + if lay.ep_size != 1 or lay.dp_size != lay.n_gpus: + raise ValueError("OD requires dense DP=N/EP1") + if ms.n_kv_heads != ms.n_heads or ms.d_ff != 4 * ms.d_model: + raise ValueError("OD attention/FF geometry mismatch") + if ms.n_experts != 1 or ms.top_k_experts != 1: + raise ValueError("OD requires one expert") + if f32val(ms.capacity_factor) != 1 or f32val(ms.aux_loss_coef) != 0: + raise ValueError("OD requires capacity=1 and aux_loss=0") + if ref.profile is None: + raise ValueError("OD requires a reference profile") + dtype = "fp32" if ref.profile == "od-fp32-ref-v1" else "bf16" + if ms.compute_dtype != dtype: + raise ValueError("OD dtype/profile mismatch") + d, layers, heads, head_layers, vocab = OD_PRESETS[od.preset] + if (ms.d_model, ms.n_layers, ms.n_heads, ms.vocab, od.head_layers) != ( + d, + layers, + heads, + vocab, + head_layers, + ): + raise ValueError("OD dimensions differ from pinned preset") + if (od.record is None) != (od.objective == "mlm"): + raise ValueError("OD record required iff objective is not MLM") + if od.record is not None and ms.seq_len != od.record.record_len() - 1: + raise ValueError("OD record length must equal seq_len+1") + if od.objective == "mlm" and ( + (od.preset == "od-tiny" and ms.seq_len > 64) + or (od.preset != "od-tiny" and ms.seq_len != 1024) + ): + raise ValueError("OD MLM sequence length differs from pinned preset") + if ds.sample_format != "u16[seq_len+1] token ids": + raise ValueError("OD requires u16 records") + if not all( + math.isfinite(f32val(v)) + for v in (od.mask_ratio, od.distill_temp, od.rps_weight) + ): + raise ValueError("nonfinite OD objective quantity") + if not 0 < f32val(od.mask_ratio) <= 1 or f32val(od.distill_temp) <= 0: + raise ValueError("OD mask ratio/temperature must be positive") + if f32val(od.rps_weight) < 0: + raise ValueError("OD RPS weight must be nonnegative") + # Shape-only count of the pinned flat OD state_dict (no tensors allocated). + count = vocab * d + layers * (12 * d * d + 9 * d + 4 * (d // heads)) + 2 * d + if od.objective != "mlm": + count += 4 * d + head_layers * (16 * d * d + 11 * d) + count += d * d + 4 * d + 4 + case unreachable: + raise ValueError(f"unsupported architecture {unreachable}") + if ms.param_count != count: + raise ValueError(f"param_count {ms.param_count} != shape count {count}") + if len(self.auditors) > 128 or len(set(self.auditors)) != len(self.auditors): + raise ValueError("auditor list must be bounded and unique") + if not all( + math.isfinite(f32val(value)) + for value in ( + ms.capacity_factor, + ms.aux_loss_coef, + ms.init_std, + inn.eps, + inn.wd, + inn.grad_clip, + inn.muon_momentum, + *inn.betas, + inn.lr_schedule.peak_lr, + self.outer.lr, + self.outer.momentum, + self.outer.topk_frac, + self.outer.ef_beta, + self.outer.preclip_norm, + self.outer.cclip_tau, + self.verify.q_base, + self.verify.influence_cap, + self.operator_budget.honeypot.rate, + ) + ): + raise ValueError("nonfinite training quantity") + return self + + def batch_samples(self) -> int: + return ( + self.inner.H + * self.inner.micro_batch + * self.inner.grad_accum + * self.reference_spec.layout.n_gpus + ) + + def run_id(self) -> str: + raise ValueError("training hash is not a run ID; use RunManifestV2.run_id()") + + def training_hash(self) -> str: + return sha256_hex(canonicalize(self.body(), allow_float=False)) + + +class PolicyHashes(WireModel): + admission_policy_hash: Hex64 + economics_policy_hash: Hex64 + aggregation_policy_hash: Hex64 + dispute_policy_hash: Hex64 + audit_policy_hash: Hex64 + + +class NetworkSpec(PolicyHashes): + relay_registry_hash: Hex64 + audit_mode: Literal["anchored-full"] + full_anchor_version: Literal[1] + capabilities: Annotated[ + list[Literal["island-replay", "all-level-disputes", "transport-receipts"]], + Field(min_length=3, max_length=3), + ] + + @model_validator(mode="after") + def _capabilities(self) -> NetworkSpec: + if len(set(self.capabilities)) != 3: + raise ValueError("all three capabilities required exactly once") + return self + + +class RunManifestV2(WireModel): + manifest_version: Literal[2] + training: TrainingManifestV2 + network: NetworkSpec + + def run_id(self) -> str: + return self.digest() + + def validate_run_id(self, run_id: str) -> None: + if run_id != self.run_id(): + raise ValueError("wrapper hash differs from explicit run ID") + + +class RosterEntryV2(WireModel): + hotkey: SS58 + slot: U64 + q_i: F32 + admission_id: Hex64 + coldkey_group: Identifier + state: AdmissionState + eligible_weight: Annotated[StrictInt, Field(ge=0, le=WEIGHT_QUANTUM)] + + @model_validator(mode="after") + def _finite(self) -> RosterEntryV2: + if not math.isfinite(f32val(self.q_i)): + raise ValueError("nonfinite roster audit probability") + return self + + +class RoundOpenV2(WireModel): + w: U64 + prev_final_hash: Hex64 + theta_hash: Hex64 + outer_state_hash: Hex64 + center_hash: Hex64 + roster_hash: Hex64 + honeypot_commit: Hex64 + d_open: Positive + d_assign: Positive + d_commit: Positive + d_audit: Positive + d_upload: Positive + d_final: Positive + contract_version: Literal[2] + policy_hashes: PolicyHashes + registry_epoch: U64 + start_state_index_hash: Hex64 + audit_mode: Literal["anchored-full"] + roster: Annotated[list[RosterEntryV2], Field(max_length=1024)] + + @model_validator(mode="after") + def _unique_roster(self) -> RoundOpenV2: + if not ( + self.d_open < self.d_assign < self.d_commit < self.d_audit < self.d_final + and self.d_commit < self.d_upload <= self.d_final + ): + raise ValueError("invalid round deadline ordering") + if len({r.slot for r in self.roster}) != len(self.roster): + raise ValueError("duplicate roster slot") + if len({r.hotkey for r in self.roster}) != len(self.roster): + raise ValueError("duplicate roster hotkey") + return self + + +class StartStateV2(WireModel): + run_id: Hex64 + w: U64 + hotkey: SS58 + theta_hash: Hex64 + state_object_sha256: Hex64 + opt_state_hash: Hex64 + ef_object_sha256: Hex64 + ef_hash: Hex64 + parent_anchor_hash: Hex64 + global_step0: U64 + anchor_verdict_hash: Hex64 + + +class AcceptV2(Accept): + work_screen_hash: Hex64 + + def validate_manifest(self, manifest: RunManifestV2) -> None: + ref = manifest.training.reference_spec + if ( + self.n_gpus != ref.layout.n_gpus + or self.image_digest != ref.image_digest + or self.driver_version not in ref.driver_allowlist + ): + raise ValueError("Accept differs from pinned logical layout/reference environment") + + +class CommitV2(Commit): + delta_bytes: Annotated[StrictInt, Field(ge=1, le=MAX_OBJECT_BYTES)] + + def validate_assignment(self, manifest: RunManifestV2, sample_count: int) -> None: + if sample_count != manifest.training.batch_samples(): + raise ValueError("assignment sample count differs from manifest") + if self.tokens != sample_count * manifest.training.model.seq_len: + raise ValueError("tokens differ from server-derived entitlement") + inner = manifest.training.inner + if self.n_leaves != inner.H // inner.J + 1: + raise ValueError("leaf count differs from manifest") + + +class DeltaManifestV2(DeltaManifest): + grant_hash: Hex64 + master_acceptance_hash: Hex64 + size: Annotated[StrictInt, Field(ge=1, le=MAX_OBJECT_BYTES)] + chunks: Annotated[list[Chunk], Field(min_length=1, max_length=512)] + + +class AuditChallengeV2(AuditChallenge): + anchor_hash: Hex64 + audit_mode: Literal["anchored-full"] + mode: Literal["full"] + segments: Annotated[list[tuple[U64, U64]], Field(max_length=256)] + + +class ArtifactRef(WireModel): + sha256: Hex64 + size: Annotated[StrictInt, Field(ge=1, le=MAX_OBJECT_BYTES)] + + +class AuditJobV2(WireModel): + run_id: Hex64 + job_id: Hex64 + auditor_id: SS58 + attempt: Annotated[StrictInt, Field(ge=1, le=2)] + lease_nonce: Hex64 + lease_expires: Positive + absolute_deadline: Positive + reservation_id: Hex64 + replay_step_budget: Positive + anchor_age: Annotated[StrictInt, Field(ge=0, le=1)] + manifest: RunManifestV2 + challenge_envelope: dict[str, JsonValue] + commit_envelope: dict[str, JsonValue] + sample_ids: Annotated[list[U64], Field(min_length=1, max_length=1_000_000)] + start_state: StartStateV2 + preimages: Annotated[list[LeafPreimage], Field(min_length=1, max_length=4096)] + ef_in: ArtifactRef + v0: ArtifactRef + created_beacon: Positive + + @model_validator(mode="after") + def _bounds(self) -> AuditJobV2: + self.manifest.validate_run_id(self.run_id) + h = self.manifest.training.inner.H + if not h <= self.replay_step_budget <= 2 * h: + raise ValueError("replay budget must cover H and not exceed 2H") + if not self.created_beacon < self.lease_expires <= self.absolute_deadline: + raise ValueError("invalid audit lease deadline") + if self.lease_expires > self.created_beacon + 100: + raise ValueError("lease exceeds 100 beacon rounds") + if self.absolute_deadline > self.created_beacon + 200: + raise ValueError("absolute audit deadline exceeds 200 beacon rounds") + if len(self.sample_ids) != self.manifest.training.batch_samples(): + raise ValueError("job assignment length mismatch") + if self.start_state.run_id != self.run_id: + raise ValueError("anchor run mismatch") + self.validate_embedded(self.created_beacon) + commit = CommitV2.model_validate(self.commit_envelope["body"]) + challenge = AuditChallengeV2.model_validate(self.challenge_envelope["body"]) + if ( + commit.w != challenge.w + or commit.hotkey != challenge.target + or self.start_state.w != commit.w + or self.start_state.hotkey != commit.hotkey + or challenge.anchor_hash != self.start_state.digest() + or self.absolute_deadline > challenge.serve_deadline + ): + raise ValueError("audit job assignment/anchor/deadline binding mismatch") + commit.validate_assignment(self.manifest, len(self.sample_ids)) + if any(p.run_id != self.run_id or p.w != commit.w for p in self.preimages): + raise ValueError("audit preimage run/round mismatch") + if not all( + math.isfinite(f32val(v)) for p in self.preimages for v in (p.loss_f32, p.norm_f32) + ): + raise ValueError("nonfinite audit metric quantity") + return self + + def validate_embedded(self, now_round: int) -> None: + """Live-expiry path; never waive expiry using an unauthenticated history assertion.""" + from hypertrain.protocol.envelope_v2 import parse_envelope, verify_envelope + + if type(now_round) is not int or not self.created_beacon <= now_round < self.lease_expires: + raise ValueError("embedded intake requires a live authenticated job lease") + for raw, expected in ( + (self.challenge_envelope, "AuditChallengeV2"), + (self.commit_envelope, "CommitV2"), + ): + env = parse_envelope(raw) + if env.type != expected or env.run_id != self.run_id or not verify_envelope(raw): + raise ValueError("audit job embeds invalid authenticated envelope") + # L0 owns authenticated original acceptance evidence for historical records. + # This callable supports only live expiry; no unsigned flag bypasses that rule. + if env.exp_drand < now_round: + raise ValueError("embedded envelope expired at verified intake beacon") + if expected == "CommitV2" and env.signer != env.body["hotkey"]: + raise ValueError("embedded commit hotkey differs from signer") + + +class RankWorkResult(WireModel): + rank: U64 + fingerprint: Hex64 + elapsed_ns: U64 + allocated_bytes: U64 + reserved_bytes: U64 + total_bytes: Positive + + @model_validator(mode="after") + def _memory(self) -> RankWorkResult: + if max(self.allocated_bytes, self.reserved_bytes) * 100 > self.total_bytes * 85: + raise ValueError("full-round peak exceeds 85%") + return self + + +class WorkScreenV2(WireModel): + evidence_kind: Literal["work-proof-v1"] + admission_id: Hex64 + nonce: Hex64 + policy_hash: Hex64 + image_digest: ImageDigest + layout: Layout + challenge_hash: Hex64 + rank_results: Annotated[list[RankWorkResult], Field(min_length=1, max_length=1024)] + artifact_hashes: Hashes + + @model_validator(mode="after") + def _ranks(self) -> WorkScreenV2: + if sorted(r.rank for r in self.rank_results) != list(range(self.layout.n_gpus)): + raise ValueError("work screen needs every logical rank exactly once") + return self + + +class HardwareHint(WireModel): + device_name: Text + device_count: Positive + driver: Text + + +class JoinRequest(WireModel): + run_id: Hex64 + request_id: Hex64 + hotkey: SS58 + coldkey: SS58 + expires_beacon: Positive + policy_hash: Hex64 + hardware_hint: HardwareHint + hot_sig: Annotated[str, StringConstraints(pattern=r"^[0-9a-f]{128}$")] + cold_sig: Annotated[str, StringConstraints(pattern=r"^[0-9a-f]{128}$")] + + +class ArtifactLimits(WireModel): + max_object_bytes: Annotated[StrictInt, Field(ge=1, le=MAX_OBJECT_BYTES)] + max_body_bytes: Literal[65536] + max_chunk_manifest_bytes: Literal[1048576] + + +class JoinChallenge(WireModel): + admission_id: Hex64 + nonce: Hex64 + seed_beacon: Positive + deadline_beacon: Positive + manifest_hash: Hex64 + theta_hash: Hex64 + assignment_hash: Hex64 + layout: Layout + artifact_limits: ArtifactLimits + policy_hash: Hex64 + + @model_validator(mode="after") + def _deadline(self) -> JoinChallenge: + if self.deadline_beacon != self.seed_beacon + 100: + raise ValueError("join deadline must be seed+100") + return self + + +class WorkProof(WireModel): + admission_id: Hex64 + challenge_hash: Hex64 + leaves_root: Hex64 + delta_hash: Hex64 + artifact_refs: Annotated[list[ArtifactRef], Field(min_length=1, max_length=512)] + + +class RotateRequest(WireModel): + hotkey: SS58 + new_hotkey: SS58 + operation_id: Hex64 + + +class AdmissionPolicyV2(WireModel): + q_base: Literal["0000803e"] # f32 0.25 + E: Literal[4] + clean_finalizations: Literal[12] + work_screen_epoch_rounds: Positive + max_pending: Literal[32] + max_trial_replays: Literal[2] + max_trials_per_beacon: Literal[2] + join_per_beacon: Literal[2] + join_burst: Literal[4] + ip_prefix_per_beacon: Literal[32] + ip_prefix_burst: Literal[64] + challenge_rounds: Literal[100] + artifact_limits: ArtifactLimits + + +class EconomicsPolicyV2(WireModel): + ledger_mode: Literal["test", "production"] + G_max_units: U64 + R_collectible_units: U64 + S_min_units: U64 + beta_ppm: Positive + gammaV_units: Literal[0] + s_lower_ppm: PPM + q_floor: Literal[1_000_000] + genesis_allocation_hash: Hex64 | None + reward_authority: SS58 + round_reward_units: U64 + max_total_issuance: U64 + shadow_bootstrap_rounds: Literal[12] + + @model_validator(mode="after") + def _mode(self) -> EconomicsPolicyV2: + if (self.ledger_mode == "test") != (self.genesis_allocation_hash is not None): + raise ValueError("only test mode requires pinned genesis allocation") + return self + + +class AggregationPolicyV2(WireModel): + arithmetic: Literal["flat-cclip-cap-v1"] + order: Literal["utf8"] + center: Literal["prev_outer_update"] + trust_mode: Literal["uniform-verified"] + weight_quantum: Literal[16777216] + miner_cap_units: Literal[4194304] + probation_cap_units: Literal[4194304] + owner_group_caps: Annotated[dict[Identifier, Positive], Field(max_length=1024)] + preclip_norm: F32 + cclip_tau: F32 + cclip_iters: Positive + + @model_validator(mode="after") + def _finite(self) -> AggregationPolicyV2: + if not all(math.isfinite(f32val(v)) for v in (self.preclip_norm, self.cclip_tau)): + raise ValueError("nonfinite aggregation quantity") + return self + + +class DisputePolicyV2(WireModel): + referees: Annotated[list[SS58], Field(min_length=1, max_length=128)] + max_open: Literal[32] + max_transcript_entries: Literal[256] + max_entry_bytes: Literal[65536] + fanout: Literal[2] + max_referee_reassignments: Literal[1] + max_referee_cost_units: U64 + + +class AuditPolicyV2(WireModel): + lease_rounds: Literal[100] + max_attempts: Literal[2] + max_concurrent_per_auditor: Literal[1] + max_running_jobs: Literal[2] + max_queued_jobs: Literal[32] + max_anchor_age_rounds: Literal[1] + max_steps_per_attempt: Positive + + def validate_training(self, training: TrainingManifestV2) -> None: + if self.max_steps_per_attempt != 2 * training.inner.H: + raise ValueError("audit policy budget must equal 2H") + + +class IslandJobV1(WireModel): + job_version: Literal[1] + run_id: Hex64 + w: U64 + manifest: RunManifestV2 + sample_ids: Annotated[list[U64], Field(min_length=1, max_length=1_000_000)] + global_step0: U64 + start_state_sha256: Hex64 + ef_in_sha256: Hex64 + v0_sha256: Hex64 + object_paths: Annotated[dict[Identifier, Text], Field(min_length=1, max_length=512)] + deadline: Positive + + @model_validator(mode="after") + def _binding(self) -> IslandJobV1: + self.manifest.validate_run_id(self.run_id) + if len(self.sample_ids) != self.manifest.training.batch_samples(): + raise ValueError("island job assignment length mismatch") + for path in self.object_paths.values(): + if path.startswith("/") or ".." in path.split("/") or "\\" in path: + raise ValueError("local object paths must be relative and confined") + return self + + +class WeightedEntryV2(WireModel): + hotkey: SS58 + admission_id: Hex64 + probation: StrictBool + weight_units: Annotated[StrictInt, Field(ge=0, le=WEIGHT_QUANTUM // 4)] + commit_hash: Hex64 + delta_manifest_hash: Hex64 + + +class WeightAllocationV2(WireModel): + policy_hash: Hex64 + entries: Annotated[list[WeightedEntryV2], Field(min_length=4, max_length=1024)] + + @model_validator(mode="after") + def _caps(self) -> WeightAllocationV2: + ids = [e.hotkey for e in self.entries] + if ids != sorted(set(ids), key=lambda value: value.encode("utf-8")): + raise ValueError("weights must use unique UTF-8 miner order") + if sum(e.weight_units for e in self.entries) != WEIGHT_QUANTUM: + raise ValueError("weights must sum exactly to Q; no renormalization") + if sum(e.weight_units for e in self.entries if e.probation) > WEIGHT_QUANTUM // 4: + raise ValueError("probation cohort exceeds Q/4") + return self + + +class OriginAllocation(WireModel): + origin_id: Hex64 + owner: SS58 + units: Positive + mature_at: U64 + + +class TestGenesis(WireModel): + run_id: Hex64 + allocation_hash: Hex64 + total_units: Positive + origins: Annotated[list[OriginAllocation], Field(min_length=1, max_length=1024)] + authority_sig: Annotated[str, StringConstraints(pattern=r"^[0-9a-f]{128}$")] + + @model_validator(mode="after") + def _conservation(self) -> TestGenesis: + if len({o.origin_id for o in self.origins}) != len(self.origins): + raise ValueError("duplicate genesis origin") + if sum(o.units for o in self.origins) != self.total_units: + raise ValueError("genesis allocations do not conserve total") + return self + + +class RewardFinalize(WireModel): + run_id: Hex64 + w: U64 + finalize_hash: Hex64 + tape_hash: Hex64 + verdict_root: Hex64 + allocation_hash: Hex64 + budget_units: Positive + origin_ids: Annotated[list[Hex64], Field(min_length=1, max_length=1024)] + mature_at: U64 + authority_sig: Annotated[str, StringConstraints(pattern=r"^[0-9a-f]{128}$")] + + @model_validator(mode="after") + def _origins(self) -> RewardFinalize: + if len(set(self.origin_ids)) != len(self.origin_ids): + raise ValueError("duplicate reward origin") + return self + + +class ShadowRewardFinalizeV1(RewardFinalize): + shadow: Literal[True] + shadow_ordinal: Annotated[StrictInt, Field(ge=0, lt=12)] + reservation_hash: Hex64 + + @model_validator(mode="before") + @classmethod + def _signed_domain(cls, data: JsonValue) -> JsonValue: + if isinstance(data, dict) and data.get("shadow") is not True: + raise ValueError("shadow domain requires JSON true") + return data + + +class ShadowReservationRequestV1(WireModel): + """Trusted store intake binds original custody before external signing.""" + + run_id: Hex64 + admission_id: Hex64 + hotkey: SS58 + coldkey: SS58 + trial_epoch: U64 + commit_hash: Hex64 + finalize_hash: Hex64 + custody_hash: Hex64 + graph_hash: Hex64 + economic_admission_hash: Hex64 + nonce: Hex64 + assignment_hash: Hex64 + finalized_beacon: Positive + mature_at: Positive + + +class ShadowReservationV1(ShadowReservationRequestV1): + shadow_ordinal: Annotated[StrictInt, Field(ge=0, lt=12)] + + +class ShadowReplayReceiptV1(WireModel): + run_id: Hex64 + admission_id: Hex64 + hotkey: SS58 + coldkey: SS58 + trial_epoch: U64 + shadow_ordinal: Annotated[StrictInt, Field(ge=0, lt=12)] + reservation_hash: Hex64 + challenge_hash: Hex64 + signed_challenge_hash: Hex64 + assignment_hash: Hex64 + sample_ids_hash: Hex64 + job_hash: Hex64 + reference_publication_hash: Hex64 + miner_publication_hash: Hex64 + reference_result_hash: Hex64 + miner_result_hash: Hex64 + reference_operation_hash: Hex64 + miner_operation_hash: Hex64 + reference_work_proof_hash: Hex64 + miner_work_proof_hash: Hex64 + commit_hash: Hex64 + audit_challenge_hash: Hex64 + verdict_hash: Hex64 + trial_finalize_hash: Hex64 + final_state_hash: Hex64 + backend_qualification_authority_hash: Hex64 + completed_beacon: Positive + + +class EscrowOperation(WireModel): + operation_id: Hex64 + owner: SS58 + units: Positive + origin_ids: Annotated[list[Hex64], Field(min_length=1, max_length=1024)] + admission_id: Hex64 | None + dispute_id: Hex64 | None + + @model_validator(mode="after") + def _origins(self) -> EscrowOperation: + if len(set(self.origin_ids)) != len(self.origin_ids): + raise ValueError("duplicate escrow origin") + return self + + +class EscrowLock(EscrowOperation): + kind: Literal["LOCK_ADMISSION", "LOCK_CONTEST"] + + @model_validator(mode="after") + def _reference(self) -> EscrowLock: + if (self.kind == "LOCK_ADMISSION") != (self.admission_id is not None): + raise ValueError("admission lock/reference mismatch") + if (self.kind == "LOCK_CONTEST") != (self.dispute_id is not None): + raise ValueError("contest lock/reference mismatch") + return self + + +class EscrowRelease(EscrowOperation): + lock_id: Hex64 + finality_hash: Hex64 + closed_dispute_root: Hex64 + + +class EscrowTransfer(EscrowOperation): + recipient: SS58 + + +class EscrowReceipt(WireModel): + operation_id: Hex64 + event_hash: Hex64 + ledger_mode: Literal["test", "production"] + available_units: U64 + admission_locked_units: U64 + dispute_locked_units: U64 + + +class DisputeV2(WireModel): + hotkey: SS58 + verdict_hash: Hex64 + action: Literal["accept", "contest"] + bond_lock: Hex64 | None + + @model_validator(mode="after") + def _funded(self) -> DisputeV2: + if (self.action == "contest") != (self.bond_lock is not None): + raise ValueError("contest requires funded bond_lock; accept must omit it") + return self + + +class BisectV2(WireModel): + dispute_id: Hex64 + seq: Annotated[StrictInt, Field(ge=0, le=255)] + level: Literal["step", "layer", "op"] + ctx: Annotated[list[U64], Field(max_length=2)] + interval: tuple[U64, U64] + N: Literal[2] + hashes: Annotated[list[Hex64], Field(min_length=3, max_length=3)] + party: SS58 + previous_transcript_hash: Hex64 + + @model_validator(mode="after") + def _context(self) -> BisectV2: + if len(self.ctx) != {"step": 0, "layer": 1, "op": 2}[self.level]: + raise ValueError("bisection context does not match level") + if self.interval[0] >= self.interval[1]: + raise ValueError("bisection interval must be nonempty") + return self + + +class ResolutionV2(WireModel): + dispute_id: Hex64 + transcript_hash: Hex64 + reason: Literal[ + "MATCH", + "FRAUD", + "WITHHELD", + "PARTY_TIMEOUT", + "INFRASTRUCTURE", + "AUDITOR_FAULT", + "REFEREE_FAILURE", + ] + loser: SS58 | None + evidence_hash: Hex64 + + @model_validator(mode="after") + def _loser(self) -> ResolutionV2: + no_loser = self.reason in {"MATCH", "INFRASTRUCTURE", "REFEREE_FAILURE"} + if no_loser != (self.loser is None): + raise ValueError("resolution reason/loser mismatch") + return self + + +MESSAGE_TYPES_V2: dict[str, type[BaseModel]] = { + cls.__name__: cls + for cls in ( + RunManifestV2, + RoundOpenV2, + StartStateV2, + AcceptV2, + CommitV2, + DeltaManifestV2, + AuditChallengeV2, + AuditJobV2, + WorkScreenV2, + JoinRequest, + JoinChallenge, + WorkProof, + RotateRequest, + AdmissionPolicyV2, + EconomicsPolicyV2, + AggregationPolicyV2, + DisputePolicyV2, + AuditPolicyV2, + TestGenesis, + RewardFinalize, + EscrowLock, + ShadowRewardFinalizeV1, + ShadowReplayReceiptV1, + ShadowReservationRequestV1, + ShadowReservationV1, + EscrowRelease, + EscrowTransfer, + EscrowReceipt, + DisputeV2, + BisectV2, + ResolutionV2, + Receipt, + StateServe, + ReplayVerdict, + Rollback, + Finalize, + IslandJobV1, + WeightAllocationV2, + ) +} diff --git a/src/hypertrain/protocol/relay_envelope.py b/src/hypertrain/protocol/relay_envelope.py new file mode 100644 index 0000000..1b06f86 --- /dev/null +++ b/src/hypertrain/protocol/relay_envelope.py @@ -0,0 +1,75 @@ +"""Relay domain reuses bounded canonical intake, never ht/1 or ht/2 signatures.""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, JsonValue + +from hypertrain.protocol.envelope import MalformedEnvelope +from hypertrain.protocol.envelope_v2 import Intake as TrainingIntake +from hypertrain.protocol.envelope_v2 import ( + SignedEnvelope, + _parse, + _seal, + _signing_message, + _verify, +) +from hypertrain.protocol.jcs import CanonicalizationError +from hypertrain.protocol.keys import KeyError_, Keypair +from hypertrain.protocol.relay_messages import RELAY_MESSAGE_TYPES + +VERSION = "ht-relay/1" +DOMAIN = b"hypertrain/relay/1|" + + +class RelayEnvelope(SignedEnvelope): + v: Literal["ht-relay/1"] + + +def signing_message( + msg_type: str, + body: dict[str, JsonValue], + exp_drand: int, + run_id: str, +) -> bytes: + return _signing_message(DOMAIN, msg_type, body, exp_drand, run_id) + + +def seal( + keypair: Keypair, + msg_type: str, + run_id: str, + body: BaseModel | dict[str, JsonValue], + exp: int, +) -> dict[str, JsonValue]: + env = _seal( + keypair, + msg_type, + run_id, + body, + exp, + version=VERSION, + domain=DOMAIN, + models=RELAY_MESSAGE_TYPES, + ) + _parse(env, RelayEnvelope, RELAY_MESSAGE_TYPES) + return env + + +def parse_envelope(raw: bytes | str | dict[str, JsonValue]) -> RelayEnvelope: + env, _ = _parse(raw, RelayEnvelope, RELAY_MESSAGE_TYPES) + assert isinstance(env, RelayEnvelope) + return env + + +def verify_envelope(raw: bytes | str | dict[str, JsonValue]) -> bool: + try: + return _verify(parse_envelope(raw), DOMAIN) + except (MalformedEnvelope, KeyError_, CanonicalizationError): + return False + + +class Intake(TrainingIntake): + def accept(self, raw: bytes | str | dict[str, JsonValue], now_round: int) -> BaseModel: + return self._accept(raw, now_round, RelayEnvelope, RELAY_MESSAGE_TYPES, DOMAIN) diff --git a/src/hypertrain/protocol/relay_messages.py b/src/hypertrain/protocol/relay_messages.py new file mode 100644 index 0000000..c5e6d69 --- /dev/null +++ b/src/hypertrain/protocol/relay_messages.py @@ -0,0 +1,508 @@ +"""Transport-only relay contracts; opportunity acceptance never implies byte custody.""" + +from __future__ import annotations + +from typing import Annotated, Literal +from urllib.parse import urlsplit + +from pydantic import Field, StrictBool, StrictInt, model_validator + +from hypertrain.protocol.hashing import sha256_hex +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.messages import SS58, Hex64 +from hypertrain.protocol.messages_v2 import ( + MAX_OBJECT_BYTES, + U64, + Hashes, + Identifier, + Positive, + Text, + WireModel, +) + +CHUNK_BYTES = 4 << 20 +ObjectSize = Annotated[StrictInt, Field(ge=1, le=MAX_OBJECT_BYTES)] + + +class RelayKey(WireModel): + key_id: Identifier + pubkey: SS58 + valid_from_round: U64 + valid_until_round: Positive + + @model_validator(mode="after") + def _validity(self) -> RelayKey: + if self.valid_from_round >= self.valid_until_round: + raise ValueError("empty relay key validity interval") + return self + + +class RelaySpec(WireModel): + id: Identifier + region: Identifier + https_url: Text + pubkeys: Annotated[list[RelayKey], Field(min_length=1, max_length=16)] + max_object_bytes: ObjectSize + max_inflight_bytes: Positive + codecs: Annotated[ + list[Literal["ht-sparse-v1", "ht-dense-int8-v1"]], Field(min_length=1, max_length=2) + ] + mode: Literal["transport"] + + @model_validator(mode="after") + def _endpoint(self) -> RelaySpec: + url = urlsplit(self.https_url) + if ( + url.scheme != "https" + or not url.hostname + or url.username + or url.password + or url.query + or url.fragment + ): + raise ValueError("relay endpoint must be credential-free HTTPS") + if len({k.key_id for k in self.pubkeys}) != len(self.pubkeys): + raise ValueError("duplicate relay key ID") + if len(set(self.codecs)) != len(self.codecs): + raise ValueError("duplicate codec") + if self.max_inflight_bytes < self.max_object_bytes: + raise ValueError("inflight limit cannot be less than object limit") + return self + + +class RelayRegistryV1(WireModel): + registry_version: Literal[1] + epoch: U64 + previous_registry_hash: Hex64 | None + specs: Annotated[list[RelaySpec], Field(min_length=1, max_length=128)] + + @model_validator(mode="after") + def _ids(self) -> RelayRegistryV1: + if len({s.id for s in self.specs}) != len(self.specs): + raise ValueError("duplicate relay ID") + if (self.epoch == 0) != (self.previous_registry_hash is None): + raise ValueError("successor registry must reference predecessor") + return self + + +class RelayNetworkManifest(WireModel): + run_id: Hex64 + base_manifest_hash: Hex64 + registry_hash: Hex64 + specs_hash: Hex64 + assignment_policy: Literal["master-observed-median3-v1"] + + +class RelayAssignment(WireModel): + w: U64 + hotkey: SS58 + primary_id: Identifier + fallback_ids: Annotated[list[Identifier], Field(max_length=128)] + assignment_epoch: U64 + manifest_hash: Hex64 + exp_drand: Positive + + @model_validator(mode="after") + def _fallbacks(self) -> RelayAssignment: + if self.primary_id in self.fallback_ids or len(set(self.fallback_ids)) != len( + self.fallback_ids + ): + raise ValueError("relay assignment IDs must be unique") + return self + + +class UploadGrant(WireModel): + w: U64 + hotkey: SS58 + relay_id: Identifier + assignment_epoch: U64 + delta_hash: Hex64 + size: ObjectSize + chunk_manifest_hash: Hex64 + retain_until: Positive + nonce: Hex64 + exp_drand: Positive + + @model_validator(mode="after") + def _retention(self) -> UploadGrant: + if self.retain_until <= self.exp_drand: + raise ValueError("grant retention must outlive upload expiration") + return self + + +class UploadChunk(WireModel): + index: Annotated[StrictInt, Field(ge=0, le=511)] + off: U64 + len: Annotated[StrictInt, Field(ge=1, le=CHUNK_BYTES)] + chunk_sha256: Hex64 + + +class UploadChunkManifest(WireModel): + size: ObjectSize + chunks: Annotated[list[UploadChunk], Field(min_length=1, max_length=512)] + + @model_validator(mode="after") + def _coverage(self) -> UploadChunkManifest: + offset = 0 + for index, chunk in enumerate(self.chunks): + if chunk.index != index or chunk.off != offset: + raise ValueError("chunks must be indexed contiguous exact coverage") + if index < len(self.chunks) - 1 and chunk.len != CHUNK_BYTES: + raise ValueError("nonterminal chunk must contain 4MiB") + offset += chunk.len + if offset != self.size: + raise ValueError("chunk coverage does not match object size") + return self + + def chunk_manifest_hash(self) -> str: + """Grant binds the canonical expected chunk LIST, not a second object wrapper.""" + return sha256_hex(canonicalize([c.body() for c in self.chunks], allow_float=False)) + + def validate_grant(self, grant: UploadGrant) -> None: + if self.size != grant.size or self.chunk_manifest_hash() != grant.chunk_manifest_hash: + raise ValueError("chunk list differs from signed grant") + + +class AcceptedUploadAck(WireModel): + grant_hash: Hex64 + w: U64 + hotkey: SS58 + relay_id: Identifier + assignment_epoch: U64 + delta_hash: Hex64 + size: ObjectSize + chunk_manifest_hash: Hex64 + key_id: Identifier + accepted_beacon: Positive + service_deadline: Positive + + @model_validator(mode="after") + def _deadline(self) -> AcceptedUploadAck: + if self.accepted_beacon > self.service_deadline: + raise ValueError("upload opportunity accepted after deadline") + return self + + +class ChunkCustodyAck(UploadChunk): + upload_ack_hash: Hex64 + grant_hash: Hex64 + durable_chunk_key: Text + key_id: Identifier + received_beacon: Positive + retain_until: Positive + + @model_validator(mode="after") + def _horizon(self) -> ChunkCustodyAck: + if self.received_beacon >= self.retain_until: + raise ValueError("chunk custody requires future retention") + return self + + +class RelayReceipt(WireModel): + w: U64 + hotkey: SS58 + delta_hash: Hex64 + size: ObjectSize + grant_hash: Hex64 + chunk_manifest_hash: Hex64 + durable_object_key: Text + key_id: Identifier + received_drand: Positive + retain_until: Positive + + @model_validator(mode="after") + def _horizon(self) -> RelayReceipt: + if self.received_drand >= self.retain_until: + raise ValueError("receipt requires future retention") + return self + + +class MasterAcceptanceV2(WireModel): + w: U64 + hotkey: SS58 + delta_hash: Hex64 + receipt_hash: Hex64 + received_drand: Positive + + +class RetentionExtension(WireModel): + grant_hash: Hex64 + custody_hashes: Annotated[list[Hex64], Field(min_length=1, max_length=512)] + seq: Positive + previous_retention_hash: Hex64 + dispute_ids: Hashes + issued_beacon: Positive + retain_until: Positive + + @model_validator(mode="after") + def _horizon(self) -> RetentionExtension: + if self.issued_beacon >= self.retain_until: + raise ValueError("extension requires future horizon") + if len(set(self.custody_hashes)) != len(self.custody_hashes): + raise ValueError("duplicate extension custody scope") + return self + + +class RetentionExtensionAck(WireModel): + extension_hash: Hex64 + key_id: Identifier + accepted_beacon: Positive + + +class CustodyRelease(WireModel): + grant_hash: Hex64 + custody_hashes: Annotated[list[Hex64], Field(min_length=1, max_length=512)] + retention_hash: Hex64 + finality_hash: Hex64 + vesting_beacon: Positive + closed_dispute_root: Hex64 + release_beacon: Positive + + @model_validator(mode="after") + def _vesting(self) -> CustodyRelease: + if self.release_beacon < self.vesting_beacon: + raise ValueError("release precedes vesting") + if len(set(self.custody_hashes)) != len(self.custody_hashes): + raise ValueError("duplicate release custody scope") + return self + + +class RetrievalRequest(WireModel): + request_id: Hex64 + nonce: Hex64 + relay_id: Identifier + key_id: Identifier + assignment_hash: Hex64 + grant_hash: Hex64 + custody_ack_hashes: Hashes + receipt_hash: Hex64 | None + retention_hash: Hex64 + object_or_chunk_hash: Hex64 + size: ObjectSize + requested_beacon: Positive + deadline_beacon: Positive + + @model_validator(mode="after") + def _deadline(self) -> RetrievalRequest: + if not self.requested_beacon < self.deadline_beacon <= self.requested_beacon + 10: + raise ValueError("retrieval deadline must be in (requested, requested+10]") + if not self.custody_ack_hashes and self.receipt_hash is None: + raise ValueError("retrieval requires authenticated byte custody") + if len(set(self.custody_ack_hashes)) != len(self.custody_ack_hashes): + raise ValueError("duplicate retrieval custody scope") + return self + + def validate_horizon(self, retain_until: int, *, released: bool = False) -> None: + """Upload cutoff is intentionally absent from retrieval authorization.""" + if released or self.deadline_beacon != min(self.requested_beacon + 10, retain_until): + raise ValueError("retrieval exceeds effective signed custody obligation") + + +class RetrievalResponse(WireModel): + request_hash: Hex64 + nonce: Hex64 + key_id: Identifier + status: Literal["SERVED", "NOT_FOUND", "UNAVAILABLE"] + object_hash: Hex64 + size: ObjectSize + served_beacon: Positive + body_sha256: Hex64 + error_code: Identifier | None + + @model_validator(mode="after") + def _status(self) -> RetrievalResponse: + if (self.status == "SERVED") != (self.error_code is None): + raise ValueError("retrieval status/error code mismatch") + return self + + def validate_request(self, request: RetrievalRequest, payload: bytes | None) -> None: + if ( + self.request_hash != request.digest() + or self.nonce != request.nonce + or self.key_id != request.key_id + or self.object_hash != request.object_or_chunk_hash + or self.size != request.size + or self.served_beacon > request.deadline_beacon + or self.served_beacon < request.requested_beacon + ): + raise ValueError("retrieval response differs from authenticated request") + if self.status == "SERVED": + if ( + payload is None + or len(payload) != self.size + or sha256_hex(payload) != self.body_sha256 + or self.body_sha256 != self.object_hash + ): + raise ValueError("served bytes fail independent hash/size verification") + elif payload is not None: + raise ValueError("failed retrieval must not return object bytes") + + +class AvailabilityContext(WireModel): + master_healthy: StrictBool + beacon_healthy: StrictBool + reference_healthy: StrictBool + observer_healthy: StrictBool + + +class RelayFailureEvidence(WireModel): + run_id: Hex64 + relay_id: Identifier + key_id: Identifier + assignment_hash: Hex64 + grant_hash: Hex64 + upload_ack_hash: Hex64 + chunk_manifest_hash: Hex64 + chunk_ack_hashes: Hashes + receipt_hash: Hex64 | None + retention_hash: Hex64 + request: RetrievalRequest | None + response: RetrievalResponse | None + observer_ids: Annotated[list[SS58], Field(min_length=1, max_length=2)] + observed_beacons: Annotated[list[Positive], Field(min_length=1, max_length=2)] + result: Literal[ + "EQUIVOCATION", + "HASH_INCONSISTENCY", + "CONTRACTUAL_SERVICE_FAILURE", + "UNCONFIRMED_AVAILABILITY", + ] + availability_context: AvailabilityContext + evidence_hash: Hex64 + + @model_validator(mode="after") + def _observers(self) -> RelayFailureEvidence: + if len(set(self.observer_ids)) != len(self.observer_ids): + raise ValueError("observers must be independent identities") + if len(self.observer_ids) != len(self.observed_beacons): + raise ValueError("each observer needs its verified beacon observation") + if self.result == "CONTRACTUAL_SERVICE_FAILURE": + if self.receipt_hash is None and not self.chunk_ack_hashes: + raise ValueError("opportunity ack cannot establish byte custody") + if len(self.observer_ids) != 2 or not all(self.availability_context.body().values()): + raise ValueError("attribution requires healthy independent observers") + return self + + +def has_full_custody( + grant: UploadGrant, + manifest: UploadChunkManifest, + opportunity: AcceptedUploadAck, + chunks: list[ChunkCustodyAck], + receipt: RelayReceipt | None = None, +) -> bool: + """Exact, timely same-key coverage only. Caller verifies all signed envelopes first.""" + if ( + manifest.size != grant.size + or manifest.chunk_manifest_hash() != grant.chunk_manifest_hash + or opportunity.grant_hash != grant.digest() + or opportunity.w != grant.w + or opportunity.hotkey != grant.hotkey + or opportunity.relay_id != grant.relay_id + or opportunity.assignment_epoch != grant.assignment_epoch + or opportunity.delta_hash != grant.delta_hash + or opportunity.size != grant.size + or opportunity.chunk_manifest_hash != grant.chunk_manifest_hash + or opportunity.service_deadline > grant.exp_drand + ): + return False + if receipt is not None: + return ( + receipt.grant_hash == grant.digest() + and receipt.w == grant.w + and receipt.hotkey == grant.hotkey + and receipt.delta_hash == grant.delta_hash + and receipt.size == grant.size + and receipt.chunk_manifest_hash == grant.chunk_manifest_hash + and receipt.key_id == opportunity.key_id + and opportunity.accepted_beacon + <= receipt.received_drand + <= opportunity.service_deadline + and receipt.retain_until >= grant.retain_until + ) + if len(chunks) != len(manifest.chunks): + return False + ordered = sorted(chunks, key=lambda c: c.index) + return all( + (ack.index, ack.off, ack.len, ack.chunk_sha256) + == (expected.index, expected.off, expected.len, expected.chunk_sha256) + and ack.upload_ack_hash == opportunity.digest() + and ack.grant_hash == grant.digest() + and ack.key_id == opportunity.key_id + and opportunity.accepted_beacon <= ack.received_beacon <= opportunity.service_deadline + and ack.retain_until >= grant.retain_until + for ack, expected in zip(ordered, manifest.chunks, strict=True) + ) + + +def extend_retention( + extension: RetentionExtension, + ack: RetentionExtensionAck, + *, + grant_hash: str, + custody_hashes: list[str], + key_id: str, + previous_hash: str, + previous_seq: int, + previous_horizon: int, +) -> int: + """Apply only an acknowledged, scoped successor signed before prior expiry.""" + if ( + extension.grant_hash != grant_hash + or set(extension.custody_hashes) != set(custody_hashes) + or extension.previous_retention_hash != previous_hash + or extension.seq != previous_seq + 1 + or extension.retain_until <= previous_horizon + or not extension.issued_beacon <= ack.accepted_beacon < previous_horizon + or ack.extension_hash != extension.digest() + or ack.key_id != key_id + ): + raise ValueError("unacknowledged, late or conflicting retention extension") + return extension.retain_until + + +def validate_release( + release: CustodyRelease, + *, + grant_hash: str, + custody_hashes: list[str], + retention_hash: str, + retain_until: int, + finality_beacon: int, + disputes_closed_beacon: int | None, + epoch_rounds: int, + retrieval_deadlines: list[int], +) -> int: + """Return earliest deletion beacon; future lanes persist finality and release evidence.""" + if ( + disputes_closed_beacon is None + or epoch_rounds < 1 + or release.grant_hash != grant_hash + or release.retention_hash != retention_hash + or set(release.custody_hashes) != set(custody_hashes) + or release.release_beacon + < max(finality_beacon, release.vesting_beacon, disputes_closed_beacon) + epoch_rounds + ): + raise ValueError("release requires closed disputes, finality, vesting and epoch grace") + return max(release.release_beacon, retain_until, *retrieval_deadlines) + + +RELAY_MESSAGE_TYPES: dict[str, type[WireModel]] = { + cls.__name__: cls + for cls in ( + RelayRegistryV1, + RelayNetworkManifest, + RelayAssignment, + UploadGrant, + UploadChunkManifest, + AcceptedUploadAck, + ChunkCustodyAck, + RelayReceipt, + MasterAcceptanceV2, + RetentionExtension, + RetentionExtensionAck, + CustodyRelease, + RetrievalRequest, + RetrievalResponse, + RelayFailureEvidence, + ) +} diff --git a/src/hypertrain/protocol/schema_v2.py b/src/hypertrain/protocol/schema_v2.py new file mode 100644 index 0000000..f92d124 --- /dev/null +++ b/src/hypertrain/protocol/schema_v2.py @@ -0,0 +1,40 @@ +"""V2-only schema export; caller supplies destination, frozen v1 schemas stay untouched.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from pydantic import BaseModel + +from hypertrain.protocol.envelope_v2 import EnvelopeV2 +from hypertrain.protocol.messages_v2 import MESSAGE_TYPES_V2, TrainingManifestV2 +from hypertrain.protocol.relay_envelope import RelayEnvelope +from hypertrain.protocol.relay_messages import RELAY_MESSAGE_TYPES + +SCHEMA_MODELS_V2: dict[str, type[BaseModel]] = { + **MESSAGE_TYPES_V2, + **RELAY_MESSAGE_TYPES, + "TrainingManifestV2": TrainingManifestV2, + "EnvelopeV2": EnvelopeV2, + "RelayEnvelope": RelayEnvelope, +} + + +def render() -> dict[str, bytes]: + return { + f"{name}.json": ( + json.dumps(model.model_json_schema(), sort_keys=True, indent=2) + "\n" + ).encode() + for name, model in SCHEMA_MODELS_V2.items() + } + + +def write(out: Path) -> list[Path]: + out.mkdir(parents=True, exist_ok=True) + paths = [] + for name, data in render().items(): + path = out / name + path.write_bytes(data) + paths.append(path) + return paths diff --git a/src/hypertrain/relay/__init__.py b/src/hypertrain/relay/__init__.py new file mode 100644 index 0000000..5e828a9 --- /dev/null +++ b/src/hypertrain/relay/__init__.py @@ -0,0 +1 @@ +"""Regional transport only; no training authority or regional arithmetic.""" diff --git a/src/hypertrain/relay/app.py b/src/hypertrain/relay/app.py new file mode 100644 index 0000000..d4a018a --- /dev/null +++ b/src/hypertrain/relay/app.py @@ -0,0 +1,293 @@ +"""Relay HTTP boundary; runtime keys/config mounted separately from coordinator secrets.""" + +from __future__ import annotations + +import base64 +import binascii +import hashlib +import hmac +import os +import tempfile +import time +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from pathlib import Path + +import anyio +import httpx +from fastapi import FastAPI, HTTPException, Request +from fastapi.responses import JSONResponse, StreamingResponse +from pydantic import BaseModel, ConfigDict, ValidationError + +from hypertrain.beacon.core import BeaconError, round_at_or_after +from hypertrain.beacon.drand import QUICKNET_GENESIS, QUICKNET_PERIOD, DrandQuicknet +from hypertrain.data.store import LocalFSStore, S3Credentials, S3Store, StoreError +from hypertrain.data.stream_store import StreamStore, blocks, spool +from hypertrain.protocol import envelope_v2, relay_envelope +from hypertrain.protocol.envelope import EnvelopeError +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.keys import Keypair +from hypertrain.protocol.relay_envelope import RelayEnvelope +from hypertrain.protocol.relay_messages import ( + RelayNetworkManifest, + RelayRegistryV1, + UploadChunkManifest, + UploadGrant, +) +from hypertrain.relay.client import forward_receipt, http_client +from hypertrain.relay.core import Relay, RelayError + + +class AcceptRequest(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + grant: RelayEnvelope + manifest: UploadChunkManifest + assignment: RelayEnvelope + round_open: envelope_v2.EnvelopeV2 + + +class RelayConfig(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + run_id: str + master_public: str + master_url: str + registry: RelayEnvelope + network_manifest: RelayEnvelope + relay_id: str + region: str + active_key: str + signing_files: dict[str, str] + observer_public_keys: list[str] + s3_endpoint: str | None = None + s3_bucket: str | None = None + s3_prefix: str = "relay/" + s3_region: str = "auto" + local_backing: str | None = None + + +def create_app(relay: Relay, drain_token: str) -> FastAPI: + if len(drain_token) < 32: + raise RelayError("drain credential must contain >=32 characters") + app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None) + + @app.exception_handler(RelayError) + @app.exception_handler(ValidationError) + @app.exception_handler(EnvelopeError) + async def rejected(request: Request, error: Exception) -> JSONResponse: + return JSONResponse({"error": type(error).__name__}, status_code=400) + + @app.exception_handler(StoreError) + @app.exception_handler(httpx.HTTPError) + @app.exception_handler(BeaconError) + async def unavailable(request: Request, error: Exception) -> JSONResponse: + return JSONResponse({"error": type(error).__name__}, status_code=503) + + async def body(request: Request) -> bytes: + data = bytearray() + async for part in request.stream(): + data.extend(part) + if len(data) > (1 << 20) + 4096: + raise HTTPException(413, "control message too large") + return bytes(data) + + async def upload_capability(request: Request, grant_hash: str) -> None: + value = request.headers.get("X-Upload-Grant", "") + if len(value) > 8192: + raise RelayError("grant header exceeds bound") + try: + raw = relay_envelope.parse_envelope(base64.b64decode(value, validate=True)) + except binascii.Error: + raise RelayError("malformed signed upload capability") from None + grant = relay.master_message(raw, UploadGrant, await relay.now()) + if grant.digest() != grant_hash: + raise RelayError("upload capability path mismatch") + + @app.get("/livez") + async def live() -> JSONResponse: + return JSONResponse({"live": True}) + + @app.get("/readyz") + async def ready() -> JSONResponse: + try: + value = await relay.ready() + except RelayError: + value = False + return JSONResponse({"ready": value}, status_code=200 if value else 503) + + @app.get("/probe") + async def probe() -> JSONResponse: + return JSONResponse( + {"relay_id": relay.spec.id, "region": relay.spec.region, "beacon": await relay.now()} + ) + + @app.post("/v1/uploads/{grant_hash}/accept") + async def accept(grant_hash: str, request: Request) -> JSONResponse: + parsed = AcceptRequest.model_validate(envelope_v2.load_json(await body(request))) + if UploadGrant.model_validate(parsed.grant.body).digest() != grant_hash: + raise RelayError("path grant differs from signed capability") + ack = await relay.accept( + parsed.grant, parsed.manifest, parsed.assignment, parsed.round_open + ) + return JSONResponse(ack.model_dump(mode="json")) + + @app.put("/v1/uploads/{grant_hash}/chunks/{index}") + async def chunk(grant_hash: str, index: int, request: Request) -> JSONResponse: + await upload_capability(request, grant_hash) + ack = await relay.chunk(grant_hash, index, request.stream()) + return JSONResponse(ack.model_dump(mode="json")) + + @app.post("/v1/uploads/{grant_hash}/complete") + async def complete(grant_hash: str, request: Request) -> JSONResponse: + await upload_capability(request, grant_hash) + receipt = await relay.complete(grant_hash) + return JSONResponse(receipt.model_dump(mode="json")) + + @app.get("/v1/uploads/{grant_hash}/acks") + async def acks(grant_hash: str, request: Request, cursor: int = 0) -> JSONResponse: + await upload_capability(request, grant_hash) + if not 0 <= cursor <= 512: + raise RelayError("ack cursor exceeds bounded chunk list") + return JSONResponse( + [e.model_dump(mode="json") for e in await relay.acks(grant_hash, cursor)] + ) + + @app.post("/v1/uploads/{grant_hash}/retention") + async def retention(grant_hash: str, request: Request) -> JSONResponse: + raw = relay_envelope.parse_envelope(await body(request)) + if raw.body.get("grant_hash") != grant_hash: + raise RelayError("retention path mismatch") + ack = await relay.retention(raw) + return JSONResponse(ack.model_dump(mode="json")) + + @app.post("/v1/uploads/{grant_hash}/release") + async def release(grant_hash: str, request: Request) -> JSONResponse: + raw = relay_envelope.parse_envelope(await body(request)) + if raw.body.get("grant_hash") != grant_hash: + raise RelayError("release path mismatch") + result = await relay.release(raw) + return JSONResponse(result.model_dump(mode="json")) + + @app.post("/v1/retrievals") + async def retrieval(request: Request) -> JSONResponse: + result = await relay.retrieve(relay_envelope.parse_envelope(await body(request))) + return JSONResponse(result.model_dump(mode="json")) + + @app.get("/v1/objects/{sha}") + async def object_stream(sha: str, request: Request) -> StreamingResponse: + size = await relay.authorize_object(sha, request.headers.get("X-Retrieval-Hash", "")) + + async def stream() -> AsyncIterator[bytes]: + with spool() as file: + await relay.backing.get(sha, size, file) + async for data in blocks(file): + yield data + + return StreamingResponse( + stream(), media_type="application/octet-stream", headers={"Content-Length": str(size)} + ) + + @app.post("/internal/drain") + async def drain(request: Request) -> JSONResponse: + token = request.headers.get("Authorization", "").removeprefix("Bearer ") + if not hmac.compare_digest(token, drain_token): + raise HTTPException(403, "drain denied") + relay.draining = True + await relay.recover() + return JSONResponse({"draining": True}) + + return app + + +@asynccontextmanager +async def runtime(app: FastAPI) -> AsyncIterator[None]: + config_bytes = await anyio.to_thread.run_sync( + Path(os.environ.get("RELAY_CONFIG", "/config/relay.json")).read_bytes + ) + config = RelayConfig.model_validate(envelope_v2.load_json(config_bytes)) + if ( + config.registry.signer != config.master_public + or config.registry.run_id != config.run_id + or config.registry.type != "RelayRegistryV1" + or not relay_envelope.verify_envelope(config.registry.model_dump()) + ): + raise RelayError("registry lacks pinned master signature") + registry = RelayRegistryV1.model_validate(config.registry.body) + if ( + config.network_manifest.signer != config.master_public + or config.network_manifest.run_id != config.run_id + or config.network_manifest.type != "RelayNetworkManifest" + or not relay_envelope.verify_envelope(config.network_manifest.model_dump()) + ): + raise RelayError("network manifest lacks pinned master signature") + network = RelayNetworkManifest.model_validate(config.network_manifest.body) + if ( + network.run_id != config.run_id + or network.registry_hash != registry.digest() + or network.specs_hash + != hashlib.sha256( + canonicalize([s.body() for s in registry.specs], allow_float=False) + ).hexdigest() + ): + raise RelayError("network manifest registry linkage mismatch") + secret_root = Path(os.environ.get("RELAY_SECRETS", "/run/relay-secrets")) + keys: dict[str, Keypair] = {} + for key_id, filename in config.signing_files.items(): + if Path(filename).name != filename: + raise RelayError("signing file must remain in mounted secret directory") + keys[key_id] = Keypair(bytes.fromhex((secret_root / filename).read_text().strip())) + drain_token = (secret_root / "drain.token").read_text().strip() + async with http_client() as client: + if config.local_backing is not None: + store: LocalFSStore | S3Store = LocalFSStore(Path(config.local_backing)) + elif config.s3_endpoint is not None and config.s3_bucket is not None: + with tempfile.NamedTemporaryFile() as credentials: + credentials.write((secret_root / "object.json").read_bytes()) + credentials.flush() + os.fchmod(credentials.fileno(), 0o600) + object_credentials = S3Credentials.from_secret_file(Path(credentials.name)) + store = S3Store( + config.s3_endpoint, + config.s3_bucket, + object_credentials, + region=config.s3_region, + prefix=config.s3_prefix, + ) + else: + raise RelayError("durable external object backing not configured") + drand = DrandQuicknet() + + async def now() -> int: + beacon = await anyio.to_thread.run_sync(drand.latest) + expected = round_at_or_after(int(time.time()), QUICKNET_GENESIS, QUICKNET_PERIOD) - 1 + if not beacon.bls_verified or abs(expected - beacon.round) > 2: + raise RelayError("verified beacon stale; eligibility/deadlines frozen") + return beacon.round + + async def forward(raw: RelayEnvelope) -> RelayEnvelope: + return await forward_receipt(client, config.master_url, raw) + + relay = Relay( + run_id=config.run_id, + master=config.master_public, + registry=registry, + network_manifest_hash=network.digest(), + observers=set(config.observer_public_keys), + relay_id=config.relay_id, + region=config.region, + keys=keys, + active_key=config.active_key, + backing=StreamStore(store, client), + now=now, + forward=forward, + ) + if min(config.registry.exp_drand, config.network_manifest.exp_drand) < await now(): + raise RelayError("pinned registry/network manifest expired") + live_app = create_app(relay, drain_token) + app.mount("/", live_app) + # Recovery is real and fail-closed; missing L0 receipt route cannot become MASTER_ACCEPTED. + await relay.recover() + yield + relay.draining = True + + +app = FastAPI(lifespan=runtime, docs_url=None, redoc_url=None, openapi_url=None) diff --git a/src/hypertrain/relay/client.py b/src/hypertrain/relay/client.py new file mode 100644 index 0000000..b7c6584 --- /dev/null +++ b/src/hypertrain/relay/client.py @@ -0,0 +1,290 @@ +"""Miner bounded upload helper, master independent receipt/retrieval verification.""" + +from __future__ import annotations + +import base64 +import hashlib +from collections.abc import AsyncIterator +from pathlib import Path +from typing import BinaryIO +from urllib.parse import urlsplit + +import anyio +import httpx +from pydantic import JsonValue + +from hypertrain.data.stream_store import BLOCK, receive, spool +from hypertrain.protocol import relay_envelope +from hypertrain.protocol.envelope import SignatureError +from hypertrain.protocol.keys import Keypair +from hypertrain.protocol.relay_envelope import RelayEnvelope +from hypertrain.protocol.relay_messages import ( + AcceptedUploadAck, + ChunkCustodyAck, + MasterAcceptanceV2, + RelayReceipt, + RelayRegistryV1, + RetrievalRequest, + RetrievalResponse, + UploadChunk, + UploadChunkManifest, + UploadGrant, +) +from hypertrain.relay.core import RelayError + + +def http_client() -> httpx.AsyncClient: + """Installed httpx only, bounded pool/timeouts, no capability redirects.""" + return httpx.AsyncClient( + timeout=httpx.Timeout(connect=5, read=30, write=30, pool=5), + limits=httpx.Limits(max_connections=8, max_keepalive_connections=4), + follow_redirects=False, + ) + + +def endpoint(registry: RelayRegistryV1, relay_id: str) -> str: + specs = [s for s in registry.specs if s.id == relay_id] + if len(specs) != 1: + raise RelayError("endpoint missing from signed registry") + return specs[0].https_url.rstrip("/") + + +def receipt_verified( + raw: RelayEnvelope, + *, + registry: RelayRegistryV1, + relay_id: str, + run_id: str, + grant: UploadGrant, + received_before: int, +) -> RelayReceipt: + """Historical receipt validity uses issued beacon, not today's rotated key.""" + receipt = RelayReceipt.model_validate(raw.body) + specs = [s for s in registry.specs if s.id == relay_id] + valid = len(specs) == 1 and any( + k.key_id == receipt.key_id + and k.pubkey == raw.signer + and k.valid_from_round <= receipt.received_drand < k.valid_until_round + for k in specs[0].pubkeys + ) + if ( + not valid + or raw.run_id != run_id + or raw.type != "RelayReceipt" + or not relay_envelope.verify_envelope(raw.model_dump()) + or receipt.grant_hash != grant.digest() + or receipt.delta_hash != grant.delta_hash + or receipt.size != grant.size + or receipt.w != grant.w + or receipt.hotkey != grant.hotkey + or receipt.chunk_manifest_hash != grant.chunk_manifest_hash + or receipt.retain_until < grant.retain_until + or receipt.received_drand > received_before + ): + raise SignatureError("forged/mismatched durability receipt") + return receipt + + +def chunk_manifest(path: Path) -> UploadChunkManifest: + size = 0 + chunks: list[UploadChunk] = [] + with path.open("rb") as file: + while data := file.read(4 << 20): + chunks.append( + UploadChunk( + index=len(chunks), + off=size, + len=len(data), + chunk_sha256=hashlib.sha256(data).hexdigest(), + ) + ) + size += len(data) + return UploadChunkManifest(size=size, chunks=chunks) + + +async def _part(file: BinaryIO, size: int) -> AsyncIterator[bytes]: + remaining = size + while remaining: + data = file.read(min(BLOCK, remaining)) + if not data: + raise RelayError("miner file shortened after manifest") + remaining -= len(data) + yield data + await anyio.lowlevel.checkpoint() + + +class RelayClient: + def __init__(self, registry: RelayRegistryV1, client: httpx.AsyncClient) -> None: + self.registry, self.client = registry, client + + async def upload( + self, + path: Path, + *, + grant_raw: RelayEnvelope, + assignment: RelayEnvelope, + round_open: dict[str, JsonValue], + ) -> RelayEnvelope: + """Original bytes only; exact duplicate uploads preserve signed acknowledgments.""" + grant = UploadGrant.model_validate(grant_raw.body) + capability = base64.b64encode(grant_raw.model_dump_json().encode()).decode() + manifest = chunk_manifest(path) + manifest.validate_grant(grant) + url = endpoint(self.registry, grant.relay_id) + response = await self.client.post( + f"{url}/v1/uploads/{grant.digest()}/accept", + json={ + "grant": grant_raw.model_dump(mode="json"), + "manifest": manifest.body(), + "assignment": assignment.model_dump(mode="json"), + "round_open": round_open, + }, + ) + response.raise_for_status() + opportunity_raw = relay_envelope.parse_envelope(response.content) + opportunity = AcceptedUploadAck.model_validate(opportunity_raw.body) + keys = next(s.pubkeys for s in self.registry.specs if s.id == grant.relay_id) + if ( + not relay_envelope.verify_envelope(opportunity_raw.model_dump()) + or opportunity_raw.run_id != grant_raw.run_id + or opportunity.grant_hash != grant.digest() + or not any( + k.pubkey == opportunity_raw.signer + and k.key_id == opportunity.key_id + and k.valid_from_round <= opportunity.accepted_beacon < k.valid_until_round + for k in keys + ) + ): + raise SignatureError("invalid opportunity acceptance") + with path.open("rb") as file: + for chunk in manifest.chunks: + response = await self.client.put( + f"{url}/v1/uploads/{grant.digest()}/chunks/{chunk.index}", + content=_part(file, chunk.len), + headers={"Content-Length": str(chunk.len), "X-Upload-Grant": capability}, + ) + response.raise_for_status() + raw = relay_envelope.parse_envelope(response.content) + ack = ChunkCustodyAck.model_validate(raw.body) + if ( + raw.signer != opportunity_raw.signer + or raw.run_id != grant_raw.run_id + or not relay_envelope.verify_envelope(raw.model_dump()) + or ack.upload_ack_hash != opportunity.digest() + or ack.grant_hash != grant.digest() + or ack.key_id != opportunity.key_id + or (ack.index, ack.off, ack.len, ack.chunk_sha256) + != (chunk.index, chunk.off, chunk.len, chunk.chunk_sha256) + ): + raise SignatureError("chunk custody mismatch") + response = await self.client.post( + f"{url}/v1/uploads/{grant.digest()}/complete", headers={"X-Upload-Grant": capability} + ) + response.raise_for_status() + raw = relay_envelope.parse_envelope(response.content) + receipt_verified( + raw, + registry=self.registry, + relay_id=grant.relay_id, + run_id=grant_raw.run_id, + grant=grant, + received_before=opportunity.service_deadline, + ) + return raw + + async def retrieval( + self, raw: RelayEnvelope, destination: BinaryIO, *, relay_public: str + ) -> RelayEnvelope: + request = RetrievalRequest.model_validate(raw.body) + url = endpoint(self.registry, request.relay_id) + response = await self.client.post(f"{url}/v1/retrievals", content=raw.model_dump_json()) + response.raise_for_status() + signed = relay_envelope.parse_envelope(response.content) + body = RetrievalResponse.model_validate(signed.body) + if ( + signed.run_id != raw.run_id + or signed.signer != relay_public + or signed.type != "RetrievalResponse" + or not relay_envelope.verify_envelope(signed.model_dump()) + or body.request_hash != request.digest() + or body.nonce != request.nonce + or body.key_id != request.key_id + or body.size != request.size + or body.object_hash != request.object_or_chunk_hash + or not request.requested_beacon <= body.served_beacon <= request.deadline_beacon + ): + raise SignatureError("retrieval response nonce/key/time binding invalid") + if body.status != "SERVED": + body.validate_request(request, None) + return signed + if body.body_sha256 != request.object_or_chunk_hash: + raise RelayError("signed SERVED hash conflicts with committed bytes") + async with self.client.stream( + "GET", + f"{url}/v1/objects/{request.object_or_chunk_hash}", + headers={"X-Retrieval-Hash": request.digest()}, + ) as stream: + stream.raise_for_status() + await receive( + stream.aiter_raw(BLOCK), destination, request.size, request.object_or_chunk_hash + ) + return signed + + async def accept_receipt( + self, + raw: RelayEnvelope, + *, + grant: UploadGrant, + request_raw: RelayEnvelope, + master: Keypair, + now: int, + deadline: int, + ) -> RelayEnvelope: + """Master requires independent original-byte retrieval, never receipt-only acceptance.""" + receipt = receipt_verified( + raw, + registry=self.registry, + relay_id=grant.relay_id, + run_id=request_raw.run_id, + grant=grant, + received_before=deadline, + ) + request = RetrievalRequest.model_validate(request_raw.body) + if ( + request.receipt_hash != receipt.digest() + or request.object_or_chunk_hash != grant.delta_hash + or request.size != grant.size + or now > deadline + ): + raise RelayError("master receipt retrieval/deadline mismatch") + with spool() as file: + response_raw = await self.retrieval(request_raw, file, relay_public=raw.signer) + response = RetrievalResponse.model_validate(response_raw.body) + if response.status != "SERVED" or max(now, response.served_beacon) > deadline: + raise RelayError("master cannot accept unreadable/late original object") + accepted = MasterAcceptanceV2( + w=grant.w, + hotkey=grant.hotkey, + delta_hash=grant.delta_hash, + receipt_hash=receipt.digest(), + received_drand=max(now, response.served_beacon), + ) + return relay_envelope.parse_envelope( + relay_envelope.seal( + master, "MasterAcceptanceV2", raw.run_id, accepted, grant.retain_until + ) + ) + + +async def forward_receipt( + client: httpx.AsyncClient, master_url: str, raw: RelayEnvelope +) -> RelayEnvelope: + """Public relay receipt route; relay holds no coordinator/admin credential.""" + parsed = urlsplit(master_url) + if parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password: + raise RelayError("master endpoint must be pinned credential-free HTTPS") + response = await client.post( + master_url.rstrip("/") + "/v2/relay-receipts", content=raw.model_dump_json() + ) + response.raise_for_status() + return relay_envelope.parse_envelope(response.content) diff --git a/src/hypertrain/relay/core.py b/src/hypertrain/relay/core.py new file mode 100644 index 0000000..91bc42e --- /dev/null +++ b/src/hypertrain/relay/core.py @@ -0,0 +1,967 @@ +"""Durable transport custody, shared replica journal, pinned signature authority. + +Metadata is an internal journal, never a replacement wire schema. One CAS includes +reservations, original signed acknowledgments, and state changes. No local cache +is authoritative. Missing master routes leave forwarding pending, never accepted. +""" + +from __future__ import annotations + +import hashlib +import secrets +from collections.abc import AsyncIterator, Awaitable, Callable +from contextlib import asynccontextmanager +from dataclasses import dataclass +from typing import BinaryIO, Literal + +import anyio +import httpx +from pydantic import BaseModel, ConfigDict, Field + +from hypertrain.data.store import CorruptObjectError, ObjectNotFound +from hypertrain.data.stream_store import StreamStore, receive, spool +from hypertrain.protocol import envelope_v2, relay_envelope +from hypertrain.protocol.envelope import SignatureError +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.keys import Keypair +from hypertrain.protocol.messages_v2 import RoundOpenV2 +from hypertrain.protocol.relay_envelope import RelayEnvelope +from hypertrain.protocol.relay_messages import ( + AcceptedUploadAck, + ChunkCustodyAck, + CustodyRelease, + MasterAcceptanceV2, + RelayAssignment, + RelayFailureEvidence, + RelayReceipt, + RelayRegistryV1, + RelaySpec, + RetentionExtension, + RetentionExtensionAck, + RetrievalRequest, + RetrievalResponse, + UploadChunkManifest, + UploadGrant, + WireModel, + extend_retention, + has_full_custody, + validate_release, +) + + +class RelayError(ValueError): + """Authenticated contract, authorization, quota or lifecycle rejection.""" + + +@dataclass(frozen=True, slots=True) +class Settlement: + """Authenticated L0 journal facts, not caller-provided JSON or a wire replacement.""" + + finality_hash: str + finality_beacon: int + closed_dispute_root: str + closed_beacon: int | None + vesting_beacon: int + + +class Pin(BaseModel): + horizon: int + retention_hash: str + seq: int = 0 + disputes: list[str] = Field(default_factory=list) + delete_after: int | None = None + + +class RetrievalRecord(BaseModel): + request: RelayEnvelope + response: RelayEnvelope + + +class ExtensionRecord(BaseModel): + extension: RelayEnvelope + acknowledgment: RelayEnvelope + + +class UploadRecord(BaseModel): + grant: RelayEnvelope + assignment: RelayEnvelope + round_open: envelope_v2.EnvelopeV2 + manifest: UploadChunkManifest + opportunity: RelayEnvelope + key_public: str + chunks: dict[str, RelayEnvelope] = Field(default_factory=dict) + receipt: RelayEnvelope | None = None + master_acceptance: RelayEnvelope | None = None + extensions: dict[str, ExtensionRecord] = Field(default_factory=dict) + releases: dict[str, RelayEnvelope] = Field(default_factory=dict) + pins: dict[str, Pin] = Field(default_factory=dict) + retrievals: dict[str, RetrievalRecord] = Field(default_factory=dict) + forward_error: str | None = None + + +class Journal(BaseModel): + model_config = ConfigDict(extra="forbid") + uploads: dict[str, UploadRecord] = Field(default_factory=dict) + nonces: dict[str, str] = Field(default_factory=dict) + failures: dict[str, RelayEnvelope] = Field(default_factory=dict) + disabled_keys: list[str] = Field(default_factory=list) + publications: dict[str, list[str]] = Field(default_factory=dict) + deletions: dict[str, str] = Field(default_factory=dict) + + @classmethod + def parse(cls, data: bytes) -> Journal: + return cls.model_validate_json(data) if data else cls() + + def serialize(self) -> bytes: + return self.model_dump_json().encode() + + +class Relay: + """Mutable service coordinator; durable state always reread through shared CAS.""" + + def __init__( + self, + *, + run_id: str, + master: str, + registry: RelayRegistryV1, + network_manifest_hash: str, + observers: set[str], + relay_id: str, + region: str, + keys: dict[str, Keypair], + active_key: str, + backing: StreamStore, + now: Callable[[], Awaitable[int]], + forward: Callable[[RelayEnvelope], Awaitable[RelayEnvelope]] | None = None, + settlement: Callable[[str], Awaitable[Settlement]] | None = None, + epoch_rounds: int = 100, + max_uploads: int = 64, + ) -> None: + specs = [s for s in registry.specs if s.id == relay_id and s.region == region] + if len(specs) != 1 or active_key not in keys: + raise RelayError("relay region/key absent from pinned registry") + self.run_id, self.master, self.registry = run_id, master, registry + self.network_manifest_hash = network_manifest_hash + self.observers = frozenset(observers) + self.spec: RelaySpec = specs[0] + self.keys, self.active_key, self.backing = keys, active_key, backing + self.now, self.forward = now, forward + self.settlement = settlement + self.epoch_rounds, self.max_uploads = epoch_rounds, max_uploads + self.draining = False + self.inflight = 0 + self.changed = anyio.Event() + self.journal_key = hashlib.sha256(f"relay-journal|{run_id}|{relay_id}".encode()).hexdigest() + self.limiter = anyio.CapacityLimiter(4) + for key in self.spec.pubkeys: + if key.key_id in keys and keys[key.key_id].ss58 != key.pubkey: + raise RelayError("private signing key does not match registry") + + async def state(self) -> Journal: + data, _ = await self.backing.journal(self.journal_key) + return Journal.parse(data) + + async def _change(self, change: Callable[[Journal], None]) -> Journal: + state = await self.backing.mutate( + self.journal_key, Journal.parse, Journal.serialize, change + ) + event, self.changed = self.changed, anyio.Event() + event.set() + return state + + def master_message[T: WireModel](self, raw: RelayEnvelope, cls: type[T], now: int) -> T: + if ( + raw.run_id != self.run_id + or raw.signer != self.master + or raw.exp_drand < now + or raw.type != cls.__name__ + or not relay_envelope.verify_envelope(raw.model_dump()) + ): + raise SignatureError("wrong relay master authority/domain/run/type/expiry") + return cls.model_validate(raw.body) + + def _seal(self, model: WireModel, key_id: str, horizon: int) -> RelayEnvelope: + return relay_envelope.parse_envelope( + relay_envelope.seal( + self.keys[key_id], type(model).__name__, self.run_id, model, horizon + ) + ) + + def _record(self, state: Journal, grant_hash: str) -> UploadRecord: + try: + return state.uploads[grant_hash] + except KeyError: + raise RelayError("unknown upload grant") from None + + def _key(self, now: int, state: Journal) -> str: + key_id = self.active_key + if key_id in state.disabled_keys or not any( + k.key_id == key_id and k.valid_from_round <= now < k.valid_until_round + for k in self.spec.pubkeys + ): + raise RelayError("active key revoked or outside signed validity interval") + return key_id + + async def accept( + self, + raw: RelayEnvelope, + manifest: UploadChunkManifest, + assignment_raw: RelayEnvelope, + round_raw: envelope_v2.EnvelopeV2, + ) -> RelayEnvelope: + now = await self.now() + grant = self.master_message(raw, UploadGrant, now) + assignment = self.master_message(assignment_raw, RelayAssignment, now) + if ( + round_raw.run_id != self.run_id + or round_raw.signer != self.master + or round_raw.type != "RoundOpenV2" + or round_raw.exp_drand < now + or not envelope_v2.verify_envelope(round_raw.model_dump()) + ): + raise SignatureError("upload deadline requires signed RoundOpenV2") + opened = RoundOpenV2.model_validate(round_raw.body) + manifest.validate_grant(grant) + if ( + grant.relay_id != self.spec.id + or grant.assignment_epoch != self.registry.epoch + or opened.registry_epoch != self.registry.epoch + or opened.w != grant.w + or assignment.w != grant.w + or assignment.hotkey != grant.hotkey + or assignment.assignment_epoch != grant.assignment_epoch + or assignment.exp_drand < now + or assignment.manifest_hash != self.network_manifest_hash + or self.spec.id not in [assignment.primary_id, *assignment.fallback_ids] + or grant.size > self.spec.max_object_bytes + ): + raise RelayError("grant is not bound to this region/assignment/round/size") + deadline = min(grant.exp_drand, opened.d_upload) + if now > deadline: + raise RelayError("upload deadline passed") + grant_hash = grant.digest() + + def commit(state: Journal) -> None: + if grant_hash in state.uploads: + existing = state.uploads[grant_hash] + if existing.manifest != manifest or existing.assignment.body != assignment_raw.body: + raise RelayError("conflicting immutable upload") + return + if self.draining: + raise RelayError("relay draining; new grants disabled") + key_id = self._key(now, state) + if len(state.uploads) >= self.max_uploads: + raise RelayError("retained upload quota exhausted") + reserved = sum( + UploadGrant.model_validate(u.grant.body).size + for u in state.uploads.values() + if u.receipt is None + and AcceptedUploadAck.model_validate(u.opportunity.body).service_deadline >= now + ) + if reserved + grant.size > self.spec.max_inflight_bytes: + raise RelayError("shared inflight byte quota exhausted") + subject = f"{grant.w}|{grant.hotkey}" + if any( + f"{u.grant.body['w']}|{u.grant.body['hotkey']}" == subject + and u.grant.body != raw.body + for u in state.uploads.values() + ): + raise RelayError("conflicting run/round/miner reservation") + if grant.nonce in state.nonces: + raise RelayError("grant nonce already reserved") + state.nonces[grant.nonce] = grant_hash + ack = AcceptedUploadAck( + grant_hash=grant_hash, + w=grant.w, + hotkey=grant.hotkey, + relay_id=grant.relay_id, + assignment_epoch=grant.assignment_epoch, + delta_hash=grant.delta_hash, + size=grant.size, + chunk_manifest_hash=grant.chunk_manifest_hash, + key_id=key_id, + accepted_beacon=now, + service_deadline=deadline, + ) + state.uploads[grant_hash] = UploadRecord( + grant=raw, + assignment=assignment_raw, + round_open=round_raw, + manifest=manifest, + opportunity=self._seal(ack, key_id, grant.retain_until), + key_public=self.keys[key_id].ss58, + ) + + state = await self._change(commit) + return state.uploads[grant_hash].opportunity + + @asynccontextmanager + async def _publication(self, sha: str) -> AsyncIterator[None]: + """Durable CAS exclusion from before write through custody acknowledgment. + + Abandoned deletion generations never expire: an old physical delete could + still execute. Recovery requires fencing that operation, not a timed lease. + """ + generation = secrets.token_hex(32) + + def reserve(state: Journal) -> None: + if sha in state.deletions: + raise RelayError("object has an outstanding durable deletion reservation") + state.publications.setdefault(sha, []).append(generation) + + await self._change(reserve) + try: + yield + finally: + + def release(state: Journal) -> None: + owners = state.publications[sha] + owners.remove(generation) + if not owners: + del state.publications[sha] + + with anyio.fail_after(30, shield=True): + await self._change(release) + + async def chunk( + self, grant_hash: str, index: int, source: AsyncIterator[bytes] + ) -> RelayEnvelope: + record = self._record(await self.state(), grant_hash) + if not 0 <= index < len(record.manifest.chunks): + raise RelayError("chunk index invalid") + async with self._publication(record.manifest.chunks[index].chunk_sha256): + return await self._chunk(grant_hash, index, source) + + async def _chunk( + self, grant_hash: str, index: int, source: AsyncIterator[bytes] + ) -> RelayEnvelope: + async with self.limiter: + now = await self.now() + record = self._record(await self.state(), grant_hash) + grant = UploadGrant.model_validate(record.grant.body) + ack = AcceptedUploadAck.model_validate(record.opportunity.body) + if now > ack.service_deadline or not 0 <= index < len(record.manifest.chunks): + raise RelayError("chunk index/deadline invalid") + expected = record.manifest.chunks[index] + if self.inflight + expected.len > self.spec.max_inflight_bytes: + raise RelayError("active stream byte quota exhausted") + with spool() as file: + self.inflight += expected.len + try: + await receive(source, file, expected.len, expected.chunk_sha256) + await self.backing.put(file, expected.chunk_sha256, expected.len) + finally: + self.inflight -= expected.len + received = await self.now() + if received > ack.service_deadline: + raise RelayError("chunk persisted after agreed deadline; no timely custody") + custody = ChunkCustodyAck( + index=expected.index, + off=expected.off, + len=expected.len, + chunk_sha256=expected.chunk_sha256, + upload_ack_hash=ack.digest(), + grant_hash=grant_hash, + durable_chunk_key=expected.chunk_sha256, + key_id=ack.key_id, + received_beacon=received, + retain_until=grant.retain_until, + ) + signed = self._seal(custody, ack.key_id, grant.retain_until) + + def commit(state: Journal) -> None: + target = self._record(state, grant_hash) + if str(index) not in target.chunks: + target.chunks[str(index)] = signed + target.pins[custody.digest()] = Pin( + horizon=grant.retain_until, retention_hash=custody.digest() + ) + + state = await self._change(commit) + target = state.uploads[grant_hash] + if len(target.chunks) == len(target.manifest.chunks): + await self.complete(grant_hash) + return target.chunks[str(index)] + + async def complete(self, grant_hash: str) -> RelayEnvelope: + record = self._record(await self.state(), grant_hash) + grant = UploadGrant.model_validate(record.grant.body) + async with self._publication(grant.delta_hash): + return await self._complete(grant_hash) + + async def _complete(self, grant_hash: str) -> RelayEnvelope: + record = self._record(await self.state(), grant_hash) + grant = UploadGrant.model_validate(record.grant.body) + opportunity = AcceptedUploadAck.model_validate(record.opportunity.body) + if record.receipt is None: + chunks = [ChunkCustodyAck.model_validate(e.body) for e in record.chunks.values()] + if not has_full_custody(grant, record.manifest, opportunity, chunks): + raise RelayError("completion requires exact timely full custody") + with spool() as whole: + for chunk in record.manifest.chunks: + with spool() as file: + await self.backing.get(chunk.chunk_sha256, chunk.len, file) + while data := file.read(64 << 10): + whole.write(data) + whole.seek(0) + await self.backing.put(whole, grant.delta_hash, grant.size) + now = await self.now() + if now > opportunity.service_deadline: + raise RelayError("complete receipt cannot backdate missed upload deadline") + if now >= grant.retain_until: + raise RelayError("custody horizon expired before complete receipt") + receipt = RelayReceipt( + w=grant.w, + hotkey=grant.hotkey, + delta_hash=grant.delta_hash, + size=grant.size, + grant_hash=grant_hash, + chunk_manifest_hash=grant.chunk_manifest_hash, + durable_object_key=grant.delta_hash, + key_id=opportunity.key_id, + received_drand=now, + retain_until=grant.retain_until, + ) + signed = self._seal(receipt, opportunity.key_id, grant.retain_until) + + def commit(state: Journal) -> None: + target = self._record(state, grant_hash) + if target.receipt is None: + target.receipt = signed + target.pins[receipt.digest()] = Pin( + horizon=grant.retain_until, retention_hash=receipt.digest() + ) + + record = (await self._change(commit)).uploads[grant_hash] + assert record.receipt is not None + if record.master_acceptance is None and self.forward is not None: + try: + acceptance_raw = await self.forward(record.receipt) + except httpx.HTTPError as error: + failure_name = type(error).__name__ + + def pending(state: Journal) -> None: + self._record(state, grant_hash).forward_error = failure_name + + await self._change(pending) + raise + accepted = self.master_message(acceptance_raw, MasterAcceptanceV2, await self.now()) + receipt = RelayReceipt.model_validate(record.receipt.body) + if ( + accepted.w != grant.w + or accepted.hotkey != grant.hotkey + or accepted.delta_hash != grant.delta_hash + or accepted.receipt_hash != receipt.digest() + or accepted.received_drand > opportunity.service_deadline + ): + raise RelayError("master acceptance differs from original receipt/deadline") + + def forwarded(state: Journal) -> None: + target = self._record(state, grant_hash) + if target.master_acceptance is None: + target.master_acceptance = acceptance_raw + target.forward_error = None + + await self._change(forwarded) + return record.receipt + + async def recover(self) -> None: + """Startup/event-triggered recovery; no invented successful master service.""" + for grant_hash, record in (await self.state()).uploads.items(): + if ( + len(record.chunks) == len(record.manifest.chunks) + and record.master_acceptance is None + ): + try: + await self.complete(grant_hash) + except RelayError as error: + # A missed completion deadline must not prevent retained-byte retrieval. + failure_name = type(error).__name__ + + def pending( + state: Journal, + grant_hash: str = grant_hash, + failure_name: str = failure_name, + ) -> None: + self._record(state, grant_hash).forward_error = failure_name + + await self._change(pending) + except httpx.HTTPError: + # Exact failure recorded by complete; custody remains retrievable. + continue + + def _scope(self, record: UploadRecord, hashes: list[str]) -> list[Pin]: + if ( + not hashes + or len(set(hashes)) != len(hashes) + or any(h not in record.pins for h in hashes) + ): + raise RelayError("unknown exact custody scope") + return [record.pins[h] for h in hashes] + + def _lineage(self, pins: list[Pin], hashes: list[str]) -> str: + if len(pins) > 1 and all(p.seq == 0 for p in pins): + return hashlib.sha256(canonicalize(sorted(hashes), allow_float=False)).hexdigest() + first = pins[0] + if any((p.seq, p.retention_hash) != (first.seq, first.retention_hash) for p in pins): + raise RelayError("mixed retention lineage") + return first.retention_hash + + async def retention(self, raw: RelayEnvelope) -> RelayEnvelope: + now = await self.now() + extension = self.master_message(raw, RetentionExtension, now) + + def commit(state: Journal) -> None: + record = self._record(state, extension.grant_hash) + if extension.digest() in record.extensions: + return + pins = self._scope(record, extension.custody_hashes) + if any(p.delete_after is not None for p in pins): + raise RelayError("released custody cannot be extended") + first = pins[0] + previous_hash = self._lineage(pins, extension.custody_hashes) + key_id = AcceptedUploadAck.model_validate(record.opportunity.body).key_id + ack = RetentionExtensionAck( + extension_hash=extension.digest(), key_id=key_id, accepted_beacon=now + ) + horizon = extend_retention( + extension, + ack, + grant_hash=extension.grant_hash, + custody_hashes=extension.custody_hashes, + key_id=key_id, + previous_hash=previous_hash, + previous_seq=first.seq, + previous_horizon=min(p.horizon for p in pins), + ) + record.extensions[extension.digest()] = ExtensionRecord( + extension=raw, acknowledgment=self._seal(ack, key_id, horizon) + ) + for pin in pins: + pin.horizon, pin.seq, pin.retention_hash = ( + horizon, + extension.seq, + extension.digest(), + ) + pin.disputes = extension.dispute_ids + + record = (await self._change(commit)).uploads[extension.grant_hash] + return record.extensions[extension.digest()].acknowledgment + + async def release(self, raw: RelayEnvelope) -> RelayEnvelope: + """L0 settlement adapter reads authenticated finality/dispute/vesting journal.""" + now = await self.now() + release = self.master_message(raw, CustodyRelease, now) + if self.settlement is None: + raise RelayError("authenticated L0 settlement journal not configured") + settled = await self.settlement(release.grant_hash) + if ( + settled.finality_hash != release.finality_hash + or settled.closed_dispute_root != release.closed_dispute_root + or settled.vesting_beacon != release.vesting_beacon + ): + raise RelayError("release lacks matching authenticated finality/closure evidence") + + def commit(state: Journal) -> None: + record = self._record(state, release.grant_hash) + pins = self._scope(record, release.custody_hashes) + if self._lineage(pins, release.custody_hashes) != release.retention_hash: + raise RelayError("release mixed retention lineage") + deadlines = [ + RetrievalRequest.model_validate(r.request.body).deadline_beacon + for r in record.retrievals.values() + ] + bound = validate_release( + release, + grant_hash=release.grant_hash, + custody_hashes=release.custody_hashes, + retention_hash=release.retention_hash, + retain_until=max(p.horizon for p in pins), + finality_beacon=settled.finality_beacon, + disputes_closed_beacon=settled.closed_beacon, + epoch_rounds=self.epoch_rounds, + retrieval_deadlines=deadlines, + ) + if now < release.release_beacon: + raise RelayError("release grace has not elapsed") + record.releases[release.digest()] = raw + for pin in pins: + pin.delete_after = bound + pin.disputes = [] + + await self._change(commit) + return raw + + async def retrieve(self, raw: RelayEnvelope) -> RelayEnvelope: + now = await self.now() + request = self.master_message(raw, RetrievalRequest, now) + record = self._record(await self.state(), request.grant_hash) + opportunity = AcceptedUploadAck.model_validate(record.opportunity.body) + if ( + request.relay_id != self.spec.id + or request.key_id != opportunity.key_id + or request.assignment_hash + != RelayAssignment.model_validate(record.assignment.body).digest() + or not request.requested_beacon <= now <= request.deadline_beacon + ): + raise RelayError("retrieval wrong assignment/key/beacon") + hashes = [*request.custody_ack_hashes] + if request.receipt_hash is not None: + hashes.append(request.receipt_hash) + pins = self._scope(record, hashes) + if self._lineage(pins, hashes) != request.retention_hash: + raise RelayError("retrieval wrong signed retention lineage") + request.validate_horizon( + min(p.horizon for p in pins), released=any(p.delete_after is not None for p in pins) + ) + grant = UploadGrant.model_validate(record.grant.body) + exact = False + if request.receipt_hash is not None and record.receipt is not None: + receipt = RelayReceipt.model_validate(record.receipt.body) + exact = receipt.digest() == request.receipt_hash and ( + request.object_or_chunk_hash, + request.size, + ) == (grant.delta_hash, grant.size) + if request.custody_ack_hashes: + chunks = [ + ChunkCustodyAck.model_validate(e.body) + for e in record.chunks.values() + if ChunkCustodyAck.model_validate(e.body).digest() in request.custody_ack_hashes + ] + exact |= ( + len(chunks) == 1 + and (request.object_or_chunk_hash, request.size) + == (chunks[0].chunk_sha256, chunks[0].len) + ) or ( + has_full_custody(grant, record.manifest, opportunity, chunks) + and (request.object_or_chunk_hash, request.size) == (grant.delta_hash, grant.size) + ) + if not exact: + raise RelayError("retrieval bytes not covered by exact signed custody") + previous = record.retrievals.get(request.request_id) + if previous is not None: + if previous.request.body != raw.body: + raise RelayError("conflicting retrieval reservation") + return previous.response + status: Literal["SERVED", "NOT_FOUND", "UNAVAILABLE"] = "SERVED" + error: str | None = None + try: + with spool() as file: + await self.backing.get(request.object_or_chunk_hash, request.size, file) + except ObjectNotFound: + status, error = "NOT_FOUND", "missing-custody" + except CorruptObjectError: + status, error = "UNAVAILABLE", "corrupt-custody" + served = await self.now() + if served > request.deadline_beacon: + raise RelayError("retrieval deadline elapsed") + response = RetrievalResponse( + request_hash=request.digest(), + nonce=request.nonce, + key_id=request.key_id, + status=status, + object_hash=request.object_or_chunk_hash, + size=request.size, + served_beacon=served, + body_sha256=request.object_or_chunk_hash, + error_code=error, + ) + signed = self._seal(response, request.key_id, min(p.horizon for p in pins)) + + def commit(state: Journal) -> None: + target = self._record(state, request.grant_hash) + current_pins = self._scope(target, hashes) + if self._lineage(current_pins, hashes) != request.retention_hash: + raise RelayError("retrieval retention changed during backing read") + request.validate_horizon( + min(p.horizon for p in current_pins), + released=any(p.delete_after is not None for p in current_pins), + ) + if request.request_id in target.retrievals: + if target.retrievals[request.request_id].request.body != raw.body: + raise RelayError("conflicting retrieval reservation") + return + if request.nonce in state.nonces: + raise RelayError("retrieval nonce replay") + if any( + RetrievalRequest.model_validate(r.request.body).object_or_chunk_hash + == request.object_or_chunk_hash + and RetrievalRequest.model_validate(r.request.body).deadline_beacon >= served + for r in target.retrievals.values() + ): + raise RelayError("one outstanding challenge per exact object/key") + if len(target.retrievals) >= 512: + raise RelayError("retrieval journal quota exhausted") + state.nonces[request.nonce] = request.digest() + target.retrievals[request.request_id] = RetrievalRecord(request=raw, response=signed) + + target = (await self._change(commit)).uploads[request.grant_hash] + return target.retrievals[request.request_id].response + + async def authorize_object(self, sha: str, request_hash: str) -> int: + """Only a persisted signed nonce challenge opens an object stream.""" + now = await self.now() + for record in (await self.state()).uploads.values(): + for retrieval in record.retrievals.values(): + request = RetrievalRequest.model_validate(retrieval.request.body) + response = RetrievalResponse.model_validate(retrieval.response.body) + if ( + request.digest() == request_hash + and request.object_or_chunk_hash == sha + and response.status == "SERVED" + and now <= request.deadline_beacon + ): + pins = self._scope( + record, + [ + *request.custody_ack_hashes, + *([request.receipt_hash] if request.receipt_hash else []), + ], + ) + if not any(p.delete_after is not None for p in pins): + return request.size + raise RelayError("object stream requires live authenticated custody challenge") + + async def ready(self) -> bool: + if self.draining or self.inflight >= self.spec.max_inflight_bytes: + return False + now = await self.now() + state = await self.state() + self._key(now, state) + reserved = sum( + UploadGrant.model_validate(record.grant.body).size + for record in state.uploads.values() + if record.receipt is None + and AcceptedUploadAck.model_validate(record.opportunity.body).service_deadline >= now + ) + if len(state.uploads) >= self.max_uploads or reserved >= self.spec.max_inflight_bytes: + return False + # Persisted CAS proves write privilege; independent object probe proves backing readability. + await self._change(lambda state: None) + with spool() as file: + file.write(b"relay-durable-probe-v1") + sha = hashlib.sha256(b"relay-durable-probe-v1").hexdigest() + await self.backing.put(file, sha, 22) + with spool() as file: + await self.backing.get(sha, 22, file) + return True + + async def acks(self, grant_hash: str, cursor: int = 0) -> list[RelayEnvelope]: + record = self._record(await self.state(), grant_hash) + return [record.chunks[str(i)] for i in sorted(map(int, record.chunks)) if i >= cursor] + + def _eligible_deletions(self, state: Journal, now: int) -> list[str]: + """Evidence remains pinned without authenticated release, even past horizon.""" + candidates: set[str] = set() + protected: set[str] = set() + for record in state.uploads.values(): + entries = [ + ( + ChunkCustodyAck.model_validate(e.body).digest(), + ChunkCustodyAck.model_validate(e.body).chunk_sha256, + ) + for e in record.chunks.values() + ] + if record.receipt is not None: + receipt = RelayReceipt.model_validate(record.receipt.body) + entries.append((receipt.digest(), receipt.delta_hash)) + for digest, sha in entries: + pin = record.pins[digest] + if pin.delete_after is not None and now >= pin.delete_after and not pin.disputes: + candidates.add(sha) + else: + protected.add(sha) + return sorted(candidates - protected - state.publications.keys() - state.deletions.keys()) + + async def eligible_deletions(self) -> list[str]: + return self._eligible_deletions(await self.state(), await self.now()) + + async def collect(self) -> list[str]: + """CAS reserves current eligibility through physical delete across replicas.""" + now = await self.now() + generation = secrets.token_hex(32) + + def reserve(state: Journal) -> None: + for sha in self._eligible_deletions(state, now): + state.deletions[sha] = generation + + reserved = await self._change(reserve) + eligible = [sha for sha, owner in reserved.deletions.items() if owner == generation] + for sha in eligible: + await self.backing.delete(sha) + + def finish(state: Journal, sha: str = sha) -> None: + if state.deletions.get(sha) != generation: + raise RelayError("deletion generation changed before physical completion") + del state.deletions[sha] + + await self._change(finish) + return eligible + + async def failure( + self, + raw: RelayEnvelope, + artifacts: list[RelayEnvelope], + observations: list[RelayEnvelope], + payload: BinaryIO | None = None, + ) -> RelayFailureEvidence: + """L0 authenticates observer health. Relay verifies signed custody/equivocation. + + This records key-specific service failures; never malicious intent/miner slash. + """ + now = await self.now() + evidence = self.master_message(raw, RelayFailureEvidence, now) + record = self._record(await self.state(), evidence.grant_hash) + grant = UploadGrant.model_validate(record.grant.body) + opportunity = AcceptedUploadAck.model_validate(record.opportunity.body) + if ( + evidence.run_id != self.run_id + or evidence.relay_id != self.spec.id + or evidence.key_id != opportunity.key_id + or evidence.assignment_hash + != RelayAssignment.model_validate(record.assignment.body).digest() + or evidence.upload_ack_hash != opportunity.digest() + or evidence.chunk_manifest_hash != grant.chunk_manifest_hash + or max(evidence.observed_beacons) > now + or evidence.evidence_hash + != hashlib.sha256( + canonicalize([e.model_dump(mode="json") for e in artifacts], allow_float=False) + ).hexdigest() + ): + raise RelayError("failure evidence binding mismatch") + for artifact in artifacts: + if artifact.run_id != self.run_id or not relay_envelope.verify_envelope( + artifact.model_dump() + ): + raise SignatureError("invalid evidence signature") + if evidence.result == "CONTRACTUAL_SERVICE_FAILURE": + authenticated = { + observation.signer + for observation in observations + if observation.signer in self.observers + and observation.run_id == self.run_id + and observation.type == "RelayFailureEvidence" + and observation.body == raw.body + and relay_envelope.verify_envelope(observation.model_dump()) + } + if authenticated != set(evidence.observer_ids) or len(authenticated) != 2: + raise SignatureError("service failure needs two pinned observer signatures") + verified = False + match evidence.result: + case "UNCONFIRMED_AVAILABILITY": + verified = True + case "EQUIVOCATION": + for left in artifacts: + for right in artifacts: + if ( + left.signer == right.signer == record.key_public + and left.type == right.type + and left.type + in {"RelayReceipt", "ChunkCustodyAck", "AcceptedUploadAck"} + and left.body.get("grant_hash") + == right.body.get("grant_hash") + == grant.digest() + and left.body.get("index") == right.body.get("index") + and left.body != right.body + ): + verified = True + case "HASH_INCONSISTENCY": + request, response = evidence.request, evidence.response + if request is not None and response is not None and payload is not None: + signed_response = [ + e + for e in artifacts + if e.type == "RetrievalResponse" + and e.signer == record.key_public + and e.body == response.body() + ] + served = record.retrievals.get(request.request_id) + payload.seek(0) + digest, size = hashlib.sha256(), 0 + while data := payload.read(64 << 10): + size += len(data) + if size > request.size: + break + digest.update(data) + verified = ( + bool(signed_response) + and served is not None + and served.request.body == request.body() + and response.status == "SERVED" + and response.request_hash == request.digest() + and response.nonce == request.nonce + and response.key_id == request.key_id + and request.requested_beacon + <= response.served_beacon + <= request.deadline_beacon + and ( + response.body_sha256 != request.object_or_chunk_hash + or size != request.size + or digest.hexdigest() != response.body_sha256 + ) + ) + case "CONTRACTUAL_SERVICE_FAILURE": + custody_scope = [ + *evidence.chunk_ack_hashes, + *([evidence.receipt_hash] if evidence.receipt_hash else []), + ] + obligated = self._scope(record, custody_scope) + if ( + self._lineage(obligated, custody_scope) != evidence.retention_hash + or any(p.delete_after is not None for p in obligated) + or max(evidence.observed_beacons) > min(p.horizon for p in obligated) + ): + raise RelayError("failure observed outside exact signed retention obligation") + chunks = [ + ChunkCustodyAck.model_validate(e.body) + for e in record.chunks.values() + if ChunkCustodyAck.model_validate(e.body).digest() in evidence.chunk_ack_hashes + ] + receipt = ( + RelayReceipt.model_validate(record.receipt.body) + if record.receipt is not None + and evidence.receipt_hash + == RelayReceipt.model_validate(record.receipt.body).digest() + else None + ) + if evidence.request is None: + verified = ( + has_full_custody(grant, record.manifest, opportunity, chunks, receipt) + and min(evidence.observed_beacons) > opportunity.service_deadline + and record.master_acceptance is None + ) + else: + request = evidence.request + if ( + request.grant_hash != evidence.grant_hash + or request.assignment_hash != evidence.assignment_hash + or request.relay_id != evidence.relay_id + or request.key_id != evidence.key_id + or request.custody_ack_hashes != evidence.chunk_ack_hashes + or request.receipt_hash != evidence.receipt_hash + or request.retention_hash != evidence.retention_hash + ): + raise RelayError("retrieval failure differs from exact requested custody") + served = record.retrievals.get(request.request_id) + verified = ( + served is not None + and served.request.body == request.body() + and evidence.response is not None + and served.response.body == evidence.response.body() + and evidence.response.status != "SERVED" + and min(evidence.observed_beacons) > request.deadline_beacon + ) + if not verified: + raise RelayError("failure lacks exact authenticated byte-custody predicate") + + def commit(state: Journal) -> None: + state.failures[evidence.digest()] = raw + if ( + evidence.result != "UNCONFIRMED_AVAILABILITY" + and evidence.key_id not in state.disabled_keys + ): + state.disabled_keys.append(evidence.key_id) + + await self._change(commit) + return evidence diff --git a/src/hypertrain/trainer/compress.py b/src/hypertrain/trainer/compress.py index 76b591f..46d47bc 100644 --- a/src/hypertrain/trainer/compress.py +++ b/src/hypertrain/trainer/compress.py @@ -16,6 +16,7 @@ import hashlib import math import struct +from collections.abc import Mapping import numpy as np import torch @@ -176,7 +177,14 @@ def compress(cfg: CompressConfig, delta: Params, ef_in: Params) -> tuple[bytes, return b"".join(out), ef_out -def decompress(payload: bytes) -> tuple[str, Params]: +def _payload_metadata( + payload: bytes, + expected_shapes: Mapping[str, tuple[int, ...]] | None, + max_payload_bytes: int | None, +) -> tuple[str, dict[str, tuple[tuple[int, ...], int, int]]]: + """Parse all framing and validate bodies before any tensor allocation.""" + if max_payload_bytes is not None and len(payload) > max_payload_bytes: + raise ValueError("delta payload exceeds byte limit") codec = next((c for c, m in MAGIC.items() if payload.startswith(m)), None) if codec is None: raise ValueError("unknown delta payload magic") @@ -184,23 +192,65 @@ def decompress(payload: bytes) -> tuple[str, Params]: try: (count,) = struct.unpack_from(" (len(payload) - off) // 20: + raise ValueError("tensor count exceeds payload framing") + if expected_shapes is not None and count != len(expected_shapes): + raise ValueError("tensor count differs from expected shapes") + out: dict[str, tuple[tuple[int, ...], int, int]] = {} for _ in range(count): (ln,) = struct.unpack_from(" len(payload) - off - 8: + raise ValueError("truncated tensor name") + raw_name = payload[off + 8 : off + 8 + ln] + name = raw_name.decode("utf-8") + if not name or (out and raw_name <= next(reversed(out)).encode("utf-8")): + raise ValueError("tensor names must be unique and UTF-8 sorted") + if expected_shapes is not None and name not in expected_shapes: + raise ValueError("unexpected tensor name") off += 8 + ln (ndim,) = struct.unpack_from(" (len(payload) - off - 4) // 8: + raise ValueError("tensor rank exceeds payload framing") + if expected_shapes is not None and ndim != len(expected_shapes[name]): + raise ValueError("tensor rank differs from expected shape") shape = struct.unpack_from(f"<{ndim}Q", payload, off + 4) + if expected_shapes is not None and shape != expected_shapes[name]: + raise ValueError("tensor dimensions differ from expected shape") off += 4 + 8 * ndim (blen,) = struct.unpack_from(" len(payload) - off - 8: raise ValueError("truncated tensor body") + start = off + 8 off += 8 + blen - if name in out: - raise ValueError("duplicate tensor in payload") - dec = _int8_decode if codec == "dense-int8" else _sparse_decode - out[name] = dec(body, tuple(shape)) + body = memoryview(payload)[start:off] + n = math.prod(shape) + if codec == "dense-int8": + blocks = (n + BLOCK - 1) // BLOCK + if blen != 4 + 4 * blocks + n or struct.unpack_from("> (2 * (k % 4)): + raise ValueError("nonzero sparse padding bits") + out[name] = (tuple(shape), start, off) except struct.error as exc: raise ValueError("truncated delta payload") from exc if off != len(payload): @@ -208,5 +258,28 @@ def decompress(payload: bytes) -> tuple[str, Params]: return codec, out +def validate_payload( + payload: bytes, + expected_shapes: Mapping[str, tuple[int, ...]], + *, + max_payload_bytes: int = 65536, +) -> str: + """Bound raw bytes and validate manifest geometry without decoding tensors.""" + codec, _ = _payload_metadata(payload, expected_shapes, max_payload_bytes) + return codec + + +def decompress( + payload: bytes, + *, + expected_shapes: Mapping[str, tuple[int, ...]] | None = None, + max_payload_bytes: int | None = None, +) -> tuple[str, Params]: + """Decode only after every tensor passes framing and optional manifest bounds.""" + codec, metadata = _payload_metadata(payload, expected_shapes, max_payload_bytes) + dec = _int8_decode if codec == "dense-int8" else _sparse_decode + return codec, {n: dec(payload[start:end], shape) for n, (shape, start, end) in metadata.items()} + + def payload_hash(payload: bytes) -> str: return hashlib.sha256(payload).hexdigest() diff --git a/src/hypertrain/trainer/config.py b/src/hypertrain/trainer/config.py index 46d8e50..59695cc 100644 --- a/src/hypertrain/trainer/config.py +++ b/src/hypertrain/trainer/config.py @@ -7,6 +7,7 @@ from typing import Any from hypertrain.protocol.messages import ODSpec, RunManifest, f32val +from hypertrain.protocol.messages_v2 import RunManifestV2, TrainingManifestV2 DTYPES = ("fp32", "bf16") STATE_POLICIES = ("reset", "derived", "carry") @@ -135,6 +136,16 @@ def __post_init__(self) -> None: def from_manifest(cls, body: Mapping[str, Any]) -> TrainConfig: """Validate ``body`` as a protocol RunManifest (strict, f32-hex reals) and convert.""" m = RunManifest.model_validate(body) + return cls._convert(m) + + @classmethod + def from_manifest_v2(cls, wrapper: RunManifestV2) -> TrainConfig: + """Consume the strict wrapper; embedded training never supplies the run ID.""" + wrapper = RunManifestV2.model_validate(wrapper.body()) + return cls._convert(wrapper.training) + + @classmethod + def _convert(cls, m: RunManifest | TrainingManifestV2) -> TrainConfig: ms, inn, out, ref = m.model, m.inner, m.outer, m.reference_spec model = ModelConfig( n_layers=ms.n_layers, diff --git a/src/hypertrain/trainer/island.py b/src/hypertrain/trainer/island.py index c4e4f5a..4861c60 100644 --- a/src/hypertrain/trainer/island.py +++ b/src/hypertrain/trainer/island.py @@ -31,6 +31,7 @@ from hypertrain.protocol.hashing import MerkleTree from hypertrain.protocol.messages import Layout, RunManifest +from hypertrain.protocol.messages_v2 import RunManifestV2 from hypertrain.trainer.compress import compress, payload_hash, state_hash from hypertrain.trainer.config import ModelConfig, TrainConfig from hypertrain.trainer.determinism import require_threads @@ -64,8 +65,56 @@ class ForbiddenCollective(RuntimeError): """A non-fixed-order reduction was attempted on verified tensors.""" +@dataclass(frozen=True, slots=True) +class TraceContext: + rank: int + step: int + microbatch: int + layer: int + op: str + + +TraceHook = Callable[[TraceContext, Tensor], Tensor] +CancelHook = Callable[[], None] + + +class HookComm: + """Actual collective boundary hooks; identity hooks leave operation order unchanged.""" + + def __init__(self, comm: Comm, hook: TraceHook | None, cancel: CancelHook | None) -> None: + self.comm, self.hook, self.cancel = comm, hook, cancel + self.rank, self.world = comm.rank, comm.world + self.step = 0 + + def emit(self, op: str, x: Tensor) -> Tensor: + if self.cancel is not None: + self.cancel() + return ( + x if self.hook is None else self.hook(TraceContext(self.rank, self.step, -1, -1, op), x) + ) + + def all_gather(self, t: Tensor) -> list[Tensor]: + got = self.comm.all_gather(self.emit("all_gather.input", t)) + return [self.emit(f"all_gather.rank.{r}", x) for r, x in enumerate(got)] + + def all_to_all(self, x: Tensor, send: list[int], recv: list[int]) -> Tensor: + return self.emit( + "all_to_all.output", self.comm.all_to_all(self.emit("all_to_all.input", x), send, recv) + ) + + def all_reduce_sum(self, t: Tensor) -> Tensor: + return self.comm.all_reduce_sum(t) + + def island_from_manifest(body: Mapping[str, Any]) -> tuple[TrainConfig, Layout, str]: """Validate ``body`` as a protocol RunManifest; return (train config, layout, run_id).""" + if body.get("manifest_version") == 2: + wrapper = RunManifestV2.model_validate(body) + return ( + TrainConfig.from_manifest_v2(wrapper), + wrapper.training.reference_spec.layout, + wrapper.run_id(), + ) m = RunManifest.model_validate(body) lay = m.reference_spec.layout cfg = TrainConfig.from_manifest(body) @@ -293,12 +342,13 @@ def gather_pieces( values: Mapping[str, Tensor], which: Sequence[Piece], src: Callable[[Piece], int], + device: torch.device, ) -> dict[Piece, Tensor]: """Deterministic gather: rank src(p) publishes values[p.name]; one padded all_gather per tensor name (bounds the transient buffer to world x one tensor).""" out: dict[Piece, Tensor] = {} for name in sorted({p[0] for p in which}): - out |= _gather_group(comm, geo, values, [p for p in which if p[0] == name], src) + out |= _gather_group(comm, geo, values, [p for p in which if p[0] == name], src, device) return out @@ -308,12 +358,12 @@ def _gather_group( values: Mapping[str, Tensor], which: Sequence[Piece], src: Callable[[Piece], int], + device: torch.device, ) -> dict[Piece, Tensor]: by_rank = [[p for p in which if src(p) == r] for r in range(comm.world)] sizes = [sum(math.prod(geo.piece_shape(p)) for p in ps) for ps in by_rank] mine = [values[p[0]].reshape(-1) for p in by_rank[comm.rank]] - dev = next(iter(values.values())).device - pad = torch.zeros(max(sizes) - sizes[comm.rank], dtype=torch.float32, device=dev) + pad = torch.zeros(max(sizes) - sizes[comm.rank], dtype=torch.float32, device=device) got = comm.all_gather(torch.cat([*mine, pad])) out: dict[Piece, Tensor] = {} for r, ps in enumerate(by_rank): @@ -385,12 +435,21 @@ def ep_dispatch(comm: Comm, geo: Geometry, rows: Tensor, counts: Tensor) -> tupl torch.tensor(perm, dtype=torch.int64, device=rows.device), [sum(c[i] for c in rc) for i in range(eq)], ) - return permute_rows(all_to_all(rows, comm, send, recv), plan.perm), plan + if isinstance(comm, HookComm): + plan = EPPlan( + plan.send, plan.recv, comm.emit("ep.dispatch.permutation", plan.perm), plan.per_expert + ) + dispatched = permute_rows(all_to_all(rows, comm, send, recv), plan.perm) + if isinstance(comm, HookComm): + dispatched = comm.emit("ep.dispatch.rows", dispatched) + return dispatched, plan def ep_combine(comm: Comm, plan: EPPlan, outs: Tensor) -> Tensor: """Inverse of ep_dispatch: [expert][src] rows back to the sender, sorted by global expert.""" back = permute_rows(outs, torch.argsort(plan.perm, stable=True)) + if isinstance(comm, HookComm): + back = comm.emit("ep.combine.rows", back) return all_to_all(back, comm, plan.recv, plan.send) @@ -430,7 +489,14 @@ def moe( def _rank_grads( - cfg: TrainConfig, comm: Comm, geo: Geometry, theta: Params, ids: Sequence[int], get: SampleFn + cfg: TrainConfig, + comm: Comm, + geo: Geometry, + theta: Params, + ids: Sequence[int], + get: SampleFn, + t: int = 0, + hook: TraceHook | None = None, ) -> tuple[Params, float]: """Mirror of loop._train_step on this rank's samples (no optimizer step).""" names = sorted(theta) @@ -441,7 +507,11 @@ def _rank_grads( for j in range(ga): params = {n: theta[n].detach().requires_grad_(True) for n in names} tokens = _load_batch(cfg, ids[j * mb : (j + 1) * mb], get, theta[names[0]].device) - ce, aux = forward(cfg.model, params, tokens, moe=moe) + + def emit(layer: int, op: str, x: Tensor, microbatch: int = j) -> Tensor: + return x if hook is None else hook(TraceContext(comm.rank, t, microbatch, layer, op), x) + + ce, aux = forward(cfg.model, params, tokens, moe=moe, hook=emit if hook else None) total = (ce + cfg.model.aux_loss_coef * aux) / ga gs = torch.autograd.grad(total, [params[n] for n in names], allow_unused=True) for n, g in zip(names, gs, strict=True): @@ -476,7 +546,11 @@ def _reduce(comm: Comm, geo: Geometry, grads: Params, reduction: str) -> Params: def _grad_norm(comm: Comm, geo: Geometry, red: Params) -> float: """optim.global_grad_norm over the full (all-shard) gradient, summed in fixed order.""" names = sorted(red) - parts = torch.tensor([float(red[n].double().pow(2).sum()) for n in names], dtype=torch.float64) + parts = torch.tensor( + [float(red[n].double().pow(2).sum()) for n in names], + dtype=torch.float64, + device=red[names[0]].device, + ) got = comm.all_gather(parts) total = 0.0 for i, n in enumerate(names): @@ -488,8 +562,8 @@ def _grad_norm(comm: Comm, geo: Geometry, red: Params) -> float: return math.sqrt(total) -def _mean_loss(comm: Comm, loss: float) -> float: - got = comm.all_gather(torch.tensor([loss], dtype=torch.float64)) +def _mean_loss(comm: Comm, loss: float, device: torch.device) -> float: + got = comm.all_gather(torch.tensor([loss], dtype=torch.float64, device=device)) acc = float(got[0][0]) for x in got[1:]: acc += float(x[0]) @@ -507,6 +581,10 @@ def train_island( carry: OptState | None = None, v0: Params | None = None, reduction: str = "all_gather", + hook: TraceHook | None = None, + cancel: CancelHook | None = None, + after_step: Callable[[int, Params], None] | None = None, + checkpoint: Callable[[int, Params, OptState], None] | None = None, ) -> RoundResult: """One H-step round on this rank; every rank returns the same full RoundResult. @@ -515,7 +593,20 @@ def train_island( guard = forbid_reductions() if isinstance(comm, DistComm) else nullcontext() with guard: return _train_island( - cfg, lay, comm, theta_start, a, get_sample, ef_in, carry, v0, reduction + cfg, + lay, + comm, + theta_start, + a, + get_sample, + ef_in, + carry, + v0, + reduction, + hook, + cancel, + after_step, + checkpoint, ) @@ -530,13 +621,33 @@ def _train_island( carry: OptState | None = None, v0: Params | None = None, reduction: str = "all_gather", + hook: TraceHook | None = None, + cancel: CancelHook | None = None, + after_step: Callable[[int, Params], None] | None = None, + checkpoint: Callable[[int, Params, OptState], None] | None = None, ) -> RoundResult: require_threads(cfg.cpu_threads) check_launch(lay, comm.world) + traced = HookComm(comm, hook, cancel) + comm = traced if sorted(theta_start) != sorted(param_shapes(cfg.model)): raise ValueError("theta_start does not match the model parameter set") if any(x.dtype != torch.float32 for x in theta_start.values()): raise ValueError("theta_start must be fp32 master weights") + if any(tuple(theta_start[n].shape) != s for n, s in param_shapes(cfg.model).items()): + raise ValueError("theta_start shapes differ from model") + if carry is not None: + adam_names = {n for n in theta_start if not uses_muon(cfg.inner, n)} + if set(carry.m) != set(theta_start) or set(carry.v) != adam_names: + raise ValueError("carried full state does not match the model") + if carry.step != a.global_step0: + raise ValueError("carried optimizer step differs from assignment") + if any( + x.dtype != torch.float32 or x.shape != theta_start[n].shape + for values in (carry.m, carry.v) + for n, x in values.items() + ): + raise ValueError("carried full state shape/dtype mismatch") if ( cfg.model.is_moe and cfg.model.n_experts % lay.ep_size @@ -565,34 +676,64 @@ def _train_island( adam = [p for p in pieces if not uses_muon(cfg.inner, p[0])] def full() -> tuple[Params, OptState]: - th = _assemble(geo, gather_pieces(comm, geo, theta, pieces, geo.owner), names) - m = _assemble(geo, gather_pieces(comm, geo, st.m, pieces, geo.owner), names) + device = theta[names[0]].device + th = _assemble(geo, gather_pieces(comm, geo, theta, pieces, geo.owner, device), names) + m = _assemble(geo, gather_pieces(comm, geo, st.m, pieces, geo.owner, device), names) adam_names = sorted({p[0] for p in adam}) - v = _assemble(geo, gather_pieces(comm, geo, st.v, adam, geo.owner), adam_names) + v = _assemble(geo, gather_pieces(comm, geo, st.v, adam, geo.owner, device), adam_names) return th, OptState(m, v, st.step) th_full, st_full = full() leaves = [_make_leaf(cfg, ia, 0, th_full, st_full, 0.0, 0.0)] + if checkpoint is not None: + checkpoint(0, th_full, st_full) window_start = th_full del th_full, st_full # memory: only window_start (theta) is needed between leaves per = mb * ga for t in range(1, H + 1): + traced.step = t + if cancel is not None: + cancel() ids = ia.batch_ids(cfg, t)[comm.rank * per : (comm.rank + 1) * per] - grads, loss_local = _rank_grads(cfg, comm, geo, theta, ids, get_sample) + grads, loss_local = _rank_grads(cfg, comm, geo, theta, ids, get_sample, t, hook) red = _reduce(comm, geo, grads, reduction) + if hook is not None: + red = { + n: hook(TraceContext(comm.rank, t, -1, cfg.model.n_layers + 1, f"gradient.{n}"), x) + for n, x in red.items() + } del grads norm = _grad_norm(comm, geo, red) lr = lr_at(cfg.inner, a.global_step0 + t - 1, t - 1) step(cfg.inner, {n: theta[n] for n in owned}, red, st, lr, norm=norm) + if after_step is not None: + after_step(t, theta) + if hook is not None: + for n in owned: + theta[n] = hook( + TraceContext(comm.rank, t, -1, cfg.model.n_layers + 1, f"optimizer.theta.{n}"), + theta[n], + ) + st.m[n] = hook( + TraceContext(comm.rank, t, -1, cfg.model.n_layers + 1, f"optimizer.m.{n}"), + st.m[n], + ) + if n in st.v: + st.v[n] = hook( + TraceContext(comm.rank, t, -1, cfg.model.n_layers + 1, f"optimizer.v.{n}"), + st.v[n], + ) del red if lay.zero1 and lay.n_gpus > 1: - synced = gather_pieces(comm, geo, theta, pieces, geo.owner) + synced = gather_pieces(comm, geo, theta, pieces, geo.owner, theta[names[0]].device) for n in names: theta[n].copy_(synced[geo.my_piece(n)]) del synced - loss = _mean_loss(comm, loss_local) + loss = _mean_loss(comm, loss_local, theta[names[0]].device) if t % J == 0: th_full, st_full = full() + if checkpoint is not None: + checkpoint(t, th_full, st_full) leaves.append( _make_leaf(cfg, ia, t, th_full, st_full, loss, _norm(th_full, window_start)) ) diff --git a/src/hypertrain/trainer/model.py b/src/hypertrain/trainer/model.py index cf31188..e9acf15 100644 --- a/src/hypertrain/trainer/model.py +++ b/src/hypertrain/trainer/model.py @@ -226,28 +226,49 @@ def _moe( MoeFn = Callable[[ModelConfig, Params, str, Tensor, torch.dtype], tuple[Tensor, Tensor]] +ForwardHook = Callable[[int, str, Tensor], Tensor] def forward( - cfg: ModelConfig, p: Params, tokens: Tensor, moe: MoeFn | None = None + cfg: ModelConfig, + p: Params, + tokens: Tensor, + moe: MoeFn | None = None, + hook: ForwardHook | None = None, ) -> tuple[Tensor, Tensor]: """tokens int64 [B, T+1] -> (mean CE loss fp32, aux loss fp32); ``moe`` swaps the MoE block.""" if cfg.arch != "decoder": - return _arch_impl(cfg).forward(cfg, p, tokens, moe=moe) + return _arch_impl(cfg).forward(cfg, p, tokens, moe=moe, hook=hook) + + def emit(layer: int, op: str, x: Tensor) -> Tensor: + return x if hook is None else hook(layer, op, x) + moe_fn = moe or _moe dt = compute_dtype(cfg) inp, tgt = tokens[:, :-1], tokens[:, 1:] - x = embed(p["emb.weight"], inp).to(dt) + emit(-1, "input", tokens) + x = emit(0, "embed", embed(p["emb.weight"], inp).to(dt)) aux = torch.zeros((), dtype=torch.float32, device=tokens.device) for i in range(cfg.n_layers): pre = f"layers.{i:03d}." - x = x + _attn(cfg, p, pre, _rms(x, p[pre + "attn_norm"]), dt) - h = _rms(x, p[pre + "mlp_norm"]) + h = emit(i, "attn_norm", _rms(x, p[pre + "attn_norm"])) + attn = emit(i, "attn", _attn(cfg, p, pre, h, dt)) + x = emit(i, "attn_residual", x + attn) + h = emit(i, "mlp_norm", _rms(x, p[pre + "mlp_norm"])) if cfg.is_moe: m, a = moe_fn(cfg, p, pre, h, dt) - x, aux = x + m, aux + a + m = emit(i, "mlp", m) + x, aux = emit(i, "mlp_residual", x + m), aux + a else: - x = x + _swiglu(h, p[pre + "w1"].to(dt), p[pre + "w3"].to(dt), p[pre + "w2"].to(dt)) - logits = (_rms(x, p["norm.weight"]) @ p["head.weight"].to(dt)).float() - loss = F.cross_entropy(logits.reshape(-1, cfg.vocab), tgt.reshape(-1)) + m = emit( + i, + "mlp", + _swiglu(h, p[pre + "w1"].to(dt), p[pre + "w3"].to(dt), p[pre + "w2"].to(dt)), + ) + x = emit(i, "mlp_residual", x + m) + norm = emit(cfg.n_layers, "final_norm", _rms(x, p["norm.weight"])) + logits = emit(cfg.n_layers, "head", (norm @ p["head.weight"].to(dt)).float()) + loss = emit( + cfg.n_layers, "loss", F.cross_entropy(logits.reshape(-1, cfg.vocab), tgt.reshape(-1)) + ) return loss, aux diff --git a/tests/aggregator/test_rollback_v2.py b/tests/aggregator/test_rollback_v2.py new file mode 100644 index 0000000..ebb805d --- /dev/null +++ b/tests/aggregator/test_rollback_v2.py @@ -0,0 +1,587 @@ +"""Real tiny carried work; two repaired rounds, signed independent negative controls.""" + +from dataclasses import dataclass, replace +from functools import partial + +import numpy as np +import pytest +import torch +from test_weighted_v2 import KEY, H, economics, policy, roster + +from hypertrain.aggregator.core import OuterState, TapeError, load_state +from hypertrain.aggregator.rollback_v2 import ( + RepairAuthority, + RepairResult, + RepairSource, + RepairTape, + make_repair_tape, + repair_message, + replay_repair_tape, +) +from hypertrain.aggregator.tape_v2 import Exclusion, VerifiedInput, make_tape +from hypertrain.auditor.replay import AnchorCache, VerifiedAnchor, pack_state +from hypertrain.challenge.store import assignment_hash +from hypertrain.challenge.trust_v2 import FundedStatus, ReplayEvidence, SettlementStatus +from hypertrain.data.store import LocalFSStore +from hypertrain.protocol.envelope_v2 import seal +from hypertrain.protocol.example import example_manifest +from hypertrain.protocol.hashing import sha256_hex +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.keys import Keypair +from hypertrain.protocol.messages import Chunk, f32hex +from hypertrain.protocol.messages_v2 import CommitV2, DeltaManifestV2, RunManifestV2 +from hypertrain.trainer.config import TrainConfig +from hypertrain.trainer.island import Comm, IslandAssignment, emulate, train_island +from hypertrain.trainer.loop import Assignment, train_round +from hypertrain.trainer.model import init_params, param_shapes + + +def tiny_manifest() -> RunManifestV2: + b = example_manifest().body() + b["model"].update( + n_layers=1, + d_model=8, + n_heads=2, + n_kv_heads=2, + d_ff=12, + vocab=16, + seq_len=4, + n_experts=1, + top_k_experts=1, + compute_dtype="fp32", + ) + b["inner"].update( + H=2, + J=1, + micro_batch=1, + grad_accum=1, + state_policy="carry", + rewarmup_steps=0, + ) + b["inner"]["lr_schedule"].update(warmup=0, stable=100, peak_lr=f32hex(0.003)) + b["reference_spec"]["layout"].update(pp=1, n_gpus=1, dp_size=1, ep_size=1, zero1=False) + cfg = TrainConfig.from_manifest(b) + b["model"]["param_count"] = sum(int(np.prod(s)) for s in param_shapes(cfg.model).values()) + return RunManifestV2.model_validate( + { + "manifest_version": 2, + "training": b, + "network": { + "admission_policy_hash": H, + "economics_policy_hash": economics().digest(), + "aggregation_policy_hash": policy().digest(), + "dispute_policy_hash": H, + "audit_policy_hash": H, + "relay_registry_hash": H, + "audit_mode": "anchored-full", + "full_anchor_version": 1, + "capabilities": ["island-replay", "all-level-disputes", "transport-receipts"], + }, + } + ) + + +def sample(i: int) -> np.ndarray: + return (np.arange(5, dtype=np.uint32) + i) % 16 + + +@dataclass(frozen=True) +class Scenario: + store: LocalFSStore + manifest: RunManifestV2 + previous: str + sources: tuple[tuple[RepairSource, ...], ...] + original_second_sources: tuple[RepairSource, ...] + authorities: tuple[RepairAuthority, ...] + excluded: tuple[Exclusion, ...] + first: RepairResult + second: RepairResult + + +@pytest.fixture(scope="module") +def scenario(tmp_path_factory: pytest.TempPathFactory) -> Scenario: + store = LocalFSStore(tmp_path_factory.mktemp("rollback")) + manifest = tiny_manifest() + cfg = TrainConfig.from_manifest_v2(manifest) + lay = manifest.training.reference_spec.layout + theta = init_params(cfg.model) + cache = AnchorCache() + anchors = [cache.genesis(manifest, roster(i).hotkey, theta) for i in range(5)] + previous = store.put(OuterState.init({n: x.numpy() for n, x in theta.items()}).to_bytes()) + current, predecessor = previous, "0" * 64 + sources, authorities = [], [] + for w in range(2): + works, round_sources, next_anchors = [], [], [] + start = {n: torch.from_numpy(x.copy()) for n, x in load_state(store, current).theta.items()} + for i, anchor in enumerate(anchors): + r = roster(i) + ids = (w * 20 + i * 2, w * 20 + i * 2 + 1) + result = emulate( + 1, + partial( + train_island, + cfg, + lay, + theta_start=start, + a=IslandAssignment(manifest.run_id(), w, ids, w * 2, 1), + get_sample=sample, + ef_in=anchor.ef, + carry=anchor.state, + ), + )[0] + dh = store.put(result.delta_payload) + commit = CommitV2( + w=w, + hotkey=r.hotkey, + leaf_scheme="ht-leaf-v1", + n_leaves=3, + leaves_root=result.leaves_root, + metrics_root=H, + final_theta_hash=result.final_theta_hash, + ef_in_hash=result.ef_in_hash, + ef_out_hash=result.ef_out_hash, + delta_hash=dh, + delta_bytes=len(result.delta_payload), + tokens=8, + ) + delta = DeltaManifestV2( + w=w, + hotkey=r.hotkey, + delta_hash=dh, + uri="object", + size=len(result.delta_payload), + format="ht-dense-int8-v1", + chunks=[Chunk(off=0, len=len(result.delta_payload), sha256=dh)], + grant_hash=H, + master_acceptance_hash=H, + ) + ah = assignment_hash(manifest.run_id(), w, i, ids) + work = VerifiedInput( + r, + commit, + delta, + ReplayEvidence( + manifest.run_id(), + w, + r.hotkey, + ah, + anchor.anchor_hash, + result.leaves_root, + dh, + result.final_theta_hash, + result.ef_in_hash, + result.ef_out_hash, + H, + "MATCH", + "anchored-full", + ), + FundedStatus(manifest.run_id(), r.hotkey, H, "test", 1000, 100, H, True), + SettlementStatus(manifest.run_id(), w, r.hotkey, False, False, H), + ah, + 12, + ) + miner = Keypair(bytes([i + 1]) * 32) + ce = canonicalize(seal(miner, "CommitV2", manifest.run_id(), commit, 100)) + de = canonicalize(seal(miner, "DeltaManifestV2", manifest.run_id(), delta, 100)) + for blob in ( + canonicalize(commit.model_dump(mode="json")), + canonicalize(delta.model_dump(mode="json")), + ce, + de, + ): + store.put(blob) + round_sources.append(RepairSource(work, ce, de, sha256_hex(ce + de), ids, anchor)) + works.append(work) + next_anchors.append( + VerifiedAnchor( + manifest.run_id(), + r.hotkey, + w, + anchor.layout_hash, + result.final_state, + result.ef_out, + H, + sha256_hex(pack_state(result.final_theta, result.final_state)), + result.final_theta, + "cpu", + ) + ) + tape = make_tape( + store, + manifest, + policy(), + canonicalize(economics().body()), + KEY, + w=w, + prev_state=current, + predecessor_tape_hash=predecessor, + inputs=works, + reference_reward_units=100, + ) + predecessor = store.put(tape.to_bytes()) + authorities.append( + RepairAuthority( + KEY.ss58, + predecessor, + H, + sha256_hex(canonicalize(manifest.training.reference_spec.model_dump(mode="json"))), + AnchorCache.layout_hash(manifest), + "cpu", + ) + ) + sources.append(tuple(round_sources)) + current, anchors = tape.body.out_state, next_anchors + excluded = (Exclusion(hotkey=roster(4).hotkey, reason="FRAUD", evidence_hash=H),) + + def repair(comm: Comm) -> RepairResult: + return make_repair_tape( + store, + manifest, + policy(), + canonicalize(economics().body()), + KEY, + authorities[0], + comm, + sample, + w=0, + prev_state=previous, + predecessor_tape_hash="0" * 64, + sources=sources[0][:4], + excluded=excluded, + reference_reward_units=100, + ) + + first = emulate(1, repair)[0] + repaired_anchors = {a.hotkey: a for a in first.anchors} + second_sources = tuple( + replace(s, anchor=repaired_anchors[s.work.roster.hotkey]) for s in sources[1][:4] + ) + second = emulate( + 1, + lambda comm: make_repair_tape( + store, + manifest, + policy(), + canonicalize(economics().body()), + KEY, + authorities[1], + comm, + sample, + w=1, + prev_state=first.tape.body.arithmetic.out_state, + predecessor_tape_hash=store.put(first.tape.to_bytes()), + sources=second_sources, + excluded=excluded, + reference_reward_units=100, + ), + )[0] + return Scenario( + store, + manifest, + previous, + (sources[0][:4], second_sources), + sources[1][:4], + tuple(authorities), + excluded, + first, + second, + ) + + +def accept( + s: Scenario, + tape: RepairTape, + *, + w: int, + sources: tuple[RepairSource, ...], + authority: RepairAuthority, + previous: str, + predecessor: str, +) -> RepairResult: + return emulate( + 1, + lambda comm: replay_repair_tape( + s.store, + tape, + s.manifest, + policy(), + canonicalize(economics().body()), + authority, + comm, + sample, + w=w, + prev_state=previous, + predecessor_tape_hash=predecessor, + sources=sources, + excluded=s.excluded, + reference_reward_units=100, + ), + )[0] + + +def test_two_round_repair_retains_state_and_recomputes_changed_theta(scenario: Scenario) -> None: + # Given: two original tapes, exclusion; fresh reader has no repair cache. + s = scenario + # When: independently rerun both rounds, retain the first replay's actual m/v/EF. + first = accept( + s, + RepairTape.from_bytes(s.first.tape.to_bytes()), + w=0, + sources=s.sources[0], + authority=s.authorities[0], + previous=s.previous, + predecessor="0" * 64, + ) + anchors = {a.hotkey: a for a in first.anchors} + sources = tuple(replace(x, anchor=anchors[x.work.roster.hotkey]) for x in s.sources[1]) + second = accept( + s, + RepairTape.from_bytes(s.second.tape.to_bytes()), + w=1, + sources=sources, + authority=s.authorities[1], + previous=first.tape.body.arithmetic.out_state, + predecessor=sha256_hex(first.tape.to_bytes()), + ) + # Then: w+1 original deltas are not substituted; actual carry reaches step4. + assert all(a.state.step == 4 for a in second.anchors) + assert second.state.to_bytes() == s.second.state.to_bytes() + assert any( + e.delta_hash != x.work.commit.delta_hash + for e in second.tape.body.recomputed + for x in sources + if e.hotkey == x.work.roster.hotkey + ) + assert all( + e.start_optimizer_hash == a.final_optimizer_hash + for e in second.tape.body.recomputed + for a in first.tape.body.recomputed + if e.hotkey == a.hotkey + ) + + +def test_repaired_inner_work_matches_independent_single_rank_trainer(scenario: Scenario) -> None: + # Given: exact N1 layout; separate train_round implementation, real retained state. + s = scenario + cfg = TrainConfig.from_manifest_v2(s.manifest) + theta = {n: torch.from_numpy(x.copy()) for n, x in s.first.state.theta.items()} + expected = {e.hotkey: e for e in s.second.tape.body.recomputed} + # When: independently run original w+1 assignments at the repaired global theta. + for source in s.sources[1]: + result = train_round( + cfg, + theta, + Assignment(s.manifest.run_id(), 1, source.sample_ids, 2), + sample, + ef_in=source.anchor.ef, + carry=source.anchor.state, + ) + # Then: no stale delta, reset optimizer, lost EF or changed trainer can pass. + e = expected[source.work.roster.hotkey] + assert result.delta_hash == e.delta_hash + assert result.leaves_root == e.leaves_root + assert pack_state(result.final_theta, result.final_state) == s.store.get( + e.final_state_object + ) + assert pack_state(result.ef_out) == s.store.get(e.final_ef_object) + + +@pytest.mark.parametrize( + "mutation", + [ + "predecessor", + "delta", + "anchor", + "layout", + "reference", + "qualification", + "source_authority", + "source_tape", + "acceptance", + "assignment", + "old_carry", + "original_carry", + "anchor_layout", + "anchor_backend", + "source_delta_authority", + "signer", + ], +) +def test_independent_replay_rejects_resigned_false_provenance( + scenario: Scenario, + mutation: str, +) -> None: + # Given: mutate candidate, re-sign with real coordinator; expected authorities unchanged. + s = scenario + body = s.second.tape.body.model_dump(mode="json") + sources = s.sources[1] + other = "22" * 32 + match mutation: + case "predecessor": + body["arithmetic"]["predecessor_tape_hash"] = other + case "delta": + body["recomputed"][0]["delta_hash"] = s.sources[1][0].work.commit.delta_hash + case "anchor": + body["recomputed"][0]["source_anchor_hash"] = other + case "layout" | "reference" | "qualification" | "source_tape": + field = { + "layout": "layout_hash", + "reference": "reference_hash", + "qualification": "qualification_hash", + "source_tape": "source_tape_hash", + }[mutation] + body[field] = other + case "acceptance": + body["recomputed"][0]["acceptance_hash"] = other + case "assignment": + body["recomputed"][0]["sample_ids"] = [999, 1000] + case "source_authority": + fake = canonicalize( + seal(KEY, "CommitV2", s.manifest.run_id(), sources[0].work.commit, 100) + ) + sources = (replace(sources[0], commit_envelope=fake), *sources[1:]) + case "old_carry": + original = s.sources[0][0].anchor + sources = (replace(sources[0], anchor=original), *sources[1:]) + case "original_carry": + sources = s.original_second_sources + case "anchor_layout": + anchor = replace(sources[0].anchor, layout_hash=other) + sources = (replace(sources[0], anchor=anchor), *sources[1:]) + case "anchor_backend": + anchor = replace(sources[0].anchor, backend="cuda") + sources = (replace(sources[0], anchor=anchor), *sources[1:]) + case "source_delta_authority": + fake = canonicalize( + seal( + KEY, + "DeltaManifestV2", + s.manifest.run_id(), + sources[0].work.delta_manifest, + 100, + ) + ) + sources = (replace(sources[0], delta_envelope=fake), *sources[1:]) + case "signer": + pass + parsed = s.second.tape.body.model_validate(body) + key = Keypair(bytes([99]) * 32) if mutation == "signer" else KEY + tape = RepairTape(body=parsed, signer=key.ss58, sig=key.sign(repair_message(parsed)).hex()) + # When / Then + with pytest.raises(TapeError): + accept( + s, + tape, + w=1, + sources=sources, + authority=s.authorities[1], + previous=s.first.tape.body.arithmetic.out_state, + predecessor=sha256_hex(s.first.tape.to_bytes()), + ) + + +@pytest.mark.parametrize( + "part,value", + [ + ("ef", float("nan")), + ("m", float("nan")), + ("v", float("inf")), + ("theta", float("nan")), + ], +) +@pytest.mark.parametrize("replay", [False, True]) +def test_nonfinite_authenticated_anchor_rejects_before_publication( + scenario: Scenario, + monkeypatch: pytest.MonkeyPatch, + part: str, + value: float, + replay: bool, +) -> None: + # Given: same authenticated metadata; poisoned actual decoded carry tensors. + s = scenario + anchor = s.sources[1][0].anchor + original = getattr(anchor, part) if part in ("ef", "theta") else getattr(anchor.state, part) + values = {n: x.clone() for n, x in original.items()} + next(iter(values.values())).view(-1)[0] = value + bad = ( + replace(anchor, **{part: values}) + if part in ("ef", "theta") + else replace(anchor, state=replace(anchor.state, **{part: values})) + ) + sources = (replace(s.sources[1][0], anchor=bad), *s.sources[1][1:]) + + def no_publication(raw: bytes) -> str: + pytest.fail("invalid carry published an object") + + monkeypatch.setattr(s.store, "put", no_publication) + # When / Then: authentic signature never authorizes nonfinite state. + with pytest.raises(TapeError, match="nonfinite repair"): + if replay: + accept( + s, + s.second.tape, + w=1, + sources=sources, + authority=s.authorities[1], + previous=s.first.tape.body.arithmetic.out_state, + predecessor=sha256_hex(s.first.tape.to_bytes()), + ) + else: + emulate( + 1, + lambda comm: make_repair_tape( + s.store, + s.manifest, + policy(), + canonicalize(economics().body()), + KEY, + s.authorities[1], + comm, + sample, + w=1, + prev_state=s.first.tape.body.arithmetic.out_state, + predecessor_tape_hash=sha256_hex(s.first.tape.to_bytes()), + sources=sources, + excluded=s.excluded, + reference_reward_units=100, + ), + ) + + +def test_finite_input_terminal_overflow_rejects_before_publication( + scenario: Scenario, + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Given: finite full carry, overflow in actual H-step model forward/optimizer. + s = scenario + anchor = s.sources[0][0].anchor + values = {n: torch.full_like(x, 3e38) for n, x in anchor.state.m.items()} + assert all(bool(torch.isfinite(x).all()) for x in values.values()) + bad = replace(anchor, state=replace(anchor.state, m=values)) + sources = (replace(s.sources[0][0], anchor=bad), *s.sources[0][1:]) + + def no_publication(raw: bytes) -> str: + pytest.fail("terminal overflow published an object") + + monkeypatch.setattr(s.store, "put", no_publication) + # When / Then + with pytest.raises((TapeError, ValueError)): + emulate( + 1, + lambda comm: make_repair_tape( + s.store, + s.manifest, + policy(), + canonicalize(economics().body()), + KEY, + s.authorities[0], + comm, + sample, + w=0, + prev_state=s.previous, + predecessor_tape_hash="0" * 64, + sources=sources, + excluded=s.excluded, + reference_reward_units=100, + ), + ) diff --git a/tests/aggregator/test_scalar_state_roundtrip.py b/tests/aggregator/test_scalar_state_roundtrip.py new file mode 100644 index 0000000..9b2b9bc --- /dev/null +++ b/tests/aggregator/test_scalar_state_roundtrip.py @@ -0,0 +1,205 @@ +"""Rank0 OD temperatures are not length-one vectors; no runtime training needed.""" + +from pathlib import Path + +import numpy as np +import pytest +import torch +from safetensors.numpy import load as st_load +from safetensors.numpy import save as st_save + +from hypertrain.aggregator.checkpoint import MODEL, write_checkpoint, write_network_checkpoint +from hypertrain.aggregator.core import MalformedDelta, OuterState, load_delta +from hypertrain.auditor.replay import pack_state, unpack_state +from hypertrain.data.store import LocalFSStore +from hypertrain.protocol.envelope_v2 import seal +from hypertrain.protocol.jcs import canonicalize +from hypertrain.trainer.compress import compress +from hypertrain.trainer.config import CompressConfig +from hypertrain.trainer.optim import OptState + + +def arrays() -> dict[str, np.ndarray]: + return { + "log_temp.choice": np.array(0.25, dtype=np.float32), + "log_temp.noul": np.array(-0.5, dtype=np.float32), + "log_temp.score": np.array(1.5, dtype=np.float32), + "vector": np.array([0.25], dtype=np.float32), + "matrix": np.arange(12, dtype=np.float32).reshape(3, 4)[:, ::2], + } + + +def equal(expected: dict[str, np.ndarray], actual: dict[str, np.ndarray]) -> None: + assert expected.keys() == actual.keys() + for name, value in expected.items(): + assert actual[name].shape == value.shape + assert actual[name].dtype == value.dtype + np.testing.assert_array_equal(actual[name], value) + + +def test_outer_state_roundtrip_preserves_all_parts_scalar_and_vector() -> None: + # Given: real OD names, scalar temperatures, true [1] vector, noncontiguous matrix. + values = arrays() + state = OuterState( + values, + {n: np.array(x * 2, dtype=np.float32) for n, x in values.items()}, + {n: np.array(x * 3, dtype=np.float32) for n, x in values.items()}, + ) + # When + raw = state.to_bytes() + restored = OuterState.from_bytes(raw) + # Then: theta, momentum and CClip center all preserve rank/dtype/value exactly. + for part in ("theta", "u", "center"): + equal(getattr(state, part), getattr(restored, part)) + assert restored.to_bytes() == raw + assert restored.hashes() == state.hashes() + + +def test_nonscalar_serialization_bytes_match_preexisting_encoding() -> None: + # Given: old encoding is the compatibility oracle for non-scalars only. + values = {n: a for n, a in arrays().items() if a.shape} + state = OuterState.init(values) + legacy = { + f"{part}/{name}": np.ascontiguousarray(value, dtype=" None: + store = LocalFSStore(tmp_path) + theta = {"log_temp.choice": np.array(0.25, dtype=np.float32)} + payload, _ = compress( + CompressConfig("dense-int8", ef_beta=0), + {n: torch.from_numpy(a) for n, a in theta.items()}, + {n: torch.zeros((), dtype=torch.float32) for n in theta}, + ) + key = store.put(payload) + restored = OuterState.from_bytes(OuterState.init(theta).to_bytes()) + assert load_delta(store, key, restored.theta)["log_temp.choice"].shape == () + with pytest.raises(MalformedDelta, match="shape"): + load_delta(store, key, {"log_temp.choice": np.array([0.25], dtype=np.float32)}) + + +def test_full_training_state_and_ef_roundtrip_preserves_temperatures() -> None: + # Existing pack/unpack path already carries theta/m/v/step and separate full EF. + theta = {n: torch.from_numpy(x.copy()) for n, x in arrays().items()} + optimizer = OptState( + {n: x.clone() * 2 for n, x in theta.items()}, + {n: x.clone().square() for n, x in theta.items()}, + 7, + ) + ef = {n: x.clone() * 0.01 for n, x in theta.items()} + loaded, state = unpack_state(pack_state(theta, optimizer)) + residual, absent = unpack_state(pack_state(ef)) + assert state is not None and state.step == 7 and absent is None + for expected, actual in ( + (theta, loaded), + (optimizer.m, state.m), + (optimizer.v, state.v), + (ef, residual), + ): + equal( + {n: x.numpy() for n, x in expected.items()}, {n: x.numpy() for n, x in actual.items()} + ) + + +def test_ordinary_checkpoint_preserves_scalar_model(tmp_path: Path) -> None: + from agg_helpers import COORD + + tape = {"body": {"kind": "global", "run_id": "11" * 32, "w": 0, "inputs": []}} + write_checkpoint( + tmp_path / "scalar", COORD, arrays(), [tape], license="Apache-2.0", dataset={"name": "tiny"} + ) + equal(arrays(), st_load((tmp_path / "scalar" / MODEL).read_bytes())) + non_scalar = {n: a for n, a in arrays().items() if a.shape} + write_checkpoint( + tmp_path / "non-scalar", + COORD, + non_scalar, + [tape], + license="Apache-2.0", + dataset={"name": "tiny"}, + ) + assert (tmp_path / "non-scalar" / MODEL).read_bytes() == st_save( + {n: np.ascontiguousarray(a, dtype=" None: + """Only serialization seam isolated; real replay arithmetic has its own tiny controls.""" + from types import SimpleNamespace + + from test_weighted_v2 import KEY, economics, fixture, policy + + import hypertrain.aggregator.checkpoint as checkpoint + from hypertrain.protocol.messages_v2 import PolicyHashes + + store = LocalFSStore(tmp_path / "objects") + manifest, works, previous = fixture(store) + for raw in (canonicalize(policy().body()), canonicalize(economics().body())): + store.put(raw) + state = OuterState.init(arrays()) + output = store.put(state.to_bytes()) + tape = SimpleNamespace( + body=SimpleNamespace( + w=0, out_state=output, out_hashes=state.hashes(), run_id=manifest.run_id() + ) + ) + final = seal( + KEY, + "Finalize", + manifest.run_id(), + { + "w": 0, + "final_theta_hash_w1": state.hashes()["theta_hash"], + "included": [], + "entitlements_root": "11" * 32, + }, + 100, + ) + opening = seal( + KEY, + "RoundOpenV2", + manifest.run_id(), + { + "w": 0, + "roster": [work.roster.body() for work in works], + "theta_hash": state.hashes()["theta_hash"], + "prev_final_hash": "0" * 64, + "outer_state_hash": "0" * 64, + "center_hash": "0" * 64, + "roster_hash": "0" * 64, + "honeypot_commit": "0" * 64, + "d_open": 1, + "d_assign": 2, + "d_commit": 3, + "d_audit": 4, + "d_upload": 5, + "d_final": 6, + "contract_version": 2, + "registry_epoch": 0, + "policy_hashes": {n: getattr(manifest.network, n) for n in PolicyHashes.model_fields}, + "start_state_index_hash": "0" * 64, + "audit_mode": "anchored-full", + }, + 100, + ) + item = { + "tape_hash": output, + "prev_state": previous, + "predecessor_tape_hash": "0" * 64, + "inputs": [], + "round_open": opening, + "finalize": final, + } + monkeypatch.setattr(checkpoint, "_network_tape", lambda *args: tape) + monkeypatch.setattr(checkpoint, "_network_subjects", lambda *args: None) + monkeypatch.setattr(checkpoint, "require_network_roster", lambda *args: None) + monkeypatch.setattr(checkpoint, "_network_replay", lambda *args: (state, None)) + write_network_checkpoint(tmp_path / "network", KEY, store, manifest, [item]) + equal(arrays(), st_load((tmp_path / "network" / MODEL).read_bytes())) diff --git a/tests/aggregator/test_tape_v2.py b/tests/aggregator/test_tape_v2.py new file mode 100644 index 0000000..12b932f --- /dev/null +++ b/tests/aggregator/test_tape_v2.py @@ -0,0 +1,48 @@ +"""Ordinary tape compatibility remains independent of rollback evidence.""" + +from pathlib import Path + +from test_weighted_v2 import KEY, economics, fixture, policy + +from hypertrain.aggregator.tape_v2 import TapeV2, make_tape, replay_tape +from hypertrain.data.store import LocalFSStore +from hypertrain.protocol.hashing import sha256_hex +from hypertrain.protocol.jcs import canonicalize + + +def test_ordinary_tape_bytes_and_output_remain_original(tmp_path: Path) -> None: + # Given: accepted ordinary payloads; no repair version/domain injected. + store = LocalFSStore(tmp_path) + manifest, works, previous = fixture(store) + tape = make_tape( + store, + manifest, + policy(), + canonicalize(economics().body()), + KEY, + w=0, + prev_state=previous, + predecessor_tape_hash="11" * 32, + inputs=works, + reference_reward_units=100, + ) + raw = tape.to_bytes() + # When + state = replay_tape( + store, + TapeV2.from_bytes(raw), + manifest, + policy(), + canonicalize(economics().body()), + signer=KEY.ss58, + w=0, + prev_state=previous, + predecessor_tape_hash="11" * 32, + inputs=works, + reference_reward_units=100, + ) + # Then + assert raw == canonicalize(tape.body_json(), allow_float=False) + assert set(tape.body_json()) == {"body", "signer", "sig"} + assert tape.body.v == "ht-tape-v2" + assert sha256_hex(state.to_bytes()) == tape.body.out_state diff --git a/tests/aggregator/test_weighted_v2.py b/tests/aggregator/test_weighted_v2.py new file mode 100644 index 0000000..2a241fe --- /dev/null +++ b/tests/aggregator/test_weighted_v2.py @@ -0,0 +1,639 @@ +from __future__ import annotations + +import tracemalloc +from dataclasses import replace +from itertools import product +from pathlib import Path +from time import perf_counter, process_time + +import numpy as np +import pytest +import torch + +from hypertrain.aggregator.core import OuterParams, OuterState, TapeError, preclip, sqnorm +from hypertrain.aggregator.tape_v2 import TapeV2, VerifiedInput, make_tape, replay_tape +from hypertrain.aggregator.weighted_v2 import ( + Candidate, + InsufficientEligibleWeight, + _completion_capacity, + aggregate_weighted, + allocate_weights, +) +from hypertrain.auditor.replay import AuditInputs, audit_full +from hypertrain.challenge.trust_v2 import FundedStatus, ReplayEvidence, SettlementStatus +from hypertrain.data.store import LocalFSStore +from hypertrain.protocol.envelope_v2 import tape_signing_message +from hypertrain.protocol.example import example_manifest +from hypertrain.protocol.hashing import sha256_hex +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.keys import Keypair +from hypertrain.protocol.messages import AuditChallenge, Chunk, Commit, f32hex +from hypertrain.protocol.messages_v2 import ( + AggregationPolicyV2, + CommitV2, + DeltaManifestV2, + EconomicsPolicyV2, + RosterEntryV2, + RunManifestV2, +) +from hypertrain.trainer.compress import compress +from hypertrain.trainer.config import CompressConfig, InnerConfig, ModelConfig, TrainConfig +from hypertrain.trainer.loop import Assignment, train_round +from hypertrain.trainer.model import init_params + +Q = 1 << 24 +H = "11" * 32 +KEY = Keypair(bytes(range(32))) + + +def policy(groups: dict[str, int] | None = None) -> AggregationPolicyV2: + return AggregationPolicyV2( + arithmetic="flat-cclip-cap-v1", + order="utf8", + center="prev_outer_update", + trust_mode="uniform-verified", + weight_quantum=Q, + miner_cap_units=Q // 4, + probation_cap_units=Q // 4, + owner_group_caps=groups or {}, + preclip_norm=f32hex(10), + cclip_tau=f32hex(10), + cclip_iters=1, + ) + + +def economics() -> EconomicsPolicyV2: + return EconomicsPolicyV2( + ledger_mode="test", + G_max_units=100, + R_collectible_units=100, + S_min_units=1000, + beta_ppm=1_000_000, + gammaV_units=0, + s_lower_ppm=1_000_000, + q_floor=1_000_000, + genesis_allocation_hash=H, + reward_authority=KEY.ss58, + round_reward_units=100, + max_total_issuance=100000, + shadow_bootstrap_rounds=12, + ) + + +def roster(i: int, probation: bool = False, owner: str | None = None) -> RosterEntryV2: + return RosterEntryV2( + hotkey=Keypair(bytes([i + 1]) * 32).ss58, + slot=i, + q_i=f32hex(1), + admission_id=H, + coldkey_group=owner or f"owner{i}", + state="PROBATION" if probation else "ACTIVE", + eligible_weight=Q // 4, + ) + + +def candidate(i: int, probation: bool = False, owner: str | None = None) -> Candidate: + return Candidate(roster(i, probation, owner), H, H) + + +def test_uniform_exact_dyadic_weights_when_four_active() -> None: + # Given + candidates = [candidate(i) for i in range(4)] + # When + allocation = allocate_weights(candidates[::-1], policy()) + # Then + assert [e.weight_units for e in allocation.entries] == [Q // 4] * 4 + assert allocation == allocate_weights(candidates, policy()) + + +@pytest.mark.parametrize("n,probation", [(3, False), (4, True), (10, True)]) +def test_infeasible_caps_when_capacity_insufficient(n: int, probation: bool) -> None: + # Given / When / Then + with pytest.raises(InsufficientEligibleWeight): + allocate_weights([candidate(i, probation) for i in range(n)], policy()) + + +def test_intersecting_owner_probation_caps_when_feasible_completion_required() -> None: + # Given: spending owner A on probation would strand the active capacity. + candidates = [ + candidate(0, owner="A"), + candidate(1, True, "A"), + candidate(2, owner="B"), + candidate(3, True, "C"), + candidate(4, owner="D"), + ] + p = policy({"A": Q // 4, "B": Q // 4, "C": Q // 4, "D": Q // 4}) + # When + allocation = allocate_weights(candidates, p) + # Then: constrained completion gives A's active all its owner capacity. + assert sum(e.weight_units for e in allocation.entries) == Q + by_id = {e.hotkey: e.weight_units for e in allocation.entries} + assert by_id[candidates[0].roster.hotkey] == Q // 4 + assert by_id[candidates[1].roster.hotkey] == 0 + + +def test_probation_bound_and_utf8_remainder_when_multiple_newcomers() -> None: + # Given + candidates = [candidate(i, i >= 4) for i in range(7)] + # When + allocation = allocate_weights(candidates, policy()) + # Then + assert sum(e.weight_units for e in allocation.entries if e.probation) == Q // 4 + assert sum(e.weight_units for e in allocation.entries) == Q + newcomers = [e.weight_units for e in allocation.entries if e.probation] + assert max(newcomers) - min(newcomers) <= 1 + + +def test_owner_capacity_rejected_when_known_group_owns_entire_roster() -> None: + # Given / When / Then + with pytest.raises(InsufficientEligibleWeight): + allocate_weights( + [candidate(i, owner="shared") for i in range(8)], policy({"shared": Q // 2}) + ) + + +def test_rescaled_copy_rejected_when_norm_screen_passes_own_assignment_replay() -> None: + # Given: two honest non-IID trajectories; exact real trainer/replay, two steps. + cfg = TrainConfig( + model=ModelConfig( + n_layers=1, + d_model=16, + n_heads=2, + n_kv_heads=2, + d_ff=24, + vocab=32, + seq_len=4, + n_experts=0, + ), + inner=InnerConfig(lr=0.003, H=2, J=1, micro_batch=1, state_policy="reset"), + compress=CompressConfig("dense-int8", ef_beta=0), + n_stages=1, + ) + theta = init_params(cfg.model) + + def sample(i: int) -> np.ndarray: + return (np.arange(5, dtype=np.uint32) + i) % 32 + + assignments = [Assignment(H, 0, (1, 2)), Assignment(H, 0, (17, 18))] + results = [train_round(cfg, theta, a, sample) for a in assignments] + c = Commit( + w=0, + hotkey=KEY.ss58, + leaf_scheme="ht-leaf-v1", + n_leaves=3, + leaves_root=results[0].leaves_root, + metrics_root=H, + final_theta_hash=results[0].final_theta_hash, + ef_in_hash=results[0].ef_in_hash, + ef_out_hash=results[0].ef_out_hash, + delta_hash=results[0].delta_hash, + delta_bytes=len(results[0].delta_payload), + tokens=8, + ) + challenge = AuditChallenge( + w=0, + target=KEY.ss58, + beacon_round=1, + beacon_sig_sha256=H, + mode="full", + segments=[], + reasons=["random"], + serve_deadline=100, + ) + audits = [ + AuditInputs( + cfg, + H, + challenge, + c.model_copy( + update={ + "leaves_root": res.leaves_root, + "final_theta_hash": res.final_theta_hash, + "ef_in_hash": res.ef_in_hash, + "ef_out_hash": res.ef_out_hash, + "delta_hash": res.delta_hash, + "delta_bytes": len(res.delta_payload), + } + ), + a, + [x.digest.hex() for x in res.leaves], + [x.preimage for x in res.leaves], + theta, + ) + for a, res in zip(assignments, results, strict=True) + ] + assert all(audit_full(a, sample).result == "MATCH" for a in audits) + honest = {n: (theta[n] - results[0].final_theta[n]).numpy() for n in theta} + copied = {n: (theta[n] - results[1].final_theta[n]).numpy() for n in theta} + scale = np.float32((sqnorm(honest) / sqnorm(copied)) ** 0.5) + copied = {n: a * scale for n, a in copied.items()} + _, clipped = preclip(copied, sqnorm(honest) ** 0.5 * 1.01) + assert not clipped + td = {n: torch.from_numpy(a) for n, a in copied.items()} + payload, _ = compress(cfg.compress, td, {n: torch.zeros_like(a) for n, a in td.items()}) + attacked = replace( + audits[0], + commit=c.model_copy( + update={"delta_hash": sha256_hex(payload), "delta_bytes": len(payload)} + ), + ) + # When + verdict = audit_full(attacked, sample) + # Then: median norm is not provenance; own assignment final compression rejects. + assert verdict.result == "MISMATCH" + + +def fixture(store: LocalFSStore, n: int = 4) -> tuple[RunManifestV2, list[VerifiedInput], str]: + p, econ = policy(), economics() + training = example_manifest().model_dump(mode="json") + training["model"]["param_count"] = 4329216 + manifest = RunManifestV2.model_validate( + { + "manifest_version": 2, + "training": training, + "network": { + "admission_policy_hash": H, + "economics_policy_hash": econ.digest(), + "aggregation_policy_hash": p.digest(), + "dispute_policy_hash": H, + "audit_policy_hash": H, + "relay_registry_hash": H, + "audit_mode": "anchored-full", + "full_anchor_version": 1, + "capabilities": ["island-replay", "all-level-disputes", "transport-receipts"], + }, + } + ) + previous = store.put(OuterState.init({"x": np.array([2.0, 3.0], np.float32)}).to_bytes()) + works = [] + for i in range(n): + r = roster(i) + delta = {"x": torch.tensor([float(i + 1), float(4 - i)])} + payload, _ = compress(CompressConfig("dense-int8", ef_beta=0), delta, {"x": torch.zeros(2)}) + delta_hash = store.put(payload) + c = CommitV2( + w=0, + hotkey=r.hotkey, + leaf_scheme="ht-leaf-v1", + n_leaves=7, + leaves_root=H, + metrics_root=H, + final_theta_hash=H, + ef_in_hash=H, + ef_out_hash=H, + delta_hash=delta_hash, + delta_bytes=len(payload), + tokens=manifest.training.batch_samples() * manifest.training.model.seq_len, + ) + d = DeltaManifestV2( + w=0, + hotkey=r.hotkey, + delta_hash=delta_hash, + uri="object", + size=len(payload), + format="ht-dense-int8-v1", + chunks=[Chunk(off=0, len=len(payload), sha256=delta_hash)], + grant_hash=H, + master_acceptance_hash=H, + ) + store.put(canonicalize(c.model_dump(mode="json"))) + store.put(canonicalize(d.model_dump(mode="json"))) + replay = ReplayEvidence( + manifest.run_id(), + 0, + r.hotkey, + H, + H, + H, + delta_hash, + H, + H, + H, + H, + "MATCH", + "anchored-full", + ) + funding = FundedStatus(manifest.run_id(), r.hotkey, H, "test", 1000, 100, H, True) + settlement = SettlementStatus(manifest.run_id(), 0, r.hotkey, False, False, H) + works.append(VerifiedInput(r, c, d, replay, funding, settlement, H, 12)) + return manifest, works, previous + + +def test_real_payload_tape_when_replayed_independently(tmp_path: Path) -> None: + # Given + store = LocalFSStore(tmp_path) + manifest, works, previous = fixture(store) + args = dict( + w=0, prev_state=previous, predecessor_tape_hash=H, inputs=works, reference_reward_units=100 + ) + tape = make_tape(store, manifest, policy(), canonicalize(economics().body()), KEY, **args) + # When + output = replay_tape( + store, + TapeV2.from_bytes(tape.to_bytes()), + manifest, + policy(), + canonicalize(economics().body()), + signer=KEY.ss58, + **args, + ) + # Then + assert sha256_hex(output.to_bytes()) == tape.body.out_state + assert not np.array_equal(output.theta["x"], np.array([2.0, 3.0], np.float32)) + + +@pytest.mark.parametrize( + "mutation", + [ + "omitted", + "duplicate", + "assignment", + "copied", + "unfunded", + "dispute", + "shadow", + "weight", + "policy", + "predecessor", + "signature", + "missing", + ], +) +def test_tape_rejects_when_original_inputs_or_gates_altered(tmp_path: Path, mutation: str) -> None: + # Given + store = LocalFSStore(tmp_path) + manifest, works, previous = fixture(store, 5 if mutation == "weight" else 4) + args = dict( + w=0, prev_state=previous, predecessor_tape_hash=H, inputs=works, reference_reward_units=100 + ) + tape = make_tape(store, manifest, policy(), canonicalize(economics().body()), KEY, **args) + p = policy() + match mutation: + case "omitted": + args["inputs"] = works[:-1] + case "duplicate": + args["inputs"] = works + [works[0]] + case "assignment": + works[0] = replace(works[0], assignment_hash="22" * 32) + case "copied": + works[0] = replace( + works[0], replay=replace(works[0].replay, delta_hash=works[1].commit.delta_hash) + ) + case "unfunded": + works[0] = replace(works[0], funding=replace(works[0].funding, locked_units=0)) + case "dispute": + works[0] = replace( + works[0], settlement=replace(works[0].settlement, unresolved_dispute=True) + ) + case "shadow": + works[0] = replace(works[0], clean_finalizations=0) + case "weight": + entries = list(tape.body.allocation.entries) + entries[0] = entries[0].model_copy(update={"weight_units": entries[0].weight_units - 1}) + entries[1] = entries[1].model_copy(update={"weight_units": entries[1].weight_units + 1}) + allocation = tape.body.allocation.model_validate( + {"policy_hash": p.digest(), "entries": [e.body() for e in entries]} + ) + body = tape.body.model_copy(update={"allocation": allocation}) + tape = TapeV2( + body=body, + signer=KEY.ss58, + sig=KEY.sign(tape_signing_message(body.body(), manifest.run_id())).hex(), + ) + case "policy": + p = policy({"owner0": 1}) + case "predecessor": + args["predecessor_tape_hash"] = "22" * 32 + case "signature": + tape = tape.model_copy(update={"sig": "00" * 64}) + case "missing": + store._path(works[0].commit.delta_hash).unlink() + # When / Then + with pytest.raises((ValueError, TapeError, RuntimeError)): + replay_tape( + store, tape, manifest, p, canonicalize(economics().body()), signer=KEY.ss58, **args + ) + + +def test_nonuniform_cclip_matches_scalar_oracle_when_owner_cap_saturates() -> None: + # Given: previous update center, not momentum; nonuniform caps and clipping. + candidates = [candidate(i, owner="small" if i == 0 else None) for i in range(5)] + p = policy({"small": Q // 16}).model_copy( + update={"preclip_norm": f32hex(2), "cclip_tau": f32hex(0.5), "cclip_iters": 2} + ) + allocation = allocate_weights(candidates, p) + previous = OuterState( + {"x": np.array([4.0], np.float32)}, + {"x": np.array([7.0], np.float32)}, + {"x": np.array([0.3], np.float32)}, + ) + deltas = { + c.roster.hotkey: {"x": np.array([i + 1.0], np.float32)} for i, c in enumerate(candidates) + } + params = OuterParams( + "nesterov", f32hex(0.1), f32hex(0.9), p.preclip_norm, p.cclip_tau, p.cclip_iters + ) + center = np.float32(0.3) + for _ in range(2): + acc = np.float32(0) + for e in allocation.entries: + d = deltas[e.hotkey]["x"][0] + if d > 2: + d = np.float32(d * np.float32(2 / float(d))) + diff = np.float32(d - center) + scale = np.float32(min(1.0, 0.5 / abs(float(diff)))) + acc = np.float32(acc + np.float32(e.weight_units / Q) * np.float32(diff * scale)) + center = np.float32(center + acc) + u = np.float32(np.float32(0.9) * np.float32(7) + center) + theta = np.float32(np.float32(4) - np.float32(0.1) * np.float32(center + np.float32(0.9) * u)) + # When + result = aggregate_weighted(previous, deltas, allocation, p, params) + # Then + assert result.state.center["x"][0] == center + assert result.state.u["x"][0] == u + assert result.state.theta["x"][0] == theta + + +def test_tape_body_hash_deterministic_when_input_order_changes(tmp_path: Path) -> None: + # Given + store = LocalFSStore(tmp_path) + manifest, works, previous = fixture(store) + # When + tapes = [ + make_tape( + store, + manifest, + policy(), + canonicalize(economics().body()), + KEY, + w=0, + prev_state=previous, + predecessor_tape_hash=H, + inputs=order, + reference_reward_units=100, + ) + for order in (works, works[::-1]) + ] + # Then: sr25519 signatures need not be deterministic; signed body digest must be. + assert tapes[0].body.digest() == tapes[1].body.digest() + + +def test_completion_matches_exhaustive_cut_oracle_when_two_cohorts_two_owners() -> None: + # Given: all 3^7 capacity combinations; active source arc cannot bind. + edges = [(0, 1), (0, 2), (1, 3), (1, 4), (2, 3), (2, 4), (3, 5), (4, 5)] + cuts = [ + {0} | {i + 1 for i, bit in enumerate(bits) if bit} + for bits in product((False, True), repeat=4) + ] + for values in product(range(3), repeat=7): + capacities = (8, *values) + graph = [[0] * 6 for _ in range(6)] + for (u, v), cap in zip(edges, capacities, strict=True): + graph[u][v] = cap + expected = min( + sum(graph[u][v] for u, v in edges if u in cut and v not in cut) for cut in cuts + ) + # When + actual = _completion_capacity( + [capacities[2], capacities[3]], + [capacities[4], capacities[5]], + [capacities[6], capacities[7]], + capacities[1], + ) + # Then: independent exhaustive min-cut, not a second augmenting-path algorithm. + assert actual == expected + + +def test_four_miner_feasibility_matches_exhaustive_reference_when_caps_intersect() -> None: + # Given: Q/4 ceiling forces the unique complete vector; enumerate every cohort mask. + base = [candidate(i) for i in range(4)] + partitions = [("A", "A", "B", "B"), ("A", "A", "A", "B"), ("A",) * 4] + for probation in product((False, True), repeat=4): + for owners in partitions: + for owner_cap in (Q // 4, Q // 2, Q): + candidates = [ + replace( + c, + roster=c.roster.model_copy( + update={ + "state": "PROBATION" if probation[i] else "ACTIVE", + "coldkey_group": owners[i], + } + ), + ) + for i, c in enumerate(base) + ] + expected = sum(probation) <= 1 and all( + owners.count(group) * (Q // 4) <= owner_cap for group in set(owners) + ) + # When / Then:144 exact integer cases, at most four identities. + if expected: + allocation = allocate_weights( + candidates, policy({"A": owner_cap, "B": owner_cap}) + ) + assert [e.weight_units for e in allocation.entries] == [Q // 4] * 4 + else: + with pytest.raises(InsufficientEligibleWeight): + allocate_weights(candidates, policy({"A": owner_cap, "B": owner_cap})) + + +def test_maxmin_matches_exhaustive_integer_reference_when_small_residual_budget() -> None: + # Given: four large owners can carry at most Q-8; small miners must carry at least8. + ordered = sorted([candidate(i) for i in range(8)], key=lambda c: c.roster.hotkey.encode()) + fixed = ordered[:4] + small = ordered[4:] + for probation in product((False, True), repeat=4): + for shared_cap in (3, 5, 8): + eligible = (2, 3, 4, 5) + candidates = [ + replace( + c, + roster=c.roster.model_copy( + update={"coldkey_group": f"fixed{i}", "eligible_weight": Q // 4} + ), + ) + for i, c in enumerate(fixed) + ] + candidates += [ + replace( + c, + roster=c.roster.model_copy( + update={ + "coldkey_group": "shared" if i < 2 else f"small{i}", + "state": "PROBATION" if probation[i] else "ACTIVE", + "eligible_weight": eligible[i], + } + ), + ) + for i, c in enumerate(small) + ] + p = policy( + { + "fixed0": Q // 4, + "fixed1": Q // 4, + "fixed2": Q // 4, + "fixed3": Q // 4 - 8, + "shared": shared_cap, + } + ) + feasible = [ + x + for x in product(*(range(cap + 1) for cap in eligible)) + if sum(x) >= 8 and x[0] + x[1] <= shared_cap + ] + expected = max(feasible, key=lambda x: (tuple(sorted(x)), x)) + # When + allocation = allocate_weights(candidates[::-1], p) + # Then: exhaustive sorted-leximin optimum, UTF-8 remainder preference. + assert tuple(e.weight_units for e in allocation.entries[4:]) == expected + + +@pytest.mark.parametrize("n", [128, 1024]) +def test_scale_roster_when_unique_owner_caps_saturate_in_many_stages(n: int) -> None: + # Given: n distinct owners, n/4 saturation breakpoints, two intersecting cohorts. + from hypertrain.protocol.keys import encode_hotkey + + template = roster(0) + candidates = [ + Candidate( + template.model_copy( + update={ + "hotkey": encode_hotkey(i.to_bytes(32, "little")), + "slot": i, + "coldkey_group": f"owner{i}", + "state": "PROBATION" if i % 2 else "ACTIVE", + } + ), + H, + H, + ) + for i in range(n) + ] + p = policy({f"owner{i}": i // 2 + 1 for i in range(n // 2)}) + # When: label measurements; never assert time or scheduling thresholds. + cpu, wall = process_time(), perf_counter() + allocation = allocate_weights(candidates, p) + cpu, wall = process_time() - cpu, perf_counter() - wall + tracemalloc.start() + traced = allocate_weights(candidates[::-1], p) + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + print(f"SCALE n={n} cpu_s={cpu:.6f} wall_s={wall:.6f} traced_peak_bytes={peak}") + # Then: deterministic arithmetic contract, not a timing-luck admission test. + assert len(allocation.entries) == n + assert traced == allocation + assert sum(e.weight_units for e in allocation.entries) == Q + assert sum(e.weight_units for e in allocation.entries if e.probation) <= Q // 4 + owners = {c.roster.hotkey: c.roster.coldkey_group for c in candidates} + assert all( + e.weight_units <= min(Q // 4, p.owner_group_caps.get(owners[e.hotkey], Q)) + for e in allocation.entries + ) + + +def test_oversized_roster_rejected_before_sort_when_exceeding_wire_ceiling() -> None: + # Given: count check precedes duplicate-key validation or expensive arithmetic. + candidates = [candidate(0)] * 1025 + # When / Then + with pytest.raises(InsufficientEligibleWeight, match="oversized candidate set"): + allocate_weights(candidates, policy()) diff --git a/tests/auditor/test_island_bisect_v2.py b/tests/auditor/test_island_bisect_v2.py new file mode 100644 index 0000000..b0575f7 --- /dev/null +++ b/tests/auditor/test_island_bisect_v2.py @@ -0,0 +1,181 @@ +"""Real same-layout trace equality and localized forward/collective/update faults.""" + +from __future__ import annotations + +import hashlib +import importlib.util +import sys +from pathlib import Path + +import numpy as np +import pytest +import torch + +import hypertrain.trainer # noqa: F401 +from hypertrain.auditor.bisect import RefereeError, tensor_digest +from hypertrain.auditor.island_bisect import IslandExecution, IslandParty, adjudicate +from hypertrain.auditor.replay import pack_state +from hypertrain.protocol.keys import Keypair +from hypertrain.trainer.config import TrainConfig +from hypertrain.trainer.island import TraceContext, emulate, train_island +from hypertrain.trainer.loop import Assignment +from hypertrain.trainer.model import init_params + + +def helpers(name: str): + spec = importlib.util.spec_from_file_location("l5_" + name, Path(__file__).parents[1] / name) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def execution(n: int = 2, ep: int = 1, objective: str | None = None) -> IslandExecution: + if objective: + helper = helpers("layout/test_od_island_v2.py") + manifest = helper.od_manifest(objective, n, True) + cfg = TrainConfig.from_manifest_v2(manifest) + get = helper.samples(cfg) + else: + manifest = helpers("miner/test_island_launch_v2.py").tiny_manifest(n, ep) + cfg = TrainConfig.from_manifest_v2(manifest) + + def get(i: int): + return (np.arange(5, dtype=np.uint32) + i) % 16 + + return IslandExecution( + cfg=cfg, + layout=manifest.training.reference_spec.layout, + assignment=Assignment( + manifest.run_id(), 0, tuple(range(manifest.training.batch_samples())) + ), + theta=init_params(cfg.model), + get_sample=get, + ) + + +@pytest.mark.parametrize( + "n,ep,objective", + [ + (2, 1, None), + (2, 2, None), + (4, 2, None), + (2, 1, "mlm"), + (2, 1, "decision"), + (2, 1, "distill"), + ], +) +def test_honest_traces_when_real_island_hooks(n: int, ep: int, objective: str | None) -> None: + # Given + run = execution(n, ep, objective) + plain = emulate( + n, + lambda c: train_island( + run.cfg, + run.layout, + c, + run.theta, + run.assignment, + run.get_sample, + ), + ) + # When + traced = emulate(n, run.trace) + repeated = emulate(n, run.trace) + a = IslandParty(Keypair(b"\x31" * 32).ss58, [c for _, c in traced], J=run.cfg.inner.J) + b = IslandParty(Keypair(b"\x32" * 32).ss58, [c for _, c in repeated], J=run.cfg.inner.J) + # Then + for rank, (result, _) in enumerate(traced): + assert result.leaf_digests == plain[rank].leaf_digests + assert result.delta_payload == plain[rank].delta_payload + assert pack_state(result.final_theta, result.final_state) == pack_state( + plain[rank].final_theta, + plain[rank].final_state, + ) + assert a.entries == b.entries + assert {entry.rank for entry in a.entries} == set(range(n)) + for level, ctx in [("step", ()), ("layer", (1,)), ("op", (1, 0))]: + at = list(range(a.span(level, ctx) + 1)) + assert a.hashes(level, ctx, at) == b.hashes(level, ctx, at) + assert any(e.op.startswith("optimizer.") for e in a.entries) + assert any(e.op == "all_gather.input" for e in a.entries) + if ep > 1: + assert any(e.op == "ep.dispatch.permutation" for e in a.entries) + truth = IslandParty(Keypair(b"\x33" * 32).ss58, [c for _, c in repeated], J=run.cfg.inner.J) + assert adjudicate(("11" * 32, "22" * 32), (a, b), truth).reason == "MATCH" + + +@pytest.mark.parametrize("op", ["attn", "all_to_all.input", "optimizer.theta."]) +def test_fault_localizes_when_actual_hook_mutates(op: str) -> None: + # Given + run = execution(2, 2) + plain = emulate(2, run.trace) + hit = [False, False] + + def fault(ctx: TraceContext, x: torch.Tensor) -> torch.Tensor: + if ( + ctx.rank == 0 + and ctx.step == 1 + and not hit[0] + and ctx.op.startswith(op) + and x.numel() + and x.is_floating_point() + ): + hit[0] = True + out = x.clone() + out.reshape(-1)[0] += 0.1 + return out + return x + + # When + changed = emulate(2, lambda c: run.trace(c, fault=fault)) + good = IslandParty(Keypair(b"\x41" * 32).ss58, [c for _, c in plain], J=1) + bad = IslandParty(Keypair(b"\x42" * 32).ss58, [c for _, c in changed], J=1) + ref = IslandParty(Keypair(b"\x43" * 32).ss58, [c for _, c in plain], J=1) + evidence = adjudicate(("11" * 32, "22" * 32), (good, bad), ref) + # Then + assert hit[0] and changed[0][0].delta_hash != plain[0][0].delta_hash + assert evidence.reason == "FRAUD" and evidence.loser == bad.hotkey + assert "rank=0" in evidence.op_spec and "op=" + op in evidence.op_spec + + +def test_referee_rejects_when_agreed_prefix_unreproducible() -> None: + # Given + run = execution() + traces = emulate(2, run.trace) + a = IslandParty(Keypair(b"\x51" * 32).ss58, [c for _, c in traces], J=1) + b = IslandParty(Keypair(b"\x52" * 32).ss58, [c for _, c in traces], J=1) + ref = IslandParty(Keypair(b"\x53" * 32).ss58, [c for _, c in traces], J=1) + theta = {k: x.clone() + 0.1 for k, x in run.theta.items()} + from hypertrain.trainer.optim import init_state + + forged = pack_state(theta, init_state(run.cfg.inner, theta)) + a.states[0] = b.states[0] = forged + # When / Then + with pytest.raises(RefereeError, match="agreed input"): + adjudicate(("11" * 32, "22" * 32), (a, b), ref) + + +def test_tensor_digest_when_cpu_encoding_is_frozen() -> None: + # Given + tensor = torch.arange(6, dtype=torch.bfloat16).reshape(2, 3) + expected = hashlib.sha256( + b"torch.bfloat16|(2, 3)|" + tensor.view(torch.uint8).numpy().tobytes() + ) + # When / Then + assert tensor_digest(tensor) == expected.digest() + + +def test_context_rejects_when_invalid_level_or_rank() -> None: + # Given + run = execution() + traces = emulate(2, run.trace) + party = IslandParty(Keypair(b"\x61" * 32).ss58, [c for _, c in traces], J=1) + # When / Then + with pytest.raises(RefereeError): + party.hashes("op", (1, 999999), [0]) + with pytest.raises(AssertionError): + party.span("unsupported", ()) + with pytest.raises(RefereeError): + IslandParty(party.hotkey, [traces[1][1], traces[0][1]], J=1) diff --git a/tests/auditor/test_island_replay_v2.py b/tests/auditor/test_island_replay_v2.py new file mode 100644 index 0000000..b049080 --- /dev/null +++ b/tests/auditor/test_island_replay_v2.py @@ -0,0 +1,536 @@ +from __future__ import annotations + +import importlib.util +import threading +import time +from pathlib import Path + +import numpy as np +import pytest +from pydantic import ValidationError + +import hypertrain.trainer # noqa: F401 +from hypertrain.auditor.honeypot import fresh_honeypots +from hypertrain.auditor.replay import ( + AnchorCache, + AuditInputError, + audit_island, + optimizer_hash, + pack_state, + replay_island_windows, +) +from hypertrain.auditor.worker import LeaseGuard +from hypertrain.protocol.envelope_v2 import seal +from hypertrain.protocol.hashing import sha256_hex +from hypertrain.protocol.keys import Keypair +from hypertrain.protocol.messages_v2 import ( + AuditChallengeV2, + AuditJobV2, + CommitV2, + RunManifestV2, + StartStateV2, +) +from hypertrain.trainer.compress import state_hash +from hypertrain.trainer.config import TrainConfig +from hypertrain.trainer.island import emulate, train_island +from hypertrain.trainer.loop import Assignment +from hypertrain.trainer.model import init_params + + +def fixture(objective: str | None = None, state_policy: str = "carry"): + path = Path(__file__).parents[1] / ( + "layout/test_od_island_v2.py" if objective else "miner/test_island_launch_v2.py" + ) + spec = importlib.util.spec_from_file_location("replay_l1_fixture", path) + assert spec and spec.loader + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + w = mod.od_manifest(objective, 2, True) if objective else mod.tiny_manifest(2) + b = w.body() + b["training"]["inner"]["state_policy"] = state_policy + w = RunManifestV2.model_validate(b) + cfg = TrainConfig.from_manifest_v2(w) + get = mod.samples(cfg) if objective else lambda i: (np.arange(5, dtype=np.uint32) + i) % 16 + theta = init_params(cfg.model) + key = Keypair(bytes([44]) * 32) + cache = AnchorCache() + anchor = cache.genesis(w, key.ss58, theta) + a = Assignment(w.run_id(), 0, tuple(range(w.training.batch_samples()))) + result = emulate( + 2, + lambda c: train_island( + cfg, + w.training.reference_spec.layout, + c, + theta, + a, + get, + carry=anchor.state if state_policy == "carry" else None, + ef_in=anchor.ef, + v0=anchor.state.v if state_policy == "derived" else None, + ), + )[0] + blobs = ( + pack_state(theta, anchor.state), + pack_state(anchor.ef), + pack_state(anchor.state.v if state_policy == "derived" else {}), + ) + start = StartStateV2( + run_id=w.run_id(), + w=0, + hotkey=key.ss58, + theta_hash=state_hash(theta), + state_object_sha256=sha256_hex(blobs[0]), + opt_state_hash=optimizer_hash(anchor.state), + ef_object_sha256=sha256_hex(blobs[1]), + ef_hash=state_hash(anchor.ef), + parent_anchor_hash=anchor.anchor_hash, + global_step0=0, + anchor_verdict_hash=anchor.proof_hash, + ) + commit = CommitV2( + w=0, + hotkey=key.ss58, + leaf_scheme="ht-leaf-v1", + n_leaves=cfg.inner.n_leaves, + leaves_root=result.leaves_root, + metrics_root="11" * 32, + final_theta_hash=result.final_theta_hash, + ef_in_hash=result.ef_in_hash, + ef_out_hash=result.ef_out_hash, + delta_hash=result.delta_hash, + delta_bytes=len(result.delta_payload), + tokens=len(a.sample_ids) * cfg.model.seq_len, + ) + challenge = AuditChallengeV2( + w=0, + target=key.ss58, + beacon_round=1, + beacon_sig_sha256="22" * 32, + mode="full", + segments=[], + reasons=["final"], + serve_deadline=100, + anchor_hash=start.digest(), + audit_mode="anchored-full", + ) + auditor = Keypair(bytes([45]) * 32) + job = AuditJobV2( + run_id=w.run_id(), + job_id="33" * 32, + auditor_id=auditor.ss58, + attempt=1, + lease_nonce="44" * 32, + lease_expires=50, + absolute_deadline=100, + reservation_id="55" * 32, + replay_step_budget=2 * cfg.inner.H, + anchor_age=0, + manifest=w, + challenge_envelope=seal(auditor, "AuditChallengeV2", w.run_id(), challenge, 100), + commit_envelope=seal(key, "CommitV2", w.run_id(), commit, 100), + sample_ids=list(a.sample_ids), + start_state=start, + preimages=[x.preimage for x in result.leaves], + ef_in=dict(sha256=sha256_hex(blobs[1]), size=len(blobs[1])), + v0=dict(sha256=sha256_hex(blobs[2]), size=len(blobs[2])), + created_beacon=1, + ) + return job, cache, blobs, get, result, key + + +@pytest.mark.parametrize("objective", [None, "mlm", "decision", "distill"]) +def test_full_carry_and_window_equality(objective: str | None) -> None: + job, cache, blobs, get, expected, _ = fixture(objective) + outputs = emulate(2, lambda c: audit_island(job, c, get, *blobs, cache, now_round=2)) + assert all(out.result == "MATCH" for out, _ in outputs) + assert outputs[0][1].delta_payload == expected.delta_payload + cfg = TrainConfig.from_manifest_v2(job.manifest) + prior = cache.prior(job.manifest, job.start_state) + a = Assignment(job.run_id, 0, tuple(job.sample_ids)) + traced_steps = [set(), set()] + + def hook(ctx, x): + if ctx.op == "attn": + traced_steps[ctx.rank].add(ctx.step) + return x + + windows = emulate( + 2, + lambda c: replay_island_windows( + cfg, + job.manifest, + c, + init_params(cfg.model), + prior.state, + a, + 0, + cfg.inner.H // cfg.inner.J, + get, + hook=hook, + ), + ) + assert [p.digest() for p in windows[0][0]] == [p.digest() for p in job.preimages[1:]] + assert traced_steps == [set(range(1, cfg.inner.H + 1))] * 2 + + +def test_fake_prefix_final_and_ef_corruption() -> None: + job, cache, blobs, get, expected, key = fixture() + cfg = TrainConfig.from_manifest_v2(job.manifest) + prior = cache.prior(job.manifest, job.start_state) + bad_state = prior.state.clone() + bad_state.m[sorted(bad_state.m)[0]].add_(0.001) + bad = pack_state(init_params(cfg.model), bad_state) + start = job.start_state.model_copy(update={"state_object_sha256": sha256_hex(bad)}) + corrupt = job.model_copy(update={"start_state": start}) + with pytest.raises(AuditInputError, match="fabricated carried prefix"): + emulate(2, lambda c: audit_island(corrupt, c, get, bad, *blobs[1:], cache, now_round=2)) + with pytest.raises(AuditInputError, match="EF object"): + emulate( + 2, lambda c: audit_island(job, c, get, blobs[0], b"bad", blobs[2], cache, now_round=2) + ) + body = dict(job.commit_envelope["body"]) + body["delta_hash"] = "66" * 32 + bad_commit = seal(key, "CommitV2", job.run_id, body, 100) + mutated = job.model_copy(update={"commit_envelope": bad_commit}) + assert ( + emulate(2, lambda c: audit_island(mutated, c, get, *blobs, cache, now_round=2))[0][0].result + == "MISMATCH" + ) + + +def test_old_anchor_work_budget_and_expired_lease_refuse() -> None: + job, cache, blobs, get, _, _ = fixture() + for changes in ({"anchor_age": 2}, {"replay_step_budget": 5}): + with pytest.raises(ValidationError): + AuditJobV2.model_validate({**job.model_dump(mode="json"), **changes}) + cache.entries.clear() + with pytest.raises(AuditInputError, match="ANCHOR_BUDGET_EXCEEDED"): + emulate(2, lambda c: audit_island(job, c, get, *blobs, cache, now_round=2)) + with LeaseGuard(50, 100, time.time() + 120) as guard: + guard.advance(50) + with pytest.raises(AuditInputError, match="lease expired"): + guard.check() + with pytest.raises(ValueError, match="live authenticated"): + job.validate_embedded(50) + + +def test_honeypot_keys_not_reused_after_reveal() -> None: + first = fresh_honeypots([], ["honest", "last_step"]) + second = fresh_honeypots(first, ["honest", "last_step"]) + assert not {p.hotkey for p in first} & {p.hotkey for p in second} + + +def test_live_beacon_expiry_cancels_before_collectives() -> None: + job, cache, blobs, get, _, _ = fixture() + with pytest.raises(ValueError, match="live authenticated"): + emulate( + 2, + lambda c: audit_island(job, c, get, *blobs, cache, now_round=2, beacon_now=lambda: 50), + ) + + +def test_chain_rejects_expiry_before_reserved_predecessor_execution() -> None: + from hypertrain.auditor.replay import audit_island_chain + + # Given: a verified beacon already expired the current reservation. + job, cache, blobs, get, _, _ = fixture() + # When/Then: do not spend predecessor work under an expired current lease. + with pytest.raises(ValueError, match="live authenticated"): + emulate( + 2, + lambda c: audit_island_chain( + job, + job, + c, + get, + blobs, + blobs, + cache, + now_round=2, + beacon_now=lambda: 50, + ), + ) + + +def test_expiry_during_terminal_compression_cannot_publish_anchor( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from hypertrain.trainer import island + + # Given: live replay; compression is the last noncollective work boundary. + job, cache, blobs, get, _, _ = fixture() + expired = threading.Event() + original = island.compress + + def compress(*args, **kwargs): + result = original(*args, **kwargs) + expired.set() + return result + + monkeypatch.setattr(island, "compress", compress) + # When/Then: terminal lease expiry cannot insert a MATCH anchor. + with pytest.raises(ValueError, match="live authenticated"): + emulate( + 2, + lambda c: audit_island( + job, + c, + get, + *blobs, + cache, + now_round=2, + beacon_now=lambda: 50 if expired.is_set() else 2, + ), + ) + assert all(key[2] == -1 for key in cache.entries) + + +def test_missing_predecessor_reconstructs_once_under_two_h_budget() -> None: + from hypertrain.auditor.replay import audit_island_chain + + job, cache, blobs, get, result, key = fixture() + emulate(2, lambda c: audit_island(job, c, get, *blobs, cache, now_round=2)) + parent = cache.entries[(job.run_id, key.ss58, 0, cache.layout_hash(job.manifest))] + cfg = TrainConfig.from_manifest_v2(job.manifest) + theta = {k: x + 0.0001 for k, x in result.final_theta.items()} + next_blobs = (pack_state(theta, parent.state), pack_state(parent.ef), pack_state({})) + start = StartStateV2( + run_id=job.run_id, + w=1, + hotkey=key.ss58, + theta_hash=state_hash(theta), + state_object_sha256=sha256_hex(next_blobs[0]), + opt_state_hash=optimizer_hash(parent.state), + ef_object_sha256=sha256_hex(next_blobs[1]), + ef_hash=state_hash(parent.ef), + parent_anchor_hash=parent.anchor_hash, + global_step0=2, + anchor_verdict_hash=parent.proof_hash, + ) + a = Assignment(job.run_id, 1, tuple(job.sample_ids), 2) + trained = emulate( + 2, + lambda c: train_island( + cfg, + job.manifest.training.reference_spec.layout, + c, + theta, + a, + get, + carry=parent.state, + ef_in=parent.ef, + ), + )[0] + body = dict(job.commit_envelope["body"]) + body.update( + w=1, + leaves_root=trained.leaves_root, + final_theta_hash=trained.final_theta_hash, + delta_hash=trained.delta_hash, + delta_bytes=len(trained.delta_payload), + ef_in_hash=trained.ef_in_hash, + ef_out_hash=trained.ef_out_hash, + ) + ch = dict(job.challenge_envelope["body"]) + ch.update(w=1, anchor_hash=start.digest()) + current = AuditJobV2.model_validate( + { + **job.model_dump(mode="json"), + "job_id": "77" * 32, + "start_state": start.body(), + "anchor_age": 1, + "ef_in": {"sha256": sha256_hex(next_blobs[1]), "size": len(next_blobs[1])}, + "preimages": [ + x.preimage.body() + if hasattr(x.preimage, "body") + else x.preimage.model_dump(mode="json") + for x in trained.leaves + ], + "commit_envelope": seal(key, "CommitV2", job.run_id, body, 100), + "challenge_envelope": seal( + Keypair(bytes([45]) * 32), "AuditChallengeV2", job.run_id, ch, 100 + ), + } + ) + del cache.entries[(job.run_id, key.ss58, 0, cache.layout_hash(job.manifest))] + out = emulate( + 2, lambda c: audit_island_chain(current, job, c, get, next_blobs, blobs, cache, now_round=2) + ) + assert out[0][0].result == "MATCH" + short = current.model_copy(update={"replay_step_budget": cfg.inner.H}) + with pytest.raises(AuditInputError, match="reservation mismatch"): + emulate( + 2, + lambda c: audit_island_chain(short, job, c, get, next_blobs, blobs, cache, now_round=2), + ) + + +def test_verified_anchor_restart_corruption_and_od_warm_start(tmp_path: Path) -> None: + from hypertrain.auditor.replay import tensor_root + from hypertrain.models.opendecision import extend_params + + job, cache, blobs, get, _, key = fixture("mlm") + emulate(2, lambda c: audit_island(job, c, get, *blobs, cache, now_round=2)) + source = cache.entries[(job.run_id, key.ss58, 0, cache.layout_hash(job.manifest))] + path = cache.persist(tmp_path, source) + fresh = AnchorCache() + restored = fresh.restore( + path, + job.manifest, + source.anchor_hash, + source.proof_hash, + tensor_root(source.theta, source.state), + state_hash(source.ef), + expected_hotkey=source.hotkey, + expected_round=source.w, + expected_backend=source.backend, + ) + decision, _, _, _, _, _ = fixture("decision") + body = decision.manifest.body() + body["training"]["model"]["od"]["warm_start"] = True + target = RunManifestV2.model_validate(body) + theta = extend_params(restored.theta, TrainConfig.from_manifest_v2(target).model) + warm = fresh.warm_start(target, key.ss58, theta, restored) + assert warm.state.step == 0 + with pytest.raises(AuditInputError, match="checkpoint tensor"): + fresh.warm_start(target, key.ss58, {n: x * 1.01 for n, x in theta.items()}, restored) + (path / "ef").write_bytes(b"corruption") + with pytest.raises(AuditInputError, match="authentication mismatch"): + fresh.restore( + path, + job.manifest, + source.anchor_hash, + source.proof_hash, + tensor_root(source.theta, source.state), + state_hash(source.ef), + expected_hotkey=source.hotkey, + expected_round=source.w, + expected_backend=source.backend, + ) + + +def test_real_cpu_match_restore_rejects_metadata_only_provenance_mutation(tmp_path: Path) -> None: + import json + + from hypertrain.auditor.replay import tensor_root + + job, cache, blobs, get, _, keypair = fixture() + results = emulate(2, lambda c: audit_island(job, c, get, *blobs, cache, now_round=2)) + assert all(out.result == "MATCH" for out, _ in results) + key = (job.run_id, keypair.ss58, 0, cache.layout_hash(job.manifest)) + source = cache.entries[key] + assert source.backend == "cpu" + path = cache.persist(tmp_path, source) + metadata = (path / "metadata").read_bytes() + original = json.loads(metadata) + state_root, ef_hash = tensor_root(source.theta, source.state), state_hash(source.ef) + fresh = AnchorCache() + + def restore(): + return fresh.restore( + path, + job.manifest, + source.anchor_hash, + source.proof_hash, + state_root, + ef_hash, + expected_hotkey=source.hotkey, + expected_round=source.w, + expected_backend="cpu", + ) + + for field, value in (("backend", "cuda"), ("hotkey", Keypair(bytes([46]) * 32).ss58), ("w", 1)): + assert restore().backend == "cpu" + assert key in fresh.entries + (path / "metadata").write_text(json.dumps({**original, field: value})) + with pytest.raises(AuditInputError, match="authentication mismatch"): + restore() + assert fresh.entries == {} # failed reload invalidates prior insertion + (path / "metadata").write_bytes(metadata) + restored = restore() + assert (restored.hotkey, restored.w, restored.backend) == (source.hotkey, 0, "cpu") + assert tensor_root(restored.theta, restored.state) == state_root + assert state_hash(restored.ef) == ef_hash + + +@pytest.mark.parametrize("state_policy", ["carry", "reset", "derived"]) +def test_real_parent_auditor_runtime(tmp_path: Path, state_policy: str) -> None: + import json + + from hypertrain.auditor.worker import execute_island_audit + from hypertrain.protocol.hashing import MerkleTree + + job, cache, blobs, get, _, key = fixture(state_policy=state_policy) + rows = [get(i).astype(" None: + # Given: pin/pause committed before seed arrives; process restarts. + database = tmp_path / "freshness.sqlite" + vector = VECTORS[0] + r = vector["round"] + now = QUICKNET_GENESIS + (r - 1) * QUICKNET_PERIOD + beacon = quicknet_push_beacon() + with sqlite3.connect(database) as db: + gate = FreshnessGate(db, "run") + gate.pin("draw", r) + with pytest.raises(BeaconUnavailable): + gate.refresh( + "draw", + beacon, + now_seconds=now, + genesis_seconds=QUICKNET_GENESIS, + period_seconds=QUICKNET_PERIOD, + max_lag_rounds=0, + ) + assert gate.status("draw").paused + with sqlite3.connect(database) as db: + gate = FreshnessGate(db, "run") + with pytest.raises(BeaconVerificationError): + gate.pin("draw", r + 1) + beacon.ingest(vector) + # When + draw = gate.refresh( + "draw", + beacon, + now_seconds=now, + genesis_seconds=QUICKNET_GENESIS, + period_seconds=QUICKNET_PERIOD, + max_lag_rounds=0, + ) + # Then + assert draw.round == r + assert gate.require_fresh("draw").signature_hash == vector["randomness"] + + +def test_old_seed_not_replaced_when_fresh_health_round_required() -> None: + # Given + with sqlite3.connect(":memory:") as db: + vector = VECTORS[0] + r = vector["round"] + beacon = quicknet_push_beacon() + beacon.ingest(vector) + gate = FreshnessGate(db, "run") + gate.pin("draw", r) + now = QUICKNET_GENESIS + r * QUICKNET_PERIOD + # When / Then + with pytest.raises(BeaconUnavailable): + gate.refresh( + "draw", + beacon, + now_seconds=now, + genesis_seconds=QUICKNET_GENESIS, + period_seconds=QUICKNET_PERIOD, + max_lag_rounds=0, + ) + assert gate.status("draw").pinned_round == r + with pytest.raises(BeaconUnavailable): + gate.require_fresh("draw") + + +class WrongRoundBeacon: + def round_at_or_after(self, t: int) -> int: + return 1 + + def get(self, round: int) -> BeaconRound: + v = VECTORS[0] + return BeaconRound(v["round"] + 1, v["signature"], v["randomness"], True) + + +def test_invalid_signature_rejected_when_provider_claims_verified() -> None: + # Given + with sqlite3.connect(":memory:") as db: + gate = FreshnessGate(db, "run") + r = VECTORS[0]["round"] + 1 + gate.pin("draw", r) + # When / Then + with pytest.raises(BeaconUnavailable): + gate.refresh( + "draw", + WrongRoundBeacon(), + now_seconds=QUICKNET_GENESIS + (r - 1) * QUICKNET_PERIOD, + genesis_seconds=QUICKNET_GENESIS, + period_seconds=QUICKNET_PERIOD, + max_lag_rounds=0, + ) + assert gate.status("draw").paused diff --git a/tests/challenge/authority_fixture_v2.py b/tests/challenge/authority_fixture_v2.py new file mode 100644 index 0000000..dccbac2 --- /dev/null +++ b/tests/challenge/authority_fixture_v2.py @@ -0,0 +1,256 @@ +"""Hermetic signed admission metadata oracle, NOT independent training evidence.""" + +from __future__ import annotations + +import importlib.util +import json +import sqlite3 +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +import hypertrain.trainer # noqa: F401 +from hypertrain.auditor.replay import AnchorCache, pack_state, unpack_state +from hypertrain.data.trial_assignment import trial_samples +from hypertrain.miner.island_launch import IslandArtifacts +from hypertrain.protocol.hashing import MerkleTree, sha256_hex +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.messages import Finalize +from hypertrain.protocol.messages_v2 import ( + EscrowLock, + RankWorkResult, + RunManifestV2, + WorkProof, + WorkScreenV2, +) +from hypertrain.trainer.compress import state_hash +from hypertrain.trainer.config import TrainConfig +from hypertrain.trainer.model import init_params + +spec = importlib.util.spec_from_file_location( + "hermetic_authority_service", Path(__file__).parent / "test_service_network_v2.py" +) +assert spec is not None and spec.loader is not None +service = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = service +spec.loader.exec_module(service) + + +def metadata_artifacts(job, directory): + """Independent byte/job oracle; no manufactured optimizer trajectory claim.""" + record = json.loads((directory / "metadata-oracle.json").read_bytes()) + if record["job_hash"] != job.digest(): + raise ValueError("metadata oracle job binding differs") + for name, digest in record["files"].items(): + if sha256_hex((directory / "rank-0" / name).read_bytes()) != digest: + raise ValueError("metadata oracle artifact differs") + source = directory.parent + theta, optimizer = unpack_state((source / "start_state").read_bytes()) + expected = init_params(TrainConfig.from_manifest_v2(job.manifest).model) + anchor = AnchorCache().genesis(job.manifest, service.HOT[0].ss58, expected) + if ( + optimizer is None + or optimizer.step != 0 + or pack_state(theta, optimizer) != pack_state(anchor.theta, anchor.state) + ): + raise ValueError("metadata oracle genesis differs") + if sha256_hex((source / "start_state").read_bytes()) != job.start_state_sha256: + raise ValueError("metadata oracle source digest differs") + width = (job.manifest.training.model.seq_len + 1) * 2 + samples = (source / "samples").read_bytes() + proofs = json.loads((source / "sample_proofs").read_bytes()) + for i, sample_id in enumerate(job.sample_ids): + if not MerkleTree.verify( + samples[i * width : (i + 1) * width], + sample_id, + [bytes.fromhex(p) for p in proofs[i]], + bytes.fromhex(job.manifest.training.dataset.merkle_root), + job.manifest.training.dataset.n_samples, + ): + raise ValueError("metadata oracle sample membership differs") + return IslandArtifacts( + directory, + *( + directory / "rank-0" / name + for name in ("state.safetensors", "ef.safetensors", "delta.bin", "leaves.json") + ), + tuple(RankWorkResult.model_validate(r) for r in record["ranks"]), + ) + + +def _screen(job, directory, challenge, *, now_beacon, backend): + """Replace only expensive execution with explicit signed metadata reference.""" + assert backend == "cpu" + published = directory / "published" + rank = published / "rank-0" + rank.mkdir(parents=True) + theta, state = unpack_state((directory / "start_state").read_bytes()) + assert state is not None + # No optimizer update is claimed: metadata-only fixture, not work qualification. + values = { + "state.safetensors": pack_state(theta, state), + "ef.safetensors": (directory / "ef_in").read_bytes(), + "delta.bin": b"hermetic-authority-metadata-not-training", + "leaves.json": canonicalize({"job_hash": job.digest(), "fixture": "metadata-only"}), + } + refs = [] + for name, raw in values.items(): + (rank / name).write_bytes(raw) + refs.append({"sha256": sha256_hex(raw), "size": len(raw)}) + ranks = [ + RankWorkResult( + rank=i, + fingerprint=sha256_hex(canonicalize([job.digest(), i])), + elapsed_ns=0, + allocated_bytes=0, + reserved_bytes=0, + total_bytes=1, + ) + for i in range(job.manifest.training.reference_spec.layout.n_gpus) + ] + (published / "metadata-oracle.json").write_bytes( + canonicalize( + { + "job_hash": job.digest(), + "files": {name: sha256_hex(raw) for name, raw in values.items()}, + "ranks": [r.body() for r in ranks], + } + ) + ) + metadata_artifacts(job, published) + proof = WorkProof( + admission_id=challenge.admission_id, + challenge_hash=challenge.digest(), + leaves_root=sha256_hex(values["leaves.json"]), + delta_hash=sha256_hex(values["delta.bin"]), + artifact_refs=refs, + ) + screen = WorkScreenV2( + evidence_kind="work-proof-v1", + admission_id=challenge.admission_id, + nonce=challenge.nonce, + policy_hash=challenge.policy_hash, + image_digest=job.manifest.training.reference_spec.image_digest, + layout=challenge.layout, + challenge_hash=challenge.digest(), + rank_results=ranks, + artifact_hashes=[r["sha256"] for r in refs], + ) + return screen, proof + + +def build_authority(source: Path) -> tuple[str, dict[str, bytes]]: + """Use real service predicates for 4x12 finalized signatures, no cached state.""" + from hypertrain.challenge import admission as admission_module + + with pytest.MonkeyPatch.context() as patch: + setup_type = service.fixture.Setup + + def carry(manifest, policy, admission_policy, rows, tree): + body = manifest.body() + body["training"]["inner"].update(state_policy="carry", rewarmup_steps=0) + return setup_type( + RunManifestV2.model_validate(body), policy, admission_policy, rows, tree + ) + + patch.setattr(service.fixture, "Setup", carry) + patch.setattr(service.time, "time", lambda: 1700000000.0) + serial = iter(range(1, 1000)) + patch.setattr(admission_module.secrets, "token_hex", lambda n: f"{next(serial):0{2 * n}x}") + patch.setattr(admission_module, "screen_work", _screen) + (source.parent / "live").mkdir() + generator = service.network.__wrapped__( + source.parent / "live", SimpleNamespace(param="mlm") + ) + network = next(generator) + try: + store = network.store + _, admission, _ = store._services(network.manifest.run_id()) + for i in range(4): + if i: + network.push(network.now + 101) + identity = network.join(i).json()["admission_id"] + lock = EscrowLock( + operation_id=f"{i + 1:064x}", + owner=service.COLD[i].ss58, + units=1000, + origin_ids=[network.origin_ids[i]], + admission_id=identity, + dispute_id=None, + kind="LOCK_ADMISSION", + ) + admission.lock( + canonicalize(network.signed(service.COLD[i], "EscrowLock", lock)), + now=network.now, + ) + for _ in range(12): + network.push(network.now + 1) + raw = admission.challenge(identity, now=network.now) + admission.prepare_reference(identity, now=network.now) + row = store._db.execute( + "SELECT * FROM admissions_v2 WHERE admission_id=?", (identity,) + ).fetchone() + proof = WorkProof.model_validate_json(row["reference_json"]) + from hypertrain.protocol.messages_v2 import JoinChallenge + + c = JoinChallenge.model_validate(raw["body"]) + job, directory = store.stage_trial_v2( + network.manifest.run_id(), + c, + trial_samples( + network.manifest, identity, c.nonce, store._beacon_v2(c.seed_beacon) + ), + row["trial_epoch"], + ) + artifacts = metadata_artifacts(job, directory / "published") + screen = WorkScreenV2( + evidence_kind="work-proof-v1", + admission_id=identity, + nonce=c.nonce, + policy_hash=c.policy_hash, + image_digest=network.manifest.training.reference_spec.image_digest, + layout=c.layout, + challenge_hash=c.digest(), + rank_results=list(artifacts.ranks), + artifact_hashes=[a.sha256 for a in proof.artifact_refs], + ) + admission.proof( + canonicalize(network.signed(service.HOT[i], "WorkProof", proof)), + canonicalize(network.signed(service.HOT[i], "WorkScreenV2", screen)), + now=network.now, + ) + final = Finalize( + w=row["trial_epoch"], + included=[service.HOT[i].ss58], + entitlements_root=proof.digest(), + final_theta_hash_w1=state_hash( + unpack_state(artifacts.state.read_bytes())[0] + ), + ) + status = admission.finalize_trial( + identity, + canonicalize(network.signed(service.COORD, "Finalize", final)), + now=network.now, + ) + assert ( + status.record.state == "ACTIVE" + and status.eligible + and status.record.clean_count == 12 + ) + source.mkdir() + with sqlite3.connect(source / "challenge.db") as saved: + store._db.backup(saved) + import shutil + + for name in ("objects", "trials-v2", "ledger"): + shutil.copytree(store.state_dir / name, source / name) + seeds = [bytes(range(32)), b"\x71" * 32, bytes(31) + b"\x01"] + seeds += [bytes([i]) * 32 for i in (0x72, 0x73, 0x74, 80, 81, 82, 83, 90, 91, 92, 93)] + from hypertrain.protocol.keys import Keypair + + return network.manifest.run_id(), {Keypair(seed).ss58: seed for seed in seeds} + finally: + generator.close() + store._db.close() diff --git a/tests/challenge/protected_opening_fixture.py b/tests/challenge/protected_opening_fixture.py new file mode 100644 index 0000000..f389d1c --- /dev/null +++ b/tests/challenge/protected_opening_fixture.py @@ -0,0 +1,1865 @@ +"""Explicit CLI only: real 728/N1/H2 admission custody. Never collected by pytest.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import selectors +import subprocess +import sys +import time +from pathlib import Path +from typing import Literal, TypedDict + +APPROVAL = "RUN-80-H2-PLUS-GENESIS-640CPU-1800WALL" +TARGETS = (12, 12, 12, 4) +REPO = Path(__file__).resolve().parents[2] + + +def manager(unit: str) -> dict[str, str]: + result = subprocess.run( + ["systemctl", "show", unit], capture_output=True, text=True, check=True, timeout=1 + ) + return dict(line.split("=", 1) for line in result.stdout.splitlines() if "=" in line) + + +def native_seconds(value: str) -> float: + import re + + if value == "0": + return 0.0 + match = re.fullmatch(r"([0-9]+(?:\.[0-9]+)?)(us|ms|s|min)", value) + if match is None: + raise RuntimeError("native guard must be finite with exact duration unit") + return float(match[1]) * {"us": 0.000001, "ms": 0.001, "s": 1, "min": 60}[match[2]] + + +def terminal_collect(root: Path, pins: Path) -> None: + """Native ExecStopPost: main has exited; retain own bounded future write/exit charge.""" + authority = load(pins) + outer = manager(os.environ["CAPACITY_CONTROLLER_UNIT"]) + group = Path("/sys/fs/cgroup") / outer["ControlGroup"].lstrip("/") + own = Path("/proc/self/cgroup").read_text().strip().split("::", 1)[1] + if ( + outer["MainPID"] != "0" + or own != outer["ControlGroup"] + or outer["Slice"] != authority["owned_slice"] + or os.environ["INVOCATION_ID"] != outer["InvocationID"] + or outer["KillMode"] != "control-group" + or outer["KillSignal"] != "9" + or not 0 < native_seconds(outer["TimeoutStopUSec"]) <= 1 + ): + raise RuntimeError("collector lacks exact native terminal invocation/guard") + parent = group.parent + if ( + (parent / "cpu.max").read_text().strip() != "50000 100000" + or (parent / "memory.max").read_text().strip() != "1073741824" + or (parent / "memory.swap.max").read_text().strip() != "0" + or sorted(os.sched_getaffinity(0)) != [0, 1, 2, 3] + or os.getpriority(os.PRIO_PROCESS, 0) != 19 + ): + raise RuntimeError("collector parent quota differs") + # One native stop hook <=1s, .5CPU quota plus one100ms-period burst. + reserve = 550_000 + budget = load(root / "budget.json") + measured = int( + dict(line.split() for line in (parent / "cpu.stat").read_text().splitlines())["usage_usec"] + ) + if ( + type(authority["final_tail_reserve_usec"]) is not int + or authority["final_tail_reserve_usec"] < reserve + or budget["carry_cpu_debit"] + measured + reserve > 640_000_000 + or time.time() + 1 > budget["cutoff"] + or any( + pid != str(os.getpid()) + for path in parent.rglob("cgroup.procs") + for pid in path.read_text().split() + ) + ): + raise RuntimeError("terminal collector descendants/budget/tail not certified") + save( + root / "owned-final-counter.json", + { + "measured_owned_cpu_usec": measured, + "reserved_write_exit_cpu_usec": reserve, + "reading_not_final_future_exit": True, + "parent_retained": True, + "invocation_id": os.environ["INVOCATION_ID"], + "slice": outer["Slice"], + "shared_PID1_journald": "NOT_ATTRIBUTED", + "resume_allowed": False, + }, + ) + + +def authenticate(root: Path, scope: Path) -> dict: + """Live controller, manager timer and pinned slice, before any product import.""" + record = load(scope / "before.json") + budget = load(root / "budget.json") + pid = record["controller_pid"] + if ( + scope.parent != root / "scopes" + or budget["cpu_limit_usec"] != 640_000_000 + or budget["cutoff"] + != budget["started"] + (1_800_000_000 - budget["carry_wall_debit"]) / 1_000_000 + or type(budget["carry_cpu_debit"]) is not int + or not 0 <= budget["carry_cpu_debit"] < 640_000_000 + or type(budget["carry_wall_debit"]) is not int + or not 0 <= budget["carry_wall_debit"] < 1_800_000_000 + or record["carry_cpu_debit"] != budget["carry_cpu_debit"] + or record["carry_wall_debit"] != budget["carry_wall_debit"] + or record["sources"] != sources() + or record["cutoff"] != budget["cutoff"] + or record["started"] != budget["started"] + or record["deadline"] > budget["cutoff"] + or time.time() >= record["deadline"] + or record["remaining_cpu_usec"] <= 0 + or Path(f"/proc/{pid}/stat").read_text().split(") ", 1)[1].split()[19] + != record["controller_start"] + or str(Path(__file__).resolve()).encode() not in Path(f"/proc/{pid}/cmdline").read_bytes() + ): + raise RuntimeError("untrusted inside controller/cutoff") + import fcntl + + with (root / "controller.lock").open("a") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + pass + else: + raise RuntimeError("external controller lock absent") + shared_stat = Path("/tmp/hypertrain-network-cpu.lock").stat() + shared_key = ( + f"{os.major(shared_stat.st_dev):02x}:{os.minor(shared_stat.st_dev):02x}:" + f"{shared_stat.st_ino}" + ) + if not any( + line.split()[4] == str(pid) and line.split()[5] == shared_key + for line in Path("/proc/locks").read_text().splitlines() + if len(line.split()) >= 6 + ): + raise RuntimeError("external controller does not own original shared CPU lock") + lock_stat = (root / "controller.lock").stat() + lock_key = ( + f"{os.major(lock_stat.st_dev):02x}:{os.minor(lock_stat.st_dev):02x}:{lock_stat.st_ino}" + ) + if not any( + line.split()[4] == str(pid) and line.split()[5] == lock_key + for line in Path("/proc/locks").read_text().splitlines() + if len(line.split()) >= 6 + ): + raise RuntimeError("controller PID does not own original lock") + unit, timer, deadline_service = (manager(record[k]) for k in ("unit", "timer", "timer_service")) + group = "/" + Path("/proc/self/cgroup").read_text().strip().split("::", 1)[1].lstrip("/") + slice_info = manager(record["slice"]) + parent = Path("/sys/fs/cgroup") / slice_info["ControlGroup"].lstrip("/") + kernel = { + n: (parent / n).read_text().strip() for n in ("memory.max", "memory.swap.max", "cpu.max") + } + if ( + unit["MainPID"] != str(os.getpid()) + or unit["ActiveState"] != "active" + or unit["ControlGroup"] != group + or unit["Slice"] != record["slice"] + or timer["ActiveState"] != "active" + or timer["Unit"] != record["timer_service"] + or time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(int(record["deadline"]))) + not in timer["TimersCalendar"] + or record["unit"] not in deadline_service["ExecStart"] + or deadline_service["Slice"] != record["slice"] + or "systemctl" not in deadline_service["ExecStart"] + or "stop" not in deadline_service["ExecStart"] + or kernel != {"memory.max": "1073741824", "memory.swap.max": "0", "cpu.max": "50000 100000"} + or group != slice_info["ControlGroup"] + "/" + record["unit"] + or sorted(os.sched_getaffinity(0)) != [0, 1, 2, 3] + or os.getpriority(os.PRIO_PROCESS, 0) != 19 + ): + raise RuntimeError("inside lacks exact controller manager/kernel authority") + saved = load(scope / "controller-kernel.json") + if saved != { + "controller_pid": pid, + "controller_start": record["controller_start"], + "unit": record["unit"], + "slice": record["slice"], + "kernel": kernel, + "cutoff": record["cutoff"], + }: + raise RuntimeError("controller kernel receipt differs") + controller_group = Path(f"/proc/{pid}/cgroup").read_text().strip().split("::", 1)[1] + if not controller_group.startswith(slice_info["ControlGroup"] + "/"): + raise RuntimeError("external controller escaped shared budget slice") + return record + + +def save(path: Path, value: dict) -> None: + """Atomic fsync metadata checkpoint; no imported training code.""" + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".pending") + with temporary.open("w") as stream: + json.dump(value, stream, sort_keys=True, separators=(",", ":")) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def load(path: Path) -> dict: + with path.open() as stream: + return json.load(stream) + + +def carry_budget(root: Path, pins: dict) -> tuple[int, int]: + """Require a root-reviewed original-bound debit, in integer microseconds.""" + raw = Path(pins["carry_receipt"]).read_bytes() + if hashlib.sha256(raw).hexdigest() != pins["carry_receipt_sha256"]: + raise RuntimeError("carry receipt differs from reviewed root authority") + receipt = json.loads(raw) + original = Path(receipt["original_root"]) + fresh = receipt["status"] == "ROOT_APPROVED_FRESH_ALLOCATION" + if ( + receipt["status"] not in {"ROOT_APPROVED_CARRY", "ROOT_APPROVED_FRESH_ALLOCATION"} + or receipt["next_root"] != str(root) + or original == root + or ( + fresh + and ( + receipt["user_approval_sha256"] != pins["user_approval_sha256"] + or not receipt["user_approval_sha256"] + or receipt["allocation_id"] == receipt["original_allocation_id"] + ) + ) + or receipt["current_sources"] != sources() + or receipt["original_budget_sha256"] + != hashlib.sha256((original / "budget.json").read_bytes()).hexdigest() + or hashlib.sha256(Path(receipt["original_terminal_path"]).read_bytes()).hexdigest() + != receipt["original_terminal_sha256"] + ): + raise RuntimeError("carry receipt original/current subject differs") + cpu, wall = receipt["carry_cpu_debit"], receipt["carry_wall_debit"] + if fresh and (cpu != 0 or wall != 0 or receipt["original_unknown_cpu_reserve"] != 640_000_000): + raise RuntimeError("fresh allocation must preserve old unknown liability separately") + if ( + type(cpu) is not int + or not 0 <= cpu <= 640_000_000 + or type(wall) is not int + or not 0 <= wall <= 1_800_000_000 + ): + raise RuntimeError("carry debit exceeds strict cumulative CPU/wall ceiling") + if cpu == 640_000_000 or wall == 1_800_000_000: + raise RuntimeError("no certified remaining inclusive budget") + return cpu, wall + + +def inclusive_window(cpu: int, wall: int, owned: int, reserve: int) -> int: + """Owned parent counter plus separately certified external/final reserves only.""" + if any(type(value) is not int or value < 0 for value in (cpu, wall, owned, reserve)): + raise RuntimeError("invalid strict owned budget counter/reserve") + total = owned + reserve + if cpu + total >= 640_000_000 or wall >= 1_800_000_000: + raise RuntimeError("inclusive owned budget exhausted") + return total + + +def cpu_fallback_deadline( + remaining_usec: int, sample_time: float, arm_time: float, wall_end: float +) -> float: + """Absolute trigger, anchored before sampling; never extend for delayed arming.""" + import math + + if ( + type(remaining_usec) is not int + or remaining_usec <= 0 + or not all(math.isfinite(value) for value in (sample_time, arm_time, wall_end)) + or not 0 <= arm_time - sample_time <= 1 + ): + raise RuntimeError("invalid/exhausted CPU sample or unbounded native arming delay") + # 100ms quota period plus four100ms retained-runtime slots: .45 CPU-s. + # Two seconds cover stop/ExecStopPost; actual arm delay is charged conservatively. + endpoint = min(wall_end, sample_time + (remaining_usec - 450_000) / 500_000) + trigger = endpoint - 2 - (arm_time - sample_time) + if trigger <= arm_time: + raise RuntimeError("no positive native CPU fallback window") + return trigger + + +def sources() -> dict[str, str]: + """Freeze final product plus fixture; no runtime imports or profile requalification.""" + paths = list((REPO / "src/hypertrain").rglob("*.py")) + paths += [ + Path(__file__), + REPO / "tests/challenge/test_service_network_v2.py", + REPO / "tests/ledger/test_escrow_v2.py", + ] + return { + str(p.relative_to(REPO)): hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted(paths) + } + + +def supervise(root: Path, approved_pins: Path) -> None: + """Separate controller owns whole-tree kill, cumulative CPU accounting and wall deadline.""" + import fcntl + import uuid + + root.mkdir(parents=True, exist_ok=True) + with ( + (root / "controller.lock").open("a") as owned, + Path("/tmp/hypertrain-network-cpu.lock").open("a") as shared, + ): + fcntl.flock(owned, fcntl.LOCK_EX | fcntl.LOCK_NB) + # Subscribe/queue at caller monitor; wait here without starting the budget. + fcntl.flock(shared, fcntl.LOCK_EX | fcntl.LOCK_NB) + frozen = sources() + pins = load(approved_pins) + if ( + pins.get("subject") != "LOCALTEST_EXACT728_CURRENT_STORE" + or pins.get("backend") != "cpu-test" + ): + raise RuntimeError("local test current-subject authority missing") + if pins.get("full_sources") != frozen: + raise RuntimeError("reviewed complete fixture/controller/source pins stale") + if ( + pins["fixture_sha256"] != frozen["tests/challenge/protected_opening_fixture.py"] + or pins["store_sha256"] != frozen["src/hypertrain/challenge/store.py"] + or pins["service_fixture_sha256"] + != frozen["tests/challenge/test_service_network_v2.py"] + ): + raise RuntimeError("reviewed admission pins stale; no automatic refresh/signing") + for relative, digest in pins["modules"].items(): + if frozen["src/hypertrain/" + relative] != digest: + raise RuntimeError("reviewed product module pin stale") + import ast + + store_tree = ast.parse((REPO / "src/hypertrain/challenge/store.py").read_text()) + implementation = next( + n + for n in ast.walk(store_tree) + if isinstance(n, ast.FunctionDef) and n.name == "_service_implementation_v2" + ) + names = next( + ast.literal_eval(n.value) + for n in implementation.body + if isinstance(n, ast.Assign) + and any(isinstance(t, ast.Name) and t.id == "owned" for t in n.targets) + ) + actual_map = { + n.name: ast.dump(n, include_attributes=False) + for n in ast.walk(store_tree) + if isinstance(n, (ast.FunctionDef, ast.ClassDef)) and n.name in names + } + actual_map.update(pins["modules"]) + if ( + hashlib.sha256( + json.dumps(actual_map, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + != pins["implementation_hash"] + ): + raise RuntimeError("reviewed capacity implementation pin stale") + source_path = root / "sources.json" + if source_path.exists() and load(source_path) != frozen: + raise RuntimeError("source changed; original fixture cannot resume/requalify") + save(source_path, frozen) + scope_directories = list((root / "scopes").iterdir()) if (root / "scopes").exists() else [] + if any( + not path.is_dir() or not (path / "before.json").is_file() for path in scope_directories + ): + raise RuntimeError("missing original scope identity; budget custody lost") + prior = [path / "before.json" for path in scope_directories] + if prior: + raise RuntimeError( + "resume requires post-terminal native reconciliation; no pre-exit reset" + ) + controller_tag = "ht-opening-budget-" + hashlib.sha256(str(root).encode()).hexdigest() + retained = subprocess.run( + ["journalctl", "--no-pager", "--output=json", "-t", controller_tag], + capture_output=True, + text=True, + check=True, + ) + original_debits = [json.loads(line) for line in retained.stdout.splitlines()] + if len(original_debits) != len(prior): + raise RuntimeError("missing/extra original accounting history; no budget reset") + carry_cpu, carry_wall = carry_budget(root, pins) + for path in prior: + prior_record = load(path) + if type(prior_record.get("attempt")) is not int or prior_record["attempt"] < 1: + raise RuntimeError("invalid original controller attempt") + if ( + prior_record.get("carry_cpu_debit") != carry_cpu + or prior_record.get("carry_wall_debit") != carry_wall + ): + raise RuntimeError("original carry debit changed; no budget reset") + prior.sort(key=lambda path: load(path)["attempt"]) + consumed = carry_cpu + started = time.time() + original_budget = load(root / "budget.json") if prior else None + if original_budget is not None and ( + type(original_budget.get("cpu_limit_usec")) is not int + or original_budget["cpu_limit_usec"] != 640_000_000 + ): + raise RuntimeError("invalid strict original CPU budget") + for attempt_number, before in enumerate(prior, 1): + after = before.with_name("after.json") + if not after.exists(): + raise RuntimeError("missing full slice+controller accounting; no automatic resume") + record, debit, kernel = ( + load(before), + load(after), + load(before.with_name("kernel-final.json")), + ) + assert original_budget is not None + if type(debit.get("identity", {}).get("attempt")) is not int: + raise RuntimeError("invalid original debit identity attempt") + for original_receipt in (debit, kernel): + if any( + type(original_receipt["identity"].get(k)) is not int + or original_receipt["identity"][k] < 1 + for k in ("attempt", "controller_pid") + ): + raise RuntimeError("invalid strict original receipt PID/attempt") + scope_id = before.parent.name + expected_unit = "ht-opening-fixture-" + scope_id + ".service" + identity = { + k: record[k] + for k in ( + "attempt", + "run_root", + "unit", + "slice", + "controller_pid", + "controller_start", + "sources", + "started", + "cutoff", + ) + } + for original_receipt, fields in ( + ( + record, + ("attempt", "controller_pid", "remaining_cpu_usec", "prior_total_cpu_usec"), + ), + ( + debit, + ( + "cpu_usec", + "total_cpu_usec", + "kernel_cpu_usec", + "controller_cpu_usec", + "logger_pid", + ), + ), + (kernel, ("cpu_usec",)), + ): + if any( + type(original_receipt.get(k)) is not int or original_receipt[k] < 0 + for k in fields + ): + raise RuntimeError("invalid strict nonbool nonnegative original CPU counter") + kernel_raw = before.with_name("kernel-final.json").read_bytes() + if ( + record["attempt"] != attempt_number + or record["run_root"] != str(root) + or record["unit"] != expected_unit + or record["slice"] != pins["owned_slice"] + or record["sources"] != frozen + or record["prior_total_cpu_usec"] != consumed + or record["remaining_cpu_usec"] != 640_000_000 - consumed + or record["started"] != original_budget["started"] + or record["cutoff"] != original_budget["cutoff"] + or original_budget["cpu_limit_usec"] != 640_000_000 + or record["cutoff"] != record["started"] + (1_800_000_000 - carry_wall) / 1_000_000 + or record["deadline"] > record["cutoff"] + or kernel["identity"] != identity + or debit["identity"] != identity + or kernel["manifest_sha256"] + != hashlib.sha256( + json.dumps( + load(root / "fixture.json")["manifest"], + sort_keys=True, + separators=(",", ":"), + ).encode() + ).hexdigest() + or debit["manifest_sha256"] != kernel["manifest_sha256"] + or debit["kernel_sha256"] != hashlib.sha256(kernel_raw).hexdigest() + or debit["invocation_id"] != kernel["invocation_id"] + or debit["units"] != [record["unit"], record["timer"], record["timer_service"]] + or record["timer"] != expected_unit.removesuffix(".service") + "-deadline.timer" + or record["timer_service"] + != expected_unit.removesuffix(".service") + "-deadline.service" + or debit["cleaned"] is not True + or debit["kernel_cpu_usec"] < kernel["cpu_usec"] + or debit["cpu_usec"] != debit["kernel_cpu_usec"] + debit["controller_cpu_usec"] + or debit["total_cpu_usec"] != consumed + debit["cpu_usec"] + or debit["total_cpu_usec"] > 640_000_000 + ): + raise RuntimeError("original accounting identity/cumulative debit inconsistent") + # Independent manager journal, not a caller-rehashed local receipt. + for suffix, original_receipt in (("kernel", kernel), ("controller", debit)): + result = subprocess.run( + [ + "journalctl", + "--no-pager", + "--output=json", + "-t", + f"ht-opening-{scope_id}-kernel" if suffix == "kernel" else controller_tag, + ], + capture_output=True, + text=True, + check=True, + ) + entries = [json.loads(line) for line in result.stdout.splitlines()] + matches = [ + entry + for entry in entries + if entry.get("MESSAGE") + == json.dumps(original_receipt, sort_keys=True, separators=(",", ":")) + and entry.get("_UID") == str(os.getuid()) + ] + if ( + len(matches) != 1 + or ( + suffix == "kernel" + and ( + matches[0].get("_SYSTEMD_INVOCATION_ID") != kernel["invocation_id"] + or matches[0].get("_SYSTEMD_UNIT") != record["unit"] + ) + ) + or ( + suffix == "controller" + and matches[0].get("_PID") != str(debit["logger_pid"]) + ) + ): + raise RuntimeError("lost/inconsistent independent original accounting journal") + consumed = debit["total_cpu_usec"] + started = record["started"] + remaining = 640_000_000 - consumed + wall_end = started + (1_800_000_000 - carry_wall) / 1_000_000 + outer = manager(os.environ["CAPACITY_CONTROLLER_UNIT"]) + slice_name = pins["owned_slice"] + parent_info = manager(slice_name) + parent_group = Path("/sys/fs/cgroup") / parent_info["ControlGroup"].lstrip("/") + # Timestamp BEFORE counter read; a later lookup must never renew this sample. + sample_monotonic = time.monotonic() + sample_wall = time.time() + sampled_cpu = int( + dict(line.split() for line in (parent_group / "cpu.stat").read_text().splitlines())[ + "usage_usec" + ] + ) + controller_path = Path("/proc/self/cgroup").read_text().strip().split("::", 1)[1] + if ( + outer["MainPID"] != str(os.getpid()) + or outer["Slice"] != slice_name + or outer["ControlGroup"] != controller_path + or not controller_path.startswith(parent_info["ControlGroup"] + "/") + or outer["KillMode"] != "control-group" + or outer["KillSignal"] != "9" + or parent_info["ActiveState"] != "active" + ): + raise RuntimeError("controller must start inside persistent owned budget slice") + fallback = manager(pins["whole_parent_fallback_service"]) + fallback_timer = manager(pins["whole_parent_fallback_timer"]) + fallback_deadline = pins["whole_parent_fallback_deadline"] + expected_argv = "/usr/bin/systemctl --no-block stop " + slice_name + expected_collector = ( + str(Path(__file__).resolve()) + + " " + + str(root) + + " --collect --approved-pins " + + str(approved_pins) + ) + if ( + "argv[]=" + expected_argv + " ;" not in fallback["ExecStart"] + or fallback_timer["ActiveState"] != "active" + or fallback_timer["Unit"] != pins["whole_parent_fallback_service"] + or native_seconds(fallback_timer["AccuracyUSec"]) > 0.000001 + or native_seconds(fallback_timer["RandomizedDelayUSec"]) != 0 + or time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(int(fallback_deadline))) + not in fallback_timer["TimersCalendar"] + or fallback["Slice"] != slice_name + or fallback["KillMode"] != "control-group" + or fallback["KillSignal"] != "9" + or not 0 < native_seconds(fallback["RuntimeMaxUSec"]) <= 1 + or not 0 < native_seconds(fallback["TimeoutStopUSec"]) <= 1 + or not time.time() < fallback_deadline <= wall_end - 1 + or not 0 < native_seconds(outer["RuntimeMaxUSec"]) <= fallback_deadline - started + or not 0 < native_seconds(outer["TimeoutStopUSec"]) <= 1 + or "argv[]=" + sys.executable + " " + expected_collector + " ;" + not in outer["ExecStopPost"] + ): + raise RuntimeError("missing independently armed whole-owned-parent fallback") + if any( + (parent_group / key).read_text().strip() != value + for key, value in { + "cpu.max": "50000 100000", + "memory.max": "1073741824", + "memory.swap.max": "0", + }.items() + ): + raise RuntimeError("owned parent kernel caps differ") + external = 0 # Shared PID1/journald excluded from the owned-test allocation. + tail = pins["final_tail_reserve_usec"] + # Independent native boundary: 2s stop/hook +1s arming +.45CPU quota allowance. + if type(tail) is not int or tail < 2_000_000: + raise RuntimeError("missing bounded owned write/exit reserve") + reserve = external + tail + + def parent_cpu() -> int: + return int( + dict(line.split() for line in (parent_group / "cpu.stat").read_text().splitlines())[ + "usage_usec" + ] + ) + + baseline = pins["owned_parent_baseline_usec"] + if type(baseline) is not int or baseline != 0: + raise RuntimeError("owned parent must be new empty allocation with zero baseline") + if prior: + final_prior = load(prior[-1].with_name("owned-final-counter.json")) + if final_prior["identity"] != load(prior[-1].with_name("after.json"))["identity"]: + raise RuntimeError("missing original final owned counter identity") + baseline = final_prior["cpu_usec"] + if type(baseline) is not int or not 0 <= baseline <= parent_cpu(): + raise RuntimeError("persistent owned counter decreased or disappeared") + + def owned_cpu() -> int: + return parent_cpu() - baseline + + if (parent_group / "cpu.max.burst").read_text().strip() != "0" or sorted( + os.sched_getaffinity(0) + ) != [0, 1, 2, 3]: + raise RuntimeError("CPU fallback quota burst/placement differs") + arm_time = time.monotonic() + latest_trigger = cpu_fallback_deadline( + remaining - sampled_cpu + baseline, + sample_monotonic, + arm_time, + sample_monotonic + wall_end - sample_wall, + ) + native_trigger = sample_monotonic + fallback_deadline - sample_wall + controller_start = int(outer["ExecMainStartTimestampMonotonic"]) / 1_000_000 + controller_terminal = ( + controller_start + + native_seconds(outer["RuntimeMaxUSec"]) + + native_seconds(outer["TimeoutStopUSec"]) + ) + if native_trigger > latest_trigger or controller_terminal > native_trigger + 2: + raise RuntimeError("native whole-parent lifetime exceeds CPU-derived absolute deadline") + inclusive_window(consumed, carry_wall, owned_cpu(), reserve) + # .5CPU quota bounds future use; reserve includes sample/stop/final tail. + deadline = min( + wall_end - 1, + fallback_deadline - 1, + time.time() + (remaining - owned_cpu() - reserve) / 500_000, + ) + if remaining <= 2_000_000 or deadline <= time.time(): + raise RuntimeError("original CPU/wall budget exhausted") + scope = root / "scopes" / uuid.uuid4().hex + scope.mkdir(parents=True) + unit = "ht-opening-fixture-" + scope.name + ".service" + timer_service = unit.removesuffix(".service") + "-deadline.service" + timer = timer_service.removesuffix(".service") + ".timer" + budget_path = root / "budget.json" + if not budget_path.exists(): + save( + budget_path, + { + "started": started, + "cutoff": wall_end, + "cpu_limit_usec": 640_000_000, + "carry_cpu_debit": carry_cpu, + "carry_wall_debit": carry_wall, + }, + ) + budget = load(budget_path) + if budget != { + "started": started, + "cutoff": wall_end, + "cpu_limit_usec": 640_000_000, + "carry_cpu_debit": carry_cpu, + "carry_wall_debit": carry_wall, + }: + raise RuntimeError("immutable overall budget changed") + controller_start = ( + Path(f"/proc/{os.getpid()}/stat").read_text().split(") ", 1)[1].split()[19] + ) + save( + scope / "before.json", + { + "unit": unit, + "started": started, + "deadline": deadline, + "remaining_cpu_usec": remaining, + "sources": frozen, + "cutoff": budget["cutoff"], + "controller_pid": os.getpid(), + "controller_start": controller_start, + "slice": slice_name, + "timer": timer, + "timer_service": timer_service, + "attempt": len(prior) + 1, + "run_root": str(root), + "prior_total_cpu_usec": consumed, + "carry_cpu_debit": carry_cpu, + "carry_wall_debit": carry_wall, + "owned_counter_baseline_usec": baseline, + "cpu_sample_usec": sampled_cpu, + "cpu_sample_monotonic": sample_monotonic, + "cpu_guard_checked_monotonic": arm_time, + "latest_native_trigger_monotonic": latest_trigger, + }, + ) + save( + scope / "controller-kernel.json", + { + "controller_pid": os.getpid(), + "controller_start": controller_start, + "unit": unit, + "slice": slice_name, + "kernel": { + n: (parent_group / n).read_text().strip() + for n in ("memory.max", "memory.swap.max", "cpu.max") + }, + "cutoff": budget["cutoff"], + }, + ) + subprocess.run( + [ + "systemd-run", + "--unit=" + timer_service, + "--slice=" + slice_name, + "--on-calendar=" + + time.strftime("%Y-%m-%d %H:%M:%S UTC", time.gmtime(int(deadline))), + "--timer-property=AccuracySec=1us", + "--timer-property=RandomizedDelaySec=0", + "/usr/bin/systemctl", + "stop", + unit, + ], + check=True, + ) + proc = None + cpu = 0 + group = None + import signal + + def cancelled(signum: int, frame: object) -> None: + raise InterruptedError("capacity controller cancelled; owned cleanup required") + + signal.signal(signal.SIGTERM, cancelled) + signal.signal(signal.SIGINT, cancelled) + try: + with ( + (scope / "stdout.log").open("xb") as stdout, + (scope / "stderr.log").open("xb") as stderr, + ): + proc = subprocess.Popen( + [ + "systemd-run", + "--quiet", + "--wait", + "--pipe", + "--unit=" + unit, + "--slice=" + slice_name, + "--property=Type=exec", + "--property=ExecStopPost=" + + sys.executable + + " " + + str(Path(__file__).resolve()) + + " --account " + + str(scope), + "--property=MemoryMax=1073741824", + "--property=MemorySwapMax=0", + "--property=CPUQuota=50%", + "--property=CPUQuotaPeriodSec=100ms", + "--property=CPUAffinity=0 1 2 3", + "--property=Nice=19", + "--property=OOMPolicy=kill", + "--property=KillMode=control-group", + "--property=KillSignal=SIGKILL", + "--property=TimeoutStopSec=1", + "--property=Restart=no", + "--property=RuntimeMaxSec=" + str(max(1, int(deadline - time.time()))), + "--working-directory=" + str(REPO), + "/usr/bin/env", + "OMP_NUM_THREADS=1", + "MKL_NUM_THREADS=1", + "CUBLAS_WORKSPACE_CONFIG=:4096:8", + "PYTHONDONTWRITEBYTECODE=1", + sys.executable, + str(Path(__file__).resolve()), + "--inside", + str(root), + "--scope", + str(scope), + ], + stdout=stdout, + stderr=stderr, + ) + with selectors.DefaultSelector() as events: + pidfd = os.pidfd_open(proc.pid) + try: + events.register(pidfd, selectors.EVENT_READ) + while True: + # Budget timer only; trials await the synchronous original API. + exited = events.select(timeout=0.25) + if group is None: + result = subprocess.run( + ["systemctl", "show", unit, "-p", "ControlGroup", "--value"], + capture_output=True, + text=True, + check=True, + timeout=1, + ) + if result.stdout.strip(): + group = parent_group + if group is not None and (group / "cpu.stat").exists(): + stats = dict( + line.split() + for line in (group / "cpu.stat").read_text().splitlines() + ) + cpu = int(stats["usage_usec"]) - baseline + if consumed + cpu + reserve >= 640_000_000 or time.time() >= deadline: + subprocess.run( + [ + "systemctl", + "stop", + unit, + ], + check=True, + timeout=1, + ) + if exited: + break + finally: + os.close(pidfd) + code = proc.wait() + if code: + raise RuntimeError(f"fixture unit exit {code}; inspect durable scope logs") + finally: + receipt_path = scope / "kernel-final.json" + accounted = receipt_path.exists() + if accounted: + final_kernel = load(receipt_path) + identity = { + k: load(scope / "before.json")[k] + for k in ( + "attempt", + "run_root", + "unit", + "slice", + "controller_pid", + "controller_start", + "sources", + "started", + "cutoff", + ) + } + if ( + type(final_kernel.get("cpu_usec")) is not int + or final_kernel["cpu_usec"] < 0 + or final_kernel.get("identity") != identity + ): + raise RuntimeError("invalid current original kernel counter/identity") + logged = subprocess.run( + [ + "journalctl", + "--no-pager", + "--output=json", + "-t", + f"ht-opening-{scope.name}-kernel", + ], + capture_output=True, + text=True, + check=True, + ) + entries = [json.loads(line) for line in logged.stdout.splitlines()] + if ( + sum( + entry.get("MESSAGE") + == json.dumps(final_kernel, sort_keys=True, separators=(",", ":")) + and entry.get("_SYSTEMD_INVOCATION_ID") == final_kernel["invocation_id"] + for entry in entries + ) + != 1 + ): + raise RuntimeError("missing independent current kernel accounting") + cpu = final_kernel["cpu_usec"] + elif group is not None and (group / "cpu.stat").exists(): + assert group is not None + cpu = int( + dict(line.split() for line in (group / "cpu.stat").read_text().splitlines())[ + "usage_usec" + ] + ) + if (parent_group / "cpu.stat").exists(): + cpu = max( + cpu, + int( + dict( + line.split() + for line in (parent_group / "cpu.stat").read_text().splitlines() + )["usage_usec"] + ), + ) + # Failed dispatch may never load the worker/deadline service. + for owned_unit in (unit, timer, timer_service): + if manager(owned_unit)["LoadState"] != "not-found": + subprocess.run(["systemctl", "stop", owned_unit], check=True, timeout=1) + if manager(owned_unit)["ActiveState"] not in {"inactive", "failed"}: + raise RuntimeError("owned capacity cleanup not terminal") + if proc is not None: + proc.wait(timeout=15) + if ( + group is not None + and group.exists() + and any( + pid != str(os.getpid()) + for p in group.rglob("cgroup.procs") + for pid in p.read_text().split() + ) + ): + raise RuntimeError("fixture descendants survived; custody not completed") + if not accounted: + raise RuntimeError("missing kernel accounting; cannot resume") + measured = owned_cpu() + total = inclusive_window(consumed, carry_wall, measured, reserve) + if consumed + total > 640_000_000 or time.time() >= budget["cutoff"]: + raise RuntimeError("inclusive controller/genesis budget exhausted") + debit = { + "cpu_usec": total, + "total_cpu_usec": consumed + total, + "units": [unit, timer, timer_service], + "cleaned": True, + "identity": identity, + "kernel_sha256": hashlib.sha256(receipt_path.read_bytes()).hexdigest(), + "invocation_id": final_kernel["invocation_id"], + "kernel_cpu_usec": measured, + "controller_cpu_usec": reserve, + "manifest_sha256": final_kernel["manifest_sha256"], + "accounting_kind": "OWNED_PARENT_PLUS_CERTIFIED_RESERVES", + "measured_owned_cpu_usec": measured, + "external_bound_usec": external, + "final_tail_reserve_usec": tail, + } + with subprocess.Popen( + ["systemd-cat", "--identifier=" + controller_tag], + stdin=subprocess.PIPE, + text=True, + ) as logger: + debit["logger_pid"] = logger.pid + logger.communicate( + json.dumps(debit, sort_keys=True, separators=(",", ":")) + "\n", timeout=1 + ) + if logger.returncode != 0: + raise RuntimeError("original controller journal write failed") + save(scope / "after.json", debit) + subprocess.run( + ["systemctl", "reset-failed", unit, timer, timer_service], check=False, timeout=1 + ) + final_owned = owned_cpu() + if final_owned - measured > tail: + raise RuntimeError("final owned accounting tail exceeded certified reserve") + inclusive_window(consumed, carry_wall, final_owned, reserve) + save( + scope / "owned-final-counter.json", + { + "cpu_usec": parent_cpu(), + "slice": slice_name, + "identity": identity, + "parent_retained": True, + "exit_tail_reserve_usec": tail, + }, + ) + + +def generate(root: Path, scope: Path) -> None: + """Original signed HTTP paths; independent fullH reference and miner artifacts each trial.""" + authority = authenticate(root, scope) + import importlib.util + import shutil + import threading + from contextlib import ExitStack + from dataclasses import dataclass + from types import SimpleNamespace + from typing import Any + + import numpy as np + import pytest + from fastapi.testclient import TestClient + from numpy.typing import NDArray + from pydantic import JsonValue + + import hypertrain.trainer # noqa: F401 + from hypertrain.auditor.replay import unpack_state + from hypertrain.challenge.app import Config, create_app + from hypertrain.data.trial_assignment import trial_samples + from hypertrain.gpu_ops import work_screen + from hypertrain.ledger import Params + from hypertrain.miner.island_launch import IslandArtifacts, launch_island, validate_artifacts + from hypertrain.protocol import envelope_v2 + from hypertrain.protocol.envelope import body_digest + from hypertrain.protocol.hashing import MerkleTree, sha256_hex + from hypertrain.protocol.jcs import canonicalize + from hypertrain.protocol.messages import Finalize + from hypertrain.protocol.messages_v2 import ( + AdmissionPolicyV2, + AggregationPolicyV2, + EconomicsPolicyV2, + EscrowLock, + IslandJobV1, + JoinChallenge, + RunManifestV2, + WorkProof, + ) + from hypertrain.trainer.compress import state_hash + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.model import init_params + + if sources() != load(root / "sources.json"): + raise RuntimeError("source map changed before fixture start") + cgroup = Path("/sys/fs/cgroup") / Path("/proc/self/cgroup").read_text().strip().split("::", 1)[ + 1 + ].lstrip("/") + kernel = { + n: (cgroup / n).read_text().strip() + for n in ("memory.max", "memory.swap.max", "cpu.max", "memory.oom.group") + } + if ( + kernel + != { + "memory.max": "1073741824", + "memory.swap.max": "0", + "cpu.max": "50000 100000", + "memory.oom.group": "1", + } + or sorted(os.sched_getaffinity(0)) != [0, 1, 2, 3] + or os.getpriority(os.PRIO_PROCESS, 0) != 19 + ): + raise RuntimeError("whole fixture kernel resource mismatch") + save( + root / "kernel.json", + {**kernel, "cgroup": str(cgroup), "affinity": [0, 1, 2, 3], "nice": 19}, + ) + spec = importlib.util.spec_from_file_location( + "protected_original_service", REPO / "tests/challenge/test_service_network_v2.py" + ) + assert spec is not None and spec.loader is not None + service = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = service + spec.loader.exec_module(service) + original_setup = service.fixture.setup + + @dataclass(frozen=True) + class Setup: + manifest: RunManifestV2 + policy: EconomicsPolicyV2 + admission_policy: AdmissionPolicyV2 + rows: NDArray[np.uint32] + tree: MerkleTree + + def exact_setup() -> Setup: + setup = original_setup() + body = setup.manifest.body() + body["training"]["model"].update(d_ff=8, seq_len=2, param_count=728) + body["training"]["inner"].update( + H=2, J=1, micro_batch=1, grad_accum=1, state_policy="carry" + ) + body["training"]["outer"]["opt"] = "nesterov" + body["training"]["reference_spec"]["layout"].update( + pp=1, n_gpus=1, dp_size=1, ep_size=1, zero1=False + ) + rows: NDArray[np.uint32] = (np.arange(256 * 3).reshape(256, 3) % 16).astype(" AggregationPolicyV2: + return AggregationPolicyV2.model_validate({**values, "cclip_iters": 1}) + + with ExitStack() as stack: + patch = stack.enter_context(pytest.MonkeyPatch.context()) + patch.setattr(service.fixture, "setup", exact_setup) + patch.setattr(service, "AggregationPolicyV2", aggregation) + fixture_path = root / "fixture.json" + if fixture_path.exists(): + checkpoint = load(fixture_path) + manifest = RunManifestV2.model_validate(checkpoint["manifest"]) + live = root / "live" + config = Config( + "hypertrain", + live / "state", + "https://master.test", + None, + live / "admin", + None, + live / "coord.key", + service.OWNER.ss58, + Params( + "hypertrain", + manifest.training.beacon.genesis_time, + 4320, + 1, + manifest.training.verify.E_vest_rounds, + ), + ) + from hypertrain.beacon.core import BeaconRound + + app = create_app( + config, + verify_beacon=lambda data: BeaconRound( + data["round"], data["signature"], data["randomness"], False + ), + ) + client = stack.enter_context(TestClient(app)) + network = service.Network(client, manifest, None, checkpoint["origin_ids"]) + network.now = network.store._now(network.store._db) + network.push(network.now) + else: + live = root / "live" + live.mkdir() + generator = service.network.__wrapped__(live, SimpleNamespace()) + network = next(generator) + stack.callback(generator.close) + save( + fixture_path, + {"manifest": network.manifest.body(), "origin_ids": network.origin_ids}, + ) + run = network.manifest.run_id() + escrow, admission, _ = network.store._services(run) + escrow.verify() + + class WorkContext(TypedDict): + trial: int + hotkey: str + operation: str + + context: WorkContext = {"trial": -1, "hotkey": "", "operation": ""} + + def verify_completed(row: dict) -> None: + """Revalidate finalized facts, signatures and both publications; never repair.""" + trial, hotkey = row["epoch"], row["hotkey"] + directory = root / "trials" / str(trial) + completed = load(directory / "completed.json") + raw = (directory / "completed.json").read_bytes() + if ( + hashlib.sha256(raw).hexdigest() + != load(directory / "completed-sha256.json")["sha256"] + ): + raise RuntimeError("completed journal hash differs") + result_row = network.store._db.execute( + "SELECT * FROM admission_trial_results WHERE epoch=?", (trial,) + ).fetchone() + facts = { + k: row[k] + for k in ( + "epoch", + "admission_id", + "nonce", + "received_beacon", + "finalized_beacon", + "evidence_hash", + ) + } + if ( + completed["manifest"] != run + or completed["trial"] != trial + or completed["hotkey"] != hotkey + or completed["trial_record"] != facts + or completed["result_record"] != dict(result_row) + or result_row["outcome"] != "MATCH" + or row["finalized_beacon"] is None + ): + raise RuntimeError("completed trial original journal differs") + signed_challenge = load(directory / "challenge.json") + challenge = JoinChallenge.model_validate(signed_challenge["body"]) + proof_bundle = load(directory / "proof.json") + final = load(directory / "finalize.json") + envelopes = [ + (signed_challenge, "JoinChallenge", service.COORD.ss58), + (proof_bundle["proof"], "WorkProof", hotkey), + (proof_bundle["screen"], "WorkScreenV2", hotkey), + (final, "Finalize", service.COORD.ss58), + ] + for wire, kind, signer in envelopes: + env = envelope_v2.parse_envelope(wire) + if (env.run_id, env.type, env.signer) != ( + run, + kind, + signer, + ) or not envelope_v2.verify_envelope(wire): + raise RuntimeError("completed trial signature authority differs") + expected = WorkProof.model_validate_json(result_row["reference"]) + if ( + challenge.body() != json.loads(result_row["challenge"]) + or challenge.manifest_hash != run + or challenge.nonce != row["nonce"] + or challenge.admission_id != row["admission_id"] + or proof_bundle["proof"] != json.loads(result_row["proof"]) + or proof_bundle["screen"] != json.loads(result_row["screen"]) + or expected.body() != proof_bundle["proof"]["body"] + or row["evidence_hash"] != expected.digest() + or final["body"]["w"] != trial + or final["body"]["included"] != [hotkey] + or final["body"]["entitlements_root"] != expected.digest() + ): + raise RuntimeError("completed proof/finality binding differs") + reservation = network.store._db.execute( + "SELECT digest,receipt FROM admission_reservations WHERE reservation=?", + (f"trial-final|{row['admission_id']}|{trial}",), + ).fetchone() + original_challenge = network.store._db.execute( + "SELECT receipt FROM admission_reservations WHERE reservation=?", + (f"challenge|{row['admission_id']}|{row['nonce']}",), + ).fetchone() + if ( + reservation is None + or reservation["digest"] != body_digest(final["body"]) + or json.loads(reservation["receipt"]) != final + or original_challenge is None + or json.loads(original_challenge[0]) != signed_challenge + ): + raise RuntimeError("original finalized signature reservation differs") + for operation in ("reference", "miner"): + counter = root / "work" / f"{trial}-{operation}" + before, after = load(counter / "before.json"), load(counter / "after.json") + job = IslandJobV1.model_validate(before["job"]) + original_directory = ( + network.store.state_dir / "trials-v2" / row["nonce"] + if operation == "reference" + else root / "miner" / str(trial) + ) + if ( + before["manifest"], + before["trial"], + before["hotkey"], + before["operation"], + before["directory"], + ) != (run, trial, hotkey, operation, str(original_directory)): + raise RuntimeError("original publication identity differs") + if ( + job.manifest != network.manifest + or job.w != trial + or before["job_hash"] != job.digest() + or before["inputs"] + != { + n: sha256_hex((original_directory / p).read_bytes()) + for n, p in job.object_paths.items() + } + or after + != { + **before, + "publication": { + n: sha256_hex( + (original_directory / "published/rank-0" / n).read_bytes() + ) + for n in ( + "state.safetensors", + "ef.safetensors", + "delta.bin", + "leaves.json", + ) + }, + "summary": sha256_hex( + (original_directory / "published/rank-0/summary.json").read_bytes() + ), + } + or completed[operation] != after + ): + raise RuntimeError("original input/publication custody differs") + artifacts = validate_artifacts(job, original_directory / "published") + if [ + sha256_hex(p.read_bytes()) + for p in (artifacts.state, artifacts.ef, artifacts.delta, artifacts.leaves) + ] != [r.sha256 for r in expected.artifact_refs]: + raise RuntimeError("original independent publication differs from signed proof") + theta, _ = unpack_state(artifacts.state.read_bytes()) + if final["body"]["final_theta_hash_w1"] != state_hash(theta): + raise RuntimeError("completed final theta differs") + record = completed["status"]["record"] + ordinal = network.store._db.execute( + "SELECT COUNT(*) FROM admission_trials WHERE admission_id=? AND epoch<=? " + "AND finalized_beacon IS NOT NULL", + (row["admission_id"], trial), + ).fetchone()[0] + if ( + record["admission_id"] != row["admission_id"] + or record["hotkey"] != hotkey + or record["clean_count"] != ordinal + ): + raise RuntimeError("completed status differs from original finalization count") + if record["state"] != ("ACTIVE" if ordinal >= 12 else "PROBATION") or record[ + "canary_blocks" + ] != list(range(1, min((ordinal - 1) // 4 + 1, 3) + 1)): + raise RuntimeError("completed original canary/state history differs") + + for completed_row in network.store._db.execute( + "SELECT t.*,a.hotkey FROM admission_trials t " + "JOIN admissions_v2 a ON a.admission_id=t.admission_id " + "WHERE t.finalized_beacon IS NOT NULL" + ): + verify_completed(dict(completed_row)) + for identity_row in network.store._db.execute( + "SELECT admission_id,clean_count FROM admissions_v2" + ): + actual_count = network.store._db.execute( + "SELECT COUNT(*) FROM admission_trials WHERE admission_id=? " + "AND finalized_beacon IS NOT NULL", + (identity_row["admission_id"],), + ).fetchone()[0] + if actual_count != identity_row["clean_count"]: + raise RuntimeError("clean_count lacks original completed trial custody") + + def counted( + job: IslandJobV1, + directory: Path, + *, + backend: Literal["cpu", "cuda"], + cancel: threading.Event | None = None, + trace: bool = False, + ) -> IslandArtifacts: + authenticate(root, scope) + trial, hotkey, operation = context["trial"], context["hotkey"], context["operation"] + counter = root / "work" / f"{trial}-{operation}" + before = { + "manifest": run, + "trial": trial, + "hotkey": hotkey, + "operation": operation, + "job_hash": job.digest(), + "job": job.body(), + "directory": str(directory), + "inputs": { + name: sha256_hex((directory / rel).read_bytes()) + for name, rel in job.object_paths.items() + }, + } + counter.mkdir(parents=True, exist_ok=True) + if (counter / "before.json").exists(): + if ( + load(counter / "before.json") != before + or not (directory / "published").exists() + ): + raise RuntimeError( + "interrupted/changed attempt cannot be repeated automatically" + ) + else: + if len(list((root / "work").glob("*/before.json"))) >= 80: + raise RuntimeError("80 trajectory hard counter exhausted") + save(counter / "before.json", before) + artifacts = launch_island(job, directory, backend=backend, cancel=cancel, trace=trace) + publication = { + p.name: sha256_hex(p.read_bytes()) + for p in (artifacts.state, artifacts.ef, artifacts.delta, artifacts.leaves) + } + after = { + **before, + "publication": publication, + "summary": sha256_hex((artifacts.directory / "rank-0/summary.json").read_bytes()), + } + if (counter / "after.json").exists() and load(counter / "after.json") != after: + raise RuntimeError("completed publication changed") + if not (counter / "after.json").exists(): + save(counter / "after.json", after) + return artifacts + + patch.setattr(work_screen, "launch_island", counted) + for index, target in enumerate(TARGETS): + hotkey = service.HOT[index].ss58 + row = network.store._db.execute( + "SELECT * FROM admissions_v2 WHERE hotkey=?", (hotkey,) + ).fetchone() + if row is None: + if index: + network.push(network.now + 101) + joined = network.join(index) + assert joined.status_code == 200, joined.text + identity = joined.json()["admission_id"] + else: + identity = row["admission_id"] + lock_path = root / "locks" / f"{index}.json" + if not lock_path.exists(): + lock = EscrowLock( + operation_id=f"{index + 1:064x}", + owner=service.COLD[index].ss58, + units=1000, + origin_ids=[network.origin_ids[index]], + admission_id=identity, + dispute_id=None, + kind="LOCK_ADMISSION", + ) + save(lock_path, network.signed(service.COLD[index], "EscrowLock", lock)) + response = network.client.post(network.url + "/escrow/lock", json=load(lock_path)) + assert response.status_code == 200, response.text + while admission.store.by_id(identity).clean_count < target: + current = network.store._db.execute( + "SELECT * FROM admissions_v2 WHERE admission_id=?", (identity,) + ).fetchone() + pending = network.store._db.execute( + "SELECT outcome FROM admission_trial_results WHERE epoch=?", + (current["trial_epoch"],), + ).fetchone() + if pending is None or pending["outcome"] != "OPEN": + emitted = ( + int((time.time() - network.manifest.training.beacon.genesis_time) // 3) + 1 + ) + network.push(max(network.now + 1, emitted)) + response = network.client.get(network.url + "/join/" + identity + "/challenge") + assert response.status_code == 200, response.text + challenge = JoinChallenge.model_validate(response.json()["body"]) + trial = network.store._db.execute( + "SELECT trial_epoch FROM admissions_v2 WHERE admission_id=?", (identity,) + ).fetchone()[0] + context["trial"] = trial + context["hotkey"] = hotkey + context["operation"] = "reference" + checkpoint_dir = root / "trials" / str(trial) + save(checkpoint_dir / "challenge.json", response.json()) + response = network.client.post( + network.url + "/admin/join/" + identity + "/reference", headers=service.admin() + ) + assert response.status_code == 200, response.text + samples = tuple( + trial_samples( + network.manifest, + identity, + challenge.nonce, + network.store._beacon_v2(challenge.seed_beacon), + ) + ) + job, reference_directory = network.store.stage_trial_v2( + run, challenge, samples, trial + ) + miner_directory = root / "miner" / str(trial) + miner_directory.mkdir(parents=True, exist_ok=True) + for relative in job.object_paths.values(): + source, destination = reference_directory / relative, miner_directory / relative + if destination.exists(): + assert destination.read_bytes() == source.read_bytes() + else: + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + context["operation"] = "miner" + screen, proof = work_screen.screen_work( + job, + miner_directory, + challenge, + now_beacon=network.now, + backend="cpu", + launch=counted, + ) + reference = validate_artifacts(job, reference_directory / "published") + actual = validate_artifacts(job, miner_directory / "published") + assert reference.directory != actual.directory + assert [ + p.read_bytes() + for p in (reference.state, reference.ef, reference.delta, reference.leaves) + ] == [ + p.read_bytes() for p in (actual.state, actual.ef, actual.delta, actual.leaves) + ] + signed_path = checkpoint_dir / "proof.json" + if not signed_path.exists(): + save( + signed_path, + { + "proof": network.signed(service.HOT[index], "WorkProof", proof), + "screen": network.signed(service.HOT[index], "WorkScreenV2", screen), + }, + ) + response = network.client.post(network.url + "/join/proof", json=load(signed_path)) + assert response.status_code == 200, response.text + theta, _ = unpack_state(actual.state.read_bytes()) + final = Finalize( + w=trial, + final_theta_hash_w1=state_hash(theta), + included=[hotkey], + entitlements_root=proof.digest(), + ) + final_path = checkpoint_dir / "finalize.json" + if not final_path.exists(): + save(final_path, network.signed(service.COORD, "Finalize", final)) + response = network.client.post( + network.url + "/admin/join/" + identity + "/finalize", + json=load(final_path), + headers=service.admin(), + ) + assert response.status_code == 200, response.text + assert ( + network.store._db.execute( + "SELECT outcome FROM admission_trial_results WHERE epoch=?", (trial,) + ).fetchone()[0] + == "MATCH" + ) + save( + checkpoint_dir / "completed.json", + { + "manifest": run, + "trial": trial, + "hotkey": hotkey, + "status": response.json(), + "reference": load(root / "work" / f"{trial}-reference/after.json"), + "miner": load(root / "work" / f"{trial}-miner/after.json"), + "trial_record": dict( + network.store._db.execute( + "SELECT * FROM admission_trials WHERE epoch=?", (trial,) + ).fetchone() + ), + "result_record": dict( + network.store._db.execute( + "SELECT * FROM admission_trial_results WHERE epoch=?", (trial,) + ).fetchone() + ), + }, + ) + save( + checkpoint_dir / "completed-sha256.json", + {"sha256": sha256_hex((checkpoint_dir / "completed.json").read_bytes())}, + ) + status = admission.status(hotkey, now=network.now) + assert status.eligible and status.record.clean_count == target + assert status.record.state == ("ACTIVE" if index < 3 else "PROBATION") + assert status.record.canary_blocks == ((1, 2, 3) if index < 3 else (1,)) + escrow.verify() + statuses = [admission.status(k.ss58, now=network.now) for k in service.HOT] + assert all(s.eligible and s.funding.locked_units == 1000 for s in statuses) + assert len(list((root / "work").glob("*/after.json"))) == 80 + assert ( + network.store._db.execute( + "SELECT COUNT(*) FROM admission_trial_results WHERE outcome='MATCH'" + ).fetchone()[0] + == 40 + ) + # Missing completion custody stops; never synthesize it from clean_count. + for row in network.store._db.execute( + "SELECT t.epoch,t.admission_id,a.hotkey FROM admission_trials t " + "JOIN admissions_v2 a ON a.admission_id=t.admission_id " + "WHERE t.finalized_beacon IS NOT NULL" + ): + directory = root / "trials" / str(row["epoch"]) + if not (directory / "completed.json").exists(): + raise RuntimeError("missing original completion custody; cannot skip/reconstruct") + profile = network.store._service_profile_v2(run) + # Existing charged genesis runner: test-only manager placement, no product unlock. + authenticate(root, scope) + from hypertrain.protocol.messages import Receipt, f32hex + from hypertrain.protocol.messages_v2 import PolicyHashes, RosterEntryV2, RoundOpenV2 + + profile_bytes = canonicalize(profile.model_dump(mode="json")) + owner_path = root / "genesis-owner.json" + if not owner_path.exists(): + save( + owner_path, + network.signed( + service.OWNER, + "Receipt", + Receipt(w=0, commit_hash=sha256_hex(profile_bytes), received_round=network.now), + ), + ) + network.store.bootstrap_service_capacity_v2( + run, profile_bytes, canonicalize(load(owner_path)) + ) + entries = [ + RosterEntryV2( + hotkey=s.record.hotkey, + slot=i, + q_i=f32hex(1), + admission_id=s.record.admission_id, + coldkey_group=s.record.coldkey, + state=s.record.state, + eligible_weight=4194304, + ) + for i, s in enumerate(statuses) + ] + final_round = min( + network.now + 100, + int((authority["deadline"] - network.manifest.training.beacon.genesis_time) // 3) + 1, + ) + intent = RoundOpenV2( + w=0, + prev_final_hash="0" * 64, + theta_hash="0" * 64, + outer_state_hash="0" * 64, + center_hash="0" * 64, + roster_hash=sha256_hex(canonicalize([e.body() for e in entries])), + honeypot_commit="0" * 64, + d_open=network.now + 20, + d_assign=network.now + 21, + d_commit=network.now + 40, + d_audit=network.now + 41, + d_upload=network.now + 42, + d_final=final_round, + contract_version=2, + policy_hashes=PolicyHashes.model_validate( + {k: getattr(network.manifest.network, k) for k in PolicyHashes.model_fields} + ), + registry_epoch=network.store._registry_v2(network.manifest).epoch, + start_state_index_hash="0" * 64, + audit_mode="anchored-full", + roster=entries, + ) + intent_path = root / "genesis-intent.json" + if not intent_path.exists(): + save(intent_path, network.signed(service.COORD, "RoundOpenV2", intent)) + original_popen = subprocess.Popen + + def contained_popen(argv: list[str], **kwargs: Any) -> subprocess.Popen: + if ( + argv + and Path(argv[0]).name == "systemd-run" + and any(x.startswith("--unit=ht-cap-") for x in argv) + ): + authenticate(root, scope) + unit_name = next(x.split("=", 1)[1] for x in argv if x.startswith("--unit=")) + journal = root / "genesis-units" / f"{unit_name}.json" + if journal.exists(): + raise RuntimeError("genesis manager unit recorded; never reset or rerun") + save( + journal, + { + "unit": unit_name, + "slice": authority["slice"], + "deadline": authority["deadline"], + "argv": argv, + }, + ) + argv = [argv[0], "--slice=" + authority["slice"], *argv[1:]] + if unit_name.endswith(".service") and "--wait" in argv: + if "--property=CPUQuota=50%" not in argv: + raise RuntimeError("original genesis caller did not bind 50% quota") + return original_popen(argv, **kwargs) + + patch.setattr(subprocess, "Popen", contained_popen) + prepared_path = root / "genesis-prepared.json" + if prepared_path.exists(): + if load(prepared_path) != network.store._record_v2(run, "genesis-prepared", "run"): + raise RuntimeError("original prepared genesis custody differs") + else: + # _prepare commits service-charge through CapacityAttempt BEFORE manager launch. + prepared = network.store._prepare_capacity_genesis_v2( + run, canonicalize(load(intent_path)) + ) + save(prepared_path, prepared) + if time.time() >= authority["deadline"]: + raise RuntimeError("overall admission+genesis cutoff elapsed") + # Real prepared objects/current runtime, real signed HTTP publication; no tensor parent. + from hypertrain.challenge.store import ChallengeError + + prepared = load(prepared_path) + body = RoundOpenV2.model_validate(load(intent_path)["body"]) + result = prepared["result"] + body = body.model_copy( + update={ + "theta_hash": result["starts"][0]["theta_hash"], + "outer_state_hash": result["outer_hashes"]["outer_state_hash"], + "center_hash": result["outer_hashes"]["center_hash"], + "start_state_index_hash": sha256_hex(canonicalize(result["starts"])), + } + ) + signed_opening = network.signed(service.COORD, "RoundOpenV2", body) + save(root / "opening-request.json", signed_opening) + + def forbidden_parent(*args: Any, **kwargs: Any) -> None: + raise RuntimeError("protected opening reached parent tensor/child launch") + + import hypertrain.auditor.replay as replay_module + import hypertrain.trainer.model as model_module + + patch.setattr(model_module, "init_params", forbidden_parent) + patch.setattr(replay_module.AnchorCache, "genesis", forbidden_parent) + patch.setattr(replay_module, "pack_state", forbidden_parent) + patch.setattr(replay_module, "unpack_state", forbidden_parent) + patch.setattr(network.store, "_restore_anchor_v2", forbidden_parent) + original_record = network.store._record_v2 + negatives = [] + for fault in ("source", "roster", "prepared-digest", "cas"): + changes = network.store._db.total_changes + calls = 0 + + def stale_record(r: str, kind: str, identity: str, fault: str = fault) -> dict: + nonlocal calls + value = original_record(r, kind, identity) + if fault == "source" and kind == "service-admission": + value = {**value, "body": {**value["body"], "implementation_hash": "99" * 32}} + if kind == "genesis-prepared": + calls += 1 + if fault == "prepared-digest" or (fault == "cas" and calls == 2): + value = {**value, "request_hash": "99" * 32} + return value + + candidate = signed_opening + if fault == "roster": + wrong = body.model_copy( + update={ + "roster": list(reversed(body.roster)), + "roster_hash": sha256_hex( + canonicalize([e.body() for e in reversed(body.roster)]) + ), + } + ) + candidate = network.signed(service.COORD, "RoundOpenV2", wrong) + with pytest.MonkeyPatch.context() as negative_patch: + negative_patch.setattr(network.store, "_record_v2", stale_record) + try: + network.store.open_round_v2(run, canonicalize(candidate)) + except ChallengeError as exc: + expected = { + "source": "SERVICE_PROFILE_AUTHORITY", + "roster": "SERVICE_PREPARED_OPEN_BINDING", + "prepared-digest": "SERVICE_PREPARED_OPEN_BINDING", + "cas": "SERVICE_PROTECTED_OPEN_CAS", + }[fault] + if expected not in str(exc): + raise + negatives.append( + { + "fault": fault, + "error": str(exc), + "sql_changes": network.store._db.total_changes - changes, + "injected_read_race": fault in ("source", "prepared-digest", "cas"), + } + ) + else: + raise RuntimeError("stale opening unexpectedly published") + if ( + network.store._db.total_changes != changes + or (network.store.state_dir / "anchors-v2").exists() + ): + raise RuntimeError("stale opening published before authority/CAS guard") + save(root / "opening-negatives.json", {"controls": negatives}) + calls_before = len(list((root / "work").glob("*/before.json"))) + response = network.client.post( + network.url + "/admin/rounds", json=signed_opening, headers=service.admin() + ) + save( + root / "opening-response.json", + {"status_code": response.status_code, "body": response.json()}, + ) + assert response.status_code == 200, response.text + assert response.json() == signed_opening + assert len(list((root / "work").glob("*/before.json"))) == calls_before == 80 + assert original_record(run, "round", "0") == signed_opening + for start in result["starts"]: + assert original_record(run, "start", "0:" + start["hotkey"]) == start + assert ( + original_record(run, "anchor", "-1:" + start["hotkey"])["anchor_hash"] + == start["parent_anchor_hash"] + ) + changes = network.store._db.total_changes + repeated = network.client.post( + network.url + "/admin/rounds", json=signed_opening, headers=service.admin() + ) + assert repeated.status_code == 409 and network.store._db.total_changes == changes + save( + root / "opening-repeat.json", + {"status_code": repeated.status_code, "body": repeated.json(), "sql_changes": 0}, + ) + save( + root / "eligible.json", + { + "manifest": network.manifest.body(), + "run_id": run, + "profile": profile.model_dump(mode="json"), + "beacon": network.now, + "source_map": load(root / "sources.json"), + "clean_counts": list(TARGETS), + "trajectories": 80, + "finalizations": 40, + "genesis_receipt": load(prepared_path), + "opening": response.json(), + }, + ) + print("ELIGIBLE_EXACT728_3ACTIVE12_1PROBATION4", flush=True) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("root", type=Path) + parser.add_argument("--approve", choices=[APPROVAL]) + parser.add_argument("--inside", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--account", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--collect", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--scope", type=Path, help=argparse.SUPPRESS) + parser.add_argument("--approved-pins", type=Path) + args = parser.parse_args() + root = args.root.resolve() + if args.collect: + if args.approved_pins is None: + parser.error("collector requires original root pins") + terminal_collect(root, args.approved_pins.resolve()) + elif args.account: + record = load(root / "before.json") + group = Path("/sys/fs/cgroup") / manager(record["slice"])["ControlGroup"].lstrip("/") + cpu = ( + int( + dict(line.split() for line in (group / "cpu.stat").read_text().splitlines())[ + "usage_usec" + ] + ) + - record["owned_counter_baseline_usec"] + ) + identity = { + k: record[k] + for k in ( + "attempt", + "run_root", + "unit", + "slice", + "controller_pid", + "controller_start", + "sources", + "started", + "cutoff", + ) + } + receipt = { + "cpu_usec": cpu, + "memory_events": (group / "memory.events").read_text(), + "identity": identity, + "manifest_sha256": hashlib.sha256( + json.dumps( + load(root.parent.parent / "fixture.json")["manifest"], + sort_keys=True, + separators=(",", ":"), + ).encode() + ).hexdigest(), + "invocation_id": os.environ["INVOCATION_ID"], + } + subprocess.run( + ["systemd-cat", "--identifier=" + f"ht-opening-{root.name}-kernel"], + input=json.dumps(receipt, sort_keys=True, separators=(",", ":")) + "\n", + text=True, + check=True, + ) + save(root / "kernel-final.json", receipt) + elif args.inside: + if args.scope is None: + parser.error("missing external controller scope") + generate(root, args.scope.resolve()) + elif args.approve != APPROVAL: + parser.error("explicit user approval required; no default trajectory execution") + else: + if args.approved_pins is None: + parser.error("independently approved final admission pins required") + supervise(root, args.approved_pins.resolve()) + + +if __name__ == "__main__": + main() diff --git a/tests/challenge/protected_opening_fixture_static.py b/tests/challenge/protected_opening_fixture_static.py new file mode 100644 index 0000000..94facf4 --- /dev/null +++ b/tests/challenge/protected_opening_fixture_static.py @@ -0,0 +1,214 @@ +"""Three original reviewer source checks; stdlib AST only, no fixture import.""" + +import ast +import hashlib +import json +import runpy +import tempfile +from pathlib import Path + + +def main() -> None: + source = Path(__file__).with_name("protected_opening_fixture.py").read_text() + tree = ast.parse(source) + functions = {n.name: n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef)} + generate = functions["generate"] + assert isinstance(generate.body[1], ast.Assign) + assert isinstance(generate.body[1].value, ast.Call) + assert isinstance(generate.body[1].value.func, ast.Name) + assert generate.body[1].value.func.id == "authenticate" + authority = ast.get_source_segment(source, functions["authenticate"]) + assert authority is not None + for field in ( + "controller_start", + "/proc/locks", + "MainPID", + "ControlGroup", + "TimersCalendar", + "controller-kernel.json", + "budget.json", + "cutoff", + ): + assert field in authority + body = ast.get_source_segment(source, generate) + assert body is not None + assert body.index("verify_completed(dict(completed_row))") < body.index("for index, target") + custody = ast.get_source_segment(source, functions["verify_completed"]) + assert custody is not None + for field in ( + "completed-sha256.json", + "verify_envelope", + "trial_record", + "result_record", + "reference", + "miner", + "validate_artifacts", + "final_theta_hash_w1", + ): + assert field in custody + assert "recovered_finality" not in source + supervisor = ast.get_source_segment(source, functions["supervise"]) + placement = ast.get_source_segment(source, functions["contained_popen"]) + assert supervisor is not None and placement is not None + assert "parent_group" in supervisor and "controller_path" in supervisor + assert "original_controller_group" not in supervisor + assert "resource.getrusage" not in supervisor + assert "os.cpu_count" not in source + assert "ROOT_VERIFIED_NATIVE_EDGE_BOUND" not in source + assert "terminal_collect" in functions + collector = ast.get_source_segment(source, functions["terminal_collect"]) + assert "MainPID" in collector and "INVOCATION_ID" in collector + assert "measured + reserve" in collector and "time.time() + 1" in collector + assert "resume_allowed" in collector + assert "whole_parent_fallback_deadline" in supervisor + assert "argv[]=" in supervisor and "TimeoutStopUSec" in supervisor + assert "MemoryOOMGroup" not in supervisor + assert '"--property=OOMPolicy=kill"' in supervisor + assert '"memory.oom.group": "1"' in body + assert ( + "cpu.stat" in supervisor and "640_000_000" in supervisor and "1_800_000_000" in supervisor + ) + assert "carry_budget(root, pins)" in supervisor + assert "inclusive_window(consumed, carry_wall, measured, reserve)" in supervisor + assert supervisor.index('["systemctl", "reset-failed"') < supervisor.index( + "final_owned = owned_cpu()" + ) + assert "signal.SIGTERM, cancelled" in supervisor and "signal.SIGINT, cancelled" in supervisor + assert ( + '"--property=OOMPolicy=kill"' in supervisor + and '"--property=KillMode=control-group"' in supervisor + ) + assert "missing full slice+controller accounting" in supervisor + for field in ( + "type(original_receipt.get(k)) is not int", + "original_receipt[k] < 0", + "prior_total_cpu_usec", + "kernel_sha256", + "invocation_id", + "manifest_sha256", + "journalctl", + "_SYSTEMD_INVOCATION_ID", + "lost/inconsistent independent original accounting journal", + ): + assert field in supervisor + assert supervisor.index('consumed = debit["total_cpu_usec"]') < supervisor.index( + 'slice_name = pins["owned_slice"]' + ) + assert 'consumed += load(after)["cpu_usec"]' not in supervisor + assert "--slice=" in placement and "CPUQuota=50%" in placement + assert "_prepare_capacity_genesis_v2" in body + assert '"genesis_receipt": load(prepared_path)' in body + assert '"/admin/rounds"' in body and '"opening": response.json()' in body + assert "SERVICE_PROTECTED_OPEN_CAS" in body and "opening-negatives.json" in body + shared_lock = next( + n + for n in ast.walk(functions["supervise"]) + if isinstance(n, ast.Call) + and ast.unparse(n.func) == "fcntl.flock" + and n.args + and ast.unparse(n.args[0]) == "shared" + ) + assert ast.unparse(shared_lock) == "fcntl.flock(shared, fcntl.LOCK_EX | fcntl.LOCK_NB)" + shared_lock_source = ast.get_source_segment(source, shared_lock) + assert shared_lock_source is not None + assert supervisor.index(shared_lock_source) < supervisor.index("started = time.time()") + assert '"/tmp/hypertrain-network-cpu.lock"' in supervisor and '"--no-fork"' not in supervisor + assert "reviewed admission pins stale" in supervisor + assert not any(isinstance(n, ast.FunctionDef) and n.name.startswith("test_") for n in tree.body) + module = runpy.run_path(str(Path(__file__).with_name("protected_opening_fixture.py"))) + with tempfile.TemporaryDirectory() as directory: + original = Path(directory) / "original" + original.mkdir() + (original / "budget.json").write_text('{"cpu_limit_usec":640000000}') + terminal = original / "terminal.json" + terminal.write_text('{"exit":1}') + candidate = Path(directory) / "next" + receipt = { + "status": "ROOT_APPROVED_CARRY", + "original_root": str(original), + "next_root": str(candidate), + "current_sources": module["sources"](), + "original_budget_sha256": hashlib.sha256( + (original / "budget.json").read_bytes() + ).hexdigest(), + "original_terminal_path": str(terminal), + "original_terminal_sha256": hashlib.sha256(terminal.read_bytes()).hexdigest(), + "carry_cpu_debit": 10, + "carry_wall_debit": 20, + } + receipt_path = Path(directory) / "carry.json" + + def consume(value): + receipt_path.write_text(json.dumps(value)) + return module["carry_budget"]( + candidate, + { + "carry_receipt": str(receipt_path), + "carry_receipt_sha256": hashlib.sha256(receipt_path.read_bytes()).hexdigest(), + }, + ) + + assert consume(receipt) == (10, 20) + for field, value in ( + ("carry_cpu_debit", True), + ("carry_cpu_debit", -1), + ("carry_cpu_debit", 640_000_001), + ("carry_wall_debit", float("inf")), + ("carry_wall_debit", 1_800_000_001), + ("carry_cpu_debit", 640_000_000), + ): + try: + consume({**receipt, field: value}) + except RuntimeError: + pass + else: + raise AssertionError("invalid carry admitted") + (original / "budget.json").unlink() + try: + consume(receipt) + except FileNotFoundError: + pass + else: + raise AssertionError("missing original budget admitted") + assert module["inclusive_window"](0, 0, 100, 200) == 300 + deadline = module["cpu_fallback_deadline"] + immediate = deadline(640_000_000, 100.0, 100.0, 1900.0) + delayed = deadline(640_000_000, 100.0, 100.75, 1900.0) + assert immediate == 1377.1 + assert delayed == immediate - 0.75 + # Stalled Python observer cannot extend native endpoint or consume beyond640. + assert (immediate + 2 - 100) * 500_000 + 450_000 <= 640_000_000 + assert deadline(640_000_000, 100.0, 100.5, 110.0) == 107.5 + assert deadline(630_000_000, 100.0, 100.0, 1900.0) == 1357.1 + for remaining, sample, arm, cutoff in ( + (0, 100.0, 100.0, 1900.0), + (1_000_000, 100.0, 100.0, 1900.0), + (640_000_000, 100.0, 102.0, 1900.0), + (640_000_000, 100.0, 100.0, 99.0), + ): + try: + deadline(remaining, sample, arm, cutoff) + except RuntimeError: + pass + else: + raise AssertionError("exhausted/delayed native deadline admitted") + for cpu, wall, owned, reserve in ( + (639_999_000, 0, 900, 100), + (0, 1_800_000_000, 0, 0), + (0, 0, True, 0), + (0, 0, -1, 0), + ): + try: + module["inclusive_window"](cpu, wall, owned, reserve) + except RuntimeError: + pass + else: + raise AssertionError("owned counter/reserve exhaustion ignored") + print( + "CARRY_NEGATIVES_INCLUSIVE_CLEANUP_PASS; " + "STATIC_AUTHORITY_CUSTODY_GENESIS_CHECKS_OK; 80_RUNTIME_UNRUN" + ) + + +if __name__ == "__main__": + main() diff --git a/tests/challenge/test_admission_v2.py b/tests/challenge/test_admission_v2.py new file mode 100644 index 0000000..6b5e3ad --- /dev/null +++ b/tests/challenge/test_admission_v2.py @@ -0,0 +1,653 @@ +"""Real shadow trajectories plus durable identity, outbox and probation negatives.""" + +from __future__ import annotations + +import importlib +import sys +from pathlib import Path + +import pytest + +from hypertrain.beacon.core import FixtureBeacon +from hypertrain.challenge.admission import Admission +from hypertrain.challenge.admission_store import AdmissionError +from hypertrain.data.store import LocalFSStore +from hypertrain.ledger.escrow_v2 import Settlement, digest +from hypertrain.miner.admission import sign_join, sign_proof, sign_rotation +from hypertrain.protocol.envelope_v2 import seal +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.messages import Finalize +from hypertrain.protocol.messages_v2 import ( + EscrowLock, + HardwareHint, + JoinChallenge, + JoinRequest, + RotateRequest, + WorkProof, + WorkScreenV2, +) + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "ledger")) +fixtures = importlib.import_module("test_escrow_v2") +COLD, COORD, HOT, OTHER, H = fixtures.COLD, fixtures.COORD, fixtures.HOT, fixtures.OTHER, fixtures.H +Setup, fund, ledger, setup, stage = ( + fixtures.Setup, + fixtures.fund, + fixtures.ledger, + fixtures.setup, + fixtures.stage, +) + + +def service(s: Setup, path: Path) -> Admission: + e = ledger(s, path / "ledger.db") + + def reference(c: JoinChallenge, samples: tuple[int, ...], epoch: int) -> tuple: + directory = path / f"reference-{c.nonce}" + return stage(s, directory, samples, epoch), directory + + return Admission( + e, + s.admission_policy, + COORD, + beacon=FixtureBeacon(current=100000).get, + stage_reference=reference, + objects=LocalFSStore(path / "objects"), + ip_secret=b"private-prefix-key", + qualified=lambda _: True, + conservative_bound=lambda: True, + backend="cpu", + ) + + +def request(s: Setup, request_id: str = H, hot=HOT, cold=COLD) -> JoinRequest: + return sign_join( + hot, + cold, + run_id=s.manifest.run_id(), + request_id=request_id, + expires_beacon=10000, + policy_hash=digest(s.admission_policy), + hardware_hint=HardwareHint(device_name="advisory", device_count=999, driver="cpu-test"), + ) + + +def apply(a: Admission, s: Setup, now: int = 1) -> str: + return a.join(canonicalize(request(s).body()), now=now, ip_prefix="loopback").admission_id + + +def current_proof(a: Admission, admission_id: str, now: int) -> tuple[WorkProof, WorkScreenV2]: + row = a.store.db.execute( + "SELECT * FROM admissions_v2 WHERE admission_id=?", (admission_id,) + ).fetchone() + assert row is not None and row["reference_json"] is not None + proof = WorkProof.model_validate_json(row["reference_json"]) + c = JoinChallenge.model_validate_json(row["challenge"]) + from hypertrain.data.trial_assignment import trial_samples + from hypertrain.gpu_ops.work_screen import screen_work + + samples = trial_samples(a.escrow.manifest, admission_id, c.nonce, a.beacon(c.seed_beacon)) + job, directory = a.stage_reference(c, samples, row["trial_epoch"]) + import shutil + + miner_directory = directory.with_name("miner-" + c.nonce) + miner_directory.mkdir(exist_ok=True) + for relative in job.object_paths.values(): + shutil.copyfile(directory / relative, miner_directory / relative) + directory = miner_directory + screen, actual = screen_work(job, directory, c, now_beacon=now, backend="cpu") + assert actual == proof + return proof, screen + + +def submit(a: Admission, proof: WorkProof, screen: WorkScreenV2, now: int = 2): + return a.proof( + canonicalize(sign_proof(HOT, a.escrow.run_id, proof, 500)), + canonicalize(seal(HOT, "WorkScreenV2", a.escrow.run_id, screen, 500)), + now=now, + ) + + +def test_join_is_idempotent_when_restart(tmp_path: Path) -> None: + # Given + s = setup() + a = service(s, tmp_path) + admission = apply(a, s) + # When + restored = service(s, tmp_path) + same = restored.join(canonicalize(request(s).body()), now=1, ip_prefix="different") + # Then + assert same.admission_id == admission and same.state == "APPLIED" + assert restored.store.db.execute("SELECT COUNT(*) FROM admissions_v2").fetchone()[0] == 1 + assert restored.status(HOT.ss58, now=1).shadow_only + + +@pytest.mark.parametrize("fault", ["hot_sig", "cold_sig", "run_id", "late", "conflict", "coldkey"]) +def test_join_rejects_when_auth_or_replay_wrong(tmp_path: Path, fault: str) -> None: + # Given + s = setup() + a = service(s, tmp_path) + apply(a, s) + req = request(s) + if fault in ("hot_sig", "cold_sig", "run_id"): + req = JoinRequest.model_validate( + {**req.body(), fault: "0" * 128 if fault.endswith("sig") else H} + ) + if fault == "conflict": + req = request(s, hot=OTHER) + if fault == "coldkey": + req = request(s, "22" * 32, hot=OTHER) + # When / Then + with pytest.raises(AdmissionError): + a.join(canonicalize(req.body()), now=10001 if fault == "late" else 1, ip_prefix="loopback") + assert a.store.db.execute("SELECT COUNT(*) FROM admissions_v2").fetchone()[0] == 1 + + +def test_quota_rejects_when_exhausted_same_beacon(tmp_path: Path) -> None: + # Given + a = service(setup(), tmp_path) + with a.escrow.tx(): + for _ in range(4): + a.store.quota("cold", 1, 2, 4) + # When / Then + with a.escrow.tx(), pytest.raises(AdmissionError): + a.store.quota("cold", 1, 2, 4) + + +def test_trial_capacity_rejects_third_shadow_reference(tmp_path: Path) -> None: + from hypertrain.protocol.keys import Keypair + + # Given + s = setup() + a = service(s, tmp_path) + ids = [] + for i in range(3): + k = Keypair(bytes([30 + i]) * 32) + r = request(s, f"{i + 1:02x}" * 32, k, k) + ids.append(a.join(canonicalize(r.body()), now=1, ip_prefix="loopback").admission_id) + a.challenge(ids[0], now=2) + a.challenge(ids[1], now=2) + # When / Then + with pytest.raises(AdmissionError, match="TRIAL_CAPACITY"): + a.challenge(ids[2], now=2) + assert a.store.db.execute("SELECT COUNT(*) FROM admission_trials").fetchone()[0] == 2 + + +def test_challenge_nonce_and_signed_receipt_survive_restart(tmp_path: Path) -> None: + # Given + s = setup() + a = service(s, tmp_path) + admission = apply(a, s) + first = a.challenge(admission, now=2) + # When + restored = service(s, tmp_path) + second = restored.challenge(admission, now=3) + # Then: original signature/receipt, not merely a matching regenerated body. + assert first == second + assert restored.store.db.execute("SELECT COUNT(*) FROM admission_trials").fetchone()[0] == 1 + + +def test_resume_rejects_illegal_applied_transition(tmp_path: Path) -> None: + # Given + s = setup() + a = service(s, tmp_path) + apply(a, s) + # When / Then + with pytest.raises(AdmissionError): + a.resume(HOT.ss58, now=2) + assert a.store.record(HOT.ss58).state == "APPLIED" + + +def test_resume_requires_fresh_challenge_without_erasing_history(tmp_path: Path) -> None: + # Given + s = setup() + a = service(s, tmp_path) + admission = apply(a, s) + first = a.challenge(admission, now=2) + a.suspend(HOT.ss58, reason="infrastructure", now=2) + # When + a.resume(HOT.ss58, now=3) + second = a.challenge(admission, now=4) + # Then: old nonce cannot masquerade as fresh probation. + assert first["body"] != second["body"] + assert a.store.record(HOT.ss58).clean_count == 0 + + +def test_trial_capacity_released_when_challenge_expires(tmp_path: Path) -> None: + # Given + s = setup() + a = service(s, tmp_path) + admission = apply(a, s) + a.challenge(admission, now=2) + # When + a.challenge(admission, now=103) + # Then: expiry is infrastructure suspension, not a fabricated clean result. + assert a.store.record(HOT.ss58).state == "SUSPENDED" + assert ( + a.store.db.execute( + "SELECT COUNT(*) FROM admission_trial_results WHERE outcome='OPEN'" + ).fetchone()[0] + == 0 + ) + assert a.store.record(HOT.ss58).clean_count == 0 + + +@pytest.mark.parametrize("boundary", ["before_ledger", "after_ledger", "after_receipt"]) +def test_outbox_single_lock_when_crash_boundary(tmp_path: Path, boundary: str) -> None: + # Given: reserve durable operation before any accounting effect. + s = setup() + a = service(s, tmp_path) + admission = apply(a, s) + origin, _ = fund(a.escrow) + op = EscrowLock( + operation_id="22" * 32, + owner=COLD.ss58, + units=1000, + origin_ids=[origin], + admission_id=admission, + dispute_id=None, + kind="LOCK_ADMISSION", + ) + a.store.enqueue_lock(op, COLD.ss58) + if boundary in ("after_ledger", "after_receipt"): + a.escrow.lock(op, signer=COLD.ss58) + if boundary == "after_receipt": + a.store.reconcile_lock(op.operation_id) + # When: reconstruct process state and reconcile original operation. + restored = service(s, tmp_path) + receipt = restored.store.reconcile_lock(op.operation_id) + # Then: no second lock, no granted eligibility from an unacknowledged outbox. + assert restored.escrow.balances().admission_locked == 1000 + assert restored.store.reconcile_lock(op.operation_id) == receipt + assert not restored.status(HOT.ss58, now=2).eligible + assert restored.escrow.balances().conserved() + + +def test_rotation_carries_locks_strikes_dispute_and_history(tmp_path: Path) -> None: + # Given + s = setup() + a = service(s, tmp_path) + admission = apply(a, s) + origin, _ = fund(a.escrow) + op = EscrowLock( + operation_id="22" * 32, + owner=COLD.ss58, + units=1000, + origin_ids=[origin], + admission_id=admission, + dispute_id=None, + kind="LOCK_ADMISSION", + ) + a.store.enqueue_lock(op, COLD.ss58) + a.store.reconcile_lock(op.operation_id) + a.suspend(HOT.ss58, reason="pending", now=2, pending_dispute=True) + a.store.db.execute("UPDATE admissions_v2 SET strikes=2 WHERE admission_id=?", (admission,)) + rotate = RotateRequest(hotkey=HOT.ss58, new_hotkey=OTHER.ss58, operation_id="33" * 32) + # When + r = a.rotate(canonicalize(sign_rotation(COLD, a.escrow.run_id, rotate, 200)), now=2) + # Then + assert r.admission_id == admission and r.pending_dispute and r.strikes == 2 + assert a.status(OTHER.ss58, now=2).funding.locked_units == 1000 + with pytest.raises(AdmissionError): + a.status(HOT.ss58, now=2) + with pytest.raises(AdmissionError): + a.resume(OTHER.ss58, now=2) + + +def test_rotation_rejects_hotkey_instead_of_coldkey(tmp_path: Path) -> None: + # Given + s = setup() + a = service(s, tmp_path) + apply(a, s) + op = RotateRequest(hotkey=HOT.ss58, new_hotkey=OTHER.ss58, operation_id="33" * 32) + # When / Then + with pytest.raises(ValueError): + a.rotate(canonicalize(sign_rotation(HOT, a.escrow.run_id, op, 200)), now=2) + assert a.store.record(HOT.ss58).state == "APPLIED" + + +def accepted_settlement( + a: Admission, admission_id: str, coldkey: str, reference: str, evidence_hash: str +) -> None: + """Persist actual service acceptance; callback checks the mutation transaction.""" + a.store.db.execute( + "CREATE TABLE IF NOT EXISTS accepted_settlements(reference TEXT PRIMARY KEY, body TEXT)" + ) + status = Settlement( + finality_hash=H, + closed_dispute_root=H, + release_beacon=2, + unresolved=False, + outcome="MATCH", + run_id=a.escrow.run_id, + admission_id=admission_id, + coldkey=coldkey, + dispute_id=reference, + evidence_hash=evidence_hash, + ) + a.store.db.execute( + "INSERT INTO accepted_settlements VALUES(?,?)", (reference, status.model_dump_json()) + ) + + def read(reference: str) -> Settlement: + assert a.store.db.in_transaction + row = a.store.db.execute( + "SELECT body FROM accepted_settlements WHERE reference=?", (reference,) + ).fetchone() + assert row is not None + return Settlement.model_validate_json(row[0]) + + a.escrow.settlement = read + + +def recovery_snapshot(a: Admission, hotkey: str) -> tuple: + return ( + a.store.record(hotkey), + tuple( + tuple(row) + for row in a.store.db.execute("SELECT * FROM admission_pending ORDER BY dispute_id") + ), + tuple( + tuple(row) + for row in a.store.db.execute("SELECT * FROM admission_transitions ORDER BY seq") + ), + ) + + +def test_pending_recovery_rejects_cross_admission_accepted_match(tmp_path: Path) -> None: + # Given: two actual dual-signed joins, second alone has accepted MATCH. + s = setup() + a = service(s, tmp_path) + first = apply(a, s) + second = a.join( + canonicalize(request(s, "22" * 32, OTHER, OTHER).body()), now=1, ip_prefix="loopback" + ).admission_id + a.suspend( + HOT.ss58, + reason="pending", + now=2, + pending_dispute=True, + dispute_id="33" * 32, + evidence_hash="44" * 32, + ) + a.suspend( + OTHER.ss58, + reason="pending", + now=2, + pending_dispute=True, + dispute_id="55" * 32, + evidence_hash="66" * 32, + ) + accepted_settlement(a, second, OTHER.ss58, "55" * 32, "66" * 32) + before = recovery_snapshot(a, HOT.ss58) + # When / Then: trusted accepted record cannot clear first identity. + with pytest.raises(AdmissionError): + a.resolve_pending(HOT.ss58, "55" * 32, now=2) + assert recovery_snapshot(a, HOT.ss58) == before + assert a.store.record(HOT.ss58).admission_id == first + + +def test_pending_recovery_rejects_old_dispute_for_current_incident(tmp_path: Path) -> None: + # Given: one real identity recovered once, then new pending evidence. + s = setup() + a = service(s, tmp_path) + admission = apply(a, s) + a.suspend( + HOT.ss58, + reason="old", + now=2, + pending_dispute=True, + dispute_id="33" * 32, + evidence_hash="44" * 32, + ) + accepted_settlement(a, admission, COLD.ss58, "33" * 32, "44" * 32) + a.resolve_pending(HOT.ss58, "33" * 32, now=2) + a.suspend( + HOT.ss58, + reason="current", + now=3, + pending_dispute=True, + dispute_id="55" * 32, + evidence_hash="66" * 32, + ) + before = recovery_snapshot(a, HOT.ss58) + # When / Then + with pytest.raises(AdmissionError): + a.resolve_pending(HOT.ss58, "33" * 32, now=3) + assert recovery_snapshot(a, HOT.ss58) == before + + +@pytest.mark.parametrize( + "field,value", + [ + ("admission_id", H), + ("coldkey", OTHER.ss58), + ("run_id", H), + ("evidence_hash", H), + ("dispute_id", H), + ("evidence_hash", None), + ], +) +def test_pending_recovery_rejects_wrong_accepted_linkage(tmp_path: Path, field: str, value) -> None: + # Given: correct lookup reference, but the accepted record linkage differs. + s = setup() + a = service(s, tmp_path) + admission = apply(a, s) + a.suspend( + HOT.ss58, + reason="current", + now=2, + pending_dispute=True, + dispute_id="33" * 32, + evidence_hash="44" * 32, + ) + accepted_settlement(a, admission, COLD.ss58, "33" * 32, "44" * 32) + row = a.store.db.execute("SELECT body FROM accepted_settlements").fetchone() + status = Settlement.model_validate_json(row[0]) + wrong = Settlement.model_validate({**status.body(), field: value}) + a.store.db.execute("UPDATE accepted_settlements SET body=?", (wrong.model_dump_json(),)) + before = recovery_snapshot(a, HOT.ss58) + # When / Then + with pytest.raises(AdmissionError): + a.resolve_pending(HOT.ss58, "33" * 32, now=2) + assert recovery_snapshot(a, HOT.ss58) == before + + +def test_rotation_preserves_exact_pending_reference_across_restart(tmp_path: Path) -> None: + # Given + s = setup() + a = service(s, tmp_path) + admission = apply(a, s) + a.suspend( + HOT.ss58, + reason="pending", + now=2, + pending_dispute=True, + dispute_id="33" * 32, + evidence_hash="44" * 32, + ) + rotate = RotateRequest(hotkey=HOT.ss58, new_hotkey=OTHER.ss58, operation_id="77" * 32) + a.rotate(canonicalize(sign_rotation(COLD, a.escrow.run_id, rotate, 200)), now=2) + restored = service(s, tmp_path) + accepted_settlement(restored, admission, COLD.ss58, "33" * 32, "44" * 32) + before = recovery_snapshot(restored, OTHER.ss58) + # When / Then: retired key and wrong reference leave the binding intact. + with pytest.raises(AdmissionError): + restored.resolve_pending(HOT.ss58, "33" * 32, now=2) + with pytest.raises(AdmissionError): + restored.resolve_pending(OTHER.ss58, "88" * 32, now=2) + assert recovery_snapshot(restored, OTHER.ss58) == before + restored.resolve_pending(OTHER.ss58, "33" * 32, now=2) + assert restored.store.record(OTHER.ss58).state == "PROBATION" + assert not restored.store.record(OTHER.ss58).pending_dispute + assert restored.store.db.execute("SELECT resolved FROM admission_pending").fetchone()[0] == 1 + + +def test_legacy_unlinked_pending_recovery_fails_closed(tmp_path: Path) -> None: + # Given: pre-fix pending boolean, no fabricated dispute identity. + s = setup() + a = service(s, tmp_path) + admission = apply(a, s) + a.suspend(HOT.ss58, reason="legacy", now=2, pending_dispute=True) + accepted_settlement(a, admission, COLD.ss58, "33" * 32, "44" * 32) + before = recovery_snapshot(a, HOT.ss58) + # When / Then + with pytest.raises(AdmissionError): + a.resolve_pending(HOT.ss58, "33" * 32, now=2) + assert recovery_snapshot(a, HOT.ss58) == before + + +@pytest.fixture(scope="module") +def trial_launches(): + """Count real reference/miner launches at the existing shared launcher seam.""" + from hypertrain.gpu_ops import work_screen + from hypertrain.miner import island_launch + + original = island_launch.launch_island + calls = [] + + def counted(job, directory, **kwargs): + calls.append((job.model_dump(mode="json", exclude={"deadline"}), directory)) + return original(job, directory, **kwargs) + + with pytest.MonkeyPatch.context() as patch: + patch.setattr(island_launch, "launch_island", counted) + patch.setattr(work_screen, "launch_island", counted) + yield calls + + +@pytest.fixture(scope="module") +def actual_proof(tmp_path_factory: pytest.TempPathFactory, trial_launches): + # Actual full-round launch and independently committed reference, no worker PASS mock. + path = tmp_path_factory.mktemp("real-admission") + s = setup() + a = service(s, path) + admission = apply(a, s) + a.challenge(admission, now=2) + before = len(trial_launches) + a.prepare_reference(admission, now=2) + proof, screen = current_proof(a, admission, 2) + nonce = screen.nonce + pair = trial_launches[before:] + assert len(pair) == 2 and pair[0][0] == pair[1][0] + assert [p.name for _, p in pair] == ["reference-" + nonce, "miner-" + nonce] + return s, a, proof, screen + + +@pytest.mark.parametrize( + "fault", ["signature", "late", "challenge", "nonce", "layout", "artifact", "unsupported"] +) +def test_proof_rejects_when_not_bound_to_reference(actual_proof, fault: str) -> None: + # Given + s, a, p, screen = actual_proof + p = WorkProof.model_validate(p.body()) + screen = WorkScreenV2.model_validate(screen.body()) + if fault == "challenge": + p = WorkProof.model_validate({**p.body(), "challenge_hash": H}) + if fault == "nonce": + screen = WorkScreenV2.model_validate({**screen.body(), "nonce": H}) + if fault == "artifact": + screen = WorkScreenV2.model_validate({**screen.body(), "artifact_hashes": [H]}) + if fault == "unsupported": + from pydantic import ValidationError + + with pytest.raises(ValidationError): + WorkScreenV2.model_validate({**screen.body(), "evidence_kind": "nvidia-cc"}) + return + if fault == "layout": + screen = WorkScreenV2.model_validate( + {**screen.body(), "layout": {**screen.layout.model_dump(mode="json"), "zero1": True}} + ) + signer = OTHER if fault == "signature" else HOT + # When / Then + with pytest.raises(ValueError): + a.proof( + canonicalize(sign_proof(signer, a.escrow.run_id, p, 500)), + canonicalize(seal(HOT, "WorkScreenV2", a.escrow.run_id, screen, 500)), + now=103 if fault == "late" else 2, + ) + assert a.escrow.balances().issued == 0 + + +def test_full_shadow_probation_and_graduation_when_funded(tmp_path: Path, trial_launches) -> None: + # Given: each finalized participation executes its own nonce-bound full reference. + s = setup() + a = service(s, tmp_path) + admission = apply(a, s) + origin, _ = fund(a.escrow) + op = EscrowLock( + operation_id="22" * 32, + owner=COLD.ss58, + units=1000, + origin_ids=[origin], + admission_id=admission, + dispute_id=None, + kind="LOCK_ADMISSION", + ) + a.lock(canonicalize(seal(COLD, "EscrowLock", a.escrow.run_id, op, 500)), now=1) + before = len(trial_launches) + for i in range(12): + now = i + 2 + a.challenge(admission, now=now) + a.prepare_reference(admission, now=now) + proof, screen = current_proof(a, admission, now) + assert len(trial_launches) - before == 2 * (i + 1) + reference, miner = trial_launches[-2:] + assert reference[0] == miner[0] + assert [reference[1].name, miner[1].name] == [ + "reference-" + screen.nonce, + "miner-" + screen.nonce, + ] + submit(a, proof, screen, now) + row = a.store.db.execute( + "SELECT * FROM admissions_v2 WHERE admission_id=?", (admission,) + ).fetchone() + from hypertrain.auditor.replay import unpack_state + from hypertrain.trainer.compress import state_hash + + theta, _ = unpack_state(a.objects.get(proof.artifact_refs[0].sha256)) + final = Finalize( + w=row["trial_epoch"], + final_theta_hash_w1=state_hash(theta), + included=[HOT.ss58], + entitlements_root=digest(proof), + ) + # When: authenticated finalize consumes the real committed reference once. + status = a.finalize_trial( + admission, canonicalize(seal(COORD, "Finalize", a.escrow.run_id, final, 500)), now=now + ) + # Then: exact schedule; calendar time never adds clean participation. + assert status.record.clean_count == i + 1 and status.actual_q_ppm == 1_000_000 + assert status.nominal_q_ppm == ( + 1_000_000 if i + 1 < 4 else 500_000 if i + 1 < 8 else 250_000 + ) + if i < 3: + assert status.shadow_only + if i < 11: + assert status.record.state == "PROBATION" + assert status.record.state == "ACTIVE" and status.eligible + assert status.record.canary_blocks == (1, 2, 3) + assert a.escrow.balances().issued == 30000 # no reward minted by trial PASS + assert a.escrow.balances().conserved() + # Honest graduation does not waive ongoing full replay: a new defective assignment suspends. + a.challenge(admission, now=14) + a.prepare_reference(admission, now=14) + proof, screen = current_proof(a, admission, 14) + assert len(trial_launches) - before == 26 + bad = WorkProof.model_validate({**proof.body(), "delta_hash": H}) + r = submit(a, bad, screen, 14) + assert r.state == "SUSPENDED" and r.pending_dispute + assert not a.status(HOT.ss58, now=14).eligible + + +def test_zero_origins_cannot_graduate_even_after_calendar_age(tmp_path: Path) -> None: + # Given + s = setup() + a = service(s, tmp_path) + apply(a, s) + # When + status = a.graduate(HOT.ss58, now=1000) + # Then + assert ( + not status.eligible and status.record.clean_count == 0 and status.record.state == "APPLIED" + ) diff --git a/tests/challenge/test_audit_deadline_v2.py b/tests/challenge/test_audit_deadline_v2.py new file mode 100644 index 0000000..e0ad94a --- /dev/null +++ b/tests/challenge/test_audit_deadline_v2.py @@ -0,0 +1,412 @@ +"""Real tiny audit geometry publication binds the accepted lease descriptor.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + +from hypertrain.auditor.replay import AnchorCache, optimizer_hash, pack_state +from hypertrain.challenge.store import ChallengeError +from hypertrain.miner.island_launch import launch_island +from hypertrain.protocol import envelope_v2 +from hypertrain.protocol.hashing import sha256_hex +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.messages import LeafPreimage, Receipt +from hypertrain.protocol.messages_v2 import ( + AuditChallengeV2, + AuditJobV2, + CommitV2, + IslandJobV1, + PolicyHashes, + RoundOpenV2, + StartStateV2, +) +from hypertrain.trainer.compress import state_hash +from hypertrain.trainer.config import TrainConfig +from hypertrain.trainer.model import init_params + +spec = importlib.util.spec_from_file_location( + "audit_publication_fixture", Path(__file__).parent / "test_service_network_v2.py" +) +assert spec is not None and spec.loader is not None +fixtures = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = fixtures +spec.loader.exec_module(fixtures) +network = fixtures.network + + +def test_audit_roster_rejects_unaccepted_round_without_mutation(network): + store = network.store + before = store._db.total_changes + with pytest.raises(ChallengeError, match="missing accepted round: 0"): + store.require_roster_v2(network.manifest.run_id(), 0) + assert store._db.total_changes == before + + +def test_real_audit_publication_completes_with_exact_lease_descriptor(network, monkeypatch): + store, manifest = network.store, network.manifest + run = manifest.run_id() + hot, auditor = fixtures.HOT[0], fixtures.AUDITORS[0] + # Real signed admission/proof/finality path used by neighboring service integration. + path = Path(__file__).parents[1] / "e2e/test_service_network_v2_e2e.py" + spec = importlib.util.spec_from_file_location("audit_round_acceptance_fixture", path) + assert spec is not None and spec.loader is not None + rounds = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = rounds + spec.loader.exec_module(rounds) + rounds._graduate_four(network) + cfg = TrainConfig.from_manifest_v2(manifest) + theta = init_params(cfg.model) + cache = AnchorCache() + starts, roster = [], [] + admission = store._services(run)[1] + for slot, key in enumerate(fixtures.HOT): + status = admission.status(key.ss58, now=network.now) + assert status.eligible and status.record.state == "ACTIVE" + actual = cache.genesis(manifest, key.ss58, theta) + starts.append( + StartStateV2( + run_id=run, + w=0, + hotkey=key.ss58, + theta_hash=state_hash(theta), + state_object_sha256=sha256_hex(pack_state(theta, actual.state)), + opt_state_hash=optimizer_hash(actual.state), + ef_object_sha256=sha256_hex(pack_state(actual.ef)), + ef_hash=state_hash(actual.ef), + parent_anchor_hash=actual.anchor_hash, + global_step0=0, + anchor_verdict_hash=actual.proof_hash, + ) + ) + roster.append( + { + "hotkey": key.ss58, + "slot": slot, + "q_i": "0000803f", + "admission_id": status.record.admission_id, + "coldkey_group": status.record.coldkey, + "state": status.record.state, + "eligible_weight": 4194304, + } + ) + opening = RoundOpenV2( + w=0, + prev_final_hash="0" * 64, + theta_hash=state_hash(theta), + outer_state_hash="0" * 64, + center_hash="0" * 64, + roster_hash=sha256_hex(canonicalize(roster)), + honeypot_commit="0" * 64, + d_open=network.now + 1, + d_assign=network.now + 2, + d_commit=network.now + 3, + d_audit=network.now + 4, + d_upload=network.now + 5, + d_final=network.now + 7, + contract_version=2, + policy_hashes=PolicyHashes.model_validate( + {name: getattr(manifest.network, name) for name in PolicyHashes.model_fields} + ), + registry_epoch=0, + start_state_index_hash=sha256_hex(canonicalize([s.body() for s in starts])), + audit_mode="anchored-full", + roster=roster, + ) + before = store._db.total_changes + with pytest.raises(ChallengeError, match="missing accepted round: 0"): + store.require_roster_v2(run, 0) + assert store._db.total_changes == before + signed_opening = network.signed(fixtures.COORD, "RoundOpenV2", opening) + response = network.client.post( + network.url + "/admin/rounds", json=signed_opening, headers=fixtures.admin() + ) + assert response.status_code == 200, response.text + assert store._record_v2(run, "round", "0") == signed_opening + assert store.require_roster_v2(run, 0) is None + anchor = cache.genesis(manifest, hot.ss58, theta) + store._persist_anchor_v2(manifest, anchor, cache) + source = store.state_dir / "jobs-v2/0" / hot.ss58 + source.mkdir(parents=True) + dataset = store._record_v2(run, "dataset", "inputs") + sample_ids = list(range(manifest.training.batch_samples())) + width = (cfg.model.seq_len + 1) * ( + 2 if manifest.training.dataset.sample_format.startswith("u16") else 4 + ) + raw_samples = store.objects.get(dataset["samples_hash"]) + proofs = json.loads(store.objects.get(dataset["proofs_hash"])) + inputs = { + "start_state": pack_state(theta), + "ef_in": pack_state(anchor.ef), + "v0": pack_state({}), + "samples": b"".join(raw_samples[i * width : (i + 1) * width] for i in sample_ids), + "sample_proofs": canonicalize([proofs[i] for i in sample_ids]), + } + hashes = {} + for name, raw in inputs.items(): + (source / name).write_bytes(raw) + hashes[name] = store.objects.put(raw) + original = IslandJobV1( + job_version=1, + run_id=run, + w=0, + manifest=manifest, + sample_ids=sample_ids, + global_step0=0, + start_state_sha256=hashes["start_state"], + ef_in_sha256=hashes["ef_in"], + v0_sha256=hashes["v0"], + object_paths={k: k for k in inputs}, + deadline=1, + ) + job_id, nonce = "aa" * 32, "bb" * 32 + created = network.now + expiry = created + 99 + deadline = manifest.training.beacon.genesis_time + (expiry - 1) * 3 + published = launch_island( + original.model_copy(update={"deadline": deadline}), source, backend="cpu" + ) + summary = json.loads((published.directory / "rank-0/summary.json").read_bytes())["commitments"] + leaves = [LeafPreimage.model_validate(p) for p in json.loads(published.leaves.read_bytes())] + start_blob = store.objects.put(pack_state(theta, anchor.state)) + start = StartStateV2( + run_id=run, + w=0, + hotkey=hot.ss58, + theta_hash=state_hash(theta), + state_object_sha256=start_blob, + opt_state_hash=optimizer_hash(anchor.state), + ef_object_sha256=hashes["ef_in"], + ef_hash=state_hash(anchor.ef), + parent_anchor_hash=anchor.anchor_hash, + global_step0=0, + anchor_verdict_hash=anchor.proof_hash, + ) + commit = CommitV2( + w=0, + hotkey=hot.ss58, + leaf_scheme="ht-leaf-v1", + n_leaves=len(leaves), + metrics_root="11" * 32, + tokens=len(sample_ids) * cfg.model.seq_len, + delta_bytes=published.delta.stat().st_size, + ef_in_hash=state_hash(anchor.ef), + **{k: summary[k] for k in ("leaves_root", "final_theta_hash", "ef_out_hash", "delta_hash")}, + ) + challenge = AuditChallengeV2( + w=0, + target=hot.ss58, + beacon_round=created, + beacon_sig_sha256="11" * 32, + mode="full", + segments=[], + reasons=["final"], + serve_deadline=expiry, + anchor_hash=start.digest(), + audit_mode="anchored-full", + ) + audit = AuditJobV2( + run_id=run, + job_id=job_id, + auditor_id=auditor.ss58, + attempt=1, + lease_nonce=nonce, + lease_expires=expiry, + absolute_deadline=expiry, + reservation_id=nonce, + replay_step_budget=cfg.inner.H, + anchor_age=0, + manifest=manifest, + challenge_envelope=network.signed(fixtures.COORD, "AuditChallengeV2", challenge), + commit_envelope=network.signed(hot, "CommitV2", commit), + sample_ids=sample_ids, + start_state=start, + preimages=leaves, + ef_in={"sha256": hashes["ef_in"], "size": len(inputs["ef_in"])}, + v0={"sha256": hashes["v0"], "size": len(inputs["v0"])}, + created_beacon=created, + ) + with store._tx(): + store._put_record_v2(run, "island-job", "0:" + hot.ss58, original.body()) + store._put_record_v2(run, "audit-job", job_id, audit.body()) + store._put_record_v2(run, "assignment", "0:" + hot.ss58, {"assignment_hash": "11" * 32}) + store._put_record_v2(run, "finality", "0", {"unresolved_audits": 1}) + store._db.execute( + "INSERT INTO audit_leases_v2(id,run_id,w,hotkey,challenge,created,absolute," + "state,attempts,auditor,nonce,expires) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)", + ( + job_id, + run, + 0, + hot.ss58, + "{}", + created, + expiry, + "running", + 1, + auditor.ss58, + nonce, + expiry, + ), + ) + from hypertrain.auditor import worker + from hypertrain.auditor.bisect import RefereeError + from hypertrain.miner.island_launch import IslandFailure + + audit_launches = [] + + def independent_launch(job, directory, **kwargs): + audit_launches.append(job) + assert kwargs["trace"] is True + assert directory == store.state_dir / "audit-geometry-v2" / job_id / nonce + assert directory != source + return launch_island(job, directory, **kwargs) + + monkeypatch.setattr(worker, "launch_island", independent_launch) + result = store.execute_audit_v2( + run, + job_id, + canonicalize( + network.signed( + auditor, "Receipt", Receipt(w=0, commit_hash=nonce, received_round=created) + ) + ), + ) + assert audit_launches == [original.model_copy(update={"deadline": deadline})] + assert result["verdict"]["result"] == "MATCH" + signed = network.signed(auditor, "ReplayVerdict", result["verdict"]) + record = store._record_v2(run, "audit-geometry", job_id + ":" + nonce) + descriptor = envelope_v2.load_json(store.objects.get(record["descriptor_hash"])) + assert descriptor["job"] == original.model_copy(update={"deadline": deadline}).body() + assert store._record_v2(run, "island-job", "0:" + hot.ss58) == original.body() + publication = Path(descriptor["directory"]) / "published" + for name in ("summary.json", "trace.json"): + path = publication / "rank-0" / name + original_bytes = path.read_bytes() + changed = json.loads(original_bytes) + if name == "summary.json": + changed["job_hash"] = "00" * 32 + else: + changed[0]["rank"] = 1 + path.write_bytes(canonicalize(changed)) + with pytest.raises((IslandFailure, RefereeError, ValueError)): + store.complete_audit_v2(run, job_id, canonicalize(signed)) + assert ( + store._db.execute("SELECT state FROM audit_leases_v2 WHERE id=?", (job_id,)).fetchone()[ + 0 + ] + == "running" + ) + path.write_bytes(original_bytes) + for fault in ("deadline", "lease", "signature"): + bad = dict(record) + if fault == "signature": + env = envelope_v2.parse_envelope(record["receipt"]) + bad["receipt"] = network.signed(hot, "Receipt", env.body) + else: + changed = json.loads(canonicalize(descriptor)) + if fault == "deadline": + changed["job"]["deadline"] += 1 + else: + changed["lease_nonce"] = "cc" * 32 + bad["descriptor_hash"] = store.objects.put(canonicalize(changed)) + bad["receipt"] = network.signed( + fixtures.COORD, + "Receipt", + Receipt(w=0, commit_hash=bad["descriptor_hash"], received_round=created), + ) + store._put_record_v2(run, "audit-geometry", job_id + ":" + nonce, bad) + with pytest.raises(ChallengeError, match="geometry authority"): + store.complete_audit_v2(run, job_id, canonicalize(signed)) + assert ( + store._db.execute("SELECT state FROM audit_leases_v2 WHERE id=?", (job_id,)).fetchone()[ + 0 + ] + == "running" + ) + store._put_record_v2(run, "audit-geometry", job_id + ":" + nonce, record) + assert store.complete_audit_v2(run, job_id, canonicalize(signed)) == {"result": "MATCH"} + assert store.complete_audit_v2(run, job_id, canonicalize(signed)) == {"result": "MATCH"} + geometry = store._record_v2(run, "trace-geometry", sha256_hex(canonicalize(result["verdict"]))) + assert geometry["job"] == descriptor["job"] + # Given: genuine accepted historical publication, a live current dispute. + import time + + from hypertrain.challenge.disputes_v2 import Contest, Turn + from hypertrain.miner import island_launch + from hypertrain.miner.island_launch import IslandFailure + + verdict_hash = sha256_hex(canonicalize(result["verdict"])) + contest = Contest.model_validate(store._record_v2(run, "contest", verdict_hash)) + dispute_id, transcript = "dd" * 32, "ee" * 32 + turn = Turn( + run_id=run, + dispute_id=dispute_id, + contest=contest, + seq=0, + level="step", + ctx=[], + interval=(0, 1), + expected_party=hot.ss58, + transcript_hash=transcript, + turn_deadline=created + 109, + absolute_deadline=created + 129, + lock_id="ff" * 32, + ) + with store._tx(): + store._db.execute( + "INSERT INTO disputes_v2 VALUES(?,?,?,?,?)", + (dispute_id, verdict_hash, hot.ss58, turn.model_dump_json(), "{}"), + ) + now = created + 100 + monkeypatch.setattr(store, "_now", lambda _: now) + monkeypatch.setattr(store, "_snapshot_current_v2", lambda _: None) + monkeypatch.setattr(time, "time", lambda: deadline + 1) + launches = [] + + def replay(job, directory, **kwargs): + launches.append(job) + assert job.deadline == manifest.training.beacon.genesis_time + (created + 128) * 3 + accepted = store._record_v2(run, "referee-job", dispute_id + ":" + transcript) + receipt = envelope_v2.parse_envelope(accepted["receipt"]) + assert receipt.signer == fixtures.COORD.ss58 + assert envelope_v2.verify_envelope(accepted["receipt"]) + frozen = envelope_v2.load_json(store.objects.get(accepted["descriptor_hash"])) + assert frozen["job"] == job.body() + assert frozen["historical_geometry_hash"] == sha256_hex(canonicalize(geometry)) + return launch_island(job, directory, **kwargs) + + monkeypatch.setattr(island_launch, "launch_island", replay) + # When: invalid historical publication or expired current dispute cannot launch. + wrong = json.loads(canonicalize(geometry)) + wrong["job"]["deadline"] += 1 + store._put_record_v2(run, "trace-geometry", verdict_hash, wrong) + with pytest.raises(IslandFailure, match="job binding"): + store.referee_v2(run, dispute_id) + assert not launches + store._put_record_v2(run, "trace-geometry", verdict_hash, geometry) + now = created + 129 + with pytest.raises(ChallengeError, match="absolute horizon"): + store.referee_v2(run, dispute_id) + assert not launches + now = created + 100 + # Then: actual fresh replay validates separate outputs, registers independent evidence. + evidence = store.referee_v2(run, dispute_id) + assert evidence["evidence"]["reason"] == "MATCH" + assert len(launches) == 1 + assert launches[0] == original.model_copy( + update={"deadline": manifest.training.beacon.genesis_time + (created + 128) * 3} + ) + assert store._record_v2(run, "trace-geometry", verdict_hash) == geometry + assert store._record_v2(run, "island-job", "0:" + hot.ss58) == original.body() + assert ( + store._db.execute( + "SELECT body FROM dispute_evidence_v2 WHERE hash=?", (evidence["evidence_hash"],) + ).fetchone() + is not None + ) + assert not store._lease_guards_v2 diff --git a/tests/challenge/test_authority_snapshot_v2.py b/tests/challenge/test_authority_snapshot_v2.py new file mode 100644 index 0000000..65a8c5b --- /dev/null +++ b/tests/challenge/test_authority_snapshot_v2.py @@ -0,0 +1,223 @@ +"""Portable finalized admissions retain original authority; no training in this test.""" + +from __future__ import annotations + +import importlib.util +import json +import shutil +import sqlite3 +import sys +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient +from pydantic import Field, create_model + +from hypertrain.challenge.app import create_app +from hypertrain.protocol import envelope_v2 +from hypertrain.protocol.hashing import sha256_hex +from hypertrain.protocol.jcs import canonicalize + +ROOT = Path(__file__).resolve().parents[2] +spec = importlib.util.spec_from_file_location( + "portable_authority_adapter", ROOT / "scripts/network_authority_snapshot.py" +) +assert spec is not None and spec.loader is not None +adapter = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = adapter +spec.loader.exec_module(adapter) +fixture_spec = importlib.util.spec_from_file_location( + "hermetic_authority_contract", Path(__file__).parent / "authority_fixture_v2.py" +) +assert fixture_spec is not None and fixture_spec.loader is not None +authority_fixture = importlib.util.module_from_spec(fixture_spec) +sys.modules[fixture_spec.name] = authority_fixture +fixture_spec.loader.exec_module(authority_fixture) + + +@pytest.fixture(scope="module") +def package(tmp_path_factory): + root = tmp_path_factory.mktemp("hermetic-od-authority") + source, directory = root / "source", root / "bundle" + run_id, originals = authority_fixture.build_authority(source) + with pytest.MonkeyPatch.context() as patch: + # Test-local run pin and declared metadata oracle; production adapter unchanged. + patch.setattr(adapter, "RUN", run_id) + patch.setattr( + adapter, + "Bundle", + create_model( + "HermeticBundle", + __base__=adapter.Bundle, + run_id=(str, Field(default=run_id, pattern="^" + run_id + "$")), + ), + ) + patch.setattr( + adapter, + "Bundle", + create_model( + "HermeticBundle", + __base__=adapter.Bundle, + run_id=(str, Field(default=run_id, pattern="^" + run_id + "$")), + ), + ) + patch.setattr(adapter, "validate_artifacts", authority_fixture.metadata_artifacts) + digest = adapter.export(source, directory) + yield directory, digest, originals + + +def test_original_export_validates_and_bootstraps_actual_service(package, tmp_path): + directory, digest, originals = package + bundle = adapter.validate(directory, digest) + secrets = tmp_path / "roles" + secrets.mkdir(mode=0o700) + for role, public in bundle.public_roles.items(): + path = secrets / (role + ".seed") + path.write_bytes(originals[public]) + path.chmod(0o600) + token = secrets / "admin.token" + token.write_text("portable-test-admin") + token.chmod(0o600) + state = tmp_path / "service" + with adapter.bootstrap(directory, digest, state, secrets, "https://master.test") as config: + app = create_app(config, verify_beacon=adapter.fixture_beacon) + with TestClient(app): + store = app.state.store + escrow, admission, _ = store._services(adapter.RUN) + escrow.verify() + rows = store._db.execute("SELECT * FROM admissions_v2").fetchall() + assert len(rows) == 4 + for row in rows: + restored_now = store._now(store._db) + assert restored_now >= bundle.beacon_round + status = admission.status(row["hotkey"], now=restored_now) + assert status.record.state == "ACTIVE" and status.eligible + assert status.record.clean_count == 12 and status.funding.locked_units == 1000 + assert store._db.execute("SELECT COUNT(*) FROM accepted_v2").fetchone()[0] == 1 + assert store._db.execute("SELECT COUNT(*) FROM admission_trials").fetchone()[0] == 48 + store._db.close() + bad = secrets / "coord.seed" + bad.write_bytes(bytes(32)) + rejected = tmp_path / "rejected" + with pytest.raises(ValueError, match="public identity"): + with adapter.bootstrap(directory, digest, rejected, secrets, "https://master.test"): + pytest.fail("wrong original role accepted") + assert not rejected.exists() + + +@pytest.mark.parametrize( + "fault", + [ + "object", + "reference", + "foreign", + "traversal", + "digest", + "screen", + "strike", + "pending", + "funding", + "rotation", + "signature", + "finality", + "assignment", + ], +) +def test_modified_package_rejects_before_restore(package, tmp_path, fault): + original, digest, _ = package + directory = tmp_path / "changed" + shutil.copytree(original, directory) + body = json.loads((directory / "bundle.json").read_bytes()) + if fault == "object": + relative = next(k for k in body["files"] if k.startswith("objects/")) + (directory / relative).unlink() + elif fault in ( + "reference", + "foreign", + "screen", + "strike", + "pending", + "funding", + "rotation", + "signature", + "finality", + "assignment", + ): + with sqlite3.connect(directory / "challenge.db") as db: + if fault == "reference": + db.execute("UPDATE admission_trial_results SET reference='{}' WHERE epoch=0") + elif fault == "screen": + db.execute("UPDATE admissions_v2 SET screen_until=0") + elif fault == "strike": + db.execute("UPDATE admissions_v2 SET strikes=1") + elif fault == "pending": + db.execute("UPDATE admissions_v2 SET pending_dispute=1") + elif fault == "funding": + db.execute("UPDATE escrow_units SET units=0 WHERE bucket='admission_locked'") + elif fault == "rotation": + db.execute("UPDATE admission_history SET coldkey='wrong-owner'") + elif fault == "signature": + signed = json.loads( + db.execute( + "SELECT proof FROM admission_trial_results WHERE epoch=0" + ).fetchone()[0] + ) + signed["sig"] = "00" * 64 + db.execute( + "UPDATE admission_trial_results SET proof=? WHERE epoch=0", + (canonicalize(signed).decode(),), + ) + elif fault == "finality": + key, raw = db.execute( + "SELECT reservation,receipt FROM admission_reservations " + "WHERE reservation LIKE 'trial-final|%' LIMIT 1" + ).fetchone() + signed = envelope_v2.parse_envelope(raw) + changed = dict(signed.body) + changed["included"] = [authority_fixture.service.AUDITORS[0].ss58] + resigned = envelope_v2.seal( + authority_fixture.service.COORD, + "Finalize", + adapter.RUN, + changed, + signed.exp_drand, + ) + db.execute( + "UPDATE admission_reservations SET receipt=? WHERE reservation=?", + (canonicalize(resigned).decode(), key), + ) + elif fault == "assignment": + db.execute("UPDATE admission_trials SET nonce=? WHERE epoch=0", ("ff" * 32,)) + else: + db.execute("UPDATE runs SET run_id=?", ("ff" * 32,)) + raw = (directory / "challenge.db").read_bytes() + body["files"]["challenge.db"] = {"sha256": sha256_hex(raw), "size": len(raw)} + elif fault == "traversal": + body["files"]["../escape"] = body["files"]["challenge.db"] + else: + digest = "00" * 32 + if fault in ( + "reference", + "foreign", + "traversal", + "screen", + "strike", + "pending", + "funding", + "rotation", + "signature", + "finality", + "assignment", + ): + raw = canonicalize(body) + (directory / "bundle.json").write_bytes(raw) + digest = sha256_hex(raw) + restored = tmp_path / "restored" + with pytest.raises((ValueError, FileNotFoundError)) as rejected: + with adapter.bootstrap( + directory, digest, restored, tmp_path / "absent-secrets", "https://master.test" + ): + pytest.fail("tampered authority accepted") + assert not restored.exists() + if fault == "screen": + assert "current funded admission is ineligible" in str(rejected.value) diff --git a/tests/challenge/test_backend_authority_v2.py b/tests/challenge/test_backend_authority_v2.py new file mode 100644 index 0000000..8aa7335 --- /dev/null +++ b/tests/challenge/test_backend_authority_v2.py @@ -0,0 +1,877 @@ +"""Trusted backend dispatch boundaries; synthetic CUDA evidence is not GPU proof.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from dataclasses import replace +from pathlib import Path + +import pytest + +from hypertrain.challenge.store import ChallengeError, ChallengeStore +from hypertrain.data.store import LocalFSStore +from hypertrain.gpu_ops.journal import fsha +from hypertrain.gpu_ops.launcher import Reject +from hypertrain.protocol import envelope_v2 +from hypertrain.protocol.hashing import sha256_hex +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.messages import Receipt +from hypertrain.protocol.messages_v2 import EconomicsPolicyV2, IslandJobV1, RunManifestV2 + + +def module(name, path): + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + loaded = importlib.util.module_from_spec(spec) + sys.modules[name] = loaded + spec.loader.exec_module(loaded) + return loaded + + +TREE = Path(__file__).parents[2] +service = module("backend_service_fixture", TREE / "tests/challenge/test_service_network_v2.py") +gpu = module("backend_gpu_fixture", TREE / "tests/gpu_ops/test_network_gpu_qualification.py") +network = service.network + + +@pytest.fixture +def authority(network, tmp_path, monkeypatch): + """Real validators over synthetic rescued files, isolated from live CPU run.""" + configs_seen = [] + check = gpu.driver.check_inputs + + def capture(cfg): + configs_seen.append(cfg) + return check(cfg) + + monkeypatch.setattr(gpu.driver, "check_inputs", capture) + seed_dir = tmp_path / "seed" + seed_dir.mkdir() + gpu.test_exact_profile_accepts_wire_defaults_without_training(seed_dir) + original = configs_seen[0] + job = IslandJobV1.model_validate_json(original.seed_job.read_bytes()) + body = job.manifest.body() + body["network"] = network.manifest.network.model_dump(mode="json") + body["training"]["beacon"] = network.manifest.training.beacon.model_dump(mode="json") + body["training"]["coord_pubkey"] = service.COORD.ss58 + body["training"]["auditors"] = [key.ss58 for key in service.AUDITORS] + manifest = RunManifestV2.model_validate(body) + job = job.model_copy(update={"manifest": manifest, "run_id": manifest.run_id()}) + original.seed_job.write_text(job.model_dump_json()) + results = gpu.results(tmp_path / "rescued") + configs = [] + for index, path in enumerate(results): + for round_index in range(2): + for rank in range(2): + summary = path.parent / f"round-{round_index}/published/rank-{rank}/summary.json" + summary.write_text(json.dumps({"backend": "cuda"})) + receipts = seed_dir / f"receipt-{index}.json" + data = json.loads(original.lifecycle_receipts.read_bytes()) + data.update(instance_id=index + 1, machine_id=index + 10, role=f"h{index}") + receipts.write_text(json.dumps(data)) + cfg = original.model_copy( + update={ + "instance_id": index + 1, + "machine_id": index + 10, + "role": f"h{index}", + "lifecycle_receipts": receipts, + "lifecycle_receipts_sha256": fsha(receipts), + } + ) + config = seed_dir / f"config-{index}.json" + config.write_text(cfg.model_dump_json()) + configs.append(config) + result = json.loads(path.read_bytes()) + result.update( + config_sha256=fsha(config), profile_sha256=cfg.profile_sha256, sources=cfg.sources + ) + result["environment"].update( + image_digest=manifest.training.reference_spec.image_digest, + torch_path="/venv/main/lib/torch/__init__.py", + ) + path.write_text(json.dumps(result)) + objects = LocalFSStore(tmp_path / "objects") + for field in ( + "economics_policy_hash", + "admission_policy_hash", + "dispute_policy_hash", + "aggregation_policy_hash", + "relay_registry_hash", + ): + objects.put(network.store.objects.get(getattr(network.manifest.network, field))) + store = ChallengeStore( + tmp_path / "authority-store", + network.store.params, + service.COORD, + service.OWNER.ss58, + network.store.verify_beacon, + objects, + ) + # Existing verified fixture beacon, copied as authority input rather than a timer. + for row in network.store._db.execute("SELECT * FROM beacon"): + store._db.execute("INSERT INTO beacon VALUES(?,?,?,?)", tuple(row)) + evidence = { + "run_id": manifest.run_id(), + "backend": "cuda", + "reference_hash": sha256_hex( + canonicalize(manifest.training.reference_spec.model_dump(mode="json")) + ), + "layout_hash": manifest.training.reference_spec.layout.model_dump_json(), + "configs": [fsha(p) for p in configs], + "results": [fsha(p) for p in results], + "summaries": [ + fsha(path.parent / f"round-{round_index}/published/rank-{rank}/summary.json") + for path in results + for round_index in range(2) + for rank in range(2) + ], + } + receipt = Receipt(w=0, commit_hash=sha256_hex(canonicalize(evidence)), received_round=1) + raw = canonicalize(envelope_v2.seal(service.OWNER, "Receipt", manifest.run_id(), receipt, 100)) + yield store, manifest, tuple(configs), tuple(results), raw + store.close_v2_notifications() + store._db.close() + + +def register(store, manifest): + signed = envelope_v2.seal(service.OWNER, "RunManifestV2", manifest.run_id(), manifest, 100) + store._db.execute( + "INSERT INTO runs VALUES(?,?,?,'created',NULL)", + (manifest.run_id(), manifest.model_dump_json(), json.dumps(signed)), + ) + + +def saved_geometry(manifest, hotkey, directory, *, published=False): + """Complete commitment fixture, not a trained trajectory or CUDA measurement.""" + import torch + + from hypertrain.auditor.replay import AnchorCache, pack_state, tensor_root + from hypertrain.miner.island_launch import validate_artifacts + from hypertrain.protocol.hashing import MerkleTree + from hypertrain.trainer.compress import compress, state_hash + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.island import IslandAssignment + from hypertrain.trainer.loop import _make_leaf + from hypertrain.trainer.model import init_params + + cfg = TrainConfig.from_manifest_v2(manifest) + theta = init_params(cfg.model) + anchor = AnchorCache().genesis(manifest, hotkey, theta) + count = manifest.training.batch_samples() + samples = [ + b"".join( + ((i + j) % cfg.model.vocab).to_bytes(4, "little") for j in range(cfg.model.seq_len + 1) + ) + for i in range(count) + ] + sample_tree = MerkleTree(samples) + inputs = { + "start_state": pack_state(theta, anchor.state), + "ef_in": pack_state(anchor.ef), + "v0": pack_state({}), + "samples": b"".join(samples), + "sample_proofs": canonicalize( + [[p.hex() for p in sample_tree.proof(i)] for i in range(count)] + ), + } + directory.mkdir(parents=True, exist_ok=True) + for name, raw in inputs.items(): + (directory / name).write_bytes(raw) + job = IslandJobV1( + job_version=1, + run_id=manifest.run_id(), + w=0, + manifest=manifest, + sample_ids=list(range(count)), + global_step0=0, + start_state_sha256=sha256_hex(inputs["start_state"]), + ef_in_sha256=sha256_hex(inputs["ef_in"]), + v0_sha256=sha256_hex(inputs["v0"]), + object_paths={name: name for name in inputs}, + deadline=1, + ) + if published: + publication = directory / "published" + publication.mkdir() + for name, raw in inputs.items(): + (publication / name).write_bytes(raw) + assignment = IslandAssignment( + job.run_id, + job.w, + tuple(job.sample_ids), + 0, + manifest.training.reference_spec.layout.n_gpus, + ) + leaves, checkpoints = [], {} + for step in range(0, cfg.inner.H + 1, cfg.inner.J): + state = replace(anchor.state, step=step) + leaves.append(_make_leaf(cfg, assignment, step, theta, state, 0.0, 0.0).preimage) + checkpoints[step] = pack_state(theta, state) + delta, ef = compress( + cfg.compress, {k: torch.zeros_like(x) for k, x in theta.items()}, anchor.ef + ) + final = replace(anchor.state, step=cfg.inner.H) + commitments = { + "leaves_root": MerkleTree([bytes.fromhex(p.digest()) for p in leaves]).root.hex(), + "final_theta_hash": state_hash(theta), + "state_root": tensor_root(theta, final), + "ef_out_hash": state_hash(ef), + "delta_hash": sha256_hex(delta), + "leaves": [p.digest() for p in leaves], + } + for rank in range(manifest.training.reference_spec.layout.n_gpus): + target = publication / f"rank-{rank}" + (target / "checkpoints").mkdir(parents=True) + for step, raw in checkpoints.items(): + (target / "checkpoints" / f"{step}.safetensors").write_bytes(raw) + values = { + "state.safetensors": pack_state(theta, final), + "ef.safetensors": pack_state(ef), + "delta.bin": delta, + "leaves.json": canonicalize([p.model_dump(mode="json") for p in leaves]), + "trace.json": canonicalize( + [ + { + "rank": rank, + "step": 1, + "microbatch": 0, + "layer": 0, + "op": "fixture.identity", + "shape": [1], + "dtype": "torch.float32", + "sha256": sha256_hex(b"backend-dispatch-metadata"), + } + ] + ), + } + for name, raw in values.items(): + (target / name).write_bytes(raw) + (target / "summary.json").write_bytes( + canonicalize( + { + "job_hash": job.digest(), + "backend": "cpu", + "optimizer_step": cfg.inner.H, + "commitments": commitments, + "work": { + "rank": rank, + "fingerprint": sha256_hex(canonicalize([job.digest(), rank])), + "elapsed_ns": 0, + "allocated_bytes": 0, + "reserved_bytes": 0, + "total_bytes": 1, + }, + } + ) + ) + validate_artifacts(job, publication) + return job + + +def test_reviewed_cuda_overrides_test_economics_and_freezes(authority): + store, manifest, configs, results, raw = authority + store._bootstrap_backend_v2(manifest, configs, results, raw) + register(store, manifest) + escrow, admission, _ = store._services(manifest.run_id()) + assert admission.backend == "cuda" and escrow.policy.ledger_mode == "test" + before = escrow.balances() + store._bootstrap_backend_v2(manifest, configs, results, raw) + assert escrow.balances() == before + store._services_v2.clear() + assert store._services(manifest.run_id())[1].backend == "cuda" + + +def test_existing_cpu_contract_cannot_promote_after_services(network, authority): + assert network.store._services(network.manifest.run_id())[1].backend == "cpu" + store, manifest, configs, results, raw = authority + register(store, manifest) + assert store._services(manifest.run_id())[1].backend == "cpu" + store._services_v2.clear() + with pytest.raises(ChallengeError, match="precede"): + store._bootstrap_backend_v2(manifest, configs, results, raw) + + +@pytest.mark.parametrize("fault", ["owner", "runtime", "artifact", "config", "cpu_summary"]) +def test_invalid_reviewed_authority_rejects_without_install(authority, fault): + store, manifest, configs, results, raw = authority + if fault == "owner": + env = envelope_v2.parse_envelope(raw) + raw = canonicalize( + envelope_v2.seal(service.COORD, "Receipt", manifest.run_id(), env.body, 100) + ) + elif fault == "runtime": + data = json.loads(results[0].read_bytes()) + data["environment"]["drivers"] = ["wrong", "wrong"] + results[0].write_text(json.dumps(data)) + elif fault == "artifact": + (results[0].parent / "round-1/published/rank-0/state.safetensors").write_bytes(b"changed") + elif fault == "cpu_summary": + (results[0].parent / "round-0/published/rank-0/summary.json").write_text( + json.dumps({"backend": "cpu"}) + ) + else: + configs[0].write_bytes(b"{}") + if fault in ("runtime", "cpu_summary", "config"): + # Even owner-reviewed bytes must pass runtime/config validation. + evidence = { + "run_id": manifest.run_id(), + "backend": "cuda", + "reference_hash": sha256_hex( + canonicalize(manifest.training.reference_spec.model_dump(mode="json")) + ), + "layout_hash": manifest.training.reference_spec.layout.model_dump_json(), + "configs": [fsha(p) for p in configs], + "results": [fsha(p) for p in results], + "summaries": [ + fsha(path.parent / f"round-{w}/published/rank-{rank}/summary.json") + for path in results + for w in range(2) + for rank in range(2) + ], + } + receipt = Receipt(w=0, commit_hash=sha256_hex(canonicalize(evidence)), received_round=1) + raw = canonicalize( + envelope_v2.seal(service.OWNER, "Receipt", manifest.run_id(), receipt, 100) + ) + with pytest.raises((ChallengeError, ValueError, Reject)): + store._bootstrap_backend_v2(manifest, configs, results, raw) + assert ( + store._db.execute("SELECT COUNT(*) FROM records_v2 WHERE kind='qualification'").fetchone()[ + 0 + ] + == 0 + ) + + +def test_conflicting_reviewed_record_rejects(authority): + store, manifest, configs, results, raw = authority + store._bootstrap_backend_v2(manifest, configs, results, raw) + data = json.loads(results[0].read_bytes()) + data["elapsed_seconds"] = 2 + results[0].write_text(json.dumps(data)) + record = store._record_v2( + manifest.run_id(), "qualification", manifest.training.reference_spec.image_digest + ) + evidence = envelope_v2.load_json(store.objects.get(record["authority_hash"])) + evidence["results"] = [fsha(p) for p in results] + receipt = Receipt(w=0, commit_hash=sha256_hex(canonicalize(evidence)), received_round=1) + raw = canonicalize(envelope_v2.seal(service.OWNER, "Receipt", manifest.run_id(), receipt, 100)) + with pytest.raises(ChallengeError, match="conflicts"): + store._bootstrap_backend_v2(manifest, configs, results, raw) + assert ( + store._record_v2( + manifest.run_id(), "qualification", manifest.training.reference_spec.image_digest + ) + == record + ) + + +def test_changed_authority_and_cpu_anchor_reject(authority): + from hypertrain.auditor.replay import AnchorCache + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.model import init_params + + store, manifest, configs, results, raw = authority + store._bootstrap_backend_v2(manifest, configs, results, raw) + register(store, manifest) + assert store._backend_v2(manifest) == "cuda" + cache = AnchorCache() + anchor = cache.genesis( + manifest, service.HOT[0].ss58, init_params(TrainConfig.from_manifest_v2(manifest).model) + ) + store._persist_anchor_v2(manifest, anchor, cache) + assert ( + store._restore_anchor_v2(manifest.run_id(), anchor.hotkey, -1, AnchorCache()).backend + == "genesis" + ) + with pytest.raises(ChallengeError, match="anchor backend"): + store._persist_anchor_v2(manifest, replace(anchor, backend="cpu", w=0), cache) + store._db.execute( + "UPDATE records_v2 SET data=json_set(data,'$.backend','cpu') WHERE kind='anchor'" + ) + with pytest.raises(ChallengeError, match="anchor backend"): + store._restore_anchor_v2(manifest.run_id(), anchor.hotkey, -1, AnchorCache()) + store._db.execute( + "UPDATE records_v2 SET data=json_set(data,'$.backend','cpu') WHERE kind='qualification'" + ) + with pytest.raises(ChallengeError, match="authority differs"): + store._backend_v2(manifest) + + +def test_production_requires_qualification(authority): + store, manifest, *_ = authority + econ = EconomicsPolicyV2.model_validate_json( + store.objects.get(manifest.network.economics_policy_hash) + ) + production = econ.model_copy( + update={"ledger_mode": "production", "genesis_allocation_hash": None} + ) + body = manifest.body() + body["network"]["economics_policy_hash"] = store.objects.put(canonicalize(production.body())) + manifest = RunManifestV2.model_validate(body) + with pytest.raises(ChallengeError, match="qualification required"): + store._backend_v2(manifest) + + +def test_cuda_dispatch_rejects_missing_accepted_round(authority): + store, manifest, configs, results, raw = authority + store._bootstrap_backend_v2(manifest, configs, results, raw) + register(store, manifest) + before = store._db.total_changes + with pytest.raises(ChallengeError, match="missing accepted round: 0"): + store.require_roster_v2(manifest.run_id(), 0) + assert store._db.total_changes == before + + +@pytest.mark.parametrize("lease_state", ["live", "expired", "cancelled"]) +def test_cuda_dispatch_at_admission_audit_and_geometry(authority, monkeypatch, lease_state): + from hypertrain.auditor import worker + from hypertrain.challenge import admission as admission_module + from hypertrain.miner import island_launch + from hypertrain.miner.admission import sign_join + from hypertrain.protocol.messages import LeafPreimage + from hypertrain.protocol.messages_v2 import ( + AuditChallengeV2, + AuditJobV2, + CommitV2, + HardwareHint, + PolicyHashes, + RoundOpenV2, + StartStateV2, + ) + + store, manifest, configs, results, raw = authority + store._bootstrap_backend_v2(manifest, configs, results, raw) + register(store, manifest) + admission = store._services(manifest.run_id())[1] + assert admission.backend == "cuda" + calls = [] + joined = admission.join( + canonicalize( + sign_join( + service.HOT[0], + service.COLD[0], + run_id=manifest.run_id(), + request_id="ab" * 32, + expires_beacon=100, + policy_hash=manifest.network.admission_policy_hash, + hardware_hint=HardwareHint( + device_name="advisory", device_count=2, driver="advisory" + ), + ).body() + ), + now=1, + ip_prefix="local", + ) + store._db.execute("INSERT INTO beacon VALUES(?,?,?,?)", (2, "00" * 96, "00" * 32, 0)) + admission.challenge(joined.admission_id, now=2) + seed = IslandJobV1.model_validate_json( + gpu.driver.Qualification.model_validate_json(configs[0].read_bytes()).seed_job.read_bytes() + ) + monkeypatch.setattr( + admission, + "stage_reference", + lambda challenge, samples, epoch: ( + seed.model_copy(update={"w": epoch, "sample_ids": list(samples)}), + store.state_dir, + ), + ) + + def screen(*args, **kwargs): + calls.append(("admission", kwargs["backend"])) + raise ChallengeError(409, "admission dispatch sentinel") + + monkeypatch.setattr(admission_module, "screen_work", screen) + with pytest.raises(ChallengeError, match="admission dispatch sentinel"): + admission.prepare_reference(joined.admission_id, now=2) + hot, auditor = service.HOT[0], service.AUDITORS[0] + run = manifest.run_id() + from hypertrain.auditor.replay import AnchorCache, optimizer_hash, pack_state + from hypertrain.trainer.compress import state_hash + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.model import init_params + + theta = init_params(TrainConfig.from_manifest_v2(manifest).model) + anchor = AnchorCache().genesis(manifest, hot.ss58, theta) + proposed_start = StartStateV2( + run_id=run, + w=0, + hotkey=hot.ss58, + theta_hash=state_hash(theta), + state_object_sha256=sha256_hex(pack_state(theta, anchor.state)), + opt_state_hash=optimizer_hash(anchor.state), + ef_object_sha256=sha256_hex(pack_state(anchor.ef)), + ef_hash=state_hash(anchor.ef), + parent_anchor_hash=anchor.anchor_hash, + global_step0=0, + anchor_verdict_hash=anchor.proof_hash, + ) + status = admission.status(hot.ss58, now=2) + roster = [ + { + "hotkey": hot.ss58, + "slot": 0, + "q_i": "0000803f", + "admission_id": status.record.admission_id, + "coldkey_group": status.record.coldkey, + "state": status.record.state, + "eligible_weight": 4194304, + } + ] + opening = RoundOpenV2( + w=0, + prev_final_hash="0" * 64, + theta_hash=state_hash(theta), + outer_state_hash="0" * 64, + center_hash="0" * 64, + roster_hash=sha256_hex(canonicalize(roster)), + honeypot_commit="0" * 64, + d_open=3, + d_assign=4, + d_commit=30, + d_audit=31, + d_upload=32, + d_final=34, + contract_version=2, + registry_epoch=0, + audit_mode="anchored-full", + roster=roster, + policy_hashes=PolicyHashes.model_validate( + {name: getattr(manifest.network, name) for name in PolicyHashes.model_fields} + ), + start_state_index_hash=sha256_hex(canonicalize([proposed_start.body()])), + ) + store.open_round_v2( + run, canonicalize(envelope_v2.seal(service.COORD, "RoundOpenV2", run, opening, 100)) + ) + h = store.objects.put(b"input") + start = StartStateV2( + run_id=run, + w=0, + hotkey=hot.ss58, + theta_hash=h, + state_object_sha256=h, + opt_state_hash=h, + ef_object_sha256=h, + ef_hash=h, + parent_anchor_hash=h, + global_step0=0, + anchor_verdict_hash=h, + ) + commit = CommitV2( + w=0, + hotkey=hot.ss58, + leaf_scheme="ht-leaf-v1", + n_leaves=7, + leaves_root=h, + metrics_root=h, + final_theta_hash=h, + ef_in_hash=h, + ef_out_hash=h, + delta_hash=h, + delta_bytes=1, + tokens=manifest.training.batch_samples() * manifest.training.model.seq_len, + ) + challenge = AuditChallengeV2( + w=0, + target=hot.ss58, + beacon_round=1, + beacon_sig_sha256=h, + mode="full", + segments=[], + reasons=["final"], + serve_deadline=100, + anchor_hash=start.digest(), + audit_mode="anchored-full", + ) + job = AuditJobV2( + run_id=run, + job_id=h, + auditor_id=auditor.ss58, + attempt=1, + lease_nonce=h, + lease_expires=50, + absolute_deadline=100, + reservation_id=h, + replay_step_budget=30, + anchor_age=0, + manifest=manifest, + challenge_envelope=envelope_v2.seal(service.COORD, "AuditChallengeV2", run, challenge, 100), + commit_envelope=envelope_v2.seal(hot, "CommitV2", run, commit, 100), + sample_ids=list(range(manifest.training.batch_samples())), + start_state=start, + preimages=[ + LeafPreimage( + run_id=run, + w=0, + t=t, + stages=[dict(theta=h, m=h, v=h)], + batch_ids_sha256=h, + rng_ctr=0, + loss_f32="00000000", + norm_f32="00000000", + ) + for t in range(0, 31, 5) + ], + ef_in=dict(sha256=h, size=5), + v0=dict(sha256=h, size=5), + created_beacon=1, + ) + geometry = saved_geometry(manifest, hot.ss58, store.state_dir / "jobs-v2/0" / hot.ss58) + with store._tx(): + store._db.execute( + "INSERT INTO audit_leases_v2(id,run_id,w,hotkey,challenge,created,absolute," + "state,attempts,auditor,nonce,expires) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)", + (h, run, 0, hot.ss58, "{}", 1, 100, "running", 1, auditor.ss58, h, 50), + ) + store._put_record_v2(run, "audit-job", h, job.body()) + store._put_record_v2(run, "island-job", "0:" + hot.ss58, geometry.body()) + samples = store.objects.put(bytes(60 * 17 * 2)) + proofs = store.objects.put(canonicalize([[]] * 60)) + store._put_record_v2( + run, "dataset", "inputs", {"samples_hash": samples, "proofs_hash": proofs} + ) + monkeypatch.setattr(store, "_fresh_v2", lambda *args: 50 if lease_state == "expired" else 1) + monkeypatch.setattr(store, "_restore_anchor_v2", lambda *args: None) + + def audit(*args, **kwargs): + calls.append(("audit", kwargs["backend"])) + assert kwargs["deadline_unix"] == manifest.training.beacon.genesis_time + 49 * 3 + if lease_state == "cancelled": + args[3].revoke() + args[3].check() + return island_launch.launch_island( + kwargs["publication_job"], + kwargs["publication_directory"], + backend=kwargs["backend"], + trace=True, + cancel=args[3].cancelled, + ) + + def launch(*args, **kwargs): + calls.append(("geometry", kwargs["backend"])) + assert args[0] == geometry.model_copy( + update={"deadline": manifest.training.beacon.genesis_time + 49 * 3} + ) + assert store._record_v2(run, "island-job", "0:" + hot.ss58) == geometry.body() + raise ChallengeError(409, "dispatch sentinel, no GPU kernels") + + monkeypatch.setattr(worker, "execute_island_audit", audit) + monkeypatch.setattr(island_launch, "launch_island", launch) + receipt = envelope_v2.seal( + auditor, "Receipt", run, Receipt(w=0, commit_hash=h, received_round=1), 100 + ) + from hypertrain.auditor.replay import AuditInputError + + expected_error = ( + pytest.raises(ChallengeError, match="dispatch sentinel, no GPU kernels") + if lease_state == "live" + else pytest.raises((ChallengeError, AuditInputError, ValueError)) + ) + with expected_error: + store.execute_audit_v2(run, h, canonicalize(receipt)) + expected = [("admission", "cuda")] + if lease_state != "expired": + expected.append(("audit", "cuda")) + if lease_state == "live": + expected.append(("geometry", "cuda")) + assert calls == expected + assert not store._lease_guards_v2 + + +@pytest.mark.parametrize("mode", ["cuda", "cpu", "cpu_anchor", "missing", "mutated"]) +def test_referee_uses_frozen_backend_before_evidence(authority, monkeypatch, mode, network): + from hypertrain.auditor.replay import AnchorCache + from hypertrain.challenge.disputes_v2 import Contest, Turn + from hypertrain.miner import island_launch + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.model import init_params + + if mode == "cpu": + import os + + from hypertrain.protocol.messages_v2 import DisputeV2, EscrowLock + + snapshot = Path(os.environ["HT_NETWORK_VERIFIED_SNAPSHOT"]) + assert (snapshot / "challenge.db").is_file() + rounds = module( + "cpu_referee_verified_round", + TREE / "tests/e2e/test_service_network_v2_e2e.py", + ) + calls = [] + + def check_referee(current): + store, manifest = current.store, current.manifest + run = manifest.run_id() + assert store._backend_v2(manifest) == "cpu" + assert store._record_v2(run, "execution-backend", "run")["authority_hash"] is None + opening = store._record_v2(run, "round", "0") + assert envelope_v2.verify_envelope(opening) + assert opening["signer"] == service.COORD.ss58 + assert store.require_roster_v2(run, 0) is None + hot = service.HOT[0] + row = store._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='contest' " + "AND json_extract(data,'$.miner')=?", + (run, hot.ss58), + ).fetchone() + contest = Contest.model_validate_json(row[0]) + h = sha256_hex(canonicalize([run, contest.verdict_hash, hot.ss58])) + origin = store._db.execute( + "SELECT origin FROM escrow_units WHERE owner=? AND bucket='available' " + "AND units>=100", + (service.COLD[0].ss58,), + ).fetchone()[0] + lock = EscrowLock( + operation_id="cd" * 32, + owner=service.COLD[0].ss58, + units=100, + origin_ids=[origin], + admission_id=None, + dispute_id=h, + kind="LOCK_CONTEST", + ) + response = current.client.post( + current.url + "/escrow/lock", + json=current.signed(service.COLD[0], "EscrowLock", lock), + ) + assert response.status_code == 200, response.text + response = current.client.post( + current.url + "/dispute", + json=current.signed( + hot, + "DisputeV2", + DisputeV2( + hotkey=hot.ss58, + verdict_hash=contest.verdict_hash, + action="contest", + bond_lock=lock.operation_id, + ), + ), + ) + assert response.status_code == 200, response.text + disputes = store._services(run)[2] + turn = disputes.get(h) + geometry = store._record_v2(run, "trace-geometry", contest.verdict_hash) + before = store._db.execute("SELECT COUNT(*) FROM dispute_evidence_v2").fetchone()[0] + + def launch(job, directory, **kwargs): + calls.append(kwargs["backend"]) + assert kwargs["backend"] == "cpu" and kwargs["trace"] is True + record = store._record_v2(run, "referee-job", h + ":" + turn.transcript_hash) + assert envelope_v2.verify_envelope(record["receipt"]) + assert record["receipt"]["signer"] == service.COORD.ss58 + descriptor = envelope_v2.load_json(store.objects.get(record["descriptor_hash"])) + assert descriptor["job"] == job.body() + assert descriptor["backend"] == "cpu" + assert descriptor["historical_geometry_hash"] == sha256_hex(canonicalize(geometry)) + assert directory == store.state_dir / "referee-v2" / h / turn.transcript_hash + raise ChallengeError(409, "referee dispatch sentinel") + + monkeypatch.setattr(island_launch, "launch_island", launch) + with pytest.raises(ChallengeError, match="referee dispatch sentinel"): + store.referee_v2(run, h) + assert calls == ["cpu"] + assert disputes.get(h) == turn + assert ( + store._db.execute("SELECT COUNT(*) FROM dispute_evidence_v2").fetchone()[0] + == before + ) + assert store._record_v2(run, "trace-geometry", contest.verdict_hash) == geometry + assert not store._lease_guards_v2 + + monkeypatch.setattr(rounds, "_funded_dispute_watch", check_referee) + rounds.test_four_actual_funded_identities_graduate_shadow_only(network, monkeypatch) + assert calls == ["cpu"] + return + + store, manifest, configs, results, raw = authority + if mode != "cpu": + store._bootstrap_backend_v2(manifest, configs, results, raw) + register(store, manifest) + _, _, disputes = store._services(manifest.run_id()) + cache = AnchorCache() + hot = service.HOT[0].ss58 + anchor = cache.genesis(manifest, hot, init_params(TrainConfig.from_manifest_v2(manifest).model)) + store._persist_anchor_v2(manifest, anchor, cache) + h = "bc" * 32 + contest = Contest( + verdict_hash=h, + challenge_hash=h, + w=0, + miner=hot, + auditor=service.AUDITORS[0].ss58, + referee=service.REFEREE.ss58, + step_span=30, + layer_span=1, + op_span=1, + ) + turn = Turn( + run_id=manifest.run_id(), + dispute_id=h, + contest=contest, + seq=0, + level="step", + ctx=[], + interval=(0, 30), + expected_party=hot, + transcript_hash=h, + turn_deadline=50, + absolute_deadline=100, + lock_id=h, + ) + original = store.state_dir / "claim" + job = saved_geometry(manifest, hot, original, published=True) + with store._tx(): + store._db.execute( + "INSERT INTO disputes_v2 VALUES(?,?,?,?,?)", (h, h, hot, turn.model_dump_json(), "{}") + ) + store._put_record_v2( + manifest.run_id(), + "trace-geometry", + h, + {"job": job.body(), "directory": str(original / "published")}, + ) + store._put_record_v2( + manifest.run_id(), "verdict", "0:" + hot, {"body": {"result": "MATCH"}} + ) + if mode == "cpu_anchor": + store._db.execute( + "UPDATE records_v2 SET data=json_set(data,'$.backend','cpu') WHERE kind='anchor'" + ) + elif mode == "missing": + store._db.execute("DELETE FROM records_v2 WHERE kind='qualification'") + elif mode == "mutated": + record = store._record_v2( + manifest.run_id(), "qualification", manifest.training.reference_spec.image_digest + ) + env = envelope_v2.parse_envelope(record["receipt"]) + record["receipt"] = envelope_v2.seal( + service.COORD, "Receipt", manifest.run_id(), env.body, env.exp_drand + ) + store._put_record_v2( + manifest.run_id(), + "qualification", + manifest.training.reference_spec.image_digest, + record, + ) + monkeypatch.setattr(store, "_snapshot_current_v2", lambda _: None) + calls = [] + + def launch(*args, **kwargs): + calls.append(kwargs["backend"]) + raise ChallengeError(409, "referee dispatch sentinel") + + monkeypatch.setattr(island_launch, "launch_island", launch) + expected_error = ( + pytest.raises(ChallengeError, match="referee dispatch sentinel") + if mode in ("cuda", "cpu") + else pytest.raises(ChallengeError) + ) + with expected_error: + store.referee_v2(manifest.run_id(), h) + assert calls == ([mode] if mode in ("cuda", "cpu") else []) + assert disputes.get(h) == turn + assert store._db.execute("SELECT COUNT(*) FROM dispute_evidence_v2").fetchone()[0] == 0 diff --git a/tests/challenge/test_capacity_outer_v2.py b/tests/challenge/test_capacity_outer_v2.py new file mode 100644 index 0000000..216d535 --- /dev/null +++ b/tests/challenge/test_capacity_outer_v2.py @@ -0,0 +1,2558 @@ +"""Bounded outer metadata and exactly one actual tiny Torch child; no training.""" + +from __future__ import annotations + +import importlib.util +import json +import os +import struct +import sys +import time +from pathlib import Path + +import pytest + +from hypertrain.aggregator.capacity_worker import ( + OuterInput, + OuterRequest, + collect_result, +) +from hypertrain.data.store import LocalFSStore +from hypertrain.gpu_ops.journal import durable_write +from hypertrain.protocol.hashing import sha256_hex +from hypertrain.protocol.jcs import canonicalize + +TREE = Path(__file__).parents[2] +spec = importlib.util.spec_from_file_location( + "outer_capacity_fixture", TREE / "tests/challenge/test_service_capacity_v2.py" +) +assert spec and spec.loader +fixture = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = fixture +spec.loader.exec_module(fixture) +network = fixture.network +tiny = fixture.tiny + + +def request_fixture(tiny_run, directory): + from hypertrain.challenge.trust_v2 import FundedStatus, ReplayEvidence, SettlementStatus + from hypertrain.protocol.messages_v2 import ( + CommitV2, + DeltaManifestV2, + EconomicsPolicyV2, + RunManifestV2, + ) + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.model import param_shapes + + store, original = tiny_run + entries = fixture.roster(4) + h = "11" * 32 + econ = EconomicsPolicyV2( + ledger_mode="test", + G_max_units=100, + R_collectible_units=100, + S_min_units=1000, + beta_ppm=1000000, + gammaV_units=0, + s_lower_ppm=1000000, + q_floor=1000000, + genesis_allocation_hash=h, + reward_authority=fixture.service.COORD.ss58, + round_reward_units=100, + max_total_issuance=100000, + shadow_bootstrap_rounds=12, + ) + body = original.body() + body["network"]["economics_policy_hash"] = econ.digest() + manifest = RunManifestV2.model_validate(body) + staged = LocalFSStore(directory / "objects") + staged.put(canonicalize(econ.body())) + staged.put(store.objects.get(manifest.network.aggregation_policy_hash)) + shapes = param_shapes(TrainConfig.from_manifest_v2(manifest).model) + works = [] + objects = {econ.digest(), manifest.network.aggregation_policy_hash} + for entry in entries: + parts = [b"ht-dense-int8-v1", struct.pack(" None: + # Given + d, f, _, _, raw = service(tmp_path) + first = d.open(raw, now=1) + deadline = first.absolute_deadline + # When + turn, sent = reply(d, f.HOT, ["11" * 32, "22" * 32, "33" * 32]) + duplicate = d.bisect(sent, now=3) + assert duplicate == turn # duplicate never renews turn deadline + turn, _ = reply(d, f.AUDITOR, ["11" * 32, "aa" * 32, "bb" * 32]) + assert turn.level == "layer" and turn.ctx == [1] + turn, _ = reply(d, f.HOT, ["11" * 32, "22" * 32, "33" * 32]) + turn, _ = reply(d, f.AUDITOR, ["11" * 32, "aa" * 32, "bb" * 32]) + assert turn.level == "op" and turn.ctx == [1, 0] + turn, _ = reply(d, f.HOT, ["11" * 32, "22" * 32, "33" * 32]) + turn, _ = reply(d, f.AUDITOR, ["11" * 32, "aa" * 32, "bb" * 32]) + restarted = DisputesV2( + d.escrow, d.policy, f.COORD, contest_of=d.contest_of, owner_of=d.owner_of, span=d.span + ) + # Then + assert restarted.get(first.dispute_id) == turn + assert restarted.bisect(sent, now=4) == duplicate + assert turn.paused and turn.seq == 6 and turn.absolute_deadline == deadline + assert restarted.open(raw, now=4) == turn + assert d.escrow.balances().dispute_locked == 100 + + +def test_reassign_rejects_coordinator_in_pinned_policy(tmp_path: Path) -> None: + # Given: policy pinned before ledger construction, actual matured origins/contest lock. + d, f, ref, _, raw = service(tmp_path, coordinator_allowlisted=True) + opened = d.open(raw, now=1) + reply(d, f.HOT, ["11" * 32] * 3) + before, _ = reply(d, f.AUDITOR, ["11" * 32] * 3) + balances = d.escrow.balances() + assert d.policy.digest() == d.escrow.manifest.network.dispute_policy_hash + assert f.COORD.ss58 in d.policy.referees and balances.dispute_locked == 100 + # When / Then + with pytest.raises(DisputeError, match="REFEREE_REASSIGNMENT_BOUND"): + d.reassign(opened.dispute_id, f.COORD.ss58, now=3) + assert d.get(opened.dispute_id) == before + assert before.contest.referee == ref.ss58 + assert d.escrow.balances() == balances + + +def test_settlement_preserves_only_accepted_recovery_linkage(tmp_path: Path) -> None: + # Given: trusted contest callback supplies stable accepted-record linkage before opening. + d, f, ref, _, raw = service(tmp_path) + accepted = d.contest_of("11" * 32).model_copy( + update={ + "admission_id": "55" * 32, + "coldkey": f.COLD.ss58, + "evidence_hash": "66" * 32, + } + ) + d.contest_of = lambda _: accepted + turn = d.open(raw, now=1) + evidence = RefereeEvidence( + dispute_id=turn.dispute_id, + transcript_hash=turn.transcript_hash, + reason="MATCH", + loser=None, + predecessor_hash="11" * 32, + inputs_hash="22" * 32, + output_hash="33" * 32, + op_spec="accepted-anchor-replay", + ) + d.register_evidence(evidence) + resolution = ResolutionV2( + dispute_id=turn.dispute_id, + transcript_hash=turn.transcript_hash, + reason="MATCH", + loser=None, + evidence_hash=evidence.digest(), + ) + d.resolution(canonicalize(seal(ref, "ResolutionV2", d.run_id, resolution, 100000)), now=3) + # When / Then: incident evidence is not replaced by resolution evidence or a caller flag. + status = d.settlement(turn.dispute_id) + assert status == d.settlement(turn.lock_id) + assert ( + status.run_id, + status.admission_id, + status.coldkey, + status.dispute_id, + status.evidence_hash, + ) == ( + d.run_id, + accepted.admission_id, + f.COLD.ss58, + turn.dispute_id, + accepted.evidence_hash, + ) + assert not status.unresolved and status.release_beacon == 3 + with pytest.raises(DisputeError): + d.settlement("77" * 32) + + +@pytest.mark.parametrize("level,ctx", [("step", []), ("layer", [1]), ("op", [1, 0])]) +@pytest.mark.parametrize("auditor", [False, True]) +def test_duplicate_point_contradiction_rejects_without_transition( + tmp_path: Path, + level: str, + ctx: list[int], + auditor: bool, +) -> None: + # Given: valid funded live turn at [1,2], three signed coordinates [1,2,2]. + d, f, _, _, raw = service(tmp_path) + opened = d.open(raw, now=1) + turn = opened.model_copy(update={"level": level, "ctx": ctx, "interval": (1, 2)}) + with d.escrow.tx(): + d._save(turn) + if auditor: + turn, _ = reply(d, f.HOT, ["11" * 32, "22" * 32, "22" * 32]) + before = d.get(opened.dispute_id) + entries = d.db.execute("SELECT COUNT(*) FROM dispute_entries_v2").fetchone()[0] + events = d.db.execute("SELECT COUNT(*) FROM dispute_events_v2").fetchone()[0] + balances = d.escrow.balances() + # When / Then: wire-valid but semantically contradictory claim cannot advance. + key = f.AUDITOR if auditor else f.HOT + with pytest.raises(DisputeError, match="CONTRADICTORY_REPEATED_POINT"): + reply(d, key, ["11" * 32, "22" * 32, "aa" * 32]) + assert d.get(opened.dispute_id) == before + assert d.db.execute("SELECT COUNT(*) FROM dispute_entries_v2").fetchone()[0] == entries + assert d.db.execute("SELECT COUNT(*) FROM dispute_events_v2").fetchone()[0] == events + assert d.escrow.balances() == balances + after, _ = reply(d, key, ["11" * 32, "bb" * 32, "bb" * 32]) + assert after.interval[0] < after.interval[1] + + +def test_referee_reassignment_when_infrastructure_once_only(tmp_path: Path) -> None: + # Given + d, f, _, next_ref, raw = service(tmp_path) + turn = d.open(raw, now=1) + reply(d, f.HOT, ["11" * 32] * 3) + paused, _ = reply(d, f.AUDITOR, ["11" * 32] * 3) + # When + reassigned = d.reassign(turn.dispute_id, next_ref.ss58, now=3) + # Then + assert reassigned.absolute_deadline == paused.absolute_deadline + assert reassigned.contest.referee == next_ref.ss58 and reassigned.reassignments == 1 + with pytest.raises(DisputeError): + d.reassign(turn.dispute_id, d.policy.referees[0], now=4) + assert d.pause(turn.dispute_id).absolute_deadline == paused.absolute_deadline + + +@pytest.mark.parametrize("miner_loses", [True, False]) +def test_losing_lock_when_fraud_evidence_accepted(tmp_path: Path, miner_loses: bool) -> None: + # Given: internal replay evidence, not a public claimant assertion. + d, f, ref, _, raw = service(tmp_path) + accepted = d.contest_of("11" * 32).model_copy(update={"auditor_coldkey": f.OTHER.ss58}) + d.contest_of = lambda _: accepted + origin = d.db.execute( + "SELECT origin FROM escrow_units WHERE owner=? AND units>0", (f.OTHER.ss58,) + ).fetchone()[0] + other_lock = EscrowLock( + operation_id="88" * 32, + owner=f.OTHER.ss58, + units=200, + origin_ids=[origin], + admission_id="99" * 32, + dispute_id=None, + kind="LOCK_ADMISSION", + ) + d.escrow.lock(other_lock, signer=f.OTHER.ss58) + turn = d.open(raw, now=1) + loser = f.HOT.ss58 if miner_loses else f.AUDITOR.ss58 + evidence = RefereeEvidence( + dispute_id=turn.dispute_id, + transcript_hash=turn.transcript_hash, + reason="FRAUD", + loser=loser, + predecessor_hash="11" * 32, + inputs_hash="22" * 32, + output_hash="33" * 32, + op_spec="verified.actual.op", + ) + d.register_evidence(evidence) + resolution = ResolutionV2( + dispute_id=turn.dispute_id, + transcript_hash=turn.transcript_hash, + reason="FRAUD", + loser=loser, + evidence_hash=evidence.digest(), + ) + d.resolution(canonicalize(seal(ref, "ResolutionV2", d.run_id, resolution, 100000)), now=3) + # When + d.settle_lock(turn.dispute_id, now=3, referee_cost=10) + d.settle_lock(turn.dispute_id, now=3, referee_cost=10) + # Then + status = d.settlement(turn.dispute_id) + assert status.loser == loser and status.referee == ref.ss58 + assert status.loser_coldkey == (f.COLD.ss58 if miner_loses else f.OTHER.ss58) + assert (status.lock_id, status.lock_owner, status.dispute_id) == ( + turn.lock_id, + f.COLD.ss58, + turn.dispute_id, + ) + assert status.resolution_hash == resolution.digest() + assert status.outcome == ("FRAUD" if miner_loses else "MATCH") + assert d.escrow.balances().burned == (90 if miner_loses else 0) + assert d.escrow.balances().dispute_locked == 0 + assert d.escrow.balances(f.COLD.ss58).available == (19900 if miner_loses else 20000) + assert d.escrow.balances(f.OTHER.ss58).available == (9810 if miner_loses else 9800) + assert d.escrow.balances(f.OTHER.ss58).admission_locked == 200 + assert d.escrow.balances().conserved() + + +def test_settlement_rejects_unresolved_or_pre_horizon_lock(tmp_path: Path) -> None: + # Given: actual funded contest with no accepted resolution. + d, _, _, _, raw = service(tmp_path) + turn = d.open(raw, now=1) + before = d.escrow.balances() + with pytest.raises(DisputeError, match="UNRESOLVED_CONTEST"): + d.settle_lock(turn.dispute_id, now=2) + d.exclude(turn.dispute_id, now=turn.absolute_deadline) + from hypertrain.ledger.escrow_v2 import EscrowError + + # When / Then: matching closed authority still cannot bypass its release horizon. + with pytest.raises(EscrowError, match="DISPUTED_OR_UNVESTED"): + d.settle_lock(turn.dispute_id, now=turn.absolute_deadline - 1) + assert d.escrow.balances() == before + + +def test_state_serve_when_request_persisted_and_replayed(tmp_path: Path) -> None: + from hypertrain.protocol.messages import StateServe + + # Given + d, f, _, _, raw = service(tmp_path) + turn = d.open(raw, now=1) + serve = StateServe( + hotkey=f.HOT.ss58, + challenge_hash=turn.contest.challenge_hash, + t=1, + uri="https://objects.invalid/checkpoint", + tensor_root="33" * 32, + merkle_proof_leaf_in_leaves_root=[], + ) + signed = canonicalize(seal(f.HOT, "StateServe", d.run_id, serve, 100000)) + with pytest.raises(DisputeError): + d.state_serve(signed, now=2) + d.state_request(turn.dispute_id, 1, now=2) + # When + assert d.state_serve(signed, now=2) == serve + assert d.state_serve(signed, now=3) == serve + # Then + assert d.db.execute("SELECT COUNT(*) FROM dispute_states_v2").fetchone()[0] == 1 + + +@pytest.mark.parametrize( + "field,value", + [ + ("seq", 1), + ("previous_transcript_hash", "aa" * 32), + ("party", "auditor"), + ("ctx", [1]), + ("interval", (0, 1)), + ], +) +def test_rejects_when_turn_binding_changes(tmp_path: Path, field: str, value) -> None: + # Given + d, f, _, _, raw = service(tmp_path) + turn = d.open(raw, now=1) + model = dict( + dispute_id=turn.dispute_id, + seq=0, + level="step", + ctx=[], + interval=(0, 2), + N=2, + hashes=["11" * 32] * 3, + party=f.HOT.ss58, + previous_transcript_hash=turn.transcript_hash, + ) + model[field] = f.AUDITOR.ss58 if value == "auditor" else value + from pydantic import ValidationError + + # When / Then + from hypertrain.protocol.envelope import UnauthorizedSigner + + with pytest.raises((DisputeError, ValidationError, UnauthorizedSigner)): + m = BisectV2.model_validate(model) + d.bisect(canonicalize(seal(f.HOT, "BisectV2", d.run_id, m, 100000)), now=2) + assert d.get(turn.dispute_id) == turn + + +def test_unfunded_flood_when_hundred_contests(tmp_path: Path) -> None: + # Given + d, _, _, _, raw = service(tmp_path, funded=False) + # When / Then + for _ in range(100): + with pytest.raises(DisputeError, match="UNFUNDED"): + d.open(raw, now=1) + assert d.db.execute("SELECT COUNT(*) FROM disputes_v2").fetchone()[0] == 0 + assert d.db.execute("SELECT COUNT(*) FROM dispute_events_v2").fetchone()[0] == 0 + + +def test_queue_capacity_when_thirty_two_funded_contests(tmp_path: Path) -> None: + # Given + d, f, ref, _, _ = service(tmp_path) + origin = d.db.execute( + "SELECT origin FROM escrow_units WHERE owner=? AND units>0", (f.COLD.ss58,) + ).fetchone()[0] + for i in range(33): + key = Keypair(bytes([150 + i]) * 32) + contest = Contest( + verdict_hash=f"{i + 1:064x}", + challenge_hash="22" * 32, + w=0, + miner=key.ss58, + auditor=f.AUDITOR.ss58, + referee=ref.ss58, + step_span=2, + layer_span=2, + op_span=2, + ) + d.contest_of = lambda _, contest=contest: contest + d.owner_of = lambda _: f.COLD.ss58 + identifier = sha256_hex(canonicalize([d.run_id, contest.verdict_hash, key.ss58])) + lock = EscrowLock( + operation_id=f"{i + 200:064x}", + owner=f.COLD.ss58, + units=100, + origin_ids=[origin], + admission_id=None, + dispute_id=identifier, + kind="LOCK_CONTEST", + ) + d.escrow.lock(lock, signer=f.COLD.ss58) + raw = canonicalize( + seal( + key, + "DisputeV2", + d.run_id, + DisputeV2( + hotkey=key.ss58, + verdict_hash=contest.verdict_hash, + action="contest", + bond_lock=lock.operation_id, + ), + 100000, + ) + ) + # When / Then + if i < 32: + d.open(raw, now=1) + else: + with pytest.raises(DisputeError, match="QUEUE_CAPACITY"): + d.open(raw, now=1) + assert d.db.execute("SELECT COUNT(*) FROM disputes_v2").fetchone()[0] == 32 + + +@pytest.mark.parametrize("miner_silent", [False, True]) +def test_timeout_when_authenticated_delivery_healthy(tmp_path: Path, miner_silent: bool) -> None: + # Given + d, f, ref, _, raw = service(tmp_path) + turn = d.open(raw, now=1) + if not miner_silent: + turn, _ = reply(d, f.HOT, ["11" * 32] * 3) + key = f.HOT if miner_silent else f.AUDITOR + now = turn.turn_deadline + 1 + proofs = availability(d, turn, (f.COORD, ref), now) + with pytest.raises(DisputeError, match="NO_AUTHENTICATED"): + d.timeout(turn.dispute_id, proofs, now=now) + ack_turn(d, key, 2) + # When + settled = d.timeout(turn.dispute_id, proofs, now=now) + d.settle_lock(turn.dispute_id, now=now) + # Then + assert settled.resolution.loser == key.ss58 + assert settled.resolution.reason == ("PARTY_TIMEOUT" if miner_silent else "AUDITOR_FAULT") + assert d.escrow.balances().burned == 0 and d.escrow.balances().dispute_locked == 0 + + +def test_timeout_rejects_when_unhealthy_or_duplicate_observer(tmp_path: Path) -> None: + # Given + d, f, ref, _, raw = service(tmp_path) + turn = d.open(raw, now=1) + ack_turn(d, f.HOT, 2) + now = turn.turn_deadline + 1 + good = availability(d, turn, (f.COORD, ref), now) + # When / Then + with pytest.raises(DisputeError, match="UNCONFIRMED"): + d.timeout(turn.dispute_id, (good[0], good[0]), now=now) + with pytest.raises(DisputeError, match="UNCONFIRMED"): + d.timeout( + turn.dispute_id, + (good[0], good[1].model_copy(update={"service_healthy": False})), + now=now, + ) + assert d.get(turn.dispute_id).resolution is None + + +def test_resolution_when_only_independent_verified_referee(tmp_path: Path) -> None: + # Given + d, f, ref, _, raw = service(tmp_path) + turn = d.open(raw, now=1) + evidence = RefereeEvidence( + dispute_id=turn.dispute_id, + transcript_hash=turn.transcript_hash, + reason="MATCH", + loser=None, + predecessor_hash="11" * 32, + inputs_hash="22" * 32, + output_hash="33" * 32, + op_spec="full-match", + ) + resolution = ResolutionV2( + dispute_id=turn.dispute_id, + transcript_hash=turn.transcript_hash, + reason="MATCH", + loser=None, + evidence_hash=evidence.digest(), + ) + from hypertrain.protocol.envelope import UnauthorizedSigner + + with pytest.raises(UnauthorizedSigner): + d.resolution( + canonicalize(seal(f.AUDITOR, "ResolutionV2", d.run_id, resolution, 100000)), now=2 + ) + with pytest.raises(DisputeError, match="UNVERIFIED"): + d.resolution(canonicalize(seal(ref, "ResolutionV2", d.run_id, resolution, 100000)), now=2) + d.register_evidence(evidence) + # When + settled = d.resolution( + canonicalize(seal(ref, "ResolutionV2", d.run_id, resolution, 100000)), now=2 + ) + d.settle_lock(turn.dispute_id, now=2) + # Then + assert settled.resolution.reason == "MATCH" and d.escrow.balances().dispute_locked == 0 + + +def test_absolute_bound_when_infrastructure_exhausted(tmp_path: Path) -> None: + # Given + d, _, _, _, raw = service(tmp_path) + turn = d.open(raw, now=1) + # When + settled = d.exclude(turn.dispute_id, now=turn.absolute_deadline) + d.settle_lock(turn.dispute_id, now=turn.absolute_deadline) + # Then + assert settled.resolution.reason == "INFRASTRUCTURE" and settled.resolution.loser is None + assert d.escrow.balances().burned == 0 + + +def test_events_when_subscribed_before_transition(tmp_path: Path) -> None: + # Given: condition acquired before trigger; wait_for rechecks durable table under the same lock. + d, _, _, _, raw = service(tmp_path) + ready = threading.Event() + result = [] + + def consumer(): + with d.condition: + ready.set() + result.extend(d.events(fixtures().HOT.ss58, 0, timeout=2)) + + worker = threading.Thread(target=consumer) + worker.start() + assert ready.wait(2) + # When + d.open(raw, now=1) + worker.join(3) + # Then + assert not worker.is_alive() and len(result) == 1 + with pytest.raises(DisputeError): + d.events(result[0].party, -1, timeout=0) + with pytest.raises(DisputeError): + d.events(result[0].party, 0, timeout=31) diff --git a/tests/challenge/test_role_dispatch_consumers_v2.py b/tests/challenge/test_role_dispatch_consumers_v2.py new file mode 100644 index 0000000..b6c09b1 --- /dev/null +++ b/tests/challenge/test_role_dispatch_consumers_v2.py @@ -0,0 +1,415 @@ +"""Private service launch forwarding; metadata dispatch evidence, no CUDA/training.""" + +from __future__ import annotations + +import importlib.util +import json +import shutil +import sys +import tarfile +import threading +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from hypertrain.challenge.store import ChallengeError +from hypertrain.protocol.envelope import body_digest +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.messages_v2 import EscrowLock + +spec = importlib.util.spec_from_file_location( + "role_backend_fixture", Path(__file__).parent / "test_backend_authority_v2.py" +) +assert spec is not None and spec.loader is not None +backend = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = backend +spec.loader.exec_module(backend) +authority = backend.authority +network = backend.network + + +def rescued_publication(job, directory, custody_name): + """Actual validated metadata custody publication, not a trained trajectory.""" + from hypertrain.auditor.replay import pack_state, tensor_root, unpack_state + from hypertrain.miner.island_launch import validate_artifacts + from hypertrain.protocol.hashing import MerkleTree, sha256_hex + from hypertrain.trainer.compress import state_hash + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.island import IslandAssignment + from hypertrain.trainer.loop import _make_leaf + from hypertrain.trainer.optim import init_state + + custody = directory / custody_name + backend.saved_geometry(job.manifest, backend.service.HOT[0].ss58, custody, published=True) + published = custody / "published" + for relative in job.object_paths.values(): + shutil.copyfile(directory / relative, published / relative) + cfg = TrainConfig.from_manifest_v2(job.manifest) + theta, carry = unpack_state((directory / job.object_paths["start_state"]).read_bytes()) + state = init_state(cfg.inner, theta, carry if cfg.inner.state_policy == "carry" else None) + assignment = IslandAssignment( + job.run_id, + job.w, + tuple(job.sample_ids), + job.global_step0, + job.manifest.training.reference_spec.layout.n_gpus, + ) + leaves = [ + _make_leaf( + cfg, assignment, t, theta, replace(state, step=job.global_step0 + t), 0.0, 0.0 + ).preimage + for t in range(0, cfg.inner.H + 1, cfg.inner.J) + ] + for rank in range(job.manifest.training.reference_spec.layout.n_gpus): + target = published / f"rank-{rank}" + for p in leaves: + (target / "checkpoints" / f"{p.t}.safetensors").write_bytes( + pack_state(theta, replace(state, step=job.global_step0 + p.t)) + ) + final = replace(state, step=job.global_step0 + cfg.inner.H) + (target / "state.safetensors").write_bytes(pack_state(theta, final)) + (target / "leaves.json").write_bytes( + canonicalize([p.model_dump(mode="json") for p in leaves]) + ) + summary = json.loads((target / "summary.json").read_bytes()) + ef, _ = unpack_state((target / "ef.safetensors").read_bytes()) + summary["backend"] = "cuda" + summary["job_hash"] = job.digest() + summary["commitments"] = { + "leaves_root": MerkleTree([bytes.fromhex(p.digest()) for p in leaves]).root.hex(), + "final_theta_hash": state_hash(theta), + "state_root": tensor_root(theta, final), + "ef_out_hash": state_hash(ef), + "delta_hash": sha256_hex((target / "delta.bin").read_bytes()), + "leaves": [p.digest() for p in leaves], + } + (target / "summary.json").write_bytes(canonicalize(summary)) + return validate_artifacts(job, published) + + +def real_adapter(monkeypatch, operation, directory): + from hypertrain.gpu_ops.journal import Journal + + spec = importlib.util.spec_from_file_location( + "consumer_real_runtime", + Path(__file__).parents[2] / "experiments/gpu_network_v2/orchestrate.py", + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + runtime = object.__new__(module.NetworkRuntime) + runtime.lifecycle = SimpleNamespace(cleanup_started=lambda: False) + journal_directory = directory / "runtime-journal" + journal_directory.mkdir() + runtime.journal = Journal(journal_directory) + # Only remote/provider execution is replaced; real adapter owns installation. + monkeypatch.setattr(runtime, "operation", operation) + return runtime + + +def accepted_spec(kind, job, context): + from hypertrain.protocol.hashing import sha256_hex + + binding = body_digest(context) + hotkey = context["hotkey"] + spec = SimpleNamespace( + job=job, + trace=kind != "reference", + operation=kind, + hotkey=hotkey, + binding=binding, + role="h0", + cutoff=job.deadline, + ) + name = "op-" + sha256_hex((job.run_id + kind + hotkey + binding + job.digest()).encode()) + return spec, name + + +def record_rescue(runtime, spec, directory, name, artifacts): + """Model original rescued custody bytes and journal, not canonical local output.""" + from hypertrain.gpu_ops.journal import fsha + + archive = directory / (name + ".tar") + with tarfile.open(archive, "w") as saved: + saved.add(artifacts.directory, arcname="out/" + name + "/published", recursive=True) + digest = fsha(archive) + runtime.journal.append("network_rescued", role=spec.role, tar_sha256=digest, tar=str(archive)) + runtime.journal.append("network_operation_done", role=spec.role, name=name, tar_sha256=digest) + return artifacts + + +def test_real_adapter_reference_canonical_publication(network, monkeypatch): + assert network.join().status_code == 200 + _, admission, _ = network.store._services(network.manifest.run_id()) + row = network.store._db.execute("SELECT * FROM admissions_v2").fetchone() + lock = EscrowLock( + operation_id="fc" * 32, + owner=backend.service.COLD[0].ss58, + units=1000, + origin_ids=[network.origin_ids[0]], + admission_id=row["admission_id"], + dispute_id=None, + kind="LOCK_ADMISSION", + ) + admission.lock( + canonicalize(network.signed(backend.service.COLD[0], "EscrowLock", lock)), now=network.now + ) + network.push(network.now + 1) + admission.challenge(row["admission_id"], now=network.now) + # Test-only remote CUDA-labelled metadata seam, not GPU qualification. + admission.backend = "cuda" + calls = [] + + def operation(spec, directory, *, cancel): + assert cancel is not None and not cancel.is_set() + calls.append(spec.job) + name = "op-" + backend.sha256_hex( + ( + spec.job.run_id + spec.operation + spec.hotkey + spec.binding + spec.job.digest() + ).encode() + ) + artifacts = rescued_publication(spec.job, directory, name) + return record_rescue(runtime, spec, directory, name, artifacts) + + runtime = real_adapter(monkeypatch, operation, network.store.state_dir) + + def factory(kind, job, directory, context): + assert kind == "reference" and context["challenge_hash"] == body_digest( + context["challenge"] + ) + spec, _ = accepted_spec(kind, job, context) + return runtime.launch_adapter(spec) + + network.store._role_launch = factory + result = network.store.trial_reference_v2(network.manifest.run_id(), row["admission_id"]) + assert result["reference_commitment"] and len(calls) == 1 + assert network.store._db.execute("SELECT reference_json FROM admissions_v2").fetchone()[0] + + +def test_real_adapter_reference_rejects_wrong_custody_path(network, monkeypatch): + from hypertrain.gpu_ops.launcher import Reject + + original = rescued_publication + + def wrong_path(job, directory, name): + return original(job, directory, name + "-wrong") + + monkeypatch.setattr(sys.modules[__name__], "rescued_publication", wrong_path) + with pytest.raises(Reject, match="^operation_canonical_custody_path$"): + test_real_adapter_reference_canonical_publication(network, monkeypatch) + assert ( + network.store._db.execute("SELECT reference_json FROM admissions_v2").fetchone()[0] is None + ) + + +def test_real_adapter_audit_canonical_publication(authority, monkeypatch): + from hypertrain.auditor import worker + from hypertrain.auditor.replay import Outcome + from hypertrain.protocol.messages import Receipt + + store, manifest, *_ = authority + backend.test_cuda_dispatch_at_admission_audit_and_geometry(authority, monkeypatch, "live") + shutil.rmtree(store.state_dir / "audit-v2") + shutil.rmtree(store.state_dir / "audit-geometry-v2") + calls = [] + + def operation(spec, directory, *, cancel): + assert cancel is not None and not cancel.is_set() + calls.append(spec.job) + name = "op-" + backend.sha256_hex( + ( + spec.job.run_id + spec.operation + spec.hotkey + spec.binding + spec.job.digest() + ).encode() + ) + artifacts = rescued_publication(spec.job, directory, name) + return record_rescue(runtime, spec, directory, name, artifacts) + + runtime = real_adapter(monkeypatch, operation, store.state_dir) + + def factory(kind, job, directory, context): + assert kind == "audit" and context["descriptor"]["job"] == job.body() + spec, _ = accepted_spec(kind, job, {**context, "hotkey": context["hotkey"]}) + return runtime.launch_adapter(spec) + + store._role_launch = factory + + def audit(*args, **kwargs): + artifacts = kwargs["launch"]( + kwargs["publication_job"], + kwargs["publication_directory"], + backend=kwargs["backend"], + cancel=args[3].cancelled, + trace=True, + ) + return Outcome("MISMATCH", 0, "ab" * 32), artifacts + + monkeypatch.setattr(worker, "execute_island_audit", audit) + lease = store._db.execute("SELECT * FROM audit_leases_v2").fetchone() + signer = next(key for key in backend.service.AUDITORS if key.ss58 == lease["auditor"]) + raw = backend.envelope_v2.seal( + signer, + "Receipt", + manifest.run_id(), + Receipt(w=0, commit_hash=lease["nonce"], received_round=1), + 100, + ) + result = store.execute_audit_v2(manifest.run_id(), lease["id"], canonicalize(raw)) + assert result["verdict"]["result"] == "MISMATCH" and len(calls) == 1 + descriptor = store._record_v2( + manifest.run_id(), "audit-geometry", lease["id"] + ":" + lease["nonce"] + ) + body = backend.envelope_v2.load_json(store.objects.get(descriptor["descriptor_hash"])) + assert (Path(body["directory"]) / "published/rank-0/trace.json").is_file() + assert not store._lease_guards_v2 + + +@pytest.mark.parametrize("consumer", ["audit", "referee"]) +def test_accepted_role_factory_called_before_kernel(authority, monkeypatch, consumer): + from hypertrain.auditor import worker + from hypertrain.protocol.messages import Receipt + + store, manifest, _, _, _ = authority + if consumer == "audit": + backend.test_cuda_dispatch_at_admission_audit_and_geometry(authority, monkeypatch, "live") + # The seeding probe stops after creating owned inputs; a new execution + # requires an empty attempt directory, not overwrite of durable files. + shutil.rmtree(store.state_dir / "audit-v2") + shutil.rmtree(store.state_dir / "audit-geometry-v2") + else: + backend.test_referee_uses_frozen_backend_before_evidence(authority, monkeypatch, "cuda") + calls = [] + + def factory(operation, job, directory, context): + assert operation == consumer and job.manifest == manifest + assert context["run_id"] == job.run_id == manifest.run_id() + assert context["descriptor_hash"] == body_digest(context["descriptor"]) + assert context["descriptor"]["job"] == job.body() + signed = backend.envelope_v2.parse_envelope(context["descriptor_receipt"]) + assert signed.signer == backend.service.COORD.ss58 + assert signed.body["commit_hash"] == context["descriptor_hash"] + assert backend.envelope_v2.verify_envelope(context["descriptor_receipt"]) + assert str(directory).startswith(str(store.state_dir)) + if operation == "audit": + assert context["audit_job"]["lease_nonce"] == context["descriptor"]["lease_nonce"] + assert context["audit_job"]["job_id"] == context["job_id"] + else: + assert context["turn"]["transcript_hash"] == context["descriptor"]["transcript_hash"] + assert context["turn"]["contest"]["referee"] == context["descriptor"]["referee"] + calls.append(operation) + + def launch(local, target, *, backend, cancel, trace=False): + assert local == job and target == directory and backend == "cuda" + assert isinstance(cancel, threading.Event) and not cancel.is_set() + assert trace is True + raise ChallengeError(409, "trusted launch dispatch sentinel") + + return launch + + store._role_launch = factory + if consumer == "audit": + + def preserve_launch(*args, **kwargs): + assert kwargs["launch"] is not None + return kwargs["launch"]( + kwargs["publication_job"], + kwargs["publication_directory"], + backend=kwargs["backend"], + cancel=args[3].cancelled, + trace=True, + ) + + monkeypatch.setattr(worker, "execute_island_audit", preserve_launch) + lease = store._db.execute("SELECT * FROM audit_leases_v2").fetchone() + signer = next(key for key in backend.service.AUDITORS if key.ss58 == lease["auditor"]) + receipt = backend.envelope_v2.seal( + signer, + "Receipt", + manifest.run_id(), + Receipt(w=0, commit_hash=lease["nonce"], received_round=1), + 100, + ) + with pytest.raises(ChallengeError, match="trusted launch dispatch sentinel"): + store.execute_audit_v2(manifest.run_id(), lease["id"], canonicalize(receipt)) + else: + with pytest.raises(ChallengeError, match="trusted launch dispatch sentinel"): + store.referee_v2(manifest.run_id(), "bc" * 32) + assert calls == [consumer] + assert not store._lease_guards_v2 + + +@pytest.mark.parametrize("failure", ["host", "source", "deadline", "cancel", "publication"]) +def test_reference_trusted_forwarding_and_refusal(network, failure): + import json + + from hypertrain.gpu_ops.work_screen import WorkScreenError + from hypertrain.miner.island_launch import IslandFailure, validate_artifacts + + assert network.join().status_code == 200 + _, admission, _ = network.store._services(network.manifest.run_id()) + row = network.store._db.execute("SELECT * FROM admissions_v2").fetchone() + lock = EscrowLock( + operation_id="fa" * 32, + owner=backend.service.COLD[0].ss58, + units=1000, + origin_ids=[network.origin_ids[0]], + admission_id=row["admission_id"], + dispute_id=None, + kind="LOCK_ADMISSION", + ) + admission.lock( + canonicalize(network.signed(backend.service.COLD[0], "EscrowLock", lock)), now=network.now + ) + network.push(network.now + 1) + admission.challenge(row["admission_id"], now=network.now) + called = [] + + def factory(operation, job, directory, context): + assert operation == "reference" + assert context["challenge_hash"] == body_digest(context["challenge"]) + accepted = backend.envelope_v2.parse_envelope(context["challenge_envelope"]) + assert accepted.body == context["challenge"] + assert backend.envelope_v2.verify_envelope(context["challenge_envelope"]) + assert context["admission_id"] == row["admission_id"] + assert job.sample_ids and (directory / "start_state").is_file() + assert ( + job.deadline + == network.manifest.training.beacon.genesis_time + + (context["challenge"]["deadline_beacon"] - 1) * 3 + ) + if failure in ("host", "source", "deadline"): + raise ChallengeError(409, "reviewed " + failure + " binding rejected") + if failure == "cancel": + key = f"reference:{row['admission_id']}:{context['challenge']['nonce']}" + network.store._lease_guards_v2[key].revoke() + + fixture_module = backend + + def launch(local, target, *, backend, cancel, trace=False): + assert local == job and target == directory and backend == "cpu" + assert isinstance(cancel, threading.Event) and not cancel.is_set() + called.append(operation) + # Return originally valid artifact objects after corrupting original bytes. + saved = fixture_module.saved_geometry( + job.manifest, context["hotkey"], directory / "forged", published=True + ) + artifacts = validate_artifacts(saved, directory / "forged/published") + path = artifacts.directory / "rank-0/summary.json" + body = json.loads(path.read_bytes()) + body["job_hash"] = "00" * 32 + path.write_bytes(canonicalize(body)) + return artifacts + + return launch + + network.store._role_launch = factory + with pytest.raises((ChallengeError, IslandFailure, WorkScreenError, FileNotFoundError)): + network.store.trial_reference_v2(network.manifest.run_id(), row["admission_id"]) + assert called == (["reference"] if failure == "publication" else []) + assert not network.store._lease_guards_v2 + assert ( + network.store._db.execute("SELECT reference_commitment FROM admissions_v2").fetchone()[0] + is None + ) diff --git a/tests/challenge/test_service_capacity_v2.py b/tests/challenge/test_service_capacity_v2.py new file mode 100644 index 0000000..593e8e4 --- /dev/null +++ b/tests/challenge/test_service_capacity_v2.py @@ -0,0 +1,790 @@ +"""First increment: real signed resource intake, metadata only, no miner sweep.""" + +from __future__ import annotations + +import ast +import importlib.util +import resource +import struct +import sys +from pathlib import Path + +import pytest + +from hypertrain.challenge.store import ChallengeError, ChallengeStore +from hypertrain.data.store import LocalFSStore +from hypertrain.protocol import envelope_v2 +from hypertrain.protocol.hashing import sha256_hex +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.keys import Keypair +from hypertrain.protocol.messages import Receipt, f32hex +from hypertrain.protocol.messages_v2 import PolicyHashes, RoundOpenV2, RunManifestV2 + +TREE = Path(__file__).parents[2] +spec = importlib.util.spec_from_file_location( + "capacity_service_fixture", TREE / "tests/challenge/test_service_network_v2.py" +) +assert spec is not None and spec.loader is not None +service = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = service +spec.loader.exec_module(service) +network = service.network + + +@pytest.fixture +def tiny(network, tmp_path): + """Create another real owner-signed run with exact resource-profile dimensions.""" + from hypertrain.protocol.messages_v2 import AggregationPolicyV2, AuditPolicyV2 + + store = network.store + body = network.manifest.body() + m = body["training"]["model"] + m.update( + n_layers=1, + d_model=8, + n_heads=2, + n_kv_heads=2, + d_ff=8, + vocab=16, + seq_len=2, + param_count=728, + n_experts=1, + top_k_experts=1, + compute_dtype="fp32", + ) + inner = body["training"]["inner"] + inner.update(H=2, J=1, micro_batch=1, grad_accum=1, opt="adamw", state_policy="carry") + inner["lr_schedule"].update(warmup=0) + body["training"]["dataset"]["n_samples"] = 256 + body["training"]["outer"]["opt"] = "nesterov" + body["training"]["reference_spec"]["layout"].update( + pp=1, n_gpus=1, dp_size=1, ep_size=1, zero1=False + ) + old = AggregationPolicyV2.model_validate_json( + store.objects.get(network.manifest.network.aggregation_policy_hash) + ) + policy = old.model_copy(update={"cclip_iters": 1}) + store.objects.put(canonicalize(policy.body())) + body["network"]["aggregation_policy_hash"] = policy.digest() + audit = AuditPolicyV2.model_validate_json( + store.objects.get(network.manifest.network.audit_policy_hash) + ).model_copy(update={"max_steps_per_attempt": 4}) + store.objects.put(canonicalize(audit.body())) + body["network"]["audit_policy_hash"] = audit.digest() + manifest = RunManifestV2.model_validate(body) + objects = LocalFSStore(tmp_path / "capacity-objects") + for name in (*PolicyHashes.model_fields, "relay_registry_hash"): + objects.put(network.store.objects.get(getattr(manifest.network, name))) + store = ChallengeStore( + tmp_path / "capacity-state", + network.store.params, + service.COORD, + service.OWNER.ss58, + verify_beacon=network.store.verify_beacon, + objects=objects, + ) + store.push_beacon(service.fixture_beacon(1)) + raw = canonicalize( + envelope_v2.seal(service.OWNER, "RunManifestV2", manifest.run_id(), manifest, 100) + ) + store.create_run_v2(raw) + yield store, manifest + store.close_v2_notifications() + store._db.close() + + +def signed_profile(store, manifest, signer=None): + profile = store._service_profile_v2(manifest.run_id()) + body = canonicalize(profile.model_dump(mode="json")) + receipt = Receipt(w=0, commit_hash=sha256_hex(body), received_round=1) + raw = canonicalize( + envelope_v2.seal(signer or service.OWNER, "Receipt", manifest.run_id(), receipt, 100) + ) + return body, raw + + +def roster(n): + return [ + { + "hotkey": Keypair(bytes([i + 1]) * 32).ss58, + "slot": i, + "q_i": f32hex(1), + "admission_id": sha256_hex(str(i).encode()), + "coldkey_group": f"owner{i}", + "state": "ACTIVE", + "eligible_weight": 1 << 22, + } + for i in range(n) + ] + + +def opening(manifest, n=32): + entries = roster(n) + return RoundOpenV2( + w=0, + prev_final_hash="0" * 64, + theta_hash="0" * 64, + outer_state_hash="0" * 64, + center_hash="0" * 64, + roster_hash=sha256_hex(canonicalize(entries)), + honeypot_commit="0" * 64, + d_open=2, + d_assign=3, + d_commit=20, + d_audit=21, + d_upload=22, + d_final=100, + contract_version=2, + policy_hashes=PolicyHashes.model_validate( + {k: getattr(manifest.network, k) for k in PolicyHashes.model_fields} + ), + registry_epoch=0, + start_state_index_hash="0" * 64, + audit_mode="anchored-full", + roster=entries, + ) + + +def test_owner_admission_accepts_real32_metadata_when_profile_exact(tiny, monkeypatch): + # Given: fail-fast tensor allocator proves metadata intake never trains/allocates state. + store, manifest = tiny + + def fail(*args, **kwargs): + pytest.fail("unexpected tensor state allocation") + + monkeypatch.setattr("hypertrain.trainer.model.init_params", fail) + body, raw = signed_profile(store, manifest) + # When + digest = store.bootstrap_service_capacity_v2(manifest.run_id(), body, raw) + estimate = store.require_roster_v2(manifest.run_id(), roster=roster(32)) + # Then + assert digest == sha256_hex(body) + assert estimate["count"] == 32 + assert estimate["audit_steps"] == 256 and estimate["repair_steps"] == 512 + assert estimate["tape_bytes_bound"] < 1 << 20 + assert estimate["memory_estimate_bytes"] < 1 << 30 + print( + f"CAPACITY_EXISTING_FIXTURE_MAX_RSS_KIB={resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}" + ) + assert store.bootstrap_service_capacity_v2(manifest.run_id(), body, raw) == digest + + +@pytest.mark.parametrize("signer", [service.COORD, service.HOT[0]]) +def test_wrong_role_rejects_when_receipt_not_owner(tiny, signer): + store, manifest = tiny + body, raw = signed_profile(store, manifest, signer) + with pytest.raises(ChallengeError): + store.bootstrap_service_capacity_v2(manifest.run_id(), body, raw) + assert not store._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='service-admission'", (manifest.run_id(),) + ).fetchone() + + +@pytest.mark.parametrize( + "fault", ["bool", "version_bool", "float", "extra", "limit", "backend", "run"] +) +def test_invalid_internal_profile_rejects_when_signed_by_owner(tiny, fault): + store, manifest = tiny + body, _ = signed_profile(store, manifest) + data = envelope_v2.load_json(body) + match fault: + case "bool": + data["max_complete_roster"] = True + case "version_bool": + data["version"] = True + case "float": + body = body.replace(b'"max_complete_roster":32', b'"max_complete_roster":32.0') + case "extra": + data["qualified"] = True + case "limit": + data["max_complete_roster"] = 128 + case "backend": + data["backend_binding_hash"] = "22" * 32 + case "run": + data["run_id"] = "22" * 32 + if fault != "float": + body = canonicalize(data) + receipt = Receipt(w=0, commit_hash=sha256_hex(body), received_round=1) + raw = canonicalize(envelope_v2.seal(service.OWNER, "Receipt", manifest.run_id(), receipt, 100)) + with pytest.raises((ChallengeError, ValueError)): + store.bootstrap_service_capacity_v2(manifest.run_id(), body, raw) + + +@pytest.mark.parametrize("n,admit", [(17, False), (33, True)]) +def test_bad_roster_rejects_before_stateallocation_when_admission_missing_or_count_excess( + tiny, monkeypatch, n, admit +): + store, manifest = tiny + if admit: + body, raw = signed_profile(store, manifest) + store.bootstrap_service_capacity_v2(manifest.run_id(), body, raw) + monkeypatch.setattr( + "hypertrain.trainer.model.init_params", lambda *a, **k: pytest.fail("allocation") + ) + raw = canonicalize( + envelope_v2.seal(service.COORD, "RoundOpenV2", manifest.run_id(), opening(manifest, n), 100) + ) + before = store._db.total_changes + with pytest.raises(ChallengeError, match="ROSTER_LIMIT"): + store.open_round_v2(manifest.run_id(), raw) + assert store._db.total_changes == before + + +def test_legacy16_guard_preserved_when_no_resource_admission(network): + assert network.store.require_roster_v2(network.manifest.run_id(), roster=roster(16)) is None + + +def test_profile_tamper_rejects_when_original_journal_body_changed(tiny): + store, manifest = tiny + body, raw = signed_profile(store, manifest) + store.bootstrap_service_capacity_v2(manifest.run_id(), body, raw) + with store._tx(): + record = store._record_v2(manifest.run_id(), "service-admission", "run") + record["body"]["max_complete_roster"] = 128 + store._put_record_v2(manifest.run_id(), "service-admission", "run", record) + with pytest.raises(ChallengeError, match="AUTHORITY"): + store.require_roster_v2(manifest.run_id(), roster=roster(32)) + + +def test_metadata_bytes_reject_when_full_descriptor_bound_exceeds_tape_budget(tiny): + store, manifest = tiny + body, raw = signed_profile(store, manifest) + store.bootstrap_service_capacity_v2(manifest.run_id(), body, raw) + entries = roster(32) + for entry in entries: + entry["coldkey_group"] = "x" * 40000 + with pytest.raises(ChallengeError, match="RESOURCE_ESTIMATE"): + store.require_roster_v2(manifest.run_id(), roster=entries) + + +@pytest.mark.parametrize("n", [16, 17, 32]) +def test_profile_open_denies_before_allocation_or_publication_when_runtime_unenforced( + tiny, monkeypatch, n +): + store, manifest = tiny + body, raw = signed_profile(store, manifest) + store.bootstrap_service_capacity_v2(manifest.run_id(), body, raw) + seen = {"init_params": 0, "AnchorCache": 0, "put": 0} + + def stop(name): + def forbidden(*args, **kwargs): + seen[name] += 1 + pytest.fail(f"runtime denial reached {name}") + + return forbidden + + monkeypatch.setattr("hypertrain.trainer.model.init_params", stop("init_params")) + monkeypatch.setattr("hypertrain.auditor.replay.AnchorCache", stop("AnchorCache")) + monkeypatch.setattr(store.objects, "put", stop("put")) + raw = canonicalize( + envelope_v2.seal(service.COORD, "RoundOpenV2", manifest.run_id(), opening(manifest, n), 100) + ) + before = store._db.total_changes + with pytest.raises(ChallengeError, match="SERVICE_RUNTIME_NOT_ENFORCED"): + store.open_round_v2(manifest.run_id(), raw) + assert seen == {"init_params": 0, "AnchorCache": 0, "put": 0} + assert store._db.total_changes == before + assert not store._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind IN " + "('round-resource','round','round-meta','start','finality','service-charge')", + (manifest.run_id(),), + ).fetchone() + + +def test_existing_reservation_survives_open_denial_before_pool_or_allocation(tiny, monkeypatch): + store, manifest = tiny + body, raw = signed_profile(store, manifest) + store.bootstrap_service_capacity_v2(manifest.run_id(), body, raw) + with store._tx(): + store._put_record_v2( + manifest.run_id(), + "round-resource", + "prior", + { + "count": 1, + "audit_steps": 8, + "repair_steps": 16, + }, + ) + monkeypatch.setattr( + "hypertrain.trainer.model.init_params", lambda *a, **k: pytest.fail("allocation") + ) + raw = canonicalize( + envelope_v2.seal(service.COORD, "RoundOpenV2", manifest.run_id(), opening(manifest), 100) + ) + before = store._db.total_changes + with pytest.raises(ChallengeError, match="SERVICE_RUNTIME_NOT_ENFORCED"): + store.open_round_v2(manifest.run_id(), raw) + assert store._db.total_changes == before + assert store._record_v2(manifest.run_id(), "round-resource", "prior")["count"] == 1 + assert not store._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='round-resource' AND id='0'", + (manifest.run_id(),), + ).fetchone() + + +def test_legacy_open_without_profile_reaches_existing_allocator_seam(tiny, monkeypatch): + store, manifest = tiny + seen = [] + + def stop(*args): + seen.append("init_params") + raise RuntimeError("legacy allocation sentinel") + + store.clock = lambda: manifest.training.beacon.genesis_time + monkeypatch.setattr("hypertrain.trainer.model.init_params", stop) + raw = canonicalize( + envelope_v2.seal( + service.COORD, "RoundOpenV2", manifest.run_id(), opening(manifest, 16), 100 + ) + ) + with pytest.raises(RuntimeError, match="legacy allocation sentinel"): + store.open_round_v2(manifest.run_id(), raw) + assert seen == ["init_params"] + + +def test_open_denial_never_refunds_existing_committed_debit(tiny): + store, manifest = reserved(tiny) + run_id = manifest.run_id() + assert store._service_charge_v2(run_id, 0, "prior", 1, kind="audit", units=512) + before = store._db.total_changes + raw = canonicalize( + envelope_v2.seal(service.COORD, "RoundOpenV2", run_id, opening(manifest), 100) + ) + with pytest.raises(ChallengeError, match="SERVICE_RUNTIME_NOT_ENFORCED"): + store.open_round_v2(run_id, raw) + assert store._db.total_changes == before + assert not store._service_charge_v2(run_id, 0, "prior", 1, kind="audit", units=512) + with pytest.raises(ChallengeError, match="BUDGET_EXHAUSTED"): + store._service_charge_v2(run_id, 0, "prior", 2, kind="audit", units=1) + + +def reserved(tiny): + store, manifest = tiny + body, raw = signed_profile(store, manifest) + store.bootstrap_service_capacity_v2(manifest.run_id(), body, raw) + entries = roster(32) + estimate = store.require_roster_v2(manifest.run_id(), roster=entries) + with store._tx(): + store._put_record_v2( + manifest.run_id(), + "round", + "0", + envelope_v2.seal( + service.COORD, "RoundOpenV2", manifest.run_id(), opening(manifest), 100 + ), + ) + store._put_record_v2( + manifest.run_id(), + "round-resource", + "0", + { + **estimate, + "deadline": 100, + "max_attempts": 2, + "max_running_jobs": 2, + }, + ) + return store, manifest + + +@pytest.mark.parametrize("method", ["_assignment_v2", "verified_inputs_v2", "aggregate_v2"]) +def test_original_roster_guard_rejects_before_load_when_reservation_changed( + tiny, monkeypatch, method +): + store, manifest = reserved(tiny) + with store._tx(): + record = store._record_v2(manifest.run_id(), "round", "0") + record["body"]["roster"].pop() + store._put_record_v2(manifest.run_id(), "round", "0", record) + monkeypatch.setattr( + "hypertrain.trainer.model.init_params", lambda *a, **k: pytest.fail("state load") + ) + with pytest.raises(ChallengeError, match="RESOURCE_BINDING"): + if method == "_assignment_v2": + store._assignment_v2(manifest.run_id(), 0, service.HOT[0].ss58) + else: + getattr(store, method)(manifest.run_id(), 0) + + +def test_heavy32_execution_remains_blocked_when_only_accounting_admitted(tiny, monkeypatch): + store, manifest = reserved(tiny) + events = {"allocation": 0, "state_load": 0, "decode": 0, "worker_launch": 0} + + def count(name): + def forbidden(*args, **kwargs): + events[name] += 1 + pytest.fail(name) + + return forbidden + + for symbol, name in ( + ("hypertrain.trainer.model.init_params", "allocation"), + ("hypertrain.aggregator.core.load_state", "state_load"), + ("hypertrain.trainer.compress.decompress", "decode"), + ("hypertrain.auditor.worker.execute_island_audit", "worker_launch"), + ): + monkeypatch.setattr(symbol, count(name)) + with pytest.raises(ChallengeError, match="RUNTIME_NOT_ENFORCED"): + store.aggregate_v2(manifest.run_id(), 0) + assert events == {"allocation": 0, "state_load": 0, "decode": 0, "worker_launch": 0} + + +@pytest.mark.parametrize("kind,limit", [("object", 65536), ("tape", 1 << 20), ("context", 1 << 20)]) +def test_actual_byte_limits_reject_when_one_byte_over_profile(tiny, kind, limit): + store, manifest = reserved(tiny) + store._service_bytes_v2(manifest.run_id(), 0, b"x" * limit, kind=kind) + with pytest.raises(ChallengeError, match="BYTES"): + store._service_bytes_v2(manifest.run_id(), 0, b"x" * (limit + 1), kind=kind) + + +@pytest.mark.parametrize("kind,limit", [("audit", 512), ("outer", 50_000_000), ("repair", 1024)]) +def test_charges_survive_failure_restart_without_duplicate_debit(tiny, kind, limit): + store, manifest = reserved(tiny) + run_id = manifest.run_id() + assert store._service_charge_v2(run_id, 0, "job", 1, kind=kind, units=limit) + # A failed execution does not undo the committed debit; next service process restores SQLite. + restored = ChallengeStore( + store.state_dir, + store.params, + service.COORD, + service.OWNER.ss58, + verify_beacon=store.verify_beacon, + objects=store.objects, + ) + try: + assert not restored._service_charge_v2(run_id, 0, "job", 1, kind=kind, units=limit) + with pytest.raises(ChallengeError, match="BUDGET_EXHAUSTED"): + restored._service_charge_v2(run_id, 0, "job", 2, kind=kind, units=1) + with pytest.raises(ChallengeError, match="CONFLICT"): + restored._service_charge_v2(run_id, 0, "job", 1, kind=kind, units=1) + assert ( + restored._db.execute( + "SELECT COUNT(*) FROM records_v2 WHERE run_id=? AND kind='service-charge'", + (run_id,), + ).fetchone()[0] + == 1 + ) + finally: + restored.close_v2_notifications() + restored._db.close() + + +def test_source_binding_ignores_unrelated_methods_but_rejects_capacity_changes(tmp_path): + source = TREE / "src/hypertrain" + paths = ( + "aggregator/core.py", + "aggregator/tape_v2.py", + "aggregator/rollback_v2.py", + "trainer/compress.py", + ) + original = ChallengeStore._service_implementation_v2(source, paths) + tree = ast.parse((source / "challenge/store.py").read_text()) + # Inspect AST selection directly, without rewriting production/source fixtures. + import unittest.mock + + unrelated = ast.FunctionDef( + name="unrelated", + args=ast.arguments(posonlyargs=[], args=[], kwonlyargs=[], kw_defaults=[], defaults=[]), + body=[ast.Pass()], + decorator_list=[], + ) + tree.body.append(unrelated) + with unittest.mock.patch("hypertrain.challenge.store.ast.parse", return_value=tree): + assert ChallengeStore._service_implementation_v2(source, paths) == original + node = next( + n + for n in ast.walk(tree) + if isinstance(n, ast.FunctionDef) and n.name == "_service_bytes_v2" + ) + node.body.append( + ast.Raise( + exc=ast.Call(func=ast.Name(id="RuntimeError", ctx=ast.Load()), args=[], keywords=[]) + ) + ) + with unittest.mock.patch("hypertrain.challenge.store.ast.parse", return_value=tree): + assert ChallengeStore._service_implementation_v2(source, paths) != original + + +def test_finalization_cannot_reset_charges_and_nested_failure_cannot_erase_debit(tiny): + store, manifest = reserved(tiny) + run_id = manifest.run_id() + with store._tx(): + with pytest.raises(ChallengeError, match="REQUIRES_COMMITTED_TRANSACTION"): + store._service_charge_v2(run_id, 0, "job", 1, kind="audit", units=1) + assert store._service_charge_v2(run_id, 0, "job", 1, kind="audit", units=512) + with store._tx(): + store._put_record_v2(run_id, "finalize", "0", {"w": 0}) + with pytest.raises(ChallengeError, match="BUDGET_EXHAUSTED"): + store._service_charge_v2(run_id, 0, "next", 1, kind="audit", units=1) + + +@pytest.mark.parametrize( + "case", ["valid", "later-shape", "codec", "format", "oversize", "stored-oversize", "legacy"] +) +def test_delta_intake_checks_trusted_geometry_before_durable_acceptance(tiny, monkeypatch, case): + import torch + + import hypertrain.trainer.compress as codec + from hypertrain.protocol.messages_v2 import CommitV2, DeltaManifestV2 + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.model import param_shapes + + store, manifest = tiny if case == "legacy" else reserved(tiny) + run_id, miner = manifest.run_id(), service.HOT[0] + config = TrainConfig.from_manifest_v2(manifest) + shapes = param_shapes(config.model) + parts = [codec.MAGIC["dense-int8"], struct.pack(" str: + return "/v2/runs/" + self.manifest.run_id() + + def push(self, number: int) -> None: + self.now = number + self.store.clock = lambda: self.manifest.training.beacon.genesis_time + (number - 1) * 3 + response = self.client.post( + "/v1/admin/beacon", json=fixture_beacon(number), headers=admin() + ) + assert response.status_code == 200, response.text + + def signed(self, key, type, body, expiry=10000): + return seal(key, type, self.manifest.run_id(), body, expiry) + + def join(self, index=0): + request = sign_join( + HOT[index], + COLD[index], + run_id=self.manifest.run_id(), + request_id=sha256_hex(f"join-{index}".encode()), + expires_beacon=10000, + policy_hash=self.manifest.network.admission_policy_hash, + hardware_hint=HardwareHint(device_name="cpu", device_count=999, driver="advisory"), + ) + return self.client.post(self.url + "/join", content=canonicalize(request.body())) + + +def admin(): + return {"Authorization": "Bearer integration-admin"} + + +@pytest.mark.parametrize( + "fault", + [ + "none", + "received", + "accepted", + "stored", + "expiry", + "owner", + "graph", + "scope", + "cutoff", + "backend", + "source", + "workload", + "extra", + ], +) +def test_shadow_operator_addressed_contract(tmp_path, fault): + """Metadata contract only. Never accepted CUDA/store bootstrap authority.""" + import copy + + from pydantic import ValidationError + + from hypertrain.challenge.store import ( + ChallengeError, + shadow_operator_expected, + shadow_operator_receipt, + ) + from hypertrain.protocol.envelope import EnvelopeError + + record = { + "domain": "shadow-operator-admission-v1", + "run_id": "11" * 32, + "owner": OWNER.ss58, + "admission_hash": "22" * 32, + "manifest_hash": "11" * 32, + "economics_policy_hash": "33" * 32, + "graph_hash": "44" * 32, + "workload": {"normal": 116, "ceiling": 126}, + "backend_qualification_authority_hash": "55" * 32, + "source_map_hash": "66" * 32, + "profile_hash": "77" * 32, + "hosts": {"h0": {"instance_id": 1}, "h1": {"instance_id": 2}}, + "budget": {"scope": "FULL116_CONTINUATION"}, + "accepted_beacon": 2, + "cutoff_unix": 100, + "expires_beacon": 5, + } + expected = copy.deepcopy(record) + received, accepted_at, expiry, signer, clock = 2, 2, 5, OWNER, 99 + if fault == "received": + received = 3 + elif fault == "accepted": + record["accepted_beacon"] = 3 + elif fault == "stored": + accepted_at = 3 + elif fault == "expiry": + expiry = 6 + elif fault == "owner": + signer = HOT[0] + elif fault == "graph": + record["graph_hash"] = "88" * 32 + elif fault == "scope": + record["budget"]["scope"] = "FIRST4_HISTORICAL_QUALIFICATION_ONLY" + elif fault == "cutoff": + clock = 100 + elif fault == "backend": + record["backend_qualification_authority_hash"] = "99" * 32 + elif fault == "source": + record["source_map_hash"] = "aa" * 32 + elif fault == "workload": + record["workload"]["normal"] = 4 + elif fault == "extra": + record["trusted"] = True + receipt = seal( + signer, + "Receipt", + record["run_id"], + { + "w": 0, + "commit_hash": sha256_hex(canonicalize(record)), + "received_round": received, + }, + expiry, + ) + raw = canonicalize({"record": record, "receipt": receipt}) + + def validate(): + parsed, digest, original = shadow_operator_receipt( + raw, OWNER.ss58, record["run_id"], accepted_at + ) + shadow_operator_expected(parsed, expected, clock) + return digest, original + + if fault == "none": + first = validate() + assert validate() == first + # Historical recovery at original beacon survives current expiry; never grants new work. + assert shadow_operator_receipt(raw, OWNER.ss58, record["run_id"], 2)[1] == first[0] + else: + with pytest.raises((ChallengeError, EnvelopeError, ValidationError)): + validate() + + +@pytest.mark.parametrize("fault", ["missing", "tampered"]) +def test_shadow_operator_cpu_and_missing_qualification_refused(network, tmp_path, fault): + """Real existing Store CPU backend is NOT an operator admission positive.""" + from hypertrain.challenge.store import ChallengeError + from hypertrain.data.store import StoreError + + run_id = network.manifest.run_id() + assert network.store._backend_v2(network.manifest) == "cpu" + if fault == "tampered": + with network.store._tx(): + network.store._put_record_v2( + run_id, + "qualification", + network.manifest.training.reference_spec.image_digest, + {"authority_hash": "ab" * 32, "receipt": {}}, + ) + with pytest.raises((ChallengeError, StoreError, ValueError, KeyError)): + network.store._backend_v2(network.manifest) + with pytest.raises(ChallengeError): + network.store._shadow_operator_v2(run_id) + assert ( + network.store._db.execute( + "SELECT COUNT(*) FROM records_v2 WHERE kind='shadow-operator'" + ).fetchone()[0] + == 0 + ) + + +def test_shadow_operator_retention_transaction_only(network, tmp_path): + """Persistence unit only: not public intake or accepted economic/GPU authority.""" + from hypertrain.challenge.store import ChallengeError + + store, run_id = network.store, network.manifest.run_id() + payload = {"metadata_persistence_unit": True} + digest = sha256_hex(canonicalize(payload)) + events = [tuple(row) for row in store._db.execute("SELECT * FROM escrow_events")] + store._retain_shadow_operator_v2(run_id, payload, digest, 1, tmp_path / "graph") + store._retain_shadow_operator_v2(run_id, payload, digest, 1, tmp_path / "graph") + before = store._record_v2(run_id, "shadow-operator", "run") + with pytest.raises(ChallengeError, match="immutable"): + store._retain_shadow_operator_v2(run_id, payload, digest, 2, tmp_path / "graph") + assert store._record_v2(run_id, "shadow-operator", "run") == before + assert [tuple(row) for row in store._db.execute("SELECT * FROM escrow_events")] == events + with pytest.raises(RuntimeError): + with store._tx(): + store._retain_shadow_operator_v2("22" * 32, payload, digest, 1, tmp_path / "graph") + raise RuntimeError("rollback") + assert ( + store._db.execute( + "SELECT COUNT(*) FROM records_v2 WHERE kind='shadow-operator'" + ).fetchone()[0] + == 1 + ) + + +def fixture_beacon(number): + from hypertrain.beacon.core import FixtureBeacon + + b = FixtureBeacon(current=number).get(number) + return {"round": b.round, "signature": b.signature, "randomness": b.randomness} + + +@pytest.fixture +def network(tmp_path: Path, request): + from hypertrain.beacon.core import BeaconRound + + s = fixture.setup() + objective = getattr(request, "param", None) + if objective is not None: + import numpy as np + + from hypertrain.protocol.hashing import MerkleTree + from hypertrain.trainer.config import TrainConfig + + od_spec = importlib.util.spec_from_file_location( + "service_od_fixture", Path(__file__).parents[1] / "layout/test_od_island_v2.py" + ) + od_module = importlib.util.module_from_spec(od_spec) + od_spec.loader.exec_module(od_module) + wrapper = od_module.od_manifest(objective, 2, True) + od_body = wrapper.body() + od_body["training"]["coord_pubkey"] = COORD.ss58 + od_body["training"]["reference_spec"]["image_digest"] = ( + s.manifest.training.reference_spec.image_digest + ) + od_body["training"]["reference_spec"]["driver_allowlist"] = ["cpu"] + od_body["training"]["reference_spec"]["sm_count"] = 1 + od_body["training"]["dataset"].update( + n_samples=32, assign_unit=1, sample_format="u16[seq_len+1] token ids" + ) + cfg = TrainConfig.from_manifest_v2(wrapper) + get = od_module.samples(cfg) + rows = np.asarray([get(i) for i in range(32)], dtype="= pin.horizon + assert all(p.delete_after is not None for p in state.uploads[grant.digest()].pins.values()) + assert grant.delta_hash not in await relay.eligible_deletions() + + anyio.run(check_release) + + # Restore actual pre-accept authority, not fabricated health/admission/finality. + network.store.close_v2_notifications() + network.store._services_v2.clear() + db = sqlite3.connect(tmp_path / "before-accept.db") + db.backup(network.store._db) + db.close() + network.now = network.store._now(network.store._db) + shutil.rmtree(network.store.state_dir / "relay-backing") + shutil.copytree(tmp_path / "relay-before-release", network.store.state_dir / "relay-backing") + + # Restore both authorities at the same snapshot; preserve original signatures. + async def custody(): + record = (await relay.state()).uploads[grant.digest()] + return record.opportunity, list(record.chunks.values()) + + opportunity_env, chunk_envs = anyio.run(custody) + opportunity = AcceptedUploadAck.model_validate(opportunity_env.body) + chunks = [ChunkCustodyAck.model_validate(e.body) for e in chunk_envs] + artifacts = [ + opportunity_env.model_dump(mode="json"), + *[e.model_dump(mode="json") for e in chunk_envs], + ] + network.push(opportunity.service_deadline + 1) + before = network.store._services(run_id)[0].balances() + + def evidence_for(items, *, ids=None, hash_override=None): + ids = [a.ss58 for a in AUDITORS] if ids is None else ids + evidence = RelayFailureEvidence( + run_id=run_id, + relay_id="local", + key_id="k1", + assignment_hash=sha256_hex(canonicalize(assigned["body"])), + grant_hash=grant.digest(), + upload_ack_hash=opportunity.digest(), + chunk_manifest_hash=grant.chunk_manifest_hash, + chunk_ack_hashes=[c.digest() for c in chunks], + receipt_hash=None, + retention_hash=sha256_hex(canonicalize(sorted(c.digest() for c in chunks))), + request=None, + response=None, + observer_ids=ids, + observed_beacons=[network.now] * len(ids), + result="CONTRACTUAL_SERVICE_FAILURE", + availability_context=AvailabilityContext( + master_healthy=True, + beacon_healthy=True, + reference_healthy=True, + observer_healthy=True, + ), + evidence_hash=hash_override or sha256_hex(canonicalize(items)), + ) + signed = relay_envelope.seal( + COORD, "RelayFailureEvidence", run_id, evidence, network.now + 20 + ) + return evidence, { + "evidence": signed, + "artifacts": items, + "observations": [ + relay_envelope.seal(a, "RelayFailureEvidence", run_id, evidence, network.now + 20) + for a in AUDITORS + ], + } + + evidence, body = evidence_for(artifacts) + # Signed booleans alone do not establish observer health. + denied = network.client.post(network.url + "/admin/relay-failure", json=body, headers=admin()) + assert denied.status_code == 409, denied.text + for auditor in AUDITORS: + url = network.url + f"/relay-observers/{grant.digest()}/{auditor.ss58}" + challenge = network.client.get(url) + assert challenge.status_code == 200, challenge.text + forged = network.client.post( + url, json=network.signed(HOT[0], "Receipt", challenge.json()["body"]) + ) + assert forged.status_code == 409 + healthy = network.client.post( + url, json=network.signed(auditor, "Receipt", challenge.json()["body"]) + ) + assert healthy.status_code == 200, healthy.text + # Original opportunity only cannot frame byte custody; mixed scope rejected. + _, prefix = evidence_for([artifacts[0]]) + assert ( + network.client.post( + network.url + "/admin/relay-failure", json=prefix, headers=admin() + ).status_code + == 422 + ) + mixed = json.loads(json.dumps(body)) + mixed["observations"][1] = mixed["observations"][0] + assert ( + network.client.post( + network.url + "/admin/relay-failure", json=mixed, headers=admin() + ).status_code + == 403 + ) + stale_body = json.loads(json.dumps(body)) + stale_body["observations"] = [ + relay_envelope.seal(a, "RelayFailureEvidence", run_id, evidence, network.now) + for a in AUDITORS + ] + assert ( + network.client.post( + network.url + "/admin/relay-failure", json=stale_body, headers=admin() + ).status_code + == 403 + ) + altered = evidence.model_copy(update={"assignment_hash": "0" * 64}) + wrong_scope = { + **body, + "evidence": relay_envelope.seal(COORD, "RelayFailureEvidence", run_id, altered, 10000), + "observations": [ + relay_envelope.seal(a, "RelayFailureEvidence", run_id, altered, 10000) for a in AUDITORS + ], + } + assert ( + network.client.post( + network.url + "/admin/relay-failure", json=wrong_scope, headers=admin() + ).status_code + == 422 + ) + accepted = network.client.post(network.url + "/admin/relay-failure", json=body, headers=admin()) + assert accepted.status_code == 200, accepted.text + assert accepted.json() == { + "result": "CONTRACTUAL_SERVICE_FAILURE", + "disabled": True, + "miner_penalty": 0, + } + receipt_artifacts = [artifacts[0], receipt_env.model_dump(mode="json")] + receipt_evidence = evidence.model_copy( + update={ + "chunk_ack_hashes": [], + "receipt_hash": receipt.digest(), + "retention_hash": receipt.digest(), + "evidence_hash": sha256_hex(canonicalize(receipt_artifacts)), + } + ) + receipt_body = { + "evidence": relay_envelope.seal( + COORD, "RelayFailureEvidence", run_id, receipt_evidence, 10000 + ), + "artifacts": receipt_artifacts, + "observations": [ + relay_envelope.seal(a, "RelayFailureEvidence", run_id, receipt_evidence, 10000) + for a in AUDITORS + ], + } + accepted_receipt = network.client.post( + network.url + "/admin/relay-failure", json=receipt_body, headers=admin() + ) + assert accepted_receipt.status_code == 200, accepted_receipt.text + # Rescue invokes actual signed retrieval/chunk bytes; no backdated acceptance. + rescued = network.client.post( + network.url + f"/admin/relay-fallback/{evidence.digest()}", headers=admin() + ) + assert rescued.status_code == 200, rescued.text + assert rescued.json()["rescued"] and rescued.json()["master_acceptance"] is None + assert rescued.json()["assignment"] == assigned + assert network.store.objects.get(grant.delta_hash) == relay.backing.store.get(grant.delta_hash) + assert network.store._services(run_id)[0].balances() == before + + # Signed successor retains current custody key; current/next validity explicit. + successor_body = registry.body() + successor_body.update(epoch=registry.epoch + 1, previous_registry_hash=registry.digest()) + next_key = Keypair(b"\x75" * 32) + successor_body["specs"][0]["pubkeys"].append( + { + "key_id": "k2", + "pubkey": next_key.ss58, + "valid_from_round": network.now, + "valid_until_round": 10000, + } + ) + successor = RelayRegistryV1.model_validate(successor_body) + rotated = network.client.post( + network.url + "/admin/relay-registry", + json=relay_envelope.seal(COORD, "RelayRegistryV1", run_id, successor, 10000), + headers=admin(), + ) + assert rotated.status_code == 200, rotated.text + + async def rotation_ready(): + rotated_relay = Relay( + run_id=run_id, + master=COORD.ss58, + registry=successor, + network_manifest_hash=assigned["body"]["manifest_hash"], + observers={a.ss58 for a in AUDITORS}, + relay_id="local", + region="local", + keys={"k1": RELAY, "k2": next_key}, + active_key="k2", + backing=relay.backing, + now=now, + settlement=settled, + ) + assert await rotated_relay.ready() + original = (await rotated_relay.state()).uploads[grant.digest()] + assert original.opportunity == opportunity_env + assert original.chunks == {str(c.body["index"]): c for c in chunk_envs} + + anyio.run(rotation_ready) + dropped = successor.body() + dropped.update(epoch=successor.epoch + 1, previous_registry_hash=successor.digest()) + dropped["specs"][0]["pubkeys"] = [dropped["specs"][0]["pubkeys"][1]] + rejected = network.client.post( + network.url + "/admin/relay-registry", + json=relay_envelope.seal( + COORD, "RelayRegistryV1", run_id, RelayRegistryV1.model_validate(dropped), 10000 + ), + headers=admin(), + ) + assert rejected.status_code == 409, rejected.text + # Timely branch: extend registry via signed successor before granting a miner + # whose accepted commit exists but whose relay assignment was not yet issued. + network.store.close_v2_notifications() + network.store._services_v2.clear() + db = sqlite3.connect(tmp_path / "before-accept.db") + db.backup(network.store._db) + db.close() + network.now = network.store._now(network.store._db) + network.push(network.now) + publish(alternate_registry) + assigned_other = network.client.get(network.url + f"/rounds/0/relay/{HOT[1].ss58}") + assert assigned_other.status_code == 200, assigned_other.text + assigned_other = assigned_other.json() + assert assigned_other["assignment"]["body"]["fallback_ids"] == ["fallback"] + path = network.store.state_dir / "jobs-v2/0" / HOT[1].ss58 / "published/rank-0/delta.bin" + from hypertrain.relay.client import RelayClient, chunk_manifest + + chunks_other = chunk_manifest(path) + response = network.client.post( + network.url + "/upload-grant", + json=relay_envelope.seal(HOT[1], "UploadChunkManifest", run_id, chunks_other, 10000), + ) + assert response.status_code == 200, response.text + original_grant = UploadGrant.model_validate(response.json()["body"]) + uncertain = RelayFailureEvidence( + run_id=run_id, + relay_id="local", + key_id="k1", + assignment_hash=sha256_hex(canonicalize(assigned_other["assignment"]["body"])), + grant_hash=original_grant.digest(), + upload_ack_hash="0" * 64, + chunk_manifest_hash=original_grant.chunk_manifest_hash, + chunk_ack_hashes=[], + receipt_hash=None, + retention_hash="0" * 64, + request=None, + response=None, + observer_ids=[AUDITORS[0].ss58], + observed_beacons=[network.now], + result="UNCONFIRMED_AVAILABILITY", + availability_context=AvailabilityContext( + master_healthy=False, + beacon_healthy=False, + reference_healthy=False, + observer_healthy=False, + ), + evidence_hash=sha256_hex(canonicalize([])), + ) + response = network.client.post( + network.url + "/admin/relay-failure", + json={ + "evidence": relay_envelope.seal( + COORD, "RelayFailureEvidence", run_id, uncertain, 10000 + ), + "artifacts": [], + "observations": [], + }, + headers=admin(), + ) + assert response.status_code == 200 and not response.json()["disabled"] + response = network.client.post( + network.url + f"/admin/relay-fallback/{uncertain.digest()}", headers=admin() + ) + assert response.status_code == 200, response.text + fallback = response.json() + reassigned = UploadGrant.model_validate(fallback["grant"]["body"]) + assert reassigned.relay_id == "fallback" and reassigned.exp_drand == original_grant.exp_drand + assert reassigned.delta_hash == original_grant.delta_hash + assert fallback["assignment"] == assigned_other["assignment"] + + async def actual_fallback(): + alternate = Relay( + run_id=run_id, + master=COORD.ss58, + registry=alternate_registry, + network_manifest_hash=assigned_other["assignment"]["body"]["manifest_hash"], + observers={a.ss58 for a in AUDITORS}, + relay_id="fallback", + region="z-local", + keys={"f1": alternate_key}, + active_key="f1", + backing=StreamStore( + LocalFSStore(network.store.state_dir / "fallback-backing"), backing_http + ), + now=now, + ) + async with real_client( + transport=httpx.ASGITransport(app=relay_app(alternate, "x" * 32)) + ) as client: + receipt = await RelayClient(alternate_registry, client).upload( + path, + grant_raw=relay_envelope.parse_envelope(fallback["grant"]), + assignment=relay_envelope.parse_envelope(fallback["assignment"]), + round_open=network.store._record_v2(run_id, "round", "0"), + ) + accepted = await network.store.relay_receipt_v2( + canonicalize(receipt.model_dump(mode="json")), client + ) + assert accepted["body"]["delta_hash"] == original_grant.delta_hash + + anyio.run(actual_fallback) + drained = network.client.post(network.url + "/admin/relay-drain/local", headers=admin()) + assert drained.status_code == 200 + new_grant = network.client.post( + network.url + "/upload-grant", + json=relay_envelope.seal(HOT[1], "UploadChunkManifest", run_id, chunks_other, 10000), + ) + assert new_grant.status_code == 409, new_grant.text + + async def drain(): + async with real_client( + transport=httpx.ASGITransport(app=app), base_url="https://relay.test" + ) as client: + assert ( + await client.post( + "/internal/drain", headers={"Authorization": "Bearer " + "x" * 32} + ) + ).status_code == 200 + assert (await client.get("/readyz")).status_code == 503 + + anyio.run(drain) + anyio.run(backing_http.aclose) + + +@pytest.mark.parametrize("invalid", [None, "partial-receipt", "duplicate", "gap", "contradiction"]) +def test_complete_receipt_prefix_rescue_http(network, monkeypatch, tmp_path, invalid): + """Exact two-chunk receipt/prefix regression; no live-round/graduation sweep.""" + import json + + import anyio + import httpx + + from hypertrain.data.store import LocalFSStore + from hypertrain.data.stream_store import StreamStore + from hypertrain.protocol import envelope_v2, relay_envelope + from hypertrain.protocol.relay_messages import ( + AcceptedUploadAck, + AvailabilityContext, + ChunkCustodyAck, + RelayAssignment, + RelayFailureEvidence, + RelayNetworkManifest, + RelayReceipt, + UploadChunk, + UploadChunkManifest, + UploadGrant, + ) + from hypertrain.relay.app import create_app as relay_app + from hypertrain.relay.core import Relay + + spec = importlib.util.spec_from_file_location( + "receipt_prefix_shadow", Path(__file__).parents[1] / "e2e/test_service_network_v2_e2e.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + # One real accepted MATCH reference supplies observer-health byte authority. + module.test_actual_shadow_trial_does_not_mutate_production(network) + run_id = network.manifest.run_id() + old = network.store._registry_v2(network.manifest) + body = old.body() + body.update(epoch=1, previous_registry_hash=old.digest()) + body["specs"][0].update(max_object_bytes=8 << 20, max_inflight_bytes=16 << 20) + registry = RelayRegistryV1.model_validate(body) + response = network.client.post( + network.url + "/admin/relay-registry", + json=relay_envelope.seal(COORD, "RelayRegistryV1", run_id, registry, 10000), + headers=admin(), + ) + assert response.status_code == 200, response.text + payload = b"a" * (4 << 20) + b"b" + chunks = UploadChunkManifest( + size=len(payload), + chunks=[ + UploadChunk(index=0, off=0, len=4 << 20, chunk_sha256=sha256_hex(payload[:-1])), + UploadChunk(index=1, off=4 << 20, len=1, chunk_sha256=sha256_hex(payload[-1:])), + ], + ) + net = RelayNetworkManifest( + run_id=run_id, + base_manifest_hash=network.manifest.training.training_hash(), + registry_hash=registry.digest(), + specs_hash=sha256_hex(canonicalize([s.body() for s in registry.specs])), + assignment_policy="master-observed-median3-v1", + ) + assignment = RelayAssignment( + w=0, + hotkey=HOT[0].ss58, + primary_id="local", + fallback_ids=[], + assignment_epoch=registry.epoch, + manifest_hash=net.digest(), + exp_drand=network.now + 20, + ) + grant = UploadGrant( + w=0, + hotkey=HOT[0].ss58, + relay_id="local", + assignment_epoch=registry.epoch, + delta_hash=sha256_hex(payload), + size=len(payload), + chunk_manifest_hash=chunks.chunk_manifest_hash(), + retain_until=network.now + 200, + nonce="ad" * 32, + exp_drand=assignment.exp_drand, + ) + signed_grant = relay_envelope.seal(COORD, "UploadGrant", run_id, grant, grant.exp_drand) + signed_assignment = relay_envelope.seal( + COORD, "RelayAssignment", run_id, assignment, assignment.exp_drand + ) + # Accepted master journal adapter: original typed signatures, not supplied trust flags. + with network.store._tx(): + network.store._put_record_v2(run_id, "grant", grant.digest(), signed_grant) + network.store._put_record_v2(run_id, "chunk-manifest", grant.digest(), chunks.body()) + network.store._put_record_v2( + run_id, "relay-assignment", "0:" + HOT[0].ss58, signed_assignment + ) + opened = { + "w": 0, + **{ + k: "12" * 32 + for k in ( + "prev_final_hash", + "theta_hash", + "outer_state_hash", + "center_hash", + "roster_hash", + "honeypot_commit", + "start_state_index_hash", + ) + }, + "d_open": 1, + "d_assign": 2, + "d_commit": network.now + 10, + "d_audit": network.now + 11, + "d_upload": grant.exp_drand, + "d_final": network.now + 30, + "contract_version": 2, + "policy_hashes": { + k: getattr(network.manifest.network, k) + for k in ( + "admission_policy_hash", + "economics_policy_hash", + "aggregation_policy_hash", + "dispute_policy_hash", + "audit_policy_hash", + ) + }, + "registry_epoch": registry.epoch, + "audit_mode": "anchored-full", + "roster": [], + } + real_client = httpx.AsyncClient + backing = real_client() + + async def now(): + return network.now + + relay = Relay( + run_id=run_id, + master=COORD.ss58, + registry=registry, + network_manifest_hash=net.digest(), + observers={a.ss58 for a in AUDITORS}, + relay_id="local", + region="local", + keys={"k1": RELAY}, + active_key="k1", + backing=StreamStore(LocalFSStore(tmp_path / "prefix-backing"), backing), + now=now, + ) + + async def uploaded(): + opportunity = await relay.accept( + relay_envelope.parse_envelope(signed_grant), + chunks, + relay_envelope.parse_envelope(signed_assignment), + envelope_v2.parse_envelope(network.signed(COORD, "RoundOpenV2", opened)), + ) + + async def source(part): + yield part + + first = await relay.chunk(grant.digest(), 0, source(payload[:-1])) + await relay.chunk(grant.digest(), 1, source(payload[-1:])) + return opportunity, first, await relay.complete(grant.digest()) + + opportunity_env, prefix_env, receipt_env = anyio.run(uploaded) + opportunity = AcceptedUploadAck.model_validate(opportunity_env.body) + prefix = ChunkCustodyAck.model_validate(prefix_env.body) + receipt = RelayReceipt.model_validate(receipt_env.body) + if invalid == "partial-receipt": + receipt = receipt.model_copy(update={"size": prefix.len, "delta_hash": prefix.chunk_sha256}) + receipt_env = relay_envelope.parse_envelope( + relay_envelope.seal(RELAY, "RelayReceipt", run_id, receipt, grant.retain_until) + ) + if invalid in {"gap", "contradiction"}: + prefix = prefix.model_copy( + update={"off": 1} if invalid == "gap" else {"chunk_sha256": "ff" * 32} + ) + prefix_env = relay_envelope.parse_envelope( + relay_envelope.seal(RELAY, "ChunkCustodyAck", run_id, prefix, grant.retain_until) + ) + artifacts = [ + opportunity_env.model_dump(mode="json"), + prefix_env.model_dump(mode="json"), + receipt_env.model_dump(mode="json"), + ] + if invalid == "duplicate": + artifacts.insert(2, prefix_env.model_dump(mode="json")) + network.push(grant.exp_drand + 1) + for auditor in AUDITORS: + url = network.url + f"/relay-observers/{grant.digest()}/{auditor.ss58}" + challenge = network.client.get(url) + assert challenge.status_code == 200, challenge.text + response = network.client.post( + url, json=network.signed(auditor, "Receipt", challenge.json()["body"]) + ) + assert response.status_code == 200, response.text + evidence = RelayFailureEvidence( + run_id=run_id, + relay_id="local", + key_id="k1", + assignment_hash=assignment.digest(), + grant_hash=grant.digest(), + upload_ack_hash=opportunity.digest(), + chunk_manifest_hash=grant.chunk_manifest_hash, + chunk_ack_hashes=[prefix.digest()], + receipt_hash=receipt.digest(), + retention_hash=receipt.digest(), + request=None, + response=None, + observer_ids=[a.ss58 for a in AUDITORS], + observed_beacons=[network.now] * 2, + result="CONTRACTUAL_SERVICE_FAILURE", + availability_context=AvailabilityContext( + master_healthy=True, beacon_healthy=True, reference_healthy=True, observer_healthy=True + ), + evidence_hash=sha256_hex(canonicalize(artifacts)), + ) + failure = network.client.post( + network.url + "/admin/relay-failure", + json={ + "evidence": relay_envelope.seal(COORD, "RelayFailureEvidence", run_id, evidence, 10000), + "artifacts": artifacts, + "observations": [ + relay_envelope.seal(a, "RelayFailureEvidence", run_id, evidence, 10000) + for a in AUDITORS + ], + }, + headers=admin(), + ) + if invalid == "partial-receipt": + assert failure.status_code == 422, failure.text + anyio.run(backing.aclose) + return + assert failure.status_code == 200, failure.text + app = relay_app(relay, "x" * 32) + monkeypatch.setattr( + httpx, + "AsyncClient", + lambda *a, **kw: real_client(*a, **{**kw, "transport": httpx.ASGITransport(app=app)}), + ) + rescued = network.client.post( + network.url + f"/admin/relay-fallback/{evidence.digest()}", headers=admin() + ) + if invalid is not None: + assert rescued.status_code == 422, rescued.text + else: + assert rescued.status_code == 200, rescued.text + assert rescued.json()["grant"] == signed_grant + assert rescued.json()["assignment"] == signed_assignment + assert rescued.json()["master_acceptance"] is None + assert network.store.objects.get(grant.delta_hash) == payload + records = [ + json.loads(r[0])["body"] + for r in network.store._db.execute("SELECT data FROM records_v2 WHERE kind='retrieval'") + ] + assert len(records) == 1 + assert records[0]["receipt_hash"] == receipt.digest() + assert records[0]["custody_ack_hashes"] == [] + assert records[0]["size"] == len(payload) + assert records[0]["retention_hash"] == receipt.digest() + assert records[0]["deadline_beacon"] == min(network.now + 10, receipt.retain_until) + anyio.run(backing.aclose) + + +@pytest.mark.parametrize( + "fault", + [ + None, + "round-mismatch", + "arbitrary-subject", + "ambiguous", + "explicit-identical", + "wrong-signer", + "wrong-run", + "unknown-commit", + ], +) +def test_upload_grant_exact_subject_with_two_live_rounds(network, tmp_path, fault): + import anyio + import httpx + + from hypertrain.data.store import LocalFSStore + from hypertrain.data.stream_store import StreamStore, spool + from hypertrain.protocol import envelope_v2, relay_envelope + from hypertrain.protocol.messages_v2 import CommitV2 + from hypertrain.protocol.relay_messages import ( + RelayAssignment, + RelayNetworkManifest, + RelayReceipt, + RetrievalRequest, + RetrievalResponse, + UploadChunk, + UploadChunkManifest, + UploadGrant, + ) + from hypertrain.relay.core import Relay + + assert network.join().status_code == 200 + run_id = network.manifest.run_id() + registry = network.store._registry_v2(network.manifest) + relay_net = RelayNetworkManifest( + run_id=run_id, + base_manifest_hash=network.manifest.training.training_hash(), + registry_hash=registry.digest(), + specs_hash=sha256_hex(canonicalize([s.body() for s in registry.specs])), + assignment_policy="master-observed-median3-v1", + ) + payloads = [b"old", b"new"] + if fault in {"ambiguous", "explicit-identical"}: + payloads[1] = payloads[0] + assignments, chunks, commit_hashes = [], [], [] + for w, payload in enumerate(payloads): + chunk = UploadChunkManifest( + size=3, chunks=[UploadChunk(index=0, off=0, len=3, chunk_sha256=sha256_hex(payload))] + ) + chunks.append(chunk) + assignment = RelayAssignment( + w=w, + hotkey=HOT[0].ss58, + primary_id="local", + fallback_ids=[], + assignment_epoch=0, + manifest_hash=relay_net.digest(), + exp_drand=552 + w * 31, + ) + signed = relay_envelope.seal( + COORD, "RelayAssignment", run_id, assignment, assignment.exp_drand + ) + assignments.append(signed) + commit = CommitV2.model_validate( + { + "w": w + 1 if fault == "round-mismatch" and w == 1 else w, + "hotkey": HOT[0].ss58, + "leaf_scheme": "ht-leaf-v1", + "n_leaves": 2, + "leaves_root": "01" * 32, + "metrics_root": "02" * 32, + "final_theta_hash": "03" * 32, + "tokens": 8, + "ef_in_hash": "04" * 32, + "ef_out_hash": "05" * 32, + "delta_hash": sha256_hex(payload), + "delta_bytes": 3, + } + ) + commit_hashes.append(sha256_hex(canonicalize(commit.model_dump(mode="json")))) + committed_envelope = network.signed(HOT[0], "CommitV2", commit) + if w == 1 and fault == "wrong-signer": + committed_envelope = network.signed(HOT[1], "CommitV2", commit) + if w == 1 and fault == "wrong-run": + committed_envelope = envelope_v2.seal(HOT[0], "CommitV2", "ff" * 32, commit, 10000) + with network.store._tx(): + network.store.objects.put(payload) + network.store._put_record_v2(run_id, "relay-assignment", f"{w}:{HOT[0].ss58}", signed) + network.store._put_record_v2(run_id, "commit", f"{w}:{HOT[0].ss58}", committed_envelope) + # Exact accepted journal boundary, real signed subjects; no graduation/train. + network.push(385) + requested = chunks[1] + if fault == "arbitrary-subject": + requested = UploadChunkManifest( + size=3, chunks=[UploadChunk(index=0, off=0, len=3, chunk_sha256=sha256_hex(b"bad"))] + ) + before = list( + network.store._db.execute( + "SELECT id,data FROM records_v2 WHERE kind='relay-assignment' ORDER BY id" + ) + ) + response = network.client.post( + network.url + "/upload-grant", + params={"commit_hash": "00" * 32 if fault == "unknown-commit" else commit_hashes[1]} + if fault + in {"explicit-identical", "wrong-signer", "wrong-run", "unknown-commit", "round-mismatch"} + else {}, + json=relay_envelope.seal(HOT[0], "UploadChunkManifest", run_id, requested, 583), + ) + if fault not in {None, "explicit-identical"}: + assert response.status_code == 409, response.text + assert ( + network.store._db.execute( + "SELECT COUNT(*) FROM records_v2 WHERE kind='grant'" + ).fetchone()[0] + == 0 + ) + return + assert response.status_code == 200, response.text + grant1 = UploadGrant.model_validate(response.json()["body"]) + assert grant1.w == 1 and grant1.delta_hash == sha256_hex(payloads[1]) + assert grant1.exp_drand == 583 + old = network.client.post( + network.url + "/upload-grant", + params={"commit_hash": commit_hashes[0]} if fault == "explicit-identical" else {}, + json=relay_envelope.seal(HOT[0], "UploadChunkManifest", run_id, chunks[0], 552), + ) + assert old.status_code == 200, old.text + grant0 = UploadGrant.model_validate(old.json()["body"]) + assert grant0.w == 0 and grant0.exp_drand == 552 + assert [tuple(r) for r in before] == [ + tuple(r) + for r in network.store._db.execute( + "SELECT id,data FROM records_v2 WHERE kind='relay-assignment' ORDER BY id" + ) + ] + + async def old_custody(): + async with httpx.AsyncClient() as client: + + async def now(): + return network.now + + relay = Relay( + run_id=run_id, + master=COORD.ss58, + registry=registry, + network_manifest_hash=relay_net.digest(), + observers={a.ss58 for a in AUDITORS}, + relay_id="local", + region="local", + keys={"k1": RELAY}, + active_key="k1", + backing=StreamStore(LocalFSStore(tmp_path / "old-custody"), client), + now=now, + ) + opened = { + "w": 0, + **{ + k: "12" * 32 + for k in ( + "prev_final_hash", + "theta_hash", + "outer_state_hash", + "center_hash", + "roster_hash", + "honeypot_commit", + "start_state_index_hash", + ) + }, + "d_open": 1, + "d_assign": 2, + "d_commit": 400, + "d_audit": 401, + "d_upload": 552, + "d_final": 600, + "contract_version": 2, + "policy_hashes": { + k: getattr(network.manifest.network, k) + for k in ( + "admission_policy_hash", + "economics_policy_hash", + "aggregation_policy_hash", + "dispute_policy_hash", + "audit_policy_hash", + ) + }, + "registry_epoch": 0, + "audit_mode": "anchored-full", + "roster": [], + } + await relay.accept( + relay_envelope.parse_envelope(old.json()), + chunks[0], + relay_envelope.parse_envelope(assignments[0]), + envelope_v2.parse_envelope(network.signed(COORD, "RoundOpenV2", opened)), + ) + + async def source(): + yield payloads[0] + + await relay.chunk(grant0.digest(), 0, source()) + receipt = RelayReceipt.model_validate((await relay.complete(grant0.digest())).body) + request = RetrievalRequest( + request_id="ab" * 32, + nonce="ac" * 32, + relay_id="local", + key_id="k1", + assignment_hash=sha256_hex(canonicalize(assignments[0]["body"])), + grant_hash=grant0.digest(), + custody_ack_hashes=[], + receipt_hash=receipt.digest(), + retention_hash=receipt.digest(), + object_or_chunk_hash=grant0.delta_hash, + size=3, + requested_beacon=385, + deadline_beacon=395, + ) + served = await relay.retrieve( + relay_envelope.parse_envelope( + relay_envelope.seal(COORD, "RetrievalRequest", run_id, request, 395) + ) + ) + assert RetrievalResponse.model_validate(served.body).status == "SERVED" + with spool() as file: + await relay.backing.get(grant0.delta_hash, 3, file) + file.seek(0) + assert file.read() == payloads[0] + + anyio.run(old_custody) + + +def test_network_miner_upload_grant_sends_accepted_commit_body_subject( + network, monkeypatch, tmp_path +): + import json + from types import SimpleNamespace + from urllib.parse import parse_qs, urlsplit + + import httpx + + from hypertrain.miner.core import NetworkMiner + from hypertrain.protocol.envelope import body_digest + from hypertrain.protocol.messages_v2 import PolicyHashes, RoundOpenV2 + + job = fixture.stage(network.setup, tmp_path / "client-job", (0,), 0) + directory = tmp_path / "published" + (directory / "rank-0").mkdir(parents=True) + commitments = { + k: "01" * 32 for k in ("leaves_root", "final_theta_hash", "ef_out_hash", "delta_hash") + } + commitments["delta_hash"] = sha256_hex(b"new") + (directory / "rank-0/summary.json").write_bytes(canonicalize({"commitments": commitments})) + files = {} + for name, raw in ( + ("state", b"state"), + ("ef", b"ef"), + ("delta", b"new"), + ("leaves", canonicalize([{"loss_f32": "00000000", "norm_f32": "00000000"}])), + ): + files[name] = directory / name + files[name].write_bytes(raw) + import hypertrain.miner.island_launch as launch + + monkeypatch.setattr( + launch, "launch_island", lambda *a, **kw: SimpleNamespace(directory=directory, **files) + ) + opening = RoundOpenV2( + w=1, + **{ + k: "12" * 32 + for k in ( + "prev_final_hash", + "theta_hash", + "outer_state_hash", + "center_hash", + "roster_hash", + "honeypot_commit", + "start_state_index_hash", + ) + }, + d_open=380, + d_assign=385, + d_commit=413, + d_audit=414, + d_upload=583, + d_final=600, + contract_version=2, + registry_epoch=0, + audit_mode="anchored-full", + roster=[], + policy_hashes=PolicyHashes.model_validate( + {k: getattr(network.manifest.network, k) for k in PolicyHashes.model_fields} + ), + ) + saved = {} + + class CapturedSubject(Exception): + pass + + def call(method, path, **kwargs): + if path.endswith("/rounds/1"): + return { + "round_open": network.signed(COORD, "RoundOpenV2", opening), + "assignment": [{"hotkey": HOT[0].ss58, "assignment_hash": "11" * 32}], + "starts": [{"hotkey": HOT[0].ss58, "ef_hash": "22" * 32}], + } + if path.endswith("/job/" + HOT[0].ss58): + return {"job": job.body(), "objects": {}} + if path.endswith("/commit"): + saved["commit"] = kwargs["json"] + return {} + if path.endswith("/relay/" + HOT[0].ss58): + return {} + if urlsplit(path).path.endswith("/upload-grant"): + assert parse_qs(urlsplit(path).query)["commit_hash"] == [ + body_digest(saved["commit"]["body"]) + ] + assert body_digest(saved["commit"]["body"]) != body_digest(saved["commit"]) + assert body_digest(saved["commit"]["body"]) != commitments["delta_hash"] + raise CapturedSubject() + return {} + + # Use exact pinned policy object from registered authority, not fixture hash. + policy = network.store.objects.get(network.manifest.network.economics_policy_hash) + + def wire(request): + return httpx.Response(200, content=policy if request.method == "GET" else b"{}") + + actor = NetworkMiner.__new__(NetworkMiner) + actor.manifest, actor.kp, actor.run_id = network.manifest, HOT[0], network.manifest.run_id() + actor.url, actor.directory = network.url, tmp_path / "client" + actor.cfg = SimpleNamespace(device="cpu") + actor.status = lambda: {"work_screen_hash": "33" * 32, "record": {"admission_id": "44" * 32}} + with httpx.Client(transport=httpx.MockTransport(wire)) as client: + actor.api = SimpleNamespace(call=call, c=client, base="https://master.test") + with pytest.raises(CapturedSubject): + actor.run_round(1) + assert json.loads(canonicalize(saved["commit"]))["body"]["w"] == 1 diff --git a/tests/challenge/test_trust_v2.py b/tests/challenge/test_trust_v2.py new file mode 100644 index 0000000..c2f2e28 --- /dev/null +++ b/tests/challenge/test_trust_v2.py @@ -0,0 +1,253 @@ +from __future__ import annotations + +from dataclasses import replace + +import pytest + +from hypertrain.aggregator.core import FinalityError +from hypertrain.challenge.finality_v2 import ( + RoundStatus, + exclude, + finish_rollback, + require_apply, + require_open, + settle, +) +from hypertrain.challenge.trust_v2 import FundedStatus, audit_floor, nominal_q, select_audit +from hypertrain.protocol.example import example_manifest +from hypertrain.protocol.keys import Keypair +from hypertrain.protocol.messages import Finalize, Rollback, f32hex +from hypertrain.protocol.messages_v2 import EconomicsPolicyV2, RosterEntryV2 + +H = "11" * 32 +K = Keypair(bytes([1]) * 32).ss58 + + +def econ() -> EconomicsPolicyV2: + return EconomicsPolicyV2( + ledger_mode="test", + G_max_units=100, + R_collectible_units=100, + S_min_units=1000, + beta_ppm=1_000_000, + gammaV_units=0, + s_lower_ppm=1_000_000, + q_floor=1_000_000, + genesis_allocation_hash=H, + reward_authority=K, + round_reward_units=100, + max_total_issuance=100000, + shadow_bootstrap_rounds=12, + ) + + +def test_economic_floor_when_origin_backed_funding_sufficient() -> None: + # Given + r = RosterEntryV2( + hotkey=K, + slot=0, + q_i=f32hex(1), + admission_id=H, + coldkey_group="owner", + state="ACTIVE", + eligible_weight=1 << 22, + ) + f = FundedStatus(H, K, H, "test", 1000, 100, H, True) + # When + floor = audit_floor( + econ(), + example_manifest().verify, + r, + f, + run_id=H, + clean_finalizations=12, + reference_reward_units=100, + ) + # Then + assert floor.q_ppm == 1_000_000 + assert floor.required_locked_units == 1000 + assert floor.influence_allowed + + +@pytest.mark.parametrize( + "fault", + [ + "unfunded", + "unknown_bound", + "zero_detection", + "economic_infeasible", + "uncollectible_reward", + "shadow", + "premature_active", + ], +) +def test_influence_rejects_when_economic_or_admission_bound_missing(fault: str) -> None: + # Given + p = econ() + r = RosterEntryV2( + hotkey=K, + slot=0, + q_i=f32hex(1), + admission_id=H, + coldkey_group="owner", + state="ACTIVE", + eligible_weight=1 << 22, + ) + f = FundedStatus(H, K, H, "test", 1000, 100, H, True) + clean = 12 + match fault: + case "unfunded": + f = replace(f, locked_units=0) + case "unknown_bound": + f = replace(f, conservative_bound=False) + case "zero_detection": + p = p.model_copy(update={"s_lower_ppm": 0}) + case "economic_infeasible": + p = p.model_copy(update={"G_max_units": 1101}) + case "uncollectible_reward": + p = p.model_copy(update={"G_max_units": 1100}) + f = replace(f, collectible_reward_units=0) + case "shadow": + r = r.model_copy(update={"state": "PROBATION"}) + clean = 3 + case "premature_active": + clean = 11 + # When + floor = audit_floor( + p, + example_manifest().verify, + r, + f, + run_id=H, + clean_finalizations=clean, + reference_reward_units=100, + ) + # Then + assert not floor.influence_allowed + + +def test_ceiling_economics_when_fractional_penalty_conversion() -> None: + # Given + p = econ().model_copy( + update={"G_max_units": 101, "beta_ppm": 333_333, "s_lower_ppm": 500_000, "S_min_units": 0} + ) + r = RosterEntryV2( + hotkey=K, + slot=0, + q_i=f32hex(1), + admission_id=H, + coldkey_group="owner", + state="ACTIVE", + eligible_weight=1 << 22, + ) + f = FundedStatus(H, K, H, "test", 600, 0, H, True) + # When + floor = audit_floor( + p, + example_manifest().verify, + r, + f, + run_id=H, + clean_finalizations=12, + reference_reward_units=0, + ) + # Then: ceil(101/(.5*.333333)) =607, not606. + assert floor.required_locked_units == 607 + assert not floor.influence_allowed + + +@pytest.mark.parametrize( + "clean,expected", [(0, 1000000), (3, 1000000), (4, 500000), (7, 500000), (8, 250000)] +) +def test_nominal_schedule_when_clean_finalizations_advance(clean: int, expected: int) -> None: + # Given / When / Then + assert nominal_q("PROBATION", clean) == expected + + +@pytest.mark.parametrize("q,selected", [(0, False), (1000000, True)]) +def test_audit_selection_when_zero_or_one_probability(q: int, selected: bool) -> None: + # Given / When + selection = select_audit(H, 0, H, K, q) + # Then + assert selection.selected is selected + assert selection.final_transition is selected + assert selection.compression_and_ef is selected + assert selection.mode == ("anchored-full" if selected else None) + + +def status(w: int) -> RoundStatus: + return RoundStatus(H, w, True, "PENDING", 0, 1, False, H, None) + + +def test_third_round_blocked_when_first_unresolved() -> None: + # Given + history = [status(0), status(1)] + require_open(H, 1, history[:1]) + require_apply(H, 1, history[:1]) + # When / Then + with pytest.raises(FinalityError): + require_open(H, 2, history) + + +def test_open_resumes_when_authenticated_settlement_complete() -> None: + # Given + s = replace(status(0), unresolved_audits=0) + final = Finalize(w=0, final_theta_hash_w1=H, included=[K], entitlements_root=H) + s = settle(s, final, H) + # When / Then + require_open(H, 2, [s, status(1)]) + assert s.closed + + +def test_exclusion_waits_for_rollback_when_update_applied() -> None: + # Given + s = exclude(replace(status(0), unresolved_audits=0), H) + # When / Then + with pytest.raises(FinalityError): + require_open(H, 2, [s, status(1)]) + assert not s.closed + + +def test_open_resumes_when_complete_two_round_rollback_verified() -> None: + # Given + s = exclude(replace(status(0), unresolved_audits=0), H) + repair = Rollback( + w=0, + excluded=[K], + old_theta_hash_w2=H, + new_theta_hash_w2="22" * 32, + new_outer_state_hash="33" * 32, + recomputed=["agg_w", "step_w", "agg_w1", "step_w1"], + cause_hashes=[H], + ) + # When + s = finish_rollback(s, repair, "22" * 32, "33" * 32) + # Then + require_open(H, 2, [s, status(1)]) + assert s.closed + + +def test_finalized_model_history_preserved_when_late_fraud() -> None: + # Given + s = replace(status(0), disposition="SETTLED", unresolved_audits=0, resolution_hash=H) + # When / Then + with pytest.raises(FinalityError): + exclude(s, "22" * 32) + + +@pytest.mark.parametrize("fault", ["missing", "duplicate", "dispute", "partial_rollback"]) +def test_finality_fails_closed_when_incomplete_history_or_settlement(fault: str) -> None: + # Given + history = [replace(status(0), disposition="SETTLED", unresolved_audits=0), status(1)] + match fault: + case "missing": + history.pop(0) + case "duplicate": + history[1] = history[0] + case "dispute": + history[0] = replace(history[0], unresolved_disputes=1) + case "partial_rollback": + history[0] = replace(history[0], disposition="EXCLUDED", rollback_complete=False) + # When / Then + with pytest.raises(FinalityError): + require_apply(H, 2, history) diff --git a/tests/e2e/test_network_gpu_prepare.py b/tests/e2e/test_network_gpu_prepare.py new file mode 100644 index 0000000..7b5134a --- /dev/null +++ b/tests/e2e/test_network_gpu_prepare.py @@ -0,0 +1,299 @@ +"""Metadata-only frozen preparation; no trials, training, provider or CUDA.""" + +import hashlib +import io +import json +import os +import shutil +import sqlite3 +import stat +import subprocess +import sys +import tarfile +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] + + +@pytest.fixture(scope="module") +def freeze_path(tmp_path_factory): + """Public source metadata only; preparation creates its own signed CPU authority.""" + from hypertrain.gpu_ops import network_qualification + + snapshot = tmp_path_factory.mktemp("public-preparation-source") + tree = snapshot / "tree" + required = set(network_qualification.sources(ROOT)) | { + "scripts/network_gpu_operation.py", + "scripts/network_gpu_seed.py", + "scripts/network_service_proof.py", + "scripts/network_authority_snapshot.py", + } + for relative in sorted(required): + source, target = ROOT / relative, tree / relative + assert source.is_file() and not source.is_symlink() + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, target) + profile_path = tree / "experiments/gpu_network_v2/profile.json" + profile = json.loads(profile_path.read_bytes()) + profile["runtime"].update(image_digest="sha256:" + "ab" * 32, driver_allowlist=["595.84"]) + assert profile["runtime"]["qualified"] is False + profile_path.write_text(json.dumps(profile, sort_keys=True)) + + def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + def artifact(path): + return {"path": str(path), "bytes": path.stat().st_size, "sha256": sha(path)} + + canonical = network_qualification.sources(tree) + operation = { + **canonical, + "scripts/network_gpu_operation.py": sha(tree / "scripts/network_gpu_operation.py"), + } + staged = {relative: sha(tree / relative) for relative in sorted(required)} + maps = { + "qualification-sources.json": canonical, + "operation-sources.json": operation, + "sources.json": staged, + "files.json": { + relative: { + "origin": str(tree / relative), + "size": (tree / relative).stat().st_size, + "sha256": digest, + } + for relative, digest in staged.items() + }, + } + for name, values in maps.items(): + (snapshot / name).write_text(json.dumps(values, sort_keys=True)) + qualification = snapshot / "qualification-source.tar" + qualification.write_bytes(network_qualification.bundle(tree)) + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w") as archive: + for relative in sorted(staged): + data = (tree / relative).read_bytes() + member = tarfile.TarInfo(relative) + member.size, member.mode, member.mtime = len(data), 0o600, 0 + archive.addfile(member, io.BytesIO(data)) + effective = snapshot / "effective-source.tar" + effective.write_bytes(buffer.getvalue()) + freeze = { + "snapshot": str(snapshot), + "archive": artifact(effective), + "qualification_archive": artifact(qualification), + "maps": {name: artifact(snapshot / name) for name in maps}, + "approved_pins": { + name: staged[name] + for name in ( + "scripts/network_service_proof.py", + "scripts/network_gpu_operation.py", + "experiments/gpu_network_v2/orchestrate.py", + ) + }, + "service_source_observed": {}, + "qualification_parser": {}, + "image_candidate": {"ref": "metadata-only@" + profile["runtime"]["image_digest"]}, + "paid_GO": False, + "launchable_private_config": False, + } + path = snapshot / "freeze.json" + path.write_text(json.dumps(freeze, sort_keys=True)) + return path + + +def test_prepare_executable_frozen_artifacts_and_refusal(tmp_path, freeze_path): + output = tmp_path / "prepared" + command = [ + sys.executable, + str(ROOT / "scripts/network_gpu_prepare.py"), + "--freeze", + str(freeze_path), + "--output", + str(output), + "--cpu-genesis-unix", + "1700000000", + "--candidate-driver", + "580.95.05", + "--candidate-driver", + "580.159.03", + ] + result = subprocess.run( + command, + capture_output=True, + text=True, + timeout=60, + env=dict(os.environ, PYTHONDONTWRITEBYTECODE="1"), + ) + assert result.returncode == 0, result.stderr + prepared = json.loads(result.stdout) + assert prepared["status"] == "LOCAL_PREPARED_NOT_LAUNCHABLE" + assert prepared["candidate_input_scope"]["drivers"] == ["580.95.05", "580.159.03"] + manifest = json.loads((output / "public/seed/manifest.json").read_bytes()) + assert manifest["training"]["reference_spec"]["driver_allowlist"] == ["580.95.05", "580.159.03"] + original = json.loads(freeze_path.read_bytes()) + source_map = json.loads(Path(original["maps"]["sources.json"]["path"]).read_bytes()) + for relative, expected in source_map.items(): + path = output / "source-freeze/tree" / relative + if relative != "experiments/gpu_network_v2/profile.json": + assert hashlib.sha256(path.read_bytes()).hexdigest() == expected + assert ( + prepared["count"]["required_total"] == 126 and prepared["count"]["required_per_host"] == 63 + ) + assert set(prepared["remaining"].values()) == {"NOT_PRESENT"} + assert "missing_or_unqualified_exact_profile_runtime" in prepared["launch_refusal"] + assert prepared["controller_refusal"] == "reviewed_layout_clock_or_config" + assert prepared["kernels_executed"] == prepared["provider_calls"] == 0 + assert list((output / "production-service").iterdir()) == [] + for relative, digest in json.loads((output / "public/index.json").read_bytes()).items(): + assert hashlib.sha256((output / "public" / relative).read_bytes()).hexdigest() == digest + assert "private" not in relative and not relative.endswith((".key", ".seed", ".hex")) + for path in (output / "private").rglob("*"): + if path.is_file(): + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + with sqlite3.connect(output / "cpu-service/challenge.db") as db: + assert db.execute("SELECT COUNT(*) FROM runs").fetchone()[0] == 1 + assert db.execute("SELECT COUNT(*) FROM admission_trials").fetchone()[0] == 0 + assert db.execute("SELECT COUNT(*) FROM escrow_origins").fetchone()[0] == 4 + assert db.execute("SELECT bls FROM beacon").fetchone()[0] == 0 + assert len(json.loads((output / "public/membership.json").read_bytes())) == 4 + probe = """ +import ast,importlib.util,json,sys +from pathlib import Path +from types import SimpleNamespace +spec=importlib.util.spec_from_file_location('prepare',sys.argv[1]) +prepare=importlib.util.module_from_spec(spec);spec.loader.exec_module(prepare) +_,_,maps,seed,driver,controller=prepare.load_frozen(Path(sys.argv[2])) +output=Path(sys.argv[3]) +job=seed.validate(output/'public/seed',sys.argv[4]) +from hypertrain.protocol.keys import Keypair +from hypertrain.protocol import envelope_v2 +from hypertrain.protocol.jcs import canonicalize +from hypertrain.gpu_ops.launcher import Reject +owner=Keypair((output/'private/roles/owner.seed').read_bytes()) +hot=Keypair((output/'private/roles/hot-0.seed').read_bytes()) +subjects=[('source',maps['operation-sources.json']), + ('index',json.loads((output/'public/index.json').read_bytes())), + ('freeze',json.loads(Path(sys.argv[2]).read_bytes()))] +for name,subject in subjects: + raw=json.loads((output/'public'/f'{name}-receipt.json').read_bytes()) + assert envelope_v2.verify_envelope(raw) and raw['signer']==owner.ss58 and raw['run_id']==job.run_id + assert raw['body']['commit_hash']==driver.sha256_hex(canonicalize(subject)) +source=Path(sys.argv[2]).parent/'tree/src/hypertrain/gpu_ops/network_qualification.py' +node=next(n for n in ast.parse(source.read_text()).body + if isinstance(n,ast.FunctionDef) and n.name=='inspect_environment') +guard=next(n for n in node.body + if isinstance(n,ast.If) and 'qualification_driver_mismatch' in ast.unparse(n)) +code=compile(ast.fix_missing_locations(ast.Module(body=[guard],type_ignores=[])),str(source),'exec') +cfg=SimpleNamespace(driver_allowlist=['580.95.05','580.159.03']) +exec(code,{'cfg':cfg,'drivers':['580.95.05','580.159.03'],'Reject':Reject}) +for unexpected in ('595.71','610','595.71.05','610.57.04'): + try: + exec(code,{'cfg':cfg,'drivers':['580.95.05',unexpected],'Reject':Reject}) + except Reject as error: + assert str(error)=='qualification_driver_mismatch' + else: + raise AssertionError('unexpected driver accepted') +# Use the canonical owned-host consumer against an actually absent receipt; +# a prepared source bundle is not an observed allocation or GPU attestation. +runtime=controller.NetworkRuntime( + SimpleNamespace(receipt=lambda role:None,cleanup_started=lambda:False, + j=controller.Journal(output/'absent-owned-role-journal')),{} +) +factory=prepare.service_factory(driver,runtime,owner,job.run_id,maps['operation-sources.json'],{hot.ss58:{'role':'h0'}},lambda:1) +try: + factory('reference',job,output/'future-operation',{'run_id':job.run_id,'hotkey':hot.ss58}) +except Reject as error: + assert str(error)=='owned-role actual owned role observation absent' +else: + raise AssertionError('dynamic context invented an owned host') +print('DYNAMIC_CONTEXT_REFUSES_ABSENT_OWNED_ROLE') +""" + dynamic = subprocess.run( + [ + sys.executable, + "-c", + probe, + str(ROOT / "scripts/network_gpu_prepare.py"), + str(output / "source-freeze/freeze.json"), + str(output), + prepared["seed_index_sha256"], + ], + capture_output=True, + text=True, + timeout=30, + ) + assert dynamic.returncode == 0, dynamic.stderr + assert "DYNAMIC_CONTEXT_REFUSES_ABSENT_OWNED_ROLE" in dynamic.stdout + again = subprocess.run(command, capture_output=True, text=True, timeout=60) + assert again.returncode != 0 and "new output" in again.stderr + + +def test_freeze_changed_map_rejects_before_private_writes(tmp_path, freeze_path): + freeze = json.loads(freeze_path.read_bytes()) + freeze["maps"]["sources.json"]["sha256"] = "0" * 64 + altered = tmp_path / "freeze.json" + altered.write_text(json.dumps(freeze)) + output = tmp_path / "prepared" + result = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts/network_gpu_prepare.py"), + "--freeze", + str(altered), + "--output", + str(output), + "--cpu-genesis-unix", + "1700000000", + ], + capture_output=True, + text=True, + timeout=10, + ) + assert result.returncode != 0 and "frozen archive/map changed" in result.stderr + assert not output.exists() + + +def test_parser_requires_explicit_cpu_clock(capsys): + import importlib.util + + spec = importlib.util.spec_from_file_location( + "prepare_parser", ROOT / "scripts/network_gpu_prepare.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + old = sys.argv + sys.argv = ["network_gpu_prepare.py", "--freeze", "unused-freeze.json", "--output", "unused"] + try: + with pytest.raises(SystemExit) as error: + module.main() + assert error.value.code == 2 + finally: + sys.argv = old + + +@pytest.mark.parametrize("unexpected", ["595.71", "610", "595.71.05", "610.57.04"]) +def test_unapproved_candidate_rejects_before_artifacts(tmp_path, freeze_path, unexpected): + output = tmp_path / "prepared" + result = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts/network_gpu_prepare.py"), + "--freeze", + str(freeze_path), + "--output", + str(output), + "--cpu-genesis-unix", + "1700000000", + "--candidate-driver", + "580.95.05", + "--candidate-driver", + unexpected, + ], + capture_output=True, + text=True, + timeout=10, + ) + assert result.returncode != 0 and "exact approved ordered pair" in result.stderr + assert not output.exists() diff --git a/tests/e2e/test_network_service_proof.py b/tests/e2e/test_network_service_proof.py new file mode 100644 index 0000000..be00805 --- /dev/null +++ b/tests/e2e/test_network_service_proof.py @@ -0,0 +1,1612 @@ +"""Small D3 dispatch/authority/cancellation contracts; no graduation or model training.""" + +from __future__ import annotations + +import contextlib +import ctypes +import importlib.util +import json +import os +import shutil +import socket +import sqlite3 +import ssl +import subprocess +import sys +import threading +from pathlib import Path + +import certifi +import pytest +from pydantic import Field, ValidationError, create_model + +SCRIPTS = Path(__file__).parents[2] / "scripts" +sys.path.insert(0, str(SCRIPTS)) +spec = importlib.util.spec_from_file_location( + "service_proof_driver", SCRIPTS / "network_service_proof.py" +) +assert spec is not None and spec.loader is not None +driver = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = driver +spec.loader.exec_module(driver) + + +@pytest.mark.parametrize("present", [False, True]) +@pytest.mark.parametrize("raises", [False, True]) +def test_cli_ca_environment_restores_native_presence_and_value(monkeypatch, present, raises): + """Tiny native environment restoration check; no signed package or CLI graph.""" + original = os.environ + if present: + monkeypatch.setenv("SSL_CERT_FILE", certifi.where()) + else: + monkeypatch.delenv("SSL_CERT_FILE", raising=False) + before = os.environ.get("SSL_CERT_FILE") + getenv = ctypes.CDLL(None).getenv + getenv.argtypes = [ctypes.c_char_p] + getenv.restype = ctypes.c_char_p + expected = ( + pytest.raises(RuntimeError, match="CA exception control") + if raises + else contextlib.nullcontext() + ) + with expected: + with monkeypatch.context() as patch: + patch.setenv("SSL_CERT_FILE", certifi.where()) + os.environ["SSL_CERT_FILE"] = "temporary-operation-ca.pem" + assert getenv(b"SSL_CERT_FILE") == b"temporary-operation-ca.pem" + if raises: + raise RuntimeError("CA exception control") + assert os.environ is original and os.environ.get("SSL_CERT_FILE") == before + assert getenv(b"SSL_CERT_FILE") == (None if before is None else os.fsencode(before)) + + +@pytest.fixture(scope="module") +def cli_ca(tmp_path_factory): + """Real temporary CLI trust anchor; independent of metadata HTTP transport.""" + directory = tmp_path_factory.mktemp("driver-cli-ca") + cert, key = directory / "ca.pem", directory / "ca.key" + subprocess.run( + [ + "openssl", + "req", + "-x509", + "-newkey", + "ec", + "-pkeyopt", + "ec_paramgen_curve:P-256", + "-nodes", + "-days", + "1", + "-keyout", + str(key), + "-out", + str(cert), + "-subj", + "/CN=master.test", + "-addext", + "subjectAltName=DNS:master.test", + ], + check=True, + capture_output=True, + ) + return cert, key + + +def verify_cli_ca(context, cert, key): + """Bounded real TLS handshake in memory; wrong CA must fail verification.""" + assert context.verify_mode == ssl.CERT_REQUIRED and context.check_hostname + server = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server.load_cert_chain(cert, key) + incoming = [ssl.MemoryBIO(), ssl.MemoryBIO()] + outgoing = [ssl.MemoryBIO(), ssl.MemoryBIO()] + peers = [ + context.wrap_bio(incoming[0], outgoing[0], server_hostname="master.test"), + server.wrap_bio(incoming[1], outgoing[1], server_side=True), + ] + completed = [False, False] + for _ in range(10): + for i, peer in enumerate(peers): + if not completed[i]: + try: + peer.do_handshake() + completed[i] = True + except ssl.SSLWantReadError: + pass + incoming[1 - i].write(outgoing[i].read()) + if all(completed): + return + pytest.fail("TLS handshake did not complete within bounded exchanges") + + +fixture_spec = importlib.util.spec_from_file_location( + "driver_hermetic_authority", SCRIPTS.parent / "tests/challenge/authority_fixture_v2.py" +) +assert fixture_spec is not None and fixture_spec.loader is not None +authority_fixture = importlib.util.module_from_spec(fixture_spec) +sys.modules[fixture_spec.name] = authority_fixture +fixture_spec.loader.exec_module(authority_fixture) + + +@pytest.fixture(scope="module") +def signed_package(tmp_path_factory): + """Real signed intake/escrow predicates, explicit metadata-only execution oracle.""" + root = tmp_path_factory.mktemp("driver-signed-metadata") + source, directory = root / "source", root / "bundle" + run_id, originals = authority_fixture.build_authority(source) + with pytest.MonkeyPatch.context() as patch: + patch.setattr(driver.authority, "RUN", run_id) + patch.setattr( + driver.authority, + "Bundle", + create_model( + "DriverHermeticBundle", + __base__=driver.authority.Bundle, + run_id=(str, Field(default=run_id, pattern="^" + run_id + "$")), + ), + ) + patch.setattr(driver.authority, "validate_artifacts", authority_fixture.metadata_artifacts) + digest = driver.authority.export(source, directory) + bundle = driver.authority.validate(directory, digest) + secrets = root / "roles" + secrets.mkdir(mode=0o700) + for role, public in bundle.public_roles.items(): + path = secrets / f"{role}.seed" + path.write_bytes(originals[public]) + path.chmod(0o600) + token = secrets / "admin.token" + token.write_text("hermetic-driver-token") + token.chmod(0o600) + template = root / "service" + with driver.authority.bootstrap( + directory, digest, template, secrets, "https://master.test" + ): + offset = float(os.environ["HT_FIXTURE_CLOCK_OFFSET"]) + from hypertrain.beacon.core import FixtureBeacon + + beacon = FixtureBeacon(current=bundle.beacon_round).get(bundle.beacon_round) + payload = { + "round": beacon.round, + "signature": beacon.signature, + "randomness": beacon.randomness, + } + (template / "runtime.json").write_text( + json.dumps( + { + "fixture_clock_offset": offset, + "beacon": payload, + } + ) + ) + yield directory, digest, secrets, template, offset, payload + + +def settings(tmp_path): + return driver.Settings( + master_https="https://master.test", + relay_https="https://relay.test", + ca=tmp_path / "ca.pem", + snapshot=tmp_path / "snapshot", + snapshot_digest="a" * 64, + secrets=tmp_path / "keys", + service_state=tmp_path / "service", + evidence=tmp_path / "proof", + verified_beacon=tmp_path / "beacon.json", + fixture_clock_directory=tmp_path / "clock", + fixture_clock_offset=0, + ) + + +def test_decoder_count_charges_full_live_geometry_not_extra_qualification(): + profile = json.loads((SCRIPTS.parent / "experiments/gpu_network_v2/profile.json").read_bytes()) + count = driver.decoder_execution_count(profile) + assert count["shadow"] == 96 and count["live"] == 16 + assert count["qualification"] == 4 and count["faulttrace_remaining"] == 10 + assert count["required_total"] == 126 and count["required_per_host"] == 63 + assert count["excess"] == 0 and count["kernels_executed"] == 0 + + +def test_decoder_count_cli_blocks_before_authority_or_provider(tmp_path, monkeypatch, capsys): + profile = SCRIPTS.parent / "experiments/gpu_network_v2/profile.json" + + def forbidden(*args, **kwargs): + raise AssertionError("workload launch before budget reconciliation") + + monkeypatch.setattr(driver, "proof", forbidden) + monkeypatch.setattr(driver.subprocess, "Popen", forbidden) + assert driver.main(["--decoder-count-profile", str(profile)]) == 0 + result = json.loads(capsys.readouterr().out) + assert result["reason"] == "D2_EXECUTION_COUNT_ONLY" and result["backend_authority"] == "NONE" + assert not list(tmp_path.iterdir()) + + +@pytest.mark.parametrize( + "section, field, value", + [("inner", "H", 2), ("schedule", "shadow_participations_per_identity", 11)], +) +def test_decoder_count_rejects_reduced_work(section, field, value): + profile = json.loads((SCRIPTS.parent / "experiments/gpu_network_v2/profile.json").read_bytes()) + profile[section][field] = value + with pytest.raises(driver.DriverError, match="original execution contract"): + driver.decoder_execution_count(profile) + + +def test_exact_decoder_four_identity_legacy_bootstrap_without_capacity_receipt(tmp_path): + """Real signed registration/genesis/joins/locks, no screen/training/CUDA claim.""" + from fastapi.testclient import TestClient + + from hypertrain.challenge.app import Config, create_app + from hypertrain.challenge.store import ChallengeError + from hypertrain.data.store import LocalFSStore + from hypertrain.ledger import Params + from hypertrain.protocol import envelope_v2 + from hypertrain.protocol.jcs import canonicalize + from hypertrain.protocol.keys import Keypair + + seed_spec = importlib.util.spec_from_file_location( + "driver_decoder_seed", SCRIPTS / "network_gpu_seed.py" + ) + assert seed_spec is not None and seed_spec.loader is not None + seed = importlib.util.module_from_spec(seed_spec) + sys.modules[seed_spec.name] = seed + seed_spec.loader.exec_module(seed) + public, private = tmp_path / "seed", tmp_path / "secrets" + digest = seed.generate( + public, private, "sha256:" + "ab" * 32, ["candidate-not-observed"], 1000, 2000 + ) + job = seed.validate(public, digest) + index = json.loads((public / "seed.json").read_bytes()) + token = private / "admin.token" + token.write_text("decoder-contract-test") + token.chmod(0o600) + # Existing loader accepts hex; raw generated seed begins with whitespace byte0x0b. + coordinator = private / "service-coord.hex" + coordinator.write_text((private / "coord.seed").read_bytes().hex()) + coordinator.chmod(0o600) + config = Config( + "hypertrain", + tmp_path / "service", + "https://master.test", + None, + token, + None, + coordinator, + index["roles"]["owner"], + Params("hypertrain", 1000, 4320, 1, job.manifest.training.verify.E_vest_rounds), + ) + app = create_app( + config, + objects=LocalFSStore(public / "objects"), + verify_beacon=driver.authority.fixture_beacon, + ) + store = app.state.store + store.clock = lambda: 1000.0 + try: + with TestClient(app) as client: + headers = {"Authorization": "Bearer decoder-contract-test"} + from hypertrain.beacon.core import FixtureBeacon + + b = FixtureBeacon(current=1).get(1) + response = client.post( + "/v1/admin/beacon", + headers=headers, + json={"round": 1, "signature": b.signature, "randomness": b.randomness}, + ) + assert response.status_code == 200 + response = client.post( + "/v2/admin/runs", + headers=headers, + content=(public / "manifest-envelope.json").read_bytes(), + ) + assert response.status_code == 201 and response.json()["run_id"] == job.run_id + root = f"/v2/runs/{job.run_id}" + response = client.post( + root + "/admin/genesis", + headers=headers, + content=(public / "genesis-envelope.json").read_bytes(), + ) + assert response.status_code == 200 + genesis = driver.authority.TestGenesis.model_validate( + envelope_v2.parse_envelope((public / "genesis-envelope.json").read_bytes()).body + ) + roster = [] + for i in range(4): + hot = Keypair((private / f"hot-{i}.seed").read_bytes()) + cold = Keypair((private / f"cold-{i}.seed").read_bytes()) + admission_id = driver.decoder_join_lock( + client, + job.manifest, + hot, + cold, + [o.origin_id for o in genesis.origins if o.owner == cold.ss58], + 1000, + f"{i + 1:064x}", + f"{i + 10:064x}", + 100, + ) + status = client.get(root + "/admission/" + hot.ss58) + assert status.status_code == 200 + assert status.json()["record"]["clean_count"] == 0 + assert not status.json()["eligible"] + assert status.json()["funding"]["locked_units"] == 1000 + roster.append({"hotkey": hot.ss58, "admission_id": admission_id}) + assert store.require_roster_v2(job.run_id, roster=roster) is None + owner = Keypair((private / "owner.seed").read_bytes()) + body = canonicalize( + { + "version": 1, + "profile_id": "tiny-cpu-service-32-v1", + "run_id": job.run_id, + "backend_binding_hash": "0" * 64, + "implementation_hash": "0" * 64, + "max_complete_roster": 32, + "memory_reservation_bytes": 1 << 30, + "max_outer_work_units": 50_000_000, + "max_audit_step_units": 512, + "max_repair_step_units": 1024, + "max_object_bytes": 65536, + "max_tape_bytes": 1 << 20, + } + ) + receipt = canonicalize( + envelope_v2.seal( + owner, + "Receipt", + job.run_id, + {"w": 0, "commit_hash": driver.sha256_hex(body), "received_round": 1}, + 10000, + ) + ) + with pytest.raises(ChallengeError) as error: + store.bootstrap_service_capacity_v2(job.run_id, body, receipt) + assert error.value.status == 409 and error.value.detail == "SERVICE_PROFILE_GEOMETRY" + assert store._backend_v2(job.manifest) == "cpu" + assert store._db.execute("SELECT COUNT(*) FROM admissions_v2").fetchone()[0] == 4 + assert store._db.execute("SELECT COUNT(*) FROM admission_trials").fetchone()[0] == 0 + escrow, _, _ = store._services(job.run_id) + escrow.verify() + assert not store._db.execute( + "SELECT 1 FROM records_v2 WHERE kind='service-admission'" + ).fetchone() + finally: + store.close_v2_notifications() + store._db.close() + + +def test_decoder_operation_cancel_rejects_before_http(tmp_path): + from types import SimpleNamespace + + seed_spec = importlib.util.spec_from_file_location( + "driver_cancel_seed", SCRIPTS / "network_gpu_seed.py" + ) + assert seed_spec is not None and seed_spec.loader is not None + seed = importlib.util.module_from_spec(seed_spec) + seed_spec.loader.exec_module(seed) + from hypertrain.protocol.keys import Keypair + + public, private = tmp_path / "seed", tmp_path / "roles" + digest = seed.generate(public, private, "sha256:" + "ab" * 32, ["candidate"], 1000, 2000) + job = seed.validate(public, digest) + index = json.loads((public / "seed.json").read_bytes()) + hot = Keypair((private / "hot-0.seed").read_bytes()) + operation = { + "operation": "probe", + "binding": "11" * 32, + "hotkey": hot.ss58, + "owner": index["roles"]["owner"], + "role": "h0", + "instance_id": 1, + "machine_id": 2, + "job": job.model_dump(mode="json"), + "binding_kind": "trial", + "sources": {}, + "image_digest": job.manifest.training.reference_spec.image_digest, + "cutoff": job.deadline, + } + cancel = threading.Event() + cancel.set() + with pytest.raises(driver.DriverError, match="identity/kind/cancellation"): + driver.decoder_operation( + SimpleNamespace(), SimpleNamespace(), operation, {}, hot, {}, tmp_path, cancel + ) + + +@pytest.mark.parametrize("fault", ["trial", "signer", "identity"]) +def test_decoder_runtime_consumer_rejects_wrong_trial_without_dispatch(tmp_path, fault): + from types import SimpleNamespace + + import httpx + + from hypertrain.protocol import envelope_v2 + from hypertrain.protocol.keys import Keypair + from hypertrain.protocol.messages_v2 import ArtifactLimits, JoinChallenge + + seed_spec = importlib.util.spec_from_file_location( + "driver_bound_seed", SCRIPTS / "network_gpu_seed.py" + ) + assert seed_spec is not None and seed_spec.loader is not None + seed = importlib.util.module_from_spec(seed_spec) + seed_spec.loader.exec_module(seed) + public, private = tmp_path / "seed", tmp_path / "roles" + digest = seed.generate(public, private, "sha256:" + "ab" * 32, ["candidate"], 1000, 2000) + job = seed.validate(public, digest) + index = json.loads((public / "seed.json").read_bytes()) + hot = Keypair((private / "hot-0.seed").read_bytes()) + coord = Keypair((private / "coord.seed").read_bytes()) + challenge = JoinChallenge( + admission_id="22" * 32, + nonce="33" * 32, + seed_beacon=1, + deadline_beacon=101, + manifest_hash=job.run_id, + theta_hash="44" * 32, + assignment_hash="55" * 32, + layout=job.manifest.training.reference_spec.layout, + artifact_limits=ArtifactLimits( + max_object_bytes=65536, max_body_bytes=65536, max_chunk_manifest_bytes=1048576 + ), + policy_hash=job.manifest.network.admission_policy_hash, + ) + operation = { + "operation": "probe", + "binding": "11" * 32, + "hotkey": hot.ss58, + "owner": index["roles"]["owner"], + "role": "h0", + "instance_id": 1, + "machine_id": 2, + "job": job.model_dump(mode="json"), + "binding_kind": "trial", + "sources": {}, + "image_digest": job.manifest.training.reference_spec.image_digest, + "cutoff": job.deadline, + } + if fault == "signer": + challenge = challenge.model_copy(update={"admission_id": "11" * 32}) + coord = hot + if fault == "identity": + operation["hotkey"] = coord.ss58 + calls = [] + + def respond(request): + calls.append(request.url.path) + return httpx.Response( + 200, json=envelope_v2.seal(coord, "JoinChallenge", job.run_id, challenge, 101) + ) + + with httpx.Client( + base_url="https://master.test", transport=httpx.MockTransport(respond) + ) as client: + with pytest.raises(driver.DriverError): + driver.decoder_operation( + client, + SimpleNamespace(), + operation, + {}, + hot, + {"owner": index["roles"]["owner"], "run_id": job.run_id}, + tmp_path, + threading.Event(), + ) + assert calls == ( + [] if fault == "identity" else [f"/v2/runs/{job.run_id}/join/" + "11" * 32 + "/challenge"] + ) + + +def test_decoder_probe_consumer_real_routes_receipt_and_debit_metadata(tmp_path, monkeypatch): + """One real reference/proof transition; metadata oracle, not independent gradients.""" + from types import SimpleNamespace + + import hypertrain.challenge.admission as admission_module + import network_gpu_operation as operation_module + from hypertrain.data.trial_assignment import trial_samples + from hypertrain.gpu_ops.journal import Journal, sha256 + from hypertrain.protocol import envelope_v2 + from hypertrain.protocol.jcs import canonicalize + + runtime_spec = importlib.util.spec_from_file_location( + "driver_real_runtime", SCRIPTS.parent / "experiments/gpu_network_v2/orchestrate.py" + ) + assert runtime_spec is not None and runtime_spec.loader is not None + runtime_module = importlib.util.module_from_spec(runtime_spec) + runtime_spec.loader.exec_module(runtime_module) + network_module = authority_fixture.service + monkeypatch.setattr(network_module.time, "time", lambda: 1700000000.0) + original_setup = network_module.fixture.Setup + + def carry(manifest, policy, admission_policy, rows, tree): + body = manifest.body() + body["training"]["inner"].update(state_policy="carry", rewarmup_steps=0) + return original_setup( + driver.RunManifestV2.model_validate(body), policy, admission_policy, rows, tree + ) + + monkeypatch.setattr(network_module.fixture, "Setup", carry) + (tmp_path / "network").mkdir() + generator = network_module.network.__wrapped__( + tmp_path / "network", SimpleNamespace(param="mlm") + ) + n = next(generator) + try: + hot, cold = network_module.HOT[0], network_module.COLD[0] + origins = [n.origin_ids[0]] + admission_id = driver.decoder_join_lock( + n.client, n.manifest, hot, cold, origins, 1000, "61" * 32, "62" * 32, 10000 + ) + n.push(2) + signed = n.client.get(n.url + "/join/" + admission_id + "/challenge").json() + challenge = driver.JoinChallenge.model_validate(envelope_v2.parse_envelope(signed).body) + with n.store._lock: + epoch = n.store._db.execute( + "SELECT trial_epoch FROM admissions_v2 WHERE admission_id=?", (admission_id,) + ).fetchone()[0] + job, inputs = n.store.stage_trial_v2( + n.manifest.run_id(), + challenge, + trial_samples( + n.manifest, + admission_id, + challenge.nonce, + n.store._beacon_v2(challenge.seed_beacon), + ), + epoch, + ) + profile = json.loads( + (SCRIPTS.parent / "experiments/gpu_network_v2/profile.json").read_bytes() + ) + profile["layout"] = job.manifest.training.reference_spec.layout.model_dump() + profile["runtime"].update( + qualified=True, image_digest=job.manifest.training.reference_spec.image_digest + ) + journal_dir = tmp_path / "journal" + journal_dir.mkdir() + journal = Journal(journal_dir) + accepted_manifest = { + "role": "h0", + "instance_id": 1, + "machine_id": 2, + "image_digest": job.manifest.training.reference_spec.image_digest, + } + lifecycle = SimpleNamespace( + j=journal, + dir=journal_dir, + roles=["h0", "h1"], + cleanup_started=lambda: False, + deadline=lambda: job.deadline + 100, + frozen=lambda role: accepted_manifest, + cfg={"cleanup_deadline_unix": job.deadline + 1000}, + receipt=lambda role: {"instance_id": 1}, + host={"h0": {"machine_id": 2}}, + ) + runtime = runtime_module.NetworkRuntime(lifecycle, profile) + runtime.reserve("h0", "metadata-qual-h0", "qualification", 2) + runtime.reserve("h1", "metadata-qual-h1", "qualification", 2) + journal.append("network_workload_promoted", metadata_only=True) + journal.append( + "network_admission_authority", owner=network_module.OWNER.ss58, metadata_only=True + ) + journal.append("supervisor_ready", metadata_only=True) + journal.append("ssh_trusted", role="h0", metadata_only=True) + op = operation_module.Operation( + operation="probe", + binding=admission_id, + hotkey=hot.ss58, + owner=network_module.OWNER.ss58, + role="h0", + instance_id=1, + machine_id=2, + job=job, + binding_kind="trial", + sources={}, + image_digest=job.manifest.training.reference_spec.image_digest, + cutoff=job.deadline, + trace=False, + ) + context = { + "role": op.role, + "binding": op.binding, + "operation": op.operation, + "hotkey": op.hotkey, + "spec_sha256": sha256(op.model_dump_json().encode()), + } + receipt = envelope_v2.seal( + network_module.OWNER, + "Receipt", + job.run_id, + { + "w": job.w, + "commit_hash": sha256(canonicalize(context)), + "received_round": n.now, + }, + 10000, + ) + runtime.accept_operation( + op, canonicalize(receipt), owner=network_module.OWNER.ss58, beacon=n.now + ) + from hypertrain.gpu_ops.launcher import Reject + + with pytest.raises(Reject, match="source_or_run_changed"): + runtime.operation(op, tmp_path / "refused", cancel=threading.Event()) + assert not journal.all("network_execution_intent", phase="workload") + cancelled = threading.Event() + cancelled.set() + with pytest.raises(Reject, match="context_not_qualified"): + runtime.operation(op, tmp_path / "refused", cancel=cancelled) + screen_original = authority_fixture._screen + + def reference(job, directory, challenge, **kwargs): + runtime.reserve("h0", "reference-" + job.digest(), "workload", 1) + return screen_original(job, directory, challenge, **kwargs) + + monkeypatch.setattr(admission_module, "screen_work", reference) + + def dispatch(operation, destination, *, cancel): + runtime.reserve(op.role, "probe-" + job.digest(), "workload", 1) + import shutil + + miner_inputs = tmp_path / "miner-inputs" + shutil.copytree(inputs, miner_inputs, ignore=shutil.ignore_patterns("published")) + screen, proof = screen_original( + job, miner_inputs, challenge, now_beacon=n.now, backend="cpu" + ) + return { + "status": "CAPTURED_NOT_ACCEPTED", + "cli_outcome": { + "proof": envelope_v2.seal( + hot, "WorkProof", job.run_id, proof, challenge.deadline_beacon + ), + "screen": envelope_v2.seal( + hot, "WorkScreenV2", job.run_id, screen, challenge.deadline_beacon + ), + }, + } + + # Exact constructor-only factory contract; launch result remains independently validated. + factory = driver.decoder_service_factory( + runtime, + lambda operation, j, d, c: ( + op.model_copy(update={"operation": operation}).model_dump(mode="json"), + receipt, + {"owner": network_module.OWNER.ss58, "beacon": n.now}, + ), + ) + from hypertrain.gpu_ops.launcher import Reject + + with pytest.raises(Reject, match="receipt_changed"): + factory("reference", job, inputs, {"challenge": signed}) + assembled = op.model_copy(update={"operation": "reference"}) + context_body = { + "role": assembled.role, + "binding": assembled.binding, + "operation": assembled.operation, + "hotkey": assembled.hotkey, + "spec_sha256": sha256(assembled.model_dump_json().encode()), + } + original_receipt = envelope_v2.seal( + network_module.OWNER, + "Receipt", + job.run_id, + { + "w": job.w, + "commit_hash": sha256(canonicalize(context_body)), + "received_round": n.now, + }, + 10000, + ) + plan = driver.decoder_context_plan( + runtime, + network_module.OWNER.ss58, + job.run_id, + {}, + {hot.ss58: {"role": "h0"}}, + {"reference:" + job.digest(): original_receipt}, + n.now, + ) + context = { + "run_id": job.run_id, + "hotkey": hot.ss58, + "admission_id": admission_id, + "epoch": job.w, + "challenge": challenge.body(), + "challenge_hash": challenge.digest(), + "challenge_envelope": signed, + } + raw, kept, pins = plan("reference", job, inputs, context) + assert raw == assembled.model_dump(mode="json") and kept == original_receipt + driver.decoder_service_factory(runtime, plan)("reference", job, inputs, context) + with pytest.raises(driver.DriverError, match="run differs"): + plan("reference", job, inputs, {**context, "run_id": "00" * 32}) + monkeypatch.setattr(runtime, "operation", dispatch) + result = driver.decoder_operation( + n.client, + runtime, + op.model_dump(mode="json"), + receipt, + hot, + { + "beacon": n.now, + "owner": network_module.OWNER.ss58, + "run_id": job.run_id, + "admin_headers": network_module.admin(), + }, + tmp_path / "artifacts", + threading.Event(), + ) + assert result["state"] == "PROBATION" + assert n.store._db.execute( + "SELECT proof_digest FROM admissions_v2 WHERE admission_id=?", (admission_id,) + ).fetchone()[0] + assert ( + sum(r["executions"] for r in journal.all("network_execution_intent", phase="workload")) + == 2 + ) + assert not n.store._db.execute( + "SELECT 1 FROM admission_trial_results WHERE outcome='MATCH'" + ).fetchone() + escrow, _, _ = n.store._services(job.run_id) + escrow.verify() + finally: + generator.close() + n.store._db.close() + + +@pytest.mark.parametrize( + "fault", + [ + None, + "missing-status", + "forged-status", + "wrong-round", + "missing-acceptance", + "wrong-signer", + "wrong-challenge", + "wrong-commit", + "wrong-run", + ], +) +def test_decoder_live_actual_cli_wrapper_shape_and_consumer(tmp_path, monkeypatch, fault, cli_ca): + """Actual cli.main plus remote wrapper; training/network producer replaced by metadata.""" + from types import SimpleNamespace + + import httpx + + import network_gpu_operation as operation + from hypertrain.miner import cli + from hypertrain.protocol import envelope_v2 + from hypertrain.protocol.keys import Keypair + + seed_spec = importlib.util.spec_from_file_location( + "driver_live_seed", SCRIPTS / "network_gpu_seed.py" + ) + assert seed_spec is not None and seed_spec.loader is not None + seed = importlib.util.module_from_spec(seed_spec) + seed_spec.loader.exec_module(seed) + public, private = tmp_path / "seed", tmp_path / "roles" + digest = seed.generate(public, private, "sha256:" + "ab" * 32, ["candidate"], 1000, 2000) + job = seed.validate(public, digest) + index = json.loads((public / "seed.json").read_bytes()) + hot = Keypair((private / "hot-0.seed").read_bytes()) + directory = tmp_path / "operation" + directory.mkdir() + for relative in job.object_paths.values(): + shutil.copyfile(public / relative, directory / relative) + spec = operation.Operation( + operation="live", + binding="33" * 32, + hotkey=hot.ss58, + owner=index["roles"]["owner"], + role="h0", + instance_id=1, + machine_id=2, + job=job, + binding_kind="commit-lease", + sources={}, + image_digest=job.manifest.training.reference_spec.image_digest, + cutoff=job.deadline, + miner_config="miner.toml", + ca="ca.pem", + master="https://master.test", + ) + common = {"w": job.w, "hotkey": hot.ss58} + bodies = { + "AcceptV2": { + **common, + "assignment_hash": "11" * 32, + "image_digest": spec.image_digest, + "driver_version": "candidate", + "n_gpus": 2, + "work_screen_hash": "12" * 32, + }, + "CommitV2": { + **common, + "leaf_scheme": "ht-leaf-v1", + "n_leaves": 7, + **{ + k: "13" * 32 + for k in ( + "leaves_root", + "metrics_root", + "final_theta_hash", + "ef_in_hash", + "ef_out_hash", + "delta_hash", + ) + }, + "delta_bytes": 1, + "tokens": job.manifest.training.batch_samples() * job.manifest.training.model.seq_len, + }, + "DeltaManifestV2": { + **common, + "delta_hash": "13" * 32, + "uri": "sha256:" + "13" * 32, + "size": 1, + "format": "ht-sparse-v1", + "chunks": [{"off": 0, "len": 1, "sha256": "14" * 32}], + "grant_hash": "15" * 32, + "master_acceptance_hash": "16" * 32, + }, + } + bodies["WorkProof"] = { + "admission_id": "17" * 32, + "challenge_hash": driver.sha256_hex(driver.canonicalize(bodies["CommitV2"])), + "leaves_root": bodies["CommitV2"]["leaves_root"], + "delta_hash": bodies["CommitV2"]["delta_hash"], + "artifact_refs": [{"sha256": "18" * 32, "size": 1}], + } + kinds = ["AcceptV2", "CommitV2", "WorkProof", "DeltaManifestV2"] + signed = [envelope_v2.seal(hot, kind, job.run_id, bodies[kind], 10000) for kind in kinds] + assert "w" not in signed[2]["body"] and "hotkey" not in signed[2]["body"] + monkeypatch.setattr(operation, "check", lambda *args: None) + monkeypatch.setattr(operation, "confined", lambda tree, relative: tree / relative) + monkeypatch.setattr(operation.time, "time", lambda: 1001) + monkeypatch.setattr(operation.signal, "signal", lambda *args: None) + cfg = SimpleNamespace( + workdir=directory / "miner", + keyfile=private / "hot-0.seed", + run_id=job.run_id, + owner_hotkey=spec.owner, + api=spec.master, + device="cuda", + image_digest=spec.image_digest, + ) + monkeypatch.setattr(cli.MinerConfig, "load", lambda *args, **kwargs: cfg) + import hypertrain.miner.core as core + + transport_original = httpx.Client + responses = [] + + def respond(request): + responses.append(request.url.path) + return httpx.Response(200, json={"w": job.w, "hotkey": hot.ss58, "accepted": True}) + + class MetadataTransport(transport_original): + def __init__(self, *args, **kwargs): + super().__init__( + *args, + base_url="https://master.test", + transport=httpx.MockTransport(respond), + **kwargs, + ) + + monkeypatch.setattr(httpx, "Client", MetadataTransport) + + class MetadataMiner: + def __init__(self, config, client): + self.manifest = job.manifest + self.kp = hot + self.api = SimpleNamespace(c=client) + + def __enter__(self): + return self + + def __exit__(self, *args): + self.api.c.close() + + def run_round(self, w): + assert w == job.w + for value in signed: + self.api.c.post( + "/v2/runs/" + job.run_id + "/train", content=driver.canonicalize(value) + ).raise_for_status() + return "UPLOADED" + + monkeypatch.setattr(core, "NetworkMiner", MetadataMiner) + monkeypatch.setattr(cli, "NetworkMiner", MetadataMiner) + cert, key = cli_ca + shutil.copyfile(cert, tmp_path / spec.ca) + original_env = os.environ + before_env = dict(original_env) + if fault is None: + # A real cli.main exception after CA assignment must also restore its parent. + exception_directory = tmp_path / "exception-operation" + exception_directory.mkdir() + with pytest.raises(ssl.SSLCertVerificationError): + with monkeypatch.context() as patch: + for name in ("SSL_CERT_FILE", "REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE"): + patch.setenv(name, before_env.get(name, certifi.where())) + if name not in before_env: + patch.delenv(name) + for name, value in before_env.items(): + if name.startswith("HYPERTRAIN_MINER_"): + patch.setenv(name, value) + patch.setattr(cfg, "workdir", exception_directory / "miner") + + def reject_wrong_ca(*args, **kwargs): + verify_cli_ca(ssl.create_default_context(cafile=certifi.where()), cert, key) + + patch.setattr(MetadataMiner, "run_round", reject_wrong_ca) + operation._capture_miner_cli(spec, tmp_path, exception_directory, threading.Event()) + assert os.environ is original_env and dict(os.environ) == before_env + # Capture real cli.main outcome using wrapper's capture path; explicit metadata HTTP only. + with monkeypatch.context() as patch: + for name in ("SSL_CERT_FILE", "REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE"): + patch.setenv(name, before_env.get(name, certifi.where())) + if name not in before_env: + patch.delenv(name) + for name, value in before_env.items(): + if name.startswith("HYPERTRAIN_MINER_"): + patch.setenv(name, value) + captured = operation._capture_miner_cli(spec, tmp_path, directory, threading.Event()) + assert os.environ["SSL_CERT_FILE"] == str(tmp_path / spec.ca) + verify_cli_ca(ssl.create_default_context(), cert, key) + assert os.environ is original_env and dict(os.environ) == before_env + # Same HTTPX construction later used by clock intake, immediately after the CLI. + with transport_original(): + pass + phase, published, challenge, commit, proof = captured + assert challenge is None and commit is None and proof is None + assert not {"status", "publication", "publication_custody_sha256", "environment"} & phase.keys() + assert not published.exists() + for name in ("operation-result.json", "publication-custody.json", "execution-custody.json"): + assert not (directory / name).exists() + assert phase["signed_api"] == json.loads((directory / "signed-api.json").read_bytes()) + assert phase["accepted_api"] == json.loads((directory / "accepted-api.json").read_bytes()) + # Explicit metadata-only Runtime result double, never completed work/Store acceptance. + result = { + **phase, + "status": "CAPTURED_NOT_ACCEPTED", + "operation": spec.operation, + "binding": spec.binding, + "hotkey": spec.hotkey, + "role": spec.role, + "job_sha256": job.digest(), + } + if fault is None: + # Reuse actual capture; real production completion still rejects missing publication. + with monkeypatch.context() as patch: + patch.setattr(operation, "_capture_miner_cli", lambda *args: captured) + with pytest.raises(FileNotFoundError): + operation.execute(spec, tmp_path, directory) + assert not (directory / "publication-custody.json").exists() + assert not (directory / "operation-result.json").exists() + import torch + + with monkeypatch.context() as patch: + patch.setattr(torch.cuda, "is_available", lambda: False) + with pytest.raises(ValueError, match="custody CUDA environment unavailable"): + operation.execution_environment(spec) + assert result["cli_outcome"] == {"status": "UPLOADED"} + assert [x["type"] for x in result["signed_api"]] == kinds + assert [x["request"] for x in result["accepted_api"]] == result["signed_api"] + if fault == "missing-status": + result["cli_outcome"] = {} + if fault == "forged-status": + result["cli_outcome"] = {"status": "PASS"} + if fault == "wrong-round": + result["signed_api"][0] = envelope_v2.seal( + hot, "AcceptV2", job.run_id, {**bodies["AcceptV2"], "w": job.w + 1}, 10000 + ) + if fault == "missing-acceptance": + result["accepted_api"] = [] + if fault in ("wrong-signer", "wrong-challenge", "wrong-commit", "wrong-run"): + signer = Keypair(bytes([99]) * 32) if fault == "wrong-signer" else hot + body = ( + {**bodies["WorkProof"], "challenge_hash": "19" * 32} + if fault == "wrong-challenge" + else bodies["WorkProof"] + ) + run = "20" * 32 if fault == "wrong-run" else job.run_id + position = 1 if fault == "wrong-commit" else 2 + if fault == "wrong-commit": + body = {**bodies["CommitV2"], "delta_hash": "21" * 32} + forged = envelope_v2.seal(signer, kinds[position], run, body, 10000) + result["signed_api"][position] = forged + result["accepted_api"][position]["request"] = forged + runtime = SimpleNamespace( + accept_operation=lambda *args, **kwargs: None, operation=lambda *args, **kwargs: result + ) + + def assignment(request): + return httpx.Response(200, json={"assignment": [{"hotkey": hot.ss58}]}) + + with transport_original( + base_url="https://master.test", transport=httpx.MockTransport(assignment) + ) as client: + args = ( + client, + runtime, + spec.model_dump(mode="json"), + {}, + hot, + {"owner": spec.owner, "run_id": job.run_id, "beacon": 1}, + directory, + threading.Event(), + ) + if fault is None: + assert driver.decoder_operation(*args) is result + else: + with pytest.raises(driver.DriverError): + driver.decoder_operation(*args) + assert len(responses) == 4 + + +def test_decoder_live_graph_full_metadata_sequence(tmp_path, monkeypatch): + """Full driver call ordering; HTTP/tape oracles explicit, no production ACTIVE claim.""" + from types import SimpleNamespace + + import httpx + + from hypertrain.protocol import envelope_v2 + from hypertrain.protocol.keys import Keypair + + seed_spec = importlib.util.spec_from_file_location( + "driver_graph_seed", SCRIPTS / "network_gpu_seed.py" + ) + assert seed_spec is not None and seed_spec.loader is not None + seed = importlib.util.module_from_spec(seed_spec) + seed_spec.loader.exec_module(seed) + public, private = tmp_path / "seed", tmp_path / "roles" + digest = seed.generate(public, private, "sha256:" + "ab" * 32, ["candidate"], 1000, 2000) + job = seed.validate(public, digest) + coordinator = Keypair((private / "coord.seed").read_bytes()) + original_seal = envelope_v2.seal + hot = [Keypair((private / f"hot-{i}.seed").read_bytes()) for i in range(4)] + cold = [Keypair((private / f"cold-{i}.seed").read_bytes()) for i in range(4)] + auditors = [Keypair((private / f"auditor-{i}.seed").read_bytes()) for i in range(2)] * 2 + roster = [ + { + "hotkey": k.ss58, + "slot": i, + "q_i": "3f800000", + "admission_id": f"{i + 1:064x}", + "coldkey_group": cold[i].ss58, + "state": "ACTIVE", + "eligible_weight": 4194304, + } + for i, k in enumerate(hot) + ] + opening = driver.RoundOpenV2( + w=0, + prev_final_hash="0" * 64, + theta_hash="11" * 32, + outer_state_hash="0" * 64, + center_hash="0" * 64, + roster_hash=driver.sha256_hex(driver.canonicalize(roster)), + honeypot_commit="0" * 64, + d_open=2, + d_assign=3, + d_commit=4, + d_audit=5, + d_upload=6, + d_final=10, + contract_version=2, + policy_hashes=driver.PolicyHashes.model_validate( + {k: getattr(job.manifest.network, k) for k in driver.PolicyHashes.model_fields} + ), + registry_epoch=0, + start_state_index_hash="12" * 32, + audit_mode="anchored-full", + roster=roster, + ) + actions = [] + sequence = iter(range(4)) + + def respond(request): + path = request.url.path + actions.append(path) + if path.endswith("/worker/lease"): + i = next(sequence) + start = driver.StartStateV2( + run_id=job.run_id, + w=0, + hotkey=hot[i].ss58, + **{ + k: "31" * 32 + for k in ( + "theta_hash", + "state_object_sha256", + "opt_state_hash", + "ef_object_sha256", + "ef_hash", + "parent_anchor_hash", + "anchor_verdict_hash", + ) + }, + global_step0=0, + ) + lease = { + "run_id": job.run_id, + "job_id": f"{i + 1:064x}", + "auditor_id": auditors[i].ss58, + "attempt": 1, + "lease_nonce": f"{i + 10:064x}", + "lease_expires": 6, + "absolute_deadline": 10, + "reservation_id": "32" * 32, + "replay_step_budget": 30, + "anchor_age": 0, + "manifest": job.manifest.body(), + "challenge_envelope": original_seal( + coordinator, + "AuditChallengeV2", + job.run_id, + { + "w": 0, + "target": hot[i].ss58, + "beacon_round": 5, + "beacon_sig_sha256": "38" * 32, + "mode": "full", + "segments": [], + "reasons": ["random"], + "serve_deadline": 10, + "anchor_hash": start.digest(), + "audit_mode": "anchored-full", + }, + 10, + ), + "commit_envelope": original_seal( + hot[i], + "CommitV2", + job.run_id, + { + "w": 0, + "hotkey": hot[i].ss58, + "leaf_scheme": "ht-leaf-v1", + "n_leaves": 7, + **{ + k: "39" * 32 + for k in ( + "leaves_root", + "metrics_root", + "final_theta_hash", + "ef_in_hash", + "ef_out_hash", + "delta_hash", + ) + }, + "delta_bytes": 1, + "tokens": job.manifest.training.batch_samples() * 16, + }, + 10, + ), + "sample_ids": job.sample_ids, + "start_state": start.body(), + "preimages": [ + { + "run_id": job.run_id, + "w": 0, + "t": 0, + "stages": [{"theta": "41" * 32, "m": "42" * 32, "v": "43" * 32}], + "batch_ids_sha256": "44" * 32, + "rng_ctr": 0, + "loss_f32": "00000000", + "norm_f32": "00000000", + } + ], + "ef_in": {"sha256": "33" * 32, "size": 1}, + "v0": {"sha256": "34" * 32, "size": 1}, + "created_beacon": 5, + } + return httpx.Response( + 200, json=original_seal(coordinator, "AuditJobV2", job.run_id, lease, 10) + ) + if path.endswith("/execute"): + return httpx.Response( + 200, + json={ + "verdict": { + "result": "MATCH", + "first_bad_leaf": None, + "challenge_hash": "35" * 32, + "recomputed_leaves_root": "36" * 32, + "replay_env": { + "image_digest": job.manifest.training.reference_spec.image_digest, + "driver": "metadata", + "gpu_uuid_sha256": "37" * 32, + "sm_count": 1, + }, + } + }, + ) + if path.endswith("/aggregate"): + return httpx.Response( + 200, + json={ + "tape_hash": "21" * 32, + "prev_state": "22" * 32, + "out_state": "23" * 32, + "theta_hash": "24" * 32, + }, + ) + return httpx.Response(200, json={}) + + # Bounded metadata downstream arithmetic/objects; no recomputation claim from these sentinels. + entries = [SimpleNamespace(hotkey=k.ss58, probation=False, weight_units=4194304) for k in hot] + monkeypatch.setattr( + driver.TapeV2, + "from_bytes", + lambda raw: SimpleNamespace( + body=SimpleNamespace(allocation=SimpleNamespace(entries=entries)) + ), + ) + policy = driver.AggregationPolicyV2.model_validate_json( + driver.LocalFSStore(public / "objects").get(job.manifest.network.aggregation_policy_hash) + ) + monkeypatch.setattr(driver.AggregationPolicyV2, "model_validate_json", lambda raw: policy) + monkeypatch.setattr(driver, "network_inputs", lambda values: values) + monkeypatch.setattr( + driver, "replay_tape", lambda *args, **kwargs: SimpleNamespace(to_bytes=lambda: b"outer") + ) + writes = [] + monkeypatch.setattr(driver, "write_network_checkpoint", lambda *args: writes.append(args)) + monkeypatch.setattr(driver, "verify_network_checkpoint", lambda *args: []) + monkeypatch.setattr(driver, "decoder_operation", lambda *args: actions.append("miner-API")) + context = { + "round_open": original_seal(coordinator, "RoundOpenV2", job.run_id, opening, 10), + "miners": [ + {"operation": {}, "receipt": {}, "key": k, "pins": {}, "artifacts": tmp_path} + for k in hot + ], + "auditors": auditors, + "accepted_inputs": lambda: [], + "accepted_finality": lambda: { + "disposition": "SETTLED", + "unresolved_audits": 0, + "unresolved_disputes": 0, + }, + } + with httpx.Client( + base_url="https://master.test", transport=httpx.MockTransport(respond) + ) as client: + lineage = driver.decoder_live_graph( + client, + SimpleNamespace(), + job.manifest, + [context], + coordinator, + {"Authorization": "Bearer metadata"}, + lambda n: actions.append(f"beacon-{n}"), + SimpleNamespace(get=lambda digest: b"outer"), + tmp_path / "checkpoint", + threading.Event(), + ) + assert len(lineage) == 1 and len(writes) == 1 + assert actions.count("miner-API") == 4 + assert sum(p.endswith("/execute") for p in actions) == 4 + assert sum(p.endswith("/complete") for p in actions) == 4 + assert actions[-2].endswith("/aggregate") and actions[-1].endswith("/finalize") + + +def test_parser_help_exits_before_work(capsys): + with pytest.raises(SystemExit) as error: + driver.main(["--help"]) + assert error.value.code == 0 + + +@pytest.mark.parametrize( + "endpoint", ["http://master.test", "https://user:secret@master.test", "https://master.test?x=1"] +) +def test_endpoint_requires_explicit_https_without_ambient_credentials(tmp_path, endpoint): + body = settings(tmp_path).model_dump() + body["master_https"] = endpoint + with pytest.raises(ValidationError): + driver.Settings.model_validate(body) + + +def test_digest_authority_rejects_before_service_write(tmp_path, signed_package): + directory, _, _, _, _, _ = signed_package + cfg = settings(tmp_path).model_copy(update={"snapshot": directory}) + with pytest.raises(ValueError, match="admitted bundle hash"): + driver.prepare(cfg) + assert not cfg.evidence.exists() + + +def test_wrong_role_rejects_before_live_write(tmp_path, monkeypatch, signed_package): + directory, digest, roles, template, _, _ = signed_package + cfg = settings(tmp_path).model_copy(update={"snapshot": directory, "snapshot_digest": digest}) + shutil.copytree(template, cfg.service_state) + shutil.copytree(roles, cfg.secrets) + (cfg.secrets / "coord.seed").write_bytes(bytes(32)) + monkeypatch.setattr(driver, "durable_destination", lambda path: True) + with pytest.raises(driver.DriverError, match="wrong role key"): + driver.prepare(cfg) + assert not cfg.evidence.exists() + + +def test_exact_product_cli_command(): + assert driver.miner_command(1) == [ + sys.executable, + "-m", + "hypertrain.miner.cli", + "run-v2", + "--round", + "1", + ] + + +def test_cancel_kills_real_ready_child_without_delay(tmp_path): + cancel = threading.Event() + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen() + listener.settimeout(10) + errors = [] + code = ( + "import socket; s=socket.create_connection(('127.0.0.1'," + + str(listener.getsockname()[1]) + + "));" + "s.sendall(b'READY'); s.recv(1)" + ) + + def child(): + try: + driver.run_child( + [sys.executable, "-c", code], dict(os.environ), tmp_path / "child.log", cancel, 10 + ) + except driver.DriverError as error: + errors.append(str(error)) + + worker = threading.Thread(target=child) + worker.start() + connection, _ = listener.accept() + connection.settimeout(10) + assert connection.recv(5) == b"READY" + cancel.set() + assert connection.recv(1) == b"" + worker.join(timeout=10) + connection.close() + listener.close() + assert not worker.is_alive() and len(errors) == 1 + + +def test_precancel_never_launches_child(tmp_path, monkeypatch): + cancel = threading.Event() + cancel.set() + + def forbidden(*args, **kwargs): + raise AssertionError("child launch") + + monkeypatch.setattr(driver.subprocess, "Popen", forbidden) + with pytest.raises(driver.DriverError, match="cancelled before miner"): + driver.run_child( + [sys.executable, "-c", "pass"], dict(os.environ), tmp_path / "none.log", cancel, 10 + ) + assert not (tmp_path / "none.log").exists() + + +def test_missing_service_adapter_rejects_before_evidence(tmp_path, monkeypatch, signed_package): + directory, digest, _, _, _, _ = signed_package + monkeypatch.setattr(driver, "durable_destination", lambda path: True) + cfg = settings(tmp_path).model_copy(update={"snapshot": directory, "snapshot_digest": digest}) + with pytest.raises(driver.DriverError, match="MISSING_ADAPTER"): + driver.prepare(cfg) + assert not cfg.evidence.exists() + + +def test_pytest_retention_destination_rejects(tmp_path): + assert not driver.durable_destination(tmp_path / "output") + + +def test_failed_live_work_exports_journal_and_indexes_before_return(tmp_path, monkeypatch): + cfg = settings(tmp_path) + cfg.service_state.mkdir() + with sqlite3.connect(cfg.service_state / "challenge.db") as db: + db.execute("CREATE TABLE accepted(id TEXT)") + db.execute("INSERT INTO accepted VALUES('original')") + objects = cfg.service_state / "objects" + objects.mkdir() + (objects / "original").write_bytes(b"signed-evidence") + config = tmp_path / "driver.json" + config.write_text(cfg.model_dump_json()) + + def failed_work(settings, cancel): + settings.evidence.mkdir() + raise driver.DriverError("actual service failed") + + monkeypatch.setattr(driver, "proof", failed_work) + monkeypatch.setattr(driver.signal, "signal", lambda *args: None) + with pytest.raises(driver.DriverError, match="actual service failed"): + driver.main(["--config", str(config)]) + result = json.loads((cfg.evidence / "result.json").read_bytes()) + index = json.loads((cfg.evidence / "index.json").read_bytes()) + assert result["status"] == "FAIL" + assert "service-export/challenge.db" in index["files"] + assert result["index_sha256"] == driver.sha256_hex((cfg.evidence / "index.json").read_bytes()) + assert (cfg.evidence / "service-export/objects/original").read_bytes() == b"signed-evidence" + for relative, entry in index["files"].items(): + data = (cfg.evidence / relative).read_bytes() + assert driver.sha256_hex(data) == entry["sha256"] and len(data) == entry["size"] + + +@pytest.mark.parametrize("offset", [float("nan"), float("inf"), -float("inf")]) +def test_nonfinite_clock_rejects_at_configuration(tmp_path, offset): + body = settings(tmp_path).model_dump() + body["fixture_clock_offset"] = offset + with pytest.raises(ValidationError): + driver.Settings.model_validate(body) + assert not settings(tmp_path).evidence.exists() + + +@pytest.mark.parametrize("mutation", ["offset", "beacon", "nonfinite", "valid"]) +def test_exact_bootstrap_clock_binding(tmp_path, monkeypatch, mutation): + from types import SimpleNamespace + + from hypertrain.beacon.core import FixtureBeacon + + cfg = settings(tmp_path) + cfg.service_state.mkdir() + beacon = FixtureBeacon(current=52).get(52) + payload = { + "round": beacon.round, + "signature": beacon.signature, + "randomness": beacon.randomness, + } + cfg.verified_beacon.write_text(json.dumps(payload)) + offset = 123.5 + runtime = {"fixture_clock_offset": offset, "beacon": payload} + if mutation == "beacon": + other = FixtureBeacon(current=53).get(53) + runtime["beacon"] = { + "round": other.round, + "signature": other.signature, + "randomness": other.randomness, + } + if mutation == "nonfinite": + runtime["fixture_clock_offset"] = float("nan") + (cfg.service_state / "runtime.json").write_text(json.dumps(runtime)) + cfg = cfg.model_copy( + update={"fixture_clock_offset": 999999 if mutation == "offset" else offset} + ) + manifest = SimpleNamespace(training=SimpleNamespace(beacon=SimpleNamespace(genesis_time=1000))) + bundle = SimpleNamespace(beacon_round=52) + monkeypatch.setattr(driver.time, "time", lambda: 1000 + 51 * 3 + offset + 10) + monkeypatch.setattr(driver.time, "monotonic", lambda: 20) + if mutation == "valid": + assert driver.bound_clock(cfg, bundle, manifest) == 1000 + 51 * 3 - 10 + else: + with pytest.raises(driver.DriverError): + driver.bound_clock(cfg, bundle, manifest) + assert not cfg.evidence.exists() + + +@pytest.mark.parametrize("failure", ["copy", "hash", "index_write", "none"]) +def test_export_errors_never_publish_unindexed_pass(tmp_path, monkeypatch, failure): + cfg = settings(tmp_path) + cfg.service_state.mkdir() + with sqlite3.connect(cfg.service_state / "challenge.db") as db: + db.execute("CREATE TABLE original(id TEXT)") + objects = cfg.service_state / "objects" + objects.mkdir() + (objects / "artifact").write_bytes(b"original") + cfg.evidence.mkdir() + if failure == "copy": + monkeypatch.setattr( + driver.shutil, + "copytree", + lambda *args, **kwargs: (_ for _ in ()).throw(OSError("injected copy failure")), + ) + if failure == "hash": + original_read = Path.read_bytes + + def fail_read(path): + if path.name == "artifact": + raise OSError("injected hash read failure") + return original_read(path) + + monkeypatch.setattr(Path, "read_bytes", fail_read) + if failure == "index_write": + original_publish = driver.durable_json + failed = [] + + def fail_index(path, value): + if path.name == "index.json" and not failed: + failed.append(True) + raise OSError("injected index write failure") + return original_publish(path, value) + + monkeypatch.setattr(driver, "durable_json", fail_index) + if failure == "none": + driver.finalize_evidence(cfg, True, None) + else: + with pytest.raises(driver.DriverError, match="incomplete evidence export"): + driver.finalize_evidence(cfg, True, None) + result = json.loads((cfg.evidence / "result.json").read_bytes()) + index = json.loads((cfg.evidence / "index.json").read_bytes()) + assert result["status"] == ("PASS" if failure == "none" else "FAIL") + assert result["export_complete"] == (failure == "none") + assert index["export_complete"] == (failure == "none") + assert result["index_sha256"] == driver.sha256_hex((cfg.evidence / "index.json").read_bytes()) + assert (objects / "artifact").exists() + + +def test_rescue_copy_error_preserves_original_workload_exception(tmp_path, monkeypatch): + cfg = settings(tmp_path) + cfg.service_state.mkdir() + with sqlite3.connect(cfg.service_state / "challenge.db") as db: + db.execute("CREATE TABLE original(id TEXT)") + (cfg.service_state / "objects").mkdir() + config = tmp_path / "config.json" + config.write_text(cfg.model_dump_json()) + original_error = driver.DriverError("original live failure") + + def fail_work(settings, cancel): + settings.evidence.mkdir() + raise original_error + + def fail_copy(*args, **kwargs): + raise OSError("rescue copy failure") + + monkeypatch.setattr(driver, "proof", fail_work) + monkeypatch.setattr(driver.shutil, "copytree", fail_copy) + monkeypatch.setattr(driver.signal, "signal", lambda *args: None) + with pytest.raises(driver.DriverError) as observed: + driver.main(["--config", str(config)]) + assert observed.value is original_error + result = json.loads((cfg.evidence / "result.json").read_bytes()) + index = json.loads((cfg.evidence / "index.json").read_bytes()) + assert result["status"] == "FAIL" and result["workload_error"] == str(original_error) + assert result["errors"] and not index["export_complete"] + + +@pytest.mark.parametrize("fault", ["offset", "beacon", "valid"]) +def test_prepare_clock_probe_rejects_before_any_output( + tmp_path, monkeypatch, fault, signed_package +): + from hypertrain.beacon.core import FixtureBeacon + + directory, digest, roles, template, offset, payload = signed_package + cfg = settings(tmp_path) + cfg = cfg.model_copy( + update={ + "snapshot": directory, + "snapshot_digest": digest, + "fixture_clock_offset": offset + 999999 if fault == "offset" else offset, + "fixture_clock_directory": cfg.service_state / "fixture-clock", + "ca": Path(certifi.where()), + } + ) + shutil.copytree(template, cfg.service_state) + shutil.copytree(roles, cfg.secrets) + monkeypatch.setattr(driver, "durable_destination", lambda *args: True) + cfg.verified_beacon.write_text(json.dumps(payload)) + if fault == "beacon": + b = FixtureBeacon(current=payload["round"] + 1).get(payload["round"] + 1) + (cfg.service_state / "runtime.json").write_text( + json.dumps( + { + "fixture_clock_offset": offset, + "beacon": { + "round": b.round, + "signature": b.signature, + "randomness": b.randomness, + }, + } + ) + ) + if fault == "valid": + bundle, manifest = driver.prepare(cfg) + assert bundle.run_id == manifest.run_id() == driver.authority.RUN + with sqlite3.connect(cfg.evidence / "initial-service.db") as db: + assert ( + db.execute("SELECT COUNT(*) FROM admissions_v2 WHERE state='ACTIVE'").fetchone()[0] + == 4 + ) + assert ( + db.execute( + "SELECT COUNT(*) FROM admission_trial_results WHERE outcome='MATCH'" + ).fetchone()[0] + == 48 + ) + else: + with pytest.raises(driver.DriverError): + driver.prepare(cfg) + assert not cfg.evidence.exists() diff --git a/tests/e2e/test_rollback_boundary_v2.py b/tests/e2e/test_rollback_boundary_v2.py new file mode 100644 index 0000000..b0aebb9 --- /dev/null +++ b/tests/e2e/test_rollback_boundary_v2.py @@ -0,0 +1,736 @@ +"""Exact independent history/CAS probes; tiny actual inner carry, no activation grid.""" + +from __future__ import annotations + +import importlib.util +import sqlite3 +import sys +import threading +from functools import partial +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pytest + +import hypertrain.trainer # noqa: F401 +from hypertrain.aggregator.core import OuterState, TapeError +from hypertrain.aggregator.rollback_v2 import ( + RepairAuthority, + RepairSource, + _qualification_body, + _replay_history, + make_repair_tape, + replay_repair_tape, +) +from hypertrain.aggregator.tape_v2 import make_tape +from hypertrain.auditor.replay import AnchorCache, optimizer_hash, pack_state +from hypertrain.challenge.store import ChallengeStore, assignment_hash +from hypertrain.challenge.trust_v2 import FundedStatus, ReplayEvidence, SettlementStatus +from hypertrain.protocol.envelope_v2 import seal +from hypertrain.protocol.hashing import sha256_hex +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.keys import Keypair +from hypertrain.protocol.messages import Chunk +from hypertrain.protocol.messages_v2 import ( + AuditChallengeV2, + AuditJobV2, + CommitV2, + DeltaManifestV2, + StartStateV2, +) +from hypertrain.trainer.compress import state_hash +from hypertrain.trainer.config import TrainConfig +from hypertrain.trainer.island import IslandAssignment, emulate, train_island +from hypertrain.trainer.model import init_params + +sys.path.insert(0, str(Path(__file__).parents[1] / "aggregator")) +spec = importlib.util.spec_from_file_location( + "rollback_tiny", Path(__file__).parents[1] / "aggregator/test_rollback_v2.py" +) +assert spec is not None and spec.loader is not None +tiny = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = tiny +spec.loader.exec_module(tiny) + + +class MemoryStore: + def __init__(self): + self.objects = {} + + def put(self, raw): + key = sha256_hex(raw) + self.objects[key] = raw + return key + + def get(self, key): + return self.objects[key] + + +def test_exact_cas_probe_binds_original_envelope_even_same_digest_and_receipt(): + # Given: same SQLite snapshot contract as independent reviewer, all original data unchanged. + store = ChallengeStore.__new__(ChallengeStore) + store._lock = threading.RLock() + store._db = sqlite3.connect(":memory:", isolation_level=None) + store._db.executescript(""" + CREATE TABLE records_v2(run_id,kind,id,data); + CREATE TABLE admissions_v2(hotkey,clean_count); + CREATE TABLE disputes_v2(id,turn); + CREATE TABLE accepted_v2(key,digest,envelope,accepted_beacon,receipt); + CREATE TABLE escrow_units(origin,owner,bucket,ref,units); + INSERT INTO accepted_v2 VALUES('source','unchanged',X'010203',1,'{}'); + """) + before = store._rollback_snapshot_v2("run") + # When: mutate original signed bytes, retaining normalized digest/receipt/state. + store._db.execute("BEGIN") + store._db.execute("UPDATE accepted_v2 SET envelope=X'010204'") + after = store._rollback_snapshot_v2("run") + store._db.execute("ROLLBACK") + # Then: CAS detects exact historical byte change; rollback restores snapshot. + assert after != before + assert store._rollback_snapshot_v2("run") == before + + +@pytest.fixture(scope="module") +def ordinary_history(): + from hypertrain.aggregator.tape_v2 import VerifiedInput + + store = MemoryStore() + from hypertrain.protocol.hashing import MerkleTree + from hypertrain.protocol.messages_v2 import RunManifestV2 + + manifest_body = tiny.tiny_manifest().body() + rows = np.array([tiny.sample(i) for i in range(32)], dtype="=100", + (cold.ss58,), + ).fetchone()[0] + lock = EscrowLock( + operation_id="df" * 32, + owner=cold.ss58, + units=100, + origin_ids=[origin], + admission_id=None, + dispute_id=dispute_id, + kind="LOCK_CONTEST", + ) + response = n.client.post(n.url + "/escrow/lock", json=n.signed(cold, "EscrowLock", lock)) + assert response.status_code == 200, response.text + response = n.client.post( + n.url + "/dispute", + json=n.signed( + hot, + "DisputeV2", + DisputeV2( + hotkey=hot.ss58, + verdict_hash=contest["verdict_hash"], + action="contest", + bond_lock=lock.operation_id, + ), + ), + ) + assert response.status_code == 200, response.text + disputes = n.store._services(run)[2] + geometry = n.store._record_v2(run, "trace-geometry", contest["verdict_hash"]) + party = IslandParty.published( + hot.ss58, IslandJobV1.model_validate(geometry["job"]), Path(geometry["directory"]) + ) + for _ in range(64): + turn = disputes.get(dispute_id) + if turn.paused: + break + key = ( + hot + if turn.expected_party == hot.ss58 + else next(k for k in fixtures.AUDITORS if k.ss58 == turn.expected_party) + ) + at = [turn.interval[0], -(-sum(turn.interval) // 2), turn.interval[1]] + hashes = party.hashes(turn.level, tuple(turn.ctx), at) + if key == hot: + hashes = [ + digest if index == 0 else sha256_hex(bytes.fromhex(digest)) + for index, digest in zip(at, hashes, strict=True) + ] + answer = BisectV2( + dispute_id=dispute_id, + seq=turn.seq, + level=turn.level, + ctx=turn.ctx, + interval=turn.interval, + N=2, + hashes=hashes, + party=key.ss58, + previous_transcript_hash=turn.transcript_hash, + ) + response = n.client.post(n.url + "/bisect", json=n.signed(key, "BisectV2", answer)) + assert response.status_code == 200, response.text + turn = disputes.get(dispute_id) + assert turn.paused + response = n.client.post( + n.url + f"/admin/disputes/{dispute_id}/referee", headers=fixtures.admin() + ) + assert response.status_code == 200, response.text + assert response.json()["evidence"]["reason"] == "FRAUD" + assert response.json()["evidence"]["loser"] == hot.ss58 + resolution = ResolutionV2( + dispute_id=dispute_id, + transcript_hash=turn.transcript_hash, + reason="FRAUD", + loser=hot.ss58, + evidence_hash=response.json()["evidence_hash"], + ) + response = n.client.post( + n.url + "/resolution", json=n.signed(fixtures.REFEREE, "ResolutionV2", resolution) + ) + assert response.status_code == 200, response.text + return resolution.digest() diff --git a/tests/e2e/test_service_network_v2_e2e.py b/tests/e2e/test_service_network_v2_e2e.py new file mode 100644 index 0000000..a72739f --- /dev/null +++ b/tests/e2e/test_service_network_v2_e2e.py @@ -0,0 +1,2150 @@ +"""Tiny real CPU shadow execution and complete-roster checkpoint boundaries.""" + +from __future__ import annotations + +import importlib.util +import json +import os +import shutil +import sqlite3 +import sys +import tempfile +from pathlib import Path + +import pytest + +spec = importlib.util.spec_from_file_location( + "service_network_fixture", Path(__file__).parents[1] / "challenge/test_service_network_v2.py" +) +assert spec is not None and spec.loader is not None +fixtures = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = fixtures +spec.loader.exec_module(fixtures) +network = fixtures.network + + +@pytest.fixture +def od_carry_network(tmp_path, request, monkeypatch): + """Pin carry before service registration; preserve real dataset/policy setup.""" + from hypertrain.protocol.messages_v2 import RunManifestV2 + + setup_type = fixtures.fixture.Setup + + def carry_setup(manifest, policy, admission_policy, rows, tree): + body = manifest.body() + body["training"]["inner"].update(state_policy="carry", rewarmup_steps=0) + return setup_type(RunManifestV2.model_validate(body), policy, admission_policy, rows, tree) + + monkeypatch.setattr(fixtures.fixture, "Setup", carry_setup) + yield from fixtures.network.__wrapped__(tmp_path, request) + + +@pytest.mark.parametrize("od_carry_network", ["mlm", "decision", "distill"], indirect=True) +def test_od_carry_admission_reference_requires_real_full_state(od_carry_network): + """First production prerequisite: genuine corresponding-manifest graduation work.""" + assert od_carry_network.manifest.training.inner.state_policy == "carry" + assert od_carry_network.manifest.training.reference_spec.layout.n_gpus == 2 + # Positive production prerequisite, not an expected-failure or patched launcher. + test_actual_shadow_trial_does_not_mutate_production(od_carry_network) + + +def test_actual_shadow_trial_does_not_mutate_production(network): + from hypertrain.auditor.replay import unpack_state + from hypertrain.protocol.messages import Finalize + from hypertrain.protocol.messages_v2 import EscrowLock, WorkProof, WorkScreenV2 + from hypertrain.trainer.compress import state_hash + + # Given: funded signed public application; no production round. + identity = network.join().json()["admission_id"] + lock = EscrowLock( + operation_id="cb" * 32, + owner=fixtures.COLD[0].ss58, + units=1000, + origin_ids=[network.origin_ids[0]], + admission_id=identity, + dispute_id=None, + kind="LOCK_ADMISSION", + ) + response = network.client.post( + network.url + "/escrow/lock", json=network.signed(fixtures.COLD[0], "EscrowLock", lock) + ) + assert response.status_code == 200, response.text + network.push(2) + challenge = network.client.get(network.url + "/join/" + identity + "/challenge") + assert challenge.status_code == 200, challenge.text + # When: actual torchrun reference, signed proof, signed exact shadow finality. + response = network.client.post( + network.url + "/admin/join/" + identity + "/reference", headers=fixtures.admin() + ) + assert response.status_code == 200, response.text + row = network.store._db.execute( + "SELECT * FROM admissions_v2 WHERE admission_id=?", (identity,) + ).fetchone() + proof = WorkProof.model_validate_json(row["reference_json"]) + from hypertrain.data.trial_assignment import trial_samples + from hypertrain.miner.island_launch import validate_artifacts + from hypertrain.protocol.messages_v2 import JoinChallenge + + challenge_body = JoinChallenge.model_validate(challenge.json()["body"]) + directory = network.store.state_dir / "trials-v2" / challenge_body.nonce + job, same_directory = network.store.stage_trial_v2( + network.manifest.run_id(), + challenge_body, + tuple( + trial_samples( + network.manifest, + identity, + challenge_body.nonce, + network.store._beacon_v2(challenge_body.seed_beacon), + ) + ), + row["trial_epoch"], + ) + artifacts = validate_artifacts(job, directory / "published") + screen = WorkScreenV2( + evidence_kind="work-proof-v1", + admission_id=identity, + nonce=challenge_body.nonce, + policy_hash=challenge_body.policy_hash, + image_digest=job.manifest.training.reference_spec.image_digest, + layout=challenge_body.layout, + challenge_hash=challenge_body.digest(), + rank_results=list(artifacts.ranks), + artifact_hashes=[ref.sha256 for ref in proof.artifact_refs], + ) + response = network.client.post( + network.url + "/join/proof", + json={ + "proof": network.signed(fixtures.HOT[0], "WorkProof", proof), + "screen": network.signed(fixtures.HOT[0], "WorkScreenV2", screen), + }, + ) + assert response.status_code == 200, response.text + theta, _ = unpack_state(network.store.objects.get(proof.artifact_refs[0].sha256)) + final = Finalize( + w=row["trial_epoch"], + final_theta_hash_w1=state_hash(theta), + included=[fixtures.HOT[0].ss58], + entitlements_root=proof.digest(), + ) + response = network.client.post( + network.url + "/admin/join/" + identity + "/finalize", + json=network.signed(fixtures.COORD, "Finalize", final), + headers=fixtures.admin(), + ) + assert response.status_code == 200, response.text + # Then: one genuine clean participation, no production cursor/tape/issuance change. + assert response.json()["record"]["clean_count"] == 1 + assert ( + network.store._db.execute( + "SELECT COUNT(*) FROM records_v2 WHERE kind IN ('round','applied','finality')" + ).fetchone()[0] + == 0 + ) + assert network.store._services(network.manifest.run_id())[0].balances().issued == 40000 + + +@pytest.mark.parametrize("network", ["mlm", "decision", "distill"], indirect=True) +def test_od_two_rank_signed_service_reference_and_proof(network): + # All OD objectives use the real signed admission/reference/proof/finality routes, + # strict record samples, Merkle proof inputs and real N2 torchrun. + test_actual_shadow_trial_does_not_mutate_production(network) + row = network.store._db.execute("SELECT reference_json FROM admissions_v2").fetchone() + assert row[0] + assert network.manifest.training.reference_spec.layout.n_gpus == 2 + + +@pytest.mark.parametrize( + "fault", ["round", "included", "run", "type", "opening_round", "predecessor"] +) +def test_resigned_checkpoint_finality_subject_rejects(tmp_path, fault): + from hypertrain.aggregator.checkpoint import ( + CheckpointError, + verify_network_checkpoint, + write_network_checkpoint, + ) + from hypertrain.data.store import LocalFSStore + from hypertrain.protocol import envelope_v2 + from hypertrain.protocol.hashing import sha256_hex + from hypertrain.protocol.jcs import canonicalize + from hypertrain.protocol.messages_v2 import RunManifestV2 + + source = Path(os.environ["HT_NETWORK_CHECKPOINT_SNAPSHOT"]) + stable = Path(tempfile.gettempdir()) / "hypertrain-l0-checkpoint-proof" + if source != stable: + shutil.copytree(source, stable, dirs_exist_ok=True) + source = stable + directory = tmp_path / "bad-checkpoint" + shutil.copytree(source, directory) + lineage = json.loads((directory / "lineage.json").read_bytes()) + item = lineage["rounds"][0] + target = "finalize" + final = item[target] + match fault: + case "round": + final["body"]["w"] += 1 + case "included": + final["body"]["included"] = [] + case "run": + final["run_id"] = "ff" * 32 + case "type": + final["type"] = "RunManifestV2" + case "opening_round": + target = "round_open" + item[target]["body"]["w"] += 1 + case "predecessor": + item["predecessor_tape_hash"] = "ab" * 32 + if fault not in ("type", "predecessor"): + signed = item[target] + item[target] = envelope_v2.seal( + fixtures.COORD, signed["type"], signed["run_id"], signed["body"], signed["exp_drand"] + ) + raw = canonicalize(lineage) + (directory / "lineage.json").write_bytes(raw) + document = json.loads((directory / "MANIFEST.json").read_bytes()) + document["body"]["files"]["lineage.json"] = sha256_hex(raw) + document["sig"] = fixtures.COORD.sign( + b"hypertrain/checkpoint/2|" + sha256_hex(canonicalize(document["body"])).encode() + ).hex() + (directory / "MANIFEST.json").write_bytes(canonicalize(document)) + assert verify_network_checkpoint(directory, fixtures.COORD.ss58) + with pytest.raises((CheckpointError, ValueError)): + write_network_checkpoint( + tmp_path / "writer-output", + fixtures.COORD, + LocalFSStore(directory / "objects"), + RunManifestV2.model_validate(lineage["manifest"]), + lineage["rounds"], + ) + assert not (tmp_path / "writer-output").exists() + + +def _live_checkpoint(network): + import anyio + import httpx + + from hypertrain.aggregator.checkpoint import verify_network_checkpoint, write_network_checkpoint + from hypertrain.aggregator.cli import main + from hypertrain.auditor.replay import AnchorCache, optimizer_hash, pack_state + from hypertrain.data.store import LocalFSStore + from hypertrain.data.stream_store import StreamStore + from hypertrain.miner.island_launch import launch_island + from hypertrain.protocol import relay_envelope + from hypertrain.protocol.hashing import sha256_hex + from hypertrain.protocol.jcs import canonicalize + from hypertrain.protocol.messages import Finalize, f32hex + from hypertrain.protocol.messages_v2 import ( + ArtifactRef, + CommitV2, + PolicyHashes, + RoundOpenV2, + StartStateV2, + WorkProof, + ) + from hypertrain.protocol.relay_messages import UploadChunk, UploadChunkManifest + from hypertrain.relay.app import create_app + from hypertrain.relay.client import RelayClient + from hypertrain.relay.core import Relay + from hypertrain.trainer.compress import state_hash + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.model import init_params + + manifest = network.manifest + cache = AnchorCache() + theta = init_params(TrainConfig.from_manifest_v2(manifest).model) + starts, roster = [], [] + for i, hot in enumerate(fixtures.HOT): + status = network.store._services(manifest.run_id())[1].status(hot.ss58, now=network.now) + anchor = cache.genesis(manifest, hot.ss58, theta) + starts.append( + StartStateV2( + run_id=manifest.run_id(), + w=0, + hotkey=hot.ss58, + theta_hash=state_hash(theta), + state_object_sha256=sha256_hex(pack_state(theta, anchor.state)), + opt_state_hash=optimizer_hash(anchor.state), + ef_object_sha256=sha256_hex(pack_state(anchor.ef)), + ef_hash=state_hash(anchor.ef), + parent_anchor_hash=anchor.anchor_hash, + global_step0=0, + anchor_verdict_hash=anchor.proof_hash, + ) + ) + roster.append( + { + "hotkey": hot.ss58, + "slot": i, + "q_i": f32hex(1), + "admission_id": status.record.admission_id, + "coldkey_group": status.record.coldkey, + "state": "ACTIVE", + "eligible_weight": 4194304, + } + ) + n = manifest.network + opening = RoundOpenV2( + w=0, + prev_final_hash="0" * 64, + theta_hash=state_hash(theta), + outer_state_hash="0" * 64, + center_hash="0" * 64, + roster_hash=sha256_hex(canonicalize(roster)), + honeypot_commit="0" * 64, + d_open=network.now + 1, + d_assign=network.now + 2, + d_commit=network.now + 30, + d_audit=network.now + 31, + d_upload=network.now + 200, + d_final=network.now + 300, + contract_version=2, + policy_hashes=PolicyHashes.model_validate( + {k: getattr(n, k) for k in PolicyHashes.model_fields} + ), + registry_epoch=0, + start_state_index_hash=sha256_hex(canonicalize([s.body() for s in starts])), + audit_mode="anchored-full", + roster=roster, + ) + response = network.client.post( + network.url + "/admin/rounds", + json=network.signed(fixtures.COORD, "RoundOpenV2", opening), + headers=fixtures.admin(), + ) + assert response.status_code == 200, response.text + network.push(opening.d_assign) + view = network.client.get(network.url + "/rounds/0") + assert view.status_code == 200, view.text + payloads = [] + for i, hot in enumerate(fixtures.HOT): + if i == 0 and os.environ.get("HT_NETWORK_REAL_CLIENT") == "1": + continue + assignment = next(a for a in view.json()["assignment"] if a["hotkey"] == hot.ss58) + row = network.store._db.execute( + "SELECT screen_json FROM admissions_v2 WHERE hotkey=?", (hot.ss58,) + ).fetchone() + screen = json.loads(row[0]) + accept = { + "w": 0, + "hotkey": hot.ss58, + "assignment_hash": assignment["assignment_hash"], + "image_digest": manifest.training.reference_spec.image_digest, + "driver_version": "test", + "n_gpus": 1, + "work_screen_hash": sha256_hex(canonicalize(screen)), + } + accept["driver_version"] = manifest.training.reference_spec.driver_allowlist[0] + response = network.client.post( + network.url + "/accept", json=network.signed(hot, "AcceptV2", accept) + ) + assert response.status_code == 200, response.text + staged = network.store.island_job_v2(manifest.run_id(), 0, hot.ss58) + from hypertrain.protocol.messages_v2 import IslandJobV1 + + job = IslandJobV1.model_validate(staged["job"]) + directory = network.store.state_dir / "jobs-v2/0" / hot.ss58 + artifacts = launch_island(job, directory, backend="cpu", trace=True) + summary = json.loads((artifacts.directory / "rank-0/summary.json").read_bytes()) + leaves = json.loads(artifacts.leaves.read_bytes()) + from hypertrain.protocol.hashing import MerkleTree + + commitments = summary["commitments"] + commit = CommitV2.model_validate( + { + "w": 0, + "hotkey": hot.ss58, + "leaf_scheme": "ht-leaf-v1", + "n_leaves": len(leaves), + "metrics_root": MerkleTree( + [bytes.fromhex(p["loss_f32"] + p["norm_f32"]) for p in leaves] + ).root.hex(), + "tokens": len(job.sample_ids) * manifest.training.model.seq_len, + "ef_in_hash": starts[i].ef_hash, + **{ + k: commitments[k] + for k in ("leaves_root", "final_theta_hash", "ef_out_hash", "delta_hash") + }, + "delta_bytes": artifacts.delta.stat().st_size, + } + ) + response = network.client.post( + network.url + "/commit", json=network.signed(hot, "CommitV2", commit) + ) + assert response.status_code == 200, response.text + refs = [] + for path in (artifacts.state, artifacts.ef, artifacts.delta, artifacts.leaves): + network.store.objects.put(path.read_bytes()) + refs.append(ArtifactRef(sha256=sha256_hex(path.read_bytes()), size=path.stat().st_size)) + descriptor = WorkProof( + admission_id=roster[i]["admission_id"], + challenge_hash=sha256_hex(canonicalize(commit.model_dump(mode="json"))), + leaves_root=commit.leaves_root, + delta_hash=commit.delta_hash, + artifact_refs=refs, + ) + response = network.client.post( + network.url + "/leaves", json=network.signed(hot, "WorkProof", descriptor) + ) + assert response.status_code == 200, response.text + payloads.append((hot, artifacts.delta)) + + async def transport_all(): + async with httpx.AsyncClient() as backing_http: + + async def now(): + return network.now + + registry = network.store._registry_v2(manifest) + relay_assignment = network.store.relay_assignment_v2( + manifest.run_id(), 0, fixtures.HOT[0].ss58 + ) + relay = Relay( + run_id=manifest.run_id(), + master=fixtures.COORD.ss58, + registry=registry, + network_manifest_hash=sha256_hex( + canonicalize(relay_assignment["network_manifest"]["body"]) + ), + observers={fixtures.AUDITORS[0].ss58, fixtures.AUDITORS[1].ss58}, + relay_id="local", + region="local", + keys={"k1": fixtures.RELAY}, + active_key="k1", + backing=StreamStore( + LocalFSStore(network.store.state_dir / "relay-backing"), backing_http + ), + now=now, + ) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(relay, "x" * 32)), + base_url="https://relay.test", + ) as client: + if os.environ.get("HT_NETWORK_REAL_CLIENT") == "1": + await anyio.to_thread.run_sync( + lambda: _exact_network_client(network, relay, create_app(relay, "x" * 32)) + ) + for hot, path in payloads: + assigned = network.store.relay_assignment_v2(manifest.run_id(), 0, hot.ss58) + raw = path.read_bytes() + chunks = UploadChunkManifest( + size=len(raw), + chunks=[ + UploadChunk(index=0, off=0, len=len(raw), chunk_sha256=sha256_hex(raw)) + ], + ) + grant = network.store.upload_grant_v2( + manifest.run_id(), + canonicalize( + relay_envelope.seal( + hot, + "UploadChunkManifest", + manifest.run_id(), + chunks, + opening.d_upload, + ) + ), + ) + receipt = await RelayClient(registry, client).upload( + path, + grant_raw=relay_envelope.parse_envelope(grant), + assignment=relay_envelope.parse_envelope(assigned["assignment"]), + round_open=network.signed(fixtures.COORD, "RoundOpenV2", opening), + ) + acceptance = await network.store.relay_receipt_v2( + canonicalize(receipt.model_dump(mode="json")), client + ) + delta = { + "w": 0, + "hotkey": hot.ss58, + "delta_hash": sha256_hex(raw), + "size": len(raw), + "uri": "sha256:" + sha256_hex(raw), + "format": "ht-sparse-v1", + "chunks": [{"off": 0, "len": len(raw), "sha256": sha256_hex(raw)}], + "grant_hash": grant["body"] and sha256_hex(canonicalize(grant["body"])), + "master_acceptance_hash": sha256_hex(canonicalize(acceptance["body"])), + } + response = network.client.post( + network.url + "/delta", json=network.signed(hot, "DeltaManifestV2", delta) + ) + assert response.status_code == 200, response.text + ack = await network.store.relay_control_v2( + manifest.run_id(), delta["grant_hash"], "extend", client + ) + assert ( + ack["body"]["extension_hash"] + == network.store._record_v2( + manifest.run_id(), "retention", delta["grant_hash"] + )["hash"] + ) + ready = await client.get("/readyz") + assert ready.status_code == 200, ready.text + drained = await client.post( + "/internal/drain", headers={"Authorization": "Bearer " + "x" * 32} + ) + assert drained.status_code == 200 + assert (await client.get("/readyz")).status_code == 503 + + anyio.run(transport_all) + network.push(opening.d_audit) + response = network.client.post(network.url + "/admin/rounds/0/audits", headers=fixtures.admin()) + assert response.status_code == 200, response.text + for index in range(4): + auditor = fixtures.AUDITORS[index % 2] + lease_request = network.signed( + auditor, + "Receipt", + {"w": 0, "commit_hash": f"{index + 1:064x}", "received_round": network.now}, + ) + response = network.client.post(network.url + "/worker/lease", json=lease_request) + assert response.status_code == 200, response.text + job = response.json()["body"] + execute = network.signed( + auditor, + "Receipt", + {"w": 0, "commit_hash": job["lease_nonce"], "received_round": network.now}, + ) + response = network.client.post( + network.url + "/worker/jobs/" + job["job_id"] + "/execute", json=execute + ) + assert response.status_code == 200, response.text + assert response.json()["verdict"]["result"] == "MATCH" + response = network.client.post( + network.url + "/worker/jobs/" + job["job_id"] + "/complete", + json=network.signed(auditor, "ReplayVerdict", response.json()["verdict"]), + ) + assert response.status_code == 200, response.text + response = network.client.post( + network.url + "/admin/rounds/0/aggregate", headers=fixtures.admin() + ) + assert response.status_code == 200, response.text + applied = response.json() + final = Finalize( + w=0, + final_theta_hash_w1=applied["theta_hash"], + included=sorted(k.ss58 for k in fixtures.HOT), + entitlements_root="0" * 64, + ) + response = network.client.post( + network.url + "/admin/rounds/0/finalize", + json=network.signed(fixtures.COORD, "Finalize", final), + headers=fixtures.admin(), + ) + assert response.status_code == 200, response.text + conserved = network.store._services(manifest.run_id())[0].balances() + repeated = network.client.post( + network.url + "/admin/rounds/0/finalize", + json=network.signed(fixtures.COORD, "Finalize", final), + headers=fixtures.admin(), + ) + assert repeated.status_code == 200 and repeated.json() == response.json() + assert network.store._services(manifest.run_id())[0].balances() == conserved + record = network.store._record_v2(manifest.run_id(), "tape-inputs", "0") + rounds = [ + { + "round_open": network.signed(fixtures.COORD, "RoundOpenV2", opening), + "finalize": network.signed(fixtures.COORD, "Finalize", final), + "inputs": record["inputs"], + "tape_hash": applied["tape_hash"], + "prev_state": applied["prev_state"], + "predecessor_tape_hash": applied["predecessor_tape_hash"], + "reference_reward_units": network.store._services(manifest.run_id())[ + 0 + ].policy.R_collectible_units, + } + ] + checkpoint = network.store.state_dir / "checkpoint" + write_network_checkpoint(checkpoint, fixtures.COORD, network.store.objects, manifest, rounds) + assert verify_network_checkpoint(checkpoint, fixtures.COORD.ss58) == [] + assert main(["verify-checkpoint-v2", str(checkpoint), "--signer", fixtures.COORD.ss58]) == 0 + _funded_dispute_watch(network) + + +def _exact_network_client(network, relay, relay_app, *, w=0, all_miners=False): + """Exact product client, real HTTP master plus real TLS relay, no authority mocks.""" + import contextlib + import io + import socket + import ssl + import subprocess + import threading + + import httpx + import uvicorn + + from hypertrain.miner.cli import main + + class ReadyServer(uvicorn.Server): + def __init__(self, config): + super().__init__(config) + self.ready = threading.Event() + + async def startup(self, sockets=None): + await super().startup(sockets=sockets) + self.ready.set() + + @contextlib.contextmanager + def server(app, cert=None, key=None): + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + config = uvicorn.Config(app, log_level="error", ssl_certfile=cert, ssl_keyfile=key) + process = ReadyServer(config) + thread = threading.Thread(target=lambda: process.run(sockets=[sock]), daemon=True) + thread.start() + assert process.ready.wait(5) + try: + yield port + finally: + process.should_exit = True + thread.join(timeout=5) + assert not thread.is_alive() + + directory = network.store.state_dir / "tls-client" + directory.mkdir(exist_ok=True) + cert, key = directory / "cert.pem", directory / "server.key" + subprocess.run( + [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-days", + "1", + "-keyout", + str(key), + "-out", + str(cert), + "-subj", + "/CN=localhost", + "-addext", + "subjectAltName=DNS:localhost", + ], + check=True, + capture_output=True, + ) + original_client = httpx.AsyncClient + with ( + server(relay_app, str(cert), str(key)) as relay_port, + server(network.client.app) as master_port, + ): + + async def route(request): + if request.url.host == "relay.test": + request.url = request.url.copy_with(host="localhost", port=relay_port) + + def real_async(*args, **kw): + kw.setdefault("verify", ssl.create_default_context(cafile=str(cert))) + kw["event_hooks"] = {"request": [route]} + return original_client(*args, **kw) + + httpx.AsyncClient = real_async + try: + for index in range(4 if all_miners else 1): + file = directory / f"miner-{index}.key" + file.write_bytes(bytes([80 + index]) * 32) + file.chmod(0o600) + config = directory / f"miner-{index}.toml" + config.write_text( + "\n".join( + f"{name} = {json.dumps(str(value))}" + for name, value in { + "api": f"http://127.0.0.1:{master_port}", + "keyfile": file, + "workdir": directory / "work", + "state_source": "cpu", + "image_digest": network.manifest.training.reference_spec.image_digest, + "run_id": network.manifest.run_id(), + "owner_hotkey": fixtures.OWNER.ss58, + "device": "cpu", + }.items() + ) + ) + output = io.StringIO() + with contextlib.redirect_stdout(output): + assert main(["run-v2", "--config", str(config), "--round", str(w)]) == 0 + assert json.loads(output.getvalue()) == {"status": "UPLOADED"} + assert network.store._record_v2( + network.manifest.run_id(), "delta", f"{w}:" + fixtures.HOT[index].ss58 + ) + finally: + httpx.AsyncClient = original_client + + +def _funded_dispute_watch(network): + import httpx + + from hypertrain.auditor.island_bisect import IslandParty + from hypertrain.data.store import LocalFSStore + from hypertrain.miner.dispute_watch import DisputeWatch, HttpDisputeTransport, PublishedStates + from hypertrain.protocol.hashing import sha256_hex + from hypertrain.protocol.jcs import canonicalize + from hypertrain.protocol.messages_v2 import DisputeV2, EscrowLock, IslandJobV1, ResolutionV2 + + hot = fixtures.HOT[0] + row = network.store._db.execute( + "SELECT data FROM records_v2 WHERE kind='contest' AND json_extract(data,'$.miner')=?", + (hot.ss58,), + ).fetchone() + contest = json.loads(row[0]) + verdict_hash = contest["verdict_hash"] + dispute_id = sha256_hex(canonicalize([network.manifest.run_id(), verdict_hash, hot.ss58])) + origin = network.store._db.execute( + "SELECT origin FROM escrow_units WHERE owner=? AND bucket='available' AND units>=100", + (fixtures.COLD[0].ss58,), + ).fetchone()[0] + lock = EscrowLock( + operation_id="cd" * 32, + owner=fixtures.COLD[0].ss58, + units=100, + origin_ids=[origin], + admission_id=None, + dispute_id=dispute_id, + kind="LOCK_CONTEST", + ) + response = network.client.post( + network.url + "/escrow/lock", json=network.signed(fixtures.COLD[0], "EscrowLock", lock) + ) + assert response.status_code == 200, response.text + response = network.client.post( + network.url + "/dispute", + json=network.signed( + hot, + "DisputeV2", + DisputeV2( + hotkey=hot.ss58, + verdict_hash=verdict_hash, + action="contest", + bond_lock=lock.operation_id, + ), + ), + ) + assert response.status_code == 200, response.text + # A signed Resolution still cannot manufacture registered referee evidence. + turn = response.json() + forged = ResolutionV2( + dispute_id=dispute_id, + transcript_hash=turn["transcript_hash"], + reason="MATCH", + loser=None, + evidence_hash="0" * 64, + ) + response = network.client.post( + network.url + "/resolution", json=network.signed(fixtures.REFEREE, "ResolutionV2", forged) + ) + assert response.status_code == 409 and "UNVERIFIED_REFEREE_EVIDENCE" in response.text + geometry = network.store._record_v2(network.manifest.run_id(), "trace-geometry", verdict_hash) + job = IslandJobV1.model_validate(geometry["job"]) + published = Path(geometry["directory"]) + + def request(req): + response = network.client.request( + req.method, + str(req.url).replace("https://service.test", ""), + content=req.content, + headers=dict(req.headers), + ) + return httpx.Response( + response.status_code, content=response.content, headers=dict(response.headers) + ) + + with httpx.Client( + transport=httpx.MockTransport(request), + headers={ + "X-Dispute-Signature": hot.sign( + f"hypertrain/watch/2|{network.manifest.run_id()}|{hot.ss58}".encode() + ).hex() + }, + ) as client: + states = PublishedStates( + job, published, LocalFSStore(network.store.state_dir / "watch-objects") + ) + watcher = DisputeWatch( + network.store.state_dir / "watch-client", + hot, + HttpDisputeTransport(client, "https://service.test" + network.url), + run_id=network.manifest.run_id(), + coordinator=fixtures.COORD.ss58, + party=lambda _: IslandParty.published(hot.ss58, job, published), + state_serve=states.serve, + beacon=lambda: network.now, + ) + assert watcher.run_once(timeout=0) == 1 + response = network.client.post( + network.url + f"/admin/disputes/{dispute_id}/state/0", headers=fixtures.admin() + ) + assert response.status_code == 200, response.text + assert watcher.run_once(timeout=0) == 1 + watcher.close() + assert network.store._db.execute("SELECT COUNT(*) FROM dispute_states_v2").fetchone()[0] == 1 + response = network.client.post( + network.url + f"/admin/disputes/{dispute_id}/referee", headers=fixtures.admin() + ) + assert response.status_code == 200, response.text + assert response.json()["evidence"]["reason"] == "MATCH" + current = network.store._services(network.manifest.run_id())[2].get(dispute_id) + resolution = ResolutionV2( + dispute_id=dispute_id, + transcript_hash=current.transcript_hash, + reason="MATCH", + loser=None, + evidence_hash=response.json()["evidence_hash"], + ) + response = network.client.post( + network.url + "/resolution", + json=network.signed(fixtures.REFEREE, "ResolutionV2", resolution), + ) + assert response.status_code == 200, response.text + response = network.client.post( + network.url + "/admission/recover", + json=network.signed( + hot, "Receipt", {"w": 0, "commit_hash": dispute_id, "received_round": network.now} + ), + ) + assert response.status_code == 200, response.text + assert response.json()["record"]["pending_dispute"] is False + assert network.store._services(network.manifest.run_id())[0].balances().dispute_locked == 0 + _funded_auditor_fraud(network) + # Subsequent independently funded miner contest: delivered auditor turn expires, + # two permitted healthy witnesses attribute AUDITOR_FAULT, never miner fraud. + from hypertrain.challenge.disputes_v2 import Availability, EventAck, signed_message + + hot2 = fixtures.HOT[1] + row = network.store._db.execute( + "SELECT data FROM records_v2 WHERE kind='contest' AND json_extract(data,'$.miner')=?", + (hot2.ss58,), + ).fetchone() + second = json.loads(row[0]) + second_id = sha256_hex( + canonicalize([network.manifest.run_id(), second["verdict_hash"], hot2.ss58]) + ) + origin = network.store._db.execute( + "SELECT origin FROM escrow_units WHERE owner=? AND bucket='available' AND units>=100", + (fixtures.COLD[1].ss58,), + ).fetchone()[0] + lock2 = EscrowLock( + operation_id="ce" * 32, + owner=fixtures.COLD[1].ss58, + units=100, + origin_ids=[origin], + admission_id=None, + dispute_id=second_id, + kind="LOCK_CONTEST", + ) + response = network.client.post( + network.url + "/escrow/lock", json=network.signed(fixtures.COLD[1], "EscrowLock", lock2) + ) + assert response.status_code == 200, response.text + response = network.client.post( + network.url + "/dispute", + json=network.signed( + hot2, + "DisputeV2", + DisputeV2( + hotkey=hot2.ss58, + verdict_hash=second["verdict_hash"], + action="contest", + bond_lock=lock2.operation_id, + ), + ), + ) + assert response.status_code == 200, response.text + disputes = network.store._services(network.manifest.run_id())[2] + turn = disputes.get(second_id) + party = IslandParty.published(hot2.ss58, job, published) + from hypertrain.protocol.messages_v2 import BisectV2 + + answer = BisectV2( + dispute_id=second_id, + seq=turn.seq, + level=turn.level, + ctx=turn.ctx, + interval=turn.interval, + N=2, + hashes=party.hashes( + turn.level, + tuple(turn.ctx), + [turn.interval[0], -(-sum(turn.interval) // 2), turn.interval[1]], + ), + party=hot2.ss58, + previous_transcript_hash=turn.transcript_hash, + ) + # Geometry from this miner's original published path, not another assignment. + own_geometry = network.store._record_v2( + network.manifest.run_id(), "trace-geometry", second["verdict_hash"] + ) + own_party = IslandParty.published( + hot2.ss58, IslandJobV1.model_validate(own_geometry["job"]), Path(own_geometry["directory"]) + ) + answer = answer.model_copy( + update={ + "hashes": own_party.hashes( + turn.level, + tuple(turn.ctx), + [turn.interval[0], -(-sum(turn.interval) // 2), turn.interval[1]], + ) + } + ) + response = network.client.post( + network.url + "/bisect", json=network.signed(hot2, "BisectV2", answer) + ) + assert response.status_code == 200, response.text + turn = disputes.get(second_id) + auditor = next(k for k in fixtures.AUDITORS if k.ss58 == turn.expected_party) + events = disputes.events(auditor.ss58, 0) + event = next(e for e in events if e.turn.dispute_id == second_id and e.turn.seq == turn.seq) + ack = EventAck( + run_id=network.manifest.run_id(), + cursor=event.cursor, + event_hash=sha256_hex(canonicalize(event.body())), + party=auditor.ss58, + received_beacon=network.now, + sig="0" * 128, + ) + ack = ack.model_copy(update={"sig": auditor.sign(signed_message(ack)).hex()}) + response = network.client.post(network.url + "/disputes/ack", json=ack.body()) + assert response.status_code == 200, response.text + network.push(turn.turn_deadline + 1) + witnesses = [] + for key in (fixtures.COORD, fixtures.REFEREE): + witness = Availability( + run_id=network.manifest.run_id(), + dispute_id=second_id, + seq=turn.seq, + transcript_hash=turn.transcript_hash, + observed_beacon=network.now, + service_healthy=True, + beacon_healthy=True, + reference_healthy=True, + signer=key.ss58, + sig="0" * 128, + ) + witnesses.append( + witness.model_copy(update={"sig": key.sign(signed_message(witness)).hex()}).body() + ) + response = network.client.post( + network.url + f"/admin/disputes/{second_id}/timeout", + json={"witnesses": witnesses}, + headers=fixtures.admin(), + ) + assert response.status_code == 200, response.text + assert response.json()["resolution"]["reason"] == "AUDITOR_FAULT" + assert network.store._services(network.manifest.run_id())[0].balances().burned == 0 + + +def test_checkpoint_and_cli_reject_full_seventeen_roster_before_store(tmp_path): + from hypertrain.aggregator.checkpoint import CheckpointError, write_network_checkpoint + from hypertrain.aggregator.cli import main + from hypertrain.protocol.hashing import sha256_hex + from hypertrain.protocol.jcs import canonicalize + from hypertrain.protocol.keys import Keypair + from hypertrain.protocol.messages import f32hex + from hypertrain.protocol.messages_v2 import PolicyHashes, RoundOpenV2 + + s = fixtures.fixture.setup() + roster = [ + { + "hotkey": Keypair(bytes([i + 1]) * 32).ss58, + "slot": i, + "q_i": f32hex(1), + "admission_id": sha256_hex(str(i).encode()), + "coldkey_group": f"o{i}", + "state": "ACTIVE", + "eligible_weight": 4194304, + } + for i in range(17) + ] + opening = RoundOpenV2( + w=0, + prev_final_hash="0" * 64, + theta_hash="0" * 64, + outer_state_hash="0" * 64, + center_hash="0" * 64, + roster_hash=sha256_hex(canonicalize(roster)), + honeypot_commit="0" * 64, + d_open=2, + d_assign=3, + d_commit=4, + d_audit=5, + d_upload=6, + d_final=20, + contract_version=2, + policy_hashes=PolicyHashes.model_validate( + {k: getattr(s.manifest.network, k) for k in PolicyHashes.model_fields} + ), + registry_epoch=0, + start_state_index_hash="0" * 64, + audit_mode="anchored-full", + roster=roster, + ) + rounds = [{"round_open": {"body": opening.body()}}] + + class ForbiddenStore: + def get(self, *args): + raise AssertionError("storage touched before complete roster gate") + + # Given / When / Then + with pytest.raises(CheckpointError, match="ROSTER_LIMIT_16"): + write_network_checkpoint( + tmp_path / "checkpoint", fixtures.COORD, ForbiddenStore(), s.manifest, rounds + ) + assert not (tmp_path / "checkpoint").exists() + bundle = tmp_path / "bundle.json" + bundle.write_text(json.dumps({"manifest": s.manifest.body(), "rounds": rounds})) + for command in ("aggregate-v2", "checkpoint-v2"): + args = [command, str(bundle)] + if command == "checkpoint-v2": + args.append(str(tmp_path / "out")) + args += ["--objects", str(tmp_path / "not-created"), "--keyfile", str(tmp_path / "absent")] + with pytest.raises(CheckpointError, match="ROSTER_LIMIT_16"): + main(args) + assert not (tmp_path / "not-created").exists() + + +def _funded_auditor_fraud(network): + from hypertrain.auditor.island_bisect import IslandParty + from hypertrain.protocol.hashing import sha256_hex + from hypertrain.protocol.jcs import canonicalize + from hypertrain.protocol.messages_v2 import ( + BisectV2, + DisputeV2, + EscrowLock, + IslandJobV1, + ResolutionV2, + ) + + run_id = network.manifest.run_id() + escrow, _, disputes = network.store._services(run_id) + hot, cold = fixtures.HOT[2], fixtures.COLD[2] + row = network.store._db.execute( + "SELECT data FROM records_v2 WHERE kind='contest' AND json_extract(data,'$.miner')=?", + (hot.ss58,), + ).fetchone() + contest = json.loads(row[0]) + dispute_id = sha256_hex(canonicalize([run_id, contest["verdict_hash"], hot.ss58])) + origin = network.store._db.execute( + "SELECT origin FROM escrow_units WHERE owner=? AND bucket='available' AND units>=100", + (cold.ss58,), + ).fetchone()[0] + before = escrow.balances(cold.ss58) + lock = EscrowLock( + operation_id="cf" * 32, + owner=cold.ss58, + units=100, + origin_ids=[origin], + admission_id=None, + dispute_id=dispute_id, + kind="LOCK_CONTEST", + ) + response = network.client.post( + network.url + "/escrow/lock", json=network.signed(cold, "EscrowLock", lock) + ) + assert response.status_code == 200, response.text + response = network.client.post( + network.url + "/dispute", + json=network.signed( + hot, + "DisputeV2", + DisputeV2( + hotkey=hot.ss58, + verdict_hash=contest["verdict_hash"], + action="contest", + bond_lock=lock.operation_id, + ), + ), + ) + assert response.status_code == 200, response.text + geometry = network.store._record_v2(run_id, "trace-geometry", contest["verdict_hash"]) + party = IslandParty.published( + hot.ss58, IslandJobV1.model_validate(geometry["job"]), Path(geometry["directory"]) + ) + for _ in range(64): + turn = disputes.get(dispute_id) + if turn.paused: + break + key = ( + hot + if turn.expected_party == hot.ss58 + else next(k for k in fixtures.AUDITORS if k.ss58 == turn.expected_party) + ) + at = [turn.interval[0], -(-sum(turn.interval) // 2), turn.interval[1]] + hashes = party.hashes(turn.level, tuple(turn.ctx), at) + if key != hot: + hashes = [ + digest if index == 0 else sha256_hex(bytes.fromhex(digest)) + for index, digest in zip(at, hashes, strict=True) + ] + answer = BisectV2( + dispute_id=dispute_id, + seq=turn.seq, + level=turn.level, + ctx=turn.ctx, + interval=turn.interval, + N=2, + hashes=hashes, + party=key.ss58, + previous_transcript_hash=turn.transcript_hash, + ) + response = network.client.post( + network.url + "/bisect", json=network.signed(key, "BisectV2", answer) + ) + assert response.status_code == 200, response.text + turn = disputes.get(dispute_id) + assert turn.paused and turn.level == "op" and tuple(turn.interval) == (0, 1) + response = network.client.post( + network.url + f"/admin/disputes/{dispute_id}/referee", headers=fixtures.admin() + ) + assert response.status_code == 200, response.text + evidence = response.json() + assert evidence["evidence"]["reason"] == "FRAUD" + assert evidence["evidence"]["loser"] == turn.contest.auditor + resolution = ResolutionV2( + dispute_id=dispute_id, + transcript_hash=turn.transcript_hash, + reason="FRAUD", + loser=turn.contest.auditor, + evidence_hash=evidence["evidence_hash"], + ) + # A re-signed wrong loser cannot consume the independent evidence or collateral. + response = network.client.post( + network.url + "/resolution", + json=network.signed( + fixtures.REFEREE, "ResolutionV2", resolution.model_copy(update={"loser": hot.ss58}) + ), + ) + assert response.status_code == 409, response.text + for _ in range(2): + response = network.client.post( + network.url + "/resolution", + json=network.signed(fixtures.REFEREE, "ResolutionV2", resolution), + ) + assert response.status_code == 200, response.text + accepted = disputes.settlement(lock.operation_id) + assert ( + accepted.loser, + accepted.lock_id, + accepted.lock_owner, + accepted.referee, + accepted.resolution_hash, + ) == ( + turn.contest.auditor, + lock.operation_id, + cold.ss58, + fixtures.REFEREE.ss58, + resolution.digest(), + ) + assert accepted.outcome == "MATCH" and not accepted.unresolved + after = escrow.balances(cold.ss58) + assert after.burned == before.burned and after.available == before.available + assert after.dispute_locked == before.dispute_locked + + +def test_four_actual_funded_identities_graduate_shadow_only(network, monkeypatch): + # Given: four independent coldkeys, conserved funded origin locks. + snapshot = os.environ.get("HT_NETWORK_VERIFIED_SNAPSHOT") + if snapshot: + source = Path(snapshot) + existing = sqlite3.connect(source / "challenge.db") + run_id, raw = existing.execute("SELECT run_id,manifest FROM runs").fetchone() + from hypertrain.protocol.messages_v2 import RunManifestV2 + + network.manifest = RunManifestV2.model_validate_json(raw) + assert run_id == network.manifest.run_id() + rows = existing.execute( + "SELECT hotkey,state,clean_count,admission_id FROM admissions_v2 ORDER BY hotkey" + ).fetchall() + assert {r[0] for r in rows} == {k.ss58 for k in fixtures.HOT} + assert all(r[1] == "ACTIVE" and r[2] == 12 for r in rows) + for _, _, _, admission_id in rows: + assert ( + existing.execute( + "SELECT COUNT(*) FROM admission_trial_results r " + "JOIN admission_trials t ON t.epoch=r.epoch " + "WHERE t.admission_id=? AND r.outcome='MATCH'", + (admission_id,), + ).fetchone()[0] + == 12 + ) + network.store.close_v2_notifications() + network.store._services_v2.clear() + network.store._db.execute("PRAGMA foreign_keys=OFF") + existing.backup(network.store._db) + network.store._db.execute("PRAGMA foreign_keys=ON") + existing.close() + # Public training cursor is recomputed from independently verified live objects. + network.store._db.execute("DELETE FROM records_v2 WHERE kind NOT IN ('dataset')") + network.store._db.execute("DELETE FROM accepted_v2") + network.store._db.execute("DELETE FROM audit_leases_v2") + network.store._db.execute("DELETE FROM escrow_finalized") + network.store._db.execute( + "DELETE FROM beacon_pins_v2 WHERE operation_id LIKE 'open:%' " + "OR operation_id LIKE 'assignment:%' OR operation_id LIKE 'audit:%' " + "OR operation_id LIKE 'aggregate:%' OR operation_id LIKE 'finalize:%'" + ) + for directory in ("objects", "trials-v2"): + shutil.copytree( + source / directory, network.store.state_dir / directory, dirs_exist_ok=True + ) + network.now = network.store._now(network.store._db) + import time + + # Historical signed authority keeps its original beacon/expiry. Translate + # only the test wall clock; hard timers retain real monotonic durations. + baseline = time.monotonic() + snapshot_time = network.manifest.training.beacon.genesis_time + (network.now - 1) * 3 + offset = time.time() - snapshot_time + clock_dir = network.store.state_dir / "fixture-clock" + clock_dir.mkdir() + (clock_dir / "sitecustomize.py").write_text( + "import os,time\n" + "_real_time=time.time\n" + "_offset=float(os.environ['HT_FIXTURE_CLOCK_OFFSET'])\n" + "time.time=lambda: _real_time()-_offset\n" + ) + monkeypatch.setenv("HT_FIXTURE_CLOCK_OFFSET", str(offset)) + monkeypatch.setenv( + "PYTHONPATH", str(clock_dir) + os.pathsep + os.environ.get("PYTHONPATH", "") + ) + monkeypatch.setattr(time, "time", lambda: snapshot_time + time.monotonic() - baseline) + network.push(network.now) + network.store.clock = lambda: ( + network.manifest.training.beacon.genesis_time + (network.now - 1) * 3 + ) + _live_checkpoint(network) + return + _graduate_four(network) + _live_checkpoint(network) + + +def _graduate_four(network): + """Execute all required signed references/proofs/finalizations, no authority shortcuts.""" + from hypertrain.auditor.replay import unpack_state + from hypertrain.data.trial_assignment import trial_samples + from hypertrain.miner.island_launch import validate_artifacts + from hypertrain.protocol.messages import Finalize + from hypertrain.protocol.messages_v2 import EscrowLock, JoinChallenge, WorkProof, WorkScreenV2 + from hypertrain.trainer.compress import state_hash + + identities = [] + for i in range(4): + current = network.store._db.execute( + "SELECT * FROM admissions_v2 WHERE hotkey=?", (fixtures.HOT[i].ss58,) + ).fetchone() + if current is not None and current["clean_count"] == 12: + assert current["state"] == "ACTIVE" + continue + if i and current is None: + network.push(network.now + 101) + identity = current["admission_id"] if current else network.join(i).json()["admission_id"] + identities.append(identity) + lock = EscrowLock( + operation_id=f"{i + 1:064x}", + owner=fixtures.COLD[i].ss58, + units=1000, + origin_ids=[network.origin_ids[i]], + admission_id=identity, + dispute_id=None, + kind="LOCK_ADMISSION", + ) + if current is None: + response = network.client.post( + network.url + "/escrow/lock", + json=network.signed(fixtures.COLD[i], "EscrowLock", lock), + ) + assert response.status_code == 200, response.text + else: + status = network.store._services(network.manifest.run_id())[1].status( + fixtures.HOT[i].ss58, now=network.now + ) + assert status.funding.locked_units == 1000 + # When: twelve actual full reference executions per identity, exact proof/finality. + for _trial in range(current["clean_count"] if current else 0, 12): + pending = network.store._db.execute( + "SELECT r.outcome FROM admissions_v2 a JOIN admission_trial_results r " + "ON r.epoch=a.trial_epoch WHERE a.admission_id=?", + (identity,), + ).fetchone() + if pending is None or pending["outcome"] != "OPEN": + network.push(network.now + 1) + response = network.client.get(network.url + "/join/" + identity + "/challenge") + assert response.status_code == 200, response.text + challenge = JoinChallenge.model_validate(response.json()["body"]) + response = network.client.post( + network.url + "/admin/join/" + identity + "/reference", headers=fixtures.admin() + ) + assert response.status_code == 200, response.text + row = network.store._db.execute( + "SELECT * FROM admissions_v2 WHERE admission_id=?", (identity,) + ).fetchone() + proof = WorkProof.model_validate_json(row["reference_json"]) + job, directory = network.store.stage_trial_v2( + network.manifest.run_id(), + challenge, + tuple( + trial_samples( + network.manifest, + identity, + challenge.nonce, + network.store._beacon_v2(challenge.seed_beacon), + ) + ), + row["trial_epoch"], + ) + artifacts = validate_artifacts(job, directory / "published") + from hypertrain.protocol.hashing import sha256_hex + + assert [ + sha256_hex(path.read_bytes()) + for path in (artifacts.state, artifacts.ef, artifacts.delta, artifacts.leaves) + ] == [ref.sha256 for ref in proof.artifact_refs] + screen = WorkScreenV2( + evidence_kind="work-proof-v1", + admission_id=identity, + nonce=challenge.nonce, + policy_hash=challenge.policy_hash, + image_digest=job.manifest.training.reference_spec.image_digest, + layout=challenge.layout, + challenge_hash=challenge.digest(), + rank_results=list(artifacts.ranks), + artifact_hashes=[r.sha256 for r in proof.artifact_refs], + ) + response = network.client.post( + network.url + "/join/proof", + json={ + "proof": network.signed(fixtures.HOT[i], "WorkProof", proof), + "screen": network.signed(fixtures.HOT[i], "WorkScreenV2", screen), + }, + ) + assert response.status_code == 200, response.text + theta, _ = unpack_state(network.store.objects.get(proof.artifact_refs[0].sha256)) + final = Finalize( + w=row["trial_epoch"], + final_theta_hash_w1=state_hash(theta), + included=[fixtures.HOT[i].ss58], + entitlements_root=proof.digest(), + ) + response = network.client.post( + network.url + "/admin/join/" + identity + "/finalize", + json=network.signed(fixtures.COORD, "Finalize", final), + headers=fixtures.admin(), + ) + assert response.status_code == 200, response.text + if _trial == 0: + assert response.json()["shadow_only"] is True + assert response.json()["eligible"] is False + assert ( + network.store._services(network.manifest.run_id())[0].balances().issued == 40000 + ) + # Then: actual ACTIVE admission, funded locks unchanged, no shadow mint/global advance. + assert response.json()["record"]["state"] == "ACTIVE" + assert response.json()["record"]["clean_count"] == 12 + assert ( + network.store._db.execute( + "SELECT COUNT(*) FROM records_v2 WHERE kind='applied'" + ).fetchone()[0] + == 0 + ) + assert network.store._services(network.manifest.run_id())[0].balances().issued == 40000 + + +@pytest.mark.parametrize("od_carry_network", ["mlm", "decision", "distill"], indirect=True) +def test_od_four_active_actual_client_two_round_carry_checkpoint(od_carry_network, monkeypatch): + """Production service path with same-manifest admission and actual HTTP/TLS miners.""" + network = od_carry_network + _od_verified_admissions(network, monkeypatch) + _od_production_rounds(network) + + +def _od_verified_admissions(network, monkeypatch): + """Reuse only an authenticated matching manifest's original finalized work.""" + import hashlib + import time + + from hypertrain.challenge.store import ChallengeError, ChallengeStore + from hypertrain.miner.island_launch import validate_artifacts + from hypertrain.protocol import envelope_v2 + from hypertrain.protocol.keys import Keypair + from hypertrain.protocol.messages_v2 import ( + JoinChallenge, + RunManifestV2, + WorkProof, + WorkScreenV2, + ) + + objective = network.manifest.training.model.od.objective + snapshot = Path(tempfile.gettempdir()) / f"hypertrain-od-carry-{objective}-active" + resume = os.environ.get("HT_OD_ADMISSION_RESUME") + source = snapshot if snapshot.exists() else Path(resume) if resume else None + if source is not None: + with ( + sqlite3.connect(f"file:{source / 'challenge.db'}?mode=ro", uri=True) as saved, + sqlite3.connect(":memory:") as compatible, + ): + run_id, raw = saved.execute("SELECT run_id,manifest FROM runs").fetchone() + manifest = RunManifestV2.model_validate_json(raw) + expected = network.manifest.body() + expected["training"]["beacon"]["genesis_time"] = manifest.training.beacon.genesis_time + assert manifest.body() == expected and manifest.run_id() == run_id + network.store.close_v2_notifications() + network.store._services_v2.clear() + saved.backup(compatible) + namespace = compatible.execute( + "SELECT value FROM meta WHERE key='admission-prefix-namespace-v2'" + ).fetchone() + if namespace is None: + # Test-only history import: preserve unknown legacy quotas, never translate them. + legacy = compatible.execute( + "SELECT * FROM admission_quota ORDER BY identity" + ).fetchall() + compatible.execute( + "ALTER TABLE admission_quota RENAME TO fixture_legacy_admission_quota" + ) + compatible.commit() + assert ( + compatible.execute( + "SELECT * FROM fixture_legacy_admission_quota ORDER BY identity" + ).fetchall() + == legacy + ) + compatible.backup(network.store._db) + network.manifest = manifest + for name in ("objects", "trials-v2"): + shutil.copytree(source / name, network.store.state_dir / name, dirs_exist_ok=True) + network.now = network.store._now(network.store._db) + baseline = time.monotonic() + historical = manifest.training.beacon.genesis_time + (network.now - 1) * 3 + offset = time.time() - historical + clock = network.store.state_dir / "fixture-clock" + clock.mkdir() + (clock / "sitecustomize.py").write_text( + "import os,time\n" + "_real_time=time.time\n" + "_offset=float(os.environ['HT_FIXTURE_CLOCK_OFFSET'])\n" + "time.time=lambda: _real_time()-_offset\n" + ) + monkeypatch.setenv("HT_FIXTURE_CLOCK_OFFSET", str(offset)) + monkeypatch.setenv("PYTHONPATH", str(clock) + os.pathsep + os.environ.get("PYTHONPATH", "")) + monkeypatch.setattr(time, "time", lambda: historical + time.monotonic() - baseline) + network.push(network.now) + if not snapshot.exists(): + _graduate_four(network) + snapshot.mkdir() + with sqlite3.connect(snapshot / "challenge.db") as saved: + network.store._db.backup(saved) + for name in ("objects", "trials-v2"): + shutil.copytree(network.store.state_dir / name, snapshot / name) + + admission = network.store._services(network.manifest.run_id())[1] + namespace = network.store._db.execute( + "SELECT value FROM meta WHERE key='admission-prefix-namespace-v2'" + ).fetchone()[0] + assert namespace == hashlib.sha256(admission.ip_secret).hexdigest() + assert network.store._prefix_secret_v2() == admission.ip_secret + reopened = ChallengeStore( + network.store.state_dir, + network.store.params, + Keypair(b"\xcc" * 32), + owner_hotkey=network.store.owner_hotkey, + verify_beacon=network.store.verify_beacon, + objects=network.store.objects, + ) + try: + with pytest.raises(ChallengeError, match="admission prefix namespace differs"): + reopened._prefix_secret_v2() + finally: + reopened.close_v2_notifications() + reopened._db.close() + rows = network.store._db.execute("SELECT * FROM admissions_v2 ORDER BY hotkey").fetchall() + assert {r["hotkey"] for r in rows} == {key.ss58 for key in fixtures.HOT} + for row in rows: + status = admission.status(row["hotkey"], now=network.now) + assert status.record.state == "ACTIVE" and status.record.clean_count == 12 + assert status.record.canary_blocks == (1, 2, 3) + assert status.eligible and not status.shadow_only + assert status.funding.locked_units == 1000 + trials = network.store._db.execute( + "SELECT t.*,r.challenge,r.reference,r.proof,r.screen,r.outcome " + "FROM admission_trials t JOIN admission_trial_results r ON r.epoch=t.epoch " + "WHERE t.admission_id=? ORDER BY t.epoch", + (row["admission_id"],), + ).fetchall() + assert len(trials) == 12 + for trial in trials: + challenge = JoinChallenge.model_validate_json(trial["challenge"]) + proof = WorkProof.model_validate_json(trial["reference"]) + signed = envelope_v2.parse_envelope(trial["proof"]) + screen_env = envelope_v2.parse_envelope(trial["screen"]) + assert envelope_v2.verify_envelope(signed.model_dump()) + assert envelope_v2.verify_envelope(screen_env.model_dump()) + assert signed.run_id == screen_env.run_id == network.manifest.run_id() + assert signed.signer == screen_env.signer == row["hotkey"] + assert WorkProof.model_validate(signed.body) == proof + assert proof.digest() == trial["evidence_hash"] and trial["outcome"] == "MATCH" + from hypertrain.data.trial_assignment import trial_samples + + job, directory = network.store.stage_trial_v2( + network.manifest.run_id(), + challenge, + tuple( + trial_samples( + network.manifest, + row["admission_id"], + challenge.nonce, + network.store._beacon_v2(challenge.seed_beacon), + ) + ), + trial["epoch"], + ) + artifacts = validate_artifacts(job, directory / "published") + assert ( + list(artifacts.ranks) == WorkScreenV2.model_validate(screen_env.body).rank_results + ) + from hypertrain.protocol.hashing import sha256_hex + + assert [ + sha256_hex(path.read_bytes()) + for path in (artifacts.state, artifacts.ef, artifacts.delta, artifacts.leaves) + ] == [ref.sha256 for ref in proof.artifact_refs] + final_raw = network.store._db.execute( + "SELECT receipt FROM admission_reservations WHERE reservation=?", + (f"trial-final|{row['admission_id']}|{trial['epoch']}",), + ).fetchone()[0] + final = envelope_v2.parse_envelope(final_raw) + assert envelope_v2.verify_envelope(final.model_dump()) + assert final.signer == fixtures.COORD.ss58 and final.run_id == network.manifest.run_id() + assert final.body["entitlements_root"] == proof.digest() + assert final.body["w"] == trial["epoch"] + assert final.body["included"] == [row["hotkey"]] + from hypertrain.auditor.replay import unpack_state + from hypertrain.trainer.compress import state_hash + + assert final.body["final_theta_hash_w1"] == state_hash( + unpack_state(artifacts.state.read_bytes())[0] + ) + assert network.store._services(network.manifest.run_id())[0].balances().issued == 40000 + assert ( + network.store._db.execute( + "SELECT COUNT(*) FROM records_v2 WHERE kind IN ('round','applied','finality')" + ).fetchone()[0] + == 0 + ) + + +def _od_production_rounds(network, *, after_audits=None): + """Two real carried client rounds, independent full audits, signed replay checkpoint.""" + import anyio + import httpx + import torch + + from hypertrain.aggregator.checkpoint import ( + network_inputs, + verify_network_checkpoint, + write_network_checkpoint, + ) + from hypertrain.aggregator.core import load_state + from hypertrain.aggregator.tape_v2 import TapeV2, replay_tape + from hypertrain.auditor.replay import AnchorCache, optimizer_hash, pack_state, unpack_state + from hypertrain.data.store import LocalFSStore + from hypertrain.data.stream_store import StreamStore + from hypertrain.protocol import envelope_v2 + from hypertrain.protocol.hashing import sha256_hex + from hypertrain.protocol.jcs import canonicalize + from hypertrain.protocol.messages import Finalize, f32hex + from hypertrain.protocol.messages_v2 import ( + AggregationPolicyV2, + PolicyHashes, + RoundOpenV2, + StartStateV2, + ) + from hypertrain.relay.app import create_app + from hypertrain.relay.core import Relay + from hypertrain.trainer.compress import state_hash + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.model import init_params + + manifest = network.manifest + run_id = manifest.run_id() + cfg = TrainConfig.from_manifest_v2(manifest) + assert cfg.inner.state_policy == "carry" + rounds, terminal = [], {} + for w in range(2): + cache = AnchorCache() + theta = init_params(cfg.model) + predecessor, prev_final = "0" * 64, "0" * 64 + if w: + previous = network.store._record_v2(run_id, "applied", str(w - 1)) + theta = { + k: torch.from_numpy(v.copy()) + for k, v in load_state(network.store.objects, previous["out_state"]).theta.items() + } + predecessor = previous["tape_hash"] + prev_final = sha256_hex(canonicalize(rounds[-1]["finalize"]["body"])) + starts, roster = [], [] + for i, hot in enumerate(fixtures.HOT): + status = network.store._services(run_id)[1].status(hot.ss58, now=network.now) + anchor = ( + network.store._restore_anchor_v2(run_id, hot.ss58, w - 1, cache) + if w + else cache.genesis(manifest, hot.ss58, theta) + ) + assert anchor.state.step == w * cfg.inner.H + if w: + prior_theta, prior_state = unpack_state(terminal[hot.ss58][0]) + prior_ef, _ = unpack_state(terminal[hot.ss58][1]) + assert optimizer_hash(anchor.state) == optimizer_hash(prior_state) + assert pack_state(anchor.theta, anchor.state) == pack_state( + prior_theta, prior_state + ) + assert pack_state(anchor.ef) == pack_state(prior_ef) + starts.append( + StartStateV2( + run_id=run_id, + w=w, + hotkey=hot.ss58, + theta_hash=state_hash(theta), + state_object_sha256=sha256_hex(pack_state(theta, anchor.state)), + opt_state_hash=optimizer_hash(anchor.state), + ef_object_sha256=sha256_hex(pack_state(anchor.ef)), + ef_hash=state_hash(anchor.ef), + parent_anchor_hash=anchor.anchor_hash, + global_step0=anchor.state.step, + anchor_verdict_hash=anchor.proof_hash, + ) + ) + roster.append( + { + "hotkey": hot.ss58, + "slot": i, + "q_i": f32hex(1), + "admission_id": status.record.admission_id, + "coldkey_group": status.record.coldkey, + "state": "ACTIVE", + "eligible_weight": 4194304, + } + ) + opening = RoundOpenV2( + w=w, + prev_final_hash=prev_final, + theta_hash=state_hash(theta), + outer_state_hash="0" * 64, + center_hash="0" * 64, + roster_hash=sha256_hex(canonicalize(roster)), + honeypot_commit="0" * 64, + d_open=network.now + 1, + d_assign=network.now + 2, + d_commit=network.now + 30, + d_audit=network.now + 31, + d_upload=network.now + 200, + d_final=network.now + 300, + contract_version=2, + policy_hashes=PolicyHashes.model_validate( + {k: getattr(manifest.network, k) for k in PolicyHashes.model_fields} + ), + registry_epoch=0, + start_state_index_hash=sha256_hex(canonicalize([s.body() for s in starts])), + audit_mode="anchored-full", + roster=roster, + ) + response = network.client.post( + network.url + "/admin/rounds", + json=network.signed(fixtures.COORD, "RoundOpenV2", opening), + headers=fixtures.admin(), + ) + assert response.status_code == 200, response.text + network.push(opening.d_assign) + + view = network.client.get(network.url + f"/rounds/{w}") + assert view.status_code == 200, view.text + + async def clients(round_index): + async with httpx.AsyncClient() as backing: + + async def now(): + return network.now + + assignment = network.store.relay_assignment_v2( + run_id, round_index, fixtures.HOT[0].ss58 + ) + relay = Relay( + run_id=run_id, + master=fixtures.COORD.ss58, + registry=network.store._registry_v2(manifest), + network_manifest_hash=sha256_hex( + canonicalize(assignment["network_manifest"]["body"]) + ), + observers={key.ss58 for key in fixtures.AUDITORS}, + relay_id="local", + region="local", + keys={"k1": fixtures.RELAY}, + active_key="k1", + backing=StreamStore( + LocalFSStore(network.store.state_dir / f"relay-{round_index}"), backing + ), + now=now, + ) + await anyio.to_thread.run_sync( + lambda: _exact_network_client( + network, relay, create_app(relay, "x" * 32), w=round_index, all_miners=True + ) + ) + + anyio.run(clients, w) + for hot in fixtures.HOT: + identity = f"{w}:{hot.ss58}" + commit = network.store._record_v2(run_id, "commit", identity) + assert envelope_v2.verify_envelope(commit) + assert commit["body"]["tokens"] == manifest.training.batch_samples() * cfg.model.seq_len + assert manifest.training.batch_samples() == ( + cfg.inner.H * cfg.inner.micro_batch * cfg.inner.grad_accum * 2 + ) + published = ( + network.store.state_dir + / "tls-client/work" + / run_id + / hot.ss58 + / str(w) + / "published" + ) + states = [ + (published / f"rank-{rank}/state.safetensors").read_bytes() for rank in range(2) + ] + assert states[0] == states[1] + end_theta, end_state = unpack_state(states[0]) + assert end_state.step == (w + 1) * cfg.inner.H + assert any(torch.count_nonzero(v) for v in end_state.m.values()) + assert state_hash(end_theta) == commit["body"]["final_theta_hash"] + terminal[hot.ss58] = (states[0], (published / "rank-0/ef.safetensors").read_bytes()) + + network.push(opening.d_audit) + response = network.client.post( + network.url + f"/admin/rounds/{w}/audits", headers=fixtures.admin() + ) + assert response.status_code == 200, response.text + for index in range(4): + auditor = fixtures.AUDITORS[index % 2] + response = network.client.post( + network.url + "/worker/lease", + json=network.signed( + auditor, + "Receipt", + { + "w": w, + "commit_hash": f"{w * 4 + index + 1:064x}", + "received_round": network.now, + }, + ), + ) + assert response.status_code == 200, response.text + job = response.json()["body"] + assert job["start_state"]["global_step0"] == w * cfg.inner.H + response = network.client.post( + network.url + "/worker/jobs/" + job["job_id"] + "/execute", + json=network.signed( + auditor, + "Receipt", + {"w": w, "commit_hash": job["lease_nonce"], "received_round": network.now}, + ), + ) + assert response.status_code == 200, response.text + assert response.json()["verdict"]["result"] == "MATCH" + response = network.client.post( + network.url + "/worker/jobs/" + job["job_id"] + "/complete", + json=network.signed(auditor, "ReplayVerdict", response.json()["verdict"]), + ) + assert response.status_code == 200, response.text + if after_audits is not None: + after_audits(network) + return + response = network.client.post( + network.url + f"/admin/rounds/{w}/aggregate", headers=fixtures.admin() + ) + assert response.status_code == 200, response.text + applied = response.json() + tape = TapeV2.from_bytes(network.store.objects.get(applied["tape_hash"])) + weights = tape.body.allocation.entries + assert {e.hotkey for e in weights} == {k.ss58 for k in fixtures.HOT} + assert sum(e.weight_units for e in weights) == 16777216 + assert all(e.weight_units == 4194304 and not e.probation for e in weights) + assert all(x.roster.state == "ACTIVE" for x in tape.body.inputs) + assert tape.body.predecessor_tape_hash == predecessor + inputs = network.store._record_v2(run_id, "tape-inputs", str(w))["inputs"] + replayed = replay_tape( + network.store.objects, + tape, + manifest, + AggregationPolicyV2.model_validate_json( + network.store.objects.get(manifest.network.aggregation_policy_hash) + ), + network.store.objects.get(manifest.network.economics_policy_hash), + signer=fixtures.COORD.ss58, + w=w, + prev_state=applied["prev_state"], + predecessor_tape_hash=predecessor, + inputs=network_inputs(inputs), + reference_reward_units=network.store._services(run_id)[0].policy.R_collectible_units, + ) + assert replayed.to_bytes() == network.store.objects.get(applied["out_state"]) + final = Finalize( + w=w, + final_theta_hash_w1=applied["theta_hash"], + included=sorted(k.ss58 for k in fixtures.HOT), + entitlements_root="0" * 64, + ) + response = network.client.post( + network.url + f"/admin/rounds/{w}/finalize", + json=network.signed(fixtures.COORD, "Finalize", final), + headers=fixtures.admin(), + ) + assert response.status_code == 200, response.text + assert response.json()["status"] == "SETTLED" + balances = network.store._services(run_id)[0].balances() + assert balances.issued == 40000 + (w + 1) * 1000000 + origins = network.store._db.execute( + "SELECT hotkey,issued FROM escrow_origins WHERE reward_round=? ORDER BY hotkey", (w,) + ).fetchall() + assert [(r[0], r[1]) for r in origins] == [ + (hot, 250000) for hot in sorted(k.ss58 for k in fixtures.HOT) + ] + repeated = network.client.post( + network.url + f"/admin/rounds/{w}/finalize", + json=network.signed(fixtures.COORD, "Finalize", final), + headers=fixtures.admin(), + ) + assert repeated.status_code == 200 and repeated.json() == response.json() + assert network.store._services(run_id)[0].balances() == balances + assert network.store._record_v2(run_id, "replayed-finality", str(w))["shadow"] is False + rounds.append( + { + "round_open": network.store._record_v2(run_id, "round", str(w)), + "finalize": network.store._record_v2(run_id, "finalize", str(w)), + "inputs": inputs, + "tape_hash": applied["tape_hash"], + "prev_state": applied["prev_state"], + "predecessor_tape_hash": predecessor, + "reference_reward_units": network.store._services(run_id)[ + 0 + ].policy.R_collectible_units, + } + ) + checkpoint = network.store.state_dir / "od-carry-checkpoint" + body = write_network_checkpoint( + checkpoint, fixtures.COORD, network.store.objects, manifest, rounds + ) + assert body["rounds"] == [0, 1] + assert verify_network_checkpoint(checkpoint, fixtures.COORD.ss58) == [] + + +@pytest.mark.parametrize("od_carry_network", ["mlm"], indirect=True) +def test_public_watch_layer_over_actual_fastapi_tls(od_carry_network, monkeypatch): + """Funded original audits; real public CLI/TLS routes and committed long-poll wake.""" + network = od_carry_network + snapshot = Path(tempfile.gettempdir()) / "hypertrain-od-carry-mlm-active" + assert (snapshot / "challenge.db").is_file(), "original verified48 snapshot required" + _od_verified_admissions(network, monkeypatch) + _od_production_rounds(network, after_audits=lambda n: _public_watch_fastapi_tls(n, monkeypatch)) + + +def _public_watch_fastapi_tls(network, monkeypatch): + import contextlib + import io + import socket + import ssl + import subprocess + import threading + + import httpx + import uvicorn + + from hypertrain.auditor.island_bisect import IslandParty + from hypertrain.challenge.disputes_v2 import EventAck, Turn, WatchEvent, authentic + from hypertrain.miner.cli import main + from hypertrain.protocol import envelope_v2 + from hypertrain.protocol.hashing import sha256_hex + from hypertrain.protocol.jcs import canonicalize + from hypertrain.protocol.messages_v2 import BisectV2, DisputeV2, EscrowLock, IslandJobV1 + + hot, run_id = fixtures.HOT[0], network.manifest.run_id() + escrow, _, disputes = network.store._services(run_id) + contest = json.loads( + network.store._db.execute( + "SELECT data FROM records_v2 WHERE kind='contest' AND json_extract(data,'$.miner')=?", + (hot.ss58,), + ).fetchone()[0] + ) + verdict_hash = contest["verdict_hash"] + dispute_id = sha256_hex(canonicalize([run_id, verdict_hash, hot.ss58])) + origin = network.store._db.execute( + "SELECT origin FROM escrow_units WHERE owner=? AND bucket='available' AND units>=100", + (fixtures.COLD[0].ss58,), + ).fetchone()[0] + lock = EscrowLock( + operation_id="ce" * 32, + owner=fixtures.COLD[0].ss58, + units=100, + origin_ids=[origin], + admission_id=None, + dispute_id=dispute_id, + kind="LOCK_CONTEST", + ) + response = network.client.post( + network.url + "/escrow/lock", json=network.signed(fixtures.COLD[0], "EscrowLock", lock) + ) + assert response.status_code == 200, response.text + geometry = network.store._record_v2(run_id, "trace-geometry", verdict_hash) + job = IslandJobV1.model_validate(geometry["job"]) + published = Path(geometry["directory"]) + party = IslandParty.published(hot.ss58, job, published) + job_path = published.parent / "job.json" + job_path.write_bytes(canonicalize(job.body())) + directory = network.store.state_dir / "fastapi-watch" + directory.mkdir() + cert, server_key = directory / "cert.pem", directory / "server.key" + subprocess.run( + [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-days", + "1", + "-keyout", + str(server_key), + "-out", + str(cert), + "-subj", + "/CN=localhost", + "-addext", + "subjectAltName=DNS:localhost", + ], + check=True, + capture_output=True, + ) + monkeypatch.setenv("SSL_CERT_FILE", str(cert)) + key = directory / "miner.key" + key.write_bytes(bytes([80]) * 32) + key.chmod(0o600) + work = directory / "work" + watch_path = work / run_id / hot.ss58 / "watch/watch.db" + waiting, completed = threading.Event(), threading.Event() + + class SubscribedCondition(threading.Condition): + def wait(self, timeout=None): + waiting.set() + return super().wait(timeout) + + class ReadyServer(uvicorn.Server): + def __init__(self, config): + super().__init__(config) + self.ready = threading.Event() + + async def startup(self, sockets=None): + await super().startup(sockets=sockets) + self.ready.set() + + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + server = ReadyServer( + uvicorn.Config( + network.client.app, + log_level="error", + ssl_certfile=str(cert), + ssl_keyfile=str(server_key), + ) + ) + serving = threading.Thread(target=lambda: server.run(sockets=[sock]), daemon=True) + config = directory / "miner.toml" + config.write_text( + "\n".join( + f"{name} = {json.dumps(str(value))}" + for name, value in { + "api": f"https://localhost:{port}", + "keyfile": key, + "workdir": work, + "state_source": "file", + "image_digest": job.manifest.training.reference_spec.image_digest, + "run_id": run_id, + "owner_hotkey": fixtures.OWNER.ss58, + "device": "cpu", + }.items() + ) + ) + failures, result, output = [], [], io.StringIO() + + def watch(): + try: + with contextlib.redirect_stdout(output): + result.append( + main( + [ + "watch", + "--config", + str(config), + "--job", + str(job_path), + "--timeout", + "10", + ] + ) + ) + except BaseException as error: + failures.append(error) + finally: + completed.set() + + worker = threading.Thread(target=watch, daemon=True) + serving.start() + try: + assert server.ready.wait(10) + with httpx.Client( + base_url=f"https://localhost:{port}" + network.url, + verify=ssl.create_default_context(cafile=str(cert)), + ) as client: + opened = client.post( + "/dispute", + json=network.signed( + hot, + "DisputeV2", + DisputeV2( + hotkey=hot.ss58, + verdict_hash=verdict_hash, + action="contest", + bond_lock=lock.operation_id, + ), + ), + ) + assert opened.status_code == 200, opened.text + before = escrow.balances() + params = {"party": hot.ss58, "cursor": 0, "timeout": 0} + signature = hot.sign(f"hypertrain/watch/2|{run_id}|{hot.ss58}".encode()).hex() + events = client.get( + "/disputes", params=params, headers={"X-Dispute-Signature": signature} + ) + assert events.status_code == 200, events.text + event = WatchEvent.model_validate(events.json()[0]) + ack = EventAck( + run_id=run_id, + cursor=event.cursor, + event_hash=event.digest(), + party=hot.ss58, + received_beacon=network.now, + sig="00" * 64, + ) + tables = ("dispute_events_v2", "dispute_acks_v2", "dispute_entries_v2") + unchanged = { + t: [tuple(r) for r in network.store._db.execute(f'SELECT * FROM "{t}"')] + for t in tables + } + assert client.get("/disputes", params=params).status_code == 403 + wrong = hot.sign(f"hypertrain/watch/2|{'ff' * 32}|{hot.ss58}".encode()).hex() + assert ( + client.get( + "/disputes", params=params, headers={"X-Dispute-Signature": wrong} + ).status_code + == 403 + ) + assert client.post("/disputes/ack", json=ack.body()).status_code == 409 + assert { + t: [tuple(r) for r in network.store._db.execute(f'SELECT * FROM "{t}"')] + for t in tables + } == unchanged + assert escrow.balances() == before and not watch_path.exists() + initial = io.StringIO() + with contextlib.redirect_stdout(initial): + assert ( + main( + ["watch", "--config", str(config), "--job", str(job_path), "--timeout", "0"] + ) + == 0 + ) + assert json.loads(initial.getvalue()) == {"processed": 1} + step = BisectV2.model_validate_json( + network.store._db.execute( + "SELECT body FROM dispute_entries_v2 WHERE id=? AND seq=0", (dispute_id,) + ).fetchone()[0] + ) + current = disputes.get(dispute_id) + auditor = next(k for k in fixtures.AUDITORS if k.ss58 == current.expected_party) + assert current.level == "step" and current.interval == (0, 2) + counterpart = BisectV2( + dispute_id=dispute_id, + seq=current.seq, + level=current.level, + ctx=current.ctx, + interval=current.interval, + N=2, + hashes=[step.hashes[0], "aa" * 32, "bb" * 32], + party=auditor.ss58, + previous_transcript_hash=current.transcript_hash, + ) + original_condition = disputes.condition + disputes.condition = SubscribedCondition() + try: + worker.start() + assert waiting.wait(15), failures + assert not completed.is_set() + transition = client.post( + "/bisect", json=network.signed(auditor, "BisectV2", counterpart) + ) + assert transition.status_code == 200, transition.text + layer = Turn.model_validate(transition.json()) + assert layer.level == "layer" and layer.expected_party == hot.ss58 + assert completed.wait(20), failures + worker.join(1) + assert not worker.is_alive() and not failures and result == [0] + assert json.loads(output.getvalue()) == {"processed": 1} + finally: + worker.join(20) + disputes.condition = original_condition + row = network.store._db.execute( + "SELECT body,envelope FROM dispute_entries_v2 WHERE id=? AND seq=?", + (dispute_id, layer.seq), + ).fetchone() + accepted = BisectV2.model_validate_json(row[0]) + signed = envelope_v2.parse_envelope(row[1]) + assert envelope_v2.verify_envelope(signed.model_dump()) + assert signed.signer == hot.ss58 and signed.run_id == run_id + assert ( + accepted.level, + accepted.ctx, + accepted.party, + accepted.previous_transcript_hash, + ) == ("layer", layer.ctx, hot.ss58, layer.transcript_hash) + lo, hi = layer.interval + assert accepted.hashes == party.hashes( + "layer", tuple(layer.ctx), [lo, -(-(lo + hi) // 2), hi] + ) + with sqlite3.connect(watch_path) as saved: + cursor, raw_event, raw_ack, durable, done = saved.execute( + "SELECT cursor,event,ack,envelope,done FROM watch_outbox " + "ORDER BY cursor DESC LIMIT 1" + ).fetchone() + assert ( + saved.execute("SELECT MAX(cursor) FROM watch_outbox WHERE done=1").fetchone()[0] + == cursor + ) + received = WatchEvent.model_validate_json(raw_event) + saved_ack = EventAck.model_validate_json(raw_ack) + assert ( + cursor > event.cursor and done == 1 and authentic(received) and authentic(saved_ack) + ) + assert received.turn == layer and saved_ack.event_hash == received.digest() + assert BisectV2.model_validate(envelope_v2.parse_envelope(durable).body) == accepted + assert ( + network.store._db.execute( + "SELECT ack FROM dispute_acks_v2 WHERE cursor=? AND party=?", (cursor, hot.ss58) + ).fetchone()[0] + == saved_ack.model_dump_json() + ) + assert escrow.balances() == before + finally: + server.should_exit = True + serving.join(10) + if worker.ident is not None: + worker.join(20) + sock.close() + assert not serving.is_alive() and not worker.is_alive() diff --git a/tests/gpu_ops/test_network_admit_cli.py b/tests/gpu_ops/test_network_admit_cli.py new file mode 100644 index 0000000..9e9731d --- /dev/null +++ b/tests/gpu_ops/test_network_admit_cli.py @@ -0,0 +1,2553 @@ +"""Reviewed CLI boundaries and loopback lifecycle; no paid or CUDA evidence.""" + +from __future__ import annotations + +import importlib.util +import json +import os +import select +import subprocess +import sys +import time +from pathlib import Path +from typing import Literal + +import pytest + +import hypertrain +from hypertrain.gpu_ops.journal import Journal, fsha, sha256 +from hypertrain.gpu_ops.launcher import Reject +from hypertrain.protocol import envelope_v2 +from hypertrain.protocol.example import example_manifest +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.keys import Keypair +from hypertrain.protocol.messages_v2 import IslandJobV1, RunManifestV2 + +TREE = Path(hypertrain.__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location( + "network_admit_cli", TREE / "experiments/gpu_network_v2/orchestrate.py" +) +assert SPEC and SPEC.loader +cli = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = cli +SPEC.loader.exec_module(cli) +OWNER = Keypair(bytes(range(32))) + + +def test_operator_graph_self_reference_excluded_only(): + """Metadata hashing unit, no runtime/bootstrap/economic admission positive.""" + import network_service_proof as driver + + graph = driver.ContinuationGraph.model_construct( + economic_authority=Path("/private/operator"), + private_files={"/private/operator": "11" * 32, "/private/owner": "22" * 32}, + run_id="33" * 32, + owner=OWNER.ss58, + ) + digest = driver.continuation_graph_hash(graph) + changed_self = graph.model_copy( + update={ + "private_files": {**graph.private_files, "/private/operator": "44" * 32}, + } + ) + assert driver.continuation_graph_hash(changed_self) == digest + changed_key = graph.model_copy( + update={ + "private_files": {**graph.private_files, "/private/owner": "55" * 32}, + } + ) + assert driver.continuation_graph_hash(changed_key) != digest + + +@pytest.mark.parametrize( + "fault", + [ + None, + "permutation", + "uuid", + "worker", + "source", + "empty", + "extra", + "missing:scripts/network_service_proof.py", + "missing:src/hypertrain/gpu_ops/network_qualification.py", + "missing:src/hypertrain/miner/island_worker.py", + "tampered:scripts/network_service_proof.py", + "tampered:src/hypertrain/gpu_ops/network_qualification.py", + "tampered:src/hypertrain/miner/island_worker.py", + "legacy", + "signer", + "run", + "expiry", + "future", + "profile", + "count", + "bool-count", + "duplicate", + "malformed-uuid", + "name", + "driver", + ], +) +def test_operator_allocation_original_csv_signed_worker(tmp_path, fault): + """Shipped CSV parser and real source signatures; no CUDA-positive claim.""" + from types import SimpleNamespace + + import network_service_proof as driver + from hypertrain.gpu_ops.network_qualification import sources + + pins = sources(TREE) + pins["scripts/network_service_proof.py"] = fsha(TREE / "scripts/network_service_proof.py") + if fault == "empty": + pins.clear() + elif fault == "extra": + pins["unexpected.py"] = "01" * 32 + elif fault and fault.startswith("missing:"): + del pins[fault.split(":", 1)[1]] + elif fault and fault.startswith("tampered:"): + pins[fault.split(":", 1)[1]] = "02" * 32 + worker = TREE / "src/hypertrain/miner/island_worker.py" + run_id = "ab" * 32 + profile = {"runtime": {"driver_allowlist": ["580.95.05"]}} + profile_hash = sha256(canonicalize(profile)) + record = { + "allocation_contract": "canonical-shipped-source-v1", + "files": pins, + "profile_sha256": profile_hash, + "original_worker_sha256": fsha(worker), + "physical_allocation": ">=2", + "training_ranks": 2, + "run_id": run_id, + } + if fault == "worker": + record["original_worker_sha256"] = "cd" * 32 + if fault == "legacy": + del record["allocation_contract"] + record["files"] = dict.fromkeys( + ("allocation_cli.py", "gpu_allocation.py", "qualify_r9.py"), "01" * 32 + ) + source = {"files": dict(record["files"]), "profile_sha256": profile_hash} + if fault == "source": + source["files"]["src/hypertrain/gpu_ops/network_qualification.py"] = "ef" * 32 + if fault == "profile": + source["profile_sha256"] = "ef" * 32 + files = {} + for name, body in (("allocation_worker_receipt", record), ("source_receipt", source)): + path = tmp_path / (name + ".json") + env = envelope_v2.seal( + Keypair(b"\xcc" * 32) if fault == "signer" else OWNER, + "Receipt", + "cd" * 32 if fault == "run" else run_id, + { + "w": 0, + "received_round": 2 if fault == "future" else 1, + "commit_hash": sha256(canonicalize(body)), + }, + 1 if fault == "expiry" else 100, + ) + if name == "allocation_worker_receipt" and fault == "worker": + env["sig"] = "00" * 64 + path.write_bytes(canonicalize({"record": body, "receipt": env})) + files[name] = path + uuids = [f"GPU-{i:08x}-1111-2222-3333-444444444444" for i in range(4)] + rows = [f"{u}, NVIDIA GeForce RTX 5090, 580.95.05" for u in reversed(uuids)] + if fault == "duplicate": + rows[-1] = rows[0] + elif fault == "malformed-uuid": + rows[0] = rows[0].replace(uuids[-1], "GPU-" + "-" * 36) + elif fault == "name": + rows[0] = rows[0].replace("RTX 5090", "RTX 4090") + elif fault == "driver": + rows[0] = rows[0].replace("580.95.05", "unqualified") + inventory = "\n".join(rows) + selected = uuids[:2] + if fault == "permutation": + selected = selected[::-1] + elif fault == "uuid": + selected = [uuids[0], "GPU-ffffffff-1111-2222-3333-444444444444"] + allocation = {"allocated_count": 4, "inventory": inventory, "selected_uuids": selected} + if fault == "count": + allocation["allocated_count"] = 3 + elif fault == "bool-count": + allocation["allocated_count"] = True + runtime = SimpleNamespace( + journal=SimpleNamespace(last=lambda *a, **k: allocation), + profile=profile, + ) + checked = {"files": files, "record": {"tree": str(TREE)}} + if fault is None: + assert ( + len( + driver.continuation_allocation_authority( + runtime, checked, OWNER.ss58, run_id, 1, "h0", {"quote": {"num_gpus": 4}} + ) + ) + == 64 + ) + else: + with pytest.raises(driver.DriverError): + driver.continuation_allocation_authority( + runtime, + checked, + OWNER.ss58, + run_id, + 2 if fault == "expiry" else 1, + "h0", + {"quote": {"num_gpus": 4}}, + ) + + +@pytest.fixture(scope="module") +def retained_custody_work(tmp_path_factory): + """Reuse original genuine miner/reference fixture once, without a third run.""" + path = TREE / "tests/ledger/test_escrow_v2.py" + spec = importlib.util.spec_from_file_location("custody_original_ledger_fixture", path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + actual = module.actual_reward.__wrapped__(tmp_path_factory) + base = tmp_path_factory.getbasetemp() + miner = next(base.glob("real-reward*/published")) + reference = next(base.glob("real-reference*/published")) + return module, actual, miner, reference + + +def test_probe_retains_original_shadow_reward_authority( + retained_custody_work, monkeypatch, tmp_path +): + # Given: original independent executions exist; capture must not launch again. + import threading + + import hypertrain.miner.island_launch as island + import network_gpu_operation as adapter + from hypertrain.auditor.replay import unpack_state + from hypertrain.data.trial_assignment import trial_assignment_hash + from hypertrain.protocol.envelope_v2 import seal, verify_envelope + from hypertrain.protocol.hashing import sha256_hex + from hypertrain.protocol.messages_v2 import ArtifactLimits, JoinChallenge + from hypertrain.trainer.compress import state_hash + + fixture, _, miner_path, reference_path = retained_custody_work + job = IslandJobV1.model_validate_json((miner_path / "job.json").read_bytes()) + refjob = IslandJobV1.model_validate_json((reference_path / "job.json").read_bytes()) + assert job.model_dump(exclude={"deadline"}) == refjob.model_dump(exclude={"deadline"}) + assert miner_path != reference_path + + def prohibited(*args, **kwargs): + raise AssertionError("custody capture cannot execute another trajectory") + + monkeypatch.setattr(island, "launch_island", prohibited) + monkeypatch.setattr(island, "_launch_unbounded", prohibited) + start, _ = unpack_state((miner_path / job.object_paths["start_state"]).read_bytes()) + challenge = JoinChallenge( + admission_id=fixture.H, + nonce="22" * 32, + seed_beacon=1, + deadline_beacon=101, + manifest_hash=job.run_id, + theta_hash=state_hash(start), + assignment_hash=trial_assignment_hash(job.manifest, job.w, tuple(job.sample_ids)), + layout=job.manifest.training.reference_spec.layout, + artifact_limits=ArtifactLimits( + max_object_bytes=268435456, max_body_bytes=65536, max_chunk_manifest_bytes=1048576 + ), + policy_hash=job.manifest.network.admission_policy_hash, + ) + signed_challenge = seal(fixture.COORD, "JoinChallenge", job.run_id, challenge, 200) + + class CpuCustodyHelper(adapter.Operation): + """Test-only helper descriptor; never submitted to production execute/check.""" + + backend: Literal["cpu"] = "cpu" + + operation = CpuCustodyHelper( + operation="probe", + binding=fixture.H, + hotkey=fixture.HOT.ss58, + owner=fixture.COLD.ss58, + role="h0", + instance_id=11, + machine_id=22, + image_digest=job.manifest.training.reference_spec.image_digest, + job=job, + binding_kind="trial", + backend="cpu", + trace=False, + cutoff=job.deadline, + sources={ + "scripts/network_gpu_operation.py": fsha(TREE / "scripts/network_gpu_operation.py") + }, + challenge="original-challenge.json", + context_files={}, + ) + miner = island.validate_artifacts(job, miner_path) + reference = island.validate_artifacts(refjob, reference_path) + # When: validate/hash retained objects; actual miner signs its own CommitV2. + signed_commit = adapter.probe_commit(operation, miner, fixture.HOT, signed_challenge) + captured = adapter.publication_custody( + operation, + miner_path.parent, + miner, + signed_challenge, + signed_commit, + ) + ref_operation = operation.model_copy(update={"operation": "reference", "job": refjob}) + independent = adapter.publication_custody( + ref_operation, + reference_path.parent, + reference, + signed_challenge, + ) + # Then: independent original publications, no launcher-signed miner or MATCH assertion. + assert verify_envelope(captured["body"]["miner_commit"]["envelope"]) + assert captured["body"]["miner_commit"]["envelope"]["signer"] == fixture.HOT.ss58 + assert independent["body"]["miner_commit"] is None + assert independent["body"]["operation"] == "reference" + assert captured["body"]["operation_sha256"] != independent["body"]["operation_sha256"] + assert captured["body"]["job_sha256"] == job.digest() + assert independent["body"]["job_sha256"] == refjob.digest() + assert captured["body"]["sample_ids_sha256"] == independent["body"]["sample_ids_sha256"] + assert captured["body"]["trial_authority"]["nonce"] == challenge.nonce + assert captured["body"]["environment"]["backend"] == "cpu" + assert captured["body"]["layout"] == challenge.layout.model_dump(mode="json") + assert captured["body"]["sources"] == operation.sources + assert ( + captured["body"]["rank_summaries"][0]["commitments"] + == (independent["body"]["rank_summaries"][0]["commitments"]) + ) + assert captured["sha256"] == sha256_hex(canonicalize(captured["body"])) + assert captured["status"] == independent["status"] == "CAPTURED_NOT_ACCEPTED" + from pydantic import ValidationError + + with pytest.raises(ValidationError): + adapter.Operation.model_validate(operation.model_dump()) + # Captured output proves hashing finished; exact cancellation state drives rejection. + cancel = threading.Event() + cancel.set() + assert cancel.is_set() + with pytest.raises(ValueError, match="deadline/cancellation"): + adapter.publish_custody_result(operation, tmp_path, {}, captured, cancel) + assert not (tmp_path / "operation-result.json").exists() + assert not (tmp_path / "publication-custody.json").exists() + cancel.clear() + clock = {"now": operation.cutoff} + monkeypatch.setattr(adapter.time, "time", lambda: clock["now"]) + with pytest.raises(ValueError, match="deadline/cancellation"): + adapter.publish_custody_result(operation, tmp_path, {}, captured, cancel) + assert not (tmp_path / "operation-result.json").exists() + assert not (tmp_path / "execution-custody.json").exists() + # Cancellation delivered by the custody write must block subsequent result publication. + clock["now"] = operation.cutoff - 1 + original_write = adapter.durable_write + + def cancel_at_custody_write(path, payload): + original_write(path, payload) + if path.name == "publication-custody.json": + cancel.set() + + monkeypatch.setattr(adapter, "durable_write", cancel_at_custody_write) + with pytest.raises(ValueError, match="deadline/cancellation"): + adapter.publish_custody_result(operation, tmp_path, {}, captured, cancel) + assert (tmp_path / "publication-custody.json").exists() + assert not (tmp_path / "operation-result.json").exists() + assert not (tmp_path / "execution-custody.json").exists() + with pytest.raises(ValueError, match="signing identity"): + adapter.probe_commit(operation, miner, fixture.OTHER, signed_challenge) + wrong_commit = seal(fixture.OTHER, "CommitV2", job.run_id, signed_commit["body"], 200) + with pytest.raises(ValueError, match="miner commit"): + adapter.publication_custody( + operation, miner_path.parent, miner, signed_challenge, wrong_commit + ) + wrong_challenge = seal(fixture.OTHER, "JoinChallenge", job.run_id, challenge, 200) + with pytest.raises(ValueError, match="trial authority"): + adapter.publication_custody( + operation, miner_path.parent, miner, wrong_challenge, signed_commit + ) + malformed = {**signed_commit, "sig": "00" * 64} + with pytest.raises(ValueError, match="miner commit"): + adapter.publication_custody( + operation, miner_path.parent, miner, signed_challenge, malformed + ) + signed_changed = seal( + fixture.HOT, "CommitV2", job.run_id, {**signed_commit["body"], "delta_hash": "33" * 32}, 200 + ) + with pytest.raises(ValueError, match="miner commit"): + adapter.publication_custody( + operation, miner_path.parent, miner, signed_challenge, signed_changed + ) + original = miner.delta.read_bytes() + try: + miner.delta.write_bytes(bytes([original[0] ^ 1]) + original[1:]) + with pytest.raises(island.IslandFailure, match="terminal compression/EF mismatch"): + adapter.publication_custody( + operation, miner_path.parent, miner, signed_challenge, signed_commit + ) + finally: + miner.delta.write_bytes(original) + + +def test_reference_context_preserves_original_signed_challenge_and_trace( + retained_custody_work, + tmp_path, + monkeypatch, +): + import threading + from types import SimpleNamespace + + import network_gpu_operation as producer + import network_service_proof as service + from hypertrain.auditor.replay import unpack_state + from hypertrain.data.trial_assignment import trial_assignment_hash + from hypertrain.gpu_ops.journal import Journal + from hypertrain.gpu_ops.work_screen import screen_work + from hypertrain.protocol.envelope_v2 import Intake, seal + from hypertrain.protocol.hashing import sha256_hex + from hypertrain.protocol.messages import Receipt + from hypertrain.protocol.messages_v2 import ArtifactLimits, JoinChallenge + from hypertrain.trainer.compress import state_hash + + fixture, _, _, reference_path = retained_custody_work + job = IslandJobV1.model_validate_json((reference_path / "job.json").read_bytes()) + start, _ = unpack_state((reference_path / job.object_paths["start_state"]).read_bytes()) + challenge = JoinChallenge( + admission_id=fixture.H, + nonce="44" * 32, + seed_beacon=1, + deadline_beacon=101, + manifest_hash=job.run_id, + theta_hash=state_hash(start), + assignment_hash=trial_assignment_hash(job.manifest, job.w, tuple(job.sample_ids)), + layout=job.manifest.training.reference_spec.layout, + artifact_limits=ArtifactLimits( + max_object_bytes=268435456, max_body_bytes=65536, max_chunk_manifest_bytes=1048576 + ), + policy_hash=job.manifest.network.admission_policy_hash, + ) + signed = seal(fixture.COORD, "JoinChallenge", job.run_id, challenge, 200) + wire = canonicalize(signed) + relative = "reference-context-" + job.digest() + "/original-join-challenge.json" + sources = {"scripts/network_gpu_operation.py": fsha(TREE / "scripts/network_gpu_operation.py")} + expected = producer.Operation( + operation="reference", + binding=fixture.H, + hotkey=fixture.HOT.ss58, + owner=OWNER.ss58, + role="h0", + instance_id=11, + machine_id=22, + job=job, + binding_kind="trial", + sources=sources, + image_digest=job.manifest.training.reference_spec.image_digest, + cutoff=job.deadline, + trace=True, + challenge=relative, + context_files={relative: sha256_hex(wire)}, + ) + body = { + "role": "h0", + "binding": fixture.H, + "operation": "reference", + "hotkey": fixture.HOT.ss58, + "spec_sha256": sha256_hex(expected.model_dump_json().encode()), + } + receipt = seal( + OWNER, + "Receipt", + job.run_id, + Receipt( + w=job.w, + commit_hash=sha256_hex(canonicalize(body)), + received_round=1, + ), + 200, + ) + context = { + "run_id": job.run_id, + "hotkey": fixture.HOT.ss58, + "challenge_envelope": signed, + "challenge": challenge.body(), + "challenge_hash": challenge.digest(), + "admission_id": fixture.H, + "epoch": job.w, + } + transferred = [] + launches = [] + cancel = threading.Event() + + class OriginalOperationObserved(Exception): + pass + + (tmp_path / "runtime-journal").mkdir() + + class RuntimeBoundary: + journal = Journal(tmp_path / "runtime-journal") + cancel_during_staging = False + lifecycle = SimpleNamespace(cleanup_started=lambda: False) + + def owned_host(self, role): + return { + "instance_id": 11, + "machine_id": 22, + "image_digest": job.manifest.training.reference_spec.image_digest, + } + + def accept_operation(self, spec, raw, *, owner, beacon): + original = Intake(job.run_id, {"Receipt": owner.__eq__}).accept(raw, beacon) + assert original.commit_hash == sha256_hex(canonicalize(body)) + assert spec == expected + self.journal.append( + "network_operation_authority", + role=spec.role, + binding=spec.binding, + operation=spec.operation, + hotkey=spec.hotkey, + w=job.w, + receipt=json.loads(raw), + spec_sha256=sha256_hex(spec.model_dump_json().encode()), + ) + + def stage_context(self, role, paths, remote, name, cutoff, cancel): + assert cutoff == job.deadline and cancel is original_cancel + assert paths[relative].read_bytes() == wire + transferred.append((relative, fsha(paths[relative]), role)) + if self.cancel_during_staging: + cancel.set() + + def launch_adapter(self, spec): + return cli.NetworkRuntime.launch_adapter(self, spec) + + def operation(self, spec, directory, cancel=None): + assert spec.trace is True and cancel is original_cancel + assert transferred + launches.append((spec, cancel)) + raise OriginalOperationObserved + + original_cancel = cancel + runtime = RuntimeBoundary() + plan = service.decoder_context_plan( + runtime, + OWNER.ss58, + job.run_id, + sources, + {fixture.HOT.ss58: {"role": "h0"}}, + {"reference:" + job.digest(): receipt}, + 1, + ) + raw, retained, _ = plan("reference", job, tmp_path, context) + assert raw == expected.model_dump(mode="json") and retained == receipt + assert (tmp_path / "original-join-challenge.json").read_bytes() == wire + factory = service.decoder_service_factory(runtime, plan) + launch = factory("reference", job, tmp_path, context) + assert not transferred and not launches + monkeypatch.setattr(service.time, "time", lambda: job.deadline - 1) + # Actual screen_work omits trace. The scoped wrapper must supply it to the real adapter. + with pytest.raises(OriginalOperationObserved): + screen_work( + job, + reference_path, + challenge, + now_beacon=1, + backend="cuda", + launch=launch, + cancel=cancel, + ) + assert transferred == [(relative, sha256_hex(wire), "h0")] + assert launches == [(expected, cancel)] + assert challenge.nonce == producer.trial_authority(expected, signed).nonce + cancel.set() + with pytest.raises(ValueError, match="CANCELLED"): + screen_work( + job, + reference_path, + challenge, + now_beacon=1, + backend="cuda", + launch=launch, + cancel=cancel, + ) + assert len(transferred) == len(launches) == 1 + # A real Event set while staging prevents the original operation call, without sleeps. + runtime.cancel_during_staging = True + cancel.clear() + with pytest.raises(service.DriverError, match="deadline/cancellation"): + screen_work( + job, + reference_path, + challenge, + now_beacon=1, + backend="cuda", + launch=launch, + cancel=cancel, + ) + assert cancel.is_set() and launches == [(expected, original_cancel)] + # Original trace-free fixture is not silently upgraded to accepted trace custody. + artifacts = producer.validate_artifacts(job, reference_path) + + class CpuReferenceCustodyHelper(producer.Operation): + """Helper-only CPU fixture; actual factory descriptor remains CUDA-only.""" + + backend: Literal["cpu"] = "cpu" + + cpu_helper = CpuReferenceCustodyHelper.model_validate( + {**expected.model_dump(), "backend": "cpu"} + ) + with pytest.raises(ValueError, match="trace absent"): + producer.publication_custody(cpu_helper, reference_path.parent, artifacts, signed) + wrong = { + **context, + "challenge_envelope": seal( + fixture.OTHER, + "JoinChallenge", + job.run_id, + challenge, + 200, + ), + } + with pytest.raises(service.DriverError, match="authority"): + plan("reference", job, tmp_path, wrong) + with pytest.raises(service.DriverError, match="accepted challenge"): + plan("reference", job, tmp_path, {**context, "epoch": job.w + 1}) + (tmp_path / "original-join-challenge.json").write_bytes(b"different original bytes") + with pytest.raises(service.DriverError, match="file conflicts"): + plan("reference", job, tmp_path, context) + + +@pytest.mark.parametrize("required", [False, True]) +def test_original_probe_trace_is_explicit_and_preserves_ordinary_default( + retained_custody_work, + monkeypatch, + tmp_path, + required, +): + import threading + from types import SimpleNamespace + + import hypertrain.gpu_ops.work_screen as screen + from hypertrain.auditor.replay import unpack_state + from hypertrain.data.trial_assignment import trial_assignment_hash + from hypertrain.miner.core import NetworkMiner + from hypertrain.protocol.envelope_v2 import seal + from hypertrain.protocol.messages_v2 import ArtifactLimits, JoinChallenge + from hypertrain.trainer.compress import state_hash + + fixture, _, miner_path, _ = retained_custody_work + job = IslandJobV1.model_validate_json((miner_path / "job.json").read_bytes()) + start, _ = unpack_state((miner_path / job.object_paths["start_state"]).read_bytes()) + challenge = JoinChallenge( + admission_id=fixture.H, + nonce="55" * 32, + seed_beacon=1, + deadline_beacon=101, + manifest_hash=job.run_id, + theta_hash=state_hash(start), + assignment_hash=trial_assignment_hash(job.manifest, job.w, tuple(job.sample_ids)), + layout=job.manifest.training.reference_spec.layout, + artifact_limits=ArtifactLimits( + max_object_bytes=268435456, max_body_bytes=65536, max_chunk_manifest_bytes=1048576 + ), + policy_hash=job.manifest.network.admission_policy_hash, + ) + path = tmp_path / "challenge.json" + path.write_bytes(canonicalize(seal(fixture.COORD, "JoinChallenge", job.run_id, challenge, 200))) + observed = [] + cancel = threading.Event() + + class LaunchObserved(Exception): + pass + + def original_launch(job, directory, *, backend, cancel=None, trace=False): + observed.append((job, directory, backend, cancel, trace)) + raise LaunchObserved + + def original_screen(job, directory, challenge, *, now_beacon, backend, launch, cancel): + assert now_beacon == 1 + # Original screen_work dispatch does not inject trace; only Miner option may supply it. + return launch(job, directory, backend=backend, cancel=cancel) + + monkeypatch.setattr(screen, "screen_work", original_screen) + miner = NetworkMiner.__new__(NetworkMiner) + miner.run_id = job.run_id + miner.cfg = SimpleNamespace(device="cpu") + miner.manifest = job.manifest + miner.api = SimpleNamespace(call=lambda method, url: {"now_round": 1}) + miner.url = "/v2/runs/" + job.run_id + # Bound deterministic clock to before original cutoff; test never launches or signs output. + import hypertrain.miner.core as core + + monkeypatch.setattr(core.time, "time", lambda: job.deadline - 1) + options = {"trace": True} if required else {} + with pytest.raises(LaunchObserved): + miner.probe(miner_path / "job.json", path, launch=original_launch, cancel=cancel, **options) + assert observed == [(job, miner_path, "cpu", cancel, required)] + if required: + from hypertrain.miner.core import MinerError + from hypertrain.miner.island_launch import validate_artifacts + + original = validate_artifacts(job, miner_path) + retained_calls = [] + + def retained_without_trace(job, directory, *, backend, cancel=None, trace=False): + retained_calls.append(trace) + return original + + # Existing genuine trace-free publication is refused, not retrained or filled in. + with pytest.raises(MinerError, match="required trace absent"): + miner.probe( + miner_path / "job.json", + path, + launch=retained_without_trace, + cancel=cancel, + trace=True, + ) + assert retained_calls == [True] + + +@pytest.fixture +def contract(tmp_path: Path): + import hypertrain.trainer # noqa: F401 + from hypertrain.auditor.replay import AnchorCache, pack_state + from hypertrain.protocol.hashing import MerkleTree + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.model import init_params + + files = {} + + def put(name, value, binary=False): + path = tmp_path / (name if binary else name + ".json") + path.write_bytes(value if binary else json.dumps(value).encode()) + files[name] = {"path": str(path), "sha256": fsha(path)} + return path + + registry = put( + "registry", + {"schemaVersion": 2, "config": {"size": 10}, "layers": [{"size": 42}]}, + ) + image = "sha256:" + fsha(registry) + profile = json.loads((TREE / "experiments/gpu_network_v2/profile.json").read_bytes()) + profile["runtime"].update(image_digest=image, driver_allowlist=["595.84"]) + profile_path = put("profile", profile) + body = example_manifest().body() + for section in ("model", "outer"): + body[section].update(profile[section]) + body["inner"].update({k: v for k, v in profile["inner"].items() if k != "lr_schedule"}) + body["inner"]["lr_schedule"].update(profile["inner"]["lr_schedule"]) + body["reference_spec"].update(image_digest=image, driver_allowlist=["595.84"], sm_count=170) + body["reference_spec"]["layout"].update(profile["layout"]) + rows = [bytes([i % 64, 0]) * 17 for i in range(4096)] + merkle = MerkleTree(rows) + body["dataset"].update( + n_samples=4096, + merkle_root=merkle.root.hex(), + sample_format="u16[seq_len+1] token ids", + ) + wrapper = RunManifestV2.model_validate( + { + "manifest_version": 2, + "training": body, + "network": { + **{ + k: "11" * 32 + for k in ( + "admission_policy_hash", + "economics_policy_hash", + "aggregation_policy_hash", + "dispute_policy_hash", + "audit_policy_hash", + "relay_registry_hash", + ) + }, + "audit_mode": "anchored-full", + "full_anchor_version": 1, + "capabilities": [ + "island-replay", + "all-level-disputes", + "transport-receipts", + ], + }, + } + ) + theta = init_params(TrainConfig.from_manifest_v2(wrapper).model) + anchor = AnchorCache().genesis(wrapper, OWNER.ss58, theta) + objects = { + "start_state": put("start_state", pack_state(theta, anchor.state), True), + "ef_in": put("ef_in", pack_state(anchor.ef), True), + "v0": put("v0", pack_state({}), True), + "samples": put("samples", b"".join(rows[:60]), True), + "sample_proofs": put( + "sample_proofs", [[p.hex() for p in merkle.proof(i)] for i in range(60)] + ), + } + now = int(time.time()) + job = IslandJobV1( + job_version=1, + run_id=wrapper.run_id(), + w=0, + manifest=wrapper, + sample_ids=list(range(60)), + global_step0=0, + start_state_sha256=fsha(objects["start_state"]), + ef_in_sha256=fsha(objects["ef_in"]), + v0_sha256=fsha(objects["v0"]), + object_paths={k: p.name for k, p in objects.items()}, + deadline=now + 3600, + ) + put("job", job.model_dump(mode="json")) + key = tmp_path / "key" + key.write_text("a" * 32) + key.chmod(0o600) + cfg = { + "base_url": "http://127.0.0.1:1", + "key_file": str(key), + "image": "candidate@" + image, + "disk_gb": 80, + "hard_deadline_seconds": 3600, + "hard_grace_seconds": 0, + "boot_timeout_seconds": 300, + "keyscan_attempts": 1, + "cleanup_deadline_unix": now + 3600, + "remote_python": "/opt/hypertrain/venv/bin/python", + "remote_root": str(tmp_path / "remote-{role}"), + "cleanup_contract": "network-v2", + "network_profile_file": str(profile_path), + "evidence_path": str(tmp_path / "evidence.json"), + "create_margin_seconds": 600, + "hosts": [ + {"role": r, "machine_id": 5000 + i, "max_dph_total": 2} + for i, r in enumerate(("h0", "h1")) + ], + } + put("config", cfg) + put( + "lifecycle_hashes", + { + p: fsha(TREE / p) + for p in ( + "src/hypertrain/gpu_ops/launcher.py", + "src/hypertrain/gpu_ops/supervisor.py", + ) + }, + ) + spec = importlib.util.spec_from_file_location( + "cli_fixture_driver", TREE / "scripts/network_gpu_qualification.py" + ) + assert spec and spec.loader + driver = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = driver + spec.loader.exec_module(driver) + sources, source_bundle = cli.effective_bundle(driver, TREE, profile_path) + put("sources", sources) + put("bundle", source_bundle, True) + offers = [ + { + "id": 900 + i, + "machine_id": 5000 + i, + "gpu_name": "RTX 5090", + "num_gpus": 2, + "cpu_cores_effective": 8, + "cpu_ram": 32768, + "disk_space": 80, + "dph_total": "0.4", + "storage_cost": "0.1", + "inet_up_cost": "0.001", + "inet_down_cost": "0.001", + "verified_unix": now, + "requested_unix": now, + "received_unix": now, + } + for i in range(2) + ] + snapshot = { + "authenticated": True, + "instances": [], + "volumes": [], + "charges": [], + "uncapped_charges": False, + "verified_unix": now, + "requested_unix": now, + "received_unix": now, + "prior_attempts_usd": "0", + "active_liabilities_usd": "0", + "available_credit_usd": "20", + "account_id": 4242, + } + authorization = { + "experiment": "hypertrain-network-v2", + "currency": "USD", + "ceiling_usd": "15", + "max_instances": 2, + "max_seconds_per_instance": 3600, + "source_message": "TEST_ONLY", + "date": "2026-10-09", + } + evidence = { + "root_verified_bounded_lifecycle": True, + "registry_image_verified": True, + "candidate_image_digest": image, + "staged_contract_sha256": "22" * 32, + "deadline_supervisor_contract_sha256": "33" * 32, + } + for k, v in { + "offers": offers, + "financial": snapshot, + "authorization": authorization, + "evidence": evidence, + }.items(): + put(k, v) + put( + "plan", + cli.admission_plan(profile, evidence, authorization, snapshot, offers, qualification=True), + ) + put( + "observations", + [{"requested_unix": now, "received_unix": now, "verified_unix": now}], + ) + from hypertrain.miner.admission import sign_join + from hypertrain.protocol.messages_v2 import HardwareHint + + join = sign_join( + OWNER, + OWNER, + run_id=wrapper.run_id(), + request_id="aa" * 32, + expires_beacon=20, + policy_hash=wrapper.network.admission_policy_hash, + hardware_hint=HardwareHint(device_name="test", device_count=2, driver="595.84"), + ) + put("join", join.model_dump(mode="json")) + put( + "seed_context", + { + "run_id": wrapper.run_id(), + "owner": OWNER.ss58, + "hotkey": OWNER.ss58, + "coldkey": OWNER.ss58, + "request_id": join.request_id, + "policy_hash": join.policy_hash, + "job_sha256": files["job"]["sha256"], + "join_sha256": files["join"]["sha256"], + "admission_id": sha256( + f"join|{wrapper.run_id()}|{join.coldkey}|{join.request_id}".encode() + ), + "state": "PROBATION", + }, + ) + record = { + "action": "admit", + "owner": OWNER.ss58, + "files": files, + "tree": str(TREE), + "parent_pid": os.getppid(), + "cutoff_unix": now + 3600, + "staging_bytes": 2_000_000, + "public_dataset_reviewed": True, + "full_genesis_reviewed": True, + "quota": {"total": 126, "per_role": 63, "qualification": 4, "remaining": 122}, + "roles": { + r: { + "job": "job", + "name": "qualification", + "hotkey": OWNER.ss58, + "join": "join", + "seed_context": "seed_context", + "objects": {k: k for k in objects}, + } + for r in ("h0", "h1") + }, + } + path = tmp_path / "review.json" + + def seal(): + receipt = envelope_v2.seal( + OWNER, + "Receipt", + wrapper.run_id(), + {"w": 0, "commit_hash": sha256(canonicalize(record)), "received_round": 10}, + 20, + ) + path.write_bytes(canonicalize({"record": record, "receipt": receipt})) + + seal() + return record, path, seal + + +def test_exact_owner_preflight_and_cli_plan(contract, capsys): + _, path, _ = contract + checked = cli.reviewed_launch(path, OWNER.ss58, 10, "admit") + assert checked["plan"]["reserve_usd"] == "5" + assert cli.main(["plan", str(TREE / "experiments/gpu_network_v2/profile.json")]) == 0 + assert json.loads(capsys.readouterr().out)["paid_execution"] is False + + +def test_effective_materialized_tree_matches_frozen_driver_sources(contract, tmp_path): + import tarfile + + record, path, _ = contract + checked = cli.reviewed_launch(path, OWNER.ss58, 10, "admit") + staged = tmp_path / "materialized" + with tarfile.open(checked["files"]["bundle"]) as archive: + archive.extractall(staged, filter="data") + assert checked["driver"].sources(staged) == json.loads(checked["files"]["sources"].read_bytes()) + assert fsha(staged / "experiments/gpu_network_v2/profile.json") == fsha( + checked["files"]["profile"] + ) + + +def test_approved_generated_seed_consumed_without_provider(contract, tmp_path, monkeypatch): + """Original generator validates index/signatures; root Receipt pins exact seed bytes.""" + from hypertrain.miner.admission import sign_join + from hypertrain.protocol.messages_v2 import HardwareHint + + record, path, _ = contract + spec = importlib.util.spec_from_file_location( + "cli_seed_fixture", TREE / "scripts/network_gpu_seed.py" + ) + assert spec and spec.loader + seed = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = seed + spec.loader.exec_module(seed) + destination, secrets = tmp_path / "generated", tmp_path / "generated-secrets" + profile = json.loads(Path(record["files"]["profile"]["path"]).read_bytes()) + admitted = seed.generate( + destination, + secrets, + profile["runtime"]["image_digest"], + profile["runtime"]["driver_allowlist"], + int(time.time()), + record["cutoff_unix"], + ) + job = seed.validate(destination, admitted) + index = json.loads((destination / "seed.json").read_bytes()) + owner = Keypair((secrets / "owner.seed").read_bytes()) + hot, cold = (Keypair((secrets / f"{r}-0.seed").read_bytes()) for r in ("hot", "cold")) + join = sign_join( + hot, + cold, + run_id=job.run_id, + request_id="ab" * 32, + expires_beacon=20, + policy_hash=job.manifest.network.admission_policy_hash, + hardware_hint=HardwareHint( + device_name="synthetic-functional", device_count=2, driver="595.84" + ), + ) + join_path = tmp_path / "generated-join.json" + join_path.write_bytes(canonicalize(join.body())) + for key in ("job", "start_state", "ef_in", "v0", "samples", "sample_proofs"): + p = destination / ("job.json" if key == "job" else key) + record["files"][key] = {"path": str(p), "sha256": fsha(p)} + for name in ( + "seed.json", + "manifest-envelope.json", + "job-envelope.json", + "reference-envelope.json", + "genesis-envelope.json", + ): + p = destination / name + record["files"]["generated-" + name] = {"path": str(p), "sha256": fsha(p)} + assert record["files"]["generated-seed.json"]["sha256"] == admitted + record["files"]["join"] = {"path": str(join_path), "sha256": fsha(join_path)} + context = Path(record["files"]["seed_context"]["path"]) + context.write_bytes( + canonicalize( + { + "run_id": job.run_id, + "owner": owner.ss58, + "hotkey": hot.ss58, + "coldkey": cold.ss58, + "request_id": join.request_id, + "policy_hash": join.policy_hash, + "job_sha256": record["files"]["job"]["sha256"], + "join_sha256": fsha(join_path), + "admission_id": sha256(f"join|{job.run_id}|{cold.ss58}|{join.request_id}".encode()), + "state": "PROBATION", + } + ) + ) + record["files"]["seed_context"]["sha256"] = fsha(context) + record["owner"] = owner.ss58 + for role in record["roles"].values(): + role["hotkey"] = hot.ss58 + # Snapshot shared source after generator work, before signing the admission record. + driver = sys.modules["cli_fixture_driver"] + sources, bundle = cli.effective_bundle(driver, TREE, Path(record["files"]["profile"]["path"])) + for key, value in (("sources", json.dumps(sources).encode()), ("bundle", bundle)): + p = Path(record["files"][key]["path"]) + p.write_bytes(value) + record["files"][key]["sha256"] = fsha(p) + receipt = envelope_v2.seal( + owner, + "Receipt", + job.run_id, + {"w": 0, "commit_hash": sha256(canonicalize(record)), "received_round": 10}, + 20, + ) + path.write_bytes(canonicalize({"record": record, "receipt": receipt})) + invoked = [] + monkeypatch.setattr(cli, "Provider", lambda *a, **kw: invoked.append(True)) + checked = cli.reviewed_launch(path, owner.ss58, 10, "admit") + assert checked["record"]["roles"]["h0"]["hotkey"] == index["hotkey"] + assert job.manifest.training.dataset.sample_format == "u32[seq_len+1] token ids" + assert invoked == [] + + +def test_continue_runtime_profile_preserves_original_materialized_sources( + contract, monkeypatch, tmp_path +): + record, path, seal = contract + original = Path(record["files"]["profile"]["path"]) + record["files"]["source_profile"] = { + "path": str(original), + "sha256": fsha(original), + } + measured = tmp_path / "measured-profile.json" + profile = json.loads(original.read_bytes()) + profile["runtime"]["qualified"] = True + profile["execution_and_transfer_bound_seconds"] = 1 + measured.write_text(json.dumps(profile)) + record["files"]["profile"] = {"path": str(measured), "sha256": fsha(measured)} + config = Path(record["files"]["config"]["path"]) + cfg = json.loads(config.read_bytes()) + cfg["network_profile_file"] = str(measured) + config.write_text(json.dumps(cfg)) + record["files"]["config"]["sha256"] = fsha(config) + record["action"] = "continue" + seal() + + class RuntimeGateReached(Exception): + pass + + def gate(*args): + raise RuntimeGateReached + + monkeypatch.setattr(cli, "runtime_admission", gate) + with pytest.raises(RuntimeGateReached): + cli.reviewed_launch(path, OWNER.ss58, 10, "continue") + record["files"]["source_profile"] = record["files"]["profile"] + seal() + with pytest.raises(Reject, match="source_changed"): + cli.reviewed_launch(path, OWNER.ss58, 10, "continue") + + +@pytest.mark.parametrize( + "fault", + [ + "overlay", + "future-financial", + "future-offer", + "future-observation", + "invalid-hotkey", + "unbound-hotkey", + "foreign-context", + "wrong-genesis", + ], +) +def test_original_independent_probes_reject_zero_provider(contract, monkeypatch, fault): + record, path, seal = contract + invoked = [] + monkeypatch.setattr(cli, "Provider", lambda *a, **kw: invoked.append(True)) + if fault in ("invalid-hotkey", "unbound-hotkey"): + record["roles"]["h1"]["hotkey"] = ( + "arbitrary-not-an-SS58-key" + if fault == "invalid-hotkey" + else Keypair(bytes([9]) * 32).ss58 + ) + else: + key = { + "overlay": "sources", + "future-financial": "financial", + "future-offer": "offers", + "future-observation": "observations", + "foreign-context": "seed_context", + "wrong-genesis": "start_state", + }[fault] + p = Path(record["files"][key]["path"]) + if fault == "wrong-genesis": + from hypertrain.auditor.replay import pack_state, unpack_state + + theta, state = unpack_state(p.read_bytes()) + assert state is not None + next(iter(state.m.values())).add_(1) + p.write_bytes(pack_state(theta, state)) + job_path = Path(record["files"]["job"]["path"]) + job = json.loads(job_path.read_bytes()) + job["start_state_sha256"] = fsha(p) + job_path.write_text(json.dumps(job)) + record["files"]["job"]["sha256"] = fsha(job_path) + ctx_path = Path(record["files"]["seed_context"]["path"]) + ctx = json.loads(ctx_path.read_bytes()) + ctx["job_sha256"] = fsha(job_path) + ctx_path.write_text(json.dumps(ctx)) + record["files"]["seed_context"]["sha256"] = fsha(ctx_path) + else: + value = json.loads(p.read_bytes()) + if fault == "overlay": + value["experiments/gpu_network_v2/profile.json"] = fsha( + TREE / "experiments/gpu_network_v2/profile.json" + ) + elif fault == "foreign-context": + value["owner"] = Keypair(bytes([9]) * 32).ss58 + else: + row = value if fault == "future-financial" else value[0] + row["verified_unix"] = int(time.time()) + 86400 + p.write_text(json.dumps(value)) + record["files"][key]["sha256"] = fsha(p) + seal() + with pytest.raises((Reject, ValueError)): + cli.main( + [ + "admit", + str(path), + "--owner", + OWNER.ss58, + "--beacon", + "10", + "--run-dir", + str(path.parent / "run"), + ] + ) + assert invoked == [] + + +def test_sequential_remote_calls_share_absolute_cutoff(tmp_path, monkeypatch): + from types import SimpleNamespace + + import hypertrain.gpu_ops.remote as remote + + clock = [100.0] + calls = [] + monkeypatch.setattr(cli.time, "time", lambda: clock[0]) + + def run(argv, **kw): + calls.append(kw["timeout"]) + clock[0] += 2 + return subprocess.CompletedProcess(argv, 0, b"ok", b"") + + monkeypatch.setattr(remote.subprocess, "run", run) + j = Journal(tmp_path) + orch = SimpleNamespace(cleanup_started=lambda: False) + ssh = cli.DeadlineSsh( + host="127.0.0.1", + port=22, + user="root", + identity=tmp_path / "identity", + known_hosts=tmp_path / "known", + journal=j, + role="h0", + timeout=60, + ) + ssh.cutoff = 103.0 + ssh.lifecycle = orch + local = tmp_path / "blob" + local.write_bytes(b"bytes") + ssh.put(local, "/root/blob") + ssh.get("/root/blob", tmp_path / "fetched") + with pytest.raises(Reject, match="absolute_deadline"): + ssh.remote_sha256("/root/blob", tmp_path) + assert calls == [3.0, 1.0] + + +def test_real_qualification_staging_depletion_censors_cleanup(contract, tmp_path, monkeypatch): + import hypertrain.gpu_ops.remote as remote + + record, path, _ = contract + checked = cli.reviewed_launch(path, OWNER.ss58, 10, "admit") + run = tmp_path / "transport-run" + run.mkdir() + (run / "logs").mkdir() + j = Journal(run) + clock = [time.time()] + first = clock[0] - 297 + j.append("create_intent", role="h0", unix=first) + orch = cli.Orchestrator( + checked["config"], + j, + cli.Provider("http://127.0.0.1:1", "a" * 32, run, j, False), + run, + ) + monkeypatch.setattr(cli.time, "time", lambda: clock[0]) + for method in ("boot", "attach", "trust"): + monkeypatch.setattr(orch, method, lambda role: None) + monkeypatch.setattr(orch, "deadline", lambda: first + 3600) + ssh = remote.Ssh( + host="127.0.0.1", + port=22, + user="root", + identity=run / "identity", + known_hosts=run / "known", + journal=j, + role="h0", + timeout=60, + ) + monkeypatch.setattr(orch, "ssh", lambda role: ssh) + calls = [] + + def transport(argv, **kwargs): + calls.append(kwargs["timeout"]) + clock[0] += 2 + return subprocess.CompletedProcess( + argv, + 0, + "" if kwargs.get("text") else b"", + "" if kwargs.get("text") else b"", + ) + + monkeypatch.setattr(remote.subprocess, "run", transport) + cleanup = [] + monkeypatch.setattr( + orch, + "cleanup", + lambda force: cleanup.append({"censored": True, "custody_complete": False, "fresh": True}), + ) + runtime = cli.NetworkRuntime(orch, checked["profile"]) + with pytest.raises(Reject, match="absolute_deadline"): + try: + cli.qualify_role(checked, orch, runtime, "h0") + except Exception: + orch.cleanup(force=True) + raise + assert calls == [3.0, 1.0] + assert cleanup == [{"censored": True, "custody_complete": False, "fresh": True}] + assert not j.last("network_qualification_captured") + + +@pytest.mark.parametrize( + "fault", + [ + "owner", + "signature", + "stale", + "future", + "one-gpu", + "quota", + "seed", + "image", + "source", + "theta-only", + "bool-clock", + "parent", + ], +) +def test_preflight_rejects_before_any_provider(contract, monkeypatch, fault): + record, path, seal = contract + invoked = [] + monkeypatch.setattr(cli, "Provider", lambda *a, **kw: invoked.append(True)) + owner = OWNER.ss58 + if fault == "owner": + owner = Keypair(bytes([9]) * 32).ss58 + elif fault == "signature": + raw = json.loads(path.read_bytes()) + raw["receipt"]["sig"] = "00" * 64 + path.write_text(json.dumps(raw)) + elif fault == "quota": + record["quota"]["remaining"] = 126 + seal() + elif fault == "parent": + record["parent_pid"] = os.getpid() + seal() + elif fault == "seed": + Path(record["files"]["job"]["path"]).unlink() + elif fault == "source": + record["files"]["bundle"]["sha256"] = "00" * 32 + seal() + else: + key = ( + "financial" + if fault in ("stale", "future", "bool-clock") + else "offers" + if fault == "one-gpu" + else "profile" + if fault == "image" + else "start_state" + ) + p = Path(record["files"][key]["path"]) + if fault == "theta-only": + from hypertrain.auditor.replay import pack_state, unpack_state + + theta, _ = unpack_state(p.read_bytes()) + p.write_bytes(pack_state(theta)) + else: + value = json.loads(p.read_bytes()) + if fault in ("stale", "future"): + value["verified_unix"] += -31 if fault == "stale" else 100 + elif fault == "bool-clock": + value["verified_unix"] = True + elif fault == "one-gpu": + value[0]["num_gpus"] = 1 + else: + value["runtime"]["image_digest"] = None + p.write_text(json.dumps(value)) + record["files"][key]["sha256"] = fsha(p) + seal() + with pytest.raises((Reject, TypeError, ValueError, KeyError)): + cli.main( + [ + "admit", + str(path), + "--owner", + owner, + "--beacon", + "10", + "--run-dir", + str(path.parent / "run"), + ] + ) + assert invoked == [] + + +def test_loopback_ready_before_two_gpu_create_and_parent_death(contract, mock_factory, tmp_path): + record, path, seal = contract + quotes = json.loads(Path(record["files"]["offers"]["path"]).read_bytes()) + for row in quotes: + row.update(num_gpus=2, cpu_cores_effective=8, cpu_ram=32768, disk_space=80) + mock = mock_factory(offers=quotes) + cfg_path = Path(record["files"]["config"]["path"]) + cfg = json.loads(cfg_path.read_bytes()) + cfg["base_url"] = mock.base + cfg_path.write_text(json.dumps(cfg)) + record["files"]["config"]["sha256"] = fsha(cfg_path) + parent = subprocess.Popen( + [sys.executable, "-c", "import sys;print('READY',flush=True);sys.stdin.read()"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + assert parent.stdout and select.select([parent.stdout], [], [], 10)[0] + assert parent.stdout.readline() == b"READY\n" + record["parent_pid"] = parent.pid + seal() + run = tmp_path / "run" + try: + checked = cli.reviewed_launch(path, OWNER.ss58, 10, "admit") + result = cli.launch_action("admit", checked, run) + assert result["status"] == "ADMITTED_NOT_QUALIFIED" + j = Journal(run) + ready = j.last("supervisor_ready") + assert ready + kinds = [r["kind"] for r in j.records()] + assert kinds.index("supervisor_ready") < kinds.index("create_intent") + assert ready["supervisor_pid"] not in (os.getpid(), parent.pid) + assert len(j.all("create_intent")) == 2 and len(j.all("receipt")) == 2 + state = mock.state() + assert state["put_count"] == 2 + assert all( + e["path"] in ("/api/v0/asks/900/", "/api/v0/asks/901/") + for e in state["events"] + if e["method"] == "PUT" + ) + assert not any("charges" in e["path"] for e in state["events"]) + assert ( + cli.Orchestrator( + checked["config"], + j, + cli.Provider(mock.base, "a" * 32, run, j, False), + run, + ).deadline() + == min(r["unix"] for r in j.all("create_intent")) + 3600 + ) + parent.terminate() + parent.wait(timeout=10) + fd = os.pidfd_open(ready["supervisor_pid"]) + try: + assert select.select([fd], [], [], 30)[0] + finally: + os.close(fd) + assert j.last("supervisor_parent_exit") + assert len(j.all("absence_confirmed")) == 2 + assert j.last("supervisor_censored") + assert not j.all("network_execution_intent") + finally: + if parent.poll() is None: + parent.terminate() + parent.wait(timeout=10) + + +def test_admission_rejects_nonindependent_ready_before_create( + contract, mock_factory, tmp_path, monkeypatch +): + record, path, seal = contract + mock = mock_factory() + cfg_path = Path(record["files"]["config"]["path"]) + cfg = json.loads(cfg_path.read_bytes()) + cfg["base_url"] = mock.base + cfg_path.write_text(json.dumps(cfg)) + record["files"]["config"]["sha256"] = fsha(cfg_path) + seal() + checked = cli.reviewed_launch(path, OWNER.ss58, 10, "admit") + monkeypatch.setattr( + cli.Orchestrator, + "ensure_supervisor", + lambda self: {"supervisor_pid": record["parent_pid"]}, + ) + with pytest.raises(Reject, match="independent_supervisor"): + cli.launch_action("admit", checked, tmp_path / "run") + assert mock.state()["put_count"] == 0 + + +def test_insufficient_cutoff_rejects_before_provider(contract, monkeypatch, tmp_path): + _, path, _ = contract + checked = cli.reviewed_launch(path, OWNER.ss58, 10, "admit") + checked["record"]["cutoff_unix"] = int(time.time()) + 30 + invoked = [] + monkeypatch.setattr(cli, "Provider", lambda *a, **kw: invoked.append(True)) + with pytest.raises(Reject, match="cutoff_work_margin"): + cli.launch_action("admit", checked, tmp_path / "run") + assert invoked == [] + + +def test_qualification_failure_debits_four_and_cleans_both(contract, tmp_path, monkeypatch): + record, path, seal = contract + record["action"] = "qualification" + financial = Path(record["files"]["financial"]["path"]) + snapshot = json.loads(financial.read_bytes()) + snapshot["instances"] = [{"id": 1}, {"id": 2}] + financial.write_text(json.dumps(snapshot)) + record["files"]["financial"]["sha256"] = fsha(financial) + seal() + checked = cli.reviewed_launch(path, OWNER.ss58, 10, "qualification") + run = tmp_path / "run" + run.mkdir() + (run / "config.json").write_bytes(checked["files"]["config"].read_bytes()) + j = Journal(run) + j.append("transaction_open", pid=record["parent_pid"], live=False) + j.append( + "network_admission_authority", + owner=record["owner"], + sources_sha256=fsha(checked["files"]["sources"]), + ) + j.append("admitted", deadline_unix=time.time() + 3600) + j.append("supervisor_ready", supervisor_pid=record["parent_pid"]) + for i, role in enumerate(("h0", "h1")): + j.append("create_intent", role=role) + j.append("receipt", role=role, instance_id=i + 1) + cleaned = [] + monkeypatch.setattr( + cli.Orchestrator, "cleanup", lambda self, force: cleaned.append(set(self.roles)) + ) + + def failure(checked, orch, runtime, role): + runtime.reserve(role, "qualification", "qualification", 2) + raise Reject("injected_rank_failure") + + monkeypatch.setattr(cli, "qualify_role", failure) + with pytest.raises(Reject, match="injected_rank_failure"): + cli.launch_action("qualification", checked, run) + assert sum(r["executions"] for r in j.all("network_execution_intent")) == 4 + assert cleaned == [{"h0", "h1"}] + assert not j.last("network_workload_promoted") + + +@pytest.mark.parametrize("fault", [None, "owner", "instance", "cancel"]) +def test_continue_dispatches_signed_graph_not_raw_jobs( + contract, tmp_path, monkeypatch, fault, capsys +): + """Transport/result seams synthetic; actual reservation/continuation loop stays integrated.""" + from types import SimpleNamespace + + record, path, seal = contract + record["action"] = "continue" + seal() + # This fixture exercises the action boundary; measured CUDA preflight is separately negative. + record["action"] = "qualification" + financial = Path(record["files"]["financial"]["path"]) + snapshot = json.loads(financial.read_bytes()) + snapshot["instances"] = [{"id": 1}, {"id": 2}] + financial.write_text(json.dumps(snapshot)) + record["files"]["financial"]["sha256"] = fsha(financial) + seal() + checked = cli.reviewed_launch(path, OWNER.ss58, 10, "qualification") + record["action"] = "continue" + checked["record"]["action"] = "continue" + run = tmp_path / "run" + run.mkdir() + (run / "config.json").write_bytes(checked["files"]["config"].read_bytes()) + j = Journal(run) + j.append("transaction_open", pid=record["parent_pid"]) + j.append( + "network_admission_authority", + owner=record["owner"], + sources_sha256=fsha(checked["files"]["sources"]), + ) + j.append("admitted", deadline_unix=time.time() + 3600) + j.append("supervisor_ready", supervisor_pid=record["parent_pid"]) + for i, role in enumerate(("h0", "h1")): + j.append("create_intent", role=role) + j.append("receipt", role=role, instance_id=i + 1) + j.append("staged", role=role) + j.append( + "cli_import_attested", + role=role, + sources_sha256=fsha(checked["files"]["sources"]), + bundle_sha256=fsha(checked["files"]["bundle"]), + ) + j.append( + "network_execution_intent", + role=role, + name="qualification", + phase="qualification", + executions=2, + ) + j.append("network_qualification_captured", role=role, name="qualification") + result_path = tmp_path / f"result-{role}.json" + result_path.write_text("{}") + receipt_path = tmp_path / f"receipt-{role}.json" + receipt_path.write_text( + json.dumps( + { + "passed": True, + "root_verified": True, + "result_sha256": fsha(result_path), + "artifact_manifest_sha256": sha256(json.dumps({}, sort_keys=True).encode()), + } + ) + ) + checked["files"][f"result-{role}"] = result_path + checked["files"][f"receipt-{role}"] = receipt_path + checked["record"]["roles"][role].update( + qualification_result=f"result-{role}", + qualification_receipt=f"receipt-{role}", + ) + checked["financial"]["instances"] = [{"id": 1}, {"id": 2}] + + graph_path = tmp_path / "graph.json" + checked["run_id"] = json.loads(checked["files"]["job"].read_bytes())["run_id"] + graph_path.write_text( + json.dumps( + { + "owner": OWNER.ss58, + "run_id": checked["run_id"], + "operation_sources": { + "experiments/gpu_network_v2/profile.json": fsha(checked["files"]["profile"]) + }, + } + ) + ) + checked["files"]["graph"] = graph_path + checked["files"]["graph_driver"] = TREE / "scripts/network_service_proof.py" + checked["profile"]["execution_and_transfer_bound_seconds"] = 1 + checked["profile"]["runtime"]["qualified"] = True + evidence_path = tmp_path / "continue-evidence.json" + evidence_path.write_text(json.dumps({"root_reviewed_two_host_full_round_transfer": True})) + checked["files"]["evidence"] = evidence_path + checked["plan"].update(admit=True, long_workload_allowed=True) + monkeypatch.setattr(cli, "runtime_admission", lambda *a: 100) + checked["driver"].Result = SimpleNamespace( + model_validate_json=lambda raw: SimpleNamespace( + role="h0" if raw == b"h0" else "h1", + instance_id=1 if raw == b"h0" else 2, + machine_id=5000 if raw == b"h0" else 5001, + config_sha256=sha256(b"config"), + sources=json.loads(checked["files"]["sources"].read_bytes()), + environment=SimpleNamespace(image_digest=checked["profile"]["runtime"]["image_digest"]), + ) + ) + for role in ("h0", "h1"): + (run / f"cli-qualification-{role}.json").write_bytes(b"config") + checked["files"][f"result-{role}"].write_bytes(role.encode()) + rp = checked["files"][f"receipt-{role}"] + value = json.loads(rp.read_bytes()) + value["result_sha256"] = fsha(checked["files"][f"result-{role}"]) + rp.write_text(json.dumps(value)) + rescues = [] + monkeypatch.setattr( + cli.NetworkRuntime, + "rescue", + lambda self, role, **kw: rescues.append(role) or {"files": {}}, + ) + monkeypatch.setattr( + cli.Orchestrator, + "ssh", + lambda self, role: SimpleNamespace(run=lambda *a, **kw: SimpleNamespace(returncode=0)), + ) + monkeypatch.setattr(cli, "deadline_ssh", lambda orch, role, cutoff: orch.ssh(role)) + monkeypatch.setattr( + cli.Orchestrator, + "cleanup", + lambda self, force: {"all_absent": True, "custody_complete": True}, + ) + calls = [] + + def intake(path, runtime, owner, run_id, manifest): + # External production graph input seam; exact owner/owned-instance negative persists. + if fault == "owner": + raise Reject("continuation graph owner/run differs") + if fault == "instance": + raise Reject("continuation graph owned instance differs") + assert runtime.owned_host("h0")["instance_id"] == 1 + assert (owner, run_id) == (OWNER.ss58, checked["run_id"]) + + def graph(path, runtime, actual, cancel): + assert j.last("network_workload_promoted") and not cancel.is_set() + assert actual is checked and path == graph_path + calls.append(cancel) + # Physical launch substitution; actual reservation journal records each kernel once. + for role in ("h0", "h1"): + for i in range(56): + runtime.reserve(role, f"signed-{i}", "workload", 1) + return { + "status": "SIGNED_GRAPH_SETTLED_PENDING_RELEASE_REVIEW", + "run_id": checked["run_id"], + "graph_sha256": fsha(path), + "fault_proof_complete": False, + "trial_finalizations": [{}] * 48, + "lineage": [{}] * 2, + "normal_total": 116, + "per_host": 58, + } + + driver = SimpleNamespace( + __file__=str(checked["files"]["graph_driver"]), + continuation_graph=intake, + decoder_continue=graph, + ) + checked["continuation_driver"] = driver + monkeypatch.setattr(cli.NetworkRuntime, "execute_plan", lambda *a: pytest.fail("raw plan")) + monkeypatch.setattr(cli.NetworkRuntime, "run_staged", lambda *a: pytest.fail("raw staged")) + monkeypatch.setattr(cli, "reviewed_launch", lambda *a: checked) + original_launch = cli.launch_action + if fault == "cancel": + + def cancelled(*args, **kwargs): + kwargs["cancel"].set() + return original_launch(*args, **kwargs) + + monkeypatch.setattr(cli, "launch_action", cancelled) + argv = [ + "continue", + str(path), + "--owner", + OWNER.ss58, + "--beacon", + "10", + "--run-dir", + str(run), + ] + if fault is not None: + with pytest.raises(Reject, match="owner/run|owned instance|cancelled"): + cli.main(argv) + assert not calls and sum(r["executions"] for r in j.all("network_execution_intent")) == 4 + else: + cli.main(argv) + result = json.loads(capsys.readouterr().out) + assert result["status"] == "SIGNED_GRAPH_SETTLED_PENDING_RELEASE_REVIEW" + assert len(calls) == 1 + assert sum(r["executions"] for r in j.all("network_execution_intent")) == 116 + assert all( + sum(r["executions"] for r in j.all("network_execution_intent", role=role)) == 58 + for role in ("h0", "h1") + ) + assert result["results"]["fault_proof_complete"] is False + + +def test_signed_operation_reserves_once_and_failed_intent_never_retries( + contract, tmp_path, monkeypatch +): + import threading + from types import SimpleNamespace + + import network_gpu_operation as adapter + from hypertrain.gpu_ops.network_qualification import sources + from hypertrain.protocol.messages_v2 import IslandJobV1 + + record, path, seal = contract + checked = cli.reviewed_launch(path, OWNER.ss58, 10, "admit") + job = IslandJobV1.model_validate_json(checked["files"]["job"].read_bytes()) + profile = checked["profile"] + profile["runtime"]["qualified"] = True + source_map = sources(TREE) + source_map["scripts/network_gpu_operation.py"] = fsha(TREE / "scripts/network_gpu_operation.py") + cutoff = int(time.time()) + 600 + job = job.model_copy(update={"deadline": cutoff}) + spec = adapter.Operation( + operation="reference", + binding="ab" * 32, + binding_kind="trial", + hotkey=OWNER.ss58, + owner=OWNER.ss58, + role="h0", + instance_id=1, + machine_id=100, + sources=source_map, + image_digest=profile["runtime"]["image_digest"], + job=job, + cutoff=cutoff, + trace=False, + ) + run = tmp_path / "physical-operation" + run.mkdir() + journal = Journal(run) + known_hosts = tmp_path / "known_hosts" + known_hosts.write_text("metadata known host") + lifecycle = SimpleNamespace( + j=journal, + roles=("h0", "h1"), + host={"h0": {"machine_id": 100}}, + cfg={"cleanup_deadline_unix": time.time() + 3600}, + deadline=lambda: time.time() + 3600, + dir=run, + cleanup_started=lambda: False, + receipt=lambda r: {"instance_id": 1}, + remote_root=lambda r: "/private/root", + ) + runtime = cli.NetworkRuntime(lifecycle, profile) + runtime.reserve("h0", "qualify", "qualification", 2) + runtime.reserve("h1", "qualify", "qualification", 2) + journal.append("network_workload_promoted") + journal.append("network_admission_authority", owner=OWNER.ss58) + journal.append("network_stage_intent", role="h0", sources=source_map) + journal.append("supervisor_ready", supervisor_pid=1) + journal.append("ssh_trusted", role="h0", known_hosts_sha256=fsha(known_hosts)) + subject = { + "role": spec.role, + "binding": spec.binding, + "operation": spec.operation, + "hotkey": spec.hotkey, + "spec_sha256": sha256(spec.model_dump_json().encode()), + } + receipt = envelope_v2.seal( + OWNER, + "Receipt", + job.run_id, + {"w": job.w, "commit_hash": sha256(canonicalize(subject)), "received_round": 10}, + 20, + ) + runtime.accept_operation(spec, canonicalize(receipt), owner=OWNER.ss58, beacon=10) + ssh = SimpleNamespace(known_hosts=known_hosts, run=lambda *a: SimpleNamespace(returncode=1)) + monkeypatch.setattr(cli, "deadline_ssh", lambda *a: ssh) + monkeypatch.setattr(cli.NetworkRuntime, "rescue", lambda *a, **kw: {}) + with pytest.raises(Reject, match="operation_stage_failed"): + runtime.operation(spec, tmp_path, cancel=threading.Event()) + assert sum(r["executions"] for r in journal.all("network_execution_intent")) == 5 + with pytest.raises(Reject, match="not_retryable"): + runtime.operation(spec, tmp_path, cancel=threading.Event()) + assert sum(r["executions"] for r in journal.all("network_execution_intent")) == 5 + + +@pytest.mark.parametrize( + "fault", [None, "owner", "instance", "absent", "cancel", "hot", "cold", "coord", "auditor"] +) +def test_signed_graph_actual_input_boundary(contract, tmp_path, fault): + """Real graph validator/authority refusal, no server, provider or CUDA launch.""" + import threading + from types import SimpleNamespace + + import network_service_proof as driver + from hypertrain.gpu_ops.network_qualification import sources + from hypertrain.miner.core import load_keyfile + + key = tmp_path / "owner.key" + key.write_text("00" * 32) + key.chmod(0o600) + owner = load_keyfile(key).ss58 + record, _, _ = contract + seed = Path(record["files"][record["roles"]["h0"]["job"]]["path"]) + job = IslandJobV1.model_validate_json(seed.read_bytes()) + body = job.manifest.body() + body["training"]["coord_pubkey"] = owner + body["training"]["auditors"] = [owner] + manifest = RunManifestV2.model_validate(body) + seed.write_text( + job.model_copy(update={"manifest": manifest, "run_id": manifest.run_id()}).model_dump_json() + ) + hot_files, cold_files = [], [] + for i in range(4): + for files, prefix, offset in ((hot_files, "hot", 1), (cold_files, "cold", 5)): + p = tmp_path / f"{prefix}-{i}.key" + p.write_text(bytes([i + offset] * 32).hex()) + p.chmod(0o600) + files.append(p) + source_map = sources(TREE) + source_map["scripts/network_gpu_operation.py"] = fsha(TREE / "scripts/network_gpu_operation.py") + hosts = { + r: {"instance_id": i + 1, "machine_id": i + 10, "image_digest": "sha256:" + "ab" * 32} + for i, r in enumerate(("h0", "h1")) + } + raw = { + "run_id": manifest.run_id(), + "owner": owner, + "owner_key_file": str(key), + "coord_key_file": str(key), + "auditor_key_files": [str(key)] * 4, + "identities": [ + { + "hotkey": load_keyfile(hot_files[i]).ss58, + "coldkey": load_keyfile(cold_files[i]).ss58, + "role": f"h{i // 2}", + "hot_key_file": str(hot_files[i]), + "cold_key_file": str(cold_files[i]), + "origin_ids": ["region:test"], + "admission_units": 1, + } + for i in range(4) + ], + "hosts": hosts, + "state_dir": str(tmp_path / "production"), + "evidence": str(tmp_path / "evidence"), + "master_https": "https://master.test:8443", + "relay_https": "https://relay.test:8444", + "metagraph_https": "https://metagraph.test:8445", + "listen_host": "127.0.0.1", + "listen_port": 8443, + "ca": str(key), + "cert": str(key), + "tls_key": str(key), + "admin_token_file": str(key), + "qualification_receipt": str(key), + "economic_authority": str(key), + "operation_sources": source_map, + "private_files": {str(p): fsha(p) for p in [key, *hot_files, *cold_files]}, + } + if fault == "owner": + raw["owner"] = "wrong" + if fault == "instance": + raw["hosts"] = {r: dict(v) for r, v in hosts.items()} + raw["hosts"]["h0"]["instance_id"] = 999 + if fault in ("hot", "cold"): + raw["identities"][3][fault + "_key_file"] = str(key) + if fault == "coord": + raw["coord_key_file"] = str(hot_files[0]) + if fault == "auditor": + raw["auditor_key_files"][3] = str(hot_files[0]) + path = tmp_path / "graph.json" + path.write_text(json.dumps(raw)) + runtime = SimpleNamespace( + lifecycle=SimpleNamespace(roles=("h0", "h1"), deadline=lambda: time.time() + 3600), + owned_host=hosts.__getitem__, + ) + if fault in ("owner", "instance", "hot", "cold", "coord", "auditor"): + with pytest.raises(driver.DriverError, match="owner/run|owned instance|signing identity"): + driver.continuation_graph(path, runtime, owner, raw["run_id"], manifest) + elif fault in ("absent", "cancel"): + cancel = threading.Event() + if fault == "cancel": + cancel.set() + checked = { + "record": { + "owner": owner, + "cutoff_unix": time.time() + 3600, + "roles": {"h0": {"job": "seed"}}, + }, + "run_id": raw["run_id"], + "files": {"seed": seed}, + } + with pytest.raises(driver.DriverError, match="production store absent|cancelled"): + driver.decoder_continue(path, runtime, checked, cancel) + else: + assert ( + driver.continuation_graph(path, runtime, owner, raw["run_id"], manifest).hosts == hosts + ) + + +def _backend_fixture(): + """Reuse original synthetic rescued-file fixture and real bootstrap writer.""" + spec = importlib.util.spec_from_file_location( + "continuation_backend_fixture", TREE / "tests/challenge/test_backend_authority_v2.py" + ) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +backend_fixture = _backend_fixture() +network = backend_fixture.network +continuation_authority = backend_fixture.authority + + +def test_continue_authentic_flat_store_intake(continuation_authority, tmp_path): + from types import SimpleNamespace + + import network_service_proof as driver + + store, manifest, configs, results, raw = continuation_authority + store._bootstrap_backend_v2(manifest, configs, results, raw) + backend_fixture.register(store, manifest) + accepted = store._record_v2( + manifest.run_id(), "qualification", manifest.training.reference_spec.image_digest + ) + assert "evidence" not in accepted and accepted["configs"] == [fsha(p) for p in configs] + receipt = tmp_path / "accepted-receipt.json" + receipt.write_bytes(canonicalize(accepted["receipt"])) + for role, config in zip(("h0", "h1"), configs, strict=True): + (tmp_path / f"cli-qualification-{role}.json").write_bytes(config.read_bytes()) + hosts = { + f"h{i}": { + "instance_id": i + 1, + "machine_id": i + 10, + "image_digest": manifest.training.reference_spec.image_digest, + } + for i in range(2) + } + runtime = SimpleNamespace(lifecycle=SimpleNamespace(dir=tmp_path), owned_host=hosts.__getitem__) + graph = SimpleNamespace(run_id=manifest.run_id(), qualification_receipt=receipt) + checked = { + "record": {"roles": {f"h{i}": {"qualification_result": f"r{i}"} for i in range(2)}}, + "files": {f"r{i}": p for i, p in enumerate(results)}, + } + driver.continuation_qualification(store, manifest, graph, runtime, checked) + (tmp_path / "cli-qualification-h1.json").write_bytes(configs[0].read_bytes()) + with pytest.raises(driver.DriverError, match="owned instance"): + driver.continuation_qualification(store, manifest, graph, runtime, checked) + + +@pytest.mark.parametrize("fault", [None, "empty", "hash", "kind"]) +def test_continue_actual_done_result_custody(tmp_path, fault): + import network_service_proof as driver + + j = Journal(tmp_path) + for role in ("h0", "h1"): + for kind, count in (("reference", 24), ("probe", 24), ("live", 4), ("audit", 4)): + for i in range(count): + name = kind + str(i) + j.append( + "network_execution_intent", role=role, name=name, phase="workload", executions=1 + ) + j.append( + "network_operation_started", + role=role, + name=name, + operation="live" if fault == "kind" else kind, + ) + if fault != "empty": + j.append( + "network_operation_done", + role=role, + name=name, + result_sha256="" if fault == "hash" else "ab" * 32, + tar_sha256="cd" * 32, + ) + if fault is not None: + with pytest.raises(driver.DriverError, match="coverage|hash absent"): + driver.continuation_results(j, ("h0", "h1")) + else: + results = driver.continuation_results(j, ("h0", "h1")) + assert len(results) == 112 and all(r["result_sha256"] and r["tar_sha256"] for r in results) + + +def test_continue_reviewed_launch_authentic_store_private_identity( + contract, continuation_authority, tmp_path, monkeypatch +): + """Real CLI intake joins original store authority and private signers, never launch.""" + from decimal import Decimal + from types import SimpleNamespace + + import uvicorn + + from hypertrain.gpu_ops import network_qualification as engine + from hypertrain.miner.admission import sign_join + from hypertrain.protocol.messages_v2 import HardwareHint + + record, record_path, _ = contract + store, original, configs, results, _ = continuation_authority + owner = backend_fixture.service.OWNER + files = record["files"] + + def put(name, value): + path = tmp_path / (name + ".json") + path.write_bytes(canonicalize(value)) + files[name] = {"path": str(path), "sha256": fsha(path)} + return path + + job = IslandJobV1.model_validate_json(Path(files["job"]["path"]).read_bytes()) + body = job.manifest.body() + body["network"] = original.network.model_dump(mode="json") + body["training"]["coord_pubkey"] = original.training.coord_pubkey + body["training"]["auditors"] = original.training.auditors + manifest = RunManifestV2.model_validate(body) + job = job.model_copy(update={"manifest": manifest, "run_id": manifest.run_id()}) + job_path = put("job", job.model_dump(mode="json")) + record.update(action="continue", owner=owner.ss58) + profile = json.loads(Path(files["profile"]["path"]).read_bytes()) + profile["runtime"]["qualified"] = True + profile["execution_and_transfer_bound_seconds"] = 1 + bounds = cli.artifact_budget(profile) + profile["artifacts"].update( + image_and_dependency_bytes=1000, + staging_bytes_per_host=1000, + rescue_bytes_per_second=bounds["rescue_min_bytes_per_second"], + ) + evidence = { + "profile_model": profile["model"], + "layout": profile["layout"], + "device": "cuda", + "torch": "2.14.0+cu130", + "full_round_transfer_bound_seconds": 1, + "no_slower_hosts": True, + "image_digest": profile["runtime"]["image_digest"], + "driver_allowlist": profile["runtime"]["driver_allowlist"], + "python": profile["runtime"]["remote_python"], + "cuda": profile["runtime"]["cuda"], + "full_round_artifacts_sha256": sha256( + canonicalize([json.loads(p.read_bytes())["rounds"] for p in results]) + ), + "outer_tape_verified": True, + "measured_rescue_bytes_per_second": profile["artifacts"]["rescue_bytes_per_second"], + "image_and_dependency_bytes": 1000, + "staging_bytes_per_host": 1000, + } + put("evidence", evidence) # Synthetic test-only runtime metadata, never production authority. + profile["runtime_evidence_sha256"] = sha256(canonicalize(evidence)) + profile_path = put("profile", profile) + source_profile = TREE / "experiments/gpu_network_v2/profile.json" + files["source_profile"] = {"path": str(source_profile), "sha256": fsha(source_profile)} + qualification_driver = sys.modules["cli_fixture_driver"] + sources, bundle = cli.effective_bundle(qualification_driver, TREE, source_profile) + put("sources", sources) + bundle_path = Path(files["bundle"]["path"]) + bundle_path.write_bytes(bundle) + files["bundle"]["sha256"] = fsha(bundle_path) + cfg = json.loads(Path(files["config"]["path"]).read_bytes()) + cfg["network_profile_file"] = str(profile_path) + for i, host in enumerate(cfg["hosts"]): + host["machine_id"] = i + 10 + put("config", cfg) + offers = json.loads(Path(files["offers"]["path"]).read_bytes()) + for i, offer in enumerate(offers): + offer["machine_id"] = i + 10 + put("offers", offers) + financial = json.loads(Path(files["financial"]["path"]).read_bytes()) + financial["instances"] = [{"id": 1}, {"id": 2}] + put("financial", financial) + cost = cli.budget.evaluate( + snapshot0_credit=Decimal("20"), + current_credit=Decimal("20"), + offers=offers, + hard_deadline_seconds=3600, + disk_gb=80, + egress_gb=30, + phase_cap=Decimal("15"), + reserve=Decimal("5"), + ) + put("plan", {**cost, "long_workload_allowed": True}) + run = tmp_path / "preflight-runtime" + run.mkdir() + j = Journal(run) + for i, (config_path, result_path) in enumerate(zip(configs, results, strict=True)): + role = f"h{i}" + hot, cold = backend_fixture.service.HOT[i * 2], backend_fixture.service.COLD[i * 2] + join = sign_join( + hot, + cold, + run_id=manifest.run_id(), + request_id=f"{i + 1:064x}", + expires_beacon=20, + policy_hash=manifest.network.admission_policy_hash, + hardware_hint=HardwareHint(device_name="test", device_count=2, driver="595.84"), + ) + join_path = put("join-" + role, join.model_dump(mode="json")) + context_name = "seed-context-" + role + put( + context_name, + { + "run_id": manifest.run_id(), + "owner": owner.ss58, + "hotkey": hot.ss58, + "coldkey": cold.ss58, + "request_id": join.request_id, + "policy_hash": join.policy_hash, + "job_sha256": fsha(job_path), + "join_sha256": fsha(join_path), + "admission_id": sha256( + f"join|{manifest.run_id()}|{cold.ss58}|{join.request_id}".encode() + ), + "state": "PROBATION", + }, + ) + record["roles"][role].update( + hotkey=hot.ss58, + join="join-" + role, + seed_context=context_name, + qualification_result="result-" + role, + ) + qualification = engine.Qualification.model_validate_json(config_path.read_bytes()) + lifecycle_receipts = json.loads(qualification.lifecycle_receipts.read_bytes()) + lifecycle_receipts["image_digest"] = profile["runtime"]["image_digest"] + qualification.lifecycle_receipts.write_bytes(canonicalize(lifecycle_receipts)) + qualification = qualification.model_copy( + update={ + "profile": profile_path, + "profile_sha256": fsha(profile_path), + "seed_job": job_path, + "hotkey": hot.ss58, + "registry_manifest": Path(files["registry"]["path"]), + "image_digest": profile["runtime"]["image_digest"], + "lifecycle_receipts_sha256": fsha(qualification.lifecycle_receipts), + } + ) + config_path.write_text(qualification.model_dump_json()) + (run / f"cli-qualification-{role}.json").write_bytes(config_path.read_bytes()) + result = json.loads(result_path.read_bytes()) + result.update(config_sha256=fsha(config_path), profile_sha256=fsha(profile_path)) + result["environment"]["image_digest"] = profile["runtime"]["image_digest"] + result_path.write_bytes(canonicalize(result)) + files["result-" + role] = {"path": str(result_path), "sha256": fsha(result_path)} + j.append("receipt", role=role, instance_id=i + 1) + accepted_evidence = { + "run_id": manifest.run_id(), + "backend": "cuda", + "reference_hash": sha256( + canonicalize(manifest.training.reference_spec.model_dump(mode="json")) + ), + "layout_hash": manifest.training.reference_spec.layout.model_dump_json(), + "configs": [fsha(p) for p in configs], + "results": [fsha(p) for p in results], + "summaries": [ + fsha(p.parent / f"round-{w}/published/rank-{r}/summary.json") + for p in results + for w in range(2) + for r in range(2) + ], + } + accepted_receipt = envelope_v2.seal( + owner, + "Receipt", + manifest.run_id(), + {"w": 0, "commit_hash": sha256(canonicalize(accepted_evidence)), "received_round": 1}, + 100, + ) + store._bootstrap_backend_v2(manifest, configs, results, canonicalize(accepted_receipt)) + backend_fixture.register(store, manifest) + assert "evidence" not in store._record_v2( + manifest.run_id(), "qualification", manifest.training.reference_spec.image_digest + ) + + def key(name, seed): + path = tmp_path / (name + ".key") + path.write_bytes(seed) + path.chmod(0o600) + return path + + owner_path = key("private-owner", b"\x71" * 32) + coord_path = key("private-coord", bytes(range(32))) + auditor_paths = [ + key("private-auditor-0", bytes(31) + b"\x01"), + key("private-auditor-1", b"\x72" * 32), + ] * 2 + identities = [ + { + "hotkey": backend_fixture.service.HOT[i].ss58, + "coldkey": backend_fixture.service.COLD[i].ss58, + "role": f"h{i // 2}", + "hot_key_file": str(key(f"private-hot-{i}", bytes([80 + i]) * 32)), + "cold_key_file": str(key(f"private-cold-{i}", bytes([90 + i]) * 32)), + "origin_ids": ["test-only"], + "admission_units": 1, + } + for i in range(4) + ] + accepted_path = put("accepted-qualification", accepted_receipt) + hosts = { + f"h{i}": { + "instance_id": i + 1, + "machine_id": i + 10, + "image_digest": profile["runtime"]["image_digest"], + } + for i in range(2) + } + operation_sources = dict(sources) + operation_sources["scripts/network_gpu_operation.py"] = fsha( + TREE / "scripts/network_gpu_operation.py" + ) + operation_sources["experiments/gpu_network_v2/profile.json"] = fsha(profile_path) + graph_input = { + "run_id": manifest.run_id(), + "owner": owner.ss58, + "owner_key_file": str(owner_path), + "coord_key_file": str(coord_path), + "auditor_key_files": [str(p) for p in auditor_paths], + "identities": identities, + "hosts": hosts, + "state_dir": str(store.state_dir), + "evidence": str(tmp_path / "new-evidence"), + "master_https": "https://master.test:8443", + "relay_https": "https://relay.test:8444", + "metagraph_https": "https://metagraph.test:8445", + "listen_host": "127.0.0.1", + "listen_port": 8443, + "ca": str(owner_path), + "cert": str(owner_path), + "tls_key": str(owner_path), + "admin_token_file": str(owner_path), + "qualification_receipt": str(accepted_path), + "economic_authority": str(accepted_path), + "operation_sources": operation_sources, + "private_files": { + str(p): fsha(p) + for p in [ + owner_path, + coord_path, + *auditor_paths, + accepted_path, + *(Path(i[k]) for i in identities for k in ("hot_key_file", "cold_key_file")), + ] + }, + } + graph_path = put("graph", graph_input) + files["graph_driver"] = { + "path": str(TREE / "scripts/network_service_proof.py"), + "sha256": fsha(TREE / "scripts/network_service_proof.py"), + } + lifecycle = cli.Orchestrator(cfg, j, SimpleNamespace(pace={}, last={}), run) + runtime = cli.NetworkRuntime(lifecycle, profile) + prohibited = [] + + def forbidden(*args, **kwargs): + prohibited.append(True) + pytest.fail("preflight crossed provider/SSH/server boundary") + + monkeypatch.setattr(cli.Provider, "call", forbidden) + monkeypatch.setattr(cli.Ssh, "run", forbidden) + monkeypatch.setattr(cli.Ssh, "put", forbidden) + monkeypatch.setattr(uvicorn.Server, "run", forbidden) + + def intake(): + for name in ("financial", "offers", "observations"): + value = json.loads(Path(files[name]["path"]).read_bytes()) + for row in value if isinstance(value, list) else [value]: + row.update( + verified_unix=int(time.time()), + requested_unix=int(time.time()), + received_unix=int(time.time()), + ) + put(name, value) + receipt = envelope_v2.seal( + owner, + "Receipt", + manifest.run_id(), + {"w": 0, "commit_hash": sha256(canonicalize(record)), "received_round": 10}, + 20, + ) + record_path.write_bytes(canonicalize({"record": record, "receipt": receipt})) + return cli.reviewed_launch(record_path, owner.ss58, 10, "continue") + + checked = intake() + driver = checked["continuation_driver"] + assert Path(driver.__file__).resolve() == TREE / "scripts/network_service_proof.py" + assert "jobs" not in checked["files"] and checked["record"]["action"] == "continue" + graph = driver.continuation_graph(graph_path, runtime, owner.ss58, checked["run_id"], manifest) + driver.continuation_qualification(store, manifest, graph, runtime, checked) + identities[3]["hot_key_file"] = str(owner_path) + put("graph", graph_input) + rejected = intake() + driver = rejected["continuation_driver"] + with pytest.raises(driver.DriverError, match="miner signing identity"): + driver.continuation_graph(graph_path, runtime, owner.ss58, rejected["run_id"], manifest) + identities[3]["hot_key_file"] = str(tmp_path / "private-hot-3.key") + put("graph", graph_input) + checked = intake() + driver = checked["continuation_driver"] + graph = driver.continuation_graph(graph_path, runtime, owner.ss58, checked["run_id"], manifest) + (run / "cli-qualification-h1.json").write_bytes(configs[0].read_bytes()) + with pytest.raises(driver.DriverError, match="owned instance"): + driver.continuation_qualification(store, manifest, graph, runtime, checked) + assert not prohibited + assert not j.all("network_execution_intent") and not j.last("network_workload_promoted") + + +@pytest.mark.parametrize("primary_error", [False, True]) +@pytest.mark.parametrize("join_error", [False, True]) +def test_continue_shutdown_exception_closes_store(primary_error, join_error): + import threading + from types import SimpleNamespace + + import network_service_proof as driver + + closed = [] + store = SimpleNamespace( + _lease_guards_v2={}, + close_v2_notifications=lambda: closed.append("notifications"), + _db=SimpleNamespace(close=lambda: closed.append("database")), + ) + server = SimpleNamespace(should_exit=False, force_exit=False) + + def join(**kwargs): + if join_error: + raise RuntimeError("HTTPS join failed") + + thread = SimpleNamespace(join=join, is_alive=lambda: True) + error = ValueError("primary graph error") + expected = ValueError if primary_error else RuntimeError if join_error else driver.DriverError + with pytest.raises(expected) as caught: + try: + if primary_error: + raise error + finally: + driver.continuation_close( + store, server, thread, None, threading.Event(), threading.Event() + ) + assert closed == ["notifications", "database"] and server.should_exit + assert server.force_exit is (not join_error) + if primary_error: + assert caught.value is error and "HTTPS" in error.__notes__[0] + + +@pytest.mark.parametrize("cancelled", [False, True]) +def test_continue_private_staging_shared_cutoff(tmp_path, monkeypatch, cancelled): + import threading + from types import SimpleNamespace + + import hypertrain.gpu_ops.remote as remote + + clock, calls = [100.0], [] + cancel = threading.Event() + (tmp_path / "logs").mkdir() + j = Journal(tmp_path) + ssh = remote.Ssh("127.0.0.1", 22, "root", tmp_path / "key", tmp_path / "known", j, "h0", 60) + lifecycle = SimpleNamespace( + dir=tmp_path, + j=j, + cfg={"cleanup_deadline_unix": 403}, + deadline=lambda: 1000, + cleanup_started=lambda: False, + ssh=lambda role: ssh, + remote_root=lambda role: "/private", + ) + runtime = cli.NetworkRuntime(lifecycle, {}) + monkeypatch.setattr(cli.time, "time", lambda: clock[0]) + + def transport(argv, **kwargs): + calls.append(kwargs["timeout"]) + clock[0] += 2 + if cancelled: + cancel.set() + return subprocess.CompletedProcess( + argv, 0, "" if kwargs.get("text") else b"", "" if kwargs.get("text") else b"" + ) + + monkeypatch.setattr(remote.subprocess, "run", transport) + p = tmp_path / "blob" + p.write_bytes(b"input") + with pytest.raises(Reject, match="absolute_deadline"): + runtime.stage_context("h0", {"context/blob": p}, "context", "test", 900, cancel) + assert calls == ([3] if cancelled else [3, 1]) + + +def test_continue_round_one_late_predecessor(contract, tmp_path, monkeypatch): + import threading + from types import SimpleNamespace + + import httpx + + import network_service_proof as driver + + record, _, _ = contract + job = IslandJobV1.model_validate_json( + Path(record["files"][record["roles"]["h0"]["job"]]["path"]).read_bytes() + ) + coord = Keypair(b"z" * 32) + body = job.manifest.body() + body["training"]["coord_pubkey"] = coord.ss58 + manifest = RunManifestV2.model_validate(body) + keys = [Keypair(bytes([i + 1] * 32)) for i in range(4)] + roster = [ + { + "hotkey": k.ss58, + "slot": i, + "q_i": "3f800000", + "admission_id": f"{i + 1:064x}", + "coldkey_group": k.ss58, + "state": "ACTIVE", + "eligible_weight": 4194304, + } + for i, k in enumerate(keys) + ] + settled, replays, built = [], [], [] + finals = {} + + def respond(request): + if request.url.path.endswith("aggregate"): + w = int(request.url.path.split("/")[-2]) + return httpx.Response( + 200, + json={ + "tape_hash": f"{w + 1:064x}", + "prev_state": "22" * 32, + "out_state": "23" * 32, + "theta_hash": "24" * 32, + }, + ) + if request.url.path.endswith("finalize"): + w = int(request.url.path.split("/")[-2]) + finals[w] = json.loads(request.content) + return httpx.Response(200, json={}) + + def build(w): + if w: + assert settled == [0] and 0 in finals + built.append(w) + opening = driver.RoundOpenV2( + w=w, + prev_final_hash=sha256(canonicalize(finals[0]["body"])) if w else "0" * 64, + theta_hash="11" * 32, + outer_state_hash="0" * 64, + center_hash="0" * 64, + roster_hash=sha256(canonicalize(roster)), + honeypot_commit="0" * 64, + d_open=2 + w * 10, + d_assign=3 + w * 10, + d_commit=4 + w * 10, + d_audit=5 + w * 10, + d_upload=6 + w * 10, + d_final=10 + w * 10, + contract_version=2, + policy_hashes={ + k: getattr(manifest.network, k) for k in driver.PolicyHashes.model_fields + }, + registry_epoch=0, + start_state_index_hash="12" * 32, + audit_mode="anchored-full", + roster=roster, + ) + + def finality(): + settled.append(w) + return {"disposition": "SETTLED", "unresolved_audits": 0, "unresolved_disputes": 0} + + return { + "round_open": envelope_v2.seal(coord, "RoundOpenV2", manifest.run_id(), opening, 30), + "miners": lambda: [ + {"operation": {}, "receipt": {}, "key": k, "pins": {}, "artifacts": tmp_path} + for k in keys + ], + "auditors": [], + "accepted_inputs": lambda: [], + "accepted_finality": finality, + } + + entries = [SimpleNamespace(hotkey=k.ss58, probation=False, weight_units=4194304) for k in keys] + monkeypatch.setattr( + driver.TapeV2, + "from_bytes", + lambda raw: SimpleNamespace( + body=SimpleNamespace(allocation=SimpleNamespace(entries=entries)) + ), + ) + monkeypatch.setattr(driver.AggregationPolicyV2, "model_validate_json", lambda raw: None) + monkeypatch.setattr(driver, "network_inputs", lambda raw: raw) + + def replay(*args, **kwargs): + replays.append(kwargs["predecessor_tape_hash"]) + return SimpleNamespace(to_bytes=lambda: b"outer") + + monkeypatch.setattr(driver, "replay_tape", replay) + monkeypatch.setattr(driver, "decoder_operation", lambda *a: None) + monkeypatch.setattr(driver, "write_network_checkpoint", lambda *a: None) + monkeypatch.setattr(driver, "verify_network_checkpoint", lambda *a: []) + with httpx.Client( + base_url="https://metadata.test", transport=httpx.MockTransport(respond) + ) as c: + lineage = driver.decoder_live_graph( + c, + None, + manifest, + [lambda: build(0), lambda: build(1)], + coord, + {}, + lambda n: None, + SimpleNamespace(get=lambda digest: b"outer"), + tmp_path / "checkpoint", + threading.Event(), + ) + assert built == settled == [0, 1] + assert replays == ["0" * 64, f"{1:064x}"] + assert lineage[1]["predecessor_tape_hash"] == lineage[0]["tape_hash"] diff --git a/tests/gpu_ops/test_network_gpu_qualification.py b/tests/gpu_ops/test_network_gpu_qualification.py new file mode 100644 index 0000000..93b311b --- /dev/null +++ b/tests/gpu_ops/test_network_gpu_qualification.py @@ -0,0 +1,350 @@ +"""Lightweight boundary/custody tests; no CUDA or training qualification.""" + +from __future__ import annotations + +import importlib.util +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from hypertrain.gpu_ops import network_qualification as driver +from hypertrain.gpu_ops.journal import fsha +from hypertrain.gpu_ops.launcher import Reject +from hypertrain.protocol.example import example_manifest +from hypertrain.protocol.messages_v2 import IslandJobV1, RunManifestV2 + +TREE = Path(__file__).parents[2] + + +def results(tmp_path: Path) -> list[Path]: + """Two synthetic CUDA-labelled JSON sets exercise parsing, never execute GPUs.""" + paths = [] + for i in range(2): + directory = tmp_path / f"host-{i}" + hashes = [] + for index in range(2): + root = directory / f"round-{index}/published" + root.mkdir(parents=True) + files = {} + for rank in range(2): + for rel in [ + "state.safetensors", + "ef.safetensors", + "delta.bin", + "leaves.json", + "trace.json", + ] + [f"checkpoints/{t}.safetensors" for t in range(0, 31, 5)]: + path = root / f"rank-{rank}" / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(bytes([index])) + files[str(path.relative_to(root))] = fsha(path) + hashes.append(files) + result = driver.Result( + status="CUDA_ROUNDS_CAPTURED_NOT_WORKLOAD_PASS", + config_sha256="11" * 32, + profile_sha256="22" * 32, + sources={"src/x.py": "33" * 32}, + environment=driver.Environment( + backend="cuda", + torch="2.14.0+cu130", + cuda="13.0", + python="/opt/hypertrain/venv/bin/python", + torch_path="/venv/torch/__init__.py", + drivers=["595.84", "595.84"], + sm_counts=[170, 170], + image_digest="sha256:" + "44" * 32, + ), + instance_id=i + 1, + machine_id=i + 10, + role=f"h{i}", + rounds=hashes, + elapsed_seconds=1, + workload_allowed=False, + ) + path = directory / "result.json" + path.write_text(result.model_dump_json()) + paths.append(path) + return paths + + +def test_bitwise_comparison_preserves_workload_gate(tmp_path: Path) -> None: + # Given + paths = results(tmp_path) + # When + result = driver.compare(paths) + # Then + assert result == {"two_host_bitwise_match": True, "workload_allowed": False} + + +def test_tampered_artifact_rejects_claimed_matching_hashes(tmp_path: Path) -> None: + # Given + paths = results(tmp_path) + (paths[1].parent / "round-1/published/rank-0/state.safetensors").write_bytes(b"tampered") + # When / Then + with pytest.raises(Reject, match="custody_changed"): + driver.compare(paths) + + +def test_same_machine_rejects_two_host_claim(tmp_path: Path) -> None: + # Given + paths = results(tmp_path) + data = json.loads(paths[1].read_bytes()) + data["machine_id"] = 10 + paths[1].write_text(json.dumps(data)) + # When / Then + with pytest.raises(Reject, match="binding_or_bitwise"): + driver.compare(paths) + + +@pytest.mark.parametrize( + "field,value", [("backend", "cpu"), ("torch", "2.14.0+cpu"), ("sm_counts", [170])] +) +def test_cpu_or_incomplete_environment_rejects( + tmp_path: Path, field: str, value: str | list[int] +) -> None: + # Given + paths = results(tmp_path) + data = json.loads(paths[0].read_bytes())["environment"] + data[field] = value + # When / Then + with pytest.raises(ValidationError): + driver.Environment.model_validate(data) + + +def test_source_manifest_detects_profile_change(tmp_path: Path) -> None: + # Given + for rel in [ + "src/hypertrain/x.py", + "experiments/gpu_network_v2/run.py", + "experiments/gpu_network_v2/orchestrate.py", + "experiments/gpu_network_v2/profile.json", + "scripts/network_gpu_qualification.py", + "docker/Dockerfile.gpu", + "pyproject.toml", + "uv.lock", + ]: + path = tmp_path / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"initial") + initial = driver.sources(tmp_path) + (tmp_path / "experiments/gpu_network_v2/profile.json").write_bytes(b"changed") + # When + changed = driver.sources(tmp_path) + # Then + assert initial != changed + + +def test_missing_runtime_proof_rejects_before_launch(tmp_path: Path) -> None: + # Given + config = tmp_path / "config.json" + config.write_text("{}") + # When / Then + with pytest.raises(ValidationError): + driver.run(config) + + +def test_partial_round_coverage_rejects_even_identical_results(tmp_path: Path) -> None: + # Given + paths = results(tmp_path) + for path in paths: + data = json.loads(path.read_bytes()) + del data["rounds"][0]["rank-1/checkpoints/30.safetensors"] + path.write_text(json.dumps(data)) + # When / Then + with pytest.raises(Reject, match="coverage_missing"): + driver.compare(paths) + + +def test_exact_profile_accepts_wire_defaults_without_training(tmp_path: Path) -> None: + # Given + profile_path = TREE / "experiments/gpu_network_v2/profile.json" + profile = json.loads(profile_path.read_bytes()) + body = example_manifest().body() + body["model"].update(profile["model"]) + body["inner"].update({k: v for k, v in profile["inner"].items() if k != "lr_schedule"}) + body["inner"]["lr_schedule"].update(profile["inner"]["lr_schedule"]) + body["outer"].update(profile["outer"]) + registry = tmp_path / "registry.json" + registry.write_bytes(b'{"schemaVersion":2,"layers":[{"size":42}]}') + image = "sha256:" + fsha(registry) + body["reference_spec"].update(image_digest=image, driver_allowlist=["595.84"], sm_count=170) + body["reference_spec"]["layout"].update(profile["layout"]) + wrapper = RunManifestV2.model_validate( + { + "manifest_version": 2, + "training": body, + "network": { + **{ + k: "11" * 32 + for k in ( + "admission_policy_hash", + "economics_policy_hash", + "aggregation_policy_hash", + "dispute_policy_hash", + "audit_policy_hash", + "relay_registry_hash", + ) + }, + "audit_mode": "anchored-full", + "full_anchor_version": 1, + "capabilities": ["island-replay", "all-level-disputes", "transport-receipts"], + }, + } + ) + job = IslandJobV1( + job_version=1, + run_id=wrapper.run_id(), + w=0, + manifest=wrapper, + sample_ids=list(range(60)), + global_step0=0, + start_state_sha256="22" * 32, + ef_in_sha256="33" * 32, + v0_sha256="44" * 32, + object_paths={ + "start_state": "start", + "ef_in": "ef", + "v0": "v0", + "samples": "samples", + "sample_proofs": "proofs", + }, + deadline=5000, + ) + job_path = tmp_path / "job.json" + job_path.write_text(job.model_dump_json()) + receipts = tmp_path / "receipts.json" + receipts.write_text( + json.dumps( + { + "instance_id": 1, + "machine_id": 2, + "role": "h0", + "admitted_unix": 100, + "image_digest": image, + "known_hosts_sha256": "55" * 32, + "lifecycle_contract_sha256": "66" * 32, + "supervisor_pid": 123, + "deadline_unix": 3700, + "cuda_observed": True, + } + ) + ) + cfg = driver.Qualification( + tree=TREE, + profile=profile_path, + profile_sha256=fsha(profile_path), + sources=driver.sources(TREE), + registry_manifest=registry, + image_digest=image, + driver_allowlist=["595.84"], + seed_job=job_path, + hotkey=body["coord_pubkey"], + output=tmp_path / "out", + admitted_unix=100, + instance_id=1, + machine_id=2, + role="h0", + lifecycle_contract_sha256="66" * 32, + known_hosts_sha256="55" * 32, + lifecycle_receipts=receipts, + lifecycle_receipts_sha256=fsha(receipts), + ) + # When + accepted = driver.check_inputs(cfg) + # Then + assert accepted.run_id == wrapper.run_id() + + +def test_script_reexports_exact_packaged_engine() -> None: + # Given: existing source-host tools still load the script by filename. + spec = importlib.util.spec_from_file_location( + "qualification_wrapper_test", TREE / "scripts/network_gpu_qualification.py" + ) + assert spec and spec.loader + wrapper = importlib.util.module_from_spec(spec) + # When + spec.loader.exec_module(wrapper) + # Then: there is only one strict schema/function implementation. + for name in ( + "Boundary", + "Qualification", + "Environment", + "Result", + "sources", + "bundle", + "check_inputs", + "inspect_environment", + "capture", + "run", + "compare", + "main", + ): + assert getattr(wrapper, name) is getattr(driver, name) + + +def test_package_artifact_import_without_repository_cwd(tmp_path: Path) -> None: + # Given: final packaging gate supplies an actual built wheel; source tests use package root. + artifact = Path(os.environ.get("HT_QUALIFICATION_WHEEL", Path(driver.__file__).parents[2])) + code = """ +import sys +from pathlib import Path +sys.path.insert(0, sys.argv[1]) +from pydantic import ValidationError +from hypertrain.gpu_ops import network_qualification as engine +from hypertrain.challenge.store import ChallengeStore +from hypertrain.protocol.example import example_manifest +from hypertrain.protocol.messages_v2 import RunManifestV2 +from hypertrain.trainer.config import TrainConfig +from hypertrain.trainer.model import param_shapes +assert str(Path(engine.__file__)).startswith(sys.argv[1]) +assert not Path("scripts/network_gpu_qualification.py").exists() +assert engine.Qualification.__module__ == "hypertrain.gpu_ops.network_qualification" +payload = {"tree":"tree", "profile":"profile", "profile_sha256":"11"*32, +"sources":{}, "registry_manifest":"registry", "image_digest":"sha256:"+"22"*32, +"driver_allowlist":["driver"], "seed_job":"job", "hotkey":"key", "output":"out", +"admitted_unix":1, "instance_id":1, "machine_id":2, "role":"h0", +"lifecycle_contract_sha256":"33"*32, "known_hosts_sha256":"44"*32, +"lifecycle_receipts":"receipt", "lifecycle_receipts_sha256":"55"*32} +assert engine.Qualification.model_validate(payload).tree == Path("tree") +for bad in ({**payload,"instance_id":True}, {**payload,"unexpected":1}): + try: + engine.Qualification.model_validate(bad) + except ValidationError: + pass + else: + raise AssertionError("strict schema weakened") +body = example_manifest().body() +body["model"]["param_count"] = sum(__import__("math").prod(shape) for shape in + param_shapes(TrainConfig.from_manifest(example_manifest()).model).values()) +manifest = RunManifestV2.model_validate({"manifest_version":2,"training":body,"network":{ +**{n:"11"*32 for n in ("admission_policy_hash","economics_policy_hash", +"aggregation_policy_hash","dispute_policy_hash","audit_policy_hash","relay_registry_hash")}, +"audit_mode":"anchored-full","full_anchor_version":1, +"capabilities":["island-replay","all-level-disputes","transport-receipts"]}}) +store = ChallengeStore.__new__(ChallengeStore) +try: + store._bootstrap_backend_v2(manifest, (Path("missing-config"),)*2, + (Path("missing-result"),)*2, b"") +except FileNotFoundError as error: + assert error.filename == "missing-config", str(error) +else: + raise AssertionError("expected real config read after packaged importer") +print("PACKAGED_QUALIFICATION_STRICT_SCHEMA_AND_STORE_IMPORT_PASS") +""" + # When: isolated interpreter ignores checkout PYTHONPATH and runs outside repository. + result = subprocess.run( + [sys.executable, "-I", "-c", code, str(artifact.resolve())], + cwd=tmp_path, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + # Then: real package/store seam imports without a sibling scripts tree. + assert result.returncode == 0, result.stderr + assert "PACKAGED_QUALIFICATION_STRICT_SCHEMA_AND_STORE_IMPORT_PASS" in result.stdout diff --git a/tests/gpu_ops/test_network_gpu_seed.py b/tests/gpu_ops/test_network_gpu_seed.py new file mode 100644 index 0000000..1e057fb --- /dev/null +++ b/tests/gpu_ops/test_network_gpu_seed.py @@ -0,0 +1,107 @@ +"""Real tiny D2 inputs; no qualification result, CUDA execution or training.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + +import hypertrain.trainer # noqa: F401 +from hypertrain.auditor.replay import unpack_state +from hypertrain.protocol.hashing import sha256_hex +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.keys import Keypair +from hypertrain.trainer.config import TrainConfig +from hypertrain.trainer.model import init_params + +ROOT = Path(__file__).resolve().parents[2] +spec = importlib.util.spec_from_file_location( + "network_seed_builder", ROOT / "scripts/network_gpu_seed.py" +) +assert spec is not None and spec.loader is not None +seed = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = seed +spec.loader.exec_module(seed) + + +@pytest.fixture +def generated(tmp_path): + public, private = tmp_path / "public", tmp_path / "private" + digest = seed.generate( + public, private, "sha256:" + "ab" * 32, ["candidate-driver-not-observed"], 1000, 2000 + ) + return public, private, digest + + +def test_real_seed_exact_profile_genesis_and_tokenizer(generated): + import torch + + public, private, digest = generated + job = seed.validate(public, digest) + m = job.manifest.training + assert m.model.param_count == 61856 and m.model.compute_dtype == "bf16" + assert m.reference_spec.layout.model_dump() == dict( + pp=1, n_gpus=2, dp_size=1, ep_size=2, zero1=True + ) + assert m.inner.H == 30 and m.inner.J == 5 and m.inner.state_policy == "carry" + theta, state = unpack_state((public / "start_state").read_bytes()) + assert sum(x.numel() for x in theta.values()) == 61856 + assert state is not None and state.step == 0 and set(state.m) == set(state.v) == set(theta) + assert all(x.dtype == torch.float32 for x in theta.values()) + assert all(torch.count_nonzero(x) == 0 for x in (*state.m.values(), *state.v.values())) + ef, _ = unpack_state((public / "ef_in").read_bytes()) + v0, _ = unpack_state((public / "v0").read_bytes()) + assert all(torch.count_nonzero(x) == 0 for x in ef.values()) and not v0 + expected = init_params(TrainConfig.from_manifest_v2(job.manifest).model) + assert all(torch.equal(theta[k], expected[k]) for k in theta) + index = json.loads((public / "seed.json").read_bytes()) + assert "NOT_CUDA_QUALIFIED" in index["claim"] + assert not json.loads((public / "reference.json").read_bytes())["cuda_observed"] + for role, key in index["roles"].items(): + secret = private / (role + ".seed") + assert secret.stat().st_mode & 0o777 == 0o600 + assert Keypair(secret.read_bytes()).ss58 == key + assert not list(public.rglob("*.seed")) + + +@pytest.mark.parametrize("fault", ["genesis", "samples", "profile", "signature", "hash"]) +def test_seed_tampering_rejects(generated, fault): + public, _, digest = generated + index = json.loads((public / "seed.json").read_bytes()) + if fault == "hash": + digest = "00" * 32 + else: + name = { + "genesis": "start_state", + "samples": "samples", + "profile": "profile.json", + "signature": "job-envelope.json", + }[fault] + path = public / name + if fault == "profile": + body = json.loads(path.read_bytes()) + body["model"]["d_model"] = 64 + path.write_bytes(canonicalize(body)) + elif fault == "signature": + body = json.loads(path.read_bytes()) + body["sig"] = "00" * 64 + path.write_bytes(canonicalize(body)) + else: + raw = bytearray(path.read_bytes()) + raw[-1] ^= 1 + path.write_bytes(raw) + index["files"][name] = sha256_hex(path.read_bytes()) + raw = canonicalize(index) + (public / "seed.json").write_bytes(raw) + digest = sha256_hex(raw) + with pytest.raises(ValueError): + seed.validate(public, digest) + + +def test_missing_runtime_inputs_reject_before_output(tmp_path): + with pytest.raises(ValueError): + seed.generate(tmp_path / "out", tmp_path / "keys", "sha256:" + "ab" * 32, [], 1000, 2000) + assert not (tmp_path / "out").exists() and not (tmp_path / "keys").exists() diff --git a/tests/gpu_ops/test_network_lifecycle.py b/tests/gpu_ops/test_network_lifecycle.py new file mode 100644 index 0000000..f300c26 --- /dev/null +++ b/tests/gpu_ops/test_network_lifecycle.py @@ -0,0 +1,545 @@ +"""Real SSH/separate supervisor, no CREATE or production provider traffic.""" + +from __future__ import annotations + +import getpass +import importlib.util +import json +import os +import select +import signal +import socket +import subprocess +import sys +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest + +from hypertrain.gpu_ops.journal import Journal +from hypertrain.gpu_ops.launcher import Orchestrator, Reject +from hypertrain.gpu_ops.provider import Provider +from hypertrain.gpu_ops.remote import ensure_identity + +TREE = Path(__file__).parents[2] + + +def network_module(): + spec = importlib.util.spec_from_file_location( + "network_lifecycle", TREE / "experiments/gpu_network_v2/orchestrate.py" + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize("fault", [None, "export", "changed-key", "rank", "continuation"]) +def test_separate_supervisor_parent_exit_rescues_both_roles_before_delete( + tmp_path: Path, + fault: str | None, +) -> None: + ssh_servers = [] + ports = [] + run = tmp_path / "run" + run.mkdir() + journal = Journal(run) + identity = ensure_identity(run) + for role in ("h0", "h1"): + host = tmp_path / f"host-{role}" + subprocess.run( + ["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", str(host)], + check=True, + capture_output=True, + ) + authorized = tmp_path / f"authorized-{role}" + authorized.write_bytes(identity.with_suffix(".pub").read_bytes()) + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + ports.append(sock.getsockname()[1]) + cfg_path = tmp_path / f"sshd-{role}" + cfg_path.write_text( + f"ListenAddress 127.0.0.1\nPort {ports[-1]}\nHostKey {host}\n" + f"PidFile {tmp_path}/{role}.pid\nAuthorizedKeysFile {authorized}\nStrictModes no\n" + "PermitRootLogin prohibit-password\nPasswordAuthentication no\n" + f"KbdInteractiveAuthentication no\nUsePAM {'yes' if os.getuid() == 0 else 'no'}\n" + f"AllowUsers {getpass.getuser()}\nLogLevel VERBOSE\n" + ) + proc = subprocess.Popen( + ["/usr/sbin/sshd", "-D", "-e", "-f", str(cfg_path)], + stderr=subprocess.PIPE, + start_new_session=True, + ) + ssh_servers.append(proc) + assert proc.stderr is not None + assert select.select([proc.stderr], [], [], 15)[0] + assert b"Server listening" in proc.stderr.readline() + + deleted = set() + methods = [] + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + return + + def do_GET(self): + methods.append(("GET", self.path)) + if self.path.startswith("/api/v0/users/current/"): + body, status = {"credit": 20, "balance": 0}, 200 + elif self.path.startswith("/api/v0/volumes/"): + body, status = {"volumes": []}, 200 + elif self.path.startswith("/api/v1/instances/"): + body = { + "success": True, + "instances": [{"id": i} for i in (1, 2) if i not in deleted], + } + status = 200 + else: + iid = int(self.path.split("?", 1)[0].rstrip("/").split("/")[-1]) + body, status = ( + ({"success": True, "instances": {"id": iid}}, 200) + if iid not in deleted + else ({"instances": None}, 200) + ) + data = json.dumps(body).encode() + self.send_response(status) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_DELETE(self): + methods.append(("DELETE", self.path)) + iid = int(self.path.split("?", 1)[0].rstrip("/").split("/")[-1]) + deleted.add(iid) + data = b'{"success":true}' + self.send_response(200) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_PUT(self): + methods.append(("PUT", self.path)) + self.send_error(403) + + api = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + api_thread = threading.Thread(target=api.serve_forever) + api_thread.start() + key = tmp_path / "key" + key.write_text("a" * 32) + key.chmod(0o600) + profile_path = TREE / "experiments/gpu_network_v2/profile.json" + cfg = { + "base_url": f"http://127.0.0.1:{api.server_port}", + "key_file": str(key), + "hosts": [{"role": r, "machine_id": i} for i, r in enumerate(("h0", "h1"), 1)], + "cleanup_contract": "network-v2", + "network_profile_file": str(profile_path), + "evidence_path": str(tmp_path / "cleanup-evidence.json"), + "remote_root": str(tmp_path / "remote-{role}"), + "remote_python": sys.executable, + "ssh_user": getpass.getuser(), + "image": "candidate@sha256:" + "11" * 32, + "provider_min_interval_seconds": 0, + "poll_seconds": 0, + "delete_attempts": 1, + "absence_attempts": 1, + "supervisor_check_seconds": 30, + } + (run / "config.json").write_text(json.dumps(cfg)) + parent = subprocess.Popen( + [sys.executable, "-c", "import sys; sys.stdin.buffer.read()"], stdin=subprocess.PIPE + ) + journal.append("transaction_open", live=False, pid=parent.pid) + journal.append("admitted", deadline_unix=__import__("time").time() + 3600) + orch = Orchestrator(cfg, journal, Provider(cfg["base_url"], "a" * 32, run, journal, False), run) + runtime = network_module().NetworkRuntime(orch, json.loads(profile_path.read_bytes())) + for i, role in enumerate(("h0", "h1"), 1): + journal.append("receipt", role=role, instance_id=i) + journal.append("instance_ready", role=role, ssh_host="127.0.0.1", ssh_port=ports[i - 1]) + orch.trust(role) + job = tmp_path / f"job-{role}" + for rank in (0, 1): + path = job / f"rank-{rank}" / "checkpoints" / "30.safetensors" + path.parent.mkdir(parents=True) + path.write_bytes(f"{role}-{rank}".encode()) + (job / "driver-result.json").write_text(json.dumps({"role": role})) + runtime.stage(role, TREE, {"driver": job}) + prior_archives = {} + if fault == "continuation": + for role in orch.roles: + runtime.reserve(role, "qualification", "qualification", 2) + prior_archives[role] = runtime.rescue(role) + journal.append("network_workload_promoted") + for role in orch.roles: + runtime.reserve(role, "continuation", "workload", 1) + ssh = orch.ssh(role) + root = orch.remote_root(role) + assert ( + ssh.run( + f"mkdir -p {root}/out/continuation/checkpoints && " + f"printf new-checkpoint > {root}/out/continuation/checkpoints/60.bin", + "continuation", + run / "logs", + ).returncode + == 0 + ) + elif fault == "export": + root0 = Path(orch.remote_root("h0")) + (root0 / "out/escape").symlink_to("/etc/hosts") + elif fault == "changed-key": + known = orch.ssh("h0").known_hosts + subprocess.run( + ["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", str(tmp_path / "wrong")], + check=True, + capture_output=True, + ) + wrong = (tmp_path / "wrong.pub").read_text().split()[:2] + known.write_text(f"[127.0.0.1]:{ports[0]} {' '.join(wrong)}\n") + elif fault == "rank": + runtime.reserve("h0", "failed-driver", "qualification", 2) + failed = orch.ssh("h0").run( + f"{sys.executable} -c 'raise SystemExit(7)'", "failed-rank", run / "logs" + ) + assert failed.returncode == 7 + journal.append( + "network_qualification_failed", role="h0", name="failed-driver", rc=failed.returncode + ) + supervisor = subprocess.Popen( + [sys.executable, "-m", "hypertrain.gpu_ops.supervisor", str(run)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + start_new_session=True, + ) + try: + assert supervisor.stdout is not None + assert select.select([supervisor.stdout], [], [], 15)[0] + assert supervisor.stdout.readline() == b"READY\n" + parent.terminate() + parent.wait(timeout=10) + fd = os.pidfd_open(supervisor.pid) + try: + assert select.select([fd], [], [], 45)[0] + finally: + os.close(fd) + custody_fault = fault in ("export", "changed-key") + assert supervisor.wait(timeout=2) == (4 if custody_fault else 0) + assert deleted == {1, 2} + assert len(journal.all("absence_confirmed")) == 2 + assert journal.last("supervisor_parent_exit") + assert len(journal.all("network_rescued")) == ( + 4 if fault == "continuation" else 1 if custody_fault else 2 + ) + if fault == "continuation": + for role, old in prior_archives.items(): + final = journal.last("network_rescued", role=role) + assert final and final["tar"] != old["tar"] + assert "continuation/checkpoints/60.bin" in final["files"] + assert Path(old["tar"]).is_file() + assert runtime.rescue(role) == final + for rec in journal.all("network_rescued"): + assert any("driver-result.json" in n for n in rec["files"]) + assert len([n for n in rec["files"] if "checkpoints/30.safetensors" in n]) == 2 + records = journal.records() + last_rescue = max( + i + for i, r in enumerate(records) + if r["kind"] in ("network_rescued", "network_rescue_failed") + ) + assert all(i > last_rescue for i, r in enumerate(records) if r["kind"] == "delete_ack") + assert not any(m == "PUT" for m, _ in methods) + assert bool(journal.last("supervisor_censored")) == custody_fault + if custody_fault: + again = orch.cleanup(force=True) + assert again["all_absent"] and again["censored"] + assert not again["custody_complete"] + if fault == "rank": + assert sum(r["executions"] for r in journal.all("network_execution_intent")) == 2 + assert journal.last("network_workload_promoted") is None + finally: + if supervisor.poll() is None: + os.killpg(supervisor.pid, signal.SIGKILL) + supervisor.wait(timeout=5) + if parent.poll() is None: + parent.terminate() + parent.wait(timeout=5) + api.shutdown() + api.server_close() + api_thread.join() + for proc in ssh_servers: + os.killpg(proc.pid, signal.SIGTERM) + proc.wait(timeout=5) + + +def test_exact_qualification_reservations_and_immutable_deadline(tmp_path: Path) -> None: + import time + + cfg = { + "base_url": "http://127.0.0.1:1", + "hosts": [{"role": "h0"}, {"role": "h1"}], + "cleanup_contract": "network-v2", + } + j = Journal(tmp_path) + j.append("admitted", deadline_unix=time.time() + 7200) + first = j.append("create_intent", role="h0") + orch = Orchestrator(cfg, j, Provider(cfg["base_url"], "a" * 32, tmp_path, j, False), tmp_path) + expected = first["unix"] + 3600 + j.append("admitted", deadline_unix=time.time() + 10000) + j.append("create_intent", role="h1") + assert orch.deadline() == expected + mod = network_module() + runtime = mod.NetworkRuntime( + orch, json.loads((TREE / "experiments/gpu_network_v2/profile.json").read_bytes()) + ) + runtime.reserve("h0", "qual", "qualification", 2) + with pytest.raises(Reject, match="not_retryable"): + runtime.reserve("h0", "qual", "qualification", 2) + runtime.reserve("h1", "qual", "qualification", 2) + assert sum(r["executions"] for r in j.all("network_execution_intent")) == 4 + with pytest.raises(Reject, match="not_promoted"): + runtime.reserve("h0", "work", "workload", 1) + with pytest.raises(Reject, match="not_verified"): + runtime.record_qualification("h0", "qual", {"passed": False}) + with pytest.raises(Reject, match="exhausted"): + runtime.reserve("h0", "retryqual", "qualification", 2) + j.append("network_workload_promoted", remaining_total=122, remaining_per_host=61) + for role in ("h0", "h1"): + runtime.reserve(role, "trace-bundle", "workload", 12) + for i in range(49): + runtime.reserve(role, f"work-{i}", "workload", 1) + with pytest.raises(Reject, match="exhausted"): + runtime.reserve(role, "overflow", "workload", 1) + assert sum(r["executions"] for r in j.all("network_execution_intent")) == 126 + + +def test_qualification_entrypoint_consumes_frozen_schema( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from types import SimpleNamespace + + from hypertrain.gpu_ops.journal import fsha + + j = Journal(tmp_path) + first = j.append("create_intent", role="h0") + j.append("receipt", role="h0", instance_id=1) + j.append("supervisor_ready") + j.append("staged", role="h0") + cfg = { + "base_url": "http://127.0.0.1:1", + "hosts": [{"role": "h0"}], + "cleanup_contract": "network-v2", + "remote_root": str(tmp_path / "remote-{role}"), + } + orch = Orchestrator(cfg, j, Provider(cfg["base_url"], "a" * 32, tmp_path, j, False), tmp_path) + known = tmp_path / "known_hosts-h0" + known.write_text("strict-key-custody") + image = "sha256:" + "11" * 32 + root = Path(orch.remote_root("h0")) + payload = { + "tree": str(root), + "profile": str(root / "profile.json"), + "profile_sha256": "22" * 32, + "sources": {}, + "registry_manifest": str(root / "registry.json"), + "image_digest": image, + "driver_allowlist": ["580"], + "seed_job": str(root / "seed.json"), + "hotkey": "h0", + "output": str(root / "out/qualification"), + "admitted_unix": int(first["unix"]), + "instance_id": 1, + "machine_id": 2, + "role": "h0", + "lifecycle_contract_sha256": "33" * 32, + "known_hosts_sha256": fsha(known), + "lifecycle_receipts": str(root / "receipts.json"), + "lifecycle_receipts_sha256": "44" * 32, + } + commands = [] + + def get(remote, local): + local.write_text(json.dumps(payload)) + + def run(command, tag, logs): + commands.append(command) + return SimpleNamespace(returncode=0) + + ssh = SimpleNamespace(get=get, run=run, known_hosts=known, timeout=0) + monkeypatch.setattr(orch, "ssh", lambda role: ssh) + mod = network_module() + monkeypatch.setattr( + mod, + "staged_command", + lambda *a, **kw: "python experiments/gpu_network_v2/run.py out/probe cuda", + ) + runtime = mod.NetworkRuntime( + orch, json.loads((TREE / "experiments/gpu_network_v2/profile.json").read_bytes()) + ) + orch.cfg.update(image="candidate@" + image, remote_python="python") + result = runtime.run_qualification("h0", "probe", "qualification.json") + assert result["executions"] == 2 and result["workload_allowed"] is False + assert commands == ["python scripts/network_gpu_qualification.py run qualification.json"] + driver = sys.modules["network_driver_contract"] + parsed = driver.Qualification.model_validate(payload) + assert parsed.tree == root + with pytest.raises(__import__("pydantic").ValidationError): + driver.Qualification.model_validate({**payload, "instance_id": True}) + + +@pytest.mark.parametrize("left,gap", [(0.0, 0.0), (0.5, 0.0), (0.5, 1.0), (1.0, 0.25)]) +def test_cleanup_recovery_deadline_covers_both_roles_and_pacing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, left: float, gap: float +) -> None: + import hypertrain.gpu_ops.launcher as launcher + from hypertrain.gpu_ops.provider import Response + + clock = [100.0] + monkeypatch.setattr(launcher.time, "time", lambda: clock[0]) + cfg = { + "base_url": "http://127.0.0.1:1", + "hosts": [{"role": r} for r in ("h0", "h1")], + "cleanup_contract": "network-v2", + "cleanup_deadline_unix": 100.0 + left, + "provider_min_interval_seconds": gap, + } + j = Journal(tmp_path) + j.append("admitted", deadline_unix=3700) + for role in ("h0", "h1"): + j.append("create_intent", role=role, label=role, baseline_ids=[]) + waits, calls = [], [] + + def sleep(seconds): + waits.append(seconds) + clock[0] += seconds + + provider = Provider(cfg["base_url"], "a" * 32, tmp_path, j, False, sleep=sleep) + orch = Orchestrator(cfg, j, provider, tmp_path) + monkeypatch.setattr(provider, "monotonic", lambda: clock[0]) + endpoint = provider.endpoint("GET", "/api/v1/instances/") + provider.last[endpoint] = provider.monotonic() + + def call(method, path, tag, body=None, timeout=30): + calls.append(timeout) + clock[0] += timeout + return Response(429, {}, 15.0, "mock", None) + + monkeypatch.setattr(provider, "call", call) + owned, unresolved = orch.owned() + assert owned == [] and unresolved == ["h0", "h1"] + assert calls == ([left - gap] if left > gap else []) + assert waits == ([gap] if 0 < gap < left else []) + clock[0] = 100.0 + left + assert orch.absent_once(1, "expired") == "unknown" + j.append("receipt", role="h0", instance_id=1) + assert orch.delete("h0") is False + assert len(calls) <= 1 + cfg["network_profile_file"] = str(TREE / "experiments/gpu_network_v2/profile.json") + result = orch.cleanup(force=True) + assert result["all_absent"] is False and result["censored"] is True + assert result["custody_complete"] is False + assert cfg["cleanup_deadline_unix"] == 100.0 + left + + +def test_cleanup_absence_show_and_delete_share_shrinking_clock( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import hypertrain.gpu_ops.launcher as launcher + from hypertrain.gpu_ops.provider import Response + + clock = [100.0] + monkeypatch.setattr(launcher.time, "time", lambda: clock[0]) + cfg = { + "base_url": "http://127.0.0.1:1", + "hosts": [{"role": "h0"}, {"role": "h1"}], + "cleanup_deadline_unix": 101.0, + "provider_min_interval_seconds": 0, + } + j = Journal(tmp_path) + provider = Provider(cfg["base_url"], "a" * 32, tmp_path, j, False) + orch = Orchestrator(cfg, j, provider, tmp_path) + waits, calls = [], [] + monkeypatch.setattr(provider, "sleep", waits.append) + + def call(method, path, tag, body=None, timeout=30): + calls.append((method, timeout)) + clock[0] += 0.25 + return Response( + 200, + {"success": True, "instances": [] if "/api/v1/" in path else None}, + None, + "mock", + None, + ) + + monkeypatch.setattr(provider, "call", call) + for role in ("h0", "h1"): + j.append("receipt", role=role, instance_id=1) + assert orch.absent_once(1, "h0") == "absent" + assert orch.delete("h0") is True + assert orch.delete("h1") is True + assert orch.confirm_absence("h1") is False + assert calls == [("GET", 1.0), ("GET", 0.75), ("DELETE", 0.5), ("DELETE", 0.25)] + assert waits == [] + + +@pytest.mark.parametrize("status,parsed", [(429, {}), (200, {"success": True, "instances": "bad"})]) +def test_unknown_inventory_never_confirms_absence( + tmp_path: Path, status: int, parsed: dict +) -> None: + from hypertrain.gpu_ops.provider import Response + + cfg = {"base_url": "http://127.0.0.1:1", "hosts": [{"role": "h0"}]} + j = Journal(tmp_path) + provider = Provider(cfg["base_url"], "a" * 32, tmp_path, j, False) + provider.call = lambda *args, **kwargs: Response(status, parsed, None, "mock", None) + orch = Orchestrator(cfg, j, provider, tmp_path) + assert orch.absent_once(1, "unknown") == "unknown" + + +def test_promotion_requires_real_qualification_and_remaining_original_clock( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + import time + + mod = network_module() + cfg = { + "base_url": "http://127.0.0.1:1", + "hosts": [{"role": "h0"}, {"role": "h1"}], + "cleanup_contract": "network-v2", + } + j = Journal(tmp_path) + j.append("admitted", deadline_unix=time.time() + 3600) + orch = Orchestrator(cfg, j, Provider(cfg["base_url"], "a" * 32, tmp_path, j, False), tmp_path) + profile = json.loads((TREE / "experiments/gpu_network_v2/profile.json").read_bytes()) + profile["execution_and_transfer_bound_seconds"] = 40 + runtime = mod.NetworkRuntime(orch, profile) + for role in orch.roles: + runtime.reserve(role, "qual", "qualification", 2) + # Reservation state-machine oracle only; actual runtime_admission stays tested separately. + monkeypatch.setattr(mod, "runtime_admission", lambda *args: 3300) + with pytest.raises(Reject, match="promotion_gate"): + runtime.promote({}, {"admit": True, "long_workload_allowed": True}) + for role in orch.roles: + verified = { + "passed": True, + "root_verified": True, + "result_sha256": "11" * 32, + "artifact_manifest_sha256": "22" * 32, + } + done = runtime.record_qualification(role, "qual", verified) + assert runtime.record_qualification(role, "qual", verified) == done + with pytest.raises(Reject, match="changed_on_resume"): + runtime.record_qualification(role, "qual", {**verified, "result_sha256": "44" * 32}) + promoted = runtime.promote( + {"root_reviewed_two_host_full_round_transfer": True}, + {"admit": True, "long_workload_allowed": True}, + ) + assert promoted["remaining_total"] == 122 + assert promoted["remaining_per_host"] == 61 + j.append("admitted", deadline_unix=time.time() + 300) + with pytest.raises(Reject, match="promotion_gate"): + runtime.promote( + {"root_reviewed_two_host_full_round_transfer": True}, + {"admit": True, "long_workload_allowed": True}, + ) diff --git a/tests/gpu_ops/test_network_role_dispatch.py b/tests/gpu_ops/test_network_role_dispatch.py new file mode 100644 index 0000000..ec7645b --- /dev/null +++ b/tests/gpu_ops/test_network_role_dispatch.py @@ -0,0 +1,1022 @@ +"""Actual strict loopback SSH; hermetic publication seam only, no kernel/GPU claim.""" + +from __future__ import annotations + +import ast +import getpass +import importlib.util +import json +import os +import select +import shlex +import socket +import subprocess +import sys +import threading +import time +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from hypertrain.gpu_ops.journal import Journal, fsha, sha256 +from hypertrain.gpu_ops.launcher import Orchestrator, Reject +from hypertrain.gpu_ops.provider import Provider +from hypertrain.gpu_ops.remote import ensure_identity +from hypertrain.protocol import envelope_v2 +from hypertrain.protocol.example import example_manifest +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.keys import Keypair +from hypertrain.protocol.messages_v2 import IslandJobV1, RunManifestV2 + +TREE = Path(__file__).parents[2] + + +def module(name, path): + spec = importlib.util.spec_from_file_location(name, path) + assert spec and spec.loader + value = importlib.util.module_from_spec(spec) + sys.modules[name] = value + spec.loader.exec_module(value) + return value + + +RUNTIME = module("role_runtime", TREE / "experiments/gpu_network_v2/orchestrate.py") +OP = module("role_operation", TREE / "scripts/network_gpu_operation.py") +OWNER = Keypair(bytes(range(32))) + + +@pytest.fixture +def context(tmp_path, mock_factory): + run = tmp_path / "run" + run.mkdir() + (run / "logs").mkdir() + journal = Journal(run) + identity = ensure_identity(run) + host = tmp_path / "host" + subprocess.run( + ["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", str(host)], + check=True, + capture_output=True, + ) + authorized = tmp_path / "authorized" + authorized.write_bytes(identity.with_suffix(".pub").read_bytes()) + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + config = tmp_path / "sshd" + config.write_text( + f"ListenAddress 127.0.0.1\nPort {port}\nHostKey {host}\nPidFile {tmp_path}/sshd.pid\n" + f"AuthorizedKeysFile {authorized}\nStrictModes no\nPermitRootLogin prohibit-password\n" + "PasswordAuthentication no\nKbdInteractiveAuthentication no\n" + f"UsePAM {'yes' if os.getuid() == 0 else 'no'}\nAllowUsers {getpass.getuser()}\n" + "LogLevel VERBOSE\n" + ) + server = subprocess.Popen( + ["/usr/sbin/sshd", "-D", "-e", "-f", str(config)], + stderr=subprocess.PIPE, + start_new_session=True, + ) + assert server.stderr and select.select([server.stderr], [], [], 15)[0] + assert b"Server listening" in server.stderr.readline() + provider = mock_factory() + cutoff = int(time.time()) + 3500 + profile = json.loads((TREE / "experiments/gpu_network_v2/profile.json").read_bytes()) + profile["runtime"].update( + image_digest="sha256:" + "11" * 32, + driver_allowlist=["595.84"], + qualified=True, + remote_python=sys.executable, + ) + cfg = { + "base_url": provider.base, + "hosts": [{"role": "h0", "machine_id": 5000}, {"role": "h1", "machine_id": 5001}], + "cleanup_contract": "network-v2", + "remote_root": str(tmp_path / "remote-{role}"), + "remote_python": sys.executable, + "ssh_user": getpass.getuser(), + "image": "candidate@" + profile["runtime"]["image_digest"], + "cleanup_deadline_unix": cutoff, + } + orch = Orchestrator(cfg, journal, Provider(provider.base, "a" * 32, run, journal, False), run) + journal.append("admitted", deadline_unix=cutoff) + journal.append("network_admission_authority", owner=OWNER.ss58) + journal.append("supervisor_ready", supervisor_pid=99999999) + for role, i in (("h0", 1), ("h1", 2)): + journal.append("receipt", role=role, instance_id=i) + journal.append("instance_ready", role=role, ssh_host="127.0.0.1", ssh_port=port) + orch.trust(role) + journal.append("staged", role=role, tar_sha256="00" * 32) + journal.append( + "network_execution_intent", + role=role, + name="qualification", + phase="qualification", + executions=2, + ) + journal.append("network_workload_promoted") + body = example_manifest().body() + for section in ("model", "outer"): + body[section].update(profile[section]) + body["inner"].update({k: v for k, v in profile["inner"].items() if k != "lr_schedule"}) + body["inner"]["lr_schedule"].update(profile["inner"]["lr_schedule"]) + body["reference_spec"].update( + image_digest=profile["runtime"]["image_digest"], + driver_allowlist=["595.84"], + layout=profile["layout"], + ) + manifest = RunManifestV2.model_validate( + { + "manifest_version": 2, + "training": body, + "network": { + **{ + k: "11" * 32 + for k in ( + "admission_policy_hash", + "economics_policy_hash", + "aggregation_policy_hash", + "dispute_policy_hash", + "audit_policy_hash", + "relay_registry_hash", + ) + }, + "audit_mode": "anchored-full", + "full_anchor_version": 1, + "capabilities": ["island-replay", "all-level-disputes", "transport-receipts"], + }, + } + ) + inputs = tmp_path / "inputs" + inputs.mkdir() + for key in ("start_state", "ef_in", "v0", "samples", "sample_proofs"): + (inputs / key).write_bytes(key.encode()) + job = IslandJobV1( + job_version=1, + run_id=manifest.run_id(), + w=0, + manifest=manifest, + sample_ids=list(range(60)), + global_step0=0, + start_state_sha256=fsha(inputs / "start_state"), + ef_in_sha256=fsha(inputs / "ef_in"), + v0_sha256=fsha(inputs / "v0"), + object_paths={k: k for k in ("start_state", "ef_in", "v0", "samples", "sample_proofs")}, + deadline=cutoff, + ) + engine = __import__("hypertrain.gpu_ops.network_qualification", fromlist=["sources"]) + sources = engine.sources(TREE) + sources["scripts/network_gpu_operation.py"] = fsha(TREE / "scripts/network_gpu_operation.py") + spec = OP.Operation( + operation="reference", + binding="ab" * 32, + binding_kind="trial", + hotkey=OWNER.ss58, + owner=OWNER.ss58, + role="h0", + instance_id=1, + machine_id=5000, + job=job, + sources=sources, + image_digest=profile["runtime"]["image_digest"], + cutoff=cutoff - 310, + trace=False, + ) + runtime = RUNTIME.NetworkRuntime(orch, profile) + context = { + "role": spec.role, + "binding": spec.binding, + "operation": spec.operation, + "hotkey": spec.hotkey, + "spec_sha256": sha256(spec.model_dump_json().encode()), + } + receipt = envelope_v2.seal( + OWNER, + "Receipt", + job.run_id, + {"w": 0, "commit_hash": sha256(canonicalize(context)), "received_round": 10}, + 20, + ) + runtime.accept_operation(spec, canonicalize(receipt), owner=OWNER.ss58, beacon=10) + try: + yield runtime, spec, inputs, provider + finally: + server.terminate() + server.wait(timeout=15) + + +@pytest.mark.parametrize( + "fault", + [ + None, + "remote-failure", + "CPU-publication", + "canonical-reference", + "canonical-audit", + "canonical-collision", + "canonical-symlink", + "canonical-cancel", + "canonical-expired", + ], +) +def test_loopback_debit_transport_custody_and_nonretryable(context, monkeypatch, fault): + runtime, spec, inputs, provider = context + canonical_case = fault is not None and fault.startswith("canonical-") + if fault == "canonical-audit": + spec = spec.model_copy( + update={"operation": "audit", "binding_kind": "commit-lease", "trace": True} + ) + receipt = envelope_v2.seal( + OWNER, + "Receipt", + spec.job.run_id, + { + "w": 0, + "commit_hash": sha256( + canonicalize( + { + "role": spec.role, + "binding": spec.binding, + "operation": spec.operation, + "hotkey": spec.hotkey, + "spec_sha256": sha256(spec.model_dump_json().encode()), + } + ) + ), + "received_round": 10, + }, + 20, + ) + runtime.accept_operation(spec, canonicalize(receipt), owner=OWNER.ss58, beacon=10) + ssh_run = RUNTIME.DeadlineSsh.run + calls = [] + + def dispatch(ssh, command, tag, logs): + if " scripts/network_gpu_operation.py " in command: + assert runtime.journal.last("network_execution_intent", role="h0")["executions"] == 1 + remote = ( + Path(runtime.lifecycle.remote_root("h0")) + / "out" + / runtime.journal.last("network_operation_started")["name"] + ) + result = { + "status": "CAPTURED_NOT_ACCEPTED", + "binding": spec.binding, + "job_sha256": spec.job.digest(), + "hotkey": spec.hotkey, + "role": spec.role, + "operation": spec.operation, + "publication": "published", + } + code = ( + "import pathlib,json; p=pathlib.Path(" + + repr(str(remote)) + + "); (p/'published/rank-0').mkdir(parents=True); (p/'published/rank-1').mkdir(); " + + "[(p/f'published/rank-{r}/summary.json').write_text(" + + '\'{"backend":"cuda"}\') for r in (0,1)]; ' + + "(p/'published/state').write_bytes(b'original'); " + + "[(p/f'published/rank-{r}'/f).write_bytes(f.encode()) for r in (0,1) " + + "for f in ('state.safetensors','ef.safetensors','delta.bin'," + + "'leaves.json','trace.json')]; " + + "[(p/f'published/rank-{r}/checkpoints').mkdir() for r in (0,1)]; " + + "[(p/f'published/rank-{r}/checkpoints/0.safetensors').write_bytes(b'checkpoint') " + + "for r in (0,1)]; " + + "(p/'operation-result.json').write_text(" + + repr(json.dumps(result)) + + ")" + ) + calls.append("hermetic-publication-not-CUDA") + if fault == "CPU-publication": + code = code.replace("cuda", "cpu") + if fault == "remote-failure": + code += ";raise SystemExit(7)" + command = shlex.quote(sys.executable) + " -c " + shlex.quote(code) + if "experiments/gpu_network_v2/orchestrate.py export" in command: + command = command.replace( + "cd " + shlex.quote(runtime.lifecycle.remote_root("h0")) + " && env PYTHONPATH=src", + "cd " + shlex.quote(str(TREE)) + " && env PYTHONPATH=src", + ) + command = command.replace( + "export out rescue.tar", + "export " + + shlex.quote(runtime.lifecycle.remote_root("h0") + "/out") + + " " + + shlex.quote(runtime.lifecycle.remote_root("h0") + "/rescue.tar"), + ) + return ssh_run(ssh, command, tag, logs) + + monkeypatch.setattr(RUNTIME.DeadlineSsh, "run", dispatch) + from hypertrain.gpu_ops.remote import Ssh + + original_run = Ssh.run + + def rescue_run(ssh, command, tag, logs): + if tag == "network-export": + command = ( + "cd " + + shlex.quote(str(TREE)) + + " && env PYTHONPATH=src " + + shlex.quote(sys.executable) + + " experiments/gpu_network_v2/orchestrate.py export " + + shlex.quote(runtime.lifecycle.remote_root("h0") + "/out") + + " " + + shlex.quote(runtime.lifecycle.remote_root("h0") + "/rescue.tar") + + " experiments/gpu_network_v2/profile.json" + ) + return original_run(ssh, command, tag, logs) + + monkeypatch.setattr(Ssh, "run", rescue_run) + import hypertrain.miner.island_launch as island + + def validate(job, directory): + assert job == spec.job and (directory / "state").read_bytes() == b"original" + return SimpleNamespace( + directory=directory, + state=directory / "rank-0/state.safetensors", + ef=directory / "rank-0/ef.safetensors", + delta=directory / "rank-0/delta.bin", + leaves=directory / "rank-0/leaves.json", + ) + + monkeypatch.setattr(island, "validate_artifacts", validate) + if canonical_case: + if fault == "canonical-collision": + (inputs / "published").mkdir() + (inputs / "published/state").write_bytes(b"stale") + if fault == "canonical-symlink": + (inputs / "published").symlink_to(inputs) + event = threading.Event() + initial_validate = validate + + def final_validate(job, directory): + result = initial_validate(job, directory) + if directory.parent.name.startswith("canonical-"): + if fault == "canonical-cancel": + event.set() + if fault == "canonical-expired": + monkeypatch.setattr(RUNTIME.time, "time", lambda: spec.cutoff + 1) + return result + + monkeypatch.setattr(island, "validate_artifacts", final_validate) + launch = runtime.launch_adapter(spec) + if fault in ( + "canonical-collision", + "canonical-symlink", + "canonical-cancel", + "canonical-expired", + ): + with pytest.raises(Reject, match="collision|expired"): + launch(spec.job, inputs, backend="cuda", trace=spec.trace, cancel=event) + if fault == "canonical-collision": + assert (inputs / "published/state").read_bytes() == b"stale" + if fault in ("canonical-cancel", "canonical-expired"): + assert not (inputs / "published").exists() + assert not [p for p in inputs.glob("canonical-*") if p.is_dir()] + if fault in ("canonical-collision", "canonical-symlink"): + with pytest.raises(Reject, match="failed_not_retryable"): + launch(spec.job, inputs, backend="cuda", trace=spec.trace, cancel=event) + else: + artifacts = launch(spec.job, inputs, backend="cuda", trace=spec.trace, cancel=event) + assert artifacts.directory == inputs / "published" + rescued = ( + inputs / runtime.journal.last("network_operation_started")["name"] / "published" + ) + original = { + str(p.relative_to(rescued)): p.read_bytes() + for p in rescued.rglob("*") + if p.is_file() + } + assert { + str(p.relative_to(artifacts.directory)): p.read_bytes() + for p in artifacts.directory.rglob("*") + if p.is_file() + } == original + assert len(calls) == 1 + duplicate = launch(spec.job, inputs, backend="cuda", trace=spec.trace, cancel=event) + assert duplicate.directory == artifacts.directory and len(calls) == 1 + # Execute actual reference loader and audit equality, not a mocked path. + from hypertrain.challenge.store import ChallengeError + + admission = ast.parse((TREE / "src/hypertrain/challenge/admission.py").read_text()) + statement = next( + node + for node in ast.walk(admission) + if isinstance(node, ast.Assign) + and any( + isinstance(target, ast.Name) and target.id == "data" for target in node.targets + ) + and "published" in ast.unparse(node.value) + ) + for name in ("state.safetensors", "ef.safetensors", "delta.bin", "leaves.json"): + env = {"directory": inputs, "name": name} + exec( + compile( + ast.Module(body=[statement], type_ignores=[]), + "actual-reference-loader", + "exec", + ), + env, + ) + assert ( + env["data"] + == getattr( + artifacts, + { + "state.safetensors": "state", + "ef.safetensors": "ef", + "delta.bin": "delta", + "leaves.json": "leaves", + }[name], + ).read_bytes() + ) + store = ast.parse((TREE / "src/hypertrain/challenge/store.py").read_text()) + check = next( + node + for node in ast.walk(store) + if isinstance(node, ast.If) + and ast.unparse(node.test) + == "artifacts.directory != geometry_directory / 'published'" + ) + exec( + compile( + ast.Module(body=[check], type_ignores=[]), "actual-audit-path-check", "exec" + ), + { + "artifacts": artifacts, + "geometry_directory": inputs, + "ChallengeError": ChallengeError, + }, + ) + (artifacts.directory / "rank-0/trace.json").write_bytes(b"different") + with pytest.raises(Reject, match="collision"): + launch(spec.job, inputs, backend="cuda", trace=spec.trace, cancel=event) + assert len(calls) == 1 + elif fault is None: + artifacts = runtime.operation(spec, inputs) + assert artifacts.directory.name == "published" + assert runtime.journal.last("network_operation_done") + else: + with pytest.raises(Reject): + runtime.operation(spec, inputs) + assert runtime.journal.last("network_operation_done") is None + assert calls == ["hermetic-publication-not-CUDA"] + assert runtime.journal.last("network_rescued")["files"] + with pytest.raises(Reject, match="not_retryable|not_qualified"): + runtime.operation(spec, inputs) + assert provider.state()["put_count"] == 0 + + +@pytest.mark.parametrize("change", ["binding", "hotkey", "cutoff", "sources", "instance_id"]) +def test_unaccepted_context_rejects_before_debit(context, change): + runtime, spec, inputs, _ = context + value = { + "binding": "cc" * 32, + "hotkey": Keypair(bytes([8]) * 32).ss58, + "cutoff": 1, + "sources": {}, + "instance_id": 99, + }[change] + before = len(runtime.journal.all("network_execution_intent")) + with pytest.raises(Reject): + runtime.operation(spec.model_copy(update={change: value}), inputs) + assert len(runtime.journal.all("network_execution_intent")) == before + + +def test_cancelled_rejects_before_launch(context): + runtime, spec, inputs, _ = context + event = threading.Event() + event.set() + with pytest.raises(Reject, match="not_qualified"): + runtime.operation(spec, inputs, cancel=event) + + +def test_changed_dedicated_known_hosts_rejects_before_debit(context): + runtime, spec, inputs, _ = context + runtime.lifecycle.ssh("h0").known_hosts.write_text("changed") + before = len(runtime.journal.all("network_execution_intent")) + with pytest.raises(Reject, match="known_hosts"): + runtime.operation(spec, inputs) + assert len(runtime.journal.all("network_execution_intent")) == before + + +def test_canonical_seams_default_and_remote_hook_signatures(): + import inspect + + from hypertrain.auditor.worker import execute_island_audit + from hypertrain.gpu_ops.work_screen import screen_work + + assert inspect.signature(screen_work).parameters["launch"].default is None + assert inspect.signature(execute_island_audit).parameters["launch"].default is None + + +def test_work_screen_trusted_launch_requires_original_validation(context, monkeypatch): + runtime, spec, inputs, _ = context + import hypertrain.gpu_ops.work_screen as screen + from hypertrain.data.trial_assignment import trial_assignment_hash + from hypertrain.protocol.messages_v2 import ArtifactLimits, JoinChallenge + + challenge = JoinChallenge( + admission_id=spec.binding, + nonce="11" * 32, + seed_beacon=1, + deadline_beacon=101, + manifest_hash=spec.job.run_id, + theta_hash="22" * 32, + assignment_hash=trial_assignment_hash( + spec.job.manifest, spec.job.w, tuple(spec.job.sample_ids) + ), + layout=spec.job.manifest.training.reference_spec.layout, + artifact_limits=ArtifactLimits( + max_object_bytes=100, max_body_bytes=65536, max_chunk_manifest_bytes=1048576 + ), + policy_hash="11" * 32, + ) + import hypertrain.auditor.replay as replay + import hypertrain.trainer.compress as compress + + monkeypatch.setattr(replay, "unpack_state", lambda blob: ({}, None)) + monkeypatch.setattr(compress, "state_hash", lambda theta: "22" * 32) + called = [] + + def launch(job, directory, **kwargs): + called.append((job, kwargs)) + return SimpleNamespace(directory=inputs) + + def refused(job, directory): + raise ValueError("independent artifact validator rejects hermetic metadata") + + monkeypatch.setattr(screen, "validate_artifacts", refused) + with pytest.raises(ValueError, match="validator rejects"): + screen.screen_work(spec.job, inputs, challenge, now_beacon=2, backend="cuda", launch=launch) + assert len(called) == 1 and called[0][0] == spec.job + + +def test_operation_owner_receipt_context_cannot_change(context): + runtime, spec, inputs, _ = context + altered = spec.model_copy(update={"trace": True}) + body = { + "role": altered.role, + "binding": altered.binding, + "operation": altered.operation, + "hotkey": altered.hotkey, + "spec_sha256": sha256(altered.model_dump_json().encode()), + } + raw = envelope_v2.seal( + OWNER, + "Receipt", + spec.job.run_id, + {"w": 0, "commit_hash": sha256(canonicalize(body)), "received_round": 10}, + 20, + ) + with pytest.raises(Reject, match="immutable"): + runtime.accept_operation(altered, canonicalize(raw), owner=OWNER.ss58, beacon=10) + + +@pytest.fixture +def probe_metadata(tmp_path, monkeypatch): + from hypertrain.data.trial_assignment import trial_assignment_hash + from hypertrain.miner.core import NetworkMiner + from hypertrain.protocol.messages_v2 import ArtifactLimits, JoinChallenge + + body = example_manifest().body() + body["coord_pubkey"] = OWNER.ss58 + profile = json.loads((TREE / "experiments/gpu_network_v2/profile.json").read_bytes()) + for section in ("model", "outer"): + body[section].update(profile[section]) + body["inner"].update({k: v for k, v in profile["inner"].items() if k != "lr_schedule"}) + body["inner"]["lr_schedule"].update(profile["inner"]["lr_schedule"]) + body["reference_spec"].update(layout=profile["layout"]) + manifest = RunManifestV2.model_validate( + { + "manifest_version": 2, + "training": body, + "network": { + **{ + k: "11" * 32 + for k in ( + "admission_policy_hash", + "economics_policy_hash", + "aggregation_policy_hash", + "dispute_policy_hash", + "audit_policy_hash", + "relay_registry_hash", + ) + }, + "audit_mode": "anchored-full", + "full_anchor_version": 1, + "capabilities": ["island-replay", "all-level-disputes", "transport-receipts"], + }, + } + ) + inputs = tmp_path / "probe" + inputs.mkdir() + paths = {k: k for k in ("start_state", "ef_in", "v0", "samples", "sample_proofs")} + for p in paths.values(): + (inputs / p).write_bytes(p.encode()) + job = IslandJobV1( + job_version=1, + run_id=manifest.run_id(), + w=0, + manifest=manifest, + sample_ids=list(range(manifest.training.batch_samples())), + global_step0=0, + start_state_sha256=fsha(inputs / "start_state"), + ef_in_sha256=fsha(inputs / "ef_in"), + v0_sha256=fsha(inputs / "v0"), + object_paths=paths, + deadline=int(time.time()) + 30, + ) + challenge = JoinChallenge( + admission_id="ab" * 32, + nonce="11" * 32, + seed_beacon=1, + deadline_beacon=101, + manifest_hash=job.run_id, + theta_hash="22" * 32, + assignment_hash=trial_assignment_hash(manifest, 0, tuple(job.sample_ids)), + layout=manifest.training.reference_spec.layout, + artifact_limits=ArtifactLimits( + max_object_bytes=100, max_body_bytes=65536, max_chunk_manifest_bytes=1048576 + ), + policy_hash="11" * 32, + ) + job_path = inputs / "job.json" + job_path.write_text(job.model_dump_json()) + challenge_path = inputs / "challenge.json" + challenge_path.write_bytes( + canonicalize(envelope_v2.seal(OWNER, "JoinChallenge", job.run_id, challenge, 101)) + ) + miner = object.__new__(NetworkMiner) + miner.cfg = SimpleNamespace(device="cuda") + miner.run_id = job.run_id + miner.manifest = manifest + miner.kp = OWNER + miner.url = "/unused" + uploads = [] + miner.api = SimpleNamespace( + call=lambda *args: {"now_round": 2}, + base="", + c=SimpleNamespace(put=lambda *a, **k: uploads.append(a)), + ) + import hypertrain.auditor.replay as replay + import hypertrain.trainer.compress as compress + + monkeypatch.setattr(replay, "unpack_state", lambda blob: ({}, None)) + monkeypatch.setattr(compress, "state_hash", lambda theta: "22" * 32) + return miner, job, job_path, challenge_path, uploads + + +@pytest.mark.parametrize("fault", ["cancel", "deadline", "authority"]) +def test_actual_probe_rejects_before_cached_kernel_or_publication( + probe_metadata, monkeypatch, fault +): + import hypertrain.gpu_ops.work_screen as screen + from hypertrain.miner.core import MinerError + + miner, job, path, challenge, uploads = probe_metadata + event = threading.Event() + if fault == "cancel": + event.set() + if fault == "deadline": + path.write_text(job.model_copy(update={"deadline": 1}).model_dump_json()) + if fault == "authority": + miner.run_id = "cc" * 32 + + def forbidden(*args, **kwargs): + raise AssertionError("cached kernel must not run") + + monkeypatch.setattr(screen, "launch_island", forbidden) + with pytest.raises(MinerError): + miner.probe(path, challenge, launch=forbidden, cancel=event) + assert not uploads + + +def test_actual_probe_forwards_guard_and_event_despite_cached_launcher(probe_metadata, monkeypatch): + import hypertrain.gpu_ops.work_screen as screen + import hypertrain.miner.island_launch as island + from hypertrain.gpu_ops.work_screen import WorkScreenError + + miner, job, path, challenge, uploads = probe_metadata + event = threading.Event() + calls = [] + + def forbidden(*args, **kwargs): + raise AssertionError("cached/global launcher bypass") + + monkeypatch.setattr(screen, "launch_island", forbidden) + monkeypatch.setattr(island, "launch_island", forbidden) + + def guarded(actual, directory, *, backend, cancel, trace=False): + assert actual == job and backend == "cuda" and cancel is event and not trace + calls.append(actual) + event.set() + return SimpleNamespace(directory=directory) + + monkeypatch.setattr( + screen, "validate_artifacts", lambda actual, directory: SimpleNamespace(directory=directory) + ) + with pytest.raises(WorkScreenError, match="CANCELLED"): + miner.probe(path, challenge, launch=guarded, cancel=event) + assert calls == [job] and not uploads + + +def test_actual_probe_inflight_cancel_reaps_process_retains_custody( + probe_metadata, monkeypatch, tmp_path +): + import hypertrain.miner.island_launch as island + + miner, job, path, challenge, uploads = probe_metadata + cancel = threading.Event() + ready = threading.Event() + processes = [] + errors = [] + listener = socket.socket(socket.AF_UNIX) + address = str(tmp_path / "ready.sock") + listener.bind(address) + listener.listen() + listener.settimeout(10) + child = ( + "import socket,signal,sys;s=socket.socket(socket.AF_UNIX);" + "s.connect(sys.argv[1]);s.sendall(b'R');signal.pause()" + ) + monkeypatch.setattr(island, "launch_argv", lambda *args: [sys.executable, "-c", child, address]) + original = subprocess.Popen + + def spawn(*args, **kwargs): + process = original(*args, **kwargs) + processes.append(process) + return process + + monkeypatch.setattr(island.subprocess, "Popen", spawn) + + def guarded(actual, directory, *, backend, cancel, trace=False): + assert actual == job and backend == "cuda" and cancel is event + (directory / "charged-intent").write_bytes(b"nonretryable") + island._launch_unbounded(actual, directory, backend, dict(os.environ), cancel) + raise AssertionError("cancelled process cannot publish") + + event = cancel + + def consume(): + try: + miner.probe(path, challenge, launch=guarded, cancel=cancel) + except BaseException as error: + errors.append(error) + finally: + ready.set() + + worker = threading.Thread(target=consume) + # Completion/cancel signals exist before triggering the real consumer. + worker.start() + try: + connection, _ = listener.accept() + with connection: + assert connection.recv(1) == b"R" + cancel.set() + assert ready.wait(10), "consumer did not reap on subscribed cancellation" + finally: + cancel.set() + worker.join(timeout=10) + listener.close() + for process in processes: + if process.poll() is None: + process.kill() + process.wait(timeout=10) + assert len(processes) == 1 and processes[0].returncode is not None + assert len(errors) == 1 and isinstance(errors[0], island.IslandFailure) + assert (path.parent / "charged-intent").read_bytes() == b"nonretryable" + assert (path.parent / "stdout.log").exists() and not uploads + + +def test_remote_probe_wrapper_explicit_consumer_guard_preserves_failed_outputs( + probe_metadata, monkeypatch +): + import signal + + import httpx + + import hypertrain.gpu_ops.work_screen as screen + import hypertrain.miner.core as core + import hypertrain.miner.island_launch as island + + miner, job, path, challenge, uploads = probe_metadata + cfg = SimpleNamespace( + run_id=job.run_id, + owner_hotkey=OWNER.ss58, + keyfile=path, + api="https://metadata.invalid", + device="cuda", + image_digest=job.manifest.training.reference_spec.image_digest, + workdir=path.parent / "miner", + ) + spec = OP.Operation( + operation="probe", + binding="ab" * 32, + binding_kind="trial", + hotkey=OWNER.ss58, + owner=OWNER.ss58, + role="h0", + instance_id=1, + machine_id=5000, + job=job, + sources={}, + image_digest=cfg.image_digest, + cutoff=job.deadline, + trace=False, + miner_config="config", + ca="ca", + master=cfg.api, + challenge=challenge.name, + ) + (path.parent / "config").write_bytes(b"metadata config") + (path.parent / "ca").write_bytes(b"metadata CA") + monkeypatch.setattr(OP, "check", lambda *args: None) + monkeypatch.setattr(core.MinerConfig, "load", lambda *args, **kwargs: cfg) + monkeypatch.setattr(core, "load_keyfile", lambda *args: OWNER) + callbacks = {} + monkeypatch.setattr( + OP.signal, "signal", lambda sig, handler: callbacks.__setitem__(sig, handler) + ) + manifest = envelope_v2.seal(OWNER, "RunManifestV2", job.run_id, job.manifest, 101) + requests = [] + + def respond(request): + requests.append(request) + assert request.method == "GET" + return httpx.Response(200, json={"manifest_envelope": manifest, "now_round": 2}) + + init = httpx.Client.__init__ + + def client_init(self, *args, **kwargs): + kwargs["transport"] = httpx.MockTransport(respond) + init(self, *args, **kwargs) + + monkeypatch.setattr(httpx.Client, "__init__", client_init) + original = island.launch_island + + def forbidden(*args, **kwargs): + raise AssertionError("cached probe launcher bypass") + + monkeypatch.setattr(screen, "launch_island", forbidden) + seen = [] + + def kernel(actual, directory, backend, env, cancel): + assert actual == job and backend == "cuda" and not cancel.is_set() + seen.append(cancel) + (directory / "retained.log").write_bytes(b"failed charged publication") + callbacks[signal.SIGTERM](signal.SIGTERM, None) + assert cancel.is_set() + raise island.IslandFailure("subscribed cancellation") + + monkeypatch.setattr(island, "_launch_unbounded", kernel) + path.unlink() # Wrapper owns its durable job write; accepted descriptor remains in spec. + with pytest.raises(island.IslandFailure, match="subscribed"): + OP.execute(spec, path.parent, path.parent) + assert len(seen) == 1 and island.launch_island is original + assert (path.parent / "published/retained.log").read_bytes() == b"failed charged publication" + assert requests and all(r.method == "GET" for r in requests) and not uploads + + +@pytest.mark.parametrize( + "fault", + [ + "two", + "duplicate", + "failed-response", + "cancel", + "replace-interrupt", + "symlink", + "count-bound", + "response-bound", + ], +) +def test_real_cli_capture_multiple_original_requests(probe_metadata, monkeypatch, fault): + import signal + + import httpx + + import hypertrain.miner.core as core + + miner, job, path, challenge, _ = probe_metadata + cfg = SimpleNamespace( + run_id=job.run_id, + owner_hotkey=OWNER.ss58, + keyfile=path, + api="https://capture.invalid", + device="cuda", + image_digest=job.manifest.training.reference_spec.image_digest, + workdir=path.parent / "miner", + ) + spec = OP.Operation( + operation="live", + binding="ab" * 32, + binding_kind="commit-lease", + hotkey=OWNER.ss58, + owner=OWNER.ss58, + role="h0", + instance_id=1, + machine_id=5000, + job=job, + sources={}, + image_digest=cfg.image_digest, + cutoff=job.deadline, + trace=True, + miner_config="config", + ca="ca", + master=cfg.api, + ) + (path.parent / "config").write_bytes(b"metadata config") + (path.parent / "ca").write_bytes(b"metadata CA") + monkeypatch.setattr(OP, "check", lambda *args: None) + monkeypatch.setattr(core.MinerConfig, "load", lambda *args, **kwargs: cfg) + monkeypatch.setattr(core, "load_keyfile", lambda *args: OWNER) + callbacks = {} + monkeypatch.setattr( + OP.signal, "signal", lambda sig, handler: callbacks.__setitem__(sig, handler) + ) + manifest = envelope_v2.seal(OWNER, "RunManifestV2", job.run_id, job.manifest, 101) + sent = [] + replies = [] + + def respond(request): + if request.method == "GET": + return httpx.Response(200, json={"manifest_envelope": manifest, "now_round": 2}) + sent.append(request.content) + content = b' { "original": ' + str(len(sent)).encode() + b" }\n" + status = 409 if fault == "failed-response" and len(sent) == 2 else 200 + if fault == "response-bound" and len(sent) == 2: + content = b"x" * ((1 << 20) + 1025) + replies.append((status, content)) + if fault == "cancel" and len(sent) == 2: + callbacks[signal.SIGTERM](signal.SIGTERM, None) + return httpx.Response(status, content=content) + + init = httpx.Client.__init__ + + def client_init(self, *args, **kwargs): + kwargs["transport"] = httpx.MockTransport(respond) + init(self, *args, **kwargs) + + monkeypatch.setattr(httpx.Client, "__init__", client_init) + signed = [] + for index in range(5 if fault == "count-bound" else 2): + body = { + "w": 0, + "commit_hash": ("11" if fault == "duplicate" else f"{index + 1:02x}") * 32, + "received_round": 10, + } + raw = envelope_v2.seal(OWNER, "Receipt", job.run_id, body, 101) + signed.append(json.dumps(raw, indent=2).encode() + b"\n") + + def round_metadata(self, w): + assert w == job.w + for data in signed: + response = self.api.c.post(self.api.base + self.url + "/capture-metadata", content=data) + response.raise_for_status() + raise core.MinerError("metadata stop after actual CLI capture") + + monkeypatch.setattr(core.NetworkMiner, "run_round", round_metadata) + replace = OP.os.replace + + def interrupt(source, destination, **kwargs): + if destination == "signed-api.json" and len(sent) == 1: + raise OSError("interrupt second capture replacement") + return replace(source, destination, **kwargs) + + if fault == "replace-interrupt": + monkeypatch.setattr(OP.os, "replace", interrupt) + outside = path.parent / "outside" + outside.write_bytes(b"preserve") + if fault == "symlink": + (path.parent / "signed-api.json").symlink_to(outside) + path.unlink() + with pytest.raises( + (ValueError, OSError, httpx.HTTPStatusError), match="capture|product miner|409|interrupt" + ): + OP.execute(spec, path.parent, path.parent) + assert not (path.parent / "operation-result.json").exists() + assert outside.read_bytes() == b"preserve" and not list(path.parent.glob(".*.next")) + if fault == "symlink": + assert not sent and (path.parent / "signed-api.json").is_symlink() + return + records = json.loads((path.parent / "signed-api.json").read_bytes()) + expected = 1 if fault == "replace-interrupt" else 4 if fault == "count-bound" else 2 + assert records == [json.loads(data) for data in signed[:expected]] + # Exact whitespace/signature bytes survive; duplicate requests are not deduplicated. + for index in range(expected): + assert (path.parent / f"signed-api-{index}.bin").read_bytes() == signed[index] + assert sent == signed[: len(sent)] + accepted = json.loads((path.parent / "accepted-api.json").read_bytes()) + accepted_count = ( + 1 if fault in ("failed-response", "replace-interrupt", "response-bound") else expected + ) + assert len(accepted) == accepted_count + for index, row in enumerate(accepted): + assert row["index"] == index and row["request_hex"] == sent[index].hex() + assert row["response_hex"] == replies[index][1].hex() and row["status"] == 200 + assert row["response"] == json.loads(replies[index][1]) + provenance = json.loads((path.parent / f"response-api-{index}.json").read_bytes()) + assert provenance["url"].endswith("/capture-metadata") and provenance["method"] == "POST" + if fault == "failed-response": + failed = json.loads((path.parent / "response-api-1.json").read_bytes()) + assert failed["status"] == 409 and failed["response_hex"] == replies[1][1].hex() diff --git a/tests/gpu_ops/test_phase_b.py b/tests/gpu_ops/test_phase_b.py index 7b16d07..9d434c4 100644 --- a/tests/gpu_ops/test_phase_b.py +++ b/tests/gpu_ops/test_phase_b.py @@ -207,11 +207,73 @@ def test_phase_b_oom_log_watch_kills_fast(mock_factory: Any, tmp_path: Path) -> assert len([r for r in rows if r["kind"] == "absence_confirmed"]) == 2 -def test_phase_b_boot_timeout_rescues_and_deletes_pair(mock_factory: Any, tmp_path: Path) -> None: +def test_phase_b_boot_timeout_rescues_and_deletes_pair( + mock_factory: Any, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import threading + from types import FunctionType, SimpleNamespace + + from hypertrain.gpu_ops import launcher, supervisor + m = mock_factory(offers=[offer8(i) for i in range(2)], never_running=True) rd = tmp_path / "run" - proc = run_b(phase_b_cfg(tmp_path, m.base, boot_timeout_seconds=900), rd) - assert proc.returncode == 3, proc.stdout + proc.stderr + cfg = phase_b_cfg(tmp_path, m.base, boot_timeout_seconds=900) + phase = _load("orchestrate") + loading = threading.Event() # Registered before the real provider request. + clock = SimpleNamespace(now=0.0, advanced=False) + original_boot = launcher.Orchestrator.boot + + class BootClockFixture(phase.PhaseBOrchestrator): + def show(self, iid, tag): + response, state = super().show(iid, tag) + receipt = self.receipt("h0") + if ( + tag == "show-h0" + and response.ok() + and isinstance(state, dict) + and state.get("id") == receipt["instance_id"] + and state.get("label") == receipt["label"] + and state.get("actual_status") == "loading" + ): + loading.set() + return response, state + + def boot(self, role): + clock.now = time.time() + start = clock.now + sleeper = self.p.sleep + + def advance(seconds): + assert loading.is_set() + assert not clock.advanced and clock.now == start + assert seconds == self.poll + clock.now += self.cfg["boot_timeout_seconds"] + clock.advanced = True + + bounded = FunctionType( + original_boot.__code__, + {**original_boot.__globals__, "time": SimpleNamespace(time=lambda: clock.now)}, + original_boot.__name__, + original_boot.__defaults__, + original_boot.__closure__, + ) + self.p.sleep = advance + try: + return bounded(self, role) + finally: + self.p.sleep = sleeper + + # Only this node's boot function sees the virtual clock. Supervisor and + # production admission/cleanup deadlines retain their real wall clock. + monkeypatch.setattr(phase, "PhaseBOrchestrator", BootClockFixture) + monkeypatch.setattr(launcher, "Orchestrator", launcher.Orchestrator) + monkeypatch.setattr(supervisor, "Orchestrator", supervisor.Orchestrator) + monkeypatch.setenv("HT_GPU_VIRTUAL_TIME", "1") + code = phase.main( + [str(B / "orchestrate.py"), "run", "--config", str(cfg), "--run-dir", str(rd)] + ) + assert code == 3 + assert loading.is_set() and clock.advanced rows = journal(rd) closed = [r for r in rows if r["kind"] == "transaction_closed"] assert closed and closed[0]["outcome"] == "failed:boot_timeout:h0" diff --git a/tests/gpu_ops/test_runtime_custody.py b/tests/gpu_ops/test_runtime_custody.py new file mode 100644 index 0000000..9fe1635 --- /dev/null +++ b/tests/gpu_ops/test_runtime_custody.py @@ -0,0 +1,373 @@ +"""Internal completion boundary; one real CPU fixture pair, synthetic CUDA metadata.""" + +from __future__ import annotations + +import importlib.util +import io +import json +import shutil +import sys +import tarfile +import threading +from pathlib import Path +from types import SimpleNamespace + +import pytest + +import hypertrain +from hypertrain.gpu_ops.journal import Journal, fsha, sha256 +from hypertrain.protocol.envelope_v2 import seal +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.messages import Receipt +from hypertrain.protocol.messages_v2 import ArtifactLimits, IslandJobV1, JoinChallenge + +STAGE = Path(__file__).resolve().parents[2] +PRODUCT = Path(hypertrain.__file__).resolve().parents[2] +sys.path.insert(0, str(PRODUCT / "scripts")) +SPEC = importlib.util.spec_from_file_location( + "isolated_runtime_custody", STAGE / "experiments/gpu_network_v2/orchestrate.py" +) +assert SPEC and SPEC.loader +cli = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = cli +SPEC.loader.exec_module(cli) + + +@pytest.fixture(scope="module") +def retained_custody_work(tmp_path_factory): + """Original actual_reward miner/reference pair once; publications cloned per case.""" + spec = importlib.util.spec_from_file_location( + "custody_original_ledger_fixture", PRODUCT / "tests/ledger/test_escrow_v2.py" + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + actual = module.actual_reward.__wrapped__(tmp_path_factory) + base = tmp_path_factory.getbasetemp() + miner = next(base.glob("real-reward*/published")) + reference = next(base.glob("real-reference*/published")) + return module, actual, miner, reference + + +@pytest.fixture +def completed(tmp_path, monkeypatch, request, retained_custody_work): + """Clone genuine module output; CUDA labels/runtime metadata explicitly synthetic.""" + import network_gpu_operation as producer + from hypertrain.auditor.replay import unpack_state + from hypertrain.data.trial_assignment import trial_assignment_hash + from hypertrain.gpu_ops.network_qualification import sources + from hypertrain.gpu_ops.work_screen import commitments + from hypertrain.miner.island_launch import validate_artifacts + from hypertrain.protocol.messages_v2 import ArtifactRef, WorkProof + from hypertrain.trainer.compress import state_hash + + fixture, _, retained, reference = retained_custody_work + job = IslandJobV1.model_validate_json((retained / "job.json").read_bytes()) + refjob = IslandJobV1.model_validate_json((reference / "job.json").read_bytes()) + assert job.model_dump(exclude={"deadline"}) == refjob.model_dump(exclude={"deadline"}) + assert retained != reference + deadline_beacon = (job.deadline - job.manifest.training.beacon.genesis_time) // 3 + 1 + signed_cutoff = job.manifest.training.beacon.genesis_time + (deadline_beacon - 1) * 3 + job = job.model_copy(update={"deadline": signed_cutoff + getattr(request, "param", 0)}) + clock = {"now": job.deadline - 10} + monkeypatch.setattr(cli.time, "time", lambda: clock["now"]) + coord, miner, owner = fixture.COORD, fixture.HOT, fixture.OTHER + assert coord.ss58 == job.manifest.training.coord_pubkey + start, _ = unpack_state((retained / job.object_paths["start_state"]).read_bytes()) + challenge = JoinChallenge( + admission_id="11" * 32, + nonce="22" * 32, + seed_beacon=deadline_beacon - 100, + deadline_beacon=deadline_beacon, + manifest_hash=job.run_id, + theta_hash=state_hash(start), + assignment_hash=trial_assignment_hash(job.manifest, job.w, tuple(job.sample_ids)), + layout=job.manifest.training.reference_spec.layout, + artifact_limits=ArtifactLimits( + max_object_bytes=268435456, max_body_bytes=65536, max_chunk_manifest_bytes=1048576 + ), + policy_hash=job.manifest.network.admission_policy_hash, + ) + wire = seal(coord, "JoinChallenge", job.run_id, challenge, deadline_beacon) + source_map = sources(PRODUCT) + source_map["scripts/network_gpu_operation.py"] = fsha( + PRODUCT / "scripts/network_gpu_operation.py" + ) + source_map["experiments/gpu_network_v2/orchestrate.py"] = fsha(Path(cli.__file__)) + spec = producer.Operation( + operation="probe", + binding=challenge.admission_id, + hotkey=miner.ss58, + owner=owner.ss58, + role="h0", + instance_id=11, + machine_id=22, + job=job, + binding_kind="trial", + sources=source_map, + image_digest=job.manifest.training.reference_spec.image_digest, + cutoff=job.deadline, + trace=False, + challenge="challenge.json", + context_files={"challenge.json": sha256(canonicalize(wire))}, + ) + name = "op-" + sha256( + (job.run_id + spec.operation + spec.hotkey + spec.binding + job.digest()).encode() + ) + original = tmp_path / name + original.mkdir() + publication = original / "published" + shutil.copytree(retained, publication) + # No training or GPU invocation. Relabel ONLY this copied CPU boundary fixture. + summary_path = publication / "rank-0/summary.json" + summary = json.loads(summary_path.read_bytes()) + summary["backend"] = "cuda" + summary["job_hash"] = sha256(canonicalize(job.model_dump(mode="json"))) + summary_path.write_bytes(canonicalize(summary)) + (publication / "job.json").write_bytes(canonicalize(job.model_dump(mode="json"))) + observed = {"backend": "cuda", "image_digest": spec.image_digest, "test_only": True} + monkeypatch.setattr(producer, "execution_environment", lambda op: observed) + artifacts = validate_artifacts(job, publication) + commit = producer.probe_commit(spec, artifacts, miner, wire) + root, delta = commitments(artifacts) + proof = WorkProof( + admission_id=spec.binding, + challenge_hash=challenge.digest(), + leaves_root=root, + delta_hash=delta, + artifact_refs=[ + ArtifactRef(sha256=fsha(p), size=p.stat().st_size) + for p in (artifacts.state, artifacts.ef, artifacts.delta, artifacts.leaves) + ], + ) + proof_wire = seal(miner, "WorkProof", job.run_id, proof, deadline_beacon) + custody = producer.publication_custody(spec, original, artifacts, wire, commit, proof_wire) + result = { + "operation": spec.operation, + "binding": spec.binding, + "hotkey": spec.hotkey, + "role": spec.role, + "job_sha256": job.digest(), + "publication": "published", + "status": "CAPTURED_NOT_ACCEPTED", + } + producer.publish_custody_result(spec, original, result, custody, threading.Event()) + (tmp_path / "journal").mkdir() + journal = Journal(tmp_path / "journal") + journal.append("network_admission_authority", owner=owner.ss58) + journal.append("supervisor_ready") + journal.append("ssh_trusted", role="h0") + cleanup = {"started": False} + lifecycle = SimpleNamespace( + dir=tmp_path, + j=journal, + cfg={"cleanup_deadline_unix": job.deadline + 300}, + host={"h0": {"machine_id": 22}}, + receipt=lambda role: {"instance_id": 11}, + deadline=lambda: job.deadline, + cleanup_started=lambda: cleanup["started"], + ) + runtime = cli.NetworkRuntime( + lifecycle, + { + "runtime": {"qualified": True, "image_digest": spec.image_digest}, + "layout": job.manifest.training.reference_spec.layout.model_dump(), + }, + ) + subject = { + "role": spec.role, + "binding": spec.binding, + "operation": spec.operation, + "hotkey": spec.hotkey, + "spec_sha256": sha256(spec.model_dump_json().encode()), + } + receipt = seal( + owner, + "Receipt", + job.run_id, + Receipt(w=job.w, commit_hash=sha256(canonicalize(subject)), received_round=1), + 101, + ) + runtime.accept_operation(spec, canonicalize(receipt), owner=owner.ss58, beacon=1) + journal.append("network_execution_intent", role="h0", name=name, phase="workload", executions=1) + journal.append( + "network_operation_started", + role="h0", + name=name, + binding=spec.binding, + job_sha256=job.digest(), + operation=spec.operation, + hotkey=spec.hotkey, + sources_sha256=sha256(json.dumps(spec.sources, sort_keys=True).encode()), + cutoff=spec.cutoff, + ) + archive_path = tmp_path / "rescued.tar" + + def archive(): + with tarfile.open(archive_path, "w") as tar: + for path in sorted(original.rglob("*")): + if path.is_file(): + data = path.read_bytes() + member = tarfile.TarInfo("out/" + name + "/" + str(path.relative_to(original))) + member.size = len(data) + tar.addfile(member, io.BytesIO(data)) + journal.append( + "network_rescued", role="h0", tar=str(archive_path), tar_sha256=fsha(archive_path) + ) + journal.append( + "network_operation_done", + role="h0", + name=name, + result_sha256=fsha(original / "operation-result.json"), + tar_sha256=fsha(archive_path), + ) + + archive() + return SimpleNamespace( + runtime=runtime, + spec=spec, + directory=tmp_path, + original=original, + custody=custody, + result=result, + clock=clock, + archive=archive, + archive_path=archive_path, + owner=owner, + cleanup=cleanup, + signed_cutoff=signed_cutoff, + ) + + +@pytest.mark.parametrize("offset", [0, 1]) +def test_completion_original_signed_done_boundary(completed, offset): + n = completed + done = n.runtime.journal.last("network_operation_done") + n.runtime.journal.append( + "network_operation_done", + **{k: v for k, v in done.items() if k not in ("kind", "unix", "pid")}, + unix=n.signed_cutoff + offset, + ) + before = n.runtime.journal.records() + if offset: + with pytest.raises(cli.Reject, match="completion_original_done_missing"): + n.runtime.completion_custody(n.spec, n.directory, tree=PRODUCT) + else: + actual = n.runtime.completion_custody(n.spec, n.directory, tree=PRODUCT) + assert actual["body"]["done"]["unix"] == n.signed_cutoff + assert actual["body"]["status"] == "NOTSTOREACCEPTED" + assert n.runtime.journal.records() == before + + +@pytest.mark.parametrize("completed", [10], indirect=True) +def test_owner_signed_later_job_cannot_extend_original_challenge(completed): + n = completed + assert n.spec.job.deadline == n.signed_cutoff + 10 + done = n.runtime.journal.last("network_operation_done") + n.runtime.journal.append( + "network_operation_done", + **{k: v for k, v in done.items() if k not in ("kind", "unix", "pid")}, + unix=n.signed_cutoff + 1, + ) + with pytest.raises(cli.Reject, match="completion_original_signed_deadline_changed"): + n.runtime.completion_custody(n.spec, n.directory, tree=PRODUCT) + + +def test_authenticated_completion_descriptor_is_not_store_acceptance(completed): + n = completed + before = n.runtime.journal.records() + actual = n.runtime.completion_custody(n.spec, n.directory, tree=PRODUCT) + assert actual["sha256"] == sha256(canonicalize(actual["body"])) + assert actual["body"]["status"] == "NOTSTOREACCEPTED" + assert actual["body"]["production_backend_acceptance"] is False + assert actual["body"]["economic_acceptance"] is False + assert actual["body"]["publication"] == n.custody["body"] + assert n.runtime.completion_custody(n.spec, n.directory, tree=PRODUCT) == actual + assert n.runtime.journal.records() == before + + +@pytest.mark.parametrize( + "fault", + [ + "rescue", + "host", + "challenge", + "owner", + "source", + "done", + "execution", + "cancel", + "cutoff", + "cleanup", + ], +) +def test_completion_refuses_changed_authority_or_custody(completed, fault): + n = completed + cancel = threading.Event() + if fault == "rescue": + n.archive_path.write_bytes(n.archive_path.read_bytes() + b"changed rescue") + elif fault == "host": + n.runtime.lifecycle.host["h0"]["machine_id"] = 23 + elif fault == "owner": + authority = n.runtime.journal.last("network_operation_authority") + receipt = {**authority["receipt"], "sig": "00" * 64} + n.runtime.journal.append( + "network_operation_authority", + **{k: v for k, v in authority.items() if k not in ("kind", "unix", "pid", "receipt")}, + receipt=receipt, + ) + elif fault == "source": + n.runtime.journal.append( + "network_operation_source", + role="h0", + run_id=n.spec.job.run_id, + sources_sha256="00" * 32, + ) + elif fault == "done": + n.runtime.journal.append( + "network_operation_done", + role="h0", + name="op-" + + sha256( + ( + n.spec.job.run_id + + n.spec.operation + + n.spec.hotkey + + n.spec.binding + + n.spec.job.digest() + ).encode() + ), + result_sha256="00" * 32, + tar_sha256=fsha(n.archive_path), + ) + elif fault == "execution": + path = n.original / "execution-custody.json" + raw = json.loads(path.read_bytes()) + raw["instance_id"] = 12 + path.write_bytes(canonicalize(raw)) + n.archive() + elif fault == "challenge": + # Rehash outer custody/result/archive; wrong challenge signature must still reject. + n.custody["body"]["trial_authority"]["envelope"]["sig"] = "00" * 64 + trial = n.custody["body"]["trial_authority"] + trial["sha256"] = sha256(canonicalize(trial["envelope"])) + n.custody["sha256"] = sha256(canonicalize(n.custody["body"])) + (n.original / "publication-custody.json").write_bytes(canonicalize(n.custody)) + n.result["publication_custody_sha256"] = fsha(n.original / "publication-custody.json") + (n.original / "operation-result.json").write_bytes( + json.dumps(n.result, sort_keys=True).encode() + ) + n.archive() + elif fault == "cancel": + cancel.set() + elif fault == "cutoff": + n.clock["now"] = n.spec.cutoff + else: + n.cleanup["started"] = True + from hypertrain.protocol.envelope_v2 import SignatureError + + before = n.runtime.journal.records() + with pytest.raises((cli.Reject, SignatureError)): + n.runtime.completion_custody(n.spec, n.directory, tree=PRODUCT, cancel=cancel) + assert n.runtime.journal.records() == before diff --git a/tests/gpu_ops/test_work_screen_v2.py b/tests/gpu_ops/test_work_screen_v2.py new file mode 100644 index 0000000..c3046d1 --- /dev/null +++ b/tests/gpu_ops/test_work_screen_v2.py @@ -0,0 +1,85 @@ +"""Actual complete tiny island work; negative bindings, no hardware-count bounty.""" + +from __future__ import annotations + +import importlib +import sys +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from hypertrain.data.trial_assignment import trial_assignment_hash +from hypertrain.gpu_ops.work_screen import WorkScreenError, screen_work +from hypertrain.protocol.messages_v2 import JoinChallenge, RankWorkResult, WorkScreenV2 + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "ledger")) +fixtures = importlib.import_module("test_escrow_v2") +H, digest, setup, stage = fixtures.H, fixtures.digest, fixtures.setup, fixtures.stage + + +def challenge(job) -> JoinChallenge: + s = setup() + return JoinChallenge( + admission_id=H, + nonce=H, + seed_beacon=2, + deadline_beacon=102, + manifest_hash=job.run_id, + theta_hash=job.manifest.training.init_state_hash, + assignment_hash=trial_assignment_hash(job.manifest, job.w, tuple(job.sample_ids)), + layout=job.manifest.training.reference_spec.layout, + artifact_limits=s.admission_policy.artifact_limits, + policy_hash=digest(s.admission_policy), + ) + + +def test_full_round_work_screen_when_real_launch(tmp_path: Path) -> None: + # Given + s = setup() + job = stage(s, tmp_path, (0,), 0) + c = challenge(job) + # When + screen, proof = screen_work(job, tmp_path, c, now_beacon=2, backend="cpu") + # Then: real publication, every logical rank, bounded full state/delta/EF/leaves. + assert screen.evidence_kind == "work-proof-v1" and len(screen.rank_results) == 1 + assert proof.delta_hash == proof.artifact_refs[2].sha256 + assert screen.artifact_hashes == [r.sha256 for r in proof.artifact_refs] + assert (tmp_path / "published/rank-0/state.safetensors").is_file() + assert screen.rank_results[0].elapsed_ns > 0 + + +@pytest.mark.parametrize("fault", ["late", "assignment", "manifest", "theta"]) +def test_work_screen_rejects_when_challenge_wrong(tmp_path: Path, fault: str) -> None: + # Given + job = stage(setup(), tmp_path, (0,), 0) + c = challenge(job) + fields = {"assignment": "assignment_hash", "manifest": "manifest_hash", "theta": "theta_hash"} + if fault in fields: + c = JoinChallenge.model_validate({**c.body(), fields[fault]: "ff" * 32}) + # When / Then + with pytest.raises(WorkScreenError): + screen_work(job, tmp_path, c, now_beacon=103 if fault == "late" else 2, backend="cpu") + assert not (tmp_path / "published").exists() + + +@pytest.mark.parametrize("allocated,reserved", [(86, 10), (10, 86)]) +def test_peak_memory_rejects_when_full_round_exceeds_budget(allocated: int, reserved: int) -> None: + # Given / When / Then + with pytest.raises(ValidationError): + RankWorkResult( + rank=0, + fingerprint=H, + elapsed_ns=1, + allocated_bytes=allocated, + reserved_bytes=reserved, + total_bytes=100, + ) + + +@pytest.mark.parametrize("kind", ["nvidia-cc", "tpm", "snp", "tdx"]) +def test_remote_hardware_evidence_rejects_when_unsupported(kind: str) -> None: + # Given: hint is never an admission substitute. + # When / Then + with pytest.raises(ValidationError): + WorkScreenV2.model_validate(dict(evidence_kind=kind)) diff --git a/tests/layout/test_island_network_v2.py b/tests/layout/test_island_network_v2.py new file mode 100644 index 0000000..8462615 --- /dev/null +++ b/tests/layout/test_island_network_v2.py @@ -0,0 +1,218 @@ +"""Island network: independent DP arithmetic, EP collectives, identity hooks and device checks.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import numpy as np +import pytest +import torch +from reference import reference_round + +import hypertrain.trainer # noqa: F401 +from hypertrain.auditor.replay import pack_state +from hypertrain.trainer.config import TrainConfig +from hypertrain.trainer.island import TraceContext, emulate, train_island +from hypertrain.trainer.loop import Assignment, train_round +from hypertrain.trainer.model import init_params + + +def helpers(): + spec = importlib.util.spec_from_file_location( + "l1_launch_fixtures", Path(__file__).parents[1] / "miner/test_island_launch_v2.py" + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize( + "n,ep,zero1", [(1, 1, False), (2, 1, True), (4, 1, False), (2, 2, True), (4, 2, True)] +) +def test_same_layout_hooks_and_reference(n: int, ep: int, zero1: bool) -> None: + w = helpers().tiny_manifest(n, ep, zero1) + cfg = TrainConfig.from_manifest_v2(w) + lay = w.training.reference_spec.layout + theta = init_params(cfg.model) + a = Assignment(w.run_id(), 0, tuple(range(w.training.batch_samples()))) + + def get(i: int): + return (np.arange(5, dtype=np.uint32) + i) % 16 + + plain = emulate(n, lambda c: train_island(cfg, lay, c, theta, a, get)) + traces: list[list[tuple[TraceContext, bytes]]] = [[] for _ in range(n)] + + def hook(ctx: TraceContext, x: torch.Tensor) -> torch.Tensor: + raw = x.detach().cpu().contiguous().reshape(-1).view(torch.uint8).numpy().tobytes() + traces[ctx.rank].append((ctx, raw)) + return x + + traced = emulate(n, lambda c: train_island(cfg, lay, c, theta, a, get, hook=hook)) + for actual, expected in zip(traced, plain, strict=True): + assert actual.leaf_digests == expected.leaf_digests + assert actual.delta_payload == expected.delta_payload + assert pack_state(actual.final_theta, actual.final_state) == pack_state( + expected.final_theta, expected.final_state + ) + assert all(any(ctx.op == "attn" for ctx, _ in trace) for trace in traces) + assert all(any(ctx.op.startswith("optimizer.") for ctx, _ in trace) for trace in traces) + again: list[list[tuple[TraceContext, bytes]]] = [[] for _ in range(n)] + + def capture(ctx: TraceContext, x: torch.Tensor) -> torch.Tensor: + raw = x.detach().cpu().contiguous().reshape(-1).view(torch.uint8).numpy().tobytes() + again[ctx.rank].append((ctx, raw)) + return x + + emulate(n, lambda c: train_island(cfg, lay, c, theta, a, get, hook=capture)) + assert traces == again + if ep == 1: + oracle = reference_round(cfg, lay, theta, a, get) + assert plain[0].delta_hash == oracle["delta_hash"] + assert plain[0].leaves_root == oracle["leaves_root"] + else: + assert any(ctx.op == "all_to_all.output" for ctx, _ in traces[0]) + if n == 1: + assert plain[0].leaf_digests == train_round(cfg, theta, a, get).leaf_digests + + +@pytest.mark.parametrize("ep", [1, 2]) +def test_decoder_bf16_same_layout_cpu(ep: int) -> None: + from hypertrain.protocol.messages_v2 import RunManifestV2 + + # Given: identical signed N2 BF16 layouts, not a CUDA oracle. + body = helpers().tiny_manifest(2, ep).body() + body["training"]["model"]["compute_dtype"] = "bf16" + w = RunManifestV2.model_validate(body) + cfg = TrainConfig.from_manifest_v2(w) + theta = init_params(cfg.model) + a = Assignment(w.run_id(), 0, tuple(range(w.training.batch_samples()))) + # When: each logical rank executes its assigned rows. + outputs = emulate( + 2, + lambda c: train_island( + cfg, + w.training.reference_spec.layout, + c, + theta, + a, + lambda i: (np.arange(5, dtype=np.uint32) + i) % 16, + ), + ) + # Then: every leaf/full-state/EF/delta agrees within the exact layout. + assert outputs[0].leaf_digests == outputs[1].leaf_digests + assert outputs[0].delta_payload == outputs[1].delta_payload + assert pack_state(outputs[0].final_theta, outputs[0].final_state) == pack_state( + outputs[1].final_theta, outputs[1].final_state + ) + assert pack_state(outputs[0].ef_out) == pack_state(outputs[1].ef_out) + + +def test_independent_reference_kills_divide_by_one(monkeypatch: pytest.MonkeyPatch) -> None: + from hypertrain.trainer import island + + w = helpers().tiny_manifest(2) + cfg = TrainConfig.from_manifest_v2(w) + lay = w.training.reference_spec.layout + theta = init_params(cfg.model) + a = Assignment(w.run_id(), 0, tuple(range(w.training.batch_samples()))) + + def get(i: int): + return (np.arange(5, dtype=np.uint32) + i) % 16 + + reference = reference_round(cfg, lay, theta, a, get) + original = island._reduce + + def wrong(comm, geo, grads, reduction): + return {k: x * lay.n_gpus for k, x in original(comm, geo, grads, reduction).items()} + + monkeypatch.setattr(island, "_reduce", wrong) + mutated = emulate(2, lambda c: train_island(cfg, lay, c, theta, a, get))[0] + assert mutated.leaves_root != reference["leaves_root"] + + +def test_collective_scalar_device_is_explicit() -> None: + from hypertrain.trainer.island import Geometry, _grad_norm, _mean_loss + + w = helpers().tiny_manifest(1) + cfg = TrainConfig.from_manifest_v2(w) + lay = w.training.reference_spec.layout + + class DeviceComm: + rank = 0 + world = 1 + + def all_gather(self, x: torch.Tensor) -> list[torch.Tensor]: + assert x.device.type == "cpu" + return [x] + + red = init_params(cfg.model) + with torch.device("meta"): + assert _grad_norm(DeviceComm(), Geometry(cfg.model, lay, 0), red) > 0 + assert _mean_loss(DeviceComm(), 1.0, torch.device("cpu")) == 1 + pack_state(red) + + +def test_cancel_boundary_refuses_execution() -> None: + w = helpers().tiny_manifest(2) + cfg = TrainConfig.from_manifest_v2(w) + + def cancelled() -> None: + raise RuntimeError("cancelled lease") + + with pytest.raises(RuntimeError, match="cancelled lease"): + emulate( + 2, + lambda c: train_island( + cfg, + w.training.reference_spec.layout, + c, + init_params(cfg.model), + Assignment(w.run_id(), 0, tuple(range(4))), + lambda i: np.zeros(5, np.uint32), + cancel=cancelled, + ), + ) + + +def test_ep_gradient_replica_sum_independent_reference() -> None: + from hypertrain.trainer.island import Geometry, _reduce + + w = helpers().tiny_manifest(4, 2, True) + cfg = TrainConfig.from_manifest_v2(w) + lay = w.training.reference_spec.layout + + def run(comm): + geo = Geometry(cfg.model, lay, comm.rank) + full = init_params(cfg.model) + grads = {k: torch.full_like(geo.local(k, x), comm.rank + 1) for k, x in full.items()} + result = _reduce(comm, geo, grads, "all_gather") + for k, x in result.items(): + expected = (4 if comm.rank % 2 == 0 else 6) / 4 if geo.is_expert(k) else 10 / 4 + assert torch.equal(x, torch.full_like(x, expected)) + return True + + assert emulate(4, run) == [True] * 4 + + +def test_zero_owner_rank_participates_without_host_device_fallback() -> None: + from hypertrain.trainer.island import Geometry, gather_pieces + + # Given: one optimizer piece, four ranks; three ranks own no state. + w = helpers().tiny_manifest(4) + cfg = TrainConfig.from_manifest_v2(w) + lay = w.training.reference_spec.layout + name = "emb.weight" + expected = torch.ones((cfg.model.vocab, cfg.model.d_model), device="cpu") + + def run(comm): + geo = Geometry(cfg.model, lay, comm.rank) + values = {name: expected} if comm.rank == 0 else {} + # When: every rank joins the same real fixed-order gather. + got = gather_pieces(comm, geo, values, [(name, 0)], lambda _: 0, expected.device) + # Then: empty-owner ranks receive the exact tensor, no inferred CPU allocation. + assert torch.equal(got[(name, 0)], expected) + return True + + assert emulate(4, run) == [True] * 4 diff --git a/tests/layout/test_od_island_v2.py b/tests/layout/test_od_island_v2.py new file mode 100644 index 0000000..daa5189 --- /dev/null +++ b/tests/layout/test_od_island_v2.py @@ -0,0 +1,155 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import numpy as np +import pytest +import torch +from opendecision.records import RecordShape, pack_record, qmax_for +from pydantic import ValidationError + +import hypertrain.trainer # noqa: F401 +from hypertrain.auditor.honeypot import Honeypot, honeypot_island +from hypertrain.auditor.replay import pack_state +from hypertrain.models.opendecision import check_manifest, extend_params +from hypertrain.protocol.messages import RunManifest +from hypertrain.protocol.messages_v2 import RunManifestV2 +from hypertrain.trainer.config import TrainConfig +from hypertrain.trainer.island import TraceContext, emulate, train_island +from hypertrain.trainer.loop import Assignment +from hypertrain.trainer.model import init_params + + +def od_manifest(objective: str, n: int, zero1: bool, dtype: str = "fp32") -> RunManifestV2: + spec = importlib.util.spec_from_file_location( + "od_l1_base", Path(__file__).parents[1] / "protocol/test_od_fields.py" + ) + assert spec and spec.loader + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + b = mod.od_body() + rec = dict(state_len=8, n_questions=1, n_options=2, opt_len=2, instr_len=2) + b["model"].update( + seq_len=8 if objective == "mlm" else 18, + param_count=116288 if objective == "mlm" else 187140, + compute_dtype=dtype, + ) + b["model"]["od"].update(objective=objective, record=None if objective == "mlm" else rec) + b["inner"].update(H=2, J=1, micro_batch=1, grad_accum=1, state_policy="reset", rewarmup_steps=0) + b["reference_spec"]["profile"] = "od-fp32-ref-v1" if dtype == "fp32" else "od-bf16-det-eager-v1" + b["reference_spec"]["layout"].update(pp=1, n_gpus=n, dp_size=n, ep_size=1, zero1=zero1) + return RunManifestV2.model_validate( + { + "manifest_version": 2, + "training": b, + "network": { + **{ + k: "11" * 32 + for k in ( + "admission_policy_hash", + "economics_policy_hash", + "aggregation_policy_hash", + "dispute_policy_hash", + "audit_policy_hash", + "relay_registry_hash", + ) + }, + "audit_mode": "anchored-full", + "full_anchor_version": 1, + "capabilities": ["island-replay", "all-level-disputes", "transport-receipts"], + }, + } + ) + + +def samples(cfg: TrainConfig): + def get(i: int): + if cfg.model.od.objective == "mlm": + return 3 + (np.arange(cfg.model.seq_len + 1, dtype=np.uint32) + i) % 200 + shape = RecordShape(**cfg.model.od.record.model_dump()) + return pack_record( + { + "state": [3 + i, 4, 5], + "instr": [[6]], + "opts": [[[7], [8]]], + "qtype": [i % 2], + "y": [i % 2], + "teacher": [[0.7, 0.2, 0.1]], + }, + shape, + qmax_for(cfg.model.vocab), + ).astype(np.uint32) + + return get + + +@pytest.mark.parametrize("objective", ["mlm", "decision", "distill"]) +@pytest.mark.parametrize("n", [1, 2, 4]) +@pytest.mark.parametrize("zero1", [False, True]) +def test_od_objectives_layout_reference_and_hooks(objective: str, n: int, zero1: bool) -> None: + from reference import reference_round + + w = od_manifest(objective, n, zero1) + check_manifest(w) + cfg = TrainConfig.from_manifest_v2(w) + lay = w.training.reference_spec.layout + theta = init_params(cfg.model) + a = Assignment(w.run_id(), 0, tuple(range(w.training.batch_samples()))) + get = samples(cfg) + plain = emulate(n, lambda c: train_island(cfg, lay, c, theta, a, get)) + seen = [[] for _ in range(n)] + + def hook(ctx: TraceContext, x: torch.Tensor) -> torch.Tensor: + raw = x.detach().cpu().contiguous().reshape(-1).view(torch.uint8).numpy().tobytes() + seen[ctx.rank].append((ctx, raw)) + return x + + traced = emulate(n, lambda c: train_island(cfg, lay, c, theta, a, get, hook=hook)) + for x in traced: + assert x.leaf_digests == plain[0].leaf_digests + assert x.delta_payload == plain[0].delta_payload + assert pack_state(x.final_theta, x.final_state) == pack_state( + plain[0].final_theta, plain[0].final_state + ) + assert plain[0].leaves_root == reference_round(cfg, lay, theta, a, get)["leaves_root"] + assert all( + any(ctx.op == "head" and ctx.layer == cfg.model.n_layers for ctx, _ in ops) for ops in seen + ) + repeated = [[] for _ in range(n)] + + def capture(ctx: TraceContext, x: torch.Tensor) -> torch.Tensor: + raw = x.detach().cpu().contiguous().reshape(-1).view(torch.uint8).numpy().tobytes() + repeated[ctx.rank].append((ctx, raw)) + return x + + emulate(n, lambda c: train_island(cfg, lay, c, theta, a, get, hook=capture)) + assert seen == repeated + if n > 1 or zero1: + with pytest.raises(ValidationError): + RunManifest.model_validate(w.training.body()) + if objective != "mlm": + mlm = TrainConfig.from_manifest_v2(od_manifest("mlm", 1, False)) + extended = extend_params(init_params(mlm.model), cfg.model) + assert all( + torch.equal(extended[k], init_params(mlm.model)[k]) for k in init_params(mlm.model) + ) + pots = emulate( + n, + lambda c: honeypot_island(Honeypot(w.training.coord_pubkey, "honest"), w, c, theta, a, get), + ) + assert pots[0].leaf_digests == plain[0].leaf_digests + + +@pytest.mark.parametrize("objective", ["mlm", "decision", "distill"]) +@pytest.mark.parametrize("zero1", [False, True]) +def test_od_bf16_same_layout_cpu(objective: str, zero1: bool) -> None: + w = od_manifest(objective, 2, zero1, "bf16") + cfg = TrainConfig.from_manifest_v2(w) + theta = init_params(cfg.model) + a = Assignment(w.run_id(), 0, tuple(range(w.training.batch_samples()))) + result = emulate( + 2, lambda c: train_island(cfg, w.training.reference_spec.layout, c, theta, a, samples(cfg)) + ) + assert result[0].leaf_digests == result[1].leaf_digests + assert result[0].delta_payload == result[1].delta_payload diff --git a/tests/ledger/test_escrow_v2.py b/tests/ledger/test_escrow_v2.py new file mode 100644 index 0000000..98c6097 --- /dev/null +++ b/tests/ledger/test_escrow_v2.py @@ -0,0 +1,1343 @@ +"""Origin accounting negatives with real signatures and persistent SQLite transactions.""" + +from __future__ import annotations + +import importlib +import json +import sqlite3 +import threading +import time +from collections.abc import Callable +from dataclasses import dataclass, replace +from pathlib import Path + +import numpy as np +import pytest +from pydantic import ValidationError + +import hypertrain.trainer # noqa: F401 +from hypertrain.auditor.replay import pack_state, unpack_state +from hypertrain.ledger.escrow_v2 import ( + EscrowError, + EscrowV2, + FinalityEvidence, + RewardWork, + Settlement, + ShadowReservationEvidence, + authority_message, + digest, + genesis_allocation_hash, + genesis_origin_id, + reward_origin_id, + shadow_origin_id, +) +from hypertrain.protocol.envelope_v2 import EnvelopeV2, UnauthorizedSigner, seal +from hypertrain.protocol.example import example_manifest +from hypertrain.protocol.hashing import MerkleTree, sha256_hex +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.keys import Keypair +from hypertrain.protocol.messages import Finalize, ReplayEnv, ReplayVerdict, f32hex +from hypertrain.protocol.messages_v2 import ( + AdmissionPolicyV2, + ArtifactLimits, + AuditChallengeV2, + CommitV2, + EconomicsPolicyV2, + EscrowLock, + EscrowOperation, + EscrowRelease, + EscrowTransfer, + IslandJobV1, + OriginAllocation, + RewardFinalize, + RunManifestV2, + ShadowReplayReceiptV1, + ShadowReservationRequestV1, + ShadowReservationV1, + ShadowRewardFinalizeV1, + TestGenesis, +) +from hypertrain.trainer.compress import state_hash +from hypertrain.trainer.config import TrainConfig +from hypertrain.trainer.model import init_params, param_shapes + +torch = importlib.import_module("torch") + +COORD = Keypair(bytes(range(32))) +AUDITOR = Keypair(bytes(31) + b"\x01") +HOT = Keypair(b"\x07" * 32) +COLD = Keypair(b"\x08" * 32) +OTHER = Keypair(b"\x09" * 32) +H = "11" * 32 +VERIFIED_SHADOW_STATES: dict[str, str] = {} + + +@dataclass(frozen=True, slots=True) +class Setup: + manifest: RunManifestV2 + policy: EconomicsPolicyV2 + admission_policy: AdmissionPolicyV2 + rows: np.ndarray + tree: MerkleTree + + +def setup(mode: str = "test") -> Setup: + admission = AdmissionPolicyV2( + q_base="0000803e", + E=4, + clean_finalizations=12, + work_screen_epoch_rounds=1000, + max_pending=32, + max_trial_replays=2, + max_trials_per_beacon=2, + join_per_beacon=2, + join_burst=4, + ip_prefix_per_beacon=32, + ip_prefix_burst=64, + challenge_rounds=100, + artifact_limits=ArtifactLimits( + max_object_bytes=1_000_000, max_body_bytes=65536, max_chunk_manifest_bytes=1048576 + ), + ) + alloc_hash = genesis_allocation_hash([(COLD.ss58, 20000, 1), (OTHER.ss58, 10000, 1)]) + policy = EconomicsPolicyV2.model_validate( + dict( + ledger_mode=mode, + G_max_units=100, + R_collectible_units=100, + S_min_units=1000, + beta_ppm=1_000_000, + gammaV_units=0, + s_lower_ppm=1_000_000, + q_floor=1_000_000, + genesis_allocation_hash=alloc_hash if mode == "test" else None, + reward_authority=COORD.ss58, + round_reward_units=1_000_000, + max_total_issuance=20_000_000, + shadow_bootstrap_rounds=12, + ) + ) + b = example_manifest().body() + b["model"].update( + n_layers=1, + d_model=8, + n_heads=2, + n_kv_heads=2, + d_ff=12, + vocab=16, + seq_len=4, + n_experts=1, + top_k_experts=1, + ) + b["inner"].update(H=1, J=1, micro_batch=1, grad_accum=1, state_policy="reset") + b["inner"]["lr_schedule"].update(warmup=0, stable=100, peak_lr=f32hex(0.001)) + b["model"]["param_count"] = sum( + int(np.prod(s)) for s in param_shapes(TrainConfig.from_manifest(b).model).values() + ) + rows = (np.arange(16 * 5).reshape(16, 5) % 16).astype(" IslandJobV1: + cfg = TrainConfig.from_manifest_v2(s.manifest) + theta = init_params(cfg.model) + inputs = dict( + start_state=pack_state(theta), + ef_in=pack_state({k: torch.zeros_like(v) for k, v in theta.items()}), + v0=pack_state({}), + samples=b"".join(s.rows[i].tobytes() for i in samples), + sample_proofs=json.dumps([[h.hex() for h in s.tree.proof(i)] for i in samples]).encode(), + ) + directory.mkdir(parents=True, exist_ok=True) + for name, data in inputs.items(): + (directory / name).write_bytes(data) + return IslandJobV1( + job_version=1, + run_id=s.manifest.run_id(), + w=w, + manifest=s.manifest, + sample_ids=list(samples), + global_step0=0, + start_state_sha256=sha256_hex(inputs["start_state"]), + ef_in_sha256=sha256_hex(inputs["ef_in"]), + v0_sha256=sha256_hex(inputs["v0"]), + object_paths={k: k for k in inputs}, + deadline=int(time.time()) + 120, + ) + + +def ledger( + s: Setup, + database: Path | sqlite3.Connection, + settlement: Callable[[str], Settlement] | None = None, + replay: Callable[[FinalityEvidence], str] | None = None, +) -> EscrowV2: + def no_unverified_finality(evidence: FinalityEvidence) -> str: + raise EscrowError("NO_REPLAY_ACCEPTANCE") + + return EscrowV2( + database, + s.manifest, + s.policy, + auditors=frozenset({AUDITOR.ss58}), + replay_finality=replay or no_unverified_finality, + settlement=settlement + or ( + lambda _: Settlement( + finality_hash=H, + closed_dispute_root=H, + release_beacon=10, + unresolved=False, + outcome="MATCH", + ) + ), + owner_of=lambda h: COLD.ss58 if h == HOT.ss58 else OTHER.ss58, + assignment_of=lambda w, h: (0,), + shadow_assignment_of=lambda binding: (0,), + shadow_settlement=settlement, + ) + + +def fund(e: EscrowV2) -> tuple[str, str]: + alloc = [(COLD.ss58, 20000, 1), (OTHER.ss58, 10000, 1)] + ah = genesis_allocation_hash(alloc) + origins = [ + OriginAllocation( + origin_id=genesis_origin_id(e.run_id, ah, i), owner=h, units=u, mature_at=m + ) + for i, (h, u, m) in enumerate(alloc) + ] + g = TestGenesis( + run_id=e.run_id, + allocation_hash=ah, + total_units=30000, + origins=origins, + authority_sig="0" * 128, + ) + g = TestGenesis.model_validate( + {**g.body(), "authority_sig": COORD.sign(authority_message(g)).hex()} + ) + e.test_genesis(g) + e.mature(H, tuple(o.origin_id for o in origins), 1) + return origins[0].origin_id, origins[1].origin_id + + +def lock(origin: str, operation: str = "22" * 32, units: int = 1000) -> EscrowLock: + return EscrowLock( + operation_id=operation, + owner=COLD.ss58, + units=units, + origin_ids=[origin], + admission_id=H, + dispute_id=None, + kind="LOCK_ADMISSION", + ) + + +def test_conservation_when_transfer_lock_release_pay_restart(tmp_path: Path) -> None: + # Given: independently funded origins, not v1 entitlement bytes. + s = setup() + e = ledger(s, tmp_path / "ledger.db") + origin, _ = fund(e) + e.transfer( + EscrowTransfer( + operation_id="33" * 32, + owner=COLD.ss58, + recipient=OTHER.ss58, + units=500, + origin_ids=[origin], + admission_id=None, + dispute_id=None, + ), + signer=COLD.ss58, + ) + receipt = e.lock(lock(origin), signer=COLD.ss58) + e.release( + EscrowRelease( + operation_id="44" * 32, + owner=COLD.ss58, + units=1000, + origin_ids=[origin], + admission_id=H, + dispute_id=None, + lock_id="22" * 32, + finality_hash=H, + closed_dispute_root=H, + ), + signer=COLD.ss58, + now_beacon=10, + ) + e.pay( + EscrowOperation( + operation_id="55" * 32, + owner=COLD.ss58, + units=500, + origin_ids=[origin], + admission_id=None, + dispute_id=None, + ), + signer=COLD.ss58, + ) + # When: reopen exact pinned mode/run. + restored = ledger(s, tmp_path / "ledger.db") + # Then: buckets and receipt retain exact conserved amounts. + assert restored.balances().conserved() + assert restored.balances().paid == 500 + assert restored.balances(COLD.ss58).available == 19000 + assert restored.lock(lock(origin), signer=COLD.ss58) == receipt + + +@pytest.mark.parametrize("fault", ["unknown_origin", "owner", "double_spend", "operation_conflict"]) +def test_lock_rejects_when_funding_invalid(tmp_path: Path, fault: str) -> None: + # Given + e = ledger(setup(), tmp_path / "ledger.db") + origin, _ = fund(e) + e.lock(lock(origin), signer=COLD.ss58) + op = lock(origin, "33" * 32, 19001) + signer = COLD.ss58 + if fault == "unknown_origin": + op = lock("ff" * 32, "33" * 32) + if fault == "owner": + signer = OTHER.ss58 + if fault == "operation_conflict": + op = lock(origin, units=999) + before = e.balances() + # When / Then + with pytest.raises(EscrowError): + e.lock(op, signer=signer) + assert e.balances() == before + + +def test_legacy_entitlement_cannot_fund_empty_ledger(tmp_path: Path) -> None: + # Given: v1 has no admitted v2 origins or bridge. + e = ledger(setup(), tmp_path / "ledger.db") + # When / Then: even a plausible v1 entitlement hash creates no units. + with pytest.raises(EscrowError, match="UNFUNDED"): + e.lock(lock(H), signer=COLD.ss58) + assert e.balances().issued == 0 + + +def test_legacy_source_payout_leaves_v2_zero_after_import_attempt(tmp_path: Path) -> None: + from hypertrain.ledger import Ledger, Params + + # Given: actual v1 entitlement, independently payable on its own ledger. + source = Ledger(tmp_path / "v1", Params("hypertrain", 0, 100, 1, 1)) + source.commit(0, COLD.ss58, 10) + source.verdict(0, COLD.ss58, "MATCH", 4, 1_000_000, 20) + source.finalize(0, 30) + e = ledger(setup(), tmp_path / "v2.db") + # When: attempted copied funding rejected; original source subsequently pays. + with pytest.raises(EscrowError): + e.lock(lock(H), signer=COLD.ss58) + source.get_weights(1, 100, 100) + # Then: no value exists in both ledgers. + assert source.state().paid > 0 and e.balances().issued == 0 + + +def test_immature_origin_cannot_lock(tmp_path: Path) -> None: + # Given + s = setup() + e = ledger(s, tmp_path / "ledger.db") + ah = s.policy.genesis_allocation_hash + assert ah is not None + origins = [ + OriginAllocation( + origin_id=genesis_origin_id(e.run_id, ah, i), owner=h, units=u, mature_at=m + ) + for i, (h, u, m) in enumerate([(COLD.ss58, 20000, 1), (OTHER.ss58, 10000, 1)]) + ] + g = TestGenesis( + run_id=e.run_id, + allocation_hash=ah, + total_units=30000, + origins=origins, + authority_sig="0" * 128, + ) + g = TestGenesis.model_validate( + {**g.body(), "authority_sig": COORD.sign(authority_message(g)).hex()} + ) + e.test_genesis(g) + # When / Then + with pytest.raises(EscrowError): + e.lock(lock(origins[0].origin_id), signer=COLD.ss58) + assert e.balances().reward_pending == 30000 + + +@pytest.mark.parametrize("fault", ["authority", "mode", "origin_domain", "allocation"]) +def test_genesis_rejects_when_authority_or_mode_wrong(tmp_path: Path, fault: str) -> None: + # Given + s = setup("production" if fault == "mode" else "test") + e = ledger(s, tmp_path / "ledger.db") + ah = genesis_allocation_hash([(COLD.ss58, 20000, 1), (OTHER.ss58, 10000, 1)]) + origins = [ + OriginAllocation( + origin_id=genesis_origin_id(e.run_id, ah, i), owner=h, units=u, mature_at=m + ) + for i, (h, u, m) in enumerate([(COLD.ss58, 20000, 1), (OTHER.ss58, 10000, 1)]) + ] + if fault == "origin_domain": + origins[0] = OriginAllocation(origin_id=H, owner=COLD.ss58, units=20000, mature_at=1) + g = TestGenesis( + run_id=e.run_id, + allocation_hash="ff" * 32 if fault == "allocation" else ah, + total_units=30000, + origins=origins, + authority_sig="0" * 128, + ) + signer = OTHER if fault == "authority" else COORD + g = TestGenesis.model_validate( + {**g.body(), "authority_sig": signer.sign(authority_message(g)).hex()} + ) + # When / Then + with pytest.raises(EscrowError): + e.test_genesis(g) + assert e.balances().issued == 0 + + +@pytest.mark.parametrize("unresolved,beacon", [(True, 10), (False, 9)]) +def test_release_rejects_when_disputed_or_unvested( + tmp_path: Path, unresolved: bool, beacon: int +) -> None: + # Given + status = Settlement( + finality_hash=H, + closed_dispute_root=H, + release_beacon=10, + unresolved=unresolved, + outcome="MATCH", + ) + e = ledger(setup(), tmp_path / "ledger.db", settlement=lambda _: status) + origin, _ = fund(e) + e.lock(lock(origin), signer=COLD.ss58) + op = EscrowRelease( + operation_id="33" * 32, + owner=COLD.ss58, + units=1000, + origin_ids=[origin], + admission_id=H, + dispute_id=None, + lock_id="22" * 32, + finality_hash=H, + closed_dispute_root=H, + ) + # When / Then + with pytest.raises(EscrowError): + e.release(op, signer=COLD.ss58, now_beacon=beacon) + assert e.balances().admission_locked == 1000 + + +def test_contest_cost_and_burn_when_proven_loss(tmp_path: Path) -> None: + # Given + status = Settlement( + finality_hash=H, closed_dispute_root=H, release_beacon=10, unresolved=False, outcome="FRAUD" + ) + e = ledger(setup(), tmp_path / "ledger.db", settlement=lambda _: status) + origin, _ = fund(e) + op = EscrowLock( + operation_id="22" * 32, + owner=COLD.ss58, + units=1000, + origin_ids=[origin], + admission_id=None, + dispute_id=H, + kind="LOCK_CONTEST", + ) + e.lock(op, signer=COLD.ss58) + slash = EscrowRelease( + operation_id="33" * 32, + owner=COLD.ss58, + units=1000, + origin_ids=[origin], + admission_id=None, + dispute_id=H, + lock_id=op.operation_id, + finality_hash=H, + closed_dispute_root=H, + ) + # When + receipt = e.slash( + slash, + authority=COORD.ss58, + now_beacon=10, + referee=OTHER.ss58, + referee_cost=100, + max_referee_cost=100, + ) + # Then + assert e.balances().burned == 900 and e.balances().dispute_locked == 0 + assert e.balances(OTHER.ss58).available == 10100 and e.balances().conserved() + assert ( + e.slash( + slash, + authority=COORD.ss58, + now_beacon=10, + referee=OTHER.ss58, + referee_cost=100, + max_referee_cost=100, + ) + == receipt + ) + + +def test_auditor_infrastructure_never_slashes_miner(tmp_path: Path) -> None: + # Given + status = Settlement( + finality_hash=H, + closed_dispute_root=H, + release_beacon=10, + unresolved=False, + outcome="INFRASTRUCTURE", + ) + e = ledger(setup(), tmp_path / "ledger.db", settlement=lambda _: status) + origin, _ = fund(e) + e.lock(lock(origin), signer=COLD.ss58) + op = EscrowRelease( + operation_id="33" * 32, + owner=COLD.ss58, + units=1000, + origin_ids=[origin], + admission_id=H, + dispute_id=None, + lock_id="22" * 32, + finality_hash=H, + closed_dispute_root=H, + ) + # When / Then + with pytest.raises(EscrowError): + e.slash(op, authority=COORD.ss58, now_beacon=10) + assert e.balances().burned == 0 + + +def test_second_genesis_rejects_conflicting_allocation(tmp_path: Path) -> None: + # Given + s = setup() + e = ledger(s, tmp_path / "ledger.db") + fund(e) + ah = s.policy.genesis_allocation_hash + assert ah is not None + origins = [ + OriginAllocation( + origin_id=genesis_origin_id(e.run_id, ah, i), owner=h, units=u, mature_at=m + ) + for i, (h, u, m) in enumerate([(COLD.ss58, 20001, 1), (OTHER.ss58, 9999, 1)]) + ] + g = TestGenesis( + run_id=e.run_id, + allocation_hash=ah, + total_units=30000, + origins=origins, + authority_sig="0" * 128, + ) + g = TestGenesis.model_validate( + {**g.body(), "authority_sig": COORD.sign(authority_message(g)).hex()} + ) + # When / Then + with pytest.raises(EscrowError): + e.test_genesis(g) + assert e.balances().issued == 30000 + + +def test_origin_state_corruption_rejects_when_reopened(tmp_path: Path) -> None: + # Given + s = setup() + path = tmp_path / "ledger.db" + e = ledger(s, path) + fund(e) + e.db.execute("UPDATE escrow_units SET owner=? WHERE owner=?", (HOT.ss58, COLD.ss58)) + # When / Then: total conservation alone would miss ownership corruption. + with pytest.raises(EscrowError, match="PERSISTED_STATE_MISMATCH"): + ledger(s, path) + + +def test_concurrent_locks_when_same_funded_origin(tmp_path: Path) -> None: + # Given: two genuine connections, a barrier instead of timing luck. + s = setup() + path = tmp_path / "ledger.db" + e = ledger(s, path) + origin, _ = fund(e) + engines = [ledger(s, path), ledger(s, path)] + barrier = threading.Barrier(2) + outcomes = [] + + def take(index: int) -> None: + barrier.wait(timeout=10) + try: + engines[index].lock(lock(origin, f"{index + 2:02x}" * 32, 15000), signer=COLD.ss58) + outcomes.append("locked") + except EscrowError: + outcomes.append("unfunded") + + # When + threads = [threading.Thread(target=take, args=(i,)) for i in range(2)] + for t in threads: + t.start() + for t in threads: + t.join(timeout=10) + assert not t.is_alive() + # Then + assert sorted(outcomes) == ["locked", "unfunded"] + assert e.balances().admission_locked == 15000 and e.balances().conserved() + + +@pytest.fixture(scope="module") +def actual_reward( + tmp_path_factory: pytest.TempPathFactory, +) -> tuple[Setup, FinalityEvidence, RewardFinalize]: + from hypertrain.gpu_ops.work_screen import commitments + from hypertrain.miner.island_launch import launch_island + + s = setup("production") + # Two legitimate one-round awards exactly fill this fixture's pinned shared cap. + policy = EconomicsPolicyV2.model_validate({**s.policy.body(), "max_total_issuance": 2_000_000}) + body = s.manifest.body() + body["network"]["economics_policy_hash"] = digest(policy) + s = replace(s, policy=policy, manifest=RunManifestV2.model_validate(body)) + directory = tmp_path_factory.mktemp("real-reward") + job = stage(s, directory, (0,), 0) + a = launch_island(job, directory, backend="cpu") + root, delta = commitments(a) + theta, _ = unpack_state(a.state.read_bytes()) + c = CommitV2( + w=0, + hotkey=HOT.ss58, + leaf_scheme="ht-leaf-v1", + n_leaves=2, + leaves_root=root, + metrics_root=H, + final_theta_hash=state_hash(theta), + ef_in_hash=H, + ef_out_hash=H, + delta_hash=delta, + delta_bytes=a.delta.stat().st_size, + tokens=4, + ) + challenge = AuditChallengeV2( + w=0, + target=HOT.ss58, + beacon_round=1, + beacon_sig_sha256=H, + mode="full", + segments=[], + reasons=["probation"], + serve_deadline=200, + anchor_hash=H, + audit_mode="anchored-full", + ) + # Independent actual same-layout full replay in another process group. + refdir = tmp_path_factory.mktemp("real-reference") + refjob = stage(s, refdir, (0,), 0) + reference = launch_island(refjob, refdir, backend="cpu") + assert ( + commitments(reference) == (root, delta) + and reference.state.read_bytes() == a.state.read_bytes() + ) + v = ReplayVerdict( + challenge_hash=digest(challenge), + first_bad_leaf=None, + result="MATCH", + recomputed_leaves_root=commitments(reference)[0], + replay_env=ReplayEnv( + image_digest=s.manifest.training.reference_spec.image_digest, + driver="cpu-test", + gpu_uuid_sha256=H, + sm_count=1, + ), + ) + final = Finalize( + w=0, final_theta_hash_w1=state_hash(theta), included=[HOT.ss58], entitlements_root=H + ) + final_env = EnvelopeV2.model_validate(seal(COORD, "Finalize", s.manifest.run_id(), final, 200)) + evidence = FinalityEvidence( + final_env, + ( + RewardWork( + EnvelopeV2.model_validate(seal(HOT, "CommitV2", s.manifest.run_id(), c, 200)), + EnvelopeV2.model_validate( + seal(AUDITOR, "ReplayVerdict", s.manifest.run_id(), v, 200) + ), + EnvelopeV2.model_validate( + seal(COORD, "AuditChallengeV2", s.manifest.run_id(), challenge, 200) + ), + COLD.ss58, + (0,), + ), + ), + sha256_hex(reference.state.read_bytes()), + False, + 10, + 20, + ) + VERIFIED_SHADOW_STATES[evidence.tape_hash] = state_hash( + unpack_state(reference.state.read_bytes())[0] + ) + origins = [ + OriginAllocation( + origin_id=reward_origin_id(s.manifest.run_id(), 0, HOT.ss58, digest(final)), + owner=COLD.ss58, + units=1_000_000, + mature_at=20, + ) + ] + reward = RewardFinalize( + run_id=s.manifest.run_id(), + w=0, + finalize_hash=digest(final), + tape_hash=evidence.tape_hash, + verdict_root=sha256_hex(canonicalize([digest(v)])), + allocation_hash=sha256_hex(canonicalize([o.body() for o in origins])), + budget_units=1_000_000, + origin_ids=[o.origin_id for o in origins], + mature_at=20, + authority_sig="0" * 128, + ) + reward = RewardFinalize.model_validate( + {**reward.body(), "authority_sig": COORD.sign(authority_message(reward)).hex()} + ) + return s, evidence, reward + + +def verified_shadow(evidence: FinalityEvidence) -> str: + # A transcript binds independently executed full replay state bytes, not a success sample. + work = evidence.works[0] + c = CommitV2.model_validate(work.commit.body) + final = Finalize.model_validate(evidence.finalize.body) + actual = VERIFIED_SHADOW_STATES[evidence.tape_hash] + assert c.final_theta_hash == final.final_theta_hash_w1 == actual + return actual + + +def reserve_actual_shadow( + e: EscrowV2, evidence: FinalityEvidence, admission_id: str = H +) -> ShadowReservationV1: + """Reuse independently executed module work; reserve before reward signing.""" + final = Finalize.model_validate(evidence.finalize.body) + commit = CommitV2.model_validate(evidence.works[0].commit.body) + request = ShadowReservationRequestV1( + run_id=e.run_id, + admission_id=admission_id, + hotkey=commit.hotkey, + coldkey=evidence.works[0].owner, + trial_epoch=final.w, + commit_hash=digest(commit), + finalize_hash=digest(final), + custody_hash=evidence.tape_hash, + graph_hash=H, + economic_admission_hash=H, + nonce=H, + assignment_hash=H, + finalized_beacon=evidence.finalized_beacon, + mature_at=evidence.finalized_beacon + e.manifest.training.verify.E_vest_rounds, + ) + return e.reserve_shadow(request, original_trial_custody(evidence)) + + +def original_trial_custody(evidence: FinalityEvidence) -> ShadowReservationEvidence: + """Use original retained reference bytes, not the acceptance replay callback.""" + item = evidence.works[0] + return ShadowReservationEvidence( + finalize=evidence.finalize, + commit=item.commit, + owner=item.owner, + sample_ids=item.sample_ids, + custody_hash=evidence.tape_hash, + reference_final_state_hash=VERIFIED_SHADOW_STATES[evidence.tape_hash], + ) + + +def sign_actual_shadow( + reward: RewardFinalize, binding: ShadowReservationV1 +) -> ShadowRewardFinalizeV1: + origin = OriginAllocation( + origin_id=shadow_origin_id(binding), + owner=binding.coldkey, + units=reward.budget_units, + mature_at=binding.mature_at, + ) + unsigned = ShadowRewardFinalizeV1.model_validate( + { + **reward.body(), + "shadow": True, + "shadow_ordinal": binding.shadow_ordinal, + "reservation_hash": digest(binding), + "mature_at": binding.mature_at, + "origin_ids": [origin.origin_id], + "allocation_hash": sha256_hex(canonicalize([origin.body()])), + } + ) + return unsigned.model_copy( + update={"authority_sig": COORD.sign(authority_message(unsigned)).hex()} + ) + + +def test_production_mints_once_and_matures_once_when_real_verified_shadow( + tmp_path: Path, actual_reward: tuple[Setup, FinalityEvidence, RewardFinalize] +) -> None: + # Given: actual two-execution trajectory with pinned production issuance. + s, evidence, reward = actual_reward + e = ledger( + s, + tmp_path / "ledger.db", + replay=verified_shadow, + settlement=lambda _: Settlement( + finality_hash=reward.finalize_hash, + closed_dispute_root=H, + release_beacon=20, + unresolved=False, + outcome="MATCH", + ), + ) + binding = reserve_actual_shadow(e, evidence) + shadow_reward = sign_actual_shadow(reward, binding) + evidence = replace(evidence, shadow=True) + # When + receipt = e.reward_finalize(shadow_reward, evidence) + assert e.reward_finalize(shadow_reward, evidence) == receipt + # Then: still pending until authorized maturity; no admission/live-model influence. + assert e.balances().reward_pending == 1_000_000 and e.balances().admission_locked == 0 + with pytest.raises(EscrowError): + e.mature(H, tuple(shadow_reward.origin_ids), 19) + first = e.mature(H, tuple(shadow_reward.origin_ids), 20) + assert e.mature(H, tuple(shadow_reward.origin_ids), 20) == first + with pytest.raises(EscrowError): + e.mature("22" * 32, tuple(shadow_reward.origin_ids), 20) + assert e.balances().available == 1_000_000 and e.balances().conserved() + + +def test_reward_maturity_rejects_when_dispute_open(tmp_path: Path, actual_reward) -> None: + # Given + s, evidence, reward = actual_reward + e = ledger( + s, + tmp_path / "ledger.db", + replay=verified_shadow, + settlement=lambda _: Settlement( + finality_hash=reward.finalize_hash, + closed_dispute_root=H, + release_beacon=20, + unresolved=True, + outcome="MATCH", + ), + ) + binding = reserve_actual_shadow(e, evidence) + shadow_reward = sign_actual_shadow(reward, binding) + e.reward_finalize(shadow_reward, replace(evidence, shadow=True)) + # When / Then + with pytest.raises(EscrowError): + e.mature(H, tuple(shadow_reward.origin_ids), 20) + assert e.balances().reward_pending == 1_000_000 + + +def test_reward_callbacks_share_atomic_mutation_snapshot(tmp_path: Path, actual_reward) -> None: + # Given: authenticated records live in the actual service connection. + s, evidence, reward = actual_reward + connection = sqlite3.connect(tmp_path / "ledger.db", isolation_level=None) + calls = [] + + def replay(facts: FinalityEvidence) -> str: + assert connection.in_transaction + calls.append("replay") + return verified_shadow(facts) + + e = ledger(s, connection, replay=replay) + + def owner(hotkey: str) -> str: + assert connection.in_transaction + calls.append("owner") + return COLD.ss58 + + def assignment(w: int, hotkey: str) -> tuple[int, ...]: + assert connection.in_transaction + calls.append("assignment") + return (0,) + + e.owner_of, e.assignment_of = owner, assignment + # When + e.reward_finalize(reward, evidence) + # Then: authority checks and journal/buckets share one SQLite snapshot. + assert calls == ["replay", "assignment", "owner"] + assert e.balances().reward_pending == 1_000_000 and e.balances().conserved() + + +def test_production_reward_rejects_when_policy_run_rebound(tmp_path: Path, actual_reward) -> None: + from dataclasses import replace + + # Given: different pinned total cap, real work rebound to that run would be required. + s, evidence, reward = actual_reward + policy = EconomicsPolicyV2.model_validate({**s.policy.body(), "max_total_issuance": 999999}) + body = s.manifest.body() + body["network"]["economics_policy_hash"] = digest(policy) + changed = replace(s, policy=policy, manifest=RunManifestV2.model_validate(body)) + e = ledger(changed, tmp_path / "ledger.db", replay=verified_shadow) + # When / Then: old-run authority cannot cross pinned economic policy. + with pytest.raises(EscrowError): + e.reward_finalize(reward, evidence) + assert e.balances().issued == 0 + + +@pytest.mark.parametrize( + "fault", + ["authority", "budget", "origin", "allocation", "audit", "signature", "cross_run", "maturity"], +) +def test_production_reward_rejects_when_invalid( + tmp_path: Path, actual_reward: tuple[Setup, FinalityEvidence, RewardFinalize], fault: str +) -> None: + from dataclasses import replace + + # Given + s, evidence, reward = actual_reward + e = ledger(s, tmp_path / "ledger.db", replay=verified_shadow) + data = reward.body() + if fault == "budget": + data["budget_units"] = 1_000_001 + if fault == "origin": + data["origin_ids"] = [H] + if fault == "allocation": + data["allocation_hash"] = H + if fault == "cross_run": + data["run_id"] = H + if fault == "maturity": + data["mature_at"] = 19 + if fault in ("audit", "signature"): + item = evidence.works[0] + v = ReplayVerdict.model_validate(item.verdict.body) + if fault == "audit": + v = ReplayVerdict.model_validate({**v.model_dump(mode="json"), "result": "MISMATCH"}) + env = EnvelopeV2.model_validate( + seal( + OTHER if fault == "signature" else AUDITOR, + "ReplayVerdict", + s.manifest.run_id(), + v, + 200, + ) + ) + evidence = replace(evidence, works=(replace(item, verdict=env),)) + reward = RewardFinalize.model_validate(data) + signer = OTHER if fault == "authority" else COORD + reward = RewardFinalize.model_validate( + {**reward.body(), "authority_sig": signer.sign(authority_message(reward)).hex()} + ) + # When / Then + with pytest.raises((EscrowError, ValueError)): + e.reward_finalize(reward, evidence) + assert e.balances().issued == 0 + + +def shadow_ledger(s: Setup, path: Path, final_hash: str) -> EscrowV2: + return ledger( + s, + path, + replay=verified_shadow, + settlement=lambda _: Settlement( + finality_hash=final_hash, + closed_dispute_root=H, + release_beacon=20, + unresolved=False, + outcome="MATCH", + ), + ) + + +def test_shadow_and_live_reward_domains_do_not_collide(tmp_path: Path, actual_reward) -> None: + # Given: the same genuine independently replayed w=0 work in separate signed domains. + s, evidence, live = actual_reward + e = shadow_ledger(s, tmp_path / "ledger.db", live.finalize_hash) + binding = reserve_actual_shadow(e, evidence) + shadow = sign_actual_shadow(live, binding) + original_live = live.model_dump_json() + live_signature_message = authority_message(live) + # When + shadow_receipt = e.reward_finalize(shadow, replace(evidence, shadow=True)) + live_receipt = e.reward_finalize(live, evidence) + # Then: two conserved issuances, distinct origins, unchanged live wire/signature domain. + assert shadow_receipt.operation_id != live_receipt.operation_id + assert set(shadow.origin_ids).isdisjoint(live.origin_ids) + assert e.balances().issued == 2_000_000 and e.balances().conserved() + assert live.model_dump_json() == original_live + assert authority_message(live) == live_signature_message + assert live_signature_message == ( + b"hypertrain/origin/2|RewardFinalize|" + + e.run_id.encode() + + b"|" + + sha256_hex(canonicalize(live.model_dump(mode="json", exclude={"authority_sig"}))).encode() + ) + assert set(RewardFinalize.model_fields) == { + "run_id", + "w", + "finalize_hash", + "tape_hash", + "verdict_root", + "allocation_hash", + "budget_units", + "origin_ids", + "mature_at", + "authority_sig", + } + assert e.db.execute("SELECT w FROM escrow_finalized").fetchone()[0] == 0 + assert e.db.execute("SELECT ordinal FROM escrow_shadow_finalized").fetchone()[0] == 0 + + +def test_shadow_and_live_share_pinned_issuance_cap(tmp_path: Path, actual_reward) -> None: + # Given: two original awards fill the genuine fixture cap, not an inserted balance. + s, evidence, live = actual_reward + e = shadow_ledger(s, tmp_path / "ledger.db", live.finalize_hash) + shadow = sign_actual_shadow(live, reserve_actual_shadow(e, evidence)) + e.reward_finalize(shadow, replace(evidence, shadow=True)) + e.reward_finalize(live, evidence) + # When / Then: another pending award cannot overreserve shared issuance. + with pytest.raises(EscrowError, match="TOTAL_ISSUANCE_BUDGET"): + reserve_actual_shadow(e, evidence, "22" * 32) + assert e.balances().issued == s.policy.max_total_issuance + assert e.db.execute("SELECT COUNT(*) FROM admission_reservations").fetchone()[0] == 1 + + +def test_live_origin_mapping_keeps_original_commit_body_hotkey( + tmp_path: Path, actual_reward +) -> None: + # Given: genuine HOT-signed work; original intake rejects a different envelope signer. + s, evidence, live = actual_reward + e = shadow_ledger(s, tmp_path / "ledger.db", live.finalize_hash) + item = evidence.works[0] + commit = EnvelopeV2.model_validate( + seal( + OTHER, + "CommitV2", + e.run_id, + item.commit.body, + 200, + ) + ) + facts = replace(evidence, works=(replace(item, commit=commit),)) + before = e.balances() + events = e.db.execute("SELECT COUNT(*) FROM escrow_events").fetchone()[0] + with pytest.raises(UnauthorizedSigner, match="hotkey differs from signer"): + e.reward_finalize(live, facts) + assert e.balances() == before + assert e.db.execute("SELECT COUNT(*) FROM escrow_events").fetchone()[0] == events + # When + receipt = e.reward_finalize(live, evidence) + # Then: the valid origin maps HOT work to its authorized COLD economic owner. + row = e.db.execute("SELECT origin,hotkey FROM escrow_origins").fetchone() + assert (row["origin"], row["hotkey"]) == (live.origin_ids[0], HOT.ss58) + owners = e.db.execute( + "SELECT DISTINCT owner FROM escrow_units WHERE origin=?", (live.origin_ids[0],) + ).fetchall() + assert [owner["owner"] for owner in owners] == [COLD.ss58] + assert receipt.event_hash + assert e.balances().issued == 1_000_000 and e.balances().conserved() + + +def test_live_issuance_preserves_pending_shadow_headroom(tmp_path: Path, actual_reward) -> None: + # Given: original retained work earned U; the next trusted-intake request reserves the last U. + # Distinct admission IDs exercise the internal ledger seam, not a fabricated public history. + s, evidence, live = actual_reward + e = shadow_ledger(s, tmp_path / "ledger.db", live.finalize_hash) + first = reserve_actual_shadow(e, evidence) + e.reward_finalize(sign_actual_shadow(live, first), replace(evidence, shadow=True)) + pending = reserve_actual_shadow(e, evidence, "22" * 32) + assert pending.shadow_ordinal == 1 + assert e.balances().issued == 1_000_000 + # When / Then: original live w=0 validates, but cannot spend the reserved issuance headroom. + with pytest.raises(EscrowError, match="TOTAL_ISSUANCE_BUDGET"): + e.reward_finalize(live, evidence) + assert e.balances().issued == 1_000_000 and e.balances().conserved() + assert e.db.execute("SELECT COUNT(*) FROM escrow_finalized").fetchone()[0] == 0 + assert e.db.execute("SELECT COUNT(*) FROM escrow_events").fetchone()[0] == 1 + assert reserve_actual_shadow(e, evidence, "22" * 32) == pending + + +@pytest.mark.parametrize( + "fault", + [ + "missing_reservation", + "signature", + "cross_type_signature", + "ordinal", + "domain", + "binding", + "allocation", + "budget", + "early_vesting", + "owner", + "audit", + "assignment", + ], +) +def test_shadow_reward_rejects_unbound_authority(tmp_path: Path, actual_reward, fault: str) -> None: + # Given: valid retained work, one reserved ordinal, independently signed shadow body. + s, evidence, live = actual_reward + e = shadow_ledger(s, tmp_path / "ledger.db", live.finalize_hash) + binding = reserve_actual_shadow(e, evidence) + reward = sign_actual_shadow(live, binding) + facts = replace(evidence, shadow=True) + if fault == "missing_reservation": + e = shadow_ledger(s, tmp_path / "other.db", live.finalize_hash) + body = reward.body() + if fault == "ordinal": + body["shadow_ordinal"] = 1 + if fault == "binding": + body["reservation_hash"] = "22" * 32 + if fault == "allocation": + body["origin_ids"] = [H] + if fault == "budget": + body["budget_units"] = 1_000_001 + if fault == "early_vesting": + body["mature_at"] = 19 + reward = ShadowRewardFinalizeV1.model_validate(body) + reward = reward.model_copy( + update={ + "authority_sig": (OTHER if fault == "signature" else COORD) + .sign(authority_message(reward)) + .hex() + } + ) + if fault == "cross_type_signature": + reward = reward.model_copy(update={"authority_sig": live.authority_sig}) + if fault == "domain": + facts = evidence + if fault == "owner": + facts = replace(facts, works=(replace(facts.works[0], owner=OTHER.ss58),)) + if fault == "audit": + item = facts.works[0] + verdict = ReplayVerdict.model_validate(item.verdict.body) + envelope = EnvelopeV2.model_validate( + seal( + HOT, + "ReplayVerdict", + e.run_id, + verdict, + 200, + ) + ) + facts = replace(facts, works=(replace(item, verdict=envelope),)) + if fault == "assignment": + e.shadow_assignment_of = lambda binding: (1,) + # When / Then + with pytest.raises((EscrowError, ValueError)): + e.reward_finalize(reward, facts) + assert e.balances().issued == 0 and e.balances().conserved() + assert e.db.execute("SELECT COUNT(*) FROM escrow_events").fetchone()[0] == 0 + + +def test_shadow_reservation_survives_crash_and_conflicting_request( + tmp_path: Path, actual_reward +) -> None: + # Given: durable reservation returned before signing; process then disappears. + s, evidence, live = actual_reward + path = tmp_path / "ledger.db" + e = shadow_ledger(s, path, live.finalize_hash) + binding = reserve_actual_shadow(e, evidence) + e.db.close() + # When + e = shadow_ledger(s, path, live.finalize_hash) + same = reserve_actual_shadow(e, evidence) + # Then: recover original binding without reassignment or issuance. + assert same == binding and same.shadow_ordinal == 0 + with pytest.raises(EscrowError, match="SHADOW_RESERVATION_PENDING"): + reserve_actual_shadow(e, evidence, "22" * 32) + request = ShadowReservationRequestV1.model_validate( + {k: v for k, v in binding.body().items() if k != "shadow_ordinal"} + ) + with pytest.raises(EscrowError, match="SHADOW_RESERVATION_CONFLICT"): + e.reserve_shadow( + request.model_copy(update={"custody_hash": "33" * 32}), + replace(original_trial_custody(evidence), custody_hash="33" * 32), + ) + assert e.balances().issued == 0 + + +@pytest.mark.parametrize("accepted", [False, True]) +def test_shadow_reservation_recovers_before_mutable_authority( + tmp_path: Path, actual_reward, accepted: bool +) -> None: + # Given: original signed bodies and custody already own an immutable reservation. + s, evidence, live = actual_reward + e = shadow_ledger(s, tmp_path / "ledger.db", live.finalize_hash) + binding = reserve_actual_shadow(e, evidence) + if accepted: + e.reward_finalize(sign_actual_shadow(live, binding), replace(evidence, shadow=True)) + + def unavailable_owner(hotkey: str) -> str: + raise EscrowError("OWNER_CHANGED_AFTER_RESERVATION") + + def unavailable_replay(facts: FinalityEvidence) -> str: + raise EscrowError("ACCEPTANCE_REPLAY_UNAVAILABLE") + + e.owner_of, e.replay_finality = unavailable_owner, unavailable_replay + # When + recovered = reserve_actual_shadow(e, evidence) + # Then: immutable authenticated replay ignores new-issuance predicates, not signatures. + assert recovered == binding + forged = replace( + original_trial_custody(evidence), + commit=EnvelopeV2.model_validate( + seal(OTHER, "CommitV2", e.run_id, evidence.works[0].commit.body, 200) + ), + ) + request = ShadowReservationRequestV1.model_validate( + {k: v for k, v in binding.body().items() if k != "shadow_ordinal"} + ) + with pytest.raises(ValueError): + e.reserve_shadow(request, forged) + + +def test_shadow_reservation_needs_no_acceptance_replay(tmp_path: Path, actual_reward) -> None: + # Given: only original independently executed trial custody exists before reservation. + s, evidence, live = actual_reward + e = shadow_ledger(s, tmp_path / "ledger.db", live.finalize_hash) + + def unavailable(facts: FinalityEvidence) -> str: + raise EscrowError("NO_ACCEPTED_AUDITOR_RECEIPT_YET") + + e.replay_finality = unavailable + # When + binding = reserve_actual_shadow(e, evidence) + # Then: ordinal can be signed now; acceptance must still invoke real independent replay. + assert binding.shadow_ordinal == 0 and not e.db.in_transaction + with pytest.raises(EscrowError, match="NO_ACCEPTED_AUDITOR_RECEIPT_YET"): + e.reward_finalize(sign_actual_shadow(live, binding), replace(evidence, shadow=True)) + assert e.balances().issued == 0 + + +def test_shadow_reservation_requires_durable_outer_commit(tmp_path: Path, actual_reward) -> None: + # Given + s, evidence, live = actual_reward + e = shadow_ledger(s, tmp_path / "ledger.db", live.finalize_hash) + # When / Then: never return an ordinal whose surrounding transaction can still roll back. + with e.tx(): + with pytest.raises(EscrowError, match="SHADOW_RESERVATION_REQUIRES_COMMIT"): + reserve_actual_shadow(e, evidence) + assert e.db.execute("SELECT COUNT(*) FROM admission_reservations").fetchone()[0] == 0 + + +def test_shadow_acceptance_rolls_back_with_outer_transaction(tmp_path: Path, actual_reward) -> None: + # Given: signed reward after durable reservation; acceptance shares caller transaction. + s, evidence, live = actual_reward + e = shadow_ledger(s, tmp_path / "ledger.db", live.finalize_hash) + binding = reserve_actual_shadow(e, evidence) + reward = sign_actual_shadow(live, binding) + # When: caller's evidence/settlement acceptance fails after ledger insertion. + with pytest.raises(EscrowError, match="CALLER_ROLLBACK"): + with e.tx(): + e.reward_finalize(reward, replace(evidence, shadow=True)) + raise EscrowError("CALLER_ROLLBACK") + # Then: original reservation survives, no funds/event/finality accepted. + assert e.balances().issued == 0 + assert e.db.execute("SELECT COUNT(*) FROM escrow_events").fetchone()[0] == 0 + assert e.db.execute("SELECT COUNT(*) FROM escrow_shadow_finalized").fetchone()[0] == 0 + assert reserve_actual_shadow(e, evidence) == binding + receipt = e.reward_finalize(reward, replace(evidence, shadow=True)) + assert receipt.event_hash and e.balances().issued == 1_000_000 + + +def test_shadow_accepted_reply_recovers_after_restart(tmp_path: Path, actual_reward) -> None: + # Given: acceptance committed, original HTTP reply lost. + s, evidence, live = actual_reward + path = tmp_path / "ledger.db" + e = shadow_ledger(s, path, live.finalize_hash) + binding = reserve_actual_shadow(e, evidence) + reward = sign_actual_shadow(live, binding) + receipt = e.reward_finalize(reward, replace(evidence, shadow=True)) + e.db.close() + e = shadow_ledger(s, path, live.finalize_hash) + # When + replay = e.reward_finalize(reward, replace(evidence, shadow=True)) + # Then + assert replay == receipt and e.balances().issued == 1_000_000 + changed = reward.model_copy(update={"tape_hash": H}) + changed = changed.model_copy( + update={"authority_sig": COORD.sign(authority_message(changed)).hex()} + ) + with pytest.raises(EscrowError, match="OPERATION_CONFLICT"): + e.reward_finalize(changed, replace(evidence, shadow=True)) + assert e.db.execute("SELECT COUNT(*) FROM escrow_events").fetchone()[0] == 1 + + +@pytest.mark.parametrize("fault", ["too_early", "unresolved", "fraud", "finality", "release"]) +def test_shadow_maturity_uses_shadow_finality_and_disputes( + tmp_path: Path, actual_reward, fault: str +) -> None: + # Given: shadow issuance has no live finalized w entry. + s, evidence, live = actual_reward + e = shadow_ledger(s, tmp_path / "ledger.db", live.finalize_hash) + reward = sign_actual_shadow(live, reserve_actual_shadow(e, evidence)) + e.reward_finalize(reward, replace(evidence, shadow=True)) + e.shadow_settlement = lambda _: Settlement( + finality_hash=H if fault == "finality" else live.finalize_hash, + closed_dispute_root=H, + release_beacon=21 if fault == "release" else 20, + unresolved=fault == "unresolved", + outcome="FRAUD" if fault == "fraud" else "MATCH", + ) + # When / Then + with pytest.raises(EscrowError): + e.mature(H, tuple(reward.origin_ids), 19 if fault == "too_early" else 20) + assert e.balances().available == 0 and e.balances().reward_pending == 1_000_000 + + +@pytest.mark.parametrize("number", [True, -1, 12, 1.0, "0"]) +def test_shadow_wire_rejects_non_strict_or_out_of_range_ordinal(actual_reward, number) -> None: + # Given: original live body with explicit signed shadow domain fields. + _, _, live = actual_reward + body = {**live.body(), "shadow": True, "shadow_ordinal": number, "reservation_hash": H} + # When / Then + with pytest.raises(ValidationError): + ShadowRewardFinalizeV1.model_validate(body) + + +@pytest.mark.parametrize("domain", [1, "true", False, None]) +def test_shadow_wire_requires_actual_boolean_domain(actual_reward, domain) -> None: + # Given + _, _, live = actual_reward + # When / Then + with pytest.raises(ValidationError): + ShadowRewardFinalizeV1.model_validate( + { + **live.body(), + "shadow": domain, + "shadow_ordinal": 0, + "reservation_hash": H, + } + ) + + +def test_shadow_replay_wire_rejects_unbound_fields(actual_reward) -> None: + # Given: complete strict custody shape, not a claimed successful replay. + s, _, _ = actual_reward + body = {name: H for name in ShadowReplayReceiptV1.model_fields} + body.update( + run_id=s.manifest.run_id(), + hotkey=HOT.ss58, + coldkey=COLD.ss58, + trial_epoch=0, + shadow_ordinal=0, + completed_beacon=10, + ) + accepted = ShadowReplayReceiptV1.model_validate(body) + # When / Then + with pytest.raises(ValidationError): + ShadowReplayReceiptV1.model_validate({**accepted.body(), "completed_beacon": True}) + with pytest.raises(ValidationError): + ShadowReplayReceiptV1.model_validate({**accepted.body(), "success": True}) diff --git a/tests/miner/test_capacity_runtime_v2.py b/tests/miner/test_capacity_runtime_v2.py new file mode 100644 index 0000000..a9c3021 --- /dev/null +++ b/tests/miner/test_capacity_runtime_v2.py @@ -0,0 +1,211 @@ +"""Real kernel-backed small stdlib children. No Torch or exact1GiB workload.""" + +from __future__ import annotations + +import json +import os +import select +import signal +import socket +import sys +import threading +import time + +import pytest + +from hypertrain.miner import island_launch as launch + + +@pytest.fixture +def small(monkeypatch, tmp_path): + monkeypatch.setattr(launch, "_CAPACITY_MEMORY", 64 << 20) + monkeypatch.setattr(launch, "_CAPACITY_CPU_PERCENT", 50) + return launch.CapacityAttempt("11" * 32, "22" * 32, tmp_path / "runtime.lock", lambda: True) + + +@pytest.mark.parametrize("mode", ["success", "oom", "cancel", "deadline"]) +def test_actual_small_kernel_attempt_and_cleanup(tmp_path, small, mode): + """Subscribe readiness before triggering real bounded command and await pidfd exit.""" + cancel = threading.Event() + events = [] + path = str(tmp_path / "ready.sock") + deadline = int(time.time()) + (4 if mode == "deadline" else 20) + script = f"""import os,socket,signal +d=os.fork() +if d==0: + signal.pause();os._exit(0) +s=socket.socket(socket.AF_UNIX);s.connect({path!r}) +s.sendall((str(os.getpid())+','+str(d)).encode()+b'\\n') +assert s.recv(1)==b'G' +if {mode!r}=='oom': + x=bytearray(96*1024*1024) +elif {mode!r}=='success': + os.kill(d,signal.SIGKILL);os.waitpid(d,0) +else: + signal.pause() +""" + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as listener: + listener.bind(path) + listener.listen(1) + listener.settimeout(15) + + def ready(): + connection, _ = listener.accept() + with connection: + connection.settimeout(10) + raw = connection.recv(128).decode().strip() + descriptors = [os.pidfd_open(int(pid)) for pid in raw.split(",")] + try: + connection.sendall(b"G") + if mode == "cancel": + cancel.set() + for fd in descriptors: + assert select.select([fd], [], [], 15)[0] + events.append("both-exited") + finally: + for fd in descriptors: + os.close(fd) + + watcher = threading.Thread(target=ready) + watcher.start() + runtime = tmp_path / "attempt" + try: + if mode == "success": + launch.run_capacity_argv( + [sys.executable, "-c", script], + runtime, + deadline, + small, + env=dict(os.environ), + cancel=cancel, + ) + else: + with pytest.raises(launch.IslandFailure, match="execution failed"): + launch.run_capacity_argv( + [sys.executable, "-c", script], + runtime, + deadline, + small, + env=dict(os.environ), + cancel=cancel, + ) + finally: + watcher.join(timeout=20) + assert not watcher.is_alive() and events == ["both-exited"] + observed = json.loads((runtime / "capacity-observed.json").read_text()) + assert observed["memory.max"] == str(64 << 20) + assert observed["cpu.max"] == "50000 100000" + assert observed["memory.oom.group"] == "1" and observed["memory.swap.max"] == "0" + assert observed["affinity"] == [0, 1, 2, 3] and observed["nice"] == 19 + result = json.loads((runtime / "capacity-result.json").read_text()) + if mode == "oom": + assert "Result=oom-kill" in result["status"] + counters = dict( + line.split() + for line in result["status"].splitlines() + if line.startswith(("oom_kill ", "oom_group_kill ")) + ) + assert int(counters["oom_kill"]) >= 1 + assert int(counters["oom_group_kill"]) == 1 + assert (runtime / "capacity-cleaned.json").is_file() + with pytest.raises(launch.IslandFailure, match="already recorded"): + launch.run_capacity_argv( + [sys.executable, "-c", "raise SystemExit(99)"], + runtime, + int(time.time()) + 20, + small, + env=dict(os.environ), + ) + + +def test_charge_denial_launches_nothing(tmp_path): + attempt = launch.CapacityAttempt("33" * 32, "44" * 32, tmp_path / "lock", lambda: False) + with pytest.raises(launch.IslandFailure, match="already charged"): + launch.run_capacity_argv( + [sys.executable, "-c", "raise SystemExit(99)"], + tmp_path / "attempt", + int(time.time()) + 10, + attempt, + env=dict(os.environ), + ) + assert not (tmp_path / "attempt/capacity-attempt.json").exists() + + +def test_missing_manager_permission_has_no_unbounded_fallback(tmp_path, small, monkeypatch): + monkeypatch.setenv("PATH", "") + with pytest.raises(launch.IslandFailure, match="manager unavailable|cleanup incomplete"): + launch.run_capacity_argv( + [sys.executable, "-c", "raise SystemExit(99)"], + tmp_path / "attempt", + int(time.time()) + 10, + small, + env=dict(os.environ), + ) + assert (tmp_path / "attempt/capacity-attempt.json").exists() + assert not (tmp_path / "attempt/capacity-observed.json").exists() + + +def test_real_runner_parent_death_retains_absolute_manager_timer(tmp_path, small): + """Fork only a stdlib waiting controller; manager timer survives its SIGKILL.""" + path = str(tmp_path / "dead.sock") + runtime = tmp_path / "attempt" + deadline = int(time.time()) + 6 + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as listener: + listener.bind(path) + listener.listen(1) + listener.settimeout(15) + parent = os.fork() + if parent == 0: + try: + code = ( + "import os,socket,signal;s=socket.socket(socket.AF_UNIX);" + f"s.connect({path!r});s.sendall(str(os.getpid()).encode());" + "s.recv(1);signal.pause()" + ) + launch.run_capacity_argv( + [sys.executable, "-c", code], runtime, deadline, small, env=dict(os.environ) + ) + finally: + os._exit(1) + try: + connection, _ = listener.accept() + with connection: + connection.settimeout(15) + child = int(connection.recv(128)) + fd = os.pidfd_open(child) + try: + connection.sendall(b"G") + os.kill(parent, signal.SIGKILL) + os.waitpid(parent, 0) + assert not select.select([fd], [], [], 0)[0] + assert connection.recv(1) == b"" + assert select.select([fd], [], [], 5)[0] + assert time.time() >= deadline + finally: + os.close(fd) + record = json.loads((runtime / "capacity-attempt.json").read_text()) + status = launch._manager( + "systemctl", "show", record["timer_service"], "-p", "Result", "-p", "ExecMainStatus" + ) + assert "Result=success" in status and "ExecMainStatus=0" in status + with pytest.raises(launch.IslandFailure, match="already recorded"): + launch.run_capacity_argv( + [sys.executable, "-c", "pass"], + runtime, + int(time.time()) + 20, + small, + env=dict(os.environ), + ) + finally: + if (runtime / "capacity-attempt.json").exists(): + record = json.loads((runtime / "capacity-attempt.json").read_text()) + for key in ("unit", "timer", "timer_service"): + launch._manager("systemctl", "stop", record[key], check=False) + launch._manager( + "systemctl", + "reset-failed", + record["unit"], + record["timer"], + record["timer_service"], + check=False, + ) diff --git a/tests/miner/test_client_network_v2.py b/tests/miner/test_client_network_v2.py new file mode 100644 index 0000000..35dcb0b --- /dev/null +++ b/tests/miner/test_client_network_v2.py @@ -0,0 +1,124 @@ +"""Explicit v2 client authority and manifest-N launcher integration.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import httpx +import pytest + +from hypertrain.miner.core import MinerConfig, MinerError, NetworkMiner +from hypertrain.protocol.envelope_v2 import seal +from hypertrain.protocol.keys import Keypair +from hypertrain.protocol.messages_v2 import RunManifestV2 + + +def test_client_requires_owner_authority_when_manifest_signed_by_other(tmp_path: Path): + # Given: valid signature, wrong pinned owner. + import importlib.util + import sys + + spec = importlib.util.spec_from_file_location( + "client_fixture", Path(__file__).parents[1] / "ledger/test_escrow_v2.py" + ) + fixture = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = fixture + spec.loader.exec_module(fixture) + manifest = fixture.setup().manifest + key = Keypair(b"a" * 32) + keyfile = tmp_path / "miner.key" + keyfile.write_bytes(b"a" * 32) + keyfile.chmod(0o600) + cfg = MinerConfig( + "https://service.test", + keyfile, + tmp_path, + "unused", + manifest.training.reference_spec.image_digest, + run_id=manifest.run_id(), + owner_hotkey=fixture.COORD.ss58, + ) + transport = httpx.MockTransport( + lambda _: httpx.Response( + 200, + json={ + "manifest_envelope": seal(key, "RunManifestV2", manifest.run_id(), manifest, 1000) + }, + ) + ) + # When / Then + with httpx.Client(transport=transport) as client, pytest.raises(MinerError, match="authority"): + NetworkMiner(cfg, client) + + +def test_two_rank_manifest_launches_real_runtime(tmp_path: Path): + # Given: real staged signed dataset, strict two-rank wrapper. + import importlib.util + import sys + + spec = importlib.util.spec_from_file_location( + "rank_fixture", Path(__file__).parents[1] / "ledger/test_escrow_v2.py" + ) + fixture = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = fixture + spec.loader.exec_module(fixture) + s = fixture.setup() + body = s.manifest.body() + body["training"]["reference_spec"]["layout"].update(n_gpus=2, dp_size=2, ep_size=1, zero1=True) + wrapper = RunManifestV2.model_validate(body) + s = fixture.Setup(wrapper, s.policy, s.admission_policy, s.rows, s.tree) + job = fixture.stage(s, tmp_path / "attempt", (0, 1), 0) + (tmp_path / "attempt/job.json").write_text(job.model_dump_json()) + keyfile = tmp_path / "key" + keyfile.write_bytes(b"b" * 32) + keyfile.chmod(0o600) + cfg = MinerConfig( + "https://service.test", + keyfile, + tmp_path / "work", + "unused", + wrapper.training.reference_spec.image_digest, + run_id=wrapper.run_id(), + owner_hotkey=fixture.COORD.ss58, + ) + transport = httpx.MockTransport( + lambda _: httpx.Response( + 200, + json={ + "manifest_envelope": seal( + fixture.COORD, "RunManifestV2", wrapper.run_id(), wrapper, 1000 + ) + }, + ) + ) + with httpx.Client(transport=transport) as client: + network = NetworkMiner(cfg, client) + # When + result = network.launch(tmp_path / "attempt/job.json") + # Then: two real rank artifacts, one identical island commitment. + summary0 = json.loads((tmp_path / "attempt/published/rank-0/summary.json").read_bytes()) + summary1 = json.loads((tmp_path / "attempt/published/rank-1/summary.json").read_bytes()) + assert summary0["commitments"] == summary1["commitments"] + assert Path(result["state"]).exists() + + +def test_run_v2_cli_dispatches_exact_round_argument(monkeypatch, capsys, tmp_path): + from hypertrain.miner import cli + + # Parser-only contract; actual NetworkMiner.run_round has separate real HTTP/TLS proof. + calls = [] + + class ParserActor: + def __init__(self, cfg, client): + pass + + def run_round(self, w): + calls.append(w) + return "UPLOADED" + + monkeypatch.setattr(cli, "NetworkMiner", ParserActor) + monkeypatch.setattr(cli.MinerConfig, "load", lambda _: object()) + assert cli.main(["run-v2", "--round", "7"]) == 0 + assert calls == [7] + assert json.loads(capsys.readouterr().out) == {"status": "UPLOADED"} diff --git a/tests/miner/test_dispute_watch_v2.py b/tests/miner/test_dispute_watch_v2.py new file mode 100644 index 0000000..4883bdb --- /dev/null +++ b/tests/miner/test_dispute_watch_v2.py @@ -0,0 +1,468 @@ +"""Independent watch over real service events, restart outbox and published StateServe.""" + +from __future__ import annotations + +import importlib.util +import sys +from dataclasses import replace +from pathlib import Path + +import pytest + +from hypertrain.auditor.island_bisect import IslandParty +from hypertrain.challenge.disputes_v2 import ( + DisputeError, + EventAck, + WatchEvent, + authentic, + signed_message, +) +from hypertrain.data.store import LocalFSStore +from hypertrain.gpu_ops.journal import flock +from hypertrain.miner.dispute_watch import DisputeWatch, PublishedStates +from hypertrain.miner.island_launch import launch_island +from hypertrain.protocol.envelope_v2 import parse_envelope +from hypertrain.protocol.hashing import MerkleTree +from hypertrain.protocol.messages import StateServe +from hypertrain.trainer.island import emulate + + +def helpers(name: str): + spec = importlib.util.spec_from_file_location( + "l5_watch_" + name, Path(__file__).parents[1] / name + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +class Transport: + def __init__(self, service, now: int = 2) -> None: + self.service, self.now = service, now + self.crash = False + self.accepted: list[bytes] = [] + self.serves: list[StateServe] = [] + + def events(self, party: str, cursor: int, *, timeout: float) -> list[WatchEvent]: + return self.service.events(party, cursor, timeout=timeout) + + def acknowledge(self, ack: EventAck) -> None: + self.service.acknowledge(ack, now=self.now) + + def submit(self, envelope: bytes) -> None: + env = parse_envelope(envelope) + if env.type == "BisectV2": + self.service.bisect(envelope, now=self.now) + else: + self.serves.append(StateServe.model_validate(env.body)) + self.accepted.append(envelope) + if self.crash: + self.crash = False + raise ConnectionError("accepted before connection lost") + + +def setup_watch(tmp_path: Path): + challenge = helpers("challenge/test_disputes_v2.py") + service, f, ref, _, raw = challenge.service(tmp_path) + turn = service.open(raw, now=1) + run = helpers("auditor/test_island_bisect_v2.py").execution() + traced = emulate(2, run.trace) + party = IslandParty(f.HOT.ss58, [c for _, c in traced], J=1) + transport = Transport(service) + setup = replace(f.setup(), manifest=service.escrow.manifest) + job = f.stage( + setup, + tmp_path / "job-input", + tuple(range(setup.manifest.training.batch_samples())), + turn.contest.w, + ) + + def state(event: WatchEvent) -> StateServe: + raise AssertionError("state not requested") + + def watch(): + return DisputeWatch( + tmp_path / "watch", + f.HOT, + transport, + run_id=service.run_id, + coordinator=f.COORD.ss58, + party=lambda _: party, + state_serve=state, + beacon=lambda: transport.now, + job=job, + ) + + return service, f, ref, turn, party, transport, watch + + +def test_cursor_exactly_once_when_restart_after_accepted_send(tmp_path: Path) -> None: + # Given + service, _, _, turn, _, transport, factory = setup_watch(tmp_path) + watch = factory() + transport.crash = True + # When + with pytest.raises(ConnectionError): + watch.run_once(timeout=0) + watch.close() + restored = factory() + assert restored.run_once(timeout=0) == 1 + # Then + assert transport.accepted[0] == transport.accepted[1] + assert service.get(turn.dispute_id).seq == 1 + assert service.db.execute("SELECT COUNT(*) FROM dispute_entries_v2").fetchone()[0] == 1 + assert restored.cursor == 1 and restored.run_once(timeout=0) == 0 + restored.close() + + +def test_watch_replies_when_layer_and_op_turns_arrive(tmp_path: Path) -> None: + # Given + service, f, _, turn, party, transport, factory = setup_watch(tmp_path) + watch = factory() + challenge = helpers("challenge/test_disputes_v2.py") + service.span = lambda c, level, ctx: party.span(level, ctx) + # When: actual hashes from watch, first differing remote endpoint at every span. + watch.run_once(timeout=0) + for level in ("layer", "op"): + for _ in range(20): + current = service.get(turn.dispute_id) + first = service.db.execute( + "SELECT body FROM dispute_entries_v2 WHERE id=? AND seq=?", + ( + turn.dispute_id, + current.seq - 1, + ), + ).fetchone()[0] + from hypertrain.protocol.messages_v2 import BisectV2 + + claim = BisectV2.model_validate_json(first) + challenge.reply(service, f.AUDITOR, [claim.hashes[0], "aa" * 32, "bb" * 32]) + current = service.get(turn.dispute_id) + watch.run_once(timeout=0) + if current.level == level: + break + else: + raise AssertionError("bounded transcript did not progress") + # Then + levels = [parse_envelope(raw).body["level"] for raw in transport.accepted] + assert list(dict.fromkeys(levels)) == ["step", "layer", "op"] + watch.close() + + +def test_watch_lock_when_second_process_owns_directory(tmp_path: Path) -> None: + # Given + _, _, _, _, _, _, factory = setup_watch(tmp_path) + watch = factory() + # When / Then + with flock(watch.directory / "watch.lock"), pytest.raises(BlockingIOError): + watch.run_once(timeout=0) + assert watch.cursor == 0 + watch.close() + + +def test_event_rejects_when_coordinator_signature_changed(tmp_path: Path) -> None: + # Given + service, f, _, _, _, transport, factory = setup_watch(tmp_path) + event = service.events(f.HOT.ss58, 0)[0] + watch = factory() + transport.events = lambda *args, **kwargs: [event.model_copy(update={"sig": "00" * 64})] + # When / Then + with pytest.raises(DisputeError, match="AUTHORITY"): + watch.run_once(timeout=0) + assert watch.cursor == 0 and not transport.accepted + watch.close() + + +@pytest.mark.parametrize("kind,wrong_round", [("turn", True), ("state", True), ("turn", False)]) +def test_watch_rejects_signed_same_run_wrong_round(tmp_path: Path, monkeypatch, kind, wrong_round): + """Authentic event; no emulation, trace replay or training needed for early rejection.""" + challenge = helpers("challenge/test_disputes_v2.py") + service, f, _, _, raw = challenge.service(tmp_path) + turn = service.open(raw, now=1) + event = service.events(f.HOT.ss58, 0, timeout=0)[0] + wrong = event.model_copy( + update={ + "kind": kind, + "checkpoint": 1 if kind == "state" else None, + "turn": event.turn.model_copy( + update={ + "contest": turn.contest.model_copy( + update={"w": turn.contest.w + int(wrong_round)} + ) + } + ), + "sig": "", + } + ) + wrong = wrong.model_copy(update={"sig": f.COORD.sign(signed_message(wrong)).hex()}) + assert authentic(wrong) + transport = Transport(service) + calls = [] + monkeypatch.setattr(transport, "events", lambda *a, **kw: [wrong]) + monkeypatch.setattr(transport, "submit", lambda raw: calls.append("send")) + monkeypatch.setattr(transport, "acknowledge", lambda raw: calls.append("ack")) + + def party(dispute): + from types import SimpleNamespace + + calls.append("trace") + return SimpleNamespace(hotkey=f.HOT.ss58, hashes=lambda *a: ["11" * 32] * 3) + + setup = replace(f.setup(), manifest=service.escrow.manifest) + watch = DisputeWatch( + tmp_path / "watch", + f.HOT, + transport, + run_id=service.run_id, + coordinator=f.COORD.ss58, + party=party, + state_serve=lambda event: calls.append("state"), + beacon=lambda: transport.now, + job=f.stage( + setup, + tmp_path / "job-input", + tuple(range(setup.manifest.training.batch_samples())), + turn.contest.w, + ), + ) + try: + if wrong_round: + with pytest.raises(DisputeError, match="WATCH_PUBLICATION_CONTEXT"): + watch.run_once(timeout=0) + assert calls == [] and watch.cursor == 0 + assert watch.db.execute("SELECT COUNT(*) FROM watch_outbox").fetchone()[0] == 0 + else: + assert watch.run_once(timeout=0) == 1 + assert calls == ["trace", "ack", "send"] and watch.cursor == event.cursor + finally: + watch.close() + + +def test_state_serve_when_real_published_checkpoint_requested(tmp_path: Path) -> None: + # Given + challenge = helpers("challenge/test_disputes_v2.py") + service, f, _, _, raw = challenge.service(tmp_path) + launch = helpers("miner/test_island_launch_v2.py") + job = launch.staged_job(tmp_path / "runtime", 2, 2) + artifacts = launch_island(job, tmp_path / "runtime", backend="cpu", trace=True) + objects = LocalFSStore(tmp_path / "objects") + states = PublishedStates(job, artifacts.directory, objects) + party = IslandParty.published(f.HOT.ss58, job, artifacts.directory) + turn = service.open(raw, now=1) + event = service.state_request(turn.dispute_id, 1, now=2) + # Event uses the runtime's actual job identity; production L0 pins the same run/w. + event = event.model_copy( + update={ + "run_id": job.run_id, + "turn": event.turn.model_copy( + update={ + "contest": event.turn.contest.model_copy(update={"w": job.w}), + } + ), + } + ) + # When + serve = states.serve(event) + # Then + assert serve.tensor_root == party.state_root(1) + assert serve.t == states.leaves[1].t + assert MerkleTree.verify( + bytes.fromhex(states.leaves[1].digest()), + 1, + [bytes.fromhex(h) for h in serve.merkle_proof_leaf_in_leaves_root], + states.tree.root, + len(states.leaves), + ) + assert serve.uri.startswith("file:") + assert objects.get(Path(serve.uri).name) == party.state_blob(1) + bad = event.model_copy(update={"checkpoint": 9999}) + with pytest.raises(DisputeError): + states.serve(bad) + + +def test_public_watch_layer_after_socket_long_poll(tmp_path: Path) -> None: + """Actual CLI/socket transport, canonical captured trace, subscribed signed transition.""" + import contextlib + import io + import json + import sqlite3 + import threading + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + + from hypertrain.miner.cli import main + from hypertrain.protocol.envelope_v2 import seal + from hypertrain.protocol.jcs import canonicalize + from hypertrain.protocol.keys import decode_hotkey, verify + from hypertrain.protocol.messages_v2 import BisectV2 + + challenge = helpers("challenge/test_disputes_v2.py") + service, f, _, _, raw = challenge.service(tmp_path) + setup = replace(f.setup(), manifest=service.escrow.manifest) + directory = tmp_path / "publication" + job = f.stage(setup, directory, tuple(range(setup.manifest.training.batch_samples())), 0) + job_path = directory / "job.json" + job_path.write_bytes(canonicalize(job.body())) + artifacts = launch_island(job, directory, backend="cpu", trace=True) + party = IslandParty.published(f.HOT.ss58, job, artifacts.directory) + states = PublishedStates(job, artifacts.directory, LocalFSStore(tmp_path / "objects")) + service.span = lambda contest, level, ctx: party.span(level, ctx) + turn = service.open(raw, now=1) + work = tmp_path / "client" + watch_path = work / service.run_id / f.HOT.ss58 / "watch" + initial = DisputeWatch( + watch_path, + f.HOT, + Transport(service), + run_id=service.run_id, + coordinator=f.COORD.ss58, + party=lambda _: party, + state_serve=states.serve, + beacon=lambda: 2, + job=job, + ) + try: + assert initial.run_once(timeout=0) == 1 + first_cursor = initial.cursor + finally: + initial.close() + step = BisectV2.model_validate_json( + service.db.execute("SELECT body FROM dispute_entries_v2 WHERE seq=0").fetchone()[0] + ) + waiting, completed = threading.Event(), threading.Event() + + class SubscribedCondition(threading.Condition): + def wait(self, timeout=None): + waiting.set() + return super().wait(timeout) + + service.condition = SubscribedCondition() + envelope = seal(f.COORD, "RunManifestV2", service.run_id, job.manifest, 100000) + run_url = "/v2/runs/" + service.run_id + requests = [] + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def response(self, body, status=200): + encoded = json.dumps(body).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) + + def do_GET(self): + from urllib.parse import parse_qs, urlsplit + + url = urlsplit(self.path) + requests.append(url.path) + if url.path == run_url: + self.response({"manifest_envelope": envelope, "now_round": 2}) + return + assert url.path == run_url + "/disputes" + query = parse_qs(url.query) + identity = query["party"][0] + assert verify( + decode_hotkey(identity), + f"hypertrain/watch/2|{service.run_id}|{identity}".encode(), + bytes.fromhex(self.headers["X-Dispute-Signature"]), + ) + events = service.events( + identity, int(query["cursor"][0]), timeout=float(query["timeout"][0]) + ) + self.response([event.model_dump(mode="json") for event in events]) + + def do_POST(self): + body = self.rfile.read(int(self.headers["Content-Length"])) + requests.append(self.path) + if self.path == run_url + "/disputes/ack": + ack = EventAck.model_validate_json(body) + service.acknowledge(ack, now=2) + self.response({"accepted": ack.cursor}) + else: + assert self.path == run_url + "/bisect" + self.response(service.bisect(body, now=2).model_dump(mode="json")) + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + server.daemon_threads = True + # Binding/listening precedes trigger; serving thread completion is bounded below. + serving = threading.Thread(target=server.serve_forever, daemon=True) + key = tmp_path / "miner.key" + key.write_bytes(b"\x07" * 32) + key.chmod(0o600) + config = tmp_path / "miner.toml" + config.write_text( + f'api = "http://127.0.0.1:{server.server_port}"\n' + f'keyfile = "{key}"\nworkdir = "{work}"\nstate_source = "file"\n' + f'image_digest = "{job.manifest.training.reference_spec.image_digest}"\n' + f'run_id = "{service.run_id}"\nowner_hotkey = "{f.COORD.ss58}"\n' + 'device = "cpu"\n' + ) + output, failures, result = io.StringIO(), [], [] + + def client(): + try: + with contextlib.redirect_stdout(output): + result.append( + main( + ["watch", "--config", str(config), "--job", str(job_path), "--timeout", "5"] + ) + ) + except BaseException as error: + failures.append(error) + finally: + completed.set() + + worker = threading.Thread(target=client, daemon=True) + serving.start() + try: + worker.start() + assert waiting.wait(15), failures + assert not completed.is_set() + layer, _ = challenge.reply(service, f.AUDITOR, [step.hashes[0], "aa" * 32, "bb" * 32]) + assert layer.level == "layer" and layer.expected_party == f.HOT.ss58 + assert completed.wait(15) + worker.join(1) + assert not worker.is_alive() and not failures and result == [0] + assert json.loads(output.getvalue()) == {"processed": 1} + accepted = BisectV2.model_validate_json( + service.db.execute( + "SELECT body FROM dispute_entries_v2 WHERE id=? AND seq=?", + (turn.dispute_id, layer.seq), + ).fetchone()[0] + ) + assert ( + accepted.level, + accepted.ctx, + accepted.party, + accepted.previous_transcript_hash, + ) == ("layer", layer.ctx, f.HOT.ss58, layer.transcript_hash) + lo, hi = layer.interval + assert accepted.hashes == party.hashes( + "layer", tuple(layer.ctx), [lo, -(-(lo + hi) // 2), hi] + ) + event = service.events(f.HOT.ss58, first_cursor, timeout=0)[0] + assert ( + service.db.execute( + "SELECT COUNT(*) FROM dispute_acks_v2 WHERE cursor=? AND party=?", + (event.cursor, f.HOT.ss58), + ).fetchone()[0] + == 1 + ) + with sqlite3.connect(watch_path / "watch.db") as saved: + cursor, durable, done = saved.execute( + "SELECT cursor,envelope,done FROM watch_outbox ORDER BY cursor DESC LIMIT 1" + ).fetchone() + assert cursor == event.cursor and done == 1 + assert BisectV2.model_validate(parse_envelope(durable).body) == accepted + assert run_url + "/disputes" in requests + assert run_url + "/disputes/ack" in requests and run_url + "/bisect" in requests + finally: + server.shutdown() + server.server_close() + serving.join(5) + worker.join(15) + assert not serving.is_alive() and not worker.is_alive() diff --git a/tests/miner/test_genesis_cpu_binding.py b/tests/miner/test_genesis_cpu_binding.py new file mode 100644 index 0000000..87cf25c --- /dev/null +++ b/tests/miner/test_genesis_cpu_binding.py @@ -0,0 +1,69 @@ +"""Actual protected caller construction, metadata only; no kernel or tensor child.""" + +import ast +from pathlib import Path +from types import SimpleNamespace + +from hypertrain.miner.island_launch import CapacityAttempt + + +def test_original_genesis_caller_selects_half_cpu_without_global_override(tmp_path: Path) -> None: + # Given: the actual source assignment, not a copied test constructor. + root = Path(__file__).parents[2] / "src/hypertrain" + store_tree = ast.parse((root / "challenge/store.py").read_text()) + caller = next( + n + for n in ast.walk(store_tree) + if isinstance(n, ast.FunctionDef) and n.name == "_prepare_capacity_genesis_v2" + ) + assignment = next( + n + for n in caller.body + if isinstance(n, ast.Assign) + and any(isinstance(t, ast.Name) and t.id == "capacity" for t in n.targets) + ) + charges = [] + namespace = { + "CapacityAttempt": CapacityAttempt, + "identity": "11" * 32, + "digest": "22" * 32, + "run_id": "33" * 32, + "estimate": {"outer_work_units": 728}, + "self": SimpleNamespace( + state_dir=tmp_path, _service_charge_v2=lambda *a, **k: charges.append((a, k)) or True + ), + } + # When: compile only that metadata construction; no preparation/manager call. + exec( + compile(ast.Module(body=[assignment], type_ignores=[]), "actual-genesis-caller", "exec"), + namespace, + ) + attempt = namespace["capacity"] + # Then: default runner remains400%; this original protected attempt explicitly50%. + assert attempt.cpu_quota == "50%" and attempt.charge() + assert charges == [(("33" * 32, -1, "genesis", 1), {"kind": "outer", "units": 728})] + assert ( + CapacityAttempt("44" * 32, "55" * 32, tmp_path / "legacy", lambda: True).cpu_quota is None + ) + runner_tree = ast.parse((root / "miner/island_launch.py").read_text()) + default = next( + n + for n in runner_tree.body + if isinstance(n, ast.Assign) + and any(isinstance(t, ast.Name) and t.id == "_CAPACITY_CPU_PERCENT" for t in n.targets) + ) + assert ast.literal_eval(default.value) == 400 + runner = next( + n + for n in runner_tree.body + if isinstance(n, ast.FunctionDef) and n.name == "run_capacity_argv" + ) + runner_code = ast.unparse(runner) + assert "capacity.cpu_quota" in runner_code and "observed['cpu.max'] != cpu_max" in runner_code + assert runner_code.count("'cpu_quota': cpu_quota") == 2 + opening = next( + n + for n in ast.walk(store_tree) + if isinstance(n, ast.FunctionDef) and n.name == "open_round_v2" + ) + assert "execution.get('cpu_quota') != attempt['cpu_quota']" in ast.unparse(opening) diff --git a/tests/miner/test_island_launch_v2.py b/tests/miner/test_island_launch_v2.py new file mode 100644 index 0000000..1d8f8f4 --- /dev/null +++ b/tests/miner/test_island_launch_v2.py @@ -0,0 +1,867 @@ +"""Real tiny torchrun parent integration; no CUDA inference from CPU checks.""" + +from __future__ import annotations + +import hashlib +import json +import subprocess +import threading +import time +from pathlib import Path + +import numpy as np +import pytest +import torch + +import hypertrain.trainer # noqa: F401 +from hypertrain.auditor.replay import pack_state, unpack_state +from hypertrain.miner.island_launch import IslandFailure, launch_argv, launch_island +from hypertrain.protocol.example import example_manifest +from hypertrain.protocol.hashing import MerkleTree +from hypertrain.protocol.messages import f32hex +from hypertrain.protocol.messages_v2 import IslandJobV1, RunManifestV2 +from hypertrain.trainer.config import TrainConfig +from hypertrain.trainer.island import emulate, train_island +from hypertrain.trainer.loop import Assignment +from hypertrain.trainer.model import init_params, param_shapes + + +def tiny_manifest(n: int = 2, ep: int = 1, zero1: bool = True) -> RunManifestV2: + b = example_manifest().body() + b["model"].update( + n_layers=1, + d_model=8, + n_heads=2, + n_kv_heads=2, + d_ff=12, + vocab=16, + seq_len=4, + n_experts=2 if ep > 1 else 1, + top_k_experts=1, + compute_dtype="fp32", + ) + b["inner"].update(H=2, J=1, micro_batch=1, grad_accum=1, state_policy="reset", rewarmup_steps=0) + b["inner"]["lr_schedule"].update(warmup=0, stable=100, peak_lr=f32hex(0.001)) + b["reference_spec"]["layout"].update(pp=1, n_gpus=n, dp_size=n // ep, ep_size=ep, zero1=zero1) + b["model"]["param_count"] = sum( + int(np.prod(s)) for s in param_shapes(TrainConfig.from_manifest(b).model).values() + ) + return RunManifestV2.model_validate( + { + "manifest_version": 2, + "training": b, + "network": { + **{ + k: "11" * 32 + for k in ( + "admission_policy_hash", + "economics_policy_hash", + "aggregation_policy_hash", + "dispute_policy_hash", + "audit_policy_hash", + "relay_registry_hash", + ) + }, + "audit_mode": "anchored-full", + "full_anchor_version": 1, + "capabilities": ["island-replay", "all-level-disputes", "transport-receipts"], + }, + } + ) + + +def staged_job(directory: Path, n: int = 2, ep: int = 1) -> IslandJobV1: + w = tiny_manifest(n, ep) + cfg = TrainConfig.from_manifest_v2(w) + theta = init_params(cfg.model) + rows = np.arange(16 * 5, dtype=np.uint32).reshape(16, 5) % 16 + tree = MerkleTree([r.astype(" None: + job = staged_job(tmp_path, n, ep) + output = launch_island(job, tmp_path, backend="cpu") + cfg = TrainConfig.from_manifest_v2(job.manifest) + rows = np.frombuffer((tmp_path / "samples").read_bytes(), " None: + job = staged_job(tmp_path, 8) + assert "--nproc-per-node=8" in launch_argv(job, tmp_path, "cuda") + assert len(job.sample_ids) == 16 + assert "--max-restarts=0" in launch_argv(job, tmp_path, "cuda") + with pytest.raises(IslandFailure, match="deadline"): + launch_island( + job.model_copy(update={"deadline": int(time.time()) - 1}), tmp_path, backend="cpu" + ) + + +def test_wrong_input_and_symlink_leave_no_publication(tmp_path: Path) -> None: + job = staged_job(tmp_path) + (tmp_path / "start_state").write_bytes(b"corrupt") + with pytest.raises(IslandFailure, match="input hash mismatch"): + launch_island(job, tmp_path, backend="cpu") + assert not (tmp_path / "published").exists() + (tmp_path / "start_state").unlink() + (tmp_path / "start_state").symlink_to("/etc/hosts") + with pytest.raises(IslandFailure, match="escapes"): + launch_island(job, tmp_path, backend="cpu") + + +def test_rank_death_and_live_cancel_teardown( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from hypertrain.miner import island_launch + + job = staged_job(tmp_path) + original = subprocess.Popen + cancel = threading.Event() + spawned = [] + + def start(*args, **kwargs): + proc = original(*args, **kwargs) + spawned.append(proc) + cancel.set() + return proc + + monkeypatch.setattr(island_launch.subprocess, "Popen", start) + with pytest.raises(IslandFailure, match="rank failure"): + launch_island(job, tmp_path, backend="cpu", cancel=cancel) + assert spawned[0].returncode is not None + assert not (tmp_path / "published").exists() + + +def test_blocked_child_cancels_on_registered_event( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import socket + import sys + + from hypertrain.miner import island_launch + + # Given: register readiness before a child blocks indefinitely in signal.pause. + job = staged_job(tmp_path) + original = subprocess.Popen + cancelled = threading.Event() + reader, writer = socket.socketpair() + with reader, writer: + reader.settimeout(10) + script = f"import os,signal; os.write({writer.fileno()},b'R'); signal.pause()" + spawned = [] + + def start(*args, **kwargs): + proc = original(*args, **kwargs, pass_fds=(writer.fileno(),)) + spawned.append(proc) + assert reader.recv(1) == b"R" + cancelled.set() + return proc + + monkeypatch.setattr( + island_launch, "launch_argv", lambda *args: [sys.executable, "-c", script] + ) + monkeypatch.setattr(island_launch.subprocess, "Popen", start) + # When/Then: cancellation wakes teardown, reaps the group, publishes nothing. + with pytest.raises(IslandFailure, match="rank failure"): + launch_island(job, tmp_path, backend="cpu", cancel=cancelled) + assert spawned[0].returncode is not None + assert not (tmp_path / "published").exists() + + +def test_noisy_child_streams_logs_and_joins_success_watcher( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + import sys + import tracemalloc + + from hypertrain.miner import island_launch + + job = staged_job(tmp_path) + original_popen = subprocess.Popen + original_thread = threading.Thread + watchers = [] + + def spawn(*args, **kwargs): + assert kwargs["stdout"] != subprocess.PIPE + assert kwargs["stderr"] != subprocess.PIPE + return original_popen(*args, **kwargs) + + def thread(*args, **kwargs): + result = original_thread(*args, **kwargs) + watchers.append(result) + return result + + def artifacts(job, directory): + return island_launch.IslandArtifacts( + directory, + directory / "state", + directory / "ef", + directory / "delta", + directory / "leaves", + (), + ) + + def forbidden_communicate(*args, **kwargs): + pytest.fail("launcher must not buffer subprocess output with communicate") + + script = "import os; b=b'x'*65536; " + "; ".join( + ["[os.write(1,b) for _ in range(64)]", "[os.write(2,b) for _ in range(64)]"] + ) + monkeypatch.setattr(island_launch, "launch_argv", lambda *args: [sys.executable, "-c", script]) + monkeypatch.setattr(island_launch, "validate_artifacts", artifacts) + monkeypatch.setattr(island_launch.subprocess, "Popen", spawn) + monkeypatch.setattr(original_popen, "communicate", forbidden_communicate) + monkeypatch.setattr(island_launch.threading, "Thread", thread) + cancel = threading.Event() + tracemalloc.start() + try: + result = launch_island(job, tmp_path, backend="cpu", cancel=cancel) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + assert peak < 1024 * 1024 # eight MiB logs never become parent byte buffers + assert (result.directory / "stdout.log").stat().st_size == 4 * 1024 * 1024 + assert (result.directory / "stderr.log").stat().st_size == 4 * 1024 * 1024 + assert len(watchers) == 1 and not watchers[0].is_alive() + assert not cancel.is_set() # completion wakes watcher without faking caller cancellation + + +def test_noisy_failure_retains_only_bounded_error_tail( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + import sys + + from hypertrain.miner import island_launch + + job = staged_job(tmp_path) + script = "import os; os.write(2,b'x'*100000+b'END'); raise SystemExit(7)" + monkeypatch.setattr(island_launch, "launch_argv", lambda *args: [sys.executable, "-c", script]) + with pytest.raises(IslandFailure, match="rank failure 7") as error: + launch_island(job, tmp_path, backend="cpu") + assert str(error.value).endswith("END") + assert len(str(error.value)) <= 4020 + assert not (tmp_path / "published").exists() + + +def test_capacity_branch_uses_real_bounded_failure_before_publication(tmp_path, monkeypatch): + import os + import sys + + from hypertrain.miner import island_launch + + job = staged_job(tmp_path) + monkeypatch.setattr(island_launch, "_CAPACITY_MEMORY", 64 << 20) + monkeypatch.setattr(island_launch, "_CAPACITY_CPU_PERCENT", 50) + calls = [] + + def charge(): + calls.append("charged") + return True + + def argv(original, attempt, backend): + assert original == job and backend == "cpu" + assert json.loads((attempt / "job.json").read_text()) == job.model_dump(mode="json") + for relative in job.object_paths.values(): + assert (attempt / relative).read_bytes() == (tmp_path / relative).read_bytes() + return [sys.executable, "-c", "raise SystemExit(7)"] + + monkeypatch.setattr(island_launch, "launch_argv", argv) + capacity = island_launch.CapacityAttempt( + "77" * 32, "66" * 32, tmp_path / "runtime.lock", charge + ) + with pytest.raises(IslandFailure, match="execution failed"): + launch_island(job, tmp_path, backend="cpu", capacity=capacity) + assert calls == ["charged"] + assert (tmp_path / "capacity-runtime/capacity-cleaned.json").exists() + assert not (tmp_path / "published").exists() + assert os.path.exists(tmp_path / "capacity-runtime/capacity-result.json") + + +def test_disagreement_corruption_and_publication_resume(tmp_path: Path) -> None: + from hypertrain.miner.island_launch import validate_artifacts + + job = staged_job(tmp_path) + result = launch_island(job, tmp_path, backend="cpu", trace=True) + summary = result.directory / "rank-1" / "summary.json" + raw = json.loads(summary.read_bytes()) + raw["commitments"]["delta_hash"] = "11" * 32 + summary.write_text(json.dumps(raw)) + with pytest.raises(IslandFailure, match="disagreement"): + validate_artifacts(job, result.directory) + raw["commitments"]["delta_hash"] = hashlib.sha256( + (result.directory / "rank-1" / "delta.bin").read_bytes() + ).hexdigest() + summary.write_text(json.dumps(raw)) + result.delta.write_bytes(b"corruption") + with pytest.raises(IslandFailure, match="compression"): + launch_island(job, tmp_path, backend="cpu") + + +def test_published_v0_corruption_rejected(tmp_path: Path) -> None: + from hypertrain.miner.island_launch import validate_artifacts + + # Given: a complete independently validated N2 publication. + job = staged_job(tmp_path) + result = launch_island(job, tmp_path, backend="cpu") + (result.directory / "v0").write_bytes(b"corrupt derived-state object") + # When/Then: validation never accepts a changed authenticated input. + with pytest.raises(IslandFailure, match="published input hash"): + validate_artifacts(job, result.directory) + + +def test_checkpoint_optimizer_counter_is_validated(tmp_path: Path) -> None: + from hypertrain.miner.island_launch import validate_artifacts + + # Given: stage hashes do not include the optimizer step scalar. + job = staged_job(tmp_path) + result = launch_island(job, tmp_path, backend="cpu") + checkpoint = result.directory / "rank-0/checkpoints/1.safetensors" + theta, state = unpack_state(checkpoint.read_bytes()) + assert state is not None + state.step += 1 + checkpoint.write_bytes(pack_state(theta, state)) + # When/Then: an otherwise byte-identical checkpoint cannot serve a false counter. + with pytest.raises(IslandFailure, match="checkpoint leaf state"): + validate_artifacts(job, result.directory) + + +def test_experiment_runtime_and_cost_admission_fail_closed() -> None: + import importlib.util + + path = Path(__file__).parents[2] / "experiments/gpu_network_v2/orchestrate.py" + spec = importlib.util.spec_from_file_location("network_experiment", path) + assert spec and spec.loader + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + profile = json.loads((path.parent / "profile.json").read_bytes()) + with pytest.raises(mod.Reject, match="unqualified"): + mod.runtime_admission(profile, {}) + evidence = { + "profile_model": profile["model"], + "layout": profile["layout"], + "device": "cuda", + "torch": "2.14.0+cu130", + "full_round_transfer_bound_seconds": 40, + "no_slower_hosts": True, + "measured_rescue_bytes_per_second": 100000000, + "image_and_dependency_bytes": 10000000000, + "staging_bytes_per_host": 1000000, + "image_digest": "sha256:" + "11" * 32, + "driver_allowlist": ["test-driver"], + "python": "/opt/hypertrain/venv/bin/python", + "cuda": "13.0", + "full_round_artifacts_sha256": "11" * 32, + "outer_tape_verified": True, + } + profile["artifacts"].update( + rescue_bytes_per_second=100000000, + image_and_dependency_bytes=10000000000, + staging_bytes_per_host=1000000, + ) + profile["runtime"].update( + image_digest=evidence["image_digest"], + driver_allowlist=evidence["driver_allowlist"], + qualified=True, + ) + from hypertrain.protocol.jcs import canonicalize + + profile.update( + execution_and_transfer_bound_seconds=40, + runtime_evidence_sha256=hashlib.sha256(canonicalize(evidence)).hexdigest(), + ) + assert mod.runtime_admission(profile, evidence) == 3300 + auth = { + "experiment": "hypertrain-network-v2", + "currency": "USD", + "ceiling_usd": "15", + "max_instances": 2, + "max_seconds_per_instance": 3600, + "source_message": "test explicit authorization", + "date": "2026-10-08", + } + snap = { + "authenticated": True, + "instances": [], + "volumes": [], + "uncapped_charges": False, + "verified_unix": int(time.time()), + "prior_attempts_usd": "0", + "active_liabilities_usd": "0", + "available_credit_usd": "15", + } + offers = [ + { + "machine_id": i, + "gpu_name": "RTX 5090", + "num_gpus": 2, + "dph_total": "2", + "storage_cost": "0", + "inet_up_cost": "0", + "inet_down_cost": "0", + "cpu_cores_effective": 8, + "cpu_ram": 32768, + "disk_space": 80, + "verified_unix": int(time.time()), + } + for i in [1, 2] + ] + assert mod.admission_plan(profile, evidence, auth, snap, offers)["admit"] + with pytest.raises(mod.Reject, match="authorization"): + mod.admission_plan(profile, evidence, {}, snap, offers) + with pytest.raises(mod.Reject, match="unknown"): + mod.admission_plan(profile, evidence, auth, {**snap, "instances": None}, offers) + with pytest.raises(mod.Reject, match="ceiling"): + mod.admission_plan(profile, evidence, auth, {**snap, "prior_attempts_usd": "8"}, offers) + + +def test_exact_staged_contract_over_real_ssh(tmp_path: Path) -> None: + import getpass + import importlib.util + import os + import select + import signal + import socket + import sys + + from hypertrain.gpu_ops.journal import Journal + from hypertrain.gpu_ops.launcher import Orchestrator + from hypertrain.gpu_ops.provider import Provider + from hypertrain.gpu_ops.remote import ensure_identity + + tree = Path(__file__).parents[2] + spec = importlib.util.spec_from_file_location( + "network_ssh_runtime", tree / "experiments/gpu_network_v2/orchestrate.py" + ) + assert spec and spec.loader + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + run = tmp_path / "lifecycle" + run.mkdir() + journal = Journal(run) + identity = ensure_identity(run) + host = tmp_path / "host-key" + subprocess.run( + ["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", str(host)], + check=True, + capture_output=True, + ) + authorized = tmp_path / "authorized_keys" + authorized.write_bytes(identity.with_suffix(".pub").read_bytes()) + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + config = tmp_path / "sshd_config" + config.write_text( + f"ListenAddress 127.0.0.1\nPort {port}\nHostKey {host}\n" + f"PidFile {tmp_path}/sshd.pid\nAuthorizedKeysFile {authorized}\nStrictModes no\n" + "PermitRootLogin prohibit-password\nPasswordAuthentication no\n" + f"KbdInteractiveAuthentication no\nUsePAM {'yes' if os.getuid() == 0 else 'no'}\n" + f"AllowUsers {getpass.getuser()}\nLogLevel VERBOSE\n" + ) + server = subprocess.Popen( + ["/usr/sbin/sshd", "-D", "-e", "-f", str(config)], + stderr=subprocess.PIPE, + start_new_session=True, + ) + try: + assert server.stderr is not None + assert select.select([server.stderr], [], [], 15)[0] + assert b"Server listening" in server.stderr.readline() + cfg = { + "base_url": "http://127.0.0.1:1", + "hosts": [{"role": "h0", "machine_id": 1}], + "remote_root": str(tmp_path / "remote"), + "remote_python": sys.executable, + "ssh_user": getpass.getuser(), + "image": "test@sha256:" + "11" * 32, + } + provider = Provider(cfg["base_url"], "a" * 32, run, journal, live=False) + lifecycle = Orchestrator(cfg, journal, provider, run) + journal.append("admitted", deadline_unix=time.time() + 3600) + journal.append("receipt", role="h0", instance_id=1) + journal.append("instance_ready", role="h0", ssh_host="127.0.0.1", ssh_port=port) + journal.append("supervisor_ready", supervisor_pid=os.getpid()) + lifecycle.trust("h0") + job_dir = tmp_path / "job" + job = staged_job(job_dir) + (job_dir / "job.json").write_text(job.model_dump_json()) + profile = json.loads((tree / "experiments/gpu_network_v2/profile.json").read_bytes()) + profile["execution_and_transfer_bound_seconds"] = 40 + runtime = mod.NetworkRuntime(lifecycle, profile) + runtime.stage("h0", tree, {"tiny": job_dir}) + done = runtime.run_staged("h0", "tiny", backend="cpu") + assert done["ssh_rc"] == 0 + assert runtime.run_staged("h0", "tiny", backend="cpu") == done + assert len(journal.all("network_job_started")) == 1 + rescued = runtime.rescue("h0") + assert any("rank-0/checkpoints/" in name for name in rescued["files"]) + assert any("rank-1/checkpoints/" in name for name in rescued["files"]) + assert lifecycle.rescue("h0") == "verified" + assert not journal.all("provider_call") + finally: + os.killpg(server.pid, signal.SIGTERM) + server.wait(timeout=15) + + +def test_tiny_full_h_artifact_readiness_and_path_preserving_rescue(tmp_path: Path) -> None: + import importlib.util + + from hypertrain.trainer.config import ModelConfig + from hypertrain.trainer.optim import OptState + + spec = importlib.util.spec_from_file_location( + "network_budget", Path(__file__).parents[2] / "experiments/gpu_network_v2/orchestrate.py" + ) + assert spec and spec.loader + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + profile = json.loads((Path(spec.origin).parent / "profile.json").read_bytes()) + bounds = mod.artifact_budget(profile) + assert profile["inner"]["H"] == 30 and profile["inner"]["J"] == 5 + assert profile["inner"]["state_policy"] == "carry" + assert profile["schedule"]["executions_total"] == 126 + cfg = ModelConfig( + **{ + k: v + for k, v in profile["model"].items() + if k + not in ("param_count", "capacity_factor", "aux_loss_coef", "init_std", "master_dtype") + } + ) + shapes = param_shapes(cfg) + assert sum(int(np.prod(x)) for x in shapes.values()) == bounds["param_count"] == 61856 + assert len(shapes) == bounds["tensor_count"] + theta = init_params(cfg) + st = OptState( + {k: torch.zeros_like(x) for k, x in theta.items()}, + {k: torch.zeros_like(x) for k, x in theta.items()}, + 30, + ) + assert len(pack_state(theta, st)) <= bounds["state_bytes"] + assert bounds["disk_artifacts_peak_bytes"] < 80000000000 + with pytest.raises(mod.Reject, match="measured"): + mod.readiness_budget(profile, {}) + profile["artifacts"].update( + image_and_dependency_bytes=10000000000, + staging_bytes_per_host=1000000, + rescue_bytes_per_second=bounds["rescue_min_bytes_per_second"], + ) + ev = { + "image_and_dependency_bytes": 10000000000, + "staging_bytes_per_host": 1000000, + "measured_rescue_bytes_per_second": bounds["rescue_min_bytes_per_second"], + } + assert mod.readiness_budget(profile, ev)["priced_transfer_gb_per_host"] > 2 + with pytest.raises(mod.Reject, match="unqualified"): + mod.staged_command(profile, "/work", "tiny", "/venv/main/bin/python", "wrong", "cuda") + profile["runtime"].update( + image_digest="sha256:" + "11" * 32, driver_allowlist=["test"], qualified=True + ) + command = mod.staged_command( + profile, + "/work", + "tiny", + "/opt/hypertrain/venv/bin/python", + profile["runtime"]["image_digest"], + "cuda", + ) + assert command.endswith("run.py out/tiny cuda") + assert "CUBLAS_WORKSPACE_CONFIG=:4096:8" in command + assert "CUDA_DISABLE_PTX_JIT=1" in command + assert "MKL_NUM_THREADS=1" in command + out = tmp_path / "out" + for rank in range(2): + p = out / "job" / "published" / f"rank-{rank}" / "checkpoints" / "30.safetensors" + p.parent.mkdir(parents=True) + p.write_bytes(bytes([rank]) * 128) + archive = tmp_path / "rescue.tar" + mod.export_artifacts(out, archive, profile) + files = mod.verify_export(archive, bounds["rescue_tar_bytes_per_host"]) + assert len(files) == 2 + assert len({x["sha256"] for x in files.values()}) == 2 + assert all("checkpoints/30.safetensors" in name for name in files) + + +def test_fresh_account_snapshot_uses_get_only() -> None: + import importlib.util + + from hypertrain.gpu_ops.provider import Response + + path = Path(__file__).parents[2] / "experiments/gpu_network_v2/orchestrate.py" + spec = importlib.util.spec_from_file_location("network_get_refresh", path) + assert spec and spec.loader + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + methods = [] + + class ReadOnly: + def call(self, method, path, tag): + methods.append((method, path)) + if path == "/api/v0/users/current/": + data = {"credit": "20.953246133119993"} + elif path == "/api/v1/instances/": + data = {"success": True, "instances": []} + elif path == "/api/v0/volumes/": + data = {"success": True, "volumes": []} + else: + data = {"success": True, "results": [], "next_token": None} + return Response(200, data, None, "mock", None) + + snapshot = mod.refresh_readonly(ReadOnly()) + assert snapshot["instances"] == snapshot["volumes"] == [] + assert [m for m, _ in methods] == ["GET"] * 4 + + +def test_qualification_cost_admits_without_measured_runtime_but_not_long_workload() -> None: + import importlib.util + + path = Path(__file__).parents[2] / "experiments/gpu_network_v2/orchestrate.py" + spec = importlib.util.spec_from_file_location("network_qualification", path) + assert spec and spec.loader + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + profile = json.loads((path.parent / "profile.json").read_bytes()) + profile["runtime"]["image_digest"] = "sha256:" + "11" * 32 + evidence = { + "root_verified_bounded_lifecycle": True, + "registry_image_verified": True, + "candidate_image_digest": profile["runtime"]["image_digest"], + "staged_contract_sha256": "22" * 32, + "deadline_supervisor_contract_sha256": "33" * 32, + } + auth = { + "experiment": "hypertrain-network-v2", + "currency": "USD", + "ceiling_usd": "15", + "max_instances": 2, + "max_seconds_per_instance": 3600, + "source_message": "bounded qualification allowed", + "date": "2026-10-08", + } + snapshot = { + "authenticated": True, + "instances": [], + "volumes": [], + "uncapped_charges": False, + "verified_unix": int(time.time()), + "prior_attempts_usd": "0", + "active_liabilities_usd": "0", + "available_credit_usd": "20.953246133119993", + } + offers = [ + { + "machine_id": n, + "gpu_name": "RTX 5090", + "num_gpus": 2, + "dph_total": "1", + "storage_cost": "0.1", + "inet_up_cost": "0.001", + "inet_down_cost": "0.001", + "cpu_cores_effective": 8, + "cpu_ram": 32768, + "disk_space": 80, + "verified_unix": int(time.time()), + } + for n in (1, 2) + ] + plan = mod.admission_plan(profile, evidence, auth, snapshot, offers, qualification=True) + assert plan["admit"] and not plan["long_workload_allowed"] + assert float(plan["hard_deadline_hours"]) == 1 # failure still priced for whole liability + assert float(plan["worst_case_total_usd"]) <= 15 + with pytest.raises(mod.Reject, match="unqualified"): + mod.admission_plan(profile, evidence, auth, snapshot, offers) + with pytest.raises(mod.Reject, match="contract_missing"): + mod.admission_plan(profile, {}, auth, snapshot, offers, qualification=True) + with pytest.raises(mod.Reject, match="ceiling"): + mod.admission_plan( + profile, + evidence, + auth, + {**snapshot, "prior_attempts_usd": "10"}, + offers, + qualification=True, + ) + profile["runtime"]["driver_allowlist"] = ["candidate-driver"] + cmd = mod.staged_command( + profile, + "/work", + "probe", + profile["runtime"]["remote_python"], + profile["runtime"]["image_digest"], + "cuda", + qualification=True, + ) + assert cmd.endswith("probe cuda") + with pytest.raises(mod.Reject, match="unqualified"): + mod.staged_command( + profile, + "/work", + "work", + profile["runtime"]["remote_python"], + profile["runtime"]["image_digest"], + "cuda", + ) + + +def test_d2_execution_plan_invokes_launcher_and_rescues_every_host_on_fault(tmp_path: Path) -> None: + import importlib.util + + from hypertrain.gpu_ops.journal import Journal + + path = Path(__file__).parents[2] / "experiments/gpu_network_v2/orchestrate.py" + spec = importlib.util.spec_from_file_location("network_plan", path) + assert spec and spec.loader + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + profile = json.loads((path.parent / "profile.json").read_bytes()) + + class Lifecycle: + roles = ["h0", "h1"] + j = Journal(tmp_path) + + runtime = mod.NetworkRuntime(Lifecycle(), profile) + jobs = [ + {"role": role, "name": f"job-{i}", "equivalents": 1, "backend": "cuda"} + for role in Lifecycle.roles + for i in range(63) + ] + launched, rescued = [], [] + + def run(role, name, equivalents, backend): + launched.append((role, name, backend)) + if role == "h0" and name == "job-0": + raise mod.Reject("actual rank failure") + return {"role": role} + + def rescue(role): + rescued.append(role) + return {"role": role} + + runtime.run_staged, runtime.rescue = run, rescue + # This check validates scheduling/failure control only, never synthetic GPU qualification. + original = mod.runtime_admission + mod.runtime_admission = lambda *args: 3300 + try: + with pytest.raises(mod.Reject, match="actual rank failure"): + runtime.execute_plan({}, jobs) + finally: + mod.runtime_admission = original + assert set(rescued) == {"h0", "h1"} + assert any(role == "h1" for role, _, _ in launched) + assert all(backend == "cuda" for _, _, backend in launched) + + +@pytest.mark.parametrize("objective", ["mlm", "decision", "distill"]) +def test_real_od_two_rank_worker_paths(tmp_path: Path, objective: str) -> None: + import importlib.util + + spec = importlib.util.spec_from_file_location( + "worker_od_fixture", Path(__file__).parents[1] / "layout/test_od_island_v2.py" + ) + assert spec and spec.loader + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + wrapper = mod.od_manifest(objective, 2, True) + cfg = TrainConfig.from_manifest_v2(wrapper) + theta = init_params(cfg.model) + get = mod.samples(cfg) + count = wrapper.training.batch_samples() + raw_rows = [get(i).astype(" RunManifestV2: + body = training or example_manifest().body() + if training is None: + body["model"]["param_count"] = 4_329_216 + body["reference_spec"]["driver_allowlist"] = ["580"] + return RunManifestV2.model_validate( + { + "manifest_version": 2, + "training": body, + "network": { + **{ + name: H + for name in ( + "admission_policy_hash", + "economics_policy_hash", + "aggregation_policy_hash", + "dispute_policy_hash", + "audit_policy_hash", + "relay_registry_hash", + ) + }, + "audit_mode": "anchored-full", + "full_anchor_version": 1, + "capabilities": ["island-replay", "all-level-disputes", "transport-receipts"], + }, + } + ) + + +def commit(tokens: int = 7680) -> CommitV2: + return CommitV2( + w=3, + hotkey=MINER.ss58, + leaf_scheme="ht-leaf-v1", + n_leaves=7, + leaves_root=H, + metrics_root=H, + final_theta_hash=H, + ef_in_hash=H, + ef_out_hash=H, + delta_hash=H, + delta_bytes=1024, + tokens=tokens, + ) + + +@pytest.mark.parametrize("n", [1, 2, 4, 8]) +@pytest.mark.parametrize("objective", ["mlm", "decision", "distill"]) +@pytest.mark.parametrize("zero1", [False, True]) +def test_od_general_n_when_v2(n: int, objective: str, zero1: bool) -> None: + # Given + body = od_body() + body["model"]["param_count"] = 116_288 + body["inner"].update(H=1, J=1, micro_batch=1, grad_accum=1) + body["reference_spec"]["layout"].update(n_gpus=n, dp_size=n, ep_size=1, zero1=zero1) + if objective != "mlm": + rec = {"state_len": 8, "n_questions": 1, "n_options": 2, "opt_len": 2, "instr_len": 2} + body["model"]["od"].update(objective=objective, record=rec) + body["model"].update(seq_len=18, param_count=187_140) + # When + model = wrapper(body) + # Then + assert model.training.batch_samples() == n + assert model.run_id() != model.training.training_hash() + if n != 1 or zero1: + with pytest.raises(ValidationError): + RunManifest.model_validate(body) + + +def test_assign_unit_when_only_island_batch_is_divisible() -> None: + # Given + body = od_body() + body["model"]["param_count"] = 116_288 + body["inner"].update(H=1, J=1, micro_batch=1, grad_accum=1) + body["reference_spec"]["layout"].update(n_gpus=8, dp_size=8) + body["dataset"].update(assign_unit=8, unit_sha256_root=H, n_samples=4096) + # When + model = wrapper(body) + # Then + assert model.training.batch_samples() == 8 + with pytest.raises(ValidationError): + RunManifest.model_validate(body) + + +@pytest.mark.parametrize("field,value", [("param_count", 1), ("d_ff", 128), ("n_kv_heads", 2)]) +def test_od_invalid_geometry_when_v2(field: str, value: int) -> None: + body = od_body() # Given + body["model"][field] = value + with pytest.raises(ValidationError): # When / Then + wrapper(body) + + +@pytest.mark.parametrize("n,ep,pp", [(4, 2, 1), (4, 1, 2), (3, 1, 1)]) +def test_invalid_layout_when_v2(n: int, ep: int, pp: int) -> None: + body = od_body() # Given + body["reference_spec"]["layout"].update(n_gpus=n, dp_size=4, ep_size=ep, pp=pp) + with pytest.raises(ValidationError): # When / Then + wrapper(body) + + +def test_wrapper_when_training_run_id_requested() -> None: + model = wrapper() # Given + with pytest.raises(ValueError): # When / Then + model.training.run_id() + assert model.run_id() == sha256_hex(canonicalize(model.body(), allow_float=False)) + + +def test_commit_when_token_entitlement_or_leaf_count_differs() -> None: + manifest = wrapper() # Given + data = commit().model_dump(mode="json") + inner = manifest.training.inner + samples = manifest.training.batch_samples() + data.update(tokens=samples * manifest.training.model.seq_len, n_leaves=inner.H // inner.J + 1) + # When / Then + CommitV2.model_validate(data).validate_assignment(manifest, samples) + data["tokens"] += 1 + with pytest.raises(ValueError): + CommitV2.model_validate(data).validate_assignment(manifest, samples) + + +def test_accept_when_claimed_rank_count_differs() -> None: + manifest = wrapper() # Given + ref = manifest.training.reference_spec + data = dict( + w=1, + hotkey=MINER.ss58, + assignment_hash=H, + image_digest=ref.image_digest, + driver_version=ref.driver_allowlist[0], + n_gpus=ref.layout.n_gpus, + work_screen_hash=H, + ) + # When / Then + AcceptV2.model_validate(data).validate_manifest(manifest) + with pytest.raises(ValueError): + wrong = AcceptV2.model_validate({**data, "n_gpus": ref.layout.n_gpus + 1}) + wrong.validate_manifest(manifest) + + +@pytest.mark.parametrize( + "change", + [ + {"w": True}, + {"tokens": "1"}, + {"tokens": -1}, + {"tokens": 1.0}, + {"delta_bytes": 2**31 + 1}, + {"extra": 1}, + {"tokens": 2**53}, + ], +) +def test_strict_body_when_invalid_number(change: dict[str, JsonValue]) -> None: + data = commit().model_dump(mode="json") # Given + data.update(change) + with pytest.raises(MalformedEnvelope): # When / Then + envelope_v2.seal(MINER, "CommitV2", RUN, data, 100) + + +def test_exact_preimage_when_sealed() -> None: + body = commit().model_dump(mode="json") # Given + expected = b"hypertrain/2|CommitV2|" + sha256_hex(canonicalize(body)).encode() + # When / Then + assert envelope_v2.signing_message("CommitV2", body, 100, RUN) == ( + expected + b"|100|" + RUN.encode() + ) + assert v1_signing_message("Commit", body, 100, RUN) == ( + b"hypertrain/1|Commit|" + sha256_hex(canonicalize(body)).encode() + b"|100|" + RUN.encode() + ) + + +def test_replay_when_same_body_or_conflicting_body_after_restart() -> None: + intake = envelope_v2.Intake(RUN) # Given + first = envelope_v2.seal(MINER, "CommitV2", RUN, commit(), 100) + intake.accept(first, 50) + restarted = envelope_v2.Intake(RUN, reservations=dict(intake.reservations)) + # When + duplicate = restarted.accept(envelope_v2.seal(MINER, "CommitV2", RUN, commit(), 101), 50) + # Then + assert duplicate == commit() and len(restarted.reservations) == 1 + with pytest.raises(ReplayError): + restarted.accept(envelope_v2.seal(MINER, "CommitV2", RUN, commit(1), 100), 50) + + +def test_workproof_when_consecutive_accepted_commit_subjects_share_admission() -> None: + intake = envelope_v2.Intake(RUN) + commits = [commit().model_copy(update={"w": w}) for w in (0, 1)] + proofs = [] + for model in commits: + accepted = intake.accept(envelope_v2.seal(MINER, "CommitV2", RUN, model, 100), 50) + proof = WorkProof( + admission_id=H, + challenge_hash=envelope_v1.body_digest(accepted.model_dump(mode="json")), + leaves_root=model.leaves_root, + delta_hash=model.delta_hash, + artifact_refs=[ArtifactRef(sha256=H, size=1)], + ) + assert intake.accept(envelope_v2.seal(MINER, "WorkProof", RUN, proof, 100), 50) == proof + assert envelope_v2.replay_key("WorkProof", RUN, MINER.ss58, proof) == ( + RUN, + "WorkProof", + MINER.ss58, + H, + proof.challenge_hash, + ) + proofs.append(proof) + restarted = envelope_v2.Intake(RUN, reservations=dict(intake.reservations)) + for proof in proofs: + assert restarted.accept(envelope_v2.seal(MINER, "WorkProof", RUN, proof, 101), 50) == proof + assert len(restarted.reservations) == 4 + # Proof payload is not nonce material; same subject retains conflict detection. + for change in ( + {"leaves_root": "22" * 32}, + {"delta_hash": "22" * 32}, + {"artifact_refs": [ArtifactRef(sha256="22" * 32, size=2)]}, + ): + with pytest.raises(ReplayError): + restarted.accept( + envelope_v2.seal(MINER, "WorkProof", RUN, proofs[1].model_copy(update=change), 100), + 50, + ) + with pytest.raises(RunMismatchError): + envelope_v2.Intake("22" * 32).accept( + envelope_v2.seal(MINER, "WorkProof", RUN, proofs[0], 100), 50 + ) + + +def test_workproof_when_distinct_admission_trial_challenges_share_identity() -> None: + intake = envelope_v2.Intake(RUN) + proof = WorkProof( + admission_id=H, + challenge_hash=H, + leaves_root=H, + delta_hash=H, + artifact_refs=[ArtifactRef(sha256=H, size=1)], + ) + for subject in (H, "22" * 32): + current = proof.model_copy(update={"challenge_hash": subject}) + assert intake.accept(envelope_v2.seal(MINER, "WorkProof", RUN, current, 100), 50) == current + assert len(intake.reservations) == 2 + + +def test_workproof_existing_service_binding_rejects_cross_subject_reuse(tmp_path, request) -> None: + # Boundary fixture seeds signed accepted commits, not training/admission qualification. + spec = importlib.util.spec_from_file_location( + "nonce_service_fixture", Path(__file__).parents[1] / "challenge/test_service_network_v2.py" + ) + assert spec is not None and spec.loader is not None + fixture = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = fixture + spec.loader.exec_module(fixture) + network_fixture = fixture.network.__wrapped__(tmp_path, request) + network = next(network_fixture) + try: + run = network.manifest.run_id() + hot = fixture.HOT[0] + admission_id = network.join().json()["admission_id"] + proofs, commits = [], [] + for w in (0, 1): + leaf = LeafPreimage( + run_id=run, + w=w, + t=0, + stages=[dict(theta=H, m=H, v=H)], + batch_ids_sha256=H, + rng_ctr=0, + loss_f32="00000000", + norm_f32="00000000", + ) + raw = canonicalize([leaf.model_dump(mode="json")]) + object_hash = network.store.objects.put(raw) + from hypertrain.protocol.hashing import MerkleTree + + model = commit().model_copy( + update={ + "w": w, + "hotkey": hot.ss58, + "n_leaves": 1, + "leaves_root": MerkleTree([bytes.fromhex(leaf.digest())]).root.hex(), + } + ) + with network.store._tx(): + network.store._put_record_v2( + run, "commit", f"{w}:{hot.ss58}", network.signed(hot, "CommitV2", model) + ) + proof = WorkProof( + admission_id=admission_id, + challenge_hash=envelope_v1.body_digest(model.model_dump(mode="json")), + leaves_root=model.leaves_root, + delta_hash=model.delta_hash, + artifact_refs=[ArtifactRef(sha256=object_hash, size=len(raw))], + ) + signed = network.signed(hot, "WorkProof", proof) + for _ in range(2): + response = network.client.post(network.url + "/leaves", json=signed) + assert response.status_code == 200, response.text + proofs.append(proof) + commits.append(model) + conflict = proofs[1].model_copy(update={"delta_hash": "22" * 32}) + response = network.client.post( + network.url + "/leaves", json=network.signed(hot, "WorkProof", conflict) + ) + assert response.status_code == 409, response.text + # New nonce alone is insufficient: subject must name an accepted commit. + wrong = proofs[1].model_copy(update={"challenge_hash": "33" * 32}) + response = network.client.post( + network.url + "/leaves", json=network.signed(hot, "WorkProof", wrong) + ) + assert response.status_code == 422, response.text + # Old signed proof cannot be rebound to a different run. + response = network.client.post( + network.url + "/leaves", json=envelope_v2.seal(hot, "WorkProof", RUN, proofs[0], 10000) + ) + assert response.status_code == 409, response.text + # New accepted round with old signed leaf bytes fails existing round/Merkle binding. + third = commits[1].model_copy(update={"w": 2}) + with network.store._tx(): + network.store._put_record_v2( + run, "commit", f"2:{hot.ss58}", network.signed(hot, "CommitV2", third) + ) + rebound = proofs[1].model_copy( + update={"challenge_hash": envelope_v1.body_digest(third.model_dump(mode="json"))} + ) + response = network.client.post( + network.url + "/leaves", json=network.signed(hot, "WorkProof", rebound) + ) + assert response.status_code == 422, response.text + assert ( + network.store._db.execute( + "SELECT COUNT(*) FROM records_v2 WHERE kind='leaves'" + ).fetchone()[0] + == 2 + ) + finally: + network_fixture.close() + + +def test_signed_boundary_when_run_expiry_role_or_domain_changed() -> None: + env = envelope_v2.seal(MINER, "CommitV2", RUN, commit(), 100) # Given + # When / Then + assert envelope_v2.verify_envelope(env) + with pytest.raises(RunMismatchError): + envelope_v2.Intake(H).accept(env, 50) + with pytest.raises(ExpiredError): + envelope_v2.Intake(RUN).accept(env, 101) + with pytest.raises(UnauthorizedSigner): + intake = envelope_v2.Intake(RUN, allowed_signers={"CommitV2": lambda s: s == COLD.ss58}) + intake.accept(env, 50) + altered = {**env, "sig": "0" * 128} + with pytest.raises(SignatureError): + envelope_v2.Intake(RUN).accept(altered, 50) + assert not relay_envelope.verify_envelope(env) + assert not envelope_v2.verify_envelope({**env, "v": "ht/1"}) + + +@pytest.mark.parametrize( + "raw", + [ + '{"a":1,"a":2}', + '{"body":{"w":1,"w":2}}', + '{"w":1.0}', + '{"w":NaN}', + '{"w":Infinity}', + "[" * 33 + "0" + "]" * 33, + '{"value":"' + "x" * 4097 + '"}', + '{"value":' + str(2**53) + "}", + ], +) +def test_parser_when_duplicates_floats_or_bounds_exceeded(raw: str) -> None: + with pytest.raises(MalformedEnvelope): # Given / When / Then + envelope_v2.load_json(raw) + + +def test_parser_when_valid_json_and_unknown_envelope_fields() -> None: + env = envelope_v2.seal(MINER, "CommitV2", RUN, commit(), 100) # Given + raw = json.dumps(env).encode() + # When / Then + assert envelope_v2.Intake(RUN).accept(raw, 50) == commit() + with pytest.raises(MalformedEnvelope): + envelope_v2.parse_envelope({**env, "unexpected": 1}) + with pytest.raises(MalformedEnvelope): + envelope_v2.load_json(raw, max_bytes=8) + + +def test_join_when_both_keys_sign_and_identity_changes() -> None: + request = JoinRequest( # Given + run_id=RUN, + request_id=H, + hotkey=MINER.ss58, + coldkey=COLD.ss58, + expires_beacon=100, + policy_hash=H, + hardware_hint={"device_name": "5090", "device_count": 8, "driver": "580"}, + hot_sig="0" * 128, + cold_sig="0" * 128, + ) + msg = envelope_v2.join_signing_message(request) + data = request.body() + data.update(hot_sig=MINER.sign(msg).hex(), cold_sig=COLD.sign(msg).hex()) + signed = JoinRequest.model_validate(data) + # When / Then + assert envelope_v2.verify_join(signed, RUN, 100) + assert not envelope_v2.verify_join(signed, H, 100) + assert not envelope_v2.verify_join(signed, RUN, 101) + changed = JoinRequest.model_validate({**data, "coldkey": MINER.ss58}) + assert not envelope_v2.verify_join(changed, RUN, 50) + + +def relay_fixture() -> tuple[ + UploadGrant, UploadChunkManifest, AcceptedUploadAck, list[ChunkCustodyAck] +]: + manifest = UploadChunkManifest( + size=(4 << 20) + 8, + chunks=[ + {"index": 0, "off": 0, "len": 4 << 20, "chunk_sha256": H}, + {"index": 1, "off": 4 << 20, "len": 8, "chunk_sha256": "22" * 32}, + ], + ) + grant = UploadGrant( + w=1, + hotkey=MINER.ss58, + relay_id="eu", + assignment_epoch=1, + delta_hash=H, + size=manifest.size, + chunk_manifest_hash=manifest.chunk_manifest_hash(), + retain_until=500, + nonce=H, + exp_drand=100, + ) + ack = AcceptedUploadAck( + grant_hash=grant.digest(), + w=1, + hotkey=MINER.ss58, + relay_id="eu", + assignment_epoch=1, + delta_hash=H, + size=grant.size, + chunk_manifest_hash=grant.chunk_manifest_hash, + key_id="key1", + accepted_beacon=10, + service_deadline=90, + ) + chunks = [ + ChunkCustodyAck( + **c.body(), + upload_ack_hash=ack.digest(), + grant_hash=grant.digest(), + durable_chunk_key=f"chunk/{c.index}", + key_id="key1", + received_beacon=20, + retain_until=500, + ) + for c in manifest.chunks + ] + return grant, manifest, ack, chunks + + +def test_full_custody_when_opportunity_prefix_or_complete_cover() -> None: + grant, manifest, ack, chunks = relay_fixture() # Given + # When / Then + assert not has_full_custody(grant, manifest, ack, []) + assert not has_full_custody(grant, manifest, ack, chunks[:1]) + assert has_full_custody(grant, manifest, ack, chunks) + assert not has_full_custody(grant, manifest, ack, [chunks[0], chunks[0]]) + changed = chunks[1].body() + changed["key_id"] = "other" + assert not has_full_custody( + grant, manifest, ack, [chunks[0], ChunkCustodyAck.model_validate(changed)] + ) + changed["key_id"] = "key1" + changed["received_beacon"] = 91 + assert not has_full_custody( + grant, manifest, ack, [chunks[0], ChunkCustodyAck.model_validate(changed)] + ) + + +def test_full_custody_when_complete_receipt() -> None: + grant, manifest, ack, _ = relay_fixture() # Given + receipt = RelayReceipt( + w=1, + hotkey=MINER.ss58, + delta_hash=H, + size=grant.size, + grant_hash=grant.digest(), + chunk_manifest_hash=grant.chunk_manifest_hash, + durable_object_key="objects/test", + key_id="key1", + received_drand=30, + retain_until=500, + ) + # When / Then + assert has_full_custody(grant, manifest, ack, [], receipt) + env = relay_envelope.seal(MINER, "RelayReceipt", RUN, receipt, 100) + assert relay_envelope.verify_envelope(env) + assert relay_envelope.Intake(RUN).accept(env, 50) == receipt + assert not envelope_v2.verify_envelope(env) + + +def retrieval(requested: int = 101, deadline: int = 111) -> RetrievalRequest: + return RetrievalRequest( + request_id=H, + nonce=H, + relay_id="eu", + key_id="key1", + assignment_hash=H, + grant_hash=H, + custody_ack_hashes=[H], + receipt_hash=None, + retention_hash=H, + object_or_chunk_hash=sha256_hex(b"hello"), + size=5, + requested_beacon=requested, + deadline_beacon=deadline, + ) + + +def test_retrieval_when_after_upload_and_near_retention_expiry() -> None: + request = retrieval() # Given: upload cutoff 100, request 101 + # When / Then + request.validate_horizon(500) + retrieval(495, 500).validate_horizon(500) + with pytest.raises(ValueError): + request.validate_horizon(110) + with pytest.raises(ValueError): + request.validate_horizon(500, released=True) + with pytest.raises(ValidationError): + retrieval(500, 500) + + +def test_retrieval_response_when_bytes_are_independently_hashed() -> None: + request = retrieval() # Given + response = RetrievalResponse( + request_hash=request.digest(), + nonce=H, + key_id="key1", + status="SERVED", + object_hash=sha256_hex(b"hello"), + size=5, + served_beacon=102, + body_sha256=sha256_hex(b"hello"), + error_code=None, + ) + # When / Then + response.validate_request(request, b"hello") + with pytest.raises(ValueError): + response.validate_request(request, b"wrong") + + +def test_retention_when_extension_ack_before_old_horizon() -> None: + extension = RetentionExtension( # Given + grant_hash=H, + custody_hashes=[H], + seq=1, + previous_retention_hash=H, + dispute_ids=[H], + issued_beacon=490, + retain_until=600, + ) + ack = RetentionExtensionAck( + extension_hash=extension.digest(), + key_id="key1", + accepted_beacon=499, + ) + args = dict( + grant_hash=H, + custody_hashes=[H], + key_id="key1", + previous_hash=H, + previous_seq=0, + previous_horizon=500, + ) + # When / Then + assert extend_retention(extension, ack, **args) == 600 + late = RetentionExtensionAck( + extension_hash=extension.digest(), + key_id="key1", + accepted_beacon=500, + ) + with pytest.raises(ValueError): + extend_retention(extension, late, **args) + + +def test_release_when_disputes_closed_and_all_horizons_expired() -> None: + release = CustodyRelease( # Given + grant_hash=H, + custody_hashes=[H], + retention_hash=H, + finality_hash=H, + vesting_beacon=500, + closed_dispute_root=H, + release_beacon=520, + ) + args = dict( + grant_hash=H, + custody_hashes=[H], + retention_hash=H, + retain_until=600, + finality_beacon=490, + epoch_rounds=10, + retrieval_deadlines=[610], + ) + # When / Then + assert validate_release(release, disputes_closed_beacon=510, **args) == 610 + with pytest.raises(ValueError): + validate_release(release, disputes_closed_beacon=None, **args) + + +@pytest.mark.parametrize("level,ctx", [("step", []), ("layer", [1]), ("op", [1, 2])]) +def test_bisection_when_context_matches_level(level: str, ctx: list[int]) -> None: + data = dict( + dispute_id=H, + seq=0, + level=level, + ctx=ctx, + interval=[0, 8], + N=2, + hashes=[H] * 3, + party=MINER.ss58, + previous_transcript_hash=H, + ) # Given + # When / Then + assert BisectV2.model_validate(data).ctx == ctx + with pytest.raises(ValidationError): + BisectV2.model_validate({**data, "ctx": [1, 2, 3]}) + with pytest.raises(ValidationError): + BisectV2.model_validate({**data, "N": True}) + + +def test_dispute_when_contest_is_unfunded() -> None: + with pytest.raises(ValidationError): # Given / When / Then + DisputeV2(hotkey=MINER.ss58, verdict_hash=H, action="contest", bond_lock=None) + + +def test_weight_caps_when_probation_cohort_exceeds_quarter() -> None: + entries = [ + dict( + hotkey=Keypair(bytes([i]) * 32).ss58, + admission_id=H, + probation=False, + weight_units=1 << 22, + commit_hash=H, + delta_manifest_hash=H, + ) + for i in range(4) + ] # Given + entries.sort(key=lambda e: e["hotkey"]) + # When / Then + assert WeightAllocationV2.model_validate({"policy_hash": H, "entries": entries}) + entries[0]["probation"] = True + entries[1]["probation"] = True + with pytest.raises(ValidationError): + WeightAllocationV2.model_validate({"policy_hash": H, "entries": entries}) + + +def test_economics_when_test_genesis_used_in_production() -> None: + data = dict( + ledger_mode="production", + G_max_units=100, + R_collectible_units=100, + S_min_units=1000, + beta_ppm=1_000_000, + gammaV_units=0, + s_lower_ppm=1_000_000, + q_floor=1_000_000, + genesis_allocation_hash=H, + reward_authority=MINER.ss58, + round_reward_units=100, + max_total_issuance=1000, + shadow_bootstrap_rounds=12, + ) # Given + with pytest.raises(ValidationError): # When / Then + EconomicsPolicyV2.model_validate(data) + + +def test_schema_when_exported_to_temporary_directory(tmp_path: Path) -> None: + files = render() # Given + paths = write(tmp_path / "v2") # When + # Then + assert {p.name: p.read_bytes() for p in paths} == files + assert b'"additionalProperties": false' in files["RunManifestV2.json"] + + +def test_local_job_when_path_escapes_attempt_directory() -> None: + manifest = wrapper() # Given + data = dict( + job_version=1, + run_id=manifest.run_id(), + w=1, + manifest=manifest, + sample_ids=list(range(manifest.training.batch_samples())), + global_step0=0, + start_state_sha256=H, + ef_in_sha256=H, + v0_sha256=H, + object_paths={"state": "../state"}, + deadline=100, + ) + with pytest.raises(ValidationError): # When / Then + IslandJobV1.model_validate(data) + data["object_paths"] = {"state": "objects/state"} + assert IslandJobV1.model_validate(data).manifest == manifest + + +def test_receipts_when_distinct_commits_share_round() -> None: + # Given + intake = envelope_v2.Intake(RUN) + first = Receipt(w=3, commit_hash=H, received_round=50) + intake.accept(envelope_v2.seal(COLD, "Receipt", RUN, first, 100), 50) + second = Receipt(w=3, commit_hash="22" * 32, received_round=50) + # When + accepted = intake.accept(envelope_v2.seal(COLD, "Receipt", RUN, second, 100), 50) + # Then + assert accepted == second and len(intake.reservations) == 2 + restarted = envelope_v2.Intake(RUN, reservations=dict(intake.reservations)) + conflict = Receipt(w=3, commit_hash=H, received_round=51) + with pytest.raises(ReplayError): + restarted.accept(envelope_v2.seal(COLD, "Receipt", RUN, conflict, 100), 51) + + +def test_rotation_when_persisted_coldkey_authorizes_distinct_hotkeys() -> None: + # Given: service callback represents the persisted old coldkey, not the old hotkey. + new = Keypair(bytes([9]) * 32) + body = dict(hotkey=MINER.ss58, new_hotkey=new.ss58, operation_id=H) + intake = envelope_v2.Intake( + RUN, allowed_signers={"RotateRequest": lambda signer: signer == COLD.ss58} + ) + # When + accepted = intake.accept(envelope_v2.seal(COLD, "RotateRequest", RUN, body, 100), 50) + # Then + assert accepted.model_dump(mode="json") == body + with pytest.raises(UnauthorizedSigner): + intake.accept(envelope_v2.seal(MINER, "RotateRequest", RUN, body, 100), 50) + with pytest.raises(ReplayError): + intake.accept( + envelope_v2.seal(COLD, "RotateRequest", RUN, {**body, "new_hotkey": MINER.ss58}, 100), + 50, + ) + + +def audit_job_body(*, signer: Keypair = MINER, expiry: int = 100) -> dict[str, JsonValue]: + manifest = wrapper() + run = manifest.run_id() + start = StartStateV2( + run_id=run, + w=3, + hotkey=MINER.ss58, + theta_hash=H, + state_object_sha256=H, + opt_state_hash=H, + ef_object_sha256=H, + ef_hash=H, + parent_anchor_hash=H, + global_step0=0, + anchor_verdict_hash=H, + ) + challenge = AuditChallengeV2( + w=3, + target=MINER.ss58, + beacon_round=10, + beacon_sig_sha256=H, + mode="full", + segments=[], + reasons=["final"], + serve_deadline=100, + anchor_hash=start.digest(), + audit_mode="anchored-full", + ) + preimages = [ + LeafPreimage( + run_id=run, + w=3, + t=t, + stages=[dict(theta=H, m=H, v=H)], + batch_ids_sha256=H, + rng_ctr=0, + loss_f32="00000000", + norm_f32="00000000", + ).model_dump(mode="json") + for t in range(0, 31, 5) + ] + data: dict[str, JsonValue] = dict( + run_id=run, + job_id=H, + auditor_id=COLD.ss58, + attempt=1, + lease_nonce=H, + lease_expires=50, + absolute_deadline=100, + reservation_id=H, + replay_step_budget=30, + anchor_age=0, + manifest=manifest.body(), + challenge_envelope=envelope_v2.seal(COLD, "AuditChallengeV2", run, challenge, expiry), + commit_envelope=envelope_v2.seal( + signer, + "CommitV2", + run, + commit(manifest.training.batch_samples() * manifest.training.model.seq_len), + expiry, + ), + sample_ids=list(range(manifest.training.batch_samples())), + start_state=start.body(), + preimages=preimages, + ef_in=dict(sha256=H, size=1), + v0=dict(sha256=H, size=1), + created_beacon=10, + ) + return data + + +def test_audit_job_when_commit_signed_by_another_identity() -> None: + data = audit_job_body(signer=COLD) # Given + with pytest.raises(ValidationError): # When / Then + AuditJobV2.model_validate(data) + + +@pytest.mark.parametrize("expired_field", ["commit_envelope", "challenge_envelope"]) +def test_audit_job_when_embedded_record_expired_before_creation(expired_field: str) -> None: + # Given: no authenticated original acceptance evidence exists in this wire contract. + data = audit_job_body() + expired = audit_job_body(expiry=1) + data[expired_field] = expired[expired_field] + with pytest.raises(ValidationError): # When / Then + AuditJobV2.model_validate(data) + + +def test_audit_job_when_expiry_is_live_at_authenticated_creation() -> None: + data = audit_job_body(expiry=10) # Given + accepted = AuditJobV2.model_validate(data) # When + assert accepted.created_beacon == 10 # Then: inclusive expiry, not replay-time renewal. + + +def test_audit_intake_when_created_beacon_cannot_waive_live_expiry() -> None: + # Given: signed job created while records were live; no original acceptance receipt supplied. + data = audit_job_body(expiry=10) + raw = envelope_v2.seal(COLD, "AuditJobV2", data["run_id"], data, 100) + with pytest.raises(ValueError): # When / Then + envelope_v2.Intake(data["run_id"]).accept(raw, 11) + + +def test_audit_intake_when_all_embedded_signatures_are_live() -> None: + data = audit_job_body() # Given + raw = envelope_v2.seal(COLD, "AuditJobV2", data["run_id"], data, 100) + accepted = envelope_v2.Intake(data["run_id"]).accept(raw, 20) # When + assert isinstance(accepted, AuditJobV2) # Then + + +@pytest.mark.parametrize("value", ["0000807f", "0000c07f"]) +@pytest.mark.parametrize( + "section,field", + [ + ("model", "capacity_factor"), + ("model", "aux_loss_coef"), + ("model", "init_std"), + ("inner", "eps"), + ("inner", "wd"), + ("inner", "grad_clip"), + ("inner", "muon_momentum"), + ("outer", "lr"), + ("outer", "momentum"), + ("outer", "topk_frac"), + ("outer", "ef_beta"), + ("outer", "preclip_norm"), + ("outer", "cclip_tau"), + ("verify", "influence_cap"), + ], +) +def test_training_when_computational_f32_is_nonfinite(section: str, field: str, value: str) -> None: + body = wrapper().body() # Given + body["training"]["inner"]["opt"] = "muon" + body["training"][section][field] = value + with pytest.raises(ValidationError): # When / Then + RunManifestV2.model_validate(body) + + +@pytest.mark.parametrize( + "path", + [ + ["inner", "betas", 0], + ["inner", "betas", 1], + ["inner", "lr_schedule", "peak_lr"], + ["operator_budget", "honeypot", "rate"], + ], +) +def test_training_when_nested_f32_is_nonfinite(path: list[str | int]) -> None: + body = wrapper().body() # Given + body["training"][path[0]][path[1]][path[2]] = "0000807f" + with pytest.raises(ValidationError): # When / Then + RunManifestV2.model_validate(body) + + +@pytest.mark.parametrize("model", [EscrowLock, EscrowRelease, EscrowTransfer]) +def test_escrow_when_origin_reference_repeated( + model: type[EscrowLock] | type[EscrowRelease] | type[EscrowTransfer], +) -> None: + # Given + data = dict( + operation_id=H, owner=COLD.ss58, units=1, origin_ids=[H, H], admission_id=H, dispute_id=None + ) + if model is EscrowLock: + data.update(kind="LOCK_ADMISSION") + elif model is EscrowRelease: + data.update(lock_id=H, finality_hash=H, closed_dispute_root=H) + else: + data.update(recipient=MINER.ss58) + with pytest.raises(ValidationError): # When / Then + model.model_validate(data) + + +def test_roster_when_probability_is_nonfinite() -> None: + data = dict( + hotkey=MINER.ss58, + slot=0, + q_i="0000807f", + admission_id=H, + coldkey_group="group", + state="ACTIVE", + eligible_weight=0, + ) # Given + with pytest.raises(ValidationError): # When / Then + RosterEntryV2.model_validate(data) + + +@pytest.mark.parametrize("field", ["preclip_norm", "cclip_tau"]) +def test_aggregation_when_computational_f32_is_nonfinite(field: str) -> None: + data = dict( + arithmetic="flat-cclip-cap-v1", + order="utf8", + center="prev_outer_update", + trust_mode="uniform-verified", + weight_quantum=1 << 24, + miner_cap_units=1 << 22, + probation_cap_units=1 << 22, + owner_group_caps={}, + preclip_norm="0000803f", + cclip_tau="0000803f", + cclip_iters=1, + ) # Given + data[field] = "0000807f" + with pytest.raises(ValidationError): # When / Then + AggregationPolicyV2.model_validate(data) + + +@pytest.mark.parametrize("field", ["mask_ratio", "rps_weight", "distill_temp"]) +def test_od_when_objective_f32_is_nonfinite(field: str) -> None: + body = od_body() # Given + body["model"]["param_count"] = 116_288 + body["model"]["od"][field] = "0000c07f" + with pytest.raises(ValidationError): # When / Then + wrapper(body) + + +@pytest.mark.parametrize("field", ["loss_f32", "norm_f32"]) +def test_audit_job_when_metric_f32_is_nonfinite(field: str) -> None: + data = audit_job_body() # Given + data["preimages"][0][field] = "0000807f" + with pytest.raises(ValidationError): # When / Then + AuditJobV2.model_validate(data) + + +def test_v1_when_opaque_published_baseline_is_replayed(tmp_path: Path) -> None: + # Given: fixed captured bytes, never freshly sign or regenerate expected values. + fixture = json.loads((Path(__file__).parent / "fixtures/network_v1_baseline.json").read_bytes()) + for message in fixture["messages"]: + body_bytes = bytes.fromhex(message["body_hex"]) + envelope_bytes = bytes.fromhex(message["envelope_hex"]) + body, envelope = json.loads(body_bytes), json.loads(envelope_bytes) + # When / Then: parse the stored signature, compare the entire body/preimage/envelope. + model = envelope_v1.Intake(message["run_id"]).accept(envelope, 50) + assert canonicalize(model.model_dump(mode="json"), allow_float=False) == body_bytes + assert envelope_v1.signing_message( + message["type"], body, message["exp_drand"], message["run_id"] + ) == bytes.fromhex(message["preimage_hex"]) + assert canonicalize(envelope, allow_float=False) == envelope_bytes + assert envelope_v1.verify_envelope(envelope) + assert not envelope_v1.verify_envelope({**envelope, "sig": "0" * 128}) + assert canonicalize(example_manifest().body(), allow_float=False) == bytes.fromhex( + fixture["messages"][0]["body_hex"] + ) + leaf_bytes = bytes.fromhex(fixture["leaf"]["body_hex"]) + leaf = LeafPreimage.model_validate_json(leaf_bytes) + assert canonicalize(leaf.model_dump(mode="json"), allow_float=False) == leaf_bytes + assert leaf.digest() == fixture["leaf"]["digest"] + tape_fixture = fixture["tape"] + tape = json.loads(bytes.fromhex(tape_fixture["signed_hex"])) + store = LocalFSStore(tmp_path / "objects") + for key, value in tape_fixture["objects"].items(): + assert store.put(bytes.fromhex(value)) == key + # When: actual v1 signature verification and arithmetic replay, tiny two-element state. + result = replay_tape(store, tape, tape["signer"]) + # Then + assert verify_tape(tape) and not verify_tape({**tape, "sig": "0" * 128}) + assert canonicalize(tape["body"], allow_float=False) == bytes.fromhex(tape_fixture["body_hex"]) + assert tape_message(tape["body"]["run_id"], tape["body"]) == bytes.fromhex( + tape_fixture["preimage_hex"] + ) + assert tape_bytes(tape) == bytes.fromhex(tape_fixture["signed_hex"]) + assert sha256_hex(result.to_bytes()) == tape["body"]["out_state"] diff --git a/tests/relay/fixtures/kind_contract_docker.py b/tests/relay/fixtures/kind_contract_docker.py new file mode 100644 index 0000000..1305d4d --- /dev/null +++ b/tests/relay/fixtures/kind_contract_docker.py @@ -0,0 +1,106 @@ +"""Mock executable only: records pinned-kind argv, stops at first control-plane exec.""" + +import json +import os +import pathlib +import sys + +args = sys.argv[1:] +trace = pathlib.Path(os.environ["KIND_MOCK_TRACE"]) +state = pathlib.Path(os.environ["KIND_MOCK_STATE"]) +with trace.open("a") as stream: + stream.write(json.dumps(args) + "\n") +node = json.loads(state.read_text()) if state.exists() else None +op = args[0] +if op in {"-v", "--version", "version"}: + print("Docker version 29.1.3, build mocked") +elif op == "info": + print(json.dumps({"CgroupVersion": "2", "CgroupDriver": "systemd", "MemoryLimit": True})) +elif op == "ps": + if node: + print("hypertrain-v2-contract-control-plane") +elif op == "pull": + print("PINNED_PULL_MOCKED") +elif op == "network": + if args[1] == "ls": + print("c" * 64) + elif args[1] == "inspect": + if "-f" in args or "--format" in args: + print("172.18.0.0/16 " if "Subnet" in " ".join(args) else "1500") + else: + print( + json.dumps( + [ + { + "Id": "c" * 64, + "Name": "kind", + "Created": "2026-10-09T00:00:00Z", + "Labels": {}, + "Containers": {}, + "IPAM": {"Config": [{"Subnet": "172.18.0.0/16"}]}, + } + ] + ) + ) +elif op == "inspect": + if "--type=image" in args: + sys.exit(1) + if "--format" in args or "-f" in args: + fmt = args[args.index("--format") + 1] if "--format" in args else args[args.index("-f") + 1] + if "Role" in fmt or "role" in fmt: + print("control-plane") + elif "IPAddress" in fmt: + print("172.18.0.2,") + else: + print("1") + else: + assert node is not None + print(json.dumps([node])) +elif op == "create": + assert "--cpus=3.5" in args and "--memory=4g" in args and "--memory-swap=4g" in args + node = { + "Id": "a" * 64, + "State": {"Running": False, "Pid": 0}, + "HostConfig": { + "NanoCpus": 3500000000, + "Memory": 4 << 30, + "MemorySwap": 4 << 30, + "CpusetCpus": "0-3", + }, + "Config": { + "Labels": { + "io.hypertrain.proof": "hypertrain-v2-contract", + "io.x-k8s.kind.cluster": "hypertrain-v2-contract", + } + }, + } + state.write_text(json.dumps(node)) + print(node["Id"]) +elif op == "start": + journal = json.loads(pathlib.Path(os.environ["KIND_MOCK_JOURNAL"]).read_text()) + assert ( + journal["node_capped_before_start"] and journal["create_host_config"] == node["HostConfig"] + ) + node["State"]["Running"] = True + node["State"]["Pid"] = 12345 + state.write_text(json.dumps(node)) + print(node["Id"]) +elif op == "logs": + receipt = json.loads(pathlib.Path(os.environ["KIND_MOCK_JOURNAL"]).read_text()) + assert receipt["node_kernel_receipt"]["verified"] + print("Reached target Multi-User System.") + print("Reached target multi-user.target") +elif op == "exec": + receipt = json.loads(pathlib.Path(os.environ["KIND_MOCK_JOURNAL"]).read_text()) + assert receipt["node_kernel_receipt"]["verified"] + sys.stderr.write("MOCK_CONTROLPLANE_STOP_AFTER_CAPPED_START") + sys.exit(73) +elif op == "rm": + state.unlink(missing_ok=True) +elif op == "stop": + assert args[-1] == node["Id"] and args[1] == "--time=0" + node["State"] = {"Running": False, "Pid": 0} + state.write_text(json.dumps(node)) +else: + sys.stderr.write("UNIMPLEMENTED_MOCK_ARGV:" + json.dumps(args)) + sys.exit(74) diff --git a/tests/relay/fixtures/podman-network-list.json b/tests/relay/fixtures/podman-network-list.json new file mode 100644 index 0000000..d44ee3e --- /dev/null +++ b/tests/relay/fixtures/podman-network-list.json @@ -0,0 +1,251 @@ +[ + { + "name": "cortex_default", + "id": "1ff113a89f6a33ac1816bad08799111532b13ee2fa472359a1048711f087db67", + "driver": "bridge", + "network_interface": "podman2", + "created": "2026-09-30T01:57:13.7083285Z", + "subnets": [ + { + "subnet": "10.89.1.0/24", + "gateway": "10.89.1.1" + } + ], + "ipv6_enabled": false, + "internal": false, + "dns_enabled": true, + "labels": { + "com.docker.compose.project": "cortex", + "io.podman.compose.project": "cortex" + }, + "ipam_options": { + "driver": "host-local" + } + }, + { + "name": "kind", + "id": "58c6b9ecba025b0d27e3a87e77d24eb64efbaad7de92c7ab8d158ca2edab6de0", + "driver": "bridge", + "network_interface": "podman1", + "created": "2026-09-26T23:02:51.493664027Z", + "subnets": [ + { + "subnet": "fc00:f853:ccd:e793::/64", + "gateway": "fc00:f853:ccd:e793::1" + }, + { + "subnet": "10.89.0.0/24", + "gateway": "10.89.0.1" + } + ], + "ipv6_enabled": true, + "internal": false, + "dns_enabled": true, + "ipam_options": { + "driver": "host-local" + } + }, + { + "name": "podman", + "id": "2f259bab93aaaaa2542ba43ef33eb990d0999ee1b9924b557b7be53c0b7a1bb9", + "driver": "bridge", + "network_interface": "podman0", + "created": "2026-10-09T02:47:00.741104709Z", + "subnets": [ + { + "subnet": "10.88.0.0/16", + "gateway": "10.88.0.1" + } + ], + "ipv6_enabled": false, + "internal": false, + "dns_enabled": false, + "ipam_options": { + "driver": "host-local" + } + }, + { + "name": "pr446-7302-st01a106e1_default", + "id": "fd95587dbf25d02ca838a4e9be8e52c00dbdbae63a3bf9f94929236ece6715e1", + "driver": "bridge", + "network_interface": "podman10", + "created": "2026-10-04T15:57:14.084749884Z", + "subnets": [ + { + "subnet": "10.89.9.0/24", + "gateway": "10.89.9.1" + } + ], + "ipv6_enabled": false, + "internal": false, + "dns_enabled": true, + "labels": { + "com.docker.compose.project": "pr446-7302-st01a106e1", + "io.podman.compose.project": "pr446-7302-st01a106e1" + }, + "ipam_options": { + "driver": "host-local" + } + }, + { + "name": "pr446-final-st01a106e1_default", + "id": "706b34207639f082a6b721a393815e2d4827223786dc76ebb801bd6069f6049c", + "driver": "bridge", + "network_interface": "podman9", + "created": "2026-10-04T15:11:23.029601071Z", + "subnets": [ + { + "subnet": "10.89.8.0/24", + "gateway": "10.89.8.1" + } + ], + "ipv6_enabled": false, + "internal": false, + "dns_enabled": true, + "labels": { + "com.docker.compose.project": "pr446-final-st01a106e1", + "io.podman.compose.project": "pr446-final-st01a106e1" + }, + "ipam_options": { + "driver": "host-local" + } + }, + { + "name": "pr446-hard-st01a106e1_default", + "id": "6114fb5405fced752f56c70c6bb0a09b3a7dadd43bac424e112106c5697dc574", + "driver": "bridge", + "network_interface": "podman7", + "created": "2026-10-04T14:12:54.05607753Z", + "subnets": [ + { + "subnet": "10.89.6.0/24", + "gateway": "10.89.6.1" + } + ], + "ipv6_enabled": false, + "internal": false, + "dns_enabled": true, + "labels": { + "com.docker.compose.project": "pr446-hard-st01a106e1", + "io.podman.compose.project": "pr446-hard-st01a106e1" + }, + "ipam_options": { + "driver": "host-local" + } + }, + { + "name": "pr446-hard2-st01a106e1_default", + "id": "77ecef72f1325eb5d227e80b859c05eac63438718e4032bf4262130220f7947f", + "driver": "bridge", + "network_interface": "podman8", + "created": "2026-10-04T15:01:50.62069166Z", + "subnets": [ + { + "subnet": "10.89.7.0/24", + "gateway": "10.89.7.1" + } + ], + "ipv6_enabled": false, + "internal": false, + "dns_enabled": true, + "labels": { + "com.docker.compose.project": "pr446-hard2-st01a106e1", + "io.podman.compose.project": "pr446-hard2-st01a106e1" + }, + "ipam_options": { + "driver": "host-local" + } + }, + { + "name": "pr446-st01a106e1-3e50_default", + "id": "b90d1f5d4c639ae2c9fff03e9a1a1e7741a227771450f473e1e8899d4750205d", + "driver": "bridge", + "network_interface": "podman6", + "created": "2026-10-04T12:46:23.158727481Z", + "subnets": [ + { + "subnet": "10.89.5.0/24", + "gateway": "10.89.5.1" + } + ], + "ipv6_enabled": false, + "internal": false, + "dns_enabled": true, + "labels": { + "com.docker.compose.project": "pr446-st01a106e1-3e50", + "io.podman.compose.project": "pr446-st01a106e1-3e50" + }, + "ipam_options": { + "driver": "host-local" + } + }, + { + "name": "pr446-st01a106e1-archive_default", + "id": "e7406253d55dc305e846d45fd2293bee72e14a85bba2d8d7cd430d142f734ee8", + "driver": "bridge", + "network_interface": "podman5", + "created": "2026-10-04T12:35:58.995606259Z", + "subnets": [ + { + "subnet": "10.89.4.0/24", + "gateway": "10.89.4.1" + } + ], + "ipv6_enabled": false, + "internal": false, + "dns_enabled": true, + "labels": { + "com.docker.compose.project": "pr446-st01a106e1-archive", + "io.podman.compose.project": "pr446-st01a106e1-archive" + }, + "ipam_options": { + "driver": "host-local" + } + }, + { + "name": "pr446-st01a106e1-final_default", + "id": "0300b821481bc5b0e023f857253e384c63f8b27c08e4fab130e5fa5918cf911b", + "driver": "bridge", + "network_interface": "podman4", + "created": "2026-10-04T12:32:57.975149655Z", + "subnets": [ + { + "subnet": "10.89.3.0/24", + "gateway": "10.89.3.1" + } + ], + "ipv6_enabled": false, + "internal": false, + "dns_enabled": true, + "labels": { + "com.docker.compose.project": "pr446-st01a106e1-final", + "io.podman.compose.project": "pr446-st01a106e1-final" + }, + "ipam_options": { + "driver": "host-local" + } + }, + { + "name": "pr446-st01a106e1_default", + "id": "1a6c269841b2b0de14107e6863bfb5b634429eaf5e903f107c9a234c789b851a", + "driver": "bridge", + "network_interface": "podman3", + "created": "2026-10-04T12:30:11.852280791Z", + "subnets": [ + { + "subnet": "10.89.2.0/24", + "gateway": "10.89.2.1" + } + ], + "ipv6_enabled": false, + "internal": false, + "dns_enabled": true, + "labels": { + "com.docker.compose.project": "pr446-st01a106e1", + "io.podman.compose.project": "pr446-st01a106e1" + }, + "ipam_options": { + "driver": "host-local" + } + } +] diff --git a/tests/relay/fixtures/real-kind-hostconfig-normalization.json b/tests/relay/fixtures/real-kind-hostconfig-normalization.json new file mode 100644 index 0000000..5047180 --- /dev/null +++ b/tests/relay/fixtures/real-kind-hostconfig-normalization.json @@ -0,0 +1,174 @@ +{ + "before": { + "Binds": [ + "/lib/modules:/lib/modules:ro" + ], + "ContainerIDFile": "", + "LogConfig": { + "Type": "json-file", + "Config": {} + }, + "NetworkMode": "none", + "PortBindings": {}, + "RestartPolicy": { + "Name": "on-failure", + "MaximumRetryCount": 1 + }, + "AutoRemove": false, + "VolumeDriver": "", + "VolumesFrom": null, + "ConsoleSize": [ + 0, + 0 + ], + "CapAdd": null, + "CapDrop": null, + "CgroupnsMode": "private", + "Dns": null, + "DnsOptions": [], + "DnsSearch": [], + "ExtraHosts": null, + "GroupAdd": null, + "IpcMode": "private", + "Cgroup": "", + "Links": null, + "OomScoreAdj": 0, + "PidMode": "", + "Privileged": true, + "PublishAllPorts": false, + "ReadonlyRootfs": false, + "SecurityOpt": [ + "seccomp=unconfined", + "apparmor=unconfined", + "label=disable" + ], + "Tmpfs": { + "/run": "", + "/tmp": "" + }, + "UTSMode": "", + "UsernsMode": "", + "ShmSize": 67108864, + "Runtime": "runc", + "Isolation": "", + "CpuShares": 0, + "Memory": 4294967296, + "NanoCpus": 3500000000, + "CgroupParent": "", + "BlkioWeight": 0, + "BlkioWeightDevice": [], + "BlkioDeviceReadBps": [], + "BlkioDeviceWriteBps": [], + "BlkioDeviceReadIOps": [], + "BlkioDeviceWriteIOps": [], + "CpuPeriod": 0, + "CpuQuota": 0, + "CpuRealtimePeriod": 0, + "CpuRealtimeRuntime": 0, + "CpusetCpus": "0-3", + "CpusetMems": "", + "Devices": [], + "DeviceCgroupRules": null, + "DeviceRequests": null, + "MemoryReservation": 0, + "MemorySwap": 4294967296, + "MemorySwappiness": null, + "OomKillDisable": false, + "PidsLimit": null, + "Ulimits": [], + "CpuCount": 0, + "CpuPercent": 0, + "IOMaximumIOps": 0, + "IOMaximumBandwidth": 0, + "MaskedPaths": null, + "ReadonlyPaths": null, + "Init": false + }, + "after": { + "Binds": [ + "/lib/modules:/lib/modules:ro" + ], + "ContainerIDFile": "", + "LogConfig": { + "Type": "json-file", + "Config": {} + }, + "NetworkMode": "none", + "PortBindings": {}, + "RestartPolicy": { + "Name": "on-failure", + "MaximumRetryCount": 1 + }, + "AutoRemove": false, + "VolumeDriver": "", + "VolumesFrom": null, + "ConsoleSize": [ + 0, + 0 + ], + "CapAdd": null, + "CapDrop": null, + "CgroupnsMode": "private", + "Dns": null, + "DnsOptions": [], + "DnsSearch": [], + "ExtraHosts": null, + "GroupAdd": null, + "IpcMode": "private", + "Cgroup": "", + "Links": null, + "OomScoreAdj": 0, + "PidMode": "", + "Privileged": true, + "PublishAllPorts": false, + "ReadonlyRootfs": false, + "SecurityOpt": [ + "seccomp=unconfined", + "apparmor=unconfined", + "label=disable" + ], + "Tmpfs": { + "/run": "", + "/tmp": "" + }, + "UTSMode": "", + "UsernsMode": "", + "ShmSize": 67108864, + "Runtime": "runc", + "Isolation": "", + "CpuShares": 0, + "Memory": 4294967296, + "NanoCpus": 3500000000, + "CgroupParent": "", + "BlkioWeight": 0, + "BlkioWeightDevice": [], + "BlkioDeviceReadBps": [], + "BlkioDeviceWriteBps": [], + "BlkioDeviceReadIOps": [], + "BlkioDeviceWriteIOps": [], + "CpuPeriod": 0, + "CpuQuota": 0, + "CpuRealtimePeriod": 0, + "CpuRealtimeRuntime": 0, + "CpusetCpus": "0-3", + "CpusetMems": "", + "Devices": [], + "DeviceCgroupRules": null, + "DeviceRequests": null, + "MemoryReservation": 0, + "MemorySwap": 4294967296, + "MemorySwappiness": null, + "OomKillDisable": null, + "PidsLimit": null, + "Ulimits": [], + "CpuCount": 0, + "CpuPercent": 0, + "IOMaximumIOps": 0, + "IOMaximumBandwidth": 0, + "MaskedPaths": null, + "ReadonlyPaths": null, + "Init": false + }, + "source_before": "preserved/d3-poststart-real-5dc86-20261009/cleanup.json:create_host_config", + "source_after": "preserved/d3-poststart-real-5dc86-20261009/stopped-inspect.json:HostConfig" +} diff --git a/tests/relay/test_lifecycle_v2.py b/tests/relay/test_lifecycle_v2.py new file mode 100644 index 0000000..2dcb54c --- /dev/null +++ b/tests/relay/test_lifecycle_v2.py @@ -0,0 +1,676 @@ +"""Restart, signed retention, nonce, drain/key and exact custody failure checks.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +import anyio +import httpx +import pytest +from test_transport_v2 import MASTER, MINER, NEXT, H, Rig + +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.relay_messages import ( + AcceptedUploadAck, + AvailabilityContext, + ChunkCustodyAck, + CustodyRelease, + RelayAssignment, + RelayFailureEvidence, + RelayReceipt, + RetentionExtension, + RetrievalRequest, + RetrievalResponse, +) +from hypertrain.relay.app import create_app +from hypertrain.relay.client import receipt_verified +from hypertrain.relay.core import RelayError, Settlement + + +@pytest.mark.parametrize("scope", ["chunk", "receipt", "failed-delete"]) +def test_gc_when_new_same_hash_custody_races_physical_delete( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, scope: str +) -> None: + async def scenario() -> None: + from hypertrain.protocol import envelope_v2 + from hypertrain.protocol.relay_messages import UploadGrant + + async with httpx.AsyncClient() as client: + rig = Rig(tmp_path, client) + data = b"abc" if scope == "chunk" else b"a" * (4 << 20) + b"abc" + old, _, _ = await rig.accept(data) + for index, off in enumerate(range(0, len(data), 4 << 20)): + await rig.put(old, data[off : off + (4 << 20)], index) + record = (await rig.relay.state()).uploads[old.digest()] + rig.clock = 201 + + async def settlement(grant_hash: str) -> Settlement: + return Settlement(H, 100, H, 120, 110) + + rig.relay.settlement = settlement + for digest in record.pins: + if ( + scope == "receipt" + and digest != RelayReceipt.model_validate(record.receipt.body).digest() + ): + continue + await rig.relay.release( + rig.seal( + CustodyRelease( + grant_hash=old.digest(), + custody_hashes=[digest], + retention_hash=digest, + finality_hash=H, + vesting_beacon=110, + closed_dispute_root=H, + release_beacon=180, + ) + ) + ) + + # Subscribe before collect; only physical-delete scheduling is intercepted. + deleting, resume = anyio.Event(), anyio.Event() + physical_delete = rig.store.delete + collected: list[str] = [] + + async def paused_delete(sha: str) -> None: + if sha == old.delta_hash: + deleting.set() + await resume.wait() + if scope == "failed-delete": + raise OSError("physical deletion outcome uncertain") + await physical_delete(sha) + + monkeypatch.setattr(rig.store, "delete", paused_delete) + peer = rig.new_relay() + peer.active_key = "k2" + new, manifest, _, assignment, opened = rig.upload(data, nonce="78" * 32, w=2) + new = UploadGrant.model_validate({**new.body(), "retain_until": 400, "exp_drand": 250}) + assignment = rig.seal( + RelayAssignment.model_validate( + { + **assignment.body, + "exp_drand": 250, + } + ) + ) + opened = envelope_v2.parse_envelope( + envelope_v2.seal( + MASTER, + "RoundOpenV2", + opened.run_id, + {**opened.body, "d_upload": 245, "d_final": 260}, + 300, + ) + ) + await peer.accept(rig.seal(new), manifest, assignment, opened) + + async def collect() -> None: + if scope == "failed-delete": + with pytest.raises(OSError, match="outcome uncertain"): + await rig.relay.collect() + else: + collected.extend(await rig.relay.collect()) + + async def source(part: bytes): + yield part + + with anyio.fail_after(10): + async with anyio.create_task_group() as tasks: + tasks.start_soon(collect) + await deleting.wait() + # A fresh replica reads the durable deletion generation, not local locks. + fresh = rig.new_relay() + assert old.delta_hash in (await fresh.state()).deletions + try: + with pytest.raises(RelayError, match="deletion reservation"): + for index, off in enumerate(range(0, len(data), 4 << 20)): + await peer.chunk( + new.digest(), index, source(data[off : off + (4 << 20)]) + ) + target = (await fresh.state()).uploads[new.digest()] + assert target.receipt is None + if scope != "receipt": + assert not target.chunks + finally: + resume.set() + + if scope == "failed-delete": + assert old.delta_hash in (await peer.state()).deletions + with pytest.raises(RelayError, match="deletion reservation"): + await peer.chunk(new.digest(), 0, source(data)) + assert not (await peer.state()).uploads[new.digest()].chunks + return + assert old.delta_hash in collected + assert not rig.store.store._path(old.delta_hash).exists() + assert not (await peer.state()).deletions + # Reservation ends only after deletion; retry writes before acknowledging custody. + for index, off in enumerate(range(0, len(data), 4 << 20)): + await peer.chunk(new.digest(), index, source(data[off : off + (4 << 20)])) + assert (await peer.state()).uploads[new.digest()].receipt is not None + assert rig.store.store.get(new.delta_hash) == data + assert new.delta_hash not in await rig.relay.collect() + + anyio.run(scenario) + + +def test_restart_when_custody_persisted_before_forwarding(tmp_path: Path) -> None: + async def scenario() -> None: + async with httpx.AsyncClient() as client: + rig = Rig(tmp_path, client) + grant, _, opportunity = await rig.accept(b"abc") + chunk = await rig.put(grant, b"abc") + receipt = await rig.relay.complete(grant.digest()) + rig.relay = rig.new_relay() + await rig.relay.recover() + record = (await rig.relay.state()).uploads[grant.digest()] + assert (record.opportunity, record.chunks["0"], record.receipt) == ( + opportunity, + chunk, + receipt, + ) + assert record.master_acceptance is None + + anyio.run(scenario) + + +def test_recovery_when_complete_custody_misses_receipt_deadline(tmp_path: Path) -> None: + # Given: crash at the full-custody boundary before receipt assembly. + async def scenario() -> None: + async with httpx.AsyncClient() as client: + rig = Rig(tmp_path, client) + grant, _, _ = await rig.accept(b"abc") + await rig.put(grant, b"abc") + + def interrupted(state) -> None: + state.uploads[grant.digest()].receipt = None + + await rig.relay._change(interrupted) + chunk = (await rig.relay.state()).uploads[grant.digest()].chunks["0"] + rig.clock = 41 + # When: a restarted relay reconstructs at a beacon after upload cutoff. + relay = rig.new_relay() + await relay.recover() + response = await relay.retrieve(rig.retrieval(grant, chunk)) + # Then: no backdated receipt; exact retained chunk still serves. + record = (await relay.state()).uploads[grant.digest()] + assert record.receipt is None + assert record.forward_error == "RelayError" + assert RetrievalResponse.model_validate(response.body).status == "SERVED" + + anyio.run(scenario) + + +@pytest.mark.parametrize("fault", ["nonce", "key", "horizon", "scope"]) +def test_retrieval_when_wrong_nonce_key_or_custody(tmp_path: Path, fault: str) -> None: + async def scenario() -> None: + async with httpx.AsyncClient() as client: + rig = Rig(tmp_path, client) + grant, _, _ = await rig.accept(b"abc") + await rig.put(grant, b"abc") + receipt = await rig.relay.complete(grant.digest()) + original = rig.retrieval(grant, receipt) + request = RetrievalRequest.model_validate(original.body) + if fault == "nonce": + await rig.relay.retrieve(original) + altered = {**request.body(), "request_id": "78" * 32} + elif fault == "key": + altered = {**request.body(), "key_id": "k2"} + elif fault == "horizon": + altered = {**request.body(), "retention_hash": H} + else: + altered = {**request.body(), "object_or_chunk_hash": H} + with pytest.raises(RelayError): + await rig.relay.retrieve(rig.seal(RetrievalRequest.model_validate(altered))) + + anyio.run(scenario) + + +@pytest.mark.parametrize("fault", ["late", "forged", "previous", "sequence"]) +def test_extension_when_no_timely_authenticated_successor(tmp_path: Path, fault: str) -> None: + async def scenario() -> None: + async with httpx.AsyncClient() as client: + rig = Rig(tmp_path, client) + grant, _, _ = await rig.accept(b"abc") + await rig.put(grant, b"abc") + receipt = RelayReceipt.model_validate((await rig.relay.complete(grant.digest())).body) + rig.clock = 200 if fault == "late" else 190 + extension = RetentionExtension( + grant_hash=grant.digest(), + custody_hashes=[receipt.digest()], + seq=2 if fault == "sequence" else 1, + previous_retention_hash=H if fault == "previous" else receipt.digest(), + dispute_ids=[H], + issued_beacon=rig.clock, + retain_until=300, + ) + with pytest.raises(ValueError): + await rig.relay.retention( + rig.seal(extension, MINER if fault == "forged" else MASTER) + ) + pin = (await rig.relay.state()).uploads[grant.digest()].pins[receipt.digest()] + assert pin.horizon == 200 + assert not (await rig.relay.state()).disabled_keys + + anyio.run(scenario) + + +def test_retention_when_dispute_extends_before_original_expiry(tmp_path: Path) -> None: + async def scenario() -> None: + async with httpx.AsyncClient() as client: + rig = Rig(tmp_path, client) + grant, _, _ = await rig.accept(b"abc") + await rig.put(grant, b"abc") + receipt_raw = await rig.relay.complete(grant.digest()) + receipt = RelayReceipt.model_validate(receipt_raw.body) + rig.clock = 190 + extension = RetentionExtension( + grant_hash=grant.digest(), + custody_hashes=[receipt.digest()], + seq=1, + previous_retention_hash=receipt.digest(), + dispute_ids=[H], + issued_beacon=190, + retain_until=300, + ) + raw = rig.seal(extension) + ack = await rig.relay.retention(raw) + assert ack == await rig.new_relay().retention(raw) + rig.clock = 210 + response = await rig.relay.retrieve( + rig.retrieval(grant, receipt_raw, horizon=300, retention_hash=extension.digest()) + ) + assert RetrievalResponse.model_validate(response.body).status == "SERVED" + assert not await rig.relay.eligible_deletions() + + anyio.run(scenario) + + +def test_near_horizon_when_retrieval_uses_signed_retention_not_upload(tmp_path: Path) -> None: + async def scenario() -> None: + async with httpx.AsyncClient() as client: + rig = Rig(tmp_path, client) + grant, _, _ = await rig.accept(b"abc") + await rig.put(grant, b"abc") + receipt = await rig.relay.complete(grant.digest()) + rig.clock = 198 + raw = rig.retrieval(grant, receipt) + assert RetrievalRequest.model_validate(raw.body).deadline_beacon == 200 + assert ( + RetrievalResponse.model_validate((await rig.relay.retrieve(raw)).body).status + == "SERVED" + ) + + anyio.run(scenario) + + +def test_drain_when_existing_upload_can_finish_new_upload_rejects(tmp_path: Path) -> None: + async def scenario() -> None: + async with httpx.AsyncClient() as client: + rig = Rig(tmp_path, client) + grant, _, _ = await rig.accept(b"abc") + app = create_app(rig.relay, "t" * 32) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="https://relay.test" + ) as wire: + assert (await wire.post("/internal/drain")).status_code == 403 + response = await wire.post( + "/internal/drain", headers={"Authorization": "Bearer " + "t" * 32} + ) + assert response.status_code == 200 + assert (await wire.get("/readyz")).status_code == 503 + assert (await wire.get("/livez")).status_code == 200 + await rig.put(grant, b"abc") + _, manifest, raw, assignment, opened = rig.upload(b"next", nonce="56" * 32, w=2) + with pytest.raises(RelayError): + await rig.relay.accept(raw, manifest, assignment, opened) + assert (await rig.relay.state()).uploads[grant.digest()].receipt is not None + + anyio.run(scenario) + + +def test_rotation_when_old_receipt_remains_verifiable_new_key_signs(tmp_path: Path) -> None: + async def scenario() -> None: + async with httpx.AsyncClient() as client: + rig = Rig(tmp_path, client) + grant, _, _ = await rig.accept(b"abc") + await rig.put(grant, b"abc") + old = await rig.relay.complete(grant.digest()) + rig.clock = 90 + rig.relay.active_key = "k2" + _, manifest, raw, assignment, opened = rig.upload(b"next", nonce="56" * 32, w=2) + # Fresh signed upload deadlines, no historical acceptance backdating. + from hypertrain.protocol import envelope_v2 + from hypertrain.protocol.relay_messages import UploadGrant + + fresh = UploadGrant.model_validate({**raw.body, "exp_drand": 120}) + raw = rig.seal(fresh) + assignment_body = RelayAssignment.model_validate({**assignment.body, "exp_drand": 120}) + opened_body = {**opened.body, "d_upload": 115, "d_final": 130} + opened = envelope_v2.parse_envelope( + envelope_v2.seal(MASTER, "RoundOpenV2", opened.run_id, opened_body, 150) + ) + ack = await rig.relay.accept(raw, manifest, rig.seal(assignment_body), opened) + assert ack.signer == NEXT.ss58 + receipt_verified( + old, + registry=rig.registry, + relay_id="eu-1", + run_id=old.run_id, + grant=grant, + received_before=35, + ) + + anyio.run(scenario) + + +def test_release_when_unresolved_dispute_blocks_deletion(tmp_path: Path) -> None: + async def scenario() -> None: + async with httpx.AsyncClient() as client: + rig = Rig(tmp_path, client) + grant, _, _ = await rig.accept(b"abc") + await rig.put(grant, b"abc") + receipt = RelayReceipt.model_validate((await rig.relay.complete(grant.digest())).body) + rig.clock = 190 + raw = rig.seal( + CustodyRelease( + grant_hash=grant.digest(), + custody_hashes=[receipt.digest()], + retention_hash=receipt.digest(), + finality_hash=H, + vesting_beacon=100, + closed_dispute_root=H, + release_beacon=180, + ) + ) + + async def settlement(grant_hash: str) -> Settlement: + assert grant_hash == grant.digest() + return Settlement(H, 100, H, None, 100) + + rig.relay.settlement = settlement + with pytest.raises(ValueError): + await rig.relay.release(raw) + assert not await rig.relay.eligible_deletions() + + anyio.run(scenario) + + +def test_release_when_closed_graced_horizons_expire_only_then_eligible(tmp_path: Path) -> None: + async def scenario() -> None: + async with httpx.AsyncClient() as client: + rig = Rig(tmp_path, client) + grant, _, _ = await rig.accept(b"abc") + await rig.put(grant, b"abc") + record = (await rig.relay.state()).uploads[grant.digest()] + rig.clock = 201 + + async def settlement(grant_hash: str) -> Settlement: + return Settlement(H, 100, H, 120, 110) + + rig.relay.settlement = settlement + for digest in record.pins: + await rig.relay.release( + rig.seal( + CustodyRelease( + grant_hash=grant.digest(), + custody_hashes=[digest], + retention_hash=digest, + finality_hash=H, + vesting_beacon=110, + closed_dispute_root=H, + release_beacon=180, + ) + ) + ) + assert await rig.relay.eligible_deletions() == [grant.delta_hash] + + anyio.run(scenario) + + +@pytest.mark.parametrize("full", [False, True]) +def test_failure_when_completion_requires_full_custody_not_miner_assertions( + tmp_path: Path, full: bool +) -> None: + async def scenario() -> None: + async with httpx.AsyncClient() as client: + rig = Rig(tmp_path, client) + data = b"a" * (4 << 20) + b"suffix" + grant, _, _ = await rig.accept(data) + await rig.put(grant, data[: 4 << 20]) + if full: + await rig.put(grant, b"suffix", 1) + record = (await rig.relay.state()).uploads[grant.digest()] + opportunity = AcceptedUploadAck.model_validate(record.opportunity.body) + artifacts = list(record.chunks.values()) + evidence = RelayFailureEvidence( + run_id=rig.relay.run_id, + relay_id="eu-1", + key_id="k1", + assignment_hash=RelayAssignment.model_validate(record.assignment.body).digest(), + grant_hash=grant.digest(), + upload_ack_hash=opportunity.digest(), + chunk_manifest_hash=grant.chunk_manifest_hash, + chunk_ack_hashes=[ + ChunkCustodyAck.model_validate(e.body).digest() for e in artifacts + ], + receipt_hash=None, + retention_hash=( + hashlib.sha256( + canonicalize( + sorted( + ChunkCustodyAck.model_validate(e.body).digest() for e in artifacts + ), + allow_float=False, + ) + ).hexdigest() + if full + else ChunkCustodyAck.model_validate(artifacts[0].body).digest() + ), + request=None, + response=None, + observer_ids=[MASTER.ss58, MINER.ss58], + observed_beacons=[36, 36], + result="CONTRACTUAL_SERVICE_FAILURE", + availability_context=AvailabilityContext( + master_healthy=True, + beacon_healthy=True, + reference_healthy=True, + observer_healthy=True, + ), + evidence_hash=hashlib.sha256( + canonicalize([e.model_dump(mode="json") for e in artifacts], allow_float=False) + ).hexdigest(), + ) + rig.clock = 36 + if full: + await rig.relay.failure( + rig.seal(evidence), artifacts, [rig.seal(evidence), rig.seal(evidence, MINER)] + ) + assert (await rig.relay.state()).disabled_keys == ["k1"] + else: + with pytest.raises(RelayError): + await rig.relay.failure( + rig.seal(evidence), + artifacts, + [rig.seal(evidence), rig.seal(evidence, MINER)], + ) + assert not (await rig.relay.state()).disabled_keys + + anyio.run(scenario) + + +def test_missing_prefix_when_post_upload_retrieval_only_faults_acknowledged_bytes( + tmp_path: Path, +) -> None: + async def scenario() -> None: + async with httpx.AsyncClient() as client: + rig = Rig(tmp_path, client) + payload = b"a" * (4 << 20) + b"unreceived-suffix" + grant, _, _ = await rig.accept(payload) + custody = await rig.put(grant, payload[: 4 << 20]) + ack = ChunkCustodyAck.model_validate(custody.body) + rig.clock = 41 + rig.store.store._path(ack.chunk_sha256).unlink() + request_raw = rig.retrieval(grant, custody) + response_raw = await rig.relay.retrieve(request_raw) + response = RetrievalResponse.model_validate(response_raw.body) + assert response.status == "NOT_FOUND" + record = (await rig.relay.state()).uploads[grant.digest()] + opportunity = AcceptedUploadAck.model_validate(record.opportunity.body) + artifacts = [custody, request_raw, response_raw] + evidence = RelayFailureEvidence( + run_id=rig.relay.run_id, + relay_id="eu-1", + key_id="k1", + assignment_hash=RelayAssignment.model_validate(record.assignment.body).digest(), + grant_hash=grant.digest(), + upload_ack_hash=opportunity.digest(), + chunk_manifest_hash=grant.chunk_manifest_hash, + chunk_ack_hashes=[ack.digest()], + receipt_hash=None, + retention_hash=ack.digest(), + request=RetrievalRequest.model_validate(request_raw.body), + response=response, + observer_ids=[MASTER.ss58, MINER.ss58], + observed_beacons=[52, 52], + result="CONTRACTUAL_SERVICE_FAILURE", + availability_context=AvailabilityContext( + master_healthy=True, + beacon_healthy=True, + reference_healthy=True, + observer_healthy=True, + ), + evidence_hash=hashlib.sha256( + canonicalize([e.model_dump(mode="json") for e in artifacts], allow_float=False) + ).hexdigest(), + ) + rig.clock = 52 + with pytest.raises(ValueError): + await rig.relay.failure(rig.seal(evidence), artifacts, []) + assert record.receipt is None + framed = RelayFailureEvidence.model_validate({**evidence.body(), "retention_hash": H}) + with pytest.raises(RelayError): + await rig.relay.failure( + rig.seal(framed), artifacts, [rig.seal(framed), rig.seal(framed, MINER)] + ) + await rig.relay.failure( + rig.seal(evidence), artifacts, [rig.seal(evidence), rig.seal(evidence, MINER)] + ) + saved = (await rig.relay.state()).failures[evidence.digest()] + assert saved.body["receipt_hash"] is None + assert saved.body["chunk_ack_hashes"] == [ack.digest()] + + anyio.run(scenario) + + +def test_forward_unavailable_when_custody_still_recovers_without_fake_acceptance( + tmp_path: Path, +) -> None: + async def scenario() -> None: + async with httpx.AsyncClient() as client: + rig = Rig(tmp_path, client) + grant, _, _ = await rig.accept(b"abc") + + async def failed(raw): + raise httpx.ConnectError("master unavailable") + + rig.relay.forward = failed + with pytest.raises(httpx.ConnectError): + await rig.put(grant, b"abc") + await rig.relay.recover() + record = (await rig.relay.state()).uploads[grant.digest()] + assert record.receipt is not None + assert record.master_acceptance is None + assert record.forward_error == "ConnectError" + assert ( + await rig.relay.retrieve(rig.retrieval(grant, record.receipt)) + ).type == "RetrievalResponse" + + anyio.run(scenario) + + +def test_ready_when_durable_probe_and_key_valid(tmp_path: Path) -> None: + async def scenario() -> None: + async with httpx.AsyncClient() as client: + rig = Rig(tmp_path, client) + assert await rig.relay.ready() + rig.clock = 101 + with pytest.raises(RelayError): + await rig.relay.ready() + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(rig.relay, "t" * 32)), + base_url="https://relay.test", + ) as wire: + assert (await wire.get("/readyz")).status_code == 503 + + anyio.run(scenario) + + +def test_ready_when_shared_upload_capacity_exhausted(tmp_path: Path) -> None: + # Given: a durable outstanding upload exhausts the shared reservation quota. + async def scenario() -> None: + async with httpx.AsyncClient() as client: + rig = Rig(tmp_path, client) + rig.relay.max_uploads = 1 + await rig.accept(b"abc") + # When: readiness is evaluated without any active request in this process. + ready = await rig.relay.ready() + # Then: new traffic is not routed into an exhausted replica. + assert not ready + + anyio.run(scenario) + + +def test_retrieval_when_release_races_durable_read(tmp_path: Path) -> None: + # Given: an authenticated chunk challenge is in flight while finality releases custody. + async def scenario() -> None: + async with httpx.AsyncClient() as client: + rig = Rig(tmp_path, client) + grant, _, _ = await rig.accept(b"abc") + chunk = await rig.put(grant, b"abc") + custody = ChunkCustodyAck.model_validate(chunk.body) + request = rig.retrieval(grant, chunk) + entered, resume = anyio.Event(), anyio.Event() + original = rig.store.get + + async def interrupted(sha, size, file): + await original(sha, size, file) + entered.set() + await resume.wait() + + rig.store.get = interrupted + + async def settlement(grant_hash: str) -> Settlement: + return Settlement(H, 1, H, 1, 1) + + rig.relay.settlement = settlement + + async def read() -> None: + with pytest.raises(ValueError): + await rig.relay.retrieve(request) + + # When: the persisted release lands after the backing read starts. + with anyio.fail_after(5): + async with anyio.create_task_group() as tasks: + tasks.start_soon(read) + await entered.wait() + await rig.relay.release( + rig.seal( + CustodyRelease( + grant_hash=grant.digest(), + custody_hashes=[custody.digest()], + retention_hash=custody.digest(), + finality_hash=H, + vesting_beacon=1, + closed_dispute_root=H, + release_beacon=11, + ) + ) + ) + resume.set() + # Then: no nonce/response reservation is minted against released custody. + assert not (await rig.relay.state()).uploads[grant.digest()].retrievals + + anyio.run(scenario) diff --git a/tests/relay/test_transport_v2.py b/tests/relay/test_transport_v2.py new file mode 100644 index 0000000..2ad3c3d --- /dev/null +++ b/tests/relay/test_transport_v2.py @@ -0,0 +1,2960 @@ +"""Real relay HTTP/storage integration; tiny objects except exact chunk-boundary test.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +import anyio +import httpx +import pytest + +from hypertrain.data.store import CorruptObjectError, LocalFSStore +from hypertrain.data.stream_store import StreamStore, blocks, spool +from hypertrain.protocol import envelope_v2, relay_envelope +from hypertrain.protocol.envelope import SignatureError +from hypertrain.protocol.keys import Keypair +from hypertrain.protocol.messages_v2 import RoundOpenV2 +from hypertrain.protocol.relay_envelope import RelayEnvelope +from hypertrain.protocol.relay_messages import ( + AcceptedUploadAck, + ChunkCustodyAck, + RelayAssignment, + RelayKey, + RelayReceipt, + RelayRegistryV1, + RelaySpec, + RetrievalRequest, + RetrievalResponse, + UploadChunk, + UploadChunkManifest, + UploadGrant, + has_full_custody, +) +from hypertrain.relay.app import create_app +from hypertrain.relay.client import RelayClient, receipt_verified +from hypertrain.relay.core import Relay, RelayError + +RUN = "ab" * 32 +MASTER = Keypair(bytes([1]) * 32) +MINER = Keypair(bytes([2]) * 32) +RELAY = Keypair(bytes([3]) * 32) +NEXT = Keypair(bytes([4]) * 32) +H = "12" * 32 + + +def test_shipped_gateway_target_port_policy(tmp_path): + import subprocess + + import yaml + + root = Path(__file__).resolve().parents[2] + fixtures = list(yaml.safe_load_all((root / "deploy/k8s/local/fixtures.yaml").read_text())) + gateway = next( + d for d in fixtures if d["kind"] == "Service" and d["metadata"]["name"] == "gateway" + ) + target = gateway["spec"]["ports"][0]["targetPort"] + assert gateway["spec"]["ports"][0]["port"] == 443 and target == 8443 + for overlay in ("local", "eu", "us", "apac"): + rendered = subprocess.run( + ["kubectl", "kustomize", str(root / "deploy/k8s/overlays" / overlay)], + capture_output=True, + text=True, + check=True, + timeout=15, + ).stdout + (tmp_path / (overlay + ".yaml")).write_text(rendered) + documents = list(yaml.safe_load_all(rendered)) + policy = next( + d + for d in documents + if d["kind"] == "NetworkPolicy" and d["metadata"]["name"] == "relay-authorized-flows" + ) + assert policy["metadata"]["namespace"] == "hypertrain-relay-" + overlay + assert policy["spec"]["podSelector"] == {"matchLabels": {"app": "hypertrain-relay"}} + rules = [ + r + for r in policy["spec"]["egress"] + if not r.get("ports") + or any( + p.get("protocol", "TCP") == "TCP" and p.get("port", target) == target + for p in r["ports"] + ) + ] + assert rules == [ + { + "to": [ + { + "namespaceSelector": { + "matchLabels": {"hypertrain.network/role": "gateway"} + }, + "podSelector": {"matchLabels": {"hypertrain.network/role": "gateway"}}, + } + ], + "ports": [{"protocol": "TCP", "port": target}], + } + ] + for namespace_role, pod_role in ( + ("gateway", "gateway"), + ("gateway", "store"), + ("store", "gateway"), + ("store", "store"), + ): + allowed = any( + peer["namespaceSelector"]["matchLabels"]["hypertrain.network/role"] + == namespace_role + and peer["podSelector"]["matchLabels"]["hypertrain.network/role"] == pod_role + for rule in rules + for peer in rule["to"] + ) + assert allowed == (namespace_role == pod_role == "gateway") + assert any({"protocol": "TCP", "port": 443} in r["ports"] for r in policy["spec"]["egress"]) + deny = next( + d + for d in documents + if d["kind"] == "NetworkPolicy" and d["metadata"]["name"] == "relay-deny-default" + ) + assert deny["spec"] == {"podSelector": {}, "policyTypes": ["Ingress", "Egress"]} + + +class Rig: + """Mutable injected beacon; every other boundary uses actual protocol/storage.""" + + def __init__(self, path: Path, client: httpx.AsyncClient) -> None: + self.path, self.clock, self.client = path, 20, client + self.registry = RelayRegistryV1( + registry_version=1, + epoch=0, + previous_registry_hash=None, + specs=[ + RelaySpec( + id="eu-1", + region="eu", + https_url="https://relay.test", + pubkeys=[ + RelayKey( + key_id="k1", + pubkey=RELAY.ss58, + valid_from_round=1, + valid_until_round=100, + ), + RelayKey( + key_id="k2", + pubkey=NEXT.ss58, + valid_from_round=80, + valid_until_round=500, + ), + ], + max_object_bytes=16 << 20, + max_inflight_bytes=32 << 20, + codecs=["ht-sparse-v1"], + mode="transport", + ) + ], + ) + self.store = StreamStore(LocalFSStore(path), client) + self.relay = self.new_relay() + + async def now(self) -> int: + return self.clock + + def new_relay(self) -> Relay: + return Relay( + run_id=RUN, + master=MASTER.ss58, + registry=self.registry, + network_manifest_hash=H, + observers={MASTER.ss58, MINER.ss58}, + relay_id="eu-1", + region="eu", + keys={"k1": RELAY, "k2": NEXT}, + active_key="k1", + backing=self.store, + now=self.now, + epoch_rounds=10, + ) + + def seal(self, model, key: Keypair = MASTER, exp: int = 500) -> RelayEnvelope: + return relay_envelope.parse_envelope( + relay_envelope.seal(key, type(model).__name__, RUN, model, exp) + ) + + def upload(self, data: bytes, *, nonce: str = H, w: int = 1): + chunks = [ + UploadChunk( + index=i, + off=off, + len=len(data[off : off + (4 << 20)]), + chunk_sha256=hashlib.sha256(data[off : off + (4 << 20)]).hexdigest(), + ) + for i, off in enumerate(range(0, len(data), 4 << 20)) + ] + manifest = UploadChunkManifest(size=len(data), chunks=chunks) + grant = UploadGrant( + w=w, + hotkey=MINER.ss58, + relay_id="eu-1", + assignment_epoch=0, + delta_hash=hashlib.sha256(data).hexdigest(), + size=len(data), + chunk_manifest_hash=manifest.chunk_manifest_hash(), + retain_until=200, + nonce=nonce, + exp_drand=40, + ) + assignment = RelayAssignment( + w=w, + hotkey=MINER.ss58, + primary_id="eu-1", + fallback_ids=[], + assignment_epoch=0, + manifest_hash=H, + exp_drand=40, + ) + opened = RoundOpenV2.model_validate( + { + "w": w, + **{ + k: H + for k in ( + "prev_final_hash", + "theta_hash", + "outer_state_hash", + "center_hash", + "roster_hash", + "honeypot_commit", + "start_state_index_hash", + ) + }, + "d_open": 1, + "d_assign": 2, + "d_commit": 10, + "d_audit": 15, + "d_upload": 35, + "d_final": 50, + "contract_version": 2, + "policy_hashes": { + k: H + for k in ( + "admission_policy_hash", + "economics_policy_hash", + "aggregation_policy_hash", + "dispute_policy_hash", + "audit_policy_hash", + ) + }, + "registry_epoch": 0, + "audit_mode": "anchored-full", + "roster": [], + } + ) + raw_round = envelope_v2.parse_envelope( + envelope_v2.seal(MASTER, "RoundOpenV2", RUN, opened, 50) + ) + return grant, manifest, self.seal(grant), self.seal(assignment), raw_round + + async def accept(self, data: bytes): + grant, manifest, raw, assignment, opened = self.upload(data) + opportunity = await self.relay.accept(raw, manifest, assignment, opened) + return grant, manifest, opportunity + + async def put(self, grant: UploadGrant, data: bytes, index: int = 0) -> RelayEnvelope: + async def source(): + yield data + + return await self.relay.chunk(grant.digest(), index, source()) + + def retrieval( + self, + grant: UploadGrant, + custody: RelayEnvelope, + *, + requested: int | None = None, + nonce: str = "45" * 32, + horizon: int = 200, + retention_hash: str | None = None, + ) -> RelayEnvelope: + stamp = self.clock if requested is None else requested + if custody.type == "RelayReceipt": + body = RelayReceipt.model_validate(custody.body) + hashes, receipt_hash, sha, size = ( + [], + body.digest(), + body.delta_hash, + body.size, + ) + else: + body = ChunkCustodyAck.model_validate(custody.body) + hashes, receipt_hash, sha, size = ( + [body.digest()], + None, + body.chunk_sha256, + body.len, + ) + assignment = RelayAssignment.model_validate(self.upload(b"a")[3].body) + request = RetrievalRequest( + request_id=nonce, + nonce=nonce, + relay_id="eu-1", + key_id="k1", + assignment_hash=assignment.digest(), + grant_hash=grant.digest(), + custody_ack_hashes=hashes, + receipt_hash=receipt_hash, + retention_hash=retention_hash or body.digest(), + object_or_chunk_hash=sha, + size=size, + requested_beacon=stamp, + deadline_beacon=min(stamp + 10, horizon), + ) + return self.seal(request) + + +def test_original_payload_when_http_upload_and_master_retrieval(tmp_path: Path) -> None: + async def scenario() -> None: + async with httpx.AsyncClient() as backing: + rig = Rig(tmp_path / "objects", backing) + app = create_app(rig.relay, "d" * 32) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="https://relay.test" + ) as wire: + client = RelayClient(rig.registry, wire) + payload = b"original sparse bytes\x00\xff" * 300 + path = tmp_path / "delta" + path.write_bytes(payload) + grant, _, raw, assignment, opened = rig.upload(payload) + receipt = await client.upload( + path, + grant_raw=raw, + assignment=assignment, + round_open=opened.model_dump(mode="json"), + ) + rig.clock = 41 # retrieval survives upload cutoff + with spool() as file: + response = await client.retrieval( + rig.retrieval(grant, receipt), file, relay_public=RELAY.ss58 + ) + assert file.read() == payload + assert RetrievalResponse.model_validate(response.body).status == "SERVED" + assert receipt == await rig.relay.complete(grant.digest()) + + anyio.run(scenario) + + +@pytest.mark.parametrize("fault", ["signature", "region", "epoch", "expired", "run", "manifest"]) +def test_grant_rejected_when_not_authorized(tmp_path: Path, fault: str) -> None: + async def scenario() -> None: + async with httpx.AsyncClient() as wire: + rig = Rig(tmp_path, wire) + grant, manifest, raw, assignment, opened = rig.upload(b"abc") + if fault == "signature": + raw = rig.seal(grant, RELAY) + elif fault == "region": + raw = rig.seal(UploadGrant.model_validate({**grant.body(), "relay_id": "us-1"})) + elif fault == "epoch": + raw = rig.seal(UploadGrant.model_validate({**grant.body(), "assignment_epoch": 1})) + elif fault == "expired": + rig.clock = 41 + elif fault == "run": + raw = relay_envelope.parse_envelope( + relay_envelope.seal(MASTER, "UploadGrant", "cd" * 32, grant, 500) + ) + else: + manifest = UploadChunkManifest( + size=3, chunks=[UploadChunk(index=0, off=0, len=3, chunk_sha256=H)] + ) + with pytest.raises((RelayError, SignatureError, ValueError)): + await rig.relay.accept(raw, manifest, assignment, opened) + assert not (await rig.relay.state()).uploads + + anyio.run(scenario) + + +@pytest.mark.parametrize("payload", [b"ab", b"abcd", b"xyz"]) +def test_chunk_rejected_when_size_or_hash_differs(tmp_path: Path, payload: bytes) -> None: + async def scenario() -> None: + async with httpx.AsyncClient() as wire: + rig = Rig(tmp_path, wire) + grant, _, _ = await rig.accept(b"abc") + with pytest.raises(CorruptObjectError): + await rig.put(grant, payload) + assert not (await rig.relay.state()).uploads[grant.digest()].chunks + + anyio.run(scenario) + + +def test_full_coverage_when_miners_stop_after_opportunity_or_prefix( + tmp_path: Path, +) -> None: + async def scenario() -> None: + async with httpx.AsyncClient() as wire: + rig = Rig(tmp_path, wire) + payload = b"a" * (4 << 20) + b"suffix" + grant, manifest, raw = await rig.accept(payload) + opportunity = AcceptedUploadAck.model_validate(raw.body) + assert not has_full_custody(grant, manifest, opportunity, []) + prefix_raw = await rig.put(grant, payload[: 4 << 20]) + prefix = ChunkCustodyAck.model_validate(prefix_raw.body) + assert not has_full_custody(grant, manifest, opportunity, [prefix]) + with pytest.raises(RelayError): + await rig.relay.complete(grant.digest()) + rig.clock = 36 + assert not has_full_custody(grant, manifest, opportunity, [prefix, prefix]) + assert not (await rig.relay.state()).disabled_keys + + anyio.run(scenario) + + +def test_automatic_completion_when_last_chunk_has_custody(tmp_path: Path) -> None: + async def scenario() -> None: + async with httpx.AsyncClient() as wire: + rig = Rig(tmp_path, wire) + payload = b"a" * (4 << 20) + b"suffix" + grant, manifest, _ = await rig.accept(payload) + await rig.put(grant, payload[: 4 << 20]) + await rig.put(grant, payload[4 << 20 :], 1) + record = (await rig.relay.state()).uploads[grant.digest()] + assert record.receipt is not None + assert has_full_custody( + grant, + manifest, + AcceptedUploadAck.model_validate(record.opportunity.body), + [ChunkCustodyAck.model_validate(e.body) for e in record.chunks.values()], + ) + with spool() as file: + await rig.store.get(grant.delta_hash, grant.size, file) + assert file.read() == payload + + anyio.run(scenario) + + +def test_replicas_when_racing_same_upload_keep_original_receipts( + tmp_path: Path, +) -> None: + async def scenario() -> None: + async with httpx.AsyncClient() as wire: + rig = Rig(tmp_path, wire) + peer = rig.new_relay() + grant, manifest, raw, assignment, opened = rig.upload(b"abc") + started = anyio.Event() + outputs: list[RelayEnvelope] = [] + + async def accept(relay: Relay) -> None: + await started.wait() + outputs.append(await relay.accept(raw, manifest, assignment, opened)) + + async with anyio.create_task_group() as tasks: + tasks.start_soon(accept, rig.relay) + tasks.start_soon(accept, peer) + started.set() + assert outputs[0] == outputs[1] + chunk = await rig.put(grant, b"abc") + with spool() as file: + file.write(b"abc") + file.seek(0) + assert chunk == await peer.chunk(grant.digest(), 0, blocks(file)) + assert len((await peer.state()).uploads) == 1 + assert (await peer.state()).uploads[grant.digest()].receipt == await rig.relay.complete( + grant.digest() + ) + + anyio.run(scenario) + + +def test_forged_receipt_when_master_verifies_durable_claim(tmp_path: Path) -> None: + async def scenario() -> None: + async with httpx.AsyncClient() as wire: + rig = Rig(tmp_path, wire) + grant, _, _ = await rig.accept(b"abc") + await rig.put(grant, b"abc") + original = await rig.relay.complete(grant.digest()) + fake = rig.seal(RelayReceipt.model_validate(original.body), MINER) + with pytest.raises(SignatureError): + receipt_verified( + fake, + registry=rig.registry, + relay_id="eu-1", + run_id=RUN, + grant=grant, + received_before=35, + ) + assert (await rig.relay.state()).uploads[grant.digest()].master_acceptance is None + + anyio.run(scenario) + + +def test_s3_streaming_when_conditional_metadata_and_durable_chunks( + tmp_path: Path, +) -> None: + async def scenario() -> None: + import importlib.util + import sys + + spec = importlib.util.spec_from_file_location( + "relay_smoke_fixture", + Path(__file__).parents[2] / "scripts/relay_kind_smoke.py", + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + secret_dir = tmp_path / "secret" + secret_dir.mkdir() + secret = secret_dir / "object.json" + secret.write_text('{"access_key_id":"test","secret_access_key":"secret"}') + secret.chmod(0o600) + module.FIXTURE = secret_dir + # The fixture stores under an isolated injected path; never touches cluster. + module.OBJECT_ROOT = tmp_path / "durable" + from hypertrain.data.store import S3Credentials, S3Store + + app = module.fixture_store() + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://objects.test" + ) as wire: + store = StreamStore( + S3Store( + "http://objects.test", + "proof", + S3Credentials.from_secret_file(secret), + region="local", + prefix="relay/eu/", + ), + wire, + ) + rig = Rig(tmp_path / "unused", wire) + rig.store = store + rig.relay.backing = store + grant, _, opportunity = await rig.accept(b"abc") + await rig.put(grant, b"abc") + receipt = await rig.relay.complete(grant.digest()) + peer = rig.new_relay() + peer.backing = store + state = await peer.state() + assert state.uploads[grant.digest()].opportunity == opportunity + assert state.uploads[grant.digest()].receipt == receipt + with spool() as file: + await store.get(grant.delta_hash, grant.size, file) + assert file.read() == b"abc" + + anyio.run(scenario) + + +def test_smoke_master_when_restart_preserves_grant_and_original_acceptance( + tmp_path: Path, +) -> None: + # Given: real SigV4 backing and a SQLite transport master in isolated fixture roots. + async def scenario() -> None: + import importlib.util + import sys + + spec = importlib.util.spec_from_file_location( + "relay_master_fixture", + Path(__file__).parents[2] / "scripts/relay_kind_smoke.py", + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + secret = tmp_path / "secrets" + secret.mkdir() + (secret / "master.seed").write_text((bytes([1]) * 32).hex()) + module.FIXTURE = secret + module.MASTER_ROOT = tmp_path / "master" + module.RUN = RUN + async with httpx.AsyncClient() as backing: + rig = Rig(tmp_path / "objects", backing) + module.registry = lambda: rig.registry + module.backing = lambda client, region: rig.store + grant, _, raw, _, _ = rig.upload(b"abc") + await rig.accept(b"abc") + await rig.put(grant, b"abc") + receipt = await rig.relay.complete(grant.digest()) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=module.fixture_master()), + base_url="https://master.test", + ) as first: + assert ( + await first.post("/register", content=raw.model_dump_json()) + ).status_code == 200 + accepted = await first.post("/v2/relay-receipts", content=receipt.model_dump_json()) + assert accepted.status_code == 200 + # When: a fresh app accepts the same receipt after process restart. + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=module.fixture_master()), + base_url="https://master.test", + ) as restarted: + duplicate = await restarted.post( + "/v2/relay-receipts", content=receipt.model_dump_json() + ) + # Then: durable grant authorization and exact original signed acceptance survive. + assert duplicate.status_code == 200 + assert duplicate.content == accepted.content + + anyio.run(scenario) + + +def test_master_acceptance_when_exact_original_object_read_before_cutoff( + tmp_path: Path, +) -> None: + async def scenario() -> None: + async with httpx.AsyncClient() as backing: + rig = Rig(tmp_path, backing) + grant, _, _ = await rig.accept(b"abc") + await rig.put(grant, b"abc") + receipt = await rig.relay.complete(grant.digest()) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(rig.relay, "t" * 32)), + base_url="https://relay.test", + ) as wire: + client = RelayClient(rig.registry, wire) + accepted = await client.accept_receipt( + receipt, + grant=grant, + request_raw=rig.retrieval(grant, receipt), + master=MASTER, + now=20, + deadline=35, + ) + assert accepted.signer == MASTER.ss58 + assert accepted.body["delta_hash"] == hashlib.sha256(b"abc").hexdigest() + assert relay_envelope.verify_envelope(accepted.model_dump()) + + anyio.run(scenario) + + +def test_unsigned_chunk_http_when_known_grant_hash_is_not_capability( + tmp_path: Path, +) -> None: + async def scenario() -> None: + async with httpx.AsyncClient() as backing: + rig = Rig(tmp_path, backing) + grant, _, _ = await rig.accept(b"abc") + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(rig.relay, "t" * 32)), + base_url="https://relay.test", + ) as wire: + response = await wire.put(f"/v1/uploads/{grant.digest()}/chunks/0", content=b"abc") + assert response.status_code == 400 + assert not (await rig.relay.state()).uploads[grant.digest()].chunks + + anyio.run(scenario) + + +def test_smoke_harness_when_local_source_and_positive_resource_admission( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import importlib.util + import json + import sys + + spec = importlib.util.spec_from_file_location( + "relay_harness_boundaries", + Path(__file__).parents[2] / "scripts/relay_kind_smoke.py", + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + image = "sha256:" + "ab" * 32 + manifest = tmp_path / "source.json" + manifest.write_text( + json.dumps( + { + "files": { + name: hashlib.sha256((module.ROOT / name).read_bytes()).hexdigest() + for name in ( + "src/hypertrain/relay/core.py", + "src/hypertrain/relay/app.py", + "src/hypertrain/data/stream_store.py", + ) + } + } + ) + ) + digest = hashlib.sha256(manifest.read_bytes()).hexdigest() + calls = [] + + def docker(*args, **kwargs): + calls.append(args) + if args[:3] == ("docker", "image", "inspect"): + return json.dumps([{"Id": image}]) + if args[:2] == ("docker", "run"): + assert "--cpus=0.5" in args and "--memory-swap=256m" in args + assert "--tmpfs" in args + return "LOCAL_SOURCE_BYTES_VERIFIED\n" + return "" + + monkeypatch.setattr(module, "command", docker) + # The admitted installed image, not an actively edited checkout, owns these bytes. + monkeypatch.setattr(module, "ROOT", tmp_path / "absent-live-checkout") + assert module.verify_local_image(image, manifest, digest).startswith("hypertrain-relay-proof:") + with pytest.raises(ValueError, match="digest mismatch"): + module.verify_local_image(image, manifest, "00" * 32) + node = { + "CpusetCpus": "0-3", + "NanoCpus": 3_500_000_000, + "Memory": 4 << 30, + "MemorySwap": 4 << 30, + } + module.validate_node_limits(node) + for key, bad in ( + ("NanoCpus", 0), + ("NanoCpus", 4_000_000_000), + ("Memory", 0), + ("MemorySwap", -1), + ("CpusetCpus", "0-7"), + ): + with pytest.raises(ValueError): + module.validate_node_limits({**node, key: bad}) + monkeypatch.undo() + assert "OpenSSL" in module.command("openssl", "version") + with pytest.raises(ExceptionGroup) as captured: + module.command("openssl", "rand", "-base64", "3200000", timeout=10) + assert "diagnostic output exceeds" in str(captured.value.exceptions) + + +def test_integration_preparation_uses_owner_validator( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import importlib.util + import sys + from types import SimpleNamespace + + calls = [] + + def validate(source, digest): + calls.append((source, digest)) + raise ValueError("owner authority rejection") + + monkeypatch.setitem( + sys.modules, "network_authority_snapshot", SimpleNamespace(validate=validate) + ) + spec = importlib.util.spec_from_file_location( + "relay_authority_delegate", + Path(__file__).parents[2] / "scripts/relay_kind_smoke.py", + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + with pytest.raises(ValueError, match="owner authority rejection"): + module.inspect_integration_snapshot(tmp_path, "11" * 32) + assert calls == [(tmp_path, "11" * 32)] + + +def test_integration_preparation_render_uses_real_roles_original_dns( + tmp_path: Path, +) -> None: + import importlib.util + import sys + + spec = importlib.util.spec_from_file_location( + "relay_integration_render", + Path(__file__).parents[2] / "scripts/relay_kind_smoke.py", + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + with pytest.raises(ValueError, match="image digest"): + module.integration_manifests("floating:latest") + result = module.integration_manifests("example.test/hypertrain@sha256:" + "11" * 32) + pods = [i for i in result["items"] if i["kind"] == "Pod"] + job = next(i for i in result["items"] if i["kind"] == "Job") + assert {p["metadata"]["name"] for p in pods} == {"service", "relay", "gateway"} + assert job["spec"]["backoffLimit"] == 0 and job["spec"]["activeDeadlineSeconds"] == 9600 + specs = [p["spec"] for p in pods] + [job["spec"]["template"]["spec"]] + assert all(not p["automountServiceAccountToken"] for p in specs) + assert sum(int(p["containers"][0]["resources"]["limits"]["cpu"][:-1]) for p in specs) <= 3500 + for spec in specs: + command = spec["containers"][0]["command"] + assert "--fixture" not in command and "--integration-role" in command + assert job["spec"]["template"]["spec"]["hostAliases"][0]["hostnames"] == [ + "master.test", + "relay.test", + ] + + +def test_integration_preparation_dispatch_forbids_launch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import importlib.util + import sys + + spec = importlib.util.spec_from_file_location( + "relay_integration_dispatch", + Path(__file__).parents[2] / "scripts/relay_kind_smoke.py", + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + monkeypatch.setattr( + sys, + "argv", + ["harness", "--mode", "integration", "--prepare-integration", "--create"], + ) + with pytest.raises(SystemExit) as error: + module.main() + assert error.value.code == 2 + + +def test_integration_preparation_deadline_recovery_preserves_baseline( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import importlib.util + import json + import sys + + spec = importlib.util.spec_from_file_location( + "relay_deadline_recovery", + Path(__file__).parents[2] / "scripts/relay_kind_smoke.py", + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + journal = tmp_path / "cleanup.json" + journal.write_text( + json.dumps( + { + "state": "CREATE_INTENT", + "cluster": "hypertrain-v2-proof", + "experiment": "hypertrain-v2-proof", + "proof_dir": str(tmp_path), + "deadline_monotonic": 0, + "cleanup_cutoff_monotonic": float("inf"), + "boot_id": Path("/proc/sys/kernel/random/boot_id").read_text().strip(), + "containers": ["bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"], + "volumes": ["user-volume"], + "networks": ["cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"], + "provider_baselines": { + "docker": { + "containers": [ + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + ], + "volumes": ["user-volume"], + "networks": [ + "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + ], + }, + "podman": { + "containers": [], + "volumes": [], + "networks": [], + "network_names": [], + }, + }, + } + ) + ) + calls = [] + + def command(*args, **kwargs): + calls.append(args) + if args[0] == "podman": + return "[]" if args[1:3] == ("network", "ls") else "" + if args[:3] == ("docker", "ps", "-aq"): + return ( + "" + if "--filter" in args + else "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\n" + ) + if args[:3] == ("docker", "volume", "ls"): + return "user-volume\n" + if args[:3] == ("docker", "network", "ls"): + return "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\n" + return "" + + def gone(pid): + raise ProcessLookupError + + monkeypatch.setattr(module, "command", command) + monkeypatch.setattr(module.os, "pidfd_open", gone) + module.deadline_supervisor(journal, 99999) + assert json.loads(journal.read_bytes())["state"] == "CLEANUP_VERIFIED" + assert not any(args[:3] == ("kind", "delete", "cluster") for args in calls) + assert not any("rm" in args for args in calls) + + +def test_launch_safety_ready_pipe_and_absolute_deadline( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import importlib.util + import json + import os + import sys + + spec = importlib.util.spec_from_file_location( + "relay_launch_ready", Path(__file__).parents[2] / "scripts/relay_kind_smoke.py" + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + journal = tmp_path / "cleanup.json" + journal.write_text( + json.dumps( + { + "boot_id": Path("/proc/sys/kernel/random/boot_id").read_text().strip(), + "deadline_monotonic": 42, + } + ) + ) + monkeypatch.setattr(module.time, "monotonic", lambda: 40) + waits = [] + + class Poll: + def register(self, fd, event): + assert fd == 999 and event == module.select.POLLIN + + def poll(self, timeout): + waits.append(timeout) + return [True] + + monkeypatch.setattr(module.os, "pidfd_open", lambda pid: 999) + close = os.close + monkeypatch.setattr(module.os, "close", lambda fd: None if fd == 999 else close(fd)) + monkeypatch.setattr(module.select, "poll", Poll) + cleaned = [] + monkeypatch.setattr(module, "cleanup_owned", lambda path: cleaned.append(path)) + module.deadline_supervisor(journal, 123) + assert waits == [2000] and cleaned == [journal] + store = module.CleanupJournal(journal) + with pytest.raises(ValueError, match="immutable journal field"): + store.write_text(json.dumps({"deadline_monotonic": 43})) + + +def test_inventory_actual_podman_network_fixture(tmp_path: Path) -> None: + import importlib.util + import json + import sys + + spec = importlib.util.spec_from_file_location( + "relay_real_network_schema", Path(__file__).parents[2] / "scripts/relay_kind_smoke.py" + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + fixture = (Path(__file__).parent / "fixtures/podman-network-list.json").read_text() + rows = json.loads(fixture) + calls = [] + + def execute(*args): + calls.append(args) + return fixture if args[1:3] == ("network", "ls") else "" + + actual = module.provider_inventory("podman", execute) + assert actual["networks"] == sorted(row["id"] for row in rows) + assert actual["network_names"] == sorted(row["name"] for row in rows) + assert len(actual["networks"]) == 11 and all( + len(identity) == 64 for identity in actual["networks"] + ) + assert ("podman", "network", "ls", "--format", "json") in calls + for bad in ("kind\n", "", "HTTP429", "{}", '[{"name":"kind","id":"short"}]'): + with pytest.raises(ValueError): + module.provider_inventory("podman", lambda *a, value=bad: value) + docker_calls = [] + module.provider_inventory("docker", lambda *a: docker_calls.append(a) or "") + assert ("docker", "network", "ls", "-q", "--no-trunc") in docker_calls + + +@pytest.mark.parametrize("failure", ["docker", "podman", "command", "cutoff", None]) +def test_cleanup_absence_attempt_never_inherits_historical_success( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure: str | None +) -> None: + import importlib.util + import json + import sys + + spec = importlib.util.spec_from_file_location( + "relay_absence_attempt", Path(__file__).parents[2] / "scripts/relay_kind_smoke.py" + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + podman_ids = [f"{i:064x}" for i in range(1, 97)] + baseline = { + "docker": {"containers": [], "volumes": [], "networks": []}, + "podman": {"containers": podman_ids, "volumes": [], "networks": [], "network_names": []}, + } + journal = tmp_path / "cleanup.json" + journal.write_text( + json.dumps( + { + "state": "CLEANUP_VERIFIED", + "absence_verified": True, + "provider_after": baseline, + "provider_baselines": baseline, + "cluster": "hypertrain-v2-stale", + "experiment": "hypertrain-v2-stale", + "proof_dir": str(tmp_path), + "containers": [], + "volumes": [], + "networks": [], + "cleanup_cutoff_monotonic": 1200, + "cleanup_budget_cutoff_monotonic": 1200, + } + ) + ) + monkeypatch.setattr(module.time, "monotonic", lambda: 1200 if failure == "cutoff" else 0) + counts = {"docker": 0, "podman": 0} + + def command(*args, **kwargs): + state = json.loads(journal.read_bytes()) + assert state["absence_verified"] is False and state["provider_after"] is None + if args[0] == "podman" and args[1:3] == ("network", "ls"): + return "[]" + if "--filter" in args: + return "" + if args[1:3] == ("ps", "-aq"): + counts[args[0]] += 1 + if counts[args[0]] == 2: + if failure == args[0]: + return "HTTP429 malformed inventory" + if failure == "command": + raise RuntimeError("inventory command failed") + return "\n".join(baseline[args[0]]["containers"]) + return "" + + monkeypatch.setattr(module, "command", command) + if failure is None: + module.cleanup_owned(journal) + else: + with pytest.raises((ValueError, RuntimeError, TimeoutError)): + module.cleanup_owned(journal) + outcome = json.loads(journal.read_bytes()) + assert outcome["absence_history"][0]["absence_verified"] is True + assert outcome["cleanup_budget_cutoff_monotonic"] == 1200 + assert outcome["absence_verified"] is (failure is None) + if failure is None: + assert ( + outcome["provider_after"] == baseline + and len(outcome["provider_after"]["podman"]["containers"]) == 96 + ) + else: + assert outcome["provider_after"] is None and outcome["proof_status"] == "CENSORED" + + +def test_provider_fix_pinned_kind_version_and_fail_closed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import importlib.util + import json + import os + import shutil + import subprocess + import sys + + harness = Path(__file__).parents[2] / "scripts/relay_kind_smoke.py" + spec = importlib.util.spec_from_file_location("relay_provider_fix", harness) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + backend = tmp_path / "backend" + shim = tmp_path / "shim" + backend.mkdir() + shim.mkdir() + trace = tmp_path / "calls.jsonl" + journal = tmp_path / "journal.json" + journal.write_text(json.dumps({"cluster": "hypertrain-v2-provider"})) + docker = backend / "docker" + docker.write_text( + "#!/usr/bin/python3\nimport json,pathlib,sys\n" + f"with pathlib.Path({str(trace)!r}).open('a') as f:" + "f.write(json.dumps(sys.argv[1:])+'\\n')\n" + "if sys.argv[1:] in (['-v'],['--version']):print('Docker version 29.1.3, build mock')\n" + ) + docker.chmod(0o700) + wrapper = shim / "docker" + wrapper.write_text( + "#!" + sys.executable + "\nimport runpy,sys\n" + f"m=runpy.run_path({str(harness)!r});m['docker_create_hook'](m['Path']({str(journal)!r})," + f"{str(docker)!r},sys.argv[1:])\n" + ) + wrapper.chmod(0o700) + for name in ("podman", "nerdctl"): + fake = shim / name + fake.write_text("#!/usr/bin/python3\nraise AssertionError('provider fallback forbidden')\n") + fake.chmod(0o700) + kind = shutil.which("kind") + assert kind is not None, "pinned kind executable required in PATH" + pinned = json.loads((harness.parents[1] / "deploy/k8s/versions.json").read_bytes()) + version = subprocess.run( + [kind, "version"], capture_output=True, text=True, check=True, timeout=10 + ).stdout.split() + assert version[:2] == ["kind", pinned["kind_version"]] + environment = {**os.environ, "PATH": str(shim), "PYTHONPATH": str(harness.parents[1] / "src")} + environment.pop("KIND_EXPERIMENTAL_PROVIDER", None) + result = subprocess.run( + [str(kind), "get", "clusters"], env=environment, capture_output=True, text=True, timeout=30 + ) + assert result.returncode == 0, result.stderr + calls = [json.loads(line) for line in trace.read_text().splitlines()] + assert ["-v"] in calls and any(c and c[0] == "ps" for c in calls) + version = subprocess.run( + [str(wrapper), "--version"], env=environment, capture_output=True, text=True, timeout=10 + ) + assert version.stdout == "Docker version 29.1.3, build mock\n" + docker.write_text("#!/usr/bin/python3\nimport sys\nsys.exit(1)\n") + environment["KIND_EXPERIMENTAL_PROVIDER"] = "docker" + failed = subprocess.run( + [str(kind), "get", "clusters"], env=environment, capture_output=True, text=True, timeout=20 + ) + assert failed.returncode != 0 and "provider fallback forbidden" not in failed.stderr + for bad in ("HTTP429", "not-json", "", "[]"): + if bad == "[]": + assert module.provider_inventory( + "podman", lambda *a: "[]" if a[1:3] == ("network", "ls") else "" + ) == { + "containers": [], + "volumes": [], + "networks": [], + "network_names": [], + } + else: + with pytest.raises(ValueError, match="malformed"): + module.provider_inventory("podman", lambda *a, value=bad: value) + monkeypatch.setenv("KIND_EXPERIMENTAL_PROVIDER", "podman") + monkeypatch.setattr(module.shutil, "which", lambda name: str(kind)) + with pytest.raises(ValueError, match="fallback forbidden"): + module.command("kind", "get", "clusters") + monkeypatch.setattr( + module, + "command", + lambda *a, **k: json.dumps( + [ + { + "Id": "a" * 64, + "HostConfig": {}, + "Config": {"Labels": {}}, + } + ] + ), + ) + with pytest.raises(ValueError, match="pre-start caps"): + module.require_capped_node(journal, "missing-marker") + + +def test_shim_raw_bytes_exit_and_pinned_kind_empty_preflight(tmp_path: Path) -> None: + import json + import os + import shutil + import subprocess + import sys + + harness = Path(__file__).parents[2] / "scripts/relay_kind_smoke.py" + journal = tmp_path / "journal.json" + journal.write_text('{"cluster":"hypertrain-v2-byteproof"}') + backend = tmp_path / "backend" + backend.mkdir() + docker = backend / "docker" + shim = tmp_path / "shim" + shim.mkdir() + wrapper = shim / "docker" + wrapper.write_text( + "#!" + sys.executable + "\nimport runpy,sys\n" + f"m=runpy.run_path({str(harness)!r});m['docker_create_hook'](m['Path']({str(journal)!r})," + f"{str(docker)!r},sys.argv[1:])\n" + ) + wrapper.chmod(0o700) + environment = { + **os.environ, + "PATH": str(shim), + "PYTHONPATH": str(harness.parents[1] / "src"), + "KIND_EXPERIMENTAL_PROVIDER": "docker", + } + for args, output, errors, status in ( + (["ps"], b"", b"", 0), + (["ps"], b"one-node\n", b"", 0), + (["ps"], b"one-node", b"", 0), + (["--version"], b"Docker version raw\n", b"", 0), + (["-v"], b"Docker version raw", b"", 0), + (["ps"], b"partial\xff", b"daemon error\xfe", 73), + ): + docker.write_text( + "#!/usr/bin/python3\nimport os,sys\n" + f"os.write(1,{output!r});os.write(2,{errors!r});sys.exit({status})\n" + ) + docker.chmod(0o700) + result = subprocess.run( + [str(wrapper), *args], env=environment, capture_output=True, timeout=10 + ) + assert (result.stdout, result.stderr, result.returncode) == (output, errors, status) + for number in (9, 15): + docker.write_text( + "#!/usr/bin/python3\nimport os\n" + f"os.write(1,b'no-final-newline');os.write(2,b'child-signal');os.kill(os.getpid(),{number})\n" + ) + result = subprocess.run( + [str(wrapper), "ps"], env=environment, capture_output=True, timeout=10 + ) + assert (result.stdout, result.stderr, result.returncode) == ( + b"no-final-newline", + b"child-signal", + -number, + ) + trace = tmp_path / "kind-calls.jsonl" + docker.write_text( + "#!/usr/bin/python3\nimport json,pathlib,sys\n" + f"with pathlib.Path({str(trace)!r}).open('a') as f:" + "f.write(json.dumps(sys.argv[1:])+'\\n')\n" + "if sys.argv[1]=='ps':sys.exit(0)\n" + "if sys.argv[1]=='info':" + "print(json.dumps({'CgroupVersion':'2','CgroupDriver':'systemd'}));sys.exit(0)\n" + "sys.stderr.write('MOCK_NEXT_STAGE_NO_NODE_CREATION');sys.exit(73)\n" + ) + for provider in ("podman", "nerdctl"): + fake = shim / provider + fake.write_text("#!/usr/bin/python3\nraise AssertionError('FALLBACK_FORBIDDEN')\n") + fake.chmod(0o700) + kind = shutil.which("kind") + if kind is None: + raise FileNotFoundError("pinned kind executable required in PATH") + result = subprocess.run( + [ + str(kind), + "create", + "cluster", + "--name", + "hypertrain-v2-byteproof", + "--image", + "kindest/node:v1.35.0@sha256:452d707d4862f52530247495d180205e029056831160e22870e37e3f6c1ac31f", + ], + env=environment, + capture_output=True, + timeout=20, + ) + calls = [json.loads(line) for line in trace.read_text().splitlines()] + assert any(c[0] == "ps" and "--format" in c for c in calls) + assert ( + b"node(s) already exist" not in result.stderr and b"FALLBACK_FORBIDDEN" not in result.stderr + ) + assert b"MOCK_NEXT_STAGE_NO_NODE_CREATION" in result.stderr + assert result.returncode != 0 + assert not any(c[0] in {"run", "create", "start"} for c in calls) + + +@pytest.mark.parametrize("kernel_mode", ["valid", "wrong", "missing", "foreign", "forged"]) +def test_pinned_kind_contract_through_capped_start(tmp_path: Path, kernel_mode: str) -> None: + import json + import os + import shutil + import subprocess + import sys + import time + + harness = Path(__file__).parents[2] / "scripts/relay_kind_smoke.py" + trace = tmp_path / "trace.jsonl" + state = tmp_path / "state.json" + journal = tmp_path / "journal.json" + journal.write_text( + json.dumps( + { + "cluster": "hypertrain-v2-contract", + "containers": [], + "deadline_monotonic": time.monotonic() + 120, + "cleanup_cutoff_monotonic": time.monotonic() + 180, + } + ) + ) + backend = tmp_path / "backend" + shim = tmp_path / "shim" + backend.mkdir() + shim.mkdir() + docker = backend / "docker" + docker.write_text( + "#!" + + sys.executable + + "\n" + + (Path(__file__).parent / "fixtures/kind_contract_docker.py").read_text() + ) + docker.chmod(0o700) + wrapper = shim / "docker" + owned_scope = "/system.slice/docker-" + "a" * 64 + ".scope" + observed_scope = owned_scope if kernel_mode != "foreign" else "/system.slice/unrelated.scope" + if kernel_mode == "forged": + observed_scope = "/system.slice/docker-" + "b" * 64 + ".scope" + wrapper.write_text( + "#!" + sys.executable + "\nimport runpy,sys\n" + f"m=runpy.run_path({str(harness)!r})\n" + "P=m['Path'];original=P.read_text\n" + "def read(self,*a,**kw):\n" + " if str(self).startswith(('/proc/12345/','/sys/fs/cgroup/system.slice/')):\n" + f" with P({str(trace)!r}).open('a') as f:" + "f.write(m['json'].dumps(['kernel_read',str(self)])+'\\n')\n" + f" if str(self).startswith('/proc/'):return {'0::' + observed_scope + '/init.scope'!r}\n" + f" if {kernel_mode!r}=='missing':raise FileNotFoundError('mock missing kernel control')\n" + " controls={'cpu.max':'350000 100000','memory.max':str(4<<30)," + f"'memory.swap.max':{'1' if kernel_mode == 'wrong' else '0'!r}," + "'cpuset.cpus.effective':'0-3'}\n" + " if self.parent.name=='init.scope':" + "controls.update({'cpu.max':'max 100000','memory.max':'max','memory.swap.max':'max'})\n" + " return controls[self.name]\n" + " return original(self,*a,**kw)\n" + "P.read_text=read\n" + f"m['docker_create_hook'](P({str(journal)!r})," + f"{str(docker)!r},sys.argv[1:])\n" + ) + wrapper.chmod(0o700) + for provider in ("podman", "nerdctl"): + fake = shim / provider + fake.write_text("#!/usr/bin/python3\nraise AssertionError('FALLBACK_FORBIDDEN')\n") + fake.chmod(0o700) + environment = { + **os.environ, + "PATH": str(shim), + "PYTHONPATH": str(harness.parents[1] / "src"), + "KIND_EXPERIMENTAL_PROVIDER": "docker", + "KIND_MOCK_TRACE": str(trace), + "KIND_MOCK_STATE": str(state), + "KIND_MOCK_JOURNAL": str(journal), + } + kind = shutil.which("kind") + if kind is None: + raise FileNotFoundError("pinned kind executable required in PATH") + image = ( + "kindest/node:v1.35.0@sha256:" + "452d707d4862f52530247495d180205e029056831160e22870e37e3f6c1ac31f" + ) + unknown = subprocess.run( + [str(wrapper), "pull", "untrusted/image:latest"], + env=environment, + capture_output=True, + timeout=10, + ) + assert unknown.returncode != 0 and b"exact trusted pinned" in unknown.stderr + result = subprocess.run( + [ + str(kind), + "create", + "cluster", + "--name", + "hypertrain-v2-contract", + "--image", + image, + "--kubeconfig", + str(tmp_path / "kubeconfig"), + "--retain", + ], + env=environment, + capture_output=True, + timeout=90, + ) + calls = [json.loads(line) for line in trace.read_text().splitlines()] + assert result.returncode != 0 + assert b"unsupported local kind Docker operation" not in result.stderr + assert b"FALLBACK_FORBIDDEN" not in result.stderr + operations = [c[0] for c in calls] + assert "pull" in operations and operations.index("create") < operations.index("start") + assert operations.index("start") < operations.index("kernel_read") + receipt = json.loads(journal.read_text()) + assert ( + receipt["node_capped_before_start"] and receipt["create_host_config"]["Memory"] == 4 << 30 + ) + if kernel_mode != "valid": + assert operations.index("kernel_read") < operations.index("stop") + assert not any(operation in {"exec", "logs"} for operation in operations) + assert not receipt["node_kernel_receipt"]["verified"] + assert receipt["node_kernel_receipt"]["stopped"] + assert json.loads(state.read_text())["State"]["Running"] is False + return + assert b"MOCK_CONTROLPLANE_STOP_AFTER_CAPPED_START" in result.stderr + assert operations.index("kernel_read") < operations.index("exec") + assert receipt["node_kernel_receipt"]["verified"] + assert receipt["node_kernel_receipt"]["authority_cgroup"] == "/sys/fs/cgroup" + owned_scope + assert receipt["node_kernel_receipt"]["descendants"] + listed = subprocess.run( + [str(kind), "get", "clusters"], env=environment, capture_output=True, timeout=20 + ) + assert listed.returncode == 0 + deleted = subprocess.run( + [ + str(kind), + "delete", + "cluster", + "--name", + "hypertrain-v2-contract", + "--kubeconfig", + str(tmp_path / "kubeconfig"), + ], + env=environment, + capture_output=True, + timeout=20, + ) + assert deleted.returncode == 0 and not state.exists() + + +def test_launch_safety_create_capped_before_start( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import importlib.util + import json + import sys + import time + + spec = importlib.util.spec_from_file_location( + "relay_create_policy", Path(__file__).parents[2] / "scripts/relay_kind_smoke.py" + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + journal = tmp_path / "cleanup.json" + journal.write_text( + json.dumps( + { + "cluster": "hypertrain-v2-hook", + "containers": [], + "deadline_monotonic": time.monotonic() + 60, + "cleanup_cutoff_monotonic": time.monotonic() + 120, + } + ) + ) + calls = [] + host = {"CpusetCpus": "0-3", "NanoCpus": 3500000000, "Memory": 4 << 30, "MemorySwap": 4 << 30} + + def command(*args, **kwargs): + calls.append(args) + if args[1] == "create": + assert "--cpus=3.5" in args and "--memory-swap=4g" in args + return "owned" + if args[1] == "inspect": + return json.dumps( + [ + { + "Id": "owned", + "HostConfig": host, + "State": {"Running": False}, + "Config": { + "Labels": { + "io.hypertrain.proof": "hypertrain-v2-hook", + "io.x-k8s.kind.cluster": "hypertrain-v2-hook", + } + }, + } + ] + ) + assert args[1] == "start" and json.loads(journal.read_bytes())["node_capped_before_start"] + return "owned" + + monkeypatch.setattr(module, "command", command) + monkeypatch.setattr( + module.os, "posix_spawn", lambda executable, argv, env: command(*argv) or 99 + ) + monkeypatch.setattr(module.os, "waitpid", lambda pid, flags: (99, 0)) + monkeypatch.setattr(module.sys, "exit", lambda status: None) + module.docker_create_hook( + journal, + "/usr/bin/docker", + ["create", "--label=io.x-k8s.kind.cluster=hypertrain-v2-hook", "node"], + ) + assert [call[1] for call in calls] == ["create", "inspect"] + host["NanoCpus"] = 0 + with pytest.raises(ValueError): + module.docker_create_hook(journal, "/usr/bin/docker", ["start", "owned"]) + + +@pytest.mark.parametrize( + "field", ["container_id", "running", "pid", "host_config", "proof_label", "stop_error"] +) +def test_poststart_rejection_diagnostics( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, field: str +) -> None: + import copy + import json + import runpy + + module = runpy.run_path(str(Path(__file__).parents[2] / "scripts/relay_kind_smoke.py")) + identity = "a" * 64 + host = {"NanoCpus": 3500000000, "Memory": 4 << 30, "MemorySwap": 4 << 30, "CpusetCpus": "0-3"} + info = { + "Id": identity, + "HostConfig": host, + "State": {"Running": False, "Pid": 0}, + "Config": { + "Labels": { + "io.hypertrain.proof": "hypertrain-v2-diag", + "io.x-k8s.kind.cluster": "hypertrain-v2-diag", + } + }, + } + journal = tmp_path / "journal.json" + journal.write_text( + json.dumps( + { + "cluster": "hypertrain-v2-diag", + "owned_containers": [identity], + "create_host_config": host, + "deadline_monotonic": 100, + "cleanup_cutoff_monotonic": 120, + } + ) + ) + chronology = [] + + def start(function): + chronology.append("start") + return 0 + + def command(*args, **kwargs): + if args[1] == "stop": + receipt = json.loads(journal.read_text())["node_kernel_receipt"] + assert receipt["error"] == "started node identity/PID mismatch" + assert receipt["rejection_reasons"] and not receipt["cgroup_read_reached"] + chronology.append("stop_after_persist") + if field == "stop_error": + raise RuntimeError("mock stop error") + return "" + chronology.append("inspect") + observed = copy.deepcopy(info) + if chronology.count("inspect") == 2: + observed["State"] = {"Running": True, "Pid": 12345} + if field == "container_id": + observed["Id"] = "b" * 64 + elif field == "running": + observed["State"]["Running"] = False + elif field == "pid": + observed["State"]["Pid"] = True + elif field == "host_config": + observed["HostConfig"]["Memory"] = 123 + else: + observed["Config"]["Labels"]["io.hypertrain.proof"] = "wrong-cluster" + return json.dumps([observed]) + + monkeypatch.setattr(module["time"], "monotonic", lambda: 10) + monkeypatch.setattr(module["anyio"], "run", start) + module["docker_create_hook"].__globals__["command"] = command + with pytest.raises(ValueError, match="started node identity/PID mismatch"): + module["docker_create_hook"](journal, "/mock/docker", ["start", identity]) + receipt = json.loads(journal.read_text())["node_kernel_receipt"] + reason = "proof_label" if field == "stop_error" else field + assert receipt["rejection_reasons"] == [reason] + assert receipt["observed"] and receipt["expected"] + assert receipt["expected"]["container_id"] == identity + assert not receipt["cgroup_read_reached"] and not receipt["verified"] + assert chronology == ["inspect", "start", "inspect", "stop_after_persist"] + if field == "host_config": + assert receipt["host_config_changed_fields"] == ["Memory"] + assert receipt["observed"]["resources"]["Memory"] == 123 + assert receipt["expected"]["resources"]["Memory"] == 4 << 30 + if field == "stop_error": + assert receipt["stop_error"] == "mock stop error" + assert "stopped" not in receipt + else: + assert receipt["stopped"] + + +@pytest.mark.parametrize("depletion", ["prestart", "pidread", "cleanup"]) +def test_kernel_gate_absolute_budget( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, depletion: str +) -> None: + import json + import runpy + + module = runpy.run_path(str(Path(__file__).parents[2] / "scripts/relay_kind_smoke.py")) + identity = "a" * 64 + now = [10.0] + journal = tmp_path / "journal.json" + host = {"NanoCpus": 3500000000, "Memory": 4 << 30, "MemorySwap": 4 << 30, "CpusetCpus": "0-3"} + journal.write_text( + json.dumps( + { + "cluster": "hypertrain-v2-budget", + "owned_containers": [identity], + "create_host_config": host, + "deadline_monotonic": 12.0, + "cleanup_cutoff_monotonic": 100.0, + "cleanup_budget_cutoff_monotonic": 14.0, + } + ) + ) + calls = [] + reads = [] + + def command(*args, **kwargs): + calls.append((args[1], kwargs["timeout"])) + if args[1] == "stop": + assert args[-1] == identity and kwargs["timeout"] == 1.0 + return "" + if depletion == "prestart": + now[0] = 13.0 + return json.dumps( + [ + { + "Id": identity, + "HostConfig": host, + "Config": {"Labels": {"io.hypertrain.proof": "hypertrain-v2-budget"}}, + "State": {"Pid": 12345, "Running": True}, + } + ] + ) + + def start(function): + calls.append(("start", None)) + if depletion == "cleanup": + now[0] = 14.0 + return 0 + + original = Path.read_text + + def read(path, *args, **kwargs): + if str(path) == "/proc/12345/cgroup": + reads.append(str(path)) + now[0] = 13.0 + return "0::/system.slice/docker-" + identity + ".scope/init.scope" + return original(path, *args, **kwargs) + + monkeypatch.setattr(module["time"], "monotonic", lambda: now[0]) + monkeypatch.setattr(module["anyio"], "run", start) + monkeypatch.setattr(Path, "read_text", read) + module["docker_create_hook"].__globals__["command"] = command + with pytest.raises(TimeoutError, match="absolute budget"): + module["docker_create_hook"](journal, "/mock/docker", ["start", identity]) + assert calls[0] == ("inspect", 2.0) + if depletion == "cleanup": + assert [c[0] for c in calls] == ["inspect", "start"] + assert not reads + else: + assert calls[-1] == ("stop", 1.0) + receipt = json.loads(journal.read_text())["node_kernel_receipt"] + assert receipt["stopped"] and not receipt["verified"] + assert len(reads) == (1 if depletion == "pidread" else 0) + + +@pytest.mark.parametrize( + "changed", + [ + None, + "Memory", + "MemorySwap", + "NanoCpus", + "CpuQuota", + "CpuPeriod", + "CpusetCpus", + "Privileged", + "SecurityOpt", + "CgroupnsMode", + "Init", + "Binds", + "Tmpfs", + "OomKillDisable", + ], +) +def test_real_hostconfig_normalization_gate( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, changed: str | None +) -> None: + import copy + import json + import runpy + + module = runpy.run_path(str(Path(__file__).parents[2] / "scripts/relay_kind_smoke.py")) + fixture = json.loads( + (Path(__file__).parent / "fixtures/real-kind-hostconfig-normalization.json").read_bytes() + ) + before, after = fixture["before"], fixture["after"] + assert [name for name in before if before[name] != after[name]] == ["OomKillDisable"] + assert before["OomKillDisable"] is False and after["OomKillDisable"] is None + matcher = module["admitted_host_config_matches"] + assert matcher(after, before) + protected = {**before, "OomKillDisable": True} + assert matcher(protected, protected) + assert not matcher(after, protected) + assert not matcher({**before, "OomKillDisable": 0}, before) + observed = copy.deepcopy(after) + mutations = { + "Memory": 0, + "MemorySwap": -1, + "NanoCpus": 0, + "CpuQuota": 350000, + "CpuPeriod": 100000, + "CpusetCpus": "0-4", + "Privileged": False, + "SecurityOpt": [], + "CgroupnsMode": "host", + "Init": True, + "Binds": [], + "Tmpfs": {}, + "OomKillDisable": True, + } + if changed: + observed[changed] = mutations[changed] + assert not matcher(observed, before) + identity = "a" * 64 + journal = tmp_path / "journal.json" + journal.write_text( + json.dumps( + { + "cluster": "hypertrain-v2-hostconfig", + "experiment": "hypertrain-v2-hostconfig", + "node_capped_before_start": True, + "owned_containers": [identity], + "create_host_config": before, + "deadline_monotonic": 100, + "cleanup_cutoff_monotonic": 120, + } + ) + ) + chronology = [] + + def command(*args, **kwargs): + if args[1] == "stop": + receipt = json.loads(journal.read_bytes())["node_kernel_receipt"] + assert receipt["rejection_reasons"] == ["host_config"] + assert not receipt["cgroup_read_reached"] + chronology.append("stop") + return "" + chronology.append("inspect") + running = "start" in chronology + return json.dumps( + [ + { + "Id": identity, + "HostConfig": observed if running else before, + "State": {"Running": running, "Pid": 12345 if running else 0}, + "Config": {"Labels": {"io.hypertrain.proof": "hypertrain-v2-hostconfig"}}, + } + ] + ) + + def start(function): + chronology.append("start") + return 0 + + original = Path.read_text + + def read(path, *args, **kwargs): + if str(path) == "/proc/12345/cgroup": + chronology.append("cgroup_read") + return "0::/system.slice/docker-" + identity + ".scope" + if str(path).startswith("/sys/fs/cgroup/system.slice/docker-"): + return { + "cpu.max": "350000 100000", + "memory.max": str(4 << 30), + "memory.swap.max": "0", + "cpuset.cpus.effective": "0-3", + }[path.name] + return original(path, *args, **kwargs) + + monkeypatch.setattr(module["time"], "monotonic", lambda: 10) + monkeypatch.setattr(module["anyio"], "run", start) + monkeypatch.setattr(Path, "read_text", read) + module["docker_create_hook"].__globals__["command"] = command + if changed: + with pytest.raises(ValueError, match="started node identity/PID mismatch"): + module["docker_create_hook"](journal, "/mock/docker", ["start", identity]) + assert "cgroup_read" not in chronology and chronology[-1] == "stop" + else: + module["docker_create_hook"](journal, "/mock/docker", ["start", identity]) + receipt = json.loads(journal.read_bytes())["node_kernel_receipt"] + assert receipt["verified"] and receipt["cgroup_read_reached"] + assert receipt["host_config_changed_fields"] == ["OomKillDisable"] + assert "stop" not in chronology + module["require_capped_node"](journal, identity) + + +def test_image_archive_stdin_through_capture_and_native_shim(tmp_path: Path) -> None: + import hashlib + import io + import json + import runpy + import sys + import tarfile + + harness = Path(__file__).parents[2] / "scripts/relay_kind_smoke.py" + module = runpy.run_path(str(harness)) + journal = tmp_path / "journal.json" + journal.write_text('{"cluster":"hypertrain-v2-stream"}') + archive = tmp_path / "image.tar" + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w") as tar: + for name, content in ( + ( + "manifest.json", + b'[{"Config":"config.json","RepoTags":["tiny:test"],"Layers":["layer.tar"]}]', + ), + ("config.json", b'{"architecture":"amd64","os":"linux"}'), + ("layer.tar", b"\x00\xffarchive-bytes\x00"), + ): + header = tarfile.TarInfo(name) + header.size = len(content) + tar.addfile(header, io.BytesIO(content)) + payload = buffer.getvalue() + archive.write_bytes(payload) + docker = tmp_path / "docker" + docker.write_text( + "#!" + sys.executable + "\nimport hashlib,os,sys\n" + "data=sys.stdin.buffer.read()\n" + f"assert len(data)=={len(payload)}\n" + f"assert hashlib.sha256(data).hexdigest()=={hashlib.sha256(payload).hexdigest()!r}\n" + "os.write(1,b'ARCHIVE_BYTES_EXACT_NO_LF');os.write(2,b'ctr mock stderr\\xff')\n" + "sys.exit(73 if sys.argv[-1]=='fail' else 0)\n" + ) + docker.chmod(0o700) + shim = tmp_path / "shim" + shim.write_text( + "#!" + sys.executable + "\nimport runpy,sys\n" + f"m=runpy.run_path({str(harness)!r});m['docker_create_hook'](m['Path']({str(journal)!r})," + f"{str(docker)!r},sys.argv[1:])\n" + ) + shim.chmod(0o700) + kind = tmp_path / "kind" + kind.write_text( + "#!" + sys.executable + "\nimport subprocess,sys\n" + "assert sys.stdin.buffer.read()==b''\n" + f"with open({str(archive)!r},'rb') as source:\n" + f" p=subprocess.run([{str(shim)!r},'exec','--privileged','-i','owned-node','ctr'," + "'--namespace=k8s.io','images','import','--all-platforms','--digests'," + "'--snapshotter=overlayfs','-',sys.argv[-1]],stdin=source)\n" + "sys.exit(p.returncode)\n" + ) + kind.chmod(0o700) + assert module["command"](str(kind), "load", "ok") == "ARCHIVE_BYTES_EXACT_NO_LF" + with pytest.raises(RuntimeError, match="exit73: ctr mock stderr"): + module["command"](str(kind), "load", "fail") + assert json.loads(journal.read_bytes()) == {"cluster": "hypertrain-v2-stream"} + + +def test_launch_safety_independent_systemd_units(tmp_path: Path) -> None: + import json + import os + import runpy + import secrets + import select + import socket + import subprocess + import sys + import time + + token = "hypertrain-v2-tiny-" + secrets.token_hex(6) + runner, supervisor = token + "-runner", token + "-supervisor" + proof_path = tmp_path / ("long-proof-directory-" + "a" * 80) + proof_path.mkdir() + harness = Path(__file__).parents[2] / "scripts/relay_kind_smoke.py" + helpers = runpy.run_path(str(harness)) + provider_baselines = { + provider: helpers["provider_inventory"](provider) for provider in ("docker", "podman") + } + for provider in provider_baselines: + assert not helpers["command"]( + provider, "ps", "-aq", "--filter", "label=io.x-k8s.kind.cluster=" + token + ), "pre-existing experiment label conflicts with ownership" + listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + path = tmp_path / "probe.sock" + listener.bind(str(path)) + listener.listen(1) + listener.settimeout(30) + journal = proof_path / "cleanup.json" + journal.write_text( + json.dumps( + { + "state": "ADMITTED_NOT_CREATED", + "cluster": token, + "experiment": token, + "provider_baselines": provider_baselines, + "containers": provider_baselines["docker"]["containers"], + "volumes": provider_baselines["docker"]["volumes"], + "networks": provider_baselines["docker"]["networks"], + "owned_containers": [], + "owned_volumes": [], + "owned_networks": [], + "runner_unit": runner + ".service", + "supervisor_unit": supervisor + ".service", + "proof_dir": str(proof_path.resolve()), + "deadline_monotonic": time.monotonic() + 40, + "cleanup_cutoff_monotonic": time.monotonic() + 60, + "boot_id": Path("/proc/sys/kernel/random/boot_id").read_text().strip(), + } + ) + ) + runner_code = ( + "import runpy,socket,signal; m=runpy.run_path(" + repr(str(harness)) + ");" + "m['start_supervisor'](m['Path'](" + repr(str(journal)) + "));" + "s=socket.socket(socket.AF_UNIX);s.connect(" + repr(str(path)) + ");" + "s.sendall(b'READY');s.close();signal.pause()" + ) + try: + subprocess.run( + [ + "systemd-run", + "--unit=" + runner, + "--property=CPUQuota=45%", + "--property=MemoryMax=384M", + "--property=MemorySwapMax=0", + "--property=AllowedCPUs=0-3", + "--property=Nice=19", + "--property=RuntimeMaxSec=60", + "--setenv=PYTHONPATH=" + str(harness.parents[1] / "src"), + sys.executable, + "-c", + runner_code, + ], + check=True, + capture_output=True, + ) + with listener.accept()[0] as connection: + connection.settimeout(30) + assert connection.recv(64) == b"READY" + observed = json.loads(journal.read_bytes())["supervisor_ready"] + assert supervisor in observed["cgroup"] and runner not in observed["cgroup"] + assert observed["controls"]["cpu.max"] == "5000 100000" + assert int(observed["controls"]["memory.max"]) == 128 << 20 + assert observed["controls"]["memory.swap.max"] == "0" + pid = int( + subprocess.check_output( + [ + "systemctl", + "show", + supervisor + ".service", + "--property=MainPID", + "--value", + ], + text=True, + ) + ) + fd = os.pidfd_open(pid) + try: + poll = select.poll() + poll.register(fd, select.POLLIN) + subprocess.run( + ["systemctl", "kill", "--signal=SIGKILL", runner + ".service"], + check=True, + capture_output=True, + ) + assert poll.poll(30000), "actual supervisor must finish after exact parent death" + finally: + os.close(fd) + log = (proof_path / "supervisor.log").read_text() + assert "SUPERVISOR_CLEANUP_DONE" in log and "Traceback" not in log + settled = json.loads(journal.read_bytes()) + assert settled["state"] == "CLEANUP_VERIFIED" and settled["absence_verified"] is True + assert settled["provider_baselines"] == provider_baselines + assert settled["provider_after"] == provider_baselines + assert all( + settled[key] == [] for key in ("owned_containers", "owned_volumes", "owned_networks") + ) + finally: + listener.close() + for unit in (runner, supervisor): + subprocess.run(["systemctl", "stop", unit + ".service"], capture_output=True) + subprocess.run(["systemctl", "reset-failed", unit + ".service"], capture_output=True) + + +@pytest.mark.parametrize("rescue_failure", [False, True]) +def test_launch_safety_rescue_before_delete_exact_owned( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, rescue_failure: bool +) -> None: + import importlib.util + import json + import sys + + spec = importlib.util.spec_from_file_location( + "relay_launch_rescue", Path(__file__).parents[2] / "scripts/relay_kind_smoke.py" + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + cluster = "hypertrain-v2-abc123" + journal = tmp_path / "cleanup.json" + journal.write_text( + json.dumps( + { + "state": "CREATE_INTENT", + "cluster": cluster, + "experiment": cluster, + "proof_dir": str(tmp_path), + "kubeconfig": str(tmp_path / "kubeconfig"), + "cleanup_cutoff_monotonic": float("inf"), + "containers": ["bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"], + "volumes": ["user-volume"], + "networks": ["cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"], + "provider_baselines": { + "docker": { + "containers": [ + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + ], + "volumes": ["user-volume"], + "networks": [ + "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + ], + }, + "podman": { + "containers": [], + "volumes": [], + "networks": [], + "network_names": [], + }, + }, + } + ) + ) + calls = [] + deleted = False + clock = [0] + timeouts = [] + monkeypatch.setattr(module.time, "monotonic", lambda: clock[0]) + state = json.loads(journal.read_bytes()) + journal.write_text(json.dumps({**state, "cleanup_cutoff_monotonic": 1200})) + + def command(*args, **kwargs): + nonlocal deleted + calls.append(args) + if args[0] == "podman": + return "[]" if args[1:3] == ("network", "ls") else "" + timeouts.append(kwargs["timeout"]) + assert kwargs["timeout"] <= 1200 - clock[0] + clock[0] += 50 + if args[:3] == ("docker", "ps", "-aq"): + if "--filter" in args: + return ( + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n" + if not deleted + else "" + ) + return ( + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\n" + if deleted + else "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\n" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n" + ) + if args[:2] == ("docker", "inspect"): + return json.dumps( + [ + { + "Id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "Config": {"Labels": {"io.hypertrain.proof": cluster}}, + "Mounts": [], + "NetworkSettings": {"Networks": {}}, + } + ] + ) + if args[:3] == ("docker", "volume", "ls"): + return "user-volume\n" + if args[:3] == ("docker", "network", "ls"): + return "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\n" + if args[:3] == ("kind", "delete", "cluster"): + assert any(c[:2] == ("docker", "cp") for c in calls) + assert any(c[:3] == ("kind", "export", "logs") for c in calls) + deleted = True + if args[:2] == ("docker", "cp") and rescue_failure: + raise ValueError("node copy failed") + return "{}" + + monkeypatch.setattr(module, "command", command) + module.cleanup_owned(journal) + result = json.loads(journal.read_bytes()) + assert result["absence_verified"] and result["state"] == "CLEANUP_VERIFIED" + assert (result["proof_status"] == "CENSORED") == rescue_failure + assert not any(args[:2] == ("docker", "rm") for args in calls) + assert len(timeouts) > 8 and clock[0] > 400 + clock[0] = 1200 + journal.write_text(json.dumps({**result, "state": "CREATE_INTENT"})) + with pytest.raises(TimeoutError, match="aggregate cleanup cutoff"): + module.cleanup_owned(journal) + assert json.loads(journal.read_bytes())["proof_status"] == "CENSORED" + clock[0] = 0 + journal.write_text(json.dumps({**result, "state": "CREATE_INTENT"})) + deleted = False + monkeypatch.setattr( + module, + "command", + lambda *a, **k: ( + json.dumps( + [ + { + "Id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "Config": {"Labels": {}}, + } + ] + ) + if a[:2] == ("docker", "inspect") + else ( + ("[]" if a[1:3] == ("network", "ls") else "") + if a[0] == "podman" + else "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\n" + ) + ), + ) + with pytest.raises(ValueError, match="ownership conflicts"): + module.cleanup_owned(journal) + + +def test_smoke_storage_when_prefix_delete_preserves_other_region( + tmp_path: Path, +) -> None: + async def scenario() -> None: + import importlib.util + import sys + + from hypertrain.data.store import ObjectNotFound, S3Credentials, S3Store + + spec = importlib.util.spec_from_file_location( + "relay_harness_gc", + Path(__file__).parents[2] / "scripts/relay_kind_smoke.py", + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + secret = tmp_path / "secret" + secret.mkdir() + (secret / "object.json").write_text( + '{"access_key_id":"local","secret_access_key":"secret"}' + ) + module.FIXTURE, module.OBJECT_ROOT = secret, tmp_path / "objects" + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=module.fixture_store()), + base_url="http://objects.test", + ) as wire: + credentials = S3Credentials("local", "secret") + stores = [ + StreamStore( + S3Store( + "http://objects.test", + "proof", + credentials, + region="local", + prefix=f"relay/{region}/", + ), + wire, + ) + for region in ("eu", "us") + ] + sha = hashlib.sha256(b"abc").hexdigest() + for store in stores: + with spool() as file: + file.write(b"abc") + await store.put(file, sha, 3) + await stores[0].delete(sha) + with spool() as file: + with pytest.raises(ObjectNotFound): + await stores[0].get(sha, 3, file) + with spool() as file: + await stores[1].get(sha, 3, file) + assert file.read() == b"abc" + name = "56" * 32 + await stores[0].compare_swap(name, b"first", None) + data, etag = await stores[0].journal(name) + assert data == b"first" + await stores[0].compare_swap(name, b"second", etag) + newer, new_etag = await stores[0].journal(name) + assert newer == b"second" and new_etag != etag + + anyio.run(scenario) + + +def test_smoke_gc_when_signed_extension_blocks_delete_and_region_prefix_isolated( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + async def scenario() -> None: + import importlib.util + import sys + + spec = importlib.util.spec_from_file_location( + "relay_harness_retention", + Path(__file__).parents[2] / "scripts/relay_kind_smoke.py", + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + async with httpx.AsyncClient() as client: + rig = Rig(tmp_path / "eu", client) + other = StreamStore(LocalFSStore(tmp_path / "us"), client) + module.RUN = RUN + module.registry = lambda: rig.registry + module.key = lambda name: MASTER if name == "master" else RELAY + module.backing = lambda client, region: rig.store if region == "eu" else other + data = b"a" * (4 << 20) + b"tail" + grant, manifest, _ = await rig.accept(data) + await rig.put(grant, data[: 4 << 20]) + await rig.put(grant, b"tail", 1) + with spool() as file: + file.write(data[: 4 << 20]) + await other.put(file, manifest.chunks[0].chunk_sha256, 4 << 20) + original_client = httpx.AsyncClient + from fastapi import FastAPI + + gateway = FastAPI() + gateway.mount("/eu", create_app(rig.relay, "t" * 32)) + + def http_client(*args, **kwargs): + if "verify" in kwargs: + return original_client( + transport=httpx.ASGITransport(app=gateway), + base_url="https://gateway.test", + ) + return original_client() + + monkeypatch.setattr(module.httpx, "AsyncClient", http_client) + await module.fixture_gc() + assert not rig.store.store._path(grant.delta_hash).exists() + assert other.store.get(manifest.chunks[0].chunk_sha256) == data[: 4 << 20] + + anyio.run(scenario) + + +@pytest.fixture +def archive_intake(tmp_path): + import io + import json + import runpy + import tarfile + from types import SimpleNamespace + + module = runpy.run_path(str(Path(__file__).parents[2] / "scripts/relay_kind_smoke.py")) + reference = "d3-isolated-recovery-20261009-r5:recovered" + files = {f"src/hypertrain/file{i}.py": "a" * 64 for i in range(109)} + source = tmp_path / "source.json" + source.write_text(json.dumps({"files": files})) + config = json.dumps( + {"os": "linux", "architecture": "amd64", "config": {"User": "65532:65532"}} + ).encode() + image_id = "sha256:" + hashlib.sha256(config).hexdigest() + layer = b"tiny layer, no actual image import" + layer_id = "sha256:" + hashlib.sha256(layer).hexdigest() + oci = json.dumps( + { + "config": {"digest": image_id, "size": len(config)}, + "layers": [{"digest": layer_id, "size": len(layer)}], + } + ).encode() + manifest_id = "sha256:" + hashlib.sha256(oci).hexdigest() + documents = { + "manifest.json": json.dumps( + [ + { + "Config": "blobs/sha256/" + image_id[7:], + "RepoTags": [reference], + "Layers": ["blobs/sha256/" + layer_id[7:]], + } + ] + ).encode(), + "index.json": json.dumps( + { + "manifests": [ + { + "digest": manifest_id, + "size": len(oci), + "annotations": { + "io.containerd.image.name": "docker.io/library/" + reference + }, + } + ] + } + ).encode(), + "blobs/sha256/" + image_id[7:]: config, + "blobs/sha256/" + layer_id[7:]: layer, + "blobs/sha256/" + manifest_id[7:]: oci, + } + archive = tmp_path / "image.tar" + with tarfile.open(archive, "w") as tar: + for name, data in documents.items(): + member = tarfile.TarInfo(name) + member.size = len(data) + tar.addfile(member, io.BytesIO(data)) + archive.chmod(0o444) + args = SimpleNamespace( + mode="transport", + local_image_id=None, + image=None, + image_archive=archive, + archive_sha256=hashlib.sha256(archive.read_bytes()).hexdigest(), + archive_image_id=image_id, + archive_reference=reference, + source_manifest=source, + source_manifest_sha256=hashlib.sha256(source.read_bytes()).hexdigest(), + ) + return module, args, files + + +@pytest.mark.parametrize( + "fault", + [ + None, + "integration", + "image-id", + "remote", + "hash", + "reference", + "config", + "source", + "writable", + "symlink", + "parent-link", + "hardlink", + ], +) +def test_transport_archive_intake_custody(archive_intake, tmp_path, fault): + import os + + module, args, files = archive_intake + if fault == "integration": + args.mode = "integration" + elif fault == "image-id": + args.local_image_id = args.archive_image_id + elif fault == "remote": + args.image = "image@sha256:" + "b" * 64 + elif fault in ("hash", "config", "source"): + setattr( + args, + { + "hash": "archive_sha256", + "config": "archive_image_id", + "source": "source_manifest_sha256", + }[fault], + ("sha256:" if fault == "config" else "") + "b" * 64, + ) + elif fault == "reference": + args.archive_reference = "other:tag" + elif fault == "writable": + args.image_archive.chmod(0o644) + elif fault == "hardlink": + os.link(args.image_archive, tmp_path / "alias.tar") + elif fault == "symlink": + link = tmp_path / "link.tar" + link.symlink_to(args.image_archive) + args.image_archive = link + elif fault == "parent-link": + link = tmp_path / "parent" + link.symlink_to(tmp_path, target_is_directory=True) + args.image_archive = link / "image.tar" + if fault: + with pytest.raises((ValueError, OSError)): + with module["transport_archive"](args): + pytest.fail("invalid archive custody admitted") + else: + with module["transport_archive"](args) as (path, reference, actual): + assert path.startswith(f"/proc/{os.getpid()}/fd/") + assert reference == args.archive_reference and actual == files + # Rename/substitute original path cannot replace the held descriptor. + args.image_archive.rename(tmp_path / "original.tar") + args.image_archive.write_bytes(b"substituted path") + assert hashlib.sha256(Path(path).read_bytes()).hexdigest() == args.archive_sha256 + + +@pytest.mark.parametrize("fault", [None, "identity", "reference", "source-job"]) +def test_transport_archive_owned_node_verification(archive_intake, monkeypatch, fault): + import json + + module, args, files = archive_intake + calls, jobs = [], [] + + def command(*argv, **kwargs): + calls.append(argv) + if argv[:3] == ("docker", "exec", "owned-control-plane"): + return json.dumps( + { + "status": { + "id": args.archive_image_id + if fault != "identity" + else "sha256:" + "b" * 64, + "repoTags": ["docker.io/library/" + args.archive_reference] + if fault != "reference" + else [], + } + } + ) + if "logs" in argv: + return ( + "WRONG" + if fault == "source-job" + else "ARCHIVE_NODE_SOURCE109_RELAY_IMPORT_VERIFIED\n" + ) + return "" + + monkeypatch.setitem(module["verify_archive_node"].__globals__, "command", command) + monkeypatch.setitem( + module["verify_archive_node"].__globals__, "apply", lambda argv, job: jobs.append(job) + ) + if fault: + with pytest.raises(ValueError): + module["verify_archive_node"]( + "owned-control-plane", + args.archive_reference, + args.archive_image_id, + files, + ["kubectl"], + ) + else: + module["verify_archive_node"]( + "owned-control-plane", args.archive_reference, args.archive_image_id, files, ["kubectl"] + ) + container = jobs[0]["spec"]["template"]["spec"]["containers"][0] + assert ( + container["image"] == args.archive_reference and container["imagePullPolicy"] == "Never" + ) + assert jobs[0]["spec"]["activeDeadlineSeconds"] == 60 + assert ( + "actual==expected" in container["command"][-1] + and "import hypertrain.relay.app" in container["command"][-1] + ) + assert "hypertrain.__file__" in container["command"][-1] + assert "root.rglob('*')" in container["command"][-1] + pod = jobs[0]["spec"]["template"]["spec"] + assert pod["securityContext"] == { + "runAsUser": 65532, + "runAsGroup": 65532, + "runAsNonRoot": True, + "fsGroup": 65532, + } + assert pod["volumes"] == [ + {"name": "data", "emptyDir": {"medium": "Memory", "sizeLimit": "16Mi"}} + ] + assert container["volumeMounts"] == [{"name": "data", "mountPath": "/data"}] + assert all( + argv[:2] not in (("docker", "run"), ("docker", "inspect"), ("docker", "tag")) + for argv in calls + ) + + +def test_transport_archive_lease_rejects_write(archive_intake): + import os + + module, args, _ = archive_intake + with pytest.raises((ValueError, BlockingIOError)): + with module["transport_archive"](args): + # Nonblocking write triggers the original read-lease break signal, no sleep. + fd = os.open(args.image_archive, os.O_WRONLY | os.O_NONBLOCK) + os.close(fd) + pytest.fail("immutable archive accepted concurrent write") + + +def test_transport_archive_native_route_preserves_image_id(archive_intake, monkeypatch): + module, args, files = archive_intake + calls = [] + monkeypatch.setitem( + module["load_transport_image"].__globals__, + "command", + lambda *argv, **kwargs: calls.append((argv, kwargs)), + ) + archive = ("/proc/123/fd/9", args.archive_reference, files) + module["load_transport_image"]("owned-cluster", args.archive_reference, archive) + module["load_transport_image"]("owned-cluster", "hypertrain-relay-proof:" + "b" * 64) + assert calls == [ + ( + ("kind", "load", "image-archive", archive[0], "--name", "owned-cluster"), + {"timeout": 240}, + ), + ( + ( + "kind", + "load", + "docker-image", + "hypertrain-relay-proof:" + "b" * 64, + "--name", + "owned-cluster", + ), + {"timeout": 240}, + ), + ] + + +@pytest.mark.parametrize("failure", ["fixture", "denied", "cleanup", None]) +def test_transport_archive_smoke_failure_and_every_workload( + archive_intake, monkeypatch, tmp_path, failure +): + import json + import re + from contextlib import nullcontext + from types import SimpleNamespace + + module, args, files = archive_intake + globals_ = module["_smoke"].__globals__ + root = Path(__file__).parents[2] + versions = json.loads((root / "deploy/k8s/versions.json").read_text()) + cni = b"bounded mock CNI" + versions["calico_manifest_sha256"] = hashlib.sha256(cni).hexdigest() + monkeypatch.setattr( + globals_["Versions"], "model_validate_json", lambda _: SimpleNamespace(**versions) + ) + args.__dict__.update( + render=False, + create=True, + cluster="hypertrain-v2-tiny", + deadline_seconds=60, + cpus="0-3", + cpu_quota=4, + cleanup=True, + proof_dir=tmp_path / "proof", + integration_root=None, + work_end=module["time"].monotonic() + 60, + cleanup_end=module["time"].monotonic() + 1260, + clock_boot_id=Path("/proc/sys/kernel/random/boot_id").read_text().strip(), + ) + cleanup = [] + workloads = [] + created = False + miner = None + + def check_resource(resource): + if isinstance(resource, dict): + if "image" in resource: + assert resource["image"] == args.archive_reference + assert resource["imagePullPolicy"] == "Never" + workloads.append(resource["name"]) + for value in resource.values(): + check_resource(value) + elif isinstance(resource, list): + for value in resource: + check_resource(value) + + def command(*argv, **kwargs): + nonlocal created, miner + if argv[:2] == ("kind", "version"): + return versions["kind_version"] + if argv[:2] == ("kind", "create"): + created = True + if argv[:4] == ("docker", "ps", "-aq", "--no-trunc"): + return "owned-node" if created else "" + if argv[:2] == ("docker", "inspect"): + return json.dumps( + [ + { + "Id": "owned-node", + "HostConfig": { + "CpusetCpus": "0-3", + "NanoCpus": 3500000000, + "Memory": 1 << 30, + "MemorySwap": 1 << 30, + }, + "Mounts": [], + "NetworkSettings": {"Networks": {}}, + } + ] + ) + if argv[0] == "openssl": + Path(argv[argv.index("-out") + 1]).write_bytes(b"tiny mock certificate") + Path(argv[argv.index("-keyout") + 1]).write_bytes(b"tiny mock private key") + if argv[:2] == ("kubectl", "kustomize"): + return ( + (root / "deploy/k8s/base/deployment.yaml") + .read_text() + .replace( + "image: ghcr.io/cortexlm/hypertrain\n", + "image: " + versions["server_image"] + "\n", + ) + ) + if "apply" in argv: + text = kwargs["input_text"] + if text.startswith("{"): + check_resource(json.loads(text)) + elif "image: " + args.archive_reference in text: + lines = text.splitlines() + for i, line in enumerate(lines): + if re.match(r"\s*image:", line): + assert line.strip() == "image: " + args.archive_reference + assert lines[i + 1].strip() == "imagePullPolicy: Never" + workloads.extend( + re.findall(r"(?m)^\s*- name: (objects|master|gateway|miner|relay)$", text) + ) + if "kind: Job" in text: + miner = { + "spec": { + "suspend": True, + "template": { + "metadata": {}, + "spec": { + "containers": [ + { + "name": "miner", + "image": args.archive_reference, + "imagePullPolicy": "Never", + } + ], + "volumes": [ + { + "name": "secrets", + "secret": {"secretName": "proof-secrets"}, + } + ], + }, + }, + } + } + if failure == "fixture": + raise RuntimeError("late fixture failure") + if "get" in argv and "job/mock-miner" in argv: + return json.dumps(miner) + if "get" in argv and "pods" in argv: + return json.dumps( + {"items": [{"metadata": {"name": "relay-1"}}, {"metadata": {"name": "relay-2"}}]} + ) + if "run" in argv and "unauthorized" in argv: + override = next(a for a in argv if a.startswith("--overrides=")) + check_resource(json.loads(override.removeprefix("--overrides="))) + return json.dumps({"metadata": {"name": "unauthorized"}}) + if "logs" in argv: + if "job/prefix-upload" in argv: + return '{"event": "PREFIX_DURABLE"}' + if "job/mock-miner" in argv: + return '{"event": "FOUR_ORIGINAL_OBJECTS_VERIFIED", "receipts": [1]}' + if "job/replica-resume" in argv or "job/regional-fallback" in argv: + return '{"receipts": [1]}' + if "job/transport-gc" in argv: + return '{"event": "TRANSPORT_RETENTION_GC_PREFIX_ISOLATION"}' + if "unauthorized" in argv: + return "unexpected connect success" if failure == "denied" else "TimeoutError" + return "" + + def cleanup_owned(path): + state = json.loads(path.read_text()) + assert state["deadline_monotonic"] == args.work_end + assert state["cleanup_cutoff_monotonic"] == args.cleanup_end + assert state.get("proof_status") != "PASS" + recorded = json.loads((path.parent / "proof-result.json").read_text()) + assert recorded["transport_cluster_smoke"] == "PENDING" + assert recorded["archive_node_verification"] == "node-source-verified" + cleanup.append(path) + if failure == "cleanup": + raise RuntimeError("cleanup failed") + module["CleanupJournal"](path).write_text( + json.dumps({**state, "state": "CLEANUP_VERIFIED"}) + ) + + for name in ("runner_admission", "start_supervisor", "require_capped_node"): + monkeypatch.setitem(globals_, name, lambda *_, **__: None) + monkeypatch.setitem( + globals_, "provider_inventory", lambda _: {"containers": [], "volumes": [], "networks": []} + ) + monkeypatch.setitem(globals_, "command", command) + monkeypatch.setitem(globals_, "cleanup_owned", cleanup_owned) + monkeypatch.setitem(globals_, "verify_archive_node", lambda *_: "node-source-verified") + monkeypatch.setattr(globals_["shutil"], "which", lambda _: "/no-real-docker") + monkeypatch.setattr( + globals_["httpx"], + "Client", + lambda **_: nullcontext( + SimpleNamespace( + get=lambda _: SimpleNamespace( + content=cni, text=cni.decode(), raise_for_status=lambda: None + ) + ) + ), + ) + if failure: + with pytest.raises((RuntimeError, AssertionError)): + module["_smoke"](args, ("/proc/mock/fd/9", args.archive_reference, files)) + else: + module["_smoke"](args, ("/proc/mock/fd/9", args.archive_reference, files)) + assert len(cleanup) == 1 + result = json.loads((args.proof_dir / "proof-result.json").read_text()) + state = json.loads((args.proof_dir / "cleanup.json").read_text()) + assert result["archive_node_verification"] == "node-source-verified" + assert result["transport_cluster_smoke"] == ("PENDING" if failure else "PASS") + assert state["proof_status"] == ("INTERRUPTED" if failure else "PASS") + if failure != "fixture": + assert {"objects", "master", "gateway", "miner", "relay", "unauthorized"} <= set(workloads) + assert workloads.count("relay") == 2 + + +@pytest.mark.parametrize( + "fault", [None, "wrong-unit", "cap", "future", "missing", "boot", "expired", "duration"] +) +def test_transport_archive_absolute_cutoff_authority(archive_intake, monkeypatch, fault): + module, args, _ = archive_intake + globals_ = module["archive_cutoffs"].__globals__ + unit = "d3-tiny.service" + original_read = Path.read_text + + def read_text(path, *args, **kwargs): + if str(path) == "/proc/self/cgroup": + return "0::/system.slice/" + ("foreign.service" if fault == "wrong-unit" else unit) + return original_read(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", read_text) + monkeypatch.setattr(globals_["time"], "monotonic", lambda: 5000.0) + args.__dict__.update( + launch_unit=unit, + work_end=14800.0, + cleanup_end=16000.0, + caller_cap=16120.0, + deadline_seconds=10800, + clock_boot_id=Path("/proc/sys/kernel/random/boot_id").read_text().strip(), + ) + if fault == "cap": + args.caller_cap = 15999.0 + elif fault == "future": + args.work_end = 15801.0 + elif fault == "missing": + args.work_end = None + elif fault == "boot": + args.clock_boot_id = "stale" + elif fault == "expired": + args.work_end = 5000.0 + elif fault == "duration": + args.cleanup_end += 1 + monkeypatch.setitem( + globals_, "command", lambda *_: pytest.fail("manager clock lookup forbidden") + ) + if fault: + with pytest.raises(ValueError): + module["archive_cutoffs"](args) + else: + module["archive_cutoffs"](args) + assert args.work_end == 14800.0 and args.cleanup_end == 16000.0 + + +def test_transport_archive_expired_admission_rejects_before_commands(archive_intake, monkeypatch): + module, args, files = archive_intake + globals_ = module["_smoke"].__globals__ + args.work_end = 11800.0 + args.deadline_seconds = 10800 + monkeypatch.setattr(globals_["time"], "monotonic", lambda: 11800.0) + monkeypatch.setitem( + globals_, "command", lambda *args, **kwargs: pytest.fail("expired admission command") + ) + with pytest.raises(TimeoutError, match="original work deadline"): + module["_smoke"](args, ("/proc/mock/fd/9", args.archive_reference, files)) + + +@pytest.mark.parametrize( + "scenario", ["runner-first", "cancel", "cutoff", "mismatch", "observation", "blind", "foreign"] +) +def test_transport_caller_lock_survives_exit_until_cleanup( + archive_intake, monkeypatch, tmp_path, scenario +): + import fcntl + import json + import signal + import socket + + module, args, _ = archive_intake + globals_ = module["caller_launch"].__globals__ + lock_path = tmp_path / "cpu.lock" + lock_path.touch(mode=0o600) + barrier_path = tmp_path / ".transport-admission.json" + proof = tmp_path / "caller-proof" + args.__dict__.update( + caller_unit="d3-caller-tiny", + cluster="hypertrain-v2-caller-tiny", + deadline_seconds=10800, + proof_dir=proof, + caller_lock=lock_path, + ) + monkeypatch.setenv("TMPDIR", str(tmp_path)) + monkeypatch.setitem(globals_, "ROOT", tmp_path) + monkeypatch.setitem(globals_, "caller_resources", lambda: 13120.0) + monkeypatch.setattr(globals_["time"], "monotonic", lambda: 1000.0) + baseline = {p: {"containers": [], "volumes": [], "networks": []} for p in ("docker", "podman")} + handlers = {} + monkeypatch.setattr( + globals_["signal"], "signal", lambda sig, handler: handlers.setdefault(sig, handler) + ) + actions = [] + state = { + "state": "CREATED", + "cluster": args.cluster, + "experiment": args.cluster, + "proof_dir": str(proof), + "boot_id": ( + "foreign-boot" + if scenario == "foreign" + else Path("/proc/sys/kernel/random/boot_id").read_text().strip() + ), + "owned_containers": ["a" * 64], + "supervisor_ready": { + "state": "READY", + "unit": args.cluster + "-supervisor", + "cgroup": "0::/system.slice/" + args.cluster + "-supervisor.service", + }, + "cleanup_cutoff_monotonic": 13000.0, + "deadline_monotonic": 11800.0, + "provider_baselines": baseline, + } + parent, child = socket.socketpair() + + def assert_owned(): + with lock_path.open("a") as candidate, pytest.raises(BlockingIOError): + fcntl.flock(candidate, fcntl.LOCK_EX | fcntl.LOCK_NB) + + class Events: + def __init__(self, path): + assert path == proof + actions.append("subscribed") + + def wait(self, cutoff, pid): + assert cutoff == 13000.0 + assert_owned() + if "runner-exited" in actions: + assert scenario in ("mismatch", "foreign") + actions.append("original-cutoff-locked") + raise TimeoutError("original cutoff") + assert parent.recv(1) == b"E" # Explicit queued runner-exit notification. + actions.append("runner-exited") + if scenario == "cancel": + handlers[signal.SIGTERM](signal.SIGTERM, None) + assert_owned() + actions.append("cancelled-locked") + if scenario in ("cutoff", "blind"): + raise TimeoutError("original cutoff") + assert_owned() + state.update(state="CLEANUP_VERIFIED", absence_verified=True, provider_after=baseline) + module["CleanupJournal"](proof / "cleanup.json").write_text(json.dumps(state)) + (proof / "supervisor.log").write_text("SUPERVISOR_CLEANUP_DONE\n") + actions.append("supervisor-terminal") + + def close(self): + assert_owned() + expected = ( + "INCOMPLETE" + if scenario in ("cutoff", "mismatch", "foreign", "blind") + else "VERIFIED" + ) + assert json.loads(barrier_path.read_text())["state"] == expected + actions.append("durable-before-release") + + def command(*argv, **kwargs): + assert_owned() + if argv[0] == "systemd-run": + assert actions == ["subscribed"] + assert "--property=RuntimeMaxSec=12000" in argv + assert "--property=CPUQuota=40%" in argv and "--property=MemoryMax=256M" in argv + assert argv[argv.index("--launch-unit") + 1] == "d3-caller-tiny.service" + assert argv[argv.index("--work-end") + 1] == "11800.0" + assert argv[argv.index("--cleanup-end") + 1] == "13000.0" + assert ( + argv[argv.index("--clock-boot-id") + 1] == state["boot_id"] or scenario == "foreign" + ) + proof.mkdir() + module["CleanupJournal"](proof / "cleanup.json").write_text(json.dumps(state)) + child.sendall(b"E") + actions.append("dispatched") + return "" + assert "--value" not in argv # No postdispatch epoch observation exists. + if scenario == "blind": + raise RuntimeError("manager permanently blind") + if scenario == "observation" and "runner-exited" not in actions: + raise RuntimeError("primary supervisor observation failed") + return "MainPID=0\nActiveState=inactive\n" + + monkeypatch.setitem(globals_, "CleanupEvents", Events) + monkeypatch.setitem(globals_, "command", command) + original_bytes = Path.read_bytes + + def read_bytes(path): + if scenario == "blind" and path == proof / "cleanup.json" and "dispatched" in actions: + raise OSError("journal permanently blind") + return original_bytes(path) + + monkeypatch.setattr(Path, "read_bytes", read_bytes) + monkeypatch.setitem( + globals_, + "provider_inventory", + lambda provider, execute=None: ( + {"containers": ["foreign"], "volumes": [], "networks": []} + if scenario == "mismatch" and "supervisor-terminal" in actions + else baseline[provider] + ), + ) + try: + if scenario == "cancel": + with pytest.raises(KeyboardInterrupt, match="verified cleanup"): + module["caller_launch"](args, ["--create"]) + elif scenario in ("cutoff", "mismatch", "foreign", "blind"): + with pytest.raises((TimeoutError, ValueError, OSError)): + module["caller_launch"](args, ["--create"]) + if scenario != "blind": + assert ( + json.loads((proof / "cleanup.json").read_text())["caller_status"] + == "INCOMPLETE" + ) + assert json.loads(barrier_path.read_text())["state"] == "INCOMPLETE" + args.proof_dir = tmp_path / "another-proof" + with pytest.raises(ValueError, match="INCOMPLETE"): + module["caller_launch"](args, ["--create"]) + assert actions.count("dispatched") == 1 + if scenario not in ("cutoff", "blind"): + assert "original-cutoff-locked" in actions + args.create = True + args.work_end = 11800.0 + with pytest.raises(ValueError, match="CREATE refused"): + module["_smoke"](args, ("/proc/mock/fd/9", args.archive_reference, {})) + elif scenario == "observation": + with pytest.raises(RuntimeError, match="primary"): + module["caller_launch"](args, ["--create"]) + assert json.loads(barrier_path.read_text())["state"] == "VERIFIED" + else: + module["caller_launch"](args, ["--create"]) + with lock_path.open("a") as released: + fcntl.flock(released, fcntl.LOCK_EX | fcntl.LOCK_NB) + assert actions[-1] == "durable-before-release" + finally: + parent.close() + child.close() + + +def test_transport_caller_native_file_event_subscription(archive_intake, tmp_path): + module, _, _ = archive_intake + proof = tmp_path / "native-proof" + events = module["CleanupEvents"](proof) + try: + proof.mkdir() + events.wait(module["time"].monotonic() + 1, 0) # Arms child watch and rechecks. + (proof / "supervisor.log").write_text("SUPERVISOR_CLEANUP_DONE\n") + events.wait(module["time"].monotonic() + 1, 0) # Already queued inotify event. + finally: + events.close() + + +def test_transport_caller_contention_rejects_before_dispatch(archive_intake, monkeypatch, tmp_path): + import fcntl + + module, args, _ = archive_intake + globals_ = module["caller_launch"].__globals__ + lock_path = tmp_path / "busy.lock" + lock_path.touch(mode=0o600) + args.__dict__.update( + caller_unit="d3-tiny", + caller_lock=lock_path, + deadline_seconds=10800, + proof_dir=tmp_path / "busy-proof", + ) + monkeypatch.setitem(globals_, "caller_resources", lambda: 13120.0) + monkeypatch.setitem( + globals_, "command", lambda *_, **__: pytest.fail("dispatch during contention") + ) + with lock_path.open("a") as owner: + fcntl.flock(owner, fcntl.LOCK_EX | fcntl.LOCK_NB) + with pytest.raises(BlockingIOError): + module["caller_launch"](args, []) + assert not args.proof_dir.exists() + + +def test_transport_caller_inclusive_resource_partition(archive_intake, monkeypatch): + from fractions import Fraction + + module, _, _ = archive_intake + globals_ = module["caller_resources"].__globals__ + original_read = Path.read_text + controls = { + "cpu.max": "5000 100000", + "memory.max": str(128 << 20), + "memory.swap.max": "0", + "cpuset.cpus.effective": "0-3", + } + + def read_text(path, *args, **kwargs): + if str(path) == "/proc/self/cgroup": + return "0::/system.slice/d3-caller-owner.service" + if str(path).startswith("/sys/fs/cgroup/"): + return controls[path.name] + return original_read(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", read_text) + monkeypatch.setitem( + globals_, + "command", + lambda *_: "RuntimeMaxUSec=3h 22min\nActiveEnterTimestampMonotonic=1000000000", + ) + assert module["caller_resources"]() == 13120.0 + assert sum(map(Fraction, ["3.5", ".40", ".05", ".05"])) == 4 + assert 4096 + 256 + 128 + 128 == 4608 + controls["memory.max"] = str(129 << 20) + with pytest.raises(ValueError, match="caller requires"): + module["caller_resources"]() diff --git a/tests/test_publish_exclusions.py b/tests/test_publish_exclusions.py new file mode 100644 index 0000000..1e06e7c --- /dev/null +++ b/tests/test_publish_exclusions.py @@ -0,0 +1,512 @@ +"""Publisher selector/copy/scanner functions, without cloning, commits or pushes.""" + +import json +import os +import subprocess +from pathlib import Path + +import pytest + + +def test_real_gitleaks_exact_values_and_same_line_new_secrets(tmp_path: Path) -> None: + # Given: optional local official binary; release gate supplies it explicitly. + binary = os.environ.get("HYPERTRAIN_GITLEAKS_BIN") + if binary is None: + import shutil + + binary = shutil.which("gitleaks") + if binary is None: + pytest.skip("real gitleaks unavailable; release check remains unverified") + source = Path(__file__).resolve().parents[1] + config = source / ".gitleaks.toml" + reviewed = [ + "scripts/canary.sh", + "scripts/publish.sh", + "tests/protocol/test_network_v2.py", + "tests/gpu_ops/test_gpu_ops.py", + "tests/storage/fakes.py", + "tests/protocol/fixtures/network_v1_baseline.json", + ] + for relative in reviewed: + path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes((source / relative).read_bytes()) + + def scan() -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + binary, + "detect", + "--no-git", + "--source", + str(tmp_path), + "--config", + str(config), + "--log-level", + "error", + "--redact=100", + ], + cwd=tmp_path, + capture_output=True, + text=True, + timeout=30, + check=False, + env={ + k: v + for k, v in os.environ.items() + if k not in ("GITLEAKS_CONFIG", "GITLEAKS_CONFIG_TOML") + }, + ) + + # When / Then: exact reviewed files, including all seven original findings, scan clean. + clean = scan() + assert clean.returncode == 0, clean.stderr + fake = "Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8Ii9Jj0Kk" + contexts = [ + ("scripts/publish.sh", "d3030488eb18aaafc0feceb256666bcc63fc15c4e6ae0bb008980affd96976a2"), + ( + "tests/protocol/fixtures/network_v1_baseline.json", + "d3030488eb18aaafc0feceb256666bcc63fc15c4e6ae0bb008980affd96976a2", + ), + ("tests/gpu_ops/test_gpu_ops.py", "7875e0437f396a1d"), + ("tests/storage/fakes.py", "SECRETSECRETSECRETSECRET1234567890abcd"), + ] + for relative, value in contexts: + path = tmp_path / relative + original = path.read_text() + line = next(x for x in original.splitlines() if value in x) + # Added same-line API key is a separate finding; unchanged public scalar stays allowed. + path.write_text(original.replace(line, line + '; api_key="' + fake + '"')) + assert scan().returncode == 1 + # A different value at that very same file/line/rule is not exempt. + path.write_text(original.replace(line, line.replace(value, fake))) + assert scan().returncode == 1 + path.write_text(original) + # A known synthetic value on a different file is not a global token exemption. + (tmp_path / "other.py").write_text('secret = "' + "SECRET" * 4 + '1234567890abcd"') + assert scan().returncode == 1 + + +SCANNER_CASES = [ + ( + "deploy/k8s/versions.json", + "server_build_job", + ( + "https://github.com/" + + "CortexLM/hypertrain/actions/runs/" + + "37815568607/job/113454067723" + ), + None, + ), + ( + "docs/network-v2.md", + "--freeze", + ( + "/root/distributed-decision-training/" + + ".omo/evidence/hypertrain-network/" + + "D2-FINAL-SOURCE-FREEZE.json" + ), + None, + ), + ( + "docs/network-v2.md", + "--freeze", + ( + "/root/distributed-decision-training/" + + ".omo/evidence/hypertrain-network/" + + "D2-FINAL-SOURCE-FREEZE.json" + ), + None, + ), + ( + "scripts/network_gpu_prepare.py", + "operation_receipts", + "CREATED_ONLY_AT_RUNTIME_FROM_ACCEPTED_CONTEXTS_BY_" + "service_factory", + None, + ), + ("tests/miner/test_island_launch_v2.py", "experiment", "hypertrain-network-v2", "monkeypatch"), + ( + "tests/protocol/fixtures/network_v1_baseline.json", + "protocol/keys.py", + "d3030488eb18aaafc0feceb256666bcc63fc15c4e6ae0bb008980affd96976a2", + "source_sha256", + ), +] + + +def run_scanner(root: Path, path: str, content: str) -> subprocess.CompletedProcess[str]: + source = Path(__file__).resolve().parents[1] + script = (source / "scripts/publish.sh").read_text() + scanner = script[script.index("scan() {") : script.index("\n# raw tree")] + target = root / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content) + (root / ".secretscan-allow").write_text("") + # Actual scanner, not a second token or exception implementation. + result = subprocess.run( + [ + "bash", + "-c", + 'ROOT="$PWD"; SCAN_PATH="$1"; list_files() { printf "%s\\n" "$SCAN_PATH"; }; ' + + scanner + + "\nscan", + "scanner", + path, + ], + cwd=root, + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert result.stderr == "", result.stderr + return result + + +def scanner_content(path: str, key: str, value: str, parent: str | None) -> str: + if path.endswith(".json"): + payload = {key: value} + return json.dumps({parent: payload} if parent else payload, indent=2) + if path.endswith(".md"): + return " " + key + " " + value + " \\\n" + declaration = ' "' + key + '": "' + value + '",\n' + return (parent + ":\n" if parent else "") + declaration + + +@pytest.mark.parametrize("path,key,value,parent", SCANNER_CASES) +def test_scanner_exact_public_context_only( + tmp_path: Path, + path: str, + key: str, + value: str, + parent: str | None, +) -> None: + # Given / When: six reviewed scalar contexts pass without whole-file exemption. + content = scanner_content(path, key, value, parent) + assert run_scanner(tmp_path, path, content).returncode == 0 + # Then: value/context/file mutation loses the specific exemption. + assert run_scanner(tmp_path, path, content.replace(value, value + "9")).returncode == 1 + assert run_scanner(tmp_path, path, content.replace(key, "api_key")).returncode == 1 + assert run_scanner(tmp_path, "other/" + path, content).returncode == 1 + + +@pytest.mark.parametrize("path,key,value,parent", SCANNER_CASES) +@pytest.mark.parametrize("secret", ["token", "private", "hex"]) +def test_scanner_public_scalar_never_hides_same_line_secret( + tmp_path: Path, + path: str, + key: str, + value: str, + parent: str | None, + secret: str, +) -> None: + content = scanner_content(path, key, value, parent) + # Given: generated fake provider token/private key/credential hex after public scalar. + suffix = { + "token": " ghp_" + "Ab2Cd3" * 6, + "private": " -----BEGIN " + "PRIVATE KEY-----", + "hex": ' api_key="' + "a1" * 32 + '"', + }[secret] + line = next(line for line in content.splitlines() if value in line) + content = content.replace(line, line + suffix) + # When / Then: original raw secret patterns still see appended bytes. + assert run_scanner(tmp_path, path, content).returncode == 1 + + +def test_scanner_json_parent_and_other_credential_controls(tmp_path: Path) -> None: + path, key, value, parent = SCANNER_CASES[-1] + # Wrong JSON parent must not turn a credential-looking filename into public metadata. + assert run_scanner(tmp_path, path, scanner_content(path, key, value, "private")).returncode == 1 + for content in ( + 'api_key = "' + "b2" * 32 + '"', + 'monkeypatch:\n token: "' + "A1b2C3d4" * 5 + '"', + 'secret:\n digest: "' + "c3" * 32 + '"', + ): + assert run_scanner(tmp_path, "tests/control.py", content).returncode == 1 + + +def test_scanner_exact_public_entropy_never_exempts_other_values(tmp_path: Path) -> None: + import ast + import hashlib + import re + + source = Path(__file__).resolve().parents[1] + script = (source / "scripts/publish.sh").read_text() + tokens = ast.literal_eval( + script.split("public_entropy=", 1)[1].split("\npublic_token_grammar=", 1)[0].strip() + ) + checked = set() + contexts = [] + for relative, allowed in tokens.items(): + for line in (source / relative).read_text().splitlines(): + for token in re.findall(r"[A-Za-z0-9+/_=-]{40,}", line): + digest = hashlib.sha256(token.encode()).hexdigest() + if digest not in allowed: + continue + checked.add((relative, digest)) + contexts.append((relative, line, token)) + qualification = "qualification-usd50-20261009T181257Z/" + "NEXT30-008-RESULT-VERIFY" + expected = { + ("docs/RESULTS.md", qualification), + ("docs/network-v2.md", qualification), + ( + "docs/network-v2.md", + "import/source109/TLS/CNI/upload/" + "fixture-master/failover/GC/deny", + ), + ( + "scripts/relay_kind_smoke.py", + "--setenv=KIND_EXPERIMENTAL_PROVIDER=" + "docker", + ), + ( + "scripts/verify_image.sh", + "com/CortexLM/opendecision/archive/" + "e125ff756d57b726fbce44c430e3d2207e7bbda8", + ), + } + assert checked == { + (relative, hashlib.sha256(token.encode()).hexdigest()) for relative, token in expected + } + # Removed by canonical allocation extraction; still exercise its exact + # public-path exception and every secret control as an explicit fixture. + legacy = "omo/evidence/hypertrain-network/" + "qualification-usd50-20261009T181257Z" + relative = "tests/gpu_ops/test_network_admit_cli.py" + assert hashlib.sha256(legacy.encode()).hexdigest() in tokens[relative] + contexts.append((relative, 'retained = ".' + legacy + '"', legacy)) + for relative, line, token in contexts: + assert run_scanner(tmp_path, relative, line + "\n").returncode == 0 + assert run_scanner(tmp_path, relative, "public note: " + line + "\n").returncode == 0 + assert run_scanner(tmp_path, "other/" + relative, line + "\n").returncode == 1 + assert run_scanner(tmp_path, relative, line.replace(token, token + "9")).returncode == 1 + fake = "Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8Ii9Jj0Kk" + assert run_scanner(tmp_path, relative, line + '; api_key="' + fake + '"').returncode == 1 + assert run_scanner(tmp_path, relative, line + " ghp_" + "Ab2Cd3" * 6).returncode == 1 + assert ( + run_scanner(tmp_path, relative, line + " -----BEGIN " + "PRIVATE KEY-----").returncode + == 1 + ) + + +def test_actual_publisher_selector_preserves_public_excludes_private(tmp_path: Path) -> None: + source = Path(__file__).resolve().parents[1] + script = (source / "scripts/publish.sh").read_text() + # Execute the actual function, not a second implementation of its regex rules. + selector = script[script.index("list_files() {") : script.index("\ncopy_files() {")] + (tmp_path / ".publishignore").write_bytes((source / ".publishignore").read_bytes()) + (tmp_path / ".secretscan-allow").write_bytes((source / ".secretscan-allow").read_bytes()) + public = { + ".publishignore", + ".secretscan-allow", + "README.md", + "src/hypertrain/data/store.py", + "tests/data/test_store.py", + "tests/protocol/fixtures/network_v1_baseline.json", + "tests/beacon/vectors.json", + "tests/gpu_ops/fixtures/leftover-instance.json", + "tests/fixtures/miner.toml", + "tests/models/od_fixtures.py", + "tests/challenge/test_authority_snapshot_v2.py", + "docs/schemas/example-run-manifest.json", + "docs/operator.md", + "deploy/trust-root-v3-row.example.toml", + "deploy/k8s/local/fixtures.yaml", + "scripts/network_authority_snapshot.py", + "experiments/gpu_network_v2/profile.json", + } + private = { + ".omo/evidence/local/proof.json", + ".herdr-web-ui/session.json", + "private/config.json", + "secrets/object.json", + "roles/role.json", + "role-keys/coordinator", + "private-keys/operator", + "authority/original.json", + "authority-state/state.json", + "portable-authority/objects/original", + "portable-bundle/config.json", + "portable-bundles/bundle-a/object", + "snapshots/original.json", + "preserved-snapshots/original.json", + "evidence/index.json", + "rescue/service/object", + "service-export/objects/original", + "integration-root/roles/original", + "experiments/gpu_network_v2/rescue/result.json", + "experiments/gpu_network_v2/evidence/index.json", + "experiments/gpu_network_v2/snapshots/original.json", + "experiments/gpu_network_v2/secrets/object.json", + "experiments/gpu_network_v2/roles/role.json", + "experiments/gpu_network_v2/private-roles/coordinator", + "experiments/gpu_network_v2/portable-bundle/object", + "local/hot-0.seed", + "local/coordinator.priv", + "local/certificate.p12", + "local/certificate.pfx", + "local/secret.pkcs8", + "local/private_key", + "local/private-key", + "local/admin.token", + "local/drain.token", + "local/challenge.db", + "local/challenge.db-wal", + "local/challenge.db-shm", + "local/bundle.json", + "local/key.pem", + "local/operator.key", + "local/id_ed25519", + "local/known_hosts", + "data/raw/public-but-not-publishable", + } + for relative in (public | private) - {".publishignore", ".secretscan-allow"}: + path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("synthetic selector input\n") + # Neither a file link nor a directory link imports its excluded target. + (tmp_path / "docs/linked-role.json").symlink_to(tmp_path / "secrets/object.json") + (tmp_path / "docs/linked-roles").symlink_to(tmp_path / "roles", target_is_directory=True) + selected = subprocess.run( + [ + "bash", + "-c", + "set -euo pipefail\nROOT=$1\n" + selector + "\nlist_files", + "selector", + str(tmp_path), + ], + check=True, + capture_output=True, + text=True, + timeout=10, + ).stdout.splitlines() + assert set(selected) == public + assert selected == sorted(public) + + +def test_root_transport_barrier_excluded_from_selected_copy(tmp_path: Path) -> None: + # Given: synthetic custody only; never inspect the active root barrier. + repo = Path(__file__).resolve().parents[1] + script = (repo / "scripts/publish.sh").read_text() + functions = script[script.index("list_files() {") : script.index("\nscan() {")] + source, destination = tmp_path / "source", tmp_path / "copy" + source.mkdir() + destination.mkdir() + (source / ".publishignore").write_bytes((repo / ".publishignore").read_bytes()) + (source / ".transport-admission.json").write_bytes(b"PRIVATE\n") + public = source / "docs/.transport-admission.json" + public.parent.mkdir() + public.write_bytes(b"PUBLIC\n") + # When: original dry selection and copy functions, no scan/remote/publish body. + result = subprocess.run( + [ + "bash", + "-c", + "set -euo pipefail\nROOT=$1\n" + + functions + + '\nlist_files | tee "$2/selected.txt" | copy_files "$2"', + "copy", + str(source), + str(destination), + ], + check=True, + capture_output=True, + text=True, + timeout=10, + ) + # Then: exact root excluded; nested public names and bytes still ship. + assert (destination / "selected.txt").read_text().splitlines() == [ + ".publishignore", + "docs/.transport-admission.json", + ] + assert not (destination / ".transport-admission.json").exists() + assert (destination / "docs/.transport-admission.json").read_bytes() == b"PUBLIC\n" + assert result.stderr == "" + + +def test_bare_selected_copy_has_publisher_runtime_controls(tmp_path: Path) -> None: + # Given: actual selector/copier, empty destination; no injected control files. + source = Path(__file__).resolve().parents[1] + script = (source / "scripts/publish.sh").read_text() + functions = script[script.index("list_files() {") : script.index("\n# raw tree")] + subprocess.run( + [ + "bash", + "-c", + "set -euo pipefail\nROOT=$1\n" + functions + '\nlist_files | copy_files "$2"', + "copy", + str(source), + str(tmp_path), + ], + check=True, + capture_output=True, + text=True, + timeout=30, + ) + # When: shipped functions run entirely from the bare selected tree. + copied_script = (tmp_path / "scripts/publish.sh").read_text() + copied_functions = copied_script[ + copied_script.index("list_files() {") : copied_script.index("\n# raw tree") + ] + result = subprocess.run( + ["bash", "-c", "set -euo pipefail\nROOT=$PWD\n" + copied_functions + "\nlist_files\nscan"], + cwd=tmp_path, + check=False, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stdout + result.stderr + # Then: runtime inputs ship byte-exactly; no external selector/scanner policy. + for control in (".publishignore", ".secretscan-allow"): + assert control in result.stdout.splitlines() + assert (tmp_path / control).read_bytes() == (source / control).read_bytes() + assert result.stderr == "" + + +@pytest.mark.parametrize("replacement", ["none", "file", "parent"]) +def test_actual_copy_rejects_post_selection_symlinks(tmp_path: Path, replacement: str) -> None: + repo = Path(__file__).resolve().parents[1] + script = (repo / "scripts/publish.sh").read_text() + functions = script[script.index("list_files() {") : script.index("\nscan() {")] + source, destination, outside = (tmp_path / name for name in ("source", "copy", "outside")) + for directory in (source, destination, outside): + directory.mkdir() + (source / ".publishignore").write_bytes((repo / ".publishignore").read_bytes()) + public = source / "docs/operator.md" + public.parent.mkdir() + public.write_bytes(b"public operator example\n") + public.chmod(0o755) + external = outside / "operator.md" + external.write_bytes(b"synthetic private external bytes\n") + command = ["bash", "-c", "set -euo pipefail\nROOT=$1\n" + functions] + # Selection completion is the deterministic hook; mutate before actual copy. + selected = subprocess.run( + [*command[:-1], command[-1] + "\nlist_files", "selector", str(source)], + check=True, + capture_output=True, + text=True, + timeout=10, + ).stdout + assert selected == ".publishignore\ndocs/operator.md\n" + if replacement == "file": + public.unlink() + public.symlink_to(external) + elif replacement == "parent": + public.parent.rename(source / "previous-docs") + (source / "docs").symlink_to(outside, target_is_directory=True) + copied = subprocess.run( + [*command[:-1], command[-1] + '\ncopy_files "$2"', "copy", str(source), str(destination)], + input=selected, + capture_output=True, + text=True, + timeout=10, + ) + if replacement == "none": + assert copied.returncode == 0, copied.stderr + assert (destination / "docs/operator.md").read_bytes() == b"public operator example\n" + assert (destination / "docs/operator.md").stat().st_mode & 0o777 == 0o755 + else: + assert copied.returncode != 0 + assert not (destination / "docs/operator.md").exists() + assert all( + path.read_bytes() != external.read_bytes() + for path in destination.rglob("*") + if path.is_file() + ) diff --git a/tests/trainer/test_arch_dispatch.py b/tests/trainer/test_arch_dispatch.py index ca2d36a..3a150e8 100644 --- a/tests/trainer/test_arch_dispatch.py +++ b/tests/trainer/test_arch_dispatch.py @@ -47,8 +47,15 @@ def stage_of(self, name: str, cfg: Any, n: int) -> Any: def init_params(self, cfg: Any) -> Any: return self._rec("init_params", cfg) - def forward(self, cfg: Any, p: Any, tokens: Any, moe: Any = None) -> Any: - return self._rec("forward", cfg, p, tokens, moe=moe) + def forward( + self, + cfg: tm.ModelConfig, + p: tm.Params, + tokens: torch.Tensor, + moe: tm.MoeFn | None = None, + hook: tm.ForwardHook | None = None, + ) -> str: + return self._rec("forward", cfg, p, tokens, moe=moe, hook=hook) @pytest.fixture @@ -75,10 +82,30 @@ def test_dispatch_routes_to_arch_impl(od_cfg: TrainConfig, monkeypatch: pytest.M assert param_shapes(cfg) == "param_shapes" assert stage_of("tok.weight", cfg, 1) == "stage_of" assert init_params(cfg) == "init_params" - sentinel = object() - assert forward(cfg, {}, torch.zeros(1, 2, dtype=torch.long), moe=sentinel) == "forward" # type: ignore[arg-type] + + def sentinel( + cfg: tm.ModelConfig, p: tm.Params, pre: str, x: torch.Tensor, dt: torch.dtype + ) -> tuple[torch.Tensor, torch.Tensor]: + raise AssertionError("dispatch must forward, not invoke, the MoE callable") + + def hook(layer: int, op: str, x: torch.Tensor) -> torch.Tensor: + return x + 1 + + tokens = torch.zeros(1, 2, dtype=torch.long) + assert forward(cfg, {}, tokens, moe=sentinel) == "forward" assert [c[0] for c in stub.calls] == ["param_shapes", "stage_of", "init_params", "forward"] assert stub.calls[-1][2]["moe"] is sentinel + assert stub.calls[-1][2]["hook"] is None + assert forward(cfg, {}, tokens, moe=sentinel, hook=hook) == "forward" + assert [c[0] for c in stub.calls] == [ + "param_shapes", + "stage_of", + "init_params", + "forward", + "forward", + ] + assert stub.calls[-1][2]["moe"] is sentinel + assert stub.calls[-1][2]["hook"] is hook def test_decoder_never_touches_arch_impl(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/trainer/test_codec_metadata.py b/tests/trainer/test_codec_metadata.py new file mode 100644 index 0000000..1d78acd --- /dev/null +++ b/tests/trainer/test_codec_metadata.py @@ -0,0 +1,215 @@ +"""Manifest geometry and framing must be checked before decoding any tensor.""" + +import struct + +import pytest +import torch + +import hypertrain.trainer.compress as codec +from hypertrain.trainer.config import CompressConfig + + +def framed(kind: str, entries: list[tuple[bytes, tuple[int, ...], bytes]]) -> bytes: + out = [codec.MAGIC[kind], struct.pack(" bytes: + return ( + struct.pack(" None: + # Given: actual encoder, both temperature scalar and genuine one-element vector. + delta = { + "scalar": torch.tensor(2.0), + "vector": torch.tensor([-1.0]), + "w": torch.arange(12, dtype=torch.float32).reshape(3, 4), + } + cfg = CompressConfig(kind, topk_frac=0.25, bits=2 if kind == "sparseloco" else 8) + payload, ef = codec.compress(cfg, delta, {n: torch.zeros_like(x) for n, x in delta.items()}) + shapes = {n: tuple(x.shape) for n, x in delta.items()} + # When + validated = codec.validate_payload(payload, shapes) + decoded_kind, decoded = codec.decompress(payload, expected_shapes=shapes) + # Then: encoder bytes and legacy API results remain identical. + assert validated == decoded_kind == kind + _, legacy = codec.decompress(payload) + assert ( + codec.compress(cfg, delta, {n: torch.zeros_like(x) for n, x in delta.items()})[0] == payload + ) + for name, value in decoded.items(): + assert value.dtype == torch.float32 and tuple(value.shape) == shapes[name] + assert torch.equal(value, legacy[name]) + assert torch.equal(ef[name], delta[name] - value) + + +@pytest.mark.parametrize("kind", list(codec.MAGIC)) +@pytest.mark.parametrize( + "attack", + [ + "huge-dim", + "zero-dim", + "scalar-vector", + "missing", + "extra", + "wrong-name", + "duplicate", + "unsorted", + "bad-count", + "bad-rank", + "bad-name-length", + "bad-body-length", + "truncated", + "trailing", + "magic", + "utf8", + "bad-scale", + "bad-topk", + "bad-index", + "bad-padding", + "byte-limit", + ], +) +def test_hostile_metadata_rejects_before_decode_or_allocation( + kind: str, + attack: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Given: a valid first tensor ensures later errors cannot trigger early decode. + entries = [(b"a", (), body(kind)), (b"b", (1,), body(kind))] + shapes = {"a": (), "b": (1,)} + if attack == "huge-dim": + entries[1] = (b"b", (2**63,), body(kind)) + elif attack == "zero-dim": + entries[1] = (b"b", (0,), body(kind)) + elif attack == "scalar-vector": + entries[0] = (b"a", (1,), body(kind)) + elif attack == "missing": + entries.pop() + elif attack == "extra": + entries.append((b"c", (), body(kind))) + elif attack == "wrong-name": + entries[1] = (b"c", (1,), body(kind)) + elif attack == "duplicate": + entries[1] = (b"a", (1,), body(kind)) + elif attack == "unsorted": + entries.reverse() + elif attack == "utf8": + entries[1] = (b"\xff", (1,), body(kind)) + elif attack in ("bad-scale", "bad-topk", "bad-index", "bad-padding"): + if kind == "dense-int8": + bad = struct.pack(" None: + counts["decode"] += 1 + pytest.fail("metadata rejection reached decoder") + + def forbidden_allocate(*args: object, **kwargs: object) -> None: + counts["allocate"] += 1 + pytest.fail("metadata rejection allocated tensor") + + for name in ("_int8_decode", "_sparse_decode"): + monkeypatch.setattr(codec, name, forbidden_decode) + for name in ("zeros", "empty", "ones", "full", "tensor", "from_numpy"): + monkeypatch.setattr(torch, name, forbidden_allocate) + # When / Then: both metadata-only and validated-decode entries reject with zero allocation. + with pytest.raises(ValueError): + codec.validate_payload(payload, shapes, max_payload_bytes=limit) + with pytest.raises(ValueError): + codec.decompress(payload, expected_shapes=shapes, max_payload_bytes=limit) + assert counts == {"decode": 0, "allocate": 0} + + +@pytest.mark.parametrize("kind", list(codec.MAGIC)) +def test_exact_raw_byte_bound(kind: str) -> None: + # Given: valid scalar encoding padded only via its legal UTF-8 parameter name. + overhead = len(framed(kind, [(b"", (), body(kind))])) + name = "x" * (65536 - overhead) + payload = framed(kind, [(name.encode(), (), body(kind))]) + assert len(payload) == 65536 + # When / Then: equality is accepted; one byte smaller bound and one byte excess reject. + assert codec.validate_payload(payload, {name: ()}) == kind + with pytest.raises(ValueError, match="byte limit"): + codec.validate_payload(payload, {name: ()}, max_payload_bytes=65535) + with pytest.raises(ValueError, match="byte limit"): + codec.validate_payload(payload + b"x", {name: ()}) + + +@pytest.mark.parametrize( + "kind,bad", + [ + ("dense-int8", struct.pack(" None: + # Given: same expected numel, invalid block/scale/code/count/index/length only. + shape = (1,) if kind == "dense-int8" else (2,) + payload = framed(kind, [(b"w", shape, bad)]) + calls = [] + + def forbidden(*args: object) -> None: + calls.append("decode") + pytest.fail("invalid body reached tensor decoder") + + monkeypatch.setattr(codec, "_int8_decode", forbidden) + monkeypatch.setattr(codec, "_sparse_decode", forbidden) + # When / Then + with pytest.raises(ValueError): + codec.decompress(payload, expected_shapes={"w": shape}) + assert calls == [] diff --git a/tests/trainer/test_config_v2.py b/tests/trainer/test_config_v2.py new file mode 100644 index 0000000..586f59d --- /dev/null +++ b/tests/trainer/test_config_v2.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + +from hypertrain.protocol.example import example_manifest +from hypertrain.trainer.config import TrainConfig +from hypertrain.trainer.island import island_from_manifest + + +def test_v1_conversion_and_golden_unchanged() -> None: + m = example_manifest() + assert m.run_id() == "ad8beb30084a2e53f817c45f1cbfdf1491162575f10f2c3d190645081b2ba2ff" + cfg, _, run = island_from_manifest(m.body()) + assert cfg == TrainConfig.from_manifest(m.body()) + assert run == m.run_id() + + +def test_v2_adapter_wrapper_is_authoritative() -> None: + spec = importlib.util.spec_from_file_location( + "config_fixture", Path(__file__).parents[1] / "miner/test_island_launch_v2.py" + ) + assert spec and spec.loader + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + w = mod.tiny_manifest(8) + cfg, lay, run = island_from_manifest(w.body()) + assert cfg == TrainConfig.from_manifest_v2(w) + assert lay.n_gpus == 8 + assert run == w.run_id() != w.training.training_hash() From a652e55068671724755bf99f6a451c1a5ead0163 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:43:23 +0000 Subject: [PATCH 2/3] fix: provide candidate verifier writable temporary storage --- scripts/verify_image.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/verify_image.sh b/scripts/verify_image.sh index 73e7967..6359f8f 100755 --- a/scripts/verify_image.sh +++ b/scripts/verify_image.sh @@ -29,6 +29,7 @@ PY test "$(jq -r '.Labels["io.cortex.challenge.source-manifest"]' <<<"$config")" = "$(sha256sum "$manifest" | cut -d ' ' -f 1)" test "$(jq -c '.Entrypoint' <<<"$config")" = '["hypertrain-miner"]' "$engine" run --rm -i --network=none --read-only \ + --tmpfs /tmp:rw,nosuid,nodev,mode=1777 \ -e PYTHONDONTWRITEBYTECODE=1 -e OMP_NUM_THREADS=1 -e MKL_NUM_THREADS=1 \ -v "$(realpath "$manifest"):/candidate-source.json:ro" \ --entrypoint /opt/hypertrain/venv/bin/python "$image" - <<'PY' From 386370d8648bf06527e40bc53f2da45fcb800654 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:59:19 +0000 Subject: [PATCH 3/3] fix: limit image prerequisite CI to main publication --- .github/workflows/images.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml index 3acae19..4cff028 100644 --- a/.github/workflows/images.yml +++ b/.github/workflows/images.yml @@ -17,6 +17,7 @@ concurrency: jobs: ci: + if: github.ref == 'refs/heads/main' uses: ./.github/workflows/ci.yml publish: