toki
8f00606c03
제품 결과와 harness·process·artifact 실패가 하나의 성공 값으로 덮이지 않도록 durable evidence와 모든 소비자 계약을 함께 마이그레이션한다.
857 lines
34 KiB
Python
857 lines
34 KiB
Python
"""Source-aware timing and usage evidence for exactly one benchmark attempt.
|
|
|
|
This module owns three things and nothing else: the closed measurement schema,
|
|
a bounded workspace write observer, and the no-clobber sidecar publisher and
|
|
strict loader for ``attempt-measurement.json``.
|
|
|
|
Every required value is either an ``observed`` value carrying its unit, clock
|
|
and source, or an explicit ``unavailable`` value carrying the reason it could
|
|
not be observed. Nothing here decomposes, sums, subtracts or reconstructs a
|
|
value the caller or the harness did not report: a missing provider total stays
|
|
unavailable, overlapping intervals stay overlapping, and a filesystem
|
|
modification time is never presented as proof of the first write.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import stat
|
|
import threading
|
|
import time
|
|
from collections import deque
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any, Callable, Mapping, Optional
|
|
|
|
from scripts.agent_benchmark.lifecycle import (
|
|
CLOCK_FILESYSTEM_MTIME,
|
|
CLOCK_HARNESS_MONOTONIC,
|
|
CLOCK_NONE,
|
|
EVENT_FIRST_OUTPUT,
|
|
EVENT_SUBMITTED,
|
|
METRIC_CLOCKS,
|
|
METRIC_NAMES,
|
|
METRIC_PREFIX,
|
|
METRIC_SOURCES,
|
|
METRIC_UNITS,
|
|
SOURCE_HARNESS,
|
|
SOURCE_WORKSPACE_POLL,
|
|
UNIT_NANOSECONDS,
|
|
InvocationResult,
|
|
HarnessOutcome,
|
|
LifecycleMetricError,
|
|
LifecycleValidationError,
|
|
ParsedMetric,
|
|
ProcessOutcome,
|
|
ProductOutcome,
|
|
metric_record,
|
|
publish_bytes_no_replace,
|
|
validate_metric,
|
|
)
|
|
|
|
MEASUREMENT_FILENAME = "attempt-measurement.json"
|
|
MEASUREMENT_VERSION = 2
|
|
MEASUREMENT_RECORD = "attempt_measurement"
|
|
|
|
STATUS_OBSERVED = "observed"
|
|
STATUS_UNAVAILABLE = "unavailable"
|
|
|
|
REASON_NOT_REPORTED = "not_reported"
|
|
REASON_NOT_OBSERVED = "not_observed"
|
|
REASON_AMBIGUOUS_TOTAL = "ambiguous_total"
|
|
REASON_OBSERVER_UNAVAILABLE = "observer_unavailable"
|
|
UNAVAILABLE_REASONS = (
|
|
REASON_NOT_REPORTED, REASON_NOT_OBSERVED, REASON_AMBIGUOUS_TOTAL,
|
|
REASON_OBSERVER_UNAVAILABLE,
|
|
)
|
|
|
|
TIMELINE_NAMES = (
|
|
"submitted_at", "first_output_at", "first_write_observed_at",
|
|
"first_write_mtime", "total_duration",
|
|
)
|
|
OBSERVATION_UNITS = tuple(sorted(set(METRIC_UNITS.values())))
|
|
|
|
DURATION_NS_PER_SECOND = 10 ** 9
|
|
# The sampling cadence matches the lifecycle controller's own poll interval and
|
|
# is published as the observation precision. Each sample is bounded by the
|
|
# entry and depth caps below, and sampling ends at the first observed write.
|
|
OBSERVER_INTERVAL_SECONDS = 0.02
|
|
OBSERVER_JOIN_SECONDS = 10.0
|
|
OBSERVER_MAX_ENTRIES = 4096
|
|
OBSERVER_MAX_DEPTH = 16
|
|
MAX_OBSERVATION_RECORDS = 1000
|
|
DIGEST_PREFIX = "sha256:"
|
|
_PATH_DIGEST_DOMAIN = b"iop-benchmark-workspace-path-v1\0"
|
|
|
|
|
|
class MeasurementError(Exception):
|
|
"""Raised when measurement evidence cannot be produced or trusted."""
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Observation:
|
|
"""One required value that is either observed or explicitly unavailable."""
|
|
|
|
status: str
|
|
value: Optional[int]
|
|
unit: str
|
|
clock: str
|
|
source: str
|
|
reason: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class WorkspaceWriteObservation:
|
|
"""The bounded observer's report about the first observed workspace write."""
|
|
|
|
observed: bool
|
|
monotonic_ns: Optional[int]
|
|
mtime_ns: Optional[int]
|
|
path_digest: str
|
|
precision_ns: int
|
|
samples: int
|
|
reason: str = ""
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class WorkspaceScan:
|
|
"""One closed workspace snapshot; incomplete snapshots are never compared."""
|
|
|
|
files: dict[str, tuple[int, int, int]]
|
|
status: str
|
|
|
|
@property
|
|
def complete(self) -> bool:
|
|
return self.status == "complete"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AttemptMeasurement:
|
|
"""One immutable measurement record bound to exactly one attempt."""
|
|
|
|
run_id: str
|
|
cell_id: str
|
|
repetition: int
|
|
attempt: int
|
|
caller: str
|
|
spec_digest: str
|
|
product: ProductOutcome
|
|
harness: HarnessOutcome
|
|
process: ProcessOutcome
|
|
timeline: dict[str, Observation]
|
|
usage: dict[str, Observation]
|
|
observer: WorkspaceWriteObservation
|
|
observations: tuple[ParsedMetric, ...]
|
|
|
|
|
|
def observed(value: int, unit: str, clock: str, source: str) -> Observation:
|
|
"""Build one observed value with its exact unit, clock and source."""
|
|
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
|
raise MeasurementError("observed value must be a non-negative integer")
|
|
return Observation(STATUS_OBSERVED, value, unit, clock, source, "")
|
|
|
|
|
|
def unavailable(reason: str, source: str) -> Observation:
|
|
"""Build one explicitly unavailable value; never substitute a zero."""
|
|
if reason not in UNAVAILABLE_REASONS:
|
|
raise MeasurementError("unavailable reason is not a closed value")
|
|
return Observation(STATUS_UNAVAILABLE, None, "", "", source, reason)
|
|
|
|
|
|
def observation_record(observation: Observation) -> dict[str, Any]:
|
|
"""Return the canonical projection of one observed/unavailable value."""
|
|
if observation.status == STATUS_OBSERVED:
|
|
return {
|
|
"status": STATUS_OBSERVED,
|
|
"value": observation.value,
|
|
"unit": observation.unit,
|
|
"clock": observation.clock,
|
|
"source": observation.source,
|
|
}
|
|
if observation.status != STATUS_UNAVAILABLE:
|
|
raise MeasurementError("observation status is not a closed value")
|
|
return {
|
|
"status": STATUS_UNAVAILABLE,
|
|
"value": None,
|
|
"reason": observation.reason,
|
|
"source": observation.source,
|
|
}
|
|
|
|
|
|
def _observation_from_record(raw: Any) -> Observation:
|
|
if not isinstance(raw, dict):
|
|
raise MeasurementError("observation is invalid")
|
|
status = raw.get("status")
|
|
if status == STATUS_OBSERVED:
|
|
if set(raw) != {"status", "value", "unit", "clock", "source"}:
|
|
raise MeasurementError("observed value schema is invalid")
|
|
value, unit, clock = raw["value"], raw["unit"], raw["clock"]
|
|
if (
|
|
isinstance(value, bool) or not isinstance(value, int) or value < 0
|
|
or unit not in OBSERVATION_UNITS
|
|
or clock not in METRIC_CLOCKS or raw["source"] not in METRIC_SOURCES
|
|
):
|
|
raise MeasurementError("observed value is invalid")
|
|
return Observation(STATUS_OBSERVED, value, unit, clock, raw["source"], "")
|
|
if status == STATUS_UNAVAILABLE:
|
|
if set(raw) != {"status", "value", "reason", "source"}:
|
|
raise MeasurementError("unavailable value schema is invalid")
|
|
if (
|
|
raw["value"] is not None
|
|
or raw["reason"] not in UNAVAILABLE_REASONS
|
|
or raw["source"] not in METRIC_SOURCES
|
|
):
|
|
raise MeasurementError("unavailable value is invalid")
|
|
return Observation(STATUS_UNAVAILABLE, None, "", "", raw["source"], raw["reason"])
|
|
raise MeasurementError("observation status is not a closed value")
|
|
|
|
|
|
def _is_digest(value: Any) -> bool:
|
|
"""True for one exact lowercase sha256 identity string."""
|
|
if not isinstance(value, str) or not value.startswith(DIGEST_PREFIX):
|
|
return False
|
|
body = value[len(DIGEST_PREFIX):]
|
|
return len(body) == 64 and all(char in "0123456789abcdef" for char in body)
|
|
|
|
|
|
def path_digest(relative_path: str) -> str:
|
|
"""Digest one workspace-relative path so no caller-chosen name persists."""
|
|
return DIGEST_PREFIX + hashlib.sha256(
|
|
_PATH_DIGEST_DOMAIN + os.fsencode(relative_path)
|
|
).hexdigest()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Bounded workspace write observer
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _scan_workspace(root: Path) -> WorkspaceScan:
|
|
"""Snapshot contained regular files without following any link.
|
|
|
|
Every directory entry consumes one shared budget, including directories,
|
|
links and non-regular files. A cap, depth or I/O boundary returns an
|
|
incomplete result rather than a partial snapshot that could be compared.
|
|
"""
|
|
found: dict[str, tuple[int, int, int]] = {}
|
|
pending: deque[tuple[Path, int]] = deque([(root, 0)])
|
|
consumed = 0
|
|
while pending:
|
|
current, depth = pending.popleft()
|
|
try:
|
|
with os.scandir(current) as scan:
|
|
for entry in scan:
|
|
# Conservatively report exhaustion as soon as the bounded
|
|
# budget has been consumed. This avoids reading one more
|
|
# entry merely to distinguish an exactly-full directory.
|
|
if consumed >= OBSERVER_MAX_ENTRIES:
|
|
return WorkspaceScan(found, "exhausted")
|
|
consumed += 1
|
|
try:
|
|
if entry.is_symlink():
|
|
continue
|
|
if entry.is_dir(follow_symlinks=False):
|
|
if depth >= OBSERVER_MAX_DEPTH:
|
|
return WorkspaceScan(found, "exhausted")
|
|
pending.append((Path(entry.path), depth + 1))
|
|
continue
|
|
info = entry.stat(follow_symlinks=False)
|
|
if not stat.S_ISREG(info.st_mode):
|
|
continue
|
|
relative = os.path.relpath(entry.path, root)
|
|
except (OSError, ValueError):
|
|
return WorkspaceScan(found, "unavailable")
|
|
found[relative] = (info.st_mtime_ns, info.st_size, info.st_ino)
|
|
except OSError:
|
|
return WorkspaceScan(found, "unavailable")
|
|
return WorkspaceScan(found, "complete")
|
|
|
|
|
|
class WorkspaceWriteObserver:
|
|
"""Sample one workspace at a bounded interval and keep the first write seen.
|
|
|
|
The observer starts before the caller is invoked so that its baseline is
|
|
older than any caller write. It reports its own harness observation time,
|
|
the filesystem modification time it read, its source and its polling
|
|
precision. It never claims that a terminal snapshot proves the first write.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
root: str | Path,
|
|
*,
|
|
interval_seconds: float = OBSERVER_INTERVAL_SECONDS,
|
|
clock: Callable[[], int] = time.monotonic_ns,
|
|
) -> None:
|
|
self.root = Path(root)
|
|
if interval_seconds <= 0:
|
|
raise MeasurementError("observer interval must be positive")
|
|
self.interval_seconds = float(interval_seconds)
|
|
self._clock = clock
|
|
self._stop = threading.Event()
|
|
self._thread: Optional[threading.Thread] = None
|
|
self._baseline: dict[str, tuple[int, int, int]] = {}
|
|
self._baseline_complete = False
|
|
self._samples = 0
|
|
self._first: Optional[tuple[int, int, str]] = None
|
|
self._unavailable_reason = ""
|
|
self._started = False
|
|
self._stopped = False
|
|
self._final_observation: Optional[WorkspaceWriteObservation] = None
|
|
|
|
@property
|
|
def precision_ns(self) -> int:
|
|
return int(self.interval_seconds * DURATION_NS_PER_SECOND)
|
|
|
|
@property
|
|
def stopped(self) -> bool:
|
|
"""True once the sampling thread has been joined and is gone."""
|
|
return self._stopped
|
|
|
|
def start(self) -> None:
|
|
"""Take the immutable baseline, then start the sampling thread."""
|
|
if self._started:
|
|
raise MeasurementError("observer has already started")
|
|
if not self.root.is_dir() or self.root.is_symlink():
|
|
raise MeasurementError("observer root must be an existing directory")
|
|
self._started = True
|
|
baseline = _scan_workspace(self.root)
|
|
if not baseline.complete:
|
|
self._unavailable_reason = REASON_OBSERVER_UNAVAILABLE
|
|
return
|
|
self._baseline = baseline.files
|
|
self._baseline_complete = True
|
|
self._thread = threading.Thread(target=self._sample_until_stopped, daemon=True)
|
|
self._thread.start()
|
|
|
|
def _sample_until_stopped(self) -> None:
|
|
while not self._stop.is_set():
|
|
if self._sample_once():
|
|
return
|
|
self._stop.wait(self.interval_seconds)
|
|
|
|
def _sample_once(self) -> bool:
|
|
"""Return True once the first created or changed file has been seen."""
|
|
current = _scan_workspace(self.root)
|
|
self._samples += 1
|
|
if not current.complete:
|
|
self._unavailable_reason = REASON_OBSERVER_UNAVAILABLE
|
|
return True
|
|
changed = sorted(
|
|
(relative, info) for relative, info in current.files.items()
|
|
if self._baseline.get(relative) != info
|
|
)
|
|
if not changed:
|
|
return False
|
|
relative, info = changed[0]
|
|
# A detection instant is sampled only after the complete snapshot has
|
|
# found the change; it never labels scan work as observation time.
|
|
now = self._clock()
|
|
self._first = (now, info[0], relative)
|
|
return True
|
|
|
|
def stop(self) -> WorkspaceWriteObservation:
|
|
"""Stop and join the sampling thread, then freeze the observation.
|
|
|
|
Cleanup never raises, so it is safe on every terminal path; a thread
|
|
that refuses to leave is reported through :attr:`stopped` instead. A
|
|
fully stopped result is frozen and returned unchanged by every later
|
|
call, so a file written after shutdown can never become the
|
|
invocation's first write. A join that times out is not frozen so a
|
|
later call can retry cleanup once the sampler has exited.
|
|
"""
|
|
if self._final_observation is not None:
|
|
return self._final_observation
|
|
self._stop.set()
|
|
thread = self._thread
|
|
if thread is None:
|
|
self._stopped = self._started
|
|
else:
|
|
thread.join(OBSERVER_JOIN_SECONDS)
|
|
self._stopped = not thread.is_alive()
|
|
if self._stopped:
|
|
self._thread = None
|
|
if (
|
|
self._stopped
|
|
and self._baseline_complete
|
|
and self._first is None
|
|
and not self._unavailable_reason
|
|
):
|
|
# The joined sampler cannot race this final bounded scan. It closes
|
|
# the interval between its final poll and caller cleanup.
|
|
self._sample_once()
|
|
if self._first is None:
|
|
observation = WorkspaceWriteObservation(
|
|
False, None, None, "", self.precision_ns, self._samples,
|
|
self._unavailable_reason or REASON_NOT_OBSERVED,
|
|
)
|
|
else:
|
|
monotonic_ns, mtime_ns, relative = self._first
|
|
observation = WorkspaceWriteObservation(
|
|
True, monotonic_ns, mtime_ns, path_digest(relative),
|
|
self.precision_ns, self._samples,
|
|
)
|
|
if self._stopped:
|
|
self._final_observation = observation
|
|
return observation
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Measurement construction
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _event_instant(result: InvocationResult, kind: str) -> Observation:
|
|
for event in result.events:
|
|
if event.kind == kind:
|
|
return observed(
|
|
event.monotonic_ns, UNIT_NANOSECONDS,
|
|
CLOCK_HARNESS_MONOTONIC, SOURCE_HARNESS,
|
|
)
|
|
return unavailable(REASON_NOT_OBSERVED, SOURCE_HARNESS)
|
|
|
|
|
|
def _write_instant(value: Any, clock: str, observation: WorkspaceWriteObservation) -> Observation:
|
|
"""Report one observer value, or say plainly that it was not observed."""
|
|
if (not observation.observed or isinstance(value, bool)
|
|
or not isinstance(value, int) or value < 0):
|
|
return unavailable(
|
|
observation.reason or REASON_NOT_OBSERVED, SOURCE_WORKSPACE_POLL
|
|
)
|
|
return observed(value, UNIT_NANOSECONDS, clock, SOURCE_WORKSPACE_POLL)
|
|
|
|
|
|
def _timeline(
|
|
result: InvocationResult, observation: WorkspaceWriteObservation
|
|
) -> dict[str, Observation]:
|
|
"""Build the timeline without comparing values across clock domains."""
|
|
duration = result.duration_ns
|
|
return {
|
|
"submitted_at": _event_instant(result, EVENT_SUBMITTED),
|
|
"first_output_at": _event_instant(result, EVENT_FIRST_OUTPUT),
|
|
"first_write_observed_at": _write_instant(
|
|
observation.monotonic_ns, CLOCK_HARNESS_MONOTONIC, observation
|
|
),
|
|
# The filesystem clock is reported beside the harness clock and never
|
|
# subtracted from it; a caller of this record cannot mix the two.
|
|
"first_write_mtime": _write_instant(
|
|
observation.mtime_ns, CLOCK_FILESYSTEM_MTIME, observation
|
|
),
|
|
"total_duration": (
|
|
observed(duration, UNIT_NANOSECONDS, CLOCK_HARNESS_MONOTONIC, SOURCE_HARNESS)
|
|
if isinstance(duration, int) and not isinstance(duration, bool) and duration >= 0
|
|
else unavailable(REASON_NOT_OBSERVED, SOURCE_HARNESS)
|
|
),
|
|
}
|
|
|
|
|
|
def _usage(metrics: tuple[ParsedMetric, ...]) -> dict[str, Observation]:
|
|
"""Project only whole caller totals; never sum or split labelled intervals.
|
|
|
|
A stage or call label marks one part of a larger report, so only unstaged,
|
|
uncalled observations can be a total. Two totals with identical labels are
|
|
a contradiction and fail closed; two totals bound to different models are
|
|
ambiguous and stay unavailable rather than being merged.
|
|
"""
|
|
totals: dict[str, list[ParsedMetric]] = {}
|
|
for metric in metrics:
|
|
if metric.stage or metric.call_id:
|
|
continue
|
|
candidates = totals.setdefault(metric.name, [])
|
|
if any(item.model == metric.model for item in candidates):
|
|
raise MeasurementError("caller reported a duplicate total")
|
|
candidates.append(metric)
|
|
usage: dict[str, Observation] = {}
|
|
for name in METRIC_NAMES:
|
|
candidates = totals.get(name, [])
|
|
if len(candidates) == 1:
|
|
metric = candidates[0]
|
|
usage[name] = observed(metric.value, metric.unit, metric.clock, metric.source)
|
|
elif candidates:
|
|
usage[name] = unavailable(REASON_AMBIGUOUS_TOTAL, candidates[0].source)
|
|
else:
|
|
usage[name] = unavailable(REASON_NOT_REPORTED, SOURCE_HARNESS)
|
|
return usage
|
|
|
|
|
|
def build_measurement(
|
|
*,
|
|
run_id: str,
|
|
cell_id: str,
|
|
repetition: int,
|
|
attempt: int,
|
|
caller: str,
|
|
result: InvocationResult,
|
|
observation: WorkspaceWriteObservation,
|
|
) -> AttemptMeasurement:
|
|
"""Join lifecycle events, caller observations and the observer into one record."""
|
|
if not isinstance(result, InvocationResult):
|
|
raise MeasurementError("invocation result is invalid")
|
|
if not isinstance(observation, WorkspaceWriteObservation):
|
|
raise MeasurementError("workspace observation is invalid")
|
|
if any(not isinstance(text, str) or not text for text in (run_id, cell_id, caller)):
|
|
raise MeasurementError("measurement identity is invalid")
|
|
if any(
|
|
isinstance(number, bool) or not isinstance(number, int) or number < 1
|
|
for number in (repetition, attempt)
|
|
):
|
|
raise MeasurementError("measurement identity is invalid")
|
|
if not _is_digest(result.spec_digest):
|
|
raise MeasurementError("measurement invocation identity is invalid")
|
|
metrics = tuple(result.metrics)
|
|
if len(metrics) > MAX_OBSERVATION_RECORDS:
|
|
raise MeasurementError("observation count exceeds the bounded record")
|
|
try:
|
|
for metric in metrics:
|
|
validate_metric(metric)
|
|
except LifecycleMetricError as exc:
|
|
raise MeasurementError("caller observation is invalid") from exc
|
|
published = [
|
|
event.kind for event in result.events
|
|
if event.kind.startswith(METRIC_PREFIX)
|
|
and event.kind[len(METRIC_PREFIX):] in METRIC_UNITS
|
|
]
|
|
if published != [METRIC_PREFIX + metric.name for metric in metrics]:
|
|
raise MeasurementError("observation set does not match published events")
|
|
return AttemptMeasurement(
|
|
run_id=run_id,
|
|
cell_id=cell_id,
|
|
repetition=repetition,
|
|
attempt=attempt,
|
|
caller=caller,
|
|
spec_digest=result.spec_digest,
|
|
product=result.product,
|
|
harness=result.harness,
|
|
process=result.process,
|
|
timeline=_timeline(result, observation),
|
|
usage=_usage(metrics),
|
|
observer=observation,
|
|
observations=metrics,
|
|
)
|
|
|
|
|
|
def measurement_record(measurement: AttemptMeasurement) -> dict[str, Any]:
|
|
"""Return the canonical durable projection of one measurement."""
|
|
if not isinstance(measurement, AttemptMeasurement):
|
|
raise MeasurementError("measurement is invalid")
|
|
observer = measurement.observer
|
|
return {
|
|
"record": MEASUREMENT_RECORD,
|
|
"measurement_version": MEASUREMENT_VERSION,
|
|
"attempt": {
|
|
"run_id": measurement.run_id,
|
|
"cell_id": measurement.cell_id,
|
|
"repetition": measurement.repetition,
|
|
"attempt": measurement.attempt,
|
|
},
|
|
"caller": measurement.caller,
|
|
"spec_digest": measurement.spec_digest,
|
|
"product": {
|
|
"status": measurement.product.status,
|
|
"reason": measurement.product.reason,
|
|
},
|
|
"harness": {
|
|
"status": measurement.harness.status,
|
|
"reason": measurement.harness.reason,
|
|
"ordered_terminal": measurement.harness.ordered_terminal,
|
|
"cleanup_complete": measurement.harness.cleanup_complete,
|
|
},
|
|
"process": {
|
|
"status": measurement.process.status,
|
|
"exit_code": measurement.process.exit_code,
|
|
"signal": measurement.process.signal,
|
|
},
|
|
"timeline": {
|
|
name: observation_record(measurement.timeline[name])
|
|
for name in TIMELINE_NAMES
|
|
},
|
|
"usage": {
|
|
name: observation_record(measurement.usage[name]) for name in METRIC_NAMES
|
|
},
|
|
"observer": {
|
|
"source": SOURCE_WORKSPACE_POLL,
|
|
"status": STATUS_OBSERVED if observer.observed else STATUS_UNAVAILABLE,
|
|
"path_digest": observer.path_digest,
|
|
"precision_ns": observer.precision_ns,
|
|
"samples": observer.samples,
|
|
"reason": "" if observer.observed else (
|
|
observer.reason or REASON_NOT_OBSERVED
|
|
),
|
|
},
|
|
"observations": [metric_record(metric) for metric in measurement.observations],
|
|
}
|
|
|
|
|
|
def measurement_bytes(measurement: AttemptMeasurement) -> bytes:
|
|
"""Serialize one measurement into canonical, sorted, ASCII bytes."""
|
|
return json.dumps(
|
|
measurement_record(measurement), sort_keys=True, separators=(",", ":"),
|
|
ensure_ascii=True,
|
|
).encode("ascii") + b"\n"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Durable publication and strict loading
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def measurement_path(attempt_root: str | Path) -> Path:
|
|
return Path(attempt_root) / MEASUREMENT_FILENAME
|
|
|
|
|
|
def publish_measurement(
|
|
attempt_root: str | Path, measurement: AttemptMeasurement
|
|
) -> Path:
|
|
"""Publish the sidecar once; a collision never mutates the prior bytes."""
|
|
path = measurement_path(attempt_root)
|
|
data = measurement_bytes(measurement)
|
|
try:
|
|
publish_bytes_no_replace(path, data)
|
|
except OSError as exc:
|
|
raise MeasurementError("measurement publication refused an existing target") from exc
|
|
except Exception as exc: # lifecycle publication failure is never silent
|
|
raise MeasurementError("measurement publication failed") from exc
|
|
return path
|
|
|
|
|
|
def _read_regular_bytes(path: Path) -> bytes:
|
|
"""Read one durable file without following links or trusting its type."""
|
|
try:
|
|
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
|
|
except OSError as exc:
|
|
raise MeasurementError("measurement is unavailable") from exc
|
|
try:
|
|
if not stat.S_ISREG(os.fstat(fd).st_mode):
|
|
raise MeasurementError("measurement must be a regular file")
|
|
chunks: list[bytes] = []
|
|
while True:
|
|
chunk = os.read(fd, 1 << 20)
|
|
if not chunk:
|
|
return b"".join(chunks)
|
|
chunks.append(chunk)
|
|
finally:
|
|
os.close(fd)
|
|
|
|
|
|
def _metric_from_record(raw: Any) -> ParsedMetric:
|
|
fields = {
|
|
"name", "value", "unit", "clock", "source", "stage", "model", "call_id",
|
|
"overlap",
|
|
}
|
|
if not isinstance(raw, dict) or set(raw) != fields:
|
|
raise MeasurementError("observation schema is invalid")
|
|
try:
|
|
return validate_metric(ParsedMetric(
|
|
raw["name"], raw["value"], raw["unit"], raw["clock"], raw["source"],
|
|
raw["stage"], raw["model"], raw["call_id"], raw["overlap"],
|
|
))
|
|
except (LifecycleMetricError, TypeError) as exc:
|
|
raise MeasurementError("observation is invalid") from exc
|
|
|
|
|
|
def _observer_from_record(raw: Any) -> WorkspaceWriteObservation:
|
|
fields = {"source", "status", "path_digest", "precision_ns", "samples", "reason"}
|
|
if not isinstance(raw, dict) or set(raw) != fields:
|
|
raise MeasurementError("observer schema is invalid")
|
|
precision, samples = raw["precision_ns"], raw["samples"]
|
|
if (
|
|
raw["source"] != SOURCE_WORKSPACE_POLL
|
|
or raw["status"] not in (STATUS_OBSERVED, STATUS_UNAVAILABLE)
|
|
or isinstance(precision, bool) or not isinstance(precision, int) or precision <= 0
|
|
or isinstance(samples, bool) or not isinstance(samples, int) or samples < 0
|
|
or not isinstance(raw["path_digest"], str)
|
|
or not isinstance(raw["reason"], str)
|
|
):
|
|
raise MeasurementError("observer record is invalid")
|
|
seen = raw["status"] == STATUS_OBSERVED
|
|
digest = raw["path_digest"]
|
|
if (seen != bool(digest) or (digest and not _is_digest(digest))
|
|
or (seen and raw["reason"])
|
|
or (not seen and raw["reason"] not in UNAVAILABLE_REASONS)):
|
|
raise MeasurementError("observer record is invalid")
|
|
return WorkspaceWriteObservation(
|
|
seen, None, None, digest, precision, samples, raw["reason"]
|
|
)
|
|
|
|
|
|
def _identity_from_record(raw: Any) -> tuple[str, str, int, int]:
|
|
if not isinstance(raw, dict) or set(raw) != {
|
|
"run_id", "cell_id", "repetition", "attempt"
|
|
}:
|
|
raise MeasurementError("attempt identity schema is invalid")
|
|
repetition, attempt = raw["repetition"], raw["attempt"]
|
|
if (
|
|
not isinstance(raw["run_id"], str) or not raw["run_id"]
|
|
or not isinstance(raw["cell_id"], str) or not raw["cell_id"]
|
|
or isinstance(repetition, bool) or not isinstance(repetition, int) or repetition < 1
|
|
or isinstance(attempt, bool) or not isinstance(attempt, int) or attempt < 1
|
|
):
|
|
raise MeasurementError("attempt identity is invalid")
|
|
return raw["run_id"], raw["cell_id"], repetition, attempt
|
|
|
|
|
|
def _observation_map(raw: Any, names: tuple[str, ...], label: str) -> dict[str, Observation]:
|
|
if not isinstance(raw, dict) or set(raw) != set(names):
|
|
raise MeasurementError(f"{label} schema is invalid")
|
|
return {name: _observation_from_record(raw[name]) for name in names}
|
|
|
|
|
|
def load_measurement(attempt_root: str | Path) -> AttemptMeasurement:
|
|
"""Load one sidecar and revalidate every closed field before use."""
|
|
raw_bytes = _read_regular_bytes(measurement_path(attempt_root))
|
|
try:
|
|
record = json.loads(raw_bytes.decode("ascii"))
|
|
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
raise MeasurementError("measurement is not canonical JSON") from exc
|
|
fields = {
|
|
"record", "measurement_version", "attempt", "caller", "spec_digest",
|
|
"product", "harness", "process", "timeline", "usage", "observer",
|
|
"observations",
|
|
}
|
|
if not isinstance(record, dict) or set(record) != fields:
|
|
raise MeasurementError("measurement schema is invalid")
|
|
if (
|
|
record["record"] != MEASUREMENT_RECORD
|
|
or record["measurement_version"] != MEASUREMENT_VERSION
|
|
or not isinstance(record["caller"], str) or not record["caller"]
|
|
or not isinstance(record["spec_digest"], str) or not _is_digest(record["spec_digest"])
|
|
or not isinstance(record["observations"], list)
|
|
):
|
|
raise MeasurementError("measurement identity is invalid")
|
|
run_id, cell_id, repetition, attempt = _identity_from_record(record["attempt"])
|
|
try:
|
|
product = ProductOutcome(**record["product"])
|
|
harness = HarnessOutcome(**record["harness"])
|
|
process = ProcessOutcome(**record["process"])
|
|
except (TypeError, LifecycleValidationError, ValueError) as exc:
|
|
raise MeasurementError("measurement outcomes are invalid") from exc
|
|
measurement = AttemptMeasurement(
|
|
run_id=run_id,
|
|
cell_id=cell_id,
|
|
repetition=repetition,
|
|
attempt=attempt,
|
|
caller=record["caller"],
|
|
spec_digest=record["spec_digest"],
|
|
product=product,
|
|
harness=harness,
|
|
process=process,
|
|
timeline=_observation_map(record["timeline"], TIMELINE_NAMES, "timeline"),
|
|
usage=_observation_map(record["usage"], METRIC_NAMES, "usage"),
|
|
observer=_observer_from_record(record["observer"]),
|
|
observations=tuple(_metric_from_record(item) for item in record["observations"]),
|
|
)
|
|
if len(measurement.observations) > MAX_OBSERVATION_RECORDS:
|
|
raise MeasurementError("observation count exceeds the bounded record")
|
|
if measurement_record(measurement) != record or measurement_bytes(measurement) != raw_bytes:
|
|
raise MeasurementError("measurement is non-canonical")
|
|
_require_derived_coherence(measurement)
|
|
return measurement
|
|
|
|
|
|
def _require_derived_coherence(measurement: AttemptMeasurement) -> None:
|
|
"""Refuse any usage or write value that its own observations do not support."""
|
|
if measurement.usage != _usage(measurement.observations):
|
|
raise MeasurementError("usage does not match the recorded observations")
|
|
semantics = {
|
|
"submitted_at": (CLOCK_HARNESS_MONOTONIC, SOURCE_HARNESS),
|
|
"first_output_at": (CLOCK_HARNESS_MONOTONIC, SOURCE_HARNESS),
|
|
"first_write_observed_at": (CLOCK_HARNESS_MONOTONIC, SOURCE_WORKSPACE_POLL),
|
|
"first_write_mtime": (CLOCK_FILESYSTEM_MTIME, SOURCE_WORKSPACE_POLL),
|
|
"total_duration": (CLOCK_HARNESS_MONOTONIC, SOURCE_HARNESS),
|
|
}
|
|
for name, value in measurement.timeline.items():
|
|
clock, source = semantics[name]
|
|
if value.source != source:
|
|
raise MeasurementError("timeline source does not match its evidence")
|
|
if value.status == STATUS_OBSERVED and (
|
|
value.unit != UNIT_NANOSECONDS or value.clock != clock
|
|
):
|
|
raise MeasurementError("timeline clock does not match its evidence")
|
|
seen = measurement.observer.observed
|
|
write = measurement.timeline["first_write_observed_at"].status
|
|
mtime = measurement.timeline["first_write_mtime"].status
|
|
if ((not seen and STATUS_OBSERVED in (write, mtime))
|
|
or (seen and write != STATUS_OBSERVED)
|
|
or (seen and mtime != STATUS_OBSERVED)):
|
|
raise MeasurementError("workspace write does not match the observer")
|
|
|
|
|
|
def validate_measurement_lifecycle_binding(
|
|
measurement: AttemptMeasurement, lifecycle: Mapping[str, Any]
|
|
) -> None:
|
|
"""Bind a sidecar to the immutable lifecycle metric and timeline evidence.
|
|
|
|
The lifecycle result/journal validation belongs to ``RunStore``. This
|
|
helper consumes that already-validated terminal projection and refuses a
|
|
coherent sidecar rewrite whose observations or derived values no longer
|
|
correspond to its immutable event stream.
|
|
"""
|
|
if not isinstance(lifecycle, Mapping):
|
|
raise MeasurementError("lifecycle evidence is invalid")
|
|
expected_outcomes = {
|
|
"product": {
|
|
"status": measurement.product.status,
|
|
"reason": measurement.product.reason,
|
|
},
|
|
"harness": {
|
|
"status": measurement.harness.status,
|
|
"reason": measurement.harness.reason,
|
|
"ordered_terminal": measurement.harness.ordered_terminal,
|
|
"cleanup_complete": measurement.harness.cleanup_complete,
|
|
},
|
|
"process": {
|
|
"status": measurement.process.status,
|
|
"exit_code": measurement.process.exit_code,
|
|
"signal": measurement.process.signal,
|
|
},
|
|
}
|
|
if any(lifecycle.get(name) != value for name, value in expected_outcomes.items()):
|
|
raise MeasurementError("measurement outcomes do not match lifecycle evidence")
|
|
events = lifecycle.get("events")
|
|
if not isinstance(events, list):
|
|
raise MeasurementError("lifecycle events are invalid")
|
|
|
|
metric_events: list[ParsedMetric] = []
|
|
instants: dict[str, int] = {}
|
|
for event in events:
|
|
if not isinstance(event, Mapping):
|
|
raise MeasurementError("lifecycle event is invalid")
|
|
kind, source = event.get("kind"), event.get("source")
|
|
if kind in (EVENT_SUBMITTED, EVENT_FIRST_OUTPUT):
|
|
value = event.get("monotonic_ns")
|
|
if source != SOURCE_HARNESS or isinstance(value, bool) or not isinstance(value, int):
|
|
raise MeasurementError("lifecycle timeline evidence is invalid")
|
|
if kind in instants:
|
|
raise MeasurementError("lifecycle timeline evidence is ambiguous")
|
|
instants[kind] = value
|
|
if not isinstance(kind, str) or not kind.startswith(METRIC_PREFIX):
|
|
continue
|
|
name = kind[len(METRIC_PREFIX):]
|
|
try:
|
|
raw = json.loads(str(event.get("detail", "")))
|
|
metric = _metric_from_record(raw)
|
|
except (json.JSONDecodeError, MeasurementError) as exc:
|
|
raise MeasurementError("lifecycle metric evidence is invalid") from exc
|
|
if metric.name != name or source != metric.source:
|
|
raise MeasurementError("lifecycle metric evidence is invalid")
|
|
metric_events.append(metric)
|
|
|
|
if tuple(metric_events) != measurement.observations:
|
|
raise MeasurementError("measurement observations do not match lifecycle evidence")
|
|
required_instants = {
|
|
"submitted_at": EVENT_SUBMITTED,
|
|
"first_output_at": EVENT_FIRST_OUTPUT,
|
|
}
|
|
for timeline_name, event_kind in required_instants.items():
|
|
timeline = measurement.timeline[timeline_name]
|
|
event_value = instants.get(event_kind)
|
|
if event_value is None:
|
|
if timeline.status != STATUS_UNAVAILABLE:
|
|
raise MeasurementError("timeline claims a missing lifecycle event")
|
|
elif timeline.status != STATUS_OBSERVED or timeline.value != event_value:
|
|
raise MeasurementError("timeline does not match lifecycle evidence")
|
|
duration = lifecycle.get("duration_ns")
|
|
total = measurement.timeline["total_duration"]
|
|
if isinstance(duration, bool) or not isinstance(duration, int) or duration < 0:
|
|
raise MeasurementError("lifecycle duration is invalid")
|
|
if total.status != STATUS_OBSERVED or total.value != duration:
|
|
raise MeasurementError("timeline duration does not match lifecycle evidence")
|