1731 lines
72 KiB
Bash
Executable file
1731 lines
72 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
readonly EX_USAGE=64
|
|
readonly EX_UNAVAILABLE=69
|
|
readonly MAX_MANIFEST_BYTES=32768
|
|
readonly MAX_EVIDENCE_BYTES=131072
|
|
readonly FIELD_MILESTONE="logged-smoke-fixture"
|
|
readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
|
|
readonly REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd -P)"
|
|
readonly MANIFEST_SCHEMA="${SCRIPT_DIR}/fixtures/iop-agent-smoke-manifest.schema.json"
|
|
|
|
daemon_pid=""
|
|
daemon_start_token=""
|
|
daemon_log=""
|
|
|
|
usage() {
|
|
cat <<'EOF'
|
|
Usage:
|
|
e2e-iop-agent-logged-smoke.sh --preflight-only
|
|
e2e-iop-agent-logged-smoke.sh --self-test
|
|
e2e-iop-agent-logged-smoke.sh --validate-manifest FILE
|
|
e2e-iop-agent-logged-smoke.sh \
|
|
--binary FILE \
|
|
--repo-config FILE \
|
|
--local-config FILE \
|
|
--provider-catalog FILE \
|
|
--project-a DIRECTORY \
|
|
--project-b DIRECTORY \
|
|
--expected-head GIT_SHA \
|
|
--output DIRECTORY
|
|
|
|
The full mode is destructive only inside the two explicitly registered clean
|
|
project clones and the device-local runtime roots declared by --local-config.
|
|
It requires Darwin, jq, a clean exact source revision, authenticated provider
|
|
CLIs, two distinct clean clones, and no pre-existing daemon process or socket.
|
|
Raw command output remains under --output. manifest.json contains only bounded,
|
|
allow-listed, path-redacted evidence.
|
|
EOF
|
|
}
|
|
|
|
fail() {
|
|
printf 'logged-smoke: %s\n' "$*" >&2
|
|
exit 1
|
|
}
|
|
|
|
host_gate() {
|
|
local os_name
|
|
os_name="$(uname -s)"
|
|
if [[ "${os_name}" != "Darwin" ]]; then
|
|
printf 'logged-smoke: Darwin host required; observed %s before provider login or process launch\n' "${os_name}" >&2
|
|
exit "${EX_UNAVAILABLE}"
|
|
fi
|
|
}
|
|
|
|
sha256_file() {
|
|
shasum -a 256 "$1" | awk '{print "sha256:" $1}'
|
|
}
|
|
|
|
sha256_text() {
|
|
printf '%s' "$1" | shasum -a 256 | awk '{print "sha256:" $1}'
|
|
}
|
|
|
|
canonical_directory() {
|
|
[[ -d "$1" ]] || return 1
|
|
(cd "$1" && pwd -P)
|
|
}
|
|
|
|
canonical_file() {
|
|
[[ -f "$1" ]] || return 1
|
|
local directory
|
|
directory="$(cd "$(dirname "$1")" && pwd -P)"
|
|
printf '%s/%s\n' "${directory}" "$(basename "$1")"
|
|
}
|
|
|
|
process_start_token() {
|
|
local token
|
|
token="$(ps -o lstart= -p "$1" 2>/dev/null | awk '{$1=$1; print}')"
|
|
if [[ -n "${token}" ]]; then
|
|
printf 'ps:%s\n' "${token}"
|
|
fi
|
|
}
|
|
|
|
cleanup_owned_daemon() {
|
|
if [[ -z "${daemon_pid}" || -z "${daemon_start_token}" ]]; then
|
|
return 0
|
|
fi
|
|
local current
|
|
current="$(process_start_token "${daemon_pid}" || true)"
|
|
if [[ "${current}" != "${daemon_start_token}" ]]; then
|
|
printf 'logged-smoke: skip cleanup for PID %s because its start identity changed\n' "${daemon_pid}" >&2
|
|
daemon_pid=""
|
|
daemon_start_token=""
|
|
return 0
|
|
fi
|
|
kill -TERM "${daemon_pid}" 2>/dev/null || true
|
|
local waited=0
|
|
while kill -0 "${daemon_pid}" 2>/dev/null; do
|
|
if (( waited >= 100 )); then
|
|
current="$(process_start_token "${daemon_pid}" || true)"
|
|
if [[ "${current}" == "${daemon_start_token}" ]]; then
|
|
kill -KILL "${daemon_pid}" 2>/dev/null || true
|
|
fi
|
|
break
|
|
fi
|
|
sleep 0.1
|
|
waited=$((waited + 1))
|
|
done
|
|
wait "${daemon_pid}" 2>/dev/null || true
|
|
daemon_pid=""
|
|
daemon_start_token=""
|
|
}
|
|
|
|
crash_owned_daemon_preserving_invocations() {
|
|
[[ -n "${daemon_pid}" && -n "${daemon_start_token}" ]] ||
|
|
fail "owned daemon identity is unavailable for restart crash"
|
|
local current
|
|
current="$(process_start_token "${daemon_pid}" || true)"
|
|
[[ "${current}" == "${daemon_start_token}" ]] ||
|
|
fail "owned daemon identity changed before restart crash"
|
|
kill -KILL "${daemon_pid}"
|
|
wait "${daemon_pid}" 2>/dev/null || true
|
|
daemon_pid=""
|
|
daemon_start_token=""
|
|
if [[ -S "${daemon_socket}" ]]; then
|
|
rm "${daemon_socket}"
|
|
fi
|
|
[[ ! -e "${daemon_socket}" ]] ||
|
|
fail "owned daemon socket remained after restart crash"
|
|
}
|
|
|
|
diagnose_failure() {
|
|
local status=$?
|
|
if (( status != 0 )) && [[ -n "${daemon_log}" && -f "${daemon_log}" ]]; then
|
|
printf '%s\n' '--- bounded daemon diagnostic (last 80 lines) ---' >&2
|
|
tail -n 80 "${daemon_log}" >&2 || true
|
|
fi
|
|
cleanup_owned_daemon
|
|
exit "${status}"
|
|
}
|
|
|
|
trap diagnose_failure EXIT
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
|
|
validate_evidence_shape() {
|
|
local kind="$1"
|
|
local locator="$2"
|
|
local evidence_file="$3"
|
|
local manifest="$4"
|
|
local filter=""
|
|
case "${kind}:${locator}" in
|
|
source_build:build-evidence.json)
|
|
filter='
|
|
. as $e | $manifest as $m |
|
|
type == "object" and
|
|
exact_keys(["head","source_tree","declared_binary_sha256","rebuilt_binary_sha256","provider_smoke_sha256","schema_sha256"]) and
|
|
$e.head == $m.source.head and $e.source_tree == $m.source.source_tree and
|
|
$e.declared_binary_sha256 == $m.source.binary_sha256 and
|
|
$e.rebuilt_binary_sha256 == $m.source.rebuilt_binary_sha256 and
|
|
$e.schema_sha256 == $m.source.schema_sha256 and
|
|
($e.provider_smoke_sha256 | digest)'
|
|
;;
|
|
discovery_quota:discovery-quota.json)
|
|
filter='
|
|
type == "object" and exact_keys(["profiles"]) and
|
|
(.profiles | type == "array" and length >= 1 and length <= 16 and
|
|
all(.[];
|
|
type == "object" and
|
|
exact_keys(["profile_id_hash","discovery","quota_status","output_sha256"]) and
|
|
(.profile_id_hash | digest) and .discovery == "ready" and
|
|
.quota_status == "observed" and (.output_sha256 | digest)))'
|
|
;;
|
|
preview:preview-evidence.json)
|
|
filter='
|
|
type == "object" and
|
|
exact_keys(["project_a","project_b","selected","unblocked"]) and
|
|
(.project_a | digest) and (.project_b | digest) and
|
|
.selected == true and .unblocked == true'
|
|
;;
|
|
state_transition:start-evidence.json)
|
|
filter='
|
|
type == "object" and
|
|
exact_keys(["start_a","start_b","running_state"]) and
|
|
(.start_a | digest) and (.start_b | digest) and (.running_state | digest)'
|
|
;;
|
|
cancellation:cancellation-evidence.json)
|
|
filter='
|
|
type == "object" and
|
|
exact_keys(["stopped_state_sha256","cancelled_process_sha256","sibling_process_sha256","cancelled_process_live","sibling_process_live"]) and
|
|
(.stopped_state_sha256 | digest) and (.cancelled_process_sha256 | digest) and
|
|
(.sibling_process_sha256 | digest) and
|
|
.cancelled_process_live == false and .sibling_process_live == true'
|
|
;;
|
|
invocation_locator:locator-a.json|invocation_locator:locator-b.json)
|
|
filter='
|
|
type == "object" and exact_keys(["snapshots"]) and
|
|
(.snapshots | type == "array" and length >= 2 and length <= 4 and
|
|
all(.[];
|
|
type == "object" and
|
|
exact_keys(["project_id_hash","state_revision","project_status","invocations"]) and
|
|
(.project_id_hash | digest) and
|
|
(.state_revision | type == "string" and test("^[0-9]+$")) and
|
|
(.project_status | state) and
|
|
(.invocations | type == "array" and length >= 1 and length <= 16 and
|
|
all(.[];
|
|
type == "object" and
|
|
exact_keys(["attempt","dispatch_ordinal","state","locators"]) and
|
|
(.attempt | type == "string" and test("^[0-9]+$")) and
|
|
(.dispatch_ordinal | type == "string" and test("^[0-9]+$")) and
|
|
(.state | type == "string" and length >= 1 and length <= 64) and
|
|
(.locators | type == "array" and length >= 1 and length <= 2 and
|
|
any(.[]; .kind == "process") and
|
|
all(.[];
|
|
type == "object" and exact_keys(["kind","revision"]) and
|
|
(.kind == "process" or .kind == "session") and
|
|
(.revision | digest)))))))'
|
|
;;
|
|
restart:restart-evidence.json)
|
|
filter='
|
|
type == "object" and
|
|
exact_keys(["project_id_hash","pre","post","pre_process_live","post_process_live","same_attempt","same_process_locator","same_process_identity","duplicate_invocation"]) and
|
|
(.project_id_hash | digest) and
|
|
(.pre | restart_snapshot) and (.post | restart_snapshot) and
|
|
(.pre.state_revision | tonumber) < (.post.state_revision | tonumber) and
|
|
.pre.attempt_id_sha256 == .post.attempt_id_sha256 and
|
|
.pre.process_locator_revision == .post.process_locator_revision and
|
|
.pre.process_identity_sha256 == .post.process_identity_sha256 and
|
|
.pre_process_live == true and .post_process_live == true and
|
|
.same_attempt == true and .same_process_locator == true and
|
|
.same_process_identity == true and .duplicate_invocation == false'
|
|
;;
|
|
project_log:project-log-a.json|project_log:project-log-b.json)
|
|
filter='
|
|
type == "object" and exact_keys(["project_id_hash","logs"]) and
|
|
(.project_id_hash | digest) and
|
|
(.logs | type == "array" and length >= 1 and length <= 16 and
|
|
all(.[];
|
|
type == "object" and
|
|
exact_keys(["archive_ordinal","event_log_sha256","timeline_sha256"]) and
|
|
(.archive_ordinal | type == "string" and test("^[0-9]+$")) and
|
|
(.event_log_sha256 | digest) and (.timeline_sha256 | digest)))'
|
|
;;
|
|
terminal_archive:archive-a.json|terminal_archive:archive-b.json)
|
|
filter='
|
|
type == "object" and exact_keys(["project_id_hash","archives"]) and
|
|
(.project_id_hash | digest) and
|
|
(.archives | type == "array" and length >= 1 and length <= 16 and
|
|
all(.[];
|
|
type == "object" and
|
|
exact_keys(["archive_ordinal","record_count","checksum","terminal","manifest_sha256","event_log_sha256","timeline_sha256"]) and
|
|
(.archive_ordinal | type == "string" and test("^[0-9]+$")) and
|
|
(.record_count | type == "number" and . > 0) and
|
|
(.checksum | digest) and .terminal == true and
|
|
(.manifest_sha256 | digest) and (.event_log_sha256 | digest) and
|
|
(.timeline_sha256 | digest)))'
|
|
;;
|
|
terminal_archive:terminal-evidence.json)
|
|
filter='
|
|
type == "object" and
|
|
exact_keys(["status_a_sha256","status_b_sha256","archive_a_sha256","archive_b_sha256","terminal_status"]) and
|
|
(.status_a_sha256 | digest) and (.status_b_sha256 | digest) and
|
|
(.archive_a_sha256 | digest) and (.archive_b_sha256 | digest) and
|
|
.terminal_status == "completed"'
|
|
;;
|
|
*)
|
|
fail "evidence kind/locator pair is not allow-listed: ${kind}:${locator}"
|
|
;;
|
|
esac
|
|
jq -e --slurpfile manifest "${manifest}" "
|
|
def digest: type == \"string\" and test(\"^sha256:[0-9a-f]{64}$\");
|
|
def exact_keys(\$allowed): (keys | sort) == (\$allowed | sort);
|
|
def state: . == \"observed\" or . == \"started\" or . == \"running\" or
|
|
. == \"stopped\" or . == \"blocked\" or . == \"completed\" or
|
|
. == \"terminal_deferred\";
|
|
def restart_snapshot:
|
|
type == \"object\" and
|
|
exact_keys([\"state_revision\",\"attempt_id_sha256\",\"process_locator_revision\",\"process_identity_sha256\"]) and
|
|
(.state_revision | type == \"string\" and test(\"^[0-9]+$\")) and
|
|
(.attempt_id_sha256 | digest) and (.process_locator_revision | digest) and
|
|
(.process_identity_sha256 | digest);
|
|
\$manifest[0] as \$manifest | ${filter}
|
|
" "${evidence_file}" >/dev/null ||
|
|
fail "evidence file violates its ${kind} shape: ${locator}"
|
|
}
|
|
|
|
validate_evidence_file() {
|
|
local manifest_dir="$1"
|
|
local kind="$2"
|
|
local locator="$3"
|
|
local expected_sha256="$4"
|
|
local manifest="$5"
|
|
[[ "${locator}" == "$(basename "${locator}")" ]] ||
|
|
fail "evidence locator escapes the manifest directory: ${locator}"
|
|
local evidence_file="${manifest_dir}/${locator}"
|
|
[[ ! -L "${evidence_file}" ]] || fail "evidence locator is a symlink: ${locator}"
|
|
[[ -f "${evidence_file}" ]] || fail "evidence locator is not a regular file: ${locator}"
|
|
local size
|
|
size="$(wc -c <"${evidence_file}" | tr -d ' ')"
|
|
(( size > 0 && size <= MAX_EVIDENCE_BYTES )) ||
|
|
fail "evidence ${locator} size ${size} is outside 1..${MAX_EVIDENCE_BYTES} bytes"
|
|
if LC_ALL=C grep -Eiq \
|
|
'authorization|bearer|api[_-]?key|credential|secret|token|/Users/|/home/|\\\\Users\\\\' \
|
|
"${evidence_file}"; then
|
|
fail "evidence contains a forbidden key or value: ${locator}"
|
|
fi
|
|
jq -e . "${evidence_file}" >/dev/null || fail "evidence is not valid JSON: ${locator}"
|
|
[[ "$(sha256_file "${evidence_file}")" == "${expected_sha256}" ]] ||
|
|
fail "evidence digest mismatch: ${locator}"
|
|
validate_evidence_shape "${kind}" "${locator}" "${evidence_file}" "${manifest}"
|
|
}
|
|
|
|
validate_manifest() {
|
|
local manifest="$1"
|
|
[[ -f "${MANIFEST_SCHEMA}" ]] || fail "missing manifest schema: ${MANIFEST_SCHEMA}"
|
|
[[ ! -L "${manifest}" ]] || fail "manifest must not be a symlink"
|
|
[[ -f "${manifest}" ]] || fail "manifest is not a regular file"
|
|
command -v jq >/dev/null 2>&1 || fail "jq is required for manifest validation"
|
|
local size
|
|
size="$(wc -c <"${manifest}" | tr -d ' ')"
|
|
(( size > 0 && size <= MAX_MANIFEST_BYTES )) ||
|
|
fail "manifest size ${size} is outside 1..${MAX_MANIFEST_BYTES} bytes"
|
|
if LC_ALL=C grep -Eiq \
|
|
'authorization|bearer|api[_-]?key|credential|secret|token|/Users/|/home/|\\\\Users\\\\' \
|
|
"${manifest}"; then
|
|
fail "manifest contains a forbidden key or value"
|
|
fi
|
|
jq -e '
|
|
def digest: type == "string" and test("^sha256:[0-9a-f]{64}$");
|
|
def exact_keys($allowed): (keys | sort) == ($allowed | sort);
|
|
def state: . == "observed" or . == "started" or . == "running" or
|
|
. == "stopped" or . == "blocked" or . == "completed" or
|
|
. == "terminal_deferred";
|
|
def evidence($root; $kind; $digest):
|
|
any($root.evidence.records[]; .kind == $kind and .sha256 == $digest);
|
|
def verified($root; $kind):
|
|
type == "object" and exact_keys(["status","evidence_sha256"]) and
|
|
.status == "verified" and (.evidence_sha256 | digest) and
|
|
evidence($root; $kind; .evidence_sha256);
|
|
def index_of($value): to_entries | map(select(.value == $value) | .key) |
|
|
if length == 0 then -1 else .[0] end;
|
|
def completed_absorbing:
|
|
(index_of("completed")) as $completed |
|
|
$completed >= 0 and all(.[$completed:][]; . == "completed") and
|
|
(index_of("blocked")) == -1 and (index_of("terminal_deferred")) == -1;
|
|
. as $root |
|
|
type == "object" and
|
|
exact_keys(["schema_version","run_id","source","environment","lifecycle","projects","evidence"]) and
|
|
.schema_version == "2" and
|
|
(.run_id | digest) and
|
|
(.source | type == "object" and
|
|
exact_keys(["head","source_tree","binary_sha256","rebuilt_binary_sha256","binary_build_head","repo_config_sha256","local_config_sha256","provider_catalog_sha256","build_evidence_sha256","schema_sha256"]) and
|
|
(.head | test("^[0-9a-f]{40,64}$")) and
|
|
(.source_tree | test("^[0-9a-f]{40,64}$")) and
|
|
.binary_build_head == .head and
|
|
.rebuilt_binary_sha256 == .binary_sha256 and
|
|
(.binary_sha256 | digest) and (.build_evidence_sha256 | digest) and
|
|
evidence($root; "source_build"; .build_evidence_sha256) and
|
|
(.schema_sha256 | digest) and .schema_sha256 == $root.evidence.schema_sha256 and
|
|
(.repo_config_sha256 | digest) and
|
|
(.local_config_sha256 | digest) and (.provider_catalog_sha256 | digest)) and
|
|
(.environment | type == "object" and exact_keys(["os","arch"]) and
|
|
.os == "darwin" and (.arch == "arm64" or .arch == "amd64")) and
|
|
(.lifecycle | type == "object" and
|
|
exact_keys(["discovery","preview","manual_start","cancellation","sibling_continuation","new_invocation","restart_recovery","terminal_completion"]) and
|
|
(.discovery | verified($root; "discovery_quota")) and
|
|
(.preview | verified($root; "preview")) and
|
|
(.manual_start | verified($root; "state_transition")) and
|
|
(.cancellation | verified($root; "cancellation")) and
|
|
(.sibling_continuation | verified($root; "cancellation")) and
|
|
(.new_invocation | verified($root; "invocation_locator")) and
|
|
(.restart_recovery | verified($root; "restart")) and
|
|
(.terminal_completion | verified($root; "terminal_archive"))) and
|
|
(.projects | type == "array" and length == 2 and
|
|
([.[].slot] | sort) == ["a","b"] and
|
|
all(.[];
|
|
type == "object" and
|
|
exact_keys(["slot","project_id_hash","workspace_revision","status_trace","terminal_status","initial_invocation_sha256","post_restart_invocation_sha256","locator_evidence_sha256","project_log_sha256","terminal_archive_sha256"]) and
|
|
(.project_id_hash | digest) and
|
|
(.workspace_revision | test("^[0-9a-f]{40,64}$")) and
|
|
(.status_trace | type == "array" and length >= 3 and length <= 16 and
|
|
all(.[]; state) and completed_absorbing) and
|
|
(.terminal_status == "completed") and
|
|
(.initial_invocation_sha256 | digest) and
|
|
(.post_restart_invocation_sha256 | digest) and
|
|
(.locator_evidence_sha256 | digest) and
|
|
(.project_log_sha256 | digest) and
|
|
(.terminal_archive_sha256 | digest) and
|
|
evidence($root; "invocation_locator"; .locator_evidence_sha256) and
|
|
evidence($root; "project_log"; .project_log_sha256) and
|
|
evidence($root; "terminal_archive"; .terminal_archive_sha256))) and
|
|
((.projects[] | select(.slot == "a")) as $a |
|
|
($a.status_trace | index_of("running")) >= 0 and
|
|
($a.status_trace | index_of("stopped")) >
|
|
($a.status_trace | index_of("running")) and
|
|
($a.status_trace | index_of("completed")) >
|
|
($a.status_trace | index_of("stopped")) and
|
|
$a.initial_invocation_sha256 != $a.post_restart_invocation_sha256) and
|
|
((.projects[] | select(.slot == "b")) as $b |
|
|
($b.status_trace | index_of("running")) >= 0 and
|
|
($b.status_trace | index_of("stopped")) == -1 and
|
|
($b.status_trace | index_of("completed")) >
|
|
($b.status_trace | index_of("running")) and
|
|
$b.initial_invocation_sha256 == $b.post_restart_invocation_sha256) and
|
|
(.evidence | type == "object" and
|
|
exact_keys(["manifest_locator","schema_sha256","records"]) and
|
|
.manifest_locator == "manifest.json" and
|
|
(.schema_sha256 | digest) and
|
|
(.records | type == "array" and length == 13 and
|
|
([.[].locator] | unique | length) == length and
|
|
all(.[];
|
|
type == "object" and exact_keys(["kind","locator","sha256"]) and
|
|
(.kind == "source_build" or .kind == "discovery_quota" or
|
|
.kind == "preview" or .kind == "state_transition" or
|
|
.kind == "cancellation" or .kind == "invocation_locator" or
|
|
.kind == "restart" or .kind == "project_log" or
|
|
.kind == "terminal_archive") and
|
|
(.locator | test("^[a-z0-9][a-z0-9._-]{0,127}\\.json$")) and
|
|
(.sha256 | digest))))
|
|
' "${manifest}" >/dev/null || fail "manifest violates the bounded allow-list contract"
|
|
local manifest_dir
|
|
manifest_dir="$(cd "$(dirname "${manifest}")" && pwd -P)"
|
|
[[ "$(jq -r '.evidence.schema_sha256' "${manifest}")" == "$(sha256_file "${MANIFEST_SCHEMA}")" ]] ||
|
|
fail "manifest schema digest differs from the tracked schema"
|
|
jq -e '
|
|
[.evidence.records[] | {kind,locator}] == [
|
|
{kind:"source_build",locator:"build-evidence.json"},
|
|
{kind:"discovery_quota",locator:"discovery-quota.json"},
|
|
{kind:"preview",locator:"preview-evidence.json"},
|
|
{kind:"state_transition",locator:"start-evidence.json"},
|
|
{kind:"cancellation",locator:"cancellation-evidence.json"},
|
|
{kind:"invocation_locator",locator:"locator-a.json"},
|
|
{kind:"invocation_locator",locator:"locator-b.json"},
|
|
{kind:"restart",locator:"restart-evidence.json"},
|
|
{kind:"project_log",locator:"project-log-a.json"},
|
|
{kind:"project_log",locator:"project-log-b.json"},
|
|
{kind:"terminal_archive",locator:"archive-a.json"},
|
|
{kind:"terminal_archive",locator:"archive-b.json"},
|
|
{kind:"terminal_archive",locator:"terminal-evidence.json"}
|
|
]
|
|
' "${manifest}" >/dev/null || fail "manifest evidence record set is not exact"
|
|
while IFS=$'\t' read -r kind locator expected_sha256; do
|
|
validate_evidence_file \
|
|
"${manifest_dir}" "${kind}" "${locator}" "${expected_sha256}" "${manifest}"
|
|
done < <(jq -r '.evidence.records[] | [.kind,.locator,.sha256] | @tsv' "${manifest}")
|
|
printf 'logged-smoke: manifest valid (%s bytes)\n' "${size}"
|
|
}
|
|
|
|
self_test() {
|
|
command -v jq >/dev/null 2>&1 || fail "jq is required for self-test"
|
|
command -v shasum >/dev/null 2>&1 || fail "shasum is required for self-test"
|
|
local temporary
|
|
temporary="$(mktemp -d "${TMPDIR:-/tmp}/iop-agent-smoke-self-test.XXXXXX")"
|
|
local bundle="${temporary}/bundle"
|
|
mkdir "${bundle}"
|
|
local safe="${bundle}/manifest.json"
|
|
local next="${bundle}/manifest.next.json"
|
|
local baseline="${temporary}/manifest.baseline.json"
|
|
local digest_a="sha256:$(printf '0%.0s' {1..64})"
|
|
local digest_b="sha256:$(printf '1%.0s' {1..64})"
|
|
local digest_c="sha256:$(printf '2%.0s' {1..64})"
|
|
local schema_digest
|
|
schema_digest="$(sha256_file "${MANIFEST_SCHEMA}")"
|
|
|
|
jq -n \
|
|
--arg a "${digest_a}" --arg schema "${schema_digest}" \
|
|
'{
|
|
head:"0000000000000000000000000000000000000000",
|
|
source_tree:"0000000000000000000000000000000000000000",
|
|
declared_binary_sha256:$a,
|
|
rebuilt_binary_sha256:$a,
|
|
provider_smoke_sha256:$a,
|
|
schema_sha256:$schema
|
|
}' >"${bundle}/build-evidence.json"
|
|
jq -n --arg a "${digest_a}" \
|
|
'{profiles:[{profile_id_hash:$a,discovery:"ready",quota_status:"observed",output_sha256:$a}]}' \
|
|
>"${bundle}/discovery-quota.json"
|
|
jq -n --arg a "${digest_a}" --arg b "${digest_b}" \
|
|
'{project_a:$a,project_b:$b,selected:true,unblocked:true}' \
|
|
>"${bundle}/preview-evidence.json"
|
|
jq -n --arg a "${digest_a}" --arg b "${digest_b}" --arg c "${digest_c}" \
|
|
'{start_a:$a,start_b:$b,running_state:$c}' >"${bundle}/start-evidence.json"
|
|
jq -n --arg a "${digest_a}" --arg b "${digest_b}" --arg c "${digest_c}" \
|
|
'{
|
|
stopped_state_sha256:$a,
|
|
cancelled_process_sha256:$b,
|
|
sibling_process_sha256:$c,
|
|
cancelled_process_live:false,
|
|
sibling_process_live:true
|
|
}' >"${bundle}/cancellation-evidence.json"
|
|
jq -n --arg project "${digest_a}" --arg revision "${digest_b}" \
|
|
'{
|
|
snapshots:[
|
|
{
|
|
project_id_hash:$project,state_revision:"2",project_status:"running",
|
|
invocations:[{
|
|
attempt:"1",dispatch_ordinal:"1",state:"running",
|
|
locators:[{kind:"process",revision:$revision}]
|
|
}]
|
|
},
|
|
{
|
|
project_id_hash:$project,state_revision:"8",project_status:"completed",
|
|
invocations:[{
|
|
attempt:"2",dispatch_ordinal:"2",state:"completed",
|
|
locators:[{kind:"process",revision:$revision}]
|
|
}]
|
|
}
|
|
]
|
|
}' >"${bundle}/locator-a.json"
|
|
jq -n --arg project "${digest_b}" --arg revision "${digest_c}" \
|
|
'{
|
|
snapshots:[
|
|
{
|
|
project_id_hash:$project,state_revision:"2",project_status:"running",
|
|
invocations:[{
|
|
attempt:"1",dispatch_ordinal:"1",state:"running",
|
|
locators:[{kind:"process",revision:$revision}]
|
|
}]
|
|
},
|
|
{
|
|
project_id_hash:$project,state_revision:"8",project_status:"completed",
|
|
invocations:[{
|
|
attempt:"1",dispatch_ordinal:"1",state:"completed",
|
|
locators:[{kind:"process",revision:$revision}]
|
|
}]
|
|
}
|
|
]
|
|
}' >"${bundle}/locator-b.json"
|
|
jq -n \
|
|
--arg project "${digest_b}" --arg attempt "${digest_a}" \
|
|
--arg locator "${digest_b}" --arg process "${digest_c}" \
|
|
'{
|
|
project_id_hash:$project,
|
|
pre:{
|
|
state_revision:"4",attempt_id_sha256:$attempt,
|
|
process_locator_revision:$locator,process_identity_sha256:$process
|
|
},
|
|
post:{
|
|
state_revision:"5",attempt_id_sha256:$attempt,
|
|
process_locator_revision:$locator,process_identity_sha256:$process
|
|
},
|
|
pre_process_live:true,
|
|
post_process_live:true,
|
|
same_attempt:true,
|
|
same_process_locator:true,
|
|
same_process_identity:true,
|
|
duplicate_invocation:false
|
|
}' >"${bundle}/restart-evidence.json"
|
|
jq -n --arg project "${digest_a}" --arg a "${digest_a}" --arg b "${digest_b}" \
|
|
'{project_id_hash:$project,logs:[{archive_ordinal:"1",event_log_sha256:$a,timeline_sha256:$b}]}' \
|
|
>"${bundle}/project-log-a.json"
|
|
jq -n --arg project "${digest_b}" --arg a "${digest_b}" --arg b "${digest_c}" \
|
|
'{project_id_hash:$project,logs:[{archive_ordinal:"1",event_log_sha256:$a,timeline_sha256:$b}]}' \
|
|
>"${bundle}/project-log-b.json"
|
|
jq -n --arg project "${digest_a}" --arg a "${digest_a}" --arg b "${digest_b}" --arg c "${digest_c}" \
|
|
'{
|
|
project_id_hash:$project,
|
|
archives:[{
|
|
archive_ordinal:"1",record_count:1,checksum:$a,terminal:true,
|
|
manifest_sha256:$a,event_log_sha256:$b,timeline_sha256:$c
|
|
}]
|
|
}' >"${bundle}/archive-a.json"
|
|
jq -n --arg project "${digest_b}" --arg a "${digest_a}" --arg b "${digest_b}" --arg c "${digest_c}" \
|
|
'{
|
|
project_id_hash:$project,
|
|
archives:[{
|
|
archive_ordinal:"1",record_count:1,checksum:$b,terminal:true,
|
|
manifest_sha256:$c,event_log_sha256:$a,timeline_sha256:$b
|
|
}]
|
|
}' >"${bundle}/archive-b.json"
|
|
jq -n --arg a "${digest_a}" --arg b "${digest_b}" --arg c "${digest_c}" \
|
|
'{
|
|
status_a_sha256:$a,status_b_sha256:$b,
|
|
archive_a_sha256:$b,archive_b_sha256:$c,terminal_status:"completed"
|
|
}' >"${bundle}/terminal-evidence.json"
|
|
|
|
local build_sha discovery_sha preview_sha start_sha cancellation_sha
|
|
local locator_a_sha locator_b_sha restart_sha log_a_sha log_b_sha
|
|
local archive_a_sha archive_b_sha terminal_sha
|
|
build_sha="$(sha256_file "${bundle}/build-evidence.json")"
|
|
discovery_sha="$(sha256_file "${bundle}/discovery-quota.json")"
|
|
preview_sha="$(sha256_file "${bundle}/preview-evidence.json")"
|
|
start_sha="$(sha256_file "${bundle}/start-evidence.json")"
|
|
cancellation_sha="$(sha256_file "${bundle}/cancellation-evidence.json")"
|
|
locator_a_sha="$(sha256_file "${bundle}/locator-a.json")"
|
|
locator_b_sha="$(sha256_file "${bundle}/locator-b.json")"
|
|
restart_sha="$(sha256_file "${bundle}/restart-evidence.json")"
|
|
log_a_sha="$(sha256_file "${bundle}/project-log-a.json")"
|
|
log_b_sha="$(sha256_file "${bundle}/project-log-b.json")"
|
|
archive_a_sha="$(sha256_file "${bundle}/archive-a.json")"
|
|
archive_b_sha="$(sha256_file "${bundle}/archive-b.json")"
|
|
terminal_sha="$(sha256_file "${bundle}/terminal-evidence.json")"
|
|
jq -n \
|
|
--arg a "${digest_a}" \
|
|
--arg b "${digest_b}" \
|
|
--arg c "${digest_c}" \
|
|
--arg schema "${schema_digest}" \
|
|
--arg build "${build_sha}" \
|
|
--arg discovery "${discovery_sha}" \
|
|
--arg preview "${preview_sha}" \
|
|
--arg start "${start_sha}" \
|
|
--arg cancellation "${cancellation_sha}" \
|
|
--arg locator_a "${locator_a_sha}" \
|
|
--arg locator_b "${locator_b_sha}" \
|
|
--arg restart "${restart_sha}" \
|
|
--arg log_a "${log_a_sha}" \
|
|
--arg log_b "${log_b_sha}" \
|
|
--arg archive_a "${archive_a_sha}" \
|
|
--arg archive_b "${archive_b_sha}" \
|
|
--arg terminal "${terminal_sha}" \
|
|
'{
|
|
schema_version:"2",
|
|
run_id:$a,
|
|
source:{
|
|
head:"0000000000000000000000000000000000000000",
|
|
source_tree:"0000000000000000000000000000000000000000",
|
|
binary_sha256:$a,
|
|
rebuilt_binary_sha256:$a,
|
|
binary_build_head:"0000000000000000000000000000000000000000",
|
|
repo_config_sha256:$a,
|
|
local_config_sha256:$a,
|
|
provider_catalog_sha256:$a,
|
|
build_evidence_sha256:$build,
|
|
schema_sha256:$schema
|
|
},
|
|
environment:{os:"darwin",arch:"arm64"},
|
|
lifecycle:{
|
|
discovery:{status:"verified",evidence_sha256:$discovery},
|
|
preview:{status:"verified",evidence_sha256:$preview},
|
|
manual_start:{status:"verified",evidence_sha256:$start},
|
|
cancellation:{status:"verified",evidence_sha256:$cancellation},
|
|
sibling_continuation:{status:"verified",evidence_sha256:$cancellation},
|
|
new_invocation:{status:"verified",evidence_sha256:$locator_a},
|
|
restart_recovery:{status:"verified",evidence_sha256:$restart},
|
|
terminal_completion:{status:"verified",evidence_sha256:$terminal}
|
|
},
|
|
projects:[
|
|
{
|
|
slot:"a",project_id_hash:$a,
|
|
workspace_revision:"0000000000000000000000000000000000000000",
|
|
status_trace:["started","running","stopped","running","completed"],
|
|
terminal_status:"completed",
|
|
initial_invocation_sha256:$a,
|
|
post_restart_invocation_sha256:$b,
|
|
locator_evidence_sha256:$locator_a,
|
|
project_log_sha256:$log_a,
|
|
terminal_archive_sha256:$archive_a
|
|
},
|
|
{
|
|
slot:"b",project_id_hash:$b,
|
|
workspace_revision:"0000000000000000000000000000000000000000",
|
|
status_trace:["started","running","completed"],
|
|
terminal_status:"completed",
|
|
initial_invocation_sha256:$c,
|
|
post_restart_invocation_sha256:$c,
|
|
locator_evidence_sha256:$locator_b,
|
|
project_log_sha256:$log_b,
|
|
terminal_archive_sha256:$archive_b
|
|
}
|
|
],
|
|
evidence:{
|
|
manifest_locator:"manifest.json",
|
|
schema_sha256:$schema,
|
|
records:[
|
|
{kind:"source_build",locator:"build-evidence.json",sha256:$build},
|
|
{kind:"discovery_quota",locator:"discovery-quota.json",sha256:$discovery},
|
|
{kind:"preview",locator:"preview-evidence.json",sha256:$preview},
|
|
{kind:"state_transition",locator:"start-evidence.json",sha256:$start},
|
|
{kind:"cancellation",locator:"cancellation-evidence.json",sha256:$cancellation},
|
|
{kind:"invocation_locator",locator:"locator-a.json",sha256:$locator_a},
|
|
{kind:"invocation_locator",locator:"locator-b.json",sha256:$locator_b},
|
|
{kind:"restart",locator:"restart-evidence.json",sha256:$restart},
|
|
{kind:"project_log",locator:"project-log-a.json",sha256:$log_a},
|
|
{kind:"project_log",locator:"project-log-b.json",sha256:$log_b},
|
|
{kind:"terminal_archive",locator:"archive-a.json",sha256:$archive_a},
|
|
{kind:"terminal_archive",locator:"archive-b.json",sha256:$archive_b},
|
|
{kind:"terminal_archive",locator:"terminal-evidence.json",sha256:$terminal}
|
|
]
|
|
}
|
|
}' >"${safe}"
|
|
validate_manifest "${safe}" >/dev/null
|
|
cp "${safe}" "${baseline}"
|
|
|
|
reject_manifest_mutation() {
|
|
local name="$1"
|
|
local filter="$2"
|
|
jq --arg digest "${digest_b}" "${filter}" "${baseline}" >"${next}"
|
|
mv "${next}" "${safe}"
|
|
if (validate_manifest "${safe}" >/dev/null 2>&1); then
|
|
fail "self-test accepted fabricated evidence: ${name}"
|
|
fi
|
|
cp "${baseline}" "${safe}"
|
|
}
|
|
|
|
reject_manifest_mutation "post-terminal-running" \
|
|
'.projects[1].status_trace = ["started","running","completed","running"]'
|
|
reject_manifest_mutation "reused-cancelled-invocation" \
|
|
'.projects[0].post_restart_invocation_sha256 = .projects[0].initial_invocation_sha256'
|
|
reject_manifest_mutation "binary-head-mismatch" \
|
|
'.source.binary_build_head = "1111111111111111111111111111111111111111"'
|
|
reject_manifest_mutation "changed-record-digest" \
|
|
'.evidence.records[0].sha256 = $digest'
|
|
reject_manifest_mutation "changed-schema-digest" \
|
|
'.source.schema_sha256 = $digest | .evidence.schema_sha256 = $digest'
|
|
reject_manifest_mutation "escaped-locator" \
|
|
'.evidence.records[0].locator = "../build-evidence.json"'
|
|
reject_manifest_mutation "duplicate-locator" \
|
|
'.evidence.records[12].locator = .evidence.records[11].locator'
|
|
|
|
mv "${bundle}/preview-evidence.json" "${bundle}/preview-evidence.saved.json"
|
|
if (validate_manifest "${safe}" >/dev/null 2>&1); then
|
|
fail "self-test accepted a deleted evidence file"
|
|
fi
|
|
mv "${bundle}/preview-evidence.saved.json" "${bundle}/preview-evidence.json"
|
|
|
|
mv "${bundle}/preview-evidence.json" "${bundle}/preview-evidence.saved.json"
|
|
ln -s "preview-evidence.saved.json" "${bundle}/preview-evidence.json"
|
|
if (validate_manifest "${safe}" >/dev/null 2>&1); then
|
|
fail "self-test accepted a symlinked evidence file"
|
|
fi
|
|
rm "${bundle}/preview-evidence.json"
|
|
mv "${bundle}/preview-evidence.saved.json" "${bundle}/preview-evidence.json"
|
|
|
|
cp "${bundle}/preview-evidence.json" "${bundle}/preview-evidence.saved.json"
|
|
printf ' ' >>"${bundle}/preview-evidence.json"
|
|
if (validate_manifest "${safe}" >/dev/null 2>&1); then
|
|
fail "self-test accepted tampered evidence bytes"
|
|
fi
|
|
mv "${bundle}/preview-evidence.saved.json" "${bundle}/preview-evidence.json"
|
|
|
|
reject_evidence_mutation() {
|
|
local name="$1"
|
|
local locator="$2"
|
|
local filter="$3"
|
|
local evidence_saved="${temporary}/${locator}.saved"
|
|
cp "${bundle}/${locator}" "${evidence_saved}"
|
|
jq --arg replacement "${digest_a}" "${filter}" "${evidence_saved}" \
|
|
>"${bundle}/${locator}"
|
|
local changed_sha
|
|
changed_sha="$(sha256_file "${bundle}/${locator}")"
|
|
jq --arg locator "${locator}" --arg digest "${changed_sha}" '
|
|
(.evidence.records[] | select(.locator == $locator)).sha256 = $digest |
|
|
if $locator == "restart-evidence.json" then
|
|
.lifecycle.restart_recovery.evidence_sha256 = $digest
|
|
elif $locator == "preview-evidence.json" then
|
|
.lifecycle.preview.evidence_sha256 = $digest
|
|
else .
|
|
end
|
|
' "${baseline}" >"${safe}"
|
|
if (validate_manifest "${safe}" >/dev/null 2>&1); then
|
|
fail "self-test accepted fabricated evidence content: ${name}"
|
|
fi
|
|
mv "${evidence_saved}" "${bundle}/${locator}"
|
|
cp "${baseline}" "${safe}"
|
|
}
|
|
|
|
reject_evidence_mutation "kind-schema-mismatch" "preview-evidence.json" \
|
|
'{head:"0000000000000000000000000000000000000000"}'
|
|
reject_evidence_mutation "same-attempt-new-process" "restart-evidence.json" \
|
|
'.post.process_locator_revision = $replacement |
|
|
.post.process_identity_sha256 = $replacement |
|
|
.same_process_locator = false |
|
|
.same_process_identity = false |
|
|
.duplicate_invocation = false'
|
|
reject_evidence_mutation "contradictory-duplicate-flag" "restart-evidence.json" \
|
|
'.duplicate_invocation = true'
|
|
|
|
local fixture_project="${temporary}/fixture-project"
|
|
mkdir -p "${fixture_project}"
|
|
seed_field_task "${fixture_project}" a
|
|
grep -F 'IOP_AGENT_LOGGED_SMOKE_A_OK' \
|
|
"${fixture_project}/agent-task/m-${FIELD_MILESTONE}/01_field_runtime/PLAN-cloud-G10.md" \
|
|
>/dev/null ||
|
|
fail "self-test field task is missing its exact marker contract"
|
|
grep -F '`field-smoke-a.txt`' \
|
|
"${fixture_project}/agent-task/m-${FIELD_MILESTONE}/01_field_runtime/PLAN-cloud-G10.md" \
|
|
>/dev/null ||
|
|
fail "self-test field task is missing its bounded write set"
|
|
|
|
sleep 30 &
|
|
daemon_pid=$!
|
|
daemon_start_token="$(process_start_token "${daemon_pid}")"
|
|
sleep 30 &
|
|
local unrelated_pid=$!
|
|
cleanup_owned_daemon
|
|
if kill -0 "${unrelated_pid}" 2>/dev/null; then
|
|
kill -TERM "${unrelated_pid}" 2>/dev/null || true
|
|
wait "${unrelated_pid}" 2>/dev/null || true
|
|
else
|
|
fail "exact-PID cleanup terminated an unrelated fixture process"
|
|
fi
|
|
rm -rf "${temporary}"
|
|
printf '%s\n' 'logged-smoke: self-test passed (derived-evidence rejection matrix and exact-PID cleanup)'
|
|
}
|
|
|
|
seed_field_task() {
|
|
local project="$1"
|
|
local slot="$2"
|
|
local task_directory="${project}/agent-task/m-${FIELD_MILESTONE}/01_field_runtime"
|
|
local marker_file="field-smoke-${slot}.txt"
|
|
local marker="IOP_AGENT_LOGGED_SMOKE_${slot^^}_OK"
|
|
|
|
[[ ! -e "${task_directory}" ]] ||
|
|
fail "field project ${slot} already contains the smoke fixture task"
|
|
[[ ! -e "${project}/${marker_file}" ]] ||
|
|
fail "field project ${slot} already contains the smoke marker"
|
|
mkdir -p "${task_directory}"
|
|
|
|
cat >"${task_directory}/PLAN-cloud-G10.md" <<EOF
|
|
# Logged Smoke Provider Task ${slot^^}
|
|
|
|
## For the Implementing Agent
|
|
|
|
Create \`${marker_file}\` with exactly one line containing \`${marker}\`.
|
|
Then fill every implementation-owned field in
|
|
\`agent-task/m-${FIELD_MILESTONE}/01_field_runtime/CODE_REVIEW-cloud-G10.md\`
|
|
with concise English evidence and leave the pair ready for official review.
|
|
Do not modify any other file.
|
|
|
|
## Implementation Checklist
|
|
|
|
- [ ] Create the exact marker file and content.
|
|
- [ ] Record the exact marker validation in the review artifact.
|
|
|
|
## Verification
|
|
|
|
\`\`\`bash
|
|
test "\$(cat ${marker_file})" = "${marker}"
|
|
\`\`\`
|
|
|
|
After verification succeeds, run \`sleep 20\` before the final response. Do not
|
|
modify files during that bounded delay; it is the live restart checkpoint.
|
|
|
|
## Modified Files Summary
|
|
|
|
| File | Purpose |
|
|
|------|---------|
|
|
| \`${marker_file}\` | Record the provider-owned field mutation. |
|
|
| \`agent-task/m-${FIELD_MILESTONE}/01_field_runtime/CODE_REVIEW-cloud-G10.md\` | Record bounded implementation evidence. |
|
|
EOF
|
|
|
|
cat >"${task_directory}/CODE_REVIEW-cloud-G10.md" <<EOF
|
|
# Code Review Reference - Logged Smoke Provider Task ${slot^^}
|
|
|
|
## Implementation Checklist
|
|
|
|
- [ ] Create \`${marker_file}\` with the exact required content.
|
|
- [ ] Validate the marker without changing any other file.
|
|
|
|
## Implementation Notes
|
|
|
|
_Record actual implementation notes._
|
|
|
|
## Verification Results
|
|
|
|
_Paste actual stdout/stderr._
|
|
EOF
|
|
}
|
|
|
|
run_agent() {
|
|
local output_file="$1"
|
|
shift
|
|
"${binary}" "$@" \
|
|
--repo-config "${repo_config}" \
|
|
--local-config "${local_config}" \
|
|
--provider-catalog "${provider_catalog}" \
|
|
--output json >"${output_file}"
|
|
}
|
|
|
|
status_value() {
|
|
jq -er '.status' "$1"
|
|
}
|
|
|
|
record_status() {
|
|
local project_id="$1"
|
|
local slot="$2"
|
|
local sequence="$3"
|
|
run_agent "${output}/status-${slot}-${sequence}.json" status "${project_id}"
|
|
}
|
|
|
|
start_daemon() {
|
|
daemon_log="${output}/daemon.log"
|
|
"${binary}" serve \
|
|
--repo-config "${repo_config}" \
|
|
--local-config "${local_config}" \
|
|
--provider-catalog "${provider_catalog}" >>"${daemon_log}" 2>&1 &
|
|
daemon_pid=$!
|
|
daemon_start_token="$(process_start_token "${daemon_pid}")"
|
|
[[ -n "${daemon_start_token}" ]] || fail "could not capture daemon process identity"
|
|
local waited=0
|
|
while [[ ! -S "${daemon_socket}" ]]; do
|
|
kill -0 "${daemon_pid}" 2>/dev/null || fail "daemon exited before socket readiness"
|
|
(( waited < 200 )) || fail "daemon socket readiness timed out"
|
|
sleep 0.1
|
|
waited=$((waited + 1))
|
|
done
|
|
}
|
|
|
|
wait_for_state() {
|
|
local project_id="$1"
|
|
local slot="$2"
|
|
local desired="$3"
|
|
local sequence_base="$4"
|
|
local max_wait="${5:-600}"
|
|
local waited=0
|
|
while (( waited < max_wait )); do
|
|
record_status "${project_id}" "${slot}" "$((sequence_base + waited))"
|
|
local state
|
|
state="$(status_value "${output}/status-${slot}-$((sequence_base + waited)).json")"
|
|
if [[ "${state}" == "${desired}" ]]; then
|
|
return 0
|
|
fi
|
|
if [[ "${state}" == "blocked" ]]; then
|
|
fail "project ${slot} blocked while waiting for ${desired}"
|
|
fi
|
|
sleep 1
|
|
waited=$((waited + 1))
|
|
done
|
|
fail "project ${slot} timed out waiting for ${desired}"
|
|
}
|
|
|
|
wait_for_live_invocation() {
|
|
local project_id="$1"
|
|
local slot="$2"
|
|
local sequence_base="$3"
|
|
local waited=0
|
|
while (( waited < 600 )); do
|
|
record_status "${project_id}" "${slot}" "$((sequence_base + waited))"
|
|
local state
|
|
state="$(status_value "${output}/status-${slot}-$((sequence_base + waited)).json")"
|
|
if [[ "${state}" == "blocked" || "${state}" == "completed" ]]; then
|
|
fail "project ${slot} reached ${state} before a live invocation was observed"
|
|
fi
|
|
if [[ -f "${state_file}" ]]; then
|
|
local process_identity
|
|
if process_identity="$(state_process_identity "${state_file}" "${project_id}" 2>/dev/null)"; then
|
|
local process_pid
|
|
local process_token
|
|
IFS=$'\t' read -r process_pid process_token <<<"${process_identity}"
|
|
if [[ -n "${process_pid}" && -n "${process_token}" ]] &&
|
|
[[ "$(process_start_token "${process_pid}" || true)" == "${process_token}" ]]; then
|
|
state_invocation_identity "${state_file}" "${project_id}" >/dev/null
|
|
return 0
|
|
fi
|
|
fi
|
|
fi
|
|
sleep 1
|
|
waited=$((waited + 1))
|
|
done
|
|
fail "project ${slot} timed out waiting for a live invocation"
|
|
}
|
|
|
|
wait_for_durable_review_while_live() {
|
|
local project_id="$1"
|
|
local slot="$2"
|
|
local process_pid="$3"
|
|
local process_token="$4"
|
|
local overlay_reference
|
|
local review_path
|
|
IFS=$'\t' read -r overlay_reference review_path < <(
|
|
jq -er --arg project "${project_id}" '
|
|
[
|
|
.state.Projects[$project].Works[]?
|
|
| select(.Locators.process? != null and .Locators.overlay? != null)
|
|
| {
|
|
ordinal:.DispatchOrdinal,
|
|
overlay:.Locators.overlay.Opaque,
|
|
review:.Unit.Metadata.review_path
|
|
}
|
|
]
|
|
| sort_by(.ordinal)
|
|
| if length > 0 then
|
|
.[-1] |
|
|
select(
|
|
(.overlay | type) == "string" and
|
|
(.review | type) == "string"
|
|
) |
|
|
[.overlay,.review] | @tsv
|
|
else error("a retained review locator is required")
|
|
end
|
|
' "${state_file}"
|
|
)
|
|
[[ "${overlay_reference}" =~ ^overlay:[0-9a-f]{64}$ ]] ||
|
|
fail "project ${slot} overlay locator is not bounded"
|
|
[[ "${review_path}" == agent-task/* && "${review_path}" != *".."* ]] ||
|
|
fail "project ${slot} review path is not contained"
|
|
local overlay_id="${overlay_reference#overlay:}"
|
|
local task_view="${overlay_root}/tasks/${overlay_id}/view"
|
|
local review_file="${task_view}/${review_path}"
|
|
local marker_file="${task_view}/field-smoke-${slot}.txt"
|
|
local marker="IOP_AGENT_LOGGED_SMOKE_${slot^^}_OK"
|
|
local waited=0
|
|
while (( waited < 12000 )); do
|
|
[[ "$(process_start_token "${process_pid}" || true)" == "${process_token}" ]] ||
|
|
fail "project ${slot} exited before durable restart evidence was ready"
|
|
if [[ -f "${review_file}" && -f "${marker_file}" ]] &&
|
|
[[ "$(tr -d '\r\n' <"${marker_file}")" == "${marker}" ]] &&
|
|
! grep -F '_Paste actual ' "${review_file}" >/dev/null; then
|
|
return 0
|
|
fi
|
|
sleep 0.05
|
|
waited=$((waited + 1))
|
|
done
|
|
fail "project ${slot} timed out waiting for durable review evidence"
|
|
}
|
|
|
|
status_trace_json() {
|
|
local slot="$1"
|
|
local ordered=()
|
|
while IFS=$'\t' read -r _ file; do
|
|
ordered+=("${file}")
|
|
done < <(
|
|
for file in "${output}"/status-"${slot}"-*.json; do
|
|
local sequence="${file##*-}"
|
|
sequence="${sequence%.json}"
|
|
printf '%010d\t%s\n' "${sequence}" "${file}"
|
|
done | sort -n
|
|
)
|
|
jq -s '[.[].status] | reduce .[] as $state ([]; if length == 0 or .[-1] != $state then . + [$state] else . end) | .[:16]' \
|
|
"${ordered[@]}"
|
|
}
|
|
|
|
state_process_identity() {
|
|
local state_file="$1"
|
|
local project_id="$2"
|
|
jq -er --arg project "${project_id}" '
|
|
[
|
|
.state.Projects[$project].Works[]?
|
|
| select(.Locators.process? != null)
|
|
| {
|
|
ordinal:.DispatchOrdinal,
|
|
process:(.Locators.process.Opaque | fromjson)
|
|
}
|
|
]
|
|
| sort_by(.ordinal)
|
|
| if length > 0 then
|
|
.[-1].process
|
|
| select((.pid | type) == "number" and (.start_token | type) == "string")
|
|
| [.pid, .start_token]
|
|
| @tsv
|
|
else error("a process locator is required")
|
|
end
|
|
' "${state_file}"
|
|
}
|
|
|
|
state_invocation_identity() {
|
|
local state_file="$1"
|
|
local project_id="$2"
|
|
local state_revision attempt_id locator_revision process_pid process_token
|
|
IFS=$'\t' read -r state_revision attempt_id locator_revision process_pid process_token < <(
|
|
state_live_invocation_fields "${state_file}" "${project_id}"
|
|
)
|
|
sha256_text "${attempt_id}"$'\t'"${locator_revision}"$'\t'"${process_pid}"$'\t'"${process_token}"
|
|
}
|
|
|
|
state_live_invocation_fields() {
|
|
local state_file="$1"
|
|
local project_id="$2"
|
|
jq -er --arg project "${project_id}" '
|
|
.revision as $revision |
|
|
[
|
|
.state.Projects[$project].Works[]?
|
|
| select(.Locators.process? != null)
|
|
| {
|
|
ordinal:.DispatchOrdinal,
|
|
attempt_id:.AttemptID,
|
|
locator_revision:.Locators.process.Revision,
|
|
process:(.Locators.process.Opaque | fromjson)
|
|
}
|
|
]
|
|
| sort_by(.ordinal)
|
|
| if length > 0 then
|
|
.[-1] as $invocation |
|
|
$invocation.process |
|
|
select(
|
|
($invocation.attempt_id | type) == "string" and
|
|
($invocation.locator_revision | type) == "string" and
|
|
(.pid | type) == "number" and
|
|
(.start_token | type) == "string"
|
|
) |
|
|
[
|
|
($revision | tostring),
|
|
$invocation.attempt_id,
|
|
$invocation.locator_revision,
|
|
(.pid | tostring),
|
|
.start_token
|
|
] |
|
|
@tsv
|
|
else error("a live invocation identity is required")
|
|
end
|
|
' "${state_file}"
|
|
}
|
|
|
|
capture_locator_summary() {
|
|
local state_file="$1"
|
|
local project_id="$2"
|
|
local project_hash="$3"
|
|
local destination="$4"
|
|
jq -e --arg project "${project_id}" --arg project_hash "${project_hash}" '
|
|
.revision as $revision |
|
|
.state.Projects[$project] as $project_state |
|
|
{
|
|
project_id_hash:$project_hash,
|
|
state_revision:($revision | tostring),
|
|
project_status:$project_state.Status,
|
|
invocations:[
|
|
$project_state.Works[]?
|
|
| select(.Locators.process? != null)
|
|
| {
|
|
attempt:(.Attempt | tostring),
|
|
dispatch_ordinal:(.DispatchOrdinal | tostring),
|
|
state:.State,
|
|
locators:[
|
|
.Locators
|
|
| to_entries[]
|
|
| select(.key == "process" or .key == "session")
|
|
| {kind:.key,revision:.value.Revision}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
| select((.invocations | length) >= 1)
|
|
| select(all(.invocations[]; (.locators | length) >= 1))
|
|
' "${state_file}" >"${destination}" ||
|
|
fail "state has no bounded invocation locator summary"
|
|
}
|
|
|
|
capture_archive_summary() {
|
|
local project_id="$1"
|
|
local project_hash="$2"
|
|
local slot="$3"
|
|
local destination="$4"
|
|
local list_file="${output}/archive-list-${slot}.txt"
|
|
: >"${list_file}"
|
|
while IFS= read -r candidate; do
|
|
if jq -e --arg project "${project_id}" '
|
|
.project_id == $project and .terminal == true and
|
|
(.record_count | type == "number" and . > 0) and
|
|
(.checksum | test("^sha256:[0-9a-f]{64}$"))
|
|
' "${candidate}" >/dev/null 2>&1; then
|
|
printf '%s\n' "${candidate}" >>"${list_file}"
|
|
fi
|
|
done < <(find "${log_root}" -type f -name '*.manifest.json' -print | LC_ALL=C sort)
|
|
[[ -s "${list_file}" ]] || fail "project ${slot} has no terminal project-log archive"
|
|
|
|
local records="${output}/archive-records-${slot}.jsonl"
|
|
: >"${records}"
|
|
while IFS= read -r manifest_path; do
|
|
local stem="${manifest_path%.manifest.json}"
|
|
local event_log="${stem}.jsonl"
|
|
local timeline="${stem}.timeline.jsonl"
|
|
[[ -f "${event_log}" && -f "${timeline}" ]] ||
|
|
fail "project ${slot} archive is missing event or timeline evidence"
|
|
jq -s -e '
|
|
length > 0 and
|
|
(. as $records |
|
|
all(range(1; length);
|
|
$records[.].sequence == $records[. - 1].sequence + 1)) and
|
|
.[-1].terminal == true
|
|
' "${event_log}" >/dev/null ||
|
|
fail "project ${slot} project-log sequence is not ordered and terminal"
|
|
jq -s -e 'length > 0 and .[-1].terminal == true' "${timeline}" >/dev/null ||
|
|
fail "project ${slot} timeline is not terminal"
|
|
jq -n \
|
|
--arg manifest_sha256 "$(sha256_file "${manifest_path}")" \
|
|
--arg event_log_sha256 "$(sha256_file "${event_log}")" \
|
|
--arg timeline_sha256 "$(sha256_file "${timeline}")" \
|
|
--argjson manifest "$(jq -c . "${manifest_path}")" \
|
|
'{
|
|
archive_ordinal:($manifest.archive_ordinal | tostring),
|
|
record_count:$manifest.record_count,
|
|
checksum:$manifest.checksum,
|
|
terminal:$manifest.terminal,
|
|
manifest_sha256:$manifest_sha256,
|
|
event_log_sha256:$event_log_sha256,
|
|
timeline_sha256:$timeline_sha256
|
|
}' >>"${records}"
|
|
done <"${list_file}"
|
|
jq -s --arg project_hash "${project_hash}" '
|
|
{project_id_hash:$project_hash,archives:sort_by(.archive_ordinal)}
|
|
' "${records}" >"${destination}"
|
|
}
|
|
|
|
preflight_only=false
|
|
self_test_only=false
|
|
validate_only=""
|
|
binary=""
|
|
repo_config=""
|
|
local_config=""
|
|
provider_catalog=""
|
|
project_a=""
|
|
project_b=""
|
|
expected_head=""
|
|
output=""
|
|
|
|
while (( $# > 0 )); do
|
|
case "$1" in
|
|
--help|-h)
|
|
usage
|
|
exit 0
|
|
;;
|
|
--preflight-only)
|
|
preflight_only=true
|
|
shift
|
|
;;
|
|
--self-test)
|
|
self_test_only=true
|
|
shift
|
|
;;
|
|
--validate-manifest)
|
|
(( $# >= 2 )) || { usage >&2; exit "${EX_USAGE}"; }
|
|
validate_only="$2"
|
|
shift 2
|
|
;;
|
|
--binary|--repo-config|--local-config|--provider-catalog|--project-a|--project-b|--expected-head|--output)
|
|
(( $# >= 2 )) || { usage >&2; exit "${EX_USAGE}"; }
|
|
case "$1" in
|
|
--binary) binary="$2" ;;
|
|
--repo-config) repo_config="$2" ;;
|
|
--local-config) local_config="$2" ;;
|
|
--provider-catalog) provider_catalog="$2" ;;
|
|
--project-a) project_a="$2" ;;
|
|
--project-b) project_b="$2" ;;
|
|
--expected-head) expected_head="$2" ;;
|
|
--output) output="$2" ;;
|
|
esac
|
|
shift 2
|
|
;;
|
|
*)
|
|
printf 'logged-smoke: unknown argument: %s\n' "$1" >&2
|
|
usage >&2
|
|
exit "${EX_USAGE}"
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [[ -n "${validate_only}" ]]; then
|
|
validate_manifest "${validate_only}"
|
|
trap - EXIT
|
|
exit 0
|
|
fi
|
|
if [[ "${self_test_only}" == true ]]; then
|
|
self_test
|
|
trap - EXIT
|
|
exit 0
|
|
fi
|
|
if [[ "${preflight_only}" == true ]]; then
|
|
host_gate
|
|
printf '%s\n' 'logged-smoke: Darwin host gate passed before provider login or process launch'
|
|
trap - EXIT
|
|
exit 0
|
|
fi
|
|
|
|
for value in binary repo_config local_config provider_catalog project_a project_b expected_head output; do
|
|
[[ -n "${!value}" ]] || { usage >&2; exit "${EX_USAGE}"; }
|
|
done
|
|
|
|
host_gate
|
|
for tool in git go jq shasum ps pgrep stat awk sed tail find; do
|
|
command -v "${tool}" >/dev/null 2>&1 || fail "required tool is unavailable: ${tool}"
|
|
done
|
|
|
|
binary="$(canonical_file "${binary}")" || fail "binary must be one existing regular file"
|
|
repo_config="$(canonical_file "${repo_config}")" || fail "repo config must be one existing regular file"
|
|
local_config="$(canonical_file "${local_config}")" || fail "local config must be one existing regular file"
|
|
provider_catalog="$(canonical_file "${provider_catalog}")" || fail "provider catalog must be one existing regular file"
|
|
project_a="$(canonical_directory "${project_a}")" || fail "project A must be one existing directory"
|
|
project_b="$(canonical_directory "${project_b}")" || fail "project B must be one existing directory"
|
|
[[ -x "${binary}" ]] || fail "binary is not executable"
|
|
[[ "${project_a}" != "${project_b}" ]] || fail "project clones must be distinct"
|
|
[[ "${project_a}" != "${REPO_ROOT}" && "${project_b}" != "${REPO_ROOT}" ]] ||
|
|
fail "field projects must not be the harness source checkout"
|
|
[[ "${output}" == /* && -d "$(dirname "${output}")" ]] ||
|
|
fail "output must be an absolute path under an existing parent directory"
|
|
|
|
source_head="$(git -C "${REPO_ROOT}" rev-parse HEAD)"
|
|
source_tree="$(git -C "${REPO_ROOT}" rev-parse 'HEAD^{tree}')"
|
|
schema_digest="$(sha256_file "${MANIFEST_SCHEMA}")"
|
|
[[ "${source_head}" == "${expected_head}" ]] || fail "source HEAD differs from --expected-head"
|
|
[[ -z "$(git -C "${REPO_ROOT}" status --porcelain)" ]] || fail "source checkout is not clean"
|
|
for project in "${project_a}" "${project_b}"; do
|
|
[[ "$(git -C "${project}" rev-parse --show-toplevel)" == "${project}" ]] ||
|
|
fail "field project is not a repository root"
|
|
[[ "$(git -C "${project}" rev-parse HEAD)" == "${expected_head}" ]] ||
|
|
fail "field project revision differs from --expected-head"
|
|
[[ -z "$(git -C "${project}" status --porcelain)" ]] ||
|
|
fail "field project is not clean"
|
|
done
|
|
seed_field_task "${project_a}" a
|
|
seed_field_task "${project_b}" b
|
|
|
|
state_root="$(awk '$1 == "state_root:" {sub(/^[^:]+:[[:space:]]*/, ""); gsub(/^["'\'']|["'\'']$/, ""); print; exit}' "${local_config}")"
|
|
[[ "${state_root}" == /* && -d "${state_root}" ]] ||
|
|
fail "local config state_root must be a simple existing absolute scalar"
|
|
[[ "$(stat -f '%Lp' "${state_root}")" == "700" ]] ||
|
|
fail "state_root mode must be 0700"
|
|
[[ "$(stat -f '%u' "${state_root}")" == "$(id -u)" ]] ||
|
|
fail "state_root must be owned by the current user"
|
|
overlay_root="$(awk '$1 == "overlay_root:" {sub(/^[^:]+:[[:space:]]*/, ""); gsub(/^["'\'']|["'\'']$/, ""); print; exit}' "${local_config}")"
|
|
[[ "${overlay_root}" == /* && -d "${overlay_root}" ]] ||
|
|
fail "local config overlay_root must be a simple existing absolute scalar"
|
|
log_root="$(awk '$1 == "log_root:" {sub(/^[^:]+:[[:space:]]*/, ""); gsub(/^["'\'']|["'\'']$/, ""); print; exit}' "${local_config}")"
|
|
[[ "${log_root}" == /* ]] ||
|
|
fail "local config log_root must be a simple absolute scalar"
|
|
state_file="${state_root}/state.json"
|
|
daemon_socket="${state_root}/iop-agent.sock"
|
|
[[ ! -e "${daemon_socket}" ]] || fail "daemon socket already exists"
|
|
if pgrep -f "${binary} serve" >/dev/null 2>&1; then
|
|
fail "a matching daemon process already exists"
|
|
fi
|
|
[[ ! -e "${output}" ]] || fail "output directory already exists"
|
|
mkdir -m 0700 "${output}"
|
|
|
|
incoming_binary_digest="$(sha256_file "${binary}")"
|
|
rebuilt_binary="${output}/iop-agent-exact"
|
|
provider_smoke_binary="${output}/iop-provider-smoke-exact"
|
|
go build -trimpath -o "${rebuilt_binary}" ./apps/agent/cmd/agent
|
|
go build -trimpath -o "${provider_smoke_binary}" ./cmd/iop-provider-smoke
|
|
rebuilt_binary_digest="$(sha256_file "${rebuilt_binary}")"
|
|
[[ "${incoming_binary_digest}" == "${rebuilt_binary_digest}" ]] ||
|
|
fail "declared binary digest differs from a fresh exact-HEAD rebuild"
|
|
binary="${rebuilt_binary}"
|
|
jq -n \
|
|
--arg head "${source_head}" \
|
|
--arg source_tree "${source_tree}" \
|
|
--arg declared_binary_sha256 "${incoming_binary_digest}" \
|
|
--arg rebuilt_binary_sha256 "${rebuilt_binary_digest}" \
|
|
--arg provider_smoke_sha256 "$(sha256_file "${provider_smoke_binary}")" \
|
|
--arg schema_sha256 "${schema_digest}" \
|
|
'{
|
|
head:$head,
|
|
source_tree:$source_tree,
|
|
declared_binary_sha256:$declared_binary_sha256,
|
|
rebuilt_binary_sha256:$rebuilt_binary_sha256,
|
|
provider_smoke_sha256:$provider_smoke_sha256,
|
|
schema_sha256:$schema_sha256
|
|
}' >"${output}/build-evidence.json"
|
|
|
|
"${binary}" validate \
|
|
--repo-config "${repo_config}" \
|
|
--local-config "${local_config}" \
|
|
--provider-catalog "${provider_catalog}" \
|
|
--output json >"${output}/validate.json"
|
|
"${binary}" provider list \
|
|
--provider-catalog "${provider_catalog}" \
|
|
--output json >"${output}/providers.json"
|
|
|
|
profile_list="${output}/profile-list.txt"
|
|
awk '
|
|
$1 == "profiles:" { in_profiles = 1; next }
|
|
in_profiles && $1 == "-" && $2 == "id:" { print $3 }
|
|
' "${provider_catalog}" >"${profile_list}"
|
|
[[ "$(wc -l <"${profile_list}" | tr -d ' ')" == "$(jq -er '.profiles' "${output}/validate.json")" ]] ||
|
|
fail "could not resolve every catalog profile for runtime discovery"
|
|
discovery_records="${output}/discovery-records.jsonl"
|
|
: >"${discovery_records}"
|
|
profile_sequence=0
|
|
while IFS= read -r profile_id; do
|
|
[[ "${profile_id}" =~ ^[A-Za-z0-9._-]+$ ]] ||
|
|
fail "catalog profile identity is not shell-safe"
|
|
profile_sequence=$((profile_sequence + 1))
|
|
profile_log="${output}/provider-profile-${profile_sequence}.log"
|
|
"${provider_smoke_binary}" \
|
|
-config "${provider_catalog}" \
|
|
-profile "${profile_id}" \
|
|
-operations status \
|
|
-redact >"${profile_log}" 2>&1
|
|
grep -E '^preflight .* state=ready .* redacted=true$' "${profile_log}" >/dev/null ||
|
|
fail "profile ${profile_sequence} did not produce ready discovery evidence"
|
|
grep -E '^operation=status .* readiness=ready terminal=complete$' "${profile_log}" >/dev/null ||
|
|
fail "profile ${profile_sequence} did not produce ready quota/status evidence"
|
|
jq -n \
|
|
--arg profile_id_hash "$(sha256_text "${profile_id}")" \
|
|
--arg output_sha256 "$(sha256_file "${profile_log}")" \
|
|
'{profile_id_hash:$profile_id_hash,discovery:"ready",quota_status:"observed",output_sha256:$output_sha256}' \
|
|
>>"${discovery_records}"
|
|
done <"${profile_list}"
|
|
jq -s 'select(length > 0) | {profiles:.}' "${discovery_records}" \
|
|
>"${output}/discovery-quota.json" ||
|
|
fail "runtime discovery produced no profile evidence"
|
|
|
|
run_agent "${output}/projects.json" project list
|
|
project_a_id="$(jq -er --arg workspace "${project_a}" '[.projects[] | select(.workspace == $workspace) | .id] | if length == 1 then .[0] else error("project A registration mismatch") end' "${output}/projects.json")"
|
|
project_b_id="$(jq -er --arg workspace "${project_b}" '[.projects[] | select(.workspace == $workspace) | .id] | if length == 1 then .[0] else error("project B registration mismatch") end' "${output}/projects.json")"
|
|
[[ "${project_a_id}" != "${project_b_id}" ]] || fail "project registrations resolve to one identity"
|
|
project_a_hash="$(sha256_text "${project_a_id}")"
|
|
project_b_hash="$(sha256_text "${project_b_id}")"
|
|
|
|
run_agent "${output}/milestones-a.json" milestone list "${project_a_id}"
|
|
run_agent "${output}/milestones-b.json" milestone list "${project_b_id}"
|
|
run_agent "${output}/preview-a.json" preview "${project_a_id}"
|
|
run_agent "${output}/preview-b.json" preview "${project_b_id}"
|
|
jq -e '.selected == true and (.next_work | length > 0) and (.blockers | length == 0)' "${output}/preview-a.json" >/dev/null
|
|
jq -e '.selected == true and (.next_work | length > 0) and (.blockers | length == 0)' "${output}/preview-b.json" >/dev/null
|
|
jq -n \
|
|
--arg preview_a_sha256 "$(sha256_file "${output}/preview-a.json")" \
|
|
--arg preview_b_sha256 "$(sha256_file "${output}/preview-b.json")" \
|
|
'{project_a:$preview_a_sha256,project_b:$preview_b_sha256,selected:true,unblocked:true}' \
|
|
>"${output}/preview-evidence.json"
|
|
|
|
run_agent "${output}/start-a.json" start "${project_a_id}"
|
|
run_agent "${output}/start-b.json" start "${project_b_id}"
|
|
record_status "${project_a_id}" a 0
|
|
record_status "${project_b_id}" b 0
|
|
start_daemon
|
|
wait_for_state "${project_a_id}" a running 100
|
|
wait_for_state "${project_b_id}" b running 100
|
|
wait_for_live_invocation "${project_a_id}" a 200
|
|
wait_for_live_invocation "${project_b_id}" b 200
|
|
[[ -f "${state_file}" ]] || fail "manager state was not persisted"
|
|
cp "${state_file}" "${output}/state-running.json"
|
|
initial_invocation_a="$(state_invocation_identity "${output}/state-running.json" "${project_a_id}")"
|
|
initial_invocation_b="$(state_invocation_identity "${output}/state-running.json" "${project_b_id}")"
|
|
IFS=$'\t' read -r process_a_pid process_a_token < <(
|
|
state_process_identity "${output}/state-running.json" "${project_a_id}"
|
|
)
|
|
IFS=$'\t' read -r process_b_pid process_b_token < <(
|
|
state_process_identity "${output}/state-running.json" "${project_b_id}"
|
|
)
|
|
[[ "$(process_start_token "${process_a_pid}")" == "${process_a_token}" ]] ||
|
|
fail "project A did not have one live exact invocation while running"
|
|
[[ "$(process_start_token "${process_b_pid}")" == "${process_b_token}" ]] ||
|
|
fail "project B did not have one live exact invocation while running"
|
|
jq -n \
|
|
--arg start_a_sha256 "$(sha256_file "${output}/start-a.json")" \
|
|
--arg start_b_sha256 "$(sha256_file "${output}/start-b.json")" \
|
|
--arg running_state_sha256 "$(sha256_file "${output}/state-running.json")" \
|
|
'{start_a:$start_a_sha256,start_b:$start_b_sha256,running_state:$running_state_sha256}' \
|
|
>"${output}/start-evidence.json"
|
|
|
|
run_agent "${output}/stop-a.json" stop "${project_a_id}"
|
|
record_status "${project_a_id}" a 300
|
|
record_status "${project_b_id}" b 300
|
|
[[ "$(status_value "${output}/status-a-300.json")" == "stopped" ]] ||
|
|
fail "project A did not stop"
|
|
[[ "$(status_value "${output}/status-b-300.json")" == "running" ]] ||
|
|
fail "project B did not remain running during project A cancellation"
|
|
cp "${state_file}" "${output}/state-stopped.json"
|
|
waited=0
|
|
while [[ "$(process_start_token "${process_a_pid}" || true)" == "${process_a_token}" ]]; do
|
|
(( waited < 200 )) ||
|
|
fail "project A exact invocation remained live after cancellation"
|
|
sleep 0.1
|
|
waited=$((waited + 1))
|
|
done
|
|
[[ "$(process_start_token "${process_b_pid}")" == "${process_b_token}" ]] ||
|
|
fail "project A cancellation changed the sibling invocation identity"
|
|
jq -n \
|
|
--arg stopped_state_sha256 "$(sha256_file "${output}/state-stopped.json")" \
|
|
--arg cancelled_process_sha256 "$(sha256_text "${process_a_pid}:${process_a_token}")" \
|
|
--arg sibling_process_sha256 "$(sha256_text "${process_b_pid}:${process_b_token}")" \
|
|
'{
|
|
stopped_state_sha256:$stopped_state_sha256,
|
|
cancelled_process_sha256:$cancelled_process_sha256,
|
|
sibling_process_sha256:$sibling_process_sha256,
|
|
cancelled_process_live:false,
|
|
sibling_process_live:true
|
|
}' >"${output}/cancellation-evidence.json"
|
|
|
|
wait_for_durable_review_while_live \
|
|
"${project_b_id}" b "${process_b_pid}" "${process_b_token}"
|
|
cp "${state_file}" "${output}/state-before-restart-live.json"
|
|
IFS=$'\t' read -r \
|
|
pre_restart_revision pre_restart_attempt pre_restart_locator \
|
|
pre_restart_pid pre_restart_start_identity < <(
|
|
state_live_invocation_fields \
|
|
"${output}/state-before-restart-live.json" "${project_b_id}"
|
|
)
|
|
[[ "$(process_start_token "${pre_restart_pid}")" == "${pre_restart_start_identity}" ]] ||
|
|
fail "project B restart checkpoint was not live before daemon shutdown"
|
|
crash_owned_daemon_preserving_invocations
|
|
[[ "$(process_start_token "${pre_restart_pid}" || true)" == "${pre_restart_start_identity}" ]] ||
|
|
fail "daemon crash changed the retained live invocation"
|
|
record_status "${project_b_id}" b 350
|
|
[[ "$(status_value "${output}/status-b-350.json")" == "running" ]] ||
|
|
fail "project B did not remain running between daemon instances"
|
|
start_daemon
|
|
wait_for_live_invocation "${project_b_id}" b 400
|
|
cp "${state_file}" "${output}/state-after-restart-live.json"
|
|
IFS=$'\t' read -r \
|
|
post_restart_revision post_restart_attempt post_restart_locator \
|
|
post_restart_pid post_restart_start_identity < <(
|
|
state_live_invocation_fields \
|
|
"${output}/state-after-restart-live.json" "${project_b_id}"
|
|
)
|
|
[[ "$(process_start_token "${post_restart_pid}")" == "${post_restart_start_identity}" ]] ||
|
|
fail "project B restart checkpoint was not live after daemon recovery"
|
|
|
|
same_attempt=false
|
|
same_process_locator=false
|
|
same_process_identity=false
|
|
duplicate_invocation=true
|
|
[[ "${pre_restart_attempt}" == "${post_restart_attempt}" ]] && same_attempt=true
|
|
[[ "${pre_restart_locator}" == "${post_restart_locator}" ]] && same_process_locator=true
|
|
if [[ "${pre_restart_pid}" == "${post_restart_pid}" &&
|
|
"${pre_restart_start_identity}" == "${post_restart_start_identity}" ]]; then
|
|
same_process_identity=true
|
|
fi
|
|
if [[ "${same_attempt}" == true &&
|
|
"${same_process_locator}" == true &&
|
|
"${same_process_identity}" == true ]]; then
|
|
duplicate_invocation=false
|
|
fi
|
|
(( post_restart_revision > pre_restart_revision )) ||
|
|
fail "daemon recovery did not advance the persisted state revision"
|
|
[[ "${duplicate_invocation}" == false ]] ||
|
|
fail "daemon recovery changed the exact live invocation identity"
|
|
post_restart_invocation_b="$(state_invocation_identity \
|
|
"${output}/state-after-restart-live.json" "${project_b_id}")"
|
|
[[ "${initial_invocation_b}" == "${post_restart_invocation_b}" ]] ||
|
|
fail "project B restart recovery changed its retained invocation identity"
|
|
jq -n \
|
|
--arg project_id_hash "${project_b_hash}" \
|
|
--arg pre_revision "${pre_restart_revision}" \
|
|
--arg post_revision "${post_restart_revision}" \
|
|
--arg pre_attempt "$(sha256_text "${pre_restart_attempt}")" \
|
|
--arg post_attempt "$(sha256_text "${post_restart_attempt}")" \
|
|
--arg pre_locator "${pre_restart_locator}" \
|
|
--arg post_locator "${post_restart_locator}" \
|
|
--arg pre_process "$(sha256_text "${pre_restart_pid}:${pre_restart_start_identity}")" \
|
|
--arg post_process "$(sha256_text "${post_restart_pid}:${post_restart_start_identity}")" \
|
|
--argjson same_attempt "${same_attempt}" \
|
|
--argjson same_process_locator "${same_process_locator}" \
|
|
--argjson same_process_identity "${same_process_identity}" \
|
|
--argjson duplicate_invocation "${duplicate_invocation}" \
|
|
'{
|
|
project_id_hash:$project_id_hash,
|
|
pre:{
|
|
state_revision:$pre_revision,
|
|
attempt_id_sha256:$pre_attempt,
|
|
process_locator_revision:$pre_locator,
|
|
process_identity_sha256:$pre_process
|
|
},
|
|
post:{
|
|
state_revision:$post_revision,
|
|
attempt_id_sha256:$post_attempt,
|
|
process_locator_revision:$post_locator,
|
|
process_identity_sha256:$post_process
|
|
},
|
|
pre_process_live:true,
|
|
post_process_live:true,
|
|
same_attempt:$same_attempt,
|
|
same_process_locator:$same_process_locator,
|
|
same_process_identity:$same_process_identity,
|
|
duplicate_invocation:$duplicate_invocation
|
|
}' >"${output}/restart-evidence.json"
|
|
|
|
run_agent "${output}/resume-a.json" resume "${project_a_id}"
|
|
record_status "${project_a_id}" a 500
|
|
wait_for_state "${project_b_id}" b completed 600 1800
|
|
wait_for_state "${project_a_id}" a completed 800 1800
|
|
cleanup_owned_daemon
|
|
record_status "${project_a_id}" a 1100
|
|
record_status "${project_b_id}" b 1100
|
|
cp "${state_file}" "${output}/state-terminal.json"
|
|
post_restart_invocation_a="$(state_invocation_identity "${output}/state-terminal.json" "${project_a_id}")"
|
|
[[ "${initial_invocation_a}" != "${post_restart_invocation_a}" ]] ||
|
|
fail "project A resume reused the cancelled invocation identity"
|
|
capture_locator_summary \
|
|
"${output}/state-running.json" "${project_a_id}" "${project_a_hash}" \
|
|
"${output}/locator-a-initial.json"
|
|
capture_locator_summary \
|
|
"${output}/state-terminal.json" "${project_a_id}" "${project_a_hash}" \
|
|
"${output}/locator-a-terminal.json"
|
|
jq -s '{snapshots:.}' \
|
|
"${output}/locator-a-initial.json" "${output}/locator-a-terminal.json" \
|
|
>"${output}/locator-a.json"
|
|
capture_locator_summary \
|
|
"${output}/state-running.json" "${project_b_id}" "${project_b_hash}" \
|
|
"${output}/locator-b-initial.json"
|
|
capture_locator_summary \
|
|
"${output}/state-after-restart-live.json" "${project_b_id}" "${project_b_hash}" \
|
|
"${output}/locator-b-restarted.json"
|
|
capture_locator_summary \
|
|
"${output}/state-terminal.json" "${project_b_id}" "${project_b_hash}" \
|
|
"${output}/locator-b-terminal.json"
|
|
jq -s '{snapshots:.}' \
|
|
"${output}/locator-b-initial.json" \
|
|
"${output}/locator-b-restarted.json" \
|
|
"${output}/locator-b-terminal.json" \
|
|
>"${output}/locator-b.json"
|
|
|
|
trace_a="$(status_trace_json a)"
|
|
trace_b="$(status_trace_json b)"
|
|
binary_digest="$(sha256_file "${binary}")"
|
|
repo_digest="$(sha256_file "${repo_config}")"
|
|
local_digest="$(sha256_file "${local_config}")"
|
|
catalog_digest="$(sha256_file "${provider_catalog}")"
|
|
project_a_revision="$(git -C "${project_a}" rev-parse HEAD)"
|
|
project_b_revision="$(git -C "${project_b}" rev-parse HEAD)"
|
|
capture_archive_summary \
|
|
"${project_a_id}" "${project_a_hash}" a "${output}/archive-a.json"
|
|
capture_archive_summary \
|
|
"${project_b_id}" "${project_b_hash}" b "${output}/archive-b.json"
|
|
jq '{project_id_hash,logs:[.archives[] | {
|
|
archive_ordinal,event_log_sha256,timeline_sha256
|
|
}]}' "${output}/archive-a.json" >"${output}/project-log-a.json"
|
|
jq '{project_id_hash,logs:[.archives[] | {
|
|
archive_ordinal,event_log_sha256,timeline_sha256
|
|
}]}' "${output}/archive-b.json" >"${output}/project-log-b.json"
|
|
jq -n \
|
|
--arg status_a_sha256 "$(sha256_file "${output}/status-a-1100.json")" \
|
|
--arg status_b_sha256 "$(sha256_file "${output}/status-b-1100.json")" \
|
|
--arg archive_a_sha256 "$(sha256_file "${output}/archive-a.json")" \
|
|
--arg archive_b_sha256 "$(sha256_file "${output}/archive-b.json")" \
|
|
'{
|
|
status_a_sha256:$status_a_sha256,
|
|
status_b_sha256:$status_b_sha256,
|
|
archive_a_sha256:$archive_a_sha256,
|
|
archive_b_sha256:$archive_b_sha256,
|
|
terminal_status:"completed"
|
|
}' >"${output}/terminal-evidence.json"
|
|
run_id="$(sha256_text "${source_head}:${binary_digest}:${project_a_hash}:${project_b_hash}")"
|
|
arch="$(uname -m)"
|
|
case "${arch}" in
|
|
arm64|x86_64) ;;
|
|
*) fail "unsupported Darwin architecture ${arch}" ;;
|
|
esac
|
|
[[ "${arch}" == "x86_64" ]] && arch="amd64"
|
|
|
|
jq -n \
|
|
--arg run_id "${run_id}" \
|
|
--arg head "${source_head}" \
|
|
--arg source_tree "${source_tree}" \
|
|
--arg binary_sha256 "${binary_digest}" \
|
|
--arg rebuilt_binary_sha256 "${rebuilt_binary_digest}" \
|
|
--arg binary_build_head "${source_head}" \
|
|
--arg repo_config_sha256 "${repo_digest}" \
|
|
--arg local_config_sha256 "${local_digest}" \
|
|
--arg provider_catalog_sha256 "${catalog_digest}" \
|
|
--arg build_evidence_sha256 "$(sha256_file "${output}/build-evidence.json")" \
|
|
--arg arch "${arch}" \
|
|
--arg project_a_hash "${project_a_hash}" \
|
|
--arg project_b_hash "${project_b_hash}" \
|
|
--arg project_a_revision "${project_a_revision}" \
|
|
--arg project_b_revision "${project_b_revision}" \
|
|
--argjson trace_a "${trace_a}" \
|
|
--argjson trace_b "${trace_b}" \
|
|
--arg initial_invocation_a "${initial_invocation_a}" \
|
|
--arg initial_invocation_b "${initial_invocation_b}" \
|
|
--arg post_restart_invocation_a "${post_restart_invocation_a}" \
|
|
--arg post_restart_invocation_b "${post_restart_invocation_b}" \
|
|
--arg locator_a_sha256 "$(sha256_file "${output}/locator-a.json")" \
|
|
--arg locator_b_sha256 "$(sha256_file "${output}/locator-b.json")" \
|
|
--arg project_log_a_sha256 "$(sha256_file "${output}/project-log-a.json")" \
|
|
--arg project_log_b_sha256 "$(sha256_file "${output}/project-log-b.json")" \
|
|
--arg archive_a_sha256 "$(sha256_file "${output}/archive-a.json")" \
|
|
--arg archive_b_sha256 "$(sha256_file "${output}/archive-b.json")" \
|
|
--arg discovery_sha256 "$(sha256_file "${output}/discovery-quota.json")" \
|
|
--arg preview_sha256 "$(sha256_file "${output}/preview-evidence.json")" \
|
|
--arg start_sha256 "$(sha256_file "${output}/start-evidence.json")" \
|
|
--arg cancellation_sha256 "$(sha256_file "${output}/cancellation-evidence.json")" \
|
|
--arg restart_sha256 "$(sha256_file "${output}/restart-evidence.json")" \
|
|
--arg terminal_sha256 "$(sha256_file "${output}/terminal-evidence.json")" \
|
|
--arg schema_digest "${schema_digest}" \
|
|
'{
|
|
schema_version:"2",
|
|
run_id:$run_id,
|
|
source:{
|
|
head:$head,
|
|
source_tree:$source_tree,
|
|
binary_sha256:$binary_sha256,
|
|
rebuilt_binary_sha256:$rebuilt_binary_sha256,
|
|
binary_build_head:$binary_build_head,
|
|
repo_config_sha256:$repo_config_sha256,
|
|
local_config_sha256:$local_config_sha256,
|
|
provider_catalog_sha256:$provider_catalog_sha256,
|
|
build_evidence_sha256:$build_evidence_sha256,
|
|
schema_sha256:$schema_digest
|
|
},
|
|
environment:{os:"darwin",arch:$arch},
|
|
lifecycle:{
|
|
discovery:{status:"verified",evidence_sha256:$discovery_sha256},
|
|
preview:{status:"verified",evidence_sha256:$preview_sha256},
|
|
manual_start:{status:"verified",evidence_sha256:$start_sha256},
|
|
cancellation:{status:"verified",evidence_sha256:$cancellation_sha256},
|
|
sibling_continuation:{status:"verified",evidence_sha256:$cancellation_sha256},
|
|
new_invocation:{status:"verified",evidence_sha256:$locator_a_sha256},
|
|
restart_recovery:{status:"verified",evidence_sha256:$restart_sha256},
|
|
terminal_completion:{status:"verified",evidence_sha256:$terminal_sha256}
|
|
},
|
|
projects:[
|
|
{
|
|
slot:"a",
|
|
project_id_hash:$project_a_hash,
|
|
workspace_revision:$project_a_revision,
|
|
status_trace:$trace_a,
|
|
terminal_status:"completed",
|
|
initial_invocation_sha256:$initial_invocation_a,
|
|
post_restart_invocation_sha256:$post_restart_invocation_a,
|
|
locator_evidence_sha256:$locator_a_sha256,
|
|
project_log_sha256:$project_log_a_sha256,
|
|
terminal_archive_sha256:$archive_a_sha256
|
|
},
|
|
{
|
|
slot:"b",
|
|
project_id_hash:$project_b_hash,
|
|
workspace_revision:$project_b_revision,
|
|
status_trace:$trace_b,
|
|
terminal_status:"completed",
|
|
initial_invocation_sha256:$initial_invocation_b,
|
|
post_restart_invocation_sha256:$post_restart_invocation_b,
|
|
locator_evidence_sha256:$locator_b_sha256,
|
|
project_log_sha256:$project_log_b_sha256,
|
|
terminal_archive_sha256:$archive_b_sha256
|
|
}
|
|
],
|
|
evidence:{
|
|
manifest_locator:"manifest.json",
|
|
schema_sha256:$schema_digest,
|
|
records:[
|
|
{kind:"source_build",locator:"build-evidence.json",sha256:$build_evidence_sha256},
|
|
{kind:"discovery_quota",locator:"discovery-quota.json",sha256:$discovery_sha256},
|
|
{kind:"preview",locator:"preview-evidence.json",sha256:$preview_sha256},
|
|
{kind:"state_transition",locator:"start-evidence.json",sha256:$start_sha256},
|
|
{kind:"cancellation",locator:"cancellation-evidence.json",sha256:$cancellation_sha256},
|
|
{kind:"invocation_locator",locator:"locator-a.json",sha256:$locator_a_sha256},
|
|
{kind:"invocation_locator",locator:"locator-b.json",sha256:$locator_b_sha256},
|
|
{kind:"restart",locator:"restart-evidence.json",sha256:$restart_sha256},
|
|
{kind:"project_log",locator:"project-log-a.json",sha256:$project_log_a_sha256},
|
|
{kind:"project_log",locator:"project-log-b.json",sha256:$project_log_b_sha256},
|
|
{kind:"terminal_archive",locator:"archive-a.json",sha256:$archive_a_sha256},
|
|
{kind:"terminal_archive",locator:"archive-b.json",sha256:$archive_b_sha256},
|
|
{kind:"terminal_archive",locator:"terminal-evidence.json",sha256:$terminal_sha256}
|
|
]
|
|
}
|
|
}' >"${output}/manifest.json"
|
|
|
|
validate_manifest "${output}/manifest.json"
|
|
trap - EXIT
|
|
printf 'logged-smoke: PASS manifest=%s\n' "${output}/manifest.json"
|