123set -euo pipefail45GST_INSPECT_MAX_BYTES=10485766ENCODER_PREFLIGHT_MAX_BYTES=104857678run_bounded_gst_inspect() {9 local timeout_path="$1"10 local gst_inspect_path="$2"11 local output_path="$3"12 local output_size13 shift 31415 if ! "$timeout_path" --signal=TERM --kill-after=2s 5s \16 "$gst_inspect_path" "$@" 2>/dev/null \17 | head -c "$((GST_INSPECT_MAX_BYTES + 1))" >"$output_path"18 then19 return 120 fi21 output_size="$(wc -c <"$output_path" | tr -d '[:space:]')"22 [[ "$output_size" =~ ^[0-9]+$ && "$output_size" -le "$GST_INSPECT_MAX_BYTES" ]]23}2425run_bounded_encoder_preflight() {26 local timeout_path="$1"27 local sigil_path="$2"28 local output_path="$3"29 local factory="$4"30 local render_node="$5"31 local output_size3233 if ! "$timeout_path" --signal=TERM --kill-after=2s 5s \34 "$sigil_path" capture encoder-preflight \35 --vaapi-encoder "$factory" \36 --vaapi-render-node "$render_node" \37 --rate-control cbr --rate-control cqp 2>&1 \38 | head -c "$((ENCODER_PREFLIGHT_MAX_BYTES + 1))" >"$output_path"39 then40 return 141 fi42 output_size="$(wc -c <"$output_path" | tr -d '[:space:]')"43 [[ "$output_size" =~ ^[0-9]+$ \44 && "$output_size" -le "$ENCODER_PREFLIGHT_MAX_BYTES" ]]45}4647classify_native_uat_mode() {48 local native_size="$1"49 local fixed_size="$2"5051 native_mode_relation=''52 [[ "$native_size" =~ ^[1-9][0-9]*x[1-9][0-9]*$ ]] || return 153 [[ "$fixed_size" =~ ^[1-9][0-9]*x[1-9][0-9]*$ ]] || return 154 if [[ "$native_size" == "$fixed_size" ]]; then55 native_mode_relation=identical-to-fixed56 else57 native_mode_relation=distinct-from-fixed58 fi59}6061select_gstva_h264_encoder() {62 local sigil_path="$1"63 local gst_inspect_path="$2"64 local device_root="${3:-/dev/dri}"65 local require_character="${4:-true}"66 local override_node="${5:-}"67 local override_factory="${6:-}"68 local timeout_path="${7:?timeout command is required}"69 local output_root="${8:?bounded inspection output directory is required}"70 local factory71 local factories72 local probe_output73 local candidate74 local pair75 local candidate_count=07677 render_node=''78 va_encoder=''79 va_candidates=''80 [[ -d "$output_root" && ! -L "$output_root" ]] || return 181 if ! LC_ALL=C run_bounded_gst_inspect \82 "$timeout_path" "$gst_inspect_path" "$output_root/gst-inspect.registry";83 then84 return 185 fi86 factories="$(awk '87 $2 ~ /^va(renderD[0-9]+)?h264(lp)?enc:$/ {88 sub(/:$/, "", $2)89 print $290 }91 ' "$output_root/gst-inspect.registry" | sort -u)" || return 19293 for candidate in "$device_root"/renderD*; do94 [[ "$(basename "$candidate")" =~ ^renderD[0-9]+$ ]] || continue95 [[ -e "$candidate" && ! -L "$candidate" && -r "$candidate" && -w "$candidate" ]] \96 || continue97 if [[ "$require_character" == true && ! -c "$candidate" ]]; then98 continue99 fi100101 while IFS= read -r factory; do102 [[ -n "$factory" ]] || continue103 probe_output="$output_root/encoder-preflight.$(basename "$candidate").$factory"104 if ! LC_ALL=C run_bounded_encoder_preflight \105 "$timeout_path" "$sigil_path" "$probe_output" "$factory" "$candidate";106 then107 continue108 fi109 pair="$candidate $factory"110 if [[ -n "$va_candidates" ]]; then111 va_candidates="$va_candidates112$pair"113 else114 va_candidates="$pair"115 fi116 candidate_count=$((candidate_count + 1))117 if [[ -n "$override_node" && "$candidate" == "$override_node" \118 && "$factory" == "$override_factory" ]]; then119 render_node="$candidate"120 va_encoder="$factory"121 elif [[ -z "$override_node" && "$candidate_count" -eq 1 ]]; then122 render_node="$candidate"123 va_encoder="$factory"124 fi125 done <<<"$factories"126 done127128 if [[ -n "$override_node" ]]; then129 [[ -n "$render_node" && -n "$va_encoder" ]]130 return131 fi132 if [[ "$candidate_count" -eq 1 ]]; then133 return 0134 fi135 render_node=''136 va_encoder=''137 [[ "$candidate_count" -eq 0 ]] && return 1138 return 2139}140141if [[ "${SIGIL_HARDWARE_UAT_SOURCE_ONLY:-}" == 1 ]]; then142 if [[ "${BASH_SOURCE[0]}" != "$0" ]]; then143 return 0144 fi145 exit 0146fi147148usage() {149 echo "usage: $0 UAT_ROOT UAT_COMMIT WORKFLOW_RUN_ID [--render-node PATH --va-encoder FACTORY]" >&2150}151152if [[ $# -lt 3 ]]; then153 usage154 exit 64155fi156157uat_root="$1"158uat_commit="$2"159workflow_run_id="$3"160shift 3161render_node_override=''162va_encoder_override=''163render_node_override_set=false164va_encoder_override_set=false165while [[ $# -gt 0 ]]; do166 case "$1" in167 --render-node)168 [[ $# -ge 2 && "$render_node_override_set" == false && -n "$2" ]] \169 || { usage; exit 64; }170 render_node_override="$2"171 render_node_override_set=true172 shift 2173 ;;174 --va-encoder)175 [[ $# -ge 2 && "$va_encoder_override_set" == false && -n "$2" ]] \176 || { usage; exit 64; }177 va_encoder_override="$2"178 va_encoder_override_set=true179 shift 2180 ;;181 *)182 usage183 exit 64184 ;;185 esac186done187if [[ "$render_node_override_set" == true || "$va_encoder_override_set" == true ]]; then188 [[ "$render_node_override_set" == true && "$va_encoder_override_set" == true ]] || {189 echo "--render-node and --va-encoder must be provided together" >&2190 exit 64191 }192fi193short_commit="${uat_commit:0:12}"194195[[ "$uat_commit" =~ ^[0-9a-f]{40}$ ]] || {196 echo "invalid UAT commit" >&2197 exit 64198}199[[ "$workflow_run_id" =~ ^[0-9]+$ ]] || {200 echo "invalid workflow run ID" >&2201 exit 64202}203case "$uat_root" in204 "$HOME"/.local/state/goq-hardware-uat."$short_commit".??????) ;;205 *) echo "unsafe UAT root: $uat_root" >&2; exit 64 ;;206esac207for trusted_path in "$HOME/.local" "$HOME/.local/state" "$uat_root" "$uat_root/incoming"; do208 [[ -d "$trusted_path" && ! -L "$trusted_path" ]] || {209 echo "UAT path is missing or symlinked: $trusted_path" >&2210 exit 64211 }212done213[[ "$(stat -c '%a:%U' "$uat_root")" == "700:$(id -un)" ]] || {214 echo "UAT root is not owner-only" >&2215 exit 64216}217218umask 077219XDG_RUNTIME_DIR="/run/user/$(id -u)"220export XDG_RUNTIME_DIR221222lock_file="$HOME/.local/state/goq-hardware-uat.lock"223[[ ! -L "$lock_file" ]] || {224 echo "unsafe UAT lock path: $lock_file" >&2225 exit 64226}227exec {uat_lock_fd}>"$lock_file"228if ! flock -n "$uat_lock_fd"; then229 echo "another hardware UAT owns the host-wide lock" >&2230 exit 75231fi232233root_suffix="${uat_root##*.}"234[[ "$root_suffix" =~ ^[[:alnum:]]{6}$ ]] || {235 echo "invalid UAT root suffix" >&2236 exit 64237}238invocation_id="$short_commit-$workflow_run_id-$root_suffix"239240incoming="$uat_root/incoming"241archive="$incoming/sigil-hardware-uat-$short_commit-bazzite-x86_64.tar.gz"242payload_root="$uat_root/candidate"243private="$uat_root/private"244evidence="$uat_root/evidence"245raw="$private/raw"246for new_path in "$payload_root" "$private" "$evidence"; do247 [[ ! -e "$new_path" && ! -L "$new_path" ]] || {248 echo "UAT output path already exists: $new_path" >&2249 exit 64250 }251done252install -d -m 0700 "$payload_root" "$private" "$evidence" "$raw"253for new_path in "$payload_root" "$private" "$evidence" "$raw"; do254 [[ -d "$new_path" && ! -L "$new_path" ]]255 [[ "$(stat -c '%a:%U' "$new_path")" == "700:$(id -un)" ]]256done257258real_config="$HOME/.config/sigil-spark/host.toml"259real_sigil="$HOME/.local/libexec/sigil-spark/current/sigil"260real_identity="$(sed -n 's/^identity_path = "\(.*\)"$/\1/p' "$real_config")"261uinput_block="$(awk '262 /^\[uinput\]$/ { emit = 1 }263 emit && /^\[/ && $0 != "[uinput]" { exit }264 emit { print }265' "$real_config")"266[[ -n "$real_identity" && -f "$real_identity" ]]267[[ "$uinput_block" == '[uinput]'$'\n'* ]]268[[ -x "$real_sigil" ]]269original_ready() {270 local invocation pid271 systemctl --user is-active --quiet sigil-host.service || return 1272 pid="$(systemctl --user show -p MainPID --value sigil-host.service)"273 invocation="$(systemctl --user show -p InvocationID --value sigil-host.service)"274 [[ "$pid" =~ ^[0-9]+$ && "$pid" -gt 1 ]] || return 1275 [[ "$invocation" =~ ^[0-9a-f]{32}$ ]] || return 1276 cmp -s "$real_sigil" "/proc/$pid/exe" || return 1277 journalctl --user _SYSTEMD_INVOCATION_ID="$invocation" \278 _SYSTEMD_USER_UNIT=sigil-host.service _PID="$pid" --no-pager -o cat \279 | grep -Fx status=ready >/dev/null280}281original_ready282original_enabled="$(systemctl --user is-enabled sigil-host.service)"283original_main_pid="$(systemctl --user show -p MainPID --value sigil-host.service)"284[[ "$original_main_pid" =~ ^[0-9]+$ && "$original_main_pid" -gt 1 ]]285sha256sum "$real_config" >"$private/original-config.sha256"286readlink "$HOME/.local/libexec/sigil-spark/current" >"$private/original-current-link"287stat -c '%i:%s:%Y:%a:%U:%G' "$real_identity" >"$private/original-identity.stat"288sha256sum "$real_identity" >"$private/original-identity.sha256"289290[[ -f "$archive" && -f "$archive.sha256" ]]291[[ -f "$incoming/release-manifest.json" && -f "$incoming/uat-provenance.txt" ]]292(293 cd "$incoming"294 sha256sum -c "$(basename -- "$archive.sha256")"295)296grep -Fxq "source_commit=$uat_commit" "$incoming/uat-provenance.txt"297grep -Fxq "workflow_run_id=$workflow_run_id" "$incoming/uat-provenance.txt"298299tar -tzf "$archive" >"$private/archive-members.txt"300if grep -Eq '(^/|(^|/)\.\.(/|$))' "$private/archive-members.txt"; then301 echo "archive contains an unsafe member" >&2302 exit 1303fi304tar --no-same-owner -xzf "$archive" -C "$payload_root"305(306 cd "$payload_root/payload"307 sha256sum -c PACKAGE-SHA256SUMS308)309(310 cd "$payload_root/payload/release"311 sha256sum -c SHA256SUMS312)313314sigil="$payload_root/payload/release/sigil"315probe="$payload_root/payload/release/sigil-probe"316manifest="$payload_root/payload/release/release-manifest.json"317cmp "$manifest" "$incoming/release-manifest.json"318jq -e --arg commit "$uat_commit" '319 .git_commit == $commit320 and .git_dirty == false321 and .binary_provenance == "self-built-clean-head"322 and .binary_provenance_verified == true323 and .release_kind == "development"324 and .release_tag == "development"325 and .target == "x86_64-unknown-linux-gnu.2.17"326 and .features == ["in-process-gstreamer"]327 and .demo_auth_bypass == false328 and .demo_direct_node == false329' "$manifest" >/dev/null330[[ -x "$sigil" && -x "$probe" ]]331332readelf -d "$sigil" >"$private/sigil.dynamic.txt"333if grep -Eq '\((RPATH|RUNPATH)\)' "$private/sigil.dynamic.txt"; then334 echo "candidate contains an unexpected runtime library path" >&2335 exit 1336fi337for soname in \338 libgstreamer-1.0.so \339 libgstapp-1.0.so \340 libgstbase-1.0.so \341 libgstvideo-1.0.so \342 libgobject-2.0.so \343 libglib-2.0.so344do345 grep -Fq "Shared library: [$soname" "$private/sigil.dynamic.txt"346done347readelf --version-info "$sigil" >"$private/sigil.versions.txt"348max_glibc="$({349 sed -n 's/.*Name: GLIBC_\([0-9][0-9.]*\).*/\1/p' "$private/sigil.versions.txt"350 echo 0351} | sort -V | tail -n 1)"352[[ "$(printf '%s\n' "$max_glibc" 2.17 | sort -V | tail -n 1)" == 2.17 ]] || {353 echo "candidate requires GLIBC_$max_glibc, above 2.17" >&2354 exit 1355}356ldd "$sigil" >"$private/sigil.ldd"357ldd "$probe" >"$private/sigil-probe.ldd"358if grep -q 'not found' "$private/sigil.ldd"; then359 echo "candidate host has an unresolved dynamic library" >&2360 exit 1361fi362if grep -q 'not found' "$private/sigil-probe.ldd"; then363 echo "candidate probe has an unresolved dynamic library" >&2364 exit 1365fi366timeout --signal=TERM --kill-after=2s 5s "$sigil" --version367timeout --signal=TERM --kill-after=2s 5s "$probe" --version368369host_identity="$private/host.key"370probe_identity="$private/probe.key"371state="$private/state"372external_cqp_state="$private/external-cqp-state"373fixed_config="$private/host-1280x800.toml"374external_cqp_config="$private/host-1280x800-external-cqp.toml"375native_config="$private/host-native.toml"376invitation="$private/probe.goq-invite"377378"$sigil" identity init --output "$host_identity" >"$private/host-identity.log"379"$sigil" identity init --output "$probe_identity" >"$private/probe-identity.log"380host_node_id="$(sed -n 's/^node_id=//p' "$private/host-identity.log")"381probe_node_id="$(sed -n 's/^node_id=//p' "$private/probe-identity.log")"382[[ -n "$host_node_id" && -n "$probe_node_id" ]]383384pw_dump="$(command -v pw-dump)"385pw_link="$(command -v pw-link)"386xprop="$(command -v xprop)"387gst_launch="$(command -v gst-launch-1.0)"388gst_inspect="$(command -v gst-inspect-1.0)"389timeout_command="$(command -v timeout)"390ffmpeg="$(command -v ffmpeg)"391[[ -S "$XDG_RUNTIME_DIR/pipewire-0" ]]392selection_status=0393select_gstva_h264_encoder \394 "$sigil" "$gst_inspect" /dev/dri true \395 "$render_node_override" "$va_encoder_override" \396 "$timeout_command" "$private" || selection_status=$?397if [[ "$selection_status" -ne 0 ]]; then398 if [[ -n "$render_node_override" ]]; then399 echo "the requested render-node and VA-encoder pair is not eligible for this UAT" >&2400 elif [[ "$selection_status" -eq 2 ]]; then401 echo "multiple GstVA H.264 encoder pairs are eligible; select one exact pair with --render-node and --va-encoder" >&2402 else403 echo "no accessible render node has a GstVA H.264 factory satisfying the CBR and CQP UAT contract" >&2404 fi405 if [[ -n "$va_candidates" ]]; then406 while IFS= read -r pair; do407 printf 'eligible_pair=%s\n' "$pair" >&2408 done <<<"$va_candidates"409 fi410 exit 1411fi412if [[ -n "$render_node_override" ]]; then413 vaapi_selection_mode=explicit414else415 vaapi_selection_mode=automatic-unique416fi417{418 echo "vaapi_render_node=$render_node"419 echo "vaapi_encoder=$va_encoder"420 echo "vaapi_selection_mode=$vaapi_selection_mode"421} >"$private/vaapi-selection.env"422423cat >"$fixed_config" <<EOF424identity_path = "$host_identity"425state_path = "$state"426source = "gamescope-pipewire"427width = 1280428height = 800429framerate = 60430codec = "h264"431input_mode = "uinput"432ffmpeg_path = "$ffmpeg"433434$uinput_block435436[gamescope_pipewire]437node_name = "gamescope"438media_class = "Video/Source"439xwayland_display = ":0"440pw_dump_path = "$pw_dump"441gst_launch_path = "$gst_launch"442gst_inspect_path = "$gst_inspect"443encoder_backend = "in-process-gstreamer"444vaapi_encoder = "$va_encoder"445vaapi_render_node = "$render_node"446rate_control = "cbr"447bitrate_kbps = 12000448EOF449450cat >"$native_config" <<EOF451identity_path = "$host_identity"452state_path = "$state"453source = "gamescope-pipewire"454framerate = 60455codec = "h264"456input_mode = "uinput"457ffmpeg_path = "$ffmpeg"458459$uinput_block460461[gamescope_pipewire]462node_name = "gamescope"463media_class = "Video/Source"464xwayland_display = ":0"465pw_dump_path = "$pw_dump"466gst_launch_path = "$gst_launch"467gst_inspect_path = "$gst_inspect"468encoder_backend = "in-process-gstreamer"469vaapi_encoder = "$va_encoder"470vaapi_render_node = "$render_node"471rate_control = "cbr"472bitrate_kbps = 12000473EOF474475cat >"$external_cqp_config" <<EOF476identity_path = "$host_identity"477state_path = "$external_cqp_state"478source = "gamescope-pipewire"479width = 1280480height = 800481framerate = 60482codec = "h264"483input_mode = "log"484ffmpeg_path = "$ffmpeg"485486[gamescope_pipewire]487node_name = "gamescope"488media_class = "Video/Source"489xwayland_display = ":0"490pw_dump_path = "$pw_dump"491gst_launch_path = "$gst_launch"492gst_inspect_path = "$gst_inspect"493encoder_backend = "external-gst-launch"494vaapi_encoder = "$va_encoder"495vaapi_render_node = "$render_node"496rate_control = "cqp"497quantizer = 24498EOF499chmod 0600 "$fixed_config" "$external_cqp_config" "$native_config"500501"$sigil" config check --config "$fixed_config" >"$private/fixed-config-check.log"502"$sigil" config check --config "$external_cqp_config" \503 >"$private/external-cqp-config-check.log"504"$sigil" config check --config "$native_config" >"$private/native-config-check.log"505grep -Fxq config=ok "$private/fixed-config-check.log"506grep -Fxq capture_preflight=ok "$private/fixed-config-check.log"507grep -Fxq encoded_mode=1280x800@60 "$private/fixed-config-check.log"508grep -Fxq config=ok "$private/external-cqp-config-check.log"509grep -Fxq capture_preflight=ok "$private/external-cqp-config-check.log"510grep -Fxq encoded_mode=1280x800@60 "$private/external-cqp-config-check.log"511grep -Fxq config=ok "$private/native-config-check.log"512grep -Fxq capture_preflight=ok "$private/native-config-check.log"513native_size="$(sed -n 's/^encoded_mode=\([0-9][0-9]*x[0-9][0-9]*\)@60$/\1/p' "$private/native-config-check.log")"514classify_native_uat_mode "$native_size" 1280x800515516"$sigil" invitation create \517 --config "$fixed_config" \518 --peer "$probe_node_id" \519 --expires-in-seconds 900 \520 --pointer-keyboard \521 --output "$invitation" \522 >"$private/invitation-create.log"523[[ "$(stat -c %a "$invitation")" == 600 ]]524525restore_required=false526capture_probe_pid=''527rollback_units=()528rollback_helper="$private/rollback-original.sh"529fixed_candidate_unit="goq-uat-fixed-candidate-$invocation_id"530native_candidate_unit="goq-uat-native-candidate-$invocation_id"531candidate_units=("$fixed_candidate_unit" "$native_candidate_unit")532active_candidate_unit=""533534cat >"$rollback_helper" <<EOF535#!/usr/bin/env bash536set -u537XDG_RUNTIME_DIR='$XDG_RUNTIME_DIR'538export XDG_RUNTIME_DIR539real_sigil='$real_sigil'540real_config='$real_config'541for candidate_unit in '$fixed_candidate_unit' '$native_candidate_unit'; do542 systemctl --user stop "\$candidate_unit.service" >/dev/null 2>&1 || true543 systemctl --user reset-failed "\$candidate_unit.service" >/dev/null 2>&1 || true544done545original_ready() {546 local invocation pid547 systemctl --user is-active --quiet sigil-host.service || return 1548 pid="\$(systemctl --user show -p MainPID --value sigil-host.service)"549 invocation="\$(systemctl --user show -p InvocationID --value sigil-host.service)"550 [[ "\$pid" =~ ^[0-9]+\$ && "\$pid" -gt 1 ]] || return 1551 [[ "\$invocation" =~ ^[0-9a-f]{32}\$ ]] || return 1552 cmp -s "\$real_sigil" "/proc/\$pid/exe" || return 1553 journalctl --user _SYSTEMD_INVOCATION_ID="\$invocation" \554 _SYSTEMD_USER_UNIT=sigil-host.service _PID="\$pid" --no-pager -o cat \555 | grep -Fx status=ready >/dev/null556}557for _ in \$(seq 1 60); do558 systemctl --user reset-failed sigil-host.service >/dev/null 2>&1 || true559 systemctl --user start sigil-host.service >/dev/null 2>&1 || true560 sleep 5561 if systemctl --user is-active --quiet sigil-host.service && original_ready; then562 exit 0563 fi564done565exit 1566EOF567chmod 0700 "$rollback_helper"568569stop_candidate() {570 local candidate_unit571 for candidate_unit in "${candidate_units[@]}"; do572 systemctl --user stop "$candidate_unit.service" >/dev/null 2>&1 || true573 systemctl --user reset-failed "$candidate_unit.service" >/dev/null 2>&1 || true574 done575 active_candidate_unit=""576}577578disarm_rollbacks() {579 local unit580 for unit in "${rollback_units[@]}"; do581 systemctl --user stop "$unit.timer" "$unit.service" 2>/dev/null || true582 done583}584585restore_original() {586 stop_candidate587 if $restore_required || ! systemctl --user is-active --quiet sigil-host.service; then588 systemctl --user reset-failed sigil-host.service || true589 systemctl --user start sigil-host.service || return 1590 for _ in $(seq 1 60); do591 if systemctl --user is-active --quiet sigil-host.service && original_ready; then592 restore_required=false593 disarm_rollbacks594 return 0595 fi596 sleep 0.5597 done598 return 1599 fi600 disarm_rollbacks601}602603cleanup() {604 local status=$?605 trap - EXIT606 trap '' INT TERM HUP607 if [[ -n "$capture_probe_pid" ]] && kill -0 "$capture_probe_pid" 2>/dev/null; then608 kill "$capture_probe_pid" 2>/dev/null || true609 wait "$capture_probe_pid" 2>/dev/null || true610 capture_probe_pid=''611 fi612 if ! restore_original; then613 echo "CRITICAL: original Sigil service did not return ready; rollback timer remains armed" >&2614 status=1615 fi616 exit "$status"617}618trap cleanup EXIT619trap 'exit 130' INT620trap 'exit 143' TERM621trap 'exit 129' HUP622623arm_rollback() {624 local unit="$1"625 systemd-run --user --unit="$unit" --on-active=20m "$rollback_helper" >/dev/null626 systemctl --user is-active --quiet "$unit.timer"627 rollback_units+=("$unit")628}629630start_candidate() {631 local mode="$1"632 local config="$2"633 host_log="$raw/$mode-host.log"634 case "$mode" in635 fixed) active_candidate_unit="$fixed_candidate_unit" ;;636 native) active_candidate_unit="$native_candidate_unit" ;;637 *) echo "unknown candidate mode: $mode" >&2; return 1 ;;638 esac639 : >"$host_log"640 systemd-run --user --quiet --collect \641 --unit="$active_candidate_unit" \642 --service-type=exec \643 --property="StandardOutput=append:$host_log" \644 --property="StandardError=append:$host_log" \645 --property=TimeoutStopSec=10s \646 --setenv=RUST_LOG=info,sigil::server=debug \647 "$sigil" serve --config "$config" --max-runtime-seconds 1080648 for _ in $(seq 1 300); do649 grep -Fxq status=ready "$host_log" 2>/dev/null && break650 systemctl --user is-active --quiet "$active_candidate_unit.service"651 sleep 0.1652 done653 grep -Fxq status=ready "$host_log"654 grep -Fxq "node_id=$host_node_id" "$host_log"655}656657validate_probe() {658 local log="$1"659 local transport="$2"660 local size="$3"661 local request_id="$4"662 local slow_consumer_ms="${5:-}"663 local expected_path_mode="${6:-direct}"664 grep -Fxq probe=ok "$log"665 grep -Fxq "dimensions=$size" "$log"666 grep -Fxq "transport=$transport" "$log"667 grep -Fxq sequence_gaps=0 "$log"668 grep -Fxq first_configured_idr=ok "$log"669 grep -Fxq frame_sequence=monotonic "$log"670 grep -Fxq recovery_barrier=configured-idr "$log"671 grep -Fxq keyframe_recovery=ok "$log"672 grep -Fxq "keyframe_request_id=$request_id" "$log"673 grep -Fxq "path_mode=$expected_path_mode" "$log"674 grep -Fxq "media_path_mode=$expected_path_mode" "$log"675 grep -Fxq "input_path_mode=$expected_path_mode" "$log"676 grep -Fxq "forced_relay=$([[ "$expected_path_mode" == relay ]] && printf ok || printf not-requested)" "$log"677 awk -F= '$1 == "keyframe_recovery_micros" && $2 ~ /^[0-9]+$/ && $2 <= 2000000 { ok=1 } END { exit !ok }' "$log"678 awk -F= '$1 == "input_ack_micros" && $2 ~ /^[0-9]+$/ { ok=1 } END { exit !ok }' "$log"679 if [[ "$transport" == iroh-moq ]]; then680 grep -Fxq "control_path_mode=$expected_path_mode" "$log"681 grep -Fxq control_alpn=sigil/control/1 "$log"682 grep -Fxq moq_group_capacity=1 "$log"683 grep -Fxq moq_unrecovered_group_gaps=0 "$log"684 grep -Fxq moq_historical_suffix_frames=0 "$log"685 fi686 if [[ -n "$slow_consumer_ms" ]]; then687 grep -Fxq slow_consumer=ok "$log"688 grep -Fxq "slow_consumer_stall_ms=$slow_consumer_ms" "$log"689 grep -Fxq slow_consumer_first_post_stall=configured-idr "$log"690 grep -Fxq slow_consumer_historical_suffix_frames=0 "$log"691 awk -F= '$1 == "slow_consumer_recovery_micros" && $2 ~ /^[0-9]+$/ && $2 <= 2000000 { ok=1 } END { exit !ok }' "$log"692 awk -F= '$1 == "slow_consumer_cancellation_delta" && $2 ~ /^[0-9]+$/ && $2 > 0 { ok=1 } END { exit !ok }' "$log"693 awk -F= '$1 == "slow_consumer_group_advance" && $2 ~ /^[0-9]+$/ && $2 > 0 { ok=1 } END { exit !ok }' "$log"694 awk -F= '$1 == "slow_consumer_sequence_advance" && $2 ~ /^[0-9]+$/ && $2 > 0 { ok=1 } END { exit !ok }' "$log"695 awk -F= -v minimum="$((slow_consumer_ms * 500))" '$1 == "slow_consumer_capture_advance_micros" && $2 ~ /^[0-9]+$/ && $2 >= minimum { ok=1 } END { exit !ok }' "$log"696 awk -F= '$1 == "slow_consumer_input_ack_micros" && $2 ~ /^[0-9]+$/ && $2 <= 2000000 { ok=1 } END { exit !ok }' "$log"697 else698 grep -Fxq slow_consumer=not-requested "$log"699 fi700}701702wait_for_count() {703 local log="$1"704 local text="$2"705 local expected="$3"706 local deadline=$((SECONDS + 20))707 local count708 while ((SECONDS < deadline)); do709 count="$(grep -Fc -- "$text" "$log" 2>/dev/null || true)"710 [[ "$count" -ge "$expected" ]] && return 0711 [[ -n "$active_candidate_unit" ]] \712 && systemctl --user is-active --quiet "$active_candidate_unit.service" \713 || return 1714 sleep 0.1715 done716 return 1717}718719run_probe_cycle() {720 local mode="$1"721 local transport="$2"722 local size="$3"723 local request_id="$4"724 local invitation_path="${5:-}"725 local slow_consumer_ms="${6:-}"726 local expected_path_mode="${7:-direct}"727 local pointer_feedback_smoke="${8:-false}"728 local log="$raw/$mode-$transport-$request_id.log"729 local command=(730 "$probe"731 --node-id "$host_node_id"732 --identity "$probe_identity"733 --frames 30734 --timeout-seconds 20735 --expect-size "$size"736 --keyframe-smoke737 --keyframe-request-id "$request_id"738 )739 [[ "$transport" == grouped-v3 ]] && command+=(--media-v3)740 [[ -n "$invitation_path" ]] && command+=(--invitation "$invitation_path")741 [[ -n "$slow_consumer_ms" ]] && command+=(--slow-consumer-ms "$slow_consumer_ms")742 [[ "$expected_path_mode" != relay ]] || command+=(--relay-only)743 if [[ "$pointer_feedback_smoke" == true ]]; then744 command+=(--pointer-smoke --pointer-feedback-smoke)745 fi746 timeout --signal=TERM --kill-after=5s 45s "${command[@]}" >"$log" 2>&1747 validate_probe "$log" "$transport" "$size" "$request_id" "$slow_consumer_ms" "$expected_path_mode"748 if [[ "$pointer_feedback_smoke" == true ]]; then749 grep -Fxq pointer_smoke=ok "$log"750 grep -Fxq pointer_feedback_smoke=ok "$log"751 grep -Eq '^pointer_sync_position=[0-9]+,[0-9]+$' "$log"752 grep -Eq '^pointer_motion_position=[0-9]+,[0-9]+$' "$log"753 fi754}755756validate_host_recovery() {757 local log="$1"758 local plain759 plain="$private/$(basename -- "$log").plain"760 wait_for_count "$log" 'MoQ control client released' 10761 wait_for_count "$log" 'media v3 client released' 10762 wait_for_count "$log" 'input client released' 20763 sed -E $'s/\x1B\\[[0-9;]*[[:alpha:]]//g' "$log" >"$plain"764 [[ "$(grep -Fc 'MoQ control client released' "$plain")" -eq 10 ]]765 [[ "$(grep -Fc 'media v3 client released' "$plain")" -eq 10 ]]766 [[ "$(grep -Fc 'input client released' "$plain")" -eq 20 ]]767 [[ "$(grep -Fc 'forced-IDR recovery acknowledged' "$plain")" -eq 20 ]]768 [[ "$(grep -Fc 'forced_idr_disposition=Requested' "$plain")" -eq 20 ]]769 [[ "$(grep -Fc 'coalesced=false' "$plain")" -ge 20 ]]770 ! grep -Eq 'forced-IDR request failed|forced-IDR recovery was not acknowledged|forced-IDR acknowledgement task failed|forced-IDR acknowledgement task ended without a result' "$plain"771}772773capture_inventory() {774 local phase="$1"775 ps -u "$(id -u)" -o pid=,comm=,args= >"$evidence/$phase-processes.txt"776 awk -v own_pid="$$" '777 $1 == own_pid { next }778 ($2 == "sigil" || $2 == "sigil-host") &&779 ($0 ~ / serve( |$)/ || $0 ~ / capture probe( |$)/) { print; next }780 $2 ~ /^gst-launch/ && $0 ~ /pipewiresrc/ { print }781 ' "$evidence/$phase-processes.txt" >"$evidence/$phase-conflicts.txt"782 if [[ -s "$evidence/$phase-conflicts.txt" ]]; then783 echo "a Sigil or GStreamer PipeWire capture process survived $phase" >&2784 cat "$evidence/$phase-conflicts.txt" >&2785 return 1786 fi787 "$pw_link" -l >"$evidence/$phase-pipewire-links.txt"788 if grep -Fiq gamescope "$evidence/$phase-pipewire-links.txt"; then789 echo "a Gamescope PipeWire consumer link survived $phase" >&2790 cat "$evidence/$phase-pipewire-links.txt" >&2791 return 1792 fi793 "$pw_dump" >"$evidence/$phase-pipewire.json"794}795796validate_capture_log() {797 local log="$1"798 grep -Fxq probe=ok "$log"799 grep -Fxq dropped_after_encode_before_probe_consumer=0 "$log"800}801802run_capture_probe() {803 local name="$1"804 local config="$2"805 local size="$3"806 local minimum_fps="${4:-}"807 local command=(808 "$sigil" capture probe --source gamescope-pipewire809 --config "$config" --frames 300 --expect-size "$size"810 )811 [[ -z "$minimum_fps" ]] || command+=(--minimum-fps "$minimum_fps")812 capture_inventory "pre-$name"813 timeout --signal=TERM --kill-after=5s 90s "${command[@]}" \814 >"$evidence/$name-capture.log"815 validate_capture_log "$evidence/$name-capture.log"816 capture_inventory "post-$name"817}818819fixed_timer="goq-uat-fixed-$invocation_id"820arm_rollback "$fixed_timer"821restore_required=true822systemctl --user stop sigil-host.service823if systemctl --user is-active --quiet sigil-host.service; then824 echo "original Sigil service did not stop" >&2825 exit 1826fi827[[ "$(systemctl --user show -p MainPID --value sigil-host.service)" -eq 0 ]]828829run_capture_probe fixed "$fixed_config" 1280x800 55830831capture_inventory pre-external-cqp832timeout --signal=TERM --kill-after=5s 90s \833 "$sigil" capture probe --source gamescope-pipewire \834 --config "$external_cqp_config" --frames 300 --expect-size 1280x800 \835 --minimum-fps 55 >"$evidence/external-cqp-capture.log" &836capture_probe_pid=$!837external_link_ready=false838for _ in $(seq 1 200); do839 if ! kill -0 "$capture_probe_pid" 2>/dev/null; then840 break841 fi842 if "$pw_link" -l | grep -Fiq gamescope; then843 external_link_ready=true844 break845 fi846 sleep 0.05847done848if [[ "$external_link_ready" != true ]]; then849 echo "external CQP probe never opened a Gamescope PipeWire consumer link" >&2850 exit 1851fi852if timeout --signal=TERM --kill-after=2s 10s \853 "$sigil" capture probe --source gamescope-pipewire \854 --config "$fixed_config" --frames 1 --expect-size 1280x800 \855 >"$evidence/contended-capture.out" 2>"$evidence/contended-capture.error"; then856 echo "a second Gamescope capture probe overlapped external CQP capture" >&2857 exit 1858fi859grep -Fq 'another Sigil daemon or capture probe already owns this lifecycle scope' \860 "$evidence/contended-capture.error"861wait "$capture_probe_pid"862capture_probe_pid=''863validate_capture_log "$evidence/external-cqp-capture.log"864capture_inventory post-external-cqp865866start_candidate fixed "$fixed_config"867run_probe_cycle fixed iroh-moq 1280x800 1001 "$invitation" 1500 relay868wait_for_count "$raw/fixed-host.log" 'MoQ control client released' 1869wait_for_count "$raw/fixed-host.log" 'input client released' 1870enrollment="$("$sigil" enrollment show --config "$fixed_config")"871grep -Fxq enrollment=active <<<"$enrollment"872grep -Fxq "peer_node_id=$probe_node_id" <<<"$enrollment"873grep -Fxq grants=view,pointer-keyboard <<<"$enrollment"874if timeout --signal=TERM --kill-after=5s 20s \875 "$probe" --node-id "$host_node_id" --identity "$probe_identity" \876 --invitation "$invitation" --frames 4 >"$raw/invitation-replay.log" 2>&1; then877 echo "redeemed invitation replay was accepted" >&2878 exit 1879fi880grep -Fq 'host rejected control stream: Portal peer is not authorized' \881 "$raw/invitation-replay.log"882mv "$invitation" "$private/redeemed-probe.goq-invite"883for cycle in $(seq 2 10); do884 run_probe_cycle fixed iroh-moq 1280x800 "$((1000 + cycle))"885done886for cycle in $(seq 1 10); do887 run_probe_cycle fixed grouped-v3 1280x800 "$((1100 + cycle))"888done889validate_host_recovery "$raw/fixed-host.log"890891fixed_candidate_pid="$(systemctl --user show -p MainPID --value \892 "$fixed_candidate_unit.service")"893[[ "$fixed_candidate_pid" =~ ^[0-9]+$ && "$fixed_candidate_pid" -gt 1 ]]894pointer_reconnects_before="$(grep -Fc \895 'Gamescope Xwayland pointer feedback reconnected' "$raw/fixed-host.log" || true)"896for cycle in 1 2; do897 systemctl --user restart gamescope-session-plus@steam.service898 gamescope_ready=false899 for _ in $(seq 1 600); do900 if systemctl --user is-active --quiet gamescope-session-plus@steam.service \901 && DISPLAY=:0 "$xprop" -root GAMESCOPE_MOUSE_FOCUS_DISPLAY >/dev/null 2>&1 \902 && "$pw_dump" | jq -e '903 .[]904 | select(.type == "PipeWire:Interface:Node")905 | select(.info.props["node.name"] == "gamescope")906 | select(.info.props["media.class"] == "Video/Source")907 ' >/dev/null908 then909 gamescope_ready=true910 break911 fi912 sleep 0.1913 done914 [[ "$gamescope_ready" == true ]]915 [[ "$(systemctl --user show -p MainPID --value \916 "$fixed_candidate_unit.service")" == "$fixed_candidate_pid" ]]917 expected_reconnects="$((pointer_reconnects_before + cycle))"918 for _ in $(seq 1 100); do919 reconnects="$(grep -Fc 'Gamescope Xwayland pointer feedback reconnected' \920 "$raw/fixed-host.log" || true)"921 [[ "$reconnects" -ge "$expected_reconnects" ]] && break922 sleep 0.1923 done924 [[ "$reconnects" -ge "$expected_reconnects" ]]925 run_probe_cycle "pointer-recovery-$cycle" iroh-moq 1280x800 \926 "$((1200 + cycle))" "" "" direct true927 sed -E $'s/\x1B\\[[0-9;]*[[:alpha:]]//g' \928 "$raw/pointer-recovery-$cycle-iroh-moq-$((1200 + cycle)).log" \929 >"$evidence/pointer-recovery-$cycle.log"930done931stop_candidate932933native_timer="goq-uat-native-$invocation_id"934arm_rollback "$native_timer"935systemctl --user stop "$fixed_timer.timer" "$fixed_timer.service" 2>/dev/null || true936run_capture_probe native "$native_config" "$native_size"937938start_candidate native "$native_config"939run_probe_cycle native iroh-moq "$native_size" 2001 "" 1500940for cycle in $(seq 2 10); do941 run_probe_cycle native iroh-moq "$native_size" "$((2000 + cycle))"942done943for cycle in $(seq 1 10); do944 run_probe_cycle native grouped-v3 "$native_size" "$((2100 + cycle))"945done946validate_host_recovery "$raw/native-host.log"947948install -d -m 0700 "$evidence/probes"949for log in \950 "$raw"/fixed-iroh-moq-*.log \951 "$raw"/fixed-grouped-v3-*.log \952 "$raw"/native-iroh-moq-*.log \953 "$raw"/native-grouped-v3-*.log954do955 grep -vE '^session_id=' "$log" >"$evidence/probes/$(basename -- "$log")"956done957958summarize_group() {959 local prefix="$1"960 local values="$private/$prefix.recovery-values"961 local files=("$evidence/probes/$prefix"-*.log)962 local raw_files=("$raw/$prefix"-*.log)963 local count unique_sessions p50_index p95_index p50 p95 maximum gaps964 [[ "${#files[@]}" -eq 10 ]]965 [[ "${#raw_files[@]}" -eq 10 ]]966 awk -F= '$1 == "keyframe_recovery_micros" { print $2 }' "${files[@]}" | sort -n >"$values"967 count="$(wc -l <"$values" | tr -d ' ')"968 [[ "$count" -eq 10 ]]969 unique_sessions="$(awk -F= '$1 == "session_id" { print $2 }' "${raw_files[@]}" | sort -u | wc -l | tr -d ' ')"970 [[ "$unique_sessions" -eq 10 ]]971 p50_index=$(((count * 50 + 99) / 100))972 p95_index=$(((count * 95 + 99) / 100))973 p50="$(sed -n "${p50_index}p" "$values")"974 p95="$(sed -n "${p95_index}p" "$values")"975 maximum="$(tail -n 1 "$values")"976 gaps="$(awk -F= '$1 == "sequence_gaps" { sum += $2 } END { print sum + 0 }' "${files[@]}")"977 [[ "$gaps" -eq 0 && "$p95" -le 2000000 ]]978 printf '%s_cycles=%s\n' "$prefix" "$count"979 printf '%s_unique_sessions=%s\n' "$prefix" "$unique_sessions"980 printf '%s_sequence_gaps=%s\n' "$prefix" "$gaps"981 printf '%s_recovery_p50_micros=%s\n' "$prefix" "$p50"982 printf '%s_recovery_p95_micros=%s\n' "$prefix" "$p95"983 printf '%s_recovery_max_micros=%s\n' "$prefix" "$maximum"984}985986all_raw_probes=(987 "$raw"/fixed-iroh-moq-*.log988 "$raw"/fixed-grouped-v3-*.log989 "$raw"/native-iroh-moq-*.log990 "$raw"/native-grouped-v3-*.log991)992[[ "${#all_raw_probes[@]}" -eq 40 ]]993fixed_unique_sessions="$(awk -F= '$1 == "session_id" { print $2 }' \994 "$raw"/fixed-iroh-moq-*.log "$raw"/fixed-grouped-v3-*.log \995 | sort -u | wc -l | tr -d ' ')"996native_unique_sessions="$(awk -F= '$1 == "session_id" { print $2 }' \997 "$raw"/native-iroh-moq-*.log "$raw"/native-grouped-v3-*.log \998 | sort -u | wc -l | tr -d ' ')"999[[ "$fixed_unique_sessions" -eq 20 && "$native_unique_sessions" -eq 20 ]]1000total_sessions="${#all_raw_probes[@]}"1001fixed_observed_fps="$(sed -n 's/^observed_fps=//p' "$evidence/fixed-capture.log")"1002external_cqp_observed_fps="$(sed -n 's/^observed_fps=//p' \1003 "$evidence/external-cqp-capture.log")"1004native_observed_fps="$(sed -n 's/^observed_fps=//p' "$evidence/native-capture.log")"1005[[ -n "$fixed_observed_fps" && -n "$external_cqp_observed_fps" \1006 && -n "$native_observed_fps" ]]1007if awk -v fps="$native_observed_fps" 'BEGIN { exit !(fps >= 55) }'; then1008 native_55fps_status=met1009else1010 native_55fps_status=below1011fi10121013{1014 echo candidate_kind=unsigned-development-hardware-uat1015 echo "source_commit=$uat_commit"1016 echo "workflow_run_id=$workflow_run_id"1017 echo fixed_size=1280x8001018 echo fixed_minimum_fps=551019 echo "fixed_observed_fps=$fixed_observed_fps"1020 echo external_cqp_minimum_fps=551021 echo "external_cqp_observed_fps=$external_cqp_observed_fps"1022 echo "vaapi_render_node=$render_node"1023 echo "vaapi_encoder=$va_encoder"1024 echo "vaapi_selection_mode=$vaapi_selection_mode"1025 echo cross_state_capture_contention=rejected1026 echo gamescope_pointer_restart_cycles=21027 echo gamescope_pointer_feedback_recovery=pass1028 echo gamescope_pointer_daemon_pid_stable=pass1029 echo "native_size=$native_size"1030 echo "native_mode_relation=$native_mode_relation"1031 echo native_performance_gate=observational1032 echo "native_observed_fps=$native_observed_fps"1033 echo "native_55fps_status=$native_55fps_status"1034 echo persistent_authenticated_identity=pass1035 echo invitation_redemption=pass1036 echo invitation_replay=rejected1037 echo "maximum_required_glibc=$max_glibc"1038 echo session_id_scope=daemon-invocation1039 echo "fixed_daemon_unique_sessions=$fixed_unique_sessions"1040 echo "native_daemon_unique_sessions=$native_unique_sessions"1041 echo "total_sessions=$total_sessions"1042 echo slow_media_consumer=pass1043 echo slow_media_consumer_resolutions=1280x800,"$native_size"1044 echo forced_relay=pass1045 echo forced_relay_scope=probe-control-media-input1046 summarize_group fixed-iroh-moq1047 summarize_group fixed-grouped-v31048 summarize_group native-iroh-moq1049 summarize_group native-grouped-v31050} >"$private/summary.pending"10511052restore_original1053[[ "$(systemctl --user is-active sigil-host.service)" == active ]]1054[[ "$(systemctl --user is-enabled sigil-host.service)" == "$original_enabled" ]]1055sha256sum -c "$private/original-config.sha256"1056sha256sum -c "$private/original-identity.sha256"1057[[ "$(readlink "$HOME/.local/libexec/sigil-spark/current")" == "$(cat "$private/original-current-link")" ]]1058[[ "$(stat -c '%i:%s:%Y:%a:%U:%G' "$real_identity")" == "$(cat "$private/original-identity.stat")" ]]10591060{1061 echo hardware_uat=pass1062 cat "$private/summary.pending"1063 echo original_service_restored=pass1064 echo original_config_preserved=pass1065 echo original_identity_preserved=pass1066 echo original_release_preserved=pass1067} >"$evidence/summary.env"1068chmod 0600 \1069 "$evidence/summary.env" \1070 "$evidence"/probes/*.log \1071 "$evidence"/*-capture.log \1072 "$evidence"/pointer-recovery-*.log1073cat "$evidence/summary.env"