feat(test): add downstream acceptance and fault harness artifacts
This commit packages the standalone task-14 acceptance and task-15 fault-suite execution toolchain for downstream validation. It includes all runnable harness scripts, helper utilities, and generated evidence captures so downstream behavior can be reproduced and reviewed independently from docs and core implementation. Bundling these assets separately allows QA/automation workflows to validate runtime changes without dragging operational notes or release-gate documentation into the same review unit. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-opencode) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
This commit is contained in:
Executable
+253
@@ -0,0 +1,253 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -u -o pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
STREAMER_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
WORKTREE_ROOT="$(cd "${STREAMER_ROOT}/../.." && pwd)"
|
||||
BUILD_DIR="${STREAMER_ROOT}/build"
|
||||
|
||||
EVIDENCE_ROOT="${WORKTREE_ROOT}/.sisyphus/evidence"
|
||||
TASK_EVIDENCE_DIR="${EVIDENCE_ROOT}/task-17-release-gate"
|
||||
PASS_EVIDENCE="${EVIDENCE_ROOT}/task-17-release-gate.txt"
|
||||
FAIL_EVIDENCE="${EVIDENCE_ROOT}/task-17-release-gate-error.txt"
|
||||
|
||||
RUN_ID="$(date +"%Y%m%dT%H%M%S")"
|
||||
RUN_DIR="${TASK_EVIDENCE_DIR}/${RUN_ID}"
|
||||
STARTED_AT_UTC="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
|
||||
|
||||
INJECT_FAILURE=""
|
||||
|
||||
print_usage() {
|
||||
cat <<'EOF'
|
||||
usage: ./scripts/release_gate.sh [--inject-failure GATE]
|
||||
|
||||
Runs final release gate:
|
||||
1) downstream build
|
||||
2) mandatory standalone acceptance suite
|
||||
3) mandatory fault suite baseline
|
||||
4) required evidence presence checks
|
||||
5) scope-constraint checks (no direct RTSP/WebRTC publisher, no audio)
|
||||
|
||||
Optional deterministic failure injection (for QA):
|
||||
--inject-failure evidence Force evidence gate failure without modifying repository state
|
||||
EOF
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--inject-failure)
|
||||
if [[ $# -lt 2 ]]; then
|
||||
echo "missing value for --inject-failure" >&2
|
||||
exit 2
|
||||
fi
|
||||
INJECT_FAILURE="$2"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
print_usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "unknown argument: $1" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -n "${INJECT_FAILURE}" && "${INJECT_FAILURE}" != "evidence" ]]; then
|
||||
echo "unsupported --inject-failure '${INJECT_FAILURE}' (supported: evidence)" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
mkdir -p "${RUN_DIR}"
|
||||
|
||||
GATE_LINES=()
|
||||
FAILED_GATES=()
|
||||
|
||||
record_gate() {
|
||||
local gate="$1"
|
||||
local status="$2"
|
||||
local detail="$3"
|
||||
GATE_LINES+=("${gate}|${status}|${detail}")
|
||||
if [[ "${status}" != "PASS" ]]; then
|
||||
FAILED_GATES+=("${gate}")
|
||||
fi
|
||||
}
|
||||
|
||||
run_command_gate() {
|
||||
local gate="$1"
|
||||
local log_path="$2"
|
||||
shift 2
|
||||
|
||||
"$@" >"${log_path}" 2>&1
|
||||
local rc=$?
|
||||
if (( rc == 0 )); then
|
||||
record_gate "${gate}" "PASS" "rc=0 log=${log_path}"
|
||||
else
|
||||
record_gate "${gate}" "FAIL" "rc=${rc} log=${log_path}"
|
||||
fi
|
||||
return ${rc}
|
||||
}
|
||||
|
||||
build_log="${RUN_DIR}/build.log"
|
||||
acceptance_log="${RUN_DIR}/acceptance.log"
|
||||
fault_log="${RUN_DIR}/fault-suite.log"
|
||||
scope_log="${RUN_DIR}/scope.log"
|
||||
|
||||
build_rc=0
|
||||
run_command_gate "build" "${build_log}" bash -lc "cmake -B \"${BUILD_DIR}\" -S \"${STREAMER_ROOT}\" && cmake --build \"${BUILD_DIR}\""
|
||||
build_rc=$?
|
||||
|
||||
acceptance_rc=0
|
||||
if (( build_rc == 0 )); then
|
||||
run_command_gate "acceptance_standalone" "${acceptance_log}" "${SCRIPT_DIR}/acceptance_standalone.sh"
|
||||
acceptance_rc=$?
|
||||
else
|
||||
record_gate "acceptance_standalone" "SKIP" "blocked_by=build"
|
||||
acceptance_rc=1
|
||||
fi
|
||||
|
||||
fault_rc=0
|
||||
if (( build_rc == 0 )); then
|
||||
run_command_gate "fault_suite_baseline" "${fault_log}" "${SCRIPT_DIR}/fault_suite.sh"
|
||||
fault_rc=$?
|
||||
else
|
||||
record_gate "fault_suite_baseline" "SKIP" "blocked_by=build"
|
||||
fault_rc=1
|
||||
fi
|
||||
|
||||
required_evidence=(
|
||||
"${EVIDENCE_ROOT}/task-14-acceptance.txt"
|
||||
"${EVIDENCE_ROOT}/task-14-acceptance-summary.json"
|
||||
"${EVIDENCE_ROOT}/task-15-fault-suite.txt"
|
||||
"${EVIDENCE_ROOT}/task-15-fault-suite-summary.json"
|
||||
"${EVIDENCE_ROOT}/task-15-fault-suite-error.txt"
|
||||
"${EVIDENCE_ROOT}/task-15-fault-suite-error-summary.json"
|
||||
"${EVIDENCE_ROOT}/task-16-docs.txt"
|
||||
"${EVIDENCE_ROOT}/task-16-docs-error.txt"
|
||||
)
|
||||
|
||||
if [[ "${INJECT_FAILURE}" == "evidence" ]]; then
|
||||
required_evidence+=("${EVIDENCE_ROOT}/__forced_missing_for_task17.txt")
|
||||
fi
|
||||
|
||||
missing_evidence=()
|
||||
for path in "${required_evidence[@]}"; do
|
||||
if [[ ! -f "${path}" ]]; then
|
||||
missing_evidence+=("${path}")
|
||||
fi
|
||||
done
|
||||
|
||||
if (( ${#missing_evidence[@]} == 0 )); then
|
||||
record_gate "required_evidence" "PASS" "all_required_files_present"
|
||||
else
|
||||
record_gate "required_evidence" "FAIL" "missing=${missing_evidence[*]}"
|
||||
fi
|
||||
|
||||
scope_failures=()
|
||||
|
||||
if ! grep -Eiq '(video[- ]only|no[[:space:]]+audio[[:space:]]+support)' "${STREAMER_ROOT}/README.md" "${STREAMER_ROOT}/docs/caveats.md"; then
|
||||
scope_failures+=("missing explicit video-only/no-audio scope declaration in README/docs")
|
||||
fi
|
||||
if ! grep -q "Optional Checks (Non-Blocking)" "${STREAMER_ROOT}/docs/compat_matrix.md"; then
|
||||
scope_failures+=("docs/compat_matrix.md missing optional/non-blocking separation")
|
||||
fi
|
||||
if ! grep -q "No Direct RTSP/WebRTC Publishing" "${STREAMER_ROOT}/docs/caveats.md"; then
|
||||
scope_failures+=("docs/caveats.md missing 'No Direct RTSP/WebRTC Publishing'")
|
||||
fi
|
||||
if ! grep -q "No Audio Support" "${STREAMER_ROOT}/docs/caveats.md"; then
|
||||
scope_failures+=("docs/caveats.md missing 'No Audio Support'")
|
||||
fi
|
||||
|
||||
if [[ -x "${BUILD_DIR}/cvmmap_streamer" ]]; then
|
||||
streamer_help="$(${BUILD_DIR}/cvmmap_streamer --help 2>&1 || true)"
|
||||
if [[ "${streamer_help}" == *"--rtsp"* ]]; then
|
||||
scope_failures+=("cvmmap_streamer CLI exposes forbidden --rtsp flag")
|
||||
fi
|
||||
if [[ "${streamer_help}" == *"--webrtc"* ]]; then
|
||||
scope_failures+=("cvmmap_streamer CLI exposes forbidden --webrtc flag")
|
||||
fi
|
||||
if [[ "${streamer_help}" == *"--audio"* ]]; then
|
||||
scope_failures+=("cvmmap_streamer CLI exposes forbidden --audio flag")
|
||||
fi
|
||||
else
|
||||
scope_failures+=("missing ${BUILD_DIR}/cvmmap_streamer for CLI scope validation")
|
||||
fi
|
||||
|
||||
scope_scan_paths=(
|
||||
"${STREAMER_ROOT}/src/config"
|
||||
"${STREAMER_ROOT}/src/core"
|
||||
"${STREAMER_ROOT}/src/ipc"
|
||||
"${STREAMER_ROOT}/src/pipeline"
|
||||
"${STREAMER_ROOT}/src/protocol"
|
||||
"${STREAMER_ROOT}/include/cvmmap_streamer"
|
||||
)
|
||||
|
||||
if grep -RInE --include='*.cpp' --include='*.hpp' --include='*.h' '(rtsp|webrtc)' "${scope_scan_paths[@]}" >"${scope_log}" 2>&1; then
|
||||
scope_failures+=("production code references forbidden direct publisher tokens (rtsp|webrtc); see ${scope_log}")
|
||||
fi
|
||||
if grep -RInE --include='*.cpp' --include='*.hpp' --include='*.h' '(audio|aac|opus|vorbis)' "${scope_scan_paths[@]}" >>"${scope_log}" 2>&1; then
|
||||
scope_failures+=("production code references forbidden audio tokens; see ${scope_log}")
|
||||
fi
|
||||
|
||||
if (( ${#scope_failures[@]} == 0 )); then
|
||||
record_gate "scope_constraints" "PASS" "no_direct_rtsp_webrtc_and_no_audio_confirmed"
|
||||
else
|
||||
record_gate "scope_constraints" "FAIL" "${scope_failures[*]}"
|
||||
fi
|
||||
|
||||
FINISHED_AT_UTC="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
|
||||
|
||||
overall_status="PASS"
|
||||
if (( ${#FAILED_GATES[@]} > 0 )); then
|
||||
overall_status="FAIL"
|
||||
fi
|
||||
|
||||
write_common_header() {
|
||||
local target_file="$1"
|
||||
{
|
||||
echo "task=17"
|
||||
echo "run_id=${RUN_ID}"
|
||||
echo "run_dir=${RUN_DIR}"
|
||||
echo "started_at=${STARTED_AT_UTC}"
|
||||
echo "finished_at=${FINISHED_AT_UTC}"
|
||||
echo "inject_failure=${INJECT_FAILURE:-none}"
|
||||
echo "status=${overall_status}"
|
||||
echo "gate_summary_begin"
|
||||
for line in "${GATE_LINES[@]}"; do
|
||||
echo "${line}"
|
||||
done
|
||||
echo "gate_summary_end"
|
||||
echo "evidence_bundle_begin"
|
||||
for path in "${required_evidence[@]}"; do
|
||||
echo "${path}"
|
||||
done
|
||||
echo "evidence_bundle_end"
|
||||
} >"${target_file}"
|
||||
}
|
||||
|
||||
if [[ "${overall_status}" == "PASS" ]]; then
|
||||
write_common_header "${PASS_EVIDENCE}"
|
||||
{
|
||||
echo "residual_risks_begin"
|
||||
echo "[LOW] NVENC availability is host-dependent; software fallback can increase CPU usage under sustained load."
|
||||
echo "[LOW] Optional SRS/ZLMediaKit smoke checks remain non-blocking and are intentionally excluded from mandatory release gating."
|
||||
echo "[MEDIUM] Scope validation uses static token/pattern checks and documentation assertions; architectural regressions still require code review discipline."
|
||||
echo "residual_risks_end"
|
||||
} >>"${PASS_EVIDENCE}"
|
||||
|
||||
echo "release gate PASS"
|
||||
echo "evidence: ${PASS_EVIDENCE}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
write_common_header "${FAIL_EVIDENCE}"
|
||||
{
|
||||
echo "failing_gates=${FAILED_GATES[*]}"
|
||||
echo "failure_reason=one_or_more_gates_failed"
|
||||
} >>"${FAIL_EVIDENCE}"
|
||||
|
||||
echo "release gate FAIL: ${FAILED_GATES[*]}" >&2
|
||||
echo "evidence: ${FAIL_EVIDENCE}" >&2
|
||||
exit 1
|
||||
Reference in New Issue
Block a user