#!/usr/bin/env bash set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" STREAMER_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" BUILD_DIR="${STREAMER_ROOT}/build" EVIDENCE_ROOT="${STREAMER_ROOT}/.sisyphus/evidence" TASK_EVIDENCE_DIR="${EVIDENCE_ROOT}/task-15-fault-suite" SUMMARY_HELPER="${SCRIPT_DIR}/fault_summary_helper.py" MODE="baseline" if [[ $# -gt 0 ]]; then case "$1" in --mode) if [[ $# -lt 2 ]]; then echo "missing value for --mode" >&2 exit 2 fi MODE="$2" shift 2 ;; --degraded) MODE="degraded" shift ;; *) echo "unknown argument: $1" >&2 exit 2 ;; esac fi if [[ "${MODE}" != "baseline" && "${MODE}" != "degraded" ]]; then echo "invalid --mode '${MODE}' (expected: baseline|degraded)" >&2 exit 2 fi RUN_ID="" RUN_DIR="" MANIFEST_TSV="" SUMMARY_JSON="" LATEST_SUMMARY_JSON="${EVIDENCE_ROOT}/task-15-fault-suite-summary.json" EVIDENCE_TEXT="${EVIDENCE_ROOT}/task-15-fault-suite.txt" STARTED_AT_UTC="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" mkdir -p "${TASK_EVIDENCE_DIR}" allocate_run_dir() { local attempts=0 while (( attempts < 50 )); do local candidate_id candidate_id="$(date +"%Y%m%dT%H%M%S")-$(date +"%N")-p$$-$RANDOM" local candidate_dir="${TASK_EVIDENCE_DIR}/${candidate_id}-${MODE}" if mkdir "${candidate_dir}" 2>/dev/null; then RUN_ID="${candidate_id}" RUN_DIR="${candidate_dir}" MANIFEST_TSV="${RUN_DIR}/rows.tsv" SUMMARY_JSON="${RUN_DIR}/summary.json" return 0 fi attempts=$((attempts + 1)) sleep 0.01 done echo "failed to allocate unique fault-suite run directory" >&2 return 1 } allocate_run_dir || exit 1 echo -e "order\tscenario_id\tname\tstatus\treason\tduration_ms\tcommand_rc\tlog_path" > "${MANIFEST_TSV}" append_manifest_row() { local order="$1" local scenario_id="$2" local name="$3" local status="$4" local reason="$5" local duration_ms="$6" local command_rc="$7" local log_path="$8" echo -e "${order}\t${scenario_id}\t${name}\t${status}\t${reason}\t${duration_ms}\t${command_rc}\t${log_path}" >> "${MANIFEST_TSV}" } run_expected_failure() { local order="$1" local scenario_id="$2" local name="$3" local expected_substring="$4" shift 4 local row_dir="${RUN_DIR}/${order}-${scenario_id}" mkdir -p "${row_dir}" local log_path="${row_dir}/command.log" local row_start_ms row_start_ms="$(date +%s%3N)" set +e "$@" >"${log_path}" 2>&1 local command_rc=$? set -e local row_end_ms row_end_ms="$(date +%s%3N)" local duration_ms=$((row_end_ms - row_start_ms)) local status="FAIL" local reason="expected non-zero rc and log token '${expected_substring}'" if (( command_rc != 0 )) && grep -Fq "${expected_substring}" "${log_path}"; then status="PASS" reason="command failed as expected" fi append_manifest_row \ "${order}" \ "${scenario_id}" \ "${name}" \ "${status}" \ "${reason}" \ "${duration_ms}" \ "${command_rc}" \ "${log_path}" printf "[%s] %s => %s (%s)\n" "${scenario_id}" "${name}" "${status}" "${reason}" } main() { local required=( "${BUILD_DIR}/cvmmap_streamer" "${BUILD_DIR}/rtmp_output_tester" ) local missing=() for bin in "${required[@]}"; do if [[ ! -x "${bin}" ]]; then missing+=("${bin}") fi done if (( ${#missing[@]} > 0 )); then { echo "task=15" echo "mode=${MODE}" echo "run_id=${RUN_ID}" echo "run_dir=${RUN_DIR}" echo "manifest=${MANIFEST_TSV}" echo "missing_binaries=${missing[*]}" } > "${EVIDENCE_TEXT}" echo "missing binaries: ${missing[*]}" >&2 return 1 fi run_expected_failure 1 "removed_encoder_backend" "removed encoder backend rejected" \ "invalid encoder backend: 'gstreamer_legacy' was removed; use ffmpeg" \ "${BUILD_DIR}/cvmmap_streamer" \ --run-mode pipeline \ --input-uri cvmmap://default \ --encoder-backend gstreamer_legacy run_expected_failure 2 "removed_rtmp_transport" "removed RTMP transport rejected" \ "invalid rtmp transport: 'legacy_custom' was removed; use libavformat or ffmpeg_process" \ "${BUILD_DIR}/cvmmap_streamer" \ --run-mode pipeline \ --input-uri cvmmap://default \ --rtmp \ --rtmp-url rtmp://127.0.0.1/live/test \ --rtmp-transport legacy_custom run_expected_failure 3 "removed_rtmp_mode_cli" "removed RTMP mode flag rejected" \ "unknown argument: --rtmp-mode (removed; RTMP always uses enhanced mode)" \ "${BUILD_DIR}/cvmmap_streamer" \ --rtmp-mode enhanced local mode_row_dir="${RUN_DIR}/4-removed_rtmp_mode_toml" mkdir -p "${mode_row_dir}" local mode_config="${mode_row_dir}/removed_rtmp_mode.toml" cat >"${mode_config}" <<'EOF' [outputs.rtmp] enabled = true urls = ["rtmp://127.0.0.1/live/test"] mode = "enhanced" EOF run_expected_failure 4 "removed_rtmp_mode_toml" "removed RTMP mode TOML rejected" \ "invalid RTMP config: outputs.rtmp.mode was removed; RTMP always uses enhanced mode" \ "${BUILD_DIR}/cvmmap_streamer" \ --config "${mode_config}" run_expected_failure 5 "missing_rtmp_url" "missing RTMP URL rejected" \ "invalid RTMP config: enabled RTMP output requires at least one URL" \ "${BUILD_DIR}/cvmmap_streamer" \ --run-mode pipeline \ --input-uri cvmmap://default \ --rtmp run_expected_failure 6 "invalid_rtp_endpoint" "invalid RTP endpoint rejected" \ "invalid RTP config: endpoint must be in ':' format" \ "${BUILD_DIR}/cvmmap_streamer" \ --run-mode pipeline \ --input-uri cvmmap://default \ --rtp \ --rtp-endpoint invalid run_expected_failure 7 "ffmpeg_process_bad_binary" "ffmpeg_process child failure surfaces" \ "child exited before publish completed" \ "${BUILD_DIR}/rtmp_output_tester" \ --rtmp-url rtmp://127.0.0.1/live/test \ --transport ffmpeg_process \ --ffmpeg-path /nonexistent/ffmpeg \ --codec h264 \ --frames 256 \ --width 640 \ --height 360 \ --frame-interval-ms 1 \ --linger-ms 0 local finished_at_utc finished_at_utc="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" python3 "${SUMMARY_HELPER}" \ --manifest "${MANIFEST_TSV}" \ --output "${SUMMARY_JSON}" \ --run-id "${RUN_ID}" \ --run-dir "${RUN_DIR}" \ --started-at "${STARTED_AT_UTC}" \ --finished-at "${finished_at_utc}" \ --mode "${MODE}" cp -f "${SUMMARY_JSON}" "${LATEST_SUMMARY_JSON}" 2>/dev/null || true local total_count pass_count fail_count all_pass total_count="$(python3 - <<'PY' "${SUMMARY_JSON}" import json, sys data = json.load(open(sys.argv[1], "r", encoding="utf-8")) counts = data.get("counts", {}) print(counts.get("total", 0)) PY )" pass_count="$(python3 - <<'PY' "${SUMMARY_JSON}" import json, sys data = json.load(open(sys.argv[1], "r", encoding="utf-8")) counts = data.get("counts", {}) print(counts.get("pass", 0)) PY )" fail_count="$(python3 - <<'PY' "${SUMMARY_JSON}" import json, sys data = json.load(open(sys.argv[1], "r", encoding="utf-8")) counts = data.get("counts", {}) print(counts.get("fail", 0)) PY )" all_pass="$(python3 - <<'PY' "${SUMMARY_JSON}" import json, sys data = json.load(open(sys.argv[1], "r", encoding="utf-8")) print("true" if data.get("all_pass", False) else "false") PY )" { echo "task=15" echo "mode=${MODE}" echo "run_id=${RUN_ID}" echo "run_dir=${RUN_DIR}" echo "manifest=${MANIFEST_TSV}" echo "summary_json=${SUMMARY_JSON}" echo "started_at=${STARTED_AT_UTC}" echo "finished_at=${finished_at_utc}" echo "counts_total=${total_count}" echo "counts_pass=${pass_count}" echo "counts_fail=${fail_count}" echo "all_pass=${all_pass}" echo "scenarios=removed_encoder_backend,removed_rtmp_transport,removed_rtmp_mode_cli,removed_rtmp_mode_toml,missing_rtmp_url,invalid_rtp_endpoint,ffmpeg_process_bad_binary" } > "${EVIDENCE_TEXT}" if [[ "${all_pass}" == "true" ]]; then echo "fault suite PASS (${pass_count}/${total_count})" echo "summary: ${SUMMARY_JSON}" return 0 fi echo "fault suite FAIL (${pass_count}/${total_count})" >&2 echo "summary: ${SUMMARY_JSON}" >&2 return 1 } main "$@"