Compare commits

...

2 Commits

Author SHA1 Message Date
crosstyan 51d03d4279 feat(encode): add Jetson Multimedia API encoder backend
Integrate a native Jetson Multimedia API encoder path and keep the existing encoded access-unit contract for RTMP, RTP, and recording consumers.

This adds conditional Jetson MMAPI detection in CMake, builds the required NVIDIA sample common classes into a dedicated support library, and compiles the new backend only when the platform dependencies are present.

The runtime selector now lets encoder.backend=auto prefer Jetson MM for NVIDIA hardware requests while keeping encoder.backend=ffmpeg as an explicit FFmpeg path. Device selection semantics are updated so nvidia requires the Jetson backend, auto can fall back to FFmpeg software, and software remains FFmpeg-only.

The Jetson backend converts supported raw inputs through swscale, feeds NvVideoEncoder in YUV420M, emits Annex-B access units, and harvests decoder configuration from a warmup keyframe so downstream packetizers keep their existing contract.

This also splits FFmpeg encoder option handling into a shared header, updates runtime config/help text and tester defaults, and refreshes compatibility/caveat documentation to reflect the new selection behavior.

Build-tree runtime RPATH handling is tightened so GCC 15 builds keep the matching libstdc++ visible locally. Verification covered GCC 15 builds, RTP H.264/H.265 tester runs, RTMP H.264 stub publish, local live SRS smoke with cvmmap://zed1, and remote execution on 192.168.2.155 using the deployed bundle lib directory for the GCC 15 runtime.
2026-04-15 18:37:44 +08:00
crosstyan 16a1a38645 Make MCAP and depth support optional 2026-04-14 17:13:29 +08:00
20 changed files with 2298 additions and 575 deletions
+369 -81
View File
@@ -8,6 +8,82 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
include(GNUInstallDirs)
set(CVMMAP_STREAMER_INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}")
function(cvmmap_streamer_append_runtime_dir out_var runtime_library)
execute_process(
COMMAND "${CMAKE_CXX_COMPILER}" -print-file-name=${runtime_library}
OUTPUT_VARIABLE runtime_library_path
OUTPUT_STRIP_TRAILING_WHITESPACE)
if (IS_ABSOLUTE "${runtime_library_path}")
get_filename_component(runtime_library_dir "${runtime_library_path}" DIRECTORY)
set(${out_var} "${${out_var}};${runtime_library_dir}" PARENT_SCOPE)
endif()
endfunction()
function(cvmmap_streamer_resolve_feature_mode out_var option_name requested_mode available unavailable_reason)
set(_valid_modes AUTO ON OFF)
list(FIND _valid_modes "${requested_mode}" _requested_mode_index)
if (_requested_mode_index EQUAL -1)
message(FATAL_ERROR
"Invalid ${option_name}='${requested_mode}' (expected: AUTO|ON|OFF)")
endif()
if (requested_mode STREQUAL "OFF")
set(${out_var} FALSE PARENT_SCOPE)
return()
endif()
if (requested_mode STREQUAL "ON")
if (NOT available)
message(FATAL_ERROR
"${option_name}=ON requires ${unavailable_reason}")
endif()
set(${out_var} TRUE PARENT_SCOPE)
return()
endif()
set(${out_var} ${available} PARENT_SCOPE)
endfunction()
set(CVMMAP_STREAMER_BUILD_RPATH "")
# Build-tree binaries compiled with a non-system GCC need that toolchain's
# C++ runtime on the dynamic loader search path.
cvmmap_streamer_append_runtime_dir(CVMMAP_STREAMER_BUILD_RPATH libstdc++.so.6)
cvmmap_streamer_append_runtime_dir(CVMMAP_STREAMER_BUILD_RPATH libgcc_s.so.1)
list(REMOVE_DUPLICATES CVMMAP_STREAMER_BUILD_RPATH)
set(
CVMMAP_STREAMER_ENABLE_MCAP
"AUTO"
CACHE STRING
"Enable MCAP recording support: AUTO, ON, or OFF")
set_property(CACHE CVMMAP_STREAMER_ENABLE_MCAP PROPERTY STRINGS AUTO ON OFF)
set(
CVMMAP_STREAMER_ENABLE_MCAP_DEPTH
"AUTO"
CACHE STRING
"Enable MCAP depth recording support: AUTO, ON, or OFF")
set_property(CACHE CVMMAP_STREAMER_ENABLE_MCAP_DEPTH PROPERTY STRINGS AUTO ON OFF)
set(
CVMMAP_STREAMER_MCAP_INCLUDE_DIR
"${CMAKE_CURRENT_LIST_DIR}/third_party/mcap/include"
CACHE PATH
"Path to MCAP headers")
set(
CVMMAP_STREAMER_ENABLE_JETSON_MM
"AUTO"
CACHE STRING
"Enable Jetson Multimedia API encoder backend: AUTO, ON, or OFF")
set_property(CACHE CVMMAP_STREAMER_ENABLE_JETSON_MM PROPERTY STRINGS AUTO ON OFF)
set(
CVMMAP_STREAMER_JETSON_MMAPI_ROOT
"/usr/src/jetson_multimedia_api"
CACHE PATH
"Path to the Jetson Multimedia API root")
find_package(Threads REQUIRED)
find_package(OpenSSL REQUIRED)
if (NOT TARGET OpenSSL::SSL AND DEFINED OPENSSL_SSL_LIBRARY)
@@ -55,7 +131,7 @@ if (CVMMAP_CNATS_PROVIDER STREQUAL "system")
find_package(cnats CONFIG REQUIRED)
find_package(cvmmap-core CONFIG QUIET PATHS "${CVMMAP_LOCAL_CORE_DIR}" NO_DEFAULT_PATH)
if (NOT TARGET cvmmap::client)
find_package(cvmmap-core CONFIG QUIET)
find_package(cvmmap-core CONFIG REQUIRED PATHS "${CVMMAP_LOCAL_CORE_DIR}" NO_DEFAULT_PATH)
endif()
else()
if (NOT EXISTS "${CVMMAP_LOCAL_NATS_STATIC}")
@@ -73,15 +149,182 @@ endif()
find_package(ZeroMQ QUIET)
find_package(spdlog REQUIRED)
find_package(Protobuf REQUIRED)
if (Protobuf_VERSION VERSION_LESS 3.13)
set(_cvmmap_streamer_protoc_wrapper "${CMAKE_BINARY_DIR}/cvmmap-streamer-protoc-wrapper.sh")
file(WRITE "${_cvmmap_streamer_protoc_wrapper}"
"#!/bin/sh\nexec \"${Protobuf_PROTOC_EXECUTABLE}\" --experimental_allow_proto3_optional \"$@\"\n")
file(CHMOD
"${_cvmmap_streamer_protoc_wrapper}"
PERMISSIONS
OWNER_READ OWNER_WRITE OWNER_EXECUTE
GROUP_READ GROUP_EXECUTE
WORLD_READ WORLD_EXECUTE)
if (TARGET protobuf::protoc)
set_target_properties(protobuf::protoc PROPERTIES IMPORTED_LOCATION "${_cvmmap_streamer_protoc_wrapper}")
endif()
set(Protobuf_PROTOC_EXECUTABLE "${_cvmmap_streamer_protoc_wrapper}")
endif()
find_package(PkgConfig REQUIRED)
find_package(rvl CONFIG QUIET)
set(_CVMMAP_STREAMER_MCAP_HEADERS_AVAILABLE FALSE)
if (EXISTS "${CVMMAP_STREAMER_MCAP_INCLUDE_DIR}/mcap/mcap.hpp")
set(_CVMMAP_STREAMER_MCAP_HEADERS_AVAILABLE TRUE)
if (NOT TARGET mcap::mcap)
add_library(cvmmap_streamer_mcap_headers INTERFACE)
target_include_directories(cvmmap_streamer_mcap_headers INTERFACE "${CVMMAP_STREAMER_MCAP_INCLUDE_DIR}")
add_library(mcap::mcap ALIAS cvmmap_streamer_mcap_headers)
endif()
elseif (NOT TARGET mcap::mcap)
add_library(mcap::mcap INTERFACE IMPORTED GLOBAL)
endif()
add_subdirectory(third_party)
pkg_check_modules(FFMPEG REQUIRED IMPORTED_TARGET libavcodec libavformat libavutil libswscale)
pkg_check_modules(PROTOBUF_PKG QUIET IMPORTED_TARGET protobuf)
pkg_check_modules(ZSTD REQUIRED IMPORTED_TARGET libzstd)
pkg_check_modules(LZ4 REQUIRED IMPORTED_TARGET liblz4)
pkg_check_modules(ZSTD QUIET IMPORTED_TARGET libzstd)
pkg_check_modules(LZ4 QUIET IMPORTED_TARGET liblz4)
set(_CVMMAP_STREAMER_MCAP_MISSING_DEPS)
if (NOT _CVMMAP_STREAMER_MCAP_HEADERS_AVAILABLE)
list(APPEND _CVMMAP_STREAMER_MCAP_MISSING_DEPS "MCAP headers at ${CVMMAP_STREAMER_MCAP_INCLUDE_DIR}")
endif()
if (NOT ZSTD_FOUND)
list(APPEND _CVMMAP_STREAMER_MCAP_MISSING_DEPS "pkg-config package libzstd")
endif()
if (NOT LZ4_FOUND)
list(APPEND _CVMMAP_STREAMER_MCAP_MISSING_DEPS "pkg-config package liblz4")
endif()
if (_CVMMAP_STREAMER_MCAP_MISSING_DEPS)
list(JOIN _CVMMAP_STREAMER_MCAP_MISSING_DEPS ", " _CVMMAP_STREAMER_MCAP_UNAVAILABLE_REASON)
set(_CVMMAP_STREAMER_MCAP_AVAILABLE FALSE)
else()
set(_CVMMAP_STREAMER_MCAP_UNAVAILABLE_REASON "MCAP dependencies")
set(_CVMMAP_STREAMER_MCAP_AVAILABLE TRUE)
endif()
cvmmap_streamer_resolve_feature_mode(
CVMMAP_STREAMER_HAS_MCAP_BOOL
CVMMAP_STREAMER_ENABLE_MCAP
"${CVMMAP_STREAMER_ENABLE_MCAP}"
${_CVMMAP_STREAMER_MCAP_AVAILABLE}
"${_CVMMAP_STREAMER_MCAP_UNAVAILABLE_REASON}")
set(RVL_LOCAL_ROOT "${CMAKE_CURRENT_LIST_DIR}/../rvl_impl" CACHE PATH "Path to a local rvl_impl checkout")
set(RVL_LOCAL_BUILD "${RVL_LOCAL_ROOT}/build/core" CACHE PATH "Path to local rvl_impl build artifacts")
set(_CVMMAP_STREAMER_RVL_AVAILABLE FALSE)
set(_CVMMAP_STREAMER_RVL_UNAVAILABLE_REASON "rvl::rvl target or local rvl_impl build artifacts")
if (TARGET rvl::rvl)
set(_CVMMAP_STREAMER_RVL_AVAILABLE TRUE)
elseif (
EXISTS "${RVL_LOCAL_ROOT}/core/include/rvl/rvl.hpp"
AND EXISTS "${RVL_LOCAL_BUILD}/librvl_core.a")
set(_CVMMAP_STREAMER_RVL_AVAILABLE TRUE)
endif()
if (CVMMAP_STREAMER_ENABLE_MCAP STREQUAL "OFF" AND CVMMAP_STREAMER_ENABLE_MCAP_DEPTH STREQUAL "ON")
message(FATAL_ERROR
"CVMMAP_STREAMER_ENABLE_MCAP_DEPTH=ON requires CVMMAP_STREAMER_ENABLE_MCAP to be AUTO or ON")
endif()
if (CVMMAP_STREAMER_ENABLE_MCAP_DEPTH STREQUAL "ON" AND NOT CVMMAP_STREAMER_HAS_MCAP_BOOL)
message(FATAL_ERROR
"CVMMAP_STREAMER_ENABLE_MCAP_DEPTH=ON requires MCAP support, but ${_CVMMAP_STREAMER_MCAP_UNAVAILABLE_REASON} is unavailable")
endif()
if (NOT CVMMAP_STREAMER_HAS_MCAP_BOOL AND CVMMAP_STREAMER_ENABLE_MCAP_DEPTH STREQUAL "AUTO")
set(CVMMAP_STREAMER_HAS_MCAP_DEPTH_BOOL FALSE)
else()
cvmmap_streamer_resolve_feature_mode(
CVMMAP_STREAMER_HAS_MCAP_DEPTH_BOOL
CVMMAP_STREAMER_ENABLE_MCAP_DEPTH
"${CVMMAP_STREAMER_ENABLE_MCAP_DEPTH}"
${_CVMMAP_STREAMER_RVL_AVAILABLE}
"${_CVMMAP_STREAMER_RVL_UNAVAILABLE_REASON}")
endif()
if (CVMMAP_STREAMER_HAS_MCAP_BOOL)
set(CVMMAP_STREAMER_HAS_MCAP 1)
else()
set(CVMMAP_STREAMER_HAS_MCAP 0)
endif()
if (CVMMAP_STREAMER_HAS_MCAP_DEPTH_BOOL)
set(CVMMAP_STREAMER_HAS_MCAP_DEPTH 1)
else()
set(CVMMAP_STREAMER_HAS_MCAP_DEPTH 0)
endif()
set(_CVMMAP_STREAMER_JETSON_MM_INCLUDE_DIR "${CVMMAP_STREAMER_JETSON_MMAPI_ROOT}/include")
set(_CVMMAP_STREAMER_JETSON_MM_COMMON_DIR "${CVMMAP_STREAMER_JETSON_MMAPI_ROOT}/samples/common/classes")
set(_CVMMAP_STREAMER_JETSON_MM_COMMON_SOURCES
"${_CVMMAP_STREAMER_JETSON_MM_COMMON_DIR}/NvBuffer.cpp"
"${_CVMMAP_STREAMER_JETSON_MM_COMMON_DIR}/NvElement.cpp"
"${_CVMMAP_STREAMER_JETSON_MM_COMMON_DIR}/NvElementProfiler.cpp"
"${_CVMMAP_STREAMER_JETSON_MM_COMMON_DIR}/NvLogging.cpp"
"${_CVMMAP_STREAMER_JETSON_MM_COMMON_DIR}/NvV4l2Element.cpp"
"${_CVMMAP_STREAMER_JETSON_MM_COMMON_DIR}/NvV4l2ElementPlane.cpp"
"${_CVMMAP_STREAMER_JETSON_MM_COMMON_DIR}/NvVideoEncoder.cpp")
set(_CVMMAP_STREAMER_JETSON_MM_MISSING_DEPS)
if (NOT EXISTS "${_CVMMAP_STREAMER_JETSON_MM_INCLUDE_DIR}/NvVideoEncoder.h")
list(APPEND _CVMMAP_STREAMER_JETSON_MM_MISSING_DEPS
"Jetson Multimedia API headers at ${_CVMMAP_STREAMER_JETSON_MM_INCLUDE_DIR}")
endif()
foreach(_cvmmap_streamer_jetson_mm_source IN LISTS _CVMMAP_STREAMER_JETSON_MM_COMMON_SOURCES)
if (NOT EXISTS "${_cvmmap_streamer_jetson_mm_source}")
list(APPEND _CVMMAP_STREAMER_JETSON_MM_MISSING_DEPS
"Jetson Multimedia API common source ${_cvmmap_streamer_jetson_mm_source}")
endif()
endforeach()
find_library(
CVMMAP_STREAMER_NVBUFSURFACE_LIBRARY
NAMES nvbufsurface
PATHS
/usr/lib/aarch64-linux-gnu/nvidia
/usr/lib/aarch64-linux-gnu/tegra
/usr/lib/aarch64-linux-gnu
PATH_SUFFIXES nvidia tegra)
if (NOT CVMMAP_STREAMER_NVBUFSURFACE_LIBRARY)
list(APPEND _CVMMAP_STREAMER_JETSON_MM_MISSING_DEPS "libnvbufsurface")
endif()
find_library(CVMMAP_STREAMER_LIBV4L2_LIBRARY NAMES v4l2)
if (NOT CVMMAP_STREAMER_LIBV4L2_LIBRARY)
list(APPEND _CVMMAP_STREAMER_JETSON_MM_MISSING_DEPS "libv4l2")
endif()
if (_CVMMAP_STREAMER_JETSON_MM_MISSING_DEPS)
list(JOIN _CVMMAP_STREAMER_JETSON_MM_MISSING_DEPS ", " _CVMMAP_STREAMER_JETSON_MM_UNAVAILABLE_REASON)
set(_CVMMAP_STREAMER_JETSON_MM_AVAILABLE FALSE)
else()
set(_CVMMAP_STREAMER_JETSON_MM_UNAVAILABLE_REASON "Jetson Multimedia API dependencies")
set(_CVMMAP_STREAMER_JETSON_MM_AVAILABLE TRUE)
endif()
cvmmap_streamer_resolve_feature_mode(
CVMMAP_STREAMER_HAS_JETSON_MM_BOOL
CVMMAP_STREAMER_ENABLE_JETSON_MM
"${CVMMAP_STREAMER_ENABLE_JETSON_MM}"
${_CVMMAP_STREAMER_JETSON_MM_AVAILABLE}
"${_CVMMAP_STREAMER_JETSON_MM_UNAVAILABLE_REASON}")
if (CVMMAP_STREAMER_HAS_JETSON_MM_BOOL)
set(CVMMAP_STREAMER_HAS_JETSON_MM 1)
add_library(cvmmap_streamer_jetson_mmapi STATIC
${_CVMMAP_STREAMER_JETSON_MM_COMMON_SOURCES})
target_include_directories(cvmmap_streamer_jetson_mmapi
PUBLIC
"${_CVMMAP_STREAMER_JETSON_MM_INCLUDE_DIR}")
target_link_libraries(cvmmap_streamer_jetson_mmapi
PUBLIC
Threads::Threads
${CVMMAP_STREAMER_LIBV4L2_LIBRARY}
${CVMMAP_STREAMER_NVBUFSURFACE_LIBRARY})
else()
set(CVMMAP_STREAMER_HAS_JETSON_MM 0)
endif()
if (NOT TARGET cvmmap::client)
if (
@@ -104,9 +347,7 @@ if (NOT TARGET cvmmap::client)
endif()
endif()
if (NOT TARGET rvl::rvl)
set(RVL_LOCAL_ROOT "${CMAKE_CURRENT_LIST_DIR}/../rvl_impl" CACHE PATH "Path to a local rvl_impl checkout")
set(RVL_LOCAL_BUILD "${RVL_LOCAL_ROOT}/build/core")
if (CVMMAP_STREAMER_HAS_MCAP_DEPTH AND NOT TARGET rvl::rvl)
if (
EXISTS "${RVL_LOCAL_ROOT}/core/include/rvl/rvl.hpp"
AND EXISTS "${RVL_LOCAL_BUILD}/librvl_core.a")
@@ -185,36 +426,29 @@ target_link_libraries(cvmmap_streamer_foxglove_proto PUBLIC cvmmap_streamer_prot
target_link_libraries(cvmmap_streamer_depth_proto PUBLIC cvmmap_streamer_protobuf)
target_link_libraries(cvmmap_streamer_control_proto PUBLIC cvmmap_streamer_protobuf)
add_library(cvmmap_streamer_mcap_runtime STATIC
src/record/mcap_runtime.cpp)
target_include_directories(cvmmap_streamer_mcap_runtime
PUBLIC)
target_link_libraries(cvmmap_streamer_mcap_runtime
PUBLIC
mcap::mcap
PkgConfig::ZSTD
PkgConfig::LZ4)
add_library(cvmmap_streamer_feature_flags INTERFACE)
target_compile_definitions(cvmmap_streamer_feature_flags
INTERFACE
CVMMAP_STREAMER_HAS_MCAP=${CVMMAP_STREAMER_HAS_MCAP}
CVMMAP_STREAMER_HAS_MCAP_DEPTH=${CVMMAP_STREAMER_HAS_MCAP_DEPTH}
CVMMAP_STREAMER_HAS_JETSON_MM=${CVMMAP_STREAMER_HAS_JETSON_MM})
add_library(cvmmap_streamer_record_support STATIC
src/encode/encoder_backend.cpp
src/encode/ffmpeg_encoder_backend.cpp
src/record/protobuf_descriptor.cpp
src/record/mcap_record_sink.cpp
src/record/mp4_record_writer.cpp)
if (CVMMAP_STREAMER_HAS_JETSON_MM_BOOL)
target_sources(cvmmap_streamer_record_support PRIVATE src/encode/jetson_mm_encoder_backend.cpp)
endif()
target_include_directories(cvmmap_streamer_record_support
PUBLIC
"${CMAKE_CURRENT_LIST_DIR}/include"
"${CMAKE_CURRENT_BINARY_DIR}")
target_link_libraries(cvmmap_streamer_record_support
PUBLIC
cvmmap_streamer_foxglove_proto
cvmmap_streamer_depth_proto
cvmmap_streamer_mcap_runtime
cvmmap_streamer_feature_flags
PkgConfig::FFMPEG
PkgConfig::ZSTD
PkgConfig::LZ4
rvl::rvl
mcap::mcap
msft_proxy4::proxy
cvmmap_streamer_protobuf)
if (TARGET spdlog::spdlog)
@@ -225,6 +459,46 @@ endif()
if (TARGET PkgConfig::PROTOBUF_PKG)
target_link_libraries(cvmmap_streamer_record_support PUBLIC PkgConfig::PROTOBUF_PKG)
endif()
if (CVMMAP_STREAMER_HAS_JETSON_MM_BOOL)
target_link_libraries(cvmmap_streamer_record_support PUBLIC cvmmap_streamer_jetson_mmapi)
endif()
if (CVMMAP_STREAMER_HAS_MCAP)
add_library(cvmmap_streamer_mcap_runtime STATIC
src/record/mcap_runtime.cpp)
target_link_libraries(cvmmap_streamer_mcap_runtime
PUBLIC
cvmmap_streamer_feature_flags
mcap::mcap
PkgConfig::ZSTD
PkgConfig::LZ4)
add_library(cvmmap_streamer_mcap_record_support STATIC
src/record/mcap_record_sink.cpp)
target_include_directories(cvmmap_streamer_mcap_record_support
PUBLIC
"${CMAKE_CURRENT_LIST_DIR}/include"
"${CMAKE_CURRENT_BINARY_DIR}")
target_link_libraries(cvmmap_streamer_mcap_record_support
PUBLIC
cvmmap_streamer_feature_flags
cvmmap_streamer_record_support
cvmmap_streamer_foxglove_proto
cvmmap_streamer_depth_proto
cvmmap_streamer_mcap_runtime
mcap::mcap)
if (CVMMAP_STREAMER_HAS_MCAP_DEPTH)
target_link_libraries(cvmmap_streamer_mcap_record_support PUBLIC rvl::rvl)
endif()
if (TARGET spdlog::spdlog)
target_link_libraries(cvmmap_streamer_mcap_record_support PUBLIC spdlog::spdlog)
elseif (TARGET spdlog)
target_link_libraries(cvmmap_streamer_mcap_record_support PUBLIC spdlog)
endif()
if (TARGET PkgConfig::PROTOBUF_PKG)
target_link_libraries(cvmmap_streamer_mcap_record_support PUBLIC PkgConfig::PROTOBUF_PKG)
endif()
endif()
add_library(cvmmap_streamer_common STATIC
src/ipc/help.cpp
@@ -249,13 +523,14 @@ set(CVMMAP_STREAMER_LINK_DEPS
Threads::Threads
cvmmap_streamer_record_support
PkgConfig::FFMPEG
PkgConfig::ZSTD
PkgConfig::LZ4
cvmmap::client
cvmmap::nats
CLI11::CLI11
tomlplusplus::tomlplusplus
mcap::mcap)
cvmmap_streamer_feature_flags)
if (CVMMAP_STREAMER_HAS_MCAP)
list(APPEND CVMMAP_STREAMER_LINK_DEPS cvmmap_streamer_mcap_record_support)
endif()
if (TARGET cppzmq::cppzmq)
list(APPEND CVMMAP_STREAMER_LINK_DEPS cppzmq::cppzmq)
@@ -283,6 +558,15 @@ endif()
target_link_libraries(cvmmap_streamer_common PUBLIC ${CVMMAP_STREAMER_LINK_DEPS})
function(cvmmap_streamer_apply_runtime_rpath target)
set_target_properties(${target} PROPERTIES
INSTALL_RPATH "${CVMMAP_STREAMER_INSTALL_RPATH}")
if (CVMMAP_STREAMER_BUILD_RPATH)
set_target_properties(${target} PROPERTIES
BUILD_RPATH "${CVMMAP_STREAMER_BUILD_RPATH}")
endif()
endfunction()
function(add_cvmmap_binary target source)
add_executable(${target} ${source} ${ARGN})
target_include_directories(${target}
@@ -295,6 +579,7 @@ function(add_cvmmap_binary target source)
set_target_properties(${target} PROPERTIES
OUTPUT_NAME "${target}"
RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/bin")
cvmmap_streamer_apply_runtime_rpath(${target})
endfunction()
add_cvmmap_binary(cvmmap_streamer src/main_streamer.cpp)
@@ -303,65 +588,68 @@ add_cvmmap_binary(rtp_output_tester src/testers/rtp_output_tester.cpp)
add_cvmmap_binary(rtmp_stub_tester src/testers/rtmp_stub_tester.cpp)
add_cvmmap_binary(rtmp_output_tester src/testers/rtmp_output_tester.cpp)
add_cvmmap_binary(ipc_snapshot_tester src/testers/ipc_snapshot_tester.cpp)
add_cvmmap_binary(mcap_depth_record_tester src/testers/mcap_depth_record_tester.cpp)
add_cvmmap_binary(mcap_body_record_tester src/testers/mcap_body_record_tester.cpp)
add_cvmmap_binary(mcap_body_inspector src/testers/mcap_body_inspector.cpp)
add_cvmmap_binary(mcap_pose_record_tester src/testers/mcap_pose_record_tester.cpp)
add_cvmmap_binary(mcap_multi_record_tester src/testers/mcap_multi_record_tester.cpp)
if (CVMMAP_STREAMER_HAS_MCAP)
add_cvmmap_binary(mcap_body_record_tester src/testers/mcap_body_record_tester.cpp)
add_cvmmap_binary(mcap_body_inspector src/testers/mcap_body_inspector.cpp)
add_cvmmap_binary(mcap_pose_record_tester src/testers/mcap_pose_record_tester.cpp)
add_executable(mcap_reader_tester src/testers/mcap_reader_tester.cpp)
target_include_directories(mcap_reader_tester
PRIVATE
"${CMAKE_CURRENT_LIST_DIR}/include"
"${CMAKE_CURRENT_BINARY_DIR}")
target_link_libraries(mcap_reader_tester
PRIVATE
CLI11::CLI11
cvmmap_streamer_foxglove_proto
cvmmap_streamer_depth_proto
cvmmap_streamer_mcap_runtime
mcap::mcap
PkgConfig::ZSTD
PkgConfig::LZ4)
if (TARGET spdlog::spdlog)
target_link_libraries(mcap_reader_tester PRIVATE spdlog::spdlog)
elseif (TARGET spdlog)
target_link_libraries(mcap_reader_tester PRIVATE spdlog)
endif()
target_link_libraries(mcap_reader_tester PRIVATE cvmmap_streamer_protobuf)
if (TARGET PkgConfig::PROTOBUF_PKG)
target_link_libraries(mcap_reader_tester PRIVATE PkgConfig::PROTOBUF_PKG)
endif()
set_target_properties(mcap_reader_tester PROPERTIES
OUTPUT_NAME "mcap_reader_tester"
RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/bin")
add_executable(mcap_reader_tester src/testers/mcap_reader_tester.cpp)
target_include_directories(mcap_reader_tester
PRIVATE
"${CMAKE_CURRENT_LIST_DIR}/include"
"${CMAKE_CURRENT_BINARY_DIR}")
target_link_libraries(mcap_reader_tester
PRIVATE
CLI11::CLI11
cvmmap_streamer_feature_flags
cvmmap_streamer_foxglove_proto
cvmmap_streamer_depth_proto
cvmmap_streamer_mcap_runtime
cvmmap_streamer_protobuf)
if (TARGET spdlog::spdlog)
target_link_libraries(mcap_reader_tester PRIVATE spdlog::spdlog)
elseif (TARGET spdlog)
target_link_libraries(mcap_reader_tester PRIVATE spdlog)
endif()
if (TARGET PkgConfig::PROTOBUF_PKG)
target_link_libraries(mcap_reader_tester PRIVATE PkgConfig::PROTOBUF_PKG)
endif()
set_target_properties(mcap_reader_tester PROPERTIES
OUTPUT_NAME "mcap_reader_tester"
RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/bin")
cvmmap_streamer_apply_runtime_rpath(mcap_reader_tester)
add_executable(mcap_replay_tester src/testers/mcap_replay_tester.cpp)
target_include_directories(mcap_replay_tester
PRIVATE
"${CMAKE_CURRENT_LIST_DIR}/include"
"${CMAKE_CURRENT_BINARY_DIR}")
target_link_libraries(mcap_replay_tester
PRIVATE
Threads::Threads
CLI11::CLI11
cvmmap_streamer_foxglove_proto
cvmmap_streamer_mcap_runtime
mcap::mcap
PkgConfig::ZSTD
PkgConfig::LZ4)
if (TARGET spdlog::spdlog)
target_link_libraries(mcap_replay_tester PRIVATE spdlog::spdlog)
elseif (TARGET spdlog)
target_link_libraries(mcap_replay_tester PRIVATE spdlog)
add_executable(mcap_replay_tester src/testers/mcap_replay_tester.cpp)
target_include_directories(mcap_replay_tester
PRIVATE
"${CMAKE_CURRENT_LIST_DIR}/include"
"${CMAKE_CURRENT_BINARY_DIR}")
target_link_libraries(mcap_replay_tester
PRIVATE
Threads::Threads
CLI11::CLI11
cvmmap_streamer_feature_flags
cvmmap_streamer_foxglove_proto
cvmmap_streamer_mcap_runtime
cvmmap_streamer_protobuf)
if (TARGET spdlog::spdlog)
target_link_libraries(mcap_replay_tester PRIVATE spdlog::spdlog)
elseif (TARGET spdlog)
target_link_libraries(mcap_replay_tester PRIVATE spdlog)
endif()
if (TARGET PkgConfig::PROTOBUF_PKG)
target_link_libraries(mcap_replay_tester PRIVATE PkgConfig::PROTOBUF_PKG)
endif()
set_target_properties(mcap_replay_tester PROPERTIES
OUTPUT_NAME "mcap_replay_tester"
RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/bin")
cvmmap_streamer_apply_runtime_rpath(mcap_replay_tester)
endif()
target_link_libraries(mcap_replay_tester PRIVATE cvmmap_streamer_protobuf)
if (TARGET PkgConfig::PROTOBUF_PKG)
target_link_libraries(mcap_replay_tester PRIVATE PkgConfig::PROTOBUF_PKG)
if (CVMMAP_STREAMER_HAS_MCAP_DEPTH)
add_cvmmap_binary(mcap_depth_record_tester src/testers/mcap_depth_record_tester.cpp)
add_cvmmap_binary(mcap_multi_record_tester src/testers/mcap_multi_record_tester.cpp)
endif()
set_target_properties(mcap_replay_tester PROPERTIES
OUTPUT_NAME "mcap_replay_tester"
RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/bin")
set(CVMMAP_STREAMER_INSTALL_TARGETS cvmmap_streamer)
+19 -13
View File
@@ -16,30 +16,36 @@ Legacy flags such as `--shm-name`, `--zmq-endpoint`, `--input-mode`, and the dum
## Encoder Path
### FFmpeg Is The Only Encoder Backend
### Jetson Hardware Encode Uses The Jetson Multimedia API Backend
The public backend surface is:
The public backend surface is still:
- `--encoder-backend auto`
- `--encoder-backend ffmpeg`
Both resolve to the FFmpeg encoder path. The removed GStreamer backend is no longer available.
`--encoder-backend ffmpeg` keeps the existing FFmpeg encoder path.
`--encoder-backend auto` may now select the in-repo Jetson Multimedia API backend on Jetson builds when hardware encoding is requested.
### NVENC Is Optional
The removed GStreamer backend is still unavailable.
When `--encoder-device nvidia` is selected, FFmpeg must expose `h264_nvenc` and `hevc_nvenc`.
### NVIDIA Device Mode On Jetson No Longer Depends On FFmpeg Hardware Encoders
Useful local checks:
On Jetson builds with `CVMMAP_STREAMER_ENABLE_JETSON_MM` enabled, `--encoder-device nvidia` uses the Jetson Multimedia API encoder directly rather than desktop `*_nvenc` or Jetson FFmpeg `*_v4l2m2m` encoder exposure.
Selection behavior is now:
- `--encoder-device software` -> FFmpeg software encode (`libx264`/`libx265`)
- `--encoder-device nvidia` -> Jetson Multimedia API backend only; fail if unavailable
- `--encoder-device auto` -> try Jetson Multimedia API first on Jetson builds, then fall back to FFmpeg software
Useful local checks on Jetson:
```bash
ffmpeg -hide_banner -encoders | rg 'nvenc|libx264|libx265'
test -e /dev/v4l2-nvenc
gst-inspect-1.0 nvv4l2h264enc nvv4l2h265enc
```
If NVENC is unavailable, use:
```bash
--encoder-device software
```
FFmpeg encoder enumeration is no longer the authoritative Jetson hardware-encode check for this repo.
### Low-Latency Defaults
@@ -47,7 +53,7 @@ The current low-latency defaults are:
- `gop=30`
- `b_frames=0`
- NVENC preset/tune tuned for low latency
- encoder-family-specific low-latency options where FFmpeg exposes them
This favors immediacy over compression efficiency.
+2 -2
View File
@@ -27,7 +27,7 @@ Notes:
- RTMP is Enhanced RTMP only.
- The custom RTMP packetizer and domestic mode are removed.
- `encoder.backend` remains `auto|ffmpeg`; both resolve to FFmpeg.
- `encoder.backend` remains `auto|ffmpeg`; `auto` may select Jetson Multimedia API on Jetson builds, while `ffmpeg` forces the FFmpeg encoder path.
## Optional Checks (Non-Blocking)
@@ -60,7 +60,7 @@ Current recording scope:
| Setting | Value |
|---------|-------|
| Encoder backend | `auto` -> FFmpeg |
| Encoder backend | `auto` -> Jetson MM on Jetson hardware requests, else FFmpeg |
| RTMP transport | `libavformat` |
| RTMP mode | Enhanced only |
| Encoder device | `auto` |
@@ -8,6 +8,15 @@
#include <string_view>
#include <vector>
#ifndef CVMMAP_STREAMER_HAS_MCAP
#define CVMMAP_STREAMER_HAS_MCAP 0
#endif
#ifndef CVMMAP_STREAMER_HAS_MCAP_DEPTH
#define CVMMAP_STREAMER_HAS_MCAP_DEPTH 0
#endif
namespace cvmmap_streamer {
enum class CodecType {
@@ -89,6 +98,7 @@ struct OutputsConfig {
struct McapRecordConfig {
bool enabled{false};
bool depth_enabled{true};
std::string path{"capture.mcap"};
std::string topic{"/camera/video"};
std::string depth_topic{"/camera/depth"};
@@ -13,6 +13,15 @@
#include <string>
#include <string_view>
#ifndef CVMMAP_STREAMER_HAS_MCAP
#define CVMMAP_STREAMER_HAS_MCAP 0
#endif
#ifndef CVMMAP_STREAMER_HAS_MCAP_DEPTH
#define CVMMAP_STREAMER_HAS_MCAP_DEPTH 0
#endif
namespace cvmmap_streamer::record {
enum class DepthEncoding {
+2
View File
@@ -0,0 +1,2 @@
#!/bin/sh
exec /usr/bin/protoc --experimental_allow_proto3_optional "$@"
+182 -44
View File
@@ -263,16 +263,27 @@ CLI::Validator validate_rtp_endpoint() {
std::string{});
}
std::optional<std::string_view> match_cli_flag(std::string_view arg, std::string_view flag) {
if (arg == flag) {
return flag;
}
if (arg.size() > flag.size() && arg.rfind(flag, 0) == 0 && arg[flag.size()] == '=') {
return flag;
}
return std::nullopt;
}
std::optional<std::string> find_disallowed_boolean_assignment(int argc, char **argv) {
struct FlagPair {
std::string_view positive;
std::string_view negative;
};
constexpr std::array<FlagPair, 6> kFlagPairs{{
constexpr std::array<FlagPair, 7> kFlagPairs{{
{"--rtmp", "--no-rtmp"},
{"--rtp", "--no-rtp"},
{"--mcap", "--no-mcap"},
{"--mcap-depth", "--no-mcap-depth"},
{"--realtime-sync", "--no-realtime-sync"},
{"--force-idr-on-reset", "--no-force-idr-on-reset"},
{"--keep-stream-on-reset", "--no-keep-stream-on-reset"},
@@ -292,6 +303,81 @@ std::optional<std::string> find_disallowed_boolean_assignment(int argc, char **a
return std::nullopt;
}
std::optional<std::string> find_unsupported_mcap_argument(int argc, char **argv) {
constexpr std::array<std::string_view, 10> kMcapFlags{{
"--mcap",
"--no-mcap",
"--mcap-path",
"--mcap-topic",
"--mcap-calibration-topic",
"--mcap-pose-topic",
"--mcap-body-topic",
"--mcap-frame-id",
"--mcap-compression",
"--mcap-depth-topic",
}};
constexpr std::array<std::string_view, 3> kMcapDepthFlags{{
"--mcap-depth",
"--no-mcap-depth",
"--mcap-depth-calibration-topic",
}};
constexpr bool mcap_supported = CVMMAP_STREAMER_HAS_MCAP != 0;
constexpr bool mcap_depth_supported = CVMMAP_STREAMER_HAS_MCAP_DEPTH != 0;
for (int i = 1; i < argc; ++i) {
const std::string_view arg{argv[i]};
if (!mcap_supported) {
for (const auto flag : kMcapFlags) {
if (match_cli_flag(arg, flag)) {
return "unsupported argument in this build: " + std::string(flag) +
" (MCAP recording support is not compiled in)";
}
}
for (const auto flag : kMcapDepthFlags) {
if (match_cli_flag(arg, flag)) {
return "unsupported argument in this build: " + std::string(flag) +
" (MCAP recording support is not compiled in)";
}
}
continue;
}
if (!mcap_depth_supported) {
for (const auto flag : kMcapDepthFlags) {
if (match_cli_flag(arg, flag)) {
return "unsupported argument in this build: " + std::string(flag) +
" (MCAP depth support is not compiled in)";
}
}
}
}
return std::nullopt;
}
bool runtime_supports_mcap() {
return CVMMAP_STREAMER_HAS_MCAP != 0;
}
bool runtime_supports_mcap_depth() {
return CVMMAP_STREAMER_HAS_MCAP_DEPTH != 0;
}
std::expected<void, std::string> validate_mcap_capability_request(const McapRecordConfig &config) {
if (!config.enabled) {
return {};
}
if (!runtime_supports_mcap()) {
return std::unexpected(
"invalid MCAP config: MCAP recording was requested, but this build was compiled without MCAP support");
}
if (config.depth_enabled && !runtime_supports_mcap_depth()) {
return std::unexpected(
"invalid MCAP config: depth recording was requested, but this build was compiled without MCAP depth support");
}
return {};
}
template <typename T>
std::optional<T> toml_value(const toml::table &table, std::string_view path) {
auto node = table.at_path(path);
@@ -462,6 +548,9 @@ std::expected<void, std::string> apply_toml_file(RuntimeConfig &config, const st
if (auto value = toml_value<bool>(table, "record.mcap.enabled")) {
config.record.mcap.enabled = *value;
}
if (auto value = toml_value<bool>(table, "record.mcap.depth_enabled")) {
config.record.mcap.depth_enabled = *value;
}
if (auto value = toml_value<std::string>(table, "record.mcap.path")) {
config.record.mcap.enabled = true;
config.record.mcap.path = *value;
@@ -710,6 +799,7 @@ std::expected<RuntimeConfig, std::string> parse_runtime_config(int argc, char **
std::optional<std::uint16_t> rtp_payload_type_override{};
std::optional<std::string> rtp_sdp_override{};
std::optional<bool> mcap_enabled_override{};
std::optional<bool> mcap_depth_enabled_override{};
std::optional<std::string> mcap_path_override{};
std::optional<std::string> mcap_topic_override{};
std::optional<std::string> mcap_depth_topic_override{};
@@ -834,6 +924,7 @@ std::expected<RuntimeConfig, std::string> parse_runtime_config(int argc, char **
->check(require_non_empty("--sdp"))
->excludes(rtp_sdp);
#if CVMMAP_STREAMER_HAS_MCAP
app.add_flag("--mcap,!--no-mcap", mcap_enabled_override, "Enable or disable MCAP recording")
->group("MCAP Record")
->default_str(defaults.record.mcap.enabled ? "true" : "false")
@@ -843,49 +934,77 @@ std::expected<RuntimeConfig, std::string> parse_runtime_config(int argc, char **
->type_name("PATH")
->check(require_non_empty("--mcap-path"))
->default_str(defaults.record.mcap.path);
app.add_option("--mcap-topic", mcap_topic_override, "Foxglove compressed video topic name")
app.add_option(
"--mcap-topic",
mcap_topic_override,
"Foxglove compressed video topic name")
->group("MCAP Record")
->type_name("TOPIC")
->check(require_non_empty("--mcap-topic"))
->default_str(defaults.record.mcap.topic);
app.add_option("--mcap-depth-topic", mcap_depth_topic_override, "Depth image topic name")
->group("MCAP Record")
->type_name("TOPIC")
->check(require_non_empty("--mcap-depth-topic"))
->default_str(defaults.record.mcap.depth_topic);
app.add_option("--mcap-calibration-topic", mcap_calibration_topic_override, "RGB camera calibration topic name")
app.add_option(
"--mcap-calibration-topic",
mcap_calibration_topic_override,
"RGB camera calibration topic name")
->group("MCAP Record")
->type_name("TOPIC")
->check(require_non_empty("--mcap-calibration-topic"))
->default_str(defaults.record.mcap.calibration_topic);
app.add_option(
"--mcap-depth-calibration-topic",
mcap_depth_calibration_topic_override,
"Depth camera calibration topic name")
->group("MCAP Record")
->type_name("TOPIC")
->check(require_non_empty("--mcap-depth-calibration-topic"))
->default_str(defaults.record.mcap.depth_calibration_topic);
app.add_option("--mcap-pose-topic", mcap_pose_topic_override, "Pose topic name")
->group("MCAP Record")
->type_name("TOPIC")
->check(require_non_empty("--mcap-pose-topic"))
->default_str(defaults.record.mcap.pose_topic);
app.add_option("--mcap-body-topic", mcap_body_topic_override, "Body tracking topic name")
app.add_option(
"--mcap-body-topic",
mcap_body_topic_override,
"Body tracking topic name")
->group("MCAP Record")
->type_name("TOPIC")
->check(require_non_empty("--mcap-body-topic"))
->default_str(defaults.record.mcap.body_topic);
app.add_option("--mcap-frame-id", mcap_frame_id_override, "Frame ID written into MCAP messages")
app.add_option(
"--mcap-frame-id",
mcap_frame_id_override,
"Frame ID written into MCAP messages")
->group("MCAP Record")
->type_name("ID")
->check(require_non_empty("--mcap-frame-id"))
->default_str(defaults.record.mcap.frame_id);
app.add_option("--mcap-compression", mcap_compression_override, "MCAP chunk compression mode")
app.add_option(
"--mcap-compression",
mcap_compression_override,
"MCAP chunk compression mode")
->group("MCAP Record")
->type_name("MODE")
->transform(canonicalize_option(canonicalize_mcap_compression))
->default_str(std::string(to_string(defaults.record.mcap.compression)));
#if CVMMAP_STREAMER_HAS_MCAP_DEPTH
app.add_flag(
"--mcap-depth,!--no-mcap-depth",
mcap_depth_enabled_override,
"Enable or disable MCAP depth recording")
->group("MCAP Depth Record")
->default_str(defaults.record.mcap.depth_enabled ? "true" : "false")
->disable_flag_override();
app.add_option(
"--mcap-depth-topic",
mcap_depth_topic_override,
"Depth image topic name")
->group("MCAP Depth Record")
->type_name("TOPIC")
->check(require_non_empty("--mcap-depth-topic"))
->default_str(defaults.record.mcap.depth_topic);
app.add_option(
"--mcap-depth-calibration-topic",
mcap_depth_calibration_topic_override,
"Depth camera calibration topic name")
->group("MCAP Depth Record")
->type_name("TOPIC")
->check(require_non_empty("--mcap-depth-calibration-topic"))
->default_str(defaults.record.mcap.depth_calibration_topic);
#endif
#endif
app.add_option("--queue-size", queue_size_override, "Pipeline queue depth")
->group("Latency")
@@ -939,6 +1058,10 @@ std::expected<RuntimeConfig, std::string> parse_runtime_config(int argc, char **
->check(CLI::NonNegativeNumber)
->default_str(std::to_string(defaults.latency.emit_stall_ms));
if (auto unsupported_mcap_argument = find_unsupported_mcap_argument(argc, argv)) {
return std::unexpected(*unsupported_mcap_argument);
}
if (auto invalid_boolean_assignment = find_disallowed_boolean_assignment(argc, argv)) {
return std::unexpected(*invalid_boolean_assignment);
}
@@ -1078,6 +1201,9 @@ std::expected<RuntimeConfig, std::string> parse_runtime_config(int argc, char **
if (mcap_enabled_override) {
config.record.mcap.enabled = *mcap_enabled_override;
}
if (mcap_depth_enabled_override) {
config.record.mcap.depth_enabled = *mcap_depth_enabled_override;
}
if (queue_size_override) {
config.latency.queue_size = *queue_size_override;
@@ -1135,7 +1261,7 @@ std::expected<void, std::string> validate_runtime_config(const RuntimeConfig &co
}
if (config.outputs.rtmp.enabled) {
if (config.encoder.backend == EncoderBackendType::Auto) {
// auto resolves to FFmpeg; nothing else is supported.
// auto may select the Jetson Multimedia API backend on Jetson before falling back to FFmpeg software.
} else if (config.encoder.backend != EncoderBackendType::FFmpeg) {
return std::unexpected("invalid backend/output matrix: RTMP requires encoder.backend=ffmpeg or auto");
}
@@ -1157,30 +1283,41 @@ std::expected<void, std::string> validate_runtime_config(const RuntimeConfig &co
}
}
if (config.record.mcap.enabled && config.record.mcap.path.empty()) {
return std::unexpected("invalid MCAP config: enabled MCAP output requires path");
}
if (config.record.mcap.topic.empty()) {
return std::unexpected("invalid MCAP config: topic must not be empty");
}
if (config.record.mcap.depth_topic.empty()) {
return std::unexpected("invalid MCAP config: depth_topic must not be empty");
}
if (config.record.mcap.calibration_topic.empty()) {
return std::unexpected("invalid MCAP config: calibration_topic must not be empty");
}
if (!config.record.mcap.depth_calibration_topic.empty() &&
config.record.mcap.depth_calibration_topic == config.record.mcap.calibration_topic) {
return std::unexpected("invalid MCAP config: depth_calibration_topic must differ from calibration_topic");
}
if (config.record.mcap.pose_topic.empty()) {
return std::unexpected("invalid MCAP config: pose_topic must not be empty");
}
if (config.record.mcap.body_topic.empty()) {
return std::unexpected("invalid MCAP config: body_topic must not be empty");
}
if (config.record.mcap.frame_id.empty()) {
return std::unexpected("invalid MCAP config: frame_id must not be empty");
if (config.record.mcap.enabled) {
if (auto capability_validation = validate_mcap_capability_request(config.record.mcap); !capability_validation) {
return std::unexpected(capability_validation.error());
}
if (config.record.mcap.path.empty()) {
return std::unexpected("invalid MCAP config: enabled MCAP output requires path");
}
if (config.record.mcap.topic.empty()) {
return std::unexpected("invalid MCAP config: topic must not be empty");
}
if (config.record.mcap.calibration_topic.empty()) {
return std::unexpected("invalid MCAP config: calibration_topic must not be empty");
}
if (config.record.mcap.pose_topic.empty()) {
return std::unexpected("invalid MCAP config: pose_topic must not be empty");
}
if (config.record.mcap.body_topic.empty()) {
return std::unexpected("invalid MCAP config: body_topic must not be empty");
}
if (config.record.mcap.frame_id.empty()) {
return std::unexpected("invalid MCAP config: frame_id must not be empty");
}
if (config.record.mcap.depth_enabled) {
if (config.record.mcap.depth_topic.empty()) {
return std::unexpected("invalid MCAP config: depth_topic must not be empty when depth recording is enabled");
}
if (config.record.mcap.depth_calibration_topic.empty()) {
return std::unexpected(
"invalid MCAP config: depth_calibration_topic must not be empty when depth recording is enabled");
}
if (config.record.mcap.depth_calibration_topic == config.record.mcap.calibration_topic) {
return std::unexpected(
"invalid MCAP config: depth_calibration_topic must differ from calibration_topic when depth recording is enabled");
}
}
}
if (config.latency.queue_size == 0) {
@@ -1213,6 +1350,7 @@ std::string summarize_runtime_config(const RuntimeConfig &config) {
ss << ", rtp.endpoint=" << (config.outputs.rtp.endpoint ? *config.outputs.rtp.endpoint : "<unset>");
ss << ", rtp.payload_type=" << static_cast<unsigned>(config.outputs.rtp.payload_type);
ss << ", mcap.enabled=" << (config.record.mcap.enabled ? "true" : "false");
ss << ", mcap.depth_enabled=" << (config.record.mcap.depth_enabled ? "true" : "false");
ss << ", mcap.path=" << config.record.mcap.path;
ss << ", mcap.topic=" << config.record.mcap.topic;
ss << ", mcap.depth_topic=" << config.record.mcap.depth_topic;
+125 -1
View File
@@ -1,14 +1,138 @@
#include "cvmmap_streamer/encode/encoder_backend.hpp"
#ifndef CVMMAP_STREAMER_HAS_JETSON_MM
#define CVMMAP_STREAMER_HAS_JETSON_MM 0
#endif
#include <optional>
#include <spdlog/spdlog.h>
namespace cvmmap_streamer::encode {
EncoderBackend make_ffmpeg_backend();
#if CVMMAP_STREAMER_HAS_JETSON_MM
EncoderBackend make_jetson_mm_backend();
#endif
namespace {
class SelectingEncoderBackend {
public:
[[nodiscard]]
std::string_view backend_name() const {
return active_backend_ ? (*active_backend_)->backend_name() : std::string_view{"selecting"};
}
[[nodiscard]]
bool using_hardware() const {
return active_backend_ ? (*active_backend_)->using_hardware() : false;
}
[[nodiscard]]
Status init(const RuntimeConfig &config, const ipc::FrameInfo &frame_info) {
shutdown();
#if CVMMAP_STREAMER_HAS_JETSON_MM
if (config.encoder.device != EncoderDeviceType::Software) {
auto jetson_backend = make_jetson_mm_backend();
auto jetson_init = jetson_backend->init(config, frame_info);
if (jetson_init) {
active_backend_.emplace(std::move(jetson_backend));
return {};
}
if (config.encoder.device == EncoderDeviceType::Nvidia) {
return std::unexpected(jetson_init.error());
}
spdlog::warn(
"JETSON_MM_ENCODER_FALLBACK reason='{}' fallback=ffmpeg_software",
format_error(jetson_init.error()));
}
if (config.encoder.device == EncoderDeviceType::Nvidia) {
return unexpected_error(
ERR_BACKEND_UNAVAILABLE,
"Jetson hardware encoder backend is unavailable in this build");
}
#else
if (config.encoder.device == EncoderDeviceType::Nvidia) {
return unexpected_error(
ERR_BACKEND_UNAVAILABLE,
"Jetson hardware encoder backend is unavailable in this build");
}
#endif
auto ffmpeg_backend = make_ffmpeg_backend();
auto ffmpeg_config = config;
if (config.encoder.device == EncoderDeviceType::Auto) {
ffmpeg_config.encoder.device = EncoderDeviceType::Software;
}
auto ffmpeg_init = ffmpeg_backend->init(ffmpeg_config, frame_info);
if (!ffmpeg_init) {
return std::unexpected(ffmpeg_init.error());
}
active_backend_.emplace(std::move(ffmpeg_backend));
return {};
}
[[nodiscard]]
Result<EncodedStreamInfo> stream_info() const {
if (!active_backend_) {
return unexpected_error(ERR_NOT_READY, "encoder backend is unavailable before initialization");
}
return (*active_backend_)->stream_info();
}
[[nodiscard]]
Status poll() {
if (!active_backend_) {
return {};
}
return (*active_backend_)->poll();
}
[[nodiscard]]
Status push_frame(const RawVideoFrame &frame) {
if (!active_backend_) {
return unexpected_error(ERR_NOT_READY, "encoder backend not initialized");
}
return (*active_backend_)->push_frame(frame);
}
[[nodiscard]]
Result<std::vector<EncodedAccessUnit>> drain() {
if (!active_backend_) {
return std::vector<EncodedAccessUnit>{};
}
return (*active_backend_)->drain();
}
[[nodiscard]]
Result<std::vector<EncodedAccessUnit>> flush() {
if (!active_backend_) {
return std::vector<EncodedAccessUnit>{};
}
return (*active_backend_)->flush();
}
void shutdown() {
if (active_backend_) {
(*active_backend_)->shutdown();
active_backend_.reset();
}
}
private:
std::optional<EncoderBackend> active_backend_{};
};
} // namespace
Result<EncoderBackend> make_encoder_backend(const RuntimeConfig &config) {
switch (config.encoder.backend) {
case EncoderBackendType::FFmpeg:
case EncoderBackendType::Auto:
return make_ffmpeg_backend();
case EncoderBackendType::Auto:
return pro::make_proxy<EncoderBackendFacade, SelectingEncoderBackend>();
}
return unexpected_error(ERR_INTERNAL, "unknown encoder backend");
+136 -94
View File
@@ -1,4 +1,5 @@
#include "cvmmap_streamer/encode/encoder_backend.hpp"
#include "ffmpeg_encoder_options.hpp"
extern "C" {
#include <libavcodec/avcodec.h>
@@ -52,10 +53,9 @@ public:
Status init(const RuntimeConfig &config, const ipc::FrameInfo &frame_info) {
shutdown();
config_ = &config;
frame_info_ = frame_info;
codec_ = config.encoder.codec;
encoder_pix_fmt_ = pick_encoder_pixel_format(config.encoder.device);
config_ = &config;
frame_info_ = frame_info;
codec_ = config.encoder.codec;
auto input_pixel_format = to_av_pixel_format(frame_info.pixel_format);
if (!input_pixel_format) {
@@ -63,45 +63,15 @@ public:
}
input_pix_fmt_ = *input_pixel_format;
auto encoder_name = pick_encoder_name(config);
if (!encoder_name) {
return std::unexpected(encoder_name.error());
}
using_hardware_ = encoder_name->find("nvenc") != std::string::npos;
const auto *encoder = avcodec_find_encoder_by_name(encoder_name->c_str());
if (encoder == nullptr) {
return unexpected_error(ERR_BACKEND_UNAVAILABLE, "FFmpeg encoder '" + *encoder_name + "' is unavailable");
auto opened_encoder = open_encoder(config);
if (!opened_encoder) {
return std::unexpected(opened_encoder.error());
}
context_ = avcodec_alloc_context3(encoder);
if (context_ == nullptr) {
return unexpected_error(ERR_ALLOCATION_FAILED, "failed to allocate FFmpeg encoder context");
}
context_->codec_type = AVMEDIA_TYPE_VIDEO;
context_->codec_id = encoder->id;
context_->width = static_cast<int>(frame_info.width);
context_->height = static_cast<int>(frame_info.height);
context_->pix_fmt = encoder_pix_fmt_;
context_->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
context_->time_base = AVRational{1, 1000000000};
context_->framerate = AVRational{30, 1};
context_->gop_size = static_cast<int>(config.encoder.gop);
context_->max_b_frames = static_cast<int>(config.encoder.b_frames);
context_->thread_count = 1;
auto codec_setup = configure_codec(*encoder_name, config);
if (!codec_setup) {
return codec_setup;
}
const auto open_result = avcodec_open2(context_, encoder, nullptr);
if (open_result < 0) {
return unexpected_error(
ERR_ENCODER,
"failed to open FFmpeg encoder '" + *encoder_name + "': " + av_error_string(open_result));
}
context_ = opened_encoder->context;
encoder_name_ = std::string(opened_encoder->candidate.name);
encoder_pix_fmt_ = opened_encoder->candidate.pixel_format;
using_hardware_ = opened_encoder->candidate.using_hardware;
scaler_ = sws_getCachedContext(
nullptr,
@@ -151,10 +121,11 @@ public:
stream_info_ = build_stream_info();
spdlog::info(
"FFMPEG_ENCODER_PATH codec={} device={} encoder={} pix_fmt={}",
"FFMPEG_ENCODER_PATH codec={} device={} encoder={} hardware={} pix_fmt={}",
cvmmap_streamer::to_string(codec_),
device_to_string(config.encoder.device),
*encoder_name,
encoder_name_,
using_hardware_,
av_get_pix_fmt_name(encoder_pix_fmt_));
return {};
}
@@ -218,11 +189,15 @@ public:
}
frame_->pict_type = frame.force_keyframe ? AV_PICTURE_TYPE_I : AV_PICTURE_TYPE_NONE;
#if defined(AV_FRAME_FLAG_KEY)
if (frame.force_keyframe) {
frame_->flags |= AV_FRAME_FLAG_KEY;
} else {
frame_->flags &= ~AV_FRAME_FLAG_KEY;
}
#else
frame_->key_frame = frame.force_keyframe ? 1 : 0;
#endif
frame_->pts = static_cast<std::int64_t>(frame.source_timestamp_ns - *first_source_timestamp_ns_);
const auto send_result = avcodec_send_frame(context_, frame_);
if (send_result < 0) {
@@ -270,6 +245,7 @@ public:
}
first_source_timestamp_ns_.reset();
stream_info_.reset();
encoder_name_.clear();
using_hardware_ = false;
}
@@ -294,14 +270,6 @@ private:
}
}
[[nodiscard]]
static AVPixelFormat pick_encoder_pixel_format(EncoderDeviceType device) {
if (device == EncoderDeviceType::Software) {
return AV_PIX_FMT_YUV420P;
}
return AV_PIX_FMT_NV12;
}
[[nodiscard]]
static std::string_view device_to_string(EncoderDeviceType device) {
switch (device) {
@@ -316,57 +284,130 @@ private:
}
[[nodiscard]]
Result<std::string> pick_encoder_name(const RuntimeConfig &config) const {
const bool prefer_hardware = config.encoder.device != EncoderDeviceType::Software;
const bool prefer_software = config.encoder.device == EncoderDeviceType::Software;
if (codec_ == CodecType::H265) {
if (prefer_hardware && avcodec_find_encoder_by_name("hevc_nvenc") != nullptr) {
return std::string("hevc_nvenc");
}
if (!prefer_hardware || config.encoder.device == EncoderDeviceType::Auto) {
if (avcodec_find_encoder_by_name("libx265") != nullptr) {
return std::string("libx265");
}
}
if (!prefer_software && avcodec_find_encoder_by_name("hevc_nvenc") != nullptr) {
return std::string("hevc_nvenc");
}
static Result<void> set_string_option(AVCodecContext *context, const char *key, std::string_view value) {
const auto result = av_opt_set(context->priv_data, key, std::string(value).c_str(), 0);
if (result < 0) {
return unexpected_error(
ERR_BACKEND_UNAVAILABLE,
"no usable FFmpeg encoder found for h265 (looked for hevc_nvenc, libx265)");
ERR_ENCODER,
"failed to set FFmpeg encoder option '" + std::string(key) + "=" + std::string(value) + "': " + av_error_string(result));
}
if (prefer_hardware && avcodec_find_encoder_by_name("h264_nvenc") != nullptr) {
return std::string("h264_nvenc");
}
if (!prefer_hardware || config.encoder.device == EncoderDeviceType::Auto) {
if (avcodec_find_encoder_by_name("libx264") != nullptr) {
return std::string("libx264");
}
}
if (!prefer_software && avcodec_find_encoder_by_name("h264_nvenc") != nullptr) {
return std::string("h264_nvenc");
}
return unexpected_error(
ERR_BACKEND_UNAVAILABLE,
"no usable FFmpeg encoder found for h264 (looked for h264_nvenc, libx264)");
return {};
}
[[nodiscard]]
Status configure_codec(std::string_view encoder_name, const RuntimeConfig &config) {
av_opt_set(context_->priv_data, "preset", encoder_name.find("nvenc") != std::string_view::npos ? "p1" : "veryfast", 0);
if (encoder_name.find("nvenc") != std::string_view::npos) {
av_opt_set(context_->priv_data, "tune", "ull", 0);
av_opt_set(context_->priv_data, "zerolatency", "1", 0);
av_opt_set(context_->priv_data, "rc-lookahead", "0", 0);
} else {
av_opt_set(context_->priv_data, "tune", "zerolatency", 0);
if (encoder_name == "libx265") {
av_opt_set(context_->priv_data, "x265-params", "repeat-headers=1:scenecut=0", 0);
static Result<void> set_int_option(AVCodecContext *context, const char *key, std::int64_t value) {
const auto result = av_opt_set_int(context->priv_data, key, value, 0);
if (result < 0) {
return unexpected_error(
ERR_ENCODER,
"failed to set FFmpeg encoder option '" + std::string(key) + "=" + std::to_string(value) + "': " + av_error_string(result));
}
return {};
}
struct OpenedEncoder {
AVCodecContext *context{nullptr};
FfmpegEncoderCandidate candidate{};
};
[[nodiscard]]
Result<OpenedEncoder> open_encoder(const RuntimeConfig &config) const {
const auto candidates = ffmpeg_encoder_candidates(codec_, config.encoder.device);
const auto attempted_candidates = ffmpeg_encoder_candidate_list(candidates);
std::string last_error{};
for (const auto &candidate : candidates) {
const auto *encoder = avcodec_find_encoder_by_name(candidate.name.data());
if (encoder == nullptr) {
last_error = "FFmpeg encoder '" + std::string(candidate.name) + "' is unavailable";
spdlog::warn(
"FFmpeg encoder '{}' unavailable in {} mode, trying next candidate",
candidate.name,
device_to_string(config.encoder.device));
continue;
}
auto *context = avcodec_alloc_context3(encoder);
if (context == nullptr) {
return unexpected_error(ERR_ALLOCATION_FAILED, "failed to allocate FFmpeg encoder context");
}
context->codec_type = AVMEDIA_TYPE_VIDEO;
context->codec_id = encoder->id;
context->width = static_cast<int>(frame_info_.width);
context->height = static_cast<int>(frame_info_.height);
context->pix_fmt = candidate.pixel_format;
context->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
context->time_base = AVRational{1, 1000000000};
context->framerate = AVRational{30, 1};
context->gop_size = static_cast<int>(config.encoder.gop);
context->max_b_frames = static_cast<int>(config.encoder.b_frames);
context->thread_count = 1;
auto codec_setup = configure_codec(context, candidate, config);
if (!codec_setup) {
last_error = codec_setup.error().detail;
avcodec_free_context(&context);
spdlog::warn(
"FFmpeg encoder '{}' configuration failed in {} mode: {}. trying next candidate",
candidate.name,
device_to_string(config.encoder.device),
codec_setup.error().detail);
continue;
}
const auto open_result = avcodec_open2(context, encoder, nullptr);
if (open_result < 0) {
last_error = "failed to open FFmpeg encoder '" + std::string(candidate.name) + "': " + av_error_string(open_result);
avcodec_free_context(&context);
spdlog::warn(
"FFmpeg encoder '{}' failed to open in {} mode: {}. trying next candidate",
candidate.name,
device_to_string(config.encoder.device),
av_error_string(open_result));
continue;
}
return OpenedEncoder{.context = context, .candidate = candidate};
}
av_opt_set_int(context_->priv_data, "forced-idr", config.latency.force_idr_on_reset ? 1 : 0, 0);
if (last_error.empty()) {
last_error = "no usable FFmpeg encoder found";
}
const auto error_code = config.encoder.device == EncoderDeviceType::Auto ? ERR_ENCODER : ERR_BACKEND_UNAVAILABLE;
return unexpected_error(error_code, last_error + " (attempted: " + attempted_candidates + ")");
}
[[nodiscard]]
static Status configure_codec(AVCodecContext *context, const FfmpegEncoderCandidate &candidate, const RuntimeConfig &config) {
if (const auto preset = ffmpeg_encoder_preset(candidate); preset) {
if (auto set = set_string_option(context, "preset", *preset); !set) {
return set;
}
}
if (const auto tune = ffmpeg_encoder_tune(candidate); tune) {
if (auto set = set_string_option(context, "tune", *tune); !set) {
return set;
}
}
if (const auto x265_params = ffmpeg_encoder_x265_params(candidate); x265_params) {
if (auto set = set_string_option(context, "x265-params", *x265_params); !set) {
return set;
}
}
if (ffmpeg_encoder_supports_nvenc_latency_flags(candidate)) {
if (auto set = set_string_option(context, "zerolatency", "1"); !set) {
return set;
}
if (auto set = set_string_option(context, "rc-lookahead", "0"); !set) {
return set;
}
}
if (ffmpeg_encoder_supports_forced_idr_option(candidate)) {
if (auto set = set_int_option(context, "forced-idr", config.latency.force_idr_on_reset ? 1 : 0); !set) {
return set;
}
}
return {};
}
@@ -478,6 +519,7 @@ private:
AVPixelFormat encoder_pix_fmt_{AV_PIX_FMT_NONE};
std::optional<std::uint64_t> first_source_timestamp_ns_{};
std::optional<EncodedStreamInfo> stream_info_{};
std::string encoder_name_{};
bool using_hardware_{false};
};
+197
View File
@@ -0,0 +1,197 @@
#pragma once
#include "cvmmap_streamer/config/runtime_config.hpp"
extern "C" {
#include <libavutil/pixfmt.h>
}
#include <optional>
#include <string>
#include <string_view>
#include <vector>
namespace cvmmap_streamer::encode {
enum class FfmpegEncoderFamily {
Nvenc,
V4l2M2m,
Omx,
LibX264,
LibX265,
};
struct FfmpegEncoderCandidate {
std::string_view name{};
FfmpegEncoderFamily family{FfmpegEncoderFamily::LibX264};
bool using_hardware{false};
AVPixelFormat pixel_format{AV_PIX_FMT_NONE};
};
[[nodiscard]]
inline std::vector<FfmpegEncoderCandidate> ffmpeg_encoder_candidates(const CodecType codec, const EncoderDeviceType device) {
std::vector<FfmpegEncoderCandidate> candidates{};
auto append_hardware_candidates = [&] {
if (codec == CodecType::H265) {
candidates.push_back(FfmpegEncoderCandidate{
.name = "hevc_nvenc",
.family = FfmpegEncoderFamily::Nvenc,
.using_hardware = true,
.pixel_format = AV_PIX_FMT_NV12,
});
candidates.push_back(FfmpegEncoderCandidate{
.name = "hevc_v4l2m2m",
.family = FfmpegEncoderFamily::V4l2M2m,
.using_hardware = true,
.pixel_format = AV_PIX_FMT_NV12,
});
return;
}
candidates.push_back(FfmpegEncoderCandidate{
.name = "h264_nvenc",
.family = FfmpegEncoderFamily::Nvenc,
.using_hardware = true,
.pixel_format = AV_PIX_FMT_NV12,
});
candidates.push_back(FfmpegEncoderCandidate{
.name = "h264_v4l2m2m",
.family = FfmpegEncoderFamily::V4l2M2m,
.using_hardware = true,
.pixel_format = AV_PIX_FMT_NV12,
});
candidates.push_back(FfmpegEncoderCandidate{
.name = "h264_omx",
.family = FfmpegEncoderFamily::Omx,
.using_hardware = true,
.pixel_format = AV_PIX_FMT_NV12,
});
};
const auto append_software_candidate = [&] {
if (codec == CodecType::H265) {
candidates.push_back(FfmpegEncoderCandidate{
.name = "libx265",
.family = FfmpegEncoderFamily::LibX265,
.using_hardware = false,
.pixel_format = AV_PIX_FMT_YUV420P,
});
return;
}
candidates.push_back(FfmpegEncoderCandidate{
.name = "libx264",
.family = FfmpegEncoderFamily::LibX264,
.using_hardware = false,
.pixel_format = AV_PIX_FMT_YUV420P,
});
};
switch (device) {
case EncoderDeviceType::Auto:
append_hardware_candidates();
append_software_candidate();
break;
case EncoderDeviceType::Nvidia:
append_hardware_candidates();
break;
case EncoderDeviceType::Software:
append_software_candidate();
break;
}
return candidates;
}
[[nodiscard]]
inline std::string ffmpeg_encoder_candidate_list(const std::vector<FfmpegEncoderCandidate> &candidates) {
std::string joined{};
for (const auto &candidate : candidates) {
if (!joined.empty()) {
joined += ", ";
}
joined += candidate.name;
}
return joined;
}
[[nodiscard]]
inline std::optional<std::string_view> ffmpeg_encoder_preset(const FfmpegEncoderCandidate &candidate) {
switch (candidate.family) {
case FfmpegEncoderFamily::Nvenc:
return "p1";
case FfmpegEncoderFamily::LibX264:
case FfmpegEncoderFamily::LibX265:
return "veryfast";
case FfmpegEncoderFamily::V4l2M2m:
case FfmpegEncoderFamily::Omx:
return std::nullopt;
}
return std::nullopt;
}
[[nodiscard]]
inline std::optional<std::string_view> ffmpeg_encoder_tune(const FfmpegEncoderCandidate &candidate) {
switch (candidate.family) {
case FfmpegEncoderFamily::Nvenc:
return "ull";
case FfmpegEncoderFamily::LibX264:
return "zerolatency";
case FfmpegEncoderFamily::V4l2M2m:
case FfmpegEncoderFamily::Omx:
case FfmpegEncoderFamily::LibX265:
return std::nullopt;
}
return std::nullopt;
}
[[nodiscard]]
inline std::optional<std::string_view> ffmpeg_encoder_rate_control_mode(const FfmpegEncoderCandidate &candidate) {
if (candidate.family == FfmpegEncoderFamily::Nvenc) {
return "vbr";
}
return std::nullopt;
}
[[nodiscard]]
inline std::optional<std::string_view> ffmpeg_encoder_quality_key(const FfmpegEncoderCandidate &candidate) {
switch (candidate.family) {
case FfmpegEncoderFamily::Nvenc:
return "cq";
case FfmpegEncoderFamily::LibX264:
case FfmpegEncoderFamily::LibX265:
return "crf";
case FfmpegEncoderFamily::V4l2M2m:
case FfmpegEncoderFamily::Omx:
return std::nullopt;
}
return std::nullopt;
}
[[nodiscard]]
inline std::optional<std::string_view> ffmpeg_encoder_x265_params(const FfmpegEncoderCandidate &candidate) {
if (candidate.family == FfmpegEncoderFamily::LibX265) {
return "repeat-headers=1:scenecut=0";
}
return std::nullopt;
}
[[nodiscard]]
inline bool ffmpeg_encoder_supports_nvenc_latency_flags(const FfmpegEncoderCandidate &candidate) {
return candidate.family == FfmpegEncoderFamily::Nvenc;
}
[[nodiscard]]
inline bool ffmpeg_encoder_supports_forced_idr_option(const FfmpegEncoderCandidate &candidate) {
switch (candidate.family) {
case FfmpegEncoderFamily::Nvenc:
case FfmpegEncoderFamily::LibX264:
case FfmpegEncoderFamily::LibX265:
return true;
case FfmpegEncoderFamily::V4l2M2m:
case FfmpegEncoderFamily::Omx:
return false;
}
return false;
}
} // namespace cvmmap_streamer::encode
+750
View File
@@ -0,0 +1,750 @@
#include "cvmmap_streamer/encode/encoder_backend.hpp"
extern "C" {
#include <libavutil/frame.h>
#include <libavutil/imgutils.h>
#include <libavutil/pixfmt.h>
#include <libswscale/swscale.h>
}
#include <NvBuffer.h>
#include <NvVideoEncoder.h>
#include <nvbufsurface.h>
#include <linux/videodev2.h>
#include <array>
#include <chrono>
#include <condition_variable>
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <mutex>
#include <optional>
#include <string>
#include <string_view>
#include <sys/time.h>
#include <unordered_map>
#include <utility>
#include <vector>
#include <spdlog/spdlog.h>
namespace cvmmap_streamer::encode {
namespace {
constexpr std::uint32_t kEncoderBufferCount = 6;
constexpr std::uint32_t kCapturePlaneSizeImage = 2u * 1024u * 1024u;
constexpr std::uint32_t kDefaultFrameRateNum = 30;
constexpr std::uint32_t kDefaultFrameRateDen = 1;
constexpr auto kWarmupTimeout = std::chrono::seconds(2);
constexpr auto kFlushTimeout = std::chrono::seconds(2);
constexpr std::uint8_t kAnnexBStartCode[4]{0x00, 0x00, 0x00, 0x01};
[[nodiscard]]
Result<AVPixelFormat> to_av_pixel_format(ipc::PixelFormat format) {
switch (format) {
case ipc::PixelFormat::BGR:
return AV_PIX_FMT_BGR24;
case ipc::PixelFormat::RGB:
return AV_PIX_FMT_RGB24;
case ipc::PixelFormat::BGRA:
return AV_PIX_FMT_BGRA;
case ipc::PixelFormat::RGBA:
return AV_PIX_FMT_RGBA;
case ipc::PixelFormat::GRAY:
return AV_PIX_FMT_GRAY8;
default:
return unexpected_error(
ERR_UNSUPPORTED,
"unsupported raw pixel format for Jetson backend (supported: BGR/RGB/BGRA/RGBA/GRAY)");
}
}
[[nodiscard]]
std::uint32_t codec_capture_plane_format(CodecType codec) {
return codec == CodecType::H265 ? V4L2_PIX_FMT_H265 : V4L2_PIX_FMT_H264;
}
[[nodiscard]]
std::uint32_t default_bitrate_bits_per_second(const ipc::FrameInfo &frame_info) {
const auto pixels_per_second =
static_cast<std::uint64_t>(frame_info.width) *
static_cast<std::uint64_t>(frame_info.height) *
static_cast<std::uint64_t>(kDefaultFrameRateNum);
const auto estimated = pixels_per_second / 8u;
return static_cast<std::uint32_t>(std::clamp<std::uint64_t>(estimated, 2'000'000u, 25'000'000u));
}
[[nodiscard]]
std::uint64_t timeval_to_token_us(const timeval &timestamp) {
return static_cast<std::uint64_t>(timestamp.tv_sec) * 1'000'000ull +
static_cast<std::uint64_t>(timestamp.tv_usec);
}
[[nodiscard]]
timeval token_us_to_timeval(const std::uint64_t token_us) {
timeval timestamp{};
timestamp.tv_sec = static_cast<time_t>(token_us / 1'000'000ull);
timestamp.tv_usec = static_cast<suseconds_t>(token_us % 1'000'000ull);
return timestamp;
}
[[nodiscard]]
std::optional<std::pair<std::size_t, std::size_t>> next_start_code(std::span<const std::uint8_t> bytes, std::size_t offset) {
for (std::size_t i = offset; i + 3 <= bytes.size(); ++i) {
if (bytes[i] == 0x00 && bytes[i + 1] == 0x00 && bytes[i + 2] == 0x01) {
return std::pair{i, static_cast<std::size_t>(3)};
}
if (i + 4 <= bytes.size() &&
bytes[i] == 0x00 &&
bytes[i + 1] == 0x00 &&
bytes[i + 2] == 0x00 &&
bytes[i + 3] == 0x01) {
return std::pair{i, static_cast<std::size_t>(4)};
}
}
return std::nullopt;
}
[[nodiscard]]
std::vector<std::span<const std::uint8_t>> split_annexb_nalus(std::span<const std::uint8_t> access_unit) {
std::vector<std::span<const std::uint8_t>> nalus{};
for (std::size_t cursor = 0;;) {
auto current_sc = next_start_code(access_unit, cursor);
if (!current_sc) {
break;
}
const auto payload_begin = current_sc->first + current_sc->second;
auto next_sc = next_start_code(access_unit, payload_begin);
const auto payload_end = next_sc ? next_sc->first : access_unit.size();
if (payload_begin < payload_end) {
nalus.push_back(access_unit.subspan(payload_begin, payload_end - payload_begin));
}
if (!next_sc) {
break;
}
cursor = next_sc->first;
}
return nalus;
}
[[nodiscard]]
bool is_parameter_set_nalu(const CodecType codec, std::span<const std::uint8_t> nalu) {
if (nalu.empty()) {
return false;
}
if (codec == CodecType::H265) {
if (nalu.size() < 2) {
return false;
}
const auto nal_type = static_cast<std::uint8_t>((nalu[0] >> 1) & 0x3fu);
return nal_type == 32 || nal_type == 33 || nal_type == 34;
}
const auto nal_type = static_cast<std::uint8_t>(nalu[0] & 0x1fu);
return nal_type == 7 || nal_type == 8;
}
[[nodiscard]]
std::vector<std::uint8_t> extract_decoder_config_annexb(const CodecType codec, std::span<const std::uint8_t> access_unit) {
std::vector<std::uint8_t> decoder_config{};
for (const auto nalu : split_annexb_nalus(access_unit)) {
if (!is_parameter_set_nalu(codec, nalu)) {
continue;
}
decoder_config.insert(decoder_config.end(), std::begin(kAnnexBStartCode), std::end(kAnnexBStartCode));
decoder_config.insert(decoder_config.end(), nalu.begin(), nalu.end());
}
return decoder_config;
}
class JetsonMmEncoderBackend {
public:
~JetsonMmEncoderBackend() {
shutdown();
}
[[nodiscard]]
std::string_view backend_name() const {
return "jetson_mm";
}
[[nodiscard]]
bool using_hardware() const {
return true;
}
[[nodiscard]]
Status init(const RuntimeConfig &config, const ipc::FrameInfo &frame_info) {
shutdown();
frame_info_ = frame_info;
codec_ = config.encoder.codec;
config_ = &config;
auto input_pixel_format = to_av_pixel_format(frame_info.pixel_format);
if (!input_pixel_format) {
return std::unexpected(input_pixel_format.error());
}
input_pix_fmt_ = *input_pixel_format;
scaler_ = sws_getCachedContext(
nullptr,
static_cast<int>(frame_info.width),
static_cast<int>(frame_info.height),
input_pix_fmt_,
static_cast<int>(frame_info.width),
static_cast<int>(frame_info.height),
AV_PIX_FMT_YUV420P,
SWS_BILINEAR,
nullptr,
nullptr,
nullptr);
if (scaler_ == nullptr) {
return unexpected_error(ERR_EXTERNAL_LIBRARY, "failed to create Jetson swscale conversion context");
}
converted_frame_ = av_frame_alloc();
if (converted_frame_ == nullptr) {
return unexpected_error(ERR_ALLOCATION_FAILED, "failed to allocate Jetson conversion frame");
}
converted_frame_->format = AV_PIX_FMT_YUV420P;
converted_frame_->width = static_cast<int>(frame_info.width);
converted_frame_->height = static_cast<int>(frame_info.height);
const auto frame_buffer = av_frame_get_buffer(converted_frame_, 32);
if (frame_buffer < 0) {
return unexpected_error(ERR_ALLOCATION_FAILED, "failed to allocate Jetson conversion frame buffer");
}
encoder_ = NvVideoEncoder::createVideoEncoder("cvmmap_streamer_jetson_mm");
if (encoder_ == nullptr) {
return unexpected_error(ERR_BACKEND_UNAVAILABLE, "failed to create Jetson NvVideoEncoder");
}
if (encoder_->setCapturePlaneFormat(
codec_capture_plane_format(codec_),
frame_info.width,
frame_info.height,
kCapturePlaneSizeImage) < 0) {
return unexpected_error(ERR_BACKEND_UNAVAILABLE, "failed to set Jetson encoder capture plane format");
}
if (encoder_->setOutputPlaneFormat(V4L2_PIX_FMT_YUV420M, frame_info.width, frame_info.height) < 0) {
return unexpected_error(ERR_BACKEND_UNAVAILABLE, "failed to set Jetson encoder output plane format");
}
if (encoder_->setBitrate(default_bitrate_bits_per_second(frame_info)) < 0) {
return unexpected_error(ERR_BACKEND_UNAVAILABLE, "failed to set Jetson encoder bitrate");
}
if (codec_ == CodecType::H264) {
if (encoder_->setProfile(V4L2_MPEG_VIDEO_H264_PROFILE_MAIN) < 0) {
return unexpected_error(ERR_BACKEND_UNAVAILABLE, "failed to set Jetson H.264 profile");
}
if (encoder_->setLevel(V4L2_MPEG_VIDEO_H264_LEVEL_5_1) < 0) {
return unexpected_error(ERR_BACKEND_UNAVAILABLE, "failed to set Jetson H.264 level");
}
} else {
if (encoder_->setProfile(V4L2_MPEG_VIDEO_H265_PROFILE_MAIN) < 0) {
return unexpected_error(ERR_BACKEND_UNAVAILABLE, "failed to set Jetson H.265 profile");
}
}
if (encoder_->setRateControlMode(V4L2_MPEG_VIDEO_BITRATE_MODE_CBR) < 0) {
return unexpected_error(ERR_BACKEND_UNAVAILABLE, "failed to set Jetson encoder rate control mode");
}
if (encoder_->setIDRInterval(config.encoder.gop) < 0) {
return unexpected_error(ERR_BACKEND_UNAVAILABLE, "failed to set Jetson encoder IDR interval");
}
if (encoder_->setIFrameInterval(config.encoder.gop) < 0) {
return unexpected_error(ERR_BACKEND_UNAVAILABLE, "failed to set Jetson encoder I-frame interval");
}
if (encoder_->setNumBFrames(config.encoder.b_frames) < 0) {
return unexpected_error(ERR_BACKEND_UNAVAILABLE, "failed to set Jetson encoder B-frame count");
}
if (encoder_->setInsertSpsPpsAtIdrEnabled(true) < 0) {
return unexpected_error(ERR_BACKEND_UNAVAILABLE, "failed to enable Jetson SPS/PPS insertion at IDR");
}
if (encoder_->setFrameRate(kDefaultFrameRateNum, kDefaultFrameRateDen) < 0) {
return unexpected_error(ERR_BACKEND_UNAVAILABLE, "failed to set Jetson encoder frame rate");
}
if (encoder_->setMaxPerfMode(1) < 0) {
return unexpected_error(ERR_BACKEND_UNAVAILABLE, "failed to enable Jetson max performance mode");
}
if (encoder_->output_plane.setupPlane(V4L2_MEMORY_MMAP, kEncoderBufferCount, true, false) < 0) {
return unexpected_error(ERR_BACKEND_UNAVAILABLE, "failed to set up Jetson output plane buffers");
}
if (encoder_->capture_plane.setupPlane(V4L2_MEMORY_MMAP, kEncoderBufferCount, true, false) < 0) {
return unexpected_error(ERR_BACKEND_UNAVAILABLE, "failed to set up Jetson capture plane buffers");
}
if (!encoder_->capture_plane.setDQThreadCallback(&JetsonMmEncoderBackend::capture_plane_dq_callback)) {
return unexpected_error(ERR_BACKEND_UNAVAILABLE, "failed to register Jetson capture dequeue callback");
}
if (encoder_->output_plane.setStreamStatus(true) < 0) {
return unexpected_error(ERR_BACKEND_UNAVAILABLE, "failed to start Jetson output plane");
}
if (encoder_->capture_plane.setStreamStatus(true) < 0) {
return unexpected_error(ERR_BACKEND_UNAVAILABLE, "failed to start Jetson capture plane");
}
if (encoder_->capture_plane.startDQThread(this) < 0) {
return unexpected_error(ERR_BACKEND_UNAVAILABLE, "failed to start Jetson capture dequeue thread");
}
capture_thread_started_ = true;
for (std::uint32_t i = 0; i < encoder_->capture_plane.getNumBuffers(); ++i) {
v4l2_buffer capture_buffer{};
std::array<v4l2_plane, MAX_PLANES> capture_planes{};
capture_buffer.index = i;
capture_buffer.m.planes = capture_planes.data();
if (encoder_->capture_plane.qBuffer(capture_buffer, nullptr) < 0) {
return unexpected_error(ERR_BACKEND_UNAVAILABLE, "failed to queue empty Jetson capture buffer");
}
}
stream_info_ = EncodedStreamInfo{
.codec = codec_,
.width = frame_info.width,
.height = frame_info.height,
.time_base_num = 1,
.time_base_den = 1'000'000'000u,
.frame_rate_num = kDefaultFrameRateNum,
.frame_rate_den = kDefaultFrameRateDen,
.bitstream_format = EncodedBitstreamFormat::AnnexB,
.decoder_config = {},
};
auto warmup = run_warmup();
if (!warmup) {
return warmup;
}
spdlog::info(
"JETSON_MM_ENCODER_READY codec={} width={} height={} gop={} b_frames={}",
cvmmap_streamer::to_string(codec_),
frame_info.width,
frame_info.height,
config.encoder.gop,
config.encoder.b_frames);
return {};
}
[[nodiscard]]
Result<EncodedStreamInfo> stream_info() const {
if (!stream_info_) {
return unexpected_error(ERR_NOT_READY, "Jetson backend stream info is unavailable before initialization");
}
return *stream_info_;
}
[[nodiscard]]
Status poll() {
return check_async_error();
}
[[nodiscard]]
Status push_frame(const RawVideoFrame &frame) {
auto status = check_async_error();
if (!status) {
return status;
}
if (encoder_ == nullptr || converted_frame_ == nullptr || scaler_ == nullptr) {
return unexpected_error(ERR_NOT_READY, "Jetson backend not initialized");
}
if (frame.bytes.empty()) {
return {};
}
const auto ticket_us = next_ticket_us_++;
const auto stream_pts_ns = ensure_stream_pts(frame.source_timestamp_ns);
auto converted = convert_frame(frame);
if (!converted) {
return converted;
}
return submit_converted_frame(ticket_us, frame.source_timestamp_ns, stream_pts_ns, frame.force_keyframe, false);
}
[[nodiscard]]
Result<std::vector<EncodedAccessUnit>> drain() {
auto status = check_async_error();
if (!status) {
return std::unexpected(status.error());
}
std::vector<EncodedAccessUnit> access_units{};
std::lock_guard lock(mutex_);
access_units.reserve(ready_access_units_.size());
while (!ready_access_units_.empty()) {
access_units.push_back(std::move(ready_access_units_.front()));
ready_access_units_.erase(ready_access_units_.begin());
}
return access_units;
}
[[nodiscard]]
Result<std::vector<EncodedAccessUnit>> flush() {
auto status = check_async_error();
if (!status) {
return std::unexpected(status.error());
}
if (encoder_ == nullptr || eos_sent_) {
return drain();
}
auto eos_submit = submit_end_of_stream();
if (!eos_submit) {
return std::unexpected(eos_submit.error());
}
std::unique_lock lock(mutex_);
const auto completed = condition_.wait_for(lock, kFlushTimeout, [&] {
return capture_eos_ || async_error_.has_value();
});
if (!completed && !capture_eos_) {
return unexpected_error(ERR_ENCODER, "timed out waiting for Jetson encoder EOS");
}
if (async_error_) {
return std::unexpected(*async_error_);
}
lock.unlock();
return drain();
}
void shutdown() {
if (encoder_ != nullptr) {
if (capture_thread_started_) {
encoder_->capture_plane.stopDQThread();
encoder_->capture_plane.waitForDQThread(1000);
capture_thread_started_ = false;
}
encoder_->output_plane.setStreamStatus(false);
encoder_->capture_plane.setStreamStatus(false);
delete encoder_;
encoder_ = nullptr;
}
if (converted_frame_ != nullptr) {
av_frame_free(&converted_frame_);
}
if (scaler_ != nullptr) {
sws_freeContext(scaler_);
scaler_ = nullptr;
}
first_real_source_timestamp_ns_.reset();
stream_info_.reset();
eos_sent_ = false;
capture_eos_ = false;
next_output_buffer_index_ = 0;
next_ticket_us_ = 1;
input_pix_fmt_ = AV_PIX_FMT_NONE;
config_ = nullptr;
std::lock_guard lock(mutex_);
pending_tickets_.clear();
ready_access_units_.clear();
async_error_.reset();
}
private:
struct TicketMetadata {
std::uint64_t source_timestamp_ns{0};
std::uint64_t stream_pts_ns{0};
bool warmup{false};
};
[[nodiscard]]
static bool capture_plane_dq_callback(
v4l2_buffer *v4l2_buf,
NvBuffer *buffer,
NvBuffer * /*shared_buffer*/,
void *data) {
auto *self = static_cast<JetsonMmEncoderBackend *>(data);
return self != nullptr ? self->handle_capture_buffer(v4l2_buf, buffer) : false;
}
[[nodiscard]]
bool handle_capture_buffer(v4l2_buffer *v4l2_buf, NvBuffer *buffer) {
if (v4l2_buf == nullptr || buffer == nullptr) {
store_async_error(unexpected_error(ERR_ENCODER, "Jetson capture dequeue callback received a null buffer").error());
return false;
}
if (buffer->planes[0].bytesused == 0) {
std::lock_guard lock(mutex_);
capture_eos_ = true;
condition_.notify_all();
return false;
}
v4l2_ctrl_videoenc_outputbuf_metadata metadata{};
if (encoder_->getMetadata(v4l2_buf->index, metadata) < 0) {
store_async_error(unexpected_error(ERR_ENCODER, "failed to read Jetson encoder output metadata").error());
return false;
}
TicketMetadata ticket{};
{
std::lock_guard lock(mutex_);
const auto ticket_it = pending_tickets_.find(timeval_to_token_us(v4l2_buf->timestamp));
if (ticket_it == pending_tickets_.end()) {
store_async_error(unexpected_error(ERR_PROTOCOL, "Jetson encoder returned an unknown frame ticket").error());
return false;
}
ticket = ticket_it->second;
pending_tickets_.erase(ticket_it);
}
std::vector<std::uint8_t> annexb_bytes(
buffer->planes[0].data,
buffer->planes[0].data + buffer->planes[0].bytesused);
const auto keyframe = metadata.KeyFrame != 0;
if (keyframe && stream_info_ && stream_info_->decoder_config.empty()) {
stream_info_->decoder_config = extract_decoder_config_annexb(codec_, annexb_bytes);
}
if (!ticket.warmup) {
EncodedAccessUnit access_unit{};
access_unit.codec = codec_;
access_unit.source_timestamp_ns = ticket.source_timestamp_ns;
access_unit.stream_pts_ns = ticket.stream_pts_ns;
access_unit.keyframe = keyframe;
access_unit.annexb_bytes = std::move(annexb_bytes);
std::lock_guard lock(mutex_);
ready_access_units_.push_back(std::move(access_unit));
}
condition_.notify_all();
if (encoder_->capture_plane.qBuffer(*v4l2_buf, nullptr) < 0) {
store_async_error(unexpected_error(ERR_ENCODER, "failed to requeue Jetson capture buffer").error());
return false;
}
return true;
}
void store_async_error(Error error) {
std::lock_guard lock(mutex_);
if (!async_error_) {
async_error_ = std::move(error);
}
condition_.notify_all();
}
[[nodiscard]]
Status check_async_error() const {
std::lock_guard lock(mutex_);
if (async_error_) {
return std::unexpected(*async_error_);
}
return {};
}
[[nodiscard]]
std::uint64_t ensure_stream_pts(const std::uint64_t source_timestamp_ns) {
if (!first_real_source_timestamp_ns_) {
first_real_source_timestamp_ns_ = source_timestamp_ns;
}
return source_timestamp_ns - *first_real_source_timestamp_ns_;
}
[[nodiscard]]
Status convert_frame(const RawVideoFrame &frame) {
const auto make_writable = av_frame_make_writable(converted_frame_);
if (make_writable < 0) {
return unexpected_error(ERR_EXTERNAL_LIBRARY, "failed to make Jetson conversion frame writable");
}
AVFrame input_frame{};
input_frame.format = input_pix_fmt_;
input_frame.width = static_cast<int>(frame_info_.width);
input_frame.height = static_cast<int>(frame_info_.height);
if (av_image_fill_arrays(
input_frame.data,
input_frame.linesize,
const_cast<std::uint8_t *>(frame.bytes.data()),
input_pix_fmt_,
input_frame.width,
input_frame.height,
1) < 0) {
return unexpected_error(ERR_INVALID_ARGUMENT, "failed to map input frame into Jetson conversion image arrays");
}
if (frame.row_stride_bytes != 0) {
input_frame.linesize[0] = static_cast<int>(frame.row_stride_bytes);
}
sws_scale(
scaler_,
input_frame.data,
input_frame.linesize,
0,
input_frame.height,
converted_frame_->data,
converted_frame_->linesize);
return {};
}
void fill_black_frame() {
av_frame_make_writable(converted_frame_);
for (int y = 0; y < converted_frame_->height; ++y) {
std::memset(converted_frame_->data[0] + y * converted_frame_->linesize[0], 16, static_cast<std::size_t>(converted_frame_->width));
}
for (int y = 0; y < converted_frame_->height / 2; ++y) {
std::memset(converted_frame_->data[1] + y * converted_frame_->linesize[1], 128, static_cast<std::size_t>(converted_frame_->width / 2));
std::memset(converted_frame_->data[2] + y * converted_frame_->linesize[2], 128, static_cast<std::size_t>(converted_frame_->width / 2));
}
}
[[nodiscard]]
Status sync_output_buffer_for_device(NvBuffer &buffer) const {
for (std::uint32_t plane = 0; plane < buffer.n_planes; ++plane) {
NvBufSurface *surface = nullptr;
if (NvBufSurfaceFromFd(buffer.planes[plane].fd, reinterpret_cast<void **>(&surface)) != 0 || surface == nullptr) {
return unexpected_error(ERR_EXTERNAL_LIBRARY, "failed to resolve Jetson output plane surface");
}
if (NvBufSurfaceSyncForDevice(surface, 0, static_cast<int>(plane)) != 0) {
return unexpected_error(ERR_EXTERNAL_LIBRARY, "failed to sync Jetson output plane buffer for device");
}
}
return {};
}
[[nodiscard]]
Result<NvBuffer *> acquire_output_buffer(v4l2_buffer &v4l2_buf, std::array<v4l2_plane, MAX_PLANES> &planes) {
planes.fill(v4l2_plane{});
std::memset(&v4l2_buf, 0, sizeof(v4l2_buf));
v4l2_buf.m.planes = planes.data();
NvBuffer *buffer = nullptr;
if (next_output_buffer_index_ < encoder_->output_plane.getNumBuffers()) {
buffer = encoder_->output_plane.getNthBuffer(next_output_buffer_index_);
v4l2_buf.index = next_output_buffer_index_++;
} else if (encoder_->output_plane.dqBuffer(v4l2_buf, &buffer, nullptr, 1000) < 0) {
return unexpected_error(ERR_ENCODER, "failed to dequeue Jetson output plane buffer");
}
if (buffer == nullptr) {
return unexpected_error(ERR_ENCODER, "Jetson output plane returned a null buffer");
}
return buffer;
}
[[nodiscard]]
Status submit_converted_frame(
const std::uint64_t ticket_us,
const std::uint64_t source_timestamp_ns,
const std::uint64_t stream_pts_ns,
const bool force_keyframe,
const bool warmup) {
std::array<v4l2_plane, MAX_PLANES> planes{};
v4l2_buffer v4l2_buf{};
auto acquired = acquire_output_buffer(v4l2_buf, planes);
if (!acquired) {
return std::unexpected(acquired.error());
}
NvBuffer *buffer = *acquired;
if (force_keyframe && encoder_->forceIDR() < 0) {
return unexpected_error(ERR_ENCODER, "failed to force a Jetson IDR frame");
}
for (std::uint32_t plane = 0; plane < buffer->n_planes; ++plane) {
auto &output_plane = buffer->planes[plane];
const auto plane_width = plane == 0 ? frame_info_.width : frame_info_.width / 2;
const auto plane_height = plane == 0 ? frame_info_.height : frame_info_.height / 2;
for (std::uint32_t row = 0; row < plane_height; ++row) {
std::memcpy(
output_plane.data + row * output_plane.fmt.stride,
converted_frame_->data[plane] + row * converted_frame_->linesize[plane],
plane_width);
}
output_plane.bytesused = output_plane.fmt.stride * plane_height;
v4l2_buf.m.planes[plane].bytesused = output_plane.bytesused;
}
v4l2_buf.flags |= V4L2_BUF_FLAG_TIMESTAMP_COPY;
v4l2_buf.timestamp = token_us_to_timeval(ticket_us);
auto sync = sync_output_buffer_for_device(*buffer);
if (!sync) {
return sync;
}
{
std::lock_guard lock(mutex_);
pending_tickets_[ticket_us] = TicketMetadata{
.source_timestamp_ns = source_timestamp_ns,
.stream_pts_ns = stream_pts_ns,
.warmup = warmup,
};
}
if (encoder_->output_plane.qBuffer(v4l2_buf, nullptr) < 0) {
std::lock_guard lock(mutex_);
pending_tickets_.erase(ticket_us);
return unexpected_error(ERR_ENCODER, "failed to queue Jetson output frame");
}
return {};
}
[[nodiscard]]
Status submit_end_of_stream() {
std::array<v4l2_plane, MAX_PLANES> planes{};
v4l2_buffer v4l2_buf{};
auto acquired = acquire_output_buffer(v4l2_buf, planes);
if (!acquired) {
return std::unexpected(acquired.error());
}
v4l2_buf.m.planes[0].bytesused = 0;
if (encoder_->output_plane.qBuffer(v4l2_buf, nullptr) < 0) {
return unexpected_error(ERR_ENCODER, "failed to queue Jetson encoder EOS");
}
eos_sent_ = true;
return {};
}
[[nodiscard]]
Status run_warmup() {
fill_black_frame();
const auto warmup_ticket = next_ticket_us_++;
auto warmup_submit = submit_converted_frame(warmup_ticket, 0, 0, true, true);
if (!warmup_submit) {
return warmup_submit;
}
std::unique_lock lock(mutex_);
const auto completed = condition_.wait_for(lock, kWarmupTimeout, [&] {
return async_error_.has_value() || (stream_info_ && !stream_info_->decoder_config.empty());
});
if (!completed || !stream_info_ || stream_info_->decoder_config.empty()) {
return unexpected_error(ERR_ENCODER, "failed to harvest Jetson decoder configuration from warmup frame");
}
if (async_error_) {
return std::unexpected(*async_error_);
}
return {};
}
const RuntimeConfig *config_{nullptr};
ipc::FrameInfo frame_info_{};
CodecType codec_{CodecType::H264};
NvVideoEncoder *encoder_{nullptr};
SwsContext *scaler_{nullptr};
AVFrame *converted_frame_{nullptr};
AVPixelFormat input_pix_fmt_{AV_PIX_FMT_NONE};
std::optional<std::uint64_t> first_real_source_timestamp_ns_{};
std::optional<EncodedStreamInfo> stream_info_{};
std::uint32_t next_output_buffer_index_{0};
std::uint64_t next_ticket_us_{1};
bool capture_thread_started_{false};
bool eos_sent_{false};
mutable std::mutex mutex_{};
std::condition_variable condition_{};
std::unordered_map<std::uint64_t, TicketMetadata> pending_tickets_{};
std::vector<EncodedAccessUnit> ready_access_units_{};
std::optional<Error> async_error_{};
bool capture_eos_{false};
};
} // namespace
EncoderBackend make_jetson_mm_backend() {
return pro::make_proxy<EncoderBackendFacade, JetsonMmEncoderBackend>();
}
} // namespace cvmmap_streamer::encode
+42 -35
View File
@@ -9,41 +9,48 @@ namespace cvmmap_streamer {
namespace {
constexpr std::array<std::string_view, 34> kHelpLines{
"Usage:",
" --help, -h\tshow this message",
"",
"Options:",
" --version\tprint version information",
" --config <path>\tload runtime config from TOML",
" --input-uri <uri>\tcvmmap source URI (example: cvmmap://default)",
" --run-mode <mode>\tpipeline|ingest",
" --codec <codec>\th264|h265",
" --encoder-backend <backend>\tauto|ffmpeg",
" --encoder-device <device>\tauto|nvidia|software",
" --gop <frames>\tencoder GOP length",
" --b-frames <count>\tencoder B-frame count",
" --keep-stream-on-reset <bool>\tkeep RTMP/RTP sessions alive across upstream stream_reset events",
" --rtp\t\tenable RTP output",
" --rtp-endpoint <host:port>\tRTP destination",
" --rtp-payload-type <pt>\tRTP payload type (96-127)",
" --rtp-sdp <path>\twrite optional SDP sidecar",
" --rtmp\t\tenable RTMP output",
" --rtmp-url <url>\tadd RTMP destination (repeatable)",
" --rtmp-transport <mode>\tlibavformat|ffmpeg_process",
" --rtmp-ffmpeg <path>\tffmpeg binary for ffmpeg_process transport",
" --mcap\t\tenable MCAP recording",
" --mcap-path <path>\tMCAP output file",
" --mcap-topic <topic>\tMCAP topic name",
" --mcap-depth-topic <topic>\tMCAP depth topic name",
" --mcap-body-topic <topic>\tMCAP body topic name",
" --mcap-frame-id <id>\tFoxglove CompressedVideo frame_id",
" --mcap-compression <mode>\tnone|lz4|zstd",
"",
"Examples:",
" cvmmap_streamer --help",
" cvmmap_streamer --run-mode pipeline --input-uri cvmmap://default --help",
" rtp_receiver_tester --help"};
constexpr auto kHelpLines = std::to_array<std::string_view>({
"Usage:",
" --help, -h\tshow this message",
"",
"Options:",
" --version\tprint version information",
" --config <path>\tload runtime config from TOML",
" --input-uri <uri>\tcvmmap source URI (example: cvmmap://default)",
" --run-mode <mode>\tpipeline|ingest",
" --codec <codec>\th264|h265",
" --encoder-backend <backend>\tauto|ffmpeg",
" --encoder-device <device>\tauto|nvidia|software",
" --gop <frames>\tencoder GOP length",
" --b-frames <count>\tencoder B-frame count",
" --keep-stream-on-reset <bool>\tkeep RTMP/RTP sessions alive across upstream stream_reset events",
" --rtp\t\tenable RTP output",
" --rtp-endpoint <host:port>\tRTP destination",
" --rtp-payload-type <pt>\tRTP payload type (96-127)",
" --rtp-sdp <path>\twrite optional SDP sidecar",
" --rtmp\t\tenable RTMP output",
" --rtmp-url <url>\tadd RTMP destination (repeatable)",
" --rtmp-transport <mode>\tlibavformat|ffmpeg_process",
" --rtmp-ffmpeg <path>\tffmpeg binary for ffmpeg_process transport",
#if CVMMAP_STREAMER_HAS_MCAP
" --mcap\t\tenable MCAP recording",
" --mcap-path <path>\tMCAP output file",
" --mcap-topic <topic>\tMCAP topic name",
" --mcap-body-topic <topic>\tMCAP body topic name",
" --mcap-frame-id <id>\tFoxglove CompressedVideo frame_id",
" --mcap-compression <mode>\tnone|lz4|zstd",
#if CVMMAP_STREAMER_HAS_MCAP_DEPTH
" --mcap-depth\t\tenable MCAP depth recording",
" --mcap-depth-topic <topic>\tMCAP depth topic name (implies --mcap)",
" record.mcap.depth_enabled\tTOML toggle for optional depth recording",
#endif
#endif
"",
"Examples:",
" cvmmap_streamer --help",
" cvmmap_streamer --run-mode pipeline --input-uri cvmmap://default --help",
" rtp_receiver_tester --help"
});
}
+194 -63
View File
@@ -1,5 +1,6 @@
#include "cvmmap_streamer/config/runtime_config.hpp"
#include "cvmmap_streamer/core/frame_source.hpp"
#include "cvmmap_streamer/core/status.hpp"
#include "cvmmap_streamer/encode/encoder_backend.hpp"
#include "cvmmap_streamer/ipc/contracts.hpp"
#include "cvmmap_streamer/metrics/latency_tracker.hpp"
@@ -11,6 +12,14 @@
#include "cvmmap_streamer/record/mp4_record_writer.hpp"
#include "proto/cvmmap_streamer/recorder_control.pb.h"
#ifndef CVMMAP_STREAMER_HAS_MCAP
#define CVMMAP_STREAMER_HAS_MCAP 0
#endif
#ifndef CVMMAP_STREAMER_HAS_MCAP_DEPTH
#define CVMMAP_STREAMER_HAS_MCAP_DEPTH 0
#endif
#include <cvmmap/client.hpp>
#include <cvmmap/nats_client.hpp>
#include <cvmmap/parser.hpp>
@@ -432,6 +441,32 @@ float stream_fps(const encode::EncodedStreamInfo &stream_info) {
static_cast<float>(stream_info.frame_rate_den);
}
template <class Config>
[[nodiscard]] bool runtime_mcap_depth_enabled(const Config &config) {
if constexpr (requires { config.record.mcap.depth_enabled; }) {
return config.record.mcap.depth_enabled;
} else {
return true;
}
}
[[nodiscard]] constexpr bool has_mcap_support() {
return CVMMAP_STREAMER_HAS_MCAP != 0;
}
[[nodiscard]] constexpr bool has_mcap_depth_support() {
return CVMMAP_STREAMER_HAS_MCAP_DEPTH != 0;
}
[[nodiscard]] std::string mcap_disabled_message() {
return "MCAP recording support is not compiled into this build";
}
[[nodiscard]] std::string mcap_depth_disabled_message() {
return "MCAP depth recording support is not compiled into this build";
}
#if CVMMAP_STREAMER_HAS_MCAP
struct McapRecorderState {
mutable std::mutex mutex{};
RuntimeConfig base_config{};
@@ -449,27 +484,6 @@ struct McapRecorderState {
} status{};
};
struct Mp4RecorderStatus {
bool can_record{false};
bool is_recording{false};
bool last_frame_ok{false};
std::uint32_t frames_ingested{0};
std::uint32_t frames_encoded{0};
std::string active_path{};
std::string error_message{};
};
struct Mp4RecorderState {
mutable std::mutex mutex{};
RuntimeConfig base_config{};
std::optional<ipc::FrameInfo> current_frame_info{};
std::optional<record::Mp4InputPixelFormat> current_input_pixel_format{};
float current_fps{30.0f};
std::optional<record::Mp4RecordWriter> writer{};
std::optional<std::uint64_t> first_frame_timestamp_ns{};
Mp4RecorderStatus status{};
};
[[nodiscard]]
protocol::RpcError make_recorder_rpc_error(
const protocol::RpcErrorCode code,
@@ -502,49 +516,11 @@ RuntimeConfig make_mcap_record_config(
return record_config;
}
[[nodiscard]]
record::Mp4EncodeTuning make_mp4_encode_tuning(const RuntimeConfig &base_config) {
return record::Mp4EncodeTuning{
.quality = record::kDefaultMp4Quality,
.gop = base_config.encoder.gop,
.b_frames = base_config.encoder.b_frames,
};
}
[[nodiscard]]
std::expected<record::Mp4InputPixelFormat, std::string> mp4_input_pixel_format(
const ipc::FrameInfo &frame_info) {
if (frame_info.depth != ipc::Depth::U8) {
return std::unexpected("MP4 recorder requires 8-bit color frames");
}
switch (frame_info.pixel_format) {
case ipc::PixelFormat::BGR:
return record::Mp4InputPixelFormat::Bgr24;
case ipc::PixelFormat::RGB:
return record::Mp4InputPixelFormat::Rgb24;
case ipc::PixelFormat::BGRA:
return record::Mp4InputPixelFormat::Bgra32;
case ipc::PixelFormat::RGBA:
return record::Mp4InputPixelFormat::Rgba32;
case ipc::PixelFormat::GRAY:
return record::Mp4InputPixelFormat::Gray8;
case ipc::PixelFormat::YUV:
case ipc::PixelFormat::YUYV:
return std::unexpected("MP4 recorder does not support packed YUV snapshot frames");
}
return std::unexpected("MP4 recorder does not support the snapshot pixel format");
}
void reset_mcap_status_after_stop(McapRecorderState::Status &status) {
status.is_recording = false;
status.active_path.clear();
}
void reset_mp4_status_after_stop(Mp4RecorderStatus &status) {
status.is_recording = false;
status.active_path.clear();
}
[[nodiscard]]
recorder_pb::McapRecorderState to_proto_mcap_state(
const McapRecorderState::Status &status) {
@@ -653,6 +629,121 @@ std::expected<McapRecorderState::Status, protocol::RpcError> get_mcap_recording_
recorder_state.status.can_record = true;
return recorder_state.status;
}
#else
struct McapRecorderState {
struct Status {};
};
[[nodiscard]]
protocol::RpcError make_mcap_unsupported_rpc_error() {
return protocol::RpcError{
.code = protocol::RpcErrorCode::Unsupported,
.message = mcap_disabled_message(),
};
}
[[nodiscard]]
std::expected<McapRecorderState::Status, protocol::RpcError> start_mcap_recording(
McapRecorderState &,
const recorder_pb::McapStartRequest &) {
return std::unexpected(make_mcap_unsupported_rpc_error());
}
[[nodiscard]]
std::expected<McapRecorderState::Status, protocol::RpcError> stop_mcap_recording(
McapRecorderState &,
const recorder_pb::McapStopRequest &) {
return std::unexpected(make_mcap_unsupported_rpc_error());
}
[[nodiscard]]
std::expected<McapRecorderState::Status, protocol::RpcError> get_mcap_recording_status(
McapRecorderState &,
const recorder_pb::McapStatusRequest &) {
return std::unexpected(make_mcap_unsupported_rpc_error());
}
void update_mcap_stream_info(
McapRecorderState &,
const encode::EncodedStreamInfo &) {}
Status write_mcap_access_unit(
McapRecorderState *,
const encode::EncodedAccessUnit &) {
return unexpected_error(ERR_UNSUPPORTED, mcap_disabled_message());
}
Status write_mcap_body_message(
McapRecorderState *,
const record::RawBodyTrackingMessageView &) {
return unexpected_error(ERR_UNSUPPORTED, mcap_disabled_message());
}
Status write_mcap_depth_map(
McapRecorderState *,
const record::RawDepthMapView &) {
return unexpected_error(ERR_UNSUPPORTED, mcap_depth_disabled_message());
}
#endif
struct Mp4RecorderStatus {
bool can_record{false};
bool is_recording{false};
bool last_frame_ok{false};
std::uint32_t frames_ingested{0};
std::uint32_t frames_encoded{0};
std::string active_path{};
std::string error_message{};
};
struct Mp4RecorderState {
mutable std::mutex mutex{};
RuntimeConfig base_config{};
std::optional<ipc::FrameInfo> current_frame_info{};
std::optional<record::Mp4InputPixelFormat> current_input_pixel_format{};
float current_fps{30.0f};
std::optional<record::Mp4RecordWriter> writer{};
std::optional<std::uint64_t> first_frame_timestamp_ns{};
Mp4RecorderStatus status{};
};
[[nodiscard]]
record::Mp4EncodeTuning make_mp4_encode_tuning(const RuntimeConfig &base_config) {
return record::Mp4EncodeTuning{
.quality = record::kDefaultMp4Quality,
.gop = base_config.encoder.gop,
.b_frames = base_config.encoder.b_frames,
};
}
[[nodiscard]]
std::expected<record::Mp4InputPixelFormat, std::string> mp4_input_pixel_format(
const ipc::FrameInfo &frame_info) {
if (frame_info.depth != ipc::Depth::U8) {
return std::unexpected("MP4 recorder requires 8-bit color frames");
}
switch (frame_info.pixel_format) {
case ipc::PixelFormat::BGR:
return record::Mp4InputPixelFormat::Bgr24;
case ipc::PixelFormat::RGB:
return record::Mp4InputPixelFormat::Rgb24;
case ipc::PixelFormat::BGRA:
return record::Mp4InputPixelFormat::Bgra32;
case ipc::PixelFormat::RGBA:
return record::Mp4InputPixelFormat::Rgba32;
case ipc::PixelFormat::GRAY:
return record::Mp4InputPixelFormat::Gray8;
case ipc::PixelFormat::YUV:
case ipc::PixelFormat::YUYV:
return std::unexpected("MP4 recorder does not support packed YUV snapshot frames");
}
return std::unexpected("MP4 recorder does not support the snapshot pixel format");
}
void reset_mp4_status_after_stop(Mp4RecorderStatus &status) {
status.is_recording = false;
status.active_path.clear();
}
void close_mp4_writer_with_error(
Mp4RecorderState &recorder_state,
@@ -893,6 +984,7 @@ void write_mp4_frame(
recorder_state->status.frames_encoded += 1;
}
#if CVMMAP_STREAMER_HAS_MCAP
void update_mcap_stream_info(
McapRecorderState &recorder_state,
const encode::EncodedStreamInfo &stream_info) {
@@ -981,6 +1073,7 @@ Status write_mcap_depth_map(
recorder_state->status.last_frame_ok = true;
return {};
}
#endif
[[nodiscard]]
Status publish_access_units(
@@ -1126,10 +1219,12 @@ int run_pipeline(const RuntimeConfig &config) {
std::optional<protocol::UdpRtpPublisher> rtp_publisher{};
std::optional<protocol::RtmpOutput> rtmp_output{};
#if CVMMAP_STREAMER_HAS_MCAP
McapRecorderState mcap_recorder{};
Mp4RecorderState mp4_recorder{};
mcap_recorder.base_config = config;
mcap_recorder.status.can_record = true;
#endif
Mp4RecorderState mp4_recorder{};
mp4_recorder.base_config = config;
cvmmap::NatsControlClient nats_client(
input_endpoints->nats_target_key,
@@ -1141,7 +1236,7 @@ int run_pipeline(const RuntimeConfig &config) {
.ipc_prefix = input_endpoints->ipc_prefix,
.base_name = input_endpoints->base_name,
.nats_target_key = input_endpoints->nats_target_key,
.recording_formats = "mp4,mcap",
.recording_formats = has_mcap_support() ? "mp4,mcap" : "mp4",
});
std::mutex nats_event_mutex{};
std::deque<std::vector<std::uint8_t>> pending_body_packets{};
@@ -1200,6 +1295,7 @@ int run_pipeline(const RuntimeConfig &config) {
}
return make_ok_mp4_response<recorder_pb::Mp4StatusResponse>(*status);
});
#if CVMMAP_STREAMER_HAS_MCAP
recorder_rpc_server.register_proto_endpoint<
recorder_pb::McapStartRequest,
recorder_pb::McapStartResponse>(
@@ -1239,6 +1335,7 @@ int run_pipeline(const RuntimeConfig &config) {
}
return make_ok_mcap_response<recorder_pb::McapStatusResponse>(*status);
});
#endif
if (!recorder_rpc_server.start()) {
spdlog::error("pipeline streamer recorder service failed on '{}'", config.input.nats_url);
nats_client.Stop();
@@ -1348,8 +1445,8 @@ int run_pipeline(const RuntimeConfig &config) {
live_output_continuity.note_new_session(stream_info);
}
#if CVMMAP_STREAMER_HAS_MCAP
update_mcap_stream_info(mcap_recorder, stream_info);
update_mp4_source_info(mp4_recorder, target_info, stream_fps(stream_info));
if (config.record.mcap.enabled) {
std::lock_guard lock(mcap_recorder.mutex);
if (!mcap_recorder.sink) {
@@ -1367,6 +1464,12 @@ int run_pipeline(const RuntimeConfig &config) {
mcap_recorder.status.active_path = config.record.mcap.path;
}
}
#else
if (config.record.mcap.enabled) {
return unexpected_error(ERR_UNSUPPORTED, mcap_disabled_message());
}
#endif
update_mp4_source_info(mp4_recorder, target_info, stream_fps(stream_info));
started = true;
restart_pending = false;
restart_target_info.reset();
@@ -1481,6 +1584,7 @@ int run_pipeline(const RuntimeConfig &config) {
continue;
}
#if CVMMAP_STREAMER_HAS_MCAP
auto write_body = write_mcap_body_message(&mcap_recorder, record::RawBodyTrackingMessageView{
.timestamp_ns = body_tracking_timestamp_ns(*parsed_body),
.bytes = body_bytes,
@@ -1490,6 +1594,7 @@ int run_pipeline(const RuntimeConfig &config) {
restart_backend(reason, active_info);
break;
}
#endif
}
if (backend && !using_encoded_input) {
@@ -1532,7 +1637,11 @@ int run_pipeline(const RuntimeConfig &config) {
stats,
rtp_publisher ? &*rtp_publisher : nullptr,
rtmp_output ? &*rtmp_output : nullptr,
#if CVMMAP_STREAMER_HAS_MCAP
&mcap_recorder,
#else
nullptr,
#endif
keep_live_outputs_on_reset ? &live_output_continuity : nullptr,
latency_tracker);
if (!drain) {
@@ -1656,7 +1765,11 @@ int run_pipeline(const RuntimeConfig &config) {
stats,
rtp_publisher ? &*rtp_publisher : nullptr,
rtmp_output ? &*rtmp_output : nullptr,
#if CVMMAP_STREAMER_HAS_MCAP
&mcap_recorder,
#else
nullptr,
#endif
keep_live_outputs_on_reset ? &live_output_continuity : nullptr,
latency_tracker);
if (!publish) {
@@ -1678,7 +1791,13 @@ int run_pipeline(const RuntimeConfig &config) {
}
}
if (!snapshot->depth.empty()) {
#if CVMMAP_STREAMER_HAS_MCAP
if (!snapshot->depth.empty() && config.record.mcap.enabled && runtime_mcap_depth_enabled(config)) {
#if !CVMMAP_STREAMER_HAS_MCAP_DEPTH
const auto reason = "pipeline depth MCAP write failed: " + mcap_depth_disabled_message();
restart_backend(reason, active_info);
continue;
#else
if (snapshot->depth_unit == ipc::DepthUnit::Unknown) {
if (!warned_unknown_depth_unit) {
spdlog::warn(
@@ -1700,7 +1819,9 @@ int run_pipeline(const RuntimeConfig &config) {
restart_backend(reason, active_info);
continue;
}
#endif
}
#endif
stats.pushed_frames += 1;
if (!want_encoded_input) {
@@ -1711,7 +1832,11 @@ int run_pipeline(const RuntimeConfig &config) {
stats,
rtp_publisher ? &*rtp_publisher : nullptr,
rtmp_output ? &*rtmp_output : nullptr,
#if CVMMAP_STREAMER_HAS_MCAP
&mcap_recorder,
#else
nullptr,
#endif
keep_live_outputs_on_reset ? &live_output_continuity : nullptr,
latency_tracker);
if (!drain) {
@@ -1741,7 +1866,11 @@ int run_pipeline(const RuntimeConfig &config) {
stats,
rtp_publisher ? &*rtp_publisher : nullptr,
rtmp_output ? &*rtmp_output : nullptr,
#if CVMMAP_STREAMER_HAS_MCAP
&mcap_recorder,
#else
nullptr,
#endif
keep_live_outputs_on_reset ? &live_output_continuity : nullptr,
latency_tracker);
if (!drain) {
@@ -1757,10 +1886,12 @@ int run_pipeline(const RuntimeConfig &config) {
if (!stop_mp4) {
spdlog::warn("pipeline MP4 recorder stop during shutdown failed: {}", stop_mp4.error().message);
}
#if CVMMAP_STREAMER_HAS_MCAP
auto stop_mcap = stop_mcap_recording(mcap_recorder, recorder_pb::McapStopRequest{});
if (!stop_mcap) {
spdlog::warn("pipeline MCAP recorder stop during shutdown failed: {}", stop_mcap.error().message);
}
#endif
recorder_rpc_server.stop();
spdlog::info(
+197 -34
View File
@@ -1,16 +1,28 @@
#include <mcap/writer.hpp>
#include "cvmmap_streamer/record/mcap_record_sink.hpp"
#ifndef CVMMAP_STREAMER_HAS_MCAP
#define CVMMAP_STREAMER_HAS_MCAP 0
#endif
#ifndef CVMMAP_STREAMER_HAS_MCAP_DEPTH
#define CVMMAP_STREAMER_HAS_MCAP_DEPTH 0
#endif
#if CVMMAP_STREAMER_HAS_MCAP
#include <mcap/writer.hpp>
#include "protobuf_descriptor.hpp"
#include "proto/cvmmap_streamer/BundleManifest.pb.h"
#include "proto/cvmmap_streamer/DepthMap.pb.h"
#include "proto/foxglove/CameraCalibration.pb.h"
#include "proto/foxglove/CompressedVideo.pb.h"
#include "proto/foxglove/PoseInFrame.pb.h"
#if CVMMAP_STREAMER_HAS_MCAP_DEPTH
#include <rvl/rvl.hpp>
#endif
#include <google/protobuf/timestamp.pb.h>
#endif
#include <cmath>
#include <cstddef>
@@ -26,10 +38,23 @@
namespace cvmmap_streamer::record {
template <class Config>
[[nodiscard]] bool mcap_depth_requested(const Config &config) {
if constexpr (requires { config.record.mcap.depth_enabled; }) {
return config.record.mcap.depth_enabled;
} else {
return true;
}
}
#if CVMMAP_STREAMER_HAS_MCAP
namespace {
#if CVMMAP_STREAMER_HAS_MCAP_DEPTH
constexpr float kRvlDepthQuantization = 200.0f;
constexpr float kMinDepthMaxMeters = 20.0f;
#endif
constexpr std::string_view kBodyTrackingMessageEncoding = "cvmmap.body_tracking.v1";
[[nodiscard]]
@@ -268,6 +293,7 @@ std::expected<std::vector<std::uint8_t>, std::string> decoder_config_to_annexb(
return avcc_to_annexb(decoder_config);
}
#if CVMMAP_STREAMER_HAS_MCAP_DEPTH
[[nodiscard]]
bool can_encode_lossless_u16_mm(const RawDepthMapView &depth_map) {
if (normalize_depth_source_unit(depth_map.source_unit) != ipc::DepthUnit::Millimeter) {
@@ -367,6 +393,7 @@ std::expected<EncodedDepthPayload, std::string> encode_depth_payload(const RawDe
return std::unexpected(std::string("failed to RVL-encode depth map: ") + error.what());
}
}
#endif
[[nodiscard]]
std::expected<void, std::string> append_repeated_double(
@@ -619,18 +646,26 @@ std::expected<McapRecordSink, std::string> McapRecordSink::create(
state->writer.addChannel(channel);
state->video_channel_id = channel.id;
const auto depth_descriptor_set = build_file_descriptor_set(cvmmap_streamer::DepthMap::descriptor());
std::string depth_schema_bytes{};
if (!depth_descriptor_set.SerializeToString(&depth_schema_bytes)) {
return std::unexpected("failed to serialize cvmmap_streamer.DepthMap descriptor set");
#if CVMMAP_STREAMER_HAS_MCAP_DEPTH
if (mcap_depth_requested(config)) {
const auto depth_descriptor_set = build_file_descriptor_set(cvmmap_streamer::DepthMap::descriptor());
std::string depth_schema_bytes{};
if (!depth_descriptor_set.SerializeToString(&depth_schema_bytes)) {
return std::unexpected("failed to serialize cvmmap_streamer.DepthMap descriptor set");
}
mcap::Schema depth_schema("cvmmap_streamer.DepthMap", "protobuf", depth_schema_bytes);
state->writer.addSchema(depth_schema);
mcap::Channel depth_channel(config.record.mcap.depth_topic, "protobuf", depth_schema.id);
state->writer.addChannel(depth_channel);
state->depth_channel_id = depth_channel.id;
}
mcap::Schema depth_schema("cvmmap_streamer.DepthMap", "protobuf", depth_schema_bytes);
state->writer.addSchema(depth_schema);
mcap::Channel depth_channel(config.record.mcap.depth_topic, "protobuf", depth_schema.id);
state->writer.addChannel(depth_channel);
state->depth_channel_id = depth_channel.id;
#else
if (mcap_depth_requested(config)) {
return std::unexpected("MCAP depth recording support is not compiled into this build");
}
#endif
const auto calibration_descriptor_set = build_file_descriptor_set(foxglove::CameraCalibration::descriptor());
std::string calibration_schema_bytes{};
@@ -661,6 +696,7 @@ std::expected<McapRecordSink, std::string> McapRecordSink::create(
return sink;
}
std::expected<void, std::string> McapRecordSink::update_stream_info(const encode::EncodedStreamInfo &stream_info) {
if (state_ == nullptr) {
return std::unexpected("MCAP sink is not open");
@@ -717,8 +753,14 @@ std::expected<void, std::string> McapRecordSink::write_depth_map(const RawDepthM
if (state_ == nullptr) {
return std::unexpected("MCAP sink is not open");
}
if (state_->depth_channel_id == 0) {
return std::unexpected("MCAP depth recording support is not enabled for this sink");
}
#if !CVMMAP_STREAMER_HAS_MCAP_DEPTH
static_cast<void>(depth_map);
return std::unexpected("MCAP depth recording support is not compiled into this build");
#else
const auto source_unit = normalize_depth_source_unit(depth_map.source_unit);
auto encoded = encode_depth_payload(depth_map);
if (!encoded) {
return std::unexpected(encoded.error());
@@ -734,13 +776,20 @@ std::expected<void, std::string> McapRecordSink::write_depth_map(const RawDepthM
state_->depth_channel_id,
state_->depth_sequence,
*encoded);
#endif
}
std::expected<void, std::string> McapRecordSink::write_depth_map_u16(const RawDepthMapU16View &depth_map) {
if (state_ == nullptr) {
return std::unexpected("MCAP sink is not open");
}
if (state_->depth_channel_id == 0) {
return std::unexpected("MCAP depth recording support is not enabled for this sink");
}
#if !CVMMAP_STREAMER_HAS_MCAP_DEPTH
static_cast<void>(depth_map);
return std::unexpected("MCAP depth recording support is not compiled into this build");
#else
auto encoded = encode_depth_payload(depth_map);
if (!encoded) {
return std::unexpected(encoded.error());
@@ -756,6 +805,7 @@ std::expected<void, std::string> McapRecordSink::write_depth_map_u16(const RawDe
state_->depth_channel_id,
state_->depth_sequence,
*encoded);
#endif
}
std::expected<void, std::string> McapRecordSink::write_camera_calibration(const RawCameraCalibrationView &calibration) {
@@ -777,6 +827,9 @@ std::expected<void, std::string> McapRecordSink::write_depth_camera_calibration(
if (state_ == nullptr) {
return std::unexpected("MCAP sink is not open");
}
if (state_->depth_channel_id == 0) {
return std::unexpected("MCAP depth recording support is not enabled for this sink");
}
return write_calibration_message(
state_->writer,
calibration.timestamp_ns,
@@ -959,9 +1012,14 @@ std::expected<void, std::string> validate_new_stream_config(
if (config.topic.empty()) {
return std::unexpected("video topic is empty");
}
if (config.depth_topic.empty()) {
return std::unexpected("depth topic is empty");
#if CVMMAP_STREAMER_HAS_MCAP_DEPTH
const bool depth_enabled = !config.depth_topic.empty();
#else
if (!config.depth_topic.empty()) {
return std::unexpected("MCAP depth recording support is not compiled into this build");
}
constexpr bool depth_enabled = false;
#endif
if (config.frame_id.empty()) {
return std::unexpected("frame_id is empty");
}
@@ -984,9 +1042,11 @@ std::expected<void, std::string> validate_new_stream_config(
};
std::vector<std::string> config_topics{};
config_topics.push_back(config.topic);
if (depth_enabled) {
config_topics.push_back(config.depth_topic);
}
for (const auto *topic : {
&config.topic,
&config.depth_topic,
&config.calibration_topic,
&config.depth_calibration_topic,
&config.pose_topic,
@@ -1003,16 +1063,9 @@ std::expected<void, std::string> validate_new_stream_config(
}
}
for (const auto *topic : {
&config.topic,
&config.depth_topic,
&config.calibration_topic,
&config.depth_calibration_topic,
&config.pose_topic,
&config.body_topic,
}) {
if (!topic->empty() && topic_in_use(*topic)) {
return std::unexpected("duplicate MCAP topic: " + *topic);
for (const auto &topic : config_topics) {
if (topic_in_use(topic)) {
return std::unexpected("duplicate MCAP topic: " + topic);
}
}
return {};
@@ -1063,6 +1116,7 @@ std::expected<MultiMcapRecordSink, std::string> MultiMcapRecordSink::create(
state->writer.addSchema(video_schema);
state->video_schema_id = video_schema.id;
#if CVMMAP_STREAMER_HAS_MCAP_DEPTH
const auto depth_descriptor_set = build_file_descriptor_set(cvmmap_streamer::DepthMap::descriptor());
std::string depth_schema_bytes{};
if (!depth_descriptor_set.SerializeToString(&depth_schema_bytes)) {
@@ -1071,6 +1125,7 @@ std::expected<MultiMcapRecordSink, std::string> MultiMcapRecordSink::create(
mcap::Schema depth_schema("cvmmap_streamer.DepthMap", "protobuf", depth_schema_bytes);
state->writer.addSchema(depth_schema);
state->depth_schema_id = depth_schema.id;
#endif
if (!state->bundle_topic.empty()) {
const auto bundle_descriptor_set = build_file_descriptor_set(cvmmap_streamer::BundleManifest::descriptor());
@@ -1126,9 +1181,13 @@ std::expected<MultiMcapRecordSink::StreamId, std::string> MultiMcapRecordSink::a
state_->writer.addChannel(video_channel);
stream.video_channel_id = video_channel.id;
mcap::Channel depth_channel(config.depth_topic, "protobuf", state_->depth_schema_id);
state_->writer.addChannel(depth_channel);
stream.depth_channel_id = depth_channel.id;
#if CVMMAP_STREAMER_HAS_MCAP_DEPTH
if (!config.depth_topic.empty()) {
mcap::Channel depth_channel(config.depth_topic, "protobuf", state_->depth_schema_id);
state_->writer.addChannel(depth_channel);
stream.depth_channel_id = depth_channel.id;
}
#endif
if (auto updated = update_stream_info_impl(stream, stream_info); !updated) {
return std::unexpected(updated.error());
@@ -1200,7 +1259,13 @@ std::expected<void, std::string> MultiMcapRecordSink::write_depth_map(
if (stream == nullptr) {
return std::unexpected(error);
}
if (stream->depth_channel_id == 0) {
return std::unexpected("MCAP depth recording support is not enabled for this stream");
}
#if !CVMMAP_STREAMER_HAS_MCAP_DEPTH
static_cast<void>(depth_map);
return std::unexpected("MCAP depth recording support is not compiled into this build");
#else
const auto source_unit = normalize_depth_source_unit(depth_map.source_unit);
auto encoded = encode_depth_payload(depth_map);
if (!encoded) {
@@ -1217,6 +1282,7 @@ std::expected<void, std::string> MultiMcapRecordSink::write_depth_map(
stream->depth_channel_id,
stream->depth_sequence,
*encoded);
#endif
}
std::expected<void, std::string> MultiMcapRecordSink::write_depth_map_u16(
@@ -1227,7 +1293,13 @@ std::expected<void, std::string> MultiMcapRecordSink::write_depth_map_u16(
if (stream == nullptr) {
return std::unexpected(error);
}
if (stream->depth_channel_id == 0) {
return std::unexpected("MCAP depth recording support is not enabled for this stream");
}
#if !CVMMAP_STREAMER_HAS_MCAP_DEPTH
static_cast<void>(depth_map);
return std::unexpected("MCAP depth recording support is not compiled into this build");
#else
auto encoded = encode_depth_payload(depth_map);
if (!encoded) {
return std::unexpected(encoded.error());
@@ -1243,6 +1315,7 @@ std::expected<void, std::string> MultiMcapRecordSink::write_depth_map_u16(
stream->depth_channel_id,
stream->depth_sequence,
*encoded);
#endif
}
std::expected<void, std::string> MultiMcapRecordSink::write_camera_calibration(
@@ -1272,6 +1345,9 @@ std::expected<void, std::string> MultiMcapRecordSink::write_depth_camera_calibra
if (stream == nullptr) {
return std::unexpected(error);
}
if (stream->depth_channel_id == 0) {
return std::unexpected("MCAP depth recording support is not enabled for this stream");
}
return write_calibration_message(
state_->writer,
calibration.timestamp_ns,
@@ -1407,4 +1483,91 @@ void MultiMcapRecordSink::close() {
state_ = nullptr;
}
#else
namespace {
[[nodiscard]] std::string mcap_unavailable_error() {
return "MCAP recording support is not compiled into this build";
}
[[nodiscard]] std::string mcap_depth_unavailable_error() {
return "MCAP depth recording support is not compiled into this build";
}
} // namespace
McapRecordSink::~McapRecordSink() = default;
McapRecordSink::McapRecordSink(McapRecordSink &&other) noexcept = default;
McapRecordSink &McapRecordSink::operator=(McapRecordSink &&other) noexcept = default;
std::expected<McapRecordSink, std::string> McapRecordSink::create(
const RuntimeConfig &,
const encode::EncodedStreamInfo &) {
return std::unexpected(mcap_unavailable_error());
}
std::expected<void, std::string> McapRecordSink::update_stream_info(const encode::EncodedStreamInfo &) {
return std::unexpected(mcap_unavailable_error());
}
std::expected<void, std::string> McapRecordSink::write_access_unit(const encode::EncodedAccessUnit &) {
return std::unexpected(mcap_unavailable_error());
}
std::expected<void, std::string> McapRecordSink::write_depth_map(const RawDepthMapView &) {
return std::unexpected(mcap_depth_unavailable_error());
}
std::expected<void, std::string> McapRecordSink::write_depth_map_u16(const RawDepthMapU16View &) {
return std::unexpected(mcap_depth_unavailable_error());
}
std::expected<void, std::string> McapRecordSink::write_camera_calibration(const RawCameraCalibrationView &) {
return std::unexpected(mcap_unavailable_error());
}
std::expected<void, std::string> McapRecordSink::write_depth_camera_calibration(const RawCameraCalibrationView &) {
return std::unexpected(mcap_depth_unavailable_error());
}
std::expected<void, std::string> McapRecordSink::write_pose(const RawPoseView &) {
return std::unexpected(mcap_unavailable_error());
}
std::expected<void, std::string> McapRecordSink::write_body_tracking_message(const RawBodyTrackingMessageView &) {
return std::unexpected(mcap_unavailable_error());
}
bool McapRecordSink::is_open() const {
return false;
}
std::string_view McapRecordSink::path() const {
return {};
}
void McapRecordSink::close() {}
MultiMcapRecordSink::~MultiMcapRecordSink() = default;
MultiMcapRecordSink::MultiMcapRecordSink(MultiMcapRecordSink &&other) noexcept = default;
MultiMcapRecordSink &MultiMcapRecordSink::operator=(MultiMcapRecordSink &&other) noexcept = default;
std::expected<MultiMcapRecordSink, std::string> MultiMcapRecordSink::create(
std::string,
McapCompression,
std::string) {
return std::unexpected(mcap_unavailable_error());
}
std::expected<MultiMcapRecordSink::StreamId, std::string> MultiMcapRecordSink::add_stream(
const McapRecordStreamConfig &,
const encode::EncodedStreamInfo &) {
return std::unexpected(const {
return {};
}
void MultiMcapRecordSink::close() {}
#endif
}
+56 -100
View File
@@ -1,4 +1,5 @@
#include "cvmmap_streamer/record/mp4_record_writer.hpp"
#include "../encode/ffmpeg_encoder_options.hpp"
extern "C" {
#include <libavcodec/avcodec.h>
@@ -23,11 +24,6 @@ namespace {
inline constexpr std::uint64_t kNanosPerSecond = 1'000'000'000ull;
struct EncoderCandidate {
std::string name{};
bool using_hardware{false};
AVPixelFormat pixel_format{AV_PIX_FMT_NONE};
};
struct ResolvedEncoderSettings {
std::string mapped_preset{};
@@ -99,52 +95,6 @@ std::uint64_t frame_period_ns(const AVRational frame_rate) {
}
[[nodiscard]]
std::vector<EncoderCandidate> encoder_candidates(const CodecType codec, const EncoderDeviceType device) {
const std::string hardware_name = codec == CodecType::H265 ? "hevc_nvenc" : "h264_nvenc";
const std::string software_name = codec == CodecType::H265 ? "libx265" : "libx264";
switch (device) {
case EncoderDeviceType::Auto:
return {
EncoderCandidate{.name = hardware_name, .using_hardware = true, .pixel_format = AV_PIX_FMT_NV12},
EncoderCandidate{.name = software_name, .using_hardware = false, .pixel_format = AV_PIX_FMT_YUV420P},
};
case EncoderDeviceType::Nvidia:
return {
EncoderCandidate{.name = hardware_name, .using_hardware = true, .pixel_format = AV_PIX_FMT_NV12},
};
case EncoderDeviceType::Software:
return {
EncoderCandidate{.name = software_name, .using_hardware = false, .pixel_format = AV_PIX_FMT_YUV420P},
};
}
return {};
}
[[nodiscard]]
std::string mapped_preset_value(const EncoderCandidate &candidate) {
return candidate.using_hardware ? "p1" : "veryfast";
}
[[nodiscard]]
std::optional<std::string> mapped_tune_value(const EncoderCandidate &candidate) {
if (candidate.using_hardware) {
return std::optional<std::string>{"ull"};
}
if (candidate.name == "libx264") {
return std::optional<std::string>{"zerolatency"};
}
return std::nullopt;
}
[[nodiscard]]
std::optional<std::string> x265_params_value(const EncoderCandidate &candidate) {
if (candidate.name != "libx265") {
return std::nullopt;
}
return std::optional<std::string>{"repeat-headers=1:scenecut=0"};
}
[[nodiscard]]
std::expected<void, std::string> set_string_option(AVCodecContext *context, const char *key, const std::string &value) {
@@ -167,7 +117,7 @@ std::expected<void, std::string> set_int_option(AVCodecContext *context, const c
[[nodiscard]]
std::expected<ResolvedEncoderSettings, std::string> configure_codec_context(
AVCodecContext *context,
const EncoderCandidate &candidate,
const encode::FfmpegEncoderCandidate &candidate,
const CodecType codec,
const std::uint32_t width,
const std::uint32_t height,
@@ -186,51 +136,52 @@ std::expected<ResolvedEncoderSettings, std::string> configure_codec_context(
context->thread_count = 1;
ResolvedEncoderSettings resolved{
.mapped_preset = mapped_preset_value(candidate),
.mapped_tune = mapped_tune_value(candidate),
.quality_value = tuning.quality,
.gop = tuning.gop,
.b_frames = tuning.b_frames,
};
if (auto set = set_string_option(context, "preset", resolved.mapped_preset); !set) {
return std::unexpected(set.error());
if (const auto preset = encode::ffmpeg_encoder_preset(candidate); preset) {
resolved.mapped_preset = std::string(*preset);
if (auto set = set_string_option(context, "preset", resolved.mapped_preset); !set) {
return std::unexpected(set.error());
}
}
if (resolved.mapped_tune) {
if (const auto tune = encode::ffmpeg_encoder_tune(candidate); tune) {
resolved.mapped_tune = std::string(*tune);
if (auto set = set_string_option(context, "tune", *resolved.mapped_tune); !set) {
return std::unexpected(set.error());
}
}
if (candidate.using_hardware) {
resolved.rate_control_mode = "vbr";
resolved.quality_key = "cq";
if (const auto rc_mode = encode::ffmpeg_encoder_rate_control_mode(candidate); rc_mode) {
resolved.rate_control_mode = std::string(*rc_mode);
if (auto set = set_string_option(context, "rc", *resolved.rate_control_mode); !set) {
return std::unexpected(set.error());
}
if (auto set = set_int_option(context, "cq", resolved.quality_value); !set) {
}
if (const auto quality_key = encode::ffmpeg_encoder_quality_key(candidate); quality_key) {
resolved.quality_key = std::string(*quality_key);
if (auto set = set_int_option(context, resolved.quality_key.c_str(), resolved.quality_value); !set) {
return std::unexpected(set.error());
}
}
if (const auto x265_params = encode::ffmpeg_encoder_x265_params(candidate); x265_params) {
if (auto set = set_string_option(context, "x265-params", std::string(*x265_params)); !set) {
return std::unexpected(set.error());
}
}
if (encode::ffmpeg_encoder_supports_nvenc_latency_flags(candidate)) {
if (auto set = set_string_option(context, "zerolatency", "1"); !set) {
return std::unexpected(set.error());
}
if (auto set = set_string_option(context, "rc-lookahead", "0"); !set) {
return std::unexpected(set.error());
}
} else {
resolved.quality_key = "crf";
if (auto set = set_int_option(context, "crf", resolved.quality_value); !set) {
}
if (encode::ffmpeg_encoder_supports_forced_idr_option(candidate)) {
if (auto set = set_int_option(context, "forced-idr", 1); !set) {
return std::unexpected(set.error());
}
if (const auto x265_params = x265_params_value(candidate); x265_params) {
if (auto set = set_string_option(context, "x265-params", *x265_params); !set) {
return std::unexpected(set.error());
}
}
}
if (auto set = set_int_option(context, "forced-idr", 1); !set) {
return std::unexpected(set.error());
}
return resolved;
@@ -238,7 +189,7 @@ std::expected<ResolvedEncoderSettings, std::string> configure_codec_context(
struct OpenedEncoder {
AVCodecContext *context{nullptr};
EncoderCandidate candidate{};
encode::FfmpegEncoderCandidate candidate{};
ResolvedEncoderSettings resolved{};
};
@@ -250,19 +201,19 @@ std::expected<OpenedEncoder, std::string> open_encoder(
const std::uint32_t height,
const AVRational framerate,
const Mp4EncodeTuning &tuning) {
const auto candidates = encode::ffmpeg_encoder_candidates(codec, device);
const auto attempted_candidates = encode::ffmpeg_encoder_candidate_list(candidates);
std::string last_error{};
for (const auto &candidate : encoder_candidates(codec, device)) {
const auto *encoder = avcodec_find_encoder_by_name(candidate.name.c_str());
for (const auto &candidate : candidates) {
const auto *encoder = avcodec_find_encoder_by_name(candidate.name.data());
if (encoder == nullptr) {
last_error = "FFmpeg encoder '" + candidate.name + "' is unavailable";
if (device == EncoderDeviceType::Auto) {
spdlog::warn(
"MP4 encoder '{}' unavailable in auto mode, trying next candidate",
candidate.name);
continue;
}
return std::unexpected(last_error);
last_error = "FFmpeg encoder '" + std::string(candidate.name) + "' is unavailable";
spdlog::warn(
"MP4 encoder '{}' unavailable in {} mode, trying next candidate",
candidate.name,
to_string(device));
continue;
}
auto *context = avcodec_alloc_context3(encoder);
@@ -272,22 +223,26 @@ std::expected<OpenedEncoder, std::string> open_encoder(
auto resolved = configure_codec_context(context, candidate, codec, width, height, framerate, tuning);
if (!resolved) {
last_error = resolved.error();
avcodec_free_context(&context);
return std::unexpected(resolved.error());
spdlog::warn(
"MP4 encoder '{}' configuration failed in {} mode: {}. trying next candidate",
candidate.name,
to_string(device),
resolved.error());
continue;
}
const auto open_result = avcodec_open2(context, encoder, nullptr);
if (open_result < 0) {
last_error = "failed to open FFmpeg encoder '" + candidate.name + "': " + av_error_string(open_result);
last_error = "failed to open FFmpeg encoder '" + std::string(candidate.name) + "': " + av_error_string(open_result);
avcodec_free_context(&context);
if (device == EncoderDeviceType::Auto) {
spdlog::warn(
"MP4 encoder '{}' failed to open in auto mode: {}. trying software fallback",
candidate.name,
av_error_string(open_result));
continue;
}
return std::unexpected(last_error);
spdlog::warn(
"MP4 encoder '{}' failed to open in {} mode: {}. trying next candidate",
candidate.name,
to_string(device),
av_error_string(open_result));
continue;
}
return OpenedEncoder{
@@ -298,9 +253,9 @@ std::expected<OpenedEncoder, std::string> open_encoder(
}
if (last_error.empty()) {
last_error = "no usable FFmpeg MP4 encoder candidates were configured";
last_error = "no usable FFmpeg encoder found";
}
return std::unexpected(last_error);
return std::unexpected(last_error + " (attempted: " + attempted_candidates + ")");
}
} // namespace
@@ -414,8 +369,10 @@ struct Mp4RecordWriter::Impl {
return std::unexpected("failed to write MP4 header: " + av_error_string(header_result));
}
const auto quality_key = resolved_settings.quality_key.empty() ? std::string("auto") : resolved_settings.quality_key;
const auto quality_value = resolved_settings.quality_key.empty() ? std::string("n/a") : std::to_string(resolved_settings.quality_value);
spdlog::info(
"MP4_RECORD_READY codec={} encoder={} hardware={} width={} height={} fps={}/{} rc={} {}={} gop={} b_frames={} input={} output={}",
"MP4_RECORD_READY codec={} encoder={} hardware={} width={} height={} fps={}/{} rc={} quality={} gop={} b_frames={} input={} output={}",
cvmmap_streamer::to_string(codec),
encoder_name,
using_hardware,
@@ -424,8 +381,7 @@ struct Mp4RecordWriter::Impl {
frame_rate.num,
frame_rate.den,
resolved_settings.rate_control_mode.value_or("auto"),
resolved_settings.quality_key,
resolved_settings.quality_value,
quality_key + "=" + quality_value,
resolved_settings.gop,
resolved_settings.b_frames,
input_pixel_format_name(input_pixel_format),
-1
View File
@@ -77,7 +77,6 @@ int main(int argc, char **argv) {
config.record.mcap.enabled = true;
config.record.mcap.path = output_path.string();
config.record.mcap.topic = "/camera/video";
config.record.mcap.depth_topic = "/camera/depth";
config.record.mcap.body_topic = std::string(kBodyTopic);
config.record.mcap.frame_id = "camera";
config.record.mcap.compression = cvmmap_streamer::McapCompression::None;
+1 -53
View File
@@ -1,14 +1,12 @@
#include <mcap/reader.hpp>
#include "proto/cvmmap_streamer/BundleManifest.pb.h"
#include "proto/cvmmap_streamer/DepthMap.pb.h"
#include "cvmmap_streamer/common.h"
#include "cvmmap_streamer/record/mcap_record_sink.hpp"
#include "proto/foxglove/CameraCalibration.pb.h"
#include "proto/foxglove/CompressedVideo.pb.h"
#include "proto/foxglove/PoseInFrame.pb.h"
#include <rvl/rvl.hpp>
#include <spdlog/spdlog.h>
#include <cstdint>
@@ -86,18 +84,14 @@ int main(int argc, char **argv) {
auto zed1 = sink->add_stream(cvmmap_streamer::record::McapRecordStreamConfig{
.topic = "/zed1/video",
.depth_topic = "/zed1/depth",
.calibration_topic = "/zed1/calibration",
.depth_calibration_topic = "/zed1/depth_calibration",
.pose_topic = "/zed1/pose",
.body_topic = "/zed1/body",
.frame_id = "zed1",
}, stream_info);
auto zed2 = sink->add_stream(cvmmap_streamer::record::McapRecordStreamConfig{
.topic = "/zed2/video",
.depth_topic = "/zed2/depth",
.calibration_topic = "/zed2/calibration",
.depth_calibration_topic = "/zed2/depth_calibration",
.pose_topic = "/zed2/pose",
.body_topic = "/zed2/body",
.frame_id = "zed2",
@@ -107,10 +101,6 @@ int main(int argc, char **argv) {
return exit_code(TesterExitCode::CreateError);
}
const std::vector<std::uint16_t> depth_pixels{
1000, 2000,
1500, 2500,
};
const std::vector<double> distortion{0.0, 0.0, 0.0, 0.0, 0.0};
const std::vector<double> intrinsic_matrix{
500.0, 0.0, 320.0,
@@ -168,15 +158,6 @@ int main(int argc, char **argv) {
return exit_code(TesterExitCode::WriteError);
}
if (auto write = sink->write_depth_map_u16(stream_id, cvmmap_streamer::record::RawDepthMapU16View{
.timestamp_ns = 100,
.width = 2,
.height = 2,
.pixels = depth_pixels,
}); !write) {
spdlog::error("failed to write depth map: {}", write.error());
return exit_code(TesterExitCode::WriteError);
}
if (auto write = sink->write_camera_calibration(stream_id, cvmmap_streamer::record::RawCameraCalibrationView{
.timestamp_ns = 100,
@@ -191,19 +172,7 @@ int main(int argc, char **argv) {
spdlog::error("failed to write calibration: {}", write.error());
return exit_code(TesterExitCode::WriteError);
}
if (auto write = sink->write_depth_camera_calibration(stream_id, cvmmap_streamer::record::RawCameraCalibrationView{
.timestamp_ns = 100,
.width = 320,
.height = 240,
.distortion_model = "plumb_bob",
.distortion = distortion,
.intrinsic_matrix = intrinsic_matrix,
.rectification_matrix = rectification_matrix,
.projection_matrix = projection_matrix,
}); !write) {
spdlog::error("failed to write depth calibration: {}", write.error());
return exit_code(TesterExitCode::WriteError);
}
if (auto write = sink->write_pose(stream_id, cvmmap_streamer::record::RawPoseView{
.timestamp_ns = 100,
@@ -251,23 +220,6 @@ int main(int argc, char **argv) {
continue;
}
if (it->schema->name == "cvmmap_streamer.DepthMap") {
cvmmap_streamer::DepthMap depth{};
if (!depth.ParseFromArray(it->message.data, static_cast<int>(it->message.dataSize))) {
spdlog::error("failed to parse cvmmap_streamer.DepthMap");
reader.close();
return exit_code(TesterExitCode::VerificationError);
}
const auto decoded = rvl::decompress_image(std::span<const std::uint8_t>(
reinterpret_cast<const std::uint8_t *>(depth.data().data()),
depth.data().size()));
if (decoded.pixels.size() != depth_pixels.size()) {
spdlog::error("decoded depth pixel count mismatch");
reader.close();
return exit_code(TesterExitCode::VerificationError);
}
continue;
}
if (it->schema->name == "cvmmap_streamer.BundleManifest") {
cvmmap_streamer::BundleManifest bundle{};
@@ -319,14 +271,10 @@ int main(int argc, char **argv) {
for (const auto &topic : {
"/bundle",
"/zed1/video",
"/zed1/depth",
"/zed1/calibration",
"/zed1/depth_calibration",
"/zed1/pose",
"/zed2/video",
"/zed2/depth",
"/zed2/calibration",
"/zed2/depth_calibration",
"/zed2/pose",
}) {
if (topic_counts[topic] != 1) {
+3 -50
View File
@@ -1,13 +1,11 @@
#include <mcap/reader.hpp>
#include "proto/cvmmap_streamer/DepthMap.pb.h"
#include "cvmmap_streamer/common.h"
#include "cvmmap_streamer/record/mcap_record_sink.hpp"
#include "proto/foxglove/CameraCalibration.pb.h"
#include "proto/foxglove/CompressedVideo.pb.h"
#include "proto/foxglove/PoseInFrame.pb.h"
#include <rvl/rvl.hpp>
#include <spdlog/spdlog.h>
#include <cmath>
@@ -56,7 +54,6 @@ int main(int argc, char **argv) {
config.record.mcap.enabled = true;
config.record.mcap.path = output_path.string();
config.record.mcap.topic = "/camera/video";
config.record.mcap.depth_topic = "/camera/depth";
config.record.mcap.calibration_topic = "/camera/calibration";
config.record.mcap.pose_topic = "/camera/pose";
config.record.mcap.frame_id = "camera";
@@ -82,19 +79,6 @@ int main(int argc, char **argv) {
return exit_code(TesterExitCode::WriteError);
}
const std::vector<std::uint16_t> depth_pixels{
1000, 2000,
0, 1500,
};
if (auto write = sink->write_depth_map_u16(cvmmap_streamer::record::RawDepthMapU16View{
.timestamp_ns = 101,
.width = 2,
.height = 2,
.pixels = depth_pixels,
}); !write) {
spdlog::error("failed to write depth map: {}", write.error());
return exit_code(TesterExitCode::WriteError);
}
const std::vector<double> distortion{0.0, 0.0, 0.0, 0.0, 0.0};
const std::vector<double> intrinsic_matrix{
@@ -146,7 +130,6 @@ int main(int argc, char **argv) {
}
std::uint64_t video_messages{0};
std::uint64_t depth_messages{0};
std::uint64_t calibration_messages{0};
std::uint64_t pose_messages{0};
@@ -179,35 +162,6 @@ int main(int argc, char **argv) {
continue;
}
if (it->schema->name == "cvmmap_streamer.DepthMap") {
cvmmap_streamer::DepthMap depth{};
if (!depth.ParseFromArray(it->message.data, static_cast<int>(it->message.dataSize))) {
spdlog::error("failed to parse cvmmap_streamer.DepthMap");
reader.close();
return exit_code(TesterExitCode::VerificationError);
}
if (it->channel->topic != "/camera/depth" ||
depth.frame_id() != "camera" ||
depth.encoding() != cvmmap_streamer::DepthMap::RVL_U16_LOSSLESS) {
spdlog::error("depth message metadata verification failed");
reader.close();
return exit_code(TesterExitCode::VerificationError);
}
const auto decoded = rvl::decompress_image(std::span<const std::uint8_t>(
reinterpret_cast<const std::uint8_t *>(depth.data().data()),
depth.data().size()));
if (decoded.pixels.size() != depth_pixels.size() ||
decoded.pixels[0] != 1000 ||
decoded.pixels[1] != 2000 ||
decoded.pixels[2] != 0 ||
decoded.pixels[3] != 1500) {
spdlog::error("depth RVL round-trip verification failed");
reader.close();
return exit_code(TesterExitCode::VerificationError);
}
depth_messages += 1;
continue;
}
if (it->schema->name == "foxglove.CameraCalibration") {
foxglove::CameraCalibration calibration{};
@@ -262,18 +216,17 @@ int main(int argc, char **argv) {
reader.close();
if (video_messages != 1 || depth_messages != 1 || calibration_messages != 1 || pose_messages != 1) {
if (video_messages != 1 || calibration_messages != 1 || pose_messages != 1) {
spdlog::error(
"unexpected message counts: video={} depth={} calibration={} pose={}",
"unexpected message counts: video={} calibration={} pose={}",
video_messages,
depth_messages,
calibration_messages,
pose_messages);
return exit_code(TesterExitCode::VerificationError);
}
spdlog::info(
"validated same-file MCAP video+depth+calibration+pose recording at '{}'",
"validated same-file MCAP video+calibration+pose recording at '{}'",
output_path.string());
return exit_code(TesterExitCode::Success);
}
+2 -2
View File
@@ -52,7 +52,7 @@ struct Config {
[[nodiscard]]
std::expected<Config, int> parse_args(int argc, char **argv) {
Config config{};
CLI::App app{"rtmp_output_tester - publish synthetic encoded video to RTMP using the configured sink"};
CLI::App app{"rtmp_output_tester - publish synthetic encoded video to RTMP using the runtime encoder selection path"};
app.add_option("--rtmp-url", config.rtmp_url, "RTMP destination URL")->required();
app.add_option("--transport", config.transport, "RTMP transport backend (libavformat|ffmpeg_process)")
->check(CLI::IsMember({"libavformat", "ffmpeg_process"}));
@@ -150,7 +150,7 @@ int main(int argc, char **argv) {
}
cvmmap_streamer::RuntimeConfig config = cvmmap_streamer::RuntimeConfig::defaults();
config.encoder.backend = cvmmap_streamer::EncoderBackendType::FFmpeg;
config.encoder.backend = cvmmap_streamer::EncoderBackendType::Auto;
config.encoder.device = *encoder_device;
config.encoder.codec = *codec;
config.encoder.gop = 15;
+2 -2
View File
@@ -48,7 +48,7 @@ struct Config {
[[nodiscard]]
std::expected<Config, int> parse_args(int argc, char **argv) {
Config config{};
CLI::App app{"rtp_output_tester - publish synthetic encoded video to RTP using the FFmpeg encoder path"};
CLI::App app{"rtp_output_tester - publish synthetic encoded video to RTP using the runtime encoder selection path"};
app.add_option("--host", config.host, "RTP destination host")->required();
app.add_option("--port", config.port, "RTP destination port")->required()->check(CLI::Range(1, 65535));
app.add_option("--payload-type", config.payload_type, "RTP payload type (96-127)")->check(CLI::Range(96, 127));
@@ -128,7 +128,7 @@ int main(int argc, char **argv) {
}
cvmmap_streamer::RuntimeConfig config = cvmmap_streamer::RuntimeConfig::defaults();
config.encoder.backend = cvmmap_streamer::EncoderBackendType::FFmpeg;
config.encoder.backend = cvmmap_streamer::EncoderBackendType::Auto;
config.encoder.device = *encoder_device;
config.encoder.codec = *codec;
config.encoder.gop = 15;