fix: avoid zstd segfault in vendored mcap writer
Replace the vendored MCAP Zstd chunk writer's ZSTD_compress2-based path with ZSTD_compressCCtx and keep the selected compression level on the wrapper state. The previous implementation crashed in Debug and real zed_svo_to_mcap runs on this machine when the first compressed chunk was flushed with --mcap-compression zstd. The same exports succeeded with none and lz4, which narrowed the failure to the shared vendored MCAP Zstd path rather than exporter sync logic. Also extend mcap_multi_record_tester to accept a compression argument so the Zstd path can be exercised directly during regression testing. Verified with: - ./build-debug/bin/mcap_multi_record_tester /tmp/mcap_multi_zstd_test.mcap zstd - ./build-debug/bin/zed_svo_to_mcap --input jump/experiment/2/2026-03-18T11-27-15/2026-03-18T11-27-15_zed1.svo2 --output /tmp/zed1_single_zstd_fixed.mcap --mcap-compression zstd --end-frame 1 - ./build-debug/bin/zed_svo_to_mcap --segment-dir jump/experiment/2/2026-03-18T11-27-15 --output /tmp/multi_zstd_fixed.mcap --mcap-compression zstd --end-frame 1
This commit is contained in:
@@ -13,7 +13,9 @@
|
||||
#include <cstdint>
|
||||
#include <filesystem>
|
||||
#include <map>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
namespace {
|
||||
@@ -31,6 +33,20 @@ constexpr int exit_code(const TesterExitCode code) {
|
||||
return static_cast<int>(code);
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
std::optional<cvmmap_streamer::McapCompression> parse_compression(std::string_view raw) {
|
||||
if (raw == "none") {
|
||||
return cvmmap_streamer::McapCompression::None;
|
||||
}
|
||||
if (raw == "lz4") {
|
||||
return cvmmap_streamer::McapCompression::Lz4;
|
||||
}
|
||||
if (raw == "zstd") {
|
||||
return cvmmap_streamer::McapCompression::Zstd;
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
@@ -43,13 +59,21 @@ int main(int argc, char **argv) {
|
||||
argc > 1
|
||||
? std::filesystem::path(argv[1])
|
||||
: std::filesystem::temp_directory_path() / "cvmmap_streamer_multi_record_test.mcap";
|
||||
const auto compression =
|
||||
argc > 2
|
||||
? parse_compression(argv[2]).value_or(cvmmap_streamer::McapCompression::None)
|
||||
: cvmmap_streamer::McapCompression::None;
|
||||
if (argc > 2 && !parse_compression(argv[2])) {
|
||||
spdlog::error("invalid compression '{}': expected none|lz4|zstd", argv[2]);
|
||||
return exit_code(TesterExitCode::CreateError);
|
||||
}
|
||||
if (output_path.has_parent_path()) {
|
||||
std::filesystem::create_directories(output_path.parent_path());
|
||||
}
|
||||
|
||||
auto sink = cvmmap_streamer::record::MultiMcapRecordSink::create(
|
||||
output_path.string(),
|
||||
cvmmap_streamer::McapCompression::None);
|
||||
compression);
|
||||
if (!sink) {
|
||||
spdlog::error("failed to create MCAP sink: {}", sink.error());
|
||||
return exit_code(TesterExitCode::CreateError);
|
||||
|
||||
+1
@@ -312,6 +312,7 @@ public:
|
||||
private:
|
||||
std::vector<std::byte> uncompressedBuffer_;
|
||||
std::vector<std::byte> compressedBuffer_;
|
||||
CompressionLevel compressionLevel_ = CompressionLevel::Default;
|
||||
ZSTD_CCtx_s* zstdContext_ = nullptr;
|
||||
};
|
||||
#endif
|
||||
|
||||
+11
-7
@@ -241,10 +241,13 @@ int ZStdCompressionLevel(CompressionLevel level) {
|
||||
|
||||
// ZStdWriter //////////////////////////////////////////////////////////////////
|
||||
|
||||
ZStdWriter::ZStdWriter(CompressionLevel compressionLevel, uint64_t chunkSize) {
|
||||
ZStdWriter::ZStdWriter(CompressionLevel compressionLevel, uint64_t chunkSize)
|
||||
: compressionLevel_(compressionLevel) {
|
||||
zstdContext_ = ZSTD_createCCtx();
|
||||
ZSTD_CCtx_setParameter(zstdContext_, ZSTD_c_compressionLevel,
|
||||
internal::ZStdCompressionLevel(compressionLevel));
|
||||
if (zstdContext_ == nullptr) {
|
||||
std::cerr << "ZSTD_createCCtx failed\n";
|
||||
std::abort();
|
||||
}
|
||||
uncompressedBuffer_.reserve(chunkSize);
|
||||
}
|
||||
|
||||
@@ -259,15 +262,16 @@ void ZStdWriter::handleWrite(const std::byte* data, uint64_t size) {
|
||||
void ZStdWriter::end() {
|
||||
const auto dstCapacity = ZSTD_compressBound(uncompressedBuffer_.size());
|
||||
compressedBuffer_.resize(dstCapacity);
|
||||
const size_t dstSize = ZSTD_compress2(zstdContext_, compressedBuffer_.data(), dstCapacity,
|
||||
uncompressedBuffer_.data(), uncompressedBuffer_.size());
|
||||
const size_t dstSize =
|
||||
ZSTD_compressCCtx(zstdContext_, compressedBuffer_.data(), dstCapacity,
|
||||
uncompressedBuffer_.data(), uncompressedBuffer_.size(),
|
||||
internal::ZStdCompressionLevel(compressionLevel_));
|
||||
if (ZSTD_isError(dstSize)) {
|
||||
const auto errCode = ZSTD_getErrorCode(dstSize);
|
||||
std::cerr << "ZSTD_compress2 failed: " << ZSTD_getErrorName(dstSize) << " ("
|
||||
std::cerr << "ZSTD_compressCCtx failed: " << ZSTD_getErrorName(dstSize) << " ("
|
||||
<< ZSTD_getErrorString(errCode) << ")\n";
|
||||
std::abort();
|
||||
}
|
||||
ZSTD_CCtx_reset(zstdContext_, ZSTD_reset_session_only);
|
||||
compressedBuffer_.resize(dstSize);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user