fix: avoid zstd segfault in vendored mcap writer
Replace the vendored MCAP Zstd chunk writer's ZSTD_compress2-based path with ZSTD_compressCCtx and keep the selected compression level on the wrapper state. The previous implementation crashed in Debug and real zed_svo_to_mcap runs on this machine when the first compressed chunk was flushed with --mcap-compression zstd. The same exports succeeded with none and lz4, which narrowed the failure to the shared vendored MCAP Zstd path rather than exporter sync logic. Also extend mcap_multi_record_tester to accept a compression argument so the Zstd path can be exercised directly during regression testing. Verified with: - ./build-debug/bin/mcap_multi_record_tester /tmp/mcap_multi_zstd_test.mcap zstd - ./build-debug/bin/zed_svo_to_mcap --input jump/experiment/2/2026-03-18T11-27-15/2026-03-18T11-27-15_zed1.svo2 --output /tmp/zed1_single_zstd_fixed.mcap --mcap-compression zstd --end-frame 1 - ./build-debug/bin/zed_svo_to_mcap --segment-dir jump/experiment/2/2026-03-18T11-27-15 --output /tmp/multi_zstd_fixed.mcap --mcap-compression zstd --end-frame 1
This commit is contained in:
@@ -13,7 +13,9 @@
|
|||||||
#include <cstdint>
|
#include <cstdint>
|
||||||
#include <filesystem>
|
#include <filesystem>
|
||||||
#include <map>
|
#include <map>
|
||||||
|
#include <optional>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
#include <string_view>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
@@ -31,6 +33,20 @@ constexpr int exit_code(const TesterExitCode code) {
|
|||||||
return static_cast<int>(code);
|
return static_cast<int>(code);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[[nodiscard]]
|
||||||
|
std::optional<cvmmap_streamer::McapCompression> parse_compression(std::string_view raw) {
|
||||||
|
if (raw == "none") {
|
||||||
|
return cvmmap_streamer::McapCompression::None;
|
||||||
|
}
|
||||||
|
if (raw == "lz4") {
|
||||||
|
return cvmmap_streamer::McapCompression::Lz4;
|
||||||
|
}
|
||||||
|
if (raw == "zstd") {
|
||||||
|
return cvmmap_streamer::McapCompression::Zstd;
|
||||||
|
}
|
||||||
|
return std::nullopt;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
int main(int argc, char **argv) {
|
int main(int argc, char **argv) {
|
||||||
@@ -43,13 +59,21 @@ int main(int argc, char **argv) {
|
|||||||
argc > 1
|
argc > 1
|
||||||
? std::filesystem::path(argv[1])
|
? std::filesystem::path(argv[1])
|
||||||
: std::filesystem::temp_directory_path() / "cvmmap_streamer_multi_record_test.mcap";
|
: std::filesystem::temp_directory_path() / "cvmmap_streamer_multi_record_test.mcap";
|
||||||
|
const auto compression =
|
||||||
|
argc > 2
|
||||||
|
? parse_compression(argv[2]).value_or(cvmmap_streamer::McapCompression::None)
|
||||||
|
: cvmmap_streamer::McapCompression::None;
|
||||||
|
if (argc > 2 && !parse_compression(argv[2])) {
|
||||||
|
spdlog::error("invalid compression '{}': expected none|lz4|zstd", argv[2]);
|
||||||
|
return exit_code(TesterExitCode::CreateError);
|
||||||
|
}
|
||||||
if (output_path.has_parent_path()) {
|
if (output_path.has_parent_path()) {
|
||||||
std::filesystem::create_directories(output_path.parent_path());
|
std::filesystem::create_directories(output_path.parent_path());
|
||||||
}
|
}
|
||||||
|
|
||||||
auto sink = cvmmap_streamer::record::MultiMcapRecordSink::create(
|
auto sink = cvmmap_streamer::record::MultiMcapRecordSink::create(
|
||||||
output_path.string(),
|
output_path.string(),
|
||||||
cvmmap_streamer::McapCompression::None);
|
compression);
|
||||||
if (!sink) {
|
if (!sink) {
|
||||||
spdlog::error("failed to create MCAP sink: {}", sink.error());
|
spdlog::error("failed to create MCAP sink: {}", sink.error());
|
||||||
return exit_code(TesterExitCode::CreateError);
|
return exit_code(TesterExitCode::CreateError);
|
||||||
|
|||||||
+1
@@ -312,6 +312,7 @@ public:
|
|||||||
private:
|
private:
|
||||||
std::vector<std::byte> uncompressedBuffer_;
|
std::vector<std::byte> uncompressedBuffer_;
|
||||||
std::vector<std::byte> compressedBuffer_;
|
std::vector<std::byte> compressedBuffer_;
|
||||||
|
CompressionLevel compressionLevel_ = CompressionLevel::Default;
|
||||||
ZSTD_CCtx_s* zstdContext_ = nullptr;
|
ZSTD_CCtx_s* zstdContext_ = nullptr;
|
||||||
};
|
};
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
+11
-7
@@ -241,10 +241,13 @@ int ZStdCompressionLevel(CompressionLevel level) {
|
|||||||
|
|
||||||
// ZStdWriter //////////////////////////////////////////////////////////////////
|
// ZStdWriter //////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
ZStdWriter::ZStdWriter(CompressionLevel compressionLevel, uint64_t chunkSize) {
|
ZStdWriter::ZStdWriter(CompressionLevel compressionLevel, uint64_t chunkSize)
|
||||||
|
: compressionLevel_(compressionLevel) {
|
||||||
zstdContext_ = ZSTD_createCCtx();
|
zstdContext_ = ZSTD_createCCtx();
|
||||||
ZSTD_CCtx_setParameter(zstdContext_, ZSTD_c_compressionLevel,
|
if (zstdContext_ == nullptr) {
|
||||||
internal::ZStdCompressionLevel(compressionLevel));
|
std::cerr << "ZSTD_createCCtx failed\n";
|
||||||
|
std::abort();
|
||||||
|
}
|
||||||
uncompressedBuffer_.reserve(chunkSize);
|
uncompressedBuffer_.reserve(chunkSize);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -259,15 +262,16 @@ void ZStdWriter::handleWrite(const std::byte* data, uint64_t size) {
|
|||||||
void ZStdWriter::end() {
|
void ZStdWriter::end() {
|
||||||
const auto dstCapacity = ZSTD_compressBound(uncompressedBuffer_.size());
|
const auto dstCapacity = ZSTD_compressBound(uncompressedBuffer_.size());
|
||||||
compressedBuffer_.resize(dstCapacity);
|
compressedBuffer_.resize(dstCapacity);
|
||||||
const size_t dstSize = ZSTD_compress2(zstdContext_, compressedBuffer_.data(), dstCapacity,
|
const size_t dstSize =
|
||||||
uncompressedBuffer_.data(), uncompressedBuffer_.size());
|
ZSTD_compressCCtx(zstdContext_, compressedBuffer_.data(), dstCapacity,
|
||||||
|
uncompressedBuffer_.data(), uncompressedBuffer_.size(),
|
||||||
|
internal::ZStdCompressionLevel(compressionLevel_));
|
||||||
if (ZSTD_isError(dstSize)) {
|
if (ZSTD_isError(dstSize)) {
|
||||||
const auto errCode = ZSTD_getErrorCode(dstSize);
|
const auto errCode = ZSTD_getErrorCode(dstSize);
|
||||||
std::cerr << "ZSTD_compress2 failed: " << ZSTD_getErrorName(dstSize) << " ("
|
std::cerr << "ZSTD_compressCCtx failed: " << ZSTD_getErrorName(dstSize) << " ("
|
||||||
<< ZSTD_getErrorString(errCode) << ")\n";
|
<< ZSTD_getErrorString(errCode) << ")\n";
|
||||||
std::abort();
|
std::abort();
|
||||||
}
|
}
|
||||||
ZSTD_CCtx_reset(zstdContext_, ZSTD_reset_session_only);
|
|
||||||
compressedBuffer_.resize(dstSize);
|
compressedBuffer_.resize(dstSize);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user