feat(streamer): add ffmpeg encoder and mcap recording
This commit is contained in:
@@ -0,0 +1,108 @@
|
||||
#include <array>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
|
||||
namespace mcap::internal {
|
||||
|
||||
/**
|
||||
* Compute CRC32 lookup tables as described at:
|
||||
* https://github.com/komrad36/CRC#option-6-1-byte-tabular
|
||||
*
|
||||
* An iteration of CRC computation can be performed on 8 bits of input at once. By pre-computing a
|
||||
* table of the values of CRC(?) for all 2^8 = 256 possible byte values, during the final
|
||||
* computation we can replace a loop over 8 bits with a single lookup in the table.
|
||||
*
|
||||
* For further speedup, we can also pre-compute the values of CRC(?0) for all possible bytes when a
|
||||
* zero byte is appended. Then we can process two bytes of input at once by computing CRC(AB) =
|
||||
* CRC(A0) ^ CRC(B), using one lookup in the CRC(?0) table and one lookup in the CRC(?) table.
|
||||
*
|
||||
* The same technique applies for any number of bytes to be processed at once, although the speed
|
||||
* improvements diminish.
|
||||
*
|
||||
* @param Polynomial The binary representation of the polynomial to use (reversed, i.e. most
|
||||
* significant bit represents x^0).
|
||||
* @param NumTables The number of bytes of input that will be processed at once.
|
||||
*/
|
||||
template <size_t Polynomial, size_t NumTables>
|
||||
struct CRC32Table {
|
||||
private:
|
||||
std::array<uint32_t, 256 * NumTables> table = {};
|
||||
|
||||
public:
|
||||
constexpr CRC32Table() {
|
||||
for (uint32_t i = 0; i < 256; i++) {
|
||||
uint32_t r = i;
|
||||
r = ((r & 1) * Polynomial) ^ (r >> 1);
|
||||
r = ((r & 1) * Polynomial) ^ (r >> 1);
|
||||
r = ((r & 1) * Polynomial) ^ (r >> 1);
|
||||
r = ((r & 1) * Polynomial) ^ (r >> 1);
|
||||
r = ((r & 1) * Polynomial) ^ (r >> 1);
|
||||
r = ((r & 1) * Polynomial) ^ (r >> 1);
|
||||
r = ((r & 1) * Polynomial) ^ (r >> 1);
|
||||
r = ((r & 1) * Polynomial) ^ (r >> 1);
|
||||
table[i] = r;
|
||||
}
|
||||
for (size_t i = 256; i < table.size(); i++) {
|
||||
uint32_t value = table[i - 256];
|
||||
table[i] = table[value & 0xff] ^ (value >> 8);
|
||||
}
|
||||
}
|
||||
|
||||
constexpr uint32_t operator[](size_t index) const {
|
||||
return table[index];
|
||||
}
|
||||
};
|
||||
|
||||
inline uint32_t getUint32LE(const std::byte* data) {
|
||||
return (uint32_t(data[0]) << 0) | (uint32_t(data[1]) << 8) | (uint32_t(data[2]) << 16) |
|
||||
(uint32_t(data[3]) << 24);
|
||||
}
|
||||
|
||||
static constexpr CRC32Table<0xedb88320, 8> CRC32_TABLE;
|
||||
|
||||
/**
|
||||
* Initialize a CRC32 to all 1 bits.
|
||||
*/
|
||||
static constexpr uint32_t CRC32_INIT = 0xffffffff;
|
||||
|
||||
/**
|
||||
* Update a streaming CRC32 calculation.
|
||||
*
|
||||
* For performance, this implementation processes the data 8 bytes at a time, using the algorithm
|
||||
* presented at: https://github.com/komrad36/CRC#option-9-8-byte-tabular
|
||||
*/
|
||||
inline uint32_t crc32Update(const uint32_t prev, const std::byte* const data, const size_t length) {
|
||||
// Process bytes one by one until we reach the proper alignment.
|
||||
uint32_t r = prev;
|
||||
size_t offset = 0;
|
||||
for (; (uintptr_t(data + offset) & alignof(uint32_t)) != 0 && offset < length; offset++) {
|
||||
r = CRC32_TABLE[(r ^ uint8_t(data[offset])) & 0xff] ^ (r >> 8);
|
||||
}
|
||||
if (offset == length) {
|
||||
return r;
|
||||
}
|
||||
|
||||
// Process 8 bytes (2 uint32s) at a time.
|
||||
size_t remainingBytes = length - offset;
|
||||
for (; remainingBytes >= 8; offset += 8, remainingBytes -= 8) {
|
||||
r ^= getUint32LE(data + offset);
|
||||
uint32_t r2 = getUint32LE(data + offset + 4);
|
||||
r = CRC32_TABLE[0 * 256 + ((r2 >> 24) & 0xff)] ^ CRC32_TABLE[1 * 256 + ((r2 >> 16) & 0xff)] ^
|
||||
CRC32_TABLE[2 * 256 + ((r2 >> 8) & 0xff)] ^ CRC32_TABLE[3 * 256 + ((r2 >> 0) & 0xff)] ^
|
||||
CRC32_TABLE[4 * 256 + ((r >> 24) & 0xff)] ^ CRC32_TABLE[5 * 256 + ((r >> 16) & 0xff)] ^
|
||||
CRC32_TABLE[6 * 256 + ((r >> 8) & 0xff)] ^ CRC32_TABLE[7 * 256 + ((r >> 0) & 0xff)];
|
||||
}
|
||||
|
||||
// Process any remaining bytes one by one.
|
||||
for (; offset < length; offset++) {
|
||||
r = CRC32_TABLE[(r ^ uint8_t(data[offset])) & 0xff] ^ (r >> 8);
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
/** Finalize a CRC32 by inverting the output value. */
|
||||
inline uint32_t crc32Final(uint32_t crc) {
|
||||
return crc ^ 0xffffffff;
|
||||
}
|
||||
|
||||
} // namespace mcap::internal
|
||||
@@ -0,0 +1,120 @@
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace mcap {
|
||||
|
||||
/**
|
||||
* @brief Status codes for MCAP readers and writers.
|
||||
*/
|
||||
enum class StatusCode {
|
||||
Success = 0,
|
||||
NotOpen,
|
||||
InvalidSchemaId,
|
||||
InvalidChannelId,
|
||||
FileTooSmall,
|
||||
ReadFailed,
|
||||
MagicMismatch,
|
||||
InvalidFile,
|
||||
InvalidRecord,
|
||||
InvalidOpCode,
|
||||
InvalidChunkOffset,
|
||||
InvalidFooter,
|
||||
DecompressionFailed,
|
||||
DecompressionSizeMismatch,
|
||||
UnrecognizedCompression,
|
||||
OpenFailed,
|
||||
MissingStatistics,
|
||||
InvalidMessageReadOptions,
|
||||
NoMessageIndexesAvailable,
|
||||
UnsupportedCompression,
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Wraps a status code and string message carrying additional context.
|
||||
*/
|
||||
struct [[nodiscard]] Status {
|
||||
StatusCode code;
|
||||
std::string message;
|
||||
|
||||
Status()
|
||||
: code(StatusCode::Success) {}
|
||||
|
||||
Status(StatusCode _code)
|
||||
: code(_code) {
|
||||
switch (code) {
|
||||
case StatusCode::Success:
|
||||
break;
|
||||
case StatusCode::NotOpen:
|
||||
message = "not open";
|
||||
break;
|
||||
case StatusCode::InvalidSchemaId:
|
||||
message = "invalid schema id";
|
||||
break;
|
||||
case StatusCode::InvalidChannelId:
|
||||
message = "invalid channel id";
|
||||
break;
|
||||
case StatusCode::FileTooSmall:
|
||||
message = "file too small";
|
||||
break;
|
||||
case StatusCode::ReadFailed:
|
||||
message = "read failed";
|
||||
break;
|
||||
case StatusCode::MagicMismatch:
|
||||
message = "magic mismatch";
|
||||
break;
|
||||
case StatusCode::InvalidFile:
|
||||
message = "invalid file";
|
||||
break;
|
||||
case StatusCode::InvalidRecord:
|
||||
message = "invalid record";
|
||||
break;
|
||||
case StatusCode::InvalidOpCode:
|
||||
message = "invalid opcode";
|
||||
break;
|
||||
case StatusCode::InvalidChunkOffset:
|
||||
message = "invalid chunk offset";
|
||||
break;
|
||||
case StatusCode::InvalidFooter:
|
||||
message = "invalid footer";
|
||||
break;
|
||||
case StatusCode::DecompressionFailed:
|
||||
message = "decompression failed";
|
||||
break;
|
||||
case StatusCode::DecompressionSizeMismatch:
|
||||
message = "decompression size mismatch";
|
||||
break;
|
||||
case StatusCode::UnrecognizedCompression:
|
||||
message = "unrecognized compression";
|
||||
break;
|
||||
case StatusCode::OpenFailed:
|
||||
message = "open failed";
|
||||
break;
|
||||
case StatusCode::MissingStatistics:
|
||||
message = "missing statistics";
|
||||
break;
|
||||
case StatusCode::InvalidMessageReadOptions:
|
||||
message = "message read options conflict";
|
||||
break;
|
||||
case StatusCode::NoMessageIndexesAvailable:
|
||||
message = "file has no message indices";
|
||||
break;
|
||||
case StatusCode::UnsupportedCompression:
|
||||
message = "unsupported compression";
|
||||
break;
|
||||
default:
|
||||
message = "unknown";
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Status(StatusCode _code, const std::string& _message)
|
||||
: code(_code)
|
||||
, message(_message) {}
|
||||
|
||||
bool ok() const {
|
||||
return code == StatusCode::Success;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace mcap
|
||||
@@ -0,0 +1,193 @@
|
||||
#pragma once
|
||||
|
||||
#include "types.hpp"
|
||||
#include <cstring>
|
||||
|
||||
// Do not compile on systems with non-8-bit bytes
|
||||
static_assert(std::numeric_limits<unsigned char>::digits == 8);
|
||||
|
||||
namespace mcap {
|
||||
|
||||
namespace internal {
|
||||
|
||||
constexpr uint64_t MinHeaderLength = /* magic bytes */ sizeof(Magic) +
|
||||
/* opcode */ 1 +
|
||||
/* record length */ 8 +
|
||||
/* profile length */ 4 +
|
||||
/* library length */ 4;
|
||||
constexpr uint64_t FooterLength = /* opcode */ 1 +
|
||||
/* record length */ 8 +
|
||||
/* summary start */ 8 +
|
||||
/* summary offset start */ 8 +
|
||||
/* summary crc */ 4 +
|
||||
/* magic bytes */ sizeof(Magic);
|
||||
|
||||
inline std::string ToHex(uint8_t byte) {
|
||||
std::string result{2, '\0'};
|
||||
result[0] = "0123456789ABCDEF"[(uint8_t(byte) >> 4) & 0x0F];
|
||||
result[1] = "0123456789ABCDEF"[uint8_t(byte) & 0x0F];
|
||||
return result;
|
||||
}
|
||||
inline std::string ToHex(std::byte byte) {
|
||||
return ToHex(uint8_t(byte));
|
||||
}
|
||||
|
||||
inline std::string to_string(const std::string& arg) {
|
||||
return arg;
|
||||
}
|
||||
inline std::string to_string(std::string_view arg) {
|
||||
return std::string(arg);
|
||||
}
|
||||
inline std::string to_string(const char* arg) {
|
||||
return std::string(arg);
|
||||
}
|
||||
template <typename... T>
|
||||
[[nodiscard]] inline std::string StrCat(T&&... args) {
|
||||
using mcap::internal::to_string;
|
||||
using std::to_string;
|
||||
return ("" + ... + to_string(std::forward<T>(args)));
|
||||
}
|
||||
|
||||
inline uint32_t KeyValueMapSize(const KeyValueMap& map) {
|
||||
size_t size = 0;
|
||||
for (const auto& [key, value] : map) {
|
||||
size += 4 + key.size() + 4 + value.size();
|
||||
}
|
||||
return (uint32_t)(size);
|
||||
}
|
||||
|
||||
inline const std::string CompressionString(Compression compression) {
|
||||
switch (compression) {
|
||||
case Compression::None:
|
||||
default:
|
||||
return std::string{};
|
||||
case Compression::Lz4:
|
||||
return "lz4";
|
||||
case Compression::Zstd:
|
||||
return "zstd";
|
||||
}
|
||||
}
|
||||
|
||||
inline uint16_t ParseUint16(const std::byte* data) {
|
||||
return uint16_t(data[0]) | (uint16_t(data[1]) << 8);
|
||||
}
|
||||
|
||||
inline uint32_t ParseUint32(const std::byte* data) {
|
||||
return uint32_t(data[0]) | (uint32_t(data[1]) << 8) | (uint32_t(data[2]) << 16) |
|
||||
(uint32_t(data[3]) << 24);
|
||||
}
|
||||
|
||||
inline Status ParseUint32(const std::byte* data, uint64_t maxSize, uint32_t* output) {
|
||||
if (maxSize < 4) {
|
||||
const auto msg = StrCat("cannot read uint32 from ", maxSize, " bytes");
|
||||
return Status{StatusCode::InvalidRecord, msg};
|
||||
}
|
||||
*output = ParseUint32(data);
|
||||
return StatusCode::Success;
|
||||
}
|
||||
|
||||
inline uint64_t ParseUint64(const std::byte* data) {
|
||||
return uint64_t(data[0]) | (uint64_t(data[1]) << 8) | (uint64_t(data[2]) << 16) |
|
||||
(uint64_t(data[3]) << 24) | (uint64_t(data[4]) << 32) | (uint64_t(data[5]) << 40) |
|
||||
(uint64_t(data[6]) << 48) | (uint64_t(data[7]) << 56);
|
||||
}
|
||||
|
||||
inline Status ParseUint64(const std::byte* data, uint64_t maxSize, uint64_t* output) {
|
||||
if (maxSize < 8) {
|
||||
const auto msg = StrCat("cannot read uint64 from ", maxSize, " bytes");
|
||||
return Status{StatusCode::InvalidRecord, msg};
|
||||
}
|
||||
*output = ParseUint64(data);
|
||||
return StatusCode::Success;
|
||||
}
|
||||
|
||||
inline Status ParseStringView(const std::byte* data, uint64_t maxSize, std::string_view* output) {
|
||||
uint32_t size = 0;
|
||||
if (auto status = ParseUint32(data, maxSize, &size); !status.ok()) {
|
||||
const auto msg = StrCat("cannot read string size: ", status.message);
|
||||
return Status{StatusCode::InvalidRecord, msg};
|
||||
}
|
||||
if (uint64_t(size) > (maxSize - 4)) {
|
||||
const auto msg = StrCat("string size ", size, " exceeds remaining bytes ", (maxSize - 4));
|
||||
return Status(StatusCode::InvalidRecord, msg);
|
||||
}
|
||||
*output = std::string_view(reinterpret_cast<const char*>(data + 4), size);
|
||||
return StatusCode::Success;
|
||||
}
|
||||
|
||||
inline Status ParseString(const std::byte* data, uint64_t maxSize, std::string* output) {
|
||||
uint32_t size = 0;
|
||||
if (auto status = ParseUint32(data, maxSize, &size); !status.ok()) {
|
||||
return status;
|
||||
}
|
||||
if (uint64_t(size) > (maxSize - 4)) {
|
||||
const auto msg = StrCat("string size ", size, " exceeds remaining bytes ", (maxSize - 4));
|
||||
return Status(StatusCode::InvalidRecord, msg);
|
||||
}
|
||||
*output = std::string(reinterpret_cast<const char*>(data + 4), size);
|
||||
return StatusCode::Success;
|
||||
}
|
||||
|
||||
inline Status ParseByteArray(const std::byte* data, uint64_t maxSize, ByteArray* output) {
|
||||
uint32_t size = 0;
|
||||
if (auto status = ParseUint32(data, maxSize, &size); !status.ok()) {
|
||||
return status;
|
||||
}
|
||||
if (uint64_t(size) > (maxSize - 4)) {
|
||||
const auto msg = StrCat("byte array size ", size, " exceeds remaining bytes ", (maxSize - 4));
|
||||
return Status(StatusCode::InvalidRecord, msg);
|
||||
}
|
||||
output->resize(size);
|
||||
// output->data() may return nullptr if 'output' is empty, but memcpy() does not accept nullptr.
|
||||
// 'output' will be empty only if the 'size' is equal to 0.
|
||||
if (size > 0) {
|
||||
std::memcpy(output->data(), data + 4, size);
|
||||
}
|
||||
return StatusCode::Success;
|
||||
}
|
||||
|
||||
inline Status ParseKeyValueMap(const std::byte* data, uint64_t maxSize, KeyValueMap* output) {
|
||||
uint32_t sizeInBytes = 0;
|
||||
if (auto status = ParseUint32(data, maxSize, &sizeInBytes); !status.ok()) {
|
||||
return status;
|
||||
}
|
||||
if (sizeInBytes > (maxSize - 4)) {
|
||||
const auto msg =
|
||||
StrCat("key-value map size ", sizeInBytes, " exceeds remaining bytes ", (maxSize - 4));
|
||||
return Status(StatusCode::InvalidRecord, msg);
|
||||
}
|
||||
|
||||
// Account for the byte size prefix in sizeInBytes to make the bounds checking
|
||||
// below simpler
|
||||
sizeInBytes += 4;
|
||||
|
||||
output->clear();
|
||||
uint64_t pos = 4;
|
||||
while (pos < sizeInBytes) {
|
||||
std::string_view key;
|
||||
if (auto status = ParseStringView(data + pos, sizeInBytes - pos, &key); !status.ok()) {
|
||||
const auto msg = StrCat("cannot read key-value map key at pos ", pos, ": ", status.message);
|
||||
return Status{StatusCode::InvalidRecord, msg};
|
||||
}
|
||||
pos += 4 + key.size();
|
||||
std::string_view value;
|
||||
if (auto status = ParseStringView(data + pos, sizeInBytes - pos, &value); !status.ok()) {
|
||||
const auto msg = StrCat("cannot read key-value map value for key \"", key, "\" at pos ", pos,
|
||||
": ", status.message);
|
||||
return Status{StatusCode::InvalidRecord, msg};
|
||||
}
|
||||
pos += 4 + value.size();
|
||||
output->emplace(key, value);
|
||||
}
|
||||
return StatusCode::Success;
|
||||
}
|
||||
|
||||
inline std::string MagicToHex(const std::byte* data) {
|
||||
return internal::ToHex(data[0]) + internal::ToHex(data[1]) + internal::ToHex(data[2]) +
|
||||
internal::ToHex(data[3]) + internal::ToHex(data[4]) + internal::ToHex(data[5]) +
|
||||
internal::ToHex(data[6]) + internal::ToHex(data[7]);
|
||||
}
|
||||
|
||||
} // namespace internal
|
||||
|
||||
} // namespace mcap
|
||||
@@ -0,0 +1,303 @@
|
||||
// Adapted from <https://github.com/ekg/intervaltree/blob/master/IntervalTree.h>
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <cassert>
|
||||
#include <iostream>
|
||||
#include <limits>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
namespace mcap::internal {
|
||||
|
||||
template <class Scalar, typename Value>
|
||||
class Interval {
|
||||
public:
|
||||
Scalar start;
|
||||
Scalar stop;
|
||||
Value value;
|
||||
Interval(const Scalar& s, const Scalar& e, const Value& v)
|
||||
: start(std::min(s, e))
|
||||
, stop(std::max(s, e))
|
||||
, value(v) {}
|
||||
};
|
||||
|
||||
template <class Scalar, typename Value>
|
||||
Value intervalStart(const Interval<Scalar, Value>& i) {
|
||||
return i.start;
|
||||
}
|
||||
|
||||
template <class Scalar, typename Value>
|
||||
Value intervalStop(const Interval<Scalar, Value>& i) {
|
||||
return i.stop;
|
||||
}
|
||||
|
||||
template <class Scalar, typename Value>
|
||||
std::ostream& operator<<(std::ostream& out, const Interval<Scalar, Value>& i) {
|
||||
out << "Interval(" << i.start << ", " << i.stop << "): " << i.value;
|
||||
return out;
|
||||
}
|
||||
|
||||
template <class Scalar, class Value>
|
||||
class IntervalTree {
|
||||
public:
|
||||
using interval = Interval<Scalar, Value>;
|
||||
using interval_vector = std::vector<interval>;
|
||||
|
||||
struct IntervalStartCmp {
|
||||
bool operator()(const interval& a, const interval& b) {
|
||||
return a.start < b.start;
|
||||
}
|
||||
};
|
||||
|
||||
struct IntervalStopCmp {
|
||||
bool operator()(const interval& a, const interval& b) {
|
||||
return a.stop < b.stop;
|
||||
}
|
||||
};
|
||||
|
||||
IntervalTree()
|
||||
: left(nullptr)
|
||||
, right(nullptr)
|
||||
, center(Scalar(0)) {}
|
||||
|
||||
~IntervalTree() = default;
|
||||
|
||||
std::unique_ptr<IntervalTree> clone() const {
|
||||
return std::unique_ptr<IntervalTree>(new IntervalTree(*this));
|
||||
}
|
||||
|
||||
IntervalTree(const IntervalTree& other)
|
||||
: intervals(other.intervals)
|
||||
, left(other.left ? other.left->clone() : nullptr)
|
||||
, right(other.right ? other.right->clone() : nullptr)
|
||||
, center(other.center) {}
|
||||
|
||||
IntervalTree& operator=(IntervalTree&&) = default;
|
||||
IntervalTree(IntervalTree&&) = default;
|
||||
|
||||
IntervalTree& operator=(const IntervalTree& other) {
|
||||
center = other.center;
|
||||
intervals = other.intervals;
|
||||
left = other.left ? other.left->clone() : nullptr;
|
||||
right = other.right ? other.right->clone() : nullptr;
|
||||
return *this;
|
||||
}
|
||||
|
||||
IntervalTree(interval_vector&& ivals, std::size_t depth = 16, std::size_t minbucket = 64,
|
||||
std::size_t maxbucket = 512, Scalar leftextent = 0, Scalar rightextent = 0)
|
||||
: left(nullptr)
|
||||
, right(nullptr) {
|
||||
--depth;
|
||||
const auto minmaxStop = std::minmax_element(ivals.begin(), ivals.end(), IntervalStopCmp());
|
||||
const auto minmaxStart = std::minmax_element(ivals.begin(), ivals.end(), IntervalStartCmp());
|
||||
if (!ivals.empty()) {
|
||||
center = (minmaxStart.first->start + minmaxStop.second->stop) / 2;
|
||||
}
|
||||
if (leftextent == 0 && rightextent == 0) {
|
||||
// sort intervals by start
|
||||
std::sort(ivals.begin(), ivals.end(), IntervalStartCmp());
|
||||
} else {
|
||||
assert(std::is_sorted(ivals.begin(), ivals.end(), IntervalStartCmp()));
|
||||
}
|
||||
if (depth == 0 || (ivals.size() < minbucket && ivals.size() < maxbucket)) {
|
||||
std::sort(ivals.begin(), ivals.end(), IntervalStartCmp());
|
||||
intervals = std::move(ivals);
|
||||
assert(is_valid().first);
|
||||
return;
|
||||
} else {
|
||||
Scalar leftp = 0;
|
||||
Scalar rightp = 0;
|
||||
|
||||
if (leftextent || rightextent) {
|
||||
leftp = leftextent;
|
||||
rightp = rightextent;
|
||||
} else {
|
||||
leftp = ivals.front().start;
|
||||
rightp = std::max_element(ivals.begin(), ivals.end(), IntervalStopCmp())->stop;
|
||||
}
|
||||
|
||||
interval_vector lefts;
|
||||
interval_vector rights;
|
||||
|
||||
for (typename interval_vector::const_iterator i = ivals.begin(); i != ivals.end(); ++i) {
|
||||
const interval& cur = *i;
|
||||
if (cur.stop < center) {
|
||||
lefts.push_back(cur);
|
||||
} else if (cur.start > center) {
|
||||
rights.push_back(cur);
|
||||
} else {
|
||||
assert(cur.start <= center);
|
||||
assert(center <= cur.stop);
|
||||
intervals.push_back(cur);
|
||||
}
|
||||
}
|
||||
|
||||
if (!lefts.empty()) {
|
||||
left.reset(new IntervalTree(std::move(lefts), depth, minbucket, maxbucket, leftp, center));
|
||||
}
|
||||
if (!rights.empty()) {
|
||||
right.reset(
|
||||
new IntervalTree(std::move(rights), depth, minbucket, maxbucket, center, rightp));
|
||||
}
|
||||
}
|
||||
assert(is_valid().first);
|
||||
}
|
||||
|
||||
// Call f on all intervals near the range [start, stop]:
|
||||
template <class UnaryFunction>
|
||||
void visit_near(const Scalar& start, const Scalar& stop, UnaryFunction f) const {
|
||||
if (!intervals.empty() && !(stop < intervals.front().start)) {
|
||||
for (auto& i : intervals) {
|
||||
f(i);
|
||||
}
|
||||
}
|
||||
if (left && start <= center) {
|
||||
left->visit_near(start, stop, f);
|
||||
}
|
||||
if (right && stop >= center) {
|
||||
right->visit_near(start, stop, f);
|
||||
}
|
||||
}
|
||||
|
||||
// Call f on all intervals crossing pos
|
||||
template <class UnaryFunction>
|
||||
void visit_overlapping(const Scalar& pos, UnaryFunction f) const {
|
||||
visit_overlapping(pos, pos, f);
|
||||
}
|
||||
|
||||
// Call f on all intervals overlapping [start, stop]
|
||||
template <class UnaryFunction>
|
||||
void visit_overlapping(const Scalar& start, const Scalar& stop, UnaryFunction f) const {
|
||||
auto filterF = [&](const interval& cur) {
|
||||
if (cur.stop >= start && cur.start <= stop) {
|
||||
// Only apply f if overlapping
|
||||
f(cur);
|
||||
}
|
||||
};
|
||||
visit_near(start, stop, filterF);
|
||||
}
|
||||
|
||||
// Call f on all intervals contained within [start, stop]
|
||||
template <class UnaryFunction>
|
||||
void visit_contained(const Scalar& start, const Scalar& stop, UnaryFunction f) const {
|
||||
auto filterF = [&](const interval& cur) {
|
||||
if (start <= cur.start && cur.stop <= stop) {
|
||||
f(cur);
|
||||
}
|
||||
};
|
||||
visit_near(start, stop, filterF);
|
||||
}
|
||||
|
||||
interval_vector find_overlapping(const Scalar& start, const Scalar& stop) const {
|
||||
interval_vector result;
|
||||
visit_overlapping(start, stop, [&](const interval& cur) {
|
||||
result.emplace_back(cur);
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
interval_vector find_contained(const Scalar& start, const Scalar& stop) const {
|
||||
interval_vector result;
|
||||
visit_contained(start, stop, [&](const interval& cur) {
|
||||
result.push_back(cur);
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
bool empty() const {
|
||||
if (left && !left->empty()) {
|
||||
return false;
|
||||
}
|
||||
if (!intervals.empty()) {
|
||||
return false;
|
||||
}
|
||||
if (right && !right->empty()) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
template <class UnaryFunction>
|
||||
void visit_all(UnaryFunction f) const {
|
||||
if (left) {
|
||||
left->visit_all(f);
|
||||
}
|
||||
std::for_each(intervals.begin(), intervals.end(), f);
|
||||
if (right) {
|
||||
right->visit_all(f);
|
||||
}
|
||||
}
|
||||
|
||||
std::pair<Scalar, Scalar> extent() const {
|
||||
struct Extent {
|
||||
std::pair<Scalar, Scalar> x{std::numeric_limits<Scalar>::max(),
|
||||
std::numeric_limits<Scalar>::min()};
|
||||
void operator()(const interval& cur) {
|
||||
x.first = std::min(x.first, cur.start);
|
||||
x.second = std::max(x.second, cur.stop);
|
||||
}
|
||||
};
|
||||
Extent extent;
|
||||
|
||||
visit_all([&](const interval& cur) {
|
||||
extent(cur);
|
||||
});
|
||||
return extent.x;
|
||||
}
|
||||
|
||||
// Check all constraints.
|
||||
// If first is false, second is invalid.
|
||||
std::pair<bool, std::pair<Scalar, Scalar>> is_valid() const {
|
||||
const auto minmaxStop =
|
||||
std::minmax_element(intervals.begin(), intervals.end(), IntervalStopCmp());
|
||||
const auto minmaxStart =
|
||||
std::minmax_element(intervals.begin(), intervals.end(), IntervalStartCmp());
|
||||
|
||||
std::pair<bool, std::pair<Scalar, Scalar>> result = {
|
||||
true, {std::numeric_limits<Scalar>::max(), std::numeric_limits<Scalar>::min()}};
|
||||
if (!intervals.empty()) {
|
||||
result.second.first = std::min(result.second.first, minmaxStart.first->start);
|
||||
result.second.second = std::min(result.second.second, minmaxStop.second->stop);
|
||||
}
|
||||
if (left) {
|
||||
auto valid = left->is_valid();
|
||||
result.first &= valid.first;
|
||||
result.second.first = std::min(result.second.first, valid.second.first);
|
||||
result.second.second = std::min(result.second.second, valid.second.second);
|
||||
if (!result.first) {
|
||||
return result;
|
||||
}
|
||||
if (valid.second.second >= center) {
|
||||
result.first = false;
|
||||
return result;
|
||||
}
|
||||
}
|
||||
if (right) {
|
||||
auto valid = right->is_valid();
|
||||
result.first &= valid.first;
|
||||
result.second.first = std::min(result.second.first, valid.second.first);
|
||||
result.second.second = std::min(result.second.second, valid.second.second);
|
||||
if (!result.first) {
|
||||
return result;
|
||||
}
|
||||
if (valid.second.first <= center) {
|
||||
result.first = false;
|
||||
return result;
|
||||
}
|
||||
}
|
||||
if (!std::is_sorted(intervals.begin(), intervals.end(), IntervalStartCmp())) {
|
||||
result.first = false;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private:
|
||||
interval_vector intervals;
|
||||
std::unique_ptr<IntervalTree> left;
|
||||
std::unique_ptr<IntervalTree> right;
|
||||
Scalar center;
|
||||
};
|
||||
|
||||
} // namespace mcap::internal
|
||||
@@ -0,0 +1,4 @@
|
||||
#pragma once
|
||||
|
||||
#include "reader.hpp"
|
||||
#include "writer.hpp"
|
||||
@@ -0,0 +1,147 @@
|
||||
#pragma once
|
||||
|
||||
#include "types.hpp"
|
||||
#include <algorithm>
|
||||
#include <variant>
|
||||
|
||||
namespace mcap::internal {
|
||||
|
||||
// Helper for writing compile-time exhaustive variant visitors.
|
||||
template <class>
|
||||
inline constexpr bool always_false_v = false;
|
||||
|
||||
/**
|
||||
* @brief A job to read a specific message at offset `offset` from the decompressed chunk
|
||||
* stored in `chunkReaderIndex`. A timestamp is provided to order this job relative to other jobs.
|
||||
*/
|
||||
struct ReadMessageJob {
|
||||
Timestamp timestamp;
|
||||
RecordOffset offset;
|
||||
size_t chunkReaderIndex;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief A job to decompress the chunk starting at `chunkStartOffset`. The message indices
|
||||
* starting directly after the chunk record and ending at `messageIndexEndOffset` will be used to
|
||||
* find specific messages within the chunk.
|
||||
*/
|
||||
struct DecompressChunkJob {
|
||||
Timestamp messageStartTime;
|
||||
Timestamp messageEndTime;
|
||||
ByteOffset chunkStartOffset;
|
||||
ByteOffset messageIndexEndOffset;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief A union of jobs that an indexed MCAP reader executes.
|
||||
*/
|
||||
using ReadJob = std::variant<ReadMessageJob, DecompressChunkJob>;
|
||||
|
||||
/**
|
||||
* @brief A priority queue of jobs for an indexed MCAP reader to execute.
|
||||
*/
|
||||
struct ReadJobQueue {
|
||||
private:
|
||||
bool reverse_ = false;
|
||||
std::vector<ReadJob> heap_;
|
||||
|
||||
/**
|
||||
* @brief return the timestamp key that should be used to compare jobs.
|
||||
*/
|
||||
static Timestamp TimeComparisonKey(const ReadJob& job, bool reverse) {
|
||||
Timestamp result = 0;
|
||||
std::visit(
|
||||
[&](auto&& arg) {
|
||||
using T = std::decay_t<decltype(arg)>;
|
||||
if constexpr (std::is_same_v<T, ReadMessageJob>) {
|
||||
result = arg.timestamp;
|
||||
} else if constexpr (std::is_same_v<T, DecompressChunkJob>) {
|
||||
if (reverse) {
|
||||
result = arg.messageEndTime;
|
||||
} else {
|
||||
result = arg.messageStartTime;
|
||||
}
|
||||
} else {
|
||||
static_assert(always_false_v<T>, "non-exhaustive visitor!");
|
||||
}
|
||||
},
|
||||
job);
|
||||
return result;
|
||||
}
|
||||
static RecordOffset PositionComparisonKey(const ReadJob& job, bool reverse) {
|
||||
RecordOffset result;
|
||||
std::visit(
|
||||
[&](auto&& arg) {
|
||||
using T = std::decay_t<decltype(arg)>;
|
||||
if constexpr (std::is_same_v<T, ReadMessageJob>) {
|
||||
result = arg.offset;
|
||||
} else if constexpr (std::is_same_v<T, DecompressChunkJob>) {
|
||||
if (reverse) {
|
||||
result.offset = arg.messageIndexEndOffset;
|
||||
} else {
|
||||
result.offset = arg.chunkStartOffset;
|
||||
}
|
||||
} else {
|
||||
static_assert(always_false_v<T>, "non-exhaustive visitor!");
|
||||
}
|
||||
},
|
||||
job);
|
||||
return result;
|
||||
}
|
||||
|
||||
static bool CompareForward(const ReadJob& a, const ReadJob& b) {
|
||||
auto aTimestamp = TimeComparisonKey(a, false);
|
||||
auto bTimestamp = TimeComparisonKey(b, false);
|
||||
if (aTimestamp == bTimestamp) {
|
||||
return PositionComparisonKey(a, false) > PositionComparisonKey(b, false);
|
||||
}
|
||||
return aTimestamp > bTimestamp;
|
||||
}
|
||||
|
||||
static bool CompareReverse(const ReadJob& a, const ReadJob& b) {
|
||||
auto aTimestamp = TimeComparisonKey(a, true);
|
||||
auto bTimestamp = TimeComparisonKey(b, true);
|
||||
if (aTimestamp == bTimestamp) {
|
||||
return PositionComparisonKey(a, true) < PositionComparisonKey(b, true);
|
||||
}
|
||||
return aTimestamp < bTimestamp;
|
||||
}
|
||||
|
||||
public:
|
||||
explicit ReadJobQueue(bool reverse)
|
||||
: reverse_(reverse) {}
|
||||
void push(DecompressChunkJob&& decompressChunkJob) {
|
||||
heap_.emplace_back(std::move(decompressChunkJob));
|
||||
if (!reverse_) {
|
||||
std::push_heap(heap_.begin(), heap_.end(), CompareForward);
|
||||
} else {
|
||||
std::push_heap(heap_.begin(), heap_.end(), CompareReverse);
|
||||
}
|
||||
}
|
||||
|
||||
void push(ReadMessageJob&& readMessageJob) {
|
||||
heap_.emplace_back(std::move(readMessageJob));
|
||||
if (!reverse_) {
|
||||
std::push_heap(heap_.begin(), heap_.end(), CompareForward);
|
||||
} else {
|
||||
std::push_heap(heap_.begin(), heap_.end(), CompareReverse);
|
||||
}
|
||||
}
|
||||
|
||||
ReadJob pop() {
|
||||
if (!reverse_) {
|
||||
std::pop_heap(heap_.begin(), heap_.end(), CompareForward);
|
||||
} else {
|
||||
std::pop_heap(heap_.begin(), heap_.end(), CompareReverse);
|
||||
}
|
||||
auto popped = heap_.back();
|
||||
heap_.pop_back();
|
||||
return popped;
|
||||
}
|
||||
|
||||
size_t len() const {
|
||||
return heap_.size();
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace mcap::internal
|
||||
@@ -0,0 +1,743 @@
|
||||
#pragma once
|
||||
|
||||
#include "intervaltree.hpp"
|
||||
#include "read_job_queue.hpp"
|
||||
#include "types.hpp"
|
||||
#include "visibility.hpp"
|
||||
#include <cstdio>
|
||||
#include <fstream>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
#include <vector>
|
||||
|
||||
namespace mcap {
|
||||
|
||||
enum struct ReadSummaryMethod {
|
||||
/**
|
||||
* @brief Parse the Summary section to produce seeking indexes and summary
|
||||
* statistics. If the Summary section is not present or corrupt, a failure
|
||||
* Status is returned and the seeking indexes and summary statistics are not
|
||||
* populated.
|
||||
*/
|
||||
NoFallbackScan,
|
||||
/**
|
||||
* @brief If the Summary section is missing or incomplete, allow falling back
|
||||
* to reading the file sequentially to produce seeking indexes and summary
|
||||
* statistics.
|
||||
*/
|
||||
AllowFallbackScan,
|
||||
/**
|
||||
* @brief Read the file sequentially from Header to DataEnd to produce seeking
|
||||
* indexes and summary statistics.
|
||||
*/
|
||||
ForceScan,
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief An abstract interface for reading MCAP data.
|
||||
*/
|
||||
struct MCAP_PUBLIC IReadable {
|
||||
virtual ~IReadable() = default;
|
||||
|
||||
/**
|
||||
* @brief Returns the size of the file in bytes.
|
||||
*
|
||||
* @return uint64_t The total number of bytes in the MCAP file.
|
||||
*/
|
||||
virtual uint64_t size() const = 0;
|
||||
/**
|
||||
* @brief This method is called by MCAP reader classes when they need to read
|
||||
* a portion of the file.
|
||||
*
|
||||
* @param output A pointer to a pointer to the buffer to write to. This method
|
||||
* is expected to either maintain an internal buffer, read data into it, and
|
||||
* update this pointer to point at the internal buffer, or update this
|
||||
* pointer to point directly at the source data if possible. The pointer and
|
||||
* data must remain valid and unmodified until the next call to read().
|
||||
* @param offset The offset in bytes from the beginning of the file to read.
|
||||
* @param size The number of bytes to read.
|
||||
* @return uint64_t Number of bytes actually read. This may be less than the
|
||||
* requested size if the end of the file is reached. The output pointer must
|
||||
* be readable from `output` to `output + size`. If the read fails, this
|
||||
* method should return 0.
|
||||
*/
|
||||
virtual uint64_t read(std::byte** output, uint64_t offset, uint64_t size) = 0;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief IReadable implementation wrapping a FILE* pointer created by fopen()
|
||||
* and a read buffer.
|
||||
*/
|
||||
class MCAP_PUBLIC FileReader final : public IReadable {
|
||||
public:
|
||||
FileReader(std::FILE* file);
|
||||
|
||||
uint64_t size() const override;
|
||||
uint64_t read(std::byte** output, uint64_t offset, uint64_t size) override;
|
||||
|
||||
private:
|
||||
// Numeric type returned by the tell/seek operations. Necessary because long on Windows is 32
|
||||
// bits so the standard C library interfaces don't work for files larger than 2GiB.
|
||||
#if defined _WIN32 || defined __CYGWIN__
|
||||
typedef __int64 offset_type;
|
||||
#else
|
||||
typedef long offset_type;
|
||||
#endif
|
||||
|
||||
static_assert((offset_type)(uint64_t)std::numeric_limits<offset_type>::max() ==
|
||||
std::numeric_limits<offset_type>::max(),
|
||||
"offset_type should fit in uint64_t");
|
||||
|
||||
std::FILE* file_;
|
||||
std::vector<std::byte> buffer_;
|
||||
uint64_t size_;
|
||||
uint64_t position_;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief IReadable implementation wrapping a std::ifstream input file stream.
|
||||
*/
|
||||
class MCAP_PUBLIC FileStreamReader final : public IReadable {
|
||||
public:
|
||||
FileStreamReader(std::ifstream& stream);
|
||||
|
||||
uint64_t size() const override;
|
||||
uint64_t read(std::byte** output, uint64_t offset, uint64_t size) override;
|
||||
|
||||
private:
|
||||
std::ifstream& stream_;
|
||||
std::vector<std::byte> buffer_;
|
||||
uint64_t size_;
|
||||
uint64_t position_;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief An abstract interface for compressed readers.
|
||||
*/
|
||||
class MCAP_PUBLIC ICompressedReader : public IReadable {
|
||||
public:
|
||||
virtual ~ICompressedReader() override = default;
|
||||
|
||||
/**
|
||||
* @brief Reset the reader state, clearing any internal buffers and state, and
|
||||
* initialize with new compressed data.
|
||||
*
|
||||
* @param data Compressed data to read from.
|
||||
* @param size Size of the compressed data in bytes.
|
||||
* @param uncompressedSize Size of the data in bytes after decompression. A
|
||||
* buffer of this size will be allocated for the uncompressed data.
|
||||
*/
|
||||
virtual void reset(const std::byte* data, uint64_t size, uint64_t uncompressedSize) = 0;
|
||||
/**
|
||||
* @brief Report the current status of decompression. A StatusCode other than
|
||||
* `StatusCode::Success` after `reset()` is called indicates the decompression
|
||||
* was not successful and the reader is in an invalid state.
|
||||
*/
|
||||
virtual Status status() const = 0;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief A "null" compressed reader that directly passes through uncompressed
|
||||
* data. No internal buffers are allocated.
|
||||
*/
|
||||
class MCAP_PUBLIC BufferReader final : public ICompressedReader {
|
||||
public:
|
||||
void reset(const std::byte* data, uint64_t size, uint64_t uncompressedSize) override;
|
||||
uint64_t read(std::byte** output, uint64_t offset, uint64_t size) override;
|
||||
uint64_t size() const override;
|
||||
Status status() const override;
|
||||
|
||||
BufferReader() = default;
|
||||
BufferReader(const BufferReader&) = delete;
|
||||
BufferReader& operator=(const BufferReader&) = delete;
|
||||
BufferReader(BufferReader&&) = delete;
|
||||
BufferReader& operator=(BufferReader&&) = delete;
|
||||
|
||||
private:
|
||||
const std::byte* data_;
|
||||
uint64_t size_;
|
||||
};
|
||||
|
||||
#ifndef MCAP_COMPRESSION_NO_ZSTD
|
||||
/**
|
||||
* @brief ICompressedReader implementation that decompresses Zstandard
|
||||
* (https://facebook.github.io/zstd/) data.
|
||||
*/
|
||||
class MCAP_PUBLIC ZStdReader final : public ICompressedReader {
|
||||
public:
|
||||
void reset(const std::byte* data, uint64_t size, uint64_t uncompressedSize) override;
|
||||
uint64_t read(std::byte** output, uint64_t offset, uint64_t size) override;
|
||||
uint64_t size() const override;
|
||||
Status status() const override;
|
||||
|
||||
/**
|
||||
* @brief Decompresses an entire Zstd-compressed chunk into `output`.
|
||||
*
|
||||
* @param data The Zstd-compressed input chunk.
|
||||
* @param compressedSize The size of the Zstd-compressed input.
|
||||
* @param uncompressedSize The size of the data once uncompressed.
|
||||
* @param output The output vector. This will be resized to `uncompressedSize` to fit the data,
|
||||
* or 0 if the decompression encountered an error.
|
||||
* @return Status
|
||||
*/
|
||||
static Status DecompressAll(const std::byte* data, uint64_t compressedSize,
|
||||
uint64_t uncompressedSize, ByteArray* output);
|
||||
ZStdReader() = default;
|
||||
ZStdReader(const ZStdReader&) = delete;
|
||||
ZStdReader& operator=(const ZStdReader&) = delete;
|
||||
ZStdReader(ZStdReader&&) = delete;
|
||||
ZStdReader& operator=(ZStdReader&&) = delete;
|
||||
|
||||
private:
|
||||
Status status_;
|
||||
ByteArray uncompressedData_;
|
||||
};
|
||||
#endif
|
||||
|
||||
#ifndef MCAP_COMPRESSION_NO_LZ4
|
||||
/**
|
||||
* @brief ICompressedReader implementation that decompresses LZ4
|
||||
* (https://lz4.github.io/lz4/) data.
|
||||
*/
|
||||
class MCAP_PUBLIC LZ4Reader final : public ICompressedReader {
|
||||
public:
|
||||
void reset(const std::byte* data, uint64_t size, uint64_t uncompressedSize) override;
|
||||
uint64_t read(std::byte** output, uint64_t offset, uint64_t size) override;
|
||||
uint64_t size() const override;
|
||||
Status status() const override;
|
||||
|
||||
/**
|
||||
* @brief Decompresses an entire LZ4-encoded chunk into `output`.
|
||||
*
|
||||
* @param data The LZ4-compressed input chunk.
|
||||
* @param size The size of the LZ4-compressed input.
|
||||
* @param uncompressedSize The size of the data once uncompressed.
|
||||
* @param output The output vector. This will be resized to `uncompressedSize` to fit the data,
|
||||
* or 0 if the decompression encountered an error.
|
||||
* @return Status
|
||||
*/
|
||||
Status decompressAll(const std::byte* data, uint64_t size, uint64_t uncompressedSize,
|
||||
ByteArray* output);
|
||||
LZ4Reader();
|
||||
LZ4Reader(const LZ4Reader&) = delete;
|
||||
LZ4Reader& operator=(const LZ4Reader&) = delete;
|
||||
LZ4Reader(LZ4Reader&&) = delete;
|
||||
LZ4Reader& operator=(LZ4Reader&&) = delete;
|
||||
~LZ4Reader() override;
|
||||
|
||||
private:
|
||||
void* decompressionContext_ = nullptr; // LZ4F_dctx*
|
||||
Status status_;
|
||||
const std::byte* compressedData_;
|
||||
ByteArray uncompressedData_;
|
||||
uint64_t compressedSize_;
|
||||
uint64_t uncompressedSize_;
|
||||
};
|
||||
#endif
|
||||
|
||||
struct LinearMessageView;
|
||||
|
||||
/**
|
||||
* @brief Options for reading messages out of an MCAP file.
|
||||
*/
|
||||
struct MCAP_PUBLIC ReadMessageOptions {
|
||||
public:
|
||||
/**
|
||||
* @brief Only messages with log timestamps greater or equal to startTime will be included.
|
||||
*/
|
||||
Timestamp startTime = 0;
|
||||
/**
|
||||
* @brief Only messages with log timestamps less than endTime will be included.
|
||||
*/
|
||||
Timestamp endTime = MaxTime;
|
||||
/**
|
||||
* @brief If provided, `topicFilter` is called on all topics found in the MCAP file. If
|
||||
* `topicFilter` returns true for a given channel, messages from that channel will be included.
|
||||
* if not provided, messages from all channels are provided.
|
||||
*/
|
||||
std::function<bool(std::string_view)> topicFilter;
|
||||
enum struct ReadOrder { FileOrder, LogTimeOrder, ReverseLogTimeOrder };
|
||||
/**
|
||||
* @brief Set the expected order that messages should be returned in.
|
||||
* if readOrder == FileOrder, messages will be returned in the order they appear in the MCAP file.
|
||||
* if readOrder == LogTimeOrder, messages will be returned in ascending log time order.
|
||||
* if readOrder == ReverseLogTimeOrder, messages will be returned in descending log time order.
|
||||
*/
|
||||
ReadOrder readOrder = ReadOrder::FileOrder;
|
||||
|
||||
ReadMessageOptions(Timestamp start, Timestamp end)
|
||||
: startTime(start)
|
||||
, endTime(end) {}
|
||||
|
||||
ReadMessageOptions() = default;
|
||||
|
||||
/**
|
||||
* @brief validate the configuration.
|
||||
*/
|
||||
Status validate() const;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Provides a read interface to an MCAP file.
|
||||
*/
|
||||
class MCAP_PUBLIC McapReader final {
|
||||
public:
|
||||
~McapReader();
|
||||
|
||||
/**
|
||||
* @brief Opens an MCAP file for reading from an already constructed IReadable
|
||||
* implementation.
|
||||
*
|
||||
* @param reader An implementation of the IReader interface that provides raw
|
||||
* MCAP data.
|
||||
* @return Status StatusCode::Success on success. If a non-success Status is
|
||||
* returned, the data source is not considered open and McapReader is not
|
||||
* usable until `open()` is called and a success response is returned.
|
||||
*/
|
||||
Status open(IReadable& reader);
|
||||
/**
|
||||
* @brief Opens an MCAP file for reading from a given filename.
|
||||
*
|
||||
* @param filename Filename to open.
|
||||
* @return Status StatusCode::Success on success. If a non-success Status is
|
||||
* returned, the data source is not considered open and McapReader is not
|
||||
* usable until `open()` is called and a success response is returned.
|
||||
*/
|
||||
Status open(std::string_view filename);
|
||||
/**
|
||||
* @brief Opens an MCAP file for reading from a std::ifstream input file
|
||||
* stream.
|
||||
*
|
||||
* @param stream Input file stream to read MCAP data from.
|
||||
* @return Status StatusCode::Success on success. If a non-success Status is
|
||||
* returned, the file is not considered open and McapReader is not usable
|
||||
* until `open()` is called and a success response is returned.
|
||||
*/
|
||||
Status open(std::ifstream& stream);
|
||||
|
||||
/**
|
||||
* @brief Closes the MCAP file, clearing any internal data structures and
|
||||
* state and dropping the data source reference.
|
||||
*
|
||||
*/
|
||||
void close();
|
||||
|
||||
/**
|
||||
* @brief Read and parse the Summary section at the end of the MCAP file, if
|
||||
* available. This will populate internal indexes to allow for efficient
|
||||
* summarization and random access. This method will automatically be called
|
||||
* upon requesting summary data or first seek if Summary section parsing is
|
||||
* allowed by the configuration options.
|
||||
*/
|
||||
Status readSummary(
|
||||
ReadSummaryMethod method, const ProblemCallback& onProblem = [](const Status&) {});
|
||||
|
||||
/**
|
||||
* @brief Returns an iterable view with `begin()` and `end()` methods for
|
||||
* iterating Messages in the MCAP file. If a non-zero `startTime` is provided,
|
||||
* this will first parse the Summary section (by calling `readSummary()`) if
|
||||
* allowed by the configuration options and it has not been parsed yet.
|
||||
*
|
||||
* @param startTime Optional start time in nanoseconds. Messages before this
|
||||
* time will not be returned.
|
||||
* @param endTime Optional end time in nanoseconds. Messages equal to or after
|
||||
* this time will not be returned.
|
||||
*/
|
||||
LinearMessageView readMessages(Timestamp startTime = 0, Timestamp endTime = MaxTime);
|
||||
/**
|
||||
* @brief Returns an iterable view with `begin()` and `end()` methods for
|
||||
* iterating Messages in the MCAP file. If a non-zero `startTime` is provided,
|
||||
* this will first parse the Summary section (by calling `readSummary()`) if
|
||||
* allowed by the configuration options and it has not been parsed yet.
|
||||
*
|
||||
* @param onProblem A callback that will be called when a parsing error
|
||||
* occurs. Problems can either be recoverable, indicating some data could
|
||||
* not be read, or non-recoverable, stopping the iteration.
|
||||
* @param startTime Optional start time in nanoseconds. Messages before this
|
||||
* time will not be returned.
|
||||
* @param endTime Optional end time in nanoseconds. Messages equal to or after
|
||||
* this time will not be returned.
|
||||
*/
|
||||
LinearMessageView readMessages(const ProblemCallback& onProblem, Timestamp startTime = 0,
|
||||
Timestamp endTime = MaxTime);
|
||||
|
||||
/**
|
||||
* @brief Returns an iterable view with `begin()` and `end()` methods for
|
||||
* iterating Messages in the MCAP file.
|
||||
* Uses the options from `options` to select the messages that are yielded.
|
||||
*/
|
||||
LinearMessageView readMessages(const ProblemCallback& onProblem,
|
||||
const ReadMessageOptions& options);
|
||||
|
||||
/**
|
||||
* @brief Returns starting and ending byte offsets that must be read to
|
||||
* iterate all messages in the given time range. If `readSummary()` has been
|
||||
* successfully called and the recording contains Chunk records, this range
|
||||
* will be narrowed to Chunk records that contain messages in the given time
|
||||
* range. Otherwise, this range will be the entire Data section if the Data
|
||||
* End record has been found or the entire file otherwise.
|
||||
*
|
||||
* This method is automatically used by `readMessages()`, and only needs to be
|
||||
* called directly if the caller is manually constructing an iterator.
|
||||
*
|
||||
* @param startTime Start time in nanoseconds.
|
||||
* @param endTime Optional end time in nanoseconds.
|
||||
* @return Start and end byte offsets.
|
||||
*/
|
||||
std::pair<ByteOffset, ByteOffset> byteRange(Timestamp startTime,
|
||||
Timestamp endTime = MaxTime) const;
|
||||
|
||||
/**
|
||||
* @brief Returns a pointer to the IReadable data source backing this reader.
|
||||
* Will return nullptr if the reader is not open.
|
||||
*/
|
||||
IReadable* dataSource();
|
||||
|
||||
/**
|
||||
* @brief Returns the parsed Header record, if it has been encountered.
|
||||
*/
|
||||
const std::optional<Header>& header() const;
|
||||
/**
|
||||
* @brief Returns the parsed Footer record, if it has been encountered.
|
||||
*/
|
||||
const std::optional<Footer>& footer() const;
|
||||
/**
|
||||
* @brief Returns the parsed Statistics record, if it has been encountered.
|
||||
*/
|
||||
const std::optional<Statistics>& statistics() const;
|
||||
|
||||
/**
|
||||
* @brief Returns all of the parsed Channel records. Call `readSummary()`
|
||||
* first to fully populate this data structure.
|
||||
*/
|
||||
const std::unordered_map<ChannelId, ChannelPtr> channels() const;
|
||||
/**
|
||||
* @brief Returns all of the parsed Schema records. Call `readSummary()`
|
||||
* first to fully populate this data structure.
|
||||
*/
|
||||
const std::unordered_map<SchemaId, SchemaPtr> schemas() const;
|
||||
|
||||
/**
|
||||
* @brief Look up a Channel record by channel ID. If the Channel has not been
|
||||
* encountered yet or does not exist in the file, this will return nullptr.
|
||||
*
|
||||
* @param channelId Channel ID to search for
|
||||
* @return ChannelPtr A shared pointer to a Channel record, or nullptr
|
||||
*/
|
||||
ChannelPtr channel(ChannelId channelId) const;
|
||||
/**
|
||||
* @brief Look up a Schema record by schema ID. If the Schema has not been
|
||||
* encountered yet or does not exist in the file, this will return nullptr.
|
||||
*
|
||||
* @param schemaId Schema ID to search for
|
||||
* @return SchemaPtr A shared pointer to a Schema record, or nullptr
|
||||
*/
|
||||
SchemaPtr schema(SchemaId schemaId) const;
|
||||
|
||||
/**
|
||||
* @brief Returns all of the parsed ChunkIndex records. Call `readSummary()`
|
||||
* first to fully populate this data structure.
|
||||
*/
|
||||
const std::vector<ChunkIndex>& chunkIndexes() const;
|
||||
|
||||
/**
|
||||
* @brief Returns all of the parsed MetadataIndex records. Call `readSummary()`
|
||||
* first to fully populate this data structure.
|
||||
* The multimap's keys are the `name` field from each indexed Metadata.
|
||||
*/
|
||||
const std::multimap<std::string, MetadataIndex>& metadataIndexes() const;
|
||||
|
||||
/**
|
||||
* @brief Returns all of the parsed AttachmentIndex records. Call `readSummary()`
|
||||
* first to fully populate this data structure.
|
||||
* The multimap's keys are the `name` field from each indexed Attachment.
|
||||
*/
|
||||
const std::multimap<std::string, AttachmentIndex>& attachmentIndexes() const;
|
||||
|
||||
// The following static methods are used internally for parsing MCAP records
|
||||
// and do not need to be called directly unless you are implementing your own
|
||||
// reader functionality or tests.
|
||||
|
||||
static Status ReadRecord(IReadable& reader, uint64_t offset, Record* record);
|
||||
static Status ReadFooter(IReadable& reader, uint64_t offset, Footer* footer);
|
||||
|
||||
static Status ParseHeader(const Record& record, Header* header);
|
||||
static Status ParseFooter(const Record& record, Footer* footer);
|
||||
static Status ParseSchema(const Record& record, Schema* schema);
|
||||
static Status ParseChannel(const Record& record, Channel* channel);
|
||||
static Status ParseMessage(const Record& record, Message* message);
|
||||
static Status ParseChunk(const Record& record, Chunk* chunk);
|
||||
static Status ParseMessageIndex(const Record& record, MessageIndex* messageIndex);
|
||||
static Status ParseChunkIndex(const Record& record, ChunkIndex* chunkIndex);
|
||||
static Status ParseAttachment(const Record& record, Attachment* attachment);
|
||||
static Status ParseAttachmentIndex(const Record& record, AttachmentIndex* attachmentIndex);
|
||||
static Status ParseStatistics(const Record& record, Statistics* statistics);
|
||||
static Status ParseMetadata(const Record& record, Metadata* metadata);
|
||||
static Status ParseMetadataIndex(const Record& record, MetadataIndex* metadataIndex);
|
||||
static Status ParseSummaryOffset(const Record& record, SummaryOffset* summaryOffset);
|
||||
static Status ParseDataEnd(const Record& record, DataEnd* dataEnd);
|
||||
|
||||
/**
|
||||
* @brief Converts a compression string ("", "zstd", "lz4") to the Compression enum.
|
||||
*/
|
||||
static std::optional<Compression> ParseCompression(const std::string_view compression);
|
||||
|
||||
private:
|
||||
using ChunkInterval = internal::Interval<ByteOffset, ChunkIndex>;
|
||||
friend LinearMessageView;
|
||||
|
||||
IReadable* input_ = nullptr;
|
||||
std::FILE* file_ = nullptr;
|
||||
std::unique_ptr<FileReader> fileInput_;
|
||||
std::unique_ptr<FileStreamReader> fileStreamInput_;
|
||||
std::optional<Header> header_;
|
||||
std::optional<Footer> footer_;
|
||||
std::optional<Statistics> statistics_;
|
||||
std::vector<ChunkIndex> chunkIndexes_;
|
||||
internal::IntervalTree<ByteOffset, ChunkIndex> chunkRanges_;
|
||||
std::multimap<std::string, AttachmentIndex> attachmentIndexes_;
|
||||
std::multimap<std::string, MetadataIndex> metadataIndexes_;
|
||||
std::unordered_map<SchemaId, SchemaPtr> schemas_;
|
||||
std::unordered_map<ChannelId, ChannelPtr> channels_;
|
||||
ByteOffset dataStart_ = 0;
|
||||
ByteOffset dataEnd_ = EndOffset;
|
||||
bool parsedSummary_ = false;
|
||||
|
||||
void reset_();
|
||||
Status readSummarySection_(IReadable& reader);
|
||||
Status readSummaryFromScan_(IReadable& reader);
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief A low-level interface for parsing MCAP-style TLV records from a data
|
||||
* source.
|
||||
*/
|
||||
struct MCAP_PUBLIC RecordReader {
|
||||
ByteOffset offset;
|
||||
ByteOffset endOffset;
|
||||
|
||||
RecordReader(IReadable& dataSource, ByteOffset startOffset, ByteOffset endOffset = EndOffset);
|
||||
|
||||
void reset(IReadable& dataSource, ByteOffset startOffset, ByteOffset endOffset);
|
||||
|
||||
std::optional<Record> next();
|
||||
|
||||
const Status& status() const;
|
||||
|
||||
ByteOffset curRecordOffset() const;
|
||||
|
||||
private:
|
||||
IReadable* dataSource_ = nullptr;
|
||||
Status status_;
|
||||
Record curRecord_;
|
||||
};
|
||||
|
||||
struct MCAP_PUBLIC TypedChunkReader {
|
||||
std::function<void(const SchemaPtr, ByteOffset)> onSchema;
|
||||
std::function<void(const ChannelPtr, ByteOffset)> onChannel;
|
||||
std::function<void(const Message&, ByteOffset)> onMessage;
|
||||
std::function<void(const Record&, ByteOffset)> onUnknownRecord;
|
||||
|
||||
TypedChunkReader();
|
||||
TypedChunkReader(const TypedChunkReader&) = delete;
|
||||
TypedChunkReader& operator=(const TypedChunkReader&) = delete;
|
||||
TypedChunkReader(TypedChunkReader&&) = delete;
|
||||
TypedChunkReader& operator=(TypedChunkReader&&) = delete;
|
||||
|
||||
void reset(const Chunk& chunk, Compression compression);
|
||||
|
||||
bool next();
|
||||
|
||||
ByteOffset offset() const;
|
||||
|
||||
const Status& status() const;
|
||||
|
||||
private:
|
||||
RecordReader reader_;
|
||||
Status status_;
|
||||
BufferReader uncompressedReader_;
|
||||
#ifndef MCAP_COMPRESSION_NO_LZ4
|
||||
LZ4Reader lz4Reader_;
|
||||
#endif
|
||||
#ifndef MCAP_COMPRESSION_NO_ZSTD
|
||||
ZStdReader zstdReader_;
|
||||
#endif
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief A mid-level interface for parsing and validating MCAP records from a
|
||||
* data source.
|
||||
*/
|
||||
struct MCAP_PUBLIC TypedRecordReader {
|
||||
std::function<void(const Header&, ByteOffset)> onHeader;
|
||||
std::function<void(const Footer&, ByteOffset)> onFooter;
|
||||
std::function<void(const SchemaPtr, ByteOffset, std::optional<ByteOffset>)> onSchema;
|
||||
std::function<void(const ChannelPtr, ByteOffset, std::optional<ByteOffset>)> onChannel;
|
||||
std::function<void(const Message&, ByteOffset, std::optional<ByteOffset>)> onMessage;
|
||||
std::function<void(const Chunk&, ByteOffset)> onChunk;
|
||||
std::function<void(const MessageIndex&, ByteOffset)> onMessageIndex;
|
||||
std::function<void(const ChunkIndex&, ByteOffset)> onChunkIndex;
|
||||
std::function<void(const Attachment&, ByteOffset)> onAttachment;
|
||||
std::function<void(const AttachmentIndex&, ByteOffset)> onAttachmentIndex;
|
||||
std::function<void(const Statistics&, ByteOffset)> onStatistics;
|
||||
std::function<void(const Metadata&, ByteOffset)> onMetadata;
|
||||
std::function<void(const MetadataIndex&, ByteOffset)> onMetadataIndex;
|
||||
std::function<void(const SummaryOffset&, ByteOffset)> onSummaryOffset;
|
||||
std::function<void(const DataEnd&, ByteOffset)> onDataEnd;
|
||||
std::function<void(const Record&, ByteOffset, std::optional<ByteOffset>)> onUnknownRecord;
|
||||
std::function<void(ByteOffset)> onChunkEnd;
|
||||
|
||||
TypedRecordReader(IReadable& dataSource, ByteOffset startOffset,
|
||||
ByteOffset endOffset = EndOffset);
|
||||
|
||||
TypedRecordReader(const TypedRecordReader&) = delete;
|
||||
TypedRecordReader& operator=(const TypedRecordReader&) = delete;
|
||||
TypedRecordReader(TypedRecordReader&&) = delete;
|
||||
TypedRecordReader& operator=(TypedRecordReader&&) = delete;
|
||||
|
||||
bool next();
|
||||
|
||||
ByteOffset offset() const;
|
||||
|
||||
const Status& status() const;
|
||||
|
||||
private:
|
||||
RecordReader reader_;
|
||||
TypedChunkReader chunkReader_;
|
||||
Status status_;
|
||||
bool parsingChunk_;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Uses message indices to read messages out of an MCAP in log time order.
|
||||
* The underlying MCAP must be chunked, with a summary section and message indexes.
|
||||
* The required McapWriterOptions are:
|
||||
* - noChunking: false
|
||||
* - noMessageIndex: false
|
||||
* - noSummary: false
|
||||
*/
|
||||
struct MCAP_PUBLIC IndexedMessageReader {
|
||||
public:
|
||||
IndexedMessageReader(McapReader& reader, const ReadMessageOptions& options,
|
||||
const std::function<void(const Message&, RecordOffset)> onMessage);
|
||||
|
||||
/**
|
||||
* @brief reads the next message out of the MCAP.
|
||||
*
|
||||
* @return true if a message was found.
|
||||
* @return false if no more messages are to be read. If there was some error reading the MCAP,
|
||||
* `status()` will return a non-Success status.
|
||||
*/
|
||||
bool next();
|
||||
|
||||
/**
|
||||
* @brief gets the status of the reader.
|
||||
*
|
||||
* @return Status
|
||||
*/
|
||||
Status status() const;
|
||||
|
||||
private:
|
||||
struct ChunkSlot {
|
||||
ByteArray decompressedChunk;
|
||||
ByteOffset chunkStartOffset;
|
||||
int unreadMessages = 0;
|
||||
};
|
||||
size_t findFreeChunkSlot();
|
||||
void decompressChunk(const Chunk& chunk, ChunkSlot& slot);
|
||||
Status status_;
|
||||
McapReader& mcapReader_;
|
||||
RecordReader recordReader_;
|
||||
#ifndef MCAP_COMPRESSION_NO_LZ4
|
||||
LZ4Reader lz4Reader_;
|
||||
#endif
|
||||
ReadMessageOptions options_;
|
||||
std::unordered_set<ChannelId> selectedChannels_;
|
||||
std::function<void(const Message&, RecordOffset)> onMessage_;
|
||||
internal::ReadJobQueue queue_;
|
||||
std::vector<ChunkSlot> chunkSlots_;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief An iterable view of Messages in an MCAP file.
|
||||
*/
|
||||
struct MCAP_PUBLIC LinearMessageView {
|
||||
struct MCAP_PUBLIC Iterator {
|
||||
using iterator_category = std::input_iterator_tag;
|
||||
using difference_type = int64_t;
|
||||
using value_type = MessageView;
|
||||
using pointer = const MessageView*;
|
||||
using reference = const MessageView&;
|
||||
|
||||
reference operator*() const;
|
||||
pointer operator->() const;
|
||||
Iterator& operator++();
|
||||
void operator++(int);
|
||||
MCAP_PUBLIC friend bool operator==(const Iterator& a, const Iterator& b);
|
||||
MCAP_PUBLIC friend bool operator!=(const Iterator& a, const Iterator& b);
|
||||
|
||||
private:
|
||||
friend LinearMessageView;
|
||||
|
||||
Iterator() = default;
|
||||
Iterator(LinearMessageView& view);
|
||||
|
||||
class Impl {
|
||||
public:
|
||||
Impl(LinearMessageView& view);
|
||||
|
||||
Impl(const Impl&) = delete;
|
||||
Impl& operator=(const Impl&) = delete;
|
||||
Impl(Impl&&) = delete;
|
||||
Impl& operator=(Impl&&) = delete;
|
||||
|
||||
void increment();
|
||||
reference dereference() const;
|
||||
bool has_value() const;
|
||||
|
||||
LinearMessageView& view_;
|
||||
|
||||
std::optional<TypedRecordReader> recordReader_;
|
||||
std::optional<IndexedMessageReader> indexedMessageReader_;
|
||||
Message curMessage_;
|
||||
std::optional<MessageView> curMessageView_;
|
||||
|
||||
private:
|
||||
void onMessage(const Message& message, RecordOffset offset);
|
||||
};
|
||||
|
||||
bool begun_ = false;
|
||||
std::unique_ptr<Impl> impl_;
|
||||
};
|
||||
|
||||
LinearMessageView(McapReader& mcapReader, const ProblemCallback& onProblem);
|
||||
LinearMessageView(McapReader& mcapReader, ByteOffset dataStart, ByteOffset dataEnd,
|
||||
Timestamp startTime, Timestamp endTime, const ProblemCallback& onProblem);
|
||||
LinearMessageView(McapReader& mcapReader, const ReadMessageOptions& options, ByteOffset dataStart,
|
||||
ByteOffset dataEnd, const ProblemCallback& onProblem);
|
||||
|
||||
LinearMessageView(const LinearMessageView&) = delete;
|
||||
LinearMessageView& operator=(const LinearMessageView&) = delete;
|
||||
LinearMessageView(LinearMessageView&&) = default;
|
||||
LinearMessageView& operator=(LinearMessageView&&) = delete;
|
||||
|
||||
Iterator begin();
|
||||
Iterator end();
|
||||
|
||||
private:
|
||||
McapReader& mcapReader_;
|
||||
ByteOffset dataStart_;
|
||||
ByteOffset dataEnd_;
|
||||
ReadMessageOptions readMessageOptions_;
|
||||
const ProblemCallback onProblem_;
|
||||
};
|
||||
|
||||
} // namespace mcap
|
||||
|
||||
#ifdef MCAP_IMPLEMENTATION
|
||||
# include "reader.inl"
|
||||
#endif
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,407 @@
|
||||
#pragma once
|
||||
|
||||
#include "errors.hpp"
|
||||
#include "visibility.hpp"
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <limits>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
namespace mcap {
|
||||
|
||||
#define MCAP_LIBRARY_VERSION "2.1.3"
|
||||
|
||||
using SchemaId = uint16_t;
|
||||
using ChannelId = uint16_t;
|
||||
using Timestamp = uint64_t;
|
||||
using ByteOffset = uint64_t;
|
||||
using KeyValueMap = std::unordered_map<std::string, std::string>;
|
||||
using ByteArray = std::vector<std::byte>;
|
||||
using ProblemCallback = std::function<void(const Status&)>;
|
||||
|
||||
constexpr char SpecVersion = '0';
|
||||
constexpr char LibraryVersion[] = MCAP_LIBRARY_VERSION;
|
||||
constexpr uint8_t Magic[] = {137, 77, 67, 65, 80, SpecVersion, 13, 10}; // "\x89MCAP0\r\n"
|
||||
constexpr uint64_t DefaultChunkSize = 1024 * 768;
|
||||
constexpr ByteOffset EndOffset = std::numeric_limits<ByteOffset>::max();
|
||||
constexpr Timestamp MaxTime = std::numeric_limits<Timestamp>::max();
|
||||
|
||||
/**
|
||||
* @brief Supported MCAP compression algorithms.
|
||||
*/
|
||||
enum struct Compression {
|
||||
None,
|
||||
Lz4,
|
||||
Zstd,
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Compression level to use when compression is enabled. Slower generally
|
||||
* produces smaller files, at the expense of more CPU time. These levels map to
|
||||
* different internal settings for each compression algorithm.
|
||||
*/
|
||||
enum struct CompressionLevel {
|
||||
Fastest,
|
||||
Fast,
|
||||
Default,
|
||||
Slow,
|
||||
Slowest,
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief MCAP record types.
|
||||
*/
|
||||
enum struct OpCode : uint8_t {
|
||||
Header = 0x01,
|
||||
Footer = 0x02,
|
||||
Schema = 0x03,
|
||||
Channel = 0x04,
|
||||
Message = 0x05,
|
||||
Chunk = 0x06,
|
||||
MessageIndex = 0x07,
|
||||
ChunkIndex = 0x08,
|
||||
Attachment = 0x09,
|
||||
AttachmentIndex = 0x0A,
|
||||
Statistics = 0x0B,
|
||||
Metadata = 0x0C,
|
||||
MetadataIndex = 0x0D,
|
||||
SummaryOffset = 0x0E,
|
||||
DataEnd = 0x0F,
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Get the string representation of an OpCode.
|
||||
*/
|
||||
MCAP_PUBLIC
|
||||
constexpr std::string_view OpCodeString(OpCode opcode);
|
||||
|
||||
/**
|
||||
* @brief A generic Type-Length-Value record using a uint8 type and uint64
|
||||
* length. This is the generic form of all MCAP records.
|
||||
*/
|
||||
struct MCAP_PUBLIC Record {
|
||||
OpCode opcode;
|
||||
uint64_t dataSize;
|
||||
std::byte* data;
|
||||
|
||||
uint64_t recordSize() const {
|
||||
return sizeof(opcode) + sizeof(dataSize) + dataSize;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Appears at the beginning of every MCAP file (after the magic byte
|
||||
* sequence) and contains the recording profile (see
|
||||
* <https://github.com/foxglove/mcap/tree/main/docs/specification/profiles>) and
|
||||
* a string signature of the recording library.
|
||||
*/
|
||||
struct MCAP_PUBLIC Header {
|
||||
std::string profile;
|
||||
std::string library;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief The final record in an MCAP file (before the trailing magic byte
|
||||
* sequence). Contains byte offsets from the start of the file to the Summary
|
||||
* and Summary Offset sections, along with an optional CRC of the combined
|
||||
* Summary and Summary Offset sections. A `summaryStart` and
|
||||
* `summaryOffsetStart` of zero indicates no Summary section is available.
|
||||
*/
|
||||
struct MCAP_PUBLIC Footer {
|
||||
ByteOffset summaryStart;
|
||||
ByteOffset summaryOffsetStart;
|
||||
uint32_t summaryCrc;
|
||||
|
||||
Footer() = default;
|
||||
Footer(ByteOffset _summaryStart, ByteOffset _summaryOffsetStart)
|
||||
: summaryStart(_summaryStart)
|
||||
, summaryOffsetStart(_summaryOffsetStart)
|
||||
, summaryCrc(0) {}
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Describes a schema used for message encoding and decoding and/or
|
||||
* describing the shape of messages. One or more Channel records map to a single
|
||||
* Schema.
|
||||
*/
|
||||
struct MCAP_PUBLIC Schema {
|
||||
SchemaId id;
|
||||
std::string name;
|
||||
std::string encoding;
|
||||
ByteArray data;
|
||||
|
||||
Schema() = default;
|
||||
|
||||
Schema(const std::string_view _name, const std::string_view _encoding,
|
||||
const std::string_view _data)
|
||||
: name(_name)
|
||||
, encoding(_encoding)
|
||||
, data{reinterpret_cast<const std::byte*>(_data.data()),
|
||||
reinterpret_cast<const std::byte*>(_data.data() + _data.size())} {}
|
||||
|
||||
Schema(const std::string_view _name, const std::string_view _encoding, const ByteArray& _data)
|
||||
: name(_name)
|
||||
, encoding(_encoding)
|
||||
, data{_data} {}
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Describes a Channel that messages are written to. A Channel represents
|
||||
* a single connection from a publisher to a topic, so each topic will have one
|
||||
* Channel per publisher. Channels optionally reference a Schema, for message
|
||||
* encodings that are not self-describing (e.g. JSON) or when schema information
|
||||
* is available (e.g. JSONSchema).
|
||||
*/
|
||||
struct MCAP_PUBLIC Channel {
|
||||
ChannelId id;
|
||||
std::string topic;
|
||||
std::string messageEncoding;
|
||||
SchemaId schemaId;
|
||||
KeyValueMap metadata;
|
||||
|
||||
Channel() = default;
|
||||
|
||||
Channel(const std::string_view _topic, const std::string_view _messageEncoding,
|
||||
SchemaId _schemaId, const KeyValueMap& _metadata = {})
|
||||
: topic(_topic)
|
||||
, messageEncoding(_messageEncoding)
|
||||
, schemaId(_schemaId)
|
||||
, metadata(_metadata) {}
|
||||
};
|
||||
|
||||
using SchemaPtr = std::shared_ptr<Schema>;
|
||||
using ChannelPtr = std::shared_ptr<Channel>;
|
||||
|
||||
/**
|
||||
* @brief A single Message published to a Channel.
|
||||
*/
|
||||
struct MCAP_PUBLIC Message {
|
||||
ChannelId channelId;
|
||||
/**
|
||||
* @brief An optional sequence number. If non-zero, sequence numbers should be
|
||||
* unique per channel and increasing over time.
|
||||
*/
|
||||
uint32_t sequence;
|
||||
/**
|
||||
* @brief Nanosecond timestamp when this message was recorded or received for
|
||||
* recording.
|
||||
*/
|
||||
Timestamp logTime;
|
||||
/**
|
||||
* @brief Nanosecond timestamp when this message was initially published. If
|
||||
* not available, this should be set to `logTime`.
|
||||
*/
|
||||
Timestamp publishTime;
|
||||
/**
|
||||
* @brief Size of the message payload in bytes, pointed to via `data`.
|
||||
*/
|
||||
uint64_t dataSize;
|
||||
/**
|
||||
* @brief A pointer to the message payload. For readers, this pointer is only
|
||||
* valid for the lifetime of an onMessage callback or before the message
|
||||
* iterator is advanced.
|
||||
*/
|
||||
const std::byte* data = nullptr;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief An collection of Schemas, Channels, and Messages that supports
|
||||
* compression and indexing.
|
||||
*/
|
||||
struct MCAP_PUBLIC Chunk {
|
||||
Timestamp messageStartTime;
|
||||
Timestamp messageEndTime;
|
||||
ByteOffset uncompressedSize;
|
||||
uint32_t uncompressedCrc;
|
||||
std::string compression;
|
||||
ByteOffset compressedSize;
|
||||
const std::byte* records = nullptr;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief A list of timestamps to byte offsets for a single Channel. This record
|
||||
* appears after each Chunk, one per Channel that appeared in that Chunk.
|
||||
*/
|
||||
struct MCAP_PUBLIC MessageIndex {
|
||||
ChannelId channelId;
|
||||
std::vector<std::pair<Timestamp, ByteOffset>> records;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Chunk Index records are found in the Summary section, providing
|
||||
* summary information for a single Chunk and pointing to each Message Index
|
||||
* record associated with that Chunk.
|
||||
*/
|
||||
struct MCAP_PUBLIC ChunkIndex {
|
||||
Timestamp messageStartTime;
|
||||
Timestamp messageEndTime;
|
||||
ByteOffset chunkStartOffset;
|
||||
ByteOffset chunkLength;
|
||||
std::unordered_map<ChannelId, ByteOffset> messageIndexOffsets;
|
||||
ByteOffset messageIndexLength;
|
||||
std::string compression;
|
||||
ByteOffset compressedSize;
|
||||
ByteOffset uncompressedSize;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief An Attachment is an arbitrary file embedded in an MCAP file, including
|
||||
* a name, media type, timestamps, and optional CRC. Attachment records are
|
||||
* written in the Data section, outside of Chunks.
|
||||
*/
|
||||
struct MCAP_PUBLIC Attachment {
|
||||
Timestamp logTime;
|
||||
Timestamp createTime;
|
||||
std::string name;
|
||||
std::string mediaType;
|
||||
uint64_t dataSize;
|
||||
const std::byte* data = nullptr;
|
||||
uint32_t crc;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Attachment Index records are found in the Summary section, providing
|
||||
* summary information for a single Attachment.
|
||||
*/
|
||||
struct MCAP_PUBLIC AttachmentIndex {
|
||||
ByteOffset offset;
|
||||
ByteOffset length;
|
||||
Timestamp logTime;
|
||||
Timestamp createTime;
|
||||
uint64_t dataSize;
|
||||
std::string name;
|
||||
std::string mediaType;
|
||||
|
||||
AttachmentIndex() = default;
|
||||
AttachmentIndex(const Attachment& attachment, ByteOffset fileOffset)
|
||||
: offset(fileOffset)
|
||||
, length(9 +
|
||||
/* name */ 4 + attachment.name.size() +
|
||||
/* log_time */ 8 +
|
||||
/* create_time */ 8 +
|
||||
/* media_type */ 4 + attachment.mediaType.size() +
|
||||
/* data */ 8 + attachment.dataSize +
|
||||
/* crc */ 4)
|
||||
, logTime(attachment.logTime)
|
||||
, createTime(attachment.createTime)
|
||||
, dataSize(attachment.dataSize)
|
||||
, name(attachment.name)
|
||||
, mediaType(attachment.mediaType) {}
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief The Statistics record is found in the Summary section, providing
|
||||
* counts and timestamp ranges for the entire file.
|
||||
*/
|
||||
struct MCAP_PUBLIC Statistics {
|
||||
uint64_t messageCount;
|
||||
uint16_t schemaCount;
|
||||
uint32_t channelCount;
|
||||
uint32_t attachmentCount;
|
||||
uint32_t metadataCount;
|
||||
uint32_t chunkCount;
|
||||
Timestamp messageStartTime;
|
||||
Timestamp messageEndTime;
|
||||
std::unordered_map<ChannelId, uint64_t> channelMessageCounts;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Holds a named map of key/value strings containing arbitrary user data.
|
||||
* Metadata records are found in the Data section, outside of Chunks.
|
||||
*/
|
||||
struct MCAP_PUBLIC Metadata {
|
||||
std::string name;
|
||||
KeyValueMap metadata;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Metadata Index records are found in the Summary section, providing
|
||||
* summary information for a single Metadata record.
|
||||
*/
|
||||
struct MCAP_PUBLIC MetadataIndex {
|
||||
uint64_t offset;
|
||||
uint64_t length;
|
||||
std::string name;
|
||||
|
||||
MetadataIndex() = default;
|
||||
MetadataIndex(const Metadata& metadata, ByteOffset fileOffset);
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Summary Offset records are found in the Summary Offset section.
|
||||
* Records in the Summary section are grouped together, and for each record type
|
||||
* found in the Summary section, a Summary Offset references the file offset and
|
||||
* length where that type of Summary record can be found.
|
||||
*/
|
||||
struct MCAP_PUBLIC SummaryOffset {
|
||||
OpCode groupOpCode;
|
||||
ByteOffset groupStart;
|
||||
ByteOffset groupLength;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief The final record in the Data section, signaling the end of Data and
|
||||
* beginning of Summary. Optionally contains a CRC of the entire Data section.
|
||||
*/
|
||||
struct MCAP_PUBLIC DataEnd {
|
||||
uint32_t dataSectionCrc;
|
||||
};
|
||||
|
||||
struct MCAP_PUBLIC RecordOffset {
|
||||
ByteOffset offset;
|
||||
std::optional<ByteOffset> chunkOffset;
|
||||
|
||||
RecordOffset() = default;
|
||||
explicit RecordOffset(ByteOffset offset_)
|
||||
: offset(offset_) {}
|
||||
RecordOffset(ByteOffset offset_, ByteOffset chunkOffset_)
|
||||
: offset(offset_)
|
||||
, chunkOffset(chunkOffset_) {}
|
||||
|
||||
bool operator==(const RecordOffset& other) const;
|
||||
bool operator>(const RecordOffset& other) const;
|
||||
|
||||
bool operator!=(const RecordOffset& other) const {
|
||||
return !(*this == other);
|
||||
}
|
||||
bool operator>=(const RecordOffset& other) const {
|
||||
return ((*this == other) || (*this > other));
|
||||
}
|
||||
bool operator<(const RecordOffset& other) const {
|
||||
return !(*this >= other);
|
||||
}
|
||||
bool operator<=(const RecordOffset& other) const {
|
||||
return !(*this > other);
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Returned when iterating over Messages in a file, MessageView contains
|
||||
* a reference to one Message, a pointer to its Channel, and an optional pointer
|
||||
* to that Channel's Schema. The Channel pointer is guaranteed to be valid,
|
||||
* while the Schema pointer may be null if the Channel references schema_id 0.
|
||||
*/
|
||||
struct MCAP_PUBLIC MessageView {
|
||||
const Message& message;
|
||||
const ChannelPtr channel;
|
||||
const SchemaPtr schema;
|
||||
const RecordOffset messageOffset;
|
||||
|
||||
MessageView(const Message& _message, const ChannelPtr _channel, const SchemaPtr _schema,
|
||||
RecordOffset offset)
|
||||
: message(_message)
|
||||
, channel(_channel)
|
||||
, schema(_schema)
|
||||
, messageOffset(offset) {}
|
||||
};
|
||||
|
||||
} // namespace mcap
|
||||
|
||||
#ifdef MCAP_IMPLEMENTATION
|
||||
# include "types.inl"
|
||||
#endif
|
||||
@@ -0,0 +1,86 @@
|
||||
#include "internal.hpp"
|
||||
|
||||
namespace mcap {
|
||||
|
||||
constexpr std::string_view OpCodeString(OpCode opcode) {
|
||||
switch (opcode) {
|
||||
case OpCode::Header:
|
||||
return "Header";
|
||||
case OpCode::Footer:
|
||||
return "Footer";
|
||||
case OpCode::Schema:
|
||||
return "Schema";
|
||||
case OpCode::Channel:
|
||||
return "Channel";
|
||||
case OpCode::Message:
|
||||
return "Message";
|
||||
case OpCode::Chunk:
|
||||
return "Chunk";
|
||||
case OpCode::MessageIndex:
|
||||
return "MessageIndex";
|
||||
case OpCode::ChunkIndex:
|
||||
return "ChunkIndex";
|
||||
case OpCode::Attachment:
|
||||
return "Attachment";
|
||||
case OpCode::AttachmentIndex:
|
||||
return "AttachmentIndex";
|
||||
case OpCode::Statistics:
|
||||
return "Statistics";
|
||||
case OpCode::Metadata:
|
||||
return "Metadata";
|
||||
case OpCode::MetadataIndex:
|
||||
return "MetadataIndex";
|
||||
case OpCode::SummaryOffset:
|
||||
return "SummaryOffset";
|
||||
case OpCode::DataEnd:
|
||||
return "DataEnd";
|
||||
default:
|
||||
return "Unknown";
|
||||
}
|
||||
}
|
||||
|
||||
MetadataIndex::MetadataIndex(const Metadata& metadata, ByteOffset fileOffset)
|
||||
: offset(fileOffset)
|
||||
, length(9 + 4 + metadata.name.size() + 4 + internal::KeyValueMapSize(metadata.metadata))
|
||||
, name(metadata.name) {}
|
||||
|
||||
bool RecordOffset::operator==(const RecordOffset& other) const {
|
||||
if (chunkOffset != std::nullopt && other.chunkOffset != std::nullopt) {
|
||||
if (*chunkOffset != *other.chunkOffset) {
|
||||
// messages are in separate chunks, cannot be equal.
|
||||
return false;
|
||||
}
|
||||
// messages are in the same chunk, compare chunk-level offsets.
|
||||
return (offset == other.offset);
|
||||
}
|
||||
if (chunkOffset != std::nullopt || other.chunkOffset != std::nullopt) {
|
||||
// one message is in a chunk and one is not, cannot be equal.
|
||||
return false;
|
||||
}
|
||||
// neither message is in a chunk, compare file-level offsets.
|
||||
return (offset == other.offset);
|
||||
}
|
||||
|
||||
bool RecordOffset::operator>(const RecordOffset& other) const {
|
||||
if (chunkOffset != std::nullopt) {
|
||||
if (other.chunkOffset != std::nullopt) {
|
||||
if (*chunkOffset == *other.chunkOffset) {
|
||||
// messages are in the same chunk, compare chunk-level offsets.
|
||||
return (offset > other.offset);
|
||||
}
|
||||
// messages are in separate chunks, compare file-level offsets
|
||||
return (*chunkOffset > *other.chunkOffset);
|
||||
} else {
|
||||
// this message is in a chunk, other is not, compare file-level offsets.
|
||||
return (*chunkOffset > other.offset);
|
||||
}
|
||||
}
|
||||
if (other.chunkOffset != std::nullopt) {
|
||||
// other message is in a chunk, this is not, compare file-level offsets.
|
||||
return (offset > *other.chunkOffset);
|
||||
}
|
||||
// neither message is in a chunk, compare file-level offsets.
|
||||
return (offset > other.offset);
|
||||
}
|
||||
|
||||
} // namespace mcap
|
||||
@@ -0,0 +1,28 @@
|
||||
/** Defines an MCAP_PUBLIC visibility attribute macro, which is used on all public interfaces.
|
||||
* This can be defined before including `mcap.hpp` to directly control symbol visibility.
|
||||
* If not defined externally, this library attempts to export symbols from the translation unit
|
||||
* where MCAP_IMPLEMENTATION is defined, and import them anywhere else.
|
||||
*/
|
||||
#ifndef MCAP_PUBLIC
|
||||
#if defined _WIN32 || defined __CYGWIN__
|
||||
# ifdef MCAP_IMPLEMENTATION
|
||||
# ifdef __GNUC__
|
||||
# define MCAP_PUBLIC __attribute__((dllexport))
|
||||
# else
|
||||
# define MCAP_PUBLIC __declspec(dllexport)
|
||||
# endif
|
||||
# else
|
||||
# ifdef __GNUC__
|
||||
# define MCAP_PUBLIC __attribute__((dllimport))
|
||||
# else
|
||||
# define MCAP_PUBLIC __declspec(dllimport)
|
||||
# endif
|
||||
# endif
|
||||
#else
|
||||
# if __GNUC__ >= 4
|
||||
# define MCAP_PUBLIC __attribute__((visibility("default")))
|
||||
# else
|
||||
# define MCAP_PUBLIC
|
||||
# endif
|
||||
#endif
|
||||
#endif
|
||||
@@ -0,0 +1,514 @@
|
||||
#pragma once
|
||||
|
||||
#include "types.hpp"
|
||||
#include "visibility.hpp"
|
||||
#include <cstdio>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <unordered_set>
|
||||
#include <vector>
|
||||
|
||||
// Forward declaration
|
||||
#ifndef MCAP_COMPRESSION_NO_ZSTD
|
||||
struct ZSTD_CCtx_s;
|
||||
#endif
|
||||
|
||||
namespace mcap {
|
||||
|
||||
/**
|
||||
* @brief Configuration options for McapWriter.
|
||||
*/
|
||||
struct MCAP_PUBLIC McapWriterOptions {
|
||||
/**
|
||||
* @brief Disable CRC calculations for Chunks.
|
||||
*/
|
||||
bool noChunkCRC = false;
|
||||
/**
|
||||
* @brief Disable CRC calculations for Attachments.
|
||||
*/
|
||||
bool noAttachmentCRC = false;
|
||||
/**
|
||||
* @brief Enable CRC calculations for all records in the data section.
|
||||
*/
|
||||
bool enableDataCRC = false;
|
||||
/**
|
||||
* @brief Disable CRC calculations for the summary section.
|
||||
*/
|
||||
bool noSummaryCRC = false;
|
||||
/**
|
||||
* @brief Do not write Chunks to the file, instead writing Schema, Channel,
|
||||
* and Message records directly into the Data section.
|
||||
*/
|
||||
bool noChunking = false;
|
||||
/**
|
||||
* @brief Do not write Message Index records to the file. If
|
||||
* `noMessageIndex=true` and `noChunkIndex=false`, Chunk Index records will
|
||||
* still be written to the Summary section, providing a coarse message index.
|
||||
*/
|
||||
bool noMessageIndex = false;
|
||||
/**
|
||||
* @brief Do not write Summary or Summary Offset sections to the file, placing
|
||||
* the Footer record immediately after DataEnd. This can provide some speed
|
||||
* boost to file writing and produce smaller files, at the expense of
|
||||
* requiring a conversion process later if fast summarization or indexed
|
||||
* access is desired.
|
||||
*/
|
||||
bool noSummary = false;
|
||||
/**
|
||||
* @brief Target uncompressed Chunk payload size in bytes. Once a Chunk's
|
||||
* uncompressed data is about to exceed this size, the Chunk will be
|
||||
* compressed (if enabled) and written to disk. Note that this is a 'soft'
|
||||
* ceiling as some Chunks could exceed this size due to either indexing
|
||||
* data or when a single message is larger than `chunkSize`, in which case,
|
||||
* the Chunk will contain only this one large message.
|
||||
* This option is ignored if `noChunking=true`.
|
||||
*/
|
||||
uint64_t chunkSize = DefaultChunkSize;
|
||||
/**
|
||||
* @brief Compression algorithm to use when writing Chunks. This option is
|
||||
* ignored if `noChunking=true`.
|
||||
*/
|
||||
Compression compression = Compression::Zstd;
|
||||
/**
|
||||
* @brief Compression level to use when writing Chunks. Slower generally
|
||||
* produces smaller files, at the expense of more CPU time. These levels map
|
||||
* to different internal settings for each compression algorithm.
|
||||
*/
|
||||
CompressionLevel compressionLevel = CompressionLevel::Default;
|
||||
/**
|
||||
* @brief By default, Chunks that do not benefit from compression will be
|
||||
* written uncompressed. This option can be used to force compression on all
|
||||
* Chunks. This option is ignored if `noChunking=true`.
|
||||
*/
|
||||
bool forceCompression = false;
|
||||
/**
|
||||
* @brief The recording profile. See
|
||||
* https://mcap.dev/spec/registry#well-known-profiles
|
||||
* for more information on well-known profiles.
|
||||
*/
|
||||
std::string profile;
|
||||
/**
|
||||
* @brief A freeform string written by recording libraries. For this library,
|
||||
* the default is "libmcap {Major}.{Minor}.{Patch}".
|
||||
*/
|
||||
std::string library = "libmcap " MCAP_LIBRARY_VERSION;
|
||||
|
||||
// The following options are less commonly used, providing more fine-grained
|
||||
// control of index records and the Summary section
|
||||
|
||||
bool noRepeatedSchemas = false;
|
||||
bool noRepeatedChannels = false;
|
||||
bool noAttachmentIndex = false;
|
||||
bool noMetadataIndex = false;
|
||||
bool noChunkIndex = false;
|
||||
bool noStatistics = false;
|
||||
bool noSummaryOffsets = false;
|
||||
|
||||
McapWriterOptions(const std::string_view _profile)
|
||||
: profile(_profile) {}
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief An abstract interface for writing MCAP data.
|
||||
*/
|
||||
class MCAP_PUBLIC IWritable {
|
||||
public:
|
||||
bool crcEnabled = false;
|
||||
|
||||
IWritable() noexcept;
|
||||
virtual ~IWritable() = default;
|
||||
|
||||
/**
|
||||
* @brief Called whenever the writer needs to write data to the output MCAP
|
||||
* file.
|
||||
*
|
||||
* @param data A pointer to the data to write.
|
||||
* @param size Size of the data in bytes.
|
||||
*/
|
||||
void write(const std::byte* data, uint64_t size);
|
||||
/**
|
||||
* @brief Called when the writer is finished writing data to the output MCAP
|
||||
* file.
|
||||
*/
|
||||
virtual void end() = 0;
|
||||
/**
|
||||
* @brief Returns the current size of the file in bytes. This must be equal to
|
||||
* the sum of all `size` parameters passed to `write()`.
|
||||
*/
|
||||
virtual uint64_t size() const = 0;
|
||||
/**
|
||||
* @brief Returns the CRC32 of the uncompressed data.
|
||||
*/
|
||||
uint32_t crc();
|
||||
/**
|
||||
* @brief Resets the CRC32 calculation.
|
||||
*/
|
||||
void resetCrc();
|
||||
|
||||
/**
|
||||
* @brief flushes any buffered data to the output. This is called by McapWriter after every
|
||||
* completed chunk. Callers may also retain a reference to the writer and call flush() at their
|
||||
* own cadence. Defaults to a no-op.
|
||||
*/
|
||||
virtual void flush() {}
|
||||
|
||||
protected:
|
||||
virtual void handleWrite(const std::byte* data, uint64_t size) = 0;
|
||||
|
||||
private:
|
||||
uint32_t crc_;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Implements the IWritable interface used by McapWriter by wrapping a
|
||||
* FILE* pointer created by fopen().
|
||||
*/
|
||||
class MCAP_PUBLIC FileWriter final : public IWritable {
|
||||
public:
|
||||
~FileWriter() override;
|
||||
|
||||
Status open(std::string_view filename);
|
||||
|
||||
void handleWrite(const std::byte* data, uint64_t size) override;
|
||||
void end() override;
|
||||
void flush() override;
|
||||
uint64_t size() const override;
|
||||
|
||||
private:
|
||||
std::FILE* file_ = nullptr;
|
||||
uint64_t size_ = 0;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Implements the IWritable interface used by McapWriter by wrapping a
|
||||
* std::ostream stream.
|
||||
*/
|
||||
class MCAP_PUBLIC StreamWriter final : public IWritable {
|
||||
public:
|
||||
StreamWriter(std::ostream& stream);
|
||||
|
||||
void handleWrite(const std::byte* data, uint64_t size) override;
|
||||
void end() override;
|
||||
void flush() override;
|
||||
uint64_t size() const override;
|
||||
|
||||
private:
|
||||
std::ostream& stream_;
|
||||
uint64_t size_ = 0;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief An abstract interface for writing Chunk data. Chunk data is buffered
|
||||
* in memory and written to disk as a single record, to support optimal
|
||||
* compression and calculating the final Chunk data size.
|
||||
*/
|
||||
class MCAP_PUBLIC IChunkWriter : public IWritable {
|
||||
public:
|
||||
virtual ~IChunkWriter() override = default;
|
||||
|
||||
/**
|
||||
* @brief Called when the writer wants to close the current output Chunk.
|
||||
* After this call, `data()` and `size()` should return the data and size of
|
||||
* the compressed data.
|
||||
*/
|
||||
virtual void end() override = 0;
|
||||
/**
|
||||
* @brief Returns the size in bytes of the uncompressed data.
|
||||
*/
|
||||
virtual uint64_t size() const override = 0;
|
||||
|
||||
/**
|
||||
* @brief Returns the size in bytes of the compressed data. This will only be
|
||||
* called after `end()`.
|
||||
*/
|
||||
virtual uint64_t compressedSize() const = 0;
|
||||
/**
|
||||
* @brief Returns true if `write()` has never been called since initialization
|
||||
* or the last call to `clear()`.
|
||||
*/
|
||||
virtual bool empty() const = 0;
|
||||
/**
|
||||
* @brief Clear the internal state of the writer, discarding any input or
|
||||
* output buffers.
|
||||
*/
|
||||
void clear();
|
||||
/**
|
||||
* @brief Returns a pointer to the uncompressed data.
|
||||
*/
|
||||
virtual const std::byte* data() const = 0;
|
||||
/**
|
||||
* @brief Returns a pointer to the compressed data. This will only be called
|
||||
* after `end()`.
|
||||
*/
|
||||
virtual const std::byte* compressedData() const = 0;
|
||||
|
||||
protected:
|
||||
virtual void handleClear() = 0;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief An in-memory IChunkWriter implementation backed by a
|
||||
* growable buffer.
|
||||
*/
|
||||
class MCAP_PUBLIC BufferWriter final : public IChunkWriter {
|
||||
public:
|
||||
void handleWrite(const std::byte* data, uint64_t size) override;
|
||||
void end() override;
|
||||
uint64_t size() const override;
|
||||
uint64_t compressedSize() const override;
|
||||
bool empty() const override;
|
||||
void handleClear() override;
|
||||
const std::byte* data() const override;
|
||||
const std::byte* compressedData() const override;
|
||||
|
||||
private:
|
||||
std::vector<std::byte> buffer_;
|
||||
};
|
||||
|
||||
#ifndef MCAP_COMPRESSION_NO_LZ4
|
||||
/**
|
||||
* @brief An in-memory IChunkWriter implementation that holds data in a
|
||||
* temporary buffer before flushing to an LZ4-compressed buffer.
|
||||
*/
|
||||
class MCAP_PUBLIC LZ4Writer final : public IChunkWriter {
|
||||
public:
|
||||
LZ4Writer(CompressionLevel compressionLevel, uint64_t chunkSize);
|
||||
|
||||
void handleWrite(const std::byte* data, uint64_t size) override;
|
||||
void end() override;
|
||||
uint64_t size() const override;
|
||||
uint64_t compressedSize() const override;
|
||||
bool empty() const override;
|
||||
void handleClear() override;
|
||||
const std::byte* data() const override;
|
||||
const std::byte* compressedData() const override;
|
||||
|
||||
private:
|
||||
std::vector<std::byte> uncompressedBuffer_;
|
||||
std::vector<std::byte> compressedBuffer_;
|
||||
CompressionLevel compressionLevel_;
|
||||
};
|
||||
#endif
|
||||
|
||||
#ifndef MCAP_COMPRESSION_NO_ZSTD
|
||||
/**
|
||||
* @brief An in-memory IChunkWriter implementation that holds data in a
|
||||
* temporary buffer before flushing to an ZStandard-compressed buffer.
|
||||
*/
|
||||
class MCAP_PUBLIC ZStdWriter final : public IChunkWriter {
|
||||
public:
|
||||
ZStdWriter(CompressionLevel compressionLevel, uint64_t chunkSize);
|
||||
~ZStdWriter() override;
|
||||
|
||||
void handleWrite(const std::byte* data, uint64_t size) override;
|
||||
void end() override;
|
||||
uint64_t size() const override;
|
||||
uint64_t compressedSize() const override;
|
||||
bool empty() const override;
|
||||
void handleClear() override;
|
||||
const std::byte* data() const override;
|
||||
const std::byte* compressedData() const override;
|
||||
|
||||
private:
|
||||
std::vector<std::byte> uncompressedBuffer_;
|
||||
std::vector<std::byte> compressedBuffer_;
|
||||
ZSTD_CCtx_s* zstdContext_ = nullptr;
|
||||
};
|
||||
#endif
|
||||
|
||||
/**
|
||||
* @brief Provides a write interface to an MCAP file.
|
||||
*/
|
||||
class MCAP_PUBLIC McapWriter final {
|
||||
public:
|
||||
~McapWriter();
|
||||
|
||||
/**
|
||||
* @brief Open a new MCAP file for writing and write the header.
|
||||
*
|
||||
* If the writer was already opened, this calls `close`() first to reset the state.
|
||||
* A writer may be re-used after being reset via `close`() or `terminate`().
|
||||
*
|
||||
* @param filename Filename of the MCAP file to write.
|
||||
* @param options Options for MCAP writing. `profile` is required.
|
||||
* @return A non-success status if the file could not be opened for writing.
|
||||
*/
|
||||
Status open(std::string_view filename, const McapWriterOptions& options);
|
||||
|
||||
/**
|
||||
* @brief Open a new MCAP file for writing and write the header.
|
||||
*
|
||||
* If the writer was already opened, this calls `close`() first to reset the state.
|
||||
* A writer may be re-used after being reset via `close`() or `terminate`().
|
||||
*
|
||||
* @param writer An implementation of the IWritable interface. Output bytes
|
||||
* will be written to this object.
|
||||
* @param options Options for MCAP writing. `profile` is required.
|
||||
*/
|
||||
void open(IWritable& writer, const McapWriterOptions& options);
|
||||
|
||||
/**
|
||||
* @brief Open a new MCAP file for writing and write the header.
|
||||
*
|
||||
* @param stream Output stream to write to.
|
||||
* @param options Options for MCAP writing. `profile` is required.
|
||||
*/
|
||||
void open(std::ostream& stream, const McapWriterOptions& options);
|
||||
|
||||
/**
|
||||
* @brief Write the MCAP footer, flush pending writes to the output stream,
|
||||
* and reset internal state. The writer may be re-used with another call to open afterwards.
|
||||
*/
|
||||
void close();
|
||||
|
||||
/**
|
||||
* @brief Reset internal state without writing the MCAP footer or flushing
|
||||
* pending writes. This should only be used in error cases as the output MCAP
|
||||
* file will be truncated. The writer may be re-used with another call to open afterwards.
|
||||
*/
|
||||
void terminate();
|
||||
|
||||
/**
|
||||
* @brief Add a new schema to the MCAP file and set `schema.id` to a generated
|
||||
* schema id. The schema id is used when adding channels to the file.
|
||||
*
|
||||
* Schemas are not cleared when the state is reset via `close`() or `terminate`().
|
||||
* If you're re-using a writer for multiple files in a row, the schemas only need
|
||||
* to be added once, before first use.
|
||||
*
|
||||
* This method does not de-duplicate schemas.
|
||||
*
|
||||
* @param schema Description of the schema to register. The `id` field is
|
||||
* ignored and will be set to a generated schema id.
|
||||
*/
|
||||
void addSchema(Schema& schema);
|
||||
|
||||
/**
|
||||
* @brief Add a new channel to the MCAP file and set `channel.id` to a
|
||||
* generated channel id. The channel id is used when adding messages to the
|
||||
* file.
|
||||
*
|
||||
* Channels are not cleared when the state is reset via `close`() or `terminate`().
|
||||
* If you're re-using a writer for multiple files in a row, the channels only need
|
||||
* to be added once, before first use.
|
||||
*
|
||||
* This method does not de-duplicate channels.
|
||||
*
|
||||
* @param channel Description of the channel to register. The `id` value is
|
||||
* ignored and will be set to a generated channel id.
|
||||
*/
|
||||
void addChannel(Channel& channel);
|
||||
|
||||
/**
|
||||
* @brief Write a message to the output stream.
|
||||
*
|
||||
* @param msg Message to add.
|
||||
* @return A non-zero error code on failure.
|
||||
*/
|
||||
Status write(const Message& message);
|
||||
|
||||
/**
|
||||
* @brief Write an attachment to the output stream.
|
||||
*
|
||||
* @param attachment Attachment to add. The `attachment.crc` will be
|
||||
* calculated and set if configuration options allow CRC calculation.
|
||||
* @return A non-zero error code on failure.
|
||||
*/
|
||||
Status write(Attachment& attachment);
|
||||
|
||||
/**
|
||||
* @brief Write a metadata record to the output stream.
|
||||
*
|
||||
* @param metadata Named group of key/value string pairs to add.
|
||||
* @return A non-zero error code on failure.
|
||||
*/
|
||||
Status write(const Metadata& metadata);
|
||||
|
||||
/**
|
||||
* @brief Current MCAP file-level statistics. This is written as a Statistics
|
||||
* record in the Summary section of the MCAP file.
|
||||
*/
|
||||
const Statistics& statistics() const;
|
||||
|
||||
/**
|
||||
* @brief Returns a pointer to the IWritable data destination backing this
|
||||
* writer. Will return nullptr if the writer is not open.
|
||||
*/
|
||||
IWritable* dataSink();
|
||||
|
||||
/**
|
||||
* @brief finishes the current chunk in progress and writes it to the file, if a chunk
|
||||
* is in progress.
|
||||
*/
|
||||
void closeLastChunk();
|
||||
|
||||
// The following static methods are used for serialization of records and
|
||||
// primitives to an output stream. They are not intended to be used directly
|
||||
// unless you are implementing a lower level writer or tests
|
||||
|
||||
static void writeMagic(IWritable& output);
|
||||
|
||||
static uint64_t write(IWritable& output, const Header& header);
|
||||
static uint64_t write(IWritable& output, const Footer& footer, bool crcEnabled);
|
||||
static uint64_t write(IWritable& output, const Schema& schema);
|
||||
static uint64_t write(IWritable& output, const Channel& channel);
|
||||
static uint64_t getRecordSize(const Message& message);
|
||||
static uint64_t write(IWritable& output, const Message& message);
|
||||
static uint64_t write(IWritable& output, const Attachment& attachment);
|
||||
static uint64_t write(IWritable& output, const Metadata& metadata);
|
||||
static uint64_t write(IWritable& output, const Chunk& chunk);
|
||||
static uint64_t write(IWritable& output, const MessageIndex& index);
|
||||
static uint64_t write(IWritable& output, const ChunkIndex& index);
|
||||
static uint64_t write(IWritable& output, const AttachmentIndex& index);
|
||||
static uint64_t write(IWritable& output, const MetadataIndex& index);
|
||||
static uint64_t write(IWritable& output, const Statistics& stats);
|
||||
static uint64_t write(IWritable& output, const SummaryOffset& summaryOffset);
|
||||
static uint64_t write(IWritable& output, const DataEnd& dataEnd);
|
||||
static uint64_t write(IWritable& output, const Record& record);
|
||||
|
||||
static void write(IWritable& output, const std::string_view str);
|
||||
static void write(IWritable& output, const ByteArray bytes);
|
||||
static void write(IWritable& output, OpCode value);
|
||||
static void write(IWritable& output, uint16_t value);
|
||||
static void write(IWritable& output, uint32_t value);
|
||||
static void write(IWritable& output, uint64_t value);
|
||||
static void write(IWritable& output, const std::byte* data, uint64_t size);
|
||||
static void write(IWritable& output, const KeyValueMap& map, uint32_t size = 0);
|
||||
|
||||
private:
|
||||
McapWriterOptions options_{""};
|
||||
uint64_t chunkSize_ = DefaultChunkSize;
|
||||
IWritable* output_ = nullptr;
|
||||
std::unique_ptr<FileWriter> fileOutput_;
|
||||
std::unique_ptr<StreamWriter> streamOutput_;
|
||||
std::unique_ptr<BufferWriter> uncompressedChunk_;
|
||||
#ifndef MCAP_COMPRESSION_NO_LZ4
|
||||
std::unique_ptr<LZ4Writer> lz4Chunk_;
|
||||
#endif
|
||||
#ifndef MCAP_COMPRESSION_NO_ZSTD
|
||||
std::unique_ptr<ZStdWriter> zstdChunk_;
|
||||
#endif
|
||||
std::vector<Schema> schemas_;
|
||||
std::vector<Channel> channels_;
|
||||
std::vector<AttachmentIndex> attachmentIndex_;
|
||||
std::vector<MetadataIndex> metadataIndex_;
|
||||
std::vector<ChunkIndex> chunkIndex_;
|
||||
Statistics statistics_{};
|
||||
std::unordered_set<SchemaId> writtenSchemas_;
|
||||
std::unordered_map<ChannelId, MessageIndex> currentMessageIndex_;
|
||||
Timestamp currentChunkStart_ = MaxTime;
|
||||
Timestamp currentChunkEnd_ = 0;
|
||||
Compression compression_ = Compression::None;
|
||||
uint64_t uncompressedSize_ = 0;
|
||||
bool opened_ = false;
|
||||
|
||||
IWritable& getOutput();
|
||||
IChunkWriter* getChunkWriter();
|
||||
void writeChunk(IWritable& output, IChunkWriter& chunkData);
|
||||
};
|
||||
|
||||
} // namespace mcap
|
||||
|
||||
#ifdef MCAP_IMPLEMENTATION
|
||||
# include "writer.inl"
|
||||
#endif
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user