22#include <unordered_map>
35 std::uint16_t unsignedWidth(std::uint64_t maxv) {
41 return w ? w : std::uint16_t{1};
48 std::uint16_t signedWidth(std::int64_t minv, std::int64_t maxv) {
49 const auto need = [](std::int64_t v) -> std::uint16_t {
51 while (w < 64 && (v < -(std::int64_t{1} << (w - 1)) || v > (std::int64_t{1} << (w - 1)) - 1)) ++w;
54 return std::max<std::uint16_t>({std::uint16_t{1}, need(minv), need(maxv)});
59 std::uint32_t offsetBits = 0;
60 std::uint16_t elemBits = 32;
61 std::uint16_t elements = 1;
63 std::vector<std::uint32_t> pallet;
64 std::unordered_map<std::string, std::uint32_t> palletIndex;
65 std::uint32_t commonDefault = 0;
68 bool isPallet()
const {
75 std::size_t recordBits()
const {
76 if (isCommon())
return 0;
79 return isPallet() ? elemBits : std::size_t{elemBits} * elements;
95 Writer(std::uint32_t magic,
const TableInfo& info,
const RecordSource& source,
const TableState& state)
96 : _magic{magic}, _info{info}, _source{source}, _state{state} {
97 std::size_t colidx = 0;
98 for (
const Column& c : info.schema) {
99 if (!c.noninline) _cols.emplace_back(colidx, c);
100 else if (c.isRelation) _relationCol = colidx;
113 if (
auto w = _block.write(_stringRegion); !w)
return std::unexpected{w.error()};
115 return _magic ==
Wdc1Magic ? _emitWdc1() : _emitWdc3();
130 std::vector<std::int64_t> lo(_cols.size(), 0), hi(_cols.size(), 0);
131 for (std::size_t r = 0; r < _source.size(); ++r)
132 for (std::size_t f = 0; f < _cols.size(); ++f)
134 for (std::uint16_t e = 0; e < _cols[f].second.arrayLen; ++e) {
135 const std::int64_t v = _source.getInt(r, _cols[f].first, e);
136 lo[f] = std::min(lo[f], v);
137 hi[f] = std::max(hi[f], v);
140 _plan.resize(_cols.size());
141 for (std::size_t f = 0; f < _cols.size(); ++f) {
142 const Column& col = _cols[f].second;
143 FieldPlan& p = _plan[f];
144 p.elements = col.arrayLen;
145 const auto kind = f < _state.wdcKinds.size()
152 else if (palletable && origPallet)
158 if (p.isPallet()) p.elemBits = 1;
159 else if (p.isCommon()) p.elemBits = 0;
161 p.elemBits = std::min<std::uint16_t>(
162 col.isSigned ? signedWidth(lo[f], hi[f]) : unsignedWidth(
static_cast<std::uint64_t
>(hi[f])), col.bits);
163 else p.elemBits = 32;
168 for (std::size_t r = 0; r < _source.size(); ++r)
169 for (std::size_t f = 0; f < _cols.size(); ++f)
170 if (_plan[f].isPallet()) {
171 std::string key = _palletKey(r, f);
172 if (_plan[f].palletIndex.try_emplace(key,
static_cast<std::uint32_t
>(_plan[f].palletIndex.size())).
174 for (std::uint16_t e = 0; e < _plan[f].elements; ++e) {
175 std::uint32_t slot = 0;
176 std::memcpy(&slot, key.data() + std::size_t{e} * 4, 4);
177 _plan[f].pallet.push_back(slot);
180 for (FieldPlan& p : _plan)
185 const std::uint64_t palletCount = p.palletIndex.size();
186 p.elemBits = unsignedWidth(palletCount);
193 void _layoutRecord() {
194 std::size_t recordBits = 0;
195 for (FieldPlan& p : _plan) {
197 p.offsetBits =
static_cast<std::uint32_t
>(recordBits);
198 recordBits += p.recordBits();
200 _recordSize =
static_cast<std::uint32_t
>((recordBits + 7) / 8);
204 _bitpackedAt = _recordSize;
205 for (
const FieldPlan& p : _plan)
207 _bitpackedAt = p.offsetBits / 8;
217 void _deriveCopies() {
218 const std::uint32_t recordCount =
static_cast<std::uint32_t
>(_source.size());
219 if (_recordSize >= 8) {
220 std::unordered_map<std::string, std::uint32_t> first;
221 _reals.reserve(recordCount);
222 for (std::uint32_t r = 0; r < recordCount; ++r) {
223 std::string key = _valueKey(r);
224 const std::uint32_t
id = _source.idOf(r);
225 if (
const auto it = first.find(key); it != first.end()) _copies.emplace_back(
id, it->second);
227 first.emplace(std::move(key),
id);
233 _reals.resize(recordCount);
234 for (std::uint32_t r = 0; r < recordCount; ++r) _reals[r] = r;
242 void _deriveCommon() {
243 _common.resize(_plan.size());
244 std::vector<std::vector<std::pair<std::uint32_t, std::uint32_t>>> raw(_plan.size());
245 for (
const std::uint32_t ri : _reals) {
246 const std::uint32_t
id = _source.idOf(ri);
247 for (std::size_t f = 0; f < _cols.size(); ++f)
248 if (_plan[f].isCommon()) raw[f].emplace_back(
id, _source.getSlot(ri, _cols[f].first, 0));
250 for (std::size_t f = 0; f < _cols.size(); ++f) {
251 if (!_plan[f].isCommon())
continue;
252 std::unordered_map<std::uint32_t, std::uint32_t> hist;
253 for (
const auto& [
id, v] : raw[f]) ++hist[v];
254 std::uint32_t def = 0, best = 0;
255 for (
const auto& [v, c] : hist)
260 _plan[f].commonDefault = def;
261 for (
const auto& [
id, v] : raw[f])
262 if (v != def) _common[f].emplace_back(
id, v);
263 std::ranges::sort(_common[f]);
265 for (
const auto& entries : _common) _commonTotal +=
static_cast<std::uint32_t
>(entries.size() * 8);
266 for (
const FieldPlan& p : _plan)
267 if (p.isPallet()) _palletTotal +=
static_cast<std::uint32_t
>(p.pallet.size() * 4);
274 void _encodeRecords() {
275 const std::uint32_t realCount =
static_cast<std::uint32_t
>(_reals.size());
276 _recordsAt = _frameBytesBeforeRecords();
277 _stringsAt = _recordsAt + std::size_t{realCount} * _recordSize;
279 std::ignore = _block.add(
"");
281 _lookup.emplace(
"", 0);
283 _recordRegion.assign(std::size_t{realCount} * _recordSize, std::byte{0});
284 _ids.resize(realCount);
285 for (std::uint32_t i = 0; i < realCount; ++i) {
286 const std::size_t base = std::size_t{i} * _recordSize;
287 BitWriter writer(_recordRegion.data() + base, _recordSize);
288 _encodeRecord(_reals[i], writer, _recordsAt + base);
289 _ids[i] = _source.idOf(_reals[i]);
298 void _encodeRecord(std::size_t r, BitWriter& writer, std::size_t recordAbs) {
299 for (std::size_t f = 0; f < _cols.size(); ++f) {
300 const std::size_t colidx = _cols[f].first;
301 const Column& col = _cols[f].second;
302 const FieldPlan& p = _plan[f];
303 if (p.isCommon())
continue;
305 const std::string key = _palletKey(r, f);
306 const auto it = p.palletIndex.find(key);
307 writer.write(p.offsetBits, p.elemBits, it != p.palletIndex.end() ? it->second : 0);
310 for (std::uint16_t e = 0; e < col.arrayLen; ++e) {
311 const std::size_t bitOff = p.offsetBits + std::size_t{e} * p.elemBits;
313 const std::string value{_source.getString(r, colidx, e)};
314 std::uint32_t blockOff = 0;
315 if (
const auto it = _lookup.find(value); it != _lookup.end()) blockOff = it->second;
317 blockOff = _block.add(value);
318 _lookup.emplace(value, blockOff);
322 writer.write(bitOff, 32, blockOff);
327 const std::size_t fieldAbs = recordAbs + bitOff / 8;
328 writer.write(bitOff, 32,
static_cast<std::uint32_t
>(_stringsAt + blockOff - fieldAbs));
331 else if (col.type ==
ColumnType::Float) writer.write(bitOff, 32, _source.getSlot(r, colidx, e));
334 const std::uint64_t mask = p.elemBits >= 64 ? ~std::uint64_t{0} : (std::uint64_t{1} << p.elemBits) - 1;
335 writer.write(bitOff, p.elemBits,
static_cast<std::uint64_t
>(_source.getInt(r, colidx, e)) & mask);
346 std::string _valueKey(std::size_t r)
const {
348 std::size_t colidx = 0;
349 for (
const Column& col : _info.schema) {
352 for (std::uint16_t e = 0; e < col.arrayLen; ++e) {
353 const std::string_view v = _source.getString(r, colidx, e);
354 const std::uint32_t n =
static_cast<std::uint32_t
>(v.size());
355 key.append(
reinterpret_cast<const char*
>(&n), 4);
359 for (std::uint16_t e = 0; e < col.arrayLen; ++e) {
360 const std::uint32_t s = _source.getSlot(r, colidx, e);
361 key.append(
reinterpret_cast<const char*
>(&s), 4);
364 for (std::uint16_t e = 0; e < col.arrayLen; ++e) {
365 const std::int64_t v = _source.getInt(r, colidx, e);
366 key.append(
reinterpret_cast<const char*
>(&v), 8);
379 std::string _palletKey(std::size_t r, std::size_t f)
const {
380 const std::uint16_t elements = _plan[f].elements;
381 std::string key(std::size_t{elements} * 4,
'\0');
382 for (std::uint16_t e = 0; e < elements; ++e) {
383 const std::uint32_t slot = _source.getSlot(r, _cols[f].first, e);
384 std::memcpy(key.data() + std::size_t{e} * 4, &slot, 4);
391 void _scanIdRange() {
392 _minId = _source.size() ? _source.idOf(0) : 0;
394 for (std::size_t r = 0; r < _source.size(); ++r) {
395 const std::uint32_t
id = _source.idOf(r);
396 _minId = std::min(_minId,
id);
397 _maxId = std::max(_maxId,
id);
404 std::size_t _frameBytesBeforeRecords()
const {
405 const std::size_t fieldCount = _plan.size();
409 return sizeof(Wdc1Header) + fieldCount *
sizeof(WdcFieldStructure);
410 const std::size_t prefix = _magic ==
Wdc5Magic ?
sizeof(Wdc5HeaderPrefix) : 0;
411 return prefix +
sizeof(Wdc3Header) +
sizeof(Wdc3SectionHeader) + fieldCount *
sizeof(WdcFieldStructure) +
412 fieldCount *
sizeof(WdcFieldStorage) + _palletTotal + _commonTotal;
416 bool _noninlineId()
const {
422 std::size_t _relationshipBytes()
const {
423 return _relationCol == std::numeric_limits<std::size_t>::max() ? 0 : 12 + _reals.size() * 8;
427 static void _append(
FileBuffer& out,
const void* p, std::size_t n) {
428 const auto* b =
static_cast<const std::byte*
>(p);
429 out.insert(out.end(), b, b + n);
435 void _appendFieldStructure(
FileBuffer& out)
const {
436 for (
const FieldPlan& p : _plan) {
439 :
static_cast<std::int16_t
>(32 - p.elemBits);
440 WdcFieldStructure fstruct{size,
static_cast<std::uint16_t
>(p.offsetBits / 8)};
441 _append(out, &fstruct,
sizeof fstruct);
449 void _appendFieldStorage(
FileBuffer& out)
const {
451 for (
const FieldPlan& p : _plan) {
453 fs.fieldOffsetBits =
static_cast<std::uint16_t
>(p.offsetBits);
454 fs.fieldSizeBits =
static_cast<std::uint16_t
>(p.isPallet()
458 : std::size_t{p.elemBits} * p.elements));
459 fs.storageType = p.storage;
461 fs.val1 = p.offsetBits - _bitpackedAt * 8;
462 fs.val2 = fs.fieldSizeBits;
465 fs.additionalDataSize =
static_cast<std::uint32_t
>(p.pallet.size() * 4);
468 else if (p.isCommon()) {
469 fs.val1 = p.commonDefault;
470 fs.additionalDataSize =
static_cast<std::uint32_t
>(_common[cf].size() * 8);
477 _append(out, &fs,
sizeof fs);
483 void _appendPalletCommon(
FileBuffer& out)
const {
484 for (
const FieldPlan& p : _plan)
486 for (
const std::uint32_t v : p.pallet) _append(out, &v, 4);
487 for (
const auto& fieldEntries : _common)
488 for (
const auto& [
id, value] : fieldEntries) {
489 _append(out, &
id, 4);
490 _append(out, &value, 4);
496 void _appendIdsCopies(
FileBuffer& out)
const {
498 for (
const std::uint32_t
id : _ids) _append(out, &
id, 4);
499 for (
const auto& [newId, srcId] : _copies) {
500 _append(out, &newId, 4);
501 _append(out, &srcId, 4);
510 const std::uint32_t realCount =
static_cast<std::uint32_t
>(_reals.size());
512 header.recordCount = realCount;
513 header.fieldCount =
static_cast<std::uint32_t
>(_plan.size());
514 header.recordSize = _recordSize;
515 header.stringTableSize =
static_cast<std::uint32_t
>(_stringRegion.size());
516 header.tableHash = _state.wdcTableHash;
517 header.layoutHash = _state.wdcLayoutHash;
518 header.minId = _minId;
519 header.maxId = _maxId;
520 header.locale = _state.wdcLocale;
521 header.copyTableSize =
static_cast<std::uint32_t
>(_copies.size() * 8);
524 header.totalFieldCount = header.fieldCount;
525 header.bitpackedDataOffset = _bitpackedAt;
526 header.lookupColumnCount = _relationshipBytes() ? 1 : 0;
527 header.offsetMapOffset = 0;
528 header.idListSize = _noninlineId() ? realCount * 4 : 0;
529 header.fieldStorageInfoSize = header.fieldCount *
static_cast<std::uint32_t
>(
sizeof(WdcFieldStorage));
530 header.commonDataSize = _commonTotal;
531 header.palletDataSize = _palletTotal;
532 header.relationshipDataSize =
static_cast<std::uint32_t
>(_relationshipBytes());
536 _stringsAt + _stringRegion.size() + header.idListSize + header.copyTableSize + header.
537 fieldStorageInfoSize + _palletTotal + _commonTotal + header.relationshipDataSize);
538 _append(out, &header,
sizeof header);
539 _appendFieldStructure(out);
540 out.insert(out.end(), _recordRegion.begin(), _recordRegion.end());
541 out.insert(out.end(), _stringRegion.begin(), _stringRegion.end());
542 _appendIdsCopies(out);
543 _appendFieldStorage(out);
544 _appendPalletCommon(out);
545 _appendRelationship(out);
555 const std::uint32_t realCount =
static_cast<std::uint32_t
>(_reals.size());
556 const std::uint32_t fieldCount =
static_cast<std::uint32_t
>(_plan.size());
557 const bool hasIdList = _noninlineId();
560 header.magic = _magic;
561 header.recordCount = realCount;
562 header.fieldCount = fieldCount;
563 header.recordSize = _recordSize;
564 header.stringTableSize =
static_cast<std::uint32_t
>(_stringRegion.size());
565 header.tableHash = _state.wdcTableHash;
566 header.layoutHash = _state.wdcLayoutHash;
567 header.minId = _minId;
568 header.maxId = _maxId;
569 header.locale = _state.wdcLocale;
572 header.totalFieldCount = fieldCount;
573 header.bitpackedDataOffset = _bitpackedAt;
574 header.lookupColumnCount = _relationshipBytes() ? 1 : 0;
575 header.fieldStorageInfoSize = fieldCount *
static_cast<std::uint32_t
>(
sizeof(WdcFieldStorage));
576 header.palletDataSize = _palletTotal;
577 header.commonDataSize = _commonTotal;
578 header.sectionCount = 1;
580 Wdc3SectionHeader section;
581 section.fileOffset =
static_cast<std::uint32_t
>(_recordsAt);
582 section.recordCount = realCount;
583 section.stringTableSize =
static_cast<std::uint32_t
>(_stringRegion.size());
584 section.idListSize = hasIdList ? realCount * 4 : 0;
585 section.relationshipDataSize =
static_cast<std::uint32_t
>(_relationshipBytes());
586 section.copyTableCount =
static_cast<std::uint32_t
>(_copies.size());
590 _stringsAt + _stringRegion.size() + std::size_t{realCount} * 4
591 + _copies.size() * 8 + _relationshipBytes());
595 Wdc5HeaderPrefix prefix;
596 if (_state.wdc5Prefix.size() ==
sizeof prefix)
597 std::memcpy(&prefix, _state.wdc5Prefix.data(),
sizeof prefix);
598 _append(out, &header.magic,
sizeof header.magic);
599 _append(out, &prefix,
sizeof prefix);
600 _append(out, &header.recordCount,
sizeof header -
sizeof header.magic);
602 else _append(out, &header,
sizeof header);
603 _append(out, §ion,
sizeof section);
604 _appendFieldStructure(out);
605 _appendFieldStorage(out);
606 _appendPalletCommon(out);
607 out.insert(out.end(), _recordRegion.begin(), _recordRegion.end());
608 out.insert(out.end(), _stringRegion.begin(), _stringRegion.end());
609 _appendIdsCopies(out);
610 _appendRelationship(out);
617 void _appendRelationship(
FileBuffer& out)
const {
618 if (_relationCol == std::numeric_limits<std::size_t>::max())
return;
619 const std::uint32_t num =
static_cast<std::uint32_t
>(_reals.size());
620 std::uint32_t rmin = std::numeric_limits<std::uint32_t>::max(), rmax = 0;
621 std::vector<std::pair<std::uint32_t, std::uint32_t>> entries;
622 entries.reserve(_reals.size());
623 for (std::uint32_t i = 0; i < _reals.size(); ++i) {
624 const auto foreign =
static_cast<std::uint32_t
>(_source.getInt(_reals[i], _relationCol, 0));
625 entries.emplace_back(foreign, i);
626 rmin = std::min(rmin, foreign);
627 rmax = std::max(rmax, foreign);
629 if (entries.empty()) rmin = 0;
630 _append(out, &num, 4);
631 _append(out, &rmin, 4);
632 _append(out, &rmax, 4);
633 for (
const auto& [foreign, index] : entries) {
634 _append(out, &foreign, 4);
635 _append(out, &index, 4);
639 std::uint32_t _magic;
640 const TableInfo& _info;
641 const RecordSource& _source;
642 const TableState& _state;
643 std::vector<std::pair<std::size_t, Column>> _cols;
644 std::size_t _relationCol = std::numeric_limits<std::size_t>::max();
645 std::vector<FieldPlan> _plan;
646 std::uint32_t _recordSize = 0;
647 std::uint32_t _bitpackedAt = 0;
648 std::vector<std::uint32_t> _reals;
649 std::vector<std::pair<std::uint32_t, std::uint32_t>> _copies;
650 std::vector<std::vector<std::pair<std::uint32_t, std::uint32_t>>> _common;
651 std::uint32_t _commonTotal = 0;
652 std::uint32_t _palletTotal = 0;
653 formats::StringBlock _block;
654 std::unordered_map<std::string, std::uint32_t> _lookup;
657 std::vector<std::uint32_t> _ids;
658 std::size_t _recordsAt = 0;
659 std::size_t _stringsAt = 0;
660 std::uint32_t _minId = 0, _maxId = 0;
671 std::format(
"{}: a pre-Cata client does not use the WDC formats", info.
name));
677 "{}: a WDC table with encrypted sections can only be written by preserving "
678 "its original image, which is not available here", info.
name));
682 return Writer{magic, info, source, state}.emit();
The encode source: the codecs read the record vector through this.
Named ClientVersion constants for the exact client builds format features appeared (or vanished) at —...
constexpr ClientVersion Cata
Cataclysm (any 4.x client).
std::uint16_t idFieldIndex(std::span< const Column > schema)
The inline field index of an inline id column (0 when non-inline).
bool idIsNoninline(std::span< const Column > schema)
Whether the id column is stored outside the record ($noninline$ id).
WdcCompression
How a column is packed (field_storage_info.storageType).
@ Bitpacked
Bit-addressed unsigned value (signed in WDC1 when the entry's flags word carries bit 0x01).
@ None
Stored inline at a byte offset, fieldSizeBits wide.
@ BitpackedSigned
Bit-addressed signed value, sign-extended (WDC2+ only; WDC1 spells it Bitpacked + flags 0x01).
@ CommonData
Sparse per-id override table; default when an id is absent.
@ PalletArray
Bit-addressed index into palletData (array per slot).
@ Pallet
Bit-addressed index into palletData (one value per slot).
@ WdcFlagNoninlineId
The id lives in the idList, not the record.
Result< FileBuffer > writeWdc(std::uint32_t magic, const TableInfo &info, const RecordSource &source, const TableState &state, EncryptedPolicy policy)
Encode a canonical image of flavor magic: one unencrypted section, integers bitpacked to their minimu...
constexpr std::uint32_t Wdc1Magic
The WDC1 magic as memcpy'd off the file front (the bytes "WDC1").
constexpr std::uint32_t Wdc5Magic
The WDC5 magic as memcpy'd off the file front (the bytes "WDC5").
@ Int
An integer column; Column::bits / Column::isSigned give the shape.
@ Float
A 32-bit IEEE float column.
@ String
A string-block reference column (u32 offset on disk).
EncryptedPolicy
How write() treats a table that still holds keyless (undecryptable) encrypted sections.
@ Preserve
Re-emit the original image verbatim; edits are not applied.
std::expected< T, Error > Result
Every fallible wowlib operation returns Result<T>; bindings translate the error branch into a target-...
std::unexpected< Error > makeError(ErrorCode code, std::string message, std::uint32_t nativeError=0)
Shorthand for constructing the error branch of a Result.
@ InvalidEntityState
An entity's members disagree (e.g.
@ NotSupported
Operation not supported by this backend/provider.
std::vector< std::byte > FileBuffer
Owning byte buffer for file contents read out of a client storage.
Schema reflection over generated client-database record structs: the column list, record stride and f...
StringBlock — the decoded representation of a chunk of zero-terminated strings (MOTX,...
The per-table identity a codec needs beyond the record data: the client version (format selection,...
ClientVersion version
The record's client version.
std::string_view name
The WoWDBDefs table name.
The preserved decode state a table carries between read() and write() so a WDBC/WDB2 write is byte-pe...
std::vector< std::byte > wdcOriginal
Raw image kept when the file has encrypted sections.
std::vector< EncryptedSection > encrypted
Encrypted WDC sections skipped on read.
The WDC-family codec: the non-templated entry points Table<Record> dispatches to for every column-com...