read.cpp

Go to the documentation of this file.

 
#include <wowlib/db/wdc/wdc.hpp>
 
#include <algorithm>
#include <cstring>
#include <format>
#include <limits>
#include <string>
#include <vector>
 
#include <wowlib/core/client_builds.hpp>
#include <wowlib/db/codec_detail.hpp>
#include <wowlib/db/schema.hpp>
 
namespace wowlib::db::wdc {
  namespace {
    std::string readCString(std::span<const std::byte> file, std::size_t at, std::size_t limit) {
      limit = std::min(limit, file.size());
      if (at >= limit) return {};
      const auto* bytes = reinterpret_cast<const char*>(file.data());
      std::size_t end = at;
      while (end < limit && bytes[end] != '\0') ++end;
      return std::string{bytes + at, end - at};
    }

    std::uint64_t signedFit(std::uint64_t raw, std::size_t bits, bool isSigned) {
      if (!isSigned || bits == 0 || bits >= 64) return raw;
      const std::uint64_t signBit = std::uint64_t{1} << (bits - 1);
      if (raw & signBit) return raw | ~((std::uint64_t{1} << bits) - 1);
      return raw;
    }

    class Decoder {
    public:
      Decoder(const WdcImage& img, const TableInfo& info, RecordSink& sink)
        : _img{img}, _info{info}, _sink{sink}, _additional{img.fieldAdditionalOffsets()} {
        // The single non-inline $relation$ column, when the table has one —
        // its values live in the relationship block, not the record image.
        std::size_t colidx = 0;
        for (const Column& col : info.schema) {
          if (col.isRelation && col.noninline) _relationCol = colidx;
          ++colidx;
        }
        // Cross-section string geometry. WDC2+ string references are relative
        // to the referencing field's position inside the CLIENT'S loading
        // blob — all sections' records back to back, then all sections'
        // string blocks — not inside the file. Precompute each section's
        // record/string start inside that blob so references can be mapped
        // back to file offsets (single-section files degenerate to "file
        // offset + reference", which is why the difference only shows on
        // multi-section tables).
        _recordsBefore.reserve(img.sections.size());
        std::uint64_t recSum = 0;
        for (const WdcSection& sec : img.sections) {
          _recordsBefore.push_back(recSum);
          recSum += sec.records.size();
        }
        _totalRecordBytes = recSum;
      }

      Result<void> run(TableState& state) {
        for (const WdcSection& sec : _img.sections) {
          // A key-flagged section whose records are all zero is genuinely
          // undecryptable (the storage lacked the TACT key and shipped
          // zeros); a key-flagged section with real bytes was decrypted by
          // the storage and decodes normally.
          if (sec.encrypted && db::detail::allZero(sec.records)) {
            reportEncrypted(sec, state);
            continue;
          }
          if (auto r = _decodeSection(sec); !r) return r;
        }
        return {};
      }
 
    private:
      void reportEncrypted(const WdcSection& sec, TableState& state) const {
        EncryptedSection report{.keyHash = sec.header.tactKeyHash, .recordCount = sec.header.recordCount};
        if (!sec.encryptedIds.empty()) report.ids = sec.encryptedIds;
        else {
          const auto ids = std::span{
            reinterpret_cast<const std::uint32_t*>(sec.idList.data()),
            sec.idList.size() / 4
          };
          report.ids.assign(ids.begin(), ids.end());
        }
        state.encrypted.push_back(std::move(report));
      }

      Result<void> _decodeSection(const WdcSection& sec) {
        const std::size_t first = _sink.size();
        if (_img.isSparse()) {
          if (auto r = _decodeSparseSection(sec); !r) return r;
        }
        else {
          const std::size_t stride = _img.header.recordSize;
          const std::size_t sectionIndex = _indexOf(sec);
          for (std::uint32_t r = 0; r < sec.header.recordCount; ++r) {
            const std::uint32_t id = _recordId(sec, r);
            const auto recordBytes = sec.records.subspan(std::size_t{r} * stride, stride);
            const std::uint64_t recordBlob = _recordsBefore[sectionIndex] + std::uint64_t{r} * stride;
            const std::size_t rec = _sink.add();
            _decodeRecord(rec, recordBytes, recordBlob, id);
          }
        }
        applyRelationship(sec, first);
        expandCopies(sec);
        return {};
      }

      Result<void> _decodeSparseSection(const WdcSection& sec) {
        for (std::uint32_t r = 0; r < sec.header.offsetMapIdCount; ++r) {
          std::uint32_t offset = 0, id = 0;
          std::uint16_t size = 0;
          std::memcpy(&offset, sec.offsetMap.data() + std::size_t{r} * 6, 4);
          std::memcpy(&size, sec.offsetMap.data() + std::size_t{r} * 6 + 4, 2);
          if (std::size_t{r} * 4 + 4 <= sec.offsetMapIds.size()) std::memcpy(
            &id, sec.offsetMapIds.data() + std::size_t{r} * 4, 4);
          if (offset + size > _img.file.size())
            return makeError(ErrorCode::TableTruncated,
                              std::format("{}: sparse record {} at {:#x} overruns the file", _info.name, r, offset));
          const std::size_t rec = _sink.add();
          decodeSparseRecord(rec, _img.file.subspan(offset, size), id);
        }
        return {};
      }

      std::uint32_t _recordId(const WdcSection& sec, std::uint32_t r) const {
        if (_img.idIsNoninline()) {
          if (std::size_t{r} * 4 + 4 <= sec.idList.size()) {
            std::uint32_t id = 0;
            std::memcpy(&id, sec.idList.data() + std::size_t{r} * 4, 4);
            return id;
          }
          return 0;
        }
        const auto recordBytes = sec.records.
                                      subspan(std::size_t{r} * _img.header.recordSize, _img.header.recordSize);
        return static_cast<std::uint32_t>(_img.fieldRaw(_img.header.idIndex, 0, 1, recordBytes, 0, _additional));
      }

      void _decodeRecord(std::size_t rec,
                         std::span<const std::byte> recordBytes,
                         std::uint64_t recordBlob,
                         std::uint32_t id) {
        std::size_t f = 0, colidx = 0;
        for (const Column& col : _info.schema) {
          if (col.noninline) {
            if (col.isId) _sink.setInt(rec, colidx, 0, static_cast<std::int64_t>(id));
          }
          else {
            for (std::uint16_t e = 0; e < col.arrayLen; ++e)
              decodeInlineElement(rec, f, col, e, colidx, recordBytes, recordBlob, id);
            ++f;
          }
          ++colidx;
        }
      }

      void decodeInlineElement(std::size_t rec,
                                 std::size_t f,
                                 const Column& col,
                                 std::uint16_t e,
                                 std::size_t colidx,
                                 std::span<const std::byte> recordBytes,
                                 std::uint64_t recordBlob,
                                 std::uint32_t id) {
        const std::uint64_t raw = _img.fieldRaw(f, e, col.arrayLen, recordBytes, id, _additional);
        switch (col.type) {
        case ColumnType::Int: {
          const std::uint64_t v = signedFit(raw, _img.elemBitWidth(f, col.arrayLen),
                                             _img.fieldIsSigned(f) || col.isSigned);
          _sink.setInt(rec, colidx, e, static_cast<std::int64_t>(v));
          break;
        }
        case ColumnType::Float:
          _sink.setFloat(rec, colidx, e, std::bit_cast<float>(static_cast<std::uint32_t>(raw)));
          break;
        case ColumnType::String: {
          // A string ARRAY stores one reference per element, each relative
          // to its OWN 4-byte slot, so the element base advances by e*4.
          const std::size_t elemByte = std::size_t{_img.fieldStorage[f].fieldOffsetBits} / 8 + std::size_t{e} * 4;
          _sink.setString(rec, colidx, e, _resolveString(recordBlob + elemByte, static_cast<std::uint32_t>(raw)));
          break;
        }
        case ColumnType::LocString:
          break;
          // WDC (Cata+) has no localized-column layout; schemas never carry one.
        }
      }

      std::string _resolveString(std::uint64_t refBlob, std::uint32_t raw) const {
        if (_img.stringMode == StringRefMode::BlockRelative) {
          const WdcSection& sec = _img.sections.front();
          return readCString(_img.file, std::size_t{sec.stringBase} + raw,
                               std::size_t{sec.stringBase} + sec.strings.size());
        }
        // Sign-extend: a reference from a late section's field to an early
        // section's string is a forward blob distance too (strings all sit
        // after records), but keep the signed interpretation for safety.
        const std::int64_t target = static_cast<std::int64_t>(refBlob) + static_cast<std::int32_t>(raw);
        if (target < static_cast<std::int64_t>(_totalRecordBytes)) return {};
        std::uint64_t t = static_cast<std::uint64_t>(target) - _totalRecordBytes;
        for (std::size_t s = 0; s < _img.sections.size(); ++s) {
          const WdcSection& sec = _img.sections[s];
          if (t < sec.strings.size())
            return readCString(_img.file, std::size_t{sec.stringBase} + t,
                                 std::size_t{sec.stringBase} + sec.strings.size());
          t -= sec.strings.size();
        }
        return {};
      }

      void decodeSparseRecord(std::size_t rec, std::span<const std::byte> bytes, std::uint32_t id) {
        std::size_t cursor = 0, colidx = 0;
        for (const Column& col : _info.schema) {
          if (col.noninline) {
            if (col.isId) _sink.setInt(rec, colidx, 0, static_cast<std::int64_t>(id));
            ++colidx;
            continue;
          }
          for (std::uint16_t e = 0; e < col.arrayLen; ++e) {
            switch (col.type) {
            case ColumnType::Int:
              _sink.setInt(rec, colidx, e, db::detail::readInt(bytes, cursor, col.bits / 8u, col.isSigned));
              break;
            case ColumnType::Float: {
              std::uint32_t v = 0;
              if (cursor + 4 <= bytes.size()) std::memcpy(&v, bytes.data() + cursor, 4);
              cursor += 4;
              _sink.setFloat(rec, colidx, e, std::bit_cast<float>(v));
              break;
            }
            case ColumnType::String: {
              std::string s = readCString(bytes, cursor, bytes.size());
              cursor += s.size() + 1;
              _sink.setString(rec, colidx, e, s);
              break;
            }
            case ColumnType::LocString:
              break;
            }
          }
          ++colidx;
        }
      }

      void applyRelationship(const WdcSection& sec, std::size_t first) {
        if (_relationCol == std::numeric_limits<std::size_t>::max() || sec.relationship.size() < 12) return;
        std::uint32_t num = 0;
        std::memcpy(&num, sec.relationship.data(), 4);
        const bool byId = _img.magic != Wdc1Magic && _img.magic != Wdc3Magic && (_img.header.flags &
          WdcFlagSecondary) != 0;
        for (std::uint32_t e = 0; e < num; ++e) {
          const std::size_t at = 12 + std::size_t{e} * 8;
          if (at + 8 > sec.relationship.size()) break;
          std::uint32_t foreign = 0, key = 0;
          std::memcpy(&foreign, sec.relationship.data() + at, 4);
          std::memcpy(&key, sec.relationship.data() + at + 4, 4);
          const std::size_t rec = byId ? _sink.findById(key) : first + key;
          if (rec < _sink.size())
            _sink.setInt(rec, _relationCol, 0, static_cast<std::int64_t>(foreign));
        }
      }

      void expandCopies(const WdcSection& sec) {
        for (std::size_t c = 0; c + 8 <= sec.copyTable.size(); c += 8) {
          std::uint32_t newId = 0, srcId = 0;
          std::memcpy(&newId, sec.copyTable.data() + c, 4);
          std::memcpy(&srcId, sec.copyTable.data() + c + 4, 4);
          if (const std::size_t src = _sink.findById(srcId); src != _sink.size()) _sink.cloneWithId(src, newId);
        }
      }

      std::size_t _indexOf(const WdcSection& sec) const {
        return static_cast<std::size_t>(&sec - _img.sections.data());
      }
 
      const WdcImage& _img;
      const TableInfo& _info;
      RecordSink& _sink;
      std::vector<std::uint32_t> _additional; 
      std::size_t _relationCol = std::numeric_limits<std::size_t>::max();
      std::vector<std::uint64_t> _recordsBefore; 
      std::uint64_t _totalRecordBytes = 0; 
    };
  }
 
  Result<void> readWdc(const TableInfo& info, std::span<const std::byte> data, RecordSink& sink, TableState& state) {
    if (info.version < builds::Cata)
      return makeError(ErrorCode::TableMagicUnknown, std::format("{}: a {}.{} client does not use the WDC formats",
                                                                  info.name, info.version.major, info.version.minor));
 
    auto parsed = WdcImage::parse(data);
    if (!parsed) return std::unexpected{parsed.error()};
    const WdcImage& img = *parsed;
 
    const std::size_t inlineColumns = db::detail::inlineColumnCount(info.schema);
    if (img.header.fieldCount != inlineColumns)
      return makeError(ErrorCode::SchemaMismatch,
                        std::format(
                          "{}: the file stores {} inline fields but the generated schema has {} "
                          "(layout_hash {:#010x})", info.name, img.header.fieldCount, inlineColumns,
                          img.header.layoutHash));
 
    sink.clear();
    state.reset();
    state.sourceMagic = img.magic;
    state.wdcTableHash = img.header.tableHash;
    state.wdcLayoutHash = img.header.layoutHash;
    state.wdcLocale = img.header.locale;
    state.wdcKinds.assign(inlineColumns, static_cast<std::uint8_t>(WdcCompression::None));
    for (std::size_t f = 0; f < inlineColumns && f < img.fieldStorage.size(); ++f)
      state.wdcKinds[f] = static_cast<std::uint8_t>(img.fieldStorage[f].storageType);
    if (img.magic == Wdc5Magic) {
      const auto* pb = reinterpret_cast<const std::byte*>(&img.wdc5);
      state.wdc5Prefix.assign(pb, pb + sizeof img.wdc5);
    }
 
    if (auto r = Decoder{img, info, sink}.run(state); !r) return r;
 
    // While any section stays undecryptable the original bytes are the only
    // faithful serialization — keep them for the verbatim re-emit.
    if (!state.encrypted.empty()) state.wdcOriginal.assign(data.begin(), data.end());
    return {};
  }
}