offset_block.hpp

Go to the documentation of this file.

#pragma once

 
#include <meta>
 
#include <cstddef>
#include <cstring>
#include <format>
#include <functional>
#include <optional>
#include <span>
#include <string>
#include <type_traits>
#include <vector>
 
#include <welder/vocabulary.hpp>
 
#include <wowlib/core/buffer.hpp>
#include <wowlib/core/client_version.hpp>
#include <wowlib/core/error.hpp>
#include <wowlib/formats/common/annotations.hpp>
#include <wowlib/formats/common/chunked_file.hpp>
 
namespace wowlib::formats::m2 {
  struct M2OffsetBase {
    // excluded: the parameter type is this unwelded base, not the entity
    [[=welder::mark::exclude]]
    bool operator==(const M2OffsetBase&) const = default;
  };

      function) everything is inline in the entity's own buffer. */
  struct OffsetReadContext {
    std::function<std::span<const std::byte>(std::size_t i)> sequenceBase;
  };

      data is written inline after the entity's own image. */
  struct OffsetWriteContext {
    std::function<FileBuffer * (std::size_t i
    )
    >
    sequenceSink;
  };
 
  // --- member-kind classification --------------------------------------------
  // The offset serializer is TYPE-driven, exactly like the chunk serializer:
  // a member's C++ type decides how it maps onto the layout. These concepts
  // name the three kinds so the dispatch reads as prose rather than as a raw
  // `IsVectorV<M> || is_same_v<M, std::string>` test.

  template <typename M> concept OffsetStringMember = std::is_same_v<M, std::string>;

  template <typename M> concept OffsetArrayMember = formats::detail::IsVectorV<M> || OffsetStringMember<M>;

  template <typename M> concept InlineRecordMember = std::is_class_v<M> && !std::is_trivially_copyable_v<M> && !
    formats::detail::IsVectorV<M> && !OffsetStringMember<M>;

  template <typename M> concept InlineScalarMember = std::is_trivially_copyable_v<M> && !OffsetArrayMember<M>;

  template <typename T, ClientVersion V>
  consteval std::size_t layoutSize() {
    if constexpr (OffsetArrayMember<T>) return 8; // an M2Array{count, offset} reference: two u32s
    else if constexpr (std::is_trivially_copyable_v<T>) return sizeof(T);
    else {
      static_assert(InlineRecordMember<T>);
      std::size_t total = 0;
      static constexpr auto Members = formats::detail::membersOf<T>();
      template for (constexpr auto m : Members) {
        static_assert(!formats::detail::annotation<formats::detail::GatedBySpec, m>().has_value(),
                      "gated_by members are only supported at the entity top level");
        if constexpr (formats::detail::versionActive<V, m>()) {
          using MT = [:std::meta::type_of(m):];
          total += layoutSize<MT, V>();
        }
      }
      return total;
    }
  }

  template <typename Derived>
  struct M2OffsetBlock : M2OffsetBase {
    [[=welder::doc("Deserialize file bytes into this entity, replacing its "
      "contents. Offsets resolve against the given buffer; "
      "sequence-gated data is read inline.")]]
    Result<void>
    read(std::span<const std::byte> data [[=welder::doc("the file (or containing-chunk payload) bytes")]]) {
      std::size_t cursor = 0;
      auto& self = static_cast<Derived&>(*this);
      return _readMembers(self, data, cursor, data, OffsetReadContext{});
    }

        @param data the file (or containing-chunk payload) bytes.
        @param ctx  per-sequence base resolution for `SequenceData` members.
        @return nothing, or the first structural error. */
    [[=welder::mark::exclude]]
    Result<void> read(std::span<const std::byte> data, const OffsetReadContext& ctx) {
      std::size_t cursor = 0;
      auto& self = static_cast<Derived&>(*this);
      return _readMembers(self, data, cursor, data, ctx);
    }
 
    [[nodiscard]]
    [[=welder::doc("Serialize this entity in wowlib's canonical layout (an "
        "offset format has no byte-perfect round-trip guarantee; "
        "a written entity re-reads equal instead)."),
      =welder::returns("the file bytes")]]
    Result<FileBuffer> write() const {
      return write(OffsetWriteContext{});
    }

    /** Serialize with external sequence data routed through @a ctx (the M2
        .anim splitting path).
        @param ctx per-sequence sink resolution for `SequenceData` members.
        @return the file bytes, or the first error. */
    [[=welder::mark::exclude]]
    Result<FileBuffer> write(const OffsetWriteContext& ctx) const {
      FileBuffer out;
      if (auto r = _writeImage(static_cast<const Derived&>(*this), out, ctx); !r) return std::unexpected{r.error()};
      return out;
    }

    /** Append this entity's serialized image to @a out (the chunk serializer's
        SelfSerializing write hook — offsets are relative to the start of the
        appended image).
        @param out the destination buffer (appended, not cleared).
        @return nothing, or the first error. */
    [[=welder::mark::exclude]]
    Result<void> write(FileBuffer& out) const {
      return _writeImage(static_cast<const Derived&>(*this), out, OffsetWriteContext{});
    }


    [[=welder::mark::exclude]]
    bool empty() const {
      return false;
    }
 
    [[nodiscard]]
    [[=welder::doc(R"(
        Check the logical integrity contracts this file must satisfy to LOAD in
        the client — companion-array counts, lookup ranges — which write()
        deliberately never enforces. Call it before writing when you want to
        know the result will load. A file read from a client and left
        unmodified reports no errors; warnings mark states real client files
        ship.)"),
      =welder::returns("every violated contract, in member order")]]
    ValidationReport validate() const {
      ValidationReport report;
      formats::detail::validateEntity(static_cast<const Derived&>(*this), report);
      return report;
    }
 
    [[nodiscard]]
    [[=welder::doc("Validate and raise on the first error instead of returning "
        "a report — the assert-style face of validate()."),
      =welder::returns("nothing; raises when validate() finds any error")]]
    Result<void> ensureValid() const {
      return validate().toResult();
    }

    [[=welder::mark::exclude]]
    std::size_t imageSize() const {
      const auto& self = static_cast<const Derived&>(*this);
      static constexpr auto Members = formats::detail::membersOf<Derived>();
      std::size_t total = 0;
      template for (constexpr auto m : Members) {
        if constexpr (formats::detail::versionActive<Derived::Version, m>())
          if (_memberPresent<m>(self)) {
            using MT = [:std::meta::type_of(m):];
            total += layoutSize<MT, Derived::Version>();
          }
      }
      return total;
    }

    [[=welder::mark::exclude]]
    static consteval std::size_t memberOffset(std::string_view name) {
      // static: `template for` needs a constant address for the range
      static constexpr auto Members = formats::detail::membersOf<Derived>();
      static constexpr auto Order = _memberOrder<Derived>();
      std::size_t off = 0;
      bool found = false;
      template for (constexpr std::size_t idx : Order) {
        constexpr auto m = Members[idx];
        if constexpr (formats::detail::versionActive<Derived::Version, m>()) {
          if (!found && std::meta::identifier_of(m) == name) found = true;
          if (!found) {
            if (formats::detail::annotation<formats::detail::GatedBySpec, m>().has_value()) throw
              "member_offset: a gated_by member precedes the target";
            using MT = [:std::meta::type_of(m):];
            off += layoutSize<MT, Derived::Version>();
          }
        }
      }
      if (!found) throw "member_offset: the entity has no such member";
      return off;
    }
 
    [[=welder::mark::exclude]]
    bool operator==(const M2OffsetBlock&) const = default;
 
  protected:
    struct M2ArrayRef {
      std::uint32_t count = 0;
      std::uint32_t offset = 0;
    };
 
    static_assert(sizeof(M2ArrayRef) == 8 && std::is_trivially_copyable_v<M2ArrayRef>);
 
    // --- layout order ---------------------------------------------------------

    static consteval std::optional<formats::detail::OffsetAfterSpec>
    _offsetAnchorOf(std::meta::info member) {
      auto anns = std::meta::annotations_of_with_type(member, ^^formats::detail::OffsetAfterSpec);
      if (anns.empty()) return std::nullopt;
      return std::meta::extract<formats::detail::OffsetAfterSpec>(anns[0]);
    }

    template <typename T>
    static consteval auto _memberOrder() {
      constexpr auto members = formats::detail::membersOf<T>();
      const auto own = std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked());
      const auto isOwn = [&](std::meta::info m) {
        for (auto o : own)
          if (o == m) return true;
        return false;
      };
      for (std::size_t i = 0; i < members.size(); ++i)
        if (!isOwn(members[i]) && !_offsetAnchorOf(members[i]).has_value())
          throw "an offset entity's trait-base member needs =offset_after(\"own member\")";
      std::vector<std::size_t> order;
      for (auto o : own) {
        for (std::size_t i = 0; i < members.size(); ++i)
          if (members[i] == o) order.push_back(i);
        if (!std::meta::has_identifier(o)) continue;
        for (std::size_t i = 0; i < members.size(); ++i)
          if (!isOwn(members[i]))
            if (auto a = _offsetAnchorOf(members[i]); a && a->view() == std::meta::identifier_of(o)) order.
              push_back(i);
      }
      if (order.size() != members.size()) throw "an offset_after anchor names no own member of the entity";
      return std::define_static_array(order);
    }
 
    // --- per-member presence predicates ---------------------------------------


    template <std::meta::info Mem, typename T>
    static bool _memberPresent(const T& rec) {
      if constexpr (constexpr auto gate = formats::detail::annotation<formats::detail::GatedBySpec, Mem>(); gate.
        has_value()) {
        static_assert(requires { rec.globalFlags; }, "gated_by members need a global_flags member to gate on");
        return (static_cast<std::uint32_t>(rec.globalFlags) & gate->mask) != 0;
      }
      else return true;
    }

    template <bool IsSequenceData, typename T>
    static bool _resolvesExternally(const T& rec) {
      if constexpr (!IsSequenceData) return false;
      else if constexpr (requires { rec.globalSequence; }) return rec.globalSequence == 0xFFFF;
      else return true;
    }
 
    // --- read engine ----------------------------------------------------------

    template <typename T>
    static Result<void> _readMembers(T& dst,
                                      const std::span<const std::byte> image,
                                      std::size_t& cursor,
                                      const std::span<const std::byte> base,
                                      const OffsetReadContext& ctx) {
      static constexpr auto Members = formats::detail::membersOf<T>();
      static constexpr auto Order = _memberOrder<T>();
      std::optional<Error> failed;
      template for (constexpr std::size_t idx : Order) {
        constexpr auto m = Members[idx];
        if constexpr (formats::detail::versionActive<Derived::Version, m>()) {
          constexpr std::string_view name = std::meta::identifier_of(m);
          constexpr bool isSeq = formats::detail::annotation<formats::detail::SequenceDataSpec, m>().has_value();
          if (!failed && _memberPresent<m>(dst))
            if (auto r = _readMember(dst.[:m:], image, cursor, base, ctx, _resolvesExternally<isSeq>(dst), name); !r)
              failed = r.error();
        }
      }
      if (failed) return std::unexpected{*failed};
      return {};
    }

    template <typename M>
    static Result<void> _readMember(M& dst,
                                     const std::span<const std::byte> image,
                                     std::size_t& cursor,
                                     const std::span<const std::byte> base,
                                     const OffsetReadContext& ctx,
                                     const bool external,
                                     const std::string_view what) {
      if constexpr (OffsetArrayMember<M>)
        return _readArrayMember(dst, image, cursor, base, ctx, external, what);
      else if constexpr (InlineRecordMember<M>) return _readMembers(dst, image, cursor, base, ctx);
      else {
        static_assert(InlineScalarMember<M>, "offset members are arrays, records, or scalars");
        return _readScalar(dst, image, cursor, what);
      }
    }


    template <typename M>
    static Result<void> _readScalar(M& dst,
                                     const std::span<const std::byte> image,
                                     std::size_t& cursor,
                                     const std::string_view what) {
      if (image.size() < sizeof(M) || image.size() - sizeof(M) < cursor)
        return _offsetError(ErrorCode::OffsetOutOfBounds, what,
                             std::format("{}-byte value at image offset {:#x} overruns the image", sizeof(M), cursor));
      std::memcpy(&dst, image.data() + cursor, sizeof(M));
      cursor += sizeof(M);
      return {};
    }


    static Result<void> _readArrayRef(M2ArrayRef& ref,
                                        const std::span<const std::byte> image,
                                        std::size_t& cursor,
                                        const std::string_view what) {
      if (cursor > image.size() || image.size() - cursor < sizeof(M2ArrayRef))
        return _offsetError(ErrorCode::OffsetOutOfBounds, what, "M2Array slot overruns the image");
      std::memcpy(&ref, image.data() + cursor, sizeof ref);
      cursor += sizeof ref;
      return {};
    }


    template <typename M>
    static Result<void> _readArrayMember(M& dst,
                                           const std::span<const std::byte> image,
                                           std::size_t& cursor,
                                           const std::span<const std::byte> base,
                                           const OffsetReadContext& ctx,
                                           const bool external,
                                           const std::string_view what) {
      M2ArrayRef ref;
      if (auto r = _readArrayRef(ref, image, cursor, what); !r) return r;
      if constexpr (OffsetStringMember<M>) return _readStringBlock(dst, ref, base, what);
      else return _readVectorBlock(dst, ref, base, ctx, external, what);
    }


    static Result<void> _readStringBlock(std::string& dst,
                                           const M2ArrayRef ref,
                                           const std::span<const std::byte> base,
                                           const std::string_view what) {
      dst.clear();
      if (ref.count == 0) return {};
      if (std::uint64_t{ref.offset} + ref.count > base.size())
        return _offsetError(ErrorCode::OffsetOutOfBounds, what,
                             std::format("string [{} at {:#x}] overruns the {}-byte buffer", ref.count, ref.offset,
                                         base.size()));
      const auto* chars = reinterpret_cast<const char*>(base.data() + ref.offset);
      const auto* nul = static_cast<const char*>(std::memchr(chars, 0, ref.count));
      dst.assign(chars, nul ? static_cast<std::size_t>(nul - chars) : ref.count);
      return {};
    }

    template <typename M>
    static Result<void> _readVectorBlock(M& dst,
                                           const M2ArrayRef ref,
                                           const std::span<const std::byte> base,
                                           const OffsetReadContext& ctx,
                                           const bool external,
                                           const std::string_view what) {
      using U = typename M::value_type;
      dst.clear();
      if (ref.count == 0) return {};
      // An external sequence element whose context base is empty (missing .anim
      // file): leave the member empty rather than fail.
      if (base.empty()) return {};
      constexpr std::size_t elem = layoutSize<U, Derived::Version>();
      if (std::uint64_t{ref.offset} + std::uint64_t{ref.count} * elem > base.size())
        return _offsetError(ErrorCode::OffsetOutOfBounds, what,
                             std::format("array [{} x {} at {:#x}] overruns the {}-byte buffer", ref.count, elem,
                                         ref.offset, base.size()));
      dst.resize(ref.count);
      if constexpr (std::is_trivially_copyable_v<U>) {
        std::memcpy(dst.data(), base.data() + ref.offset, std::size_t{ref.count} * elem);
        return {};
      }
      else return _readArrayElements(dst, ref, base, ctx, external, what);
    }

    template <typename M>
    static Result<void> _readArrayElements(M& dst,
                                             const M2ArrayRef ref,
                                             const std::span<const std::byte> base,
                                             const OffsetReadContext& ctx,
                                             const bool external,
                                             const std::string_view what) {
      using U = typename M::value_type;
      constexpr std::size_t elem = layoutSize<U, Derived::Version>();
      for (std::size_t i = 0; i < dst.size(); ++i) {
        const auto elementImage = base.subspan(ref.offset + i * elem, elem);
        const std::span<const std::byte> elementBase = external && ctx.sequenceBase ? ctx.sequenceBase(i) : base;
        std::size_t elementCursor = 0;
        if (auto r = _readMember(dst[i], elementImage, elementCursor, elementBase, ctx, false, what); !r) return r;
      }
      return {};
    }
 
    // --- write engine ---------------------------------------------------------


    static void _putBytes(FileBuffer& image,
                           const std::size_t imagePos,
                           std::size_t& cursor,
                           const void* bytes,
                           const std::size_t n) {
      std::memcpy(image.data() + imagePos + cursor, bytes, n);
      cursor += n;
    }

    static std::size_t
    _allocBlock(FileBuffer& target, const std::size_t bytes) {
      target.resize((target.size() + 15) / 16 * 16);
      const std::size_t at = target.size();
      target.resize(target.size() + bytes);
      return at;
    }


    static Result<void> _writeImage(const Derived& self, FileBuffer& out, const OffsetWriteContext& ctx) {
      FileBuffer buffer;
      buffer.resize(self.imageSize());
      std::size_t cursor = 0;
      if (auto r = _writeMembers(self, buffer, 0, cursor, buffer, ctx); !r) return r;
      out.insert(out.end(), buffer.begin(), buffer.end());
      return {};
    }

    template <typename T>
    static Result<void> _writeMembers(const T& src,
                                       FileBuffer& image,
                                       const std::size_t imagePos,
                                       std::size_t& cursor,
                                       FileBuffer& blocks,
                                       const OffsetWriteContext& ctx) {
      static constexpr auto Members = formats::detail::membersOf<T>();
      static constexpr auto Order = _memberOrder<T>();
      std::optional<Error> failed;
      template for (constexpr std::size_t idx : Order) {
        constexpr auto m = Members[idx];
        if constexpr (formats::detail::versionActive<Derived::Version, m>()) {
          constexpr std::string_view name = std::meta::identifier_of(m);
          constexpr bool isSeq = formats::detail::annotation<formats::detail::SequenceDataSpec, m>().has_value();
          if (!failed && _memberPresent<m>(src))
            if (auto r = _writeMember(src.[:m:], image, imagePos, cursor, blocks, ctx,
                                       _resolvesExternally<isSeq>(src), name); !r) failed = r.error();
        }
      }
      if (failed) return std::unexpected{*failed};
      return {};
    }

    template <typename M>
    static Result<void> _writeMember(const M& src,
                                      FileBuffer& image,
                                      const std::size_t imagePos,
                                      std::size_t& cursor,
                                      FileBuffer& blocks,
                                      const OffsetWriteContext& ctx,
                                      const bool external,
                                      const std::string_view what) {
      if constexpr (OffsetArrayMember<M>)
        return _writeArrayMember(src, image, imagePos, cursor, blocks, ctx, external, what);
      else if constexpr (InlineRecordMember<M>) return _writeMembers(src, image, imagePos, cursor, blocks, ctx);
      else {
        static_assert(InlineScalarMember<M>, "offset members are arrays, records, or scalars");
        _putBytes(image, imagePos, cursor, &src, sizeof(M));
        return {};
      }
    }


    template <typename M>
    static Result<void> _writeArrayMember(const M& src,
                                            FileBuffer& image,
                                            const std::size_t imagePos,
                                            std::size_t& cursor,
                                            FileBuffer& blocks,
                                            const OffsetWriteContext& ctx,
                                            const bool external,
                                            const std::string_view what) {
      if constexpr (OffsetStringMember<M>) return _writeStringBlock(src, image, imagePos, cursor, blocks);
      else
        return _writeVectorBlock(src, image, imagePos, cursor, blocks, ctx, external, what);
    }


    static Result<void> _writeStringBlock(const std::string& src,
                                            FileBuffer& image,
                                            const std::size_t imagePos,
                                            std::size_t& cursor,
                                            FileBuffer& blocks) {
      const std::size_t at = _allocBlock(blocks, src.size() + 1);
      std::memcpy(blocks.data() + at, src.data(), src.size());
      blocks[at + src.size()] = std::byte{0};
      const M2ArrayRef ref{static_cast<std::uint32_t>(src.size() + 1), static_cast<std::uint32_t>(at)};
      _putBytes(image, imagePos, cursor, &ref, sizeof ref);
      return {};
    }

    template <typename M>
    static Result<void> _writeVectorBlock(const M& src,
                                            FileBuffer& image,
                                            const std::size_t imagePos,
                                            std::size_t& cursor,
                                            FileBuffer& blocks,
                                            const OffsetWriteContext& ctx,
                                            const bool external,
                                            const std::string_view what) {
      using U = typename M::value_type;
      if (src.empty()) {
        const M2ArrayRef empty{0, 0};
        _putBytes(image, imagePos, cursor, &empty, sizeof empty);
        return {};
      }
      constexpr std::size_t elem = layoutSize<U, Derived::Version>();
      const std::size_t at = _allocBlock(blocks, src.size() * elem);
      const M2ArrayRef ref{static_cast<std::uint32_t>(src.size()), static_cast<std::uint32_t>(at)};
      _putBytes(image, imagePos, cursor, &ref, sizeof ref);
      if constexpr (std::is_trivially_copyable_v<U>) {
        std::memcpy(blocks.data() + at, src.data(), src.size() * elem);
        return {};
      }
      else return _writeArrayElements(src, at, blocks, ctx, external, what);
    }

    template <typename M>
    static Result<void> _writeArrayElements(const M& src,
                                              const std::size_t blockAt,
                                              FileBuffer& blocks,
                                              const OffsetWriteContext& ctx,
                                              const bool external,
                                              const std::string_view what) {
      using U = typename M::value_type;
      constexpr std::size_t elem = layoutSize<U, Derived::Version>();
      for (std::size_t i = 0; i < src.size(); ++i) {
        FileBuffer* sink = external && ctx.sequenceSink ? ctx.sequenceSink(i) : nullptr;
        std::size_t elementCursor = 0;
        if (auto r = _writeMember(src[i], blocks, blockAt + i * elem, elementCursor, sink ? *sink : blocks, ctx,
                                   false, what); !r) return r;
      }
      return {};
    }
 
    // --- diagnostics ----------------------------------------------------------


    static std::unexpected<Error> _offsetError(const ErrorCode code,
                                                const std::string_view member,
                                                const std::string_view what) {
      return makeError(code, std::format("offset member '{}': {}", member, what));
    }
  };

  template <typename E> concept OffsetEntity = std::derived_from<E, M2OffsetBase> && requires {
    { E::Version } -> std::convertible_to<ClientVersion>;
  };
}