diff --git a/ZEngine/ZEngine/Helpers/ThreadPool.h b/ZEngine/ZEngine/Helpers/ThreadPool.h index a3c6124b..42a718a8 100644 --- a/ZEngine/ZEngine/Helpers/ThreadPool.h +++ b/ZEngine/ZEngine/Helpers/ThreadPool.h @@ -110,6 +110,16 @@ namespace ZEngine::Helpers /// The callback runs before any queued tasks on each worker thread. /// @param fn Callback — fn(ctx, worker_idx). Must be thread-safe. /// @param ctx Caller context forwarded to fn. + void InitClosureSlab(Core::Memory::ArenaAllocator* arena, size_t bytes) + { + m_closure_slab.Init(arena, bytes); + } + + Core::Memory::TLSFSlab* GetClosureSlab() + { + return m_closure_slab.Pool ? &m_closure_slab : nullptr; + } + void RegisterWorkerInit(WorkerInitFn fn, void* ctx) { m_init_ctx.value.store(ctx, std::memory_order_relaxed); @@ -124,13 +134,9 @@ namespace ZEngine::Helpers m_cancellation.value.store(true, std::memory_order_release); for (size_t i = 0; i < WorkerCount; ++i) m_workers[i].cv.notify_one(); - // Yield until all workers have exited — ensures Worker::mutex and - // Worker::cv are not destroyed while a thread is still using them. - // yield() lets the OS schedule worker threads so they can see the - // cancellation token and decrement the counter; a pure spin-wait - // would starve workers on a loaded CI runner. while (m_active_workers.value.load(std::memory_order_acquire) > 0) std::this_thread::yield(); + m_closure_slab.Shutdown(); } private: @@ -141,6 +147,7 @@ namespace ZEngine::Helpers std::condition_variable cv; }; + Core::Memory::TLSFSlab m_closure_slab{}; Worker m_workers[MAX_WORKERS]; PaddedAtomic m_cursor{}; PaddedAtomic m_cancellation{}; @@ -200,17 +207,30 @@ namespace ZEngine::Helpers Pool->Submit(ctx, fn); } - // Lambda shim — one heap allocation per lambda call (for captures). - // Use the C-style overload directly to stay on the zero-alloc path. template static void Submit(T&& f) { - using Fn = std::decay_t; - auto* p = new Fn(std::forward(f)); - Pool->Submit(p, [](void* ctx) { - auto* fn = static_cast(ctx); + using Fn = std::decay_t; + + static constexpr size_t fn_offset = (sizeof(Core::Memory::TLSFSlab*) + alignof(Fn) - 1) & ~(alignof(Fn) - 1); + static constexpr size_t block_size = fn_offset + sizeof(Fn); + + Core::Memory::TLSFSlab* slab = Pool ? Pool->GetClosureSlab() : nullptr; + uint8_t* block = slab ? static_cast(slab->Alloc(block_size)) : static_cast(::operator new(block_size)); + + *reinterpret_cast(block) = slab; + new (block + fn_offset) Fn(std::forward(f)); + + Pool->Submit(block, [](void* ctx) { + uint8_t* raw = static_cast(ctx); + Core::Memory::TLSFSlab* slab = *reinterpret_cast(raw); + auto* fn = reinterpret_cast(raw + fn_offset); (*fn)(); - delete fn; + fn->~Fn(); + if (slab) + slab->Free(raw); + else + ::operator delete(raw); }); } diff --git a/ZEngine/ZEngine/Importers/AssetCodec.cpp b/ZEngine/ZEngine/Importers/AssetCodec.cpp index 1fca0840..7f186a49 100644 --- a/ZEngine/ZEngine/Importers/AssetCodec.cpp +++ b/ZEngine/ZEngine/Importers/AssetCodec.cpp @@ -337,7 +337,7 @@ namespace ZEngine::Importers::AssetCodec out_cubemap = Rendering::Buffers::Bitmap(header.FaceWidth, header.FaceHeight, header.LayerCount, header.Channel, Rendering::Buffers::BitmapFormat::FLOAT); out_cubemap.Type = Rendering::Buffers::BitmapType::CUBE; - in.read(reinterpret_cast(out_cubemap.Buffer.data()), static_cast(header.BufferByteSize)); + in.read(reinterpret_cast(out_cubemap.Buffer), static_cast(header.BufferByteSize)); return in.good(); } @@ -375,13 +375,13 @@ namespace ZEngine::Importers::AssetCodec .FaceHeight = cubemap.Height, .Channel = cubemap.Channel, .LayerCount = cubemap.Depth, - .BufferByteSize = static_cast(cubemap.Buffer.size()), + .BufferByteSize = static_cast(cubemap.BufferSize), }; const auto* hdr_bytes = reinterpret_cast(&header); auto w1 = file->Write({hdr_bytes, sizeof(header)}, 0); - const auto* data_bytes = reinterpret_cast(cubemap.Buffer.data()); - auto w2 = file->Write({data_bytes, cubemap.Buffer.size()}, sizeof(header)); + const auto* data_bytes = reinterpret_cast(cubemap.Buffer); + auto w2 = file->Write({data_bytes, cubemap.BufferSize}, sizeof(header)); auto flush = file->Flush(); file->Close(); ctx.Close(file); diff --git a/ZEngine/ZEngine/Importers/EnvironmentMapImporter.cpp b/ZEngine/ZEngine/Importers/EnvironmentMapImporter.cpp index c0ad7b61..ad0d7715 100644 --- a/ZEngine/ZEngine/Importers/EnvironmentMapImporter.cpp +++ b/ZEngine/ZEngine/Importers/EnvironmentMapImporter.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -40,11 +41,12 @@ namespace ZEngine::Importers return Core::VFS::VFSResult::Fail(Core::VFS::VFSError::IOError); } - Bitmap equirect = {width, height, 4, BitmapFormat::FLOAT, image_data}; + Core::Memory::TLSFSlab* slab = Helpers::GetWorkerSlab(); + Bitmap equirect(width, height, 4, BitmapFormat::FLOAT, image_data, slab); stbi_image_free(const_cast(image_data)); - Bitmap vertical_cross = Bitmap::EquirectangularMapToVerticalCross(equirect); - Bitmap cubemap = Bitmap::VerticalCrossToCubemap(vertical_cross); + Bitmap vertical_cross = Bitmap::EquirectangularMapToVerticalCross(equirect, slab); + Bitmap cubemap = Bitmap::VerticalCrossToCubemap(vertical_cross, slab); // Write to project://_cache/envmaps/.zenvmap via VFS. // Keyed by UUID — regenerable, gitignored, transparent to game code. diff --git a/ZEngine/ZEngine/Rendering/Buffers/Bitmap.h b/ZEngine/ZEngine/Rendering/Buffers/Bitmap.h index b349e9a8..9b75310d 100644 --- a/ZEngine/ZEngine/Rendering/Buffers/Bitmap.h +++ b/ZEngine/ZEngine/Rendering/Buffers/Bitmap.h @@ -1,10 +1,10 @@ #pragma once #include #include +#include #include #include #include -#include namespace ZEngine::Rendering::Buffers { @@ -29,56 +29,25 @@ namespace ZEngine::Rendering::Buffers * The A and B values are normalized coordinates in the range [-1, 1], calculated from pixel coordinates (i, j) * and the face size. * - * Reference: "Real-Time Rendering, Fourth Edition" by Tomas Akenine-M�ller, Eric Haines, Naty Hoffman + * Reference: "Real-Time Rendering, Fourth Edition" by Tomas Akenine-M?ller, Eric Haines, Naty Hoffman */ static ZEngine::Core::Maths::Vec3f FaceCoordToXYZ(int i, int j, int face_id, int face_size) { const float A = 2.0f * float(i) / face_size; const float B = 2.0f * float(j) / face_size; - /* - * The right face of the cube is mapped to the negative x-axis, so the x-coordinate is set to -1.0f. - * The y and z coordinates are set based on the normalized pixel coordinates. - */ if (face_id == 0) return ZEngine::Core::Maths::Vec3f(-1.0f, A - 1.0f, B - 1.0f); - - /* - * The left face is mapped to the positive x-axis, so the x-coordinate is set to A - 1.0f. - * The y-coordinate is set to -1.0f, and the z-coordinate is set based on the normalized pixel coordinates. - */ if (face_id == 1) return ZEngine::Core::Maths::Vec3f(A - 1.0f, -1.0f, 1.0f - B); - - /* - * The top face is mapped to the positive y-axis, so the y-coordinate is set to A - 1.0f. - * The x-coordinate is set to 1.0f, and the z-coordinate is set based on the normalized pixel coordinates. - */ if (face_id == 2) return ZEngine::Core::Maths::Vec3f(1.0f, A - 1.0f, 1.0f - B); - - /* - * The bottom face is mapped to the negative y-axis, so the y-coordinate is set to 1.0f. - * The x-coordinate is set to 1.0f - A, and the z-coordinate is set based on the normalized pixel - * coordinates - */ if (face_id == 3) return ZEngine::Core::Maths::Vec3f(1.0f - A, 1.0f, 1.0f - B); - - /* - * The front face is mapped to the positive z-axis, so the z-coordinate is set to 1.0f. - *The x and y coordinates are set based on the normalized pixel coordinates. - */ if (face_id == 4) return ZEngine::Core::Maths::Vec3f(B - 1.0f, A - 1.0f, 1.0f); - - /* - * The back face is mapped to the negative z-axis, so the z-coordinate is set to -1.0f. - * The x and y coordinates are set based on the normalized pixel coordinates. - */ if (face_id == 5) return ZEngine::Core::Maths::Vec3f(1.0f - B, A - 1.0f, -1.0f); - return ZEngine::Core::Maths::Vec3f{}; } }; @@ -86,16 +55,72 @@ namespace ZEngine::Rendering::Buffers struct Bitmap { Bitmap() = default; - Bitmap(int width, int height, int channel, BitmapFormat format) : Width(width), Height(height), Channel(channel), Format(format), Buffer(width * height * channel * BytePerChannel(format)) {} - Bitmap(int width, int height, int depth, int channel, BitmapFormat format) : Width(width), Height(height), Depth(depth), Channel(channel), Format(format), Buffer(width * height * depth * channel * BytePerChannel(format)) {} - Bitmap(int width, int height, int channel, BitmapFormat format, const void* data) : Width(width), Height(height), Channel(channel), Format(format), Buffer(width * height * channel * BytePerChannel(format)) + + /// @brief Allocate a zeroed buffer. When slab is non-null the buffer is slab-backed + /// and freed via slab on destruction; otherwise heap-allocated (new[]). + Bitmap(int width, int height, int channel, BitmapFormat format, Core::Memory::TLSFSlab* slab = nullptr) : Width(width), Height(height), Channel(channel), Format(format), Slab(slab) + { + Alloc(static_cast(width) * height * channel * BytePerChannel(format)); + } + + /// @brief Cubemap / depth variant. + Bitmap(int width, int height, int depth, int channel, BitmapFormat format, Core::Memory::TLSFSlab* slab = nullptr) : Width(width), Height(height), Depth(depth), Channel(channel), Format(format), Slab(slab) + { + Alloc(static_cast(width) * height * depth * channel * BytePerChannel(format)); + } + + /// @brief Allocate and copy from data. slab parameter routes the buffer allocation. + Bitmap(int width, int height, int channel, BitmapFormat format, const void* data, Core::Memory::TLSFSlab* slab = nullptr) : Width(width), Height(height), Channel(channel), Format(format), Slab(slab) { + size_t sz = static_cast(width) * height * channel * BytePerChannel(format); if (data) { - ZENGINE_VALIDATE_ASSERT(Helpers::secure_memcpy(Buffer.data(), Buffer.size(), data, Buffer.size()) == Helpers::MEMORY_OP_SUCCESS, "Failed to perform memory copy operation") + AllocNoZero(sz); + if (Buffer) + ZENGINE_VALIDATE_ASSERT(Helpers::secure_memcpy(Buffer, BufferSize, data, BufferSize) == Helpers::MEMORY_OP_SUCCESS, "Bitmap: memcpy from source data failed") + } + else + { + Alloc(sz); } } - ~Bitmap() = default; + + ~Bitmap() + { + Free(); + } + + Bitmap(const Bitmap&) = delete; + Bitmap& operator=(const Bitmap&) = delete; + + Bitmap(Bitmap&& o) noexcept : Width(o.Width), Height(o.Height), Depth(o.Depth), Channel(o.Channel), Type(o.Type), Format(o.Format), Buffer(o.Buffer), BufferSize(o.BufferSize), Slab(o.Slab) + { + o.Buffer = nullptr; + o.BufferSize = 0; + o.Slab = nullptr; + } + + Bitmap& operator=(Bitmap&& o) noexcept + { + if (this != &o) + { + Free(); + Width = o.Width; + Height = o.Height; + Depth = o.Depth; + Channel = o.Channel; + Type = o.Type; + Format = o.Format; + Buffer = o.Buffer; + BufferSize = o.BufferSize; + Slab = o.Slab; + + o.Buffer = nullptr; + o.BufferSize = 0; + o.Slab = nullptr; + } + return *this; + } void SetPixel(int x, int y, const ZEngine::Core::Maths::Vec4f& pixel) { @@ -114,7 +139,7 @@ namespace ZEngine::Rendering::Buffers else if (Format == BitmapFormat::FLOAT) { const int ofs = Channel * (y * Width + x); - float* data = reinterpret_cast(Buffer.data()); + float* data = reinterpret_cast(Buffer); if (Channel > 0) data[ofs + 0] = pixel.x; if (Channel > 1) @@ -136,40 +161,32 @@ namespace ZEngine::Rendering::Buffers else if (Format == BitmapFormat::FLOAT) { const int ofs = Channel * (y * Width + x); - const float* data = reinterpret_cast(Buffer.data()); + const float* data = reinterpret_cast(Buffer); return ZEngine::Core::Maths::Vec4f(Channel > 0 ? data[ofs + 0] : 0.0f, Channel > 1 ? data[ofs + 1] : 0.0f, Channel > 2 ? data[ofs + 2] : 0.0f, Channel > 3 ? data[ofs + 3] : 0.0f); } - return ZEngine::Core::Maths::Vec4f(); } inline static int BytePerChannel(BitmapFormat format) { if (format == BitmapFormat::UNSIGNED_BYTE) - { return 1; - } - else if (format == BitmapFormat::FLOAT) - { + if (format == BitmapFormat::FLOAT) return 4; - } - return 0; } - inline static Bitmap EquirectangularMapToVerticalCross(const Bitmap& input_map) + /// @brief slab is forwarded to all internal Bitmap allocations within this call. + inline static Bitmap EquirectangularMapToVerticalCross(const Bitmap& input_map, Core::Memory::TLSFSlab* slab = nullptr) { if (input_map.Type != BitmapType::TEXTURE_2D) - { return Bitmap(); - } - - const int face_size = input_map.Width / 4; - const int width = face_size * 3; - const int height = face_size * 4; + const int face_size = input_map.Width / 4; + const int width = face_size * 3; + const int height = face_size * 4; - Bitmap vertical_cross = Bitmap(width, height, input_map.Channel, input_map.Format); + Bitmap vertical_cross(width, height, input_map.Channel, input_map.Format, slab); const ZEngine::Core::Maths::IVec2 face_offsets[] = { ZEngine::Core::Maths::IVec2{ face_size, face_size * 3}, @@ -218,16 +235,17 @@ namespace ZEngine::Rendering::Buffers return vertical_cross; } - inline static Bitmap VerticalCrossToCubemap(const Bitmap& input_map) + /// @brief slab is forwarded to the cubemap buffer allocation. + inline static Bitmap VerticalCrossToCubemap(const Bitmap& input_map, Core::Memory::TLSFSlab* slab = nullptr) { - const int face_width = input_map.Width / 3; - const int face_height = input_map.Height / 4; + const int face_width = input_map.Width / 3; + const int face_height = input_map.Height / 4; - Bitmap cubemap = Bitmap(face_width, face_height, 6, input_map.Channel, input_map.Format); + Bitmap cubemap(face_width, face_height, 6, input_map.Channel, input_map.Format, slab); cubemap.Type = CUBE; - const uint8_t* source = input_map.Buffer.data(); - uint8_t* destination = cubemap.Buffer.data(); + const uint8_t* source = input_map.Buffer; + uint8_t* destination = cubemap.Buffer; int pixel_size = cubemap.Channel * BytePerChannel(cubemap.Format); const int RIGHT_FACE = 0; @@ -249,57 +267,85 @@ namespace ZEngine::Rendering::Buffers switch (face) { case RIGHT_FACE: - { pixel_pos_x = i; pixel_pos_y = face_height + j; break; - } case LEFT_FACE: - { pixel_pos_x = 2 * face_width + i; pixel_pos_y = 1 * face_height + j; break; - } case UP_FACE: - { pixel_pos_x = 2 * face_width - (i + 1); pixel_pos_y = 1 * face_height - (j + 1); break; - } case DOWN_FACE: - { pixel_pos_x = 2 * face_width - (i + 1); pixel_pos_y = 3 * face_height - (j + 1); break; - } case FRONT_FACE: - { pixel_pos_x = 2 * face_width - (i + 1); pixel_pos_y = input_map.Height - (j + 1); break; - } case BACK_FACE: - { pixel_pos_x = face_width + i; pixel_pos_y = face_height + j; break; - } } - ZENGINE_VALIDATE_ASSERT(Helpers::secure_memcpy(destination, pixel_size, source + (pixel_pos_y * input_map.Width + pixel_pos_x) * pixel_size, pixel_size) == Helpers::MEMORY_OP_SUCCESS, "Failed to perform memory copy operation") + ZENGINE_VALIDATE_ASSERT(Helpers::secure_memcpy(destination, pixel_size, source + (pixel_pos_y * input_map.Width + pixel_pos_x) * pixel_size, pixel_size) == Helpers::MEMORY_OP_SUCCESS, "Bitmap: pixel copy failed in VerticalCrossToCubemap") destination += pixel_size; } } } - return cubemap; } - int Width = 0; - int Height = 0; - int Depth = 1; - int Channel = 3; - BitmapType Type = BitmapType::TEXTURE_2D; - BitmapFormat Format = BitmapFormat::UNSIGNED_BYTE; - std::vector Buffer = {}; + int Width = 0; + int Height = 0; + int Depth = 1; + int Channel = 3; + BitmapType Type = BitmapType::TEXTURE_2D; + BitmapFormat Format = BitmapFormat::UNSIGNED_BYTE; + uint8_t* Buffer = nullptr; ///< Pixel data. Owned by this Bitmap. + size_t BufferSize = 0; ///< Size of Buffer in bytes. + Core::Memory::TLSFSlab* Slab = nullptr; ///< Non-null when Buffer is slab-backed. + + private: + void Alloc(size_t n) + { + if (n == 0) + return; + BufferSize = n; + if (Slab) + { + Buffer = static_cast(Slab->Alloc(n)); + Helpers::secure_memset(Buffer, 0, n, n); + } + else + { + Buffer = new uint8_t[n](); + } + } + + void AllocNoZero(size_t n) + { + if (n == 0) + return; + BufferSize = n; + Buffer = Slab ? static_cast(Slab->Alloc(n)) : new uint8_t[n]; + } + + void Free() + { + if (Buffer) + { + if (Slab) + Slab->Free(Buffer); + else + delete[] Buffer; + Buffer = nullptr; + BufferSize = 0; + Slab = nullptr; + } + } }; -} // namespace ZEngine::Rendering::Buffers \ No newline at end of file +} // namespace ZEngine::Rendering::Buffers diff --git a/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp b/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp index d94a56bf..bc93ec12 100644 --- a/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp +++ b/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp @@ -58,6 +58,7 @@ namespace ZEngine::Rendering InitGlobalBuffers(); InitTextureTimelines(); InitUploadSlabs(static_cast(Helpers::ThreadPoolHelper::Pool->WorkerCount)); + Helpers::ThreadPoolHelper::Pool->InitClosureSlab(m_device->Arena, ZKilo(512)); registry->SetOnReadyCallback(this, [](void* ctx, const uuids::uuid& uuid, AssetHandle handle) { auto* rrm = static_cast(ctx); @@ -1348,9 +1349,9 @@ namespace ZEngine::Rendering ZENGINE_CORE_ERROR("Failed to deserialize .zenvmap: {}", captured_filename) return; } - size_t bytes = cubemap.Buffer.size(); + size_t bytes = cubemap.BufferSize; buffer.resize(bytes); - Helpers::secure_memmove(buffer.data(), bytes, cubemap.Buffer.data(), bytes); + Helpers::secure_memmove(buffer.data(), bytes, cubemap.Buffer, bytes); } else { @@ -1361,28 +1362,38 @@ namespace ZEngine::Rendering ZENGINE_CORE_ERROR("Failed to load texture: {}", captured_filename) return; } - std::vector output_buf; + Core::Memory::TLSFSlab* slab = Helpers::GetWorkerSlab(); + size_t float_buf_bytes = 0; + float* output_buf = nullptr; if (ch == STBI_rgb) { - size_t total = w * h; - output_buf.resize(total * 4); - stbir_resize_float(image_data, w, h, 0, output_buf.data(), w, h, 0, 4); + size_t total = (size_t) (w * h); + float_buf_bytes = total * 4 * sizeof(float); + output_buf = slab ? static_cast(slab->Alloc(float_buf_bytes)) : new float[total * 4]; + stbir_resize_float(image_data, w, h, 0, output_buf, w, h, 0, 4); for (size_t i = 0; i < total; ++i) output_buf[i * 4 + 3] = 255.f; } else { - output_buf.resize((size_t) (w * h * ch)); + float_buf_bytes = (size_t) (w * h * ch) * sizeof(float); + output_buf = slab ? static_cast(slab->Alloc(float_buf_bytes)) : new float[w * h * ch]; + Helpers::secure_memcpy(output_buf, float_buf_bytes, image_data, float_buf_bytes); } stbi_image_free((void*) image_data); - Rendering::Buffers::Bitmap in = {w, h, 4, Rendering::Buffers::BitmapFormat::FLOAT, output_buf.data()}; - Rendering::Buffers::Bitmap vertical_cross = Rendering::Buffers::Bitmap::EquirectangularMapToVerticalCross(in); - Rendering::Buffers::Bitmap cubemap = Rendering::Buffers::Bitmap::VerticalCrossToCubemap(vertical_cross); + Rendering::Buffers::Bitmap in(w, h, 4, Rendering::Buffers::BitmapFormat::FLOAT, output_buf, slab); + if (slab) + slab->Free(output_buf); + else + delete[] output_buf; + + Rendering::Buffers::Bitmap vertical_cross = Rendering::Buffers::Bitmap::EquirectangularMapToVerticalCross(in, slab); + Rendering::Buffers::Bitmap cubemap = Rendering::Buffers::Bitmap::VerticalCrossToCubemap(vertical_cross, slab); - size_t bytes = cubemap.Buffer.size(); + size_t bytes = cubemap.BufferSize; buffer.resize(bytes); - Helpers::secure_memmove(buffer.data(), bytes, cubemap.Buffer.data(), bytes); + Helpers::secure_memmove(buffer.data(), bytes, cubemap.Buffer, bytes); } } else