Loading...
Searching...
No Matches
vulkan_memory.h
Go to the documentation of this file.
#ifndef HALIDE_RUNTIME_VULKAN_MEMORY_H
#define HALIDE_RUNTIME_VULKAN_MEMORY_H
#include "internal/block_allocator.h"
#include "internal/memory_resources.h"
#include "vulkan_internal.h"
// Uncomment to enable verbose memory allocation debugging
// #define HL_VK_DEBUG_MEM 1
namespace Halide {
namespace Runtime {
namespace Internal {
namespace Vulkan {
// --------------------------------------------------------------------------
// Enable external client to override Vulkan allocation callbacks (if they so desire)
WEAK ScopedSpinLock::AtomicFlag custom_allocation_callbacks_lock = 0;
WEAK const VkAllocationCallbacks *custom_allocation_callbacks = nullptr; // nullptr => use Vulkan runtime implementation
// --------------------------------------------------------------------------
// Runtime configuration parameters to adjust the behaviour of the block allocator
struct VulkanMemoryConfig {
size_t maximum_pool_size = 0; //< Maximum number of bytes to allocate for the entire pool (including all blocks). Specified in bytes. Zero means no constraint
size_t minimum_block_size = 4 * 1024; //< Default block size is 4KB
size_t maximum_block_size = 0; //< Specified in bytes. Zero means no constraint
size_t maximum_block_count = 0; //< Maximum number of blocks to allocate. Zero means no constraint
size_t nearest_multiple = 32; //< Always round up the requested region sizes to the given integer value. Zero means no constraint
};
WEAK VulkanMemoryConfig memory_allocator_config;
// --------------------------------------------------------------------------
/** Vulkan Memory Allocator class interface for managing large
* memory requests stored as contiguous blocks of memory, which
* are then sub-allocated into smaller regions of
* memory to avoid the excessive cost of vkAllocate and the limited
* number of available allocation calls through the API.
*/
class VulkanMemoryAllocator {
public:
// disable copy constructors and assignment
VulkanMemoryAllocator(const VulkanMemoryAllocator &) = delete;
VulkanMemoryAllocator &operator=(const VulkanMemoryAllocator &) = delete;
// disable non-factory constrction
VulkanMemoryAllocator() = delete;
~VulkanMemoryAllocator() = delete;
// Factory methods for creation / destruction
static VulkanMemoryAllocator *create(void *user_context, const VulkanMemoryConfig &config,
VkDevice dev, VkPhysicalDevice phys_dev,
const SystemMemoryAllocatorFns &system_allocator,
const VkAllocationCallbacks *alloc_callbacks = nullptr);
static int destroy(void *user_context, VulkanMemoryAllocator *allocator);
// Public interface methods
MemoryRegion *reserve(void *user_context, const MemoryRequest &request);
int conform(void *user_context, MemoryRequest *request); //< conforms the given memory request into one that can be allocated
int release(void *user_context, MemoryRegion *region); //< unmark and cache the region for reuse
int reclaim(void *user_context, MemoryRegion *region); //< free the region and consolidate
int retain(void *user_context, MemoryRegion *region); //< retain the region and increase its use count
bool collect(void *user_context); //< returns true if any blocks were removed
int release(void *user_context);
int destroy(void *user_context);
void *map(void *user_context, MemoryRegion *region);
int unmap(void *user_context, MemoryRegion *region);
MemoryRegion *create_crop(void *user_context, MemoryRegion *region, uint64_t offset);
int destroy_crop(void *user_context, MemoryRegion *region);
MemoryRegion *owner_of(void *user_context, MemoryRegion *region);
VkDevice current_device() const {
return this->device;
}
VkPhysicalDevice current_physical_device() const {
return this->physical_device;
}
VkPhysicalDeviceLimits current_physical_device_limits() const {
return this->physical_device_limits;
}
const VkAllocationCallbacks *callbacks() const {
return this->alloc_callbacks;
}
static const VulkanMemoryConfig &default_config();
static int allocate_block(void *instance_ptr, MemoryBlock *block);
static int deallocate_block(void *instance_ptr, MemoryBlock *block);
static int conform_block_request(void *instance_ptr, MemoryRequest *request);
static int allocate_region(void *instance_ptr, MemoryRegion *region);
static int deallocate_region(void *instance_ptr, MemoryRegion *region);
static int conform_region_request(void *instance_ptr, MemoryRequest *request);
size_t bytes_allocated_for_blocks() const;
size_t blocks_allocated() const;
size_t bytes_allocated_for_regions() const;
size_t regions_allocated() const;
private:
static constexpr uint32_t invalid_memory_heap = uint32_t(-1);
static constexpr uint32_t invalid_usage_flags = uint32_t(-1);
static constexpr uint32_t invalid_memory_type = uint32_t(VK_MAX_MEMORY_TYPES);
// Initializes a new instance
int initialize(void *user_context, const VulkanMemoryConfig &config,
VkDevice dev, VkPhysicalDevice phys_dev,
const SystemMemoryAllocatorFns &system_allocator,
const VkAllocationCallbacks *alloc_callbacks = nullptr);
uint32_t select_memory_usage(void *user_context, MemoryProperties properties) const;
uint32_t preferred_flags_for_memory_properties(void *user_context, MemoryProperties properties) const;
uint32_t required_flags_for_memory_properties(void *user_context, MemoryProperties properties) const;
bool is_valid_memory_type_for_heap(void *user_context,
VkPhysicalDeviceMemoryProperties *memory_properties,
uint32_t memory_type_index,
uint32_t heap_index) const;
bool is_preferred_memory_type_for_flags(void *user_context,
VkPhysicalDeviceMemoryProperties *memory_properties,
uint32_t memory_type_index,
uint32_t preferred_flags,
uint32_t required_flags) const;
bool is_valid_memory_type_for_flags(void *user_context,
VkPhysicalDeviceMemoryProperties *memory_properties,
uint32_t memory_type_index,
uint32_t required_flags) const;
int lookup_requirements(void *user_context, size_t size, uint32_t usage_flags, VkMemoryRequirements *memory_requirements);
size_t block_byte_count = 0;
size_t block_count = 0;
size_t region_byte_count = 0;
size_t region_count = 0;
void *owner_context = nullptr;
VulkanMemoryConfig config;
VkDevice device = nullptr;
VkPhysicalDevice physical_device = nullptr;
VkPhysicalDeviceLimits physical_device_limits = {};
const VkAllocationCallbacks *alloc_callbacks = nullptr;
BlockAllocator *block_allocator = nullptr;
};
VulkanMemoryAllocator *VulkanMemoryAllocator::create(void *user_context,
const VulkanMemoryConfig &cfg, VkDevice dev, VkPhysicalDevice phys_dev,
const SystemMemoryAllocatorFns &system_allocator,
const VkAllocationCallbacks *alloc_callbacks) {
if (system_allocator.allocate == nullptr) {
error(user_context) << "VulkanBlockAllocator: Unable to create instance! Missing system allocator interface!\n";
return nullptr;
}
VulkanMemoryAllocator *result = reinterpret_cast<VulkanMemoryAllocator *>(
system_allocator.allocate(user_context, sizeof(VulkanMemoryAllocator)));
if (result == nullptr) {
error(user_context) << "VulkanMemoryAllocator: Failed to create instance! Out of memory!\n";
return nullptr; // caller must handle error case for out-of-memory
}
result->initialize(user_context, cfg, dev, phys_dev, system_allocator, alloc_callbacks);
return result;
}
int VulkanMemoryAllocator::destroy(void *user_context, VulkanMemoryAllocator *instance) {
if (instance == nullptr) {
error(user_context) << "VulkanBlockAllocator: Unable to destroy instance! Invalide instance pointer!\n";
return halide_error_code_internal_error;
}
BlockAllocator::MemoryAllocators allocators = instance->block_allocator->current_allocators();
instance->destroy(user_context);
if (allocators.system.deallocate == nullptr) {
error(user_context) << "VulkanBlockAllocator: Unable to destroy instance! Missing system allocator interface!\n";
return halide_error_code_internal_error;
}
allocators.system.deallocate(user_context, instance);
return halide_error_code_success;
}
int VulkanMemoryAllocator::initialize(void *user_context,
const VulkanMemoryConfig &cfg, VkDevice dev, VkPhysicalDevice phys_dev,
const SystemMemoryAllocatorFns &system_allocator,
const VkAllocationCallbacks *callbacks) {
owner_context = user_context;
config = cfg;
device = dev;
physical_device = phys_dev;
alloc_callbacks = callbacks;
region_count = 0;
region_byte_count = 0;
block_count = 0;
block_byte_count = 0;
BlockAllocator::MemoryAllocators allocators;
allocators.system = system_allocator;
allocators.block = {VulkanMemoryAllocator::allocate_block, VulkanMemoryAllocator::deallocate_block, VulkanMemoryAllocator::conform_block_request};
allocators.region = {VulkanMemoryAllocator::allocate_region, VulkanMemoryAllocator::deallocate_region, VulkanMemoryAllocator::conform_region_request};
BlockAllocator::Config block_allocator_config = {0};
block_allocator_config.maximum_pool_size = cfg.maximum_pool_size;
block_allocator_config.maximum_block_count = cfg.maximum_block_count;
block_allocator_config.maximum_block_size = cfg.maximum_block_size;
block_allocator_config.minimum_block_size = cfg.minimum_block_size;
block_allocator_config.nearest_multiple = cfg.nearest_multiple;
block_allocator = BlockAllocator::create(user_context, block_allocator_config, allocators);
if (block_allocator == nullptr) {
error(user_context) << "VulkanMemoryAllocator: Failed to create BlockAllocator! Out of memory?!\n";
return halide_error_code_out_of_memory;
}
// get the physical device properties to determine limits and allocation requirements
VkPhysicalDeviceProperties physical_device_properties = {0};
memset(&physical_device_limits, 0, sizeof(VkPhysicalDeviceLimits));
vkGetPhysicalDeviceProperties(physical_device, &physical_device_properties);
memcpy(&physical_device_limits, &(physical_device_properties.limits), sizeof(VkPhysicalDeviceLimits));
return halide_error_code_success;
}
MemoryRegion *VulkanMemoryAllocator::reserve(void *user_context, const MemoryRequest &request) {
#if defined(HL_VK_DEBUG_MEM)
debug(nullptr) << "VulkanMemoryAllocator: Reserving memory ("
<< "user_context=" << user_context << " "
<< "block_allocator=" << (void *)(block_allocator) << " "
<< "request_size=" << (uint32_t)(request.size) << " "
<< "device=" << (void *)(device) << " "
<< "physical_device=" << (void *)(physical_device) << ") ...\n";
#endif
if ((device == nullptr) || (physical_device == nullptr)) {
error(user_context) << "VulkanMemoryAllocator: Unable to reserve memory! Invalid device handle!\n";
return nullptr;
}
if (block_allocator == nullptr) {
error(user_context) << "VulkanMemoryAllocator: Unable to reserve memory! Invalid block allocator!\n";
return nullptr;
}
return block_allocator->reserve(this, request);
}
void *VulkanMemoryAllocator::map(void *user_context, MemoryRegion *region) {
#if defined(HL_VK_DEBUG_MEM)
debug(nullptr) << "VulkanMemoryAllocator: Mapping region ("
<< "user_context=" << user_context << " "
<< "device=" << (void *)(device) << " "
<< "physical_device=" << (void *)(physical_device) << " "
<< "region=" << (void *)(region) << " "
<< "region_size=" << (uint32_t)region->size << " "
<< "region_offset=" << (uint32_t)region->offset << " "
<< "crop_offset=" << (uint32_t)region->range.head_offset << ") ...\n";
#endif
if ((device == nullptr) || (physical_device == nullptr)) {
error(user_context) << "VulkanMemoryAllocator: Unable to map memory! Invalid device handle!\n";
return nullptr;
}
if (block_allocator == nullptr) {
error(user_context) << "VulkanMemoryAllocator: Unable to map memory! Invalid block allocator!\n";
return nullptr;
}
MemoryRegion *owner = owner_of(user_context, region);
RegionAllocator *region_allocator = RegionAllocator::find_allocator(user_context, owner);
if (region_allocator == nullptr) {
error(user_context) << "VulkanMemoryAllocator: Unable to map region! Invalid region allocator handle!\n";
return nullptr; // NOTE: caller must handle nullptr
}
BlockResource *block_resource = region_allocator->block_resource();
if (block_resource == nullptr) {
error(user_context) << "VulkanMemoryAllocator: Unable to map region! Invalid block resource handle!\n";
return nullptr; // NOTE: caller must handle nullptr
}
VkDeviceMemory *device_memory = reinterpret_cast<VkDeviceMemory *>(block_resource->memory.handle);
if (device_memory == nullptr) {
error(user_context) << "VulkanMemoryAllocator: Unable to map region! Invalid device memory handle!\n";
return nullptr; // NOTE: caller must handle nullptr
}
void *mapped_ptr = nullptr;
VkDeviceSize memory_offset = region->offset + region->range.head_offset;
VkDeviceSize memory_size = region->size - region->range.tail_offset - region->range.head_offset;
if (((double)region->size - (double)region->range.tail_offset - (double)region->range.head_offset) <= 0.0) {
error(user_context) << "VulkanMemoryAllocator: Unable to map region! Invalid memory range !\n";
return nullptr;
}
#if defined(HL_VK_DEBUG_MEM)
debug(nullptr) << "VulkanMemoryAllocator: MapMemory ("
<< "user_context=" << user_context << "\n"
<< " region_size=" << (uint32_t)region->size << "\n"
<< " region_offset=" << (uint32_t)region->offset << "\n"
<< " region_range.head_offset=" << (uint32_t)region->range.head_offset << "\n"
<< " region_range.tail_offset=" << (uint32_t)region->range.tail_offset << "\n"
<< " memory_offset=" << (uint32_t)memory_offset << "\n"
<< " memory_size=" << (uint32_t)memory_size << "\n)\n";
#endif
VkResult result = vkMapMemory(device, *device_memory, memory_offset, memory_size, 0, (void **)(&mapped_ptr));
if (result != VK_SUCCESS) {
error(user_context) << "VulkanMemoryAllocator: Mapping region failed! vkMapMemory returned error code: " << vk_get_error_name(result) << "\n";
return nullptr;
}
return mapped_ptr;
}
int VulkanMemoryAllocator::unmap(void *user_context, MemoryRegion *region) {
#if defined(HL_VK_DEBUG_MEM)
debug(nullptr) << "VulkanMemoryAllocator: Unmapping region ("
<< "user_context=" << user_context << " "
<< "device=" << (void *)(device) << " "
<< "physical_device=" << (void *)(physical_device) << " "
<< "region=" << (void *)(region) << " "
<< "region_size=" << (uint32_t)region->size << " "
<< "region_offset=" << (uint32_t)region->offset << " "
<< "crop_offset=" << (uint32_t)region->range.head_offset << ") ...\n";
#endif
if ((device == nullptr) || (physical_device == nullptr)) {
error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid device handle!\n";
return halide_error_code_generic_error;
}
MemoryRegion *owner = owner_of(user_context, region);
RegionAllocator *region_allocator = RegionAllocator::find_allocator(user_context, owner);
if (region_allocator == nullptr) {
error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid region allocator handle!\n";
return halide_error_code_internal_error;
}
BlockResource *block_resource = region_allocator->block_resource();
if (block_resource == nullptr) {
error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid block resource handle!\n";
return halide_error_code_internal_error;
}
VkDeviceMemory *device_memory = reinterpret_cast<VkDeviceMemory *>(block_resource->memory.handle);
if (device_memory == nullptr) {
error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid device memory handle!\n";
return halide_error_code_internal_error;
}
vkUnmapMemory(device, *device_memory);
return halide_error_code_success;
}
MemoryRegion *VulkanMemoryAllocator::create_crop(void *user_context, MemoryRegion *region, uint64_t offset) {
#if defined(HL_VK_DEBUG_MEM)
debug(nullptr) << "VulkanMemoryAllocator: Cropping region ("
<< "user_context=" << user_context << " "
<< "device=" << (void *)(device) << " "
<< "physical_device=" << (void *)(physical_device) << " "
<< "region=" << (void *)(region) << " "
<< "region_size=" << (uint32_t)region->size << " "
<< "region_offset=" << (uint32_t)region->offset << " "
<< "crop_offset=" << (int64_t)offset << ") ...\n";
#endif
if ((device == nullptr) || (physical_device == nullptr)) {
error(user_context) << "VulkanMemoryAllocator: Unable to crop region! Invalid device handle!\n";
return nullptr;
}
MemoryRegion *owner = owner_of(user_context, region);
RegionAllocator *region_allocator = RegionAllocator::find_allocator(user_context, owner);
if (region_allocator == nullptr) {
error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid region allocator handle!\n";
return nullptr; // NOTE: caller must handle nullptr
}
// increment usage count
int error_code = region_allocator->retain(this, owner);
if (error_code != halide_error_code_success) {
error(user_context) << "VulkanMemoryAllocator: Unable to crop region! Failed to retain memory region!\n";
return nullptr; // NOTE: caller must handle nullptr
}
// create a new region to return, and copy all the other region's properties
const BlockAllocator::MemoryAllocators &allocators = block_allocator->current_allocators();
if (allocators.system.allocate == nullptr) {
error(user_context) << "VulkanMemoryAllocator: Unable to create crop! Missing system allocator interface!\n";
return nullptr;
}
MemoryRegion *memory_region = reinterpret_cast<MemoryRegion *>(
allocators.system.allocate(user_context, sizeof(MemoryRegion)));
if (memory_region == nullptr) {
error(user_context) << "VulkanMemoryAllocator: Failed to allocate memory region! Out of memory!\n";
return nullptr;
}
memcpy(memory_region, owner, sizeof(MemoryRegion));
// point the handle to the owner of the allocated region, and update the head offset
memory_region->is_owner = false;
memory_region->handle = (void *)owner;
memory_region->range.head_offset = owner->range.head_offset + offset;
return memory_region;
}
int VulkanMemoryAllocator::destroy_crop(void *user_context, MemoryRegion *region) {
if (region == nullptr) {
error(user_context) << "VulkanMemoryAllocator: Failed to destroy crop! Invalid memory region!\n";
return halide_error_code_generic_error;
}
MemoryRegion *owner = owner_of(user_context, region);
RegionAllocator *region_allocator = RegionAllocator::find_allocator(user_context, owner);
if (region_allocator == nullptr) {
error(user_context) << "VulkanMemoryAllocator: Unable to destroy crop region! Invalid region allocator handle!\n";
return halide_error_code_internal_error;
}
// decrement usage count
int error_code = region_allocator->release(this, owner);
if (error_code != halide_error_code_success) {
error(user_context) << "VulkanBlockAllocator: Unable to destroy crop region! Region allocator failed to release memory region!\n";
return error_code;
}
// discard the copied region struct
const BlockAllocator::MemoryAllocators &allocators = block_allocator->current_allocators();
if (allocators.system.deallocate == nullptr) {
error(user_context) << "VulkanBlockAllocator: Unable to destroy crop region! Missing system allocator interface!\n";
return halide_error_code_internal_error;
}
allocators.system.deallocate(user_context, region);
return halide_error_code_success;
}
MemoryRegion *VulkanMemoryAllocator::owner_of(void *user_context, MemoryRegion *region) {
if (region->is_owner) {
return region;
} else {
// If this is a cropped region, use the handle to retrieve the owner of the allocation
return reinterpret_cast<MemoryRegion *>(region->handle);
}
}
int VulkanMemoryAllocator::release(void *user_context, MemoryRegion *region) {
#if defined(HL_VK_DEBUG_MEM)
debug(nullptr) << "VulkanMemoryAllocator: Releasing region ("
<< "user_context=" << user_context << " "
<< "region=" << (void *)(region) << " "
<< "size=" << (uint32_t)region->size << " "
<< "offset=" << (uint32_t)region->offset << ") ...\n";
#endif
if ((device == nullptr) || (physical_device == nullptr)) {
error(user_context) << "VulkanMemoryAllocator: Unable to release region! Invalid device handle!\n";
return halide_error_code_generic_error;
}
if (block_allocator == nullptr) {
error(user_context) << "VulkanMemoryAllocator: Unable to release region! Invalid block allocator!\n";
return halide_error_code_generic_error;
}
return block_allocator->release(this, region);
}
int VulkanMemoryAllocator::reclaim(void *user_context, MemoryRegion *region) {
#if defined(HL_VK_DEBUG_MEM)
debug(nullptr) << "VulkanMemoryAllocator: Reclaiming region ("
<< "user_context=" << user_context << " "
<< "region=" << (void *)(region) << " "
<< "size=" << (uint32_t)region->size << " "
<< "offset=" << (uint32_t)region->offset << ") ...\n";
#endif
if ((device == nullptr) || (physical_device == nullptr)) {
error(user_context) << "VulkanMemoryAllocator: Unable to reclaim region! Invalid device handle!\n";
return halide_error_code_generic_error;
}
if (block_allocator == nullptr) {
error(user_context) << "VulkanMemoryAllocator: Unable to reclaim region! Invalid block allocator!\n";
return halide_error_code_generic_error;
}
return block_allocator->reclaim(this, region);
}
int VulkanMemoryAllocator::retain(void *user_context, MemoryRegion *region) {
#if defined(HL_VK_DEBUG_MEM)
debug(nullptr) << "VulkanMemoryAllocator: Retaining region ("
<< "user_context=" << user_context << " "
<< "region=" << (void *)(region) << " "
<< "size=" << (uint32_t)region->size << " "
<< "offset=" << (uint32_t)region->offset << ") ...\n";
#endif
if ((device == nullptr) || (physical_device == nullptr)) {
error(user_context) << "VulkanMemoryAllocator: Unable to retain region! Invalid device handle!\n";
return halide_error_code_generic_error;
}
if (block_allocator == nullptr) {
error(user_context) << "VulkanMemoryAllocator: Unable to retain region! Invalid block allocator!\n";
return halide_error_code_generic_error;
}
return block_allocator->retain(this, region);
}
bool VulkanMemoryAllocator::collect(void *user_context) {
#if defined(HL_VK_DEBUG_MEM)
debug(nullptr) << "VulkanMemoryAllocator: Collecting unused memory ("
<< "user_context=" << user_context << ") ... \n";
#endif
if ((device == nullptr) || (physical_device == nullptr) || (block_allocator == nullptr)) {
return false;
}
return block_allocator->collect(this);
}
int VulkanMemoryAllocator::release(void *user_context) {
#if defined(HL_VK_DEBUG_MEM)
debug(nullptr) << "VulkanMemoryAllocator: Releasing block allocator ("
<< "user_context=" << user_context << ") ... \n";
#endif
if ((device == nullptr) || (physical_device == nullptr)) {
error(user_context) << "VulkanMemoryAllocator: Unable to release allocator! Invalid device handle!\n";
return halide_error_code_generic_error;
}
if (block_allocator == nullptr) {
error(user_context) << "VulkanMemoryAllocator: Unable to release allocator! Invalid block allocator!\n";
return halide_error_code_generic_error;
}
return block_allocator->release(this);
}
int VulkanMemoryAllocator::destroy(void *user_context) {
#if defined(HL_VK_DEBUG_MEM)
debug(nullptr) << "VulkanMemoryAllocator: Destroying allocator ("
<< "user_context=" << user_context << ") ... \n";
#endif
if (block_allocator != nullptr) {
BlockAllocator::destroy(user_context, block_allocator);
block_allocator = nullptr;
}
region_count = 0;
region_byte_count = 0;
block_count = 0;
block_byte_count = 0;
return halide_error_code_success;
}
const VulkanMemoryConfig &
VulkanMemoryAllocator::default_config() {
static VulkanMemoryConfig result;
return result;
}
// --
int VulkanMemoryAllocator::lookup_requirements(void *user_context, size_t size, uint32_t usage_flags, VkMemoryRequirements *memory_requirements) {
#if defined(HL_VK_DEBUG_MEM)
debug(nullptr) << "VulkanMemoryAllocator: Looking up requirements ("
<< "user_context=" << user_context << " "
<< "size=" << (uint32_t)block->size << ", "
<< "usage_flags=" << usage_flags << ") ... \n";
#endif
VkBufferCreateInfo create_info = {
VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, // struct type
nullptr, // struct extending this
0, // create flags
size, // buffer size (in bytes)
usage_flags, // buffer usage flags
VK_SHARING_MODE_EXCLUSIVE, // sharing mode
0, nullptr};
// Create a buffer to determine alignment requirements
VkBuffer buffer = VK_NULL_HANDLE;
VkResult result = vkCreateBuffer(this->device, &create_info, this->alloc_callbacks, &buffer);
if (result != VK_SUCCESS) {
#if defined(HL_VK_DEBUG_MEM)
debug(nullptr) << "VulkanMemoryAllocator: Failed to create buffer to find requirements!\n\t"
<< "vkCreateBuffer returned: " << vk_get_error_name(result) << "\n";
#endif
error(user_context) << "VulkanRegionAllocator: Failed to create buffer to gather memory requirements!\n";
return halide_error_code_device_malloc_failed;
}
vkGetBufferMemoryRequirements(this->device, buffer, memory_requirements);
vkDestroyBuffer(this->device, buffer, this->alloc_callbacks);
return halide_error_code_success;
}
int VulkanMemoryAllocator::conform_block_request(void *instance_ptr, MemoryRequest *request) {
VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
if (instance == nullptr) {
return halide_error_code_success;
}
void *user_context = instance->owner_context;
#if defined(HL_VK_DEBUG_MEM)
debug(nullptr) << "VulkanMemoryAllocator: Conforming block request ("
<< "user_context=" << user_context << " "
<< "request=" << (void *)(request) << ") ... \n";
#endif
if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
error(user_context) << "VulkanRegionAllocator: Unable to conform block request! Invalid device handle!\n";
return halide_error_code_internal_error;
}
VkMemoryRequirements memory_requirements = {0};
uint32_t usage_flags = instance->select_memory_usage(user_context, request->properties);
int error_code = instance->lookup_requirements(user_context, request->size, usage_flags, &memory_requirements);
if (error_code != halide_error_code_success) {
error(user_context) << "VulkanRegionAllocator: Failed to conform block request! Unable to lookup requirements!\n";
return error_code;
}
#if defined(HL_VK_DEBUG_MEM)
debug(nullptr) << "VulkanMemoryAllocator: Block allocated ("
<< "size=" << (uint32_t)request->size << ", "
<< "required_alignment=" << (uint32_t)memory_requirements.alignment << ", "
<< "required_size=" << (uint32_t)memory_requirements.size << ", "
<< "uniform_buffer_offset_alignment=" << (uint32_t)instance->physical_device_limits.minUniformBufferOffsetAlignment << ", "
<< "storage_buffer_offset_alignment=" << (uint32_t)instance->physical_device_limits.minStorageBufferOffsetAlignment << ", "
<< "dedicated=" << (request->dedicated ? "true" : "false") << ")\n";
#endif
request->size = memory_requirements.size;
request->properties.alignment = memory_requirements.alignment;
return halide_error_code_success;
}
int VulkanMemoryAllocator::allocate_block(void *instance_ptr, MemoryBlock *block) {
VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
if (instance == nullptr) {
return halide_error_code_success;
}
void *user_context = instance->owner_context;
if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid device handle!\n";
return halide_error_code_internal_error;
}
if (block == nullptr) {
error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid pointer!\n";
return halide_error_code_internal_error;
}
#if defined(HL_VK_DEBUG_MEM)
debug(nullptr) << "VulkanMemoryAllocator: Allocating block ("
<< "user_context=" << user_context << " "
<< "block=" << (void *)(block) << " "
<< "size=" << (uint64_t)block->size << ", "
<< "dedicated=" << (block->dedicated ? "true" : "false") << " "
<< "usage=" << halide_memory_usage_name(block->properties.usage) << " "
<< "caching=" << halide_memory_caching_name(block->properties.caching) << " "
<< "visibility=" << halide_memory_visibility_name(block->properties.visibility) << ")\n";
#endif
VkDeviceMemory *device_memory = (VkDeviceMemory *)vk_host_malloc(nullptr, sizeof(VkDeviceMemory), 0, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT, instance->alloc_callbacks);
if (device_memory == nullptr) {
error(user_context) << "VulkanBlockAllocator: Unable to allocate block! Failed to allocate device memory handle!\n";
return halide_error_code_out_of_memory;
}
VkPhysicalDeviceMemoryProperties memory_properties;
vkGetPhysicalDeviceMemoryProperties(instance->physical_device, &memory_properties);
// Attempt to allocate device memory with the requested flags, trying each type until successful
uint32_t selected_type = invalid_memory_type;
uint32_t preferred_flags = instance->preferred_flags_for_memory_properties(user_context, block->properties);
uint32_t required_flags = instance->required_flags_for_memory_properties(user_context, block->properties);
// Try preferred flags first
for (uint32_t type_index = 0; type_index < memory_properties.memoryTypeCount; type_index++) {
if (!instance->is_preferred_memory_type_for_flags(user_context, &memory_properties, type_index, preferred_flags, required_flags)) {
continue;
}
// Allocate memory
VkMemoryAllocateInfo alloc_info = {
VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, // struct type
nullptr, // struct extending this
block->size, // size of allocation in bytes
type_index // memory type index from physical device
};
VkResult result = vkAllocateMemory(instance->device, &alloc_info, instance->alloc_callbacks, device_memory);
if (result == VK_SUCCESS) {
selected_type = type_index;
break;
}
#if defined(HL_VK_DEBUG_MEM)
debug(user_context) << "VulkanMemoryAllocator: Allocation request for preferred flags was not successful (\n"
<< "\tblock=" << (void *)(block) << "\n"
<< "\ttype_index=" << type_index << "\n"
<< "\tsize=" << (uint64_t)block->size << "\n"
<< "\tdedicated=" << (block->dedicated ? "true" : "false") << "\n"
<< "\tusage=" << halide_memory_usage_name(block->properties.usage) << "\n"
<< "\tcaching=" << halide_memory_caching_name(block->properties.caching) << "\n"
<< "\tvisibility=" << halide_memory_visibility_name(block->properties.visibility) << "\n"
<< "\tblocks_allocated=" << (uint64_t)instance->blocks_allocated() << "\n"
<< "\tbytes_allocated_for_blocks=" << (uint64_t)instance->bytes_allocated_for_blocks() << "\n"
<< "\tregions_allocated=" << (uint64_t)instance->regions_allocated() << "\n"
<< "\tbytes_allocated_for_regions=" << (uint64_t)instance->bytes_allocated_for_regions() << "\n)\n"
<< "vkAllocateMemory returned: "
<< vk_get_error_name(result) << "\n";
#endif
}
// try any valid type on any valid pool
if (selected_type == invalid_memory_type) {
for (uint32_t type_index = 0; type_index < memory_properties.memoryTypeCount; type_index++) {
if (!instance->is_valid_memory_type_for_flags(user_context, &memory_properties, type_index, required_flags)) {
continue;
}
// Allocate memory
VkMemoryAllocateInfo alloc_info = {
VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, // struct type
nullptr, // struct extending this
block->size, // size of allocation in bytes
type_index // memory type index from physical device
};
VkResult result = vkAllocateMemory(instance->device, &alloc_info, instance->alloc_callbacks, device_memory);
if (result == VK_SUCCESS) {
selected_type = type_index;
break;
}
#if defined(HL_VK_DEBUG_MEM)
debug(user_context) << "VulkanMemoryAllocator: Allocation request for valid flags was not successful (\n"
<< "\tblock=" << (void *)(block) << "\n"
<< "\ttype_index=" << type_index << "\n"
<< "\tsize=" << (uint64_t)block->size << "\n"
<< "\tdedicated=" << (block->dedicated ? "true" : "false") << "\n"
<< "\tusage=" << halide_memory_usage_name(block->properties.usage) << "\n"
<< "\tcaching=" << halide_memory_caching_name(block->properties.caching) << "\n"
<< "\tvisibility=" << halide_memory_visibility_name(block->properties.visibility) << "\n"
<< "\tblocks_allocated=" << (uint64_t)instance->blocks_allocated() << "\n"
<< "\tbytes_allocated_for_blocks=" << (uint64_t)instance->bytes_allocated_for_blocks() << "\n"
<< "\tregions_allocated=" << (uint64_t)instance->regions_allocated() << "\n"
<< "\tbytes_allocated_for_regions=" << (uint64_t)instance->bytes_allocated_for_regions() << "\n)\n"
<< "vkAllocateMemory returned: "
<< vk_get_error_name(result) << "\n";
#endif
}
}
if (selected_type == invalid_memory_type) {
vk_host_free(nullptr, device_memory, instance->alloc_callbacks);
device_memory = VK_NULL_HANDLE;
error(user_context) << "VulkanMemoryAllocator: Allocation failed for block (\n"
<< "\tblock=" << (void *)(block) << "\n"
<< "\tsize=" << (uint64_t)block->size << "\n"
<< "\tdedicated=" << (block->dedicated ? "true" : "false") << "\n"
<< "\tusage=" << halide_memory_usage_name(block->properties.usage) << "\n"
<< "\tcaching=" << halide_memory_caching_name(block->properties.caching) << "\n"
<< "\tvisibility=" << halide_memory_visibility_name(block->properties.visibility) << "\n"
<< "\tblocks_allocated=" << (uint64_t)instance->blocks_allocated() << "\n"
<< "\tbytes_allocated_for_blocks=" << (uint64_t)instance->bytes_allocated_for_blocks() << "\n"
<< "\tregions_allocated=" << (uint64_t)instance->regions_allocated() << "\n"
<< "\tbytes_allocated_for_regions=" << (uint64_t)instance->bytes_allocated_for_regions() << "\n)\n";
return halide_error_code_device_malloc_failed;
}
#ifdef DEBUG_RUNTIME
debug(nullptr) << "vkAllocateMemory: Allocated memory for device region (" << (uint64_t)block->size << " bytes) ...\n";
#endif
block->handle = (void *)device_memory;
instance->block_byte_count += block->size;
instance->block_count++;
return halide_error_code_success;
}
int VulkanMemoryAllocator::deallocate_block(void *instance_ptr, MemoryBlock *block) {
VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
if (instance == nullptr) {
return halide_error_code_success;
}
void *user_context = instance->owner_context;
#if defined(HL_VK_DEBUG_MEM)
debug(nullptr) << "VulkanMemoryAllocator: Deallocating block ("
<< "user_context=" << user_context << " "
<< "block=" << (void *)(block) << ") ... \n";
#endif
if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid device handle!\n";
return halide_error_code_internal_error;
}
if (block == nullptr) {
error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid pointer!\n";
return halide_error_code_internal_error;
}
#if defined(HL_VK_DEBUG_MEM)
debug(nullptr) << "VulkanBlockAllocator: deallocating block ("
<< "size=" << (uint32_t)block->size << ", "
<< "dedicated=" << (block->dedicated ? "true" : "false") << " "
<< "usage=" << halide_memory_usage_name(block->properties.usage) << " "
<< "caching=" << halide_memory_caching_name(block->properties.caching) << " "
<< "visibility=" << halide_memory_visibility_name(block->properties.visibility) << ")\n";
#endif
// Nothing to do if device memory was already freed
if (block->handle == nullptr) {
return halide_error_code_success;
}
VkDeviceMemory *device_memory = reinterpret_cast<VkDeviceMemory *>(block->handle);
if (device_memory == nullptr) {
error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid device memory handle!\n";
return halide_error_code_internal_error;
}
vkFreeMemory(instance->device, *device_memory, instance->alloc_callbacks);
#ifdef DEBUG_RUNTIME
debug(nullptr) << "vkFreeMemory: Deallocated memory for device region (" << (uint64_t)block->size << " bytes) ...\n";
#endif
if (instance->block_count > 0) {
instance->block_count--;
} else {
error(nullptr) << "VulkanRegionAllocator: Block counter invalid ... reseting to zero!\n";
instance->block_count = 0;
}
if (int64_t(instance->block_byte_count) - int64_t(block->size) >= 0) {
instance->block_byte_count -= block->size;
} else {
error(nullptr) << "VulkanRegionAllocator: Block byte counter invalid ... reseting to zero!\n";
instance->block_byte_count = 0;
}
block->handle = nullptr;
vk_host_free(nullptr, device_memory, instance->alloc_callbacks);
device_memory = nullptr;
return halide_error_code_success;
}
size_t VulkanMemoryAllocator::blocks_allocated() const {
return block_count;
}
size_t VulkanMemoryAllocator::bytes_allocated_for_blocks() const {
return block_byte_count;
}
uint32_t VulkanMemoryAllocator::required_flags_for_memory_properties(
void *user_context, MemoryProperties properties) const {
uint32_t required_flags = 0; //< must have in order to enable requested access
switch (properties.visibility) {
case MemoryVisibility::DeviceOnly:
required_flags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
break;
case MemoryVisibility::DeviceToHost:
required_flags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
break;
case MemoryVisibility::HostToDevice:
required_flags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
break;
case MemoryVisibility::DefaultVisibility:
case MemoryVisibility::HostOnly:
break;
case MemoryVisibility::InvalidVisibility:
error(user_context) << "VulkanMemoryAllocator: Unable to convert type! Invalid memory visibility request!\n\t"
<< "caching=" << halide_memory_caching_name(properties.caching) << "\n";
return invalid_memory_type;
};
return required_flags;
}
uint32_t VulkanMemoryAllocator::preferred_flags_for_memory_properties(
void *user_context, MemoryProperties properties) const {
uint32_t preferred_flags = 0; //< preferred memory flags for requested access type
switch (properties.visibility) {
case MemoryVisibility::HostOnly:
preferred_flags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
break;
case MemoryVisibility::DeviceToHost:
preferred_flags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
break;
case MemoryVisibility::HostToDevice:
case MemoryVisibility::DefaultVisibility:
case MemoryVisibility::DeviceOnly:
break;
case MemoryVisibility::InvalidVisibility:
error(user_context) << "VulkanMemoryAllocator: Unable to convert type! Invalid memory visibility request!\n\t"
<< "caching=" << halide_memory_caching_name(properties.caching) << "\n";
return invalid_memory_type;
};
uint32_t required_flags = required_flags_for_memory_properties(user_context, properties);
switch (properties.caching) {
case MemoryCaching::CachedCoherent:
if (required_flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) {
preferred_flags |= VK_MEMORY_PROPERTY_HOST_CACHED_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
}
break;
case MemoryCaching::UncachedCoherent:
if (required_flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) {
preferred_flags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
}
break;
case MemoryCaching::Cached:
if (required_flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) {
preferred_flags |= VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
}
break;
case MemoryCaching::Uncached:
case MemoryCaching::DefaultCaching:
break;
case MemoryCaching::InvalidCaching:
default:
error(user_context) << "VulkanMemoryAllocator: Unable to convert type! Invalid memory caching request!\n\t"
<< "caching=" << halide_memory_caching_name(properties.caching) << "\n";
return invalid_memory_type;
};
return preferred_flags;
}
bool VulkanMemoryAllocator::is_valid_memory_type_for_heap(void *user_context,
VkPhysicalDeviceMemoryProperties *memory_properties,
uint32_t memory_type_index,
uint32_t heap_index) const {
if (memory_type_index < memory_properties->memoryTypeCount) {
if (memory_properties->memoryTypes[memory_type_index].heapIndex == heap_index) {
return true;
}
}
return false;
}
bool VulkanMemoryAllocator::is_valid_memory_type_for_flags(void *user_context,
VkPhysicalDeviceMemoryProperties *memory_properties,
uint32_t memory_type_index,
uint32_t required_flags) const {
if (memory_type_index < memory_properties->memoryTypeCount) {
const VkMemoryPropertyFlags property_flags = memory_properties->memoryTypes[memory_type_index].propertyFlags;
if (required_flags) {
if ((property_flags & required_flags) != required_flags) {
return false;
}
}
return true;
}
return false;
}
bool VulkanMemoryAllocator::is_preferred_memory_type_for_flags(void *user_context,
VkPhysicalDeviceMemoryProperties *memory_properties,
uint32_t memory_type_index,
uint32_t preferred_flags,
uint32_t required_flags) const {
if (memory_type_index < memory_properties->memoryTypeCount) {
const VkMemoryPropertyFlags property_flags = memory_properties->memoryTypes[memory_type_index].propertyFlags;
if (required_flags) {
if ((property_flags & required_flags) != required_flags) {
return false;
}
}
if (preferred_flags) {
if ((property_flags & preferred_flags) != preferred_flags) {
return false;
}
}
return true;
}
return false;
}
// --
int VulkanMemoryAllocator::conform(void *user_context, MemoryRequest *request) {
// NOTE: Vulkan will only allow us to bind device memory to a buffer if the memory requirements are met.
// So now we have to check those (on every allocation) and potentially recreate the buffer if the requirements
// don't match the requested VkBuffer's properties. Note that this is the internal storage for the driver,
// whose size may be required to larger than our requested size (even though we will only ever touch the
// size of the region we're managing as within our block)
VkMemoryRequirements memory_requirements = {0};
uint32_t usage_flags = select_memory_usage(user_context, request->properties);
int error_code = lookup_requirements(user_context, request->size, usage_flags, &memory_requirements);
if (error_code != halide_error_code_success) {
error(user_context) << "VulkanRegionAllocator: Failed to conform block request! Unable to lookup requirements!\n";
return error_code;
}
#if defined(HL_VK_DEBUG_MEM)
debug(nullptr) << "VulkanMemoryAllocator: Buffer requirements ("
<< "requested_size=" << (uint32_t)region->size << ", "
<< "required_alignment=" << (uint32_t)memory_requirements.alignment << ", "
<< "required_size=" << (uint32_t)memory_requirements.size << ")\n";
#endif
// Enforce any alignment constraints reported by the device limits for each usage type
if (usage_flags & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) {
if ((request->alignment % this->physical_device_limits.minStorageBufferOffsetAlignment) != 0) {
request->alignment = this->physical_device_limits.minStorageBufferOffsetAlignment;
}
} else if (usage_flags & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT) {
if ((request->alignment % this->physical_device_limits.minUniformBufferOffsetAlignment) != 0) {
request->alignment = this->physical_device_limits.minUniformBufferOffsetAlignment;
}
}
// Ensure the request ends on an aligned address
if (request->alignment > config.nearest_multiple) {
request->properties.nearest_multiple = request->alignment;
}
size_t actual_alignment = conform_alignment(request->alignment, memory_requirements.alignment);
size_t actual_offset = aligned_offset(request->offset, actual_alignment);
size_t actual_size = conform_size(actual_offset, memory_requirements.size, actual_alignment, request->properties.nearest_multiple);
#if defined(HL_VK_DEBUG_MEM)
if ((request->size != actual_size) || (request->alignment != actual_alignment) || (request->offset != actual_offset)) {
debug(nullptr) << "VulkanMemoryAllocator: Adjusting request to match requirements (\n"
<< " size = " << (uint64_t)request->size << " => " << (uint64_t)actual_size << ",\n"
<< " alignment = " << (uint64_t)request->alignment << " => " << (uint64_t)actual_alignment << ",\n"
<< " offset = " << (uint64_t)request->offset << " => " << (uint64_t)actual_offset << ",\n"
<< " required.size = " << (uint64_t)memory_requirements.size << ",\n"
<< " required.alignment = " << (uint64_t)memory_requirements.alignment << "\n)\n";
}
#endif
request->size = actual_size;
request->alignment = actual_alignment;
request->offset = actual_offset;
return halide_error_code_success;
}
int VulkanMemoryAllocator::conform_region_request(void *instance_ptr, MemoryRequest *request) {
VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
if (instance == nullptr) {
return halide_error_code_success;
}
void *user_context = instance->owner_context;
#if defined(HL_VK_DEBUG_MEM)
debug(nullptr) << "VulkanMemoryAllocator: Conforming region request ("
<< "user_context=" << user_context << " "
<< "request=" << (void *)(region) << ") ... \n";
#endif
if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
error(user_context) << "VulkanRegionAllocator: Unable to conform region request! Invalid device handle!\n";
return halide_error_code_internal_error;
}
#if defined(HL_VK_DEBUG_MEM)
debug(nullptr) << "VulkanRegionAllocator: Conforming region request ("
<< "size=" << (uint32_t)request->size << ", "
<< "offset=" << (uint32_t)request->offset << ", "
<< "dedicated=" << (request->dedicated ? "true" : "false") << " "
<< "usage=" << halide_memory_usage_name(request->properties.usage) << " "
<< "caching=" << halide_memory_caching_name(request->properties.caching) << " "
<< "visibility=" << halide_memory_visibility_name(request->properties.visibility) << ")\n";
#endif
return instance->conform(user_context, request);
}
int VulkanMemoryAllocator::allocate_region(void *instance_ptr, MemoryRegion *region) {
VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
if (instance == nullptr) {
return halide_error_code_success;
}
void *user_context = instance->owner_context;
#if defined(HL_VK_DEBUG_MEM)
debug(nullptr) << "VulkanMemoryAllocator: Allocating region ("
<< "user_context=" << user_context << " "
<< "region=" << (void *)(region) << ") ... \n";
#endif
if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
error(user_context) << "VulkanRegionAllocator: Unable to allocate region! Invalid device handle!\n";
return halide_error_code_internal_error;
}
if (region == nullptr) {
error(user_context) << "VulkanRegionAllocator: Unable to allocate region! Invalid pointer!\n";
return halide_error_code_internal_error;
}
#if defined(HL_VK_DEBUG_MEM)
debug(nullptr) << "VulkanRegionAllocator: Allocating region ("
<< "size=" << (uint32_t)region->size << ", "
<< "offset=" << (uint32_t)region->offset << ", "
<< "dedicated=" << (region->dedicated ? "true" : "false") << " "
<< "usage=" << halide_memory_usage_name(region->properties.usage) << " "
<< "caching=" << halide_memory_caching_name(region->properties.caching) << " "
<< "visibility=" << halide_memory_visibility_name(region->properties.visibility) << ")\n";
#endif
uint32_t usage_flags = instance->select_memory_usage(user_context, region->properties);
VkBufferCreateInfo create_info = {
VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, // struct type
nullptr, // struct extending this
0, // create flags
region->size, // buffer size (in bytes)
usage_flags, // buffer usage flags
VK_SHARING_MODE_EXCLUSIVE, // sharing mode
0, nullptr};
VkBuffer *buffer = (VkBuffer *)vk_host_malloc(nullptr, sizeof(VkBuffer), 0, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT, instance->alloc_callbacks);
if (buffer == nullptr) {
error(user_context) << "VulkanRegionAllocator: Unable to allocate region! Failed to allocate buffer handle!\n";
return halide_error_code_out_of_memory;
}
VkResult result = vkCreateBuffer(instance->device, &create_info, instance->alloc_callbacks, buffer);
if (result != VK_SUCCESS) {
// Allocation failed ... collect unused regions and try again ...
instance->collect(user_context);
result = vkCreateBuffer(instance->device, &create_info, instance->alloc_callbacks, buffer);
}
if (result != VK_SUCCESS) {
vk_host_free(nullptr, buffer, instance->alloc_callbacks);
buffer = nullptr;
error(user_context) << "VulkanRegionAllocator: Failed to create buffer!\n\t"
<< "vkCreateBuffer returned: " << vk_get_error_name(result) << "\n";
return halide_error_code_device_malloc_failed;
}
// NOTE: Vulkan will only allow us to bind device memory to a buffer if the memory requirements are met.
// So now we have to check those (on every allocation) and potentially recreate the buffer if the requirements
// don't match the requested VkBuffer's properties. Note that this is the internal storage for the driver,
// whose size may be required to larger than our requested size (even though we will only ever touch the
// size of the region we're managing as within our block)
VkMemoryRequirements memory_requirements = {0};
vkGetBufferMemoryRequirements(instance->device, *buffer, &memory_requirements);
#if defined(HL_VK_DEBUG_MEM)
debug(nullptr) << "VulkanMemoryAllocator: Buffer requirements ("
<< "requested_size=" << (uint32_t)region->size << ", "
<< "required_alignment=" << (uint32_t)memory_requirements.alignment << ", "
<< "required_size=" << (uint32_t)memory_requirements.size << ")\n";
#endif
if (memory_requirements.size > region->size) {
vkDestroyBuffer(instance->device, *buffer, instance->alloc_callbacks);
#ifdef DEBUG_RUNTIME
debug(nullptr) << "VulkanMemoryAllocator: Reallocating buffer to match required size ("
<< (uint64_t)region->size << " => " << (uint64_t)memory_requirements.size << " bytes) ...\n";
#endif
create_info.size = memory_requirements.size;
VkResult result = vkCreateBuffer(instance->device, &create_info, instance->alloc_callbacks, buffer);
if (result != VK_SUCCESS) {
error(user_context) << "VulkanRegionAllocator: Failed to recreate buffer!\n\t"
<< "vkCreateBuffer returned: " << vk_get_error_name(result) << "\n";
return halide_error_code_device_malloc_failed;
}
}
#ifdef DEBUG_RUNTIME
debug(nullptr) << "vkCreateBuffer: Created buffer for device region (" << (uint64_t)region->size << " bytes) ...\n";
#endif
RegionAllocator *region_allocator = RegionAllocator::find_allocator(user_context, region);
if (region_allocator == nullptr) {
error(user_context) << "VulkanBlockAllocator: Unable to allocate region! Invalid region allocator!\n";
return halide_error_code_internal_error;
}
BlockResource *block_resource = region_allocator->block_resource();
if (block_resource == nullptr) {
error(user_context) << "VulkanBlockAllocator: Unable to allocate region! Invalid block resource handle!\n";
return halide_error_code_internal_error;
}
VkDeviceMemory *device_memory = reinterpret_cast<VkDeviceMemory *>(block_resource->memory.handle);
if (device_memory == nullptr) {
error(user_context) << "VulkanBlockAllocator: Unable to allocate region! Invalid device memory handle!\n";
return halide_error_code_internal_error;
}
// Finally, bind buffer to the device memory
result = vkBindBufferMemory(instance->device, *buffer, *device_memory, region->offset);
if (result != VK_SUCCESS) {
error(user_context) << "VulkanRegionAllocator: Failed to bind buffer!\n\t"
<< "vkBindBufferMemory returned: " << vk_get_error_name(result) << "\n";
return halide_error_code_generic_error;
}
region->handle = (void *)buffer;
region->is_owner = true;
instance->region_byte_count += region->size;
instance->region_count++;
return halide_error_code_success;
}
int VulkanMemoryAllocator::deallocate_region(void *instance_ptr, MemoryRegion *region) {
VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
if (instance == nullptr) {
return halide_error_code_success;
}
void *user_context = instance->owner_context;
#if defined(HL_VK_DEBUG_MEM)
debug(nullptr) << "VulkanMemoryAllocator: Deallocating region ("
<< "user_context=" << user_context << " "
<< "region=" << (void *)(region) << ") ... \n";
#endif
if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
error(user_context) << "VulkanRegionAllocator: Unable to deallocate region! Invalid device handle!\n";
return halide_error_code_internal_error;
}
if (region == nullptr) {
error(user_context) << "VulkanRegionAllocator: Unable to deallocate region! Invalid pointer!\n";
return halide_error_code_internal_error;
}
#if defined(HL_VK_DEBUG_MEM)
debug(nullptr) << "VulkanRegionAllocator: Deallocating region ("
<< "size=" << (uint32_t)region->size << ", "
<< "offset=" << (uint32_t)region->offset << ", "
<< "dedicated=" << (region->dedicated ? "true" : "false") << " "
<< "usage=" << halide_memory_usage_name(region->properties.usage) << " "
<< "caching=" << halide_memory_caching_name(region->properties.caching) << " "
<< "visibility=" << halide_memory_visibility_name(region->properties.visibility) << ")\n";
#endif
if (region->handle == nullptr) {
error(user_context) << "VulkanRegionAllocator: Unable to deallocate region! Invalid handle!\n";
return halide_error_code_internal_error;
}
VkBuffer *buffer = reinterpret_cast<VkBuffer *>(region->handle);
if (buffer == nullptr) {
error(user_context) << "VulkanRegionAllocator: Unable to deallocate region! Invalid buffer handle!\n";
return halide_error_code_internal_error;
}
vkDestroyBuffer(instance->device, *buffer, instance->alloc_callbacks);
#ifdef DEBUG_RUNTIME
debug(nullptr) << "vkDestroyBuffer: Destroyed buffer for device region (" << (uint64_t)region->size << " bytes) ...\n";
#endif
halide_error_code_t error_code = halide_error_code_success;
region->handle = nullptr;
if (instance->region_count > 0) {
instance->region_count--;
} else {
error(user_context) << "VulkanRegionAllocator: Region counter invalid ... reseting to zero!\n";
instance->region_count = 0;
error_code = halide_error_code_internal_error;
}
if (int64_t(instance->region_byte_count) - int64_t(region->size) >= 0) {
instance->region_byte_count -= region->size;
} else {
error(user_context) << "VulkanRegionAllocator: Region byte counter invalid ... reseting to zero!\n";
instance->region_byte_count = 0;
error_code = halide_error_code_internal_error;
}
vk_host_free(nullptr, buffer, instance->alloc_callbacks);
buffer = nullptr;
return error_code;
}
size_t VulkanMemoryAllocator::regions_allocated() const {
return region_count;
}
size_t VulkanMemoryAllocator::bytes_allocated_for_regions() const {
return region_byte_count;
}
uint32_t VulkanMemoryAllocator::select_memory_usage(void *user_context, MemoryProperties properties) const {
uint32_t result = 0;
switch (properties.usage) {
case MemoryUsage::UniformStorage:
result |= VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT;
break;
case MemoryUsage::DynamicStorage:
case MemoryUsage::StaticStorage:
result |= VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
break;
case MemoryUsage::TransferSrc:
result |= VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT;
break;
case MemoryUsage::TransferDst:
result |= VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
break;
case MemoryUsage::TransferSrcDst:
result |= VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
break;
case MemoryUsage::DefaultUsage:
case MemoryUsage::InvalidUsage:
default:
error(user_context) << "VulkanRegionAllocator: Unable to convert type! Invalid memory usage request!\n\t"
<< "usage=" << halide_memory_usage_name(properties.usage) << "\n";
return invalid_usage_flags;
};
if (result == invalid_usage_flags) {
error(user_context) << "VulkanRegionAllocator: Failed to find appropriate memory usage for given properties:\n\t"
<< "usage=" << halide_memory_usage_name(properties.usage) << " "
<< "caching=" << halide_memory_caching_name(properties.caching) << " "
<< "visibility=" << halide_memory_visibility_name(properties.visibility) << "\n";
return invalid_usage_flags;
}
return result;
}
// --------------------------------------------------------------------------
namespace {
// --------------------------------------------------------------------------
// Halide System allocator for host allocations
void *vk_system_malloc(void *user_context, size_t size) {
return malloc(size);
}
void vk_system_free(void *user_context, void *ptr) {
free(ptr);
}
// Vulkan host-side allocation
void *vk_host_malloc(void *user_context, size_t size, size_t alignment, VkSystemAllocationScope scope, const VkAllocationCallbacks *callbacks) {
if (callbacks) {
return callbacks->pfnAllocation(user_context, size, alignment, scope);
} else {
return vk_system_malloc(user_context, size);
}
}
void vk_host_free(void *user_context, void *ptr, const VkAllocationCallbacks *callbacks) {
if (callbacks) {
return callbacks->pfnFree(user_context, ptr);
} else {
return vk_system_free(user_context, ptr);
}
}
VulkanMemoryAllocator *vk_create_memory_allocator(void *user_context,
VkDevice device,
VkPhysicalDevice physical_device,
const VkAllocationCallbacks *alloc_callbacks) {
SystemMemoryAllocatorFns system_allocator = {vk_system_malloc, vk_system_free};
VulkanMemoryConfig config = memory_allocator_config;
// Parse the allocation config string (if specified).
//
// `HL_VK_ALLOC_CONFIG=N:N:N` will tell Halide to configure the Vulkan memory
// allocator use the given constraints specified as three integer values
// separated by a `:` or `;`. These values correspond to `minimum_block_size`,
// `maximum_block_size` and `maximum_block_count`.
//
const char *alloc_config = vk_get_alloc_config_internal(user_context);
if (!StringUtils::is_empty(alloc_config)) {
StringTable alloc_config_values;
alloc_config_values.parse(user_context, alloc_config, HL_VK_ENV_DELIM);
if (alloc_config_values.size() > 0) {
config.maximum_pool_size = atoi(alloc_config_values[0]) * 1024 * 1024;
print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.maximum_pool_size << " for maximum pool size (in bytes)\n";
}
if (alloc_config_values.size() > 1) {
config.minimum_block_size = atoi(alloc_config_values[1]) * 1024 * 1024;
print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.minimum_block_size << " for minimum block size (in bytes)\n";
}
if (alloc_config_values.size() > 2) {
config.maximum_block_size = atoi(alloc_config_values[2]) * 1024 * 1024;
print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.maximum_block_size << " for maximum block size (in bytes)\n";
}
if (alloc_config_values.size() > 3) {
config.maximum_block_count = atoi(alloc_config_values[3]);
print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.maximum_block_count << " for maximum block count\n";
}
if (alloc_config_values.size() > 4) {
config.nearest_multiple = atoi(alloc_config_values[4]);
print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.nearest_multiple << " for nearest multiple\n";
}
}
return VulkanMemoryAllocator::create(user_context,
config, device, physical_device,
system_allocator, alloc_callbacks);
}
int vk_destroy_memory_allocator(void *user_context, VulkanMemoryAllocator *allocator) {
if (allocator != nullptr) {
allocator->collect(user_context);
VulkanMemoryAllocator::destroy(user_context, allocator);
allocator = nullptr;
}
return halide_error_code_success;
}
// --------------------------------------------------------------------------
int vk_clear_device_buffer(void *user_context,
VulkanMemoryAllocator *allocator,
VkCommandBuffer command_buffer,
VkQueue command_queue,
VkBuffer device_buffer) {
#ifdef DEBUG_RUNTIME
debug(user_context)
<< " vk_clear_device_buffer (user_context: " << user_context << ", "
<< "allocator: " << (void *)allocator << ", "
<< "command_buffer: " << (void *)command_buffer << ", "
<< "command_queue: " << (void *)command_queue << ", "
<< "device_buffer: " << (void *)device_buffer << ")\n";
#endif
// begin the command buffer
VkCommandBufferBeginInfo command_buffer_begin_info =
{
VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, // struct type
nullptr, // pointer to struct extending this
VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT, // flags
nullptr // pointer to parent command buffer
};
VkResult result = vkBeginCommandBuffer(command_buffer, &command_buffer_begin_info);
if (result != VK_SUCCESS) {
error(user_context) << "Vulkan: vkBeginCommandBuffer returned " << vk_get_error_name(result) << "\n";
return halide_error_code_generic_error;
}
// fill buffer with zero values up to the size of the buffer
vkCmdFillBuffer(command_buffer, device_buffer, 0, VK_WHOLE_SIZE, 0);
// end the command buffer
result = vkEndCommandBuffer(command_buffer);
if (result != VK_SUCCESS) {
error(user_context) << "Vulkan: vkEndCommandBuffer returned " << vk_get_error_name(result) << "\n";
return halide_error_code_generic_error;
}
// submit the command buffer
VkSubmitInfo submit_info =
{
VK_STRUCTURE_TYPE_SUBMIT_INFO, // struct type
nullptr, // pointer to struct extending this
0, // wait semaphore count
nullptr, // semaphores
nullptr, // pipeline stages where semaphore waits occur
1, // how many command buffers to execute
&command_buffer, // the command buffers
0, // number of semaphores to signal
nullptr // the semaphores to signal
};
result = vkQueueSubmit(command_queue, 1, &submit_info, VK_NULL_HANDLE);
if (result != VK_SUCCESS) {
error(user_context) << "Vulkan: vkQueueSubmit returned " << vk_get_error_name(result) << "\n";
return halide_error_code_generic_error;
}
// wait for memset to finish
result = vkQueueWaitIdle(command_queue);
if (result != VK_SUCCESS) {
error(user_context) << "Vulkan: vkQueueWaitIdle returned " << vk_get_error_name(result) << "\n";
return halide_error_code_generic_error;
}
return halide_error_code_success;
}
// --------------------------------------------------------------------------
} // namespace
} // namespace Vulkan
} // namespace Internal
} // namespace Runtime
} // namespace Halide
// --------------------------------------------------------------------------
extern "C" {
// --------------------------------------------------------------------------
WEAK void halide_vulkan_set_allocation_callbacks(const VkAllocationCallbacks *callbacks) {
using namespace Halide::Runtime::Internal::Vulkan;
ScopedSpinLock lock(&custom_allocation_callbacks_lock);
custom_allocation_callbacks = callbacks;
}
WEAK const VkAllocationCallbacks *halide_vulkan_get_allocation_callbacks(void *user_context) {
using namespace Halide::Runtime::Internal::Vulkan;
ScopedSpinLock lock(&custom_allocation_callbacks_lock);
return custom_allocation_callbacks;
}
// --------------------------------------------------------------------------
} // extern "C"
#endif // HALIDE_RUNTIME_VULKAN_MEMORY_H
debug
#define debug(n)
For optional debugging during codegen, use the debug macro as follows:
Definition Debug.h:52
halide_error_code_t
halide_error_code_t
The error codes that may be returned by a Halide pipeline.
Definition HalideRuntime.h:1074
halide_error_code_internal_error
@ halide_error_code_internal_error
There is a bug in the Halide compiler.
Definition HalideRuntime.h:1164
halide_error_code_generic_error
@ halide_error_code_generic_error
An uncategorized error occurred.
Definition HalideRuntime.h:1079
halide_error_code_success
@ halide_error_code_success
There was no error.
Definition HalideRuntime.h:1076
halide_error_code_device_malloc_failed
@ halide_error_code_device_malloc_failed
The Halide runtime encountered an error while trying to allocate memory on device.
Definition HalideRuntime.h:1141
halide_error_code_out_of_memory
@ halide_error_code_out_of_memory
A call to halide_malloc returned NULL.
Definition HalideRuntime.h:1119
block_allocator.h
Halide::Runtime::Internal::BlockAllocator
Allocator class interface for managing large contiguous blocks of memory, which are then sub-allocate...
Definition block_allocator.h:25
Halide::Runtime::Internal::BlockAllocator::destroy
static void destroy(void *user_context, BlockAllocator *block_allocator)
Definition block_allocator.h:138
Halide::Runtime::Internal::BlockAllocator::create
static BlockAllocator * create(void *user_context, const Config &config, const MemoryAllocators &allocators)
Definition block_allocator.h:124
Halide::Runtime::Internal::BlockAllocator::current_allocators
const MemoryAllocators & current_allocators() const
Definition block_allocator.h:629
Halide::Runtime::Internal::RegionAllocator
Allocator class interface for sub-allocating a contiguous memory block into smaller regions of memory...
Definition region_allocator.h:24
Halide::Runtime::Internal::RegionAllocator::find_allocator
static RegionAllocator * find_allocator(void *user_context, MemoryRegion *memory_region)
Definition region_allocator.h:240
Halide::Runtime::Internal::RegionAllocator::retain
int retain(void *user_context, MemoryRegion *memory_region)
Definition region_allocator.h:232
Halide::Runtime::Internal::RegionAllocator::release
int release(void *user_context, MemoryRegion *memory_region)
Definition region_allocator.h:210
Halide::Runtime::Internal::RegionAllocator::block_resource
BlockResource * block_resource() const
Definition region_allocator.h:832
Halide::Runtime::Internal::StringTable::parse
size_t parse(void *user_context, const char *str, const char *delim)
Definition string_table.h:159
Halide::Runtime::Internal::StringTable::size
size_t size() const
Definition string_table.h:54
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator
Vulkan Memory Allocator class interface for managing large memory requests stored as contiguous block...
Definition vulkan_memory.h:42
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::reclaim
int reclaim(void *user_context, MemoryRegion *region)
Definition vulkan_memory.h:466
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::default_config
static const VulkanMemoryConfig & default_config()
Definition vulkan_memory.h:549
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::reserve
MemoryRegion * reserve(void *user_context, const MemoryRequest &request)
Definition vulkan_memory.h:227
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::VulkanMemoryAllocator
VulkanMemoryAllocator(const VulkanMemoryAllocator &)=delete
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::release
int release(void *user_context, MemoryRegion *region)
Definition vulkan_memory.h:447
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::allocate_block
static int allocate_block(void *instance_ptr, MemoryBlock *block)
Definition vulkan_memory.h:630
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::conform
int conform(void *user_context, MemoryRequest *request)
Definition vulkan_memory.h:983
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::deallocate_region
static int deallocate_region(void *instance_ptr, MemoryRegion *region)
Definition vulkan_memory.h:1207
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::operator=
VulkanMemoryAllocator & operator=(const VulkanMemoryAllocator &)=delete
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::destroy
static int destroy(void *user_context, VulkanMemoryAllocator *allocator)
Definition vulkan_memory.h:174
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::unmap
int unmap(void *user_context, MemoryRegion *region)
Definition vulkan_memory.h:316
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::conform_region_request
static int conform_region_request(void *instance_ptr, MemoryRequest *request)
Definition vulkan_memory.h:1043
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::~VulkanMemoryAllocator
~VulkanMemoryAllocator()=delete
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::create_crop
MemoryRegion * create_crop(void *user_context, MemoryRegion *region, uint64_t offset)
Definition vulkan_memory.h:355
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::destroy_crop
int destroy_crop(void *user_context, MemoryRegion *region)
Definition vulkan_memory.h:408
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::regions_allocated
size_t regions_allocated() const
Definition vulkan_memory.h:1277
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::blocks_allocated
size_t blocks_allocated() const
Definition vulkan_memory.h:844
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::allocate_region
static int allocate_region(void *instance_ptr, MemoryRegion *region)
Definition vulkan_memory.h:1075
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::conform_block_request
static int conform_block_request(void *instance_ptr, MemoryRequest *request)
Definition vulkan_memory.h:588
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::owner_of
MemoryRegion * owner_of(void *user_context, MemoryRegion *region)
Definition vulkan_memory.h:438
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::current_device
VkDevice current_device() const
Definition vulkan_memory.h:76
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::current_physical_device_limits
VkPhysicalDeviceLimits current_physical_device_limits() const
Definition vulkan_memory.h:82
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::collect
bool collect(void *user_context)
Definition vulkan_memory.h:504
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::bytes_allocated_for_blocks
size_t bytes_allocated_for_blocks() const
Definition vulkan_memory.h:848
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::callbacks
const VkAllocationCallbacks * callbacks() const
Definition vulkan_memory.h:85
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::deallocate_block
static int deallocate_block(void *instance_ptr, MemoryBlock *block)
Definition vulkan_memory.h:776
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::VulkanMemoryAllocator
VulkanMemoryAllocator()=delete
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::map
void * map(void *user_context, MemoryRegion *region)
Definition vulkan_memory.h:250
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::current_physical_device
VkPhysicalDevice current_physical_device() const
Definition vulkan_memory.h:79
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::bytes_allocated_for_regions
size_t bytes_allocated_for_regions() const
Definition vulkan_memory.h:1281
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::retain
int retain(void *user_context, MemoryRegion *region)
Definition vulkan_memory.h:485
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::create
static VulkanMemoryAllocator * create(void *user_context, const VulkanMemoryConfig &config, VkDevice dev, VkPhysicalDevice phys_dev, const SystemMemoryAllocatorFns &system_allocator, const VkAllocationCallbacks *alloc_callbacks=nullptr)
Definition vulkan_memory.h:152
memory_resources.h
halide_memory_caching_name
WEAK const char * halide_memory_caching_name(MemoryCaching value)
Definition memory_resources.h:293
halide_memory_usage_name
WEAK const char * halide_memory_usage_name(MemoryUsage value)
Definition memory_resources.h:260
halide_memory_visibility_name
WEAK const char * halide_memory_visibility_name(MemoryVisibility value)
Definition memory_resources.h:233
Halide::Runtime::Internal::Vulkan
Definition vulkan_context.h:17
Halide::Runtime::Internal::Vulkan::memory_allocator_config
WEAK VulkanMemoryConfig memory_allocator_config
Definition vulkan_memory.h:32
Halide::Runtime::Internal::Vulkan::custom_allocation_callbacks
WEAK const VkAllocationCallbacks * custom_allocation_callbacks
Definition vulkan_memory.h:20
Halide::Runtime::Internal::Vulkan::custom_allocation_callbacks_lock
WEAK ScopedSpinLock::AtomicFlag custom_allocation_callbacks_lock
Definition vulkan_memory.h:19
Halide::Runtime::Internal::Vulkan::alloc_config
WEAK char alloc_config[1024]
Definition vulkan_extensions.h:31
Halide::Runtime::Internal
Definition constants.h:13
Halide::Runtime::Internal::conform_alignment
ALWAYS_INLINE size_t conform_alignment(size_t requested, size_t required)
Definition memory_resources.h:135
Halide::Runtime::Internal::MemoryUsage::TransferSrcDst
@ TransferSrcDst
Definition memory_resources.h:42
Halide::Runtime::Internal::MemoryUsage::DefaultUsage
@ DefaultUsage
Definition memory_resources.h:43
Halide::Runtime::Internal::MemoryUsage::DynamicStorage
@ DynamicStorage
Definition memory_resources.h:38
Halide::Runtime::Internal::MemoryUsage::UniformStorage
@ UniformStorage
Definition memory_resources.h:39
Halide::Runtime::Internal::MemoryUsage::InvalidUsage
@ InvalidUsage
Definition memory_resources.h:36
Halide::Runtime::Internal::MemoryUsage::StaticStorage
@ StaticStorage
Definition memory_resources.h:37
Halide::Runtime::Internal::MemoryUsage::TransferSrc
@ TransferSrc
Definition memory_resources.h:40
Halide::Runtime::Internal::MemoryUsage::TransferDst
@ TransferDst
Definition memory_resources.h:41
Halide::Runtime::Internal::conform_size
ALWAYS_INLINE size_t conform_size(size_t offset, size_t size, size_t alignment, size_t nearest_multiple)
Definition memory_resources.h:151
Halide::Runtime::Internal::MemoryVisibility::DefaultVisibility
@ DefaultVisibility
Definition memory_resources.h:30
Halide::Runtime::Internal::MemoryVisibility::DeviceToHost
@ DeviceToHost
Definition memory_resources.h:28
Halide::Runtime::Internal::MemoryVisibility::InvalidVisibility
@ InvalidVisibility
Definition memory_resources.h:25
Halide::Runtime::Internal::MemoryVisibility::HostToDevice
@ HostToDevice
Definition memory_resources.h:29
Halide::Runtime::Internal::MemoryVisibility::HostOnly
@ HostOnly
Definition memory_resources.h:26
Halide::Runtime::Internal::MemoryVisibility::DeviceOnly
@ DeviceOnly
Definition memory_resources.h:27
Halide::Runtime::Internal::MemoryCaching::Cached
@ Cached
Definition memory_resources.h:49
Halide::Runtime::Internal::MemoryCaching::InvalidCaching
@ InvalidCaching
Definition memory_resources.h:48
Halide::Runtime::Internal::MemoryCaching::UncachedCoherent
@ UncachedCoherent
Definition memory_resources.h:52
Halide::Runtime::Internal::MemoryCaching::DefaultCaching
@ DefaultCaching
Definition memory_resources.h:53
Halide::Runtime::Internal::MemoryCaching::Uncached
@ Uncached
Definition memory_resources.h:50
Halide::Runtime::Internal::MemoryCaching::CachedCoherent
@ CachedCoherent
Definition memory_resources.h:51
Halide::Runtime::Internal::aligned_offset
ALWAYS_INLINE size_t aligned_offset(size_t offset, size_t alignment)
Definition memory_resources.h:128
Halide::Runtime
Definition constants.h:12
Halide
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
Definition AbstractGenerator.h:19
Halide::print
Expr print(const std::vector< Expr > &values)
Create an Expr that prints out its value whenever it is evaluated.
uint64_t
unsigned __INT64_TYPE__ uint64_t
Definition runtime_internal.h:23
int64_t
signed __INT64_TYPE__ int64_t
Definition runtime_internal.h:22
atoi
int atoi(const char *)
malloc
void * malloc(size_t)
memcpy
void * memcpy(void *s1, const void *s2, size_t n)
memset
void * memset(void *s, int val, size_t n)
uint32_t
unsigned __INT32_TYPE__ uint32_t
Definition runtime_internal.h:25
WEAK
#define WEAK
Definition runtime_internal.h:52
free
void free(void *)
Halide::Runtime::Internal::BlockAllocator::Config
Definition block_allocator.h:43
Halide::Runtime::Internal::BlockAllocator::Config::maximum_block_count
size_t maximum_block_count
Definition block_allocator.h:48
Halide::Runtime::Internal::BlockAllocator::Config::nearest_multiple
size_t nearest_multiple
Definition block_allocator.h:49
Halide::Runtime::Internal::BlockAllocator::Config::minimum_block_size
size_t minimum_block_size
Definition block_allocator.h:46
Halide::Runtime::Internal::BlockAllocator::Config::maximum_pool_size
size_t maximum_pool_size
Definition block_allocator.h:45
Halide::Runtime::Internal::BlockAllocator::Config::maximum_block_size
size_t maximum_block_size
Definition block_allocator.h:47
Halide::Runtime::Internal::BlockAllocator::MemoryAllocators
Definition block_allocator.h:36
Halide::Runtime::Internal::BlockAllocator::MemoryAllocators::region
MemoryRegionAllocatorFns region
Definition block_allocator.h:39
Halide::Runtime::Internal::BlockAllocator::MemoryAllocators::block
MemoryBlockAllocatorFns block
Definition block_allocator.h:38
Halide::Runtime::Internal::BlockAllocator::MemoryAllocators::system
SystemMemoryAllocatorFns system
Definition block_allocator.h:37
Halide::Runtime::Internal::BlockResource
Definition memory_resources.h:103
Halide::Runtime::Internal::BlockResource::memory
MemoryBlock memory
Definition memory_resources.h:104
Halide::Runtime::Internal::MemoryBlock
Definition memory_resources.h:65
Halide::Runtime::Internal::MemoryBlock::handle
void * handle
Definition memory_resources.h:66
Halide::Runtime::Internal::MemoryBlock::dedicated
bool dedicated
Definition memory_resources.h:68
Halide::Runtime::Internal::MemoryBlock::properties
MemoryProperties properties
Definition memory_resources.h:69
Halide::Runtime::Internal::MemoryBlock::size
size_t size
Definition memory_resources.h:67
Halide::Runtime::Internal::MemoryProperties
Definition memory_resources.h:56
Halide::Runtime::Internal::MemoryProperties::caching
MemoryCaching caching
Definition memory_resources.h:59
Halide::Runtime::Internal::MemoryProperties::alignment
size_t alignment
Definition memory_resources.h:60
Halide::Runtime::Internal::MemoryProperties::nearest_multiple
size_t nearest_multiple
Definition memory_resources.h:61
Halide::Runtime::Internal::MemoryProperties::visibility
MemoryVisibility visibility
Definition memory_resources.h:57
Halide::Runtime::Internal::MemoryProperties::usage
MemoryUsage usage
Definition memory_resources.h:58
Halide::Runtime::Internal::MemoryRange::head_offset
size_t head_offset
Definition memory_resources.h:74
Halide::Runtime::Internal::MemoryRange::tail_offset
size_t tail_offset
Definition memory_resources.h:75
Halide::Runtime::Internal::MemoryRegion
Definition memory_resources.h:79
Halide::Runtime::Internal::MemoryRegion::handle
void * handle
Definition memory_resources.h:80
Halide::Runtime::Internal::MemoryRegion::dedicated
bool dedicated
Definition memory_resources.h:84
Halide::Runtime::Internal::MemoryRegion::is_owner
bool is_owner
Definition memory_resources.h:85
Halide::Runtime::Internal::MemoryRegion::size
size_t size
Definition memory_resources.h:82
Halide::Runtime::Internal::MemoryRegion::range
MemoryRange range
Definition memory_resources.h:83
Halide::Runtime::Internal::MemoryRegion::properties
MemoryProperties properties
Definition memory_resources.h:86
Halide::Runtime::Internal::MemoryRegion::offset
size_t offset
Definition memory_resources.h:81
Halide::Runtime::Internal::MemoryRequest
Definition memory_resources.h:90
Halide::Runtime::Internal::MemoryRequest::size
size_t size
Definition memory_resources.h:92
Halide::Runtime::Internal::MemoryRequest::offset
size_t offset
Definition memory_resources.h:91
Halide::Runtime::Internal::MemoryRequest::dedicated
bool dedicated
Definition memory_resources.h:94
Halide::Runtime::Internal::MemoryRequest::alignment
size_t alignment
Definition memory_resources.h:93
Halide::Runtime::Internal::MemoryRequest::properties
MemoryProperties properties
Definition memory_resources.h:95
Halide::Runtime::Internal::ScopedSpinLock
Definition scoped_spin_lock.h:9
Halide::Runtime::Internal::ScopedSpinLock::AtomicFlag
char AtomicFlag
Definition scoped_spin_lock.h:11
Halide::Runtime::Internal::StringUtils::is_empty
static bool is_empty(const char *str)
Definition string_storage.h:13
Halide::Runtime::Internal::SystemMemoryAllocatorFns
Definition memory_resources.h:193
Halide::Runtime::Internal::SystemMemoryAllocatorFns::deallocate
DeallocateSystemFn deallocate
Definition memory_resources.h:195
Halide::Runtime::Internal::SystemMemoryAllocatorFns::allocate
AllocateSystemFn allocate
Definition memory_resources.h:194
Halide::Runtime::Internal::Vulkan::VulkanMemoryConfig
Definition vulkan_memory.h:25
Halide::Runtime::Internal::Vulkan::VulkanMemoryConfig::maximum_block_size
size_t maximum_block_size
Definition vulkan_memory.h:28
Halide::Runtime::Internal::Vulkan::VulkanMemoryConfig::nearest_multiple
size_t nearest_multiple
Definition vulkan_memory.h:30
Halide::Runtime::Internal::Vulkan::VulkanMemoryConfig::maximum_pool_size
size_t maximum_pool_size
Definition vulkan_memory.h:26
Halide::Runtime::Internal::Vulkan::VulkanMemoryConfig::minimum_block_size
size_t minimum_block_size
Definition vulkan_memory.h:27
Halide::Runtime::Internal::Vulkan::VulkanMemoryConfig::maximum_block_count
size_t maximum_block_count
Definition vulkan_memory.h:29
HL_VK_ENV_DELIM
#define HL_VK_ENV_DELIM
Definition vulkan_interface.h:14
vulkan_internal.h
halide_vulkan_set_allocation_callbacks
WEAK void halide_vulkan_set_allocation_callbacks(const VkAllocationCallbacks *callbacks)
Definition vulkan_memory.h:1495
halide_vulkan_get_allocation_callbacks
WEAK const VkAllocationCallbacks * halide_vulkan_get_allocation_callbacks(void *user_context)
Definition vulkan_memory.h:1501