Loading...
Searching...
No Matches
HalideBuffer.h
Go to the documentation of this file.
/** \file
* Defines a Buffer type that wraps from halide_buffer_t and adds
* functionality, and methods for more conveniently iterating over the
* samples in a halide_buffer_t outside of Halide code. */
#ifndef HALIDE_RUNTIME_BUFFER_H
#define HALIDE_RUNTIME_BUFFER_H
#include <algorithm>
#include <atomic>
#include <cassert>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <limits>
#include <memory>
#include <type_traits>
#include <vector>
#ifdef __APPLE__
#include <AvailabilityVersions.h>
#include <TargetConditionals.h>
#endif
#if defined(__has_feature)
#if __has_feature(memory_sanitizer)
#include <sanitizer/msan_interface.h>
#endif
#endif
#include "HalideRuntime.h"
#ifdef _MSC_VER
#include <malloc.h>
#define HALIDE_ALLOCA _alloca
#else
#define HALIDE_ALLOCA __builtin_alloca
#endif
// gcc 5.1 has a false positive warning on this code
#if __GNUC__ == 5 && __GNUC_MINOR__ == 1
#pragma GCC diagnostic ignored "-Warray-bounds"
#endif
#ifndef HALIDE_RUNTIME_BUFFER_CHECK_INDICES
#define HALIDE_RUNTIME_BUFFER_CHECK_INDICES 0
#endif
#ifndef HALIDE_RUNTIME_BUFFER_ALLOCATION_ALIGNMENT
// Conservatively align buffer allocations to 128 bytes by default.
// This is enough alignment for all the platforms currently in use.
// Redefine this in your compiler settings if you desire more/less alignment.
#define HALIDE_RUNTIME_BUFFER_ALLOCATION_ALIGNMENT 128
#endif
static_assert(((HALIDE_RUNTIME_BUFFER_ALLOCATION_ALIGNMENT & (HALIDE_RUNTIME_BUFFER_ALLOCATION_ALIGNMENT - 1)) == 0),
"HALIDE_RUNTIME_BUFFER_ALLOCATION_ALIGNMENT must be a power of 2.");
// Unfortunately, not all C++17 runtimes support aligned_alloc
// (it may depends on OS/SDK version); this is provided as an opt-out
// if you are compiling on a platform that doesn't provide a (good)
// implementation. (Note that we actually use the C11 `::aligned_alloc()`
// rather than the C++17 `std::aligned_alloc()` because at least one platform
// we found supports the former but not the latter.)
#ifndef HALIDE_RUNTIME_BUFFER_USE_ALIGNED_ALLOC
// clang-format off
#ifdef _WIN32
// Windows (regardless of which compiler) doesn't implement aligned_alloc(),
// even in C++17 mode, and has stated they probably never will, as the issue
// is in the incompatibility that free() needs to be able to free both pointers
// returned by malloc() and aligned_alloc(). So, always default it off here.
#define HALIDE_RUNTIME_BUFFER_USE_ALIGNED_ALLOC 0
#elif defined(__ANDROID_API__) && __ANDROID_API__ < 28
// Android doesn't provide aligned_alloc until API 28
#define HALIDE_RUNTIME_BUFFER_USE_ALIGNED_ALLOC 0
#elif defined(__APPLE__)
#if TARGET_OS_OSX && (__MAC_OS_X_VERSION_MIN_REQUIRED < __MAC_10_15)
// macOS doesn't provide aligned_alloc until 10.15
#define HALIDE_RUNTIME_BUFFER_USE_ALIGNED_ALLOC 0
#elif TARGET_OS_IPHONE && (__IPHONE_OS_VERSION_MIN_REQUIRED < __IPHONE_14_0)
// iOS doesn't provide aligned_alloc until 14.0
#define HALIDE_RUNTIME_BUFFER_USE_ALIGNED_ALLOC 0
#else
// Assume it's ok on all other Apple targets
#define HALIDE_RUNTIME_BUFFER_USE_ALIGNED_ALLOC 1
#endif
#else
#if defined(__GLIBCXX__) && !defined(_GLIBCXX_HAVE_ALIGNED_ALLOC)
// ARM GNU-A baremetal compiler doesn't provide aligned_alloc as of 12.2
#define HALIDE_RUNTIME_BUFFER_USE_ALIGNED_ALLOC 0
#else
// Not Windows, Android, or Apple: just assume it's ok
#define HALIDE_RUNTIME_BUFFER_USE_ALIGNED_ALLOC 1
#endif
#endif
// clang-format on
#endif // HALIDE_RUNTIME_BUFFER_USE_ALIGNED_ALLOC
namespace Halide {
namespace Runtime {
// Forward-declare our Buffer class
template<typename T, int Dims, int InClassDimStorage>
class Buffer;
// A helper to check if a parameter pack is entirely implicitly
// int-convertible to use with std::enable_if
template<typename... Args>
struct AllInts : std::false_type {};
template<>
struct AllInts<> : std::true_type {};
template<typename T, typename... Args>
struct AllInts<T, Args...> {
static const bool value = std::is_convertible<T, int>::value && AllInts<Args...>::value;
};
// Floats and doubles are technically implicitly int-convertible, but
// doing so produces a warning we treat as an error, so just disallow
// it here.
template<typename... Args>
struct AllInts<float, Args...> : std::false_type {};
template<typename... Args>
struct AllInts<double, Args...> : std::false_type {};
namespace Internal {
// A helper to detect if there are any zeros in a container
template<typename Container>
bool any_zero(const Container &c) {
for (int i : c) {
if (i == 0) {
return true;
}
}
return false;
}
struct DefaultAllocatorFns {
static inline void *(*default_allocate_fn)(size_t) = nullptr;
static inline void (*default_deallocate_fn)(void *) = nullptr;
};
} // namespace Internal
/** A struct acting as a header for allocations owned by the Buffer
* class itself. */
struct AllocationHeader {
void (*deallocate_fn)(void *);
std::atomic<int> ref_count;
// Note that ref_count always starts at 1
explicit AllocationHeader(void (*deallocate_fn)(void *))
: deallocate_fn(deallocate_fn), ref_count(1) {
}
};
/** This indicates how to deallocate the device for a Halide::Runtime::Buffer. */
enum struct BufferDeviceOwnership : int {
Allocated, ///> halide_device_free will be called when device ref count goes to zero
WrappedNative, ///> halide_device_detach_native will be called when device ref count goes to zero
Unmanaged, ///> No free routine will be called when device ref count goes to zero
AllocatedDeviceAndHost, ///> Call device_and_host_free when DevRefCount goes to zero.
Cropped, ///> Call halide_device_release_crop when DevRefCount goes to zero.
};
/** A similar struct for managing device allocations. */
struct DeviceRefCount {
// This is only ever constructed when there's something to manage,
// so start at one.
std::atomic<int> count{1};
BufferDeviceOwnership ownership{BufferDeviceOwnership::Allocated};
};
constexpr int AnyDims = -1;
/** A templated Buffer class that wraps halide_buffer_t and adds
* functionality. When using Halide from C++, this is the preferred
* way to create input and output buffers. The overhead of using this
* class relative to a naked halide_buffer_t is minimal - it uses another
* ~16 bytes on the stack, and does no dynamic allocations when using
* it to represent existing memory of a known maximum dimensionality.
*
* The template parameter T is the element type. For buffers where the
* element type is unknown, or may vary, use void or const void.
*
* The template parameter Dims is the number of dimensions. For buffers where
* the dimensionality type is unknown at, or may vary, use AnyDims.
*
* InClassDimStorage is the maximum number of dimensions that can be represented
* using space inside the class itself. Set it to the maximum dimensionality
* you expect this buffer to be. If the actual dimensionality exceeds
* this, heap storage is allocated to track the shape of the buffer.
* InClassDimStorage defaults to 4, which should cover nearly all usage.
*
* The class optionally allocates and owns memory for the image using
* a shared pointer allocated with the provided allocator. If they are
* null, malloc and free are used. Any device-side allocation is
* considered as owned if and only if the host-side allocation is
* owned. */
template<typename T = void,
int Dims = AnyDims,
int InClassDimStorage = (Dims == AnyDims ? 4 : std::max(Dims, 1))>
class Buffer {
/** The underlying halide_buffer_t */
halide_buffer_t buf = {};
/** Some in-class storage for shape of the dimensions. */
halide_dimension_t shape[InClassDimStorage];
/** The allocation owned by this Buffer. NULL if the Buffer does not
* own the memory. */
AllocationHeader *alloc = nullptr;
/** A reference count for the device allocation owned by this
* buffer. */
mutable DeviceRefCount *dev_ref_count = nullptr;
/** True if T is of type void or const void */
static const bool T_is_void = std::is_same<typename std::remove_const<T>::type, void>::value;
/** A type function that adds a const qualifier if T is a const type. */
template<typename T2>
using add_const_if_T_is_const = typename std::conditional<std::is_const<T>::value, const T2, T2>::type;
/** T unless T is (const) void, in which case (const)
* uint8_t. Useful for providing return types for operator() */
using not_void_T = typename std::conditional<T_is_void,
add_const_if_T_is_const<uint8_t>,
T>::type;
/** T with constness removed. Useful for return type of copy(). */
using not_const_T = typename std::remove_const<T>::type;
/** The type the elements are stored as. Equal to not_void_T
* unless T is a pointer, in which case uint64_t. Halide stores
* all pointer types as uint64s internally, even on 32-bit
* systems. */
using storage_T = typename std::conditional<std::is_pointer<T>::value, uint64_t, not_void_T>::type;
public:
/** True if the Halide type is not void (or const void). */
static constexpr bool has_static_halide_type = !T_is_void;
/** Get the Halide type of T. Callers should not use the result if
* has_static_halide_type is false. */
static constexpr halide_type_t static_halide_type() {
return halide_type_of<typename std::remove_cv<not_void_T>::type>();
}
/** Does this Buffer own the host memory it refers to? */
bool owns_host_memory() const {
return alloc != nullptr;
}
static constexpr bool has_static_dimensions = (Dims != AnyDims);
/** Callers should not use the result if
* has_static_dimensions is false. */
static constexpr int static_dimensions() {
return Dims;
}
static_assert(!has_static_dimensions || static_dimensions() >= 0);
private:
/** Increment the reference count of any owned allocation */
void incref() const {
if (owns_host_memory()) {
alloc->ref_count++;
}
if (buf.device) {
if (!dev_ref_count) {
// I seem to have a non-zero dev field but no
// reference count for it. I must have been given a
// device allocation by a Halide pipeline, and have
// never been copied from since. Take sole ownership
// of it.
dev_ref_count = new DeviceRefCount;
}
dev_ref_count->count++;
}
}
// Note that this is called "cropped" but can also encompass a slice/embed
// operation as well.
struct DevRefCountCropped : DeviceRefCount {
// We will only store Buffers that have a dynamic number of dimensions.
// Buffers that cropped or sliced from need to be first converted to
// one with variable size. This is required because we cannot possibly
// know what the actual dimensionality is of the buffer this is a
// crop or slice from. Since cropping a sliced buffer is also possible,
// no optimizations can be made for cropped buffers either.
Buffer<T, AnyDims> cropped_from;
explicit DevRefCountCropped(const Buffer<T, AnyDims> &cropped_from)
: cropped_from(cropped_from) {
ownership = BufferDeviceOwnership::Cropped;
}
};
/** Setup the device ref count for a buffer to indicate it is a crop (or slice, embed, etc) of cropped_from */
void crop_from(const Buffer<T, AnyDims> &cropped_from) {
assert(dev_ref_count == nullptr);
dev_ref_count = new DevRefCountCropped(cropped_from);
}
/** Decrement the reference count of any owned allocation and free host
* and device memory if it hits zero. Sets alloc to nullptr. */
void decref(bool device_only = false) {
if (owns_host_memory() && !device_only) {
int new_count = --(alloc->ref_count);
if (new_count == 0) {
void (*fn)(void *) = alloc->deallocate_fn;
alloc->~AllocationHeader();
fn(alloc);
}
buf.host = nullptr;
alloc = nullptr;
set_host_dirty(false);
}
int new_count = 0;
if (dev_ref_count) {
new_count = --(dev_ref_count->count);
}
if (new_count == 0) {
if (buf.device) {
assert(!(alloc && device_dirty()) &&
"Implicitly freeing a dirty device allocation while a host allocation still lives. "
"Call device_free explicitly if you want to drop dirty device-side data. "
"Call copy_to_host explicitly if you want the data copied to the host allocation "
"before the device allocation is freed.");
int result = halide_error_code_success;
if (dev_ref_count && dev_ref_count->ownership == BufferDeviceOwnership::WrappedNative) {
result = buf.device_interface->detach_native(nullptr, &buf);
} else if (dev_ref_count && dev_ref_count->ownership == BufferDeviceOwnership::AllocatedDeviceAndHost) {
result = buf.device_interface->device_and_host_free(nullptr, &buf);
} else if (dev_ref_count && dev_ref_count->ownership == BufferDeviceOwnership::Cropped) {
result = buf.device_interface->device_release_crop(nullptr, &buf);
} else if (dev_ref_count == nullptr || dev_ref_count->ownership == BufferDeviceOwnership::Allocated) {
result = buf.device_interface->device_free(nullptr, &buf);
}
// No reasonable way to return the error, but we can at least assert-fail in debug builds.
assert((result == halide_error_code_success) && "device_interface call returned a nonzero result in Buffer::decref()");
(void)result;
}
if (dev_ref_count) {
if (dev_ref_count->ownership == BufferDeviceOwnership::Cropped) {
delete (DevRefCountCropped *)dev_ref_count;
} else {
delete dev_ref_count;
}
}
}
dev_ref_count = nullptr;
buf.device = 0;
buf.device_interface = nullptr;
}
void free_shape_storage() {
if (buf.dim != shape) {
delete[] buf.dim;
buf.dim = nullptr;
}
}
template<int DimsSpecified>
void make_static_shape_storage() {
static_assert(Dims == AnyDims || Dims == DimsSpecified,
"Number of arguments to Buffer() does not match static dimensionality");
buf.dimensions = DimsSpecified;
if constexpr (Dims == AnyDims) {
if constexpr (DimsSpecified <= InClassDimStorage) {
buf.dim = shape;
} else {
static_assert(DimsSpecified >= 1);
buf.dim = new halide_dimension_t[DimsSpecified];
}
} else {
static_assert(InClassDimStorage >= Dims);
buf.dim = shape;
}
}
void make_shape_storage(const int dimensions) {
if (Dims != AnyDims && Dims != dimensions) {
assert(false && "Number of arguments to Buffer() does not match static dimensionality");
}
// This should usually be inlined, so if dimensions is statically known,
// we can skip the call to new
buf.dimensions = dimensions;
buf.dim = (dimensions <= InClassDimStorage) ? shape : new halide_dimension_t[dimensions];
}
void copy_shape_from(const halide_buffer_t &other) {
// All callers of this ensure that buf.dimensions == other.dimensions.
make_shape_storage(other.dimensions);
std::copy(other.dim, other.dim + other.dimensions, buf.dim);
}
template<typename T2, int D2, int S2>
void move_shape_from(Buffer<T2, D2, S2> &&other) {
if (other.shape == other.buf.dim) {
copy_shape_from(other.buf);
} else {
buf.dim = other.buf.dim;
other.buf.dim = nullptr;
}
other.buf = halide_buffer_t();
}
/** Initialize the shape from a halide_buffer_t. */
void initialize_from_buffer(const halide_buffer_t &b,
BufferDeviceOwnership ownership) {
memcpy(&buf, &b, sizeof(halide_buffer_t));
copy_shape_from(b);
if (b.device) {
dev_ref_count = new DeviceRefCount;
dev_ref_count->ownership = ownership;
}
}
/** Initialize the shape from an array of ints */
void initialize_shape(const int *sizes) {
for (int i = 0; i < buf.dimensions; i++) {
buf.dim[i].min = 0;
buf.dim[i].extent = sizes[i];
if (i == 0) {
buf.dim[i].stride = 1;
} else {
buf.dim[i].stride = buf.dim[i - 1].stride * buf.dim[i - 1].extent;
}
}
}
/** Initialize the shape from a vector of extents */
void initialize_shape(const std::vector<int> &sizes) {
assert(buf.dimensions == (int)sizes.size());
initialize_shape(sizes.data());
}
/** Initialize the shape from the static shape of an array */
template<typename Array, size_t N>
void initialize_shape_from_array_shape(int next, Array (&vals)[N]) {
buf.dim[next].min = 0;
buf.dim[next].extent = (int)N;
if (next == 0) {
buf.dim[next].stride = 1;
} else {
initialize_shape_from_array_shape(next - 1, vals[0]);
buf.dim[next].stride = buf.dim[next - 1].stride * buf.dim[next - 1].extent;
}
}
/** Base case for the template recursion above. */
template<typename T2>
void initialize_shape_from_array_shape(int, const T2 &) {
}
/** Get the dimensionality of a multi-dimensional C array */
template<typename Array, size_t N>
static int dimensionality_of_array(Array (&vals)[N]) {
return dimensionality_of_array(vals[0]) + 1;
}
template<typename T2>
static int dimensionality_of_array(const T2 &) {
return 0;
}
/** Get the underlying halide_type_t of an array's element type. */
template<typename Array, size_t N>
static halide_type_t scalar_type_of_array(Array (&vals)[N]) {
return scalar_type_of_array(vals[0]);
}
template<typename T2>
static halide_type_t scalar_type_of_array(const T2 &) {
return halide_type_of<typename std::remove_cv<T2>::type>();
}
/** Crop a single dimension without handling device allocation. */
void crop_host(int d, int min, int extent) {
assert(dim(d).min() <= min);
assert(dim(d).max() >= min + extent - 1);
ptrdiff_t shift = min - dim(d).min();
if (buf.host != nullptr) {
buf.host += (shift * dim(d).stride()) * type().bytes();
}
buf.dim[d].min = min;
buf.dim[d].extent = extent;
}
/** Crop as many dimensions as are in rect, without handling device allocation. */
void crop_host(const std::vector<std::pair<int, int>> &rect) {
assert(rect.size() <= static_cast<decltype(rect.size())>(std::numeric_limits<int>::max()));
int limit = (int)rect.size();
assert(limit <= dimensions());
for (int i = 0; i < limit; i++) {
crop_host(i, rect[i].first, rect[i].second);
}
}
void complete_device_crop(Buffer<T, Dims, InClassDimStorage> &result_host_cropped) const {
assert(buf.device_interface != nullptr);
if (buf.device_interface->device_crop(nullptr, &this->buf, &result_host_cropped.buf) == halide_error_code_success) {
// TODO: Figure out what to do if dev_ref_count is nullptr. Should incref logic run here?
// is it possible to get to this point without incref having run at least once since
// the device field was set? (I.e. in the internal logic of crop. incref might have been
// called.)
if (dev_ref_count != nullptr && dev_ref_count->ownership == BufferDeviceOwnership::Cropped) {
result_host_cropped.crop_from(((DevRefCountCropped *)dev_ref_count)->cropped_from);
} else {
result_host_cropped.crop_from(*this);
}
}
}
/** slice a single dimension without handling device allocation. */
void slice_host(int d, int pos) {
static_assert(Dims == AnyDims);
assert(dimensions() > 0);
assert(d >= 0 && d < dimensions());
assert(pos >= dim(d).min() && pos <= dim(d).max());
buf.dimensions--;
ptrdiff_t shift = pos - buf.dim[d].min;
if (buf.host != nullptr) {
buf.host += (shift * buf.dim[d].stride) * type().bytes();
}
for (int i = d; i < buf.dimensions; i++) {
buf.dim[i] = buf.dim[i + 1];
}
buf.dim[buf.dimensions] = {0, 0, 0};
}
void complete_device_slice(Buffer<T, AnyDims, InClassDimStorage> &result_host_sliced, int d, int pos) const {
assert(buf.device_interface != nullptr);
if (buf.device_interface->device_slice(nullptr, &this->buf, d, pos, &result_host_sliced.buf) == halide_error_code_success) {
// TODO: Figure out what to do if dev_ref_count is nullptr. Should incref logic run here?
// is it possible to get to this point without incref having run at least once since
// the device field was set? (I.e. in the internal logic of slice. incref might have been
// called.)
if (dev_ref_count != nullptr && dev_ref_count->ownership == BufferDeviceOwnership::Cropped) {
// crop_from() is correct here, despite the fact that we are slicing.
result_host_sliced.crop_from(((DevRefCountCropped *)dev_ref_count)->cropped_from);
} else {
// crop_from() is correct here, despite the fact that we are slicing.
result_host_sliced.crop_from(*this);
}
}
}
public:
typedef T ElemType;
/** Read-only access to the shape */
class Dimension {
const halide_dimension_t &d;
public:
/** The lowest coordinate in this dimension */
HALIDE_ALWAYS_INLINE int min() const {
return d.min;
}
/** The number of elements in memory you have to step over to
* increment this coordinate by one. */
HALIDE_ALWAYS_INLINE int stride() const {
return d.stride;
}
/** The extent of the image along this dimension */
HALIDE_ALWAYS_INLINE int extent() const {
return d.extent;
}
/** The highest coordinate in this dimension */
HALIDE_ALWAYS_INLINE int max() const {
return min() + extent() - 1;
}
/** An iterator class, so that you can iterate over
* coordinates in a dimensions using a range-based for loop. */
struct iterator {
int val;
int operator*() const {
return val;
}
bool operator!=(const iterator &other) const {
return val != other.val;
}
iterator &operator++() {
val++;
return *this;
}
};
/** An iterator that points to the min coordinate */
HALIDE_ALWAYS_INLINE iterator begin() const {
return {min()};
}
/** An iterator that points to one past the max coordinate */
HALIDE_ALWAYS_INLINE iterator end() const {
return {min() + extent()};
}
explicit Dimension(const halide_dimension_t &dim)
: d(dim) {
}
};
/** Access the shape of the buffer */
HALIDE_ALWAYS_INLINE Dimension dim(int i) const {
assert(i >= 0 && i < this->dimensions());
return Dimension(buf.dim[i]);
}
/** Access to the mins, strides, extents. Will be deprecated. Do not use. */
// @{
int min(int i) const {
return dim(i).min();
}
int extent(int i) const {
return dim(i).extent();
}
int stride(int i) const {
return dim(i).stride();
}
// @}
/** The total number of elements this buffer represents. Equal to
* the product of the extents */
size_t number_of_elements() const {
return buf.number_of_elements();
}
/** Get the dimensionality of the buffer. */
int dimensions() const {
if constexpr (has_static_dimensions) {
return Dims;
} else {
return buf.dimensions;
}
}
/** Get the type of the elements. */
halide_type_t type() const {
return buf.type;
}
/** A pointer to the element with the lowest address. If all
* strides are positive, equal to the host pointer. */
T *begin() const {
assert(buf.host != nullptr); // Cannot call begin() on an unallocated Buffer.
return (T *)buf.begin();
}
/** A pointer to one beyond the element with the highest address. */
T *end() const {
assert(buf.host != nullptr); // Cannot call end() on an unallocated Buffer.
return (T *)buf.end();
}
/** The total number of bytes spanned by the data in memory. */
size_t size_in_bytes() const {
return buf.size_in_bytes();
}
/** Reset the Buffer to be equivalent to a default-constructed Buffer
* of the same static type (if any); Buffer<void> will have its runtime
* type reset to uint8. */
void reset() {
*this = Buffer();
}
Buffer()
: shape() {
buf.type = static_halide_type();
// If Dims are statically known, must create storage that many.
// otherwise, make a zero-dimensional buffer.
constexpr int buf_dimensions = (Dims == AnyDims) ? 0 : Dims;
make_static_shape_storage<buf_dimensions>();
}
/** Make a Buffer from a halide_buffer_t */
explicit Buffer(const halide_buffer_t &buf,
BufferDeviceOwnership ownership = BufferDeviceOwnership::Unmanaged) {
assert(T_is_void || buf.type == static_halide_type());
initialize_from_buffer(buf, ownership);
}
/** Give Buffers access to the members of Buffers of different dimensionalities and types. */
template<typename T2, int D2, int S2>
friend class Buffer;
private:
template<typename T2, int D2, int S2>
static void static_assert_can_convert_from() {
static_assert((!std::is_const<T2>::value || std::is_const<T>::value),
"Can't convert from a Buffer<const T> to a Buffer<T>");
static_assert(std::is_same<typename std::remove_const<T>::type,
typename std::remove_const<T2>::type>::value ||
T_is_void || Buffer<T2, D2, S2>::T_is_void,
"type mismatch constructing Buffer");
static_assert(Dims == AnyDims || D2 == AnyDims || Dims == D2,
"Can't convert from a Buffer with static dimensionality to a Buffer with different static dimensionality");
}
public:
static void set_default_allocate_fn(void *(*allocate_fn)(size_t)) {
Internal::DefaultAllocatorFns::default_allocate_fn = allocate_fn;
}
static void set_default_deallocate_fn(void (*deallocate_fn)(void *)) {
Internal::DefaultAllocatorFns::default_deallocate_fn = deallocate_fn;
}
/** Determine if a Buffer<T, Dims, InClassDimStorage> can be constructed from some other Buffer type.
* If this can be determined at compile time, fail with a static assert; otherwise
* return a boolean based on runtime typing. */
template<typename T2, int D2, int S2>
static bool can_convert_from(const Buffer<T2, D2, S2> &other) {
static_assert_can_convert_from<T2, D2, S2>();
if (Buffer<T2, D2, S2>::T_is_void && !T_is_void) {
if (other.type() != static_halide_type()) {
return false;
}
}
if (Dims != AnyDims) {
if (other.dimensions() != Dims) {
return false;
}
}
return true;
}
/** Fail an assertion at runtime or compile-time if an Buffer<T, Dims, InClassDimStorage>
* cannot be constructed from some other Buffer type. */
template<typename T2, int D2, int S2>
static void assert_can_convert_from(const Buffer<T2, D2, S2> &other) {
// Explicitly call static_assert_can_convert_from() here so
// that we always get compile-time checking, even if compiling with
// assertions disabled.
static_assert_can_convert_from<T2, D2, S2>();
assert(can_convert_from(other));
}
/** Copy constructor. Does not copy underlying data. */
Buffer(const Buffer<T, Dims, InClassDimStorage> &other)
: buf(other.buf),
alloc(other.alloc) {
other.incref();
dev_ref_count = other.dev_ref_count;
copy_shape_from(other.buf);
}
/** Construct a Buffer from a Buffer of different dimensionality
* and type. Asserts that the type and dimensionality matches (at runtime,
* if one of the types is void). Note that this constructor is
* implicit. This, for example, lets you pass things like
* Buffer<T> or Buffer<const void> to functions expected
* Buffer<const T>. */
template<typename T2, int D2, int S2>
Buffer(const Buffer<T2, D2, S2> &other)
: buf(other.buf),
alloc(other.alloc) {
assert_can_convert_from(other);
other.incref();
dev_ref_count = other.dev_ref_count;
copy_shape_from(other.buf);
}
/** Move constructor */
Buffer(Buffer<T, Dims, InClassDimStorage> &&other) noexcept
: buf(other.buf),
alloc(other.alloc),
dev_ref_count(other.dev_ref_count) {
other.dev_ref_count = nullptr;
other.alloc = nullptr;
move_shape_from(std::forward<Buffer<T, Dims, InClassDimStorage>>(other));
}
/** Move-construct a Buffer from a Buffer of different
* dimensionality and type. Asserts that the types match (at
* runtime if one of the types is void). */
template<typename T2, int D2, int S2>
Buffer(Buffer<T2, D2, S2> &&other)
: buf(other.buf),
alloc(other.alloc),
dev_ref_count(other.dev_ref_count) {
assert_can_convert_from(other);
other.dev_ref_count = nullptr;
other.alloc = nullptr;
move_shape_from(std::forward<Buffer<T2, D2, S2>>(other));
}
/** Assign from another Buffer of possibly-different
* dimensionality and type. Asserts that the types match (at
* runtime if one of the types is void). */
template<typename T2, int D2, int S2>
Buffer<T, Dims, InClassDimStorage> &operator=(const Buffer<T2, D2, S2> &other) {
if ((const void *)this == (const void *)&other) {
return *this;
}
assert_can_convert_from(other);
other.incref();
decref();
dev_ref_count = other.dev_ref_count;
alloc = other.alloc;
free_shape_storage();
buf = other.buf;
copy_shape_from(other.buf);
return *this;
}
/** Standard assignment operator */
Buffer<T, Dims, InClassDimStorage> &operator=(const Buffer<T, Dims, InClassDimStorage> &other) {
// The cast to void* here is just to satisfy clang-tidy
if ((const void *)this == (const void *)&other) {
return *this;
}
other.incref();
decref();
dev_ref_count = other.dev_ref_count;
alloc = other.alloc;
free_shape_storage();
buf = other.buf;
copy_shape_from(other.buf);
return *this;
}
/** Move from another Buffer of possibly-different
* dimensionality and type. Asserts that the types match (at
* runtime if one of the types is void). */
template<typename T2, int D2, int S2>
Buffer<T, Dims, InClassDimStorage> &operator=(Buffer<T2, D2, S2> &&other) {
assert_can_convert_from(other);
decref();
alloc = other.alloc;
other.alloc = nullptr;
dev_ref_count = other.dev_ref_count;
other.dev_ref_count = nullptr;
free_shape_storage();
buf = other.buf;
move_shape_from(std::forward<Buffer<T2, D2, S2>>(other));
return *this;
}
/** Standard move-assignment operator */
Buffer<T, Dims, InClassDimStorage> &operator=(Buffer<T, Dims, InClassDimStorage> &&other) noexcept {
decref();
alloc = other.alloc;
other.alloc = nullptr;
dev_ref_count = other.dev_ref_count;
other.dev_ref_count = nullptr;
free_shape_storage();
buf = other.buf;
move_shape_from(std::forward<Buffer<T, Dims, InClassDimStorage>>(other));
return *this;
}
/** Check the product of the extents fits in memory. */
void check_overflow() {
size_t size = type().bytes();
for (int i = 0; i < dimensions(); i++) {
size *= dim(i).extent();
}
// We allow 2^31 or 2^63 bytes, so drop the top bit.
size = (size << 1) >> 1;
for (int i = 0; i < dimensions(); i++) {
size /= dim(i).extent();
}
assert(size == (size_t)type().bytes() && "Error: Overflow computing total size of buffer.");
}
/** Allocate memory for this Buffer. Drops the reference to any
* owned memory. */
void allocate(void *(*allocate_fn)(size_t) = nullptr,
void (*deallocate_fn)(void *) = nullptr) {
// Drop any existing allocation
deallocate();
// Conservatively align images to (usually) 128 bytes. This is enough
// alignment for all the platforms we might use. Also ensure that the allocation
// is such that the logical size is an integral multiple of 128 bytes (or a bit more).
constexpr size_t alignment = HALIDE_RUNTIME_BUFFER_ALLOCATION_ALIGNMENT;
const auto align_up = [=](size_t value) -> size_t {
return (value + alignment - 1) & ~(alignment - 1);
};
size_t size = size_in_bytes();
#if HALIDE_RUNTIME_BUFFER_USE_ALIGNED_ALLOC
// Only use aligned_alloc() if no custom allocators are specified.
if (!allocate_fn && !deallocate_fn && !Internal::DefaultAllocatorFns::default_allocate_fn && !Internal::DefaultAllocatorFns::default_deallocate_fn) {
// As a practical matter, sizeof(AllocationHeader) is going to be no more than 16 bytes
// on any supported platform, so we will just overallocate by 'alignment'
// so that the user storage also starts at an aligned point. This is a bit
// wasteful, but probably not a big deal.
static_assert(sizeof(AllocationHeader) <= alignment);
void *alloc_storage = ::aligned_alloc(alignment, align_up(size) + alignment);
assert((uintptr_t)alloc_storage == align_up((uintptr_t)alloc_storage));
alloc = new (alloc_storage) AllocationHeader(free);
buf.host = (uint8_t *)((uintptr_t)alloc_storage + alignment);
return;
}
// else fall thru
#endif
if (!allocate_fn) {
allocate_fn = Internal::DefaultAllocatorFns::default_allocate_fn;
if (!allocate_fn) {
allocate_fn = malloc;
}
}
if (!deallocate_fn) {
deallocate_fn = Internal::DefaultAllocatorFns::default_deallocate_fn;
if (!deallocate_fn) {
deallocate_fn = free;
}
}
static_assert(sizeof(AllocationHeader) <= alignment);
// malloc() and friends must return a pointer aligned to at least alignof(std::max_align_t);
// make sure this is OK for AllocationHeader, since it always goes at the start
static_assert(alignof(AllocationHeader) <= alignof(std::max_align_t));
const size_t requested_size = align_up(size + alignment +
std::max(0, (int)sizeof(AllocationHeader) -
(int)sizeof(std::max_align_t)));
void *alloc_storage = allocate_fn(requested_size);
alloc = new (alloc_storage) AllocationHeader(deallocate_fn);
uint8_t *unaligned_ptr = ((uint8_t *)alloc) + sizeof(AllocationHeader);
buf.host = (uint8_t *)align_up((uintptr_t)unaligned_ptr);
}
/** Drop reference to any owned host or device memory, possibly
* freeing it, if this buffer held the last reference to
* it. Retains the shape of the buffer. Does nothing if this
* buffer did not allocate its own memory. */
void deallocate() {
decref();
}
/** Drop reference to any owned device memory, possibly freeing it
* if this buffer held the last reference to it. Asserts that
* device_dirty is false. */
void device_deallocate() {
decref(true);
}
/** Allocate a new image of the given size with a runtime
* type. Only used when you do know what size you want but you
* don't know statically what type the elements are. Pass zeros
* to make a buffer suitable for bounds query calls. */
template<typename... Args,
typename = typename std::enable_if<AllInts<Args...>::value>::type>
Buffer(halide_type_t t, int first, Args... rest) {
if (!T_is_void) {
assert(static_halide_type() == t);
}
int extents[] = {first, (int)rest...};
buf.type = t;
constexpr int buf_dimensions = 1 + (int)(sizeof...(rest));
make_static_shape_storage<buf_dimensions>();
initialize_shape(extents);
if (!Internal::any_zero(extents)) {
check_overflow();
allocate();
}
}
/** Allocate a new image of the given size. Pass zeros to make a
* buffer suitable for bounds query calls. */
// @{
// The overload with one argument is 'explicit', so that
// (say) int is not implicitly convertible to Buffer<int>
explicit Buffer(int first) {
static_assert(!T_is_void,
"To construct an Buffer<void>, pass a halide_type_t as the first argument to the constructor");
int extents[] = {first};
buf.type = static_halide_type();
constexpr int buf_dimensions = 1;
make_static_shape_storage<buf_dimensions>();
initialize_shape(extents);
if (first != 0) {
check_overflow();
allocate();
}
}
template<typename... Args,
typename = typename std::enable_if<AllInts<Args...>::value>::type>
Buffer(int first, int second, Args... rest) {
static_assert(!T_is_void,
"To construct an Buffer<void>, pass a halide_type_t as the first argument to the constructor");
int extents[] = {first, second, (int)rest...};
buf.type = static_halide_type();
constexpr int buf_dimensions = 2 + (int)(sizeof...(rest));
make_static_shape_storage<buf_dimensions>();
initialize_shape(extents);
if (!Internal::any_zero(extents)) {
check_overflow();
allocate();
}
}
// @}
/** Allocate a new image of unknown type using a vector of ints as the size. */
Buffer(halide_type_t t, const std::vector<int> &sizes) {
if (!T_is_void) {
assert(static_halide_type() == t);
}
buf.type = t;
// make_shape_storage() will do a runtime check that dimensionality matches.
make_shape_storage((int)sizes.size());
initialize_shape(sizes);
if (!Internal::any_zero(sizes)) {
check_overflow();
allocate();
}
}
/** Allocate a new image of known type using a vector of ints as the size. */
explicit Buffer(const std::vector<int> &sizes)
: Buffer(static_halide_type(), sizes) {
}
private:
// Create a copy of the sizes vector, ordered as specified by order.
static std::vector<int> make_ordered_sizes(const std::vector<int> &sizes, const std::vector<int> &order) {
assert(order.size() == sizes.size());
std::vector<int> ordered_sizes(sizes.size());
for (size_t i = 0; i < sizes.size(); ++i) {
ordered_sizes[i] = sizes.at(order[i]);
}
return ordered_sizes;
}
public:
/** Allocate a new image of unknown type using a vector of ints as the size and
* a vector of indices indicating the storage order for each dimension. The
* length of the sizes vector and the storage-order vector must match. For instance,
* to allocate an interleaved RGB buffer, you would pass {2, 0, 1} for storage_order. */
Buffer(halide_type_t t, const std::vector<int> &sizes, const std::vector<int> &storage_order)
: Buffer(t, make_ordered_sizes(sizes, storage_order)) {
transpose(storage_order);
}
Buffer(const std::vector<int> &sizes, const std::vector<int> &storage_order)
: Buffer(static_halide_type(), sizes, storage_order) {
}
/** Make an Buffer that refers to a statically sized array. Does not
* take ownership of the data, and does not set the host_dirty flag. */
template<typename Array, size_t N>
explicit Buffer(Array (&vals)[N]) {
const int buf_dimensions = dimensionality_of_array(vals);
buf.type = scalar_type_of_array(vals);
buf.host = (uint8_t *)vals;
make_shape_storage(buf_dimensions);
initialize_shape_from_array_shape(buf.dimensions - 1, vals);
}
/** Initialize an Buffer of runtime type from a pointer and some
* sizes. Assumes dense row-major packing and a min coordinate of
* zero. Does not take ownership of the data and does not set the
* host_dirty flag. */
template<typename... Args,
typename = typename std::enable_if<AllInts<Args...>::value>::type>
explicit Buffer(halide_type_t t, add_const_if_T_is_const<void> *data, int first, Args &&...rest) {
if (!T_is_void) {
assert(static_halide_type() == t);
}
int extents[] = {first, (int)rest...};
buf.type = t;
buf.host = (uint8_t *)const_cast<void *>(data);
constexpr int buf_dimensions = 1 + (int)(sizeof...(rest));
make_static_shape_storage<buf_dimensions>();
initialize_shape(extents);
}
/** Initialize an Buffer from a pointer and some sizes. Assumes
* dense row-major packing and a min coordinate of zero. Does not
* take ownership of the data and does not set the host_dirty flag. */
template<typename... Args,
typename = typename std::enable_if<AllInts<Args...>::value>::type>
explicit Buffer(T *data, int first, Args &&...rest) {
int extents[] = {first, (int)rest...};
buf.type = static_halide_type();
buf.host = (uint8_t *)const_cast<typename std::remove_const<T>::type *>(data);
constexpr int buf_dimensions = 1 + (int)(sizeof...(rest));
make_static_shape_storage<buf_dimensions>();
initialize_shape(extents);
}
/** Initialize an Buffer from a pointer and a vector of
* sizes. Assumes dense row-major packing and a min coordinate of
* zero. Does not take ownership of the data and does not set the
* host_dirty flag. */
explicit Buffer(T *data, const std::vector<int> &sizes) {
buf.type = static_halide_type();
buf.host = (uint8_t *)const_cast<typename std::remove_const<T>::type *>(data);
make_shape_storage((int)sizes.size());
initialize_shape(sizes);
}
/** Initialize an Buffer of runtime type from a pointer and a
* vector of sizes. Assumes dense row-major packing and a min
* coordinate of zero. Does not take ownership of the data and
* does not set the host_dirty flag. */
explicit Buffer(halide_type_t t, add_const_if_T_is_const<void> *data, const std::vector<int> &sizes) {
if (!T_is_void) {
assert(static_halide_type() == t);
}
buf.type = t;
buf.host = (uint8_t *)const_cast<void *>(data);
make_shape_storage((int)sizes.size());
initialize_shape(sizes);
}
/** Initialize an Buffer from a pointer to the min coordinate and
* an array describing the shape. Does not take ownership of the
* data, and does not set the host_dirty flag. */
explicit Buffer(halide_type_t t, add_const_if_T_is_const<void> *data, int d, const halide_dimension_t *shape) {
if (!T_is_void) {
assert(static_halide_type() == t);
}
buf.type = t;
buf.host = (uint8_t *)const_cast<void *>(data);
make_shape_storage(d);
for (int i = 0; i < d; i++) {
buf.dim[i] = shape[i];
}
}
/** Initialize a Buffer from a pointer to the min coordinate and
* a vector describing the shape. Does not take ownership of the
* data, and does not set the host_dirty flag. */
explicit Buffer(halide_type_t t, add_const_if_T_is_const<void> *data,
const std::vector<halide_dimension_t> &shape)
: Buffer(t, data, (int)shape.size(), shape.data()) {
}
/** Initialize an Buffer from a pointer to the min coordinate and
* an array describing the shape. Does not take ownership of the
* data and does not set the host_dirty flag. */
explicit Buffer(T *data, int d, const halide_dimension_t *shape) {
buf.type = static_halide_type();
buf.host = (uint8_t *)const_cast<typename std::remove_const<T>::type *>(data);
make_shape_storage(d);
for (int i = 0; i < d; i++) {
buf.dim[i] = shape[i];
}
}
/** Initialize a Buffer from a pointer to the min coordinate and
* a vector describing the shape. Does not take ownership of the
* data, and does not set the host_dirty flag. */
explicit Buffer(T *data, const std::vector<halide_dimension_t> &shape)
: Buffer(data, (int)shape.size(), shape.data()) {
}
/** Destructor. Will release any underlying owned allocation if
* this is the last reference to it. Will assert fail if there are
* weak references to this Buffer outstanding. */
~Buffer() {
decref();
free_shape_storage();
}
/** Get a pointer to the raw halide_buffer_t this wraps. */
// @{
halide_buffer_t *raw_buffer() {
return &buf;
}
const halide_buffer_t *raw_buffer() const {
return &buf;
}
// @}
/** Provide a cast operator to halide_buffer_t *, so that
* instances can be passed directly to Halide filters. */
operator halide_buffer_t *() {
return &buf;
}
/** Return a typed reference to this Buffer. Useful for converting
* a reference to a Buffer<void> to a reference to, for example, a
* Buffer<const uint8_t>, or converting a Buffer<T>& to Buffer<const T>&.
* You can also optionally sspecify a new value for Dims; this is useful
* mainly for removing the dimensionality constraint on a Buffer with
* explicit dimensionality. Does a runtime assert if the source buffer type
* is void or the new dimensionality is incompatible. */
template<typename T2, int D2 = Dims>
HALIDE_ALWAYS_INLINE Buffer<T2, D2, InClassDimStorage> &as() & {
Buffer<T2, D2, InClassDimStorage>::assert_can_convert_from(*this);
return *((Buffer<T2, D2, InClassDimStorage> *)this);
}
/** Return a const typed reference to this Buffer. Useful for converting
* a reference to a Buffer<void> to a reference to, for example, a
* Buffer<const uint8_t>, or converting a Buffer<T>& to Buffer<const T>&.
* You can also optionally sspecify a new value for Dims; this is useful
* mainly for removing the dimensionality constraint on a Buffer with
* explicit dimensionality. Does a runtime assert if the source buffer type
* is void or the new dimensionality is incompatible. */
template<typename T2, int D2 = Dims>
HALIDE_ALWAYS_INLINE const Buffer<T2, D2, InClassDimStorage> &as() const & {
Buffer<T2, D2, InClassDimStorage>::assert_can_convert_from(*this);
return *((const Buffer<T2, D2, InClassDimStorage> *)this);
}
/** Return an rval reference to this Buffer. Useful for converting
* a reference to a Buffer<void> to a reference to, for example, a
* Buffer<const uint8_t>, or converting a Buffer<T>& to Buffer<const T>&.
* You can also optionally sspecify a new value for Dims; this is useful
* mainly for removing the dimensionality constraint on a Buffer with
* explicit dimensionality. Does a runtime assert if the source buffer type
* is void or the new dimensionality is incompatible. */
template<typename T2, int D2 = Dims>
HALIDE_ALWAYS_INLINE Buffer<T2, D2, InClassDimStorage> as() && {
Buffer<T2, D2, InClassDimStorage>::assert_can_convert_from(*this);
return *((Buffer<T2, D2, InClassDimStorage> *)this);
}
/** as_const() is syntactic sugar for .as<const T>(), to avoid the need
* to recapitulate the type argument. */
// @{
HALIDE_ALWAYS_INLINE
Buffer<typename std::add_const<T>::type, Dims, InClassDimStorage> &as_const() & {
// Note that we can skip the assert_can_convert_from(), since T -> const T
// conversion is always legal.
return *((Buffer<typename std::add_const<T>::type, Dims, InClassDimStorage> *)this);
}
HALIDE_ALWAYS_INLINE
const Buffer<typename std::add_const<T>::type, Dims, InClassDimStorage> &as_const() const & {
return *((const Buffer<typename std::add_const<T>::type, Dims, InClassDimStorage> *)this);
}
HALIDE_ALWAYS_INLINE
Buffer<typename std::add_const<T>::type, Dims, InClassDimStorage> as_const() && {
return *((Buffer<typename std::add_const<T>::type, Dims, InClassDimStorage> *)this);
}
// @}
/** Add some syntactic sugar to allow autoconversion from Buffer<T> to Buffer<const T>& when
* passing arguments */
template<typename T2 = T, typename = typename std::enable_if<!std::is_const<T2>::value>::type>
operator Buffer<typename std::add_const<T2>::type, Dims, InClassDimStorage> &() & {
return as_const();
}
/** Add some syntactic sugar to allow autoconversion from Buffer<T> to Buffer<void>& when
* passing arguments */
template<typename TVoid,
typename T2 = T,
typename = typename std::enable_if<std::is_same<TVoid, void>::value &&
!std::is_void<T2>::value &&
!std::is_const<T2>::value>::type>
operator Buffer<TVoid, Dims, InClassDimStorage> &() & {
return as<TVoid, Dims>();
}
/** Add some syntactic sugar to allow autoconversion from Buffer<const T> to Buffer<const void>& when
* passing arguments */
template<typename TVoid,
typename T2 = T,
typename = typename std::enable_if<std::is_same<TVoid, void>::value &&
!std::is_void<T2>::value &&
std::is_const<T2>::value>::type>
operator Buffer<const TVoid, Dims, InClassDimStorage> &() & {
return as<const TVoid, Dims>();
}
/** Conventional names for the first three dimensions. */
// @{
int width() const {
return (dimensions() > 0) ? dim(0).extent() : 1;
}
int height() const {
return (dimensions() > 1) ? dim(1).extent() : 1;
}
int channels() const {
return (dimensions() > 2) ? dim(2).extent() : 1;
}
// @}
/** Conventional names for the min and max value of each dimension */
// @{
int left() const {
return dim(0).min();
}
int right() const {
return dim(0).max();
}
int top() const {
return dim(1).min();
}
int bottom() const {
return dim(1).max();
}
// @}
/** Make a new image which is a deep copy of this image. Use crop
* or slice followed by copy to make a copy of only a portion of
* the image. The new image has the same nesting order of dimensions
* (e.g. channels innermost), but resets the strides to the default
* (each stride is the product of the extents of the inner dimensions).
* Note that this means any strides of zero get broadcast into a non-zero stride.
*
* Note that the returned Buffer is always of a non-const type T (ie:
*
* Buffer<const T>.copy() -> Buffer<T> rather than Buffer<const T>
*
* which is always safe, since we are making a deep copy. (The caller
* can easily cast it back to Buffer<const T> if desired, which is
* always safe and free.)
*/
Buffer<not_const_T, Dims, InClassDimStorage> copy(void *(*allocate_fn)(size_t) = nullptr,
void (*deallocate_fn)(void *) = nullptr) const {
Buffer<not_const_T, Dims, InClassDimStorage> dst = Buffer<not_const_T, Dims, InClassDimStorage>::make_with_shape_of(*this, allocate_fn, deallocate_fn);
dst.copy_from(*this);
return dst;
}
/** Like copy(), but the copy is created in interleaved memory layout
* (vs. keeping the same memory layout as the original). Requires that 'this'
* has exactly 3 dimensions.
*/
Buffer<not_const_T, Dims, InClassDimStorage> copy_to_interleaved(void *(*allocate_fn)(size_t) = nullptr,
void (*deallocate_fn)(void *) = nullptr) const {
static_assert(Dims == AnyDims || Dims == 3);
assert(dimensions() == 3);
Buffer<not_const_T, Dims, InClassDimStorage> dst = Buffer<not_const_T, Dims, InClassDimStorage>::make_interleaved(nullptr, width(), height(), channels());
dst.set_min(min(0), min(1), min(2));
dst.allocate(allocate_fn, deallocate_fn);
dst.copy_from(*this);
return dst;
}
/** Like copy(), but the copy is created in planar memory layout
* (vs. keeping the same memory layout as the original).
*/
Buffer<not_const_T, Dims, InClassDimStorage> copy_to_planar(void *(*allocate_fn)(size_t) = nullptr,
void (*deallocate_fn)(void *) = nullptr) const {
std::vector<int> mins, extents;
const int dims = dimensions();
mins.reserve(dims);
extents.reserve(dims);
for (int d = 0; d < dims; ++d) {
mins.push_back(dim(d).min());
extents.push_back(dim(d).extent());
}
Buffer<not_const_T, Dims, InClassDimStorage> dst = Buffer<not_const_T, Dims, InClassDimStorage>(nullptr, extents);
dst.set_min(mins);
dst.allocate(allocate_fn, deallocate_fn);
dst.copy_from(*this);
return dst;
}
/** Make a copy of the Buffer which shares the underlying host and/or device
* allocations as the existing Buffer. This is purely syntactic sugar for
* cases where you have a const reference to a Buffer but need a temporary
* non-const copy (e.g. to make a call into AOT-generated Halide code), and want a terse
* inline way to create a temporary. \code
* void call_my_func(const Buffer<const uint8_t>& input) {
* my_func(input.alias(), output);
* }\endcode
*/
Buffer<T, Dims, InClassDimStorage> alias() const {
return *this;
}
/** Fill a Buffer with the values at the same coordinates in
* another Buffer. Restricts itself to coordinates contained
* within the intersection of the two buffers. If the two Buffers
* are not in the same coordinate system, you will need to
* translate the argument Buffer first. E.g. if you're blitting a
* sprite onto a framebuffer, you'll want to translate the sprite
* to the correct location first like so: \code
* framebuffer.copy_from(sprite.translated({x, y})); \endcode
*/
template<typename T2, int D2, int S2>
void copy_from(Buffer<T2, D2, S2> src) {
static_assert(!std::is_const<T>::value, "Cannot call copy_from() on a Buffer<const T>");
assert(!device_dirty() && "Cannot call Halide::Runtime::Buffer::copy_from on a device dirty destination.");
assert(!src.device_dirty() && "Cannot call Halide::Runtime::Buffer::copy_from on a device dirty source.");
Buffer<T, Dims, InClassDimStorage> dst(*this);
static_assert(Dims == AnyDims || D2 == AnyDims || Dims == D2);
assert(src.dimensions() == dst.dimensions());
// Trim the copy to the region in common
const int d = dimensions();
for (int i = 0; i < d; i++) {
int min_coord = std::max(dst.dim(i).min(), src.dim(i).min());
int max_coord = std::min(dst.dim(i).max(), src.dim(i).max());
if (max_coord < min_coord) {
// The buffers do not overlap.
return;
}
dst.crop(i, min_coord, max_coord - min_coord + 1);
src.crop(i, min_coord, max_coord - min_coord + 1);
}
// If T is void, we need to do runtime dispatch to an
// appropriately-typed lambda. We're copying, so we only care
// about the element size. (If not, this should optimize away
// into a static dispatch to the right-sized copy.)
if (T_is_void ? (type().bytes() == 1) : (sizeof(not_void_T) == 1)) {
using MemType = uint8_t;
auto &typed_dst = reinterpret_cast<Buffer<MemType, Dims, InClassDimStorage> &>(dst);
auto &typed_src = reinterpret_cast<Buffer<const MemType, D2, S2> &>(src);
typed_dst.for_each_value([&](MemType &dst, MemType src) { dst = src; }, typed_src);
} else if (T_is_void ? (type().bytes() == 2) : (sizeof(not_void_T) == 2)) {
using MemType = uint16_t;
auto &typed_dst = reinterpret_cast<Buffer<MemType, Dims, InClassDimStorage> &>(dst);
auto &typed_src = reinterpret_cast<Buffer<const MemType, D2, S2> &>(src);
typed_dst.for_each_value([&](MemType &dst, MemType src) { dst = src; }, typed_src);
} else if (T_is_void ? (type().bytes() == 4) : (sizeof(not_void_T) == 4)) {
using MemType = uint32_t;
auto &typed_dst = reinterpret_cast<Buffer<MemType, Dims, InClassDimStorage> &>(dst);
auto &typed_src = reinterpret_cast<Buffer<const MemType, D2, S2> &>(src);
typed_dst.for_each_value([&](MemType &dst, MemType src) { dst = src; }, typed_src);
} else if (T_is_void ? (type().bytes() == 8) : (sizeof(not_void_T) == 8)) {
using MemType = uint64_t;
auto &typed_dst = reinterpret_cast<Buffer<MemType, Dims, InClassDimStorage> &>(dst);
auto &typed_src = reinterpret_cast<Buffer<const MemType, D2, S2> &>(src);
typed_dst.for_each_value([&](MemType &dst, MemType src) { dst = src; }, typed_src);
} else {
assert(false && "type().bytes() must be 1, 2, 4, or 8");
}
set_host_dirty();
}
/** Make an image that refers to a sub-range of this image along
* the given dimension. Asserts that the crop region is within
* the existing bounds: you cannot "crop outwards", even if you know there
* is valid Buffer storage (e.g. because you already cropped inwards). */
Buffer<T, Dims, InClassDimStorage> cropped(int d, int min, int extent) const {
// Make a fresh copy of the underlying buffer (but not a fresh
// copy of the allocation, if there is one).
Buffer<T, Dims, InClassDimStorage> im = *this;
// This guarantees the prexisting device ref is dropped if the
// device_crop call fails and maintains the buffer in a consistent
// state.
im.device_deallocate();
im.crop_host(d, min, extent);
if (buf.device_interface != nullptr) {
complete_device_crop(im);
}
return im;
}
/** Crop an image in-place along the given dimension. This does
* not move any data around in memory - it just changes the min
* and extent of the given dimension. */
void crop(int d, int min, int extent) {
// An optimization for non-device buffers. For the device case,
// a temp buffer is required, so reuse the not-in-place version.
// TODO(zalman|abadams): Are nop crops common enough to special
// case the device part of the if to do nothing?
if (buf.device_interface != nullptr) {
*this = cropped(d, min, extent);
} else {
crop_host(d, min, extent);
}
}
/** Make an image that refers to a sub-rectangle of this image along
* the first N dimensions. Asserts that the crop region is within
* the existing bounds. The cropped image may drop any device handle
* if the device_interface cannot accomplish the crop in-place. */
Buffer<T, Dims, InClassDimStorage> cropped(const std::vector<std::pair<int, int>> &rect) const {
// Make a fresh copy of the underlying buffer (but not a fresh
// copy of the allocation, if there is one).
Buffer<T, Dims, InClassDimStorage> im = *this;
// This guarantees the prexisting device ref is dropped if the
// device_crop call fails and maintains the buffer in a consistent
// state.
im.device_deallocate();
im.crop_host(rect);
if (buf.device_interface != nullptr) {
complete_device_crop(im);
}
return im;
}
/** Crop an image in-place along the first N dimensions. This does
* not move any data around in memory, nor does it free memory. It
* just rewrites the min/extent of each dimension to refer to a
* subregion of the same allocation. */
void crop(const std::vector<std::pair<int, int>> &rect) {
// An optimization for non-device buffers. For the device case,
// a temp buffer is required, so reuse the not-in-place version.
// TODO(zalman|abadams): Are nop crops common enough to special
// case the device part of the if to do nothing?
if (buf.device_interface != nullptr) {
*this = cropped(rect);
} else {
crop_host(rect);
}
}
/** Make an image which refers to the same data with using
* translated coordinates in the given dimension. Positive values
* move the image data to the right or down relative to the
* coordinate system. Drops any device handle. */
Buffer<T, Dims, InClassDimStorage> translated(int d, int dx) const {
Buffer<T, Dims, InClassDimStorage> im = *this;
im.translate(d, dx);
return im;
}
/** Translate an image in-place along one dimension by changing
* how it is indexed. Does not move any data around in memory. */
void translate(int d, int delta) {
assert(d >= 0 && d < this->dimensions());
device_deallocate();
buf.dim[d].min += delta;
}
/** Make an image which refers to the same data translated along
* the first N dimensions. */
Buffer<T, Dims, InClassDimStorage> translated(const std::vector<int> &delta) const {
Buffer<T, Dims, InClassDimStorage> im = *this;
im.translate(delta);
return im;
}
/** Translate an image along the first N dimensions by changing
* how it is indexed. Does not move any data around in memory. */
void translate(const std::vector<int> &delta) {
device_deallocate();
assert(delta.size() <= static_cast<decltype(delta.size())>(std::numeric_limits<int>::max()));
int limit = (int)delta.size();
assert(limit <= dimensions());
for (int i = 0; i < limit; i++) {
translate(i, delta[i]);
}
}
/** Set the min coordinate of an image in the first N dimensions. */
// @{
void set_min(const std::vector<int> &mins) {
assert(mins.size() <= static_cast<decltype(mins.size())>(dimensions()));
device_deallocate();
for (size_t i = 0; i < mins.size(); i++) {
buf.dim[i].min = mins[i];
}
}
template<typename... Args>
void set_min(Args... args) {
set_min(std::vector<int>{args...});
}
// @}
/** Test if a given coordinate is within the bounds of an image. */
// @{
bool contains(const std::vector<int> &coords) const {
assert(coords.size() <= static_cast<decltype(coords.size())>(dimensions()));
for (size_t i = 0; i < coords.size(); i++) {
if (coords[i] < dim((int)i).min() || coords[i] > dim((int)i).max()) {
return false;
}
}
return true;
}
template<typename... Args>
bool contains(Args... args) const {
return contains(std::vector<int>{args...});
}
// @}
/** Make a buffer which refers to the same data in the same layout
* using a swapped indexing order for the dimensions given. So
* A = B.transposed(0, 1) means that A(i, j) == B(j, i), and more
* strongly that A.address_of(i, j) == B.address_of(j, i). */
Buffer<T, Dims, InClassDimStorage> transposed(int d1, int d2) const {
Buffer<T, Dims, InClassDimStorage> im = *this;
im.transpose(d1, d2);
return im;
}
/** Transpose a buffer in-place by changing how it is indexed. For
* example, transpose(0, 1) on a two-dimensional buffer means that
* the value referred to by coordinates (i, j) is now reached at
* the coordinates (j, i), and vice versa. This is done by
* reordering the per-dimension metadata rather than by moving
* data around in memory, so other views of the same memory will
* not see the data as having been transposed. */
void transpose(int d1, int d2) {
assert(d1 >= 0 && d1 < this->dimensions());
assert(d2 >= 0 && d2 < this->dimensions());
std::swap(buf.dim[d1], buf.dim[d2]);
}
/** A generalized transpose: instead of swapping two dimensions,
* pass a vector that lists each dimension index exactly once, in
* the desired order. This does not move any data around in memory
* - it just permutes how it is indexed. */
void transpose(const std::vector<int> &order) {
assert((int)order.size() == dimensions());
if (dimensions() < 2) {
// My, that was easy
return;
}
std::vector<int> order_sorted = order;
for (size_t i = 1; i < order_sorted.size(); i++) {
for (size_t j = i; j > 0 && order_sorted[j - 1] > order_sorted[j]; j--) {
std::swap(order_sorted[j], order_sorted[j - 1]);
transpose(j, j - 1);
}
}
}
/** Make a buffer which refers to the same data in the same
* layout using a different ordering of the dimensions. */
Buffer<T, Dims, InClassDimStorage> transposed(const std::vector<int> &order) const {
Buffer<T, Dims, InClassDimStorage> im = *this;
im.transpose(order);
return im;
}
/** Make a lower-dimensional buffer that refers to one slice of
* this buffer. */
Buffer<T, (Dims == AnyDims ? AnyDims : Dims - 1)>
sliced(int d, int pos) const {
static_assert(Dims == AnyDims || Dims > 0, "Cannot slice a 0-dimensional buffer");
assert(dimensions() > 0);
Buffer<T, AnyDims, InClassDimStorage> im = *this;
// This guarantees the prexisting device ref is dropped if the
// device_slice call fails and maintains the buffer in a consistent
// state.
im.device_deallocate();
im.slice_host(d, pos);
if (buf.device_interface != nullptr) {
complete_device_slice(im, d, pos);
}
return im;
}
/** Make a lower-dimensional buffer that refers to one slice of this
* buffer at the dimension's minimum. */
Buffer<T, (Dims == AnyDims ? AnyDims : Dims - 1)>
sliced(int d) const {
static_assert(Dims == AnyDims || Dims > 0, "Cannot slice a 0-dimensional buffer");
assert(dimensions() > 0);
return sliced(d, dim(d).min());
}
/** Rewrite the buffer to refer to a single lower-dimensional
* slice of itself along the given dimension at the given
* coordinate. Does not move any data around or free the original
* memory, so other views of the same data are unaffected. Can
* only be called on a Buffer with dynamic dimensionality. */
void slice(int d, int pos) {
static_assert(Dims == AnyDims, "Cannot call slice() on a Buffer with static dimensionality.");
assert(dimensions() > 0);
// An optimization for non-device buffers. For the device case,
// a temp buffer is required, so reuse the not-in-place version.
// TODO(zalman|abadams): Are nop slices common enough to special
// case the device part of the if to do nothing?
if (buf.device_interface != nullptr) {
*this = sliced(d, pos);
} else {
slice_host(d, pos);
}
}
/** Slice a buffer in-place at the dimension's minimum. */
void slice(int d) {
slice(d, dim(d).min());
}
/** Make a new buffer that views this buffer as a single slice in a
* higher-dimensional space. The new dimension has extent one and
* the given min. This operation is the opposite of slice. As an
* example, the following condition is true:
*
\code
im2 = im.embedded(1, 17);
&im(x, y, c) == &im2(x, 17, y, c);
\endcode
*/
Buffer<T, (Dims == AnyDims ? AnyDims : Dims + 1)>
embedded(int d, int pos = 0) const {
Buffer<T, AnyDims, InClassDimStorage> im(*this);
im.embed(d, pos);
return im;
}
/** Embed a buffer in-place, increasing the
* dimensionality. */
void embed(int d, int pos = 0) {
static_assert(Dims == AnyDims, "Cannot call embed() on a Buffer with static dimensionality.");
assert(d >= 0 && d <= dimensions());
add_dimension();
translate(dimensions() - 1, pos);
for (int i = dimensions() - 1; i > d; i--) {
transpose(i, i - 1);
}
}
/** Add a new dimension with a min of zero and an extent of
* one. The stride is the extent of the outermost dimension times
* its stride. The new dimension is the last dimension. This is a
* special case of embed. */
void add_dimension() {
static_assert(Dims == AnyDims, "Cannot call add_dimension() on a Buffer with static dimensionality.");
const int dims = buf.dimensions;
buf.dimensions++;
if (buf.dim != shape) {
// We're already on the heap. Reallocate.
halide_dimension_t *new_shape = new halide_dimension_t[buf.dimensions];
for (int i = 0; i < dims; i++) {
new_shape[i] = buf.dim[i];
}
delete[] buf.dim;
buf.dim = new_shape;
} else if (dims == InClassDimStorage) {
// Transition from the in-class storage to the heap
make_shape_storage(buf.dimensions);
for (int i = 0; i < dims; i++) {
buf.dim[i] = shape[i];
}
} else {
// We still fit in the class
}
buf.dim[dims] = {0, 1, 0};
if (dims == 0) {
buf.dim[dims].stride = 1;
} else {
buf.dim[dims].stride = buf.dim[dims - 1].extent * buf.dim[dims - 1].stride;
}
}
/** Add a new dimension with a min of zero, an extent of one, and
* the specified stride. The new dimension is the last
* dimension. This is a special case of embed. */
void add_dimension_with_stride(int s) {
add_dimension();
buf.dim[buf.dimensions - 1].stride = s;
}
/** Methods for managing any GPU allocation. */
// @{
// Set the host dirty flag. Called by every operator()
// access. Must be inlined so it can be hoisted out of loops.
HALIDE_ALWAYS_INLINE
void set_host_dirty(bool v = true) {
assert((!v || !device_dirty()) && "Cannot set host dirty when device is already dirty. Call copy_to_host() before accessing the buffer from host.");
buf.set_host_dirty(v);
}
// Check if the device allocation is dirty. Called by
// set_host_dirty, which is called by every accessor. Must be
// inlined so it can be hoisted out of loops.
HALIDE_ALWAYS_INLINE
bool device_dirty() const {
return buf.device_dirty();
}
bool host_dirty() const {
return buf.host_dirty();
}
void set_device_dirty(bool v = true) {
assert((!v || !host_dirty()) && "Cannot set device dirty when host is already dirty.");
buf.set_device_dirty(v);
}
int copy_to_host(void *ctx = nullptr) {
if (device_dirty()) {
return buf.device_interface->copy_to_host(ctx, &buf);
}
return halide_error_code_success;
}
int copy_to_device(const struct halide_device_interface_t *device_interface, void *ctx = nullptr) {
if (host_dirty()) {
return device_interface->copy_to_device(ctx, &buf, device_interface);
}
return halide_error_code_success;
}
int device_malloc(const struct halide_device_interface_t *device_interface, void *ctx = nullptr) {
return device_interface->device_malloc(ctx, &buf, device_interface);
}
int device_free(void *ctx = nullptr) {
if (dev_ref_count) {
assert(dev_ref_count->ownership == BufferDeviceOwnership::Allocated &&
"Can't call device_free on an unmanaged or wrapped native device handle. "
"Free the source allocation or call device_detach_native instead.");
// Multiple people may be holding onto this dev field
assert(dev_ref_count->count == 1 &&
"Multiple Halide::Runtime::Buffer objects share this device "
"allocation. Freeing it would create dangling references. "
"Don't call device_free on Halide buffers that you have copied or "
"passed by value.");
}
int ret = halide_error_code_success;
if (buf.device_interface) {
ret = buf.device_interface->device_free(ctx, &buf);
}
if (dev_ref_count) {
delete dev_ref_count;
dev_ref_count = nullptr;
}
return ret;
}
int device_wrap_native(const struct halide_device_interface_t *device_interface,
uint64_t handle, void *ctx = nullptr) {
assert(device_interface);
dev_ref_count = new DeviceRefCount;
dev_ref_count->ownership = BufferDeviceOwnership::WrappedNative;
return device_interface->wrap_native(ctx, &buf, handle, device_interface);
}
int device_detach_native(void *ctx = nullptr) {
assert(dev_ref_count &&
dev_ref_count->ownership == BufferDeviceOwnership::WrappedNative &&
"Only call device_detach_native on buffers wrapping a native "
"device handle via device_wrap_native. This buffer was allocated "
"using device_malloc, or is unmanaged. "
"Call device_free or free the original allocation instead.");
// Multiple people may be holding onto this dev field
assert(dev_ref_count->count == 1 &&
"Multiple Halide::Runtime::Buffer objects share this device "
"allocation. Freeing it could create dangling references. "
"Don't call device_detach_native on Halide buffers that you "
"have copied or passed by value.");
int ret = halide_error_code_success;
if (buf.device_interface) {
ret = buf.device_interface->detach_native(ctx, &buf);
}
delete dev_ref_count;
dev_ref_count = nullptr;
return ret;
}
int device_and_host_malloc(const struct halide_device_interface_t *device_interface, void *ctx = nullptr) {
return device_interface->device_and_host_malloc(ctx, &buf, device_interface);
}
int device_and_host_free(const struct halide_device_interface_t *device_interface, void *ctx = nullptr) {
if (dev_ref_count) {
assert(dev_ref_count->ownership == BufferDeviceOwnership::AllocatedDeviceAndHost &&
"Can't call device_and_host_free on a device handle not allocated with device_and_host_malloc. "
"Free the source allocation or call device_detach_native instead.");
// Multiple people may be holding onto this dev field
assert(dev_ref_count->count == 1 &&
"Multiple Halide::Runtime::Buffer objects share this device "
"allocation. Freeing it would create dangling references. "
"Don't call device_and_host_free on Halide buffers that you have copied or "
"passed by value.");
}
int ret = halide_error_code_success;
if (buf.device_interface) {
ret = buf.device_interface->device_and_host_free(ctx, &buf);
}
if (dev_ref_count) {
delete dev_ref_count;
dev_ref_count = nullptr;
}
return ret;
}
int device_sync(void *ctx = nullptr) {
return buf.device_sync(ctx);
}
bool has_device_allocation() const {
return buf.device != 0;
}
/** Return the method by which the device field is managed. */
BufferDeviceOwnership device_ownership() const {
if (dev_ref_count == nullptr) {
return BufferDeviceOwnership::Allocated;
}
return dev_ref_count->ownership;
}
// @}
/** If you use the (x, y, c) indexing convention, then Halide
* Buffers are stored planar by default. This function constructs
* an interleaved RGB or RGBA image that can still be indexed
* using (x, y, c). Passing it to a generator requires that the
* generator has been compiled with support for interleaved (also
* known as packed or chunky) memory layouts. */
static Buffer<void, Dims, InClassDimStorage> make_interleaved(halide_type_t t, int width, int height, int channels) {
static_assert(Dims == AnyDims || Dims == 3, "make_interleaved() must be called on a Buffer that can represent 3 dimensions.");
Buffer<void, Dims, InClassDimStorage> im(t, channels, width, height);
// Note that this is equivalent to calling transpose({2, 0, 1}),
// but slightly more efficient.
im.transpose(0, 1);
im.transpose(1, 2);
return im;
}
/** If you use the (x, y, c) indexing convention, then Halide
* Buffers are stored planar by default. This function constructs
* an interleaved RGB or RGBA image that can still be indexed
* using (x, y, c). Passing it to a generator requires that the
* generator has been compiled with support for interleaved (also
* known as packed or chunky) memory layouts. */
static Buffer<T, Dims, InClassDimStorage> make_interleaved(int width, int height, int channels) {
return make_interleaved(static_halide_type(), width, height, channels);
}
/** Wrap an existing interleaved image. */
static Buffer<add_const_if_T_is_const<void>, Dims, InClassDimStorage>
make_interleaved(halide_type_t t, T *data, int width, int height, int channels) {
static_assert(Dims == AnyDims || Dims == 3, "make_interleaved() must be called on a Buffer that can represent 3 dimensions.");
Buffer<add_const_if_T_is_const<void>, Dims, InClassDimStorage> im(t, data, channels, width, height);
im.transpose(0, 1);
im.transpose(1, 2);
return im;
}
/** Wrap an existing interleaved image. */
static Buffer<T, Dims, InClassDimStorage> make_interleaved(T *data, int width, int height, int channels) {
return make_interleaved(static_halide_type(), data, width, height, channels);
}
/** Make a zero-dimensional Buffer */
static Buffer<add_const_if_T_is_const<void>, Dims, InClassDimStorage> make_scalar(halide_type_t t) {
static_assert(Dims == AnyDims || Dims == 0, "make_scalar() must be called on a Buffer that can represent 0 dimensions.");
Buffer<add_const_if_T_is_const<void>, AnyDims, InClassDimStorage> buf(t, 1);
buf.slice(0, 0);
return buf;
}
/** Make a zero-dimensional Buffer */
static Buffer<T, Dims, InClassDimStorage> make_scalar() {
static_assert(Dims == AnyDims || Dims == 0, "make_scalar() must be called on a Buffer that can represent 0 dimensions.");
Buffer<T, AnyDims, InClassDimStorage> buf(1);
buf.slice(0, 0);
return buf;
}
/** Make a zero-dimensional Buffer that points to non-owned, existing data */
static Buffer<T, Dims, InClassDimStorage> make_scalar(T *data) {
static_assert(Dims == AnyDims || Dims == 0, "make_scalar() must be called on a Buffer that can represent 0 dimensions.");
Buffer<T, AnyDims, InClassDimStorage> buf(data, 1);
buf.slice(0, 0);
return buf;
}
/** Make a buffer with the same shape and memory nesting order as
* another buffer. It may have a different type. */
template<typename T2, int D2, int S2>
// NOLINTNEXTLINE(performance-unnecessary-value-param)
static Buffer<T, Dims, InClassDimStorage> make_with_shape_of(Buffer<T2, D2, S2> src,
void *(*allocate_fn)(size_t) = nullptr,
void (*deallocate_fn)(void *) = nullptr) {
// Note that src is taken by value because its dims are mutated
// in-place by the helper. Do not change to taking it by reference.
static_assert(Dims == D2 || Dims == AnyDims);
const halide_type_t dst_type = T_is_void ? src.type() : halide_type_of<typename std::remove_cv<not_void_T>::type>();
return Buffer<>::make_with_shape_of_helper(dst_type, src.dimensions(), src.buf.dim,
allocate_fn, deallocate_fn);
}
private:
static Buffer<> make_with_shape_of_helper(halide_type_t dst_type,
int dimensions,
halide_dimension_t *shape,
void *(*allocate_fn)(size_t),
void (*deallocate_fn)(void *)) {
// Reorder the dimensions of src to have strides in increasing order
std::vector<int> swaps;
for (int i = dimensions - 1; i > 0; i--) {
for (int j = i; j > 0; j--) {
if (shape[j - 1].stride > shape[j].stride) {
std::swap(shape[j - 1], shape[j]);
swaps.push_back(j);
}
}
}
// Rewrite the strides to be dense (this messes up src, which
// is why we took it by value).
for (int i = 0; i < dimensions; i++) {
if (i == 0) {
shape[i].stride = 1;
} else {
shape[i].stride = shape[i - 1].extent * shape[i - 1].stride;
}
}
// Undo the dimension reordering
while (!swaps.empty()) {
int j = swaps.back();
std::swap(shape[j - 1], shape[j]);
swaps.pop_back();
}
// Use an explicit runtime type, and make dst a Buffer<void>, to allow
// using this method with Buffer<void> for either src or dst.
Buffer<> dst(dst_type, nullptr, dimensions, shape);
dst.allocate(allocate_fn, deallocate_fn);
return dst;
}
template<typename... Args>
HALIDE_ALWAYS_INLINE
ptrdiff_t
offset_of(int d, int first, Args... rest) const {
#if HALIDE_RUNTIME_BUFFER_CHECK_INDICES
assert(first >= this->buf.dim[d].min);
assert(first < this->buf.dim[d].min + this->buf.dim[d].extent);
#endif
return offset_of(d + 1, rest...) + (ptrdiff_t)this->buf.dim[d].stride * (first - this->buf.dim[d].min);
}
HALIDE_ALWAYS_INLINE
ptrdiff_t offset_of(int d) const {
return 0;
}
template<typename... Args>
HALIDE_ALWAYS_INLINE storage_T *address_of(Args... args) const {
if (T_is_void) {
return (storage_T *)(this->buf.host) + offset_of(0, args...) * type().bytes();
} else {
return (storage_T *)(this->buf.host) + offset_of(0, args...);
}
}
HALIDE_ALWAYS_INLINE
ptrdiff_t offset_of(const int *pos) const {
ptrdiff_t offset = 0;
for (int i = this->dimensions() - 1; i >= 0; i--) {
#if HALIDE_RUNTIME_BUFFER_CHECK_INDICES
assert(pos[i] >= this->buf.dim[i].min);
assert(pos[i] < this->buf.dim[i].min + this->buf.dim[i].extent);
#endif
offset += (ptrdiff_t)this->buf.dim[i].stride * (pos[i] - this->buf.dim[i].min);
}
return offset;
}
HALIDE_ALWAYS_INLINE
storage_T *address_of(const int *pos) const {
if (T_is_void) {
return (storage_T *)this->buf.host + offset_of(pos) * type().bytes();
} else {
return (storage_T *)this->buf.host + offset_of(pos);
}
}
public:
/** Get a pointer to the address of the min coordinate. */
T *data() const {
return (T *)(this->buf.host);
}
/** Access elements. Use im(...) to get a reference to an element,
* and use &im(...) to get the address of an element. If you pass
* fewer arguments than the buffer has dimensions, the rest are
* treated as their min coordinate. The non-const versions set the
* host_dirty flag to true.
*/
//@{
template<typename... Args,
typename = typename std::enable_if<AllInts<Args...>::value>::type>
HALIDE_ALWAYS_INLINE const not_void_T &operator()(int first, Args... rest) const {
static_assert(!T_is_void,
"Cannot use operator() on Buffer<void> types");
constexpr int expected_dims = 1 + (int)(sizeof...(rest));
static_assert(Dims == AnyDims || Dims == expected_dims, "Buffer with static dimensions was accessed with the wrong number of coordinates in operator()");
assert(!device_dirty());
return *((const not_void_T *)(address_of(first, rest...)));
}
HALIDE_ALWAYS_INLINE
const not_void_T &operator()() const {
static_assert(!T_is_void,
"Cannot use operator() on Buffer<void> types");
constexpr int expected_dims = 0;
static_assert(Dims == AnyDims || Dims == expected_dims, "Buffer with static dimensions was accessed with the wrong number of coordinates in operator()");
assert(!device_dirty());
return *((const not_void_T *)(data()));
}
HALIDE_ALWAYS_INLINE
const not_void_T &
operator()(const int *pos) const {
static_assert(!T_is_void,
"Cannot use operator() on Buffer<void> types");
assert(!device_dirty());
return *((const not_void_T *)(address_of(pos)));
}
template<typename... Args,
typename = typename std::enable_if<AllInts<Args...>::value>::type>
HALIDE_ALWAYS_INLINE not_void_T &operator()(int first, Args... rest) {
static_assert(!T_is_void,
"Cannot use operator() on Buffer<void> types");
constexpr int expected_dims = 1 + (int)(sizeof...(rest));
static_assert(Dims == AnyDims || Dims == expected_dims, "Buffer with static dimensions was accessed with the wrong number of coordinates in operator()");
set_host_dirty();
return *((not_void_T *)(address_of(first, rest...)));
}
HALIDE_ALWAYS_INLINE
not_void_T &
operator()() {
static_assert(!T_is_void,
"Cannot use operator() on Buffer<void> types");
constexpr int expected_dims = 0;
static_assert(Dims == AnyDims || Dims == expected_dims, "Buffer with static dimensions was accessed with the wrong number of coordinates in operator()");
set_host_dirty();
return *((not_void_T *)(data()));
}
HALIDE_ALWAYS_INLINE
not_void_T &
operator()(const int *pos) {
static_assert(!T_is_void,
"Cannot use operator() on Buffer<void> types");
set_host_dirty();
return *((not_void_T *)(address_of(pos)));
}
// @}
/** Tests that all values in this buffer are equal to val. */
bool all_equal(not_void_T val) const {
bool all_equal = true;
for_each_element([&](const int *pos) { all_equal &= (*this)(pos) == val; });
return all_equal;
}
Buffer<T, Dims, InClassDimStorage> &fill(not_void_T val) {
set_host_dirty();
for_each_value([=](T &v) { v = val; });
return *this;
}
private:
/** Helper functions for for_each_value. */
// @{
template<int N>
struct for_each_value_task_dim {
std::ptrdiff_t extent;
std::ptrdiff_t stride[N];
};
// Given an array of strides, and a bunch of pointers to pointers
// (all of different types), advance the pointers using the
// strides.
template<typename Ptr, typename... Ptrs>
HALIDE_ALWAYS_INLINE static void advance_ptrs(const std::ptrdiff_t *stride, Ptr &ptr, Ptrs &...ptrs) {
ptr += *stride;
advance_ptrs(stride + 1, ptrs...);
}
HALIDE_ALWAYS_INLINE
static void advance_ptrs(const std::ptrdiff_t *) {
}
template<typename Fn, typename Ptr, typename... Ptrs>
HALIDE_NEVER_INLINE static void for_each_value_helper(Fn &&f, int d, bool innermost_strides_are_one,
const for_each_value_task_dim<sizeof...(Ptrs) + 1> *t, Ptr ptr, Ptrs... ptrs) {
if (d == 0) {
if (innermost_strides_are_one) {
Ptr end = ptr + t[0].extent;
while (ptr != end) {
f(*ptr++, (*ptrs++)...);
}
} else {
for (std::ptrdiff_t i = t[0].extent; i != 0; i--) {
f(*ptr, (*ptrs)...);
advance_ptrs(t[0].stride, ptr, ptrs...);
}
}
} else {
for (std::ptrdiff_t i = t[d].extent; i != 0; i--) {
for_each_value_helper(f, d - 1, innermost_strides_are_one, t, ptr, ptrs...);
advance_ptrs(t[d].stride, ptr, ptrs...);
}
}
}
// Return pair is <new_dimensions, innermost_strides_are_one>
template<int N>
HALIDE_NEVER_INLINE static std::pair<int, bool> for_each_value_prep(for_each_value_task_dim<N> *t,
const halide_buffer_t **buffers) {
const int dimensions = buffers[0]->dimensions;
assert(dimensions > 0);
// Check the buffers all have clean host allocations
for (int i = 0; i < N; i++) {
if (buffers[i]->device) {
assert(buffers[i]->host &&
"Buffer passed to for_each_value has device allocation but no host allocation. Call allocate() and copy_to_host() first");
assert(!buffers[i]->device_dirty() &&
"Buffer passed to for_each_value is dirty on device. Call copy_to_host() first");
} else {
assert(buffers[i]->host &&
"Buffer passed to for_each_value has no host or device allocation");
}
}
// Extract the strides in all the dimensions
for (int i = 0; i < dimensions; i++) {
for (int j = 0; j < N; j++) {
assert(buffers[j]->dimensions == dimensions);
assert(buffers[j]->dim[i].extent == buffers[0]->dim[i].extent &&
buffers[j]->dim[i].min == buffers[0]->dim[i].min);
const int s = buffers[j]->dim[i].stride;
t[i].stride[j] = s;
}
t[i].extent = buffers[0]->dim[i].extent;
// Order the dimensions by stride, so that the traversal is cache-coherent.
// Use the last dimension for this, because this is the source in copies.
// It appears to be better to optimize read order than write order.
for (int j = i; j > 0 && t[j].stride[N - 1] < t[j - 1].stride[N - 1]; j--) {
std::swap(t[j], t[j - 1]);
}
}
// flatten dimensions where possible to make a larger inner
// loop for autovectorization.
int d = dimensions;
for (int i = 1; i < d; i++) {
bool flat = true;
for (int j = 0; j < N; j++) {
flat = flat && t[i - 1].stride[j] * t[i - 1].extent == t[i].stride[j];
}
if (flat) {
t[i - 1].extent *= t[i].extent;
for (int j = i; j < d - 1; j++) {
t[j] = t[j + 1];
}
i--;
d--;
}
}
// Note that we assert() that dimensions > 0 above
// (our one-and-only caller will only call us that way)
// so the unchecked access to t[0] should be safe.
bool innermost_strides_are_one = true;
for (int i = 0; i < N; i++) {
innermost_strides_are_one &= (t[0].stride[i] == 1);
}
return {d, innermost_strides_are_one};
}
template<typename Fn, typename... Args, int N = sizeof...(Args) + 1>
void for_each_value_impl(Fn &&f, Args &&...other_buffers) const {
if (dimensions() > 0) {
const size_t alloc_size = dimensions() * sizeof(for_each_value_task_dim<N>);
Buffer<>::for_each_value_task_dim<N> *t =
(Buffer<>::for_each_value_task_dim<N> *)HALIDE_ALLOCA(alloc_size);
// Move the preparatory code into a non-templated helper to
// save code size.
const halide_buffer_t *buffers[] = {&buf, (&other_buffers.buf)...};
auto [new_dims, innermost_strides_are_one] = Buffer<>::for_each_value_prep(t, buffers);
if (new_dims > 0) {
Buffer<>::for_each_value_helper(f, new_dims - 1,
innermost_strides_are_one,
t,
data(), (other_buffers.data())...);
return;
}
// else fall thru
}
// zero-dimensional case
f(*data(), (*other_buffers.data())...);
}
// @}
public:
/** Call a function on every value in the buffer, and the
* corresponding values in some number of other buffers of the
* same size. The function should take a reference, const
* reference, or value of the correct type for each buffer. This
* effectively lifts a function of scalars to an element-wise
* function of buffers. This produces code that the compiler can
* autovectorize. This is slightly cheaper than for_each_element,
* because it does not need to track the coordinates.
*
* Note that constness of Buffers is preserved: a const Buffer<T> (for either
* 'this' or the other-buffers arguments) will allow mutation of the
* buffer contents, while a Buffer<const T> will not. Attempting to specify
* a mutable reference for the lambda argument of a Buffer<const T>
* will result in a compilation error. */
// @{
template<typename Fn, typename... Args, int N = sizeof...(Args) + 1>
HALIDE_ALWAYS_INLINE const Buffer<T, Dims, InClassDimStorage> &for_each_value(Fn &&f, Args &&...other_buffers) const {
for_each_value_impl(f, std::forward<Args>(other_buffers)...);
return *this;
}
template<typename Fn, typename... Args, int N = sizeof...(Args) + 1>
HALIDE_ALWAYS_INLINE
Buffer<T, Dims, InClassDimStorage> &
for_each_value(Fn &&f, Args &&...other_buffers) {
for_each_value_impl(f, std::forward<Args>(other_buffers)...);
return *this;
}
// @}
private:
// Helper functions for for_each_element
struct for_each_element_task_dim {
int min, max;
};
/** If f is callable with this many args, call it. The first
* argument is just to make the overloads distinct. Actual
* overload selection is done using the enable_if. */
template<typename Fn,
typename... Args,
typename = decltype(std::declval<Fn>()(std::declval<Args>()...))>
HALIDE_ALWAYS_INLINE static void for_each_element_variadic(int, int, const for_each_element_task_dim *, Fn &&f, Args... args) {
f(args...);
}
/** If the above overload is impossible, we add an outer loop over
* an additional argument and try again. */
template<typename Fn,
typename... Args>
HALIDE_ALWAYS_INLINE static void for_each_element_variadic(double, int d, const for_each_element_task_dim *t, Fn &&f, Args... args) {
for (int i = t[d].min; i <= t[d].max; i++) {
for_each_element_variadic(0, d - 1, t, std::forward<Fn>(f), i, args...);
}
}
/** Determine the minimum number of arguments a callable can take
* using the same trick. */
template<typename Fn,
typename... Args,
typename = decltype(std::declval<Fn>()(std::declval<Args>()...))>
HALIDE_ALWAYS_INLINE static int num_args(int, Fn &&, Args...) {
return (int)(sizeof...(Args));
}
/** The recursive version is only enabled up to a recursion limit
* of 256. This catches callables that aren't callable with any
* number of ints. */
template<typename Fn,
typename... Args>
HALIDE_ALWAYS_INLINE static int num_args(double, Fn &&f, Args... args) {
static_assert(sizeof...(args) <= 256,
"Callable passed to for_each_element must accept either a const int *,"
" or up to 256 ints. No such operator found. Expect infinite template recursion.");
return num_args(0, std::forward<Fn>(f), 0, args...);
}
/** A version where the callable takes a position array instead,
* with compile-time recursion on the dimensionality. This
* overload is preferred to the one below using the same int vs
* double trick as above, but is impossible once d hits -1 using
* std::enable_if. */
template<int d,
typename Fn,
typename = typename std::enable_if<(d >= 0)>::type>
HALIDE_ALWAYS_INLINE static void for_each_element_array_helper(int, const for_each_element_task_dim *t, Fn &&f, int *pos) {
for (pos[d] = t[d].min; pos[d] <= t[d].max; pos[d]++) {
for_each_element_array_helper<d - 1>(0, t, std::forward<Fn>(f), pos);
}
}
/** Base case for recursion above. */
template<int d,
typename Fn,
typename = typename std::enable_if<(d < 0)>::type>
HALIDE_ALWAYS_INLINE static void for_each_element_array_helper(double, const for_each_element_task_dim *t, Fn &&f, int *pos) {
f(pos);
}
/** A run-time-recursive version (instead of
* compile-time-recursive) that requires the callable to take a
* pointer to a position array instead. Dispatches to the
* compile-time-recursive version once the dimensionality gets
* small. */
template<typename Fn>
static void for_each_element_array(int d, const for_each_element_task_dim *t, Fn &&f, int *pos) {
if (d == -1) {
f(pos);
} else if (d == 0) {
// Once the dimensionality gets small enough, dispatch to
// a compile-time-recursive version for better codegen of
// the inner loops.
for_each_element_array_helper<0, Fn>(0, t, std::forward<Fn>(f), pos);
} else if (d == 1) {
for_each_element_array_helper<1, Fn>(0, t, std::forward<Fn>(f), pos);
} else if (d == 2) {
for_each_element_array_helper<2, Fn>(0, t, std::forward<Fn>(f), pos);
} else if (d == 3) {
for_each_element_array_helper<3, Fn>(0, t, std::forward<Fn>(f), pos);
} else {
for (pos[d] = t[d].min; pos[d] <= t[d].max; pos[d]++) {
for_each_element_array(d - 1, t, std::forward<Fn>(f), pos);
}
}
}
/** We now have two overloads for for_each_element. This one
* triggers if the callable takes a const int *.
*/
template<typename Fn,
typename = decltype(std::declval<Fn>()((const int *)nullptr))>
static void for_each_element(int, int dims, const for_each_element_task_dim *t, Fn &&f, int check = 0) {
const int size = dims * sizeof(int);
int *pos = (int *)HALIDE_ALLOCA(size);
// At least one version of GCC will (incorrectly) report that pos "may be used uninitialized".
// Add this memset to silence it.
memset(pos, 0, size);
for_each_element_array(dims - 1, t, std::forward<Fn>(f), pos);
}
/** This one triggers otherwise. It treats the callable as
* something that takes some number of ints. */
template<typename Fn>
HALIDE_ALWAYS_INLINE static void for_each_element(double, int dims, const for_each_element_task_dim *t, Fn &&f) {
int args = num_args(0, std::forward<Fn>(f));
assert(dims >= args);
for_each_element_variadic(0, args - 1, t, std::forward<Fn>(f));
}
template<typename Fn>
void for_each_element_impl(Fn &&f) const {
for_each_element_task_dim *t =
(for_each_element_task_dim *)HALIDE_ALLOCA(dimensions() * sizeof(for_each_element_task_dim));
for (int i = 0; i < dimensions(); i++) {
t[i].min = dim(i).min();
t[i].max = dim(i).max();
}
for_each_element(0, dimensions(), t, std::forward<Fn>(f));
}
public:
/** Call a function at each site in a buffer. This is likely to be
* much slower than using Halide code to populate a buffer, but is
* convenient for tests. If the function has more arguments than the
* buffer has dimensions, the remaining arguments will be zero. If it
* has fewer arguments than the buffer has dimensions then the last
* few dimensions of the buffer are not iterated over. For example,
* the following code exploits this to set a floating point RGB image
* to red:
\code
Buffer<float, 3> im(100, 100, 3);
im.for_each_element([&](int x, int y) {
im(x, y, 0) = 1.0f;
im(x, y, 1) = 0.0f;
im(x, y, 2) = 0.0f:
});
\endcode
* The compiled code is equivalent to writing the a nested for loop,
* and compilers are capable of optimizing it in the same way.
*
* If the callable can be called with an int * as the sole argument,
* that version is called instead. Each location in the buffer is
* passed to it in a coordinate array. This version is higher-overhead
* than the variadic version, but is useful for writing generic code
* that accepts buffers of arbitrary dimensionality. For example, the
* following sets the value at all sites in an arbitrary-dimensional
* buffer to their first coordinate:
\code
im.for_each_element([&](const int *pos) {im(pos) = pos[0];});
\endcode
* It is also possible to use for_each_element to iterate over entire
* rows or columns by cropping the buffer to a single column or row
* respectively and iterating over elements of the result. For example,
* to set the diagonal of the image to 1 by iterating over the columns:
\code
Buffer<float, 3> im(100, 100, 3);
im.sliced(1, 0).for_each_element([&](int x, int c) {
im(x, x, c) = 1.0f;
});
\endcode
* Or, assuming the memory layout is known to be dense per row, one can
* memset each row of an image like so:
\code
Buffer<float, 3> im(100, 100, 3);
im.sliced(0, 0).for_each_element([&](int y, int c) {
memset(&im(0, y, c), 0, sizeof(float) * im.width());
});
\endcode
*/
// @{
template<typename Fn>
HALIDE_ALWAYS_INLINE const Buffer<T, Dims, InClassDimStorage> &for_each_element(Fn &&f) const {
for_each_element_impl(f);
return *this;
}
template<typename Fn>
HALIDE_ALWAYS_INLINE
Buffer<T, Dims, InClassDimStorage> &
for_each_element(Fn &&f) {
for_each_element_impl(f);
return *this;
}
// @}
private:
template<typename Fn>
struct FillHelper {
Fn f;
Buffer<T, Dims, InClassDimStorage> *buf;
template<typename... Args,
typename = decltype(std::declval<Fn>()(std::declval<Args>()...))>
void operator()(Args... args) {
(*buf)(args...) = f(args...);
}
FillHelper(Fn &&f, Buffer<T, Dims, InClassDimStorage> *buf)
: f(std::forward<Fn>(f)), buf(buf) {
}
};
public:
/** Fill a buffer by evaluating a callable at every site. The
* callable should look much like a callable passed to
* for_each_element, but it should return the value that should be
* stored to the coordinate corresponding to the arguments. */
template<typename Fn,
typename = typename std::enable_if<!std::is_arithmetic<typename std::decay<Fn>::type>::value>::type>
Buffer<T, Dims, InClassDimStorage> &fill(Fn &&f) {
// We'll go via for_each_element. We need a variadic wrapper lambda.
FillHelper<Fn> wrapper(std::forward<Fn>(f), this);
return for_each_element(wrapper);
}
/** Check if an input buffer passed extern stage is a querying
* bounds. Compared to doing the host pointer check directly,
* this both adds clarity to code and will facilitate moving to
* another representation for bounds query arguments. */
bool is_bounds_query() const {
return buf.is_bounds_query();
}
/** Convenient check to verify that all of the interesting bytes in the Buffer
* are initialized under MSAN. Note that by default, we use for_each_value() here so that
* we skip any unused padding that isn't part of the Buffer; this isn't efficient,
* but in MSAN mode, it doesn't matter. (Pass true for the flag to force check
* the entire Buffer storage.) */
void msan_check_mem_is_initialized(bool entire = false) const {
#if defined(__has_feature)
#if __has_feature(memory_sanitizer)
if (entire) {
__msan_check_mem_is_initialized(data(), size_in_bytes());
} else {
for_each_value([](T &v) { __msan_check_mem_is_initialized(&v, sizeof(T)); ; });
}
#endif
#endif
}
};
} // namespace Runtime
} // namespace Halide
#undef HALIDE_ALLOCA
#endif // HALIDE_RUNTIME_IMAGE_H
HALIDE_RUNTIME_BUFFER_ALLOCATION_ALIGNMENT
#define HALIDE_RUNTIME_BUFFER_ALLOCATION_ALIGNMENT
Definition HalideBuffer.h:53
HALIDE_ALLOCA
#define HALIDE_ALLOCA
Definition HalideBuffer.h:37
HalideRuntime.h
This file declares the routines used by Halide internally in its runtime.
halide_dimension_t
struct halide_dimension_t halide_dimension_t
HALIDE_NEVER_INLINE
#define HALIDE_NEVER_INLINE
Definition HalideRuntime.h:50
halide_error_code_success
@ halide_error_code_success
There was no error.
Definition HalideRuntime.h:1076
HALIDE_ALWAYS_INLINE
#define HALIDE_ALWAYS_INLINE
Definition HalideRuntime.h:49
halide_buffer_t
struct halide_buffer_t halide_buffer_t
The raw representation of an image passed around by generated Halide code.
Halide::Buffer<>
Halide::PyTorch::Buffer::Dimension::Dimension
Dimension(const halide_dimension_t &dim)
Definition HalideBuffer.h:627
Halide::PyTorch::Buffer::type
halide_type_t type() const
Get the type of the elements.
Definition HalideBuffer.h:667
Halide::PyTorch::Buffer::Buffer
Buffer()
Definition HalideBuffer.h:696
Halide::Runtime::Buffer::Dimension
Read-only access to the shape.
Definition HalideBuffer.h:576
Halide::Runtime::Buffer::Dimension::min
HALIDE_ALWAYS_INLINE int min() const
The lowest coordinate in this dimension.
Definition HalideBuffer.h:581
Halide::Runtime::Buffer::Dimension::Dimension
Dimension(const halide_dimension_t &dim)
Definition HalideBuffer.h:627
Halide::Runtime::Buffer::Dimension::max
HALIDE_ALWAYS_INLINE int max() const
The highest coordinate in this dimension.
Definition HalideBuffer.h:597
Halide::Runtime::Buffer::Dimension::end
HALIDE_ALWAYS_INLINE iterator end() const
An iterator that points to one past the max coordinate.
Definition HalideBuffer.h:623
Halide::Runtime::Buffer::Dimension::stride
HALIDE_ALWAYS_INLINE int stride() const
The number of elements in memory you have to step over to increment this coordinate by one.
Definition HalideBuffer.h:587
Halide::Runtime::Buffer::Dimension::begin
HALIDE_ALWAYS_INLINE iterator begin() const
An iterator that points to the min coordinate.
Definition HalideBuffer.h:618
Halide::Runtime::Buffer::Dimension::extent
HALIDE_ALWAYS_INLINE int extent() const
The extent of the image along this dimension.
Definition HalideBuffer.h:592
Halide::Runtime::Buffer
A templated Buffer class that wraps halide_buffer_t and adds functionality.
Definition HalideBuffer.h:224
Halide::Runtime::Buffer::operator=
Buffer< T, Dims, InClassDimStorage > & operator=(const Buffer< T2, D2, S2 > &other)
Assign from another Buffer of possibly-different dimensionality and type.
Definition HalideBuffer.h:820
Halide::Runtime::Buffer::copy_to_planar
Buffer< not_const_T, Dims, InClassDimStorage > copy_to_planar(void *(*allocate_fn)(size_t)=nullptr, void(*deallocate_fn)(void *)=nullptr) const
Like copy(), but the copy is created in planar memory layout (vs.
Definition HalideBuffer.h:1371
Halide::Runtime::Buffer::transposed
Buffer< T, Dims, InClassDimStorage > transposed(const std::vector< int > &order) const
Make a buffer which refers to the same data in the same layout using a different ordering of the dime...
Definition HalideBuffer.h:1654
Halide::Runtime::Buffer::height
int height() const
Definition HalideBuffer.h:1304
Halide::Runtime::Buffer::translate
void translate(int d, int delta)
Translate an image in-place along one dimension by changing how it is indexed.
Definition HalideBuffer.h:1549
Halide::Runtime::Buffer::Buffer
Buffer(const halide_buffer_t &buf, BufferDeviceOwnership ownership=BufferDeviceOwnership::Unmanaged)
Make a Buffer from a halide_buffer_t.
Definition HalideBuffer.h:706
Halide::Runtime::Buffer::allocate
void allocate(void *(*allocate_fn)(size_t)=nullptr, void(*deallocate_fn)(void *)=nullptr)
Allocate memory for this Buffer.
Definition HalideBuffer.h:897
Halide::Runtime::Buffer::copy
Buffer< not_const_T, Dims, InClassDimStorage > copy(void *(*allocate_fn)(size_t)=nullptr, void(*deallocate_fn)(void *)=nullptr) const
Make a new image which is a deep copy of this image.
Definition HalideBuffer.h:1346
Halide::Runtime::Buffer::embedded
Buffer< T,(Dims==AnyDims ? AnyDims :Dims+1)> embedded(int d, int pos=0) const
Make a new buffer that views this buffer as a single slice in a higher-dimensional space.
Definition HalideBuffer.h:1727
Halide::Runtime::Buffer::Buffer
friend class Buffer
Give Buffers access to the members of Buffers of different dimensionalities and types.
Definition HalideBuffer.h:714
Halide::Runtime::Buffer::add_dimension
void add_dimension()
Add a new dimension with a min of zero and an extent of one.
Definition HalideBuffer.h:1749
Halide::Runtime::Buffer::ElemType
T ElemType
Definition HalideBuffer.h:573
Halide::Runtime::Buffer::slice
void slice(int d)
Slice a buffer in-place at the dimension's minimum.
Definition HalideBuffer.h:1712
Halide::Runtime::Buffer::set_default_allocate_fn
static void set_default_allocate_fn(void *(*allocate_fn)(size_t))
Definition HalideBuffer.h:730
Halide::Runtime::Buffer::owns_host_memory
bool owns_host_memory() const
Does this Buffer own the host memory it refers to?
Definition HalideBuffer.h:272
Halide::Runtime::Buffer::width
int width() const
Conventional names for the first three dimensions.
Definition HalideBuffer.h:1301
Halide::Runtime::Buffer::right
int right() const
Definition HalideBuffer.h:1318
Halide::Runtime::Buffer::transpose
void transpose(const std::vector< int > &order)
A generalized transpose: instead of swapping two dimensions, pass a vector that lists each dimension ...
Definition HalideBuffer.h:1636
Halide::Runtime::Buffer::set_min
void set_min(const std::vector< int > &mins)
Set the min coordinate of an image in the first N dimensions.
Definition HalideBuffer.h:1577
Halide::Runtime::Buffer::for_each_element
HALIDE_ALWAYS_INLINE Buffer< T, Dims, InClassDimStorage > & for_each_element(Fn &&f)
Definition HalideBuffer.h:2551
Halide::Runtime::Buffer::Buffer
Buffer(halide_type_t t, add_const_if_T_is_const< void > *data, const std::vector< int > &sizes)
Initialize an Buffer of runtime type from a pointer and a vector of sizes.
Definition HalideBuffer.h:1133
Halide::Runtime::Buffer::as
HALIDE_ALWAYS_INLINE Buffer< T2, D2, InClassDimStorage > as() &&
Return an rval reference to this Buffer.
Definition HalideBuffer.h:1244
Halide::Runtime::Buffer::copy_to_host
int copy_to_host(void *ctx=nullptr)
Definition HalideBuffer.h:1813
Halide::Runtime::Buffer::Buffer
Buffer(halide_type_t t, const std::vector< int > &sizes)
Allocate a new image of unknown type using a vector of ints as the size.
Definition HalideBuffer.h:1031
Halide::Runtime::Buffer::device_malloc
int device_malloc(const struct halide_device_interface_t *device_interface, void *ctx=nullptr)
Definition HalideBuffer.h:1827
Halide::Runtime::Buffer::device_free
int device_free(void *ctx=nullptr)
Definition HalideBuffer.h:1831
Halide::Runtime::Buffer::extent
int extent(int i) const
Definition HalideBuffer.h:643
Halide::Runtime::Buffer::contains
bool contains(Args... args) const
Definition HalideBuffer.h:1604
Halide::Runtime::Buffer::crop
void crop(const std::vector< std::pair< int, int > > &rect)
Crop an image in-place along the first N dimensions.
Definition HalideBuffer.h:1525
Halide::Runtime::Buffer::as_const
HALIDE_ALWAYS_INLINE const Buffer< typename std::add_const< T >::type, Dims, InClassDimStorage > & as_const() const &
Definition HalideBuffer.h:1260
Halide::Runtime::Buffer::set_device_dirty
void set_device_dirty(bool v=true)
Definition HalideBuffer.h:1808
Halide::Runtime::Buffer::operator()
HALIDE_ALWAYS_INLINE const not_void_T & operator()(const int *pos) const
Definition HalideBuffer.h:2133
Halide::Runtime::Buffer::Buffer
Buffer(T *data, int d, const halide_dimension_t *shape)
Initialize an Buffer from a pointer to the min coordinate and an array describing the shape.
Definition HalideBuffer.h:1169
Halide::Runtime::Buffer::Buffer
Buffer(Buffer< T2, D2, S2 > &&other)
Move-construct a Buffer from a Buffer of different dimensionality and type.
Definition HalideBuffer.h:806
Halide::Runtime::Buffer::host_dirty
bool host_dirty() const
Definition HalideBuffer.h:1804
Halide::Runtime::Buffer::slice
void slice(int d, int pos)
Rewrite the buffer to refer to a single lower-dimensional slice of itself along the given dimension a...
Definition HalideBuffer.h:1696
Halide::Runtime::Buffer::top
int top() const
Definition HalideBuffer.h:1322
Halide::Runtime::Buffer::operator()
HALIDE_ALWAYS_INLINE const not_void_T & operator()(int first, Args... rest) const
Access elements.
Definition HalideBuffer.h:2112
Halide::Runtime::Buffer::set_host_dirty
HALIDE_ALWAYS_INLINE void set_host_dirty(bool v=true)
Methods for managing any GPU allocation.
Definition HalideBuffer.h:1791
Halide::Runtime::Buffer::msan_check_mem_is_initialized
void msan_check_mem_is_initialized(bool entire=false) const
Convenient check to verify that all of the interesting bytes in the Buffer are initialized under MSAN...
Definition HalideBuffer.h:2600
Halide::Runtime::Buffer::as_const
HALIDE_ALWAYS_INLINE Buffer< typename std::add_const< T >::type, Dims, InClassDimStorage > as_const() &&
Definition HalideBuffer.h:1265
Halide::Runtime::Buffer::operator=
Buffer< T, Dims, InClassDimStorage > & operator=(Buffer< T, Dims, InClassDimStorage > &&other) noexcept
Standard move-assignment operator.
Definition HalideBuffer.h:869
Halide::Runtime::Buffer::device_detach_native
int device_detach_native(void *ctx=nullptr)
Definition HalideBuffer.h:1862
Halide::Runtime::Buffer::device_wrap_native
int device_wrap_native(const struct halide_device_interface_t *device_interface, uint64_t handle, void *ctx=nullptr)
Definition HalideBuffer.h:1854
Halide::Runtime::Buffer::has_static_halide_type
static constexpr bool has_static_halide_type
True if the Halide type is not void (or const void).
Definition HalideBuffer.h:263
Halide::Runtime::Buffer::translated
Buffer< T, Dims, InClassDimStorage > translated(const std::vector< int > &delta) const
Make an image which refers to the same data translated along the first N dimensions.
Definition HalideBuffer.h:1557
Halide::Runtime::Buffer::dim
HALIDE_ALWAYS_INLINE Dimension dim(int i) const
Access the shape of the buffer.
Definition HalideBuffer.h:633
Halide::Runtime::Buffer::Buffer
Buffer(int first, int second, Args... rest)
Definition HalideBuffer.h:1015
Halide::Runtime::Buffer::as_const
HALIDE_ALWAYS_INLINE Buffer< typename std::add_const< T >::type, Dims, InClassDimStorage > & as_const() &
as_const() is syntactic sugar for .as<const T>(), to avoid the need to recapitulate the type argument...
Definition HalideBuffer.h:1253
Halide::Runtime::Buffer::transposed
Buffer< T, Dims, InClassDimStorage > transposed(int d1, int d2) const
Make a buffer which refers to the same data in the same layout using a swapped indexing order for the...
Definition HalideBuffer.h:1613
Halide::Runtime::Buffer::for_each_value
HALIDE_ALWAYS_INLINE Buffer< T, Dims, InClassDimStorage > & for_each_value(Fn &&f, Args &&...other_buffers)
Definition HalideBuffer.h:2348
Halide::Runtime::Buffer::operator()
HALIDE_ALWAYS_INLINE not_void_T & operator()()
Definition HalideBuffer.h:2153
Halide::Runtime::Buffer::device_ownership
BufferDeviceOwnership device_ownership() const
Return the method by which the device field is managed.
Definition HalideBuffer.h:1920
Halide::Runtime::Buffer::check_overflow
void check_overflow()
Check the product of the extents fits in memory.
Definition HalideBuffer.h:882
Halide::Runtime::Buffer::can_convert_from
static bool can_convert_from(const Buffer< T2, D2, S2 > &other)
Determine if a Buffer<T, Dims, InClassDimStorage> can be constructed from some other Buffer type.
Definition HalideBuffer.h:741
Halide::Runtime::Buffer::copy_to_interleaved
Buffer< not_const_T, Dims, InClassDimStorage > copy_to_interleaved(void *(*allocate_fn)(size_t)=nullptr, void(*deallocate_fn)(void *)=nullptr) const
Like copy(), but the copy is created in interleaved memory layout (vs.
Definition HalideBuffer.h:1357
Halide::Runtime::Buffer::device_and_host_malloc
int device_and_host_malloc(const struct halide_device_interface_t *device_interface, void *ctx=nullptr)
Definition HalideBuffer.h:1884
Halide::Runtime::Buffer::device_sync
int device_sync(void *ctx=nullptr)
Definition HalideBuffer.h:1911
Halide::Runtime::Buffer::make_interleaved
static Buffer< void, Dims, InClassDimStorage > make_interleaved(halide_type_t t, int width, int height, int channels)
If you use the (x, y, c) indexing convention, then Halide Buffers are stored planar by default.
Definition HalideBuffer.h:1934
Halide::Runtime::Buffer::Buffer
Buffer(const std::vector< int > &sizes)
Allocate a new image of known type using a vector of ints as the size.
Definition HalideBuffer.h:1046
Halide::Runtime::Buffer::bottom
int bottom() const
Definition HalideBuffer.h:1326
Halide::Runtime::Buffer::embed
void embed(int d, int pos=0)
Embed a buffer in-place, increasing the dimensionality.
Definition HalideBuffer.h:1735
Halide::Runtime::Buffer::static_halide_type
static constexpr halide_type_t static_halide_type()
Get the Halide type of T.
Definition HalideBuffer.h:267
Halide::Runtime::Buffer::Buffer
Buffer(T *data, int first, Args &&...rest)
Initialize an Buffer from a pointer and some sizes.
Definition HalideBuffer.h:1109
Halide::Runtime::Buffer::copy_to_device
int copy_to_device(const struct halide_device_interface_t *device_interface, void *ctx=nullptr)
Definition HalideBuffer.h:1820
Halide::Runtime::Buffer::Buffer
Buffer(Array(&vals)[N])
Make an Buffer that refers to a statically sized array.
Definition HalideBuffer.h:1078
Halide::Runtime::Buffer::raw_buffer
const halide_buffer_t * raw_buffer() const
Definition HalideBuffer.h:1199
Halide::Runtime::Buffer::operator()
HALIDE_ALWAYS_INLINE not_void_T & operator()(int first, Args... rest)
Definition HalideBuffer.h:2142
Halide::Runtime::Buffer::make_interleaved
static Buffer< T, Dims, InClassDimStorage > make_interleaved(int width, int height, int channels)
If you use the (x, y, c) indexing convention, then Halide Buffers are stored planar by default.
Definition HalideBuffer.h:1950
Halide::Runtime::Buffer::type
halide_type_t type() const
Get the type of the elements.
Definition HalideBuffer.h:667
Halide::Runtime::Buffer::device_and_host_free
int device_and_host_free(const struct halide_device_interface_t *device_interface, void *ctx=nullptr)
Definition HalideBuffer.h:1888
Halide::Runtime::Buffer::Buffer
Buffer(int first)
Allocate a new image of the given size.
Definition HalideBuffer.h:999
Halide::Runtime::Buffer::raw_buffer
halide_buffer_t * raw_buffer()
Get a pointer to the raw halide_buffer_t this wraps.
Definition HalideBuffer.h:1195
Halide::Runtime::Buffer::end
T * end() const
A pointer to one beyond the element with the highest address.
Definition HalideBuffer.h:679
Halide::Runtime::Buffer::device_dirty
HALIDE_ALWAYS_INLINE bool device_dirty() const
Definition HalideBuffer.h:1800
Halide::Runtime::Buffer::cropped
Buffer< T, Dims, InClassDimStorage > cropped(const std::vector< std::pair< int, int > > &rect) const
Make an image that refers to a sub-rectangle of this image along the first N dimensions.
Definition HalideBuffer.h:1504
Halide::Runtime::Buffer::static_dimensions
static constexpr int static_dimensions()
Callers should not use the result if has_static_dimensions is false.
Definition HalideBuffer.h:280
Halide::Runtime::Buffer::transpose
void transpose(int d1, int d2)
Transpose a buffer in-place by changing how it is indexed.
Definition HalideBuffer.h:1626
Halide::Runtime::Buffer::deallocate
void deallocate()
Drop reference to any owned host or device memory, possibly freeing it, if this buffer held the last ...
Definition HalideBuffer.h:961
Halide::Runtime::Buffer::size_in_bytes
size_t size_in_bytes() const
The total number of bytes spanned by the data in memory.
Definition HalideBuffer.h:685
Halide::Runtime::Buffer::has_device_allocation
bool has_device_allocation() const
Definition HalideBuffer.h:1915
Halide::Runtime::Buffer::reset
void reset()
Reset the Buffer to be equivalent to a default-constructed Buffer of the same static type (if any); B...
Definition HalideBuffer.h:692
Halide::Runtime::Buffer::channels
int channels() const
Definition HalideBuffer.h:1307
Halide::Runtime::Buffer::Buffer
Buffer(halide_type_t t, int first, Args... rest)
Allocate a new image of the given size with a runtime type.
Definition HalideBuffer.h:978
Halide::Runtime::Buffer::dimensions
int dimensions() const
Get the dimensionality of the buffer.
Definition HalideBuffer.h:658
Halide::Runtime::Buffer::~Buffer
~Buffer()
Destructor.
Definition HalideBuffer.h:1188
Halide::Runtime::Buffer::Buffer
Buffer(halide_type_t t, add_const_if_T_is_const< void > *data, int d, const halide_dimension_t *shape)
Initialize an Buffer from a pointer to the min coordinate and an array describing the shape.
Definition HalideBuffer.h:1146
Halide::Runtime::Buffer::min
int min(int i) const
Access to the mins, strides, extents.
Definition HalideBuffer.h:640
Halide::Runtime::Buffer::for_each_element
HALIDE_ALWAYS_INLINE const Buffer< T, Dims, InClassDimStorage > & for_each_element(Fn &&f) const
Call a function at each site in a buffer.
Definition HalideBuffer.h:2543
Halide::Runtime::Buffer::device_deallocate
void device_deallocate()
Drop reference to any owned device memory, possibly freeing it if this buffer held the last reference...
Definition HalideBuffer.h:968
Halide::Runtime::Buffer::operator()
HALIDE_ALWAYS_INLINE const not_void_T & operator()() const
Definition HalideBuffer.h:2122
Halide::Runtime::Buffer::make_scalar
static Buffer< T, Dims, InClassDimStorage > make_scalar()
Make a zero-dimensional Buffer.
Definition HalideBuffer.h:1978
Halide::Runtime::Buffer::add_dimension_with_stride
void add_dimension_with_stride(int s)
Add a new dimension with a min of zero, an extent of one, and the specified stride.
Definition HalideBuffer.h:1781
Halide::Runtime::Buffer::Buffer
Buffer(Buffer< T, Dims, InClassDimStorage > &&other) noexcept
Move constructor.
Definition HalideBuffer.h:793
Halide::Runtime::Buffer::cropped
Buffer< T, Dims, InClassDimStorage > cropped(int d, int min, int extent) const
Make an image that refers to a sub-range of this image along the given dimension.
Definition HalideBuffer.h:1468
Halide::Runtime::Buffer::crop
void crop(int d, int min, int extent)
Crop an image in-place along the given dimension.
Definition HalideBuffer.h:1488
Halide::Runtime::Buffer::fill
Buffer< T, Dims, InClassDimStorage > & fill(Fn &&f)
Fill a buffer by evaluating a callable at every site.
Definition HalideBuffer.h:2581
Halide::Runtime::Buffer::make_scalar
static Buffer< T, Dims, InClassDimStorage > make_scalar(T *data)
Make a zero-dimensional Buffer that points to non-owned, existing data.
Definition HalideBuffer.h:1986
Halide::Runtime::Buffer::alias
Buffer< T, Dims, InClassDimStorage > alias() const
Make a copy of the Buffer which shares the underlying host and/or device allocations as the existing ...
Definition HalideBuffer.h:1397
Halide::Runtime::Buffer::set_min
void set_min(Args... args)
Definition HalideBuffer.h:1586
Halide::Runtime::Buffer::number_of_elements
size_t number_of_elements() const
The total number of elements this buffer represents.
Definition HalideBuffer.h:653
Halide::Runtime::Buffer::assert_can_convert_from
static void assert_can_convert_from(const Buffer< T2, D2, S2 > &other)
Fail an assertion at runtime or compile-time if an Buffer<T, Dims, InClassDimStorage> cannot be const...
Definition HalideBuffer.h:759
Halide::Runtime::Buffer::translate
void translate(const std::vector< int > &delta)
Translate an image along the first N dimensions by changing how it is indexed.
Definition HalideBuffer.h:1565
Halide::Runtime::Buffer::Buffer
Buffer(const Buffer< T, Dims, InClassDimStorage > &other)
Copy constructor.
Definition HalideBuffer.h:768
Halide::Runtime::Buffer::operator()
HALIDE_ALWAYS_INLINE not_void_T & operator()(const int *pos)
Definition HalideBuffer.h:2164
Halide::Runtime::Buffer::data
T * data() const
Get a pointer to the address of the min coordinate.
Definition HalideBuffer.h:2099
Halide::Runtime::Buffer::fill
Buffer< T, Dims, InClassDimStorage > & fill(not_void_T val)
Definition HalideBuffer.h:2179
Halide::Runtime::Buffer::Buffer
Buffer(const std::vector< int > &sizes, const std::vector< int > &storage_order)
Definition HalideBuffer.h:1071
Halide::Runtime::Buffer::operator=
Buffer< T, Dims, InClassDimStorage > & operator=(Buffer< T2, D2, S2 > &&other)
Move from another Buffer of possibly-different dimensionality and type.
Definition HalideBuffer.h:855
Halide::Runtime::Buffer::Buffer
Buffer(halide_type_t t, const std::vector< int > &sizes, const std::vector< int > &storage_order)
Allocate a new image of unknown type using a vector of ints as the size and a vector of indices indic...
Definition HalideBuffer.h:1066
Halide::Runtime::Buffer::Buffer
Buffer(halide_type_t t, add_const_if_T_is_const< void > *data, const std::vector< halide_dimension_t > &shape)
Initialize a Buffer from a pointer to the min coordinate and a vector describing the shape.
Definition HalideBuffer.h:1161
Halide::Runtime::Buffer::has_static_dimensions
static constexpr bool has_static_dimensions
Definition HalideBuffer.h:276
Halide::Runtime::Buffer::sliced
Buffer< T,(Dims==AnyDims ? AnyDims :Dims - 1)> sliced(int d, int pos) const
Make a lower-dimensional buffer that refers to one slice of this buffer.
Definition HalideBuffer.h:1663
Halide::Runtime::Buffer::make_interleaved
static Buffer< add_const_if_T_is_const< void >, Dims, InClassDimStorage > make_interleaved(halide_type_t t, T *data, int width, int height, int channels)
Wrap an existing interleaved image.
Definition HalideBuffer.h:1956
Halide::Runtime::Buffer::for_each_value
HALIDE_ALWAYS_INLINE const Buffer< T, Dims, InClassDimStorage > & for_each_value(Fn &&f, Args &&...other_buffers) const
Call a function on every value in the buffer, and the corresponding values in some number of other bu...
Definition HalideBuffer.h:2340
Halide::Runtime::Buffer::is_bounds_query
bool is_bounds_query() const
Check if an input buffer passed extern stage is a querying bounds.
Definition HalideBuffer.h:2591
Halide::Runtime::Buffer::sliced
Buffer< T,(Dims==AnyDims ? AnyDims :Dims - 1)> sliced(int d) const
Make a lower-dimensional buffer that refers to one slice of this buffer at the dimension's minimum.
Definition HalideBuffer.h:1684
Halide::Runtime::Buffer::left
int left() const
Conventional names for the min and max value of each dimension.
Definition HalideBuffer.h:1314
Halide::Runtime::Buffer::copy_from
void copy_from(Buffer< T2, D2, S2 > src)
Fill a Buffer with the values at the same coordinates in another Buffer.
Definition HalideBuffer.h:1411
Halide::Runtime::Buffer::translated
Buffer< T, Dims, InClassDimStorage > translated(int d, int dx) const
Make an image which refers to the same data with using translated coordinates in the given dimension.
Definition HalideBuffer.h:1541
Halide::Runtime::Buffer::stride
int stride(int i) const
Definition HalideBuffer.h:646
Halide::Runtime::Buffer::make_interleaved
static Buffer< T, Dims, InClassDimStorage > make_interleaved(T *data, int width, int height, int channels)
Wrap an existing interleaved image.
Definition HalideBuffer.h:1965
Halide::Runtime::Buffer::set_default_deallocate_fn
static void set_default_deallocate_fn(void(*deallocate_fn)(void *))
Definition HalideBuffer.h:733
Halide::Runtime::Buffer::make_with_shape_of
static Buffer< T, Dims, InClassDimStorage > make_with_shape_of(Buffer< T2, D2, S2 > src, void *(*allocate_fn)(size_t)=nullptr, void(*deallocate_fn)(void *)=nullptr)
Make a buffer with the same shape and memory nesting order as another buffer.
Definition HalideBuffer.h:1997
Halide::Runtime::Buffer::Buffer
Buffer(const Buffer< T2, D2, S2 > &other)
Construct a Buffer from a Buffer of different dimensionality and type.
Definition HalideBuffer.h:783
Halide::Runtime::Buffer::contains
bool contains(const std::vector< int > &coords) const
Test if a given coordinate is within the bounds of an image.
Definition HalideBuffer.h:1593
Halide::Runtime::Buffer::Buffer
Buffer(T *data, const std::vector< halide_dimension_t > &shape)
Initialize a Buffer from a pointer to the min coordinate and a vector describing the shape.
Definition HalideBuffer.h:1181
Halide::Runtime::Buffer::Buffer
Buffer(T *data, const std::vector< int > &sizes)
Initialize an Buffer from a pointer and a vector of sizes.
Definition HalideBuffer.h:1122
Halide::Runtime::Buffer::operator=
Buffer< T, Dims, InClassDimStorage > & operator=(const Buffer< T, Dims, InClassDimStorage > &other)
Standard assignment operator.
Definition HalideBuffer.h:836
Halide::Runtime::Buffer::begin
T * begin() const
A pointer to the element with the lowest address.
Definition HalideBuffer.h:673
Halide::Runtime::Buffer::all_equal
bool all_equal(not_void_T val) const
Tests that all values in this buffer are equal to val.
Definition HalideBuffer.h:2173
Halide::Runtime::Buffer::Buffer
Buffer(halide_type_t t, add_const_if_T_is_const< void > *data, int first, Args &&...rest)
Initialize an Buffer of runtime type from a pointer and some sizes.
Definition HalideBuffer.h:1092
Halide::Runtime::Buffer::as
HALIDE_ALWAYS_INLINE Buffer< T2, D2, InClassDimStorage > & as() &
Return a typed reference to this Buffer.
Definition HalideBuffer.h:1218
Halide::Runtime::Buffer::as
HALIDE_ALWAYS_INLINE const Buffer< T2, D2, InClassDimStorage > & as() const &
Return a const typed reference to this Buffer.
Definition HalideBuffer.h:1231
Halide::Runtime::Buffer::Buffer
Buffer()
Definition HalideBuffer.h:696
Halide::Runtime::Buffer::make_scalar
static Buffer< add_const_if_T_is_const< void >, Dims, InClassDimStorage > make_scalar(halide_type_t t)
Make a zero-dimensional Buffer.
Definition HalideBuffer.h:1970
Halide::Runtime::Internal
Definition constants.h:13
Halide::Runtime::Internal::any_zero
bool any_zero(const Container &c)
Definition HalideBuffer.h:151
Halide::Runtime::AnyDims
constexpr int AnyDims
Definition HalideBuffer.h:195
Halide::Runtime::BufferDeviceOwnership
BufferDeviceOwnership
This indicates how to deallocate the device for a Halide::Runtime::Buffer.
Definition HalideBuffer.h:179
Halide::Runtime::BufferDeviceOwnership::Allocated
@ Allocated
Definition HalideBuffer.h:180
Halide::Runtime::BufferDeviceOwnership::AllocatedDeviceAndHost
@ AllocatedDeviceAndHost
Definition HalideBuffer.h:183
Halide::Runtime::BufferDeviceOwnership::WrappedNative
@ WrappedNative
Definition HalideBuffer.h:181
Halide::Runtime::BufferDeviceOwnership::Unmanaged
@ Unmanaged
Definition HalideBuffer.h:182
Halide::Runtime::BufferDeviceOwnership::Cropped
@ Cropped
Definition HalideBuffer.h:184
Halide
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
Definition AbstractGenerator.h:19
Halide::min
Expr min(const FuncRef &a, const FuncRef &b)
Explicit overloads of min and max for FuncRef.
Definition Func.h:603
Halide::max
Expr max(const FuncRef &a, const FuncRef &b)
Definition Func.h:606
uint64_t
unsigned __INT64_TYPE__ uint64_t
Definition runtime_internal.h:23
uintptr_t
__UINTPTR_TYPE__ uintptr_t
Definition runtime_internal.h:75
malloc
void * malloc(size_t)
align_up
ALWAYS_INLINE T align_up(T p, size_t alignment)
Definition runtime_internal.h:221
uint8_t
unsigned __INT8_TYPE__ uint8_t
Definition runtime_internal.h:29
ptrdiff_t
__PTRDIFF_TYPE__ ptrdiff_t
Definition runtime_internal.h:32
uint16_t
unsigned __INT16_TYPE__ uint16_t
Definition runtime_internal.h:27
memcpy
void * memcpy(void *s1, const void *s2, size_t n)
size_t
__SIZE_TYPE__ size_t
Definition runtime_internal.h:31
memset
void * memset(void *s, int val, size_t n)
uint32_t
unsigned __INT32_TYPE__ uint32_t
Definition runtime_internal.h:25
free
void free(void *)
Halide::PyTorch::Buffer::Dimension::iterator
An iterator class, so that you can iterate over coordinates in a dimensions using a range-based for l...
Definition HalideBuffer.h:603
Halide::Runtime::AllInts< T, Args... >::value
static const bool value
Definition HalideBuffer.h:136
Halide::Runtime::AllInts
Definition HalideBuffer.h:129
Halide::Runtime::AllocationHeader
A struct acting as a header for allocations owned by the Buffer class itself.
Definition HalideBuffer.h:168
Halide::Runtime::AllocationHeader::ref_count
std::atomic< int > ref_count
Definition HalideBuffer.h:170
Halide::Runtime::AllocationHeader::AllocationHeader
AllocationHeader(void(*deallocate_fn)(void *))
Definition HalideBuffer.h:173
Halide::Runtime::AllocationHeader::deallocate_fn
void(* deallocate_fn)(void *)
Definition HalideBuffer.h:169
Halide::Runtime::Buffer::Dimension::iterator
An iterator class, so that you can iterate over coordinates in a dimensions using a range-based for l...
Definition HalideBuffer.h:603
Halide::Runtime::Buffer::Dimension::iterator::operator!=
bool operator!=(const iterator &other) const
Definition HalideBuffer.h:608
Halide::Runtime::Buffer::Dimension::iterator::operator++
iterator & operator++()
Definition HalideBuffer.h:611
Halide::Runtime::Buffer::Dimension::iterator::val
int val
Definition HalideBuffer.h:604
Halide::Runtime::Buffer::Dimension::iterator::operator*
int operator*() const
Definition HalideBuffer.h:605
Halide::Runtime::DeviceRefCount
A similar struct for managing device allocations.
Definition HalideBuffer.h:188
Halide::Runtime::DeviceRefCount::ownership
BufferDeviceOwnership ownership
Definition HalideBuffer.h:192
Halide::Runtime::DeviceRefCount::count
std::atomic< int > count
Definition HalideBuffer.h:191
Halide::Runtime::Internal::DefaultAllocatorFns
Definition HalideBuffer.h:160
Halide::Runtime::Internal::DefaultAllocatorFns::default_allocate_fn
static void *(* default_allocate_fn)(size_t)
Definition HalideBuffer.h:161
Halide::Runtime::Internal::DefaultAllocatorFns::default_deallocate_fn
static void(* default_deallocate_fn)(void *)
Definition HalideBuffer.h:162
halide_buffer_t
The raw representation of an image passed around by generated Halide code.
Definition HalideRuntime.h:1569
halide_buffer_t::dimensions
int32_t dimensions
The dimensionality of the buffer.
Definition HalideRuntime.h:1588
halide_buffer_t::dim
halide_dimension_t * dim
The shape of the buffer.
Definition HalideRuntime.h:1592
halide_buffer_t::device
uint64_t device
A device-handle for e.g.
Definition HalideRuntime.h:1571
halide_buffer_t::host
uint8_t * host
A pointer to the start of the data in main memory.
Definition HalideRuntime.h:1579
halide_device_interface_t
Each GPU API provides a halide_device_interface_t struct pointing to the code that manages device all...
Definition HalideRuntime.h:797
halide_device_interface_t::device_and_host_malloc
int(* device_and_host_malloc)(void *user_context, struct halide_buffer_t *buf, const struct halide_device_interface_t *device_interface)
Definition HalideRuntime.h:807
halide_device_interface_t::wrap_native
int(* wrap_native)(void *user_context, struct halide_buffer_t *buf, uint64_t handle, const struct halide_device_interface_t *device_interface)
Definition HalideRuntime.h:817
halide_device_interface_t::copy_to_device
int(* copy_to_device)(void *user_context, struct halide_buffer_t *buf, const struct halide_device_interface_t *device_interface)
Definition HalideRuntime.h:805
halide_device_interface_t::device_malloc
int(* device_malloc)(void *user_context, struct halide_buffer_t *buf, const struct halide_device_interface_t *device_interface)
Definition HalideRuntime.h:798
halide_dimension_t
Definition HalideRuntime.h:1516
halide_dimension_t::extent
int32_t extent
Definition HalideRuntime.h:1539
halide_dimension_t::stride
int32_t stride
Definition HalideRuntime.h:1539
halide_type_t
A runtime tag for a type in the halide type system.
Definition HalideRuntime.h:503