cache_aligned_allocator.cpp
Go to the documentation of this file.
/*
Copyright (c) 2005-2020 Intel Corporation
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
#include "tbb/tbb_config.h"
#include "tbb/cache_aligned_allocator.h"
#include "tbb/tbb_allocator.h"
#include "tbb/tbb_exception.h"
#include "tbb_misc.h"
#include "dynamic_link.h"
#include <cstdlib>
#if _WIN32||_WIN64
#include "tbb/machine/windows_api.h"
#else
#include <dlfcn.h>
#endif /* _WIN32||_WIN64 */
#if __TBB_WEAK_SYMBOLS_PRESENT
#pragma weak scalable_malloc
#pragma weak scalable_free
#pragma weak scalable_aligned_malloc
#pragma weak scalable_aligned_free
extern "C" {
void* scalable_malloc( size_t );
void scalable_free( void* );
void* scalable_aligned_malloc( size_t, size_t );
void scalable_aligned_free( void* );
}
#endif /* __TBB_WEAK_SYMBOLS_PRESENT */
namespace tbb {
namespace internal {
static void* DummyMalloc( size_t size );
static void DummyFree( void * ptr );
static void* (*MallocHandler)( size_t size ) = &DummyMalloc;
static void (*FreeHandler)( void* pointer ) = &DummyFree;
static void* dummy_padded_allocate( size_t bytes, size_t alignment );
static void dummy_padded_free( void * ptr );
// ! Allocates memory using standard malloc. It is used when scalable_allocator is not available
static void* padded_allocate( size_t bytes, size_t alignment );
// ! Allocates memory using standard free. It is used when scalable_allocator is not available
static void padded_free( void* p );
static void* (*padded_allocate_handler)( size_t bytes, size_t alignment ) = &dummy_padded_allocate;
static void (*padded_free_handler)( void* p ) = &dummy_padded_free;
static const dynamic_link_descriptor MallocLinkTable[] = {
DLD(scalable_malloc, MallocHandler),
DLD(scalable_free, FreeHandler),
DLD(scalable_aligned_malloc, padded_allocate_handler),
DLD(scalable_aligned_free, padded_free_handler),
};
#if TBB_USE_DEBUG
#define DEBUG_SUFFIX "_debug"
#else
#define DEBUG_SUFFIX
#endif /* TBB_USE_DEBUG */
// MALLOCLIB_NAME is the name of the TBB memory allocator library.
#if _WIN32||_WIN64
#define MALLOCLIB_NAME "tbbmalloc" DEBUG_SUFFIX ".dll"
#elif __APPLE__
#define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX ".dylib"
#elif __FreeBSD__ || __NetBSD__ || __OpenBSD__ || __sun || _AIX || __ANDROID__
#define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX ".so"
#elif __linux__ // Note that order of these #elif's is important!
#define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX __TBB_STRING(.so.TBB_COMPATIBLE_INTERFACE_VERSION)
#else
#error Unknown OS
#endif
void initialize_handler_pointers() {
__TBB_ASSERT( MallocHandler==&DummyMalloc, NULL );
bool success = dynamic_link( MALLOCLIB_NAME, MallocLinkTable, 4 );
if( !success ) {
// If unsuccessful, set the handlers to the default routines.
// This must be done now, and not before FillDynamicLinks runs, because if other
// threads call the handlers, we want them to go through the DoOneTimeInitializations logic,
// which forces them to wait.
FreeHandler = &std::free;
MallocHandler = &std::malloc;
padded_allocate_handler = &padded_allocate;
padded_free_handler = &padded_free;
}
#if !__TBB_RML_STATIC
PrintExtraVersionInfo( "ALLOCATOR", success?"scalable_malloc":"malloc" );
#endif
}
static tbb::atomic<do_once_state> initialization_state;
void initialize_cache_aligned_allocator() {
atomic_do_once( &initialize_handler_pointers, initialization_state );
}
static void* DummyMalloc( size_t size ) {
initialize_cache_aligned_allocator();
__TBB_ASSERT( MallocHandler!=&DummyMalloc, NULL );
return (*MallocHandler)( size );
}
static void DummyFree( void * ptr ) {
initialize_cache_aligned_allocator();
__TBB_ASSERT( FreeHandler!=&DummyFree, NULL );
(*FreeHandler)( ptr );
}
static void* dummy_padded_allocate( size_t bytes, size_t alignment ) {
initialize_cache_aligned_allocator();
__TBB_ASSERT( padded_allocate_handler!=&dummy_padded_allocate, NULL );
return (*padded_allocate_handler)(bytes, alignment);
}
static void dummy_padded_free( void * ptr ) {
initialize_cache_aligned_allocator();
__TBB_ASSERT( padded_free_handler!=&dummy_padded_free, NULL );
(*padded_free_handler)( ptr );
}
// TODO: use CPUID to find actual line size, though consider backward compatibility
static size_t NFS_LineSize = 128;
size_t NFS_GetLineSize() {
return NFS_LineSize;
}
#if _MSC_VER && !defined(__INTEL_COMPILER)
// unary minus operator applied to unsigned type, result still unsigned
#pragma warning( disable: 4146 4706 )
#endif
void* NFS_Allocate( size_t n, size_t element_size, void* /*hint*/ ) {
//TODO: make this functionality available via an adaptor over generic STL like allocator
const size_t nfs_cache_line_size = NFS_LineSize;
__TBB_ASSERT( nfs_cache_line_size <= NFS_MaxLineSize, "illegal value for NFS_LineSize" );
__TBB_ASSERT( is_power_of_two(nfs_cache_line_size), "must be power of two" );
size_t bytes = n*element_size;
if (bytes<n || bytes+nfs_cache_line_size<bytes) {
// Overflow
throw_exception(eid_bad_alloc);
}
// scalable_aligned_malloc considers zero size request an error, and returns NULL
if (bytes==0) bytes = 1;
void* result = (*padded_allocate_handler)( bytes, nfs_cache_line_size );
if (!result)
throw_exception(eid_bad_alloc);
__TBB_ASSERT( is_aligned(result, nfs_cache_line_size), "The address returned isn't aligned to cache line size" );
return result;
}
void NFS_Free( void* p ) {
(*padded_free_handler)( p );
}
static void* padded_allocate( size_t bytes, size_t alignment ) {
unsigned char* result = NULL;
unsigned char* base = (unsigned char*)std::malloc(alignment+bytes);
if( base ) {
// Round up to the next line
result = (unsigned char*)((uintptr_t)(base+alignment)&-alignment);
// Record where block actually starts.
((uintptr_t*)result)[-1] = uintptr_t(base);
}
return result;
}
static void padded_free( void* p ) {
if( p ) {
__TBB_ASSERT( (uintptr_t)p>=0x4096, "attempt to free block not obtained from cache_aligned_allocator" );
// Recover where block actually starts
unsigned char* base = ((unsigned char**)p)[-1];
__TBB_ASSERT( (void*)((uintptr_t)(base+NFS_LineSize)&-NFS_LineSize)==p, "not allocated by NFS_Allocate?" );
std::free(base);
}
}
void* __TBB_EXPORTED_FUNC allocate_via_handler_v3( size_t n ) {
void* result = (*MallocHandler) (n);
if (!result) {
throw_exception(eid_bad_alloc);
}
return result;
}
void __TBB_EXPORTED_FUNC deallocate_via_handler_v3( void *p ) {
if( p ) {
(*FreeHandler)( p );
}
}
bool __TBB_EXPORTED_FUNC is_malloc_used_v3() {
if (MallocHandler == &DummyMalloc) {
void* void_ptr = (*MallocHandler)(1);
(*FreeHandler)(void_ptr);
}
__TBB_ASSERT( MallocHandler!=&DummyMalloc && FreeHandler!=&DummyFree, NULL );
// Cast to void avoids type mismatch errors on some compilers (e.g. __IBMCPP__)
__TBB_ASSERT( !(((void*)MallocHandler==(void*)&std::malloc) ^ ((void*)FreeHandler==(void*)&std::free)),
"Both shim pointers must refer to routines from the same package (either TBB or CRT)" );
return (void*)MallocHandler == (void*)&std::malloc;
}
} // namespace internal
} // namespace tbb
cache_aligned_allocator.h
windows_api.h
__TBB_EXPORTED_FUNC
#define __TBB_EXPORTED_FUNC
Definition: scalable_allocator.h:38
tbb_allocator.h
tbb_config.h
tbb_exception.h
__TBB_ASSERT
#define __TBB_ASSERT(predicate, comment)
No-op version of __TBB_ASSERT.
Definition: tbb_stddef.h:165
dynamic_link
OPEN_INTERNAL_NAMESPACE bool dynamic_link(const char *, const dynamic_link_descriptor *, size_t, dynamic_link_handle *handle, int)
Definition: dynamic_link.cpp:564
dynamic_link.h
DLD
#define DLD(s, h)
The helper to construct dynamic_link_descriptor structure.
Definition: dynamic_link.h:56
tbb_misc.h
void
void
Definition: ittnotify_static.h:91
size
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t size
Definition: ittnotify_static.h:109
p
void const char const char int ITT_FORMAT __itt_group_sync p
Definition: ittnotify_static.h:91
scalable_malloc
void *__TBB_EXPORTED_FUNC scalable_malloc(size_t size)
scalable_free
void __TBB_EXPORTED_FUNC scalable_free(void *ptr)
tbb::internal::NFS_Allocate
void *__TBB_EXPORTED_FUNC NFS_Allocate(size_t n_element, size_t element_size, void *hint)
Allocate memory on cache/sector line boundary.
Definition: cache_aligned_allocator.cpp:176
tbb::internal::NFS_MaxLineSize
const size_t NFS_MaxLineSize
Compile-time constant that is upper bound on cache line/sector size.
Definition: tbb_stddef.h:216
tbb::internal::NFS_Free
void __TBB_EXPORTED_FUNC NFS_Free(void *)
Free memory allocated by NFS_Allocate.
Definition: cache_aligned_allocator.cpp:198
scalable_aligned_malloc
void *__TBB_EXPORTED_FUNC scalable_aligned_malloc(size_t size, size_t alignment)
tbb::internal::NFS_GetLineSize
size_t __TBB_EXPORTED_FUNC NFS_GetLineSize()
Cache/sector line size.
Definition: cache_aligned_allocator.cpp:167
scalable_aligned_free
void __TBB_EXPORTED_FUNC scalable_aligned_free(void *ptr)
tbb
The graph class.
Definition: serial/tbb/parallel_for.h:46
tbb::internal::DummyMalloc
static void * DummyMalloc(size_t size)
Dummy routine used for first indirect call via MallocHandler.
Definition: cache_aligned_allocator.cpp:137
tbb::internal::deallocate_via_handler_v3
void __TBB_EXPORTED_FUNC deallocate_via_handler_v3(void *p)
Deallocates memory using FreeHandler.
Definition: cache_aligned_allocator.cpp:232
tbb::internal::atomic_do_once
void atomic_do_once(const F &initializer, atomic< do_once_state > &state)
One-time initialization function.
Definition: tbb_misc.h:213
tbb::internal::dummy_padded_free
static void dummy_padded_free(void *ptr)
Dummy routine used for first indirect call via padded_free_handler.
Definition: cache_aligned_allocator.cpp:158
tbb::internal::initialize_cache_aligned_allocator
void initialize_cache_aligned_allocator()
Defined in cache_aligned_allocator.cpp.
Definition: cache_aligned_allocator.cpp:132
tbb::internal::MallocLinkTable
static const dynamic_link_descriptor MallocLinkTable[]
Table describing how to link the handlers.
Definition: cache_aligned_allocator.cpp:82
tbb::internal::padded_allocate
static void * padded_allocate(size_t bytes, size_t alignment)
Definition: cache_aligned_allocator.cpp:202
tbb::internal::throw_exception
void throw_exception(exception_id eid)
Versionless convenience wrapper for throw_exception_v4()
Definition: tbb_exception.h:105
tbb::internal::MallocHandler
static void *(* MallocHandler)(size_t size)
Handler for memory allocation.
Definition: cache_aligned_allocator.cpp:58
tbb::internal::NFS_LineSize
static size_t NFS_LineSize
Definition: cache_aligned_allocator.cpp:165
tbb::internal::PrintExtraVersionInfo
void PrintExtraVersionInfo(const char *category, const char *format,...)
Prints arbitrary extra TBB version information on stderr.
Definition: tbb_misc.cpp:211
tbb::internal::dummy_padded_allocate
static void * dummy_padded_allocate(size_t bytes, size_t alignment)
Dummy routine used for first indirect call via padded_allocate_handler.
Definition: cache_aligned_allocator.cpp:151
tbb::internal::initialization_state
static tbb::atomic< do_once_state > initialization_state
Definition: cache_aligned_allocator.cpp:131
tbb::internal::is_power_of_two
bool is_power_of_two(integer_type arg)
A function to check if passed integer is a power of 2.
Definition: tbb_stddef.h:376
tbb::internal::is_aligned
bool is_aligned(T *pointer, uintptr_t alignment)
A function to check if passed in pointer is aligned on a specific border.
Definition: tbb_stddef.h:370
tbb::internal::padded_allocate_handler
static void *(* padded_allocate_handler)(size_t bytes, size_t alignment)
Handler for padded memory allocation.
Definition: cache_aligned_allocator.cpp:76
tbb::internal::allocate_via_handler_v3
void *__TBB_EXPORTED_FUNC allocate_via_handler_v3(size_t n)
Allocates memory using MallocHandler.
Definition: cache_aligned_allocator.cpp:224
tbb::internal::DummyFree
static void DummyFree(void *ptr)
Dummy routine used for first indirect call via FreeHandler.
Definition: cache_aligned_allocator.cpp:144
tbb::internal::FreeHandler
static void(* FreeHandler)(void *pointer)
Handler for memory deallocation.
Definition: cache_aligned_allocator.cpp:61
tbb::internal::padded_free_handler
static void(* padded_free_handler)(void *p)
Handler for padded memory deallocation.
Definition: cache_aligned_allocator.cpp:79
tbb::internal::is_malloc_used_v3
bool __TBB_EXPORTED_FUNC is_malloc_used_v3()
Returns true if standard malloc/free are used to work with memory.
Definition: cache_aligned_allocator.cpp:238
tbb::internal::eid_bad_alloc
@ eid_bad_alloc
Definition: tbb_exception.h:68
tbb::internal::padded_free
static void padded_free(void *p)
Definition: cache_aligned_allocator.cpp:214
tbb::internal::initialize_handler_pointers
void initialize_handler_pointers()
Initialize the allocation/free handler pointers.
Definition: cache_aligned_allocator.cpp:113
internal
Definition: _flow_graph_async_msg_impl.h:24
dynamic_link_descriptor
Association between a handler name and location of pointer to it.
Definition: dynamic_link.h:60

Copyright © 2005-2020 Intel Corporation. All Rights Reserved.

Intel, Pentium, Intel Xeon, Itanium, Intel XScale and VTune are registered trademarks or trademarks of Intel Corporation or its subsidiaries in the United States and other countries.

* Other names and brands may be claimed as the property of others.