godot/thirdparty/embree/common/sys/alloc.cpp
jfons a69cc9f13d
Upgrade Embree to the latest official release.
Since Embree v3.13.0 supports AARCH64, switch back to the
official repo instead of using Embree-aarch64.

`thirdparty/embree/patches/godot-changes.patch` should now contain
an accurate diff of the changes done to the library.

(cherry picked from commit 767e374dce)
2021-05-22 15:14:07 +02:00

328 lines
8.5 KiB
C++

// Copyright 2009-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
#include "alloc.h"
#include "intrinsics.h"
#include "sysinfo.h"
#include "mutex.h"
////////////////////////////////////////////////////////////////////////////////
/// All Platforms
////////////////////////////////////////////////////////////////////////////////
namespace embree
{
void* alignedMalloc(size_t size, size_t align)
{
if (size == 0)
return nullptr;
assert((align & (align-1)) == 0);
void* ptr = _mm_malloc(size,align);
if (size != 0 && ptr == nullptr)
// -- GODOT start --
// throw std::bad_alloc();
abort();
// -- GODOT end --
return ptr;
}
void alignedFree(void* ptr)
{
if (ptr)
_mm_free(ptr);
}
static bool huge_pages_enabled = false;
static MutexSys os_init_mutex;
__forceinline bool isHugePageCandidate(const size_t bytes)
{
if (!huge_pages_enabled)
return false;
/* use huge pages only when memory overhead is low */
const size_t hbytes = (bytes+PAGE_SIZE_2M-1) & ~size_t(PAGE_SIZE_2M-1);
return 66*(hbytes-bytes) < bytes; // at most 1.5% overhead
}
}
////////////////////////////////////////////////////////////////////////////////
/// Windows Platform
////////////////////////////////////////////////////////////////////////////////
#ifdef _WIN32
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#include <malloc.h>
namespace embree
{
bool win_enable_selockmemoryprivilege (bool verbose)
{
HANDLE hToken;
if (!OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY | TOKEN_ADJUST_PRIVILEGES, &hToken)) {
if (verbose) std::cout << "WARNING: OpenProcessToken failed while trying to enable SeLockMemoryPrivilege: " << GetLastError() << std::endl;
return false;
}
TOKEN_PRIVILEGES tp;
tp.PrivilegeCount = 1;
tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
if (!LookupPrivilegeValueW(nullptr, L"SeLockMemoryPrivilege", &tp.Privileges[0].Luid)) {
if (verbose) std::cout << "WARNING: LookupPrivilegeValue failed while trying to enable SeLockMemoryPrivilege: " << GetLastError() << std::endl;
return false;
}
SetLastError(ERROR_SUCCESS);
if (!AdjustTokenPrivileges(hToken, FALSE, &tp, sizeof(tp), nullptr, 0)) {
if (verbose) std::cout << "WARNING: AdjustTokenPrivileges failed while trying to enable SeLockMemoryPrivilege" << std::endl;
return false;
}
if (GetLastError() == ERROR_NOT_ALL_ASSIGNED) {
if (verbose) std::cout << "WARNING: AdjustTokenPrivileges failed to enable SeLockMemoryPrivilege: Add SeLockMemoryPrivilege for current user and run process in elevated mode (Run as administrator)." << std::endl;
return false;
}
return true;
}
bool os_init(bool hugepages, bool verbose)
{
Lock<MutexSys> lock(os_init_mutex);
if (!hugepages) {
huge_pages_enabled = false;
return true;
}
if (GetLargePageMinimum() != PAGE_SIZE_2M) {
huge_pages_enabled = false;
return false;
}
huge_pages_enabled = true;
return true;
}
void* os_malloc(size_t bytes, bool& hugepages)
{
if (bytes == 0) {
hugepages = false;
return nullptr;
}
/* try direct huge page allocation first */
if (isHugePageCandidate(bytes))
{
int flags = MEM_COMMIT | MEM_RESERVE | MEM_LARGE_PAGES;
char* ptr = (char*) VirtualAlloc(nullptr,bytes,flags,PAGE_READWRITE);
if (ptr != nullptr) {
hugepages = true;
return ptr;
}
}
/* fall back to 4k pages */
int flags = MEM_COMMIT | MEM_RESERVE;
char* ptr = (char*) VirtualAlloc(nullptr,bytes,flags,PAGE_READWRITE);
// -- GODOT start --
// if (ptr == nullptr) throw std::bad_alloc();
if (ptr == nullptr) abort();
// -- GODOT end --
hugepages = false;
return ptr;
}
size_t os_shrink(void* ptr, size_t bytesNew, size_t bytesOld, bool hugepages)
{
if (hugepages) // decommitting huge pages seems not to work under Windows
return bytesOld;
const size_t pageSize = hugepages ? PAGE_SIZE_2M : PAGE_SIZE_4K;
bytesNew = (bytesNew+pageSize-1) & ~(pageSize-1);
bytesOld = (bytesOld+pageSize-1) & ~(pageSize-1);
if (bytesNew >= bytesOld)
return bytesOld;
if (!VirtualFree((char*)ptr+bytesNew,bytesOld-bytesNew,MEM_DECOMMIT))
// -- GODOT start --
// throw std::bad_alloc();
abort();
// -- GODOT end --
return bytesNew;
}
void os_free(void* ptr, size_t bytes, bool hugepages)
{
if (bytes == 0)
return;
if (!VirtualFree(ptr,0,MEM_RELEASE))
// -- GODOT start --
// throw std::bad_alloc();
abort();
// -- GODOT end --
}
void os_advise(void *ptr, size_t bytes)
{
}
}
#endif
////////////////////////////////////////////////////////////////////////////////
/// Unix Platform
////////////////////////////////////////////////////////////////////////////////
#if defined(__UNIX__)
#include <sys/mman.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include <sstream>
#if defined(__MACOSX__)
#include <mach/vm_statistics.h>
#endif
namespace embree
{
bool os_init(bool hugepages, bool verbose)
{
Lock<MutexSys> lock(os_init_mutex);
if (!hugepages) {
huge_pages_enabled = false;
return true;
}
#if defined(__LINUX__)
int hugepagesize = 0;
std::ifstream file;
file.open("/proc/meminfo",std::ios::in);
if (!file.is_open()) {
if (verbose) std::cout << "WARNING: Could not open /proc/meminfo. Huge page support cannot get enabled!" << std::endl;
huge_pages_enabled = false;
return false;
}
std::string line;
while (getline(file,line))
{
std::stringstream sline(line);
while (!sline.eof() && sline.peek() == ' ') sline.ignore();
std::string tag; getline(sline,tag,' ');
while (!sline.eof() && sline.peek() == ' ') sline.ignore();
std::string val; getline(sline,val,' ');
while (!sline.eof() && sline.peek() == ' ') sline.ignore();
std::string unit; getline(sline,unit,' ');
if (tag == "Hugepagesize:" && unit == "kB") {
hugepagesize = std::stoi(val)*1024;
break;
}
}
if (hugepagesize != PAGE_SIZE_2M)
{
if (verbose) std::cout << "WARNING: Only 2MB huge pages supported. Huge page support cannot get enabled!" << std::endl;
huge_pages_enabled = false;
return false;
}
#endif
huge_pages_enabled = true;
return true;
}
void* os_malloc(size_t bytes, bool& hugepages)
{
if (bytes == 0) {
hugepages = false;
return nullptr;
}
/* try direct huge page allocation first */
if (isHugePageCandidate(bytes))
{
#if defined(__MACOSX__)
void* ptr = mmap(0, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, VM_FLAGS_SUPERPAGE_SIZE_2MB, 0);
if (ptr != MAP_FAILED) {
hugepages = true;
return ptr;
}
#elif defined(MAP_HUGETLB)
void* ptr = mmap(0, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON | MAP_HUGETLB, -1, 0);
if (ptr != MAP_FAILED) {
hugepages = true;
return ptr;
}
#endif
}
/* fallback to 4k pages */
void* ptr = (char*) mmap(0, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
// -- GODOT start --
// if (ptr == MAP_FAILED) throw std::bad_alloc();
if (ptr == MAP_FAILED) abort();
// -- GODOT end --
hugepages = false;
/* advise huge page hint for THP */
os_advise(ptr,bytes);
return ptr;
}
size_t os_shrink(void* ptr, size_t bytesNew, size_t bytesOld, bool hugepages)
{
const size_t pageSize = hugepages ? PAGE_SIZE_2M : PAGE_SIZE_4K;
bytesNew = (bytesNew+pageSize-1) & ~(pageSize-1);
bytesOld = (bytesOld+pageSize-1) & ~(pageSize-1);
if (bytesNew >= bytesOld)
return bytesOld;
if (munmap((char*)ptr+bytesNew,bytesOld-bytesNew) == -1)
// -- GODOT start --
// throw std::bad_alloc();
abort();
// -- GODOT end --
return bytesNew;
}
void os_free(void* ptr, size_t bytes, bool hugepages)
{
if (bytes == 0)
return;
/* for hugepages we need to also align the size */
const size_t pageSize = hugepages ? PAGE_SIZE_2M : PAGE_SIZE_4K;
bytes = (bytes+pageSize-1) & ~(pageSize-1);
if (munmap(ptr,bytes) == -1)
// -- GODOT start --
// throw std::bad_alloc();
abort();
// -- GODOT end --
}
/* hint for transparent huge pages (THP) */
void os_advise(void* pptr, size_t bytes)
{
#if defined(MADV_HUGEPAGE)
madvise(pptr,bytes,MADV_HUGEPAGE);
#endif
}
}
#endif