// license:BSD-3-Clause // copyright-holders:Aaron Giles /*************************************************************************** drccache.c Universal dynamic recompiler cache management. ***************************************************************************/ #include "emu.h" #include "drccache.h" #include namespace { template constexpr T *ALIGN_PTR_UP(T *p, U align) { return reinterpret_cast((uintptr_t(p) + (align - 1)) & ~uintptr_t(align - 1)); } template constexpr T *ALIGN_PTR_DOWN(T *p, U align) { return reinterpret_cast(uintptr_t(p) & ~uintptr_t(align - 1)); } } // anonymous namespace //************************************************************************** // DRC CACHE //************************************************************************** //------------------------------------------------- // drc_cache - constructor //------------------------------------------------- drc_cache::drc_cache(size_t bytes) : m_cache({ NEAR_CACHE_SIZE, bytes - NEAR_CACHE_SIZE }, osd::virtual_memory_allocation::READ_WRITE_EXECUTE), m_near(reinterpret_cast(m_cache.get())), m_neartop(m_near), m_base(ALIGN_PTR_UP(m_near + NEAR_CACHE_SIZE, m_cache.page_size())), m_top(m_base), m_limit(m_near + m_cache.size()), m_end(m_limit), m_codegen(nullptr), m_size(m_cache.size()), m_executable(false), m_rwx(false) { // alignment and page size must be powers of two, cache must be page-aligned assert(!(CACHE_ALIGNMENT & (CACHE_ALIGNMENT - 1))); assert(!(m_cache.page_size() & (m_cache.page_size() - 1))); assert(!(uintptr_t(m_near) & (m_cache.page_size() - 1))); assert(m_cache.page_size() >= CACHE_ALIGNMENT); std::fill(std::begin(m_free), std::end(m_free), nullptr); std::fill(std::begin(m_nearfree), std::end(m_nearfree), nullptr); if (!m_cache) { throw emu_fatalerror("drc_cache: Error allocating virtual memory"); } else if (!m_cache.set_access(0, m_size, osd::virtual_memory_allocation::READ_WRITE)) { throw emu_fatalerror("drc_cache: Error marking cache read/write"); } else if (m_cache.set_access(m_base - m_near, m_end - m_base, osd::virtual_memory_allocation::READ_WRITE_EXECUTE)) { osd_printf_verbose("drc_cache: RWX pages supported\n"); m_rwx = true; } else { osd_printf_verbose("drc_cache: Using W^X mode\n"); m_rwx = false; } } //------------------------------------------------- // ~drc_cache - destructor //------------------------------------------------- drc_cache::~drc_cache() { } //------------------------------------------------- // flush - flush the cache contents //------------------------------------------------- void drc_cache::flush() { // can't flush in the middle of codegen assert(!m_codegen); // just reset the top back to the base and re-seed m_top = m_base; codegen_init(); } //------------------------------------------------- // alloc - allocate permanent memory from the // cache //------------------------------------------------- void *drc_cache::alloc(size_t bytes) { assert(bytes > 0); // pick first from the free list if (bytes < MAX_PERMANENT_ALLOC) { free_link **const linkptr = &m_free[(bytes + CACHE_ALIGNMENT - 1) / CACHE_ALIGNMENT]; free_link *const link = *linkptr; if (link) { *linkptr = link->m_next; return link; } } // if no space, we just fail drccodeptr const ptr = ALIGN_PTR_DOWN(m_end - bytes, CACHE_ALIGNMENT); drccodeptr const limit = ALIGN_PTR_DOWN(ptr, m_cache.page_size()); if (m_top > limit) return nullptr; // otherwise update the end of the cache m_limit = limit; m_end = ptr; return ptr; } //------------------------------------------------- // alloc_near - allocate permanent memory from // the near part of the cache //------------------------------------------------- void *drc_cache::alloc_near(size_t bytes) { assert(bytes > 0); // pick first from the free list if (bytes < MAX_PERMANENT_ALLOC) { free_link **const linkptr = &m_nearfree[(bytes + CACHE_ALIGNMENT - 1) / CACHE_ALIGNMENT]; free_link *const link = *linkptr; if (link) { *linkptr = link->m_next; return link; } } // if no space, we just fail drccodeptr const ptr = ALIGN_PTR_UP(m_neartop, CACHE_ALIGNMENT); if ((ptr + bytes) > m_base) return nullptr; // otherwise update the top of the near part of the cache m_neartop = ptr + bytes; return ptr; } //------------------------------------------------- // alloc_temporary - allocate temporary memory // from the cache //------------------------------------------------- void *drc_cache::alloc_temporary(size_t bytes) { // can't allocate in the middle of codegen assert(!m_codegen); // if no space, we just fail drccodeptr const ptr = m_top; if ((ptr + bytes) > m_limit) return nullptr; // otherwise, update the cache top codegen_init(); m_top = ALIGN_PTR_UP(ptr + bytes, CACHE_ALIGNMENT); return ptr; } //------------------------------------------------- // free - release permanent memory allocated from // the cache //------------------------------------------------- void drc_cache::dealloc(void *memory, size_t bytes) { drccodeptr const mem = reinterpret_cast(memory); assert(bytes < MAX_PERMANENT_ALLOC); assert(((mem >= m_near) && (mem < m_base)) || ((mem >= m_end) && (mem < (m_near + m_size)))); // determine which free list to add to free_link **const linkptr = &((mem < m_base) ? m_nearfree : m_free)[(bytes + CACHE_ALIGNMENT - 1) / CACHE_ALIGNMENT]; // link is into the free list for our size free_link *const link = reinterpret_cast(memory); link->m_next = *linkptr; *linkptr = link; } void drc_cache::codegen_init() { if (m_executable) { if (!m_rwx) m_cache.set_access(0, m_size, osd::virtual_memory_allocation::READ_WRITE); m_executable = false; } } void drc_cache::codegen_complete() { if (!m_executable) { if (!m_rwx) m_cache.set_access(m_base - m_near, ALIGN_PTR_UP(m_top, m_cache.page_size()) - m_base, osd::virtual_memory_allocation::READ_EXECUTE); m_executable = true; } } //------------------------------------------------- // begin_codegen - begin code generation //------------------------------------------------- drccodeptr *drc_cache::begin_codegen(uint32_t reserve_bytes) { // can't restart in the middle of codegen assert(!m_codegen); assert(m_oob_list.empty()); // if no space, we just fail if ((m_top + reserve_bytes) > m_limit) return nullptr; // otherwise, return a pointer to the cache top codegen_init(); m_codegen = m_top; return &m_top; } //------------------------------------------------- // end_codegen - complete code generation //------------------------------------------------- drccodeptr drc_cache::end_codegen() { drccodeptr const result = m_codegen; // run the OOB handlers while (!m_oob_list.empty()) { // call the callback m_oob_list.front().m_callback(&m_top, m_oob_list.front().m_param1, m_oob_list.front().m_param2); assert((m_top - m_codegen) < CODEGEN_MAX_BYTES); // add it to the free list m_oob_free.splice(m_oob_free.begin(), m_oob_list, m_oob_list.begin()); } // update the cache top osd::invalidate_instruction_cache(m_codegen, m_top - m_codegen); m_top = ALIGN_PTR_UP(m_top, CACHE_ALIGNMENT); m_codegen = nullptr; return result; } //------------------------------------------------- // request_oob_codegen - request callback for // out-of-band codegen //------------------------------------------------- void drc_cache::request_oob_codegen(drc_oob_delegate &&callback, void *param1, void *param2) { assert(m_codegen); // pull an item from the free list std::list::iterator oob; if (m_oob_free.empty()) { oob = m_oob_list.emplace(m_oob_list.end()); } else { oob = m_oob_free.begin(); m_oob_list.splice(m_oob_list.end(), m_oob_free, oob); } // fill it in oob->m_callback = std::move(callback); oob->m_param1 = param1; oob->m_param2 = param2; }