mirror of
https://github.com/NVIDIA/open-gpu-kernel-modules.git
synced 2024-12-13 15:08:59 +01:00
1350 lines
38 KiB
C
1350 lines
38 KiB
C
/*******************************************************************************
|
|
Copyright (c) 2016-2021 NVIDIA Corporation
|
|
|
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
of this software and associated documentation files (the "Software"), to
|
|
deal in the Software without restriction, including without limitation the
|
|
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
|
sell copies of the Software, and to permit persons to whom the Software is
|
|
furnished to do so, subject to the following conditions:
|
|
|
|
The above copyright notice and this permission notice shall be
|
|
included in all copies or substantial portions of the Software.
|
|
|
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
|
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
DEALINGS IN THE SOFTWARE.
|
|
|
|
*******************************************************************************/
|
|
|
|
#include "uvm_mem.h"
|
|
#include "uvm_mmu.h"
|
|
#include "uvm_processors.h"
|
|
#include "uvm_va_space.h"
|
|
#include "uvm_gpu.h"
|
|
#include "uvm_global.h"
|
|
#include "uvm_kvmalloc.h"
|
|
#include "uvm_push.h"
|
|
#include "uvm_range_allocator.h"
|
|
#include "uvm_hal.h"
|
|
#include "uvm_linux.h"
|
|
|
|
static uvm_range_allocator_t g_free_ranges;
|
|
static bool g_mem_initialized;
|
|
|
|
NV_STATUS uvm_mem_global_init(void)
|
|
{
|
|
NV_STATUS status = uvm_range_allocator_init(UVM_MEM_VA_SIZE, &g_free_ranges);
|
|
if (status != NV_OK)
|
|
return status;
|
|
|
|
g_mem_initialized = true;
|
|
|
|
return NV_OK;
|
|
}
|
|
|
|
void uvm_mem_global_exit(void)
|
|
{
|
|
if (!g_mem_initialized)
|
|
return;
|
|
|
|
uvm_range_allocator_deinit(&g_free_ranges);
|
|
}
|
|
|
|
static bool vidmem_can_be_mapped(uvm_mem_t *vidmem, bool is_user_space)
|
|
{
|
|
UVM_ASSERT(uvm_mem_is_vidmem(vidmem));
|
|
|
|
// Mapping a vidmem allocation on a user VA space is currently unsupported,
|
|
// because there is no use case.
|
|
if (is_user_space)
|
|
return false;
|
|
|
|
return true;
|
|
}
|
|
|
|
static bool sysmem_can_be_mapped(uvm_mem_t *sysmem)
|
|
{
|
|
UVM_ASSERT(uvm_mem_is_sysmem(sysmem));
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return true;
|
|
}
|
|
|
|
static bool mem_can_be_mapped_on_cpu(uvm_mem_t *mem, bool is_user_space)
|
|
{
|
|
if (uvm_mem_is_sysmem(mem))
|
|
return sysmem_can_be_mapped(mem);
|
|
|
|
if (!vidmem_can_be_mapped(mem, is_user_space))
|
|
return false;
|
|
|
|
return mem->backing_gpu->parent->numa_info.enabled && PAGE_ALIGNED(mem->chunk_size);
|
|
}
|
|
|
|
static bool mem_can_be_mapped_on_cpu_kernel(uvm_mem_t *mem)
|
|
{
|
|
return mem_can_be_mapped_on_cpu(mem, false);
|
|
}
|
|
|
|
static bool mem_can_be_mapped_on_cpu_user(uvm_mem_t *mem)
|
|
{
|
|
return mem_can_be_mapped_on_cpu(mem, true);
|
|
}
|
|
|
|
static bool mem_can_be_mapped_on_gpu(uvm_mem_t *mem, uvm_gpu_t *gpu, bool is_user_space)
|
|
{
|
|
if (uvm_mem_is_sysmem(mem))
|
|
return sysmem_can_be_mapped(mem);
|
|
|
|
if (!vidmem_can_be_mapped(mem, is_user_space))
|
|
return false;
|
|
|
|
return uvm_mem_is_local_vidmem(mem, gpu);
|
|
}
|
|
|
|
static bool mem_can_be_mapped_on_gpu_kernel(uvm_mem_t *mem, uvm_gpu_t *gpu)
|
|
{
|
|
return mem_can_be_mapped_on_gpu(mem, gpu, false);
|
|
}
|
|
|
|
static bool mem_can_be_mapped_on_gpu_user(uvm_mem_t *mem, uvm_gpu_t *gpu)
|
|
{
|
|
return mem_can_be_mapped_on_gpu(mem, gpu, true);
|
|
}
|
|
|
|
bool uvm_mem_mapped_on_gpu_user(uvm_mem_t *mem, uvm_gpu_t *gpu)
|
|
{
|
|
if (mem->user == NULL)
|
|
return false;
|
|
|
|
return uvm_global_processor_mask_test(&mem->user->mapped_on, gpu->global_id);
|
|
}
|
|
|
|
bool uvm_mem_mapped_on_gpu_kernel(uvm_mem_t *mem, uvm_gpu_t *gpu)
|
|
{
|
|
return uvm_global_processor_mask_test(&mem->kernel.mapped_on, gpu->global_id);
|
|
}
|
|
|
|
bool uvm_mem_mapped_on_cpu_user(uvm_mem_t *mem)
|
|
{
|
|
if (mem->user == NULL)
|
|
return false;
|
|
|
|
return uvm_global_processor_mask_test(&mem->user->mapped_on, UVM_GLOBAL_ID_CPU);
|
|
}
|
|
|
|
bool uvm_mem_mapped_on_cpu_kernel(uvm_mem_t *mem)
|
|
{
|
|
return uvm_global_processor_mask_test(&mem->kernel.mapped_on, UVM_GLOBAL_ID_CPU);
|
|
}
|
|
|
|
static void mem_set_mapped_on_gpu_user(uvm_mem_t *mem, uvm_gpu_t *gpu)
|
|
{
|
|
UVM_ASSERT(mem->user != NULL);
|
|
UVM_ASSERT(mem_can_be_mapped_on_gpu_user(mem, gpu));
|
|
UVM_ASSERT(!uvm_mem_mapped_on_gpu_user(mem, gpu));
|
|
|
|
uvm_global_processor_mask_set(&mem->user->mapped_on, gpu->global_id);
|
|
}
|
|
|
|
static void mem_set_mapped_on_gpu_kernel(uvm_mem_t *mem, uvm_gpu_t *gpu)
|
|
{
|
|
UVM_ASSERT(mem_can_be_mapped_on_gpu_kernel(mem, gpu));
|
|
UVM_ASSERT(!uvm_mem_mapped_on_gpu_kernel(mem, gpu));
|
|
|
|
uvm_global_processor_mask_set(&mem->kernel.mapped_on, gpu->global_id);
|
|
}
|
|
|
|
static void mem_set_mapped_on_cpu_user(uvm_mem_t *mem)
|
|
{
|
|
UVM_ASSERT(mem->user != NULL);
|
|
UVM_ASSERT(mem_can_be_mapped_on_cpu_user(mem));
|
|
UVM_ASSERT(!uvm_mem_mapped_on_cpu_user(mem));
|
|
|
|
uvm_global_processor_mask_set(&mem->user->mapped_on, UVM_GLOBAL_ID_CPU);
|
|
}
|
|
|
|
static void mem_set_mapped_on_cpu_kernel(uvm_mem_t *mem)
|
|
{
|
|
UVM_ASSERT(mem_can_be_mapped_on_cpu_kernel(mem));
|
|
UVM_ASSERT(!uvm_mem_mapped_on_cpu_kernel(mem));
|
|
|
|
uvm_global_processor_mask_set(&mem->kernel.mapped_on, UVM_GLOBAL_ID_CPU);
|
|
}
|
|
|
|
static void mem_clear_mapped_on_gpu_kernel(uvm_mem_t *mem, uvm_gpu_t *gpu)
|
|
{
|
|
uvm_global_processor_mask_clear(&mem->kernel.mapped_on, gpu->global_id);
|
|
}
|
|
|
|
static void mem_clear_mapped_on_gpu_user(uvm_mem_t *mem, uvm_gpu_t *gpu)
|
|
{
|
|
UVM_ASSERT(mem->user != NULL);
|
|
|
|
uvm_global_processor_mask_clear(&mem->user->mapped_on, gpu->global_id);
|
|
}
|
|
|
|
static void mem_clear_mapped_on_cpu_user(uvm_mem_t *mem)
|
|
{
|
|
UVM_ASSERT(mem->user != NULL);
|
|
|
|
uvm_global_processor_mask_clear(&mem->user->mapped_on, UVM_GLOBAL_ID_CPU);
|
|
}
|
|
|
|
static void mem_clear_mapped_on_cpu_kernel(uvm_mem_t *mem)
|
|
{
|
|
uvm_global_processor_mask_clear(&mem->kernel.mapped_on, UVM_GLOBAL_ID_CPU);
|
|
}
|
|
|
|
static bool sysmem_mapped_on_gpu_phys(uvm_mem_t *sysmem, uvm_gpu_t *gpu)
|
|
{
|
|
UVM_ASSERT(uvm_mem_is_sysmem(sysmem));
|
|
|
|
return uvm_global_processor_mask_test(&sysmem->sysmem.mapped_on_phys, gpu->global_id);
|
|
}
|
|
|
|
static void sysmem_set_mapped_on_gpu_phys(uvm_mem_t *sysmem, uvm_gpu_t *gpu)
|
|
{
|
|
UVM_ASSERT(uvm_mem_is_sysmem(sysmem));
|
|
UVM_ASSERT(!sysmem_mapped_on_gpu_phys(sysmem, gpu));
|
|
|
|
uvm_global_processor_mask_set(&sysmem->sysmem.mapped_on_phys, gpu->global_id);
|
|
}
|
|
|
|
static void sysmem_clear_mapped_on_gpu_phys(uvm_mem_t *sysmem, uvm_gpu_t *gpu)
|
|
{
|
|
UVM_ASSERT(uvm_mem_is_sysmem(sysmem));
|
|
|
|
uvm_global_processor_mask_clear(&sysmem->sysmem.mapped_on_phys, gpu->global_id);
|
|
}
|
|
|
|
NV_STATUS uvm_mem_translate_gpu_attributes(const UvmGpuMappingAttributes *attrs,
|
|
uvm_va_space_t *va_space,
|
|
uvm_gpu_t **gpu_out,
|
|
uvm_mem_gpu_mapping_attrs_t *attrs_out)
|
|
{
|
|
uvm_gpu_t *gpu;
|
|
|
|
switch (attrs->gpuMappingType) {
|
|
case UvmGpuMappingTypeDefault:
|
|
break;
|
|
case UvmGpuMappingTypeReadWriteAtomic:
|
|
attrs_out->protection = UVM_PROT_READ_WRITE_ATOMIC;
|
|
break;
|
|
case UvmGpuMappingTypeReadWrite:
|
|
attrs_out->protection = UVM_PROT_READ_WRITE;
|
|
break;
|
|
case UvmGpuMappingTypeReadOnly:
|
|
attrs_out->protection = UVM_PROT_READ_ONLY;
|
|
break;
|
|
default:
|
|
return NV_ERR_INVALID_ARGUMENT;
|
|
}
|
|
|
|
switch (attrs->gpuCachingType) {
|
|
case UvmGpuCachingTypeDefault:
|
|
break;
|
|
case UvmGpuCachingTypeForceUncached:
|
|
attrs_out->is_cacheable = false;
|
|
break;
|
|
case UvmGpuCachingTypeForceCached:
|
|
attrs_out->is_cacheable = true;
|
|
break;
|
|
default:
|
|
return NV_ERR_INVALID_ARGUMENT;
|
|
}
|
|
|
|
gpu = uvm_va_space_get_gpu_by_uuid(va_space, &attrs->gpuUuid);
|
|
if (!gpu)
|
|
return NV_ERR_INVALID_DEVICE;
|
|
|
|
if (gpu_out)
|
|
*gpu_out = gpu;
|
|
|
|
return NV_OK;
|
|
}
|
|
|
|
static struct page *uvm_virt_to_page(const void *addr)
|
|
{
|
|
if (virt_addr_valid(addr))
|
|
return virt_to_page(addr);
|
|
|
|
if (is_vmalloc_addr(addr))
|
|
return vmalloc_to_page(addr);
|
|
|
|
return NULL;
|
|
}
|
|
|
|
uvm_chunk_sizes_mask_t uvm_mem_kernel_chunk_sizes(uvm_gpu_t *gpu)
|
|
{
|
|
// Get the mmu mode hal directly as the internal address space tree has not
|
|
// been created yet.
|
|
uvm_mmu_mode_hal_t *hal = gpu->parent->arch_hal->mmu_mode_hal(gpu->big_page.internal_size);
|
|
NvU32 page_sizes = hal->page_sizes();
|
|
|
|
return (uvm_chunk_sizes_mask_t)(page_sizes & UVM_CHUNK_SIZES_MASK);
|
|
}
|
|
|
|
static NvU32 mem_pick_chunk_size(uvm_mem_t *mem)
|
|
{
|
|
NvU32 biggest_page_size;
|
|
NvU32 chunk_size;
|
|
|
|
if (uvm_mem_is_sysmem(mem))
|
|
return PAGE_SIZE;
|
|
|
|
biggest_page_size = uvm_mmu_biggest_page_size_up_to(&mem->backing_gpu->address_space_tree, UVM_CHUNK_SIZE_MAX);
|
|
|
|
if (mem->size < mem->backing_gpu->big_page.internal_size)
|
|
chunk_size = UVM_PAGE_SIZE_4K;
|
|
else if (mem->size < biggest_page_size)
|
|
chunk_size = mem->backing_gpu->big_page.internal_size;
|
|
else
|
|
chunk_size = biggest_page_size;
|
|
|
|
// When UVM_PAGE_SIZE_DEFAULT is used on NUMA-enabled GPUs, we force
|
|
// chunk_size to be PAGE_SIZE at least, to allow CPU mappings.
|
|
if (mem->backing_gpu->parent->numa_info.enabled)
|
|
chunk_size = max(chunk_size, (NvU32)PAGE_SIZE);
|
|
|
|
return chunk_size;
|
|
}
|
|
|
|
static NvU32 mem_pick_gpu_page_size(uvm_mem_t *mem, uvm_gpu_t *gpu, uvm_page_tree_t *gpu_page_tree)
|
|
{
|
|
if (uvm_mem_is_vidmem(mem)) {
|
|
// For vidmem allocations the chunk size is picked out of the supported
|
|
// page sizes and can be used directly.
|
|
return mem->chunk_size;
|
|
}
|
|
|
|
// For sysmem, check whether the GPU supports mapping it with large pages.
|
|
if (gpu->parent->can_map_sysmem_with_large_pages) {
|
|
// If it's supported, pick the largest page size not bigger than
|
|
// the chunk size.
|
|
return uvm_mmu_biggest_page_size_up_to(gpu_page_tree, mem->chunk_size);
|
|
}
|
|
|
|
// Otherwise just use 4K.
|
|
return UVM_PAGE_SIZE_4K;
|
|
}
|
|
|
|
static void mem_free_vidmem_chunks(uvm_mem_t *mem)
|
|
{
|
|
size_t i;
|
|
|
|
UVM_ASSERT(uvm_mem_is_vidmem(mem));
|
|
|
|
if (!mem->vidmem.chunks)
|
|
return;
|
|
|
|
for (i = 0; i < mem->chunks_count; ++i) {
|
|
// On allocation error PMM guarantees the chunks array to be zeroed so
|
|
// just check for NULL.
|
|
if (mem->vidmem.chunks[i] == NULL)
|
|
break;
|
|
uvm_pmm_gpu_free(&mem->backing_gpu->pmm, mem->vidmem.chunks[i], NULL);
|
|
}
|
|
|
|
uvm_kvfree(mem->vidmem.chunks);
|
|
mem->vidmem.chunks = NULL;
|
|
}
|
|
|
|
static void mem_free_sysmem_dma_chunks(uvm_mem_t *mem)
|
|
{
|
|
size_t i;
|
|
NvU32 gpu_index;
|
|
|
|
UVM_ASSERT(uvm_mem_is_sysmem_dma(mem));
|
|
gpu_index = uvm_global_id_gpu_index(mem->dma_owner->global_id);
|
|
|
|
if (!mem->sysmem.pages || !mem->sysmem.va)
|
|
goto end;
|
|
|
|
for (i = 0; i < mem->chunks_count; ++i) {
|
|
if (!mem->sysmem.va[i])
|
|
break;
|
|
|
|
uvm_gpu_dma_free_page(mem->dma_owner->parent,
|
|
mem->sysmem.va[i],
|
|
mem->sysmem.dma_addrs[gpu_index][i]);
|
|
}
|
|
|
|
end:
|
|
sysmem_clear_mapped_on_gpu_phys(mem, mem->dma_owner);
|
|
|
|
uvm_kvfree(mem->sysmem.dma_addrs[gpu_index]);
|
|
mem->sysmem.dma_addrs[gpu_index] = NULL;
|
|
|
|
uvm_kvfree(mem->sysmem.pages);
|
|
mem->sysmem.pages = NULL;
|
|
|
|
uvm_kvfree(mem->sysmem.va);
|
|
mem->sysmem.va = NULL;
|
|
}
|
|
|
|
static void mem_free_sysmem_chunks(uvm_mem_t *mem)
|
|
{
|
|
size_t i;
|
|
|
|
UVM_ASSERT(uvm_mem_is_sysmem(mem));
|
|
|
|
if (!mem->sysmem.pages)
|
|
return;
|
|
|
|
for (i = 0; i < mem->chunks_count; ++i) {
|
|
if (!mem->sysmem.pages[i])
|
|
break;
|
|
__free_pages(mem->sysmem.pages[i], get_order(mem->chunk_size));
|
|
}
|
|
|
|
uvm_kvfree(mem->sysmem.pages);
|
|
mem->sysmem.pages = NULL;
|
|
}
|
|
|
|
static void mem_free_chunks(uvm_mem_t *mem)
|
|
{
|
|
if (uvm_mem_is_vidmem(mem))
|
|
mem_free_vidmem_chunks(mem);
|
|
else if (uvm_mem_is_sysmem_dma(mem))
|
|
mem_free_sysmem_dma_chunks(mem);
|
|
else
|
|
mem_free_sysmem_chunks(mem);
|
|
}
|
|
|
|
static NV_STATUS mem_alloc_dma_addrs(uvm_mem_t *mem, const uvm_gpu_t *gpu)
|
|
{
|
|
NvU64 *dma_addrs = NULL;
|
|
NvU32 gpu_index = uvm_global_id_gpu_index(gpu->global_id);
|
|
|
|
dma_addrs = uvm_kvmalloc_zero(sizeof(*dma_addrs) * mem->chunks_count);
|
|
if (!dma_addrs)
|
|
return NV_ERR_NO_MEMORY;
|
|
|
|
mem->sysmem.dma_addrs[gpu_index] = dma_addrs;
|
|
|
|
return NV_OK;
|
|
}
|
|
|
|
static gfp_t sysmem_allocation_gfp_flags(int order, bool zero)
|
|
{
|
|
gfp_t gfp_flags = NV_UVM_GFP_FLAGS;
|
|
|
|
if (zero)
|
|
gfp_flags |= __GFP_ZERO;
|
|
|
|
// High-order page allocations require the __GFP_COMP flag to work with
|
|
// vm_insert_page.
|
|
if (order > 0)
|
|
gfp_flags |= __GFP_COMP;
|
|
|
|
return gfp_flags;
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// There is a tighter coupling between allocation and mapping because of the
|
|
// allocator UVM must use. Hence, this function does the equivalent of
|
|
// uvm_mem_map_gpu_phys().
|
|
//
|
|
// In case of failure, the caller is required to handle cleanup by calling
|
|
// uvm_mem_free
|
|
static NV_STATUS mem_alloc_sysmem_dma_chunks(uvm_mem_t *mem, struct mm_struct *mm, gfp_t gfp_flags)
|
|
{
|
|
size_t i;
|
|
NV_STATUS status;
|
|
NvU64 *dma_addrs;
|
|
|
|
UVM_ASSERT_MSG(mem->chunk_size == PAGE_SIZE,
|
|
"mem->chunk_size is 0x%x. PAGE_SIZE is only supported.",
|
|
mem->chunk_size);
|
|
UVM_ASSERT(uvm_mem_is_sysmem_dma(mem));
|
|
|
|
mem->sysmem.pages = uvm_kvmalloc_zero(sizeof(*mem->sysmem.pages) * mem->chunks_count);
|
|
mem->sysmem.va = uvm_kvmalloc_zero(sizeof(*mem->sysmem.va) * mem->chunks_count);
|
|
if (!mem->sysmem.pages || !mem->sysmem.va)
|
|
goto err_no_mem;
|
|
|
|
status = mem_alloc_dma_addrs(mem, mem->dma_owner);
|
|
if (status != NV_OK)
|
|
goto error;
|
|
|
|
dma_addrs = mem->sysmem.dma_addrs[uvm_global_id_gpu_index(mem->dma_owner->global_id)];
|
|
|
|
for (i = 0; i < mem->chunks_count; ++i) {
|
|
mem->sysmem.va[i] = uvm_gpu_dma_alloc_page(mem->dma_owner->parent, gfp_flags, &dma_addrs[i]);
|
|
if (!mem->sysmem.va[i])
|
|
goto err_no_mem;
|
|
|
|
mem->sysmem.pages[i] = uvm_virt_to_page(mem->sysmem.va[i]);
|
|
if (!mem->sysmem.pages[i])
|
|
goto err_no_mem;
|
|
}
|
|
|
|
sysmem_set_mapped_on_gpu_phys(mem, mem->dma_owner);
|
|
|
|
return NV_OK;
|
|
|
|
err_no_mem:
|
|
status = NV_ERR_NO_MEMORY;
|
|
error:
|
|
mem_free_sysmem_dma_chunks(mem);
|
|
return status;
|
|
}
|
|
|
|
// In case of failure, the caller is required to handle cleanup by calling
|
|
// uvm_mem_free
|
|
static NV_STATUS mem_alloc_sysmem_chunks(uvm_mem_t *mem, struct mm_struct *mm, gfp_t gfp_flags)
|
|
{
|
|
size_t i;
|
|
int order;
|
|
|
|
UVM_ASSERT(uvm_mem_is_sysmem(mem) && !uvm_mem_is_sysmem_dma(mem));
|
|
|
|
mem->sysmem.pages = uvm_kvmalloc_zero(sizeof(*mem->sysmem.pages) * mem->chunks_count);
|
|
if (!mem->sysmem.pages)
|
|
return NV_ERR_NO_MEMORY;
|
|
|
|
order = get_order(mem->chunk_size);
|
|
for (i = 0; i < mem->chunks_count; ++i) {
|
|
mem->sysmem.pages[i] = alloc_pages(gfp_flags, order);
|
|
if (!mem->sysmem.pages[i])
|
|
return NV_ERR_NO_MEMORY;
|
|
}
|
|
|
|
return NV_OK;
|
|
}
|
|
|
|
// In case of failure, the caller is required to handle cleanup by calling
|
|
// uvm_mem_free
|
|
static NV_STATUS mem_alloc_vidmem_chunks(uvm_mem_t *mem, bool zero)
|
|
{
|
|
NV_STATUS status;
|
|
|
|
UVM_ASSERT(uvm_mem_is_vidmem(mem));
|
|
|
|
// TODO: Bug 2446832: A non-zeroing request may not be obeyed because PMM
|
|
// does not support explicit allocation of non-zeroed (or zeroed) chunks.
|
|
//
|
|
// The zeroing case can be implemented even without resolving that bug, by
|
|
// clearing the chunks after PMM allocation. But this functionality has not
|
|
// been implemented, because the only expected use case is a memory that
|
|
// gets mapped on user space, and vidmem never is.
|
|
UVM_ASSERT(!zero);
|
|
|
|
mem->vidmem.chunks = uvm_kvmalloc_zero(mem->chunks_count * sizeof(*mem->vidmem.chunks));
|
|
if (!mem->vidmem.chunks)
|
|
return NV_ERR_NO_MEMORY;
|
|
|
|
status = uvm_pmm_gpu_alloc_kernel(&mem->backing_gpu->pmm,
|
|
mem->chunks_count,
|
|
mem->chunk_size,
|
|
UVM_PMM_ALLOC_FLAGS_NONE,
|
|
mem->vidmem.chunks,
|
|
NULL);
|
|
if (status != NV_OK) {
|
|
UVM_ERR_PRINT("pmm_gpu_alloc(count=%zd, size=0x%x) failed: %s\n",
|
|
mem->chunks_count,
|
|
mem->chunk_size,
|
|
nvstatusToString(status));
|
|
return status;
|
|
}
|
|
|
|
return NV_OK;
|
|
}
|
|
|
|
static NV_STATUS mem_alloc_chunks(uvm_mem_t *mem, struct mm_struct *mm, bool zero)
|
|
{
|
|
if (uvm_mem_is_sysmem(mem)) {
|
|
gfp_t gfp_flags;
|
|
uvm_memcg_context_t memcg_context;
|
|
NV_STATUS status;
|
|
|
|
UVM_ASSERT(PAGE_ALIGNED(mem->chunk_size));
|
|
gfp_flags = sysmem_allocation_gfp_flags(get_order(mem->chunk_size), zero);
|
|
if (UVM_CGROUP_ACCOUNTING_SUPPORTED() && mm)
|
|
gfp_flags |= NV_UVM_GFP_FLAGS_ACCOUNT;
|
|
|
|
uvm_memcg_context_start(&memcg_context, mm);
|
|
if (uvm_mem_is_sysmem_dma(mem))
|
|
status = mem_alloc_sysmem_dma_chunks(mem, mm, gfp_flags);
|
|
else
|
|
status = mem_alloc_sysmem_chunks(mem, mm, gfp_flags);
|
|
|
|
uvm_memcg_context_end(&memcg_context);
|
|
return status;
|
|
}
|
|
|
|
return mem_alloc_vidmem_chunks(mem, zero);
|
|
}
|
|
|
|
static const char *mem_physical_source(uvm_mem_t *mem)
|
|
{
|
|
if (uvm_mem_is_vidmem(mem))
|
|
return uvm_gpu_name(mem->backing_gpu);
|
|
|
|
return "CPU";
|
|
}
|
|
|
|
NV_STATUS uvm_mem_map_kernel(uvm_mem_t *mem, const uvm_global_processor_mask_t *mask)
|
|
{
|
|
uvm_gpu_t *gpu;
|
|
NV_STATUS status;
|
|
|
|
if (!mask)
|
|
return NV_OK;
|
|
|
|
if (uvm_global_processor_mask_test(mask, UVM_GLOBAL_ID_CPU)) {
|
|
status = uvm_mem_map_cpu_kernel(mem);
|
|
if (status != NV_OK)
|
|
return status;
|
|
}
|
|
|
|
for_each_global_gpu_in_mask(gpu, mask) {
|
|
status = uvm_mem_map_gpu_kernel(mem, gpu);
|
|
if (status != NV_OK)
|
|
return status;
|
|
}
|
|
return NV_OK;
|
|
}
|
|
|
|
NV_STATUS uvm_mem_alloc(const uvm_mem_alloc_params_t *params, uvm_mem_t **mem_out)
|
|
{
|
|
NV_STATUS status;
|
|
uvm_mem_t *mem = NULL;
|
|
|
|
UVM_ASSERT(params->size > 0);
|
|
|
|
mem = uvm_kvmalloc_zero(sizeof(*mem));
|
|
if (mem == NULL)
|
|
return NV_ERR_NO_MEMORY;
|
|
|
|
mem->backing_gpu = params->backing_gpu;
|
|
mem->dma_owner = params->dma_owner;
|
|
UVM_ASSERT(!mem->dma_owner || !mem->backing_gpu);
|
|
|
|
mem->size = params->size;
|
|
mem->chunk_size = params->page_size;
|
|
if (mem->chunk_size == UVM_PAGE_SIZE_DEFAULT)
|
|
mem->chunk_size = mem_pick_chunk_size(mem);
|
|
|
|
UVM_ASSERT(mem->chunk_size > 0);
|
|
|
|
mem->physical_allocation_size = UVM_ALIGN_UP(mem->size, mem->chunk_size);
|
|
mem->chunks_count = mem->physical_allocation_size / mem->chunk_size;
|
|
|
|
status = mem_alloc_chunks(mem, params->mm, params->zero);
|
|
if (status != NV_OK)
|
|
goto error;
|
|
|
|
*mem_out = mem;
|
|
return NV_OK;
|
|
|
|
error:
|
|
uvm_mem_free(mem);
|
|
return status;
|
|
}
|
|
|
|
static NV_STATUS mem_init_user_mapping(uvm_mem_t *mem, uvm_va_space_t *user_va_space, void *user_addr)
|
|
{
|
|
UVM_ASSERT(user_va_space);
|
|
UVM_ASSERT(user_addr);
|
|
|
|
// If the user structure exists, the VA space and address should match
|
|
if (mem->user != NULL) {
|
|
UVM_ASSERT(mem->user->va_space == user_va_space);
|
|
UVM_ASSERT(mem->user->addr == user_addr);
|
|
return NV_OK;
|
|
}
|
|
|
|
UVM_ASSERT(IS_ALIGNED((NvU64)user_addr, mem->chunk_size));
|
|
UVM_ASSERT(mem->physical_allocation_size == mem->size);
|
|
|
|
mem->user = uvm_kvmalloc_zero(sizeof(*mem->user));
|
|
if (mem->user == NULL)
|
|
return NV_ERR_NO_MEMORY;
|
|
|
|
mem->user->va_space = user_va_space;
|
|
mem->user->addr = user_addr;
|
|
|
|
return NV_OK;
|
|
}
|
|
|
|
static void mem_deinit_user_mapping(uvm_mem_t *mem)
|
|
{
|
|
if (mem->user == NULL)
|
|
return;
|
|
|
|
if (!uvm_global_processor_mask_empty(&mem->user->mapped_on))
|
|
return;
|
|
|
|
uvm_kvfree(mem->user);
|
|
mem->user = NULL;
|
|
}
|
|
|
|
static NvU64 reserved_gpu_va(uvm_mem_t *mem, uvm_gpu_t *gpu)
|
|
{
|
|
UVM_ASSERT(mem->kernel.range_alloc.aligned_start + mem->physical_allocation_size < gpu->parent->uvm_mem_va_size);
|
|
|
|
return gpu->parent->uvm_mem_va_base + mem->kernel.range_alloc.aligned_start;
|
|
}
|
|
|
|
static struct page *mem_cpu_page(uvm_mem_t *mem, NvU64 offset)
|
|
{
|
|
struct page *base_page = mem->sysmem.pages[offset / mem->chunk_size];
|
|
|
|
UVM_ASSERT_MSG(PAGE_ALIGNED(offset), "offset 0x%llx\n", offset);
|
|
|
|
offset = offset % mem->chunk_size;
|
|
return pfn_to_page(page_to_pfn(base_page) + offset / PAGE_SIZE);
|
|
}
|
|
|
|
static NV_STATUS mem_map_cpu_to_sysmem_kernel(uvm_mem_t *mem)
|
|
{
|
|
struct page **pages = mem->sysmem.pages;
|
|
size_t num_pages = mem->physical_allocation_size / PAGE_SIZE;
|
|
pgprot_t prot = PAGE_KERNEL;
|
|
|
|
UVM_ASSERT(uvm_mem_is_sysmem(mem));
|
|
|
|
// If chunk size is different than PAGE_SIZE then create a temporary array
|
|
// of all the pages to map so that vmap() can be used.
|
|
if (mem->chunk_size != PAGE_SIZE) {
|
|
size_t page_index;
|
|
pages = uvm_kvmalloc(sizeof(*pages) * num_pages);
|
|
if (!pages)
|
|
return NV_ERR_NO_MEMORY;
|
|
for (page_index = 0; page_index < num_pages; ++page_index)
|
|
pages[page_index] = mem_cpu_page(mem, page_index * PAGE_SIZE);
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
mem->kernel.cpu_addr = vmap(pages, num_pages, VM_MAP, prot);
|
|
|
|
if (mem->chunk_size != PAGE_SIZE)
|
|
uvm_kvfree(pages);
|
|
|
|
if (!mem->kernel.cpu_addr)
|
|
return NV_ERR_NO_MEMORY;
|
|
|
|
return NV_OK;
|
|
}
|
|
|
|
static NV_STATUS mem_map_cpu_to_vidmem_kernel(uvm_mem_t *mem)
|
|
{
|
|
struct page **pages;
|
|
size_t num_chunk_pages = mem->chunk_size / PAGE_SIZE;
|
|
size_t num_pages = mem->physical_allocation_size / PAGE_SIZE;
|
|
size_t page_index;
|
|
size_t chunk_index;
|
|
|
|
UVM_ASSERT(uvm_mem_is_vidmem(mem));
|
|
|
|
pages = uvm_kvmalloc(sizeof(*pages) * num_pages);
|
|
if (!pages)
|
|
return NV_ERR_NO_MEMORY;
|
|
|
|
page_index = 0;
|
|
|
|
for (chunk_index = 0; chunk_index < mem->chunks_count; ++chunk_index) {
|
|
uvm_gpu_chunk_t *chunk = mem->vidmem.chunks[chunk_index];
|
|
struct page *page = uvm_gpu_chunk_to_page(&mem->backing_gpu->pmm, chunk);
|
|
size_t chunk_page_index;
|
|
|
|
for (chunk_page_index = 0; chunk_page_index < num_chunk_pages; ++chunk_page_index)
|
|
pages[page_index++] = page + chunk_page_index;
|
|
}
|
|
UVM_ASSERT(page_index == num_pages);
|
|
|
|
mem->kernel.cpu_addr = vmap(pages, num_pages, VM_MAP, PAGE_KERNEL);
|
|
|
|
uvm_kvfree(pages);
|
|
|
|
if (!mem->kernel.cpu_addr)
|
|
return NV_ERR_NO_MEMORY;
|
|
|
|
return NV_OK;
|
|
}
|
|
|
|
void uvm_mem_unmap_cpu_kernel(uvm_mem_t *mem)
|
|
{
|
|
if (!uvm_mem_mapped_on_cpu_kernel(mem))
|
|
return;
|
|
|
|
vunmap(mem->kernel.cpu_addr);
|
|
mem->kernel.cpu_addr = NULL;
|
|
mem_clear_mapped_on_cpu_kernel(mem);
|
|
}
|
|
|
|
static NV_STATUS mem_map_cpu_to_sysmem_user(uvm_mem_t *mem, struct vm_area_struct *vma)
|
|
{
|
|
NV_STATUS status;
|
|
NvU64 offset;
|
|
|
|
UVM_ASSERT(mem->user != NULL);
|
|
UVM_ASSERT(uvm_mem_is_sysmem(mem));
|
|
uvm_assert_mmap_lock_locked(vma->vm_mm);
|
|
|
|
// TODO: Bug 1995015: high-order page allocations need to be allocated as
|
|
// compound pages in order to be able to use vm_insert_page on them. This
|
|
// is not currently being exercised because the only allocations using this
|
|
// are semaphore pools (which typically use a single page).
|
|
for (offset = 0; offset < mem->physical_allocation_size; offset += PAGE_SIZE) {
|
|
int ret = vm_insert_page(vma, (unsigned long)mem->user->addr + offset, mem_cpu_page(mem, offset));
|
|
if (ret) {
|
|
UVM_ASSERT_MSG(ret == -ENOMEM, "ret: %d\n", ret);
|
|
status = errno_to_nv_status(ret);
|
|
goto error;
|
|
}
|
|
}
|
|
|
|
return NV_OK;
|
|
|
|
error:
|
|
unmap_mapping_range(&mem->user->va_space->mapping, (size_t)mem->user->addr, mem->physical_allocation_size, 1);
|
|
return status;
|
|
}
|
|
|
|
void uvm_mem_unmap_cpu_user(uvm_mem_t *mem)
|
|
{
|
|
if (!uvm_mem_mapped_on_cpu_user(mem))
|
|
return;
|
|
|
|
unmap_mapping_range(&mem->user->va_space->mapping, (size_t)mem->user->addr, mem->physical_allocation_size, 1);
|
|
mem_clear_mapped_on_cpu_user(mem);
|
|
mem_deinit_user_mapping(mem);
|
|
}
|
|
|
|
NV_STATUS uvm_mem_map_cpu_user(uvm_mem_t *mem, uvm_va_space_t *user_va_space, struct vm_area_struct *vma)
|
|
{
|
|
NV_STATUS status;
|
|
void *user_addr;
|
|
|
|
UVM_ASSERT(mem);
|
|
UVM_ASSERT(mem_can_be_mapped_on_cpu_user(mem));
|
|
|
|
if (uvm_mem_mapped_on_cpu_user(mem))
|
|
return NV_OK;
|
|
|
|
UVM_ASSERT((vma->vm_end - vma->vm_start) == mem->size);
|
|
|
|
user_addr = (void *) (uintptr_t)vma->vm_start;
|
|
|
|
status = mem_init_user_mapping(mem, user_va_space, user_addr);
|
|
if (status != NV_OK)
|
|
return status;
|
|
|
|
status = mem_map_cpu_to_sysmem_user(mem, vma);
|
|
if (status != NV_OK)
|
|
goto cleanup;
|
|
|
|
mem_set_mapped_on_cpu_user(mem);
|
|
|
|
return NV_OK;
|
|
|
|
cleanup:
|
|
mem_deinit_user_mapping(mem);
|
|
return status;
|
|
}
|
|
|
|
NV_STATUS uvm_mem_map_cpu_kernel(uvm_mem_t *mem)
|
|
{
|
|
NV_STATUS status;
|
|
|
|
UVM_ASSERT(mem);
|
|
UVM_ASSERT(mem_can_be_mapped_on_cpu_kernel(mem));
|
|
|
|
if (uvm_mem_mapped_on_cpu_kernel(mem))
|
|
return NV_OK;
|
|
|
|
if (uvm_mem_is_sysmem(mem))
|
|
status = mem_map_cpu_to_sysmem_kernel(mem);
|
|
else
|
|
status = mem_map_cpu_to_vidmem_kernel(mem);
|
|
|
|
if (status != NV_OK)
|
|
return status;
|
|
|
|
mem_set_mapped_on_cpu_kernel(mem);
|
|
|
|
return NV_OK;
|
|
}
|
|
|
|
static void sysmem_unmap_gpu_phys(uvm_mem_t *mem, uvm_gpu_t *gpu)
|
|
{
|
|
NvU64 *dma_addrs = mem->sysmem.dma_addrs[uvm_global_id_gpu_index(gpu->global_id)];
|
|
NvU32 i;
|
|
|
|
UVM_ASSERT(uvm_mem_is_sysmem(mem));
|
|
UVM_ASSERT(gpu != mem->dma_owner);
|
|
UVM_ASSERT(dma_addrs);
|
|
|
|
for (i = 0; i < mem->chunks_count; ++i) {
|
|
if (dma_addrs[i] == 0) {
|
|
// The DMA address can only be 0 when cleaning up after a failed
|
|
// partial map_gpu_sysmem_iommu() operation.
|
|
break;
|
|
}
|
|
uvm_gpu_unmap_cpu_pages(gpu, dma_addrs[i], mem->chunk_size);
|
|
dma_addrs[i] = 0;
|
|
}
|
|
|
|
uvm_kvfree(dma_addrs);
|
|
mem->sysmem.dma_addrs[uvm_global_id_gpu_index(gpu->global_id)] = NULL;
|
|
}
|
|
|
|
static NV_STATUS sysmem_map_gpu_phys(uvm_mem_t *mem, uvm_gpu_t *gpu)
|
|
{
|
|
NV_STATUS status;
|
|
size_t i;
|
|
|
|
UVM_ASSERT(uvm_mem_is_sysmem(mem));
|
|
UVM_ASSERT(gpu != mem->dma_owner);
|
|
|
|
status = mem_alloc_dma_addrs(mem, gpu);
|
|
if (status != NV_OK)
|
|
return status;
|
|
|
|
for (i = 0; i < mem->chunks_count; ++i) {
|
|
status = uvm_gpu_map_cpu_pages(gpu,
|
|
mem->sysmem.pages[i],
|
|
mem->chunk_size,
|
|
&mem->sysmem.dma_addrs[uvm_global_id_gpu_index(gpu->global_id)][i]);
|
|
if (status != NV_OK)
|
|
goto error;
|
|
}
|
|
|
|
return NV_OK;
|
|
|
|
error:
|
|
sysmem_unmap_gpu_phys(mem, gpu);
|
|
return status;
|
|
}
|
|
|
|
static uvm_gpu_chunk_t *mem_get_chunk(uvm_mem_t *mem, size_t mem_offset, size_t *offset_in_chunk)
|
|
{
|
|
size_t chunk_index = uvm_div_pow2_64(mem_offset, mem->chunk_size);
|
|
|
|
if (offset_in_chunk)
|
|
*offset_in_chunk = mem_offset & (mem->chunk_size - 1);
|
|
|
|
UVM_ASSERT(uvm_mem_is_vidmem(mem));
|
|
return mem->vidmem.chunks[chunk_index];
|
|
}
|
|
|
|
static uvm_gpu_phys_address_t mem_gpu_physical_vidmem(uvm_mem_t *mem, size_t offset)
|
|
{
|
|
size_t chunk_offset;
|
|
uvm_gpu_chunk_t *chunk = mem_get_chunk(mem, offset, &chunk_offset);
|
|
return uvm_gpu_phys_address(UVM_APERTURE_VID, chunk->address + chunk_offset);
|
|
}
|
|
|
|
static uvm_gpu_phys_address_t mem_gpu_physical_sysmem(uvm_mem_t *mem, uvm_gpu_t *gpu, size_t offset)
|
|
{
|
|
NvU64 *dma_addrs = mem->sysmem.dma_addrs[uvm_global_id_gpu_index(gpu->global_id)];
|
|
NvU64 dma_addr = dma_addrs[offset / mem->chunk_size];
|
|
|
|
UVM_ASSERT(uvm_mem_is_sysmem(mem));
|
|
UVM_ASSERT(sysmem_mapped_on_gpu_phys(mem, gpu));
|
|
|
|
return uvm_gpu_phys_address(UVM_APERTURE_SYS, dma_addr + offset % mem->chunk_size);
|
|
}
|
|
|
|
static bool mem_check_range(uvm_mem_t *mem, NvU64 offset, NvU64 size)
|
|
{
|
|
UVM_ASSERT(size != 0);
|
|
UVM_ASSERT_MSG(UVM_ALIGN_DOWN(offset, mem->chunk_size) == UVM_ALIGN_DOWN(offset + size - 1, mem->chunk_size),
|
|
"offset %llu size %llu page_size %u\n",
|
|
offset,
|
|
size,
|
|
mem->chunk_size);
|
|
UVM_ASSERT_MSG(offset / mem->chunk_size < mem->chunks_count, "offset %llu\n", offset);
|
|
return true;
|
|
}
|
|
|
|
uvm_gpu_phys_address_t uvm_mem_gpu_physical(uvm_mem_t *mem, uvm_gpu_t *gpu, NvU64 offset, NvU64 size)
|
|
{
|
|
UVM_ASSERT(mem_check_range(mem, offset, size));
|
|
|
|
if (uvm_mem_is_vidmem(mem)) {
|
|
UVM_ASSERT(uvm_mem_is_local_vidmem(mem, gpu));
|
|
|
|
return mem_gpu_physical_vidmem(mem, offset);
|
|
}
|
|
|
|
return mem_gpu_physical_sysmem(mem, gpu, offset);
|
|
}
|
|
|
|
uvm_gpu_address_t uvm_mem_gpu_address_copy(uvm_mem_t *mem, uvm_gpu_t *accessing_gpu, NvU64 offset, NvU64 size)
|
|
{
|
|
uvm_gpu_address_t copy_addr;
|
|
size_t chunk_offset;
|
|
uvm_gpu_chunk_t *chunk;
|
|
|
|
UVM_ASSERT(mem_check_range(mem, offset, size));
|
|
|
|
if (uvm_mem_is_sysmem(mem) || uvm_mem_is_local_vidmem(mem, accessing_gpu))
|
|
return uvm_mem_gpu_address_physical(mem, accessing_gpu, offset, size);
|
|
|
|
// Peer GPUs may need to use some form of translation (identity mappings,
|
|
// indirect peers) to copy.
|
|
chunk = mem_get_chunk(mem, offset, &chunk_offset);
|
|
copy_addr = uvm_pmm_gpu_peer_copy_address(&mem->backing_gpu->pmm, chunk, accessing_gpu);
|
|
copy_addr.address += chunk_offset;
|
|
return copy_addr;
|
|
}
|
|
|
|
typedef struct uvm_mem_pte_maker_data_struct
|
|
{
|
|
uvm_mem_t *mem;
|
|
const uvm_mem_gpu_mapping_attrs_t *attrs;
|
|
} uvm_mem_pte_maker_data_t;
|
|
|
|
static NvU64 mem_pte_maker(uvm_page_table_range_vec_t *range_vec, NvU64 offset, void *vp_data)
|
|
{
|
|
uvm_mem_pte_maker_data_t *data = (uvm_mem_pte_maker_data_t *)vp_data;
|
|
uvm_page_tree_t *tree = range_vec->tree;
|
|
uvm_gpu_t *gpu = tree->gpu;
|
|
uvm_gpu_phys_address_t phys = uvm_mem_gpu_physical(data->mem, gpu, offset, range_vec->page_size);
|
|
|
|
return tree->hal->make_pte(phys.aperture,
|
|
phys.address,
|
|
data->attrs->protection,
|
|
data->attrs->is_cacheable ? UVM_MMU_PTE_FLAGS_CACHED : UVM_MMU_PTE_FLAGS_NONE);
|
|
}
|
|
|
|
static void mem_unmap_gpu(uvm_mem_t *mem, uvm_gpu_t *gpu, uvm_page_table_range_vec_t **range_vec)
|
|
{
|
|
NV_STATUS status;
|
|
uvm_membar_t tlb_membar = UVM_MEMBAR_SYS;
|
|
|
|
if (uvm_mem_is_local_vidmem(mem, gpu))
|
|
tlb_membar = UVM_MEMBAR_GPU;
|
|
|
|
status = uvm_page_table_range_vec_clear_ptes(*range_vec, tlb_membar);
|
|
if (status != NV_OK)
|
|
UVM_ERR_PRINT("Clearing PTEs failed: %s, GPU %s\n", nvstatusToString(status), uvm_gpu_name(gpu));
|
|
|
|
uvm_page_table_range_vec_destroy(*range_vec);
|
|
*range_vec = NULL;
|
|
}
|
|
|
|
static NV_STATUS mem_map_gpu(uvm_mem_t *mem,
|
|
uvm_gpu_t *gpu,
|
|
NvU64 gpu_va,
|
|
uvm_page_tree_t *tree,
|
|
const uvm_mem_gpu_mapping_attrs_t *attrs,
|
|
uvm_page_table_range_vec_t **range_vec)
|
|
{
|
|
NV_STATUS status;
|
|
NvU32 page_size;
|
|
uvm_pmm_alloc_flags_t pmm_flags = UVM_PMM_ALLOC_FLAGS_EVICT;
|
|
|
|
uvm_mem_pte_maker_data_t pte_maker_data = {
|
|
.mem = mem,
|
|
.attrs = attrs
|
|
};
|
|
|
|
if (!uvm_gpu_can_address(gpu, gpu_va, mem->size))
|
|
return NV_ERR_OUT_OF_RANGE;
|
|
|
|
page_size = mem_pick_gpu_page_size(mem, gpu, tree);
|
|
UVM_ASSERT_MSG(uvm_mmu_page_size_supported(tree, page_size), "page_size 0x%x\n", page_size);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
status = uvm_page_table_range_vec_create(tree,
|
|
gpu_va,
|
|
mem->physical_allocation_size,
|
|
page_size,
|
|
pmm_flags,
|
|
range_vec);
|
|
if (status != NV_OK) {
|
|
UVM_ERR_PRINT("Failed to init page mapping at [0x%llx, 0x%llx): %s, GPU %s\n",
|
|
gpu_va,
|
|
gpu_va + mem->physical_allocation_size,
|
|
nvstatusToString(status),
|
|
uvm_gpu_name(gpu));
|
|
return status;
|
|
}
|
|
|
|
status = uvm_page_table_range_vec_write_ptes(*range_vec, UVM_MEMBAR_NONE, mem_pte_maker, &pte_maker_data);
|
|
if (status != NV_OK) {
|
|
UVM_ERR_PRINT("Failed to write PTEs for mapping at [0x%llx, 0x%llx): %s, GPU %s\n",
|
|
gpu_va,
|
|
gpu_va + mem->physical_allocation_size,
|
|
nvstatusToString(status),
|
|
uvm_gpu_name(gpu));
|
|
goto error;
|
|
}
|
|
|
|
return NV_OK;
|
|
|
|
error:
|
|
mem_unmap_gpu(mem, gpu, range_vec);
|
|
return status;
|
|
}
|
|
|
|
static NV_STATUS mem_init_gpu_kernel_range(uvm_mem_t *mem)
|
|
{
|
|
if (mem->kernel.range_alloc.node != NULL)
|
|
return NV_OK;
|
|
|
|
return uvm_range_allocator_alloc(&g_free_ranges,
|
|
mem->physical_allocation_size,
|
|
mem->chunk_size,
|
|
&mem->kernel.range_alloc);
|
|
}
|
|
|
|
static void mem_deinit_gpu_kernel_range(uvm_mem_t *mem)
|
|
{
|
|
if (mem->kernel.range_alloc.node == NULL)
|
|
return;
|
|
|
|
// Do not remove the range allocation if there is any GPU where the memory
|
|
// is still mapped on kernel space.
|
|
if (UVM_GLOBAL_ID_IS_VALID(uvm_global_processor_mask_find_first_gpu_id(&mem->kernel.mapped_on)))
|
|
return;
|
|
|
|
uvm_range_allocator_free(&g_free_ranges, &mem->kernel.range_alloc);
|
|
}
|
|
|
|
NV_STATUS uvm_mem_map_gpu_kernel(uvm_mem_t *mem, uvm_gpu_t *gpu)
|
|
{
|
|
NV_STATUS status;
|
|
NvU64 gpu_va;
|
|
uvm_page_table_range_vec_t **range_vec;
|
|
uvm_mem_gpu_mapping_attrs_t attrs = {
|
|
.protection = UVM_PROT_READ_WRITE_ATOMIC,
|
|
.is_cacheable = uvm_mem_is_vidmem(mem)
|
|
};
|
|
|
|
UVM_ASSERT(mem_can_be_mapped_on_gpu_kernel(mem, gpu));
|
|
|
|
if (uvm_mem_mapped_on_gpu_kernel(mem, gpu))
|
|
return NV_OK;
|
|
|
|
status = uvm_mem_map_gpu_phys(mem, gpu);
|
|
if (status != NV_OK)
|
|
return status;
|
|
|
|
status = mem_init_gpu_kernel_range(mem);
|
|
if (status != NV_OK)
|
|
return status;
|
|
|
|
gpu_va = uvm_parent_gpu_canonical_address(gpu->parent, reserved_gpu_va(mem, gpu));
|
|
range_vec = &mem->kernel.range_vecs[uvm_global_id_gpu_index(gpu->global_id)];
|
|
|
|
status = mem_map_gpu(mem, gpu, gpu_va, &gpu->address_space_tree, &attrs, range_vec);
|
|
if (status != NV_OK)
|
|
goto cleanup;
|
|
|
|
mem_set_mapped_on_gpu_kernel(mem, gpu);
|
|
|
|
return NV_OK;
|
|
|
|
cleanup:
|
|
mem_deinit_gpu_kernel_range(mem);
|
|
|
|
return status;
|
|
}
|
|
|
|
NV_STATUS uvm_mem_map_gpu_user(uvm_mem_t *mem,
|
|
uvm_gpu_t *gpu,
|
|
uvm_va_space_t *user_va_space,
|
|
void *user_addr,
|
|
const uvm_mem_gpu_mapping_attrs_t *attrs)
|
|
{
|
|
NV_STATUS status;
|
|
uvm_gpu_va_space_t *gpu_va_space;
|
|
uvm_page_table_range_vec_t **range_vec;
|
|
|
|
UVM_ASSERT(mem_can_be_mapped_on_gpu_user(mem, gpu));
|
|
uvm_assert_rwsem_locked(&user_va_space->lock);
|
|
|
|
if (uvm_mem_mapped_on_gpu_user(mem, gpu))
|
|
return NV_OK;
|
|
|
|
status = uvm_mem_map_gpu_phys(mem, gpu);
|
|
if (status != NV_OK)
|
|
return status;
|
|
|
|
status = mem_init_user_mapping(mem, user_va_space, user_addr);
|
|
if (status != NV_OK)
|
|
return status;
|
|
|
|
gpu_va_space = uvm_gpu_va_space_get(mem->user->va_space, gpu);
|
|
range_vec = &mem->user->range_vecs[uvm_global_id_gpu_index(gpu->global_id)];
|
|
|
|
status = mem_map_gpu(mem, gpu, (NvU64)mem->user->addr, &gpu_va_space->page_tables, attrs, range_vec);
|
|
if (status != NV_OK)
|
|
goto cleanup;
|
|
|
|
mem_set_mapped_on_gpu_user(mem, gpu);
|
|
|
|
return NV_OK;
|
|
|
|
cleanup:
|
|
mem_deinit_user_mapping(mem);
|
|
return status;
|
|
}
|
|
|
|
void uvm_mem_unmap_gpu_user(uvm_mem_t *mem, uvm_gpu_t *gpu)
|
|
{
|
|
if (!uvm_mem_mapped_on_gpu_user(mem, gpu))
|
|
return;
|
|
|
|
mem_unmap_gpu(mem, gpu, &mem->user->range_vecs[uvm_global_id_gpu_index(gpu->global_id)]);
|
|
mem_clear_mapped_on_gpu_user(mem, gpu);
|
|
mem_deinit_user_mapping(mem);
|
|
}
|
|
|
|
void uvm_mem_unmap_gpu_kernel(uvm_mem_t *mem, uvm_gpu_t *gpu)
|
|
{
|
|
if (!uvm_mem_mapped_on_gpu_kernel(mem, gpu))
|
|
return;
|
|
|
|
mem_unmap_gpu(mem, gpu, &mem->kernel.range_vecs[uvm_global_id_gpu_index(gpu->global_id)]);
|
|
mem_clear_mapped_on_gpu_kernel(mem, gpu);
|
|
mem_deinit_gpu_kernel_range(mem);
|
|
}
|
|
|
|
static bool mem_can_be_phys_mapped_on_gpu(uvm_mem_t *mem, uvm_gpu_t *gpu)
|
|
{
|
|
if (uvm_mem_is_sysmem(mem))
|
|
return sysmem_can_be_mapped(mem);
|
|
else
|
|
return uvm_mem_is_local_vidmem(mem, gpu);
|
|
}
|
|
|
|
NV_STATUS uvm_mem_map_gpu_phys(uvm_mem_t *mem, uvm_gpu_t *gpu)
|
|
{
|
|
NV_STATUS status;
|
|
|
|
UVM_ASSERT(mem_can_be_phys_mapped_on_gpu(mem, gpu));
|
|
|
|
if (uvm_mem_is_vidmem(mem))
|
|
return NV_OK;
|
|
|
|
if (gpu == mem->dma_owner)
|
|
return NV_OK;
|
|
|
|
if (sysmem_mapped_on_gpu_phys(mem, gpu))
|
|
return NV_OK;
|
|
|
|
status = sysmem_map_gpu_phys(mem, gpu);
|
|
if (status != NV_OK)
|
|
return status;
|
|
|
|
sysmem_set_mapped_on_gpu_phys(mem, gpu);
|
|
return NV_OK;
|
|
}
|
|
|
|
void uvm_mem_unmap_gpu_phys(uvm_mem_t *mem, uvm_gpu_t *gpu)
|
|
{
|
|
UVM_ASSERT(mem);
|
|
UVM_ASSERT(gpu);
|
|
|
|
if (uvm_mem_is_vidmem(mem))
|
|
return;
|
|
|
|
// GPU for which the mapping is managed by the dma_alloc_coherent
|
|
// API will be unmapped when the allocation is freed.
|
|
if (gpu == mem->dma_owner)
|
|
return;
|
|
|
|
if (!sysmem_mapped_on_gpu_phys(mem, gpu))
|
|
return;
|
|
|
|
uvm_mem_unmap_gpu_user(mem, gpu);
|
|
uvm_mem_unmap_gpu_kernel(mem, gpu);
|
|
|
|
sysmem_unmap_gpu_phys(mem, gpu);
|
|
sysmem_clear_mapped_on_gpu_phys(mem, gpu);
|
|
}
|
|
|
|
void uvm_mem_free(uvm_mem_t *mem)
|
|
{
|
|
uvm_gpu_t *gpu;
|
|
|
|
if (mem == NULL)
|
|
return;
|
|
|
|
uvm_mem_unmap_cpu_user(mem);
|
|
uvm_mem_unmap_cpu_kernel(mem);
|
|
|
|
if (mem->user != NULL) {
|
|
for_each_global_gpu_in_mask(gpu, &mem->user->mapped_on) {
|
|
uvm_mem_unmap_gpu_user(mem, gpu);
|
|
|
|
// If we unmapped the last device, the user mapping is freed, so
|
|
// exit the loop before the iterator accesses a non-existing mask.
|
|
if (mem->user == NULL)
|
|
break;
|
|
}
|
|
}
|
|
|
|
for_each_global_gpu_in_mask(gpu, &mem->kernel.mapped_on)
|
|
uvm_mem_unmap_gpu_kernel(mem, gpu);
|
|
|
|
if (uvm_mem_is_sysmem(mem)) {
|
|
for_each_global_gpu_in_mask(gpu, &mem->sysmem.mapped_on_phys)
|
|
uvm_mem_unmap_gpu_phys(mem, gpu);
|
|
}
|
|
|
|
mem_free_chunks(mem);
|
|
|
|
uvm_kvfree(mem);
|
|
}
|
|
|
|
void *uvm_mem_get_cpu_addr_kernel(uvm_mem_t *mem)
|
|
{
|
|
UVM_ASSERT(uvm_mem_mapped_on_cpu_kernel(mem));
|
|
|
|
return mem->kernel.cpu_addr;
|
|
}
|
|
|
|
NvU64 uvm_mem_get_gpu_va_kernel(uvm_mem_t *mem, uvm_gpu_t *gpu)
|
|
{
|
|
UVM_ASSERT(uvm_mem_mapped_on_gpu_kernel(mem, gpu));
|
|
|
|
return reserved_gpu_va(mem, gpu);
|
|
}
|
|
|
|
uvm_gpu_address_t uvm_mem_gpu_address_virtual_kernel(uvm_mem_t *mem, uvm_gpu_t *gpu)
|
|
{
|
|
return uvm_gpu_address_virtual(uvm_mem_get_gpu_va_kernel(mem, gpu));
|
|
}
|
|
|
|
uvm_gpu_address_t uvm_mem_gpu_address_physical(uvm_mem_t *mem, uvm_gpu_t *gpu, NvU64 offset, NvU64 size)
|
|
{
|
|
return uvm_gpu_address_from_phys(uvm_mem_gpu_physical(mem, gpu, offset, size));
|
|
}
|