/******************************************************************************* Copyright (c) 2015-2022 NVIDIA Corporation Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. *******************************************************************************/ #include "uvm_rm_mem.h" #include "uvm_test.h" #include "uvm_test_ioctl.h" #include "uvm_va_space.h" #include "uvm_kvmalloc.h" static NV_STATUS map_cpu(uvm_rm_mem_t *rm_mem) { void *cpu_va; // Map TEST_NV_CHECK_RET(uvm_rm_mem_map_cpu(rm_mem)); TEST_CHECK_RET(uvm_rm_mem_mapped_on_cpu(rm_mem)); // Mapping if already mapped is OK TEST_NV_CHECK_RET(uvm_rm_mem_map_cpu(rm_mem)); // Unmap uvm_rm_mem_unmap_cpu(rm_mem); // Unmapping already unmapped also OK uvm_rm_mem_unmap_cpu(rm_mem); // Map again TEST_NV_CHECK_RET(uvm_rm_mem_map_cpu(rm_mem)); cpu_va = uvm_rm_mem_get_cpu_va(rm_mem); TEST_CHECK_RET(cpu_va != NULL); // Check that the CPU VA is writable. // memset and memcpy might cause alignment faults on aarch64. // See Bug 2668765 for more details. Since this is a test ioctl and // therefore not a critical performance path, it's safe to use memset_io on // all platforms. memset_io(cpu_va, 0, rm_mem->size); return NV_OK; } static NV_STATUS check_alignment(uvm_rm_mem_t *rm_mem, uvm_gpu_t *gpu, NvU64 alignment) { // Alignment requirements only apply to mappings in the UVM-owned VA space if (alignment != 0) { NvU64 gpu_uvm_va = uvm_rm_mem_get_gpu_uvm_va(rm_mem, gpu); TEST_CHECK_RET(IS_ALIGNED(gpu_uvm_va, alignment)); } return NV_OK; } static NV_STATUS map_gpu_owner(uvm_rm_mem_t *rm_mem, NvU64 alignment) { uvm_gpu_t *gpu = rm_mem->gpu_owner; NvU64 gpu_uvm_va; NvU64 gpu_proxy_va = 0; // The memory should have been automatically mapped in the GPU owner TEST_CHECK_RET(uvm_rm_mem_mapped_on_gpu(rm_mem, gpu)); gpu_uvm_va = uvm_rm_mem_get_gpu_uvm_va(rm_mem, gpu); // In SR-IOV heavy, there are two VA spaces per GPU, so there are two // mappings for a single rm_mem object on a GPU, even if the memory is // located in vidmem. if (uvm_gpu_uses_proxy_channel_pool(gpu)) { TEST_CHECK_RET(uvm_rm_mem_mapped_on_gpu_proxy(rm_mem, gpu)); gpu_proxy_va = uvm_rm_mem_get_gpu_proxy_va(rm_mem, gpu); } else { TEST_CHECK_RET(!uvm_rm_mem_mapped_on_gpu_proxy(rm_mem, gpu)); } TEST_NV_CHECK_RET(check_alignment(rm_mem, gpu, alignment)); // Mappings are not ref counted, so additional map calls are no-ops; the // GPU VA should remain the same for all the applicable VA spaces. TEST_NV_CHECK_RET(uvm_rm_mem_map_gpu(rm_mem, gpu, alignment)); TEST_CHECK_RET(gpu_uvm_va == uvm_rm_mem_get_gpu_uvm_va(rm_mem, gpu)); if (uvm_gpu_uses_proxy_channel_pool(gpu)) TEST_CHECK_RET(gpu_proxy_va == uvm_rm_mem_get_gpu_proxy_va(rm_mem, gpu)); // Unmapping the GPU owner is a no-op uvm_rm_mem_unmap_gpu(rm_mem, gpu); TEST_CHECK_RET(uvm_rm_mem_mapped_on_gpu(rm_mem, gpu)); TEST_CHECK_RET(gpu_uvm_va == uvm_rm_mem_get_gpu_uvm_va(rm_mem, gpu)); if (uvm_gpu_uses_proxy_channel_pool(gpu)) { TEST_CHECK_RET(uvm_rm_mem_mapped_on_gpu_proxy(rm_mem, gpu)); TEST_CHECK_RET(gpu_proxy_va == uvm_rm_mem_get_gpu_proxy_va(rm_mem, gpu)); } else { TEST_CHECK_RET(!uvm_rm_mem_mapped_on_gpu_proxy(rm_mem, gpu)); } return NV_OK; } static NV_STATUS map_other_gpus(uvm_rm_mem_t *rm_mem, uvm_va_space_t *va_space, NvU64 alignment) { uvm_gpu_t *gpu_owner = rm_mem->gpu_owner; uvm_gpu_t *gpu; for_each_va_space_gpu(gpu, va_space) { if (gpu == gpu_owner) continue; TEST_NV_CHECK_RET(uvm_rm_mem_map_gpu(rm_mem, gpu, alignment)); TEST_CHECK_RET(uvm_rm_mem_mapped_on_gpu(rm_mem, gpu)); // Mappings are not ref counted, so additional map calls are no-ops TEST_NV_CHECK_RET(uvm_rm_mem_map_gpu(rm_mem, gpu, alignment)); // The previous GPU map calls added mappings to the proxy VA space // when in SR-IOV heavy mode TEST_CHECK_RET(uvm_rm_mem_mapped_on_gpu_proxy(rm_mem, gpu) == uvm_gpu_uses_proxy_channel_pool(gpu)); // Unmapping removes all mappings uvm_rm_mem_unmap_gpu(rm_mem, gpu); TEST_CHECK_RET(!uvm_rm_mem_mapped_on_gpu(rm_mem, gpu)); TEST_CHECK_RET(!uvm_rm_mem_mapped_on_gpu_proxy(rm_mem, gpu)); // Additional unmappings are no-ops uvm_rm_mem_unmap_gpu(rm_mem, gpu); TEST_CHECK_RET(!uvm_rm_mem_mapped_on_gpu(rm_mem, gpu)); TEST_CHECK_RET(!uvm_rm_mem_mapped_on_gpu_proxy(rm_mem, gpu)); // Subsequent mappings should behave as they did in the beginning. TEST_NV_CHECK_RET(uvm_rm_mem_map_gpu(rm_mem, gpu, alignment)); TEST_CHECK_RET(uvm_rm_mem_mapped_on_gpu(rm_mem, gpu)); TEST_CHECK_RET(uvm_rm_mem_mapped_on_gpu_proxy(rm_mem, gpu) == uvm_gpu_uses_proxy_channel_pool(gpu)); TEST_NV_CHECK_RET(check_alignment(rm_mem, gpu, alignment)); } return NV_OK; } static NV_STATUS test_all_gpus_in_va(uvm_va_space_t *va_space) { uvm_gpu_t *gpu; uvm_rm_mem_t *rm_mem = NULL; NV_STATUS status = NV_OK; // Create allocations of these types static const uvm_rm_mem_type_t mem_types[] = { UVM_RM_MEM_TYPE_SYS, UVM_RM_MEM_TYPE_GPU }; // Create allocations of these sizes static const size_t sizes[] = { 1, 4, 16, 128, 1024, 4096, 1024 * 1024, 4 * 1024 * 1024 }; static const NvU64 alignments[] = { 0, 8, UVM_PAGE_SIZE_4K >> 1, UVM_PAGE_SIZE_4K, UVM_PAGE_SIZE_4K << 1, UVM_PAGE_SIZE_64K, UVM_PAGE_SIZE_2M, UVM_PAGE_SIZE_2M << 3, UVM_PAGE_SIZE_2M << 5 }; uvm_assert_rwsem_locked(&va_space->lock); TEST_CHECK_RET(!uvm_processor_mask_empty(&va_space->registered_gpus)); for_each_va_space_gpu(gpu, va_space) { int i, j, k; for (i = 0; i < ARRAY_SIZE(sizes); ++i) { for (j = 0; j < ARRAY_SIZE(mem_types); ++j) { for (k = 0; k < ARRAY_SIZE(alignments); ++k) { bool test_cpu_mappings = true; // Create an allocation in the GPU's address space TEST_NV_CHECK_RET(uvm_rm_mem_alloc(gpu, mem_types[j], sizes[i], alignments[k], &rm_mem)); test_cpu_mappings = mem_types[j] == UVM_RM_MEM_TYPE_SYS || !uvm_conf_computing_mode_enabled(gpu); // Test CPU mappings if (test_cpu_mappings) TEST_NV_CHECK_GOTO(map_cpu(rm_mem), error); // Test mappings in the GPU owning the allocation TEST_NV_CHECK_GOTO(map_gpu_owner(rm_mem, alignments[k]), error); // For sysmem allocations, test mappings on all other GPUs if (rm_mem->type == UVM_RM_MEM_TYPE_SYS) TEST_NV_CHECK_GOTO(map_other_gpus(rm_mem, va_space, alignments[k]), error); uvm_rm_mem_free(rm_mem); } } } } return NV_OK; error: uvm_rm_mem_free(rm_mem); return status; } NV_STATUS uvm_test_rm_mem_sanity(UVM_TEST_RM_MEM_SANITY_PARAMS *params, struct file *filp) { NV_STATUS status; uvm_va_space_t *va_space = uvm_va_space_get(filp); uvm_va_space_down_read_rm(va_space); status = test_all_gpus_in_va(va_space); uvm_va_space_up_read_rm(va_space); return status; }