diff --git a/kernel-open/common/inc/os-interface.h b/kernel-open/common/inc/os-interface.h index 9d5ac0a46..0bf90e4c3 100644 --- a/kernel-open/common/inc/os-interface.h +++ b/kernel-open/common/inc/os-interface.h @@ -169,7 +169,7 @@ NvBool NV_API_CALL os_is_grid_supported (void); NvU32 NV_API_CALL os_get_grid_csp_support (void); void NV_API_CALL os_bug_check (NvU32, const char *); NV_STATUS NV_API_CALL os_lock_user_pages (void *, NvU64, void **, NvU32); -NV_STATUS NV_API_CALL os_lookup_user_io_memory (void *, NvU64, NvU64 **); +NV_STATUS NV_API_CALL os_lookup_user_io_memory (void *, NvU64, NvU64 **, void **); NV_STATUS NV_API_CALL os_unlock_user_pages (NvU64, void *); NV_STATUS NV_API_CALL os_match_mmap_offset (void *, NvU64, NvU64 *); NV_STATUS NV_API_CALL os_get_euid (NvU32 *); diff --git a/kernel-open/nvidia/os-mlock.c b/kernel-open/nvidia/os-mlock.c index b0c32c1b4..d90c20f3a 100644 --- a/kernel-open/nvidia/os-mlock.c +++ b/kernel-open/nvidia/os-mlock.c @@ -111,10 +111,59 @@ static NV_STATUS get_io_ptes(struct vm_area_struct *vma, return NV_OK; } +/*! + * @brief Pins user IO pages that have been mapped to the user processes virtual + * address space with remap_pfn_range. + * + * @param[in] vma VMA that contains the virtual address range given by the + * start and the page count. + * @param[in] start Beginning of the virtual address range of the IO pages. + * @param[in] page_count Number of pages to pin from start. + * @param[in,out] page_array Storage array for pointers to the pinned pages. + * Must be large enough to contain at least page_count + * pointers. + * + * @return NV_OK if the pages were pinned successfully, error otherwise. + */ +static NV_STATUS get_io_pages(struct vm_area_struct *vma, + NvUPtr start, + NvU64 page_count, + struct page **page_array) +{ + NV_STATUS rmStatus = NV_OK; + NvU64 i, pinned = 0; + unsigned long pfn; + + for (i = 0; i < page_count; i++) + { + if ((nv_follow_pfn(vma, (start + (i * PAGE_SIZE)), &pfn) < 0) || + (!pfn_valid(pfn))) + { + rmStatus = NV_ERR_INVALID_ADDRESS; + break; + } + + // Page-backed memory mapped to userspace with remap_pfn_range + page_array[i] = pfn_to_page(pfn); + get_page(page_array[i]); + pinned++; + } + + if (pinned < page_count) + { + for (i = 0; i < pinned; i++) + put_page(page_array[i]); + rmStatus = NV_ERR_INVALID_ADDRESS; + } + + return rmStatus; +} + NV_STATUS NV_API_CALL os_lookup_user_io_memory( void *address, NvU64 page_count, - NvU64 **pte_array + NvU64 **pte_array, + void **page_array ) { NV_STATUS rmStatus; @@ -162,9 +211,18 @@ NV_STATUS NV_API_CALL os_lookup_user_io_memory( goto done; } - rmStatus = get_io_ptes(vma, start, page_count, (NvU64 **)result_array); - if (rmStatus == NV_OK) - *pte_array = (NvU64 *)result_array; + if (pfn_valid(pfn)) + { + rmStatus = get_io_pages(vma, start, page_count, (struct page **)result_array); + if (rmStatus == NV_OK) + *page_array = (void *)result_array; + } + else + { + rmStatus = get_io_ptes(vma, start, page_count, (NvU64 **)result_array); + if (rmStatus == NV_OK) + *pte_array = (NvU64 *)result_array; + } done: nv_mmap_read_unlock(mm); diff --git a/src/nvidia/arch/nvalloc/unix/include/os-interface.h b/src/nvidia/arch/nvalloc/unix/include/os-interface.h index c01966dd8..500d913bd 100644 --- a/src/nvidia/arch/nvalloc/unix/include/os-interface.h +++ b/src/nvidia/arch/nvalloc/unix/include/os-interface.h @@ -165,7 +165,7 @@ NvBool NV_API_CALL os_is_grid_supported (void); NvU32 NV_API_CALL os_get_grid_csp_support (void); void NV_API_CALL os_bug_check (NvU32, const char *); NV_STATUS NV_API_CALL os_lock_user_pages (void *, NvU64, void **, NvU32); -NV_STATUS NV_API_CALL os_lookup_user_io_memory (void *, NvU64, NvU64 **); +NV_STATUS NV_API_CALL os_lookup_user_io_memory (void *, NvU64, NvU64 **, void **); NV_STATUS NV_API_CALL os_unlock_user_pages (NvU64, void *); NV_STATUS NV_API_CALL os_match_mmap_offset (void *, NvU64, NvU64 *); NV_STATUS NV_API_CALL os_get_euid (NvU32 *); diff --git a/src/nvidia/arch/nvalloc/unix/src/escape.c b/src/nvidia/arch/nvalloc/unix/src/escape.c index 1046b19f9..a0d876e23 100644 --- a/src/nvidia/arch/nvalloc/unix/src/escape.c +++ b/src/nvidia/arch/nvalloc/unix/src/escape.c @@ -167,11 +167,25 @@ static void RmCreateOsDescriptor(NVOS32_PARAMETERS *pApi, API_SECURITY_INFO secI } else if (rmStatus == NV_ERR_INVALID_ADDRESS) { - rmStatus = os_lookup_user_io_memory(pDescriptor, pageCount, &pPteArray); + rmStatus = os_lookup_user_io_memory(pDescriptor, pageCount, + &pPteArray, &pPageArray); if (rmStatus == NV_OK) { - pApi->data.AllocOsDesc.descriptor = (NvP64)(NvUPtr)pPteArray; - pApi->data.AllocOsDesc.descriptorType = NVOS32_DESCRIPTOR_TYPE_OS_IO_MEMORY; + if (pPageArray != NULL) + { + pApi->data.AllocOsDesc.descriptor = (NvP64)(NvUPtr)pPageArray; + pApi->data.AllocOsDesc.descriptorType = NVOS32_DESCRIPTOR_TYPE_OS_PAGE_ARRAY; + } + else if (pPteArray != NULL) + { + pApi->data.AllocOsDesc.descriptor = (NvP64)(NvUPtr)pPteArray; + pApi->data.AllocOsDesc.descriptorType = NVOS32_DESCRIPTOR_TYPE_OS_IO_MEMORY; + } + else + { + NV_ASSERT_FAILED("unknown memory import type"); + rmStatus = NV_ERR_NOT_SUPPORTED; + } } } if (rmStatus != NV_OK)