mirror of
https://github.com/NVIDIA/open-gpu-kernel-modules.git
synced 2024-12-04 15:24:17 +01:00
435 lines
20 KiB
C
435 lines
20 KiB
C
/*******************************************************************************
|
|
Copyright (c) 2016-2022 NVIDIA Corporation
|
|
|
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
of this software and associated documentation files (the "Software"), to
|
|
deal in the Software without restriction, including without limitation the
|
|
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
|
sell copies of the Software, and to permit persons to whom the Software is
|
|
furnished to do so, subject to the following conditions:
|
|
|
|
The above copyright notice and this permission notice shall be
|
|
included in all copies or substantial portions of the Software.
|
|
|
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
|
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
DEALINGS IN THE SOFTWARE.
|
|
|
|
*******************************************************************************/
|
|
|
|
#ifndef _UVM_HMM_H_
|
|
#define _UVM_HMM_H_
|
|
|
|
#include "nvtypes.h"
|
|
#include "uvm_forward_decl.h"
|
|
#include "uvm_va_block_types.h"
|
|
#include "uvm_va_policy.h"
|
|
#include "uvm_linux.h"
|
|
#include "uvm_range_tree.h"
|
|
#include "uvm_lock.h"
|
|
|
|
typedef struct
|
|
{
|
|
// This stores pointers to uvm_va_block_t for HMM blocks.
|
|
uvm_range_tree_t blocks;
|
|
uvm_mutex_t blocks_lock;
|
|
|
|
// TODO: Bug 3351822: [UVM-HMM] Remove temporary testing changes.
|
|
// This flag is set true by default for each va_space so most processes
|
|
// don't see partially implemented UVM-HMM behavior but can be enabled by
|
|
// test code for a given va_space so the test process can do some interim
|
|
// testing. It needs to be a separate flag instead of modifying
|
|
// uvm_disable_hmm or va_space->flags since those are user inputs and are
|
|
// visible/checked by test code.
|
|
// Remove this when UVM-HMM is fully integrated into chips_a.
|
|
bool disable;
|
|
} uvm_hmm_va_space_t;
|
|
|
|
#if UVM_IS_CONFIG_HMM()
|
|
// Tells whether HMM is enabled for the given va_space.
|
|
// If it is not enabled, all of the functions below are no-ops.
|
|
bool uvm_hmm_is_enabled(uvm_va_space_t *va_space);
|
|
|
|
// Self-explanatory name: reports if HMM is enabled system-wide.
|
|
bool uvm_hmm_is_enabled_system_wide(void);
|
|
|
|
// Initialize HMM for the given the va_space.
|
|
// Locking: the va_space->va_space_mm.mm mmap_lock must be write locked
|
|
// and the va_space lock must be held in write mode.
|
|
NV_STATUS uvm_hmm_va_space_initialize(uvm_va_space_t *va_space);
|
|
|
|
// Initialize HMM for the given the va_space for testing.
|
|
// Bug 1750144: UVM: Add HMM (Heterogeneous Memory Management) support to
|
|
// the UVM driver. Remove this when enough HMM functionality is implemented.
|
|
// Locking: the va_space->va_space_mm.mm mmap_lock must be write locked
|
|
// and the va_space lock must be held in write mode.
|
|
NV_STATUS uvm_hmm_va_space_initialize_test(uvm_va_space_t *va_space);
|
|
|
|
// Destroy any HMM state for the given the va_space.
|
|
// Locking: va_space lock must be held in write mode.
|
|
void uvm_hmm_va_space_destroy(uvm_va_space_t *va_space);
|
|
|
|
// Find an existing HMM va_block.
|
|
// This function can be called without having retained and locked the mm,
|
|
// but in that case, the only allowed operations on the returned block are
|
|
// locking the block, reading its state, and performing eviction. GPU fault
|
|
// handling and user-initiated migrations are not allowed.
|
|
// Return values are the same as uvm_va_block_find().
|
|
// Locking: This must be called with va_space lock held in at least read
|
|
// mode.
|
|
NV_STATUS uvm_hmm_va_block_find(uvm_va_space_t *va_space,
|
|
NvU64 addr,
|
|
uvm_va_block_t **va_block_ptr);
|
|
|
|
// Find or create a new HMM va_block.
|
|
//
|
|
// Return NV_ERR_INVALID_ADDRESS if there is no VMA associated with the
|
|
// address 'addr' or the VMA does not have at least PROT_READ permission.
|
|
// The caller is also responsible for checking that there is no UVM
|
|
// va_range covering the given address before calling this function.
|
|
// If va_block_context is not NULL, the VMA is cached in
|
|
// va_block_context->hmm.vma.
|
|
// Locking: This function must be called with mm retained and locked for
|
|
// at least read and the va_space lock at least for read.
|
|
NV_STATUS uvm_hmm_va_block_find_create(uvm_va_space_t *va_space,
|
|
NvU64 addr,
|
|
uvm_va_block_context_t *va_block_context,
|
|
uvm_va_block_t **va_block_ptr);
|
|
|
|
// Find the VMA for the given address and set va_block_context->hmm.vma.
|
|
// Return NV_ERR_INVALID_ADDRESS if va_block_context->mm is NULL or there
|
|
// is no VMA associated with the address 'addr' or the VMA does not have at
|
|
// least PROT_READ permission.
|
|
// Locking: This function must be called with mm retained and locked for
|
|
// at least read or mm equal to NULL.
|
|
NV_STATUS uvm_hmm_find_vma(uvm_va_block_context_t *va_block_context, NvU64 addr);
|
|
|
|
// If va_block is a HMM va_block, check that va_block_context->hmm.vma is
|
|
// not NULL and covers the given region. This always returns true and is
|
|
// intended to only be used with UVM_ASSERT().
|
|
// Locking: This function must be called with the va_block lock held and if
|
|
// va_block is a HMM block, va_block_context->mm must be retained and
|
|
// locked for at least read.
|
|
bool uvm_hmm_va_block_context_vma_is_valid(uvm_va_block_t *va_block,
|
|
uvm_va_block_context_t *va_block_context,
|
|
uvm_va_block_region_t region);
|
|
|
|
// Find or create a HMM va_block and mark it so the next va_block split
|
|
// will fail for testing purposes.
|
|
// Locking: This function must be called with mm retained and locked for
|
|
// at least read and the va_space lock at least for read.
|
|
NV_STATUS uvm_hmm_test_va_block_inject_split_error(uvm_va_space_t *va_space, NvU64 addr);
|
|
|
|
// Reclaim any HMM va_blocks that overlap the given range.
|
|
// Note that 'end' is inclusive. If mm is NULL, any HMM va_block in the
|
|
// range will be reclaimed which assumes that the mm is being torn down
|
|
// and was not retained.
|
|
// Return values:
|
|
// NV_ERR_NO_MEMORY: Reclaim required a block split, which failed.
|
|
// NV_OK: There were no HMM blocks in the range, or all HMM
|
|
// blocks in the range were successfully reclaimed.
|
|
// Locking: If mm is not NULL, it must equal va_space_mm.mm, the caller
|
|
// must retain it with uvm_va_space_mm_or_current_retain() or be sure that
|
|
// mm->mm_users is not zero, and it must be locked for at least read mode.
|
|
// Also, the va_space lock must be held in write mode.
|
|
// TODO: Bug 3372166: add asynchronous va_block reclaim.
|
|
NV_STATUS uvm_hmm_va_block_reclaim(uvm_va_space_t *va_space,
|
|
struct mm_struct *mm,
|
|
NvU64 start,
|
|
NvU64 end);
|
|
|
|
// This is called to update the va_space tree of HMM va_blocks after an
|
|
// existing va_block is split.
|
|
// Locking: the va_space lock must be held in write mode.
|
|
void uvm_hmm_va_block_split_tree(uvm_va_block_t *existing_va_block, uvm_va_block_t *new_block);
|
|
|
|
// Find a HMM policy range that needs to be split. The callback function
|
|
// 'split_needed_cb' returns true if the policy range needs to be split.
|
|
// If a policy range is split, the existing range is truncated to
|
|
// [existing_start, addr - 1] and a new policy node with the same policy
|
|
// values as the existing node is created covering [addr, existing_end].
|
|
// Before: [----------- existing ------------]
|
|
// After: [---- existing ----][---- new ----]
|
|
// ^addr
|
|
// Locking: the va_space must be write locked.
|
|
NV_STATUS uvm_hmm_split_as_needed(uvm_va_space_t *va_space,
|
|
NvU64 addr,
|
|
uvm_va_policy_is_split_needed_t split_needed_cb,
|
|
void *data);
|
|
|
|
// Set the preferred location policy for the given range.
|
|
// Note that 'last_address' is inclusive.
|
|
// Locking: the va_space->va_space_mm.mm mmap_lock must be locked
|
|
// and the va_space lock must be held in write mode.
|
|
NV_STATUS uvm_hmm_set_preferred_location(uvm_va_space_t *va_space,
|
|
uvm_processor_id_t preferred_location,
|
|
NvU64 base,
|
|
NvU64 last_address);
|
|
|
|
// Set the accessed by policy for the given range. This also tries to
|
|
// map the range. Note that 'last_address' is inclusive.
|
|
// Locking: the va_space->va_space_mm.mm mmap_lock must be locked
|
|
// and the va_space lock must be held in write mode.
|
|
NV_STATUS uvm_hmm_set_accessed_by(uvm_va_space_t *va_space,
|
|
uvm_processor_id_t processor_id,
|
|
bool set_bit,
|
|
NvU64 base,
|
|
NvU64 last_address);
|
|
|
|
// Set the read duplication policy for the given range.
|
|
// Note that 'last_address' is inclusive.
|
|
// Locking: the va_space->va_space_mm.mm mmap_lock must be write locked
|
|
// and the va_space lock must be held in write mode.
|
|
// TODO: Bug 3660922: need to implement HMM read duplication support.
|
|
static NV_STATUS uvm_hmm_set_read_duplication(uvm_va_space_t *va_space,
|
|
uvm_read_duplication_policy_t new_policy,
|
|
NvU64 base,
|
|
NvU64 last_address)
|
|
{
|
|
if (!uvm_hmm_is_enabled(va_space))
|
|
return NV_ERR_INVALID_ADDRESS;
|
|
return NV_OK;
|
|
}
|
|
|
|
// This function assigns va_block_context->policy to the policy covering
|
|
// the given address 'addr' and assigns the ending address '*endp' to the
|
|
// minimum of va_block->end, va_block_context->hmm.vma->vm_end - 1, and the
|
|
// ending address of the policy range. Note that va_block_context->hmm.vma
|
|
// is expected to be initialized before calling this function.
|
|
// Locking: This function must be called with
|
|
// va_block_context->hmm.vma->vm_mm retained and locked for least read and
|
|
// the va_block lock held.
|
|
void uvm_hmm_find_policy_end(uvm_va_block_t *va_block,
|
|
uvm_va_block_context_t *va_block_context,
|
|
unsigned long addr,
|
|
NvU64 *endp);
|
|
|
|
// This function finds the VMA for the page index 'page_index' and assigns
|
|
// it to va_block_context->vma, sets va_block_context->policy to the policy
|
|
// covering the given address, and sets the ending page range '*outerp'
|
|
// to the minimum of *outerp, va_block_context->hmm.vma->vm_end - 1, the
|
|
// ending address of the policy range, and va_block->end.
|
|
// Return NV_ERR_INVALID_ADDRESS if no VMA is found; otherwise, NV_OK.
|
|
// Locking: This function must be called with
|
|
// va_block_context->hmm.vma->vm_mm retained and locked for least read and
|
|
// the va_block lock held.
|
|
NV_STATUS uvm_hmm_find_policy_vma_and_outer(uvm_va_block_t *va_block,
|
|
uvm_va_block_context_t *va_block_context,
|
|
uvm_page_index_t page_index,
|
|
uvm_page_index_t *outerp);
|
|
|
|
// Clear thrashing policy information from all HMM va_blocks.
|
|
// Locking: va_space lock must be held in write mode.
|
|
NV_STATUS uvm_hmm_clear_thrashing_policy(uvm_va_space_t *va_space);
|
|
|
|
// Return the expanded region around 'address' limited to the intersection
|
|
// of va_block start/end, vma start/end, and policy start/end.
|
|
// va_block_context must not be NULL, va_block_context->hmm.vma must be
|
|
// valid (this is usually set by uvm_hmm_va_block_find_create()), and
|
|
// va_block_context->policy must be valid.
|
|
// Locking: the caller must hold mm->mmap_lock in at least read mode, the
|
|
// va_space lock must be held in at least read mode, and the va_block lock
|
|
// held.
|
|
uvm_va_block_region_t uvm_hmm_get_prefetch_region(uvm_va_block_t *va_block,
|
|
uvm_va_block_context_t *va_block_context,
|
|
NvU64 address);
|
|
|
|
// Return the logical protection allowed of a HMM va_block for the page at
|
|
// the given address.
|
|
// va_block_context must not be NULL and va_block_context->hmm.vma must be
|
|
// valid (this is usually set by uvm_hmm_va_block_find_create()).
|
|
// Locking: the caller must hold va_block_context->mm mmap_lock in at least
|
|
// read mode.
|
|
uvm_prot_t uvm_hmm_compute_logical_prot(uvm_va_block_t *va_block,
|
|
uvm_va_block_context_t *va_block_context,
|
|
NvU64 addr);
|
|
|
|
NV_STATUS uvm_test_hmm_init(UVM_TEST_HMM_INIT_PARAMS *params, struct file *filp);
|
|
|
|
NV_STATUS uvm_test_split_invalidate_delay(UVM_TEST_SPLIT_INVALIDATE_DELAY_PARAMS *params,
|
|
struct file *filp);
|
|
|
|
NV_STATUS uvm_hmm_va_range_info(uvm_va_space_t *va_space,
|
|
struct mm_struct *mm,
|
|
UVM_TEST_VA_RANGE_INFO_PARAMS *params);
|
|
|
|
// Return true if GPU fault new residency location should be system memory.
|
|
// va_block_context must not be NULL and va_block_context->hmm.vma must be
|
|
// valid (this is usually set by uvm_hmm_va_block_find_create()).
|
|
// TODO: Bug 3660968: Remove this hack as soon as HMM migration is
|
|
// implemented for VMAs other than anonymous memory.
|
|
// Locking: the va_block lock must be held. If the va_block is a HMM
|
|
// va_block, the va_block_context->mm must be retained and locked for least
|
|
// read.
|
|
bool uvm_hmm_must_use_sysmem(uvm_va_block_t *va_block,
|
|
uvm_va_block_context_t *va_block_context);
|
|
|
|
#else // UVM_IS_CONFIG_HMM()
|
|
|
|
static bool uvm_hmm_is_enabled(uvm_va_space_t *va_space)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
static bool uvm_hmm_is_enabled_system_wide(void)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
static NV_STATUS uvm_hmm_va_space_initialize(uvm_va_space_t *va_space)
|
|
{
|
|
return NV_OK;
|
|
}
|
|
|
|
static NV_STATUS uvm_hmm_va_space_initialize_test(uvm_va_space_t *va_space)
|
|
{
|
|
return NV_WARN_NOTHING_TO_DO;
|
|
}
|
|
|
|
static void uvm_hmm_va_space_destroy(uvm_va_space_t *va_space)
|
|
{
|
|
}
|
|
|
|
static NV_STATUS uvm_hmm_va_block_find(uvm_va_space_t *va_space,
|
|
NvU64 addr,
|
|
uvm_va_block_t **va_block_ptr)
|
|
{
|
|
return NV_ERR_INVALID_ADDRESS;
|
|
}
|
|
|
|
static NV_STATUS uvm_hmm_va_block_find_create(uvm_va_space_t *va_space,
|
|
NvU64 addr,
|
|
uvm_va_block_context_t *va_block_context,
|
|
uvm_va_block_t **va_block_ptr)
|
|
{
|
|
return NV_ERR_INVALID_ADDRESS;
|
|
}
|
|
|
|
static NV_STATUS uvm_hmm_find_vma(uvm_va_block_context_t *va_block_context, NvU64 addr)
|
|
{
|
|
return NV_OK;
|
|
}
|
|
|
|
static bool uvm_hmm_va_block_context_vma_is_valid(uvm_va_block_t *va_block,
|
|
uvm_va_block_context_t *va_block_context,
|
|
uvm_va_block_region_t region)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
static NV_STATUS uvm_hmm_test_va_block_inject_split_error(uvm_va_space_t *va_space, NvU64 addr)
|
|
{
|
|
return NV_ERR_INVALID_ADDRESS;
|
|
}
|
|
|
|
static NV_STATUS uvm_hmm_va_block_reclaim(uvm_va_space_t *va_space,
|
|
struct mm_struct *mm,
|
|
NvU64 start,
|
|
NvU64 end)
|
|
{
|
|
return NV_OK;
|
|
}
|
|
|
|
static void uvm_hmm_va_block_split_tree(uvm_va_block_t *existing_va_block, uvm_va_block_t *new_block)
|
|
{
|
|
}
|
|
|
|
static NV_STATUS uvm_hmm_split_as_needed(uvm_va_space_t *va_space,
|
|
NvU64 addr,
|
|
uvm_va_policy_is_split_needed_t split_needed_cb,
|
|
void *data)
|
|
{
|
|
return NV_OK;
|
|
}
|
|
|
|
static NV_STATUS uvm_hmm_set_preferred_location(uvm_va_space_t *va_space,
|
|
uvm_processor_id_t preferred_location,
|
|
NvU64 base,
|
|
NvU64 last_address)
|
|
{
|
|
return NV_ERR_INVALID_ADDRESS;
|
|
}
|
|
|
|
static NV_STATUS uvm_hmm_set_accessed_by(uvm_va_space_t *va_space,
|
|
uvm_processor_id_t processor_id,
|
|
bool set_bit,
|
|
NvU64 base,
|
|
NvU64 last_address)
|
|
{
|
|
return NV_ERR_INVALID_ADDRESS;
|
|
}
|
|
|
|
static NV_STATUS uvm_hmm_set_read_duplication(uvm_va_space_t *va_space,
|
|
uvm_read_duplication_policy_t new_policy,
|
|
NvU64 base,
|
|
NvU64 last_address)
|
|
{
|
|
return NV_ERR_INVALID_ADDRESS;
|
|
}
|
|
|
|
static void uvm_hmm_find_policy_end(uvm_va_block_t *va_block,
|
|
uvm_va_block_context_t *va_block_context,
|
|
unsigned long addr,
|
|
NvU64 *endp)
|
|
{
|
|
}
|
|
|
|
static NV_STATUS uvm_hmm_find_policy_vma_and_outer(uvm_va_block_t *va_block,
|
|
uvm_va_block_context_t *va_block_context,
|
|
uvm_page_index_t page_index,
|
|
uvm_page_index_t *outerp)
|
|
{
|
|
return NV_OK;
|
|
}
|
|
|
|
static NV_STATUS uvm_hmm_clear_thrashing_policy(uvm_va_space_t *va_space)
|
|
{
|
|
return NV_OK;
|
|
}
|
|
|
|
static uvm_va_block_region_t uvm_hmm_get_prefetch_region(uvm_va_block_t *va_block,
|
|
uvm_va_block_context_t *va_block_context,
|
|
NvU64 address)
|
|
{
|
|
return (uvm_va_block_region_t){};
|
|
}
|
|
|
|
static uvm_prot_t uvm_hmm_compute_logical_prot(uvm_va_block_t *va_block,
|
|
uvm_va_block_context_t *va_block_context,
|
|
NvU64 addr)
|
|
{
|
|
return UVM_PROT_NONE;
|
|
}
|
|
|
|
static NV_STATUS uvm_test_hmm_init(UVM_TEST_HMM_INIT_PARAMS *params, struct file *filp)
|
|
{
|
|
return NV_WARN_NOTHING_TO_DO;
|
|
}
|
|
|
|
static NV_STATUS uvm_test_split_invalidate_delay(UVM_TEST_SPLIT_INVALIDATE_DELAY_PARAMS *params,
|
|
struct file *filp)
|
|
{
|
|
return NV_ERR_INVALID_STATE;
|
|
}
|
|
|
|
static NV_STATUS uvm_hmm_va_range_info(uvm_va_space_t *va_space,
|
|
struct mm_struct *mm,
|
|
UVM_TEST_VA_RANGE_INFO_PARAMS *params)
|
|
{
|
|
return NV_ERR_INVALID_ADDRESS;
|
|
}
|
|
|
|
static bool uvm_hmm_must_use_sysmem(uvm_va_block_t *va_block,
|
|
uvm_va_block_context_t *va_block_context)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
#endif // UVM_IS_CONFIG_HMM()
|
|
|
|
#endif // _UVM_HMM_H_
|