mirror of
https://github.com/NVIDIA/open-gpu-kernel-modules.git
synced 2024-12-12 21:08:56 +01:00
470 lines
16 KiB
C
470 lines
16 KiB
C
/*******************************************************************************
|
|
Copyright (c) 2022 NVIDIA Corporation
|
|
|
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
of this software and associated documentation files (the "Software"), to
|
|
deal in the Software without restriction, including without limitation the
|
|
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
|
sell copies of the Software, and to permit persons to whom the Software is
|
|
furnished to do so, subject to the following conditions:
|
|
|
|
The above copyright notice and this permission notice shall be
|
|
included in all copies or substantial portions of the Software.
|
|
|
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
|
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
DEALINGS IN THE SOFTWARE.
|
|
|
|
*******************************************************************************/
|
|
|
|
#include "uvm_common.h"
|
|
#include "uvm_linux.h"
|
|
#include "uvm_types.h"
|
|
#include "uvm_va_policy.h"
|
|
#include "uvm_va_block.h"
|
|
#include "uvm_va_space.h"
|
|
#include "uvm_va_range.h"
|
|
|
|
uvm_va_policy_t uvm_va_policy_default __read_mostly = {
|
|
.preferred_location = UVM_ID_INVALID,
|
|
.read_duplication = UVM_READ_DUPLICATION_UNSET,
|
|
};
|
|
|
|
bool uvm_va_policy_is_read_duplicate(uvm_va_policy_t *policy, uvm_va_space_t *va_space)
|
|
{
|
|
return policy->read_duplication == UVM_READ_DUPLICATION_ENABLED &&
|
|
uvm_va_space_can_read_duplicate(va_space, NULL);
|
|
}
|
|
|
|
uvm_va_policy_t *uvm_va_policy_get(uvm_va_block_t *va_block, NvU64 addr)
|
|
{
|
|
uvm_assert_mutex_locked(&va_block->lock);
|
|
|
|
if (uvm_va_block_is_hmm(va_block)) {
|
|
uvm_va_policy_node_t *node = uvm_va_policy_node_find(va_block, addr);
|
|
|
|
return node ? &node->policy : &uvm_va_policy_default;
|
|
}
|
|
else
|
|
return uvm_va_range_get_policy(va_block->va_range);
|
|
}
|
|
|
|
#if UVM_IS_CONFIG_HMM()
|
|
|
|
static struct kmem_cache *g_uvm_va_policy_node_cache __read_mostly;
|
|
|
|
static uvm_va_policy_node_t *uvm_va_policy_node_container(uvm_range_tree_node_t *tree_node)
|
|
{
|
|
return container_of(tree_node, uvm_va_policy_node_t, node);
|
|
}
|
|
|
|
static uvm_va_policy_t *uvm_va_policy_container(uvm_range_tree_node_t *tree_node)
|
|
{
|
|
if (!tree_node)
|
|
return NULL;
|
|
|
|
return &uvm_va_policy_node_container(tree_node)->policy;
|
|
}
|
|
|
|
NV_STATUS uvm_va_policy_init(void)
|
|
{
|
|
g_uvm_va_policy_node_cache = NV_KMEM_CACHE_CREATE("uvm_va_policy_node_t", uvm_va_policy_node_t);
|
|
if (!g_uvm_va_policy_node_cache)
|
|
return NV_ERR_NO_MEMORY;
|
|
|
|
return NV_OK;
|
|
}
|
|
|
|
void uvm_va_policy_exit(void)
|
|
{
|
|
kmem_cache_destroy_safe(&g_uvm_va_policy_node_cache);
|
|
}
|
|
|
|
static uvm_va_policy_node_t *uvm_va_policy_node_alloc(NvU64 start, NvU64 end)
|
|
{
|
|
uvm_va_policy_node_t *node;
|
|
|
|
UVM_ASSERT(PAGE_ALIGNED(start));
|
|
UVM_ASSERT(PAGE_ALIGNED(end + 1));
|
|
|
|
node = nv_kmem_cache_zalloc(g_uvm_va_policy_node_cache, NV_UVM_GFP_FLAGS);
|
|
if (!node)
|
|
return NULL;
|
|
|
|
node->node.start = start;
|
|
node->node.end = end;
|
|
|
|
return node;
|
|
}
|
|
|
|
static void uvm_va_policy_node_free(uvm_va_policy_node_t *node)
|
|
{
|
|
kmem_cache_free(g_uvm_va_policy_node_cache, node);
|
|
}
|
|
|
|
static uvm_va_policy_node_t *uvm_va_policy_node_create(uvm_va_block_t *va_block, NvU64 start, NvU64 end)
|
|
{
|
|
uvm_va_policy_node_t *node;
|
|
NV_STATUS status;
|
|
|
|
UVM_ASSERT(uvm_va_block_is_hmm(va_block));
|
|
uvm_assert_mutex_locked(&va_block->lock);
|
|
UVM_ASSERT(start >= va_block->start);
|
|
UVM_ASSERT(end <= va_block->end);
|
|
|
|
node = uvm_va_policy_node_alloc(start, end);
|
|
if (!node)
|
|
return NULL;
|
|
|
|
node->policy = uvm_va_policy_default;
|
|
|
|
status = uvm_range_tree_add(&va_block->hmm.va_policy_tree, &node->node);
|
|
UVM_ASSERT(status == NV_OK);
|
|
|
|
return node;
|
|
}
|
|
|
|
uvm_va_policy_node_t *uvm_va_policy_node_find(uvm_va_block_t *va_block, NvU64 addr)
|
|
{
|
|
uvm_range_tree_node_t *tree_node;
|
|
|
|
UVM_ASSERT(uvm_va_block_is_hmm(va_block));
|
|
uvm_assert_mutex_locked(&va_block->lock);
|
|
|
|
tree_node = uvm_range_tree_find(&va_block->hmm.va_policy_tree, addr);
|
|
if (!tree_node)
|
|
return NULL;
|
|
|
|
return uvm_va_policy_node_container(tree_node);
|
|
}
|
|
|
|
uvm_va_policy_node_t *uvm_va_policy_node_iter_first(uvm_va_block_t *va_block, NvU64 start, NvU64 end)
|
|
{
|
|
uvm_range_tree_node_t *tree_node;
|
|
|
|
UVM_ASSERT(uvm_va_block_is_hmm(va_block));
|
|
uvm_assert_mutex_locked(&va_block->lock);
|
|
|
|
tree_node = uvm_range_tree_iter_first(&va_block->hmm.va_policy_tree, start, end);
|
|
if (!tree_node)
|
|
return NULL;
|
|
|
|
return uvm_va_policy_node_container(tree_node);
|
|
}
|
|
|
|
uvm_va_policy_node_t *uvm_va_policy_node_iter_next(uvm_va_block_t *va_block,
|
|
uvm_va_policy_node_t *node,
|
|
NvU64 end)
|
|
{
|
|
uvm_range_tree_node_t *tree_node;
|
|
|
|
if (!node)
|
|
return NULL;
|
|
|
|
tree_node = uvm_range_tree_iter_next(&va_block->hmm.va_policy_tree, &node->node, end);
|
|
if (!tree_node)
|
|
return NULL;
|
|
|
|
return uvm_va_policy_node_container(tree_node);
|
|
}
|
|
|
|
NV_STATUS uvm_va_policy_node_split(uvm_va_block_t *va_block,
|
|
uvm_va_policy_node_t *old,
|
|
NvU64 new_end,
|
|
uvm_va_policy_node_t **new_ptr)
|
|
{
|
|
uvm_va_policy_node_t *new;
|
|
|
|
UVM_ASSERT(uvm_va_block_is_hmm(va_block));
|
|
uvm_assert_mutex_locked(&va_block->lock);
|
|
|
|
UVM_ASSERT(new_end > old->node.start);
|
|
UVM_ASSERT(new_end < old->node.end);
|
|
|
|
new = uvm_va_policy_node_alloc(new_end + 1, old->node.end);
|
|
if (!new)
|
|
return NV_ERR_NO_MEMORY;
|
|
|
|
new->policy = old->policy;
|
|
|
|
uvm_range_tree_split(&va_block->hmm.va_policy_tree, &old->node, &new->node);
|
|
|
|
if (new_ptr)
|
|
*new_ptr = new;
|
|
|
|
return NV_OK;
|
|
}
|
|
|
|
void uvm_va_policy_node_split_move(uvm_va_block_t *old_va_block,
|
|
uvm_va_block_t *new_va_block)
|
|
{
|
|
uvm_va_policy_node_t *node, *next;
|
|
NV_STATUS status;
|
|
|
|
UVM_ASSERT(uvm_va_block_is_hmm(old_va_block));
|
|
UVM_ASSERT(uvm_va_block_is_hmm(new_va_block));
|
|
uvm_assert_mutex_locked(&old_va_block->lock);
|
|
|
|
UVM_ASSERT(old_va_block->end + 1 == new_va_block->start);
|
|
|
|
uvm_for_each_va_policy_node_in_safe(node, next, old_va_block, new_va_block->start, new_va_block->end) {
|
|
uvm_range_tree_remove(&old_va_block->hmm.va_policy_tree, &node->node);
|
|
UVM_ASSERT(node->node.start >= new_va_block->start);
|
|
UVM_ASSERT(node->node.end <= new_va_block->end);
|
|
status = uvm_range_tree_add(&new_va_block->hmm.va_policy_tree, &node->node);
|
|
UVM_ASSERT(status == NV_OK);
|
|
}
|
|
}
|
|
|
|
void uvm_va_policy_clear(uvm_va_block_t *va_block, NvU64 start, NvU64 end)
|
|
{
|
|
uvm_va_policy_node_t *node, *new;
|
|
uvm_range_tree_node_t *tree_node;
|
|
NV_STATUS status;
|
|
|
|
UVM_ASSERT(uvm_va_block_is_hmm(va_block));
|
|
uvm_assert_mutex_locked(&va_block->lock);
|
|
|
|
tree_node = uvm_range_tree_iter_first(&va_block->hmm.va_policy_tree, start, end);
|
|
while (tree_node) {
|
|
node = uvm_va_policy_node_container(tree_node);
|
|
tree_node = uvm_range_tree_iter_next(&va_block->hmm.va_policy_tree, &node->node, end);
|
|
|
|
if (node->node.start < start) {
|
|
if (node->node.end <= end) {
|
|
uvm_range_tree_shrink_node(&va_block->hmm.va_policy_tree, &node->node, node->node.start, start - 1);
|
|
continue;
|
|
}
|
|
|
|
status = uvm_va_policy_node_split(va_block, node, start - 1, &new);
|
|
// If we can't split, save the policy before the part being cleared
|
|
// but forget the policy after the range to be cleared.
|
|
// Since policy isn't a guarantee, at least this is safe.
|
|
if (status != NV_OK) {
|
|
uvm_range_tree_shrink_node(&va_block->hmm.va_policy_tree, &node->node, node->node.start, start - 1);
|
|
continue;
|
|
}
|
|
|
|
node = new;
|
|
}
|
|
|
|
if (node->node.end > end) {
|
|
uvm_range_tree_shrink_node(&va_block->hmm.va_policy_tree, &node->node, end + 1, node->node.end);
|
|
continue;
|
|
}
|
|
|
|
uvm_range_tree_remove(&va_block->hmm.va_policy_tree, &node->node);
|
|
uvm_va_policy_node_free(node);
|
|
}
|
|
}
|
|
|
|
static void uvm_va_policy_node_set(uvm_va_policy_node_t *node,
|
|
uvm_va_policy_type_t which,
|
|
uvm_processor_id_t processor_id,
|
|
uvm_read_duplication_policy_t new_policy)
|
|
{
|
|
switch (which) {
|
|
case UVM_VA_POLICY_PREFERRED_LOCATION:
|
|
UVM_ASSERT(!UVM_ID_IS_INVALID(processor_id));
|
|
node->policy.preferred_location = processor_id;
|
|
break;
|
|
|
|
case UVM_VA_POLICY_ACCESSED_BY:
|
|
UVM_ASSERT(!UVM_ID_IS_INVALID(processor_id));
|
|
uvm_processor_mask_set(&node->policy.accessed_by, processor_id);
|
|
break;
|
|
|
|
case UVM_VA_POLICY_READ_DUPLICATION:
|
|
UVM_ASSERT(new_policy == UVM_READ_DUPLICATION_ENABLED ||
|
|
new_policy == UVM_READ_DUPLICATION_DISABLED);
|
|
node->policy.read_duplication = new_policy;
|
|
break;
|
|
|
|
default:
|
|
UVM_ASSERT_MSG(0, "Unknown policy type %u\n", which);
|
|
break;
|
|
}
|
|
}
|
|
|
|
static void uvm_va_policy_node_clear(uvm_va_block_t *va_block,
|
|
uvm_va_policy_node_t *node,
|
|
uvm_va_policy_type_t which,
|
|
uvm_processor_id_t processor_id,
|
|
uvm_read_duplication_policy_t new_policy)
|
|
{
|
|
switch (which) {
|
|
case UVM_VA_POLICY_PREFERRED_LOCATION:
|
|
UVM_ASSERT(UVM_ID_IS_INVALID(processor_id));
|
|
node->policy.preferred_location = processor_id;
|
|
break;
|
|
|
|
case UVM_VA_POLICY_ACCESSED_BY:
|
|
UVM_ASSERT(!UVM_ID_IS_INVALID(processor_id));
|
|
uvm_processor_mask_clear(&node->policy.accessed_by, processor_id);
|
|
break;
|
|
|
|
case UVM_VA_POLICY_READ_DUPLICATION:
|
|
default:
|
|
// Read duplication is never set back to UVM_READ_DUPLICATION_UNSET.
|
|
UVM_ASSERT(0);
|
|
break;
|
|
}
|
|
|
|
// Check to see if the node is now the default and can be removed.
|
|
if (UVM_ID_IS_INVALID(node->policy.preferred_location) &&
|
|
uvm_processor_mask_empty(&node->policy.accessed_by) &&
|
|
node->policy.read_duplication == UVM_READ_DUPLICATION_UNSET) {
|
|
uvm_range_tree_remove(&va_block->hmm.va_policy_tree, &node->node);
|
|
uvm_va_policy_node_free(node);
|
|
}
|
|
}
|
|
|
|
static uvm_va_policy_node_t *create_node_and_set(uvm_va_block_t *va_block,
|
|
NvU64 start,
|
|
NvU64 end,
|
|
uvm_va_policy_type_t which,
|
|
uvm_processor_id_t processor_id,
|
|
uvm_read_duplication_policy_t new_policy)
|
|
{
|
|
uvm_va_policy_node_t *node;
|
|
|
|
// Create a new node for the missing range.
|
|
node = uvm_va_policy_node_create(va_block, start, end);
|
|
if (!node)
|
|
return node;
|
|
|
|
uvm_va_policy_node_set(node, which, processor_id, new_policy);
|
|
|
|
return node;
|
|
}
|
|
|
|
static bool va_policy_node_split_needed(uvm_va_policy_node_t *node,
|
|
NvU64 start,
|
|
NvU64 end,
|
|
uvm_va_policy_type_t which,
|
|
bool is_default,
|
|
uvm_processor_id_t processor_id,
|
|
uvm_read_duplication_policy_t new_policy)
|
|
{
|
|
// If the node doesn't extend beyond the range being set, it doesn't need
|
|
// to be split.
|
|
if (node->node.start >= start && node->node.end <= end)
|
|
return false;
|
|
|
|
// If the new policy value doesn't match the old value, a split is needed.
|
|
switch (which) {
|
|
case UVM_VA_POLICY_PREFERRED_LOCATION:
|
|
return !uvm_id_equal(node->policy.preferred_location, processor_id);
|
|
|
|
case UVM_VA_POLICY_ACCESSED_BY:
|
|
if (is_default)
|
|
return uvm_processor_mask_test(&node->policy.accessed_by, processor_id);
|
|
else
|
|
return !uvm_processor_mask_test(&node->policy.accessed_by, processor_id);
|
|
|
|
case UVM_VA_POLICY_READ_DUPLICATION:
|
|
return node->policy.read_duplication != new_policy;
|
|
|
|
default:
|
|
UVM_ASSERT(0);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
NV_STATUS uvm_va_policy_set_range(uvm_va_block_t *va_block,
|
|
NvU64 start,
|
|
NvU64 end,
|
|
uvm_va_policy_type_t which,
|
|
bool is_default,
|
|
uvm_processor_id_t processor_id,
|
|
uvm_read_duplication_policy_t new_policy)
|
|
{
|
|
uvm_va_policy_node_t *node, *next, *new;
|
|
NvU64 addr;
|
|
NvU64 node_start;
|
|
NvU64 node_end;
|
|
|
|
UVM_ASSERT(uvm_va_block_is_hmm(va_block));
|
|
uvm_assert_mutex_locked(&va_block->lock);
|
|
UVM_ASSERT(PAGE_ALIGNED(start));
|
|
UVM_ASSERT(PAGE_ALIGNED(end + 1));
|
|
UVM_ASSERT(start < end);
|
|
UVM_ASSERT(start >= va_block->start);
|
|
UVM_ASSERT(start < va_block->end);
|
|
UVM_ASSERT(end <= va_block->end);
|
|
|
|
// Note that the policy range ends have already been split so we only need
|
|
// to fill in the middle or remove nodes.
|
|
node = uvm_va_policy_node_iter_first(va_block, start, end);
|
|
|
|
if (!node) {
|
|
// There is no policy in the given range so it is already the default.
|
|
if (is_default)
|
|
return NV_OK;
|
|
|
|
// Create a new node for the missing range.
|
|
node = create_node_and_set(va_block,
|
|
start,
|
|
end,
|
|
which,
|
|
processor_id,
|
|
new_policy);
|
|
if (!node)
|
|
return NV_ERR_NO_MEMORY;
|
|
|
|
return NV_OK;
|
|
}
|
|
|
|
for (addr = start; node; addr = node_end + 1, node = next) {
|
|
node_start = node->node.start;
|
|
node_end = node->node.end;
|
|
|
|
// Nodes should have been split before setting policy so verify that.
|
|
UVM_ASSERT(!va_policy_node_split_needed(node, start, end, which, is_default, processor_id, new_policy));
|
|
|
|
next = uvm_va_policy_node_iter_next(va_block, node, end);
|
|
|
|
if (is_default) {
|
|
uvm_va_policy_node_clear(va_block, node, which, processor_id, new_policy);
|
|
// Note that node may have been deleted.
|
|
}
|
|
else {
|
|
uvm_va_policy_node_set(node, which, processor_id, new_policy);
|
|
|
|
// TODO: Bug 1707562: Add support for merging policy ranges.
|
|
}
|
|
|
|
if (!is_default && addr < node_start) {
|
|
// Create a new node for the missing range on the left.
|
|
new = create_node_and_set(va_block,
|
|
addr,
|
|
node_start - 1,
|
|
which,
|
|
processor_id,
|
|
new_policy);
|
|
if (!new)
|
|
return NV_ERR_NO_MEMORY;
|
|
}
|
|
else if (!is_default && !next && node_end < end) {
|
|
// Create a new node for the missing range on the right.
|
|
new = create_node_and_set(va_block,
|
|
node_end + 1,
|
|
end,
|
|
which,
|
|
processor_id,
|
|
new_policy);
|
|
if (!new)
|
|
return NV_ERR_NO_MEMORY;
|
|
break;
|
|
}
|
|
}
|
|
|
|
return NV_OK;
|
|
}
|
|
|
|
#endif // UVM_IS_CONFIG_HMM()
|