mirror of
https://github.com/NVIDIA/open-gpu-kernel-modules.git
synced 2024-12-13 15:08:59 +01:00
5674 lines
137 KiB
C
5674 lines
137 KiB
C
/*
|
|
* SPDX-FileCopyrightText: Copyright (c) 1999-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
* SPDX-License-Identifier: MIT
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
|
* copy of this software and associated documentation files (the "Software"),
|
|
* to deal in the Software without restriction, including without limitation
|
|
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
* and/or sell copies of the Software, and to permit persons to whom the
|
|
* Software is furnished to do so, subject to the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice shall be included in
|
|
* all copies or substantial portions of the Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
|
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
* DEALINGS IN THE SOFTWARE.
|
|
*/
|
|
|
|
#include "nvmisc.h"
|
|
#include "os-interface.h"
|
|
#include "nv-linux.h"
|
|
#include "nv-p2p.h"
|
|
#include "nv-reg.h"
|
|
#include "nv-msi.h"
|
|
#include "nv-pci-table.h"
|
|
|
|
#if defined(NV_UVM_ENABLE)
|
|
#include "nv_uvm_interface.h"
|
|
#endif
|
|
|
|
#if defined(NV_VGPU_KVM_BUILD)
|
|
#include "nv-vgpu-vfio-interface.h"
|
|
#endif
|
|
|
|
|
|
#include "nvlink_proto.h"
|
|
#include "nvlink_caps.h"
|
|
|
|
|
|
#include "nv-frontend.h"
|
|
#include "nv-hypervisor.h"
|
|
#include "nv-ibmnpu.h"
|
|
#include "nv-rsync.h"
|
|
#include "nv-kthread-q.h"
|
|
#include "nv-pat.h"
|
|
#include "nv-dmabuf.h"
|
|
|
|
#if !defined(CONFIG_RETPOLINE)
|
|
#include "nv-retpoline.h"
|
|
#endif
|
|
|
|
#include <linux/firmware.h>
|
|
|
|
#include <sound/core.h> /* HDA struct snd_card */
|
|
|
|
#include <asm/cache.h>
|
|
|
|
#if defined(NV_SOUND_HDAUDIO_H_PRESENT)
|
|
#include "sound/hdaudio.h"
|
|
#endif
|
|
|
|
#if defined(NV_SOUND_HDA_CODEC_H_PRESENT)
|
|
#include <sound/core.h>
|
|
#include <sound/hda_codec.h>
|
|
#include <sound/hda_verbs.h>
|
|
#endif
|
|
|
|
#if defined(NV_SEQ_READ_ITER_PRESENT)
|
|
#include <linux/uio.h>
|
|
#include <linux/seq_file.h>
|
|
#include <linux/kernfs.h>
|
|
#endif
|
|
|
|
#include <linux/dmi.h> /* System DMI info */
|
|
|
|
#include <linux/ioport.h>
|
|
|
|
#include "conftest/patches.h"
|
|
|
|
#define RM_THRESHOLD_TOTAL_IRQ_COUNT 100000
|
|
#define RM_THRESHOLD_UNAHNDLED_IRQ_COUNT 99900
|
|
#define RM_UNHANDLED_TIMEOUT_US 100000
|
|
|
|
const NvBool nv_is_rm_firmware_supported_os = NV_TRUE;
|
|
|
|
// Deprecated, use NV_REG_ENABLE_GPU_FIRMWARE instead
|
|
char *rm_firmware_active = NULL;
|
|
NV_MODULE_STRING_PARAMETER(rm_firmware_active);
|
|
|
|
#define NV_FIRMWARE_GSP_FILENAME "nvidia/" NV_VERSION_STRING "/gsp.bin"
|
|
#define NV_FIRMWARE_GSP_LOG_FILENAME "nvidia/" NV_VERSION_STRING "/gsp_log.bin"
|
|
|
|
MODULE_FIRMWARE(NV_FIRMWARE_GSP_FILENAME);
|
|
|
|
/*
|
|
* Global NVIDIA capability state, for GPU driver
|
|
*/
|
|
nv_cap_t *nvidia_caps_root = NULL;
|
|
|
|
/*
|
|
* our global state; one per device
|
|
*/
|
|
NvU32 num_nv_devices = 0;
|
|
NvU32 num_probed_nv_devices = 0;
|
|
|
|
nv_linux_state_t *nv_linux_devices;
|
|
|
|
/*
|
|
* And one for the control device
|
|
*/
|
|
nv_linux_state_t nv_ctl_device = { { 0 } };
|
|
extern NvU32 nv_dma_remap_peer_mmio;
|
|
|
|
nv_kthread_q_t nv_kthread_q;
|
|
nv_kthread_q_t nv_deferred_close_kthread_q;
|
|
|
|
struct rw_semaphore nv_system_pm_lock;
|
|
|
|
#if defined(CONFIG_PM)
|
|
static nv_power_state_t nv_system_power_state;
|
|
static nv_pm_action_depth_t nv_system_pm_action_depth;
|
|
struct semaphore nv_system_power_state_lock;
|
|
#endif
|
|
|
|
void *nvidia_p2p_page_t_cache;
|
|
static void *nvidia_pte_t_cache;
|
|
void *nvidia_stack_t_cache;
|
|
static nvidia_stack_t *__nv_init_sp;
|
|
|
|
static int nv_tce_bypass_mode = NV_TCE_BYPASS_MODE_DEFAULT;
|
|
|
|
struct semaphore nv_linux_devices_lock;
|
|
|
|
static NvTristate nv_chipset_is_io_coherent = NV_TRISTATE_INDETERMINATE;
|
|
|
|
// True if all the successfully probed devices support ATS
|
|
// Assigned at device probe (module init) time
|
|
NvBool nv_ats_supported = NVCPU_IS_PPC64LE
|
|
|
|
|
|
|
|
|
|
|
|
;
|
|
|
|
// allow an easy way to convert all debug printfs related to events
|
|
// back and forth between 'info' and 'errors'
|
|
#if defined(NV_DBG_EVENTS)
|
|
#define NV_DBG_EVENTINFO NV_DBG_ERRORS
|
|
#else
|
|
#define NV_DBG_EVENTINFO NV_DBG_INFO
|
|
#endif
|
|
|
|
#if defined(HDA_MAX_CODECS)
|
|
#define NV_HDA_MAX_CODECS HDA_MAX_CODECS
|
|
#else
|
|
#define NV_HDA_MAX_CODECS 8
|
|
#endif
|
|
|
|
/***
|
|
*** STATIC functions, only in this file
|
|
***/
|
|
|
|
/* nvos_ functions.. do not take a state device parameter */
|
|
static int nvos_count_devices(void);
|
|
|
|
|
|
static nv_alloc_t *nvos_create_alloc(struct device *, NvU64);
|
|
|
|
|
|
|
|
static int nvos_free_alloc(nv_alloc_t *);
|
|
|
|
/***
|
|
*** EXPORTS to Linux Kernel
|
|
***/
|
|
|
|
static irqreturn_t nvidia_isr_common_bh (void *);
|
|
static void nvidia_isr_bh_unlocked (void *);
|
|
static int nvidia_ctl_open (struct inode *, struct file *);
|
|
static int nvidia_ctl_close (struct inode *, struct file *);
|
|
|
|
const char *nv_device_name = MODULE_NAME;
|
|
static const char *nvidia_stack_cache_name = MODULE_NAME "_stack_cache";
|
|
static const char *nvidia_pte_cache_name = MODULE_NAME "_pte_cache";
|
|
static const char *nvidia_p2p_page_cache_name = MODULE_NAME "_p2p_page_cache";
|
|
|
|
static int nvidia_open (struct inode *, struct file *);
|
|
static int nvidia_close (struct inode *, struct file *);
|
|
static unsigned int nvidia_poll (struct file *, poll_table *);
|
|
static int nvidia_ioctl (struct inode *, struct file *, unsigned int, unsigned long);
|
|
|
|
/* character device entry points*/
|
|
nvidia_module_t nv_fops = {
|
|
.owner = THIS_MODULE,
|
|
.module_name = MODULE_NAME,
|
|
.instance = MODULE_INSTANCE_NUMBER,
|
|
.open = nvidia_open,
|
|
.close = nvidia_close,
|
|
.ioctl = nvidia_ioctl,
|
|
.mmap = nvidia_mmap,
|
|
.poll = nvidia_poll,
|
|
};
|
|
|
|
#if defined(CONFIG_PM)
|
|
static int nv_pmops_suspend (struct device *dev);
|
|
static int nv_pmops_resume (struct device *dev);
|
|
static int nv_pmops_freeze (struct device *dev);
|
|
static int nv_pmops_thaw (struct device *dev);
|
|
static int nv_pmops_restore (struct device *dev);
|
|
static int nv_pmops_poweroff (struct device *dev);
|
|
static int nv_pmops_runtime_suspend (struct device *dev);
|
|
static int nv_pmops_runtime_resume (struct device *dev);
|
|
|
|
struct dev_pm_ops nv_pm_ops = {
|
|
.suspend = nv_pmops_suspend,
|
|
.resume = nv_pmops_resume,
|
|
.freeze = nv_pmops_freeze,
|
|
.thaw = nv_pmops_thaw,
|
|
.poweroff = nv_pmops_poweroff,
|
|
.restore = nv_pmops_restore,
|
|
.runtime_suspend = nv_pmops_runtime_suspend,
|
|
.runtime_resume = nv_pmops_runtime_resume,
|
|
};
|
|
#endif
|
|
|
|
/***
|
|
*** see nv.h for functions exported to other parts of resman
|
|
***/
|
|
|
|
/***
|
|
*** STATIC functions
|
|
***/
|
|
|
|
#if defined(NVCPU_X86_64)
|
|
#define NV_AMD_SEV_BIT BIT(1)
|
|
|
|
static
|
|
NvBool nv_is_sev_supported(
|
|
void
|
|
)
|
|
{
|
|
unsigned int eax, ebx, ecx, edx;
|
|
|
|
/* Check for the SME/SEV support leaf */
|
|
eax = 0x80000000;
|
|
ecx = 0;
|
|
native_cpuid(&eax, &ebx, &ecx, &edx);
|
|
if (eax < 0x8000001f)
|
|
return NV_FALSE;
|
|
|
|
eax = 0x8000001f;
|
|
ecx = 0;
|
|
native_cpuid(&eax, &ebx, &ecx, &edx);
|
|
/* Check whether SEV is supported */
|
|
if (!(eax & NV_AMD_SEV_BIT))
|
|
return NV_FALSE;
|
|
|
|
return NV_TRUE;
|
|
}
|
|
#endif
|
|
|
|
static
|
|
void nv_sev_init(
|
|
void
|
|
)
|
|
{
|
|
#if defined(MSR_AMD64_SEV) && defined(NVCPU_X86_64)
|
|
NvU32 lo_val, hi_val;
|
|
|
|
if (!nv_is_sev_supported())
|
|
return;
|
|
|
|
rdmsr(MSR_AMD64_SEV, lo_val, hi_val);
|
|
|
|
os_sev_status = lo_val;
|
|
#if defined(MSR_AMD64_SEV_ENABLED)
|
|
os_sev_enabled = (os_sev_status & MSR_AMD64_SEV_ENABLED);
|
|
#endif
|
|
#endif
|
|
}
|
|
|
|
static
|
|
nv_alloc_t *nvos_create_alloc(
|
|
struct device *dev,
|
|
|
|
NvU64 num_pages
|
|
|
|
|
|
|
|
)
|
|
{
|
|
nv_alloc_t *at;
|
|
|
|
NvU64 pt_size;
|
|
|
|
|
|
|
|
unsigned int i;
|
|
|
|
NV_KMALLOC(at, sizeof(nv_alloc_t));
|
|
if (at == NULL)
|
|
{
|
|
nv_printf(NV_DBG_ERRORS, "NVRM: failed to allocate alloc info\n");
|
|
return NULL;
|
|
}
|
|
|
|
memset(at, 0, sizeof(nv_alloc_t));
|
|
|
|
at->dev = dev;
|
|
|
|
pt_size = num_pages * sizeof(nvidia_pte_t *);
|
|
//
|
|
// Check for multiplication overflow and check whether num_pages value can fit in at->num_pages.
|
|
//
|
|
if ((num_pages != 0) && ((pt_size / num_pages) != sizeof(nvidia_pte_t*)))
|
|
{
|
|
nv_printf(NV_DBG_ERRORS, "NVRM: Invalid page table allocation - Number of pages exceeds max value.\n");
|
|
NV_KFREE(at, sizeof(nv_alloc_t));
|
|
return NULL;
|
|
}
|
|
|
|
at->num_pages = num_pages;
|
|
if (at->num_pages != num_pages)
|
|
{
|
|
nv_printf(NV_DBG_ERRORS, "NVRM: Invalid page table allocation - requested size overflows.\n");
|
|
NV_KFREE(at, sizeof(nv_alloc_t));
|
|
return NULL;
|
|
}
|
|
|
|
|
|
|
|
|
|
if (os_alloc_mem((void **)&at->page_table, pt_size) != NV_OK)
|
|
{
|
|
nv_printf(NV_DBG_ERRORS, "NVRM: failed to allocate page table\n");
|
|
NV_KFREE(at, sizeof(nv_alloc_t));
|
|
return NULL;
|
|
}
|
|
|
|
memset(at->page_table, 0, pt_size);
|
|
|
|
|
|
|
|
NV_ATOMIC_SET(at->usage_count, 0);
|
|
|
|
for (i = 0; i < at->num_pages; i++)
|
|
{
|
|
at->page_table[i] = NV_KMEM_CACHE_ALLOC(nvidia_pte_t_cache);
|
|
if (at->page_table[i] == NULL)
|
|
{
|
|
nv_printf(NV_DBG_ERRORS,
|
|
"NVRM: failed to allocate page table entry\n");
|
|
nvos_free_alloc(at);
|
|
return NULL;
|
|
}
|
|
memset(at->page_table[i], 0, sizeof(nvidia_pte_t));
|
|
}
|
|
|
|
at->pid = os_get_current_process();
|
|
|
|
return at;
|
|
}
|
|
|
|
static
|
|
int nvos_free_alloc(
|
|
nv_alloc_t *at
|
|
)
|
|
{
|
|
unsigned int i;
|
|
|
|
if (at == NULL)
|
|
return -1;
|
|
|
|
if (NV_ATOMIC_READ(at->usage_count))
|
|
return 1;
|
|
|
|
for (i = 0; i < at->num_pages; i++)
|
|
{
|
|
if (at->page_table[i] != NULL)
|
|
NV_KMEM_CACHE_FREE(at->page_table[i], nvidia_pte_t_cache);
|
|
}
|
|
os_free_mem(at->page_table);
|
|
|
|
NV_KFREE(at, sizeof(nv_alloc_t));
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void
|
|
nv_module_resources_exit(nv_stack_t *sp)
|
|
{
|
|
nv_kmem_cache_free_stack(sp);
|
|
|
|
NV_KMEM_CACHE_DESTROY(nvidia_p2p_page_t_cache);
|
|
NV_KMEM_CACHE_DESTROY(nvidia_pte_t_cache);
|
|
NV_KMEM_CACHE_DESTROY(nvidia_stack_t_cache);
|
|
}
|
|
|
|
static int __init
|
|
nv_module_resources_init(nv_stack_t **sp)
|
|
{
|
|
int rc = -ENOMEM;
|
|
|
|
nvidia_stack_t_cache = NV_KMEM_CACHE_CREATE(nvidia_stack_cache_name,
|
|
nvidia_stack_t);
|
|
if (nvidia_stack_t_cache == NULL)
|
|
{
|
|
nv_printf(NV_DBG_ERRORS,
|
|
"NVRM: nvidia_stack_t cache allocation failed.\n");
|
|
goto exit;
|
|
}
|
|
|
|
nvidia_pte_t_cache = NV_KMEM_CACHE_CREATE(nvidia_pte_cache_name,
|
|
nvidia_pte_t);
|
|
if (nvidia_pte_t_cache == NULL)
|
|
{
|
|
nv_printf(NV_DBG_ERRORS,
|
|
"NVRM: nvidia_pte_t cache allocation failed.\n");
|
|
goto exit;
|
|
}
|
|
|
|
nvidia_p2p_page_t_cache = NV_KMEM_CACHE_CREATE(nvidia_p2p_page_cache_name,
|
|
nvidia_p2p_page_t);
|
|
if (nvidia_p2p_page_t_cache == NULL)
|
|
{
|
|
nv_printf(NV_DBG_ERRORS,
|
|
"NVRM: nvidia_p2p_page_t cache allocation failed.\n");
|
|
goto exit;
|
|
}
|
|
|
|
rc = nv_kmem_cache_alloc_stack(sp);
|
|
if (rc < 0)
|
|
{
|
|
goto exit;
|
|
}
|
|
|
|
exit:
|
|
if (rc < 0)
|
|
{
|
|
nv_kmem_cache_free_stack(*sp);
|
|
|
|
NV_KMEM_CACHE_DESTROY(nvidia_p2p_page_t_cache);
|
|
NV_KMEM_CACHE_DESTROY(nvidia_pte_t_cache);
|
|
NV_KMEM_CACHE_DESTROY(nvidia_stack_t_cache);
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
|
|
static void
|
|
nvlink_drivers_exit(void)
|
|
{
|
|
|
|
#if NVCPU_IS_64_BITS
|
|
nvswitch_exit();
|
|
#endif
|
|
|
|
|
|
#if defined(NVCPU_PPC64LE)
|
|
ibmnpu_exit();
|
|
#endif
|
|
|
|
nvlink_core_exit();
|
|
}
|
|
|
|
|
|
|
|
static int __init
|
|
nvlink_drivers_init(void)
|
|
{
|
|
int rc = 0;
|
|
|
|
rc = nvlink_core_init();
|
|
if (rc < 0)
|
|
{
|
|
nv_printf(NV_DBG_INFO, "NVRM: NVLink core init failed.\n");
|
|
return rc;
|
|
}
|
|
|
|
#if defined(NVCPU_PPC64LE)
|
|
rc = ibmnpu_init();
|
|
if (rc < 0)
|
|
{
|
|
nv_printf(NV_DBG_INFO, "NVRM: IBM NPU init failed.\n");
|
|
nvlink_core_exit();
|
|
return rc;
|
|
}
|
|
#endif
|
|
|
|
|
|
#if NVCPU_IS_64_BITS
|
|
rc = nvswitch_init();
|
|
if (rc < 0)
|
|
{
|
|
nv_printf(NV_DBG_INFO, "NVRM: NVSwitch init failed.\n");
|
|
#if defined(NVCPU_PPC64LE)
|
|
ibmnpu_exit();
|
|
#endif
|
|
nvlink_core_exit();
|
|
}
|
|
#endif
|
|
|
|
|
|
return rc;
|
|
}
|
|
|
|
|
|
static void
|
|
nv_module_state_exit(nv_stack_t *sp)
|
|
{
|
|
nv_state_t *nv = NV_STATE_PTR(&nv_ctl_device);
|
|
|
|
nv_teardown_pat_support();
|
|
|
|
nv_kthread_q_stop(&nv_deferred_close_kthread_q);
|
|
nv_kthread_q_stop(&nv_kthread_q);
|
|
|
|
nv_lock_destroy_locks(sp, nv);
|
|
}
|
|
|
|
static int
|
|
nv_module_state_init(nv_stack_t *sp)
|
|
{
|
|
int rc;
|
|
nv_state_t *nv = NV_STATE_PTR(&nv_ctl_device);
|
|
|
|
nv->os_state = (void *)&nv_ctl_device;
|
|
|
|
if (!nv_lock_init_locks(sp, nv))
|
|
{
|
|
return -ENOMEM;
|
|
}
|
|
|
|
rc = nv_kthread_q_init(&nv_kthread_q, "nv_queue");
|
|
if (rc != 0)
|
|
{
|
|
goto exit;
|
|
}
|
|
|
|
rc = nv_kthread_q_init(&nv_deferred_close_kthread_q, "nv_queue");
|
|
if (rc != 0)
|
|
{
|
|
nv_kthread_q_stop(&nv_kthread_q);
|
|
goto exit;
|
|
}
|
|
|
|
rc = nv_init_pat_support(sp);
|
|
if (rc < 0)
|
|
{
|
|
nv_kthread_q_stop(&nv_deferred_close_kthread_q);
|
|
nv_kthread_q_stop(&nv_kthread_q);
|
|
goto exit;
|
|
}
|
|
|
|
nv_linux_devices = NULL;
|
|
NV_INIT_MUTEX(&nv_linux_devices_lock);
|
|
init_rwsem(&nv_system_pm_lock);
|
|
|
|
#if defined(CONFIG_PM)
|
|
NV_INIT_MUTEX(&nv_system_power_state_lock);
|
|
nv_system_power_state = NV_POWER_STATE_RUNNING;
|
|
nv_system_pm_action_depth = NV_PM_ACTION_DEPTH_DEFAULT;
|
|
#endif
|
|
|
|
NV_SPIN_LOCK_INIT(&nv_ctl_device.snapshot_timer_lock);
|
|
|
|
exit:
|
|
if (rc < 0)
|
|
{
|
|
nv_lock_destroy_locks(sp, nv);
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
static void __init
|
|
nv_registry_keys_init(nv_stack_t *sp)
|
|
{
|
|
NV_STATUS status;
|
|
nv_state_t *nv = NV_STATE_PTR(&nv_ctl_device);
|
|
NvU32 data;
|
|
|
|
/*
|
|
* Determine the TCE bypass mode here so it can be used during
|
|
* device probe. Also determine whether we should allow
|
|
* user-mode NUMA onlining of device memory.
|
|
*/
|
|
if (NVCPU_IS_PPC64LE)
|
|
{
|
|
status = rm_read_registry_dword(sp, nv,
|
|
NV_REG_TCE_BYPASS_MODE,
|
|
&data);
|
|
if ((status == NV_OK) && ((int)data != NV_TCE_BYPASS_MODE_DEFAULT))
|
|
{
|
|
nv_tce_bypass_mode = data;
|
|
}
|
|
|
|
if (NVreg_EnableUserNUMAManagement)
|
|
{
|
|
/* Force on the core RM registry key to match. */
|
|
status = rm_write_registry_dword(sp, nv, "RMNumaOnlining", 1);
|
|
WARN_ON(status != NV_OK);
|
|
}
|
|
}
|
|
|
|
status = rm_read_registry_dword(sp, nv, NV_DMA_REMAP_PEER_MMIO, &data);
|
|
if (status == NV_OK)
|
|
{
|
|
nv_dma_remap_peer_mmio = data;
|
|
}
|
|
}
|
|
|
|
static void __init
|
|
nv_report_applied_patches(void)
|
|
{
|
|
unsigned i;
|
|
|
|
for (i = 0; __nv_patches[i].short_description; i++)
|
|
{
|
|
if (i == 0)
|
|
{
|
|
nv_printf(NV_DBG_ERRORS, "NVRM: Applied patches:\n");
|
|
}
|
|
|
|
nv_printf(NV_DBG_ERRORS,
|
|
"NVRM: Patch #%d: %s\n", i + 1, __nv_patches[i].short_description);
|
|
}
|
|
}
|
|
|
|
static void
|
|
nv_drivers_exit(void)
|
|
{
|
|
|
|
|
|
|
|
nv_pci_unregister_driver();
|
|
|
|
nvidia_unregister_module(&nv_fops);
|
|
}
|
|
|
|
static int __init
|
|
nv_drivers_init(void)
|
|
{
|
|
int rc;
|
|
|
|
rc = nvidia_register_module(&nv_fops);
|
|
if (rc < 0)
|
|
{
|
|
nv_printf(NV_DBG_ERRORS,
|
|
"NVRM: failed to register character device.\n");
|
|
return rc;
|
|
}
|
|
|
|
rc = nv_pci_register_driver();
|
|
if (rc < 0)
|
|
{
|
|
nv_printf(NV_DBG_ERRORS, "NVRM: No NVIDIA PCI devices found.\n");
|
|
rc = -ENODEV;
|
|
goto exit;
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
exit:
|
|
if (rc < 0)
|
|
{
|
|
nvidia_unregister_module(&nv_fops);
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
static void
|
|
nv_module_exit(nv_stack_t *sp)
|
|
{
|
|
nv_module_state_exit(sp);
|
|
|
|
rm_shutdown_rm(sp);
|
|
|
|
nv_destroy_rsync_info();
|
|
|
|
nvlink_drivers_exit();
|
|
|
|
|
|
nv_cap_drv_exit();
|
|
|
|
nv_module_resources_exit(sp);
|
|
}
|
|
|
|
static int __init
|
|
nv_module_init(nv_stack_t **sp)
|
|
{
|
|
int rc;
|
|
|
|
rc = nv_module_resources_init(sp);
|
|
if (rc < 0)
|
|
{
|
|
return rc;
|
|
}
|
|
|
|
rc = nv_cap_drv_init();
|
|
if (rc < 0)
|
|
{
|
|
nv_printf(NV_DBG_ERRORS, "NVRM: nv-cap-drv init failed.\n");
|
|
goto cap_drv_exit;
|
|
}
|
|
|
|
|
|
rc = nvlink_drivers_init();
|
|
if (rc < 0)
|
|
{
|
|
goto cap_drv_exit;
|
|
}
|
|
|
|
|
|
nv_init_rsync_info();
|
|
nv_sev_init();
|
|
|
|
if (!rm_init_rm(*sp))
|
|
{
|
|
nv_printf(NV_DBG_ERRORS, "NVRM: rm_init_rm() failed!\n");
|
|
rc = -EIO;
|
|
goto nvlink_exit;
|
|
}
|
|
|
|
rc = nv_module_state_init(*sp);
|
|
if (rc < 0)
|
|
{
|
|
goto init_rm_exit;
|
|
}
|
|
|
|
return rc;
|
|
|
|
init_rm_exit:
|
|
rm_shutdown_rm(*sp);
|
|
|
|
nvlink_exit:
|
|
nv_destroy_rsync_info();
|
|
|
|
nvlink_drivers_exit();
|
|
|
|
|
|
cap_drv_exit:
|
|
nv_cap_drv_exit();
|
|
nv_module_resources_exit(*sp);
|
|
|
|
return rc;
|
|
}
|
|
|
|
/*
|
|
* In this function we check for the cases where GPU exclusion is not
|
|
* honored, and issue a warning.
|
|
*
|
|
* Only GPUs that support a mechanism to query UUID prior to
|
|
* initializing the GPU can be excluded, so that we can detect and
|
|
* exclude them during device probe. This function checks that an
|
|
* initialized GPU was not specified in the exclusion list, and issues a
|
|
* warning if so.
|
|
*/
|
|
static void
|
|
nv_assert_not_in_gpu_exclusion_list(
|
|
nvidia_stack_t *sp,
|
|
nv_state_t *nv
|
|
)
|
|
{
|
|
char *uuid = rm_get_gpu_uuid(sp, nv);
|
|
|
|
if (uuid == NULL)
|
|
{
|
|
NV_DEV_PRINTF(NV_DBG_INFO, nv, "Unable to read UUID");
|
|
return;
|
|
}
|
|
|
|
if (nv_is_uuid_in_gpu_exclusion_list(uuid))
|
|
{
|
|
NV_DEV_PRINTF(NV_DBG_WARNINGS, nv,
|
|
"Could not exclude GPU %s because PBI is not supported\n",
|
|
uuid);
|
|
WARN_ON(1);
|
|
}
|
|
|
|
os_free_mem(uuid);
|
|
|
|
return;
|
|
}
|
|
|
|
static int __init nv_caps_root_init(void)
|
|
{
|
|
nvidia_caps_root = os_nv_cap_init("driver/" MODULE_NAME);
|
|
|
|
return (nvidia_caps_root == NULL) ? -ENOENT : 0;
|
|
}
|
|
|
|
static void nv_caps_root_exit(void)
|
|
{
|
|
os_nv_cap_destroy_entry(nvidia_caps_root);
|
|
nvidia_caps_root = NULL;
|
|
}
|
|
|
|
int __init nvidia_init_module(void)
|
|
{
|
|
int rc;
|
|
NvU32 count;
|
|
nvidia_stack_t *sp = NULL;
|
|
const NvBool is_nvswitch_present = os_is_nvswitch_present();
|
|
|
|
nv_memdbg_init();
|
|
|
|
rc = nv_procfs_init();
|
|
if (rc < 0)
|
|
{
|
|
nv_printf(NV_DBG_ERRORS, "NVRM: failed to initialize procfs.\n");
|
|
return rc;
|
|
}
|
|
|
|
rc = nv_caps_root_init();
|
|
if (rc < 0)
|
|
{
|
|
nv_printf(NV_DBG_ERRORS, "NVRM: failed to initialize capabilities.\n");
|
|
goto procfs_exit;
|
|
}
|
|
|
|
rc = nv_module_init(&sp);
|
|
if (rc < 0)
|
|
{
|
|
nv_printf(NV_DBG_ERRORS, "NVRM: failed to initialize module.\n");
|
|
goto caps_root_exit;
|
|
}
|
|
|
|
count = nvos_count_devices();
|
|
if ((count == 0) && (!is_nvswitch_present))
|
|
{
|
|
nv_printf(NV_DBG_ERRORS, "NVRM: No NVIDIA GPU found.\n");
|
|
rc = -ENODEV;
|
|
goto module_exit;
|
|
}
|
|
|
|
rc = nv_drivers_init();
|
|
if (rc < 0)
|
|
{
|
|
goto module_exit;
|
|
}
|
|
|
|
if (num_probed_nv_devices != count)
|
|
{
|
|
nv_printf(NV_DBG_ERRORS,
|
|
"NVRM: The NVIDIA probe routine was not called for %d device(s).\n",
|
|
count - num_probed_nv_devices);
|
|
nv_printf(NV_DBG_ERRORS,
|
|
"NVRM: This can occur when a driver such as: \n"
|
|
"NVRM: nouveau, rivafb, nvidiafb or rivatv "
|
|
"\nNVRM: was loaded and obtained ownership of the NVIDIA device(s).\n");
|
|
nv_printf(NV_DBG_ERRORS,
|
|
"NVRM: Try unloading the conflicting kernel module (and/or\n"
|
|
"NVRM: reconfigure your kernel without the conflicting\n"
|
|
"NVRM: driver(s)), then try loading the NVIDIA kernel module\n"
|
|
"NVRM: again.\n");
|
|
}
|
|
|
|
if ((num_probed_nv_devices == 0) && (!is_nvswitch_present))
|
|
{
|
|
rc = -ENODEV;
|
|
nv_printf(NV_DBG_ERRORS, "NVRM: No NVIDIA devices probed.\n");
|
|
goto drivers_exit;
|
|
}
|
|
|
|
if (num_probed_nv_devices != num_nv_devices)
|
|
{
|
|
nv_printf(NV_DBG_ERRORS,
|
|
"NVRM: The NVIDIA probe routine failed for %d device(s).\n",
|
|
num_probed_nv_devices - num_nv_devices);
|
|
}
|
|
|
|
if ((num_nv_devices == 0) && (!is_nvswitch_present))
|
|
{
|
|
rc = -ENODEV;
|
|
nv_printf(NV_DBG_ERRORS,
|
|
"NVRM: None of the NVIDIA devices were initialized.\n");
|
|
goto drivers_exit;
|
|
}
|
|
|
|
/*
|
|
* Initialize registry keys after PCI driver registration has
|
|
* completed successfully to support per-device module
|
|
* parameters.
|
|
*/
|
|
nv_registry_keys_init(sp);
|
|
|
|
nv_report_applied_patches();
|
|
|
|
nv_printf(NV_DBG_ERRORS, "NVRM: loading %s\n", pNVRM_ID);
|
|
|
|
#if defined(NV_UVM_ENABLE)
|
|
rc = nv_uvm_init();
|
|
if (rc != 0)
|
|
{
|
|
goto drivers_exit;
|
|
}
|
|
#endif
|
|
|
|
__nv_init_sp = sp;
|
|
|
|
return 0;
|
|
|
|
drivers_exit:
|
|
nv_drivers_exit();
|
|
|
|
module_exit:
|
|
nv_module_exit(sp);
|
|
|
|
caps_root_exit:
|
|
nv_caps_root_exit();
|
|
|
|
procfs_exit:
|
|
nv_procfs_exit();
|
|
|
|
return rc;
|
|
}
|
|
|
|
void nvidia_exit_module(void)
|
|
{
|
|
nvidia_stack_t *sp = __nv_init_sp;
|
|
|
|
#if defined(NV_UVM_ENABLE)
|
|
nv_uvm_exit();
|
|
#endif
|
|
|
|
nv_drivers_exit();
|
|
|
|
nv_module_exit(sp);
|
|
|
|
nv_caps_root_exit();
|
|
|
|
nv_procfs_exit();
|
|
|
|
nv_memdbg_exit();
|
|
}
|
|
|
|
static void *nv_alloc_file_private(void)
|
|
{
|
|
nv_linux_file_private_t *nvlfp;
|
|
unsigned int i;
|
|
|
|
NV_KMALLOC(nvlfp, sizeof(nv_linux_file_private_t));
|
|
if (!nvlfp)
|
|
return NULL;
|
|
|
|
memset(nvlfp, 0, sizeof(nv_linux_file_private_t));
|
|
|
|
for (i = 0; i < NV_FOPS_STACK_INDEX_COUNT; ++i)
|
|
{
|
|
NV_INIT_MUTEX(&nvlfp->fops_sp_lock[i]);
|
|
}
|
|
init_waitqueue_head(&nvlfp->waitqueue);
|
|
NV_SPIN_LOCK_INIT(&nvlfp->fp_lock);
|
|
|
|
return nvlfp;
|
|
}
|
|
|
|
static void nv_free_file_private(nv_linux_file_private_t *nvlfp)
|
|
{
|
|
nvidia_event_t *nvet;
|
|
|
|
if (nvlfp == NULL)
|
|
return;
|
|
|
|
for (nvet = nvlfp->event_data_head; nvet != NULL; nvet = nvlfp->event_data_head)
|
|
{
|
|
nvlfp->event_data_head = nvlfp->event_data_head->next;
|
|
NV_KFREE(nvet, sizeof(nvidia_event_t));
|
|
}
|
|
|
|
if (nvlfp->mmap_context.page_array != NULL)
|
|
{
|
|
os_free_mem(nvlfp->mmap_context.page_array);
|
|
}
|
|
|
|
NV_KFREE(nvlfp, sizeof(nv_linux_file_private_t));
|
|
}
|
|
|
|
|
|
static int nv_is_control_device(
|
|
struct inode *inode
|
|
)
|
|
{
|
|
return (minor((inode)->i_rdev) == NV_CONTROL_DEVICE_MINOR);
|
|
}
|
|
|
|
/*
|
|
* Search the global list of nv devices for the one with the given minor device
|
|
* number. If found, nvl is returned with nvl->ldata_lock taken.
|
|
*/
|
|
static nv_linux_state_t *find_minor(NvU32 minor)
|
|
{
|
|
nv_linux_state_t *nvl;
|
|
|
|
LOCK_NV_LINUX_DEVICES();
|
|
nvl = nv_linux_devices;
|
|
while (nvl != NULL)
|
|
{
|
|
if (nvl->minor_num == minor)
|
|
{
|
|
down(&nvl->ldata_lock);
|
|
break;
|
|
}
|
|
nvl = nvl->next;
|
|
}
|
|
|
|
UNLOCK_NV_LINUX_DEVICES();
|
|
return nvl;
|
|
}
|
|
|
|
/*
|
|
* Search the global list of nv devices for the one with the given gpu_id.
|
|
* If found, nvl is returned with nvl->ldata_lock taken.
|
|
*/
|
|
static nv_linux_state_t *find_gpu_id(NvU32 gpu_id)
|
|
{
|
|
nv_linux_state_t *nvl;
|
|
|
|
LOCK_NV_LINUX_DEVICES();
|
|
nvl = nv_linux_devices;
|
|
while (nvl != NULL)
|
|
{
|
|
nv_state_t *nv = NV_STATE_PTR(nvl);
|
|
if (nv->gpu_id == gpu_id)
|
|
{
|
|
down(&nvl->ldata_lock);
|
|
break;
|
|
}
|
|
nvl = nvl->next;
|
|
}
|
|
|
|
UNLOCK_NV_LINUX_DEVICES();
|
|
return nvl;
|
|
}
|
|
|
|
/*
|
|
* Search the global list of nv devices for the one with the given UUID. Devices
|
|
* with missing UUID information are ignored. If found, nvl is returned with
|
|
* nvl->ldata_lock taken.
|
|
*/
|
|
nv_linux_state_t *find_uuid(const NvU8 *uuid)
|
|
{
|
|
nv_linux_state_t *nvl = NULL;
|
|
nv_state_t *nv;
|
|
const NvU8 *dev_uuid;
|
|
|
|
LOCK_NV_LINUX_DEVICES();
|
|
|
|
for (nvl = nv_linux_devices; nvl; nvl = nvl->next)
|
|
{
|
|
nv = NV_STATE_PTR(nvl);
|
|
down(&nvl->ldata_lock);
|
|
dev_uuid = nv_get_cached_uuid(nv);
|
|
if (dev_uuid && memcmp(dev_uuid, uuid, GPU_UUID_LEN) == 0)
|
|
goto out;
|
|
up(&nvl->ldata_lock);
|
|
}
|
|
|
|
out:
|
|
UNLOCK_NV_LINUX_DEVICES();
|
|
return nvl;
|
|
}
|
|
|
|
/*
|
|
* Search the global list of nv devices. The search logic is:
|
|
*
|
|
* 1) If any device has the given UUID, return it
|
|
*
|
|
* 2) If no device has the given UUID but at least one device is missing
|
|
* its UUID (for example because rm_init_adapter has not run on it yet),
|
|
* return that device.
|
|
*
|
|
* 3) If no device has the given UUID and all UUIDs are present, return NULL.
|
|
*
|
|
* In cases 1 and 2, nvl is returned with nvl->ldata_lock taken.
|
|
*
|
|
* The reason for this weird logic is because UUIDs aren't always available. See
|
|
* bug 1642200.
|
|
*/
|
|
static nv_linux_state_t *find_uuid_candidate(const NvU8 *uuid)
|
|
{
|
|
nv_linux_state_t *nvl = NULL;
|
|
nv_state_t *nv;
|
|
const NvU8 *dev_uuid;
|
|
int use_missing;
|
|
int has_missing = 0;
|
|
|
|
LOCK_NV_LINUX_DEVICES();
|
|
|
|
/*
|
|
* Take two passes through the list. The first pass just looks for the UUID.
|
|
* The second looks for the target or missing UUIDs. It would be nice if
|
|
* this could be done in a single pass by remembering which nvls are missing
|
|
* UUIDs, but we have to hold the nvl lock after we check for the UUID.
|
|
*/
|
|
for (use_missing = 0; use_missing <= 1; use_missing++)
|
|
{
|
|
for (nvl = nv_linux_devices; nvl; nvl = nvl->next)
|
|
{
|
|
nv = NV_STATE_PTR(nvl);
|
|
down(&nvl->ldata_lock);
|
|
dev_uuid = nv_get_cached_uuid(nv);
|
|
if (dev_uuid)
|
|
{
|
|
/* Case 1: If a device has the given UUID, return it */
|
|
if (memcmp(dev_uuid, uuid, GPU_UUID_LEN) == 0)
|
|
goto out;
|
|
}
|
|
else
|
|
{
|
|
/* Case 2: If no device has the given UUID but at least one
|
|
* device is missing its UUID, return that device. */
|
|
if (use_missing)
|
|
goto out;
|
|
has_missing = 1;
|
|
}
|
|
up(&nvl->ldata_lock);
|
|
}
|
|
|
|
/* Case 3: If no device has the given UUID and all UUIDs are present,
|
|
* return NULL. */
|
|
if (!has_missing)
|
|
break;
|
|
}
|
|
|
|
out:
|
|
UNLOCK_NV_LINUX_DEVICES();
|
|
return nvl;
|
|
}
|
|
|
|
void nv_dev_free_stacks(nv_linux_state_t *nvl)
|
|
{
|
|
NvU32 i;
|
|
for (i = 0; i < NV_DEV_STACK_COUNT; i++)
|
|
{
|
|
if (nvl->sp[i])
|
|
{
|
|
nv_kmem_cache_free_stack(nvl->sp[i]);
|
|
nvl->sp[i] = NULL;
|
|
}
|
|
}
|
|
}
|
|
|
|
static int nv_dev_alloc_stacks(nv_linux_state_t *nvl)
|
|
{
|
|
NvU32 i;
|
|
int rc;
|
|
|
|
for (i = 0; i < NV_DEV_STACK_COUNT; i++)
|
|
{
|
|
rc = nv_kmem_cache_alloc_stack(&nvl->sp[i]);
|
|
if (rc != 0)
|
|
{
|
|
nv_dev_free_stacks(nvl);
|
|
return rc;
|
|
}
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int validate_numa_start_state(nv_linux_state_t *nvl)
|
|
{
|
|
int rc = 0;
|
|
int numa_status = nv_get_numa_status(nvl);
|
|
|
|
if (numa_status != NV_IOCTL_NUMA_STATUS_DISABLED)
|
|
{
|
|
if (nv_ctl_device.numa_memblock_size == 0)
|
|
{
|
|
nv_printf(NV_DBG_ERRORS, "NVRM: numa memblock size of zero "
|
|
"found during device start");
|
|
rc = -EINVAL;
|
|
}
|
|
else
|
|
{
|
|
/* Keep the individual devices consistent with the control device */
|
|
nvl->numa_memblock_size = nv_ctl_device.numa_memblock_size;
|
|
}
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
NV_STATUS NV_API_CALL nv_get_num_dpaux_instances(nv_state_t *nv, NvU32 *num_instances)
|
|
{
|
|
*num_instances = nv->num_dpaux_instance;
|
|
return NV_OK;
|
|
}
|
|
|
|
void NV_API_CALL
|
|
nv_schedule_uvm_isr(nv_state_t *nv)
|
|
{
|
|
#if defined(NV_UVM_ENABLE)
|
|
nv_uvm_event_interrupt(nv_get_cached_uuid(nv));
|
|
#endif
|
|
}
|
|
|
|
/*
|
|
* Brings up the device on the first file open. Assumes nvl->ldata_lock is held.
|
|
*/
|
|
static int nv_start_device(nv_state_t *nv, nvidia_stack_t *sp)
|
|
{
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
#if defined(NV_LINUX_PCIE_MSI_SUPPORTED)
|
|
NvU32 msi_config = 0;
|
|
#endif
|
|
int rc = 0;
|
|
NvBool kthread_init = NV_FALSE;
|
|
NvBool power_ref = NV_FALSE;
|
|
|
|
rc = nv_get_rsync_info();
|
|
if (rc != 0)
|
|
{
|
|
return rc;
|
|
}
|
|
|
|
rc = validate_numa_start_state(nvl);
|
|
if (rc != 0)
|
|
{
|
|
goto failed;
|
|
}
|
|
|
|
if (nv_dev_is_pci(nvl->dev) && (nv->pci_info.device_id == 0))
|
|
{
|
|
nv_printf(NV_DBG_ERRORS, "NVRM: open of non-existent GPU with minor number %d\n", nvl->minor_num);
|
|
rc = -ENXIO;
|
|
goto failed;
|
|
}
|
|
|
|
if (!(nv->flags & NV_FLAG_PERSISTENT_SW_STATE))
|
|
{
|
|
if (rm_ref_dynamic_power(sp, nv, NV_DYNAMIC_PM_COARSE) != NV_OK)
|
|
{
|
|
rc = -EINVAL;
|
|
goto failed;
|
|
}
|
|
power_ref = NV_TRUE;
|
|
}
|
|
else
|
|
{
|
|
if (rm_ref_dynamic_power(sp, nv, NV_DYNAMIC_PM_FINE) != NV_OK)
|
|
{
|
|
rc = -EINVAL;
|
|
goto failed;
|
|
}
|
|
power_ref = NV_TRUE;
|
|
}
|
|
|
|
rc = nv_init_ibmnpu_devices(nv);
|
|
if (rc != 0)
|
|
{
|
|
nv_printf(NV_DBG_ERRORS,
|
|
"NVRM: failed to initialize ibmnpu devices attached to GPU with minor number %d\n",
|
|
nvl->minor_num);
|
|
goto failed;
|
|
}
|
|
|
|
if (!(nv->flags & NV_FLAG_PERSISTENT_SW_STATE))
|
|
{
|
|
rc = nv_dev_alloc_stacks(nvl);
|
|
if (rc != 0)
|
|
goto failed;
|
|
}
|
|
|
|
#if defined(NV_LINUX_PCIE_MSI_SUPPORTED)
|
|
if (nv_dev_is_pci(nvl->dev))
|
|
{
|
|
if (!(nv->flags & NV_FLAG_PERSISTENT_SW_STATE))
|
|
{
|
|
rm_read_registry_dword(sp, nv, NV_REG_ENABLE_MSI, &msi_config);
|
|
if (msi_config == 1)
|
|
{
|
|
if (pci_find_capability(nvl->pci_dev, PCI_CAP_ID_MSIX))
|
|
{
|
|
nv_init_msix(nv);
|
|
}
|
|
if (pci_find_capability(nvl->pci_dev, PCI_CAP_ID_MSI) &&
|
|
!(nv->flags & NV_FLAG_USES_MSIX))
|
|
{
|
|
nv_init_msi(nv);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
#endif
|
|
|
|
if (((!(nv->flags & NV_FLAG_USES_MSI)) && (!(nv->flags & NV_FLAG_USES_MSIX)))
|
|
&& (nv->interrupt_line == 0) && !(nv->flags & NV_FLAG_SOC_DISPLAY))
|
|
{
|
|
NV_DEV_PRINTF(NV_DBG_ERRORS, nv,
|
|
"No interrupts of any type are available. Cannot use this GPU.\n");
|
|
rc = -EIO;
|
|
goto failed;
|
|
}
|
|
|
|
rc = 0;
|
|
if (!(nv->flags & NV_FLAG_PERSISTENT_SW_STATE))
|
|
{
|
|
if (nv->flags & NV_FLAG_SOC_DISPLAY)
|
|
{
|
|
|
|
|
|
|
|
}
|
|
else if (!(nv->flags & NV_FLAG_USES_MSIX))
|
|
{
|
|
rc = request_threaded_irq(nv->interrupt_line, nvidia_isr,
|
|
nvidia_isr_kthread_bh, nv_default_irq_flags(nv),
|
|
nv_device_name, (void *)nvl);
|
|
}
|
|
#if defined(NV_LINUX_PCIE_MSI_SUPPORTED)
|
|
else
|
|
{
|
|
rc = nv_request_msix_irq(nvl);
|
|
}
|
|
#endif
|
|
}
|
|
if (rc != 0)
|
|
{
|
|
if ((nv->interrupt_line != 0) && (rc == -EBUSY))
|
|
{
|
|
NV_DEV_PRINTF(NV_DBG_ERRORS, nv,
|
|
"Tried to get IRQ %d, but another driver\n",
|
|
(unsigned int) nv->interrupt_line);
|
|
nv_printf(NV_DBG_ERRORS, "NVRM: has it and is not sharing it.\n");
|
|
nv_printf(NV_DBG_ERRORS, "NVRM: You may want to verify that no audio driver");
|
|
nv_printf(NV_DBG_ERRORS, " is using the IRQ.\n");
|
|
}
|
|
NV_DEV_PRINTF(NV_DBG_ERRORS, nv, "request_irq() failed (%d)\n", rc);
|
|
goto failed;
|
|
}
|
|
|
|
if (!(nv->flags & NV_FLAG_PERSISTENT_SW_STATE))
|
|
{
|
|
rc = os_alloc_mutex(&nvl->isr_bh_unlocked_mutex);
|
|
if (rc != 0)
|
|
goto failed;
|
|
nv_kthread_q_item_init(&nvl->bottom_half_q_item, nvidia_isr_bh_unlocked, (void *)nv);
|
|
rc = nv_kthread_q_init(&nvl->bottom_half_q, nv_device_name);
|
|
if (rc != 0)
|
|
goto failed;
|
|
kthread_init = NV_TRUE;
|
|
|
|
rc = nv_kthread_q_init(&nvl->queue.nvk, "nv_queue");
|
|
if (rc)
|
|
goto failed;
|
|
nv->queue = &nvl->queue;
|
|
}
|
|
|
|
if (!rm_init_adapter(sp, nv))
|
|
{
|
|
if (!(nv->flags & NV_FLAG_USES_MSIX) &&
|
|
!(nv->flags & NV_FLAG_SOC_DISPLAY))
|
|
{
|
|
free_irq(nv->interrupt_line, (void *) nvl);
|
|
}
|
|
else if (nv->flags & NV_FLAG_SOC_DISPLAY)
|
|
{
|
|
|
|
|
|
|
|
}
|
|
#if defined(NV_LINUX_PCIE_MSI_SUPPORTED)
|
|
else
|
|
{
|
|
nv_free_msix_irq(nvl);
|
|
}
|
|
#endif
|
|
NV_DEV_PRINTF(NV_DBG_ERRORS, nv,
|
|
"rm_init_adapter failed, device minor number %d\n",
|
|
nvl->minor_num);
|
|
rc = -EIO;
|
|
goto failed;
|
|
}
|
|
|
|
{
|
|
const NvU8 *uuid = rm_get_gpu_uuid_raw(sp, nv);
|
|
|
|
if (uuid != NULL)
|
|
{
|
|
#if defined(NV_UVM_ENABLE)
|
|
nv_uvm_notify_start_device(uuid);
|
|
#endif
|
|
}
|
|
}
|
|
|
|
if (!(nv->flags & NV_FLAG_PERSISTENT_SW_STATE))
|
|
{
|
|
nv_acpi_register_notifier(nvl);
|
|
}
|
|
|
|
nv->flags |= NV_FLAG_OPEN;
|
|
|
|
/*
|
|
* Now that RM init is done, allow dynamic power to control the GPU in FINE
|
|
* mode, if enabled. (If the mode is COARSE, this unref will do nothing
|
|
* which will cause the GPU to remain powered up.)
|
|
* This is balanced by a FINE ref increment at the beginning of
|
|
* nv_stop_device().
|
|
*/
|
|
rm_unref_dynamic_power(sp, nv, NV_DYNAMIC_PM_FINE);
|
|
|
|
return 0;
|
|
|
|
failed:
|
|
#if defined(NV_LINUX_PCIE_MSI_SUPPORTED)
|
|
if (nv->flags & NV_FLAG_USES_MSI)
|
|
{
|
|
nv->flags &= ~NV_FLAG_USES_MSI;
|
|
NV_PCI_DISABLE_MSI(nvl->pci_dev);
|
|
if(nvl->irq_count)
|
|
NV_KFREE(nvl->irq_count, nvl->num_intr * sizeof(nv_irq_count_info_t));
|
|
}
|
|
if (nv->flags & NV_FLAG_USES_MSIX)
|
|
{
|
|
nv->flags &= ~NV_FLAG_USES_MSIX;
|
|
pci_disable_msix(nvl->pci_dev);
|
|
NV_KFREE(nvl->irq_count, nvl->num_intr*sizeof(nv_irq_count_info_t));
|
|
NV_KFREE(nvl->msix_entries, nvl->num_intr*sizeof(struct msix_entry));
|
|
}
|
|
|
|
if (nvl->msix_bh_mutex)
|
|
{
|
|
os_free_mutex(nvl->msix_bh_mutex);
|
|
nvl->msix_bh_mutex = NULL;
|
|
}
|
|
#endif
|
|
|
|
if (nv->queue && !(nv->flags & NV_FLAG_PERSISTENT_SW_STATE))
|
|
{
|
|
nv->queue = NULL;
|
|
nv_kthread_q_stop(&nvl->queue.nvk);
|
|
}
|
|
|
|
if (kthread_init && !(nv->flags & NV_FLAG_PERSISTENT_SW_STATE))
|
|
nv_kthread_q_stop(&nvl->bottom_half_q);
|
|
|
|
if (nvl->isr_bh_unlocked_mutex)
|
|
{
|
|
os_free_mutex(nvl->isr_bh_unlocked_mutex);
|
|
nvl->isr_bh_unlocked_mutex = NULL;
|
|
}
|
|
|
|
nv_dev_free_stacks(nvl);
|
|
|
|
nv_unregister_ibmnpu_devices(nv);
|
|
|
|
if (power_ref)
|
|
{
|
|
rm_unref_dynamic_power(sp, nv, NV_DYNAMIC_PM_COARSE);
|
|
}
|
|
|
|
nv_put_rsync_info();
|
|
|
|
return rc;
|
|
}
|
|
|
|
/*
|
|
* Makes sure the device is ready for operations and increases nvl->usage_count.
|
|
* Assumes nvl->ldata_lock is held.
|
|
*/
|
|
static int nv_open_device(nv_state_t *nv, nvidia_stack_t *sp)
|
|
{
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
int rc;
|
|
NV_STATUS status;
|
|
|
|
if (os_is_vgx_hyper())
|
|
{
|
|
/* fail open if GPU is being unbound */
|
|
if (nv->flags & NV_FLAG_UNBIND_LOCK)
|
|
{
|
|
NV_DEV_PRINTF(NV_DBG_ERRORS, nv,
|
|
"Open failed as GPU is locked for unbind operation\n");
|
|
return -ENODEV;
|
|
}
|
|
}
|
|
|
|
NV_DEV_PRINTF(NV_DBG_INFO, nv, "Opening GPU with minor number %d\n",
|
|
nvl->minor_num);
|
|
|
|
status = nv_check_gpu_state(nv);
|
|
if (status == NV_ERR_GPU_IS_LOST)
|
|
{
|
|
NV_DEV_PRINTF(NV_DBG_INFO, nv, "Device in removal process\n");
|
|
return -ENODEV;
|
|
}
|
|
|
|
|
|
if (unlikely(NV_ATOMIC_READ(nvl->usage_count) >= NV_S32_MAX))
|
|
return -EMFILE;
|
|
|
|
|
|
if ( ! (nv->flags & NV_FLAG_OPEN))
|
|
{
|
|
/* Sanity check: !NV_FLAG_OPEN requires usage_count == 0 */
|
|
if (NV_ATOMIC_READ(nvl->usage_count) != 0)
|
|
{
|
|
NV_DEV_PRINTF(NV_DBG_ERRORS, nv,
|
|
"Minor device %u is referenced without being open!\n",
|
|
nvl->minor_num);
|
|
WARN_ON(1);
|
|
return -EBUSY;
|
|
}
|
|
|
|
rc = nv_start_device(nv, sp);
|
|
if (rc != 0)
|
|
return rc;
|
|
}
|
|
else if (rm_is_device_sequestered(sp, nv))
|
|
{
|
|
/* Do not increment the usage count of sequestered devices. */
|
|
NV_DEV_PRINTF(NV_DBG_ERRORS, nv, "Device is currently unavailable\n");
|
|
return -EBUSY;
|
|
}
|
|
|
|
NV_ATOMIC_INC(nvl->usage_count);
|
|
return 0;
|
|
}
|
|
|
|
static void nv_init_mapping_revocation(nv_linux_state_t *nvl,
|
|
struct file *file,
|
|
nv_linux_file_private_t *nvlfp,
|
|
struct inode *inode)
|
|
{
|
|
down(&nvl->mmap_lock);
|
|
|
|
/* Set up struct address_space for use with unmap_mapping_range() */
|
|
nv_address_space_init_once(&nvlfp->mapping);
|
|
nvlfp->mapping.host = inode;
|
|
nvlfp->mapping.a_ops = inode->i_mapping->a_ops;
|
|
#if defined(NV_ADDRESS_SPACE_HAS_BACKING_DEV_INFO)
|
|
nvlfp->mapping.backing_dev_info = inode->i_mapping->backing_dev_info;
|
|
#endif
|
|
file->f_mapping = &nvlfp->mapping;
|
|
|
|
/* Add nvlfp to list of open files in nvl for mapping revocation */
|
|
list_add(&nvlfp->entry, &nvl->open_files);
|
|
|
|
up(&nvl->mmap_lock);
|
|
}
|
|
|
|
/*
|
|
** nvidia_open
|
|
**
|
|
** nv driver open entry point. Sessions are created here.
|
|
*/
|
|
int
|
|
nvidia_open(
|
|
struct inode *inode,
|
|
struct file *file
|
|
)
|
|
{
|
|
nv_state_t *nv = NULL;
|
|
nv_linux_state_t *nvl = NULL;
|
|
int rc = 0;
|
|
nv_linux_file_private_t *nvlfp = NULL;
|
|
nvidia_stack_t *sp = NULL;
|
|
unsigned int i;
|
|
unsigned int k;
|
|
|
|
nv_printf(NV_DBG_INFO, "NVRM: nvidia_open...\n");
|
|
|
|
nvlfp = nv_alloc_file_private();
|
|
if (nvlfp == NULL)
|
|
{
|
|
nv_printf(NV_DBG_ERRORS, "NVRM: failed to allocate file private!\n");
|
|
return -ENOMEM;
|
|
}
|
|
|
|
rc = nv_kmem_cache_alloc_stack(&sp);
|
|
if (rc != 0)
|
|
{
|
|
nv_free_file_private(nvlfp);
|
|
return rc;
|
|
}
|
|
|
|
for (i = 0; i < NV_FOPS_STACK_INDEX_COUNT; ++i)
|
|
{
|
|
rc = nv_kmem_cache_alloc_stack(&nvlfp->fops_sp[i]);
|
|
if (rc != 0)
|
|
{
|
|
nv_kmem_cache_free_stack(sp);
|
|
for (k = 0; k < i; ++k)
|
|
{
|
|
nv_kmem_cache_free_stack(nvlfp->fops_sp[k]);
|
|
}
|
|
nv_free_file_private(nvlfp);
|
|
return rc;
|
|
}
|
|
}
|
|
|
|
NV_SET_FILE_PRIVATE(file, nvlfp);
|
|
nvlfp->sp = sp;
|
|
|
|
/* for control device, just jump to its open routine */
|
|
/* after setting up the private data */
|
|
if (nv_is_control_device(inode))
|
|
{
|
|
rc = nvidia_ctl_open(inode, file);
|
|
if (rc != 0)
|
|
goto failed;
|
|
return rc;
|
|
}
|
|
|
|
rc = nv_down_read_interruptible(&nv_system_pm_lock);
|
|
if (rc < 0)
|
|
goto failed;
|
|
|
|
/* Takes nvl->ldata_lock */
|
|
nvl = find_minor(NV_DEVICE_MINOR_NUMBER(inode));
|
|
if (!nvl)
|
|
{
|
|
rc = -ENODEV;
|
|
up_read(&nv_system_pm_lock);
|
|
goto failed;
|
|
}
|
|
|
|
nvlfp->nvptr = nvl;
|
|
nv = NV_STATE_PTR(nvl);
|
|
|
|
if ((nv->flags & NV_FLAG_EXCLUDE) != 0)
|
|
{
|
|
char *uuid = rm_get_gpu_uuid(sp, nv);
|
|
NV_DEV_PRINTF(NV_DBG_ERRORS, nv,
|
|
"open() not permitted for excluded %s\n",
|
|
(uuid != NULL) ? uuid : "GPU");
|
|
if (uuid != NULL)
|
|
os_free_mem(uuid);
|
|
rc = -EPERM;
|
|
goto failed1;
|
|
}
|
|
|
|
rc = nv_open_device(nv, sp);
|
|
/* Fall-through on error */
|
|
|
|
nv_assert_not_in_gpu_exclusion_list(sp, nv);
|
|
|
|
failed1:
|
|
up(&nvl->ldata_lock);
|
|
|
|
up_read(&nv_system_pm_lock);
|
|
failed:
|
|
if (rc != 0)
|
|
{
|
|
if (nvlfp != NULL)
|
|
{
|
|
nv_kmem_cache_free_stack(sp);
|
|
for (i = 0; i < NV_FOPS_STACK_INDEX_COUNT; ++i)
|
|
{
|
|
nv_kmem_cache_free_stack(nvlfp->fops_sp[i]);
|
|
}
|
|
nv_free_file_private(nvlfp);
|
|
NV_SET_FILE_PRIVATE(file, NULL);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
nv_init_mapping_revocation(nvl, file, nvlfp, inode);
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
static void validate_numa_shutdown_state(nv_linux_state_t *nvl)
|
|
{
|
|
int numa_status = nv_get_numa_status(nvl);
|
|
WARN_ON((numa_status != NV_IOCTL_NUMA_STATUS_OFFLINE) &&
|
|
(numa_status != NV_IOCTL_NUMA_STATUS_DISABLED));
|
|
}
|
|
|
|
void nv_shutdown_adapter(nvidia_stack_t *sp,
|
|
nv_state_t *nv,
|
|
nv_linux_state_t *nvl)
|
|
{
|
|
validate_numa_shutdown_state(nvl);
|
|
|
|
rm_disable_adapter(sp, nv);
|
|
|
|
// It's safe to call nv_kthread_q_stop even if queue is not initialized
|
|
nv_kthread_q_stop(&nvl->bottom_half_q);
|
|
|
|
if (nv->queue != NULL)
|
|
{
|
|
nv->queue = NULL;
|
|
nv_kthread_q_stop(&nvl->queue.nvk);
|
|
}
|
|
|
|
if (nvl->isr_bh_unlocked_mutex)
|
|
{
|
|
os_free_mutex(nvl->isr_bh_unlocked_mutex);
|
|
nvl->isr_bh_unlocked_mutex = NULL;
|
|
}
|
|
|
|
if (!(nv->flags & NV_FLAG_USES_MSIX) &&
|
|
!(nv->flags & NV_FLAG_SOC_DISPLAY))
|
|
{
|
|
free_irq(nv->interrupt_line, (void *)nvl);
|
|
if (nv->flags & NV_FLAG_USES_MSI)
|
|
{
|
|
NV_PCI_DISABLE_MSI(nvl->pci_dev);
|
|
if(nvl->irq_count)
|
|
NV_KFREE(nvl->irq_count, nvl->num_intr * sizeof(nv_irq_count_info_t));
|
|
}
|
|
}
|
|
else if (nv->flags & NV_FLAG_SOC_DISPLAY)
|
|
{
|
|
|
|
|
|
|
|
}
|
|
#if defined(NV_LINUX_PCIE_MSI_SUPPORTED)
|
|
else
|
|
{
|
|
nv_free_msix_irq(nvl);
|
|
pci_disable_msix(nvl->pci_dev);
|
|
nv->flags &= ~NV_FLAG_USES_MSIX;
|
|
NV_KFREE(nvl->msix_entries, nvl->num_intr*sizeof(struct msix_entry));
|
|
NV_KFREE(nvl->irq_count, nvl->num_intr*sizeof(nv_irq_count_info_t));
|
|
}
|
|
#endif
|
|
|
|
if (nvl->msix_bh_mutex)
|
|
{
|
|
os_free_mutex(nvl->msix_bh_mutex);
|
|
nvl->msix_bh_mutex = NULL;
|
|
}
|
|
|
|
rm_shutdown_adapter(sp, nv);
|
|
}
|
|
|
|
/*
|
|
* Tears down the device on the last file close. Assumes nvl->ldata_lock is
|
|
* held.
|
|
*/
|
|
static void nv_stop_device(nv_state_t *nv, nvidia_stack_t *sp)
|
|
{
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
static int persistence_mode_notice_logged;
|
|
|
|
/*
|
|
* The GPU needs to be powered on to go through the teardown sequence.
|
|
* This balances the FINE unref at the end of nv_start_device().
|
|
*/
|
|
rm_ref_dynamic_power(sp, nv, NV_DYNAMIC_PM_FINE);
|
|
|
|
#if defined(NV_UVM_ENABLE)
|
|
{
|
|
const NvU8* uuid;
|
|
// Inform UVM before disabling adapter. Use cached copy
|
|
uuid = nv_get_cached_uuid(nv);
|
|
if (uuid != NULL)
|
|
{
|
|
// this function cannot fail
|
|
nv_uvm_notify_stop_device(uuid);
|
|
}
|
|
}
|
|
#endif
|
|
/* Adapter is already shutdown as part of nvidia_pci_remove */
|
|
if (!nv->removed)
|
|
{
|
|
if (nv->flags & NV_FLAG_PERSISTENT_SW_STATE)
|
|
{
|
|
rm_disable_adapter(sp, nv);
|
|
}
|
|
else
|
|
{
|
|
nv_acpi_unregister_notifier(nvl);
|
|
nv_shutdown_adapter(sp, nv, nvl);
|
|
}
|
|
}
|
|
|
|
if (!(nv->flags & NV_FLAG_PERSISTENT_SW_STATE))
|
|
{
|
|
nv_dev_free_stacks(nvl);
|
|
}
|
|
|
|
if ((nv->flags & NV_FLAG_PERSISTENT_SW_STATE) &&
|
|
(!persistence_mode_notice_logged) && (!os_is_vgx_hyper()))
|
|
{
|
|
nv_printf(NV_DBG_ERRORS, "NVRM: Persistence mode is deprecated and"
|
|
" will be removed in a future release. Please use"
|
|
" nvidia-persistenced instead.\n");
|
|
persistence_mode_notice_logged = 1;
|
|
}
|
|
|
|
/* leave INIT flag alone so we don't reinit every time */
|
|
nv->flags &= ~NV_FLAG_OPEN;
|
|
|
|
nv_unregister_ibmnpu_devices(nv);
|
|
|
|
if (!(nv->flags & NV_FLAG_PERSISTENT_SW_STATE))
|
|
{
|
|
rm_unref_dynamic_power(sp, nv, NV_DYNAMIC_PM_COARSE);
|
|
}
|
|
else
|
|
{
|
|
/* If in legacy persistence mode, only unref FINE refcount. */
|
|
rm_unref_dynamic_power(sp, nv, NV_DYNAMIC_PM_FINE);
|
|
}
|
|
|
|
nv_put_rsync_info();
|
|
}
|
|
|
|
/*
|
|
* Decreases nvl->usage_count, stopping the device when it reaches 0. Assumes
|
|
* nvl->ldata_lock is held.
|
|
*/
|
|
static void nv_close_device(nv_state_t *nv, nvidia_stack_t *sp)
|
|
{
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
|
|
if (NV_ATOMIC_READ(nvl->usage_count) == 0)
|
|
{
|
|
nv_printf(NV_DBG_ERRORS,
|
|
"NVRM: Attempting to close unopened minor device %u!\n",
|
|
nvl->minor_num);
|
|
WARN_ON(1);
|
|
return;
|
|
}
|
|
|
|
if (NV_ATOMIC_DEC_AND_TEST(nvl->usage_count))
|
|
nv_stop_device(nv, sp);
|
|
}
|
|
|
|
/*
|
|
** nvidia_close
|
|
**
|
|
** Primary driver close entry point.
|
|
*/
|
|
|
|
static void
|
|
nvidia_close_callback(
|
|
nv_linux_file_private_t *nvlfp
|
|
)
|
|
{
|
|
nv_linux_state_t *nvl = nvlfp->nvptr;
|
|
nv_state_t *nv = NV_STATE_PTR(nvl);
|
|
nvidia_stack_t *sp = nvlfp->sp;
|
|
unsigned int i;
|
|
NvBool bRemove = NV_FALSE;
|
|
|
|
rm_cleanup_file_private(sp, nv, &nvlfp->nvfp);
|
|
|
|
down(&nvl->mmap_lock);
|
|
list_del(&nvlfp->entry);
|
|
up(&nvl->mmap_lock);
|
|
|
|
down(&nvl->ldata_lock);
|
|
nv_close_device(nv, sp);
|
|
|
|
bRemove = (!NV_IS_DEVICE_IN_SURPRISE_REMOVAL(nv)) &&
|
|
(NV_ATOMIC_READ(nvl->usage_count) == 0) &&
|
|
rm_get_device_remove_flag(sp, nv->gpu_id);
|
|
|
|
for (i = 0; i < NV_FOPS_STACK_INDEX_COUNT; ++i)
|
|
{
|
|
nv_kmem_cache_free_stack(nvlfp->fops_sp[i]);
|
|
}
|
|
|
|
nv_free_file_private(nvlfp);
|
|
|
|
/*
|
|
* In case of surprise removal of device, we have 2 cases as below:
|
|
*
|
|
* 1> When nvidia_pci_remove is scheduled prior to nvidia_close.
|
|
* nvidia_pci_remove will not destroy linux layer locks & nv linux state
|
|
* struct but will set variable nv->removed for nvidia_close.
|
|
* Once all the clients are closed, last nvidia_close will clean up linux
|
|
* layer locks and nv linux state struct.
|
|
*
|
|
* 2> When nvidia_close is scheduled prior to nvidia_pci_remove.
|
|
* This will be treated as normal working case. nvidia_close will not do
|
|
* any cleanup related to linux layer locks and nv linux state struct.
|
|
* nvidia_pci_remove when scheduled will do necessary cleanup.
|
|
*/
|
|
if ((NV_ATOMIC_READ(nvl->usage_count) == 0) && nv->removed)
|
|
{
|
|
nvidia_frontend_remove_device((void *)&nv_fops, nvl);
|
|
nv_lock_destroy_locks(sp, nv);
|
|
NV_KFREE(nvl, sizeof(nv_linux_state_t));
|
|
}
|
|
else
|
|
{
|
|
up(&nvl->ldata_lock);
|
|
|
|
#if defined(NV_PCI_STOP_AND_REMOVE_BUS_DEVICE)
|
|
if (bRemove)
|
|
{
|
|
NV_PCI_STOP_AND_REMOVE_BUS_DEVICE(nvl->pci_dev);
|
|
}
|
|
#endif
|
|
}
|
|
|
|
nv_kmem_cache_free_stack(sp);
|
|
}
|
|
|
|
static void nvidia_close_deferred(void *data)
|
|
{
|
|
nv_linux_file_private_t *nvlfp = data;
|
|
|
|
down_read(&nv_system_pm_lock);
|
|
|
|
nvidia_close_callback(nvlfp);
|
|
|
|
up_read(&nv_system_pm_lock);
|
|
}
|
|
|
|
int
|
|
nvidia_close(
|
|
struct inode *inode,
|
|
struct file *file
|
|
)
|
|
{
|
|
int rc;
|
|
nv_linux_file_private_t *nvlfp = NV_GET_LINUX_FILE_PRIVATE(file);
|
|
nv_linux_state_t *nvl = nvlfp->nvptr;
|
|
nv_state_t *nv = NV_STATE_PTR(nvl);
|
|
|
|
NV_DEV_PRINTF(NV_DBG_INFO, nv, "nvidia_close on GPU with minor number %d\n", NV_DEVICE_MINOR_NUMBER(inode));
|
|
|
|
if (nv_is_control_device(inode))
|
|
{
|
|
return nvidia_ctl_close(inode, file);
|
|
}
|
|
|
|
NV_SET_FILE_PRIVATE(file, NULL);
|
|
|
|
rc = nv_down_read_interruptible(&nv_system_pm_lock);
|
|
if (rc == 0)
|
|
{
|
|
nvidia_close_callback(nvlfp);
|
|
up_read(&nv_system_pm_lock);
|
|
}
|
|
else
|
|
{
|
|
nv_kthread_q_item_init(&nvlfp->deferred_close_q_item,
|
|
nvidia_close_deferred,
|
|
nvlfp);
|
|
rc = nv_kthread_q_schedule_q_item(&nv_deferred_close_kthread_q,
|
|
&nvlfp->deferred_close_q_item);
|
|
WARN_ON(rc == 0);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
unsigned int
|
|
nvidia_poll(
|
|
struct file *file,
|
|
poll_table *wait
|
|
)
|
|
{
|
|
unsigned int mask = 0;
|
|
nv_linux_file_private_t *nvlfp = NV_GET_LINUX_FILE_PRIVATE(file);
|
|
unsigned long eflags;
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_FILEP(file);
|
|
nv_state_t *nv = NV_STATE_PTR(nvl);
|
|
NV_STATUS status;
|
|
|
|
status = nv_check_gpu_state(nv);
|
|
if (status == NV_ERR_GPU_IS_LOST)
|
|
{
|
|
NV_DEV_PRINTF(NV_DBG_INFO, nv, "GPU is lost, skipping nvidia_poll\n");
|
|
return POLLHUP;
|
|
}
|
|
|
|
if ((file->f_flags & O_NONBLOCK) == 0)
|
|
poll_wait(file, &nvlfp->waitqueue, wait);
|
|
|
|
NV_SPIN_LOCK_IRQSAVE(&nvlfp->fp_lock, eflags);
|
|
|
|
if ((nvlfp->event_data_head != NULL) || nvlfp->dataless_event_pending)
|
|
{
|
|
mask = (POLLPRI | POLLIN);
|
|
nvlfp->dataless_event_pending = NV_FALSE;
|
|
}
|
|
|
|
NV_SPIN_UNLOCK_IRQRESTORE(&nvlfp->fp_lock, eflags);
|
|
|
|
return mask;
|
|
}
|
|
|
|
#define NV_CTL_DEVICE_ONLY(nv) \
|
|
{ \
|
|
if (((nv)->flags & NV_FLAG_CONTROL) == 0) \
|
|
{ \
|
|
status = -EINVAL; \
|
|
goto done; \
|
|
} \
|
|
}
|
|
|
|
#define NV_ACTUAL_DEVICE_ONLY(nv) \
|
|
{ \
|
|
if (((nv)->flags & NV_FLAG_CONTROL) != 0) \
|
|
{ \
|
|
status = -EINVAL; \
|
|
goto done; \
|
|
} \
|
|
}
|
|
|
|
/*
|
|
* Fills the ci array with the state of num_entries devices. Returns -EINVAL if
|
|
* num_entries isn't big enough to hold all available devices.
|
|
*/
|
|
static int nvidia_read_card_info(nv_ioctl_card_info_t *ci, size_t num_entries)
|
|
{
|
|
nv_state_t *nv;
|
|
nv_linux_state_t *nvl;
|
|
size_t i = 0;
|
|
int rc = 0;
|
|
|
|
/* Clear each card's flags field the lazy way */
|
|
memset(ci, 0, num_entries * sizeof(ci[0]));
|
|
|
|
LOCK_NV_LINUX_DEVICES();
|
|
|
|
if (num_entries < num_nv_devices)
|
|
{
|
|
rc = -EINVAL;
|
|
goto out;
|
|
}
|
|
|
|
for (nvl = nv_linux_devices; nvl && i < num_entries; nvl = nvl->next)
|
|
{
|
|
nv = NV_STATE_PTR(nvl);
|
|
|
|
/* We do not include excluded GPUs in the list... */
|
|
if ((nv->flags & NV_FLAG_EXCLUDE) != 0)
|
|
continue;
|
|
|
|
ci[i].valid = NV_TRUE;
|
|
ci[i].pci_info.domain = nv->pci_info.domain;
|
|
ci[i].pci_info.bus = nv->pci_info.bus;
|
|
ci[i].pci_info.slot = nv->pci_info.slot;
|
|
ci[i].pci_info.vendor_id = nv->pci_info.vendor_id;
|
|
ci[i].pci_info.device_id = nv->pci_info.device_id;
|
|
ci[i].gpu_id = nv->gpu_id;
|
|
ci[i].interrupt_line = nv->interrupt_line;
|
|
ci[i].reg_address = nv->regs->cpu_address;
|
|
ci[i].reg_size = nv->regs->size;
|
|
ci[i].minor_number = nvl->minor_num;
|
|
if (nv_dev_is_pci(nvl->dev))
|
|
{
|
|
ci[i].fb_address = nv->fb->cpu_address;
|
|
ci[i].fb_size = nv->fb->size;
|
|
}
|
|
i++;
|
|
}
|
|
|
|
out:
|
|
UNLOCK_NV_LINUX_DEVICES();
|
|
return rc;
|
|
}
|
|
|
|
int
|
|
nvidia_ioctl(
|
|
struct inode *inode,
|
|
struct file *file,
|
|
unsigned int cmd,
|
|
unsigned long i_arg)
|
|
{
|
|
NV_STATUS rmStatus;
|
|
int status = 0;
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_FILEP(file);
|
|
nv_state_t *nv = NV_STATE_PTR(nvl);
|
|
nv_linux_file_private_t *nvlfp = NV_GET_LINUX_FILE_PRIVATE(file);
|
|
nvidia_stack_t *sp = NULL;
|
|
nv_ioctl_xfer_t ioc_xfer;
|
|
void *arg_ptr = (void *) i_arg;
|
|
void *arg_copy = NULL;
|
|
size_t arg_size = 0;
|
|
int arg_cmd;
|
|
|
|
nv_printf(NV_DBG_INFO, "NVRM: ioctl(0x%x, 0x%x, 0x%x)\n",
|
|
_IOC_NR(cmd), (unsigned int) i_arg, _IOC_SIZE(cmd));
|
|
|
|
status = nv_down_read_interruptible(&nv_system_pm_lock);
|
|
if (status < 0)
|
|
return status;
|
|
|
|
down(&nvlfp->fops_sp_lock[NV_FOPS_STACK_INDEX_IOCTL]);
|
|
sp = nvlfp->fops_sp[NV_FOPS_STACK_INDEX_IOCTL];
|
|
|
|
rmStatus = nv_check_gpu_state(nv);
|
|
if (rmStatus == NV_ERR_GPU_IS_LOST)
|
|
{
|
|
nv_printf(NV_DBG_INFO, "NVRM: GPU is lost, skipping nvidia_ioctl\n");
|
|
status = -EINVAL;
|
|
goto done;
|
|
}
|
|
|
|
arg_size = _IOC_SIZE(cmd);
|
|
arg_cmd = _IOC_NR(cmd);
|
|
|
|
if (arg_cmd == NV_ESC_IOCTL_XFER_CMD)
|
|
{
|
|
if (arg_size != sizeof(nv_ioctl_xfer_t))
|
|
{
|
|
nv_printf(NV_DBG_ERRORS,
|
|
"NVRM: invalid ioctl XFER structure size!\n");
|
|
status = -EINVAL;
|
|
goto done;
|
|
}
|
|
|
|
if (NV_COPY_FROM_USER(&ioc_xfer, arg_ptr, sizeof(ioc_xfer)))
|
|
{
|
|
nv_printf(NV_DBG_ERRORS,
|
|
"NVRM: failed to copy in ioctl XFER data!\n");
|
|
status = -EFAULT;
|
|
goto done;
|
|
}
|
|
|
|
arg_cmd = ioc_xfer.cmd;
|
|
arg_size = ioc_xfer.size;
|
|
arg_ptr = NvP64_VALUE(ioc_xfer.ptr);
|
|
|
|
if (arg_size > NV_ABSOLUTE_MAX_IOCTL_SIZE)
|
|
{
|
|
nv_printf(NV_DBG_ERRORS, "NVRM: invalid ioctl XFER size!\n");
|
|
status = -EINVAL;
|
|
goto done;
|
|
}
|
|
}
|
|
|
|
NV_KMALLOC(arg_copy, arg_size);
|
|
if (arg_copy == NULL)
|
|
{
|
|
nv_printf(NV_DBG_ERRORS, "NVRM: failed to allocate ioctl memory\n");
|
|
status = -ENOMEM;
|
|
goto done;
|
|
}
|
|
|
|
if (NV_COPY_FROM_USER(arg_copy, arg_ptr, arg_size))
|
|
{
|
|
nv_printf(NV_DBG_ERRORS, "NVRM: failed to copy in ioctl data!\n");
|
|
status = -EFAULT;
|
|
goto done;
|
|
}
|
|
|
|
switch (arg_cmd)
|
|
{
|
|
case NV_ESC_QUERY_DEVICE_INTR:
|
|
{
|
|
nv_ioctl_query_device_intr *query_intr = arg_copy;
|
|
|
|
NV_ACTUAL_DEVICE_ONLY(nv);
|
|
|
|
if ((arg_size < sizeof(*query_intr)) ||
|
|
(!nv->regs->map))
|
|
{
|
|
status = -EINVAL;
|
|
goto done;
|
|
}
|
|
|
|
query_intr->intrStatus =
|
|
*(nv->regs->map + (NV_RM_DEVICE_INTR_ADDRESS >> 2));
|
|
query_intr->status = NV_OK;
|
|
break;
|
|
}
|
|
|
|
/* pass out info about the card */
|
|
case NV_ESC_CARD_INFO:
|
|
{
|
|
size_t num_arg_devices = arg_size / sizeof(nv_ioctl_card_info_t);
|
|
|
|
NV_CTL_DEVICE_ONLY(nv);
|
|
|
|
status = nvidia_read_card_info(arg_copy, num_arg_devices);
|
|
break;
|
|
}
|
|
|
|
case NV_ESC_ATTACH_GPUS_TO_FD:
|
|
{
|
|
size_t num_arg_gpus = arg_size / sizeof(NvU32);
|
|
size_t i;
|
|
|
|
NV_CTL_DEVICE_ONLY(nv);
|
|
|
|
if (num_arg_gpus == 0 || nvlfp->num_attached_gpus != 0 ||
|
|
arg_size % sizeof(NvU32) != 0)
|
|
{
|
|
status = -EINVAL;
|
|
goto done;
|
|
}
|
|
|
|
NV_KMALLOC(nvlfp->attached_gpus, arg_size);
|
|
if (nvlfp->attached_gpus == NULL)
|
|
{
|
|
status = -ENOMEM;
|
|
goto done;
|
|
}
|
|
memcpy(nvlfp->attached_gpus, arg_copy, arg_size);
|
|
nvlfp->num_attached_gpus = num_arg_gpus;
|
|
|
|
for (i = 0; i < nvlfp->num_attached_gpus; i++)
|
|
{
|
|
if (nvlfp->attached_gpus[i] == 0)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (nvidia_dev_get(nvlfp->attached_gpus[i], sp))
|
|
{
|
|
while (i--)
|
|
{
|
|
if (nvlfp->attached_gpus[i] != 0)
|
|
nvidia_dev_put(nvlfp->attached_gpus[i], sp);
|
|
}
|
|
NV_KFREE(nvlfp->attached_gpus, arg_size);
|
|
nvlfp->num_attached_gpus = 0;
|
|
|
|
status = -EINVAL;
|
|
break;
|
|
}
|
|
}
|
|
|
|
break;
|
|
}
|
|
|
|
case NV_ESC_CHECK_VERSION_STR:
|
|
{
|
|
NV_CTL_DEVICE_ONLY(nv);
|
|
|
|
rmStatus = rm_perform_version_check(sp, arg_copy, arg_size);
|
|
status = ((rmStatus == NV_OK) ? 0 : -EINVAL);
|
|
break;
|
|
}
|
|
|
|
case NV_ESC_SYS_PARAMS:
|
|
{
|
|
nv_ioctl_sys_params_t *api = arg_copy;
|
|
|
|
NV_CTL_DEVICE_ONLY(nv);
|
|
|
|
if (arg_size != sizeof(nv_ioctl_sys_params_t))
|
|
{
|
|
status = -EINVAL;
|
|
goto done;
|
|
}
|
|
|
|
/* numa_memblock_size should only be set once */
|
|
if (nvl->numa_memblock_size == 0)
|
|
{
|
|
nvl->numa_memblock_size = api->memblock_size;
|
|
}
|
|
else
|
|
{
|
|
status = (nvl->numa_memblock_size == api->memblock_size) ?
|
|
0 : -EBUSY;
|
|
goto done;
|
|
}
|
|
break;
|
|
}
|
|
|
|
case NV_ESC_NUMA_INFO:
|
|
{
|
|
nv_ioctl_numa_info_t *api = arg_copy;
|
|
rmStatus = NV_OK;
|
|
|
|
NV_ACTUAL_DEVICE_ONLY(nv);
|
|
|
|
if (arg_size != sizeof(nv_ioctl_numa_info_t))
|
|
{
|
|
status = -EINVAL;
|
|
goto done;
|
|
}
|
|
|
|
api->offline_addresses.numEntries =
|
|
ARRAY_SIZE(api->offline_addresses.addresses),
|
|
|
|
rmStatus = rm_get_gpu_numa_info(sp, nv,
|
|
&(api->nid),
|
|
&(api->numa_mem_addr),
|
|
&(api->numa_mem_size),
|
|
(api->offline_addresses.addresses),
|
|
&(api->offline_addresses.numEntries));
|
|
if (rmStatus != NV_OK)
|
|
{
|
|
status = -EBUSY;
|
|
goto done;
|
|
}
|
|
|
|
api->status = nv_get_numa_status(nvl);
|
|
api->memblock_size = nv_ctl_device.numa_memblock_size;
|
|
break;
|
|
}
|
|
|
|
case NV_ESC_SET_NUMA_STATUS:
|
|
{
|
|
nv_ioctl_set_numa_status_t *api = arg_copy;
|
|
rmStatus = NV_OK;
|
|
|
|
if (!NV_IS_SUSER())
|
|
{
|
|
status = -EACCES;
|
|
goto done;
|
|
}
|
|
|
|
NV_ACTUAL_DEVICE_ONLY(nv);
|
|
|
|
if (arg_size != sizeof(nv_ioctl_set_numa_status_t))
|
|
{
|
|
status = -EINVAL;
|
|
goto done;
|
|
}
|
|
|
|
/*
|
|
* The nv_linux_state_t for the device needs to be locked
|
|
* in order to prevent additional open()/close() calls from
|
|
* manipulating the usage count for the device while we
|
|
* determine if NUMA state can be changed.
|
|
*/
|
|
down(&nvl->ldata_lock);
|
|
|
|
if (nv_get_numa_status(nvl) != api->status)
|
|
{
|
|
if (api->status == NV_IOCTL_NUMA_STATUS_OFFLINE_IN_PROGRESS)
|
|
{
|
|
/*
|
|
* Only the current client should have an open file
|
|
* descriptor for the device, to allow safe offlining.
|
|
*/
|
|
if (NV_ATOMIC_READ(nvl->usage_count) > 1)
|
|
{
|
|
status = -EBUSY;
|
|
goto unlock;
|
|
}
|
|
else
|
|
{
|
|
/*
|
|
* If this call fails, it indicates that RM
|
|
* is not ready to offline memory, and we should keep
|
|
* the current NUMA status of ONLINE.
|
|
*/
|
|
rmStatus = rm_gpu_numa_offline(sp, nv);
|
|
if (rmStatus != NV_OK)
|
|
{
|
|
status = -EBUSY;
|
|
goto unlock;
|
|
}
|
|
}
|
|
}
|
|
|
|
status = nv_set_numa_status(nvl, api->status);
|
|
if (status < 0)
|
|
{
|
|
if (api->status == NV_IOCTL_NUMA_STATUS_OFFLINE_IN_PROGRESS)
|
|
(void) rm_gpu_numa_online(sp, nv);
|
|
goto unlock;
|
|
}
|
|
|
|
if (api->status == NV_IOCTL_NUMA_STATUS_ONLINE)
|
|
{
|
|
rmStatus = rm_gpu_numa_online(sp, nv);
|
|
if (rmStatus != NV_OK)
|
|
{
|
|
status = -EBUSY;
|
|
goto unlock;
|
|
}
|
|
}
|
|
}
|
|
|
|
unlock:
|
|
up(&nvl->ldata_lock);
|
|
|
|
break;
|
|
}
|
|
|
|
case NV_ESC_EXPORT_TO_DMABUF_FD:
|
|
{
|
|
nv_ioctl_export_to_dma_buf_fd_t *params = arg_copy;
|
|
|
|
if (arg_size != sizeof(nv_ioctl_export_to_dma_buf_fd_t))
|
|
{
|
|
status = -EINVAL;
|
|
goto done;
|
|
}
|
|
|
|
NV_ACTUAL_DEVICE_ONLY(nv);
|
|
|
|
params->status = nv_dma_buf_export(nv, params);
|
|
|
|
break;
|
|
}
|
|
|
|
default:
|
|
rmStatus = rm_ioctl(sp, nv, &nvlfp->nvfp, arg_cmd, arg_copy, arg_size);
|
|
status = ((rmStatus == NV_OK) ? 0 : -EINVAL);
|
|
break;
|
|
}
|
|
|
|
done:
|
|
up(&nvlfp->fops_sp_lock[NV_FOPS_STACK_INDEX_IOCTL]);
|
|
|
|
up_read(&nv_system_pm_lock);
|
|
|
|
if (arg_copy != NULL)
|
|
{
|
|
if (status != -EFAULT)
|
|
{
|
|
if (NV_COPY_TO_USER(arg_ptr, arg_copy, arg_size))
|
|
{
|
|
nv_printf(NV_DBG_ERRORS, "NVRM: failed to copy out ioctl data\n");
|
|
status = -EFAULT;
|
|
}
|
|
}
|
|
NV_KFREE(arg_copy, arg_size);
|
|
}
|
|
|
|
return status;
|
|
}
|
|
|
|
irqreturn_t
|
|
nvidia_isr_msix(
|
|
int irq,
|
|
void *arg
|
|
)
|
|
{
|
|
irqreturn_t ret;
|
|
nv_linux_state_t *nvl = (void *) arg;
|
|
|
|
// nvidia_isr_msix() is called for each of the MSI-X vectors and they can
|
|
// run in parallel on different CPUs (cores), but this is not currently
|
|
// supported by nvidia_isr() and its children. As a big hammer fix just
|
|
// spinlock around the nvidia_isr() call to serialize them.
|
|
//
|
|
// At this point interrupts are disabled on the CPU running our ISR (see
|
|
// comments for nv_default_irq_flags()) so a plain spinlock is enough.
|
|
NV_SPIN_LOCK(&nvl->msix_isr_lock);
|
|
|
|
ret = nvidia_isr(irq, arg);
|
|
|
|
NV_SPIN_UNLOCK(&nvl->msix_isr_lock);
|
|
|
|
return ret;
|
|
}
|
|
|
|
/*
|
|
* driver receives an interrupt
|
|
* if someone waiting, then hand it off.
|
|
*/
|
|
irqreturn_t
|
|
nvidia_isr(
|
|
int irq,
|
|
void *arg
|
|
)
|
|
{
|
|
nv_linux_state_t *nvl = (void *) arg;
|
|
nv_state_t *nv = NV_STATE_PTR(nvl);
|
|
NvU32 need_to_run_bottom_half_gpu_lock_held = 0;
|
|
NvBool rm_handled = NV_FALSE, uvm_handled = NV_FALSE, rm_fault_handling_needed = NV_FALSE;
|
|
NvU32 rm_serviceable_fault_cnt = 0;
|
|
NvU32 sec, usec;
|
|
NvU16 index = 0;
|
|
NvU64 currentTime = 0;
|
|
NvBool found_irq = NV_FALSE;
|
|
|
|
rm_gpu_copy_mmu_faults_unlocked(nvl->sp[NV_DEV_STACK_ISR], nv, &rm_serviceable_fault_cnt);
|
|
rm_fault_handling_needed = (rm_serviceable_fault_cnt != 0);
|
|
|
|
#if defined (NV_UVM_ENABLE)
|
|
//
|
|
// Returns NV_OK if the UVM driver handled the interrupt
|
|
//
|
|
// Returns NV_ERR_NO_INTR_PENDING if the interrupt is not for
|
|
// the UVM driver.
|
|
//
|
|
// Returns NV_WARN_MORE_PROCESSING_REQUIRED if the UVM top-half ISR was
|
|
// unable to get its lock(s), due to other (UVM) threads holding them.
|
|
//
|
|
// RM can normally treat NV_WARN_MORE_PROCESSING_REQUIRED the same as
|
|
// NV_ERR_NO_INTR_PENDING, but in some cases the extra information may
|
|
// be helpful.
|
|
//
|
|
if (nv_uvm_event_interrupt(nv_get_cached_uuid(nv)) == NV_OK)
|
|
uvm_handled = NV_TRUE;
|
|
#endif
|
|
|
|
rm_handled = rm_isr(nvl->sp[NV_DEV_STACK_ISR], nv,
|
|
&need_to_run_bottom_half_gpu_lock_held);
|
|
|
|
/* Replicating the logic in linux kernel to track unhandled interrupt crossing a threshold */
|
|
if ((nv->flags & NV_FLAG_USES_MSI) || (nv->flags & NV_FLAG_USES_MSIX))
|
|
{
|
|
if (nvl->irq_count != NULL)
|
|
{
|
|
for (index = 0; index < nvl->current_num_irq_tracked; index++)
|
|
{
|
|
if (nvl->irq_count[index].irq == irq)
|
|
{
|
|
found_irq = NV_TRUE;
|
|
break;
|
|
}
|
|
|
|
found_irq = NV_FALSE;
|
|
}
|
|
|
|
if (!found_irq && nvl->current_num_irq_tracked < nvl->num_intr)
|
|
{
|
|
index = nvl->current_num_irq_tracked;
|
|
nvl->irq_count[index].irq = irq;
|
|
nvl->current_num_irq_tracked++;
|
|
found_irq = NV_TRUE;
|
|
}
|
|
|
|
if (found_irq)
|
|
{
|
|
nvl->irq_count[index].total++;
|
|
|
|
if(rm_handled == NV_FALSE)
|
|
{
|
|
os_get_current_time(&sec, &usec);
|
|
currentTime = ((NvU64)sec) * 1000000 + (NvU64)usec;
|
|
|
|
/* Reset unhandled count if it's been more than 0.1 seconds since the last unhandled IRQ */
|
|
if ((currentTime - nvl->irq_count[index].last_unhandled) > RM_UNHANDLED_TIMEOUT_US)
|
|
nvl->irq_count[index].unhandled = 1;
|
|
else
|
|
nvl->irq_count[index].unhandled++;
|
|
|
|
nvl->irq_count[index].last_unhandled = currentTime;
|
|
rm_handled = NV_TRUE;
|
|
}
|
|
|
|
if (nvl->irq_count[index].total >= RM_THRESHOLD_TOTAL_IRQ_COUNT)
|
|
{
|
|
if (nvl->irq_count[index].unhandled > RM_THRESHOLD_UNAHNDLED_IRQ_COUNT)
|
|
nv_printf(NV_DBG_ERRORS,"NVRM: Going over RM unhandled interrupt threshold for irq %d\n", irq);
|
|
|
|
nvl->irq_count[index].total = 0;
|
|
nvl->irq_count[index].unhandled = 0;
|
|
nvl->irq_count[index].last_unhandled = 0;
|
|
}
|
|
}
|
|
else
|
|
nv_printf(NV_DBG_ERRORS,"NVRM: IRQ number out of valid range\n");
|
|
}
|
|
}
|
|
|
|
if (need_to_run_bottom_half_gpu_lock_held)
|
|
{
|
|
return IRQ_WAKE_THREAD;
|
|
}
|
|
else
|
|
{
|
|
//
|
|
// If rm_isr does not need to run a bottom half and mmu_faults_copied
|
|
// indicates that bottom half is needed, then we enqueue a kthread based
|
|
// bottom half, as this specific bottom_half will acquire the GPU lock
|
|
//
|
|
if (rm_fault_handling_needed)
|
|
nv_kthread_q_schedule_q_item(&nvl->bottom_half_q, &nvl->bottom_half_q_item);
|
|
}
|
|
|
|
return IRQ_RETVAL(rm_handled || uvm_handled || rm_fault_handling_needed);
|
|
}
|
|
|
|
irqreturn_t
|
|
nvidia_isr_kthread_bh(
|
|
int irq,
|
|
void *data
|
|
)
|
|
{
|
|
return nvidia_isr_common_bh(data);
|
|
}
|
|
|
|
irqreturn_t
|
|
nvidia_isr_msix_kthread_bh(
|
|
int irq,
|
|
void *data
|
|
)
|
|
{
|
|
NV_STATUS status;
|
|
irqreturn_t ret;
|
|
nv_state_t *nv = (nv_state_t *) data;
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
|
|
//
|
|
// Synchronize kthreads servicing bottom halves for different MSI-X vectors
|
|
// as they share same pre-allocated alt-stack.
|
|
//
|
|
status = os_acquire_mutex(nvl->msix_bh_mutex);
|
|
// os_acquire_mutex can only fail if we cannot sleep and we can
|
|
WARN_ON(status != NV_OK);
|
|
|
|
ret = nvidia_isr_common_bh(data);
|
|
|
|
os_release_mutex(nvl->msix_bh_mutex);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static irqreturn_t
|
|
nvidia_isr_common_bh(
|
|
void *data
|
|
)
|
|
{
|
|
nv_state_t *nv = (nv_state_t *) data;
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
nvidia_stack_t *sp = nvl->sp[NV_DEV_STACK_ISR_BH];
|
|
NV_STATUS status;
|
|
|
|
status = nv_check_gpu_state(nv);
|
|
if (status == NV_ERR_GPU_IS_LOST)
|
|
{
|
|
nv_printf(NV_DBG_INFO, "NVRM: GPU is lost, skipping ISR bottom half\n");
|
|
}
|
|
else
|
|
{
|
|
rm_isr_bh(sp, nv);
|
|
}
|
|
|
|
return IRQ_HANDLED;
|
|
}
|
|
|
|
static void
|
|
nvidia_isr_bh_unlocked(
|
|
void * args
|
|
)
|
|
{
|
|
nv_state_t *nv = (nv_state_t *) args;
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
nvidia_stack_t *sp;
|
|
NV_STATUS status;
|
|
|
|
//
|
|
// Synchronize kthreads servicing unlocked bottom half as they
|
|
// share same pre-allocated stack for alt-stack
|
|
//
|
|
status = os_acquire_mutex(nvl->isr_bh_unlocked_mutex);
|
|
if (status != NV_OK)
|
|
{
|
|
nv_printf(NV_DBG_ERRORS, "NVRM: %s: Unable to take bottom_half mutex!\n",
|
|
__FUNCTION__);
|
|
WARN_ON(1);
|
|
}
|
|
|
|
sp = nvl->sp[NV_DEV_STACK_ISR_BH_UNLOCKED];
|
|
|
|
status = nv_check_gpu_state(nv);
|
|
if (status == NV_ERR_GPU_IS_LOST)
|
|
{
|
|
nv_printf(NV_DBG_INFO,
|
|
"NVRM: GPU is lost, skipping unlocked ISR bottom half\n");
|
|
}
|
|
else
|
|
{
|
|
rm_isr_bh_unlocked(sp, nv);
|
|
}
|
|
|
|
os_release_mutex(nvl->isr_bh_unlocked_mutex);
|
|
}
|
|
|
|
static void
|
|
nvidia_rc_timer_callback(
|
|
struct nv_timer *nv_timer
|
|
)
|
|
{
|
|
nv_linux_state_t *nvl = container_of(nv_timer, nv_linux_state_t, rc_timer);
|
|
nv_state_t *nv = NV_STATE_PTR(nvl);
|
|
nvidia_stack_t *sp = nvl->sp[NV_DEV_STACK_TIMER];
|
|
NV_STATUS status;
|
|
|
|
status = nv_check_gpu_state(nv);
|
|
if (status == NV_ERR_GPU_IS_LOST)
|
|
{
|
|
nv_printf(NV_DBG_INFO,
|
|
"NVRM: GPU is lost, skipping device timer callbacks\n");
|
|
return;
|
|
}
|
|
|
|
if (rm_run_rc_callback(sp, nv) == NV_OK)
|
|
{
|
|
// set another timeout 1 sec in the future:
|
|
mod_timer(&nvl->rc_timer.kernel_timer, jiffies + HZ);
|
|
}
|
|
}
|
|
|
|
/*
|
|
** nvidia_ctl_open
|
|
**
|
|
** nv control driver open entry point. Sessions are created here.
|
|
*/
|
|
static int
|
|
nvidia_ctl_open(
|
|
struct inode *inode,
|
|
struct file *file
|
|
)
|
|
{
|
|
nv_linux_state_t *nvl = &nv_ctl_device;
|
|
nv_state_t *nv = NV_STATE_PTR(nvl);
|
|
nv_linux_file_private_t *nvlfp = NV_GET_LINUX_FILE_PRIVATE(file);
|
|
|
|
nv_printf(NV_DBG_INFO, "NVRM: nvidia_ctl_open\n");
|
|
|
|
down(&nvl->ldata_lock);
|
|
|
|
/* save the nv away in file->private_data */
|
|
nvlfp->nvptr = nvl;
|
|
|
|
if (NV_ATOMIC_READ(nvl->usage_count) == 0)
|
|
{
|
|
nv->flags |= (NV_FLAG_OPEN | NV_FLAG_CONTROL);
|
|
}
|
|
|
|
NV_ATOMIC_INC(nvl->usage_count);
|
|
up(&nvl->ldata_lock);
|
|
|
|
return 0;
|
|
}
|
|
|
|
|
|
/*
|
|
** nvidia_ctl_close
|
|
*/
|
|
static int
|
|
nvidia_ctl_close(
|
|
struct inode *inode,
|
|
struct file *file
|
|
)
|
|
{
|
|
nv_alloc_t *at, *next;
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_FILEP(file);
|
|
nv_state_t *nv = NV_STATE_PTR(nvl);
|
|
nv_linux_file_private_t *nvlfp = NV_GET_LINUX_FILE_PRIVATE(file);
|
|
nvidia_stack_t *sp = nvlfp->sp;
|
|
unsigned int i;
|
|
|
|
nv_printf(NV_DBG_INFO, "NVRM: nvidia_ctl_close\n");
|
|
|
|
down(&nvl->ldata_lock);
|
|
if (NV_ATOMIC_DEC_AND_TEST(nvl->usage_count))
|
|
{
|
|
nv->flags &= ~NV_FLAG_OPEN;
|
|
}
|
|
up(&nvl->ldata_lock);
|
|
|
|
rm_cleanup_file_private(sp, nv, &nvlfp->nvfp);
|
|
|
|
if (nvlfp->free_list != NULL)
|
|
{
|
|
at = nvlfp->free_list;
|
|
while (at != NULL)
|
|
{
|
|
next = at->next;
|
|
if (at->pid == os_get_current_process())
|
|
NV_PRINT_AT(NV_DBG_MEMINFO, at);
|
|
nv_free_pages(nv, at->num_pages,
|
|
at->flags.contig,
|
|
at->cache_type,
|
|
(void *)at);
|
|
at = next;
|
|
}
|
|
}
|
|
|
|
if (nvlfp->num_attached_gpus != 0)
|
|
{
|
|
size_t i;
|
|
|
|
for (i = 0; i < nvlfp->num_attached_gpus; i++)
|
|
{
|
|
if (nvlfp->attached_gpus[i] != 0)
|
|
nvidia_dev_put(nvlfp->attached_gpus[i], sp);
|
|
}
|
|
|
|
NV_KFREE(nvlfp->attached_gpus, sizeof(NvU32) * nvlfp->num_attached_gpus);
|
|
nvlfp->num_attached_gpus = 0;
|
|
}
|
|
|
|
for (i = 0; i < NV_FOPS_STACK_INDEX_COUNT; ++i)
|
|
{
|
|
nv_kmem_cache_free_stack(nvlfp->fops_sp[i]);
|
|
}
|
|
|
|
nv_free_file_private(nvlfp);
|
|
NV_SET_FILE_PRIVATE(file, NULL);
|
|
|
|
nv_kmem_cache_free_stack(sp);
|
|
|
|
return 0;
|
|
}
|
|
|
|
|
|
void NV_API_CALL
|
|
nv_set_dma_address_size(
|
|
nv_state_t *nv,
|
|
NvU32 phys_addr_bits
|
|
)
|
|
{
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
NvU64 start_addr = nv_get_dma_start_address(nv);
|
|
NvU64 new_mask = (((NvU64)1) << phys_addr_bits) - 1;
|
|
|
|
nvl->dma_dev.addressable_range.limit = start_addr + new_mask;
|
|
|
|
/*
|
|
* The only scenario in which we definitely should not update the DMA mask
|
|
* is on POWER, when using TCE bypass mode (see nv_get_dma_start_address()
|
|
* for details), since the meaning of the DMA mask is overloaded in that
|
|
* case.
|
|
*/
|
|
if (!nvl->tce_bypass_enabled)
|
|
{
|
|
dma_set_mask(&nvl->pci_dev->dev, new_mask);
|
|
/* Certain kernels have a bug which causes pci_set_consistent_dma_mask
|
|
* to call GPL sme_active symbol, this bug has already been fixed in a
|
|
* minor release update but detect the failure scenario here to prevent
|
|
* an installation regression */
|
|
#if !NV_IS_EXPORT_SYMBOL_GPL_sme_active
|
|
dma_set_coherent_mask(&nvl->pci_dev->dev, new_mask);
|
|
#endif
|
|
}
|
|
}
|
|
|
|
static NvUPtr
|
|
nv_map_guest_pages(nv_alloc_t *at,
|
|
NvU64 address,
|
|
NvU32 page_count,
|
|
NvU32 page_idx)
|
|
{
|
|
struct page **pages;
|
|
NvU32 j;
|
|
NvUPtr virt_addr;
|
|
|
|
NV_KMALLOC(pages, sizeof(struct page *) * page_count);
|
|
if (pages == NULL)
|
|
{
|
|
nv_printf(NV_DBG_ERRORS,
|
|
"NVRM: failed to allocate vmap() page descriptor table!\n");
|
|
return 0;
|
|
}
|
|
|
|
for (j = 0; j < page_count; j++)
|
|
{
|
|
pages[j] = NV_GET_PAGE_STRUCT(at->page_table[page_idx+j]->phys_addr);
|
|
}
|
|
|
|
virt_addr = nv_vm_map_pages(pages, page_count,
|
|
at->cache_type == NV_MEMORY_CACHED, at->flags.unencrypted);
|
|
NV_KFREE(pages, sizeof(struct page *) * page_count);
|
|
|
|
return virt_addr;
|
|
}
|
|
|
|
NV_STATUS NV_API_CALL
|
|
nv_alias_pages(
|
|
nv_state_t *nv,
|
|
NvU32 page_cnt,
|
|
NvU32 contiguous,
|
|
NvU32 cache_type,
|
|
NvU64 guest_id,
|
|
NvU64 *pte_array,
|
|
void **priv_data
|
|
)
|
|
{
|
|
nv_alloc_t *at;
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
NvU32 i=0;
|
|
nvidia_pte_t *page_ptr = NULL;
|
|
|
|
at = nvos_create_alloc(nvl->dev, page_cnt);
|
|
|
|
if (at == NULL)
|
|
{
|
|
return NV_ERR_NO_MEMORY;
|
|
}
|
|
|
|
at->cache_type = cache_type;
|
|
if (contiguous)
|
|
at->flags.contig = NV_TRUE;
|
|
#if defined(NVCPU_AARCH64)
|
|
if (at->cache_type != NV_MEMORY_CACHED)
|
|
at->flags.aliased = NV_TRUE;
|
|
#endif
|
|
|
|
at->flags.guest = NV_TRUE;
|
|
|
|
at->order = get_order(at->num_pages * PAGE_SIZE);
|
|
|
|
for (i=0; i < at->num_pages; ++i)
|
|
{
|
|
page_ptr = at->page_table[i];
|
|
|
|
if (contiguous && i>0)
|
|
{
|
|
page_ptr->dma_addr = pte_array[0] + (i << PAGE_SHIFT);
|
|
}
|
|
else
|
|
{
|
|
page_ptr->dma_addr = pte_array[i];
|
|
}
|
|
|
|
page_ptr->phys_addr = page_ptr->dma_addr;
|
|
|
|
/* aliased pages will be mapped on demand. */
|
|
page_ptr->virt_addr = 0x0;
|
|
}
|
|
|
|
at->guest_id = guest_id;
|
|
*priv_data = at;
|
|
NV_ATOMIC_INC(at->usage_count);
|
|
|
|
NV_PRINT_AT(NV_DBG_MEMINFO, at);
|
|
|
|
return NV_OK;
|
|
}
|
|
|
|
/*
|
|
* This creates a dummy nv_alloc_t for peer IO mem, so that it can
|
|
* be mapped using NvRmMapMemory.
|
|
*/
|
|
NV_STATUS NV_API_CALL nv_register_peer_io_mem(
|
|
nv_state_t *nv,
|
|
NvU64 *phys_addr,
|
|
NvU64 page_count,
|
|
void **priv_data
|
|
)
|
|
{
|
|
nv_alloc_t *at;
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
NvU64 i;
|
|
NvU64 addr;
|
|
|
|
at = nvos_create_alloc(nvl->dev, page_count);
|
|
|
|
if (at == NULL)
|
|
return NV_ERR_NO_MEMORY;
|
|
|
|
// IO regions should be uncached and contiguous
|
|
at->cache_type = NV_MEMORY_UNCACHED;
|
|
at->flags.contig = NV_TRUE;
|
|
#if defined(NVCPU_AARCH64)
|
|
at->flags.aliased = NV_TRUE;
|
|
#endif
|
|
at->flags.peer_io = NV_TRUE;
|
|
|
|
at->order = get_order(at->num_pages * PAGE_SIZE);
|
|
|
|
addr = phys_addr[0];
|
|
|
|
for (i = 0; i < page_count; i++)
|
|
{
|
|
at->page_table[i]->phys_addr = addr;
|
|
addr += PAGE_SIZE;
|
|
}
|
|
|
|
// No struct page array exists for this memory.
|
|
at->user_pages = NULL;
|
|
|
|
*priv_data = at;
|
|
|
|
NV_PRINT_AT(NV_DBG_MEMINFO, at);
|
|
|
|
return NV_OK;
|
|
}
|
|
|
|
void NV_API_CALL nv_unregister_peer_io_mem(
|
|
nv_state_t *nv,
|
|
void *priv_data
|
|
)
|
|
{
|
|
nv_alloc_t *at = priv_data;
|
|
|
|
NV_PRINT_AT(NV_DBG_MEMINFO, at);
|
|
|
|
nvos_free_alloc(at);
|
|
}
|
|
|
|
/*
|
|
* By registering user pages, we create a dummy nv_alloc_t for it, so that the
|
|
* rest of the RM can treat it like any other alloc.
|
|
*
|
|
* This also converts the page array to an array of physical addresses.
|
|
*/
|
|
NV_STATUS NV_API_CALL nv_register_user_pages(
|
|
nv_state_t *nv,
|
|
NvU64 page_count,
|
|
NvU64 *phys_addr,
|
|
void *import_priv,
|
|
void **priv_data
|
|
)
|
|
{
|
|
nv_alloc_t *at;
|
|
NvU64 i;
|
|
struct page **user_pages;
|
|
nv_linux_state_t *nvl;
|
|
nvidia_pte_t *page_ptr;
|
|
|
|
nv_printf(NV_DBG_MEMINFO, "NVRM: VM: nv_register_user_pages: 0x%x\n", page_count);
|
|
user_pages = *priv_data;
|
|
nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
|
|
at = nvos_create_alloc(nvl->dev, page_count);
|
|
|
|
if (at == NULL)
|
|
{
|
|
return NV_ERR_NO_MEMORY;
|
|
}
|
|
|
|
/*
|
|
* Anonymous memory currently must be write-back cacheable, and we can't
|
|
* enforce contiguity.
|
|
*/
|
|
at->cache_type = NV_MEMORY_UNCACHED;
|
|
#if defined(NVCPU_AARCH64)
|
|
at->flags.aliased = NV_TRUE;
|
|
#endif
|
|
|
|
at->flags.user = NV_TRUE;
|
|
|
|
at->order = get_order(at->num_pages * PAGE_SIZE);
|
|
|
|
for (i = 0; i < page_count; i++)
|
|
{
|
|
/*
|
|
* We only assign the physical address and not the DMA address, since
|
|
* this allocation hasn't been DMA-mapped yet.
|
|
*/
|
|
page_ptr = at->page_table[i];
|
|
page_ptr->phys_addr = page_to_phys(user_pages[i]);
|
|
|
|
phys_addr[i] = page_ptr->phys_addr;
|
|
}
|
|
|
|
/* Save off the user pages array to be restored later */
|
|
at->user_pages = user_pages;
|
|
|
|
/* Save off the import private data to be returned later */
|
|
if (import_priv != NULL)
|
|
{
|
|
at->import_priv = import_priv;
|
|
}
|
|
|
|
*priv_data = at;
|
|
|
|
NV_PRINT_AT(NV_DBG_MEMINFO, at);
|
|
|
|
return NV_OK;
|
|
}
|
|
|
|
void NV_API_CALL nv_unregister_user_pages(
|
|
nv_state_t *nv,
|
|
NvU64 page_count,
|
|
void **import_priv,
|
|
void **priv_data
|
|
)
|
|
{
|
|
nv_alloc_t *at = *priv_data;
|
|
|
|
nv_printf(NV_DBG_MEMINFO, "NVRM: VM: nv_unregister_user_pages: 0x%x\n", page_count);
|
|
|
|
NV_PRINT_AT(NV_DBG_MEMINFO, at);
|
|
|
|
WARN_ON(!at->flags.user);
|
|
|
|
/* Restore the user pages array for the caller to handle */
|
|
*priv_data = at->user_pages;
|
|
|
|
/* Return the import private data for the caller to handle */
|
|
if (import_priv != NULL)
|
|
{
|
|
*import_priv = at->import_priv;
|
|
}
|
|
|
|
nvos_free_alloc(at);
|
|
}
|
|
|
|
/*
|
|
* This creates a dummy nv_alloc_t for existing physical allocations, so
|
|
* that it can be mapped using NvRmMapMemory and BAR2 code path.
|
|
*/
|
|
NV_STATUS NV_API_CALL nv_register_phys_pages(
|
|
nv_state_t *nv,
|
|
NvU64 *phys_addr,
|
|
NvU64 page_count,
|
|
NvU32 cache_type,
|
|
void **priv_data
|
|
)
|
|
{
|
|
nv_alloc_t *at;
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
NvU64 i;
|
|
NvU64 addr;
|
|
|
|
at = nvos_create_alloc(nvl->dev, page_count);
|
|
|
|
if (at == NULL)
|
|
return NV_ERR_NO_MEMORY;
|
|
/*
|
|
* Setting memory flags to cacheable and discontiguous.
|
|
*/
|
|
at->cache_type = cache_type;
|
|
|
|
/*
|
|
* Only physical address is available so we don't try to reuse existing
|
|
* mappings
|
|
*/
|
|
at->flags.physical = NV_TRUE;
|
|
|
|
at->order = get_order(at->num_pages * PAGE_SIZE);
|
|
|
|
for (i = 0, addr = phys_addr[0]; i < page_count; addr = phys_addr[++i])
|
|
{
|
|
at->page_table[i]->phys_addr = addr;
|
|
}
|
|
|
|
at->user_pages = NULL;
|
|
*priv_data = at;
|
|
|
|
NV_PRINT_AT(NV_DBG_MEMINFO, at);
|
|
|
|
return NV_OK;
|
|
}
|
|
|
|
NV_STATUS NV_API_CALL nv_register_sgt(
|
|
nv_state_t *nv,
|
|
NvU64 *phys_addr,
|
|
NvU64 page_count,
|
|
NvU32 cache_type,
|
|
void **priv_data,
|
|
struct sg_table *import_sgt,
|
|
void *import_priv
|
|
)
|
|
{
|
|
nv_alloc_t *at;
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
|
|
unsigned int i, j = 0;
|
|
NvU64 sg_addr, sg_off, sg_len;
|
|
struct scatterlist *sg;
|
|
|
|
at = nvos_create_alloc(nvl->dev, page_count);
|
|
|
|
if (at == NULL)
|
|
return NV_ERR_NO_MEMORY;
|
|
|
|
/* Populate phys addrs with DMA addrs from SGT */
|
|
for_each_sg(import_sgt->sgl, sg, import_sgt->nents, i)
|
|
{
|
|
/*
|
|
* It is possible for dma_map_sg() to merge scatterlist entries, so
|
|
* make sure we account for that here.
|
|
*/
|
|
for (sg_addr = sg_dma_address(sg), sg_len = sg_dma_len(sg), sg_off = 0;
|
|
(sg_off < sg_len) && (j < page_count);
|
|
sg_off += PAGE_SIZE, j++)
|
|
{
|
|
phys_addr[j] = sg_addr + sg_off;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Setting memory flags to cacheable and discontiguous.
|
|
*/
|
|
at->cache_type = cache_type;
|
|
|
|
at->import_sgt = import_sgt;
|
|
|
|
/* Save off the import private data to be returned later */
|
|
if (import_priv != NULL)
|
|
{
|
|
at->import_priv = import_priv;
|
|
}
|
|
|
|
at->order = get_order(at->num_pages * PAGE_SIZE);
|
|
|
|
*priv_data = at;
|
|
|
|
NV_PRINT_AT(NV_DBG_MEMINFO, at);
|
|
|
|
return NV_OK;
|
|
}
|
|
|
|
void NV_API_CALL nv_unregister_sgt(
|
|
nv_state_t *nv,
|
|
struct sg_table **import_sgt,
|
|
void **import_priv,
|
|
void *priv_data
|
|
)
|
|
{
|
|
nv_alloc_t *at = priv_data;
|
|
|
|
nv_printf(NV_DBG_MEMINFO, "NVRM: VM: nv_unregister_sgt\n");
|
|
|
|
NV_PRINT_AT(NV_DBG_MEMINFO, at);
|
|
|
|
/* Restore the imported SGT for the caller to handle */
|
|
*import_sgt = at->import_sgt;
|
|
|
|
/* Return the import private data for the caller to handle */
|
|
if (import_priv != NULL)
|
|
{
|
|
*import_priv = at->import_priv;
|
|
}
|
|
|
|
nvos_free_alloc(at);
|
|
}
|
|
|
|
void NV_API_CALL nv_unregister_phys_pages(
|
|
nv_state_t *nv,
|
|
void *priv_data
|
|
)
|
|
{
|
|
nv_alloc_t *at = priv_data;
|
|
NV_PRINT_AT(NV_DBG_MEMINFO, at);
|
|
|
|
nvos_free_alloc(at);
|
|
}
|
|
|
|
NV_STATUS NV_API_CALL nv_get_num_phys_pages(
|
|
void *pAllocPrivate,
|
|
NvU32 *pNumPages
|
|
)
|
|
{
|
|
nv_alloc_t *at = pAllocPrivate;
|
|
|
|
if (!pNumPages) {
|
|
return NV_ERR_INVALID_ARGUMENT;
|
|
}
|
|
|
|
*pNumPages = at->num_pages;
|
|
|
|
return NV_OK;
|
|
}
|
|
|
|
NV_STATUS NV_API_CALL nv_get_phys_pages(
|
|
void *pAllocPrivate,
|
|
void *pPages,
|
|
NvU32 *pNumPages
|
|
)
|
|
{
|
|
nv_alloc_t *at = pAllocPrivate;
|
|
struct page **pages = (struct page **)pPages;
|
|
NvU32 page_count;
|
|
int i;
|
|
|
|
if (!pNumPages || !pPages) {
|
|
return NV_ERR_INVALID_ARGUMENT;
|
|
}
|
|
|
|
page_count = NV_MIN(*pNumPages, at->num_pages);
|
|
|
|
for (i = 0; i < page_count; i++) {
|
|
pages[i] = NV_GET_PAGE_STRUCT(at->page_table[i]->phys_addr);
|
|
}
|
|
|
|
*pNumPages = page_count;
|
|
|
|
return NV_OK;
|
|
}
|
|
|
|
void* NV_API_CALL nv_alloc_kernel_mapping(
|
|
nv_state_t *nv,
|
|
void *pAllocPrivate,
|
|
NvU64 pageIndex,
|
|
NvU32 pageOffset,
|
|
NvU64 size,
|
|
void **pPrivate
|
|
)
|
|
{
|
|
nv_alloc_t *at = pAllocPrivate;
|
|
NvU32 j, page_count;
|
|
NvUPtr virt_addr;
|
|
struct page **pages;
|
|
NvBool isUserAllocatedMem;
|
|
|
|
//
|
|
// For User allocated memory (like ErrorNotifier's) which is NOT allocated
|
|
// nor owned by RM, the RM driver just stores the physical address
|
|
// corresponding to that memory and does not map it until required.
|
|
// In that case, in page tables the virt_addr == 0, so first we need to map
|
|
// those pages to obtain virtual address.
|
|
//
|
|
isUserAllocatedMem = at->flags.user &&
|
|
!at->page_table[pageIndex]->virt_addr &&
|
|
at->page_table[pageIndex]->phys_addr;
|
|
|
|
//
|
|
// User memory may NOT have kernel VA. So check this and fallback to else
|
|
// case to create one.
|
|
//
|
|
if (((size + pageOffset) <= PAGE_SIZE) &&
|
|
!at->flags.guest && !at->flags.aliased &&
|
|
!isUserAllocatedMem && !at->flags.physical)
|
|
{
|
|
*pPrivate = NULL;
|
|
return (void *)(at->page_table[pageIndex]->virt_addr + pageOffset);
|
|
}
|
|
else
|
|
{
|
|
size += pageOffset;
|
|
page_count = (size >> PAGE_SHIFT) + ((size & ~NV_PAGE_MASK) ? 1 : 0);
|
|
|
|
if (at->flags.guest)
|
|
{
|
|
virt_addr = nv_map_guest_pages(at,
|
|
nv->bars[NV_GPU_BAR_INDEX_REGS].cpu_address,
|
|
page_count, pageIndex);
|
|
}
|
|
else
|
|
{
|
|
NV_KMALLOC(pages, sizeof(struct page *) * page_count);
|
|
if (pages == NULL)
|
|
{
|
|
nv_printf(NV_DBG_ERRORS,
|
|
"NVRM: failed to allocate vmap() page descriptor table!\n");
|
|
return NULL;
|
|
}
|
|
|
|
for (j = 0; j < page_count; j++)
|
|
pages[j] = NV_GET_PAGE_STRUCT(at->page_table[pageIndex+j]->phys_addr);
|
|
|
|
virt_addr = nv_vm_map_pages(pages, page_count,
|
|
at->cache_type == NV_MEMORY_CACHED, at->flags.unencrypted);
|
|
NV_KFREE(pages, sizeof(struct page *) * page_count);
|
|
}
|
|
|
|
if (virt_addr == 0)
|
|
{
|
|
nv_printf(NV_DBG_ERRORS, "NVRM: failed to map pages!\n");
|
|
return NULL;
|
|
}
|
|
|
|
*pPrivate = (void *)(NvUPtr)page_count;
|
|
return (void *)(virt_addr + pageOffset);
|
|
}
|
|
|
|
return NULL;
|
|
}
|
|
|
|
NV_STATUS NV_API_CALL nv_free_kernel_mapping(
|
|
nv_state_t *nv,
|
|
void *pAllocPrivate,
|
|
void *address,
|
|
void *pPrivate
|
|
)
|
|
{
|
|
nv_alloc_t *at = pAllocPrivate;
|
|
NvUPtr virt_addr;
|
|
NvU32 page_count;
|
|
|
|
virt_addr = ((NvUPtr)address & NV_PAGE_MASK);
|
|
page_count = (NvUPtr)pPrivate;
|
|
|
|
if (at->flags.guest)
|
|
{
|
|
nv_iounmap((void *)virt_addr, (page_count * PAGE_SIZE));
|
|
}
|
|
else if (pPrivate != NULL)
|
|
{
|
|
nv_vm_unmap_pages(virt_addr, page_count);
|
|
}
|
|
|
|
return NV_OK;
|
|
}
|
|
|
|
NV_STATUS NV_API_CALL nv_alloc_pages(
|
|
nv_state_t *nv,
|
|
NvU32 page_count,
|
|
NvBool contiguous,
|
|
NvU32 cache_type,
|
|
NvBool zeroed,
|
|
NvBool unencrypted,
|
|
NvU64 *pte_array,
|
|
void **priv_data
|
|
)
|
|
{
|
|
nv_alloc_t *at;
|
|
NV_STATUS status = NV_ERR_NO_MEMORY;
|
|
nv_linux_state_t *nvl = NULL;
|
|
NvBool will_remap = NV_FALSE;
|
|
NvU32 i;
|
|
struct device *dev = NULL;
|
|
|
|
nv_printf(NV_DBG_MEMINFO, "NVRM: VM: nv_alloc_pages: %d pages\n", page_count);
|
|
nv_printf(NV_DBG_MEMINFO, "NVRM: VM: contig %d cache_type %d\n",
|
|
contiguous, cache_type);
|
|
|
|
//
|
|
// system memory allocation can be associated with a client instead of a gpu
|
|
// handle the case where per device state is NULL
|
|
//
|
|
if(nv)
|
|
{
|
|
nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
will_remap = nv_requires_dma_remap(nv);
|
|
dev = nvl->dev;
|
|
}
|
|
|
|
if (nv_encode_caching(NULL, cache_type, NV_MEMORY_TYPE_SYSTEM))
|
|
return NV_ERR_NOT_SUPPORTED;
|
|
|
|
at = nvos_create_alloc(dev, page_count);
|
|
if (at == NULL)
|
|
return NV_ERR_NO_MEMORY;
|
|
|
|
at->cache_type = cache_type;
|
|
|
|
if (contiguous)
|
|
at->flags.contig = NV_TRUE;
|
|
if (zeroed)
|
|
at->flags.zeroed = NV_TRUE;
|
|
#if defined(NVCPU_AARCH64)
|
|
if (at->cache_type != NV_MEMORY_CACHED)
|
|
at->flags.aliased = NV_TRUE;
|
|
#endif
|
|
if (unencrypted)
|
|
at->flags.unencrypted = NV_TRUE;
|
|
|
|
#if defined(NVCPU_PPC64LE)
|
|
/*
|
|
* Starting on Power9 systems, DMA addresses for NVLink are no longer the
|
|
* same as used over PCIe. There is an address compression scheme required
|
|
* for NVLink ONLY which impacts the upper address bits of the DMA address.
|
|
*
|
|
* This divergence between PCIe and NVLink DMA mappings breaks assumptions
|
|
* in the driver where during initialization we allocate system memory
|
|
* for the GPU to access over PCIe before NVLink is trained -- and some of
|
|
* these mappings persist on the GPU. If these persistent mappings are not
|
|
* equivalent they will cause invalid DMA accesses from the GPU once we
|
|
* switch to NVLink.
|
|
*
|
|
* To work around this we limit all system memory allocations from the driver
|
|
* during the period before NVLink is enabled to be from NUMA node 0 (CPU 0)
|
|
* which has a CPU real address with the upper address bits (above bit 42)
|
|
* set to 0. Effectively making the PCIe and NVLink DMA mappings equivalent
|
|
* allowing persistent system memory mappings already programmed on the GPU
|
|
* to remain valid after NVLink is enabled.
|
|
*
|
|
* See Bug 1920398 for more details.
|
|
*/
|
|
if (nv && nvl->npu && !nvl->dma_dev.nvlink)
|
|
at->flags.node0 = NV_TRUE;
|
|
#endif
|
|
|
|
if (at->flags.contig)
|
|
status = nv_alloc_contig_pages(nv, at);
|
|
else
|
|
status = nv_alloc_system_pages(nv, at);
|
|
|
|
if (status != NV_OK)
|
|
goto failed;
|
|
|
|
for (i = 0; i < ((contiguous) ? 1 : page_count); i++)
|
|
{
|
|
/*
|
|
* The contents of the pte_array[] depend on whether or not this device
|
|
* requires DMA-remapping. If it does, it should be the phys addresses
|
|
* used by the DMA-remapping paths, otherwise it should be the actual
|
|
* address that the device should use for DMA (which, confusingly, may
|
|
* be different than the CPU physical address, due to a static DMA
|
|
* offset).
|
|
*/
|
|
if ((nv == NULL) || will_remap)
|
|
{
|
|
pte_array[i] = at->page_table[i]->phys_addr;
|
|
}
|
|
else
|
|
{
|
|
pte_array[i] = nv_phys_to_dma(dev,
|
|
at->page_table[i]->phys_addr);
|
|
}
|
|
}
|
|
|
|
*priv_data = at;
|
|
NV_ATOMIC_INC(at->usage_count);
|
|
|
|
NV_PRINT_AT(NV_DBG_MEMINFO, at);
|
|
|
|
return NV_OK;
|
|
|
|
failed:
|
|
nvos_free_alloc(at);
|
|
|
|
return status;
|
|
}
|
|
|
|
NV_STATUS NV_API_CALL nv_free_pages(
|
|
nv_state_t *nv,
|
|
NvU32 page_count,
|
|
NvBool contiguous,
|
|
NvU32 cache_type,
|
|
void *priv_data
|
|
)
|
|
{
|
|
NV_STATUS rmStatus = NV_OK;
|
|
nv_alloc_t *at = priv_data;
|
|
|
|
nv_printf(NV_DBG_MEMINFO, "NVRM: VM: nv_free_pages: 0x%x\n", page_count);
|
|
|
|
NV_PRINT_AT(NV_DBG_MEMINFO, at);
|
|
|
|
/*
|
|
* If the 'at' usage count doesn't drop to zero here, not all of
|
|
* the user mappings have been torn down in time - we can't
|
|
* safely free the memory. We report success back to the RM, but
|
|
* defer the actual free operation until later.
|
|
*
|
|
* This is described in greater detail in the comments above the
|
|
* nvidia_vma_(open|release)() callbacks in nv-mmap.c.
|
|
*/
|
|
if (!NV_ATOMIC_DEC_AND_TEST(at->usage_count))
|
|
return NV_OK;
|
|
|
|
if (!at->flags.guest)
|
|
{
|
|
if (at->flags.contig)
|
|
nv_free_contig_pages(at);
|
|
else
|
|
nv_free_system_pages(at);
|
|
}
|
|
|
|
nvos_free_alloc(at);
|
|
|
|
return rmStatus;
|
|
}
|
|
|
|
NvBool nv_lock_init_locks
|
|
(
|
|
nvidia_stack_t *sp,
|
|
nv_state_t *nv
|
|
)
|
|
{
|
|
nv_linux_state_t *nvl;
|
|
nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
|
|
NV_INIT_MUTEX(&nvl->ldata_lock);
|
|
NV_INIT_MUTEX(&nvl->mmap_lock);
|
|
|
|
NV_ATOMIC_SET(nvl->usage_count, 0);
|
|
|
|
if (!rm_init_event_locks(sp, nv))
|
|
return NV_FALSE;
|
|
|
|
return NV_TRUE;
|
|
}
|
|
|
|
void nv_lock_destroy_locks
|
|
(
|
|
nvidia_stack_t *sp,
|
|
nv_state_t *nv
|
|
)
|
|
{
|
|
rm_destroy_event_locks(sp, nv);
|
|
}
|
|
|
|
void NV_API_CALL nv_post_event(
|
|
nv_event_t *event,
|
|
NvHandle handle,
|
|
NvU32 index,
|
|
NvU32 info32,
|
|
NvU16 info16,
|
|
NvBool data_valid
|
|
)
|
|
{
|
|
nv_linux_file_private_t *nvlfp = nv_get_nvlfp_from_nvfp(event->nvfp);
|
|
unsigned long eflags;
|
|
nvidia_event_t *nvet;
|
|
|
|
NV_SPIN_LOCK_IRQSAVE(&nvlfp->fp_lock, eflags);
|
|
|
|
if (data_valid)
|
|
{
|
|
NV_KMALLOC_ATOMIC(nvet, sizeof(nvidia_event_t));
|
|
if (nvet == NULL)
|
|
{
|
|
NV_SPIN_UNLOCK_IRQRESTORE(&nvlfp->fp_lock, eflags);
|
|
return;
|
|
}
|
|
|
|
if (nvlfp->event_data_tail != NULL)
|
|
nvlfp->event_data_tail->next = nvet;
|
|
if (nvlfp->event_data_head == NULL)
|
|
nvlfp->event_data_head = nvet;
|
|
nvlfp->event_data_tail = nvet;
|
|
nvet->next = NULL;
|
|
|
|
nvet->event = *event;
|
|
nvet->event.hObject = handle;
|
|
nvet->event.index = index;
|
|
nvet->event.info32 = info32;
|
|
nvet->event.info16 = info16;
|
|
}
|
|
//
|
|
// 'event_pending' is interpreted by nvidia_poll() and nv_get_event() to
|
|
// mean that an event without data is pending. Therefore, only set it to
|
|
// true here if newly posted event is dataless.
|
|
//
|
|
else
|
|
{
|
|
nvlfp->dataless_event_pending = NV_TRUE;
|
|
}
|
|
|
|
NV_SPIN_UNLOCK_IRQRESTORE(&nvlfp->fp_lock, eflags);
|
|
|
|
wake_up_interruptible(&nvlfp->waitqueue);
|
|
}
|
|
|
|
NvBool NV_API_CALL nv_is_rm_firmware_active(
|
|
nv_state_t *nv
|
|
)
|
|
{
|
|
if (rm_firmware_active)
|
|
{
|
|
// "all" here means all GPUs
|
|
if (strcmp(rm_firmware_active, "all") == 0)
|
|
return NV_TRUE;
|
|
}
|
|
return NV_FALSE;
|
|
}
|
|
|
|
const char *nv_firmware_path(
|
|
nv_firmware_t fw_type
|
|
)
|
|
{
|
|
switch (fw_type)
|
|
{
|
|
case NV_FIRMWARE_GSP:
|
|
return NV_FIRMWARE_GSP_FILENAME;
|
|
case NV_FIRMWARE_GSP_LOG:
|
|
return NV_FIRMWARE_GSP_LOG_FILENAME;
|
|
}
|
|
return "";
|
|
}
|
|
|
|
const void* NV_API_CALL nv_get_firmware(
|
|
nv_state_t *nv,
|
|
nv_firmware_t fw_type,
|
|
const void **fw_buf,
|
|
NvU32 *fw_size
|
|
)
|
|
{
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
const struct firmware *fw;
|
|
|
|
// path is relative to /lib/firmware
|
|
// if this fails it will print an error to dmesg
|
|
if (request_firmware(&fw, nv_firmware_path(fw_type), nvl->dev) != 0)
|
|
return NULL;
|
|
|
|
*fw_size = fw->size;
|
|
*fw_buf = fw->data;
|
|
|
|
return fw;
|
|
}
|
|
|
|
void NV_API_CALL nv_put_firmware(
|
|
const void *fw_handle
|
|
)
|
|
{
|
|
release_firmware(fw_handle);
|
|
}
|
|
|
|
nv_file_private_t* NV_API_CALL nv_get_file_private(
|
|
NvS32 fd,
|
|
NvBool ctl,
|
|
void **os_private
|
|
)
|
|
{
|
|
struct file *filp = NULL;
|
|
nv_linux_file_private_t *nvlfp = NULL;
|
|
dev_t rdev = 0;
|
|
|
|
filp = fget(fd);
|
|
|
|
if (filp == NULL || !NV_FILE_INODE(filp))
|
|
{
|
|
goto fail;
|
|
}
|
|
|
|
rdev = (NV_FILE_INODE(filp))->i_rdev;
|
|
|
|
if (MAJOR(rdev) != NV_MAJOR_DEVICE_NUMBER)
|
|
{
|
|
goto fail;
|
|
}
|
|
|
|
if (ctl)
|
|
{
|
|
if (MINOR(rdev) != NV_CONTROL_DEVICE_MINOR)
|
|
goto fail;
|
|
}
|
|
else
|
|
{
|
|
NvBool found = NV_FALSE;
|
|
int i;
|
|
|
|
for (i = 0; i <= NV_FRONTEND_CONTROL_DEVICE_MINOR_MIN; i++)
|
|
{
|
|
if ((nv_minor_num_table[i] != NULL) && (MINOR(rdev) == i))
|
|
{
|
|
found = NV_TRUE;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!found)
|
|
goto fail;
|
|
}
|
|
|
|
nvlfp = NV_GET_LINUX_FILE_PRIVATE(filp);
|
|
|
|
*os_private = filp;
|
|
|
|
return &nvlfp->nvfp;
|
|
|
|
fail:
|
|
|
|
if (filp != NULL)
|
|
{
|
|
fput(filp);
|
|
}
|
|
|
|
return NULL;
|
|
}
|
|
|
|
void NV_API_CALL nv_put_file_private(
|
|
void *os_private
|
|
)
|
|
{
|
|
struct file *filp = os_private;
|
|
fput(filp);
|
|
}
|
|
|
|
int NV_API_CALL nv_get_event(
|
|
nv_file_private_t *nvfp,
|
|
nv_event_t *event,
|
|
NvU32 *pending
|
|
)
|
|
{
|
|
nv_linux_file_private_t *nvlfp = nv_get_nvlfp_from_nvfp(nvfp);
|
|
nvidia_event_t *nvet;
|
|
unsigned long eflags;
|
|
|
|
NV_SPIN_LOCK_IRQSAVE(&nvlfp->fp_lock, eflags);
|
|
|
|
nvet = nvlfp->event_data_head;
|
|
if (nvet == NULL)
|
|
{
|
|
NV_SPIN_UNLOCK_IRQRESTORE(&nvlfp->fp_lock, eflags);
|
|
return NV_ERR_GENERIC;
|
|
}
|
|
|
|
*event = nvet->event;
|
|
|
|
if (nvlfp->event_data_tail == nvet)
|
|
nvlfp->event_data_tail = NULL;
|
|
nvlfp->event_data_head = nvet->next;
|
|
|
|
*pending = (nvlfp->event_data_head != NULL);
|
|
|
|
NV_SPIN_UNLOCK_IRQRESTORE(&nvlfp->fp_lock, eflags);
|
|
|
|
NV_KFREE(nvet, sizeof(nvidia_event_t));
|
|
|
|
return NV_OK;
|
|
}
|
|
|
|
int NV_API_CALL nv_start_rc_timer(
|
|
nv_state_t *nv
|
|
)
|
|
{
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
|
|
if (nv->rc_timer_enabled)
|
|
return -1;
|
|
|
|
nv_printf(NV_DBG_INFO, "NVRM: initializing rc timer\n");
|
|
|
|
nv_timer_setup(&nvl->rc_timer, nvidia_rc_timer_callback);
|
|
|
|
nv->rc_timer_enabled = 1;
|
|
|
|
// set the timeout for 1 second in the future:
|
|
mod_timer(&nvl->rc_timer.kernel_timer, jiffies + HZ);
|
|
|
|
nv_printf(NV_DBG_INFO, "NVRM: rc timer initialized\n");
|
|
|
|
return 0;
|
|
}
|
|
|
|
int NV_API_CALL nv_stop_rc_timer(
|
|
nv_state_t *nv
|
|
)
|
|
{
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
|
|
if (!nv->rc_timer_enabled)
|
|
return -1;
|
|
|
|
nv_printf(NV_DBG_INFO, "NVRM: stopping rc timer\n");
|
|
nv->rc_timer_enabled = 0;
|
|
del_timer_sync(&nvl->rc_timer.kernel_timer);
|
|
nv_printf(NV_DBG_INFO, "NVRM: rc timer stopped\n");
|
|
|
|
return 0;
|
|
}
|
|
|
|
#define SNAPSHOT_TIMER_FREQ (jiffies + HZ / NV_SNAPSHOT_TIMER_HZ)
|
|
|
|
static void snapshot_timer_callback(struct nv_timer *timer)
|
|
{
|
|
nv_linux_state_t *nvl = &nv_ctl_device;
|
|
nv_state_t *nv = NV_STATE_PTR(nvl);
|
|
unsigned long flags;
|
|
|
|
NV_SPIN_LOCK_IRQSAVE(&nvl->snapshot_timer_lock, flags);
|
|
if (nvl->snapshot_callback != NULL)
|
|
{
|
|
nvl->snapshot_callback(nv->profiler_context);
|
|
mod_timer(&timer->kernel_timer, SNAPSHOT_TIMER_FREQ);
|
|
}
|
|
NV_SPIN_UNLOCK_IRQRESTORE(&nvl->snapshot_timer_lock, flags);
|
|
}
|
|
|
|
void NV_API_CALL nv_start_snapshot_timer(void (*snapshot_callback)(void *context))
|
|
{
|
|
nv_linux_state_t *nvl = &nv_ctl_device;
|
|
|
|
nvl->snapshot_callback = snapshot_callback;
|
|
nv_timer_setup(&nvl->snapshot_timer, snapshot_timer_callback);
|
|
mod_timer(&nvl->snapshot_timer.kernel_timer, SNAPSHOT_TIMER_FREQ);
|
|
}
|
|
|
|
void NV_API_CALL nv_stop_snapshot_timer(void)
|
|
{
|
|
nv_linux_state_t *nvl = &nv_ctl_device;
|
|
NvBool timer_active;
|
|
unsigned long flags;
|
|
|
|
NV_SPIN_LOCK_IRQSAVE(&nvl->snapshot_timer_lock, flags);
|
|
timer_active = nvl->snapshot_callback != NULL;
|
|
nvl->snapshot_callback = NULL;
|
|
NV_SPIN_UNLOCK_IRQRESTORE(&nvl->snapshot_timer_lock, flags);
|
|
|
|
if (timer_active)
|
|
del_timer_sync(&nvl->snapshot_timer.kernel_timer);
|
|
}
|
|
|
|
void NV_API_CALL nv_flush_snapshot_timer(void)
|
|
{
|
|
nv_linux_state_t *nvl = &nv_ctl_device;
|
|
nv_state_t *nv = NV_STATE_PTR(nvl);
|
|
unsigned long flags;
|
|
|
|
NV_SPIN_LOCK_IRQSAVE(&nvl->snapshot_timer_lock, flags);
|
|
if (nvl->snapshot_callback != NULL)
|
|
nvl->snapshot_callback(nv->profiler_context);
|
|
NV_SPIN_UNLOCK_IRQRESTORE(&nvl->snapshot_timer_lock, flags);
|
|
}
|
|
|
|
static int __init
|
|
nvos_count_devices(void)
|
|
{
|
|
int count;
|
|
|
|
count = nv_pci_count_devices();
|
|
|
|
|
|
|
|
|
|
return count;
|
|
}
|
|
|
|
NvBool nvos_is_chipset_io_coherent(void)
|
|
{
|
|
if (nv_chipset_is_io_coherent == NV_TRISTATE_INDETERMINATE)
|
|
{
|
|
nvidia_stack_t *sp = NULL;
|
|
if (nv_kmem_cache_alloc_stack(&sp) != 0)
|
|
{
|
|
nv_printf(NV_DBG_ERRORS,
|
|
"NVRM: cannot allocate stack for platform coherence check callback \n");
|
|
WARN_ON(1);
|
|
return NV_FALSE;
|
|
}
|
|
|
|
nv_chipset_is_io_coherent = rm_is_chipset_io_coherent(sp);
|
|
|
|
nv_kmem_cache_free_stack(sp);
|
|
}
|
|
|
|
return nv_chipset_is_io_coherent;
|
|
}
|
|
|
|
#if defined(CONFIG_PM)
|
|
static NV_STATUS
|
|
nv_power_management(
|
|
nv_state_t *nv,
|
|
nv_pm_action_t pm_action
|
|
)
|
|
{
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
int status = NV_OK;
|
|
nvidia_stack_t *sp = NULL;
|
|
|
|
if (nv_kmem_cache_alloc_stack(&sp) != 0)
|
|
{
|
|
return NV_ERR_NO_MEMORY;
|
|
}
|
|
|
|
status = nv_check_gpu_state(nv);
|
|
if (status == NV_ERR_GPU_IS_LOST)
|
|
{
|
|
NV_DEV_PRINTF(NV_DBG_INFO, nv, "GPU is lost, skipping PM event\n");
|
|
goto failure;
|
|
}
|
|
|
|
switch (pm_action)
|
|
{
|
|
case NV_PM_ACTION_STANDBY:
|
|
/* fall through */
|
|
case NV_PM_ACTION_HIBERNATE:
|
|
{
|
|
status = rm_power_management(sp, nv, pm_action);
|
|
|
|
nv_kthread_q_stop(&nvl->bottom_half_q);
|
|
|
|
nv_disable_pat_support();
|
|
break;
|
|
}
|
|
case NV_PM_ACTION_RESUME:
|
|
{
|
|
nv_enable_pat_support();
|
|
|
|
nv_kthread_q_item_init(&nvl->bottom_half_q_item,
|
|
nvidia_isr_bh_unlocked, (void *)nv);
|
|
|
|
status = nv_kthread_q_init(&nvl->bottom_half_q, nv_device_name);
|
|
if (status != NV_OK)
|
|
break;
|
|
|
|
status = rm_power_management(sp, nv, pm_action);
|
|
break;
|
|
}
|
|
default:
|
|
status = NV_ERR_INVALID_ARGUMENT;
|
|
break;
|
|
}
|
|
|
|
failure:
|
|
nv_kmem_cache_free_stack(sp);
|
|
|
|
return status;
|
|
}
|
|
|
|
static NV_STATUS
|
|
nv_restore_user_channels(
|
|
nv_state_t *nv
|
|
)
|
|
{
|
|
NV_STATUS status = NV_OK;
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
nv_stack_t *sp = NULL;
|
|
|
|
if (nv_kmem_cache_alloc_stack(&sp) != 0)
|
|
{
|
|
return NV_ERR_NO_MEMORY;
|
|
}
|
|
|
|
down(&nvl->ldata_lock);
|
|
|
|
if ((nv->flags & NV_FLAG_OPEN) == 0)
|
|
{
|
|
goto done;
|
|
}
|
|
|
|
status = rm_restart_user_channels(sp, nv);
|
|
WARN_ON(status != NV_OK);
|
|
|
|
down(&nvl->mmap_lock);
|
|
|
|
nv_set_safe_to_mmap_locked(nv, NV_TRUE);
|
|
|
|
up(&nvl->mmap_lock);
|
|
|
|
rm_unref_dynamic_power(sp, nv, NV_DYNAMIC_PM_FINE);
|
|
|
|
done:
|
|
up(&nvl->ldata_lock);
|
|
|
|
nv_kmem_cache_free_stack(sp);
|
|
|
|
return status;
|
|
}
|
|
|
|
static NV_STATUS
|
|
nv_preempt_user_channels(
|
|
nv_state_t *nv
|
|
)
|
|
{
|
|
NV_STATUS status = NV_OK;
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
nv_stack_t *sp = NULL;
|
|
|
|
if (nv_kmem_cache_alloc_stack(&sp) != 0)
|
|
{
|
|
return NV_ERR_NO_MEMORY;
|
|
}
|
|
|
|
down(&nvl->ldata_lock);
|
|
|
|
if ((nv->flags & NV_FLAG_OPEN) == 0)
|
|
{
|
|
goto done;
|
|
}
|
|
|
|
status = rm_ref_dynamic_power(sp, nv, NV_DYNAMIC_PM_FINE);
|
|
WARN_ON(status != NV_OK);
|
|
|
|
down(&nvl->mmap_lock);
|
|
|
|
nv_set_safe_to_mmap_locked(nv, NV_FALSE);
|
|
nv_revoke_gpu_mappings_locked(nv);
|
|
|
|
up(&nvl->mmap_lock);
|
|
|
|
status = rm_stop_user_channels(sp, nv);
|
|
WARN_ON(status != NV_OK);
|
|
|
|
done:
|
|
up(&nvl->ldata_lock);
|
|
|
|
nv_kmem_cache_free_stack(sp);
|
|
|
|
return status;
|
|
}
|
|
|
|
static NV_STATUS
|
|
nvidia_suspend(
|
|
struct device *dev,
|
|
nv_pm_action_t pm_action,
|
|
NvBool is_procfs_suspend
|
|
)
|
|
{
|
|
NV_STATUS status = NV_OK;
|
|
struct pci_dev *pci_dev = NULL;
|
|
nv_linux_state_t *nvl;
|
|
nv_state_t *nv;
|
|
|
|
if (nv_dev_is_pci(dev))
|
|
{
|
|
pci_dev = to_pci_dev(dev);
|
|
nvl = pci_get_drvdata(pci_dev);
|
|
}
|
|
else
|
|
{
|
|
nvl = dev_get_drvdata(dev);
|
|
}
|
|
nv = NV_STATE_PTR(nvl);
|
|
|
|
down(&nvl->ldata_lock);
|
|
|
|
if (((nv->flags & NV_FLAG_OPEN) == 0) &&
|
|
((nv->flags & NV_FLAG_PERSISTENT_SW_STATE) == 0))
|
|
{
|
|
goto done;
|
|
}
|
|
|
|
if ((nv->flags & NV_FLAG_SUSPENDED) != 0)
|
|
{
|
|
nvl->suspend_count++;
|
|
goto pci_pm;
|
|
}
|
|
|
|
if (nv->preserve_vidmem_allocations && !is_procfs_suspend)
|
|
{
|
|
NV_DEV_PRINTF(NV_DBG_ERRORS, nv,
|
|
"PreserveVideoMemoryAllocations module parameter is set. "
|
|
"System Power Management attempted without driver procfs suspend interface. "
|
|
"Please refer to the 'Configuring Power Management Support' section in the driver README.\n");
|
|
status = NV_ERR_NOT_SUPPORTED;
|
|
goto done;
|
|
}
|
|
|
|
nvidia_modeset_suspend(nv->gpu_id);
|
|
|
|
status = nv_power_management(nv, pm_action);
|
|
|
|
if (status != NV_OK)
|
|
{
|
|
nvidia_modeset_resume(nv->gpu_id);
|
|
goto done;
|
|
}
|
|
else
|
|
{
|
|
nv->flags |= NV_FLAG_SUSPENDED;
|
|
}
|
|
|
|
pci_pm:
|
|
/*
|
|
* Check if PCI power state should be D0 during system suspend. The PCI PM
|
|
* core will change the power state only if the driver has not saved the
|
|
* state in it's suspend callback.
|
|
*/
|
|
if ((nv->d0_state_in_suspend) && (pci_dev != NULL) &&
|
|
!is_procfs_suspend && (pm_action == NV_PM_ACTION_STANDBY))
|
|
{
|
|
pci_save_state(pci_dev);
|
|
}
|
|
|
|
done:
|
|
up(&nvl->ldata_lock);
|
|
|
|
return status;
|
|
}
|
|
|
|
static NV_STATUS
|
|
nvidia_resume(
|
|
struct device *dev,
|
|
nv_pm_action_t pm_action
|
|
)
|
|
{
|
|
NV_STATUS status = NV_OK;
|
|
struct pci_dev *pci_dev;
|
|
nv_linux_state_t *nvl;
|
|
nv_state_t *nv;
|
|
|
|
if (nv_dev_is_pci(dev))
|
|
{
|
|
pci_dev = to_pci_dev(dev);
|
|
nvl = pci_get_drvdata(pci_dev);
|
|
}
|
|
else
|
|
{
|
|
nvl = dev_get_drvdata(dev);
|
|
}
|
|
nv = NV_STATE_PTR(nvl);
|
|
|
|
down(&nvl->ldata_lock);
|
|
|
|
if ((nv->flags & NV_FLAG_SUSPENDED) == 0)
|
|
{
|
|
goto done;
|
|
}
|
|
|
|
if (nvl->suspend_count != 0)
|
|
{
|
|
nvl->suspend_count--;
|
|
}
|
|
else
|
|
{
|
|
status = nv_power_management(nv, pm_action);
|
|
|
|
if (status == NV_OK)
|
|
{
|
|
nvidia_modeset_resume(nv->gpu_id);
|
|
nv->flags &= ~NV_FLAG_SUSPENDED;
|
|
}
|
|
}
|
|
|
|
done:
|
|
up(&nvl->ldata_lock);
|
|
|
|
return status;
|
|
}
|
|
|
|
static NV_STATUS
|
|
nv_resume_devices(
|
|
nv_pm_action_t pm_action,
|
|
nv_pm_action_depth_t pm_action_depth
|
|
)
|
|
{
|
|
nv_linux_state_t *nvl;
|
|
NvBool resume_devices = NV_TRUE;
|
|
NV_STATUS status;
|
|
|
|
if (pm_action_depth == NV_PM_ACTION_DEPTH_MODESET)
|
|
{
|
|
goto resume_modeset;
|
|
}
|
|
|
|
if (pm_action_depth == NV_PM_ACTION_DEPTH_UVM)
|
|
{
|
|
resume_devices = NV_FALSE;
|
|
}
|
|
|
|
LOCK_NV_LINUX_DEVICES();
|
|
|
|
for (nvl = nv_linux_devices; nvl != NULL; nvl = nvl->next)
|
|
{
|
|
if (resume_devices)
|
|
{
|
|
status = nvidia_resume(nvl->dev, pm_action);
|
|
WARN_ON(status != NV_OK);
|
|
}
|
|
}
|
|
|
|
UNLOCK_NV_LINUX_DEVICES();
|
|
|
|
status = nv_uvm_resume();
|
|
WARN_ON(status != NV_OK);
|
|
|
|
LOCK_NV_LINUX_DEVICES();
|
|
|
|
for (nvl = nv_linux_devices; nvl != NULL; nvl = nvl->next)
|
|
{
|
|
status = nv_restore_user_channels(NV_STATE_PTR(nvl));
|
|
WARN_ON(status != NV_OK);
|
|
}
|
|
|
|
UNLOCK_NV_LINUX_DEVICES();
|
|
|
|
resume_modeset:
|
|
nvidia_modeset_resume(0);
|
|
|
|
return NV_OK;
|
|
}
|
|
|
|
static NV_STATUS
|
|
nv_suspend_devices(
|
|
nv_pm_action_t pm_action,
|
|
nv_pm_action_depth_t pm_action_depth
|
|
)
|
|
{
|
|
nv_linux_state_t *nvl;
|
|
NvBool resume_devices = NV_FALSE;
|
|
NV_STATUS status = NV_OK;
|
|
|
|
nvidia_modeset_suspend(0);
|
|
|
|
if (pm_action_depth == NV_PM_ACTION_DEPTH_MODESET)
|
|
{
|
|
return NV_OK;
|
|
}
|
|
|
|
LOCK_NV_LINUX_DEVICES();
|
|
|
|
for (nvl = nv_linux_devices; nvl != NULL && status == NV_OK; nvl = nvl->next)
|
|
{
|
|
status = nv_preempt_user_channels(NV_STATE_PTR(nvl));
|
|
WARN_ON(status != NV_OK);
|
|
}
|
|
|
|
UNLOCK_NV_LINUX_DEVICES();
|
|
|
|
if (status == NV_OK)
|
|
{
|
|
status = nv_uvm_suspend();
|
|
WARN_ON(status != NV_OK);
|
|
}
|
|
if (status != NV_OK)
|
|
{
|
|
goto done;
|
|
}
|
|
|
|
if (pm_action_depth == NV_PM_ACTION_DEPTH_UVM)
|
|
{
|
|
return NV_OK;
|
|
}
|
|
|
|
LOCK_NV_LINUX_DEVICES();
|
|
|
|
for (nvl = nv_linux_devices; nvl != NULL && status == NV_OK; nvl = nvl->next)
|
|
{
|
|
status = nvidia_suspend(nvl->dev, pm_action, NV_TRUE);
|
|
WARN_ON(status != NV_OK);
|
|
}
|
|
if (status != NV_OK)
|
|
{
|
|
resume_devices = NV_TRUE;
|
|
}
|
|
|
|
UNLOCK_NV_LINUX_DEVICES();
|
|
|
|
done:
|
|
if (status != NV_OK)
|
|
{
|
|
LOCK_NV_LINUX_DEVICES();
|
|
|
|
for (nvl = nv_linux_devices; nvl != NULL; nvl = nvl->next)
|
|
{
|
|
if (resume_devices)
|
|
{
|
|
nvidia_resume(nvl->dev, pm_action);
|
|
}
|
|
|
|
nv_restore_user_channels(NV_STATE_PTR(nvl));
|
|
}
|
|
|
|
UNLOCK_NV_LINUX_DEVICES();
|
|
}
|
|
|
|
return status;
|
|
}
|
|
|
|
NV_STATUS
|
|
nv_set_system_power_state(
|
|
nv_power_state_t power_state,
|
|
nv_pm_action_depth_t pm_action_depth
|
|
)
|
|
{
|
|
NV_STATUS status;
|
|
nv_pm_action_t pm_action;
|
|
|
|
switch (power_state)
|
|
{
|
|
case NV_POWER_STATE_IN_HIBERNATE:
|
|
pm_action = NV_PM_ACTION_HIBERNATE;
|
|
break;
|
|
case NV_POWER_STATE_IN_STANDBY:
|
|
pm_action = NV_PM_ACTION_STANDBY;
|
|
break;
|
|
case NV_POWER_STATE_RUNNING:
|
|
pm_action = NV_PM_ACTION_RESUME;
|
|
break;
|
|
default:
|
|
return NV_ERR_INVALID_ARGUMENT;
|
|
}
|
|
|
|
down(&nv_system_power_state_lock);
|
|
|
|
if (nv_system_power_state == power_state)
|
|
{
|
|
status = NV_OK;
|
|
goto done;
|
|
}
|
|
|
|
if (power_state == NV_POWER_STATE_RUNNING)
|
|
{
|
|
status = nv_resume_devices(pm_action, nv_system_pm_action_depth);
|
|
up_write(&nv_system_pm_lock);
|
|
}
|
|
else
|
|
{
|
|
if (nv_system_power_state != NV_POWER_STATE_RUNNING)
|
|
{
|
|
status = NV_ERR_INVALID_ARGUMENT;
|
|
goto done;
|
|
}
|
|
|
|
nv_system_pm_action_depth = pm_action_depth;
|
|
|
|
down_write(&nv_system_pm_lock);
|
|
status = nv_suspend_devices(pm_action, nv_system_pm_action_depth);
|
|
if (status != NV_OK)
|
|
{
|
|
up_write(&nv_system_pm_lock);
|
|
goto done;
|
|
}
|
|
}
|
|
|
|
nv_system_power_state = power_state;
|
|
|
|
done:
|
|
up(&nv_system_power_state_lock);
|
|
|
|
return status;
|
|
}
|
|
|
|
int nv_pmops_suspend(
|
|
struct device *dev
|
|
)
|
|
{
|
|
NV_STATUS status;
|
|
|
|
status = nvidia_suspend(dev, NV_PM_ACTION_STANDBY, NV_FALSE);
|
|
return (status == NV_OK) ? 0 : -EIO;
|
|
}
|
|
|
|
int nv_pmops_resume(
|
|
struct device *dev
|
|
)
|
|
{
|
|
NV_STATUS status;
|
|
|
|
status = nvidia_resume(dev, NV_PM_ACTION_RESUME);
|
|
return (status == NV_OK) ? 0 : -EIO;
|
|
}
|
|
|
|
int nv_pmops_freeze(
|
|
struct device *dev
|
|
)
|
|
{
|
|
NV_STATUS status;
|
|
|
|
status = nvidia_suspend(dev, NV_PM_ACTION_HIBERNATE, NV_FALSE);
|
|
return (status == NV_OK) ? 0 : -EIO;
|
|
}
|
|
|
|
int nv_pmops_thaw(
|
|
struct device *dev
|
|
)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
int nv_pmops_restore(
|
|
struct device *dev
|
|
)
|
|
{
|
|
NV_STATUS status;
|
|
|
|
status = nvidia_resume(dev, NV_PM_ACTION_RESUME);
|
|
return (status == NV_OK) ? 0 : -EIO;
|
|
}
|
|
|
|
int nv_pmops_poweroff(
|
|
struct device *dev
|
|
)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static int
|
|
nvidia_transition_dynamic_power(
|
|
struct device *dev,
|
|
NvBool enter
|
|
)
|
|
{
|
|
struct pci_dev *pci_dev = to_pci_dev(dev);
|
|
nv_linux_state_t *nvl = pci_get_drvdata(pci_dev);
|
|
nv_state_t *nv = NV_STATE_PTR(nvl);
|
|
nvidia_stack_t *sp = NULL;
|
|
NV_STATUS status;
|
|
|
|
if ((nv->flags & (NV_FLAG_OPEN | NV_FLAG_PERSISTENT_SW_STATE)) == 0)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
if (nv_kmem_cache_alloc_stack(&sp) != 0)
|
|
{
|
|
return -ENOMEM;
|
|
}
|
|
|
|
status = rm_transition_dynamic_power(sp, nv, enter);
|
|
|
|
nv_kmem_cache_free_stack(sp);
|
|
|
|
return (status == NV_OK) ? 0 : -EIO;
|
|
}
|
|
|
|
int nv_pmops_runtime_suspend(
|
|
struct device *dev
|
|
)
|
|
{
|
|
return nvidia_transition_dynamic_power(dev, NV_TRUE);
|
|
}
|
|
|
|
int nv_pmops_runtime_resume(
|
|
struct device *dev
|
|
)
|
|
{
|
|
return nvidia_transition_dynamic_power(dev, NV_FALSE);
|
|
}
|
|
#endif /* defined(CONFIG_PM) */
|
|
|
|
nv_state_t* NV_API_CALL nv_get_adapter_state(
|
|
NvU32 domain,
|
|
NvU8 bus,
|
|
NvU8 slot
|
|
)
|
|
{
|
|
nv_linux_state_t *nvl;
|
|
|
|
LOCK_NV_LINUX_DEVICES();
|
|
for (nvl = nv_linux_devices; nvl != NULL; nvl = nvl->next)
|
|
{
|
|
nv_state_t *nv = NV_STATE_PTR(nvl);
|
|
if (nv->pci_info.domain == domain && nv->pci_info.bus == bus
|
|
&& nv->pci_info.slot == slot)
|
|
{
|
|
UNLOCK_NV_LINUX_DEVICES();
|
|
return nv;
|
|
}
|
|
}
|
|
UNLOCK_NV_LINUX_DEVICES();
|
|
|
|
return NULL;
|
|
}
|
|
|
|
nv_state_t* NV_API_CALL nv_get_ctl_state(void)
|
|
{
|
|
return NV_STATE_PTR(&nv_ctl_device);
|
|
}
|
|
|
|
NV_STATUS NV_API_CALL nv_log_error(
|
|
nv_state_t *nv,
|
|
NvU32 error_number,
|
|
const char *format,
|
|
va_list ap
|
|
)
|
|
{
|
|
NV_STATUS status = NV_OK;
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
|
|
nv_report_error(nvl->pci_dev, error_number, format, ap);
|
|
#if defined(CONFIG_CRAY_XT)
|
|
status = nvos_forward_error_to_cray(nvl->pci_dev, error_number,
|
|
format, ap);
|
|
#endif
|
|
|
|
return status;
|
|
}
|
|
|
|
NvU64 NV_API_CALL nv_get_dma_start_address(
|
|
nv_state_t *nv
|
|
)
|
|
{
|
|
#if defined(NVCPU_PPC64LE)
|
|
struct pci_dev *pci_dev;
|
|
dma_addr_t dma_addr;
|
|
NvU64 saved_dma_mask;
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
|
|
/*
|
|
* If TCE bypass is disabled via a module parameter, then just return
|
|
* the default (which is 0).
|
|
*
|
|
* Otherwise, the DMA start address only needs to be set once, and it
|
|
* won't change afterward. Just return the cached value if asked again,
|
|
* to avoid the kernel printing redundant messages to the kernel
|
|
* log when we call pci_set_dma_mask().
|
|
*/
|
|
if ((nv_tce_bypass_mode == NV_TCE_BYPASS_MODE_DISABLE) ||
|
|
(nvl->tce_bypass_enabled))
|
|
{
|
|
return nvl->dma_dev.addressable_range.start;
|
|
}
|
|
|
|
pci_dev = nvl->pci_dev;
|
|
|
|
/*
|
|
* Linux on IBM POWER8 offers 2 different DMA set-ups, sometimes
|
|
* referred to as "windows".
|
|
*
|
|
* The "default window" provides a 2GB region of PCI address space
|
|
* located below the 32-bit line. The IOMMU is used to provide a
|
|
* "rich" mapping--any page in system memory can be mapped at an
|
|
* arbitrary address within this window. The mappings are dynamic
|
|
* and pass in and out of being as pci_map*()/pci_unmap*() calls
|
|
* are made.
|
|
*
|
|
* Dynamic DMA Windows (sometimes "Huge DDW") provides a linear
|
|
* mapping of the system's entire physical address space at some
|
|
* fixed offset above the 59-bit line. IOMMU is still used, and
|
|
* pci_map*()/pci_unmap*() are still required, but mappings are
|
|
* static. They're effectively set up in advance, and any given
|
|
* system page will always map to the same PCI bus address. I.e.
|
|
* physical 0x00000000xxxxxxxx => PCI 0x08000000xxxxxxxx
|
|
*
|
|
* This driver does not support the 2G default window because
|
|
* of its limited size, and for reasons having to do with UVM.
|
|
*
|
|
* Linux on POWER8 will only provide the DDW-style full linear
|
|
* mapping when the driver claims support for 64-bit DMA addressing
|
|
* (a pre-requisite because the PCI addresses used in this case will
|
|
* be near the top of the 64-bit range). The linear mapping
|
|
* is not available in all system configurations.
|
|
*
|
|
* Detect whether the linear mapping is present by claiming
|
|
* 64-bit support and then mapping physical page 0. For historical
|
|
* reasons, Linux on POWER8 will never map a page to PCI address 0x0.
|
|
* In the "default window" case page 0 will be mapped to some
|
|
* non-zero address below the 32-bit line. In the
|
|
* DDW/linear-mapping case, it will be mapped to address 0 plus
|
|
* some high-order offset.
|
|
*
|
|
* If the linear mapping is present and sane then return the offset
|
|
* as the starting address for all DMA mappings.
|
|
*/
|
|
saved_dma_mask = pci_dev->dma_mask;
|
|
if (pci_set_dma_mask(pci_dev, DMA_BIT_MASK(64)) != 0)
|
|
{
|
|
goto done;
|
|
}
|
|
|
|
dma_addr = pci_map_single(pci_dev, NULL, 1, DMA_BIDIRECTIONAL);
|
|
if (pci_dma_mapping_error(pci_dev, dma_addr))
|
|
{
|
|
pci_set_dma_mask(pci_dev, saved_dma_mask);
|
|
goto done;
|
|
}
|
|
|
|
pci_unmap_single(pci_dev, dma_addr, 1, DMA_BIDIRECTIONAL);
|
|
|
|
/*
|
|
* From IBM: "For IODA2, native DMA bypass or KVM TCE-based implementation
|
|
* of full 64-bit DMA support will establish a window in address-space
|
|
* with the high 14 bits being constant and the bottom up-to-50 bits
|
|
* varying with the mapping."
|
|
*
|
|
* Unfortunately, we don't have any good interfaces or definitions from
|
|
* the kernel to get information about the DMA offset assigned by OS.
|
|
* However, we have been told that the offset will be defined by the top
|
|
* 14 bits of the address, and bits 40-49 will not vary for any DMA
|
|
* mappings until 1TB of system memory is surpassed; this limitation is
|
|
* essential for us to function properly since our current GPUs only
|
|
* support 40 physical address bits. We are in a fragile place where we
|
|
* need to tell the OS that we're capable of 64-bit addressing, while
|
|
* relying on the assumption that the top 24 bits will not vary in this
|
|
* case.
|
|
*
|
|
* The way we try to compute the window, then, is mask the trial mapping
|
|
* against the DMA capabilities of the device. That way, devices with
|
|
* greater addressing capabilities will only take the bits it needs to
|
|
* define the window.
|
|
*/
|
|
if ((dma_addr & DMA_BIT_MASK(32)) != 0)
|
|
{
|
|
/*
|
|
* Huge DDW not available - page 0 mapped to non-zero address below
|
|
* the 32-bit line.
|
|
*/
|
|
nv_printf(NV_DBG_WARNINGS,
|
|
"NVRM: DMA window limited by platform\n");
|
|
pci_set_dma_mask(pci_dev, saved_dma_mask);
|
|
goto done;
|
|
}
|
|
else if ((dma_addr & saved_dma_mask) != 0)
|
|
{
|
|
NvU64 memory_size = os_get_num_phys_pages() * PAGE_SIZE;
|
|
if ((dma_addr & ~saved_dma_mask) !=
|
|
((dma_addr + memory_size) & ~saved_dma_mask))
|
|
{
|
|
/*
|
|
* The physical window straddles our addressing limit boundary,
|
|
* e.g., for an adapter that can address up to 1TB, the window
|
|
* crosses the 40-bit limit so that the lower end of the range
|
|
* has different bits 63:40 than the higher end of the range.
|
|
* We can only handle a single, static value for bits 63:40, so
|
|
* we must fall back here.
|
|
*/
|
|
nv_printf(NV_DBG_WARNINGS,
|
|
"NVRM: DMA window limited by memory size\n");
|
|
pci_set_dma_mask(pci_dev, saved_dma_mask);
|
|
goto done;
|
|
}
|
|
}
|
|
|
|
nvl->tce_bypass_enabled = NV_TRUE;
|
|
nvl->dma_dev.addressable_range.start = dma_addr & ~(saved_dma_mask);
|
|
|
|
/* Update the coherent mask to match */
|
|
dma_set_coherent_mask(&pci_dev->dev, pci_dev->dma_mask);
|
|
|
|
done:
|
|
return nvl->dma_dev.addressable_range.start;
|
|
#else
|
|
return 0;
|
|
#endif
|
|
}
|
|
|
|
NV_STATUS NV_API_CALL nv_set_primary_vga_status(
|
|
nv_state_t *nv
|
|
)
|
|
{
|
|
/* IORESOURCE_ROM_SHADOW wasn't added until 2.6.10 */
|
|
#if defined(IORESOURCE_ROM_SHADOW)
|
|
nv_linux_state_t *nvl;
|
|
struct pci_dev *pci_dev;
|
|
|
|
nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
pci_dev = nvl->pci_dev;
|
|
|
|
nv->primary_vga = ((NV_PCI_RESOURCE_FLAGS(pci_dev, PCI_ROM_RESOURCE) &
|
|
IORESOURCE_ROM_SHADOW) == IORESOURCE_ROM_SHADOW);
|
|
return NV_OK;
|
|
#else
|
|
return NV_ERR_NOT_SUPPORTED;
|
|
#endif
|
|
}
|
|
|
|
NV_STATUS NV_API_CALL nv_pci_trigger_recovery(
|
|
nv_state_t *nv
|
|
)
|
|
{
|
|
NV_STATUS status = NV_ERR_NOT_SUPPORTED;
|
|
#if defined(NV_PCI_ERROR_RECOVERY)
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
|
|
/*
|
|
* Calling readl() on PPC64LE will allow the kernel to check its state for
|
|
* the device and update it accordingly. This needs to be done before
|
|
* checking if the PCI channel is offline, so that we don't check stale
|
|
* state.
|
|
*
|
|
* This will also kick off the recovery process for the device.
|
|
*/
|
|
if (NV_PCI_ERROR_RECOVERY_ENABLED())
|
|
{
|
|
if (readl(nv->regs->map) == 0xFFFFFFFF)
|
|
{
|
|
if (pci_channel_offline(nvl->pci_dev))
|
|
{
|
|
NV_DEV_PRINTF(NV_DBG_ERRORS, nv,
|
|
"PCI channel for the device is offline\n");
|
|
status = NV_OK;
|
|
}
|
|
}
|
|
}
|
|
#endif
|
|
return status;
|
|
}
|
|
|
|
NvBool NV_API_CALL nv_requires_dma_remap(
|
|
nv_state_t *nv
|
|
)
|
|
{
|
|
NvBool dma_remap = NV_FALSE;
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
dma_remap = !nv_dma_maps_swiotlb(nvl->dev);
|
|
return dma_remap;
|
|
}
|
|
|
|
/*
|
|
* Intended for use by external kernel modules to list nvidia gpu ids.
|
|
*/
|
|
NvBool nvidia_get_gpuid_list(NvU32 *gpu_ids, NvU32 *gpu_count)
|
|
{
|
|
nv_linux_state_t *nvl;
|
|
unsigned int count;
|
|
NvBool ret = NV_TRUE;
|
|
|
|
LOCK_NV_LINUX_DEVICES();
|
|
|
|
count = 0;
|
|
for (nvl = nv_linux_devices; nvl != NULL; nvl = nvl->next)
|
|
count++;
|
|
|
|
if (*gpu_count == 0)
|
|
{
|
|
goto done;
|
|
}
|
|
else if ((*gpu_count) < count)
|
|
{
|
|
ret = NV_FALSE;
|
|
goto done;
|
|
}
|
|
|
|
count = 0;
|
|
for (nvl = nv_linux_devices; nvl != NULL; nvl = nvl->next)
|
|
{
|
|
nv_state_t *nv = NV_STATE_PTR(nvl);
|
|
gpu_ids[count++] = nv->gpu_id;
|
|
}
|
|
|
|
|
|
done:
|
|
|
|
*gpu_count = count;
|
|
|
|
UNLOCK_NV_LINUX_DEVICES();
|
|
|
|
return ret;
|
|
}
|
|
|
|
/*
|
|
* Kernel-level analog to nvidia_open, intended for use by external
|
|
* kernel modules. This increments the ref count of the device with
|
|
* the given gpu_id and makes sure the device has been initialized.
|
|
*
|
|
* Clients of this interface are counted by the RM reset path, to ensure a
|
|
* GPU is not reset while the GPU is active.
|
|
*
|
|
* Returns -ENODEV if the given gpu_id does not exist.
|
|
*/
|
|
int nvidia_dev_get(NvU32 gpu_id, nvidia_stack_t *sp)
|
|
{
|
|
nv_linux_state_t *nvl;
|
|
int rc;
|
|
|
|
/* Takes nvl->ldata_lock */
|
|
nvl = find_gpu_id(gpu_id);
|
|
if (!nvl)
|
|
return -ENODEV;
|
|
|
|
rc = nv_open_device(NV_STATE_PTR(nvl), sp);
|
|
|
|
if (rc == 0)
|
|
WARN_ON(rm_set_external_kernel_client_count(sp, NV_STATE_PTR(nvl), NV_TRUE) != NV_OK);
|
|
|
|
up(&nvl->ldata_lock);
|
|
return rc;
|
|
}
|
|
|
|
/*
|
|
* Kernel-level analog to nvidia_close, intended for use by external
|
|
* kernel modules. This decrements the ref count of the device with
|
|
* the given gpu_id, potentially tearing it down.
|
|
*/
|
|
void nvidia_dev_put(NvU32 gpu_id, nvidia_stack_t *sp)
|
|
{
|
|
nv_linux_state_t *nvl;
|
|
|
|
/* Takes nvl->ldata_lock */
|
|
nvl = find_gpu_id(gpu_id);
|
|
if (!nvl)
|
|
return;
|
|
|
|
nv_close_device(NV_STATE_PTR(nvl), sp);
|
|
|
|
WARN_ON(rm_set_external_kernel_client_count(sp, NV_STATE_PTR(nvl), NV_FALSE) != NV_OK);
|
|
|
|
up(&nvl->ldata_lock);
|
|
}
|
|
|
|
/*
|
|
* Like nvidia_dev_get but uses UUID instead of gpu_id. Note that this may
|
|
* trigger initialization and teardown of unrelated devices to look up their
|
|
* UUIDs.
|
|
*
|
|
* Clients of this interface are counted by the RM reset path, to ensure a
|
|
* GPU is not reset while the GPU is active.
|
|
*/
|
|
int nvidia_dev_get_uuid(const NvU8 *uuid, nvidia_stack_t *sp)
|
|
{
|
|
nv_state_t *nv = NULL;
|
|
nv_linux_state_t *nvl = NULL;
|
|
const NvU8 *dev_uuid;
|
|
int rc = 0;
|
|
|
|
/* Takes nvl->ldata_lock */
|
|
nvl = find_uuid_candidate(uuid);
|
|
while (nvl)
|
|
{
|
|
nv = NV_STATE_PTR(nvl);
|
|
|
|
/*
|
|
* If the device is missing its UUID, this call exists solely so
|
|
* rm_get_gpu_uuid_raw will be called and we can inspect the UUID.
|
|
*/
|
|
rc = nv_open_device(nv, sp);
|
|
if (rc != 0)
|
|
goto out;
|
|
|
|
/* The UUID should always be present following nv_open_device */
|
|
dev_uuid = nv_get_cached_uuid(nv);
|
|
WARN_ON(!dev_uuid);
|
|
if (dev_uuid && memcmp(dev_uuid, uuid, GPU_UUID_LEN) == 0)
|
|
break;
|
|
|
|
/* No match, try again. */
|
|
nv_close_device(nv, sp);
|
|
up(&nvl->ldata_lock);
|
|
nvl = find_uuid_candidate(uuid);
|
|
}
|
|
|
|
if (nvl)
|
|
{
|
|
rc = 0;
|
|
WARN_ON(rm_set_external_kernel_client_count(sp, NV_STATE_PTR(nvl), NV_TRUE) != NV_OK);
|
|
}
|
|
else
|
|
rc = -ENODEV;
|
|
|
|
out:
|
|
if (nvl)
|
|
up(&nvl->ldata_lock);
|
|
return rc;
|
|
}
|
|
|
|
/*
|
|
* Like nvidia_dev_put but uses UUID instead of gpu_id.
|
|
*/
|
|
void nvidia_dev_put_uuid(const NvU8 *uuid, nvidia_stack_t *sp)
|
|
{
|
|
nv_linux_state_t *nvl;
|
|
|
|
/* Callers must already have called nvidia_dev_get_uuid() */
|
|
|
|
/* Takes nvl->ldata_lock */
|
|
nvl = find_uuid(uuid);
|
|
if (!nvl)
|
|
return;
|
|
|
|
nv_close_device(NV_STATE_PTR(nvl), sp);
|
|
|
|
WARN_ON(rm_set_external_kernel_client_count(sp, NV_STATE_PTR(nvl), NV_FALSE) != NV_OK);
|
|
|
|
up(&nvl->ldata_lock);
|
|
}
|
|
|
|
int nvidia_dev_block_gc6(const NvU8 *uuid, nvidia_stack_t *sp)
|
|
|
|
{
|
|
nv_linux_state_t *nvl;
|
|
|
|
/* Callers must already have called nvidia_dev_get_uuid() */
|
|
|
|
/* Takes nvl->ldata_lock */
|
|
nvl = find_uuid(uuid);
|
|
if (!nvl)
|
|
return -ENODEV;
|
|
|
|
if (rm_ref_dynamic_power(sp, NV_STATE_PTR(nvl), NV_DYNAMIC_PM_FINE) != NV_OK)
|
|
{
|
|
up(&nvl->ldata_lock);
|
|
return -EINVAL;
|
|
}
|
|
|
|
up(&nvl->ldata_lock);
|
|
|
|
return 0;
|
|
}
|
|
|
|
int nvidia_dev_unblock_gc6(const NvU8 *uuid, nvidia_stack_t *sp)
|
|
|
|
{
|
|
nv_linux_state_t *nvl;
|
|
|
|
/* Callers must already have called nvidia_dev_get_uuid() */
|
|
|
|
/* Takes nvl->ldata_lock */
|
|
nvl = find_uuid(uuid);
|
|
if (!nvl)
|
|
return -ENODEV;
|
|
|
|
rm_unref_dynamic_power(sp, NV_STATE_PTR(nvl), NV_DYNAMIC_PM_FINE);
|
|
|
|
up(&nvl->ldata_lock);
|
|
|
|
return 0;
|
|
}
|
|
|
|
NV_STATUS NV_API_CALL nv_get_device_memory_config(
|
|
nv_state_t *nv,
|
|
NvU64 *compr_addr_sys_phys,
|
|
NvU64 *addr_guest_phys,
|
|
NvU32 *addr_width,
|
|
NvU32 *granularity,
|
|
NvS32 *node_id
|
|
)
|
|
{
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
NV_STATUS status = NV_ERR_NOT_SUPPORTED;
|
|
|
|
if (!nv_platform_supports_numa(nvl))
|
|
{
|
|
return NV_ERR_NOT_SUPPORTED;
|
|
}
|
|
|
|
#if defined(NVCPU_PPC64LE)
|
|
nv_npu_numa_info_t *numa_info;
|
|
|
|
numa_info = &nvl->npu->numa_info;
|
|
|
|
if (node_id != NULL)
|
|
{
|
|
*node_id = nvl->numa_info.node_id;
|
|
}
|
|
|
|
if (compr_addr_sys_phys != NULL)
|
|
{
|
|
*compr_addr_sys_phys =
|
|
numa_info->compr_sys_phys_addr;
|
|
}
|
|
|
|
if (addr_guest_phys != NULL)
|
|
{
|
|
*addr_guest_phys =
|
|
numa_info->guest_phys_addr;
|
|
}
|
|
|
|
if (addr_width != NULL)
|
|
{
|
|
*addr_width = nv_volta_dma_addr_size - nv_volta_addr_space_width;
|
|
}
|
|
|
|
if (granularity != NULL)
|
|
{
|
|
*granularity = nv_volta_addr_space_width;
|
|
}
|
|
|
|
status = NV_OK;
|
|
#endif
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return status;
|
|
}
|
|
|
|
#if defined(NVCPU_PPC64LE)
|
|
|
|
NV_STATUS NV_API_CALL nv_get_nvlink_line_rate(
|
|
nv_state_t *nvState,
|
|
NvU32 *linerate
|
|
)
|
|
{
|
|
#if defined(NV_PNV_PCI_GET_NPU_DEV_PRESENT) && defined(NV_OF_GET_PROPERTY_PRESENT)
|
|
|
|
nv_linux_state_t *nvl;
|
|
struct pci_dev *npuDev;
|
|
NvU32 *pSpeedPtr = NULL;
|
|
NvU32 speed;
|
|
int len;
|
|
|
|
if (nvState != NULL)
|
|
nvl = NV_GET_NVL_FROM_NV_STATE(nvState);
|
|
else
|
|
return NV_ERR_INVALID_ARGUMENT;
|
|
|
|
if (!nvl->npu)
|
|
{
|
|
return NV_ERR_NOT_SUPPORTED;
|
|
}
|
|
|
|
npuDev = nvl->npu->devs[0];
|
|
if (!npuDev->dev.of_node)
|
|
{
|
|
nv_printf(NV_DBG_ERRORS, "NVRM: %s: OF Node not found in IBM-NPU device node\n",
|
|
__FUNCTION__);
|
|
return NV_ERR_NOT_SUPPORTED;
|
|
}
|
|
|
|
pSpeedPtr = (NvU32 *) of_get_property(npuDev->dev.of_node, "ibm,nvlink-speed", &len);
|
|
|
|
if (pSpeedPtr)
|
|
{
|
|
speed = (NvU32) be32_to_cpup(pSpeedPtr);
|
|
}
|
|
else
|
|
{
|
|
return NV_ERR_NOT_SUPPORTED;
|
|
}
|
|
|
|
if (!speed)
|
|
{
|
|
return NV_ERR_NOT_SUPPORTED;
|
|
}
|
|
else
|
|
{
|
|
*linerate = speed;
|
|
}
|
|
|
|
return NV_OK;
|
|
|
|
#endif
|
|
|
|
return NV_ERR_NOT_SUPPORTED;
|
|
}
|
|
|
|
#endif
|
|
|
|
NV_STATUS NV_API_CALL nv_indicate_idle(
|
|
nv_state_t *nv
|
|
)
|
|
{
|
|
#if defined(NV_PM_RUNTIME_AVAILABLE)
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
struct device *dev = nvl->dev;
|
|
struct file *file = nvl->sysfs_config_file;
|
|
loff_t f_pos = 0;
|
|
char buf;
|
|
|
|
pm_runtime_put_noidle(dev);
|
|
|
|
#if defined(NV_SEQ_READ_ITER_PRESENT)
|
|
{
|
|
struct kernfs_open_file *of = ((struct seq_file *)file->private_data)->private;
|
|
struct kernfs_node *kn;
|
|
|
|
mutex_lock(&of->mutex);
|
|
kn = of->kn;
|
|
if (kn != NULL && atomic_inc_unless_negative(&kn->active))
|
|
{
|
|
if ((kn->attr.ops != NULL) && (kn->attr.ops->read != NULL))
|
|
{
|
|
kn->attr.ops->read(of, &buf, 1, f_pos);
|
|
}
|
|
atomic_dec(&kn->active);
|
|
}
|
|
mutex_unlock(&of->mutex);
|
|
}
|
|
#else
|
|
#if defined(NV_KERNEL_READ_HAS_POINTER_POS_ARG)
|
|
kernel_read(file, &buf, 1, &f_pos);
|
|
#else
|
|
kernel_read(file, f_pos, &buf, 1);
|
|
#endif
|
|
#endif
|
|
|
|
return NV_OK;
|
|
#else
|
|
return NV_ERR_NOT_SUPPORTED;
|
|
#endif
|
|
}
|
|
|
|
NV_STATUS NV_API_CALL nv_indicate_not_idle(
|
|
nv_state_t *nv
|
|
)
|
|
{
|
|
#if defined(NV_PM_RUNTIME_AVAILABLE)
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
struct device *dev = nvl->dev;
|
|
|
|
pm_runtime_get_noresume(dev);
|
|
|
|
nvl->is_forced_shutdown = NV_TRUE;
|
|
pci_bus_type.shutdown(dev);
|
|
|
|
return NV_OK;
|
|
#else
|
|
return NV_ERR_NOT_SUPPORTED;
|
|
#endif
|
|
}
|
|
|
|
void NV_API_CALL nv_idle_holdoff(
|
|
nv_state_t *nv
|
|
)
|
|
{
|
|
#if defined(NV_PM_RUNTIME_AVAILABLE)
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
struct device *dev = nvl->dev;
|
|
|
|
pm_runtime_get_noresume(dev);
|
|
#endif
|
|
}
|
|
|
|
NvBool NV_API_CALL nv_dynamic_power_available(
|
|
nv_state_t *nv
|
|
)
|
|
{
|
|
#if defined(NV_PM_RUNTIME_AVAILABLE)
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
return nvl->sysfs_config_file != NULL;
|
|
#else
|
|
return NV_FALSE;
|
|
#endif
|
|
}
|
|
|
|
/* caller should hold nv_linux_devices_lock using LOCK_NV_LINUX_DEVICES */
|
|
void nv_linux_add_device_locked(nv_linux_state_t *nvl)
|
|
{
|
|
if (nv_linux_devices == NULL) {
|
|
nv_linux_devices = nvl;
|
|
}
|
|
else
|
|
{
|
|
nv_linux_state_t *tnvl;
|
|
for (tnvl = nv_linux_devices; tnvl->next != NULL; tnvl = tnvl->next);
|
|
tnvl->next = nvl;
|
|
}
|
|
}
|
|
|
|
/* caller should hold nv_linux_devices_lock using LOCK_NV_LINUX_DEVICES */
|
|
void nv_linux_remove_device_locked(nv_linux_state_t *nvl)
|
|
{
|
|
if (nvl == nv_linux_devices) {
|
|
nv_linux_devices = nvl->next;
|
|
}
|
|
else
|
|
{
|
|
nv_linux_state_t *tnvl;
|
|
for (tnvl = nv_linux_devices; tnvl->next != nvl; tnvl = tnvl->next);
|
|
tnvl->next = nvl->next;
|
|
}
|
|
}
|
|
|
|
void NV_API_CALL nv_control_soc_irqs(nv_state_t *nv, NvBool bEnable)
|
|
{
|
|
int count;
|
|
|
|
if (bEnable)
|
|
{
|
|
for (count = 0; count < nv->num_soc_irqs; count++)
|
|
{
|
|
nv->soc_irq_info[count].bh_pending = NV_FALSE;
|
|
nv->current_soc_irq = -1;
|
|
enable_irq(nv->soc_irq_info[count].irq_num);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
for (count = 0; count < nv->num_soc_irqs; count++)
|
|
{
|
|
disable_irq_nosync(nv->soc_irq_info[count].irq_num);
|
|
}
|
|
}
|
|
}
|
|
|
|
NvU32 NV_API_CALL nv_get_dev_minor(nv_state_t *nv)
|
|
{
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
|
|
return nvl->minor_num;
|
|
}
|
|
|
|
NV_STATUS NV_API_CALL nv_acquire_fabric_mgmt_cap(int fd, int *duped_fd)
|
|
{
|
|
|
|
*duped_fd = nvlink_cap_acquire(fd, NVLINK_CAP_FABRIC_MANAGEMENT);
|
|
if (*duped_fd < 0)
|
|
{
|
|
return NV_ERR_INSUFFICIENT_PERMISSIONS;
|
|
}
|
|
|
|
return NV_OK;
|
|
|
|
|
|
|
|
}
|
|
|
|
/*
|
|
* Wakes up the NVIDIA GPU HDA codec and contoller by reading
|
|
* codec proc file.
|
|
*/
|
|
void NV_API_CALL nv_audio_dynamic_power(
|
|
nv_state_t *nv
|
|
)
|
|
{
|
|
/*
|
|
* The runtime power management for nvidia HDA controller can be possible
|
|
* after commit 07f4f97d7b4b ("vga_switcheroo: Use device link for HDA
|
|
* controller"). This commit has also moved 'PCI_CLASS_MULTIMEDIA_HD_AUDIO'
|
|
* macro from <sound/hdaudio.h> to <linux/pci_ids.h>.
|
|
* If 'NV_PCI_CLASS_MULTIMEDIA_HD_AUDIO_PRESENT' is not defined, then
|
|
* this function will be stub function.
|
|
*
|
|
* Also, check if runtime PM is enabled in the kernel (with
|
|
* 'NV_PM_RUNTIME_AVAILABLE') and stub this function if it is disabled. This
|
|
* function uses kernel fields only present when the kconfig has runtime PM
|
|
* enabled.
|
|
*/
|
|
#if defined(NV_PCI_CLASS_MULTIMEDIA_HD_AUDIO_PRESENT) && defined(NV_PM_RUNTIME_AVAILABLE)
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
struct device *dev = nvl->dev;
|
|
struct pci_dev *audio_pci_dev, *pci_dev;
|
|
struct snd_card *card;
|
|
|
|
if (!nv_dev_is_pci(dev))
|
|
return;
|
|
|
|
pci_dev = to_pci_dev(dev);
|
|
|
|
audio_pci_dev = os_pci_init_handle(NV_PCI_DOMAIN_NUMBER(pci_dev),
|
|
NV_PCI_BUS_NUMBER(pci_dev),
|
|
NV_PCI_SLOT_NUMBER(pci_dev),
|
|
1, NULL, NULL);
|
|
|
|
if (audio_pci_dev == NULL)
|
|
return;
|
|
|
|
/*
|
|
* Check if HDA controller is in pm suspended state. The HDA contoller
|
|
* can not be runtime resumed if this API is called during system
|
|
* suspend/resume time and HDA controller is in pm suspended state.
|
|
*/
|
|
if (audio_pci_dev->dev.power.is_suspended)
|
|
return;
|
|
|
|
card = pci_get_drvdata(audio_pci_dev);
|
|
if (card == NULL)
|
|
return;
|
|
|
|
/*
|
|
* Commit be57bfffb7b5 ("ALSA: hda: move hda_codec.h to include/sound")
|
|
* in v4.20-rc1 moved "hda_codec.h" header file from the private sound
|
|
* folder to include/sound.
|
|
*/
|
|
#if defined(NV_SOUND_HDA_CODEC_H_PRESENT)
|
|
{
|
|
struct list_head *p;
|
|
struct hda_codec *codec = NULL;
|
|
unsigned int cmd, res;
|
|
|
|
/*
|
|
* Traverse the list of devices which the sound card maintains and
|
|
* search for HDA codec controller.
|
|
*/
|
|
list_for_each_prev(p, &card->devices)
|
|
{
|
|
struct snd_device *pdev = list_entry(p, struct snd_device, list);
|
|
|
|
if (pdev->type == SNDRV_DEV_CODEC)
|
|
{
|
|
codec = pdev->device_data;
|
|
|
|
/*
|
|
* NVIDIA HDA codec controller uses linux kernel HDA codec
|
|
* driver. Commit 05852448690d ("ALSA: hda - Support indirect
|
|
* execution of verbs") added support for overriding exec_verb.
|
|
* This codec->core.exec_verb will be codec_exec_verb() for
|
|
* NVIDIA HDA codec driver.
|
|
*/
|
|
if (codec->core.exec_verb == NULL)
|
|
{
|
|
return;
|
|
}
|
|
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (codec == NULL)
|
|
{
|
|
return;
|
|
}
|
|
|
|
/* If HDA codec controller is already runtime active, then return */
|
|
if (snd_hdac_is_power_on(&codec->core))
|
|
{
|
|
return;
|
|
}
|
|
|
|
/*
|
|
* Encode codec verb for getting vendor ID from root node.
|
|
* Refer Intel High Definition Audio Specification for more details.
|
|
*/
|
|
cmd = (codec->addr << 28) | (AC_NODE_ROOT << 20) |
|
|
(AC_VERB_PARAMETERS << 8) | AC_PAR_VENDOR_ID;
|
|
|
|
/*
|
|
* It will internally increment the runtime PM refcount,
|
|
* wake-up the audio codec controller and send the HW
|
|
* command for getting vendor ID. Once the vendor ID will be
|
|
* returned back, then it will decrement the runtime PM refcount
|
|
* and runtime suspend audio codec controller again (If refcount is
|
|
* zero) once auto suspend counter expires.
|
|
*/
|
|
codec->core.exec_verb(&codec->core, cmd, 0, &res);
|
|
}
|
|
#else
|
|
{
|
|
int codec_addr;
|
|
|
|
/*
|
|
* The filp_open() call below depends on the current task's fs_struct
|
|
* (current->fs), which may already be NULL if this is called during
|
|
* process teardown.
|
|
*/
|
|
if (current->fs == NULL)
|
|
return;
|
|
|
|
/* If device is runtime active, then return */
|
|
if (audio_pci_dev->dev.power.runtime_status == RPM_ACTIVE)
|
|
return;
|
|
|
|
for (codec_addr = 0; codec_addr < NV_HDA_MAX_CODECS; codec_addr++)
|
|
{
|
|
char filename[48];
|
|
NvU8 buf;
|
|
int ret;
|
|
|
|
ret = snprintf(filename, sizeof(filename),
|
|
"/proc/asound/card%d/codec#%d",
|
|
card->number, codec_addr);
|
|
|
|
if (ret > 0 && ret < sizeof(filename) &&
|
|
(os_open_and_read_file(filename, &buf, 1) == NV_OK))
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
#endif
|
|
#endif
|
|
}
|
|
|
|
static int nv_match_dev_state(const void *data, struct file *filp, unsigned fd)
|
|
{
|
|
nv_linux_state_t *nvl = NULL;
|
|
dev_t rdev = 0;
|
|
|
|
if (filp == NULL ||
|
|
filp->private_data == NULL ||
|
|
NV_FILE_INODE(filp) == NULL)
|
|
return 0;
|
|
|
|
rdev = (NV_FILE_INODE(filp))->i_rdev;
|
|
if (MAJOR(rdev) != NV_MAJOR_DEVICE_NUMBER)
|
|
return 0;
|
|
|
|
nvl = NV_GET_NVL_FROM_FILEP(filp);
|
|
if (nvl == NULL)
|
|
return 0;
|
|
|
|
return (data == nvl);
|
|
}
|
|
|
|
NvBool NV_API_CALL nv_is_gpu_accessible(nv_state_t *nv)
|
|
{
|
|
struct files_struct *files = current->files;
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
|
|
#ifdef NV_ITERATE_FD_PRESENT
|
|
return !!iterate_fd(files, 0, nv_match_dev_state, nvl);
|
|
#else
|
|
struct fdtable *fdtable;
|
|
int ret_val = 0;
|
|
int fd = 0;
|
|
|
|
if (files == NULL)
|
|
return 0;
|
|
|
|
spin_lock(&files->file_lock);
|
|
|
|
for (fdtable = files_fdtable(files); fd < fdtable->max_fds; fd++)
|
|
{
|
|
struct file *filp;
|
|
|
|
#ifdef READ_ONCE
|
|
filp = READ_ONCE(fdtable->fd[fd]);
|
|
#else
|
|
filp = ACCESS_ONCE(fdtable->fd[fd]);
|
|
smp_read_barrier_depends();
|
|
#endif
|
|
if (filp == NULL)
|
|
continue;
|
|
|
|
ret_val = nv_match_dev_state(nvl, filp, fd);
|
|
if (ret_val)
|
|
break;
|
|
}
|
|
|
|
spin_unlock(&files->file_lock);
|
|
|
|
return !!ret_val;
|
|
#endif
|
|
}
|
|
|
|
|
|
NvBool NV_API_CALL nv_platform_supports_s0ix(void)
|
|
{
|
|
#if defined(CONFIG_ACPI)
|
|
return (acpi_gbl_FADT.flags & ACPI_FADT_LOW_POWER_S0) != 0;
|
|
#else
|
|
return NV_FALSE;
|
|
#endif
|
|
}
|
|
|
|
NvBool NV_API_CALL nv_s2idle_pm_configured(void)
|
|
{
|
|
NvU8 buf[8];
|
|
|
|
#if defined(NV_SEQ_READ_ITER_PRESENT)
|
|
struct file *file;
|
|
ssize_t num_read;
|
|
struct kiocb kiocb;
|
|
struct iov_iter iter;
|
|
struct kvec iov = {
|
|
.iov_base = &buf,
|
|
.iov_len = sizeof(buf),
|
|
};
|
|
|
|
if (os_open_readonly_file("/sys/power/mem_sleep", (void **)&file) != NV_OK)
|
|
{
|
|
return NV_FALSE;
|
|
}
|
|
|
|
/*
|
|
* init_sync_kiocb() internally uses GPL licensed __get_task_ioprio() from
|
|
* v5.20-rc1.
|
|
*/
|
|
#if defined(NV_GET_TASK_IOPRIO_PRESENT)
|
|
memset(&kiocb, 0, sizeof(kiocb));
|
|
kiocb.ki_filp = file;
|
|
kiocb.ki_flags = iocb_flags(file);
|
|
kiocb.ki_ioprio = IOPRIO_DEFAULT;
|
|
#else
|
|
init_sync_kiocb(&kiocb, file);
|
|
#endif
|
|
|
|
kiocb.ki_pos = 0;
|
|
iov_iter_kvec(&iter, READ, &iov, 1, sizeof(buf));
|
|
|
|
num_read = seq_read_iter(&kiocb, &iter);
|
|
|
|
os_close_file((void *)file);
|
|
|
|
if (num_read != sizeof(buf))
|
|
{
|
|
return NV_FALSE;
|
|
}
|
|
#else
|
|
if (os_open_and_read_file("/sys/power/mem_sleep", buf,
|
|
sizeof(buf)) != NV_OK)
|
|
{
|
|
return NV_FALSE;
|
|
}
|
|
#endif
|
|
|
|
return (memcmp(buf, "[s2idle]", 8) == 0);
|
|
}
|
|
|
|
|
|
/*
|
|
* Function query system chassis info, to figure out if the platform is
|
|
* Laptop or Notebook.
|
|
* This function should be used when querying GPU form factor information is
|
|
* not possible via core RM or if querying both system and GPU form factor
|
|
* information is necessary.
|
|
*/
|
|
NvBool NV_API_CALL nv_is_chassis_notebook(void)
|
|
{
|
|
const char *chassis_type = dmi_get_system_info(DMI_CHASSIS_TYPE);
|
|
|
|
//
|
|
// Return true only for Laptop & Notebook
|
|
// As per SMBIOS spec Laptop = 9 and Notebook = 10
|
|
//
|
|
return (chassis_type && (!strcmp(chassis_type, "9") || !strcmp(chassis_type, "10")));
|
|
}
|
|
|
|
void NV_API_CALL nv_allow_runtime_suspend
|
|
(
|
|
nv_state_t *nv
|
|
)
|
|
{
|
|
#if defined(NV_PM_RUNTIME_AVAILABLE)
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
struct device *dev = nvl->dev;
|
|
|
|
spin_lock_irq(&dev->power.lock);
|
|
|
|
if (dev->power.runtime_auto == false)
|
|
{
|
|
dev->power.runtime_auto = true;
|
|
atomic_add_unless(&dev->power.usage_count, -1, 0);
|
|
}
|
|
|
|
spin_unlock_irq(&dev->power.lock);
|
|
#endif
|
|
}
|
|
|
|
void NV_API_CALL nv_disallow_runtime_suspend
|
|
(
|
|
nv_state_t *nv
|
|
)
|
|
{
|
|
#if defined(NV_PM_RUNTIME_AVAILABLE)
|
|
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
|
|
struct device *dev = nvl->dev;
|
|
|
|
spin_lock_irq(&dev->power.lock);
|
|
|
|
if (dev->power.runtime_auto == true)
|
|
{
|
|
dev->power.runtime_auto = false;
|
|
atomic_inc(&dev->power.usage_count);
|
|
}
|
|
|
|
spin_unlock_irq(&dev->power.lock);
|
|
#endif
|
|
}
|
|
|
|
NvU32 NV_API_CALL nv_get_os_type(void)
|
|
{
|
|
return OS_TYPE_LINUX;
|
|
}
|
|
|
|
void NV_API_CALL nv_flush_coherent_cpu_cache_range(nv_state_t *nv, NvU64 cpu_virtual, NvU64 size)
|
|
{
|
|
#if NVCPU_IS_PPC64LE
|
|
return nv_ibmnpu_cache_flush_range(nv, cpu_virtual, size);
|
|
#elif NVCPU_IS_AARCH64
|
|
|
|
NvU64 va, cbsize;
|
|
NvU64 end_cpu_virtual = cpu_virtual + size;
|
|
|
|
nv_printf(NV_DBG_INFO,
|
|
"Flushing CPU virtual range [0x%llx, 0x%llx)\n",
|
|
cpu_virtual, end_cpu_virtual);
|
|
|
|
cbsize = cache_line_size();
|
|
// Align address to line size
|
|
cpu_virtual = NV_ALIGN_UP(cpu_virtual, cbsize);
|
|
|
|
// Force eviction of any cache lines from the NUMA-onlined region.
|
|
for (va = cpu_virtual; va < end_cpu_virtual; va += cbsize)
|
|
{
|
|
asm volatile("dc civac, %0" : : "r" (va): "memory");
|
|
// Reschedule if necessary to avoid lockup warnings
|
|
cond_resched();
|
|
}
|
|
asm volatile("dsb sy" : : : "memory");
|
|
|
|
#endif
|
|
}
|
|
|
|
static struct resource *nv_next_resource(struct resource *p)
|
|
{
|
|
if (p->child != NULL)
|
|
return p->child;
|
|
|
|
while ((p->sibling == NULL) && (p->parent != NULL))
|
|
p = p->parent;
|
|
|
|
return p->sibling;
|
|
}
|
|
|
|
/*
|
|
* Function to get the correct PCI Bus memory window which can be mapped
|
|
* in the real mode emulator (emu).
|
|
* The function gets called during the initialization of the emu before
|
|
* remapping it to OS.
|
|
*/
|
|
void NV_API_CALL nv_get_updated_emu_seg(
|
|
NvU32 *start,
|
|
NvU32 *end
|
|
)
|
|
{
|
|
struct resource *p;
|
|
|
|
if (*start >= *end)
|
|
return;
|
|
|
|
for (p = iomem_resource.child; (p != NULL); p = nv_next_resource(p))
|
|
{
|
|
/* If we passed the resource we are looking for, stop */
|
|
if (p->start > *end)
|
|
{
|
|
p = NULL;
|
|
break;
|
|
}
|
|
|
|
/* Skip until we find a range that matches what we look for */
|
|
if (p->end < *start)
|
|
continue;
|
|
|
|
if ((p->end > *end) && (p->child))
|
|
continue;
|
|
|
|
if ((p->flags & IORESOURCE_MEM) != IORESOURCE_MEM)
|
|
continue;
|
|
|
|
/* Found a match, break */
|
|
break;
|
|
}
|
|
|
|
if (p != NULL)
|
|
{
|
|
*start = max((resource_size_t)*start, p->start);
|
|
*end = min((resource_size_t)*end, p->end);
|
|
}
|
|
}
|