/* * SPDX-FileCopyrightText: Copyright (c) 2015-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: MIT * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */ #include "conftest.h" #include "nvlink_os.h" #include "nvlink_linux.h" #include "nvlink_errors.h" #include "nvlink_export.h" #include "nv-linux.h" #include "nv-procfs.h" #include "nv-time.h" #include "nvlink_caps.h" #include #include #include #include #include #include #define MAX_ERROR_STRING 512 typedef struct nvlink_file_private { struct { /* A duped file descriptor for fabric_mgmt capability */ int fabric_mgmt; } capability_fds; } nvlink_file_private_t; #define NVLINK_SET_FILE_PRIVATE(filp, data) ((filp)->private_data = (data)) #define NVLINK_GET_FILE_PRIVATE(filp) ((nvlink_file_private_t *)(filp)->private_data) typedef struct { struct mutex lock; NvBool initialized; struct cdev cdev; dev_t devno; int opened; int major_devnum; } _nvlink_drvctx; // nvlink driver local state static _nvlink_drvctx nvlink_drvctx; #if defined(CONFIG_PROC_FS) #define NV_DEFINE_SINGLE_NVLINK_PROCFS_FILE(name) \ NV_DEFINE_SINGLE_PROCFS_FILE_READ_ONLY(name, nv_system_pm_lock) #endif #define NVLINK_PROCFS_DIR "driver/nvidia-nvlink" static struct proc_dir_entry *nvlink_procfs_dir = NULL; #if defined(CONFIG_PROC_FS) static int nvlink_is_procfs_available = 1; #else static int nvlink_is_procfs_available = 0; #endif static struct proc_dir_entry *nvlink_permissions = NULL; static int nv_procfs_read_permissions(struct seq_file *s, void *v) { // Restrict device node permissions - 0666. seq_printf(s, "%s: %u\n", "DeviceFileMode", 438); return 0; } NV_DEFINE_SINGLE_NVLINK_PROCFS_FILE(permissions); static void nvlink_permissions_exit(void) { if (!nvlink_permissions) { return; } NV_REMOVE_PROC_ENTRY(nvlink_permissions); nvlink_permissions = NULL; } static int nvlink_permissions_init(void) { if (!nvlink_procfs_dir) { return -EACCES; } nvlink_permissions = NV_CREATE_PROC_FILE("permissions", nvlink_procfs_dir, permissions, NULL); if (!nvlink_permissions) { return -EACCES; } return 0; } static void nvlink_procfs_exit(void) { nvlink_permissions_exit(); if (!nvlink_procfs_dir) { return; } NV_REMOVE_PROC_ENTRY(nvlink_procfs_dir); nvlink_procfs_dir = NULL; } static int nvlink_procfs_init(void) { int rc = 0; if (!nvlink_is_procfs_available) { return -EACCES; } nvlink_procfs_dir = NV_CREATE_PROC_DIR(NVLINK_PROCFS_DIR, NULL); if (!nvlink_procfs_dir) { return -EACCES; } rc = nvlink_permissions_init(); if (rc < 0) { goto cleanup; } return 0; cleanup: nvlink_procfs_exit(); return rc; } static int nvlink_fops_open(struct inode *inode, struct file *filp) { int rc = 0; nvlink_file_private_t *private = NULL; nvlink_print(NVLINK_DBG_INFO, "nvlink driver open\n"); mutex_lock(&nvlink_drvctx.lock); // nvlink lib driver is currently exclusive open. if (nvlink_drvctx.opened) { rc = -EBUSY; goto open_error; } private = (nvlink_file_private_t *)nvlink_malloc(sizeof(*private)); if (private == NULL) { rc = -ENOMEM; goto open_error; } private->capability_fds.fabric_mgmt = -1; NVLINK_SET_FILE_PRIVATE(filp, private); // mark our state as opened nvlink_drvctx.opened = NV_TRUE; open_error: mutex_unlock(&nvlink_drvctx.lock); return rc; } static int nvlink_fops_release(struct inode *inode, struct file *filp) { nvlink_file_private_t *private = NVLINK_GET_FILE_PRIVATE(filp); nvlink_print(NVLINK_DBG_INFO, "nvlink driver close\n"); mutex_lock(&nvlink_drvctx.lock); if (private->capability_fds.fabric_mgmt > 0) { nvlink_cap_release(private->capability_fds.fabric_mgmt); private->capability_fds.fabric_mgmt = -1; } nvlink_free(filp->private_data); NVLINK_SET_FILE_PRIVATE(filp, NULL); // mark the device as not opened nvlink_drvctx.opened = NV_FALSE; mutex_unlock(&nvlink_drvctx.lock); return 0; } static int nvlink_fops_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg) { nvlink_ioctrl_params ctrl_params = {0}; int param_size = _IOC_SIZE(cmd); void *param_buf = NULL; NvlStatus ret_val = 0; int rc = 0; // no buffer for simple _IO types if (param_size) { // allocate a buffer to hold user input param_buf = kzalloc(param_size, GFP_KERNEL); if (NULL == param_buf) { rc = -ENOMEM; goto nvlink_ioctl_fail; } // copy user input to kernel buffers. Simple _IOR() ioctls can skip this step. if (_IOC_DIR(cmd) & _IOC_WRITE) { // copy user input to local buffer if (copy_from_user(param_buf, (const void *)arg, param_size)) { rc = -EFAULT; goto nvlink_ioctl_fail; } } } ctrl_params.osPrivate = filp->private_data; ctrl_params.cmd = _IOC_NR(cmd); ctrl_params.buf = param_buf; ctrl_params.size = param_size; ret_val = nvlink_lib_ioctl_ctrl(&ctrl_params); if (NVL_SUCCESS != ret_val) { rc = -EINVAL; goto nvlink_ioctl_fail; } // no copy for write-only ioctl if ((param_size) && (_IOC_DIR(cmd) & _IOC_READ)) { if (copy_to_user((void *)arg, ctrl_params.buf, ctrl_params.size)) { rc = -EFAULT; goto nvlink_ioctl_fail; } } nvlink_ioctl_fail: if (param_buf) { kfree(param_buf); } return rc; } #define NV_FILE_INODE(file) (file)->f_inode static long nvlink_fops_unlocked_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { return nvlink_fops_ioctl(NV_FILE_INODE(file), file, cmd, arg); } static const struct file_operations nvlink_fops = { .owner = THIS_MODULE, .open = nvlink_fops_open, .release = nvlink_fops_release, #if defined(NV_FILE_OPERATIONS_HAS_IOCTL) .ioctl = nvlink_fops_ioctl, #endif .unlocked_ioctl = nvlink_fops_unlocked_ioctl, }; int __init nvlink_core_init(void) { NvlStatus ret_val; int rc; if (NV_TRUE == nvlink_drvctx.initialized) { nvlink_print(NVLINK_DBG_ERRORS, "nvlink core interface already initialized\n"); return -EBUSY; } mutex_init(&nvlink_drvctx.lock); ret_val = nvlink_lib_initialize(); if (NVL_SUCCESS != ret_val) { nvlink_print(NVLINK_DBG_ERRORS, "Failed to initialize driver : %d\n", ret_val); rc = -ENODEV; goto nvlink_lib_initialize_fail; } rc = alloc_chrdev_region(&nvlink_drvctx.devno, 0, NVLINK_NUM_MINOR_DEVICES, NVLINK_DEVICE_NAME); if (rc < 0) { nvlink_print(NVLINK_DBG_ERRORS, "alloc_chrdev_region failed: %d\n", rc); goto alloc_chrdev_region_fail; } nvlink_drvctx.major_devnum = MAJOR(nvlink_drvctx.devno); nvlink_print(NVLINK_DBG_INFO, "Nvlink Core is being initialized, major device number %d\n", nvlink_drvctx.major_devnum); cdev_init(&nvlink_drvctx.cdev, &nvlink_fops); nvlink_drvctx.cdev.owner = THIS_MODULE; rc = cdev_add(&nvlink_drvctx.cdev, nvlink_drvctx.devno, NVLINK_NUM_MINOR_DEVICES); if (rc < 0) { nvlink_print(NVLINK_DBG_ERRORS, " Unable to create cdev\n"); goto cdev_add_fail; } rc = nvlink_procfs_init(); if (rc < 0) { goto procfs_init_fail; } rc = nvlink_cap_init(NVLINK_PROCFS_DIR); if (rc < 0) { nvlink_print(NVLINK_DBG_ERRORS, " Unable to create capability\n"); goto cap_init_fail; } nvlink_drvctx.initialized = NV_TRUE; return 0; cap_init_fail: nvlink_procfs_exit(); procfs_init_fail: cdev_del(&nvlink_drvctx.cdev); cdev_add_fail: unregister_chrdev_region(nvlink_drvctx.devno, NVLINK_NUM_MINOR_DEVICES); alloc_chrdev_region_fail: nvlink_lib_unload(); nvlink_lib_initialize_fail: nv_mutex_destroy(&nvlink_drvctx.lock); return rc; } void nvlink_core_exit(void) { if (NV_FALSE == nvlink_drvctx.initialized) { return; } nvlink_cap_exit(); nvlink_procfs_exit(); cdev_del(&nvlink_drvctx.cdev); unregister_chrdev_region(nvlink_drvctx.devno, NVLINK_NUM_MINOR_DEVICES); nvlink_lib_unload(); nv_mutex_destroy(&nvlink_drvctx.lock); nvlink_print(NVLINK_DBG_INFO, "Unregistered Nvlink Core, major device number %d\n", nvlink_drvctx.major_devnum); } void nvlink_print ( const char *file, int line, const char *function, int log_level, const char *fmt, ... ) { va_list arglist; char nv_string[MAX_ERROR_STRING]; char *sys_log_level; switch (log_level) { case NVLINK_DBG_LEVEL_INFO: sys_log_level = KERN_INFO; break; case NVLINK_DBG_LEVEL_SETUP: sys_log_level = KERN_DEBUG; break; case NVLINK_DBG_LEVEL_USERERRORS: sys_log_level = KERN_NOTICE; break; case NVLINK_DBG_LEVEL_WARNINGS: sys_log_level = KERN_WARNING; break; case NVLINK_DBG_LEVEL_ERRORS: sys_log_level = KERN_ERR; break; default: sys_log_level = KERN_INFO; break; } va_start(arglist, fmt); vsnprintf(nv_string, sizeof(nv_string), fmt, arglist); va_end(arglist); nv_string[sizeof(nv_string) - 1] = '\0'; printk("%snvidia-nvlink: %s", sys_log_level, nv_string); } void * nvlink_malloc(NvLength size) { return kmalloc(size, GFP_KERNEL); } void nvlink_free(void *ptr) { return kfree(ptr); } char * nvlink_strcpy(char *dest, const char *src) { return strcpy(dest, src); } int nvlink_strcmp(const char *dest, const char *src) { return strcmp(dest, src); } NvLength nvlink_strlen(const char *s) { return strlen(s); } int nvlink_snprintf(char *dest, NvLength size, const char *fmt, ...) { va_list arglist; int chars_written; va_start(arglist, fmt); chars_written = vsnprintf(dest, size, fmt, arglist); va_end(arglist); return chars_written; } NvU32 nvlink_memRd32(const volatile void * address) { return (*(const volatile NvU32*)(address)); } void nvlink_memWr32(volatile void *address, NvU32 data) { (*(volatile NvU32 *)(address)) = data; } NvU64 nvlink_memRd64(const volatile void * address) { return (*(const volatile NvU64 *)(address)); } void nvlink_memWr64(volatile void *address, NvU64 data) { (*(volatile NvU64 *)(address)) = data; } void * nvlink_memset(void *dest, int value, NvLength size) { return memset(dest, value, size); } void * nvlink_memcpy(void *dest, const void *src, NvLength size) { return memcpy(dest, src, size); } int nvlink_memcmp(const void *s1, const void *s2, NvLength size) { return memcmp(s1, s2, size); } /* * Sleep for specified milliseconds. Yields the CPU to scheduler. */ void nvlink_sleep(unsigned int ms) { NV_STATUS status; status = nv_sleep_ms(ms); if (status != NV_OK) { if (printk_ratelimit()) { nvlink_print(NVLINK_DBG_ERRORS, "NVLink: requested sleep duration" " %d msec exceeded %d msec\n", ms, NV_MAX_ISR_DELAY_MS); WARN_ON(1); } } } void nvlink_assert(int cond) { if ((cond) == 0x0) { if (printk_ratelimit()) { nvlink_print(NVLINK_DBG_ERRORS, "NVLink: Assertion failed!\n"); WARN_ON(1); } dbg_breakpoint(); } } void * nvlink_allocLock() { struct semaphore *sema; sema = nvlink_malloc(sizeof(*sema)); if (sema == NULL) { nvlink_print(NVLINK_DBG_ERRORS, "Failed to allocate sema!\n"); return NULL; } sema_init(sema, 1); return sema; } void nvlink_acquireLock(void *hLock) { down(hLock); } void nvlink_releaseLock(void *hLock) { up(hLock); } void nvlink_freeLock(void *hLock) { if (NULL == hLock) { return; } NVLINK_FREE(hLock); } NvBool nvlink_isLockOwner(void *hLock) { return NV_TRUE; } NvlStatus nvlink_acquire_fabric_mgmt_cap(void *osPrivate, NvU64 capDescriptor) { int dup_fd = -1; nvlink_file_private_t *private_data = (nvlink_file_private_t *)osPrivate; if (private_data == NULL) { return NVL_BAD_ARGS; } dup_fd = nvlink_cap_acquire((int)capDescriptor, NVLINK_CAP_FABRIC_MANAGEMENT); if (dup_fd < 0) { return NVL_ERR_OPERATING_SYSTEM; } private_data->capability_fds.fabric_mgmt = dup_fd; return NVL_SUCCESS; } int nvlink_is_fabric_manager(void *osPrivate) { nvlink_file_private_t *private_data = (nvlink_file_private_t *)osPrivate; /* Make sure that fabric mgmt capbaility fd is valid */ if ((private_data == NULL) || (private_data->capability_fds.fabric_mgmt < 0)) { return 0; } return 1; } int nvlink_is_admin(void) { return NV_IS_SUSER(); }