open-gpu-kernel-modules/kernel-open/nvidia/nv-dmabuf.c

/*
 * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: MIT
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */
#include <linux/dma-buf.h>
#include "nv-dmabuf.h"



#if defined(CONFIG_DMA_SHARED_BUFFER)

//
// The Linux kernel's dma_length in struct scatterlist is unsigned int
// which limits the maximum sg length to 4GB - 1.
// To get around this limitation, the BAR1 scatterlist returned by RM
// is split into (4GB - PAGE_SIZE) sized chunks to build the sg_table.
//
#define NV_DMA_BUF_SG_MAX_LEN         ((NvU32)(NVBIT64(32) - PAGE_SIZE))

typedef struct nv_dma_buf_mem_handle
{
    NvHandle h_memory;
    NvU64    offset;
    NvU64    size;
    NvU64    bar1_va;
} nv_dma_buf_mem_handle_t;

typedef struct nv_dma_buf_file_private
{
    nv_state_t              *nv;
    NvHandle                 h_client;
    NvHandle                 h_device;
    NvHandle                 h_subdevice;
    NvU32                    total_objects;
    NvU32                    num_objects;
    NvU64                    total_size;
    NvU64                    attached_size;
    struct mutex             lock;
    nv_dma_buf_mem_handle_t *handles;
    NvU64                    bar1_va_ref_count;
    void                    *mig_info;
    NvBool                   can_mmap;
} nv_dma_buf_file_private_t;

static void
nv_dma_buf_free_file_private(
    nv_dma_buf_file_private_t *priv
)
{
    if (priv == NULL)
    {
        return;
    }

    if (priv->handles != NULL)
    {
        NV_KFREE(priv->handles, priv->total_objects * sizeof(priv->handles[0]));
        priv->handles = NULL;
    }

    mutex_destroy(&priv->lock);

    NV_KFREE(priv, sizeof(nv_dma_buf_file_private_t));
}

static nv_dma_buf_file_private_t*
nv_dma_buf_alloc_file_private(
    NvU32 num_handles
)
{
    nv_dma_buf_file_private_t *priv = NULL;

    NV_KMALLOC(priv, sizeof(nv_dma_buf_file_private_t));
    if (priv == NULL)
    {
        return NULL;
    }

    memset(priv, 0, sizeof(nv_dma_buf_file_private_t));

    mutex_init(&priv->lock);

    NV_KMALLOC(priv->handles, num_handles * sizeof(priv->handles[0]));
    if (priv->handles == NULL)
    {
        goto failed;
    }

    memset(priv->handles, 0, num_handles * sizeof(priv->handles[0]));

    return priv;

failed:
    nv_dma_buf_free_file_private(priv);

    return NULL;
}

// Must be called with RMAPI lock and GPU lock taken
static void
nv_dma_buf_undup_mem_handles_unlocked(
    nvidia_stack_t            *sp,
    NvU32                      index,
    NvU32                      num_objects,
    nv_dma_buf_file_private_t *priv
)
{
    NvU32 i = 0;

    for (i = index; i < num_objects; i++)
    {
        if (priv->handles[i].h_memory == 0)
        {
            continue;
        }

        rm_dma_buf_undup_mem_handle(sp, priv->nv, priv->h_client,
                                    priv->handles[i].h_memory);

        priv->attached_size -= priv->handles[i].size;
        priv->handles[i].h_memory = 0;
        priv->handles[i].offset = 0;
        priv->handles[i].size = 0;
        priv->num_objects--;
    }
}

static void
nv_dma_buf_undup_mem_handles(
    nvidia_stack_t            *sp,
    NvU32                      index,
    NvU32                      num_objects,
    nv_dma_buf_file_private_t *priv
)
{
    NV_STATUS status;

    status = rm_acquire_api_lock(sp);
    if (WARN_ON(status != NV_OK))
    {
        return;
    }

    status = rm_acquire_all_gpus_lock(sp);
    if (WARN_ON(status != NV_OK))
    {
        goto unlock_api_lock;
    }

    nv_dma_buf_undup_mem_handles_unlocked(sp, index, num_objects, priv);

    rm_release_all_gpus_lock(sp);

unlock_api_lock:
    rm_release_api_lock(sp);
}

static NV_STATUS
nv_dma_buf_dup_mem_handles(
    nvidia_stack_t                  *sp,
    nv_dma_buf_file_private_t       *priv,
    nv_ioctl_export_to_dma_buf_fd_t *params
)
{
    NV_STATUS status = NV_OK;
    NvU32 index = params->index;
    NvU32 count = 0;
    NvU32 i = 0;

    status = rm_acquire_api_lock(sp);
    if (status != NV_OK)
    {
        return status;
    }

    status = rm_acquire_gpu_lock(sp, priv->nv);
    if (status != NV_OK)
    {
        goto unlock_api_lock;
    }

    for (i = 0; i < params->numObjects; i++)
    {
        NvHandle h_memory_duped = 0;

        if (priv->handles[index].h_memory != 0)
        {
            status = NV_ERR_IN_USE;
            goto failed;
        }

        if (params->sizes[i] > priv->total_size - priv->attached_size)
        {
            status = NV_ERR_INVALID_ARGUMENT;
            goto failed;
        }

        status = rm_dma_buf_dup_mem_handle(sp, priv->nv,
                                           params->hClient,
                                           priv->h_client,
                                           priv->h_device,
                                           priv->h_subdevice,
                                           priv->mig_info,
                                           params->handles[i],
                                           params->offsets[i],
                                           params->sizes[i],
                                           &h_memory_duped);
        if (status != NV_OK)
        {
            goto failed;
        }

        priv->attached_size += params->sizes[i];
        priv->handles[index].h_memory = h_memory_duped;
        priv->handles[index].offset = params->offsets[i];
        priv->handles[index].size = params->sizes[i];
        priv->num_objects++;
        index++;
        count++;
    }

    if ((priv->num_objects == priv->total_objects) &&
        (priv->attached_size != priv->total_size))
    {
        status = NV_ERR_INVALID_ARGUMENT;
        goto failed;
    }

    rm_release_gpu_lock(sp, priv->nv);

    rm_release_api_lock(sp);

    return NV_OK;

failed:
    nv_dma_buf_undup_mem_handles_unlocked(sp, params->index, count, priv);

    rm_release_gpu_lock(sp, priv->nv);

unlock_api_lock:
    rm_release_api_lock(sp);

    return status;
}

// Must be called with RMAPI lock and GPU lock taken
static void
nv_dma_buf_unmap_unlocked(
    nvidia_stack_t *sp,
    nv_dma_device_t *peer_dma_dev,
    nv_dma_buf_file_private_t *priv,
    struct sg_table *sgt,
    NvU32 mapped_handle_count
)
{
    NV_STATUS status;
    NvU32 i;
    NvU64 dma_len;
    NvU64 dma_addr;
    NvBool bar1_unmap_needed;
    struct scatterlist *sg = NULL;

    bar1_unmap_needed = (priv->bar1_va_ref_count == 0);

    sg = sgt->sgl;
    for (i = 0; i < mapped_handle_count; i++)
    {
        NvU64 handle_size = priv->handles[i].size;

        dma_addr = sg_dma_address(sg);
        dma_len  = 0;

        //
        // Seek ahead in the scatterlist until the handle size is covered.
        // IOVA unmap can then be done all at once instead of doing it
        // one sg at a time.
        //
        while(handle_size != dma_len)
        {
            dma_len += sg_dma_len(sg);
            sg = sg_next(sg);
        }

        nv_dma_unmap_peer(peer_dma_dev, (dma_len / os_page_size), dma_addr);

        if (bar1_unmap_needed)
        {
            status = rm_dma_buf_unmap_mem_handle(sp, priv->nv, priv->h_client,
                                                 priv->handles[i].h_memory,
                                                 priv->handles[i].size,
                                                 priv->handles[i].bar1_va);
            WARN_ON(status != NV_OK);
        }
    }
}

static struct sg_table*
nv_dma_buf_map(
    struct dma_buf_attachment *attachment,
    enum dma_data_direction direction
)
{
    NV_STATUS status;
    nvidia_stack_t *sp = NULL;
    struct scatterlist *sg = NULL;
    struct sg_table *sgt = NULL;
    struct dma_buf *buf = attachment->dmabuf;
    struct device *dev = attachment->dev;
    nv_dma_buf_file_private_t *priv = buf->priv;
    nv_dma_device_t peer_dma_dev = {{ 0 }};
    NvBool bar1_map_needed;
    NvBool bar1_unmap_needed;
    NvU32 mapped_handle_count = 0;
    NvU32 num_sg_entries = 0;
    NvU32 i = 0;
    int rc = 0;

    //
    // We support importers that are able to handle MMIO resources
    // not backed by struct page. This will need to be revisited
    // when dma-buf support for P9 will be added.
    //
#if defined(NV_DMA_BUF_HAS_DYNAMIC_ATTACHMENT) && \
    defined(NV_DMA_BUF_ATTACHMENT_HAS_PEER2PEER)
    if (dma_buf_attachment_is_dynamic(attachment) &&
        !attachment->peer2peer)
    {
        nv_printf(NV_DBG_ERRORS,
                  "NVRM: failed to map dynamic attachment with no P2P support\n");
        return NULL;
    }
#endif

    mutex_lock(&priv->lock);

    if (priv->num_objects != priv->total_objects)
    {
        goto unlock_priv;
    }

    rc = nv_kmem_cache_alloc_stack(&sp);
    if (rc != 0)
    {
        goto unlock_priv;
    }

    status = rm_acquire_api_lock(sp);
    if (status != NV_OK)
    {
        goto free_sp;
    }

    status = rm_acquire_gpu_lock(sp, priv->nv);
    if (status != NV_OK)
    {
        goto unlock_api_lock;
    }

    NV_KMALLOC(sgt, sizeof(struct sg_table));
    if (sgt == NULL)
    {
        goto unlock_gpu_lock;
    }

    memset(sgt, 0, sizeof(struct sg_table));
    //
    // Pre-calculate number of sg entries we need based on handle size.
    // This is needed to allocate sg_table.
    //
    for (i = 0; i < priv->num_objects; i++)
    {
        NvU64 count = priv->handles[i].size + NV_DMA_BUF_SG_MAX_LEN - 1;
        do_div(count, NV_DMA_BUF_SG_MAX_LEN);
        num_sg_entries += count;
    }

    //
    // RM currently returns contiguous BAR1, so we create as many
    // sg entries as num_sg_entries calculated above.
    // When RM can alloc discontiguous BAR1, this code will need to be revisited.
    //
    rc = sg_alloc_table(sgt, num_sg_entries, GFP_KERNEL);
    if (rc != 0)
    {
        goto free_sgt;
    }

    peer_dma_dev.dev = dev;
    peer_dma_dev.addressable_range.limit = (NvU64)dev->dma_mask;
    bar1_map_needed = bar1_unmap_needed = (priv->bar1_va_ref_count == 0);

    sg = sgt->sgl;
    for (i = 0; i < priv->num_objects; i++)
    {
        NvU64 dma_addr;
        NvU64 dma_len;

        if (bar1_map_needed)
        {
            status = rm_dma_buf_map_mem_handle(sp, priv->nv, priv->h_client,
                                               priv->handles[i].h_memory,
                                               priv->handles[i].offset,
                                               priv->handles[i].size,
                                               &priv->handles[i].bar1_va);
            if (status != NV_OK)
            {
                goto unmap_handles;
            }
        }

        mapped_handle_count++;

        dma_addr = priv->handles[i].bar1_va;
        dma_len  = priv->handles[i].size;

        //
        // IOVA map the full handle at once and then breakdown the range
        // (dma_addr, dma_addr + dma_len) into smaller sg entries.
        //
        status = nv_dma_map_peer(&peer_dma_dev, priv->nv->dma_dev,
                                 0x1, (dma_len / os_page_size), &dma_addr);
        if (status != NV_OK)
        {
            if (bar1_unmap_needed)
            {
                // Unmap the recently mapped memory handle
                (void) rm_dma_buf_unmap_mem_handle(sp, priv->nv, priv->h_client,
                                                   priv->handles[i].h_memory,
                                                   priv->handles[i].size,
                                                   priv->handles[i].bar1_va);
            }

            mapped_handle_count--;

            // Unmap remaining memory handles
            goto unmap_handles;
        }

        while(dma_len != 0)
        {
            NvU32 sg_len = NV_MIN(dma_len, NV_DMA_BUF_SG_MAX_LEN);

            sg_set_page(sg, NULL, sg_len, 0);
            sg_dma_address(sg) = (dma_addr_t)dma_addr;
            sg_dma_len(sg) = sg_len;
            dma_addr += sg_len;
            dma_len -= sg_len;
            sg = sg_next(sg);
        }
    }

    priv->bar1_va_ref_count++;

    rm_release_gpu_lock(sp, priv->nv);

    rm_release_api_lock(sp);

    nv_kmem_cache_free_stack(sp);

    mutex_unlock(&priv->lock);

    return sgt;

unmap_handles:
    nv_dma_buf_unmap_unlocked(sp, &peer_dma_dev, priv, sgt, mapped_handle_count);

    sg_free_table(sgt);

free_sgt:
    NV_KFREE(sgt, sizeof(struct sg_table));

unlock_gpu_lock:
    rm_release_gpu_lock(sp, priv->nv);

unlock_api_lock:
    rm_release_api_lock(sp);

free_sp:
    nv_kmem_cache_free_stack(sp);

unlock_priv:
    mutex_unlock(&priv->lock);

    return NULL;
}

static void
nv_dma_buf_unmap(
    struct dma_buf_attachment *attachment,
    struct sg_table *sgt,
    enum dma_data_direction direction
)
{
    NV_STATUS status;
    struct dma_buf *buf = attachment->dmabuf;
    struct device *dev = attachment->dev;
    nvidia_stack_t *sp = NULL;
    nv_dma_buf_file_private_t *priv = buf->priv;
    nv_dma_device_t peer_dma_dev = {{ 0 }};
    int rc = 0;

    mutex_lock(&priv->lock);

    if (priv->num_objects != priv->total_objects)
    {
        goto unlock_priv;
    }

    rc = nv_kmem_cache_alloc_stack(&sp);
    if (WARN_ON(rc != 0))
    {
        goto unlock_priv;
    }

    status = rm_acquire_api_lock(sp);
    if (WARN_ON(status != NV_OK))
    {
        goto free_sp;
    }

    status = rm_acquire_gpu_lock(sp, priv->nv);
    if (WARN_ON(status != NV_OK))
    {
        goto unlock_api_lock;
    }

    peer_dma_dev.dev = dev;
    peer_dma_dev.addressable_range.limit = (NvU64)dev->dma_mask;

    priv->bar1_va_ref_count--;

    nv_dma_buf_unmap_unlocked(sp, &peer_dma_dev, priv, sgt, priv->num_objects);

    sg_free_table(sgt);

    NV_KFREE(sgt, sizeof(struct sg_table));

    rm_release_gpu_lock(sp, priv->nv);

unlock_api_lock:
    rm_release_api_lock(sp);

free_sp:
    nv_kmem_cache_free_stack(sp);

unlock_priv:
    mutex_unlock(&priv->lock);
}

static void
nv_dma_buf_release(
    struct dma_buf *buf
)
{
    int rc = 0;
    nvidia_stack_t *sp = NULL;
    nv_dma_buf_file_private_t *priv = buf->priv;
    nv_state_t *nv;

    if (priv == NULL)
    {
        return;
    }

    nv = priv->nv;

    rc = nv_kmem_cache_alloc_stack(&sp);
    if (WARN_ON(rc != 0))
    {
        return;
    }

    nv_dma_buf_undup_mem_handles(sp, 0, priv->num_objects, priv);

    rm_dma_buf_put_client_and_device(sp, priv->nv, priv->h_client, priv->h_device,
                                     priv->h_subdevice, priv->mig_info);

    nv_dma_buf_free_file_private(priv);
    buf->priv = NULL;

    nvidia_dev_put(nv->gpu_id, sp);

    nv_kmem_cache_free_stack(sp);

    return;
}

static int
nv_dma_buf_mmap(
    struct dma_buf *buf,
    struct vm_area_struct *vma
)
{
    // TODO: Check can_mmap flag

    return -ENOTSUPP;
}

#if defined(NV_DMA_BUF_OPS_HAS_KMAP) || \
    defined(NV_DMA_BUF_OPS_HAS_MAP)
static void*
nv_dma_buf_kmap_stub(
    struct dma_buf *buf,
    unsigned long page_num
)
{
    return NULL;
}

static void
nv_dma_buf_kunmap_stub(
    struct dma_buf *buf,
    unsigned long page_num,
    void *addr
)
{
    return;
}
#endif

#if defined(NV_DMA_BUF_OPS_HAS_KMAP_ATOMIC) || \
    defined(NV_DMA_BUF_OPS_HAS_MAP_ATOMIC)
static void*
nv_dma_buf_kmap_atomic_stub(
    struct dma_buf *buf,
    unsigned long page_num
)
{
    return NULL;
}

static void
nv_dma_buf_kunmap_atomic_stub(
    struct dma_buf *buf,
    unsigned long page_num,
    void *addr
)
{
    return;
}
#endif

//
// Note: Some of the dma-buf operations are mandatory in some kernels.
// So stubs are added to prevent dma_buf_export() failure.
// The actual implementations of these interfaces is not really required
// for the export operation to work.
//
// Same functions are used for kmap*/map* because of this commit:
// f9b67f0014cb: dma-buf: Rename dma-ops to prevent conflict with kunmap_atomic
//
static const struct dma_buf_ops nv_dma_buf_ops = {
    .map_dma_buf   = nv_dma_buf_map,
    .unmap_dma_buf = nv_dma_buf_unmap,
    .release       = nv_dma_buf_release,
    .mmap          = nv_dma_buf_mmap,
#if defined(NV_DMA_BUF_OPS_HAS_KMAP)
    .kmap          = nv_dma_buf_kmap_stub,
    .kunmap        = nv_dma_buf_kunmap_stub,
#endif
#if defined(NV_DMA_BUF_OPS_HAS_KMAP_ATOMIC)
    .kmap_atomic   = nv_dma_buf_kmap_atomic_stub,
    .kunmap_atomic = nv_dma_buf_kunmap_atomic_stub,
#endif
#if defined(NV_DMA_BUF_OPS_HAS_MAP)
    .map          = nv_dma_buf_kmap_stub,
    .unmap        = nv_dma_buf_kunmap_stub,
#endif
#if defined(NV_DMA_BUF_OPS_HAS_MAP_ATOMIC)
    .map_atomic   = nv_dma_buf_kmap_atomic_stub,
    .unmap_atomic = nv_dma_buf_kunmap_atomic_stub,
#endif
};

static NV_STATUS
nv_dma_buf_create(
    nv_state_t *nv,
    nv_ioctl_export_to_dma_buf_fd_t *params
)
{
    int rc = 0;
    NV_STATUS status;
    nvidia_stack_t *sp = NULL;
    struct dma_buf *buf = NULL;
    nv_dma_buf_file_private_t *priv = NULL;
    NvU32 gpu_id = nv->gpu_id;

    if (!nv->dma_buf_supported)
    {
        return NV_ERR_NOT_SUPPORTED;
    }

    if (params->index > (params->totalObjects - params->numObjects))
    {
        return NV_ERR_INVALID_ARGUMENT;
    }

    priv = nv_dma_buf_alloc_file_private(params->totalObjects);
    if (priv == NULL)
    {
        nv_printf(NV_DBG_ERRORS, "NVRM: failed to allocate dma-buf private\n");
        return NV_ERR_NO_MEMORY;
    }

    priv->total_objects = params->totalObjects;
    priv->total_size    = params->totalSize;
    priv->nv            = nv;
    priv->can_mmap      = NV_FALSE;

    rc = nv_kmem_cache_alloc_stack(&sp);
    if (rc != 0)
    {
        status = NV_ERR_NO_MEMORY;
        goto cleanup_priv;
    }

    rc = nvidia_dev_get(gpu_id, sp);
    if (rc != 0)
    {
        status = NV_ERR_OPERATING_SYSTEM;
        goto cleanup_sp;
    }

    status = rm_dma_buf_get_client_and_device(sp, priv->nv,
                                              params->hClient,
                                              &priv->h_client,
                                              &priv->h_device,
                                              &priv->h_subdevice,
                                              &priv->mig_info);
    if (status != NV_OK)
    {
        goto cleanup_device;
    }

    status = nv_dma_buf_dup_mem_handles(sp, priv, params);
    if (status != NV_OK)
    {
        goto cleanup_client_and_device;
    }

#if (NV_DMA_BUF_EXPORT_ARGUMENT_COUNT == 1)
    {
        DEFINE_DMA_BUF_EXPORT_INFO(exp_info);

        exp_info.ops   = &nv_dma_buf_ops;
        exp_info.size  = params->totalSize;
        exp_info.flags = O_RDWR | O_CLOEXEC;
        exp_info.priv  = priv;

        buf = dma_buf_export(&exp_info);
    }
#elif (NV_DMA_BUF_EXPORT_ARGUMENT_COUNT == 4)
    buf = dma_buf_export(priv, &nv_dma_buf_ops,
                         params->totalSize, O_RDWR | O_CLOEXEC);
#elif (NV_DMA_BUF_EXPORT_ARGUMENT_COUNT == 5)
    buf = dma_buf_export(priv, &nv_dma_buf_ops,
                         params->totalSize, O_RDWR | O_CLOEXEC, NULL);
#endif

    if (IS_ERR(buf))
    {
        nv_printf(NV_DBG_ERRORS, "NVRM: failed to create dma-buf\n");

        status = NV_ERR_OPERATING_SYSTEM;

        goto cleanup_handles;
    }

    nv_kmem_cache_free_stack(sp);

    rc = dma_buf_fd(buf, O_RDWR | O_CLOEXEC);
    if (rc < 0)
    {
        nv_printf(NV_DBG_ERRORS, "NVRM: failed to get dma-buf file descriptor\n");

        //
        // If dma-buf is successfully created, the dup'd handles
        // clean-up should be done by the release callback.
        //
        dma_buf_put(buf);

        return NV_ERR_OPERATING_SYSTEM;
    }

    params->fd = rc;

    return NV_OK;

cleanup_handles:
    nv_dma_buf_undup_mem_handles(sp, 0, priv->num_objects, priv);

cleanup_client_and_device:
    rm_dma_buf_put_client_and_device(sp, priv->nv, priv->h_client, priv->h_device,
                                     priv->h_subdevice, priv->mig_info);

cleanup_device:
    nvidia_dev_put(gpu_id, sp);

cleanup_sp:
    nv_kmem_cache_free_stack(sp);

cleanup_priv:
    nv_dma_buf_free_file_private(priv);

    return status;
}

static NV_STATUS
nv_dma_buf_reuse(
    nv_state_t *nv,
    nv_ioctl_export_to_dma_buf_fd_t *params
)
{
    int rc = 0;
    NV_STATUS status = NV_OK;
    nvidia_stack_t *sp = NULL;
    struct dma_buf *buf = NULL;
    nv_dma_buf_file_private_t *priv = NULL;

    buf = dma_buf_get(params->fd);
    if (IS_ERR(buf))
    {
        nv_printf(NV_DBG_ERRORS, "NVRM: failed to get dma-buf\n");
        return NV_ERR_OPERATING_SYSTEM;
    }


    if (buf->ops != &nv_dma_buf_ops)
    {
        nv_printf(NV_DBG_ERRORS, "NVRM: Invalid dma-buf fd\n");
        status = NV_ERR_INVALID_ARGUMENT;
        goto cleanup_dmabuf;
    }


    priv = buf->priv;

    if (priv == NULL)
    {
        status = NV_ERR_OPERATING_SYSTEM;
        goto cleanup_dmabuf;
    }

    rc = mutex_lock_interruptible(&priv->lock);
    if (rc != 0)
    {
        status = NV_ERR_OPERATING_SYSTEM;
        goto cleanup_dmabuf;
    }


    if ((priv->total_objects < params->numObjects) ||
        (params->index > (priv->total_objects - params->numObjects)))



    {
        status = NV_ERR_INVALID_ARGUMENT;
        goto unlock_priv;
    }

    rc = nv_kmem_cache_alloc_stack(&sp);
    if (rc != 0)
    {
        status = NV_ERR_NO_MEMORY;
        goto unlock_priv;
    }

    status = nv_dma_buf_dup_mem_handles(sp, priv, params);
    if (status != NV_OK)
    {
        goto cleanup_sp;
    }

cleanup_sp:
    nv_kmem_cache_free_stack(sp);

unlock_priv:
    mutex_unlock(&priv->lock);

cleanup_dmabuf:
    dma_buf_put(buf);

    return status;
}
#endif // CONFIG_DMA_SHARED_BUFFER

NV_STATUS
nv_dma_buf_export(
    nv_state_t *nv,
    nv_ioctl_export_to_dma_buf_fd_t *params
)
{
#if defined(CONFIG_DMA_SHARED_BUFFER)
    NV_STATUS status;

    if ((params == NULL) ||
        (params->totalSize == 0) ||
        (params->numObjects == 0) ||
        (params->totalObjects == 0) ||
        (params->numObjects > NV_DMABUF_EXPORT_MAX_HANDLES) ||
        (params->numObjects > params->totalObjects))
    {
        return NV_ERR_INVALID_ARGUMENT;
    }

    //
    // If fd >= 0, dma-buf already exists with this fd, so get dma-buf from fd.
    // If fd == -1, dma-buf is not created yet, so create it and then store
    // additional handles.
    //
    if (params->fd == -1)
    {
        status = nv_dma_buf_create(nv, params);
    }
    else if (params->fd >= 0)
    {
        status = nv_dma_buf_reuse(nv, params);
    }
    else
    {
        status = NV_ERR_INVALID_ARGUMENT;
    }

    return status;
#else
    return NV_ERR_NOT_SUPPORTED;
#endif // CONFIG_DMA_SHARED_BUFFER
}