worker_rdmo.c

Go to the documentation of this file.

/*
 * Copyright (c) 2024 NVIDIA CORPORATION AND AFFILIATES.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification, are permitted
 * provided that the following conditions are met:
 *     * Redistributions of source code must retain the above copyright notice, this list of
 *       conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright notice, this list of
 *       conditions and the following disclaimer in the documentation and/or other materials
 *       provided with the distribution.
 *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
 *       to endorse or promote products derived from this software without specific prior written
 *       permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
 * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */
 
#include <stdint.h>
#include <stdlib.h>
 
#include <doca_buf.h>
#include <doca_pe.h>
#include "worker_rdmo.h"
 
DOCA_LOG_REGISTER(UROM::WORKER::RDMO);
 
static uint64_t rdmo_id;         /* RDMO plugin id, id is generated by UROM lib and
                      * will be updated in init function
                      */
static uint64_t rdmo_version = 0x01; /* RDMO plugin host version */
 
/* RDMO task metadata */
struct doca_rdmo_task_data {
    union doca_data cookie; /* User cookie */
    union {
        urom_rdmo_client_init_finished client_init_cb; /* User client init task callback */
        urom_rdmo_rq_create_finished rq_create_cb;     /* User RQ create task callback */
        urom_rdmo_rq_destroy_finished rq_destroy_cb;   /* User RQ destroy task callback */
        urom_rdmo_mr_register_finished mr_reg_cb;      /* User MR register task callback */
        urom_rdmo_mr_deregister_finished mr_dereg_cb;  /* User MR deregister task callback */
    };
};
 
/*
 * RDMO notification unpack function
 *
 * @packed_notif [in]: packed RDMO notification buffer
 * @rdmo_notif [out]: set unpacked RDMO notification
 * @return: DOCA_SUCCESS on success and DOCA_ERROR otherwise
 */
static doca_error_t urom_worker_rdmo_notif_unpack(void *packed_notif, struct urom_worker_notify_rdmo **rdmo_notif)
{
    void *ptr;
 
    *rdmo_notif = packed_notif;
    ptr = packed_notif + sizeof(struct urom_worker_notify_rdmo);
 
    switch ((*rdmo_notif)->type) {
    case UROM_WORKER_CMD_RDMO_CLIENT_INIT:
        (*rdmo_notif)->client_init.addr = ptr;
        break;
    }
    return DOCA_SUCCESS;
}
 
/*
 * Calculate RDMO packed command size
 *
 * @rdmo_cmd [in]: RDMO command
 * @return: command packed size
 */
static size_t urom_worker_rdmo_cmd_packed_len(struct urom_worker_rdmo_cmd *rdmo_cmd)
{
    size_t pack_len;
 
    pack_len = sizeof(struct urom_worker_rdmo_cmd);
 
    /* Pack inline data */
    switch (rdmo_cmd->type) {
    case UROM_WORKER_CMD_RDMO_CLIENT_INIT:
        pack_len += rdmo_cmd->client_init.addr_len;
        break;
    case UROM_WORKER_CMD_RDMO_RQ_CREATE:
        pack_len += rdmo_cmd->rq_create.addr_len;
        break;
    case UROM_WORKER_CMD_RDMO_MR_REG:
        pack_len += rdmo_cmd->mr_reg.packed_rkey_len;
        pack_len += rdmo_cmd->mr_reg.packed_memh_len;
        break;
    }
 
    return pack_len;
}
 
/*
 * Pack RDMO command
 *
 * @rdmo_cmd [in]: rdmo command
 * @packed_cmd_len [in/out]: packed command buffer size
 * @packed_cmd [out]: packed command buffer
 * @return: DOCA_SUCCESS on success and DOCA_ERROR otherwise
 */
static doca_error_t urom_worker_rdmo_cmd_pack(struct urom_worker_rdmo_cmd *rdmo_cmd,
                          size_t *packed_cmd_len,
                          void *packed_cmd)
{
    void *pack_tail = packed_cmd;
    void *pack_head;
    size_t pack_len;
 
    pack_len = urom_worker_rdmo_cmd_packed_len(rdmo_cmd);
    if (pack_len > *packed_cmd_len)
        return DOCA_ERROR_INITIALIZATION;
 
    /* Pack base command */
    pack_len = sizeof(struct urom_worker_rdmo_cmd);
    pack_head = urom_rdmo_serialize_next_raw(&pack_tail, void, pack_len);
    memcpy(pack_head, rdmo_cmd, pack_len);
    *packed_cmd_len = pack_len;
 
    /* Pack inline data */
    switch (rdmo_cmd->type) {
    case UROM_WORKER_CMD_RDMO_CLIENT_INIT:
        pack_len = rdmo_cmd->client_init.addr_len;
        pack_head = urom_rdmo_serialize_next_raw(&pack_tail, void, pack_len);
        memcpy(pack_head, rdmo_cmd->client_init.addr, pack_len);
        *packed_cmd_len += pack_len;
        break;
    case UROM_WORKER_CMD_RDMO_RQ_CREATE:
        pack_len = rdmo_cmd->rq_create.addr_len;
        pack_head = urom_rdmo_serialize_next_raw(&pack_tail, void, pack_len);
        memcpy(pack_head, rdmo_cmd->rq_create.addr, pack_len);
        *packed_cmd_len += pack_len;
        break;
    case UROM_WORKER_CMD_RDMO_MR_REG:
        pack_len = rdmo_cmd->mr_reg.packed_rkey_len;
        pack_head = urom_rdmo_serialize_next_raw(&pack_tail, void, pack_len);
        memcpy(pack_head, rdmo_cmd->mr_reg.packed_rkey, pack_len);
        *packed_cmd_len += pack_len;
        pack_len = rdmo_cmd->mr_reg.packed_memh_len;
        pack_head = urom_rdmo_serialize_next_raw(&pack_tail, void, pack_len);
        memcpy(pack_head, rdmo_cmd->mr_reg.packed_memh, pack_len);
        *packed_cmd_len += pack_len;
        break;
    }
 
    return DOCA_SUCCESS;
}
 
/*
 * RDMO common command's completion callback function
 *
 * @task [in]: UROM worker task
 * @type [in]: RDMO task type
 */
static void urom_rdmo_completion(struct doca_urom_worker_cmd_task *task, enum urom_worker_rdmo_notify_type type)
{
    size_t data_len;
    doca_error_t result;
    size_t extended_mem = 0;
    struct doca_buf *response;
    struct urom_worker_notify_rdmo notify_error = {0};
    struct urom_worker_notify_rdmo *rdmo_notify = &notify_error;
    struct doca_rdmo_task_data *task_data;
 
    notify_error.type = type;
 
    task_data = (struct doca_rdmo_task_data *)doca_urom_worker_cmd_task_get_user_data(task);
    if (task_data == NULL) {
        DOCA_LOG_ERR("Failed to get task data buffer");
        goto task_release;
    }
 
    response = doca_urom_worker_cmd_task_get_response(task);
    if (response == NULL) {
        DOCA_LOG_ERR("Failed to get task response buffer");
        result = DOCA_ERROR_INVALID_VALUE;
        goto error_exit;
    }
 
    result = doca_buf_get_data(response, (void **)&rdmo_notify);
    if (result != DOCA_SUCCESS)
        goto error_exit;
 
    result = urom_worker_rdmo_notif_unpack((void *)rdmo_notify, &rdmo_notify);
    if (result != DOCA_SUCCESS)
        goto error_exit;
 
    result = doca_buf_get_data_len(response, &data_len);
    if (result != DOCA_SUCCESS) {
        DOCA_LOG_ERR("Failed to get response data length");
        goto error_exit;
    }
 
    result = doca_task_get_status(doca_urom_worker_cmd_task_as_task(task));
    if (result != DOCA_SUCCESS)
        goto error_exit;
 
    if (rdmo_notify->type == UROM_WORKER_NOTIFY_RDMO_CLIENT_INIT)
        extended_mem = rdmo_notify->client_init.addr_len;
 
    if (data_len != sizeof(*rdmo_notify) + extended_mem) {
        DOCA_LOG_ERR("Task response data length is different from notification expected length");
        result = DOCA_ERROR_INVALID_VALUE;
        goto error_exit;
    }
 
error_exit:
    switch (rdmo_notify->type) {
    case UROM_WORKER_NOTIFY_RDMO_CLIENT_INIT:
        (task_data->client_init_cb)(result,
                        task_data->cookie,
                        rdmo_notify->client_init.addr,
                        rdmo_notify->client_init.addr_len);
        break;
    case UROM_WORKER_NOTIFY_RDMO_RQ_CREATE:
        (task_data->rq_create_cb)(result, task_data->cookie, rdmo_notify->rq_create.rq_id);
        break;
    case UROM_WORKER_NOTIFY_RDMO_RQ_DESTROY:
        (task_data->rq_destroy_cb)(result, task_data->cookie, rdmo_notify->rq_destroy.rq_id);
        break;
    case UROM_WORKER_NOTIFY_RDMO_MR_REG:
        (task_data->mr_reg_cb)(result, task_data->cookie, rdmo_notify->mr_reg.rkey);
        break;
    case UROM_WORKER_NOTIFY_RDMO_MR_DEREG:
        (task_data->mr_dereg_cb)(result, task_data->cookie, rdmo_notify->mr_dereg.rkey);
        break;
    default:
        DOCA_LOG_ERR("Invalid RDMO notification type %lu", rdmo_notify->type);
        break;
    }
 
task_release:
    result = doca_urom_worker_cmd_task_release(task);
    if (result != DOCA_SUCCESS)
        DOCA_LOG_ERR("Failed to release worker command task %s", doca_error_get_descr(result));
}
 
/*
 * RDMO client init command completion callback function, user callback will be called inside the function
 *
 * @task [in]: UROM worker task
 * @task_user_data [in]: task user data
 * @ctx_user_data [in]: worker context user data
 */
static void urom_rdmo_client_init_completed(struct doca_urom_worker_cmd_task *task,
                        union doca_data task_user_data,
                        union doca_data ctx_user_data)
{
    (void)task_user_data;
    (void)ctx_user_data;
 
    urom_rdmo_completion(task, UROM_WORKER_NOTIFY_RDMO_CLIENT_INIT);
}
 
doca_error_t urom_rdmo_task_client_init(struct doca_urom_worker *worker_ctx,
                    union doca_data cookie,
                    uint64_t id,
                    void *addr,
                    uint64_t addr_len,
                    urom_rdmo_client_init_finished cb)
{
    doca_error_t result;
    size_t pack_len = 0;
    struct doca_buf *payload;
    struct doca_urom_worker_cmd_task *task;
    struct doca_rdmo_task_data *task_data;
    struct urom_worker_rdmo_cmd *rdmo_cmd;
 
    /* Allocate task */
    result = doca_urom_worker_cmd_task_allocate_init(worker_ctx, rdmo_id, &task);
    if (result != DOCA_SUCCESS)
        return result;
 
    payload = doca_urom_worker_cmd_task_get_payload(task);
    result = doca_buf_get_data(payload, (void **)&rdmo_cmd);
    if (result != DOCA_SUCCESS)
        goto task_destroy;
 
    result = doca_buf_get_data_len(payload, &pack_len);
    if (result != DOCA_SUCCESS)
        goto task_destroy;
 
    /* Populate commands attributes */
    rdmo_cmd->type = UROM_WORKER_CMD_RDMO_CLIENT_INIT;
    rdmo_cmd->client_init.id = id;
    rdmo_cmd->client_init.addr = addr;
    rdmo_cmd->client_init.addr_len = addr_len;
 
    urom_worker_rdmo_cmd_pack(rdmo_cmd, &pack_len, (void *)rdmo_cmd);
 
    doca_buf_set_data(payload, rdmo_cmd, pack_len);
 
    task_data = (struct doca_rdmo_task_data *)doca_urom_worker_cmd_task_get_user_data(task);
    task_data->client_init_cb = cb;
    task_data->cookie = cookie;
 
    doca_urom_worker_cmd_task_set_cb(task, urom_rdmo_client_init_completed);
 
    result = doca_task_submit(doca_urom_worker_cmd_task_as_task(task));
    if (result != DOCA_SUCCESS)
        goto task_destroy;
 
    return DOCA_SUCCESS;
 
task_destroy:
    doca_urom_worker_cmd_task_release(task);
    return result;
}
 
/*
 * RDMO RQ create command completion callback function, user callback will be called inside the function
 *
 * @task [in]: UROM worker task
 * @task_user_data [in]: task user data
 * @ctx_user_data [in]: worker context user data
 */
static void doca_urom_rq_create_init_completed(struct doca_urom_worker_cmd_task *task,
                           union doca_data task_user_data,
                           union doca_data ctx_user_data)
{
    (void)task_user_data;
    (void)ctx_user_data;
 
    urom_rdmo_completion(task, UROM_WORKER_NOTIFY_RDMO_RQ_CREATE);
}
 
doca_error_t urom_rdmo_task_rq_create(struct doca_urom_worker *worker_ctx,
                      union doca_data cookie,
                      void *addr,
                      uint64_t addr_len,
                      urom_rdmo_rq_create_finished cb)
{
    doca_error_t result;
    size_t pack_len = 0;
    struct doca_buf *payload;
    struct doca_urom_worker_cmd_task *task;
    struct doca_rdmo_task_data *task_data;
    struct urom_worker_rdmo_cmd *rdmo_cmd;
 
    /* Allocate task */
    result = doca_urom_worker_cmd_task_allocate_init(worker_ctx, rdmo_id, &task);
    if (result != DOCA_SUCCESS)
        return result;
 
    payload = doca_urom_worker_cmd_task_get_payload(task);
    result = doca_buf_get_data(payload, (void **)&rdmo_cmd);
    if (result != DOCA_SUCCESS)
        goto task_destroy;
 
    result = doca_buf_get_data_len(payload, &pack_len);
    if (result != DOCA_SUCCESS)
        goto task_destroy;
 
    /* Populate commands attributes */
    rdmo_cmd->type = UROM_WORKER_CMD_RDMO_RQ_CREATE;
    rdmo_cmd->rq_create.addr = addr;
    rdmo_cmd->rq_create.addr_len = addr_len;
 
    urom_worker_rdmo_cmd_pack(rdmo_cmd, &pack_len, (void *)rdmo_cmd);
 
    doca_buf_set_data(payload, rdmo_cmd, pack_len);
 
    task_data = (struct doca_rdmo_task_data *)doca_urom_worker_cmd_task_get_user_data(task);
    task_data->rq_create_cb = cb;
    task_data->cookie = cookie;
 
    doca_urom_worker_cmd_task_set_cb(task, doca_urom_rq_create_init_completed);
 
    result = doca_task_submit(doca_urom_worker_cmd_task_as_task(task));
    if (result != DOCA_SUCCESS)
        goto task_destroy;
 
    return DOCA_SUCCESS;
 
task_destroy:
    doca_urom_worker_cmd_task_release(task);
    return result;
}
 
/*
 * RDMO RQ destroy command completion callback function, user callback will be called inside the function
 *
 * @task [in]: UROM worker task
 * @task_user_data [in]: task user data
 * @ctx_user_data [in]: worker context user data
 */
static void urom_rdmo_rq_destroy_completed(struct doca_urom_worker_cmd_task *task,
                       union doca_data task_user_data,
                       union doca_data ctx_user_data)
{
    (void)task_user_data;
    (void)ctx_user_data;
 
    urom_rdmo_completion(task, UROM_WORKER_NOTIFY_RDMO_RQ_DESTROY);
}
 
doca_error_t urom_rdmo_task_rq_destroy(struct doca_urom_worker *worker_ctx,
                       union doca_data cookie,
                       uint64_t rq_id,
                       urom_rdmo_rq_destroy_finished cb)
{
    doca_error_t result;
    size_t pack_len = 0;
    struct doca_buf *payload;
    struct doca_urom_worker_cmd_task *task;
    struct doca_rdmo_task_data *task_data;
    struct urom_worker_rdmo_cmd *rdmo_cmd;
 
    /* Allocate task */
    result = doca_urom_worker_cmd_task_allocate_init(worker_ctx, rdmo_id, &task);
    if (result != DOCA_SUCCESS)
        return result;
 
    payload = doca_urom_worker_cmd_task_get_payload(task);
    result = doca_buf_get_data(payload, (void **)&rdmo_cmd);
    if (result != DOCA_SUCCESS)
        goto task_destroy;
 
    result = doca_buf_get_data_len(payload, &pack_len);
    if (result != DOCA_SUCCESS)
        goto task_destroy;
 
    /* Populate commands attributes */
    rdmo_cmd->type = UROM_WORKER_CMD_RDMO_RQ_DESTROY;
    rdmo_cmd->rq_destroy.rq_id = rq_id;
 
    urom_worker_rdmo_cmd_pack(rdmo_cmd, &pack_len, (void *)rdmo_cmd);
 
    doca_buf_set_data(payload, rdmo_cmd, pack_len);
 
    task_data = (struct doca_rdmo_task_data *)doca_urom_worker_cmd_task_get_user_data(task);
    task_data->rq_destroy_cb = cb;
    task_data->cookie = cookie;
 
    doca_urom_worker_cmd_task_set_cb(task, urom_rdmo_rq_destroy_completed);
 
    result = doca_task_submit(doca_urom_worker_cmd_task_as_task(task));
    if (result != DOCA_SUCCESS)
        goto task_destroy;
 
    return DOCA_SUCCESS;
 
task_destroy:
    doca_urom_worker_cmd_task_release(task);
    return result;
}
 
/*
 * RDMO MR register command completion callback function, user callback will be called inside the function
 *
 * @task [in]: UROM worker task
 * @task_user_data [in]: task user data
 * @ctx_user_data [in]: worker context user data
 */
static void urom_rdmo_mr_reg_completed(struct doca_urom_worker_cmd_task *task,
                       union doca_data task_user_data,
                       union doca_data ctx_user_data)
{
    (void)task_user_data;
    (void)ctx_user_data;
 
    urom_rdmo_completion(task, UROM_WORKER_NOTIFY_RDMO_MR_REG);
}
 
doca_error_t urom_rdmo_task_mr_register(struct doca_urom_worker *worker_ctx,
                    union doca_data cookie,
                    uint64_t va,
                    uint64_t len,
                    void *rkey,
                    uint64_t rkey_len,
                    void *memh,
                    uint64_t memh_len,
                    urom_rdmo_mr_register_finished cb)
{
    doca_error_t result;
    size_t pack_len = 0;
    struct doca_buf *payload;
    struct doca_urom_worker_cmd_task *task;
    struct doca_rdmo_task_data *task_data;
    struct urom_worker_rdmo_cmd *rdmo_cmd;
 
    /* Allocate task */
    result = doca_urom_worker_cmd_task_allocate_init(worker_ctx, rdmo_id, &task);
    if (result != DOCA_SUCCESS)
        return result;
 
    payload = doca_urom_worker_cmd_task_get_payload(task);
    result = doca_buf_get_data(payload, (void **)&rdmo_cmd);
    if (result != DOCA_SUCCESS)
        goto task_destroy;
 
    result = doca_buf_get_data_len(payload, &pack_len);
    if (result != DOCA_SUCCESS)
        goto task_destroy;
 
    /* Populate commands attributes */
    rdmo_cmd->type = UROM_WORKER_CMD_RDMO_MR_REG;
    rdmo_cmd->mr_reg.va = va;
    rdmo_cmd->mr_reg.len = len;
    rdmo_cmd->mr_reg.packed_rkey = rkey;
    rdmo_cmd->mr_reg.packed_rkey_len = rkey_len;
    rdmo_cmd->mr_reg.packed_memh = memh;
    rdmo_cmd->mr_reg.packed_memh_len = memh_len;
 
    urom_worker_rdmo_cmd_pack(rdmo_cmd, &pack_len, (void *)rdmo_cmd);
 
    doca_buf_set_data(payload, rdmo_cmd, pack_len);
 
    task_data = (struct doca_rdmo_task_data *)doca_urom_worker_cmd_task_get_user_data(task);
    task_data->mr_reg_cb = cb;
    task_data->cookie = cookie;
 
    doca_urom_worker_cmd_task_set_cb(task, urom_rdmo_mr_reg_completed);
 
    result = doca_task_submit(doca_urom_worker_cmd_task_as_task(task));
    if (result != DOCA_SUCCESS)
        goto task_destroy;
 
    return DOCA_SUCCESS;
 
task_destroy:
    doca_urom_worker_cmd_task_release(task);
    return result;
}
 
/*
 * RDMO MR deregister command completion callback function, user callback will be called inside the function
 *
 * @task [in]: UROM worker task
 * @task_user_data [in]: task user data
 * @ctx_user_data [in]: worker context user data
 */
static void urom_rdmo_mr_dereg_completed(struct doca_urom_worker_cmd_task *task,
                     union doca_data task_user_data,
                     union doca_data ctx_user_data)
{
    (void)task_user_data;
    (void)ctx_user_data;
 
    urom_rdmo_completion(task, UROM_WORKER_NOTIFY_RDMO_MR_DEREG);
}
 
doca_error_t urom_rdmo_task_mr_deregister(struct doca_urom_worker *worker_ctx,
                      union doca_data cookie,
                      uint64_t rkey_id,
                      urom_rdmo_mr_deregister_finished cb)
{
    doca_error_t result;
    size_t pack_len = 0;
    struct doca_buf *payload;
    struct doca_urom_worker_cmd_task *task;
    struct doca_rdmo_task_data *task_data;
    struct urom_worker_rdmo_cmd *rdmo_cmd;
 
    /* Allocate task */
    result = doca_urom_worker_cmd_task_allocate_init(worker_ctx, rdmo_id, &task);
    if (result != DOCA_SUCCESS)
        return result;
 
    payload = doca_urom_worker_cmd_task_get_payload(task);
    result = doca_buf_get_data(payload, (void **)&rdmo_cmd);
    if (result != DOCA_SUCCESS)
        goto task_destroy;
 
    result = doca_buf_get_data_len(payload, &pack_len);
    if (result != DOCA_SUCCESS)
        goto task_destroy;
 
    /* Populate commands attributes */
    rdmo_cmd->type = UROM_WORKER_CMD_RDMO_MR_DEREG;
    rdmo_cmd->mr_dereg.rkey = rkey_id;
 
    urom_worker_rdmo_cmd_pack(rdmo_cmd, &pack_len, (void *)rdmo_cmd);
 
    doca_buf_set_data(payload, rdmo_cmd, pack_len);
 
    task_data = (struct doca_rdmo_task_data *)doca_urom_worker_cmd_task_get_user_data(task);
    task_data->mr_dereg_cb = cb;
    task_data->cookie = cookie;
 
    doca_urom_worker_cmd_task_set_cb(task, urom_rdmo_mr_dereg_completed);
 
    result = doca_task_submit(doca_urom_worker_cmd_task_as_task(task));
    if (result != DOCA_SUCCESS)
        goto task_destroy;
 
    return DOCA_SUCCESS;
 
task_destroy:
    doca_urom_worker_cmd_task_release(task);
    return result;
}
 
doca_error_t urom_rdmo_init(uint64_t plugin_id, uint64_t version)
{
    if (version != rdmo_version)
        return DOCA_ERROR_UNSUPPORTED_VERSION;
 
    rdmo_id = plugin_id;
    return DOCA_SUCCESS;
}