NVIDIA DOCA SDK Data Center on a Chip Framework Documentation
DOCA DPA Device
Collaboration diagram for DOCA DPA Device:

Modules

 DOCA DPA Device - Buffer
 
 DOCA DPA Device - Comch MsgQ
 
 DOCA DPA Device - Device Emulation for PCI devices
 
 DOCA DPA Device - RDMA
 
 DOCA DPA Device - Sync Event
 

Macros

#define DOCA_DPA_DEVICE
 declares that we are compiling for the DPA Device More...
 
#define __forceinline   static inline __attribute__((always_inline))
 static inline wrapper More...
 
#define DOCA_DPA_DEV_LOG_ERR(...)   doca_dpa_dev_log(DOCA_DPA_DEV_LOG_LEVEL_ERROR, __VA_ARGS__)
 Generate a DOCA DPA device ERROR log message. More...
 
#define DOCA_DPA_DEV_LOG_WARN(...)   doca_dpa_dev_log(DOCA_DPA_DEV_LOG_LEVEL_WARNING, __VA_ARGS__)
 Generate a DOCA DPA device WARNING log message. More...
 
#define DOCA_DPA_DEV_LOG_INFO(...)   doca_dpa_dev_log(DOCA_DPA_DEV_LOG_LEVEL_INFO, __VA_ARGS__)
 Generate a DOCA DPA device INFO log message. More...
 
#define DOCA_DPA_DEV_LOG_DBG(...)   doca_dpa_dev_log(DOCA_DPA_DEV_LOG_LEVEL_DEBUG, __VA_ARGS__)
 Generate a DOCA DPA device DEBUG log message. More...
 

Typedefs

typedef __dpa_global__ enum doca_dpa_dev_log_level doca_dpa_dev_log_level_t
 DOCA DPA device log levels, sorted by verbosity from high to low. More...
 

Enumerations

enum  doca_dpa_dev_log_level {
  DOCA_DPA_DEV_LOG_LEVEL_DISABLE = 10 , DOCA_DPA_DEV_LOG_LEVEL_ERROR = 30 , DOCA_DPA_DEV_LOG_LEVEL_WARNING = 40 , DOCA_DPA_DEV_LOG_LEVEL_INFO = 50 ,
  DOCA_DPA_DEV_LOG_LEVEL_DEBUG = 60 , DOCA_DPA_DEV_LOG_LEVEL_DISABLE = 10 , DOCA_DPA_DEV_LOG_LEVEL_ERROR = 30 , DOCA_DPA_DEV_LOG_LEVEL_WARNING = 40 ,
  DOCA_DPA_DEV_LOG_LEVEL_INFO = 50 , DOCA_DPA_DEV_LOG_LEVEL_DEBUG = 60
}
 DOCA DPA device log levels, sorted by verbosity from high to low. More...
 
enum  doca_dpa_dev_completion_type_t {
  DOCA_DPA_DEV_COMP_SEND = 0x0 , DOCA_DPA_DEV_COMP_RECV_RDMA_WRITE_IMM = 0x1 , DOCA_DPA_DEV_COMP_RECV_SEND = 0x2 , DOCA_DPA_DEV_COMP_RECV_SEND_IMM = 0x3 ,
  DOCA_DPA_DEV_COMP_SEND_ERR = 0xD , DOCA_DPA_DEV_COMP_RECV_ERR = 0xE
}
 DPA completion type. More...
 
enum  doca_dpa_dev_submit_flag { DOCA_DPA_DEV_SUBMIT_FLAG_NONE = 0U , DOCA_DPA_DEV_SUBMIT_FLAG_FLUSH = (1U << 0) , DOCA_DPA_DEV_SUBMIT_FLAG_OPTIMIZE_REPORTS = (1U << 1) }
 DPA submit flag type. More...
 

Functions

DOCA_EXPERIMENTAL void doca_dpa_dev_device_set (doca_dpa_dev_t dpa_handle)
 Set a DPA device. More...
 
DOCA_EXPERIMENTAL unsigned int doca_dpa_dev_thread_rank (void)
 Obtains the thread rank. More...
 
DOCA_EXPERIMENTAL unsigned int doca_dpa_dev_num_threads (void)
 Obtains the number of threads running the kernel. More...
 
DOCA_EXPERIMENTAL doca_dpa_dev_uintptr_t doca_dpa_dev_thread_get_local_storage (void)
 Get DPA thread local storage. More...
 
DOCA_EXPERIMENTAL void doca_dpa_dev_yield (void)
 Yield a DPA thread. More...
 
DOCA_EXPERIMENTAL void doca_dpa_dev_thread_reschedule (void)
 Reschedule a DPA thread. More...
 
DOCA_EXPERIMENTAL void doca_dpa_dev_thread_finish (void)
 Finish a DPA thread. More...
 
DOCA_EXPERIMENTAL void doca_dpa_dev_thread_notify (doca_dpa_dev_notification_completion_t comp_handle)
 Notify the completion handle and trigger the attached thread. More...
 
DOCA_EXPERIMENTAL void doca_dpa_dev_log (doca_dpa_dev_log_level_t log_level, const char *format,...) __attribute__((format(printf
 Print logs to Host. More...
 
DOCA_EXPERIMENTAL void doca_dpa_dev_trace (uint64_t arg1, uint64_t arg2, uint64_t arg3, uint64_t arg4, uint64_t arg5)
 Creates trace message entry with arguments. More...
 
DOCA_EXPERIMENTAL void doca_dpa_dev_trace_flush (void)
 Flush the trace message buffer to Host. More...
 
DOCA_EXPERIMENTAL void doca_dpa_dev_hash_table_add (doca_dpa_dev_hash_table_t ht_handle, uint32_t key, uint64_t value)
 Map a specific key to the specified value in the hash table. More...
 
DOCA_EXPERIMENTAL void doca_dpa_dev_hash_table_remove (doca_dpa_dev_hash_table_t ht_handle, uint32_t key)
 Remove the key and its corresponding value from the hash table. More...
 
DOCA_EXPERIMENTAL int doca_dpa_dev_hash_table_find (doca_dpa_dev_hash_table_t ht_handle, uint32_t key, uint64_t *value)
 Returns the value to which the specified key is mapped in the hash table. More...
 
DOCA_EXPERIMENTAL int doca_dpa_dev_get_completion (doca_dpa_dev_completion_t dpa_comp_handle, doca_dpa_dev_completion_element_t *comp_element)
 Get DPA completion element. More...
 
DOCA_EXPERIMENTAL doca_dpa_dev_completion_type_t doca_dpa_dev_get_completion_type (doca_dpa_dev_completion_element_t comp_element)
 Get completion element type. More...
 
DOCA_EXPERIMENTAL uint32_t doca_dpa_dev_get_completion_user_data (doca_dpa_dev_completion_element_t comp_element)
 Get completion element user data. More...
 
DOCA_EXPERIMENTAL uint32_t doca_dpa_dev_get_completion_immediate (doca_dpa_dev_completion_element_t comp_element)
 Get completion element immediate data. More...
 
DOCA_EXPERIMENTAL void doca_dpa_dev_completion_ack (doca_dpa_dev_completion_t dpa_comp_handle, uint64_t num_comp)
 Acknowledge that the completions have been read on the completion context. More...
 
DOCA_EXPERIMENTAL void doca_dpa_dev_completion_request_notification (doca_dpa_dev_completion_t dpa_comp_handle)
 Request notification on the DPA completion. More...
 

Variables

__dpa_global__ typedef uint64_t doca_dpa_dev_t
 DPA context handle type definition. More...
 
__dpa_global__ typedef uint64_t doca_dpa_dev_uintptr_t
 DPA pointer type definition. More...
 
__dpa_global__ typedef uint64_t doca_dpa_dev_hash_table_t
 DPA hash table handle type definition. More...
 
__dpa_global__ typedef uint64_t doca_dpa_dev_completion_t
 DPA completion handle type definition. More...
 
__dpa_global__ typedef uint64_t doca_dpa_dev_completion_element_t
 DPA completion element handle type definition. More...
 
__dpa_global__ typedef uint64_t doca_dpa_dev_async_ops_t
 DPA asynchronous ops handle type definition. More...
 
__dpa_global__ typedef uint64_t doca_dpa_dev_notification_completion_t
 DPA notification completion handle type definition. More...
 

Detailed Description

DOCA DPA Device library. For more details please refer to the user guide on DOCA devzone.

Macro Definition Documentation

◆ __forceinline

#define __forceinline   static inline __attribute__((always_inline))

static inline wrapper

Definition at line 130 of file doca_dpa_dev.h.

◆ DOCA_DPA_DEV_LOG_DBG

#define DOCA_DPA_DEV_LOG_DBG (   ...)    doca_dpa_dev_log(DOCA_DPA_DEV_LOG_LEVEL_DEBUG, __VA_ARGS__)

Generate a DOCA DPA device DEBUG log message.

Note
This call affects the performance.

Definition at line 295 of file doca_dpa_dev.h.

◆ DOCA_DPA_DEV_LOG_ERR

#define DOCA_DPA_DEV_LOG_ERR (   ...)    doca_dpa_dev_log(DOCA_DPA_DEV_LOG_LEVEL_ERROR, __VA_ARGS__)

Generate a DOCA DPA device ERROR log message.

Note
This call affects the performance.

Definition at line 271 of file doca_dpa_dev.h.

◆ DOCA_DPA_DEV_LOG_INFO

#define DOCA_DPA_DEV_LOG_INFO (   ...)    doca_dpa_dev_log(DOCA_DPA_DEV_LOG_LEVEL_INFO, __VA_ARGS__)

Generate a DOCA DPA device INFO log message.

Note
This call affects the performance.

Definition at line 287 of file doca_dpa_dev.h.

◆ DOCA_DPA_DEV_LOG_WARN

#define DOCA_DPA_DEV_LOG_WARN (   ...)    doca_dpa_dev_log(DOCA_DPA_DEV_LOG_LEVEL_WARNING, __VA_ARGS__)

Generate a DOCA DPA device WARNING log message.

Note
This call affects the performance.

Definition at line 279 of file doca_dpa_dev.h.

◆ DOCA_DPA_DEVICE

#define DOCA_DPA_DEVICE

declares that we are compiling for the DPA Device

Note
Must be defined before the first API use/include of DOCA

Definition at line 32 of file doca_dpa_dev.h.

Typedef Documentation

◆ doca_dpa_dev_log_level_t

DOCA DPA device log levels, sorted by verbosity from high to low.

Include to define compatibility with current version, define experimental Symbols

Enumeration Type Documentation

◆ doca_dpa_dev_completion_type_t

DPA completion type.

Enumerator
DOCA_DPA_DEV_COMP_SEND 

Send completion

DOCA_DPA_DEV_COMP_RECV_RDMA_WRITE_IMM 

Receive RDMA Write with Immediate completion

DOCA_DPA_DEV_COMP_RECV_SEND 

Receive Send completion

DOCA_DPA_DEV_COMP_RECV_SEND_IMM 

Receive Send with Immediate completion

DOCA_DPA_DEV_COMP_SEND_ERR 

Send Error completion

DOCA_DPA_DEV_COMP_RECV_ERR 

Receive Error completion

Definition at line 92 of file doca_dpa_dev.h.

◆ doca_dpa_dev_log_level

DOCA DPA device log levels, sorted by verbosity from high to low.

Include to define compatibility with current version, define experimental Symbols

Enumerator
DOCA_DPA_DEV_LOG_LEVEL_DISABLE 

Disable log messages

DOCA_DPA_DEV_LOG_LEVEL_ERROR 

Error log level

DOCA_DPA_DEV_LOG_LEVEL_WARNING 

Warning log level

DOCA_DPA_DEV_LOG_LEVEL_INFO 

Info log level

DOCA_DPA_DEV_LOG_LEVEL_DEBUG 

Debug log level

DOCA_DPA_DEV_LOG_LEVEL_DISABLE 

Disable log messages

DOCA_DPA_DEV_LOG_LEVEL_ERROR 

Error log level

DOCA_DPA_DEV_LOG_LEVEL_WARNING 

Warning log level

DOCA_DPA_DEV_LOG_LEVEL_INFO 

Info log level

DOCA_DPA_DEV_LOG_LEVEL_DEBUG 

Debug log level

Definition at line 46 of file doca_dpa_dev.h.

◆ doca_dpa_dev_submit_flag

DPA submit flag type.

Enumerator
DOCA_DPA_DEV_SUBMIT_FLAG_NONE 
DOCA_DPA_DEV_SUBMIT_FLAG_FLUSH 
DOCA_DPA_DEV_SUBMIT_FLAG_OPTIMIZE_REPORTS 

Use flag to inform related DPA context (such as RDMA or DPA Async ops) to flush related operation and previous operations to HW, otherwise the context may aggregate the operation and not flush it immediately

Definition at line 104 of file doca_dpa_dev.h.

Function Documentation

◆ doca_dpa_dev_completion_ack()

DOCA_EXPERIMENTAL void doca_dpa_dev_completion_ack ( doca_dpa_dev_completion_t  dpa_comp_handle,
uint64_t  num_comp 
)

Acknowledge that the completions have been read on the completion context.

This function releases resources of the acked completion elements in the completion context. This acknowledgment enables receiving new num_comp completions

Parameters
[in]dpa_comp_handle- DPA completion handle
[in]num_comp- Number of completion elements which have been read

◆ doca_dpa_dev_completion_request_notification()

DOCA_EXPERIMENTAL void doca_dpa_dev_completion_request_notification ( doca_dpa_dev_completion_t  dpa_comp_handle)

Request notification on the DPA completion.

This function enables requesting new notifications on the DPA completion. Without calling this function, DPA completion context will not be notified on new arrived completion elements hence new completions will not be populated in DPA completion context.

Parameters
[in]dpa_comp_handle- DPA completion handle

◆ doca_dpa_dev_device_set()

DOCA_EXPERIMENTAL void doca_dpa_dev_device_set ( doca_dpa_dev_t  dpa_handle)

Set a DPA device.

This function must be called before calling any related API of a DPA resource (RDMA/completion context/Async ops/...) that was created on an extended DPA context (a DPA context that was created using host API doca_dpa_device_extend()). Please note:

  • User who creates only a DPA base context and has no extended DPA context in his application doesn't need to call this API.
  • User who creates DPA resources on both base and extended DPA contexts must sets the proper DPA device (also for base DPA context) before calling relevant DPA resource API.

◆ doca_dpa_dev_get_completion()

DOCA_EXPERIMENTAL int doca_dpa_dev_get_completion ( doca_dpa_dev_completion_t  dpa_comp_handle,
doca_dpa_dev_completion_element_t comp_element 
)

Get DPA completion element.

This function returns a completion element which has arrived at the completion context. If a DPA thread is attached to this completion context, the thread will be triggered on this completion element.

Parameters
[in]dpa_comp_handle- DPA completion handle
[out]comp_element- DPA completion element
Returns
This function returns 1 if there is completion element to return. Otherwise returns 0

◆ doca_dpa_dev_get_completion_immediate()

DOCA_EXPERIMENTAL uint32_t doca_dpa_dev_get_completion_immediate ( doca_dpa_dev_completion_element_t  comp_element)

Get completion element immediate data.

This function returns immediate data for a completion of type: 1- DOCA_DPA_DEV_COMP_RECV_RDMA_WRITE_IMM 2- DOCA_DPA_DEV_COMP_RECV_SEND_IMM

Note
This API is not relevant for completions of other types.
Parameters
[in]comp_element- DPA RDMA completion element
Returns
This function returns immediate of completion element

◆ doca_dpa_dev_get_completion_type()

DOCA_EXPERIMENTAL doca_dpa_dev_completion_type_t doca_dpa_dev_get_completion_type ( doca_dpa_dev_completion_element_t  comp_element)

Get completion element type.

Parameters
[in]comp_element- DPA completion element
Returns
This function returns type completion element

◆ doca_dpa_dev_get_completion_user_data()

DOCA_EXPERIMENTAL uint32_t doca_dpa_dev_get_completion_user_data ( doca_dpa_dev_completion_element_t  comp_element)

Get completion element user data.

This API returns user data which: 1- Was set previously in host API doca_dpa_async_ops_create(..., user_data, ...) When DPA Completion Context is attached to DPA Async Ops. 2- Equivalent to "connection_id" in doca_error_t doca_rdma_connection_get_id(const struct doca_rdma_connection *rdma_connection, uint32_t *connection_id) When DPA Completion Context is attached to DOCA RDMA context.

Parameters
[in]comp_element- DPA completion element
Returns
This function returns user data of completion element

◆ doca_dpa_dev_hash_table_add()

DOCA_EXPERIMENTAL void doca_dpa_dev_hash_table_add ( doca_dpa_dev_hash_table_t  ht_handle,
uint32_t  key,
uint64_t  value 
)

Map a specific key to the specified value in the hash table.

Note
Adding a new key when the hash table is full will lead to unexpected behavior.
Parameters
[in]ht_handle- DPA hash table handle
[in]key- New key to add to hash table
[in]value- New key's value to add to hash table

◆ doca_dpa_dev_hash_table_find()

DOCA_EXPERIMENTAL int doca_dpa_dev_hash_table_find ( doca_dpa_dev_hash_table_t  ht_handle,
uint32_t  key,
uint64_t *  value 
)

Returns the value to which the specified key is mapped in the hash table.

Parameters
[in]ht_handle- DPA hash table handle
[in]key- Key to find in hash table
[out]value- Key's value
Returns
This function returns 1 if the key exists in the hash table. Otherwise returns 0

◆ doca_dpa_dev_hash_table_remove()

DOCA_EXPERIMENTAL void doca_dpa_dev_hash_table_remove ( doca_dpa_dev_hash_table_t  ht_handle,
uint32_t  key 
)

Remove the key and its corresponding value from the hash table.

Parameters
[in]ht_handle- DPA hash table handle
[in]key- Key to remove from hash table

◆ doca_dpa_dev_log()

DOCA_EXPERIMENTAL void doca_dpa_dev_log ( doca_dpa_dev_log_level_t  log_level,
const char *  format,
  ... 
)

Print logs to Host.

This function prints from device to host's standard output stream or the user defined file set by doca_dpa_log_file_set_path(). The log level will determine the print according to the verbosity set by doca_dpa_set_log_level(). It is recommended to use the bellow defined MACROs for device logging for better readability. Multiple threads may call these MACROs simultaneously. Printing is a convenience service, and due to limited buffering on the host, not all print statements may appear on the host.

Parameters
[in]log_level- level for device log
[in]format- format string that contains the text to be written to host (same as from regular printf)

◆ doca_dpa_dev_num_threads()

DOCA_EXPERIMENTAL unsigned int doca_dpa_dev_num_threads ( void  )

Obtains the number of threads running the kernel.

Retrieves the number of threads assigned to a given kernel. This is either the value: 1- num_threads that was passed to host API doca_dpa_kernel_launch_update_set/add(..., num_threads, ...) 2- num_threads that was passed to host API doca_dpa_thread_group_create(..., num_threads, ...)

Returns
Returns the number of threads running the kernel

◆ doca_dpa_dev_thread_finish()

DOCA_EXPERIMENTAL void doca_dpa_dev_thread_finish ( void  )

Finish a DPA thread.

This function marks the thread not to be rescheduled. Any new completion arrived at the attach completion context will not trigger the DPA thread again.

Note
This API is relevant only for a kernel used for DPA thread ('doca_dpa_thread'), which was set previously using host API doca_dpa_thread_set_func_arg(). This API is not relevant for a kernel used in host kernel launch APIs.

◆ doca_dpa_dev_thread_get_local_storage()

DOCA_EXPERIMENTAL doca_dpa_dev_uintptr_t doca_dpa_dev_thread_get_local_storage ( void  )

Get DPA thread local storage.

This function returns DPA thread local storage which was set previously using host API doca_dpa_thread_set_local_storage().

Note
This API is relevant only for a kernel used for DPA thread ('doca_dpa_thread'), which was set previously using host API doca_dpa_thread_set_func_arg(). This API is not relevant for a kernel used in host kernel launch APIs.
Returns
This function returns DPA pointer for DPA thread local storage

◆ doca_dpa_dev_thread_notify()

DOCA_EXPERIMENTAL void doca_dpa_dev_thread_notify ( doca_dpa_dev_notification_completion_t  comp_handle)

Notify the completion handle and trigger the attached thread.

This function notifies the given completion context which leads to triggering the attached DPA thread. Please note that using this API will leads to triggering the attached thread without receiving a completion on the attached completion context, hence can't read completion info using doca_dpa_dev_get_completion*() APIs

Note
This API is relevant only for a kernel used for DPA thread ('doca_dpa_thread'), which was set previously using host API doca_dpa_thread_set_func_arg(). This API is not relevant for a kernel used in host kernel launch APIs.
Parameters
[in]comp_handle- DPA notification completion handle

◆ doca_dpa_dev_thread_rank()

DOCA_EXPERIMENTAL unsigned int doca_dpa_dev_thread_rank ( void  )

Obtains the thread rank.

Retrieves the thread rank for a given kernel on the DPA. This function returns a number in {0..N-1}, where N is either: 1- The number of threads requested for launch during a kernel submission when using host APIs: doca_dpa_kernel_launch_update_set/add(..., N, ...). 2- The number of threads in thread group running the kernel when using host APIs: doca_dpa_thread_group_create(..., N, ...)

Returns
Returns the thread rank for a given kernel on the DPA.

◆ doca_dpa_dev_thread_reschedule()

DOCA_EXPERIMENTAL void doca_dpa_dev_thread_reschedule ( void  )

Reschedule a DPA thread.

This function reschedules a DPA thread and releases the EU and make it available for rescheduling. Any new completion arrived at the attach completion context will trigger the DPA thread again.

Note
This API is relevant only for a kernel used for DPA thread ('doca_dpa_thread'), which was set previously using host API doca_dpa_thread_set_func_arg(). This API is not relevant for a kernel used in host kernel launch APIs.

◆ doca_dpa_dev_trace()

DOCA_EXPERIMENTAL void doca_dpa_dev_trace ( uint64_t  arg1,
uint64_t  arg2,
uint64_t  arg3,
uint64_t  arg4,
uint64_t  arg5 
)

Creates trace message entry with arguments.

This function prints traces arguments from device to host's standard output stream, or to the user's defined outfile set by doca_dpa_trace_file_set_path().

Note
It is recommended to use trace for enhanced performance in logging
Parameters
[in]arg1- argument #1 to format into the template
[in]arg2- argument #2 to format into the template
[in]arg3- argument #3 to format into the template
[in]arg4- argument #4 to format into the template
[in]arg5- argument #5 to format into the template

◆ doca_dpa_dev_trace_flush()

DOCA_EXPERIMENTAL void doca_dpa_dev_trace_flush ( void  )

Flush the trace message buffer to Host.

As soon as a buffer is fully occupied it is internally sent to host, however user can ask partially occupied buffer to be sent to host. Its intended use is at end of run to flush whatever messages left

Note
: Frequent call to this API might cause performance issues.

◆ doca_dpa_dev_yield()

DOCA_EXPERIMENTAL void doca_dpa_dev_yield ( void  )

Yield a DPA thread.

This function yields a DPA thread that is running a kernel

Note
This API is relevant only for a kernel used in host kernel launch APIs doca_dpa_kernel_launch_update_set/add(...). This API is not relevant for a kernel used for DPA thread ('doca_dpa_thread'), which was set previously using host API doca_dpa_thread_set_func_arg().

Variable Documentation

◆ doca_dpa_dev_async_ops_t

DPA asynchronous ops handle type definition.

Definition at line 82 of file doca_dpa_dev.h.

◆ doca_dpa_dev_completion_element_t

__dpa_global__ typedef uint64_t doca_dpa_dev_completion_element_t

DPA completion element handle type definition.

Definition at line 77 of file doca_dpa_dev.h.

◆ doca_dpa_dev_completion_t

DPA completion handle type definition.

Definition at line 72 of file doca_dpa_dev.h.

◆ doca_dpa_dev_hash_table_t

DPA hash table handle type definition.

Definition at line 67 of file doca_dpa_dev.h.

◆ doca_dpa_dev_notification_completion_t

DPA notification completion handle type definition.

Definition at line 87 of file doca_dpa_dev.h.

◆ doca_dpa_dev_t

__dpa_global__ typedef uint64_t doca_dpa_dev_t

DPA context handle type definition.

Definition at line 57 of file doca_dpa_dev.h.

◆ doca_dpa_dev_uintptr_t

DPA pointer type definition.

Definition at line 62 of file doca_dpa_dev.h.