Skip to content

Commit

Permalink
prov/opx: Add HMEM handle for GDRCopy in GET/PUT
Browse files Browse the repository at this point in the history 
Signed-off-by: Lindsay Reiser <lindsay.reiser@cornelisnetworks.com>
  • Loading branch information
@lsavers @j-xiong
lsavers authored and j-xiong committed Feb 28, 2025
1 parent 3b25276 commit a3be069
Show file tree
Hide file tree
Showing 11 changed files with 630 additions and 610 deletions.
273 changes: 144 additions & 129 deletions prov/opx/include/rdma/opx/fi_opx_endpoint.h
View file

Large diffs are not rendered by default.

8 changes: 4 additions & 4 deletions prov/opx/include/rdma/opx/fi_opx_fabric_transport.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,10 @@
#include "rdma/opx/fi_opx_hfi1_transport.h"

#define FI_OPX_FABRIC_TX_INJECT fi_opx_hfi1_tx_inject
#define FI_OPX_FABRIC_TX_SEND_EGR fi_opx_hfi1_tx_send_egr_select
#define FI_OPX_FABRIC_TX_SENDV_EGR fi_opx_hfi1_tx_sendv_egr_select
#define FI_OPX_FABRIC_TX_SEND_RZV fi_opx_hfi1_tx_send_rzv_select
#define FI_OPX_FABRIC_TX_SENDV_RZV fi_opx_hfi1_tx_sendv_rzv
#define OPX_FABRIC_TX_SEND_EGR opx_hfi1_tx_send_egr_select
#define OPX_FABRIC_TX_SENDV_EGR opx_hfi1_tx_sendv_egr_select
#define OPX_FABRIC_TX_SEND_RZV opx_hfi1_tx_send_rzv_select
#define OPX_FABRIC_TX_SENDV_RZV opx_hfi1_tx_sendv_rzv
#define FI_OPX_FABRIC_RX_RZV_RTS fi_opx_hfi1_rx_rzv_rts
#define FI_OPX_FABRIC_RX_RZV_RTS_ETRUNC fi_opx_hfi1_rx_rzv_rts_etrunc
#define FI_OPX_FABRIC_RX_RZV_CTS fi_opx_hfi1_rx_rzv_cts
Expand Down
53 changes: 27 additions & 26 deletions prov/opx/include/rdma/opx/fi_opx_hfi1_inlines.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,8 @@ size_t opx_hfi1_dput_write_header_and_payload_put(struct fi_opx_ep *opx_ep, unio
const uint64_t op64, const uint64_t dt64, const size_t payload_bytes,
const uint64_t key_or_rma_req, uint8_t **sbuf,
const enum fi_hmem_iface sbuf_iface, const uint64_t sbuf_device,
uintptr_t *rbuf, const enum opx_hfi1_type hfi1_type,
const uint32_t opcode)
const uint64_t sbuf_handle, uintptr_t *rbuf,
const enum opx_hfi1_type hfi1_type, const uint32_t opcode)
{
if (hfi1_type & (OPX_HFI1_WFR | OPX_HFI1_JKR_9B)) {
hdr->qw_9B[4] = opx_ep->rx->tx.dput_9B.hdr.qw_9B[4] | opcode | (dt64 << 16) | (op64 << 24) |
Expand All @@ -60,8 +60,8 @@ size_t opx_hfi1_dput_write_header_and_payload_put(struct fi_opx_ep *opx_ep, unio

if (tx_payload) {
assert(!iov);
OPX_HMEM_COPY_FROM((void *) tx_payload, (const void *) *sbuf, payload_bytes, OPX_HMEM_NO_HANDLE,
OPX_HMEM_DEV_REG_THRESHOLD_NOT_SET, sbuf_iface, sbuf_device);
OPX_HMEM_COPY_FROM((void *) tx_payload, (const void *) *sbuf, payload_bytes, sbuf_handle,
OPX_HMEM_DEV_REG_SEND_THRESHOLD, sbuf_iface, sbuf_device);
} else {
assert(iov);
iov->iov_base = (void *) *sbuf;
Expand Down Expand Up @@ -217,7 +217,8 @@ size_t opx_hfi1_dput_write_header_and_payload_get(struct fi_opx_ep *opx_ep, unio
const uint64_t dt64, const size_t payload_bytes,
const uintptr_t rma_request_vaddr, uint8_t **sbuf,
const enum fi_hmem_iface sbuf_iface, const uint64_t sbuf_device,
uintptr_t *rbuf, const enum opx_hfi1_type hfi1_type)
const uint64_t sbuf_handle, uintptr_t *rbuf,
const enum opx_hfi1_type hfi1_type)
{
if (hfi1_type & (OPX_HFI1_WFR | OPX_HFI1_JKR_9B)) {
hdr->qw_9B[4] = opx_ep->rx->tx.dput_9B.hdr.qw_9B[4] | FI_OPX_HFI_DPUT_OPCODE_GET | (dt64 << 16) |
Expand All @@ -234,8 +235,8 @@ size_t opx_hfi1_dput_write_header_and_payload_get(struct fi_opx_ep *opx_ep, unio
if (tx_payload) {
assert(!iov);
if (dt64 == (FI_VOID - 1)) {
OPX_HMEM_COPY_FROM((void *) tx_payload, (const void *) *sbuf, payload_bytes, OPX_HMEM_NO_HANDLE,
OPX_HMEM_DEV_REG_THRESHOLD_NOT_SET, sbuf_iface, sbuf_device);
OPX_HMEM_COPY_FROM((void *) tx_payload, (const void *) *sbuf, payload_bytes, sbuf_handle,
OPX_HMEM_DEV_REG_SEND_THRESHOLD, sbuf_iface, sbuf_device);
} else {
OPX_HMEM_ATOMIC_DISPATCH((void *) *sbuf, (void *) tx_payload, payload_bytes, dt64,
FI_ATOMIC_WRITE, sbuf_iface, sbuf_device);
Expand All @@ -258,8 +259,8 @@ size_t opx_hfi1_dput_write_header_and_payload_rzv(struct fi_opx_ep *opx_ep, unio
const uint64_t op64, const uint64_t dt64, const size_t payload_bytes,
const uint32_t opcode, const uintptr_t target_byte_counter_vaddr,
uint8_t **sbuf, const enum fi_hmem_iface sbuf_iface,
const uint64_t sbuf_device, uintptr_t *rbuf,
enum opx_hfi1_type hfi1_type)
const uint64_t sbuf_device, const uint64_t sbuf_handle,
uintptr_t *rbuf, enum opx_hfi1_type hfi1_type)
{
if (hfi1_type & (OPX_HFI1_WFR | OPX_HFI1_JKR_9B)) {
hdr->qw_9B[4] = opx_ep->rx->tx.rzv_dput_9B.hdr.qw_9B[4] | (opcode) | (payload_bytes << 48);
Expand All @@ -273,8 +274,8 @@ size_t opx_hfi1_dput_write_header_and_payload_rzv(struct fi_opx_ep *opx_ep, unio

if (tx_payload) {
assert(!iov);
OPX_HMEM_COPY_FROM((void *) tx_payload, (const void *) *sbuf, payload_bytes, OPX_HMEM_NO_HANDLE,
OPX_HMEM_DEV_REG_THRESHOLD_NOT_SET, sbuf_iface, sbuf_device);
OPX_HMEM_COPY_FROM((void *) tx_payload, (const void *) *sbuf, payload_bytes, sbuf_handle,
OPX_HMEM_DEV_REG_SEND_THRESHOLD, sbuf_iface, sbuf_device);
} else {
assert(iov);
iov->iov_base = (void *) *sbuf;
Expand All @@ -294,9 +295,9 @@ size_t opx_hfi1_dput_write_packet(struct fi_opx_ep *opx_ep, union opx_hfi1_packe
const uint64_t bth_rx, const size_t payload_bytes, const uint64_t key,
const uint64_t fetch_vaddr, const uintptr_t target_byte_counter_vaddr,
const uintptr_t rma_request_vaddr, uint64_t bytes_sent, uint8_t **sbuf,
const enum fi_hmem_iface sbuf_iface, const uint64_t sbuf_device, uint8_t **cbuf,
const enum fi_hmem_iface cbuf_iface, const uint64_t cbuf_device, uintptr_t *rbuf,
const enum opx_hfi1_type hfi1_type)
const enum fi_hmem_iface sbuf_iface, const uint64_t sbuf_device,
const uint64_t sbuf_handle, uint8_t **cbuf, const enum fi_hmem_iface cbuf_iface,
const uint64_t cbuf_device, uintptr_t *rbuf, const enum opx_hfi1_type hfi1_type)
{
uint64_t psn = (uint64_t) htonl((uint32_t) psn_orig);

Expand Down Expand Up @@ -341,25 +342,25 @@ size_t opx_hfi1_dput_write_packet(struct fi_opx_ep *opx_ep, union opx_hfi1_packe
case FI_OPX_HFI_DPUT_OPCODE_RZV:
case FI_OPX_HFI_DPUT_OPCODE_RZV_TID:
case FI_OPX_HFI_DPUT_OPCODE_RZV_NONCONTIG:
return opx_hfi1_dput_write_header_and_payload_rzv(opx_ep, hdr, tx_payload, iov, op64, dt64,
payload_bytes, opcode, target_byte_counter_vaddr,
sbuf, sbuf_iface, sbuf_device, rbuf, hfi1_type);
return opx_hfi1_dput_write_header_and_payload_rzv(
opx_ep, hdr, tx_payload, iov, op64, dt64, payload_bytes, opcode, target_byte_counter_vaddr,
sbuf, sbuf_iface, sbuf_device, sbuf_handle, rbuf, hfi1_type);
break;
case FI_OPX_HFI_DPUT_OPCODE_GET:
return opx_hfi1_dput_write_header_and_payload_get(opx_ep, hdr, tx_payload, iov, dt64, payload_bytes,
rma_request_vaddr, sbuf, sbuf_iface, sbuf_device,
rbuf, hfi1_type);
sbuf_handle, rbuf, hfi1_type);
break;
case FI_OPX_HFI_DPUT_OPCODE_PUT:
return opx_hfi1_dput_write_header_and_payload_put(opx_ep, hdr, tx_payload, iov, op64, dt64,
payload_bytes, key, sbuf, sbuf_iface, sbuf_device,
rbuf, hfi1_type, opcode);
sbuf_handle, rbuf, hfi1_type, opcode);
break;
case FI_OPX_HFI_DPUT_OPCODE_PUT_CQ:
return opx_hfi1_dput_write_header_and_payload_put(
opx_ep, hdr, tx_payload, iov, op64, dt64, payload_bytes,
target_byte_counter_vaddr, /* this is the remote rma_request */
sbuf, sbuf_iface, sbuf_device, rbuf, hfi1_type, opcode);
sbuf, sbuf_iface, sbuf_device, sbuf_handle, rbuf, hfi1_type, opcode);
break;
case FI_OPX_HFI_DPUT_OPCODE_ATOMIC_FETCH:
return opx_hfi1_dput_write_header_and_payload_atomic_fetch(
Expand All @@ -384,13 +385,13 @@ size_t opx_hfi1_dput_write_header_and_payload(
const uint64_t lrh_dlid, const uint64_t bth_rx, const size_t payload_bytes, const uint64_t key,
const uint64_t fetch_vaddr, const uintptr_t target_byte_counter_vaddr, const uintptr_t rma_request_vaddr,
uint64_t bytes_sent, uint8_t **sbuf, const enum fi_hmem_iface sbuf_iface, const uint64_t sbuf_device,
uint8_t **cbuf, const enum fi_hmem_iface cbuf_iface, const uint64_t cbuf_device, uintptr_t *rbuf,
const enum opx_hfi1_type hfi1_type)
const uint64_t sbuf_handle, uint8_t **cbuf, const enum fi_hmem_iface cbuf_iface, const uint64_t cbuf_device,
uintptr_t *rbuf, const enum opx_hfi1_type hfi1_type)
{
return opx_hfi1_dput_write_packet(opx_ep, hdr, tx_payload, NULL, opcode, psn_orig, lrh_dws, op64, dt64,
lrh_dlid, bth_rx, payload_bytes, key, fetch_vaddr, target_byte_counter_vaddr,
rma_request_vaddr, bytes_sent, sbuf, sbuf_iface, sbuf_device, cbuf,
cbuf_iface, cbuf_device, rbuf, hfi1_type);
rma_request_vaddr, bytes_sent, sbuf, sbuf_iface, sbuf_device, sbuf_handle,
cbuf, cbuf_iface, cbuf_device, rbuf, hfi1_type);
}

__OPX_FORCE_INLINE__
Expand All @@ -408,7 +409,7 @@ size_t opx_hfi1_dput_write_header_and_iov(struct fi_opx_ep *opx_ep, union opx_hf
*/
return opx_hfi1_dput_write_packet(opx_ep, hdr, NULL, iov, opcode, 0, lrh_dws, op64, dt64, lrh_dlid, bth_rx,
payload_bytes, key, fetch_vaddr, target_byte_counter_vaddr, rma_request_vaddr,
bytes_sent, sbuf, FI_HMEM_SYSTEM, 0ul, cbuf, FI_HMEM_SYSTEM, 0ul, rbuf,
hfi1_type);
bytes_sent, sbuf, FI_HMEM_SYSTEM, 0ul, OPX_HMEM_NO_HANDLE, cbuf,
FI_HMEM_SYSTEM, 0ul, rbuf, hfi1_type);
}
#endif
26 changes: 14 additions & 12 deletions prov/opx/include/rdma/opx/fi_opx_hfi1_packet.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1355,8 +1355,8 @@ union cacheline {
uint8_t byte[64];
};

union fi_opx_hfi1_dput_iov {
uint64_t qw[6];
union opx_hfi1_dput_iov {
uint64_t qw[8];
struct {
uintptr_t rbuf;
uintptr_t sbuf;
Expand All @@ -1365,6 +1365,8 @@ union fi_opx_hfi1_dput_iov {
uint64_t sbuf_device;
enum fi_hmem_iface rbuf_iface;
enum fi_hmem_iface sbuf_iface;
uint64_t pad;
uint64_t sbuf_handle;
};
};

Expand Down Expand Up @@ -1400,10 +1402,10 @@ struct fi_opx_hmem_iov {
} __attribute__((__packed__));

#define FI_OPX_MAX_HMEM_IOV ((FI_OPX_HFI1_PACKET_MTU - sizeof(uintptr_t)) / sizeof(struct fi_opx_hmem_iov))
#define FI_OPX_MAX_DPUT_IOV ((FI_OPX_HFI1_PACKET_MTU / sizeof(union fi_opx_hfi1_dput_iov) - 4) + 3)
#define FI_OPX_MAX_DPUT_IOV ((FI_OPX_HFI1_PACKET_MTU / sizeof(union opx_hfi1_dput_iov) - 4) + 3)

#define FI_OPX_MAX_DPUT_TIDPAIRS \
((FI_OPX_HFI1_PACKET_MTU - sizeof(union fi_opx_hfi1_dput_iov) - (4 * sizeof(uint32_t))) / sizeof(uint32_t))
((FI_OPX_HFI1_PACKET_MTU - sizeof(union opx_hfi1_dput_iov) - (4 * sizeof(uint32_t))) / sizeof(uint32_t))

#define OPX_IMMEDIATE_BYTE_COUNT_SHIFT (5)
#define OPX_IMMEDIATE_BYTE_COUNT_MASK (0xE0)
Expand Down Expand Up @@ -1494,23 +1496,23 @@ union fi_opx_hfi1_packet_payload {
} rendezvous;

struct {
union fi_opx_hfi1_dput_iov iov[FI_OPX_MAX_DPUT_IOV];
union opx_hfi1_dput_iov iov[FI_OPX_MAX_DPUT_IOV];
} rma_rts;

struct {
union fi_opx_hfi1_dput_iov iov[FI_OPX_MAX_DPUT_IOV];
union opx_hfi1_dput_iov iov[FI_OPX_MAX_DPUT_IOV];
} cts;

/* tid_cts extends cts*/
struct {
/* ==== CACHE LINE 0 ==== */
union fi_opx_hfi1_dput_iov iov[1];
uint32_t tid_offset;
uint32_t ntidpairs;
int32_t origin_byte_counter_adjust;
uint32_t unused;

union opx_hfi1_dput_iov iov[1];
/* ==== CACHE LINE 1 ==== */
uint32_t tid_offset;
uint32_t ntidpairs;
int32_t origin_byte_counter_adjust;
uint32_t unused;

uint32_t tidpairs[FI_OPX_MAX_DPUT_TIDPAIRS];
} tid_cts;
} __attribute__((__aligned__(32)));
Expand Down
Loading

0 comments on commit a3be069

Please sign in to comment.