Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

libhns: Support mmapping reset state from kernel and reporting software wc #1504

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

libhns: Support mmapping reset state from kernel and reporting software wc #1504

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7631f0c
Update kernel headers
Oct 14, 2024
94f1967
libhns: Get reset state from kernel by mmap
Oct 14, 2024
80d452f
libhns: Return an error when post send/recv/srq recv in reset state
Oct 14, 2024
a318264
libhns: Support reporting WC as software mode
Oct 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions kernel-headers/rdma/hns-abi.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ struct hns_roce_ib_alloc_ucontext_resp {
__u32 max_inline_data;
__u8 congest_type;
__u8 reserved0[7];
__aligned_u64 reset_mmap_key;
};

struct hns_roce_ib_alloc_ucontext {
Expand All @@ -153,4 +154,9 @@ struct hns_roce_ib_create_ah_resp {
__u8 tc_mode;
};

struct hns_roce_reset_state {
__u32 hw_ready;
__u32 reserved;
};

#endif /* HNS_ABI_USER_H */
42 changes: 37 additions & 5 deletions providers/hns/hns_roce_u.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,41 @@ static void hns_roce_destroy_context_lock(struct hns_roce_context *context)
pthread_mutex_destroy(&context->db_list_mutex);
}

static int hns_roce_mmap(struct hns_roce_device *hr_dev,
struct hns_roce_alloc_ucontext_resp *resp,
struct hns_roce_context *context, int cmd_fd)
{
uint64_t reset_mmap_key = resp->reset_mmap_key;

context->uar = mmap(NULL, hr_dev->page_size, PROT_READ | PROT_WRITE,
MAP_SHARED, cmd_fd, 0);
if (context->uar == MAP_FAILED)
return ENOMEM;

/* Check whether kernel supports mmapping reset state */
if (!reset_mmap_key)
return 0;

context->reset_state = mmap(NULL, hr_dev->page_size, PROT_READ,
MAP_SHARED, cmd_fd, reset_mmap_key);
if (context->reset_state == MAP_FAILED)
goto db_free;

return 0;

db_free:
munmap(context->uar, hr_dev->page_size);
return ENOMEM;
}

static void hns_roce_munmap(struct hns_roce_device *hr_dev,
struct hns_roce_context *context)
{
munmap(context->uar, hr_dev->page_size);
if (context->reset_state)
munmap(context->reset_state, hr_dev->page_size);
}

static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev,
int cmd_fd,
void *private_data)
Expand All @@ -215,12 +250,9 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev,
if (set_context_attr(hr_dev, context, &resp))
goto err_set_attr;

context->uar = mmap(NULL, hr_dev->page_size, PROT_READ | PROT_WRITE,
MAP_SHARED, cmd_fd, 0);
if (context->uar == MAP_FAILED)
if (hns_roce_mmap(hr_dev, &resp, context, cmd_fd))
goto err_set_attr;


verbs_set_ops(&context->ibv_ctx, &hns_common_ops);
verbs_set_ops(&context->ibv_ctx, &hr_dev->u_hw->hw_ops);

Expand All @@ -239,7 +271,7 @@ static void hns_roce_free_context(struct ibv_context *ibctx)
struct hns_roce_device *hr_dev = to_hr_dev(ibctx->device);
struct hns_roce_context *context = to_hr_ctx(ibctx);

munmap(context->uar, hr_dev->page_size);
hns_roce_munmap(hr_dev, context);
hns_roce_destroy_context_lock(context);
verbs_uninit_context(&context->ibv_ctx);
free(context);
Expand Down
10 changes: 10 additions & 0 deletions providers/hns/hns_roce_u.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,7 @@ struct hns_roce_spinlock {
struct hns_roce_context {
struct verbs_context ibv_ctx;
void *uar;
void *reset_state;
pthread_spinlock_t uar_lock;

struct {
Expand Down Expand Up @@ -266,6 +267,11 @@ struct hns_roce_cq {
unsigned int cqe_size;
struct hns_roce_v2_cqe *cqe;
struct ibv_pd *parent_domain;
struct list_head list_sq;
struct list_head list_rq;
struct list_head list_srq;
struct list_head list_xrc_srq;
struct hns_roce_v2_cqe *sw_cqe;
};

struct hns_roce_idx_que {
Expand Down Expand Up @@ -301,6 +307,7 @@ struct hns_roce_srq {
unsigned int *rdb;
unsigned int cap_flags;
unsigned short counter;
struct list_node xrc_srcq_node;
};

struct hns_roce_wq {
Expand Down Expand Up @@ -361,6 +368,9 @@ struct hns_roce_qp {
void *cur_wqe;
unsigned int rb_sq_head; /* roll back sq head */
struct hns_roce_sge_info sge_info;
struct list_node rcq_node;
struct list_node scq_node;
struct list_node srcq_node;
};

struct hns_roce_av {
Expand Down
Loading