@@ -28,6 +28,7 @@ rdma_library(ibverbs "${CMAKE_CURRENT_BINARY_DIR}/libibverbs.map"
# See Documentation/versioning.md
1 1.1.${PACKAGE_VERSION}
cmd.c
+ cmd_cq.c
cmd_fallback.c
cmd_ioctl.c
compat-1_0.c
@@ -486,79 +486,6 @@ int ibv_cmd_dealloc_mw(struct ibv_mw *mw,
return 0;
}
-int ibv_cmd_create_cq(struct ibv_context *context, int cqe,
- struct ibv_comp_channel *channel,
- int comp_vector, struct ibv_cq *cq,
- struct ibv_create_cq *cmd, size_t cmd_size,
- struct ib_uverbs_create_cq_resp *resp, size_t resp_size)
-{
- IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_CQ, resp, resp_size);
- cmd->user_handle = (uintptr_t) cq;
- cmd->cqe = cqe;
- cmd->comp_vector = comp_vector;
- cmd->comp_channel = channel ? channel->fd : -1;
- cmd->reserved = 0;
-
- if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
- return errno;
-
- (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
-
- cq->handle = resp->cq_handle;
- cq->cqe = resp->cqe;
- cq->context = context;
-
- return 0;
-}
-
-int ibv_cmd_create_cq_ex(struct ibv_context *context,
- struct ibv_cq_init_attr_ex *cq_attr,
- struct ibv_cq_ex *cq,
- struct ibv_create_cq_ex *cmd,
- size_t cmd_core_size,
- size_t cmd_size,
- struct ib_uverbs_ex_create_cq_resp *resp,
- size_t resp_core_size,
- size_t resp_size)
-{
- int err;
-
- memset(cmd, 0, cmd_core_size);
- IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size, CREATE_CQ, resp,
- resp_core_size, resp_size);
-
- if (cq_attr->comp_mask & ~(IBV_CQ_INIT_ATTR_MASK_RESERVED - 1))
- return EINVAL;
-
- cmd->user_handle = (uintptr_t)cq;
- cmd->cqe = cq_attr->cqe;
- cmd->comp_vector = cq_attr->comp_vector;
- cmd->comp_channel = cq_attr->channel ? cq_attr->channel->fd : -1;
- cmd->comp_mask = 0;
-
- if (cmd_core_size >= offsetof(struct ibv_create_cq_ex, flags) +
- sizeof(cmd->flags)) {
- if ((cq_attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_FLAGS) &&
- (cq_attr->flags & ~(IBV_CREATE_CQ_ATTR_RESERVED - 1)))
- return EOPNOTSUPP;
-
- if (cq_attr->wc_flags & IBV_WC_EX_WITH_COMPLETION_TIMESTAMP)
- cmd->flags |= IBV_CREATE_CQ_EX_KERNEL_FLAG_COMPLETION_TIMESTAMP;
- }
-
- err = write(context->cmd_fd, cmd, cmd_size);
- if (err != cmd_size)
- return errno;
-
- (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
-
- cq->handle = resp->base.cq_handle;
- cq->cqe = resp->base.cqe;
- cq->context = context;
-
- return 0;
-}
-
int ibv_cmd_poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc)
{
struct ibv_poll_cq cmd;
@@ -638,29 +565,6 @@ int ibv_cmd_resize_cq(struct ibv_cq *cq, int cqe,
return 0;
}
-int ibv_cmd_destroy_cq(struct ibv_cq *cq)
-{
- struct ibv_destroy_cq cmd;
- struct ib_uverbs_destroy_cq_resp resp;
-
- IBV_INIT_CMD_RESP(&cmd, sizeof cmd, DESTROY_CQ, &resp, sizeof resp);
- cmd.cq_handle = cq->handle;
- cmd.reserved = 0;
-
- if (write(cq->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
- return errno;
-
- (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);
-
- pthread_mutex_lock(&cq->mutex);
- while (cq->comp_events_completed != resp.comp_events_reported ||
- cq->async_events_completed != resp.async_events_reported)
- pthread_cond_wait(&cq->cond, &cq->mutex);
- pthread_mutex_unlock(&cq->mutex);
-
- return 0;
-}
-
int ibv_cmd_create_srq(struct ibv_pd *pd,
struct ibv_srq *srq, struct ibv_srq_init_attr *attr,
struct ibv_create_srq *cmd, size_t cmd_size,
new file mode 100644
@@ -0,0 +1,186 @@
+/*
+ * Copyright (c) 2018 Mellanox Technologies, Ltd. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <infiniband/cmd_write.h>
+
+static int ibv_icmd_create_cq(struct ibv_context *context, int cqe,
+ struct ibv_comp_channel *channel, int comp_vector,
+ uint32_t flags, struct ibv_cq *cq,
+ struct ibv_command_buffer *link)
+{
+ DECLARE_FBCMD_BUFFER(cmdb, UVERBS_OBJECT_CQ, UVERBS_CQ_CREATE, 7, link);
+ struct ib_uverbs_attr *handle;
+ uint32_t resp_cqe;
+ int ret;
+
+ cq->context = context;
+
+ handle = fill_attr_out_obj(cmdb, CREATE_CQ_HANDLE);
+ fill_attr_out_ptr(cmdb, CREATE_CQ_RESP_CQE, &resp_cqe);
+
+ fill_attr_in_uint32(cmdb, CREATE_CQ_CQE, cqe);
+ fill_attr_in_uint64(cmdb, CREATE_CQ_USER_HANDLE, (uintptr_t)cq);
+ if (channel)
+ fill_attr_in_fd(cmdb, CREATE_CQ_COMP_CHANNEL, channel->fd);
+ fill_attr_in_uint32(cmdb, CREATE_CQ_COMP_VECTOR, comp_vector);
+
+ if (flags) {
+ fallback_require_ex(cmdb);
+ fill_attr_in_uint32(cmdb, CREATE_CQ_FLAGS, flags);
+ }
+
+ switch (execute_ioctl_fallback(cq->context, create_cq, cmdb, &ret)) {
+ case TRY_WRITE: {
+ DECLARE_LEGACY_UHW_BUFS(link, create_cq);
+
+ *req = (struct ib_uverbs_create_cq){
+ .user_handle = (uintptr_t)cq,
+ .cqe = cqe,
+ .comp_vector = comp_vector,
+ .comp_channel = channel ? channel->fd : -1,
+ };
+
+ ret = execute_write_bufs(IB_USER_VERBS_CMD_CREATE_CQ,
+ cq->context, req, resp);
+ if (ret)
+ return ret;
+
+ cq->handle = resp->cq_handle;
+ cq->cqe = resp->cqe;
+
+ return 0;
+ }
+ case TRY_WRITE_EX: {
+ DECLARE_LEGACY_UHW_BUFS_EX(link, create_cq);
+
+ *req = (struct ib_uverbs_ex_create_cq){
+ .user_handle = (uintptr_t)cq,
+ .cqe = cqe,
+ .comp_vector = comp_vector,
+ .comp_channel = channel ? channel->fd : -1,
+ .flags = flags,
+ };
+
+ ret = execute_write_bufs_ex(IB_USER_VERBS_EX_CMD_CREATE_CQ,
+ cq->context, req, resp);
+ if (ret)
+ return ret;
+
+ cq->handle = resp->base.cq_handle;
+ cq->cqe = resp->base.cqe;
+
+ return 0;
+ }
+
+ case ERROR:
+ return ret;
+
+ case SUCCESS:
+ break;
+ }
+
+ cq->handle = read_attr_obj(CREATE_CQ_HANDLE, handle);
+ cq->cqe = resp_cqe;
+
+ return 0;
+}
+
+int ibv_cmd_create_cq(struct ibv_context *context, int cqe,
+ struct ibv_comp_channel *channel, int comp_vector,
+ struct ibv_cq *cq, struct ibv_create_cq *cmd,
+ size_t cmd_size, struct ib_uverbs_create_cq_resp *resp,
+ size_t resp_size)
+{
+ DECLARE_CMD_BUFFER_COMPAT(cmdb, UVERBS_OBJECT_CQ, UVERBS_CQ_CREATE);
+
+ return ibv_icmd_create_cq(context, cqe, channel, comp_vector, 0, cq,
+ cmdb);
+}
+
+int ibv_cmd_create_cq_ex(struct ibv_context *context,
+ struct ibv_cq_init_attr_ex *cq_attr,
+ struct ibv_cq_ex *cq,
+ struct ibv_create_cq_ex *cmd,
+ size_t cmd_size,
+ struct ib_uverbs_ex_create_cq_resp *resp,
+ size_t resp_size)
+{
+ DECLARE_CMD_BUFFER_COMPAT(cmdb, UVERBS_OBJECT_CQ, UVERBS_CQ_CREATE);
+ uint32_t flags = 0;
+
+ if (!check_comp_mask(cq_attr->comp_mask, IBV_CQ_INIT_ATTR_MASK_FLAGS))
+ return EOPNOTSUPP;
+
+ if (cq_attr->wc_flags & IBV_WC_EX_WITH_COMPLETION_TIMESTAMP)
+ flags |= IBV_CREATE_CQ_EX_KERNEL_FLAG_COMPLETION_TIMESTAMP;
+
+ return ibv_icmd_create_cq(context, cq_attr->cqe, cq_attr->channel,
+ cq_attr->comp_vector, flags,
+ ibv_cq_ex_to_cq(cq), cmdb);
+}
+
+int ibv_cmd_destroy_cq(struct ibv_cq *cq)
+{
+ DECLARE_FBCMD_BUFFER(cmdb, UVERBS_OBJECT_CQ, UVERBS_CQ_DESTROY, 2,
+ NULL);
+ DECLARE_LEGACY_CORE_BUFS(destroy_cq);
+ int ret;
+
+ fill_attr_out_ptr(cmdb, DESTROY_CQ_RESP, &resp);
+ fill_attr_in_obj(cmdb, DESTROY_CQ_HANDLE, cq->handle);
+
+ switch (execute_ioctl_fallback(cq->context, destroy_cq, cmdb, &ret)) {
+ case TRY_WRITE: {
+ *req = (struct ib_uverbs_destroy_cq){
+ .cq_handle = cq->handle,
+ };
+
+ ret = execute_write(IB_USER_VERBS_CMD_DESTROY_CQ, cq->context,
+ req, &resp);
+ break;
+ }
+
+ default:
+ break;
+ }
+
+ if (ret)
+ return ret;
+
+ pthread_mutex_lock(&cq->mutex);
+ while (cq->comp_events_completed != resp.comp_events_reported ||
+ cq->async_events_completed != resp.async_events_reported)
+ pthread_cond_wait(&cq->cond, &cq->mutex);
+ pthread_mutex_unlock(&cq->mutex);
+
+ return 0;
+}
@@ -373,10 +373,8 @@ int ibv_cmd_create_cq_ex(struct ibv_context *context,
struct ibv_cq_init_attr_ex *cq_attr,
struct ibv_cq_ex *cq,
struct ibv_create_cq_ex *cmd,
- size_t cmd_core_size,
size_t cmd_size,
struct ib_uverbs_ex_create_cq_resp *resp,
- size_t resp_core_size,
size_t resp_size);
int ibv_cmd_poll_cq(struct ibv_cq *cq, int ne, struct ibv_wc *wc);
int ibv_cmd_req_notify_cq(struct ibv_cq *cq, int solicited_only);
@@ -170,12 +170,7 @@ struct ibv_create_comp_channel {
struct ibv_create_cq {
struct ib_uverbs_cmd_hdr hdr;
- __u64 response;
- __u64 user_handle;
- __u32 cqe;
- __u32 comp_vector;
- __s32 comp_channel;
- __u32 reserved;
+ struct ib_uverbs_create_cq core_payload;
};
enum ibv_create_cq_ex_kernel_flags {
@@ -184,13 +179,7 @@ enum ibv_create_cq_ex_kernel_flags {
struct ibv_create_cq_ex {
struct ex_hdr hdr;
- __u64 user_handle;
- __u32 cqe;
- __u32 comp_vector;
- __s32 comp_channel;
- __u32 comp_mask;
- __u32 flags;
- __u32 reserved;
+ struct ib_uverbs_ex_create_cq core_payload;
};
struct ibv_poll_cq {
@@ -215,9 +204,7 @@ struct ibv_resize_cq {
struct ibv_destroy_cq {
struct ib_uverbs_cmd_hdr hdr;
- __u64 response;
- __u32 cq_handle;
- __u32 reserved;
+ struct ib_uverbs_destroy_cq core_payload;
};
#define IBV_CREATE_QP_COMMON \
@@ -168,7 +168,6 @@ int c4iw_dereg_mr(struct ibv_mr *mr)
struct ibv_cq *c4iw_create_cq(struct ibv_context *context, int cqe,
struct ibv_comp_channel *channel, int comp_vector)
{
- struct ibv_create_cq cmd;
struct c4iw_create_cq_resp resp;
struct c4iw_cq *chp;
struct c4iw_dev *dev = to_c4iw_dev(context->device);
@@ -181,7 +180,7 @@ struct ibv_cq *c4iw_create_cq(struct ibv_context *context, int cqe,
resp.reserved = 0;
ret = ibv_cmd_create_cq(context, cqe, channel, comp_vector,
- &chp->ibv_cq, &cmd, sizeof cmd,
+ &chp->ibv_cq, NULL, 0,
&resp.ibv_resp, sizeof resp);
if (ret)
goto err1;
@@ -169,7 +169,6 @@ struct ibv_cq *hfi1_create_cq(struct ibv_context *context, int cqe,
int comp_vector)
{
struct hfi1_cq *cq;
- struct ibv_create_cq cmd;
struct hfi1_create_cq_resp resp;
int ret;
size_t size;
@@ -180,7 +179,7 @@ struct ibv_cq *hfi1_create_cq(struct ibv_context *context, int cqe,
return NULL;
ret = ibv_cmd_create_cq(context, cqe, channel, comp_vector,
- &cq->ibv_cq, &cmd, sizeof cmd,
+ &cq->ibv_cq, NULL, 0,
&resp.ibv_resp, sizeof resp);
if (ret) {
free(cq);
@@ -205,8 +204,6 @@ struct ibv_cq *hfi1_create_cq_v1(struct ibv_context *context, int cqe,
int comp_vector)
{
struct ibv_cq *cq;
- struct ibv_create_cq cmd;
- struct ib_uverbs_create_cq_resp resp;
int ret;
cq = malloc(sizeof *cq);
@@ -214,7 +211,7 @@ struct ibv_cq *hfi1_create_cq_v1(struct ibv_context *context, int cqe,
return NULL;
ret = ibv_cmd_create_cq(context, cqe, channel, comp_vector,
- cq, &cmd, sizeof cmd, &resp, sizeof resp);
+ cq, NULL, 0, NULL, 0);
if (ret) {
free(cq);
return NULL;
@@ -148,7 +148,6 @@ struct ibv_cq *ipath_create_cq(struct ibv_context *context, int cqe,
int comp_vector)
{
struct ipath_cq *cq;
- struct ibv_create_cq cmd;
struct ipath_create_cq_resp resp;
int ret;
size_t size;
@@ -158,7 +157,7 @@ struct ibv_cq *ipath_create_cq(struct ibv_context *context, int cqe,
return NULL;
ret = ibv_cmd_create_cq(context, cqe, channel, comp_vector,
- &cq->ibv_cq, &cmd, sizeof cmd,
+ &cq->ibv_cq, NULL, 0,
&resp.ibv_resp, sizeof resp);
if (ret) {
free(cq);
@@ -183,8 +182,6 @@ struct ibv_cq *ipath_create_cq_v1(struct ibv_context *context, int cqe,
int comp_vector)
{
struct ibv_cq *cq;
- struct ibv_create_cq cmd;
- struct ib_uverbs_create_cq_resp resp;
int ret;
cq = malloc(sizeof *cq);
@@ -192,7 +189,7 @@ struct ibv_cq *ipath_create_cq_v1(struct ibv_context *context, int cqe,
return NULL;
ret = ibv_cmd_create_cq(context, cqe, channel, comp_vector,
- cq, &cmd, sizeof cmd, &resp, sizeof resp);
+ cq, NULL, 0, NULL, 0);
if (ret) {
free(cq);
return NULL;
@@ -413,8 +413,8 @@ static int mlx4_cmd_create_cq(struct ibv_context *context,
struct ibv_cq_init_attr_ex *cq_attr,
struct mlx4_cq *cq)
{
- struct mlx4_create_cq cmd = {};
- struct mlx4_create_cq_resp resp = {};
+ struct mlx4_create_cq cmd;
+ struct mlx4_create_cq_resp resp;
int ret;
cmd.buf_addr = (uintptr_t) cq->buf.buf;
@@ -436,8 +436,8 @@ static int mlx4_cmd_create_cq_ex(struct ibv_context *context,
struct ibv_cq_init_attr_ex *cq_attr,
struct mlx4_cq *cq)
{
- struct mlx4_create_cq_ex cmd = {};
- struct mlx4_create_cq_resp_ex resp = {};
+ struct mlx4_create_cq_ex cmd;
+ struct mlx4_create_cq_resp_ex resp;
int ret;
cmd.buf_addr = (uintptr_t) cq->buf.buf;
@@ -445,10 +445,8 @@ static int mlx4_cmd_create_cq_ex(struct ibv_context *context,
ret = ibv_cmd_create_cq_ex(context, cq_attr,
&cq->ibv_cq, &cmd.ibv_cmd,
- sizeof(cmd.ibv_cmd),
sizeof(cmd),
&resp.ibv_resp,
- sizeof(resp.ibv_resp),
sizeof(resp));
if (!ret)
cq->cqn = resp.cqn;
@@ -162,7 +162,6 @@ static struct ibv_cq *rxe_create_cq(struct ibv_context *context, int cqe,
int comp_vector)
{
struct rxe_cq *cq;
- struct ibv_create_cq cmd;
struct rxe_create_cq_resp resp;
int ret;
@@ -172,7 +171,7 @@ static struct ibv_cq *rxe_create_cq(struct ibv_context *context, int cqe,
}
ret = ibv_cmd_create_cq(context, cqe, channel, comp_vector,
- &cq->ibv_cq, &cmd, sizeof cmd,
+ &cq->ibv_cq, NULL, 0,
&resp.ibv_resp, sizeof resp);
if (ret) {
free(cq);