c-code

/*
 * Copyright (c) 2005-2012 Intel Corporation.  All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#if HAVE_CONFIG_H
#  include <config.h>
#endif /* HAVE_CONFIG_H */

#include <user_global.h>

#include <stdlib.h>
#include <string.h>
#include <user_global.h>
#include <stdio.h>
#include <fcntl.h>
#include <errno.h>
#include <stdint.h>
//#include <poll.h>
//#include <unistd.h>
#include <pthread.h>
//#include <endian.h>
//#include <byteswap.h>
#include <stddef.h>
#include <netdb.h>
#include <stats.h>

#include "cma.h"
#include <infiniband/driver.h>
#include <infiniband/marshall.h>
#include <rdma/rdma_cma.h>
#include <rdma/rdma_cma_abi.h>
#include <rdma/rdma_verbs.h>
#include <infiniband/ib.h>
#include <ns_file.h>
#include <infiniband/arch.h>
#include <user_global.h>
#include <trace.h>

extern int nsadi_enabled;
extern int NSKVCore_getModel(void) ;
extern int NSKVCore_modelToSystemClass_(int a) ;

#define CMA_INIT_CMD(req, req_size, op)		\
do {						\
	memset(req, 0, req_size);		\
	(req)->cmd = UCMA_CMD_##op;		\
	(req)->in  = req_size - sizeof(struct ucma_abi_cmd_hdr); \
} while (0)

#define CMA_INIT_CMD_RESP(req, req_size, op, resp, resp_size) \
do {						\
	CMA_INIT_CMD(req, req_size, op);	\
	(req)->out = resp_size;			\
	(req)->response = (uintptr_t) (resp);	\
} while (0)

struct cma_device {
	struct ibv_context *verbs;
	struct ibv_pd	   *pd;
	uint64_t	    guid;
	int		    port_cnt;
	int		    refcnt;
	int		    max_qpsize;
	uint8_t		    max_initiator_depth;
	uint8_t		    max_responder_resources;
};

struct cma_id_private {
	struct rdma_cm_id	id;
	struct cma_device	*cma_dev;
	void			*connect;
	size_t			connect_len;
	int			events_completed;
	int			connect_error;
	int			sync;
	pthread_cond_t		cond;
	pthread_mutex_t		mut;
	uint32_t		handle;
	struct cma_multicast	*mc_list;
	struct ibv_qp_init_attr	*qp_init_attr;
	uint8_t			initiator_depth;
	uint8_t			responder_resources;
};

struct cma_multicast {
	struct cma_multicast  *next;
	struct cma_id_private *id_priv;
	void		*context;
	int		events_completed;
	pthread_cond_t	cond;
	uint32_t	handle;
	union ibv_gid	mgid;
	uint16_t	mlid;
	struct sockaddr_storage addr;
};

struct cma_event {
	struct rdma_cm_event	event;
	uint8_t			private_data[RDMA_MAX_PRIVATE_DATA];
	struct cma_id_private	*id_priv;
	struct cma_multicast	*mc;
};

static struct cma_device *cma_dev_array;
static int cma_dev_cnt;
static pthread_mutex_t mut = PTHREAD_MUTEX_INITIALIZER;
static int abi_ver = RDMA_USER_CM_MAX_ABI_VERSION;
int af_ib_support;

static void ucma_cleanup(void)
{
	ucma_ib_cleanup();

	if (cma_dev_cnt) {
		while (cma_dev_cnt--) {
			if (cma_dev_array[cma_dev_cnt].refcnt)
				ibv_dealloc_pd(cma_dev_array[cma_dev_cnt].pd);
			ibv_close_device(cma_dev_array[cma_dev_cnt].verbs);
		}

		free(cma_dev_array);
		cma_dev_cnt = 0;
	}
}

static int check_abi_version(void)
{
	char value[8];

	if(g_trace_level >= TRACE_LEVEL_FIVE) uib_trace_entry_point(CHECK_ABI_VER);

	if ((ibv_read_sysfs_file(ibv_get_sysfs_path(),
				 "misc/rdma_cm/abi_version",
				 value, sizeof value) <= 0) &&
	    (ibv_read_sysfs_file(ibv_get_sysfs_path(),
				 "infiniband_ucma/abi_version",
				 value, sizeof value) <= 0)) {
		/*
		 * Older version of Linux do not have class/misc.  To support
		 * backports, assume the most recent version of the ABI.  If
		 * we're wrong, we'll simply fail later when calling the ABI.
		 */
		//fprintf(stderr, PFX "Warning: couldn't read ABI version.\n");
		//fprintf(stderr, PFX "Warning: assuming: %d\n", abi_ver);
		uib_trace(CHECK_ABI_VER, READ_ABI_VERSION, TRACE_LEVEL_ONE, UIB_NO_QPN,
			  -1, UIB_NO_ERRNO, TRACE_ERROR, UIB_NO_INSTANCE, UIB_NO_TRACE_DATA, NULL);
		return 0;
	}

	abi_ver = strtol(value, NULL, 10);
	if (abi_ver < RDMA_USER_CM_MIN_ABI_VERSION ||
	    abi_ver > RDMA_USER_CM_MAX_ABI_VERSION) {
		//fprintf(stderr, PFX "Fatal: kernel ABI version %d "
	   //			"doesn't match library version %d.\n",
	   //			abi_ver, RDMA_USER_CM_MAX_ABI_VERSION);
		uib_trace(CHECK_ABI_VER, ABI_VERSION, TRACE_LEVEL_ONE, UIB_NO_QPN,
			  -1, UIB_NO_ERRNO, TRACE_ERROR, UIB_NO_INSTANCE, UIB_NO_TRACE_DATA, NULL);
		return -1;
	}
	return 0;
}

/*
 * This function is called holding the mutex lock
 * cma_dev_cnt must be set before calling this function to
 * ensure that the lock is not acquired recursively.
 */
static void ucma_set_af_ib_support(void)
{
	struct rdma_cm_id *id;
	struct sockaddr_ib sib;
	int ret;

	ret = rdma_create_id(NULL, &id, NULL, RDMA_PS_IB);
	if (ret)
		return;

	memset(&sib, 0, sizeof sib);
	sib.sib_family = AF_IB;
	sib.sib_sid = htonll(RDMA_IB_IP_PS_TCP);
	sib.sib_sid_mask = htonll(RDMA_IB_IP_PS_MASK);
	af_ib_support = 1;
	ret = rdma_bind_addr(id, (struct sockaddr *) &sib);
	af_ib_support = !ret;

	rdma_destroy_id(id);
}

int ucma_init(void)
{
	struct ibv_device **dev_list = NULL;
	struct cma_device *cma_dev;
	struct ibv_device_attr attr;
	int i, ret, dev_cnt;

	if(g_trace_level >= TRACE_LEVEL_FIVE) uib_trace_entry_point(UCMA_INIT);


	/* Quick check without lock to see if we're already initialized */
	if (cma_dev_cnt)
		return 0;

	pthread_mutex_lock(&mut);
	if (cma_dev_cnt) {
		pthread_mutex_unlock(&mut);
		return 0;
	}

	ret = check_abi_version();
	if (ret) {
		uib_trace(UCMA_INIT, SOCKET_CREATE_ERROR, TRACE_LEVEL_ONE, UIB_NO_QPN,
			  ret, UIB_NO_ERRNO, TRACE_ERROR, UIB_NO_INSTANCE, UIB_NO_TRACE_DATA, NULL);
		goto err1;
	}

	dev_list = ibv_get_device_list(&dev_cnt);
	if (!dev_list) {
//		fprintf(stderr, PFX "Fatal: unable to get RDMA device list\n");
		ret = ERR(ENODEV);
		uib_trace(UCMA_INIT, NO_DEVICES, TRACE_LEVEL_ONE, UIB_NO_QPN,
			  -1, errno, TRACE_ERROR, 1, UIB_NO_TRACE_DATA, NULL);
		goto err1;
	}

	if (!dev_cnt) {
		//fprintf(stderr, PFX "Fatal: no RDMA devices found\n");
		ret = ERR(ENODEV);
		uib_trace(UCMA_INIT, NO_DEVICES, TRACE_LEVEL_ONE, UIB_NO_QPN,
			  -1, errno, TRACE_ERROR, 2, UIB_NO_TRACE_DATA, NULL);
		goto err2;
	}

	cma_dev_array = calloc(dev_cnt, sizeof *cma_dev);
	if (!cma_dev_array) {
		ret = ERR(ENOMEM);
		uib_trace(UCMA_INIT, ALLOC_ERROR, TRACE_LEVEL_ONE, UIB_NO_QPN,
			  -1, errno, TRACE_ERROR, UIB_NO_INSTANCE, UIB_NO_TRACE_DATA, NULL);
		goto err2;
	}

	for (i = 0; dev_list[i];) {
		cma_dev = &cma_dev_array[i];

		cma_dev->guid = ibv_get_device_guid(dev_list[i]);
		cma_dev->verbs = ibv_open_device(dev_list[i]);
		if (!cma_dev->verbs) {
			//fprintf(stderr, PFX "Fatal: unable to open RDMA device\n");
			ret = ERR(ENODEV);
			uib_trace(UCMA_INIT, DEVICE_OPEN_ERROR, TRACE_LEVEL_ONE, UIB_NO_QPN,
				  -1, errno, TRACE_ERROR, UIB_NO_INSTANCE, UIB_NO_TRACE_DATA, NULL);
			goto err3;
		}

		i++;
		ret = ibv_query_device(cma_dev->verbs, &attr);
		if (ret) {
			//fprintf(stderr, PFX "Fatal: unable to query RDMA device\n");
			ret = ERR(ret);
			uib_trace(UCMA_INIT, QUERY_DEVICE_ERROR, TRACE_LEVEL_ONE, UIB_NO_QPN,
				  -1, errno, TRACE_ERROR, UIB_NO_INSTANCE, UIB_NO_TRACE_DATA, NULL);
			goto err3;
		}

		cma_dev->port_cnt = attr.phys_port_cnt;
		cma_dev->max_qpsize = attr.max_qp_wr;
		cma_dev->max_initiator_depth = (uint8_t) attr.max_qp_init_rd_atom;
		cma_dev->max_responder_resources = (uint8_t) attr.max_qp_rd_atom;
	}

	cma_dev_cnt = dev_cnt;
	ucma_set_af_ib_support();
	pthread_mutex_unlock(&mut);
	ibv_free_device_list(dev_list);
	return 0;

err3:
	while (i--)
		ibv_close_device(cma_dev_array[i].verbs);
	free(cma_dev_array);
err2:
	ibv_free_device_list(dev_list);
err1:
	pthread_mutex_unlock(&mut);
	return ret;
}

struct ibv_context **rdma_get_devices(int *num_devices)
{
	struct ibv_context **devs = NULL;
	int i;

	if (ucma_init())
		goto out;

	devs = malloc(sizeof *devs * (cma_dev_cnt + 1));
	if (!devs)
		goto out;

	for (i = 0; i < cma_dev_cnt; i++)
		devs[i] = cma_dev_array[i].verbs;
	devs[i] = NULL;
out:
	if (num_devices)
		*num_devices = devs ? cma_dev_cnt : 0;
	return devs;
}

void rdma_free_devices(struct ibv_context **list)
{
	free(list);
}

void /*__attribute__((destructor))*/ rdma_cma_fini(void)
{
	ucma_cleanup();
}

struct rdma_event_channel *rdma_create_event_channel(void)
{
	struct rdma_event_channel *channel;

	if (ucma_init())
		return NULL;

	if(g_trace_level >= TRACE_LEVEL_FIVE) uib_trace_entry_point(CREATE_EVENT_CHAN);
	channel = malloc(sizeof *channel);
	if (!channel)
		return NULL;

	channel->fd = ns_open("/dev/infiniband/rdma_cm",strlen("/dev/infiniband/rdma_cm"), 0xdeadbeef);
	if (channel->fd < 0) {
		//fprintf(stderr, PFX "Fatal: unable to open /dev/infiniband/rdma_cm\n");
		uib_trace(CREATE_EVENT_CHAN, OPEN_RDMA_CM, TRACE_LEVEL_ONE, UIB_NO_QPN,
			  UIB_NO_ECODE, UIB_NO_ERRNO, TRACE_ERROR, UIB_NO_INSTANCE, UIB_NO_TRACE_DATA, NULL);
		goto err;
	}
	return channel;
err:
	free(channel);
	return NULL;
}

void rdma_destroy_event_channel(struct rdma_event_channel *channel)
{
	put_unregFile(channel->fd) ;
	ns_close(channel->fd);
	free(channel);
}

static int ucma_get_device(struct cma_id_private *id_priv, uint64_t guid)
{
	struct cma_device *cma_dev;
	int i, ret = 0;

	for (i = 0; i < cma_dev_cnt; i++) {
		cma_dev = &cma_dev_array[i];
		if (cma_dev->guid == guid)
			goto match;
	}

	return ERR(ENODEV);
match:
	pthread_mutex_lock(&mut);
	if (!cma_dev->refcnt++) {
		cma_dev->pd = ibv_alloc_pd(cma_dev_array[i].verbs);
		if (!cma_dev->pd) {
			cma_dev->refcnt--;
			ret = ERR(ENOMEM);
			goto out;
		}
	}
	id_priv->cma_dev = cma_dev;
	id_priv->id.verbs = cma_dev->verbs;
	id_priv->id.pd = cma_dev->pd;
out:
	pthread_mutex_unlock(&mut);
	return ret;
}

static void ucma_put_device(struct cma_device *cma_dev)
{
	pthread_mutex_lock(&mut);
	if (!--cma_dev->refcnt)
		ibv_dealloc_pd(cma_dev->pd);
	pthread_mutex_unlock(&mut);
}

static void ucma_free_id(struct cma_id_private *id_priv)
{
	if (id_priv->cma_dev)
		ucma_put_device(id_priv->cma_dev);
	pthread_cond_destroy(&id_priv->cond);
	pthread_mutex_destroy(&id_priv->mut);
	if (id_priv->id.route.path_rec)
		free(id_priv->id.route.path_rec);

	if (id_priv->sync)
		rdma_destroy_event_channel(id_priv->id.channel);
	if (id_priv->connect_len)
		free(id_priv->connect);
	free(id_priv);
}

static struct cma_id_private *ucma_alloc_id(struct rdma_event_channel *channel,
					    void *context,
					    enum rdma_port_space ps,
					    enum ibv_qp_type qp_type)
{
	struct cma_id_private *id_priv;

	id_priv = calloc(1, sizeof *id_priv);
	if (!id_priv)
		return NULL;

	id_priv->id.context = context;
	id_priv->id.ps = ps;
	id_priv->id.qp_type = qp_type;

	if (!channel) {
		id_priv->id.channel = rdma_create_event_channel();
		if (!id_priv->id.channel)
			goto err;
		id_priv->sync = 1;
	} else {
		id_priv->id.channel = channel;
	}

	pthread_mutex_init(&id_priv->mut, NULL);
	if (pthread_cond_init(&id_priv->cond, NULL))
		goto err;

	return id_priv;

err:	ucma_free_id(id_priv);
	return NULL;
}

static int rdma_create_id2(struct rdma_event_channel *channel,
			   struct rdma_cm_id **id, void *context,
			   enum rdma_port_space ps, enum ibv_qp_type qp_type)
{
	struct ucma_abi_create_id_resp resp;
	struct ucma_abi_create_id cmd;
	struct cma_id_private *id_priv;
	int ret;

	ret = ucma_init();
	if (ret)
		return ret;

	id_priv = ucma_alloc_id(channel, context, ps, qp_type);
	if (!id_priv)
		return ERR(ENOMEM);

	CMA_INIT_CMD_RESP(&cmd, sizeof cmd, CREATE_ID, &resp, sizeof resp);
	cmd.uid = (uintptr_t) id_priv;
	cmd.ps = ps;
	cmd.qp_type = qp_type;

	ret = ns_write(id_priv->id.channel->fd, &cmd, sizeof cmd);
	if (ret != sizeof cmd)
		goto err;

	VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);

	id_priv->handle = resp.id;
	*id = &id_priv->id;
	return 0;

err:	ucma_free_id(id_priv);
	return (ERR(-ret));
}

int rdma_create_id(struct rdma_event_channel *channel,
		   struct rdma_cm_id **id, void *context,
		   enum rdma_port_space ps)
{
	enum ibv_qp_type qp_type;

	qp_type = (ps == RDMA_PS_IPOIB || ps == RDMA_PS_UDP) ?
		  IBV_QPT_UD : IBV_QPT_RC;
	return rdma_create_id2(channel, id, context, ps, qp_type);
}

static int ucma_destroy_kern_id(int fd, uint32_t handle)
{
	struct ucma_abi_destroy_id_resp resp;
	struct ucma_abi_destroy_id cmd;
	int ret;

	CMA_INIT_CMD_RESP(&cmd, sizeof cmd, DESTROY_ID, &resp, sizeof resp);
	cmd.id = handle;

	ret = ns_write(fd, &cmd, sizeof cmd);
	if (ret != sizeof cmd){
		return (ret >= 0) ? ERR(ENODATA) : ERR(-ret);
	}

	VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);

	return resp.events_reported;
}

int rdma_destroy_id(struct rdma_cm_id *id)
{
	struct cma_id_private *id_priv;
	int ret;

	id_priv = container_of(id, struct cma_id_private, id);
	ret = ucma_destroy_kern_id(id->channel->fd, id_priv->handle);
	if (ret < 0)
		return ret;

	if (id_priv->id.event)
		rdma_ack_cm_event(id_priv->id.event);

	pthread_mutex_lock(&id_priv->mut);
	while (id_priv->events_completed < ret)
		pthread_cond_wait(&id_priv->cond, &id_priv->mut);
	pthread_mutex_unlock(&id_priv->mut);

	ucma_free_id(id_priv);
	return 0;
}

int ucma_addrlen(struct sockaddr *addr)
{
	if (!addr)
		return 0;

	switch (addr->sa_family) {
	case PF_INET:
		return sizeof(struct sockaddr_in);
	case PF_INET6:
		return sizeof(struct sockaddr_in6);
	case PF_IB:
		return af_ib_support ? sizeof(struct sockaddr_ib) : 0;
	default:
		return 0;
	}
}

static int ucma_query_addr(struct rdma_cm_id *id)
{
	struct ucma_abi_query_addr_resp resp;
	struct ucma_abi_query cmd;
	struct cma_id_private *id_priv;
	int ret;

	CMA_INIT_CMD_RESP(&cmd, sizeof cmd, QUERY, &resp, sizeof resp);
	id_priv = container_of(id, struct cma_id_private, id);
	cmd.id = id_priv->handle;
	cmd.option = UCMA_QUERY_ADDR;

	ret = ns_write(id->channel->fd, &cmd, sizeof cmd);
	if (ret != sizeof cmd)
		return (ret >= 0) ? ERR(ENODATA) : ERR(-ret);

	VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);

	memcpy(&id->route.addr.p1.src_addr, &resp.src_addr, resp.src_size);
	memcpy(&id->route.addr.p2.dst_addr, &resp.dst_addr, resp.dst_size);

	if (!id_priv->cma_dev && resp.node_guid) {
		ret = ucma_get_device(id_priv, resp.node_guid);
		if (ret)
			return ret;
		id->port_num = resp.port_num;
		id->route.addr.addr.ibaddr.pkey = resp.pkey;
	}

	return 0;
}

static int ucma_query_gid(struct rdma_cm_id *id)
{
	struct ucma_abi_query_addr_resp resp;
	struct ucma_abi_query cmd;
	struct cma_id_private *id_priv;
	struct sockaddr_ib *sib;
	int ret;

	CMA_INIT_CMD_RESP(&cmd, sizeof cmd, QUERY, &resp, sizeof resp);
	id_priv = container_of(id, struct cma_id_private, id);
	cmd.id = id_priv->handle;
	cmd.option = UCMA_QUERY_GID;

	ret = ns_write(id->channel->fd, &cmd, sizeof cmd);
	if (ret != sizeof cmd)
		return (ret >= 0) ? ERR(ENODATA) : ERR(-ret);

	VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);

	sib = (struct sockaddr_ib *) &resp.src_addr;
	memcpy(id->route.addr.addr.ibaddr.sgid.raw, sib->sib_addr.sib_raw,
	       sizeof id->route.addr.addr.ibaddr.sgid);

	sib = (struct sockaddr_ib *) &resp.dst_addr;
	memcpy(id->route.addr.addr.ibaddr.dgid.raw, sib->sib_addr.sib_raw,
	       sizeof id->route.addr.addr.ibaddr.dgid);

	return 0;
}

static void ucma_convert_path(struct ibv_path_data *path_data,
			      struct ibv_sa_path_rec *sa_path)
{
	uint32_t fl_hop;

	sa_path->dgid = path_data->path.dgid;
	sa_path->sgid = path_data->path.sgid;
	sa_path->dlid = path_data->path.dlid;
	sa_path->slid = path_data->path.slid;
	sa_path->raw_traffic = 0;

	fl_hop = ntohl(path_data->path.flowlabel_hoplimit);
	sa_path->flow_label = htonl(fl_hop >> 8);
	sa_path->hop_limit = (uint8_t) fl_hop;

	sa_path->traffic_class = path_data->path.tclass;
	sa_path->reversible = path_data->path.reversible_numpath >> 7;
	sa_path->numb_path = 1;
	sa_path->pkey = path_data->path.pkey;
	sa_path->sl = ntohs(path_data->path.qosclass_sl) & 0xF;
	sa_path->mtu_selector = 1;
	sa_path->mtu = path_data->path.mtu & 0x1F;
	sa_path->rate_selector = 1;
	sa_path->rate = path_data->path.rate & 0x1F;
	sa_path->packet_life_time_selector = 1;
	sa_path->packet_life_time = path_data->path.packetlifetime & 0x1F;

	sa_path->preference = (uint8_t) path_data->flags;
}

static int ucma_query_path(struct rdma_cm_id *id)
{
	struct ucma_abi_query_path_resp *resp;
	struct ucma_abi_query cmd;
	struct cma_id_private *id_priv;
	int ret, i, size;

	size = sizeof(*resp) + sizeof(struct ibv_path_data) * 6;
	resp = alloca(size);
	if (!resp)
		return ERR(ENOMEM);

	CMA_INIT_CMD_RESP(&cmd, sizeof cmd, QUERY, resp, size);
	id_priv = container_of(id, struct cma_id_private, id);
	cmd.id = id_priv->handle;
	cmd.option = UCMA_QUERY_PATH;

	ret = ns_write(id->channel->fd, &cmd, sizeof cmd);
	if (ret != sizeof cmd)
		return (ret >= 0) ? ERR(ENODATA) : ERR(-ret);

	VALGRIND_MAKE_MEM_DEFINED(resp, size);

	if (resp->num_paths) {
		id->route.path_rec = malloc(sizeof(*id->route.path_rec) *
					    resp->num_paths);
		if (!id->route.path_rec)
			return ERR(ENOMEM);

		id->route.num_paths = resp->num_paths;
		for (i = 0; i < resp->num_paths; i++)
			ucma_convert_path(&resp->path_data[i], &id->route.path_rec[i]);
	}

	return 0;
}

static int ucma_query_route(struct rdma_cm_id *id)
{
	struct ucma_abi_query_route_resp resp;
	struct ucma_abi_query cmd;
	struct cma_id_private *id_priv;
	int ret, i;

	CMA_INIT_CMD_RESP(&cmd, sizeof cmd, QUERY_ROUTE, &resp, sizeof resp);
	id_priv = container_of(id, struct cma_id_private, id);
	cmd.id = id_priv->handle;

	ret = ns_write(id->channel->fd, &cmd, sizeof cmd);
	if (ret != sizeof cmd)
		return (ret >= 0) ? ERR(ENODATA) : ERR(-ret);

	VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);

	if (resp.num_paths) {
		id->route.path_rec = malloc(sizeof *id->route.path_rec *
					    resp.num_paths);
		if (!id->route.path_rec)
			return ERR(ENOMEM);

		id->route.num_paths = resp.num_paths;
		for (i = 0; i < resp.num_paths; i++)
			ibv_copy_path_rec_from_kern(&id->route.path_rec[i],
						    &resp.ib_route[i]);
	}

	memcpy(id->route.addr.addr.ibaddr.sgid.raw, resp.ib_route[0].sgid,
	       sizeof id->route.addr.addr.ibaddr.sgid);
	memcpy(id->route.addr.addr.ibaddr.dgid.raw, resp.ib_route[0].dgid,
	       sizeof id->route.addr.addr.ibaddr.dgid);
	id->route.addr.addr.ibaddr.pkey = resp.ib_route[0].pkey;
	memcpy(&id->route.addr.p1.src_addr, &resp.src_addr,
	       sizeof resp.src_addr);
	memcpy(&id->route.addr.p2.dst_addr, &resp.dst_addr,
	       sizeof resp.dst_addr);

	if (!id_priv->cma_dev && resp.node_guid) {
		ret = ucma_get_device(id_priv, resp.node_guid);
		if (ret)
			return ret;
		id_priv->id.port_num = resp.port_num;
	}

	return 0;
}

static int rdma_bind_addr2(struct rdma_cm_id *id, struct sockaddr *addr,
			   socklen_t addrlen)
{
	struct ucma_abi_bind cmd;
	struct cma_id_private *id_priv;
	int ret;

	CMA_INIT_CMD(&cmd, sizeof cmd, BIND);
	id_priv = container_of(id, struct cma_id_private, id);
	cmd.id = id_priv->handle;
	cmd.addr_size = addrlen;
	memcpy(&cmd.addr, addr, addrlen);

	ret = ns_write(id->channel->fd, &cmd, sizeof cmd);
	if (ret != sizeof cmd)
		return (ret >= 0) ? ERR(ENODATA) : ERR(-ret);

	return ucma_query_addr(id);
}

int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
{
	struct ucma_abi_bind_ip cmd;
	struct cma_id_private *id_priv;
	int ret, addrlen;

	addrlen = ucma_addrlen(addr);
	if (!addrlen)
		return ERR(EINVAL);

	if (af_ib_support)
		return rdma_bind_addr2(id, addr, addrlen);

	CMA_INIT_CMD(&cmd, sizeof cmd, BIND_IP);
	id_priv = container_of(id, struct cma_id_private, id);
	cmd.id = id_priv->handle;
	memcpy(&cmd.addr, addr, addrlen);

	ret = ns_write(id->channel->fd, &cmd, sizeof cmd);
	if (ret != sizeof cmd) {
		return (ret >= 0) ? ERR(ENODATA) : ERR(-ret);
	}

	return ucma_query_route(id);
}

int ucma_complete(struct rdma_cm_id *id)
{
	struct cma_id_private *id_priv;
	int ret;

	id_priv = container_of(id, struct cma_id_private, id);
	if (!id_priv->sync)
		return 0;

	if (id_priv->id.event) {
		rdma_ack_cm_event(id_priv->id.event);
		id_priv->id.event = NULL;
	}

	ret = rdma_get_cm_event(id_priv->id.channel, &id_priv->id.event);
	if (ret)
		return ret;

	if (id_priv->id.event->status) {
		if (id_priv->id.event->event == RDMA_CM_EVENT_REJECTED)
			ret = ERR(ECONNREFUSED);
		else if (id_priv->id.event->status < 0)
			ret = ERR(id_priv->id.event->status);	// NonStop change, errors are positive
		else
			ret = ERR(id_priv->id.event->status);	// same as above
	}
	return ret;
}

static int rdma_resolve_addr2(struct rdma_cm_id *id, struct sockaddr *src_addr,
			      socklen_t src_len, struct sockaddr *dst_addr,
			      socklen_t dst_len, int timeout_ms)
{
	struct ucma_abi_resolve_addr cmd;
	struct cma_id_private *id_priv;
	int ret;

	CMA_INIT_CMD(&cmd, sizeof cmd, RESOLVE_ADDR);
	id_priv = container_of(id, struct cma_id_private, id);
	cmd.id = id_priv->handle;
	if ((cmd.src_size = src_len))
		memcpy(&cmd.src_addr, src_addr, src_len);
	memcpy(&cmd.dst_addr, dst_addr, dst_len);
	cmd.dst_size = dst_len;
	cmd.timeout_ms = timeout_ms;

	ret = ns_write(id->channel->fd, &cmd, sizeof cmd);
	if (ret != sizeof cmd)
		return (ret >= 0) ? ERR(ENODATA) : ERR(-ret);

	memcpy(&id->route.addr.p2.dst_addr, dst_addr, dst_len);
	return ucma_complete(id);
}

int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
		      struct sockaddr *dst_addr, int timeout_ms)
{
	struct ucma_abi_resolve_ip cmd;
	struct cma_id_private *id_priv;
	int ret, dst_len, src_len;

	dst_len = ucma_addrlen(dst_addr);
	if (!dst_len)
		return ERR(EINVAL);

	src_len = ucma_addrlen(src_addr);
	if (src_addr && !src_len)
		return ERR(EINVAL);

	if (af_ib_support)
		return rdma_resolve_addr2(id, src_addr, src_len, dst_addr,
					  dst_len, timeout_ms);

	CMA_INIT_CMD(&cmd, sizeof cmd, RESOLVE_IP);
	id_priv = container_of(id, struct cma_id_private, id);
	cmd.id = id_priv->handle;
	if (src_addr)
		memcpy(&cmd.src_addr, src_addr, src_len);
	memcpy(&cmd.dst_addr, dst_addr, dst_len);
	cmd.timeout_ms = timeout_ms;

	ret = ns_write(id->channel->fd, &cmd, sizeof cmd);
	if (ret != sizeof cmd)
		return (ret >= 0) ? ERR(ENODATA) : ERR(-ret);

	memcpy(&id->route.addr.p2.dst_addr, dst_addr, dst_len);
	return ucma_complete(id);
}

static int ucma_set_ib_route(struct rdma_cm_id *id)
{
	struct rdma_addrinfo hint, *rai;
	int ret;

	memset(&hint, 0, sizeof hint);
	hint.ai_flags = RAI_ROUTEONLY;
	hint.ai_family = id->route.addr.p1.src_addr.sa_family;
	hint.ai_src_len = ucma_addrlen((struct sockaddr *) &id->route.addr.p1.src_addr);
	hint.ai_src_addr = &id->route.addr.p1.src_addr;
	hint.ai_dst_len = ucma_addrlen((struct sockaddr *) &id->route.addr.p2.dst_addr);
	hint.ai_dst_addr = &id->route.addr.p2.dst_addr;

	ret = rdma_getaddrinfo(NULL, NULL, &hint, &rai);
	if (ret)
		return ret;

	if (rai->ai_route_len)
		ret = rdma_set_option(id, RDMA_OPTION_IB, RDMA_OPTION_IB_PATH,
				      rai->ai_route, rai->ai_route_len);
	else
		ret = -1;

	rdma_freeaddrinfo(rai);
	return ret;
}

int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
{
	struct ucma_abi_resolve_route cmd;
	struct cma_id_private *id_priv;
	int ret;

	id_priv = container_of(id, struct cma_id_private, id);
	if (id->verbs->device->transport_type == IBV_TRANSPORT_IB) {
		ret = ucma_set_ib_route(id);
		if (!ret)
			goto out;
	}

	CMA_INIT_CMD(&cmd, sizeof cmd, RESOLVE_ROUTE);
	cmd.id = id_priv->handle;
	cmd.timeout_ms = timeout_ms;

	ret = ns_write(id->channel->fd, &cmd, sizeof cmd);
	if (ret != sizeof cmd)
		return (ret >= 0) ? ERR(ENODATA) : ERR(-ret);

out:
	return ucma_complete(id);
}

static int ucma_is_ud_qp(enum ibv_qp_type qp_type)
{
	return (qp_type == IBV_QPT_UD);
}

static int rdma_init_qp_attr(struct rdma_cm_id *id, struct ibv_qp_attr *qp_attr,
			     int *qp_attr_mask)
{
	struct ucma_abi_init_qp_attr cmd;
	struct ibv_kern_qp_attr resp;
	struct cma_id_private *id_priv;
	int ret;

	CMA_INIT_CMD_RESP(&cmd, sizeof cmd, INIT_QP_ATTR, &resp, sizeof resp);
	id_priv = container_of(id, struct cma_id_private, id);
	cmd.id = id_priv->handle;
	cmd.qp_state = qp_attr->qp_state;

	ret = ns_write(id->channel->fd, &cmd, sizeof cmd);
	if (ret != sizeof cmd)
		return (ret >= 0) ? ERR(ENODATA) : ERR(-ret);

	VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);

	ibv_copy_qp_attr_from_kern(qp_attr, &resp);
	*qp_attr_mask = resp.qp_attr_mask;
	return 0;
}

static int ucma_modify_qp_rtr(struct rdma_cm_id *id, uint8_t resp_res)
{
	struct ibv_qp_attr qp_attr;
	int qp_attr_mask, ret;

	if (!id->qp)
		return ERR(EINVAL);

	/* Need to update QP attributes from default values. */
	qp_attr.qp_state = IBV_QPS_INIT;
	ret = rdma_init_qp_attr(id, &qp_attr, &qp_attr_mask);
	if (ret)
		return ret;

	ret = ibv_modify_qp(id->qp, &qp_attr, qp_attr_mask);
	if (ret)
		return ERR(ret);

	qp_attr.qp_state = IBV_QPS_RTR;
	ret = rdma_init_qp_attr(id, &qp_attr, &qp_attr_mask);
	if (ret)
		return ret;

	if (resp_res != RDMA_MAX_RESP_RES)
		qp_attr.max_dest_rd_atomic = resp_res;
	return rdma_seterrno(ibv_modify_qp(id->qp, &qp_attr, qp_attr_mask));
}

static int ucma_modify_qp_rts(struct rdma_cm_id *id, uint8_t init_depth)
{
	struct ibv_qp_attr qp_attr;
	int qp_attr_mask, ret;

	qp_attr.qp_state = IBV_QPS_RTS;
	ret = rdma_init_qp_attr(id, &qp_attr, &qp_attr_mask);
	if (ret)
		return ret;

	if (init_depth != RDMA_MAX_INIT_DEPTH)
		qp_attr.max_rd_atomic = init_depth;
	return rdma_seterrno(ibv_modify_qp(id->qp, &qp_attr, qp_attr_mask));
}

static int ucma_modify_qp_sqd(struct rdma_cm_id *id)
{
	struct ibv_qp_attr qp_attr;

	if (!id->qp)
		return 0;

	qp_attr.qp_state = IBV_QPS_SQD;
	return rdma_seterrno(ibv_modify_qp(id->qp, &qp_attr, IBV_QP_STATE));
}

static int ucma_modify_qp_err(struct rdma_cm_id *id)
{
	struct ibv_qp_attr qp_attr;

	if (!id->qp)
		return 0;

	VALGRIND_MAKE_MEM_DEFINED(&qp_attr, sizeof(qp_attr));

	qp_attr.qp_state = IBV_QPS_ERR;
	return rdma_seterrno(ibv_modify_qp(id->qp, &qp_attr, IBV_QP_STATE));
}

static int ucma_find_pkey(struct cma_device *cma_dev, uint8_t port_num,
			  uint16_t pkey, uint16_t *pkey_index)
{
	int ret, i;
	uint16_t chk_pkey;

	for (i = 0, ret = 0; !ret; i++) {
		ret = ibv_query_pkey(cma_dev->verbs, port_num, i, &chk_pkey);
		if (!ret && pkey == chk_pkey) {
			*pkey_index = (uint16_t) i;
			return 0;
		}
	}
	return ERR(EINVAL);
}

static int ucma_init_conn_qp3(struct cma_id_private *id_priv, struct ibv_qp *qp)
{
	struct ibv_qp_attr qp_attr;
	int ret;

	ret = ucma_find_pkey(id_priv->cma_dev, id_priv->id.port_num,
			     id_priv->id.route.addr.addr.ibaddr.pkey,
			     &qp_attr.pkey_index);
	if (ret)
		return ret;

	qp_attr.port_num = id_priv->id.port_num;
	qp_attr.qp_state = IBV_QPS_INIT;
	qp_attr.qp_access_flags = 0;

	ret = ibv_modify_qp(qp, &qp_attr, IBV_QP_STATE | IBV_QP_ACCESS_FLAGS |
					  IBV_QP_PKEY_INDEX | IBV_QP_PORT);
	return rdma_seterrno(ret);
}

static int ucma_init_conn_qp(struct cma_id_private *id_priv, struct ibv_qp *qp)
{
	struct ibv_qp_attr qp_attr;
	int qp_attr_mask, ret;

	if (abi_ver == 3)
		return ucma_init_conn_qp3(id_priv, qp);

	qp_attr.qp_state = IBV_QPS_INIT;
	ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
	if (ret)
		return ret;

	return rdma_seterrno(ibv_modify_qp(qp, &qp_attr, qp_attr_mask));
}

static int ucma_init_ud_qp3(struct cma_id_private *id_priv, struct ibv_qp *qp)
{
	struct ibv_qp_attr qp_attr;
	int ret;

	ret = ucma_find_pkey(id_priv->cma_dev, id_priv->id.port_num,
			     id_priv->id.route.addr.addr.ibaddr.pkey,
			     &qp_attr.pkey_index);
	if (ret)
		return ret;

	qp_attr.port_num = id_priv->id.port_num;
	qp_attr.qp_state = IBV_QPS_INIT;
	qp_attr.qkey = RDMA_UDP_QKEY;

	ret = ibv_modify_qp(qp, &qp_attr, IBV_QP_STATE | IBV_QP_QKEY |
					  IBV_QP_PKEY_INDEX | IBV_QP_PORT);
	if (ret)
		return ERR(ret);

	qp_attr.qp_state = IBV_QPS_RTR;
	ret = ibv_modify_qp(qp, &qp_attr, IBV_QP_STATE);
	if (ret)
		return ERR(ret);

	qp_attr.qp_state = IBV_QPS_RTS;
	qp_attr.sq_psn = 0;
	ret = ibv_modify_qp(qp, &qp_attr, IBV_QP_STATE | IBV_QP_SQ_PSN);
	return rdma_seterrno(ret);
}

static int ucma_init_ud_qp(struct cma_id_private *id_priv, struct ibv_qp *qp)
{
	struct ibv_qp_attr qp_attr;
	int qp_attr_mask, ret;

	if (abi_ver == 3)
		return ucma_init_ud_qp3(id_priv, qp);

	qp_attr.qp_state = IBV_QPS_INIT;
	ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
	if (ret)
		return ret;

	ret = ibv_modify_qp(qp, &qp_attr, qp_attr_mask);
	if (ret)
		return ERR(ret);

	qp_attr.qp_state = IBV_QPS_RTR;
	ret = ibv_modify_qp(qp, &qp_attr, IBV_QP_STATE);
	if (ret)
		return ERR(ret);

	qp_attr.qp_state = IBV_QPS_RTS;
	qp_attr.sq_psn = 0;
	ret = ibv_modify_qp(qp, &qp_attr, IBV_QP_STATE | IBV_QP_SQ_PSN);
	return rdma_seterrno(ret);
}

static void ucma_destroy_cqs(struct rdma_cm_id *id)
{
	if (id->recv_cq)
		ibv_destroy_cq(id->recv_cq);

	if (id->recv_cq_channel)
		ibv_destroy_comp_channel(id->recv_cq_channel);

	if (id->send_cq && (id->send_cq != id->recv_cq))
		ibv_destroy_cq(id->send_cq);

	if (id->send_cq_channel && (id->send_cq_channel != id->recv_cq_channel))
		ibv_destroy_comp_channel(id->send_cq_channel);
}

static int ucma_create_cqs(struct rdma_cm_id *id, uint32_t send_size, uint32_t recv_size)
{
	if (recv_size) {
		id->recv_cq_channel = ibv_create_comp_channel(id->verbs);
		if (!id->recv_cq_channel)
			goto err;

		id->recv_cq = ibv_create_cq(id->verbs, recv_size,
					    id, id->recv_cq_channel, 0);
		if (!id->recv_cq)
			goto err;
	}

	if (send_size) {
		id->send_cq_channel = ibv_create_comp_channel(id->verbs);
		if (!id->send_cq_channel)
			goto err;

		id->send_cq = ibv_create_cq(id->verbs, send_size,
					    id, id->send_cq_channel, 0);
		if (!id->send_cq)
			goto err;
	}

	return 0;
err:
	ucma_destroy_cqs(id);
	return ERR(ENOMEM);
}

int rdma_create_srq(struct rdma_cm_id *id, struct ibv_pd *pd,
		    struct ibv_srq_init_attr *attr)
{
	//struct cma_id_private *id_priv;
	struct ibv_srq *srq;
	int ret;

	//id_priv = container_of(id, struct cma_id_private, id);
	if (!pd)
		pd = id->pd;

#ifdef IBV_XRC_OPS
	if (attr->srq_type == IBV_SRQT_XRC) {
		if (!attr->ext.xrc.cq) {
			ret = ucma_create_cqs(id, 0, attr->attr.max_wr);
			if (ret)
				return ret;

			attr->ext.xrc.cq = id->recv_cq;
		}
	}

	srq = ibv_create_xsrq(pd, attr);
#else
	srq = ibv_create_srq(pd, attr);
#endif
	if (!srq) {
		ret = -1;
		goto err;
	}

	id->pd = pd;
	id->srq = srq;
	return 0;
err:
	ucma_destroy_cqs(id);
	return ret;
}

void rdma_destroy_srq(struct rdma_cm_id *id)
{
	ibv_destroy_srq(id->srq);
	if (!id->qp)
		ucma_destroy_cqs(id);
	id->srq = NULL;
}

int rdma_create_qp(struct rdma_cm_id *id, struct ibv_pd *pd,
		   struct ibv_qp_init_attr *qp_init_attr)
{
	struct cma_id_private *id_priv;
	struct ibv_qp *qp;
	int ret;

	if (id->qp)
		return ERR(EINVAL);

	id_priv = container_of(id, struct cma_id_private, id);
	if (!pd)
		pd = id->pd;
	else if (id->verbs != pd->context)
		return ERR(EINVAL);

	ret = ucma_create_cqs(id, qp_init_attr->send_cq ? 0 : qp_init_attr->cap.max_send_wr,
			      qp_init_attr->recv_cq ? 0 : qp_init_attr->cap.max_recv_wr);
	if (ret)
		return ret;

	if (!qp_init_attr->send_cq)
		qp_init_attr->send_cq = id->send_cq;
	if (!qp_init_attr->recv_cq)
		qp_init_attr->recv_cq = id->recv_cq;
	qp = ibv_create_qp(pd, qp_init_attr);
	if (!qp) {
		ret = ERR(ENOMEM);
		goto err1;
	}

	if (ucma_is_ud_qp(id->qp_type))
		ret = ucma_init_ud_qp(id_priv, qp);
	else
		ret = ucma_init_conn_qp(id_priv, qp);
	if (ret)
		goto err2;

	id->pd = pd;
	id->qp = qp;

	//
	//	NonStop Add, save port number (fabric) in IB QP
	//	used for stats.
	//
	{
		struct ibv_qp_attr attr1 ;
		struct ibv_qp_init_attr init_attr ;
		struct verbs_qp  *v_qp = container_of(qp, struct verbs_qp, qp);
		struct process_qp_info *info = v_qp->ns_qp_info;	// get pointer to the qp stats

		if(info != NULL) {
			if (ibv_query_qp(qp,&attr1,IBV_QP_PORT,&init_attr) == 0) {
					info->port = attr1.port_num - 1 ;
			} else {
				info->port = 0 ;
			}
		}  // End if info
	}

	return 0;
err2:
	ibv_destroy_qp(qp);
err1:
	ucma_destroy_cqs(id);
	return ret;
}

void rdma_destroy_qp(struct rdma_cm_id *id)
{
	ibv_destroy_qp(id->qp);
	ucma_destroy_cqs(id);
	id->qp = NULL;
}

static int ucma_valid_param(struct cma_id_private *id_priv,
			    struct rdma_conn_param *param)
{
	if (id_priv->id.ps != RDMA_PS_TCP)
		return 0;

	if (!id_priv->id.qp && !param)
		goto err;

	if (!param)
		return 0;

	if ((param->responder_resources != RDMA_MAX_RESP_RES) &&
	    (param->responder_resources > id_priv->cma_dev->max_responder_resources))
		goto err;

	if ((param->initiator_depth != RDMA_MAX_INIT_DEPTH) &&
	    (param->initiator_depth > id_priv->cma_dev->max_initiator_depth))
		goto err;

	return 0;
err:
	return ERR(EINVAL);
}

static void ucma_copy_conn_param_to_kern(struct cma_id_private *id_priv,
					 struct ucma_abi_conn_param *dst,
					 struct rdma_conn_param *src,
					 uint32_t qp_num, uint8_t srq)
{
	dst->qp_num = qp_num;
	dst->srq = srq;
	dst->responder_resources = id_priv->responder_resources;
	dst->initiator_depth = id_priv->initiator_depth;
	dst->valid = 1;

	if (id_priv->connect_len) {
		memcpy(dst->private_data, id_priv->connect, id_priv->connect_len);
		dst->private_data_len = id_priv->connect_len;
	}

	if (src) {
		dst->flow_control = src->flow_control;
		dst->retry_count = src->retry_count;
		dst->rnr_retry_count = src->rnr_retry_count;

		if (src->private_data && src->private_data_len) {
			memcpy(dst->private_data + dst->private_data_len,
			       src->private_data, src->private_data_len);
			dst->private_data_len += src->private_data_len;
		}
	} else {
		dst->retry_count = 7;
		dst->rnr_retry_count = 7;
	}
}

int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
{
	struct ucma_abi_connect cmd;
	struct cma_id_private *id_priv;
	int ret;

	if(g_trace_level >= TRACE_LEVEL_FIVE) uib_trace_entry_point(RDMA_CONN);


	id_priv = container_of(id, struct cma_id_private, id);
	ret = ucma_valid_param(id_priv, conn_param);
	if (ret)
		return ret;

	if((!nsadi_enabled)&&
		(((id->route.addr.p2.dst_addr.sa_data[3] & 0x7f)!= 1)||
		 (id->route.addr.p2.dst_addr.sa_data[4]
		           != id->route.addr.p1.src_addr.sa_data[4]))){
		//fprintf(stderr, PFX "Error: NSADI License not enabled\n");
		uib_trace(RDMA_CONN, LICENSE_ERROR, TRACE_LEVEL_ONE, UIB_NO_QPN,
			  -1, ENOTSUP, TRACE_ERROR, UIB_NO_INSTANCE, UIB_NO_TRACE_DATA, NULL);
		return ERR(ENOTSUP);
	}

	if (conn_param && conn_param->initiator_depth != RDMA_MAX_INIT_DEPTH)
		id_priv->initiator_depth = conn_param->initiator_depth;
	else
		id_priv->initiator_depth = id_priv->cma_dev->max_initiator_depth;
	if (conn_param && conn_param->responder_resources != RDMA_MAX_RESP_RES)
		id_priv->responder_resources = conn_param->responder_resources;
	else
		id_priv->responder_resources = id_priv->cma_dev->max_responder_resources;

	CMA_INIT_CMD(&cmd, sizeof cmd, CONNECT);
	cmd.id = id_priv->handle;
	if (id->qp) {
		ucma_copy_conn_param_to_kern(id_priv, &cmd.conn_param,
					     conn_param, id->qp->qp_num,
					     (id->qp->srq != NULL));
	} else if (conn_param) {
		ucma_copy_conn_param_to_kern(id_priv, &cmd.conn_param,
					     conn_param, conn_param->qp_num,
					     conn_param->srq);
	} else {
		ucma_copy_conn_param_to_kern(id_priv, &cmd.conn_param,
					     conn_param, 0, 0);
	}

	ret = ns_write(id->channel->fd, &cmd, sizeof cmd);
	if (ret != sizeof cmd)
		return (ret >= 0) ? ERR(ENODATA) : ERR(-ret);

	if (id_priv->connect_len) {
		free(id_priv->connect);
		id_priv->connect_len = 0;
	}

	return ucma_complete(id);
}

int rdma_listen(struct rdma_cm_id *id, int backlog)
{
	struct ucma_abi_listen cmd;
	struct cma_id_private *id_priv;
	int ret;

	CMA_INIT_CMD(&cmd, sizeof cmd, LISTEN);
	id_priv = container_of(id, struct cma_id_private, id);
	cmd.id = id_priv->handle;
	cmd.backlog = backlog;

	ret = ns_write(id->channel->fd, &cmd, sizeof cmd);
	if (ret != sizeof cmd)
		return (ret >= 0) ? ERR(ENODATA) : ERR(-ret);

	if (af_ib_support)
		return ucma_query_addr(id);
	else
		return ucma_query_route(id);
}

int rdma_get_request(struct rdma_cm_id *listen, struct rdma_cm_id **id)
{
	struct cma_id_private *id_priv;
	struct rdma_cm_event *event;
	int ret;

	id_priv = container_of(listen, struct cma_id_private, id);
	if (!id_priv->sync)
		return ERR(EINVAL);

	if (listen->event) {
		rdma_ack_cm_event(listen->event);
		listen->event = NULL;
	}

	ret = rdma_get_cm_event(listen->channel, &event);
	if (ret) {
		return ret;
	}

	if (event->status) {
		ret = ERR(event->status);
		goto err;
	}

	if (event->event != RDMA_CM_EVENT_CONNECT_REQUEST) {
		ret = ERR(EINVAL);
		goto err;
	}

	if (id_priv->qp_init_attr) {
		struct ibv_qp_init_attr attr;

		attr = *id_priv->qp_init_attr;
		ret = rdma_create_qp(event->id, listen->pd, &attr);
		if (ret)
			goto err;
	}

	*id = event->id;
	(*id)->event = event;
	return 0;

err:
	listen->event = event;
	return ret;
}

int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
{
	struct ucma_abi_accept cmd;
	struct cma_id_private *id_priv;
	int ret;

	if(g_trace_level >= TRACE_LEVEL_FIVE) uib_trace_entry_point(RDMA_ACCEPT);

	id_priv = container_of(id, struct cma_id_private, id);
	ret = ucma_valid_param(id_priv, conn_param);
	if (ret)
		return ret;

	if((!nsadi_enabled)&&
		(((id->route.addr.p2.dst_addr.sa_data[3] & 0x7f)!= 1)||
		 (id->route.addr.p2.dst_addr.sa_data[4]
		           != id->route.addr.p1.src_addr.sa_data[4]))){
//		fprintf(stderr, PFX "Error: NSADI License not enabled\n");
		uib_trace(RDMA_ACCEPT, LICENSE_ERROR, TRACE_LEVEL_ONE, UIB_NO_QPN,
			  -1, ENOTSUP, TRACE_ERROR, UIB_NO_INSTANCE, UIB_NO_TRACE_DATA, NULL);
		return ERR(ENOTSUP);
	}

	if (!conn_param || conn_param->initiator_depth == RDMA_MAX_INIT_DEPTH) {
		id_priv->initiator_depth = min(id_priv->initiator_depth,
					       id_priv->cma_dev->max_initiator_depth);
	} else {
		id_priv->initiator_depth = conn_param->initiator_depth;
	}
	if (!conn_param || conn_param->responder_resources == RDMA_MAX_RESP_RES) {
		id_priv->responder_resources = min(id_priv->responder_resources,
						   id_priv->cma_dev->max_responder_resources);
	} else {
		id_priv->responder_resources = conn_param->responder_resources;
	}

	if (!ucma_is_ud_qp(id->qp_type)) {
		ret = ucma_modify_qp_rtr(id, id_priv->responder_resources);
		if (ret)
			return ret;

		ret = ucma_modify_qp_rts(id, id_priv->initiator_depth);
		if (ret)
			return ret;
	}

	CMA_INIT_CMD(&cmd, sizeof cmd, ACCEPT);
	cmd.id = id_priv->handle;
	cmd.uid = (uintptr_t) id_priv;
	if (id->qp)
		ucma_copy_conn_param_to_kern(id_priv, &cmd.conn_param,
					     conn_param, id->qp->qp_num,
					     (id->qp->srq != NULL));
	else
		ucma_copy_conn_param_to_kern(id_priv, &cmd.conn_param,
					     conn_param, conn_param->qp_num,
					     conn_param->srq);

	ret = ns_write(id->channel->fd, &cmd, sizeof cmd);
	if (ret != sizeof cmd) {
		ucma_modify_qp_err(id);
		return (ret >= 0) ? ERR(ENODATA) : ERR(-ret);
	}

	if (ucma_is_ud_qp(id->qp_type))
		return 0;

	return ucma_complete(id);
}

int rdma_reject(struct rdma_cm_id *id, const void *private_data,
		uint8_t private_data_len)
{
	struct ucma_abi_reject cmd;
	struct cma_id_private *id_priv;
	int ret;

	CMA_INIT_CMD(&cmd, sizeof cmd, REJECT);

	id_priv = container_of(id, struct cma_id_private, id);
	cmd.id = id_priv->handle;
	if (private_data && private_data_len) {
		memcpy(cmd.private_data, private_data, private_data_len);
		cmd.private_data_len = private_data_len;
	}

	ret = ns_write(id->channel->fd, &cmd, sizeof cmd);
	if (ret != sizeof cmd)
		return (ret >= 0) ? ERR(ENODATA) : ERR(-ret);

	return 0;
}

int rdma_notify(struct rdma_cm_id *id, enum ibv_event_type event)
{
	struct ucma_abi_notify cmd;
	struct cma_id_private *id_priv;
	int ret;

	CMA_INIT_CMD(&cmd, sizeof cmd, NOTIFY);

	id_priv = container_of(id, struct cma_id_private, id);
	cmd.id = id_priv->handle;
	cmd.event = event;
	ret = ns_write(id->channel->fd, &cmd, sizeof cmd);
	if (ret != sizeof cmd)
		return (ret >= 0) ? ERR(ENODATA) : ERR(-ret);

	return 0;
}

int rdma_disconnect(struct rdma_cm_id *id)
{
	struct ucma_abi_disconnect cmd;
	struct cma_id_private *id_priv;
	int ret;

	switch (id->verbs->device->transport_type) {
	case IBV_TRANSPORT_IB:
		ret = ucma_modify_qp_err(id);
		break;
	case IBV_TRANSPORT_IWARP:
		ret = ucma_modify_qp_sqd(id);
		break;
	default:
		ret = ERR(EINVAL);
	}
	if (ret)
		return ret;

	CMA_INIT_CMD(&cmd, sizeof cmd, DISCONNECT);
	id_priv = container_of(id, struct cma_id_private, id);
	cmd.id = id_priv->handle;

	ret = ns_write(id->channel->fd, &cmd, sizeof cmd);
	if (ret != sizeof cmd)
		return (ret >= 0) ? ERR(ENODATA) : ERR(-ret);

	return ucma_complete(id);
}

#ifdef dddd
static int rdma_join_multicast2(struct rdma_cm_id *id, struct sockaddr *addr,
				socklen_t addrlen, void *context)
{
	struct ucma_abi_create_id_resp resp;
	struct cma_id_private *id_priv;
	struct cma_multicast *mc, **pos;
	int ret;

	id_priv = container_of(id, struct cma_id_private, id);
	mc = calloc(1, sizeof *mc);
	if (!mc)
		return ERR(ENOMEM);

	mc->context = context;
	mc->id_priv = id_priv;
	memcpy(&mc->addr, addr, addrlen);
	if (pthread_cond_init(&mc->cond, NULL)) {
		ret = -1;
		goto err1;
	}

	pthread_mutex_lock(&id_priv->mut);
	mc->next = id_priv->mc_list;
	id_priv->mc_list = mc;
	pthread_mutex_unlock(&id_priv->mut);

	if (af_ib_support) {
		struct ucma_abi_join_mcast cmd;

		CMA_INIT_CMD_RESP(&cmd, sizeof cmd, JOIN_MCAST, &resp, sizeof resp);
		cmd.id = id_priv->handle;
		memcpy(&cmd.addr, addr, addrlen);
		cmd.addr_size = addrlen;
		cmd.uid = (uintptr_t) mc;
		cmd.reserved = 0;

		ret = ns_write(id->channel->fd, &cmd, sizeof cmd);
		if (ret != sizeof cmd) {
			ret = (ret >= 0) ? ERR(ENODATA) : ERR(-ret);
			goto err2;
		}
	} else {
		struct ucma_abi_join_ip_mcast cmd;

		CMA_INIT_CMD_RESP(&cmd, sizeof cmd, JOIN_IP_MCAST, &resp, sizeof resp);
		cmd.id = id_priv->handle;
		memcpy(&cmd.addr, addr, addrlen);
		cmd.uid = (uintptr_t) mc;

		ret = ns_write(id->channel->fd, &cmd, sizeof cmd);
		if (ret != sizeof cmd) {
			ret = (ret >= 0) ? ERR(ENODATA) : ERR(-ret);
			goto err2;
		}
	}

	VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);

	mc->handle = resp.id;
	return ucma_complete(id);

err2:
	pthread_mutex_lock(&id_priv->mut);
	for (pos = &id_priv->mc_list; *pos != mc; pos = &(*pos)->next)
		;
	*pos = mc->next;
	pthread_mutex_unlock(&id_priv->mut);
err1:
	free(mc);
	return ret;
}
#endif /* dddd */

int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
			void *context)
{
#ifdef dddd
	int addrlen;

	addrlen = ucma_addrlen(addr);
	if (!addrlen)
		return ERR(EINVAL);

	return rdma_join_multicast2(id, addr, addrlen, context);
#endif /* dddd */
	return ERR(ENOSYS);
}

int rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
{
#ifdef dddd
	struct ucma_abi_destroy_id cmd;
	struct ucma_abi_destroy_id_resp resp;
	struct cma_id_private *id_priv;
	struct cma_multicast *mc, **pos;
	int ret, addrlen;

	addrlen = ucma_addrlen(addr);
	if (!addrlen)
		return ERR(EINVAL);

	id_priv = container_of(id, struct cma_id_private, id);
	pthread_mutex_lock(&id_priv->mut);
	for (pos = &id_priv->mc_list; *pos; pos = &(*pos)->next)
		if (!memcmp(&(*pos)->addr, addr, addrlen))
			break;

	mc = *pos;
	if (*pos)
		*pos = mc->next;
	pthread_mutex_unlock(&id_priv->mut);
	if (!mc)
		return ERR(EADDRNOTAVAIL);

	if (id->qp)
		ibv_detach_mcast(id->qp, &mc->mgid, mc->mlid);

	CMA_INIT_CMD_RESP(&cmd, sizeof cmd, LEAVE_MCAST, &resp, sizeof resp);
	cmd.id = mc->handle;

	ret = ns_write(id->channel->fd, &cmd, sizeof cmd);
	if (ret != sizeof cmd) {
		ret = (ret >= 0) ? ERR(ENODATA) : ERR(-ret);
		goto free;
	}

	VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);

	pthread_mutex_lock(&id_priv->mut);
	while (mc->events_completed < resp.events_reported)
		pthread_cond_wait(&mc->cond, &id_priv->mut);
	pthread_mutex_unlock(&id_priv->mut);

	ret = 0;
free:
	free(mc);
	return ret;
#endif /* dddd */
	return ERR(ENOSYS);
}

static void ucma_complete_event(struct cma_id_private *id_priv)
{
	pthread_mutex_lock(&id_priv->mut);
	id_priv->events_completed++;
	pthread_cond_signal(&id_priv->cond);
	pthread_mutex_unlock(&id_priv->mut);
}

static void ucma_complete_mc_event(struct cma_multicast *mc)
{
	pthread_mutex_lock(&mc->id_priv->mut);
	mc->events_completed++;
	pthread_cond_signal(&mc->cond);
	mc->id_priv->events_completed++;
	pthread_cond_signal(&mc->id_priv->cond);
	pthread_mutex_unlock(&mc->id_priv->mut);
}

int rdma_ack_cm_event(struct rdma_cm_event *event)
{
	struct cma_event *evt;

	if (!event)
		return ERR(EINVAL);

	evt = container_of(event, struct cma_event, event);

	if (evt->mc)
		ucma_complete_mc_event(evt->mc);
	else
		ucma_complete_event(evt->id_priv);
	free(evt);
	return 0;
}

static void ucma_process_addr_resolved(struct cma_event *evt)
{
	if (af_ib_support) {
		evt->event.status = ucma_query_addr(&evt->id_priv->id);
		if (!evt->event.status &&
		    evt->id_priv->id.verbs->device->transport_type == IBV_TRANSPORT_IB)
			evt->event.status = ucma_query_gid(&evt->id_priv->id);
	} else {
		evt->event.status = ucma_query_route(&evt->id_priv->id);
	}

	if (evt->event.status)
		evt->event.event = RDMA_CM_EVENT_ADDR_ERROR;
}

static void ucma_process_route_resolved(struct cma_event *evt)
{
	if (evt->id_priv->id.verbs->device->transport_type != IBV_TRANSPORT_IB)
		return;

	if (af_ib_support)
		evt->event.status = ucma_query_path(&evt->id_priv->id);
	else
		evt->event.status = ucma_query_route(&evt->id_priv->id);

	if (evt->event.status)
		evt->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
}

static int ucma_query_req_info(struct rdma_cm_id *id)
{
	int ret;

	if (!af_ib_support)
		return ucma_query_route(id);

	ret = ucma_query_addr(id);
	if (ret)
		return ret;

	ret = ucma_query_gid(id);
	if (ret)
		return ret;

	ret = ucma_query_path(id);
	if (ret)
		return ret;

	return 0;
}

int ns_license_data_get(char *src_str, int str_len, const char *search_str)
{
	int license_data = -1;
	int l_data = -1;
	char scan_str[256] = "";
	char *p_scan_str = scan_str;

	if(g_trace_level >= TRACE_LEVEL_FIVE) uib_trace_entry_point(LIC_DATA_GET);

	/* Scan the cached global feature string for the search string */
	if ((p_scan_str = strstr(src_str, search_str)) != NULL) {
		/* Advance to the next colon char */
		if ((p_scan_str = strstr(p_scan_str,":")) != NULL) {
			/* Advance to the char after the colon */
			++p_scan_str;
			/* Convert the feature data string to an integer */
			license_data = -2;
			errno = 0;
			l_data = strtol(p_scan_str,NULL,10);
			if (errno == 0) {
				/* Success! Return the integer data */
				license_data = l_data;
				//fprintf(stderr, "ns_license_data_get: Data found %s:%d\n", search_str, license_data);
			} else {
				//fprintf(stderr, "ns_license_data_get: Data conversion err=%d str=%s,data=%d\n", errno, scan_str, l_data);
					uib_trace(LIC_DATA_GET, DATA_CONVERSION, TRACE_LEVEL_ONE, UIB_NO_QPN,
						  license_data, errno, TRACE_ERROR, UIB_NO_INSTANCE, 1, (int64_t *)&license_data);
			}
		} else {
			//fprintf(stderr, "ns_license_data_get: No colon after %s in %s\n", search_str, src_str);
			uib_trace(LIC_DATA_GET, NO_COLON, TRACE_LEVEL_ONE, UIB_NO_QPN,
				  license_data, UIB_NO_ERRNO, TRACE_ERROR, UIB_NO_INSTANCE, UIB_NO_TRACE_DATA, NULL);
		}
	} else {
		//fprintf(stderr, "ns_license_data_get: No %s data in %s\n", search_str, src_str);
		uib_trace(LIC_DATA_GET, MISSING_DATA, TRACE_LEVEL_ONE, UIB_NO_QPN,
			  license_data, UIB_NO_ERRNO, TRACE_ERROR, UIB_NO_INSTANCE, UIB_NO_TRACE_DATA, NULL);
	}

	/*
	 * Return one of the following:
	 *      Requested license data,
	 *      -1 if string scan failed, or
	 *      -2 if data conversion failed
	 */
	return (license_data);
}



static int ns_ucma_validate_lic (struct cma_id_private *id_priv, int dir)
{
	int ret = ENOTSUP ;
	int max = 2 ;
	char li[NS_LICENSE_USER_MAX_STR_LEN] = "";
	int li_len = sizeof(li);
	int pos = -1;
	int num_local_servers = -1;

	if(g_trace_level >= TRACE_LEVEL_FIVE) uib_trace_entry_point(VALIDATE_LIC);


	if ((NSKVCore_modelToSystemClass_(NSKVCore_getModel())) == NS_LICENSE_SYS_CLASS_HIGH_END) {
		max = 8 ;
	}

	ret = ns_utguid_search (id_priv->id.route.path_rec->dgid.global.interface_id, li, &li_len);
	if (ret == 0) {
		pos = ns_license_data_get(li, li_len, NS_LICENSE_USER_POSITION);
		switch (pos) {
		case NS_LICENSE_USER_POS_RESOURCE:
			break;
		case NS_LICENSE_USER_POS_LOCAL_EXT_SRVR:
			num_local_servers = ns_license_data_get(li, li_len, NS_LICENSE_USER_NUM_LOCAL_SRV);
			if (num_local_servers > max) {
				//fprintf(stderr, "ns_ucma_validate_lic: NSADI license does not support more than %d servers \n",max);
				ret = ERANGE;
				uib_trace(VALIDATE_LIC, LICENSE_ERROR, TRACE_LEVEL_ONE, UIB_NO_QPN,
					  ret, UIB_NO_ERRNO, TRACE_ERROR, 1, UIB_NO_TRACE_DATA, NULL);
			} else {
				ret = 0 ;
			}
			break;
		case NS_LICENSE_USER_POS_REMOTE_EXT_SRVR:
			//fprintf(stderr, "ns_ucma_validate_lic: NSADI cannot be targeted %s remote external servers\n", dir?"from":"by");
			ret = ENOTSUP;
			uib_trace(VALIDATE_LIC, LICENSE_ERROR, TRACE_LEVEL_ONE, UIB_NO_QPN,
				  ret, UIB_NO_ERRNO, TRACE_ERROR, 2, UIB_NO_TRACE_DATA, NULL);
			break;
		default:
			//fprintf(stderr, "ns_ucma_validate_lic: Invalid license data, pos=%d num=%d\n",pos,num_local_servers);
			ret = EBADDATA;
			uib_trace(VALIDATE_LIC, LICENSE_ERROR, TRACE_LEVEL_ONE, UIB_NO_QPN,
				  ret, UIB_NO_ERRNO, TRACE_ERROR, 3, UIB_NO_TRACE_DATA, NULL);
			break;
		}
	} else {
		//fprintf(stderr, "ns_ucma_validate_lic: ns_utguid_search() err=%d\n",ret);
		uib_trace(VALIDATE_LIC, LICENSE_ERROR, TRACE_LEVEL_ONE, UIB_NO_QPN,
			  ret, UIB_NO_ERRNO, TRACE_ERROR, 4, UIB_NO_TRACE_DATA, NULL);
	}

	errno = ret ;
	return (ret) ;
}



static int ucma_process_conn_req(struct cma_event *evt,
				 uint32_t handle)
{
	struct cma_id_private *id_priv;
	int ret;

	if(g_trace_level >= TRACE_LEVEL_FIVE) uib_trace_entry_point(PROC_CON_REQ);

	id_priv = ucma_alloc_id(evt->id_priv->id.channel,
				evt->id_priv->id.context, evt->id_priv->id.ps,
				evt->id_priv->id.qp_type);
	if (!id_priv) {
		ucma_destroy_kern_id(evt->id_priv->id.channel->fd, handle);
		ret = ERR(ENOMEM);
		goto err1;
	}

	evt->event.listen_id = &evt->id_priv->id;
	evt->event.id = &id_priv->id;
	id_priv->handle = handle;
	id_priv->initiator_depth = evt->event.param.conn.initiator_depth;
	id_priv->responder_resources = evt->event.param.conn.responder_resources;

	if (evt->id_priv->sync) {
		ret = rdma_migrate_id(&id_priv->id, NULL);
		if (ret)
			goto err2;
	}

	ret = ucma_query_req_info(&id_priv->id);
	if (ret)
		goto err2;

	if ((ret = ns_ucma_validate_lic(id_priv,0))) {
		//fprintf(stderr, "ucma_process_conn_req: ns_ucma_validate_lic() err=%d\n",ret);
		ERR(ret);    /* TODO: (djc) this okay? */
		uib_trace(PROC_CON_REQ, LICENSE_ERROR, TRACE_LEVEL_ONE, UIB_NO_QPN,
			  ret, UIB_NO_ERRNO, TRACE_ERROR, UIB_NO_INSTANCE, UIB_NO_TRACE_DATA, NULL);
		goto err2 ;
	}

	return 0;

err2:
	rdma_destroy_id(&id_priv->id);
err1:
	ucma_complete_event(evt->id_priv);
	return ret;
}

static int ucma_process_conn_resp(struct cma_id_private *id_priv)
{
	struct ucma_abi_accept cmd;
	int ret;

	if(g_trace_level >= TRACE_LEVEL_FIVE) uib_trace_entry_point(PROC_CON_RESP);

	if ((ret = ns_ucma_validate_lic(id_priv,1))) {
		//fprintf(stderr, "ucma_process_conn_resp: ns_ucma_validate_lic() err=%d\n",ret);
		ERR(ret);    /* TODO: (djc) this okay? */
		uib_trace(PROC_CON_RESP, LICENSE_ERROR, TRACE_LEVEL_ONE, UIB_NO_QPN,
			  ret, UIB_NO_ERRNO, TRACE_ERROR, UIB_NO_INSTANCE, UIB_NO_TRACE_DATA, NULL);
		goto err ;
	}

	ret = ucma_modify_qp_rtr(&id_priv->id, RDMA_MAX_RESP_RES);
	if (ret)
		goto err;

	ret = ucma_modify_qp_rts(&id_priv->id, RDMA_MAX_INIT_DEPTH);
	if (ret)
		goto err;

	CMA_INIT_CMD(&cmd, sizeof cmd, ACCEPT);
	cmd.id = id_priv->handle;

	ret = ns_write(id_priv->id.channel->fd, &cmd, sizeof cmd);
	if (ret != sizeof cmd) {
		ret = (ret >= 0) ? ERR(ENODATA) : ERR(-ret);
		goto err;
	}

	return 0;
err:
	ucma_modify_qp_err(&id_priv->id);
	return ret;
}

static int ucma_process_join(struct cma_event *evt)
{
	evt->mc->mgid = evt->event.param.ud.ah_attr.grh.dgid;
	evt->mc->mlid = evt->event.param.ud.ah_attr.dlid;

	if (!evt->id_priv->id.qp)
		return 0;

	return rdma_seterrno(ibv_attach_mcast(evt->id_priv->id.qp,
					      &evt->mc->mgid, evt->mc->mlid));
}

static void ucma_copy_conn_event(struct cma_event *event,
				 struct ucma_abi_conn_param *src)
{
	struct rdma_conn_param *dst = &event->event.param.conn;

	dst->private_data_len = src->private_data_len;
	if (src->private_data_len) {
		dst->private_data = &event->private_data;
		memcpy(&event->private_data, src->private_data,
		       src->private_data_len);
	}

	dst->responder_resources = src->responder_resources;
	dst->initiator_depth = src->initiator_depth;
	dst->flow_control = src->flow_control;
	dst->retry_count = src->retry_count;
	dst->rnr_retry_count = src->rnr_retry_count;
	dst->srq = src->srq;
	dst->qp_num = src->qp_num;
}

static void ucma_copy_ud_event(struct cma_event *event,
			       struct ucma_abi_ud_param *src)
{
	struct rdma_ud_param *dst = &event->event.param.ud;

	dst->private_data_len = src->private_data_len;
	if (src->private_data_len) {
		dst->private_data = &event->private_data;
		memcpy(&event->private_data, src->private_data,
		       src->private_data_len);
	}

	ibv_copy_ah_attr_from_kern(&dst->ah_attr, &src->ah_attr);
	dst->qp_num = src->qp_num;
	dst->qkey = src->qkey;
}

int rdma_get_cm_event(struct rdma_event_channel *channel,
		      struct rdma_cm_event **event)
{
	struct ucma_abi_event_resp resp;
	struct ucma_abi_get_event cmd;
	struct cma_event *evt;
	int ret;
	int count_xfer ;
	int err ;
	int data[10] ;

	if(g_trace_level >= TRACE_LEVEL_FIVE) uib_trace_entry_point(GET_CM_EVENT);

	ret = ucma_init();
	if (ret)
		return ret;

	if (!event)
		return ERR(EINVAL);

	evt = malloc(sizeof *evt);
	if (!evt)
		return ERR(ENOMEM);

retry:
	memset(evt, 0, sizeof *evt);
	CMA_INIT_CMD_RESP(&cmd, sizeof cmd, GET_EVENT, &resp, sizeof resp);
	//
	//	Issue the write to get any pending events. The NSK kernel only implements
	//	a non-blocking interface, so will get an immediate return with either return of
	//	the size of the message (success, event pending condition), or ENODATA (no
	//	events at this time condition).
	//
	ret = ns_write(channel->fd, &cmd, sizeof cmd);

	if (ret != sizeof cmd) {
		if (ret == -EAGAIN) {
			// TODO: NIBVL - Need use LDONE for tag instead of -1?
			put_regFile(channel->fd) ;
			if ((ret = put_awaitio(channel->fd,-1,-1,&count_xfer,&err,&data))) {
				return (-1) ;
			}
			goto retry ;
		} else {
			free(evt);
			return (ret >= 0) ? ERR(ENODATA) : ERR(-ret);
		}
	}

	put_unregFile(channel->fd) ;
	VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);

	evt->event.event = (enum rdma_cm_event_type)resp.event;
	evt->id_priv = (void *) (uintptr_t) resp.uid;
	evt->event.id = &evt->id_priv->id;

	//
	//	NonStop. Errors (-errnos) coming from the kernel have a different
	//	value (linux) from NonStop error codes. So have to map into NonStop
	//	space.
	//
	resp.status = ns_map_errno_(resp.status) ;
	evt->event.status = resp.status;

	switch (resp.event) {
	case RDMA_CM_EVENT_ADDR_RESOLVED:
		ucma_process_addr_resolved(evt);
		break;
	case RDMA_CM_EVENT_ROUTE_RESOLVED:
		ucma_process_route_resolved(evt);
		break;
	case RDMA_CM_EVENT_CONNECT_REQUEST:
		evt->id_priv = (void *) (uintptr_t) resp.uid;
		if (ucma_is_ud_qp(evt->id_priv->id.qp_type))
			ucma_copy_ud_event(evt, &resp.param.ud);
		else
			ucma_copy_conn_event(evt, &resp.param.conn);

		ret = ucma_process_conn_req(evt, resp.id);
		if (ret) {
			//fprintf(stderr, "rdma_get_cm_event: ucma_process_conn_req() err=%d\n",ret);
			uib_trace(GET_CM_EVENT, CON_REQ_ERR, TRACE_LEVEL_ONE, UIB_NO_QPN,
				  ret, UIB_NO_ERRNO, TRACE_ERROR, UIB_NO_INSTANCE, UIB_NO_TRACE_DATA, NULL);
			goto retry;
		}
		break;
	case RDMA_CM_EVENT_CONNECT_RESPONSE:
		ucma_copy_conn_event(evt, &resp.param.conn);
		evt->event.status = ucma_process_conn_resp(evt->id_priv);
		if (!evt->event.status)
			evt->event.event = RDMA_CM_EVENT_ESTABLISHED;
		else {
			evt->event.event = RDMA_CM_EVENT_CONNECT_ERROR;
			evt->id_priv->connect_error = 1;
		}
		break;
	case RDMA_CM_EVENT_ESTABLISHED:
		if (ucma_is_ud_qp(evt->id_priv->id.qp_type)) {
			ucma_copy_ud_event(evt, &resp.param.ud);
			break;
		}

		ucma_copy_conn_event(evt, &resp.param.conn);
		break;
	case RDMA_CM_EVENT_REJECTED:
		if (evt->id_priv->connect_error) {
			ucma_complete_event(evt->id_priv);
			goto retry;
		}
		ucma_copy_conn_event(evt, &resp.param.conn);
		ucma_modify_qp_err(evt->event.id);
		break;
	case RDMA_CM_EVENT_DISCONNECTED:
		if (evt->id_priv->connect_error) {
			ucma_complete_event(evt->id_priv);
			goto retry;
		}
		ucma_copy_conn_event(evt, &resp.param.conn);
		break;
	case RDMA_CM_EVENT_MULTICAST_JOIN:
		evt->mc = (void *) (uintptr_t) resp.uid;
		evt->id_priv = evt->mc->id_priv;
		evt->event.id = &evt->id_priv->id;
		ucma_copy_ud_event(evt, &resp.param.ud);
		evt->event.param.ud.private_data = evt->mc->context;
		evt->event.status = ucma_process_join(evt);
		if (evt->event.status)
			evt->event.event = RDMA_CM_EVENT_MULTICAST_ERROR;
		break;
	case RDMA_CM_EVENT_MULTICAST_ERROR:
		evt->mc = (void *) (uintptr_t) resp.uid;
		evt->id_priv = evt->mc->id_priv;
		evt->event.id = &evt->id_priv->id;
		evt->event.param.ud.private_data = evt->mc->context;
		break;
	default:
		evt->id_priv = (void *) (uintptr_t) resp.uid;
		evt->event.id = &evt->id_priv->id;
		evt->event.status = resp.status;
		if (ucma_is_ud_qp(evt->id_priv->id.qp_type))
			ucma_copy_ud_event(evt, &resp.param.ud);
		else
			ucma_copy_conn_event(evt, &resp.param.conn);
		break;
	}

	*event = &evt->event;
	return 0;
}

const char *rdma_event_str(enum rdma_cm_event_type event)
{
	switch (event) {
	case RDMA_CM_EVENT_ADDR_RESOLVED:
		return "RDMA_CM_EVENT_ADDR_RESOLVED";
	case RDMA_CM_EVENT_ADDR_ERROR:
		return "RDMA_CM_EVENT_ADDR_ERROR";
	case RDMA_CM_EVENT_ROUTE_RESOLVED:
		return "RDMA_CM_EVENT_ROUTE_RESOLVED";
	case RDMA_CM_EVENT_ROUTE_ERROR:
		return "RDMA_CM_EVENT_ROUTE_ERROR";
	case RDMA_CM_EVENT_CONNECT_REQUEST:
		return "RDMA_CM_EVENT_CONNECT_REQUEST";
	case RDMA_CM_EVENT_CONNECT_RESPONSE:
		return "RDMA_CM_EVENT_CONNECT_RESPONSE";
	case RDMA_CM_EVENT_CONNECT_ERROR:
		return "RDMA_CM_EVENT_CONNECT_ERROR";
	case RDMA_CM_EVENT_UNREACHABLE:
		return "RDMA_CM_EVENT_UNREACHABLE";
	case RDMA_CM_EVENT_REJECTED:
		return "RDMA_CM_EVENT_REJECTED";
	case RDMA_CM_EVENT_ESTABLISHED:
		return "RDMA_CM_EVENT_ESTABLISHED";
	case RDMA_CM_EVENT_DISCONNECTED:
		return "RDMA_CM_EVENT_DISCONNECTED";
	case RDMA_CM_EVENT_DEVICE_REMOVAL:
		return "RDMA_CM_EVENT_DEVICE_REMOVAL";
	case RDMA_CM_EVENT_MULTICAST_JOIN:
		return "RDMA_CM_EVENT_MULTICAST_JOIN";
	case RDMA_CM_EVENT_MULTICAST_ERROR:
		return "RDMA_CM_EVENT_MULTICAST_ERROR";
	case RDMA_CM_EVENT_ADDR_CHANGE:
		return "RDMA_CM_EVENT_ADDR_CHANGE";
	case RDMA_CM_EVENT_TIMEWAIT_EXIT:
		return "RDMA_CM_EVENT_TIMEWAIT_EXIT";
	default:
		return "UNKNOWN EVENT";
	}
}

int rdma_set_option(struct rdma_cm_id *id, int level, int optname,
		    void *optval, size_t optlen)
{
	struct ucma_abi_set_option cmd;
	struct cma_id_private *id_priv;
	int ret;

	CMA_INIT_CMD(&cmd, sizeof cmd, SET_OPTION);
	id_priv = container_of(id, struct cma_id_private, id);
	cmd.id = id_priv->handle;
	cmd.optval = (uintptr_t) optval;
	cmd.level = level;
	cmd.optname = optname;
	cmd.optlen = optlen;

	ret = ns_write(id->channel->fd, &cmd, sizeof cmd);
	if (ret != sizeof cmd)
		return (ret >= 0) ? ERR(ENODATA) : ERR(-ret);

	return 0;
}

int rdma_migrate_id(struct rdma_cm_id *id, struct rdma_event_channel *channel)
{
	struct ucma_abi_migrate_resp resp;
	struct ucma_abi_migrate_id cmd;
	struct cma_id_private *id_priv;
	int ret, sync;

	id_priv = container_of(id, struct cma_id_private, id);
	if (id_priv->sync && !channel)
		return ERR(EINVAL);

	if ((sync = (channel == NULL))) {
		channel = rdma_create_event_channel();
		if (!channel)
			return -1;
	}

	CMA_INIT_CMD_RESP(&cmd, sizeof cmd, MIGRATE_ID, &resp, sizeof resp);
	cmd.id = id_priv->handle;
	cmd.fd = id->channel->fd;

	ret = ns_write(channel->fd, &cmd, sizeof cmd);
	if (ret != sizeof cmd) {
		if (sync)
			rdma_destroy_event_channel(channel);
		return (ret >= 0) ? ERR(ENODATA) : ERR(-ret);
	}

	VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);

	if (id_priv->sync) {
		if (id->event) {
			rdma_ack_cm_event(id->event);
			id->event = NULL;
		}
		rdma_destroy_event_channel(id->channel);
	}

	/*
	 * Eventually if we want to support migrating channels while events are
	 * being processed on the current channel, we need to block here while
	 * there are any outstanding events on the current channel for this id
	 * to prevent the user from processing events for this id on the old
	 * channel after this call returns.
	 */
	pthread_mutex_lock(&id_priv->mut);
	id_priv->sync = sync;
	id->channel = channel;
	while (id_priv->events_completed < resp.events_reported)
		pthread_cond_wait(&id_priv->cond, &id_priv->mut);
	pthread_mutex_unlock(&id_priv->mut);

	return 0;
}

static int ucma_passive_ep(struct rdma_cm_id *id, struct rdma_addrinfo *res,
			   struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr)
{
	struct cma_id_private *id_priv;
	int ret;

	if (af_ib_support)
		ret = rdma_bind_addr2(id, res->ai_src_addr, res->ai_src_len);
	else
		ret = rdma_bind_addr(id, res->ai_src_addr);
	if (ret)
		return ret;

	id_priv = container_of(id, struct cma_id_private, id);
	if (pd)
		id->pd = pd;

	if (qp_init_attr) {
		id_priv->qp_init_attr = malloc(sizeof *qp_init_attr);
		if (!id_priv->qp_init_attr)
			return ERR(ENOMEM);

		*id_priv->qp_init_attr = *qp_init_attr;
		id_priv->qp_init_attr->qp_type = (enum ibv_qp_type)res->ai_qp_type;
	}

	return 0;
}

int rdma_create_ep(struct rdma_cm_id **id, struct rdma_addrinfo *res,
		   struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr)
{
	struct rdma_cm_id *cm_id;
	struct cma_id_private *id_priv;
	int ret;

	ret = rdma_create_id2(NULL, &cm_id, NULL, (enum rdma_port_space)res->ai_port_space,
	                     (enum ibv_qp_type)res->ai_qp_type);
	if (ret)
		return ret;

	if (res->ai_flags & RAI_PASSIVE) {
		ret = ucma_passive_ep(cm_id, res, pd, qp_init_attr);
		if (ret)
			goto err;
		goto out;
	}

	if (af_ib_support)
		ret = rdma_resolve_addr2(cm_id, res->ai_src_addr, res->ai_src_len,
					 res->ai_dst_addr, res->ai_dst_len, 2000);
	else
		ret = rdma_resolve_addr(cm_id, res->ai_src_addr, res->ai_dst_addr, 2000);
	if (ret)
		goto err;

	if (res->ai_route_len) {
		ret = rdma_set_option(cm_id, RDMA_OPTION_IB, RDMA_OPTION_IB_PATH,
				      res->ai_route, res->ai_route_len);
		if (!ret)
			ret = ucma_complete(cm_id);
	} else {
		ret = rdma_resolve_route(cm_id, 2000);
	}
	if (ret)
		goto err;

	if (qp_init_attr) {
		qp_init_attr->qp_type = (enum ibv_qp_type)res->ai_qp_type;
		ret = rdma_create_qp(cm_id, pd, qp_init_attr);
		if (ret)
			goto err;
	}

	if (res->ai_connect_len) {
		id_priv = container_of(cm_id, struct cma_id_private, id);
		id_priv->connect = malloc(res->ai_connect_len);
		if (!id_priv->connect) {
			ret = ERR(ENOMEM);
			goto err;
		}
		memcpy(id_priv->connect, res->ai_connect, res->ai_connect_len);
		id_priv->connect_len = res->ai_connect_len;
	}

out:
	*id = cm_id;
	return 0;

err:
	rdma_destroy_ep(cm_id);
	return ret;
}

void rdma_destroy_ep(struct rdma_cm_id *id)
{
	struct cma_id_private *id_priv;

	if (id->qp)
		rdma_destroy_qp(id);

	if (id->srq)
		rdma_destroy_srq(id);

	id_priv = container_of(id, struct cma_id_private, id);
	if (id_priv->qp_init_attr)
		free(id_priv->qp_init_attr);

	rdma_destroy_id(id);
}

int ucma_max_qpsize(struct rdma_cm_id *id)
{
	struct cma_id_private *id_priv;
	int i, max_size = 0;

	id_priv = container_of(id, struct cma_id_private, id);
	if (id && id_priv->cma_dev) {
		max_size = id_priv->cma_dev->max_qpsize;
	} else {
		ucma_init();
		for (i = 0; i < cma_dev_cnt; i++) {
			if (!max_size || max_size > cma_dev_array[i].max_qpsize)
				max_size = cma_dev_array[i].max_qpsize;
		}
	}
	return max_size;
}

uint16_t ucma_get_port(struct sockaddr *addr)
{
	switch (addr->sa_family) {
	case AF_INET:
		return ((struct sockaddr_in *) addr)->sin_port;
	case AF_INET6:
		return ((struct sockaddr_in6 *) addr)->sin6_port;
	case AF_IB:
		return htons((uint16_t) ntohll(((struct sockaddr_ib *) addr)->sib_sid));
	default:
		return 0;
	}
}

uint16_t rdma_get_src_port(struct rdma_cm_id *id)
{
	return ucma_get_port(&id->route.addr.p1.src_addr);
}

uint16_t rdma_get_dst_port(struct rdma_cm_id *id)
{
	return ucma_get_port(&id->route.addr.p2.dst_addr);
}