[6/10] DAPL v2.0: scm: add support for IB collective drivers in socket cm provider

Commit Message

Arlin Davis Aug. 11, 2011, 12:42 a.m. UTC

Add collective member address and threading information
on a per transport basis. Call create/free service with
HCA open/close.

Signed-off-by: Arlin Davis <[e-mail]>
---
 dapl/openib_scm/dapl_ib_util.h |   18 +++++++++++++++++-
 dapl/openib_scm/device.c       |   13 +++++++++++++
 2 files changed, 30 insertions(+), 1 deletions(-)

Patch

diff --git a/dapl/openib_scm/dapl_ib_util.h b/dapl/openib_scm/dapl_ib_util.h
index 5f9fb43..0d7f9f3 100644
--- a/dapl/openib_scm/dapl_ib_util.h
+++ b/dapl/openib_scm/dapl_ib_util.h
@@ -111,7 +111,23 @@  typedef struct _ib_hca_transport
 	uint8_t			sl;
 	uint16_t		pkey;
 	int			pkey_idx;
-
+#ifdef DAT_IB_COLLECTIVES
+	/* Collective member device and address information */
+	ib_thread_state_t 	coll_thread_state;
+	DAPL_OS_THREAD 		coll_thread;
+	DAPL_OS_LOCK 		coll_lock;
+	DAPL_OS_WAIT_OBJECT 	coll_event;
+	struct dapl_llist_entry *grp_list;
+	user_progress_func_t 	*user_func;
+	int 			l_sock;
+	struct sockaddr_in	m_addr;
+	void 			*m_ctx;
+	void			*m_info;
+	void			*f_info;
+	int			m_size;
+	int			f_size;
+	int			t_id;
+#endif
 } ib_hca_transport_t;
 
 /* prototypes */
diff --git a/dapl/openib_scm/device.c b/dapl/openib_scm/device.c
index 4c50f03..41fccdf 100644
--- a/dapl/openib_scm/device.c
+++ b/dapl/openib_scm/device.c
@@ -57,6 +57,10 @@  static const char rcsid[] = "$Id:  $";
 
 #include <stdlib.h>
 
+#ifdef DAT_IB_COLLECTIVES
+#include <collectives/ib_collectives.h>
+#endif
+
 ib_thread_state_t g_ib_thread_state = 0;
 DAPL_OS_THREAD g_ib_thread;
 DAPL_OS_LOCK g_hca_lock;
@@ -440,6 +444,11 @@  found:
 		     (unsigned long long)htonll(hca_ptr->ib_trans.gid.global.
 						interface_id));
 
+#ifdef DAT_IB_COLLECTIVES
+	if (dapli_create_collective_service(hca_ptr))
+		goto bail;
+#endif
+
 	ibv_free_device_list(dev_list);
 	return dat_status;
 
@@ -471,6 +480,10 @@  DAT_RETURN dapls_ib_close_hca(IN DAPL_HCA * hca_ptr)
 {
 	dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " close_hca: %p\n", hca_ptr);
 
+#ifdef DAT_IB_COLLECTIVES
+		dapli_free_collective_service(hca_ptr);
+#endif
+
 	dapl_os_lock(&g_hca_lock);
 	if (g_ib_thread_state != IB_THREAD_RUN) {
 		dapl_os_unlock(&g_hca_lock);