From 3405180e97cd6b4d4bef6fed2a7e666eb8126906 Mon Sep 17 00:00:00 2001
From: Timo Dritschler <timo.dritschler@kit.edu>
Date: Fri, 25 Apr 2014 19:32:51 +0200
Subject: KIRO Server and Client now automatically exchange MRI on connect
Added 'kiro_client_sync' that RDMA-READs the KIRO-TRB from the server Updated
Makefile Updated test-client
---
Makefile | 8 +++---
kiro-client.c | 81 ++++++++++++++++++++++++++++++++++++++++++++---------------
kiro-client.h | 4 +--
kiro-server.c | 73 +++++++++++++++++++++++++++++++++++++++++++++++++----
kiro-server.h | 2 +-
test-client.c | 3 ++-
6 files changed, 138 insertions(+), 33 deletions(-)
diff --git a/Makefile b/Makefile
index 4e39036..f0f272b 100644
--- a/Makefile
+++ b/Makefile
@@ -24,11 +24,11 @@ test: test-trb client server
test-trb: kiro-trb.o test.c
$(CC) $(CFLAGS) $(LDFLAGS) test.c kiro-trb.o -o test-trb
-client: kiro-client.o test-client.c
- $(CC) $(CFLAGS) $(LDFLAGS) test-client.c kiro-client.o -o client
+client: kiro-client.o kiro-trb.o test-client.c
+ $(CC) $(CFLAGS) $(LDFLAGS) test-client.c kiro-client.o kiro-trb.o -o client
-server: kiro-server.o test-server.c
- $(CC) $(CFLAGS) $(LDFLAGS) test-server.c kiro-server.o -o server
+server: kiro-server.o kiro-trb.o test-server.c
+ $(CC) $(CFLAGS) $(LDFLAGS) test-server.c kiro-server.o kiro-trb.o -o server
.PHONY : clean
diff --git a/kiro-client.c b/kiro-client.c
index 41f4ff5..253251f 100644
--- a/kiro-client.c
+++ b/kiro-client.c
@@ -33,6 +33,7 @@
#include <glib.h>
#include "kiro-client.h"
#include "kiro-rdma.h"
+#include "kiro-trb.h"
#include <errno.h>
@@ -50,8 +51,9 @@ struct _KiroClientPrivate {
/* 'Real' private structures */
/* (Not accessible by properties) */
- struct rdma_event_channel *ec; // Main Event Channel
- struct rdma_cm_id *conn; // Connection to the Server
+ struct rdma_event_channel *ec; // Main Event Channel
+ struct rdma_cm_id *conn; // Connection to the Server
+ KiroTrb *buffer; // Ring Buffer used to hold data from server
};
@@ -64,12 +66,15 @@ static void kiro_client_init (KiroClient *self)
{
KiroClientPrivate *priv = KIRO_CLIENT_GET_PRIVATE(self);
memset(priv, 0, sizeof(&priv));
+ priv->buffer = g_object_new(KIRO_TYPE_TRB, NULL);
}
static void
kiro_client_finalize (GObject *object)
{
- //PASS
+ KiroClient *self = KIRO_CLIENT(object);
+ KiroClientPrivate * priv = KIRO_CLIENT_GET_PRIVATE(self);
+ g_object_unref(priv->buffer);
}
static void
@@ -162,38 +167,74 @@ int kiro_client_connect (KiroClient *self, char *address, char* port)
rdma_destroy_ep(priv->conn);
return -1;
}
+ printf("Connected to server.\n");
- priv->ec = rdma_create_event_channel();
- int oldflags = fcntl (priv->ec->fd, F_GETFL, 0);
- /* Only change the FD Mode if we were able to get its flags */
- if (oldflags >= 0) {
- oldflags |= O_NONBLOCK;
- /* Store modified flag word in the descriptor. */
- fcntl (priv->ec->fd, F_SETFL, oldflags);
- }
- if(rdma_migrate_id(priv->conn, priv->ec))
+
+ struct ibv_wc wc;
+ if(rdma_get_recv_comp(priv->conn, &wc) < 0)
{
- printf("Was unable to migrate connection to new Event Channel.\n");
+ printf("Failure waiting for POST from server.\n");
rdma_disconnect(priv->conn);
kiro_destroy_connection_context(ctx);
rdma_destroy_ep(priv->conn);
return -1;
}
+ printf("Got Message from Server.\n");
+ ctx->peer_mr = (((struct kiro_ctrl_msg *)(ctx->cf_mr_recv->mem))->peer_mri);
+ printf("Expected TRB Size is: %u\n",ctx->peer_mr.length);
- //ToDo:
- //Create TRB, request RDMA from Server, call kiro_client_sync, ???, Profit!
-
+ ctx->rdma_mr = kiro_create_rdma_memory(priv->conn->pd, ctx->peer_mr.length, IBV_ACCESS_LOCAL_WRITE);
+ if(!ctx->rdma_mr)
+ {
+ printf("Failed to allocate memory for receive buffer.\n");
+ rdma_disconnect(priv->conn);
+ kiro_destroy_connection_context(ctx);
+ rdma_destroy_ep(priv->conn);
+ return -1;
+ }
+ printf("Connection setup completed successfully!\n");
- printf("Connected to server.\n");
return 0;
-
}
int kiro_client_sync (KiroClient *self)
-{
- //PASS
+{
+ KiroClientPrivate *priv = KIRO_CLIENT_GET_PRIVATE(self);
+ struct kiro_connection_context *ctx = (struct kiro_connection_context *)priv->conn->context;
+
+ if(rdma_post_read(priv->conn, priv->conn, ctx->rdma_mr->mem, ctx->peer_mr.length, ctx->rdma_mr->mr, 0, ctx->peer_mr.addr, ctx->peer_mr.rkey))
+ {
+ printf("Failed to read from server.\n");
+ rdma_disconnect(priv->conn);
+ kiro_destroy_connection_context(ctx);
+ rdma_destroy_ep(priv->conn);
+ return -1;
+ }
+
+ struct ibv_wc wc;
+ if(rdma_get_send_comp(priv->conn, &wc) < 0)
+ {
+ printf("Failure reading from server.\n");
+ rdma_disconnect(priv->conn);
+ kiro_destroy_connection_context(ctx);
+ rdma_destroy_ep(priv->conn);
+ return -1;
+ }
+
+ if(!kiro_trb_is_setup(priv->buffer))
+ {
+ //First time setup
+ kiro_trb_ingest(priv->buffer, ctx->rdma_mr->mem);
+ }
+ else
+ {
+ //Refresh
+ kiro_trb_refresh(priv->buffer);
+ }
+
+ printf("Buffer successfully read from server.\n");
return 0;
}
diff --git a/kiro-client.h b/kiro-client.h
index 0f6881c..61e5676 100644
--- a/kiro-client.h
+++ b/kiro-client.h
@@ -77,9 +77,9 @@ GObject kiro_client_new (void);
/* client functions */
-int kiro_client_connect (KiroClient *, char *, char*);
+int kiro_client_connect (KiroClient* client, char* dest_addr, char* dest_port);
-int kiro_client_sync (KiroClient *);
+int kiro_client_sync (KiroClient* client);
G_END_DECLS
diff --git a/kiro-server.c b/kiro-server.c
index 119f0c1..d4cf7bc 100644
--- a/kiro-server.c
+++ b/kiro-server.c
@@ -33,6 +33,7 @@
#include <glib.h>
#include "kiro-server.h"
#include "kiro-rdma.h"
+#include "kiro-trb.h"
/*
@@ -51,6 +52,7 @@ struct _KiroServerPrivate {
struct rdma_event_channel *ec; // Main Event Channel
struct rdma_cm_id *base; // Base-Listening-Connection
struct kiro_connection *client; // Connection to the client
+ KiroTrb *buffer; // Memory Container
};
@@ -63,12 +65,21 @@ static void kiro_server_init (KiroServer *self)
{
KiroServerPrivate *priv = KIRO_SERVER_GET_PRIVATE(self);
memset(priv, 0, sizeof(&priv));
+
+ priv->buffer = g_object_new(KIRO_TYPE_TRB, NULL);
+ kiro_trb_reshape(priv->buffer, sizeof(uint64_t), 1000);
+ uint64_t a = 0xAFFED00F;
+ uint64_t b = 0x1337BEEF;
+ kiro_trb_push(priv->buffer, &a);
+ kiro_trb_push(priv->buffer, &b);
}
static void
kiro_server_finalize (GObject *object)
{
- //PASS
+ KiroServer *self = KIRO_SERVER(object);
+ KiroServerPrivate * priv = KIRO_SERVER_GET_PRIVATE(self);
+ g_object_unref(priv->buffer);
}
static void
@@ -218,13 +229,65 @@ int kiro_server_start (KiroServer *self, char *address, char *port)
return -1;
}
- //ToDo:
- //Create TRB, request RDMA from Server, call kiro_server_sync, ???, Profit!
+ printf("Client Connected.\n");
+
+ ctx->rdma_mr = (struct kiro_rdma_mem *)calloc(1, sizeof(struct kiro_rdma_mem));
+ if(!ctx->rdma_mr)
+ {
+ printf("Failed to allocate RDMA Memory Container.\n");
+ rdma_disconnect(priv->client->id);
+ kiro_destroy_connection_context(ctx);
+ rdma_destroy_ep(priv->base);
+ rdma_destroy_ep(priv->client->id);
+ free(priv->client);
+ return -1;
+ }
- printf("Client Connected.\n");
- return 0;
+ ctx->rdma_mr->mem = kiro_trb_get_raw_buffer(priv->buffer);
+ ctx->rdma_mr->size = kiro_trb_get_raw_size(priv->buffer);
+ ctx->rdma_mr->mr = rdma_reg_read(priv->client->id, ctx->rdma_mr->mem, ctx->rdma_mr->size);
+ if(!ctx->rdma_mr->mr)
+ {
+ printf("Failed to register RDMA Memory Region.\n");
+ rdma_disconnect(priv->client->id);
+ kiro_destroy_connection_context(ctx);
+ rdma_destroy_ep(priv->base);
+ rdma_destroy_ep(priv->client->id);
+ free(priv->client);
+ return -1;
+ }
+ struct kiro_ctrl_msg *msg = (struct kiro_ctrl_msg *)(ctx->cf_mr_send->mem);
+ msg->msg_type = KIRO_ACK_RDMA;
+ msg->peer_mri = *(ctx->rdma_mr->mr);
+
+ if(rdma_post_send(priv->client->id, priv->client, ctx->cf_mr_send->mem, ctx->cf_mr_send->size, ctx->cf_mr_send->mr, IBV_SEND_SIGNALED))
+ {
+ printf("Failure while trying to post SEND.\n");
+ rdma_disconnect(priv->client->id);
+ kiro_destroy_connection_context(ctx);
+ rdma_destroy_ep(priv->base);
+ rdma_destroy_ep(priv->client->id);
+ free(priv->client);
+ return -1;
+ }
+
+ struct ibv_wc wc;
+
+ if(rdma_get_send_comp(priv->client->id, &wc) < 0)
+ {
+ printf("Failed to post RDMA MRI to client.\n");
+ rdma_disconnect(priv->client->id);
+ kiro_destroy_connection_context(ctx);
+ rdma_destroy_ep(priv->base);
+ rdma_destroy_ep(priv->client->id);
+ free(priv->client);
+ return -1;
+ }
+ printf("RDMA MRI sent to client.\n");
+ sleep(1);
+ return 0;
}
diff --git a/kiro-server.h b/kiro-server.h
index 6e7973c..1d85419 100644
--- a/kiro-server.h
+++ b/kiro-server.h
@@ -76,7 +76,7 @@ GObject kiro_server_new (void);
/* server functions */
-int kiro_server_start (KiroServer*, char*, char*);
+int kiro_server_start (KiroServer* server, char* bind_addr, char* bind_port);
G_END_DECLS
diff --git a/test-client.c b/test-client.c
index 67dacf7..0a318f1 100644
--- a/test-client.c
+++ b/test-client.c
@@ -7,6 +7,7 @@
int main(void)
{
KiroClient *client = g_object_new(KIRO_TYPE_CLIENT, NULL);
- kiro_client_connect(client, "192.168.11.61", "60010", 5000);
+ kiro_client_connect(client, "192.168.11.61", "60010");
+ kiro_client_sync(client);
return 0;
}
\ No newline at end of file
--
cgit v1.2.3