diff mbox series

[rdma-core,6/6] verbs: Let caller pre-fetch a sub-region of ODP MR in rc_pingpong

Message ID 1544016291-10815-7-git-send-email-yishaih@mellanox.com (mailing list archive)
State Not Applicable
Headers show
Series verbs: Introduce ibv_advise_mr() verb | expand

Commit Message

Yishai Hadas Dec. 5, 2018, 1:24 p.m. UTC 
From: Moni Shoua <monis@mellanox.com>

Add option (-P) to pre-fetch the memory that the application uses.
This option is applicable only with ODP MR and it should prevent a page
fault later on when the HCA tries to read or write from or to the
memory.

The pre-fetch is done once, just after creating the MR.

Signed-off-by: Moni Shoua <monis@mellanox.com>
Reviewed-by: Guy Levi <guyle@mellanox.com>
Signed-off-by: Yishai Hadas <yishaih@mellanox.com>
---
 libibverbs/examples/rc_pingpong.c | 29 ++++++++++++++++++++++++++++-
 libibverbs/man/ibv_rc_pingpong.1  |  7 +++++--
 2 files changed, 33 insertions(+), 3 deletions(-)
diff mbox series

Patch

diff --git a/libibverbs/examples/rc_pingpong.c b/libibverbs/examples/rc_pingpong.c
index d4c5df5..8b2253d 100644
--- a/libibverbs/examples/rc_pingpong.c
+++ b/libibverbs/examples/rc_pingpong.c
@@ -57,6 +57,7 @@  enum {
 
 static int page_size;
 static int use_odp;
+static int prefetch_mr;
 static int use_ts;
 static int validate_buf;
 static int use_dm;
@@ -430,6 +431,22 @@  static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
 		goto clean_dm;
 	}
 
+	if (prefetch_mr) {
+		struct ibv_sge sg_list;
+		int ret;
+
+		sg_list.lkey = ctx->mr->lkey;
+		sg_list.addr = (uintptr_t)ctx->buf;
+		sg_list.length = size;
+
+		ret = ibv_advise_mr(ctx->pd, IBV_ADVISE_MR_ADVICE_PREFETCH_WRITE,
+				    IB_UVERBS_ADVISE_MR_FLAG_FLUSH,
+				    &sg_list, 1);
+
+		if (ret)
+			fprintf(stderr, "Couldn't prefetch MR(%d). Continue anyway\n", ret);
+	}
+
 	if (use_ts) {
 		struct ibv_cq_init_attr_ex attr_ex = {
 			.cqe = rx_depth + 1,
@@ -717,6 +734,7 @@  static void usage(const char *argv0)
 	printf("  -e, --events           sleep on CQ events (default poll)\n");
 	printf("  -g, --gid-idx=<gid index> local port gid index\n");
 	printf("  -o, --odp		    use on demand paging\n");
+	printf("  -P, --prefetch	    prefetch an ODP MR\n");
 	printf("  -t, --ts	            get CQE with timestamp\n");
 	printf("  -c, --chk	            validate received buffer\n");
 	printf("  -j, --dm	            use device memory\n");
@@ -764,13 +782,14 @@  int main(int argc, char *argv[])
 			{ .name = "events",   .has_arg = 0, .val = 'e' },
 			{ .name = "gid-idx",  .has_arg = 1, .val = 'g' },
 			{ .name = "odp",      .has_arg = 0, .val = 'o' },
+			{ .name = "prefetch", .has_arg = 0, .val = 'P' },
 			{ .name = "ts",       .has_arg = 0, .val = 't' },
 			{ .name = "chk",      .has_arg = 0, .val = 'c' },
 			{ .name = "dm",       .has_arg = 0, .val = 'j' },
 			{}
 		};
 
-		c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:eg:otcj",
+		c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:eg:oPtcj",
 				long_options, NULL);
 
 		if (c == -1)
@@ -832,6 +851,9 @@  int main(int argc, char *argv[])
 		case 'o':
 			use_odp = 1;
 			break;
+		case 'P':
+			prefetch_mr = 1;
+			break;
 		case 't':
 			use_ts = 1;
 			break;
@@ -861,6 +883,11 @@  int main(int argc, char *argv[])
 		return 1;
 	}
 
+	if (!use_odp && prefetch_mr) {
+		fprintf(stderr, "prefetch is valid only with on-demand memory region\n");
+		return 1;
+	}
+
 	if (use_ts) {
 		ts.comp_recv_max_time_delta = 0;
 		ts.comp_recv_min_time_delta = 0xffffffff;
diff --git a/libibverbs/man/ibv_rc_pingpong.1 b/libibverbs/man/ibv_rc_pingpong.1
index ad5c834..5561fe5 100644
--- a/libibverbs/man/ibv_rc_pingpong.1
+++ b/libibverbs/man/ibv_rc_pingpong.1
@@ -8,12 +8,12 @@  ibv_rc_pingpong \- simple InfiniBand RC transport test
 .B ibv_rc_pingpong
 [\-p port] [\-d device] [\-i ib port] [\-s size] [\-m size]
 [\-r rx depth] [\-n iters] [\-l sl] [\-e] [\-g gid index]
-[\-o] [\-t] \fBHOSTNAME\fR
+[\-o] [\-P] [\-t] \fBHOSTNAME\fR
 
 .B ibv_rc_pingpong
 [\-p port] [\-d device] [\-i ib port] [\-s size] [\-m size]
 [\-r rx depth] [\-n iters] [\-l sl] [\-e] [\-g gid index]
-[\-o] [\-t]
+[\-o] [\-P] [\-t]
 
 .SH DESCRIPTION
 .PP
@@ -58,6 +58,9 @@  local port \fIGIDINDEX\fR
 \fB\-o\fR, \fB\-\-odp\fR
 use on demand paging
 .TP
+\fB\-P\fR, \fB\-\-prefetch=\fR
+prefetch an ODP MR
+.TP
 \fB\-t\fR, \fB\-\-ts\fR
 get CQE with timestamp
 .TP