bpftrace probing

Change-Id: I2717b736e23552bffe1492604cbd073f22977c84

contrib/perf/bpftrace.txt

@@ -0,0 +1,46 @@
+BEGIN {
+        @poll__last_nsecs = nsecs;
+}
+
+tracepoint:syscalls:sys_enter_poll /comm == "osmo-upf"/ {
+        $elapsed = (nsecs - @poll__last_nsecs)/1000000;
+        @poll__last_nsecs = nsecs;
+        @poll__ms = hist($elapsed);
+}
+
+uprobe:/usr/local/lib/libnftables.so.1.1.0:nft_run_cmd_from_buffer {
+        @nft_run_cmd_from_buffer__start = nsecs;
+}
+
+uretprobe:/usr/local/lib/libnftables.so.1.1.0:nft_run_cmd_from_buffer {
+        $nft_run_cmd_from_buffer__duration = (nsecs - @nft_run_cmd_from_buffer__start)/1000000;
+        @nft_run_cmd_from_buffer__ms = hist($nft_run_cmd_from_buffer__duration);
+}
+
+uprobe:/usr/local/bin/osmo-upf:up_gtp_action_enable {
+        @up_gtp_action_enable__start = nsecs;
+}
+
+uretprobe:/usr/local/bin/osmo-upf:up_gtp_action_enable {
+        $up_gtp_action_enable__duration = (nsecs - @up_gtp_action_enable__start)/1000000;
+        @up_gtp_action_enable__ms = hist($up_gtp_action_enable__duration);
+}
+
+uprobe:/usr/local/lib/libosmocore.so.20.0.0:logp2 {
+        @logp2__start = nsecs;
+}
+
+uretprobe:/usr/local/lib/libosmocore.so.20.0.0:logp2 {
+        $logp2__duration = (nsecs - @logp2__start)/1000;
+        @logp2__us = hist($logp2__duration);
+}
+
+
+interval:s:10 {
+        printf("==========================================================");
+        time("%Y-%m-%d %H:%M:%S\n");
+        print(@poll__ms);
+        print(@logp2__us);
+        print(@up_gtp_action_enable__ms);
+        print(@nft_run_cmd_from_buffer__ms);
+}

contrib/perf/osmo-pfcp-tool.cfg

@@ -0,0 +1,8 @@
+log stderr
+ logging level set-all info
+
+timer pfcp x23 5
+timer pfcp x24 5000
+
+local-addr 127.0.0.2
+listen

contrib/perf/osmo-upf.cfg

@@ -0,0 +1,30 @@
+log stderr
+ logging filter all 1
+ logging color 1
+ logging print level 1
+ logging print category 1
+ logging print category-hex 0
+ logging print file basename last
+ logging print timestamp date
+ logging level set-all notice
+ logging level session debug
+ logging level nft debug
+ logging level gtp debug
+#logging level set-all debug
+
+line vty
+ bind 127.0.0.1
+ctrl
+ bind 127.0.0.1
+
+# PFCP retransmissions
+timer pfcp x23 0
+# PFCP response timeout
+timer pfcp x24 0
+
+pfcp
+ local-addr 127.0.0.1
+tunmap
+ table-name osmo-upf
+tunend
+ dev create apn23

contrib/perf/tunend.vty

@@ -0,0 +1,7 @@
+pfcp-peer 127.0.0.1
+ tx assoc-setup-req
+ sleep 1
+
+ n 3 session create tunend
+ wait responses
+ sleep 5

contrib/perf/tunmap.vty

@@ -0,0 +1,7 @@
+pfcp-peer 127.0.0.1
+ tx assoc-setup-req
+ sleep 1
+
+ n 10000 session create tunmap
+ wait responses
+ sleep 5

src/osmo-pfcp-tool/osmo_pfcp_tool_main.c

@@ -226,14 +226,15 @@ const struct log_info log_info = {
 	.num_cat = ARRAY_SIZE(pfcp_tool_default_categories),
 };
 
-int pfcp_tool_mainloop()
+int pfcp_tool_mainloop(int poll)
 {
+	int rc;
 	log_reset_context();
-	osmo_select_main_ctx(0);
+	rc = osmo_select_main_ctx(poll);
 
 	/* If the user hits Ctrl-C the third time, just terminate immediately. */
 	if (quit >= 3)
-		return 1;
+		return -1;
 
 	/* Has SIGTERM been received (and not yet been handled)? */
 	if (quit && !osmo_select_shutdown_requested()) {
@@ -243,7 +244,7 @@ int pfcp_tool_mainloop()
 		osmo_select_shutdown_request();
 		/* continue the main select loop until all write queues are serviced. */
 	}
-	return 0;
+	return rc;
 }
 
 int main(int argc, char **argv)
@@ -326,7 +327,7 @@ int main(int argc, char **argv)
 		}
 	}
 
-	pfcp_tool_mainloop();
+	pfcp_tool_mainloop(1);
 
 	pfcp_tool_vty_init_cmds();
 
@@ -341,14 +342,14 @@ int main(int argc, char **argv)
 		printf("Done reading '%s', waiting for retransmission queue...\n",
 		       pfcp_tool_cmdline_config.command_file);
 		do {
-			if (pfcp_tool_mainloop())
+			if (pfcp_tool_mainloop(0) == -1)
 				break;
 		} while (osmo_pfcp_endpoint_retrans_queue_is_busy(g_pfcp_tool->ep));
 		printf("Done\n");
 	} else {
 		printf("Listening for commands on VTY...\n");
 		do {
-			if (pfcp_tool_mainloop())
+			if (pfcp_tool_mainloop(0) == -1)
 				break;
 		} while (!osmo_select_shutdown_done());
 	}

src/osmo-pfcp-tool/pfcp_tool.c

@@ -48,6 +48,8 @@ void g_pfcp_tool_alloc(void *ctx)
 	};
 
 	INIT_LLIST_HEAD(&g_pfcp_tool->peers);
+
+	g_pfcp_tool->next_ue_addr_state = 0xc0a90001;
 }
 
 struct pfcp_tool_peer *pfcp_tool_peer_find(const struct osmo_sockaddr *remote_addr)
@@ -183,3 +185,14 @@ uint64_t peer_new_seid(struct pfcp_tool_peer *peer)
 {
 	return peer->next_seid_state++;
 }
+
+uint32_t pfcp_tool_new_teid(void)
+{
+	return g_pfcp_tool->next_teid_state++;
+}
+
+int pfcp_tool_next_ue_addr(struct osmo_sockaddr_str *dst)
+{
+	uint32_t addr = g_pfcp_tool->next_ue_addr_state++;
+	return osmo_sockaddr_str_from_32h(dst, addr, 0);
+}

src/osmo-pfcp-tool/pfcp_tool.h

@@ -100,6 +100,9 @@ struct g_pfcp_tool {
 
 	struct osmo_pfcp_endpoint *ep;
 	struct llist_head peers;
+
+	uint32_t next_teid_state;
+	uint32_t next_ue_addr_state;
 };
 
 extern struct g_pfcp_tool *g_pfcp_tool;
@@ -108,7 +111,7 @@ void g_pfcp_tool_alloc(void *ctx);
 void pfcp_tool_vty_init_cfg();
 void pfcp_tool_vty_init_cmds();
 
-int pfcp_tool_mainloop();
+int pfcp_tool_mainloop(int poll);
 
 struct pfcp_tool_peer *pfcp_tool_peer_find_or_create(const struct osmo_sockaddr *remote_addr);
 struct pfcp_tool_session *pfcp_tool_session_find_or_create(struct pfcp_tool_peer *peer, uint64_t cp_seid,
@@ -117,3 +120,6 @@ void pfcp_tool_rx_msg(struct osmo_pfcp_endpoint *ep, struct osmo_pfcp_msg *m, st
 
 int peer_tx(struct pfcp_tool_peer *peer, struct osmo_pfcp_msg *m);
 uint64_t peer_new_seid(struct pfcp_tool_peer *peer);
+
+uint32_t pfcp_tool_new_teid(void);
+int pfcp_tool_next_ue_addr(struct osmo_sockaddr_str *dst);

src/osmo-pfcp-tool/pfcp_tool_vty.c

@@ -108,7 +108,7 @@ DEFUN(c_listen, c_listen_cmd,
 DEFUN(c_sleep, c_sleep_cmd,
 	"sleep <0-999999> [<0-999>]",
 	"Let some time pass\n"
-	"Seconds to wait\n")
+	"Seconds to wait\n" "Additional milliseconds to wait\n")
 {
 	int secs = atoi(argv[0]);
 	int msecs = 0;
@@ -124,7 +124,7 @@ DEFUN(c_sleep, c_sleep_cmd,
 
 	/* Still operate the message pump while waiting for time to pass */
 	while (t.active && !osmo_select_shutdown_done()) {
-		if (pfcp_tool_mainloop())
+		if (pfcp_tool_mainloop(0) == -1)
 			break;
 	}
 
@@ -397,9 +397,16 @@ DEFUN(s_f_teid_choose, s_f_teid_choose_cmd,
 	return CMD_SUCCESS;
 }
 
-int session_tunend_tx_est_req(struct vty *vty, const char **argv, int argc)
+enum pdr_id_fixed {
+	PDR_ID_CORE = 1,
+	PDR_ID_ACCESS = 2,
+};
+
+const char * const gtp_ip_core = "10.99.0.1";
+const char * const gtp_ip_access = "10.99.0.2";
+
+int session_tunend_tx_est_req(struct pfcp_tool_session *session, bool forw, osmo_pfcp_resp_cb resp_cb)
 {
-	struct pfcp_tool_session *session = vty->index;
 	struct pfcp_tool_peer *peer = session->peer;
 	int rc;
 	struct osmo_pfcp_msg *m;
@@ -411,20 +418,18 @@ int session_tunend_tx_est_req(struct vty *vty, const char **argv, int argc)
 
 	OSMO_ASSERT(session->kind == UP_GTP_U_TUNEND);
 
-	if (!g_pfcp_tool->ep) {
-		vty_out(vty, "Endpoint not configured%s", VTY_NEWLINE);
+	if (!g_pfcp_tool->ep)
 		return CMD_WARNING;
-	}
 
-	if (argc > 0 && !strcmp("drop", argv[0]))
-		osmo_pfcp_bits_set(aa.bits, OSMO_PFCP_APPLY_ACTION_DROP, true);
-	else
+	if (forw)
 		osmo_pfcp_bits_set(aa.bits, OSMO_PFCP_APPLY_ACTION_FORW, true);
+	else
+		osmo_pfcp_bits_set(aa.bits, OSMO_PFCP_APPLY_ACTION_DROP, true);
 
 #define STR_TO_ADDR(DST, SRC) do { \
 			if (osmo_sockaddr_str_to_sockaddr(&SRC, &DST.u.sas)) { \
-				vty_out(vty, "Error in " #SRC ": " OSMO_SOCKADDR_STR_FMT "%s", \
-					OSMO_SOCKADDR_STR_FMT_ARGS(&SRC), VTY_NEWLINE); \
+				LOGP(DLGLOBAL, LOGL_ERROR, "Error in " #SRC ": " OSMO_SOCKADDR_STR_FMT "\n", \
+				     OSMO_SOCKADDR_STR_FMT_ARGS(&SRC)); \
 				return CMD_WARNING; \
 			} \
 		} while (0)
@@ -464,6 +469,10 @@ int session_tunend_tx_est_req(struct vty *vty, const char **argv, int argc)
 	osmo_pfcp_ip_addrs_set(&cp_f_seid.ip_addr, osmo_pfcp_endpoint_get_local_addr(g_pfcp_tool->ep));
 
 	m = osmo_pfcp_msg_alloc_tx_req(OTC_SELECT, &peer->remote_addr, OSMO_PFCP_MSGT_SESSION_EST_REQ);
+
+	m->ctx.resp_cb = resp_cb;
+	m->ctx.priv = session;
+
 	m->h.seid_present = true;
 	/* the UPF has yet to assign a SEID for itself, no matter what SEID we (the CPF) use for this session */
 	m->h.seid = 0;
@@ -475,7 +484,7 @@ int session_tunend_tx_est_req(struct vty *vty, const char **argv, int argc)
 		.create_pdr_count = 2,
 		.create_pdr = {
 			{
-				.pdr_id = 1,
+				.pdr_id = PDR_ID_CORE,
 				.precedence = 255,
 				.pdi = {
 					.source_iface = OSMO_PFCP_SOURCE_IFACE_CORE,
@@ -492,7 +501,7 @@ int session_tunend_tx_est_req(struct vty *vty, const char **argv, int argc)
 				.far_id = 1,
 			},
 			{
-				.pdr_id = 2,
+				.pdr_id = PDR_ID_ACCESS,
 				.precedence = 255,
 				.pdi = {
 					.source_iface = OSMO_PFCP_SOURCE_IFACE_ACCESS,
@@ -532,15 +541,14 @@ int session_tunend_tx_est_req(struct vty *vty, const char **argv, int argc)
 
 	rc = peer_tx(peer, m);
 	if (rc) {
-		vty_out(vty, "Failed to transmit: %s%s", strerror(-rc), VTY_NEWLINE);
+		LOGP(DLGLOBAL, LOGL_ERROR, "Failed to transmit: %s\n", strerror(-rc));
 		return CMD_WARNING;
 	}
 	return CMD_SUCCESS;
 }
 
-int session_tunmap_tx_est_req(struct vty *vty, const char **argv, int argc)
+int session_tunmap_tx_est_req(struct pfcp_tool_session *session, bool forw, osmo_pfcp_resp_cb resp_cb)
 {
-	struct pfcp_tool_session *session = vty->index;
 	struct pfcp_tool_peer *peer = session->peer;
 	int rc;
 	struct osmo_pfcp_msg *m;
@@ -555,15 +563,13 @@ int session_tunmap_tx_est_req(struct vty *vty, const char **argv, int argc)
 
 	struct osmo_pfcp_ie_apply_action aa = {};
 
-	if (!g_pfcp_tool->ep) {
-		vty_out(vty, "Endpoint not configured%s", VTY_NEWLINE);
+	if (!g_pfcp_tool->ep)
 		return CMD_WARNING;
-	}
 
-	if (argc > 0 && !strcmp("drop", argv[0]))
-		osmo_pfcp_bits_set(aa.bits, OSMO_PFCP_APPLY_ACTION_DROP, true);
-	else
+	if (forw)
 		osmo_pfcp_bits_set(aa.bits, OSMO_PFCP_APPLY_ACTION_FORW, true);
+	else
+		osmo_pfcp_bits_set(aa.bits, OSMO_PFCP_APPLY_ACTION_DROP, true);
 
 	if (session->tunmap.access.local.teid == 0) {
 		f_teid_access_local = (struct osmo_pfcp_ie_f_teid){
@@ -625,6 +631,10 @@ int session_tunmap_tx_est_req(struct vty *vty, const char **argv, int argc)
 	osmo_pfcp_ip_addrs_set(&cp_f_seid.ip_addr, osmo_pfcp_endpoint_get_local_addr(g_pfcp_tool->ep));
 
 	m = osmo_pfcp_msg_alloc_tx_req(OTC_SELECT, &peer->remote_addr, OSMO_PFCP_MSGT_SESSION_EST_REQ);
+
+	m->ctx.resp_cb = resp_cb;
+	m->ctx.priv = session;
+
 	m->h.seid_present = true;
 	m->h.seid = 0;
 	/* GTP tunmap: remove header from both directions, and add header in both directions */
@@ -635,7 +645,7 @@ int session_tunmap_tx_est_req(struct vty *vty, const char **argv, int argc)
 		.create_pdr_count = 2,
 		.create_pdr = {
 			{
-				.pdr_id = 1,
+				.pdr_id = PDR_ID_CORE,
 				.precedence = 255,
 				.pdi = {
 					.source_iface = OSMO_PFCP_SOURCE_IFACE_CORE,
@@ -650,7 +660,7 @@ int session_tunmap_tx_est_req(struct vty *vty, const char **argv, int argc)
 				.far_id = 1,
 			},
 			{
-				.pdr_id = 2,
+				.pdr_id = PDR_ID_ACCESS,
 				.precedence = 255,
 				.pdi = {
 					.source_iface = OSMO_PFCP_SOURCE_IFACE_ACCESS,
@@ -692,7 +702,7 @@ int session_tunmap_tx_est_req(struct vty *vty, const char **argv, int argc)
 
 	rc = peer_tx(peer, m);
 	if (rc) {
-		vty_out(vty, "Failed to transmit: %s%s", strerror(-rc), VTY_NEWLINE);
+		LOGP(DLGLOBAL, LOGL_ERROR, "Failed to transmit PFCP: %s\n", strerror(-rc));
 		return CMD_WARNING;
 	}
 	return CMD_SUCCESS;
@@ -705,39 +715,33 @@ DEFUN(session_tx_est_req, session_tx_est_req_cmd,
       "Set FAR to DROP = 1\n")
 {
 	struct pfcp_tool_session *session = vty->index;
+	bool forw = (argc == 0 || !strcmp("forw", argv[0]));
 	switch (session->kind) {
 	case UP_GTP_U_TUNEND:
-		return session_tunend_tx_est_req(vty, argv, argc);
+		return session_tunend_tx_est_req(session, forw, NULL);
 	case UP_GTP_U_TUNMAP:
-		return session_tunmap_tx_est_req(vty, argv, argc);
+		return session_tunmap_tx_est_req(session, forw, NULL);
 	default:
 		vty_out(vty, "unknown gtp action%s", VTY_NEWLINE);
 		return CMD_WARNING;
 	}
 }
 
-DEFUN(session_tx_mod_req, session_tx_mod_req_cmd,
-      "tx session-mod-req far [(forw|drop)]",
-      TX_STR "Send a Session Modification Request\n"
-      "Set FAR to FORW = 1\n"
-      "Set FAR to DROP = 1\n")
+int session_tunmap_tx_mod_req(struct pfcp_tool_session *session, bool forw, osmo_pfcp_resp_cb resp_cb)
 {
-	struct pfcp_tool_session *session = vty->index;
 	struct pfcp_tool_peer *peer = session->peer;
 	int rc;
 	struct osmo_pfcp_msg *m;
 	struct osmo_pfcp_ie_apply_action aa = {};
 	struct osmo_pfcp_ie_f_seid cp_f_seid;
 
-	if (!g_pfcp_tool->ep) {
-		vty_out(vty, "Endpoint not configured%s", VTY_NEWLINE);
+	if (!g_pfcp_tool->ep)
 		return CMD_WARNING;
-	}
 
-	if (argc > 0 && !strcmp("drop", argv[0]))
-		osmo_pfcp_bits_set(aa.bits, OSMO_PFCP_APPLY_ACTION_DROP, true);
-	else
+	if (forw)
 		osmo_pfcp_bits_set(aa.bits, OSMO_PFCP_APPLY_ACTION_FORW, true);
+	else
+		osmo_pfcp_bits_set(aa.bits, OSMO_PFCP_APPLY_ACTION_DROP, true);
 
 	cp_f_seid = (struct osmo_pfcp_ie_f_seid){
 		.seid = session->cp_seid,
@@ -745,6 +749,10 @@ DEFUN(session_tx_mod_req, session_tx_mod_req_cmd,
 	osmo_pfcp_ip_addrs_set(&cp_f_seid.ip_addr, osmo_pfcp_endpoint_get_local_addr(g_pfcp_tool->ep));
 
 	m = osmo_pfcp_msg_alloc_tx_req(OTC_SELECT, &peer->remote_addr, OSMO_PFCP_MSGT_SESSION_MOD_REQ);
+
+	m->ctx.resp_cb = resp_cb;
+	m->ctx.priv = session;
+
 	m->h.seid_present = true;
 	m->h.seid = session->up_f_seid.seid;
 	m->ies.session_mod_req = (struct osmo_pfcp_msg_session_mod_req){
@@ -766,13 +774,25 @@ DEFUN(session_tx_mod_req, session_tx_mod_req_cmd,
 	};
 
 	rc = peer_tx(peer, m);
-	if (rc) {
-		vty_out(vty, "Failed to transmit: %s%s", strerror(-rc), VTY_NEWLINE);
+	if (rc)
 		return CMD_WARNING;
-	}
 	return CMD_SUCCESS;
 }
 
+DEFUN(session_tx_mod_req, session_tx_mod_req_cmd,
+      "tx session-mod-req far [(forw|drop)]",
+      TX_STR "Send a Session Modification Request\n"
+      "Set FAR to FORW = 1\n"
+      "Set FAR to DROP = 1\n")
+{
+	struct pfcp_tool_session *session = vty->index;
+	bool forw = (argc == 0 || !strcmp("forw", argv[0]));
+	int rc = session_tunmap_tx_mod_req(session, forw, NULL);
+	if (rc != CMD_SUCCESS)
+		vty_out(vty, "Failed to send Session Modification Request%s", VTY_NEWLINE);
+	return rc;
+}
+
 DEFUN(session_tx_del_req, session_tx_del_req_cmd,
       "tx session-del-req",
       TX_STR "Send a Session Deletion Request\n")
@@ -799,6 +819,280 @@ DEFUN(session_tx_del_req, session_tx_del_req_cmd,
 	return CMD_SUCCESS;
 }
 
+/* N SESSIONS */
+
+static int responses_pending = 0;
+
+DEFUN(wait_responses, wait_responses_cmd,
+	"wait responses",
+	"Let some time pass until events have occured\n"
+	"Wait for all PFCP responses for pending PFCP requests\n")
+{
+	if (!responses_pending) {
+		vty_out(vty, "no responses pending, not waiting.%s", VTY_NEWLINE);
+		return CMD_SUCCESS;
+	}
+
+	vty_out(vty, "waiting for %d responses...%s", responses_pending, VTY_NEWLINE);
+	vty_flush(vty);
+
+	/* Still operate the message pump while waiting for time to pass */
+	while (!osmo_select_shutdown_done()) {
+		if (pfcp_tool_mainloop(0) == -1)
+			break;
+		if (responses_pending == 0)
+			break;
+	}
+
+	vty_out(vty, "...done waiting for responses%s", VTY_NEWLINE);
+	vty_flush(vty);
+	return CMD_SUCCESS;
+}
+
+/* N SESSIONS TUNEND */
+
+int one_session_mod_tunend_resp_cb(struct osmo_pfcp_msg *req, struct osmo_pfcp_msg *rx_resp, const char *errmsg)
+{
+	responses_pending--;
+	return 0;
+}
+
+int one_session_mod_tunend(struct pfcp_tool_session *session)
+{
+	int rc;
+
+	rc = session_tunmap_tx_mod_req(session, true, one_session_mod_tunend_resp_cb);
+	if (rc == CMD_SUCCESS)
+		responses_pending++;
+	return rc;
+}
+
+void est_resp_get_created_f_teid(struct pfcp_tool_gtp_tun_ep *dst, const struct osmo_pfcp_msg *rx_resp, uint16_t pdr_id)
+{
+	int i;
+	const struct osmo_pfcp_msg_session_est_resp *r;
+	if (rx_resp->h.message_type != OSMO_PFCP_MSGT_SESSION_EST_RESP)
+		return;
+
+	r = &rx_resp->ies.session_est_resp;
+
+	for (i = 0; i < r->created_pdr_count; i++) {
+		const struct osmo_pfcp_ie_created_pdr *p = &r->created_pdr[i];
+		if (p->pdr_id != pdr_id)
+			continue;
+		if (!p->local_f_teid_present)
+			continue;
+		osmo_sockaddr_str_from_sockaddr(&dst->addr,
+						&p->local_f_teid.fixed.ip_addr.v4.u.sas);
+		dst->teid = p->local_f_teid.fixed.teid;
+	}
+}
+
+int one_session_create_tunend_resp_cb(struct osmo_pfcp_msg *req, struct osmo_pfcp_msg *rx_resp, const char *errmsg)
+{
+	struct pfcp_tool_session *session = req->ctx.priv;
+	enum osmo_pfcp_cause *cause;
+	const struct osmo_pfcp_msg_session_est_resp *r = &rx_resp->ies.session_est_resp;
+
+	responses_pending--;
+
+	if (errmsg)
+		LOGP(DLPFCP, LOGL_ERROR, "%s\n", errmsg);
+
+	cause = rx_resp ? osmo_pfcp_msg_cause(rx_resp) : NULL;
+	if (!cause || *cause != OSMO_PFCP_CAUSE_REQUEST_ACCEPTED)
+		return 0;
+
+	/* store SEID */
+	if (r->up_f_seid_present)
+		session->up_f_seid = r->up_f_seid;
+	/* store access local F-TEID */
+	est_resp_get_created_f_teid(&session->tunend.access.local, rx_resp, PDR_ID_ACCESS);
+
+	/* Success response, now continue with second step: Session Mod to set the CORE's remote side GTP */
+	one_session_mod_tunend(session);
+	return 0;
+}
+
+static int one_session_create_tunend(struct pfcp_tool_peer *peer)
+{
+	struct pfcp_tool_session *session;
+	struct pfcp_tool_tunend *te;
+	struct pfcp_tool_gtp_tun_ep *dst;
+	int rc;
+
+	session = pfcp_tool_session_find_or_create(peer, peer_new_seid(peer), UP_GTP_U_TUNEND);
+	te = &session->tunend;
+
+	/* Access: set remote GTP address */
+	dst = &te->access.remote;
+	if (osmo_sockaddr_str_from_str2(&dst->addr, gtp_ip_access)) {
+		LOGP(DLGLOBAL, LOGL_ERROR, "Error setting GTP IP address from %s\n",
+		     osmo_quote_cstr_c(OTC_SELECT, gtp_ip_access, -1));
+		return CMD_WARNING;
+	}
+	dst->teid = pfcp_tool_new_teid();
+
+	/* Set local F-TEIDs == CHOOSE */
+	te->access.local = (struct pfcp_tool_gtp_tun_ep){};
+	pfcp_tool_next_ue_addr(&te->core.ue_local_addr);
+
+	/* Send initial Session Establishment Request */
+	rc = session_tunend_tx_est_req(session, false, one_session_create_tunend_resp_cb);
+	if (rc == CMD_SUCCESS)
+		responses_pending++;
+	return rc;
+}
+
+/* N SESSIONS TUNMAP */
+
+int one_session_mod_tunmap_resp_cb(struct osmo_pfcp_msg *req, struct osmo_pfcp_msg *rx_resp, const char *errmsg)
+{
+	responses_pending--;
+	return 0;
+}
+
+int one_session_mod_tunmap(struct pfcp_tool_session *session)
+{
+	struct pfcp_tool_gtp_tun_ep *dst;
+	int rc;
+
+	dst = &session->tunmap.core.remote;
+	if (osmo_sockaddr_str_from_str2(&dst->addr, gtp_ip_core)) {
+		LOGP(DLGLOBAL, LOGL_ERROR, "Error setting GTP IP address from %s\n",
+		     osmo_quote_cstr_c(OTC_SELECT, gtp_ip_core, -1));
+		return CMD_WARNING;
+	}
+	dst->teid = pfcp_tool_new_teid();
+
+	rc = session_tunmap_tx_mod_req(session, true, one_session_mod_tunmap_resp_cb);
+	if (rc == CMD_SUCCESS)
+		responses_pending++;
+	return rc;
+}
+
+int one_session_create_tunmap_resp_cb(struct osmo_pfcp_msg *req, struct osmo_pfcp_msg *rx_resp, const char *errmsg)
+{
+	struct pfcp_tool_session *session = req->ctx.priv;
+	enum osmo_pfcp_cause *cause;
+
+	responses_pending--;
+
+	if (errmsg)
+		LOGP(DLPFCP, LOGL_ERROR, "%s\n", errmsg);
+
+	cause = rx_resp ? osmo_pfcp_msg_cause(rx_resp) : NULL;
+	if (!cause || *cause != OSMO_PFCP_CAUSE_REQUEST_ACCEPTED)
+		return 0;
+
+	/* store SEID */
+	if (rx_resp->ies.session_est_resp.up_f_seid_present)
+		session->up_f_seid = rx_resp->ies.session_est_resp.up_f_seid;
+
+	/* store local F-TEIDs */
+	est_resp_get_created_f_teid(&session->tunmap.access.local, rx_resp, PDR_ID_ACCESS);
+	est_resp_get_created_f_teid(&session->tunmap.core.local, rx_resp, PDR_ID_CORE);
+
+	/* Success response, now continue with second step: Session Mod to set the CORE's remote side GTP */
+	one_session_mod_tunmap(session);
+	return 0;
+}
+
+static int one_session_create_tunmap(struct pfcp_tool_peer *peer)
+{
+	struct pfcp_tool_session *session;
+	struct pfcp_tool_tunmap *tm;
+	struct pfcp_tool_gtp_tun_ep *dst;
+	int rc;
+
+	session = pfcp_tool_session_find_or_create(peer, peer_new_seid(peer), UP_GTP_U_TUNMAP);
+	tm = &session->tunmap;
+
+	/* Access: set remote GTP address */
+	dst = &tm->access.remote;
+	if (osmo_sockaddr_str_from_str2(&dst->addr, gtp_ip_access)) {
+		LOGP(DLGLOBAL, LOGL_ERROR, "Error setting GTP IP address from %s\n",
+		     osmo_quote_cstr_c(OTC_SELECT, gtp_ip_access, -1));
+		return CMD_WARNING;
+	}
+	dst->teid = pfcp_tool_new_teid();
+
+	/* Core: set remote GTP address */
+	dst = &tm->core.remote;
+	if (osmo_sockaddr_str_from_str2(&dst->addr, gtp_ip_core)) {
+		LOGP(DLGLOBAL, LOGL_ERROR, "Error setting GTP IP address from %s\n",
+		     osmo_quote_cstr_c(OTC_SELECT, gtp_ip_core, -1));
+		return CMD_WARNING;
+	}
+	dst->teid = pfcp_tool_new_teid();
+
+	/* Set local F-TEIDs == CHOOSE */
+	tm->access.local = (struct pfcp_tool_gtp_tun_ep){};
+	tm->core.local = (struct pfcp_tool_gtp_tun_ep){};
+
+	/* Send initial Session Establishment Request */
+	rc = session_tunmap_tx_est_req(session, false, one_session_create_tunmap_resp_cb);
+	if (rc == CMD_SUCCESS)
+		responses_pending++;
+	return rc;
+	return CMD_WARNING;
+}
+
+static void n_sessions_create(struct pfcp_tool_peer *peer, int n, enum up_gtp_action_kind kind)
+{
+	int i;
+	for (i = 0; i < n; i++) {
+		int rc;
+		if (kind == UP_GTP_U_TUNMAP)
+			rc = one_session_create_tunmap(peer);
+		else
+			rc = one_session_create_tunend(peer);
+		if (rc != CMD_SUCCESS)
+			break;
+
+		/* handle any pending select work */
+		while (!osmo_select_shutdown_done()) {
+			rc = pfcp_tool_mainloop(1);
+			/* quit requested */
+			if (rc < 0)
+				return;
+			/* no fd needed service */
+			if (rc == 0)
+				break;
+		}
+	}
+}
+
+static void n_sessions_delete(struct pfcp_tool_peer *peer, int n, enum up_gtp_action_kind kind)
+{
+}
+
+DEFUN(n_sessions, n_sessions_cmd,
+      "n (<0-2147483647>|all) session (create|delete) (tunend|tunmap)",
+      "Batch run\n"
+      "Perform the action N times, or on all available entries\n"
+      "In a batch run, create and later delete a number of sessions at once.\n"
+      "Create N new sessions\n"
+      "Delete N sessions created earlier\n"
+      TUNEND_STR TUNMAP_STR)
+{
+	struct pfcp_tool_peer *peer = vty->index;
+	int n = (strcmp("all", argv[0]) == 0? -1 : atoi(argv[0]));
+	bool create = (strcmp("create", argv[1]) == 0);
+	enum up_gtp_action_kind kind;
+	if (!strcmp(argv[2], "tunmap"))
+		kind = UP_GTP_U_TUNMAP;
+	else
+		kind = UP_GTP_U_TUNEND;
+
+	if (create)
+		n_sessions_create(peer, n, kind);
+	else
+		n_sessions_delete(peer, n, kind);
+	return CMD_SUCCESS;
+}
+
+
 static void install_ve_and_config(struct cmd_element *cmd)
 {
 	install_element_ve(cmd);
@@ -818,6 +1112,7 @@ void pfcp_tool_vty_init_cmds()
 	OSMO_ASSERT(g_pfcp_tool != NULL);
 
 	install_ve_and_config(&c_sleep_cmd);
+	install_ve_and_config(&wait_responses_cmd);
 
 	install_ve_and_config(&peer_cmd);
 	install_node(&peer_node, NULL);
@@ -826,6 +1121,8 @@ void pfcp_tool_vty_init_cmds()
 	install_element(PEER_NODE, &peer_tx_heartbeat_cmd);
 	install_element(PEER_NODE, &peer_tx_assoc_setup_req_cmd);
 	install_element(PEER_NODE, &peer_retrans_req_cmd);
+	install_element(PEER_NODE, &n_sessions_cmd);
+	install_element(PEER_NODE, &wait_responses_cmd);
 
 	install_element(PEER_NODE, &session_cmd);
 	install_element(PEER_NODE, &session_endecaps_cmd);