scale: faster lookup whether local TEID is in use

Keep a hash table of all local GTP TEID assigned: speed up handing out
new TEID to new GTP tunnels, for both tunend and tunmap.

Before this, whenever osmo-upf sets up a tunnel, it would have to
linearly iterate *all* peers x sessions x GTP tunnels to ensure that a
TEID is not already in use. Now we iterate only one hashtable bucket.

Scoping: technically, a TEID would be scoped by the local GTP IP
address. For simplicity, osmo-upf hands out each TEID only once, across
all local GTP IP addresses. This also helps when analysing pcaps.

Change-Id: I920bb14da434ac29fcd49d95d207081bacb3d661

include/osmocom/upf/up_session.h

@@ -69,7 +69,7 @@ struct up_session {
 struct up_session *up_session_find_or_add(struct up_peer *peer, const struct osmo_pfcp_ie_f_seid *cp_f_seid);
 struct up_session *up_session_find_by_up_seid(struct up_peer *peer, uint64_t up_seid);
 struct up_session *up_session_find_by_cp_f_seid(struct up_peer *peer, const struct osmo_pfcp_ie_f_seid *cp_f_seid);
-struct up_session *up_session_find_by_local_teid(struct up_peer *peer, uint32_t teid);
+struct up_session *up_session_find_by_local_teid(uint32_t teid);
 
 void up_session_set_msg_ctx(struct up_session *session, struct osmo_pfcp_msg *m);
 
@@ -101,6 +101,9 @@ struct pdr {
 	bool active;
 
 	char *inactive_reason;
+
+	/* hashtable entry for g_upf->pdr_by_local_teid */
+	struct hlist_node node_by_local_teid;
 };
 
 int pdr_to_str_buf(char *buf, size_t buflen, const struct pdr *pdr);

include/osmocom/upf/upf.h

@@ -28,6 +28,7 @@
 #include <osmocom/core/socket.h>
 #include <osmocom/core/select.h>
 #include <osmocom/core/linuxlist.h>
+#include <osmocom/core/hashtable.h>
 
 struct osmo_tdef;
 struct ctrl_handle;
@@ -112,11 +113,15 @@ struct g_upf {
 		char *table_name;
 		int priority_pre;
 		int priority_post;
+
 		uint32_t next_chain_id_state;
+		/* fast look up of used nft chain ids */
+		DECLARE_HASHTABLE(nft_tun_by_chain_id, 6);
 	} tunmap;
 
 	struct {
 		uint32_t next_local_teid_state;
+		DECLARE_HASHTABLE(pdr_by_local_teid, 6);
 	} gtp;
 
 	struct llist_head netinst;
@@ -141,4 +146,4 @@ int upf_gtp_devs_open();
 void upf_gtp_devs_close();
 
 uint32_t upf_next_local_teid(void);
-uint32_t upf_next_chain_id(void);
+uint32_t upf_next_chain_id(struct hlist_node *for_node);

include/osmocom/upf/upf_nft.h

@@ -25,11 +25,25 @@
 
 #include <stdint.h>
 
+#include <osmocom/core/hashtable.h>
+
 #include <osmocom/upf/upf_tun.h>
 
 struct upf_nft_tun {
 	struct upf_tun tun;
+
+	/* Assigned id for nft ruleset, or 0 if none has been assigned. When this chain_id is invalidated again, the id
+	 * remains set here in chain_id, but chain_id_valid == false.
+	 * The assigned chain_id number is kept around to allow accurate logging also after removing a tunmap, like:
+	 *   "DGTP NOTICE [...] Disabled tunmap, nft chain IDs: access--1-> <-2--core"
+	 */
 	uint32_t chain_id;
+	/* Whether above chain_id is still reserved, i.e. whether this tun is still listed in the global hashtable
+	 * reserving chain_ids. If chain_id_valid == false, the id in chain_id must not be used in nft rules, and
+	 * node_by_chain_id is not an active entry in the nft_tun_by_chain_id hash table. */
+	bool chain_id_valid;
+	/* hashtable entry for g_upf->tunmap.nft_tun_by_chain_id */
+	struct hlist_node node_by_chain_id;
 };
 
 struct upf_tunmap {

src/osmo-upf/up_session.c

@@ -377,6 +377,8 @@ static struct pdr *pdr_find(struct up_session *session, uint16_t pdr_id)
 static void pdr_del(struct pdr *pdr)
 {
 	llist_del(&pdr->entry);
+	if (pdr->local_f_teid)
+		hlist_del(&pdr->node_by_local_teid);
 	talloc_free(pdr);
 }
 
@@ -479,6 +481,7 @@ static struct pdr *pdr_create(struct up_session *session,
 			.local_f_teid_present = true,
 			.local_f_teid = *pdr->local_f_teid,
 		};
+		hash_add(g_upf->gtp.pdr_by_local_teid, &pdr->node_by_local_teid, pdr->local_f_teid->fixed.teid);
 	} else {
 		created_pdr[*created_pdr_count] = (struct osmo_pfcp_ie_created_pdr){
 			.pdr_id = pdr->desc.pdr_id,
@@ -1051,18 +1054,12 @@ struct up_session *up_session_find_by_cp_f_seid(struct up_peer *peer, const stru
 	return NULL;
 }
 
-struct up_session *up_session_find_by_local_teid(struct up_peer *peer, uint32_t teid)
+struct up_session *up_session_find_by_local_teid(uint32_t teid)
 {
-	struct up_session *session;
+	struct pdr *pdr;
-	int bkt;
+	hash_for_each_possible(g_upf->gtp.pdr_by_local_teid, pdr, node_by_local_teid, teid) {
-	hash_for_each(peer->sessions_by_up_seid, bkt, session, node_by_up_seid) {
+		if (pdr->local_f_teid && teid == pdr->local_f_teid->fixed.teid)
-		struct pdr *pdr;
+			return pdr->session;
-		llist_for_each_entry(pdr, &session->pdrs, entry) {
-			if (!pdr->local_f_teid)
-				continue;
-			if (pdr->local_f_teid->fixed.teid == teid)
-				return session;
-		}
 	}
 	return NULL;
 }

src/osmo-upf/upf.c

@@ -66,6 +66,9 @@ void g_upf_alloc(void *ctx)
 	INIT_LLIST_HEAD(&g_upf->tunend.vty_cfg.devs);
 	INIT_LLIST_HEAD(&g_upf->tunend.devs);
 	INIT_LLIST_HEAD(&g_upf->netinst);
+
+	hash_init(g_upf->tunmap.nft_tun_by_chain_id);
+	hash_init(g_upf->gtp.pdr_by_local_teid);
 }
 
 int upf_pfcp_init(void)
@@ -121,17 +124,6 @@ int upf_gtp_devs_open()
 	return 0;
 }
 
-static bool upf_is_local_teid_in_use(uint32_t teid)
-{
-	struct up_peer *peer;
-	llist_for_each_entry(peer, &g_upf->pfcp.ep->peers, entry) {
-		struct up_session *session = up_session_find_by_local_teid(peer, teid);
-		if (session)
-			return true;
-	}
-	return false;
-}
-
 static uint32_t upf_next_local_teid_inc(void)
 {
 	g_upf->gtp.next_local_teid_state++;
@@ -145,7 +137,7 @@ uint32_t upf_next_local_teid(void)
 	uint32_t sanity;
 	for (sanity = 2342; sanity; sanity--) {
 		uint32_t next_teid = upf_next_local_teid_inc();
-		if (upf_is_local_teid_in_use(next_teid))
+		if (up_session_find_by_local_teid(next_teid))
 			continue;
 		return next_teid;
 	}
@@ -161,42 +153,28 @@ static uint32_t upf_next_chain_id_inc(void)
 }
 
 /* Return an unused chain_id, or 0 if none is found with sane effort. */
-uint32_t upf_next_chain_id(void)
+uint32_t upf_next_chain_id(struct hlist_node *for_node)
 {
 	uint32_t sanity;
 
 	/* Make sure the new chain_id is not used anywhere */
 	for (sanity = 2342; sanity; sanity--) {
-		struct up_peer *peer;
 		uint32_t chain_id = upf_next_chain_id_inc();
 		bool taken = false;
 
-		if (!g_upf->pfcp.ep)
+		struct upf_nft_tun *nft_tun;
-			return chain_id;
+		hash_for_each_possible(g_upf->tunmap.nft_tun_by_chain_id, nft_tun, node_by_chain_id, chain_id) {
-
+			if (chain_id == nft_tun->chain_id) {
-		llist_for_each_entry(peer, &g_upf->pfcp.ep->peers, entry) {
+				taken = true;
-			struct up_session *session;
-			int bkt;
-			hash_for_each(peer->sessions_by_up_seid, bkt, session, node_by_up_seid) {
-				struct up_gtp_action *a;
-				llist_for_each_entry(a, &session->active_gtp_actions, entry) {
-					if (a->kind != UP_GTP_U_TUNMAP)
-						continue;
-					if (a->tunmap.access.chain_id == chain_id
-					    || a->tunmap.core.chain_id == chain_id) {
-						taken = true;
-						break;
-					}
-				}
-				if (taken)
-					break;
-			}
-			if (taken)
 				break;
+			}
 		}
 
-		if (!taken)
+		if (!taken) {
+			/* add entry to hash table for fast lookup of used chain_ids */
+			hash_add(g_upf->tunmap.nft_tun_by_chain_id, for_node, chain_id);
 			return chain_id;
+		}
 	}
 
 	/* finding a chain_id became insane, return invalid = 0 */

src/osmo-upf/upf_nft.c

@@ -357,25 +357,46 @@ char *upf_nft_tunmap_get_ruleset_del_str(void *ctx, struct upf_tunmap *tunmap)
 	return upf_nft_ruleset_tunmap_delete_c(ctx, &args);
 }
 
-static int upf_nft_tunmap_ensure_chain_id(struct upf_nft_tun *tun)
+static int upf_nft_tunmap_chain_id_get(struct upf_nft_tun *tun)
 {
-	if (tun->chain_id)
+	if (tun->chain_id && tun->chain_id_valid)
-		return 0;
+		return -EALREADY;
-	tun->chain_id = upf_next_chain_id();
+	tun->chain_id = upf_next_chain_id(&tun->node_by_chain_id);
 	if (!tun->chain_id)
 		return -ENOSPC;
+	tun->chain_id_valid = true;
 	return 0;
 }
 
+static void upf_nft_tunmap_chain_id_put(struct upf_nft_tun *tun)
+{
+	if (!tun->chain_id || !tun->chain_id_valid)
+		return;
+	hlist_del(&tun->node_by_chain_id);
+	tun->chain_id_valid = false;
+}
+
 int upf_nft_tunmap_create(struct upf_tunmap *tunmap)
 {
-	if (upf_nft_tunmap_ensure_chain_id(&tunmap->access)
+	int rc;
-	    || upf_nft_tunmap_ensure_chain_id(&tunmap->core))
+	rc = upf_nft_tunmap_chain_id_get(&tunmap->access);
-		return -ENOSPC;
+	if (rc)
+		return rc;
+	rc = upf_nft_tunmap_chain_id_get(&tunmap->core);
+	if (rc) {
+		/* chain_id for core failed, but access side has already reserved a chain_id, release that chain_id
+		 * again. */
+		upf_nft_tunmap_chain_id_put(&tunmap->access);
+		return rc;
+	}
 	return upf_nft_run(upf_nft_tunmap_get_ruleset_str(OTC_SELECT, tunmap));
 }
 
 int upf_nft_tunmap_delete(struct upf_tunmap *tunmap)
 {
-	return upf_nft_run(upf_nft_tunmap_get_ruleset_del_str(OTC_SELECT, tunmap));
+	if (upf_nft_run(upf_nft_tunmap_get_ruleset_del_str(OTC_SELECT, tunmap)))
+		return -EIO;
+	upf_nft_tunmap_chain_id_put(&tunmap->access);
+	upf_nft_tunmap_chain_id_put(&tunmap->core);
+	return 0;
 }