WIP: run external tests in qemu

Related: OS#6425
Change-Id: I4e3cfc5547d9e6464db58d7229ec4d8f655788e5

.gitignore

@@ -57,6 +57,7 @@ tests/atlocal
 tests/package.m4
 tests/testsuite
 tests/testsuite.log
+tests/qemu/_*
 
 writtenconfig/
 

configure.ac

@@ -49,7 +49,7 @@ AC_SEARCH_LIBS([sctp_recvmsg], [sctp], [
 LIBS=$old_LIBS
 
 PKG_CHECK_MODULES(LIBASN1C, libasn1c >= 0.9.30)
-PKG_CHECK_MODULES(LIBOSMOCORE, libosmocore >= 1.9.0)
+PKG_CHECK_MODULES(LIBOSMOCORE, libosmocore > 1.9.0)
 PKG_CHECK_MODULES(LIBOSMOVTY, libosmovty >= 1.9.0)
 PKG_CHECK_MODULES(LIBOSMOCTRL, libosmoctrl >= 1.9.0)
 PKG_CHECK_MODULES(LIBOSMOGSM, libosmogsm >= 1.9.0)
@@ -70,6 +70,16 @@ fi
 AM_CONDITIONAL(ENABLE_PFCP, test "x$osmo_ac_pfcp" = "xyes")
 AC_SUBST(osmo_ac_pfcp)
 
+# Enable libnftables support for traffic counters using nft
+AC_ARG_ENABLE([nftables], [AS_HELP_STRING([--enable-nftables], [Build with libnftables support, for traffic counters using nft])],
+    [osmo_ac_nftables="$enableval"],[osmo_ac_nftables="no"])
+if test "x$osmo_ac_nftables" = "xyes" ; then
+	PKG_CHECK_MODULES(LIBNFTABLES, libnftables >= 1.0.2)
+	AC_DEFINE(ENABLE_NFTABLES, 1, [Define to build with libnftables support])
+fi
+AM_CONDITIONAL(ENABLE_NFTABLES, test "x$osmo_ac_nftables" = "xyes")
+AC_SUBST(osmo_ac_nftables)
+
 dnl checks for header files
 AC_HEADER_STDC
 
@@ -157,6 +167,17 @@ AC_MSG_CHECKING([whether to enable VTY/CTRL tests])
 AC_MSG_RESULT([$enable_ext_tests])
 AM_CONDITIONAL(ENABLE_EXT_TESTS, test "x$enable_ext_tests" = "xyes")
 
+AC_ARG_ENABLE([external_tests_qemu],
+		AC_HELP_STRING([--enable-external-tests-qemu],
+				[Run VTY/CTRL tests in QEMU [default=no]]),
+		[enable_ext_tests_qemu="$enableval"],[enable_ext_tests_qemu="no"])
+AC_MSG_CHECKING([whether to run VTY/CTRL tests in QEMU])
+AC_MSG_RESULT([$enable_ext_tests_qemu])
+AM_CONDITIONAL(ENABLE_EXT_TESTS_QEMU, test "x$enable_ext_tests_qemu" = "xyes")
+if test x"$exnable_ext_tests_qemu" = x"yes" && ! $srcdir/tests/qemu/check-depends.sh; then
+	AC_MSG_ERROR([missing programs for --enable-external-tests-qemu])
+fi
+
 # Generate manuals
 AC_ARG_ENABLE(manuals,
 	[AS_HELP_STRING(

contrib/jenkins.sh

@@ -4,9 +4,11 @@
 # environment variables:
 # * PFCP: configure PFCP support if set to "1" (default)
 # * WITH_MANUALS: build manual PDFs if set to "1"
+# * NFTABLES: configure nftables support if set to "1" (default)
 # * PUBLISH: upload manuals after building if set to "1" (ignored without WITH_MANUALS = "1")
 #
 PFCP=${PFCP:-1}
+NFTABLES=${NFTABLES:-1}
 
 if ! [ -x "$(command -v osmo-build-dep.sh)" ]; then
 	echo "Error: We need to have scripts/osmo-deps.sh from http://git.osmocom.org/osmo-ci/ in PATH !"
@@ -45,6 +47,9 @@ if [ "$PFCP" = "1" ]; then
 	osmo-build-dep.sh libosmo-pfcp
 	CONFIG="$CONFIG --enable-pfcp"
 fi
+if [ "$NFTABLES" = "1" ]; then
+	CONFIG="$CONFIG --enable-nftables --enable-external-tests-qemu"
+fi
 if [ "$WITH_MANUALS" = "1" ]; then
 	CONFIG="$CONFIG --enable-manuals"
 fi

debian/control

@@ -22,6 +22,7 @@ Build-Depends: debhelper (>= 10),
                libosmo-ranap-dev (>= 1.5.0),
                libosmo-rua-dev (>= 1.5.0),
                libosmo-pfcp-dev (>= 0.3.0),
+               libnftables-dev,
                osmo-gsm-manuals-dev (>= 1.5.0)
 Standards-Version: 3.9.8
 Vcs-Git: https://gitea.osmocom.org/cellular-infrastructure/osmo-hnbgw

debian/rules

@@ -44,11 +44,18 @@
 %:
 	dh $@ --with autoreconf
 
-# debmake generated override targets
+# libnftables is too old in Debian 10 (OS#6425)
-CONFIGURE_FLAGS += --with-systemdsystemunitdir=/lib/systemd/system --enable-manuals
-CONFIGURE_FLAGS += --enable-pfcp
 override_dh_auto_configure:
-	dh_auto_configure -- $(CONFIGURE_FLAGS)
+	CONFIGURE_FLAGS=" \
+		--enable-manuals \
+		--enable-pfcp \
+		--with-systemdsystemunitdir=/lib/systemd/system \
+	"; \
+	if pkg-config --exists libnftables --atleast-version=1.0.2; then \
+		CONFIGURE_FLAGS="$$CONFIGURE_FLAGS --enable-nftables"; \
+	fi; \
+	dh_auto_configure -- $$CONFIGURE_FLAGS
+
 #
 # Do not install libtool archive, python .pyc .pyo
 #override_dh_install:

include/osmocom/hnbgw/Makefile.am

@@ -3,6 +3,7 @@ noinst_HEADERS = \
 	context_map.h hnbgw.h hnbgw_cn.h \
 	hnbgw_hnbap.h hnbgw_rua.h hnbgw_ranap.h \
 	kpi.h \
+	nft_kpi.h \
 	ranap_rab_ass.h mgw_fsm.h tdefs.h \
 	hnbgw_pfcp.h \
 	ps_rab_ass_fsm.h \

include/osmocom/hnbgw/hnbgw.h

@@ -6,6 +6,7 @@
 #include <osmocom/core/write_queue.h>
 #include <osmocom/core/timer.h>
 #include <osmocom/core/rate_ctr.h>
+#include <osmocom/core/sockaddr_str.h>
 #include <osmocom/gsm/gsm23003.h>
 #include <osmocom/sigtran/sccp_sap.h>
 #include <osmocom/sigtran/osmo_ss7.h>
@@ -18,6 +19,8 @@
 #include <osmocom/mgcp_client/mgcp_client.h>
 #include <osmocom/mgcp_client/mgcp_client_pool.h>
 
+#include <osmocom/hnbgw/nft_kpi.h>
+
 #define STORE_UPTIME_INTERVAL	10 /* seconds */
 #define HNB_STORE_RAB_DURATIONS_INTERVAL 1 /* seconds */
 
@@ -29,6 +32,7 @@ enum {
 	DMGW,
 	DHNB,
 	DCN,
+	DNFT,
 };
 
 extern const struct log_info hnbgw_log_info;
@@ -132,6 +136,11 @@ enum hnb_rate_ctr {
 	HNB_CTR_CS_PAGING_ATTEMPTED,
 
 	HNB_CTR_RAB_ACTIVE_MILLISECONDS_TOTAL,
+
+	HNB_CTR_GTPU_PACKETS_UL,
+	HNB_CTR_GTPU_TOTAL_BYTES_UL,
+	HNB_CTR_GTPU_PACKETS_DL,
+	HNB_CTR_GTPU_TOTAL_BYTES_DL,
 };
 
 enum hnb_stat {
@@ -353,6 +362,14 @@ struct hnb_persistent {
 
 	struct rate_ctr_group *ctrs;
 	struct osmo_stat_item_group *statg;
+
+	struct {
+		struct osmo_sockaddr_str addr_remote;
+		struct {
+			struct nft_kpi_val ul;
+			struct nft_kpi_val dl;
+		} last;
+	} nft_kpi;
 };
 
 struct ue_context {
@@ -451,6 +468,8 @@ void hnb_context_release_ue_state(struct hnb_context *ctx);
 
 struct hnb_persistent *hnb_persistent_alloc(const struct umts_cell_id *id);
 struct hnb_persistent *hnb_persistent_find_by_id(const struct umts_cell_id *id);
+struct hnb_persistent *hnb_persistent_find_by_id_str(const char *id_str);
+void hnb_persistent_update_addr(struct hnb_persistent *hnbp, int new_fd);
 void hnb_persistent_free(struct hnb_persistent *hnbp);
 
 void hnbgw_vty_init(void);

include/osmocom/hnbgw/nft_kpi.h

@@ -0,0 +1,17 @@
+#pragma once
+#include <stdint.h>
+#include <stdbool.h>
+
+struct hnb_persistent;
+
+struct nft_kpi_val {
+	uint64_t packets;
+	uint64_t bytes;
+
+	bool handle_present;
+	int64_t handle;
+};
+
+int nft_kpi_init(void);
+int hnb_nft_kpi_start(struct hnb_persistent *hnbp, const struct osmo_sockaddr_str *gtpu_remote);
+int hnb_nft_kpi_end(struct hnb_persistent *hnbp);

src/osmo-hnbgw/Makefile.am

@@ -20,6 +20,7 @@ AM_CFLAGS = \
 	$(LIBOSMORANAP_CFLAGS) \
 	$(LIBOSMOHNBAP_CFLAGS) \
 	$(LIBOSMOMGCPCLIENT_CFLAGS) \
+	$(LIBNFTABLES_CFLAGS) \
 	$(NULL)
 
 AM_LDFLAGS = \
@@ -46,6 +47,7 @@ libhnbgw_la_SOURCES = \
 	mgw_fsm.c \
 	kpi_ranap.c \
 	tdefs.c \
+	nft_kpi.c \
 	$(NULL)
 
 libhnbgw_la_LIBADD = \
@@ -62,6 +64,7 @@ libhnbgw_la_LIBADD = \
 	$(LIBOSMOHNBAP_LIBS) \
 	$(LIBSCTP_LIBS) \
 	$(LIBOSMOMGCPCLIENT_LIBS) \
+	$(LIBNFTABLES_LIBS) \
 	$(NULL)
 
 if ENABLE_PFCP

src/osmo-hnbgw/hnbgw.c

@@ -342,8 +342,10 @@ void hnb_context_release(struct hnb_context *ctx)
 	}
 
 	/* remove back reference from hnb_persistent to context */
-	if (ctx->persistent)
+	if (ctx->persistent) {
+		hnb_nft_kpi_end(ctx->persistent);
 		ctx->persistent->ctx = NULL;
+	}
 
 	talloc_free(ctx);
 }
@@ -457,6 +459,24 @@ const struct rate_ctr_desc hnb_ctr_description[] = {
 
 	[HNB_CTR_RAB_ACTIVE_MILLISECONDS_TOTAL] = {
 		"rab:cs:active_milliseconds:total", "Cumulative number of milliseconds of CS RAB activity" },
+
+	[HNB_CTR_GTPU_PACKETS_UL] = {
+		"gtpu:packets:ul",
+		"Count of GTP-U packets received from the HNB",
+	},
+	[HNB_CTR_GTPU_TOTAL_BYTES_UL] = {
+		"gtpu:total_bytes:ul",
+		"Count of total GTP-U bytes received from the HNB, including the GTP-U/UDP/IP headers",
+	},
+	[HNB_CTR_GTPU_PACKETS_DL] = {
+		"gtpu:packets:dl",
+		"Count of GTP-U packets sent to the HNB",
+	},
+	[HNB_CTR_GTPU_TOTAL_BYTES_DL] = {
+		"gtpu:total_bytes:dl",
+		"Count of total GTP-U bytes sent to the HNB, including the GTP-U/UDP/IP headers",
+	},
+
 };
 
 const struct rate_ctr_group_desc hnb_ctrg_desc = {
@@ -519,9 +539,45 @@ struct hnb_persistent *hnb_persistent_find_by_id(const struct umts_cell_id *id)
 	return NULL;
 }
 
+struct hnb_persistent *hnb_persistent_find_by_id_str(const char *id_str)
+{
+	struct hnb_persistent *hnbp;
+	llist_for_each_entry (hnbp, &g_hnbgw->hnb_persistent_list, list) {
+		if (strcmp(hnbp->id_str, id_str))
+			continue;
+		return hnbp;
+	}
+	return NULL;
+}
+
+/* Read the peer's remote IP address from the Iuh conn's fd, and set up GTP-U counters for that remote address. */
+void hnb_persistent_update_addr(struct hnb_persistent *hnbp, int new_fd)
+{
+	int rc;
+	socklen_t socklen;
+	struct osmo_sockaddr osa;
+	struct osmo_sockaddr_str remote_str;
+
+	socklen = sizeof(struct osmo_sockaddr);
+	rc = getpeername(new_fd, &osa.u.sa, &socklen);
+	if (!rc)
+		rc = osmo_sockaddr_str_from_osa(&remote_str, &osa);
+	if (rc < 0) {
+		LOGP(DHNB, LOGL_ERROR, "cannot read remote hNodeB address from Iuh file descriptor\n");
+		return;
+	}
+
+	/* We got the remote address from the RUA link, and now we are blatantly assuming that the hNodeB has its GTP
+	 * endpoint on the same IP address, just with UDP port 2152 (the fixed GTP port as per 3GPP spec). */
+	remote_str.port = 2152;
+
+	hnb_nft_kpi_start(hnbp, &remote_str);
+}
+
 void hnb_persistent_free(struct hnb_persistent *hnbp)
 {
 	/* FIXME: check if in use? */
+	hnb_nft_kpi_end(hnbp);
 	llist_del(&hnbp->list);
 	talloc_free(hnbp);
 }
@@ -847,6 +903,11 @@ static const struct log_info_cat hnbgw_log_cat[] = {
 		.color = OSMO_LOGCOLOR_DARKYELLOW,
 		.description = "Core Network side (via SCCP)",
 	},
+	[DNFT] = {
+		.name = "DNFT", .loglevel = LOGL_NOTICE, .enabled = 1,
+		.color = OSMO_LOGCOLOR_BLUE,
+		.description = "nftables interaction for retrieving stats",
+	},
 };
 
 const struct log_info hnbgw_log_info = {

src/osmo-hnbgw/hnbgw_hnbap.c

@@ -553,6 +553,8 @@ static int hnbgw_rx_hnb_register_req(struct hnb_context *ctx, ANY_t *in)
 
 	ctx->hnb_registered = true;
 
+	hnb_persistent_update_addr(ctx->persistent, osmo_stream_srv_get_fd(ctx->conn));
+
 	/* Send HNBRegisterAccept */
 	rc = hnbgw_tx_hnb_register_acc(ctx);
 	hnbap_free_hnbregisterrequesties(&ies);

src/osmo-hnbgw/hnbgw_vty.c

@@ -35,6 +35,7 @@
 #include <osmocom/hnbgw/hnbgw_cn.h>
 #include <osmocom/hnbgw/context_map.h>
 #include <osmocom/hnbgw/tdefs.h>
+#include <osmocom/hnbgw/nft_kpi.h>
 #include <osmocom/sigtran/protocol/sua.h>
 #include <osmocom/sigtran/sccp_helpers.h>
 #include <osmocom/netif/stream.h>

src/osmo-hnbgw/nft_kpi.c

@@ -0,0 +1,421 @@
+/* Set up and read internet traffic counters using netfilter */
+/* Copyright (C) 2024 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+ * All Rights Reserved
+ *
+ * Author: Neels Janosch Hofmeyr <nhofmeyr@sysmocom.de>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ */
+
+#include <osmocom/core/logging.h>
+#include <osmocom/hnbgw/hnbgw.h>
+
+#include "config.h"
+
+#if !ENABLE_NFTABLES
+
+int hnb_nft_kpi_start(struct hnb_persistent *hnbp, const struct osmo_sockaddr_str *gtpu_remote)
+{
+	LOGP(DNFT, LOGL_INFO, "Built without libnftables support, not starting nft based counters for HNB %s\n",
+	     hnbp->id_str);
+	return 0;
+}
+
+int hnb_nft_kpi_end(struct hnb_persistent *hnbp)
+{
+	return 0;
+}
+
+#else
+
+#include <stdbool.h>
+#include <ctype.h>
+#include <inttypes.h>
+
+#include <nftables/libnftables.h>
+
+#include <osmocom/core/timer.h>
+#include <osmocom/hnbgw/nft_kpi.h>
+#include <osmocom/hnbgw/tdefs.h>
+
+struct nft_kpi_state {
+	struct {
+		struct nft_ctx *nft_ctx;
+		char *table_name;
+		bool table_created;
+	} nft;
+	struct osmo_timer_list period;
+};
+
+static struct nft_kpi_state g_nft_kpi_state = {};
+
+static struct nft_ctx *g_nft_ctx(void)
+{
+	struct nft_kpi_state *s = &g_nft_kpi_state;
+
+	if (s->nft.nft_ctx)
+		return s->nft.nft_ctx;
+
+	s->nft.nft_ctx = nft_ctx_new(NFT_CTX_DEFAULT);
+	if (!s->nft.nft_ctx) {
+		LOGP(DNFT, LOGL_ERROR, "cannot allocate libnftables nft_ctx\n");
+		OSMO_ASSERT(false);
+	}
+
+	nft_ctx_output_set_flags(s->nft.nft_ctx, NFT_CTX_OUTPUT_HANDLE);
+
+	return s->nft.nft_ctx;
+}
+
+static int nft_run_now(const char *buffer)
+{
+	int rc;
+	const int logmax = 256;
+
+	rc = nft_run_cmd_from_buffer(g_nft_ctx(), buffer);
+	if (rc < 0) {
+		LOGP(DNFT, LOGL_ERROR, "error running nft buffer: rc=%d buffer=%s\n",
+		     rc, osmo_quote_str_c(OTC_SELECT, buffer, -1));
+		return -EIO;
+	}
+
+	if (log_check_level(DNFT, LOGL_DEBUG)) {
+		size_t l = strlen(buffer);
+		LOGP(DNFT, LOGL_DEBUG, "ran nft buffer, %zu chars: \"%s%s\"\n",
+		     l,
+		     osmo_escape_cstr_c(OTC_SELECT, buffer, OSMO_MIN(logmax, l)),
+		     l > logmax ? "..." : "");
+	}
+
+	return 0;
+}
+
+static void nft_kpi_period_cb(void *data);
+
+static void nft_kpi_period_schedule(void)
+{
+	unsigned long period = osmo_tdef_get(hnbgw_T_defs, -34, OSMO_TDEF_S, 10);
+	if (period < 1)
+		period = 1;
+	osmo_timer_setup(&g_nft_kpi_state.period, nft_kpi_period_cb, NULL);
+	osmo_timer_schedule(&g_nft_kpi_state.period, period, 0);
+}
+
+int nft_kpi_init(void)
+{
+	struct nft_kpi_state *s = &g_nft_kpi_state;
+	char cmd[1024];
+	struct osmo_strbuf sb = { .buf = cmd, .len = sizeof(cmd) };
+
+	if (s->nft.table_created)
+		return 0;
+
+	if (!s->nft.table_name || !*s->nft.table_name)
+		s->nft.table_name = talloc_strdup(g_hnbgw, "osmo-hnbgw");
+
+	OSMO_STRBUF_PRINTF(sb, "add table inet %s { flags owner; };\n", s->nft.table_name);
+	OSMO_STRBUF_PRINTF(sb,
+			   "add chain inet %s gtpu-ul {"
+			   " type filter hook prerouting priority 0; policy accept;"
+			   " ip protocol != udp accept;"
+			   " udp sport != 2152 accept;"
+			   " udp dport != 2152 accept;"
+			   "};\n",
+			   s->nft.table_name);
+	OSMO_STRBUF_PRINTF(sb,
+			   "add chain inet %s gtpu-dl {"
+			   " type filter hook postrouting priority 0; policy accept;"
+			   " ip protocol != udp accept;"
+			   " udp sport != 2152 accept;"
+			   " udp dport != 2152 accept;"
+			   "};\n",
+			   s->nft.table_name);
+	OSMO_ASSERT(sb.chars_needed < sizeof(cmd));
+
+	if (nft_run_now(cmd))
+		return -EIO;
+
+	s->nft.table_created = true;
+	nft_kpi_period_schedule();
+	return 0;
+}
+
+/* Set up counters for the hNodeB's remote address */
+int hnb_nft_kpi_start(struct hnb_persistent *hnbp, const struct osmo_sockaddr_str *gtpu_remote)
+{
+	struct nft_kpi_state *s = &g_nft_kpi_state;
+	char cmd[1024];
+	struct osmo_strbuf sb = { .buf = cmd, .len = sizeof(cmd) };
+	int rc;
+
+	if (!osmo_sockaddr_str_is_nonzero(gtpu_remote)) {
+		LOGP(DNFT, LOGL_ERROR, "HNB %s: invalid remote GTP-U address: " OSMO_SOCKADDR_STR_FMT "\n",
+		     hnbp->id_str, OSMO_SOCKADDR_STR_FMT_ARGS(gtpu_remote));
+		return -EINVAL;
+	}
+
+	if (!osmo_sockaddr_str_cmp(gtpu_remote, &hnbp->nft_kpi.addr_remote)) {
+		/* The remote address is unchanged, no need to update the nft probe */
+		return 0;
+	}
+
+	nft_kpi_init();
+
+	/* The remote address has changed. Cancel previous probe, if any, and start a new one. */
+	if (osmo_sockaddr_str_is_nonzero(&hnbp->nft_kpi.addr_remote))
+		hnb_nft_kpi_end(hnbp);
+
+	hnbp->nft_kpi.last.ul = (struct nft_kpi_val){};
+	hnbp->nft_kpi.last.dl = (struct nft_kpi_val){};
+
+	hnbp->nft_kpi.addr_remote = *gtpu_remote;
+
+	OSMO_STRBUF_PRINTF(sb, "add rule inet %s gtpu-ul ip saddr %s counter comment \"ul:%s\";\n",
+			   s->nft.table_name,
+			   hnbp->nft_kpi.addr_remote.ip,
+			   hnbp->id_str);
+	OSMO_STRBUF_PRINTF(sb, "add rule inet %s gtpu-dl ip daddr %s counter comment \"dl:%s\";\n",
+			   s->nft.table_name,
+			   hnbp->nft_kpi.addr_remote.ip,
+			   hnbp->id_str);
+	OSMO_ASSERT(sb.chars_needed < sizeof(cmd));
+
+	rc = nft_run_now(cmd);
+	if (rc) {
+		/* There was an error running the rule, clear addr_remote to indicate that no rule exists. */
+		hnbp->nft_kpi.addr_remote = (struct osmo_sockaddr_str){};
+	}
+	return rc;
+}
+
+static void nft_kpi_read_counters(void);
+
+/* Terminate nft based counters for this HNB */
+int hnb_nft_kpi_end(struct hnb_persistent *hnbp)
+{
+	struct nft_kpi_state *s = &g_nft_kpi_state;
+	char *cmd;
+
+	/* When there is no table created, neither can there be any rules to be deleted.
+	 * The rules get removed, but the table remains present for as long as osmo-hnbgw runs. */
+	if (!s->nft.table_created)
+		return 0;
+
+	/* presence of addr_remote indicates whether an nft rule has been submitted and still needs to be removed */
+	if (!osmo_sockaddr_str_is_nonzero(&hnbp->nft_kpi.addr_remote))
+		return 0;
+
+	if (!hnbp->nft_kpi.last.ul.handle_present
+	    || !hnbp->nft_kpi.last.dl.handle_present) {
+		/* We get to know the nft handles only after creating the rule, when querying the counters. If the
+		 * handle is not known here yet, then it means we haven't read the counters yet. We have to find out the
+		 * handle now. */
+		nft_kpi_read_counters();
+	}
+
+	/* clear the addr to indicate that the nft rule no longer exists. Even if below 'delete rule' fails, just
+	 * attempt to delete the rule once. */
+	hnbp->nft_kpi.addr_remote = (struct osmo_sockaddr_str){};
+
+	cmd = talloc_asprintf(OTC_SELECT,
+			      "delete rule inet %s gtpu-ul handle %"PRId64";\n"
+			      "delete rule inet %s gtpu-dl handle %"PRId64";\n",
+			      s->nft.table_name,
+			      hnbp->nft_kpi.last.ul.handle,
+			      s->nft.table_name,
+			      hnbp->nft_kpi.last.dl.handle);
+	return nft_run_now(cmd);
+}
+
+static void update_ctr(struct rate_ctr_group *cg, int cgidx, uint64_t *last_val, uint64_t new_val)
+{
+	/* Because an hNodeB may re-connect, or even change the address it connects from, we need to store the last seen
+	 * value and add the difference to the rate counter. For example, the rate_ctr that lives in hnb_persistent has
+	 * seen 100 GTP-U packets. The hNodeB disconnects for ten seconds and then comes back. Now the nft ruleset has
+	 * been deleted and re-created, so the counters we read are back at 0, but we want to continue showing 100. When
+	 * the ruleset detects 10, we want to show 110. Hence this last_val stuff here.
+	 * last_val is also back to zero whenever the nft counters are restarted, see hnb_nft_kpi_start(), where
+	 * hnbp->nft_kpi.last.ul and last.dl are zeroed.
+	 */
+	if (new_val > *last_val)
+		rate_ctr_add2(cg, cgidx, new_val - *last_val);
+	*last_val = new_val;
+}
+
+static void hnb_update_counters(struct hnb_persistent *hnbp, bool ul, int64_t packets, int64_t bytes, int64_t handle)
+{
+	struct nft_kpi_val *val = (ul ? &hnbp->nft_kpi.last.ul : &hnbp->nft_kpi.last.dl);
+
+	/* Remember the nftables handle, which is needed to remove a rule when a hNodeB disconnects. */
+	if (handle) {
+		val->handle_present = true;
+		val->handle = handle;
+	}
+
+	update_ctr(hnbp->ctrs,
+		   ul ? HNB_CTR_GTPU_PACKETS_UL : HNB_CTR_GTPU_PACKETS_DL,
+		   &val->packets, packets);
+	update_ctr(hnbp->ctrs,
+		   ul ? HNB_CTR_GTPU_TOTAL_BYTES_UL : HNB_CTR_GTPU_TOTAL_BYTES_DL,
+		   &val->bytes, bytes);
+}
+
+/* In the string section *pos .. end, find the first occurrence of after_str and return the following token, which ends
+ * by a space or at end. If end is NULL, search until the '\0' termination of *pos.
+ * Return true if after_str was found, copy the following token into buf, and in *pos, return the position just after
+ * that token. */
+static bool get_token_after(char *buf, size_t buflen, const char **pos, const char *end, const char *after_str)
+{
+	const char *found = strstr(*pos, after_str);
+	const char *token_end;
+	size_t token_len;
+	if (!found)
+		return false;
+	if (end && found >= end) {
+		*pos = end;
+		return false;
+	}
+	found += strlen(after_str);
+	while (*found && *found == ' ' && (!end || found < end))
+		found++;
+	token_end = found;
+	while (*token_end != ' ' && (!end || token_end < end))
+		token_end++;
+	if (token_end <= found) {
+		*pos = found;
+		return false;
+	}
+	if (*found == '"' && token_end > found + 1 && *(token_end - 1) == '"') {
+		found++;
+		token_end--;
+	}
+	token_len = token_end - found;
+	token_len = OSMO_MIN(token_len, buflen - 1);
+	memcpy(buf, found, token_len);
+	buf[token_len] = '\0';
+	*pos = token_end;
+	return true;
+}
+
+static void decode_nft_response(const char *response)
+{
+	struct nft_kpi_state *s = &g_nft_kpi_state;
+	const char *pos;
+	char buf[128];
+	int count = 0;
+
+	/* find and parse all occurences of strings like:
+	 *    [...] counter packets 3 bytes 129 comment "ul:001-01-L2-R3-S4-C1" # handle 10
+	 */
+	pos = response;
+	while (*pos) {
+		const char *line_end;
+		int64_t packets;
+		int64_t bytes;
+		int64_t handle = 0;
+		bool ul;
+		struct hnb_persistent *hnbp;
+
+		if (!get_token_after(buf, sizeof(buf), &pos, NULL, "counter packets "))
+			break;
+		if (osmo_str_to_int64(&packets, buf, 10, 0, INT64_MAX))
+			break;
+		line_end = strchr(pos, '\n');
+		if (!line_end)
+			line_end = pos + strlen(pos);
+
+		if (!get_token_after(buf, sizeof(buf), &pos, line_end, "bytes "))
+			break;
+		if (osmo_str_to_int64(&bytes, buf, 10, 0, INT64_MAX))
+			break;
+
+		if (!get_token_after(buf, sizeof(buf), &pos, line_end, "comment "))
+			break;
+
+		if (osmo_str_startswith(buf, "ul:"))
+			ul = true;
+		else if (osmo_str_startswith(buf, "dl:"))
+			ul = false;
+		else
+			break;
+
+		hnbp = hnb_persistent_find_by_id_str(buf + 3);
+		if (!hnbp)
+			break;
+
+		if (!get_token_after(buf, sizeof(buf), &pos, line_end, "# handle "))
+			break;
+		if (osmo_str_to_int64(&handle, buf, 10, 0, INT64_MAX))
+			break;
+
+		hnb_update_counters(hnbp, ul, packets, bytes, handle);
+		count++;
+	}
+
+	LOGP(DNFT, LOGL_DEBUG, "read %d counters from nft table %s\n", count, s->nft.table_name);
+}
+
+static void nft_kpi_read_counters(void)
+{
+	int rc;
+	const int logmax = 256;
+	struct nft_kpi_state *s = &g_nft_kpi_state;
+	struct nft_ctx *nft = s->nft.nft_ctx;
+	char cmd[256];
+	struct osmo_strbuf sb = { .buf = cmd, .len = sizeof(cmd) };
+	const char *output;
+
+	if (!nft)
+		return;
+
+	OSMO_STRBUF_PRINTF(sb, "list table inet %s", s->nft.table_name);
+	OSMO_ASSERT(sb.chars_needed < sizeof(cmd));
+
+	rc = nft_ctx_buffer_output(nft);
+	if (rc) {
+		LOGP(DNFT, LOGL_ERROR, "error: nft_ctx_buffer_output() returned failure: rc=%d cmd=%s\n",
+		     rc, osmo_quote_str_c(OTC_SELECT, cmd, -1));
+		goto unbuffer_and_exit;
+	}
+	rc = nft_run_cmd_from_buffer(nft, cmd);
+	if (rc < 0) {
+		LOGP(DNFT, LOGL_ERROR, "error running nft cmd: rc=%d cmd=%s\n",
+		     rc, osmo_quote_str_c(OTC_SELECT, cmd, -1));
+		goto unbuffer_and_exit;
+	}
+
+	output = nft_ctx_get_output_buffer(nft);
+	if (log_check_level(DNFT, LOGL_DEBUG)) {
+		size_t l = strlen(cmd);
+		LOGP(DNFT, LOGL_DEBUG, "ran nft request, %zu chars: \"%s%s\"\n",
+		     l,
+		     osmo_escape_cstr_c(OTC_SELECT, cmd, OSMO_MIN(logmax, l)),
+		     l > logmax ? "..." : "");
+		l = strlen(output);
+		LOGP(DNFT, LOGL_DEBUG, "got nft response, %zu chars: \"%s%s\"\n",
+		     l,
+		     osmo_escape_cstr_c(OTC_SELECT, output, OSMO_MIN(logmax, l)),
+		     l > logmax ? "..." : "");
+	}
+
+	decode_nft_response(output);
+
+unbuffer_and_exit:
+	nft_ctx_unbuffer_output(nft);
+}
+
+static void nft_kpi_period_cb(void *data)
+{
+	nft_kpi_read_counters();
+	nft_kpi_period_schedule();
+}
+
+#endif // ENABLE_NFTABLES

src/osmo-hnbgw/osmo_hnbgw_main.c

@@ -328,6 +328,14 @@ int main(int argc, char **argv)
 	/* If UPF is configured, set up PFCP socket and send Association Setup Request to UPF */
 	hnbgw_pfcp_init();
 #endif
+#if ENABLE_NFTABLES
+	/* If nftables is enabled, initialize the nft table now or fail startup. This is important to immediately let
+	 * the user know if cap_net_admin privileges are missing, and not only when the first hNodeB connects. */
+	if (nft_kpi_init()) {
+		perror("Failed to initialize nft KPI, probably missing cap_net_admin");
+		exit(1);
+	}
+#endif
 
 	hnbgw_cnpool_start(&g_hnbgw->sccp.cnpool_iucs);
 	hnbgw_cnpool_start(&g_hnbgw->sccp.cnpool_iups);

src/osmo-hnbgw/tdefs.c

@@ -36,6 +36,7 @@ struct osmo_tdef hnbgw_T_defs[] = {
 	{.T = 4, .default_val = 5, .desc = "Timeout to receive RANAP RESET ACKNOWLEDGE from an MSC/SGSN" },
 	{.T = -31, .default_val = 15, .desc = "Timeout for establishing and releasing context maps (RUA <-> SCCP)" },
 	{.T = -1002, .default_val = 10, .desc = "Timeout for the HNB to respond to PS RAB Assignment Request" },
+	{.T = -34, .default_val = 1, .desc = "Period to query network traffic stats from netfilter" },
 	{ }
 };
 

tests/Makefile.am

@@ -39,12 +39,16 @@ DISTCLEANFILES = \
 	atconfig \
 	$(NULL)
 
+if ENABLE_EXT_TESTS_QEMU
+QEMU_EXT_TEST_WRAPPER = BUILDDIR=$(abs_top_builddir) $(abs_top_srcdir)/tests/qemu/ext_test_wrapper.sh
+endif
+
 if ENABLE_EXT_TESTS
 python-tests:
 	$(MAKE) vty-test
 	$(MAKE) config-tests
-	osmotestvty.py -p $(abs_top_srcdir) -w $(abs_top_builddir) -v
+	$(QEMU_EXT_TEST_WRAPPER) osmotestvty.py -p $(abs_top_srcdir) -w $(abs_top_builddir) -v
-	osmotestconfig.py -p $(abs_top_srcdir) -w $(abs_top_builddir) -v
+	$(QEMU_EXT_TEST_WRAPPER) osmotestconfig.py -p $(abs_top_srcdir) -w $(abs_top_builddir) -v
 else
 python-tests:
 	echo "Not running python-based tests (determined at configure-time)"
@@ -61,7 +65,7 @@ endif
 # pass -u to vty_script_runner.py by doing:
 #   make vty-test U=-u
 vty-test: $(top_builddir)/src/osmo-hnbgw/osmo-hnbgw
-	osmo_verify_transcript_vty.py -v \
+	$(QEMU_EXT_TEST_WRAPPER) osmo_verify_transcript_vty.py -v \
 		-n OsmoHNBGW -p 4261 \
 		-r "$(top_builddir)/src/osmo-hnbgw/osmo-hnbgw \
 		-c $(srcdir)/osmo-hnbgw-vty-test.cfg" \
@@ -72,7 +76,10 @@ vty-test: $(top_builddir)/src/osmo-hnbgw/osmo-hnbgw
 # To prevent 'make -j N' from running these tests in parallel, call a script with a linear for-loop.
 .PHONY: config-tests
 config-tests:
-	$(srcdir)/config/run_tests.sh "$(top_builddir)/src/osmo-hnbgw/osmo-hnbgw" "$(srcdir)/config/" $U
+	$(QEMU_EXT_TEST_WRAPPER) $(srcdir)/config/run_tests.sh \
+		"$(top_builddir)/src/osmo-hnbgw/osmo-hnbgw" \
+		"$(srcdir)/config/" \
+		$U
 
 # Run a specific test with: 'make ctrl-test CTRL_TEST=osmo-hnbgw.ctrl'
 CTRL_TEST ?= *.ctrl
@@ -81,7 +88,7 @@ CTRL_TEST ?= *.ctrl
 # pass -u to ctrl_script_runner.py by doing:
 #   make ctrl-test U=-u
 ctrl-test: $(top_builddir)/src/osmo-hnbgw/osmo-hnbgw
-	osmo_verify_transcript_ctrl.py -v \
+	$(QEMU_EXT_TEST_WRAPPER) osmo_verify_transcript_ctrl.py -v \
 		-p 4262 \
 		-r "$(top_builddir)/src/osmo-hnbgw/osmo-hnbgw -c $(top_srcdir)/doc/examples/osmo-hnbgw/osmo-hnbgw-cnpool.cfg" \
 		$(U) $(srcdir)/$(CTRL_TEST)

tests/qemu/check-depends.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+RET=0
+
+require_program() {
+	if [ -z "$(command -v "$1")" ]; then
+		RET=1
+		echo "ERROR: missing program: $1"
+	fi
+}
+
+require_program busybox
+require_program cpio
+require_program find
+require_program gzip
+require_program lddtree
+require_program qemu-system-x86_64
+
+exit "$RET"

tests/qemu/ext_test_wrapper.sh

@@ -0,0 +1,75 @@
+#!/bin/sh -ex
+DIR="$(cd "$(dirname "$0")" && pwd)"
+
+cat << EOF > "$DIR"/_testcmd.sh
+#!/bin/sh -ex
+cd "$PWD"
+
+EOF
+chmod +x "$DIR"/_testcmd.sh
+
+for i in "$@"; do
+	echo -n "\"$i\" " >> "$DIR"/_testcmd.sh
+done
+echo >> "$DIR"/_testcmd.sh
+
+cat << EOF > "$DIR"/_init.sh
+#!/usr/bin/busybox sh
+echo "Running _init.sh"
+set -x
+
+/usr/bin/busybox --install -s
+hostname qemu
+
+for i in \$(cat /modules); do
+	modprobe "\$i"
+done
+
+ip link set lo up
+ip a
+
+mkdir /mnt/
+mount \
+	-t 9p \
+	-o trans=virtio \
+	fs0 \
+	/mnt/
+
+# osmotestconfig.py tries to write to builddir
+cp -r /mnt/$BUILDDIR /tmp/builddir
+mount --bind /tmp/builddir /mnt/$BUILDDIR
+
+if chroot /mnt "$DIR"/_testcmd.sh; then
+	echo "QEMU_TEST_SUCCESSFUL"
+fi
+
+poweroff -f
+EOF
+chmod +x "$DIR"/_init.sh
+
+if ! [ -e "$DIR"/_linux ]; then
+	cp /boot/vmlinuz "$DIR"/_linux
+fi
+
+$DIR/initrd-build.sh
+
+KERNEL_CMDLINE="root=/dev/ram0 console=ttyS0 panic=-1 init=/init"
+
+qemu-system-x86_64 \
+	$MACHINE_ARG \
+	-smp 1 \
+	-m 512M \
+	-no-user-config -nodefaults -display none \
+	-gdb unix:"$DIR"/_gdb.pipe,server=on,wait=off \
+	-no-reboot \
+	-kernel "$DIR"/_linux \
+	-initrd "$DIR"/_initrd.gz \
+	-append "${KERNEL_CMDLINE}" \
+	-serial stdio \
+	-chardev socket,id=charserial1,path="$DIR"/_gdb-serial.pipe,server=on,wait=off \
+	-device isa-serial,chardev=charserial1,id=serial1 \
+	-fsdev local,security_model=passthrough,id=fsdev-fs0,multidevs=remap,path=/ \
+	-device virtio-9p-pci,id=fs0,fsdev=fsdev-fs0,mount_tag=fs0 \
+	2>&1 | tee "$DIR/_output"
+
+grep -q QEMU_TEST_SUCCESSFUL "$DIR/_output"

tests/qemu/initrd-build.sh

@@ -0,0 +1,114 @@
+#!/bin/sh -e
+DIR="$(cd "$(dirname "$0")" && pwd)"
+DIR_INITRD="$DIR/_initrd"
+DIR_MODULES="$(find /lib/modules/* -type d -prune | sort -r | head -n 1)"
+
+# Add one or more files to the initramfs, with parent directories.
+# usr-merge: resolve symlinks for /lib -> /usr/lib etc. so "cp --parents" does
+# not fail with "cp: cannot make directory '/tmp/initrd/lib': File exists"
+# $@: path to files
+initrd_add_file() {
+	local i
+
+	for i in "$@"; do
+		case "$i" in
+		/bin/*|/sbin/*|/lib/*|/lib64/*)
+			cp -a --parents "$i" "$DIR_INITRD"/usr
+			;;
+		*)
+			cp -a --parents "$i" "$DIR_INITRD"
+			;;
+		esac
+	done
+}
+
+# Add kernel module files with dependencies
+# $@: kernel module names
+initrd_add_mod() {
+	local i
+	local kernel="$(basename "$DIR_MODULES")"
+	local files="$(modprobe \
+		-a \
+		--dry-run \
+		--show-depends \
+		--set-version="$kernel" \
+		"$@" \
+		| grep -v "^builtin" \
+		| sort -u \
+		| cut -d ' ' -f 2)"
+
+	initrd_add_file $files
+
+	# Save the list of modules
+	for i in $@; do
+		echo "$i" >> "$DIR_INITRD"/modules
+	done
+}
+
+# Add binaries with depending libraries
+# $@: paths to binaries
+initrd_add_bin() {
+	local bin
+	local bin_path
+	local file
+
+	for bin in "$@"; do
+		local bin_path="$(which "$bin")"
+		if [ -z "$bin_path" ]; then
+			echo "ERROR: file not found: $bin"
+			exit 1
+		fi
+
+		lddtree_out="$(lddtree -l "$bin_path")"
+		if [ -z "$lddtree_out" ]; then
+			echo "ERROR: lddtree failed on '$bin_path'"
+			exit 1
+		fi
+
+		for file in $lddtree_out; do
+			initrd_add_file "$file"
+
+			# Copy resolved symlink
+			if [ -L "$file" ]; then
+				initrd_add_file "$(realpath "$file")"
+			fi
+		done
+	done
+}
+
+rm -rf "$DIR_INITRD"
+mkdir -p "$DIR_INITRD"
+cd "$DIR_INITRD"
+
+for dir in bin sbin lib lib64; do
+	ln -s usr/"$dir" "$dir"
+done
+
+mkdir -p \
+	dev/net \
+	proc \
+	run \
+	sys \
+	tmp \
+	usr/bin \
+	usr/sbin
+
+initrd_add_bin \
+	busybox
+
+initrd_add_mod \
+	9p \
+	9pnet \
+	9pnet_virtio \
+	nf_tables \
+	nfnetlink \
+	sctp
+
+initrd_add_file \
+	"$DIR_MODULES"/modules.dep
+
+cp "$DIR"/_init.sh init
+
+find . -print0 \
+	| cpio --quiet -o -0 -H newc \
+	| gzip -1 > "$DIR"/_initrd.gz