demo wip

Change-Id: If7b7b28f51f4b408333176825ba1aa1889cef505

README.md

@@ -12,9 +12,9 @@ GIT Repository
 
 You can clone from the official osmo-upf.git repository using
 
-	git clone git://git.osmocom.org/osmo-upf.git
+	git clone https://gitea.osmocom.org/cellular-infrastructure/osmo-upf
 
-There is a cgit interface at https://git.osmocom.org/osmo-upf/
+There is a web interface at https://gitea.osmocom.org/cellular-infrastructure/osmo-upf.
 
 To submit patches, see "Contributing" below.
 

configure.ac

@@ -123,7 +123,7 @@ if test "x$enable_ext_tests" = "xyes" ; then
 	fi
 	AC_CHECK_PROG(OSMOTESTEXT_CHECK,osmotestvty.py,yes)
 	 if test "x$OSMOTESTEXT_CHECK" != "xyes" ; then
-		AC_MSG_ERROR([Please install git://osmocom.org/python/osmo-python-tests to run the VTY/CTRL tests.])
+		AC_MSG_ERROR([Please install https://gitea.osmocom.org/cellular-infrastructure/osmo-python-tests to run the VTY/CTRL tests.])
 	fi
 fi
 AC_MSG_CHECKING([whether to enable VTY/CTRL tests])
@@ -206,7 +206,7 @@ AC_OUTPUT(
     doc/Makefile
     doc/examples/Makefile
     doc/manuals/Makefile
-    doc/charts/Makefile
     contrib/Makefile
     contrib/systemd/Makefile
+    contrib/demo/Makefile
     Makefile)

contrib/demo/Makefile.am

@@ -0,0 +1,16 @@
+msc: \
+	$(NULL)
+
+dot: \
+	$(builddir)/demo_overview.png \
+	$(NULL)
+
+$(builddir)/%.png: $(srcdir)/%.msc
+	mscgen -T png -o $@ $<
+
+$(builddir)/%.png: $(srcdir)/%.dot
+	dot -Tpng $< > $@
+
+.PHONY: poll
+poll:
+	while true; do $(MAKE) msc dot; sleep 1; done

contrib/demo/demo_overview.dot

@@ -0,0 +1,36 @@
+digraph G {
+rankdir=TB
+labelloc=t; label="OsmoUPF demo Overview"
+
+subgraph cluster_left {
+	label="netns 'left'";style=dotted;rankdir=TB
+	left_eth [label="veth3\n10.3.0.1/32"]
+	left_upf [label="osmo-upf\ntunend\nPFCP: 127.3.0.1:8805"]
+	left_tunend [label="GTP module tunend\napn_left 10.3.1.1\nUE: 192.168.3.42"]
+	left_tool [label="osmo-pfcp-tool\nPFCP: 127.3.0.2:8805"]
+	left_tool -> left_upf [label="PFCP"]
+}
+
+subgraph cluster_mid {
+	label="netns 'mid'";style=dotted
+	mid_eth23 [label="veth23\n10.2.3.1/32"]
+	mid_eth21 [label="veth21\n10.2.1.1/32"]
+	mid_upf [label="osmo-upf\ntunmap\nPFCP: 127.2.0.1:8805\nGTP: nft rulesets"]
+	mid_nft [label="nft tunmap"]
+	mid_tool [label="osmo-pfcp-tool\nPFCP: 127.2.0.2:8805"]
+	mid_tool -> mid_upf [label="PFCP"]
+}
+
+subgraph cluster_right {
+	label="netns 'right'";style=dotted
+	right_eth [label="veth1\n10.1.0.1/32"]
+	right_upf [label="osmo-upf\ntunend\nPFCP: 127.1.0.1:8805"]
+	right_tunend [label="GTP module tunend\napn_right 10.1.1.1\nUE: 192.168.1.42"]
+	right_tool [label="osmo-pfcp-tool\nPFCP: 127.1.0.2:8805"]
+	right_tool -> right_upf [label="PFCP"]
+}
+
+left_tunend -> mid_nft [label="GTP-U"]
+mid_nft -> right_tunend [label="GTP-U"]
+
+}

contrib/osmo-pfcp-tool-scripts/encaps_plus_tunmap.vty

@@ -1,30 +0,0 @@
-# ACCESS                HOP                           CORE
-#                       session 23 = tunmap           session 42 = encaps/decaps
-# GTP 127.0.0.13        127.0.0.12                    127.0.0.11
-# TEID l:23 r:123 <---> r:23 l:123 | l:142 r:42 <---> r:142 l:42 | 192.168.100.42
-#
-# Run two UPF, one listening on / sending from 127.0.0.11, the other on 127.0.0.12.
-# (Each has to match on the sender address of incoming GTP packets.)
-
-timer pfcp x23 0
-
-pfcp-peer 127.0.0.11
- tx assoc-setup-req
- sleep 1
- session endecaps 42
-  ue ip 192.168.100.42
-  gtp access ip 127.0.0.12
-  gtp access teid local 42 remote 142
-  tx session-est-req
- sleep 1
-
-pfcp-peer 127.0.0.12
- tx assoc-setup-req
- sleep 1
- session tunmap 23
-  gtp core ip 127.0.0.11
-  gtp core teid local 142 remote 42
-  gtp access ip 127.0.0.13
-  gtp access teid local 123 remote 23
-  tx session-est-req
- sleep 1

contrib/osmo-pfcp-tool-scripts/netns_add.sh

@@ -0,0 +1,40 @@
+#
+#  ns1         <-> nsr1                      <-> nsr2                      <-> ns2
+#  10.141.10.2 <-> 10.141.10.1 192.168.10.10 <-> 192.168.10.11 10.141.11.1 <-> 10.141.11.2
+#
+ip netns add ns1
+ip netns add nsr1
+ip netns add nsr2
+ip netns add ns2
+
+ip link add veth0 netns nsr1 type veth peer name veth0 netns ns1
+ip link add veth1 netns nsr1 type veth peer name veth1 netns nsr2
+ip link add veth0 netns nsr2 type veth peer name veth0 netns ns2
+
+ip -net nsr1 addr add 10.141.10.1/24 dev veth0
+ip -net nsr1 addr add 192.168.10.10/24 dev veth1
+ip -net nsr1 link set up dev veth0
+ip -net nsr1 link set up dev veth1
+ip -net nsr1 ro add 10.141.11.0/24 via 192.168.10.11
+
+ip -net nsr2 addr add 10.141.11.1/24 dev veth0
+ip -net nsr2 addr add 10.141.20.1/32 dev veth0
+ip -net nsr2 addr add 192.168.10.11/24 dev veth1
+ip -net nsr2 link set up dev veth0
+ip -net nsr2 link set up dev veth1
+ip -net nsr2 ro add 10.141.10.0/24 via 192.168.10.10
+
+ip netns exec nsr1 sysctl net.ipv4.ip_forward=1 > /dev/null
+ip netns exec nsr2 sysctl net.ipv4.ip_forward=1 > /dev/null
+
+ip -net ns1 addr add 10.141.10.2/24 dev veth0
+ip -net ns1 link set up dev veth0
+ip -net ns1 ro add default via 10.141.10.1
+
+ip -net ns2 addr add 10.141.11.2/24 dev veth0
+ip -net ns2 addr add 10.141.20.2/32 dev veth0
+ip -net ns2 link set up dev veth0
+ip -net ns2 ro add default via 10.141.11.1
+
+# ip netns exec ns2 iperf3 -s
+# ip netns exec ns1 iperf3 -c 192.168.10.2 -n 100G

contrib/osmo-pfcp-tool-scripts/netns_enter.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+ns="$1"
+sudo ip netns exec $ns bash -c "su $USER"

contrib/osmo-pfcp-tool-scripts/osmo-pfcp-tool-nsr1.cfg

@@ -0,0 +1,5 @@
+log stderr
+ logging level set-all info
+
+local-addr 10.141.20.1
+listen

contrib/osmo-pfcp-tool-scripts/osmo-pfcp-tool-nsr2.cfg

@@ -0,0 +1,5 @@
+log stderr
+ logging level set-all info
+
+local-addr 10.141.20.2
+listen

contrib/osmo-pfcp-tool-scripts/osmo-upf-11.cfg

@@ -21,7 +21,7 @@ ctrl
 timer pfcp x24 5000
 pfcp
  local-addr 127.0.0.11
-gtp
+tunend
  dev create apn11 127.0.0.11
-nft
+tunmap
  table-name osmo-upf-11

contrib/osmo-pfcp-tool-scripts/osmo-upf-12.cfg

@@ -21,7 +21,5 @@ ctrl
 timer pfcp x24 5000
 pfcp
  local-addr 127.0.0.12
-gtp
- dev create apn12 127.0.0.12
-nft
+tunmap
  table-name osmo-upf-12

contrib/osmo-pfcp-tool-scripts/osmo-upf-13.cfg

@@ -0,0 +1,25 @@
+log stderr
+ logging filter all 1
+ logging color 1
+ logging print level 1
+ logging print category 1
+ logging print category-hex 0
+ logging print file basename last
+ logging print extended-timestamp 1
+ logging level set-all notice
+ logging level set-all info
+ logging level session debug
+ logging level nft debug
+ logging level gtp debug
+#logging level set-all debug
+
+line vty
+ bind 127.0.0.13
+ctrl
+ bind 127.0.0.13
+
+timer pfcp x24 5000
+pfcp
+ local-addr 127.0.0.13
+tunmap
+ table-name osmo-upf-13

contrib/osmo-pfcp-tool-scripts/osmo-upf-ns2.cfg

@@ -0,0 +1,22 @@
+log stderr
+ logging filter all 1
+ logging color 1
+ logging print level 1
+ logging print category 1
+ logging print category-hex 0
+ logging print file basename last
+ logging print extended-timestamp 1
+ logging level set-all notice
+ logging level set-all info
+ logging level session debug
+ logging level nft debug
+ logging level gtp debug
+#logging level set-all debug
+
+timer pfcp x24 5000
+pfcp
+ local-addr 10.141.11.2
+tunend
+ dev create apn_ns2 10.141.11.2
+tunmap
+ table-name osmo-upf-ns2

contrib/osmo-pfcp-tool-scripts/osmo-upf-nsr1.cfg

@@ -0,0 +1,22 @@
+log stderr
+ logging filter all 1
+ logging color 1
+ logging print level 1
+ logging print category 1
+ logging print category-hex 0
+ logging print file basename last
+ logging print extended-timestamp 1
+ logging level set-all notice
+ logging level set-all info
+ logging level session debug
+ logging level nft debug
+ logging level gtp debug
+#logging level set-all debug
+
+timer pfcp x24 5000
+pfcp
+ local-addr 10.141.10.1
+tunmap
+ table-name osmo-upf-nsr1
+tunend
+ dev create apn_nsr1 192.168.10.10

contrib/osmo-pfcp-tool-scripts/osmo-upf-nsr2.cfg

@@ -0,0 +1,25 @@
+log stderr
+ logging filter all 1
+ logging color 1
+ logging print level 1
+ logging print category 1
+ logging print category-hex 0
+ logging print file basename last
+ logging print extended-timestamp 1
+ logging level set-all notice
+ logging level set-all info
+ logging level session debug
+ logging level nft debug
+ logging level gtp debug
+#logging level set-all debug
+
+line vty
+ bind 10.141.11.1
+ctrl
+ bind 10.141.11.1
+
+timer pfcp x24 5000
+pfcp
+ local-addr 10.141.11.1
+tunmap
+ table-name osmo-upf-nsr2

contrib/osmo-pfcp-tool-scripts/pfcp_ns2.vty

@@ -0,0 +1,35 @@
+# ACCESS                  UPF tunmap                        UPF tunend (CORE)
+# nsr1                    nsr2                              ns2
+#                         session 23 = tunmap               session 42 = encaps/decaps
+# GTP 192.168.10.10       192.168.10.11 | 10.141.11.1       10.141.11.2 |
+# TEID 4            <---> 3             | 2           <---> 1           | 192.168.100.42
+#
+# with netns setup:
+#  ns1         <-> nsr1                      <-> nsr2                      <-> ns2
+#  10.141.10.2 <-> 10.141.10.1 192.168.10.10 <-> 192.168.10.11 10.141.11.1 <-> 10.141.11.2
+
+timer pfcp x23 0
+
+#ns2
+pfcp-peer 10.141.11.2
+ tx assoc-setup-req
+ sleep 1
+ session tunend 42
+  ue ip 192.168.100.42
+  gtp access local f-teid 10.141.11.2 1
+  gtp access remote f-teid 10.141.11.1 2
+  tx session-est-req
+ sleep 1
+
+#nsr2
+pfcp-peer 10.141.11.1
+ tx assoc-setup-req
+ sleep 1
+ session tunmap 23
+  gtp core remote f-teid 10.141.11.2 1
+  gtp core local f-teid 10.141.11.1 2
+  gtp access local f-teid 192.168.10.11 3
+  gtp access remote f-teid 192.168.10.10 4
+  tx session-est-req
+ sleep 1
+

contrib/osmo-pfcp-tool-scripts/pfcp_ns2_only.vty

@@ -0,0 +1,23 @@
+# ACCESS                  UPF tunmap                        UPF tunend (CORE)
+# nsr1                    nsr2                              ns2
+#                         session 23 = tunmap               session 42 = encaps/decaps
+# GTP 192.168.10.10       192.168.10.11 | 10.141.11.1       10.141.11.2 |
+# TEID 4            <---> 3             | 2           <---> 1           | 192.168.100.42
+#
+# with netns setup:
+#  ns1         <-> nsr1                      <-> nsr2                      <-> ns2
+#  10.141.10.2 <-> 10.141.10.1 192.168.10.10 <-> 192.168.10.11 10.141.11.1 <-> 10.141.11.2
+
+timer pfcp x23 0
+
+#ns2
+pfcp-peer 10.141.11.2
+ tx assoc-setup-req
+ sleep 1
+ session tunend 42
+  ue ip 192.168.100.42
+  gtp access local f-teid 10.141.11.2 1
+  gtp access remote f-teid 10.141.11.1 2
+  tx session-est-req
+ sleep 1
+

contrib/osmo-pfcp-tool-scripts/pfcp_nsr1.vty

@@ -0,0 +1,22 @@
+# ACCESS                  UPF tunmap                        UPF tunend (CORE)
+# nsr1                    nsr2                              ns2
+#                         session 23 = tunmap               session 42 = encaps/decaps
+# GTP 192.168.10.10       192.168.10.11 | 10.141.11.1       10.141.11.2 |
+# TEID 4            <---> 3             | 2           <---> 1           | 192.168.100.42
+#
+# with netns setup:
+#  ns1         <-> nsr1                      <-> nsr2                      <-> ns2
+#  10.141.10.2 <-> 10.141.10.1 192.168.10.10 <-> 192.168.10.11 10.141.11.1 <-> 10.141.11.2
+
+timer pfcp x23 0
+
+#nsr1
+pfcp-peer 10.141.10.1
+ tx assoc-setup-req
+ sleep 1
+ session tunend 17
+  ue ip 192.168.101.17
+  gtp access remote f-teid 192.168.10.11 3
+  gtp access local f-teid 192.168.10.10 4
+  tx session-est-req
+ sleep 1

contrib/osmo-pfcp-tool-scripts/pfcp_nsr2.vty

@@ -0,0 +1,24 @@
+# ACCESS                  UPF tunmap                        UPF tunend (CORE)
+# nsr1                    nsr2                              ns2
+#                         session 23 = tunmap               session 42 = encaps/decaps
+# GTP 192.168.10.10       192.168.10.11 | 10.141.11.1       10.141.11.2 |
+# TEID 4            <---> 3             | 2           <---> 1           | 192.168.100.42
+#
+# with netns setup:
+#  ns1         <-> nsr1                      <-> nsr2                      <-> ns2
+#  10.141.10.2 <-> 10.141.10.1 192.168.10.10 <-> 192.168.10.11 10.141.11.1 <-> 10.141.11.2
+
+timer pfcp x23 0
+
+#nsr2
+pfcp-peer 10.141.11.1
+ tx assoc-setup-req
+ sleep 1
+ session tunmap 23
+  gtp core remote f-teid 10.141.11.2 1
+  gtp core local f-teid 10.141.11.1 2
+  gtp access local f-teid 192.168.10.11 3
+  gtp access remote f-teid 192.168.10.10 4
+  tx session-est-req
+ sleep 1
+

contrib/osmo-pfcp-tool-scripts/reproduce_nft_missing_elements.txt

@@ -0,0 +1,33 @@
+# The commit that breaks/unbreaks is 'drop flags owner'.
+# revert it to reproduce missing 'element'
+git revert 98d73e4c5b489cf8c34238417f9bcfa6d8cbbe9a
+
+# rebuild osmo-upf, then
+sudo setcap cap_net_admin+pe $(which osmo-upf)
+
+# enter testing arena
+cd osmo-upf/contrib/osmo-pfcp-tool-scripts/
+sudo ./netns_add.sh
+
+# one terminal for osmo-upf
+sudo ./netns_enter.sh nsr2
+osmo-upf -c osmo-upf-nsr2.cfg
+
+# keep osmo-upf running here; at any time to restart:
+^C
+sudo nft delete table inet osmo-upf-nsr2
+osmo-upf -c osmo-upf-nsr2.cfg
+
+# other terminal also in nsr2:
+sudo ./netns_enter.sh nsr2
+
+# uses osmo-pfcp-tool.cfg, need another IP address to not clash with osmo-upf's PFCP port
+# so notice that there is 127.0.0.2 in osmo-pfcp-tool.cfg and we need loopback:
+sudo ip link set lo up
+
+# send PFCP instructions for a tunmap to osmo-upf (redo after each restart of osmo-upf)
+osmo-pfcp-tool pfcp-nsr2.vty
+
+sudo nft list ruleset
+# should show no 'elements'.
+# ...now you can drop the 'git revert' and rebuild, then you should see 'elements' present.

contrib/osmo-pfcp-tool-scripts/session_est_without_assoc.vty

@@ -1,4 +1,6 @@
 timer pfcp x23 0
 pfcp-peer 127.0.0.1
- session endecaps
+ session tunend
+  ue ip 127.127.127.127
+  gtp access remote f-teid 127.0.0.127 127
   tx session-est-req

contrib/osmo-pfcp-tool-scripts/session_mod.vty

@@ -3,12 +3,14 @@ pfcp-peer 127.0.0.1
  tx assoc-setup-req
  sleep 1
  session
+  ue ip 127.127.127.127
+  gtp access remote f-teid 127.0.0.127 127
   tx session-est-req drop
   sleep 3
-  tx session-mod-req forw
+  tx session-mod-req far forw
   sleep 5
-  tx session-mod-req drop
+  tx session-mod-req far drop
   sleep 3
-  tx session-mod-req forw
+  tx session-mod-req far forw
   sleep 3
   tx session-del-req

contrib/osmo-pfcp-tool-scripts/tunend_plus_tunmap.vty

@@ -0,0 +1,46 @@
+# ACCESS                  UPF tunmap                        UPF tunend (CORE)
+# nsr1                    nsr2                              ns2
+#                         session 23 = tunmap               session 42 = encaps/decaps
+# GTP 192.168.10.10       192.168.10.11 | 10.141.11.1       10.141.11.2 |
+# TEID 4            <---> 3             | 2           <---> 1           | 192.168.100.42
+#
+# with netns setup:
+#  ns1         <-> nsr1                      <-> nsr2                      <-> ns2
+#  10.141.10.2 <-> 10.141.10.1 192.168.10.10 <-> 192.168.10.11 10.141.11.1 <-> 10.141.11.2
+
+timer pfcp x23 0
+
+#ns2
+pfcp-peer 10.141.11.2
+ tx assoc-setup-req
+ sleep 1
+ session tunend 42
+  ue ip 192.168.100.42
+  gtp access local f-teid 10.141.11.2 1
+  gtp access remote f-teid 10.141.11.1 2
+  tx session-est-req
+ sleep 1
+
+#nsr2
+pfcp-peer 10.141.11.1
+ tx assoc-setup-req
+ sleep 1
+ session tunmap 23
+  gtp core remote f-teid 10.141.11.2 1
+  gtp core local f-teid 10.141.11.1 2
+  gtp access local f-teid 192.168.10.11 3
+  gtp access remote f-teid 192.168.10.10 4
+  tx session-est-req
+ sleep 1
+
+#nsr1
+pfcp-peer 10.141.10.1
+ tx assoc-setup-req
+ sleep 1
+ session tunmap 17
+  gtp core remote f-teid 192.168.10.11 3
+  gtp core local f-teid 192.168.10.10 4
+  gtp access local f-teid 10.141.10.1 5
+  gtp access remote f-teid 10.141.10.2 6
+  tx session-est-req
+ sleep 1

contrib/osmo-pfcp-tool-scripts/tunend_session_est.vty

@@ -2,7 +2,10 @@ timer pfcp x23 0
 pfcp-peer 127.0.0.1
  tx assoc-setup-req
  sleep 1
- session endecaps
+ session tunend
+  ue ip 127.127.127.127
+  gtp access local f-teid choose
+  gtp access remote f-teid 127.0.0.12 142
   tx session-est-req forw
   sleep 5
   tx session-del-req

contrib/osmo-pfcp-tool-scripts/tunmap_session_est.vty

@@ -3,6 +3,10 @@ pfcp-peer 127.0.0.1
  tx assoc-setup-req
  sleep 1
  session tunmap
+  gtp core remote f-teid 127.0.0.11 42
+  gtp core local f-teid choose
+  gtp access local f-teid choose
+  gtp access remote f-teid 127.0.0.13 23
   tx session-est-req
   sleep 5
   tx session-del-req

contrib/systemd/osmo-upf.service

@@ -3,6 +3,8 @@ Description=Osmocom User Plane Function (UPF)
 
 [Service]
 Type=simple
+StateDirectory=osmocom
+WorkingDirectory=%S/osmocom
 Restart=always
 ExecStart=/usr/bin/osmo-upf -c /etc/osmocom/osmo-upf.cfg
 RestartSec=2

debian/control

@@ -18,8 +18,8 @@ Build-Depends: debhelper (>=9),
                libosmo-pfcp-dev (>= 0.1.0),
                osmo-gsm-manuals-dev (>= 1.2.0)
 Standards-Version: 3.9.8
-Vcs-Git: git://git.osmocom.org/osmo-upf.git
-Vcs-Browser: https://git.osmocom.org/osmo-upf/
+Vcs-Git: https://gitea.osmocom.org/cellular-infrastructure/osmo-upf
+Vcs-Browser: https://gitea.osmocom.org/cellular-infrastructure/osmo-upf
 Homepage: https://projects.osmocom.org/projects/osmo-upf
 
 Package: osmo-upf

debian/copyright

@@ -1,6 +1,6 @@
 Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 Upstream-Name: osmo-upf
-Source: git://git.osmocom.org/osmo-upf
+Source: https://gitea.osmocom.org/cellular-infrastructure/osmo-upf
 
 Files:     *
 Copyright: 2021-2022 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>

debian/osmo-upf.install

@@ -1,4 +1,5 @@
 etc/osmocom/osmo-upf.cfg
 lib/systemd/system/osmo-upf.service
+usr/bin/osmo-pfcp-tool
 usr/bin/osmo-upf
 usr/share/doc/osmo-upf/examples/osmo-upf/osmo-upf.cfg usr/share/doc/osmo-upf/examples

doc/Makefile.am

@@ -1,5 +1,4 @@
 SUBDIRS = \
 	examples \
 	manuals \
-	charts \
 	$(NULL)

doc/charts/Makefile.am

@@ -1,24 +0,0 @@
-msc: \
-	$(builddir)/pfcp_msgs.png \
-	$(builddir)/pfcp_msgs_gtp.png \
-	$(NULL)
-
-dot: \
-	$(builddir)/pfcp_overview.png \
-	$(builddir)/pfcp_cp_peer_fsm.png \
-	$(builddir)/pfcp_up_peer_fsm.png \
-	$(builddir)/pfcp_heartbeat_fsm.png \
-	$(builddir)/pfcp_cp_session_fsm.png \
-	$(builddir)/pfcp_up_session_fsm.png \
-	$(builddir)/pfcp_and_gtp.png \
-	$(NULL)
-
-$(builddir)/%.png: $(srcdir)/%.msc
-	mscgen -T png -o $@ $<
-
-$(builddir)/%.png: $(srcdir)/%.dot
-	dot -Tpng $< > $@
-
-.PHONY: poll
-poll:
-	while true; do $(MAKE) msc dot; sleep 1; done

doc/charts/pfcp_and_gtp.dot

@@ -1,20 +0,0 @@
-digraph G {
-rankdir=LR
-labelloc=t; label="PFCP and GTP"
-
-SGSN [label="SGSN\n123.44.0.9"]
-SGWC [label="SGW-C\n123.44.05"]
-subgraph cluster_UPF {
- label="OsmoUPF";
- SGWU [label="SGW-U\n123.44.0.6"];
- GTPk [label="kernel GTP\n123.44.0.6"]
-}
-
-SGSN -> SGWC [label="S4\nGTPv2-C"]
-SGWC -> SGWU [label="Sxa\nPFCP\nSession Establishment:\n"]
-SGSN -> GTPk [label="S4\nGTPv1-U",dir=both]
-
-MS [label="MS\n192.168.104.176"]
-MS -> SGSN [dir=both]
-
-}

doc/charts/pfcp_cp_peer_fsm.dot

@@ -1,39 +0,0 @@
-digraph G {
-rankdir=TB
-labelloc=t; label="PFCP CP peer FSM\nControl Plane side, managing association with remote UP peer"
-
-cp [label="CP function",shape="box"]
-
-cp -> WAIT_ASSOC_SETUP_RESP [label="cp_peer_associate()"]
-
-txrx [label="PFCP socket",shape="box"]
-WAIT_ASSOC_SETUP_RESP -> txrx [label="tx_assoc_setup_req()",style=dotted]
-txrx -> WAIT_ASSOC_SETUP_RESP [label="EV_RX_ASSOC_SETUP_RESP",style=dotted]
-WAIT_ASSOC_SETUP_RESP -> ASSOCIATED [label="Assoc Setup Resp"]
-
-WAIT_ASSOC_SETUP_RESP -> WAIT_ASSOC_SETUP_RESP [label="retry"]
-
-heartbeat [label="PFCP heartbeat FSM",shape=box3d]
-ASSOCIATED -> heartbeat [label="alloc()",style=dotted]
-heartbeat -> ASSOCIATED [label="EV_HEARTBEAT_FAILURE",style=dotted]
-
-txrx2 [label="PFCP socket",shape="box"]
-txrx2 -> ASSOCIATED [label="EV_RX_ASSOC_UPDATE_REQ\n3GPP TS 29.244 6.2.7.3.1",style=dotted]
-GRACEFUL_RELEASE -> txrx2 [label="tx_assoc_update_resp()",style=dotted]
-
-cp_session [label="PFCP CP session FSM",shape=box3d]
-cp -> ASSOCIATED [label="cp_peer_session_create()",style=dotted]
-ASSOCIATED -> cp_session [label="cp_session_create()",style=dotted]
-cp -> cp_session [style=invisible,arrowhead=none]
-
-ASSOCIATED -> GRACEFUL_RELEASE [label="Association Update\nindicating graceful release"]
-
-cp -> ASSOCIATED [label="cp_peer_release()",style=dotted]
-ASSOCIATED -> term [label="cp_peer_release()\nHeartbeat failure"]
-
-ASSOCIATED -> WAIT_ASSOC_SETUP_RESP [label="Heartbeat failure"]
-
-GRACEFUL_RELEASE -> term
-term [shape="octagon"]
-
-}

doc/charts/pfcp_cp_session_fsm.dot

@@ -1,28 +0,0 @@
-digraph G {
-rankdir=TB
-labelloc=t; label="PFCP CP session FSM"
-
-cp [label="CP function",shape=box]
-cp -> WAIT_ESTABLISHMENT_RESP [label="cp_session_create(cp_peer)\niff cp_peer in state ASSOCIATED"]
-
-txrx [label="PFCP socket",shape=box]
-
-WAIT_ESTABLISHMENT_RESP -> txrx [label="tx_session_est_req()",style=dotted]
-txrx -> WAIT_ESTABLISHMENT_RESP [label="EV_RX_SESSION_EST_RESP",style=dotted]
-
-WAIT_ESTABLISHMENT_RESP -> ESTABLISHED [label="Est Resp"]
-
-cp -> ESTABLISHED [label="cp_session_modify()",style=dotted]
-ESTABLISHED -> WAIT_MODIFICATION_RESP [label="cp_session_modify()"]
-WAIT_MODIFICATION_RESP -> txrx [label="tx_session_mod_req()",style=dotted]
-txrx -> WAIT_MODIFICATION_RESP [label="EV_RX_SESSION_MOD_RESP",style=dotted,constraint=false]
-WAIT_MODIFICATION_RESP -> ESTABLISHED [label="Mod Resp"]
-
-cp -> ESTABLISHED [label="cp_session_delete()",style=dotted]
-ESTABLISHED -> WAIT_DELETION_RESP [label="cp_session_delete()"]
-WAIT_DELETION_RESP -> txrx [label="tx_session_del_req()",style=dotted]
-txrx -> WAIT_DELETION_RESP [label="EV_RX_SESSION_DEL_RESP",style=dotted,constraint=false]
-WAIT_DELETION_RESP -> term
-term [shape="octagon"]
-
-}

doc/charts/pfcp_heartbeat_fsm.dot

@@ -1,21 +0,0 @@
-digraph G {
-rankdir=TB
-labelloc=t; label="PFCP heartbeat FSM"
-
-peer [label="PFCP CP/UP peer FSM",shape=box3d]
-txrx [label="PFCP socket",shape=box]
-
-peer -> IDLE [label="alloc()"]
-IDLE -> WAIT_HEARTBEAT_RESP -> IDLE
-WAIT_HEARTBEAT_RESP -> term
-term [shape="octagon"]
-
-WAIT_HEARTBEAT_RESP -> txrx [label="tx_heartbeat_req()",style=dotted]
-txrx -> WAIT_HEARTBEAT_RESP [label="HEARTBEAT_EV_RX_RESP",style=dotted]
-
-term -> peer [label="PEER_EV_HEARTBEAT_FAILURE",style=dotted]
-
-txrx2 [label="PFCP socket",shape=box]
-txrx2 -> txrx2 [label="rx Heartbeat Req\ntx Heartbeat Resp",style=dotted]
-
-}

doc/charts/pfcp_overview.dot

@@ -1,23 +0,0 @@
-digraph G {
-rankdir=TB
-labelloc=t; label="PFCP Overview\n3GPP TS 29.244 3.1, 5.8.1"
-
-subgraph cluster_N1_CP {
-	label="Node: Control Plane function";style=dotted
-	N1_E_CP [label="CP Entity"]
-}
-
-subgraph cluster_N2_UP {
-	label="Node: User Plane function\nNode ID: my-userplane.com\n(FQDN may provide multiple PFCP Entities)";style=dotted
-	N2_E_UP [label="UP Entity\n8.7.6.1"]
-	N2_E_UP2 [label="UP Entity\n8.7.6.2"]
-}
-
-subgraph cluster_N3_UP {
-	label="Node: User Plane function\nNode ID: 1.2.3.4\n(IP address means only one PFCP Entity)";style=dotted
-	N3_E_UP [label="UP Entity\n1.2.3.4\n(osmo-upf)"]
-}
-
-N1_E_CP -> N3_E_UP [label="PFCP Request"]
-N1_E_CP -> N2_E_UP
-}

doc/charts/pfcp_up_peer_fsm.dot

@@ -1,27 +0,0 @@
-digraph G {
-rankdir=TB
-labelloc=t; label="PFCP UP peer FSM\nUser Plane side, managing association with remote CP peer"
-
-txrx [label="PFCP socket",shape="box"]
-
-txrx -> NOT_ASSOCIATED [label="rx PFCP msg from\nnew remote IP"]
-txrx -> NOT_ASSOCIATED [label="EV_RX_ASSOC_SETUP_REQ",style=dotted]
-
-NOT_ASSOCIATED -> ASSOCIATED [label="Assoc Setup Req",shape="box"]
-
-heartbeat [label="PFCP heartbeat FSM",shape=box3d]
-ASSOCIATED -> heartbeat [label="alloc()",style=dotted]
-heartbeat -> ASSOCIATED [label="EV_HEARTBEAT_FAILURE",style=dotted]
-
-txrx -> ASSOCIATED [label="EV_RX_SESSION_EST_REQ",style=dotted]
-up_session [label="PFCP UP session FSM",shape=box3d]
-ASSOCIATED -> up_session [label="up_session_create()",style=dotted]
-
-txrx -> ASSOCIATED [label="EV_RX_ASSOC_UPD_REQ",style=dotted]
-ASSOCIATED -> GRACEFUL_RELEASE [label="Association Update\nindicating graceful release"]
-
-ASSOCIATED -> term [label="Heartbeat failure"]
-GRACEFUL_RELEASE -> term
-term [shape="octagon"]
-
-}

doc/charts/pfcp_up_session_fsm.dot

@@ -1,21 +0,0 @@
-digraph G {
-rankdir=TB
-labelloc=t; label="PFCP UP session FSM"
-
-peer [label="PFCP UP peer FSM",shape=box3d]
-peer -> ESTABLISHED [label="rx_session_est_req()"]
-
-txrx [label="PFCP socket",shape="box"]
-txrx2 [label="PFCP socket",shape="box"]
-
-txrx -> ESTABLISHED [label="EV_RX_SESSION_MOD_REQ",style=dotted]
-ESTABLISHED -> txrx [label="tx_session_mod_resp()",style=dotted,constraint=false]
-ESTABLISHED -> ESTABLISHED [label="Mod"]
-
-txrx2 -> ESTABLISHED [label="EV_RX_SESSION_DEL_REQ",style=dotted]
-ESTABLISHED -> txrx2 [label="tx_session_del_resp()",style=dotted,constraint=false]
-
-ESTABLISHED -> term [label="Deletion"]
-term [shape="octagon"]
-
-}

doc/examples/osmo-upf/osmo-upf-create-dev.cfg

@@ -6,11 +6,10 @@ log stderr
  logging print category-hex 0
  logging print file basename last
  logging print extended-timestamp 1
- logging level set-all info
-#logging level set-all debug
+ logging level set-all notice
 
 timer pfcp x24 5000
 pfcp
  local-addr 127.0.0.1
-gtp
+tunend
  dev create apn23

doc/examples/osmo-upf/osmo-upf-mockup.cfg

@@ -6,14 +6,12 @@ log stderr
  logging print category-hex 0
  logging print file basename last
  logging print extended-timestamp 1
- logging level set-all debug
  logging level set-all notice
- logging level set-all info
 
 timer pfcp x24 5000
 pfcp
  local-addr 127.0.0.1
-gtp
+tunend
  mockup
-nft
+tunmap
  mockup

doc/examples/osmo-upf/osmo-upf.cfg

@@ -6,9 +6,7 @@ log stderr
  logging print category-hex 0
  logging print file basename last
  logging print extended-timestamp 1
- logging level set-all debug
  logging level set-all notice
- logging level set-all info
 
 timer pfcp x24 5000
 pfcp

doc/manuals/chapters/netinst.adoc

@@ -0,0 +1,57 @@
+[[netinst]]
+== Local GTP Addresses / Network Instance
+
+PFCP features optional Network Instance IEs, in which the CPF may tell the UPF which local network interface to use for
+a PDR and/or a FAR.
+
+NOTE:: osmo-upf only evaluates the Network Instances configured in PDRs. Since osmo-upf always pairs a PDR+FAR with
+another PDR+FAR in reverse direction, each side's PDR is sufficient.
+
+Network Instance IEs affect both the tunend and the tunmap use cases, as well as which local IP address is returned
+in the PFCP response
+
+1. Look up Network Instance name in the osmo-upf.cfg `netinst` section, to obtain a local IP address.
+2. Depending on use case:
+  - tunend: create the tunnel on a GTP device matching the local IP address, see <<gtp_module>>.
+  - tunmap: use the local IP address in the netfilter ruleset, see <<nftables>>.
+3. Usually, return the chosen local IP address in the F-TEID IE of the Created PDR IE in the PFCP response.
+
+Network Instance configuration consists of {name, IP address} pairs.
+
+NOTE:: As soon as a `netinst` configuration is nonempty, receiving an undefined Network Instance name results in a PFCP
+Reject response, and a log message on cateogry `session`, level `NOTICE`. To make the PFCP return success, add the
+failing name to the `netinst` config.
+
+=== netinst for tunend
+
+The following configuration sets up two GTP devices for tunend, expecting Network Instance names `access1` or `access2`:
+
+----
+tunend
+ dev create apn1 10.0.0.1
+ dev create apn2 10.0.0.2
+netinst
+ add access1 10.0.0.1
+ add access2 10.0.0.2
+----
+
+For example, if a Create PDR IE indicates Network Instance = `access1`, a GTP tunnel is set up in GTP kernel device
+`apn1`. For `access2`, use `apn2`.
+
+=== netinst for tunmap
+
+For the tunmap use case, it is sufficient to configure `netinst` entries, without any addition to the `tunmap` section.
+The following example configures various interfaces for tunmap, to match Network Instance names received in PFCP:
+
+----
+tunmap
+ table-name osmo-upf
+netinst
+ add access1 10.0.0.1
+ add access2 10.0.0.2
+ add core1 9.0.0.1
+ add core2 9.0.0.2
+----
+
+For example, a Create PDR indicating a Network Instance of `core1` will result in an nftables rule that receives packets
+on local address `9.0.0.1`.

doc/manuals/chapters/overview.adoc

@@ -22,3 +22,189 @@ The aim is to provide:
 - 1000 modifications of tunnel state per second (add/remove/modify),
 - 4-8 Gbps throughput,
 - 100-125k concurrent GTP tunnels.
+
+A typical network scenario using OsmoUPF is illustrated in the following
+diagram:
+
+.Typical network architecture used with OsmoUPF
+[graphviz]
+----
+digraph G {
+  rankdir = LR;
+
+  UE [label="UE\n(3G phone)"]
+
+  subgraph cluster_hnbgw_mgw_upf {
+    style=dotted
+    HNBGW -> UPF [label="PFCP",constraint=false]
+    UPF [label=OsmoUPF,style=bold]
+  }
+
+  subgraph cluster_hnbgw_mgw_upf2 {
+    style=dotted
+    SGSN -> UPF2 [label="PFCP",constraint=false]
+    UPF2 [label=OsmoUPF,style=bold]
+  }
+
+  subgraph cluster_hnbgw_mgw_upf3 {
+    style=dotted
+    GGSN -> UPF3 [label="PFCP",constraint=false]
+    UPF3 [label=OsmoUPF,style=bold]
+  }
+
+  hNodeB [shape="box",label="hNodeB\n(3G femto cell)"]
+
+  UE -> hNodeB [label="Uu"]
+  hNodeB -> HNBGW [label="Iuh",style=dashed]
+  STP [label="STP\n(SCCP/M3UA)"]
+  HNBGW -> STP -> SGSN [label="IuPS",style=dashed]
+  SGSN -> GGSN [label="GTP-C",style="dashed"]
+  hNodeB -> UPF -> UPF2 -> UPF3 [label="GTP-U"]
+  UPF3 -> internet [label="apn"]
+}
+----
+
+NOTE: at the time of writing this section, the only Osmocom component providing
+a PFCP CPF interface is OsmoHNBGW. PFCP support has not yet made its way into
+OsmoSGSN nor OsmoGGSN.
+
+=== the PFCP interface
+
+PFCP is specified by 3GPP TS 29.244.
+
+OsmoUPF implements a PFCP User Plane Function interface, listening for PFCP
+requests from PFCP Control Plane Function clients, to carry out proxy-relaying
+and encapsulation/decapsulation of GTP tunnels.
+
+OsmoUPF does not support the complete PFCP feature set. It detects exactly two
+use cases that will provide service of actual GTP tunnels:
+
+.tunend use case
+----
+Access                 osmo-upf              Core
+ PGW                      |              PDN/internet
+  |                PDR1:  > FAR1:             |
+  |                IP/GTP | IP                |
+  |        ------> F-TEID |            -----> |
+  |                       |                   |
+  |                FAR2:  < PDR2:             |
+  |                IP/GTP | IP                |
+  | F-TEID <------        | UE IP addr <----- |
+----
+
+* `tunend`: GTP tunnel encapsulation/decapsulation:
+  - One Packet Detection Rule (PDR) accepts a GTP tunnel from the Access side
+    with an Outer Header Removal.
+  - This PDR uses a Forwarding Action Rule (FAR) for plain IP towards Core.
+  - Another PDR accepts plain IP on a specific IP address from Core.
+  - The second PDR uses a FAR towards Access with Outer Header Creation for GTP.
+
+.tunmap use case
+----
+Access                 osmo-upf                 Core
+ PGW                      |                     PGW
+  |                PDR1:  > FAR1:                |
+  |                IP/GTP | IP/GTP               |
+  |        ------> F-TEID |        -----> F-TEID |
+  |                       |                      |
+  |                FAR2:  < PDR2:                |
+  |                IP/GTP | IP/GTP               |
+  | F-TEID <------        | F-TEID <-----        |
+----
+
+* `tunmap`: GTP tunnel forwarding:
+  - One Packet Detection Rule (PDR) accepts a GTP tunnel from the Access side
+    with an Outer Header Removal.
+  - This PDR uses a Forwarding Action Rule (FAR) towards Core with an Outer
+    Header Creation for GTP.
+  - A second PDR+FAR pair like above, with Access and Core swapped.
+
+Access and Core must be indicated by the Source Interface IE (PDR) and
+Destination Interface IE (FAR) in PFCP.
+
+Any set of rules only partially or not at all matching the above PDR and FAR
+rules will not result in any actions on the GTP user plane, but will still
+return a successful outcome in the PFCP messages.
+
+For example, a rule set using a Source Interface other than "Access" or "Core" results
+in a PFCP no-op, returning PFCP responses with successful outcome, but not
+providing any GTP-U service.
+
+This is a direct result of:
+
+- allowing PFCP rule sets to be setup incrementally by several subsequent PFCP
+  messages, and of
+- OsmoUPF using Linux kernel features for the GTP user plane, where there is
+  either a full bidirectional GTP tunnel in place or none at all.
+
+For example, for `tunmap`, a typical CPF will establish a PFCP session in two
+steps: first request a local F-TEID from the UPF before passing on a data
+service request from Access to Core. When the Core side has responded with its
+GTP details, the PFCP session at the UPF is updated (Session Modifification),
+to form a complete PFCP rule set.
+
+.Typical sequence of establishing a GTP-U tunnel relay
+["mscgen"]
+----
+msc {
+	hscale="1";
+	sgsn[label="SGSN"],sgwc[label="SGW-C"],sgwu[label="SGW-U"],pgwc[label="PGW-C"];
+
+	sgsn << pgwc [label="Access"];
+	sgsn >> pgwc [label="Core"];
+
+	sgsn => sgwc [label="GTP Create Session Request\n\n\n"];
+
+	|||;
+
+	sgwc => sgwu [label="PFCP Session Establishment Request\n\n2x Create PDR\nF-TEID = CHOOSE"];
+
+	|||;
+
+	sgwc <= sgwu [label="PFCP Session Establishment Response\n\n2x Created PDR\nwith chosen local F-TEID"];
+
+	|||;
+
+	sgwc => pgwc [label="GTP Create Session Request\nwith chosen local F-TEID towards Core"];
+	sgwc <= pgwc [label="GTP Create Session Response\nwith remote F-TEID at Core"];
+
+	|||;
+
+	sgwc => sgwu [label="PFCP Session Modification Request\n\nUpdate FAR\nwith remote F-TEID at Core"];
+
+	|||;
+
+	sgwc <= sgwu [label="PFCP Session Modification Response\n\n\n"];
+
+	|||;
+
+	sgsn <= sgwc [label="GTP Create Session Response\n\n\n"];
+}
+----
+
+The OsmoUPF logging as well as the VTY interface yield information on whether a
+ruleset results in an actual bidirectional GTP tunnel being set up.
+
+
+=== the GTP interface
+
+OsmoUPF requires the following Linux kernel features to provide the GTP user
+plane functionality:
+
+- the Linux kernel GTP module for encapsulation/decapsulation between GTP and
+  plain IP.
+- the Linux netfilter nftables feature for relaying GTP, i.e. forwarding between
+  two GTP tunnels.
+
+Tunnel relaying with netfilter requires at least Linux kernel 5.17.
+
+To be able to interact with these Linux kernel features, the osmo-upf binary
+needs cap_net_admin privileges, as in:
+
+----
+sudo setcap cap_net_admin+pe /usr/bin/osmo-upf
+----
+
+Without above Linux kernel features, or when no cap_net_admin is available,
+OsmoUPF is only useful for testing PFCP clients: the GTP features may be run in
+mockup mode, so that OsmoUPF serves as a "dry run" PFCP server.

doc/manuals/chapters/running.adoc

@@ -0,0 +1,180 @@
+== Running OsmoUPF
+
+The OsmoUPF executable (`osmo-upf`) offers the following command-line
+arguments:
+
+=== SYNOPSIS
+
+*osmo-upf* [-h|-V] [-D] [-c 'CONFIGFILE']
+
+=== OPTIONS
+
+*-h, --help*::
+	Print a short help message about the supported options
+*-V, --version*::
+	Print the compile-time version number of the OsmoHNBGW program
+*-D, --daemonize*::
+	Fork the process as a daemon into background.
+*-c, --config-file 'CONFIGFILE'*::
+	Specify the file and path name of the configuration file to be
+	used. If none is specified, use `osmo-upf.cfg` in the current
+	working directory.
+
+=== Multiple instances
+
+Running multiple instances of `osmo-upf` on the same computer is possible if
+all interfaces (VTY, CTRL, PFCP) are separated using the appropriate
+configuration options. The IP based interfaces are binding to local host by
+default. In order to separate the processes, the user has to bind those
+services to different ports, or different specific IP addresses.
+
+The VTY and the Control interface can be bound to IP addresses from the loopback
+address range, for example:
+
+----
+line vty
+ bind 127.0.0.2
+ctrl
+ bind 127.0.0.2
+----
+
+The PFCP port is specified to be fixed as port 8805. Hence, each osmo-upf
+process needs to run on a distinct local interface:
+
+----
+pfcp
+  local-addr 10.9.0.2
+----
+
+For GTP encapsulation/decapsulation and GTP tunnel relaying, osmo-upf depends on
+the IP addresses configured at the Linux kernel GTP module, and the IP addresses
+negotiated within PFCP by the control plane function.
+
+If multiple `osmo-upf` processes are running on the same Linux kernel, each
+`osmo-upf` needs to be configured with a distinct netfilter table name, so that
+naming of individual tunnel rulesets does not collide:
+
+----
+tunmap
+ table-name osmo-upf-2
+----
+
+=== Configuring Primary Links
+
+==== Configure PFCP Server
+
+The following example configures OsmoUPF to listen for PFCP association requests
+from Control Plane Function entities on local interface 10.9.8.7, port 8805:
+
+----
+pfcp
+ local-addr 10.9.8.7
+----
+
+3GPP TS 29.244 4.2.2 specifies that PFCP Request messages shall be sent to UDP
+port 8805, i.e. the PFCP port is fixed as 8805 and currently not configurable in
+osmo-upf.
+
+Setting a 'local-addr' is required: the PFCP protocol features a Node ID, which
+uniquely identifies PFCP peers across different interfaces. According to the
+PFCP specification, the Node ID can be a fully-qualified domain name (FQDN) or
+an IP address. Currently, osmo-upf has no support for using an FQDN as Node
+ID, and so far uses the 'local-addr' as local Node ID -- hence the 'local-addr'
+must not be "0.0.0.0", which is an unfortunate consequence. This is likely to
+improve in the future, see https://osmocom.org/issues/5682 .
+
+==== Configure Linux Kernel GTP Features
+
+OsmoUPF uses two distinct Linux kernel features:
+
+* The GTP module is used for GTP encapsulation/decapsulation from/to
+  "the internet".
+
+* The netfilter module is used for GTP tunnel proxying, also known as
+  tunnel forwarding or tunnel mapping.
+
+.Linux kernel feature usage
+[graphviz]
+----
+include::upf_gtp_roles.dot[]
+----
+
+GTP kernel module configuration can be omitted for sites that serve only as GTP
+forwarding proxy, without encapsulation/decapsulation of GTP payloads.
+
+[[gtp_module]]
+===== Configure Linux Kernel GTP Module for `tunend`
+
+The Linux kernel GTP module is used for the `tunend` use case, i.e. GTP
+encapsulation/decapsulation from/to "the internet".
+
+To use the GTP kernel module, OsmoUPF requires a GTP device, which is a
+dedicated network device provided by the Linux kernel, serving as GTP tunnel
+endpoint. It is typically named like "apn0".
+
+`osmo-upf` can either create a GTP device on startup, or use a pre-existing GTP
+device. To en/decapsulate GTP, the APN device needs to be assigned an IP address
+range that matches the UE IP addresses that are configured in GTP-C / PFCP.
+
+The following configuration placed in `osmo-upf.cfg` creates a GTP device called
+`apn23` on startup of osmo-upf, which is destroyed on program exit. It listens
+for GTP on local IP address `1.2.3.4`:
+
+----
+tunend
+ dev create apn23 1.2.3.4
+----
+
+TODO:: `osmo-upf` is not yet able to configure this network device's IP address
+range, MTU etc.
+
+The following configuration placed in `osmo-upf.cfg` uses a pre-existing device
+called `apn42`:
+
+----
+tunend
+ dev use apn42 2.3.4.5
+----
+
+GTP kernel devices can be managed manually using the `gtp-link` program
+available from the 'libgtpnl' project:
+
+----
+# gtp-link add apn42
+(keep this process running)
+# ip addr add dev apn42 192.168.42.1/24
+
+$ osmo-upf -c osmo-upf.cfg
+----
+
+It is possible to configure multiple GTP devices in `osmo-upf.cfg`. Depending on
+the Network Instance name, osmo-upf creates tunnel endpoints on the GTP device
+with a matching IP address:
+
+- The Network Instance IE in the PDR on the Access side determines the local IP
+  address to use, see <<netinst>>.
+- This local IP address in turn determines the GTP device to use.
+
+It is possible for a GTP device to listen on ANY -- just omit the IP address in
+the `dev` config. In this case, all Network Instance names will be served by
+this GTP device. When using ANY, there should be exactly one GTP dev configured.
+
+[[nftables]]
+===== Configure Linux netfilter for `tunmap`
+
+The Linux kernel netfilter module is used for GTP tunnel proxying, also known as
+tunnel forwarding or tunnel mapping.
+
+Using the netfilter module usually requires no configuration in `osmo-upf.cfg`.
+
+`osmo-upf` creates a new netfilter table, under which it submits rule sets for
+GTP tunnel proxying. This table name defaults to `osmo-upf`. A custom table name
+can be configured in `osmo-upf.cfg` like this:
+
+----
+tunmap
+ table-name my-table-name
+----
+
+When running more than one osmo-upf process on a system, pick distinct table
+names to avoid name collisions in the nftables reulesets.

doc/manuals/chapters/upf_gtp_roles.dot

@@ -0,0 +1,31 @@
+digraph G {
+rankdir=LR
+sgsn [label="SGSN"]
+
+subgraph cluster_sgw {
+	style=invisible
+	sgwc [label="SGW-C"]
+	sgwu [label="OsmoUPF as SGW-U\ntunnel proxy\n*netfilter* kernel module",style=bold,shape=box]
+	sgwc -> sgwu [label="PFCP",constraint=false]
+}
+
+subgraph cluster_pgw {
+	style=invisible
+	pgwc [label="PGW-C"]
+	pgwu [label="OsmoUPF as PGW-U\ntunnel proxy\n*netfilter* kernel module",style=bold,shape=box]
+	pgwc -> pgwu [label="PFCP",constraint=false]
+}
+
+subgraph cluster_tdf {
+	style=invisible
+	tdfc [label="TDF-C"]
+	tdfu [label="OsmoUPF as TDF-U\ntunnel en-/decaps\n*GTP* kernel module",style=bold,shape=box]
+	tdfc -> tdfu [label="PFCP",constraint=false]
+}
+
+pdn [label="PDN\n'the internet'"]
+
+sgsn -> sgwc -> pgwc -> tdfc [label="GTP-C"]
+sgsn -> sgwu -> pgwu -> tdfu [label="GTP-U",dir=both]
+tdfu -> pdn [label="IP",dir=both]
+}

doc/manuals/osmoupf-usermanual.adoc

@@ -9,6 +9,10 @@ include::./common/chapters/preface.adoc[]
 
 include::{srcdir}/chapters/overview.adoc[]
 
+include::{srcdir}/chapters/running.adoc[]
+
+include::{srcdir}/chapters/netinst.adoc[]
+
 include::./common/chapters/vty.adoc[]
 
 include::./common/chapters/logging.adoc[]

include/osmocom/upf/Makefile.am

@@ -1,4 +1,5 @@
 noinst_HEADERS = \
+	netinst.h \
 	up_endpoint.h \
 	up_peer.h \
 	up_session.h \

include/osmocom/upf/netinst.h

@@ -0,0 +1,44 @@
+/*
+ * (C) 2022 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+ * All Rights Reserved.
+ *
+ * Author: Neels Janosch Hofmeyr <nhofmeyr@sysmocom.de>
+ *
+ * SPDX-License-Identifier: GPL-2.0+
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#pragma once
+
+#include <osmocom/core/linuxlist.h>
+#include <osmocom/core/sockaddr_str.h>
+
+struct vty;
+
+struct network_instance {
+	struct llist_head entry;
+
+	char *name;
+	struct osmo_sockaddr_str addr;
+};
+
+const struct network_instance *netinst_add(void *ctx, struct llist_head *list, const char *name, const char *addr,
+					   const char **errmsg);
+const struct network_instance *netinst_find(struct llist_head *list, const char *name);
+const struct network_instance *netinst_first(struct llist_head *list);
+int netinst_clear(struct llist_head *list);
+
+int netinst_vty_write(struct vty *vty, struct llist_head *list, const char *indent, const char *name_or_null);

include/osmocom/upf/up_gtp_action.h

@@ -40,7 +40,7 @@ struct up_session;
 
 enum up_gtp_action_kind {
 	UP_GTP_DROP,
-	UP_GTP_U_ENDECAPS,
+	UP_GTP_U_TUNEND,
 	UP_GTP_U_TUNMAP,
 };
 
@@ -48,13 +48,13 @@ struct up_gtp_action {
 	struct llist_head entry;
 	struct up_session *session;
 
-	uint16_t pdr_core;
 	uint16_t pdr_access;
+	uint16_t pdr_core;
 
 	enum up_gtp_action_kind kind;
 	union {
 		/* En-/De-capsulate GTP: add/remove a GTP header and forward the GTP payload from/to plain IP. */
-		struct upf_gtp_tun_desc endecaps;
+		struct upf_gtp_tunend_desc tunend;
 
 		/* Tunnel-map GTP: translate from one TEID to another and forward */
 		struct upf_nft_tunmap_desc tunmap;

include/osmocom/upf/up_session.h

@@ -41,12 +41,6 @@ enum up_session_fsm_event {
 	UP_SESSION_EV_USE_COUNT_ZERO,
 };
 
-enum up_session_kind {
-	UP_SESSION_DROP,
-	UP_SESSION_GTP_U_ENDECAPS,
-	UP_SESSION_GTP_U_FORW,
-};
-
 struct up_session {
 	struct hlist_node node_by_up_seid;
 	struct hlist_node node_by_cp_seid;
@@ -60,10 +54,14 @@ struct up_session {
 	struct osmo_use_count use_count;
 	struct osmo_use_count_entry use_count_buf[8];
 
+	/* llist of struct pdr */
 	struct llist_head pdrs;
+	/* llist of struct far */
 	struct llist_head fars;
+	/* llist of struct chosen_f_teid */
 	struct llist_head chosen_f_teids;
 
+	/* llist of struct up_gtp_action */
 	struct llist_head active_gtp_actions;
 };
 
@@ -96,8 +94,8 @@ struct pdr {
 
 	bool rx_decaps;
 	bool forw_encaps;
-	bool forw_to_core;
-	bool forw_from_core;
+	bool access_to_core;
+	bool core_to_access;
 
 	struct pdr *reverse_pdr;
 	bool active;

include/osmocom/upf/upf.h

@@ -37,6 +37,12 @@ struct nft_ctx;
 
 #define UPF_PFCP_LISTEN_DEFAULT "0.0.0.0"
 
+#define PORT_GTP0_C 3386
+#define PORT_GTP0_U 3386
+
+#define PORT_GTP1_C 2123
+#define PORT_GTP1_U 2152
+
 extern struct osmo_tdef_group g_upf_tdef_groups[];
 
 struct pfcp_vty_cfg {
@@ -44,7 +50,7 @@ struct pfcp_vty_cfg {
 	uint16_t local_port;
 };
 
-struct gtp_vty_cfg_dev {
+struct tunend_vty_cfg_dev {
 	struct llist_head entry;
 
 	/* If true, osmo-upf creates the GTP device on startup. If false, the GTP device was created by the user, and we
@@ -60,12 +66,18 @@ struct gtp_vty_cfg_dev {
 	char *local_addr;
 };
 
-struct gtp_vty_cfg {
-	/* list of struct gtp_vty_cfg_dev, GTP devices as in the config file. The actual GTP devices in use are in
+struct tunend_vty_cfg {
+	/* list of struct tunend_vty_cfg_dev, GTP devices as in the config file. The actual GTP devices in use are in
 	 * g_upf->gtp.devs. */
 	struct llist_head devs;
 };
 
+/* Item in an llist of string pointers */
+struct string_listitem {
+	struct llist_head entry;
+	char *str;
+};
+
 struct g_upf {
 	struct ctrl_handle *ctrl;
 
@@ -80,7 +92,7 @@ struct g_upf {
 		bool mockup;
 
 		/* GTP devices as in osmo-upf.cfg */
-		struct gtp_vty_cfg vty_cfg;
+		struct tunend_vty_cfg vty_cfg;
 
 		/* GTP devices actually in use, list of struct upf_gtp_dev. */
 		struct llist_head devs;
@@ -98,9 +110,12 @@ struct g_upf {
 
 		struct nft_ctx *nft_ctx;
 		char *table_name;
-		int priority;
-		uint32_t next_id_state;
+		int priority_pre;
+		int priority_post;
+		uint32_t next_chain_id_state;
 	} nft;
+
+	struct llist_head netinst;
 };
 
 extern struct g_upf *g_upf;

include/osmocom/upf/upf_gtp.h

@@ -30,12 +30,6 @@
 #define LOG_GTP_DEV(DEV, LEVEL, FMT, ARGS...) \
 	LOGP(DGTP, LEVEL, "%s: " FMT, upf_gtp_dev_to_str_c(OTC_SELECT, (DEV)), ##ARGS)
 
-#define PORT_GTP0_C 3386
-#define PORT_GTP0_U 3386
-
-#define PORT_GTP1_C 2123
-#define PORT_GTP1_U 2152
-
 struct upf_gtp_dev {
 	struct llist_head entry;
 
@@ -57,29 +51,37 @@ struct upf_gtp_dev {
 
 	uint32_t ifidx;
 
+	/* list of struct upf_gtp_tunend */
 	struct llist_head tunnels;
 };
 
-struct upf_gtp_tun_desc {
+/* Description of a GTP encapsulation / decapsulation.
+ * The active state to operate the GTP kernel module accordingly is kept in struct upf_gtp_tunend. */
+struct upf_gtp_tunend_desc {
+	struct {
+		struct osmo_sockaddr gtp_local_addr;
 		uint32_t local_teid;
-	uint32_t remote_teid;
-	struct osmo_sockaddr ue_addr;
 		struct osmo_sockaddr gtp_remote_addr;
+		uint32_t remote_teid;
+	} access;
+	struct {
+		struct osmo_sockaddr ue_local_addr;
+	} core;
 };
 
-int upf_gtp_tun_desc_cmp(const struct upf_gtp_tun_desc *a, const struct upf_gtp_tun_desc *b);
+int upf_gtp_tunend_desc_cmp(const struct upf_gtp_tunend_desc *a, const struct upf_gtp_tunend_desc *b);
 
-int upf_gtp_genl_open();
+int upf_gtp_genl_ensure_open();
 void upf_gtp_genl_close();
 
 int upf_gtp_dev_open(const char *name, bool create_gtp_dev, const char *local_addr, bool listen_for_gtpv0,
 		     bool sgsn_mode);
 struct upf_gtp_dev *upf_gtp_dev_find_by_name(const char *name);
+struct upf_gtp_dev *upf_gtp_dev_find_by_local_addr(const struct osmo_sockaddr *local_addr);
 struct upf_gtp_dev *upf_gtp_dev_first();
 
-int upf_gtp_dev_tunnel_add(struct upf_gtp_dev *dev, const struct upf_gtp_tun_desc *t);
-bool upf_gtp_dev_is_tunnel_active(struct upf_gtp_dev *dev, const struct upf_gtp_tun_desc *t);
-int upf_gtp_dev_tunnel_del(struct upf_gtp_dev *dev, const struct upf_gtp_tun_desc *t);
+int upf_gtp_dev_tunend_add(struct upf_gtp_dev *dev, const struct upf_gtp_tunend_desc *t);
+int upf_gtp_dev_tunend_del(struct upf_gtp_dev *dev, const struct upf_gtp_tunend_desc *t);
 
 int upf_gtp_dev_to_str_buf(char *buf, size_t buflen, const struct upf_gtp_dev *dev);
 char *upf_gtp_dev_to_str_c(void *ctx, const struct upf_gtp_dev *dev);

include/osmocom/upf/upf_nft.h

@@ -27,24 +27,32 @@
 
 #include <osmocom/core/socket.h>
 
-#define NFT_CHAIN_NAME_PREFIX_TUNMAP "tunmap"
-
 struct upf_nft_tunmap_desc {
 	struct {
-		struct osmo_sockaddr gtp_remote_addr;
+		struct osmo_sockaddr gtp_local_addr;
 		uint32_t local_teid;
+		struct osmo_sockaddr gtp_remote_addr;
 		uint32_t remote_teid;
+		uint32_t chain_id;
 	} access;
 	struct {
-		struct osmo_sockaddr gtp_remote_addr;
+		struct osmo_sockaddr gtp_local_addr;
 		uint32_t local_teid;
+		struct osmo_sockaddr gtp_remote_addr;
 		uint32_t remote_teid;
+		uint32_t chain_id;
 	} core;
-	uint32_t id;
 };
 
+int upf_nft_tunmap_to_str_buf(char *buf, size_t buflen, const struct upf_nft_tunmap_desc *tunmap);
+char *upf_nft_tunmap_to_str_c(void *ctx, const struct upf_nft_tunmap_desc *tunmap);
+
 int upf_nft_init();
 int upf_nft_free();
 
+char *upf_nft_tunmap_get_table_init_str(void *ctx);
+char *upf_nft_tunmap_get_vmap_init_str(void *ctx);
+char *upf_nft_tunmap_get_ruleset_str(void *ctx, struct upf_nft_tunmap_desc *tunmap);
+char *upf_nft_tunmap_get_ruleset_del_str(void *ctx, struct upf_nft_tunmap_desc *tunmap);
 int upf_nft_tunmap_create(struct upf_nft_tunmap_desc *tunmap);
 int upf_nft_tunmap_delete(struct upf_nft_tunmap_desc *tunmap);

src/osmo-pfcp-tool/osmo_pfcp_tool_main.c

@@ -206,17 +206,16 @@ static void signal_handler(int signum)
 	}
 }
 
-static const char * const osmo_pfcp_tool_copyright =
-	"OsmoPFCPTool - Osmocom Packet Forwarding Control Protocol tool for testing\r\n"
-	"Copyright (C) 2021-2022 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>\r\n"
-	"License AGPLv3+: GNU AGPL version 3 or later <http://gnu.org/licenses/agpl-3.0.html>\r\n"
-	"This is free software: you are free to change and redistribute it.\r\n"
-	"There is NO WARRANTY, to the extent permitted by law.\r\n";
 
 static struct vty_app_info pfcp_tool_vty_app_info = {
 	.name = "osmo-pfcp-tool",
 	.version = PACKAGE_VERSION,
-	.copyright = osmo_pfcp_tool_copyright,
+	.copyright =
+	"OsmoPFCPTool - Osmocom Packet Forwarding Control Protocol tool for testing\r\n"
+	"Copyright (C) 2021-2022 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>\r\n"
+	"License AGPLv3+: GNU AGPL version 3 or later <http://gnu.org/licenses/agpl-3.0.html>\r\n"
+	"This is free software: you are free to change and redistribute it.\r\n"
+	"There is NO WARRANTY, to the extent permitted by law.\r\n",
 };
 
 static const struct log_info_cat pfcp_tool_default_categories[] = {
@@ -300,13 +299,13 @@ int main(int argc, char **argv)
 		}
 	}
 
-	/* start telnet, after reading config for vty_get_bind_addr() */
-	rc = telnet_init_dynif(tall_pfcp_tool_ctx, &g_pfcp_tool, vty_get_bind_addr(), OSMO_VTY_PORT_PFCP_TOOL);
+	/* start telnet VTY */
+	rc = telnet_init_default(tall_pfcp_tool_ctx, &g_pfcp_tool, OSMO_VTY_PORT_PFCP_TOOL);
 	if (rc < 0)
 		return 2;
 
 	/* start control interface, after reading config for ctrl_vty_get_bind_addr() */
-	g_pfcp_tool->ctrl = ctrl_interface_setup_dynip(g_pfcp_tool, ctrl_vty_get_bind_addr(), OSMO_CTRL_PORT_PFCP_TOOL, NULL);
+	g_pfcp_tool->ctrl = ctrl_interface_setup(g_pfcp_tool, OSMO_CTRL_PORT_PFCP_TOOL, NULL);
 	if (!g_pfcp_tool->ctrl) {
 		fprintf(stderr, "Failed to initialize control interface. Exiting.\n");
 		return -1;

src/osmo-pfcp-tool/pfcp_tool.c

@@ -85,7 +85,7 @@ struct pfcp_tool_session *pfcp_tool_session_find(struct pfcp_tool_peer *peer, ui
 }
 
 struct pfcp_tool_session *pfcp_tool_session_find_or_create(struct pfcp_tool_peer *peer, uint64_t cp_seid,
-							   enum up_gtp_action_kind gtp_action)
+							   enum up_gtp_action_kind kind)
 {
 	struct pfcp_tool_session *session = pfcp_tool_session_find(peer, cp_seid);
 	if (session)
@@ -95,7 +95,7 @@ struct pfcp_tool_session *pfcp_tool_session_find_or_create(struct pfcp_tool_peer
 	*session = (struct pfcp_tool_session){
 		.peer = peer,
 		.cp_seid = cp_seid,
-		.gtp_action = gtp_action,
+		.kind = kind,
 	};
 	llist_add(&session->entry, &peer->sessions);
 	return session;
@@ -159,14 +159,23 @@ void pfcp_tool_rx_msg(struct osmo_pfcp_endpoint *ep, struct osmo_pfcp_msg *m, st
 	}
 }
 
+static void copy_msg(struct osmo_pfcp_msg *dst, const struct osmo_pfcp_msg *m)
+{
+	*dst = *m;
+	dst->encoded = NULL;
+	dst->ctx.peer_use_token = NULL;
+	dst->ctx.session_use_token = NULL;
+	dst->ctx.resp_cb = NULL;
+}
+
 int peer_tx(struct pfcp_tool_peer *peer, struct osmo_pfcp_msg *m)
 {
 	int rc;
-	rc = osmo_pfcp_endpoint_tx(g_pfcp_tool->ep, m);
 	if (m->is_response)
-		peer->last_resp = *m;
+		copy_msg(&peer->last_resp, m);
 	else
-		peer->last_req = *m;
+		copy_msg(&peer->last_req, m);
+	rc = osmo_pfcp_endpoint_tx(g_pfcp_tool->ep, m);
 	return rc;
 }
 

src/osmo-pfcp-tool/pfcp_tool.h

@@ -51,30 +51,43 @@ struct pfcp_tool_peer {
 	struct llist_head sessions;
 };
 
-struct pfcp_tool_teid_pair {
-	uint32_t local;
-	uint32_t remote;
+struct pfcp_tool_gtp_tun_ep {
+	struct osmo_sockaddr_str addr;
+	uint32_t teid;
+};
+
+struct pfcp_tool_gtp_tun {
+	struct pfcp_tool_gtp_tun_ep local;
+	struct pfcp_tool_gtp_tun_ep remote;
+};
+
+struct pfcp_tool_tunend_desc {
+	struct pfcp_tool_gtp_tun access;
+	struct {
+		struct osmo_sockaddr_str ue_local_addr;
+	} core;
+};
+
+struct pfcp_tool_tunmap_desc {
+	struct pfcp_tool_gtp_tun access;
+	struct pfcp_tool_gtp_tun core;
 };
 
 struct pfcp_tool_session {
 	struct llist_head entry;
 
-	enum up_gtp_action_kind gtp_action;
-
 	struct pfcp_tool_peer *peer;
 	uint64_t cp_seid;
 	struct osmo_pfcp_ie_f_seid up_f_seid;
 
-	struct {
-		struct pfcp_tool_teid_pair teid;
-		struct osmo_sockaddr_str gtp_ip;
-	} access;
+	enum up_gtp_action_kind kind;
+	union {
+		/* En-/De-capsulate GTP: add/remove a GTP header and forward the GTP payload from/to plain IP. */
+		struct pfcp_tool_tunend_desc tunend;
 
-	struct {
-		struct pfcp_tool_teid_pair teid;
-		struct osmo_sockaddr_str gtp_ip;
-		struct osmo_sockaddr_str ue_addr;
-	} core;
+		/* Tunnel-map GTP: translate from one TEID to another and forward */
+		struct pfcp_tool_tunmap_desc tunmap;
+	};
 };
 
 struct g_pfcp_tool {

src/osmo-pfcp-tool/pfcp_tool_vty.c

@@ -234,7 +234,7 @@ DEFUN(peer_retrans_req, peer_retrans_req_cmd,
 	else
 		*m = peer->last_resp;
 
-	OSMO_LOG_PFCP_MSG(m, LOGL_DEBUG, "retrans %s\n", argv[0]);
+	OSMO_LOG_PFCP_MSG(m, LOGL_INFO, "retrans %s\n", argv[0]);
 
 	rc = osmo_pfcp_endpoint_tx_data(g_pfcp_tool->ep, m);
 	if (rc) {
@@ -250,24 +250,26 @@ static struct cmd_node session_node = {
 	1,
 };
 
+#define SESSION_STR "Enter the 'session' node for the given SEID\n"
+#define TUNEND_STR "Set up GTP tunnel encapsulation/decapsulation (default)\n"
+#define TUNMAP_STR "Set up GTP tunnel mapping\n"
+#define SEID_STR "local Session Endpoint ID\n"
+
 DEFUN(session, session_cmd,
-      "session [(endecaps|tunmap)] [<0-18446744073709551615>]",
-      "Enter the 'session' node for the given SEID\n"
-      "Set up GTP tunnel encapsulation/decapsulation (default)\n"
-      "Set up GTP tunnel mapping\n"
-      "local Session Endpoint ID\n")
+      "session [(tunend|tunmap)] [<0-18446744073709551615>]",
+      SESSION_STR TUNEND_STR TUNMAP_STR SEID_STR)
 {
 	struct pfcp_tool_peer *peer = vty->index;
 	struct pfcp_tool_session *session;
-	enum up_gtp_action_kind gtp_action = UP_GTP_U_ENDECAPS;
+	enum up_gtp_action_kind kind = UP_GTP_U_TUNEND;
 
 	if (argc > 0 && !strcmp(argv[0], "tunmap"))
-		gtp_action = UP_GTP_U_TUNMAP;
+		kind = UP_GTP_U_TUNMAP;
 
 	if (argc > 1)
-		session = pfcp_tool_session_find_or_create(peer, atoll(argv[1]), gtp_action);
+		session = pfcp_tool_session_find_or_create(peer, atoll(argv[1]), kind);
 	else
-		session = pfcp_tool_session_find_or_create(peer, peer_new_seid(peer), gtp_action);
+		session = pfcp_tool_session_find_or_create(peer, peer_new_seid(peer), kind);
 
 	vty->index = session;
 	vty->node = SESSION_NODE;
@@ -275,64 +277,127 @@ DEFUN(session, session_cmd,
 	return CMD_SUCCESS;
 }
 
+/* legacy compat: "tunend" was originally named "endecaps" */
+DEFUN_CMD_ELEMENT(session, session_endecaps_cmd,
+		  "session (endecaps) [<0-18446744073709551615>]",
+		  SESSION_STR TUNEND_STR SEID_STR, CMD_ATTR_HIDDEN, 0);
+
 DEFUN(s_ue, s_ue_cmd,
       "ue ip A.B.C.D",
       "Setup the UE as it appears towards the Core network in plain IP traffic\n"
       "IP address assigned to the UE\n")
 {
 	struct pfcp_tool_session *session = vty->index;
-	if (osmo_sockaddr_str_from_str2(&session->core.ue_addr, argv[0])) {
+
+	if (session->kind != UP_GTP_U_TUNEND) {
+		vty_out(vty, "%% Error: 'ue ip' makes no sense in a 'tunmap' session%s", VTY_NEWLINE);
+		return CMD_WARNING;
+	}
+	if (osmo_sockaddr_str_from_str2(&session->tunend.core.ue_local_addr, argv[0])) {
 		vty_out(vty, "Error setting UE IP address%s", VTY_NEWLINE);
 		return CMD_WARNING;
 	}
 	return CMD_SUCCESS;
 }
 
-DEFUN(s_teid, s_teid_cmd,
-      "gtp (access|core) teid local <0-4294967295> remote <0-4294967295>",
-      "Setup TEID used in GTP\n"
-      "Set the TEIDs towards the ACCESS network (towards the radio network and the actual UE)\n"
-      "Set the TEIDs towards the CORE network (towards the internet)\n"
-      "Local TEID, which the UPF expects to see in incoming GTP packets\n"
-      "Local TEID, when 0 tell the UPF to choose (PFCP: FAR F-TEID: CHOOSE=1)\n"
-      "Remote TEID, which the UPF sends out in GTP packets\n"
-      "Remote TEID, which the GTP peer has assigned for itself\n")
-{
-	struct pfcp_tool_session *session = vty->index;
-	struct pfcp_tool_teid_pair *dst;
-	if (!strcmp(argv[0], "access"))
-		dst = &session->access.teid;
-	else
-		dst = &session->core.teid;
-	*dst = (struct pfcp_tool_teid_pair){
-		.local = atoi(argv[1]),
-		.remote = atoi(argv[2]),
-	};
-	return CMD_SUCCESS;
-}
+#define GTP_ACCESS_CORE_STRS \
+      "Setup GTP\n" \
+      "Setup GTP towards ACCESS (towards the radio network and the actual UE)\n" \
+      "Setup GTP towards CORE (towards the internet)\n"
+#define GTP_LOCAL_STR "Setup GTP on the local side (UPF's local GTP endpoint)\n"
+#define GTP_REMOTE_STR "Setup GTP on the remote side (UPF's remote GTP peer)\n"
+#define F_TEID_STR "Set the fully-qualified TEID, i.e. GTP IP address and TEID\n"
 
-DEFUN(s_gtp, s_gtp_cmd,
-      "gtp (access|core) ip A.B.C.D",
-      "Setup GTP peer\n"
-      "Set the GTP peer towards the ACCESS network (towards the radio network and the actual UE)\n"
-      "Set the GTP peer towards the CORE network (towards the internet)\n"
-      "Set the GTP peer IP address, where to send GTP packets to / receive GTP packets from\n"
-      "GTP peer IP address\n")
+DEFUN(s_f_teid, s_f_teid_cmd,
+      "gtp (access|core) (local|remote) f-teid A.B.C.D <0-4294967295>",
+      GTP_ACCESS_CORE_STRS
+      GTP_LOCAL_STR GTP_REMOTE_STR
+      F_TEID_STR
+      "GTP peer IP address\n"
+      "GTP TEID\n")
 {
 	struct pfcp_tool_session *session = vty->index;
-	struct osmo_sockaddr_str *dst;
-	if (!strcmp(argv[0], "access"))
-		dst = &session->access.gtp_ip;
+	const char *tun_side = argv[0];
+	const char *local_remote = argv[1];
+	const char *addr_str = argv[2];
+	const char *teid_str = argv[3];
+	struct pfcp_tool_gtp_tun_ep *dst;
+
+	switch (session->kind) {
+	case UP_GTP_U_TUNEND:
+		if (!strcmp(tun_side, "access")) {
+			if (!strcmp(local_remote, "local"))
+				dst = &session->tunend.access.local;
 			else
-		dst = &session->core.gtp_ip;
-	if (osmo_sockaddr_str_from_str2(dst, argv[1])) {
-		vty_out(vty, "Error setting GTP IP address%s", VTY_NEWLINE);
+				dst = &session->tunend.access.remote;
+		} else {
+			vty_out(vty, "%% Error: 'gtp core (local|remote) f-teid': 'tunend' only has GTP on"
+				" the 'access' side%s", VTY_NEWLINE);
 			return CMD_WARNING;
 		}
+		break;
+	case UP_GTP_U_TUNMAP:
+		if (!strcmp(tun_side, "access")) {
+			if (!strcmp(local_remote, "local"))
+				dst = &session->tunmap.access.local;
+			else
+				dst = &session->tunmap.access.remote;
+		} else {
+			if (!strcmp(local_remote, "local"))
+				dst = &session->tunmap.core.local;
+			else
+				dst = &session->tunmap.core.remote;
+		}
+		break;
+	default:
+		OSMO_ASSERT(0);
+	}
+
+	if (osmo_sockaddr_str_from_str2(&dst->addr, addr_str)) {
+		vty_out(vty, "Error setting GTP IP address from %s%s",
+			osmo_quote_cstr_c(OTC_SELECT, addr_str, -1), VTY_NEWLINE);
+		return CMD_WARNING;
+	}
+	dst->teid = atoi(teid_str);
 	return CMD_SUCCESS;
 }
 
-int session_endecaps_tx_est_req(struct vty *vty, const char **argv, int argc)
+DEFUN(s_f_teid_choose, s_f_teid_choose_cmd,
+      "gtp (access|core) local f-teid choose",
+      GTP_ACCESS_CORE_STRS
+      GTP_LOCAL_STR
+      F_TEID_STR
+      "Send F-TEID with CHOOSE=1, i.e. the UPF shall return the local F-TEID in a PFCP Created PDR IE\n")
+{
+	struct pfcp_tool_session *session = vty->index;
+	const char *tun_side = argv[0];
+	struct pfcp_tool_gtp_tun_ep *dst;
+
+	switch (session->kind) {
+	case UP_GTP_U_TUNEND:
+		if (!strcmp(tun_side, "access")) {
+			dst = &session->tunend.access.local;
+		} else {
+			vty_out(vty, "%% Error: 'gtp core local choose': 'tunend' only has GTP on"
+				" the 'access' side%s", VTY_NEWLINE);
+			return CMD_WARNING;
+		}
+		break;
+	case UP_GTP_U_TUNMAP:
+		if (!strcmp(tun_side, "access"))
+			dst = &session->tunmap.access.local;
+		else
+			dst = &session->tunmap.core.local;
+		break;
+	default:
+		OSMO_ASSERT(0);
+	}
+
+	*dst = (struct pfcp_tool_gtp_tun_ep){};
+	return CMD_SUCCESS;
+}
+
+int session_tunend_tx_est_req(struct vty *vty, const char **argv, int argc)
 {
 	struct pfcp_tool_session *session = vty->index;
 	struct pfcp_tool_peer *peer = session->peer;
@@ -344,6 +409,8 @@ int session_endecaps_tx_est_req(struct vty *vty, const char **argv, int argc)
 	struct osmo_sockaddr ue_addr;
 	struct osmo_pfcp_ie_f_seid cp_f_seid;
 
+	OSMO_ASSERT(session->kind == UP_GTP_U_TUNEND);
+
 	if (!g_pfcp_tool->ep) {
 		vty_out(vty, "Endpoint not configured%s", VTY_NEWLINE);
 		return CMD_WARNING;
@@ -354,12 +421,17 @@ int session_endecaps_tx_est_req(struct vty *vty, const char **argv, int argc)
 	else
 		osmo_pfcp_bits_set(aa.bits, OSMO_PFCP_APPLY_ACTION_FORW, true);
 
-	if (osmo_sockaddr_str_to_sockaddr(&session->core.ue_addr, &ue_addr.u.sas)) {
-		vty_out(vty, "Error in UE IP%s", VTY_NEWLINE);
-		return CMD_WARNING;
-	}
+#define STR_TO_ADDR(DST, SRC) do { \
+			if (osmo_sockaddr_str_to_sockaddr(&SRC, &DST.u.sas)) { \
+				vty_out(vty, "Error in " #SRC ": " OSMO_SOCKADDR_STR_FMT "%s", \
+					OSMO_SOCKADDR_STR_FMT_ARGS(&SRC), VTY_NEWLINE); \
+				return CMD_WARNING; \
+			} \
+		} while (0)
 
-	if (session->access.teid.local == 0) {
+	STR_TO_ADDR(ue_addr, session->tunend.core.ue_local_addr);
+
+	if (session->tunend.access.local.teid == 0) {
 		f_teid_access_local = (struct osmo_pfcp_ie_f_teid){
 			.choose_flag = true,
 			.choose = {
@@ -369,29 +441,22 @@ int session_endecaps_tx_est_req(struct vty *vty, const char **argv, int argc)
 	} else {
 		f_teid_access_local = (struct osmo_pfcp_ie_f_teid){
 			.fixed = {
-				.teid = session->access.teid.local,
+				.teid = session->tunend.access.local.teid,
 				.ip_addr = {
 					.v4_present = true,
-					.v4 = osmo_pfcp_endpoint_get_cfg(g_pfcp_tool->ep)->local_addr,
 				},
 			},
 		};
-		if (osmo_sockaddr_str_to_sockaddr(&session->access.gtp_ip, &f_teid_access_local.fixed.ip_addr.v4.u.sas)) {
-			vty_out(vty, "Error in GTP IP towards Access%s", VTY_NEWLINE);
-			return CMD_WARNING;
-		}
+		STR_TO_ADDR(f_teid_access_local.fixed.ip_addr.v4, session->tunend.access.local.addr);
 	}
 
 	ohc_access = (struct osmo_pfcp_ie_outer_header_creation){
 		.teid_present = true,
-		.teid = session->access.teid.remote,
+		.teid = session->tunend.access.remote.teid,
 		.ip_addr.v4_present = true,
 	};
 	osmo_pfcp_bits_set(ohc_access.desc_bits, OSMO_PFCP_OUTER_HEADER_CREATION_GTP_U_UDP_IPV4, true);
-	if (osmo_sockaddr_str_to_sockaddr(&session->access.gtp_ip, &ohc_access.ip_addr.v4.u.sas)) {
-		vty_out(vty, "Error in GTP IP towards Access%s", VTY_NEWLINE);
-		return CMD_WARNING;
-	}
+	STR_TO_ADDR(ohc_access.ip_addr.v4, session->tunend.access.remote.addr);
 
 	cp_f_seid = (struct osmo_pfcp_ie_f_seid){
 		.seid = session->cp_seid,
@@ -500,7 +565,7 @@ int session_tunmap_tx_est_req(struct vty *vty, const char **argv, int argc)
 	else
 		osmo_pfcp_bits_set(aa.bits, OSMO_PFCP_APPLY_ACTION_FORW, true);
 
-	if (session->access.teid.local == 0) {
+	if (session->tunmap.access.local.teid == 0) {
 		f_teid_access_local = (struct osmo_pfcp_ie_f_teid){
 			.choose_flag = true,
 			.choose = {
@@ -510,31 +575,25 @@ int session_tunmap_tx_est_req(struct vty *vty, const char **argv, int argc)
 	} else {
 		f_teid_access_local = (struct osmo_pfcp_ie_f_teid){
 			.fixed = {
-				.teid = session->access.teid.local,
+				.teid = session->tunmap.access.local.teid,
 				.ip_addr = {
 					.v4_present = true,
 					.v4 = osmo_pfcp_endpoint_get_cfg(g_pfcp_tool->ep)->local_addr,
 				},
 			},
 		};
-		if (osmo_sockaddr_str_to_sockaddr(&session->access.gtp_ip, &f_teid_access_local.fixed.ip_addr.v4.u.sas)) {
-			vty_out(vty, "Error in GTP IP towards Access%s", VTY_NEWLINE);
-			return CMD_WARNING;
-		}
+		STR_TO_ADDR(f_teid_access_local.fixed.ip_addr.v4, session->tunmap.access.local.addr);
 	}
 
 	ohc_access = (struct osmo_pfcp_ie_outer_header_creation){
 		.teid_present = true,
-		.teid = session->access.teid.remote,
+		.teid = session->tunmap.access.remote.teid,
 		.ip_addr.v4_present = true,
 	};
 	osmo_pfcp_bits_set(ohc_access.desc_bits, OSMO_PFCP_OUTER_HEADER_CREATION_GTP_U_UDP_IPV4, true);
-	if (osmo_sockaddr_str_to_sockaddr(&session->access.gtp_ip, &ohc_access.ip_addr.v4.u.sas)) {
-		vty_out(vty, "Error in GTP IP towards Access%s", VTY_NEWLINE);
-		return CMD_WARNING;
-	}
+	STR_TO_ADDR(ohc_access.ip_addr.v4, session->tunmap.access.remote.addr);
 
-	if (session->core.teid.local == 0) {
+	if (session->tunmap.core.local.teid == 0) {
 		f_teid_core_local = (struct osmo_pfcp_ie_f_teid){
 			.choose_flag = true,
 			.choose = {
@@ -544,28 +603,21 @@ int session_tunmap_tx_est_req(struct vty *vty, const char **argv, int argc)
 	} else {
 		f_teid_core_local = (struct osmo_pfcp_ie_f_teid){
 			.fixed = {
-				.teid = session->core.teid.local,
+				.teid = session->tunmap.core.local.teid,
 				.ip_addr = {
 					.v4_present = true,
-					.v4 = osmo_pfcp_endpoint_get_cfg(g_pfcp_tool->ep)->local_addr,
 				},
 			},
 		};
-		if (osmo_sockaddr_str_to_sockaddr(&session->core.gtp_ip, &f_teid_core_local.fixed.ip_addr.v4.u.sas)) {
-			vty_out(vty, "Error in GTP IP towards Core%s", VTY_NEWLINE);
-			return CMD_WARNING;
-		}
+		STR_TO_ADDR(f_teid_core_local.fixed.ip_addr.v4, session->tunmap.core.local.addr);
 	}
 	ohc_core = (struct osmo_pfcp_ie_outer_header_creation){
 		.teid_present = true,
-		.teid = session->core.teid.remote,
+		.teid = session->tunmap.core.remote.teid,
 		.ip_addr.v4_present = true,
 	};
 	osmo_pfcp_bits_set(ohc_core.desc_bits, OSMO_PFCP_OUTER_HEADER_CREATION_GTP_U_UDP_IPV4, true);
-	if (osmo_sockaddr_str_to_sockaddr(&session->core.gtp_ip, &ohc_core.ip_addr.v4.u.sas)) {
-		vty_out(vty, "Error in GTP IP towards Core%s", VTY_NEWLINE);
-		return CMD_WARNING;
-	}
+	STR_TO_ADDR(ohc_core.ip_addr.v4, session->tunmap.core.remote.addr);
 
 	cp_f_seid = (struct osmo_pfcp_ie_f_seid){
 		.seid = session->cp_seid,
@@ -653,9 +705,9 @@ DEFUN(session_tx_est_req, session_tx_est_req_cmd,
       "Set FAR to DROP = 1\n")
 {
 	struct pfcp_tool_session *session = vty->index;
-	switch (session->gtp_action) {
-	case UP_GTP_U_ENDECAPS:
-		return session_endecaps_tx_est_req(vty, argv, argc);
+	switch (session->kind) {
+	case UP_GTP_U_TUNEND:
+		return session_tunend_tx_est_req(vty, argv, argc);
 	case UP_GTP_U_TUNMAP:
 		return session_tunmap_tx_est_req(vty, argv, argc);
 	default:
@@ -776,12 +828,13 @@ void pfcp_tool_vty_init_cmds()
 	install_element(PEER_NODE, &peer_retrans_req_cmd);
 
 	install_element(PEER_NODE, &session_cmd);
+	install_element(PEER_NODE, &session_endecaps_cmd);
 	install_node(&session_node, NULL);
 	install_element(SESSION_NODE, &c_sleep_cmd);
 	install_element(SESSION_NODE, &session_tx_est_req_cmd);
 	install_element(SESSION_NODE, &session_tx_mod_req_cmd);
 	install_element(SESSION_NODE, &session_tx_del_req_cmd);
 	install_element(SESSION_NODE, &s_ue_cmd);
-	install_element(SESSION_NODE, &s_gtp_cmd);
-	install_element(SESSION_NODE, &s_teid_cmd);
+	install_element(SESSION_NODE, &s_f_teid_cmd);
+	install_element(SESSION_NODE, &s_f_teid_choose_cmd);
 }

src/osmo-upf/Makefile.am

@@ -30,6 +30,7 @@ bin_PROGRAMS = \
 	$(NULL)
 
 osmo_upf_SOURCES = \
+	netinst.c \
 	osmo_upf_main.c \
 	up_endpoint.c \
 	up_gtp_action.c \

src/osmo-upf/netinst.c

@@ -0,0 +1,124 @@
+/*
+ * (C) 2022 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+ * All Rights Reserved.
+ *
+ * Author: Neels Janosch Hofmeyr <nhofmeyr@sysmocom.de>
+ *
+ * SPDX-License-Identifier: GPL-2.0+
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include <string.h>
+
+#include <osmocom/core/talloc.h>
+#include <osmocom/core/logging.h>
+#include <osmocom/vty/vty.h>
+
+#include <osmocom/upf/netinst.h>
+
+/* Add a new netinst entry to the given list.
+ * \param ctx  talloc allocate new entry from ctx.
+ * \param list  append to this list.
+ * \param name  The Network Instance name as given in PFCP Network Instance IEs.
+ * \param addr  IP address string of local interface to associate with the Network Instance.
+ * \param errmsg  On error, an error description is returned in this out-argument.
+ * \return new network_instance entry, or NULL on error.
+ */
+const struct network_instance *netinst_add(void *ctx, struct llist_head *list, const char *name, const char *addr,
+					   const char **errmsg)
+{
+	struct network_instance *netinst;
+	if (errmsg)
+		*errmsg = NULL;
+
+	if (!name || !*name) {
+		if (errmsg)
+			*errmsg = "Network Instance name must not be empty";
+		return NULL;
+	}
+
+	if (netinst_find(list, name)) {
+		if (errmsg)
+			*errmsg = "Network Instance entry with this name already exists";
+		return NULL;
+	}
+
+	netinst = talloc(ctx, struct network_instance);
+	*netinst = (struct network_instance){
+		.name = talloc_strdup(netinst, name),
+	};
+	if (osmo_sockaddr_str_from_str(&netinst->addr, addr, 0)) {
+		if (errmsg)
+			*errmsg = "Network Instance address is not a valid IP address string";
+		talloc_free(netinst);
+		return NULL;
+	}
+
+	llist_add_tail(&netinst->entry, list);
+
+	return netinst;
+}
+
+const struct network_instance *netinst_find(struct llist_head *list, const char *name)
+{
+	const struct network_instance *netinst;
+
+	if (!name)
+		return NULL;
+
+	llist_for_each_entry(netinst, list, entry)
+		if (!strcmp(netinst->name, name))
+			return netinst;
+
+	return NULL;
+}
+
+const struct network_instance *netinst_first(struct llist_head *list)
+{
+	return llist_first_entry_or_null(list, struct network_instance, entry);
+}
+
+/* Clear the list of Network Instance entries, return the nr of entries that were removed. */
+int netinst_clear(struct llist_head *list)
+{
+	int count = 0;
+	while (1) {
+		struct network_instance *netinst = llist_first_entry_or_null(list, struct network_instance, entry);
+		if (!netinst)
+			break;
+		llist_del(&netinst->entry);
+		talloc_free(netinst);
+		count++;
+	}
+	return count;
+}
+
+/* Write one or all netinst entries to the VTY output.
+ * If name_or_null is NULL, print all entries. Else, print only the entry matching that name.
+ * Return number of printed entries. */
+int netinst_vty_write(struct vty *vty, struct llist_head *list, const char *indent, const char *name_or_null)
+{
+	const struct network_instance *netinst;
+	int count = 0;
+
+	llist_for_each_entry(netinst, list, entry) {
+		if (name_or_null && strcmp(netinst->name, name_or_null))
+			continue;
+		vty_out(vty, "%sadd %s %s%s", indent, netinst->name, netinst->addr.ip, VTY_NEWLINE);
+		count++;
+	}
+	return count;
+}

src/osmo-upf/osmo_upf_main.c

@@ -304,13 +304,13 @@ int main(int argc, char **argv)
 		return 1;
 	}
 
-	/* start telnet, after reading config for vty_get_bind_addr() */
-	rc = telnet_init_dynif(tall_upf_ctx, &g_upf, vty_get_bind_addr(), OSMO_VTY_PORT_UPF);
+	/* start telnet VTY */
+	rc = telnet_init_default(tall_upf_ctx, &g_upf, OSMO_VTY_PORT_UPF);
 	if (rc < 0)
 		return 2;
 
 	/* start control interface, after reading config for ctrl_vty_get_bind_addr() */
-	g_upf->ctrl = ctrl_interface_setup_dynip(g_upf, ctrl_vty_get_bind_addr(), OSMO_CTRL_PORT_UPF, NULL);
+	g_upf->ctrl = ctrl_interface_setup(g_upf, OSMO_CTRL_PORT_UPF, NULL);
 	if (!g_upf->ctrl) {
 		fprintf(stderr, "Failed to initialize control interface. Exiting.\n");
 		return -1;
@@ -331,9 +331,6 @@ int main(int argc, char **argv)
 		}
 	}
 
-	if (upf_gtp_genl_open())
-		return -1;
-
 	if (upf_gtp_devs_open())
 		return -1;
 

src/osmo-upf/up_endpoint.c

@@ -38,7 +38,7 @@ static void up_endpoint_set_msg_ctx(struct osmo_pfcp_endpoint *ep, struct osmo_p
 		if (!m->ctx.peer_fi && req->ctx.peer_fi)
 			up_peer_set_msg_ctx(req->ctx.peer_fi->priv, m);
 		if (!m->ctx.session_fi && req->ctx.session_fi)
-			up_session_set_msg_ctx(req->ctx.peer_fi->priv, m);
+			up_session_set_msg_ctx(req->ctx.session_fi->priv, m);
 	}
 
 	/* From the remote address, find the matching peer instance */
@@ -221,6 +221,12 @@ static void up_endpoint_rx_cb(struct osmo_pfcp_endpoint *ep, struct osmo_pfcp_ms
 	case OSMO_PFCP_MSGT_SESSION_REP_REQ:
 		up_ep_rx_session_rep_req(up_ep, m);
 		return;
+	case OSMO_PFCP_MSGT_HEARTBEAT_REQ:
+	case OSMO_PFCP_MSGT_HEARTBEAT_RESP:
+		/* Heartbeat is already handled in osmo_pfcp_endpoint_handle_rx() in pfcp_endpoint.c. The heartbeat
+		 * messages are also dispatched here, to the rx_cb, "on informtional basis", nothing needs to happen
+		 * here. */
+		return;
 	default:
 		OSMO_LOG_PFCP_MSG(m, LOGL_ERROR, "Unknown message type\n");
 		return;

src/osmo-upf/up_gtp_action.c

@@ -47,15 +47,15 @@ int up_gtp_action_cmp(const struct up_gtp_action *a, const struct up_gtp_action
 		return cmp;
 
 	switch (a->kind) {
-	case UP_GTP_U_ENDECAPS:
-		if ((cmp = CMP_MEMB(endecaps.local_teid)))
+	case UP_GTP_U_TUNEND:
+		if ((cmp = CMP_MEMB(tunend.access.local_teid)))
 			return cmp;
-		if ((cmp = CMP_MEMB(endecaps.remote_teid)))
+		if ((cmp = CMP_MEMB(tunend.access.remote_teid)))
 			return cmp;
-		cmp = osmo_sockaddr_cmp(&a->endecaps.gtp_remote_addr, &b->endecaps.gtp_remote_addr);
+		cmp = osmo_sockaddr_cmp(&a->tunend.access.gtp_remote_addr, &b->tunend.access.gtp_remote_addr);
 		if (cmp)
 			return cmp;
-		cmp = osmo_sockaddr_cmp(&a->endecaps.ue_addr, &b->endecaps.ue_addr);
+		cmp = osmo_sockaddr_cmp(&a->tunend.core.ue_local_addr, &b->tunend.core.ue_local_addr);
 		if (cmp)
 			return cmp;
 		break;
@@ -79,70 +79,61 @@ int up_gtp_action_cmp(const struct up_gtp_action *a, const struct up_gtp_action
 static int up_gtp_action_enable_disable(struct up_gtp_action *a, bool enable)
 {
 	struct upf_gtp_dev *gtp_dev;
+	const struct osmo_sockaddr *gtp_addr;
 	int rc;
 
 	switch (a->kind) {
-	case UP_GTP_U_ENDECAPS:
+	case UP_GTP_U_TUNEND:
 		if (g_upf->gtp.mockup) {
-			LOG_UP_GTP_ACTION(a, LOGL_NOTICE, "gtp/mockup active, skipping GTP action %s\n",
+			LOG_UP_GTP_ACTION(a, LOGL_NOTICE, "tunend/mockup active, skipping GTP action %s\n",
 					  enable ? "enable" : "disable");
 			return 0;
 		}
 
-		/* use the first available GTP device.
-		 * TODO: select by interface name?
-		 */
-		gtp_dev = upf_gtp_dev_first();
+		/* Pick GTP device matching the local F-TEID set up for the GTP tunnel (it is on the Access side) */
+		gtp_addr = &a->tunend.access.gtp_local_addr;
+		gtp_dev = upf_gtp_dev_find_by_local_addr(gtp_addr);
 		if (!gtp_dev) {
-			LOG_UP_GTP_ACTION(a, LOGL_ERROR, "No GTP device open, cannot %s\n", enable ? "enable" : "disable");
+			LOG_UP_GTP_ACTION(a, LOGL_ERROR, "No GTP device open for local address %s, cannot %s"
+					  " -- consider configuring 'tunend' / 'dev (create|use) foo %s'\n",
+					  osmo_sockaddr_to_str_c(OTC_SELECT, gtp_addr),
+					  enable ? "enable" : "disable",
+					  osmo_sockaddr_to_str_c(OTC_SELECT, gtp_addr));
 			return -EIO;
 		}
 
 		if (enable)
-			rc = upf_gtp_dev_tunnel_add(gtp_dev, &a->endecaps);
+			rc = upf_gtp_dev_tunend_add(gtp_dev, &a->tunend);
 		else
-			rc = upf_gtp_dev_tunnel_del(gtp_dev, &a->endecaps);
+			rc = upf_gtp_dev_tunend_del(gtp_dev, &a->tunend);
 		if (rc) {
-			LOG_UP_GTP_ACTION(a, LOGL_ERROR, "Failed to %s GTP tunnel: %d %s\n",
-					  enable ? "enable" : "disable", rc, strerror(-rc));
+			LOG_UP_GTP_ACTION(a, LOGL_ERROR, "Failed to %s GTP tunnel (rc=%d)\n",
+					  enable ? "enable" : "disable", rc);
 			return rc;
 		}
-		LOG_UP_GTP_ACTION(a, LOGL_NOTICE, "%s GTP tunnel\n", enable ? "Enabled" : "Disabled");
+		LOG_UP_GTP_ACTION(a, LOGL_NOTICE, "%s tunend on dev %s\n", enable ? "Enabled" : "Disabled",
+				  gtp_dev->name);
 		return 0;
 
 	case UP_GTP_U_TUNMAP:
 		if (g_upf->nft.mockup) {
-			LOG_UP_GTP_ACTION(a, LOGL_NOTICE, "nft/mockup active, skipping nftables ruleset %s\n",
+			LOG_UP_GTP_ACTION(a, LOGL_NOTICE, "tunmap/mockup active, skipping nftables ruleset %s\n",
 					  enable ? "enable" : "disable");
 			return 0;
 		}
 
-		if (enable && a->tunmap.id != 0) {
-			LOG_UP_GTP_ACTION(a, LOGL_ERROR,
-					  "Cannot enable: nft GTP tunnel mapping rule has been enabled before"
-					  " as " NFT_CHAIN_NAME_PREFIX_TUNMAP "%u\n", a->tunmap.id);
-			return -EALREADY;
-		}
-		if (!enable && a->tunmap.id == 0) {
-			LOG_UP_GTP_ACTION(a, LOGL_ERROR,
-					  "Cannot disable: nft GTP tunnel mapping rule has not been enabled"
-					  " (no " NFT_CHAIN_NAME_PREFIX_TUNMAP " id)\n");
-			return -ENOENT;
-		}
 		if (enable)
 			rc = upf_nft_tunmap_create(&a->tunmap);
 		else
 			rc = upf_nft_tunmap_delete(&a->tunmap);
 		if (rc) {
-			LOG_UP_GTP_ACTION(a, LOGL_ERROR,
-					  "Failed to %s nft GTP tunnel mapping " NFT_CHAIN_NAME_PREFIX_TUNMAP "%u:"
-					  " %d %s\n", enable ? "enable" : "disable", a->tunmap.id, rc, strerror(-rc));
+			LOG_UP_GTP_ACTION(a, LOGL_ERROR, "Failed to %s nft GTP tunnel mapping (rc=%d)\n",
+					  enable ? "enable" : "disable", rc);
 			return rc;
 		}
-		LOG_UP_GTP_ACTION(a, LOGL_NOTICE, "%s nft GTP tunnel mapping " NFT_CHAIN_NAME_PREFIX_TUNMAP "%u\n",
-				  enable ? "Enabled" : "Disabled", a->tunmap.id);
-		if (!enable)
-			a->tunmap.id = 0;
+		LOG_UP_GTP_ACTION(a, LOGL_NOTICE, "%s tunmap, nft chain IDs: access--%u-> <-%u--core\n",
+				  enable ? "Enabled" : "Disabled",
+				  a->tunmap.access.chain_id, a->tunmap.core.chain_id);
 		return 0;
 
 	default:
@@ -165,21 +156,28 @@ int up_gtp_action_to_str_buf(char *buf, size_t buflen, const struct up_gtp_actio
 {
 	struct osmo_strbuf sb = { .buf = buf, .len = buflen };
 	switch (a->kind) {
-	case UP_GTP_U_ENDECAPS:
-		OSMO_STRBUF_PRINTF(sb, "GTP:endecaps GTP-access:");
-		OSMO_STRBUF_APPEND(sb, osmo_sockaddr_to_str_buf2, &a->endecaps.gtp_remote_addr);
-		OSMO_STRBUF_PRINTF(sb, " TEID-r:0x%"PRIx32" TEID-l:0x%"PRIx32" IP-core:",
-				   a->endecaps.remote_teid, a->endecaps.local_teid);
-		OSMO_STRBUF_APPEND(sb, osmo_sockaddr_to_str_buf2, &a->endecaps.ue_addr);
+	case UP_GTP_U_TUNEND:
+		OSMO_STRBUF_PRINTF(sb, "GTP:tunend GTP-access-r:");
+		OSMO_STRBUF_APPEND(sb, osmo_sockaddr_to_str_buf2, &a->tunend.access.gtp_remote_addr);
+		OSMO_STRBUF_PRINTF(sb, " TEID-access-r:0x%"PRIx32, a->tunend.access.remote_teid);
+		OSMO_STRBUF_PRINTF(sb, " GTP-access-l:");
+		OSMO_STRBUF_APPEND(sb, osmo_sockaddr_to_str_buf2, &a->tunend.access.gtp_local_addr);
+		OSMO_STRBUF_PRINTF(sb, " TEID-access-l:0x%"PRIx32" IP-core-l:", a->tunend.access.local_teid);
+		OSMO_STRBUF_APPEND(sb, osmo_sockaddr_to_str_buf2, &a->tunend.core.ue_local_addr);
 		break;
 	case UP_GTP_U_TUNMAP:
-		OSMO_STRBUF_PRINTF(sb, "GTP:tunmap GTP-access:");
+		OSMO_STRBUF_PRINTF(sb, "GTP:tunmap GTP-access-r:");
 		OSMO_STRBUF_APPEND(sb, osmo_sockaddr_to_str_buf2, &a->tunmap.access.gtp_remote_addr);
-		OSMO_STRBUF_PRINTF(sb, " TEID-access-r:0x%"PRIx32" TEID-access-l:0x%"PRIx32" GTP-core:",
-				   a->tunmap.access.remote_teid, a->tunmap.access.local_teid);
+		OSMO_STRBUF_PRINTF(sb, " TEID-access-r:0x%"PRIx32, a->tunmap.access.remote_teid);
+		OSMO_STRBUF_PRINTF(sb, " GTP-access-l:");
+		OSMO_STRBUF_APPEND(sb, osmo_sockaddr_to_str_buf2, &a->tunmap.access.gtp_local_addr);
+		OSMO_STRBUF_PRINTF(sb, " TEID-access-l:0x%"PRIx32, a->tunmap.access.local_teid);
+		OSMO_STRBUF_PRINTF(sb, " GTP-core-r:");
 		OSMO_STRBUF_APPEND(sb, osmo_sockaddr_to_str_buf2, &a->tunmap.core.gtp_remote_addr);
-		OSMO_STRBUF_PRINTF(sb, " TEID-core-r:0x%"PRIx32" TEID-core-l:0x%"PRIx32,
-				   a->tunmap.core.remote_teid, a->tunmap.core.local_teid);
+		OSMO_STRBUF_PRINTF(sb, " TEID-core-r:0x%"PRIx32, a->tunmap.core.remote_teid);
+		OSMO_STRBUF_PRINTF(sb, " GTP-core-l:");
+		OSMO_STRBUF_APPEND(sb, osmo_sockaddr_to_str_buf2, &a->tunmap.core.gtp_local_addr);
+		OSMO_STRBUF_PRINTF(sb, " TEID-core-l:0x%"PRIx32, a->tunmap.core.local_teid);
 		break;
 	case UP_GTP_DROP:
 		OSMO_STRBUF_PRINTF(sb, "GTP:drop");
@@ -189,9 +187,10 @@ int up_gtp_action_to_str_buf(char *buf, size_t buflen, const struct up_gtp_actio
 		break;
 	}
 	if (a->session)
-		OSMO_STRBUF_PRINTF(sb, " PFCP-peer:%s SEID-l:0x%"PRIx64" PDR:%d,%d",
-				   up_peer_remote_addr_str(a->session->up_peer),
-				   a->session->up_seid, a->pdr_core, a->pdr_access);
+		OSMO_STRBUF_PRINTF(sb, " PFCP-peer:%s SEID-l:0x%"PRIx64,
+				   up_peer_remote_addr_str(a->session->up_peer), a->session->up_seid);
+	OSMO_STRBUF_PRINTF(sb, " PDR-access:%d", a->pdr_access);
+	OSMO_STRBUF_PRINTF(sb, " PDR-core:%d", a->pdr_core);
 	return sb.chars_needed;
 }
 

src/osmo-upf/up_peer.c

@@ -169,11 +169,6 @@ struct up_peer *up_peer_find_or_add(struct up_endpoint *up_endpoint, const struc
 	return up_peer_add(up_endpoint, remote_addr);
 }
 
-int up_peer_tx(struct up_peer *peer, struct osmo_pfcp_msg *m)
-{
-	return osmo_pfcp_endpoint_tx(peer->up_endpoint->pfcp_ep, m);
-}
-
 static int up_peer_fsm_timer_cb(struct osmo_fsm_inst *fi)
 {
 	//struct up_peer *peer = fi->priv;
@@ -188,7 +183,7 @@ void up_peer_set_msg_ctx(struct up_peer *peer, struct osmo_pfcp_msg *m)
 	m->ctx.peer_fi = peer->fi;
 	m->ctx.peer_use_count = &peer->use_count;
 	m->ctx.peer_use_token = (m->rx ? UP_USE_MSG_RX : UP_USE_MSG_TX);
-	osmo_use_count_get_put(m->ctx.peer_use_count, m->ctx.peer_use_token, 1);
+	OSMO_ASSERT(osmo_use_count_get_put(m->ctx.peer_use_count, m->ctx.peer_use_token, 1) == 0);
 }
 
 struct osmo_pfcp_msg *up_peer_init_tx(struct up_peer *peer, struct osmo_pfcp_msg *in_reply_to,
@@ -217,7 +212,8 @@ static int up_peer_tx_assoc_setup_resp(struct up_peer *peer, struct osmo_pfcp_ms
 	};
 
 	if (osmo_pfcp_endpoint_tx(peer->up_endpoint->pfcp_ep, resp)) {
-		OSMO_LOG_PFCP_MSG(resp, LOGL_ERROR, "Error sending response, cannot associate with peer\n");
+		OSMO_LOG_PFCP_MSG(m, LOGL_ERROR, "Error sending response to this message,"
+				  " cannot associate with peer\n");
 		return -EIO;
 	}
 	return 0;
@@ -234,7 +230,7 @@ static int up_peer_tx_assoc_rel_resp(struct up_peer *peer, struct osmo_pfcp_msg
 	};
 
 	if (osmo_pfcp_endpoint_tx(peer->up_endpoint->pfcp_ep, resp)) {
-		OSMO_LOG_PFCP_MSG(resp, LOGL_ERROR, "Error sending response\n");
+		OSMO_LOG_PFCP_MSG(m, LOGL_ERROR, "Error sending response to this message\n");
 		return -EIO;
 	}
 	return 0;
@@ -326,6 +322,8 @@ nack_response:
 		.cause = cause,
 	};
 	osmo_pfcp_endpoint_tx(peer->up_endpoint->pfcp_ep, resp);
+	if (session)
+		up_session_discard(session);
 }
 
 static void up_peer_not_associated_action(struct osmo_fsm_inst *fi, uint32_t event, void *data)

src/osmo-upf/up_session.c

@@ -34,6 +34,7 @@
 #include <osmocom/upf/up_peer.h>
 #include <osmocom/upf/up_session.h>
 #include <osmocom/upf/up_gtp_action.h>
+#include <osmocom/upf/netinst.h>
 
 static enum osmo_pfcp_cause up_session_setup_gtp(struct up_session *session);
 
@@ -47,7 +48,7 @@ void up_session_set_msg_ctx(struct up_session *session, struct osmo_pfcp_msg *m)
 	m->ctx.session_fi = session->fi;
 	m->ctx.session_use_count = &session->use_count;
 	m->ctx.session_use_token = (m->rx ? UP_USE_MSG_RX : UP_USE_MSG_TX);
-	osmo_use_count_get_put(m->ctx.session_use_count, m->ctx.session_use_token, 1);
+	OSMO_ASSERT(osmo_use_count_get_put(m->ctx.session_use_count, m->ctx.session_use_token, 1) == 0);
 }
 
 enum up_session_fsm_state {
@@ -114,11 +115,75 @@ struct chosen_f_teid *chosen_f_teid_find(struct llist_head *list, uint8_t choose
 	return NULL;
 }
 
+/* Find local interface's IP address by Network Instance name. Return 0 on success, or an OSMO_PFCP_CAUSE_* value on
+ * failure. */
+static int up_session_choose_local_ip(struct up_session *session, struct osmo_pfcp_ip_addrs *local_addr,
+				      const char *netinst_name)
+{
+	const struct network_instance *netinst;
+	struct osmo_sockaddr osa = {};
+
+	if (llist_empty(&g_upf->netinst)) {
+		/* No network instances are configured in osmo-upf.cfg. Instead use the local address configured for
+		 * PFCP, assuming that in a simplistic setup the host has only one interface. It is unlikely to be
+		 * useful for a production environment where the entire point is to hand packet data from one interface
+		 * to another, and where PFCP most probably happens on an entirely different interface, but may make
+		 * things simpler for lab testing. */
+		if (osmo_pfcp_ip_addrs_set(local_addr,
+					   osmo_pfcp_endpoint_get_local_addr(session->up_peer->up_endpoint->pfcp_ep))) {
+			LOGPFSML(session->fi, LOGL_ERROR, "Invalid local address in pfcp_endpoint cfg\n");
+			return OSMO_PFCP_CAUSE_SYSTEM_FAILURE;
+		}
+		LOGPFSML(session->fi, LOGL_NOTICE,
+			 "Cannot look up Network Instance %s: No 'netinst' is configured, setting up GTP on same local"
+			 " interface as PFCP: %s (makes sense only for lab testing)\n",
+			 osmo_quote_str_c(OTC_SELECT, netinst_name, -1),
+			 osmo_pfcp_ip_addrs_to_str_c(OTC_SELECT, local_addr));
+		return 0;
+	}
+
+	if (!netinst_name || !*netinst_name) {
+		/* Empty or no Network Instance IE in incoming PFCP request. Pick the first network instance; makes
+		 * sense only in a simplistic lab setup where packet data is forwarded to the same interface that it is
+		 * received on, and where no Network Instance is indicated by the CPF. Warn if more than one network
+		 * instance is configured to choose from. */
+		if (llist_count(&g_upf->netinst) > 1)
+			LOGPFSML(session->fi, LOGL_NOTICE,
+				 "Missing Network Instance in incoming request, using the first 'netinst' from cfg\n");
+		netinst = netinst_first(&g_upf->netinst);
+		/* there has to be a first entry, because we handled the empty list above. */
+		OSMO_ASSERT(netinst);
+	} else {
+		netinst = netinst_find(&g_upf->netinst, netinst_name);
+		if (!netinst) {
+			LOGPFSML(session->fi, LOGL_ERROR, "Network Instance from PFCP request not found: %s"
+				 " -- ensure there is a 'netinst' / 'add %s <ip-addr>' entry in your config\n",
+				 osmo_quote_str_c(OTC_SELECT, netinst_name, -1),
+				 osmo_escape_str_c(OTC_SELECT, netinst_name, -1));
+			return OSMO_PFCP_CAUSE_RULE_CREATION_MOD_FAILURE;
+		}
+	}
+
+	/* Convert netinst IP address string first to osmo_sockaddr and then to osmo_pfcp_ip_addrs. */
+	if (osmo_sockaddr_str_to_sockaddr(&netinst->addr, &osa.u.sas)
+	    || osmo_pfcp_ip_addrs_set(local_addr, &osa)) {
+		LOGPFSML(session->fi, LOGL_ERROR,
+			 "Network Instance %s from PFCP request yields no valid IP address: "
+				OSMO_SOCKADDR_STR_FMT "\n",
+			 osmo_quote_str_c(OTC_SELECT, netinst_name, -1),
+			 OSMO_SOCKADDR_STR_FMT_ARGS(&netinst->addr));
+		return OSMO_PFCP_CAUSE_RULE_CREATION_MOD_FAILURE;
+	}
+	return 0;
+}
+
 /* Choose an F-TEID (when the peer has sent CHOOSE = 1).
  * If the peer also sent a CHOOSE_ID, then remember this F-TEID choice under the given ID, and re-use that choice when
- * the same ID re-appears. The chosen IDs are saved in session->chosen_f_teids. */
+ * the same ID re-appears. The chosen IDs are saved in session->chosen_f_teids.
+ * Return 0 on success, or an OSMO_PFCP_CAUSE_* value on failure. */
 static enum osmo_pfcp_cause up_session_choose_f_teid(struct up_session *session, struct osmo_pfcp_ie_f_teid *dst,
-						     bool choose_id_present, uint8_t choose_id)
+						     bool choose_id_present, uint8_t choose_id,
+						     const char *netinst_name)
 {
 	struct up_endpoint *up_ep = session->up_peer->up_endpoint;
 	struct chosen_f_teid *chosen = NULL;
@@ -129,23 +194,26 @@ static enum osmo_pfcp_cause up_session_choose_f_teid(struct up_session *session,
 		/* Re-use a previous F-TEID */
 		*dst = chosen->f_teid;
 	} else {
-		/* Choose a new F-TEID */
+		int rc;
+
 		*dst = (struct osmo_pfcp_ie_f_teid){
-			.fixed = {
-				.teid = up_endpoint_next_teid(up_ep),
-			},
+			.choose_flag = false,
 		};
+
+		/* Determine local IP address from Network Instance value received in PFCP request */
+		rc = up_session_choose_local_ip(session, &dst->fixed.ip_addr, netinst_name);
+		if (rc)
+			return rc;
+
+		/* Choose a new TEID */
+		dst->fixed.teid = up_endpoint_next_teid(up_ep);
 		if (dst->fixed.teid == 0) {
 			LOGPFSML(session->fi, LOGL_ERROR, "Failed to allocate an unused TEID\n");
 			return OSMO_PFCP_CAUSE_PFCP_ENTITY_IN_CONGESTION;
 		}
-		LOGPFSML(session->fi, LOGL_INFO, "Allocated new local TEID 0x%x\n", dst->fixed.teid);
+		LOGPFSML(session->fi, LOGL_INFO, "Allocated new local F-TEID %s\n",
+			 osmo_pfcp_ie_f_teid_to_str_c(OTC_SELECT, dst));
 
-		if (osmo_pfcp_ip_addrs_set(&dst->fixed.ip_addr,
-					   osmo_pfcp_endpoint_get_local_addr(up_ep->pfcp_ep))) {
-			LOGPFSML(session->fi, LOGL_ERROR, "Invalid local address in pfcp_endpoint cfg\n");
-			return OSMO_PFCP_CAUSE_PFCP_ENTITY_IN_CONGESTION;
-		}
 		/* Save this choice */
 		if (choose_id_present) {
 			chosen = talloc(session, struct chosen_f_teid);
@@ -189,22 +257,23 @@ static void far_upd(struct far *far, const struct osmo_pfcp_ie_upd_far *upd)
 	if (upd->upd_forw_params_present) {
 		const struct osmo_pfcp_ie_upd_forw_params *u = &upd->upd_forw_params;
 		struct osmo_pfcp_ie_forw_params *p = &far->desc.forw_params;
+		far->desc.forw_params_present = true;
 		if (u->destination_iface_present)
 			p->destination_iface = u->destination_iface;
 		if (u->network_inst_present) {
-			p->network_inst = p->network_inst;
+			p->network_inst = u->network_inst;
 			p->network_inst_present = true;
 		}
 		if (u->outer_header_creation_present) {
-			p->outer_header_creation = p->outer_header_creation;
+			p->outer_header_creation = u->outer_header_creation;
 			p->outer_header_creation_present = true;
 		}
 		if (u->linked_te_id_present) {
-			p->linked_te_id = p->linked_te_id;
+			p->linked_te_id = u->linked_te_id;
 			p->linked_te_id_present = true;
 		}
 		if (u->destination_iface_type_present) {
-			p->destination_iface_type = p->destination_iface_type;
+			p->destination_iface_type = u->destination_iface_type;
 			p->destination_iface_type_present = true;
 		}
 	}
@@ -237,12 +306,9 @@ static int far_to_str_buf(char *buf, size_t len, const struct far *far)
 	if (f->forw_params_present) {
 		OSMO_STRBUF_PRINTF(sb, " dst:%s", osmo_pfcp_dest_iface_str(f->forw_params.destination_iface));
 		if (f->forw_params.outer_header_creation_present) {
-			OSMO_STRBUF_PRINTF(sb, " encaps-");
-			OSMO_STRBUF_APPEND(sb, osmo_pfcp_bits_to_str_buf,
-					   f->forw_params.outer_header_creation.desc_bits,
-					   osmo_pfcp_outer_header_creation_strs);
-			if (f->forw_params.outer_header_creation.teid_present)
-				OSMO_STRBUF_PRINTF(sb, " TEID-0x%x", f->forw_params.outer_header_creation.teid);
+			OSMO_STRBUF_PRINTF(sb, ",");
+			OSMO_STRBUF_APPEND(sb, osmo_pfcp_ie_outer_header_creation_to_str_buf,
+					   &f->forw_params.outer_header_creation);
 		}
 	}
 	OSMO_STRBUF_PRINTF(sb, "}");
@@ -273,6 +339,10 @@ int pdr_to_str_buf(char *buf, size_t buflen, const struct pdr *pdr)
 			OSMO_STRBUF_APPEND(sb, osmo_sockaddr_to_str_buf2, &pdr->desc.pdi.ue_ip_address.ip_addr.v6);
 		}
 	}
+	if (pdr->desc.pdi.network_inst_present) {
+		OSMO_STRBUF_PRINTF(sb, " netinst:");
+		OSMO_STRBUF_APPEND(sb, osmo_quote_str_buf3, pdr->desc.pdi.network_inst.str, -1);
+	}
 	if (pdr->local_f_teid) {
 		OSMO_STRBUF_PRINTF(sb, " ");
 		OSMO_STRBUF_APPEND(sb, osmo_pfcp_ie_f_teid_to_str_buf, pdr->local_f_teid);
@@ -316,6 +386,7 @@ static void pdr_set_far(struct pdr *pdr, struct far *far)
 	pdr->far = far;
 }
 
+/* Set up a new Packet Detection Rule, append the response to the end of the created_pdr/created_pdr_count array. */
 static struct pdr *pdr_create(struct up_session *session,
 			      const struct osmo_pfcp_ie_create_pdr *create_pdr,
 			      enum osmo_pfcp_cause *cause,
@@ -372,9 +443,13 @@ static struct pdr *pdr_create(struct up_session *session,
 		if (pdr->desc.pdi.local_f_teid.choose_flag) {
 			/* CHOOSE = 1: we need to pick our own local F-TEID */
 			struct osmo_pfcp_ie_f_teid local_f_teid;
+			const char *netinst_name = NULL;
+			if (pdr->desc.pdi.network_inst_present)
+				netinst_name = pdr->desc.pdi.network_inst.str;
 			*cause = up_session_choose_f_teid(session, &local_f_teid,
 							  pdr->desc.pdi.local_f_teid.choose.choose_id_present,
-							  pdr->desc.pdi.local_f_teid.choose.choose_id);
+							  pdr->desc.pdi.local_f_teid.choose.choose_id,
+							  netinst_name);
 			if (*cause != OSMO_PFCP_CAUSE_REQUEST_ACCEPTED) {
 				*offending_ie = OSMO_PFCP_IEI_F_TEID;
 				*offending_ie_present = true;
@@ -556,14 +631,20 @@ static void up_session_est(struct up_session *session, struct osmo_pfcp_msg *m)
 	resp->up_f_seid_present = true;
 
 	rc = osmo_pfcp_endpoint_tx(peer->up_endpoint->pfcp_ep, tx);
-	if (rc)
+	if (rc) {
+		/* sending ACK failed, discard session. It might seem like a good idea to keep the session around,
+		 * because the creation succeeded, only the ACK failed. But in the greater scheme of things, if we
+		 * cannot ACK to the PFCP peer, all is lost. Rather not keep stale sessions around. */
 		up_session_fsm_state_chg(UP_SESSION_ST_WAIT_USE_COUNT);
+		return;
+	}
 	up_session_fsm_state_chg(UP_SESSION_ST_ESTABLISHED);
 	return;
 
 nack_response:
 	resp->created_pdr_count = 0;
 	osmo_pfcp_endpoint_tx(peer->up_endpoint->pfcp_ep, tx);
+	/* No matter if sending the NACK succeeded or not, discard the session. */
 	up_session_fsm_state_chg(UP_SESSION_ST_WAIT_USE_COUNT);
 }
 
@@ -644,8 +725,13 @@ static void up_session_mod(struct up_session *session, struct osmo_pfcp_msg *m)
 		goto nack_response;
 
 	/* Success, send ACK */
-	if (osmo_pfcp_endpoint_tx(peer->up_endpoint->pfcp_ep, tx))
+	if (osmo_pfcp_endpoint_tx(peer->up_endpoint->pfcp_ep, tx)) {
+		/* sending ACK failed, discard session. It might seem like a good idea to keep the session around,
+		 * because the modification succeeded, only the ACK failed. But in the greater scheme of things, if we
+		 * cannot ACK to the PFCP peer, all is lost. Rather not keep stale sessions around. */
 		up_session_fsm_state_chg(UP_SESSION_ST_WAIT_USE_COUNT);
+		return;
+	}
 
 	LOGPFSML(fi, LOGL_NOTICE, "Session modified: %s\n", up_session_gtp_status(session));
 	return;
@@ -653,6 +739,7 @@ static void up_session_mod(struct up_session *session, struct osmo_pfcp_msg *m)
 nack_response:
 	resp->created_pdr_count = 0;
 	osmo_pfcp_endpoint_tx(peer->up_endpoint->pfcp_ep, tx);
+	/* No matter if sending the NACK succeeded or not, discard the session. */
 	up_session_fsm_state_chg(UP_SESSION_ST_WAIT_USE_COUNT);
 }
 
@@ -667,6 +754,7 @@ static void up_session_del(struct up_session *session, struct osmo_pfcp_msg *m)
 		.cause = OSMO_PFCP_CAUSE_REQUEST_ACCEPTED
 	};
 	osmo_pfcp_endpoint_tx(peer->up_endpoint->pfcp_ep, tx);
+	/* No matter if sending the deletion ACK succeeded or not, discard the session. */
 	up_session_fsm_state_chg(UP_SESSION_ST_WAIT_USE_COUNT);
 }
 
@@ -996,8 +1084,8 @@ static void pdr_classify(struct pdr *pdr)
 {
 	pdr->rx_decaps = false;
 	pdr->forw_encaps = false;
-	pdr->forw_to_core = false;
-	pdr->forw_from_core = false;
+	pdr->access_to_core = false;
+	pdr->core_to_access = false;
 	if (!pdr->far)
 		return;
 
@@ -1009,10 +1097,10 @@ static void pdr_classify(struct pdr *pdr)
 	if (!action_is_forw(&pdr->far->desc.apply_action))
 		return;
 
-	pdr->forw_to_core = (pdr->desc.pdi.source_iface == OSMO_PFCP_SOURCE_IFACE_ACCESS
+	pdr->access_to_core = (pdr->desc.pdi.source_iface == OSMO_PFCP_SOURCE_IFACE_ACCESS
 			       && pdr->far->desc.forw_params.destination_iface == OSMO_PFCP_DEST_IFACE_CORE);
 
-	pdr->forw_from_core = (pdr->desc.pdi.source_iface == OSMO_PFCP_SOURCE_IFACE_CORE
+	pdr->core_to_access = (pdr->desc.pdi.source_iface == OSMO_PFCP_SOURCE_IFACE_CORE
 			       && pdr->far->desc.forw_params.destination_iface == OSMO_PFCP_DEST_IFACE_ACCESS);
 }
 
@@ -1032,6 +1120,11 @@ void pdr_reverse_unset(struct pdr *pdr)
 	pdr->reverse_pdr = NULL;
 }
 
+/* Log that a PDR (and its reverse-PDR) is inactive.
+ * \param pdr  The Access-to-Core PDR.
+ * \param desc  Why it is inactive.
+ * \param pdr_to_str  The PDR that desc describes, can be pdr or the reverse Core-to-Access PDR.
+ */
 static void log_inactive_pdr_set(struct pdr *pdr, const char *desc, const struct pdr *pdr_to_str)
 {
 	struct pdr *rpdr = pdr->reverse_pdr;
@@ -1055,8 +1148,9 @@ static void log_inactive_pdr_set(struct pdr *pdr, const char *desc, const struct
  * The given PDR must have an outer-header-removal and a local F-TEID.
  * Its reverse-PDR must have a UE address flagged as "Destination" IP addr.
  * Its reverse-PDR's FAR must have an outer-header creation with a remote TEID.
+ * \param pdr  A rule detecting packets on Access, where pdr->reverse_pdr detects packets on Core.
  */
-static void add_gtp_action_endecaps(void *ctx, struct llist_head *dst, struct pdr *pdr)
+static void add_gtp_action_tunend(void *ctx, struct llist_head *dst, struct pdr *pdr)
 {
 	struct up_session *session = pdr->session;
 	struct up_gtp_action *a;
@@ -1067,18 +1161,20 @@ static void add_gtp_action_endecaps(void *ctx, struct llist_head *dst, struct pd
 	OSMO_ASSERT(pdr->far);
 	OSMO_ASSERT(pdr->reverse_pdr);
 	OSMO_ASSERT(pdr->reverse_pdr->far);
-
-	/* To decaps, we need to have a local TEID assigned for which to receive GTP packets. */
-	if (!pdr->local_f_teid || pdr->local_f_teid->choose_flag) {
-		log_inactive_pdr_set(pdr, "missing local TEID", pdr);
-		return;
-	}
-
-	/* To encaps, we need to have a remote TEID assigned to send out in GTP packets, and we need to know where to
-	 * send GTP to. */
 	rpdr = pdr->reverse_pdr;
 	rfar = rpdr->far;
 	rfar_forw = &rfar->desc.forw_params;
+
+	OSMO_ASSERT(pdr->access_to_core);
+	OSMO_ASSERT(rpdr->core_to_access);
+
+	/* To decaps incoming on Access, we need to have a local F-TEID assigned for which to receive GTP packets. */
+	if (!pdr->local_f_teid || pdr->local_f_teid->choose_flag) {
+		log_inactive_pdr_set(pdr, "missing local F-TEID", pdr);
+		return;
+	}
+
+	/* To encaps outgoing on Access, we need to have a remote F-TEID assigned to send out in GTP packets */
 	if (!rfar->desc.forw_params_present) {
 		log_inactive_pdr_set(pdr, "missing FAR Forwarding Parameters", rpdr);
 		return;
@@ -1096,8 +1192,7 @@ static void add_gtp_action_endecaps(void *ctx, struct llist_head *dst, struct pd
 		return;
 	}
 
-	/* To receive packets to be encapsulated, we need to know the assigned IP address for the UE, which receives the
-	 * IP packets that should be placed into GTP. */
+	/* To receive IP packets incoming on Core, we need to know the assigned IP address for the UE */
 	if (!rpdr->desc.pdi.ue_ip_address_present) {
 		log_inactive_pdr_set(pdr, "missing UE IP Address in PDI", rpdr);
 		return;
@@ -1127,21 +1222,31 @@ static void add_gtp_action_endecaps(void *ctx, struct llist_head *dst, struct pd
 	OSMO_ASSERT(a);
 	*a = (struct up_gtp_action){
 		.session = session,
-		.pdr_core = pdr->desc.pdr_id,
-		.pdr_access = rpdr->desc.pdr_id,
-		.kind = UP_GTP_U_ENDECAPS,
-		.endecaps = {
+		.pdr_access = pdr->desc.pdr_id,
+		.pdr_core = rpdr->desc.pdr_id,
+		.kind = UP_GTP_U_TUNEND,
+		.tunend = {
+			.access = {
+				.gtp_local_addr = pdr->local_f_teid->fixed.ip_addr.v4,
 				.local_teid = pdr->local_f_teid->fixed.teid,
-			.remote_teid = rfar_forw->outer_header_creation.teid,
 				.gtp_remote_addr = rfar_forw->outer_header_creation.ip_addr.v4,
-			.ue_addr = rpdr->desc.pdi.ue_ip_address.ip_addr.v4,
+				.remote_teid = rfar_forw->outer_header_creation.teid,
+			},
+			.core = {
+				.ue_local_addr = rpdr->desc.pdi.ue_ip_address.ip_addr.v4,
+			},
 		},
 	};
 
 	llist_add_tail(&a->entry, dst);
 }
 
-static void add_gtp_action_forw(void *ctx, struct llist_head *dst, struct pdr *pdr)
+/* A GTP tunnel on Access side, mapping to another GTP tunnel on Core side and vice versa.
+ * The PDR and its reverse PDR must both have an outer-header-removal and a local F-TEID.
+ * Both FARs must have an outer-header creation with a remote F-TEID.
+ * \param pdr  A rule detecting packets on Access, where pdr->reverse_pdr detects packets on Core.
+ */
+static void add_gtp_action_tunmap(void *ctx, struct llist_head *dst, struct pdr *pdr)
 {
 	struct up_session *session = pdr->session;
 	struct up_gtp_action *a;
@@ -1161,52 +1266,53 @@ static void add_gtp_action_forw(void *ctx, struct llist_head *dst, struct pdr *p
 	rfar = rpdr->far;
 	rfar_forw = &rfar->desc.forw_params;
 
-	/* To decaps, we need to have a local TEID assigned for which to receive GTP packets. */
-	/* decaps from CORE */
+	OSMO_ASSERT(pdr->access_to_core);
+	OSMO_ASSERT(rpdr->core_to_access);
+
+	/* To decaps incoming on Access, we need to have a local F-TEID assigned for which to receive GTP packets. */
 	if (!pdr->local_f_teid || pdr->local_f_teid->choose_flag) {
-		log_inactive_pdr_set(pdr, "missing local TEID (CORE side)", pdr);
+		log_inactive_pdr_set(pdr, "missing local F-TEID (Access side)", pdr);
 		return;
 	}
-	/* decaps from ACCESS */
+	/* To decaps incoming on Core, we need to have a local F-TEID assigned for which to receive GTP packets. */
 	if (!rpdr->local_f_teid || rpdr->local_f_teid->choose_flag) {
-		log_inactive_pdr_set(pdr, "missing local TEID (ACCESS side)", pdr);
+		log_inactive_pdr_set(pdr, "missing local F-TEID (Core side)", pdr);
 		return;
 	}
 
-	/* To encaps, we need to have a remote TEID assigned to send out in GTP packets, and we need to know where to
-	 * send GTP to. */
-	/* encaps towards ACCESS */
+	/* To encaps outgoing on Core, we need to have a remote F-TEID assigned to send out in GTP packets */
 	if (!far->desc.forw_params_present) {
-		log_inactive_pdr_set(pdr, "missing FAR Forwarding Parameters", pdr);
+		log_inactive_pdr_set(pdr, "missing FAR Forwarding Parameters (Access side)", pdr);
 		return;
 	}
 	if (!far_forw->outer_header_creation_present) {
-		log_inactive_pdr_set(pdr, "missing FAR Outer Header Creation", pdr);
+		log_inactive_pdr_set(pdr, "missing FAR Outer Header Creation (Access side)", pdr);
 		return;
 	}
 	if (!far_forw->outer_header_creation.teid_present) {
-		log_inactive_pdr_set(pdr, "missing TEID in FAR Outer Header Creation", pdr);
+		log_inactive_pdr_set(pdr, "missing TEID in FAR Outer Header Creation (Access side)", pdr);
 		return;
 	}
 	if (!far_forw->outer_header_creation.ip_addr.v4_present) {
-		log_inactive_pdr_set(pdr, "missing IPv4 in FAR Outer Header Creation", pdr);
+		log_inactive_pdr_set(pdr, "missing IPv4 in FAR Outer Header Creation (Access side)", pdr);
 		return;
 	}
-	/* encaps towards CORE */
+
+	/* To encaps outgoing on Access, we need to have a remote F-TEID assigned to send out in GTP packets */
 	if (!rfar->desc.forw_params_present) {
-		log_inactive_pdr_set(pdr, "missing FAR Forwarding Parameters", rpdr);
+		log_inactive_pdr_set(pdr, "missing FAR Forwarding Parameters (Access side)", rpdr);
 		return;
 	}
 	if (!rfar_forw->outer_header_creation_present) {
-		log_inactive_pdr_set(pdr, "missing FAR Outer Header Creation", rpdr);
+		log_inactive_pdr_set(pdr, "missing FAR Outer Header Creation (Access side)", rpdr);
 		return;
 	}
 	if (!rfar_forw->outer_header_creation.teid_present) {
-		log_inactive_pdr_set(pdr, "missing TEID in FAR Outer Header Creation", rpdr);
+		log_inactive_pdr_set(pdr, "missing TEID in FAR Outer Header Creation (Access side)", rpdr);
 		return;
 	}
 	if (!rfar_forw->outer_header_creation.ip_addr.v4_present) {
-		log_inactive_pdr_set(pdr, "missing IPv4 in FAR Outer Header Creation", rpdr);
+		log_inactive_pdr_set(pdr, "missing IPv4 in FAR Outer Header Creation (Access side)", rpdr);
 		return;
 	}
 
@@ -1226,19 +1332,21 @@ static void add_gtp_action_forw(void *ctx, struct llist_head *dst, struct pdr *p
 	OSMO_ASSERT(a);
 	*a = (struct up_gtp_action){
 		.session = session,
-		.pdr_core = pdr->desc.pdr_id,
-		.pdr_access = rpdr->desc.pdr_id,
+		.pdr_access = pdr->desc.pdr_id,
+		.pdr_core = rpdr->desc.pdr_id,
 		.kind = UP_GTP_U_TUNMAP,
 		.tunmap = {
-			.core = {
-				.local_teid = pdr->local_f_teid->fixed.teid,
-				.remote_teid = rfar_forw->outer_header_creation.teid,
-				.gtp_remote_addr = rfar_forw->outer_header_creation.ip_addr.v4,
-			},
 			.access = {
+				.gtp_local_addr = pdr->local_f_teid->fixed.ip_addr.v4,
+				.local_teid = pdr->local_f_teid->fixed.teid,
+				.gtp_remote_addr = rfar_forw->outer_header_creation.ip_addr.v4,
+				.remote_teid = rfar_forw->outer_header_creation.teid,
+			},
+			.core = {
+				.gtp_local_addr = rpdr->local_f_teid->fixed.ip_addr.v4,
 				.local_teid = rpdr->local_f_teid->fixed.teid,
-				.remote_teid = far_forw->outer_header_creation.teid,
 				.gtp_remote_addr = far_forw->outer_header_creation.ip_addr.v4,
+				.remote_teid = far_forw->outer_header_creation.teid,
 			},
 		},
 	};
@@ -1270,12 +1378,13 @@ static enum osmo_pfcp_cause find_gtp_actions(void *ctx, struct llist_head *dst,
 		if (pdr->reverse_pdr)
 			continue;
 
-		/* In this outer loop, only follow the forw_to_core directed PDRs, in the inner loop find the matching
-		 * forw_from_core PDR. */
-		if (!pdr->forw_to_core)
+		/* In this outer loop, only follow the access_to_core directed PDRs, in the inner loop find the matching
+		 * core_to_access PDR. i.e. we are looking only at PDRs detecting packets on the Access side, pairing up
+		 * with "reverse PDRs" detecting packets on the Core side. */
+		if (!pdr->access_to_core)
 			continue;
 
-		/* If a required TEID is not known, we cannot pair this PDR up */
+		/* If a required local addr + TEID is not known, we cannot pair this PDR up */
 		if (pdr->rx_decaps && !pdr->local_f_teid)
 			continue;
 
@@ -1286,7 +1395,7 @@ static enum osmo_pfcp_cause find_gtp_actions(void *ctx, struct llist_head *dst,
 				continue;
 
 			/* Looking for a PDR facing the other way */
-			if (!other->forw_from_core)
+			if (!other->core_to_access)
 				continue;
 			/* GTP header-ness must match, in reverse. */
 			if (pdr->rx_decaps != other->forw_encaps
@@ -1309,14 +1418,14 @@ static enum osmo_pfcp_cause find_gtp_actions(void *ctx, struct llist_head *dst,
 			continue;
 		}
 
-		/* Iterate in direction to-Core, where pdr->reverse_pdr will be the from-Core counterpart. */
-		if (!pdr->forw_to_core)
+		/* Iterate in direction Access-to-Core, where pdr->reverse_pdr will be the Core-to-Access counterpart. */
+		if (!pdr->access_to_core)
 			continue;
 
 		if (pdr->rx_decaps && !pdr->forw_encaps)
-			add_gtp_action_endecaps(ctx, dst, pdr);
+			add_gtp_action_tunend(ctx, dst, pdr);
 		else if (pdr->rx_decaps && pdr->forw_encaps)
-			add_gtp_action_forw(ctx, dst, pdr);
+			add_gtp_action_tunmap(ctx, dst, pdr);
 		else {
 			/* log the details of both PDRs in two separate log lines */
 			log_inactive_pdr_set(pdr, "not implemented", pdr);

src/osmo-upf/upf.c

@@ -51,7 +51,8 @@ void g_upf_alloc(void *ctx)
 			},
 		},
 		.nft = {
-			.priority = -300,
+			.priority_pre = -300,
+			.priority_post = 400,
 		},
 		.gtp = {
 			/* TODO: recovery count state file; use lower byte of current time, poor person's random. */
@@ -61,6 +62,7 @@ void g_upf_alloc(void *ctx)
 
 	INIT_LLIST_HEAD(&g_upf->gtp.vty_cfg.devs);
 	INIT_LLIST_HEAD(&g_upf->gtp.devs);
+	INIT_LLIST_HEAD(&g_upf->netinst);
 }
 
 int upf_pfcp_listen()
@@ -87,8 +89,8 @@ int upf_pfcp_listen()
 
 int upf_gtp_devs_open()
 {
-	struct gtp_vty_cfg *c = &g_upf->gtp.vty_cfg;
-	struct gtp_vty_cfg_dev *d;
+	struct tunend_vty_cfg *c = &g_upf->gtp.vty_cfg;
+	struct tunend_vty_cfg_dev *d;
 
 	llist_for_each_entry(d, &c->devs, entry) {
 		if (upf_gtp_dev_open(d->dev_name, d->create, d->local_addr, false, false))

src/osmo-upf/upf_gtp.c

@@ -39,7 +39,7 @@
 #include <osmocom/upf/upf_gtpu_echo.h>
 
 #define LOG_GTP_TUN(TUN, LEVEL, FMT, ARGS...) \
-	LOGP(DGTP, LEVEL, "%s: " FMT, upf_gtp_tun_to_str_c(OTC_SELECT, (TUN)), ##ARGS)
+	LOGP(DGTP, LEVEL, "%s: " FMT, upf_gtp_tunend_to_str_c(OTC_SELECT, (TUN)), ##ARGS)
 
 int upf_gtp_dev_to_str_buf(char *buf, size_t buflen, const struct upf_gtp_dev *dev)
 {
@@ -73,6 +73,26 @@ struct upf_gtp_dev *upf_gtp_dev_find_by_name(const char *name)
 	return NULL;
 }
 
+struct upf_gtp_dev *upf_gtp_dev_find_by_local_addr(const struct osmo_sockaddr *local_addr)
+{
+	struct upf_gtp_dev *dev;
+	struct upf_gtp_dev *dev_any = NULL;
+	struct osmo_sockaddr needle = *local_addr;
+
+	llist_for_each_entry(dev, &g_upf->gtp.devs, entry) {
+		/* To leave the port number out of the cmp, set the needle's port to match */
+		osmo_sockaddr_set_port(&needle.u.sa, osmo_sockaddr_port(&dev->gtpv1.local_addr.u.sa));
+
+		if (!osmo_sockaddr_cmp(&needle, &dev->gtpv1.local_addr))
+			return dev;
+		if (osmo_sockaddr_is_any(&dev->gtpv1.local_addr) == 1)
+			dev_any = dev;
+	}
+	/* No 1:1 match found, but there is a dev listening on ANY? Return that.
+	 * If there is no such dev, return NULL. */
+	return dev_any;
+}
+
 struct upf_gtp_dev *upf_gtp_dev_first()
 {
 	return llist_first_entry_or_null(&g_upf->gtp.devs, struct upf_gtp_dev, entry);
@@ -173,7 +193,7 @@ int upf_gtp_dev_open(const char *name, bool create_gtp_dev, const char *local_ad
 	struct upf_gtp_dev *dev;
 
 	if (g_upf->gtp.mockup) {
-		LOGP(DGTP, LOGL_NOTICE, "gtp/mockup active: not opening GTP device '%s'\n", name);
+		LOGP(DGTP, LOGL_NOTICE, "tunend/mockup active: not opening GTP device '%s'\n", name);
 		return 0;
 	}
 
@@ -183,6 +203,12 @@ int upf_gtp_dev_open(const char *name, bool create_gtp_dev, const char *local_ad
 
 	dev->sgsn_mode = sgsn_mode;
 
+	rc = upf_gtp_genl_ensure_open();
+	if (rc) {
+		LOG_GTP_DEV(dev, LOGL_ERROR, "Cannot set up GTP device, failed to open mnl_socket\n");
+		return rc;
+	}
+
 	if (listen_for_gtpv0) {
 		rc = osmo_sock_init_osa_ofd(&dev->gtpv0.ofd, SOCK_DGRAM, 0, &dev->gtpv0.local_addr, &any,
 					    OSMO_SOCK_F_BIND);
@@ -255,13 +281,8 @@ void upf_gtp_genl_close()
 }
 
 /* Open an MNL socket which allows to create and remove GTP devices (requires CAP_NET_ADMIN). */
-int upf_gtp_genl_open()
+int upf_gtp_genl_ensure_open()
 {
-	if (g_upf->gtp.mockup) {
-		LOGP(DGTP, LOGL_NOTICE, "gtp/mockup active: not opening mnl_socket\n");
-		return 0;
-	}
-
 	/* Already open? */
 	if (g_upf->gtp.nl && g_upf->gtp.genl_id >= 0)
 		return 0;
@@ -286,59 +307,69 @@ int upf_gtp_genl_open()
 	return 0;
 }
 
-struct upf_gtp_tun {
+struct upf_gtp_tunend {
 	struct llist_head entry;
 
 	struct upf_gtp_dev *dev;
-	struct upf_gtp_tun_desc desc;
+	struct upf_gtp_tunend_desc desc;
 	bool active;
 };
 
-static int upf_gtp_tun_to_str_buf(char *buf, size_t buflen, const struct upf_gtp_tun *tun)
+static int upf_gtp_tunend_to_str_buf(char *buf, size_t buflen, const struct upf_gtp_tunend *tun)
 {
 	struct osmo_strbuf sb = { .buf = buf, .len = buflen };
-	OSMO_STRBUF_PRINTF(sb, "%s:tun{TEID=l:0x%x,r:0x%x UE=", tun->dev->name, tun->desc.local_teid,
-			   tun->desc.remote_teid);
-	OSMO_STRBUF_APPEND(sb, osmo_sockaddr_to_str_buf2, &tun->desc.ue_addr);
-	OSMO_STRBUF_PRINTF(sb, " GTP-dst=");
-	OSMO_STRBUF_APPEND(sb, osmo_sockaddr_to_str_buf2, &tun->desc.gtp_remote_addr);
-	OSMO_STRBUF_PRINTF(sb, "}");
+	/* "tunend{dev=apn0 access(GTP-r=1.2.3.4 TEID:l=0x1234,r=0x5678) core(UE-l=10.9.8.7)}" */
+	OSMO_STRBUF_PRINTF(sb, "tunend{dev=%s access(GTP-r=", tun->dev->name);
+	OSMO_STRBUF_APPEND(sb, osmo_sockaddr_to_str_buf2, &tun->desc.access.gtp_remote_addr);
+	OSMO_STRBUF_PRINTF(sb, " TEID:l=0x%x,r=0x%x) core(UE-l=",
+			   tun->desc.access.local_teid, tun->desc.access.remote_teid);
+	OSMO_STRBUF_APPEND(sb, osmo_sockaddr_to_str_buf2, &tun->desc.core.ue_local_addr);
+	OSMO_STRBUF_PRINTF(sb, ")}");
 	return sb.chars_needed;
 }
 
-static char *upf_gtp_tun_to_str_c(void *ctx, const struct upf_gtp_tun *tun)
+static char *upf_gtp_tunend_to_str_c(void *ctx, const struct upf_gtp_tunend *tun)
 {
-	OSMO_NAME_C_IMPL(ctx, 64, "ERROR", upf_gtp_tun_to_str_buf, tun)
+	OSMO_NAME_C_IMPL(ctx, 64, "ERROR", upf_gtp_tunend_to_str_buf, tun)
 }
 
-static int upf_gtp_tun_deactivate(struct upf_gtp_tun *tun);
+static int upf_gtp_tunend_deactivate(struct upf_gtp_tunend *tun);
 
-static int upf_gtp_tun_destruct(struct upf_gtp_tun *tun)
+static int upf_gtp_tunend_destruct(struct upf_gtp_tunend *tun)
 {
 	if (tun->active)
-		upf_gtp_tun_deactivate(tun);
+		upf_gtp_tunend_deactivate(tun);
 	llist_del(&tun->entry);
 	return 0;
 }
 
-static struct upf_gtp_tun *upf_gtp_tun_alloc(struct upf_gtp_dev *dev, const struct upf_gtp_tun_desc *desc)
+#define tunend_desc_validate(TUN_DESC) \
+	do { \
+		OSMO_ASSERT(osmo_sockaddr_port(&(TUN_DESC)->access.gtp_local_addr.u.sa) == 0); \
+		OSMO_ASSERT(osmo_sockaddr_port(&(TUN_DESC)->access.gtp_remote_addr.u.sa) == 0); \
+		OSMO_ASSERT(osmo_sockaddr_port(&(TUN_DESC)->core.ue_local_addr.u.sa) == 0); \
+	} while (0)
+
+static struct upf_gtp_tunend *upf_gtp_tunend_alloc(struct upf_gtp_dev *dev, const struct upf_gtp_tunend_desc *desc)
 {
-	struct upf_gtp_tun *tun = talloc(dev, struct upf_gtp_tun);
+	struct upf_gtp_tunend *tun = talloc(dev, struct upf_gtp_tunend);
 	OSMO_ASSERT(tun);
-	*tun = (struct upf_gtp_tun){
+	tunend_desc_validate(desc);
+	*tun = (struct upf_gtp_tunend){
 		.dev = dev,
 		.desc = *desc,
 	};
 	llist_add(&tun->entry, &dev->tunnels);
-	talloc_set_destructor(tun, upf_gtp_tun_destruct);
+	talloc_set_destructor(tun, upf_gtp_tunend_destruct);
 	return tun;
 }
 
-static struct gtp_tunnel *upf_gtp_tun_to_gtp_tunnel(struct upf_gtp_tun *tun)
+static struct gtp_tunnel *upf_gtp_tunend_to_gtp_tunnel(struct upf_gtp_tunend *tun)
 {
 	struct gtp_tunnel *t;
 
-	if (tun->desc.ue_addr.u.sas.ss_family != AF_INET || tun->desc.gtp_remote_addr.u.sas.ss_family != AF_INET) {
+	if (tun->desc.core.ue_local_addr.u.sas.ss_family != AF_INET
+	    || tun->desc.access.gtp_remote_addr.u.sas.ss_family != AF_INET) {
 		LOG_GTP_TUN(tun, LOGL_ERROR, "Only capabale of IPv4\n");
 		return NULL;
 	}
@@ -347,14 +378,14 @@ static struct gtp_tunnel *upf_gtp_tun_to_gtp_tunnel(struct upf_gtp_tun *tun)
 	OSMO_ASSERT(t);
 	gtp_tunnel_set_ifidx(t, tun->dev->ifidx);
 	gtp_tunnel_set_version(t, GTP_V1);
-	gtp_tunnel_set_i_tei(t, tun->desc.local_teid);
-	gtp_tunnel_set_o_tei(t, tun->desc.remote_teid);
-	gtp_tunnel_set_ms_ip4(t, &tun->desc.ue_addr.u.sin.sin_addr);
-	gtp_tunnel_set_sgsn_ip4(t, &tun->desc.gtp_remote_addr.u.sin.sin_addr);
+	gtp_tunnel_set_i_tei(t, tun->desc.access.local_teid);
+	gtp_tunnel_set_o_tei(t, tun->desc.access.remote_teid);
+	gtp_tunnel_set_sgsn_ip4(t, &tun->desc.access.gtp_remote_addr.u.sin.sin_addr);
+	gtp_tunnel_set_ms_ip4(t, &tun->desc.core.ue_local_addr.u.sin.sin_addr);
 	return t;
 }
 
-int upf_gtp_tun_activate(struct upf_gtp_tun *tun)
+int upf_gtp_tunend_activate(struct upf_gtp_tunend *tun)
 {
 	int rc;
 	struct gtp_tunnel *t;
@@ -362,7 +393,7 @@ int upf_gtp_tun_activate(struct upf_gtp_tun *tun)
 	if (tun->active)
 		return -EALREADY;
 
-	t = upf_gtp_tun_to_gtp_tunnel(tun);
+	t = upf_gtp_tunend_to_gtp_tunnel(tun);
 	if (!t)
 		return -ENOTSUP;
 
@@ -380,37 +411,40 @@ int upf_gtp_tun_activate(struct upf_gtp_tun *tun)
 	return rc;
 }
 
-static struct upf_gtp_tun *upf_gtp_dev_tunnel_find(struct upf_gtp_dev *dev, const struct upf_gtp_tun_desc *tun_desc)
+static struct upf_gtp_tunend *upf_gtp_dev_tunend_find(struct upf_gtp_dev *dev, const struct upf_gtp_tunend_desc *tun_desc)
 {
-	struct upf_gtp_tun *tun;
+	struct upf_gtp_tunend *tun;
+	tunend_desc_validate(tun_desc);
 	llist_for_each_entry(tun, &dev->tunnels, entry) {
-		if (upf_gtp_tun_desc_cmp(tun_desc, &tun->desc))
+		if (upf_gtp_tunend_desc_cmp(tun_desc, &tun->desc))
 			continue;
 		return tun;
 	}
 	return NULL;
 }
 
-int upf_gtp_dev_tunnel_add(struct upf_gtp_dev *dev, const struct upf_gtp_tun_desc *tun_desc)
+int upf_gtp_dev_tunend_add(struct upf_gtp_dev *dev, const struct upf_gtp_tunend_desc *tun_desc)
 {
-	struct upf_gtp_tun *tun;
-	tun = upf_gtp_dev_tunnel_find(dev, tun_desc);
+	struct upf_gtp_tunend *tun;
+	tunend_desc_validate(tun_desc);
+	tun = upf_gtp_dev_tunend_find(dev, tun_desc);
 	if (!tun)
-		tun = upf_gtp_tun_alloc(dev, tun_desc);
+		tun = upf_gtp_tunend_alloc(dev, tun_desc);
 	if (tun->active)
 		return 0;
-	return upf_gtp_tun_activate(tun);
+	return upf_gtp_tunend_activate(tun);
 }
 
-int upf_gtp_dev_tunnel_del(struct upf_gtp_dev *dev, const struct upf_gtp_tun_desc *tun_desc)
+int upf_gtp_dev_tunend_del(struct upf_gtp_dev *dev, const struct upf_gtp_tunend_desc *tun_desc)
 {
-	struct upf_gtp_tun *tun;
+	struct upf_gtp_tunend *tun;
 	int rc;
-	tun = upf_gtp_dev_tunnel_find(dev, tun_desc);
+	tunend_desc_validate(tun_desc);
+	tun = upf_gtp_dev_tunend_find(dev, tun_desc);
 	if (!tun)
 		return 0;
 	if (tun->active) {
-		rc = upf_gtp_tun_deactivate(tun);
+		rc = upf_gtp_tunend_deactivate(tun);
 		if (rc)
 			return rc;
 	}
@@ -418,7 +452,7 @@ int upf_gtp_dev_tunnel_del(struct upf_gtp_dev *dev, const struct upf_gtp_tun_des
 	return 0;
 }
 
-static int upf_gtp_tun_deactivate(struct upf_gtp_tun *tun)
+static int upf_gtp_tunend_deactivate(struct upf_gtp_tunend *tun)
 {
 	int rc;
 	struct gtp_tunnel *t;
@@ -428,13 +462,13 @@ static int upf_gtp_tun_deactivate(struct upf_gtp_tun *tun)
 		return -EINVAL;
 	}
 
-	t = upf_gtp_tun_to_gtp_tunnel(tun);
+	t = upf_gtp_tunend_to_gtp_tunnel(tun);
 	if (!t)
 		return -EINVAL;
 
 	rc = gtp_del_tunnel(g_upf->gtp.genl_id, g_upf->gtp.nl, t);
 	if (rc)
-		LOG_GTP_TUN(tun, LOGL_ERROR, "Failed to delete tunnel: %d %s\n", rc, strerror(rc));
+		LOG_GTP_TUN(tun, LOGL_ERROR, "Failed to delete tunnel\n");
 	else
 		tun->active = false;
 
@@ -444,9 +478,9 @@ static int upf_gtp_tun_deactivate(struct upf_gtp_tun *tun)
 
 static int upf_gtp_dev_destruct(struct upf_gtp_dev *dev)
 {
-	struct upf_gtp_tun *t;
+	struct upf_gtp_tunend *t;
 	/* Destruct and clean up all active tunnels before deleting the device */
-	while ((t = llist_first_entry_or_null(&dev->tunnels, struct upf_gtp_tun, entry)))
+	while ((t = llist_first_entry_or_null(&dev->tunnels, struct upf_gtp_tunend, entry)))
 		talloc_free(t);
 	llist_del(&dev->entry);
 	/* osmo_fd_close() is a noop if ofd.fd == -1 */
@@ -457,7 +491,7 @@ static int upf_gtp_dev_destruct(struct upf_gtp_dev *dev)
 	return 0;
 }
 
-int upf_gtp_tun_desc_cmp(const struct upf_gtp_tun_desc *a, const struct upf_gtp_tun_desc *b)
+int upf_gtp_tunend_desc_cmp(const struct upf_gtp_tunend_desc *a, const struct upf_gtp_tunend_desc *b)
 {
 	int r;
 
@@ -469,9 +503,9 @@ int upf_gtp_tun_desc_cmp(const struct upf_gtp_tun_desc *a, const struct upf_gtp_
 		return 1;
 
 #define CMP_MEMB(MEMB) OSMO_CMP(a->MEMB, b->MEMB)
-	if ((r = CMP_MEMB(local_teid)))
+	if ((r = CMP_MEMB(access.local_teid)))
 		return r;
-	if ((r = CMP_MEMB(remote_teid)))
+	if ((r = CMP_MEMB(access.remote_teid)))
 		return r;
-	return osmo_sockaddr_cmp(&a->gtp_remote_addr, &b->gtp_remote_addr);
+	return osmo_sockaddr_cmp(&a->access.gtp_remote_addr, &b->access.gtp_remote_addr);
 }

src/osmo-upf/upf_gtpu_echo.c

@@ -155,7 +155,12 @@ int upf_gtpu_echo_setup(struct upf_gtp_dev *dev)
 		return -EINVAL;
 	}
 
+	/* the caller should already have osmo_fd_register()ed when setting up the socket. */
+	OSMO_ASSERT(osmo_fd_is_registered(&dev->gtpv1.ofd));
+	/* make sure there is no cb yet that this would be replacing. */
+	OSMO_ASSERT(dev->gtpv1.ofd.cb == NULL);
+
 	dev->gtpv1.ofd.cb = upf_gtpu_echo_read_cb;
 	dev->gtpv1.ofd.data = dev;
-	return osmo_fd_register(&dev->gtpv1.ofd);
+	return 0;
 }

src/osmo-upf/upf_nft.c

@@ -32,7 +32,31 @@
 
 static char *upf_nft_ruleset_table_create(void *ctx, const char *table_name)
 {
-	return talloc_asprintf(ctx, "add table inet %s\n", table_name);
+	return talloc_asprintf(ctx, "add table inet %s { flags owner; };\n", table_name);
+}
+
+static char *upf_nft_ruleset_vmap_init(void *ctx, const char *table_name, int priority_pre, int priority_post)
+{
+	/* add chain inet osmo-upf pre { type filter hook prerouting priority -300; policy accept; }
+	 * add chain inet osmo-upf post { type filter hook postrouting priority 400; policy accept; }
+	 * add map inet osmo-upf tunmap-pre { typeof ip daddr . @ih,32,32 : verdict; }
+	 * add map inet osmo-upf tunmap-post { typeof meta mark : verdict; }
+	 * add rule inet osmo-upf pre udp dport 2152 ip daddr . @ih,32,32 vmap @tunmap-pre
+	 * add rule inet osmo-upf post meta mark vmap @tunmap-post
+	 */
+	return talloc_asprintf(ctx,
+		"add chain inet %s pre { type filter hook prerouting priority %d; policy accept; };\n"
+		"add chain inet %s post { type filter hook postrouting priority %d; policy accept; };\n"
+		"add map inet %s tunmap-pre { typeof ip daddr . @ih,32,32 : verdict; };\n"
+		"add map inet %s tunmap-post { typeof meta mark : verdict; };\n"
+		"add rule inet %s pre udp dport %u ip daddr . @ih,32,32 vmap @tunmap-pre;\n"
+		"add rule inet %s post meta mark vmap @tunmap-post;\n",
+		table_name, priority_pre,
+		table_name, priority_post,
+		table_name,
+		table_name,
+		table_name, PORT_GTP1_U,
+		table_name);
 }
 
 static int upf_nft_run(const char *ruleset)
@@ -40,7 +64,7 @@ static int upf_nft_run(const char *ruleset)
 	int rc;
 
 	if (g_upf->nft.mockup) {
-		LOGP(DNFT, LOGL_NOTICE, "nft/mockup active: not running nft ruleset: '%s'\n", ruleset);
+		LOGP(DNFT, LOGL_NOTICE, "tunmap/mockup active: not running nft ruleset: '%s'\n", ruleset);
 		return 0;
 	}
 
@@ -56,15 +80,23 @@ static int upf_nft_run(const char *ruleset)
 		     rc, osmo_quote_str_c(OTC_SELECT, ruleset, -1));
 		return -EIO;
 	}
+
+	LOGP(DNFT, LOGL_DEBUG, "set up nft ruleset: %s\n", osmo_quote_str_c(OTC_SELECT, ruleset, -1));
 	return 0;
 }
 
 int upf_nft_init()
 {
 	int rc;
+
+	/* Always set up the default settings, also in mockup mode, so that the VTY reflects sane values */
+	if (!g_upf->nft.table_name)
+		g_upf->nft.table_name = talloc_strdup(g_upf, "osmo-upf");
+
+	/* When in mockup mode, do not set up nft_ctx and netfilter table */
 	if (g_upf->nft.mockup) {
 		LOGP(DNFT, LOGL_NOTICE,
-		     "nft/mockup active: not allocating libnftables nft_ctx. FOR TESTING PURPOSES ONLY.\n");
+		     "tunmap/mockup active: not allocating libnftables nft_ctx. FOR TESTING PURPOSES ONLY.\n");
 		return 0;
 	}
 
@@ -74,16 +106,19 @@ int upf_nft_init()
 		return -EIO;
 	}
 
-	if (!g_upf->nft.table_name)
-		g_upf->nft.table_name = talloc_strdup(g_upf, "osmo-upf");
-
-	rc = upf_nft_run(upf_nft_ruleset_table_create(OTC_SELECT, g_upf->nft.table_name));
+	rc = upf_nft_run(upf_nft_tunmap_get_table_init_str(OTC_SELECT));
 	if (rc) {
 		LOGP(DNFT, LOGL_ERROR, "Failed to create nft table %s\n",
 		     osmo_quote_str_c(OTC_SELECT, g_upf->nft.table_name, -1));
 		return rc;
 	}
 	LOGP(DNFT, LOGL_NOTICE, "Created nft table %s\n", osmo_quote_str_c(OTC_SELECT, g_upf->nft.table_name, -1));
+
+	rc = upf_nft_run(upf_nft_tunmap_get_vmap_init_str(OTC_SELECT));
+	if (rc) {
+		LOGP(DNFT, LOGL_ERROR, "Failed to initialize nft verdict map in table %s\n", g_upf->nft.table_name);
+		return rc;
+	}
 	return 0;
 }
 
@@ -98,50 +133,119 @@ int upf_nft_free()
 
 struct upf_nft_args_peer {
 	/* The source IP address in packets received from this peer */
-	const struct osmo_sockaddr *addr;
+	const struct osmo_sockaddr *addr_remote;
 	/* The TEID that we send to the peer in GTP packets. */
 	uint32_t teid_remote;
+	/* The local destination IP address in packets received from this peer */
+	const struct osmo_sockaddr *addr_local;
 	/* The TEID that the peer sends to us in GTP packets. */
 	uint32_t teid_local;
+	/* The nft chain id that forwards packets received on addr_local,teid_local. Also used for the 'mark' id in
+	 * the verdict map ruleset. */
+	uint32_t chain_id;
 };
 
 struct upf_nft_args {
 	/* global table name */
 	const char *table_name;
-	/* chain name for this specific tunnel mapping */
-	uint32_t chain_id;
-	int priority;
 
 	struct upf_nft_args_peer peer_a;
 	struct upf_nft_args_peer peer_b;
 };
 
-static int tunmap_single_direction(char *buf, size_t buflen,
+static int tunmap_add_single_direction(char *buf, size_t buflen,
 				       const struct upf_nft_args *args,
-				   const struct upf_nft_args_peer *from_peer,
-				   const struct upf_nft_args_peer *to_peer)
+				       bool dir_a2b)
 {
 	struct osmo_strbuf sb = { .buf = buf, .len = buflen };
-	OSMO_STRBUF_PRINTF(sb, "add rule inet %s " NFT_CHAIN_NAME_PREFIX_TUNMAP "%u", args->table_name, args->chain_id);
+	const struct upf_nft_args_peer *from_peer;
+	const struct upf_nft_args_peer *to_peer;
 
-	/* Match only UDP packets */
-	OSMO_STRBUF_PRINTF(sb, " meta l4proto udp");
+	if (dir_a2b) {
+		from_peer = &args->peer_a;
+		to_peer = &args->peer_b;
+	} else {
+		from_peer = &args->peer_b;
+		to_peer = &args->peer_a;
+	}
 
-	/* Match on packets coming in from from_peer */
-	OSMO_STRBUF_PRINTF(sb, " ip saddr ");
-	OSMO_STRBUF_APPEND(sb, osmo_sockaddr_to_str_buf2, from_peer->addr);
+	/* # add chain for verdict map in prerouting
+	 * add chain inet osmo-upf tunmap-pre-123
+	 * # mangle destination address at prerouting
+	 * add rule inet osmo-upf tunmap-pre-123 ip daddr set 1.1.1.1 meta mark set 123 counter accept
+	 *
+	 * # add chain for verdict map in postrouting
+	 * add chain inet osmo-upf tunmap-post-123
+	 * # mangle source address and GTP TID at postrouting
+	 * add rule inet osmo-upf tunmap-post-123 ip saddr set 2.2.2.1 @ih,32,32 set 0x00000102 counter accept
+	 *
+	 * # add elements to verdict map, jump to chain
+	 * add element inet osmo-upf tunmap-pre { 2.2.2.3 . 0x00000203 : jump tunmap-pre-123 }
+	 * add element inet osmo-upf tunmap-post { 123 : jump tunmap-post-123 }
+	 */
 
-	/* Match on the TEID in the header */
-	OSMO_STRBUF_PRINTF(sb, " @ih,32,32 0x%08x", from_peer->teid_local);
+	OSMO_STRBUF_PRINTF(sb, "add chain inet %s tunmap-pre-%u;\n",
+			   args->table_name, from_peer->chain_id);
 
-	/* Change destination address to to_peer */
+	OSMO_STRBUF_PRINTF(sb, "add rule inet %s tunmap-pre-%u",
+			   args->table_name, from_peer->chain_id);
 	OSMO_STRBUF_PRINTF(sb, " ip daddr set ");
-	OSMO_STRBUF_APPEND(sb, osmo_sockaddr_to_str_buf2, to_peer->addr);
+	OSMO_STRBUF_APPEND(sb, osmo_sockaddr_to_str_buf2, to_peer->addr_remote);
+	OSMO_STRBUF_PRINTF(sb, " meta mark set %u counter accept;\n", from_peer->chain_id);
 
-	/* Change the TEID in the header to the one to_peer expects */
-	OSMO_STRBUF_PRINTF(sb, " @ih,32,32 set 0x%08x", to_peer->teid_remote);
+	OSMO_STRBUF_PRINTF(sb, "add chain inet %s tunmap-post-%u;\n",
+			   args->table_name, from_peer->chain_id);
 
-	OSMO_STRBUF_PRINTF(sb, " counter\n");
+	OSMO_STRBUF_PRINTF(sb, "add rule inet %s tunmap-post-%u",
+			   args->table_name, from_peer->chain_id);
+	OSMO_STRBUF_PRINTF(sb, " ip saddr set ");
+	OSMO_STRBUF_APPEND(sb, osmo_sockaddr_to_str_buf2, to_peer->addr_local);
+	OSMO_STRBUF_PRINTF(sb, " @ih,32,32 set 0x%x", to_peer->teid_remote);
+	OSMO_STRBUF_PRINTF(sb, " counter accept;\n");
+
+	OSMO_STRBUF_PRINTF(sb, "add element inet %s tunmap-pre { ",
+			   args->table_name);
+	OSMO_STRBUF_APPEND(sb, osmo_sockaddr_to_str_buf2, from_peer->addr_local);
+	OSMO_STRBUF_PRINTF(sb, " . 0x%x : jump tunmap-pre-%u };\n",
+			   from_peer->teid_local, from_peer->chain_id);
+
+	OSMO_STRBUF_PRINTF(sb, "add element inet %s tunmap-post { %u : jump tunmap-post-%u };\n",
+			   args->table_name, from_peer->chain_id, from_peer->chain_id);
+
+	return sb.chars_needed;
+}
+
+static int tunmap_del_single_direction(char *buf, size_t buflen,
+				       const struct upf_nft_args *args,
+				       bool dir_a2b)
+{
+	struct osmo_strbuf sb = { .buf = buf, .len = buflen };
+	const struct upf_nft_args_peer *from_peer;
+
+	if (dir_a2b)
+		from_peer = &args->peer_a;
+	else
+		from_peer = &args->peer_b;
+
+	/* delete element inet osmo-upf tunmap-pre { 2.2.2.3 . 0x203 }
+	 * delete element inet osmo-upf tunmap-post { 123 }
+	 * delete chain inet osmo-upf tunmap-pre-123
+	 * delete chain inet osmo-upf tunmap-post-123
+	 */
+
+	OSMO_STRBUF_PRINTF(sb, "delete element inet %s tunmap-pre { ",
+			   args->table_name);
+	OSMO_STRBUF_APPEND(sb, osmo_sockaddr_to_str_buf2, from_peer->addr_local);
+	OSMO_STRBUF_PRINTF(sb, " . 0x%x };\n", from_peer->teid_local);
+
+	OSMO_STRBUF_PRINTF(sb, "delete element inet %s tunmap-post { %u };\n",
+			   args->table_name, from_peer->chain_id);
+
+	OSMO_STRBUF_PRINTF(sb, "delete chain inet %s tunmap-pre-%u;\n",
+			   args->table_name, from_peer->chain_id);
+
+	OSMO_STRBUF_PRINTF(sb, "delete chain inet %s tunmap-post-%u;\n",
+			   args->table_name, from_peer->chain_id);
 
 	return sb.chars_needed;
 }
@@ -150,70 +254,129 @@ static int upf_nft_ruleset_tunmap_create_buf(char *buf, size_t buflen, const str
 {
 	struct osmo_strbuf sb = { .buf = buf, .len = buflen };
 
-	/* Add a chain for this tunnel mapping */
-	OSMO_STRBUF_PRINTF(sb, "add chain inet %s " NFT_CHAIN_NAME_PREFIX_TUNMAP "%u { type filter hook prerouting priority %d; }\n",
-			   args->table_name, args->chain_id, args->priority);
-
 	/* Forwarding from peer_a to peer_b */
-	OSMO_STRBUF_APPEND(sb, tunmap_single_direction, args, &args->peer_a, &args->peer_b);
+	OSMO_STRBUF_APPEND(sb, tunmap_add_single_direction, args, true);
 	/* And from peer_b to peer_a */
-	OSMO_STRBUF_APPEND(sb, tunmap_single_direction, args, &args->peer_b, &args->peer_a);
+	OSMO_STRBUF_APPEND(sb, tunmap_add_single_direction, args, false);
 
 	return sb.chars_needed;
 }
 
 static char *upf_nft_ruleset_tunmap_create_c(void *ctx, const struct upf_nft_args *args)
 {
-	OSMO_NAME_C_IMPL(ctx, 512, "ERROR", upf_nft_ruleset_tunmap_create_buf, args)
+	OSMO_NAME_C_IMPL(ctx, 1024, "ERROR", upf_nft_ruleset_tunmap_create_buf, args)
 }
 
 static int upf_nft_ruleset_tunmap_delete_buf(char *buf, size_t buflen, const struct upf_nft_args *args)
 {
 	struct osmo_strbuf sb = { .buf = buf, .len = buflen };
-	OSMO_STRBUF_PRINTF(sb, "delete chain inet %s " NFT_CHAIN_NAME_PREFIX_TUNMAP "%u\n",
-			   args->table_name, args->chain_id);
+
+	/* Forwarding from peer_a to peer_b */
+	OSMO_STRBUF_APPEND(sb, tunmap_del_single_direction, args, true);
+	/* And from peer_b to peer_a */
+	OSMO_STRBUF_APPEND(sb, tunmap_del_single_direction, args, false);
+
 	return sb.chars_needed;
 }
 
 static char *upf_nft_ruleset_tunmap_delete_c(void *ctx, const struct upf_nft_args *args)
 {
-	OSMO_NAME_C_IMPL(ctx, 64, "ERROR", upf_nft_ruleset_tunmap_delete_buf, args)
+	OSMO_NAME_C_IMPL(ctx, 512, "ERROR", upf_nft_ruleset_tunmap_delete_buf, args)
+}
+
+int upf_nft_tunmap_to_str_buf(char *buf, size_t buflen, const struct upf_nft_tunmap_desc *tunmap)
+{
+	struct osmo_strbuf sb = { .buf = buf, .len = buflen };
+
+	/* ACCESS 1.1.1.2:0x102 <---> 2.2.2.1:0x201 UPF 2.2.2.3:0x203 <---> 3.3.3.2:0x302 CORE */
+	OSMO_STRBUF_PRINTF(sb, "ACCESS ");
+	OSMO_STRBUF_APPEND(sb, osmo_sockaddr_to_str_buf2, &tunmap->access.gtp_remote_addr);
+	OSMO_STRBUF_PRINTF(sb, ":0x%x <---> ", tunmap->access.remote_teid);
+	OSMO_STRBUF_APPEND(sb, osmo_sockaddr_to_str_buf2, &tunmap->access.gtp_local_addr);
+	OSMO_STRBUF_PRINTF(sb, ":0x%x UPF ", tunmap->access.local_teid);
+	OSMO_STRBUF_APPEND(sb, osmo_sockaddr_to_str_buf2, &tunmap->core.gtp_local_addr);
+	OSMO_STRBUF_PRINTF(sb, ":0x%x <---> ", tunmap->core.local_teid);
+	OSMO_STRBUF_APPEND(sb, osmo_sockaddr_to_str_buf2, &tunmap->core.gtp_remote_addr);
+	OSMO_STRBUF_PRINTF(sb, ":0x%x CORE", tunmap->core.remote_teid);
+	return sb.chars_needed;
+}
+
+char *upf_nft_tunmap_to_str_c(void *ctx, const struct upf_nft_tunmap_desc *tunmap)
+{
+	OSMO_NAME_C_IMPL(ctx, 128, "ERROR", upf_nft_tunmap_to_str_buf, tunmap)
 }
 
 static void upf_nft_args_from_tunmap_desc(struct upf_nft_args *args, const struct upf_nft_tunmap_desc *tunmap)
 {
+	OSMO_ASSERT(osmo_sockaddr_port(&tunmap->access.gtp_remote_addr.u.sa) == 0);
+	OSMO_ASSERT(osmo_sockaddr_port(&tunmap->access.gtp_local_addr.u.sa) == 0);
+	OSMO_ASSERT(osmo_sockaddr_port(&tunmap->core.gtp_remote_addr.u.sa) == 0);
+	OSMO_ASSERT(osmo_sockaddr_port(&tunmap->core.gtp_local_addr.u.sa) == 0);
+
 	*args = (struct upf_nft_args){
 		.table_name = g_upf->nft.table_name,
-		.chain_id = tunmap->id,
-		.priority = g_upf->nft.priority,
 		.peer_a = {
-			.addr = &tunmap->access.gtp_remote_addr,
+			.addr_remote = &tunmap->access.gtp_remote_addr,
 			.teid_remote = tunmap->access.remote_teid,
+			.addr_local = &tunmap->access.gtp_local_addr,
 			.teid_local = tunmap->access.local_teid,
+			.chain_id = tunmap->access.chain_id,
 		},
 		.peer_b = {
-			.addr = &tunmap->core.gtp_remote_addr,
+			.addr_remote = &tunmap->core.gtp_remote_addr,
 			.teid_remote = tunmap->core.remote_teid,
+			.addr_local = &tunmap->core.gtp_local_addr,
 			.teid_local = tunmap->core.local_teid,
+			.chain_id = tunmap->core.chain_id,
 		},
 	};
 }
 
-int upf_nft_tunmap_create(struct upf_nft_tunmap_desc *tunmap)
+char *upf_nft_tunmap_get_table_init_str(void *ctx)
+{
+	return upf_nft_ruleset_table_create(ctx, g_upf->nft.table_name);
+}
+
+char *upf_nft_tunmap_get_vmap_init_str(void *ctx)
+{
+	return upf_nft_ruleset_vmap_init(ctx, g_upf->nft.table_name, g_upf->nft.priority_pre,
+					 g_upf->nft.priority_post);
+}
+
+static uint32_t chain_id_next(void)
+{
+	g_upf->nft.next_chain_id_state++;
+	if (!g_upf->nft.next_chain_id_state)
+		g_upf->nft.next_chain_id_state++;
+	return g_upf->nft.next_chain_id_state;
+}
+
+char *upf_nft_tunmap_get_ruleset_str(void *ctx, struct upf_nft_tunmap_desc *tunmap)
 {
 	struct upf_nft_args args;
 
-	/* Give this tunnel mapping a new id, returned to the caller so that the tunnel mapping can be deleted later */
-	g_upf->nft.next_id_state++;
-	tunmap->id = g_upf->nft.next_id_state;
+	if (!tunmap->access.chain_id)
+		tunmap->access.chain_id = chain_id_next();
+	if (!tunmap->core.chain_id)
+		tunmap->core.chain_id = chain_id_next();
 
 	upf_nft_args_from_tunmap_desc(&args, tunmap);
-	return upf_nft_run(upf_nft_ruleset_tunmap_create_c(OTC_SELECT, &args));
+	return upf_nft_ruleset_tunmap_create_c(ctx, &args);
+}
+
+char *upf_nft_tunmap_get_ruleset_del_str(void *ctx, struct upf_nft_tunmap_desc *tunmap)
+{
+	struct upf_nft_args args;
+	upf_nft_args_from_tunmap_desc(&args, tunmap);
+	return upf_nft_ruleset_tunmap_delete_c(ctx, &args);
+}
+
+int upf_nft_tunmap_create(struct upf_nft_tunmap_desc *tunmap)
+{
+	return upf_nft_run(upf_nft_tunmap_get_ruleset_str(OTC_SELECT, tunmap));
 }
 
 int upf_nft_tunmap_delete(struct upf_nft_tunmap_desc *tunmap)
 {
-	struct upf_nft_args args;
-	upf_nft_args_from_tunmap_desc(&args, tunmap);
-	return upf_nft_run(upf_nft_ruleset_tunmap_delete_c(OTC_SELECT, &args));
+	return upf_nft_run(upf_nft_tunmap_get_ruleset_del_str(OTC_SELECT, tunmap));
 }

src/osmo-upf/upf_vty.c

@@ -37,11 +37,13 @@
 #include <osmocom/upf/up_peer.h>
 #include <osmocom/upf/up_session.h>
 #include <osmocom/upf/up_gtp_action.h>
+#include <osmocom/upf/netinst.h>
 
 enum upf_vty_node {
 	PFCP_NODE = _LAST_OSMOVTY_NODE + 1,
-	GTP_NODE,
-	NFT_NODE,
+	TUNEND_NODE,
+	TUNMAP_NODE,
+	NETINST_NODE,
 };
 
 static struct cmd_node cfg_pfcp_node = {
@@ -51,7 +53,7 @@ static struct cmd_node cfg_pfcp_node = {
 };
 
 #define pfcp_vty (g_upf->pfcp.vty_cfg)
-#define gtp_vty (g_upf->gtp.vty_cfg)
+#define tunend_vty (g_upf->gtp.vty_cfg)
 
 DEFUN(cfg_pfcp, cfg_pfcp_cmd,
       "pfcp",
@@ -78,29 +80,32 @@ DEFUN(cfg_pfcp_local_addr, cfg_pfcp_local_addr_cmd,
 	return CMD_SUCCESS;
 }
 
-static struct cmd_node cfg_gtp_node = {
-	GTP_NODE,
-	"%s(config-gtp)# ",
+static struct cmd_node cfg_tunend_node = {
+	TUNEND_NODE,
+	"%s(config-tunend)# ",
 	1,
 };
 
-DEFUN(cfg_gtp, cfg_gtp_cmd,
-      "gtp",
-      "Enter the 'gtp' node to configure Linux GTP kernel module usage\n")
+#define TUNEND_NODE_STR "Enter the 'tunend' node to configure Linux GTP kernel module usage\n"
+
+DEFUN(cfg_tunend, cfg_tunend_cmd, "tunend", TUNEND_NODE_STR)
 {
-	vty->node = GTP_NODE;
+	vty->node = TUNEND_NODE;
 	return CMD_SUCCESS;
 }
 
-static int config_write_gtp(struct vty *vty)
+/* legacy compat: "tunend" was originally named "gtp" */
+DEFUN_CMD_ELEMENT(cfg_tunend, cfg_gtp_cmd, "gtp", TUNEND_NODE_STR, CMD_ATTR_HIDDEN, 0);
+
+static int config_write_tunend(struct vty *vty)
 {
-	struct gtp_vty_cfg_dev *d;
-	vty_out(vty, "gtp%s", VTY_NEWLINE);
+	struct tunend_vty_cfg_dev *d;
+	vty_out(vty, "tunend%s", VTY_NEWLINE);
 
 	if (g_upf->gtp.mockup)
 		vty_out(vty, " mockup%s", VTY_NEWLINE);
 
-	llist_for_each_entry(d, &gtp_vty.devs, entry) {
+	llist_for_each_entry(d, &tunend_vty.devs, entry) {
 		if (d->create) {
 			vty_out(vty, " dev create %s", d->dev_name);
 			if (d->local_addr)
@@ -115,7 +120,7 @@ static int config_write_gtp(struct vty *vty)
 
 #define DEV_STR "Configure the GTP device to use for encaps/decaps.\n"
 
-DEFUN(cfg_gtp_mockup, cfg_gtp_mockup_cmd,
+DEFUN(cfg_tunend_mockup, cfg_tunend_mockup_cmd,
       "mockup",
       "don't actually send commands to the GTP kernel module, just return success\n")
 {
@@ -123,7 +128,7 @@ DEFUN(cfg_gtp_mockup, cfg_gtp_mockup_cmd,
 	return CMD_SUCCESS;
 }
 
-DEFUN(cfg_gtp_no_mockup, cfg_gtp_no_mockup_cmd,
+DEFUN(cfg_tunend_no_mockup, cfg_tunend_no_mockup_cmd,
       "no mockup",
       NO_STR
       "operate GTP kernel module normally\n")
@@ -132,52 +137,59 @@ DEFUN(cfg_gtp_no_mockup, cfg_gtp_no_mockup_cmd,
 	return CMD_SUCCESS;
 }
 
-DEFUN(cfg_gtp_dev_create, cfg_gtp_dev_create_cmd,
+static struct tunend_vty_cfg_dev *tunend_dev_add(int argc, const char **argv, bool create)
+{
+	struct tunend_vty_cfg_dev *d = talloc_zero(g_upf, struct tunend_vty_cfg_dev);
+	d->create = create;
+	d->dev_name = talloc_strdup(d, argv[0]);
+	if (argc > 1)
+		d->local_addr = talloc_strdup(d, argv[1]);
+	llist_add(&d->entry, &tunend_vty.devs);
+	return d;
+}
+
+DEFUN(cfg_tunend_dev_create, cfg_tunend_dev_create_cmd,
       "dev create DEVNAME [LISTEN_ADDR]",
       DEV_STR
       "Add GTP device, creating a new Linux kernel GTP device. Will listen on GTPv1 port "
       OSMO_STRINGIFY_VAL(PORT_GTP1_U)
-      " and GTPv0 port " OSMO_STRINGIFY_VAL(PORT_GTP0_U) " on the specified interface, or on ANY if LISTEN_ADDR is"
-      " omitted.\n"
+      " and GTPv0 port " OSMO_STRINGIFY_VAL(PORT_GTP0_U) " on the specified LISTEN_ADDR\n"
       "device name, e.g. 'apn0'\n"
-      "IPv4 or IPv6 address to listen on, omit for ANY\n")
+      "IPv4 or IPv6 address to listen on, omit for ANY. LISTEN_ADDR is used to pick a GTP device matching the local"
+      " address for a PFCP Network Instance, which are configured in the 'netinst' node.\n")
 {
-	struct gtp_vty_cfg_dev *d = talloc_zero(g_upf, struct gtp_vty_cfg_dev);
-	d->create = true;
-	d->dev_name = talloc_strdup(d, argv[0]);
-	if (argc > 1)
-		d->local_addr = talloc_strdup(d, argv[1]);
-	llist_add(&d->entry, &gtp_vty.devs);
-	vty_out(vty, "Added GTP device %s (create new)%s", d->dev_name, VTY_NEWLINE);
+	struct tunend_vty_cfg_dev *d = tunend_dev_add(argc, argv, true);
+	vty_out(vty, "Added GTP device %s on %s (create new)%s", d->dev_name, d->local_addr ? : "0.0.0.0", VTY_NEWLINE);
 	return CMD_SUCCESS;
 }
 
-DEFUN(cfg_gtp_dev_use, cfg_gtp_dev_use_cmd,
-      "dev use DEVNAME",
+DEFUN(cfg_tunend_dev_use, cfg_tunend_dev_use_cmd,
+      "dev use DEVNAME [LOCAL_ADDR]",
       DEV_STR
       "Add GTP device, using an existing Linux kernel GTP device, e.g. created by 'gtp-link'\n"
-      "device name, e.g. 'apn0'\n")
+      "device name, e.g. 'apn0'\n"
+      "The local GTP address this device listens on. It is assumed to be ANY when omitted."
+      " LOCAL_ADDR is used to pick a GTP device matching the local address for a PFCP Network Instance,"
+      " which are configured in the 'netinst' node.\n")
 {
-	struct gtp_vty_cfg_dev *d = talloc_zero(g_upf, struct gtp_vty_cfg_dev);
-	d->create = false;
-	d->dev_name = talloc_strdup(d, argv[0]);
-	llist_add(&d->entry, &gtp_vty.devs);
-	vty_out(vty, "Added GTP device %s (use existing)%s", d->dev_name, VTY_NEWLINE);
+	struct tunend_vty_cfg_dev *d = tunend_dev_add(argc, argv, false);
+	vty_out(vty, "Added GTP device %s on %s (use existing)%s", d->dev_name, d->local_addr ? : "0.0.0.0",
+		VTY_NEWLINE);
 	return CMD_SUCCESS;
 }
 
-DEFUN(cfg_gtp_dev_del, cfg_gtp_dev_del_cmd,
+DEFUN(cfg_tunend_dev_del, cfg_tunend_dev_del_cmd,
       "dev delete DEVNAME",
       DEV_STR
       "Remove a GTP device from the configuration, and delete the Linux kernel GTP device if it was created here.\n"
       "device name, e.g. 'apn0'\n")
 {
 	const char *dev_name = argv[0];
-	struct gtp_vty_cfg_dev *d;
+	struct tunend_vty_cfg_dev *d;
 	struct upf_gtp_dev *dev;
 
 	/* remove from VTY cfg */
-	llist_for_each_entry(d, &gtp_vty.devs, entry) {
+	llist_for_each_entry(d, &tunend_vty.devs, entry) {
 		if (strcmp(d->dev_name, dev_name))
 			continue;
 		llist_del(&d->entry);
@@ -191,33 +203,37 @@ DEFUN(cfg_gtp_dev_del, cfg_gtp_dev_del_cmd,
 	return CMD_SUCCESS;
 }
 
-static struct cmd_node cfg_nft_node = {
-	NFT_NODE,
-	"%s(config-nft)# ",
+static struct cmd_node cfg_tunmap_node = {
+	TUNMAP_NODE,
+	"%s(config-tunmap)# ",
 	1,
 };
 
-DEFUN(cfg_nft, cfg_nft_cmd,
-      "nft",
-      "Enter the 'nft' node to configure nftables usage\n")
+#define TUNMAP_NODE_STR "Enter the 'tunmap' node to configure nftables usage\n"
+
+DEFUN(cfg_tunmap, cfg_tunmap_cmd, "tunmap", TUNMAP_NODE_STR)
 {
-	vty->node = NFT_NODE;
+	vty->node = TUNMAP_NODE;
 	return CMD_SUCCESS;
 }
 
-static int config_write_nft(struct vty *vty)
+/* legacy compat: "tunmap" was originally named "nft" */
+DEFUN_CMD_ELEMENT(cfg_tunmap, cfg_nft_cmd, "nft", TUNMAP_NODE_STR, CMD_ATTR_HIDDEN, 0);
+
+static int config_write_tunmap(struct vty *vty)
 {
-	vty_out(vty, "nft%s", VTY_NEWLINE);
+	vty_out(vty, "tunmap%s", VTY_NEWLINE);
 
 	if (g_upf->nft.mockup)
 		vty_out(vty, " mockup%s", VTY_NEWLINE);
 
 	if (g_upf->nft.table_name && strcmp(g_upf->nft.table_name, "osmo-upf"))
 		vty_out(vty, " table-name %s%s", g_upf->nft.table_name, VTY_NEWLINE);
+
 	return CMD_SUCCESS;
 }
 
-DEFUN(cfg_nft_mockup, cfg_nft_mockup_cmd,
+DEFUN(cfg_tunmap_mockup, cfg_tunmap_mockup_cmd,
       "mockup",
       "don't actually send rulesets to nftables, just return success\n")
 {
@@ -225,7 +241,7 @@ DEFUN(cfg_nft_mockup, cfg_nft_mockup_cmd,
 	return CMD_SUCCESS;
 }
 
-DEFUN(cfg_nft_no_mockup, cfg_nft_no_mockup_cmd,
+DEFUN(cfg_tunmap_no_mockup, cfg_tunmap_no_mockup_cmd,
       "no mockup",
       NO_STR
       "operate nftables rulesets normally\n")
@@ -234,17 +250,149 @@ DEFUN(cfg_nft_no_mockup, cfg_nft_no_mockup_cmd,
 	return CMD_SUCCESS;
 }
 
-DEFUN(cfg_nft_table_name, cfg_nft_table_name_cmd,
+DEFUN(cfg_tunmap_table_name, cfg_tunmap_table_name_cmd,
       "table-name TABLE_NAME",
       "Set the nft inet table name to create and place GTP tunnel forwarding chains in"
       " (as in 'nft add table inet foo'). If multiple instances of osmo-upf are running on the same system, each"
-      " osmo-upf must have its own table name. Otherwise the names of created forwarding chains will collide.\n"
+      " osmo-upf must have its own table name. Otherwise the names of created forwarding chains will collide."
+      " The default table name is \"osmo-upf\".\n"
       "nft inet table name\n")
 {
 	osmo_talloc_replace_string(g_upf, &g_upf->nft.table_name, argv[0]);
 	return CMD_SUCCESS;
 }
 
+#define NFT_RULE_STR "nftables rule specifics\n"
+#define TUNMAP_STR "GTP tunmap use case (a.k.a. forwarding between two GTP tunnels)\n"
+#define TUNMAP_APPEND_STR "'tunmap append' feature is no longer available.\n"
+
+DEFUN_DEPRECATED(cfg_tunmap_nft_rule_append, cfg_tunmap_nft_rule_append_cmd,
+      "nft-rule tunmap append .NFT_RULE",
+      NFT_RULE_STR TUNMAP_STR TUNMAP_APPEND_STR TUNMAP_APPEND_STR)
+{
+	vty_out(vty, "%% deprecated config option: 'nft-rule tunmap append'%s", VTY_NEWLINE);
+	return CMD_SUCCESS;
+}
+
+DEFUN_DEPRECATED(cfg_tunmap_no_nft_rule_append, cfg_tunmap_no_nft_rule_append_cmd,
+      "no nft-rule tunmap append",
+      NO_STR NFT_RULE_STR TUNMAP_STR TUNMAP_APPEND_STR TUNMAP_APPEND_STR)
+{
+	vty_out(vty, "%% deprecated config option: 'no nft-rule tunmap append'%s", VTY_NEWLINE);
+	return CMD_SUCCESS;
+}
+
+DEFUN_DEPRECATED(show_nft_rule_append, show_nft_rule_append_cmd,
+      "show nft-rule tunmap append",
+      SHOW_STR NFT_RULE_STR TUNMAP_STR TUNMAP_APPEND_STR)
+{
+	vty_out(vty, "%% deprecated config option: 'show nft-rule tunmap append'%s", VTY_NEWLINE);
+	return CMD_SUCCESS;
+}
+
+DEFUN(show_nft_rule_tunmap_example, show_nft_rule_tunmap_example_cmd,
+      "show nft-rule tunmap example",
+      SHOW_STR NFT_RULE_STR TUNMAP_STR
+      "Print a complete nftables ruleset for a tunmap filled with example IP addresses and TEIDs\n")
+{
+	struct osmo_sockaddr_str str = {};
+	struct upf_nft_tunmap_desc d = {
+		.access = {
+			.local_teid = 0x201,
+			.remote_teid = 0x102,
+			.chain_id = 123,
+		},
+		.core = {
+			.local_teid = 0x203,
+			.remote_teid = 0x302,
+			.chain_id = 321,
+		},
+	};
+
+	osmo_sockaddr_str_from_str2(&str, "1.1.1.1");
+	osmo_sockaddr_str_to_sockaddr(&str, &d.access.gtp_remote_addr.u.sas);
+
+	osmo_sockaddr_str_from_str2(&str, "2.2.2.1");
+	osmo_sockaddr_str_to_sockaddr(&str, &d.access.gtp_local_addr.u.sas);
+
+	osmo_sockaddr_str_from_str2(&str, "2.2.2.3");
+	osmo_sockaddr_str_to_sockaddr(&str, &d.core.gtp_local_addr.u.sas);
+
+	osmo_sockaddr_str_from_str2(&str, "3.3.3.3");
+	osmo_sockaddr_str_to_sockaddr(&str, &d.core.gtp_remote_addr.u.sas);
+
+	vty_out(vty, "%% init verdict map:%s", VTY_NEWLINE);
+	vty_out(vty, "%s%s", upf_nft_tunmap_get_table_init_str(OTC_SELECT), VTY_NEWLINE);
+	vty_out(vty, "%s%s", upf_nft_tunmap_get_vmap_init_str(OTC_SELECT), VTY_NEWLINE);
+	vty_out(vty, "%% add tunmap:%s", VTY_NEWLINE);
+	vty_out(vty, "%% %s%s", upf_nft_tunmap_to_str_c(OTC_SELECT, &d), VTY_NEWLINE);
+	vty_out(vty, "%s%s", upf_nft_tunmap_get_ruleset_str(OTC_SELECT, &d), VTY_NEWLINE);
+	vty_out(vty, "%% delete tunmap:%s", VTY_NEWLINE);
+	vty_out(vty, "%s%s", upf_nft_tunmap_get_ruleset_del_str(OTC_SELECT, &d), VTY_NEWLINE);
+	return CMD_SUCCESS;
+}
+
+static struct cmd_node cfg_netinst_node = {
+	NETINST_NODE,
+	"%s(config-netinst)# ",
+	1,
+};
+
+DEFUN(cfg_netinst, cfg_netinst_cmd,
+      "netinst",
+      "Enter the Network Instance configuration node\n")
+{
+	vty->node = NETINST_NODE;
+	return CMD_SUCCESS;
+}
+
+static int config_write_netinst(struct vty *vty)
+{
+	vty_out(vty, "netinst%s", VTY_NEWLINE);
+	netinst_vty_write(vty, &g_upf->netinst, " ", NULL);
+	return CMD_SUCCESS;
+}
+
+DEFUN(cfg_netinst_add, cfg_netinst_add_cmd,
+      "add NAME ADDR",
+      "add Network Instance: associate a PFCP Network Instance name with a local IP address\n"
+      "Network Instance name as received in PFCP Network Instance IE\n"
+      "IP address of a local interface\n")
+{
+	const char *errmsg;
+	if (!netinst_add(g_upf, &g_upf->netinst, argv[0], argv[1], &errmsg)) {
+		vty_out(vty, "%% Error: netinst: cannot add %s %s: %s%s", argv[0], argv[1],
+			errmsg ? : "(unknown error)", VTY_NEWLINE);
+		return CMD_WARNING;
+	}
+	return CMD_SUCCESS;
+}
+
+DEFUN(show_netinst, show_netinst_cmd,
+      "show netinst [NAME]",
+      SHOW_STR "List configured Network Instance entries\n"
+      "Show the Network Instance with this name (show all when omitted)\n")
+{
+	const char *name_or_null = argc > 0 ? argv[0] : NULL;
+
+	if (!netinst_vty_write(vty, &g_upf->netinst, " ", name_or_null)) {
+		if (name_or_null)
+			vty_out(vty, "%% No such Network Instance entry%s", VTY_NEWLINE);
+		else
+			vty_out(vty, "%% No Network Instance entries configured%s", VTY_NEWLINE);
+	}
+	return CMD_SUCCESS;
+}
+
+DEFUN(cfg_netinst_clear, cfg_netinst_clear_cmd,
+      "clear",
+      "Remove all Network Instance entries\n")
+{
+	int count = netinst_clear(&g_upf->netinst);
+	vty_out(vty, "netinst entries removed: %d%s", count, VTY_NEWLINE);
+	return CMD_SUCCESS;
+}
+
 DEFUN(show_pdr, show_pdr_cmd,
       "show pdr",
       SHOW_STR
@@ -282,16 +430,11 @@ DEFUN(show_pdr, show_pdr_cmd,
 DEFUN(show_gtp, show_gtp_cmd,
       "show gtp",
       SHOW_STR
-      "Active GTP tunnels and forwardings\n")
+      "Active GTP tunnels, both tunend and tunmap\n")
 {
 	struct up_peer *peer;
 	int count = 0;
 
-	if (!upf_gtp_dev_first()) {
-		vty_out(vty, "No GTP device open%s", VTY_NEWLINE);
-		return CMD_SUCCESS;
-	}
-
 	llist_for_each_entry(peer, &g_upf->pfcp.ep->peers, entry) {
 		struct up_session *session;
 		int bkt;
@@ -346,25 +489,40 @@ void upf_vty_init()
 	install_element_ve(&show_pdr_cmd);
 	install_element_ve(&show_gtp_cmd);
 	install_element_ve(&show_session_cmd);
+	install_element_ve(&show_netinst_cmd);
+	install_element_ve(&show_nft_rule_append_cmd);
 
 	install_node(&cfg_pfcp_node, config_write_pfcp);
 	install_element(CONFIG_NODE, &cfg_pfcp_cmd);
 
 	install_element(PFCP_NODE, &cfg_pfcp_local_addr_cmd);
 
-	install_node(&cfg_gtp_node, config_write_gtp);
+	install_node(&cfg_tunend_node, config_write_tunend);
+	install_element(CONFIG_NODE, &cfg_tunend_cmd);
 	install_element(CONFIG_NODE, &cfg_gtp_cmd);
 
-	install_element(GTP_NODE, &cfg_gtp_mockup_cmd);
-	install_element(GTP_NODE, &cfg_gtp_no_mockup_cmd);
-	install_element(GTP_NODE, &cfg_gtp_dev_create_cmd);
-	install_element(GTP_NODE, &cfg_gtp_dev_use_cmd);
-	install_element(GTP_NODE, &cfg_gtp_dev_del_cmd);
+	install_element(TUNEND_NODE, &cfg_tunend_mockup_cmd);
+	install_element(TUNEND_NODE, &cfg_tunend_no_mockup_cmd);
+	install_element(TUNEND_NODE, &cfg_tunend_dev_create_cmd);
+	install_element(TUNEND_NODE, &cfg_tunend_dev_use_cmd);
+	install_element(TUNEND_NODE, &cfg_tunend_dev_del_cmd);
 
-	install_node(&cfg_nft_node, config_write_nft);
+	install_node(&cfg_tunmap_node, config_write_tunmap);
+	install_element(CONFIG_NODE, &cfg_tunmap_cmd);
 	install_element(CONFIG_NODE, &cfg_nft_cmd);
 
-	install_element(NFT_NODE, &cfg_nft_mockup_cmd);
-	install_element(NFT_NODE, &cfg_nft_no_mockup_cmd);
-	install_element(NFT_NODE, &cfg_nft_table_name_cmd);
+	install_element(TUNMAP_NODE, &cfg_tunmap_mockup_cmd);
+	install_element(TUNMAP_NODE, &cfg_tunmap_no_mockup_cmd);
+	install_element(TUNMAP_NODE, &cfg_tunmap_table_name_cmd);
+	install_element(TUNMAP_NODE, &cfg_tunmap_nft_rule_append_cmd);
+	install_element(TUNMAP_NODE, &cfg_tunmap_no_nft_rule_append_cmd);
+	install_element(TUNMAP_NODE, &show_nft_rule_append_cmd);
+	install_element(TUNMAP_NODE, &show_nft_rule_tunmap_example_cmd);
+
+	install_node(&cfg_netinst_node, config_write_netinst);
+	install_element(CONFIG_NODE, &cfg_netinst_cmd);
+
+	install_element(NETINST_NODE, &cfg_netinst_clear_cmd);
+	install_element(NETINST_NODE, &cfg_netinst_add_cmd);
+	install_element(NETINST_NODE, &show_netinst_cmd);
 }

tests/netinst.vty

@@ -0,0 +1,80 @@
+OsmoUPF> show ?
+...
+  netinst         List configured Network Instance entries
+...
+OsmoUPF> show netinst?
+  netinst  List configured Network Instance entries
+OsmoUPF> show netinst ?
+  [NAME]  Show the Network Instance with this name (show all when omitted)
+
+OsmoUPF> show netinst
+% No Network Instance entries configured
+OsmoUPF> show netinst foo
+% No such Network Instance entry
+
+OsmoUPF> enable
+
+OsmoUPF# show netinst
+% No Network Instance entries configured
+
+OsmoUPF# configure terminal
+OsmoUPF(config)# netinst
+
+OsmoUPF(config-netinst)# list
+...
+  clear
+  add NAME ADDR
+  show netinst [NAME]
+
+OsmoUPF(config-netinst)# clear?
+  clear  Remove all Network Instance entries
+OsmoUPF(config-netinst)# clear ?
+  <cr>  
+
+OsmoUPF(config-netinst)# add?
+  add  add Network Instance: associate a PFCP Network Instance name with a local IP address
+OsmoUPF(config-netinst)# add ?
+  NAME  Network Instance name as received in PFCP Network Instance IE
+OsmoUPF(config-netinst)# add foo ?
+  ADDR  IP address of a local interface
+
+OsmoUPF(config-netinst)# add foo bar
+% Error: netinst: cannot add foo bar: Network Instance address is not a valid IP address string
+OsmoUPF(config-netinst)# add foo 1.2.3.4
+OsmoUPF(config-netinst)# add foo 2.3.4.5
+% Error: netinst: cannot add foo 2.3.4.5: Network Instance entry with this name already exists
+OsmoUPF(config-netinst)# add bar 2.3.4.5
+OsmoUPF(config-netinst)# show netinst
+ add foo 1.2.3.4
+ add bar 2.3.4.5
+OsmoUPF(config-netinst)# add baz 1:2:3:4::0
+OsmoUPF(config-netinst)# show netinst
+ add foo 1.2.3.4
+ add bar 2.3.4.5
+ add baz 1:2:3:4::0
+OsmoUPF(config-netinst)# show netinst foo
+ add foo 1.2.3.4
+OsmoUPF(config-netinst)# show netinst bar
+ add bar 2.3.4.5
+OsmoUPF(config-netinst)# show netinst baz
+ add baz 1:2:3:4::0
+
+OsmoUPF(config-netinst)# show running-config
+...
+netinst
+ add foo 1.2.3.4
+ add bar 2.3.4.5
+ add baz 1:2:3:4::0
+...
+
+OsmoUPF(config-netinst)# clear
+netinst entries removed: 3
+OsmoUPF(config-netinst)# show netinst
+% No Network Instance entries configured
+OsmoUPF(config-netinst)# clear
+netinst entries removed: 0
+
+OsmoUPF(config-netinst)# show netinst?
+  netinst  List configured Network Instance entries
+OsmoUPF(config-netinst)# show netinst ?
+  [NAME]  Show the Network Instance with this name (show all when omitted)

tests/nft-rule.vty

@@ -0,0 +1,46 @@
+OsmoUPF> enable
+OsmoUPF# configure terminal
+OsmoUPF(config)# tunmap
+
+OsmoUPF(config-tunmap)# show nft-rule tunmap example
+% init verdict map:
+add table inet osmo-upf { flags owner; };
+
+add chain inet osmo-upf pre { type filter hook prerouting priority -300; policy accept; };
+add chain inet osmo-upf post { type filter hook postrouting priority 400; policy accept; };
+add map inet osmo-upf tunmap-pre { typeof ip daddr . @ih,32,32 : verdict; };
+add map inet osmo-upf tunmap-post { typeof meta mark : verdict; };
+add rule inet osmo-upf pre udp dport 2152 ip daddr . @ih,32,32 vmap @tunmap-pre;
+add rule inet osmo-upf post meta mark vmap @tunmap-post;
+
+% add tunmap:
+% ACCESS 1.1.1.1:0x102 <---> 2.2.2.1:0x201 UPF 2.2.2.3:0x203 <---> 3.3.3.3:0x302 CORE
+add chain inet osmo-upf tunmap-pre-123;
+add rule inet osmo-upf tunmap-pre-123 ip daddr set 3.3.3.3 meta mark set 123 counter accept;
+add chain inet osmo-upf tunmap-post-123;
+add rule inet osmo-upf tunmap-post-123 ip saddr set 2.2.2.3 @ih,32,32 set 0x302 counter accept;
+add element inet osmo-upf tunmap-pre { 2.2.2.1 . 0x201 : jump tunmap-pre-123 };
+add element inet osmo-upf tunmap-post { 123 : jump tunmap-post-123 };
+add chain inet osmo-upf tunmap-pre-321;
+add rule inet osmo-upf tunmap-pre-321 ip daddr set 1.1.1.1 meta mark set 321 counter accept;
+add chain inet osmo-upf tunmap-post-321;
+add rule inet osmo-upf tunmap-post-321 ip saddr set 2.2.2.1 @ih,32,32 set 0x102 counter accept;
+add element inet osmo-upf tunmap-pre { 2.2.2.3 . 0x203 : jump tunmap-pre-321 };
+add element inet osmo-upf tunmap-post { 321 : jump tunmap-post-321 };
+
+% delete tunmap:
+delete element inet osmo-upf tunmap-pre { 2.2.2.1 . 0x201 };
+delete element inet osmo-upf tunmap-post { 123 };
+delete chain inet osmo-upf tunmap-pre-123;
+delete chain inet osmo-upf tunmap-post-123;
+delete element inet osmo-upf tunmap-pre { 2.2.2.3 . 0x203 };
+delete element inet osmo-upf tunmap-post { 321 };
+delete chain inet osmo-upf tunmap-pre-321;
+delete chain inet osmo-upf tunmap-post-321;
+
+OsmoUPF(config-tunmap)# show nft-rule tunmap append
+% deprecated config option: 'show nft-rule tunmap append'
+OsmoUPF(config-tunmap)# nft-rule tunmap append meta nftrace set 1
+% deprecated config option: 'nft-rule tunmap append'
+OsmoUPF(config-tunmap)# no nft-rule tunmap append
+% deprecated config option: 'no nft-rule tunmap append'

tests/upf.vty

@@ -13,23 +13,86 @@ OsmoUPF(config-pfcp)# local-addr ?
   IP_ADDR  IP address
 OsmoUPF(config-pfcp)# exit
 
+OsmoUPF(config)# # ensure its old name "gtp" enters the tunend node
 OsmoUPF(config)# gtp
-OsmoUPF(config-gtp)# list
+OsmoUPF(config-tunend)# list
 ...
+  mockup
+  no mockup
   dev create DEVNAME [LISTEN_ADDR]
-  dev use DEVNAME
+  dev use DEVNAME [LOCAL_ADDR]
   dev delete DEVNAME
 
-OsmoUPF(config-gtp)# dev?
+OsmoUPF(config-tunend)# exit
+OsmoUPF(config)# tunend
+OsmoUPF(config-tunend)# list
+...
+  dev create DEVNAME [LISTEN_ADDR]
+  dev use DEVNAME [LOCAL_ADDR]
+  dev delete DEVNAME
+
+OsmoUPF(config-tunend)# dev?
   dev  Configure the GTP device to use for encaps/decaps.
-OsmoUPF(config-gtp)# dev ?
-  create  Add GTP device, creating a new Linux kernel GTP device. Will listen on GTPv1 port 2152 and GTPv0 port 3386 on the specified interface, or on ANY if LISTEN_ADDR is omitted.
+OsmoUPF(config-tunend)# dev ?
+  create  Add GTP device, creating a new Linux kernel GTP device. Will listen on GTPv1 port 2152 and GTPv0 port 3386 on the specified LISTEN_ADDR
   use     Add GTP device, using an existing Linux kernel GTP device, e.g. created by 'gtp-link'
   delete  Remove a GTP device from the configuration, and delete the Linux kernel GTP device if it was created here.
-OsmoUPF(config-gtp)# dev create ?
+OsmoUPF(config-tunend)# dev create ?
   DEVNAME  device name, e.g. 'apn0'
-OsmoUPF(config-gtp)# dev create foo ?
-  [LISTEN_ADDR]  IPv4 or IPv6 address to listen on, omit for ANY
-OsmoUPF(config-gtp)# dev delete ?
+OsmoUPF(config-tunend)# dev create foo ?
+  [LISTEN_ADDR]  IPv4 or IPv6 address to listen on, omit for ANY. LISTEN_ADDR is used to pick a GTP device matching the local address for a PFCP Network Instance, which are configured in the 'netinst' node.
+OsmoUPF(config-tunend)# dev use ?
   DEVNAME  device name, e.g. 'apn0'
-OsmoUPF(config-gtp)# exit
+OsmoUPF(config-tunend)# dev use foo ?
+  [LOCAL_ADDR]  The local GTP address this device listens on. It is assumed to be ANY when omitted. LOCAL_ADDR is used to pick a GTP device matching the local address for a PFCP Network Instance, which are configured in the 'netinst' node.
+OsmoUPF(config-tunend)# dev delete ?
+  DEVNAME  device name, e.g. 'apn0'
+OsmoUPF(config-tunend)# exit
+
+OsmoUPF(config)# # ensure its old name "nft" enters the tunmap node
+OsmoUPF(config)# nft
+OsmoUPF(config-tunmap)# list
+...
+  mockup
+  no mockup
+  table-name TABLE_NAME
+  show nft-rule tunmap example
+OsmoUPF(config-tunmap)# exit
+
+OsmoUPF(config)# tunmap
+OsmoUPF(config-tunmap)# list
+...
+  mockup
+  no mockup
+  table-name TABLE_NAME
+  show nft-rule tunmap example
+
+OsmoUPF(config-tunmap)# mockup?
+  mockup  don't actually send rulesets to nftables, just return success
+OsmoUPF(config-tunmap)# no ?
+  mockup  operate nftables rulesets normally
+
+OsmoUPF(config-tunmap)# table-name?
+  table-name  Set the nft inet table name to create and place GTP tunnel forwarding chains in (as in 'nft add table inet foo'). If multiple instances of osmo-upf are running on the same system, each osmo-upf must have its own table name. Otherwise the names of created forwarding chains will collide. The default table name is "osmo-upf".
+OsmoUPF(config-tunmap)# table-name ?
+  TABLE_NAME  nft inet table name
+
+OsmoUPF(config-tunmap)# nft-rule?
+% There is no matched command.
+OsmoUPF(config-tunmap)# nft-rule ?
+% There is no matched command.
+OsmoUPF(config-tunmap)# nft-rule tunmap ?
+% There is no matched command.
+OsmoUPF(config-tunmap)# nft-rule tunmap append ?
+% There is no matched command.
+
+OsmoUPF(config-tunmap)# show?
+  show  Show running system information
+OsmoUPF(config-tunmap)# show ?
+...
+  nft-rule        nftables rule specifics
+...
+OsmoUPF(config-tunmap)# show nft-rule ?
+  tunmap  GTP tunmap use case (a.k.a. forwarding between two GTP tunnels)
+OsmoUPF(config-tunmap)# show nft-rule tunmap ?
+  example  Print a complete nftables ruleset for a tunmap filled with example IP addresses and TEIDs