Bump version: 1.2.0.2-3868 → 1.2.1

Patch release including support for XOR-3G

Related: OS#4924
Change-Id: I9b0c4851bde641f2f818302ee41dc19cef435f80

.gitignore

@@ -2,9 +2,12 @@
 *.lo
 *.la
 *.db
+*.db-shm
+*.db-wal
 *.pyc
 .*.sw?
 .version
+.tarball-version
 Makefile
 Makefile.in
 aclocal.m4
@@ -37,6 +40,7 @@ src/gsupclient/gsup-test-client
 tests/atconfig
 tests/testsuite
 tests/testsuite.log
+tests/testsuite.dir
 
 tests/auc/auc_3g_test
 tests/auc/auc_ts_55_205_test_sets.c
@@ -45,3 +49,16 @@ tests/auc/auc_test
 tests/gsup_server/gsup_server_test
 tests/gsup/gsup_test
 tests/db/db_test
+tests/hlr_vty_test.db*
+tests/db_upgrade/*.dump
+
+# manuals
+doc/manuals/*.html
+doc/manuals/*.svg
+doc/manuals/*.pdf
+doc/manuals/*__*.png
+doc/manuals/*.check
+doc/manuals/generated/
+doc/manuals/osmomsc-usermanual.xml
+doc/manuals/common
+doc/manuals/build

Makefile.am

@@ -5,6 +5,7 @@ SUBDIRS = \
 	src \
 	include \
 	sql \
+	contrib \
 	tests \
 	$(NULL)
 
@@ -12,6 +13,9 @@ EXTRA_DIST = \
 	.version \
 	$(NULL)
 
+AM_DISTCHECK_CONFIGURE_FLAGS = \
+	--with-systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir)
+
 pkgconfigdir = $(libdir)/pkgconfig
 pkgconfig_DATA = libosmo-gsup-client.pc
 

configure.ac

@@ -34,11 +34,11 @@ PKG_PROG_PKG_CONFIG([0.20])
 
 PKG_CHECK_MODULES(TALLOC, [talloc >= 2.0.1])
 
-PKG_CHECK_MODULES(LIBOSMOCORE, libosmocore >= 0.11.0)
-PKG_CHECK_MODULES(LIBOSMOGSM, libosmogsm >= 0.11.0)
-PKG_CHECK_MODULES(LIBOSMOVTY, libosmovty >= 0.11.0)
-PKG_CHECK_MODULES(LIBOSMOCTRL, libosmoctrl >= 0.11.0)
-PKG_CHECK_MODULES(LIBOSMOABIS, libosmoabis >= 0.5.0)
+PKG_CHECK_MODULES(LIBOSMOCORE, libosmocore >= 1.3.0)
+PKG_CHECK_MODULES(LIBOSMOGSM, libosmogsm >= 1.3.0)
+PKG_CHECK_MODULES(LIBOSMOVTY, libosmovty >= 1.3.0)
+PKG_CHECK_MODULES(LIBOSMOCTRL, libosmoctrl >= 1.3.0)
+PKG_CHECK_MODULES(LIBOSMOABIS, libosmoabis >= 0.6.0)
 
 PKG_CHECK_MODULES(SQLITE3, sqlite3)
 
@@ -59,6 +59,21 @@ then
 	CPPFLAGS="$CPPFLAGS -fsanitize=address -fsanitize=undefined"
 fi
 
+AC_ARG_ENABLE([sqlite_talloc],
+		AC_HELP_STRING([--enable-sqlite-talloc],
+				[Configure SQLite3 to use talloc memory allocator [default=no]]),
+		[sqlite_talloc="$enableval"],[sqlite_talloc="no"])
+if test "x$sqlite_talloc" = "xyes" ; then
+	# Older versions of SQLite3 (at least 3.8.2) become unstable with talloc.
+	# Feel free to relax to 3.24.0 > VER > 3.8.2 if it works for you.
+	# FIXME: PKG_CHECK_MODULES() may return cached result here!
+	PKG_CHECK_MODULES(SQLITE3, sqlite3 >= 3.24.0)
+	AC_DEFINE([SQLITE_USE_TALLOC], 1, [Use talloc for SQLite3])
+fi
+AC_MSG_CHECKING([whether to use talloc for SQLite3])
+AC_MSG_RESULT([$sqlite_talloc])
+AM_CONDITIONAL([DB_SQLITE_DEBUG], [test "x$sqlite_talloc" = "xyes"])
+
 AC_ARG_ENABLE(werror,
 	[AS_HELP_STRING(
 		[--enable-werror],
@@ -92,21 +107,85 @@ AC_MSG_CHECKING([whether to enable VTY/CTRL tests])
 AC_MSG_RESULT([$enable_ext_tests])
 AM_CONDITIONAL(ENABLE_EXT_TESTS, test "x$enable_ext_tests" = "xyes")
 
+# Generate manuals
+AC_ARG_ENABLE(manuals,
+	[AS_HELP_STRING(
+		[--enable-manuals],
+		[Generate manual PDFs [default=no]],
+	)],
+	[osmo_ac_build_manuals=$enableval], [osmo_ac_build_manuals="no"])
+AM_CONDITIONAL([BUILD_MANUALS], [test x"$osmo_ac_build_manuals" = x"yes"])
+AC_ARG_VAR(OSMO_GSM_MANUALS_DIR, [path to common osmo-gsm-manuals files, overriding pkg-config and "../osmo-gsm-manuals"
+	fallback])
+if test x"$osmo_ac_build_manuals" = x"yes"
+then
+	# Find OSMO_GSM_MANUALS_DIR (env, pkg-conf, fallback)
+	if test -n "$OSMO_GSM_MANUALS_DIR"; then
+		echo "checking for OSMO_GSM_MANUALS_DIR... $OSMO_GSM_MANUALS_DIR (from env)"
+	else
+		OSMO_GSM_MANUALS_DIR="$($PKG_CONFIG osmo-gsm-manuals --variable=osmogsmmanualsdir 2>/dev/null)"
+		if test -n "$OSMO_GSM_MANUALS_DIR"; then
+			echo "checking for OSMO_GSM_MANUALS_DIR... $OSMO_GSM_MANUALS_DIR (from pkg-conf)"
+		else
+			OSMO_GSM_MANUALS_DIR="../osmo-gsm-manuals"
+			echo "checking for OSMO_GSM_MANUALS_DIR... $OSMO_GSM_MANUALS_DIR (fallback)"
+		fi
+	fi
+	if ! test -d "$OSMO_GSM_MANUALS_DIR"; then
+		AC_MSG_ERROR("OSMO_GSM_MANUALS_DIR does not exist! Install osmo-gsm-manuals or set OSMO_GSM_MANUALS_DIR.")
+	fi
+
+	# Find and run check-depends
+	CHECK_DEPENDS="$OSMO_GSM_MANUALS_DIR/check-depends.sh"
+	if ! test -x "$CHECK_DEPENDS"; then
+		CHECK_DEPENDS="osmo-gsm-manuals-check-depends"
+	fi
+	if ! $CHECK_DEPENDS; then
+		AC_MSG_ERROR("missing dependencies for --enable-manuals")
+	fi
+
+	# Put in Makefile with absolute path
+	OSMO_GSM_MANUALS_DIR="$(realpath "$OSMO_GSM_MANUALS_DIR")"
+	AC_SUBST([OSMO_GSM_MANUALS_DIR])
+fi
+
+# https://www.freedesktop.org/software/systemd/man/daemon.html
+AC_ARG_WITH([systemdsystemunitdir],
+     [AS_HELP_STRING([--with-systemdsystemunitdir=DIR], [Directory for systemd service files])],,
+     [with_systemdsystemunitdir=auto])
+AS_IF([test "x$with_systemdsystemunitdir" = "xyes" -o "x$with_systemdsystemunitdir" = "xauto"], [
+     def_systemdsystemunitdir=$($PKG_CONFIG --variable=systemdsystemunitdir systemd)
+
+     AS_IF([test "x$def_systemdsystemunitdir" = "x"],
+   [AS_IF([test "x$with_systemdsystemunitdir" = "xyes"],
+    [AC_MSG_ERROR([systemd support requested but pkg-config unable to query systemd package])])
+    with_systemdsystemunitdir=no],
+   [with_systemdsystemunitdir="$def_systemdsystemunitdir"])])
+AS_IF([test "x$with_systemdsystemunitdir" != "xno"],
+      [AC_SUBST([systemdsystemunitdir], [$with_systemdsystemunitdir])])
+AM_CONDITIONAL([HAVE_SYSTEMD], [test "x$with_systemdsystemunitdir" != "xno"])
+
 AC_MSG_RESULT([CFLAGS="$CFLAGS"])
 AC_MSG_RESULT([CPPFLAGS="$CPPFLAGS"])
 
 AC_OUTPUT(
 	Makefile
 	doc/Makefile
+	doc/examples/Makefile
 	src/Makefile
 	src/gsupclient/Makefile
 	include/Makefile
+	include/osmocom/Makefile
+	include/osmocom/hlr/Makefile
 	libosmo-gsup-client.pc
 	sql/Makefile
+	doc/manuals/Makefile
+	contrib/Makefile
+	contrib/systemd/Makefile
 	tests/Makefile
 	tests/auc/Makefile
 	tests/auc/gen_ts_55_205_test_sets/Makefile
 	tests/gsup_server/Makefile
-	tests/gsup/Makefile
 	tests/db/Makefile
+	tests/db_upgrade/Makefile
 	)

contrib/Makefile.am

@@ -0,0 +1 @@
+SUBDIRS = systemd

contrib/jenkins.sh

@@ -1,5 +1,10 @@
 #!/bin/sh
 # jenkins build helper script for osmo-hlr.  This is how we build on jenkins.osmocom.org
+#
+# environment variables:
+# * WITH_MANUALS: build manual PDFs if set to "1"
+# * PUBLISH: upload manuals after building if set to "1" (ignored without WITH_MANUALS = "1")
+#
 
 if ! [ -x "$(command -v osmo-build-dep.sh)" ]; then
 	echo "Error: We need to have scripts/osmo-deps.sh from http://git.osmocom.org/osmo-ci/ in PATH !"
@@ -22,10 +27,18 @@ verify_value_string_arrays_are_terminated.py $(find . -name "*.[hc]")
 
 export PKG_CONFIG_PATH="$inst/lib/pkgconfig:$PKG_CONFIG_PATH"
 export LD_LIBRARY_PATH="$inst/lib"
+export PATH="$inst/bin:$PATH"
 
 osmo-build-dep.sh libosmocore "" ac_cv_path_DOXYGEN=false
 osmo-build-dep.sh libosmo-abis
 
+# Additional configure options and depends
+CONFIG=""
+if [ "$WITH_MANUALS" = "1" ]; then
+	osmo-build-dep.sh osmo-gsm-manuals
+	CONFIG="--enable-manuals"
+fi
+
 set +x
 echo
 echo
@@ -36,9 +49,14 @@ set -x
 
 cd "$base"
 autoreconf --install --force
-./configure --enable-sanitize --enable-external-tests --enable-werror
+./configure --enable-sanitize --enable-external-tests --enable-werror $CONFIG
 $MAKE $PARALLEL_MAKE
 $MAKE check || cat-testlogs.sh
-$MAKE distcheck || cat-testlogs.sh
+DISTCHECK_CONFIGURE_FLAGS="$CONFIG" $MAKE distcheck || cat-testlogs.sh
 
+if [ "$WITH_MANUALS" = "1" ] && [ "$PUBLISH" = "1" ]; then
+	make -C "$base/doc/manuals" publish
+fi
+
+$MAKE maintainer-clean
 osmo-clean-workspace.sh

contrib/systemd/Makefile.am

@@ -0,0 +1,6 @@
+EXTRA_DIST = osmo-hlr.service
+
+if HAVE_SYSTEMD
+systemdsystemunit_DATA = \
+  osmo-hlr.service
+endif

contrib/systemd/osmo-hlr.service

@@ -1,5 +1,6 @@
 [Unit]
 Description=Osmocom Home Location Register (OsmoHLR)
+Documentation=https://osmocom.org/projects/osmo-hlr/wiki/OsmoHLR
 
 [Service]
 Type=simple

debian/changelog

@@ -1,3 +1,193 @@
+osmo-hlr (1.2.1) unstable; urgency=medium
+
+  [ Neels Hofmeyr ]
+  * test_nodes.vty: remove cruft
+
+  [ Harald Welte ]
+  * support the XOR algorithm for UMTS AKA
+
+ -- Harald Welte <laforge@osmocom.org>  Tue, 29 Dec 2020 20:39:53 +0100
+
+osmo-hlr (1.2.0) unstable; urgency=medium
+
+  [ Ruben Undheim ]
+  * Fix test for return codes on mipsel and alpha archs
+
+  [ Thorsten Alteholz ]
+  * fix spelling errors detected by lintian
+
+  [ Pau Espin Pedrol ]
+  * tests: Fix db_test err file to expect error code name instead of value
+
+  [ Oliver Smith ]
+  * tests/test_nodes.vty: check less libosmocore cmds
+  * tests/db_upgrade: disable for old sqlite versions
+  * gitignore: add tests/db_upgrade/*.dump
+  * gsup_client.h: fix license header: GPLv2+
+  * tests/auc: change back to python3
+
+  [ Neels Hofmeyr ]
+  * fix double free in osmo_gsup_client_enc_send()
+  * db upgrade to v2: log version 2, not 1
+  * fix upgrade to version 2: imei column default value
+  * add --db-check option
+  * hlr.sql: move comment
+  * add db_upgrade test
+  * hlr db schema 3: hlr_number -> msc_number
+  * db.c: code dup: add db_run_statements() for arrays of statements
+  * move headers to include/osmocom/hlr
+  * fix upgrade test in presence of ~/.sqliterc
+  * db upgrade: remove some code dup
+  * add osmo_gsup_msgb_alloc()
+  * Makefile convenience: add VTY_TEST var to run only one test
+  * remove gsup_test
+  * test_nodes.vty: tweak: add some '?' checks
+  * db v4: add column last_lu_seen_ps
+
+  [ Harald Welte ]
+  * AUC: Add support for setting the AMF separation bit to '1' for EUTRAN
+  * hlr: exit(2) on unsupported positional arguments on command line
+
+ -- Pau Espin Pedrol <pespin@sysmocom.de>  Fri, 03 Jan 2020 12:37:35 +0100
+
+osmo-hlr (1.1.0) unstable; urgency=medium
+
+  [ Oliver Smith ]
+  * docs: running: document --db-upgrade
+  * Add IMEI column to subscriber table
+  * Optionally store IMEI in subscriber table
+  * VTY tests: fill DB before running test
+  * VTY: integrate IMEI
+  * hlr.c: replace deprecated osmo_gsup_get_err_msg_type()
+  * hlr.c: move hlr_ctx to the top
+  * tests: use -no-install libtool flag to avoid ./lt-* scripts
+  * Cosmetic: gsup_route_find: comment addr, addrlen
+  * USSD: save MO USSD's originating MSC's vlr_number
+  * USSD: don't use gsm0480_msgb_alloc_name()
+  * hlr.c: forward GSUP messages between clients
+  * db_hlr.c: db_subscr_create(): add flags argument
+  * db_hlr.c: add db_subscr_exists_by_imsi()
+  * Create subscribers on demand
+  * Document subscribers create on demand feature
+  * debian: create -doc subpackage with pdf manuals
+  * db_test: set timezone to work around mktime bug
+  * db_hlr: zero-initialize "struct tm"
+  * rx_check_imei_req(): fix IMEI bounds checking
+  * contrib/jenkins.sh: run "make maintainer-clean"
+  * VTY: add subscriber update network-access-mode
+  * manuals: improve subscribers create on demand
+  * gitignore: ignore everything generated in db_test
+  * db_auc.c: verify hex key sizes read from DB
+
+  [ Max ]
+  * Log ip:port when adding GSUP routes
+  * Add link to project wiki to .service file
+  * Enable statsd support
+
+  [ Vadim Yanitskiy ]
+  * hlr.c: properly terminate the process on SIGTERM
+  * hlr.c: fix: also store the session state in read_cb_forward()
+  * hlr.c: fix: properly print the original message type in read_cb_forward()
+  * hlr.c: check the presence of msgb->l2h in read_cb()
+  * hlr.c: fix possible msgb memleaks in read_cb()
+  * db_hlr.c: add db_subscr_exists_by_msisdn()
+  * src/db.h: use GSM23003_MSISDN_MAX_DIGITS for MSISDN buffer size
+  * src/hlr.c: fix deprecation warning: use gsm48_decode_bcd_number2()
+  * hlr_ussd.c: fix: properly pass invokeID in handle_ussd_own_msisdn()
+  * hlr_ussd.c: rx_proc_ss_req(): fix NULL pointer dereference
+  * build: fix mess with 'db_test_SOURCES' and 'db_test_LDADD'
+  * tests/db_test: close the database when test is finished
+  * src/db.c: integrate SQLite3 with talloc allocator
+
+  [ Neels Hofmeyr ]
+  * USSD: fix routing to multiple MSC
+  * fix error logging for GSUP route
+  * add missing error log: invalid IMSI
+  * osmo-hlr: allow configuring db path from cfg file
+  * use new OSMO_IMSI_BUF_SIZE
+
+  [ Daniel Willmann ]
+  * manuals: Add script to update vty/counter documentation from docker
+  * manuals: Update vty documentation
+
+  [ Pau Espin Pedrol ]
+  * Remove undefined param passed to logging_vty_add_cmds
+  * configure.ac: Require libosmocore 1.2.0
+
+ -- Pau Espin Pedrol <pespin@sysmocom.de>  Wed, 07 Aug 2019 16:14:23 +0200
+
+osmo-hlr (1.0.0) unstable; urgency=medium
+
+  [ Stefan Sperling ]
+  * move creation of insert subscriber data messages to a common function
+
+  [ Harald Welte ]
+  * Return proper GSUP error in case of too short IMSI
+  * disable blind subscriber insertion into every VLR/SGSN
+  * gsup_server: Add "priv" pointer and make it point to 'struct hlr'
+  * move osmo_gsup_addr_send() declaration from luop.h to gsup_router.h
+  * gsup_router: Use "#pragma once" and add missing #includes
+  * Add "show gsup-connections" VTY command
+  * import gsup_client.c as new libosmo-gsup-client
+  * gsup_client: rename gsup_client_* to osmo_gsup_client_*
+  * GSUP: Log GSUP route add/remove
+  * hlr: Export + Declare global g_hlr symbol
+  * USSD: Add Core USSD handling + VTY routing config to HLR
+  * USSD: Add basic dispatch + decode of GSUP-encapsulated SS/USSD
+  * hlr_ussd: Introduce LOGPSS() macro
+  * USSD: Send ReturnError component if USSD Code unknown / EUSE disconnected
+  * USSD: Further unification of log output; Use LOGPSS when possible
+  * osmo-hlr.cfg: Don't enable DEBUG logging by default
+  * USSD: Add new "DSS" logging category and use it appropriately
+  * USSD: fix null-pointer deref in "default-route" vty/config cmd
+  * Add osmo-euse-demo as minimalistic test of a External USSD (EUSE) handler
+  * USSD: Add support for internal USSD handlers
+  * debian: Add sub-package for libosmo-gsup-client
+  * pkg-config: Fix libosmo-gsup-client pkg-config file
+  * gitignore: Add .tarball-version
+  * debian: Make libosmo-gsup-client-dev depend on libosmo-gsup-client0
+  * USSD: Fix "ussd default-route"
+  * libosmo-gsup-client: License is GPLv2-or-later
+  * osmo-hlr.cfg: Ensure well-formed config file example
+  * test_nodes.vty: Since libosmocore 1.0.0, we only have one space
+
+  [ Martin Hauke ]
+  * sql/Makefile.am: Make docsdir completely configurable
+  * debian: Fix typo in package description
+
+  [ Pau Espin Pedrol ]
+  * debian: Avoid installing duplicate cfg file in /etc
+  * sql/Makefile: Install hlr_data.sql as example together with hlr.sql
+  * sql/Makefile: Install sql files under doc/.../sql subdir
+  * sql/Makefile: Create empty /var/lib/osmocom directory at install time
+  * Install systemd services with autotools
+  * Move doc/Makefile.am to doc/examples/Makefile.am
+  * Install sample cfg file to /etc/osmocom
+
+  [ Vadim Yanitskiy ]
+  * hlr.c: move deinitialization code from SIGINT handler
+  * hlr.c: free root talloc context on exit
+  * hlr.c: track the use of talloc NULL memory contexts
+  * src/db.c: fix: make sure the database is properly closed
+  * src/db.c: don't ignore the result of db_bootstrap()
+  * hlr_vty_subscr.c: fix subscriber creation command help
+  * Update .gitignore: add missing build products
+  * tests/Makefile.am: also remove temporary sqlite files
+  * hlr_ussd.h: add #pragma once include guard
+  * hlr_ussd.h: use proper libc headers
+  * Update .gitignore: ignore osmo-euse-demo
+  * hlr_ussd.h: drop meaningless forward declaration
+  * USSD/hlr_vty.c: print error if EUSE is not found
+  * hlr_ussd.c: fix: properly print a EUSE / IUSE name
+  * hlr_ussd.c: avoid using CR and NL in IUSE responses
+
+  [ Neels Hofmeyr ]
+  * fix build: adjust test_nodes.vty to logging change
+  * tweak example config
+  * make: always allow running python tests manually
+
+ -- Harald Welte <laforge@gnumonks.org>  Sun, 20 Jan 2019 19:29:58 +0100
+
 osmo-hlr (0.2.1) unstable; urgency=medium
 
   [ Neels Hofmeyr ]

debian/control

@@ -7,12 +7,13 @@ Build-Depends: debhelper (>= 9),
                dh-autoreconf,
                dh-systemd (>= 1.5),
                autotools-dev,
-               python-minimal,
+               python3-minimal,
                libosmocore-dev,
                libosmo-abis-dev,
                libosmo-netif-dev,
                libsqlite3-dev,
-               sqlite3
+               sqlite3,
+               osmo-gsm-manuals-dev
 Standards-Version: 3.9.6
 Vcs-Browser: http://cgit.osmocom.org/osmo-hlr
 Vcs-Git: git://git.osmocom.org/osmo-hlr
@@ -32,3 +33,37 @@ Priority: extra
 Depends: osmo-hlr (= ${binary:Version}), ${misc:Depends}
 Description: Debug symbols for the osmo-hlr
  Make debugging possible
+
+Package: libosmo-gsup-client0
+Section: libs
+Architecture: any
+Multi-Arch: same
+Depends: ${shlibs:Depends},
+	 ${misc:Depends}
+Pre-Depends: ${misc:Pre-Depends}
+Description: Osmocom GSUP (General Subscriber Update Protocol) client library
+  This is a shared library that can be used to implement client programs for
+  the GSUP protocol.  The typical GSUP server is OsmoHLR, with OsmoMSC, OsmoSGSN
+  and External USSD Entities (EUSEs) using this library to implement clients.
+
+Package: libosmo-gsup-client-dev
+Architecture: any
+Multi-Arch: same
+Depends: ${misc:Depends},
+	 libosmo-gsup-client0 (= ${binary:Version}),
+	 libosmocore-dev
+Description: Development headers of Osmocom GSUP client library
+  This is a shared library that can be used to implement client programs for
+  the GSUP protocol.  The typical GSUP server is OsmoHLR, with OsmoMSC, OsmoSGSN
+  and External USSD Entities (EUSEs) using this library to implement clients.
+  .
+  This package contains the development headers.
+
+Package: osmo-hlr-doc
+Architecture: all
+Section: doc
+Priority: optional
+Depends: ${misc:Depends}
+Description: ${misc:Package} PDF documentation
+ Various manuals: user manual, VTY reference manual and/or
+ protocol/interface manuals.

debian/libosmo-gsup-client-dev.install

@@ -0,0 +1,5 @@
+usr/include/osmocom/gsupclient
+usr/lib/*/libosmo-gsup-client*.a
+usr/lib/*/libosmo-gsup-client*.so
+usr/lib/*/libosmo-gsup-client*.la
+usr/lib/*/pkgconfig/libosmo-gsup-client.pc

debian/libosmo-gsup-client0.install

@@ -0,0 +1 @@
+usr/lib/*/libosmo-gsup-client*.so.*

debian/osmo-hlr-doc.install

@@ -0,0 +1 @@
+usr/share/doc/osmo-hlr-doc/*.pdf

debian/osmo-hlr.install

@@ -1,3 +1,5 @@
+/etc/osmocom/osmo-hlr.cfg
+/lib/systemd/system/osmo-hlr.service
 /usr/bin/osmo-hlr
 /usr/bin/osmo-hlr-db-tool
 /usr/share/doc/osmo-hlr/sql/hlr.sql

debian/osmo-hlr.service

@@ -1 +0,0 @@
-../contrib/systemd/osmo-hlr.service

debian/rules

@@ -15,3 +15,10 @@ override_dh_strip:
 # Print test results in case of a failure
 override_dh_auto_test:
 	dh_auto_test || (find . -name testsuite.log -exec cat {} \; ; false)
+
+override_dh_auto_configure:
+	dh_auto_configure -- --with-systemdsystemunitdir=/lib/systemd/system --enable-manuals
+
+# Don't create .pdf.gz files (barely saves space and they can't be opened directly by most pdf readers)
+override_dh_compress:
+	dh_compress -X.pdf

doc/Makefile.am

@@ -1,22 +1,4 @@
-CFG_FILES = find $(srcdir) -name '*.cfg*' | sed -e 's,^$(srcdir),,'
-
-dist-hook:
-	for f in $$($(CFG_FILES)); do \
-		j="$(distdir)/$$f" && \
-		mkdir -p "$$(dirname $$j)" && \
-		$(INSTALL_DATA) $(srcdir)/$$f $$j; \
-	done
-
-install-data-hook:
-	for f in $$($(CFG_FILES)); do \
-		j="$(DESTDIR)$(docdir)/$$f" && \
-		mkdir -p "$$(dirname $$j)" && \
-		$(INSTALL_DATA) $(srcdir)/$$f $$j; \
-	done
-
-uninstall-hook:
-	@$(PRE_UNINSTALL)
-	for f in $$($(CFG_FILES)); do \
-		j="$(DESTDIR)$(docdir)/$$f" && \
-		$(RM) $$j; \
-	done
+SUBDIRS = \
+	examples \
+	manuals \
+	$(NULL)

doc/examples/Makefile.am

@@ -0,0 +1,27 @@
+osmoconfdir = $(sysconfdir)/osmocom
+osmoconf_DATA = osmo-hlr.cfg
+
+EXTRA_DIST = osmo-hlr.cfg
+
+CFG_FILES = find $(srcdir) -name '*.cfg*' | sed -e 's,^$(srcdir),,'
+
+dist-hook:
+	for f in $$($(CFG_FILES)); do \
+		j="$(distdir)/$$f" && \
+		mkdir -p "$$(dirname $$j)" && \
+		$(INSTALL_DATA) $(srcdir)/$$f $$j; \
+	done
+
+install-data-hook:
+	for f in $$($(CFG_FILES)); do \
+		j="$(DESTDIR)$(docdir)/examples/$$f" && \
+		mkdir -p "$$(dirname $$j)" && \
+		$(INSTALL_DATA) $(srcdir)/$$f $$j; \
+	done
+
+uninstall-hook:
+	@$(PRE_UNINSTALL)
+	for f in $$($(CFG_FILES)); do \
+		j="$(DESTDIR)$(docdir)/examples/$$f" && \
+		$(RM) $$j; \
+	done

doc/examples/osmo-hlr.cfg

@@ -2,17 +2,18 @@
 ! OsmoHLR example configuration
 !
 log stderr
-  logging filter all 1
-  logging color 1
-  logging print category 1
-  logging timestamp 1
-  logging print extended-timestamp 1
-  logging level all notice
-  logging level main notice
-  logging level db notice
-  logging level auc notice
-  logging level ss info
-  logging level linp error
+ logging filter all 1
+ logging color 1
+ logging print category 1
+ logging print category-hex 0
+ logging print level 1
+ logging print file basename last
+ logging print extended-timestamp 1
+ logging level main notice
+ logging level db notice
+ logging level auc notice
+ logging level ss info
+ logging level linp error
 !
 line vty
  bind 127.0.0.1
@@ -22,3 +23,4 @@ hlr
  gsup
   bind ip 127.0.0.1
  ussd route prefix *#100# internal own-msisdn
+ ussd route prefix *#101# internal own-imsi

doc/manuals/Makefile.am

@@ -0,0 +1,61 @@
+EXTRA_DIST = example_subscriber_add_update_delete.vty \
+    example_subscriber_cs_ps_enabled.ctrl \
+    example_subscriber_info.ctrl \
+    osmohlr-usermanual.adoc \
+    osmohlr-usermanual-docinfo.xml \
+    osmohlr-vty-reference.xml \
+    regen_doc.sh \
+    chapters \
+    vty
+
+if BUILD_MANUALS
+  ASCIIDOC = osmohlr-usermanual.adoc
+  ASCIIDOC_DEPS = $(srcdir)/chapters/*.adoc $(srcdir)/*.vty $(srcdir)/*.ctrl
+  include $(OSMO_GSM_MANUALS_DIR)/build/Makefile.asciidoc.inc
+
+  VTY_REFERENCE = osmohlr-vty-reference.xml
+  include $(OSMO_GSM_MANUALS_DIR)/build/Makefile.vty-reference.inc
+
+  OSMO_REPOSITORY = osmo-hlr
+  include $(OSMO_GSM_MANUALS_DIR)/build/Makefile.common.inc
+endif
+
+TMP_DB = generated/hlr.db
+
+update-examples: update-examples-ctrl update-examples-vty
+
+.PHONY: found-update-deps
+found-update-deps:
+	@if [ ! -f "$(top_srcdir)/sql/hlr.sql" ]; then \
+		echo "You need to define OSMO_HLR_PATH to point at an osmo-hlr.git"; \
+		exit 1; \
+	fi
+	@if [ -z "$(shell which osmo-hlr)" ]; then \
+		echo "osmo-hlr needs to be installed / available in the PATH"; \
+		exit 1; \
+	fi
+	@if [ -z "$(shell which osmo_verify_transcript_ctrl.py)" ]; then \
+		echo "You need to install git.osmocom.org/python/osmo-python-tests.git"; \
+		exit 1; \
+	fi
+	@if [ -z "$(shell which osmo_verify_transcript_vty.py)" ]; then \
+		echo "You need to install git.osmocom.org/python/osmo-python-tests.git"; \
+		exit 1; \
+	fi
+
+update-examples-ctrl: found-update-deps
+	mkdir -p generated
+	rm -f "$(TMP_DB)"
+	sqlite3 "$(TMP_DB)" < "$(top_srcdir)/sql/hlr.sql"
+	sqlite3 "$(TMP_DB)" < "$(top_srcdir)/tests/test_subscriber.sql"
+	osmo_verify_transcript_ctrl.py \
+		-r "osmo-hlr -l $(TMP_DB) -c $(top_srcdir)/doc/examples/osmo-hlr.cfg" \
+		-p 4259 --update *.ctrl
+
+update-examples-vty: found-update-deps
+	mkdir -p generated
+	rm -f "$(TMP_DB)"
+	sqlite3 "$(TMP_DB)" < "$(top_srcdir)/sql/hlr.sql"
+	osmo_verify_transcript_vty.py \
+		-r "osmo-hlr -l $(TMP_DB) -c $(top_srcdir)/doc/examples/osmo-hlr.cfg" \
+		-p 4258 --update *.vty

doc/manuals/chapters/control.adoc

@@ -0,0 +1,106 @@
+[[hlr-ctrl]]
+== Control interface
+
+The actual protocol is described in <<common-control-if>>, the variables common
+to all programs using it are described in <<ctrl_common_vars>>. This section
+describes the CTRL interface variables specific to OsmoHLR.
+
+All subscriber variables are available by different selectors, which are freely
+interchangeable:
+
+.Subscriber selectors available on OsmoHLR's Control interface
+[options="header",width="100%",cols="35%,65%"]
+|===
+|Selector|Comment
+|subscriber.*by-imsi-*'123456'.*|Subscriber selector by IMSI, replace "123456" with the actual IMSI
+|subscriber.*by-msisdn-*'123456'.*|Subscriber selector by MSISDN
+|subscriber.*by-id-*'123456'.*|Subscriber selector by database ID
+|===
+
+Each of the above selectors feature all of these control variables:
+
+.Subscriber variables available on OsmoHLR's Control interface
+[options="header",width="100%",cols="35%,8%,8%,8%,41%"]
+|===
+|Name|Access|Trap|Value|Comment
+|subscriber.by-\*.*info*|R|No||List (short) subscriber information
+|subscriber.by-\*.*info-aud*|R|No||List subscriber authentication tokens
+|subscriber.by-\*.*info-all*|R|No||List both 'info' and 'info-aud' in one
+|subscriber.by-\*.*cs-enabled*|RW|No|'1' or '0'|Enable/disable circuit-switched access
+|subscriber.by-\*.*ps-enabled*|RW|No|'1' or '0'|Enable/disable packet-switched access
+|===
+
+=== subscriber.by-*.info, info-aud, info-all
+
+Query the HLR database and return current subscriber record, in multiple lines
+of the format
+
+----
+name<tab>value
+----
+
+To keep the reply as short as possible, some values are omitted if they are
+empty. These are the returned values and their presence
+modalities; for their meaning, see <<subscriber-params>>:
+
+.Returned values by OsmoHLR's 'info', 'info-all' and 'info-aud' commands
+[options="header",width="100%",cols="15%,15%,30%,40%"]
+|===
+|Returned by 'info-all' and|Name|Format|Presence
+|'info'|id|-9223372036854775808 .. 9223372036854775807 (usually not negative)|always
+|'info'|imsi|6 to 15 decimal digits|always
+|'info'|msisdn|1 to 15 decimal digits|when non-empty
+|'info'|nam_cs|'1' if CS is enabled, or '0'|always
+|'info'|nam_ps|'1' if PS is enabled, or '0'|always
+|'info'|vlr_number|up to 15 decimal digits|when non-empty
+|'info'|sgsn_number|up to 15 decimal digits|when non-empty
+|'info'|sgsn_address||when non-empty
+|'info'|ms_purged_cs|'1' if CS is purged, or '0'|always
+|'info'|ms_purged_ps|'1' if PS is purged, or '0'|always
+|'info'|periodic_lu_timer|0..4294967295|always
+|'info'|periodic_rau_tau_timer|0..4294967295|always
+|'info'|lmsi|8 hex digits|always
+|'info-aud'|aud2g.algo|one of 'comp128v1', 'comp128v2', 'comp128v3' or 'xor'|when valid 2G auth data is set
+|'info-aud'|aud2g.ki|32 hexadecimal digits|when valid 2G auth data is set
+|'info-aud'|aud3g.algo|so far always 'milenage'|when valid 3G auth data is set
+|'info-aud'|aud3g.k|32 hexadecimal digits|when valid 3G auth data is set
+|'info-aud'|aud3g.op|32 hexadecimal digits|when valid 3G auth data is set, *not* when OPC is set
+|'info-aud'|aud3g.opc|32 hexadecimal digits|when valid 3G auth data is set, *not* when OP is set
+|'info-aud'|aud3g.ind_bitlen|0..28|when valid 3G auth data is set
+|'info-aud'|aud3g.sqn|0 .. 18446744073709551615|when valid 3G auth data is set
+|===
+
+This is an example Control Interface transcript that illustrates the various
+'info' commands:
+
+----
+include::../example_subscriber_info.ctrl[]
+----
+
+=== subscriber.by-*.ps-enabled, cs-enabled
+
+Disable or enable packet-/circuit-switched access for the given IMSI;
+
+* 'ps-enabled' switches access to GPRS or UMTS data services,
+* 'cs-enabled' switches access to voice services.
+
+When disabled, the next time this subscriber attempts to do a Location Updating
+GSUP operation for the given domain (i.e. from the SGSN for 'ps-enabled', from
+the MSC/VLR for 'cs-enabled'), it will be rejected by OsmoHLR. Currently
+connected GSUP clients will be notified via GSUP when a subscriber is being
+disabled, so that the subscriber can be dropped in case it is currently
+attached.
+
+The current 'ps-enabled'/'cs-enabled' status can be queried by 'GET' commands,
+and also by looking at 'nam_ps' and 'nam_cs' in a 'subscriber.by-*.info'
+response.
+
+A value of "1" indicates that the given domain is enabled, which is the
+default; a value of "0" disables access.
+
+This is an example transcript that illustrates 'ps-enabled' and 'cs-enabled'
+commands:
+
+----
+include::../example_subscriber_cs_ps_enabled.ctrl[]
+----

doc/manuals/chapters/overview.adoc

@@ -0,0 +1,69 @@
+[[overview]]
+== Overview
+
+This manual should help you getting started with OsmoHLR. It will cover
+aspects of configuring and running the OsmoHLR.
+
+[[intro_overview]]
+=== About OsmoHLR
+
+OsmoHLR is Osmocom's minimal implementation of a Home Location Register (HLR)
+for 2G and 3G GSM and UMTS mobile core networks. Its interfaces are:
+
+- GSUP, serving towards OsmoMSC and OsmoSGSN;
+- A local SQLite database;
+- The Osmocom typical telnet VTY and CTRL interfaces.
+
+Originally, the OpenBSC project's OsmoNITB all-in-one implementation had an
+integrated HLR, managing subscribers and SMS in the same local database. Along
+with the separate OsmoMSC and its new VLR component, OsmoHLR was implemented
+from scratch to alleviate various shortcomings of the internal HLR:
+
+- The separate HLR allows using centralized subscriber management for both
+  circuit-switched and packet-switched domains (i.e. one OsmoHLR for both
+  OsmoMSC and OsmoSGSN).
+
+- VLR and HLR brought full UMTS AKA (Authentication and Key Agreement) support,
+  i.e. Milenage authentication in both the full 3G variant as well as the
+  backwards compatible 2G variant.
+
+- In contrast to the OsmoNITB, the specific way the new OsmoMSC's VLR accesses
+  OsmoHLR brings fully asynchronous subscriber database access.
+
+Find the OsmoHLR issue tracker and wiki online at
+
+- https://osmocom.org/projects/osmo-hlr
+- https://osmocom.org/projects/osmo-hlr/wiki
+
+
+[[fig-gsm]]
+.Typical GSM network architecture used with OsmoHLR
+[graphviz]
+----
+digraph G {
+	rankdir=LR;
+	subgraph cluster_hlr {
+		label = "OsmoHLR";
+		GSUP [label="GSUP server"]
+		DB [label="SQLite DB"]
+		GSUP->DB
+		DB->CTRL [dir="back"]
+		DB->VTY [dir="back"]
+	}
+
+	Admin [label="Admin and\nMaintenance"]
+	SW [label="3rd party software\nintegration"]
+	VTY->Admin [dir="back"]
+	CTRL->SW [dir="back"]
+		
+	MSC [label="MSC/VLR"]
+	MSC->GSUP  [label="GSUP"]
+	SGSN->GSUP [label="GSUP"]
+
+	BSC->MSC
+	HNBGW->MSC
+	HNBGW->SGSN
+	PCU->SGSN
+}
+----
+

doc/manuals/chapters/running.adoc

@@ -0,0 +1,87 @@
+== Running OsmoHLR
+
+The OsmoHLR executable (`osmo-hlr`) offers the following command-line
+arguments:
+
+=== SYNOPSIS
+
+*osmo-hlr* [-h] [-c 'CONFIGFILE'] [-l 'DATABASE'] [-d 'DBGMASK'] [-D] [-s] [-T] [-e 'LOGLEVEL'] [-U] [-V]
+
+=== OPTIONS
+
+// Keep the order the same as in osmo-hlr --help!
+
+*-h, --help*::
+	Print a short help message about the supported options
+*-c, --config-file 'CONFIGFILE'*::
+	Specify the file and path name of the configuration file to be
+	used. If none is specified, use `osmo-hlr.cfg` in the current
+	working directory.
+*-l, --database 'DATABASE'*::
+	Specify the file name of the SQLite3 database to use as HLR/AUC
+	storage
+*-d, --debug 'DBGMASK','DBGLEVELS'*::
+	Set the log subsystems and levels for logging to stderr. This
+	has mostly been superseded by VTY-based logging configuration,
+	see <<logging>> for further information.
+*-D, --daemonize*::
+	Fork the process as a daemon into background.
+*-s, --disable-color*::
+	Disable colors for logging to stderr. This has mostly been
+	deprecated by VTY based logging configuration, see <<logging>>
+	for more information.
+*-T, --timestamp*::
+	Enable time-stamping of log messages to stderr. This has mostly
+	been deprecated by VTY based logging configuration, see
+	<<logging>> for more information.
+*-e, --log-level 'LOGLEVEL'*::
+	Set the global log level for logging to stderr. This has mostly
+	been deprecated by VTY based logging configuration, see
+	<<logging>> for more information.
+*-U, --db-upgrade*::
+	Allow HLR database schema upgrades. If OsmoHLR was updated and
+	requires a newer database schema, it will refuse to start unless
+	this option is specified. The updated database can not be
+	downgraded, make backups as necessary.
+*-V, --version*::
+	Print the compile-time version number of the OsmoHLR program
+
+=== Bootstrap the Database
+
+If no database exists yet, OsmoHLR will automatically create and bootstrap a
+database file with empty tables. If no `-l` command-line option is provided,
+this database file will be created in the current working directory.
+
+Alternatively, you may use the `osmo-hlr-db-tool`, which is installed along
+with `osmo-hlr`, to bootstrap an empty database, or to migrate subscriber data
+from an old 'OsmoNITB' database. See `osmo-hlr-db-tool --help`.
+
+=== Multiple instances
+
+Running multiple instances of `osmo-hlr` on the same computer is possible if
+all interfaces (VTY, CTRL) are separated using the appropriate configuration
+options. The IP based interfaces are binding to local host by default. In order
+to separate the processes, the user has to bind those services to specific but
+different IP addresses and/or ports.
+
+The VTY and the Control interface can be bound to IP addresses from the loopback
+address range, for example:
+
+----
+line vty
+ bind 127.0.0.2
+ctrl
+ bind 127.0.0.2
+----
+
+The GSUP interface can be bound to a specific IP address by the following
+configuration options:
+
+----
+hlr
+ gsup
+  bind ip 10.23.42.1
+----
+
+NOTE: At the time of writing, OsmoHLR lacks a config option to change the GSUP
+port, which is by default TCP port 4222.

doc/manuals/chapters/subscribers.adoc

@@ -0,0 +1,129 @@
+== Managing Subscribers
+
+Subscribers are kept in a local SQLite database file and can be managed via VTY
+and CTRL interfaces.
+
+This section provides some examples; also refer to the OsmoHLR VTY reference
+manual <<vty-ref-osmohlr>> as well as the Control interface described in
+<<hlr-ctrl>>.
+
+=== Example: Add/Update/Delete Subscriber via VTY
+
+The following telnet VTY session adds a subscriber complete with GSM (2G) and
+UMTS (3G and 2G) authentication tokens, and finally removes the subscriber
+again; it assumes that osmo-hlr is running and listening for telnet VTY
+connections on localhost:
+
+----
+$ telnet localhost 4258
+include::../example_subscriber_add_update_delete.vty[]
+----
+
+[[subscriber-params]]
+=== Subscriber Parameters
+
+The following parameters are managed for each subscriber of the HLR, modelled
+roughly after 3GPP TS 23.008, version 13.3.0; note that not all of these
+parameters are necessarily in active use.
+
+The `aud3g` table also applies to 2G networks: it provides UMTS AKA tokens for
+Milenage authentication, which is available both on 3G and 2G networks. On 2G,
+when both MS and network are R99 capable (like OsmoMSC and OsmoSGSN are), the
+full UMTS AKA with Milenage keys from `aud_3g`, using AUTN and extended RES
+tokens, is available. With pre-R99 MS or network configurations, the GSM AKA
+compatible variant of Milenage, still using the Milenage keys from `aud_3g` but
+transceiving only RAND and SRES, may be applicable. (See 3GPP TS 33.102, chapter
+6.8.1, Authentication and key agreement of UMTS subscribers.)
+
+.OsmoHLR's subscriber parameters
+[options="header",width="100%",cols="20%,20%,60%"]
+|===
+|Name|Example|Description
+|imsi|901700000014701|identity of the SIM/USIM, 3GPP TS 23.008 chapter 2.1.1.1
+|msisdn|2342123|number to dial to reach this subscriber (multiple MSISDNs can be stored per subscriber), 3GPP TS 23.008 chapter 2.1.2
+|imeisv|4234234234234275|identity of the mobile device and software version, 3GPP TS 23.008 chapter 2.2.3
+|aud2g.algo|comp128v3|Authentication algorithm ID for GSM AKA, corresponds to enum osmo_auth_algo
+|aud2g.ki||Subscriber's secret key (128bit)
+|aud3g.algo|milenage|Authentication algorithm ID for UMTS AKA (applies to both 3G and 2G networks), corresponds to enum osmo_auth_algo
+|aud3g.k|(32 hexadecimal digits)|Subscriber's secret key (128bit)
+|aud3g.op|(32 hexadecimal digits)|Operator's secret key (128bit)
+|aud3g.opc|(32 hexadecimal digits)|Secret key derived from OP and K (128bit), alternative to using OP which does not disclose OP to subscribers
+|aud3g.sqn|123|Sequence number of last used key (64bit unsigned)
+|aud3g.ind_bitlen|5|Nr of index bits at lower SQN end
+|apn||
+|vlr_number||3GPP TS 23.008 chapter 2.4.5
+|msc_number||3GPP TS 23.008 chapter 2.4.6
+|sgsn_number||3GPP TS 23.008 chapter 2.4.8.1
+|sgsn_address||3GPP TS 23.008 chapter 2.13.10
+|ggsn_number||3GPP TS 23.008 chapter 2.4.8.2
+|gmlc_number||3GPP TS 23.008 chapter 2.4.9.2
+|smsc_number||3GPP TS 23.008 chapter 2.4.23
+|periodic_lu_tmr||3GPP TS 23.008 chapter 2.4.24
+|periodic_rau_tau_tmr||3GPP TS 23.008 chapter 2.13.115
+|nam_cs|1|Enable/disable voice access (3GPP TS 23.008 chapter 2.1.1.2: network access mode)
+|nam_ps|0|Enable/disable data access (3GPP TS 23.008 chapter 2.1.1.2: network access mode)
+|lmsi||3GPP TS 23.008 chapter 2.1.8
+|ms_purged_cs|0|3GPP TS 23.008 chapter 2.7.5
+|ms_purged_ps|1|3GPP TS 23.008 chapter 2.7.6
+|===
+
+=== Configuring the Subscribers Create on Demand Feature
+
+Usually a HLR will only allow mobile equipment (ME) on the network, if the HLR
+has a subscriber entry with the ME's IMSI. But OsmoHLR can also be configured to
+automatically create new entries for new IMSIs, with the
+`subscriber-create-on-demand` VTY option. The obvious use case is creating the
+new subscriber entry and then allowing the ME to use both the CS
+(Circuit Switched) and PS (Packet Switched) NAM (Network Access Mode).
+
+.osmo-hlr.cfg
+----
+hlr
+ subscriber-create-on-demand 5 cs+ps
+----
+
+On the other hand, operators might only want to give network access to IMSIs, of
+which they know the owner. In order to do that, one can set the default NAM to
+`none` and manually approve new subscribers by changing the NAM (e.g. over the
+VTY, see the example below).
+
+Oftentimes it is hard to know, which IMSI belongs to which ME, but the IMEI is
+readily available. If you configure your MSC to send IMEI checking requests to
+the HLR, before sending location update requests, the subscribers created on
+demand can also have the IMEI stored in the HLR database. With OsmoMSC, this
+is done by writing `check-imei-rqd early` in the `msc` section of osmo-msc.cfg.
+Then enable storing the IMEI when receiving check IMEI requests with
+`store-imei` in the OsmoHLR configuration.
+
+.osmo-msc.cfg
+----
+msc
+ check-imei-rqd early
+----
+
+.osmo-hlr.cfg
+----
+hlr
+ subscriber-create-on-demand 5 none
+ store-imei
+----
+
+.Example: Enabling CS and PS NAM via VTY for a known IMEI
+----
+OsmoHLR> enable
+OsmoHLR# subscriber imei 35761300444848 show
+    ID: 1
+    IMSI: 123456789023000
+    MSISDN: 58192 <1>
+    IMEI: 35761300444848
+    CS disabled <2>
+    PS disabled <2>
+OsmoHLR# subscriber imei 35761300444848 update network-access-mode cs+ps
+OsmoHLR# subscriber imei 35761300444848 show
+    ID: 1
+    IMSI: 123456789023000
+    MSISDN: 58192
+    IMEI: 35761300444848
+----
+<1> Randomly generated 5 digit MSISDN
+<2> Disabled CS and PS NAM prevent the subscriber from accessing the network

doc/manuals/chapters/ussd.adoc

@@ -0,0 +1,78 @@
+[[ussd]]
+== Unstructured Supplementary Services Data (USSD)
+
+The _Unstructured Supplementary Services Data (USSD)_ is one service within
+2G/3G networks next to other services such as circuit-switched voice, packet-switched
+data and SMS (Short Message Service).
+
+It is on an abstract level quite similar to SMS in that USSD can be used to send
+textual messages.  However, there are the following differences:
+
+* USSD is between the MS (phone) and an USSD application on the network, while
+  SMS is primarily between two subscribers identified by their MSISDN
+* USSD is faster, as it doesn't suffer from the complicated three-layer CP/RP/TP
+  protocol stack of SMS with it's acknowledgement of the acknowledged acknowledgement.
+* USSD is session-oriented, i.e. a dialogue/session between subscriber and application
+  can persist for the transfer of more than one message.  The dedicated radio channel
+  on the RAN remains established throughout that dialogue.
+
+=== USSD in Osmocom
+
+Until August 2018, OsmoMSC contained some minimalistic internal USSD
+handling with no
+ability to attach/extend it with external USSD applications.
+
+From August 2018 onwards, OsmoMSC doesn't contain any internal USSD
+handlers/applications anymore.  Instead, all USSD is transported to/from
+OsmoHLR via the GSUP protocol.
+
+OsmoHLR contains some intenal USSD handlers and can route USSD messages
+to any number of external USSD entities (EUSEs).  The EUSE also use GSUP
+to communicate USSD from/to OsmoHLR.
+
+Each EUSE is identified by its name.  The name consists of a single-word
+string preceding a currently fixed ("-00-00-00-00-00-00") suffix.
+There is no authentication between EUSE and OsmoHLR: Any client program
+able to connect to the GSUP port of OsmoHLR can register as any EUSE
+(name).
+
+NOTE:: We plan to remove the requirement for this suffix as soon as we
+are done resolving all more important issues.
+
+=== USSD Configuration
+
+USSD configuration in OsmoHLR happens within the `hlr` VTY node.
+
+`euse foobar-00-00-00-00-00-00` defines an EUSE with the given name `foobar`
+
+`ussd route prefix *123 external foobar-00-00-00-00-00-00` installs a
+prefix route to the named EUSE.  All USSD short codes starting with *123 will be
+routed to the named EUSE.
+
+`ussd route prefix *#100# internal own-msisdn` installs a prefix route
+to the named internal USSD handler.  There above command will restore
+the old behavior, in which *#100# will return a text message containing
+the subscribers own phone number.  There is one other handler called
+`own-imsi` which will return the IMSI instead of the MSISDN.
+
+`ussd default-route external foobar-00-00-00-00-00-00` installs a
+default route to the named EUSE.  This means that all USSD codes for
+which no more specific route exists will be routed to the named EUSE.
+
+=== Example EUSE program
+
+We have provided an example EUSE developed in C language using existing
+Osmocom libraries for GSUP protocol handling and USSD encoding/decoding.
+It will register as `foobar` EUSE to OsmoHLR on localhost.  You can run
+it on a different machine by specifying e.g. `osmo-euse-demo 1.2.3.4 5678`
+to make it connect to OsmoHLR on IP address 1.2.3.4 and GSUP/TCP port
+5678.
+
+The idea is that you can use this as a template to develop your own USSD
+applications, or any gateways to other protocols or interfaces.
+
+You can find it in `osmo-hlr/src/osmo-euse-demo.c` or online by
+following the link to http://git.osmocom.org/osmo-hlr/tree/src/osmo-euse-demo.c
+
+This demonstration program will echo back any USSD message sent/routed
+to it, quoted like _You sent "..."_.

doc/manuals/example_subscriber_add_update_delete.vty

@@ -0,0 +1,34 @@
+OsmoHLR> enable
+OsmoHLR# subscriber imsi 123456789023000 create
+% Created subscriber 123456789023000
+    ID: 1
+    IMSI: 123456789023000
+    MSISDN: none
+
+OsmoHLR# subscriber imsi 123456789023000 update msisdn 423
+% Updated subscriber IMSI='123456789023000' to MSISDN='423'
+
+OsmoHLR# subscriber msisdn 423 update aud3g milenage k deaf0ff1ced0d0dabbedd1ced1cef00d opc cededeffacedacefacedbadfadedbeef
+OsmoHLR# subscriber msisdn 423 show
+    ID: 1
+    IMSI: 123456789023000
+    MSISDN: 423
+    3G auth: MILENAGE
+             K=deaf0ff1ced0d0dabbedd1ced1cef00d
+             OPC=cededeffacedacefacedbadfadedbeef
+             IND-bitlen=5
+
+OsmoHLR# subscriber msisdn 423 update aud2g comp128v3 ki beefedcafefaceacedaddeddecadefee
+OsmoHLR# subscriber msisdn 423 show
+    ID: 1
+    IMSI: 123456789023000
+    MSISDN: 423
+    2G auth: COMP128v3
+             KI=beefedcafefaceacedaddeddecadefee
+    3G auth: MILENAGE
+             K=deaf0ff1ced0d0dabbedd1ced1cef00d
+             OPC=cededeffacedacefacedbadfadedbeef
+             IND-bitlen=5
+
+OsmoHLR# subscriber imsi 123456789023000 delete
+% Deleted subscriber for IMSI '123456789023000'

doc/manuals/example_subscriber_cs_ps_enabled.ctrl

@@ -0,0 +1,71 @@
+GET 1 subscriber.by-msisdn-103.info
+GET_REPLY 1 subscriber.by-msisdn-103.info 
+id	3
+imsi	901990000000003
+msisdn	103
+nam_cs	1
+nam_ps	1
+ms_purged_cs	0
+ms_purged_ps	0
+periodic_lu_timer	0
+periodic_rau_tau_timer	0
+lmsi	00000000
+
+GET 2 subscriber.by-msisdn-103.ps-enabled
+GET_REPLY 2 subscriber.by-msisdn-103.ps-enabled 1
+
+SET 3 subscriber.by-msisdn-103.ps-enabled 0
+SET_REPLY 3 subscriber.by-msisdn-103.ps-enabled OK
+
+GET 4 subscriber.by-msisdn-103.ps-enabled
+GET_REPLY 4 subscriber.by-msisdn-103.ps-enabled 0
+
+GET 5 subscriber.by-msisdn-103.info
+GET_REPLY 5 subscriber.by-msisdn-103.info 
+id	3
+imsi	901990000000003
+msisdn	103
+nam_cs	1
+nam_ps	0
+ms_purged_cs	0
+ms_purged_ps	0
+periodic_lu_timer	0
+periodic_rau_tau_timer	0
+lmsi	00000000
+
+SET 6 subscriber.by-msisdn-103.cs-enabled 0
+SET_REPLY 6 subscriber.by-msisdn-103.cs-enabled OK
+
+GET 7 subscriber.by-msisdn-103.cs-enabled
+GET_REPLY 7 subscriber.by-msisdn-103.cs-enabled 0
+
+GET 8 subscriber.by-msisdn-103.info
+GET_REPLY 8 subscriber.by-msisdn-103.info 
+id	3
+imsi	901990000000003
+msisdn	103
+nam_cs	0
+nam_ps	0
+ms_purged_cs	0
+ms_purged_ps	0
+periodic_lu_timer	0
+periodic_rau_tau_timer	0
+lmsi	00000000
+
+SET 9 subscriber.by-msisdn-103.cs-enabled 1
+SET_REPLY 9 subscriber.by-msisdn-103.cs-enabled OK
+SET 10 subscriber.by-msisdn-103.ps-enabled 1
+SET_REPLY 10 subscriber.by-msisdn-103.ps-enabled OK
+
+GET 11 subscriber.by-msisdn-103.info
+GET_REPLY 11 subscriber.by-msisdn-103.info 
+id	3
+imsi	901990000000003
+msisdn	103
+nam_cs	1
+nam_ps	1
+ms_purged_cs	0
+ms_purged_ps	0
+periodic_lu_timer	0
+periodic_rau_tau_timer	0
+lmsi	00000000

doc/manuals/example_subscriber_info.ctrl

@@ -0,0 +1,42 @@
+GET 1 subscriber.by-imsi-901990000000003.info
+GET_REPLY 1 subscriber.by-imsi-901990000000003.info 
+id	3
+imsi	901990000000003
+msisdn	103
+nam_cs	1
+nam_ps	1
+ms_purged_cs	0
+ms_purged_ps	0
+periodic_lu_timer	0
+periodic_rau_tau_timer	0
+lmsi	00000000
+
+GET 2 subscriber.by-msisdn-103.info-aud
+GET_REPLY 2 subscriber.by-msisdn-103.info-aud 
+aud2g.algo	COMP128v1
+aud2g.ki	000102030405060708090a0b0c0d0e0f
+aud3g.algo	MILENAGE
+aud3g.k	000102030405060708090a0b0c0d0e0f
+aud3g.opc	101112131415161718191a1b1c1d1e1f
+aud3g.ind_bitlen	5
+aud3g.sqn	0
+
+GET 3 subscriber.by-id-3.info-all
+GET_REPLY 3 subscriber.by-id-3.info-all 
+id	3
+imsi	901990000000003
+msisdn	103
+nam_cs	1
+nam_ps	1
+ms_purged_cs	0
+ms_purged_ps	0
+periodic_lu_timer	0
+periodic_rau_tau_timer	0
+lmsi	00000000
+aud2g.algo	COMP128v1
+aud2g.ki	000102030405060708090a0b0c0d0e0f
+aud3g.algo	MILENAGE
+aud3g.k	000102030405060708090a0b0c0d0e0f
+aud3g.opc	101112131415161718191a1b1c1d1e1f
+aud3g.ind_bitlen	5
+aud3g.sqn	0

doc/manuals/osmohlr-usermanual-docinfo.xml

@@ -0,0 +1,47 @@
+<revhistory>
+  <revision>
+    <revnumber>1</revnumber>
+    <date>September 18th, 2017</date>
+    <authorinitials>NH</authorinitials>
+    <revremark>
+      Initial version; based on OsmoNITB manual version 2.
+    </revremark>
+  </revision>
+</revhistory>
+
+<authorgroup>
+  <author>
+    <firstname>Neels</firstname>
+    <surname>Hofmeyr</surname>
+    <email>nhofmeyr@sysmocom.de</email>
+    <authorinitials>NH</authorinitials>
+    <affiliation>
+      <shortaffil>sysmocom</shortaffil>
+      <orgname>sysmocom - s.f.m.c. GmbH</orgname>
+      <jobtitle>Senior Developer</jobtitle>
+    </affiliation>
+  </author>
+</authorgroup>
+
+<copyright>
+  <year>2017</year>
+  <holder>sysmocom - s.f.m.c. GmbH</holder>
+</copyright>
+
+<legalnotice>
+  <para>
+	Permission is granted to copy, distribute and/or modify this
+	document under the terms of the GNU Free Documentation License,
+	Version 1.3 or any later version published by the Free Software
+	Foundation; with the Invariant Sections being just 'Foreword',
+	'Acknowledgements' and 'Preface', with no Front-Cover Texts,
+	and no Back-Cover Texts.  A copy of the license is included in
+	the section entitled "GNU Free Documentation License".
+  </para>
+  <para>
+	The Asciidoc source code of this manual can be found at
+	<ulink url="http://git.osmocom.org/osmo-gsm-manuals/">
+		http://git.osmocom.org/osmo-gsm-manuals/
+	</ulink>
+  </para>
+</legalnotice>

doc/manuals/osmohlr-usermanual.adoc

@@ -0,0 +1,36 @@
+:gfdl-enabled:
+:program-name: OsmoHLR
+
+OsmoHLR User Manual
+====================
+Neels Hofmeyr <nhofmeyr@sysmocom.de>
+
+
+include::./common/chapters/preface.adoc[]
+
+include::{srcdir}/chapters/overview.adoc[]
+
+include::{srcdir}/chapters/running.adoc[]
+
+include::{srcdir}/chapters/subscribers.adoc[]
+
+include::{srcdir}/chapters/ussd.adoc[]
+
+include::./common/chapters/vty.adoc[]
+
+include::./common/chapters/logging.adoc[]
+
+include::{srcdir}/chapters/control.adoc[]
+
+include::./common/chapters/control_if.adoc[]
+
+include::./common/chapters/gsup.adoc[]
+
+include::./common/chapters/port_numbers.adoc[]
+
+include::./common/chapters/bibliography.adoc[]
+
+include::./common/chapters/glossary.adoc[]
+
+include::./common/chapters/gfdl.adoc[]
+

doc/manuals/osmohlr-vty-reference.xml

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ex:ts=2:sw=42sts=2:et
+  -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+-->
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML 5.0//EN"
+"http://docbook.org/xml/5.0/dtd/docbook.dtd" [
+<!ENTITY chapter-vty      SYSTEM      "./common/chapters/vty.xml" >
+<!ENTITY sections-vty     SYSTEM      "generated/docbook_vty.xml"  >
+]>
+
+<book>
+  <info>
+    <revhistory>
+        <revision>
+            <revnumber>v1</revnumber>
+            <date>18th September 2017</date>
+            <authorinitials>nh</authorinitials>
+            <revremark>Initial</revremark>
+        </revision>
+    </revhistory>
+
+    <title>OsmoHLR VTY Reference</title>
+
+    <copyright>
+      <year>2017</year>
+    </copyright>
+
+    <legalnotice>
+      <para>This work is copyright by <orgname>sysmocom - s.f.m.c. GmbH</orgname>. All rights reserved.
+      </para>
+    </legalnotice>
+  </info>
+
+  <!-- Main chapters-->
+  &chapter-vty;
+</book>
+

doc/manuals/regen_doc.sh

@@ -0,0 +1,17 @@
+#!/bin/sh -x
+
+if [ -z "$DOCKER_PLAYGROUND" ]; then
+	echo "You need to set DOCKER_PLAYGROUND"
+	exit 1
+fi
+
+SCRIPT=$(realpath "$0")
+MANUAL_DIR=$(dirname "$SCRIPT")
+
+COMMIT=${COMMIT:-$(git log -1 --format=format:%H)}
+
+cd "$DOCKER_PLAYGROUND/scripts" || exit 1
+
+OSMO_HLR_BRANCH=$COMMIT ./regen_doc.sh osmo-hlr 4258 \
+	"$MANUAL_DIR/chapters/counters_generated.adoc" \
+	"$MANUAL_DIR/vty/hlr_vty_reference.xml"

doc/manuals/vty/hlr_vty_additions.xml

@@ -0,0 +1,2 @@
+<vtydoc xmlns='urn:osmocom:xml:libosmocore:vty:doc:1.0'>
+</vtydoc>

include/Makefile.am

@@ -1,2 +1,3 @@
-nobase_include_HEADERS = osmocom/gsupclient/gsup_client.h
+SUBDIRS = osmocom
 
+nobase_include_HEADERS = osmocom/gsupclient/gsup_client.h

include/osmocom/Makefile.am

@@ -0,0 +1,3 @@
+SUBDIRS = \
+	hlr \
+	$(NULL)

include/osmocom/gsupclient/gsup_client.h

@@ -6,23 +6,24 @@
  * Author: Jacob Erlbeck
  *
  * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
  * (at your option) any later version.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * GNU General Public License for more details.
  *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #pragma once
 
 #include <osmocom/core/timer.h>
 #include <osmocom/gsm/oap_client.h>
+#include <osmocom/gsm/ipa.h>
+#include <osmocom/gsm/gsup.h>
 
 /* a loss of GSUP between MSC and HLR is considered quite serious, let's try to recover as quickly as
  * possible.  Even one new connection attempt per second should be quite acceptable until the link is
@@ -38,7 +39,7 @@ struct osmo_gsup_client;
 typedef int (*osmo_gsup_client_read_cb_t)(struct osmo_gsup_client *gsupc, struct msgb *msg);
 
 struct osmo_gsup_client {
-	const char *unit_name;
+	const char *unit_name; /* same as ipa_dev->unit_name, for backwards compat */
 
 	struct ipa_client_conn *link;
 	osmo_gsup_client_read_cb_t read_cb;
@@ -50,8 +51,16 @@ struct osmo_gsup_client {
 	struct osmo_timer_list connect_timer;
 	int is_connected;
 	int got_ipa_pong;
+
+	struct ipaccess_unit *ipa_dev; /* identification information sent to IPA server */
 };
 
+struct osmo_gsup_client *osmo_gsup_client_create2(void *talloc_ctx,
+						  struct ipaccess_unit *ipa_dev,
+						  const char *ip_addr,
+						  unsigned int tcp_port,
+						  osmo_gsup_client_read_cb_t read_cb,
+						  struct osmo_oap_client_config *oapc_config);
 struct osmo_gsup_client *osmo_gsup_client_create(void *talloc_ctx,
 						 const char *unit_name,
 						 const char *ip_addr,
@@ -61,5 +70,7 @@ struct osmo_gsup_client *osmo_gsup_client_create(void *talloc_ctx,
 
 void osmo_gsup_client_destroy(struct osmo_gsup_client *gsupc);
 int osmo_gsup_client_send(struct osmo_gsup_client *gsupc, struct msgb *msg);
+int osmo_gsup_client_enc_send(struct osmo_gsup_client *gsupc,
+			      const struct osmo_gsup_message *gsup_msg);
 struct msgb *osmo_gsup_client_msgb_alloc(void);
 

include/osmocom/hlr/Makefile.am

@@ -0,0 +1,14 @@
+noinst_HEADERS = \
+	auc.h \
+	ctrl.h \
+	db.h \
+	gsup_router.h \
+	gsup_server.h \
+	hlr.h \
+	hlr_ussd.h \
+	hlr_vty.h \
+	hlr_vty_subscr.h \
+	logging.h \
+	luop.h \
+	rand.h \
+	$(NULL)

include/osmocom/hlr/db.h

@@ -9,8 +9,10 @@ enum stmt_idx {
 	DB_STMT_SEL_BY_IMSI,
 	DB_STMT_SEL_BY_MSISDN,
 	DB_STMT_SEL_BY_ID,
+	DB_STMT_SEL_BY_IMEI,
 	DB_STMT_UPD_VLR_BY_ID,
 	DB_STMT_UPD_SGSN_BY_ID,
+	DB_STMT_UPD_IMEI_BY_IMSI,
 	DB_STMT_AUC_BY_IMSI,
 	DB_STMT_AUC_UPD_SQN,
 	DB_STMT_UPD_PURGE_CS_BY_IMSI,
@@ -20,10 +22,15 @@ enum stmt_idx {
 	DB_STMT_SUBSCR_CREATE,
 	DB_STMT_DEL_BY_ID,
 	DB_STMT_SET_MSISDN_BY_IMSI,
+	DB_STMT_DELETE_MSISDN_BY_IMSI,
 	DB_STMT_AUC_2G_INSERT,
 	DB_STMT_AUC_2G_DELETE,
 	DB_STMT_AUC_3G_INSERT,
 	DB_STMT_AUC_3G_DELETE,
+	DB_STMT_SET_LAST_LU_SEEN,
+	DB_STMT_SET_LAST_LU_SEEN_PS,
+	DB_STMT_EXISTS_BY_IMSI,
+	DB_STMT_EXISTS_BY_MSISDN,
 	_NUM_DB_STMT
 };
 
@@ -33,12 +40,17 @@ struct db_context {
 	sqlite3_stmt *stmt[_NUM_DB_STMT];
 };
 
+/* Optional feature to make SQLite3 using talloc */
+#ifdef SQLITE_USE_TALLOC
+int db_sqlite3_use_talloc(void *ctx);
+#endif
+
 void db_remove_reset(sqlite3_stmt *stmt);
 bool db_bind_text(sqlite3_stmt *stmt, const char *param_name, const char *text);
 bool db_bind_int(sqlite3_stmt *stmt, const char *param_name, int nr);
 bool db_bind_int64(sqlite3_stmt *stmt, const char *param_name, int64_t nr);
 void db_close(struct db_context *dbc);
-struct db_context *db_open(void *ctx, const char *fname, bool enable_sqlite3_logging);
+struct db_context *db_open(void *ctx, const char *fname, bool enable_sqlite3_logging, bool allow_upgrades);
 
 #include <osmocom/crypt/auth.h>
 
@@ -54,7 +66,7 @@ int db_update_sqn(struct db_context *dbc, int64_t id,
 int db_get_auc(struct db_context *dbc, const char *imsi,
 	       unsigned int auc_3g_ind, struct osmo_auth_vector *vec,
 	       unsigned int num_vec, const uint8_t *rand_auts,
-	       const uint8_t *auts);
+	       const uint8_t *auts, bool separation_bit);
 
 #include <osmocom/core/linuxlist.h>
 #include <osmocom/gsm/protocol/gsm_23_003.h>
@@ -67,10 +79,11 @@ struct hlr_subscriber {
 
 	int64_t		id;
 	char		imsi[GSM23003_IMSI_MAX_DIGITS+1];
-	char		msisdn[GT_MAX_DIGITS+1];
+	char		msisdn[GSM23003_MSISDN_MAX_DIGITS+1];
 	/* imeisv? */
-	char		vlr_number[GT_MAX_DIGITS+1];
-	char		sgsn_number[GT_MAX_DIGITS+1];
+	char		imei[GSM23003_IMEI_NUM_DIGITS+1];
+	char		vlr_number[32];
+	char		sgsn_number[32];
 	char		sgsn_address[GT_MAX_DIGITS+1];
 	/* ggsn number + address */
 	/* gmlc number */
@@ -82,8 +95,15 @@ struct hlr_subscriber {
 	uint32_t	lmsi;
 	bool		ms_purged_cs;
 	bool		ms_purged_ps;
+	time_t		last_lu_seen;
+	time_t		last_lu_seen_ps;
 };
 
+/* A format string for use with strptime(3). This format string is
+ * used to parse the last_lu_seen column stored in the HLR database.
+ * See https://sqlite.org/lang_datefunc.html, function datetime(). */
+#define DB_LAST_LU_SEEN_FMT "%Y-%m-%d %H:%M:%S"
+
 /* Like struct osmo_sub_auth_data, but the keys are in hexdump representation.
  * This is useful because SQLite requires them in hexdump format, and callers
  * like the VTY and CTRL interface also have them available as hexdump to begin
@@ -107,13 +127,20 @@ struct sub_auth_data_str {
 	} u;
 };
 
-int db_subscr_create(struct db_context *dbc, const char *imsi);
+#define DB_SUBSCR_FLAG_NAM_CS	(1 << 1)
+#define DB_SUBSCR_FLAG_NAM_PS	(1 << 2)
+
+int db_subscr_create(struct db_context *dbc, const char *imsi, uint8_t flags);
 int db_subscr_delete_by_id(struct db_context *dbc, int64_t subscr_id);
 
 int db_subscr_update_msisdn_by_imsi(struct db_context *dbc, const char *imsi,
 				    const char *msisdn);
 int db_subscr_update_aud_by_id(struct db_context *dbc, int64_t subscr_id,
 			       const struct sub_auth_data_str *aud);
+int db_subscr_update_imei_by_imsi(struct db_context *dbc, const char* imsi, const char *imei);
+
+int db_subscr_exists_by_imsi(struct db_context *dbc, const char *imsi);
+int db_subscr_exists_by_msisdn(struct db_context *dbc, const char *msisdn);
 
 int db_subscr_get_by_imsi(struct db_context *dbc, const char *imsi,
 			  struct hlr_subscriber *subscr);
@@ -121,6 +148,7 @@ int db_subscr_get_by_msisdn(struct db_context *dbc, const char *msisdn,
 			    struct hlr_subscriber *subscr);
 int db_subscr_get_by_id(struct db_context *dbc, int64_t id,
 			struct hlr_subscriber *subscr);
+int db_subscr_get_by_imei(struct db_context *dbc, const char *imei, struct hlr_subscriber *subscr);
 int db_subscr_nam(struct db_context *dbc, const char *imsi, bool nam_val, bool is_ps);
 int db_subscr_lu(struct db_context *dbc, int64_t subscr_id,
 		 const char *vlr_or_sgsn_number, bool is_ps);

include/osmocom/hlr/gsup_router.h

@@ -1,11 +1,20 @@
 #pragma once
 
 #include <stdint.h>
-#include "gsup_server.h"
+#include <osmocom/hlr/gsup_server.h>
+
+struct gsup_route {
+	struct llist_head list;
+
+	uint8_t *addr;
+	struct osmo_gsup_conn *conn;
+};
 
 struct osmo_gsup_conn *gsup_route_find(struct osmo_gsup_server *gs,
 					const uint8_t *addr, size_t addrlen);
 
+struct gsup_route *gsup_route_find_by_conn(const struct osmo_gsup_conn *conn);
+
 /* add a new route for the given address to the given conn */
 int gsup_route_add(struct osmo_gsup_conn *conn, const uint8_t *addr, size_t addrlen);
 

include/osmocom/hlr/gsup_server.h

@@ -47,6 +47,7 @@ struct osmo_gsup_conn {
 	bool supports_ps; /* client supports OSMO_GSUP_CN_DOMAIN_PS */
 };
 
+struct msgb *osmo_gsup_msgb_alloc(const char *label);
 
 int osmo_gsup_conn_send(struct osmo_gsup_conn *conn, struct msgb *msg);
 int osmo_gsup_conn_ccm_get(const struct osmo_gsup_conn *clnt, uint8_t **addr,

include/osmocom/hlr/hlr.h

@@ -25,6 +25,8 @@
 #include <stdbool.h>
 #include <osmocom/core/linuxlist.h>
 
+#define HLR_DEFAULT_DB_FILE_PATH "hlr.db"
+
 struct hlr_euse;
 
 struct hlr {
@@ -32,6 +34,7 @@ struct hlr {
 	struct osmo_gsup_server *gs;
 
 	/* DB context */
+	char *db_file_path;
 	struct db_context *dbc;
 
 	/* Control Interface */
@@ -45,9 +48,19 @@ struct hlr {
 	struct hlr_euse *euse_default;
 	struct llist_head iuse_list;
 
+	/* NCSS (call independent) session guard timeout value */
+	int ncss_guard_timeout;
+
 	struct llist_head ussd_routes;
 
 	struct llist_head ss_sessions;
+
+	bool store_imei;
+
+	bool subscr_create_on_demand;
+	/* Bitmask of DB_SUBSCR_FLAG_* */
+	uint8_t subscr_create_on_demand_flags;
+	unsigned int subscr_create_on_demand_rand_msisdn_len;
 };
 
 extern struct hlr *g_hlr;

include/osmocom/hlr/hlr_ussd.h

@@ -4,9 +4,10 @@
 
 #include <osmocom/core/linuxlist.h>
 #include <osmocom/gsm/gsup.h>
-#include "gsup_server.h"
 
-struct osmo_gsup_conn;
+#include <osmocom/hlr/gsup_server.h>
+
+#define NCSS_GUARD_TIMEOUT_DEFAULT 30
 
 struct hlr_ussd_route {
 	/* g_hlr.routes */

include/osmocom/hlr/hlr_vty.h

@@ -25,7 +25,7 @@
 #include <osmocom/core/logging.h>
 #include <osmocom/vty/vty.h>
 #include <osmocom/vty/command.h>
-#include "hlr.h"
+#include <osmocom/hlr/hlr.h>
 
 enum hlr_vty_node {
 	HLR_NODE = _LAST_OSMOVTY_NODE + 1,
@@ -35,4 +35,4 @@ enum hlr_vty_node {
 
 int hlr_vty_is_config_node(struct vty *vty, int node);
 int hlr_vty_go_parent(struct vty *vty);
-void hlr_vty_init(const struct log_info *cat);
+void hlr_vty_init(void);

include/osmocom/hlr/luop.h

@@ -27,8 +27,8 @@
 #include <osmocom/core/timer.h>
 #include <osmocom/gsm/gsup.h>
 
-#include "db.h"
-#include "gsup_server.h"
+#include <osmocom/hlr/db.h>
+#include <osmocom/hlr/gsup_server.h>
 
 #define CANCEL_TIMEOUT_SECS	30
 #define ISD_TIMEOUT_SECS	30

libosmo-gsup-client.pc.in

@@ -3,9 +3,9 @@ exec_prefix=@exec_prefix@
 libdir=@libdir@
 includedir=@includedir@
 
-Name: Osmocom GSUP and OAP Client Library
+Name: Osmocom GSUP Client Library
 Description: C Utility Library
 Version: @VERSION@
-Libs: -L${libdir} @TALLOC_LIBS@ -losmogsm -losmocore
+Libs: -L${libdir} -losmo-gsup-client
 Cflags: -I${includedir}/
 

sql/hlr.sql

@@ -1,16 +1,18 @@
-CREATE TABLE IF NOT EXISTS subscriber (
+CREATE TABLE subscriber (
 -- OsmoHLR's DB scheme is modelled roughly after TS 23.008 version 13.3.0
 	id		INTEGER PRIMARY KEY,
 	-- Chapter 2.1.1.1
 	imsi		VARCHAR(15) UNIQUE NOT NULL,
 	-- Chapter 2.1.2
 	msisdn		VARCHAR(15) UNIQUE,
-	-- Chapter 2.2.3: Most recent / current IMEI
+	-- Chapter 2.2.3: Most recent / current IMEISV
 	imeisv		VARCHAR,
+	-- Chapter 2.1.9: Most recent / current IMEI
+	imei		VARCHAR(14),
 	-- Chapter 2.4.5
 	vlr_number	VARCHAR(15),
 	-- Chapter 2.4.6
-	hlr_number	VARCHAR(15),
+	msc_number	VARCHAR(15),
 	-- Chapter 2.4.8.1
 	sgsn_number	VARCHAR(15),
 	-- Chapter 2.13.10
@@ -36,34 +38,44 @@ CREATE TABLE IF NOT EXISTS subscriber (
 	-- Chapter 2.7.5
 	ms_purged_cs	BOOLEAN NOT NULL DEFAULT 0,
 	-- Chapter 2.7.6
-	ms_purged_ps	BOOLEAN NOT NULL DEFAULT 0
+	ms_purged_ps	BOOLEAN NOT NULL DEFAULT 0,
+
+	-- Timestamp of last location update seen from subscriber
+	-- The value is a string which encodes a UTC timestamp in granularity of seconds.
+	last_lu_seen TIMESTAMP default NULL,
+	last_lu_seen_ps TIMESTAMP default NULL
 );
 
-CREATE TABLE IF NOT EXISTS subscriber_apn (
+CREATE TABLE subscriber_apn (
 	subscriber_id	INTEGER,		-- subscriber.id
 	apn		VARCHAR(256) NOT NULL
 );
 
-CREATE TABLE IF NOT EXISTS subscriber_multi_msisdn (
+CREATE TABLE subscriber_multi_msisdn (
 -- Chapter 2.1.3
 	subscriber_id	INTEGER,		-- subscriber.id
 	msisdn		VARCHAR(15) NOT NULL
 );
 
-CREATE TABLE IF NOT EXISTS auc_2g (
+CREATE TABLE auc_2g (
 	subscriber_id	INTEGER PRIMARY KEY,	-- subscriber.id
 	algo_id_2g	INTEGER NOT NULL,	-- enum osmo_auth_algo value
 	ki		VARCHAR(32) NOT NULL	-- hex string: subscriber's secret key (128bit)
 );
 
-CREATE TABLE IF NOT EXISTS auc_3g (
+CREATE TABLE auc_3g (
 	subscriber_id	INTEGER PRIMARY KEY,	-- subscriber.id
 	algo_id_3g	INTEGER NOT NULL,	-- enum osmo_auth_algo value
 	k		VARCHAR(32) NOT NULL,	-- hex string: subscriber's secret key (128bit)
 	op		VARCHAR(32),		-- hex string: operator's secret key (128bit)
 	opc		VARCHAR(32),		-- hex string: derived from OP and K (128bit)
 	sqn		INTEGER NOT NULL DEFAULT 0,	-- sequence number of key usage
-	ind_bitlen	INTEGER NOT NULL DEFAULT 5	-- nr of index bits at lower SQN end
+	-- nr of index bits at lower SQN end
+	ind_bitlen	INTEGER NOT NULL DEFAULT 5
 );
 
-CREATE UNIQUE INDEX IF NOT EXISTS idx_subscr_imsi ON subscriber (imsi);
+CREATE UNIQUE INDEX idx_subscr_imsi ON subscriber (imsi);
+
+-- Set HLR database schema version number
+-- Note: This constant is currently duplicated in src/db.c and must be kept in sync!
+PRAGMA user_version = 4;

src/Makefile.am

@@ -11,11 +11,12 @@ AM_CFLAGS = \
 	$(NULL)
 
 AM_CPPFLAGS = -I$(top_srcdir)/include \
+	-I$(top_builddir)/include \
 	$(NULL)
 
 EXTRA_DIST = \
 	populate_hlr_db.pl \
-	db_bootstrap.sed \
+	db_sql2c.sed \
 	$(NULL)
 
 BUILT_SOURCES = \
@@ -24,18 +25,6 @@ BUILT_SOURCES = \
 CLEANFILES = $(BUILT_SOURCES)
 
 noinst_HEADERS = \
-	auc.h \
-	db.h \
-	hlr.h \
-	luop.h \
-	gsup_router.h \
-	gsup_server.h \
-	logging.h \
-	rand.h \
-	ctrl.h \
-	hlr_vty.h \
-	hlr_vty_subscr.h \
-	hlr_ussd.h \
 	db_bootstrap.h \
 	$(NULL)
 
@@ -87,21 +76,6 @@ osmo_hlr_db_tool_LDADD = \
 	$(SQLITE3_LIBS) \
 	$(NULL)
 
-db_test_SOURCES = \
-	auc.c \
-	db.c \
-	db_auc.c \
-	db_test.c \
-	logging.c \
-	rand_fake.c \
-	$(NULL)
-
-db_test_LDADD = \
-	$(LIBOSMOCORE_LIBS) \
-	$(LIBOSMOGSM_LIBS) \
-	$(SQLITE3_LIBS) \
-	$(NULL)
-
 osmo_euse_demo_SOURCES = \
 	osmo-euse-demo.c \
 	$(NULL)
@@ -112,13 +86,18 @@ osmo_euse_demo_LDADD = \
 	$(LIBOSMOGSM_LIBS) \
 	$(NULL)
 
+if DB_SQLITE_DEBUG
+osmo_hlr_SOURCES += db_debug.c
+osmo_hlr_db_tool_SOURCES += db_debug.c
+endif
+
 BOOTSTRAP_SQL = $(top_srcdir)/sql/hlr.sql
 
-db_bootstrap.h: $(BOOTSTRAP_SQL) $(srcdir)/db_bootstrap.sed
-	echo "/* DO NOT EDIT THIS FILE. It is generated from osmo-hlr.git/sql/hlr.sql */" > "$@"
+db_bootstrap.h: $(BOOTSTRAP_SQL) $(srcdir)/db_sql2c.sed
+	echo "/* DO NOT EDIT THIS FILE. It is generated from files in osmo-hlr.git/sql/ */" > "$@"
 	echo "#pragma once" >> "$@"
 	echo "static const char *stmt_bootstrap_sql[] = {" >> "$@"
 	cat "$(BOOTSTRAP_SQL)" \
-		| sed -f "$(srcdir)/db_bootstrap.sed" \
+		| sed -f "$(srcdir)/db_sql2c.sed" \
 		>> "$@"
 	echo "};" >> "$@"

src/auc.c

@@ -23,8 +23,8 @@
 #include <osmocom/core/utils.h>
 #include <osmocom/crypt/auth.h>
 
-#include "logging.h"
-#include "rand.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/rand.h>
 
 #define hexb(buf) osmo_hexdump_nospc((void*)buf, sizeof(buf))
 #define hex(buf,sz) osmo_hexdump_nospc((void*)buf, sz)

src/ctrl.c

@@ -28,9 +28,9 @@
 #include <osmocom/gsm/gsm23003.h>
 #include <osmocom/ctrl/ports.h>
 
-#include "hlr.h"
-#include "ctrl.h"
-#include "db.h"
+#include <osmocom/hlr/hlr.h>
+#include <osmocom/hlr/ctrl.h>
+#include <osmocom/hlr/db.h>
 
 #define SEL_BY "by-"
 #define SEL_BY_IMSI SEL_BY "imsi-"
@@ -95,7 +95,7 @@ static bool get_subscriber(struct db_context *dbc,
 		cmd->reply = "No such subscriber.";
 		return false;
 	default:
-		cmd->reply = "An unknown error has occured during get_subscriber().";
+		cmd->reply = "An unknown error has occurred during get_subscriber().";
 		return false;
 	}
 }
@@ -132,7 +132,7 @@ static void print_subscr_info(struct ctrl_cmd *cmd,
 			      struct hlr_subscriber *subscr)
 {
 	ctrl_cmd_reply_printf(cmd,
-		"\nid\t%"PRIu64
+		"\nid\t%" PRIu64
 		FMT_S
 		FMT_S
 		FMT_BOOL
@@ -189,7 +189,7 @@ static void print_subscr_info_aud3g(struct ctrl_cmd *cmd, struct osmo_sub_auth_d
 	ctrl_cmd_reply_printf(cmd,
 		"\naud3g.%s\t%s"
 		"\naud3g.ind_bitlen\t%u"
-		"\naud3g.sqn\t%"PRIu64
+		"\naud3g.sqn\t%" PRIu64
 		,
 		aud->u.umts.opc_is_op? "op" : "opc",
 		hexdump_buf(aud->u.umts.opc),

src/db.c

@@ -23,14 +23,18 @@
 #include <sqlite3.h>
 #include <string.h>
 
-#include "logging.h"
-#include "db.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/db.h>
 #include "db_bootstrap.h"
 
+/* This constant is currently duplicated in sql/hlr.sql and must be kept in sync! */
+#define CURRENT_SCHEMA_VERSION	4
+
 #define SEL_COLUMNS \
 	"id," \
 	"imsi," \
 	"msisdn," \
+	"imei," \
 	"vlr_number," \
 	"sgsn_number," \
 	"sgsn_address," \
@@ -40,14 +44,18 @@
 	"nam_ps," \
 	"lmsi," \
 	"ms_purged_cs," \
-	"ms_purged_ps"
+	"ms_purged_ps," \
+	"last_lu_seen," \
+	"last_lu_seen_ps" \
 
 static const char *stmt_sql[] = {
 	[DB_STMT_SEL_BY_IMSI] = "SELECT " SEL_COLUMNS " FROM subscriber WHERE imsi = ?",
 	[DB_STMT_SEL_BY_MSISDN] = "SELECT " SEL_COLUMNS " FROM subscriber WHERE msisdn = ?",
 	[DB_STMT_SEL_BY_ID] = "SELECT " SEL_COLUMNS " FROM subscriber WHERE id = ?",
+	[DB_STMT_SEL_BY_IMEI] = "SELECT " SEL_COLUMNS " FROM subscriber WHERE imei = ?",
 	[DB_STMT_UPD_VLR_BY_ID] = "UPDATE subscriber SET vlr_number = $number WHERE id = $subscriber_id",
 	[DB_STMT_UPD_SGSN_BY_ID] = "UPDATE subscriber SET sgsn_number = $number WHERE id = $subscriber_id",
+	[DB_STMT_UPD_IMEI_BY_IMSI] = "UPDATE subscriber SET imei = $imei WHERE imsi = $imsi",
 	[DB_STMT_AUC_BY_IMSI] =
 		"SELECT id, algo_id_2g, ki, algo_id_3g, k, op, opc, sqn, ind_bitlen"
 		" FROM subscriber"
@@ -59,9 +67,10 @@ static const char *stmt_sql[] = {
 	[DB_STMT_UPD_PURGE_PS_BY_IMSI] = "UPDATE subscriber SET ms_purged_ps = $val WHERE imsi = $imsi",
 	[DB_STMT_UPD_NAM_CS_BY_IMSI] = "UPDATE subscriber SET nam_cs = $val WHERE imsi = $imsi",
 	[DB_STMT_UPD_NAM_PS_BY_IMSI] = "UPDATE subscriber SET nam_ps = $val WHERE imsi = $imsi",
-	[DB_STMT_SUBSCR_CREATE] = "INSERT INTO subscriber (imsi) VALUES ($imsi)",
+	[DB_STMT_SUBSCR_CREATE] = "INSERT INTO subscriber (imsi, nam_cs, nam_ps) VALUES ($imsi, $nam_cs, $nam_ps)",
 	[DB_STMT_DEL_BY_ID] = "DELETE FROM subscriber WHERE id = $subscriber_id",
 	[DB_STMT_SET_MSISDN_BY_IMSI] = "UPDATE subscriber SET msisdn = $msisdn WHERE imsi = $imsi",
+	[DB_STMT_DELETE_MSISDN_BY_IMSI] = "UPDATE subscriber SET msisdn = NULL WHERE imsi = $imsi",
 	[DB_STMT_AUC_2G_INSERT] =
 		"INSERT INTO auc_2g (subscriber_id, algo_id_2g, ki)"
 		" VALUES($subscriber_id, $algo_id_2g, $ki)",
@@ -70,6 +79,10 @@ static const char *stmt_sql[] = {
 		"INSERT INTO auc_3g (subscriber_id, algo_id_3g, k, op, opc, ind_bitlen)"
 		" VALUES($subscriber_id, $algo_id_3g, $k, $op, $opc, $ind_bitlen)",
 	[DB_STMT_AUC_3G_DELETE] = "DELETE FROM auc_3g WHERE subscriber_id = $subscriber_id",
+	[DB_STMT_SET_LAST_LU_SEEN] = "UPDATE subscriber SET last_lu_seen = datetime($val, 'unixepoch') WHERE id = $subscriber_id",
+	[DB_STMT_SET_LAST_LU_SEEN_PS] = "UPDATE subscriber SET last_lu_seen_ps = datetime($val, 'unixepoch') WHERE id = $subscriber_id",
+	[DB_STMT_EXISTS_BY_IMSI] = "SELECT 1 FROM subscriber WHERE imsi = $imsi",
+	[DB_STMT_EXISTS_BY_MSISDN] = "SELECT 1 FROM subscriber WHERE msisdn = $msisdn",
 };
 
 static void sql3_error_log_cb(void *arg, int err_code, const char *msg)
@@ -190,47 +203,299 @@ void db_close(struct db_context *dbc)
 	talloc_free(dbc);
 }
 
-static int db_bootstrap(struct db_context *dbc)
+static int db_run_statements(struct db_context *dbc, const char **statements, size_t statements_count)
 {
+	int rc;
 	int i;
-	for (i = 0; i < ARRAY_SIZE(stmt_bootstrap_sql); i++) {
-		int rc;
+	for (i = 0; i < statements_count; i++) {
+		const char *stmt_str = statements[i];
 		sqlite3_stmt *stmt;
 
-		rc = sqlite3_prepare_v2(dbc->db, stmt_bootstrap_sql[i], -1,
-					&stmt, NULL);
+		rc = sqlite3_prepare_v2(dbc->db, stmt_str, -1, &stmt, NULL);
 		if (rc != SQLITE_OK) {
-			LOGP(DDB, LOGL_ERROR, "Unable to prepare SQL statement '%s'\n",
-			     stmt_bootstrap_sql[i]);
+			LOGP(DDB, LOGL_ERROR, "Unable to prepare SQL statement '%s'\n", stmt_str);
 			return rc;
 		}
-
-		/* execute the statement */
 		rc = sqlite3_step(stmt);
 		db_remove_reset(stmt);
 		sqlite3_finalize(stmt);
 		if (rc != SQLITE_DONE) {
-			LOGP(DDB, LOGL_ERROR, "Cannot bootstrap database: SQL error: (%d) %s,"
-			     " during stmt '%s'",
-			     rc, sqlite3_errmsg(dbc->db),
-			     stmt_bootstrap_sql[i]);
+			LOGP(DDB, LOGL_ERROR, "SQL error: (%d) %s, during stmt '%s'",
+			     rc, sqlite3_errmsg(dbc->db), stmt_str);
 			return rc;
 		}
 	}
+	return rc;
+}
+
+static int db_bootstrap(struct db_context *dbc)
+{
+	int rc = db_run_statements(dbc, stmt_bootstrap_sql, ARRAY_SIZE(stmt_bootstrap_sql));
+	if (rc != SQLITE_DONE) {
+		LOGP(DDB, LOGL_ERROR, "Cannot bootstrap database\n");
+		return rc;
+	}
 	return SQLITE_OK;
 }
 
-struct db_context *db_open(void *ctx, const char *fname, bool enable_sqlite_logging)
+/* https://www.sqlite.org/fileformat2.html#storage_of_the_sql_database_schema */
+static bool db_table_exists(struct db_context *dbc, const char *table_name)
+{
+	const char *table_exists_sql = "SELECT name FROM sqlite_master WHERE type='table' AND name=?";
+	sqlite3_stmt *stmt;
+	int rc;
+
+	rc = sqlite3_prepare_v2(dbc->db, table_exists_sql, -1, &stmt, NULL);
+	if (rc != SQLITE_OK) {
+		LOGP(DDB, LOGL_ERROR, "Unable to prepare SQL statement '%s'\n", table_exists_sql);
+		return false;
+	}
+
+	if (!db_bind_text(stmt, NULL, table_name))
+		return false;
+
+	rc = sqlite3_step(stmt);
+	db_remove_reset(stmt);
+	sqlite3_finalize(stmt);
+	return (rc == SQLITE_ROW);
+}
+
+/* Indicate whether the database is initialized with tables for schema version 0.
+ * We only check for the 'subscriber' table here because Neels said so. */
+static bool db_is_bootstrapped_v0(struct db_context *dbc)
+{
+	if (!db_table_exists(dbc, "subscriber")) {
+		LOGP(DDB, LOGL_DEBUG, "Table 'subscriber' not found in database '%s'\n", dbc->fname);
+		return false;
+	}
+
+	return true;
+}
+
+static int
+db_upgrade_v1(struct db_context *dbc)
+{
+	int rc;
+	const char *statements[] = {
+		"ALTER TABLE subscriber ADD COLUMN last_lu_seen TIMESTAMP default NULL",
+		"PRAGMA user_version = 1",
+	};
+
+	rc = db_run_statements(dbc, statements, ARRAY_SIZE(statements));
+	if (rc != SQLITE_DONE) {
+		LOGP(DDB, LOGL_ERROR, "Unable to update HLR database schema to version 1\n");
+		return rc;
+	}
+	return rc;
+}
+
+static int db_upgrade_v2(struct db_context *dbc)
+{
+	int rc;
+	const char *statements[] = {
+		"ALTER TABLE subscriber ADD COLUMN imei VARCHAR(14)",
+		"PRAGMA user_version = 2",
+	};
+
+	rc = db_run_statements(dbc, statements, ARRAY_SIZE(statements));
+	if (rc != SQLITE_DONE) {
+		LOGP(DDB, LOGL_ERROR, "Unable to update HLR database schema to version 2\n");
+		return rc;
+	}
+	return rc;
+}
+
+static int db_upgrade_v3(struct db_context *dbc)
+{
+	int rc;
+
+	/* A newer SQLite version would allow simply 'ATLER TABLE subscriber RENAME COLUMN hlr_number TO msc_number'.
+	 * This is a really expensive workaround for that in order to cover earlier SQLite versions as well:
+	 * Create a new table with the new column name and copy the data over (https://www.sqlite.org/faq.html#q11).
+	 */
+#define SUBSCR_V3_CREATE  \
+"(\n" \
+"-- OsmoHLR's DB scheme is modelled roughly after TS 23.008 version 13.3.0\n" \
+"	id		INTEGER PRIMARY KEY,\n" \
+"	-- Chapter 2.1.1.1\n" \
+"	imsi		VARCHAR(15) UNIQUE NOT NULL,\n" \
+"	-- Chapter 2.1.2\n" \
+"	msisdn		VARCHAR(15) UNIQUE,\n" \
+"	-- Chapter 2.2.3: Most recent / current IMEISV\n" \
+"	imeisv		VARCHAR,\n" \
+"	-- Chapter 2.1.9: Most recent / current IMEI\n" \
+"	imei		VARCHAR(14),\n" \
+"	-- Chapter 2.4.5\n" \
+"	vlr_number	VARCHAR(15),\n" \
+"	-- Chapter 2.4.6\n" \
+"	msc_number	VARCHAR(15),\n" \
+"	-- Chapter 2.4.8.1\n" \
+"	sgsn_number	VARCHAR(15),\n" \
+"	-- Chapter 2.13.10\n" \
+"	sgsn_address	VARCHAR,\n" \
+"	-- Chapter 2.4.8.2\n" \
+"	ggsn_number	VARCHAR(15),\n" \
+"	-- Chapter 2.4.9.2\n" \
+"	gmlc_number	VARCHAR(15),\n" \
+"	-- Chapter 2.4.23\n" \
+"	smsc_number	VARCHAR(15),\n" \
+"	-- Chapter 2.4.24\n" \
+"	periodic_lu_tmr	INTEGER,\n" \
+"	-- Chapter 2.13.115\n" \
+"	periodic_rau_tau_tmr INTEGER,\n" \
+"	-- Chapter 2.1.1.2: network access mode\n" \
+"	nam_cs		BOOLEAN NOT NULL DEFAULT 1,\n" \
+"	nam_ps		BOOLEAN NOT NULL DEFAULT 1,\n" \
+"	-- Chapter 2.1.8\n" \
+"	lmsi		INTEGER,\n" \
+ \
+"	-- The below purged flags might not even be stored non-volatile,\n" \
+"	-- refer to TS 23.012 Chapter 3.6.1.4\n" \
+"	-- Chapter 2.7.5\n" \
+"	ms_purged_cs	BOOLEAN NOT NULL DEFAULT 0,\n" \
+"	-- Chapter 2.7.6\n" \
+"	ms_purged_ps	BOOLEAN NOT NULL DEFAULT 0,\n" \
+ \
+"	-- Timestamp of last location update seen from subscriber\n" \
+"	-- The value is a string which encodes a UTC timestamp in granularity of seconds.\n" \
+"	last_lu_seen TIMESTAMP default NULL\n" \
+")\n"
+
+#define SUBSCR_V2_COLUMN_NAMES \
+	"id," \
+	"imsi," \
+	"msisdn," \
+	"imeisv," \
+	"imei," \
+	"vlr_number," \
+	"hlr_number," \
+	"sgsn_number," \
+	"sgsn_address," \
+	"ggsn_number," \
+	"gmlc_number," \
+	"smsc_number," \
+	"periodic_lu_tmr," \
+	"periodic_rau_tau_tmr," \
+	"nam_cs," \
+	"nam_ps," \
+	"lmsi," \
+	"ms_purged_cs," \
+	"ms_purged_ps," \
+	"last_lu_seen"
+
+#define SUBSCR_V3_COLUMN_NAMES \
+	"id," \
+	"imsi," \
+	"msisdn," \
+	"imeisv," \
+	"imei," \
+	"vlr_number," \
+	"msc_number," \
+	"sgsn_number," \
+	"sgsn_address," \
+	"ggsn_number," \
+	"gmlc_number," \
+	"smsc_number," \
+	"periodic_lu_tmr," \
+	"periodic_rau_tau_tmr," \
+	"nam_cs," \
+	"nam_ps," \
+	"lmsi," \
+	"ms_purged_cs," \
+	"ms_purged_ps," \
+	"last_lu_seen"
+
+	const char *statements[] = {
+		"BEGIN TRANSACTION",
+		"CREATE TEMPORARY TABLE subscriber_backup" SUBSCR_V3_CREATE,
+		"INSERT INTO subscriber_backup SELECT " SUBSCR_V2_COLUMN_NAMES " FROM subscriber",
+		"DROP TABLE subscriber",
+		"CREATE TABLE subscriber" SUBSCR_V3_CREATE,
+		"INSERT INTO subscriber SELECT " SUBSCR_V3_COLUMN_NAMES " FROM subscriber_backup",
+		"DROP TABLE subscriber_backup",
+		"COMMIT",
+		"PRAGMA user_version = 3",
+	};
+
+	rc = db_run_statements(dbc, statements, ARRAY_SIZE(statements));
+	if (rc != SQLITE_DONE) {
+		LOGP(DDB, LOGL_ERROR, "Unable to update HLR database schema to version 3\n");
+		return rc;
+	}
+	return rc;
+}
+
+static int db_upgrade_v4(struct db_context *dbc)
+{
+	int rc;
+	const char *statements[] = {
+		"ALTER TABLE subscriber ADD COLUMN last_lu_seen_ps TIMESTAMP default NULL",
+		"PRAGMA user_version = 4",
+	};
+
+	rc = db_run_statements(dbc, statements, ARRAY_SIZE(statements));
+	if (rc != SQLITE_DONE) {
+		LOGP(DDB, LOGL_ERROR, "Unable to update HLR database schema to version 4\n");
+		return rc;
+	}
+	return rc;
+}
+
+typedef int (*db_upgrade_func_t)(struct db_context *dbc);
+static db_upgrade_func_t db_upgrade_path[] = {
+	db_upgrade_v1,
+	db_upgrade_v2,
+	db_upgrade_v3,
+	db_upgrade_v4,
+};
+
+static int db_get_user_version(struct db_context *dbc)
+{
+	const char *user_version_sql = "PRAGMA user_version";
+	sqlite3_stmt *stmt;
+	int version, rc;
+
+	rc = sqlite3_prepare_v2(dbc->db, user_version_sql, -1, &stmt, NULL);
+	if (rc != SQLITE_OK) {
+		LOGP(DDB, LOGL_ERROR, "Unable to prepare SQL statement '%s'\n", user_version_sql);
+		return -1;
+	}
+	rc = sqlite3_step(stmt);
+	if (rc == SQLITE_ROW) {
+		version = sqlite3_column_int(stmt, 0);
+	} else {
+		LOGP(DDB, LOGL_ERROR, "SQL statement '%s' failed: %d\n", user_version_sql, rc);
+		version = -1;
+	}
+
+	db_remove_reset(stmt);
+	sqlite3_finalize(stmt);
+	return version;
+}
+
+struct db_context *db_open(void *ctx, const char *fname, bool enable_sqlite_logging, bool allow_upgrade)
 {
 	struct db_context *dbc = talloc_zero(ctx, struct db_context);
 	unsigned int i;
 	int rc;
 	bool has_sqlite_config_sqllog = false;
+	int version;
 
 	LOGP(DDB, LOGL_NOTICE, "using database: %s\n", fname);
 	LOGP(DDB, LOGL_INFO, "Compiled against SQLite3 lib version %s\n", SQLITE_VERSION);
 	LOGP(DDB, LOGL_INFO, "Running with SQLite3 lib version %s\n", sqlite3_libversion());
 
+#ifdef SQLITE_USE_TALLOC
+	/* Configure SQLite3 to use talloc memory allocator */
+	rc = db_sqlite3_use_talloc(ctx);
+	if (rc == SQLITE_OK) {
+		LOGP(DDB, LOGL_NOTICE, "SQLite3 is configured to use talloc\n");
+	} else {
+		LOGP(DDB, LOGL_ERROR, "Failed to configure SQLite3 "
+		     "to use talloc, using default memory allocator\n");
+	}
+#endif
+
 	dbc->fname = talloc_strdup(dbc, fname);
 
 	for (i = 0; i < 0xfffff; i++) {
@@ -274,10 +539,49 @@ struct db_context *db_open(void *ctx, const char *fname, bool enable_sqlite_logg
 		LOGP(DDB, LOGL_ERROR, "Unable to set Write-Ahead Logging: %s\n",
 			err_msg);
 
-	rc = db_bootstrap(dbc);
-	if (rc != SQLITE_OK) {
-		LOGP(DDB, LOGL_ERROR, "Failed to bootstrap DB: (rc=%d) %s\n",
-			rc, sqlite3_errmsg(dbc->db));
+	version = db_get_user_version(dbc);
+	if (version < 0) {
+		LOGP(DDB, LOGL_ERROR, "Unable to read user version number from database '%s'\n", dbc->fname);
+		goto out_free;
+	}
+
+	/* An empty database will always report version zero. */
+	if (version == 0 && !db_is_bootstrapped_v0(dbc)) {
+		LOGP(DDB, LOGL_NOTICE, "Missing database tables detected; Bootstrapping database '%s'\n", dbc->fname);
+		rc = db_bootstrap(dbc);
+		if (rc != SQLITE_OK) {
+			LOGP(DDB, LOGL_ERROR, "Failed to bootstrap DB: (rc=%d) %s\n",
+			     rc, sqlite3_errmsg(dbc->db));
+			goto out_free;
+		}
+		version = CURRENT_SCHEMA_VERSION;
+	}
+
+	LOGP(DDB, LOGL_NOTICE, "Database '%s' has HLR DB schema version %d\n", dbc->fname, version);
+
+	for (; allow_upgrade && (version < ARRAY_SIZE(db_upgrade_path)); version++) {
+		db_upgrade_func_t upgrade_func = db_upgrade_path[version];
+		rc = upgrade_func(dbc);
+		if (rc != SQLITE_DONE) {
+			LOGP(DDB, LOGL_ERROR, "Failed to upgrade HLR DB schema to version %d: (rc=%d) %s\n",
+			     version+1, rc, sqlite3_errmsg(dbc->db));
+			goto out_free;
+		}
+		LOGP(DDB, LOGL_NOTICE, "Database '%s' has been upgraded to HLR DB schema version %d\n",
+		     dbc->fname, version+1);
+	}
+
+	if (version != CURRENT_SCHEMA_VERSION) {
+		if (version < CURRENT_SCHEMA_VERSION) {
+			LOGP(DDB, LOGL_NOTICE, "HLR DB schema version %d is outdated\n", version);
+			if (!allow_upgrade) {
+				LOGP(DDB, LOGL_ERROR, "Not upgrading HLR database to schema version %d; "
+				     "use the --db-upgrade option to allow HLR database upgrades\n",
+				     CURRENT_SCHEMA_VERSION);
+			}
+		} else
+			LOGP(DDB, LOGL_ERROR, "HLR DB schema version %d is unknown\n", version);
+
 		goto out_free;
 	}
 

src/db_auc.c

@@ -26,10 +26,10 @@
 
 #include <sqlite3.h>
 
-#include "logging.h"
-#include "db.h"
-#include "auc.h"
-#include "rand.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/db.h>
+#include <osmocom/hlr/auc.h>
+#include <osmocom/hlr/rand.h>
 
 #define LOGAUC(imsi, level, fmt, args ...)	LOGP(DAUC, level, "IMSI='%s': " fmt, imsi, ## args)
 
@@ -49,7 +49,7 @@ int db_update_sqn(struct db_context *dbc, int64_t subscr_id, uint64_t new_sqn)
 	/* execute the statement */
 	rc = sqlite3_step(stmt);
 	if (rc != SQLITE_DONE) {
-		LOGP(DAUC, LOGL_ERROR, "Cannot update SQN for subscriber ID=%"PRId64
+		LOGP(DAUC, LOGL_ERROR, "Cannot update SQN for subscriber ID=%" PRId64
 		     ": SQL error: (%d) %s\n",
 		     subscr_id, rc, sqlite3_errmsg(dbc->db));
 		ret = -EIO;
@@ -59,11 +59,11 @@ int db_update_sqn(struct db_context *dbc, int64_t subscr_id, uint64_t new_sqn)
 	/* verify execution result */
 	rc = sqlite3_changes(dbc->db);
 	if (!rc) {
-		LOGP(DAUC, LOGL_ERROR, "Cannot update SQN for subscriber ID=%"PRId64
+		LOGP(DAUC, LOGL_ERROR, "Cannot update SQN for subscriber ID=%" PRId64
 		     ": no auc_3g entry for such subscriber\n", subscr_id);
 		ret = -ENOENT;
 	} else if (rc != 1) {
-		LOGP(DAUC, LOGL_ERROR, "Update SQN for subscriber ID=%"PRId64
+		LOGP(DAUC, LOGL_ERROR, "Update SQN for subscriber ID=%" PRId64
 		     ": SQL modified %d rows (expected 1)\n", subscr_id, rc);
 		ret = -EIO;
 	}
@@ -73,6 +73,32 @@ out:
 	return ret;
 }
 
+/* hexparse a specific column of a sqlite prepared statement into dst (with length check)
+ * returns 0 for success, -EIO on error */
+static int hexparse_stmt(uint8_t *dst, size_t dst_len, sqlite3_stmt *stmt, int col, const char *col_name,
+			 const char *imsi)
+{
+	const uint8_t *text;
+	size_t col_len;
+
+	/* Bytes are stored as hex strings in database, hence divide length by two */
+	col_len = sqlite3_column_bytes(stmt, col) / 2;
+
+	if (col_len != dst_len) {
+		LOGAUC(imsi, LOGL_ERROR, "Error reading %s, expected length %lu but has length %lu\n", col_name,
+		       dst_len, col_len);
+		return -EIO;
+	}
+
+	text = sqlite3_column_text(stmt, col);
+	if (!text) {
+		LOGAUC(imsi, LOGL_ERROR, "Error reading %s\n", col_name);
+		return -EIO;
+	}
+	osmo_hexparse((void *)text, dst, dst_len);
+	return 0;
+}
+
 /* obtain the authentication data for a given imsi
  * returns 0 for success, negative value on error:
  * -ENOENT if the IMSI is not known, -ENOKEY if the IMSI is known but has no auth data,
@@ -113,49 +139,34 @@ int db_get_auth_data(struct db_context *dbc, const char *imsi,
 	/* obtain result values using sqlite3_column_*() */
 	if (sqlite3_column_type(stmt, 1) == SQLITE_INTEGER) {
 		/* we do have some 2G authentication data */
-		const uint8_t *ki;
-
-		aud2g->algo = sqlite3_column_int(stmt, 1);
-		ki = sqlite3_column_text(stmt, 2);
-#if 0
-		if (sqlite3_column_bytes(stmt, 2) != sizeof(aud2g->u.gsm.ki)) {
-			LOGAUC(imsi, LOGL_ERROR, "Error reading Ki: %d\n", rc);
+		if (hexparse_stmt(aud2g->u.gsm.ki, sizeof(aud2g->u.gsm.ki), stmt, 2, "Ki", imsi))
 			goto end_2g;
-		}
-#endif
-		osmo_hexparse((void*)ki, (void*)&aud2g->u.gsm.ki, sizeof(aud2g->u.gsm.ki));
+		aud2g->algo = sqlite3_column_int(stmt, 1);
 		aud2g->type = OSMO_AUTH_TYPE_GSM;
 	} else
 		LOGAUC(imsi, LOGL_DEBUG, "No 2G Auth Data\n");
-//end_2g:
+end_2g:
 	if (sqlite3_column_type(stmt, 3) == SQLITE_INTEGER) {
 		/* we do have some 3G authentication data */
-		const uint8_t *k, *op, *opc;
-
-		aud3g->algo = sqlite3_column_int(stmt, 3);
-		k = sqlite3_column_text(stmt, 4);
-		if (!k) {
-			LOGAUC(imsi, LOGL_ERROR, "Error reading K: %d\n", rc);
+		if (hexparse_stmt(aud3g->u.umts.k, sizeof(aud3g->u.umts.k), stmt, 4, "K", imsi)) {
 			ret = -EIO;
 			goto out;
 		}
-		osmo_hexparse((void*)k, (void*)&aud3g->u.umts.k, sizeof(aud3g->u.umts.k));
+		aud3g->algo = sqlite3_column_int(stmt, 3);
+
 		/* UMTS Subscribers can have either OP or OPC */
-		op = sqlite3_column_text(stmt, 5);
-		if (!op) {
-			opc = sqlite3_column_text(stmt, 6);
-			if (!opc) {
-				LOGAUC(imsi, LOGL_ERROR, "Error reading OPC: %d\n", rc);
+		if (sqlite3_column_text(stmt, 5)) {
+			if (hexparse_stmt(aud3g->u.umts.opc, sizeof(aud3g->u.umts.opc), stmt, 5, "OP", imsi)) {
 				ret = -EIO;
 				goto out;
 			}
-			osmo_hexparse((void*)opc, (void*)&aud3g->u.umts.opc,
-					sizeof(aud3g->u.umts.opc));
-			aud3g->u.umts.opc_is_op = 0;
-		} else {
-			osmo_hexparse((void*)op, (void*)&aud3g->u.umts.opc,
-					sizeof(aud3g->u.umts.opc));
 			aud3g->u.umts.opc_is_op = 1;
+		} else {
+			if (hexparse_stmt(aud3g->u.umts.opc, sizeof(aud3g->u.umts.opc), stmt, 6, "OPC", imsi)) {
+				ret = -EIO;
+				goto out;
+			}
+			aud3g->u.umts.opc_is_op = 0;
 		}
 		aud3g->u.umts.sqn = sqlite3_column_int64(stmt, 7);
 		aud3g->u.umts.ind_bitlen = sqlite3_column_int(stmt, 8);
@@ -178,7 +189,7 @@ out:
 int db_get_auc(struct db_context *dbc, const char *imsi,
 	       unsigned int auc_3g_ind, struct osmo_auth_vector *vec,
 	       unsigned int num_vec, const uint8_t *rand_auts,
-	       const uint8_t *auts)
+	       const uint8_t *auts, bool separation_bit)
 {
 	struct osmo_sub_auth_data aud2g, aud3g;
 	int64_t subscr_id;
@@ -198,6 +209,12 @@ int db_get_auc(struct db_context *dbc, const char *imsi,
 		       aud3g.u.umts.ind_bitlen, aud3g.u.umts.ind);
 		aud3g.u.umts.ind &= (1U << aud3g.u.umts.ind_bitlen) - 1;
 	}
+	/* the first bit (bit0) cannot be used as AMF anymore, but has been
+	 * re-appropriated as the separation bit.  See 3GPP TS 33.102 Annex H
+	 * together with 3GPP TS 33.401 / 33.402 / 33.501 */
+	aud3g.u.umts.amf[0] = aud3g.u.umts.amf[0] & 0x7f;
+	if (separation_bit)
+		aud3g.u.umts.amf[0] |= 0x80;
 
 	LOGAUC(imsi, LOGL_DEBUG, "Calling to generate %u vectors\n", num_vec);
 	rc = auc_compute_vectors(vec, num_vec, &aud2g, &aud3g, rand_auts, auts);

src/db_debug.c

@@ -0,0 +1,86 @@
+/*
+ * libtalloc based memory allocator for SQLite3.
+ *
+ * (C) 2019 by Vadim Yanitskiy <axilirator@gmail.com>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include <sqlite3.h>
+#include <talloc.h>
+#include <errno.h>
+
+/* Dedicated talloc context for SQLite */
+static void *db_sqlite_ctx = NULL;
+
+static void *tall_xMalloc(int size)
+{
+	return talloc_size(db_sqlite_ctx, size);
+}
+
+static void tall_xFree(void *ptr)
+{
+	talloc_free(ptr);
+}
+
+static void *tall_xRealloc(void *ptr, int size)
+{
+	return talloc_realloc_fn(db_sqlite_ctx, ptr, size);
+}
+
+static int tall_xSize(void *ptr)
+{
+	return talloc_total_size(ptr);
+}
+
+/* DUMMY: talloc doesn't round up the allocation size */
+static int tall_xRoundup(int size) { return size; }
+
+/* DUMMY: nothing to initialize */
+static int tall_xInit(void *data) { return 0; }
+
+/* DUMMY: nothing to deinitialize */
+static void tall_xShutdown(void *data) {  }
+
+/* Interface between SQLite and talloc memory allocator */
+static const struct sqlite3_mem_methods tall_sqlite_if = {
+	/* Memory allocation function */
+	.xMalloc = &tall_xMalloc,
+	/* Free a prior allocation */
+	.xFree = &tall_xFree,
+	/* Resize an allocation */
+	.xRealloc = &tall_xRealloc,
+	/* Return the size of an allocation */
+	.xSize = &tall_xSize,
+	/* Round up request size to allocation size */
+	.xRoundup = &tall_xRoundup,
+	/* Initialize the memory allocator */
+	.xInit = &tall_xInit,
+	/* Deinitialize the memory allocator */
+	.xShutdown = &tall_xShutdown,
+	/* Argument to xInit() and xShutdown() */
+	.pAppData = NULL,
+};
+
+int db_sqlite3_use_talloc(void *ctx)
+{
+	if (db_sqlite_ctx != NULL)
+		return -EEXIST;
+
+	db_sqlite_ctx = talloc_named_const(ctx, 0, "SQLite3");
+	return sqlite3_config(SQLITE_CONFIG_MALLOC, &tall_sqlite_if);
+}

src/db_hlr.c

@@ -17,9 +17,15 @@
  *
  */
 
+#define _POSIX_C_SOURCE 200809L /* for strptime(3) */
+/* These are needed as well due to the above _POSIX_C_SOURCE definition: */
+#define _DEFAULT_SOURCE		/* for struct timezone */
+#define _XOPEN_SOURCE		/* for clockid_t */
+
 #include <string.h>
 #include <errno.h>
 #include <inttypes.h>
+#include <time.h>
 
 #include <osmocom/core/utils.h>
 #include <osmocom/crypt/auth.h>
@@ -27,20 +33,21 @@
 
 #include <sqlite3.h>
 
-#include "logging.h"
-#include "hlr.h"
-#include "db.h"
-#include "gsup_server.h"
-#include "luop.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/hlr.h>
+#include <osmocom/hlr/db.h>
+#include <osmocom/hlr/gsup_server.h>
+#include <osmocom/hlr/luop.h>
 
 #define LOGHLR(imsi, level, fmt, args ...)	LOGP(DAUC, level, "IMSI='%s': " fmt, imsi, ## args)
 
 /*! Add new subscriber record to the HLR database.
  * \param[in,out] dbc  database context.
  * \param[in] imsi  ASCII string of IMSI digits, is validated.
+ * \param[in] flags  Bitmask of DB_SUBSCR_FLAG_*.
  * \returns 0 on success, -EINVAL on invalid IMSI, -EIO on database error.
  */
-int db_subscr_create(struct db_context *dbc, const char *imsi)
+int db_subscr_create(struct db_context *dbc, const char *imsi, uint8_t flags)
 {
 	sqlite3_stmt *stmt;
 	int rc;
@@ -55,6 +62,10 @@ int db_subscr_create(struct db_context *dbc, const char *imsi)
 
 	if (!db_bind_text(stmt, "$imsi", imsi))
 		return -EIO;
+	if (!db_bind_int(stmt, "$nam_cs", (flags & DB_SUBSCR_FLAG_NAM_CS) != 0))
+		return -EIO;
+	if (!db_bind_int(stmt, "$nam_ps", (flags & DB_SUBSCR_FLAG_NAM_PS) != 0))
+		return -EIO;
 
 	/* execute the statement */
 	rc = sqlite3_step(stmt);
@@ -92,7 +103,7 @@ int db_subscr_delete_by_id(struct db_context *dbc, int64_t subscr_id)
 	rc = sqlite3_step(stmt);
 	if (rc != SQLITE_DONE) {
 		LOGP(DAUC, LOGL_ERROR,
-		       "Cannot delete subscriber ID=%"PRId64": SQL error: (%d) %s\n",
+		       "Cannot delete subscriber ID=%" PRId64 ": SQL error: (%d) %s\n",
 		       subscr_id, rc, sqlite3_errmsg(dbc->db));
 		db_remove_reset(stmt);
 		return -EIO;
@@ -101,11 +112,11 @@ int db_subscr_delete_by_id(struct db_context *dbc, int64_t subscr_id)
 	/* verify execution result */
 	rc = sqlite3_changes(dbc->db);
 	if (!rc) {
-		LOGP(DAUC, LOGL_ERROR, "Cannot delete: no such subscriber: ID=%"PRId64"\n",
+		LOGP(DAUC, LOGL_ERROR, "Cannot delete: no such subscriber: ID=%" PRId64 "\n",
 		     subscr_id);
 		ret = -ENOENT;
 	} else if (rc != 1) {
-		LOGP(DAUC, LOGL_ERROR, "Delete subscriber ID=%"PRId64
+		LOGP(DAUC, LOGL_ERROR, "Delete subscriber ID=%" PRId64
 		     ": SQL modified %d rows (expected 1)\n", subscr_id, rc);
 		ret = -EIO;
 	}
@@ -135,8 +146,8 @@ int db_subscr_delete_by_id(struct db_context *dbc, int64_t subscr_id)
 
 /*! Set a subscriber's MSISDN in the HLR database.
  * \param[in,out] dbc  database context.
- * \param[in] imsi  ASCII string of IMSI digits.
- * \param[in] msisdn  ASCII string of MSISDN digits.
+ * \param[in] imsi  ASCII string of IMSI digits
+ * \param[in] msisdn  ASCII string of MSISDN digits, or NULL to remove the MSISDN.
  * \returns 0 on success, -EINVAL in case of invalid MSISDN string, -EIO on
  *          database failure, -ENOENT if no such subscriber exists.
  */
@@ -146,19 +157,22 @@ int db_subscr_update_msisdn_by_imsi(struct db_context *dbc, const char *imsi,
 	int rc;
 	int ret = 0;
 
-	if (!osmo_msisdn_str_valid(msisdn)) {
+	if (msisdn && !osmo_msisdn_str_valid(msisdn)) {
 		LOGHLR(imsi, LOGL_ERROR,
 		       "Cannot update subscriber: invalid MSISDN: '%s'\n",
 		       msisdn);
 		return -EINVAL;
 	}
 
-	sqlite3_stmt *stmt = dbc->stmt[DB_STMT_SET_MSISDN_BY_IMSI];
+	sqlite3_stmt *stmt = dbc->stmt[
+		msisdn ? DB_STMT_SET_MSISDN_BY_IMSI : DB_STMT_DELETE_MSISDN_BY_IMSI];
 
 	if (!db_bind_text(stmt, "$imsi", imsi))
 		return -EIO;
-	if (!db_bind_text(stmt, "$msisdn", msisdn))
-		return -EIO;
+	if (msisdn) {
+		if (!db_bind_text(stmt, "$msisdn", msisdn))
+			return -EIO;
+	}
 
 	/* execute the statement */
 	rc = sqlite3_step(stmt);
@@ -250,11 +264,11 @@ int db_subscr_update_aud_by_id(struct db_context *dbc, int64_t subscr_id,
 		switch (aud->algo) {
 		case OSMO_AUTH_ALG_NONE:
 		case OSMO_AUTH_ALG_MILENAGE:
+		case OSMO_AUTH_ALG_XOR:
 			break;
 		case OSMO_AUTH_ALG_COMP128v1:
 		case OSMO_AUTH_ALG_COMP128v2:
 		case OSMO_AUTH_ALG_COMP128v3:
-		case OSMO_AUTH_ALG_XOR:
 			LOGP(DAUC, LOGL_ERROR, "Cannot update auth tokens:"
 			     " auth algo not suited for 3G: %s\n",
 			     osmo_auth_alg_name(aud->algo));
@@ -312,7 +326,7 @@ int db_subscr_update_aud_by_id(struct db_context *dbc, int64_t subscr_id,
 		 * empty, and no entry is not an error then.*/
 		ret = -ENOENT;
 	else if (rc != 1) {
-		LOGP(DAUC, LOGL_ERROR, "Delete subscriber ID=%"PRId64
+		LOGP(DAUC, LOGL_ERROR, "Delete subscriber ID=%" PRId64
 		     " from %s: SQL modified %d rows (expected 1)\n",
 		     subscr_id, label, rc);
 		ret = -EIO;
@@ -377,6 +391,77 @@ out:
 	return ret;
 }
 
+/*! Set a subscriber's IMEI in the HLR database.
+ * \param[in,out] dbc  database context.
+ * \param[in] imsi  ASCII string of IMSI digits
+ * \param[in] imei  ASCII string of identifier digits, or NULL to remove the IMEI.
+ * \returns 0 on success, -ENOENT when the given subscriber does not exist,
+ *         -EIO on database errors.
+ */
+int db_subscr_update_imei_by_imsi(struct db_context *dbc, const char* imsi, const char *imei)
+{
+	int rc, ret = 0;
+	sqlite3_stmt *stmt = dbc->stmt[DB_STMT_UPD_IMEI_BY_IMSI];
+
+	if (imei && !osmo_imei_str_valid(imei, false)) {
+		LOGP(DAUC, LOGL_ERROR, "Cannot update subscriber IMSI='%s': invalid IMEI: '%s'\n", imsi, imei);
+		return -EINVAL;
+	}
+
+	if (!db_bind_text(stmt, "$imsi", imsi))
+		return -EIO;
+	if (imei && !db_bind_text(stmt, "$imei", imei))
+		return -EIO;
+
+	/* execute the statement */
+	rc = sqlite3_step(stmt);
+	if (rc != SQLITE_DONE) {
+		LOGP(DAUC, LOGL_ERROR, "Update IMEI for subscriber IMSI='%s': SQL Error: %s\n", imsi,
+		     sqlite3_errmsg(dbc->db));
+		ret = -EIO;
+		goto out;
+	}
+
+	/* verify execution result */
+	rc = sqlite3_changes(dbc->db);
+	if (!rc) {
+		LOGP(DAUC, LOGL_ERROR, "Cannot update IMEI for subscriber IMSI='%s': no such subscriber\n", imsi);
+		ret = -ENOENT;
+	} else if (rc != 1) {
+		LOGP(DAUC, LOGL_ERROR, "Update IMEI for subscriber IMSI='%s': SQL modified %d rows (expected 1)\n",
+		     imsi, rc);
+		ret = -EIO;
+	}
+
+out:
+	db_remove_reset(stmt);
+	return ret;
+}
+
+static void parse_last_lu_seen(time_t *dst, const char *last_lu_seen_str, const char *imsi, const char *label)
+{
+	struct tm tm = {0};
+	time_t val;
+	if (!last_lu_seen_str || last_lu_seen_str[0] == '\0')
+		return;
+
+	if (strptime(last_lu_seen_str, DB_LAST_LU_SEEN_FMT, &tm) == NULL) {
+		LOGP(DAUC, LOGL_ERROR, "IMSI-%s: Last LU Seen %s: Cannot parse timestamp '%s'\n",
+		     imsi, label, last_lu_seen_str);
+		return;
+	}
+
+	errno = 0;
+	val = mktime(&tm);
+	if (val == -1) {
+		LOGP(DAUC, LOGL_ERROR, "IMSI-%s: Last LU Seen %s: Cannot convert timestamp '%s' to time_t: %s\n",
+		     imsi, label, last_lu_seen_str, strerror(errno));
+		val = 0;
+	}
+
+	*dst = val;
+}
+
 /* Common code for db_subscr_get_by_*() functions. */
 static int db_sel(struct db_context *dbc, sqlite3_stmt *stmt, struct hlr_subscriber *subscr,
 		  const char **err)
@@ -404,17 +489,22 @@ static int db_sel(struct db_context *dbc, sqlite3_stmt *stmt, struct hlr_subscri
 	subscr->id = sqlite3_column_int64(stmt, 0);
 	copy_sqlite3_text_to_buf(subscr->imsi, stmt, 1);
 	copy_sqlite3_text_to_buf(subscr->msisdn, stmt, 2);
+	copy_sqlite3_text_to_buf(subscr->imei, stmt, 3);
 	/* FIXME: These should all be BLOBs as they might contain NUL */
-	copy_sqlite3_text_to_buf(subscr->vlr_number, stmt, 3);
-	copy_sqlite3_text_to_buf(subscr->sgsn_number, stmt, 4);
-	copy_sqlite3_text_to_buf(subscr->sgsn_address, stmt, 5);
-	subscr->periodic_lu_timer = sqlite3_column_int(stmt, 6);
-	subscr->periodic_rau_tau_timer = sqlite3_column_int(stmt, 7);
-	subscr->nam_cs = sqlite3_column_int(stmt, 8);
-	subscr->nam_ps = sqlite3_column_int(stmt, 9);
-	subscr->lmsi = sqlite3_column_int(stmt, 10);
-	subscr->ms_purged_cs = sqlite3_column_int(stmt, 11);
-	subscr->ms_purged_ps = sqlite3_column_int(stmt, 12);
+	copy_sqlite3_text_to_buf(subscr->vlr_number, stmt, 4);
+	copy_sqlite3_text_to_buf(subscr->sgsn_number, stmt, 5);
+	copy_sqlite3_text_to_buf(subscr->sgsn_address, stmt, 6);
+	subscr->periodic_lu_timer = sqlite3_column_int(stmt, 7);
+	subscr->periodic_rau_tau_timer = sqlite3_column_int(stmt, 8);
+	subscr->nam_cs = sqlite3_column_int(stmt, 9);
+	subscr->nam_ps = sqlite3_column_int(stmt, 10);
+	subscr->lmsi = sqlite3_column_int(stmt, 11);
+	subscr->ms_purged_cs = sqlite3_column_int(stmt, 12);
+	subscr->ms_purged_ps = sqlite3_column_int(stmt, 13);
+	parse_last_lu_seen(&subscr->last_lu_seen, (const char *)sqlite3_column_text(stmt, 14),
+			   subscr->imsi, "CS");
+	parse_last_lu_seen(&subscr->last_lu_seen_ps, (const char *)sqlite3_column_text(stmt, 15),
+			   subscr->imsi, "PS");
 
 out:
 	db_remove_reset(stmt);
@@ -433,6 +523,31 @@ out:
 	return ret;
 }
 
+/*! Check if a subscriber exists in the HLR database.
+ * \param[in, out] dbc  database context.
+ * \param[in] imsi  ASCII string of IMSI digits.
+ * \returns 0 if it exists, -ENOENT if it does not exist, -EIO on database error.
+ */
+int db_subscr_exists_by_imsi(struct db_context *dbc, const char *imsi) {
+	sqlite3_stmt *stmt = dbc->stmt[DB_STMT_EXISTS_BY_IMSI];
+	const char *err;
+	int rc;
+
+	if (!db_bind_text(stmt, NULL, imsi))
+		return -EIO;
+
+	rc = sqlite3_step(stmt);
+	db_remove_reset(stmt);
+	if (rc == SQLITE_ROW)
+		return 0; /* exists */
+	if (rc == SQLITE_DONE)
+		return -ENOENT; /* does not exist */
+
+	err = sqlite3_errmsg(dbc->db);
+	LOGP(DAUC, LOGL_ERROR, "Failed to check if subscriber exists by IMSI='%s': %s\n", imsi, err);
+	return rc;
+}
+
 /*! Retrieve subscriber data from the HLR database.
  * \param[in,out] dbc  database context.
  * \param[in] imsi  ASCII string of IMSI digits.
@@ -457,6 +572,33 @@ int db_subscr_get_by_imsi(struct db_context *dbc, const char *imsi,
 	return rc;
 }
 
+/*! Check if a subscriber exists in the HLR database.
+ * \param[in, out] dbc  database context.
+ * \param[in] msisdn  ASCII string of MSISDN digits.
+ * \returns 0 if it exists, -ENOENT if it does not exist, -EIO on database error.
+ */
+int db_subscr_exists_by_msisdn(struct db_context *dbc, const char *msisdn)
+{
+	sqlite3_stmt *stmt = dbc->stmt[DB_STMT_EXISTS_BY_MSISDN];
+	const char *err;
+	int rc;
+
+	if (!db_bind_text(stmt, NULL, msisdn))
+		return -EIO;
+
+	rc = sqlite3_step(stmt);
+	db_remove_reset(stmt);
+	if (rc == SQLITE_ROW)
+		return 0; /* exists */
+	if (rc == SQLITE_DONE)
+		return -ENOENT; /* does not exist */
+
+	err = sqlite3_errmsg(dbc->db);
+	LOGP(DAUC, LOGL_ERROR, "Failed to check if subscriber exists "
+		"by MSISDN='%s': %s\n", msisdn, err);
+	return rc;
+}
+
 /*! Retrieve subscriber data from the HLR database.
  * \param[in,out] dbc  database context.
  * \param[in] msisdn  ASCII string of MSISDN digits.
@@ -500,11 +642,33 @@ int db_subscr_get_by_id(struct db_context *dbc, int64_t id,
 
 	rc = db_sel(dbc, stmt, subscr, &err);
 	if (rc)
-		LOGP(DAUC, LOGL_ERROR, "Cannot read subscriber from db: ID=%"PRId64": %s\n",
+		LOGP(DAUC, LOGL_ERROR, "Cannot read subscriber from db: ID=%" PRId64 ": %s\n",
 		     id, err);
 	return rc;
 }
 
+/*! Retrieve subscriber data from the HLR database.
+ * \param[in,out] dbc  database context.
+ * \param[in] imei  ASCII string of identifier digits
+ * \param[out] subscr  place retrieved data in this struct.
+ * \returns 0 on success, -ENOENT if no such subscriber was found, -EIO on
+ *          database error.
+ */
+int db_subscr_get_by_imei(struct db_context *dbc, const char *imei, struct hlr_subscriber *subscr)
+{
+	sqlite3_stmt *stmt = dbc->stmt[DB_STMT_SEL_BY_IMEI];
+	const char *err;
+	int rc;
+
+	if (!db_bind_text(stmt, NULL, imei))
+		return -EIO;
+
+	rc = db_sel(dbc, stmt, subscr, &err);
+	if (rc)
+		LOGP(DAUC, LOGL_ERROR, "Cannot read subscriber from db: IMEI=%s: %s\n", imei, err);
+	return rc;
+}
+
 /*! You should use hlr_subscr_nam() instead; enable or disable PS or CS for a
  * subscriber without notifying GSUP clients.
  * \param[in,out] dbc  database context.
@@ -574,6 +738,7 @@ int db_subscr_lu(struct db_context *dbc, int64_t subscr_id,
 {
 	sqlite3_stmt *stmt;
 	int rc, ret = 0;
+	struct timespec localtime;
 
 	stmt = dbc->stmt[is_ps ? DB_STMT_UPD_SGSN_BY_ID
 			       : DB_STMT_UPD_VLR_BY_ID];
@@ -587,7 +752,7 @@ int db_subscr_lu(struct db_context *dbc, int64_t subscr_id,
 	/* execute the statement */
 	rc = sqlite3_step(stmt);
 	if (rc != SQLITE_DONE) {
-		LOGP(DAUC, LOGL_ERROR, "Update %s number for subscriber ID=%"PRId64": SQL Error: %s\n",
+		LOGP(DAUC, LOGL_ERROR, "Update %s number for subscriber ID=%" PRId64 ": SQL Error: %s\n",
 		     is_ps? "SGSN" : "VLR", subscr_id, sqlite3_errmsg(dbc->db));
 		ret = -EIO;
 		goto out;
@@ -596,17 +761,58 @@ int db_subscr_lu(struct db_context *dbc, int64_t subscr_id,
 	/* verify execution result */
 	rc = sqlite3_changes(dbc->db);
 	if (!rc) {
-		LOGP(DAUC, LOGL_ERROR, "Cannot update %s number for subscriber ID=%"PRId64
+		LOGP(DAUC, LOGL_ERROR, "Cannot update %s number for subscriber ID=%" PRId64
 		     ": no such subscriber\n",
 		     is_ps? "SGSN" : "VLR", subscr_id);
 		ret = -ENOENT;
+		goto out;
 	} else if (rc != 1) {
-		LOGP(DAUC, LOGL_ERROR, "Update %s number for subscriber ID=%"PRId64
+		LOGP(DAUC, LOGL_ERROR, "Update %s number for subscriber ID=%" PRId64
 		       ": SQL modified %d rows (expected 1)\n",
 		       is_ps? "SGSN" : "VLR", subscr_id, rc);
 		ret = -EIO;
+		goto out;
 	}
 
+	db_remove_reset(stmt);
+
+	if (osmo_clock_gettime(CLOCK_REALTIME, &localtime) != 0) {
+		LOGP(DAUC, LOGL_ERROR, "Cannot get the current time: (%d) %s\n", errno, strerror(errno));
+		ret = -errno;
+		goto out;
+	}
+
+	stmt = dbc->stmt[is_ps? DB_STMT_SET_LAST_LU_SEEN_PS : DB_STMT_SET_LAST_LU_SEEN];
+
+	if (!db_bind_int64(stmt, "$subscriber_id", subscr_id))
+		return -EIO;
+	/* The timestamp will be converted to UTC by SQLite. */
+	if (!db_bind_int64(stmt, "$val", (int64_t)localtime.tv_sec)) {
+		ret = -EIO;
+		goto out;
+	}
+
+	rc = sqlite3_step(stmt);
+	if (rc != SQLITE_DONE) {
+		LOGP(DAUC, LOGL_ERROR,
+		       "Cannot update LU timestamp for subscriber ID=%" PRId64 ": SQL error: (%d) %s\n",
+		       subscr_id, rc, sqlite3_errmsg(dbc->db));
+		ret = -EIO;
+		goto out;
+	}
+
+	/* verify execution result */
+	rc = sqlite3_changes(dbc->db);
+	if (!rc) {
+		LOGP(DAUC, LOGL_ERROR, "Cannot update LU timestamp for subscriber ID=%" PRId64
+		     ": no such subscriber\n", subscr_id);
+		ret = -ENOENT;
+		goto out;
+	} else if (rc != 1) {
+		LOGP(DAUC, LOGL_ERROR, "Update LU timestamp for subscriber ID=%" PRId64
+		     ": SQL modified %d rows (expected 1)\n", subscr_id, rc);
+		ret = -EIO;
+	}
 out:
 	db_remove_reset(stmt);
 	return ret;

src/db_sql2c.sed

@@ -1,4 +1,4 @@
-# Input to this is sql/hlr.sql.
+# Input to this are sql/*.sql files.
 #
 # We want each SQL statement line wrapped in "...\n", and each end (";") to
 # become a comma:

src/gsup_router.c

@@ -23,17 +23,17 @@
 #include <osmocom/core/linuxlist.h>
 #include <osmocom/core/talloc.h>
 
-#include "logging.h"
-#include "gsup_server.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/gsup_server.h>
+#include <osmocom/hlr/gsup_router.h>
 
-struct gsup_route {
-	struct llist_head list;
-
-	uint8_t *addr;
-	struct osmo_gsup_conn *conn;
-};
-
-/* find a route for the given address */
+/*! Find a route for the given address.
+ * \param[in] gs gsup server
+ * \param[in] addr IPA name of the client (SGSN, MSC/VLR). Although this is passed like a blob, together with the
+ *                 length, it must be nul-terminated! This is for legacy reasons, see the discussion here:
+ *                 https://gerrit.osmocom.org/#/c/osmo-hlr/+/13048/
+ * \param[in] addrlen length of addr, *including the nul-byte* (strlen(addr) + 1).
+ */
 struct osmo_gsup_conn *gsup_route_find(struct osmo_gsup_server *gs,
 					const uint8_t *addr, size_t addrlen)
 {
@@ -47,6 +47,22 @@ struct osmo_gsup_conn *gsup_route_find(struct osmo_gsup_server *gs,
 	return NULL;
 }
 
+/*! Find a GSUP connection's route (to read the IPA address from the route).
+ * \param[in] conn GSUP connection
+ * \return GSUP route
+ */
+struct gsup_route *gsup_route_find_by_conn(const struct osmo_gsup_conn *conn)
+{
+	struct gsup_route *gr;
+
+	llist_for_each_entry(gr, &conn->server->routes, list) {
+		if (gr->conn == conn)
+			return gr;
+	}
+
+	return NULL;
+}
+
 /* add a new route for the given address to the given conn */
 int gsup_route_add(struct osmo_gsup_conn *conn, const uint8_t *addr, size_t addrlen)
 {
@@ -61,7 +77,7 @@ int gsup_route_add(struct osmo_gsup_conn *conn, const uint8_t *addr, size_t addr
 	if (!gr)
 		return -ENOMEM;
 
-	LOGP(DMAIN, LOGL_INFO, "Adding GSUP route for %s\n", addr);
+	LOGP(DMAIN, LOGL_INFO, "Adding GSUP route for %s via %s:%u\n", addr, conn->conn->addr, conn->conn->port);
 
 	gr->addr = talloc_memdup(gr, addr, addrlen);
 	gr->conn = conn;

src/gsup_send.c

@@ -21,12 +21,19 @@
 
 #include <errno.h>
 
-#include "gsup_server.h"
-#include "gsup_router.h"
+#include <osmocom/hlr/gsup_server.h>
+#include <osmocom/hlr/gsup_router.h>
 
 #include <osmocom/core/logging.h>
 
-/* Send a msgb to a given address using routing */
+/*! Send a msgb to a given address using routing.
+ * \param[in] gs gsup server
+ * \param[in] addr IPA name of the client (SGSN, MSC/VLR). Although this is passed like a blob, together with the
+ *                 length, it must be nul-terminated! This is for legacy reasons, see the discussion here:
+ *                 https://gerrit.osmocom.org/#/c/osmo-hlr/+/13048/
+ * \param[in] addrlen length of addr, *including the nul-byte* (strlen(addr) + 1).
+ * \param[in] msg message buffer
+ */
 int osmo_gsup_addr_send(struct osmo_gsup_server *gs,
 			const uint8_t *addr, size_t addrlen,
 			struct msgb *msg)
@@ -35,7 +42,7 @@ int osmo_gsup_addr_send(struct osmo_gsup_server *gs,
 
 	conn = gsup_route_find(gs, addr, addrlen);
 	if (!conn) {
-		DEBUGP(DLGSUP, "Cannot find route for addr %s\n", addr);
+		DEBUGP(DLGSUP, "Cannot find route for addr %s\n", osmo_quote_str((const char*)addr, addrlen));
 		msgb_free(msg);
 		return -ENODEV;
 	}

src/gsup_server.c

@@ -27,8 +27,15 @@
 #include <osmocom/gsm/gsm48_ie.h>
 #include <osmocom/gsm/apn.h>
 
-#include "gsup_server.h"
-#include "gsup_router.h"
+#include <osmocom/hlr/gsup_server.h>
+#include <osmocom/hlr/gsup_router.h>
+
+struct msgb *osmo_gsup_msgb_alloc(const char *label)
+{
+	struct msgb *msg = msgb_alloc_headroom(1024+16, 16, label);
+	OSMO_ASSERT(msg);
+	return msg;
+}
 
 static void osmo_gsup_server_send(struct osmo_gsup_conn *conn,
 			     int proto_ext, struct msgb *msg_tx)

src/gsupclient/gsup_client.c

@@ -7,16 +7,16 @@
  * Author: Neels Hofmeyr
  *
  * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
  * (at your option) any later version.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
+ * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU Affero General Public License
+ * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  *
  */
@@ -170,16 +170,12 @@ static int gsup_client_read_cb(struct ipa_client_conn *link, struct msgb *msg)
 	struct ipaccess_head_ext *he = (struct ipaccess_head_ext *) msgb_l2(msg);
 	struct osmo_gsup_client *gsupc = (struct osmo_gsup_client *)link->data;
 	int rc;
-	struct ipaccess_unit ipa_dev = {
-		/* see gsup_client_create() on const vs non-const */
-		.unit_name = (char*)gsupc->unit_name,
-	};
 
-	OSMO_ASSERT(ipa_dev.unit_name);
+	OSMO_ASSERT(gsupc->unit_name);
 
 	msg->l2h = &hh->data[0];
 
-	rc = ipaccess_bts_handle_ccm(link, &ipa_dev, msg);
+	rc = ipaccess_bts_handle_ccm(link, gsupc->ipa_dev, msg);
 
 	if (rc < 0) {
 		LOGP(DLGSUP, LOGL_NOTICE,
@@ -262,24 +258,33 @@ static void start_test_procedure(struct osmo_gsup_client *gsupc)
 	gsup_client_send_ping(gsupc);
 }
 
-struct osmo_gsup_client *osmo_gsup_client_create(void *talloc_ctx,
-						 const char *unit_name,
-						 const char *ip_addr,
-						 unsigned int tcp_port,
-						 osmo_gsup_client_read_cb_t read_cb,
-						 struct osmo_oap_client_config *oapc_config)
+/*!
+ * Create a gsup client connecting to the specified IP address and TCP port.
+ * Use the provided ipaccess unit as the client-side identifier; ipa_dev should
+ * be allocated in talloc_ctx talloc_ctx as well.
+ * \param[in] talloc_ctx talloc context.
+ * \param[in] ipa_dev IP access unit which contains client identification information; must be allocated
+ *                    in talloc_ctx as well to ensure it lives throughout the lifetime of the connection.
+ * \param[in] ip_addr GSUP server IP address.
+ * \param[in] tcp_port GSUP server TCP port.
+ * \param[in] read_cb callback for reading from the GSUP connection.
+ * \param[in] oapc_config OPA client configuration.
+ *  \returns a GSUP client connection or NULL on failure.
+ */
+struct osmo_gsup_client *osmo_gsup_client_create2(void *talloc_ctx,
+						  struct ipaccess_unit *ipa_dev,
+						  const char *ip_addr,
+						  unsigned int tcp_port,
+						  osmo_gsup_client_read_cb_t read_cb,
+						  struct osmo_oap_client_config *oapc_config)
 {
 	struct osmo_gsup_client *gsupc;
 	int rc;
 
 	gsupc = talloc_zero(talloc_ctx, struct osmo_gsup_client);
 	OSMO_ASSERT(gsupc);
-
-	/* struct ipaccess_unit has a non-const unit_name, so let's copy to be
-	 * able to have a non-const unit_name here as well. To not taint the
-	 * public gsup_client API, let's store it in a const char* anyway. */
-	gsupc->unit_name = talloc_strdup(gsupc, unit_name);
-	OSMO_ASSERT(gsupc->unit_name);
+	gsupc->unit_name = (const char *)ipa_dev->unit_name; /* API backwards compat */
+	gsupc->ipa_dev = ipa_dev;
 
 	/* a NULL oapc_config will mark oap_state disabled. */
 	rc = osmo_oap_client_init(oapc_config, &gsupc->oap_state);
@@ -313,6 +318,22 @@ failed:
 	return NULL;
 }
 
+/**
+ * Like osmo_gsup_client_create2() except it expects a unit name instead
+ * of a full-blown ipacess_unit as the client-side identifier.
+ */
+struct osmo_gsup_client *osmo_gsup_client_create(void *talloc_ctx,
+						 const char *unit_name,
+						 const char *ip_addr,
+						 unsigned int tcp_port,
+						 osmo_gsup_client_read_cb_t read_cb,
+						 struct osmo_oap_client_config *oapc_config)
+{
+	struct ipaccess_unit *ipa_dev = talloc_zero(talloc_ctx, struct ipaccess_unit);
+	ipa_dev->unit_name = talloc_strdup(ipa_dev, unit_name);
+	return osmo_gsup_client_create2(talloc_ctx, ipa_dev, ip_addr, tcp_port, read_cb, oapc_config);
+}
+
 void osmo_gsup_client_destroy(struct osmo_gsup_client *gsupc)
 {
 	osmo_timer_del(&gsupc->connect_timer);
@@ -339,6 +360,43 @@ int osmo_gsup_client_send(struct osmo_gsup_client *gsupc, struct msgb *msg)
 	return 0;
 }
 
+/*! Encode and send a GSUP message.
+ * \param[in] gsupc    GSUP client.
+ * \param[in] gsup_msg GSUP message to be sent.
+ * \returns 0 in case of success, negative on error.
+ */
+int osmo_gsup_client_enc_send(struct osmo_gsup_client *gsupc,
+			      const struct osmo_gsup_message *gsup_msg)
+{
+	struct msgb *gsup_msgb;
+	int rc;
+
+	gsup_msgb = osmo_gsup_client_msgb_alloc();
+	if (!gsup_msgb) {
+		LOGP(DLGSUP, LOGL_ERROR, "Couldn't allocate GSUP message\n");
+		return -ENOMEM;
+	}
+
+	rc = osmo_gsup_encode(gsup_msgb, gsup_msg);
+	if (rc) {
+		LOGP(DLGSUP, LOGL_ERROR, "Couldn't encode GSUP message\n");
+		goto error;
+	}
+
+	rc = osmo_gsup_client_send(gsupc, gsup_msgb);
+	if (rc) {
+		LOGP(DLGSUP, LOGL_ERROR, "Couldn't send GSUP message\n");
+		/* Do not free, osmo_gsup_client_send() already has. */
+		return rc;
+	}
+
+	return 0;
+
+error:
+	talloc_free(gsup_msgb);
+	return rc;
+}
+
 struct msgb *osmo_gsup_client_msgb_alloc(void)
 {
 	return msgb_alloc_headroom(4000, 64, __func__);

src/hlr.c

@@ -23,6 +23,7 @@
 #include <getopt.h>
 
 #include <osmocom/core/msgb.h>
+#include <osmocom/core/stats.h>
 #include <osmocom/core/logging.h>
 #include <osmocom/core/application.h>
 #include <osmocom/gsm/gsup.h>
@@ -32,19 +33,23 @@
 #include <osmocom/vty/ports.h>
 #include <osmocom/ctrl/control_vty.h>
 #include <osmocom/gsm/apn.h>
+#include <osmocom/gsm/gsm48_ie.h>
+#include <osmocom/gsm/gsm_utils.h>
+#include <osmocom/gsm/protocol/gsm_23_003.h>
 
-#include "db.h"
-#include "hlr.h"
-#include "ctrl.h"
-#include "logging.h"
-#include "gsup_server.h"
-#include "gsup_router.h"
-#include "rand.h"
-#include "luop.h"
-#include "hlr_vty.h"
-#include "hlr_ussd.h"
+#include <osmocom/hlr/db.h>
+#include <osmocom/hlr/hlr.h>
+#include <osmocom/hlr/ctrl.h>
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/gsup_server.h>
+#include <osmocom/hlr/gsup_router.h>
+#include <osmocom/hlr/rand.h>
+#include <osmocom/hlr/luop.h>
+#include <osmocom/hlr/hlr_vty.h>
+#include <osmocom/hlr/hlr_ussd.h>
 
 struct hlr *g_hlr;
+static void *hlr_ctx = NULL;
 static int quit = 0;
 
 /* Trigger 'Insert Subscriber Data' messages to all connected GSUP clients.
@@ -54,16 +59,15 @@ static int quit = 0;
 void
 osmo_hlr_subscriber_update_notify(struct hlr_subscriber *subscr)
 {
-	/* FIXME: the below code can only be re-enabled after we make sure that an ISD
-	 * is only sent tot the currently serving VLR and/or SGSN (if there are any).
-	 * We cannot blindly flood the entire PLMN with this, as it would create subscriber
-	 * state in every VLR/SGSN out there, even those that have never seen the subscriber.
-	 * See https://osmocom.org/issues/3154 for details. */
-#if 0
         struct osmo_gsup_conn *co;
 
-	if (g_hlr->gs == NULL)
+	if (g_hlr->gs == NULL) {
+		LOGP(DLGSUP, LOGL_DEBUG,
+		     "IMSI %s: NOT Notifying peers of subscriber data change,"
+		     " there is no GSUP server\n",
+		     subscr->imsi);
 		return;
+	}
 
 	llist_for_each_entry(co, &g_hlr->gs->clients, list) {
 		struct osmo_gsup_message gsup = { };
@@ -72,20 +76,50 @@ osmo_hlr_subscriber_update_notify(struct hlr_subscriber *subscr)
 		struct msgb *msg_out;
 		uint8_t *peer;
 		int peer_len;
+		size_t peer_strlen;
+		const char *peer_compare;
 		enum osmo_gsup_cn_domain cn_domain;
 
-		if (co->supports_ps)
+		if (co->supports_ps) {
 			cn_domain = OSMO_GSUP_CN_DOMAIN_PS;
-		else if (co->supports_cs)
+			peer_compare = subscr->sgsn_number;
+		} else if (co->supports_cs) {
 			cn_domain = OSMO_GSUP_CN_DOMAIN_CS;
-		else {
-			/* We have not yet received a location update from this subscriber .*/
+			peer_compare = subscr->vlr_number;
+		} else {
+			/* We have not yet received a location update from this GSUP client.*/
 			continue;
 		}
 
+		peer_len = osmo_gsup_conn_ccm_get(co, &peer, IPAC_IDTAG_SERNR);
+		if (peer_len < 0) {
+			LOGP(DLGSUP, LOGL_ERROR,
+			       "IMSI='%s': cannot get peer name for connection %s:%u\n", subscr->imsi,
+			       co && co->conn && co->conn->server? co->conn->server->addr : "unset",
+			       co && co->conn && co->conn->server? co->conn->server->port : 0);
+			continue;
+		}
+
+		peer_strlen = strnlen((const char*)peer, peer_len);
+		if (strlen(peer_compare) != peer_strlen || strncmp(peer_compare, (const char *)peer, peer_len)) {
+			/* Mismatch. The subscriber is not subscribed with this GSUP client. */
+			/* I hope peer is always nul terminated... */
+			if (peer_strlen < peer_len)
+				LOGP(DLGSUP, LOGL_DEBUG,
+				     "IMSI %s: subscriber change: skipping %s peer %s\n",
+				     subscr->imsi, cn_domain == OSMO_GSUP_CN_DOMAIN_PS ? "PS" : "CS",
+				     osmo_quote_str((char*)peer, -1));
+			continue;
+		}
+
+		LOGP(DLGSUP, LOGL_DEBUG,
+		     "IMSI %s: subscriber change: notifying %s peer %s\n",
+		     subscr->imsi, cn_domain == OSMO_GSUP_CN_DOMAIN_PS ? "PS" : "CS",
+		     osmo_quote_str(peer_compare, -1));
+
 		if (osmo_gsup_create_insert_subscriber_data_msg(&gsup, subscr->imsi, subscr->msisdn, msisdn_enc,
 								sizeof(msisdn_enc), apn, sizeof(apn), cn_domain) != 0) {
-			LOGP(DMAIN, LOGL_ERROR,
+			LOGP(DLGSUP, LOGL_ERROR,
 			       "IMSI='%s': Cannot notify GSUP client; could not create gsup message "
 			       "for %s:%u\n", subscr->imsi,
 			       co && co->conn && co->conn->server? co->conn->server->addr : "unset",
@@ -94,9 +128,9 @@ osmo_hlr_subscriber_update_notify(struct hlr_subscriber *subscr)
 		}
 
 		/* Send ISD to MSC/SGSN */
-		msg_out = msgb_alloc_headroom(1024+16, 16, "GSUP ISD UPDATE");
+		msg_out = osmo_gsup_msgb_alloc("GSUP ISD UPDATE");
 		if (msg_out == NULL) {
-			LOGP(DMAIN, LOGL_ERROR,
+			LOGP(DLGSUP, LOGL_ERROR,
 			       "IMSI='%s': Cannot notify GSUP client; could not allocate msg buffer "
 			       "for %s:%u\n", subscr->imsi,
 			       co && co->conn && co->conn->server? co->conn->server->addr : "unset",
@@ -105,15 +139,6 @@ osmo_hlr_subscriber_update_notify(struct hlr_subscriber *subscr)
 		}
 		osmo_gsup_encode(msg_out, &gsup);
 
-		peer_len = osmo_gsup_conn_ccm_get(co, &peer, IPAC_IDTAG_SERNR);
-		if (peer_len < 0) {
-			LOGP(DMAIN, LOGL_ERROR,
-			       "IMSI='%s': cannot get peer name for connection %s:%u\n", subscr->imsi,
-			       co && co->conn && co->conn->server? co->conn->server->addr : "unset",
-			       co && co->conn && co->conn->server? co->conn->server->port : 0);
-			continue;
-		}
-
 		if (osmo_gsup_addr_send(g_hlr->gs, peer, peer_len, msg_out) < 0) {
 			LOGP(DMAIN, LOGL_ERROR,
 			       "IMSI='%s': Cannot notify GSUP client; send operation failed "
@@ -123,7 +148,78 @@ osmo_hlr_subscriber_update_notify(struct hlr_subscriber *subscr)
 			continue;
 		}
 	}
-#endif
+}
+
+static int generate_new_msisdn(char *msisdn, const char *imsi, unsigned int len)
+{
+	int i, j, rc;
+	uint8_t rand_buf[GSM23003_MSISDN_MAX_DIGITS];
+
+	OSMO_ASSERT(len <= sizeof(rand_buf));
+
+	/* Generate a random unique MSISDN (with retry) */
+	for (i = 0; i < 10; i++) {
+		/* Get a random number (with retry) */
+		for (j = 0; j < 10; j++) {
+			rc = osmo_get_rand_id(rand_buf, len);
+			if (!rc)
+				break;
+		}
+		if (rc) {
+			LOGP(DMAIN, LOGL_ERROR, "IMSI='%s': Failed to generate new MSISDN, random number generator"
+						" failed (rc=%d)\n", imsi, rc);
+			return rc;
+		}
+
+		/* Shift 0x00 ... 0xff range to 30 ... 39 (ASCII numbers) */
+		for (j = 0; j < len; j++)
+			msisdn[j] = 48 + (rand_buf[j] % 10);
+		msisdn[j] = '\0';
+
+		/* Ensure there is no subscriber with such MSISDN */
+		if (db_subscr_exists_by_msisdn(g_hlr->dbc, msisdn) == -ENOENT)
+			return 0;
+	}
+
+	/* Failure */
+	LOGP(DMAIN, LOGL_ERROR, "IMSI='%s': Failed to generate a new MSISDN, consider increasing "
+				"the length for the automatically assigned MSISDNs "
+				"(see 'subscriber-create-on-demand' command)\n", imsi);
+	return -1;
+}
+
+static int subscr_create_on_demand(const char *imsi)
+{
+	char msisdn[GSM23003_MSISDN_MAX_DIGITS + 1];
+	int rc;
+	unsigned int rand_msisdn_len = g_hlr->subscr_create_on_demand_rand_msisdn_len;
+
+	if (!g_hlr->subscr_create_on_demand)
+		return -1;
+	if (db_subscr_exists_by_imsi(g_hlr->dbc, imsi) == 0)
+		return -1;
+	if (rand_msisdn_len && generate_new_msisdn(msisdn, imsi, rand_msisdn_len) != 0)
+		return -1;
+
+	LOGP(DMAIN, LOGL_INFO, "IMSI='%s': Creating subscriber on demand\n", imsi);
+	rc = db_subscr_create(g_hlr->dbc, imsi, g_hlr->subscr_create_on_demand_flags);
+	if (rc) {
+		LOGP(DMAIN, LOGL_ERROR, "Failed to create subscriber on demand (rc=%d): IMSI='%s'\n", rc, imsi);
+		return rc;
+	}
+
+	if (!rand_msisdn_len)
+		return 0;
+
+	/* Update MSISDN of the new (just allocated) subscriber */
+	rc = db_subscr_update_msisdn_by_imsi(g_hlr->dbc, imsi, msisdn);
+	if (rc) {
+		LOGP(DMAIN, LOGL_ERROR, "IMSI='%s': Failed to assign MSISDN='%s' (rc=%d)\n", imsi, msisdn, rc);
+		return rc;
+	}
+	LOGP(DMAIN, LOGL_INFO, "IMSI='%s': Successfully assigned MSISDN='%s'\n", imsi, msisdn);
+
+	return 0;
 }
 
 /***********************************************************************
@@ -137,16 +233,22 @@ static int rx_send_auth_info(struct osmo_gsup_conn *conn,
 {
 	struct osmo_gsup_message gsup_out;
 	struct msgb *msg_out;
+	bool separation_bit = false;
 	int rc;
 
+	subscr_create_on_demand(gsup->imsi);
+
 	/* initialize return message structure */
 	memset(&gsup_out, 0, sizeof(gsup_out));
 	memcpy(&gsup_out.imsi, &gsup->imsi, sizeof(gsup_out.imsi));
 
+	if (gsup->current_rat_type == OSMO_RAT_EUTRAN_SGS)
+		separation_bit = true;
+
 	rc = db_get_auc(dbc, gsup->imsi, conn->auc_3g_ind,
 			gsup_out.auth_vectors,
 			ARRAY_SIZE(gsup_out.auth_vectors),
-			gsup->rand, gsup->auts);
+			gsup->rand, gsup->auts, separation_bit);
 	if (rc <= 0) {
 		gsup_out.message_type = OSMO_GSUP_MSGT_SEND_AUTH_INFO_ERROR;
 		switch (rc) {
@@ -173,7 +275,7 @@ static int rx_send_auth_info(struct osmo_gsup_conn *conn,
 		gsup_out.num_auth_vectors = rc;
 	}
 
-	msg_out = msgb_alloc_headroom(1024+16, 16, "GSUP AUC response");
+	msg_out = osmo_gsup_msgb_alloc("GSUP AUC response");
 	osmo_gsup_encode(msg_out, &gsup_out);
 	return osmo_gsup_conn_send(conn, msg_out);
 }
@@ -242,29 +344,35 @@ void lu_op_rx_gsup(struct lu_operation *luop,
 static int rx_upd_loc_req(struct osmo_gsup_conn *conn,
 			  const struct osmo_gsup_message *gsup)
 {
+	struct hlr_subscriber *subscr;
 	struct lu_operation *luop = lu_op_alloc_conn(conn);
 	if (!luop) {
 		LOGP(DMAIN, LOGL_ERROR, "LU REQ from conn without addr?\n");
 		return -EINVAL;
 	}
 
+	subscr = &luop->subscr;
+
 	lu_op_statechg(luop, LU_S_LU_RECEIVED);
 
-	if (gsup->cn_domain == OSMO_GSUP_CN_DOMAIN_CS)
+	switch (gsup->cn_domain) {
+	case OSMO_GSUP_CN_DOMAIN_CS:
 		conn->supports_cs = true;
-	if (gsup->cn_domain == OSMO_GSUP_CN_DOMAIN_PS) {
-		conn->supports_ps = true;
-		luop->is_ps = true;
-	} else {
+		break;
+	default:
 		/* The client didn't send a CN_DOMAIN IE; assume packet-switched in
 		 * accordance with the GSUP spec in osmo-hlr's user manual (section
 		 * 11.6.15 "CN Domain" says "if no CN Domain IE is present within
 		 * a request, the PS Domain is assumed." */
+	case OSMO_GSUP_CN_DOMAIN_PS:
 		conn->supports_ps = true;
 		luop->is_ps = true;
+		break;
 	}
 	llist_add(&luop->list, &g_lu_ops);
 
+	subscr_create_on_demand(gsup->imsi);
+
 	/* Roughly follwing "Process Update_Location_HLR" of TS 09.02 */
 
 	/* check if subscriber is known at all */
@@ -297,13 +405,21 @@ static int rx_upd_loc_req(struct osmo_gsup_conn *conn,
 		lu_op_tx_cancel_old(luop);
 	} else
 #endif
-	{
-		/* TODO: Subscriber allowed to roam in PLMN? */
-		/* TODO: Update RoutingInfo */
-		/* TODO: Reset Flag MS Purged (cs/ps) */
-		/* TODO: Control_Tracing_HLR / Control_Tracing_HLR_with_SGSN */
-		lu_op_tx_insert_subscr_data(luop);
-	}
+
+	/* Store the VLR / SGSN number with the subscriber, so we know where it was last seen. */
+	LOGP(DAUC, LOGL_DEBUG, "IMSI='%s': storing %s = %s\n",
+	     subscr->imsi, luop->is_ps ? "SGSN number" : "VLR number",
+	     osmo_quote_str((const char*)luop->peer, -1));
+	if (db_subscr_lu(g_hlr->dbc, subscr->id, (const char *)luop->peer, luop->is_ps))
+		LOGP(DAUC, LOGL_ERROR, "IMSI='%s': Cannot update %s in the database\n",
+		     subscr->imsi, luop->is_ps ? "SGSN number" : "VLR number");
+
+	/* TODO: Subscriber allowed to roam in PLMN? */
+	/* TODO: Update RoutingInfo */
+	/* TODO: Reset Flag MS Purged (cs/ps) */
+	/* TODO: Control_Tracing_HLR / Control_Tracing_HLR_with_SGSN */
+	lu_op_tx_insert_subscr_data(luop);
+
 	return 0;
 }
 
@@ -339,7 +455,7 @@ static int rx_purge_ms_req(struct osmo_gsup_conn *conn,
 		gsup_reply.cause = GMM_CAUSE_NET_FAIL;
 	}
 
-	msg_out = msgb_alloc_headroom(1024+16, 16, "GSUP AUC response");
+	msg_out = osmo_gsup_msgb_alloc("GSUP AUC response");
 	osmo_gsup_encode(msg_out, &gsup_reply);
 	return osmo_gsup_conn_send(conn, msg_out);
 }
@@ -347,41 +463,178 @@ static int rx_purge_ms_req(struct osmo_gsup_conn *conn,
 static int gsup_send_err_reply(struct osmo_gsup_conn *conn, const char *imsi,
 				enum osmo_gsup_message_type type_in, uint8_t err_cause)
 {
-	int type_err = osmo_gsup_get_err_msg_type(type_in);
+	int type_err = OSMO_GSUP_TO_MSGT_ERROR(type_in);
 	struct osmo_gsup_message gsup_reply = {0};
 	struct msgb *msg_out;
 
-	if (type_err < 0) {
-		LOGP(DMAIN, LOGL_ERROR, "unable to determine error response for %s\n",
-			osmo_gsup_message_type_name(type_in));
-		return type_err;
-	}
-
 	OSMO_STRLCPY_ARRAY(gsup_reply.imsi, imsi);
 	gsup_reply.message_type = type_err;
 	gsup_reply.cause = err_cause;
-	msg_out = msgb_alloc_headroom(1024+16, 16, "GSUP ERR response");
-	OSMO_ASSERT(msg_out);
+	msg_out = osmo_gsup_msgb_alloc("GSUP ERR response");
 	osmo_gsup_encode(msg_out, &gsup_reply);
 	LOGP(DMAIN, LOGL_NOTICE, "Tx %s\n", osmo_gsup_message_type_name(type_err));
 	return osmo_gsup_conn_send(conn, msg_out);
 }
 
+static int rx_check_imei_req(struct osmo_gsup_conn *conn, const struct osmo_gsup_message *gsup)
+{
+	struct osmo_gsup_message gsup_reply = {0};
+	struct msgb *msg_out;
+	char imei[GSM23003_IMEI_NUM_DIGITS_NO_CHK+1] = {0};
+	int rc;
+
+	/* Require IMEI */
+	if (!gsup->imei_enc) {
+		LOGP(DMAIN, LOGL_ERROR, "%s: missing IMEI\n", gsup->imsi);
+		gsup_send_err_reply(conn, gsup->imsi, gsup->message_type, GMM_CAUSE_INV_MAND_INFO);
+		return -1;
+	}
+
+	/* Decode IMEI (fails if IMEI is too long) */
+	rc = gsm48_decode_bcd_number2(imei, sizeof(imei), gsup->imei_enc, gsup->imei_enc_len, 0);
+	if (rc < 0) {
+		LOGP(DMAIN, LOGL_ERROR, "%s: failed to decode IMEI (rc: %i)\n", gsup->imsi, rc);
+		gsup_send_err_reply(conn, gsup->imsi, gsup->message_type, GMM_CAUSE_INV_MAND_INFO);
+		return -1;
+	}
+
+	/* Check if IMEI is too short */
+	if (strlen(imei) != GSM23003_IMEI_NUM_DIGITS_NO_CHK) {
+		LOGP(DMAIN, LOGL_ERROR, "%s: wrong encoded IMEI length (IMEI: '%s', %lu, %i)\n", gsup->imsi, imei,
+		     strlen(imei), GSM23003_IMEI_NUM_DIGITS_NO_CHK);
+		gsup_send_err_reply(conn, gsup->imsi, gsup->message_type, GMM_CAUSE_INV_MAND_INFO);
+		return -1;
+	}
+
+	subscr_create_on_demand(gsup->imsi);
+
+	/* Save in DB if desired */
+	if (g_hlr->store_imei) {
+		LOGP(DAUC, LOGL_DEBUG, "IMSI='%s': storing IMEI = %s\n", gsup->imsi, imei);
+		if (db_subscr_update_imei_by_imsi(g_hlr->dbc, gsup->imsi, imei) < 0) {
+			gsup_send_err_reply(conn, gsup->imsi, gsup->message_type, GMM_CAUSE_INV_MAND_INFO);
+			return -1;
+		}
+	} else {
+		/* Check if subscriber exists and print IMEI */
+		LOGP(DMAIN, LOGL_INFO, "IMSI='%s': has IMEI = %s (consider setting 'store-imei')\n", gsup->imsi, imei);
+		struct hlr_subscriber subscr;
+		if (db_subscr_get_by_imsi(g_hlr->dbc, gsup->imsi, &subscr) < 0) {
+			gsup_send_err_reply(conn, gsup->imsi, gsup->message_type, GMM_CAUSE_INV_MAND_INFO);
+			return -1;
+		}
+	}
+
+	/* Accept all IMEIs */
+	gsup_reply.imei_result = OSMO_GSUP_IMEI_RESULT_ACK;
+	gsup_reply.message_type = OSMO_GSUP_MSGT_CHECK_IMEI_RESULT;
+	msg_out = osmo_gsup_msgb_alloc("GSUP Check_IMEI response");
+	memcpy(gsup_reply.imsi, gsup->imsi, sizeof(gsup_reply.imsi));
+	osmo_gsup_encode(msg_out, &gsup_reply);
+	return osmo_gsup_conn_send(conn, msg_out);
+}
+
+static char namebuf[255];
+#define LOGP_GSUP_FWD(gsup, level, fmt, args ...) \
+	LOGP(DMAIN, level, "Forward %s (class=%s, IMSI=%s, %s->%s): " fmt, \
+	     osmo_gsup_message_type_name(gsup->message_type), \
+	     osmo_gsup_message_class_name(gsup->message_class), \
+	     gsup->imsi, \
+	     osmo_quote_str((const char *)gsup->source_name, gsup->source_name_len), \
+	     osmo_quote_str_buf2(namebuf, sizeof(namebuf), (const char *)gsup->destination_name, gsup->destination_name_len), \
+	     ## args)
+
+static int read_cb_forward(struct osmo_gsup_conn *conn, struct msgb *msg, const struct osmo_gsup_message *gsup)
+{
+	int ret = -EINVAL;
+	struct osmo_gsup_message *gsup_err;
+
+	/* FIXME: it would be better if the msgb never were deallocated immediately by osmo_gsup_addr_send(), which a
+	 * select-loop volatile talloc context could facilitate. Then we would still be able to access gsup-> members
+	 * (pointing into the msgb) even after sending failed, and we wouldn't need to copy this data before sending: */
+	/* Prepare error message (before IEs get deallocated) */
+	gsup_err = talloc_zero(hlr_ctx, struct osmo_gsup_message);
+	OSMO_STRLCPY_ARRAY(gsup_err->imsi, gsup->imsi);
+	gsup_err->message_class = gsup->message_class;
+	gsup_err->destination_name = talloc_memdup(gsup_err, gsup->destination_name, gsup->destination_name_len);
+	gsup_err->destination_name_len = gsup->destination_name_len;
+	gsup_err->message_type = gsup->message_type;
+	gsup_err->session_state = gsup->session_state;
+	gsup_err->session_id = gsup->session_id;
+	gsup_err->source_name = talloc_memdup(gsup_err, gsup->source_name, gsup->source_name_len);
+	gsup_err->source_name_len = gsup->source_name_len;
+
+	/* Check for routing IEs */
+	if (!gsup->source_name || !gsup->source_name_len || !gsup->destination_name || !gsup->destination_name_len) {
+		LOGP_GSUP_FWD(gsup, LOGL_ERROR, "missing routing IEs\n");
+		goto end;
+	}
+
+	/* Verify source name (e.g. "MSC-00-00-00-00-00-00") */
+	if (gsup_route_find(conn->server, gsup->source_name, gsup->source_name_len) != conn) {
+		LOGP_GSUP_FWD(gsup, LOGL_ERROR, "mismatching source name\n");
+		goto end;
+	}
+
+	/* Forward message without re-encoding (so we don't remove unknown IEs) */
+	LOGP_GSUP_FWD(gsup, LOGL_INFO, "checks passed, forwarding\n");
+
+	/* Remove incoming IPA header to be able to prepend an outgoing IPA header */
+	msgb_pull_to_l2(msg);
+	ret = osmo_gsup_addr_send(g_hlr->gs, gsup->destination_name, gsup->destination_name_len, msg);
+	/* AT THIS POINT, THE msg MAY BE DEALLOCATED and the data like gsup->imsi, gsup->source_name etc may all be
+	 * invalid and cause segfaults. */
+	msg = NULL;
+	gsup = NULL;
+	if (ret == -ENODEV)
+		LOGP_GSUP_FWD(gsup_err, LOGL_ERROR, "destination not connected\n");
+	else if (ret)
+		LOGP_GSUP_FWD(gsup_err, LOGL_ERROR, "unknown error %i\n", ret);
+
+end:
+	/* Send error back to source */
+	if (ret) {
+		struct msgb *msg_err = osmo_gsup_msgb_alloc("GSUP forward ERR response");
+		gsup_err->message_type = OSMO_GSUP_MSGT_E_ROUTING_ERROR;
+		osmo_gsup_encode(msg_err, gsup_err);
+		LOGP_GSUP_FWD(gsup_err, LOGL_NOTICE, "Tx %s\n", osmo_gsup_message_type_name(gsup_err->message_type));
+		osmo_gsup_conn_send(conn, msg_err);
+	}
+	talloc_free(gsup_err);
+	if (msg)
+		msgb_free(msg);
+	return ret;
+}
+
 static int read_cb(struct osmo_gsup_conn *conn, struct msgb *msg)
 {
 	static struct osmo_gsup_message gsup;
 	int rc;
 
+	if (!msgb_l2(msg) || !msgb_l2len(msg)) {
+		LOGP(DMAIN, LOGL_ERROR, "missing or empty L2 data\n");
+		msgb_free(msg);
+		return -EINVAL;
+	}
+
 	rc = osmo_gsup_decode(msgb_l2(msg), msgb_l2len(msg), &gsup);
 	if (rc < 0) {
 		LOGP(DMAIN, LOGL_ERROR, "error in GSUP decode: %d\n", rc);
+		msgb_free(msg);
 		return rc;
 	}
 
 	/* 3GPP TS 23.003 Section 2.2 clearly states that an IMSI with less than 5
 	 * digits is impossible.  Even 5 digits is a highly theoretical case */
-	if (strlen(gsup.imsi) < 5)
-		return gsup_send_err_reply(conn, gsup.imsi, gsup.message_type, GMM_CAUSE_INV_MAND_INFO);
+	if (strlen(gsup.imsi) < 5) { /* TODO: move this check to libosmogsm/gsup.c? */
+		LOGP(DMAIN, LOGL_ERROR, "IMSI too short: %s\n", osmo_quote_str(gsup.imsi, -1));
+		gsup_send_err_reply(conn, gsup.imsi, gsup.message_type, GMM_CAUSE_INV_MAND_INFO);
+		msgb_free(msg);
+		return -EINVAL;
+	}
+
+	if (gsup.destination_name_len)
+		return read_cb_forward(conn, msg, &gsup);
 
 	switch (gsup.message_type) {
 	/* requests sent to us */
@@ -427,6 +680,9 @@ static int read_cb(struct osmo_gsup_conn *conn, struct msgb *msg)
 			lu_op_rx_gsup(luop, &gsup);
 		}
 		break;
+	case OSMO_GSUP_MSGT_CHECK_IMEI_REQUEST:
+		rx_check_imei_req(conn, &gsup);
+		break;
 	default:
 		LOGP(DMAIN, LOGL_DEBUG, "Unhandled GSUP message type %s\n",
 		     osmo_gsup_message_type_name(gsup.message_type));
@@ -451,6 +707,8 @@ static void print_help()
 	printf("  -s --disable-color         Do not print ANSI colors in the log\n");
 	printf("  -T --timestamp             Prefix every log line with a timestamp.\n");
 	printf("  -e --log-level number      Set a global loglevel.\n");
+	printf("  -U --db-upgrade            Allow HLR database schema upgrades.\n");
+	printf("  -C --db-check              Quit after opening (and upgrading) the database.\n");
 	printf("  -V --version               Print the version of OsmoHLR.\n");
 }
 
@@ -458,10 +716,13 @@ static struct {
 	const char *config_file;
 	const char *db_file;
 	bool daemonize;
+	bool db_upgrade;
+	bool db_check;
 } cmdline_opts = {
 	.config_file = "osmo-hlr.cfg",
-	.db_file = "hlr.db",
+	.db_file = NULL,
 	.daemonize = false,
+	.db_upgrade = false,
 };
 
 static void handle_options(int argc, char **argv)
@@ -477,11 +738,13 @@ static void handle_options(int argc, char **argv)
 			{"disable-color", 0, 0, 's'},
 			{"log-level", 1, 0, 'e'},
 			{"timestamp", 0, 0, 'T'},
+			{"db-upgrade", 0, 0, 'U' },
+			{"db-check", 0, 0, 'C' },
 			{"version", 0, 0, 'V' },
 			{0, 0, 0, 0}
 		};
 
-		c = getopt_long(argc, argv, "hc:l:d:Dse:TV",
+		c = getopt_long(argc, argv, "hc:l:d:Dse:TUV",
 				long_options, &option_index);
 		if (c == -1)
 			break;
@@ -512,6 +775,12 @@ static void handle_options(int argc, char **argv)
 		case 'T':
 			log_set_print_timestamp(osmo_stderr_target, 1);
 			break;
+		case 'U':
+			cmdline_opts.db_upgrade = true;
+			break;
+		case 'C':
+			cmdline_opts.db_check = true;
+			break;
 		case 'V':
 			print_version(1);
 			exit(0);
@@ -523,15 +792,19 @@ static void handle_options(int argc, char **argv)
 			break;
 		}
 	}
-}
 
-static void *hlr_ctx = NULL;
+	if (argc > optind) {
+		fprintf(stderr, "Unsupported positional arguments on command line\n");
+		exit(2);
+	}
+}
 
 static void signal_hdlr(int signal)
 {
 	switch (signal) {
+	case SIGTERM:
 	case SIGINT:
-		LOGP(DMAIN, LOGL_NOTICE, "Terminating due to SIGINT\n");
+		LOGP(DMAIN, LOGL_NOTICE, "Terminating due to signal=%d\n", signal);
 		quit++;
 		break;
 	case SIGUSR1:
@@ -571,6 +844,10 @@ int main(int argc, char **argv)
 	INIT_LLIST_HEAD(&g_hlr->iuse_list);
 	INIT_LLIST_HEAD(&g_hlr->ss_sessions);
 	INIT_LLIST_HEAD(&g_hlr->ussd_routes);
+	g_hlr->db_file_path = talloc_strdup(g_hlr, HLR_DEFAULT_DB_FILE_PATH);
+
+	/* Init default (call independent) SS session guard timeout value */
+	g_hlr->ncss_guard_timeout = NCSS_GUARD_TIMEOUT_DEFAULT;
 
 	rc = osmo_init_logging2(hlr_ctx, &hlr_log_info);
 	if (rc < 0) {
@@ -578,10 +855,11 @@ int main(int argc, char **argv)
 		exit(1);
 	}
 
+	osmo_stats_init(hlr_ctx);
 	vty_init(&vty_info);
 	ctrl_vty_init(hlr_ctx);
 	handle_options(argc, argv);
-	hlr_vty_init(&hlr_log_info);
+	hlr_vty_init();
 
 	rc = vty_read_config_file(cmdline_opts.config_file, NULL);
 	if (rc < 0) {
@@ -591,12 +869,6 @@ int main(int argc, char **argv)
 		return rc;
 	}
 
-	/* start telnet after reading config for vty_get_bind_addr() */
-	rc = telnet_init_dynif(hlr_ctx, NULL, vty_get_bind_addr(),
-			       OSMO_VTY_PORT_HLR);
-	if (rc < 0)
-		return rc;
-
 	LOGP(DMAIN, LOGL_NOTICE, "hlr starting\n");
 
 	rc = rand_init();
@@ -605,12 +877,32 @@ int main(int argc, char **argv)
 		exit(1);
 	}
 
-	g_hlr->dbc = db_open(hlr_ctx, cmdline_opts.db_file, true);
+	if (cmdline_opts.db_file)
+		osmo_talloc_replace_string(g_hlr, &g_hlr->db_file_path, cmdline_opts.db_file);
+
+	g_hlr->dbc = db_open(hlr_ctx, g_hlr->db_file_path, true, cmdline_opts.db_upgrade);
 	if (!g_hlr->dbc) {
-		LOGP(DMAIN, LOGL_FATAL, "Error opening database\n");
+		LOGP(DMAIN, LOGL_FATAL, "Error opening database %s\n", osmo_quote_str(g_hlr->db_file_path, -1));
 		exit(1);
 	}
 
+	if (cmdline_opts.db_check) {
+		LOGP(DMAIN, LOGL_NOTICE, "Cmdline option --db-check: Database was opened successfully, quitting.\n");
+		db_close(g_hlr->dbc);
+		log_fini();
+		talloc_free(hlr_ctx);
+		talloc_free(tall_vty_ctx);
+		talloc_disable_null_tracking();
+		exit(0);
+	}
+
+	/* start telnet after reading config for vty_get_bind_addr() */
+	rc = telnet_init_dynif(hlr_ctx, NULL, vty_get_bind_addr(),
+			       OSMO_VTY_PORT_HLR);
+	if (rc < 0)
+		return rc;
+
+
 	g_hlr->gs = osmo_gsup_server_create(hlr_ctx, g_hlr->gsup_bind_addr, OSMO_GSUP_PORT,
 					    read_cb, &g_lu_ops, g_hlr);
 	if (!g_hlr->gs) {
@@ -623,6 +915,7 @@ int main(int argc, char **argv)
 
 	osmo_init_ignore_signals();
 	signal(SIGINT, &signal_hdlr);
+	signal(SIGTERM, &signal_hdlr);
 	signal(SIGUSR1, &signal_hdlr);
 
 	if (cmdline_opts.daemonize) {

src/hlr_db_tool.c

@@ -29,9 +29,9 @@
 #include <osmocom/core/logging.h>
 #include <osmocom/core/application.h>
 
-#include "logging.h"
-#include "db.h"
-#include "rand.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/db.h>
+#include <osmocom/hlr/rand.h>
 
 struct hlr_db_tool_ctx {
 	/* DB context */
@@ -44,8 +44,10 @@ static struct {
 	const char *db_file;
 	bool bootstrap;
 	const char *import_nitb_db;
+	bool db_upgrade;
 } cmdline_opts = {
 	.db_file = "hlr.db",
+	.db_upgrade = false,
 };
 
 static void print_help()
@@ -59,6 +61,7 @@ static void print_help()
 	printf("  -s --disable-color         Do not print ANSI colors in the log\n");
 	printf("  -T --timestamp             Prefix every log line with a timestamp.\n");
 	printf("  -e --log-level number      Set a global loglevel.\n");
+	printf("  -U --db-upgrade            Allow HLR database schema upgrades.\n");
 	printf("  -V --version               Print the version of OsmoHLR-db-tool.\n");
 	printf("\n");
 	printf("Commands:\n");
@@ -96,11 +99,12 @@ static void handle_options(int argc, char **argv)
 			{"disable-color", 0, 0, 's'},
 			{"timestamp", 0, 0, 'T'},
 			{"log-level", 1, 0, 'e'},
+			{"db-upgrade", 0, 0, 'U' },
 			{"version", 0, 0, 'V' },
 			{0, 0, 0, 0}
 		};
 
-		c = getopt_long(argc, argv, "hl:d:sTe:V",
+		c = getopt_long(argc, argv, "hl:d:sTe:UV",
 				long_options, &option_index);
 		if (c == -1)
 			break;
@@ -124,6 +128,9 @@ static void handle_options(int argc, char **argv)
 		case 'e':
 			log_set_log_level(osmo_stderr_target, atoi(optarg));
 			break;
+		case 'U':
+			cmdline_opts.db_upgrade = true;
+			break;
 		case 'V':
 			print_version(1);
 			exit(EXIT_SUCCESS);
@@ -293,9 +300,9 @@ void import_nitb_subscr(sqlite3 *nitb_db, sqlite3_stmt *stmt)
 
 	imsi = sqlite3_column_int64(stmt, 0);
 
-	snprintf(imsi_str, sizeof(imsi_str), "%"PRId64, imsi);
+	snprintf(imsi_str, sizeof(imsi_str), "%" PRId64, imsi);
 
-	rc = db_subscr_create(dbc, imsi_str);
+	rc = db_subscr_create(dbc, imsi_str, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS);
 	if (rc < 0) {
 		LOGP(DDB, LOGL_ERROR, "OsmoNITB DB import to %s: failed to create IMSI %s: %d: %s\n",
 		     dbc->fname,
@@ -409,7 +416,7 @@ int main(int argc, char **argv)
 		exit(EXIT_FAILURE);
 	}
 
-	g_hlr_db_tool_ctx->dbc = db_open(g_hlr_db_tool_ctx, cmdline_opts.db_file, true);
+	g_hlr_db_tool_ctx->dbc = db_open(g_hlr_db_tool_ctx, cmdline_opts.db_file, true, cmdline_opts.db_upgrade);
 	if (!g_hlr_db_tool_ctx->dbc) {
 		LOGP(DMAIN, LOGL_FATAL, "Error opening database\n");
 		exit(EXIT_FAILURE);

src/hlr_ussd.c

@@ -29,11 +29,12 @@
 #include <string.h>
 #include <errno.h>
 
-#include "hlr.h"
-#include "hlr_ussd.h"
-#include "gsup_server.h"
-#include "gsup_router.h"
-#include "logging.h"
+#include <osmocom/hlr/hlr.h>
+#include <osmocom/hlr/hlr_ussd.h>
+#include <osmocom/hlr/gsup_server.h>
+#include <osmocom/hlr/gsup_router.h>
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/db.h>
 
 /***********************************************************************
  * core data structures expressing config from VTY
@@ -126,8 +127,10 @@ static struct hlr_ussd_route *ussd_route_lookup_7bit(struct hlr *hlr, const char
 	struct hlr_ussd_route *rt;
 	llist_for_each_entry(rt, &hlr->ussd_routes, list) {
 		if (!strncmp(ussd_code, rt->prefix, strlen(rt->prefix))) {
-			LOGP(DSS, LOGL_DEBUG, "Found EUSE %s (prefix %s) for USSD Code '%s'\n",
-				rt->u.euse->name, rt->prefix, ussd_code);
+			LOGP(DSS, LOGL_DEBUG, "Found %s '%s' (prefix '%s') for USSD "
+				"Code '%s'\n", rt->is_external ? "EUSE" : "IUSE",
+				rt->is_external ? rt->u.euse->name : rt->u.iuse->name,
+				rt->prefix, ussd_code);
 			return rt;
 		}
 	}
@@ -147,7 +150,7 @@ struct ss_session {
 	/* link us to hlr->ss_sessions */
 	struct llist_head list;
 	/* imsi of this session */
-	char imsi[GSM23003_IMSI_MAX_DIGITS+2];
+	char imsi[OSMO_IMSI_BUF_SIZE];
 	/* ID of this session (unique per IMSI) */
 	uint32_t session_id;
 	/* state of the session */
@@ -164,6 +167,12 @@ struct ss_session {
 		const struct hlr_iuse *iuse;
 	} u;
 
+	/* subscriber's vlr_number
+	 * MO USSD: originating MSC's vlr_number
+	 * MT USSD: looked up once per session and cached here */
+	uint8_t *vlr_number;
+	size_t vlr_number_len;
+
 	/* we don't keep a pointer to the osmo_gsup_{route,conn} towards the MSC/VLR here,
 	 * as this might change during inter-VLR hand-over, and we simply look-up the serving MSC/VLR
 	 * every time we receive an USSD component from the EUSE */
@@ -206,11 +215,11 @@ struct ss_session *ss_session_alloc(struct hlr *hlr, const char *imsi, uint32_t
 
 	OSMO_STRLCPY_ARRAY(ss->imsi, imsi);
 	ss->session_id = session_id;
+
+	/* Schedule self-destruction timer */
 	osmo_timer_setup(&ss->timeout, ss_session_timeout, ss);
-	/* NOTE: The timeout is currently global and not refreshed with subsequent messages
-	 * within the SS/USSD session.  So 30s after the initial SS message, the session will
-	 * timeout! */
-	osmo_timer_schedule(&ss->timeout, 30, 0);
+	if (g_hlr->ncss_guard_timeout > 0)
+		osmo_timer_schedule(&ss->timeout, g_hlr->ncss_guard_timeout, 0);
 
 	llist_add_tail(&ss->list, &hlr->ss_sessions);
 	return ss;
@@ -220,6 +229,35 @@ struct ss_session *ss_session_alloc(struct hlr *hlr, const char *imsi, uint32_t
  * handling functions for encoding SS messages + wrapping them in GSUP
  ***********************************************************************/
 
+/* Resolve the target MSC by ss->imsi and send GSUP message. */
+static int ss_gsup_send(struct ss_session *ss, struct osmo_gsup_server *gs, struct msgb *msg)
+{
+	struct hlr_subscriber subscr = {};
+	int rc;
+
+	/* Use vlr_number as looked up by the caller, or look up now. */
+	if (!ss->vlr_number) {
+		rc = db_subscr_get_by_imsi(g_hlr->dbc, ss->imsi, &subscr);
+		if (rc < 0) {
+			LOGPSS(ss, LOGL_ERROR, "Cannot find subscriber, cannot route GSUP message\n");
+			msgb_free(msg);
+			return -EINVAL;
+		}
+		ss->vlr_number = (uint8_t *)talloc_strdup(ss, subscr.vlr_number);
+		ss->vlr_number_len = strlen(subscr.vlr_number) + 1;
+	}
+
+	/* Check for empty string (all vlr_number strings end in "\0", because otherwise gsup_route_find() fails) */
+	if (ss->vlr_number_len == 1) {
+		LOGPSS(ss, LOGL_ERROR, "Cannot send GSUP message, no VLR number stored for subscriber\n");
+		msgb_free(msg);
+		return -EINVAL;
+	}
+
+	LOGPSS(ss, LOGL_DEBUG, "Tx SS/USSD to VLR %s\n", osmo_quote_str((char *)ss->vlr_number, ss->vlr_number_len));
+	return osmo_gsup_addr_send(gs, ss->vlr_number, ss->vlr_number_len, msg);
+}
+
 static int ss_tx_to_ms(struct ss_session *ss, enum osmo_gsup_message_type gsup_msg_type,
 			bool final, struct msgb *ss_msg)
 
@@ -239,13 +277,12 @@ static int ss_tx_to_ms(struct ss_session *ss, enum osmo_gsup_message_type gsup_m
 		resp.ss_info_len = msgb_length(ss_msg);
 	}
 
-	resp_msg = gsm0480_msgb_alloc_name(__func__);
+	resp_msg = msgb_alloc_headroom(4000, 64, __func__);
 	OSMO_ASSERT(resp_msg);
 	osmo_gsup_encode(resp_msg, &resp);
 	msgb_free(ss_msg);
 
-	/* FIXME: resolve this based on the database vlr_addr */
-	return osmo_gsup_addr_send(g_hlr->gs, (uint8_t *)"MSC-00-00-00-00-00-00", 22, resp_msg);
+	return ss_gsup_send(ss, g_hlr->gs, resp_msg);
 }
 
 #if 0
@@ -280,7 +317,7 @@ static int ss_tx_ussd_7bit(struct ss_session *ss, bool final, uint8_t invoke_id,
  * Internal USSD Handlers
  ***********************************************************************/
 
-#include "db.h"
+#include <osmocom/hlr/db.h>
 
 static int handle_ussd_own_msisdn(struct osmo_gsup_conn *conn, struct ss_session *ss,
 				  const struct osmo_gsup_message *gsup, const struct ss_request *req)
@@ -295,15 +332,15 @@ static int handle_ussd_own_msisdn(struct osmo_gsup_conn *conn, struct ss_session
 		if (strlen(subscr.msisdn) == 0)
 			snprintf(buf, sizeof(buf), "You have no MSISDN!");
 		else
-			snprintf(buf, sizeof(buf), "Your extension is %s\r", subscr.msisdn);
+			snprintf(buf, sizeof(buf), "Your extension is %s", subscr.msisdn);
 		ss_tx_ussd_7bit(ss, true, req->invoke_id, buf);
 		break;
 	case -ENOENT:
-		ss_tx_error(ss, true, GSM0480_ERR_CODE_UNKNOWN_SUBSCRIBER);
+		ss_tx_error(ss, req->invoke_id, GSM0480_ERR_CODE_UNKNOWN_SUBSCRIBER);
 		break;
 	case -EIO:
 	default:
-		ss_tx_error(ss, true, GSM0480_ERR_CODE_SYSTEM_FAILURE);
+		ss_tx_error(ss, req->invoke_id, GSM0480_ERR_CODE_SYSTEM_FAILURE);
 		break;
 	}
 	return 0;
@@ -313,7 +350,7 @@ static int handle_ussd_own_imsi(struct osmo_gsup_conn *conn, struct ss_session *
 				const struct osmo_gsup_message *gsup, const struct ss_request *req)
 {
 	char buf[GSM0480_USSD_7BIT_STRING_LEN+1];
-	snprintf(buf, sizeof(buf), "Your IMSI is %s!\n", ss->imsi);
+	snprintf(buf, sizeof(buf), "Your IMSI is %s", ss->imsi);
 	ss_tx_ussd_7bit(ss, true, req->invoke_id, buf);
 	return 0;
 }
@@ -397,8 +434,15 @@ static int handle_ss(struct ss_session *ss, const struct osmo_gsup_message *gsup
 
 	LOGPSS(ss, LOGL_INFO, "SS CompType=%s, OpCode=%s\n",
 		gsm0480_comp_type_name(comp_type), gsm0480_op_code_name(req->opcode));
-	/* FIXME */
-	return 0;
+
+	/**
+	 * FIXME: As we don't store any SS related information
+	 * (e.g. call forwarding preferences) in the database,
+	 * we don't handle "structured" SS requests at all.
+	 */
+	LOGPSS(ss, LOGL_NOTICE, "Structured SS requests are not supported, rejecting...\n");
+	ss_tx_error(ss, req->invoke_id, GSM0480_ERR_CODE_FACILITY_NOT_SUPPORTED);
+	return -ENOTSUP;
 }
 
 /* Handle a USSD GSUP message for a given SS Session received from VLR or EUSE */
@@ -420,12 +464,10 @@ static int handle_ussd(struct osmo_gsup_conn *conn, struct ss_session *ss,
 	}
 
 	if (is_euse_originated) {
-		msg_out = msgb_alloc_headroom(1024+16, 16, "GSUP USSD FW");
-		OSMO_ASSERT(msg_out);
+		msg_out = osmo_gsup_msgb_alloc("GSUP USSD FW");
 		/* Received from EUSE, Forward to VLR */
 		osmo_gsup_encode(msg_out, gsup);
-		/* FIXME: resolve this based on the database vlr_addr */
-		osmo_gsup_addr_send(conn->server, (uint8_t *)"MSC-00-00-00-00-00-00", 22, msg_out);
+		ss_gsup_send(ss, conn->server, msg_out);
 	} else {
 		/* Received from VLR (MS) */
 		if (ss->is_external) {
@@ -438,14 +480,15 @@ static int handle_ussd(struct osmo_gsup_conn *conn, struct ss_session *ss,
 				LOGPSS(ss, LOGL_ERROR, "Cannot find conn for EUSE %s\n", addr);
 				ss_tx_error(ss, req->invoke_id, GSM0480_ERR_CODE_SYSTEM_FAILURE);
 			} else {
-				msg_out = msgb_alloc_headroom(1024+16, 16, "GSUP USSD FW");
-				OSMO_ASSERT(msg_out);
+				msg_out = osmo_gsup_msgb_alloc("GSUP USSD FW");
 				osmo_gsup_encode(msg_out, gsup);
 				osmo_gsup_conn_send(conn, msg_out);
 			}
 		} else {
 			/* Handle internally */
 			ss->u.iuse->handle_ussd(conn, ss, gsup, req);
+			/* Release session immediately */
+			ss_session_free(ss);
 		}
 	}
 
@@ -460,6 +503,7 @@ int rx_proc_ss_req(struct osmo_gsup_conn *conn, const struct osmo_gsup_message *
 	struct hlr *hlr = conn->server->priv;
 	struct ss_session *ss;
 	struct ss_request req = {0};
+	struct gsup_route *gsup_rt;
 
 	LOGP(DSS, LOGL_DEBUG, "%s/0x%08x: Process SS (%s)\n", gsup->imsi, gsup->session_id,
 		osmo_gsup_session_state_name(gsup->session_state));
@@ -473,6 +517,11 @@ int rx_proc_ss_req(struct osmo_gsup_conn *conn, const struct osmo_gsup_message *
 			/* FIXME: Send a Reject component? */
 			goto out_err;
 		}
+	} else if (gsup->session_state != OSMO_GSUP_SESSION_STATE_END) {
+		LOGP(DSS, LOGL_ERROR, "%s/0x%082x: Missing SS payload for '%s'\n",
+		     gsup->imsi, gsup->session_id,
+		     osmo_gsup_session_state_name(gsup->session_state));
+		goto out_err;
 	}
 
 	switch (gsup->session_state) {
@@ -489,6 +538,20 @@ int rx_proc_ss_req(struct osmo_gsup_conn *conn, const struct osmo_gsup_message *
 				gsup->imsi, gsup->session_id);
 			goto out_err;
 		}
+		/* Get IPA name from VLR conn and save as ss->vlr_number */
+		if (!conn_is_euse(conn)) {
+			gsup_rt = gsup_route_find_by_conn(conn);
+			if (gsup_rt) {
+				ss->vlr_number = (uint8_t *)talloc_strdup(ss, (const char *)gsup_rt->addr);
+				ss->vlr_number_len = strlen((const char *)gsup_rt->addr) + 1;
+				LOGPSS(ss, LOGL_DEBUG, "Destination IPA name retrieved from GSUP route: %s\n",
+				       osmo_quote_str((const char *)ss->vlr_number, ss->vlr_number_len));
+			} else {
+				LOGPSS(ss, LOGL_NOTICE, "Could not find GSUP route, therefore can't set the destination"
+							" IPA name. We'll try to look it up later, but this should not"
+							" have happened.\n");
+			}
+		}
 		if (ss_op_is_ussd(req.opcode)) {
 			if (conn_is_euse(conn)) {
 				/* EUSE->VLR: MT USSD. EUSE is known ('conn'), VLR is to be resolved */
@@ -505,6 +568,11 @@ int rx_proc_ss_req(struct osmo_gsup_conn *conn, const struct osmo_gsup_message *
 						ss->is_external = false;
 						ss->u.iuse = rt->u.iuse;
 					}
+				} else {
+					if (hlr->euse_default) {
+						ss->is_external = true;
+						ss->u.euse = hlr->euse_default;
+					}
 				}
 			}
 			/* dispatch unstructured SS to routing */
@@ -521,6 +589,11 @@ int rx_proc_ss_req(struct osmo_gsup_conn *conn, const struct osmo_gsup_message *
 				gsup->imsi, gsup->session_id);
 			goto out_err;
 		}
+
+		/* Reschedule self-destruction timer */
+		if (g_hlr->ncss_guard_timeout > 0)
+			osmo_timer_schedule(&ss->timeout, g_hlr->ncss_guard_timeout, 0);
+
 		if (ss_op_is_ussd(req.opcode)) {
 			/* dispatch unstructured SS to routing */
 			handle_ussd(conn, ss, gsup, &req);
@@ -536,13 +609,18 @@ int rx_proc_ss_req(struct osmo_gsup_conn *conn, const struct osmo_gsup_message *
 				gsup->imsi, gsup->session_id);
 			goto out_err;
 		}
-		if (ss_op_is_ussd(req.opcode)) {
-			/* dispatch unstructured SS to routing */
-			handle_ussd(conn, ss, gsup, &req);
-		} else {
-			/* dispatch non-call SS to internal code */
-			handle_ss(ss, gsup, &req);
+
+		/* SS payload is optional for END */
+		if (gsup->ss_info && gsup->ss_info_len) {
+			if (ss_op_is_ussd(req.opcode)) {
+				/* dispatch unstructured SS to routing */
+				handle_ussd(conn, ss, gsup, &req);
+			} else {
+				/* dispatch non-call SS to internal code */
+				handle_ss(ss, gsup, &req);
+			}
 		}
+
 		ss_session_free(ss);
 		break;
 	default:

src/hlr_vty.c

@@ -27,15 +27,18 @@
 
 #include <osmocom/core/talloc.h>
 #include <osmocom/vty/vty.h>
+#include <osmocom/vty/stats.h>
 #include <osmocom/vty/command.h>
 #include <osmocom/vty/logging.h>
 #include <osmocom/vty/misc.h>
 #include <osmocom/abis/ipa.h>
 
-#include "hlr.h"
-#include "hlr_vty.h"
-#include "hlr_vty_subscr.h"
-#include "gsup_server.h"
+#include <osmocom/hlr/db.h>
+#include <osmocom/hlr/hlr.h>
+#include <osmocom/hlr/hlr_vty.h>
+#include <osmocom/hlr/hlr_vty_subscr.h>
+#include <osmocom/hlr/hlr_ussd.h>
+#include <osmocom/hlr/gsup_server.h>
 
 struct cmd_node hlr_node = {
 	HLR_NODE,
@@ -70,6 +73,27 @@ DEFUN(cfg_gsup,
 static int config_write_hlr(struct vty *vty)
 {
 	vty_out(vty, "hlr%s", VTY_NEWLINE);
+	if (g_hlr->store_imei)
+		vty_out(vty, " store-imei%s", VTY_NEWLINE);
+	if (g_hlr->db_file_path && strcmp(g_hlr->db_file_path, HLR_DEFAULT_DB_FILE_PATH))
+		vty_out(vty, " database %s%s", g_hlr->db_file_path, VTY_NEWLINE);
+	if (g_hlr->subscr_create_on_demand) {
+		const char *flags_str = "none";
+		uint8_t flags = g_hlr->subscr_create_on_demand_flags;
+		unsigned int rand_msisdn_len = g_hlr->subscr_create_on_demand_rand_msisdn_len;
+
+		if ((flags & DB_SUBSCR_FLAG_NAM_CS) && (flags & DB_SUBSCR_FLAG_NAM_PS))
+			flags_str = "cs+ps";
+		else if (flags & DB_SUBSCR_FLAG_NAM_CS)
+			flags_str = "cs";
+		else if (flags & DB_SUBSCR_FLAG_NAM_PS)
+			flags_str = "ps";
+
+		if (rand_msisdn_len)
+			vty_out(vty, " subscriber-create-on-demand %i %s%s", rand_msisdn_len, flags_str, VTY_NEWLINE);
+		else
+			vty_out(vty, " subscriber-create-on-demand no-msisdn %s%s", flags_str, VTY_NEWLINE);
+	}
 	return CMD_SUCCESS;
 }
 
@@ -126,7 +150,7 @@ DEFUN(cfg_hlr_gsup_bind_ip,
  * USSD Entity
  ***********************************************************************/
 
-#include "hlr_ussd.h"
+#include <osmocom/hlr/hlr_ussd.h>
 
 #define USSD_STR "USSD Configuration\n"
 #define UROUTE_STR "Routing Configuration\n"
@@ -193,7 +217,13 @@ DEFUN(cfg_ussd_defaultroute, cfg_ussd_defaultroute_cmd,
 	USSD_STR "Configure default-route for all USSD to unknown destinations\n"
 	EXT_STR)
 {
-	struct hlr_euse *euse = euse_find(g_hlr, argv[0]);
+	struct hlr_euse *euse;
+
+	euse = euse_find(g_hlr, argv[0]);
+	if (!euse) {
+		vty_out(vty, "%% Cannot find EUSE %s%s", argv[0], VTY_NEWLINE);
+		return CMD_WARNING;
+	}
 
 	if (g_hlr->euse_default != euse) {
 		vty_out(vty, "Switching default route from %s to %s%s",
@@ -214,6 +244,15 @@ DEFUN(cfg_ussd_no_defaultroute, cfg_ussd_no_defaultroute_cmd,
 	return CMD_SUCCESS;
 }
 
+DEFUN(cfg_database, cfg_database_cmd,
+	"database PATH",
+	"Set the path to the HLR database file\n"
+	"Relative or absolute file system path to the database file (default is '" HLR_DEFAULT_DB_FILE_PATH "')\n")
+{
+	osmo_talloc_replace_string(g_hlr, &g_hlr->db_file_path, argv[0]);
+	return CMD_SUCCESS;
+}
+
 struct cmd_node euse_node = {
 	EUSE_NODE,
 	"%s(config-hlr-euse)# ",
@@ -282,9 +321,75 @@ static int config_write_euse(struct vty *vty)
 	if (g_hlr->euse_default)
 		vty_out(vty, " ussd default-route external %s%s", g_hlr->euse_default->name, VTY_NEWLINE);
 
+	if (g_hlr->ncss_guard_timeout != NCSS_GUARD_TIMEOUT_DEFAULT)
+		vty_out(vty, " ncss-guard-timeout %i%s",
+			g_hlr->ncss_guard_timeout, VTY_NEWLINE);
+
 	return 0;
 }
 
+DEFUN(cfg_ncss_guard_timeout, cfg_ncss_guard_timeout_cmd,
+	"ncss-guard-timeout <0-255>",
+	"Set guard timer for NCSS (call independent SS) session activity\n"
+	"Guard timer value (sec.), or 0 to disable")
+{
+	g_hlr->ncss_guard_timeout = atoi(argv[0]);
+	return CMD_SUCCESS;
+}
+
+DEFUN(cfg_store_imei, cfg_store_imei_cmd,
+	"store-imei",
+	"Save the IMEI in the database when receiving Check IMEI requests. Note that an MSC does not necessarily send"
+	" Check IMEI requests (for OsmoMSC, you may want to set 'check-imei-rqd 1').")
+{
+	g_hlr->store_imei = true;
+	return CMD_SUCCESS;
+}
+
+DEFUN(cfg_no_store_imei, cfg_no_store_imei_cmd,
+	"no store-imei",
+	"Do not save the IMEI in the database, when receiving Check IMEI requests.")
+{
+	g_hlr->store_imei = false;
+	return CMD_SUCCESS;
+}
+
+DEFUN(cfg_subscr_create_on_demand, cfg_subscr_create_on_demand_cmd,
+	"subscriber-create-on-demand (no-msisdn|<3-15>) (none|cs|ps|cs+ps)",
+	"Make a new record when a subscriber is first seen.\n"
+	"Do not automatically assign MSISDN.\n"
+	"Length of an automatically assigned MSISDN.\n"
+	"Do not allow any NAM (Network Access Mode) by default.\n"
+	"Allow access to circuit switched NAM by default.\n"
+	"Allow access to packet switched NAM by default.\n"
+	"Allow access to circuit and packet switched NAM by default.\n")
+{
+	unsigned int rand_msisdn_len = 0;
+	uint8_t flags = 0x00;
+
+	if (strcmp(argv[0], "no-msisdn") != 0)
+		rand_msisdn_len = atoi(argv[0]);
+
+	if (strstr(argv[1], "cs"))
+		flags |= DB_SUBSCR_FLAG_NAM_CS;
+	if (strstr(argv[1], "ps"))
+		flags |= DB_SUBSCR_FLAG_NAM_PS;
+
+	g_hlr->subscr_create_on_demand = true;
+	g_hlr->subscr_create_on_demand_rand_msisdn_len = rand_msisdn_len;
+	g_hlr->subscr_create_on_demand_flags = flags;
+
+	return CMD_SUCCESS;
+}
+
+DEFUN(cfg_no_subscr_create_on_demand, cfg_no_subscr_create_on_demand_cmd,
+	"no subscriber-create-on-demand",
+	"Do not make a new record when a subscriber is first seen.\n")
+{
+	g_hlr->subscr_create_on_demand = false;
+	return CMD_SUCCESS;
+}
+
 /***********************************************************************
  * Common Code
  ***********************************************************************/
@@ -324,10 +429,11 @@ int hlr_vty_is_config_node(struct vty *vty, int node)
 	}
 }
 
-void hlr_vty_init(const struct log_info *cat)
+void hlr_vty_init(void)
 {
-	logging_vty_add_cmds(cat);
+	logging_vty_add_cmds();
 	osmo_talloc_vty_add_cmds();
+	osmo_stats_vty_add_cmds();
 
 	install_element_ve(&show_gsup_conn_cmd);
 
@@ -339,6 +445,8 @@ void hlr_vty_init(const struct log_info *cat)
 
 	install_element(GSUP_NODE, &cfg_hlr_gsup_bind_ip_cmd);
 
+	install_element(HLR_NODE, &cfg_database_cmd);
+
 	install_element(HLR_NODE, &cfg_euse_cmd);
 	install_element(HLR_NODE, &cfg_no_euse_cmd);
 	install_node(&euse_node, config_write_euse);
@@ -347,6 +455,11 @@ void hlr_vty_init(const struct log_info *cat)
 	install_element(HLR_NODE, &cfg_ussd_no_route_pfx_cmd);
 	install_element(HLR_NODE, &cfg_ussd_defaultroute_cmd);
 	install_element(HLR_NODE, &cfg_ussd_no_defaultroute_cmd);
+	install_element(HLR_NODE, &cfg_ncss_guard_timeout_cmd);
+	install_element(HLR_NODE, &cfg_store_imei_cmd);
+	install_element(HLR_NODE, &cfg_no_store_imei_cmd);
+	install_element(HLR_NODE, &cfg_subscr_create_on_demand_cmd);
+	install_element(HLR_NODE, &cfg_no_subscr_create_on_demand_cmd);
 
 	hlr_vty_subscriber_init();
 }

src/hlr_vty_subscr.c

@@ -20,19 +20,42 @@
 #include <inttypes.h>
 #include <string.h>
 #include <errno.h>
+#include <sys/types.h>
+#include <time.h>
 
 #include <osmocom/gsm/gsm23003.h>
 #include <osmocom/vty/vty.h>
 #include <osmocom/vty/command.h>
 #include <osmocom/core/utils.h>
 
-#include "hlr.h"
-#include "db.h"
+#include <osmocom/hlr/hlr.h>
+#include <osmocom/hlr/db.h>
 
 struct vty;
 
 #define hexdump_buf(buf) osmo_hexdump_nospc((void*)buf, sizeof(buf))
 
+static char *
+get_datestr(const time_t *t, char *datebuf)
+{
+	char *p, *s = ctime_r(t, datebuf);
+
+	/* Strip trailing newline. */
+	p = strchr(s, '\n');
+	if (p)
+		*p = '\0';
+	return s;
+}
+
+static void dump_last_lu_seen(struct vty *vty, const char *domain_label, time_t last_lu_seen)
+{
+	char datebuf[26]; /* for ctime_r(3) */
+	if (!last_lu_seen)
+		return;
+	vty_out(vty, "    last LU seen on %s: %s UTC%s", domain_label, get_datestr(&last_lu_seen, datebuf),
+		VTY_NEWLINE);
+}
+
 static void subscr_dump_full_vty(struct vty *vty, struct hlr_subscriber *subscr)
 {
 	int rc;
@@ -43,6 +66,15 @@ static void subscr_dump_full_vty(struct vty *vty, struct hlr_subscriber *subscr)
 
 	vty_out(vty, "    IMSI: %s%s", *subscr->imsi ? subscr->imsi : "none", VTY_NEWLINE);
 	vty_out(vty, "    MSISDN: %s%s", *subscr->msisdn ? subscr->msisdn : "none", VTY_NEWLINE);
+
+	if (*subscr->imei) {
+		char checksum = osmo_luhn(subscr->imei, 14);
+		if (checksum == -EINVAL)
+			vty_out(vty, "    IMEI: %s (INVALID LENGTH!)%s", subscr->imei, VTY_NEWLINE);
+		else
+			vty_out(vty, "    IMEI: %s%c%s", subscr->imei, checksum, VTY_NEWLINE);
+	}
+
 	if (*subscr->vlr_number)
 		vty_out(vty, "    VLR number: %s%s", subscr->vlr_number, VTY_NEWLINE);
 	if (*subscr->sgsn_number)
@@ -63,6 +95,8 @@ static void subscr_dump_full_vty(struct vty *vty, struct hlr_subscriber *subscr)
 		vty_out(vty, "    PS disabled%s", VTY_NEWLINE);
 	if (subscr->ms_purged_ps)
 		vty_out(vty, "    PS purged%s", VTY_NEWLINE);
+	dump_last_lu_seen(vty, "CS", subscr->last_lu_seen);
+	dump_last_lu_seen(vty, "PS", subscr->last_lu_seen_ps);
 
 	if (!*subscr->imsi)
 		return;
@@ -114,6 +148,7 @@ static void subscr_dump_full_vty(struct vty *vty, struct hlr_subscriber *subscr)
 
 static int get_subscr_by_argv(struct vty *vty, const char *type, const char *id, struct hlr_subscriber *subscr)
 {
+	char imei_buf[GSM23003_IMEI_NUM_DIGITS_NO_CHK+1];
 	int rc = -1;
 	if (strcmp(type, "imsi") == 0)
 		rc = db_subscr_get_by_imsi(g_hlr->dbc, id, subscr);
@@ -121,6 +156,17 @@ static int get_subscr_by_argv(struct vty *vty, const char *type, const char *id,
 		rc = db_subscr_get_by_msisdn(g_hlr->dbc, id, subscr);
 	else if (strcmp(type, "id") == 0)
 		rc = db_subscr_get_by_id(g_hlr->dbc, atoll(id), subscr);
+	else if (strcmp(type, "imei") == 0) {
+		/* Verify IMEI with checksum digit */
+		if (osmo_imei_str_valid(id, true)) {
+			/* Cut the checksum off */
+			osmo_strlcpy(imei_buf, id, sizeof(imei_buf));
+			id = imei_buf;
+			vty_out(vty, "%% Checksum validated and stripped for search: imei = '%s'%s", id,
+				VTY_NEWLINE);
+		}
+		rc = db_subscr_get_by_imei(g_hlr->dbc, id, subscr);
+	}
 	if (rc)
 		vty_out(vty, "%% No subscriber for %s = '%s'%s",
 			type, id, VTY_NEWLINE);
@@ -130,18 +176,20 @@ static int get_subscr_by_argv(struct vty *vty, const char *type, const char *id,
 #define SUBSCR_CMD "subscriber "
 #define SUBSCR_CMD_HELP "Subscriber management commands\n"
 
-#define SUBSCR_ID "(imsi|msisdn|id) IDENT "
+#define SUBSCR_ID "(imsi|msisdn|id|imei) IDENT"
 #define SUBSCR_ID_HELP \
 	"Identify subscriber by IMSI\n" \
 	"Identify subscriber by MSISDN (phone number)\n" \
 	"Identify subscriber by database ID\n" \
-	"IMSI/MSISDN/ID of the subscriber\n"
+	"Identify subscriber by IMEI\n" \
+	"IMSI/MSISDN/ID/IMEI of the subscriber\n"
 
-#define SUBSCR 		SUBSCR_CMD SUBSCR_ID
+#define SUBSCR 		SUBSCR_CMD SUBSCR_ID " "
 #define SUBSCR_HELP	SUBSCR_CMD_HELP SUBSCR_ID_HELP
 
 #define SUBSCR_UPDATE		SUBSCR "update "
 #define SUBSCR_UPDATE_HELP	SUBSCR_HELP "Set or update subscriber data\n"
+#define SUBSCR_MSISDN_HELP	"Set MSISDN (phone number) of the subscriber\n"
 
 DEFUN(subscriber_show,
       subscriber_show_cmd,
@@ -159,6 +207,10 @@ DEFUN(subscriber_show,
 	return CMD_SUCCESS;
 }
 
+ALIAS(subscriber_show, show_subscriber_cmd,
+      "show " SUBSCR_CMD SUBSCR_ID,
+      SHOW_STR SUBSCR_CMD_HELP SUBSCR_ID_HELP);
+
 DEFUN(subscriber_create,
       subscriber_create_cmd,
       SUBSCR_CMD "imsi IDENT create",
@@ -176,7 +228,7 @@ DEFUN(subscriber_create,
 		return CMD_WARNING;
 	}
 
-	rc = db_subscr_create(g_hlr->dbc, imsi);
+	rc = db_subscr_create(g_hlr->dbc, imsi, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS);
 
 	if (rc) {
 		if (rc == -EEXIST)
@@ -224,9 +276,9 @@ DEFUN(subscriber_delete,
 
 DEFUN(subscriber_msisdn,
       subscriber_msisdn_cmd,
-      SUBSCR_UPDATE "msisdn MSISDN",
-      SUBSCR_UPDATE_HELP
-      "Set MSISDN (phone number) of the subscriber\n"
+      SUBSCR_UPDATE "msisdn (none|MSISDN)",
+      SUBSCR_UPDATE_HELP SUBSCR_MSISDN_HELP
+      "Remove MSISDN (phone number)\n"
       "New MSISDN (phone number)\n")
 {
 	struct hlr_subscriber subscr;
@@ -234,15 +286,19 @@ DEFUN(subscriber_msisdn,
 	const char *id = argv[1];
 	const char *msisdn = argv[2];
 
-	if (strlen(msisdn) > sizeof(subscr.msisdn) - 1) {
-		vty_out(vty, "%% MSISDN is too long, max. %zu characters are allowed%s",
-			sizeof(subscr.msisdn)-1, VTY_NEWLINE);
-		return CMD_WARNING;
-	}
+	if (strcmp(msisdn, "none") == 0)
+		msisdn = NULL;
+	else {
+		if (strlen(msisdn) > sizeof(subscr.msisdn) - 1) {
+			vty_out(vty, "%% MSISDN is too long, max. %zu characters are allowed%s",
+				sizeof(subscr.msisdn)-1, VTY_NEWLINE);
+			return CMD_WARNING;
+		}
 
-	if (!osmo_msisdn_str_valid(msisdn)) {
-		vty_out(vty, "%% MSISDN invalid: '%s'%s", msisdn, VTY_NEWLINE);
-		return CMD_WARNING;
+		if (!osmo_msisdn_str_valid(msisdn)) {
+			vty_out(vty, "%% MSISDN invalid: '%s'%s", msisdn, VTY_NEWLINE);
+			return CMD_WARNING;
+		}
 	}
 
 	if (get_subscr_by_argv(vty, id_type, id, &subscr))
@@ -254,11 +310,18 @@ DEFUN(subscriber_msisdn,
 		return CMD_WARNING;
 	}
 
-	vty_out(vty, "%% Updated subscriber IMSI='%s' to MSISDN='%s'%s",
-		subscr.imsi, msisdn, VTY_NEWLINE);
+	if (msisdn) {
+		vty_out(vty, "%% Updated subscriber IMSI='%s' to MSISDN='%s'%s",
+			subscr.imsi, msisdn, VTY_NEWLINE);
+
+		if (db_subscr_get_by_msisdn(g_hlr->dbc, msisdn, &subscr) == 0)
+			osmo_hlr_subscriber_update_notify(&subscr);
+	} else {
+		vty_out(vty, "%% Updated subscriber IMSI='%s': removed MSISDN%s",
+			subscr.imsi, VTY_NEWLINE);
 
-	if (db_subscr_get_by_msisdn(g_hlr->dbc, msisdn, &subscr) == 0)
 		osmo_hlr_subscriber_update_notify(&subscr);
+	}
 
 	return CMD_SUCCESS;
 }
@@ -475,9 +538,134 @@ DEFUN(subscriber_aud3g,
 	return CMD_SUCCESS;
 }
 
+DEFUN(subscriber_aud3g_xor,
+      subscriber_aud3g_xor_cmd,
+      SUBSCR_UPDATE "aud3g xor k K"
+      " [ind-bitlen] [<0-28>]",
+      SUBSCR_UPDATE_HELP
+      "Set UMTS authentication data (3G, and 2G with UMTS AKA)\n"
+      "Use XOR algorithm\n"
+      "Set Encryption Key K\n" "K as 32 hexadecimal characters\n"
+      "Set IND bit length\n" "IND bit length value (default: 5)\n")
+{
+	struct hlr_subscriber subscr;
+	int minlen = 0;
+	int maxlen = 0;
+	int rc;
+	const char *id_type = argv[0];
+	const char *id = argv[1];
+	const char *k = argv[2];
+	int ind_bitlen = argc > 4? atoi(argv[4]) : 5;
+	struct sub_auth_data_str aud3g = {
+		.type = OSMO_AUTH_TYPE_UMTS,
+		.u.umts = {
+			.k = k,
+			.opc_is_op = 0,
+			.opc = "00000000000000000000000000000000",
+			.ind_bitlen = ind_bitlen,
+		},
+	};
+
+	if (!auth_algo_parse("xor", &aud3g.algo, &minlen, &maxlen)) {
+		vty_out(vty, "%% Unknown auth algorithm: '%s'%s", "xor", VTY_NEWLINE);
+		return CMD_WARNING;
+	}
+
+	if (!is_hexkey_valid(vty, "K", aud3g.u.umts.k, minlen, maxlen))
+		return CMD_WARNING;
+
+	if (get_subscr_by_argv(vty, id_type, id, &subscr))
+		return CMD_WARNING;
+
+	rc = db_subscr_update_aud_by_id(g_hlr->dbc, subscr.id, &aud3g);
+
+	if (rc) {
+		vty_out(vty, "%% Error: cannot set 3G auth data for IMSI='%s'%s",
+			subscr.imsi, VTY_NEWLINE);
+		return CMD_WARNING;
+	}
+	return CMD_SUCCESS;
+}
+
+DEFUN(subscriber_imei,
+      subscriber_imei_cmd,
+      SUBSCR_UPDATE "imei (none|IMEI)",
+      SUBSCR_UPDATE_HELP
+      "Set IMEI of the subscriber (normally populated from MSC, no need to set this manually)\n"
+      "Forget IMEI\n"
+      "Set IMEI (use for debug only!)\n")
+{
+	struct hlr_subscriber subscr;
+	const char *id_type = argv[0];
+	const char *id = argv[1];
+	const char *imei = argv[2];
+	char imei_buf[GSM23003_IMEI_NUM_DIGITS_NO_CHK+1];
+
+	if (strcmp(imei, "none") == 0)
+		imei = NULL;
+	else {
+		/* Verify IMEI with checksum digit */
+		if (osmo_imei_str_valid(imei, true)) {
+			/* Cut the checksum off */
+			osmo_strlcpy(imei_buf, imei, sizeof(imei_buf));
+			imei = imei_buf;
+		} else if (!osmo_imei_str_valid(imei, false)) {
+			vty_out(vty, "%% IMEI invalid: '%s'%s", imei, VTY_NEWLINE);
+			return CMD_WARNING;
+		}
+	}
+
+	if (get_subscr_by_argv(vty, id_type, id, &subscr))
+		return CMD_WARNING;
+
+	if (db_subscr_update_imei_by_imsi(g_hlr->dbc, subscr.imsi, imei)) {
+		vty_out(vty, "%% Error: cannot update IMEI for subscriber IMSI='%s'%s",
+			subscr.imsi, VTY_NEWLINE);
+		return CMD_WARNING;
+	}
+
+	if (imei)
+		vty_out(vty, "%% Updated subscriber IMSI='%s' to IMEI='%s'%s",
+			subscr.imsi, imei, VTY_NEWLINE);
+	else
+		vty_out(vty, "%% Updated subscriber IMSI='%s': removed IMEI%s",
+			subscr.imsi, VTY_NEWLINE);
+
+	return CMD_SUCCESS;
+}
+
+DEFUN(subscriber_nam,
+      subscriber_nam_cmd,
+      SUBSCR_UPDATE "network-access-mode (none|cs|ps|cs+ps)",
+      SUBSCR_UPDATE_HELP
+      "Set Network Access Mode (NAM) of the subscriber\n"
+      "Do not allow access to circuit switched or packet switched services\n"
+      "Allow access to circuit switched services only\n"
+      "Allow access to packet switched services only\n"
+      "Allow access to both circuit and packet switched services\n")
+{
+	struct hlr_subscriber subscr;
+	const char *id_type = argv[0];
+	const char *id = argv[1];
+	bool nam_cs = strstr(argv[2], "cs");
+	bool nam_ps = strstr(argv[2], "ps");
+
+	if (get_subscr_by_argv(vty, id_type, id, &subscr))
+		return CMD_WARNING;
+
+	if (nam_cs != subscr.nam_cs)
+		hlr_subscr_nam(g_hlr, &subscr, nam_cs, 0);
+	if (nam_ps != subscr.nam_ps)
+		hlr_subscr_nam(g_hlr, &subscr, nam_ps, 1);
+
+	return CMD_SUCCESS;
+}
+
+
 void hlr_vty_subscriber_init(void)
 {
 	install_element_ve(&subscriber_show_cmd);
+	install_element_ve(&show_subscriber_cmd);
 	install_element(ENABLE_NODE, &subscriber_create_cmd);
 	install_element(ENABLE_NODE, &subscriber_delete_cmd);
 	install_element(ENABLE_NODE, &subscriber_msisdn_cmd);
@@ -485,4 +673,7 @@ void hlr_vty_subscriber_init(void)
 	install_element(ENABLE_NODE, &subscriber_aud2g_cmd);
 	install_element(ENABLE_NODE, &subscriber_no_aud3g_cmd);
 	install_element(ENABLE_NODE, &subscriber_aud3g_cmd);
+	install_element(ENABLE_NODE, &subscriber_aud3g_xor_cmd);
+	install_element(ENABLE_NODE, &subscriber_imei_cmd);
+	install_element(ENABLE_NODE, &subscriber_nam_cmd);
 }

src/logging.c

@@ -1,5 +1,5 @@
 #include <osmocom/core/utils.h>
-#include "logging.h"
+#include <osmocom/hlr/logging.h>
 
 const struct log_info_cat hlr_log_info_cat[] = {
 	[DMAIN] = {

src/luop.c

@@ -28,10 +28,10 @@
 #include <osmocom/gsm/gsup.h>
 #include <osmocom/gsm/apn.h>
 
-#include "gsup_server.h"
-#include "gsup_router.h"
-#include "logging.h"
-#include "luop.h"
+#include <osmocom/hlr/gsup_server.h>
+#include <osmocom/hlr/gsup_router.h>
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/luop.h>
 
 const struct value_string lu_state_names[] = {
 	{ LU_S_NULL,			"NULL" },
@@ -50,8 +50,7 @@ static void _luop_tx_gsup(struct lu_operation *luop,
 {
 	struct msgb *msg_out;
 
-	msg_out = msgb_alloc_headroom(1024+16, 16, "GSUP LUOP");
-	OSMO_ASSERT(msg_out);
+	msg_out = osmo_gsup_msgb_alloc("GSUP LUOP");
 	osmo_gsup_encode(msg_out, gsup);
 
 	osmo_gsup_addr_send(luop->gsup_server, luop->peer,

src/osmo-euse-demo.c

@@ -44,7 +44,7 @@
 
 #include <osmocom/gsupclient/gsup_client.h>
 
-#include "logging.h"
+#include <osmocom/hlr/logging.h>
 
 static struct osmo_gsup_client *g_gc;
 

tests/Makefile.am

@@ -2,7 +2,7 @@ SUBDIRS = \
 	auc \
 	gsup_server \
 	db \
-	gsup \
+	db_upgrade \
 	$(NULL)
 
 # The `:;' works around a Bash 3.2 bug when the output is not writeable.
@@ -44,20 +44,27 @@ python-tests:
 # don't run vty and ctrl tests concurrently so that the ports don't conflict
 	$(MAKE) vty-test
 	$(MAKE) ctrl-test
+	$(MAKE) db-upgrade-equivalence-test
+else
+python-tests:
+	echo "Not running python-based external tests (determined at configure-time)"
+endif
 
 VTY_TEST_DB = hlr_vty_test.db
+VTY_TEST ?= *.vty
 
 # To update the VTY script from current application behavior,
 # pass -u to vty_script_runner.py by doing:
 #   make vty-test U=-u
 vty-test:
 	-rm -f $(VTY_TEST_DB)
+	sqlite3 $(VTY_TEST_DB) < $(top_srcdir)/sql/hlr.sql
+	sqlite3 $(VTY_TEST_DB) < $(srcdir)/test_subscriber.vty.sql
 	osmo_verify_transcript_vty.py -v \
 		-n OsmoHLR -p 4258 \
 		-r "$(top_builddir)/src/osmo-hlr -c $(top_srcdir)/doc/examples/osmo-hlr.cfg -l $(VTY_TEST_DB)" \
-		$(U) $(srcdir)/*.vty
-	-rm -f $(VTY_TEST_DB)
-	-rm $(VTY_TEST_DB)-*
+		$(U) $(srcdir)/$(VTY_TEST)
+	-rm -f $(VTY_TEST_DB) $(VTY_TEST_DB)-*
 
 CTRL_TEST_DB = hlr_ctrl_test.db
 
@@ -75,10 +82,8 @@ ctrl-test:
 	-rm -f $(CTRL_TEST_DB)
 	-rm $(CTRL_TEST_DB)-*
 
-else
-python-tests:
-	echo "Not running python-based tests (determined at configure-time)"
-endif
+db-upgrade-equivalence-test:
+	$(MAKE) -C db_upgrade upgrade-equivalence-test
 
 check-local: atconfig $(TESTSUITE)
 	$(SHELL) '$(TESTSUITE)' $(TESTSUITEFLAGS)

tests/auc/Makefile.am

@@ -2,17 +2,18 @@ SUBDIRS = gen_ts_55_205_test_sets
 
 AM_CPPFLAGS = \
 	$(all_includes) \
-	-I$(top_srcdir)/src \
 	$(NULL)
 
 AM_CFLAGS = \
 	-Wall \
 	-ggdb3 \
+	-I$(top_srcdir)/include \
 	$(LIBOSMOCORE_CFLAGS) \
 	$(LIBOSMOGSM_CFLAGS) \
 	$(NULL)
 
 AM_LDFLAGS = \
+	-no-install \
 	$(NULL)
 
 EXTRA_DIST = \

tests/auc/auc_test.c

@@ -29,8 +29,8 @@
 
 #include <osmocom/crypt/auth.h>
 
-#include "logging.h"
-#include "auc.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/auc.h>
 
 #define comment_start() fprintf(stderr, "\n===== %s\n", __func__);
 #define comment_end() fprintf(stderr, "===== %s: SUCCESS\n\n", __func__);
@@ -454,6 +454,50 @@ static void test_gen_vectors_3g_only(void)
 	comment_end();
 }
 
+static void test_gen_vectors_3g_xor(void)
+{
+	struct osmo_sub_auth_data aud2g;
+	struct osmo_sub_auth_data aud3g;
+	struct osmo_auth_vector vec;
+	int rc;
+
+	comment_start();
+
+	aud2g = (struct osmo_sub_auth_data){ 0 };
+
+	aud3g = (struct osmo_sub_auth_data){
+		.type = OSMO_AUTH_TYPE_UMTS,
+		.algo = OSMO_AUTH_ALG_XOR,
+		.u.umts.sqn = 0,
+	};
+
+	osmo_hexparse("000102030405060708090a0b0c0d0e0f",
+		      aud3g.u.umts.k, sizeof(aud3g.u.umts.k));
+	osmo_hexparse("00000000000000000000000000000000",
+		      aud3g.u.umts.opc, sizeof(aud3g.u.umts.opc));
+	next_rand("b5039c57e4a75051551d1a390a71ce48", true);
+
+	vec = (struct osmo_auth_vector){ {0} };
+	VERBOSE_ASSERT(aud3g.u.umts.sqn, == 0, "%"PRIu64);
+	rc = auc_compute_vectors(&vec, 1, &aud2g, &aud3g, NULL, NULL);
+	VERBOSE_ASSERT(rc, == 1, "%d");
+	VERBOSE_ASSERT(aud3g.u.umts.sqn, == 0, "%"PRIu64);
+
+	VEC_IS(&vec,
+	       "  rand: b5039c57e4a75051551d1a390a71ce48\n"
+	       "  autn: 54e0a256565d0000b5029e54e0a25656\n"
+	       "  ck: 029e54e0a256565d141032067cc047b5\n"
+	       "  ik: 9e54e0a256565d141032067cc047b502\n"
+	       "  res: b5029e54e0a256565d141032067cc047\n"
+	       "  res_len: 10\n"
+	       "  kc: 98e880384887f9fe\n"
+	       "  sres: 0ec81877\n"
+	       "  auth_types: 03000000\n"
+	      );
+
+	comment_end();
+}
+
 void test_gen_vectors_bad_args()
 {
 	struct osmo_auth_vector vec;
@@ -622,6 +666,7 @@ int main(int argc, char **argv)
 	test_gen_vectors_2g_only();
 	test_gen_vectors_2g_plus_3g();
 	test_gen_vectors_3g_only();
+	test_gen_vectors_3g_xor();
 	test_gen_vectors_bad_args();
 
 	printf("Done\n");

tests/auc/auc_test.err

@@ -217,6 +217,29 @@ DAUC vector [2]: auth_types = 0x3
 ===== test_gen_vectors_3g_only: SUCCESS
 
 
+===== test_gen_vectors_3g_xor
+aud3g.u.umts.sqn == 0
+DAUC Computing 1 auth vector: 3G only (2G derived from 3G keys)
+DAUC 3G: k = 000102030405060708090a0b0c0d0e0f
+DAUC 3G: opc = 00000000000000000000000000000000
+DAUC 3G: for sqn ind 0, previous sqn was 0
+DAUC vector [0]: rand = b5039c57e4a75051551d1a390a71ce48
+DAUC vector [0]: sqn = 0
+DAUC vector [0]: autn = 54e0a256565d0000b5029e54e0a25656
+DAUC vector [0]: ck = 029e54e0a256565d141032067cc047b5
+DAUC vector [0]: ik = 9e54e0a256565d141032067cc047b502
+DAUC vector [0]: res = b5029e54e0a256565d141032067cc047
+DAUC vector [0]: res_len = 16
+DAUC vector [0]: deriving 2G from 3G
+DAUC vector [0]: kc = 98e880384887f9fe
+DAUC vector [0]: sres = 0ec81877
+DAUC vector [0]: auth_types = 0x3
+rc == 1
+aud3g.u.umts.sqn == 0
+vector matches expectations
+===== test_gen_vectors_3g_xor: SUCCESS
+
+
 ===== test_gen_vectors_bad_args
 
 - no auth data (a)

tests/auc/gen_ts_55_205_test_sets/main_template.c

@@ -33,8 +33,8 @@
 
 #include <osmocom/crypt/auth.h>
 
-#include "logging.h"
-#include "auc.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/auc.h>
 
 #define comment_start() fprintf(stderr, "\n===== %s\n", __func__);
 #define comment_end() fprintf(stderr, "===== %s: SUCCESS\n\n", __func__);

tests/auc/gen_ts_55_205_test_sets/pdftxt_2_c.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python
-# FIXME: use python3 once buildslaves are updated.
+#!/usr/bin/env python3
 # Convert test sets pasted from 3GPP TS 55.205 to C code.
  
 # (C) 2016 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>

tests/db/Makefile.am

@@ -1,7 +1,7 @@
 AM_CFLAGS = \
 	$(all_includes) \
-	-I$(top_srcdir)/src \
-	-I$(top_builddir)/src \
+	-I$(top_srcdir)/include \
+	-I$(top_builddir)/include \
 	-Wall \
 	-ggdb3 \
 	$(LIBOSMOCORE_CFLAGS) \
@@ -10,6 +10,10 @@ AM_CFLAGS = \
 	$(SQLITE3_CFLAGS) \
 	$(NULL)
 
+AM_LDFLAGS = \
+	-no-install \
+	$(NULL)
+
 EXTRA_DIST = \
 	db_test.ok \
 	db_test.err \
@@ -22,16 +26,20 @@ db_test_SOURCES = \
 	$(NULL)
 
 db_test_LDADD = \
-	$(top_srcdir)/src/db.c \
-	$(top_srcdir)/src/db_hlr.c \
-	$(top_srcdir)/src/db_auc.c \
-	$(top_srcdir)/src/logging.c \
+	$(top_builddir)/src/logging.o \
+	$(top_builddir)/src/db_auc.o \
+	$(top_builddir)/src/db_hlr.o \
+	$(top_builddir)/src/db.o \
 	$(LIBOSMOCORE_LIBS) \
 	$(LIBOSMOGSM_LIBS) \
 	$(LIBOSMOABIS_LIBS) \
 	$(SQLITE3_LIBS) \
 	$(NULL)
 
+if DB_SQLITE_DEBUG
+db_test_LDADD += $(top_builddir)/src/db_debug.o
+endif
+
 .PHONY: db_test.db update_exp manual manual-nonverbose manual-gdb
 db_test.db:
 	rm -f db_test.db

tests/db/db_test.c

@@ -27,8 +27,8 @@
 #include <osmocom/core/utils.h>
 #include <osmocom/core/logging.h>
 
-#include "db.h"
-#include "logging.h"
+#include <osmocom/hlr/db.h>
+#include <osmocom/hlr/logging.h>
 
 #define comment_start() fprintf(stderr, "\n===== %s\n", __func__);
 #define comment(fmt, args...) fprintf(stderr, "\n--- " fmt "\n\n", ## args);
@@ -51,7 +51,12 @@ static void _fill_invalid(void *dest, size_t size)
  * The return code is then available in g_rc. */
 #define ASSERT_RC(call, expect_rc) \
 	do { \
-		fprintf(stderr, #call " --> " #expect_rc "\n"); \
+    if ((expect_rc) == -ENOKEY) \
+		  fprintf(stderr, #call " --> -ENOKEY\n"); \
+    else if ((expect_rc) == -ENOTSUP) \
+		  fprintf(stderr, #call " --> -ENOTSUP\n"); \
+    else \
+		  fprintf(stderr, #call " --> " #expect_rc "\n"); \
 		g_rc = call; \
 		if (g_rc != (expect_rc)) \
 			fprintf(stderr, " MISMATCH: got rc = %d, expected: " \
@@ -67,7 +72,12 @@ static void _fill_invalid(void *dest, size_t size)
 	do { \
 		int rc; \
 		fill_invalid(g_subscr); \
-		fprintf(stderr, "db_subscr_get_by_" #by "(dbc, " #val ", &g_subscr) --> " \
+    if ((expect_rc) == -ENOKEY) \
+		  fprintf(stderr, "db_subscr_get_by_" #by "(dbc, " #val ", &g_subscr) --> -ENOKEY \n"); \
+    else if ((expect_rc) == -ENOTSUP) \
+		  fprintf(stderr, "db_subscr_get_by_" #by "(dbc, " #val ", &g_subscr) --> -ENOTSUP \n"); \
+    else \
+		  fprintf(stderr, "db_subscr_get_by_" #by "(dbc, " #val ", &g_subscr) --> " \
                                 #expect_rc "\n"); \
 		rc = db_subscr_get_by_##by(dbc, val, &g_subscr); \
 		if (rc != (expect_rc)) \
@@ -105,7 +115,7 @@ static void _fill_invalid(void *dest, size_t size)
 #define ASSERT_DB_GET_AUC(imsi, expect_rc) \
 	do { \
 		struct osmo_auth_vector vec[N_VECTORS]; \
-		ASSERT_RC(db_get_auc(dbc, imsi, 3, vec, N_VECTORS, NULL, NULL), expect_rc); \
+		ASSERT_RC(db_get_auc(dbc, imsi, 3, vec, N_VECTORS, NULL, NULL, false), expect_rc); \
 	} while (0)
 
 /* Not linking the real auc_compute_vectors(), just returning num_vec.
@@ -148,6 +158,7 @@ void dump_subscr(struct hlr_subscriber *subscr)
 	Pd(id);
 	Ps(imsi);
 	Ps(msisdn);
+	Ps(imei);
 	Ps(vlr_number);
 	Ps(sgsn_number);
 	Ps(sgsn_address);
@@ -207,6 +218,17 @@ void dump_aud(const char *label, struct osmo_sub_auth_data *aud)
 #undef Phex
 }
 
+void db_raw_sql(struct db_context *dbc, const char *sql)
+{
+	sqlite3_stmt *stmt;
+
+	fprintf(stderr, "raw SQL: %s\n", sql);
+	ASSERT_RC(sqlite3_prepare_v2(dbc->db, sql, -1, &stmt, NULL), SQLITE_OK);
+	ASSERT_RC(sqlite3_step(stmt), SQLITE_DONE);
+	db_remove_reset(stmt);
+	sqlite3_finalize(stmt);
+}
+
 static const char *imsi0 = "123456789000000";
 static const char *imsi1 = "123456789000001";
 static const char *imsi2 = "123456789000002";
@@ -220,40 +242,45 @@ static void test_subscr_create_update_sel_delete()
 
 	comment("Create with valid / invalid IMSI");
 
-	ASSERT_RC(db_subscr_create(dbc, imsi0), 0);
+	ASSERT_RC(db_subscr_create(dbc, imsi0, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), 0);
 	ASSERT_SEL(imsi, imsi0, 0);
 	id0 = g_subscr.id;
-	ASSERT_RC(db_subscr_create(dbc, imsi1), 0);
+	ASSERT_RC(db_subscr_create(dbc, imsi1, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), 0);
 	ASSERT_SEL(imsi, imsi1, 0);
 	id1 = g_subscr.id;
-	ASSERT_RC(db_subscr_create(dbc, imsi2), 0);
+	ASSERT_RC(db_subscr_create(dbc, imsi2, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), 0);
 	ASSERT_SEL(imsi, imsi2, 0);
 	id2 = g_subscr.id;
-	ASSERT_RC(db_subscr_create(dbc, imsi0), -EIO);
+	ASSERT_RC(db_subscr_create(dbc, imsi0, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), -EIO);
 	ASSERT_SEL(imsi, imsi0, 0);
-	ASSERT_RC(db_subscr_create(dbc, imsi1), -EIO);
-	ASSERT_RC(db_subscr_create(dbc, imsi1), -EIO);
+	ASSERT_RC(db_subscr_create(dbc, imsi1, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), -EIO);
+	ASSERT_RC(db_subscr_create(dbc, imsi1, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), -EIO);
 	ASSERT_SEL(imsi, imsi1, 0);
-	ASSERT_RC(db_subscr_create(dbc, imsi2), -EIO);
-	ASSERT_RC(db_subscr_create(dbc, imsi2), -EIO);
+	ASSERT_RC(db_subscr_create(dbc, imsi2, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), -EIO);
+	ASSERT_RC(db_subscr_create(dbc, imsi2, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), -EIO);
 	ASSERT_SEL(imsi, imsi2, 0);
 
-	ASSERT_RC(db_subscr_create(dbc, "123456789 000003"), -EINVAL);
+	ASSERT_RC(db_subscr_create(dbc, "123456789 000003", DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), -EINVAL);
 	ASSERT_SEL(imsi, "123456789000003", -ENOENT);
 
-	ASSERT_RC(db_subscr_create(dbc, "123456789000002123456"), -EINVAL);
+	ASSERT_RC(db_subscr_create(dbc, "123456789000002123456", DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS),
+		  -EINVAL);
 	ASSERT_SEL(imsi, "123456789000002123456", -ENOENT);
 
-	ASSERT_RC(db_subscr_create(dbc, "foobar123"), -EINVAL);
+	ASSERT_RC(db_subscr_create(dbc, "foobar123", DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), -EINVAL);
 	ASSERT_SEL(imsi, "foobar123", -ENOENT);
 
-	ASSERT_RC(db_subscr_create(dbc, "123"), -EINVAL);
+	ASSERT_RC(db_subscr_create(dbc, "123", DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), -EINVAL);
 	ASSERT_SEL(imsi, "123", -ENOENT);
 
-	ASSERT_RC(db_subscr_create(dbc, short_imsi), 0);
+	ASSERT_RC(db_subscr_create(dbc, short_imsi, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), 0);
 	ASSERT_SEL(imsi, short_imsi, 0);
 	id_short = g_subscr.id;
 
+	comment("Check if subscriber exists (by IMSI)");
+
+	ASSERT_RC(db_subscr_exists_by_imsi(dbc, imsi0), 0);
+	ASSERT_RC(db_subscr_exists_by_imsi(dbc, unknown_imsi), -ENOENT);
 
 	comment("Set valid / invalid MSISDN");
 
@@ -288,6 +315,11 @@ static void test_subscr_create_update_sel_delete()
 	ASSERT_SEL(imsi, imsi0, 0);
 	ASSERT_SEL(msisdn, "5432101234567891", -ENOENT);
 
+	comment("Check if subscriber exists (by MSISDN)");
+
+	ASSERT_RC(db_subscr_exists_by_msisdn(dbc, "543210123456789"), 0);
+	ASSERT_RC(db_subscr_exists_by_msisdn(dbc, "5432101234567891"), -ENOENT);
+
 	comment("Set MSISDN on non-existent / invalid IMSI");
 
 	ASSERT_RC(db_subscr_update_msisdn_by_imsi(dbc, unknown_imsi, "99"), -ENOENT);
@@ -296,6 +328,23 @@ static void test_subscr_create_update_sel_delete()
 	ASSERT_RC(db_subscr_update_msisdn_by_imsi(dbc, "foobar", "99"), -ENOENT);
 	ASSERT_SEL(msisdn, "99", -ENOENT);
 
+	comment("Set valid / invalid IMEI");
+
+	ASSERT_RC(db_subscr_update_imei_by_imsi(dbc, imsi0, "12345678901234"), 0);
+	ASSERT_SEL(imei, "12345678901234", 0);
+
+	ASSERT_RC(db_subscr_update_imei_by_imsi(dbc, imsi0, "123456789012345"), -EINVAL); /* too long */
+	ASSERT_SEL(imei, "12345678901234", 0);
+	ASSERT_SEL(imei, "123456789012345", -ENOENT);
+
+	comment("Set the same IMEI again");
+	ASSERT_RC(db_subscr_update_imei_by_imsi(dbc, imsi0, "12345678901234"), 0);
+	ASSERT_SEL(imei, "12345678901234", 0);
+
+	comment("Remove IMEI");
+	ASSERT_RC(db_subscr_update_imei_by_imsi(dbc, imsi0, NULL), 0);
+	ASSERT_SEL(imei, "12345678901234", -ENOENT);
+
 	comment("Set / unset nam_cs and nam_ps");
 
 	/*                                nam_val, is_ps */
@@ -434,6 +483,22 @@ static void test_subscr_create_update_sel_delete()
 	ASSERT_RC(db_subscr_delete_by_id(dbc, id_short), 0);
 	ASSERT_SEL(imsi, short_imsi, -ENOENT);
 
+	comment("Create and delete subscribers with non-default nam_cs and nam_ps");
+
+	ASSERT_RC(db_subscr_create(dbc, imsi0, 0x00), 0);
+	ASSERT_SEL(imsi, imsi0, 0);
+	id0 = g_subscr.id;
+	ASSERT_RC(db_subscr_create(dbc, imsi1, DB_SUBSCR_FLAG_NAM_CS), 0);
+	ASSERT_SEL(imsi, imsi1, 0);
+	id1 = g_subscr.id;
+	ASSERT_RC(db_subscr_create(dbc, imsi2, DB_SUBSCR_FLAG_NAM_PS), 0);
+	ASSERT_SEL(imsi, imsi2, 0);
+	id2 = g_subscr.id;
+
+	ASSERT_RC(db_subscr_delete_by_id(dbc, id0), 0);
+	ASSERT_RC(db_subscr_delete_by_id(dbc, id1), 0);
+	ASSERT_RC(db_subscr_delete_by_id(dbc, id2), 0);
+
 	comment_end();
 }
 
@@ -477,7 +542,7 @@ static void test_subscr_aud()
 
 	comment("Create subscriber");
 
-	ASSERT_RC(db_subscr_create(dbc, imsi0), 0);
+	ASSERT_RC(db_subscr_create(dbc, imsi0, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), 0);
 	ASSERT_SEL(imsi, imsi0, 0);
 
 	id = g_subscr.id;
@@ -689,7 +754,7 @@ static void test_subscr_aud()
 
 	comment("Re-add subscriber and verify auth data didn't come back");
 
-	ASSERT_RC(db_subscr_create(dbc, imsi0), 0);
+	ASSERT_RC(db_subscr_create(dbc, imsi0, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), 0);
 	ASSERT_SEL(imsi, imsi0, 0);
 
 	/* For this test to work, we want to get the same subscriber ID back,
@@ -705,6 +770,70 @@ static void test_subscr_aud()
 	comment_end();
 }
 
+/* Make each key too short in this test. Note that we can't set them longer than the allowed size without changing the
+ * table structure. */
+static void test_subscr_aud_invalid_len()
+{
+	int64_t id;
+
+	comment_start();
+	comment("Create subscriber");
+	ASSERT_RC(db_subscr_create(dbc, imsi0, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), 0);
+	ASSERT_SEL(imsi, imsi0, 0);
+	id = g_subscr.id;
+
+
+	/* Invalid Ki length */
+	comment("Set auth data, 2G only, with invalid Ki length");
+	ASSERT_RC(db_subscr_update_aud_by_id(dbc, id,
+		  mk_aud_2g(OSMO_AUTH_ALG_COMP128v1, "0123456789abcdef0123456789abcdef")),
+		  0);
+	/* Use raw SQL to avoid length check in db_subscr_update_aud_by_id(). This changes all rows in the table, which
+         * is fine for this test (implicit WHERE 1). */
+	db_raw_sql(dbc, "UPDATE auc_2g SET ki = '0123456789abcdef0123456789abcde'");
+	ASSERT_SEL_AUD(imsi0, -ENOKEY, id);
+
+	comment("Remove 2G auth data");
+	ASSERT_RC(db_subscr_update_aud_by_id(dbc, id,
+		  mk_aud_2g(OSMO_AUTH_ALG_NONE, NULL)),
+		  0);
+
+	/* Invalid K length */
+	comment("Set auth data, 3G only, with invalid K length");
+	ASSERT_RC(db_subscr_update_aud_by_id(dbc, id,
+		mk_aud_3g(OSMO_AUTH_ALG_MILENAGE,
+			  "BeefedCafeFaceAcedAddedDecadeFee", true,
+			  "C01ffedC1cadaeAc1d1f1edAcac1aB0a", 5)),
+		0);
+	db_raw_sql(dbc, "UPDATE auc_3g SET k = 'C01ffedC1cadaeAc1d1f1edAcac1aB0'");
+	ASSERT_SEL_AUD(imsi0, -EIO, id);
+
+	/* Invalid OP length */
+	comment("Set auth data, 3G only, with invalid OP length");
+	ASSERT_RC(db_subscr_update_aud_by_id(dbc, id,
+		mk_aud_3g(OSMO_AUTH_ALG_MILENAGE,
+			  "BeefedCafeFaceAcedAddedDecadeFee", true,
+			  "C01ffedC1cadaeAc1d1f1edAcac1aB0a", 5)),
+		0);
+	db_raw_sql(dbc, "UPDATE auc_3g SET op = 'BeefedCafeFaceAcedAddedDecadeFe'");
+	ASSERT_SEL_AUD(imsi0, -EIO, id);
+
+	/* Invalid OPC length */
+	comment("Set auth data, 3G only, with invalid OPC length");
+	ASSERT_RC(db_subscr_update_aud_by_id(dbc, id,
+		mk_aud_3g(OSMO_AUTH_ALG_MILENAGE,
+			  "BeefedCafeFaceAcedAddedDecadeFee", false,
+			  "C01ffedC1cadaeAc1d1f1edAcac1aB0a", 5)),
+		0);
+	db_raw_sql(dbc, "UPDATE auc_3g SET opc = 'BeefedCafeFaceAcedAddedDecadeFe'");
+	ASSERT_SEL_AUD(imsi0, -EIO, id);
+
+
+	comment("Delete subscriber");
+	ASSERT_RC(db_subscr_delete_by_id(dbc, id), 0);
+	comment_end();
+}
+
 static void test_subscr_sqn()
 {
 	int64_t id;
@@ -721,7 +850,7 @@ static void test_subscr_sqn()
 
 	comment("Create subscriber");
 
-	ASSERT_RC(db_subscr_create(dbc, imsi0), 0);
+	ASSERT_RC(db_subscr_create(dbc, imsi0, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), 0);
 	ASSERT_SEL(imsi, imsi0, 0);
 
 	id = g_subscr.id;
@@ -850,15 +979,17 @@ int main(int argc, char **argv)
 	log_set_log_level(osmo_stderr_target, LOGL_ERROR);
 	/* Disable SQLite logging so that we're not vulnerable on SQLite error messages changing across
 	 * library versions. */
-	dbc = db_open(ctx, "db_test.db", false);
+	dbc = db_open(ctx, "db_test.db", false, false);
 	log_set_log_level(osmo_stderr_target, 0);
 	OSMO_ASSERT(dbc);
 
 	test_subscr_create_update_sel_delete();
 	test_subscr_aud();
+	test_subscr_aud_invalid_len();
 	test_subscr_sqn();
 
 	printf("Done\n");
+	db_close(dbc);
 	return 0;
 }
 

tests/db/db_test.err

@@ -3,7 +3,7 @@
 
 --- Create with valid / invalid IMSI
 
-db_subscr_create(dbc, imsi0) --> 0
+db_subscr_create(dbc, imsi0, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> 0
 
 db_subscr_get_by_imsi(dbc, imsi0, &g_subscr) --> 0
 struct hlr_subscriber {
@@ -11,7 +11,7 @@ struct hlr_subscriber {
   .imsi = '123456789000000',
 }
 
-db_subscr_create(dbc, imsi1) --> 0
+db_subscr_create(dbc, imsi1, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> 0
 
 db_subscr_get_by_imsi(dbc, imsi1, &g_subscr) --> 0
 struct hlr_subscriber {
@@ -19,7 +19,7 @@ struct hlr_subscriber {
   .imsi = '123456789000001',
 }
 
-db_subscr_create(dbc, imsi2) --> 0
+db_subscr_create(dbc, imsi2, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> 0
 
 db_subscr_get_by_imsi(dbc, imsi2, &g_subscr) --> 0
 struct hlr_subscriber {
@@ -27,7 +27,7 @@ struct hlr_subscriber {
   .imsi = '123456789000002',
 }
 
-db_subscr_create(dbc, imsi0) --> -EIO
+db_subscr_create(dbc, imsi0, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> -EIO
 DAUC IMSI='123456789000000': Cannot create subscriber: SQL error: (2067) UNIQUE constraint failed: subscriber.imsi
 
 db_subscr_get_by_imsi(dbc, imsi0, &g_subscr) --> 0
@@ -36,10 +36,10 @@ struct hlr_subscriber {
   .imsi = '123456789000000',
 }
 
-db_subscr_create(dbc, imsi1) --> -EIO
+db_subscr_create(dbc, imsi1, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> -EIO
 DAUC IMSI='123456789000001': Cannot create subscriber: SQL error: (2067) UNIQUE constraint failed: subscriber.imsi
 
-db_subscr_create(dbc, imsi1) --> -EIO
+db_subscr_create(dbc, imsi1, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> -EIO
 DAUC IMSI='123456789000001': Cannot create subscriber: SQL error: (2067) UNIQUE constraint failed: subscriber.imsi
 
 db_subscr_get_by_imsi(dbc, imsi1, &g_subscr) --> 0
@@ -48,10 +48,10 @@ struct hlr_subscriber {
   .imsi = '123456789000001',
 }
 
-db_subscr_create(dbc, imsi2) --> -EIO
+db_subscr_create(dbc, imsi2, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> -EIO
 DAUC IMSI='123456789000002': Cannot create subscriber: SQL error: (2067) UNIQUE constraint failed: subscriber.imsi
 
-db_subscr_create(dbc, imsi2) --> -EIO
+db_subscr_create(dbc, imsi2, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> -EIO
 DAUC IMSI='123456789000002': Cannot create subscriber: SQL error: (2067) UNIQUE constraint failed: subscriber.imsi
 
 db_subscr_get_by_imsi(dbc, imsi2, &g_subscr) --> 0
@@ -60,31 +60,31 @@ struct hlr_subscriber {
   .imsi = '123456789000002',
 }
 
-db_subscr_create(dbc, "123456789 000003") --> -EINVAL
+db_subscr_create(dbc, "123456789 000003", DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> -EINVAL
 DAUC Cannot create subscriber: invalid IMSI: '123456789 000003'
 
 db_subscr_get_by_imsi(dbc, "123456789000003", &g_subscr) --> -ENOENT
 DAUC Cannot read subscriber from db: IMSI='123456789000003': No such subscriber
 
-db_subscr_create(dbc, "123456789000002123456") --> -EINVAL
+db_subscr_create(dbc, "123456789000002123456", DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> -EINVAL
 DAUC Cannot create subscriber: invalid IMSI: '123456789000002123456'
 
 db_subscr_get_by_imsi(dbc, "123456789000002123456", &g_subscr) --> -ENOENT
 DAUC Cannot read subscriber from db: IMSI='123456789000002123456': No such subscriber
 
-db_subscr_create(dbc, "foobar123") --> -EINVAL
+db_subscr_create(dbc, "foobar123", DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> -EINVAL
 DAUC Cannot create subscriber: invalid IMSI: 'foobar123'
 
 db_subscr_get_by_imsi(dbc, "foobar123", &g_subscr) --> -ENOENT
 DAUC Cannot read subscriber from db: IMSI='foobar123': No such subscriber
 
-db_subscr_create(dbc, "123") --> -EINVAL
+db_subscr_create(dbc, "123", DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> -EINVAL
 DAUC Cannot create subscriber: invalid IMSI: '123'
 
 db_subscr_get_by_imsi(dbc, "123", &g_subscr) --> -ENOENT
 DAUC Cannot read subscriber from db: IMSI='123': No such subscriber
 
-db_subscr_create(dbc, short_imsi) --> 0
+db_subscr_create(dbc, short_imsi, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> 0
 
 db_subscr_get_by_imsi(dbc, short_imsi, &g_subscr) --> 0
 struct hlr_subscriber {
@@ -93,6 +93,13 @@ struct hlr_subscriber {
 }
 
 
+--- Check if subscriber exists (by IMSI)
+
+db_subscr_exists_by_imsi(dbc, imsi0) --> 0
+
+db_subscr_exists_by_imsi(dbc, unknown_imsi) --> -ENOENT
+
+
 --- Set valid / invalid MSISDN
 
 db_subscr_get_by_imsi(dbc, imsi0, &g_subscr) --> 0
@@ -212,6 +219,13 @@ db_subscr_get_by_msisdn(dbc, "5432101234567891", &g_subscr) --> -ENOENT
 DAUC Cannot read subscriber from db: MSISDN='5432101234567891': No such subscriber
 
 
+--- Check if subscriber exists (by MSISDN)
+
+db_subscr_exists_by_msisdn(dbc, "543210123456789") --> 0
+
+db_subscr_exists_by_msisdn(dbc, "5432101234567891") --> -ENOENT
+
+
 --- Set MSISDN on non-existent / invalid IMSI
 
 db_subscr_update_msisdn_by_imsi(dbc, unknown_imsi, "99") --> -ENOENT
@@ -227,6 +241,54 @@ db_subscr_get_by_msisdn(dbc, "99", &g_subscr) --> -ENOENT
 DAUC Cannot read subscriber from db: MSISDN='99': No such subscriber
 
 
+--- Set valid / invalid IMEI
+
+db_subscr_update_imei_by_imsi(dbc, imsi0, "12345678901234") --> 0
+
+db_subscr_get_by_imei(dbc, "12345678901234", &g_subscr) --> 0
+struct hlr_subscriber {
+  .id = 1,
+  .imsi = '123456789000000',
+  .msisdn = '543210123456789',
+  .imei = '12345678901234',
+}
+
+db_subscr_update_imei_by_imsi(dbc, imsi0, "123456789012345") --> -EINVAL
+DAUC Cannot update subscriber IMSI='123456789000000': invalid IMEI: '123456789012345'
+
+db_subscr_get_by_imei(dbc, "12345678901234", &g_subscr) --> 0
+struct hlr_subscriber {
+  .id = 1,
+  .imsi = '123456789000000',
+  .msisdn = '543210123456789',
+  .imei = '12345678901234',
+}
+
+db_subscr_get_by_imei(dbc, "123456789012345", &g_subscr) --> -ENOENT
+DAUC Cannot read subscriber from db: IMEI=123456789012345: No such subscriber
+
+
+--- Set the same IMEI again
+
+db_subscr_update_imei_by_imsi(dbc, imsi0, "12345678901234") --> 0
+
+db_subscr_get_by_imei(dbc, "12345678901234", &g_subscr) --> 0
+struct hlr_subscriber {
+  .id = 1,
+  .imsi = '123456789000000',
+  .msisdn = '543210123456789',
+  .imei = '12345678901234',
+}
+
+
+--- Remove IMEI
+
+db_subscr_update_imei_by_imsi(dbc, imsi0, NULL) --> 0
+
+db_subscr_get_by_imei(dbc, "12345678901234", &g_subscr) --> -ENOENT
+DAUC Cannot read subscriber from db: IMEI=12345678901234: No such subscriber
+
+
 --- Set / unset nam_cs and nam_ps
 
 db_subscr_nam(dbc, imsi0, false, true) --> 0
@@ -704,6 +766,43 @@ db_subscr_delete_by_id(dbc, id_short) --> 0
 db_subscr_get_by_imsi(dbc, short_imsi, &g_subscr) --> -ENOENT
 DAUC Cannot read subscriber from db: IMSI='123456': No such subscriber
 
+
+--- Create and delete subscribers with non-default nam_cs and nam_ps
+
+db_subscr_create(dbc, imsi0, 0x00) --> 0
+
+db_subscr_get_by_imsi(dbc, imsi0, &g_subscr) --> 0
+struct hlr_subscriber {
+  .id = 1,
+  .imsi = '123456789000000',
+  .nam_cs = false,
+  .nam_ps = false,
+}
+
+db_subscr_create(dbc, imsi1, DB_SUBSCR_FLAG_NAM_CS) --> 0
+
+db_subscr_get_by_imsi(dbc, imsi1, &g_subscr) --> 0
+struct hlr_subscriber {
+  .id = 2,
+  .imsi = '123456789000001',
+  .nam_ps = false,
+}
+
+db_subscr_create(dbc, imsi2, DB_SUBSCR_FLAG_NAM_PS) --> 0
+
+db_subscr_get_by_imsi(dbc, imsi2, &g_subscr) --> 0
+struct hlr_subscriber {
+  .id = 3,
+  .imsi = '123456789000002',
+  .nam_cs = false,
+}
+
+db_subscr_delete_by_id(dbc, id0) --> 0
+
+db_subscr_delete_by_id(dbc, id1) --> 0
+
+db_subscr_delete_by_id(dbc, id2) --> 0
+
 ===== test_subscr_create_update_sel_delete: SUCCESS
 
 
@@ -715,13 +814,13 @@ db_get_auth_data(dbc, unknown_imsi, &g_aud2g, &g_aud3g, &g_id) --> -2
 DAUC IMSI='999999999': No such subscriber
 
 
-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> -2
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> -2
 DAUC IMSI='123456789000000': No such subscriber
 
 
 --- Create subscriber
 
-db_subscr_create(dbc, imsi0) --> 0
+db_subscr_create(dbc, imsi0, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> 0
 
 db_subscr_get_by_imsi(dbc, imsi0, &g_subscr) --> 0
 struct hlr_subscriber {
@@ -729,12 +828,12 @@ struct hlr_subscriber {
   .imsi = '123456789000000',
 }
 
-db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -126
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 
 
-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> -126
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 
@@ -753,7 +852,7 @@ DAUC IMSI='123456789000000': No 3G Auth Data
 }
 3G: none
 
-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> 3
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> 3
 DAUC IMSI='123456789000000': No 3G Auth Data
 DAUC IMSI='123456789000000': Calling to generate 3 vectors
 DAUC IMSI='123456789000000': Generated 3 vectors
@@ -811,12 +910,12 @@ DAUC IMSI='123456789000000': No 3G Auth Data
 
 db_subscr_update_aud_by_id(dbc, id, mk_aud_2g(OSMO_AUTH_ALG_NONE, NULL)) --> 0
 
-db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -126
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 
 
-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> -126
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 
@@ -836,12 +935,12 @@ DAUC IMSI='123456789000000': No 3G Auth Data
 
 db_subscr_update_aud_by_id(dbc, id, mk_aud_2g(OSMO_AUTH_ALG_NONE, "f000000000000f00000000000f000000")) --> 0
 
-db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -126
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 
 
-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> -126
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 
@@ -864,7 +963,7 @@ DAUC IMSI='123456789000000': No 2G Auth Data
   .u.umts.ind_bitlen = 5,
 }
 
-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> 3
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> 3
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': Calling to generate 3 vectors
 DAUC IMSI='123456789000000': Generated 3 vectors
@@ -938,12 +1037,12 @@ DAUC IMSI='123456789000000': No 2G Auth Data
 
 db_subscr_update_aud_by_id(dbc, id, mk_aud_3g(OSMO_AUTH_ALG_NONE, NULL, false, NULL, 0)) --> 0
 
-db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -126
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 
 
-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> -126
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 
@@ -965,7 +1064,7 @@ DAUC IMSI='123456789000000': No 2G Auth Data
   .u.umts.ind_bitlen = 5,
 }
 
-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> 3
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> 3
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': Calling to generate 3 vectors
 DAUC IMSI='123456789000000': Generated 3 vectors
@@ -973,12 +1072,12 @@ DAUC IMSI='123456789000000': Updating SQN=0 in DB
 
 db_subscr_update_aud_by_id(dbc, id, mk_aud_3g(OSMO_AUTH_ALG_NONE, "asdfasdfasd", false, "asdfasdfasdf", 99999)) --> 0
 
-db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -126
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 
 
-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> -126
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 
@@ -1006,7 +1105,7 @@ db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> 0
   .u.umts.ind_bitlen = 5,
 }
 
-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> 3
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> 3
 DAUC IMSI='123456789000000': Calling to generate 3 vectors
 DAUC IMSI='123456789000000': Generated 3 vectors
 DAUC IMSI='123456789000000': Updating SQN=0 in DB
@@ -1211,7 +1310,7 @@ DAUC Cannot read subscriber from db: IMSI='123456789000000': No such subscriber
 
 --- Re-add subscriber and verify auth data didn't come back
 
-db_subscr_create(dbc, imsi0) --> 0
+db_subscr_create(dbc, imsi0, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> 0
 
 db_subscr_get_by_imsi(dbc, imsi0, &g_subscr) --> 0
 struct hlr_subscriber {
@@ -1219,12 +1318,12 @@ struct hlr_subscriber {
   .imsi = '123456789000000',
 }
 
-db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -126
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 
 
-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> -126
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 
@@ -1233,12 +1332,97 @@ db_subscr_delete_by_id(dbc, id) --> 0
 db_subscr_get_by_imsi(dbc, imsi0, &g_subscr) --> -ENOENT
 DAUC Cannot read subscriber from db: IMSI='123456789000000': No such subscriber
 
-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> -2
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> -2
 DAUC IMSI='123456789000000': No such subscriber
 
 ===== test_subscr_aud: SUCCESS
 
 
+===== test_subscr_aud_invalid_len
+
+--- Create subscriber
+
+db_subscr_create(dbc, imsi0, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> 0
+
+db_subscr_get_by_imsi(dbc, imsi0, &g_subscr) --> 0
+struct hlr_subscriber {
+  .id = 1,
+  .imsi = '123456789000000',
+}
+
+
+--- Set auth data, 2G only, with invalid Ki length
+
+db_subscr_update_aud_by_id(dbc, id, mk_aud_2g(OSMO_AUTH_ALG_COMP128v1, "0123456789abcdef0123456789abcdef")) --> 0
+
+raw SQL: UPDATE auc_2g SET ki = '0123456789abcdef0123456789abcde'
+sqlite3_prepare_v2(dbc->db, sql, -1, &stmt, NULL) --> SQLITE_OK
+
+sqlite3_step(stmt) --> SQLITE_DONE
+
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
+DAUC IMSI='123456789000000': Error reading Ki, expected length 16 but has length 15
+DAUC IMSI='123456789000000': No 3G Auth Data
+
+
+
+--- Remove 2G auth data
+
+db_subscr_update_aud_by_id(dbc, id, mk_aud_2g(OSMO_AUTH_ALG_NONE, NULL)) --> 0
+
+
+--- Set auth data, 3G only, with invalid K length
+
+db_subscr_update_aud_by_id(dbc, id, mk_aud_3g(OSMO_AUTH_ALG_MILENAGE, "BeefedCafeFaceAcedAddedDecadeFee", true, "C01ffedC1cadaeAc1d1f1edAcac1aB0a", 5)) --> 0
+
+raw SQL: UPDATE auc_3g SET k = 'C01ffedC1cadaeAc1d1f1edAcac1aB0'
+sqlite3_prepare_v2(dbc->db, sql, -1, &stmt, NULL) --> SQLITE_OK
+
+sqlite3_step(stmt) --> SQLITE_DONE
+
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -5
+DAUC IMSI='123456789000000': No 2G Auth Data
+DAUC IMSI='123456789000000': Error reading K, expected length 16 but has length 15
+
+
+
+--- Set auth data, 3G only, with invalid OP length
+
+db_subscr_update_aud_by_id(dbc, id, mk_aud_3g(OSMO_AUTH_ALG_MILENAGE, "BeefedCafeFaceAcedAddedDecadeFee", true, "C01ffedC1cadaeAc1d1f1edAcac1aB0a", 5)) --> 0
+
+raw SQL: UPDATE auc_3g SET op = 'BeefedCafeFaceAcedAddedDecadeFe'
+sqlite3_prepare_v2(dbc->db, sql, -1, &stmt, NULL) --> SQLITE_OK
+
+sqlite3_step(stmt) --> SQLITE_DONE
+
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -5
+DAUC IMSI='123456789000000': No 2G Auth Data
+DAUC IMSI='123456789000000': Error reading OP, expected length 16 but has length 15
+
+
+
+--- Set auth data, 3G only, with invalid OPC length
+
+db_subscr_update_aud_by_id(dbc, id, mk_aud_3g(OSMO_AUTH_ALG_MILENAGE, "BeefedCafeFaceAcedAddedDecadeFee", false, "C01ffedC1cadaeAc1d1f1edAcac1aB0a", 5)) --> 0
+
+raw SQL: UPDATE auc_3g SET opc = 'BeefedCafeFaceAcedAddedDecadeFe'
+sqlite3_prepare_v2(dbc->db, sql, -1, &stmt, NULL) --> SQLITE_OK
+
+sqlite3_step(stmt) --> SQLITE_DONE
+
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -5
+DAUC IMSI='123456789000000': No 2G Auth Data
+DAUC IMSI='123456789000000': Error reading OPC, expected length 16 but has length 15
+
+
+
+--- Delete subscriber
+
+db_subscr_delete_by_id(dbc, id) --> 0
+
+===== test_subscr_aud_invalid_len: SUCCESS
+
+
 ===== test_subscr_sqn
 
 --- Set SQN for unknown subscriber
@@ -1258,7 +1442,7 @@ DAUC Cannot read subscriber from db: ID=9999: No such subscriber
 
 --- Create subscriber
 
-db_subscr_create(dbc, imsi0) --> 0
+db_subscr_create(dbc, imsi0, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> 0
 
 db_subscr_get_by_imsi(dbc, imsi0, &g_subscr) --> 0
 struct hlr_subscriber {
@@ -1266,7 +1450,7 @@ struct hlr_subscriber {
   .imsi = '123456789000000',
 }
 
-db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -126
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 
@@ -1277,7 +1461,7 @@ DAUC IMSI='123456789000000': No 3G Auth Data
 db_update_sqn(dbc, id, 123) --> -ENOENT
 DAUC Cannot update SQN for subscriber ID=1: no auc_3g entry for such subscriber
 
-db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -126
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 
@@ -1285,7 +1469,7 @@ DAUC IMSI='123456789000000': No 3G Auth Data
 db_update_sqn(dbc, id, 543) --> -ENOENT
 DAUC Cannot update SQN for subscriber ID=1: no auc_3g entry for such subscriber
 
-db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -126
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 

tests/db_upgrade/Makefile.am

@@ -0,0 +1,14 @@
+EXTRA_DIST = \
+	db_upgrade_test.sh \
+	db_upgrade_test.err \
+	db_upgrade_test.ok \
+	hlr_db_v0.sql \
+	osmo-hlr.cfg \
+	create_subscribers.vty \
+	$(NULL)
+
+update_exp:
+	$(srcdir)/db_upgrade_test.sh $(srcdir) $(builddir) >"$(srcdir)/db_upgrade_test.ok" 2>"$(srcdir)/db_upgrade_test.err"
+
+upgrade-equivalence-test:
+	$(srcdir)/db_upgrade_test.sh $(srcdir) $(builddir) do-equivalence-test

tests/db_upgrade/create_subscribers.vty

@@ -0,0 +1,47 @@
+OsmoHLR> enable
+OsmoHLR# subscriber imsi 123456789012345 create
+% Created subscriber 123456789012345
+    ID: 1
+    IMSI: 123456789012345
+    MSISDN: none
+OsmoHLR# subscriber imsi 123456789012345 update msisdn 098765432109876
+% Updated subscriber IMSI='123456789012345' to MSISDN='098765432109876'
+OsmoHLR# subscriber imsi 123456789012345 update aud2g comp128v1 ki BeefedCafeFaceAcedAddedDecadeFee
+OsmoHLR# subscriber imsi 123456789012345 update aud3g milenage k C01ffedC1cadaeAc1d1f1edAcac1aB0a opc CededEffacedAceFacedBadFadedBeef
+OsmoHLR# subscriber imsi 111111111 create
+% Created subscriber 111111111
+    ID: 2
+    IMSI: 111111111
+    MSISDN: none
+OsmoHLR# subscriber imsi 222222222 create
+% Created subscriber 222222222
+    ID: 3
+    IMSI: 222222222
+    MSISDN: none
+OsmoHLR# subscriber imsi 222222222 update msisdn 22222
+% Updated subscriber IMSI='222222222' to MSISDN='22222'
+OsmoHLR# subscriber imsi 333333 create
+% Created subscriber 333333
+    ID: 4
+    IMSI: 333333
+    MSISDN: none
+OsmoHLR# subscriber imsi 333333 update msisdn 3
+% Updated subscriber IMSI='333333' to MSISDN='3'
+OsmoHLR# subscriber imsi 333333 update aud2g comp128v2 ki 33333333333333333333333333333333
+OsmoHLR# subscriber imsi 444444444444444 create
+% Created subscriber 444444444444444
+    ID: 5
+    IMSI: 444444444444444
+    MSISDN: none
+OsmoHLR# subscriber imsi 444444444444444 update msisdn 4444
+% Updated subscriber IMSI='444444444444444' to MSISDN='4444'
+OsmoHLR# subscriber imsi 444444444444444 update aud3g milenage k 44444444444444444444444444444444 op 44444444444444444444444444444444
+OsmoHLR# subscriber imsi 5555555 create
+% Created subscriber 5555555
+    ID: 6
+    IMSI: 5555555
+    MSISDN: none
+OsmoHLR# subscriber imsi 5555555 update msisdn 55555555555555
+% Updated subscriber IMSI='5555555' to MSISDN='55555555555555'
+OsmoHLR# subscriber imsi 5555555 update aud2g xor ki 55555555555555555555555555555555
+OsmoHLR# subscriber imsi 5555555 update aud3g milenage k 55555555555555555555555555555555 opc 55555555555555555555555555555555

tests/db_upgrade/create_subscribers_step2.sql

@@ -0,0 +1,6 @@
+update subscriber set vlr_number = 'MSC-1' where id = 1;
+update subscriber set ms_purged_cs = 1 where id = 2;
+update subscriber set ms_purged_ps = 1 where id = 3;
+update subscriber set nam_cs = 0 where id = 4;
+update subscriber set nam_ps = 0 where id = 5;
+update subscriber set nam_cs = 0, nam_ps = 0 where id = 6;

tests/db_upgrade/db_upgrade_test.ok

@@ -0,0 +1,172 @@
+Creating db in schema version 0
+
+Version 0 db:
+
+Table: auc_2g
+name|type|notnull|dflt_value|pk
+algo_id_2g|INTEGER|1||0
+ki|VARCHAR(32)|1||0
+subscriber_id|INTEGER|0||1
+
+Table auc_2g contents:
+algo_id_2g|ki|subscriber_id
+1|BeefedCafeFaceAcedAddedDecadeFee|1
+2|33333333333333333333333333333333|4
+4|55555555555555555555555555555555|6
+
+Table: auc_3g
+name|type|notnull|dflt_value|pk
+algo_id_3g|INTEGER|1||0
+ind_bitlen|INTEGER|1|5|0
+k|VARCHAR(32)|1||0
+op|VARCHAR(32)|0||0
+opc|VARCHAR(32)|0||0
+sqn|INTEGER|1|0|0
+subscriber_id|INTEGER|0||1
+
+Table auc_3g contents:
+algo_id_3g|ind_bitlen|k|op|opc|sqn|subscriber_id
+5|5|C01ffedC1cadaeAc1d1f1edAcac1aB0a||CededEffacedAceFacedBadFadedBeef|0|1
+5|5|44444444444444444444444444444444|44444444444444444444444444444444||0|5
+5|5|55555555555555555555555555555555||55555555555555555555555555555555|0|6
+
+Table: subscriber
+name|type|notnull|dflt_value|pk
+ggsn_number|VARCHAR(15)|0||0
+gmlc_number|VARCHAR(15)|0||0
+hlr_number|VARCHAR(15)|0||0
+id|INTEGER|0||1
+imeisv|VARCHAR|0||0
+imsi|VARCHAR(15)|1||0
+lmsi|INTEGER|0||0
+ms_purged_cs|BOOLEAN|1|0|0
+ms_purged_ps|BOOLEAN|1|0|0
+msisdn|VARCHAR(15)|0||0
+nam_cs|BOOLEAN|1|1|0
+nam_ps|BOOLEAN|1|1|0
+periodic_lu_tmr|INTEGER|0||0
+periodic_rau_tau_tmr|INTEGER|0||0
+sgsn_address|VARCHAR|0||0
+sgsn_number|VARCHAR(15)|0||0
+smsc_number|VARCHAR(15)|0||0
+vlr_number|VARCHAR(15)|0||0
+
+Table subscriber contents:
+ggsn_number|gmlc_number|hlr_number|id|imeisv|imsi|lmsi|ms_purged_cs|ms_purged_ps|msisdn|nam_cs|nam_ps|periodic_lu_tmr|periodic_rau_tau_tmr|sgsn_address|sgsn_number|smsc_number|vlr_number
+|||1||123456789012345||0|0|098765432109876|1|1||||||MSC-1
+|||2||111111111||1|0||1|1||||||
+|||3||222222222||0|1|22222|1|1||||||
+|||4||333333||0|0|3|0|1||||||
+|||5||444444444444444||0|0|4444|1|0||||||
+|||6||5555555||0|0|55555555555555|0|0||||||
+
+Table: subscriber_apn
+name|type|notnull|dflt_value|pk
+apn|VARCHAR(256)|1||0
+subscriber_id|INTEGER|0||0
+
+Table subscriber_apn contents:
+
+Table: subscriber_multi_msisdn
+name|type|notnull|dflt_value|pk
+msisdn|VARCHAR(15)|1||0
+subscriber_id|INTEGER|0||0
+
+Table subscriber_multi_msisdn contents:
+
+Launching osmo-hlr to upgrade db:
+osmo-hlr --database $db --db-upgrade --db-check --config-file $srcdir/osmo-hlr.cfg
+rc = 0
+DMAIN hlr starting
+DDB using database: <PATH>test.db
+DDB Database <PATH>test.db' has HLR DB schema version 0
+DDB Database <PATH>test.db' has been upgraded to HLR DB schema version 1
+DDB Database <PATH>test.db' has been upgraded to HLR DB schema version 2
+DDB Database <PATH>test.db' has been upgraded to HLR DB schema version 3
+DDB Database <PATH>test.db' has been upgraded to HLR DB schema version 4
+DMAIN Cmdline option --db-check: Database was opened successfully, quitting.
+
+Resulting db:
+
+Table: auc_2g
+name|type|notnull|dflt_value|pk
+algo_id_2g|INTEGER|1||0
+ki|VARCHAR(32)|1||0
+subscriber_id|INTEGER|0||1
+
+Table auc_2g contents:
+algo_id_2g|ki|subscriber_id
+1|BeefedCafeFaceAcedAddedDecadeFee|1
+2|33333333333333333333333333333333|4
+4|55555555555555555555555555555555|6
+
+Table: auc_3g
+name|type|notnull|dflt_value|pk
+algo_id_3g|INTEGER|1||0
+ind_bitlen|INTEGER|1|5|0
+k|VARCHAR(32)|1||0
+op|VARCHAR(32)|0||0
+opc|VARCHAR(32)|0||0
+sqn|INTEGER|1|0|0
+subscriber_id|INTEGER|0||1
+
+Table auc_3g contents:
+algo_id_3g|ind_bitlen|k|op|opc|sqn|subscriber_id
+5|5|C01ffedC1cadaeAc1d1f1edAcac1aB0a||CededEffacedAceFacedBadFadedBeef|0|1
+5|5|44444444444444444444444444444444|44444444444444444444444444444444||0|5
+5|5|55555555555555555555555555555555||55555555555555555555555555555555|0|6
+
+Table: subscriber
+name|type|notnull|dflt_value|pk
+ggsn_number|VARCHAR(15)|0||0
+gmlc_number|VARCHAR(15)|0||0
+id|INTEGER|0||1
+imei|VARCHAR(14)|0||0
+imeisv|VARCHAR|0||0
+imsi|VARCHAR(15)|1||0
+last_lu_seen|TIMESTAMP|0|NULL|0
+last_lu_seen_ps|TIMESTAMP|0|NULL|0
+lmsi|INTEGER|0||0
+ms_purged_cs|BOOLEAN|1|0|0
+ms_purged_ps|BOOLEAN|1|0|0
+msc_number|VARCHAR(15)|0||0
+msisdn|VARCHAR(15)|0||0
+nam_cs|BOOLEAN|1|1|0
+nam_ps|BOOLEAN|1|1|0
+periodic_lu_tmr|INTEGER|0||0
+periodic_rau_tau_tmr|INTEGER|0||0
+sgsn_address|VARCHAR|0||0
+sgsn_number|VARCHAR(15)|0||0
+smsc_number|VARCHAR(15)|0||0
+vlr_number|VARCHAR(15)|0||0
+
+Table subscriber contents:
+ggsn_number|gmlc_number|id|imei|imeisv|imsi|last_lu_seen|last_lu_seen_ps|lmsi|ms_purged_cs|ms_purged_ps|msc_number|msisdn|nam_cs|nam_ps|periodic_lu_tmr|periodic_rau_tau_tmr|sgsn_address|sgsn_number|smsc_number|vlr_number
+||1|||123456789012345||||0|0||098765432109876|1|1||||||MSC-1
+||2|||111111111||||1|0|||1|1||||||
+||3|||222222222||||0|1||22222|1|1||||||
+||4|||333333||||0|0||3|0|1||||||
+||5|||444444444444444||||0|0||4444|1|0||||||
+||6|||5555555||||0|0||55555555555555|0|0||||||
+
+Table: subscriber_apn
+name|type|notnull|dflt_value|pk
+apn|VARCHAR(256)|1||0
+subscriber_id|INTEGER|0||0
+
+Table subscriber_apn contents:
+
+Table: subscriber_multi_msisdn
+name|type|notnull|dflt_value|pk
+msisdn|VARCHAR(15)|1||0
+subscriber_id|INTEGER|0||0
+
+Table subscriber_multi_msisdn contents:
+
+Verify that osmo-hlr can open it:
+osmo-hlr --database $db --db-check --config-file $srcdir/osmo-hlr.cfg
+rc = 0
+DMAIN hlr starting
+DDB using database: <PATH>test.db
+DDB Database <PATH>test.db' has HLR DB schema version 4
+DMAIN Cmdline option --db-check: Database was opened successfully, quitting.

tests/db_upgrade/db_upgrade_test.sh

@@ -0,0 +1,83 @@
+#!/bin/sh
+srcdir="${1:-.}"
+builddir="${2:-.}"
+do_equivalence_test="$3"
+
+set -e
+
+db="$builddir/test.db"
+osmo_hlr="$builddir/../../src/osmo-hlr"
+cfg="$srcdir/osmo-hlr.cfg"
+
+dump_sorted_schema(){
+	db_file="$1"
+	tables="$(sqlite3 -batch -noheader "$db_file" "SELECT name FROM sqlite_master WHERE type = 'table' order by name")"
+	for table in $tables; do
+		echo
+		echo "Table: $table"
+		sqlite3 -batch -header "$db_file" "SELECT name,type,\"notnull\",dflt_value,pk FROM PRAGMA_TABLE_INFO('$table') order by name;"
+		echo
+		echo "Table $table contents:"
+		columns="$(sqlite3 -batch -noheader "$db_file" "SELECT name FROM PRAGMA_TABLE_INFO('$table') order by name;")"
+		sqlite3 -batch -header "$db_file" "SELECT $(echo $columns | sed 's/ /,/g') from $table;"
+	done
+}
+
+rm -f "$db"
+echo "Creating db in schema version 0"
+sqlite3 -batch "$db" < "$srcdir/hlr_db_v0.sql"
+
+echo
+echo "Version 0 db:"
+dump_sorted_schema "$db"
+
+set +e
+
+echo
+echo "Launching osmo-hlr to upgrade db:"
+echo osmo-hlr --database '$db' --db-upgrade --db-check --config-file '$srcdir/osmo-hlr.cfg'
+"$osmo_hlr" --database "$db" --db-upgrade --db-check --config-file "$cfg" >log 2>&1
+echo "rc = $?"
+cat log | sed 's@[^ "]*/@<PATH>@g'
+
+echo
+echo "Resulting db:"
+dump_sorted_schema "$db"
+
+echo
+echo "Verify that osmo-hlr can open it:"
+echo osmo-hlr --database '$db' --db-check --config-file '$srcdir/osmo-hlr.cfg'
+"$osmo_hlr" --database "$db" --db-check --config-file "$cfg" >log 2>&1
+echo "rc = $?"
+cat log | sed 's@[^ "]*/@<PATH>@g'
+
+if [ -n "$do_equivalence_test" ]; then
+	# this part requires osmo_interact_vty.py, so this test is not part of the normal run
+	set -e -x
+	mint_db="$builddir/mint.db"
+	rm -f "$mint_db"
+
+	osmo_verify_transcript_vty.py -v \
+		-n OsmoHLR -p 4258 \
+		-r "$osmo_hlr -c $cfg -l $mint_db" \
+		"$srcdir/create_subscribers.vty"
+	sqlite3 -batch "$mint_db" < "$srcdir/create_subscribers_step2.sql"
+
+	set +x
+	test_dump="$builddir/test.dump"
+	mint_dump="$builddir/mint.dump"
+
+	dump_sorted_schema "$db" > "$test_dump"
+	dump_sorted_schema "$mint_db" > "$mint_dump"
+
+	echo
+	echo "Newly created sorted schema is:"
+	cat "$mint_dump"
+	echo
+	echo "Diff to upgraded schema:"
+	diff -u "$mint_dump" "$test_dump"
+	echo "rc=$?"
+fi
+
+rm -f log
+rm -f "$db"

tests/db_upgrade/hlr_db_v0.sql

@@ -0,0 +1,80 @@
+PRAGMA foreign_keys=OFF;
+BEGIN TRANSACTION;
+CREATE TABLE subscriber (
+-- OsmoHLR's DB scheme is modelled roughly after TS 23.008 version 13.3.0
+	id		INTEGER PRIMARY KEY,
+	-- Chapter 2.1.1.1
+	imsi		VARCHAR(15) UNIQUE NOT NULL,
+	-- Chapter 2.1.2
+	msisdn		VARCHAR(15) UNIQUE,
+	-- Chapter 2.2.3: Most recent / current IMEI
+	imeisv		VARCHAR,
+	-- Chapter 2.4.5
+	vlr_number	VARCHAR(15),
+	-- Chapter 2.4.6
+	hlr_number	VARCHAR(15),
+	-- Chapter 2.4.8.1
+	sgsn_number	VARCHAR(15),
+	-- Chapter 2.13.10
+	sgsn_address	VARCHAR,
+	-- Chapter 2.4.8.2
+	ggsn_number	VARCHAR(15),
+	-- Chapter 2.4.9.2
+	gmlc_number	VARCHAR(15),
+	-- Chapter 2.4.23
+	smsc_number	VARCHAR(15),
+	-- Chapter 2.4.24
+	periodic_lu_tmr	INTEGER,
+	-- Chapter 2.13.115
+	periodic_rau_tau_tmr INTEGER,
+	-- Chapter 2.1.1.2: network access mode
+	nam_cs		BOOLEAN NOT NULL DEFAULT 1,
+	nam_ps		BOOLEAN NOT NULL DEFAULT 1,
+	-- Chapter 2.1.8
+	lmsi		INTEGER,
+	-- The below purged flags might not even be stored non-volatile,
+	-- refer to TS 23.012 Chapter 3.6.1.4
+	-- Chapter 2.7.5
+	ms_purged_cs	BOOLEAN NOT NULL DEFAULT 0,
+	-- Chapter 2.7.6
+	ms_purged_ps	BOOLEAN NOT NULL DEFAULT 0
+);
+INSERT INTO subscriber VALUES(1,'123456789012345','098765432109876',NULL,'MSC-1',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,1,NULL,0,0);
+INSERT INTO subscriber VALUES(2,'111111111',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,1,NULL,1,0);
+INSERT INTO subscriber VALUES(3,'222222222','22222',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,1,NULL,0,1);
+INSERT INTO subscriber VALUES(4,'333333','3',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,1,NULL,0,0);
+INSERT INTO subscriber VALUES(5,'444444444444444','4444',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,0,NULL,0,0);
+INSERT INTO subscriber VALUES(6,'5555555','55555555555555',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,NULL,0,0);
+CREATE TABLE subscriber_apn (
+	subscriber_id	INTEGER,		-- subscriber.id
+	apn		VARCHAR(256) NOT NULL
+);
+CREATE TABLE subscriber_multi_msisdn (
+-- Chapter 2.1.3
+	subscriber_id	INTEGER,		-- subscriber.id
+	msisdn		VARCHAR(15) NOT NULL
+);
+CREATE TABLE auc_2g (
+	subscriber_id	INTEGER PRIMARY KEY,	-- subscriber.id
+	algo_id_2g	INTEGER NOT NULL,	-- enum osmo_auth_algo value
+	ki		VARCHAR(32) NOT NULL	-- hex string: subscriber's secret key (128bit)
+);
+INSERT INTO auc_2g VALUES(1,1,'BeefedCafeFaceAcedAddedDecadeFee');
+INSERT INTO auc_2g VALUES(4,2,'33333333333333333333333333333333');
+INSERT INTO auc_2g VALUES(6,4,'55555555555555555555555555555555');
+CREATE TABLE auc_3g (
+	subscriber_id	INTEGER PRIMARY KEY,	-- subscriber.id
+	algo_id_3g	INTEGER NOT NULL,	-- enum osmo_auth_algo value
+	k		VARCHAR(32) NOT NULL,	-- hex string: subscriber's secret key (128bit)
+	op		VARCHAR(32),		-- hex string: operator's secret key (128bit)
+	opc		VARCHAR(32),		-- hex string: derived from OP and K (128bit)
+	sqn		INTEGER NOT NULL DEFAULT 0,	-- sequence number of key usage
+	-- nr of index bits at lower SQN end
+	ind_bitlen	INTEGER NOT NULL DEFAULT 5
+);
+INSERT INTO auc_3g VALUES(1,5,'C01ffedC1cadaeAc1d1f1edAcac1aB0a',NULL,'CededEffacedAceFacedBadFadedBeef',0,5);
+INSERT INTO auc_3g VALUES(5,5,'44444444444444444444444444444444','44444444444444444444444444444444',NULL,0,5);
+INSERT INTO auc_3g VALUES(6,5,'55555555555555555555555555555555',NULL,'55555555555555555555555555555555',0,5);
+CREATE UNIQUE INDEX idx_subscr_imsi ON subscriber (imsi)
+;
+COMMIT;

tests/db_upgrade/osmo-hlr.cfg

@@ -0,0 +1,6 @@
+log stderr
+ logging level db notice
+ logging print category-hex 0
+ logging print file 0
+ logging print category 1
+ logging color 0

tests/gsup/Makefile.am

@@ -1,41 +0,0 @@
-AM_CPPFLAGS = \
-	$(all_includes) \
-	-I$(top_srcdir)/src \
-	$(NULL)
-
-AM_CFLAGS = \
-	-Wall \
-	-ggdb3 \
-	$(LIBOSMOCORE_CFLAGS) \
-	$(LIBOSMOGSM_CFLAGS) \
-	$(LIBOSMOABIS_CFLAGS) \
-	$(NULL)
-
-AM_LDFLAGS = \
-	$(NULL)
-
-EXTRA_DIST = \
-	gsup_test.ok \
-	gsup_test.err \
-	$(NULL)
-
-noinst_PROGRAMS = \
-	gsup_test \
-	$(NULL)
-
-gsup_test_SOURCES = \
-	gsup_test.c \
-	$(NULL)
-
-gsup_test_LDADD = \
-	$(top_srcdir)/src/luop.c \
-	$(top_srcdir)/src/gsup_server.c \
-	$(top_srcdir)/src/gsup_router.c \
-	$(LIBOSMOCORE_LIBS) \
-	$(LIBOSMOGSM_LIBS) \
-	$(LIBOSMOABIS_LIBS) \
-	$(NULL)
-
-.PHONY: update_exp
-update_exp:
-	$(builddir)/gsup_test >"$(srcdir)/gsup_test.ok" 2>"$(srcdir)/gsup_test.err"

tests/gsup/gsup_test.c

@@ -1,91 +0,0 @@
-/* (C) 2018 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
- *
- * All Rights Reserved
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- */
-
-#include <string.h>
-
-#include <osmocom/core/logging.h>
-#include <osmocom/core/utils.h>
-#include <osmocom/core/application.h>
-#include <osmocom/gsm/gsup.h>
-
-#include "logging.h"
-#include "luop.h"
-
-struct osmo_gsup_server;
-
-/* override osmo_gsup_addr_send() to not actually send anything. */
-int osmo_gsup_addr_send(struct osmo_gsup_server *gs,
-			const uint8_t *addr, size_t addrlen,
-			struct msgb *msg)
-{
-	LOGP(DMAIN, LOGL_DEBUG, "%s\n", msgb_hexdump(msg));
-	msgb_free(msg);
-	return 0;
-}
-
-int db_subscr_get_by_imsi(struct db_context *dbc, const char *imsi,
-			  struct hlr_subscriber *subscr)
-{
-	return 0;
-}
-
-/* Verify that the internally allocated msgb is large enough */
-void test_gsup_tx_insert_subscr_data()
-{
-	struct lu_operation luop = {
-		.state = LU_S_LU_RECEIVED,
-		.subscr = {
-			.imsi = "123456789012345",
-			.msisdn = "987654321098765",
-			.nam_cs = true,
-			.nam_ps = true,
-		},
-		.is_ps = true,
-	};
-
-	lu_op_tx_insert_subscr_data(&luop);
-}
-
-const struct log_info_cat default_categories[] = {
-	[DMAIN] = {
-		.name = "DMAIN",
-		.description = "Main Program",
-		.enabled = 1, .loglevel = LOGL_DEBUG,
-	},
-};
-
-static struct log_info info = {
-	.cat = default_categories,
-	.num_cat = ARRAY_SIZE(default_categories),
-};
-
-int main(int argc, char **argv)
-{
-	void *ctx = talloc_named_const(NULL, 0, "gsup_test");
-	osmo_init_logging2(ctx, &info);
-	log_set_print_filename(osmo_stderr_target, 0);
-	log_set_print_timestamp(osmo_stderr_target, 0);
-	log_set_use_color(osmo_stderr_target, 0);
-	log_set_print_category(osmo_stderr_target, 1);
-
-	test_gsup_tx_insert_subscr_data();
-
-	printf("Done.\n");
-	return EXIT_SUCCESS;
-}

tests/gsup/gsup_test.err

@@ -1,2 +0,0 @@
-DMAIN 10 01 08 21 43 65 87 09 21 43 f5 08 09 08 89 67 45 23 01 89 67 f5 05 07 10 01 01 12 02 01 2a 28 01 01 
-DMAIN LU OP state change: LU RECEIVED -> ISD SENT

tests/gsup/gsup_test.ok

@@ -1 +0,0 @@
-Done.

tests/gsup_server/Makefile.am

@@ -1,17 +1,18 @@
 AM_CPPFLAGS = \
 	$(all_includes) \
-	-I$(top_srcdir)/src \
 	$(NULL)
 
 AM_CFLAGS = \
 	-Wall \
 	-ggdb3 \
+	-I$(top_srcdir)/include \
 	$(LIBOSMOCORE_CFLAGS) \
 	$(LIBOSMOGSM_CFLAGS) \
 	$(LIBOSMOABIS_CFLAGS) \
 	$(NULL)
 
 AM_LDFLAGS = \
+	-no-install \
 	$(NULL)
 
 EXTRA_DIST = \

tests/gsup_server/gsup_server_test.c

@@ -20,7 +20,7 @@
 
 #include <stdio.h>
 #include <osmocom/core/utils.h>
-#include "gsup_server.h"
+#include <osmocom/hlr/gsup_server.h>
 
 #define comment_start() printf("\n===== %s\n", __func__)
 #define comment_end() printf("===== %s: SUCCESS\n\n", __func__)

tests/test_nodes.vty

@@ -1,75 +1,47 @@
+OsmoHLR> ?
+  show        Show running system information
+  list        Print command list
+  exit        Exit current mode and down to previous mode
+  help        Description of the interactive help system
+  enable      Turn on privileged mode command
+  terminal    Set terminal line parameters
+  who         Display who is on vty
+  logging     Configure logging
+  no          Negate a command or set its defaults
+  logp        Print a message on all log outputs; useful for placing markers in test logs
+  subscriber  Subscriber management commands
 OsmoHLR> list
-  show version
-  show online-help
-  list
-  exit
-  help
-  enable
-  terminal length <0-512>
-  terminal no length
-  who
-  show history
-  logging enable
 ...
-  show logging vty
-  show alarms
-  show talloc-context (application|all) (full|brief|DEPTH)
-  show talloc-context (application|all) (full|brief|DEPTH) tree ADDRESS
-  show talloc-context (application|all) (full|brief|DEPTH) filter REGEXP
   show gsup-connections
-  subscriber (imsi|msisdn|id) IDENT show
+  subscriber (imsi|msisdn|id|imei) IDENT show
+  show subscriber (imsi|msisdn|id|imei) IDENT
 
 OsmoHLR> enable
-OsmoHLR# list
-  help
-  list
-  write terminal
-  write file
-  write memory
-  write
-  show running-config
-  exit
-  disable
-  configure terminal
-  copy running-config startup-config
-  show startup-config
-  show version
-  show online-help
-  terminal length <0-512>
-  terminal no length
-  who
-  show history
-  terminal monitor
-  terminal no monitor
-  logging enable
+OsmoHLR# ?
 ...
+  subscriber  Subscriber management commands
 
 OsmoHLR# configure terminal
+OsmoHLR(config)# ?
+...
+  hlr       Configure the HLR
 OsmoHLR(config)# list
-  help
-  list
-  write terminal
-  write file
-  write memory
-  write
-  show running-config
-  exit
-  end
 ...
   hlr
 
 OsmoHLR(config)# hlr
+OsmoHLR(config-hlr)# ?
+...
+  euse                         Configure a particular External USSD Entity
+  no                           Negate a command or set its defaults
+  ussd                         USSD Configuration
+  ncss-guard-timeout           Set guard timer for NCSS (call independent SS) session activity
+  store-imei                   Save the IMEI in the database when receiving Check IMEI requests. Note that an MSC does not necessarily send Check IMEI requests (for OsmoMSC, you may want to set 'check-imei-rqd 1').
+  subscriber-create-on-demand  Make a new record when a subscriber is first seen.
 OsmoHLR(config-hlr)# list
-  help
-  list
-  write terminal
-  write file
-  write memory
-  write
-  show running-config
-  exit
-  end
+...
   gsup
+  database PATH
   euse NAME
   no euse NAME
   ussd route prefix PREFIX internal (own-msisdn|own-imsi)
@@ -77,18 +49,18 @@ OsmoHLR(config-hlr)# list
   no ussd route prefix PREFIX
   ussd default-route external EUSE
   no ussd default-route
+  ncss-guard-timeout <0-255>
+  store-imei
+  no store-imei
+  subscriber-create-on-demand (no-msisdn|<3-15>) (none|cs|ps|cs+ps)
+  no subscriber-create-on-demand
 
 OsmoHLR(config-hlr)# gsup
+OsmoHLR(config-hlr-gsup)# ?
+...
+  bind   Listen/Bind related socket option
 OsmoHLR(config-hlr-gsup)# list
-  help
-  list
-  write terminal
-  write file
-  write memory
-  write
-  show running-config
-  exit
-  end
+...
   bind ip A.B.C.D
 
 OsmoHLR(config-hlr-gsup)# exit
@@ -96,6 +68,7 @@ OsmoHLR(config-hlr)# exit
 OsmoHLR(config)# exit
 OsmoHLR# configure terminal
 OsmoHLR(config)# hlr
+OsmoHLR(config-hlr)# store-imei
 OsmoHLR(config-hlr)# gsup
 OsmoHLR(config-hlr-gsup)# end
 OsmoHLR# disable
@@ -107,25 +80,17 @@ Current configuration:
 !
 !
 log stderr
-  logging filter all 1
-  logging color 1
-  logging print category 1
-  logging print extended-timestamp 1
-  logging print file 1
-  logging level all notice
-  logging level main notice
-  logging level db notice
-  logging level auc notice
-  logging level ss info
 ...
-!
-line vty
- no login
-!
-ctrl
- bind 127.0.0.1
+ logging level main notice
+ logging level db notice
+ logging level auc notice
+ logging level ss info
+...
 hlr
+ store-imei
+ database hlr_vty_test.db
  gsup
   bind ip 127.0.0.1
  ussd route prefix *#100# internal own-msisdn
+ ussd route prefix *#101# internal own-imsi
 end

tests/test_subscr_create_on_demand.vty

@@ -0,0 +1,43 @@
+OsmoHLR> enable
+OsmoHLR# configure terminal
+OsmoHLR(config)# hlr
+
+OsmoHLR(config-hlr)# subscriber-create-on-demand no-msisdn none
+OsmoHLR(config-hlr)# show running-config
+...
+hlr
+...
+ subscriber-create-on-demand no-msisdn none
+...
+
+OsmoHLR(config-hlr)# subscriber-create-on-demand 3 none
+OsmoHLR(config-hlr)# show running-config
+...
+hlr
+...
+ subscriber-create-on-demand 3 none
+...
+
+OsmoHLR(config-hlr)# subscriber-create-on-demand 4 cs
+OsmoHLR(config-hlr)# show running-config
+...
+hlr
+...
+ subscriber-create-on-demand 4 cs
+...
+
+OsmoHLR(config-hlr)# subscriber-create-on-demand 5 ps
+OsmoHLR(config-hlr)# show running-config
+...
+hlr
+...
+ subscriber-create-on-demand 5 ps
+...
+
+OsmoHLR(config-hlr)# subscriber-create-on-demand 6 cs+ps
+OsmoHLR(config-hlr)# show running-config
+...
+hlr
+...
+ subscriber-create-on-demand 6 cs+ps
+...

tests/test_subscriber.vty

@@ -2,14 +2,18 @@ OsmoHLR> enable
 
 OsmoHLR# list
 ...
-  subscriber (imsi|msisdn|id) IDENT show
+  subscriber (imsi|msisdn|id|imei) IDENT show
+  show subscriber (imsi|msisdn|id|imei) IDENT
   subscriber imsi IDENT create
-  subscriber (imsi|msisdn|id) IDENT delete
-  subscriber (imsi|msisdn|id) IDENT update msisdn MSISDN
-  subscriber (imsi|msisdn|id) IDENT update aud2g none
-  subscriber (imsi|msisdn|id) IDENT update aud2g (comp128v1|comp128v2|comp128v3|xor) ki KI
-  subscriber (imsi|msisdn|id) IDENT update aud3g none
-  subscriber (imsi|msisdn|id) IDENT update aud3g milenage k K (op|opc) OP_C [ind-bitlen] [<0-28>]
+  subscriber (imsi|msisdn|id|imei) IDENT delete
+  subscriber (imsi|msisdn|id|imei) IDENT update msisdn (none|MSISDN)
+  subscriber (imsi|msisdn|id|imei) IDENT update aud2g none
+  subscriber (imsi|msisdn|id|imei) IDENT update aud2g (comp128v1|comp128v2|comp128v3|xor) ki KI
+  subscriber (imsi|msisdn|id|imei) IDENT update aud3g none
+  subscriber (imsi|msisdn|id|imei) IDENT update aud3g milenage k K (op|opc) OP_C [ind-bitlen] [<0-28>]
+  subscriber (imsi|msisdn|id|imei) IDENT update aud3g xor k K [ind-bitlen] [<0-28>]
+  subscriber (imsi|msisdn|id|imei) IDENT update imei (none|IMEI)
+  subscriber (imsi|msisdn|id|imei) IDENT update network-access-mode (none|cs|ps|cs+ps)
 
 OsmoHLR# subscriber?
   subscriber  Subscriber management commands
@@ -18,20 +22,36 @@ OsmoHLR# subscriber ?
   imsi    Identify subscriber by IMSI
   msisdn  Identify subscriber by MSISDN (phone number)
   id      Identify subscriber by database ID
+  imei    Identify subscriber by IMEI
 
 OsmoHLR# subscriber imsi ?
-  IDENT  IMSI/MSISDN/ID of the subscriber
+  IDENT  IMSI/MSISDN/ID/IMEI of the subscriber
 OsmoHLR# subscriber msisdn ?
-  IDENT  IMSI/MSISDN/ID of the subscriber
+  IDENT  IMSI/MSISDN/ID/IMEI of the subscriber
 OsmoHLR# subscriber id ?
-  IDENT  IMSI/MSISDN/ID of the subscriber
+  IDENT  IMSI/MSISDN/ID/IMEI of the subscriber
+OsmoHLR# subscriber imei ?
+  IDENT  IMSI/MSISDN/ID/IMEI of the subscriber
 
 OsmoHLR# subscriber imsi 123456789023000 show
 % No subscriber for imsi = '123456789023000'
-OsmoHLR# subscriber id 1 show
-% No subscriber for id = '1'
+OsmoHLR# subscriber id 101 show
+% No subscriber for id = '101'
 OsmoHLR# subscriber msisdn 12345 show
 % No subscriber for msisdn = '12345'
+OsmoHLR# subscriber imei 357613004448485 show
+% Checksum validated and stripped for search: imei = '35761300444848'
+% No subscriber for imei = '35761300444848'
+
+OsmoHLR# show subscriber imsi 123456789023000
+% No subscriber for imsi = '123456789023000'
+OsmoHLR# show subscriber id 101
+% No subscriber for id = '101'
+OsmoHLR# show subscriber msisdn 12345
+% No subscriber for msisdn = '12345'
+OsmoHLR# show subscriber imei 357613004448485
+% Checksum validated and stripped for search: imei = '35761300444848'
+% No subscriber for imei = '35761300444848'
 
 OsmoHLR# subscriber imsi 1234567890230001 create
 % Not a valid IMSI: 1234567890230001
@@ -42,16 +62,16 @@ OsmoHLR# subscriber imsi 12345 create
 
 OsmoHLR# subscriber imsi 123456789023000 create
 % Created subscriber 123456789023000
-    ID: 1
+    ID: 101
     IMSI: 123456789023000
     MSISDN: none
 
 OsmoHLR# subscriber imsi 123456789023000 show
-    ID: 1
+    ID: 101
     IMSI: 123456789023000
     MSISDN: none
-OsmoHLR# subscriber id 1 show
-    ID: 1
+OsmoHLR# subscriber id 101 show
+    ID: 101
     IMSI: 123456789023000
     MSISDN: none
 OsmoHLR# subscriber msisdn 12345 show
@@ -61,15 +81,15 @@ OsmoHLR# subscriber imsi 123456789023000 update msisdn 12345
 % Updated subscriber IMSI='123456789023000' to MSISDN='12345'
 
 OsmoHLR# subscriber imsi 123456789023000 show
-    ID: 1
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 12345
-OsmoHLR# subscriber id 1 show
-    ID: 1
+OsmoHLR# subscriber id 101 show
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 12345
 OsmoHLR# subscriber msisdn 12345 show
-    ID: 1
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 12345
 
@@ -78,23 +98,44 @@ OsmoHLR# subscriber msisdn 12345 update msisdn 423
 OsmoHLR# subscriber msisdn 12345 show
 % No subscriber for msisdn = '12345'
 
+OsmoHLR# subscriber msisdn 423 update msisdn none
+% Updated subscriber IMSI='123456789023000': removed MSISDN
+OsmoHLR# subscriber msisdn 423 show
+% No subscriber for msisdn = '423'
 OsmoHLR# subscriber imsi 123456789023000 show
-    ID: 1
+    ID: 101
+    IMSI: 123456789023000
+    MSISDN: none
+OsmoHLR# subscriber imsi 123456789023000 update msisdn 423
+% Updated subscriber IMSI='123456789023000' to MSISDN='423'
+OsmoHLR# subscriber msisdn 423 show
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 423
-OsmoHLR# subscriber id 1 show
-    ID: 1
+
+OsmoHLR# subscriber imsi 123456789023000 show
+    ID: 101
+    IMSI: 123456789023000
+    MSISDN: 423
+OsmoHLR# subscriber id 101 show
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 423
 OsmoHLR# subscriber msisdn 423 show
-    ID: 1
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 423
 
 OsmoHLR# subscriber imsi 123456789023000 update ?
-  msisdn  Set MSISDN (phone number) of the subscriber
-  aud2g   Set 2G authentication data
-  aud3g   Set UMTS authentication data (3G, and 2G with UMTS AKA)
+  msisdn               Set MSISDN (phone number) of the subscriber
+  aud2g                Set 2G authentication data
+  aud3g                Set UMTS authentication data (3G, and 2G with UMTS AKA)
+  imei                 Set IMEI of the subscriber (normally populated from MSC, no need to set this manually)
+  network-access-mode  Set Network Access Mode (NAM) of the subscriber
+
+OsmoHLR# subscriber imsi 123456789023000 update msisdn ?
+  none    Remove MSISDN (phone number)
+  MSISDN  New MSISDN (phone number)
 
 OsmoHLR# subscriber imsi 123456789023000 update aud2g ?
   none       Delete 2G authentication data
@@ -114,7 +155,7 @@ OsmoHLR# subscriber imsi 123456789023000 update aud2g comp128v1 ki val ?
 
 OsmoHLR# subscriber imsi 123456789023000 update aud2g xor ki Deaf0ff1ceD0d0DabbedD1ced1ceF00d
 OsmoHLR# subscriber imsi 123456789023000 show
-    ID: 1
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 423
     2G auth: XOR
@@ -122,39 +163,39 @@ OsmoHLR# subscriber imsi 123456789023000 show
 
 OsmoHLR# subscriber imsi 123456789023000 update aud2g comp128v1 ki BeefedCafeFaceAcedAddedDecadeFee
 OsmoHLR# subscriber imsi 123456789023000 show
-    ID: 1
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 423
     2G auth: COMP128v1
              KI=beefedcafefaceacedaddeddecadefee
-OsmoHLR# subscriber id 1 show
-    ID: 1
+OsmoHLR# subscriber id 101 show
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 423
     2G auth: COMP128v1
              KI=beefedcafefaceacedaddeddecadefee
 OsmoHLR# subscriber msisdn 423 show
-    ID: 1
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 423
     2G auth: COMP128v1
              KI=beefedcafefaceacedaddeddecadefee
 
-OsmoHLR# subscriber id 1 update aud2g comp128v2 ki CededEffacedAceFacedBadFadedBeef
-OsmoHLR# subscriber id 1 show
-    ID: 1
+OsmoHLR# subscriber id 101 update aud2g comp128v2 ki CededEffacedAceFacedBadFadedBeef
+OsmoHLR# subscriber id 101 show
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 423
     2G auth: COMP128v2
              KI=cededeffacedacefacedbadfadedbeef
 OsmoHLR# subscriber msisdn 423 show
-    ID: 1
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 423
     2G auth: COMP128v2
              KI=cededeffacedacefacedbadfadedbeef
 OsmoHLR# subscriber imsi 123456789023000 show
-    ID: 1
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 423
     2G auth: COMP128v2
@@ -162,63 +203,63 @@ OsmoHLR# subscriber imsi 123456789023000 show
 
 OsmoHLR# subscriber msisdn 423 update aud2g comp128v3 ki C01ffedC1cadaeAc1d1f1edAcac1aB0a
 OsmoHLR# subscriber msisdn 423 show
-    ID: 1
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 423
     2G auth: COMP128v3
              KI=c01ffedc1cadaeac1d1f1edacac1ab0a
 OsmoHLR# subscriber imsi 123456789023000 show
-    ID: 1
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 423
     2G auth: COMP128v3
              KI=c01ffedc1cadaeac1d1f1edacac1ab0a
-OsmoHLR# subscriber id 1 show
-    ID: 1
+OsmoHLR# subscriber id 101 show
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 423
     2G auth: COMP128v3
              KI=c01ffedc1cadaeac1d1f1edacac1ab0a
 
-OsmoHLR# subscriber id 1 update aud2g nonsense ki BeefedCafeFaceAcedAddedDecadeFee
+OsmoHLR# subscriber id 101 update aud2g nonsense ki BeefedCafeFaceAcedAddedDecadeFee
 % Unknown command.
-OsmoHLR# subscriber id 1 show
-    ID: 1
+OsmoHLR# subscriber id 101 show
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 423
     2G auth: COMP128v3
              KI=c01ffedc1cadaeac1d1f1edacac1ab0a
 
-OsmoHLR# subscriber id 1 update aud2g milenage ki BeefedCafeFaceAcedAddedDecadeFee
+OsmoHLR# subscriber id 101 update aud2g milenage ki BeefedCafeFaceAcedAddedDecadeFee
 % Unknown command.
-OsmoHLR# subscriber id 1 show
-    ID: 1
+OsmoHLR# subscriber id 101 show
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 423
     2G auth: COMP128v3
              KI=c01ffedc1cadaeac1d1f1edacac1ab0a
 
-OsmoHLR# subscriber id 1 update aud2g xor ki CoiffedCicadaeAcidifiedAcaciaBoa
+OsmoHLR# subscriber id 101 update aud2g xor ki CoiffedCicadaeAcidifiedAcaciaBoa
 % Invalid value for KI: 'CoiffedCicadaeAcidifiedAcaciaBoa'
-OsmoHLR# subscriber id 1 show
-    ID: 1
+OsmoHLR# subscriber id 101 show
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 423
     2G auth: COMP128v3
              KI=c01ffedc1cadaeac1d1f1edacac1ab0a
 
-OsmoHLR# subscriber id 1 update aud2g xor ki C01ffedC1cadaeAc1d1f1edAcac1aB0aX
+OsmoHLR# subscriber id 101 update aud2g xor ki C01ffedC1cadaeAc1d1f1edAcac1aB0aX
 % Invalid value for KI: 'C01ffedC1cadaeAc1d1f1edAcac1aB0aX'
-OsmoHLR# subscriber id 1 show
-    ID: 1
+OsmoHLR# subscriber id 101 show
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 423
     2G auth: COMP128v3
              KI=c01ffedc1cadaeac1d1f1edacac1ab0a
 
-OsmoHLR# subscriber id 1 update aud2g none
-OsmoHLR# subscriber id 1 show
-    ID: 1
+OsmoHLR# subscriber id 101 update aud2g none
+OsmoHLR# subscriber id 101 show
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 423
 
@@ -226,6 +267,7 @@ OsmoHLR# subscriber id 1 show
 OsmoHLR# subscriber imsi 123456789023000 update aud3g ?
   none      Delete 3G authentication data
   milenage  Use Milenage algorithm
+  xor       Use XOR algorithm
 
 OsmoHLR# subscriber imsi 123456789023000 update aud3g milenage ?
   k  Set Encryption Key K
@@ -248,7 +290,7 @@ OsmoHLR# subscriber imsi 123456789023000 update aud3g milenage k Deaf0ff1ceD0d0D
 
 OsmoHLR# subscriber imsi 123456789023000 update aud3g milenage k Deaf0ff1ceD0d0DabbedD1ced1ceF00d opc CededEffacedAceFacedBadFadedBeef
 OsmoHLR# subscriber imsi 123456789023000 show
-    ID: 1
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 423
     3G auth: MILENAGE
@@ -259,7 +301,7 @@ OsmoHLR# subscriber imsi 123456789023000 show
 
 OsmoHLR# subscriber imsi 123456789023000 update aud3g milenage k Deaf0ff1ceD0d0DabbedD1ced1ceF00d op DeafBeddedBabeAcceededFadedDecaf
 OsmoHLR# subscriber imsi 123456789023000 show
-    ID: 1
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 423
     3G auth: MILENAGE
@@ -269,13 +311,13 @@ OsmoHLR# subscriber imsi 123456789023000 show
 
 OsmoHLR# subscriber imsi 123456789023000 update aud3g none
 OsmoHLR# subscriber imsi 123456789023000 show
-    ID: 1
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 423
 
 OsmoHLR# subscriber imsi 123456789023000 update aud3g milenage k Deaf0ff1ceD0d0DabbedD1ced1ceF00d opc CededEffacedAceFacedBadFadedBeef ind-bitlen 23
 OsmoHLR# subscriber imsi 123456789023000 show
-    ID: 1
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 423
     3G auth: MILENAGE
@@ -286,7 +328,7 @@ OsmoHLR# subscriber imsi 123456789023000 show
 OsmoHLR# subscriber imsi 123456789023000 update aud3g milenage k CoiffedCicadaeAcidifiedAcaciaBoa opc CededEffacedAceFacedBadFadedBeef
 % Invalid value for K: 'CoiffedCicadaeAcidifiedAcaciaBoa'
 OsmoHLR# subscriber imsi 123456789023000 show
-    ID: 1
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 423
     3G auth: MILENAGE
@@ -297,7 +339,7 @@ OsmoHLR# subscriber imsi 123456789023000 show
 OsmoHLR# subscriber imsi 123456789023000 update aud3g milenage k Deaf0ff1ceD0d0DabbedD1ced1ceF00d opc CoiffedCicadaeAcidifiedAcaciaBoa
 % Invalid value for OPC: 'CoiffedCicadaeAcidifiedAcaciaBoa'
 OsmoHLR# subscriber imsi 123456789023000 show
-    ID: 1
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 423
     3G auth: MILENAGE
@@ -309,7 +351,7 @@ OsmoHLR# subscriber imsi 123456789023000 update aud3g milenage k Deaf0ff1ceD0d0D
 OsmoHLR# subscriber imsi 123456789023000 update aud3g milenage k Deaf0ff1ceD0d0DabbedD1ced1ceF00d op CoiffedCicadaeAcidifiedAcaciaBoa
 % Invalid value for OP: 'CoiffedCicadaeAcidifiedAcaciaBoa'
 OsmoHLR# subscriber imsi 123456789023000 show
-    ID: 1
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 423
     3G auth: MILENAGE
@@ -317,9 +359,9 @@ OsmoHLR# subscriber imsi 123456789023000 show
              OP=c01ffedc1cadaeac1d1f1edacac1ab0a
              IND-bitlen=5
 
-OsmoHLR# subscriber id 1 update aud2g comp128v2 ki CededEffacedAceFacedBadFadedBeef
-OsmoHLR# subscriber id 1 show
-    ID: 1
+OsmoHLR# subscriber id 101 update aud2g comp128v2 ki CededEffacedAceFacedBadFadedBeef
+OsmoHLR# subscriber id 101 show
+    ID: 101
     IMSI: 123456789023000
     MSISDN: 423
     2G auth: COMP128v2
@@ -334,16 +376,100 @@ OsmoHLR# subscriber imsi 123456789023000 delete
 
 OsmoHLR# subscriber imsi 123456789023000 show
 % No subscriber for imsi = '123456789023000'
-OsmoHLR# subscriber id 1 show
-% No subscriber for id = '1'
+OsmoHLR# subscriber id 101 show
+% No subscriber for id = '101'
 OsmoHLR# subscriber msisdn 423 show
 % No subscriber for msisdn = '423'
 
 OsmoHLR# subscriber imsi 123456789023000 create
 % Created subscriber 123456789023000
-    ID: 1
+    ID: 101
     IMSI: 123456789023000
     MSISDN: none
 
 OsmoHLR# subscriber imsi 123456789023000 delete
 % Deleted subscriber for IMSI '123456789023000'
+
+OsmoHLR# subscriber imsi 123456789023000 create
+% Created subscriber 123456789023000
+    ID: 101
+    IMSI: 123456789023000
+    MSISDN: none
+
+OsmoHLR# subscriber imsi 123456789023000 update imei ?
+  none  Forget IMEI
+  IMEI  Set IMEI (use for debug only!)
+
+OsmoHLR# subscriber imsi 123456789023000 update imei 35761300444848
+% Updated subscriber IMSI='123456789023000' to IMEI='35761300444848'
+
+OsmoHLR# subscriber imsi 123456789023000 update imei 357613004448484
+% IMEI invalid: '357613004448484'
+
+OsmoHLR# subscriber imsi 123456789023000 update imei 357613004448485
+% Updated subscriber IMSI='123456789023000' to IMEI='35761300444848'
+
+OsmoHLR# show subscriber imei 35761300444848
+    ID: 101
+    IMSI: 123456789023000
+    MSISDN: none
+    IMEI: 357613004448485
+
+OsmoHLR# show subscriber imei 357613004448485
+% Checksum validated and stripped for search: imei = '35761300444848'
+    ID: 101
+    IMSI: 123456789023000
+    MSISDN: none
+    IMEI: 357613004448485
+
+OsmoHLR# show subscriber imei 357613004448484
+% No subscriber for imei = '357613004448484'
+
+OsmoHLR# subscriber imsi 123456789023000 update imei none
+% Updated subscriber IMSI='123456789023000': removed IMEI
+
+OsmoHLR# subscriber imsi 123456789023000 show
+    ID: 101
+    IMSI: 123456789023000
+    MSISDN: none
+
+OsmoHLR# subscriber imsi 123456789023000 delete
+% Deleted subscriber for IMSI '123456789023000'
+
+OsmoHLR# show subscriber id 99
+    ID: 99
+    IMSI: 000000000000099
+    MSISDN: none
+    IMEI: 12345 (INVALID LENGTH!)
+
+OsmoHLR# subscriber imsi 123456789023000 create
+% Created subscriber 123456789023000
+    ID: 101
+    IMSI: 123456789023000
+    MSISDN: none
+OsmoHLR# subscriber imsi 123456789023000 update network-access-mode none
+OsmoHLR# subscriber imsi 123456789023000 show
+    ID: 101
+    IMSI: 123456789023000
+    MSISDN: none
+    CS disabled
+    PS disabled
+OsmoHLR# subscriber imsi 123456789023000 update network-access-mode cs
+OsmoHLR# subscriber imsi 123456789023000 show
+    ID: 101
+    IMSI: 123456789023000
+    MSISDN: none
+    PS disabled
+OsmoHLR# subscriber imsi 123456789023000 update network-access-mode ps
+OsmoHLR# subscriber imsi 123456789023000 show
+    ID: 101
+    IMSI: 123456789023000
+    MSISDN: none
+    CS disabled
+OsmoHLR# subscriber imsi 123456789023000 update network-access-mode cs+ps
+OsmoHLR# subscriber imsi 123456789023000 show
+    ID: 101
+    IMSI: 123456789023000
+    MSISDN: none
+OsmoHLR# subscriber imsi 123456789023000 delete
+% Deleted subscriber for IMSI '123456789023000'

tests/test_subscriber.vty.sql

@@ -0,0 +1,6 @@
+-- Subscriber with invalid IMEI length
+INSERT INTO subscriber (id, imsi, imei) VALUES(99, '000000000000099', '12345');
+
+-- Dummy entry with ID=100 gives all subscribers created in the VTY test an
+-- ID > 100, so we can pre-fill the database with IDs < 100.
+INSERT INTO subscriber (id, imsi) VALUES(100, '000000000000100');

tests/testsuite.at

@@ -15,13 +15,6 @@ cat $abs_srcdir/auc/auc_ts_55_205_test_sets.err > experr
 AT_CHECK([$abs_top_builddir/tests/auc/auc_ts_55_205_test_sets], [], [expout], [experr])
 AT_CLEANUP
 
-AT_SETUP([gsup])
-AT_KEYWORDS([gsup])
-cat $abs_srcdir/gsup/gsup_test.ok > expout
-cat $abs_srcdir/gsup/gsup_test.err > experr
-AT_CHECK([$abs_top_builddir/tests/gsup/gsup_test], [], [expout], [experr])
-AT_CLEANUP
-
 AT_SETUP([gsup_server])
 AT_KEYWORDS([gsup_server])
 cat $abs_srcdir/gsup_server/gsup_server_test.ok > expout
@@ -36,3 +29,13 @@ cat $abs_srcdir/db/db_test.err > experr
 sqlite3 db_test.db < $abs_top_srcdir/sql/hlr.sql
 AT_CHECK([$abs_top_builddir/tests/db/db_test], [], [expout], [experr])
 AT_CLEANUP
+
+# AT_SKIP_IF: disable for old sqlite versions, because the way we dump tables in the test doesn't work with it.
+# https://lists.osmocom.org/pipermail/openbsc/2019-November/013063.html
+AT_SETUP([db_upgrade])
+AT_SKIP_IF([ ! pkg-config sqlite3 --exists --atleast-version=3.16.2 ])
+AT_KEYWORDS([db_upgrade])
+cat $abs_srcdir/db_upgrade/db_upgrade_test.ok > expout
+cat $abs_srcdir/db_upgrade/db_upgrade_test.err > experr
+AT_CHECK([$abs_srcdir/db_upgrade/db_upgrade_test.sh $abs_srcdir/db_upgrade $abs_builddir/db_upgrade], [], [expout], [experr])
+AT_CLEANUP