Makefile.am: fix pkgconfig_DATA

Fix building debian packages:
	dh_install: Cannot find (any matches for) "usr/lib/*/pkgconfig/libosmo-mslookup.pc" (tried in "." and "debian/tmp")
	dh_install: libosmo-mslookup-dev missing files: usr/lib/*/pkgconfig/libosmo-mslookup.pc
	dh_install: missing files, aborting

Fixes: 637bbfcd92 ("add osmo-mslookup-client program")
Change-Id: Ib0bce2d09b41834f7331969eaf7c57a9787f7efb

.gitignore

@@ -2,6 +2,8 @@
 *.lo
 *.la
 *.db
+*.db-shm
+*.db-wal
 *.pyc
 .*.sw?
 .version
@@ -34,10 +36,12 @@ src/osmo-hlr
 src/osmo-hlr-db-tool
 src/osmo-euse-demo
 src/gsupclient/gsup-test-client
+src/mslookup/osmo-mslookup-client
 
 tests/atconfig
 tests/testsuite
 tests/testsuite.log
+tests/testsuite.dir
 
 tests/auc/auc_3g_test
 tests/auc/auc_ts_55_205_test_sets.c
@@ -47,6 +51,11 @@ tests/gsup_server/gsup_server_test
 tests/gsup/gsup_test
 tests/db/db_test
 tests/hlr_vty_test.db*
+tests/db_upgrade/*.dump
+tests/mslookup/mdns_test
+tests/mslookup/mslookup_client_mdns_test
+tests/mslookup/mslookup_client_test
+tests/mslookup/mslookup_test
 
 # manuals
 doc/manuals/*.html

Makefile.am

@@ -17,7 +17,8 @@ AM_DISTCHECK_CONFIGURE_FLAGS = \
 	--with-systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir)
 
 pkgconfigdir = $(libdir)/pkgconfig
-pkgconfig_DATA = libosmo-gsup-client.pc
+pkgconfig_DATA = libosmo-gsup-client.pc \
+		 libosmo-mslookup.pc
 
 @RELMAKE@
 

configure.ac

@@ -34,10 +34,10 @@ PKG_PROG_PKG_CONFIG([0.20])
 
 PKG_CHECK_MODULES(TALLOC, [talloc >= 2.0.1])
 
-PKG_CHECK_MODULES(LIBOSMOCORE, libosmocore >= 1.0.0)
-PKG_CHECK_MODULES(LIBOSMOGSM, libosmogsm >= 1.0.0)
-PKG_CHECK_MODULES(LIBOSMOVTY, libosmovty >= 1.0.0)
-PKG_CHECK_MODULES(LIBOSMOCTRL, libosmoctrl >= 1.0.0)
+PKG_CHECK_MODULES(LIBOSMOCORE, libosmocore >= 1.3.0)
+PKG_CHECK_MODULES(LIBOSMOGSM, libosmogsm >= 1.3.0)
+PKG_CHECK_MODULES(LIBOSMOVTY, libosmovty >= 1.3.0)
+PKG_CHECK_MODULES(LIBOSMOCTRL, libosmoctrl >= 1.3.0)
 PKG_CHECK_MODULES(LIBOSMOABIS, libosmoabis >= 0.6.0)
 
 PKG_CHECK_MODULES(SQLITE3, sqlite3)
@@ -59,6 +59,21 @@ then
 	CPPFLAGS="$CPPFLAGS -fsanitize=address -fsanitize=undefined"
 fi
 
+AC_ARG_ENABLE([sqlite_talloc],
+		AC_HELP_STRING([--enable-sqlite-talloc],
+				[Configure SQLite3 to use talloc memory allocator [default=no]]),
+		[sqlite_talloc="$enableval"],[sqlite_talloc="no"])
+if test "x$sqlite_talloc" = "xyes" ; then
+	# Older versions of SQLite3 (at least 3.8.2) become unstable with talloc.
+	# Feel free to relax to 3.24.0 > VER > 3.8.2 if it works for you.
+	# FIXME: PKG_CHECK_MODULES() may return cached result here!
+	PKG_CHECK_MODULES(SQLITE3, sqlite3 >= 3.24.0)
+	AC_DEFINE([SQLITE_USE_TALLOC], 1, [Use talloc for SQLite3])
+fi
+AC_MSG_CHECKING([whether to use talloc for SQLite3])
+AC_MSG_RESULT([$sqlite_talloc])
+AM_CONDITIONAL([DB_SQLITE_DEBUG], [test "x$sqlite_talloc" = "xyes"])
+
 AC_ARG_ENABLE(werror,
 	[AS_HELP_STRING(
 		[--enable-werror],
@@ -159,16 +174,23 @@ AC_OUTPUT(
 	doc/examples/Makefile
 	src/Makefile
 	src/gsupclient/Makefile
+	src/mslookup/Makefile
 	include/Makefile
+	include/osmocom/Makefile
+	include/osmocom/hlr/Makefile
+	include/osmocom/mslookup/Makefile
 	libosmo-gsup-client.pc
+	libosmo-mslookup.pc
 	sql/Makefile
 	doc/manuals/Makefile
 	contrib/Makefile
 	contrib/systemd/Makefile
+	contrib/dgsm/Makefile
 	tests/Makefile
 	tests/auc/Makefile
 	tests/auc/gen_ts_55_205_test_sets/Makefile
 	tests/gsup_server/Makefile
-	tests/gsup/Makefile
 	tests/db/Makefile
+	tests/db_upgrade/Makefile
+	tests/mslookup/Makefile
 	)

contrib/Makefile.am

@@ -1 +1,4 @@
-SUBDIRS = systemd
+SUBDIRS = \
+	systemd \
+	dgsm \
+	$(NULL)

contrib/dgsm/Makefile.am

@@ -0,0 +1,4 @@
+EXTRA_DIST = \
+	osmo-mslookup-pipe.py \
+	osmo-mslookup-socket.py \
+	$(NULL)

contrib/dgsm/osmo-mslookup-pipe.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+# vim: shiftwidth=4 tabstop=4 expandtab
+import subprocess
+import json
+
+def query_mslookup(query_str):
+    result = {'result': 'not-found'}
+    proc = subprocess.Popen(('osmo-mslookup-client', '-f', 'json', query_str),
+		            stdout=subprocess.PIPE)
+    for line in iter(proc.stdout.readline,''):
+        if not line:
+            break
+        response = json.loads(line)
+        if response.get('result') == 'result':
+                result = response
+        print('Response: %r' % response)
+    return result
+
+if __name__ == '__main__':
+    import sys
+    query_str = '1000-5000@sip.voice.12345.msisdn'
+    if len(sys.argv) > 1:
+        query_str = sys.argv[1]
+    print('Final result: %r' % query_mslookup(query_str))

contrib/dgsm/osmo-mslookup-socket.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+# vim: shiftwidth=4 tabstop=4 expandtab
+import socket
+import time
+
+MSLOOKUP_SOCKET_PATH = '/tmp/mslookup'
+
+def query_mslookup_socket(query_str, socket_path=MSLOOKUP_SOCKET_PATH):
+    mslookup_socket = socket.socket(socket.AF_UNIX, socket.SOCK_SEQPACKET)
+    mslookup_socket.setblocking(True)
+    mslookup_socket.connect(socket_path)
+    result = {'result': 'not-found'}
+    column_names = mslookup_socket.recv(1024).decode('ascii')
+    if not column_names:
+        return result
+    column_names = column_names.split('\t')
+    mslookup_socket.sendall(query_str.encode('ascii'))
+    while True:
+        csv = mslookup_socket.recv(1024).decode('ascii')
+        if not csv:
+            break
+        response = dict(zip(column_names, csv.split('\t')))
+        if response.get('result') == 'result':
+            result = response
+        print('Response: %r' % response)
+    return result
+
+if __name__ == '__main__':
+    import sys
+    print(
+        '\nPlease run separately: osmo-mslookup-client --socket /tmp/mslookup -d\n')
+    query_str = '1000-5000@sip.voice.12345.msisdn'
+    if len(sys.argv) > 1:
+        query_str = sys.argv[1]
+    print('Final result: %r' % query_mslookup_socket(query_str))

contrib/jenkins.sh

@@ -58,4 +58,5 @@ if [ "$WITH_MANUALS" = "1" ] && [ "$PUBLISH" = "1" ]; then
 	make -C "$base/doc/manuals" publish
 fi
 
+$MAKE maintainer-clean
 osmo-clean-workspace.sh

contrib/systemd/osmo-hlr.service

@@ -1,5 +1,6 @@
 [Unit]
 Description=Osmocom Home Location Register (OsmoHLR)
+Documentation=https://osmocom.org/projects/osmo-hlr/wiki/OsmoHLR
 
 [Service]
 Type=simple

debian/changelog

@@ -1,3 +1,111 @@
+osmo-hlr (1.2.0) unstable; urgency=medium
+
+  [ Ruben Undheim ]
+  * Fix test for return codes on mipsel and alpha archs
+
+  [ Thorsten Alteholz ]
+  * fix spelling errors detected by lintian
+
+  [ Pau Espin Pedrol ]
+  * tests: Fix db_test err file to expect error code name instead of value
+
+  [ Oliver Smith ]
+  * tests/test_nodes.vty: check less libosmocore cmds
+  * tests/db_upgrade: disable for old sqlite versions
+  * gitignore: add tests/db_upgrade/*.dump
+  * gsup_client.h: fix license header: GPLv2+
+  * tests/auc: change back to python3
+
+  [ Neels Hofmeyr ]
+  * fix double free in osmo_gsup_client_enc_send()
+  * db upgrade to v2: log version 2, not 1
+  * fix upgrade to version 2: imei column default value
+  * add --db-check option
+  * hlr.sql: move comment
+  * add db_upgrade test
+  * hlr db schema 3: hlr_number -> msc_number
+  * db.c: code dup: add db_run_statements() for arrays of statements
+  * move headers to include/osmocom/hlr
+  * fix upgrade test in presence of ~/.sqliterc
+  * db upgrade: remove some code dup
+  * add osmo_gsup_msgb_alloc()
+  * Makefile convenience: add VTY_TEST var to run only one test
+  * remove gsup_test
+  * test_nodes.vty: tweak: add some '?' checks
+  * db v4: add column last_lu_seen_ps
+
+  [ Harald Welte ]
+  * AUC: Add support for setting the AMF separation bit to '1' for EUTRAN
+  * hlr: exit(2) on unsupported positional arguments on command line
+
+ -- Pau Espin Pedrol <pespin@sysmocom.de>  Fri, 03 Jan 2020 12:37:35 +0100
+
+osmo-hlr (1.1.0) unstable; urgency=medium
+
+  [ Oliver Smith ]
+  * docs: running: document --db-upgrade
+  * Add IMEI column to subscriber table
+  * Optionally store IMEI in subscriber table
+  * VTY tests: fill DB before running test
+  * VTY: integrate IMEI
+  * hlr.c: replace deprecated osmo_gsup_get_err_msg_type()
+  * hlr.c: move hlr_ctx to the top
+  * tests: use -no-install libtool flag to avoid ./lt-* scripts
+  * Cosmetic: gsup_route_find: comment addr, addrlen
+  * USSD: save MO USSD's originating MSC's vlr_number
+  * USSD: don't use gsm0480_msgb_alloc_name()
+  * hlr.c: forward GSUP messages between clients
+  * db_hlr.c: db_subscr_create(): add flags argument
+  * db_hlr.c: add db_subscr_exists_by_imsi()
+  * Create subscribers on demand
+  * Document subscribers create on demand feature
+  * debian: create -doc subpackage with pdf manuals
+  * db_test: set timezone to work around mktime bug
+  * db_hlr: zero-initialize "struct tm"
+  * rx_check_imei_req(): fix IMEI bounds checking
+  * contrib/jenkins.sh: run "make maintainer-clean"
+  * VTY: add subscriber update network-access-mode
+  * manuals: improve subscribers create on demand
+  * gitignore: ignore everything generated in db_test
+  * db_auc.c: verify hex key sizes read from DB
+
+  [ Max ]
+  * Log ip:port when adding GSUP routes
+  * Add link to project wiki to .service file
+  * Enable statsd support
+
+  [ Vadim Yanitskiy ]
+  * hlr.c: properly terminate the process on SIGTERM
+  * hlr.c: fix: also store the session state in read_cb_forward()
+  * hlr.c: fix: properly print the original message type in read_cb_forward()
+  * hlr.c: check the presence of msgb->l2h in read_cb()
+  * hlr.c: fix possible msgb memleaks in read_cb()
+  * db_hlr.c: add db_subscr_exists_by_msisdn()
+  * src/db.h: use GSM23003_MSISDN_MAX_DIGITS for MSISDN buffer size
+  * src/hlr.c: fix deprecation warning: use gsm48_decode_bcd_number2()
+  * hlr_ussd.c: fix: properly pass invokeID in handle_ussd_own_msisdn()
+  * hlr_ussd.c: rx_proc_ss_req(): fix NULL pointer dereference
+  * build: fix mess with 'db_test_SOURCES' and 'db_test_LDADD'
+  * tests/db_test: close the database when test is finished
+  * src/db.c: integrate SQLite3 with talloc allocator
+
+  [ Neels Hofmeyr ]
+  * USSD: fix routing to multiple MSC
+  * fix error logging for GSUP route
+  * add missing error log: invalid IMSI
+  * osmo-hlr: allow configuring db path from cfg file
+  * use new OSMO_IMSI_BUF_SIZE
+
+  [ Daniel Willmann ]
+  * manuals: Add script to update vty/counter documentation from docker
+  * manuals: Update vty documentation
+
+  [ Pau Espin Pedrol ]
+  * Remove undefined param passed to logging_vty_add_cmds
+  * configure.ac: Require libosmocore 1.2.0
+
+ -- Pau Espin Pedrol <pespin@sysmocom.de>  Wed, 07 Aug 2019 16:14:23 +0200
+
 osmo-hlr (1.0.0) unstable; urgency=medium
 
   [ Stefan Sperling ]

debian/control

@@ -7,12 +7,13 @@ Build-Depends: debhelper (>= 9),
                dh-autoreconf,
                dh-systemd (>= 1.5),
                autotools-dev,
-               python-minimal,
+               python3-minimal,
                libosmocore-dev,
                libosmo-abis-dev,
                libosmo-netif-dev,
                libsqlite3-dev,
-               sqlite3
+               sqlite3,
+               osmo-gsm-manuals-dev
 Standards-Version: 3.9.6
 Vcs-Browser: http://cgit.osmocom.org/osmo-hlr
 Vcs-Git: git://git.osmocom.org/osmo-hlr
@@ -57,3 +58,34 @@ Description: Development headers of Osmocom GSUP client library
   and External USSD Entities (EUSEs) using this library to implement clients.
   .
   This package contains the development headers.
+
+Package: libosmo-mslookup0
+Section: libs
+Architecture: any
+Multi-Arch: same
+Depends: ${shlibs:Depends},
+         ${misc:Depends}
+Pre-Depends: ${misc:Pre-Depends}
+Description: Osmocom MS lookup library
+  This shared library contains routines for looking up mobile subscribers.
+
+Package: libosmo-mslookup-dev
+Architecture: any
+Multi-Arch: same
+Depends: ${misc:Depends},
+	 libosmo-mslookup0 (= ${binary:Version}),
+	 libosmocore-dev
+Pre-Depends: ${misc:Pre-Depends}
+Description: Development headers of Osmocom MS lookup library
+  This shared library contains routines for looking up mobile subscribers.
+  .
+  This package contains the development headers.
+
+Package: osmo-hlr-doc
+Architecture: all
+Section: doc
+Priority: optional
+Depends: ${misc:Depends}
+Description: ${misc:Package} PDF documentation
+ Various manuals: user manual, VTY reference manual and/or
+ protocol/interface manuals.

debian/libosmo-mslookup-dev.install

@@ -0,0 +1,5 @@
+usr/include/osmocom/mslookup
+usr/lib/*/libosmo-mslookup*.a
+usr/lib/*/libosmo-mslookup*.so
+usr/lib/*/libosmo-mslookup*.la
+usr/lib/*/pkgconfig/libosmo-mslookup.pc

debian/libosmo-mslookup0.install

@@ -0,0 +1 @@
+usr/lib/*/libosmo-mslookup*.so.*

debian/osmo-hlr-doc.install

@@ -0,0 +1 @@
+usr/share/doc/osmo-hlr-doc/*.pdf

debian/rules

@@ -17,4 +17,8 @@ override_dh_auto_test:
 	dh_auto_test || (find . -name testsuite.log -exec cat {} \; ; false)
 
 override_dh_auto_configure:
-	dh_auto_configure -- --with-systemdsystemunitdir=/lib/systemd/system
+	dh_auto_configure -- --with-systemdsystemunitdir=/lib/systemd/system --enable-manuals
+
+# Don't create .pdf.gz files (barely saves space and they can't be opened directly by most pdf readers)
+override_dh_compress:
+	dh_compress -X.pdf

doc/manuals/Makefile.am

@@ -4,6 +4,7 @@ EXTRA_DIST = example_subscriber_add_update_delete.vty \
     osmohlr-usermanual.adoc \
     osmohlr-usermanual-docinfo.xml \
     osmohlr-vty-reference.xml \
+    regen_doc.sh \
     chapters \
     vty
 
@@ -15,6 +16,7 @@ if BUILD_MANUALS
   VTY_REFERENCE = osmohlr-vty-reference.xml
   include $(OSMO_GSM_MANUALS_DIR)/build/Makefile.vty-reference.inc
 
+  OSMO_REPOSITORY = osmo-hlr
   include $(OSMO_GSM_MANUALS_DIR)/build/Makefile.common.inc
 endif
 

doc/manuals/chapters/running.adoc

@@ -5,7 +5,7 @@ arguments:
 
 === SYNOPSIS
 
-*osmo-hlr* [-h] [-c 'CONFIGFILE'] [-l 'DATABASE'] [-d 'DBGMASK'] [-D] [-s] [-T] [-e 'LOGLEVEL'] [-V]
+*osmo-hlr* [-h] [-c 'CONFIGFILE'] [-l 'DATABASE'] [-d 'DBGMASK'] [-D] [-s] [-T] [-e 'LOGLEVEL'] [-U] [-V]
 
 === OPTIONS
 
@@ -38,6 +38,11 @@ arguments:
 	Set the global log level for logging to stderr. This has mostly
 	been deprecated by VTY based logging configuration, see
 	<<logging>> for more information.
+*-U, --db-upgrade*::
+	Allow HLR database schema upgrades. If OsmoHLR was updated and
+	requires a newer database schema, it will refuse to start unless
+	this option is specified. The updated database can not be
+	downgraded, make backups as necessary.
 *-V, --version*::
 	Print the compile-time version number of the OsmoHLR program
 

doc/manuals/chapters/subscribers.adoc

@@ -52,7 +52,7 @@ transceiving only RAND and SRES, may be applicable. (See 3GPP TS 33.102, chapter
 |aud3g.ind_bitlen|5|Nr of index bits at lower SQN end
 |apn||
 |vlr_number||3GPP TS 23.008 chapter 2.4.5
-|hlr_number||3GPP TS 23.008 chapter 2.4.6
+|msc_number||3GPP TS 23.008 chapter 2.4.6
 |sgsn_number||3GPP TS 23.008 chapter 2.4.8.1
 |sgsn_address||3GPP TS 23.008 chapter 2.13.10
 |ggsn_number||3GPP TS 23.008 chapter 2.4.8.2
@@ -67,3 +67,63 @@ transceiving only RAND and SRES, may be applicable. (See 3GPP TS 33.102, chapter
 |ms_purged_ps|1|3GPP TS 23.008 chapter 2.7.6
 |===
 
+=== Configuring the Subscribers Create on Demand Feature
+
+Usually a HLR will only allow mobile equipment (ME) on the network, if the HLR
+has a subscriber entry with the ME's IMSI. But OsmoHLR can also be configured to
+automatically create new entries for new IMSIs, with the
+`subscriber-create-on-demand` VTY option. The obvious use case is creating the
+new subscriber entry and then allowing the ME to use both the CS
+(Circuit Switched) and PS (Packet Switched) NAM (Network Access Mode).
+
+.osmo-hlr.cfg
+----
+hlr
+ subscriber-create-on-demand 5 cs+ps
+----
+
+On the other hand, operators might only want to give network access to IMSIs, of
+which they know the owner. In order to do that, one can set the default NAM to
+`none` and manually approve new subscribers by changing the NAM (e.g. over the
+VTY, see the example below).
+
+Oftentimes it is hard to know, which IMSI belongs to which ME, but the IMEI is
+readily available. If you configure your MSC to send IMEI checking requests to
+the HLR, before sending location update requests, the subscribers created on
+demand can also have the IMEI stored in the HLR database. With OsmoMSC, this
+is done by writing `check-imei-rqd early` in the `msc` section of osmo-msc.cfg.
+Then enable storing the IMEI when receiving check IMEI requests with
+`store-imei` in the OsmoHLR configuration.
+
+.osmo-msc.cfg
+----
+msc
+ check-imei-rqd early
+----
+
+.osmo-hlr.cfg
+----
+hlr
+ subscriber-create-on-demand 5 none
+ store-imei
+----
+
+.Example: Enabling CS and PS NAM via VTY for a known IMEI
+----
+OsmoHLR> enable
+OsmoHLR# subscriber imei 35761300444848 show
+    ID: 1
+    IMSI: 123456789023000
+    MSISDN: 58192 <1>
+    IMEI: 35761300444848
+    CS disabled <2>
+    PS disabled <2>
+OsmoHLR# subscriber imei 35761300444848 update network-access-mode cs+ps
+OsmoHLR# subscriber imei 35761300444848 show
+    ID: 1
+    IMSI: 123456789023000
+    MSISDN: 58192
+    IMEI: 35761300444848
+----
+<1> Randomly generated 5 digit MSISDN
+<2> Disabled CS and PS NAM prevent the subscriber from accessing the network

doc/manuals/regen_doc.sh

@@ -0,0 +1,17 @@
+#!/bin/sh -x
+
+if [ -z "$DOCKER_PLAYGROUND" ]; then
+	echo "You need to set DOCKER_PLAYGROUND"
+	exit 1
+fi
+
+SCRIPT=$(realpath "$0")
+MANUAL_DIR=$(dirname "$SCRIPT")
+
+COMMIT=${COMMIT:-$(git log -1 --format=format:%H)}
+
+cd "$DOCKER_PLAYGROUND/scripts" || exit 1
+
+OSMO_HLR_BRANCH=$COMMIT ./regen_doc.sh osmo-hlr 4258 \
+	"$MANUAL_DIR/chapters/counters_generated.adoc" \
+	"$MANUAL_DIR/vty/hlr_vty_reference.xml"

doc/manuals/vty/hlr_vty_reference.xml

@@ -187,7 +187,7 @@
         <param name='MASK' doc='List of logging categories to log, e.g. &apos;abc:mno:xyz&apos;. Available log categories depend on the specific application, refer to the &apos;logging level&apos; command. Optionally add individual log levels like &apos;abc,1:mno,3:xyz,5&apos;, where the level numbers are LOGL_DEBUG=1 LOGL_INFO=3 LOGL_NOTICE=5 LOGL_ERROR=7 LOGL_FATAL=8' />
       </params>
     </command>
-    <command id='logging level (main|db|auc|ss|lglobal|llapd|linp|lmux|lmi|lmib|lsms|lctrl|lgtp|lstats|lgsup|loap|lss7|lsccp|lsua|lm3ua|lmgcp|ljibuf) (debug|info|notice|error|fatal)'>
+    <command id='logging level (main|db|auc|ss|lglobal|llapd|linp|lmux|lmi|lmib|lsms|lctrl|lgtp|lstats|lgsup|loap|lss7|lsccp|lsua|lm3ua|lmgcp|ljibuf|lrspro) (debug|info|notice|error|fatal)'>
       <params>
         <param name='logging' doc='Configure logging' />
         <param name='level' doc='Set the log level for a specified category' />
@@ -213,6 +213,7 @@
         <param name='lm3ua' doc='libosmo-sigtran MTP3 User Adaptation' />
         <param name='lmgcp' doc='libosmo-mgcp Media Gateway Control Protocol' />
         <param name='ljibuf' doc='libosmo-netif Jitter Buffer' />
+        <param name='lrspro' doc='Remote SIM protocol' />
         <param name='debug' doc='Log debug messages and higher levels' />
         <param name='info' doc='Log informational messages and higher levels' />
         <param name='notice' doc='Log noticeable messages and higher levels' />
@@ -302,22 +303,63 @@
         <param name='REGEXP' doc='Regular expression' />
       </params>
     </command>
+    <command id='show stats'>
+      <params>
+        <param name='show' doc='Show running system information' />
+        <param name='stats' doc='Show statistical values' />
+      </params>
+    </command>
+    <command id='show stats level (global|peer|subscriber)'>
+      <params>
+        <param name='show' doc='Show running system information' />
+        <param name='stats' doc='Show statistical values' />
+        <param name='level' doc='Set the maximum group level' />
+        <param name='global' doc='Show global groups only' />
+        <param name='peer' doc='Show global and network peer related groups' />
+        <param name='subscriber' doc='Show global, peer, and subscriber groups' />
+      </params>
+    </command>
+    <command id='show asciidoc counters'>
+      <params>
+        <param name='show' doc='Show running system information' />
+        <param name='asciidoc' doc='Asciidoc generation' />
+        <param name='counters' doc='Generate table of all registered counters' />
+      </params>
+    </command>
+    <command id='show rate-counters'>
+      <params>
+        <param name='show' doc='Show running system information' />
+        <param name='rate-counters' doc='Show all rate counters' />
+      </params>
+    </command>
     <command id='show gsup-connections'>
       <params>
         <param name='show' doc='Show running system information' />
         <param name='gsup-connections' doc='GSUP Connections from VLRs, SGSNs, EUSEs' />
       </params>
     </command>
-    <command id='subscriber (imsi|msisdn|id) IDENT show'>
+    <command id='subscriber (imsi|msisdn|id|imei) IDENT show'>
       <params>
         <param name='subscriber' doc='Subscriber management commands' />
         <param name='imsi' doc='Identify subscriber by IMSI' />
         <param name='msisdn' doc='Identify subscriber by MSISDN (phone number)' />
         <param name='id' doc='Identify subscriber by database ID' />
-        <param name='IDENT' doc='IMSI/MSISDN/ID of the subscriber' />
+        <param name='imei' doc='Identify subscriber by IMEI' />
+        <param name='IDENT' doc='IMSI/MSISDN/ID/IMEI of the subscriber' />
         <param name='show' doc='Show subscriber information' />
       </params>
     </command>
+    <command id='show subscriber (imsi|msisdn|id|imei) IDENT'>
+      <params>
+        <param name='show' doc='Show running system information' />
+        <param name='subscriber' doc='Subscriber management commands' />
+        <param name='imsi' doc='Identify subscriber by IMSI' />
+        <param name='msisdn' doc='Identify subscriber by MSISDN (phone number)' />
+        <param name='id' doc='Identify subscriber by database ID' />
+        <param name='imei' doc='Identify subscriber by IMEI' />
+        <param name='IDENT' doc='IMSI/MSISDN/ID/IMEI of the subscriber' />
+      </params>
+    </command>
   </node>
   <node id='enable'>
     <name>enable</name>
@@ -486,7 +528,7 @@
         <param name='MASK' doc='List of logging categories to log, e.g. &apos;abc:mno:xyz&apos;. Available log categories depend on the specific application, refer to the &apos;logging level&apos; command. Optionally add individual log levels like &apos;abc,1:mno,3:xyz,5&apos;, where the level numbers are LOGL_DEBUG=1 LOGL_INFO=3 LOGL_NOTICE=5 LOGL_ERROR=7 LOGL_FATAL=8' />
       </params>
     </command>
-    <command id='logging level (main|db|auc|ss|lglobal|llapd|linp|lmux|lmi|lmib|lsms|lctrl|lgtp|lstats|lgsup|loap|lss7|lsccp|lsua|lm3ua|lmgcp|ljibuf) (debug|info|notice|error|fatal)'>
+    <command id='logging level (main|db|auc|ss|lglobal|llapd|linp|lmux|lmi|lmib|lsms|lctrl|lgtp|lstats|lgsup|loap|lss7|lsccp|lsua|lm3ua|lmgcp|ljibuf|lrspro) (debug|info|notice|error|fatal)'>
       <params>
         <param name='logging' doc='Configure logging' />
         <param name='level' doc='Set the log level for a specified category' />
@@ -512,6 +554,7 @@
         <param name='lm3ua' doc='libosmo-sigtran MTP3 User Adaptation' />
         <param name='lmgcp' doc='libosmo-mgcp Media Gateway Control Protocol' />
         <param name='ljibuf' doc='libosmo-netif Jitter Buffer' />
+        <param name='lrspro' doc='Remote SIM protocol' />
         <param name='debug' doc='Log debug messages and higher levels' />
         <param name='info' doc='Log informational messages and higher levels' />
         <param name='notice' doc='Log noticeable messages and higher levels' />
@@ -521,8 +564,7 @@
     </command>
     <command id='logging level set-all (debug|info|notice|error|fatal)'>
       <params>
-        <param name='logging' doc='Configure logging' />
-        <param name='level' doc='Set the log level for a specified category' />
+        <param name='logging' doc='Configure logging' />        <param name='level' doc='Set the log level for a specified category' />
         <param name='set-all' doc='Once-off set all categories to the given log level. There is no single command to take back these changes -- each category is set to the given level, period.' />
         <param name='debug' doc='Log debug messages and higher levels' />
         <param name='info' doc='Log informational messages and higher levels' />
@@ -601,22 +643,63 @@
         <param name='REGEXP' doc='Regular expression' />
       </params>
     </command>
+    <command id='show stats'>
+      <params>
+        <param name='show' doc='Show running system information' />
+        <param name='stats' doc='Show statistical values' />
+      </params>
+    </command>
+    <command id='show stats level (global|peer|subscriber)'>
+      <params>
+        <param name='show' doc='Show running system information' />
+        <param name='stats' doc='Show statistical values' />
+        <param name='level' doc='Set the maximum group level' />
+        <param name='global' doc='Show global groups only' />
+        <param name='peer' doc='Show global and network peer related groups' />
+        <param name='subscriber' doc='Show global, peer, and subscriber groups' />
+      </params>
+    </command>
+    <command id='show asciidoc counters'>
+      <params>
+        <param name='show' doc='Show running system information' />
+        <param name='asciidoc' doc='Asciidoc generation' />
+        <param name='counters' doc='Generate table of all registered counters' />
+      </params>
+    </command>
+    <command id='show rate-counters'>
+      <params>
+        <param name='show' doc='Show running system information' />
+        <param name='rate-counters' doc='Show all rate counters' />
+      </params>
+    </command>
     <command id='show gsup-connections'>
       <params>
         <param name='show' doc='Show running system information' />
         <param name='gsup-connections' doc='GSUP Connections from VLRs, SGSNs, EUSEs' />
       </params>
     </command>
-    <command id='subscriber (imsi|msisdn|id) IDENT show'>
+    <command id='subscriber (imsi|msisdn|id|imei) IDENT show'>
       <params>
         <param name='subscriber' doc='Subscriber management commands' />
         <param name='imsi' doc='Identify subscriber by IMSI' />
         <param name='msisdn' doc='Identify subscriber by MSISDN (phone number)' />
         <param name='id' doc='Identify subscriber by database ID' />
-        <param name='IDENT' doc='IMSI/MSISDN/ID of the subscriber' />
+        <param name='imei' doc='Identify subscriber by IMEI' />
+        <param name='IDENT' doc='IMSI/MSISDN/ID/IMEI of the subscriber' />
         <param name='show' doc='Show subscriber information' />
       </params>
     </command>
+    <command id='show subscriber (imsi|msisdn|id|imei) IDENT'>
+      <params>
+        <param name='show' doc='Show running system information' />
+        <param name='subscriber' doc='Subscriber management commands' />
+        <param name='imsi' doc='Identify subscriber by IMSI' />
+        <param name='msisdn' doc='Identify subscriber by MSISDN (phone number)' />
+        <param name='id' doc='Identify subscriber by database ID' />
+        <param name='imei' doc='Identify subscriber by IMEI' />
+        <param name='IDENT' doc='IMSI/MSISDN/ID/IMEI of the subscriber' />
+      </params>
+    </command>
     <command id='subscriber imsi IDENT create'>
       <params>
         <param name='subscriber' doc='Subscriber management commands' />
@@ -625,47 +708,52 @@
         <param name='create' doc='Create subscriber by IMSI' />
       </params>
     </command>
-    <command id='subscriber (imsi|msisdn|id) IDENT delete'>
+    <command id='subscriber (imsi|msisdn|id|imei) IDENT delete'>
       <params>
         <param name='subscriber' doc='Subscriber management commands' />
         <param name='imsi' doc='Identify subscriber by IMSI' />
         <param name='msisdn' doc='Identify subscriber by MSISDN (phone number)' />
         <param name='id' doc='Identify subscriber by database ID' />
-        <param name='IDENT' doc='IMSI/MSISDN/ID of the subscriber' />
+        <param name='imei' doc='Identify subscriber by IMEI' />
+        <param name='IDENT' doc='IMSI/MSISDN/ID/IMEI of the subscriber' />
         <param name='delete' doc='Delete subscriber from database' />
       </params>
     </command>
-    <command id='subscriber (imsi|msisdn|id) IDENT update msisdn MSISDN'>
+    <command id='subscriber (imsi|msisdn|id|imei) IDENT update msisdn (none|MSISDN)'>
       <params>
         <param name='subscriber' doc='Subscriber management commands' />
         <param name='imsi' doc='Identify subscriber by IMSI' />
         <param name='msisdn' doc='Identify subscriber by MSISDN (phone number)' />
         <param name='id' doc='Identify subscriber by database ID' />
-        <param name='IDENT' doc='IMSI/MSISDN/ID of the subscriber' />
+        <param name='imei' doc='Identify subscriber by IMEI' />
+        <param name='IDENT' doc='IMSI/MSISDN/ID/IMEI of the subscriber' />
         <param name='update' doc='Set or update subscriber data' />
         <param name='msisdn' doc='Set MSISDN (phone number) of the subscriber' />
+        <param name='none' doc='Remove MSISDN (phone number)' />
         <param name='MSISDN' doc='New MSISDN (phone number)' />
       </params>
     </command>
-    <command id='subscriber (imsi|msisdn|id) IDENT update aud2g none'>
+    <command id='subscriber (imsi|msisdn|id|imei) IDENT update aud2g none'>
       <params>
         <param name='subscriber' doc='Subscriber management commands' />
         <param name='imsi' doc='Identify subscriber by IMSI' />
         <param name='msisdn' doc='Identify subscriber by MSISDN (phone number)' />
         <param name='id' doc='Identify subscriber by database ID' />
-        <param name='IDENT' doc='IMSI/MSISDN/ID of the subscriber' />
+        <param name='imei' doc='Identify subscriber by IMEI' />
+        <param name='IDENT' doc='IMSI/MSISDN/ID/IMEI of the subscriber' />
         <param name='update' doc='Set or update subscriber data' />
         <param name='aud2g' doc='Set 2G authentication data' />
         <param name='none' doc='Delete 2G authentication data' />
       </params>
     </command>
-    <command id='subscriber (imsi|msisdn|id) IDENT update aud2g (comp128v1|comp128v2|comp128v3|xor) ki KI'>
+    <command id='subscriber (imsi|msisdn|id|imei) IDENT update aud2g (comp128v1|comp128v2|comp128v3|xor) ki KI'>
       <params>
         <param name='subscriber' doc='Subscriber management commands' />
         <param name='imsi' doc='Identify subscriber by IMSI' />
         <param name='msisdn' doc='Identify subscriber by MSISDN (phone number)' />
         <param name='id' doc='Identify subscriber by database ID' />
-        <param name='IDENT' doc='IMSI/MSISDN/ID of the subscriber' />
+        <param name='imei' doc='Identify subscriber by IMEI' />
+        <param name='IDENT' doc='IMSI/MSISDN/ID/IMEI of the subscriber' />
         <param name='update' doc='Set or update subscriber data' />
         <param name='aud2g' doc='Set 2G authentication data' />
         <param name='comp128v1' doc='Use COMP128v1 algorithm' />
@@ -676,25 +764,27 @@
         <param name='KI' doc='Ki as 32 hexadecimal characters' />
       </params>
     </command>
-    <command id='subscriber (imsi|msisdn|id) IDENT update aud3g none'>
+    <command id='subscriber (imsi|msisdn|id|imei) IDENT update aud3g none'>
       <params>
         <param name='subscriber' doc='Subscriber management commands' />
         <param name='imsi' doc='Identify subscriber by IMSI' />
         <param name='msisdn' doc='Identify subscriber by MSISDN (phone number)' />
         <param name='id' doc='Identify subscriber by database ID' />
-        <param name='IDENT' doc='IMSI/MSISDN/ID of the subscriber' />
+        <param name='imei' doc='Identify subscriber by IMEI' />
+        <param name='IDENT' doc='IMSI/MSISDN/ID/IMEI of the subscriber' />
         <param name='update' doc='Set or update subscriber data' />
         <param name='aud3g' doc='Set UMTS authentication data (3G, and 2G with UMTS AKA)' />
         <param name='none' doc='Delete 3G authentication data' />
       </params>
     </command>
-    <command id='subscriber (imsi|msisdn|id) IDENT update aud3g milenage k K (op|opc) OP_C [ind-bitlen] [&lt;0-28&gt;]'>
+    <command id='subscriber (imsi|msisdn|id|imei) IDENT update aud3g milenage k K (op|opc) OP_C [ind-bitlen] [&lt;0-28&gt;]'>
       <params>
         <param name='subscriber' doc='Subscriber management commands' />
         <param name='imsi' doc='Identify subscriber by IMSI' />
         <param name='msisdn' doc='Identify subscriber by MSISDN (phone number)' />
         <param name='id' doc='Identify subscriber by database ID' />
-        <param name='IDENT' doc='IMSI/MSISDN/ID of the subscriber' />
+        <param name='imei' doc='Identify subscriber by IMEI' />
+        <param name='IDENT' doc='IMSI/MSISDN/ID/IMEI of the subscriber' />
         <param name='update' doc='Set or update subscriber data' />
         <param name='aud3g' doc='Set UMTS authentication data (3G, and 2G with UMTS AKA)' />
         <param name='milenage' doc='Use Milenage algorithm' />
@@ -707,6 +797,36 @@
         <param name='[&lt;0-28&gt;]' doc='IND bit length value (default: 5)' />
       </params>
     </command>
+    <command id='subscriber (imsi|msisdn|id|imei) IDENT update imei (none|IMEI)'>
+      <params>
+        <param name='subscriber' doc='Subscriber management commands' />
+        <param name='imsi' doc='Identify subscriber by IMSI' />
+        <param name='msisdn' doc='Identify subscriber by MSISDN (phone number)' />
+        <param name='id' doc='Identify subscriber by database ID' />
+        <param name='imei' doc='Identify subscriber by IMEI' />
+        <param name='IDENT' doc='IMSI/MSISDN/ID/IMEI of the subscriber' />
+        <param name='update' doc='Set or update subscriber data' />
+        <param name='imei' doc='Set IMEI of the subscriber (normally populated from MSC, no need to set this manually)' />
+        <param name='none' doc='Forget IMEI' />
+        <param name='IMEI' doc='Set IMEI (use for debug only!)' />
+      </params>
+    </command>
+    <command id='subscriber (imsi|msisdn|id|imei) IDENT update network-access-mode (none|cs|ps|cs+ps)'>
+      <params>
+        <param name='subscriber' doc='Subscriber management commands' />
+        <param name='imsi' doc='Identify subscriber by IMSI' />
+        <param name='msisdn' doc='Identify subscriber by MSISDN (phone number)' />
+        <param name='id' doc='Identify subscriber by database ID' />
+        <param name='imei' doc='Identify subscriber by IMEI' />
+        <param name='IDENT' doc='IMSI/MSISDN/ID/IMEI of the subscriber' />
+        <param name='update' doc='Set or update subscriber data' />
+        <param name='network-access-mode' doc='Set Network Access Mode (NAM) of the subscriber' />
+        <param name='none' doc='Do not allow access to circuit switched or packet switched services' />
+        <param name='cs' doc='Allow access to circuit switched services only' />
+        <param name='ps' doc='Allow access to packet switched services only' />
+        <param name='cs+ps' doc='Allow access to both circuit and packet switched services' />
+      </params>
+    </command>
   </node>
   <node id='config'>
     <name>config</name>
@@ -883,7 +1003,8 @@
         <param name='user' doc='Generic facility' />
         <param name='uucp' doc='UUCP facility' />
       </params>
-    </command>    <command id='log syslog local &lt;0-7&gt;'>
+    </command>
+    <command id='log syslog local &lt;0-7&gt;'>
       <params>
         <param name='log' doc='Configure logging sub-system' />
         <param name='syslog' doc='Logging via syslog' />
@@ -905,6 +1026,43 @@
         <param name='[HOSTNAME]' doc='Host name to send the GSMTAP logging to (UDP port 4729)' />
       </params>
     </command>
+    <command id='stats reporter statsd'>
+      <params>
+        <param name='stats' doc='Configure stats sub-system' />
+        <param name='reporter' doc='Configure a stats reporter' />
+        <param name='statsd' doc='Report to a STATSD server' />
+      </params>
+    </command>
+    <command id='no stats reporter statsd'>
+      <params>
+        <param name='no' doc='Negate a command or set its defaults' />
+        <param name='stats' doc='Configure stats sub-system' />
+        <param name='reporter' doc='Configure a stats reporter' />
+        <param name='statsd' doc='Report to a STATSD server' />
+      </params>
+    </command>
+    <command id='stats reporter log'>
+      <params>
+        <param name='stats' doc='Configure stats sub-system' />
+        <param name='reporter' doc='Configure a stats reporter' />
+        <param name='log' doc='Report to the logger' />
+      </params>
+    </command>
+    <command id='no stats reporter log'>
+      <params>
+        <param name='no' doc='Negate a command or set its defaults' />
+        <param name='stats' doc='Configure stats sub-system' />
+        <param name='reporter' doc='Configure a stats reporter' />
+        <param name='log' doc='Report to the logger' />
+      </params>
+    </command>
+    <command id='stats interval &lt;1-65535&gt;'>
+      <params>
+        <param name='stats' doc='Configure stats sub-system' />
+        <param name='interval' doc='Set the reporting interval' />
+        <param name='&lt;1-65535&gt;' doc='Interval in seconds' />
+      </params>
+    </command>
     <command id='hlr'>
       <params>
         <param name='hlr' doc='Configure the HLR' />
@@ -985,7 +1143,7 @@
         <param name='[last]' doc='Log source file info at the end of a log line. If omitted, log source file info just before the log text.' />
       </params>
     </command>
-    <command id='logging level (main|db|auc|ss|lglobal|llapd|linp|lmux|lmi|lmib|lsms|lctrl|lgtp|lstats|lgsup|loap|lss7|lsccp|lsua|lm3ua|lmgcp|ljibuf) (debug|info|notice|error|fatal)'>
+    <command id='logging level (main|db|auc|ss|lglobal|llapd|linp|lmux|lmi|lmib|lsms|lctrl|lgtp|lstats|lgsup|loap|lss7|lsccp|lsua|lm3ua|lmgcp|ljibuf|lrspro) (debug|info|notice|error|fatal)'>
       <params>
         <param name='logging' doc='Configure logging' />
         <param name='level' doc='Set the log level for a specified category' />
@@ -1011,6 +1169,7 @@
         <param name='lm3ua' doc='libosmo-sigtran MTP3 User Adaptation' />
         <param name='lmgcp' doc='libosmo-mgcp Media Gateway Control Protocol' />
         <param name='ljibuf' doc='libosmo-netif Jitter Buffer' />
+        <param name='lrspro' doc='Remote SIM protocol' />
         <param name='debug' doc='Log debug messages and higher levels' />
         <param name='info' doc='Log informational messages and higher levels' />
         <param name='notice' doc='Log noticeable messages and higher levels' />
@@ -1051,6 +1210,75 @@
       </params>
     </command>
   </node>
+  <node id='config-stats'>
+    <name>config-stats</name>
+    <command id='local-ip ADDR'>
+      <params>
+        <param name='local-ip' doc='Set the IP address to which we bind locally' />
+        <param name='ADDR' doc='IP Address' />
+      </params>
+    </command>
+    <command id='no local-ip'>
+      <params>
+        <param name='no' doc='Negate a command or set its defaults' />
+        <param name='local-ip' doc='Set the IP address to which we bind locally' />
+      </params>
+    </command>
+    <command id='remote-ip ADDR'>
+      <params>
+        <param name='remote-ip' doc='Set the remote IP address to which we connect' />
+        <param name='ADDR' doc='IP Address' />
+      </params>
+    </command>
+    <command id='remote-port &lt;1-65535&gt;'>
+      <params>
+        <param name='remote-port' doc='Set the remote port to which we connect' />
+        <param name='&lt;1-65535&gt;' doc='Remote port number' />
+      </params>
+    </command>
+    <command id='mtu &lt;100-65535&gt;'>
+      <params>
+        <param name='mtu' doc='Set the maximum packet size' />
+        <param name='&lt;100-65535&gt;' doc='Size in byte' />
+      </params>
+    </command>
+    <command id='no mtu'>
+      <params>
+        <param name='no' doc='Negate a command or set its defaults' />
+        <param name='mtu' doc='Set the maximum packet size' />
+      </params>
+    </command>
+    <command id='prefix PREFIX'>
+      <params>
+        <param name='prefix' doc='Set the item name prefix' />
+        <param name='PREFIX' doc='The prefix string' />
+      </params>
+    </command>
+    <command id='no prefix'>
+      <params>
+        <param name='no' doc='Negate a command or set its defaults' />
+        <param name='prefix' doc='Set the item name prefix' />
+      </params>
+    </command>
+    <command id='level (global|peer|subscriber)'>
+      <params>
+        <param name='level' doc='Set the maximum group level' />
+        <param name='global' doc='Report global groups only' />
+        <param name='peer' doc='Report global and network peer related groups' />
+        <param name='subscriber' doc='Report global, peer, and subscriber groups' />
+      </params>
+    </command>
+    <command id='enable'>
+      <params>
+        <param name='enable' doc='Enable the reporter' />
+      </params>
+    </command>
+    <command id='disable'>
+      <params>
+        <param name='disable' doc='Disable the reporter' />
+      </params>
+    </command>
+  </node>
   <node id='config-line'>
     <name>config-line</name>
     <command id='login'>
@@ -1064,10 +1292,11 @@
         <param name='login' doc='Enable password checking' />
       </params>
     </command>
-    <command id='bind A.B.C.D'>
+    <command id='bind A.B.C.D [&lt;0-65535&gt;]'>
       <params>
         <param name='bind' doc='Accept VTY telnet connections on local interface' />
         <param name='A.B.C.D' doc='Local interface IP address (default: 127.0.0.1)' />
+        <param name='[&lt;0-65535&gt;]' doc='Local TCP port number' />
       </params>
     </command>
   </node>
@@ -1087,6 +1316,12 @@
         <param name='gsup' doc='Configure GSUP options' />
       </params>
     </command>
+    <command id='database PATH'>
+      <params>
+        <param name='database' doc='Set the path to the HLR database file' />
+        <param name='PATH' doc='Relative or absolute file system path to the database file (default is &apos;hlr.db&apos;)' />
+      </params>
+    </command>
     <command id='euse NAME'>
       <params>
         <param name='euse' doc='Configure a particular External USSD Entity' />
@@ -1145,6 +1380,40 @@
         <param name='default-route' doc='Remove the default-route for all USSD to unknown destinations' />
       </params>
     </command>
+    <command id='ncss-guard-timeout &lt;0-255&gt;'>
+      <params>
+        <param name='ncss-guard-timeout' doc='Set guard timer for NCSS (call independent SS) session activity' />
+        <param name='&lt;0-255&gt;' doc='Guard timer value (sec.), or 0 to disable' />
+      </params>
+    </command>
+    <command id='store-imei'>
+      <params>
+        <param name='store-imei' doc='Save the IMEI in the database when receiving Check IMEI requests. Note that an MSC does not necessarily send Check IMEI requests (for OsmoMSC, you may want to set &apos;check-imei-rqd 1&apos;).' />
+      </params>
+    </command>
+    <command id='no store-imei'>
+      <params>
+        <param name='no' doc='Do not save the IMEI in the database, when receiving Check IMEI requests.' />
+        <param name='store-imei' doc='(null)' />
+      </params>
+    </command>
+    <command id='subscriber-create-on-demand (no-msisdn|&lt;3-15&gt;) (none|cs|ps|cs+ps)'>
+      <params>
+        <param name='subscriber-create-on-demand' doc='Make a new record when a subscriber is first seen.' />
+        <param name='no-msisdn' doc='Do not automatically assign MSISDN.' />
+        <param name='&lt;3-15&gt;' doc='Length of an automatically assigned MSISDN.' />
+        <param name='none' doc='Do not allow any NAM (Network Access Mode) by default.' />
+        <param name='cs' doc='Allow access to circuit switched NAM by default.' />
+        <param name='ps' doc='Allow access to packet switched NAM by default.' />
+        <param name='cs+ps' doc='Allow access to circuit and packet switched NAM by default.' />
+      </params>
+    </command>
+    <command id='no subscriber-create-on-demand'>
+      <params>
+        <param name='no' doc='Do not make a new record when a subscriber is first seen.' />
+        <param name='subscriber-create-on-demand' doc='(null)' />
+      </params>
+    </command>
   </node>
   <node id='config-hlr-gsup'>
     <name>config-hlr-gsup</name>

include/Makefile.am

@@ -1,2 +1,11 @@
-nobase_include_HEADERS = osmocom/gsupclient/gsup_client.h
+SUBDIRS = osmocom
 
+nobase_include_HEADERS = \
+	osmocom/gsupclient/gsup_client.h \
+	osmocom/mslookup/mdns.h \
+	osmocom/mslookup/mdns_sock.h \
+	osmocom/mslookup/mslookup_client_fake.h \
+	osmocom/mslookup/mslookup_client.h \
+	osmocom/mslookup/mslookup_client_mdns.h \
+	osmocom/mslookup/mslookup.h \
+	$(NULL)

include/osmocom/Makefile.am

@@ -0,0 +1,4 @@
+SUBDIRS = \
+	hlr \
+	mslookup \
+	$(NULL)

include/osmocom/gsupclient/gsup_client.h

@@ -6,18 +6,17 @@
  * Author: Jacob Erlbeck
  *
  * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
  * (at your option) any later version.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * GNU General Public License for more details.
  *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #pragma once
 

include/osmocom/hlr/Makefile.am

@@ -0,0 +1,14 @@
+noinst_HEADERS = \
+	auc.h \
+	ctrl.h \
+	db.h \
+	gsup_router.h \
+	gsup_server.h \
+	hlr.h \
+	hlr_ussd.h \
+	hlr_vty.h \
+	hlr_vty_subscr.h \
+	logging.h \
+	luop.h \
+	rand.h \
+	$(NULL)

include/osmocom/hlr/db.h

@@ -9,8 +9,10 @@ enum stmt_idx {
 	DB_STMT_SEL_BY_IMSI,
 	DB_STMT_SEL_BY_MSISDN,
 	DB_STMT_SEL_BY_ID,
+	DB_STMT_SEL_BY_IMEI,
 	DB_STMT_UPD_VLR_BY_ID,
 	DB_STMT_UPD_SGSN_BY_ID,
+	DB_STMT_UPD_IMEI_BY_IMSI,
 	DB_STMT_AUC_BY_IMSI,
 	DB_STMT_AUC_UPD_SQN,
 	DB_STMT_UPD_PURGE_CS_BY_IMSI,
@@ -26,6 +28,9 @@ enum stmt_idx {
 	DB_STMT_AUC_3G_INSERT,
 	DB_STMT_AUC_3G_DELETE,
 	DB_STMT_SET_LAST_LU_SEEN,
+	DB_STMT_SET_LAST_LU_SEEN_PS,
+	DB_STMT_EXISTS_BY_IMSI,
+	DB_STMT_EXISTS_BY_MSISDN,
 	_NUM_DB_STMT
 };
 
@@ -35,6 +40,11 @@ struct db_context {
 	sqlite3_stmt *stmt[_NUM_DB_STMT];
 };
 
+/* Optional feature to make SQLite3 using talloc */
+#ifdef SQLITE_USE_TALLOC
+int db_sqlite3_use_talloc(void *ctx);
+#endif
+
 void db_remove_reset(sqlite3_stmt *stmt);
 bool db_bind_text(sqlite3_stmt *stmt, const char *param_name, const char *text);
 bool db_bind_int(sqlite3_stmt *stmt, const char *param_name, int nr);
@@ -56,7 +66,7 @@ int db_update_sqn(struct db_context *dbc, int64_t id,
 int db_get_auc(struct db_context *dbc, const char *imsi,
 	       unsigned int auc_3g_ind, struct osmo_auth_vector *vec,
 	       unsigned int num_vec, const uint8_t *rand_auts,
-	       const uint8_t *auts);
+	       const uint8_t *auts, bool separation_bit);
 
 #include <osmocom/core/linuxlist.h>
 #include <osmocom/gsm/protocol/gsm_23_003.h>
@@ -69,8 +79,9 @@ struct hlr_subscriber {
 
 	int64_t		id;
 	char		imsi[GSM23003_IMSI_MAX_DIGITS+1];
-	char		msisdn[GT_MAX_DIGITS+1];
+	char		msisdn[GSM23003_MSISDN_MAX_DIGITS+1];
 	/* imeisv? */
+	char		imei[GSM23003_IMEI_NUM_DIGITS+1];
 	char		vlr_number[32];
 	char		sgsn_number[32];
 	char		sgsn_address[GT_MAX_DIGITS+1];
@@ -85,6 +96,7 @@ struct hlr_subscriber {
 	bool		ms_purged_cs;
 	bool		ms_purged_ps;
 	time_t		last_lu_seen;
+	time_t		last_lu_seen_ps;
 };
 
 /* A format string for use with strptime(3). This format string is
@@ -115,13 +127,20 @@ struct sub_auth_data_str {
 	} u;
 };
 
-int db_subscr_create(struct db_context *dbc, const char *imsi);
+#define DB_SUBSCR_FLAG_NAM_CS	(1 << 1)
+#define DB_SUBSCR_FLAG_NAM_PS	(1 << 2)
+
+int db_subscr_create(struct db_context *dbc, const char *imsi, uint8_t flags);
 int db_subscr_delete_by_id(struct db_context *dbc, int64_t subscr_id);
 
 int db_subscr_update_msisdn_by_imsi(struct db_context *dbc, const char *imsi,
 				    const char *msisdn);
 int db_subscr_update_aud_by_id(struct db_context *dbc, int64_t subscr_id,
 			       const struct sub_auth_data_str *aud);
+int db_subscr_update_imei_by_imsi(struct db_context *dbc, const char* imsi, const char *imei);
+
+int db_subscr_exists_by_imsi(struct db_context *dbc, const char *imsi);
+int db_subscr_exists_by_msisdn(struct db_context *dbc, const char *msisdn);
 
 int db_subscr_get_by_imsi(struct db_context *dbc, const char *imsi,
 			  struct hlr_subscriber *subscr);
@@ -129,6 +148,7 @@ int db_subscr_get_by_msisdn(struct db_context *dbc, const char *msisdn,
 			    struct hlr_subscriber *subscr);
 int db_subscr_get_by_id(struct db_context *dbc, int64_t id,
 			struct hlr_subscriber *subscr);
+int db_subscr_get_by_imei(struct db_context *dbc, const char *imei, struct hlr_subscriber *subscr);
 int db_subscr_nam(struct db_context *dbc, const char *imsi, bool nam_val, bool is_ps);
 int db_subscr_lu(struct db_context *dbc, int64_t subscr_id,
 		 const char *vlr_or_sgsn_number, bool is_ps);

include/osmocom/hlr/gsup_router.h

@@ -1,11 +1,20 @@
 #pragma once
 
 #include <stdint.h>
-#include "gsup_server.h"
+#include <osmocom/hlr/gsup_server.h>
+
+struct gsup_route {
+	struct llist_head list;
+
+	uint8_t *addr;
+	struct osmo_gsup_conn *conn;
+};
 
 struct osmo_gsup_conn *gsup_route_find(struct osmo_gsup_server *gs,
 					const uint8_t *addr, size_t addrlen);
 
+struct gsup_route *gsup_route_find_by_conn(const struct osmo_gsup_conn *conn);
+
 /* add a new route for the given address to the given conn */
 int gsup_route_add(struct osmo_gsup_conn *conn, const uint8_t *addr, size_t addrlen);
 

include/osmocom/hlr/gsup_server.h

@@ -47,6 +47,7 @@ struct osmo_gsup_conn {
 	bool supports_ps; /* client supports OSMO_GSUP_CN_DOMAIN_PS */
 };
 
+struct msgb *osmo_gsup_msgb_alloc(const char *label);
 
 int osmo_gsup_conn_send(struct osmo_gsup_conn *conn, struct msgb *msg);
 int osmo_gsup_conn_ccm_get(const struct osmo_gsup_conn *clnt, uint8_t **addr,

include/osmocom/hlr/hlr.h

@@ -25,6 +25,8 @@
 #include <stdbool.h>
 #include <osmocom/core/linuxlist.h>
 
+#define HLR_DEFAULT_DB_FILE_PATH "hlr.db"
+
 struct hlr_euse;
 
 struct hlr {
@@ -32,6 +34,7 @@ struct hlr {
 	struct osmo_gsup_server *gs;
 
 	/* DB context */
+	char *db_file_path;
 	struct db_context *dbc;
 
 	/* Control Interface */
@@ -43,7 +46,6 @@ struct hlr {
 
 	struct llist_head euse_list;
 	struct hlr_euse *euse_default;
-	struct llist_head iuse_list;
 
 	/* NCSS (call independent) session guard timeout value */
 	int ncss_guard_timeout;
@@ -51,6 +53,13 @@ struct hlr {
 	struct llist_head ussd_routes;
 
 	struct llist_head ss_sessions;
+
+	bool store_imei;
+
+	bool subscr_create_on_demand;
+	/* Bitmask of DB_SUBSCR_FLAG_* */
+	uint8_t subscr_create_on_demand_flags;
+	unsigned int subscr_create_on_demand_rand_msisdn_len;
 };
 
 extern struct hlr *g_hlr;

include/osmocom/hlr/hlr_ussd.h

@@ -5,7 +5,7 @@
 #include <osmocom/core/linuxlist.h>
 #include <osmocom/gsm/gsup.h>
 
-#include "gsup_server.h"
+#include <osmocom/hlr/gsup_server.h>
 
 #define NCSS_GUARD_TIMEOUT_DEFAULT 30
 

include/osmocom/hlr/hlr_vty.h

@@ -25,7 +25,7 @@
 #include <osmocom/core/logging.h>
 #include <osmocom/vty/vty.h>
 #include <osmocom/vty/command.h>
-#include "hlr.h"
+#include <osmocom/hlr/hlr.h>
 
 enum hlr_vty_node {
 	HLR_NODE = _LAST_OSMOVTY_NODE + 1,
@@ -35,4 +35,4 @@ enum hlr_vty_node {
 
 int hlr_vty_is_config_node(struct vty *vty, int node);
 int hlr_vty_go_parent(struct vty *vty);
-void hlr_vty_init(const struct log_info *cat);
+void hlr_vty_init(void);

include/osmocom/hlr/logging.h

@@ -8,6 +8,7 @@ enum {
 	DGSUP,
 	DAUC,
 	DSS,
+	DMSLOOKUP,
 };
 
 extern const struct log_info hlr_log_info;

include/osmocom/hlr/luop.h

@@ -27,8 +27,8 @@
 #include <osmocom/core/timer.h>
 #include <osmocom/gsm/gsup.h>
 
-#include "db.h"
-#include "gsup_server.h"
+#include <osmocom/hlr/db.h>
+#include <osmocom/hlr/gsup_server.h>
 
 #define CANCEL_TIMEOUT_SECS	30
 #define ISD_TIMEOUT_SECS	30

include/osmocom/mslookup/Makefile.am

@@ -0,0 +1,6 @@
+# most headers here are installed, see /include/Makefile.am
+
+noinst_HEADERS = \
+	mdns_msg.h \
+	mdns_rfc.h \
+	$(NULL)

include/osmocom/mslookup/mdns.h

@@ -0,0 +1,39 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*! \file mdns.h */
+
+#pragma once
+
+#include <osmocom/core/msgb.h>
+#include <osmocom/mslookup/mslookup.h>
+
+#define OSMO_MDNS_DOMAIN_SUFFIX_DEFAULT "mdns.osmocom.org"
+
+struct msgb *osmo_mdns_query_encode(void *ctx, uint16_t packet_id, const struct osmo_mslookup_query *query,
+				    const char *domain_suffix);
+
+struct osmo_mslookup_query *osmo_mdns_query_decode(void *ctx, const uint8_t *data, size_t data_len,
+						   uint16_t *packet_id, const char *domain_suffix);
+
+struct msgb *osmo_mdns_result_encode(void *ctx, uint16_t packet_id, const struct osmo_mslookup_query *query,
+				     const struct osmo_mslookup_result *result, const char *domain_suffix);
+
+int osmo_mdns_result_decode(void *ctx, const uint8_t *data, size_t data_len, uint16_t *packet_id,
+			    struct osmo_mslookup_query *query, struct osmo_mslookup_result *result,
+			    const char *domain_suffix);

include/osmocom/mslookup/mdns_msg.h

@@ -0,0 +1,54 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <stdint.h>
+#include "mdns_rfc.h"
+
+struct osmo_mdns_record {
+	struct llist_head list;
+	enum osmo_mdns_rfc_record_type type;
+	uint16_t length;
+	uint8_t *data;
+};
+
+struct osmo_mdns_msg_request {
+	uint16_t id;
+	char *domain;
+	enum osmo_mdns_rfc_record_type type;
+};
+
+struct osmo_mdns_msg_answer {
+	uint16_t id;
+	char *domain;
+	/*! list of osmo_mdns_record. */
+	struct llist_head records;
+};
+
+int osmo_mdns_msg_request_encode(void *ctx, struct msgb *msg, const struct osmo_mdns_msg_request *req);
+struct osmo_mdns_msg_request *osmo_mdns_msg_request_decode(void *ctx, const uint8_t *data, size_t data_len);
+
+void osmo_mdns_msg_answer_init(struct osmo_mdns_msg_answer *answer);
+int osmo_mdns_msg_answer_encode(void *ctx, struct msgb *msg, const struct osmo_mdns_msg_answer *ans);
+struct osmo_mdns_msg_answer *osmo_mdns_msg_answer_decode(void *ctx, const uint8_t *data, size_t data_len);
+int osmo_mdns_result_from_answer(struct osmo_mslookup_result *result, const struct osmo_mdns_msg_answer *ans);
+
+struct osmo_mdns_record *osmo_mdns_record_txt_keyval_encode(void *ctx, const char *key, const char *value_fmt, ...);
+int osmo_mdns_record_txt_keyval_decode(const struct osmo_mdns_record *rec,
+				       char *key_buf, size_t key_size, char *value_buf, size_t value_size);

include/osmocom/mslookup/mdns_rfc.h

@@ -0,0 +1,113 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <osmocom/core/msgb.h>
+#include <osmocom/core/endian.h>
+#include <osmocom/mslookup/mdns.h>
+
+/* RFC 1035 2.3.4 */
+#define OSMO_MDNS_RFC_MAX_NAME_LEN 255
+
+/* RFC 1035 3.3 <character-string> */
+#define OSMO_MDNS_RFC_MAX_CHARACTER_STRING_LEN 256
+
+enum osmo_mdns_rfc_record_type {
+	OSMO_MDNS_RFC_RECORD_TYPE_UNKNOWN = 0,
+
+	/* RFC 1035 3.2.2 */
+	OSMO_MDNS_RFC_RECORD_TYPE_A = 1, /* IPv4 address */
+	OSMO_MDNS_RFC_RECORD_TYPE_TXT = 16, /* Text strings */
+
+	/* RFC 3596 2.1 */
+	OSMO_MDNS_RFC_RECORD_TYPE_AAAA = 28, /* IPv6 address */
+
+	/* RFC 1035 3.2.3 */
+	OSMO_MDNS_RFC_RECORD_TYPE_ALL = 255, /* Request only: ask for all */
+};
+
+enum osmo_mdns_rfc_class {
+	OSMO_MDNS_RFC_CLASS_UNKNOWN = 0,
+
+	/* RFC 1035 3.2.4 */
+	OSMO_MDNS_RFC_CLASS_IN = 1, /* Internet and IP networks */
+
+	/* RFC 1035 3.2.5 */
+	OSMO_MDNS_RFC_CLASS_ALL = 255, /* Request only: ask for all */
+};
+
+/* RFC 1035 4.1.1 */
+struct osmo_mdns_rfc_header {
+#if OSMO_IS_LITTLE_ENDIAN
+	uint16_t id;
+	uint8_t rd:1,
+		tc:1,
+		aa:1,
+		opcode:4,
+		qr:1; /* QR (0: query, 1: response) */
+	uint8_t rcode:4,
+		z:3,
+		ra:1;
+	uint16_t qdcount; /* Number of questions */
+	uint16_t ancount; /* Number of answers */
+	uint16_t nscount; /* Number of authority records */
+	uint16_t arcount; /* Number of additional records */
+#elif OSMO_IS_BIG_ENDIAN
+/* auto-generated from the little endian part above (libosmocore/contrib/struct_endianess.py) */
+	uint16_t id;
+	uint8_t qr:1, opcode:4, aa:1, tc:1, rd:1;
+	uint8_t ra:1, z:3, rcode:4;
+	uint16_t qdcount;
+	uint16_t ancount;
+	uint16_t nscount;
+	uint16_t arcount;
+#endif
+} __attribute__ ((packed));
+
+/* RFC 1035 4.1.2 */
+struct osmo_mdns_rfc_question {
+	char *domain; /* Domain to be encoded as qname (e.g. "gsup.hlr.1234567.imsi") */
+	enum osmo_mdns_rfc_record_type qtype;
+	enum osmo_mdns_rfc_class qclass;
+};
+
+/* RFC 1035 4.1.3 */
+struct osmo_mdns_rfc_record {
+	char *domain; /* Domain to be encoded as name (e.g. "gsup.hlr.1234567.imsi") */
+	enum osmo_mdns_rfc_record_type type;
+	enum osmo_mdns_rfc_class class;
+	uint32_t ttl;
+	uint16_t rdlength;
+	uint8_t *rdata;
+};
+
+char *osmo_mdns_rfc_qname_encode(void *ctx, const char *domain);
+char *osmo_mdns_rfc_qname_decode(void *ctx, const char *qname, size_t qname_len);
+
+void osmo_mdns_rfc_header_encode(struct msgb *msg, const struct osmo_mdns_rfc_header *hdr);
+int osmo_mdns_rfc_header_decode(const uint8_t *data, size_t data_len, struct osmo_mdns_rfc_header *hdr);
+
+int osmo_mdns_rfc_question_encode(void *ctx, struct msgb *msg, const struct osmo_mdns_rfc_question *qst);
+struct osmo_mdns_rfc_question *osmo_mdns_rfc_question_decode(void *ctx, const uint8_t *data, size_t data_len);
+
+int osmo_mdns_rfc_record_encode(void *ctx, struct msgb *msg, const struct osmo_mdns_rfc_record *rec);
+struct osmo_mdns_rfc_record *osmo_mdns_rfc_record_decode(void *ctx, const uint8_t *data, size_t data_len,
+							 size_t *record_len);

include/osmocom/mslookup/mdns_sock.h

@@ -0,0 +1,33 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+#include <netdb.h>
+#include <osmocom/core/msgb.h>
+#include <osmocom/core/select.h>
+
+struct osmo_mdns_sock {
+	struct osmo_fd osmo_fd;
+	struct addrinfo *ai;
+};
+
+struct osmo_mdns_sock *osmo_mdns_sock_init(void *ctx, const char *ip, unsigned int port,
+					   int (*cb)(struct osmo_fd *fd, unsigned int what),
+					   void *data, unsigned int priv_nr);
+int osmo_mdns_sock_send(const struct osmo_mdns_sock *mdns_sock, struct msgb *msg);
+void osmo_mdns_sock_cleanup(struct osmo_mdns_sock *mdns_sock);

include/osmocom/mslookup/mslookup.h

@@ -0,0 +1,121 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*! \defgroup mslookup Distributed GSM: finding subscribers
+ *  @{
+ * \file mslookup.h
+ */
+
+#pragma once
+
+#include <osmocom/core/utils.h>
+#include <osmocom/core/sockaddr_str.h>
+#include <osmocom/gsm/protocol/gsm_23_003.h>
+
+#define OSMO_MSLOOKUP_SERVICE_MAXLEN 64
+
+bool osmo_mslookup_service_valid(const char *service);
+
+enum osmo_mslookup_id_type {
+	OSMO_MSLOOKUP_ID_NONE = 0,
+	OSMO_MSLOOKUP_ID_IMSI,
+	OSMO_MSLOOKUP_ID_MSISDN,
+};
+
+extern const struct value_string osmo_mslookup_id_type_names[];
+static inline const char *osmo_mslookup_id_type_name(enum osmo_mslookup_id_type val)
+{ return get_value_string(osmo_mslookup_id_type_names, val); }
+
+struct osmo_mslookup_id {
+	enum osmo_mslookup_id_type type;
+	union {
+		char imsi[GSM23003_IMSI_MAX_DIGITS+1];
+		char msisdn[GSM23003_MSISDN_MAX_DIGITS+1];
+	};
+};
+
+int osmo_mslookup_id_cmp(const struct osmo_mslookup_id *a, const struct osmo_mslookup_id *b);
+bool osmo_mslookup_id_valid(const struct osmo_mslookup_id *id);
+
+enum osmo_mslookup_result_code {
+	OSMO_MSLOOKUP_RC_NONE = 0,
+	/*! An intermediate valid result. The request is still open for more results. */
+	OSMO_MSLOOKUP_RC_RESULT,
+	/*! Returned when the final request timeout has elapsed without results. */
+	OSMO_MSLOOKUP_RC_NOT_FOUND,
+};
+
+extern const struct value_string osmo_mslookup_result_code_names[];
+static inline const char *osmo_mslookup_result_code_name(enum osmo_mslookup_result_code val)
+{ return get_value_string(osmo_mslookup_result_code_names, val); }
+
+/*! Information to request from a lookup. */
+struct osmo_mslookup_query {
+	/*! Which service to request, by freely invented names. For service name conventions (for voice, SMS, HLR,...),
+	 * refer to the OsmoHLR user's manual http://ftp.osmocom.org/docs/latest/osmohlr-usermanual.pdf */
+	char service[OSMO_MSLOOKUP_SERVICE_MAXLEN + 1];
+	/*! IMSI or MSISDN to look up. */
+	struct osmo_mslookup_id id;
+
+	/*! Caller provided private data, if desired. */
+	void *priv;
+};
+
+/*! Result data as passed back to a lookup client that invoked an osmo_mslookup_client_request. */
+struct osmo_mslookup_result {
+	/*! Outcome of the request. */
+	enum osmo_mslookup_result_code rc;
+
+	/*! IP address and port to reach the given service via IPv4, if any. */
+	struct osmo_sockaddr_str host_v4;
+
+	/*! IP address and port to reach the given service via IPv6, if any. */
+	struct osmo_sockaddr_str host_v6;
+
+	/*! How long ago the service last verified presence of the subscriber, in seconds, or zero if the presence is
+	 * invariable (like the home HLR record for an IMSI).
+	 * If a subscriber has recently moved to a different location, we get multiple replies and want to choose the
+	 * most recent one. If this were a timestamp, firstly the time zones would need to be taken care of.
+	 * Even if we choose UTC, a service provider with an inaccurate date/time would end up affecting the result.
+	 * The least susceptible to configuration errors or difference in local and remote clock is a value that
+	 * indicates the actual age of the record in seconds. The time that the lookup query took to be answered should
+	 * be neglectable here, since we would typically wait one second (or very few seconds) for lookup replies,
+	 * while typical Location Updating periods are in the range of 15 minutes. */
+	uint32_t age;
+
+	/*! Whether this is the last result returned for this request. */
+	bool last;
+};
+
+int osmo_mslookup_query_init_from_domain_str(struct osmo_mslookup_query *q, const char *domain);
+
+size_t osmo_mslookup_id_name_buf(char *buf, size_t buflen, const struct osmo_mslookup_id *id);
+char *osmo_mslookup_id_name_c(void *ctx, const struct osmo_mslookup_id *id);
+char *osmo_mslookup_id_name_b(char *buf, size_t buflen, const struct osmo_mslookup_id *id);
+
+size_t osmo_mslookup_result_to_str_buf(char *buf, size_t buflen,
+				     const struct osmo_mslookup_query *query,
+				     const struct osmo_mslookup_result *result);
+char *osmo_mslookup_result_name_c(void *ctx,
+				  const struct osmo_mslookup_query *query,
+				  const struct osmo_mslookup_result *result);
+char *osmo_mslookup_result_name_b(char *buf, size_t buflen,
+				  const struct osmo_mslookup_query *query,
+				  const struct osmo_mslookup_result *result);
+
+/*! @} */

include/osmocom/mslookup/mslookup_client.h

@@ -0,0 +1,132 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <osmocom/core/linuxlist.h>
+#include <osmocom/core/timer.h>
+#include <osmocom/core/sockaddr_str.h>
+#include <osmocom/mslookup/mslookup.h>
+
+struct osmo_mslookup_client;
+struct osmo_mslookup_result;
+
+typedef void (*osmo_mslookup_cb_t)(struct osmo_mslookup_client *client,
+				   uint32_t request_handle,
+				   const struct osmo_mslookup_query *query,
+				   const struct osmo_mslookup_result *result);
+
+/*! This handling information is passed along with a lookup request.
+ * It tells the osmo_mslookup_client layer how to handle responses received from various mslookup methods (at the time
+ * of writing only mDNS exists as a method, but the intention is to easily allow adding other methods in the future).
+ * This query handling info is not seen by the individual method implementations, to clarify that it is the
+ * osmo_mslookup_client layer that takes care of these details. */
+struct osmo_mslookup_query_handling {
+	/*! Wait at least this long before returning any results.
+	 *
+	 * If nonzero, result_cb will be called as soon as this delay has elapsed, either with the so far youngest age
+	 * result, or with a "not found yet" result. After this delay has elapsed, receiving results will continue
+	 * until result_timeout_milliseconds has elapsed.
+	 *
+	 * If zero, responses are fed to the result_cb right from the start, every time a younger aged result than
+	 * before comes in.
+	 *
+	 * If a result with age == 0 is received, min_wait_milliseconds is ignored, the result is returned immediately
+	 * and listening for responses ends.
+	 *
+	 * Rationale: If a subscriber has recently moved between sites, multiple results will arrive, and the youngest
+	 * age wins. It can make sense to wait a minimum time for responses before determining the winning result.
+	 *
+	 * However, if no result or no valid result has arrived within a short period, the subscriber may be at a site
+	 * that is far away or that is currently experiencing high latency. It is thus a good safety net to still
+	 * receive results for an extended period of time.
+	 *
+	 * For some services, it is possible to establish links to every received result, and whichever link succeeds
+	 * will be used (for example for SIP calls: first to pick up the call gets connected, the others are dropped
+	 * silently).
+	 */
+	uint32_t min_wait_milliseconds;
+
+	/*! Total time in milliseconds to listen for lookup responses.
+	 *
+	 * When this timeout elapses, osmo_mslookup_client_request_cancel() is called implicitly; Manually invoking
+	 * osmo_mslookup_client_request_cancel() after result_timeout_milliseconds has elapsed is not necessary, but is
+	 * still safe to do anyway.
+	 *
+	 * If zero, min_wait_milliseconds is also used as result_timeout_milliseconds; if that is also zero, a default
+	 * timeout value is used.
+	 *
+	 * If result_timeout_milliseconds <= min_wait_milliseconds, then min_wait_milliseconds is used as
+	 * result_timeout_milliseconds, i.e. the timeout triggers as soon as min_wait_milliseconds hits.
+	 *
+	 * osmo_mslookup_client_request_cancel() can be called any time to end the request.
+	 */
+	uint32_t result_timeout_milliseconds;
+
+	/*! Invoked every time a result with a younger age than the previous result has arrived.
+	 * To stop receiving results before result_timeout_milliseconds has elapsed, call
+	 * osmo_mslookup_client_request_cancel().
+	 */
+	osmo_mslookup_cb_t result_cb;
+};
+
+uint32_t osmo_mslookup_client_request(struct osmo_mslookup_client *client,
+				      const struct osmo_mslookup_query *query,
+				      const struct osmo_mslookup_query_handling *handling);
+
+void osmo_mslookup_client_request_cancel(struct osmo_mslookup_client *client, uint32_t request_handle);
+
+struct osmo_mslookup_client *osmo_mslookup_client_new(void *ctx);
+bool osmo_mslookup_client_active(struct osmo_mslookup_client *client);
+void osmo_mslookup_client_free(struct osmo_mslookup_client *client);
+
+/*! Describe a specific mslookup client method implementation. This struct is only useful for a lookup method
+ * implementation to add itself to an osmo_mslookup_client, see for example osmo_mslookup_client_add_mdns(). */
+struct osmo_mslookup_client_method {
+	struct llist_head entry;
+
+	/*! Human readable name of this lookup method. */
+	const char *name;
+
+	/*! Private data for the lookup method implementation. */
+	void *priv;
+
+	/*! Backpointer to the client this method is added to. */
+	struct osmo_mslookup_client *client;
+
+	/*! Launch a lookup query. Called from osmo_mslookup_client_request().
+	 * The implementation returns results by calling osmo_mslookup_client_rx_result(). */
+	void (*request)(struct osmo_mslookup_client_method *method,
+			const struct osmo_mslookup_query *query,
+			uint32_t request_handle);
+	/*! End a lookup query. Called from osmo_mslookup_client_request_cancel(). It is guaranteed to be called
+	 * exactly once per above request() invocation. (The API user is required to invoke
+	 * osmo_mslookup_client_request_cancel() exactly once per osmo_mslookup_client_request().) */
+	void (*request_cleanup)(struct osmo_mslookup_client_method *method,
+				uint32_t request_handle);
+
+	/*! The mslookup_client is removing this method, clean up all open requests, lists and allocations. */
+	void (*destruct)(struct osmo_mslookup_client_method *method);
+};
+
+void osmo_mslookup_client_method_add(struct osmo_mslookup_client *client,
+				     struct osmo_mslookup_client_method *method);
+bool osmo_mslookup_client_method_del(struct osmo_mslookup_client *client,
+				     struct osmo_mslookup_client_method *method);
+void osmo_mslookup_client_rx_result(struct osmo_mslookup_client *client, uint32_t request_handle,
+				    const struct osmo_mslookup_result *result);

include/osmocom/mslookup/mslookup_client_fake.h

@@ -0,0 +1,34 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+/*! MS lookup fake API for testing purposes. */
+#include <osmocom/mslookup/mslookup_client.h>
+
+struct osmo_mslookup_fake_response {
+	struct timeval time_to_reply;
+	struct osmo_mslookup_id for_id;
+	const char *for_service;
+	struct osmo_mslookup_result result;
+	bool sent;
+};
+
+struct osmo_mslookup_client_method *osmo_mslookup_client_add_fake(struct osmo_mslookup_client *client,
+								  struct osmo_mslookup_fake_response *responses,
+								  size_t responses_len);

include/osmocom/mslookup/mslookup_client_mdns.h

@@ -0,0 +1,38 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <stdint.h>
+
+struct osmo_mslookup_client;
+struct osmo_mslookup_client_method;
+
+/*! MS Lookup mDNS server bind default IP. Taken from the Administratevly Scoped block, particularly the Organizational
+ * Scoped range, https://tools.ietf.org/html/rfc2365 . */
+#define OSMO_MSLOOKUP_MDNS_IP4 "239.192.23.42"
+#define OSMO_MSLOOKUP_MDNS_IP6 "ff08::23:42" // <-- TODO: sane?
+#define OSMO_MSLOOKUP_MDNS_PORT 4266
+
+struct osmo_mslookup_client_method *osmo_mslookup_client_add_mdns(struct osmo_mslookup_client *client, const char *ip,
+								  uint16_t port, int initial_packet_id,
+								  const char *domain_suffix);
+
+const struct osmo_sockaddr_str *osmo_mslookup_client_method_mdns_get_bind_addr(struct osmo_mslookup_client_method *dns_method);
+
+const char *osmo_mslookup_client_method_mdns_get_domain_suffix(struct osmo_mslookup_client_method *dns_method);

libosmo-mslookup.pc.in

@@ -0,0 +1,11 @@
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: Osmocom MS Lookup Library
+Description: C Utility Library
+Version: @VERSION@
+Libs: -L${libdir} @TALLOC_LIBS@ -losmogsm -losmo-mslookup -losmocore
+Cflags: -I${includedir}/
+

sql/hlr.sql

@@ -5,12 +5,14 @@ CREATE TABLE subscriber (
 	imsi		VARCHAR(15) UNIQUE NOT NULL,
 	-- Chapter 2.1.2
 	msisdn		VARCHAR(15) UNIQUE,
-	-- Chapter 2.2.3: Most recent / current IMEI
+	-- Chapter 2.2.3: Most recent / current IMEISV
 	imeisv		VARCHAR,
+	-- Chapter 2.1.9: Most recent / current IMEI
+	imei		VARCHAR(14),
 	-- Chapter 2.4.5
 	vlr_number	VARCHAR(15),
 	-- Chapter 2.4.6
-	hlr_number	VARCHAR(15),
+	msc_number	VARCHAR(15),
 	-- Chapter 2.4.8.1
 	sgsn_number	VARCHAR(15),
 	-- Chapter 2.13.10
@@ -40,7 +42,8 @@ CREATE TABLE subscriber (
 
 	-- Timestamp of last location update seen from subscriber
 	-- The value is a string which encodes a UTC timestamp in granularity of seconds.
-	last_lu_seen TIMESTAMP default NULL
+	last_lu_seen TIMESTAMP default NULL,
+	last_lu_seen_ps TIMESTAMP default NULL
 );
 
 CREATE TABLE subscriber_apn (
@@ -67,11 +70,12 @@ CREATE TABLE auc_3g (
 	op		VARCHAR(32),		-- hex string: operator's secret key (128bit)
 	opc		VARCHAR(32),		-- hex string: derived from OP and K (128bit)
 	sqn		INTEGER NOT NULL DEFAULT 0,	-- sequence number of key usage
-	ind_bitlen	INTEGER NOT NULL DEFAULT 5	-- nr of index bits at lower SQN end
+	-- nr of index bits at lower SQN end
+	ind_bitlen	INTEGER NOT NULL DEFAULT 5
 );
 
 CREATE UNIQUE INDEX idx_subscr_imsi ON subscriber (imsi);
 
 -- Set HLR database schema version number
 -- Note: This constant is currently duplicated in src/db.c and must be kept in sync!
-PRAGMA user_version = 1;
+PRAGMA user_version = 4;

src/Makefile.am

@@ -1,4 +1,7 @@
-SUBDIRS = gsupclient
+SUBDIRS = \
+	gsupclient \
+	mslookup \
+	$(NULL)
 
 AM_CFLAGS = \
 	-Wall \
@@ -11,6 +14,7 @@ AM_CFLAGS = \
 	$(NULL)
 
 AM_CPPFLAGS = -I$(top_srcdir)/include \
+	-I$(top_builddir)/include \
 	$(NULL)
 
 EXTRA_DIST = \
@@ -24,18 +28,6 @@ BUILT_SOURCES = \
 CLEANFILES = $(BUILT_SOURCES)
 
 noinst_HEADERS = \
-	auc.h \
-	db.h \
-	hlr.h \
-	luop.h \
-	gsup_router.h \
-	gsup_server.h \
-	logging.h \
-	rand.h \
-	ctrl.h \
-	hlr_vty.h \
-	hlr_vty_subscr.h \
-	hlr_ussd.h \
 	db_bootstrap.h \
 	$(NULL)
 
@@ -87,21 +79,6 @@ osmo_hlr_db_tool_LDADD = \
 	$(SQLITE3_LIBS) \
 	$(NULL)
 
-db_test_SOURCES = \
-	auc.c \
-	db.c \
-	db_auc.c \
-	db_test.c \
-	logging.c \
-	rand_fake.c \
-	$(NULL)
-
-db_test_LDADD = \
-	$(LIBOSMOCORE_LIBS) \
-	$(LIBOSMOGSM_LIBS) \
-	$(SQLITE3_LIBS) \
-	$(NULL)
-
 osmo_euse_demo_SOURCES = \
 	osmo-euse-demo.c \
 	$(NULL)
@@ -112,6 +89,11 @@ osmo_euse_demo_LDADD = \
 	$(LIBOSMOGSM_LIBS) \
 	$(NULL)
 
+if DB_SQLITE_DEBUG
+osmo_hlr_SOURCES += db_debug.c
+osmo_hlr_db_tool_SOURCES += db_debug.c
+endif
+
 BOOTSTRAP_SQL = $(top_srcdir)/sql/hlr.sql
 
 db_bootstrap.h: $(BOOTSTRAP_SQL) $(srcdir)/db_sql2c.sed

src/auc.c

@@ -23,8 +23,8 @@
 #include <osmocom/core/utils.h>
 #include <osmocom/crypt/auth.h>
 
-#include "logging.h"
-#include "rand.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/rand.h>
 
 #define hexb(buf) osmo_hexdump_nospc((void*)buf, sizeof(buf))
 #define hex(buf,sz) osmo_hexdump_nospc((void*)buf, sz)

src/ctrl.c

@@ -28,9 +28,9 @@
 #include <osmocom/gsm/gsm23003.h>
 #include <osmocom/ctrl/ports.h>
 
-#include "hlr.h"
-#include "ctrl.h"
-#include "db.h"
+#include <osmocom/hlr/hlr.h>
+#include <osmocom/hlr/ctrl.h>
+#include <osmocom/hlr/db.h>
 
 #define SEL_BY "by-"
 #define SEL_BY_IMSI SEL_BY "imsi-"
@@ -95,7 +95,7 @@ static bool get_subscriber(struct db_context *dbc,
 		cmd->reply = "No such subscriber.";
 		return false;
 	default:
-		cmd->reply = "An unknown error has occured during get_subscriber().";
+		cmd->reply = "An unknown error has occurred during get_subscriber().";
 		return false;
 	}
 }

src/db.c

@@ -23,17 +23,18 @@
 #include <sqlite3.h>
 #include <string.h>
 
-#include "logging.h"
-#include "db.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/db.h>
 #include "db_bootstrap.h"
 
 /* This constant is currently duplicated in sql/hlr.sql and must be kept in sync! */
-#define CURRENT_SCHEMA_VERSION	1
+#define CURRENT_SCHEMA_VERSION	4
 
 #define SEL_COLUMNS \
 	"id," \
 	"imsi," \
 	"msisdn," \
+	"imei," \
 	"vlr_number," \
 	"sgsn_number," \
 	"sgsn_address," \
@@ -44,14 +45,17 @@
 	"lmsi," \
 	"ms_purged_cs," \
 	"ms_purged_ps," \
-	"last_lu_seen"
+	"last_lu_seen," \
+	"last_lu_seen_ps" \
 
 static const char *stmt_sql[] = {
 	[DB_STMT_SEL_BY_IMSI] = "SELECT " SEL_COLUMNS " FROM subscriber WHERE imsi = ?",
 	[DB_STMT_SEL_BY_MSISDN] = "SELECT " SEL_COLUMNS " FROM subscriber WHERE msisdn = ?",
 	[DB_STMT_SEL_BY_ID] = "SELECT " SEL_COLUMNS " FROM subscriber WHERE id = ?",
+	[DB_STMT_SEL_BY_IMEI] = "SELECT " SEL_COLUMNS " FROM subscriber WHERE imei = ?",
 	[DB_STMT_UPD_VLR_BY_ID] = "UPDATE subscriber SET vlr_number = $number WHERE id = $subscriber_id",
 	[DB_STMT_UPD_SGSN_BY_ID] = "UPDATE subscriber SET sgsn_number = $number WHERE id = $subscriber_id",
+	[DB_STMT_UPD_IMEI_BY_IMSI] = "UPDATE subscriber SET imei = $imei WHERE imsi = $imsi",
 	[DB_STMT_AUC_BY_IMSI] =
 		"SELECT id, algo_id_2g, ki, algo_id_3g, k, op, opc, sqn, ind_bitlen"
 		" FROM subscriber"
@@ -63,7 +67,7 @@ static const char *stmt_sql[] = {
 	[DB_STMT_UPD_PURGE_PS_BY_IMSI] = "UPDATE subscriber SET ms_purged_ps = $val WHERE imsi = $imsi",
 	[DB_STMT_UPD_NAM_CS_BY_IMSI] = "UPDATE subscriber SET nam_cs = $val WHERE imsi = $imsi",
 	[DB_STMT_UPD_NAM_PS_BY_IMSI] = "UPDATE subscriber SET nam_ps = $val WHERE imsi = $imsi",
-	[DB_STMT_SUBSCR_CREATE] = "INSERT INTO subscriber (imsi) VALUES ($imsi)",
+	[DB_STMT_SUBSCR_CREATE] = "INSERT INTO subscriber (imsi, nam_cs, nam_ps) VALUES ($imsi, $nam_cs, $nam_ps)",
 	[DB_STMT_DEL_BY_ID] = "DELETE FROM subscriber WHERE id = $subscriber_id",
 	[DB_STMT_SET_MSISDN_BY_IMSI] = "UPDATE subscriber SET msisdn = $msisdn WHERE imsi = $imsi",
 	[DB_STMT_DELETE_MSISDN_BY_IMSI] = "UPDATE subscriber SET msisdn = NULL WHERE imsi = $imsi",
@@ -76,6 +80,9 @@ static const char *stmt_sql[] = {
 		" VALUES($subscriber_id, $algo_id_3g, $k, $op, $opc, $ind_bitlen)",
 	[DB_STMT_AUC_3G_DELETE] = "DELETE FROM auc_3g WHERE subscriber_id = $subscriber_id",
 	[DB_STMT_SET_LAST_LU_SEEN] = "UPDATE subscriber SET last_lu_seen = datetime($val, 'unixepoch') WHERE id = $subscriber_id",
+	[DB_STMT_SET_LAST_LU_SEEN_PS] = "UPDATE subscriber SET last_lu_seen_ps = datetime($val, 'unixepoch') WHERE id = $subscriber_id",
+	[DB_STMT_EXISTS_BY_IMSI] = "SELECT 1 FROM subscriber WHERE imsi = $imsi",
+	[DB_STMT_EXISTS_BY_MSISDN] = "SELECT 1 FROM subscriber WHERE msisdn = $msisdn",
 };
 
 static void sql3_error_log_cb(void *arg, int err_code, const char *msg)
@@ -196,28 +203,38 @@ void db_close(struct db_context *dbc)
 	talloc_free(dbc);
 }
 
-static int db_bootstrap(struct db_context *dbc)
+static int db_run_statements(struct db_context *dbc, const char **statements, size_t statements_count)
 {
+	int rc;
 	int i;
-	for (i = 0; i < ARRAY_SIZE(stmt_bootstrap_sql); i++) {
-		int rc;
+	for (i = 0; i < statements_count; i++) {
+		const char *stmt_str = statements[i];
 		sqlite3_stmt *stmt;
-		rc = sqlite3_prepare_v2(dbc->db, stmt_bootstrap_sql[i], -1, &stmt, NULL);
+
+		rc = sqlite3_prepare_v2(dbc->db, stmt_str, -1, &stmt, NULL);
 		if (rc != SQLITE_OK) {
-			LOGP(DDB, LOGL_ERROR, "Unable to prepare SQL statement '%s'\n", stmt_bootstrap_sql[i]);
+			LOGP(DDB, LOGL_ERROR, "Unable to prepare SQL statement '%s'\n", stmt_str);
 			return rc;
 		}
-
 		rc = sqlite3_step(stmt);
 		db_remove_reset(stmt);
 		sqlite3_finalize(stmt);
 		if (rc != SQLITE_DONE) {
-			LOGP(DDB, LOGL_ERROR, "Cannot bootstrap database: SQL error: (%d) %s,"
-			     " during stmt '%s'",
-			     rc, sqlite3_errmsg(dbc->db), stmt_bootstrap_sql[i]);
+			LOGP(DDB, LOGL_ERROR, "SQL error: (%d) %s, during stmt '%s'",
+			     rc, sqlite3_errmsg(dbc->db), stmt_str);
 			return rc;
 		}
 	}
+	return rc;
+}
+
+static int db_bootstrap(struct db_context *dbc)
+{
+	int rc = db_run_statements(dbc, stmt_bootstrap_sql, ARRAY_SIZE(stmt_bootstrap_sql));
+	if (rc != SQLITE_DONE) {
+		LOGP(DDB, LOGL_ERROR, "Cannot bootstrap database\n");
+		return rc;
+	}
 	return SQLITE_OK;
 }
 
@@ -258,38 +275,180 @@ static bool db_is_bootstrapped_v0(struct db_context *dbc)
 static int
 db_upgrade_v1(struct db_context *dbc)
 {
-	sqlite3_stmt *stmt;
 	int rc;
-	const char *update_stmt_sql = "ALTER TABLE subscriber ADD COLUMN last_lu_seen TIMESTAMP default NULL";
-	const char *set_schema_version_sql = "PRAGMA user_version = 1";
+	const char *statements[] = {
+		"ALTER TABLE subscriber ADD COLUMN last_lu_seen TIMESTAMP default NULL",
+		"PRAGMA user_version = 1",
+	};
 
-	rc = sqlite3_prepare_v2(dbc->db, update_stmt_sql, -1, &stmt, NULL);
-	if (rc != SQLITE_OK) {
-		LOGP(DDB, LOGL_ERROR, "Unable to prepare SQL statement '%s'\n", update_stmt_sql);
-		return rc;
-	}
-	rc = sqlite3_step(stmt);
-	db_remove_reset(stmt);
-	sqlite3_finalize(stmt);
+	rc = db_run_statements(dbc, statements, ARRAY_SIZE(statements));
 	if (rc != SQLITE_DONE) {
-		LOGP(DDB, LOGL_ERROR, "Unable to update HLR database schema to version %d\n", 1);
+		LOGP(DDB, LOGL_ERROR, "Unable to update HLR database schema to version 1\n");
 		return rc;
 	}
-
-	rc = sqlite3_prepare_v2(dbc->db, set_schema_version_sql, -1, &stmt, NULL);
-	if (rc != SQLITE_OK) {
-		LOGP(DDB, LOGL_ERROR, "Unable to prepare SQL statement '%s'\n", set_schema_version_sql);
-		return rc;
-	}
-	rc = sqlite3_step(stmt);
-	if (rc != SQLITE_DONE)
-		LOGP(DDB, LOGL_ERROR, "Unable to update HLR database schema to version %d\n", 1);
-
-	db_remove_reset(stmt);
-	sqlite3_finalize(stmt);
 	return rc;
 }
 
+static int db_upgrade_v2(struct db_context *dbc)
+{
+	int rc;
+	const char *statements[] = {
+		"ALTER TABLE subscriber ADD COLUMN imei VARCHAR(14)",
+		"PRAGMA user_version = 2",
+	};
+
+	rc = db_run_statements(dbc, statements, ARRAY_SIZE(statements));
+	if (rc != SQLITE_DONE) {
+		LOGP(DDB, LOGL_ERROR, "Unable to update HLR database schema to version 2\n");
+		return rc;
+	}
+	return rc;
+}
+
+static int db_upgrade_v3(struct db_context *dbc)
+{
+	int rc;
+
+	/* A newer SQLite version would allow simply 'ATLER TABLE subscriber RENAME COLUMN hlr_number TO msc_number'.
+	 * This is a really expensive workaround for that in order to cover earlier SQLite versions as well:
+	 * Create a new table with the new column name and copy the data over (https://www.sqlite.org/faq.html#q11).
+	 */
+#define SUBSCR_V3_CREATE  \
+"(\n" \
+"-- OsmoHLR's DB scheme is modelled roughly after TS 23.008 version 13.3.0\n" \
+"	id		INTEGER PRIMARY KEY,\n" \
+"	-- Chapter 2.1.1.1\n" \
+"	imsi		VARCHAR(15) UNIQUE NOT NULL,\n" \
+"	-- Chapter 2.1.2\n" \
+"	msisdn		VARCHAR(15) UNIQUE,\n" \
+"	-- Chapter 2.2.3: Most recent / current IMEISV\n" \
+"	imeisv		VARCHAR,\n" \
+"	-- Chapter 2.1.9: Most recent / current IMEI\n" \
+"	imei		VARCHAR(14),\n" \
+"	-- Chapter 2.4.5\n" \
+"	vlr_number	VARCHAR(15),\n" \
+"	-- Chapter 2.4.6\n" \
+"	msc_number	VARCHAR(15),\n" \
+"	-- Chapter 2.4.8.1\n" \
+"	sgsn_number	VARCHAR(15),\n" \
+"	-- Chapter 2.13.10\n" \
+"	sgsn_address	VARCHAR,\n" \
+"	-- Chapter 2.4.8.2\n" \
+"	ggsn_number	VARCHAR(15),\n" \
+"	-- Chapter 2.4.9.2\n" \
+"	gmlc_number	VARCHAR(15),\n" \
+"	-- Chapter 2.4.23\n" \
+"	smsc_number	VARCHAR(15),\n" \
+"	-- Chapter 2.4.24\n" \
+"	periodic_lu_tmr	INTEGER,\n" \
+"	-- Chapter 2.13.115\n" \
+"	periodic_rau_tau_tmr INTEGER,\n" \
+"	-- Chapter 2.1.1.2: network access mode\n" \
+"	nam_cs		BOOLEAN NOT NULL DEFAULT 1,\n" \
+"	nam_ps		BOOLEAN NOT NULL DEFAULT 1,\n" \
+"	-- Chapter 2.1.8\n" \
+"	lmsi		INTEGER,\n" \
+ \
+"	-- The below purged flags might not even be stored non-volatile,\n" \
+"	-- refer to TS 23.012 Chapter 3.6.1.4\n" \
+"	-- Chapter 2.7.5\n" \
+"	ms_purged_cs	BOOLEAN NOT NULL DEFAULT 0,\n" \
+"	-- Chapter 2.7.6\n" \
+"	ms_purged_ps	BOOLEAN NOT NULL DEFAULT 0,\n" \
+ \
+"	-- Timestamp of last location update seen from subscriber\n" \
+"	-- The value is a string which encodes a UTC timestamp in granularity of seconds.\n" \
+"	last_lu_seen TIMESTAMP default NULL\n" \
+")\n"
+
+#define SUBSCR_V2_COLUMN_NAMES \
+	"id," \
+	"imsi," \
+	"msisdn," \
+	"imeisv," \
+	"imei," \
+	"vlr_number," \
+	"hlr_number," \
+	"sgsn_number," \
+	"sgsn_address," \
+	"ggsn_number," \
+	"gmlc_number," \
+	"smsc_number," \
+	"periodic_lu_tmr," \
+	"periodic_rau_tau_tmr," \
+	"nam_cs," \
+	"nam_ps," \
+	"lmsi," \
+	"ms_purged_cs," \
+	"ms_purged_ps," \
+	"last_lu_seen"
+
+#define SUBSCR_V3_COLUMN_NAMES \
+	"id," \
+	"imsi," \
+	"msisdn," \
+	"imeisv," \
+	"imei," \
+	"vlr_number," \
+	"msc_number," \
+	"sgsn_number," \
+	"sgsn_address," \
+	"ggsn_number," \
+	"gmlc_number," \
+	"smsc_number," \
+	"periodic_lu_tmr," \
+	"periodic_rau_tau_tmr," \
+	"nam_cs," \
+	"nam_ps," \
+	"lmsi," \
+	"ms_purged_cs," \
+	"ms_purged_ps," \
+	"last_lu_seen"
+
+	const char *statements[] = {
+		"BEGIN TRANSACTION",
+		"CREATE TEMPORARY TABLE subscriber_backup" SUBSCR_V3_CREATE,
+		"INSERT INTO subscriber_backup SELECT " SUBSCR_V2_COLUMN_NAMES " FROM subscriber",
+		"DROP TABLE subscriber",
+		"CREATE TABLE subscriber" SUBSCR_V3_CREATE,
+		"INSERT INTO subscriber SELECT " SUBSCR_V3_COLUMN_NAMES " FROM subscriber_backup",
+		"DROP TABLE subscriber_backup",
+		"COMMIT",
+		"PRAGMA user_version = 3",
+	};
+
+	rc = db_run_statements(dbc, statements, ARRAY_SIZE(statements));
+	if (rc != SQLITE_DONE) {
+		LOGP(DDB, LOGL_ERROR, "Unable to update HLR database schema to version 3\n");
+		return rc;
+	}
+	return rc;
+}
+
+static int db_upgrade_v4(struct db_context *dbc)
+{
+	int rc;
+	const char *statements[] = {
+		"ALTER TABLE subscriber ADD COLUMN last_lu_seen_ps TIMESTAMP default NULL",
+		"PRAGMA user_version = 4",
+	};
+
+	rc = db_run_statements(dbc, statements, ARRAY_SIZE(statements));
+	if (rc != SQLITE_DONE) {
+		LOGP(DDB, LOGL_ERROR, "Unable to update HLR database schema to version 4\n");
+		return rc;
+	}
+	return rc;
+}
+
+typedef int (*db_upgrade_func_t)(struct db_context *dbc);
+static db_upgrade_func_t db_upgrade_path[] = {
+	db_upgrade_v1,
+	db_upgrade_v2,
+	db_upgrade_v3,
+	db_upgrade_v4,
+};
+
 static int db_get_user_version(struct db_context *dbc)
 {
 	const char *user_version_sql = "PRAGMA user_version";
@@ -326,6 +485,17 @@ struct db_context *db_open(void *ctx, const char *fname, bool enable_sqlite_logg
 	LOGP(DDB, LOGL_INFO, "Compiled against SQLite3 lib version %s\n", SQLITE_VERSION);
 	LOGP(DDB, LOGL_INFO, "Running with SQLite3 lib version %s\n", sqlite3_libversion());
 
+#ifdef SQLITE_USE_TALLOC
+	/* Configure SQLite3 to use talloc memory allocator */
+	rc = db_sqlite3_use_talloc(ctx);
+	if (rc == SQLITE_OK) {
+		LOGP(DDB, LOGL_NOTICE, "SQLite3 is configured to use talloc\n");
+	} else {
+		LOGP(DDB, LOGL_ERROR, "Failed to configure SQLite3 "
+		     "to use talloc, using default memory allocator\n");
+	}
+#endif
+
 	dbc->fname = talloc_strdup(dbc, fname);
 
 	for (i = 0; i < 0xfffff; i++) {
@@ -389,23 +559,16 @@ struct db_context *db_open(void *ctx, const char *fname, bool enable_sqlite_logg
 
 	LOGP(DDB, LOGL_NOTICE, "Database '%s' has HLR DB schema version %d\n", dbc->fname, version);
 
-	if (version < CURRENT_SCHEMA_VERSION && allow_upgrade) {
-		switch (version) {
-		case 0:
-			rc = db_upgrade_v1(dbc);
-			if (rc != SQLITE_DONE) {
-				LOGP(DDB, LOGL_ERROR, "Failed to upgrade HLR DB schema to version 1: (rc=%d) %s\n",
-				     rc, sqlite3_errmsg(dbc->db));
-				goto out_free;
-			}
-			version = 1;
-			/* fall through */
-		/* case N: ... */
-		default:
-			break;
+	for (; allow_upgrade && (version < ARRAY_SIZE(db_upgrade_path)); version++) {
+		db_upgrade_func_t upgrade_func = db_upgrade_path[version];
+		rc = upgrade_func(dbc);
+		if (rc != SQLITE_DONE) {
+			LOGP(DDB, LOGL_ERROR, "Failed to upgrade HLR DB schema to version %d: (rc=%d) %s\n",
+			     version+1, rc, sqlite3_errmsg(dbc->db));
+			goto out_free;
 		}
 		LOGP(DDB, LOGL_NOTICE, "Database '%s' has been upgraded to HLR DB schema version %d\n",
-		     dbc->fname, version);
+		     dbc->fname, version+1);
 	}
 
 	if (version != CURRENT_SCHEMA_VERSION) {

src/db_auc.c

@@ -26,10 +26,10 @@
 
 #include <sqlite3.h>
 
-#include "logging.h"
-#include "db.h"
-#include "auc.h"
-#include "rand.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/db.h>
+#include <osmocom/hlr/auc.h>
+#include <osmocom/hlr/rand.h>
 
 #define LOGAUC(imsi, level, fmt, args ...)	LOGP(DAUC, level, "IMSI='%s': " fmt, imsi, ## args)
 
@@ -73,6 +73,32 @@ out:
 	return ret;
 }
 
+/* hexparse a specific column of a sqlite prepared statement into dst (with length check)
+ * returns 0 for success, -EIO on error */
+static int hexparse_stmt(uint8_t *dst, size_t dst_len, sqlite3_stmt *stmt, int col, const char *col_name,
+			 const char *imsi)
+{
+	const uint8_t *text;
+	size_t col_len;
+
+	/* Bytes are stored as hex strings in database, hence divide length by two */
+	col_len = sqlite3_column_bytes(stmt, col) / 2;
+
+	if (col_len != dst_len) {
+		LOGAUC(imsi, LOGL_ERROR, "Error reading %s, expected length %lu but has length %lu\n", col_name,
+		       dst_len, col_len);
+		return -EIO;
+	}
+
+	text = sqlite3_column_text(stmt, col);
+	if (!text) {
+		LOGAUC(imsi, LOGL_ERROR, "Error reading %s\n", col_name);
+		return -EIO;
+	}
+	osmo_hexparse((void *)text, dst, dst_len);
+	return 0;
+}
+
 /* obtain the authentication data for a given imsi
  * returns 0 for success, negative value on error:
  * -ENOENT if the IMSI is not known, -ENOKEY if the IMSI is known but has no auth data,
@@ -113,49 +139,34 @@ int db_get_auth_data(struct db_context *dbc, const char *imsi,
 	/* obtain result values using sqlite3_column_*() */
 	if (sqlite3_column_type(stmt, 1) == SQLITE_INTEGER) {
 		/* we do have some 2G authentication data */
-		const uint8_t *ki;
-
-		aud2g->algo = sqlite3_column_int(stmt, 1);
-		ki = sqlite3_column_text(stmt, 2);
-#if 0
-		if (sqlite3_column_bytes(stmt, 2) != sizeof(aud2g->u.gsm.ki)) {
-			LOGAUC(imsi, LOGL_ERROR, "Error reading Ki: %d\n", rc);
+		if (hexparse_stmt(aud2g->u.gsm.ki, sizeof(aud2g->u.gsm.ki), stmt, 2, "Ki", imsi))
 			goto end_2g;
-		}
-#endif
-		osmo_hexparse((void*)ki, (void*)&aud2g->u.gsm.ki, sizeof(aud2g->u.gsm.ki));
+		aud2g->algo = sqlite3_column_int(stmt, 1);
 		aud2g->type = OSMO_AUTH_TYPE_GSM;
 	} else
 		LOGAUC(imsi, LOGL_DEBUG, "No 2G Auth Data\n");
-//end_2g:
+end_2g:
 	if (sqlite3_column_type(stmt, 3) == SQLITE_INTEGER) {
 		/* we do have some 3G authentication data */
-		const uint8_t *k, *op, *opc;
-
-		aud3g->algo = sqlite3_column_int(stmt, 3);
-		k = sqlite3_column_text(stmt, 4);
-		if (!k) {
-			LOGAUC(imsi, LOGL_ERROR, "Error reading K: %d\n", rc);
+		if (hexparse_stmt(aud3g->u.umts.k, sizeof(aud3g->u.umts.k), stmt, 4, "K", imsi)) {
 			ret = -EIO;
 			goto out;
 		}
-		osmo_hexparse((void*)k, (void*)&aud3g->u.umts.k, sizeof(aud3g->u.umts.k));
+		aud3g->algo = sqlite3_column_int(stmt, 3);
+
 		/* UMTS Subscribers can have either OP or OPC */
-		op = sqlite3_column_text(stmt, 5);
-		if (!op) {
-			opc = sqlite3_column_text(stmt, 6);
-			if (!opc) {
-				LOGAUC(imsi, LOGL_ERROR, "Error reading OPC: %d\n", rc);
+		if (sqlite3_column_text(stmt, 5)) {
+			if (hexparse_stmt(aud3g->u.umts.opc, sizeof(aud3g->u.umts.opc), stmt, 5, "OP", imsi)) {
 				ret = -EIO;
 				goto out;
 			}
-			osmo_hexparse((void*)opc, (void*)&aud3g->u.umts.opc,
-					sizeof(aud3g->u.umts.opc));
-			aud3g->u.umts.opc_is_op = 0;
-		} else {
-			osmo_hexparse((void*)op, (void*)&aud3g->u.umts.opc,
-					sizeof(aud3g->u.umts.opc));
 			aud3g->u.umts.opc_is_op = 1;
+		} else {
+			if (hexparse_stmt(aud3g->u.umts.opc, sizeof(aud3g->u.umts.opc), stmt, 6, "OPC", imsi)) {
+				ret = -EIO;
+				goto out;
+			}
+			aud3g->u.umts.opc_is_op = 0;
 		}
 		aud3g->u.umts.sqn = sqlite3_column_int64(stmt, 7);
 		aud3g->u.umts.ind_bitlen = sqlite3_column_int(stmt, 8);
@@ -178,7 +189,7 @@ out:
 int db_get_auc(struct db_context *dbc, const char *imsi,
 	       unsigned int auc_3g_ind, struct osmo_auth_vector *vec,
 	       unsigned int num_vec, const uint8_t *rand_auts,
-	       const uint8_t *auts)
+	       const uint8_t *auts, bool separation_bit)
 {
 	struct osmo_sub_auth_data aud2g, aud3g;
 	int64_t subscr_id;
@@ -198,6 +209,12 @@ int db_get_auc(struct db_context *dbc, const char *imsi,
 		       aud3g.u.umts.ind_bitlen, aud3g.u.umts.ind);
 		aud3g.u.umts.ind &= (1U << aud3g.u.umts.ind_bitlen) - 1;
 	}
+	/* the first bit (bit0) cannot be used as AMF anymore, but has been
+	 * re-appropriated as the separation bit.  See 3GPP TS 33.102 Annex H
+	 * together with 3GPP TS 33.401 / 33.402 / 33.501 */
+	aud3g.u.umts.amf[0] = aud3g.u.umts.amf[0] & 0x7f;
+	if (separation_bit)
+		aud3g.u.umts.amf[0] |= 0x80;
 
 	LOGAUC(imsi, LOGL_DEBUG, "Calling to generate %u vectors\n", num_vec);
 	rc = auc_compute_vectors(vec, num_vec, &aud2g, &aud3g, rand_auts, auts);

src/db_debug.c

@@ -0,0 +1,86 @@
+/*
+ * libtalloc based memory allocator for SQLite3.
+ *
+ * (C) 2019 by Vadim Yanitskiy <axilirator@gmail.com>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include <sqlite3.h>
+#include <talloc.h>
+#include <errno.h>
+
+/* Dedicated talloc context for SQLite */
+static void *db_sqlite_ctx = NULL;
+
+static void *tall_xMalloc(int size)
+{
+	return talloc_size(db_sqlite_ctx, size);
+}
+
+static void tall_xFree(void *ptr)
+{
+	talloc_free(ptr);
+}
+
+static void *tall_xRealloc(void *ptr, int size)
+{
+	return talloc_realloc_fn(db_sqlite_ctx, ptr, size);
+}
+
+static int tall_xSize(void *ptr)
+{
+	return talloc_total_size(ptr);
+}
+
+/* DUMMY: talloc doesn't round up the allocation size */
+static int tall_xRoundup(int size) { return size; }
+
+/* DUMMY: nothing to initialize */
+static int tall_xInit(void *data) { return 0; }
+
+/* DUMMY: nothing to deinitialize */
+static void tall_xShutdown(void *data) {  }
+
+/* Interface between SQLite and talloc memory allocator */
+static const struct sqlite3_mem_methods tall_sqlite_if = {
+	/* Memory allocation function */
+	.xMalloc = &tall_xMalloc,
+	/* Free a prior allocation */
+	.xFree = &tall_xFree,
+	/* Resize an allocation */
+	.xRealloc = &tall_xRealloc,
+	/* Return the size of an allocation */
+	.xSize = &tall_xSize,
+	/* Round up request size to allocation size */
+	.xRoundup = &tall_xRoundup,
+	/* Initialize the memory allocator */
+	.xInit = &tall_xInit,
+	/* Deinitialize the memory allocator */
+	.xShutdown = &tall_xShutdown,
+	/* Argument to xInit() and xShutdown() */
+	.pAppData = NULL,
+};
+
+int db_sqlite3_use_talloc(void *ctx)
+{
+	if (db_sqlite_ctx != NULL)
+		return -EEXIST;
+
+	db_sqlite_ctx = talloc_named_const(ctx, 0, "SQLite3");
+	return sqlite3_config(SQLITE_CONFIG_MALLOC, &tall_sqlite_if);
+}

src/db_hlr.c

@@ -33,20 +33,21 @@
 
 #include <sqlite3.h>
 
-#include "logging.h"
-#include "hlr.h"
-#include "db.h"
-#include "gsup_server.h"
-#include "luop.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/hlr.h>
+#include <osmocom/hlr/db.h>
+#include <osmocom/hlr/gsup_server.h>
+#include <osmocom/hlr/luop.h>
 
 #define LOGHLR(imsi, level, fmt, args ...)	LOGP(DAUC, level, "IMSI='%s': " fmt, imsi, ## args)
 
 /*! Add new subscriber record to the HLR database.
  * \param[in,out] dbc  database context.
  * \param[in] imsi  ASCII string of IMSI digits, is validated.
+ * \param[in] flags  Bitmask of DB_SUBSCR_FLAG_*.
  * \returns 0 on success, -EINVAL on invalid IMSI, -EIO on database error.
  */
-int db_subscr_create(struct db_context *dbc, const char *imsi)
+int db_subscr_create(struct db_context *dbc, const char *imsi, uint8_t flags)
 {
 	sqlite3_stmt *stmt;
 	int rc;
@@ -61,6 +62,10 @@ int db_subscr_create(struct db_context *dbc, const char *imsi)
 
 	if (!db_bind_text(stmt, "$imsi", imsi))
 		return -EIO;
+	if (!db_bind_int(stmt, "$nam_cs", (flags & DB_SUBSCR_FLAG_NAM_CS) != 0))
+		return -EIO;
+	if (!db_bind_int(stmt, "$nam_ps", (flags & DB_SUBSCR_FLAG_NAM_PS) != 0))
+		return -EIO;
 
 	/* execute the statement */
 	rc = sqlite3_step(stmt);
@@ -386,14 +391,83 @@ out:
 	return ret;
 }
 
+/*! Set a subscriber's IMEI in the HLR database.
+ * \param[in,out] dbc  database context.
+ * \param[in] imsi  ASCII string of IMSI digits
+ * \param[in] imei  ASCII string of identifier digits, or NULL to remove the IMEI.
+ * \returns 0 on success, -ENOENT when the given subscriber does not exist,
+ *         -EIO on database errors.
+ */
+int db_subscr_update_imei_by_imsi(struct db_context *dbc, const char* imsi, const char *imei)
+{
+	int rc, ret = 0;
+	sqlite3_stmt *stmt = dbc->stmt[DB_STMT_UPD_IMEI_BY_IMSI];
+
+	if (imei && !osmo_imei_str_valid(imei, false)) {
+		LOGP(DAUC, LOGL_ERROR, "Cannot update subscriber IMSI='%s': invalid IMEI: '%s'\n", imsi, imei);
+		return -EINVAL;
+	}
+
+	if (!db_bind_text(stmt, "$imsi", imsi))
+		return -EIO;
+	if (imei && !db_bind_text(stmt, "$imei", imei))
+		return -EIO;
+
+	/* execute the statement */
+	rc = sqlite3_step(stmt);
+	if (rc != SQLITE_DONE) {
+		LOGP(DAUC, LOGL_ERROR, "Update IMEI for subscriber IMSI='%s': SQL Error: %s\n", imsi,
+		     sqlite3_errmsg(dbc->db));
+		ret = -EIO;
+		goto out;
+	}
+
+	/* verify execution result */
+	rc = sqlite3_changes(dbc->db);
+	if (!rc) {
+		LOGP(DAUC, LOGL_ERROR, "Cannot update IMEI for subscriber IMSI='%s': no such subscriber\n", imsi);
+		ret = -ENOENT;
+	} else if (rc != 1) {
+		LOGP(DAUC, LOGL_ERROR, "Update IMEI for subscriber IMSI='%s': SQL modified %d rows (expected 1)\n",
+		     imsi, rc);
+		ret = -EIO;
+	}
+
+out:
+	db_remove_reset(stmt);
+	return ret;
+}
+
+static void parse_last_lu_seen(time_t *dst, const char *last_lu_seen_str, const char *imsi, const char *label)
+{
+	struct tm tm = {0};
+	time_t val;
+	if (!last_lu_seen_str || last_lu_seen_str[0] == '\0')
+		return;
+
+	if (strptime(last_lu_seen_str, DB_LAST_LU_SEEN_FMT, &tm) == NULL) {
+		LOGP(DAUC, LOGL_ERROR, "IMSI-%s: Last LU Seen %s: Cannot parse timestamp '%s'\n",
+		     imsi, label, last_lu_seen_str);
+		return;
+	}
+
+	errno = 0;
+	val = mktime(&tm);
+	if (val == -1) {
+		LOGP(DAUC, LOGL_ERROR, "IMSI-%s: Last LU Seen %s: Cannot convert timestamp '%s' to time_t: %s\n",
+		     imsi, label, last_lu_seen_str, strerror(errno));
+		val = 0;
+	}
+
+	*dst = val;
+}
+
 /* Common code for db_subscr_get_by_*() functions. */
 static int db_sel(struct db_context *dbc, sqlite3_stmt *stmt, struct hlr_subscriber *subscr,
 		  const char **err)
 {
 	int rc;
 	int ret = 0;
-	const char *last_lu_seen_str;
-	struct tm tm;
 
 	/* execute the statement */
 	rc = sqlite3_step(stmt);
@@ -415,31 +489,22 @@ static int db_sel(struct db_context *dbc, sqlite3_stmt *stmt, struct hlr_subscri
 	subscr->id = sqlite3_column_int64(stmt, 0);
 	copy_sqlite3_text_to_buf(subscr->imsi, stmt, 1);
 	copy_sqlite3_text_to_buf(subscr->msisdn, stmt, 2);
+	copy_sqlite3_text_to_buf(subscr->imei, stmt, 3);
 	/* FIXME: These should all be BLOBs as they might contain NUL */
-	copy_sqlite3_text_to_buf(subscr->vlr_number, stmt, 3);
-	copy_sqlite3_text_to_buf(subscr->sgsn_number, stmt, 4);
-	copy_sqlite3_text_to_buf(subscr->sgsn_address, stmt, 5);
-	subscr->periodic_lu_timer = sqlite3_column_int(stmt, 6);
-	subscr->periodic_rau_tau_timer = sqlite3_column_int(stmt, 7);
-	subscr->nam_cs = sqlite3_column_int(stmt, 8);
-	subscr->nam_ps = sqlite3_column_int(stmt, 9);
-	subscr->lmsi = sqlite3_column_int(stmt, 10);
-	subscr->ms_purged_cs = sqlite3_column_int(stmt, 11);
-	subscr->ms_purged_ps = sqlite3_column_int(stmt, 12);
-	last_lu_seen_str = (const char *)sqlite3_column_text(stmt, 13);
-	if (last_lu_seen_str && last_lu_seen_str[0] != '\0') {
-		if (strptime(last_lu_seen_str, DB_LAST_LU_SEEN_FMT, &tm) == NULL) {
-			LOGP(DAUC, LOGL_ERROR, "Cannot parse last LU timestamp '%s' of subscriber with IMSI='%s': %s\n",
-			     last_lu_seen_str, subscr->imsi, strerror(errno));
-		} else {
-			subscr->last_lu_seen = mktime(&tm);
-			if (subscr->last_lu_seen == -1) {
-				LOGP(DAUC, LOGL_ERROR, "Cannot convert LU timestamp '%s' to time_t: %s\n",
-				     last_lu_seen_str, strerror(errno));
-				subscr->last_lu_seen = 0;
-			}
-		}
-	}
+	copy_sqlite3_text_to_buf(subscr->vlr_number, stmt, 4);
+	copy_sqlite3_text_to_buf(subscr->sgsn_number, stmt, 5);
+	copy_sqlite3_text_to_buf(subscr->sgsn_address, stmt, 6);
+	subscr->periodic_lu_timer = sqlite3_column_int(stmt, 7);
+	subscr->periodic_rau_tau_timer = sqlite3_column_int(stmt, 8);
+	subscr->nam_cs = sqlite3_column_int(stmt, 9);
+	subscr->nam_ps = sqlite3_column_int(stmt, 10);
+	subscr->lmsi = sqlite3_column_int(stmt, 11);
+	subscr->ms_purged_cs = sqlite3_column_int(stmt, 12);
+	subscr->ms_purged_ps = sqlite3_column_int(stmt, 13);
+	parse_last_lu_seen(&subscr->last_lu_seen, (const char *)sqlite3_column_text(stmt, 14),
+			   subscr->imsi, "CS");
+	parse_last_lu_seen(&subscr->last_lu_seen_ps, (const char *)sqlite3_column_text(stmt, 15),
+			   subscr->imsi, "PS");
 
 out:
 	db_remove_reset(stmt);
@@ -458,6 +523,31 @@ out:
 	return ret;
 }
 
+/*! Check if a subscriber exists in the HLR database.
+ * \param[in, out] dbc  database context.
+ * \param[in] imsi  ASCII string of IMSI digits.
+ * \returns 0 if it exists, -ENOENT if it does not exist, -EIO on database error.
+ */
+int db_subscr_exists_by_imsi(struct db_context *dbc, const char *imsi) {
+	sqlite3_stmt *stmt = dbc->stmt[DB_STMT_EXISTS_BY_IMSI];
+	const char *err;
+	int rc;
+
+	if (!db_bind_text(stmt, NULL, imsi))
+		return -EIO;
+
+	rc = sqlite3_step(stmt);
+	db_remove_reset(stmt);
+	if (rc == SQLITE_ROW)
+		return 0; /* exists */
+	if (rc == SQLITE_DONE)
+		return -ENOENT; /* does not exist */
+
+	err = sqlite3_errmsg(dbc->db);
+	LOGP(DAUC, LOGL_ERROR, "Failed to check if subscriber exists by IMSI='%s': %s\n", imsi, err);
+	return rc;
+}
+
 /*! Retrieve subscriber data from the HLR database.
  * \param[in,out] dbc  database context.
  * \param[in] imsi  ASCII string of IMSI digits.
@@ -482,6 +572,33 @@ int db_subscr_get_by_imsi(struct db_context *dbc, const char *imsi,
 	return rc;
 }
 
+/*! Check if a subscriber exists in the HLR database.
+ * \param[in, out] dbc  database context.
+ * \param[in] msisdn  ASCII string of MSISDN digits.
+ * \returns 0 if it exists, -ENOENT if it does not exist, -EIO on database error.
+ */
+int db_subscr_exists_by_msisdn(struct db_context *dbc, const char *msisdn)
+{
+	sqlite3_stmt *stmt = dbc->stmt[DB_STMT_EXISTS_BY_MSISDN];
+	const char *err;
+	int rc;
+
+	if (!db_bind_text(stmt, NULL, msisdn))
+		return -EIO;
+
+	rc = sqlite3_step(stmt);
+	db_remove_reset(stmt);
+	if (rc == SQLITE_ROW)
+		return 0; /* exists */
+	if (rc == SQLITE_DONE)
+		return -ENOENT; /* does not exist */
+
+	err = sqlite3_errmsg(dbc->db);
+	LOGP(DAUC, LOGL_ERROR, "Failed to check if subscriber exists "
+		"by MSISDN='%s': %s\n", msisdn, err);
+	return rc;
+}
+
 /*! Retrieve subscriber data from the HLR database.
  * \param[in,out] dbc  database context.
  * \param[in] msisdn  ASCII string of MSISDN digits.
@@ -530,6 +647,28 @@ int db_subscr_get_by_id(struct db_context *dbc, int64_t id,
 	return rc;
 }
 
+/*! Retrieve subscriber data from the HLR database.
+ * \param[in,out] dbc  database context.
+ * \param[in] imei  ASCII string of identifier digits
+ * \param[out] subscr  place retrieved data in this struct.
+ * \returns 0 on success, -ENOENT if no such subscriber was found, -EIO on
+ *          database error.
+ */
+int db_subscr_get_by_imei(struct db_context *dbc, const char *imei, struct hlr_subscriber *subscr)
+{
+	sqlite3_stmt *stmt = dbc->stmt[DB_STMT_SEL_BY_IMEI];
+	const char *err;
+	int rc;
+
+	if (!db_bind_text(stmt, NULL, imei))
+		return -EIO;
+
+	rc = db_sel(dbc, stmt, subscr, &err);
+	if (rc)
+		LOGP(DAUC, LOGL_ERROR, "Cannot read subscriber from db: IMEI=%s: %s\n", imei, err);
+	return rc;
+}
+
 /*! You should use hlr_subscr_nam() instead; enable or disable PS or CS for a
  * subscriber without notifying GSUP clients.
  * \param[in,out] dbc  database context.
@@ -643,7 +782,7 @@ int db_subscr_lu(struct db_context *dbc, int64_t subscr_id,
 		goto out;
 	}
 
-	stmt = dbc->stmt[DB_STMT_SET_LAST_LU_SEEN];
+	stmt = dbc->stmt[is_ps? DB_STMT_SET_LAST_LU_SEEN_PS : DB_STMT_SET_LAST_LU_SEEN];
 
 	if (!db_bind_int64(stmt, "$subscriber_id", subscr_id))
 		return -EIO;

src/gsup_router.c

@@ -23,17 +23,17 @@
 #include <osmocom/core/linuxlist.h>
 #include <osmocom/core/talloc.h>
 
-#include "logging.h"
-#include "gsup_server.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/gsup_server.h>
+#include <osmocom/hlr/gsup_router.h>
 
-struct gsup_route {
-	struct llist_head list;
-
-	uint8_t *addr;
-	struct osmo_gsup_conn *conn;
-};
-
-/* find a route for the given address */
+/*! Find a route for the given address.
+ * \param[in] gs gsup server
+ * \param[in] addr IPA name of the client (SGSN, MSC/VLR). Although this is passed like a blob, together with the
+ *                 length, it must be nul-terminated! This is for legacy reasons, see the discussion here:
+ *                 https://gerrit.osmocom.org/#/c/osmo-hlr/+/13048/
+ * \param[in] addrlen length of addr, *including the nul-byte* (strlen(addr) + 1).
+ */
 struct osmo_gsup_conn *gsup_route_find(struct osmo_gsup_server *gs,
 					const uint8_t *addr, size_t addrlen)
 {
@@ -47,6 +47,22 @@ struct osmo_gsup_conn *gsup_route_find(struct osmo_gsup_server *gs,
 	return NULL;
 }
 
+/*! Find a GSUP connection's route (to read the IPA address from the route).
+ * \param[in] conn GSUP connection
+ * \return GSUP route
+ */
+struct gsup_route *gsup_route_find_by_conn(const struct osmo_gsup_conn *conn)
+{
+	struct gsup_route *gr;
+
+	llist_for_each_entry(gr, &conn->server->routes, list) {
+		if (gr->conn == conn)
+			return gr;
+	}
+
+	return NULL;
+}
+
 /* add a new route for the given address to the given conn */
 int gsup_route_add(struct osmo_gsup_conn *conn, const uint8_t *addr, size_t addrlen)
 {
@@ -61,7 +77,7 @@ int gsup_route_add(struct osmo_gsup_conn *conn, const uint8_t *addr, size_t addr
 	if (!gr)
 		return -ENOMEM;
 
-	LOGP(DMAIN, LOGL_INFO, "Adding GSUP route for %s\n", addr);
+	LOGP(DMAIN, LOGL_INFO, "Adding GSUP route for %s via %s:%u\n", addr, conn->conn->addr, conn->conn->port);
 
 	gr->addr = talloc_memdup(gr, addr, addrlen);
 	gr->conn = conn;

src/gsup_send.c

@@ -21,12 +21,19 @@
 
 #include <errno.h>
 
-#include "gsup_server.h"
-#include "gsup_router.h"
+#include <osmocom/hlr/gsup_server.h>
+#include <osmocom/hlr/gsup_router.h>
 
 #include <osmocom/core/logging.h>
 
-/* Send a msgb to a given address using routing */
+/*! Send a msgb to a given address using routing.
+ * \param[in] gs gsup server
+ * \param[in] addr IPA name of the client (SGSN, MSC/VLR). Although this is passed like a blob, together with the
+ *                 length, it must be nul-terminated! This is for legacy reasons, see the discussion here:
+ *                 https://gerrit.osmocom.org/#/c/osmo-hlr/+/13048/
+ * \param[in] addrlen length of addr, *including the nul-byte* (strlen(addr) + 1).
+ * \param[in] msg message buffer
+ */
 int osmo_gsup_addr_send(struct osmo_gsup_server *gs,
 			const uint8_t *addr, size_t addrlen,
 			struct msgb *msg)
@@ -35,7 +42,7 @@ int osmo_gsup_addr_send(struct osmo_gsup_server *gs,
 
 	conn = gsup_route_find(gs, addr, addrlen);
 	if (!conn) {
-		DEBUGP(DLGSUP, "Cannot find route for addr %s\n", addr);
+		DEBUGP(DLGSUP, "Cannot find route for addr %s\n", osmo_quote_str((const char*)addr, addrlen));
 		msgb_free(msg);
 		return -ENODEV;
 	}

src/gsup_server.c

@@ -27,8 +27,15 @@
 #include <osmocom/gsm/gsm48_ie.h>
 #include <osmocom/gsm/apn.h>
 
-#include "gsup_server.h"
-#include "gsup_router.h"
+#include <osmocom/hlr/gsup_server.h>
+#include <osmocom/hlr/gsup_router.h>
+
+struct msgb *osmo_gsup_msgb_alloc(const char *label)
+{
+	struct msgb *msg = msgb_alloc_headroom(1024+16, 16, label);
+	OSMO_ASSERT(msg);
+	return msg;
+}
 
 static void osmo_gsup_server_send(struct osmo_gsup_conn *conn,
 			     int proto_ext, struct msgb *msg_tx)

src/gsupclient/gsup_client.c

@@ -386,7 +386,8 @@ int osmo_gsup_client_enc_send(struct osmo_gsup_client *gsupc,
 	rc = osmo_gsup_client_send(gsupc, gsup_msgb);
 	if (rc) {
 		LOGP(DLGSUP, LOGL_ERROR, "Couldn't send GSUP message\n");
-		goto error;
+		/* Do not free, osmo_gsup_client_send() already has. */
+		return rc;
 	}
 
 	return 0;

src/hlr.c

@@ -23,6 +23,7 @@
 #include <getopt.h>
 
 #include <osmocom/core/msgb.h>
+#include <osmocom/core/stats.h>
 #include <osmocom/core/logging.h>
 #include <osmocom/core/application.h>
 #include <osmocom/gsm/gsup.h>
@@ -33,19 +34,22 @@
 #include <osmocom/ctrl/control_vty.h>
 #include <osmocom/gsm/apn.h>
 #include <osmocom/gsm/gsm48_ie.h>
+#include <osmocom/gsm/gsm_utils.h>
+#include <osmocom/gsm/protocol/gsm_23_003.h>
 
-#include "db.h"
-#include "hlr.h"
-#include "ctrl.h"
-#include "logging.h"
-#include "gsup_server.h"
-#include "gsup_router.h"
-#include "rand.h"
-#include "luop.h"
-#include "hlr_vty.h"
-#include "hlr_ussd.h"
+#include <osmocom/hlr/db.h>
+#include <osmocom/hlr/hlr.h>
+#include <osmocom/hlr/ctrl.h>
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/gsup_server.h>
+#include <osmocom/hlr/gsup_router.h>
+#include <osmocom/hlr/rand.h>
+#include <osmocom/hlr/luop.h>
+#include <osmocom/hlr/hlr_vty.h>
+#include <osmocom/hlr/hlr_ussd.h>
 
 struct hlr *g_hlr;
+static void *hlr_ctx = NULL;
 static int quit = 0;
 
 /* Trigger 'Insert Subscriber Data' messages to all connected GSUP clients.
@@ -124,7 +128,7 @@ osmo_hlr_subscriber_update_notify(struct hlr_subscriber *subscr)
 		}
 
 		/* Send ISD to MSC/SGSN */
-		msg_out = msgb_alloc_headroom(1024+16, 16, "GSUP ISD UPDATE");
+		msg_out = osmo_gsup_msgb_alloc("GSUP ISD UPDATE");
 		if (msg_out == NULL) {
 			LOGP(DLGSUP, LOGL_ERROR,
 			       "IMSI='%s': Cannot notify GSUP client; could not allocate msg buffer "
@@ -146,6 +150,78 @@ osmo_hlr_subscriber_update_notify(struct hlr_subscriber *subscr)
 	}
 }
 
+static int generate_new_msisdn(char *msisdn, const char *imsi, unsigned int len)
+{
+	int i, j, rc;
+	uint8_t rand_buf[GSM23003_MSISDN_MAX_DIGITS];
+
+	OSMO_ASSERT(len <= sizeof(rand_buf));
+
+	/* Generate a random unique MSISDN (with retry) */
+	for (i = 0; i < 10; i++) {
+		/* Get a random number (with retry) */
+		for (j = 0; j < 10; j++) {
+			rc = osmo_get_rand_id(rand_buf, len);
+			if (!rc)
+				break;
+		}
+		if (rc) {
+			LOGP(DMAIN, LOGL_ERROR, "IMSI='%s': Failed to generate new MSISDN, random number generator"
+						" failed (rc=%d)\n", imsi, rc);
+			return rc;
+		}
+
+		/* Shift 0x00 ... 0xff range to 30 ... 39 (ASCII numbers) */
+		for (j = 0; j < len; j++)
+			msisdn[j] = 48 + (rand_buf[j] % 10);
+		msisdn[j] = '\0';
+
+		/* Ensure there is no subscriber with such MSISDN */
+		if (db_subscr_exists_by_msisdn(g_hlr->dbc, msisdn) == -ENOENT)
+			return 0;
+	}
+
+	/* Failure */
+	LOGP(DMAIN, LOGL_ERROR, "IMSI='%s': Failed to generate a new MSISDN, consider increasing "
+				"the length for the automatically assigned MSISDNs "
+				"(see 'subscriber-create-on-demand' command)\n", imsi);
+	return -1;
+}
+
+static int subscr_create_on_demand(const char *imsi)
+{
+	char msisdn[GSM23003_MSISDN_MAX_DIGITS + 1];
+	int rc;
+	unsigned int rand_msisdn_len = g_hlr->subscr_create_on_demand_rand_msisdn_len;
+
+	if (!g_hlr->subscr_create_on_demand)
+		return -1;
+	if (db_subscr_exists_by_imsi(g_hlr->dbc, imsi) == 0)
+		return -1;
+	if (rand_msisdn_len && generate_new_msisdn(msisdn, imsi, rand_msisdn_len) != 0)
+		return -1;
+
+	LOGP(DMAIN, LOGL_INFO, "IMSI='%s': Creating subscriber on demand\n", imsi);
+	rc = db_subscr_create(g_hlr->dbc, imsi, g_hlr->subscr_create_on_demand_flags);
+	if (rc) {
+		LOGP(DMAIN, LOGL_ERROR, "Failed to create subscriber on demand (rc=%d): IMSI='%s'\n", rc, imsi);
+		return rc;
+	}
+
+	if (!rand_msisdn_len)
+		return 0;
+
+	/* Update MSISDN of the new (just allocated) subscriber */
+	rc = db_subscr_update_msisdn_by_imsi(g_hlr->dbc, imsi, msisdn);
+	if (rc) {
+		LOGP(DMAIN, LOGL_ERROR, "IMSI='%s': Failed to assign MSISDN='%s' (rc=%d)\n", imsi, msisdn, rc);
+		return rc;
+	}
+	LOGP(DMAIN, LOGL_INFO, "IMSI='%s': Successfully assigned MSISDN='%s'\n", imsi, msisdn);
+
+	return 0;
+}
+
 /***********************************************************************
  * Send Auth Info handling
  ***********************************************************************/
@@ -157,16 +233,27 @@ static int rx_send_auth_info(struct osmo_gsup_conn *conn,
 {
 	struct osmo_gsup_message gsup_out;
 	struct msgb *msg_out;
+	bool separation_bit = false;
+	int num_auth_vectors = OSMO_GSUP_MAX_NUM_AUTH_INFO;
 	int rc;
 
+	subscr_create_on_demand(gsup->imsi);
+
 	/* initialize return message structure */
 	memset(&gsup_out, 0, sizeof(gsup_out));
 	memcpy(&gsup_out.imsi, &gsup->imsi, sizeof(gsup_out.imsi));
 
+	if (gsup->current_rat_type == OSMO_RAT_EUTRAN_SGS)
+		separation_bit = true;
+
+	if (gsup->num_auth_vectors > 0 &&
+			gsup->num_auth_vectors <= OSMO_GSUP_MAX_NUM_AUTH_INFO)
+		num_auth_vectors = gsup->num_auth_vectors;
+
 	rc = db_get_auc(dbc, gsup->imsi, conn->auc_3g_ind,
 			gsup_out.auth_vectors,
-			ARRAY_SIZE(gsup_out.auth_vectors),
-			gsup->rand, gsup->auts);
+			num_auth_vectors,
+			gsup->rand, gsup->auts, separation_bit);
 	if (rc <= 0) {
 		gsup_out.message_type = OSMO_GSUP_MSGT_SEND_AUTH_INFO_ERROR;
 		switch (rc) {
@@ -193,7 +280,7 @@ static int rx_send_auth_info(struct osmo_gsup_conn *conn,
 		gsup_out.num_auth_vectors = rc;
 	}
 
-	msg_out = msgb_alloc_headroom(1024+16, 16, "GSUP AUC response");
+	msg_out = osmo_gsup_msgb_alloc("GSUP AUC response");
 	osmo_gsup_encode(msg_out, &gsup_out);
 	return osmo_gsup_conn_send(conn, msg_out);
 }
@@ -289,6 +376,8 @@ static int rx_upd_loc_req(struct osmo_gsup_conn *conn,
 	}
 	llist_add(&luop->list, &g_lu_ops);
 
+	subscr_create_on_demand(gsup->imsi);
+
 	/* Roughly follwing "Process Update_Location_HLR" of TS 09.02 */
 
 	/* check if subscriber is known at all */
@@ -371,7 +460,7 @@ static int rx_purge_ms_req(struct osmo_gsup_conn *conn,
 		gsup_reply.cause = GMM_CAUSE_NET_FAIL;
 	}
 
-	msg_out = msgb_alloc_headroom(1024+16, 16, "GSUP AUC response");
+	msg_out = osmo_gsup_msgb_alloc("GSUP AUC response");
 	osmo_gsup_encode(msg_out, &gsup_reply);
 	return osmo_gsup_conn_send(conn, msg_out);
 }
@@ -379,21 +468,14 @@ static int rx_purge_ms_req(struct osmo_gsup_conn *conn,
 static int gsup_send_err_reply(struct osmo_gsup_conn *conn, const char *imsi,
 				enum osmo_gsup_message_type type_in, uint8_t err_cause)
 {
-	int type_err = osmo_gsup_get_err_msg_type(type_in);
+	int type_err = OSMO_GSUP_TO_MSGT_ERROR(type_in);
 	struct osmo_gsup_message gsup_reply = {0};
 	struct msgb *msg_out;
 
-	if (type_err < 0) {
-		LOGP(DMAIN, LOGL_ERROR, "unable to determine error response for %s\n",
-			osmo_gsup_message_type_name(type_in));
-		return type_err;
-	}
-
 	OSMO_STRLCPY_ARRAY(gsup_reply.imsi, imsi);
 	gsup_reply.message_type = type_err;
 	gsup_reply.cause = err_cause;
-	msg_out = msgb_alloc_headroom(1024+16, 16, "GSUP ERR response");
-	OSMO_ASSERT(msg_out);
+	msg_out = osmo_gsup_msgb_alloc("GSUP ERR response");
 	osmo_gsup_encode(msg_out, &gsup_reply);
 	LOGP(DMAIN, LOGL_NOTICE, "Tx %s\n", osmo_gsup_message_type_name(type_err));
 	return osmo_gsup_conn_send(conn, msg_out);
@@ -403,49 +485,161 @@ static int rx_check_imei_req(struct osmo_gsup_conn *conn, const struct osmo_gsup
 {
 	struct osmo_gsup_message gsup_reply = {0};
 	struct msgb *msg_out;
-	char imei[GSM23003_IMEI_NUM_DIGITS+1] = {0};
+	char imei[GSM23003_IMEI_NUM_DIGITS_NO_CHK+1] = {0};
+	int rc;
 
-	/* Encoded IMEI length check */
-	if (!gsup->imei_enc || gsup->imei_enc_len < 1 || gsup->imei_enc[0] >= sizeof(imei)) {
-		LOGP(DMAIN, LOGL_ERROR, "%s: wrong encoded IMEI length\n", gsup->imsi);
+	/* Require IMEI */
+	if (!gsup->imei_enc) {
+		LOGP(DMAIN, LOGL_ERROR, "%s: missing IMEI\n", gsup->imsi);
 		gsup_send_err_reply(conn, gsup->imsi, gsup->message_type, GMM_CAUSE_INV_MAND_INFO);
 		return -1;
 	}
 
-	/* Decode IMEI */
-	if (gsm48_decode_bcd_number(imei, sizeof(imei), gsup->imei_enc, 0) < 0) {
-		LOGP(DMAIN, LOGL_ERROR, "%s: failed to decode IMEI\n", gsup->imsi);
+	/* Decode IMEI (fails if IMEI is too long) */
+	rc = gsm48_decode_bcd_number2(imei, sizeof(imei), gsup->imei_enc, gsup->imei_enc_len, 0);
+	if (rc < 0) {
+		LOGP(DMAIN, LOGL_ERROR, "%s: failed to decode IMEI (rc: %i)\n", gsup->imsi, rc);
 		gsup_send_err_reply(conn, gsup->imsi, gsup->message_type, GMM_CAUSE_INV_MAND_INFO);
 		return -1;
 	}
 
-	/* Only print the IMEI for now, it's planned to store it here (OS#2541) */
-	LOGP(DMAIN, LOGL_INFO, "%s: has IMEI: %s\n", gsup->imsi, imei);
+	/* Check if IMEI is too short */
+	if (strlen(imei) != GSM23003_IMEI_NUM_DIGITS_NO_CHK) {
+		LOGP(DMAIN, LOGL_ERROR, "%s: wrong encoded IMEI length (IMEI: '%s', %lu, %i)\n", gsup->imsi, imei,
+		     strlen(imei), GSM23003_IMEI_NUM_DIGITS_NO_CHK);
+		gsup_send_err_reply(conn, gsup->imsi, gsup->message_type, GMM_CAUSE_INV_MAND_INFO);
+		return -1;
+	}
+
+	subscr_create_on_demand(gsup->imsi);
+
+	/* Save in DB if desired */
+	if (g_hlr->store_imei) {
+		LOGP(DAUC, LOGL_DEBUG, "IMSI='%s': storing IMEI = %s\n", gsup->imsi, imei);
+		if (db_subscr_update_imei_by_imsi(g_hlr->dbc, gsup->imsi, imei) < 0) {
+			gsup_send_err_reply(conn, gsup->imsi, gsup->message_type, GMM_CAUSE_INV_MAND_INFO);
+			return -1;
+		}
+	} else {
+		/* Check if subscriber exists and print IMEI */
+		LOGP(DMAIN, LOGL_INFO, "IMSI='%s': has IMEI = %s (consider setting 'store-imei')\n", gsup->imsi, imei);
+		struct hlr_subscriber subscr;
+		if (db_subscr_get_by_imsi(g_hlr->dbc, gsup->imsi, &subscr) < 0) {
+			gsup_send_err_reply(conn, gsup->imsi, gsup->message_type, GMM_CAUSE_INV_MAND_INFO);
+			return -1;
+		}
+	}
 
 	/* Accept all IMEIs */
 	gsup_reply.imei_result = OSMO_GSUP_IMEI_RESULT_ACK;
 	gsup_reply.message_type = OSMO_GSUP_MSGT_CHECK_IMEI_RESULT;
-	msg_out = msgb_alloc_headroom(1024+16, 16, "GSUP Check_IMEI response");
+	msg_out = osmo_gsup_msgb_alloc("GSUP Check_IMEI response");
 	memcpy(gsup_reply.imsi, gsup->imsi, sizeof(gsup_reply.imsi));
 	osmo_gsup_encode(msg_out, &gsup_reply);
 	return osmo_gsup_conn_send(conn, msg_out);
 }
 
+static char namebuf[255];
+#define LOGP_GSUP_FWD(gsup, level, fmt, args ...) \
+	LOGP(DMAIN, level, "Forward %s (class=%s, IMSI=%s, %s->%s): " fmt, \
+	     osmo_gsup_message_type_name(gsup->message_type), \
+	     osmo_gsup_message_class_name(gsup->message_class), \
+	     gsup->imsi, \
+	     osmo_quote_str((const char *)gsup->source_name, gsup->source_name_len), \
+	     osmo_quote_str_buf2(namebuf, sizeof(namebuf), (const char *)gsup->destination_name, gsup->destination_name_len), \
+	     ## args)
+
+static int read_cb_forward(struct osmo_gsup_conn *conn, struct msgb *msg, const struct osmo_gsup_message *gsup)
+{
+	int ret = -EINVAL;
+	struct osmo_gsup_message *gsup_err;
+
+	/* FIXME: it would be better if the msgb never were deallocated immediately by osmo_gsup_addr_send(), which a
+	 * select-loop volatile talloc context could facilitate. Then we would still be able to access gsup-> members
+	 * (pointing into the msgb) even after sending failed, and we wouldn't need to copy this data before sending: */
+	/* Prepare error message (before IEs get deallocated) */
+	gsup_err = talloc_zero(hlr_ctx, struct osmo_gsup_message);
+	OSMO_STRLCPY_ARRAY(gsup_err->imsi, gsup->imsi);
+	gsup_err->message_class = gsup->message_class;
+	gsup_err->destination_name = talloc_memdup(gsup_err, gsup->destination_name, gsup->destination_name_len);
+	gsup_err->destination_name_len = gsup->destination_name_len;
+	gsup_err->message_type = gsup->message_type;
+	gsup_err->session_state = gsup->session_state;
+	gsup_err->session_id = gsup->session_id;
+	gsup_err->source_name = talloc_memdup(gsup_err, gsup->source_name, gsup->source_name_len);
+	gsup_err->source_name_len = gsup->source_name_len;
+
+	/* Check for routing IEs */
+	if (!gsup->source_name || !gsup->source_name_len || !gsup->destination_name || !gsup->destination_name_len) {
+		LOGP_GSUP_FWD(gsup, LOGL_ERROR, "missing routing IEs\n");
+		goto end;
+	}
+
+	/* Verify source name (e.g. "MSC-00-00-00-00-00-00") */
+	if (gsup_route_find(conn->server, gsup->source_name, gsup->source_name_len) != conn) {
+		LOGP_GSUP_FWD(gsup, LOGL_ERROR, "mismatching source name\n");
+		goto end;
+	}
+
+	/* Forward message without re-encoding (so we don't remove unknown IEs) */
+	LOGP_GSUP_FWD(gsup, LOGL_INFO, "checks passed, forwarding\n");
+
+	/* Remove incoming IPA header to be able to prepend an outgoing IPA header */
+	msgb_pull_to_l2(msg);
+	ret = osmo_gsup_addr_send(g_hlr->gs, gsup->destination_name, gsup->destination_name_len, msg);
+	/* AT THIS POINT, THE msg MAY BE DEALLOCATED and the data like gsup->imsi, gsup->source_name etc may all be
+	 * invalid and cause segfaults. */
+	msg = NULL;
+	gsup = NULL;
+	if (ret == -ENODEV)
+		LOGP_GSUP_FWD(gsup_err, LOGL_ERROR, "destination not connected\n");
+	else if (ret)
+		LOGP_GSUP_FWD(gsup_err, LOGL_ERROR, "unknown error %i\n", ret);
+
+end:
+	/* Send error back to source */
+	if (ret) {
+		struct msgb *msg_err = osmo_gsup_msgb_alloc("GSUP forward ERR response");
+		gsup_err->message_type = OSMO_GSUP_MSGT_E_ROUTING_ERROR;
+		osmo_gsup_encode(msg_err, gsup_err);
+		LOGP_GSUP_FWD(gsup_err, LOGL_NOTICE, "Tx %s\n", osmo_gsup_message_type_name(gsup_err->message_type));
+		osmo_gsup_conn_send(conn, msg_err);
+	}
+	talloc_free(gsup_err);
+	if (msg)
+		msgb_free(msg);
+	return ret;
+}
+
 static int read_cb(struct osmo_gsup_conn *conn, struct msgb *msg)
 {
 	static struct osmo_gsup_message gsup;
 	int rc;
 
+	if (!msgb_l2(msg) || !msgb_l2len(msg)) {
+		LOGP(DMAIN, LOGL_ERROR, "missing or empty L2 data\n");
+		msgb_free(msg);
+		return -EINVAL;
+	}
+
 	rc = osmo_gsup_decode(msgb_l2(msg), msgb_l2len(msg), &gsup);
 	if (rc < 0) {
 		LOGP(DMAIN, LOGL_ERROR, "error in GSUP decode: %d\n", rc);
+		msgb_free(msg);
 		return rc;
 	}
 
 	/* 3GPP TS 23.003 Section 2.2 clearly states that an IMSI with less than 5
 	 * digits is impossible.  Even 5 digits is a highly theoretical case */
-	if (strlen(gsup.imsi) < 5)
-		return gsup_send_err_reply(conn, gsup.imsi, gsup.message_type, GMM_CAUSE_INV_MAND_INFO);
+	if (strlen(gsup.imsi) < 5) { /* TODO: move this check to libosmogsm/gsup.c? */
+		LOGP(DMAIN, LOGL_ERROR, "IMSI too short: %s\n", osmo_quote_str(gsup.imsi, -1));
+		gsup_send_err_reply(conn, gsup.imsi, gsup.message_type, GMM_CAUSE_INV_MAND_INFO);
+		msgb_free(msg);
+		return -EINVAL;
+	}
+
+	if (gsup.destination_name_len)
+		return read_cb_forward(conn, msg, &gsup);
 
 	switch (gsup.message_type) {
 	/* requests sent to us */
@@ -519,6 +713,7 @@ static void print_help()
 	printf("  -T --timestamp             Prefix every log line with a timestamp.\n");
 	printf("  -e --log-level number      Set a global loglevel.\n");
 	printf("  -U --db-upgrade            Allow HLR database schema upgrades.\n");
+	printf("  -C --db-check              Quit after opening (and upgrading) the database.\n");
 	printf("  -V --version               Print the version of OsmoHLR.\n");
 }
 
@@ -527,9 +722,10 @@ static struct {
 	const char *db_file;
 	bool daemonize;
 	bool db_upgrade;
+	bool db_check;
 } cmdline_opts = {
 	.config_file = "osmo-hlr.cfg",
-	.db_file = "hlr.db",
+	.db_file = NULL,
 	.daemonize = false,
 	.db_upgrade = false,
 };
@@ -548,6 +744,7 @@ static void handle_options(int argc, char **argv)
 			{"log-level", 1, 0, 'e'},
 			{"timestamp", 0, 0, 'T'},
 			{"db-upgrade", 0, 0, 'U' },
+			{"db-check", 0, 0, 'C' },
 			{"version", 0, 0, 'V' },
 			{0, 0, 0, 0}
 		};
@@ -586,6 +783,9 @@ static void handle_options(int argc, char **argv)
 		case 'U':
 			cmdline_opts.db_upgrade = true;
 			break;
+		case 'C':
+			cmdline_opts.db_check = true;
+			break;
 		case 'V':
 			print_version(1);
 			exit(0);
@@ -597,15 +797,19 @@ static void handle_options(int argc, char **argv)
 			break;
 		}
 	}
-}
 
-static void *hlr_ctx = NULL;
+	if (argc > optind) {
+		fprintf(stderr, "Unsupported positional arguments on command line\n");
+		exit(2);
+	}
+}
 
 static void signal_hdlr(int signal)
 {
 	switch (signal) {
+	case SIGTERM:
 	case SIGINT:
-		LOGP(DMAIN, LOGL_NOTICE, "Terminating due to SIGINT\n");
+		LOGP(DMAIN, LOGL_NOTICE, "Terminating due to signal=%d\n", signal);
 		quit++;
 		break;
 	case SIGUSR1:
@@ -642,9 +846,9 @@ int main(int argc, char **argv)
 
 	g_hlr = talloc_zero(hlr_ctx, struct hlr);
 	INIT_LLIST_HEAD(&g_hlr->euse_list);
-	INIT_LLIST_HEAD(&g_hlr->iuse_list);
 	INIT_LLIST_HEAD(&g_hlr->ss_sessions);
 	INIT_LLIST_HEAD(&g_hlr->ussd_routes);
+	g_hlr->db_file_path = talloc_strdup(g_hlr, HLR_DEFAULT_DB_FILE_PATH);
 
 	/* Init default (call independent) SS session guard timeout value */
 	g_hlr->ncss_guard_timeout = NCSS_GUARD_TIMEOUT_DEFAULT;
@@ -655,10 +859,11 @@ int main(int argc, char **argv)
 		exit(1);
 	}
 
+	osmo_stats_init(hlr_ctx);
 	vty_init(&vty_info);
 	ctrl_vty_init(hlr_ctx);
 	handle_options(argc, argv);
-	hlr_vty_init(&hlr_log_info);
+	hlr_vty_init();
 
 	rc = vty_read_config_file(cmdline_opts.config_file, NULL);
 	if (rc < 0) {
@@ -668,12 +873,6 @@ int main(int argc, char **argv)
 		return rc;
 	}
 
-	/* start telnet after reading config for vty_get_bind_addr() */
-	rc = telnet_init_dynif(hlr_ctx, NULL, vty_get_bind_addr(),
-			       OSMO_VTY_PORT_HLR);
-	if (rc < 0)
-		return rc;
-
 	LOGP(DMAIN, LOGL_NOTICE, "hlr starting\n");
 
 	rc = rand_init();
@@ -682,12 +881,32 @@ int main(int argc, char **argv)
 		exit(1);
 	}
 
-	g_hlr->dbc = db_open(hlr_ctx, cmdline_opts.db_file, true, cmdline_opts.db_upgrade);
+	if (cmdline_opts.db_file)
+		osmo_talloc_replace_string(g_hlr, &g_hlr->db_file_path, cmdline_opts.db_file);
+
+	g_hlr->dbc = db_open(hlr_ctx, g_hlr->db_file_path, true, cmdline_opts.db_upgrade);
 	if (!g_hlr->dbc) {
-		LOGP(DMAIN, LOGL_FATAL, "Error opening database\n");
+		LOGP(DMAIN, LOGL_FATAL, "Error opening database %s\n", osmo_quote_str(g_hlr->db_file_path, -1));
 		exit(1);
 	}
 
+	if (cmdline_opts.db_check) {
+		LOGP(DMAIN, LOGL_NOTICE, "Cmdline option --db-check: Database was opened successfully, quitting.\n");
+		db_close(g_hlr->dbc);
+		log_fini();
+		talloc_free(hlr_ctx);
+		talloc_free(tall_vty_ctx);
+		talloc_disable_null_tracking();
+		exit(0);
+	}
+
+	/* start telnet after reading config for vty_get_bind_addr() */
+	rc = telnet_init_dynif(hlr_ctx, NULL, vty_get_bind_addr(),
+			       OSMO_VTY_PORT_HLR);
+	if (rc < 0)
+		return rc;
+
+
 	g_hlr->gs = osmo_gsup_server_create(hlr_ctx, g_hlr->gsup_bind_addr, OSMO_GSUP_PORT,
 					    read_cb, &g_lu_ops, g_hlr);
 	if (!g_hlr->gs) {
@@ -700,6 +919,7 @@ int main(int argc, char **argv)
 
 	osmo_init_ignore_signals();
 	signal(SIGINT, &signal_hdlr);
+	signal(SIGTERM, &signal_hdlr);
 	signal(SIGUSR1, &signal_hdlr);
 
 	if (cmdline_opts.daemonize) {

src/hlr_db_tool.c

@@ -29,9 +29,9 @@
 #include <osmocom/core/logging.h>
 #include <osmocom/core/application.h>
 
-#include "logging.h"
-#include "db.h"
-#include "rand.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/db.h>
+#include <osmocom/hlr/rand.h>
 
 struct hlr_db_tool_ctx {
 	/* DB context */
@@ -302,7 +302,7 @@ void import_nitb_subscr(sqlite3 *nitb_db, sqlite3_stmt *stmt)
 
 	snprintf(imsi_str, sizeof(imsi_str), "%" PRId64, imsi);
 
-	rc = db_subscr_create(dbc, imsi_str);
+	rc = db_subscr_create(dbc, imsi_str, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS);
 	if (rc < 0) {
 		LOGP(DDB, LOGL_ERROR, "OsmoNITB DB import to %s: failed to create IMSI %s: %d: %s\n",
 		     dbc->fname,

src/hlr_ussd.c

@@ -29,11 +29,12 @@
 #include <string.h>
 #include <errno.h>
 
-#include "hlr.h"
-#include "hlr_ussd.h"
-#include "gsup_server.h"
-#include "gsup_router.h"
-#include "logging.h"
+#include <osmocom/hlr/hlr.h>
+#include <osmocom/hlr/hlr_ussd.h>
+#include <osmocom/hlr/gsup_server.h>
+#include <osmocom/hlr/gsup_router.h>
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/db.h>
 
 /***********************************************************************
  * core data structures expressing config from VTY
@@ -149,7 +150,7 @@ struct ss_session {
 	/* link us to hlr->ss_sessions */
 	struct llist_head list;
 	/* imsi of this session */
-	char imsi[GSM23003_IMSI_MAX_DIGITS+2];
+	char imsi[OSMO_IMSI_BUF_SIZE];
 	/* ID of this session (unique per IMSI) */
 	uint32_t session_id;
 	/* state of the session */
@@ -166,6 +167,12 @@ struct ss_session {
 		const struct hlr_iuse *iuse;
 	} u;
 
+	/* subscriber's vlr_number
+	 * MO USSD: originating MSC's vlr_number
+	 * MT USSD: looked up once per session and cached here */
+	uint8_t *vlr_number;
+	size_t vlr_number_len;
+
 	/* we don't keep a pointer to the osmo_gsup_{route,conn} towards the MSC/VLR here,
 	 * as this might change during inter-VLR hand-over, and we simply look-up the serving MSC/VLR
 	 * every time we receive an USSD component from the EUSE */
@@ -222,6 +229,35 @@ struct ss_session *ss_session_alloc(struct hlr *hlr, const char *imsi, uint32_t
  * handling functions for encoding SS messages + wrapping them in GSUP
  ***********************************************************************/
 
+/* Resolve the target MSC by ss->imsi and send GSUP message. */
+static int ss_gsup_send(struct ss_session *ss, struct osmo_gsup_server *gs, struct msgb *msg)
+{
+	struct hlr_subscriber subscr = {};
+	int rc;
+
+	/* Use vlr_number as looked up by the caller, or look up now. */
+	if (!ss->vlr_number) {
+		rc = db_subscr_get_by_imsi(g_hlr->dbc, ss->imsi, &subscr);
+		if (rc < 0) {
+			LOGPSS(ss, LOGL_ERROR, "Cannot find subscriber, cannot route GSUP message\n");
+			msgb_free(msg);
+			return -EINVAL;
+		}
+		ss->vlr_number = (uint8_t *)talloc_strdup(ss, subscr.vlr_number);
+		ss->vlr_number_len = strlen(subscr.vlr_number) + 1;
+	}
+
+	/* Check for empty string (all vlr_number strings end in "\0", because otherwise gsup_route_find() fails) */
+	if (ss->vlr_number_len == 1) {
+		LOGPSS(ss, LOGL_ERROR, "Cannot send GSUP message, no VLR number stored for subscriber\n");
+		msgb_free(msg);
+		return -EINVAL;
+	}
+
+	LOGPSS(ss, LOGL_DEBUG, "Tx SS/USSD to VLR %s\n", osmo_quote_str((char *)ss->vlr_number, ss->vlr_number_len));
+	return osmo_gsup_addr_send(gs, ss->vlr_number, ss->vlr_number_len, msg);
+}
+
 static int ss_tx_to_ms(struct ss_session *ss, enum osmo_gsup_message_type gsup_msg_type,
 			bool final, struct msgb *ss_msg)
 
@@ -241,13 +277,12 @@ static int ss_tx_to_ms(struct ss_session *ss, enum osmo_gsup_message_type gsup_m
 		resp.ss_info_len = msgb_length(ss_msg);
 	}
 
-	resp_msg = gsm0480_msgb_alloc_name(__func__);
+	resp_msg = msgb_alloc_headroom(4000, 64, __func__);
 	OSMO_ASSERT(resp_msg);
 	osmo_gsup_encode(resp_msg, &resp);
 	msgb_free(ss_msg);
 
-	/* FIXME: resolve this based on the database vlr_addr */
-	return osmo_gsup_addr_send(g_hlr->gs, (uint8_t *)"MSC-00-00-00-00-00-00", 22, resp_msg);
+	return ss_gsup_send(ss, g_hlr->gs, resp_msg);
 }
 
 #if 0
@@ -282,7 +317,7 @@ static int ss_tx_ussd_7bit(struct ss_session *ss, bool final, uint8_t invoke_id,
  * Internal USSD Handlers
  ***********************************************************************/
 
-#include "db.h"
+#include <osmocom/hlr/db.h>
 
 static int handle_ussd_own_msisdn(struct osmo_gsup_conn *conn, struct ss_session *ss,
 				  const struct osmo_gsup_message *gsup, const struct ss_request *req)
@@ -301,11 +336,11 @@ static int handle_ussd_own_msisdn(struct osmo_gsup_conn *conn, struct ss_session
 		ss_tx_ussd_7bit(ss, true, req->invoke_id, buf);
 		break;
 	case -ENOENT:
-		ss_tx_error(ss, true, GSM0480_ERR_CODE_UNKNOWN_SUBSCRIBER);
+		ss_tx_error(ss, req->invoke_id, GSM0480_ERR_CODE_UNKNOWN_SUBSCRIBER);
 		break;
 	case -EIO:
 	default:
-		ss_tx_error(ss, true, GSM0480_ERR_CODE_SYSTEM_FAILURE);
+		ss_tx_error(ss, req->invoke_id, GSM0480_ERR_CODE_SYSTEM_FAILURE);
 		break;
 	}
 	return 0;
@@ -429,12 +464,10 @@ static int handle_ussd(struct osmo_gsup_conn *conn, struct ss_session *ss,
 	}
 
 	if (is_euse_originated) {
-		msg_out = msgb_alloc_headroom(1024+16, 16, "GSUP USSD FW");
-		OSMO_ASSERT(msg_out);
+		msg_out = osmo_gsup_msgb_alloc("GSUP USSD FW");
 		/* Received from EUSE, Forward to VLR */
 		osmo_gsup_encode(msg_out, gsup);
-		/* FIXME: resolve this based on the database vlr_addr */
-		osmo_gsup_addr_send(conn->server, (uint8_t *)"MSC-00-00-00-00-00-00", 22, msg_out);
+		ss_gsup_send(ss, conn->server, msg_out);
 	} else {
 		/* Received from VLR (MS) */
 		if (ss->is_external) {
@@ -447,8 +480,7 @@ static int handle_ussd(struct osmo_gsup_conn *conn, struct ss_session *ss,
 				LOGPSS(ss, LOGL_ERROR, "Cannot find conn for EUSE %s\n", addr);
 				ss_tx_error(ss, req->invoke_id, GSM0480_ERR_CODE_SYSTEM_FAILURE);
 			} else {
-				msg_out = msgb_alloc_headroom(1024+16, 16, "GSUP USSD FW");
-				OSMO_ASSERT(msg_out);
+				msg_out = osmo_gsup_msgb_alloc("GSUP USSD FW");
 				osmo_gsup_encode(msg_out, gsup);
 				osmo_gsup_conn_send(conn, msg_out);
 			}
@@ -471,6 +503,7 @@ int rx_proc_ss_req(struct osmo_gsup_conn *conn, const struct osmo_gsup_message *
 	struct hlr *hlr = conn->server->priv;
 	struct ss_session *ss;
 	struct ss_request req = {0};
+	struct gsup_route *gsup_rt;
 
 	LOGP(DSS, LOGL_DEBUG, "%s/0x%08x: Process SS (%s)\n", gsup->imsi, gsup->session_id,
 		osmo_gsup_session_state_name(gsup->session_state));
@@ -484,6 +517,11 @@ int rx_proc_ss_req(struct osmo_gsup_conn *conn, const struct osmo_gsup_message *
 			/* FIXME: Send a Reject component? */
 			goto out_err;
 		}
+	} else if (gsup->session_state != OSMO_GSUP_SESSION_STATE_END) {
+		LOGP(DSS, LOGL_ERROR, "%s/0x%082x: Missing SS payload for '%s'\n",
+		     gsup->imsi, gsup->session_id,
+		     osmo_gsup_session_state_name(gsup->session_state));
+		goto out_err;
 	}
 
 	switch (gsup->session_state) {
@@ -500,6 +538,20 @@ int rx_proc_ss_req(struct osmo_gsup_conn *conn, const struct osmo_gsup_message *
 				gsup->imsi, gsup->session_id);
 			goto out_err;
 		}
+		/* Get IPA name from VLR conn and save as ss->vlr_number */
+		if (!conn_is_euse(conn)) {
+			gsup_rt = gsup_route_find_by_conn(conn);
+			if (gsup_rt) {
+				ss->vlr_number = (uint8_t *)talloc_strdup(ss, (const char *)gsup_rt->addr);
+				ss->vlr_number_len = strlen((const char *)gsup_rt->addr) + 1;
+				LOGPSS(ss, LOGL_DEBUG, "Destination IPA name retrieved from GSUP route: %s\n",
+				       osmo_quote_str((const char *)ss->vlr_number, ss->vlr_number_len));
+			} else {
+				LOGPSS(ss, LOGL_NOTICE, "Could not find GSUP route, therefore can't set the destination"
+							" IPA name. We'll try to look it up later, but this should not"
+							" have happened.\n");
+			}
+		}
 		if (ss_op_is_ussd(req.opcode)) {
 			if (conn_is_euse(conn)) {
 				/* EUSE->VLR: MT USSD. EUSE is known ('conn'), VLR is to be resolved */
@@ -557,13 +609,18 @@ int rx_proc_ss_req(struct osmo_gsup_conn *conn, const struct osmo_gsup_message *
 				gsup->imsi, gsup->session_id);
 			goto out_err;
 		}
-		if (ss_op_is_ussd(req.opcode)) {
-			/* dispatch unstructured SS to routing */
-			handle_ussd(conn, ss, gsup, &req);
-		} else {
-			/* dispatch non-call SS to internal code */
-			handle_ss(ss, gsup, &req);
+
+		/* SS payload is optional for END */
+		if (gsup->ss_info && gsup->ss_info_len) {
+			if (ss_op_is_ussd(req.opcode)) {
+				/* dispatch unstructured SS to routing */
+				handle_ussd(conn, ss, gsup, &req);
+			} else {
+				/* dispatch non-call SS to internal code */
+				handle_ss(ss, gsup, &req);
+			}
 		}
+
 		ss_session_free(ss);
 		break;
 	default:

src/hlr_vty.c

@@ -27,16 +27,18 @@
 
 #include <osmocom/core/talloc.h>
 #include <osmocom/vty/vty.h>
+#include <osmocom/vty/stats.h>
 #include <osmocom/vty/command.h>
 #include <osmocom/vty/logging.h>
 #include <osmocom/vty/misc.h>
 #include <osmocom/abis/ipa.h>
 
-#include "hlr.h"
-#include "hlr_vty.h"
-#include "hlr_vty_subscr.h"
-#include "hlr_ussd.h"
-#include "gsup_server.h"
+#include <osmocom/hlr/db.h>
+#include <osmocom/hlr/hlr.h>
+#include <osmocom/hlr/hlr_vty.h>
+#include <osmocom/hlr/hlr_vty_subscr.h>
+#include <osmocom/hlr/hlr_ussd.h>
+#include <osmocom/hlr/gsup_server.h>
 
 struct cmd_node hlr_node = {
 	HLR_NODE,
@@ -71,6 +73,27 @@ DEFUN(cfg_gsup,
 static int config_write_hlr(struct vty *vty)
 {
 	vty_out(vty, "hlr%s", VTY_NEWLINE);
+	if (g_hlr->store_imei)
+		vty_out(vty, " store-imei%s", VTY_NEWLINE);
+	if (g_hlr->db_file_path && strcmp(g_hlr->db_file_path, HLR_DEFAULT_DB_FILE_PATH))
+		vty_out(vty, " database %s%s", g_hlr->db_file_path, VTY_NEWLINE);
+	if (g_hlr->subscr_create_on_demand) {
+		const char *flags_str = "none";
+		uint8_t flags = g_hlr->subscr_create_on_demand_flags;
+		unsigned int rand_msisdn_len = g_hlr->subscr_create_on_demand_rand_msisdn_len;
+
+		if ((flags & DB_SUBSCR_FLAG_NAM_CS) && (flags & DB_SUBSCR_FLAG_NAM_PS))
+			flags_str = "cs+ps";
+		else if (flags & DB_SUBSCR_FLAG_NAM_CS)
+			flags_str = "cs";
+		else if (flags & DB_SUBSCR_FLAG_NAM_PS)
+			flags_str = "ps";
+
+		if (rand_msisdn_len)
+			vty_out(vty, " subscriber-create-on-demand %i %s%s", rand_msisdn_len, flags_str, VTY_NEWLINE);
+		else
+			vty_out(vty, " subscriber-create-on-demand no-msisdn %s%s", flags_str, VTY_NEWLINE);
+	}
 	return CMD_SUCCESS;
 }
 
@@ -127,7 +150,7 @@ DEFUN(cfg_hlr_gsup_bind_ip,
  * USSD Entity
  ***********************************************************************/
 
-#include "hlr_ussd.h"
+#include <osmocom/hlr/hlr_ussd.h>
 
 #define USSD_STR "USSD Configuration\n"
 #define UROUTE_STR "Routing Configuration\n"
@@ -221,6 +244,15 @@ DEFUN(cfg_ussd_no_defaultroute, cfg_ussd_no_defaultroute_cmd,
 	return CMD_SUCCESS;
 }
 
+DEFUN(cfg_database, cfg_database_cmd,
+	"database PATH",
+	"Set the path to the HLR database file\n"
+	"Relative or absolute file system path to the database file (default is '" HLR_DEFAULT_DB_FILE_PATH "')\n")
+{
+	osmo_talloc_replace_string(g_hlr, &g_hlr->db_file_path, argv[0]);
+	return CMD_SUCCESS;
+}
+
 struct cmd_node euse_node = {
 	EUSE_NODE,
 	"%s(config-hlr-euse)# ",
@@ -305,6 +337,59 @@ DEFUN(cfg_ncss_guard_timeout, cfg_ncss_guard_timeout_cmd,
 	return CMD_SUCCESS;
 }
 
+DEFUN(cfg_store_imei, cfg_store_imei_cmd,
+	"store-imei",
+	"Save the IMEI in the database when receiving Check IMEI requests. Note that an MSC does not necessarily send"
+	" Check IMEI requests (for OsmoMSC, you may want to set 'check-imei-rqd 1').")
+{
+	g_hlr->store_imei = true;
+	return CMD_SUCCESS;
+}
+
+DEFUN(cfg_no_store_imei, cfg_no_store_imei_cmd,
+	"no store-imei",
+	"Do not save the IMEI in the database, when receiving Check IMEI requests.")
+{
+	g_hlr->store_imei = false;
+	return CMD_SUCCESS;
+}
+
+DEFUN(cfg_subscr_create_on_demand, cfg_subscr_create_on_demand_cmd,
+	"subscriber-create-on-demand (no-msisdn|<3-15>) (none|cs|ps|cs+ps)",
+	"Make a new record when a subscriber is first seen.\n"
+	"Do not automatically assign MSISDN.\n"
+	"Length of an automatically assigned MSISDN.\n"
+	"Do not allow any NAM (Network Access Mode) by default.\n"
+	"Allow access to circuit switched NAM by default.\n"
+	"Allow access to packet switched NAM by default.\n"
+	"Allow access to circuit and packet switched NAM by default.\n")
+{
+	unsigned int rand_msisdn_len = 0;
+	uint8_t flags = 0x00;
+
+	if (strcmp(argv[0], "no-msisdn") != 0)
+		rand_msisdn_len = atoi(argv[0]);
+
+	if (strstr(argv[1], "cs"))
+		flags |= DB_SUBSCR_FLAG_NAM_CS;
+	if (strstr(argv[1], "ps"))
+		flags |= DB_SUBSCR_FLAG_NAM_PS;
+
+	g_hlr->subscr_create_on_demand = true;
+	g_hlr->subscr_create_on_demand_rand_msisdn_len = rand_msisdn_len;
+	g_hlr->subscr_create_on_demand_flags = flags;
+
+	return CMD_SUCCESS;
+}
+
+DEFUN(cfg_no_subscr_create_on_demand, cfg_no_subscr_create_on_demand_cmd,
+	"no subscriber-create-on-demand",
+	"Do not make a new record when a subscriber is first seen.\n")
+{
+	g_hlr->subscr_create_on_demand = false;
+	return CMD_SUCCESS;
+}
+
 /***********************************************************************
  * Common Code
  ***********************************************************************/
@@ -344,10 +429,11 @@ int hlr_vty_is_config_node(struct vty *vty, int node)
 	}
 }
 
-void hlr_vty_init(const struct log_info *cat)
+void hlr_vty_init(void)
 {
-	logging_vty_add_cmds(cat);
+	logging_vty_add_cmds();
 	osmo_talloc_vty_add_cmds();
+	osmo_stats_vty_add_cmds();
 
 	install_element_ve(&show_gsup_conn_cmd);
 
@@ -359,6 +445,8 @@ void hlr_vty_init(const struct log_info *cat)
 
 	install_element(GSUP_NODE, &cfg_hlr_gsup_bind_ip_cmd);
 
+	install_element(HLR_NODE, &cfg_database_cmd);
+
 	install_element(HLR_NODE, &cfg_euse_cmd);
 	install_element(HLR_NODE, &cfg_no_euse_cmd);
 	install_node(&euse_node, config_write_euse);
@@ -368,6 +456,10 @@ void hlr_vty_init(const struct log_info *cat)
 	install_element(HLR_NODE, &cfg_ussd_defaultroute_cmd);
 	install_element(HLR_NODE, &cfg_ussd_no_defaultroute_cmd);
 	install_element(HLR_NODE, &cfg_ncss_guard_timeout_cmd);
+	install_element(HLR_NODE, &cfg_store_imei_cmd);
+	install_element(HLR_NODE, &cfg_no_store_imei_cmd);
+	install_element(HLR_NODE, &cfg_subscr_create_on_demand_cmd);
+	install_element(HLR_NODE, &cfg_no_subscr_create_on_demand_cmd);
 
 	hlr_vty_subscriber_init();
 }

src/hlr_vty_subscr.c

@@ -28,8 +28,8 @@
 #include <osmocom/vty/command.h>
 #include <osmocom/core/utils.h>
 
-#include "hlr.h"
-#include "db.h"
+#include <osmocom/hlr/hlr.h>
+#include <osmocom/hlr/db.h>
 
 struct vty;
 
@@ -47,17 +47,34 @@ get_datestr(const time_t *t, char *datebuf)
 	return s;
 }
 
+static void dump_last_lu_seen(struct vty *vty, const char *domain_label, time_t last_lu_seen)
+{
+	char datebuf[26]; /* for ctime_r(3) */
+	if (!last_lu_seen)
+		return;
+	vty_out(vty, "    last LU seen on %s: %s UTC%s", domain_label, get_datestr(&last_lu_seen, datebuf),
+		VTY_NEWLINE);
+}
+
 static void subscr_dump_full_vty(struct vty *vty, struct hlr_subscriber *subscr)
 {
 	int rc;
 	struct osmo_sub_auth_data aud2g;
 	struct osmo_sub_auth_data aud3g;
-	char datebuf[26]; /* for ctime_r(3) */
 
 	vty_out(vty, "    ID: %"PRIu64"%s", subscr->id, VTY_NEWLINE);
 
 	vty_out(vty, "    IMSI: %s%s", *subscr->imsi ? subscr->imsi : "none", VTY_NEWLINE);
 	vty_out(vty, "    MSISDN: %s%s", *subscr->msisdn ? subscr->msisdn : "none", VTY_NEWLINE);
+
+	if (*subscr->imei) {
+		char checksum = osmo_luhn(subscr->imei, 14);
+		if (checksum == -EINVAL)
+			vty_out(vty, "    IMEI: %s (INVALID LENGTH!)%s", subscr->imei, VTY_NEWLINE);
+		else
+			vty_out(vty, "    IMEI: %s%c%s", subscr->imei, checksum, VTY_NEWLINE);
+	}
+
 	if (*subscr->vlr_number)
 		vty_out(vty, "    VLR number: %s%s", subscr->vlr_number, VTY_NEWLINE);
 	if (*subscr->sgsn_number)
@@ -78,8 +95,8 @@ static void subscr_dump_full_vty(struct vty *vty, struct hlr_subscriber *subscr)
 		vty_out(vty, "    PS disabled%s", VTY_NEWLINE);
 	if (subscr->ms_purged_ps)
 		vty_out(vty, "    PS purged%s", VTY_NEWLINE);
-	if (subscr->last_lu_seen)
-		vty_out(vty, "    last LU seen: %s UTC%s", get_datestr(&subscr->last_lu_seen, datebuf), VTY_NEWLINE);
+	dump_last_lu_seen(vty, "CS", subscr->last_lu_seen);
+	dump_last_lu_seen(vty, "PS", subscr->last_lu_seen_ps);
 
 	if (!*subscr->imsi)
 		return;
@@ -131,6 +148,7 @@ static void subscr_dump_full_vty(struct vty *vty, struct hlr_subscriber *subscr)
 
 static int get_subscr_by_argv(struct vty *vty, const char *type, const char *id, struct hlr_subscriber *subscr)
 {
+	char imei_buf[GSM23003_IMEI_NUM_DIGITS_NO_CHK+1];
 	int rc = -1;
 	if (strcmp(type, "imsi") == 0)
 		rc = db_subscr_get_by_imsi(g_hlr->dbc, id, subscr);
@@ -138,6 +156,17 @@ static int get_subscr_by_argv(struct vty *vty, const char *type, const char *id,
 		rc = db_subscr_get_by_msisdn(g_hlr->dbc, id, subscr);
 	else if (strcmp(type, "id") == 0)
 		rc = db_subscr_get_by_id(g_hlr->dbc, atoll(id), subscr);
+	else if (strcmp(type, "imei") == 0) {
+		/* Verify IMEI with checksum digit */
+		if (osmo_imei_str_valid(id, true)) {
+			/* Cut the checksum off */
+			osmo_strlcpy(imei_buf, id, sizeof(imei_buf));
+			id = imei_buf;
+			vty_out(vty, "%% Checksum validated and stripped for search: imei = '%s'%s", id,
+				VTY_NEWLINE);
+		}
+		rc = db_subscr_get_by_imei(g_hlr->dbc, id, subscr);
+	}
 	if (rc)
 		vty_out(vty, "%% No subscriber for %s = '%s'%s",
 			type, id, VTY_NEWLINE);
@@ -147,12 +176,13 @@ static int get_subscr_by_argv(struct vty *vty, const char *type, const char *id,
 #define SUBSCR_CMD "subscriber "
 #define SUBSCR_CMD_HELP "Subscriber management commands\n"
 
-#define SUBSCR_ID "(imsi|msisdn|id) IDENT"
+#define SUBSCR_ID "(imsi|msisdn|id|imei) IDENT"
 #define SUBSCR_ID_HELP \
 	"Identify subscriber by IMSI\n" \
 	"Identify subscriber by MSISDN (phone number)\n" \
 	"Identify subscriber by database ID\n" \
-	"IMSI/MSISDN/ID of the subscriber\n"
+	"Identify subscriber by IMEI\n" \
+	"IMSI/MSISDN/ID/IMEI of the subscriber\n"
 
 #define SUBSCR 		SUBSCR_CMD SUBSCR_ID " "
 #define SUBSCR_HELP	SUBSCR_CMD_HELP SUBSCR_ID_HELP
@@ -198,7 +228,7 @@ DEFUN(subscriber_create,
 		return CMD_WARNING;
 	}
 
-	rc = db_subscr_create(g_hlr->dbc, imsi);
+	rc = db_subscr_create(g_hlr->dbc, imsi, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS);
 
 	if (rc) {
 		if (rc == -EEXIST)
@@ -508,6 +538,81 @@ DEFUN(subscriber_aud3g,
 	return CMD_SUCCESS;
 }
 
+DEFUN(subscriber_imei,
+      subscriber_imei_cmd,
+      SUBSCR_UPDATE "imei (none|IMEI)",
+      SUBSCR_UPDATE_HELP
+      "Set IMEI of the subscriber (normally populated from MSC, no need to set this manually)\n"
+      "Forget IMEI\n"
+      "Set IMEI (use for debug only!)\n")
+{
+	struct hlr_subscriber subscr;
+	const char *id_type = argv[0];
+	const char *id = argv[1];
+	const char *imei = argv[2];
+	char imei_buf[GSM23003_IMEI_NUM_DIGITS_NO_CHK+1];
+
+	if (strcmp(imei, "none") == 0)
+		imei = NULL;
+	else {
+		/* Verify IMEI with checksum digit */
+		if (osmo_imei_str_valid(imei, true)) {
+			/* Cut the checksum off */
+			osmo_strlcpy(imei_buf, imei, sizeof(imei_buf));
+			imei = imei_buf;
+		} else if (!osmo_imei_str_valid(imei, false)) {
+			vty_out(vty, "%% IMEI invalid: '%s'%s", imei, VTY_NEWLINE);
+			return CMD_WARNING;
+		}
+	}
+
+	if (get_subscr_by_argv(vty, id_type, id, &subscr))
+		return CMD_WARNING;
+
+	if (db_subscr_update_imei_by_imsi(g_hlr->dbc, subscr.imsi, imei)) {
+		vty_out(vty, "%% Error: cannot update IMEI for subscriber IMSI='%s'%s",
+			subscr.imsi, VTY_NEWLINE);
+		return CMD_WARNING;
+	}
+
+	if (imei)
+		vty_out(vty, "%% Updated subscriber IMSI='%s' to IMEI='%s'%s",
+			subscr.imsi, imei, VTY_NEWLINE);
+	else
+		vty_out(vty, "%% Updated subscriber IMSI='%s': removed IMEI%s",
+			subscr.imsi, VTY_NEWLINE);
+
+	return CMD_SUCCESS;
+}
+
+DEFUN(subscriber_nam,
+      subscriber_nam_cmd,
+      SUBSCR_UPDATE "network-access-mode (none|cs|ps|cs+ps)",
+      SUBSCR_UPDATE_HELP
+      "Set Network Access Mode (NAM) of the subscriber\n"
+      "Do not allow access to circuit switched or packet switched services\n"
+      "Allow access to circuit switched services only\n"
+      "Allow access to packet switched services only\n"
+      "Allow access to both circuit and packet switched services\n")
+{
+	struct hlr_subscriber subscr;
+	const char *id_type = argv[0];
+	const char *id = argv[1];
+	bool nam_cs = strstr(argv[2], "cs");
+	bool nam_ps = strstr(argv[2], "ps");
+
+	if (get_subscr_by_argv(vty, id_type, id, &subscr))
+		return CMD_WARNING;
+
+	if (nam_cs != subscr.nam_cs)
+		hlr_subscr_nam(g_hlr, &subscr, nam_cs, 0);
+	if (nam_ps != subscr.nam_ps)
+		hlr_subscr_nam(g_hlr, &subscr, nam_ps, 1);
+
+	return CMD_SUCCESS;
+}
+
+
 void hlr_vty_subscriber_init(void)
 {
 	install_element_ve(&subscriber_show_cmd);
@@ -519,4 +624,6 @@ void hlr_vty_subscriber_init(void)
 	install_element(ENABLE_NODE, &subscriber_aud2g_cmd);
 	install_element(ENABLE_NODE, &subscriber_no_aud3g_cmd);
 	install_element(ENABLE_NODE, &subscriber_aud3g_cmd);
+	install_element(ENABLE_NODE, &subscriber_imei_cmd);
+	install_element(ENABLE_NODE, &subscriber_nam_cmd);
 }

src/logging.c

@@ -1,5 +1,5 @@
 #include <osmocom/core/utils.h>
-#include "logging.h"
+#include <osmocom/hlr/logging.h>
 
 const struct log_info_cat hlr_log_info_cat[] = {
 	[DMAIN] = {
@@ -25,7 +25,12 @@ const struct log_info_cat hlr_log_info_cat[] = {
 		.color = "\033[1;34m",
 		.enabled = 1, .loglevel = LOGL_NOTICE,
 	},
-
+	[DMSLOOKUP] = {
+		.name = "DMSLOOKUP",
+		.description = "Mobile Subscriber Lookup",
+		.color = "\033[1;35m",
+		.enabled = 1, .loglevel = LOGL_NOTICE,
+	},
 };
 
 const struct log_info hlr_log_info = {

src/luop.c

@@ -28,10 +28,10 @@
 #include <osmocom/gsm/gsup.h>
 #include <osmocom/gsm/apn.h>
 
-#include "gsup_server.h"
-#include "gsup_router.h"
-#include "logging.h"
-#include "luop.h"
+#include <osmocom/hlr/gsup_server.h>
+#include <osmocom/hlr/gsup_router.h>
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/luop.h>
 
 const struct value_string lu_state_names[] = {
 	{ LU_S_NULL,			"NULL" },
@@ -50,8 +50,7 @@ static void _luop_tx_gsup(struct lu_operation *luop,
 {
 	struct msgb *msg_out;
 
-	msg_out = msgb_alloc_headroom(1024+16, 16, "GSUP LUOP");
-	OSMO_ASSERT(msg_out);
+	msg_out = osmo_gsup_msgb_alloc("GSUP LUOP");
 	osmo_gsup_encode(msg_out, gsup);
 
 	osmo_gsup_addr_send(luop->gsup_server, luop->peer,

src/mslookup/Makefile.am

@@ -0,0 +1,37 @@
+# This is _NOT_ the library release version, it's an API version.
+# Please read chapter "Library interface versions" of the libtool documentation
+# before making any modifications: https://www.gnu.org/software/libtool/manual/html_node/Versioning.html
+LIBVERSION=0:0:0
+
+AM_CPPFLAGS = $(all_includes) -I$(top_srcdir)/include -I$(top_builddir)/include
+AM_CFLAGS = -fPIC -Wall $(PCSC_CFLAGS) $(TALLOC_CFLAGS) $(LIBOSMOCORE_CFLAGS)
+AM_LDFLAGS = $(COVERAGE_LDFLAGS)
+
+lib_LTLIBRARIES = libosmo-mslookup.la
+
+libosmo_mslookup_la_SOURCES = \
+	mdns.c \
+	mdns_msg.c \
+	mdns_rfc.c \
+	mdns_sock.c \
+	mslookup.c \
+	mslookup_client.c \
+	mslookup_client_fake.c \
+	mslookup_client_mdns.c \
+	$(NULL)
+
+libosmo_mslookup_la_LDFLAGS = -version-info $(LIBVERSION)
+libosmo_mslookup_la_LIBADD = \
+	$(LIBOSMOCORE_LIBS) \
+	$(LIBOSMOGSM_LIBS) \
+	$(TALLOC_LIBS) \
+	$(NULL)
+
+bin_PROGRAMS = osmo-mslookup-client
+osmo_mslookup_client_SOURCES = osmo-mslookup-client.c
+osmo_mslookup_client_LDADD = \
+	libosmo-mslookup.la \
+	$(LIBOSMOCORE_LIBS) \
+	$(TALLOC_LIBS) \
+	$(NULL)
+osmo_mslookup_client_CFLAGS = $(TALLOC_CFLAGS) $(LIBOSMOCORE_CFLAGS)

src/mslookup/mdns.c

@@ -0,0 +1,425 @@
+/* mslookup specific functions for encoding and decoding mslookup queries/results into mDNS packets, using the high
+ * level functions from mdns_msg.c and mdns_record.c to build the request/answer messages. */
+
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <osmocom/hlr/logging.h>
+#include <osmocom/core/msgb.h>
+#include <osmocom/mslookup/mslookup.h>
+#include <osmocom/mslookup/mdns_msg.h>
+#include <osmocom/mslookup/mdns_rfc.h>
+#include <errno.h>
+#include <inttypes.h>
+
+static struct msgb *osmo_mdns_msgb_alloc(const char *label)
+{
+	return msgb_alloc(1024, label);
+}
+
+/*! Combine the mslookup query service, ID and ID type into a domain string.
+ * \param[in] domain_suffix  is appended to each domain in the queries to avoid colliding with the top-level domains
+ *                           administrated by IANA. Example: "mdns.osmocom.org"
+ * \returns allocated buffer with the resulting domain (i.e. "sip.voice.123.msisdn.mdns.osmocom.org") on success,
+ * 	    NULL on failure.
+ */
+static char *domain_from_query(void *ctx, const struct osmo_mslookup_query *query, const char *domain_suffix)
+{
+	const char *id;
+
+	/* Get id from query */
+	switch (query->id.type) {
+		case OSMO_MSLOOKUP_ID_IMSI:
+			id = query->id.imsi;
+			break;
+		case OSMO_MSLOOKUP_ID_MSISDN:
+			id = query->id.msisdn;
+			break;
+		default:
+			LOGP(DMSLOOKUP, LOGL_ERROR, "can't encode mslookup query id type %i", query->id.type);
+			return NULL;
+	}
+
+	return talloc_asprintf(ctx, "%s.%s.%s.%s", query->service, id, osmo_mslookup_id_type_name(query->id.type),
+			       domain_suffix);
+}
+
+/*! Split up query service, ID and ID type from a domain string into a mslookup query.
+ * \param[in] domain  with domain_suffix, e.g. "sip.voice.123.msisdn.mdns.osmocom.org"
+ * \param[in] domain_suffix  is appended to each domain in the queries to avoid colliding with the top-level domains
+ *                           administrated by IANA. It is not part of the resulting struct osmo_mslookup_query, so we
+ *                           remove it in this function. Example: "mdns.osmocom.org"
+ */
+int query_from_domain(struct osmo_mslookup_query *query, const char *domain, const char *domain_suffix)
+{
+	int domain_len = strlen(domain) - strlen(domain_suffix) - 1;
+	char domain_buf[OSMO_MDNS_RFC_MAX_NAME_LEN];
+
+	if (domain_len <= 0 || domain_len >= sizeof(domain_buf))
+		return -EINVAL;
+
+	if (domain[domain_len] != '.' || strcmp(domain + domain_len + 1, domain_suffix) != 0)
+		return -EINVAL;
+
+	memcpy(domain_buf, domain, domain_len);
+	domain_buf[domain_len] = '\0';
+	return osmo_mslookup_query_init_from_domain_str(query, domain_buf);
+}
+
+/*! Encode a mslookup query into a mDNS packet.
+ * \param[in] domain_suffix  is appended to each domain in the queries to avoid colliding with the top-level domains
+ *                           administrated by IANA. Example: "mdns.osmocom.org"
+ * \returns msgb, or NULL on error.
+ */
+struct msgb *osmo_mdns_query_encode(void *ctx, uint16_t packet_id, const struct osmo_mslookup_query *query,
+				    const char *domain_suffix)
+{
+	struct osmo_mdns_msg_request req = {0};
+	struct msgb *msg = osmo_mdns_msgb_alloc(__func__);
+
+	req.id = packet_id;
+	req.type = OSMO_MDNS_RFC_RECORD_TYPE_ALL;
+	req.domain = domain_from_query(ctx, query, domain_suffix);
+	if (!req.domain)
+		goto error;
+	if (osmo_mdns_msg_request_encode(ctx, msg, &req))
+		goto error;
+	talloc_free(req.domain);
+	return msg;
+error:
+	msgb_free(msg);
+	talloc_free(req.domain);
+	return NULL;
+}
+
+/*! Decode a mDNS request packet into a mslookup query.
+ * \param[out] packet_id  the result must be sent with the same packet_id.
+ * \param[in] domain_suffix  is appended to each domain in the queries to avoid colliding with the top-level domains
+ *                           administrated by IANA. Example: "mdns.osmocom.org"
+ * \returns allocated mslookup query on success, NULL on error.
+ */
+struct osmo_mslookup_query *osmo_mdns_query_decode(void *ctx, const uint8_t *data, size_t data_len,
+						   uint16_t *packet_id, const char *domain_suffix)
+{
+	struct osmo_mdns_msg_request *req = NULL;
+	struct osmo_mslookup_query *query = NULL;
+
+	req = osmo_mdns_msg_request_decode(ctx, data, data_len);
+	if (!req)
+		return NULL;
+
+	query = talloc_zero(ctx, struct osmo_mslookup_query);
+	OSMO_ASSERT(query);
+	if (query_from_domain(query, req->domain, domain_suffix) < 0)
+		goto error_free;
+
+	*packet_id = req->id;
+	talloc_free(req);
+	return query;
+error_free:
+	talloc_free(req);
+	talloc_free(query);
+	return NULL;
+}
+
+/*! Parse sockaddr_str from mDNS record, so the mslookup result can be filled with it.
+ * \param[out] sockaddr_str resulting IPv4 or IPv6 sockaddr_str.
+ * \param[in] rec  single record of the abstracted list of mDNS records
+ * \returns 0 on success, -EINVAL on error.
+ */
+static int sockaddr_str_from_mdns_record(struct osmo_sockaddr_str *sockaddr_str, struct osmo_mdns_record *rec)
+{
+	switch (rec->type) {
+	case OSMO_MDNS_RFC_RECORD_TYPE_A:
+		if (rec->length != 4) {
+			LOGP(DMSLOOKUP, LOGL_ERROR, "unexpected length of A record\n");
+			return -EINVAL;
+		}
+		osmo_sockaddr_str_from_32(sockaddr_str, *(uint32_t *)rec->data, 0);
+		break;
+	case OSMO_MDNS_RFC_RECORD_TYPE_AAAA:
+		if (rec->length != 16) {
+			LOGP(DMSLOOKUP, LOGL_ERROR, "unexpected length of AAAA record\n");
+			return -EINVAL;
+		}
+		osmo_sockaddr_str_from_in6_addr(sockaddr_str, (struct in6_addr*)rec->data, 0);
+		break;
+	default:
+		LOGP(DMSLOOKUP, LOGL_ERROR, "unexpected record type\n");
+		return -EINVAL;
+	}
+	return 0;
+}
+
+/*! Encode a successful mslookup result, along with the original query and packet_id into one mDNS answer packet.
+ *
+ * The records in the packet are ordered as follows:
+ * 1) "age", ip_v4/v6, "port" (only IPv4 or IPv6 present) or
+ * 2) "age", ip_v4, "port", ip_v6, "port" (both IPv4 and v6 present).
+ * "age" and "port" are TXT records, ip_v4 is an A record, ip_v6 is an AAAA record.
+ *
+ * \param[in] packet_id  as received in osmo_mdns_query_decode().
+ * \param[in] query  the original query, so we can send the domain back in the answer (i.e. "sip.voice.1234.msisdn").
+ * \param[in] result  holds the age, IPs and ports of the queried service.
+ * \param[in] domain_suffix  is appended to each domain in the queries to avoid colliding with the top-level domains
+ *                           administrated by IANA. Example: "mdns.osmocom.org"
+ * \returns msg on success, NULL on error.
+ */
+struct msgb *osmo_mdns_result_encode(void *ctx, uint16_t packet_id, const struct osmo_mslookup_query *query,
+				     const struct osmo_mslookup_result *result, const char *domain_suffix)
+{
+	struct osmo_mdns_msg_answer ans = {};
+	struct osmo_mdns_record *rec_age = NULL;
+	struct osmo_mdns_record rec_ip_v4 = {0};
+	struct osmo_mdns_record rec_ip_v6 = {0};
+	struct osmo_mdns_record *rec_ip_v4_port = NULL;
+	struct osmo_mdns_record *rec_ip_v6_port = NULL;
+	struct in_addr rec_ip_v4_in;
+	struct in6_addr rec_ip_v6_in;
+	struct msgb *msg = osmo_mdns_msgb_alloc(__func__);
+	char buf[256];
+
+	ctx = talloc_named(ctx, 0, "osmo_mdns_result_encode");
+
+	/* Prepare answer (ans) */
+	ans.domain = domain_from_query(ctx, query, domain_suffix);
+	if (!ans.domain)
+		goto error;
+	ans.id = packet_id;
+	INIT_LLIST_HEAD(&ans.records);
+
+	/* Record for age */
+	rec_age = osmo_mdns_record_txt_keyval_encode(ctx, "age", "%"PRIu32, result->age);
+	OSMO_ASSERT(rec_age);
+	llist_add_tail(&rec_age->list, &ans.records);
+
+	/* Records for IPv4 */
+	if (osmo_sockaddr_str_is_set(&result->host_v4)) {
+		if (osmo_sockaddr_str_to_in_addr(&result->host_v4, &rec_ip_v4_in) < 0) {
+			LOGP(DMSLOOKUP, LOGL_ERROR, "failed to encode ipv4: %s\n",
+			     osmo_mslookup_result_name_b(buf, sizeof(buf), query, result));
+			goto error;
+		}
+		rec_ip_v4.type = OSMO_MDNS_RFC_RECORD_TYPE_A;
+		rec_ip_v4.data = (uint8_t *)&rec_ip_v4_in;
+		rec_ip_v4.length = sizeof(rec_ip_v4_in);
+		llist_add_tail(&rec_ip_v4.list, &ans.records);
+
+		rec_ip_v4_port = osmo_mdns_record_txt_keyval_encode(ctx, "port", "%"PRIu16, result->host_v4.port);
+		OSMO_ASSERT(rec_ip_v4_port);
+		llist_add_tail(&rec_ip_v4_port->list, &ans.records);
+	}
+
+	/* Records for IPv6 */
+	if (osmo_sockaddr_str_is_set(&result->host_v6)) {
+		if (osmo_sockaddr_str_to_in6_addr(&result->host_v6, &rec_ip_v6_in) < 0) {
+			LOGP(DMSLOOKUP, LOGL_ERROR, "failed to encode ipv6: %s\n",
+			     osmo_mslookup_result_name_b(buf, sizeof(buf), query, result));
+			goto error;
+		}
+		rec_ip_v6.type = OSMO_MDNS_RFC_RECORD_TYPE_AAAA;
+		rec_ip_v6.data = (uint8_t *)&rec_ip_v6_in;
+		rec_ip_v6.length = sizeof(rec_ip_v6_in);
+		llist_add_tail(&rec_ip_v6.list, &ans.records);
+
+		rec_ip_v6_port = osmo_mdns_record_txt_keyval_encode(ctx, "port", "%"PRIu16, result->host_v6.port);
+		OSMO_ASSERT(rec_ip_v6_port);
+		llist_add_tail(&rec_ip_v6_port->list, &ans.records);
+	}
+
+	if (osmo_mdns_msg_answer_encode(ctx, msg, &ans)) {
+		LOGP(DMSLOOKUP, LOGL_ERROR, "failed to encode mDNS answer: %s\n",
+		     osmo_mslookup_result_name_b(buf, sizeof(buf), query, result));
+		goto error;
+	}
+	talloc_free(ctx);
+	return msg;
+error:
+	msgb_free(msg);
+	talloc_free(ctx);
+	return NULL;
+}
+
+static int decode_uint32_t(const char *str, uint32_t *val)
+{
+	long long int lld;
+	char *endptr = NULL;
+	*val = 0;
+	errno = 0;
+	lld = strtoll(str, &endptr, 10);
+	if (errno || !endptr || *endptr)
+		return -EINVAL;
+	if (lld < 0 || lld > UINT32_MAX)
+		return -EINVAL;
+	*val = lld;
+	return 0;
+}
+
+static int decode_port(const char *str, uint16_t *port)
+{
+	uint32_t val;
+	if (decode_uint32_t(str, &val))
+		return -EINVAL;
+	if (val > 65535)
+		return -EINVAL;
+	*port = val;
+	return 0;
+}
+
+/*! Read expected mDNS records into mslookup result.
+ *
+ * The records in the packet must be ordered as follows:
+ * 1) "age", ip_v4/v6, "port" (only IPv4 or IPv6 present) or
+ * 2) "age", ip_v4, "port", ip_v6, "port" (both IPv4 and v6 present).
+ * "age" and "port" are TXT records, ip_v4 is an A record, ip_v6 is an AAAA record.
+ *
+ * \param[out] result  holds the age, IPs and ports of the queried service.
+ * \param[in] ans  abstracted mDNS answer with a list of resource records.
+ * \returns 0 on success, -EINVAL on error.
+ */
+int osmo_mdns_result_from_answer(struct osmo_mslookup_result *result, const struct osmo_mdns_msg_answer *ans)
+{
+	struct osmo_mdns_record *rec;
+	char txt_key[64];
+	char txt_value[64];
+	bool found_age = false;
+	bool found_ip_v4 = false;
+	bool found_ip_v6 = false;
+	struct osmo_sockaddr_str *expect_port_for = NULL;
+
+	*result = (struct osmo_mslookup_result){};
+
+	result->rc = OSMO_MSLOOKUP_RC_NONE;
+
+	llist_for_each_entry(rec, &ans->records, list) {
+		switch (rec->type) {
+			case OSMO_MDNS_RFC_RECORD_TYPE_A:
+				if (expect_port_for) {
+					LOGP(DMSLOOKUP, LOGL_ERROR,
+					     "'A' record found, but still expecting a 'port' value first\n");
+					return -EINVAL;
+				}
+				if (found_ip_v4) {
+					LOGP(DMSLOOKUP, LOGL_ERROR, "'A' record found twice in mDNS answer\n");
+					return -EINVAL;
+				}
+				found_ip_v4 = true;
+				expect_port_for = &result->host_v4;
+				if (sockaddr_str_from_mdns_record(expect_port_for, rec)) {
+					LOGP(DMSLOOKUP, LOGL_ERROR, "'A' record with invalid address data\n");
+					return -EINVAL;
+				}
+				break;
+			case OSMO_MDNS_RFC_RECORD_TYPE_AAAA:
+				if (expect_port_for) {
+					LOGP(DMSLOOKUP, LOGL_ERROR,
+					     "'AAAA' record found, but still expecting a 'port' value first\n");
+					return -EINVAL;
+				}
+				if (found_ip_v6) {
+					LOGP(DMSLOOKUP, LOGL_ERROR, "'AAAA' record found twice in mDNS answer\n");
+					return -EINVAL;
+				}
+				found_ip_v6 = true;
+				expect_port_for = &result->host_v6;
+				if (sockaddr_str_from_mdns_record(expect_port_for, rec) != 0) {
+					LOGP(DMSLOOKUP, LOGL_ERROR, "'AAAA' record with invalid address data\n");
+					return -EINVAL;
+				}
+				break;
+			case OSMO_MDNS_RFC_RECORD_TYPE_TXT:
+				if (osmo_mdns_record_txt_keyval_decode(rec, txt_key, sizeof(txt_key),
+								       txt_value, sizeof(txt_value)) != 0) {
+					LOGP(DMSLOOKUP, LOGL_ERROR, "failed to decode txt record\n");
+					return -EINVAL;
+				}
+				if (strcmp(txt_key, "age") == 0) {
+					if (found_age) {
+						LOGP(DMSLOOKUP, LOGL_ERROR, "duplicate 'TXT' record for 'age'\n");
+						return -EINVAL;
+					}
+					found_age = true;
+					if (decode_uint32_t(txt_value, &result->age)) {
+						LOGP(DMSLOOKUP, LOGL_ERROR,
+						     "'TXT' record: invalid 'age' value ('age=%s')\n", txt_value);
+						return -EINVAL;
+					}
+				} else if (strcmp(txt_key, "port") == 0) {
+					if (!expect_port_for) {
+						LOGP(DMSLOOKUP, LOGL_ERROR,
+						     "'TXT' record for 'port' without previous 'A' or 'AAAA' record\n");
+						return -EINVAL;
+					}
+					if (decode_port(txt_value, &expect_port_for->port)) {
+						LOGP(DMSLOOKUP, LOGL_ERROR,
+						     "'TXT' record: invalid 'port' value ('port=%s')\n", txt_value);
+						return -EINVAL;
+					}
+					expect_port_for = NULL;
+				} else {
+					LOGP(DMSLOOKUP, LOGL_ERROR, "unexpected key '%s' in TXT record\n", txt_key);
+					return -EINVAL;
+				}
+				break;
+			default:
+				LOGP(DMSLOOKUP, LOGL_ERROR, "unexpected record type\n");
+				return -EINVAL;
+		}
+	}
+
+	/* Check if everything was found */
+	if (!found_age || !(found_ip_v4 || found_ip_v6) || expect_port_for) {
+		LOGP(DMSLOOKUP, LOGL_ERROR, "missing resource records in mDNS answer\n");
+		return -EINVAL;
+	}
+
+	result->rc = OSMO_MSLOOKUP_RC_RESULT;
+	return 0;
+}
+
+/*! Decode a mDNS answer packet into a mslookup result, query and packet_id.
+ * \param[out] packet_id  same ID as sent in the request packet.
+ * \param[out] query  the original query (service, ID, ID type).
+ * \param[out] result  holds the age, IPs and ports of the queried service.
+ * \param[in] domain_suffix  is appended to each domain in the queries to avoid colliding with the top-level domains
+ *                           administrated by IANA. Example: "mdns.osmocom.org"
+ * \returns 0 on success, -EINVAL on error.
+ */
+int osmo_mdns_result_decode(void *ctx, const uint8_t *data, size_t data_len, uint16_t *packet_id,
+			    struct osmo_mslookup_query *query, struct osmo_mslookup_result *result,
+			    const char *domain_suffix)
+{
+	int rc = -EINVAL;
+	struct osmo_mdns_msg_answer *ans;
+	ans = osmo_mdns_msg_answer_decode(ctx, data, data_len);
+	if (!ans)
+		goto exit_free;
+
+	if (query_from_domain(query, ans->domain, domain_suffix) < 0)
+		goto exit_free;
+
+	if (osmo_mdns_result_from_answer(result, ans) < 0)
+		goto exit_free;
+
+	*packet_id = ans->id;
+	rc = 0;
+
+exit_free:
+	talloc_free(ans);
+	return rc;
+}

src/mslookup/mdns_msg.c

@@ -0,0 +1,261 @@
+/* High level mDNS encoding and decoding functions for whole messages:
+ * Request message (header, question)
+ * Answer message (header, resource record 1, ... resource record N)*/
+
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <errno.h>
+#include <string.h>
+#include <osmocom/hlr/logging.h>
+#include <osmocom/mslookup/mdns_msg.h>
+
+/*! Encode request message into one mDNS packet, consisting of the header section and one question section.
+ * \returns 0 on success, -EINVAL on error.
+ */
+int osmo_mdns_msg_request_encode(void *ctx, struct msgb *msg, const struct osmo_mdns_msg_request *req)
+{
+	struct osmo_mdns_rfc_header hdr = {0};
+	struct osmo_mdns_rfc_question qst = {0};
+
+	hdr.id = req->id;
+	hdr.qdcount = 1;
+	osmo_mdns_rfc_header_encode(msg, &hdr);
+
+	qst.domain = req->domain;
+	qst.qtype = req->type;
+	qst.qclass = OSMO_MDNS_RFC_CLASS_IN;
+	if (osmo_mdns_rfc_question_encode(ctx, msg, &qst) != 0)
+		return -EINVAL;
+
+	return 0;
+}
+
+/*! Decode request message from a mDNS packet, consisting of the header section and one question section.
+ * \returns allocated request message on success, NULL on error.
+ */
+struct osmo_mdns_msg_request *osmo_mdns_msg_request_decode(void *ctx, const uint8_t *data, size_t data_len)
+{
+	struct osmo_mdns_rfc_header hdr = {0};
+	size_t hdr_len = sizeof(struct osmo_mdns_rfc_header);
+	struct osmo_mdns_rfc_question* qst = NULL;
+	struct osmo_mdns_msg_request *ret = NULL;
+
+	if (data_len < hdr_len || osmo_mdns_rfc_header_decode(data, hdr_len, &hdr) != 0 || hdr.qr != 0)
+		return NULL;
+
+	qst = osmo_mdns_rfc_question_decode(ctx, data + hdr_len, data_len - hdr_len);
+	if (!qst)
+		return NULL;
+
+	ret = talloc_zero(ctx, struct osmo_mdns_msg_request);
+	ret->id = hdr.id;
+	ret->domain = talloc_strdup(ret, qst->domain);
+	ret->type = qst->qtype;
+
+	talloc_free(qst);
+	return ret;
+}
+
+/*! Initialize the linked list for resource records in a answer message. */
+void osmo_mdns_msg_answer_init(struct osmo_mdns_msg_answer *ans)
+{
+	*ans = (struct osmo_mdns_msg_answer){};
+	INIT_LLIST_HEAD(&ans->records);
+}
+
+/*! Encode answer message into one mDNS packet, consisting of the header section and N resource records.
+ *
+ * To keep things simple, this sends the domain with each resource record. Other DNS implementations make use of
+ * "message compression", which would send a question section with the domain before the resource records, and then
+ * point inside each resource record with an offset back to the domain in the question section (RFC 1035 4.1.4).
+ * \returns 0 on success, -EINVAL on error.
+ */
+int osmo_mdns_msg_answer_encode(void *ctx, struct msgb *msg, const struct osmo_mdns_msg_answer *ans)
+{
+	struct osmo_mdns_rfc_header hdr = {0};
+	struct osmo_mdns_record *ans_record;
+
+	hdr.id = ans->id;
+	hdr.qr = 1;
+	hdr.ancount = llist_count(&ans->records);
+	osmo_mdns_rfc_header_encode(msg, &hdr);
+
+	llist_for_each_entry(ans_record, &ans->records, list) {
+		struct osmo_mdns_rfc_record rec = {0};
+
+		rec.domain = ans->domain;
+		rec.type = ans_record->type;
+		rec.class = OSMO_MDNS_RFC_CLASS_IN;
+		rec.ttl = 0;
+		rec.rdlength = ans_record->length;
+		rec.rdata = ans_record->data;
+
+		if (osmo_mdns_rfc_record_encode(ctx, msg, &rec) != 0)
+			return -EINVAL;
+	}
+
+	return 0;
+}
+
+/*! Decode answer message from a mDNS packet.
+ *
+ * Answer messages must consist of one header and one or more resource records. An additional question section or
+ * message compression (RFC 1035 4.1.4) are not supported.
+* \returns allocated answer message on success, NULL on error.
+ */
+struct osmo_mdns_msg_answer *osmo_mdns_msg_answer_decode(void *ctx, const uint8_t *data, size_t data_len)
+{
+	struct osmo_mdns_rfc_header hdr = {};
+	size_t hdr_len = sizeof(struct osmo_mdns_rfc_header);
+	struct osmo_mdns_msg_answer *ret = talloc_zero(ctx, struct osmo_mdns_msg_answer);
+
+	/* Parse header section */
+	if (data_len < hdr_len || osmo_mdns_rfc_header_decode(data, hdr_len, &hdr) != 0 || hdr.qr != 1)
+		goto error;
+	ret->id = hdr.id;
+	data_len -= hdr_len;
+	data += hdr_len;
+
+	/* Parse resource records */
+	INIT_LLIST_HEAD(&ret->records);
+	while (data_len) {
+		size_t record_len;
+		struct osmo_mdns_rfc_record *rec;
+		struct osmo_mdns_record* ret_record;
+
+		rec = osmo_mdns_rfc_record_decode(ret, data, data_len, &record_len);
+		if (!rec)
+			goto error;
+
+		/* Copy domain to ret */
+		if (ret->domain) {
+			if (strcmp(ret->domain, rec->domain) != 0) {
+				LOGP(DMSLOOKUP, LOGL_ERROR, "domain mismatch in resource records ('%s' vs '%s')\n",
+				     ret->domain, rec->domain);
+				goto error;
+			}
+		}
+		else
+			ret->domain = talloc_strdup(ret, rec->domain);
+
+		/* Add simplified record to ret */
+		ret_record = talloc_zero(ret, struct osmo_mdns_record);
+		ret_record->type = rec->type;
+		ret_record->length = rec->rdlength;
+		ret_record->data = talloc_memdup(ret_record, rec->rdata, rec->rdlength);
+		llist_add_tail(&ret_record->list, &ret->records);
+
+		data += record_len;
+		data_len -= record_len;
+		talloc_free(rec);
+	}
+
+	/* Verify record count */
+	if (llist_count(&ret->records) != hdr.ancount) {
+		LOGP(DMSLOOKUP, LOGL_ERROR, "amount of parsed records (%i) doesn't match count in header (%i)\n",
+		     llist_count(&ret->records), hdr.ancount);
+		goto error;
+	}
+
+	return ret;
+error:
+	talloc_free(ret);
+	return NULL;
+}
+
+/*! Get a TXT resource record, which stores a key=value string.
+ * \returns allocated resource record on success, NULL on error.
+ */
+static struct osmo_mdns_record *_osmo_mdns_record_txt_encode(void *ctx, const char *key, const char *value)
+{
+	struct osmo_mdns_record *ret = talloc_zero(ctx, struct osmo_mdns_record);
+	size_t len = strlen(key) + 1 + strlen(value);
+
+	if (len > OSMO_MDNS_RFC_MAX_CHARACTER_STRING_LEN - 1)
+		return NULL;
+
+	/* redundant len is required, see RFC 1035 3.3.14 and 3.3. */
+	ret->data = (uint8_t *)talloc_asprintf(ctx, "%c%s=%s", (char)len, key, value);
+	if (!ret->data)
+		return NULL;
+	ret->type = OSMO_MDNS_RFC_RECORD_TYPE_TXT;
+	ret->length = len + 1;
+	return ret;
+}
+
+/*! Get a TXT resource record, which stores a key=value string, but build value from a format string.
+ * \returns allocated resource record on success, NULL on error.
+ */
+struct osmo_mdns_record *osmo_mdns_record_txt_keyval_encode(void *ctx, const char *key, const char *value_fmt, ...)
+{
+	va_list ap;
+	char *value = NULL;
+	struct osmo_mdns_record *r;
+
+	if (!value_fmt)
+		return _osmo_mdns_record_txt_encode(ctx, key, "");
+
+	va_start(ap, value_fmt);
+	value = talloc_vasprintf(ctx, value_fmt, ap);
+	if (!value)
+		return NULL;
+	va_end(ap);
+	r = _osmo_mdns_record_txt_encode(ctx, key, value);
+	talloc_free(value);
+	return r;
+}
+
+/*! Decode a TXT resource record, which stores a key=value string.
+ * \returns 0 on success, -EINVAL on error.
+ */
+int osmo_mdns_record_txt_keyval_decode(const struct osmo_mdns_record *rec,
+				       char *key_buf, size_t key_size, char *value_buf, size_t value_size)
+{
+	const char *key_value;
+	const char *key_value_end;
+	const char *sep;
+	const char *value;
+
+	if (rec->type != OSMO_MDNS_RFC_RECORD_TYPE_TXT)
+		return -EINVAL;
+
+	key_value = (const char *)rec->data;
+	key_value_end = key_value + rec->length;
+
+	/* Verify and then skip the redundant string length byte */
+	if (*key_value != rec->length - 1)
+		return -EINVAL;
+	key_value++;
+
+	if (key_value >= key_value_end)
+		return -EINVAL;
+
+	/* Find equals sign */
+	sep = osmo_strnchr(key_value, key_value_end - key_value, '=');
+	if (!sep)
+		return -EINVAL;
+
+	/* Parse key */
+	osmo_print_n(key_buf, key_size, key_value, sep - key_value);
+
+	/* Parse value */
+	value = sep + 1;
+	osmo_print_n(value_buf, value_size, value, key_value_end - value);
+	return 0;
+}

src/mslookup/mdns_rfc.c

@@ -0,0 +1,265 @@
+/* Low level mDNS encoding and decoding functions of the qname IE, header/question sections and resource records,
+ * as described in these RFCs:
+ * - RFC 1035 (Domain names - implementation and specification)
+ * - RFC 3596 (DNS Extensions to Support IP Version 6) */
+
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <string.h>
+#include <ctype.h>
+#include <errno.h>
+#include <osmocom/core/msgb.h>
+#include <osmocom/core/bitvec.h>
+#include <osmocom/core/logging.h>
+#include <osmocom/mslookup/mdns_rfc.h>
+
+/*
+ * Encode/decode IEs
+ */
+
+/*! Encode a domain string as qname (RFC 1035 4.1.2).
+ * \param[in] domain  multiple labels separated by dots, e.g. "sip.voice.1234.msisdn".
+ * \returns allocated buffer with length-value pairs for each label (e.g. 0x03 "sip" 0x05 "voice" ...), NULL on error.
+ */
+char *osmo_mdns_rfc_qname_encode(void *ctx, const char *domain)
+{
+	char *domain_dup;
+	char *domain_iter;
+	char buf[OSMO_MDNS_RFC_MAX_NAME_LEN + 2] = ""; /* len(qname) is len(domain) +1 */
+	struct osmo_strbuf sb = { .buf = buf, .len = sizeof(buf) };
+	char *label;
+
+	if (strlen(domain) > OSMO_MDNS_RFC_MAX_NAME_LEN)
+		return NULL;
+
+	domain_iter = domain_dup = talloc_strdup(ctx, domain);
+	while ((label = strsep(&domain_iter, "."))) {
+		size_t len = strlen(label);
+
+		/* Empty domain, dot at start, two dots in a row, or ending with a dot */
+		if (!len)
+			goto error;
+
+		OSMO_STRBUF_PRINTF(sb, "%c%s", (char)len, label);
+	}
+
+	talloc_free(domain_dup);
+	return talloc_strdup(ctx, buf);
+
+error:
+	talloc_free(domain_dup);
+	return NULL;
+}
+
+/*! Decode a domain string from a qname (RFC 1035 4.1.2).
+ * \param[in] qname  buffer with length-value pairs for each label (e.g. 0x03 "sip" 0x05 "voice" ...)
+ * \param[in] qname_max_len  amount of bytes that can be read at most from the memory location that qname points to.
+ * \returns allocated buffer with domain string, multiple labels separated by dots (e.g. "sip.voice.1234.msisdn"),
+ *	    NULL on error.
+ */
+char *osmo_mdns_rfc_qname_decode(void *ctx, const char *qname, size_t qname_max_len)
+{
+	const char *next_label, *qname_end = qname + qname_max_len;
+	char buf[OSMO_MDNS_RFC_MAX_NAME_LEN + 1];
+	int i = 0;
+
+	if (qname_max_len < 1)
+		return NULL;
+
+	while (*qname) {
+		size_t len = *qname;
+		next_label = qname + len + 1;
+
+		if (next_label >= qname_end || i + len > OSMO_MDNS_RFC_MAX_NAME_LEN)
+			return NULL;
+
+		if (i) {
+			/* Two dots in a row is not allowed */
+			if (buf[i - 1] == '.')
+				return NULL;
+
+			buf[i] = '.';
+			i++;
+		}
+
+		memcpy(buf + i, qname + 1, len);
+		i += len;
+		qname = next_label;
+	}
+	buf[i] = '\0';
+
+	return talloc_strdup(ctx, buf);
+}
+
+/*
+ * Encode/decode message sections
+ */
+
+/*! Encode header section (RFC 1035 4.1.1).
+ * \param[in] msgb  mesage buffer to which the encoded data will be appended.
+ */
+void osmo_mdns_rfc_header_encode(struct msgb *msg, const struct osmo_mdns_rfc_header *hdr)
+{
+	struct osmo_mdns_rfc_header *buf = (struct osmo_mdns_rfc_header *) msgb_put(msg, sizeof(*hdr));
+	memcpy(buf, hdr, sizeof(*hdr));
+
+	osmo_store16be(buf->id, &buf->id);
+	osmo_store16be(buf->qdcount, &buf->qdcount);
+	osmo_store16be(buf->ancount, &buf->ancount);
+	osmo_store16be(buf->nscount, &buf->nscount);
+	osmo_store16be(buf->arcount, &buf->arcount);
+}
+
+/*! Decode header section (RFC 1035 4.1.1). */
+int osmo_mdns_rfc_header_decode(const uint8_t *data, size_t data_len, struct osmo_mdns_rfc_header *hdr)
+{
+	if (data_len != sizeof(*hdr))
+		return -EINVAL;
+
+	memcpy(hdr, data, data_len);
+
+	hdr->id = osmo_load16be(&hdr->id);
+	hdr->qdcount = osmo_load16be(&hdr->qdcount);
+	hdr->ancount = osmo_load16be(&hdr->ancount);
+	hdr->nscount = osmo_load16be(&hdr->nscount);
+	hdr->arcount = osmo_load16be(&hdr->arcount);
+
+	return 0;
+}
+
+/*! Encode question section (RFC 1035 4.1.2).
+ * \param[in] msgb  mesage buffer to which the encoded data will be appended.
+ */
+int osmo_mdns_rfc_question_encode(void *ctx, struct msgb *msg, const struct osmo_mdns_rfc_question *qst)
+{
+	char *qname;
+	size_t qname_len;
+	uint8_t *qname_buf;
+
+	/* qname */
+	qname = osmo_mdns_rfc_qname_encode(ctx, qst->domain);
+	if (!qname)
+		return -EINVAL;
+	qname_len = strlen(qname) + 1;
+	qname_buf = msgb_put(msg, qname_len);
+	memcpy(qname_buf, qname, qname_len);
+	talloc_free(qname);
+
+	/* qtype and qclass */
+	msgb_put_u16(msg, qst->qtype);
+	msgb_put_u16(msg, qst->qclass);
+
+	return 0;
+}
+
+/*! Decode question section (RFC 1035 4.1.2). */
+struct osmo_mdns_rfc_question *osmo_mdns_rfc_question_decode(void *ctx, const uint8_t *data, size_t data_len)
+{
+	struct osmo_mdns_rfc_question *ret;
+	size_t qname_len = data_len - 4;
+
+	if (data_len < 6)
+		return NULL;
+
+	/* qname */
+	ret = talloc_zero(ctx, struct osmo_mdns_rfc_question);
+	if (!ret)
+		return NULL;
+	ret->domain = osmo_mdns_rfc_qname_decode(ret, (const char *)data, qname_len);
+	if (!ret->domain) {
+		talloc_free(ret);
+		return NULL;
+	}
+
+	/* qtype and qclass */
+	ret->qtype = osmo_load16be(data + qname_len);
+	ret->qclass = osmo_load16be(data + qname_len + 2);
+
+	return ret;
+}
+
+/*
+ * Encode/decode resource records
+ */
+
+/*! Encode one resource record (RFC 1035 4.1.3).
+ * \param[in] msgb  mesage buffer to which the encoded data will be appended.
+ */
+int osmo_mdns_rfc_record_encode(void *ctx, struct msgb *msg, const struct osmo_mdns_rfc_record *rec)
+{
+	char *name;
+	size_t name_len;
+	uint8_t *buf;
+
+	/* name */
+	name = osmo_mdns_rfc_qname_encode(ctx, rec->domain);
+	if (!name)
+		return -EINVAL;
+	name_len = strlen(name) + 1;
+	buf = msgb_put(msg, name_len);
+	memcpy(buf, name, name_len);
+	talloc_free(name);
+
+	/* type, class, ttl, rdlength */
+	msgb_put_u16(msg, rec->type);
+	msgb_put_u16(msg, rec->class);
+	msgb_put_u32(msg, rec->ttl);
+	msgb_put_u16(msg, rec->rdlength);
+
+	/* rdata */
+	buf = msgb_put(msg, rec->rdlength);
+	memcpy(buf, rec->rdata, rec->rdlength);
+	return 0;
+}
+
+/*! Decode one resource record (RFC 1035 4.1.3). */
+struct osmo_mdns_rfc_record *osmo_mdns_rfc_record_decode(void *ctx, const uint8_t *data, size_t data_len,
+						       size_t *record_len)
+{
+	struct osmo_mdns_rfc_record *ret = talloc_zero(ctx, struct osmo_mdns_rfc_record);
+	size_t name_len;
+
+	/* name */
+	ret->domain = osmo_mdns_rfc_qname_decode(ret, (const char *)data, data_len - 10);
+	if (!ret->domain)
+		goto error;
+	name_len = strlen(ret->domain) + 2;
+	if (name_len + 10 > data_len)
+		goto error;
+
+	/* type, class, ttl, rdlength */
+	ret->type = osmo_load16be(data + name_len);
+	ret->class = osmo_load16be(data + name_len + 2);
+	ret->ttl = osmo_load32be(data + name_len + 4);
+	ret->rdlength = osmo_load16be(data + name_len + 8);
+	if (name_len + 10 + ret->rdlength > data_len)
+		goto error;
+
+	/* rdata */
+	ret->rdata = talloc_memdup(ret, data + name_len + 10, ret->rdlength);
+	if (!ret->rdata)
+		return NULL;
+
+	*record_len = name_len + 10 + ret->rdlength;
+	return ret;
+error:
+	talloc_free(ret);
+	return NULL;
+}
+

src/mslookup/mdns_sock.c

@@ -0,0 +1,144 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <netdb.h>
+#include <stdbool.h>
+#include <talloc.h>
+#include <osmocom/core/utils.h>
+#include <osmocom/core/select.h>
+#include <osmocom/hlr/logging.h>
+#include <osmocom/mslookup/mdns_sock.h>
+
+/*! Open socket to send and receive multicast data.
+ *
+ * The socket is opened with SO_REUSEADDR, so we can bind to the same IP and port multiple times. This socket receives
+ * everything sent to that multicast IP/port, including its own data data sent from osmo_mdns_sock_send(). So whenever
+ * sending something, the receive callback will be called with the same data and should discard it.
+ *
+ * \param[in] ip  multicast IPv4 or IPv6 address.
+ * \param[in] port  port number.
+ * \param[in] cb  callback for incoming data that will be passed to osmo_fd_setup (should read from osmo_fd->fd).
+ * \param[in] data  userdata passed to osmo_fd (available in cb as osmo_fd->data).
+ * \param[in] priv_nr  additional userdata integer passed to osmo_fd (available in cb as osmo_fd->priv_nr).
+ * \returns allocated osmo_mdns_sock, NULL on error.
+ */
+struct osmo_mdns_sock *osmo_mdns_sock_init(void *ctx, const char *ip, unsigned int port,
+					   int (*cb)(struct osmo_fd *fd, unsigned int what),
+					   void *data, unsigned int priv_nr)
+{
+	struct osmo_mdns_sock *ret;
+	int sock, rc;
+	struct addrinfo hints = {0};
+	struct ip_mreq multicast_req = {0};
+	in_addr_t iface = INADDR_ANY;
+	char portbuf[10];
+	int y = 1;
+
+	snprintf(portbuf, sizeof(portbuf) -1, "%u", port);
+	ret = talloc_zero(ctx, struct osmo_mdns_sock);
+	OSMO_ASSERT(ret);
+
+	/* Fill addrinfo */
+	hints.ai_family = PF_UNSPEC;
+	hints.ai_socktype = SOCK_DGRAM;
+	hints.ai_flags = (AI_PASSIVE | AI_NUMERICHOST);
+	rc = getaddrinfo(ip, portbuf, &hints, &ret->ai);
+	if (rc != 0) {
+		LOGP(DMSLOOKUP, LOGL_ERROR, "osmo_mdns_sock_init: getaddrinfo: %s\n", gai_strerror(rc));
+		ret->ai = NULL;
+		goto error;
+	}
+
+	/* Open socket */
+	sock = socket(ret->ai->ai_family, ret->ai->ai_socktype, 0);
+	if (sock == -1) {
+		LOGP(DMSLOOKUP, LOGL_ERROR, "osmo_mdns_sock_init: socket: %s\n", strerror(errno));
+		goto error;
+	}
+
+	/* Set multicast options */
+	rc = setsockopt(sock, IPPROTO_IP, IP_MULTICAST_IF, (char*)&iface, sizeof(iface));
+	if (rc == -1) {
+		LOGP(DMSLOOKUP, LOGL_ERROR, "osmo_mdns_sock_init: setsockopt: %s\n", strerror(errno));
+		goto error;
+	}
+	memcpy(&multicast_req.imr_multiaddr, &((struct sockaddr_in*)(ret->ai->ai_addr))->sin_addr,
+	       sizeof(multicast_req.imr_multiaddr));
+	multicast_req.imr_interface.s_addr = htonl(INADDR_ANY);
+	rc = setsockopt(sock, IPPROTO_IP, IP_ADD_MEMBERSHIP, (char*)&multicast_req, sizeof(multicast_req));
+	if (rc == -1) {
+		LOGP(DMSLOOKUP, LOGL_ERROR, "osmo_mdns_sock_init: setsockopt: %s\n", strerror(errno));
+		goto error;
+	}
+
+	/* Always allow binding the same IP and port twice. This is needed in OsmoHLR (where the code becomes cleaner by
+	 * just using a different socket for server and client code) and in the mslookup_client_mdns_test. Also for
+	 * osmo-mslookup-client if it is running multiple times in parallel (i.e. two incoming calls almost at the same
+	 * time need to be resolved with the simple dialplan example that just starts new processes). */
+	rc = setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char *)&y, sizeof(y));
+	if (rc == -1) {
+		LOGP(DMSLOOKUP, LOGL_ERROR, "osmo_mdns_sock_init: setsockopt: %s\n", strerror(errno));
+		goto error;
+	}
+
+	/* Bind and register osmo_fd callback */
+	rc = bind(sock, ret->ai->ai_addr, ret->ai->ai_addrlen);
+	if (rc == -1) {
+		LOGP(DMSLOOKUP, LOGL_ERROR, "osmo_mdns_sock_init: bind: %s\n", strerror(errno));
+		goto error;
+	}
+	osmo_fd_setup(&ret->osmo_fd, sock, OSMO_FD_READ, cb, data, priv_nr);
+	if (osmo_fd_register(&ret->osmo_fd) != 0)
+		goto error;
+
+	return ret;
+error:
+	if (ret->ai)
+		freeaddrinfo(ret->ai);
+	talloc_free(ret);
+	return NULL;
+}
+
+/*! Send msgb over mdns_sock and consume msgb.
+ * \returns 0 on success, -1 on error.
+ */
+int osmo_mdns_sock_send(const struct osmo_mdns_sock *mdns_sock, struct msgb *msg)
+{
+	size_t len = msgb_length(msg);
+	int rc = sendto(mdns_sock->osmo_fd.fd, msgb_data(msg), len, 0, mdns_sock->ai->ai_addr,
+			mdns_sock->ai->ai_addrlen);
+	msgb_free(msg);
+	return (rc == len) ? 0 : -1;
+}
+
+/*! Tear down osmo_mdns_sock. */
+void osmo_mdns_sock_cleanup(struct osmo_mdns_sock *mdns_sock)
+{
+	osmo_fd_close(&mdns_sock->osmo_fd);
+	freeaddrinfo(mdns_sock->ai);
+	talloc_free(mdns_sock);
+}

src/mslookup/mslookup.c

@@ -0,0 +1,321 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <string.h>
+#include <errno.h>
+#include <osmocom/gsm/gsm23003.h>
+#include <osmocom/mslookup/mslookup.h>
+
+/*! \addtogroup mslookup
+ *
+ * Distributed GSM: finding subscribers
+ *
+ * There are various aspects of the D-GSM code base in osmo-hlr.git, here is an overview:
+ *
+ * mslookup is the main enabler of D-GSM, a concept for connecting services between independent core network stacks.
+ *
+ * D-GSM consists of:
+ * (1) mslookup client to find subscribers:
+ *     (a) external clients like ESME, SIP PBX, ... ask osmo-hlr to tell where to send SMS, voice calls, ...
+ *     (b) osmo-hlr's own mslookup client asks remote osmo-hlrs whether they know a given IMSI.
+ * (2) when a subscriber was found at a remote HLR, GSUP gets forwarded there:
+ *     (a) to deliver messages for the GSUP proxy, osmo-hlr manages many GSUP clients to establish links to remote HLRs.
+ *     (b) osmo-hlr has a GSUP proxy layer that caches data of IMSIs that get proxied to a remote HLR.
+ *     (c) decision making to distinguish local IMSIs from ones proxied to a remote HLR.
+ *
+ * (1) mslookup is a method of finding subscribers using (multicast) queries, by MSISDN or by IMSI.
+ * It is open to various lookup methods, the first one being multicast DNS.
+ * An mslookup client sends a request, and an mslookup server responds.
+ * The mslookup server is implemented by osmo-hlr. mslookup clients are arbitrary programs, like an ESME or a SIP PBX.
+ * Hence the mslookup client is public API, while the mslookup server is implemented "privately" in osmo-hlr.
+ *
+ * (1a) Public mslookup client: libosmo-mslookup
+ *   src/mslookup/mslookup.c               Things useful for both client and server.
+ *   src/mslookup/mslookup_client.c        The client API, which can use various lookup methods,
+ *                                         and consolidates results from various responders.
+ *   src/mslookup/mslookup_client_mdns.c   lookup method implementing multicast DNS, client side.
+ *
+ *   src/mslookup/osmo-mslookup-client.c   Utility program to ease invocation for (blocking) mslookup clients.
+ *
+ *   src/mslookup/mslookup_client_fake.c   lookup method generating fake results, for testing client implementations.
+ *
+ *   src/mslookup/mdns*.c                  implementation of DNS to be used by mslookup_client_mdns.c,
+ *                                         and the mslookup_server.c.
+ *
+ *   contrib/dgsm/esme_dgsm.py                 Example implementation for an mslookup enabled SMS handler.
+ *   contrib/dgsm/freeswitch_dialplan_dgsm.py  Example implementation for an mslookup enabled FreeSWITCH dialplan.
+ *   contrib/dgsm/osmo-mslookup-pipe.py        Example for writing a python client using the osmo-mslookup-client
+ *                                             cmdline.
+ *   contrib/dgsm/osmo-mslookup-socket.py      Example for writing a python client using the osmo-mslookup-client
+ *                                             unix domain socket.
+ *
+ * (1b) "Private" mslookup server in osmo-hlr:
+ *   src/mslookup_server.c        Respond to mslookup queries, independent from the particular lookup method.
+ *   src/mslookup_server_mdns.c   mDNS specific implementation for mslookup_server.c.
+ *   src/dgsm_vty.c               Configure services that mslookup server sends to remote requests.
+ *
+ * (2) Proxy and GSUP clients to remote HLR instances:
+ *
+ * (a) Be a GSUP client to forward to a remote HLR:
+ *  src/gsupclient/   The same API that is used by osmo-{msc,sgsn} is also used to forward GSUP to remote osmo-hlrs.
+ *  src/remote_hlr.c  Establish links to remote osmo-hlrs, where this osmo-hlr is a client (proxying e.g. for an MSC).
+ *
+ * (b) Keep track of remotely handled IMSIs:
+ *  src/proxy.c       Keep track of proxied IMSIs and cache important subscriber data.
+ *
+ * (c) Direct GSUP request to the right destination: either the local or a remote HLR:
+ *  src/dgsm.c        The glue that makes osmo-hlr distinguish between local IMSIs and those that are proxied to another
+ *                    osmo-hlr.
+ *  src/dgsm_vty.c    Config.
+ *
+ *  @{
+ * \file mslookup.c
+ */
+
+const struct value_string osmo_mslookup_id_type_names[] = {
+	{ OSMO_MSLOOKUP_ID_NONE, "none" },
+	{ OSMO_MSLOOKUP_ID_IMSI, "imsi" },
+	{ OSMO_MSLOOKUP_ID_MSISDN, "msisdn" },
+	{}
+};
+
+const struct value_string osmo_mslookup_result_code_names[] = {
+	{ OSMO_MSLOOKUP_RC_NONE, "none" },
+	{ OSMO_MSLOOKUP_RC_RESULT, "result" },
+	{ OSMO_MSLOOKUP_RC_NOT_FOUND, "not-found" },
+	{}
+};
+
+/*! Compare two struct osmo_mslookup_id.
+ * \returns   0 if a and b are equal,
+ *          < 0 if a (or the ID type / start of ID) is < b,
+ *          > 0 if a (or the ID type / start of ID) is > b.
+ */
+int osmo_mslookup_id_cmp(const struct osmo_mslookup_id *a, const struct osmo_mslookup_id *b)
+{
+	int cmp;
+	if (a == b)
+		return 0;
+	if (!a)
+		return -1;
+	if (!b)
+		return 1;
+
+	cmp = OSMO_CMP(a->type, b->type);
+	if (cmp)
+		return cmp;
+
+	switch (a->type) {
+	case OSMO_MSLOOKUP_ID_IMSI:
+		return strncmp(a->imsi, b->imsi, sizeof(a->imsi));
+	case OSMO_MSLOOKUP_ID_MSISDN:
+		return strncmp(a->msisdn, b->msisdn, sizeof(a->msisdn));
+	default:
+		return 0;
+	}
+}
+
+bool osmo_mslookup_id_valid(const struct osmo_mslookup_id *id)
+{
+	switch (id->type) {
+	case OSMO_MSLOOKUP_ID_IMSI:
+		return osmo_imsi_str_valid(id->imsi);
+	case OSMO_MSLOOKUP_ID_MSISDN:
+		return osmo_msisdn_str_valid(id->msisdn);
+	default:
+		return false;
+	}
+}
+
+bool osmo_mslookup_service_valid(const char *service)
+{
+	return strlen(service) > 0;
+}
+
+/*! Write ID and ID type to a buffer.
+ * \param[out] buf  nul-terminated {id}.{id_type} string (e.g. "1234.msisdn") or
+* 		    "?.none" if the ID type is invalid.
+ * \returns amount of bytes written to buf.
+ */
+size_t osmo_mslookup_id_name_buf(char *buf, size_t buflen, const struct osmo_mslookup_id *id)
+{
+	struct osmo_strbuf sb = { .buf = buf, .len = buflen };
+	switch (id->type) {
+	case OSMO_MSLOOKUP_ID_IMSI:
+		OSMO_STRBUF_PRINTF(sb, "%s", id->imsi);
+		break;
+	case OSMO_MSLOOKUP_ID_MSISDN:
+		OSMO_STRBUF_PRINTF(sb, "%s", id->msisdn);
+		break;
+	default:
+		OSMO_STRBUF_PRINTF(sb, "?");
+		break;
+	}
+	OSMO_STRBUF_PRINTF(sb, ".%s", osmo_mslookup_id_type_name(id->type));
+	return sb.chars_needed;
+}
+
+/*! Same as osmo_mslookup_id_name_buf(), but return a talloc allocated string of sufficient size. */
+char *osmo_mslookup_id_name_c(void *ctx, const struct osmo_mslookup_id *id)
+{
+	OSMO_NAME_C_IMPL(ctx, 64, "ERROR", osmo_mslookup_id_name_buf, id)
+}
+
+/*! Same as osmo_mslookup_id_name_buf(), but directly return the char* (for printf-like string formats). */
+char *osmo_mslookup_id_name_b(char *buf, size_t buflen, const struct osmo_mslookup_id *id)
+{
+	int rc = osmo_mslookup_id_name_buf(buf, buflen, id);
+	if (rc < 0 && buflen)
+		buf[0] = '\0';
+	return buf;
+}
+
+/*! Write mslookup result string to buffer.
+ * \param[in] query  with the service, ID and ID type to be written to buf like a domain string, or NULL to omit.
+ * \param[in] result with the result code, IPv4/v6 and age to be written to buf or NULL to omit.
+ * \param[out] buf  result as flat string, which looks like the following for a valid query and result with IPv4 and v6
+ *                  answer: "sip.voice.1234.msisdn -> ipv4: 42.42.42.42:1337 -> ipv6: [1234:5678:9ABC::]:1338 (age=1)",
+ *                  the result part can also be " -> timeout" or " -> rc=5" depending on the result code.
+ * \returns amount of bytes written to buf.
+ */
+size_t osmo_mslookup_result_to_str_buf(char *buf, size_t buflen,
+				     const struct osmo_mslookup_query *query,
+				     const struct osmo_mslookup_result *result)
+{
+	struct osmo_strbuf sb = { .buf = buf, .len = buflen };
+	if (query) {
+		OSMO_STRBUF_PRINTF(sb, "%s.", query->service);
+		OSMO_STRBUF_APPEND(sb, osmo_mslookup_id_name_buf, &query->id);
+	}
+	if (result && result->rc == OSMO_MSLOOKUP_RC_NONE)
+		result = NULL;
+	if (result) {
+		if (result->rc != OSMO_MSLOOKUP_RC_RESULT) {
+			OSMO_STRBUF_PRINTF(sb, " %s", osmo_mslookup_result_code_name(result->rc));
+		} else {
+			if (result->host_v4.ip[0]) {
+				OSMO_STRBUF_PRINTF(sb, " -> ipv4: " OSMO_SOCKADDR_STR_FMT,
+						   OSMO_SOCKADDR_STR_FMT_ARGS(&result->host_v4));
+			}
+			if (result->host_v6.ip[0]) {
+				OSMO_STRBUF_PRINTF(sb, " -> ipv6: " OSMO_SOCKADDR_STR_FMT,
+						   OSMO_SOCKADDR_STR_FMT_ARGS(&result->host_v6));
+			}
+			OSMO_STRBUF_PRINTF(sb, " (age=%u)", result->age);
+		}
+		OSMO_STRBUF_PRINTF(sb, " %s", result->last ? "(last)" : "(not-last)");
+	}
+	return sb.chars_needed;
+}
+
+/*! Same as osmo_mslookup_result_to_str_buf(), but return a talloc allocated string of sufficient size. */
+char *osmo_mslookup_result_name_c(void *ctx,
+				  const struct osmo_mslookup_query *query,
+				  const struct osmo_mslookup_result *result)
+{
+	OSMO_NAME_C_IMPL(ctx, 64, "ERROR", osmo_mslookup_result_to_str_buf, query, result)
+}
+
+/*! Same as osmo_mslookup_result_to_str_buf(), but directly return the char* (for printf-like string formats). */
+char *osmo_mslookup_result_name_b(char *buf, size_t buflen,
+				  const struct osmo_mslookup_query *query,
+				  const struct osmo_mslookup_result *result)
+{
+	int rc = osmo_mslookup_result_to_str_buf(buf, buflen, query, result);
+	if (rc < 0 && buflen)
+		buf[0] = '\0';
+	return buf;
+}
+
+/*! Copy part of a string to a buffer and nul-terminate it.
+ * \returns 0 on success, negative on error.
+ */
+static int token(char *dest, size_t dest_size, const char *start, const char *end)
+{
+	int len;
+	if (start >= end)
+		return -10;
+	len = end - start;
+	if (len >= dest_size)
+		return -11;
+	strncpy(dest, start, len);
+	dest[len] = '\0';
+	return 0;
+}
+
+/*! Parse a string like "foo.moo.goo.123456789012345.msisdn" into service="foo.moo.goo", id="123456789012345" and
+ * id_type="msisdn", placed in a struct osmo_mslookup_query.
+ * \param q  Write parsed query to this osmo_mslookup_query.
+ * \param domain  Human readable domain string like "sip.voice.12345678.msisdn".
+ * \returns 0 on success, negative on error.
+ */
+int osmo_mslookup_query_init_from_domain_str(struct osmo_mslookup_query *q, const char *domain)
+{
+	const char *last_dot;
+	const char *second_last_dot;
+	const char *id_type;
+	const char *id;
+	int rc;
+
+	*q = (struct osmo_mslookup_query){};
+
+	if (!domain)
+		return -1;
+
+	last_dot = strrchr(domain, '.');
+
+	if (!last_dot)
+		return -2;
+
+	if (last_dot <= domain)
+		return -3;
+
+	for (second_last_dot = last_dot - 1; second_last_dot > domain && *second_last_dot != '.'; second_last_dot--);
+	if (second_last_dot == domain || *second_last_dot != '.')
+		return -3;
+
+	id_type = last_dot + 1;
+	if (!*id_type)
+		return -4;
+
+	q->id.type = get_string_value(osmo_mslookup_id_type_names, id_type);
+
+	id = second_last_dot + 1;
+	switch (q->id.type) {
+	case OSMO_MSLOOKUP_ID_IMSI:
+		rc = token(q->id.imsi, sizeof(q->id.imsi), id, last_dot);
+		if (rc)
+			return rc;
+		if (!osmo_imsi_str_valid(q->id.imsi))
+			return -5;
+		break;
+	case OSMO_MSLOOKUP_ID_MSISDN:
+		rc = token(q->id.msisdn, sizeof(q->id.msisdn), id, last_dot);
+		if (rc)
+			return rc;
+		if (!osmo_msisdn_str_valid(q->id.msisdn))
+			return -6;
+		break;
+	default:
+		return -7;
+	}
+
+	return token(q->service, sizeof(q->service), domain, second_last_dot);
+}
+
+/*! @} */

src/mslookup/mslookup_client.c

@@ -0,0 +1,310 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <osmocom/hlr/logging.h>
+#include <osmocom/mslookup/mslookup_client.h>
+
+/*! Lookup client's internal data for a query. */
+struct osmo_mslookup_client {
+	struct llist_head lookup_methods;
+	struct llist_head requests;
+	uint32_t next_request_handle;
+};
+
+/*! Lookup client's internal data for a query.
+ * The request methods only get to see the query part, and result handling is done commonly for all request methods. */
+struct osmo_mslookup_client_request {
+	struct llist_head entry;
+	struct osmo_mslookup_client *client;
+	uint32_t request_handle;
+
+	struct osmo_mslookup_query query;
+	struct osmo_mslookup_query_handling handling;
+	struct osmo_timer_list timeout;
+	bool waiting_min_delay;
+
+	struct osmo_mslookup_result result;
+};
+
+static struct osmo_mslookup_client_request *get_request(struct osmo_mslookup_client *client, uint32_t request_handle)
+{
+	struct osmo_mslookup_client_request *r;
+	if (!request_handle)
+		return NULL;
+	llist_for_each_entry(r, &client->requests, entry) {
+		if (r->request_handle == request_handle)
+			return r;
+	}
+	return NULL;
+}
+
+struct osmo_mslookup_client *osmo_mslookup_client_new(void *ctx)
+{
+	struct osmo_mslookup_client *client = talloc_zero(ctx, struct osmo_mslookup_client);
+	OSMO_ASSERT(client);
+	INIT_LLIST_HEAD(&client->lookup_methods);
+	INIT_LLIST_HEAD(&client->requests);
+	return client;
+}
+
+/*! Return whether any lookup methods are available.
+ * \param[in] client  Client to query.
+ * \return true when a client is present that has at least one osmo_mslookup_client_method registered.
+ */
+bool osmo_mslookup_client_active(struct osmo_mslookup_client *client)
+{
+	if (!client)
+		return false;
+	if (llist_empty(&client->lookup_methods))
+		return false;
+	return true;
+}
+
+static void _osmo_mslookup_client_method_del(struct osmo_mslookup_client_method *method)
+{
+	if (method->destruct)
+		method->destruct(method);
+	llist_del(&method->entry);
+	talloc_free(method);
+}
+
+/*! Stop and free mslookup client and all registered lookup methods.
+ */
+void osmo_mslookup_client_free(struct osmo_mslookup_client *client)
+{
+	struct osmo_mslookup_client_method *m, *n;
+	if (!client)
+		return;
+	llist_for_each_entry_safe(m, n, &client->lookup_methods, entry) {
+		_osmo_mslookup_client_method_del(m);
+	}
+	talloc_free(client);
+}
+
+/*! Add an osmo_mslookup_client_method to service MS Lookup requests.
+ * Note, osmo_mslookup_client_method_del() will talloc_free() the method pointer, so it needs to be dynamically
+ * allocated.
+ * \param client  The osmo_mslookup_client instance to add to.
+ * \param method  A fully initialized method struct, allocated by talloc.
+ */
+void osmo_mslookup_client_method_add(struct osmo_mslookup_client *client,
+				     struct osmo_mslookup_client_method *method)
+{
+	method->client = client;
+	llist_add_tail(&method->entry, &client->lookup_methods);
+}
+
+/*! \return false if the method was not listed, true if the method was listed, removed and talloc_free()d.
+ */
+bool osmo_mslookup_client_method_del(struct osmo_mslookup_client *client,
+				     struct osmo_mslookup_client_method *method)
+{
+	struct osmo_mslookup_client_method *m;
+	llist_for_each_entry(m, &client->lookup_methods, entry) {
+		if (m == method) {
+			_osmo_mslookup_client_method_del(method);
+			return true;
+		}
+	}
+	return false;
+}
+
+static void osmo_mslookup_request_send_result(struct osmo_mslookup_client_request *r, bool finish)
+{
+	struct osmo_mslookup_client *client = r->client;
+	uint32_t request_handle = r->request_handle;
+
+	r->result.last = finish;
+	r->handling.result_cb(r->client, r->request_handle, &r->query, &r->result);
+
+	/* Make sure the request struct is discarded.
+	 * The result_cb() may already have triggered a cleanup, so query by request_handle. */
+	if (finish)
+		osmo_mslookup_client_request_cancel(client, request_handle);
+}
+
+void osmo_mslookup_client_rx_result(struct osmo_mslookup_client *client, uint32_t request_handle,
+				    const struct osmo_mslookup_result *result)
+{
+	struct osmo_mslookup_client_request *req = get_request(client, request_handle);
+
+	if (!req) {
+		LOGP(DMSLOOKUP, LOGL_ERROR,
+		     "Internal error: Got mslookup result for a request that does not exist (handle %u)\n",
+		     req->request_handle);
+		return;
+	}
+
+	/* Ignore incoming results that are not successful */
+	if (result->rc != OSMO_MSLOOKUP_RC_RESULT)
+		return;
+
+	/* If we already stored an earlier successful result, keep that if its age is younger. */
+	if (req->result.rc == OSMO_MSLOOKUP_RC_RESULT
+	    && result->age >= req->result.age)
+		return;
+
+	req->result = *result;
+
+	/* If age == 0, it doesn't get any better, so return the result immediately. */
+	if (req->result.age == 0) {
+		osmo_mslookup_request_send_result(req, true);
+		return;
+	}
+
+	if (req->waiting_min_delay)
+		return;
+
+	osmo_mslookup_request_send_result(req, false);
+}
+
+static void _osmo_mslookup_client_request_cleanup(struct osmo_mslookup_client_request *r)
+{
+	struct osmo_mslookup_client_method *m;
+	osmo_timer_del(&r->timeout);
+	llist_for_each_entry(m, &r->client->lookup_methods, entry) {
+		if (!m->request_cleanup)
+			continue;
+		m->request_cleanup(m, r->request_handle);
+	}
+	llist_del(&r->entry);
+	talloc_free(r);
+}
+
+static void timeout_cb(void *data);
+
+static void set_timer(struct osmo_mslookup_client_request *r, unsigned long milliseconds)
+{
+	osmo_timer_setup(&r->timeout, timeout_cb, r);
+	osmo_timer_schedule(&r->timeout, milliseconds / 1000, (milliseconds % 1000) * 1000);
+}
+
+static void timeout_cb(void *data)
+{
+	struct osmo_mslookup_client_request *r = data;
+	if (r->waiting_min_delay) {
+		/* The initial delay has passed. See if it stops here, or whether the overall timeout continues. */
+		r->waiting_min_delay = false;
+
+		if (r->handling.result_timeout_milliseconds <= r->handling.min_wait_milliseconds) {
+			/* It ends here. Return a final result. */
+			if (r->result.rc != OSMO_MSLOOKUP_RC_RESULT)
+				r->result.rc = OSMO_MSLOOKUP_RC_NOT_FOUND;
+			osmo_mslookup_request_send_result(r, true);
+			return;
+		}
+
+		/* We continue to listen for results. If one is already on record, send it now. */
+		if (r->result.rc == OSMO_MSLOOKUP_RC_RESULT)
+			osmo_mslookup_request_send_result(r, false);
+
+		set_timer(r, r->handling.result_timeout_milliseconds - r->handling.min_wait_milliseconds);
+		return;
+	}
+	/* The final timeout has passed, finish and clean up the request. */
+	switch (r->result.rc) {
+	case OSMO_MSLOOKUP_RC_RESULT:
+		/* If the rc == OSMO_MSLOOKUP_RC_RESULT, this result has already been sent.
+		 * Don't send it again, instead send an RC_NONE, last=true result. */
+		r->result.rc = OSMO_MSLOOKUP_RC_NONE;
+		break;
+	default:
+		r->result.rc = OSMO_MSLOOKUP_RC_NOT_FOUND;
+		break;
+	}
+	osmo_mslookup_request_send_result(r, true);
+}
+
+/*! Launch a subscriber lookup with the provided query.
+ * A request is cleared implicitly when the handling->result_cb is invoked; if the quer->priv pointer becomes invalid
+ * before that, a request should be canceled by calling osmo_mslookup_client_request_cancel() with the returned
+ * request_handle. A request handle of zero indicates error.
+ * \return a nonzero request_handle that allows ending the request, or 0 on invalid query data. */
+uint32_t osmo_mslookup_client_request(struct osmo_mslookup_client *client,
+				      const struct osmo_mslookup_query *query,
+				      const struct osmo_mslookup_query_handling *handling)
+{
+	struct osmo_mslookup_client_request *r;
+	struct osmo_mslookup_client_request *other;
+	struct osmo_mslookup_client_method *m;
+
+	if (!osmo_mslookup_service_valid(query->service)
+	    || !osmo_mslookup_id_valid(&query->id)) {
+		char buf[256];
+		LOGP(DMSLOOKUP, LOGL_ERROR, "Invalid query: %s\n",
+		     osmo_mslookup_result_name_b(buf, sizeof(buf), query, NULL));
+		return 0;
+	}
+
+	r = talloc_zero(client, struct osmo_mslookup_client_request);
+	OSMO_ASSERT(r);
+
+	/* A request_handle of zero means error, so make sure we don't use a zero handle. */
+	if (!client->next_request_handle)
+		client->next_request_handle++;
+	*r = (struct osmo_mslookup_client_request){
+		.client = client,
+		.query = *query,
+		.handling = *handling,
+		.request_handle = client->next_request_handle++,
+	};
+
+	if (!r->handling.result_timeout_milliseconds)
+		r->handling.result_timeout_milliseconds = r->handling.min_wait_milliseconds;
+	if (!r->handling.result_timeout_milliseconds)
+		r->handling.result_timeout_milliseconds = 1000;
+
+	/* Paranoia: make sure a request_handle exists only once, by expiring an already existing one. This is unlikely
+	 * to happen in practice: before we get near wrapping a uint32_t range, previous requests should long have
+	 * timed out or ended. */
+	llist_for_each_entry(other, &client->requests, entry) {
+		if (other->request_handle != r->request_handle)
+			continue;
+		osmo_mslookup_request_send_result(other, true);
+		/* we're sure it exists only once. */
+		break;
+	}
+
+	/* Now sure that the new request_handle does not exist a second time. */
+	llist_add_tail(&r->entry, &client->requests);
+
+	if (r->handling.min_wait_milliseconds) {
+		r->waiting_min_delay = true;
+		set_timer(r, r->handling.min_wait_milliseconds);
+	} else {
+		set_timer(r, r->handling.result_timeout_milliseconds);
+	}
+
+	/* Let the lookup implementations know */
+	llist_for_each_entry(m, &client->lookup_methods, entry) {
+		m->request(m, query, r->request_handle);
+	}
+	return r->request_handle;
+}
+
+/*! End or cancel a subscriber lookup. This *must* be invoked exactly once per osmo_mslookup_client_request() invocation,
+ * either after a lookup has concluded or to abort an ongoing lookup.
+ * \param[in] request_handle  The request_handle returned by an osmo_mslookup_client_request() invocation.
+ */
+void osmo_mslookup_client_request_cancel(struct osmo_mslookup_client *client, uint32_t request_handle)
+{
+	struct osmo_mslookup_client_request *r = get_request(client, request_handle);
+	if (!r)
+		return;
+	_osmo_mslookup_client_request_cleanup(r);
+}

src/mslookup/mslookup_client_fake.c

@@ -0,0 +1,156 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <osmocom/hlr/logging.h>
+#include <osmocom/mslookup/mslookup_client.h>
+#include <osmocom/mslookup/mslookup_client_fake.h>
+
+#include <string.h>
+
+/* Fake mslookup method */
+
+struct fake_lookup_state {
+	struct osmo_mslookup_client *client;
+	struct llist_head requests;
+	struct osmo_timer_list async_response_timer;
+	struct osmo_mslookup_fake_response *responses;
+	size_t responses_len;
+};
+
+struct fake_lookup_request {
+	struct llist_head entry;
+	uint32_t request_handle;
+	struct osmo_mslookup_query query;
+	struct timeval received_at;
+};
+
+/*! Args for osmo_timer_schedule: seconds and microseconds. */
+#define ASYNC_RESPONSE_PERIOD 0, (1e6 / 10)
+static void fake_lookup_async_response(void *state);
+
+static void fake_lookup_request(struct osmo_mslookup_client_method *method,
+				const struct osmo_mslookup_query *query,
+				uint32_t request_handle)
+{
+	struct fake_lookup_state *state = method->priv;
+	char buf[256];
+	LOGP(DMSLOOKUP, LOGL_DEBUG, "%s(%s)\n", __func__, osmo_mslookup_result_name_b(buf, sizeof(buf), query, NULL));
+
+	/* A real implementation would send packets to some remote server.
+	 * Here this is simulated: add to the list of requests, which fake_lookup_async_response() will reply upon
+	 * according to the test data listing the replies that the test wants to generate. */
+
+	struct fake_lookup_request *r = talloc_zero(method->client, struct fake_lookup_request);
+	*r = (struct fake_lookup_request){
+		.request_handle = request_handle,
+		.query = *query,
+	};
+	osmo_gettimeofday(&r->received_at, NULL);
+	llist_add_tail(&r->entry, &state->requests);
+}
+
+static void fake_lookup_request_cleanup(struct osmo_mslookup_client_method *method,
+					uint32_t request_handle)
+{
+	struct fake_lookup_state *state = method->priv;
+
+	/* Tear down any state associated with this handle. */
+	struct fake_lookup_request *r;
+	llist_for_each_entry(r, &state->requests, entry) {
+		if (r->request_handle != request_handle)
+			continue;
+		llist_del(&r->entry);
+		talloc_free(r);
+		LOGP(DMSLOOKUP, LOGL_DEBUG, "%s() ok\n", __func__);
+		return;
+	}
+	LOGP(DMSLOOKUP, LOGL_DEBUG, "%s() FAILED\n", __func__);
+}
+
+static void fake_lookup_async_response(void *data)
+{
+	struct fake_lookup_state *state = data;
+	struct fake_lookup_request *req, *n;
+	struct timeval now;
+	char str[256];
+
+	osmo_gettimeofday(&now, NULL);
+
+	llist_for_each_entry_safe(req, n, &state->requests, entry) {
+		struct osmo_mslookup_fake_response *resp;
+
+		for (resp = state->responses;
+		     (resp - state->responses) < state->responses_len;
+		     resp++) {
+			struct timeval diff;
+
+			if (resp->sent)
+				continue;
+			if (osmo_mslookup_id_cmp(&req->query.id, &resp->for_id) != 0)
+				continue;
+			if (strcmp(req->query.service, resp->for_service) != 0)
+				continue;
+
+			timersub(&now, &req->received_at, &diff);
+			if (timercmp(&diff, &resp->time_to_reply, <))
+				continue;
+
+			/* It's time to reply to this request. */
+			LOGP(DMSLOOKUP, LOGL_DEBUG, "osmo_mslookup_client_rx_result(): %s\n",
+			     osmo_mslookup_result_name_b(str, sizeof(str), &req->query, &resp->result));
+			osmo_mslookup_client_rx_result(state->client, req->request_handle, &resp->result);
+			resp->sent = true;
+
+			/* The req will have been cleaned up now, so we must not iterate over state->responses anymore
+			 * with this req. */
+			break;
+		}
+	}
+
+	osmo_timer_schedule(&state->async_response_timer, ASYNC_RESPONSE_PERIOD);
+}
+
+struct osmo_mslookup_client_method *osmo_mslookup_client_add_fake(struct osmo_mslookup_client *client,
+								  struct osmo_mslookup_fake_response *responses,
+								  size_t responses_len)
+{
+	struct osmo_mslookup_client_method *method = talloc_zero(client, struct osmo_mslookup_client_method);
+	OSMO_ASSERT(method);
+
+	struct fake_lookup_state *state = talloc_zero(method, struct fake_lookup_state);
+	OSMO_ASSERT(state);
+	*state = (struct fake_lookup_state){
+		.client = client,
+		.responses = responses,
+		.responses_len = responses_len,
+	};
+	INIT_LLIST_HEAD(&state->requests);
+
+	*method = (struct osmo_mslookup_client_method){
+		.name = "fake",
+		.priv = state,
+		.request = fake_lookup_request,
+		.request_cleanup = fake_lookup_request_cleanup,
+	};
+
+	osmo_timer_setup(&state->async_response_timer, fake_lookup_async_response, state);
+	osmo_mslookup_client_method_add(client, method);
+
+	osmo_timer_schedule(&state->async_response_timer, ASYNC_RESPONSE_PERIOD);
+	return method;
+}

src/mslookup/mslookup_client_mdns.c

@@ -0,0 +1,235 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <string.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <unistd.h>
+#include <errno.h>
+#include <osmocom/core/select.h>
+#include <osmocom/gsm/gsm_utils.h>
+#include <osmocom/hlr/logging.h>
+#include <osmocom/mslookup/mdns.h>
+#include <osmocom/mslookup/mdns_sock.h>
+#include <osmocom/mslookup/mslookup_client.h>
+#include <osmocom/mslookup/mslookup_client_mdns.h>
+
+struct osmo_mdns_method_state {
+	/* Parameters passed by _add_method_dns() */
+	struct osmo_sockaddr_str bind_addr;
+	const char *domain_suffix;
+
+	struct osmo_mdns_sock *mc;
+
+	struct osmo_mslookup_client *client;
+	struct llist_head requests;
+	uint16_t next_packet_id;
+};
+
+struct osmo_mdns_method_request {
+	struct llist_head entry;
+	uint32_t request_handle;
+	struct osmo_mslookup_query query;
+	uint16_t packet_id;
+};
+
+static int request_handle_by_query(uint32_t *request_handle, struct osmo_mdns_method_state *state,
+				   struct osmo_mslookup_query *query, uint16_t packet_id)
+{
+	struct osmo_mdns_method_request *request;
+
+	llist_for_each_entry(request, &state->requests, entry) {
+		if (strcmp(request->query.service, query->service) != 0)
+			continue;
+		if (osmo_mslookup_id_cmp(&request->query.id, &query->id) != 0)
+			continue;
+
+		/* Match! */
+		*request_handle = request->request_handle;
+		return 0;
+	}
+	return -1;
+}
+
+static int mdns_method_recv(struct osmo_fd *osmo_fd, unsigned int what)
+{
+	struct osmo_mdns_method_state *state = osmo_fd->data;
+	struct osmo_mslookup_result result;
+	struct osmo_mslookup_query query;
+	uint16_t packet_id;
+	int n;
+	uint8_t buffer[1024];
+	uint32_t request_handle = 0;
+	void *ctx = state;
+
+	n = read(osmo_fd->fd, buffer, sizeof(buffer));
+	if (n < 0) {
+		LOGP(DMSLOOKUP, LOGL_ERROR, "failed to read from socket\n");
+		return n;
+	}
+
+	if (osmo_mdns_result_decode(ctx, buffer, n, &packet_id, &query, &result, state->domain_suffix) < 0)
+		return -EINVAL;
+
+	if (request_handle_by_query(&request_handle, state, &query, packet_id) != 0)
+		return -EINVAL;
+
+	osmo_mslookup_client_rx_result(state->client, request_handle, &result);
+	return n;
+}
+
+static void mdns_method_request(struct osmo_mslookup_client_method *method, const struct osmo_mslookup_query *query,
+				uint32_t request_handle)
+{
+	char buf[256];
+	struct osmo_mdns_method_state *state = method->priv;
+	struct msgb *msg;
+	struct osmo_mdns_method_request *r = talloc_zero(method->client, struct osmo_mdns_method_request);
+
+	*r = (struct osmo_mdns_method_request){
+		.request_handle = request_handle,
+		.query = *query,
+		.packet_id = state->next_packet_id,
+	};
+	llist_add(&r->entry, &state->requests);
+	state->next_packet_id++;
+
+	msg = osmo_mdns_query_encode(method->client, r->packet_id, query, state->domain_suffix);
+	if (!msg) {
+		LOGP(DMSLOOKUP, LOGL_ERROR, "Cannot encode request: %s\n",
+		     osmo_mslookup_result_name_b(buf, sizeof(buf), query, NULL));
+	}
+
+	/* Send over the wire */
+	LOGP(DMSLOOKUP, LOGL_DEBUG, "sending mDNS query: %s.%s\n", query->service,
+	     osmo_mslookup_id_name_b(buf, sizeof(buf), &query->id));
+	if (osmo_mdns_sock_send(state->mc, msg) == -1)
+		LOGP(DMSLOOKUP, LOGL_ERROR, "sending mDNS query failed\n");
+}
+
+static void mdns_method_request_cleanup(struct osmo_mslookup_client_method *method, uint32_t request_handle)
+{
+	struct osmo_mdns_method_state *state = method->priv;
+
+	/* Tear down any state associated with this handle. */
+	struct osmo_mdns_method_request *r;
+	llist_for_each_entry(r, &state->requests, entry) {
+		if (r->request_handle != request_handle)
+			continue;
+		llist_del(&r->entry);
+		talloc_free(r);
+		return;
+	}
+}
+
+static void mdns_method_destruct(struct osmo_mslookup_client_method *method)
+{
+	struct osmo_mdns_method_state *state = method->priv;
+	struct osmo_mdns_method_request *e, *n;
+	if (!state)
+		return;
+
+	/* Drop all DNS lookup request state. Triggering a timeout event and cleanup for mslookup client users will
+	 * happen in the mslookup_client.c, we will simply stop responding from this lookup method. */
+	llist_for_each_entry_safe(e, n, &state->requests, entry) {
+		llist_del(&e->entry);
+	}
+
+	osmo_mdns_sock_cleanup(state->mc);
+}
+
+/*! Initialize the mDNS lookup method.
+ * \param[in] client  the client to attach the method to.
+ * \param[in] ip  IPv4 or IPv6 address string.
+ * \param[in] port  The port to bind to.
+ * \param[in] initial_packet_id  Used in the first mslookup query, then increased by one in each following query. All
+ *				 servers answer to each query with the same packet ID. Set to -1 to use a random
+ *				 initial ID (recommended unless you need deterministic output). This ID is for visually
+ *				 distinguishing the packets in packet sniffers, the mslookup client uses not just the
+ *				 ID, but all query parameters (service type, ID, ID type), to determine if a reply is
+ *				 relevant.
+ * \param[in] domain_suffix  is appended to each domain in the queries to avoid colliding with the top-level domains
+ *                           administrated by IANA. Example: "mdns.osmocom.org" */
+struct osmo_mslookup_client_method *osmo_mslookup_client_add_mdns(struct osmo_mslookup_client *client, const char *ip,
+								  uint16_t port, int initial_packet_id,
+								  const char *domain_suffix)
+{
+	struct osmo_mdns_method_state *state;
+	struct osmo_mslookup_client_method *m;
+
+	m = talloc_zero(client, struct osmo_mslookup_client_method);
+	OSMO_ASSERT(m);
+
+	state = talloc_zero(m, struct osmo_mdns_method_state);
+	OSMO_ASSERT(state);
+	INIT_LLIST_HEAD(&state->requests);
+	if (osmo_sockaddr_str_from_str(&state->bind_addr, ip, port)) {
+		LOGP(DMSLOOKUP, LOGL_ERROR, "mslookup mDNS: invalid address/port: %s %u\n",
+		     ip, port);
+		goto error_cleanup;
+	}
+
+	if (initial_packet_id == -1) {
+		if (osmo_get_rand_id((uint8_t *)&state->next_packet_id, 2) < 0) {
+			LOGP(DMSLOOKUP, LOGL_ERROR, "mslookup mDNS: failed to generate random initial packet ID\n");
+			goto error_cleanup;
+		}
+	} else
+		state->next_packet_id = initial_packet_id;
+
+	state->client = client;
+	state->domain_suffix = domain_suffix;
+
+	state->mc = osmo_mdns_sock_init(state, ip, port, mdns_method_recv, state, 0);
+	if (!state->mc)
+		goto error_cleanup;
+
+	*m = (struct osmo_mslookup_client_method){
+		.name = "mDNS",
+		.priv = state,
+		.request = mdns_method_request,
+		.request_cleanup = mdns_method_request_cleanup,
+		.destruct = mdns_method_destruct,
+	};
+
+	osmo_mslookup_client_method_add(client, m);
+	return m;
+
+error_cleanup:
+	talloc_free(m);
+	return NULL;
+}
+
+const struct osmo_sockaddr_str *osmo_mslookup_client_method_mdns_get_bind_addr(struct osmo_mslookup_client_method *dns_method)
+{
+	struct osmo_mdns_method_state *state;
+	if (!dns_method || !dns_method->priv)
+		return NULL;
+	state = dns_method->priv;
+	return &state->bind_addr;
+}
+
+const char *osmo_mslookup_client_method_mdns_get_domain_suffix(struct osmo_mslookup_client_method *dns_method)
+{
+	struct osmo_mdns_method_state *state;
+	if (!dns_method || !dns_method->priv)
+		return NULL;
+	state = dns_method->priv;
+	return state->domain_suffix;
+}

src/mslookup/osmo-mslookup-client.c

@@ -0,0 +1,879 @@
+/*! \file osmo-mslookup-client.c
+ * Distributed GSM: find the location of subscribers, for example by multicast DNS,
+ * to obtain HLR, SIP or SMPP server addresses (or arbitrary service names).
+ */
+/*
+ * (C) 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ * (C) 2019 by Neels Hofmeyr <neels@hofmeyr.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <stdio.h>
+#include <unistd.h>
+#include <getopt.h>
+#include <errno.h>
+#include <talloc.h>
+#include <sys/un.h>
+
+#include <osmocom/core/application.h>
+#include <osmocom/core/logging.h>
+#include <osmocom/core/select.h>
+#include <osmocom/core/socket.h>
+#include <osmocom/mslookup/mslookup_client.h>
+#include <osmocom/mslookup/mslookup_client_mdns.h>
+#include <osmocom/mslookup/mdns_sock.h>
+#include <osmocom/mslookup/mdns.h>
+
+#define CSV_HEADERS "query\tresult\tlast\tage\tv4_ip\tv4_port\tv6_ip\tv6_port"
+
+static void print_version(void)
+{
+	printf("osmo-mslookup-client version %s\n", PACKAGE_VERSION);
+	printf("\n"
+	"Copyright (C) 2019 by sysmocom - s.f.m.c. GmbH\n"
+	"Copyright (C) 2019 by Neels Hofmeyr <neels@hofmeyr.de>\n"
+	"This program is free software; you can redistribute it and/or modify\n"
+	"it under the terms of the GNU General Public License as published by\n"
+	"the Free Software Foundation; either version 2 of the License, or\n"
+	"(at your option) any later version.\n"
+	"\n");
+}
+
+static void print_help()
+{
+	print_version();
+	printf(
+"Standalone mslookup client for Distributed GSM\n"
+"\n"
+"Receiving mslookup results means listening for responses on a socket. Often,\n"
+"integration (e.g. FreeSwitch dialplan.py) makes it hard to select() on a socket\n"
+"to read responses, because that interferes with the main program (e.g.\n"
+"FreeSwitch's dialplan.py seems to be integrated with an own select() main loop\n"
+"that interferes with osmo_select_main(), or an smpp.py uses\n"
+"smpplib.client.listen() as main loop, etc.).\n"
+"\n"
+"This program provides a trivial solution, by outsourcing the mslookup main loop\n"
+"to a separate process. Communication is done via cmdline arg and stdout pipe or\n"
+"a (blocking) unix domain socket, results are returned in CSV or JSON format.\n"
+"\n"
+"This can be done one-shot, i.e. exit as soon as the response has been\n"
+"determined, or in daemon form, i.e. continuously listen for requests and return\n"
+"responses.\n"
+"\n"
+"About running a local daemon: it is unintuitive to connect to a socket to solve\n"
+"a problem of reading from a socket -- it seems like just more of the same\n"
+"problem. The reasons why the daemon is in fact useful are:\n"
+"- The osmo-mslookup-client daemon will return only those results matching\n"
+"  requests issued on that socket connection.\n"
+"- A program can simply blockingly recv() from the osmo-mslookup-client socket\n"
+"  instead of needing to run osmo_select_main() so that libosmo-mslookup is able\n"
+"  to asynchronously receive responses from remote servers.\n"
+"- Only one long-lived multicast socket needs to be opened instead of a new\n"
+"  socket for each request.\n"
+"\n"
+"Output is in CSV or json, see --format. The default is tab-separated CSV\n"
+"with these columns:\n"
+CSV_HEADERS "\n"
+"\n"
+"One-shot operation example:\n"
+"$ osmo-mslookup-client 1000-@sip.voice.12345.msisdn -f json\n"
+"{\"query\": \"sip.voice.12345.msisdn\", \"result\": \"result\", \"last\": true, \"age\": 5, \"v4\": [\"1.2.3.7\", \"23\"]}\n"
+"$\n"
+"\n"
+"Daemon operation example:\n"
+"$ osmo-mslookup-client -s /tmp/mslookup -d\n"
+"(and a client program then connects to /tmp/mslookup, find an implementation\n"
+"example below)\n"
+"\n"
+"Integrating with calling programs can be done by:\n"
+"- call osmo-mslookup-client with the query string as argument.\n"
+"  It will open a multicast DNS socket, send out a query and wait for the\n"
+"  matching response. It will print the result on stdout and exit.\n"
+"  This method launches a new process for every mslookup query,\n"
+"  and creates a short-lived multicast listener for each invocation.\n"
+"  This is fine for low activity, but does not scale well.\n"
+"\n"
+"- invoke osmo-mslookup-client --socket /tmp/mslookup -d.\n"
+"  Individual queries can be sent by connecting to that unix domain socket,\n"
+"  blockingly reading the response when it arrives and disconnecting.\n"
+"  This way only one process keeps one multicast listener open.\n"
+"  Callers can connect to this socket without spawning processes.\n"
+"  This is recommended for scale.\n"
+"\n"
+"Python example clients for {CSV,JSON}x{cmdline,socket} can be found here:\n"
+"http://git.osmocom.org/osmo-hlr/tree/contrib/dgsm/osmo-mslookup-pipe.py\n"
+"http://git.osmocom.org/osmo-hlr/tree/contrib/dgsm/osmo-mslookup-socket.py\n"
+"\n"
+"\n"
+"Options:\n"
+"\n"
+"[[delay-][timeout]@]service.number.id\n"
+"	A service query string with optional individual timeout.\n"
+"	The same format is also used on a daemon socket, if any.\n"
+"	The timeout consists of the min-delay and the timeout numbers,\n"
+"	corresponding to the --min-delay and --timeout options, in milliseconds.\n"
+"	These options apply if a query string lacks own numbers.\n"
+"	Examples:\n"
+"		gsup.hlr.1234567.imsi		Use cmdline timeout settings\n"
+"		5000@gsup.hlr.1234567.imsi	Return N results for 5 seconds\n"
+"		1000-5000@sip.voice.123.msisdn	Same, but silent for first second\n"
+"		10000-@smpp.sms.567.msisdn	Return 1 result after 10 seconds\n"
+"\n"
+"--format -f csv (default)\n"
+"	Format result lines in CSV format.\n"
+"--no-csv-headers -H\n"
+"	If the format is 'csv', by default, the first output line prints the\n"
+"	CSV headers used for CSV output format. This option disables these CSV\n"
+"	headers.\n"
+"\n"
+"--format -f json\n"
+"	Format result lines in json instead of semicolon separated, like:\n"
+"	{\"query\": \"sip.voice.12345.msisdn\", \"result\": \"ok\", \"v4\": [\"10.9.8.7\", \"5060\"]}\n"
+"\n"
+"--daemon -d\n"
+"	Keep running after a request has been serviced\n"
+"\n"
+"--mdns-ip -m " OSMO_MSLOOKUP_MDNS_IP4 " -m " OSMO_MSLOOKUP_MDNS_IP6 "\n"
+"--mdns-port -M " OSMO_STRINGIFY_VAL(OSMO_MSLOOKUP_MDNS_PORT) "\n"
+"	Set multicast IP address / port to send mDNS requests and listen for\n"
+"	mDNS reponses\n"
+"--mdns-domain-suffix -D " OSMO_MDNS_DOMAIN_SUFFIX_DEFAULT "\n"
+"	Append this suffix to each mDNS query's domain to avoid colliding with the\n"
+"	top-level domains administrated by IANA.\n"
+"\n"
+"--min-delay -t 1000 (in milliseconds)\n"
+"	Set minimum delay to wait before returning any results.\n"
+"	When this timeout has elapsed, the best current result is returned,\n"
+"	if any is available.\n"
+"	Responses arriving after the min-delay has elapsed which have a younger\n"
+"	age than previous results are returned immediately.\n"
+"	Note: When a response with age of zero comes in, the result is returned\n"
+"	immediately and the request is discarded: non-daemon mode exits, daemon\n"
+"	mode ignores later results.\n"
+"\n"
+"--timeout -T 1000 (in milliseconds)\n"
+"	Set timeout after which to stop listening for responses.\n"
+"	If this is smaller than -t, the value from -t will be used for -T as well.\n"
+"	Note: When a response with age of zero comes in, the result is returned\n"
+"	immediately and the request is discarded: non-daemon mode exits, daemon\n"
+"	mode ignores later results.\n"
+"\n"
+"--socket -s /path/to/unix-domain-socket\n"
+"	Listen to requests from and write responses to a UNIX domain socket.\n"
+"\n"
+"--send -S <query> <age> <ip1> <port1> <ip2> <port2>\n"
+"	Do not query, but send an mslookup result. This is useful only for\n"
+"	testing. Examples:\n"
+"	--send foo.123.msisdn 300 23.42.17.11 1234\n"
+"	--send foo.123.msisdn 300 2323:4242:1717:1111::42 1234\n"
+"	--send foo.123.msisdn 300 23.42.17.11 1234 2323:4242:1717:1111::42 1234\n"
+"\n"
+"--quiet -q\n"
+"	Do not print errors to stderr, do not log to stderr.\n"
+"\n"
+"--help -h\n"
+"	This help\n"
+);
+}
+
+enum result_format {
+	FORMAT_CSV = 0,
+	FORMAT_JSON,
+};
+
+static struct {
+	bool daemon;
+	struct osmo_sockaddr_str mdns_addr;
+	uint32_t min_delay;
+	uint32_t timeout;
+	const char *socket_path;
+	const char *format_str;
+	const char *mdns_domain_suffix;
+	bool csv_headers;
+	bool send;
+	bool quiet;
+} cmdline_opts = {
+	.mdns_addr = { .af=AF_INET, .ip=OSMO_MSLOOKUP_MDNS_IP4, .port=OSMO_MSLOOKUP_MDNS_PORT },
+	.min_delay = 1000,
+	.timeout = 1000,
+	.csv_headers = true,
+	.mdns_domain_suffix = OSMO_MDNS_DOMAIN_SUFFIX_DEFAULT,
+};
+
+#define print_error(fmt, args...) do { \
+		if (!cmdline_opts.quiet) \
+			fprintf(stderr, fmt, ##args); \
+	} while (0)
+
+char g_buf[1024];
+
+long long int parse_int(long long int minval, long long int maxval, const char *arg, int *rc)
+{
+	long long int val;
+	char *endptr;
+	if (rc)
+		*rc = -1;
+	if (!arg)
+		return -1;
+	errno = 0;
+	val = strtoll(arg, &endptr, 10);
+	if (errno || val < minval || val > maxval || *endptr)
+		return -1;
+	if (rc)
+		*rc = 0;
+	return val;
+}
+
+int cb_doing_nothing(struct osmo_fd *fd, unsigned int what)
+{
+	return 0;
+}
+
+/* --send: Just send a response, for manual testing. */
+int do_send(int argc, char ** argv)
+{
+	/* parse args <query> <age> <v4-ip> <v4-port> <v6-ip> <v6-port> */
+#define ARG(NR) ((argc > NR)? argv[NR] : NULL)
+	const char *query_str = ARG(0);
+	const char *age_str = ARG(1);
+	const char *ip_strs[2][2] = {
+		{ ARG(2), ARG(3) },
+		{ ARG(4), ARG(5) },
+	};
+	struct osmo_mslookup_query q = {};
+	struct osmo_mslookup_result r = { .rc = OSMO_MSLOOKUP_RC_RESULT };
+	int i;
+	int rc;
+	void *ctx = talloc_named_const(NULL, 0, __func__);
+	struct osmo_mdns_sock *sock;
+
+	if (!query_str) {
+		print_error("--send needs a query string like foo.123456.imsi\n");
+		return 1;
+	}
+	if (osmo_mslookup_query_init_from_domain_str(&q, query_str)) {
+		print_error("Invalid query string '%s', need a query string like foo.123456.imsi\n",
+			    query_str);
+		return 1;
+	}
+
+	if (!age_str) {
+		print_error("--send needs an age\n");
+		return 1;
+	}
+	r.age = parse_int(0, UINT32_MAX, age_str, &rc);
+	if (rc) {
+		print_error("invalid age\n");
+		return 1;
+	}
+
+	for (i = 0; i < 2; i++) {
+		struct osmo_sockaddr_str addr;
+		uint16_t port;
+		if (!ip_strs[i][0])
+			continue;
+		port = parse_int(1, 65535, ip_strs[i][1] ? : "2342", &rc);
+		if (rc) {
+			print_error("invalid port: %s\n", ip_strs[i][1] ? : "NULL");
+			return 1;
+		}
+		if (osmo_sockaddr_str_from_str(&addr, ip_strs[i][0], port)) {
+			print_error("invalid IP addr: %s\n", ip_strs[i][0]);
+			return 1;
+		}
+		if (addr.af == AF_INET)
+			r.host_v4 = addr;
+		else
+			r.host_v6 = addr;
+	}
+
+	printf("Sending mDNS to " OSMO_SOCKADDR_STR_FMT ": %s\n", OSMO_SOCKADDR_STR_FMT_ARGS(&cmdline_opts.mdns_addr),
+	       osmo_mslookup_result_name_c(ctx, &q, &r));
+
+	rc = 1;
+	sock = osmo_mdns_sock_init(ctx, cmdline_opts.mdns_addr.ip, cmdline_opts.mdns_addr.port,
+				   cb_doing_nothing, NULL, 0);
+	if (!sock) {
+		print_error("unable to open mDNS socket\n");
+		goto exit_cleanup;
+	}
+
+	struct msgb *msg = osmo_mdns_result_encode(ctx, 0, &q, &r, cmdline_opts.mdns_domain_suffix);
+	if (!msg) {
+		print_error("unable to encode mDNS response\n");
+		goto exit_cleanup;
+	}
+
+	if (osmo_mdns_sock_send(sock, msg)) {
+		print_error("unable to send mDNS message\n");
+		goto exit_cleanup;
+	}
+
+	rc = 0;
+exit_cleanup:
+	osmo_mdns_sock_cleanup(sock);
+	talloc_free(ctx);
+	return rc;
+}
+
+static struct {
+	void *ctx;
+	unsigned int requests_handled;
+	struct osmo_fd socket_ofd;
+	struct osmo_mslookup_client *mslookup_client;
+	struct llist_head queries;
+	struct llist_head socket_clients;
+	enum result_format format;
+} globals = {
+	.queries = LLIST_HEAD_INIT(globals.queries),
+	.socket_clients = LLIST_HEAD_INIT(globals.socket_clients),
+};
+
+typedef void (*formatter_t)(char *buf, size_t buflen, const char *query_str, const struct osmo_mslookup_result *r);
+
+void formatter_csv(char *buf, size_t buflen, const char *query_str, const struct osmo_mslookup_result *r)
+{
+	struct osmo_strbuf sb = { .buf=buf, .len=buflen };
+	OSMO_STRBUF_PRINTF(sb, "%s", query_str);
+
+	if (!r)
+		OSMO_STRBUF_PRINTF(sb, "\tERROR\t\t\t\t\t\t");
+	else {
+		OSMO_STRBUF_PRINTF(sb, "\t%s", osmo_mslookup_result_code_name(r->rc));
+		OSMO_STRBUF_PRINTF(sb, "\t%s", r->last ? "last" : "not-last");
+		OSMO_STRBUF_PRINTF(sb, "\t%u", r->age);
+		switch (r->rc) {
+		case OSMO_MSLOOKUP_RC_RESULT:
+			if (osmo_sockaddr_str_is_nonzero(&r->host_v4))
+				OSMO_STRBUF_PRINTF(sb, "\t%s\t%u", r->host_v4.ip, r->host_v4.port);
+			else
+				OSMO_STRBUF_PRINTF(sb, "\t\t");
+			if (osmo_sockaddr_str_is_nonzero(&r->host_v6))
+				OSMO_STRBUF_PRINTF(sb, "\t%s\t%u", r->host_v6.ip, r->host_v6.port);
+			else
+				OSMO_STRBUF_PRINTF(sb, "\t\t");
+			break;
+		default:
+			OSMO_STRBUF_PRINTF(sb, "\t\t\t\t\t");
+			break;
+		}
+	}
+}
+
+void formatter_json(char *buf, size_t buflen, const char *query_str, const struct osmo_mslookup_result *r)
+{
+	struct osmo_strbuf sb = { .buf=buf, .len=buflen };
+	OSMO_STRBUF_PRINTF(sb, "{\"query\": \"%s\"", query_str);
+
+	if (!r)
+		OSMO_STRBUF_PRINTF(sb, ", \"result\": \"ERROR\"");
+	else {
+		OSMO_STRBUF_PRINTF(sb, ", \"result\": \"%s\"", osmo_mslookup_result_code_name(r->rc));
+		OSMO_STRBUF_PRINTF(sb, ", \"last\": %s", r->last ? "true" : "false");
+		OSMO_STRBUF_PRINTF(sb, ", \"age\": %u", r->age);
+		if (r->rc == OSMO_MSLOOKUP_RC_RESULT) {
+			if (osmo_sockaddr_str_is_nonzero(&r->host_v4))
+				OSMO_STRBUF_PRINTF(sb, ", \"v4\": [\"%s\", \"%u\"]", r->host_v4.ip, r->host_v4.port);
+			if (osmo_sockaddr_str_is_nonzero(&r->host_v6))
+				OSMO_STRBUF_PRINTF(sb, ", \"v6\": [\"%s\", \"%u\"]", r->host_v6.ip, r->host_v6.port);
+		}
+	}
+	OSMO_STRBUF_PRINTF(sb, "}");
+}
+
+formatter_t formatters[] = {
+	[FORMAT_CSV] = formatter_csv,
+	[FORMAT_JSON] = formatter_json,
+};
+
+void respond_str_stdout(const char *str) {
+	fprintf(stdout, "%s\n", str);
+	fflush(stdout);
+}
+
+void start_query_str(const char *query_str);
+void start_query_strs(char **query_strs, size_t query_strs_len);
+
+struct socket_client {
+	struct llist_head entry;
+	struct osmo_fd ofd;
+	char query_str[1024];
+};
+
+static void socket_client_close(struct socket_client *c)
+{
+	struct osmo_fd *ofd = &c->ofd;
+
+	close(ofd->fd);
+	ofd->fd = -1;
+	osmo_fd_unregister(ofd);
+
+	llist_del(&c->entry);
+	talloc_free(c);
+}
+
+void socket_client_respond_result(struct socket_client *c, const char *response)
+{
+	write(c->ofd.fd, response, strlen(response));
+}
+
+static int socket_read_cb(struct osmo_fd *ofd)
+{
+	struct socket_client *c = ofd->data;
+	int rc;
+	char rxbuf[1024];
+	char *query_with_timeout;
+	char *query_str;
+	char *at;
+
+	rc = recv(ofd->fd, rxbuf, sizeof(rxbuf), 0);
+	if (rc == 0)
+		goto close;
+
+	if (rc < 0) {
+		if (errno == EAGAIN)
+			return 0;
+		goto close;
+	}
+
+	if (rc >= sizeof(c->query_str))
+		goto close;
+
+	rxbuf[rc] = '\0';
+	query_with_timeout = strtok(rxbuf, "\r\n");
+	at = strchr(query_with_timeout, '@');
+	query_str = at ? at + 1 : query_with_timeout;
+
+	if (c->query_str[0]) {
+		print_error("ERROR: Only one query per client connect is allowed;"
+			    " received '%s' and '%s' on the same connection\n",
+			    c->query_str, query_str);
+		formatters[globals.format](g_buf, sizeof(g_buf), query_str, NULL);
+		socket_client_respond_result(c, g_buf);
+		return 0;
+	}
+
+	OSMO_STRLCPY_ARRAY(c->query_str, query_str);
+	start_query_str(query_with_timeout);
+	printf("query: %s\n", query_with_timeout);
+	return rc;
+
+close:
+	socket_client_close(c);
+	return -1;
+}
+
+static int socket_cb(struct osmo_fd *ofd, unsigned int flags)
+{
+	int rc = 0;
+
+	if (flags & BSC_FD_READ)
+		rc = socket_read_cb(ofd);
+	if (rc < 0)
+		return rc;
+
+	return rc;
+}
+
+int socket_accept(struct osmo_fd *ofd, unsigned int flags)
+{
+	struct socket_client *c;
+	struct sockaddr_un un_addr;
+	socklen_t len;
+	int rc;
+
+	len = sizeof(un_addr);
+	rc = accept(ofd->fd, (struct sockaddr*)&un_addr, &len);
+	if (rc < 0) {
+		print_error("Failed to accept a new connection\n");
+		return -1;
+	}
+
+	c = talloc_zero(globals.ctx, struct socket_client);
+	OSMO_ASSERT(c);
+	c->ofd.fd = rc;
+	c->ofd.when = BSC_FD_READ;
+	c->ofd.cb = socket_cb;
+	c->ofd.data = c;
+
+	if (osmo_fd_register(&c->ofd) != 0) {
+		print_error("Failed to register new connection fd\n");
+		close(c->ofd.fd);
+		c->ofd.fd = -1;
+		talloc_free(c);
+		return -1;
+	}
+
+	llist_add(&c->entry, &globals.socket_clients);
+
+	if (globals.format == FORMAT_CSV && cmdline_opts.csv_headers)
+		write(c->ofd.fd, CSV_HEADERS, strlen(CSV_HEADERS));
+
+	return 0;
+}
+
+int socket_init(const char *sock_path)
+{
+	struct osmo_fd *ofd = &globals.socket_ofd;
+	int rc;
+
+	ofd->fd = osmo_sock_unix_init(SOCK_SEQPACKET, 0, sock_path, OSMO_SOCK_F_BIND);
+	if (ofd->fd < 0) {
+		print_error("Could not create unix socket: %s: %s\n", sock_path, strerror(errno));
+		return -1;
+	}
+
+	ofd->when = BSC_FD_READ;
+	ofd->cb = socket_accept;
+
+	rc = osmo_fd_register(ofd);
+	if (rc < 0) {
+		print_error("Could not register listen fd: %d\n", rc);
+		close(ofd->fd);
+		return rc;
+	}
+	return 0;
+}
+
+void socket_close()
+{
+	struct socket_client *c, *n;
+	llist_for_each_entry_safe(c, n, &globals.socket_clients, entry)
+		socket_client_close(c);
+	if (osmo_fd_is_registered(&globals.socket_ofd)) {
+		close(globals.socket_ofd.fd);
+		globals.socket_ofd.fd = -1;
+		osmo_fd_unregister(&globals.socket_ofd);
+	}
+}
+
+struct query {
+	struct llist_head entry;
+
+	char query_str[128];
+	struct osmo_mslookup_query query;
+	uint32_t handle;
+};
+
+void respond_result(const char *query_str, const struct osmo_mslookup_result *r)
+{
+	struct socket_client *c, *n;
+	formatters[globals.format](g_buf, sizeof(g_buf), query_str, r);
+	respond_str_stdout(g_buf);
+
+	llist_for_each_entry_safe(c, n, &globals.socket_clients, entry) {
+		if (!strcmp(query_str, c->query_str)) {
+			socket_client_respond_result(c, g_buf);
+			if (r->last)
+				socket_client_close(c);
+		}
+	}
+	if (r->last)
+		globals.requests_handled++;
+}
+
+void respond_err(const char *query_str)
+{
+	respond_result(query_str, NULL);
+}
+
+struct query *query_by_handle(uint32_t request_handle)
+{
+	struct query *q;
+	llist_for_each_entry(q, &globals.queries, entry) {
+		if (request_handle == q->handle)
+			return q;
+	}
+	return NULL;
+}
+
+void mslookup_result_cb(struct osmo_mslookup_client *client,
+			uint32_t request_handle,
+			const struct osmo_mslookup_query *query,
+			const struct osmo_mslookup_result *result)
+{
+	struct query *q = query_by_handle(request_handle);
+	if (!q)
+		return;
+	respond_result(q->query_str, result);
+	if (result->last) {
+		llist_del(&q->entry);
+		talloc_free(q);
+	}
+}
+
+void start_query_str(const char *query_str)
+{
+	struct query *q;
+	const char *domain_str = query_str;
+	char *at;
+	struct osmo_mslookup_query_handling h = {
+		.min_wait_milliseconds = cmdline_opts.min_delay,
+		.result_timeout_milliseconds = cmdline_opts.timeout,
+		.result_cb = mslookup_result_cb,
+	};
+
+	at = strchr(query_str, '@');
+	if (at) {
+		int rc;
+		char timeouts[16];
+		char *dash;
+		char *timeout;
+
+		domain_str = at + 1;
+
+		h.min_wait_milliseconds = h.result_timeout_milliseconds = 0;
+
+		if (osmo_print_n(timeouts, sizeof(timeouts), query_str, at - query_str) >= sizeof(timeouts)) {
+			print_error("ERROR: timeouts part too long in query string\n");
+			respond_err(domain_str);
+			return;
+		}
+
+		dash = strchr(timeouts, '-');
+		if (dash) {
+			char min_delay[16];
+			osmo_print_n(min_delay, sizeof(min_delay), timeouts, dash - timeouts);
+			h.min_wait_milliseconds = parse_int(0, UINT32_MAX, min_delay, &rc);
+			if (rc) {
+				print_error("ERROR: invalid min-delay number: %s\n", min_delay);
+				respond_err(domain_str);
+				return;
+			}
+			timeout = dash + 1;
+		} else {
+			timeout = timeouts;
+		}
+		if (*timeout) {
+			h.result_timeout_milliseconds = parse_int(0, UINT32_MAX, timeout, &rc);
+			if (rc) {
+				print_error("ERROR: invalid timeout number: %s\n", timeout);
+				respond_err(domain_str);
+				return;
+			}
+		}
+	}
+
+	if (strlen(domain_str) >= sizeof(q->query_str)) {
+		print_error("ERROR: query string is too long: '%s'\n", domain_str);
+		respond_err(domain_str);
+		return;
+	}
+
+	q = talloc_zero(globals.ctx, struct query);
+	OSMO_ASSERT(q);
+	OSMO_STRLCPY_ARRAY(q->query_str, domain_str);
+
+	if (osmo_mslookup_query_init_from_domain_str(&q->query, q->query_str)) {
+		print_error("ERROR: cannot parse query string: '%s'\n", domain_str);
+		respond_err(domain_str);
+		talloc_free(q);
+		return;
+	}
+
+	q->handle = osmo_mslookup_client_request(globals.mslookup_client, &q->query, &h);
+	if (!q->handle) {
+		print_error("ERROR: cannot send query: '%s'\n", domain_str);
+		respond_err(domain_str);
+		talloc_free(q);
+		return;
+	}
+
+	llist_add(&q->entry, &globals.queries);
+}
+
+void start_query_strs(char **query_strs, size_t query_strs_len)
+{
+	int i;
+	for (i = 0; i < query_strs_len; i++)
+		start_query_str(query_strs[i]);
+}
+
+int main(int argc, char **argv)
+{
+	int rc = EXIT_FAILURE;
+	globals.ctx = talloc_named_const(NULL, 0, "osmo-mslookup-client");
+
+	osmo_init_logging2(globals.ctx, NULL);
+	log_set_print_filename2(osmo_stderr_target, LOG_FILENAME_BASENAME);
+	log_set_print_filename_pos(osmo_stderr_target, LOG_FILENAME_POS_LINE_END);
+	log_set_print_level(osmo_stderr_target, 1);
+	log_set_print_category(osmo_stderr_target, 1);
+	log_set_print_category_hex(osmo_stderr_target, 0);
+	log_set_print_extended_timestamp(osmo_stderr_target, 1);
+	log_set_use_color(osmo_stderr_target, 0);
+
+	while (1) {
+		int c;
+		long long int val;
+		char *endptr;
+		int option_index = 0;
+
+		static struct option long_options[] = {
+			{ "format", 1, 0, 'f' },
+			{ "no-csv-headers", 0, 0, 'H' },
+			{ "daemon", 0, 0, 'd' },
+			{ "mdns-ip", 1, 0, 'm' },
+			{ "mdns-port", 1, 0, 'M' },
+			{ "mdns-domain-suffix", 1, 0, 'D' },
+			{ "timeout", 1, 0, 'T' },
+			{ "min-delay", 1, 0, 't' },
+			{ "socket", 1, 0, 's' },
+			{ "send", 0, 0, 'S' },
+			{ "quiet", 0, 0, 'q' },
+			{ "help", 0, 0, 'h' },
+			{ "version", 0, 0, 'V' },
+			{}
+		};
+
+#define PARSE_INT(TARGET, MINVAL, MAXVAL) do { \
+		int _rc; \
+		TARGET = parse_int(MINVAL, MAXVAL, optarg, &_rc); \
+		if (_rc) { \
+			print_error("Invalid " #TARGET ": %s\n", optarg); \
+			goto program_exit; \
+		} \
+	} while (0)
+
+		c = getopt_long(argc, argv, "f:Hdm:M:D:t:T:s:SqhV", long_options, &option_index);
+
+		if (c == -1)
+			break;
+
+		switch (c) {
+		case 'f':
+			cmdline_opts.format_str = optarg;
+			break;
+		case 'H':
+			cmdline_opts.csv_headers = false;
+			break;
+		case 'd':
+			cmdline_opts.daemon = true;
+			break;
+		case 'm':
+			if (osmo_sockaddr_str_from_str(&cmdline_opts.mdns_addr, optarg, cmdline_opts.mdns_addr.port)
+			    || !osmo_sockaddr_str_is_nonzero(&cmdline_opts.mdns_addr)) {
+				print_error("Invalid mDNS IP address: %s\n", optarg);
+				goto program_exit;
+			}
+			break;
+		case 'M':
+			PARSE_INT(cmdline_opts.mdns_addr.port, 1, 65535);
+			break;
+		case 'D':
+			cmdline_opts.mdns_domain_suffix = optarg;
+			break;
+		case 't':
+			PARSE_INT(cmdline_opts.min_delay, 0, UINT32_MAX);
+			break;
+		case 'T':
+			PARSE_INT(cmdline_opts.timeout, 0, UINT32_MAX);
+			break;
+		case 's':
+			cmdline_opts.socket_path = optarg;
+			break;
+		case 'S':
+			cmdline_opts.send = true;
+			break;
+		case 'q':
+			cmdline_opts.quiet = true;
+			break;
+
+		case 'h':
+			print_help();
+			rc = 0;
+			goto program_exit;
+		case 'V':
+			print_version();
+			rc = 0;
+			goto program_exit;
+
+		default:
+			/* catch unknown options *as well as* missing arguments. */
+			print_error("Error in command line options. Exiting.\n");
+			goto program_exit;
+		}
+	}
+
+	if (cmdline_opts.send) {
+		if (cmdline_opts.daemon || cmdline_opts.format_str || cmdline_opts.socket_path) {
+			print_error("--send option cannot have any listening related args.");
+		}
+		rc = do_send(argc - optind, argv + optind);
+		goto program_exit;
+	}
+
+	if (!cmdline_opts.daemon && !(argc - optind)) {
+		print_help();
+		goto program_exit;
+	}
+
+	if (cmdline_opts.daemon && !cmdline_opts.timeout) {
+		print_error("In daemon mode, --timeout must not be zero.\n");
+		goto program_exit;
+	}
+
+	if (cmdline_opts.quiet)
+		log_target_destroy(osmo_stderr_target);
+
+	if (cmdline_opts.format_str) {
+		if (osmo_str_startswith("json", cmdline_opts.format_str))
+			globals.format = FORMAT_JSON;
+		else if (osmo_str_startswith("csv", cmdline_opts.format_str))
+			globals.format = FORMAT_CSV;
+		else {
+			print_error("Invalid format: %s\n", cmdline_opts.format_str);
+			goto program_exit;
+		}
+	}
+
+	if (globals.format == FORMAT_CSV && cmdline_opts.csv_headers)
+		respond_str_stdout(CSV_HEADERS);
+
+	globals.mslookup_client = osmo_mslookup_client_new(globals.ctx);
+	if (!globals.mslookup_client
+	    || !osmo_mslookup_client_add_mdns(globals.mslookup_client,
+					      cmdline_opts.mdns_addr.ip, cmdline_opts.mdns_addr.port,
+					      -1, cmdline_opts.mdns_domain_suffix)) {
+		print_error("Failed to start mDNS client\n");
+		goto program_exit;
+	}
+
+	if (cmdline_opts.socket_path) {
+		if (socket_init(cmdline_opts.socket_path))
+			goto program_exit;
+	}
+
+	start_query_strs(&argv[optind], argc - optind);
+
+	while (1) {
+		osmo_select_main_ctx(0);
+
+		if (!cmdline_opts.daemon
+		    && globals.requests_handled
+		    && llist_empty(&globals.queries))
+			break;
+	}
+
+	rc = 0;
+program_exit:
+	osmo_mslookup_client_free(globals.mslookup_client);
+	socket_close();
+	log_fini();
+	talloc_free(globals.ctx);
+	return rc;
+}

src/osmo-euse-demo.c

@@ -44,7 +44,7 @@
 
 #include <osmocom/gsupclient/gsup_client.h>
 
-#include "logging.h"
+#include <osmocom/hlr/logging.h>
 
 static struct osmo_gsup_client *g_gc;
 

tests/Makefile.am

@@ -2,7 +2,8 @@ SUBDIRS = \
 	auc \
 	gsup_server \
 	db \
-	gsup \
+	db_upgrade \
+	mslookup \
 	$(NULL)
 
 # The `:;' works around a Bash 3.2 bug when the output is not writeable.
@@ -44,24 +45,27 @@ python-tests:
 # don't run vty and ctrl tests concurrently so that the ports don't conflict
 	$(MAKE) vty-test
 	$(MAKE) ctrl-test
+	$(MAKE) db-upgrade-equivalence-test
 else
 python-tests:
 	echo "Not running python-based external tests (determined at configure-time)"
 endif
 
 VTY_TEST_DB = hlr_vty_test.db
+VTY_TEST ?= *.vty
 
 # To update the VTY script from current application behavior,
 # pass -u to vty_script_runner.py by doing:
 #   make vty-test U=-u
 vty-test:
 	-rm -f $(VTY_TEST_DB)
+	sqlite3 $(VTY_TEST_DB) < $(top_srcdir)/sql/hlr.sql
+	sqlite3 $(VTY_TEST_DB) < $(srcdir)/test_subscriber.vty.sql
 	osmo_verify_transcript_vty.py -v \
 		-n OsmoHLR -p 4258 \
 		-r "$(top_builddir)/src/osmo-hlr -c $(top_srcdir)/doc/examples/osmo-hlr.cfg -l $(VTY_TEST_DB)" \
-		$(U) $(srcdir)/*.vty
-	-rm -f $(VTY_TEST_DB)
-	-rm $(VTY_TEST_DB)-*
+		$(U) $(srcdir)/$(VTY_TEST)
+	-rm -f $(VTY_TEST_DB) $(VTY_TEST_DB)-*
 
 CTRL_TEST_DB = hlr_ctrl_test.db
 
@@ -79,6 +83,9 @@ ctrl-test:
 	-rm -f $(CTRL_TEST_DB)
 	-rm $(CTRL_TEST_DB)-*
 
+db-upgrade-equivalence-test:
+	$(MAKE) -C db_upgrade upgrade-equivalence-test
+
 check-local: atconfig $(TESTSUITE)
 	$(SHELL) '$(TESTSUITE)' $(TESTSUITEFLAGS)
 	$(MAKE) $(AM_MAKEFLAGS) python-tests

tests/auc/Makefile.am

@@ -2,17 +2,18 @@ SUBDIRS = gen_ts_55_205_test_sets
 
 AM_CPPFLAGS = \
 	$(all_includes) \
-	-I$(top_srcdir)/src \
 	$(NULL)
 
 AM_CFLAGS = \
 	-Wall \
 	-ggdb3 \
+	-I$(top_srcdir)/include \
 	$(LIBOSMOCORE_CFLAGS) \
 	$(LIBOSMOGSM_CFLAGS) \
 	$(NULL)
 
 AM_LDFLAGS = \
+	-no-install \
 	$(NULL)
 
 EXTRA_DIST = \

tests/auc/auc_test.c

@@ -29,8 +29,8 @@
 
 #include <osmocom/crypt/auth.h>
 
-#include "logging.h"
-#include "auc.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/auc.h>
 
 #define comment_start() fprintf(stderr, "\n===== %s\n", __func__);
 #define comment_end() fprintf(stderr, "===== %s: SUCCESS\n\n", __func__);

tests/auc/gen_ts_55_205_test_sets/main_template.c

@@ -33,8 +33,8 @@
 
 #include <osmocom/crypt/auth.h>
 
-#include "logging.h"
-#include "auc.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/auc.h>
 
 #define comment_start() fprintf(stderr, "\n===== %s\n", __func__);
 #define comment_end() fprintf(stderr, "===== %s: SUCCESS\n\n", __func__);

tests/auc/gen_ts_55_205_test_sets/pdftxt_2_c.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python
-# FIXME: use python3 once buildslaves are updated.
+#!/usr/bin/env python3
 # Convert test sets pasted from 3GPP TS 55.205 to C code.
  
 # (C) 2016 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>

tests/db/Makefile.am

@@ -1,7 +1,7 @@
 AM_CFLAGS = \
 	$(all_includes) \
-	-I$(top_srcdir)/src \
-	-I$(top_builddir)/src \
+	-I$(top_srcdir)/include \
+	-I$(top_builddir)/include \
 	-Wall \
 	-ggdb3 \
 	$(LIBOSMOCORE_CFLAGS) \
@@ -10,6 +10,10 @@ AM_CFLAGS = \
 	$(SQLITE3_CFLAGS) \
 	$(NULL)
 
+AM_LDFLAGS = \
+	-no-install \
+	$(NULL)
+
 EXTRA_DIST = \
 	db_test.ok \
 	db_test.err \
@@ -22,16 +26,20 @@ db_test_SOURCES = \
 	$(NULL)
 
 db_test_LDADD = \
-	$(top_srcdir)/src/db.c \
-	$(top_srcdir)/src/db_hlr.c \
-	$(top_srcdir)/src/db_auc.c \
-	$(top_srcdir)/src/logging.c \
+	$(top_builddir)/src/logging.o \
+	$(top_builddir)/src/db_auc.o \
+	$(top_builddir)/src/db_hlr.o \
+	$(top_builddir)/src/db.o \
 	$(LIBOSMOCORE_LIBS) \
 	$(LIBOSMOGSM_LIBS) \
 	$(LIBOSMOABIS_LIBS) \
 	$(SQLITE3_LIBS) \
 	$(NULL)
 
+if DB_SQLITE_DEBUG
+db_test_LDADD += $(top_builddir)/src/db_debug.o
+endif
+
 .PHONY: db_test.db update_exp manual manual-nonverbose manual-gdb
 db_test.db:
 	rm -f db_test.db

tests/db/db_test.c

@@ -27,8 +27,8 @@
 #include <osmocom/core/utils.h>
 #include <osmocom/core/logging.h>
 
-#include "db.h"
-#include "logging.h"
+#include <osmocom/hlr/db.h>
+#include <osmocom/hlr/logging.h>
 
 #define comment_start() fprintf(stderr, "\n===== %s\n", __func__);
 #define comment(fmt, args...) fprintf(stderr, "\n--- " fmt "\n\n", ## args);
@@ -51,7 +51,12 @@ static void _fill_invalid(void *dest, size_t size)
  * The return code is then available in g_rc. */
 #define ASSERT_RC(call, expect_rc) \
 	do { \
-		fprintf(stderr, #call " --> " #expect_rc "\n"); \
+    if ((expect_rc) == -ENOKEY) \
+		  fprintf(stderr, #call " --> -ENOKEY\n"); \
+    else if ((expect_rc) == -ENOTSUP) \
+		  fprintf(stderr, #call " --> -ENOTSUP\n"); \
+    else \
+		  fprintf(stderr, #call " --> " #expect_rc "\n"); \
 		g_rc = call; \
 		if (g_rc != (expect_rc)) \
 			fprintf(stderr, " MISMATCH: got rc = %d, expected: " \
@@ -67,7 +72,12 @@ static void _fill_invalid(void *dest, size_t size)
 	do { \
 		int rc; \
 		fill_invalid(g_subscr); \
-		fprintf(stderr, "db_subscr_get_by_" #by "(dbc, " #val ", &g_subscr) --> " \
+    if ((expect_rc) == -ENOKEY) \
+		  fprintf(stderr, "db_subscr_get_by_" #by "(dbc, " #val ", &g_subscr) --> -ENOKEY \n"); \
+    else if ((expect_rc) == -ENOTSUP) \
+		  fprintf(stderr, "db_subscr_get_by_" #by "(dbc, " #val ", &g_subscr) --> -ENOTSUP \n"); \
+    else \
+		  fprintf(stderr, "db_subscr_get_by_" #by "(dbc, " #val ", &g_subscr) --> " \
                                 #expect_rc "\n"); \
 		rc = db_subscr_get_by_##by(dbc, val, &g_subscr); \
 		if (rc != (expect_rc)) \
@@ -105,7 +115,7 @@ static void _fill_invalid(void *dest, size_t size)
 #define ASSERT_DB_GET_AUC(imsi, expect_rc) \
 	do { \
 		struct osmo_auth_vector vec[N_VECTORS]; \
-		ASSERT_RC(db_get_auc(dbc, imsi, 3, vec, N_VECTORS, NULL, NULL), expect_rc); \
+		ASSERT_RC(db_get_auc(dbc, imsi, 3, vec, N_VECTORS, NULL, NULL, false), expect_rc); \
 	} while (0)
 
 /* Not linking the real auc_compute_vectors(), just returning num_vec.
@@ -148,6 +158,7 @@ void dump_subscr(struct hlr_subscriber *subscr)
 	Pd(id);
 	Ps(imsi);
 	Ps(msisdn);
+	Ps(imei);
 	Ps(vlr_number);
 	Ps(sgsn_number);
 	Ps(sgsn_address);
@@ -207,6 +218,17 @@ void dump_aud(const char *label, struct osmo_sub_auth_data *aud)
 #undef Phex
 }
 
+void db_raw_sql(struct db_context *dbc, const char *sql)
+{
+	sqlite3_stmt *stmt;
+
+	fprintf(stderr, "raw SQL: %s\n", sql);
+	ASSERT_RC(sqlite3_prepare_v2(dbc->db, sql, -1, &stmt, NULL), SQLITE_OK);
+	ASSERT_RC(sqlite3_step(stmt), SQLITE_DONE);
+	db_remove_reset(stmt);
+	sqlite3_finalize(stmt);
+}
+
 static const char *imsi0 = "123456789000000";
 static const char *imsi1 = "123456789000001";
 static const char *imsi2 = "123456789000002";
@@ -220,40 +242,45 @@ static void test_subscr_create_update_sel_delete()
 
 	comment("Create with valid / invalid IMSI");
 
-	ASSERT_RC(db_subscr_create(dbc, imsi0), 0);
+	ASSERT_RC(db_subscr_create(dbc, imsi0, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), 0);
 	ASSERT_SEL(imsi, imsi0, 0);
 	id0 = g_subscr.id;
-	ASSERT_RC(db_subscr_create(dbc, imsi1), 0);
+	ASSERT_RC(db_subscr_create(dbc, imsi1, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), 0);
 	ASSERT_SEL(imsi, imsi1, 0);
 	id1 = g_subscr.id;
-	ASSERT_RC(db_subscr_create(dbc, imsi2), 0);
+	ASSERT_RC(db_subscr_create(dbc, imsi2, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), 0);
 	ASSERT_SEL(imsi, imsi2, 0);
 	id2 = g_subscr.id;
-	ASSERT_RC(db_subscr_create(dbc, imsi0), -EIO);
+	ASSERT_RC(db_subscr_create(dbc, imsi0, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), -EIO);
 	ASSERT_SEL(imsi, imsi0, 0);
-	ASSERT_RC(db_subscr_create(dbc, imsi1), -EIO);
-	ASSERT_RC(db_subscr_create(dbc, imsi1), -EIO);
+	ASSERT_RC(db_subscr_create(dbc, imsi1, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), -EIO);
+	ASSERT_RC(db_subscr_create(dbc, imsi1, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), -EIO);
 	ASSERT_SEL(imsi, imsi1, 0);
-	ASSERT_RC(db_subscr_create(dbc, imsi2), -EIO);
-	ASSERT_RC(db_subscr_create(dbc, imsi2), -EIO);
+	ASSERT_RC(db_subscr_create(dbc, imsi2, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), -EIO);
+	ASSERT_RC(db_subscr_create(dbc, imsi2, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), -EIO);
 	ASSERT_SEL(imsi, imsi2, 0);
 
-	ASSERT_RC(db_subscr_create(dbc, "123456789 000003"), -EINVAL);
+	ASSERT_RC(db_subscr_create(dbc, "123456789 000003", DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), -EINVAL);
 	ASSERT_SEL(imsi, "123456789000003", -ENOENT);
 
-	ASSERT_RC(db_subscr_create(dbc, "123456789000002123456"), -EINVAL);
+	ASSERT_RC(db_subscr_create(dbc, "123456789000002123456", DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS),
+		  -EINVAL);
 	ASSERT_SEL(imsi, "123456789000002123456", -ENOENT);
 
-	ASSERT_RC(db_subscr_create(dbc, "foobar123"), -EINVAL);
+	ASSERT_RC(db_subscr_create(dbc, "foobar123", DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), -EINVAL);
 	ASSERT_SEL(imsi, "foobar123", -ENOENT);
 
-	ASSERT_RC(db_subscr_create(dbc, "123"), -EINVAL);
+	ASSERT_RC(db_subscr_create(dbc, "123", DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), -EINVAL);
 	ASSERT_SEL(imsi, "123", -ENOENT);
 
-	ASSERT_RC(db_subscr_create(dbc, short_imsi), 0);
+	ASSERT_RC(db_subscr_create(dbc, short_imsi, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), 0);
 	ASSERT_SEL(imsi, short_imsi, 0);
 	id_short = g_subscr.id;
 
+	comment("Check if subscriber exists (by IMSI)");
+
+	ASSERT_RC(db_subscr_exists_by_imsi(dbc, imsi0), 0);
+	ASSERT_RC(db_subscr_exists_by_imsi(dbc, unknown_imsi), -ENOENT);
 
 	comment("Set valid / invalid MSISDN");
 
@@ -288,6 +315,11 @@ static void test_subscr_create_update_sel_delete()
 	ASSERT_SEL(imsi, imsi0, 0);
 	ASSERT_SEL(msisdn, "5432101234567891", -ENOENT);
 
+	comment("Check if subscriber exists (by MSISDN)");
+
+	ASSERT_RC(db_subscr_exists_by_msisdn(dbc, "543210123456789"), 0);
+	ASSERT_RC(db_subscr_exists_by_msisdn(dbc, "5432101234567891"), -ENOENT);
+
 	comment("Set MSISDN on non-existent / invalid IMSI");
 
 	ASSERT_RC(db_subscr_update_msisdn_by_imsi(dbc, unknown_imsi, "99"), -ENOENT);
@@ -296,6 +328,23 @@ static void test_subscr_create_update_sel_delete()
 	ASSERT_RC(db_subscr_update_msisdn_by_imsi(dbc, "foobar", "99"), -ENOENT);
 	ASSERT_SEL(msisdn, "99", -ENOENT);
 
+	comment("Set valid / invalid IMEI");
+
+	ASSERT_RC(db_subscr_update_imei_by_imsi(dbc, imsi0, "12345678901234"), 0);
+	ASSERT_SEL(imei, "12345678901234", 0);
+
+	ASSERT_RC(db_subscr_update_imei_by_imsi(dbc, imsi0, "123456789012345"), -EINVAL); /* too long */
+	ASSERT_SEL(imei, "12345678901234", 0);
+	ASSERT_SEL(imei, "123456789012345", -ENOENT);
+
+	comment("Set the same IMEI again");
+	ASSERT_RC(db_subscr_update_imei_by_imsi(dbc, imsi0, "12345678901234"), 0);
+	ASSERT_SEL(imei, "12345678901234", 0);
+
+	comment("Remove IMEI");
+	ASSERT_RC(db_subscr_update_imei_by_imsi(dbc, imsi0, NULL), 0);
+	ASSERT_SEL(imei, "12345678901234", -ENOENT);
+
 	comment("Set / unset nam_cs and nam_ps");
 
 	/*                                nam_val, is_ps */
@@ -434,6 +483,22 @@ static void test_subscr_create_update_sel_delete()
 	ASSERT_RC(db_subscr_delete_by_id(dbc, id_short), 0);
 	ASSERT_SEL(imsi, short_imsi, -ENOENT);
 
+	comment("Create and delete subscribers with non-default nam_cs and nam_ps");
+
+	ASSERT_RC(db_subscr_create(dbc, imsi0, 0x00), 0);
+	ASSERT_SEL(imsi, imsi0, 0);
+	id0 = g_subscr.id;
+	ASSERT_RC(db_subscr_create(dbc, imsi1, DB_SUBSCR_FLAG_NAM_CS), 0);
+	ASSERT_SEL(imsi, imsi1, 0);
+	id1 = g_subscr.id;
+	ASSERT_RC(db_subscr_create(dbc, imsi2, DB_SUBSCR_FLAG_NAM_PS), 0);
+	ASSERT_SEL(imsi, imsi2, 0);
+	id2 = g_subscr.id;
+
+	ASSERT_RC(db_subscr_delete_by_id(dbc, id0), 0);
+	ASSERT_RC(db_subscr_delete_by_id(dbc, id1), 0);
+	ASSERT_RC(db_subscr_delete_by_id(dbc, id2), 0);
+
 	comment_end();
 }
 
@@ -477,7 +542,7 @@ static void test_subscr_aud()
 
 	comment("Create subscriber");
 
-	ASSERT_RC(db_subscr_create(dbc, imsi0), 0);
+	ASSERT_RC(db_subscr_create(dbc, imsi0, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), 0);
 	ASSERT_SEL(imsi, imsi0, 0);
 
 	id = g_subscr.id;
@@ -689,7 +754,7 @@ static void test_subscr_aud()
 
 	comment("Re-add subscriber and verify auth data didn't come back");
 
-	ASSERT_RC(db_subscr_create(dbc, imsi0), 0);
+	ASSERT_RC(db_subscr_create(dbc, imsi0, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), 0);
 	ASSERT_SEL(imsi, imsi0, 0);
 
 	/* For this test to work, we want to get the same subscriber ID back,
@@ -705,6 +770,70 @@ static void test_subscr_aud()
 	comment_end();
 }
 
+/* Make each key too short in this test. Note that we can't set them longer than the allowed size without changing the
+ * table structure. */
+static void test_subscr_aud_invalid_len()
+{
+	int64_t id;
+
+	comment_start();
+	comment("Create subscriber");
+	ASSERT_RC(db_subscr_create(dbc, imsi0, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), 0);
+	ASSERT_SEL(imsi, imsi0, 0);
+	id = g_subscr.id;
+
+
+	/* Invalid Ki length */
+	comment("Set auth data, 2G only, with invalid Ki length");
+	ASSERT_RC(db_subscr_update_aud_by_id(dbc, id,
+		  mk_aud_2g(OSMO_AUTH_ALG_COMP128v1, "0123456789abcdef0123456789abcdef")),
+		  0);
+	/* Use raw SQL to avoid length check in db_subscr_update_aud_by_id(). This changes all rows in the table, which
+         * is fine for this test (implicit WHERE 1). */
+	db_raw_sql(dbc, "UPDATE auc_2g SET ki = '0123456789abcdef0123456789abcde'");
+	ASSERT_SEL_AUD(imsi0, -ENOKEY, id);
+
+	comment("Remove 2G auth data");
+	ASSERT_RC(db_subscr_update_aud_by_id(dbc, id,
+		  mk_aud_2g(OSMO_AUTH_ALG_NONE, NULL)),
+		  0);
+
+	/* Invalid K length */
+	comment("Set auth data, 3G only, with invalid K length");
+	ASSERT_RC(db_subscr_update_aud_by_id(dbc, id,
+		mk_aud_3g(OSMO_AUTH_ALG_MILENAGE,
+			  "BeefedCafeFaceAcedAddedDecadeFee", true,
+			  "C01ffedC1cadaeAc1d1f1edAcac1aB0a", 5)),
+		0);
+	db_raw_sql(dbc, "UPDATE auc_3g SET k = 'C01ffedC1cadaeAc1d1f1edAcac1aB0'");
+	ASSERT_SEL_AUD(imsi0, -EIO, id);
+
+	/* Invalid OP length */
+	comment("Set auth data, 3G only, with invalid OP length");
+	ASSERT_RC(db_subscr_update_aud_by_id(dbc, id,
+		mk_aud_3g(OSMO_AUTH_ALG_MILENAGE,
+			  "BeefedCafeFaceAcedAddedDecadeFee", true,
+			  "C01ffedC1cadaeAc1d1f1edAcac1aB0a", 5)),
+		0);
+	db_raw_sql(dbc, "UPDATE auc_3g SET op = 'BeefedCafeFaceAcedAddedDecadeFe'");
+	ASSERT_SEL_AUD(imsi0, -EIO, id);
+
+	/* Invalid OPC length */
+	comment("Set auth data, 3G only, with invalid OPC length");
+	ASSERT_RC(db_subscr_update_aud_by_id(dbc, id,
+		mk_aud_3g(OSMO_AUTH_ALG_MILENAGE,
+			  "BeefedCafeFaceAcedAddedDecadeFee", false,
+			  "C01ffedC1cadaeAc1d1f1edAcac1aB0a", 5)),
+		0);
+	db_raw_sql(dbc, "UPDATE auc_3g SET opc = 'BeefedCafeFaceAcedAddedDecadeFe'");
+	ASSERT_SEL_AUD(imsi0, -EIO, id);
+
+
+	comment("Delete subscriber");
+	ASSERT_RC(db_subscr_delete_by_id(dbc, id), 0);
+	comment_end();
+}
+
 static void test_subscr_sqn()
 {
 	int64_t id;
@@ -721,7 +850,7 @@ static void test_subscr_sqn()
 
 	comment("Create subscriber");
 
-	ASSERT_RC(db_subscr_create(dbc, imsi0), 0);
+	ASSERT_RC(db_subscr_create(dbc, imsi0, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), 0);
 	ASSERT_SEL(imsi, imsi0, 0);
 
 	id = g_subscr.id;
@@ -856,9 +985,11 @@ int main(int argc, char **argv)
 
 	test_subscr_create_update_sel_delete();
 	test_subscr_aud();
+	test_subscr_aud_invalid_len();
 	test_subscr_sqn();
 
 	printf("Done\n");
+	db_close(dbc);
 	return 0;
 }
 

tests/db/db_test.err

@@ -3,7 +3,7 @@
 
 --- Create with valid / invalid IMSI
 
-db_subscr_create(dbc, imsi0) --> 0
+db_subscr_create(dbc, imsi0, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> 0
 
 db_subscr_get_by_imsi(dbc, imsi0, &g_subscr) --> 0
 struct hlr_subscriber {
@@ -11,7 +11,7 @@ struct hlr_subscriber {
   .imsi = '123456789000000',
 }
 
-db_subscr_create(dbc, imsi1) --> 0
+db_subscr_create(dbc, imsi1, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> 0
 
 db_subscr_get_by_imsi(dbc, imsi1, &g_subscr) --> 0
 struct hlr_subscriber {
@@ -19,7 +19,7 @@ struct hlr_subscriber {
   .imsi = '123456789000001',
 }
 
-db_subscr_create(dbc, imsi2) --> 0
+db_subscr_create(dbc, imsi2, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> 0
 
 db_subscr_get_by_imsi(dbc, imsi2, &g_subscr) --> 0
 struct hlr_subscriber {
@@ -27,7 +27,7 @@ struct hlr_subscriber {
   .imsi = '123456789000002',
 }
 
-db_subscr_create(dbc, imsi0) --> -EIO
+db_subscr_create(dbc, imsi0, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> -EIO
 DAUC IMSI='123456789000000': Cannot create subscriber: SQL error: (2067) UNIQUE constraint failed: subscriber.imsi
 
 db_subscr_get_by_imsi(dbc, imsi0, &g_subscr) --> 0
@@ -36,10 +36,10 @@ struct hlr_subscriber {
   .imsi = '123456789000000',
 }
 
-db_subscr_create(dbc, imsi1) --> -EIO
+db_subscr_create(dbc, imsi1, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> -EIO
 DAUC IMSI='123456789000001': Cannot create subscriber: SQL error: (2067) UNIQUE constraint failed: subscriber.imsi
 
-db_subscr_create(dbc, imsi1) --> -EIO
+db_subscr_create(dbc, imsi1, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> -EIO
 DAUC IMSI='123456789000001': Cannot create subscriber: SQL error: (2067) UNIQUE constraint failed: subscriber.imsi
 
 db_subscr_get_by_imsi(dbc, imsi1, &g_subscr) --> 0
@@ -48,10 +48,10 @@ struct hlr_subscriber {
   .imsi = '123456789000001',
 }
 
-db_subscr_create(dbc, imsi2) --> -EIO
+db_subscr_create(dbc, imsi2, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> -EIO
 DAUC IMSI='123456789000002': Cannot create subscriber: SQL error: (2067) UNIQUE constraint failed: subscriber.imsi
 
-db_subscr_create(dbc, imsi2) --> -EIO
+db_subscr_create(dbc, imsi2, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> -EIO
 DAUC IMSI='123456789000002': Cannot create subscriber: SQL error: (2067) UNIQUE constraint failed: subscriber.imsi
 
 db_subscr_get_by_imsi(dbc, imsi2, &g_subscr) --> 0
@@ -60,31 +60,31 @@ struct hlr_subscriber {
   .imsi = '123456789000002',
 }
 
-db_subscr_create(dbc, "123456789 000003") --> -EINVAL
+db_subscr_create(dbc, "123456789 000003", DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> -EINVAL
 DAUC Cannot create subscriber: invalid IMSI: '123456789 000003'
 
 db_subscr_get_by_imsi(dbc, "123456789000003", &g_subscr) --> -ENOENT
 DAUC Cannot read subscriber from db: IMSI='123456789000003': No such subscriber
 
-db_subscr_create(dbc, "123456789000002123456") --> -EINVAL
+db_subscr_create(dbc, "123456789000002123456", DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> -EINVAL
 DAUC Cannot create subscriber: invalid IMSI: '123456789000002123456'
 
 db_subscr_get_by_imsi(dbc, "123456789000002123456", &g_subscr) --> -ENOENT
 DAUC Cannot read subscriber from db: IMSI='123456789000002123456': No such subscriber
 
-db_subscr_create(dbc, "foobar123") --> -EINVAL
+db_subscr_create(dbc, "foobar123", DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> -EINVAL
 DAUC Cannot create subscriber: invalid IMSI: 'foobar123'
 
 db_subscr_get_by_imsi(dbc, "foobar123", &g_subscr) --> -ENOENT
 DAUC Cannot read subscriber from db: IMSI='foobar123': No such subscriber
 
-db_subscr_create(dbc, "123") --> -EINVAL
+db_subscr_create(dbc, "123", DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> -EINVAL
 DAUC Cannot create subscriber: invalid IMSI: '123'
 
 db_subscr_get_by_imsi(dbc, "123", &g_subscr) --> -ENOENT
 DAUC Cannot read subscriber from db: IMSI='123': No such subscriber
 
-db_subscr_create(dbc, short_imsi) --> 0
+db_subscr_create(dbc, short_imsi, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> 0
 
 db_subscr_get_by_imsi(dbc, short_imsi, &g_subscr) --> 0
 struct hlr_subscriber {
@@ -93,6 +93,13 @@ struct hlr_subscriber {
 }
 
 
+--- Check if subscriber exists (by IMSI)
+
+db_subscr_exists_by_imsi(dbc, imsi0) --> 0
+
+db_subscr_exists_by_imsi(dbc, unknown_imsi) --> -ENOENT
+
+
 --- Set valid / invalid MSISDN
 
 db_subscr_get_by_imsi(dbc, imsi0, &g_subscr) --> 0
@@ -212,6 +219,13 @@ db_subscr_get_by_msisdn(dbc, "5432101234567891", &g_subscr) --> -ENOENT
 DAUC Cannot read subscriber from db: MSISDN='5432101234567891': No such subscriber
 
 
+--- Check if subscriber exists (by MSISDN)
+
+db_subscr_exists_by_msisdn(dbc, "543210123456789") --> 0
+
+db_subscr_exists_by_msisdn(dbc, "5432101234567891") --> -ENOENT
+
+
 --- Set MSISDN on non-existent / invalid IMSI
 
 db_subscr_update_msisdn_by_imsi(dbc, unknown_imsi, "99") --> -ENOENT
@@ -227,6 +241,54 @@ db_subscr_get_by_msisdn(dbc, "99", &g_subscr) --> -ENOENT
 DAUC Cannot read subscriber from db: MSISDN='99': No such subscriber
 
 
+--- Set valid / invalid IMEI
+
+db_subscr_update_imei_by_imsi(dbc, imsi0, "12345678901234") --> 0
+
+db_subscr_get_by_imei(dbc, "12345678901234", &g_subscr) --> 0
+struct hlr_subscriber {
+  .id = 1,
+  .imsi = '123456789000000',
+  .msisdn = '543210123456789',
+  .imei = '12345678901234',
+}
+
+db_subscr_update_imei_by_imsi(dbc, imsi0, "123456789012345") --> -EINVAL
+DAUC Cannot update subscriber IMSI='123456789000000': invalid IMEI: '123456789012345'
+
+db_subscr_get_by_imei(dbc, "12345678901234", &g_subscr) --> 0
+struct hlr_subscriber {
+  .id = 1,
+  .imsi = '123456789000000',
+  .msisdn = '543210123456789',
+  .imei = '12345678901234',
+}
+
+db_subscr_get_by_imei(dbc, "123456789012345", &g_subscr) --> -ENOENT
+DAUC Cannot read subscriber from db: IMEI=123456789012345: No such subscriber
+
+
+--- Set the same IMEI again
+
+db_subscr_update_imei_by_imsi(dbc, imsi0, "12345678901234") --> 0
+
+db_subscr_get_by_imei(dbc, "12345678901234", &g_subscr) --> 0
+struct hlr_subscriber {
+  .id = 1,
+  .imsi = '123456789000000',
+  .msisdn = '543210123456789',
+  .imei = '12345678901234',
+}
+
+
+--- Remove IMEI
+
+db_subscr_update_imei_by_imsi(dbc, imsi0, NULL) --> 0
+
+db_subscr_get_by_imei(dbc, "12345678901234", &g_subscr) --> -ENOENT
+DAUC Cannot read subscriber from db: IMEI=12345678901234: No such subscriber
+
+
 --- Set / unset nam_cs and nam_ps
 
 db_subscr_nam(dbc, imsi0, false, true) --> 0
@@ -704,6 +766,43 @@ db_subscr_delete_by_id(dbc, id_short) --> 0
 db_subscr_get_by_imsi(dbc, short_imsi, &g_subscr) --> -ENOENT
 DAUC Cannot read subscriber from db: IMSI='123456': No such subscriber
 
+
+--- Create and delete subscribers with non-default nam_cs and nam_ps
+
+db_subscr_create(dbc, imsi0, 0x00) --> 0
+
+db_subscr_get_by_imsi(dbc, imsi0, &g_subscr) --> 0
+struct hlr_subscriber {
+  .id = 1,
+  .imsi = '123456789000000',
+  .nam_cs = false,
+  .nam_ps = false,
+}
+
+db_subscr_create(dbc, imsi1, DB_SUBSCR_FLAG_NAM_CS) --> 0
+
+db_subscr_get_by_imsi(dbc, imsi1, &g_subscr) --> 0
+struct hlr_subscriber {
+  .id = 2,
+  .imsi = '123456789000001',
+  .nam_ps = false,
+}
+
+db_subscr_create(dbc, imsi2, DB_SUBSCR_FLAG_NAM_PS) --> 0
+
+db_subscr_get_by_imsi(dbc, imsi2, &g_subscr) --> 0
+struct hlr_subscriber {
+  .id = 3,
+  .imsi = '123456789000002',
+  .nam_cs = false,
+}
+
+db_subscr_delete_by_id(dbc, id0) --> 0
+
+db_subscr_delete_by_id(dbc, id1) --> 0
+
+db_subscr_delete_by_id(dbc, id2) --> 0
+
 ===== test_subscr_create_update_sel_delete: SUCCESS
 
 
@@ -715,13 +814,13 @@ db_get_auth_data(dbc, unknown_imsi, &g_aud2g, &g_aud3g, &g_id) --> -2
 DAUC IMSI='999999999': No such subscriber
 
 
-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> -2
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> -2
 DAUC IMSI='123456789000000': No such subscriber
 
 
 --- Create subscriber
 
-db_subscr_create(dbc, imsi0) --> 0
+db_subscr_create(dbc, imsi0, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> 0
 
 db_subscr_get_by_imsi(dbc, imsi0, &g_subscr) --> 0
 struct hlr_subscriber {
@@ -729,12 +828,12 @@ struct hlr_subscriber {
   .imsi = '123456789000000',
 }
 
-db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -126
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 
 
-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> -126
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 
@@ -753,7 +852,7 @@ DAUC IMSI='123456789000000': No 3G Auth Data
 }
 3G: none
 
-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> 3
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> 3
 DAUC IMSI='123456789000000': No 3G Auth Data
 DAUC IMSI='123456789000000': Calling to generate 3 vectors
 DAUC IMSI='123456789000000': Generated 3 vectors
@@ -811,12 +910,12 @@ DAUC IMSI='123456789000000': No 3G Auth Data
 
 db_subscr_update_aud_by_id(dbc, id, mk_aud_2g(OSMO_AUTH_ALG_NONE, NULL)) --> 0
 
-db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -126
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 
 
-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> -126
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 
@@ -836,12 +935,12 @@ DAUC IMSI='123456789000000': No 3G Auth Data
 
 db_subscr_update_aud_by_id(dbc, id, mk_aud_2g(OSMO_AUTH_ALG_NONE, "f000000000000f00000000000f000000")) --> 0
 
-db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -126
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 
 
-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> -126
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 
@@ -864,7 +963,7 @@ DAUC IMSI='123456789000000': No 2G Auth Data
   .u.umts.ind_bitlen = 5,
 }
 
-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> 3
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> 3
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': Calling to generate 3 vectors
 DAUC IMSI='123456789000000': Generated 3 vectors
@@ -938,12 +1037,12 @@ DAUC IMSI='123456789000000': No 2G Auth Data
 
 db_subscr_update_aud_by_id(dbc, id, mk_aud_3g(OSMO_AUTH_ALG_NONE, NULL, false, NULL, 0)) --> 0
 
-db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -126
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 
 
-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> -126
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 
@@ -965,7 +1064,7 @@ DAUC IMSI='123456789000000': No 2G Auth Data
   .u.umts.ind_bitlen = 5,
 }
 
-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> 3
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> 3
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': Calling to generate 3 vectors
 DAUC IMSI='123456789000000': Generated 3 vectors
@@ -973,12 +1072,12 @@ DAUC IMSI='123456789000000': Updating SQN=0 in DB
 
 db_subscr_update_aud_by_id(dbc, id, mk_aud_3g(OSMO_AUTH_ALG_NONE, "asdfasdfasd", false, "asdfasdfasdf", 99999)) --> 0
 
-db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -126
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 
 
-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> -126
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 
@@ -1006,7 +1105,7 @@ db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> 0
   .u.umts.ind_bitlen = 5,
 }
 
-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> 3
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> 3
 DAUC IMSI='123456789000000': Calling to generate 3 vectors
 DAUC IMSI='123456789000000': Generated 3 vectors
 DAUC IMSI='123456789000000': Updating SQN=0 in DB
@@ -1211,7 +1310,7 @@ DAUC Cannot read subscriber from db: IMSI='123456789000000': No such subscriber
 
 --- Re-add subscriber and verify auth data didn't come back
 
-db_subscr_create(dbc, imsi0) --> 0
+db_subscr_create(dbc, imsi0, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> 0
 
 db_subscr_get_by_imsi(dbc, imsi0, &g_subscr) --> 0
 struct hlr_subscriber {
@@ -1219,12 +1318,12 @@ struct hlr_subscriber {
   .imsi = '123456789000000',
 }
 
-db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -126
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 
 
-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> -126
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 
@@ -1233,12 +1332,97 @@ db_subscr_delete_by_id(dbc, id) --> 0
 db_subscr_get_by_imsi(dbc, imsi0, &g_subscr) --> -ENOENT
 DAUC Cannot read subscriber from db: IMSI='123456789000000': No such subscriber
 
-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> -2
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> -2
 DAUC IMSI='123456789000000': No such subscriber
 
 ===== test_subscr_aud: SUCCESS
 
 
+===== test_subscr_aud_invalid_len
+
+--- Create subscriber
+
+db_subscr_create(dbc, imsi0, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> 0
+
+db_subscr_get_by_imsi(dbc, imsi0, &g_subscr) --> 0
+struct hlr_subscriber {
+  .id = 1,
+  .imsi = '123456789000000',
+}
+
+
+--- Set auth data, 2G only, with invalid Ki length
+
+db_subscr_update_aud_by_id(dbc, id, mk_aud_2g(OSMO_AUTH_ALG_COMP128v1, "0123456789abcdef0123456789abcdef")) --> 0
+
+raw SQL: UPDATE auc_2g SET ki = '0123456789abcdef0123456789abcde'
+sqlite3_prepare_v2(dbc->db, sql, -1, &stmt, NULL) --> SQLITE_OK
+
+sqlite3_step(stmt) --> SQLITE_DONE
+
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
+DAUC IMSI='123456789000000': Error reading Ki, expected length 16 but has length 15
+DAUC IMSI='123456789000000': No 3G Auth Data
+
+
+
+--- Remove 2G auth data
+
+db_subscr_update_aud_by_id(dbc, id, mk_aud_2g(OSMO_AUTH_ALG_NONE, NULL)) --> 0
+
+
+--- Set auth data, 3G only, with invalid K length
+
+db_subscr_update_aud_by_id(dbc, id, mk_aud_3g(OSMO_AUTH_ALG_MILENAGE, "BeefedCafeFaceAcedAddedDecadeFee", true, "C01ffedC1cadaeAc1d1f1edAcac1aB0a", 5)) --> 0
+
+raw SQL: UPDATE auc_3g SET k = 'C01ffedC1cadaeAc1d1f1edAcac1aB0'
+sqlite3_prepare_v2(dbc->db, sql, -1, &stmt, NULL) --> SQLITE_OK
+
+sqlite3_step(stmt) --> SQLITE_DONE
+
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -5
+DAUC IMSI='123456789000000': No 2G Auth Data
+DAUC IMSI='123456789000000': Error reading K, expected length 16 but has length 15
+
+
+
+--- Set auth data, 3G only, with invalid OP length
+
+db_subscr_update_aud_by_id(dbc, id, mk_aud_3g(OSMO_AUTH_ALG_MILENAGE, "BeefedCafeFaceAcedAddedDecadeFee", true, "C01ffedC1cadaeAc1d1f1edAcac1aB0a", 5)) --> 0
+
+raw SQL: UPDATE auc_3g SET op = 'BeefedCafeFaceAcedAddedDecadeFe'
+sqlite3_prepare_v2(dbc->db, sql, -1, &stmt, NULL) --> SQLITE_OK
+
+sqlite3_step(stmt) --> SQLITE_DONE
+
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -5
+DAUC IMSI='123456789000000': No 2G Auth Data
+DAUC IMSI='123456789000000': Error reading OP, expected length 16 but has length 15
+
+
+
+--- Set auth data, 3G only, with invalid OPC length
+
+db_subscr_update_aud_by_id(dbc, id, mk_aud_3g(OSMO_AUTH_ALG_MILENAGE, "BeefedCafeFaceAcedAddedDecadeFee", false, "C01ffedC1cadaeAc1d1f1edAcac1aB0a", 5)) --> 0
+
+raw SQL: UPDATE auc_3g SET opc = 'BeefedCafeFaceAcedAddedDecadeFe'
+sqlite3_prepare_v2(dbc->db, sql, -1, &stmt, NULL) --> SQLITE_OK
+
+sqlite3_step(stmt) --> SQLITE_DONE
+
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -5
+DAUC IMSI='123456789000000': No 2G Auth Data
+DAUC IMSI='123456789000000': Error reading OPC, expected length 16 but has length 15
+
+
+
+--- Delete subscriber
+
+db_subscr_delete_by_id(dbc, id) --> 0
+
+===== test_subscr_aud_invalid_len: SUCCESS
+
+
 ===== test_subscr_sqn
 
 --- Set SQN for unknown subscriber
@@ -1258,7 +1442,7 @@ DAUC Cannot read subscriber from db: ID=9999: No such subscriber
 
 --- Create subscriber
 
-db_subscr_create(dbc, imsi0) --> 0
+db_subscr_create(dbc, imsi0, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> 0
 
 db_subscr_get_by_imsi(dbc, imsi0, &g_subscr) --> 0
 struct hlr_subscriber {
@@ -1266,7 +1450,7 @@ struct hlr_subscriber {
   .imsi = '123456789000000',
 }
 
-db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -126
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 
@@ -1277,7 +1461,7 @@ DAUC IMSI='123456789000000': No 3G Auth Data
 db_update_sqn(dbc, id, 123) --> -ENOENT
 DAUC Cannot update SQN for subscriber ID=1: no auc_3g entry for such subscriber
 
-db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -126
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 
@@ -1285,7 +1469,7 @@ DAUC IMSI='123456789000000': No 3G Auth Data
 db_update_sqn(dbc, id, 543) --> -ENOENT
 DAUC Cannot update SQN for subscriber ID=1: no auc_3g entry for such subscriber
 
-db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -126
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data
 

tests/db_upgrade/Makefile.am

@@ -0,0 +1,14 @@
+EXTRA_DIST = \
+	db_upgrade_test.sh \
+	db_upgrade_test.err \
+	db_upgrade_test.ok \
+	hlr_db_v0.sql \
+	osmo-hlr.cfg \
+	create_subscribers.vty \
+	$(NULL)
+
+update_exp:
+	$(srcdir)/db_upgrade_test.sh $(srcdir) $(builddir) >"$(srcdir)/db_upgrade_test.ok" 2>"$(srcdir)/db_upgrade_test.err"
+
+upgrade-equivalence-test:
+	$(srcdir)/db_upgrade_test.sh $(srcdir) $(builddir) do-equivalence-test

tests/db_upgrade/create_subscribers.vty

@@ -0,0 +1,47 @@
+OsmoHLR> enable
+OsmoHLR# subscriber imsi 123456789012345 create
+% Created subscriber 123456789012345
+    ID: 1
+    IMSI: 123456789012345
+    MSISDN: none
+OsmoHLR# subscriber imsi 123456789012345 update msisdn 098765432109876
+% Updated subscriber IMSI='123456789012345' to MSISDN='098765432109876'
+OsmoHLR# subscriber imsi 123456789012345 update aud2g comp128v1 ki BeefedCafeFaceAcedAddedDecadeFee
+OsmoHLR# subscriber imsi 123456789012345 update aud3g milenage k C01ffedC1cadaeAc1d1f1edAcac1aB0a opc CededEffacedAceFacedBadFadedBeef
+OsmoHLR# subscriber imsi 111111111 create
+% Created subscriber 111111111
+    ID: 2
+    IMSI: 111111111
+    MSISDN: none
+OsmoHLR# subscriber imsi 222222222 create
+% Created subscriber 222222222
+    ID: 3
+    IMSI: 222222222
+    MSISDN: none
+OsmoHLR# subscriber imsi 222222222 update msisdn 22222
+% Updated subscriber IMSI='222222222' to MSISDN='22222'
+OsmoHLR# subscriber imsi 333333 create
+% Created subscriber 333333
+    ID: 4
+    IMSI: 333333
+    MSISDN: none
+OsmoHLR# subscriber imsi 333333 update msisdn 3
+% Updated subscriber IMSI='333333' to MSISDN='3'
+OsmoHLR# subscriber imsi 333333 update aud2g comp128v2 ki 33333333333333333333333333333333
+OsmoHLR# subscriber imsi 444444444444444 create
+% Created subscriber 444444444444444
+    ID: 5
+    IMSI: 444444444444444
+    MSISDN: none
+OsmoHLR# subscriber imsi 444444444444444 update msisdn 4444
+% Updated subscriber IMSI='444444444444444' to MSISDN='4444'
+OsmoHLR# subscriber imsi 444444444444444 update aud3g milenage k 44444444444444444444444444444444 op 44444444444444444444444444444444
+OsmoHLR# subscriber imsi 5555555 create
+% Created subscriber 5555555
+    ID: 6
+    IMSI: 5555555
+    MSISDN: none
+OsmoHLR# subscriber imsi 5555555 update msisdn 55555555555555
+% Updated subscriber IMSI='5555555' to MSISDN='55555555555555'
+OsmoHLR# subscriber imsi 5555555 update aud2g xor ki 55555555555555555555555555555555
+OsmoHLR# subscriber imsi 5555555 update aud3g milenage k 55555555555555555555555555555555 opc 55555555555555555555555555555555

tests/db_upgrade/create_subscribers_step2.sql

@@ -0,0 +1,6 @@
+update subscriber set vlr_number = 'MSC-1' where id = 1;
+update subscriber set ms_purged_cs = 1 where id = 2;
+update subscriber set ms_purged_ps = 1 where id = 3;
+update subscriber set nam_cs = 0 where id = 4;
+update subscriber set nam_ps = 0 where id = 5;
+update subscriber set nam_cs = 0, nam_ps = 0 where id = 6;

tests/db_upgrade/db_upgrade_test.ok

@@ -0,0 +1,172 @@
+Creating db in schema version 0
+
+Version 0 db:
+
+Table: auc_2g
+name|type|notnull|dflt_value|pk
+algo_id_2g|INTEGER|1||0
+ki|VARCHAR(32)|1||0
+subscriber_id|INTEGER|0||1
+
+Table auc_2g contents:
+algo_id_2g|ki|subscriber_id
+1|BeefedCafeFaceAcedAddedDecadeFee|1
+2|33333333333333333333333333333333|4
+4|55555555555555555555555555555555|6
+
+Table: auc_3g
+name|type|notnull|dflt_value|pk
+algo_id_3g|INTEGER|1||0
+ind_bitlen|INTEGER|1|5|0
+k|VARCHAR(32)|1||0
+op|VARCHAR(32)|0||0
+opc|VARCHAR(32)|0||0
+sqn|INTEGER|1|0|0
+subscriber_id|INTEGER|0||1
+
+Table auc_3g contents:
+algo_id_3g|ind_bitlen|k|op|opc|sqn|subscriber_id
+5|5|C01ffedC1cadaeAc1d1f1edAcac1aB0a||CededEffacedAceFacedBadFadedBeef|0|1
+5|5|44444444444444444444444444444444|44444444444444444444444444444444||0|5
+5|5|55555555555555555555555555555555||55555555555555555555555555555555|0|6
+
+Table: subscriber
+name|type|notnull|dflt_value|pk
+ggsn_number|VARCHAR(15)|0||0
+gmlc_number|VARCHAR(15)|0||0
+hlr_number|VARCHAR(15)|0||0
+id|INTEGER|0||1
+imeisv|VARCHAR|0||0
+imsi|VARCHAR(15)|1||0
+lmsi|INTEGER|0||0
+ms_purged_cs|BOOLEAN|1|0|0
+ms_purged_ps|BOOLEAN|1|0|0
+msisdn|VARCHAR(15)|0||0
+nam_cs|BOOLEAN|1|1|0
+nam_ps|BOOLEAN|1|1|0
+periodic_lu_tmr|INTEGER|0||0
+periodic_rau_tau_tmr|INTEGER|0||0
+sgsn_address|VARCHAR|0||0
+sgsn_number|VARCHAR(15)|0||0
+smsc_number|VARCHAR(15)|0||0
+vlr_number|VARCHAR(15)|0||0
+
+Table subscriber contents:
+ggsn_number|gmlc_number|hlr_number|id|imeisv|imsi|lmsi|ms_purged_cs|ms_purged_ps|msisdn|nam_cs|nam_ps|periodic_lu_tmr|periodic_rau_tau_tmr|sgsn_address|sgsn_number|smsc_number|vlr_number
+|||1||123456789012345||0|0|098765432109876|1|1||||||MSC-1
+|||2||111111111||1|0||1|1||||||
+|||3||222222222||0|1|22222|1|1||||||
+|||4||333333||0|0|3|0|1||||||
+|||5||444444444444444||0|0|4444|1|0||||||
+|||6||5555555||0|0|55555555555555|0|0||||||
+
+Table: subscriber_apn
+name|type|notnull|dflt_value|pk
+apn|VARCHAR(256)|1||0
+subscriber_id|INTEGER|0||0
+
+Table subscriber_apn contents:
+
+Table: subscriber_multi_msisdn
+name|type|notnull|dflt_value|pk
+msisdn|VARCHAR(15)|1||0
+subscriber_id|INTEGER|0||0
+
+Table subscriber_multi_msisdn contents:
+
+Launching osmo-hlr to upgrade db:
+osmo-hlr --database $db --db-upgrade --db-check --config-file $srcdir/osmo-hlr.cfg
+rc = 0
+DMAIN hlr starting
+DDB using database: <PATH>test.db
+DDB Database <PATH>test.db' has HLR DB schema version 0
+DDB Database <PATH>test.db' has been upgraded to HLR DB schema version 1
+DDB Database <PATH>test.db' has been upgraded to HLR DB schema version 2
+DDB Database <PATH>test.db' has been upgraded to HLR DB schema version 3
+DDB Database <PATH>test.db' has been upgraded to HLR DB schema version 4
+DMAIN Cmdline option --db-check: Database was opened successfully, quitting.
+
+Resulting db:
+
+Table: auc_2g
+name|type|notnull|dflt_value|pk
+algo_id_2g|INTEGER|1||0
+ki|VARCHAR(32)|1||0
+subscriber_id|INTEGER|0||1
+
+Table auc_2g contents:
+algo_id_2g|ki|subscriber_id
+1|BeefedCafeFaceAcedAddedDecadeFee|1
+2|33333333333333333333333333333333|4
+4|55555555555555555555555555555555|6
+
+Table: auc_3g
+name|type|notnull|dflt_value|pk
+algo_id_3g|INTEGER|1||0
+ind_bitlen|INTEGER|1|5|0
+k|VARCHAR(32)|1||0
+op|VARCHAR(32)|0||0
+opc|VARCHAR(32)|0||0
+sqn|INTEGER|1|0|0
+subscriber_id|INTEGER|0||1
+
+Table auc_3g contents:
+algo_id_3g|ind_bitlen|k|op|opc|sqn|subscriber_id
+5|5|C01ffedC1cadaeAc1d1f1edAcac1aB0a||CededEffacedAceFacedBadFadedBeef|0|1
+5|5|44444444444444444444444444444444|44444444444444444444444444444444||0|5
+5|5|55555555555555555555555555555555||55555555555555555555555555555555|0|6
+
+Table: subscriber
+name|type|notnull|dflt_value|pk
+ggsn_number|VARCHAR(15)|0||0
+gmlc_number|VARCHAR(15)|0||0
+id|INTEGER|0||1
+imei|VARCHAR(14)|0||0
+imeisv|VARCHAR|0||0
+imsi|VARCHAR(15)|1||0
+last_lu_seen|TIMESTAMP|0|NULL|0
+last_lu_seen_ps|TIMESTAMP|0|NULL|0
+lmsi|INTEGER|0||0
+ms_purged_cs|BOOLEAN|1|0|0
+ms_purged_ps|BOOLEAN|1|0|0
+msc_number|VARCHAR(15)|0||0
+msisdn|VARCHAR(15)|0||0
+nam_cs|BOOLEAN|1|1|0
+nam_ps|BOOLEAN|1|1|0
+periodic_lu_tmr|INTEGER|0||0
+periodic_rau_tau_tmr|INTEGER|0||0
+sgsn_address|VARCHAR|0||0
+sgsn_number|VARCHAR(15)|0||0
+smsc_number|VARCHAR(15)|0||0
+vlr_number|VARCHAR(15)|0||0
+
+Table subscriber contents:
+ggsn_number|gmlc_number|id|imei|imeisv|imsi|last_lu_seen|last_lu_seen_ps|lmsi|ms_purged_cs|ms_purged_ps|msc_number|msisdn|nam_cs|nam_ps|periodic_lu_tmr|periodic_rau_tau_tmr|sgsn_address|sgsn_number|smsc_number|vlr_number
+||1|||123456789012345||||0|0||098765432109876|1|1||||||MSC-1
+||2|||111111111||||1|0|||1|1||||||
+||3|||222222222||||0|1||22222|1|1||||||
+||4|||333333||||0|0||3|0|1||||||
+||5|||444444444444444||||0|0||4444|1|0||||||
+||6|||5555555||||0|0||55555555555555|0|0||||||
+
+Table: subscriber_apn
+name|type|notnull|dflt_value|pk
+apn|VARCHAR(256)|1||0
+subscriber_id|INTEGER|0||0
+
+Table subscriber_apn contents:
+
+Table: subscriber_multi_msisdn
+name|type|notnull|dflt_value|pk
+msisdn|VARCHAR(15)|1||0
+subscriber_id|INTEGER|0||0
+
+Table subscriber_multi_msisdn contents:
+
+Verify that osmo-hlr can open it:
+osmo-hlr --database $db --db-check --config-file $srcdir/osmo-hlr.cfg
+rc = 0
+DMAIN hlr starting
+DDB using database: <PATH>test.db
+DDB Database <PATH>test.db' has HLR DB schema version 4
+DMAIN Cmdline option --db-check: Database was opened successfully, quitting.

tests/db_upgrade/db_upgrade_test.sh

@@ -0,0 +1,83 @@
+#!/bin/sh
+srcdir="${1:-.}"
+builddir="${2:-.}"
+do_equivalence_test="$3"
+
+set -e
+
+db="$builddir/test.db"
+osmo_hlr="$builddir/../../src/osmo-hlr"
+cfg="$srcdir/osmo-hlr.cfg"
+
+dump_sorted_schema(){
+	db_file="$1"
+	tables="$(sqlite3 -batch -noheader "$db_file" "SELECT name FROM sqlite_master WHERE type = 'table' order by name")"
+	for table in $tables; do
+		echo
+		echo "Table: $table"
+		sqlite3 -batch -header "$db_file" "SELECT name,type,\"notnull\",dflt_value,pk FROM PRAGMA_TABLE_INFO('$table') order by name;"
+		echo
+		echo "Table $table contents:"
+		columns="$(sqlite3 -batch -noheader "$db_file" "SELECT name FROM PRAGMA_TABLE_INFO('$table') order by name;")"
+		sqlite3 -batch -header "$db_file" "SELECT $(echo $columns | sed 's/ /,/g') from $table;"
+	done
+}
+
+rm -f "$db"
+echo "Creating db in schema version 0"
+sqlite3 -batch "$db" < "$srcdir/hlr_db_v0.sql"
+
+echo
+echo "Version 0 db:"
+dump_sorted_schema "$db"
+
+set +e
+
+echo
+echo "Launching osmo-hlr to upgrade db:"
+echo osmo-hlr --database '$db' --db-upgrade --db-check --config-file '$srcdir/osmo-hlr.cfg'
+"$osmo_hlr" --database "$db" --db-upgrade --db-check --config-file "$cfg" >log 2>&1
+echo "rc = $?"
+cat log | sed 's@[^ "]*/@<PATH>@g'
+
+echo
+echo "Resulting db:"
+dump_sorted_schema "$db"
+
+echo
+echo "Verify that osmo-hlr can open it:"
+echo osmo-hlr --database '$db' --db-check --config-file '$srcdir/osmo-hlr.cfg'
+"$osmo_hlr" --database "$db" --db-check --config-file "$cfg" >log 2>&1
+echo "rc = $?"
+cat log | sed 's@[^ "]*/@<PATH>@g'
+
+if [ -n "$do_equivalence_test" ]; then
+	# this part requires osmo_interact_vty.py, so this test is not part of the normal run
+	set -e -x
+	mint_db="$builddir/mint.db"
+	rm -f "$mint_db"
+
+	osmo_verify_transcript_vty.py -v \
+		-n OsmoHLR -p 4258 \
+		-r "$osmo_hlr -c $cfg -l $mint_db" \
+		"$srcdir/create_subscribers.vty"
+	sqlite3 -batch "$mint_db" < "$srcdir/create_subscribers_step2.sql"
+
+	set +x
+	test_dump="$builddir/test.dump"
+	mint_dump="$builddir/mint.dump"
+
+	dump_sorted_schema "$db" > "$test_dump"
+	dump_sorted_schema "$mint_db" > "$mint_dump"
+
+	echo
+	echo "Newly created sorted schema is:"
+	cat "$mint_dump"
+	echo
+	echo "Diff to upgraded schema:"
+	diff -u "$mint_dump" "$test_dump"
+	echo "rc=$?"
+fi
+
+rm -f log
+rm -f "$db"

tests/db_upgrade/hlr_db_v0.sql

@@ -0,0 +1,80 @@
+PRAGMA foreign_keys=OFF;
+BEGIN TRANSACTION;
+CREATE TABLE subscriber (
+-- OsmoHLR's DB scheme is modelled roughly after TS 23.008 version 13.3.0
+	id		INTEGER PRIMARY KEY,
+	-- Chapter 2.1.1.1
+	imsi		VARCHAR(15) UNIQUE NOT NULL,
+	-- Chapter 2.1.2
+	msisdn		VARCHAR(15) UNIQUE,
+	-- Chapter 2.2.3: Most recent / current IMEI
+	imeisv		VARCHAR,
+	-- Chapter 2.4.5
+	vlr_number	VARCHAR(15),
+	-- Chapter 2.4.6
+	hlr_number	VARCHAR(15),
+	-- Chapter 2.4.8.1
+	sgsn_number	VARCHAR(15),
+	-- Chapter 2.13.10
+	sgsn_address	VARCHAR,
+	-- Chapter 2.4.8.2
+	ggsn_number	VARCHAR(15),
+	-- Chapter 2.4.9.2
+	gmlc_number	VARCHAR(15),
+	-- Chapter 2.4.23
+	smsc_number	VARCHAR(15),
+	-- Chapter 2.4.24
+	periodic_lu_tmr	INTEGER,
+	-- Chapter 2.13.115
+	periodic_rau_tau_tmr INTEGER,
+	-- Chapter 2.1.1.2: network access mode
+	nam_cs		BOOLEAN NOT NULL DEFAULT 1,
+	nam_ps		BOOLEAN NOT NULL DEFAULT 1,
+	-- Chapter 2.1.8
+	lmsi		INTEGER,
+	-- The below purged flags might not even be stored non-volatile,
+	-- refer to TS 23.012 Chapter 3.6.1.4
+	-- Chapter 2.7.5
+	ms_purged_cs	BOOLEAN NOT NULL DEFAULT 0,
+	-- Chapter 2.7.6
+	ms_purged_ps	BOOLEAN NOT NULL DEFAULT 0
+);
+INSERT INTO subscriber VALUES(1,'123456789012345','098765432109876',NULL,'MSC-1',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,1,NULL,0,0);
+INSERT INTO subscriber VALUES(2,'111111111',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,1,NULL,1,0);
+INSERT INTO subscriber VALUES(3,'222222222','22222',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,1,NULL,0,1);
+INSERT INTO subscriber VALUES(4,'333333','3',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,1,NULL,0,0);
+INSERT INTO subscriber VALUES(5,'444444444444444','4444',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,0,NULL,0,0);
+INSERT INTO subscriber VALUES(6,'5555555','55555555555555',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,NULL,0,0);
+CREATE TABLE subscriber_apn (
+	subscriber_id	INTEGER,		-- subscriber.id
+	apn		VARCHAR(256) NOT NULL
+);
+CREATE TABLE subscriber_multi_msisdn (
+-- Chapter 2.1.3
+	subscriber_id	INTEGER,		-- subscriber.id
+	msisdn		VARCHAR(15) NOT NULL
+);
+CREATE TABLE auc_2g (
+	subscriber_id	INTEGER PRIMARY KEY,	-- subscriber.id
+	algo_id_2g	INTEGER NOT NULL,	-- enum osmo_auth_algo value
+	ki		VARCHAR(32) NOT NULL	-- hex string: subscriber's secret key (128bit)
+);
+INSERT INTO auc_2g VALUES(1,1,'BeefedCafeFaceAcedAddedDecadeFee');
+INSERT INTO auc_2g VALUES(4,2,'33333333333333333333333333333333');
+INSERT INTO auc_2g VALUES(6,4,'55555555555555555555555555555555');
+CREATE TABLE auc_3g (
+	subscriber_id	INTEGER PRIMARY KEY,	-- subscriber.id
+	algo_id_3g	INTEGER NOT NULL,	-- enum osmo_auth_algo value
+	k		VARCHAR(32) NOT NULL,	-- hex string: subscriber's secret key (128bit)
+	op		VARCHAR(32),		-- hex string: operator's secret key (128bit)
+	opc		VARCHAR(32),		-- hex string: derived from OP and K (128bit)
+	sqn		INTEGER NOT NULL DEFAULT 0,	-- sequence number of key usage
+	-- nr of index bits at lower SQN end
+	ind_bitlen	INTEGER NOT NULL DEFAULT 5
+);
+INSERT INTO auc_3g VALUES(1,5,'C01ffedC1cadaeAc1d1f1edAcac1aB0a',NULL,'CededEffacedAceFacedBadFadedBeef',0,5);
+INSERT INTO auc_3g VALUES(5,5,'44444444444444444444444444444444','44444444444444444444444444444444',NULL,0,5);
+INSERT INTO auc_3g VALUES(6,5,'55555555555555555555555555555555',NULL,'55555555555555555555555555555555',0,5);
+CREATE UNIQUE INDEX idx_subscr_imsi ON subscriber (imsi)
+;
+COMMIT;

tests/db_upgrade/osmo-hlr.cfg

@@ -0,0 +1,6 @@
+log stderr
+ logging level db notice
+ logging print category-hex 0
+ logging print file 0
+ logging print category 1
+ logging color 0

tests/gsup/Makefile.am

@@ -1,41 +0,0 @@
-AM_CPPFLAGS = \
-	$(all_includes) \
-	-I$(top_srcdir)/src \
-	$(NULL)
-
-AM_CFLAGS = \
-	-Wall \
-	-ggdb3 \
-	$(LIBOSMOCORE_CFLAGS) \
-	$(LIBOSMOGSM_CFLAGS) \
-	$(LIBOSMOABIS_CFLAGS) \
-	$(NULL)
-
-AM_LDFLAGS = \
-	$(NULL)
-
-EXTRA_DIST = \
-	gsup_test.ok \
-	gsup_test.err \
-	$(NULL)
-
-noinst_PROGRAMS = \
-	gsup_test \
-	$(NULL)
-
-gsup_test_SOURCES = \
-	gsup_test.c \
-	$(NULL)
-
-gsup_test_LDADD = \
-	$(top_srcdir)/src/luop.c \
-	$(top_srcdir)/src/gsup_server.c \
-	$(top_srcdir)/src/gsup_router.c \
-	$(LIBOSMOCORE_LIBS) \
-	$(LIBOSMOGSM_LIBS) \
-	$(LIBOSMOABIS_LIBS) \
-	$(NULL)
-
-.PHONY: update_exp
-update_exp:
-	$(builddir)/gsup_test >"$(srcdir)/gsup_test.ok" 2>"$(srcdir)/gsup_test.err"

tests/gsup/gsup_test.c

@@ -1,91 +0,0 @@
-/* (C) 2018 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
- *
- * All Rights Reserved
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- */
-
-#include <string.h>
-
-#include <osmocom/core/logging.h>
-#include <osmocom/core/utils.h>
-#include <osmocom/core/application.h>
-#include <osmocom/gsm/gsup.h>
-
-#include "logging.h"
-#include "luop.h"
-
-struct osmo_gsup_server;
-
-/* override osmo_gsup_addr_send() to not actually send anything. */
-int osmo_gsup_addr_send(struct osmo_gsup_server *gs,
-			const uint8_t *addr, size_t addrlen,
-			struct msgb *msg)
-{
-	LOGP(DMAIN, LOGL_DEBUG, "%s\n", msgb_hexdump(msg));
-	msgb_free(msg);
-	return 0;
-}
-
-int db_subscr_get_by_imsi(struct db_context *dbc, const char *imsi,
-			  struct hlr_subscriber *subscr)
-{
-	return 0;
-}
-
-/* Verify that the internally allocated msgb is large enough */
-void test_gsup_tx_insert_subscr_data()
-{
-	struct lu_operation luop = {
-		.state = LU_S_LU_RECEIVED,
-		.subscr = {
-			.imsi = "123456789012345",
-			.msisdn = "987654321098765",
-			.nam_cs = true,
-			.nam_ps = true,
-		},
-		.is_ps = true,
-	};
-
-	lu_op_tx_insert_subscr_data(&luop);
-}
-
-const struct log_info_cat default_categories[] = {
-	[DMAIN] = {
-		.name = "DMAIN",
-		.description = "Main Program",
-		.enabled = 1, .loglevel = LOGL_DEBUG,
-	},
-};
-
-static struct log_info info = {
-	.cat = default_categories,
-	.num_cat = ARRAY_SIZE(default_categories),
-};
-
-int main(int argc, char **argv)
-{
-	void *ctx = talloc_named_const(NULL, 0, "gsup_test");
-	osmo_init_logging2(ctx, &info);
-	log_set_print_filename(osmo_stderr_target, 0);
-	log_set_print_timestamp(osmo_stderr_target, 0);
-	log_set_use_color(osmo_stderr_target, 0);
-	log_set_print_category(osmo_stderr_target, 1);
-
-	test_gsup_tx_insert_subscr_data();
-
-	printf("Done.\n");
-	return EXIT_SUCCESS;
-}

tests/gsup/gsup_test.err

@@ -1,2 +0,0 @@
-DMAIN 10 01 08 21 43 65 87 09 21 43 f5 08 09 08 89 67 45 23 01 89 67 f5 05 07 10 01 01 12 02 01 2a 28 01 01 
-DMAIN LU OP state change: LU RECEIVED -> ISD SENT

tests/gsup/gsup_test.ok

@@ -1 +0,0 @@
-Done.

tests/gsup_server/Makefile.am

@@ -1,17 +1,18 @@
 AM_CPPFLAGS = \
 	$(all_includes) \
-	-I$(top_srcdir)/src \
 	$(NULL)
 
 AM_CFLAGS = \
 	-Wall \
 	-ggdb3 \
+	-I$(top_srcdir)/include \
 	$(LIBOSMOCORE_CFLAGS) \
 	$(LIBOSMOGSM_CFLAGS) \
 	$(LIBOSMOABIS_CFLAGS) \
 	$(NULL)
 
 AM_LDFLAGS = \
+	-no-install \
 	$(NULL)
 
 EXTRA_DIST = \

tests/gsup_server/gsup_server_test.c

@@ -20,7 +20,7 @@
 
 #include <stdio.h>
 #include <osmocom/core/utils.h>
-#include "gsup_server.h"
+#include <osmocom/hlr/gsup_server.h>
 
 #define comment_start() printf("\n===== %s\n", __func__)
 #define comment_end() printf("===== %s: SUCCESS\n\n", __func__)

tests/mslookup/Makefile.am

@@ -0,0 +1,69 @@
+AM_CPPFLAGS = \
+	$(all_includes) \
+	$(NULL)
+
+AM_CFLAGS = \
+	-Wall \
+	-ggdb3 \
+	-I$(top_srcdir)/include \
+	$(LIBOSMOCORE_CFLAGS) \
+	$(LIBOSMOGSM_CFLAGS) \
+	$(LIBOSMOABIS_CFLAGS) \
+	$(NULL)
+
+AM_LDFLAGS = \
+	-no-install \
+	$(NULL)
+
+EXTRA_DIST = \
+	mdns_test.err \
+	mslookup_client_mdns_test.err \
+	mslookup_client_test.err \
+	mslookup_test.err \
+	$(NULL)
+
+check_PROGRAMS = \
+	mdns_test \
+	mslookup_client_mdns_test \
+	mslookup_client_test \
+	mslookup_test \
+	$(NULL)
+
+mslookup_test_SOURCES = \
+	mslookup_test.c \
+	$(NULL)
+mslookup_test_LDADD = \
+	$(top_builddir)/src/mslookup/libosmo-mslookup.la \
+	$(LIBOSMOGSM_LIBS) \
+	$(NULL)
+
+mslookup_client_test_SOURCES = \
+	mslookup_client_test.c \
+	$(NULL)
+mslookup_client_test_LDADD = \
+	$(top_builddir)/src/mslookup/libosmo-mslookup.la \
+	$(LIBOSMOGSM_LIBS) \
+	$(NULL)
+
+mslookup_client_mdns_test_SOURCES = \
+	mslookup_client_mdns_test.c \
+	$(NULL)
+mslookup_client_mdns_test_LDADD = \
+	$(top_builddir)/src/mslookup/libosmo-mslookup.la \
+	$(LIBOSMOGSM_LIBS) \
+	$(NULL)
+
+mdns_test_SOURCES = \
+	mdns_test.c \
+	$(NULL)
+mdns_test_LDADD = \
+	$(top_builddir)/src/mslookup/libosmo-mslookup.la \
+	$(LIBOSMOGSM_LIBS) \
+	$(NULL)
+
+.PHONY: update_exp
+update_exp:
+	for i in $(check_PROGRAMS); do \
+		echo "Updating $$i.err"; \
+		$(builddir)/$$i 2>"$(srcdir)/$$i.err"; \
+	done

tests/mslookup/mdns_test.c

@@ -0,0 +1,602 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include <assert.h>
+#include <errno.h>
+#include <string.h>
+#include <osmocom/core/application.h>
+#include <osmocom/core/logging.h>
+#include <osmocom/core/utils.h>
+#include <osmocom/mslookup/mdns_rfc.h>
+#include <osmocom/mslookup/mdns_msg.h>
+
+struct qname_enc_dec_test {
+	const char *domain;
+	const char *qname;
+	size_t qname_max_len; /* default: strlen(qname) + 1 */
+};
+
+static const struct qname_enc_dec_test qname_enc_dec_test_data[] = {
+	{
+		/* OK: typical mslookup domain */
+		.domain = "hlr.1234567.imsi",
+		.qname = "\x03" "hlr" "\x07" "1234567" "\x04" "imsi",
+	},
+	{
+		/* Wrong format: double dot */
+		.domain = "hlr..imsi",
+		.qname = NULL,
+	},
+	{
+		/* Wrong format: double dot */
+		.domain = "hlr",
+		.qname = "\x03hlr\0\x03imsi",
+	},
+	{
+		/* Wrong format: dot at end */
+		.domain = "hlr.",
+		.qname = NULL,
+	},
+	{
+		/* Wrong format: dot at start */
+		.domain = ".hlr",
+		.qname = NULL,
+	},
+	{
+		/* Wrong format: empty */
+		.domain = "",
+		.qname = NULL,
+	},
+	{
+		/* OK: maximum length */
+		.domain =
+			"123456789." "123456789." "123456789." "123456789." "123456789."
+			"123456789." "123456789." "123456789." "123456789." "123456789."
+			"123456789." "123456789." "123456789." "123456789." "123456789."
+			"123456789." "123456789." "123456789." "123456789." "123456789."
+			"123456789." "123456789." "123456789." "123456789." "123456789."
+			"12345"
+			,
+		.qname =
+			"\t123456789\t123456789\t123456789\t123456789\t123456789"
+			"\t123456789\t123456789\t123456789\t123456789\t123456789"
+			"\t123456789\t123456789\t123456789\t123456789\t123456789"
+			"\t123456789\t123456789\t123456789\t123456789\t123456789"
+			"\t123456789\t123456789\t123456789\t123456789\t123456789"
+			"\x05" "12345"
+	},
+	{
+		/* Error: too long domain */
+		.domain =
+			"123456789." "123456789." "123456789." "123456789." "123456789."
+			"123456789." "123456789." "123456789." "123456789." "123456789."
+			"123456789." "123456789." "123456789." "123456789." "123456789."
+			"123456789." "123456789." "123456789." "123456789." "123456789."
+			"123456789." "123456789." "123456789." "123456789." "123456789."
+			"12345toolong"
+			,
+		.qname = NULL,
+	},
+	{
+		/* Error: too long qname */
+		.domain = NULL,
+		.qname =
+			"\t123456789\t123456789\t123456789\t123456789\t123456789"
+			"\t123456789\t123456789\t123456789\t123456789\t123456789"
+			"\t123456789\t123456789\t123456789\t123456789\t123456789"
+			"\t123456789\t123456789\t123456789\t123456789\t123456789"
+			"\t123456789\t123456789\t123456789\t123456789\t123456789"
+			"\t123456789\t123456789\t123456789\t123456789\t123456789"
+	},
+	{
+		/* Error: wrong token length in qname */
+		.domain = NULL,
+		.qname = "\x03" "hlr" "\x07" "1234567" "\x05" "imsi",
+	},
+	{
+		/* Error: wrong token length in qname */
+		.domain = NULL,
+		.qname = "\x02" "hlr" "\x07" "1234567" "\x04" "imsi",
+	},
+	{
+		/* Wrong format: token length at end of qname */
+		.domain = NULL,
+		.qname = "\x03hlr\x03",
+	},
+	{
+		/* Error: overflow in label length */
+		.domain = NULL,
+		.qname = "\x03" "hlr" "\x07" "1234567" "\x04" "imsi",
+		.qname_max_len = 17,
+	},
+};
+
+void test_enc_dec_rfc_qname(void *ctx)
+{
+	char quote_buf[300];
+	int i;
+
+	fprintf(stderr, "-- %s --\n", __func__);
+
+	for (i = 0; i < ARRAY_SIZE(qname_enc_dec_test_data); i++) {
+		const struct qname_enc_dec_test *t = &qname_enc_dec_test_data[i];
+		char *res;
+
+		if (t->domain) {
+			fprintf(stderr, "domain: %s\n", osmo_quote_str_buf2(quote_buf, sizeof(quote_buf), t->domain, -1));
+			fprintf(stderr, "exp: %s\n", osmo_quote_str_buf2(quote_buf, sizeof(quote_buf), t->qname, -1));
+			res = osmo_mdns_rfc_qname_encode(ctx, t->domain);
+			fprintf(stderr, "res: %s\n", osmo_quote_str_buf2(quote_buf, sizeof(quote_buf), res, -1));
+			if (t->qname == res || (t->qname && res && strcmp(t->qname, res) == 0))
+				fprintf(stderr, "=> OK\n");
+			else
+				fprintf(stderr, "=> ERROR\n");
+			if (res)
+				talloc_free(res);
+			fprintf(stderr, "\n");
+		}
+
+		if (t->qname) {
+			size_t qname_max_len = t->qname_max_len;
+			if (qname_max_len)
+				fprintf(stderr, "qname_max_len: %lu\n", qname_max_len);
+			else
+				qname_max_len = strlen(t->qname) + 1;
+
+			fprintf(stderr, "qname: %s\n", osmo_quote_str_buf2(quote_buf, sizeof(quote_buf), t->qname, -1));
+			fprintf(stderr, "exp: %s\n", osmo_quote_str_buf2(quote_buf, sizeof(quote_buf), t->domain, -1));
+			res = osmo_mdns_rfc_qname_decode(ctx, t->qname, qname_max_len);
+			fprintf(stderr, "res: %s\n", osmo_quote_str_buf2(quote_buf, sizeof(quote_buf), res, -1));
+			if (t->domain == res || (t->domain && res && strcmp(t->domain, res) == 0))
+				fprintf(stderr, "=> OK\n");
+			else
+				fprintf(stderr, "=> ERROR\n");
+			if (res)
+				talloc_free(res);
+			fprintf(stderr, "\n");
+		}
+	}
+}
+
+#define PRINT_HDR(hdr, name) \
+	fprintf(stderr, "header %s:\n" \
+	       ".id = %i\n" \
+	       ".qr = %i\n" \
+	       ".opcode = %x\n" \
+	       ".aa = %i\n" \
+	       ".tc = %i\n" \
+	       ".rd = %i\n" \
+	       ".ra = %i\n" \
+	       ".z = %x\n" \
+	       ".rcode = %x\n" \
+	       ".qdcount = %u\n" \
+	       ".ancount = %u\n" \
+	       ".nscount = %u\n" \
+	       ".arcount = %u\n", \
+	       name, hdr.id, hdr.qr, hdr.opcode, hdr.aa, hdr.tc, hdr.rd, hdr.ra, hdr.z, hdr.rcode, hdr.qdcount, \
+	       hdr.ancount, hdr.nscount, hdr.arcount)
+
+static const struct osmo_mdns_rfc_header header_enc_dec_test_data[] = {
+	{
+		/* Typical use case for mslookup */
+		.id = 1337,
+		.qdcount = 1,
+	},
+	{
+		/* Fill out everything */
+		.id = 42,
+		.qr = 1,
+		.opcode = 0x02,
+		.aa = 1,
+		.tc = 1,
+		.rd = 1,
+		.ra = 1,
+		.z  = 0x02,
+		.rcode = 0x03,
+		.qdcount = 1234,
+		.ancount = 1111,
+		.nscount = 2222,
+		.arcount = 3333,
+	},
+};
+
+void test_enc_dec_rfc_header()
+{
+	int i;
+
+	fprintf(stderr, "-- %s --\n", __func__);
+	for (i = 0; i< ARRAY_SIZE(header_enc_dec_test_data); i++) {
+		const struct osmo_mdns_rfc_header in = header_enc_dec_test_data[i];
+		struct osmo_mdns_rfc_header out = {0};
+		struct msgb *msg = msgb_alloc(4096, "dns_test");
+
+		PRINT_HDR(in, "in");
+		osmo_mdns_rfc_header_encode(msg, &in);
+		fprintf(stderr, "encoded: %s\n", osmo_hexdump(msgb_data(msg), msgb_length(msg)));
+		assert(osmo_mdns_rfc_header_decode(msgb_data(msg), msgb_length(msg), &out) == 0);
+		PRINT_HDR(out, "out");
+
+		fprintf(stderr, "in (hexdump):  %s\n", osmo_hexdump((unsigned char *)&in, sizeof(in)));
+		fprintf(stderr, "out (hexdump): %s\n", osmo_hexdump((unsigned char *)&out, sizeof(out)));
+		assert(memcmp(&in, &out, sizeof(in)) == 0);
+
+		fprintf(stderr, "=> OK\n\n");
+		msgb_free(msg);
+	}
+}
+
+void test_enc_dec_rfc_header_einval()
+{
+	struct osmo_mdns_rfc_header out = {0};
+	struct msgb *msg = msgb_alloc(4096, "dns_test");
+	fprintf(stderr, "-- %s --\n", __func__);
+
+	assert(osmo_mdns_rfc_header_decode(msgb_data(msg), 11, &out) == -EINVAL);
+	fprintf(stderr, "=> OK\n\n");
+
+	msgb_free(msg);
+}
+
+#define PRINT_QST(qst, name) \
+	fprintf(stderr, "question %s:\n" \
+	       ".domain = %s\n" \
+	       ".qtype = %i\n" \
+	       ".qclass = %i\n", \
+	       name, (qst)->domain, (qst)->qtype, (qst)->qclass)
+
+static const struct osmo_mdns_rfc_question question_enc_dec_test_data[] = {
+	{
+		.domain = "hlr.1234567.imsi",
+		.qtype = OSMO_MDNS_RFC_RECORD_TYPE_ALL,
+		.qclass = OSMO_MDNS_RFC_CLASS_IN,
+	},
+	{
+		.domain = "hlr.1234567.imsi",
+		.qtype = OSMO_MDNS_RFC_RECORD_TYPE_A,
+		.qclass = OSMO_MDNS_RFC_CLASS_ALL,
+	},
+	{
+		.domain = "hlr.1234567.imsi",
+		.qtype = OSMO_MDNS_RFC_RECORD_TYPE_AAAA,
+		.qclass = OSMO_MDNS_RFC_CLASS_ALL,
+	},
+};
+
+void test_enc_dec_rfc_question(void *ctx)
+{
+	int i;
+
+	fprintf(stderr, "-- %s --\n", __func__);
+	for (i = 0; i< ARRAY_SIZE(question_enc_dec_test_data); i++) {
+		const struct osmo_mdns_rfc_question in = question_enc_dec_test_data[i];
+		struct osmo_mdns_rfc_question *out;
+		struct msgb *msg = msgb_alloc(4096, "dns_test");
+
+		PRINT_QST(&in, "in");
+		assert(osmo_mdns_rfc_question_encode(ctx, msg, &in) == 0);
+		fprintf(stderr, "encoded: %s\n", osmo_hexdump(msgb_data(msg), msgb_length(msg)));
+		out = osmo_mdns_rfc_question_decode(ctx, msgb_data(msg), msgb_length(msg));
+		assert(out);
+		PRINT_QST(out, "out");
+
+		if (strcmp(in.domain, out->domain) != 0)
+			fprintf(stderr, "=> ERROR: domain does not match\n");
+		else if (in.qtype != out->qtype)
+			fprintf(stderr, "=> ERROR: qtype does not match\n");
+		else if (in.qclass != out->qclass)
+			fprintf(stderr, "=> ERROR: qclass does not match\n");
+		else
+			fprintf(stderr, "=> OK\n");
+
+		fprintf(stderr, "\n");
+		msgb_free(msg);
+		talloc_free(out);
+	}
+}
+
+void test_enc_dec_rfc_question_null(void *ctx)
+{
+	uint8_t data[5] = {0};
+
+	fprintf(stderr, "-- %s --\n", __func__);
+	assert(osmo_mdns_rfc_question_decode(ctx, data, sizeof(data)) == NULL);
+	fprintf(stderr, "=> OK\n\n");
+}
+
+#define PRINT_REC(rec, name) \
+	fprintf(stderr, "question %s:\n" \
+	       ".domain = %s\n" \
+	       ".type = %i\n" \
+	       ".class = %i\n" \
+	       ".ttl = %i\n" \
+	       ".rdlength = %i\n" \
+	       ".rdata = %s\n", \
+	       name, (rec)->domain, (rec)->type, (rec)->class, (rec)->ttl, (rec)->rdlength, \
+	       osmo_quote_str((char *)(rec)->rdata, (rec)->rdlength))
+
+static const struct osmo_mdns_rfc_record record_enc_dec_test_data[] = {
+	{
+		.domain = "hlr.1234567.imsi",
+		.type = OSMO_MDNS_RFC_RECORD_TYPE_A,
+		.class = OSMO_MDNS_RFC_CLASS_IN,
+		.ttl = 1234,
+		.rdlength = 9,
+		.rdata = (uint8_t *)"10.42.2.1",
+	},
+};
+
+void test_enc_dec_rfc_record(void *ctx)
+{
+	int i;
+
+	fprintf(stderr, "-- %s --\n", __func__);
+	for (i=0; i< ARRAY_SIZE(record_enc_dec_test_data); i++) {
+		const struct osmo_mdns_rfc_record in = record_enc_dec_test_data[i];
+		struct osmo_mdns_rfc_record *out;
+		struct msgb *msg = msgb_alloc(4096, "dns_test");
+		size_t record_len;
+
+		PRINT_REC(&in, "in");
+		assert(osmo_mdns_rfc_record_encode(ctx, msg, &in) == 0);
+		fprintf(stderr, "encoded: %s\n", osmo_hexdump(msgb_data(msg), msgb_length(msg)));
+		out = osmo_mdns_rfc_record_decode(ctx, msgb_data(msg), msgb_length(msg), &record_len);
+		fprintf(stderr, "record_len: %lu\n", record_len);
+		assert(out);
+		PRINT_REC(out, "out");
+
+		if (strcmp(in.domain, out->domain) != 0)
+			fprintf(stderr, "=> ERROR: domain does not match\n");
+		else if (in.type != out->type)
+			fprintf(stderr, "=> ERROR: type does not match\n");
+		else if (in.class != out->class)
+			fprintf(stderr, "=> ERROR: class does not match\n");
+		else if (in.ttl != out->ttl)
+			fprintf(stderr, "=> ERROR: ttl does not match\n");
+		else if (in.rdlength != out->rdlength)
+			fprintf(stderr, "=> ERROR: rdlength does not match\n");
+		else if (memcmp(in.rdata, out->rdata, in.rdlength) != 0)
+			fprintf(stderr, "=> ERROR: rdata does not match\n");
+		else
+			fprintf(stderr, "=> OK\n");
+
+		fprintf(stderr, "\n");
+		msgb_free(msg);
+		talloc_free(out);
+	}
+}
+
+static uint8_t ip_v4_n[] = {23, 42, 47, 11};
+static uint8_t ip_v6_n[] = {0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88,
+			    0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x00};
+
+
+enum test_records {
+	RECORD_NONE,
+	RECORD_A,
+	RECORD_AAAA,
+	RECORD_TXT_AGE,
+	RECORD_TXT_PORT_444,
+	RECORD_TXT_PORT_666,
+	RECORD_TXT_INVALID_KEY,
+	RECORD_TXT_INVALID_NO_KEY_VALUE,
+	RECORD_INVALID,
+};
+struct result_from_answer_test {
+	const char *desc;
+	const enum test_records records[5];
+	bool error;
+	const struct osmo_mslookup_result res;
+};
+
+static void test_result_from_answer(void *ctx)
+{
+	void *print_ctx = talloc_named_const(ctx, 0, __func__);
+	struct osmo_sockaddr_str test_host_v4 = {.af = AF_INET, .port=444, .ip = "23.42.47.11"};
+	struct osmo_sockaddr_str test_host_v6 = {.af = AF_INET6, .port=666,
+						 .ip = "1122:3344:5566:7788:99aa:bbcc:ddee:ff00"};
+	struct osmo_mslookup_result test_result_v4 = {.rc = OSMO_MSLOOKUP_RC_RESULT, .age = 3,
+						      .host_v4 = test_host_v4};
+	struct osmo_mslookup_result test_result_v6 = {.rc = OSMO_MSLOOKUP_RC_RESULT, .age = 3,
+						      .host_v6 = test_host_v6};
+	struct osmo_mslookup_result test_result_v4_v6 = {.rc = OSMO_MSLOOKUP_RC_RESULT, .age = 3,
+							 .host_v4 = test_host_v4, .host_v6 = test_host_v6};
+	struct result_from_answer_test result_from_answer_data[] = {
+		{
+			.desc = "IPv4",
+			.records = {RECORD_TXT_AGE, RECORD_A, RECORD_TXT_PORT_444},
+			.res = test_result_v4
+		},
+		{
+			.desc = "IPv6",
+			.records = {RECORD_TXT_AGE, RECORD_AAAA, RECORD_TXT_PORT_666},
+			.res = test_result_v6
+		},
+		{
+			.desc = "IPv4 + IPv6",
+			.records = {RECORD_TXT_AGE, RECORD_A, RECORD_TXT_PORT_444, RECORD_AAAA, RECORD_TXT_PORT_666},
+			.res = test_result_v4_v6
+		},
+		{
+			.desc = "A twice",
+			.records = {RECORD_TXT_AGE, RECORD_A, RECORD_TXT_PORT_444, RECORD_A},
+			.error = true
+		},
+		{
+			.desc = "AAAA twice",
+			.records = {RECORD_TXT_AGE, RECORD_AAAA, RECORD_TXT_PORT_444, RECORD_AAAA},
+			.error = true
+		},
+		{
+			.desc = "invalid TXT: no key/value pair",
+			.records = {RECORD_TXT_AGE, RECORD_AAAA, RECORD_TXT_INVALID_NO_KEY_VALUE},
+			.error = true
+		},
+		{
+			.desc = "age twice",
+			.records = {RECORD_TXT_AGE, RECORD_TXT_AGE},
+			.error = true
+		},
+		{
+			.desc = "port as first record",
+			.records = {RECORD_TXT_PORT_444},
+			.error = true
+		},
+		{
+			.desc = "port without previous ip record",
+			.records = {RECORD_TXT_AGE, RECORD_TXT_PORT_444},
+			.error = true
+		},
+		{
+			.desc = "invalid TXT: invalid key",
+			.records = {RECORD_TXT_AGE, RECORD_AAAA, RECORD_TXT_INVALID_KEY},
+			.error = true
+		},
+		{
+			.desc = "unexpected record type",
+			.records = {RECORD_TXT_AGE, RECORD_INVALID},
+			.error = true
+		},
+		{
+			.desc = "missing record: age",
+			.records = {RECORD_A, RECORD_TXT_PORT_444},
+			.error = true
+		},
+		{
+			.desc = "missing record: port for ipv4",
+			.records = {RECORD_TXT_AGE, RECORD_A},
+			.error = true
+		},
+		{
+			.desc = "missing record: port for ipv4 #2",
+			.records = {RECORD_TXT_AGE, RECORD_AAAA, RECORD_TXT_PORT_666, RECORD_A},
+			.error = true
+		},
+	};
+	int i = 0;
+	int j = 0;
+
+	fprintf(stderr, "-- %s --\n", __func__);
+	for (i = 0; i < ARRAY_SIZE(result_from_answer_data); i++) {
+		struct result_from_answer_test *t = &result_from_answer_data[i];
+		struct osmo_mdns_msg_answer ans = {0};
+		struct osmo_mslookup_result res = {0};
+		void *ctx_test = talloc_named_const(ctx, 0, t->desc);
+		bool is_error;
+
+		fprintf(stderr, "---\n");
+		fprintf(stderr, "test: %s\n", t->desc);
+		fprintf(stderr, "error: %s\n", t->error ? "true" : "false");
+		fprintf(stderr, "records:\n");
+		/* Build records list */
+		INIT_LLIST_HEAD(&ans.records);
+		for (j = 0; j < ARRAY_SIZE(t->records); j++) {
+			struct osmo_mdns_record *rec = NULL;
+
+			switch (t->records[j]) {
+				case RECORD_NONE:
+					break;
+				case RECORD_A:
+					fprintf(stderr, "- A 42.42.42.42\n");
+					rec = talloc_zero(ctx_test, struct osmo_mdns_record);
+					rec->type = OSMO_MDNS_RFC_RECORD_TYPE_A;
+					rec->data = ip_v4_n;
+					rec->length = sizeof(ip_v4_n);
+					break;
+				case RECORD_AAAA:
+					fprintf(stderr, "- AAAA 1122:3344:5566:7788:99aa:bbcc:ddee:ff00\n");
+					rec = talloc_zero(ctx_test, struct osmo_mdns_record);
+					rec->type = OSMO_MDNS_RFC_RECORD_TYPE_AAAA;
+					rec->data = ip_v6_n;
+					rec->length = sizeof(ip_v6_n);
+					break;
+				case RECORD_TXT_AGE:
+					fprintf(stderr, "- TXT age=3\n");
+					rec = osmo_mdns_record_txt_keyval_encode(ctx_test, "age", "3");
+					break;
+				case RECORD_TXT_PORT_444:
+					fprintf(stderr, "- TXT port=444\n");
+					rec = osmo_mdns_record_txt_keyval_encode(ctx_test, "port", "444");
+					break;
+				case RECORD_TXT_PORT_666:
+					fprintf(stderr, "- TXT port=666\n");
+					rec = osmo_mdns_record_txt_keyval_encode(ctx_test, "port", "666");
+					break;
+				case RECORD_TXT_INVALID_KEY:
+					fprintf(stderr, "- TXT hello=world\n");
+					rec = osmo_mdns_record_txt_keyval_encode(ctx_test, "hello", "world");
+					break;
+				case RECORD_TXT_INVALID_NO_KEY_VALUE:
+					fprintf(stderr, "- TXT 12345\n");
+					rec = osmo_mdns_record_txt_keyval_encode(ctx_test, "12", "45");
+					rec->data[3] = '3';
+					break;
+				case RECORD_INVALID:
+					fprintf(stderr, "- (invalid)\n");
+					rec = talloc_zero(ctx, struct osmo_mdns_record);
+					rec->type = OSMO_MDNS_RFC_RECORD_TYPE_UNKNOWN;
+					break;
+			}
+
+			if (rec)
+				llist_add_tail(&rec->list, &ans.records);
+		}
+
+		/* Verify output */
+		is_error = (osmo_mdns_result_from_answer(&res, &ans) != 0);
+		if (t->error != is_error) {
+			fprintf(stderr, "got %s\n", is_error ? "error" : "no error");
+			OSMO_ASSERT(false);
+		}
+		if (!t->error) {
+			fprintf(stderr, "exp: %s\n", osmo_mslookup_result_name_c(print_ctx, NULL, &t->res));
+			fprintf(stderr, "res: %s\n", osmo_mslookup_result_name_c(print_ctx, NULL, &res));
+			OSMO_ASSERT(t->res.rc == res.rc);
+			OSMO_ASSERT(!osmo_sockaddr_str_cmp(&t->res.host_v4, &res.host_v4));
+			OSMO_ASSERT(!osmo_sockaddr_str_cmp(&t->res.host_v6, &res.host_v6));
+			OSMO_ASSERT(t->res.age == res.age);
+			OSMO_ASSERT(t->res.last == res.last);
+		}
+
+		talloc_free(ctx_test);
+		fprintf(stderr, "=> OK\n");
+	}
+}
+
+int main()
+{
+	void *ctx = talloc_named_const(NULL, 0, "main");
+	osmo_init_logging2(ctx, NULL);
+
+	log_set_print_filename(osmo_stderr_target, 0);
+	log_set_print_level(osmo_stderr_target, 1);
+	log_set_print_category(osmo_stderr_target, 1);
+	log_set_print_category_hex(osmo_stderr_target, 0);
+	log_set_use_color(osmo_stderr_target, 0);
+
+	test_enc_dec_rfc_qname(ctx);
+	test_enc_dec_rfc_header();
+	test_enc_dec_rfc_header_einval();
+	test_enc_dec_rfc_question(ctx);
+	test_enc_dec_rfc_question_null(ctx);
+	test_enc_dec_rfc_record(ctx);
+
+	test_result_from_answer(ctx);
+
+	return 0;
+}