Add post-upgrade script for automatic db upgrade

Closes: OS#5138
Change-Id: I34e943e5020285f63a2162010eb42675af1520bf

.gitignore

@@ -1,8 +1,13 @@
 *.o
+*.lo
+*.la
 *.db
+*.db-shm
+*.db-wal
 *.pyc
 .*.sw?
 .version
+.tarball-version
 Makefile
 Makefile.in
 aclocal.m4
@@ -21,11 +26,47 @@ m4
 *.m4
 missing
 .deps
+*~
+
+*.pc
+.libs
 
-src/osmo-hlr
 src/db_test
+src/db_bootstrap.h
+src/osmo-hlr
+src/osmo-hlr-db-tool
+src/osmo-euse-demo
+src/gsupclient/gsup-test-client
+src/mslookup/osmo-mslookup-client
 
+tests/atconfig
 tests/testsuite
+tests/testsuite.log
+tests/testsuite.dir
+
 tests/auc/auc_3g_test
 tests/auc/auc_ts_55_205_test_sets.c
 tests/auc/auc_ts_55_205_test_sets
+tests/auc/auc_test
+tests/gsup_server/gsup_server_test
+tests/gsup/gsup_test
+tests/db/db_test
+tests/hlr_vty_test.db*
+tests/db_upgrade/*.dump
+tests/mslookup/mdns_test
+tests/mslookup/mslookup_client_mdns_test
+tests/mslookup/mslookup_client_test
+tests/mslookup/mslookup_test
+
+# manuals
+doc/manuals/*.html
+doc/manuals/*.svg
+doc/manuals/*.pdf
+doc/manuals/*__*.png
+doc/manuals/*.check
+doc/manuals/generated/
+doc/manuals/osmomsc-usermanual.xml
+doc/manuals/common
+doc/manuals/build
+
+contrib/osmo-hlr.spec

Makefile.am

@@ -2,14 +2,26 @@ AUTOMAKE_OPTIONS = foreign dist-bzip2
 
 SUBDIRS = \
 	src \
+	include \
+	doc \
 	sql \
+	contrib \
 	tests \
 	$(NULL)
 
 EXTRA_DIST = \
 	.version \
+	contrib/osmo-hlr.spec.in \
+	debian \
 	$(NULL)
 
+AM_DISTCHECK_CONFIGURE_FLAGS = \
+	--with-systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir)
+
+pkgconfigdir = $(libdir)/pkgconfig
+pkgconfig_DATA = libosmo-gsup-client.pc \
+		 libosmo-mslookup.pc
+
 @RELMAKE@
 
 BUILT_SOURCES = $(top_srcdir)/.version

README.md

@@ -0,0 +1,66 @@
+osmo-hlr - Osmocom HLR Implementation
+=====================================
+
+This repository contains a C-language implementation of a GSM Home
+Location Register (HLR). It is part of the
+[Osmocom](https://osmocom.org/) Open Source Mobile Communications
+project.
+
+Warning: While the HLR logical functionality is implemented, OsmoHLR
+does not use the ETSI/3GPP TCAP/MAP protocol stack. Instead, a much
+simpler custom protocol (GSUP) is used.  This means, OsmoHLR is of
+no use outside the context of an Osmocom core network.  You can use
+it with OsmoMSC, OsmoSGSN etc. - but not with third party components.
+
+Homepage
+--------
+
+The official homepage of the project is
+https://osmocom.org/projects/osmo-hlr/wiki
+
+GIT Repository
+--------------
+
+You can clone from the official osmo-hlr.git repository using
+
+	git clone git://git.osmocom.org/osmo-hlr.git
+	git clone https://git.osmocom.org/osmo-hlr.git
+
+There is a cgit interface at https://git.osmocom.org/osmo-hlr/
+
+Documentation
+-------------
+
+User Manuals and VTY reference manuals are [optionally] built in PDF form
+as part of the build process.
+
+Pre-rendered PDF version of the current "master" can be found at
+[User Manual](https://ftp.osmocom.org/docs/latest/osmohlr-usermanual.pdf)
+as well as the VTY reference manuals
+* [VTY Reference Manual for osmo-hlr](https://ftp.osmocom.org/docs/latest/osmohlr-vty-reference.pdf)
+
+Mailing List
+------------
+
+Discussions related to osmo-hlr are happening on the
+openbsc@lists.osmocom.org mailing list, please see
+https://lists.osmocom.org/mailman/listinfo/openbsc for subscription
+options and the list archive.
+
+Please observe the [Osmocom Mailing List
+Rules](https://osmocom.org/projects/cellular-infrastructure/wiki/Mailing_List_Rules)
+when posting.
+
+Contributing
+------------
+
+Our coding standards are described at
+https://osmocom.org/projects/cellular-infrastructure/wiki/Coding_standards
+
+We us a gerrit based patch submission/review process for managing
+contributions.  Please see
+https://osmocom.org/projects/cellular-infrastructure/wiki/Gerrit for
+more details
+
+The current patch queue for osmo-hlr can be seen at
+https://gerrit.osmocom.org/#/q/project:osmo-hlr+status:open

TODO-RELEASE

@@ -0,0 +1,9 @@
+# When cleaning up this file: bump API version in corresponding Makefile.am and rename corresponding debian/lib*.install
+# according to https://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html#Updating-version-info
+# In short:
+# LIBVERSION=c:r:a
+# If the library source code has changed at all since the last update, then increment revision: c:r + 1:a.
+# If any interfaces have been added, removed, or changed since the last update: c + 1:0:0.
+# If any interfaces have been added since the last public release: c:r:a + 1.
+# If any interfaces have been removed or changed since the last public release: c:r:0.
+#library        what            description / commit summary line

configure.ac

@@ -12,6 +12,8 @@ AM_INIT_AUTOMAKE([foreign dist-bzip2 no-dist-gzip 1.9])
 
 AC_CONFIG_TESTDIR(tests)
 
+CFLAGS="$CFLAGS -std=gnu11"
+
 dnl kernel style compile messages
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 
@@ -25,6 +27,11 @@ AC_PROG_MKDIR_P
 AC_PROG_CC
 AC_PROG_INSTALL
 
+dnl patching ${archive_cmds} to affect generation of file "libtool" to fix linking with clang
+AS_CASE(["$LD"],[*clang*],
+  [AS_CASE(["${host_os}"],
+     [*linux*],[archive_cmds='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-soname $wl$soname -o $lib'])])
+
 dnl check for pkg-config (explained in detail in libosmocore/configure.ac)
 AC_PATH_PROG(PKG_CONFIG_INSTALLED, pkg-config, no)
 if test "x$PKG_CONFIG_INSTALLED" = "xno"; then
@@ -34,11 +41,11 @@ PKG_PROG_PKG_CONFIG([0.20])
 
 PKG_CHECK_MODULES(TALLOC, [talloc >= 2.0.1])
 
-PKG_CHECK_MODULES(LIBOSMOCORE, libosmocore >= 0.9.5)
-PKG_CHECK_MODULES(LIBOSMOGSM, libosmogsm >= 0.9.0)
-PKG_CHECK_MODULES(LIBOSMOVTY, libosmovty >= 0.9.0)
-PKG_CHECK_MODULES(LIBOSMOCTRL, libosmoctrl)
-PKG_CHECK_MODULES(LIBOSMOABIS, libosmoabis >= 0.3.2)
+PKG_CHECK_MODULES(LIBOSMOCORE, libosmocore >= 1.5.0)
+PKG_CHECK_MODULES(LIBOSMOGSM, libosmogsm >= 1.5.0)
+PKG_CHECK_MODULES(LIBOSMOVTY, libosmovty >= 1.5.0)
+PKG_CHECK_MODULES(LIBOSMOCTRL, libosmoctrl >= 1.5.0)
+PKG_CHECK_MODULES(LIBOSMOABIS, libosmoabis >= 1.1.0)
 
 PKG_CHECK_MODULES(SQLITE3, sqlite3)
 
@@ -47,6 +54,51 @@ AC_CONFIG_MACRO_DIR([m4])
 dnl checks for header files
 AC_HEADER_STDC
 
+AC_ARG_ENABLE(sanitize,
+	[AS_HELP_STRING(
+		[--enable-sanitize],
+		[Compile with address sanitizer enabled],
+	)],
+	[sanitize=$enableval], [sanitize="no"])
+if test x"$sanitize" = x"yes"
+then
+	CFLAGS="$CFLAGS -fsanitize=address -fsanitize=undefined"
+	CPPFLAGS="$CPPFLAGS -fsanitize=address -fsanitize=undefined"
+fi
+
+AC_ARG_ENABLE([sqlite_talloc],
+		AC_HELP_STRING([--enable-sqlite-talloc],
+				[Configure SQLite3 to use talloc memory allocator [default=no]]),
+		[sqlite_talloc="$enableval"],[sqlite_talloc="no"])
+if test "x$sqlite_talloc" = "xyes" ; then
+	# Older versions of SQLite3 (at least 3.8.2) become unstable with talloc.
+	# Feel free to relax to 3.24.0 > VER > 3.8.2 if it works for you.
+	# FIXME: PKG_CHECK_MODULES() may return cached result here!
+	PKG_CHECK_MODULES(SQLITE3, sqlite3 >= 3.24.0)
+	AC_DEFINE([SQLITE_USE_TALLOC], 1, [Use talloc for SQLite3])
+fi
+AC_MSG_CHECKING([whether to use talloc for SQLite3])
+AC_MSG_RESULT([$sqlite_talloc])
+AM_CONDITIONAL([DB_SQLITE_DEBUG], [test "x$sqlite_talloc" = "xyes"])
+
+AC_ARG_ENABLE(werror,
+	[AS_HELP_STRING(
+		[--enable-werror],
+		[Turn all compiler warnings into errors, with exceptions:
+		 a) deprecation (allow upstream to mark deprecation without breaking builds);
+		 b) "#warning" pragmas (allow to remind ourselves of errors without breaking builds)
+		]
+	)],
+	[werror=$enableval], [werror="no"])
+if test x"$werror" = x"yes"
+then
+	WERROR_FLAGS="-Werror"
+	WERROR_FLAGS+=" -Wno-error=deprecated -Wno-error=deprecated-declarations"
+	WERROR_FLAGS+=" -Wno-error=cpp" # "#warning"
+	CFLAGS="$CFLAGS $WERROR_FLAGS"
+	CPPFLAGS="$CPPFLAGS $WERROR_FLAGS"
+fi
+
 AC_ARG_ENABLE([external_tests],
 		AC_HELP_STRING([--enable-external-tests],
 				[Include the VTY/CTRL tests in make check [default=no]]),
@@ -62,13 +114,101 @@ AC_MSG_CHECKING([whether to enable VTY/CTRL tests])
 AC_MSG_RESULT([$enable_ext_tests])
 AM_CONDITIONAL(ENABLE_EXT_TESTS, test "x$enable_ext_tests" = "xyes")
 
+# mslookup_client_mdns_test (OS#4385: does not work everywhere)
+AC_ARG_ENABLE([mslookup_client_mdns_test],
+		AC_HELP_STRING([--enable-mslookup-client-mdns-test],
+				[Include the mslookup_client_mdns_test in make check [default=no]]),
+		[enable_mslookup_client_mdns_test="$enableval"],[enable_mslookup_client_mdns_test="no"])
+AC_MSG_CHECKING([whether to enable mslookup_client_mdns_test])
+AC_MSG_RESULT([$enable_mslookup_client_mdns_test])
+AM_CONDITIONAL(ENABLE_MSLOOKUP_CLIENT_MDNS_TEST, test "x$enable_mslookup_client_mdns_test" = "xyes")
+
+# Generate manuals
+AC_ARG_ENABLE(manuals,
+	[AS_HELP_STRING(
+		[--enable-manuals],
+		[Generate manual PDFs [default=no]],
+	)],
+	[osmo_ac_build_manuals=$enableval], [osmo_ac_build_manuals="no"])
+AM_CONDITIONAL([BUILD_MANUALS], [test x"$osmo_ac_build_manuals" = x"yes"])
+AC_ARG_VAR(OSMO_GSM_MANUALS_DIR, [path to common osmo-gsm-manuals files, overriding pkg-config and "../osmo-gsm-manuals"
+	fallback])
+if test x"$osmo_ac_build_manuals" = x"yes"
+then
+	# Find OSMO_GSM_MANUALS_DIR (env, pkg-conf, fallback)
+	if test -n "$OSMO_GSM_MANUALS_DIR"; then
+		echo "checking for OSMO_GSM_MANUALS_DIR... $OSMO_GSM_MANUALS_DIR (from env)"
+	else
+		OSMO_GSM_MANUALS_DIR="$($PKG_CONFIG osmo-gsm-manuals --variable=osmogsmmanualsdir 2>/dev/null)"
+		if test -n "$OSMO_GSM_MANUALS_DIR"; then
+			echo "checking for OSMO_GSM_MANUALS_DIR... $OSMO_GSM_MANUALS_DIR (from pkg-conf)"
+		else
+			OSMO_GSM_MANUALS_DIR="../osmo-gsm-manuals"
+			echo "checking for OSMO_GSM_MANUALS_DIR... $OSMO_GSM_MANUALS_DIR (fallback)"
+		fi
+	fi
+	if ! test -d "$OSMO_GSM_MANUALS_DIR"; then
+		AC_MSG_ERROR("OSMO_GSM_MANUALS_DIR does not exist! Install osmo-gsm-manuals or set OSMO_GSM_MANUALS_DIR.")
+	fi
+
+	# Find and run check-depends
+	CHECK_DEPENDS="$OSMO_GSM_MANUALS_DIR/check-depends.sh"
+	if ! test -x "$CHECK_DEPENDS"; then
+		CHECK_DEPENDS="osmo-gsm-manuals-check-depends"
+	fi
+	if ! $CHECK_DEPENDS; then
+		AC_MSG_ERROR("missing dependencies for --enable-manuals")
+	fi
+
+	# Put in Makefile with absolute path
+	OSMO_GSM_MANUALS_DIR="$(realpath "$OSMO_GSM_MANUALS_DIR")"
+	AC_SUBST([OSMO_GSM_MANUALS_DIR])
+fi
+
+# https://www.freedesktop.org/software/systemd/man/daemon.html
+AC_ARG_WITH([systemdsystemunitdir],
+     [AS_HELP_STRING([--with-systemdsystemunitdir=DIR], [Directory for systemd service files])],,
+     [with_systemdsystemunitdir=auto])
+AS_IF([test "x$with_systemdsystemunitdir" = "xyes" -o "x$with_systemdsystemunitdir" = "xauto"], [
+     def_systemdsystemunitdir=$($PKG_CONFIG --variable=systemdsystemunitdir systemd)
+
+     AS_IF([test "x$def_systemdsystemunitdir" = "x"],
+   [AS_IF([test "x$with_systemdsystemunitdir" = "xyes"],
+    [AC_MSG_ERROR([systemd support requested but pkg-config unable to query systemd package])])
+    with_systemdsystemunitdir=no],
+   [with_systemdsystemunitdir="$def_systemdsystemunitdir"])])
+AS_IF([test "x$with_systemdsystemunitdir" != "xno"],
+      [AC_SUBST([systemdsystemunitdir], [$with_systemdsystemunitdir])])
+AM_CONDITIONAL([HAVE_SYSTEMD], [test "x$with_systemdsystemunitdir" != "xno"])
+
+AC_MSG_RESULT([CFLAGS="$CFLAGS"])
+AC_MSG_RESULT([CPPFLAGS="$CPPFLAGS"])
+
 AC_OUTPUT(
 	Makefile
+	doc/Makefile
+	doc/examples/Makefile
 	src/Makefile
+	src/gsupclient/Makefile
+	src/mslookup/Makefile
+	include/Makefile
+	include/osmocom/Makefile
+	include/osmocom/hlr/Makefile
+	include/osmocom/mslookup/Makefile
+	libosmo-gsup-client.pc
+	libosmo-mslookup.pc
 	sql/Makefile
+	doc/manuals/Makefile
+	contrib/Makefile
+	contrib/systemd/Makefile
+	contrib/dgsm/Makefile
+	contrib/osmo-hlr.spec
 	tests/Makefile
 	tests/auc/Makefile
 	tests/auc/gen_ts_55_205_test_sets/Makefile
 	tests/gsup_server/Makefile
+	tests/gsup/Makefile
 	tests/db/Makefile
+	tests/db_upgrade/Makefile
+	tests/mslookup/Makefile
 	)

contrib/Makefile.am

@@ -0,0 +1,14 @@
+SUBDIRS = \
+	systemd \
+	dgsm \
+	$(NULL)
+
+EXTRA_DIST = osmo-hlr-post-upgrade.sh
+
+install-data-hook:
+	install -Dm755 $(srcdir)/osmo-hlr-post-upgrade.sh \
+		-t $(DESTDIR)$(datadir)/osmocom/
+
+uninstall-hook:
+	@$(PRE_UNINSTALL)
+	$(RM) $(DESTDIR)$(datadir)/osmocom/osmo-hlr-post-upgrade.sh

contrib/dgsm/Makefile.am

@@ -0,0 +1,6 @@
+EXTRA_DIST = \
+	esme_dgsm.py \
+	freeswitch_dialplan_dgsm.py \
+	osmo-mslookup-pipe.py \
+	osmo-mslookup-socket.py \
+	$(NULL)

contrib/dgsm/esme_dgsm.py

@@ -0,0 +1,184 @@
+#!/usr/bin/env python3
+"""
+SPDX-License-Identifier: MIT
+Copyright 2019 sysmocom s.f.m.c GmbH <info@sysmocom.de>
+
+WARNING: this is just a proof-of-concept implementation, it blocks for every
+received SMPP request and is not suitable for servicing more than one request
+at a time.
+
+Based on esme.py from RCCN (license changed with permission from author):
+https://github.com/Rhizomatica/rccn/blob/master/rccn/esme.py
+Copyright 2017 keith <keith@rhizomatica.org>
+
+Forward SMS to the receiver's SMSC, as determined with mslookup.
+Requires smpplip (pip3 install --user smpplib) and osmo-mslookup-client.
+
+Example SMPP configuration for osmo-msc.cfg:
+smpp
+ local-tcp-ip 127.0.0.1 2775
+ policy closed
+ smpp-first
+# outgoing to esme_dgsm.py
+ esme OSMPP
+  no alert-notifications
+  password foo
+  default-route
+# incoming from esme_dgsm.py
+ esme ISMPP
+  no alert-notifications
+  password foo
+"""
+import argparse
+import json
+import logging
+import smpplib
+import subprocess
+import time
+
+
+def can_handle_pdu(pdu):
+    if not isinstance(pdu, smpplib.command.DeliverSM):
+        logging.info('PDU is not a DeliverSM, ignoring')
+        return False
+
+    if int(pdu.dest_addr_ton) == smpplib.consts.SMPP_TON_INTL:
+        logging.info("Unable to handle SMS for %s: SMPP_TON_INTL" %
+                     (pdu.destination_addr))
+        return False
+
+    return True
+
+
+def query_mslookup(service_type, id, id_type='msisdn'):
+    query_str = '%s.%s.%s' % (service_type, id, id_type)
+    logging.info('mslookup: ' + query_str)
+
+    result_line = subprocess.check_output(['osmo-mslookup-client', query_str,
+                                           '-f', 'json'])
+    if isinstance(result_line, bytes):
+        result_line = result_line.decode('ascii')
+
+    logging.info('mslookup result: ' + result_line.rstrip())
+    return json.loads(result_line)
+
+
+def tx_sms(dst_host, dst_port, source, destination, registered_delivery,
+           unicode_text):
+    smpp_client = smpplib.client.Client(dst_host, dst_port, 90)
+    smpp_client.connect()
+    smpp_client.bind_transceiver(system_id=args.dst_id, password=args.dst_pass)
+    logging.info('Connected to destination SMSC (%s@%s:%s)' % (args.dst_id,
+                 dst_host, dst_port))
+
+    pdu = smpp_client.send_message(
+        source_addr_ton=smpplib.consts.SMPP_TON_ALNUM,
+        source_addr_npi=smpplib.consts.SMPP_NPI_UNK,
+        source_addr=source.decode(),
+        dest_addr_ton=smpplib.consts.SMPP_TON_SBSCR,
+        dest_addr_npi=smpplib.consts.SMPP_NPI_ISDN,
+        destination_addr=destination.decode(),
+        short_message=unicode_text,
+        registered_delivery=registered_delivery,
+    )
+
+    smpp_client.unbind()
+    smpp_client.disconnect()
+    del pdu
+    del smpp_client
+
+
+def rx_deliver_sm(pdu):
+    if not can_handle_pdu(pdu):
+        return smpplib.consts.SMPP_ESME_RSYSERR
+
+    msisdn = pdu.destination_addr.decode()
+    logging.info("Incoming SMS for: " + msisdn)
+
+    if args.sleep:
+        logging.info("Sleeping for %i seconds" % (args.sleep))
+        time.sleep(args.sleep)
+        logging.info("Sleep done")
+
+    if args.always_fail is not None:
+        return args.always_fail
+
+    result = query_mslookup("smpp.sms", msisdn)
+    if 'v4' not in result or not result['v4']:
+        logging.info('No IPv4 result from mslookup! This example only'
+                     ' makes use of IPv4, dropping.')
+        return smpplib.consts.SMPP_ESME_RSYSERR
+
+    dst_host, dst_port = result['v4']
+    tx_sms(dst_host, dst_port, pdu.source_addr,
+           pdu.destination_addr, int(pdu.registered_delivery),
+           pdu.short_message)
+
+    return smpplib.consts.SMPP_ESME_ROK
+
+
+def smpp_bind():
+    client = smpplib.client.Client(args.src_host, args.src_port, 90)
+    client.set_message_received_handler(rx_deliver_sm)
+    client.connect()
+    client.bind_transceiver(system_id=args.src_id, password=args.src_pass)
+    logging.info('Connected to source SMSC (%s@%s:%s)' % (args.src_id,
+                 args.src_host, args.src_port))
+    logging.info('Waiting for SMS...')
+    client.listen()
+
+
+def main():
+    global args
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--src-host', default='127.0.0.1',
+                        help='source SMSC (OsmoMSC) host (default: 127.0.0.1)')
+    parser.add_argument('--src-port', default=2775, type=int,
+                        help='source SMSC (OsmoMSC) port (default: 2775)')
+    parser.add_argument('--src-id', default='OSMPP',
+                        help='source system id, as configured in osmo-msc.cfg'
+                             ' (default: OSMPP)')
+    parser.add_argument('--src-pass', default='foo',
+                        help='source system password, as configured in'
+                             ' osmo-msc.cfg (default: foo)')
+    parser.add_argument('--dst-id', default='ISMPP',
+                        help='destination system id, as configured in'
+                             ' osmo-msc.cfg (default: ISMPP)')
+    parser.add_argument('--dst-pass', default='foo',
+                        help='destination system password, as configured in'
+                             ' osmo-msc.cfg (default: foo)')
+    parser.add_argument('--sleep', default=0, type=float,
+                        help='sleep time in seconds before forwarding an SMS,'
+                             ' to test multithreading (default: 0)')
+    parser.add_argument('--always-fail', default=None, metavar='SMPP_ESME_ERRCODE',
+                        help='test delivery failure: always return an error code on Deliver-SM,'
+                        ' pass an smpplib error code name like RDELIVERYFAILURE (see smpplib/consts.py),'
+                        ' or an SMPP error code in hex digits')
+    args = parser.parse_args()
+
+    logging.basicConfig(level=logging.INFO, format='[%(asctime)s]'
+                        ' (%(threadName)s) %(message)s', datefmt="%H:%M:%S")
+
+    if args.always_fail:
+        resolved = None
+        name = 'SMPP_ESME_' + args.always_fail
+        if hasattr(smpplib.consts, name):
+            resolved = getattr(smpplib.consts, name)
+        if resolved is None:
+            try:
+                resolved = int(args.always_fail, 16)
+            except ValueError:
+                resolved = None
+        if resolved is None:
+            print('Invalid argument for --always-fail: %r' % args.always_fail)
+            exit(1)
+        args.always_fail = resolved
+        logging.info('--always-fail: returning error code %s to all Deliver-SM' % hex(args.always_fail))
+
+    smpp_bind()
+
+
+if __name__ == "__main__":
+    main()
+
+# vim: expandtab tabstop=4 shiftwidth=4

contrib/dgsm/freeswitch_dialplan_dgsm.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""
+SPDX-License-Identifier: MIT
+Copyright 2019 sysmocom s.f.m.c GmbH <info@sysmocom.de>
+
+This is a freeswitch dialplan implementation, see:
+https://freeswitch.org/confluence/display/FREESWITCH/mod_python
+
+Find the right SIP server with mslookup (depending on the destination number)
+and bridge calls accordingly.
+"""
+import json
+import subprocess
+
+
+def query_mslookup(service_type, id, id_type='msisdn'):
+    query_str = '%s.%s.%s' % (service_type, id, id_type)
+    print('[dialplan-dgsm] mslookup: ' + query_str)
+
+    result_line = subprocess.check_output([
+        'osmo-mslookup-client', query_str, '-f', 'json'])
+    if isinstance(result_line, bytes):
+        result_line = result_line.decode('ascii')
+
+    print('[dialplan-dgsm] mslookup result: ' + result_line)
+    return json.loads(result_line)
+
+
+def handler(session, args):
+    """ Handle calls: bridge to the SIP server found with mslookup. """
+    print('[dialplan-dgsm] call handler')
+    msisdn = session.getVariable('destination_number')
+
+    # Run osmo-mslookup-client binary. We have also tried to directly call the
+    # C functions with ctypes but this has lead to hard-to-debug segfaults.
+    try:
+        result = query_mslookup("sip.voice", msisdn)
+
+        # This example only makes use of IPv4
+        if not result['v4']:
+            print('[dialplan-dgsm] no IPv4 result from mslookup')
+            session.hangup('UNALLOCATED_NUMBER')
+            return
+
+        sip_ip, sip_port = result['v4']
+        dial_str = 'sofia/internal/sip:{}@{}:{}'.format(
+            msisdn, sip_ip, sip_port)
+        print('[dialplan-dgsm] dial_str: ' + str(dial_str))
+
+        session.execute('bridge', dial_str)
+    except:
+        print('[dialplan-dgsm]: exception during call handler')
+        session.hangup('UNALLOCATED_NUMBER')
+
+
+def fsapi(session, stream, env, args):
+    """ Freeswitch refuses to load the module without this. """
+    stream.write(env.serialize())
+
+
+def main():
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('id', type=int)
+    parser.add_argument('-i', '--id-type', default='msisdn',
+                        help='default: "msisdn"')
+    parser.add_argument('-s', '--service', default='sip.voice',
+                        help='default: "sip.voice"')
+    args = parser.parse_args()
+
+    result = query_mslookup(args.service, args.id, args.id_type)
+    print(json.dumps(result))
+
+
+if __name__ == '__main__':
+    main()

contrib/dgsm/osmo-mslookup-pipe.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+# vim: shiftwidth=4 tabstop=4 expandtab
+import subprocess
+import json
+
+def query_mslookup(query_str):
+    result = {'result': 'not-found'}
+    proc = subprocess.Popen(('osmo-mslookup-client', '-f', 'json', query_str),
+		            stdout=subprocess.PIPE)
+    for line in iter(proc.stdout.readline,''):
+        if not line:
+            break
+        response = json.loads(line)
+        if response.get('result') == 'result':
+                result = response
+        print('Response: %r' % response)
+    return result
+
+if __name__ == '__main__':
+    import sys
+    query_str = '1000-5000@sip.voice.12345.msisdn'
+    if len(sys.argv) > 1:
+        query_str = sys.argv[1]
+    print('Final result: %r' % query_mslookup(query_str))

contrib/dgsm/osmo-mslookup-socket.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+# vim: shiftwidth=4 tabstop=4 expandtab
+import socket
+import time
+
+MSLOOKUP_SOCKET_PATH = '/tmp/mslookup'
+
+def query_mslookup_socket(query_str, socket_path=MSLOOKUP_SOCKET_PATH):
+    mslookup_socket = socket.socket(socket.AF_UNIX, socket.SOCK_SEQPACKET)
+    mslookup_socket.setblocking(True)
+    mslookup_socket.connect(socket_path)
+    result = {'result': 'not-found'}
+    column_names = mslookup_socket.recv(1024).decode('ascii')
+    if not column_names:
+        return result
+    column_names = column_names.split('\t')
+    mslookup_socket.sendall(query_str.encode('ascii'))
+    while True:
+        csv = mslookup_socket.recv(1024).decode('ascii')
+        if not csv:
+            break
+        response = dict(zip(column_names, csv.split('\t')))
+        if response.get('result') == 'result':
+            result = response
+        print('Response: %r' % response)
+    return result
+
+if __name__ == '__main__':
+    import sys
+    print(
+        '\nPlease run separately: osmo-mslookup-client --socket /tmp/mslookup -d\n')
+    query_str = '1000-5000@sip.voice.12345.msisdn'
+    if len(sys.argv) > 1:
+        query_str = sys.argv[1]
+    print('Final result: %r' % query_mslookup_socket(query_str))

contrib/ipa.py

@@ -1,278 +0,0 @@
-#!/usr/bin/python3
-# -*- mode: python-mode; py-indent-tabs-mode: nil -*-
-"""
-/*
- * Copyright (C) 2016 sysmocom s.f.m.c. GmbH
- *
- * All Rights Reserved
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- */
-"""
-
-import struct, random, sys
-
-class IPA(object):
-    """
-    Stateless IPA protocol multiplexer: add/remove/parse (extended) header
-    """
-    version = "0.0.5"
-    TCP_PORT_OML = 3002
-    TCP_PORT_RSL = 3003
-    # OpenBSC extensions: OSMO, MGCP_OLD
-    PROTO = dict(RSL=0x00, CCM=0xFE, SCCP=0xFD, OML=0xFF, OSMO=0xEE, MGCP_OLD=0xFC)
-    # ...OML Router Control, GSUP GPRS extension, Osmocom Authn Protocol
-    EXT = dict(CTRL=0, MGCP=1, LAC=2, SMSC=3, ORC=4, GSUP=5, OAP=6)
-    # OpenBSC extension: SCCP_OLD
-    MSGT = dict(PING=0x00, PONG=0x01, ID_GET=0x04, ID_RESP=0x05, ID_ACK=0x06, SCCP_OLD=0xFF)
-    _IDTAG = dict(SERNR=0, UNITNAME=1, LOCATION=2, TYPE=3, EQUIPVERS=4, SWVERSION=5, IPADDR=6, MACADDR=7, UNIT=8)
-    CTRL_GET = 'GET'
-    CTRL_SET = 'SET'
-    CTRL_REP = 'REPLY'
-    CTRL_ERR = 'ERR'
-    CTRL_TRAP = 'TRAP'
-
-    def _l(self, d, p):
-        """
-        Reverse dictionary lookup: return key for a given value
-        """
-        if p is None:
-            return 'UNKNOWN'
-        return list(d.keys())[list(d.values()).index(p)]
-
-    def _tag(self, t, v):
-        """
-        Create TAG as TLV data
-        """
-        return struct.pack(">HB", len(v) + 1, t) + v
-
-    def proto(self, p):
-        """
-        Lookup protocol name
-        """
-        return self._l(self.PROTO, p)
-
-    def ext(self, p):
-        """
-        Lookup protocol extension name
-        """
-        return self._l(self.EXT, p)
-
-    def msgt(self, p):
-        """
-        Lookup message type name
-        """
-        return self._l(self.MSGT, p)
-
-    def idtag(self, p):
-        """
-        Lookup ID tag name
-        """
-        return self._l(self._IDTAG, p)
-
-    def ext_name(self, proto, exten):
-        """
-        Return proper extension byte name depending on the protocol used
-        """
-        if self.PROTO['CCM'] == proto:
-            return self.msgt(exten)
-        if self.PROTO['OSMO'] == proto:
-            return self.ext(exten)
-        return None
-
-    def add_header(self, data, proto, ext=None):
-        """
-        Add IPA header (with extension if necessary), data must be represented as bytes
-        """
-        if ext is None:
-            return struct.pack(">HB", len(data) + 1, proto) + data
-        return struct.pack(">HBB", len(data) + 1, proto, ext) + data
-
-    def del_header(self, data):
-        """
-        Strip IPA protocol header correctly removing extension if present
-        Returns data length, IPA protocol, extension (or None if not defined for a give protocol) and the data without header
-        """
-        if not len(data):
-            return None, None, None, None
-        (dlen, proto) = struct.unpack('>HB', data[:3])
-        if self.PROTO['OSMO'] == proto or self.PROTO['CCM'] == proto: # there's extension which we have to unpack
-            return struct.unpack('>HBB', data[:4]) + (data[4:], ) # length, protocol, extension, data
-        return dlen, proto, None, data[3:] # length, protocol, _, data
-
-    def split_combined(self, data):
-        """
-        Split the data which contains multiple concatenated IPA messages into tuple (first, rest) where rest contains remaining messages, first is the single IPA message
-        """
-        (length, _, _, _) = self.del_header(data)
-        return data[:(length + 3)], data[(length + 3):]
-
-    def tag_serial(self, data):
-        """
-        Make TAG for serial number
-        """
-        return self._tag(self._IDTAG['SERNR'], data)
-
-    def tag_name(self, data):
-        """
-        Make TAG for unit name
-        """
-        return self._tag(self._IDTAG['UNITNAME'], data)
-
-    def tag_loc(self, data):
-        """
-        Make TAG for location
-        """
-        return self._tag(self._IDTAG['LOCATION'], data)
-
-    def tag_type(self, data):
-        """
-        Make TAG for unit type
-        """
-        return self._tag(self._IDTAG['TYPE'], data)
-
-    def tag_equip(self, data):
-        """
-        Make TAG for equipment version
-        """
-        return self._tag(self._IDTAG['EQUIPVERS'], data)
-
-    def tag_sw(self, data):
-        """
-        Make TAG for software version
-        """
-        return self._tag(self._IDTAG['SWVERSION'], data)
-
-    def tag_ip(self, data):
-        """
-        Make TAG for IP address
-        """
-        return self._tag(self._IDTAG['IPADDR'], data)
-
-    def tag_mac(self, data):
-        """
-        Make TAG for MAC address
-        """
-        return self._tag(self._IDTAG['MACADDR'], data)
-
-    def tag_unit(self, data):
-        """
-        Make TAG for unit ID
-        """
-        return self._tag(self._IDTAG['UNIT'], data)
-
-    def identity(self, unit=b'', mac=b'', location=b'', utype=b'', equip=b'', sw=b'', name=b'', serial=b''):
-        """
-        Make IPA IDENTITY tag list, by default returns empty concatenated bytes of tag list
-        """
-        return self.tag_unit(unit) + self.tag_mac(mac) + self.tag_loc(location) + self.tag_type(utype) + self.tag_equip(equip) + self.tag_sw(sw) + self.tag_name(name) + self.tag_serial(serial)
-
-    def ping(self):
-        """
-        Make PING message
-        """
-        return self.add_header(b'', self.PROTO['CCM'], self.MSGT['PING'])
-
-    def pong(self):
-        """
-        Make PONG message
-        """
-        return self.add_header(b'', self.PROTO['CCM'], self.MSGT['PONG'])
-
-    def id_ack(self):
-        """
-        Make ID_ACK CCM message
-        """
-        return self.add_header(b'', self.PROTO['CCM'], self.MSGT['ID_ACK'])
-
-    def id_get(self):
-        """
-        Make ID_GET CCM message
-        """
-        return self.add_header(self.identity(), self.PROTO['CCM'], self.MSGT['ID_GET'])
-
-    def id_resp(self, data):
-        """
-        Make ID_RESP CCM message
-        """
-        return self.add_header(data, self.PROTO['CCM'], self.MSGT['ID_RESP'])
-
-class Ctrl(IPA):
-    """
-    Osmocom CTRL protocol implemented on top of IPA multiplexer
-    """
-    def __init__(self):
-        random.seed()
-
-    def add_header(self, data):
-        """
-        Add CTRL header
-        """
-        return super(Ctrl, self).add_header(data.encode('utf-8'), IPA.PROTO['OSMO'], IPA.EXT['CTRL'])
-
-    def rem_header(self, data):
-        """
-        Remove CTRL header, check for appropriate protocol and extension
-        """
-        (_, proto, ext, d) = super(Ctrl, self).del_header(data)
-        if self.PROTO['OSMO'] != proto or self.EXT['CTRL'] != ext:
-            return None
-        return d
-
-    def parse(self, data, op=None):
-        """
-        Parse Ctrl string returning (var, value) pair
-        var could be None in case of ERROR message
-        value could be None in case of GET message
-        """
-        (s, i, v) = data.split(' ', 2)
-        if s == self.CTRL_ERR:
-            return None, v
-        if s == self.CTRL_GET:
-            return v, None
-        (s, i, var, val) = data.split(' ', 3)
-        if s == self.CTRL_TRAP and i != '0':
-            return None, '%s with non-zero id %s' % (s, i)
-        if op is not None and i != op:
-            if s == self.CTRL_GET + '_' + self.CTRL_REP or s == self.CTRL_SET + '_' + self.CTRL_REP:
-                return None, '%s with unexpected id %s' % (s, i)
-        return var, val
-
-    def trap(self, var, val):
-        """
-        Make TRAP message with given (vak, val) pair
-        """
-        return self.add_header("%s 0 %s %s" % (self.CTRL_TRAP, var, val))
-
-    def cmd(self, var, val=None):
-        """
-        Make SET/GET command message: returns (r, m) tuple where r is random operation id and m is assembled message
-        """
-        r = random.randint(1, sys.maxsize)
-        if val is not None:
-            return r, self.add_header("%s %s %s %s" % (self.CTRL_SET, r, var, val))
-        return r, self.add_header("%s %s %s" % (self.CTRL_GET, r, var))
-
-    def verify(self, reply, r, var, val=None):
-        """
-        Verify reply to SET/GET command: returns (b, v) tuple where v is True/False verification result and v is the variable value
-        """
-        (k, v) = self.parse(reply)
-        if k != var or (val is not None and v != val):
-            return False, v
-        return True, v
-
-if __name__ == '__main__':
-    print("IPA multiplexer v%s loaded." % IPA.version)

contrib/jenkins.sh

@@ -1,5 +1,10 @@
 #!/bin/sh
 # jenkins build helper script for osmo-hlr.  This is how we build on jenkins.osmocom.org
+#
+# environment variables:
+# * WITH_MANUALS: build manual PDFs if set to "1"
+# * PUBLISH: upload manuals after building if set to "1" (ignored without WITH_MANUALS = "1")
+#
 
 if ! [ -x "$(command -v osmo-build-dep.sh)" ]; then
 	echo "Error: We need to have scripts/osmo-deps.sh from http://git.osmocom.org/osmo-ci/ in PATH !"
@@ -14,17 +19,25 @@ deps="$base/deps"
 inst="$deps/install"
 export deps inst
 
+osmo-clean-workspace.sh
+
 mkdir "$deps" || true
-rm -rf "$inst"
 
 verify_value_string_arrays_are_terminated.py $(find . -name "*.[hc]")
 
 export PKG_CONFIG_PATH="$inst/lib/pkgconfig:$PKG_CONFIG_PATH"
 export LD_LIBRARY_PATH="$inst/lib"
+export PATH="$inst/bin:$PATH"
 
 osmo-build-dep.sh libosmocore "" ac_cv_path_DOXYGEN=false
 osmo-build-dep.sh libosmo-abis
 
+# Additional configure options and depends
+CONFIG=""
+if [ "$WITH_MANUALS" = "1" ]; then
+	CONFIG="--enable-manuals"
+fi
+
 set +x
 echo
 echo
@@ -35,9 +48,19 @@ set -x
 
 cd "$base"
 autoreconf --install --force
-./configure --enable-external-tests
+./configure \
+	--enable-sanitize \
+	--enable-external-tests \
+	--enable-mslookup-client-mdns-test \
+	--enable-werror \
+	$CONFIG
 $MAKE $PARALLEL_MAKE
-if [ "x$label" != "xFreeBSD_amd64" ]; then
-    $MAKE check || cat-testlogs.sh
-    $MAKE distcheck || cat-testlogs.sh
+$MAKE check || cat-testlogs.sh
+DISTCHECK_CONFIGURE_FLAGS="$CONFIG" $MAKE $PARALLEL_MAKE distcheck || cat-testlogs.sh
+
+if [ "$WITH_MANUALS" = "1" ] && [ "$PUBLISH" = "1" ]; then
+	make -C "$base/doc/manuals" publish
 fi
+
+$MAKE $PARALLEL_MAKE maintainer-clean
+osmo-clean-workspace.sh

contrib/osmo-hlr-post-upgrade.sh

@@ -0,0 +1,95 @@
+#!/bin/sh -e
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright 2021 sysmocom s.f.m.c GmbH <info@sysmocom.de>
+#
+# Packagers are supposed to call this script in post-upgrade, so it can safely
+# upgrade the database scheme if required.
+
+DB="/var/lib/osmocom/hlr.db"
+IS_ACTIVE=0
+
+msg() {
+	echo "osmo-hlr-post-upgrade: $@"
+}
+
+err() {
+	msg "ERROR: $@"
+}
+
+open_db() {
+	# Attempt to open the database with osmo-hlr-db-tool, it will fail if
+	# upgrading the schema is required
+	osmo-hlr-db-tool -s -l "$DB" create
+}
+
+check_upgrade_required() {
+	if ! [ -e "$DB" ]; then
+		msg "nothing to do (no existing database)"
+		exit 0
+	fi
+	
+	if open_db 2>/dev/null; then
+		msg "nothing to do (database version is up to date)"
+		exit 0
+	fi
+
+	msg "database upgrade is required"
+}
+
+stop_service() {
+	if systemctl is-active -q osmo-hlr; then
+		IS_ACTIVE=1
+		msg "stopping osmo-hlr service"
+		systemctl stop osmo-hlr
+
+		# Verify that it stopped
+		for i in $(seq 1 100); do
+			if ! systemctl is-active -q osmo-hlr; then
+				return
+			fi
+			sleep 0.1
+		done
+
+		err "failed to stop osmo-hlr service"
+		exit 1
+	else
+		msg "osmo-hlr service is not running"
+	fi
+}
+
+create_backup() {
+	backup="$DB.$(date +%Y%m%d%H%M%S).bak"
+	msg "creating backup: $backup"
+	if [ -e "$backup" ]; then
+		err "backup already exists: $backup"
+		exit 1
+	fi
+	cp "$DB" "$backup"
+}
+
+upgrade() {
+	msg "performing database upgrade"
+	osmo-hlr-db-tool -s -U -l "$DB" create
+	
+	if ! open_db 2>/dev/null; then
+		err "failed to open the database after upgrade"
+		err "osmo-hlr-db-tool output:"
+		open_db
+		# exit because of "set -e"
+	fi
+	
+	msg "database upgrade successful"
+}
+
+start_service() {
+	if [ "$IS_ACTIVE" = "1" ]; then
+		msg "starting osmo-hlr service"
+		systemctl start osmo-hlr
+	fi
+}
+
+check_upgrade_required
+stop_service
+create_backup
+upgrade
+start_service

contrib/osmo-hlr.spec.in

@@ -0,0 +1,195 @@
+#
+# spec file for package osmo-hlr
+#
+# Copyright (c) 2017 SUSE LINUX GmbH, Nuernberg, Germany.
+# Copyright (c) 2016, Martin Hauke <mardnh@gmx.de>
+#
+# All modifications and additions to the file contributed by third parties
+# remain the property of their copyright owners, unless otherwise agreed
+# upon. The license for this file, and modifications and additions to the
+# file, is the same license as for the pristine package itself (unless the
+# license for the pristine package is not an Open Source License, in which
+# case the license is the MIT License). An "Open Source License" is a
+# license that conforms to the Open Source Definition (Version 1.9)
+# published by the Open Source Initiative.
+
+Name:           osmo-hlr
+Version:        @VERSION@
+Release:        0
+Summary:        Osmocom Home Location Register for GSUP protocol towards OsmoSGSN and OsmoCSCN
+License:        AGPL-3.0-or-later AND GPL-2.0-or-later
+Group:          Productivity/Telephony/Servers
+URL:            https://osmocom.org/projects/osmo-hlr
+Source:         %{name}-%{version}.tar.xz
+BuildRequires:  autoconf
+BuildRequires:  automake
+BuildRequires:  libtool
+BuildRequires:  pkgconfig >= 0.20
+BuildRequires:  python3
+%if 0%{?suse_version}
+BuildRequires:  systemd-rpm-macros
+%endif
+BuildRequires:  pkgconfig(libosmoabis) >= 1.1.0
+BuildRequires:  pkgconfig(libosmocore) >= 1.5.0
+BuildRequires:  pkgconfig(libosmoctrl) >= 1.5.0
+BuildRequires:  pkgconfig(libosmogsm) >= 1.5.0
+BuildRequires:  pkgconfig(libosmovty) >= 1.5.0
+BuildRequires:  pkgconfig(sqlite3)
+BuildRequires:  pkgconfig(talloc) >= 2.0.1
+# only needed for populate_hlr_db.pl
+Requires:       libdbi-drivers-dbd-sqlite3
+%{?systemd_requires}
+
+%description
+The GSUP HLR is a stand-alone HLR (Home Location Register) for SIM
+and USIM based subscribers which exposes the GSUP protocol towards
+its users. OsmoSGSN supports this protocol.
+
+osmo-gsup-hlr is still very simplistic. It is a single-threaded
+architecture and uses only sqlite3 tables as back-end.  It is suitable
+for installations of the scale that OsmoNITB was able to handle.  It
+also lacks various features like fine-grained control of subscribed
+services (like supplementary services).
+
+%package -n libosmo-gsup-client0
+Summary:        Osmocom GSUP (General Subscriber Update Protocol) client library
+License:        GPL-2.0-or-later
+Group:          System/Libraries
+
+%description -n libosmo-gsup-client0
+This is a shared library that can be used to implement client programs for
+the GSUP protocol. The typical GSUP server is OsmoHLR, with OsmoMSC, OsmoSGSN
+and External USSD Entities (EUSEs) using this library to implement clients.
+
+%package -n libosmo-gsup-client-devel
+Summary:        Development files for the Osmocom GSUP client library
+License:        GPL-2.0-or-later
+Group:          Development/Libraries/C and C++
+Requires:       libosmo-gsup-client0 = %{version}
+
+%description -n libosmo-gsup-client-devel
+This is a shared library that can be used to implement client programs for
+the GSUP protocol. The typical GSUP server is OsmoHLR, with OsmoMSC, OsmoSGSN
+and External USSD Entities (EUSEs) using this library to implement clients.
+
+This subpackage contains libraries and header files for developing
+applications that want to make use of libosmo-gsup-client.
+
+%package -n libosmo-mslookup0
+Summary:        Osmocom MS lookup library
+License:        GPL-2.0-or-later
+Group:          System/Libraries
+
+%description -n libosmo-mslookup0
+This shared library contains routines for looking up mobile subscribers.
+
+%package -n libosmo-mslookup-devel
+Summary:        Development files for the Osmocom MS lookup library
+License:        GPL-2.0-or-later
+Group:          Development/Libraries/C and C++
+Requires:       libosmo-mslookup0 = %{version}
+
+%description -n libosmo-mslookup-devel
+This shared library contains routines for looking up mobile subscribers.
+
+This subpackage contains libraries and header files for developing
+applications that want to make use of libosmo-mslookup.
+
+
+%package -n osmo-mslookup-client
+Summary:        Standalone program using libosmo-mslookup
+License:        GPL-2.0-or-later
+Group:          Development/Libraries/C and C++
+
+%description -n osmo-mslookup-client
+Standalone program using libosmo-mslookup to easily integrate with programs
+that want to connect services (SIP, SMS,...) to the current location of a
+subscriber.
+
+%prep
+%setup -q
+
+%build
+echo "%{version}" >.tarball-version
+autoreconf -fi
+%configure \
+  --docdir="%{_docdir}/%{name}" \
+  --with-systemdsystemunitdir=%{_unitdir} \
+  --enable-shared \
+  --disable-static
+make V=1 %{?_smp_mflags}
+
+%install
+%make_install
+install -d "%{buildroot}/%{_localstatedir}/lib/osmocom"
+find %{buildroot} -type f -name "*.la" -delete -print
+
+%check
+make %{?_smp_mflags} check || (find . -name testsuite.log -exec cat {} +)
+
+%if 0%{?suse_version}
+%preun
+%service_del_preun %{name}.service
+
+%postun
+%service_del_postun %{name}.service
+
+%pre
+%service_add_pre %{name}.service
+%endif
+
+%post
+%if 0%{?suse_version}
+%service_add_post %{name}.service
+%endif
+/usr/share/osmocom/osmo-hlr-post-upgrade.sh
+
+%post   -n libosmo-gsup-client0 -p /sbin/ldconfig
+%postun -n libosmo-gsup-client0 -p /sbin/ldconfig
+%post   -n libosmo-mslookup0 -p /sbin/ldconfig
+%postun -n libosmo-mslookup0 -p /sbin/ldconfig
+
+%files
+%license COPYING
+%dir %{_docdir}/%{name}
+%dir %{_docdir}/%{name}/examples
+%{_docdir}/%{name}/examples/osmo-hlr.cfg
+%{_docdir}/%{name}/examples/osmo-hlr-dgsm.cfg
+%dir %{_docdir}/%{name}/sql
+%{_docdir}/%{name}/sql/hlr.sql
+%{_docdir}/%{name}/sql//hlr_data.sql
+%dir %{_sysconfdir}/osmocom
+%dir %{_localstatedir}/lib/osmocom
+%{_bindir}/osmo-hlr
+%{_bindir}/osmo-hlr-db-tool
+%dir %{_sysconfdir}/osmocom
+%config %{_sysconfdir}/osmocom/osmo-hlr.cfg
+%{_unitdir}/osmo-hlr.service
+%dir %{_datadir}/osmocom
+%{_datadir}/osmocom/osmo-hlr-post-upgrade.sh
+
+%files -n libosmo-gsup-client0
+%{_libdir}/libosmo-gsup-client.so.0*
+
+%files -n libosmo-gsup-client-devel
+%{_bindir}/osmo-euse-demo
+%dir %{_includedir}/osmocom
+%dir %{_includedir}/osmocom/gsupclient
+%{_includedir}/osmocom/gsupclient/*.h
+%{_libdir}/libosmo-gsup-client.so
+%{_libdir}/pkgconfig/libosmo-gsup-client.pc
+
+%files -n libosmo-mslookup0
+%{_libdir}/libosmo-mslookup.so.0*
+
+%files -n libosmo-mslookup-devel
+%dir %{_includedir}/osmocom
+%dir %{_includedir}/osmocom/mslookup
+%{_includedir}/osmocom/mslookup/*.h
+%{_libdir}/libosmo-mslookup.so
+%{_libdir}/pkgconfig/libosmo-mslookup.pc
+
+%files -n osmo-mslookup-client
+%{_bindir}/osmo-mslookup-client
+
+%changelog

contrib/systemd/Makefile.am

@@ -0,0 +1,6 @@
+EXTRA_DIST = osmo-hlr.service
+
+if HAVE_SYSTEMD
+systemdsystemunit_DATA = \
+  osmo-hlr.service
+endif

contrib/systemd/osmo-hlr.service

@@ -1,5 +1,6 @@
 [Unit]
 Description=Osmocom Home Location Register (OsmoHLR)
+Documentation=https://osmocom.org/projects/osmo-hlr/wiki/OsmoHLR
 
 [Service]
 Type=simple

debian/changelog

@@ -1,3 +1,485 @@
+osmo-hlr (1.3.0) unstable; urgency=medium
+
+  [ Alexander Couzens ]
+  * hlr: respect the num_auth_vectors requested
+  * hlr: remove unused internal USSD list
+
+  [ Oliver Smith ]
+  * add libosmo-mslookup abstract client
+  * add mDNS lookup method to libosmo-mslookup
+  * Makefile.am: fix pkgconfig_DATA
+  * add mDNS lookup method to libosmo-mslookup (#2)
+  * contrib/dgsm/ add example esme and dialplan
+  * mslookup_client.c: fix dereferencing null pointer
+  * mdns_msg.c: always call va_end
+  * mslookup_client_mdns.c: fix dereferencing null
+  * osmo-mslookup-client.c: fix dereferencing null
+  * osmo-mslookup-client: fix dereferencing null
+  * mdns_sock.c: fix resource leak of sock
+  * mdns_rfc.c: fix possible access of uninit. mem
+  * mslookup_client_mdns_test: disable by default
+  * mslookup_client_mdns_test: no automatic skip
+  * Cosmetic: mention OS#4491 in location cancel code
+  * hlr_vty_subscr: prettier output for last LU seen
+  * contrib: import RPM spec
+  * contrib: integrate RPM spec
+  * Makefile.am: EXTRA_DIST: debian, contrib/*.spec.in
+  * contrib/jenkins: don't build osmo-gsm-manuals
+  * configure.ac: set -std=gnu11
+
+  [ Neels Hofmeyr ]
+  * add osmo-mslookup-client program
+  * add osmo-mslookup-client program (#2)
+  * fix missing braces in LOGP_GSUP_FWD
+  * gsup_client.c: fix deprecation for client create func
+  * 1/2: refactor: add and use lu_fsm, osmo_gsup_req, osmo_ipa_name
+  * 2/2: wrap ipa_name in osmo_cni_peer_id with type enum and union
+  * gsup client: add up_down_cb(), add osmo_gsup_client_create3()
+  * db v5: prep for D-GSM: add vlr_via_proxy and sgsn_via_proxy
+  * enlarge the GSUP message headroom
+  * test_nodes.vty: remove cruft
+  * D-GSM 1/n: add mslookup server in osmo-hlr
+  * D-GSM 2/n: implement mDNS method of mslookup server
+  * D-GSM 3/n: implement roaming by mslookup in osmo-hlr
+  * gsup_server: send routing error back to the correct peer
+  * adoc: add D-GSM chapter to osmohlr-usermanual
+  * drop error log for when a subscriber does not exist
+  * vty: show subscriber: change format of 'last LU seen'
+  * vty: show subscriber: show lu d,h,m,s ago, not just seconds
+  * auc3g: officially wrap IND around IND_bitlen space
+  * make osmo_cni_peer_id_cmp() NULL safe
+  * osmo_gsup_req_new(): require from_peer != NULL
+  * gsup_server.c: properly handle negative rc from osmo_gsup_conn_ccm_get()
+  * osmo_mslookup_server_mdns_rx(): handle read() rc == 0
+  * hlr_subscr_nam(): fix condition to fix nam=false notifications
+  * esme_dgsm.py: add --always-fail option for debugging SMPP
+  * osmo-mslookup-client: fix segfault for respond_error() caller
+  * manual: describe subscriber import by SQL
+
+  [ Harald Welte ]
+  * Revert "add osmo-mslookup-client program"
+  * Revert "add mDNS lookup method to libosmo-mslookup"
+  * Use OSMO_FD_* instead of deprecated BSC_FD_*
+  * support the XOR algorithm for UMTS AKA
+  * auc_test.c: Add some comments on what the test cases actually do
+  * main: add --vty-ref-mode, use vty_dump_xml_ref_mode()
+  * manuals: generate vty reference xml at build time
+
+  [ Vadim Yanitskiy ]
+  * db: fix possible SQLite3 allocated memory leak in db_open()
+  * gsup_server: fix typo: s/omso_gsup_message/osmo_gsup_message/
+  * debian/control: change maintainer to the Osmocom team / mailing list
+  * cosmetic: fix spelling in logging message: existAnt -> existEnt
+  * doc/manuals: fix s/There/The/ in 'USSD Configuration'
+  * doc/manuals: re-organize description of internal USSD handlers
+  * USSD: fix handle_ussd(): do not free() unconditionally
+  * USSD: add special 'idle' handler to IUSE for testing
+
+  [ Eric ]
+  * configure.ac: fix libtool issue  with clang and sanitizer
+
+  [ Philipp Maier ]
+  * doc: do not use loglevel info for log category ss
+
+  [ Pau Espin Pedrol ]
+  * configure.ac: Fix trailing whitespace
+  * doc: Update VTY reference xml file
+  * Support setting rt-prio and cpu-affinity mask through VTY
+  * Set TCP NODELAY sockopt to GSUP cli and srv connections
+  * contrib/jenkins: Enable parallel make in make distcheck
+  * .gitignore: Ignore new autofoo tmp files
+  * tests: Replace deprecated API log_set_print_filename
+
+  [ Keith ]
+  * osmo-hlr-db-tool: Make import from osmo-nitb less "lossy"
+  * Correct vty inline help for show subscriber
+  * Add vty command to show summary of all or filtered subscribers
+  * Fix Coverity Warnings
+
+ -- Pau Espin Pedrol <pespin@sysmocom.de>  Tue, 23 Feb 2021 18:13:53 +0100
+
+osmo-hlr (1.2.0) unstable; urgency=medium
+
+  [ Ruben Undheim ]
+  * Fix test for return codes on mipsel and alpha archs
+
+  [ Thorsten Alteholz ]
+  * fix spelling errors detected by lintian
+
+  [ Pau Espin Pedrol ]
+  * tests: Fix db_test err file to expect error code name instead of value
+
+  [ Oliver Smith ]
+  * tests/test_nodes.vty: check less libosmocore cmds
+  * tests/db_upgrade: disable for old sqlite versions
+  * gitignore: add tests/db_upgrade/*.dump
+  * gsup_client.h: fix license header: GPLv2+
+  * tests/auc: change back to python3
+
+  [ Neels Hofmeyr ]
+  * fix double free in osmo_gsup_client_enc_send()
+  * db upgrade to v2: log version 2, not 1
+  * fix upgrade to version 2: imei column default value
+  * add --db-check option
+  * hlr.sql: move comment
+  * add db_upgrade test
+  * hlr db schema 3: hlr_number -> msc_number
+  * db.c: code dup: add db_run_statements() for arrays of statements
+  * move headers to include/osmocom/hlr
+  * fix upgrade test in presence of ~/.sqliterc
+  * db upgrade: remove some code dup
+  * add osmo_gsup_msgb_alloc()
+  * Makefile convenience: add VTY_TEST var to run only one test
+  * remove gsup_test
+  * test_nodes.vty: tweak: add some '?' checks
+  * db v4: add column last_lu_seen_ps
+
+  [ Harald Welte ]
+  * AUC: Add support for setting the AMF separation bit to '1' for EUTRAN
+  * hlr: exit(2) on unsupported positional arguments on command line
+
+ -- Pau Espin Pedrol <pespin@sysmocom.de>  Fri, 03 Jan 2020 12:37:35 +0100
+
+osmo-hlr (1.1.0) unstable; urgency=medium
+
+  [ Oliver Smith ]
+  * docs: running: document --db-upgrade
+  * Add IMEI column to subscriber table
+  * Optionally store IMEI in subscriber table
+  * VTY tests: fill DB before running test
+  * VTY: integrate IMEI
+  * hlr.c: replace deprecated osmo_gsup_get_err_msg_type()
+  * hlr.c: move hlr_ctx to the top
+  * tests: use -no-install libtool flag to avoid ./lt-* scripts
+  * Cosmetic: gsup_route_find: comment addr, addrlen
+  * USSD: save MO USSD's originating MSC's vlr_number
+  * USSD: don't use gsm0480_msgb_alloc_name()
+  * hlr.c: forward GSUP messages between clients
+  * db_hlr.c: db_subscr_create(): add flags argument
+  * db_hlr.c: add db_subscr_exists_by_imsi()
+  * Create subscribers on demand
+  * Document subscribers create on demand feature
+  * debian: create -doc subpackage with pdf manuals
+  * db_test: set timezone to work around mktime bug
+  * db_hlr: zero-initialize "struct tm"
+  * rx_check_imei_req(): fix IMEI bounds checking
+  * contrib/jenkins.sh: run "make maintainer-clean"
+  * VTY: add subscriber update network-access-mode
+  * manuals: improve subscribers create on demand
+  * gitignore: ignore everything generated in db_test
+  * db_auc.c: verify hex key sizes read from DB
+
+  [ Max ]
+  * Log ip:port when adding GSUP routes
+  * Add link to project wiki to .service file
+  * Enable statsd support
+
+  [ Vadim Yanitskiy ]
+  * hlr.c: properly terminate the process on SIGTERM
+  * hlr.c: fix: also store the session state in read_cb_forward()
+  * hlr.c: fix: properly print the original message type in read_cb_forward()
+  * hlr.c: check the presence of msgb->l2h in read_cb()
+  * hlr.c: fix possible msgb memleaks in read_cb()
+  * db_hlr.c: add db_subscr_exists_by_msisdn()
+  * src/db.h: use GSM23003_MSISDN_MAX_DIGITS for MSISDN buffer size
+  * src/hlr.c: fix deprecation warning: use gsm48_decode_bcd_number2()
+  * hlr_ussd.c: fix: properly pass invokeID in handle_ussd_own_msisdn()
+  * hlr_ussd.c: rx_proc_ss_req(): fix NULL pointer dereference
+  * build: fix mess with 'db_test_SOURCES' and 'db_test_LDADD'
+  * tests/db_test: close the database when test is finished
+  * src/db.c: integrate SQLite3 with talloc allocator
+
+  [ Neels Hofmeyr ]
+  * USSD: fix routing to multiple MSC
+  * fix error logging for GSUP route
+  * add missing error log: invalid IMSI
+  * osmo-hlr: allow configuring db path from cfg file
+  * use new OSMO_IMSI_BUF_SIZE
+
+  [ Daniel Willmann ]
+  * manuals: Add script to update vty/counter documentation from docker
+  * manuals: Update vty documentation
+
+  [ Pau Espin Pedrol ]
+  * Remove undefined param passed to logging_vty_add_cmds
+  * configure.ac: Require libosmocore 1.2.0
+
+ -- Pau Espin Pedrol <pespin@sysmocom.de>  Wed, 07 Aug 2019 16:14:23 +0200
+
+osmo-hlr (1.0.0) unstable; urgency=medium
+
+  [ Stefan Sperling ]
+  * move creation of insert subscriber data messages to a common function
+
+  [ Harald Welte ]
+  * Return proper GSUP error in case of too short IMSI
+  * disable blind subscriber insertion into every VLR/SGSN
+  * gsup_server: Add "priv" pointer and make it point to 'struct hlr'
+  * move osmo_gsup_addr_send() declaration from luop.h to gsup_router.h
+  * gsup_router: Use "#pragma once" and add missing #includes
+  * Add "show gsup-connections" VTY command
+  * import gsup_client.c as new libosmo-gsup-client
+  * gsup_client: rename gsup_client_* to osmo_gsup_client_*
+  * GSUP: Log GSUP route add/remove
+  * hlr: Export + Declare global g_hlr symbol
+  * USSD: Add Core USSD handling + VTY routing config to HLR
+  * USSD: Add basic dispatch + decode of GSUP-encapsulated SS/USSD
+  * hlr_ussd: Introduce LOGPSS() macro
+  * USSD: Send ReturnError component if USSD Code unknown / EUSE disconnected
+  * USSD: Further unification of log output; Use LOGPSS when possible
+  * osmo-hlr.cfg: Don't enable DEBUG logging by default
+  * USSD: Add new "DSS" logging category and use it appropriately
+  * USSD: fix null-pointer deref in "default-route" vty/config cmd
+  * Add osmo-euse-demo as minimalistic test of a External USSD (EUSE) handler
+  * USSD: Add support for internal USSD handlers
+  * debian: Add sub-package for libosmo-gsup-client
+  * pkg-config: Fix libosmo-gsup-client pkg-config file
+  * gitignore: Add .tarball-version
+  * debian: Make libosmo-gsup-client-dev depend on libosmo-gsup-client0
+  * USSD: Fix "ussd default-route"
+  * libosmo-gsup-client: License is GPLv2-or-later
+  * osmo-hlr.cfg: Ensure well-formed config file example
+  * test_nodes.vty: Since libosmocore 1.0.0, we only have one space
+
+  [ Martin Hauke ]
+  * sql/Makefile.am: Make docsdir completely configurable
+  * debian: Fix typo in package description
+
+  [ Pau Espin Pedrol ]
+  * debian: Avoid installing duplicate cfg file in /etc
+  * sql/Makefile: Install hlr_data.sql as example together with hlr.sql
+  * sql/Makefile: Install sql files under doc/.../sql subdir
+  * sql/Makefile: Create empty /var/lib/osmocom directory at install time
+  * Install systemd services with autotools
+  * Move doc/Makefile.am to doc/examples/Makefile.am
+  * Install sample cfg file to /etc/osmocom
+
+  [ Vadim Yanitskiy ]
+  * hlr.c: move deinitialization code from SIGINT handler
+  * hlr.c: free root talloc context on exit
+  * hlr.c: track the use of talloc NULL memory contexts
+  * src/db.c: fix: make sure the database is properly closed
+  * src/db.c: don't ignore the result of db_bootstrap()
+  * hlr_vty_subscr.c: fix subscriber creation command help
+  * Update .gitignore: add missing build products
+  * tests/Makefile.am: also remove temporary sqlite files
+  * hlr_ussd.h: add #pragma once include guard
+  * hlr_ussd.h: use proper libc headers
+  * Update .gitignore: ignore osmo-euse-demo
+  * hlr_ussd.h: drop meaningless forward declaration
+  * USSD/hlr_vty.c: print error if EUSE is not found
+  * hlr_ussd.c: fix: properly print a EUSE / IUSE name
+  * hlr_ussd.c: avoid using CR and NL in IUSE responses
+
+  [ Neels Hofmeyr ]
+  * fix build: adjust test_nodes.vty to logging change
+  * tweak example config
+  * make: always allow running python tests manually
+
+ -- Harald Welte <laforge@gnumonks.org>  Sun, 20 Jan 2019 19:29:58 +0100
+
+osmo-hlr (0.2.1) unstable; urgency=medium
+
+  [ Neels Hofmeyr ]
+  * fix luop crash: use buffer for APN that remains valid
+  * add gsup_test to catch OS#3231
+  * add error handling to osmo_gsup_configure_wildcard_apn()
+
+ -- Pau Espin Pedrol <pespin@sysmocom.de>  Fri, 04 May 2018 18:41:35 +0200
+
+osmo-hlr (0.2.0) unstable; urgency=medium
+
+  [ Neels Hofmeyr ]
+  * vty: skip installing cmds now always installed by default
+  * hlr_db_tool: fix error log strerror invocation
+  * cosmetic: add comment on ignored return value
+  * db-tool: add command 'create'
+  * db-tool: cosmetic: tweak printf output
+  * db-tool: error-exit on too many arguments
+  * add --enable-sanitize config option
+  * db_test: don't verify SQLite issued error messages, they might change
+  * cosmetic: rx_send_auth_info(): decide error cause with switch()
+  * return GMM_CAUSE_IMSI_UNKNOWN if there is no auth data
+  * db_get_auth_data / db_get_auc: clarify return values
+  * osmo-hlr: log details for unknown IMSI / no auth data / db error
+  * db_test: also test db_get_auc() return values
+  * fix test_subscriber_errors.ctrl after libosmocore change
+  * fix debug log: put 'deriving 2G from 3G' in proper place
+  * ctrl test: fix: adjust expectations after stricter ctrl parsing
+  * fix build: db_test: missing LIBOSMOABIS_CFLAGS and _LIBS
+  * configure: add --enable-werror
+  * jenkins.sh: use --enable-werror configure flag, not CFLAGS
+
+  [ Harald Welte ]
+  * hlr.c: Avoid overflow of lu_operation.subscr.imsi
+  * Fix expected test output after new 'logging print file 1' vty command
+  * osmo-hlr: Add talloc context introspection via VTY
+  * vty: Don't print error if removing auth data while none present
+  * Fix responses to PURGE MS
+
+  [ Alexander Couzens ]
+  * debian: include systemd service osmo-hlr.service
+  * doc: install example .cfg files to $(docdir)/examples/
+  * debian: install osmo-hlr.cfg to /etc/osmocom
+
+  [ Max ]
+  * Remove unused check
+  * Remove unused ipa.py
+  * Enable sanitize for CI tests
+
+  [ Pau Espin Pedrol ]
+  * luop.c: Transform FIXME from warning to pragma message
+  * contrib:jenkins.sh: Enable Werror
+  * use osmo_init_logging2
+  * Remove unused src/db_test.c
+
+  [ Alexander Huemer ]
+  * Add missing build products in .gitignore
+
+  [ Stefan Sperling ]
+  * more robust usage of osmo_timer API for osmo-hlr luop timer
+  * notify GSUP clients when HLR subscriber information changes
+  * rewrite subscriber_update_notify() without calls into luop
+  * don't forget to mark luop as a packet switched connection
+
+ -- Pau Espin Pedrol <pespin@sysmocom.de>  Thu, 03 May 2018 16:27:13 +0200
+
+osmo-hlr (0.1.0) unstable; urgency=medium
+
+  [ Neels Hofmeyr ]
+  * build with autoconf/automake, add jenkins.sh script
+  * fix build on FreeBSD: eliminate implicitly declared functions
+  * fix various compiler warnings
+  * fix DLGSUP logging cat after change in libosmocore
+  * build: recoin db_test as non-installable program
+  * build: actually make sqlite mandatory
+  * bump required libosmocore version to 0.9.5
+  * gsup: send subscriber MSISDN
+  * debug log: log computed vector kinds
+  * log: move a log from info to debug level
+  * hlr.sql: typo in comment
+  * auc.c: typo in comment
+  * main: add and use root talloc ctx
+  * main: add option parsing with db file and default options
+  * main: add VTY and '-c config-file' option
+  * sql: fix 3g_auc's column K data type
+  * cosmetic: sql: indicate VARCHAR size of key columns as 32
+  * sql: auc_3g: set sqn NOT NULL DEFAULT 0
+  * comment: sql: describe auc_2g and auc_3g columns
+  * Add test suite skeleton with empty test (auc_3g_test)
+  * tests: auc_3g_test: implement vector generation test
+  * auth: verify test sets from 3GPP TS 55.205
+  * sql: add unique constraints to IMSI and MSISDN
+  * UMTS AKA resync: fix argument ordering
+  * auc_3g_test: add AUTS resync test
+  * auc_gen_vectors(): ensure sane arguments, test
+  * auc_3g_test: allow to inc fake rand bytes upon rand request
+  * auc_3g_test: add AUTS test with N vectors, to show bug
+  * cosmetic: refactor auc_compute_vectors(), add debug log
+  * auc_compute_vectors(): fix AUTS resync for multiple vectors
+  * cosmetic: auc_3g_test: improve test debugging tools
+  * cosmetic: rename auc_3g_test.c to auc_test.c
+  * fix: properly handle error rc by osmo_gsup_conn_ccm_get()
+  * auc tests: adjust cosmetically to prepare for SQN changes
+  * auc tests: fix after SQN scheme changes from libosmocore
+  * fix debug log: adjust to new SQN increment scheme
+  * UMTS AKA: implement SQN increment according to SEQ and IND
+  * debug log: output ind slot, previous sqn, and sqn db update
+  * jenkins: add value_string termination check
+  * fix db_subscr_ps error handling
+  * add config example (mostly empty)
+  * install hlr.sql in prefix/doc/osmo-hlr/
+  * use OSMO_GSUP_PORT == 4222 instead of hardcoded 2222
+  * add basic CTRL interface tests
+  * add CTRL tests for enable-/disable-/status-ps
+  * cosmetic: prepend DB_STMT_ to enum stmt_idx entries
+  * cosmetic: rename db_subscr_get() to db_subscr_get_by_imsi()
+  * cosmetic: refactor db_bind_imsi() as db_bind_text()
+  * cosmetic: multi-line DB_STMT_AUC_BY_IMSI
+  * cosmetic: log IMSI='<imsi>', log "no such subscriber"
+  * cosmetic: log: "SQLite" with capital L
+  * cosmetic: db_hlr: SL3_TXT: clarify indenting
+  * ctrl_test_runner.py: use proper constant as test db path
+  * gitignore: tests/package.m4
+  * cosmetic: don't log about missing SQLite log cb
+  * add db_bind_int() and db_bind_int64()
+  * add db_subscr_create(), db_subscr_delete(), db_subscr_update_msisdn_by_imsi()
+  * add initial db_test: creating and deleting subscribers
+  * less noise: simplify db_remove_reset()
+  * db: use int64_t as subscriber id
+  * add db_subscr_get_by_msisdn() and db_subscr_get_by_id()
+  * refactor db_subscr_ps() to db_subscr_nam()
+  * refactor db_subscr_lu()
+  * refactor db_subscr_purge
+  * add db_subscr_update_aud_by_id(), complete db_subscr_delete_by_id()
+  * refactor db_get_auth_data return val
+  * code undup: use db_remove_reset() in db_auc.c
+  * fix db_update_sqn(): reset stmt in all error cases
+  * code undup: use db_bind_text() in db_get_auth_data()
+  * debian: 'make check' needs sqlite3, add to Build-Depends
+  * fix db_subscr_get_by_*(): clear output data; test in db_test.c
+  * implement subscriber vty interface, tests
+  * add test_nodes.vty
+  * replace ctrl_test_runner.py with transcript test_subscriber.ctrl
+  * add lu_op_free(), use in luop.c
+  * luop: fix mem leak upon error in lu_op_alloc_conn()
+  * fix mem leak in handle_cmd_ps(): free luop
+  * api doc: say that lu_op_tx_del_subscr_data() doesn't free
+  * add hlr_subsrc_nam to put GSUP client notification in proper API
+  * vty: fix output of empty IMSI
+  * db api: fix/add API docs
+  * cosmetic: tweak params of hlr_controlif_setup()
+  * ctrl: completely replace all CTRL commands
+  * test_subscriber.ctrl: test against octal/hex interpretation of id
+  * jenkins: use osmo-clean-workspace.sh before and after build
+  * tests/Makefile: use test db var instead of repeating the path
+  * db_test: fix *FLAGS
+  * automatically create db tables on osmo-hlr invocation
+  * cosmetic: sql/hlr.sql: move comments
+  * cosmetic: rename SL3_TXT macro, use osmo_strlcpy()
+  * fix default logging levels to NOTICE, not DEBUG
+  * add osmo-hlr-db-tool, program to migrate from osmo-nitb db
+
+  [ Max ]
+  * Add gerrit settings
+  * Add hardcoded APN
+  * Log error cause as a string
+  * Move GSUP msg init into separate function
+  * Use strings for GSUP message type
+  * Move lu_operation into separate file
+  * db: move duplicated code into helper functions
+  * Fix compiler's warning about printf security
+  * Add routines to update nam_ps
+  * Add global HLR struct
+  * Make subscr parameter to db_subscr_get() optional
+  * Add CTRL interface
+  * CTRL: add enable/disable packet service cmds
+  * Add .deb packaging
+  * deb: fix OBS build
+  * debian: remove obsolete dependency
+  * Attempt to fix .deb package
+  * deb: use python in shebang
+  * Another attempt at fixing .deb
+  * Use release helper from libosmocore
+  * Use value string check from osmo-ci
+
+  [ Daniel Willmann ]
+  * Add systemd service file
+  * hlr_data.sql: Insert ki and opc instead of op to example data
+  * tests/auc: Don't require python3
+
+  [ Pau Espin Pedrol ]
+  * VTY: Add hlr node and bind ip field
+  * debian: remove unneeded dependency libdbd-sqlite3
+
+  [ Harald Welte ]
+  * jenkins.sh: Proper error message if local environment isn't set up
+
+  [ Alexander Couzens ]
+  * debian/rules: show testsuite.log when tests are failing
+
+ -- Harald Welte <laforge@gnumonks.org>  Sat, 28 Oct 2017 20:37:33 +0200
+
 osmo-hlr (0.0.1) UNRELEASED; urgency=low
 
      * Initial release (Closes: OS#1948)

debian/control

@@ -1,18 +1,19 @@
 Source: osmo-hlr
 Section: net
 Priority: optional
-Maintainer: Max Suraev <msuraev@sysmocom.de>
+Maintainer: Osmocom team <openbsc@lists.osmocom.org>
 Build-Depends: debhelper (>= 9),
                pkg-config,
                dh-autoreconf,
                dh-systemd (>= 1.5),
                autotools-dev,
-               python-minimal,
-               libosmocore-dev,
-               libosmo-abis-dev,
-               libosmo-netif-dev,
+               python3-minimal,
+               libosmocore-dev (>= 1.5.0),
+               libosmo-abis-dev (>= 1.1.0),
+               libosmo-netif-dev (>= 1.1.0),
                libsqlite3-dev,
-               sqlite3
+               sqlite3,
+               osmo-gsm-manuals-dev (>= 1.1.0)
 Standards-Version: 3.9.6
 Vcs-Browser: http://cgit.osmocom.org/osmo-hlr
 Vcs-Git: git://git.osmocom.org/osmo-hlr
@@ -32,3 +33,59 @@ Priority: extra
 Depends: osmo-hlr (= ${binary:Version}), ${misc:Depends}
 Description: Debug symbols for the osmo-hlr
  Make debugging possible
+
+Package: libosmo-gsup-client0
+Section: libs
+Architecture: any
+Multi-Arch: same
+Depends: ${shlibs:Depends},
+	 ${misc:Depends}
+Pre-Depends: ${misc:Pre-Depends}
+Description: Osmocom GSUP (General Subscriber Update Protocol) client library
+  This is a shared library that can be used to implement client programs for
+  the GSUP protocol.  The typical GSUP server is OsmoHLR, with OsmoMSC, OsmoSGSN
+  and External USSD Entities (EUSEs) using this library to implement clients.
+
+Package: libosmo-gsup-client-dev
+Architecture: any
+Multi-Arch: same
+Depends: ${misc:Depends},
+	 libosmo-gsup-client0 (= ${binary:Version}),
+	 libosmocore-dev
+Description: Development headers of Osmocom GSUP client library
+  This is a shared library that can be used to implement client programs for
+  the GSUP protocol.  The typical GSUP server is OsmoHLR, with OsmoMSC, OsmoSGSN
+  and External USSD Entities (EUSEs) using this library to implement clients.
+  .
+  This package contains the development headers.
+
+Package: libosmo-mslookup0
+Section: libs
+Architecture: any
+Multi-Arch: same
+Depends: ${shlibs:Depends},
+         ${misc:Depends}
+Pre-Depends: ${misc:Pre-Depends}
+Description: Osmocom MS lookup library
+  This shared library contains routines for looking up mobile subscribers.
+
+Package: libosmo-mslookup-dev
+Architecture: any
+Multi-Arch: same
+Depends: ${misc:Depends},
+	 libosmo-mslookup0 (= ${binary:Version}),
+	 libosmocore-dev
+Pre-Depends: ${misc:Pre-Depends}
+Description: Development headers of Osmocom MS lookup library
+  This shared library contains routines for looking up mobile subscribers.
+  .
+  This package contains the development headers.
+
+Package: osmo-hlr-doc
+Architecture: all
+Section: doc
+Priority: optional
+Depends: ${misc:Depends}
+Description: ${misc:Package} PDF documentation
+ Various manuals: user manual, VTY reference manual and/or
+ protocol/interface manuals.

debian/libosmo-gsup-client-dev.install

@@ -0,0 +1,5 @@
+usr/include/osmocom/gsupclient
+usr/lib/*/libosmo-gsup-client*.a
+usr/lib/*/libosmo-gsup-client*.so
+usr/lib/*/libosmo-gsup-client*.la
+usr/lib/*/pkgconfig/libosmo-gsup-client.pc

debian/libosmo-gsup-client0.install

@@ -0,0 +1 @@
+usr/lib/*/libosmo-gsup-client*.so.*

debian/libosmo-mslookup-dev.install

@@ -0,0 +1,5 @@
+usr/include/osmocom/mslookup
+usr/lib/*/libosmo-mslookup*.a
+usr/lib/*/libosmo-mslookup*.so
+usr/lib/*/libosmo-mslookup*.la
+usr/lib/*/pkgconfig/libosmo-mslookup.pc

debian/libosmo-mslookup0.install

@@ -0,0 +1 @@
+usr/lib/*/libosmo-mslookup*.so.*

debian/osmo-hlr-doc.install

@@ -0,0 +1 @@
+usr/share/doc/osmo-hlr-doc/*.pdf

debian/osmo-hlr.install

@@ -1,2 +1,9 @@
+/etc/osmocom/osmo-hlr.cfg
+/lib/systemd/system/osmo-hlr.service
 /usr/bin/osmo-hlr
-/usr/share/doc/osmo-hlr/hlr.sql
+/usr/bin/osmo-hlr-db-tool
+/usr/share/doc/osmo-hlr/sql/hlr.sql
+/usr/share/doc/osmo-hlr/sql/hlr_data.sql
+/usr/share/doc/osmo-hlr/examples/osmo-hlr.cfg
+/usr/share/osmocom/osmo-hlr-post-upgrade.sh
+/var/lib/osmocom

debian/postinst

@@ -0,0 +1,5 @@
+#!/bin/sh -e
+# Debian's postinst script is called on both installation and upgrade. Call the
+# post-upgrade script in both cases, it won't do anything if there is nothing
+# to do.
+/usr/share/osmocom/osmo-hlr-post-upgrade.sh

debian/rules

@@ -15,3 +15,10 @@ override_dh_strip:
 # Print test results in case of a failure
 override_dh_auto_test:
 	dh_auto_test || (find . -name testsuite.log -exec cat {} \; ; false)
+
+override_dh_auto_configure:
+	dh_auto_configure -- --with-systemdsystemunitdir=/lib/systemd/system --enable-manuals
+
+# Don't create .pdf.gz files (barely saves space and they can't be opened directly by most pdf readers)
+override_dh_compress:
+	dh_compress -X.pdf

doc/Makefile.am

@@ -0,0 +1,4 @@
+SUBDIRS = \
+	examples \
+	manuals \
+	$(NULL)

doc/examples/Makefile.am

@@ -0,0 +1,27 @@
+osmoconfdir = $(sysconfdir)/osmocom
+osmoconf_DATA = osmo-hlr.cfg
+
+EXTRA_DIST = osmo-hlr.cfg
+
+CFG_FILES = find $(srcdir) -name '*.cfg*' | sed -e 's,^$(srcdir),,'
+
+dist-hook:
+	for f in $$($(CFG_FILES)); do \
+		j="$(distdir)/$$f" && \
+		mkdir -p "$$(dirname $$j)" && \
+		$(INSTALL_DATA) $(srcdir)/$$f $$j; \
+	done
+
+install-data-hook:
+	for f in $$($(CFG_FILES)); do \
+		j="$(DESTDIR)$(docdir)/examples/$$f" && \
+		mkdir -p "$$(dirname $$j)" && \
+		$(INSTALL_DATA) $(srcdir)/$$f $$j; \
+	done
+
+uninstall-hook:
+	@$(PRE_UNINSTALL)
+	for f in $$($(CFG_FILES)); do \
+		j="$(DESTDIR)$(docdir)/examples/$$f" && \
+		$(RM) $$j; \
+	done

doc/examples/osmo-hlr-dgsm.cfg

@@ -0,0 +1,22 @@
+# OsmoHLR example configuration for Distributed GSM (mslookup)
+hlr
+ gsup
+  # For D-GSM roaming, osmo-hlr's GSUP must listen on a public IP:
+  bind ip 10.9.8.7
+  # Each HLR should identify with a distinct name
+  ipa-name hlr-23
+mslookup
+ # Bind mslookup mDNS server and client on default multicast address and port:
+ # 239.192.23.42 port 4266
+ mdns bind
+ # Tell the mslookup server in osmo-hlr which IP+ports to return when a
+ # remote voice call or SMS wants to deliver to a local subscriber:
+ server
+  # local osmo-sip-connector SIP port
+  service sip.voice at 10.9.8.7 5060
+  # local osmo-msc SMPP port
+  service smpp.sms at 10.9.8.7 2775
+  # experimental: SMS-over-GSUP: this HLR's GSUP port
+  service gsup.sms at 10.9.8.7 4222
+  # only required if different from above 'gsup'/'bind ip':
+  #service gsup.hlr at 10.9.8.7 4222

doc/examples/osmo-hlr.cfg

@@ -2,13 +2,18 @@
 ! OsmoHLR example configuration
 !
 log stderr
-  logging filter all 1
-  logging color 1
-  logging print category 1
-  logging timestamp 1
-  logging print extended-timestamp 1
-  logging level all debug
-  logging level linp error
+ logging filter all 1
+ logging color 1
+ logging print category 1
+ logging print category-hex 0
+ logging print level 1
+ logging print file basename last
+ logging print extended-timestamp 1
+ logging level main notice
+ logging level db notice
+ logging level auc notice
+ logging level ss notice
+ logging level linp error
 !
 line vty
  bind 127.0.0.1
@@ -17,3 +22,5 @@ ctrl
 hlr
  gsup
   bind ip 127.0.0.1
+ ussd route prefix *#100# internal own-msisdn
+ ussd route prefix *#101# internal own-imsi

doc/manuals/Makefile.am

@@ -0,0 +1,67 @@
+EXTRA_DIST = example_subscriber_add_update_delete.vty \
+    example_subscriber_cs_ps_enabled.ctrl \
+    example_subscriber_info.ctrl \
+    osmohlr-usermanual.adoc \
+    osmohlr-usermanual-docinfo.xml \
+    osmohlr-vty-reference.xml \
+    regen_doc.sh \
+    chapters \
+    vty
+
+if BUILD_MANUALS
+  ASCIIDOC = osmohlr-usermanual.adoc
+  ASCIIDOC_DEPS = $(srcdir)/chapters/*.adoc $(srcdir)/*.vty $(srcdir)/*.ctrl
+  include $(OSMO_GSM_MANUALS_DIR)/build/Makefile.asciidoc.inc
+
+  VTY_REFERENCE = osmohlr-vty-reference.xml
+
+  BUILT_REFERENCE_XML = $(builddir)/vty/hlr_vty_reference.xml
+  $(builddir)/vty/hlr_vty_reference.xml: $(top_builddir)/src/osmo-hlr
+	mkdir -p $(builddir)/vty
+	$(top_builddir)/src/osmo-hlr --vty-ref-xml > $@
+
+  include $(OSMO_GSM_MANUALS_DIR)/build/Makefile.vty-reference.inc
+
+  OSMO_REPOSITORY = osmo-hlr
+  include $(OSMO_GSM_MANUALS_DIR)/build/Makefile.common.inc
+endif
+
+TMP_DB = generated/hlr.db
+
+update-examples: update-examples-ctrl update-examples-vty
+
+.PHONY: found-update-deps
+found-update-deps:
+	@if [ ! -f "$(top_srcdir)/sql/hlr.sql" ]; then \
+		echo "You need to define OSMO_HLR_PATH to point at an osmo-hlr.git"; \
+		exit 1; \
+	fi
+	@if [ -z "$(shell which osmo-hlr)" ]; then \
+		echo "osmo-hlr needs to be installed / available in the PATH"; \
+		exit 1; \
+	fi
+	@if [ -z "$(shell which osmo_verify_transcript_ctrl.py)" ]; then \
+		echo "You need to install git.osmocom.org/python/osmo-python-tests.git"; \
+		exit 1; \
+	fi
+	@if [ -z "$(shell which osmo_verify_transcript_vty.py)" ]; then \
+		echo "You need to install git.osmocom.org/python/osmo-python-tests.git"; \
+		exit 1; \
+	fi
+
+update-examples-ctrl: found-update-deps
+	mkdir -p generated
+	rm -f "$(TMP_DB)"
+	sqlite3 "$(TMP_DB)" < "$(top_srcdir)/sql/hlr.sql"
+	sqlite3 "$(TMP_DB)" < "$(top_srcdir)/tests/test_subscriber.sql"
+	osmo_verify_transcript_ctrl.py \
+		-r "osmo-hlr -l $(TMP_DB) -c $(top_srcdir)/doc/examples/osmo-hlr.cfg" \
+		-p 4259 --update *.ctrl
+
+update-examples-vty: found-update-deps
+	mkdir -p generated
+	rm -f "$(TMP_DB)"
+	sqlite3 "$(TMP_DB)" < "$(top_srcdir)/sql/hlr.sql"
+	osmo_verify_transcript_vty.py \
+		-r "osmo-hlr -l $(TMP_DB) -c $(top_srcdir)/doc/examples/osmo-hlr.cfg" \
+		-p 4258 --update *.vty

doc/manuals/chapters/control.adoc

@@ -0,0 +1,106 @@
+[[hlr-ctrl]]
+== Control interface
+
+The actual protocol is described in <<common-control-if>>, the variables common
+to all programs using it are described in <<ctrl_common_vars>>. This section
+describes the CTRL interface variables specific to OsmoHLR.
+
+All subscriber variables are available by different selectors, which are freely
+interchangeable:
+
+.Subscriber selectors available on OsmoHLR's Control interface
+[options="header",width="100%",cols="35%,65%"]
+|===
+|Selector|Comment
+|subscriber.*by-imsi-*'123456'.*|Subscriber selector by IMSI, replace "123456" with the actual IMSI
+|subscriber.*by-msisdn-*'123456'.*|Subscriber selector by MSISDN
+|subscriber.*by-id-*'123456'.*|Subscriber selector by database ID
+|===
+
+Each of the above selectors feature all of these control variables:
+
+.Subscriber variables available on OsmoHLR's Control interface
+[options="header",width="100%",cols="35%,8%,8%,8%,41%"]
+|===
+|Name|Access|Trap|Value|Comment
+|subscriber.by-\*.*info*|R|No||List (short) subscriber information
+|subscriber.by-\*.*info-aud*|R|No||List subscriber authentication tokens
+|subscriber.by-\*.*info-all*|R|No||List both 'info' and 'info-aud' in one
+|subscriber.by-\*.*cs-enabled*|RW|No|'1' or '0'|Enable/disable circuit-switched access
+|subscriber.by-\*.*ps-enabled*|RW|No|'1' or '0'|Enable/disable packet-switched access
+|===
+
+=== subscriber.by-*.info, info-aud, info-all
+
+Query the HLR database and return current subscriber record, in multiple lines
+of the format
+
+----
+name<tab>value
+----
+
+To keep the reply as short as possible, some values are omitted if they are
+empty. These are the returned values and their presence
+modalities; for their meaning, see <<subscriber-params>>:
+
+.Returned values by OsmoHLR's 'info', 'info-all' and 'info-aud' commands
+[options="header",width="100%",cols="15%,15%,30%,40%"]
+|===
+|Returned by 'info-all' and|Name|Format|Presence
+|'info'|id|-9223372036854775808 .. 9223372036854775807 (usually not negative)|always
+|'info'|imsi|6 to 15 decimal digits|always
+|'info'|msisdn|1 to 15 decimal digits|when non-empty
+|'info'|nam_cs|'1' if CS is enabled, or '0'|always
+|'info'|nam_ps|'1' if PS is enabled, or '0'|always
+|'info'|vlr_number|up to 15 decimal digits|when non-empty
+|'info'|sgsn_number|up to 15 decimal digits|when non-empty
+|'info'|sgsn_address||when non-empty
+|'info'|ms_purged_cs|'1' if CS is purged, or '0'|always
+|'info'|ms_purged_ps|'1' if PS is purged, or '0'|always
+|'info'|periodic_lu_timer|0..4294967295|always
+|'info'|periodic_rau_tau_timer|0..4294967295|always
+|'info'|lmsi|8 hex digits|always
+|'info-aud'|aud2g.algo|one of 'comp128v1', 'comp128v2', 'comp128v3' or 'xor'|when valid 2G auth data is set
+|'info-aud'|aud2g.ki|32 hexadecimal digits|when valid 2G auth data is set
+|'info-aud'|aud3g.algo|so far always 'milenage'|when valid 3G auth data is set
+|'info-aud'|aud3g.k|32 hexadecimal digits|when valid 3G auth data is set
+|'info-aud'|aud3g.op|32 hexadecimal digits|when valid 3G auth data is set, *not* when OPC is set
+|'info-aud'|aud3g.opc|32 hexadecimal digits|when valid 3G auth data is set, *not* when OP is set
+|'info-aud'|aud3g.ind_bitlen|0..28|when valid 3G auth data is set
+|'info-aud'|aud3g.sqn|0 .. 18446744073709551615|when valid 3G auth data is set
+|===
+
+This is an example Control Interface transcript that illustrates the various
+'info' commands:
+
+----
+include::../example_subscriber_info.ctrl[]
+----
+
+=== subscriber.by-*.ps-enabled, cs-enabled
+
+Disable or enable packet-/circuit-switched access for the given IMSI;
+
+* 'ps-enabled' switches access to GPRS or UMTS data services,
+* 'cs-enabled' switches access to voice services.
+
+When disabled, the next time this subscriber attempts to do a Location Updating
+GSUP operation for the given domain (i.e. from the SGSN for 'ps-enabled', from
+the MSC/VLR for 'cs-enabled'), it will be rejected by OsmoHLR. Currently
+connected GSUP clients will be notified via GSUP when a subscriber is being
+disabled, so that the subscriber can be dropped in case it is currently
+attached.
+
+The current 'ps-enabled'/'cs-enabled' status can be queried by 'GET' commands,
+and also by looking at 'nam_ps' and 'nam_cs' in a 'subscriber.by-*.info'
+response.
+
+A value of "1" indicates that the given domain is enabled, which is the
+default; a value of "0" disables access.
+
+This is an example transcript that illustrates 'ps-enabled' and 'cs-enabled'
+commands:
+
+----
+include::../example_subscriber_cs_ps_enabled.ctrl[]
+----

doc/manuals/chapters/dgsm.adoc

@@ -0,0 +1,491 @@
+== Distributed GSM / Multicast MS Lookup
+
+Distributed GSM (D-GSM) allows independent mobile core network stacks to provide voice, SMS and Roaming services to each
+other, without the need for centralised entities or administration authority, and in a way that is resilient against
+unstable network links between sites.
+
+D-GSM aims at communal networks, where several independent sites, let's call them villages, each have a full mobile core
+network infrastructure. It elegantly provides ad-hoc service for subscribers moving across villages, and allows villages
+to dynamically join or leave the cooperative network without the need for configuration changes at other sites.
+
+A challenge for linking separate sites is to find the current location of a subscriber. Typically, in mobile networks, a
+centralized entity keeps track of where to Page for subscribers. Running several fully independent sites with unreliable
+links between them makes it hard to provide such centralisation.
+
+D-GSM finds subscribers by mslookup, a service provided by OsmoHLR, typically using multicast DNS queries.  This allows
+routing Location Updating requests, calls, and SMS to the right site without administrative delay nor the need for a
+reliable link to a central database.
+
+D-GSM is highly resilient against single sites or links becoming temporarily unavailable. Service between still
+reachable sites simply continues; Service to a disconnected site resumes as soon as it becomes reachable again.
+
+This brings an entirely new paradigm to mobile core network infrastructure: as sites become reachable on the IP network
+and join the common IP multicast group, services between them become available immediately. Basically, the only premise
+is that IP routing and multicast works across sites, and that each site uses unique IPA names in the GSUP config.
+
+This chapter describes how D-GSM and mslookup work, and how to configure sites to use D-GSM, using Osmocom core network
+infrastructure.
+
+=== Finding Subscribers: mslookup Clients
+
+There are two fundamentally distinct subscriber lookups provided by the mslookup service.
+
+==== Find the Current Location of an MSISDN
+
+[[fig_dgsm_connect]]
+.mslookup for connecting subscribers: Alice is visiting village C; a phone call gets routed directly to her current location independently from her resident village infrastructure
+[graphviz]
+----
+digraph G {
+rankdir=LR
+
+subgraph cluster_village_b {
+	label="Village B"
+	ms_bob [label="Bob\n(from village B)",shape=box]
+	pbx_b [label="SIP B"]
+}
+
+subgraph cluster_village_c {
+	label="Village C"
+	ms_alice [label="Alice\n(from village A)",shape=box]
+	msc_c [label="MSC C"]
+	hlr_c [label="HLR C"]
+	sip_c [label="SIP C"]
+}
+
+ms_alice -> msc_c [style=dashed,arrowhead=none]
+msc_c -> hlr_c [label="attached",style=dashed,arrowhead=none]
+ms_bob -> pbx_b [label="call Alice"]
+pbx_b -> hlr_c [label="mslookup by MSISDN",style=dotted,dir=both]
+pbx_b -> sip_c -> msc_c -> ms_alice [label="call"]
+}
+----
+
+For example, if a subscriber is currently visiting another village, establish a phone call / send SMS towards that
+village.
+
+- To deliver a phone call, a SIP agent integrates an mslookup client to request the SIP service of an MSISDN's current
+  location (example: <<dgsm_conf_dialplan>>). It receives an IP address and port to send the SIP Invite to.
+
+- To deliver an SMS, an ESME integrates an mslookup client to request the SMPP service of an MSISDN's current location
+  (example: <<dgsm_conf_esme_smpp>>).
+
+The current location of a subscriber may change at any time, and, when moving across locations, a subscriber may
+suddenly lose reception to the previous location without explicitly detaching. Hence an mslookup request for the current
+location of an MSISDN may get numerous responses. To find the currently valid location, mslookup includes the age of the
+subscriber record, i.e. how long ago the subscriber was last reached. The one response with the youngest age reflects
+the current location.
+
+In order to evaluate several responses, mslookup always waits for a fixed amount of time (1 second), and then evaluates
+the available responses.
+
+Services are not limited to SIP and SMPP, arbitrarily named services can be added to the mslookup configuration.
+
+.Message sequence for locating an MSISDN to deliver a voice call
+["mscgen"]
+----
+msc {
+  hscale="2";
+  moms[label="MS,BSS\nvillage A"],momsc[label="MSC,MGW\nvillage A"],mosipcon[label="osmo-sip-connector\nvillage A"],mopbx[label="PBX\nvillage A"],mthlr[label="OsmoHLR\nvillage B"],mtsipcon[label="osmo-sip-connector\nvillage B"],mtmsc[label="MGW,MSC\nvillage B"],mtms[label="RAN,MS\nvillage B"];
+
+  moms =>> momsc [label="CC Setup"];
+  momsc =>> mosipcon [label="MNCC_SETUP_IND"];
+  mosipcon =>> mopbx [label="SIP INVITE"];
+  mopbx rbox mopbx [label="dialplan: launch mslookup by MSISDN"];
+  --- [label="multicast-DNS query to all connected sites"];
+  ...;
+  mopbx <<= mthlr [label="mDNS response\n(age)"];
+  mopbx rbox mopbx [label="wait ~ 1s for more mDNS responses"];
+  ...;
+  mopbx =>> mtsipcon [label="SIP INVITE (MT)"];
+  mtmsc <<= mtsipcon [label="MNCC_SETUP_REQ"];
+  mtms <<= mtmsc [label="Paging (CC)"];
+  moms rbox mtms [label="voice call commences"];
+
+}
+----
+
+==== Find the Home HLR for an IMSI
+
+[[fig_dgsm_roaming]]
+.mslookup for Roaming: Alice visits village B; she can attach to the local mobile network, which proxies HLR administration to her home village.
+[graphviz]
+----
+digraph G {
+rankdir=LR
+
+subgraph cluster_village_b {
+	label="Village B"
+
+	ms_alice [label="Alice\n(from village A)",shape=box]
+	msc_b [label="MSC B"]
+	hlr_b [label="HLR B"]
+}
+
+subgraph cluster_village_a {
+	label="Village A"
+	hlr_alice [label="Alice's home HLR"]
+}
+
+ms_alice -> msc_b -> hlr_b [label="Location\nUpdating"]
+hlr_b -> hlr_alice [label="mslookup by IMSI",style=dotted,dir=both]
+hlr_b -> hlr_alice [label="GSUP proxy forwarding"]
+}
+----
+
+For example, when attaching to a local network, a local resident gets serviced directly by the local village's HLR,
+while a visitor from another village gets serviced by the remote village's HLR (Roaming).
+
+A home HLR typically stays the same for a given IMSI. If the home site is reachable, there should be exactly one
+response to an mslookup request asking for it. The age of such a home-HLR response is always sent as zero.
+
+If a response's age is zero, mslookup does not wait for further responses and immediately uses the result.
+
+If there were more than one HLR accepting service for an IMSI, the one with the shortest response latency is used.
+
+=== mslookup Configuration
+
+OsmoHLR the main mslookup agent. It provides the responses for both current location services as well as for locating
+the fixed home-HLR. But naturally, depending on the mslookup request's purpose, different OsmoHLR instances will respond
+for a given subscriber.
+
+- When querying the home HLR, it is always the (typically single) home HLR instance that sends the mslookup response. As
+  soon as it finds the queried IMSI in the local HLR database, an OsmoHLR will respond to home-HLR requests.
+  In <<fig_dgsm_roaming>>, Alice's home HLR responds to the Roaming request ("where is the home HLR?").
+
+- When querying the location of an MSISDN, it is always the HLR proxy nearest to the servicing MSC that sends the
+  mslookup response. Even though the home HLR keeps the Location Updating record also for Roaming cases, it will only
+  respond to an mslookup service request if the subscriber has attached at a directly connected MSC. If attached at a
+  remote MSC, that MSC's remote HLR will be the GSUP proxy for the home HLR, and the remote HLR is responsible for
+  responding to service requests.
+  In <<fig_dgsm_roaming>>, HLR B is the nearest proxy and will answer all service requests ("where is this MSISDN?").
+  Alice's home HLR will not answer service requests, because it detects that the servicing MSC is connected via another
+  HLR proxy.
+
+[[dgsm_example_config]]
+==== Example
+
+Here is an osmo-hlr.cfg mslookup configuration example for one site, which is explained in subsequent chapters.
+
+ hlr
+  gsup
+   bind ip 10.9.8.7
+   ipa-name hlr-23
+ mslookup
+  mdns bind
+  server
+   service sip.voice at 10.9.8.7 5060
+   service smpp.sms at 10.9.8.7 2775
+
+OsmoHLR has both an mslookup server and a client.
+
+- The server responds to incoming service and home-HLR requests, when the local HLR is responsible.
+- The client is used as GSUP proxy to a remote home HLR (found by mslookup upon a locally unknown IMSI).
+- The client may also be used for forwarding SMS-over-GSUP.
+
+The mslookup service can be implemented by various methods.
+At the time of writing, the only method implemented is mDNS.
+
+==== mDNS
+
+The stock mslookup method is mDNS, multicast DNS. It consists of standard DNS encoding according to <<ietf-rfc1035>> and
+<<ietf-rfc3596>>, but sent and received on IP multicast. In the response, standard A and AAAA records return the
+service's IP address, while additional TXT records provide the service's port number and the MS attach age.
+
+TIP: To watch D-GSM mDNS conversations in wireshark, select "udp.port == 4266" (the default mslookup mDNS port
+number), right click on the packet to "Decode as...", and select "DNS".
+
+In OsmoHLR, the mDNS server and client are typically both enabled at the same time:
+
+ mslookup
+  mdns bind
+
+Server and client can also be enabled/disabled individually:
+
+ mslookup
+  server
+   mdns bind
+  client
+   mdns bind
+
+These examples use the default mslookup multicast IP address and port. It is possible to configure custom IP address and
+port, but beware that the IP address must be from a multicast range, see <<ietf-rfc5771>>:
+
+ mslookup
+  mdns bind 239.192.23.42 4266
+
+Domain names generated from mslookup queries (e.g. "sip.voice.123.msisdn") should not collide with IANA permitted
+domains. Therefore we add the "mdns.osmocom.org" suffix. It can be overridden as follows:
+
+ mslookup
+  mdns domain-suffix mdns.osmocom.org
+
+==== Server: Site Services
+
+The mslookup server requires a list of service addresses provided at the local site, in order to respond to service
+requests matching locally attached subscribers.
+
+ mslookup
+  server
+   service sip.voice at 10.9.8.7 5060
+   service smpp.sms at 10.9.8.7 2775
+
+In this example:
+
+- "10.9.8.7 5060" are the IP address and port on which the local site's osmo-sip-connector is bound to receive SIP
+  Invite requests.
+- "10.9.8.7 2775" are the local site's OsmoMSC SMPP bind address and port.
+
+Obviously, these IP addresses must be routable back to this site from all other sites. Using link-local or "ANY"
+addresses, like 127.0.0.1 or 0.0.0.0, will not work here. Instead, each service config requires a public IP address that
+all remote requestors are able to reach (not necessarily on the host that osmo-hlr is running on).
+
+If a site has more than one MSC, services can also be configured for each MSC individually, keyed by the IPA unit name
+that each MSC sends on the GSUP link:
+
+ mslookup
+  server
+   msc ipa-name msc-262-42-0
+    service sip.voice at 10.11.12.13 5060
+    service smpp.sms at 10.11.12.13 2775
+   msc ipa-name msc-901-70-0
+    service sip.voice at 10.9.8.7 5060
+    service smpp.sms at 10.9.8.7 2775
+
+Here, "msc-262-42-0" is the IPA name of a local OsmoMSC instance. To configure an OsmoMSC's IPA name on the GSUP link,
+see osmo-msc.cfg, setting `hlr` / `ipa-name`.
+
+For mslookup service responses, only Location Updatings in the Circuit Switched domain are relevant. OsmoHLR does manage
+IMSIs attaching in the Packet Switched domain (via an SGSN) similarly to Circuit Switched (via an MSC), but mslookup
+completely ignores the Packet Switched attach status.
+
+==== Server: Own GSUP Address
+
+When responding to home-HLR requests, OsmoHLR implicitly by default responds with its locally configured GSUP bind
+address (setting `hlr` / `gsup` / `bind ip`). If required, an explicit local GSUP address and port can be configured,
+for example:
+
+ hlr
+  gsup
+   bind ip 0.0.0.0
+   ipa-name hlr-23
+ mslookup
+  server
+   # osmo-hlr's own GSUP address to send in mslookup responses:
+   service gsup.hlr at 10.9.8.7 4222
+
+The gsup.hlr service can only be configured globally (because requests come from arbitrary mDNS clients, before a
+Location Updating has associated the IMSI with the requesting MSC).
+
+==== Client IPA Naming
+
+For reliable GSUP proxy routing to a remote HLR (Roaming), it is important that each GSUP client, i.e. each HLR, MSC and
+SGSN instance, has a unique IPA name.
+
+Example for configuring an OsmoHLR instance's IPA name:
+
+ hlr
+  gsup
+   ipa-name hlr-23
+
+Here, "hlr-23" is the unique identification of this OsmoHLR instance across all potentially connected D-GSM sites.
+
+Furthermore, each MSC and SGSN must have a uniquely distinct IPA name across all sites (here "msc-262-42-0" and
+"msc-901-70-0" are used as example IPA names for local MSCs).
+
+When this OsmoHLR connects to a remote HLR, be it for GSUP proxying or SMS-over-GSUP, it communicates its own IPA name
+(on GSUP link-up) as well as the IPA name of the requesting client MSC/SGSN (as Source Name in each message) to the
+remote OsmoHLR GSUP server. These names are used to route GSUP responses back to the respective requesting peer.
+
+If two MSCs were accidentally configured with identical names, a problem will occur as soon as both MSCs attempt to
+attach to the same OsmoHLR (either directly or via GSUP proxying). The MSC that shows up first will work normally, but
+any duplicate that shows up later will be rejected, since a route for its name already exists.
+
+=== Queries
+
+In URL notation, typical mslookup queries look like:
+
+ gsup.hlr.123456789.imsi
+ sip.voice.123.msisdn
+ smpp.sms.123.msisdn
+
+A query consists of
+
+- a service name ("gsup.hlr"),
+- an id ("123456789"),
+- the id type ("imsi").
+
+The calling client also defines a timeout to wait for responses.
+
+The mslookup ID types are fixed, while service names can be chosen arbitrarily.
+
+.mslookup ID types, no other ID types are understood by mslookup
+[options="header",width="100%",cols="20%,80%"]
+|===
+|ID Type|Description
+|imsi|An IMSI as existing in an OsmoHLR subscriber database
+|msisdn|A phone number as configured in an OsmoHLR subscriber database
+|===
+
+.mslookup service name conventions, arbitrary service names can be added as required
+[options="header",width="100%",cols="20%,20%,60%"]
+|===
+|Service Name|Protocol|Description
+|gsup.hlr | GSUP | Home HLR's GSUP server, to handle Location Updating related procedures
+|sip.voice | SIP | OsmoSIPConnector, to receive a SIP Invite (MT side of a call)
+|smpp.sms | SMPP | Destination OsmoMSC (or other SMPP server) to deliver an SMS to the recipient
+|gsup.sms | GSUP | GSUP peer to deliver an SMS to the recipient using SMS-over-GSUP
+|===
+
+Arbitrarily named services can be added to the mslookup configuration and queried by mslookup clients; as soon as a
+service name is present in osmo-hlr.cfg, it can be queried from any mslookup client.
+
+Service names should consist of a protocol name (like "sip", "gsup", "english") and an intended action/entity (like
+"voice", "hlr", "greeting").
+
+=== Service Client Implementation
+
+In principle, arbitrary services could query target addresses via mslookup, leaving it up to any and all kinds of
+clients to find their respective destination addresses. But of course, mslookup was designed with specific services in
+mind, namely:
+
+- SIP call agents and
+- SMS delivery (an ESME or SMSC)
+
+The following chapters describe examples of setting up a working distributed core network providing SIP voice calls and
+SMS forwarding across sites.
+
+==== mslookup Library
+
+The OsmoHLR provides an mslookup client C library, libosmo-mslookup. Service lookups can be integrated directly
+in client programs using this library. However, its mDNS implementation requires the libosmocore select() loop, which
+can be challenging to integrate in practice. An alternative solution is the osmo-mslookup-client tool.
+
+[[dgsm_osmo_mslookup_client]]
+==== osmo-mslookup-client
+
+The mslookup C library is available, but often, a simpler approach for client implementations is desirable:
+
+- When querying for a service address, the client is typically interested in the single final best result (youngest age
+  / first responding home HLR).
+- Voice call and SMS clients typically would block until an mslookup result is known. For example, the FreeSwitch
+  dialplan integration expects a result synchronously, i.e. without waiting for mslookup responses via a select() loop.
+- Integrating the libosmocore select() loop required for mDNS can break the already existing socket handling in the
+  client program.
+
+The osmo-mslookup-client cmdline tool provides a trivial way to synchronously acquire the single result for an mslookup
+request. The service client can invoke an osmo-mslookup-client process per request and read the result from stdout.
+
+Each invocation obviously spawns a separate process and opens a multicast socket for mDNS. For better scalability,
+osmo-mslookup-client can also be run as a daemon, providing results via a unix domain socket. Using synchronous write()
+and recv() allows blocking until a result is received without interfering with the client program's select() setup.
+
+By itself, osmo-mslookup-client is also helpful as a diagnostic tool:
+
+----
+$ osmo-mslookup-client sip.voice.1001.msisdn
+sip.voice.1001.msisdn	ok	10.9.8.7	5060
+
+$ osmo-mslookup-client gsup.hlr.901700000014701.imsi
+gsup.hlr.901700000014701.imsi	ok	10.9.8.7	4222
+
+$ osmo-mslookup-client gsup.hlr.111111.imsi
+gsup.hlr.111111.imsi	not-found
+
+$ osmo-mslookup-client gsup.hlr.1001.msisdn sip.voice.1001.msisdn smpp.sms.1001.msisdn foo.1001.msisdn
+gsup.hlr.1001.msisdn	ok	10.9.8.7	4222
+foo.1001.msisdn	not-found
+smpp.sms.1001.msisdn	ok	10.9.8.7	2775
+sip.voice.1001.msisdn	ok	10.9.8.7	5060
+
+$ osmo-mslookup-client --csv-headers gsup.hlr.901700000014701.imsi
+QUERY	RESULT	V4_IP	V4_PORT	V6_IP	V6_PORT
+gsup.hlr.901700000014701.imsi	ok	10.9.8.7	4222
+
+$ osmo-mslookup-client -f json gsup.hlr.901700000014701.imsi
+{"query": "gsup.hlr.901700000014701.imsi", "result": "ok", "v4": ["10.9.8.7", "4222"]}
+----
+
+For full help including example client invocations in Python, see the output of:
+
+ osmo-mslookup-client -h
+
+==== SIP Service Client
+
+[[dgsm_conf_dialplan]]
+===== FreeSwitch dialplan.py
+
+The FreeSWITCH PBX software <<freeswitch_pbx>> offers a Python integration to determine a SIP call recipient by a custom
+dialplan implementation. An example dialplan implementation for FreeSWITCH that uses D-GSM mslookup is provided in the
+osmo-hlr source tree under `contrib`, called `freeswitch_dialplan_dgsm.py`.
+
+To integrate it with your FREESWITCH setup, add a new `extension` block to your `dialplan/public.xml`:
+
+----
+    <extension name="outbound">
+      <condition field="destination_number" expression=".*">
+	<action application="set" data="hangup_after_bridge=true"/>
+	<action application="set" data="session_in_hangup_hook=true"/>
+	<action application="set" data="ringback=%(2000, 4000, 440.0, 480.0)"/>
+	<action application="python" data="freeswitch_dialplan_dgsm"/>
+      </condition>
+    </extension>
+----
+
+Make sure that the dir containing `freeswitch_dialplan_dgsm.py` is in your `PYTHONPATH` environment variable, and start
+the server:
+
+----
+$ export PYTHONPATH="$PYTHONPATH:/home/user/code/osmo-hlr/contrib/dgsm"
+$ freeswitch -nf -nonat -nonatmap -nocal -nort -c
+----
+
+==== SMS Service Client
+
+[[dgsm_conf_esme_smpp]]
+===== SMS via SMPP Port
+
+An example ESME using D-GSM mslookup, `esme_dgsm.py`, is provided in the osmo-hlr source tree under `contrib`. It
+attaches to OsmoMSC's SMPP port to send SMS to recipients determined by mslookup.
+
+OsmoMSC should be configured as "smpp-first", so that all SMS routing is determined by mslookup. If configured without
+smpp-first, OsmoMSC may try to deliver an SMS locally, even though the recipient has recently moved to a different site.
+
+An example OsmoMSC configuration to work with esme_dgsm.py:
+
+----
+smpp
+ local-tcp-ip 127.0.0.1 2775
+ system-id test-msc
+ policy closed
+ smpp-first
+ # outgoing to esme_dgsm.py
+ esme OSMPP
+  no alert-notifications
+  password foo
+  default-route
+ # incoming from esme_dgsm.py
+ esme ISMPP
+  no alert-notifications
+  password foo
+----
+
+Launch esme_dgsm.py alongside OsmoMSC:
+
+----
+./esme_dgsm.py --src-host 127.0.0.1
+----
+
+esme_dgsm.py will be notified via SMPP for each SMS to be delivered, and will forward them either to a remote
+recipient, or back to the same OsmoMSC, depending on the mslookup result. If the MSISDN is not reachable (or
+esme_dgsm.py can't handle the message for other reasons), it returns the RSYSERR code back to OsmoMSC.
+
+Note that the esme_dgsm.py is a proof of concept and should not be used in production. It has several limitations, such
+as not supporting multipart SMS messages.
+
+===== SMS-Over-GSUP
+
+The GSUP protocol defines SMS delivery messages. When OsmoMSC is configured to deliver SMS via GSUP, MO SMS are directly
+forwarded to the HLR, which will determine where to forward the SMS-over-GSUP messages using its mslookup client.
+
+FIXME implement this

doc/manuals/chapters/overview.adoc

@@ -0,0 +1,69 @@
+[[overview]]
+== Overview
+
+This manual should help you getting started with OsmoHLR. It will cover
+aspects of configuring and running the OsmoHLR.
+
+[[intro_overview]]
+=== About OsmoHLR
+
+OsmoHLR is Osmocom's minimal implementation of a Home Location Register (HLR)
+for 2G and 3G GSM and UMTS mobile core networks. Its interfaces are:
+
+- GSUP, serving towards OsmoMSC and OsmoSGSN;
+- A local SQLite database;
+- The Osmocom typical telnet VTY and CTRL interfaces.
+
+Originally, the OpenBSC project's OsmoNITB all-in-one implementation had an
+integrated HLR, managing subscribers and SMS in the same local database. Along
+with the separate OsmoMSC and its new VLR component, OsmoHLR was implemented
+from scratch to alleviate various shortcomings of the internal HLR:
+
+- The separate HLR allows using centralized subscriber management for both
+  circuit-switched and packet-switched domains (i.e. one OsmoHLR for both
+  OsmoMSC and OsmoSGSN).
+
+- VLR and HLR brought full UMTS AKA (Authentication and Key Agreement) support,
+  i.e. Milenage authentication in both the full 3G variant as well as the
+  backwards compatible 2G variant.
+
+- In contrast to the OsmoNITB, the specific way the new OsmoMSC's VLR accesses
+  OsmoHLR brings fully asynchronous subscriber database access.
+
+Find the OsmoHLR issue tracker and wiki online at
+
+- https://osmocom.org/projects/osmo-hlr
+- https://osmocom.org/projects/osmo-hlr/wiki
+
+
+[[fig-gsm]]
+.Typical GSM network architecture used with OsmoHLR
+[graphviz]
+----
+digraph G {
+	rankdir=LR;
+	subgraph cluster_hlr {
+		label = "OsmoHLR";
+		GSUP [label="GSUP server"]
+		DB [label="SQLite DB"]
+		GSUP->DB
+		DB->CTRL [dir="back"]
+		DB->VTY [dir="back"]
+	}
+
+	Admin [label="Admin and\nMaintenance"]
+	SW [label="3rd party software\nintegration"]
+	VTY->Admin [dir="back"]
+	CTRL->SW [dir="back"]
+		
+	MSC [label="MSC/VLR"]
+	MSC->GSUP  [label="GSUP"]
+	SGSN->GSUP [label="GSUP"]
+
+	BSC->MSC
+	HNBGW->MSC
+	HNBGW->SGSN
+	PCU->SGSN
+}
+----
+

doc/manuals/chapters/running.adoc

@@ -0,0 +1,87 @@
+== Running OsmoHLR
+
+The OsmoHLR executable (`osmo-hlr`) offers the following command-line
+arguments:
+
+=== SYNOPSIS
+
+*osmo-hlr* [-h] [-c 'CONFIGFILE'] [-l 'DATABASE'] [-d 'DBGMASK'] [-D] [-s] [-T] [-e 'LOGLEVEL'] [-U] [-V]
+
+=== OPTIONS
+
+// Keep the order the same as in osmo-hlr --help!
+
+*-h, --help*::
+	Print a short help message about the supported options
+*-c, --config-file 'CONFIGFILE'*::
+	Specify the file and path name of the configuration file to be
+	used. If none is specified, use `osmo-hlr.cfg` in the current
+	working directory.
+*-l, --database 'DATABASE'*::
+	Specify the file name of the SQLite3 database to use as HLR/AUC
+	storage
+*-d, --debug 'DBGMASK','DBGLEVELS'*::
+	Set the log subsystems and levels for logging to stderr. This
+	has mostly been superseded by VTY-based logging configuration,
+	see <<logging>> for further information.
+*-D, --daemonize*::
+	Fork the process as a daemon into background.
+*-s, --disable-color*::
+	Disable colors for logging to stderr. This has mostly been
+	deprecated by VTY based logging configuration, see <<logging>>
+	for more information.
+*-T, --timestamp*::
+	Enable time-stamping of log messages to stderr. This has mostly
+	been deprecated by VTY based logging configuration, see
+	<<logging>> for more information.
+*-e, --log-level 'LOGLEVEL'*::
+	Set the global log level for logging to stderr. This has mostly
+	been deprecated by VTY based logging configuration, see
+	<<logging>> for more information.
+*-U, --db-upgrade*::
+	Allow HLR database schema upgrades. If OsmoHLR was updated and
+	requires a newer database schema, it will refuse to start unless
+	this option is specified. The updated database can not be
+	downgraded, make backups as necessary.
+*-V, --version*::
+	Print the compile-time version number of the OsmoHLR program
+
+=== Bootstrap the Database
+
+If no database exists yet, OsmoHLR will automatically create and bootstrap a
+database file with empty tables. If no `-l` command-line option is provided,
+this database file will be created in the current working directory.
+
+Alternatively, you may use the `osmo-hlr-db-tool`, which is installed along
+with `osmo-hlr`, to bootstrap an empty database, or to migrate subscriber data
+from an old 'OsmoNITB' database. See <<db_import_nitb>>.
+
+=== Multiple instances
+
+Running multiple instances of `osmo-hlr` on the same computer is possible if
+all interfaces (VTY, CTRL) are separated using the appropriate configuration
+options. The IP based interfaces are binding to local host by default. In order
+to separate the processes, the user has to bind those services to specific but
+different IP addresses and/or ports.
+
+The VTY and the Control interface can be bound to IP addresses from the loopback
+address range, for example:
+
+----
+line vty
+ bind 127.0.0.2
+ctrl
+ bind 127.0.0.2
+----
+
+The GSUP interface can be bound to a specific IP address by the following
+configuration options:
+
+----
+hlr
+ gsup
+  bind ip 10.23.42.1
+----
+
+NOTE: At the time of writing, OsmoHLR lacks a config option to change the GSUP
+port, which is by default TCP port 4222.

doc/manuals/chapters/subscribers.adoc

@@ -0,0 +1,209 @@
+== Managing Subscribers
+
+Subscribers are kept in a local SQLite database file and can be managed via VTY
+and CTRL interfaces.
+
+This section provides some examples; also refer to the OsmoHLR VTY reference
+manual <<vty-ref-osmohlr>> as well as the Control interface described in
+<<hlr-ctrl>>.
+
+=== Example: Add/Update/Delete Subscriber via VTY
+
+The following telnet VTY session adds a subscriber complete with GSM (2G) and
+UMTS (3G and 2G) authentication tokens, and finally removes the subscriber
+again; it assumes that osmo-hlr is running and listening for telnet VTY
+connections on localhost:
+
+----
+$ telnet localhost 4258
+include::../example_subscriber_add_update_delete.vty[]
+----
+
+[[subscriber-params]]
+=== Subscriber Parameters
+
+The following parameters are managed for each subscriber of the HLR, modelled
+roughly after 3GPP TS 23.008, version 13.3.0; note that not all of these
+parameters are necessarily in active use.
+
+The `aud3g` table also applies to 2G networks: it provides UMTS AKA tokens for
+Milenage authentication, which is available both on 3G and 2G networks. On 2G,
+when both MS and network are R99 capable (like OsmoMSC and OsmoSGSN are), the
+full UMTS AKA with Milenage keys from `aud_3g`, using AUTN and extended RES
+tokens, is available. With pre-R99 MS or network configurations, the GSM AKA
+compatible variant of Milenage, still using the Milenage keys from `aud_3g` but
+transceiving only RAND and SRES, may be applicable. (See 3GPP TS 33.102, chapter
+6.8.1, Authentication and key agreement of UMTS subscribers.)
+
+.OsmoHLR's subscriber parameters
+[options="header",width="100%",cols="20%,20%,60%"]
+|===
+|Name|Example|Description
+|imsi|901700000014701|identity of the SIM/USIM, 3GPP TS 23.008 chapter 2.1.1.1
+|msisdn|2342123|number to dial to reach this subscriber (multiple MSISDNs can be stored per subscriber), 3GPP TS 23.008 chapter 2.1.2
+|imeisv|4234234234234275|identity of the mobile device and software version, 3GPP TS 23.008 chapter 2.2.3
+|aud2g.algo|comp128v3|Authentication algorithm ID for GSM AKA, corresponds to enum osmo_auth_algo
+|aud2g.ki||Subscriber's secret key (128bit)
+|aud3g.algo|milenage|Authentication algorithm ID for UMTS AKA (applies to both 3G and 2G networks), corresponds to enum osmo_auth_algo
+|aud3g.k|(32 hexadecimal digits)|Subscriber's secret key (128bit)
+|aud3g.op|(32 hexadecimal digits)|Operator's secret key (128bit)
+|aud3g.opc|(32 hexadecimal digits)|Secret key derived from OP and K (128bit), alternative to using OP which does not disclose OP to subscribers
+|aud3g.sqn|123|Sequence number of last used key (64bit unsigned)
+|aud3g.ind_bitlen|5|Nr of index bits at lower SQN end
+|apn||
+|vlr_number||3GPP TS 23.008 chapter 2.4.5
+|msc_number||3GPP TS 23.008 chapter 2.4.6
+|sgsn_number||3GPP TS 23.008 chapter 2.4.8.1
+|sgsn_address||3GPP TS 23.008 chapter 2.13.10
+|ggsn_number||3GPP TS 23.008 chapter 2.4.8.2
+|gmlc_number||3GPP TS 23.008 chapter 2.4.9.2
+|smsc_number||3GPP TS 23.008 chapter 2.4.23
+|periodic_lu_tmr||3GPP TS 23.008 chapter 2.4.24
+|periodic_rau_tau_tmr||3GPP TS 23.008 chapter 2.13.115
+|nam_cs|1|Enable/disable voice access (3GPP TS 23.008 chapter 2.1.1.2: network access mode)
+|nam_ps|0|Enable/disable data access (3GPP TS 23.008 chapter 2.1.1.2: network access mode)
+|lmsi||3GPP TS 23.008 chapter 2.1.8
+|ms_purged_cs|0|3GPP TS 23.008 chapter 2.7.5
+|ms_purged_ps|1|3GPP TS 23.008 chapter 2.7.6
+|===
+
+=== Configuring the Subscribers Create on Demand Feature
+
+Usually a HLR will only allow mobile equipment (ME) on the network, if the HLR
+has a subscriber entry with the ME's IMSI. But OsmoHLR can also be configured to
+automatically create new entries for new IMSIs, with the
+`subscriber-create-on-demand` VTY option. The obvious use case is creating the
+new subscriber entry and then allowing the ME to use both the CS
+(Circuit Switched) and PS (Packet Switched) NAM (Network Access Mode).
+
+.osmo-hlr.cfg
+----
+hlr
+ subscriber-create-on-demand 5 cs+ps
+----
+
+On the other hand, operators might only want to give network access to IMSIs, of
+which they know the owner. In order to do that, one can set the default NAM to
+`none` and manually approve new subscribers by changing the NAM (e.g. over the
+VTY, see the example below).
+
+Oftentimes it is hard to know, which IMSI belongs to which ME, but the IMEI is
+readily available. If you configure your MSC to send IMEI checking requests to
+the HLR, before sending location update requests, the subscribers created on
+demand can also have the IMEI stored in the HLR database. With OsmoMSC, this
+is done by writing `check-imei-rqd early` in the `msc` section of osmo-msc.cfg.
+Then enable storing the IMEI when receiving check IMEI requests with
+`store-imei` in the OsmoHLR configuration.
+
+.osmo-msc.cfg
+----
+msc
+ check-imei-rqd early
+----
+
+.osmo-hlr.cfg
+----
+hlr
+ subscriber-create-on-demand 5 none
+ store-imei
+----
+
+.Example: Enabling CS and PS NAM via VTY for a known IMEI
+----
+OsmoHLR> enable
+OsmoHLR# subscriber imei 35761300444848 show
+    ID: 1
+    IMSI: 123456789023000
+    MSISDN: 58192 <1>
+    IMEI: 35761300444848
+    CS disabled <2>
+    PS disabled <2>
+OsmoHLR# subscriber imei 35761300444848 update network-access-mode cs+ps
+OsmoHLR# subscriber imei 35761300444848 show
+    ID: 1
+    IMSI: 123456789023000
+    MSISDN: 58192
+    IMEI: 35761300444848
+----
+<1> Randomly generated 5 digit MSISDN
+<2> Disabled CS and PS NAM prevent the subscriber from accessing the network
+
+
+=== Import Subscriber Data
+
+==== Scripted Import
+
+WARNING: It is not generally a good idea to depend on the HLR database's internal table structure, but in the lack of an
+automated import procedure, this example is provided as an ad-hoc method to aid automated subscriber import. This is not
+guaranteed to remain valid.
+
+NOTE: We may add CSV and other import methods to the `osmo-hlr-db-tool`, but so far that is not implemented. Contact the
+community if you are interested in such a feature being implemented.
+
+NOTE: `sqlite3` is available from your distribution packages or `sqlite.org`.
+
+Currently, probably the easiest way to automatically import subscribers to OsmoHLR is to write out a text file with SQL
+commands per subscriber, and feed that to `sqlite3`, as described below.
+
+A difficulty is to always choose subscriber IDs that are not yet in use. For an initial import, the subscriber ID may be
+incremented per subscriber record. If adding more subscribers to an existing database, it is necessary to choose
+subscriber IDs that are not yet in use. Get the highest ID in use with:
+
+----
+sqlite3 hlr.db 'select max(id) from subscriber'
+----
+
+A full SQL example of adding a single subscriber with id 23, IMSI 001010123456789, MSISDN 1234, Ki for COMP128v1, and K
+and OPC for Milenage:
+
+----
+INSERT subscriber (id, imsi, msisdn) VALUES (23, '001010123456789', '1234');
+
+INSERT INTO auc_2g (subscriber_id, algo_id_2g, ki)
+VALUES(23, 1, '0123456789abcdef0123456789abcdef');
+
+INSERT INTO auc_3g (subscriber_id, algo_id_3g, k, op, opc)
+VALUES(23, 5, '0123456789abcdef0123456789abcdef',NULL,'0123456789abcdef0123456789abcdef');
+----
+
+Table entries to `auc_2g` and/or `auc_3g` may be omitted if no such key material is required.
+
+UMTS Milenage auth (on both 2G and 3G RAN) is configured by the `auc_3g` table. `algo_id_3g` must currently always be 5
+(MILENAGE).
+
+The algorithm IDs for `algo_id_2g` and `algo_id_3g` are:
+
+.Algorithm IDs in OsmoHLR's database
+[options="header",width="50%",cols="40%,60%"]
+|===
+|`algo_id_2g` / `algo_id_3g` | Authentication Algorithm
+| 1 | COMP128v1
+| 2 | COMP128v2
+| 3 | COMP128v3
+| 4 | XOR
+| 5 | MILENAGE
+|===
+
+Create an empty HLR database with
+
+----
+osmo-hlr-db-tool -l hlr.db create
+----
+
+Repeat above SQL commands per subscriber, incrementing the subscriber ID for each block, then feed the SQL commands for
+the subscribers to be imported to the `sqlite3` command line tool:
+
+----
+sqlite3 hlr.db < subscribers.sql
+----
+
+[[db_import_nitb]]
+==== Import OsmoNITB database
+
+To upgrade from old OsmoNITB to OsmoHLR, use `osmo-hlr-db-tool`:
+
+----
+osmo-hlr-db-tool -l hlr.db import-nitb-db nitb.db
+----
+
+Be aware that the import is lossy, only the IMSI, MSISDN, nam_cs/ps and 2G auth data are set.

doc/manuals/chapters/ussd.adoc

@@ -0,0 +1,92 @@
+[[ussd]]
+== Unstructured Supplementary Services Data (USSD)
+
+The _Unstructured Supplementary Services Data (USSD)_ is one service within
+2G/3G networks next to other services such as circuit-switched voice, packet-switched
+data and SMS (Short Message Service).
+
+It is on an abstract level quite similar to SMS in that USSD can be used to send
+textual messages.  However, there are the following differences:
+
+* USSD is between the MS (phone) and an USSD application on the network, while
+  SMS is primarily between two subscribers identified by their MSISDN
+* USSD is faster, as it doesn't suffer from the complicated three-layer CP/RP/TP
+  protocol stack of SMS with it's acknowledgement of the acknowledged acknowledgement.
+* USSD is session-oriented, i.e. a dialogue/session between subscriber and application
+  can persist for the transfer of more than one message.  The dedicated radio channel
+  on the RAN remains established throughout that dialogue.
+
+=== USSD in Osmocom
+
+Until August 2018, OsmoMSC contained some minimalistic internal USSD
+handling with no
+ability to attach/extend it with external USSD applications.
+
+From August 2018 onwards, OsmoMSC doesn't contain any internal USSD
+handlers/applications anymore.  Instead, all USSD is transported to/from
+OsmoHLR via the GSUP protocol.
+
+OsmoHLR contains some intenal USSD handlers and can route USSD messages
+to any number of external USSD entities (EUSEs).  The EUSE also use GSUP
+to communicate USSD from/to OsmoHLR.
+
+Each EUSE is identified by its name.  The name consists of a single-word
+string preceding a currently fixed ("-00-00-00-00-00-00") suffix.
+There is no authentication between EUSE and OsmoHLR: Any client program
+able to connect to the GSUP port of OsmoHLR can register as any EUSE
+(name).
+
+NOTE:: We plan to remove the requirement for this suffix as soon as we
+are done resolving all more important issues.
+
+=== USSD Configuration
+
+USSD configuration in OsmoHLR happens within the `hlr` VTY node.
+
+`euse foobar-00-00-00-00-00-00` defines an EUSE with the given name `foobar`
+
+`ussd route prefix *123 external foobar-00-00-00-00-00-00` installs a
+prefix route to the named EUSE.  All USSD short codes starting with *123 will be
+routed to the named EUSE.
+
+`ussd route prefix *#100# internal own-msisdn` installs a prefix route
+to the named internal USSD handler.  The above command will restore
+the old behavior, in which *#100# will return a text message containing
+the subscribers own phone number.  More information on internal USSD
+handlers can be found in <<iuse_handlers>>.
+
+`ussd default-route external foobar-00-00-00-00-00-00` installs a
+default route to the named EUSE.  This means that all USSD codes for
+which no more specific route exists will be routed to the named EUSE.
+
+[[iuse_handlers]]
+=== Built-in USSD handlers
+
+OsmoHLR has an Internal USSD Entity (IUSE) that allows to handle some
+USSD requests internally.  It features a set of simple handlers, which
+can be assigned to one or more USSD request prefixes:
+
+* `own-msisdn` returns subscriber's MSISDN (if assigned);
+* `own-imsi` returns subscriber's IMSI;
+* `test-idle` keeps the session idle until the MS terminates it, or
+  the guard timer expires (may be useful for testing).
+
+Additional handlers can be added on request.
+
+=== Example EUSE program
+
+We have provided an example EUSE developed in C language using existing
+Osmocom libraries for GSUP protocol handling and USSD encoding/decoding.
+It will register as `foobar` EUSE to OsmoHLR on localhost.  You can run
+it on a different machine by specifying e.g. `osmo-euse-demo 1.2.3.4 5678`
+to make it connect to OsmoHLR on IP address 1.2.3.4 and GSUP/TCP port
+5678.
+
+The idea is that you can use this as a template to develop your own USSD
+applications, or any gateways to other protocols or interfaces.
+
+You can find it in `osmo-hlr/src/osmo-euse-demo.c` or online by
+following the link to http://git.osmocom.org/osmo-hlr/tree/src/osmo-euse-demo.c
+
+This demonstration program will echo back any USSD message sent/routed
+to it, quoted like _You sent "..."_.

doc/manuals/example_subscriber_add_update_delete.vty

@@ -0,0 +1,34 @@
+OsmoHLR> enable
+OsmoHLR# subscriber imsi 123456789023000 create
+% Created subscriber 123456789023000
+    ID: 1
+    IMSI: 123456789023000
+    MSISDN: none
+
+OsmoHLR# subscriber imsi 123456789023000 update msisdn 423
+% Updated subscriber IMSI='123456789023000' to MSISDN='423'
+
+OsmoHLR# subscriber msisdn 423 update aud3g milenage k deaf0ff1ced0d0dabbedd1ced1cef00d opc cededeffacedacefacedbadfadedbeef
+OsmoHLR# subscriber msisdn 423 show
+    ID: 1
+    IMSI: 123456789023000
+    MSISDN: 423
+    3G auth: MILENAGE
+             K=deaf0ff1ced0d0dabbedd1ced1cef00d
+             OPC=cededeffacedacefacedbadfadedbeef
+             IND-bitlen=5
+
+OsmoHLR# subscriber msisdn 423 update aud2g comp128v3 ki beefedcafefaceacedaddeddecadefee
+OsmoHLR# subscriber msisdn 423 show
+    ID: 1
+    IMSI: 123456789023000
+    MSISDN: 423
+    2G auth: COMP128v3
+             KI=beefedcafefaceacedaddeddecadefee
+    3G auth: MILENAGE
+             K=deaf0ff1ced0d0dabbedd1ced1cef00d
+             OPC=cededeffacedacefacedbadfadedbeef
+             IND-bitlen=5
+
+OsmoHLR# subscriber imsi 123456789023000 delete
+% Deleted subscriber for IMSI '123456789023000'

doc/manuals/example_subscriber_cs_ps_enabled.ctrl

@@ -0,0 +1,71 @@
+GET 1 subscriber.by-msisdn-103.info
+GET_REPLY 1 subscriber.by-msisdn-103.info 
+id	3
+imsi	901990000000003
+msisdn	103
+nam_cs	1
+nam_ps	1
+ms_purged_cs	0
+ms_purged_ps	0
+periodic_lu_timer	0
+periodic_rau_tau_timer	0
+lmsi	00000000
+
+GET 2 subscriber.by-msisdn-103.ps-enabled
+GET_REPLY 2 subscriber.by-msisdn-103.ps-enabled 1
+
+SET 3 subscriber.by-msisdn-103.ps-enabled 0
+SET_REPLY 3 subscriber.by-msisdn-103.ps-enabled OK
+
+GET 4 subscriber.by-msisdn-103.ps-enabled
+GET_REPLY 4 subscriber.by-msisdn-103.ps-enabled 0
+
+GET 5 subscriber.by-msisdn-103.info
+GET_REPLY 5 subscriber.by-msisdn-103.info 
+id	3
+imsi	901990000000003
+msisdn	103
+nam_cs	1
+nam_ps	0
+ms_purged_cs	0
+ms_purged_ps	0
+periodic_lu_timer	0
+periodic_rau_tau_timer	0
+lmsi	00000000
+
+SET 6 subscriber.by-msisdn-103.cs-enabled 0
+SET_REPLY 6 subscriber.by-msisdn-103.cs-enabled OK
+
+GET 7 subscriber.by-msisdn-103.cs-enabled
+GET_REPLY 7 subscriber.by-msisdn-103.cs-enabled 0
+
+GET 8 subscriber.by-msisdn-103.info
+GET_REPLY 8 subscriber.by-msisdn-103.info 
+id	3
+imsi	901990000000003
+msisdn	103
+nam_cs	0
+nam_ps	0
+ms_purged_cs	0
+ms_purged_ps	0
+periodic_lu_timer	0
+periodic_rau_tau_timer	0
+lmsi	00000000
+
+SET 9 subscriber.by-msisdn-103.cs-enabled 1
+SET_REPLY 9 subscriber.by-msisdn-103.cs-enabled OK
+SET 10 subscriber.by-msisdn-103.ps-enabled 1
+SET_REPLY 10 subscriber.by-msisdn-103.ps-enabled OK
+
+GET 11 subscriber.by-msisdn-103.info
+GET_REPLY 11 subscriber.by-msisdn-103.info 
+id	3
+imsi	901990000000003
+msisdn	103
+nam_cs	1
+nam_ps	1
+ms_purged_cs	0
+ms_purged_ps	0
+periodic_lu_timer	0
+periodic_rau_tau_timer	0
+lmsi	00000000

doc/manuals/example_subscriber_info.ctrl

@@ -0,0 +1,42 @@
+GET 1 subscriber.by-imsi-901990000000003.info
+GET_REPLY 1 subscriber.by-imsi-901990000000003.info 
+id	3
+imsi	901990000000003
+msisdn	103
+nam_cs	1
+nam_ps	1
+ms_purged_cs	0
+ms_purged_ps	0
+periodic_lu_timer	0
+periodic_rau_tau_timer	0
+lmsi	00000000
+
+GET 2 subscriber.by-msisdn-103.info-aud
+GET_REPLY 2 subscriber.by-msisdn-103.info-aud 
+aud2g.algo	COMP128v1
+aud2g.ki	000102030405060708090a0b0c0d0e0f
+aud3g.algo	MILENAGE
+aud3g.k	000102030405060708090a0b0c0d0e0f
+aud3g.opc	101112131415161718191a1b1c1d1e1f
+aud3g.ind_bitlen	5
+aud3g.sqn	0
+
+GET 3 subscriber.by-id-3.info-all
+GET_REPLY 3 subscriber.by-id-3.info-all 
+id	3
+imsi	901990000000003
+msisdn	103
+nam_cs	1
+nam_ps	1
+ms_purged_cs	0
+ms_purged_ps	0
+periodic_lu_timer	0
+periodic_rau_tau_timer	0
+lmsi	00000000
+aud2g.algo	COMP128v1
+aud2g.ki	000102030405060708090a0b0c0d0e0f
+aud3g.algo	MILENAGE
+aud3g.k	000102030405060708090a0b0c0d0e0f
+aud3g.opc	101112131415161718191a1b1c1d1e1f
+aud3g.ind_bitlen	5
+aud3g.sqn	0

doc/manuals/osmohlr-usermanual-docinfo.xml

@@ -0,0 +1,47 @@
+<revhistory>
+  <revision>
+    <revnumber>1</revnumber>
+    <date>September 18th, 2017</date>
+    <authorinitials>NH</authorinitials>
+    <revremark>
+      Initial version; based on OsmoNITB manual version 2.
+    </revremark>
+  </revision>
+</revhistory>
+
+<authorgroup>
+  <author>
+    <firstname>Neels</firstname>
+    <surname>Hofmeyr</surname>
+    <email>nhofmeyr@sysmocom.de</email>
+    <authorinitials>NH</authorinitials>
+    <affiliation>
+      <shortaffil>sysmocom</shortaffil>
+      <orgname>sysmocom - s.f.m.c. GmbH</orgname>
+      <jobtitle>Senior Developer</jobtitle>
+    </affiliation>
+  </author>
+</authorgroup>
+
+<copyright>
+  <year>2017</year>
+  <holder>sysmocom - s.f.m.c. GmbH</holder>
+</copyright>
+
+<legalnotice>
+  <para>
+	Permission is granted to copy, distribute and/or modify this
+	document under the terms of the GNU Free Documentation License,
+	Version 1.3 or any later version published by the Free Software
+	Foundation; with the Invariant Sections being just 'Foreword',
+	'Acknowledgements' and 'Preface', with no Front-Cover Texts,
+	and no Back-Cover Texts.  A copy of the license is included in
+	the section entitled "GNU Free Documentation License".
+  </para>
+  <para>
+	The Asciidoc source code of this manual can be found at
+	<ulink url="http://git.osmocom.org/osmo-gsm-manuals/">
+		http://git.osmocom.org/osmo-gsm-manuals/
+	</ulink>
+  </para>
+</legalnotice>

doc/manuals/osmohlr-usermanual.adoc

@@ -0,0 +1,39 @@
+:gfdl-enabled:
+:program-name: OsmoHLR
+
+OsmoHLR User Manual
+====================
+Neels Hofmeyr <nhofmeyr@sysmocom.de>
+
+
+include::./common/chapters/preface.adoc[]
+
+include::{srcdir}/chapters/overview.adoc[]
+
+include::{srcdir}/chapters/running.adoc[]
+
+include::{srcdir}/chapters/subscribers.adoc[]
+
+include::{srcdir}/chapters/ussd.adoc[]
+
+include::./common/chapters/vty.adoc[]
+
+include::./common/chapters/logging.adoc[]
+
+include::{srcdir}/chapters/control.adoc[]
+
+include::./common/chapters/control_if.adoc[]
+
+include::{srcdir}/chapters/dgsm.adoc[]
+
+include::./common/chapters/gsup.adoc[]
+
+include::./common/chapters/vty_cpu_sched.adoc[]
+
+include::./common/chapters/port_numbers.adoc[]
+
+include::./common/chapters/bibliography.adoc[]
+
+include::./common/chapters/glossary.adoc[]
+
+include::./common/chapters/gfdl.adoc[]

doc/manuals/osmohlr-vty-reference.xml

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ex:ts=2:sw=42sts=2:et
+  -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+-->
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML 5.0//EN"
+"http://docbook.org/xml/5.0/dtd/docbook.dtd" [
+<!ENTITY chapter-vty      SYSTEM      "./common/chapters/vty.xml" >
+<!ENTITY sections-vty     SYSTEM      "generated/docbook_vty.xml"  >
+]>
+
+<book>
+  <info>
+    <revhistory>
+        <revision>
+            <revnumber>v1</revnumber>
+            <date>18th September 2017</date>
+            <authorinitials>nh</authorinitials>
+            <revremark>Initial</revremark>
+        </revision>
+    </revhistory>
+
+    <title>OsmoHLR VTY Reference</title>
+
+    <copyright>
+      <year>2017</year>
+    </copyright>
+
+    <legalnotice>
+      <para>This work is copyright by <orgname>sysmocom - s.f.m.c. GmbH</orgname>. All rights reserved.
+      </para>
+    </legalnotice>
+  </info>
+
+  <!-- Main chapters-->
+  &chapter-vty;
+</book>
+

doc/manuals/regen_doc.sh

@@ -0,0 +1,17 @@
+#!/bin/sh -x
+
+if [ -z "$DOCKER_PLAYGROUND" ]; then
+	echo "You need to set DOCKER_PLAYGROUND"
+	exit 1
+fi
+
+SCRIPT=$(realpath "$0")
+MANUAL_DIR=$(dirname "$SCRIPT")
+
+COMMIT=${COMMIT:-$(git log -1 --format=format:%H)}
+
+cd "$DOCKER_PLAYGROUND/scripts" || exit 1
+
+OSMO_HLR_BRANCH=$COMMIT ./regen_doc.sh osmo-hlr 4258 \
+	"$MANUAL_DIR/chapters/counters_generated.adoc" \
+	"$MANUAL_DIR/vty/hlr_vty_reference.xml"

doc/manuals/vty/hlr_vty_additions.xml

@@ -0,0 +1,2 @@
+<vtydoc xmlns='urn:osmocom:xml:libosmocore:vty:doc:1.0'>
+</vtydoc>

include/Makefile.am

@@ -0,0 +1,13 @@
+SUBDIRS = osmocom
+
+nobase_include_HEADERS = \
+	osmocom/gsupclient/cni_peer_id.h \
+	osmocom/gsupclient/gsup_client.h \
+	osmocom/gsupclient/gsup_req.h \
+	osmocom/mslookup/mdns.h \
+	osmocom/mslookup/mdns_sock.h \
+	osmocom/mslookup/mslookup_client_fake.h \
+	osmocom/mslookup/mslookup_client.h \
+	osmocom/mslookup/mslookup_client_mdns.h \
+	osmocom/mslookup/mslookup.h \
+	$(NULL)

include/osmocom/Makefile.am

@@ -0,0 +1,4 @@
+SUBDIRS = \
+	hlr \
+	mslookup \
+	$(NULL)

include/osmocom/gsupclient/cni_peer_id.h

@@ -0,0 +1,66 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+#include <unistd.h>
+#include <stdint.h>
+#include <osmocom/core/utils.h>
+
+/*! IPA Name: Arbitrary length blob, not necessarily zero-terminated.
+ * In osmo-hlr, struct hlr_subscriber is mostly used as static reference and cannot serve as talloc context, which is
+ * why this is also implemented as a fixed-maximum-size buffer instead of a talloc'd arbitrary sized buffer.
+ * NOTE: The length of val may be extended in the future if it becomes necessary.
+ * At the time of writing, this holds IPA unit name strings of very limited length.
+ */
+struct osmo_ipa_name {
+	size_t len;
+	uint8_t val[128];
+};
+
+bool osmo_ipa_name_is_empty(const struct osmo_ipa_name *ipa_name);
+int osmo_ipa_name_set(struct osmo_ipa_name *ipa_name, const uint8_t *val, size_t len);
+int osmo_ipa_name_set_str(struct osmo_ipa_name *ipa_name, const char *str_fmt, ...);
+int osmo_ipa_name_cmp(const struct osmo_ipa_name *a, const struct osmo_ipa_name *b);
+const char *osmo_ipa_name_to_str_c(void *ctx, const struct osmo_ipa_name *ipa_name);
+const char *osmo_ipa_name_to_str(const struct osmo_ipa_name *ipa_name);
+
+enum osmo_cni_peer_id_type {
+	OSMO_CNI_PEER_ID_EMPTY=0,
+	OSMO_CNI_PEER_ID_IPA_NAME,
+	/* OSMO_CNI_PEER_ID_GLOBAL_TITLE, <-- currently not implemented, but likely future possibility */
+};
+
+extern const struct value_string osmo_cni_peer_id_type_names[];
+static inline const char *osmo_cni_peer_id_type_name(enum osmo_cni_peer_id_type val)
+{ return get_value_string(osmo_cni_peer_id_type_names, val); }
+
+struct osmo_cni_peer_id {
+	enum osmo_cni_peer_id_type type;
+	union {
+		struct osmo_ipa_name ipa_name;
+	};
+};
+
+bool osmo_cni_peer_id_is_empty(const struct osmo_cni_peer_id *cni_peer_id);
+int osmo_cni_peer_id_set(struct osmo_cni_peer_id *cni_peer_id, enum osmo_cni_peer_id_type type,
+			  const uint8_t *val, size_t len);
+int osmo_cni_peer_id_set_str(struct osmo_cni_peer_id *cni_peer_id, enum osmo_cni_peer_id_type type,
+			      const char *str_fmt, ...);
+int osmo_cni_peer_id_cmp(const struct osmo_cni_peer_id *a, const struct osmo_cni_peer_id *b);
+const char *osmo_cni_peer_id_to_str(const struct osmo_cni_peer_id *cni_peer_id);
+const char *osmo_cni_peer_id_to_str_c(void *ctx, const struct osmo_cni_peer_id *cni_peer_id);

include/osmocom/gsupclient/gsup_client.h

@@ -0,0 +1,101 @@
+/* GPRS Subscriber Update Protocol client */
+
+/* (C) 2014 by Sysmocom s.f.m.c. GmbH
+ * All Rights Reserved
+ *
+ * Author: Jacob Erlbeck
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+#include <osmocom/core/timer.h>
+#include <osmocom/gsm/oap_client.h>
+#include <osmocom/gsm/ipa.h>
+#include <osmocom/gsm/gsup.h>
+
+/* a loss of GSUP between MSC and HLR is considered quite serious, let's try to recover as quickly as
+ * possible.  Even one new connection attempt per second should be quite acceptable until the link is
+ * re-established */
+#define OSMO_GSUP_CLIENT_RECONNECT_INTERVAL 1
+#define OSMO_GSUP_CLIENT_PING_INTERVAL 20
+
+struct msgb;
+struct ipa_client_conn;
+struct osmo_gsup_client;
+
+/* Expects message in msg->l2h */
+typedef int (*osmo_gsup_client_read_cb_t)(struct osmo_gsup_client *gsupc, struct msgb *msg);
+
+typedef bool (*osmo_gsup_client_up_down_cb_t)(struct osmo_gsup_client *gsupc, bool up);
+
+struct osmo_gsup_client {
+	const char *unit_name; /* same as ipa_dev->unit_name, for backwards compat */
+
+	struct ipa_client_conn *link;
+	osmo_gsup_client_read_cb_t read_cb;
+	void *data;
+
+	struct osmo_oap_client_state oap_state;
+
+	struct osmo_timer_list ping_timer;
+	struct osmo_timer_list connect_timer;
+	int is_connected;
+	int got_ipa_pong;
+
+	struct ipaccess_unit *ipa_dev; /* identification information sent to IPA server */
+
+	osmo_gsup_client_up_down_cb_t up_down_cb;
+};
+
+struct osmo_gsup_client_config {
+	/*! IP access unit which contains client identification information; must be allocated in talloc_ctx as well to
+	 * ensure it lives throughout the lifetime of the connection. */
+	struct ipaccess_unit *ipa_dev;
+	/*! GSUP server IP address to connect to. */
+	const char *ip_addr;
+	/*! GSUP server TCP port to connect to. */
+	unsigned int tcp_port;
+	/*! OPA client configuration, or NULL. */
+	struct osmo_oap_client_config *oapc_config;
+	/*! callback for reading from the GSUP connection. */
+	osmo_gsup_client_read_cb_t read_cb;
+	/*! Invoked when the GSUP link is ready for communication, and when the link drops. */
+	osmo_gsup_client_up_down_cb_t up_down_cb;
+	/*! User data stored in the returned gsupc->data, as context for the callbacks. */
+	void *data;
+	/*! Marker for future extension, always pass this as false. */
+	bool more;
+};
+struct osmo_gsup_client *osmo_gsup_client_create3(void *talloc_ctx, struct osmo_gsup_client_config *config);
+
+struct osmo_gsup_client *osmo_gsup_client_create2(void *talloc_ctx,
+						  struct ipaccess_unit *ipa_dev,
+						  const char *ip_addr,
+						  unsigned int tcp_port,
+						  osmo_gsup_client_read_cb_t read_cb,
+						  struct osmo_oap_client_config *oapc_config);
+struct osmo_gsup_client *osmo_gsup_client_create(void *talloc_ctx,
+						 const char *unit_name,
+						 const char *ip_addr,
+						 unsigned int tcp_port,
+						 osmo_gsup_client_read_cb_t read_cb,
+						 struct osmo_oap_client_config *oapc_config);
+
+void osmo_gsup_client_destroy(struct osmo_gsup_client *gsupc);
+int osmo_gsup_client_send(struct osmo_gsup_client *gsupc, struct msgb *msg);
+int osmo_gsup_client_enc_send(struct osmo_gsup_client *gsupc,
+			      const struct osmo_gsup_message *gsup_msg);
+struct msgb *osmo_gsup_client_msgb_alloc(void);
+

include/osmocom/gsupclient/gsup_req.h

@@ -0,0 +1,119 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <osmocom/gsm/gsup.h>
+#include <osmocom/gsupclient/cni_peer_id.h>
+
+struct osmo_gsup_req;
+
+#define LOG_GSUP_REQ_CAT_SRC(req, subsys, level, file, line, fmt, args...) \
+	LOGPSRC(subsys, level, file, line, "GSUP %u: %s: IMSI-%s %s: " fmt, \
+		(req) ? (req)->nr : 0, \
+		(req) ? osmo_cni_peer_id_to_str(&(req)->source_name) : "NULL", \
+		(req) ? (req)->gsup.imsi : "NULL", \
+		(req) ? osmo_gsup_message_type_name((req)->gsup.message_type) : "NULL", \
+		##args)
+#define LOG_GSUP_REQ_CAT(req, subsys, level, fmt, args...) \
+	LOG_GSUP_REQ_CAT_SRC(req, subsys, level, __FILE__, __LINE__, fmt, ##args)
+
+#define LOG_GSUP_REQ_SRC(req, level, file, line, fmt, args...) \
+	LOG_GSUP_REQ_CAT_SRC(req, DLGSUP, level, file, line, fmt, ##args)
+
+#define LOG_GSUP_REQ(req, level, fmt, args...) \
+	LOG_GSUP_REQ_SRC(req, level, __FILE__, __LINE__, fmt, ##args)
+
+typedef void (*osmo_gsup_req_send_response_t)(struct osmo_gsup_req *req, struct osmo_gsup_message *response);
+
+/* Keep track of an incoming request, to route back a response when it is ready.
+ * Particularly, a GSUP response to a request must contain various bits of information that need to be copied from the
+ * request for proxy/routing to work and for session states to remain valid. That is the main reason why (almost) all
+ * GSUP request/response should go through an osmo_gsup_req, even if it is handled synchronously.
+ */
+struct osmo_gsup_req {
+	/* The incoming GSUP message in decoded form. */
+	const struct osmo_gsup_message gsup;
+
+	/* Decoding result code. If decoding failed, this will be != 0. */
+	int decode_rc;
+
+	/* The ultimate source of this message: the source_name form the GSUP message, or, if not present, then the
+	 * immediate GSUP peer. GSUP messages going via a proxy reflect the initial source in the source_name.
+	 * This source_name is implicitly added to the routes for the conn the message was received on. */
+	struct osmo_cni_peer_id source_name;
+
+	/* If the source_name is not an immediate GSUP peer, this is set to the closest intermediate peer between here
+	 * and source_name. */
+	struct osmo_cni_peer_id via_proxy;
+
+	/* Identify this request by number, for logging. */
+	unsigned int nr;
+
+	/* osmo_gsup_req can be used by both gsup_server and gsup_client. The individual method of actually sending a
+	 * GSUP message is provided by this callback. */
+	osmo_gsup_req_send_response_t send_response_cb;
+
+	/* User supplied data pointer, may be used to provide context to send_response_cb(). */
+	void *cb_data;
+
+	/* List entry that can be used to keep a list of osmo_gsup_req instances; not used directly by osmo_gsup_req.c,
+	 * it is up to using implementations to keep a list. If this is non-NULL, osmo_gsup_req_free() calls
+	 * llist_del() on this. */
+	struct llist_head entry;
+
+	/* A decoded GSUP message still points into the received msgb. For a decoded osmo_gsup_message to remain valid,
+	 * we also need to keep the msgb. */
+	struct msgb *msg;
+};
+
+struct osmo_gsup_req *osmo_gsup_req_new(void *ctx, const struct osmo_cni_peer_id *from_peer, struct msgb *msg,
+					osmo_gsup_req_send_response_t send_response_cb, void *cb_data,
+					struct llist_head *add_to_list);
+void osmo_gsup_req_free(struct osmo_gsup_req *req);
+
+/*! See _osmo_gsup_req_respond() for details.
+ * Call _osmo_gsup_req_respond(), passing the caller's source file and line for logging. */
+#define osmo_gsup_req_respond(REQ, RESPONSE, ERROR, FINAL_RESPONSE) \
+	_osmo_gsup_req_respond(REQ, RESPONSE, ERROR, FINAL_RESPONSE, __FILE__, __LINE__)
+int _osmo_gsup_req_respond(struct osmo_gsup_req *req, struct osmo_gsup_message *response,
+			   bool error, bool final_response, const char *file, int line);
+
+/*! See _osmo_gsup_req_respond_msgt() for details.
+ * Call _osmo_gsup_req_respond_msgt(), passing the caller's source file and line for logging. */
+#define osmo_gsup_req_respond_msgt(REQ, MESSAGE_TYPE, FINAL_RESPONSE) \
+	_osmo_gsup_req_respond_msgt(REQ, MESSAGE_TYPE, FINAL_RESPONSE, __FILE__, __LINE__)
+int _osmo_gsup_req_respond_msgt(struct osmo_gsup_req *req, enum osmo_gsup_message_type message_type,
+				bool final_response, const char *file, int line);
+
+/*! See _osmo_gsup_req_respond_err() for details.
+ * Log an error message, and call _osmo_gsup_req_respond_err(), passing the caller's source file and line for logging.
+ */
+#define osmo_gsup_req_respond_err(REQ, CAUSE, FMT, args...) do { \
+		LOG_GSUP_REQ(REQ, LOGL_ERROR, "%s: " FMT "\n", \
+			     get_value_string(gsm48_gmm_cause_names, CAUSE), ##args); \
+		_osmo_gsup_req_respond_err(REQ, CAUSE, __FILE__, __LINE__); \
+	} while(0)
+void _osmo_gsup_req_respond_err(struct osmo_gsup_req *req, enum gsm48_gmm_cause cause,
+				const char *file, int line);
+
+int osmo_gsup_make_response(struct osmo_gsup_message *reply,
+			    const struct osmo_gsup_message *rx, bool error, bool final_response);
+
+size_t osmo_gsup_message_to_str_buf(char *buf, size_t bufsize, const struct osmo_gsup_message *msg);
+char *osmo_gsup_message_to_str_c(void *ctx, const struct osmo_gsup_message *msg);

include/osmocom/hlr/Makefile.am

@@ -0,0 +1,20 @@
+noinst_HEADERS = \
+	auc.h \
+	ctrl.h \
+	db.h \
+	dgsm.h \
+	gsup_router.h \
+	gsup_server.h \
+	hlr.h \
+	hlr_ussd.h \
+	hlr_vty.h \
+	hlr_vty_subscr.h \
+	logging.h \
+	lu_fsm.h \
+	mslookup_server.h \
+	mslookup_server_mdns.h \
+	proxy.h \
+	rand.h \
+	remote_hlr.h \
+	timestamp.h \
+	$(NULL)

include/osmocom/hlr/ctrl.h

@@ -24,8 +24,11 @@
 
 #include <osmocom/ctrl/control_if.h>
 
-#include "gsup_server.h"
+enum hlr_ctrl_node {
+	CTRL_NODE_SUBSCR = _LAST_CTRL_NODE,
+	CTRL_NODE_SUBSCR_BY,
+	_LAST_CTRL_NODE_HLR
+};
 
 int hlr_ctrl_cmds_install();
-struct ctrl_handle *hlr_controlif_setup(struct hlr *ctx,
-					struct osmo_gsup_server *gs);
+struct ctrl_handle *hlr_controlif_setup(struct hlr *hlr);

include/osmocom/hlr/db.h

@@ -3,14 +3,25 @@
 #include <stdbool.h>
 #include <sqlite3.h>
 
+#include <osmocom/gsupclient/cni_peer_id.h>
+
 struct hlr;
 
 enum stmt_idx {
+	DB_STMT_SEL_ALL,
+	DB_STMT_SEL_ALL_ORDER_LAST_SEEN,
+	DB_STMT_SEL_FILTER_MSISDN,
+	DB_STMT_SEL_FILTER_IMSI,
+	DB_STMT_SEL_FILTER_IMEI,
+	DB_STMT_SEL_FILTER_CS,
+	DB_STMT_SEL_FILTER_PS,
 	DB_STMT_SEL_BY_IMSI,
 	DB_STMT_SEL_BY_MSISDN,
 	DB_STMT_SEL_BY_ID,
+	DB_STMT_SEL_BY_IMEI,
 	DB_STMT_UPD_VLR_BY_ID,
 	DB_STMT_UPD_SGSN_BY_ID,
+	DB_STMT_UPD_IMEI_BY_IMSI,
 	DB_STMT_AUC_BY_IMSI,
 	DB_STMT_AUC_UPD_SQN,
 	DB_STMT_UPD_PURGE_CS_BY_IMSI,
@@ -20,10 +31,15 @@ enum stmt_idx {
 	DB_STMT_SUBSCR_CREATE,
 	DB_STMT_DEL_BY_ID,
 	DB_STMT_SET_MSISDN_BY_IMSI,
+	DB_STMT_DELETE_MSISDN_BY_IMSI,
 	DB_STMT_AUC_2G_INSERT,
 	DB_STMT_AUC_2G_DELETE,
 	DB_STMT_AUC_3G_INSERT,
 	DB_STMT_AUC_3G_DELETE,
+	DB_STMT_SET_LAST_LU_SEEN,
+	DB_STMT_SET_LAST_LU_SEEN_PS,
+	DB_STMT_EXISTS_BY_IMSI,
+	DB_STMT_EXISTS_BY_MSISDN,
 	_NUM_DB_STMT
 };
 
@@ -33,12 +49,18 @@ struct db_context {
 	sqlite3_stmt *stmt[_NUM_DB_STMT];
 };
 
+/* Optional feature to make SQLite3 using talloc */
+#ifdef SQLITE_USE_TALLOC
+int db_sqlite3_use_talloc(void *ctx);
+#endif
+
 void db_remove_reset(sqlite3_stmt *stmt);
 bool db_bind_text(sqlite3_stmt *stmt, const char *param_name, const char *text);
 bool db_bind_int(sqlite3_stmt *stmt, const char *param_name, int nr);
 bool db_bind_int64(sqlite3_stmt *stmt, const char *param_name, int64_t nr);
+bool db_bind_null(sqlite3_stmt *stmt, const char *param_name);
 void db_close(struct db_context *dbc);
-struct db_context *db_open(void *ctx, const char *fname);
+struct db_context *db_open(void *ctx, const char *fname, bool enable_sqlite3_logging, bool allow_upgrades);
 
 #include <osmocom/crypt/auth.h>
 
@@ -54,7 +76,7 @@ int db_update_sqn(struct db_context *dbc, int64_t id,
 int db_get_auc(struct db_context *dbc, const char *imsi,
 	       unsigned int auc_3g_ind, struct osmo_auth_vector *vec,
 	       unsigned int num_vec, const uint8_t *rand_auts,
-	       const uint8_t *auts);
+	       const uint8_t *auts, bool separation_bit);
 
 #include <osmocom/core/linuxlist.h>
 #include <osmocom/gsm/protocol/gsm_23_003.h>
@@ -67,10 +89,11 @@ struct hlr_subscriber {
 
 	int64_t		id;
 	char		imsi[GSM23003_IMSI_MAX_DIGITS+1];
-	char		msisdn[GT_MAX_DIGITS+1];
+	char		msisdn[GSM23003_MSISDN_MAX_DIGITS+1];
 	/* imeisv? */
-	char		vlr_number[GT_MAX_DIGITS+1];
-	char		sgsn_number[GT_MAX_DIGITS+1];
+	char		imei[GSM23003_IMEI_NUM_DIGITS+1];
+	char		vlr_number[32];
+	char		sgsn_number[32];
 	char		sgsn_address[GT_MAX_DIGITS+1];
 	/* ggsn number + address */
 	/* gmlc number */
@@ -82,8 +105,18 @@ struct hlr_subscriber {
 	uint32_t	lmsi;
 	bool		ms_purged_cs;
 	bool		ms_purged_ps;
+	time_t		last_lu_seen;
+	time_t		last_lu_seen_ps;
+	/* talloc'd IPA unit name */
+	struct osmo_ipa_name	vlr_via_proxy;
+	struct osmo_ipa_name	sgsn_via_proxy;
 };
 
+/* A format string for use with strptime(3). This format string is
+ * used to parse the last_lu_seen column stored in the HLR database.
+ * See https://sqlite.org/lang_datefunc.html, function datetime(). */
+#define DB_LAST_LU_SEEN_FMT "%Y-%m-%d %H:%M:%S"
+
 /* Like struct osmo_sub_auth_data, but the keys are in hexdump representation.
  * This is useful because SQLite requires them in hexdump format, and callers
  * like the VTY and CTRL interface also have them available as hexdump to begin
@@ -107,25 +140,57 @@ struct sub_auth_data_str {
 	} u;
 };
 
-int db_subscr_create(struct db_context *dbc, const char *imsi);
+#define DB_SUBSCR_FLAG_NAM_CS	(1 << 1)
+#define DB_SUBSCR_FLAG_NAM_PS	(1 << 2)
+
+int db_subscr_create(struct db_context *dbc, const char *imsi, uint8_t flags);
 int db_subscr_delete_by_id(struct db_context *dbc, int64_t subscr_id);
 
 int db_subscr_update_msisdn_by_imsi(struct db_context *dbc, const char *imsi,
 				    const char *msisdn);
 int db_subscr_update_aud_by_id(struct db_context *dbc, int64_t subscr_id,
 			       const struct sub_auth_data_str *aud);
+int db_subscr_update_imei_by_imsi(struct db_context *dbc, const char* imsi, const char *imei);
 
+int db_subscr_exists_by_imsi(struct db_context *dbc, const char *imsi);
+int db_subscr_exists_by_msisdn(struct db_context *dbc, const char *msisdn);
+
+int db_subscrs_get(struct db_context *dbc, const char *filter_type, const char *filter,
+		   void (*get_cb)(struct hlr_subscriber *subscr, void *data), void *data,
+		   int *count, const char **err);
 int db_subscr_get_by_imsi(struct db_context *dbc, const char *imsi,
 			  struct hlr_subscriber *subscr);
 int db_subscr_get_by_msisdn(struct db_context *dbc, const char *msisdn,
 			    struct hlr_subscriber *subscr);
 int db_subscr_get_by_id(struct db_context *dbc, int64_t id,
 			struct hlr_subscriber *subscr);
+int db_subscr_get_by_imei(struct db_context *dbc, const char *imei, struct hlr_subscriber *subscr);
 int db_subscr_nam(struct db_context *dbc, const char *imsi, bool nam_val, bool is_ps);
 int db_subscr_lu(struct db_context *dbc, int64_t subscr_id,
-		 const char *vlr_or_sgsn_number, bool is_ps);
+		 const struct osmo_ipa_name *vlr_name, bool is_ps,
+		 const struct osmo_ipa_name *via_proxy);
 
 int db_subscr_purge(struct db_context *dbc, const char *by_imsi,
 		    bool purge_val, bool is_ps);
 
-int hlr_subscr_nam(struct hlr *hlr, struct hlr_subscriber *subscr, bool nam_val, bool is_ps);
+/*! Call sqlite3_column_text() and copy result to a char[].
+ * \param[out] buf  A char[] used as sizeof() arg(!) and osmo_strlcpy() target.
+ * \param[in] stmt  An sqlite3_stmt*.
+ * \param[in] idx   Index in stmt's returned columns.
+ */
+#define copy_sqlite3_text_to_buf(buf, stmt, idx) \
+	do { \
+		const char *_txt = (const char *) sqlite3_column_text(stmt, idx); \
+		osmo_strlcpy(buf, _txt, sizeof(buf)); \
+	} while (0)
+
+/*! Call sqlite3_column_text() and copy result to a struct osmo_ipa_name.
+ * \param[out] ipa_name  A struct osmo_ipa_name* to write to.
+ * \param[in] stmt  An sqlite3_stmt*.
+ * \param[in] idx  Index in stmt's returned columns.
+ */
+#define copy_sqlite3_text_to_ipa_name(ipa_name, stmt, idx) \
+	do { \
+		const char *_txt = (const char *) sqlite3_column_text(stmt, idx); \
+		osmo_ipa_name_set_str(ipa_name, _txt); \
+	} while (0)

include/osmocom/hlr/dgsm.h

@@ -0,0 +1,46 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#pragma once
+
+#include <osmocom/mslookup/mslookup.h>
+#include <osmocom/hlr/gsup_server.h>
+#include <osmocom/hlr/logging.h>
+#include <osmocom/gsupclient/cni_peer_id.h>
+#include <osmocom/gsupclient/gsup_req.h>
+
+#define LOG_DGSM(imsi, level, fmt, args...) \
+	LOGP(DDGSM, level, "(IMSI-%s) " fmt, imsi, ##args)
+
+struct vty;
+struct remote_hlr;
+struct hlr_subscriber;
+
+extern void *dgsm_ctx;
+
+void dgsm_init(void *ctx);
+void dgsm_start(void *ctx);
+void dgsm_stop();
+
+bool dgsm_check_forward_gsup_msg(struct osmo_gsup_req *req);
+
+void dgsm_vty_init();
+void dgsm_mdns_client_config_apply(void);
+
+bool hlr_subscr_lu_age(const struct hlr_subscriber *subscr, uint32_t *age_p);

include/osmocom/hlr/gsup_router.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include <stdint.h>
+#include <osmocom/hlr/gsup_server.h>
+
+struct osmo_ipa_name;
+
+struct gsup_route {
+	struct llist_head list;
+
+	uint8_t *addr;
+	struct osmo_gsup_conn *conn;
+};
+
+struct osmo_gsup_conn *gsup_route_find(struct osmo_gsup_server *gs,
+					const uint8_t *addr, size_t addrlen);
+struct osmo_gsup_conn *gsup_route_find_by_ipa_name(struct osmo_gsup_server *gs, const struct osmo_ipa_name *ipa_name);
+
+struct gsup_route *gsup_route_find_by_conn(const struct osmo_gsup_conn *conn);
+
+/* add a new route for the given address to the given conn */
+int gsup_route_add_ipa_name(struct osmo_gsup_conn *conn, const struct osmo_ipa_name *ipa_name);
+int gsup_route_add(struct osmo_gsup_conn *conn, const uint8_t *addr, size_t addrlen);
+
+/* delete all routes for the given connection */
+int gsup_route_del_conn(struct osmo_gsup_conn *conn);
+
+int osmo_gsup_addr_send(struct osmo_gsup_server *gs,
+			const uint8_t *addr, size_t addrlen,
+			struct msgb *msg);
+int osmo_gsup_send_to_ipa_name(struct osmo_gsup_server *gs, const struct osmo_ipa_name *ipa_name, struct msgb *msg);
+int osmo_gsup_enc_send_to_ipa_name(struct osmo_gsup_server *gs, const struct osmo_ipa_name *ipa_name,
+			  const struct osmo_gsup_message *gsup);

include/osmocom/hlr/gsup_server.h

@@ -0,0 +1,78 @@
+#pragma once
+
+#include <osmocom/core/linuxlist.h>
+#include <osmocom/core/msgb.h>
+#include <osmocom/abis/ipa.h>
+#include <osmocom/abis/ipaccess.h>
+#include <osmocom/gsm/gsup.h>
+#include <osmocom/gsupclient/cni_peer_id.h>
+#include <osmocom/gsupclient/gsup_req.h>
+
+#ifndef OSMO_GSUP_MAX_CALLED_PARTY_BCD_LEN
+#define OSMO_GSUP_MAX_CALLED_PARTY_BCD_LEN	43 /* TS 24.008 10.5.4.7 */
+#endif
+
+struct osmo_gsup_conn;
+
+/* Expects message in msg->l2h */
+typedef int (*osmo_gsup_read_cb_t)(struct osmo_gsup_conn *conn, struct msgb *msg);
+
+struct osmo_gsup_server {
+	/* private data of the application/user */
+	void *priv;
+
+	/* list of osmo_gsup_conn */
+	struct llist_head clients;
+
+	struct ipa_server_link *link;
+	osmo_gsup_read_cb_t read_cb;
+	struct llist_head routes;
+
+	/* Proxy requests from this server's clients to remote GSUP servers. */
+	struct proxy *proxy;
+};
+
+
+/* a single connection to a given client (SGSN, MSC) */
+struct osmo_gsup_conn {
+	struct llist_head list;
+
+	struct osmo_gsup_server *server;
+	struct ipa_server_conn *conn;
+	//struct oap_state oap_state;
+	struct tlv_parsed ccm;
+
+	unsigned int auc_3g_ind; /*!< IND index used for UMTS AKA SQN */
+
+	/* Set when Location Update is received: */
+	bool supports_cs; /* client supports OSMO_GSUP_CN_DOMAIN_CS */
+	bool supports_ps; /* client supports OSMO_GSUP_CN_DOMAIN_PS */
+
+	/* The IPA unit name received on this link. Routes with more unit names serviced by this link may exist in
+	 * osmo_gsup_server->routes, but this is the name the immediate peer identified as in the IPA handshake. */
+	struct osmo_ipa_name peer_name;
+};
+
+struct msgb *osmo_gsup_msgb_alloc(const char *label);
+
+struct osmo_gsup_req *osmo_gsup_conn_rx(struct osmo_gsup_conn *conn, struct msgb *msg);
+int osmo_gsup_conn_send(struct osmo_gsup_conn *conn, struct msgb *msg);
+int osmo_gsup_conn_ccm_get(const struct osmo_gsup_conn *clnt, uint8_t **addr,
+			   uint8_t tag);
+
+struct osmo_gsup_server *osmo_gsup_server_create(void *ctx,
+						 const char *ip_addr,
+						 uint16_t tcp_port,
+						 osmo_gsup_read_cb_t read_cb,
+						 void *priv);
+
+void osmo_gsup_server_destroy(struct osmo_gsup_server *gsups);
+
+int osmo_gsup_configure_wildcard_apn(struct osmo_gsup_message *gsup,
+				     uint8_t *apn_buf, size_t apn_buf_size);
+int osmo_gsup_create_insert_subscriber_data_msg(struct osmo_gsup_message *gsup, const char *imsi, const char *msisdn,
+					    uint8_t *msisdn_enc, size_t msisdn_enc_size,
+				            uint8_t *apn_buf, size_t apn_buf_size,
+					    enum osmo_gsup_cn_domain cn_domain);
+int osmo_gsup_forward_to_local_peer(struct osmo_gsup_server *server, const struct osmo_cni_peer_id *to_peer,
+				    struct osmo_gsup_req *req, struct osmo_gsup_message *modified_gsup);

include/osmocom/hlr/hlr.h

@@ -0,0 +1,121 @@
+/* OsmoHLR generic header */
+
+/* (C) 2017 sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ * All Rights Reserved
+ *
+ * Author: Max Suraev <msuraev@sysmocom.de>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#pragma once
+
+#include <stdbool.h>
+#include <osmocom/core/linuxlist.h>
+#include <osmocom/gsm/ipa.h>
+#include <osmocom/core/tdef.h>
+#include <osmocom/core/sockaddr_str.h>
+
+#include <osmocom/hlr/dgsm.h>
+
+#define HLR_DEFAULT_DB_FILE_PATH "hlr.db"
+
+struct hlr_euse;
+struct osmo_gsup_conn;
+enum osmo_gsup_message_type;
+
+extern struct osmo_tdef g_hlr_tdefs[];
+
+struct hlr {
+	/* GSUP server pointer */
+	struct osmo_gsup_server *gs;
+
+	/* DB context */
+	char *db_file_path;
+	struct db_context *dbc;
+
+	/* Control Interface */
+	struct ctrl_handle *ctrl;
+	const char *ctrl_bind_addr;
+
+	/* Local bind addr */
+	char *gsup_bind_addr;
+	struct ipaccess_unit gsup_unit_name;
+
+	struct llist_head euse_list;
+	struct hlr_euse *euse_default;
+
+	/* NCSS (call independent) session guard timeout value */
+	int ncss_guard_timeout;
+
+	struct llist_head ussd_routes;
+
+	struct llist_head ss_sessions;
+
+	bool store_imei;
+
+	bool subscr_create_on_demand;
+	/* Bitmask of DB_SUBSCR_FLAG_* */
+	uint8_t subscr_create_on_demand_flags;
+	unsigned int subscr_create_on_demand_rand_msisdn_len;
+
+	struct {
+		bool allow_startup;
+		struct {
+			/* Whether the mslookup server should be active in general (all lookup methods) */
+			bool enable;
+			uint32_t local_attach_max_age;
+			struct llist_head local_site_services;
+			struct {
+				/* Whether the mDNS method of the mslookup server should be active. */
+				bool enable;
+				/* The mDNS bind address and domain suffix as set by the VTY, not necessarily in use. */
+				struct osmo_sockaddr_str bind_addr;
+				char *domain_suffix;
+				struct osmo_mslookup_server_mdns *running;
+			} mdns;
+		} server;
+
+		/* The mslookup client in osmo-hlr is used to find out which remote HLRs service a locally unknown IMSI.
+		 * (It may also be used to resolve recipients for SMS-over-GSUP in the future.) */
+		struct {
+			/* Whether to proxy/forward to remote HLRs */
+			bool enable;
+
+			/* If this is set, all GSUP for unknown IMSIs is forwarded directly to this GSUP address,
+			 * unconditionally. */
+			struct osmo_sockaddr_str gsup_gateway_proxy;
+
+			/* mslookup client request handling */
+			unsigned int result_timeout_milliseconds;
+
+			struct osmo_mslookup_client *client;
+			struct {
+				/* Whether to use mDNS for IMSI MS Lookup */
+				bool enable;
+				struct osmo_sockaddr_str query_addr;
+				char *domain_suffix;
+				struct osmo_mslookup_client_method *running;
+			} mdns;
+		} client;
+	} mslookup;
+};
+
+extern struct hlr *g_hlr;
+
+struct hlr_subscriber;
+
+void osmo_hlr_subscriber_update_notify(struct hlr_subscriber *subscr);
+int hlr_subscr_nam(struct hlr *hlr, struct hlr_subscriber *subscr, bool nam_val, bool is_ps);

include/osmocom/hlr/hlr_ussd.h

@@ -0,0 +1,60 @@
+#pragma once
+
+#include <stdbool.h>
+
+#include <osmocom/core/linuxlist.h>
+#include <osmocom/gsm/gsup.h>
+
+#include <osmocom/hlr/gsup_server.h>
+
+#define NCSS_GUARD_TIMEOUT_DEFAULT 30
+
+struct hlr_ussd_route {
+	/* g_hlr.routes */
+	struct llist_head list;
+	const char *prefix;
+	bool is_external;
+	union {
+		struct hlr_euse *euse;
+		const struct hlr_iuse *iuse;
+	} u;
+};
+
+struct hlr_euse {
+	/* list in the per-hlr list of EUSEs */
+	struct llist_head list;
+	struct hlr *hlr;
+	/* name (must match the IPA ID tag) */
+	const char *name;
+	/* human-readable description */
+	const char *description;
+
+	/* GSUP connection to the EUSE, if any */
+	struct osmo_gsup_conn *conn;
+};
+
+struct hlr_euse *euse_find(struct hlr *hlr, const char *name);
+struct hlr_euse *euse_alloc(struct hlr *hlr, const char *name);
+void euse_del(struct hlr_euse *euse);
+
+const struct hlr_iuse *iuse_find(const char *name);
+
+struct hlr_ussd_route *ussd_route_find_prefix(struct hlr *hlr, const char *prefix);
+struct hlr_ussd_route *ussd_route_prefix_alloc_int(struct hlr *hlr, const char *prefix,
+						   const struct hlr_iuse *iuse);
+struct hlr_ussd_route *ussd_route_prefix_alloc_ext(struct hlr *hlr, const char *prefix,
+						   struct hlr_euse *euse);
+void ussd_route_del(struct hlr_ussd_route *rt);
+
+void rx_proc_ss_req(struct osmo_gsup_req *req);
+void rx_proc_ss_error(struct osmo_gsup_req *req);
+
+struct ss_session;
+struct ss_request;
+
+/* Internal USSD Handler */
+struct hlr_iuse {
+	const char *name;
+	/* call-back to be called for any incoming USSD messages for this IUSE */
+	int (*handle_ussd)(struct ss_session *ss, const struct osmo_gsup_message *gsup, const struct ss_request *req);
+};

include/osmocom/hlr/hlr_vty.h

@@ -25,13 +25,19 @@
 #include <osmocom/core/logging.h>
 #include <osmocom/vty/vty.h>
 #include <osmocom/vty/command.h>
-#include "hlr.h"
+#include <osmocom/hlr/hlr.h>
 
 enum hlr_vty_node {
 	HLR_NODE = _LAST_OSMOVTY_NODE + 1,
 	GSUP_NODE,
+	EUSE_NODE,
+	MSLOOKUP_NODE,
+	MSLOOKUP_SERVER_NODE,
+	MSLOOKUP_SERVER_MSC_NODE,
+	MSLOOKUP_CLIENT_NODE,
 };
 
 int hlr_vty_is_config_node(struct vty *vty, int node);
 int hlr_vty_go_parent(struct vty *vty);
-void hlr_vty_init(struct hlr *hlr, const struct log_info *cat);
+void hlr_vty_init(void);
+void dgsm_vty_init(void);

include/osmocom/hlr/hlr_vty_subscr.h

@@ -0,0 +1,3 @@
+#pragma once
+
+void hlr_vty_subscriber_init(void);

include/osmocom/hlr/logging.h

@@ -7,6 +7,10 @@ enum {
 	DDB,
 	DGSUP,
 	DAUC,
+	DSS,
+	DMSLOOKUP,
+	DLU,
+	DDGSM,
 };
 
 extern const struct log_info hlr_log_info;

include/osmocom/hlr/lu_fsm.h

@@ -0,0 +1,22 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#pragma once
+
+void lu_rx_gsup(struct osmo_gsup_req *req);

include/osmocom/hlr/mslookup_server.h

@@ -0,0 +1,72 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#pragma once
+
+#include <osmocom/gsupclient/cni_peer_id.h>
+#include <osmocom/mslookup/mslookup.h>
+
+struct osmo_mslookup_query;
+struct osmo_mslookup_result;
+
+/*! mslookup service name used for roaming/proxying between osmo-hlr instances. */
+#define OSMO_MSLOOKUP_SERVICE_HLR_GSUP "gsup.hlr"
+
+/*! What addresses to return to mslookup queries when a subscriber is attached at the local site.
+ * Mapping of service name to IP address and port. This corresponds to the VTY config for
+ * 'mslookup' / 'server' [/ 'msc MSC-1-2-3'] / 'service sip.voice at 1.2.3.4 1234'.
+ */
+struct mslookup_service_host {
+	struct llist_head entry;
+	char service[OSMO_MSLOOKUP_SERVICE_MAXLEN+1];
+	struct osmo_sockaddr_str host_v4;
+	struct osmo_sockaddr_str host_v6;
+};
+
+/*! Sets of mslookup_service_host per connected MSC.
+ * When there are more than one MSC connected to this osmo-hlr, this allows keeping separate sets of service addresses
+ * for each MSC. The entry with mslookup_server_msc_wildcard as MSC name is used for all MSCs (if no match for that
+ * particular MSC is found). This corresponds to the VTY config for
+ * 'mslookup' / 'server' / 'msc MSC-1-2-3'.
+ */
+struct mslookup_server_msc_cfg {
+	struct llist_head entry;
+	struct osmo_ipa_name name;
+	struct llist_head service_hosts;
+};
+
+struct mslookup_service_host *mslookup_server_service_get(const struct osmo_ipa_name *msc_name, const char *service);
+
+struct mslookup_service_host *mslookup_server_msc_service_get(struct mslookup_server_msc_cfg *msc, const char *service,
+							      bool create);
+int mslookup_server_msc_service_set(struct mslookup_server_msc_cfg *msc, const char *service,
+				    const struct osmo_sockaddr_str *addr);
+int mslookup_server_msc_service_del(struct mslookup_server_msc_cfg *msc, const char *service,
+				    const struct osmo_sockaddr_str *addr);
+
+extern const struct osmo_ipa_name mslookup_server_msc_wildcard;
+struct mslookup_server_msc_cfg *mslookup_server_msc_get(const struct osmo_ipa_name *msc_name, bool create);
+
+const struct mslookup_service_host *mslookup_server_get_local_gsup_addr();
+void mslookup_server_rx(const struct osmo_mslookup_query *query,
+			     struct osmo_mslookup_result *result);
+
+bool subscriber_has_done_lu_here(const struct osmo_mslookup_query *query,
+				 uint32_t *lu_age_p, struct osmo_ipa_name *local_msc_name,
+				 char *ret_imsi, size_t ret_imsi_len);

include/osmocom/hlr/mslookup_server_mdns.h

@@ -0,0 +1,36 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#pragma once
+
+#include <stdbool.h>
+#include <osmocom/core/sockaddr_str.h>
+#include <osmocom/mslookup/mdns_sock.h>
+
+struct osmo_mslookup_server_mdns {
+	struct osmo_mslookup_server *mslookup;
+	struct osmo_sockaddr_str bind_addr;
+	char *domain_suffix;
+	struct osmo_mdns_sock *sock;
+};
+
+struct osmo_mslookup_server_mdns *osmo_mslookup_server_mdns_start(void *ctx, const struct osmo_sockaddr_str *bind_addr,
+								  const char *domain_suffix);
+void osmo_mslookup_server_mdns_stop(struct osmo_mslookup_server_mdns *server);
+void mslookup_server_mdns_config_apply();

include/osmocom/hlr/proxy.h

@@ -0,0 +1,95 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#pragma once
+
+#include <time.h>
+#include <osmocom/gsm/protocol/gsm_23_003.h>
+#include <osmocom/core/sockaddr_str.h>
+#include <osmocom/gsupclient/cni_peer_id.h>
+#include <osmocom/hlr/timestamp.h>
+
+struct osmo_gsup_req;
+struct remote_hlr;
+
+struct proxy {
+	struct llist_head subscr_list;
+	struct llist_head pending_gsup_reqs;
+
+	/* When messages arrive back from a remote HLR that this is the proxy for, reach the VLR to forward the response
+	 * to via this osmo_gsup_server. */
+	struct osmo_gsup_server *gsup_server_to_vlr;
+
+	/* How long to keep proxy entries without a refresh, in seconds. */
+	uint32_t fresh_time;
+
+	/* How often to garbage collect the proxy cache, period in seconds.
+	 * To change this and take effect immediately, rather use proxy_set_gc_period(). */
+	uint32_t gc_period;
+
+	struct osmo_timer_list gc_timer;
+};
+
+struct proxy_subscr_domain_state {
+	struct osmo_ipa_name vlr_name;
+	timestamp_t last_lu;
+
+	/* The name from which an Update Location Request was received. Copied to vlr_name as soon as the LU is
+	 * completed successfully. */
+	struct osmo_ipa_name vlr_name_preliminary;
+
+	/* Set if this is a middle proxy, i.e. a proxy behind another proxy.
+	 * That is mostly to know whether the MS is attached at a local MSC/SGSN or further away.
+	 * It could be a boolean, but store the full name for logging. Set only at successful LU acceptance. */
+	struct osmo_ipa_name vlr_via_proxy;
+};
+
+struct proxy_subscr {
+	char imsi[GSM23003_IMSI_MAX_DIGITS+1];
+	char msisdn[GSM23003_MSISDN_MAX_DIGITS+1];
+	struct osmo_sockaddr_str remote_hlr_addr;
+	struct proxy_subscr_domain_state cs, ps;
+};
+
+void proxy_init(struct osmo_gsup_server *gsup_server_to_vlr);
+void proxy_del(struct proxy *proxy);
+void proxy_set_gc_period(struct proxy *proxy, uint32_t gc_period);
+
+/* The API to access / modify proxy entries keeps the implementation opaque, to make sure that we can easily move proxy
+ * storage to SQLite db. */
+int proxy_subscr_get_by_imsi(struct proxy_subscr *dst, struct proxy *proxy, const char *imsi);
+int proxy_subscr_get_by_msisdn(struct proxy_subscr *dst, struct proxy *proxy, const char *msisdn);
+void proxy_subscrs_get_by_remote_hlr(struct proxy *proxy, const struct osmo_sockaddr_str *remote_hlr_addr,
+				     bool (*yield)(struct proxy *proxy, const struct proxy_subscr *subscr, void *data),
+				     void *data);
+int proxy_subscr_create_or_update(struct proxy *proxy, const struct proxy_subscr *proxy_subscr);
+int proxy_subscr_del(struct proxy *proxy, const char *imsi);
+
+int proxy_subscr_forward_to_remote_hlr(struct proxy *proxy, const struct proxy_subscr *proxy_subscr,
+				       struct osmo_gsup_req *req);
+void proxy_subscr_forward_to_remote_hlr_resolved(struct proxy *proxy, const struct proxy_subscr *proxy_subscr,
+						 struct remote_hlr *remote_hlr, struct osmo_gsup_req *req);
+
+int proxy_subscr_forward_to_vlr(struct proxy *proxy, const struct proxy_subscr *proxy_subscr,
+				const struct osmo_gsup_message *gsup, struct remote_hlr *from_remote_hlr);
+
+void proxy_subscr_remote_hlr_resolved(struct proxy *proxy, const struct proxy_subscr *proxy_subscr,
+				      const struct osmo_sockaddr_str *remote_hlr_addr);
+void proxy_subscr_remote_hlr_up(struct proxy *proxy, const struct proxy_subscr *proxy_subscr,
+				struct remote_hlr *remote_hlr);

include/osmocom/hlr/remote_hlr.h

@@ -0,0 +1,59 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#pragma once
+
+#include <stdbool.h>
+#include <osmocom/core/linuxlist.h>
+#include <osmocom/core/sockaddr_str.h>
+
+struct osmo_gsup_client;
+struct osmo_gsup_message;
+struct osmo_gsup_req;
+struct msgb;
+
+#define LOG_REMOTE_HLR(remote_hlr, level, fmt, args...) \
+	LOGP(DDGSM, level, "(Proxy HLR-" OSMO_SOCKADDR_STR_FMT ") " fmt, \
+	     OSMO_SOCKADDR_STR_FMT_ARGS((remote_hlr) ? &(remote_hlr)->addr : NULL), ##args)
+
+#define LOG_REMOTE_HLR_MSG(remote_hlr, gsup_msg, level, fmt, args...) \
+	LOG_REMOTE_HLR(remote_hlr, level, "%s: " fmt, osmo_gsup_message_type_name((gsup_msg)->message_type), ##args)
+
+/* GSUP client link for proxying to a remote HLR. */
+struct remote_hlr {
+	struct llist_head entry;
+	struct osmo_sockaddr_str addr;
+	struct osmo_gsup_client *gsupc;
+	struct llist_head pending_up_callbacks;
+};
+
+/*! Receive a remote_hlr address when connecting succeeded, or remote_hlr == NULL on error.
+ * \param addr  GSUP IP address and port for which the connection was requested.
+ * \param remote_hlr  The connected remote_hlr ready for sending, or NULL if connecting failed.
+ * \param data  Same a passed to remote_hlr_get_or_connect(). */
+typedef void (*remote_hlr_connect_result_cb_t)(const struct osmo_sockaddr_str *addr, struct remote_hlr *remote_hlr, void *data);
+
+struct remote_hlr *remote_hlr_get_or_connect(const struct osmo_sockaddr_str *addr, bool connect,
+					     remote_hlr_connect_result_cb_t connect_result_cb, void *data);
+void remote_hlr_destroy(struct remote_hlr *remote_hlr);
+int remote_hlr_msgb_send(struct remote_hlr *remote_hlr, struct msgb *msg);
+void remote_hlr_gsup_forward_to_remote_hlr(struct remote_hlr *remote_hlr, struct osmo_gsup_req *req,
+					   struct osmo_gsup_message *modified_gsup);
+
+bool remote_hlr_is_up(struct remote_hlr *remote_hlr);

include/osmocom/hlr/timestamp.h

@@ -1,9 +1,6 @@
-/* OsmoHLR generic header */
-
-/* (C) 2017 sysmocom s.f.m.c. GmbH <info@sysmocom.de>
- * All Rights Reserved
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
  *
- * Author: Max Suraev <msuraev@sysmocom.de>
+ * All Rights Reserved
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as published by
@@ -22,19 +19,10 @@
 
 #pragma once
 
+#include <sys/time.h>
 #include <stdbool.h>
+#include <stdint.h>
 
-struct hlr {
-	/* GSUP server pointer */
-	struct osmo_gsup_server *gs;
-
-	/* DB context */
-	struct db_context *dbc;
-
-	/* Control Interface */
-	struct ctrl_handle *ctrl;
-	const char *ctrl_bind_addr;
-
-	/* Local bind addr */
-	char *gsup_bind_addr;
-};
+typedef time_t timestamp_t;
+void timestamp_update(timestamp_t *timestamp);
+bool timestamp_age(const timestamp_t *timestamp, uint32_t *age);

include/osmocom/mslookup/Makefile.am

@@ -0,0 +1,6 @@
+# most headers here are installed, see /include/Makefile.am
+
+noinst_HEADERS = \
+	mdns_msg.h \
+	mdns_rfc.h \
+	$(NULL)

include/osmocom/mslookup/mdns.h

@@ -0,0 +1,39 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*! \file mdns.h */
+
+#pragma once
+
+#include <osmocom/core/msgb.h>
+#include <osmocom/mslookup/mslookup.h>
+
+#define OSMO_MDNS_DOMAIN_SUFFIX_DEFAULT "mdns.osmocom.org"
+
+struct msgb *osmo_mdns_query_encode(void *ctx, uint16_t packet_id, const struct osmo_mslookup_query *query,
+				    const char *domain_suffix);
+
+struct osmo_mslookup_query *osmo_mdns_query_decode(void *ctx, const uint8_t *data, size_t data_len,
+						   uint16_t *packet_id, const char *domain_suffix);
+
+struct msgb *osmo_mdns_result_encode(void *ctx, uint16_t packet_id, const struct osmo_mslookup_query *query,
+				     const struct osmo_mslookup_result *result, const char *domain_suffix);
+
+int osmo_mdns_result_decode(void *ctx, const uint8_t *data, size_t data_len, uint16_t *packet_id,
+			    struct osmo_mslookup_query *query, struct osmo_mslookup_result *result,
+			    const char *domain_suffix);

include/osmocom/mslookup/mdns_msg.h

@@ -0,0 +1,54 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <stdint.h>
+#include "mdns_rfc.h"
+
+struct osmo_mdns_record {
+	struct llist_head list;
+	enum osmo_mdns_rfc_record_type type;
+	uint16_t length;
+	uint8_t *data;
+};
+
+struct osmo_mdns_msg_request {
+	uint16_t id;
+	char *domain;
+	enum osmo_mdns_rfc_record_type type;
+};
+
+struct osmo_mdns_msg_answer {
+	uint16_t id;
+	char *domain;
+	/*! list of osmo_mdns_record. */
+	struct llist_head records;
+};
+
+int osmo_mdns_msg_request_encode(void *ctx, struct msgb *msg, const struct osmo_mdns_msg_request *req);
+struct osmo_mdns_msg_request *osmo_mdns_msg_request_decode(void *ctx, const uint8_t *data, size_t data_len);
+
+void osmo_mdns_msg_answer_init(struct osmo_mdns_msg_answer *answer);
+int osmo_mdns_msg_answer_encode(void *ctx, struct msgb *msg, const struct osmo_mdns_msg_answer *ans);
+struct osmo_mdns_msg_answer *osmo_mdns_msg_answer_decode(void *ctx, const uint8_t *data, size_t data_len);
+int osmo_mdns_result_from_answer(struct osmo_mslookup_result *result, const struct osmo_mdns_msg_answer *ans);
+
+struct osmo_mdns_record *osmo_mdns_record_txt_keyval_encode(void *ctx, const char *key, const char *value_fmt, ...);
+int osmo_mdns_record_txt_keyval_decode(const struct osmo_mdns_record *rec,
+				       char *key_buf, size_t key_size, char *value_buf, size_t value_size);

include/osmocom/mslookup/mdns_rfc.h

@@ -0,0 +1,113 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <osmocom/core/msgb.h>
+#include <osmocom/core/endian.h>
+#include <osmocom/mslookup/mdns.h>
+
+/* RFC 1035 2.3.4 */
+#define OSMO_MDNS_RFC_MAX_NAME_LEN 255
+
+/* RFC 1035 3.3 <character-string> */
+#define OSMO_MDNS_RFC_MAX_CHARACTER_STRING_LEN 256
+
+enum osmo_mdns_rfc_record_type {
+	OSMO_MDNS_RFC_RECORD_TYPE_UNKNOWN = 0,
+
+	/* RFC 1035 3.2.2 */
+	OSMO_MDNS_RFC_RECORD_TYPE_A = 1, /* IPv4 address */
+	OSMO_MDNS_RFC_RECORD_TYPE_TXT = 16, /* Text strings */
+
+	/* RFC 3596 2.1 */
+	OSMO_MDNS_RFC_RECORD_TYPE_AAAA = 28, /* IPv6 address */
+
+	/* RFC 1035 3.2.3 */
+	OSMO_MDNS_RFC_RECORD_TYPE_ALL = 255, /* Request only: ask for all */
+};
+
+enum osmo_mdns_rfc_class {
+	OSMO_MDNS_RFC_CLASS_UNKNOWN = 0,
+
+	/* RFC 1035 3.2.4 */
+	OSMO_MDNS_RFC_CLASS_IN = 1, /* Internet and IP networks */
+
+	/* RFC 1035 3.2.5 */
+	OSMO_MDNS_RFC_CLASS_ALL = 255, /* Request only: ask for all */
+};
+
+/* RFC 1035 4.1.1 */
+struct osmo_mdns_rfc_header {
+#if OSMO_IS_LITTLE_ENDIAN
+	uint16_t id;
+	uint8_t rd:1,
+		tc:1,
+		aa:1,
+		opcode:4,
+		qr:1; /* QR (0: query, 1: response) */
+	uint8_t rcode:4,
+		z:3,
+		ra:1;
+	uint16_t qdcount; /* Number of questions */
+	uint16_t ancount; /* Number of answers */
+	uint16_t nscount; /* Number of authority records */
+	uint16_t arcount; /* Number of additional records */
+#elif OSMO_IS_BIG_ENDIAN
+/* auto-generated from the little endian part above (libosmocore/contrib/struct_endianess.py) */
+	uint16_t id;
+	uint8_t qr:1, opcode:4, aa:1, tc:1, rd:1;
+	uint8_t ra:1, z:3, rcode:4;
+	uint16_t qdcount;
+	uint16_t ancount;
+	uint16_t nscount;
+	uint16_t arcount;
+#endif
+} __attribute__ ((packed));
+
+/* RFC 1035 4.1.2 */
+struct osmo_mdns_rfc_question {
+	char *domain; /* Domain to be encoded as qname (e.g. "gsup.hlr.1234567.imsi") */
+	enum osmo_mdns_rfc_record_type qtype;
+	enum osmo_mdns_rfc_class qclass;
+};
+
+/* RFC 1035 4.1.3 */
+struct osmo_mdns_rfc_record {
+	char *domain; /* Domain to be encoded as name (e.g. "gsup.hlr.1234567.imsi") */
+	enum osmo_mdns_rfc_record_type type;
+	enum osmo_mdns_rfc_class class;
+	uint32_t ttl;
+	uint16_t rdlength;
+	uint8_t *rdata;
+};
+
+char *osmo_mdns_rfc_qname_encode(void *ctx, const char *domain);
+char *osmo_mdns_rfc_qname_decode(void *ctx, const char *qname, size_t qname_len);
+
+void osmo_mdns_rfc_header_encode(struct msgb *msg, const struct osmo_mdns_rfc_header *hdr);
+int osmo_mdns_rfc_header_decode(const uint8_t *data, size_t data_len, struct osmo_mdns_rfc_header *hdr);
+
+int osmo_mdns_rfc_question_encode(void *ctx, struct msgb *msg, const struct osmo_mdns_rfc_question *qst);
+struct osmo_mdns_rfc_question *osmo_mdns_rfc_question_decode(void *ctx, const uint8_t *data, size_t data_len);
+
+int osmo_mdns_rfc_record_encode(void *ctx, struct msgb *msg, const struct osmo_mdns_rfc_record *rec);
+struct osmo_mdns_rfc_record *osmo_mdns_rfc_record_decode(void *ctx, const uint8_t *data, size_t data_len,
+							 size_t *record_len);

include/osmocom/mslookup/mdns_sock.h

@@ -0,0 +1,33 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+#include <netdb.h>
+#include <osmocom/core/msgb.h>
+#include <osmocom/core/select.h>
+
+struct osmo_mdns_sock {
+	struct osmo_fd osmo_fd;
+	struct addrinfo *ai;
+};
+
+struct osmo_mdns_sock *osmo_mdns_sock_init(void *ctx, const char *ip, unsigned int port,
+					   int (*cb)(struct osmo_fd *fd, unsigned int what),
+					   void *data, unsigned int priv_nr);
+int osmo_mdns_sock_send(const struct osmo_mdns_sock *mdns_sock, struct msgb *msg);
+void osmo_mdns_sock_cleanup(struct osmo_mdns_sock *mdns_sock);

include/osmocom/mslookup/mslookup.h

@@ -0,0 +1,121 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*! \defgroup mslookup Distributed GSM: finding subscribers
+ *  @{
+ * \file mslookup.h
+ */
+
+#pragma once
+
+#include <osmocom/core/utils.h>
+#include <osmocom/core/sockaddr_str.h>
+#include <osmocom/gsm/protocol/gsm_23_003.h>
+
+#define OSMO_MSLOOKUP_SERVICE_MAXLEN 64
+
+bool osmo_mslookup_service_valid(const char *service);
+
+enum osmo_mslookup_id_type {
+	OSMO_MSLOOKUP_ID_NONE = 0,
+	OSMO_MSLOOKUP_ID_IMSI,
+	OSMO_MSLOOKUP_ID_MSISDN,
+};
+
+extern const struct value_string osmo_mslookup_id_type_names[];
+static inline const char *osmo_mslookup_id_type_name(enum osmo_mslookup_id_type val)
+{ return get_value_string(osmo_mslookup_id_type_names, val); }
+
+struct osmo_mslookup_id {
+	enum osmo_mslookup_id_type type;
+	union {
+		char imsi[GSM23003_IMSI_MAX_DIGITS+1];
+		char msisdn[GSM23003_MSISDN_MAX_DIGITS+1];
+	};
+};
+
+int osmo_mslookup_id_cmp(const struct osmo_mslookup_id *a, const struct osmo_mslookup_id *b);
+bool osmo_mslookup_id_valid(const struct osmo_mslookup_id *id);
+
+enum osmo_mslookup_result_code {
+	OSMO_MSLOOKUP_RC_NONE = 0,
+	/*! An intermediate valid result. The request is still open for more results. */
+	OSMO_MSLOOKUP_RC_RESULT,
+	/*! Returned when the final request timeout has elapsed without results. */
+	OSMO_MSLOOKUP_RC_NOT_FOUND,
+};
+
+extern const struct value_string osmo_mslookup_result_code_names[];
+static inline const char *osmo_mslookup_result_code_name(enum osmo_mslookup_result_code val)
+{ return get_value_string(osmo_mslookup_result_code_names, val); }
+
+/*! Information to request from a lookup. */
+struct osmo_mslookup_query {
+	/*! Which service to request, by freely invented names. For service name conventions (for voice, SMS, HLR,...),
+	 * refer to the OsmoHLR user's manual http://ftp.osmocom.org/docs/latest/osmohlr-usermanual.pdf */
+	char service[OSMO_MSLOOKUP_SERVICE_MAXLEN + 1];
+	/*! IMSI or MSISDN to look up. */
+	struct osmo_mslookup_id id;
+
+	/*! Caller provided private data, if desired. */
+	void *priv;
+};
+
+/*! Result data as passed back to a lookup client that invoked an osmo_mslookup_client_request. */
+struct osmo_mslookup_result {
+	/*! Outcome of the request. */
+	enum osmo_mslookup_result_code rc;
+
+	/*! IP address and port to reach the given service via IPv4, if any. */
+	struct osmo_sockaddr_str host_v4;
+
+	/*! IP address and port to reach the given service via IPv6, if any. */
+	struct osmo_sockaddr_str host_v6;
+
+	/*! How long ago the service last verified presence of the subscriber, in seconds, or zero if the presence is
+	 * invariable (like the home HLR record for an IMSI).
+	 * If a subscriber has recently moved to a different location, we get multiple replies and want to choose the
+	 * most recent one. If this were a timestamp, firstly the time zones would need to be taken care of.
+	 * Even if we choose UTC, a service provider with an inaccurate date/time would end up affecting the result.
+	 * The least susceptible to configuration errors or difference in local and remote clock is a value that
+	 * indicates the actual age of the record in seconds. The time that the lookup query took to be answered should
+	 * be neglectable here, since we would typically wait one second (or very few seconds) for lookup replies,
+	 * while typical Location Updating periods are in the range of 15 minutes. */
+	uint32_t age;
+
+	/*! Whether this is the last result returned for this request. */
+	bool last;
+};
+
+int osmo_mslookup_query_init_from_domain_str(struct osmo_mslookup_query *q, const char *domain);
+
+size_t osmo_mslookup_id_name_buf(char *buf, size_t buflen, const struct osmo_mslookup_id *id);
+char *osmo_mslookup_id_name_c(void *ctx, const struct osmo_mslookup_id *id);
+char *osmo_mslookup_id_name_b(char *buf, size_t buflen, const struct osmo_mslookup_id *id);
+
+size_t osmo_mslookup_result_to_str_buf(char *buf, size_t buflen,
+				     const struct osmo_mslookup_query *query,
+				     const struct osmo_mslookup_result *result);
+char *osmo_mslookup_result_name_c(void *ctx,
+				  const struct osmo_mslookup_query *query,
+				  const struct osmo_mslookup_result *result);
+char *osmo_mslookup_result_name_b(char *buf, size_t buflen,
+				  const struct osmo_mslookup_query *query,
+				  const struct osmo_mslookup_result *result);
+
+/*! @} */

include/osmocom/mslookup/mslookup_client.h

@@ -0,0 +1,132 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <osmocom/core/linuxlist.h>
+#include <osmocom/core/timer.h>
+#include <osmocom/core/sockaddr_str.h>
+#include <osmocom/mslookup/mslookup.h>
+
+struct osmo_mslookup_client;
+struct osmo_mslookup_result;
+
+typedef void (*osmo_mslookup_cb_t)(struct osmo_mslookup_client *client,
+				   uint32_t request_handle,
+				   const struct osmo_mslookup_query *query,
+				   const struct osmo_mslookup_result *result);
+
+/*! This handling information is passed along with a lookup request.
+ * It tells the osmo_mslookup_client layer how to handle responses received from various mslookup methods (at the time
+ * of writing only mDNS exists as a method, but the intention is to easily allow adding other methods in the future).
+ * This query handling info is not seen by the individual method implementations, to clarify that it is the
+ * osmo_mslookup_client layer that takes care of these details. */
+struct osmo_mslookup_query_handling {
+	/*! Wait at least this long before returning any results.
+	 *
+	 * If nonzero, result_cb will be called as soon as this delay has elapsed, either with the so far youngest age
+	 * result, or with a "not found yet" result. After this delay has elapsed, receiving results will continue
+	 * until result_timeout_milliseconds has elapsed.
+	 *
+	 * If zero, responses are fed to the result_cb right from the start, every time a younger aged result than
+	 * before comes in.
+	 *
+	 * If a result with age == 0 is received, min_wait_milliseconds is ignored, the result is returned immediately
+	 * and listening for responses ends.
+	 *
+	 * Rationale: If a subscriber has recently moved between sites, multiple results will arrive, and the youngest
+	 * age wins. It can make sense to wait a minimum time for responses before determining the winning result.
+	 *
+	 * However, if no result or no valid result has arrived within a short period, the subscriber may be at a site
+	 * that is far away or that is currently experiencing high latency. It is thus a good safety net to still
+	 * receive results for an extended period of time.
+	 *
+	 * For some services, it is possible to establish links to every received result, and whichever link succeeds
+	 * will be used (for example for SIP calls: first to pick up the call gets connected, the others are dropped
+	 * silently).
+	 */
+	uint32_t min_wait_milliseconds;
+
+	/*! Total time in milliseconds to listen for lookup responses.
+	 *
+	 * When this timeout elapses, osmo_mslookup_client_request_cancel() is called implicitly; Manually invoking
+	 * osmo_mslookup_client_request_cancel() after result_timeout_milliseconds has elapsed is not necessary, but is
+	 * still safe to do anyway.
+	 *
+	 * If zero, min_wait_milliseconds is also used as result_timeout_milliseconds; if that is also zero, a default
+	 * timeout value is used.
+	 *
+	 * If result_timeout_milliseconds <= min_wait_milliseconds, then min_wait_milliseconds is used as
+	 * result_timeout_milliseconds, i.e. the timeout triggers as soon as min_wait_milliseconds hits.
+	 *
+	 * osmo_mslookup_client_request_cancel() can be called any time to end the request.
+	 */
+	uint32_t result_timeout_milliseconds;
+
+	/*! Invoked every time a result with a younger age than the previous result has arrived.
+	 * To stop receiving results before result_timeout_milliseconds has elapsed, call
+	 * osmo_mslookup_client_request_cancel().
+	 */
+	osmo_mslookup_cb_t result_cb;
+};
+
+uint32_t osmo_mslookup_client_request(struct osmo_mslookup_client *client,
+				      const struct osmo_mslookup_query *query,
+				      const struct osmo_mslookup_query_handling *handling);
+
+void osmo_mslookup_client_request_cancel(struct osmo_mslookup_client *client, uint32_t request_handle);
+
+struct osmo_mslookup_client *osmo_mslookup_client_new(void *ctx);
+bool osmo_mslookup_client_active(struct osmo_mslookup_client *client);
+void osmo_mslookup_client_free(struct osmo_mslookup_client *client);
+
+/*! Describe a specific mslookup client method implementation. This struct is only useful for a lookup method
+ * implementation to add itself to an osmo_mslookup_client, see for example osmo_mslookup_client_add_mdns(). */
+struct osmo_mslookup_client_method {
+	struct llist_head entry;
+
+	/*! Human readable name of this lookup method. */
+	const char *name;
+
+	/*! Private data for the lookup method implementation. */
+	void *priv;
+
+	/*! Backpointer to the client this method is added to. */
+	struct osmo_mslookup_client *client;
+
+	/*! Launch a lookup query. Called from osmo_mslookup_client_request().
+	 * The implementation returns results by calling osmo_mslookup_client_rx_result(). */
+	void (*request)(struct osmo_mslookup_client_method *method,
+			const struct osmo_mslookup_query *query,
+			uint32_t request_handle);
+	/*! End a lookup query. Called from osmo_mslookup_client_request_cancel(). It is guaranteed to be called
+	 * exactly once per above request() invocation. (The API user is required to invoke
+	 * osmo_mslookup_client_request_cancel() exactly once per osmo_mslookup_client_request().) */
+	void (*request_cleanup)(struct osmo_mslookup_client_method *method,
+				uint32_t request_handle);
+
+	/*! The mslookup_client is removing this method, clean up all open requests, lists and allocations. */
+	void (*destruct)(struct osmo_mslookup_client_method *method);
+};
+
+void osmo_mslookup_client_method_add(struct osmo_mslookup_client *client,
+				     struct osmo_mslookup_client_method *method);
+bool osmo_mslookup_client_method_del(struct osmo_mslookup_client *client,
+				     struct osmo_mslookup_client_method *method);
+void osmo_mslookup_client_rx_result(struct osmo_mslookup_client *client, uint32_t request_handle,
+				    const struct osmo_mslookup_result *result);

include/osmocom/mslookup/mslookup_client_fake.h

@@ -0,0 +1,34 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+/*! MS lookup fake API for testing purposes. */
+#include <osmocom/mslookup/mslookup_client.h>
+
+struct osmo_mslookup_fake_response {
+	struct timeval time_to_reply;
+	struct osmo_mslookup_id for_id;
+	const char *for_service;
+	struct osmo_mslookup_result result;
+	bool sent;
+};
+
+struct osmo_mslookup_client_method *osmo_mslookup_client_add_fake(struct osmo_mslookup_client *client,
+								  struct osmo_mslookup_fake_response *responses,
+								  size_t responses_len);

include/osmocom/mslookup/mslookup_client_mdns.h

@@ -0,0 +1,38 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <stdint.h>
+
+struct osmo_mslookup_client;
+struct osmo_mslookup_client_method;
+
+/*! MS Lookup mDNS server bind default IP. Taken from the Administratevly Scoped block, particularly the Organizational
+ * Scoped range, https://tools.ietf.org/html/rfc2365 . */
+#define OSMO_MSLOOKUP_MDNS_IP4 "239.192.23.42"
+#define OSMO_MSLOOKUP_MDNS_IP6 "ff08::23:42" // <-- TODO: sane?
+#define OSMO_MSLOOKUP_MDNS_PORT 4266
+
+struct osmo_mslookup_client_method *osmo_mslookup_client_add_mdns(struct osmo_mslookup_client *client, const char *ip,
+								  uint16_t port, int initial_packet_id,
+								  const char *domain_suffix);
+
+const struct osmo_sockaddr_str *osmo_mslookup_client_method_mdns_get_bind_addr(struct osmo_mslookup_client_method *dns_method);
+
+const char *osmo_mslookup_client_method_mdns_get_domain_suffix(struct osmo_mslookup_client_method *dns_method);

libosmo-gsup-client.pc.in

@@ -0,0 +1,11 @@
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: Osmocom GSUP Client Library
+Description: C Utility Library
+Version: @VERSION@
+Libs: -L${libdir} -losmo-gsup-client
+Cflags: -I${includedir}/
+

libosmo-mslookup.pc.in

@@ -0,0 +1,11 @@
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: Osmocom MS Lookup Library
+Description: C Utility Library
+Version: @VERSION@
+Libs: -L${libdir} @TALLOC_LIBS@ -losmogsm -losmo-mslookup -losmocore
+Cflags: -I${includedir}/
+

sql/Makefile.am

@@ -3,5 +3,12 @@ EXTRA_DIST = \
 	hlr.sql \
 	$(NULL)
 
-docsdir = $(datadir)/doc/osmo-hlr
-docs_DATA = $(srcdir)/hlr.sql
+sqldir = $(docdir)/sql
+sql_DATA = $(srcdir)/hlr.sql $(srcdir)/hlr_data.sql
+
+
+install-data-local:
+	$(MKDIR_P) $(DESTDIR)$(localstatedir)/lib/osmocom
+
+uninstall-hook:
+	rm -rf $(DESTDIR)$(localstatedir)/lib/osmocom

sql/hlr.sql

@@ -1,17 +1,18 @@
---modelled roughly after TS 23.008 version 13.3.0
-
 CREATE TABLE subscriber (
+-- OsmoHLR's DB scheme is modelled roughly after TS 23.008 version 13.3.0
 	id		INTEGER PRIMARY KEY,
 	-- Chapter 2.1.1.1
 	imsi		VARCHAR(15) UNIQUE NOT NULL,
 	-- Chapter 2.1.2
 	msisdn		VARCHAR(15) UNIQUE,
-	-- Chapter 2.2.3: Most recent / current IMEI
+	-- Chapter 2.2.3: Most recent / current IMEISV
 	imeisv		VARCHAR,
+	-- Chapter 2.1.9: Most recent / current IMEI
+	imei		VARCHAR(14),
 	-- Chapter 2.4.5
 	vlr_number	VARCHAR(15),
 	-- Chapter 2.4.6
-	hlr_number	VARCHAR(15),
+	msc_number	VARCHAR(15),
 	-- Chapter 2.4.8.1
 	sgsn_number	VARCHAR(15),
 	-- Chapter 2.13.10
@@ -37,7 +38,17 @@ CREATE TABLE subscriber (
 	-- Chapter 2.7.5
 	ms_purged_cs	BOOLEAN NOT NULL DEFAULT 0,
 	-- Chapter 2.7.6
-	ms_purged_ps	BOOLEAN NOT NULL DEFAULT 0
+	ms_purged_ps	BOOLEAN NOT NULL DEFAULT 0,
+
+	-- Timestamp of last location update seen from subscriber
+	-- The value is a string which encodes a UTC timestamp in granularity of seconds.
+	last_lu_seen TIMESTAMP default NULL,
+	last_lu_seen_ps TIMESTAMP default NULL,
+
+	-- When a LU was received via a proxy, that proxy's hlr_number is stored here,
+	-- while vlr_number reflects the MSC on the far side of that proxy.
+	vlr_via_proxy	VARCHAR,
+	sgsn_via_proxy	VARCHAR
 );
 
 CREATE TABLE subscriber_apn (
@@ -45,8 +56,8 @@ CREATE TABLE subscriber_apn (
 	apn		VARCHAR(256) NOT NULL
 );
 
--- Chapter 2.1.3
 CREATE TABLE subscriber_multi_msisdn (
+-- Chapter 2.1.3
 	subscriber_id	INTEGER,		-- subscriber.id
 	msisdn		VARCHAR(15) NOT NULL
 );
@@ -64,8 +75,12 @@ CREATE TABLE auc_3g (
 	op		VARCHAR(32),		-- hex string: operator's secret key (128bit)
 	opc		VARCHAR(32),		-- hex string: derived from OP and K (128bit)
 	sqn		INTEGER NOT NULL DEFAULT 0,	-- sequence number of key usage
-	ind_bitlen	INTEGER NOT NULL DEFAULT 5	-- nr of index bits at lower SQN end
+	-- nr of index bits at lower SQN end
+	ind_bitlen	INTEGER NOT NULL DEFAULT 5
 );
 
-CREATE UNIQUE INDEX IF NOT EXISTS idx_subscr_imsi ON subscriber (imsi);
--- SELECT algo_id_2g, ki, algo_id_3g, k, op, opc, sqn FROM subscriber LEFT JOIN auc_2g ON auc_2g.subscriber_id = subscriber.id LEFT JOIN auc_3g ON auc_3g.subscriber_id = subscriber.id WHERE imsi = ?
+CREATE UNIQUE INDEX idx_subscr_imsi ON subscriber (imsi);
+
+-- Set HLR database schema version number
+-- Note: This constant is currently duplicated in src/db.c and must be kept in sync!
+PRAGMA user_version = 5;

src/Makefile.am

@@ -1,44 +1,47 @@
+SUBDIRS = \
+	gsupclient \
+	mslookup \
+	$(NULL)
+
 AM_CFLAGS = \
 	-Wall \
 	$(LIBOSMOCORE_CFLAGS) \
 	$(LIBOSMOGSM_CFLAGS) \
 	$(LIBOSMOVTY_CFLAGS) \
 	$(LIBOSMOCTRL_CFLAGS) \
+	$(LIBOSMOMSLOOKUP_CFLAGS) \
 	$(LIBOSMOABIS_CFLAGS) \
 	$(SQLITE3_CFLAGS) \
 	$(NULL)
 
-EXTRA_DIST = \
-	populate_hlr_db.pl \
+AM_CPPFLAGS = -I$(top_srcdir)/include \
+	-I$(top_builddir)/include \
 	$(NULL)
 
+EXTRA_DIST = \
+	populate_hlr_db.pl \
+	db_sql2c.sed \
+	$(NULL)
+
+BUILT_SOURCES = \
+	db_bootstrap.h \
+	$(NULL)
+CLEANFILES = $(BUILT_SOURCES)
+
 noinst_HEADERS = \
-	auc.h \
-	db.h \
-	hlr.h \
-	luop.h \
-	gsup_router.h \
-	gsup_server.h \
-	logging.h \
-	rand.h \
-	ctrl.h \
-	hlr_vty.h \
-	hlr_vty_subscr.h \
+	db_bootstrap.h \
 	$(NULL)
 
 bin_PROGRAMS = \
 	osmo-hlr \
-	$(NULL)
-
-noinst_PROGRAMS = \
-	db_test \
+	osmo-hlr-db-tool \
+	osmo-euse-demo \
 	$(NULL)
 
 osmo_hlr_SOURCES = \
 	auc.c \
 	ctrl.c \
 	db.c \
-	luop.c \
 	db_auc.c \
 	db_hlr.c \
 	gsup_router.c \
@@ -48,28 +51,68 @@ osmo_hlr_SOURCES = \
 	rand_urandom.c \
 	hlr_vty.c \
 	hlr_vty_subscr.c \
+	gsup_send.c \
+	hlr_ussd.c \
+	proxy.c \
+	dgsm.c \
+	remote_hlr.c \
+	lu_fsm.c \
+	timestamp.c \
+	mslookup_server.c \
+	mslookup_server_mdns.c \
+	dgsm_vty.c \
 	$(NULL)
 
 osmo_hlr_LDADD = \
+	$(top_builddir)/src/gsupclient/libosmo-gsup-client.la \
+	$(top_builddir)/src/mslookup/libosmo-mslookup.la \
 	$(LIBOSMOCORE_LIBS) \
 	$(LIBOSMOGSM_LIBS) \
 	$(LIBOSMOVTY_LIBS) \
 	$(LIBOSMOCTRL_LIBS) \
+	$(LIBOSMOMSLOOKUP_LIBS) \
 	$(LIBOSMOABIS_LIBS) \
 	$(SQLITE3_LIBS) \
 	$(NULL)
 
-db_test_SOURCES = \
-	auc.c \
+osmo_hlr_db_tool_SOURCES = \
+	hlr_db_tool.c \
 	db.c \
-	db_auc.c \
-	db_test.c \
+	db_hlr.c \
 	logging.c \
-	rand_fake.c \
+	rand_urandom.c \
+	dbd_decode_binary.c \
+	$(srcdir)/gsupclient/cni_peer_id.c \
 	$(NULL)
 
-db_test_LDADD = \
+osmo_hlr_db_tool_LDADD = \
 	$(LIBOSMOCORE_LIBS) \
 	$(LIBOSMOGSM_LIBS) \
 	$(SQLITE3_LIBS) \
 	$(NULL)
+
+osmo_euse_demo_SOURCES = \
+	osmo-euse-demo.c \
+	$(NULL)
+
+osmo_euse_demo_LDADD = \
+	$(top_builddir)/src/gsupclient/libosmo-gsup-client.la \
+	$(LIBOSMOCORE_LIBS) \
+	$(LIBOSMOGSM_LIBS) \
+	$(NULL)
+
+if DB_SQLITE_DEBUG
+osmo_hlr_SOURCES += db_debug.c
+osmo_hlr_db_tool_SOURCES += db_debug.c
+endif
+
+BOOTSTRAP_SQL = $(top_srcdir)/sql/hlr.sql
+
+db_bootstrap.h: $(BOOTSTRAP_SQL) $(srcdir)/db_sql2c.sed
+	echo "/* DO NOT EDIT THIS FILE. It is generated from files in osmo-hlr.git/sql/ */" > "$@"
+	echo "#pragma once" >> "$@"
+	echo "static const char *stmt_bootstrap_sql[] = {" >> "$@"
+	cat "$(BOOTSTRAP_SQL)" \
+		| sed -f "$(srcdir)/db_sql2c.sed" \
+		>> "$@"
+	echo "};" >> "$@"

src/auc.c

@@ -23,8 +23,8 @@
 #include <osmocom/core/utils.h>
 #include <osmocom/crypt/auth.h>
 
-#include "logging.h"
-#include "rand.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/rand.h>
 
 #define hexb(buf) osmo_hexdump_nospc((void*)buf, sizeof(buf))
 #define hex(buf,sz) osmo_hexdump_nospc((void*)buf, sz)
@@ -144,6 +144,7 @@ int auc_compute_vectors(struct osmo_auth_vector *vec, unsigned int num_vec,
 
 			if (!aud2g) {
 				/* use the 2G tokens from 3G keys */
+				DBGP("vector [%u]: deriving 2G from 3G\n", i);
 				DBGVB(kc);
 				DBGVB(sres);
 				DBGVV("0x%x", auth_types);
@@ -151,7 +152,7 @@ int auc_compute_vectors(struct osmo_auth_vector *vec, unsigned int num_vec,
 			}
 			/* calculate 2G separately */
 
-			DBGP("vector [%u]: deriving 2G from 3G\n", i);
+			DBGP("vector [%u]: calculating 2G separately\n", i);
 
 			rc = osmo_auth_gen_vec(&vtmp, aud2g, rand);
 			if (rc < 0) {

src/ctrl.c

@@ -21,87 +21,382 @@
  */
 
 #include <stdbool.h>
+#include <errno.h>
+#include <inttypes.h>
+#include <string.h>
 
-#include <osmocom/ctrl/control_cmd.h>
-#include <osmocom/ctrl/control_if.h>
+#include <osmocom/gsm/gsm23003.h>
 #include <osmocom/ctrl/ports.h>
 
-#include "gsup_server.h"
-#include "logging.h"
-#include "db.h"
-#include "hlr.h"
-#include "luop.h"
-#include "ctrl.h"
+#include <osmocom/hlr/hlr.h>
+#include <osmocom/hlr/ctrl.h>
+#include <osmocom/hlr/db.h>
 
-static int handle_cmd_ps(struct hlr *ctx, struct ctrl_cmd *cmd, bool enable)
+#define SEL_BY "by-"
+#define SEL_BY_IMSI SEL_BY "imsi-"
+#define SEL_BY_MSISDN SEL_BY "msisdn-"
+#define SEL_BY_ID SEL_BY "id-"
+
+#define hexdump_buf(buf) osmo_hexdump_nospc((void*)buf, sizeof(buf))
+
+static bool startswith(const char *str, const char *start)
+{
+	return strncmp(str, start, strlen(start)) == 0;
+}
+
+static int _get_subscriber(struct db_context *dbc,
+			   const char *by_selector,
+			   struct hlr_subscriber *subscr)
+{
+	const char *val;
+	if (startswith(by_selector, SEL_BY_IMSI)) {
+		val = by_selector + strlen(SEL_BY_IMSI);
+		if (!osmo_imsi_str_valid(val))
+			return -EINVAL;
+		return db_subscr_get_by_imsi(dbc, val, subscr);
+	}
+	if (startswith(by_selector, SEL_BY_MSISDN)) {
+		val = by_selector + strlen(SEL_BY_MSISDN);
+		if (!osmo_msisdn_str_valid(val))
+			return -EINVAL;
+		return db_subscr_get_by_msisdn(dbc, val, subscr);
+	}
+	if (startswith(by_selector, SEL_BY_ID)) {
+		int64_t id;
+		char *endptr;
+		val = by_selector + strlen(SEL_BY_ID);
+		if (*val == '+')
+			return -EINVAL;
+		errno = 0;
+		id = strtoll(val, &endptr, 10);
+		if (errno || *endptr)
+			return -EINVAL;
+		return db_subscr_get_by_id(dbc, id, subscr);
+	}
+	return -ENOTSUP;
+}
+
+static bool get_subscriber(struct db_context *dbc,
+			   const char *by_selector,
+			   struct hlr_subscriber *subscr,
+			   struct ctrl_cmd *cmd)
+{
+	int rc = _get_subscriber(dbc, by_selector, subscr);
+	switch (rc) {
+	case 0:
+		return true;
+	case -ENOTSUP:
+		cmd->reply = "Not a known subscriber 'by-xxx-' selector.";
+		return false;
+	case -EINVAL:
+		cmd->reply = "Invalid value part of 'by-xxx-value' selector.";
+		return false;
+	case -ENOENT:
+		cmd->reply = "No such subscriber.";
+		return false;
+	default:
+		cmd->reply = "An unknown error has occurred during get_subscriber().";
+		return false;
+	}
+}
+
+/* Optimization: if a subscriber operation is requested by-imsi, just return
+ * the IMSI right back. */
+static const char *get_subscriber_imsi(struct db_context *dbc,
+				       const char *by_selector,
+				       struct ctrl_cmd *cmd)
+{
+	static struct hlr_subscriber subscr;
+
+	if (startswith(by_selector, SEL_BY_IMSI))
+		return by_selector + strlen(SEL_BY_IMSI);
+	if (!get_subscriber(dbc, by_selector, &subscr, cmd))
+		return NULL;
+	return subscr.imsi;
+}
+
+/* printf fmt and arg to completely omit a string if it is empty. */
+#define FMT_S "%s%s%s%s"
+#define ARG_S(name, val) \
+	(val) && *(val) ? "\n" : "", \
+	(val) && *(val) ? name : "", \
+	(val) && *(val) ? "\t" : "", \
+	(val) && *(val) ? (val) : "" \
+
+/* printf fmt and arg to completely omit bool of given value. */
+#define FMT_BOOL "%s"
+#define ARG_BOOL(name, val) \
+	val ? "\n" name "\t1" : "\n" name "\t0"
+
+static void print_subscr_info(struct ctrl_cmd *cmd,
+			      struct hlr_subscriber *subscr)
+{
+	ctrl_cmd_reply_printf(cmd,
+		"\nid\t%" PRIu64
+		FMT_S
+		FMT_S
+		FMT_BOOL
+		FMT_BOOL
+		FMT_S
+		FMT_S
+		FMT_S
+		FMT_BOOL
+		FMT_BOOL
+		"\nperiodic_lu_timer\t%u"
+		"\nperiodic_rau_tau_timer\t%u"
+		"\nlmsi\t%08x"
+		,
+		subscr->id,
+		ARG_S("imsi", subscr->imsi),
+		ARG_S("msisdn", subscr->msisdn),
+		ARG_BOOL("nam_cs", subscr->nam_cs),
+		ARG_BOOL("nam_ps", subscr->nam_ps),
+		ARG_S("vlr_number", subscr->vlr_number),
+		ARG_S("sgsn_number", subscr->sgsn_number),
+		ARG_S("sgsn_address", subscr->sgsn_address),
+		ARG_BOOL("ms_purged_cs", subscr->ms_purged_cs),
+		ARG_BOOL("ms_purged_ps", subscr->ms_purged_ps),
+		subscr->periodic_lu_timer,
+		subscr->periodic_rau_tau_timer,
+		subscr->lmsi
+		);
+}
+
+static void print_subscr_info_aud2g(struct ctrl_cmd *cmd, struct osmo_sub_auth_data *aud)
+{
+	if (aud->algo == OSMO_AUTH_ALG_NONE)
+		return;
+	ctrl_cmd_reply_printf(cmd,
+		"\naud2g.algo\t%s"
+		"\naud2g.ki\t%s"
+		,
+		osmo_auth_alg_name(aud->algo),
+		hexdump_buf(aud->u.gsm.ki));
+}
+
+static void print_subscr_info_aud3g(struct ctrl_cmd *cmd, struct osmo_sub_auth_data *aud)
+{
+	if (aud->algo == OSMO_AUTH_ALG_NONE)
+		return;
+	ctrl_cmd_reply_printf(cmd,
+		"\naud3g.algo\t%s"
+		"\naud3g.k\t%s"
+		,
+		osmo_auth_alg_name(aud->algo),
+		hexdump_buf(aud->u.umts.k));
+	/* hexdump uses a static string buffer, hence only one hexdump per
+	 * printf(). */
+	ctrl_cmd_reply_printf(cmd,
+		"\naud3g.%s\t%s"
+		"\naud3g.ind_bitlen\t%u"
+		"\naud3g.sqn\t%" PRIu64
+		,
+		aud->u.umts.opc_is_op? "op" : "opc",
+		hexdump_buf(aud->u.umts.opc),
+		aud->u.umts.ind_bitlen,
+		aud->u.umts.sqn);
+}
+
+CTRL_CMD_DEFINE_RO(subscr_info, "info");
+static int get_subscr_info(struct ctrl_cmd *cmd, void *data)
 {
 	struct hlr_subscriber subscr;
+	struct hlr *hlr = data;
+	const char *by_selector = cmd->node;
 
-	if (db_subscr_get_by_imsi(ctx->dbc, cmd->value, &subscr) < 0) {
-		cmd->reply = "Subscriber Unknown in HLR";
+	if (!get_subscriber(hlr->dbc, by_selector, &subscr, cmd))
+		return CTRL_CMD_ERROR;
+
+	print_subscr_info(cmd, &subscr);
+
+	return CTRL_CMD_REPLY;
+}
+
+CTRL_CMD_DEFINE_RO(subscr_info_aud, "info-aud");
+static int get_subscr_info_aud(struct ctrl_cmd *cmd, void *data)
+{
+	const char *imsi;
+	struct osmo_sub_auth_data aud2g;
+	struct osmo_sub_auth_data aud3g;
+	struct hlr *hlr = data;
+	const char *by_selector = cmd->node;
+	int rc;
+
+	imsi = get_subscriber_imsi(hlr->dbc, by_selector, cmd);
+	if (!imsi)
+		return CTRL_CMD_ERROR;
+
+	rc = db_get_auth_data(hlr->dbc, imsi, &aud2g, &aud3g, NULL);
+
+	switch (rc) {
+	case 0:
+		break;
+	case -ENOENT:
+	case -ENOKEY:
+		/* No auth data found, tell the print*() functions about it. */
+		aud2g.algo = OSMO_AUTH_ALG_NONE;
+		aud3g.algo = OSMO_AUTH_ALG_NONE;
+		break;
+	default:
+		cmd->reply = "Error retrieving authentication data.";
 		return CTRL_CMD_ERROR;
 	}
 
-	if (hlr_subscr_nam(ctx, &subscr, enable, true) < 0) {
-		cmd->reply = "Error updating DB";
+	print_subscr_info_aud2g(cmd, &aud2g);
+	print_subscr_info_aud3g(cmd, &aud3g);
+
+	return CTRL_CMD_REPLY;
+}
+
+CTRL_CMD_DEFINE_RO(subscr_info_all, "info-all");
+static int get_subscr_info_all(struct ctrl_cmd *cmd, void *data)
+{
+	struct hlr_subscriber subscr;
+	struct osmo_sub_auth_data aud2g;
+	struct osmo_sub_auth_data aud3g;
+	struct hlr *hlr = data;
+	const char *by_selector = cmd->node;
+	int rc;
+
+	if (!get_subscriber(hlr->dbc, by_selector, &subscr, cmd))
+		return CTRL_CMD_ERROR;
+
+	rc = db_get_auth_data(hlr->dbc, subscr.imsi, &aud2g, &aud3g, NULL);
+
+	switch (rc) {
+	case 0:
+		break;
+	case -ENOENT:
+	case -ENOKEY:
+		/* No auth data found, tell the print*() functions about it. */
+		aud2g.algo = OSMO_AUTH_ALG_NONE;
+		aud3g.algo = OSMO_AUTH_ALG_NONE;
+		break;
+	default:
+		cmd->reply = "Error retrieving authentication data.";
 		return CTRL_CMD_ERROR;
 	}
 
+	print_subscr_info(cmd, &subscr);
+	print_subscr_info_aud2g(cmd, &aud2g);
+	print_subscr_info_aud3g(cmd, &aud3g);
+
+	return CTRL_CMD_REPLY;
+}
+
+static int verify_subscr_cs_ps_enabled(struct ctrl_cmd *cmd, const char *value, void *data)
+{
+	if (!value || !*value
+	    || (strcmp(value, "0") && strcmp(value, "1")))
+		return 1;
+	return 0;
+}
+
+static int get_subscr_cs_ps_enabled(struct ctrl_cmd *cmd, void *data,
+				    bool is_ps)
+{
+	struct hlr_subscriber subscr;
+	struct hlr *hlr = data;
+	const char *by_selector = cmd->node;
+
+	if (!get_subscriber(hlr->dbc, by_selector, &subscr, cmd))
+		return CTRL_CMD_ERROR;
+
+	cmd->reply = (is_ps ? subscr.nam_ps : subscr.nam_cs)
+		     ? "1" : "0";
+	return CTRL_CMD_REPLY;
+}
+
+static int set_subscr_cs_ps_enabled(struct ctrl_cmd *cmd, void *data,
+				    bool is_ps)
+{
+	const char *imsi;
+	struct hlr *hlr = data;
+	const char *by_selector = cmd->node;
+
+	imsi = get_subscriber_imsi(hlr->dbc, by_selector, cmd);
+	if (!imsi)
+		return CTRL_CMD_ERROR;
+	if (db_subscr_nam(hlr->dbc, imsi, strcmp(cmd->value, "1") == 0, is_ps))
+		return CTRL_CMD_ERROR;
 	cmd->reply = "OK";
 	return CTRL_CMD_REPLY;
 }
 
-CTRL_CMD_DEFINE_WO_NOVRF(enable_ps, "enable-ps");
-static int set_enable_ps(struct ctrl_cmd *cmd, void *data)
+CTRL_CMD_DEFINE(subscr_ps_enabled, "ps-enabled");
+static int verify_subscr_ps_enabled(struct ctrl_cmd *cmd, const char *value, void *data)
 {
-	return handle_cmd_ps(data, cmd, true);
+	return verify_subscr_cs_ps_enabled(cmd, value, data);
+}
+static int get_subscr_ps_enabled(struct ctrl_cmd *cmd, void *data)
+{
+	return get_subscr_cs_ps_enabled(cmd, data, true);
+}
+static int set_subscr_ps_enabled(struct ctrl_cmd *cmd, void *data)
+{
+	return set_subscr_cs_ps_enabled(cmd, data, true);
 }
 
-CTRL_CMD_DEFINE_WO_NOVRF(disable_ps, "disable-ps");
-static int set_disable_ps(struct ctrl_cmd *cmd, void *data)
+CTRL_CMD_DEFINE(subscr_cs_enabled, "cs-enabled");
+static int verify_subscr_cs_enabled(struct ctrl_cmd *cmd, const char *value, void *data)
 {
-	return handle_cmd_ps(data, cmd, false);
+	return verify_subscr_cs_ps_enabled(cmd, value, data);
 }
-
-CTRL_CMD_DEFINE_WO_NOVRF(status_ps, "status-ps");
-static int set_status_ps(struct ctrl_cmd *cmd, void *data)
+static int get_subscr_cs_enabled(struct ctrl_cmd *cmd, void *data)
 {
-	struct hlr *ctx = data;
-	struct lu_operation *luop = lu_op_alloc(ctx->gs);
-	if (!luop) {
-		cmd->reply = "Internal HLR error";
-		return CTRL_CMD_ERROR;
-	}
-
-	if (!lu_op_fill_subscr(luop, ctx->dbc, cmd->value)) {
-		cmd->reply = "Subscriber Unknown in HLR";
-		return CTRL_CMD_ERROR;
-	}
-
-	cmd->reply = luop->subscr.nam_ps ? "1" : "0";
-
-	return CTRL_CMD_REPLY;
+	return get_subscr_cs_ps_enabled(cmd, data, false);
+}
+static int set_subscr_cs_enabled(struct ctrl_cmd *cmd, void *data)
+{
+	return set_subscr_cs_ps_enabled(cmd, data, false);
 }
 
 int hlr_ctrl_cmds_install()
 {
 	int rc = 0;
 
-	rc |= ctrl_cmd_install(CTRL_NODE_ROOT, &cmd_enable_ps);
-	rc |= ctrl_cmd_install(CTRL_NODE_ROOT, &cmd_disable_ps);
-	rc |= ctrl_cmd_install(CTRL_NODE_ROOT, &cmd_status_ps);
+	rc |= ctrl_cmd_install(CTRL_NODE_SUBSCR_BY, &cmd_subscr_info);
+	rc |= ctrl_cmd_install(CTRL_NODE_SUBSCR_BY, &cmd_subscr_info_aud);
+	rc |= ctrl_cmd_install(CTRL_NODE_SUBSCR_BY, &cmd_subscr_info_all);
+	rc |= ctrl_cmd_install(CTRL_NODE_SUBSCR_BY, &cmd_subscr_ps_enabled);
+	rc |= ctrl_cmd_install(CTRL_NODE_SUBSCR_BY, &cmd_subscr_cs_enabled);
 
 	return rc;
 }
 
-struct ctrl_handle *hlr_controlif_setup(struct hlr *ctx,
-					struct osmo_gsup_server *gs)
+static int hlr_ctrl_node_lookup(void *data, vector vline, int *node_type,
+				void **node_data, int *i)
+{
+	const char *token = vector_slot(vline, *i);
+
+	switch (*node_type) {
+	case CTRL_NODE_ROOT:
+		if (strcmp(token, "subscriber") != 0)
+			return 0;
+		*node_data = NULL;
+		*node_type = CTRL_NODE_SUBSCR;
+		break;
+	case CTRL_NODE_SUBSCR:
+		if (!startswith(token, "by-"))
+			return 0;
+		*node_data = (void*)token;
+		*node_type = CTRL_NODE_SUBSCR_BY;
+		break;
+	default:
+		return 0;
+	}
+
+	return 1;
+}
+
+struct ctrl_handle *hlr_controlif_setup(struct hlr *hlr)
 {
 	int rc;
-	struct ctrl_handle *hdl = ctrl_interface_setup_dynip(ctx,
-							     ctx->ctrl_bind_addr,
-							     OSMO_CTRL_PORT_HLR,
-							     NULL);
+	struct ctrl_handle *hdl = ctrl_interface_setup_dynip2(hlr,
+							      hlr->ctrl_bind_addr,
+							      OSMO_CTRL_PORT_HLR,
+							      hlr_ctrl_node_lookup,
+							      _LAST_CTRL_NODE_HLR);
 	if (!hdl)
 		return NULL;
 

src/db.c

@@ -23,13 +23,18 @@
 #include <sqlite3.h>
 #include <string.h>
 
-#include "logging.h"
-#include "db.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/db.h>
+#include "db_bootstrap.h"
+
+/* This constant is currently duplicated in sql/hlr.sql and must be kept in sync! */
+#define CURRENT_SCHEMA_VERSION	5
 
 #define SEL_COLUMNS \
 	"id," \
 	"imsi," \
 	"msisdn," \
+	"imei," \
 	"vlr_number," \
 	"sgsn_number," \
 	"sgsn_address," \
@@ -39,14 +44,28 @@
 	"nam_ps," \
 	"lmsi," \
 	"ms_purged_cs," \
-	"ms_purged_ps"
+	"ms_purged_ps," \
+	"last_lu_seen," \
+	"last_lu_seen_ps," \
+	"vlr_via_proxy," \
+	"sgsn_via_proxy"
 
 static const char *stmt_sql[] = {
+	[DB_STMT_SEL_ALL] = "SELECT " SEL_COLUMNS " FROM subscriber;",
+	[DB_STMT_SEL_ALL_ORDER_LAST_SEEN] = "SELECT " SEL_COLUMNS " FROM subscriber "
+		"WHERE last_lu_seen IS NOT NULL ORDER BY last_lu_seen;",
+	[DB_STMT_SEL_FILTER_MSISDN] = "SELECT " SEL_COLUMNS " FROM subscriber WHERE msisdn LIKE $search ORDER BY msisdn",
+	[DB_STMT_SEL_FILTER_IMSI] = "SELECT " SEL_COLUMNS " FROM subscriber WHERE imsi LIKE $search ORDER BY imsi",
+	[DB_STMT_SEL_FILTER_IMEI] = "SELECT " SEL_COLUMNS " FROM subscriber WHERE imei LIKE $search ORDER BY imei",
+	[DB_STMT_SEL_FILTER_CS] = "SELECT " SEL_COLUMNS " FROM subscriber WHERE nam_cs = $search ORDER BY last_lu_seen",
+	[DB_STMT_SEL_FILTER_PS] = "SELECT " SEL_COLUMNS " FROM subscriber WHERE nam_ps = $search ORDER BY last_lu_seen",
 	[DB_STMT_SEL_BY_IMSI] = "SELECT " SEL_COLUMNS " FROM subscriber WHERE imsi = ?",
 	[DB_STMT_SEL_BY_MSISDN] = "SELECT " SEL_COLUMNS " FROM subscriber WHERE msisdn = ?",
 	[DB_STMT_SEL_BY_ID] = "SELECT " SEL_COLUMNS " FROM subscriber WHERE id = ?",
-	[DB_STMT_UPD_VLR_BY_ID] = "UPDATE subscriber SET vlr_number = $number WHERE id = $subscriber_id",
-	[DB_STMT_UPD_SGSN_BY_ID] = "UPDATE subscriber SET sgsn_number = $number WHERE id = $subscriber_id",
+	[DB_STMT_SEL_BY_IMEI] = "SELECT " SEL_COLUMNS " FROM subscriber WHERE imei = ?",
+	[DB_STMT_UPD_VLR_BY_ID] = "UPDATE subscriber SET vlr_number = $number, vlr_via_proxy = $proxy WHERE id = $subscriber_id",
+	[DB_STMT_UPD_SGSN_BY_ID] = "UPDATE subscriber SET sgsn_number = $number, sgsn_via_proxy = $proxy WHERE id = $subscriber_id",
+	[DB_STMT_UPD_IMEI_BY_IMSI] = "UPDATE subscriber SET imei = $imei WHERE imsi = $imsi",
 	[DB_STMT_AUC_BY_IMSI] =
 		"SELECT id, algo_id_2g, ki, algo_id_3g, k, op, opc, sqn, ind_bitlen"
 		" FROM subscriber"
@@ -58,9 +77,10 @@ static const char *stmt_sql[] = {
 	[DB_STMT_UPD_PURGE_PS_BY_IMSI] = "UPDATE subscriber SET ms_purged_ps = $val WHERE imsi = $imsi",
 	[DB_STMT_UPD_NAM_CS_BY_IMSI] = "UPDATE subscriber SET nam_cs = $val WHERE imsi = $imsi",
 	[DB_STMT_UPD_NAM_PS_BY_IMSI] = "UPDATE subscriber SET nam_ps = $val WHERE imsi = $imsi",
-	[DB_STMT_SUBSCR_CREATE] = "INSERT INTO subscriber (imsi) VALUES ($imsi)",
+	[DB_STMT_SUBSCR_CREATE] = "INSERT INTO subscriber (imsi, nam_cs, nam_ps) VALUES ($imsi, $nam_cs, $nam_ps)",
 	[DB_STMT_DEL_BY_ID] = "DELETE FROM subscriber WHERE id = $subscriber_id",
 	[DB_STMT_SET_MSISDN_BY_IMSI] = "UPDATE subscriber SET msisdn = $msisdn WHERE imsi = $imsi",
+	[DB_STMT_DELETE_MSISDN_BY_IMSI] = "UPDATE subscriber SET msisdn = NULL WHERE imsi = $imsi",
 	[DB_STMT_AUC_2G_INSERT] =
 		"INSERT INTO auc_2g (subscriber_id, algo_id_2g, ki)"
 		" VALUES($subscriber_id, $algo_id_2g, $ki)",
@@ -69,6 +89,10 @@ static const char *stmt_sql[] = {
 		"INSERT INTO auc_3g (subscriber_id, algo_id_3g, k, op, opc, ind_bitlen)"
 		" VALUES($subscriber_id, $algo_id_3g, $k, $op, $opc, $ind_bitlen)",
 	[DB_STMT_AUC_3G_DELETE] = "DELETE FROM auc_3g WHERE subscriber_id = $subscriber_id",
+	[DB_STMT_SET_LAST_LU_SEEN] = "UPDATE subscriber SET last_lu_seen = datetime($val, 'unixepoch') WHERE id = $subscriber_id",
+	[DB_STMT_SET_LAST_LU_SEEN_PS] = "UPDATE subscriber SET last_lu_seen_ps = datetime($val, 'unixepoch') WHERE id = $subscriber_id",
+	[DB_STMT_EXISTS_BY_IMSI] = "SELECT 1 FROM subscriber WHERE imsi = $imsi",
+	[DB_STMT_EXISTS_BY_MSISDN] = "SELECT 1 FROM subscriber WHERE msisdn = $msisdn",
 };
 
 static void sql3_error_log_cb(void *arg, int err_code, const char *msg)
@@ -98,6 +122,8 @@ static void sql3_sql_log_cb(void *arg, sqlite3 *s3, const char *stmt, int type)
 void db_remove_reset(sqlite3_stmt *stmt)
 {
 	sqlite3_clear_bindings(stmt);
+	/* sqlite3_reset() just repeats an error code already evaluated during sqlite3_step(). */
+	/* coverity[CHECKED_RETURN] */
 	sqlite3_reset(stmt);
 }
 
@@ -167,29 +193,356 @@ bool db_bind_int64(sqlite3_stmt *stmt, const char *param_name, int64_t nr)
 	return true;
 }
 
+bool db_bind_null(sqlite3_stmt *stmt, const char *param_name)
+{
+	int rc;
+	int idx = param_name ? sqlite3_bind_parameter_index(stmt, param_name) : 1;
+	if (idx < 1) {
+		LOGP(DDB, LOGL_ERROR, "Error composing SQL, cannot bind parameter '%s'\n",
+		     param_name);
+		return false;
+	}
+	rc = sqlite3_bind_null(stmt, idx);
+	if (rc != SQLITE_OK) {
+		LOGP(DDB, LOGL_ERROR, "Error binding NULL to SQL parameter %s: %d\n",
+		     param_name ? param_name : "#1", rc);
+		db_remove_reset(stmt);
+		return false;
+	}
+	return true;
+}
+
 void db_close(struct db_context *dbc)
 {
 	unsigned int i;
+	int rc;
 
 	for (i = 0; i < ARRAY_SIZE(dbc->stmt); i++) {
 		/* it is ok to call finalize on NULL */
 		sqlite3_finalize(dbc->stmt[i]);
 	}
-	sqlite3_close(dbc->db);
+
+	/* Ask sqlite3 to close DB */
+	rc = sqlite3_close(dbc->db);
+	if (rc != SQLITE_OK) { /* Make sure it's actually closed! */
+		LOGP(DDB, LOGL_ERROR, "Couldn't close database: (rc=%d) %s\n",
+			rc, sqlite3_errmsg(dbc->db));
+	}
+
 	talloc_free(dbc);
 }
 
-struct db_context *db_open(void *ctx, const char *fname)
+static int db_run_statements(struct db_context *dbc, const char **statements, size_t statements_count)
+{
+	int rc;
+	int i;
+	for (i = 0; i < statements_count; i++) {
+		const char *stmt_str = statements[i];
+		sqlite3_stmt *stmt;
+
+		rc = sqlite3_prepare_v2(dbc->db, stmt_str, -1, &stmt, NULL);
+		if (rc != SQLITE_OK) {
+			LOGP(DDB, LOGL_ERROR, "Unable to prepare SQL statement '%s'\n", stmt_str);
+			return rc;
+		}
+		rc = sqlite3_step(stmt);
+		db_remove_reset(stmt);
+		sqlite3_finalize(stmt);
+		if (rc != SQLITE_DONE) {
+			LOGP(DDB, LOGL_ERROR, "SQL error: (%d) %s, during stmt '%s'",
+			     rc, sqlite3_errmsg(dbc->db), stmt_str);
+			return rc;
+		}
+	}
+	return rc;
+}
+
+static int db_bootstrap(struct db_context *dbc)
+{
+	int rc = db_run_statements(dbc, stmt_bootstrap_sql, ARRAY_SIZE(stmt_bootstrap_sql));
+	if (rc != SQLITE_DONE) {
+		LOGP(DDB, LOGL_ERROR, "Cannot bootstrap database\n");
+		return rc;
+	}
+	return SQLITE_OK;
+}
+
+/* https://www.sqlite.org/fileformat2.html#storage_of_the_sql_database_schema */
+static bool db_table_exists(struct db_context *dbc, const char *table_name)
+{
+	const char *table_exists_sql = "SELECT name FROM sqlite_master WHERE type='table' AND name=?";
+	sqlite3_stmt *stmt;
+	int rc;
+
+	rc = sqlite3_prepare_v2(dbc->db, table_exists_sql, -1, &stmt, NULL);
+	if (rc != SQLITE_OK) {
+		LOGP(DDB, LOGL_ERROR, "Unable to prepare SQL statement '%s'\n", table_exists_sql);
+		return false;
+	}
+
+	if (!db_bind_text(stmt, NULL, table_name))
+		return false;
+
+	rc = sqlite3_step(stmt);
+	db_remove_reset(stmt);
+	sqlite3_finalize(stmt);
+	return (rc == SQLITE_ROW);
+}
+
+/* Indicate whether the database is initialized with tables for schema version 0.
+ * We only check for the 'subscriber' table here because Neels said so. */
+static bool db_is_bootstrapped_v0(struct db_context *dbc)
+{
+	if (!db_table_exists(dbc, "subscriber")) {
+		LOGP(DDB, LOGL_DEBUG, "Table 'subscriber' not found in database '%s'\n", dbc->fname);
+		return false;
+	}
+
+	return true;
+}
+
+static int
+db_upgrade_v1(struct db_context *dbc)
+{
+	int rc;
+	const char *statements[] = {
+		"ALTER TABLE subscriber ADD COLUMN last_lu_seen TIMESTAMP default NULL",
+		"PRAGMA user_version = 1",
+	};
+
+	rc = db_run_statements(dbc, statements, ARRAY_SIZE(statements));
+	if (rc != SQLITE_DONE) {
+		LOGP(DDB, LOGL_ERROR, "Unable to update HLR database schema to version 1\n");
+		return rc;
+	}
+	return rc;
+}
+
+static int db_upgrade_v2(struct db_context *dbc)
+{
+	int rc;
+	const char *statements[] = {
+		"ALTER TABLE subscriber ADD COLUMN imei VARCHAR(14)",
+		"PRAGMA user_version = 2",
+	};
+
+	rc = db_run_statements(dbc, statements, ARRAY_SIZE(statements));
+	if (rc != SQLITE_DONE) {
+		LOGP(DDB, LOGL_ERROR, "Unable to update HLR database schema to version 2\n");
+		return rc;
+	}
+	return rc;
+}
+
+static int db_upgrade_v3(struct db_context *dbc)
+{
+	int rc;
+
+	/* A newer SQLite version would allow simply 'ATLER TABLE subscriber RENAME COLUMN hlr_number TO msc_number'.
+	 * This is a really expensive workaround for that in order to cover earlier SQLite versions as well:
+	 * Create a new table with the new column name and copy the data over (https://www.sqlite.org/faq.html#q11).
+	 */
+#define SUBSCR_V3_CREATE  \
+"(\n" \
+"-- OsmoHLR's DB scheme is modelled roughly after TS 23.008 version 13.3.0\n" \
+"	id		INTEGER PRIMARY KEY,\n" \
+"	-- Chapter 2.1.1.1\n" \
+"	imsi		VARCHAR(15) UNIQUE NOT NULL,\n" \
+"	-- Chapter 2.1.2\n" \
+"	msisdn		VARCHAR(15) UNIQUE,\n" \
+"	-- Chapter 2.2.3: Most recent / current IMEISV\n" \
+"	imeisv		VARCHAR,\n" \
+"	-- Chapter 2.1.9: Most recent / current IMEI\n" \
+"	imei		VARCHAR(14),\n" \
+"	-- Chapter 2.4.5\n" \
+"	vlr_number	VARCHAR(15),\n" \
+"	-- Chapter 2.4.6\n" \
+"	msc_number	VARCHAR(15),\n" \
+"	-- Chapter 2.4.8.1\n" \
+"	sgsn_number	VARCHAR(15),\n" \
+"	-- Chapter 2.13.10\n" \
+"	sgsn_address	VARCHAR,\n" \
+"	-- Chapter 2.4.8.2\n" \
+"	ggsn_number	VARCHAR(15),\n" \
+"	-- Chapter 2.4.9.2\n" \
+"	gmlc_number	VARCHAR(15),\n" \
+"	-- Chapter 2.4.23\n" \
+"	smsc_number	VARCHAR(15),\n" \
+"	-- Chapter 2.4.24\n" \
+"	periodic_lu_tmr	INTEGER,\n" \
+"	-- Chapter 2.13.115\n" \
+"	periodic_rau_tau_tmr INTEGER,\n" \
+"	-- Chapter 2.1.1.2: network access mode\n" \
+"	nam_cs		BOOLEAN NOT NULL DEFAULT 1,\n" \
+"	nam_ps		BOOLEAN NOT NULL DEFAULT 1,\n" \
+"	-- Chapter 2.1.8\n" \
+"	lmsi		INTEGER,\n" \
+ \
+"	-- The below purged flags might not even be stored non-volatile,\n" \
+"	-- refer to TS 23.012 Chapter 3.6.1.4\n" \
+"	-- Chapter 2.7.5\n" \
+"	ms_purged_cs	BOOLEAN NOT NULL DEFAULT 0,\n" \
+"	-- Chapter 2.7.6\n" \
+"	ms_purged_ps	BOOLEAN NOT NULL DEFAULT 0,\n" \
+ \
+"	-- Timestamp of last location update seen from subscriber\n" \
+"	-- The value is a string which encodes a UTC timestamp in granularity of seconds.\n" \
+"	last_lu_seen TIMESTAMP default NULL\n" \
+")\n"
+
+#define SUBSCR_V2_COLUMN_NAMES \
+	"id," \
+	"imsi," \
+	"msisdn," \
+	"imeisv," \
+	"imei," \
+	"vlr_number," \
+	"hlr_number," \
+	"sgsn_number," \
+	"sgsn_address," \
+	"ggsn_number," \
+	"gmlc_number," \
+	"smsc_number," \
+	"periodic_lu_tmr," \
+	"periodic_rau_tau_tmr," \
+	"nam_cs," \
+	"nam_ps," \
+	"lmsi," \
+	"ms_purged_cs," \
+	"ms_purged_ps," \
+	"last_lu_seen"
+
+#define SUBSCR_V3_COLUMN_NAMES \
+	"id," \
+	"imsi," \
+	"msisdn," \
+	"imeisv," \
+	"imei," \
+	"vlr_number," \
+	"msc_number," \
+	"sgsn_number," \
+	"sgsn_address," \
+	"ggsn_number," \
+	"gmlc_number," \
+	"smsc_number," \
+	"periodic_lu_tmr," \
+	"periodic_rau_tau_tmr," \
+	"nam_cs," \
+	"nam_ps," \
+	"lmsi," \
+	"ms_purged_cs," \
+	"ms_purged_ps," \
+	"last_lu_seen"
+
+	const char *statements[] = {
+		"BEGIN TRANSACTION",
+		"CREATE TEMPORARY TABLE subscriber_backup" SUBSCR_V3_CREATE,
+		"INSERT INTO subscriber_backup SELECT " SUBSCR_V2_COLUMN_NAMES " FROM subscriber",
+		"DROP TABLE subscriber",
+		"CREATE TABLE subscriber" SUBSCR_V3_CREATE,
+		"INSERT INTO subscriber SELECT " SUBSCR_V3_COLUMN_NAMES " FROM subscriber_backup",
+		"DROP TABLE subscriber_backup",
+		"COMMIT",
+		"PRAGMA user_version = 3",
+	};
+
+	rc = db_run_statements(dbc, statements, ARRAY_SIZE(statements));
+	if (rc != SQLITE_DONE) {
+		LOGP(DDB, LOGL_ERROR, "Unable to update HLR database schema to version 3\n");
+		return rc;
+	}
+	return rc;
+}
+
+static int db_upgrade_v4(struct db_context *dbc)
+{
+	int rc;
+	const char *statements[] = {
+		"ALTER TABLE subscriber ADD COLUMN last_lu_seen_ps TIMESTAMP default NULL",
+		"PRAGMA user_version = 4",
+	};
+
+	rc = db_run_statements(dbc, statements, ARRAY_SIZE(statements));
+	if (rc != SQLITE_DONE) {
+		LOGP(DDB, LOGL_ERROR, "Unable to update HLR database schema to version 4\n");
+		return rc;
+	}
+	return rc;
+}
+
+static int db_upgrade_v5(struct db_context *dbc)
+{
+	int rc;
+	const char *statements[] = {
+		"ALTER TABLE subscriber ADD COLUMN vlr_via_proxy VARCHAR",
+		"ALTER TABLE subscriber ADD COLUMN sgsn_via_proxy VARCHAR",
+		"PRAGMA user_version = 5",
+	};
+
+	rc = db_run_statements(dbc, statements, ARRAY_SIZE(statements));
+	if (rc != SQLITE_DONE) {
+		LOGP(DDB, LOGL_ERROR, "Unable to update HLR database schema to version 5\n");
+		return rc;
+	}
+	return rc;
+}
+
+typedef int (*db_upgrade_func_t)(struct db_context *dbc);
+static db_upgrade_func_t db_upgrade_path[] = {
+	db_upgrade_v1,
+	db_upgrade_v2,
+	db_upgrade_v3,
+	db_upgrade_v4,
+	db_upgrade_v5,
+};
+
+static int db_get_user_version(struct db_context *dbc)
+{
+	const char *user_version_sql = "PRAGMA user_version";
+	sqlite3_stmt *stmt;
+	int version, rc;
+
+	rc = sqlite3_prepare_v2(dbc->db, user_version_sql, -1, &stmt, NULL);
+	if (rc != SQLITE_OK) {
+		LOGP(DDB, LOGL_ERROR, "Unable to prepare SQL statement '%s'\n", user_version_sql);
+		return -1;
+	}
+	rc = sqlite3_step(stmt);
+	if (rc == SQLITE_ROW) {
+		version = sqlite3_column_int(stmt, 0);
+	} else {
+		LOGP(DDB, LOGL_ERROR, "SQL statement '%s' failed: %d\n", user_version_sql, rc);
+		version = -1;
+	}
+
+	db_remove_reset(stmt);
+	sqlite3_finalize(stmt);
+	return version;
+}
+
+struct db_context *db_open(void *ctx, const char *fname, bool enable_sqlite_logging, bool allow_upgrade)
 {
 	struct db_context *dbc = talloc_zero(ctx, struct db_context);
 	unsigned int i;
 	int rc;
 	bool has_sqlite_config_sqllog = false;
+	int version;
 
 	LOGP(DDB, LOGL_NOTICE, "using database: %s\n", fname);
 	LOGP(DDB, LOGL_INFO, "Compiled against SQLite3 lib version %s\n", SQLITE_VERSION);
 	LOGP(DDB, LOGL_INFO, "Running with SQLite3 lib version %s\n", sqlite3_libversion());
 
+#ifdef SQLITE_USE_TALLOC
+	/* Configure SQLite3 to use talloc memory allocator */
+	rc = db_sqlite3_use_talloc(ctx);
+	if (rc == SQLITE_OK) {
+		LOGP(DDB, LOGL_NOTICE, "SQLite3 is configured to use talloc\n");
+	} else {
+		LOGP(DDB, LOGL_ERROR, "Failed to configure SQLite3 "
+		     "to use talloc, using default memory allocator\n");
+	}
+#endif
+
 	dbc->fname = talloc_strdup(dbc, fname);
 
 	for (i = 0; i < 0xfffff; i++) {
@@ -201,9 +554,11 @@ struct db_context *db_open(void *ctx, const char *fname)
 			has_sqlite_config_sqllog = true;
 	}
 
-	rc = sqlite3_config(SQLITE_CONFIG_LOG, sql3_error_log_cb, NULL);
-	if (rc != SQLITE_OK)
-		LOGP(DDB, LOGL_NOTICE, "Unable to set SQLite3 error log callback\n");
+	if (enable_sqlite_logging) {
+		rc = sqlite3_config(SQLITE_CONFIG_LOG, sql3_error_log_cb, NULL);
+		if (rc != SQLITE_OK)
+			LOGP(DDB, LOGL_NOTICE, "Unable to set SQLite3 error log callback\n");
+	}
 
 	if (has_sqlite_config_sqllog) {
 		rc = sqlite3_config(SQLITE_CONFIG_SQLLOG, sql3_sql_log_cb, NULL);
@@ -227,9 +582,57 @@ struct db_context *db_open(void *ctx, const char *fname)
 
 	char *err_msg;
 	rc = sqlite3_exec(dbc->db, "PRAGMA journal_mode=WAL; PRAGMA synchonous = NORMAL;", 0, 0, &err_msg);
-	if (rc != SQLITE_OK)
+	if (rc != SQLITE_OK) {
 		LOGP(DDB, LOGL_ERROR, "Unable to set Write-Ahead Logging: %s\n",
 			err_msg);
+		sqlite3_free(err_msg);
+	}
+
+	version = db_get_user_version(dbc);
+	if (version < 0) {
+		LOGP(DDB, LOGL_ERROR, "Unable to read user version number from database '%s'\n", dbc->fname);
+		goto out_free;
+	}
+
+	/* An empty database will always report version zero. */
+	if (version == 0 && !db_is_bootstrapped_v0(dbc)) {
+		LOGP(DDB, LOGL_NOTICE, "Missing database tables detected; Bootstrapping database '%s'\n", dbc->fname);
+		rc = db_bootstrap(dbc);
+		if (rc != SQLITE_OK) {
+			LOGP(DDB, LOGL_ERROR, "Failed to bootstrap DB: (rc=%d) %s\n",
+			     rc, sqlite3_errmsg(dbc->db));
+			goto out_free;
+		}
+		version = CURRENT_SCHEMA_VERSION;
+	}
+
+	LOGP(DDB, LOGL_NOTICE, "Database '%s' has HLR DB schema version %d\n", dbc->fname, version);
+
+	for (; allow_upgrade && (version < ARRAY_SIZE(db_upgrade_path)); version++) {
+		db_upgrade_func_t upgrade_func = db_upgrade_path[version];
+		rc = upgrade_func(dbc);
+		if (rc != SQLITE_DONE) {
+			LOGP(DDB, LOGL_ERROR, "Failed to upgrade HLR DB schema to version %d: (rc=%d) %s\n",
+			     version+1, rc, sqlite3_errmsg(dbc->db));
+			goto out_free;
+		}
+		LOGP(DDB, LOGL_NOTICE, "Database '%s' has been upgraded to HLR DB schema version %d\n",
+		     dbc->fname, version+1);
+	}
+
+	if (version != CURRENT_SCHEMA_VERSION) {
+		if (version < CURRENT_SCHEMA_VERSION) {
+			LOGP(DDB, LOGL_NOTICE, "HLR DB schema version %d is outdated\n", version);
+			if (!allow_upgrade) {
+				LOGP(DDB, LOGL_ERROR, "Not upgrading HLR database to schema version %d; "
+				     "use the --db-upgrade option to allow HLR database upgrades\n",
+				     CURRENT_SCHEMA_VERSION);
+			}
+		} else
+			LOGP(DDB, LOGL_ERROR, "HLR DB schema version %d is unknown\n", version);
+
+		goto out_free;
+	}
 
 	/* prepare all SQL statements */
 	for (i = 0; i < ARRAY_SIZE(dbc->stmt); i++) {

src/db_auc.c

@@ -26,10 +26,10 @@
 
 #include <sqlite3.h>
 
-#include "logging.h"
-#include "db.h"
-#include "auc.h"
-#include "rand.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/db.h>
+#include <osmocom/hlr/auc.h>
+#include <osmocom/hlr/rand.h>
 
 #define LOGAUC(imsi, level, fmt, args ...)	LOGP(DAUC, level, "IMSI='%s': " fmt, imsi, ## args)
 
@@ -49,7 +49,7 @@ int db_update_sqn(struct db_context *dbc, int64_t subscr_id, uint64_t new_sqn)
 	/* execute the statement */
 	rc = sqlite3_step(stmt);
 	if (rc != SQLITE_DONE) {
-		LOGP(DAUC, LOGL_ERROR, "Cannot update SQN for subscriber ID=%"PRId64
+		LOGP(DAUC, LOGL_ERROR, "Cannot update SQN for subscriber ID=%" PRId64
 		     ": SQL error: (%d) %s\n",
 		     subscr_id, rc, sqlite3_errmsg(dbc->db));
 		ret = -EIO;
@@ -59,11 +59,11 @@ int db_update_sqn(struct db_context *dbc, int64_t subscr_id, uint64_t new_sqn)
 	/* verify execution result */
 	rc = sqlite3_changes(dbc->db);
 	if (!rc) {
-		LOGP(DAUC, LOGL_ERROR, "Cannot update SQN for subscriber ID=%"PRId64
+		LOGP(DAUC, LOGL_ERROR, "Cannot update SQN for subscriber ID=%" PRId64
 		     ": no auc_3g entry for such subscriber\n", subscr_id);
 		ret = -ENOENT;
 	} else if (rc != 1) {
-		LOGP(DAUC, LOGL_ERROR, "Update SQN for subscriber ID=%"PRId64
+		LOGP(DAUC, LOGL_ERROR, "Update SQN for subscriber ID=%" PRId64
 		     ": SQL modified %d rows (expected 1)\n", subscr_id, rc);
 		ret = -EIO;
 	}
@@ -73,8 +73,36 @@ out:
 	return ret;
 }
 
+/* hexparse a specific column of a sqlite prepared statement into dst (with length check)
+ * returns 0 for success, -EIO on error */
+static int hexparse_stmt(uint8_t *dst, size_t dst_len, sqlite3_stmt *stmt, int col, const char *col_name,
+			 const char *imsi)
+{
+	const uint8_t *text;
+	size_t col_len;
+
+	/* Bytes are stored as hex strings in database, hence divide length by two */
+	col_len = sqlite3_column_bytes(stmt, col) / 2;
+
+	if (col_len != dst_len) {
+		LOGAUC(imsi, LOGL_ERROR, "Error reading %s, expected length %lu but has length %lu\n", col_name,
+		       dst_len, col_len);
+		return -EIO;
+	}
+
+	text = sqlite3_column_text(stmt, col);
+	if (!text) {
+		LOGAUC(imsi, LOGL_ERROR, "Error reading %s\n", col_name);
+		return -EIO;
+	}
+	osmo_hexparse((void *)text, dst, dst_len);
+	return 0;
+}
+
 /* obtain the authentication data for a given imsi
- * returns -1 in case of error, 0 for unknown IMSI, 1 for success */
+ * returns 0 for success, negative value on error:
+ * -ENOENT if the IMSI is not known, -ENOKEY if the IMSI is known but has no auth data,
+ * -EIO on db failure */
 int db_get_auth_data(struct db_context *dbc, const char *imsi,
 		     struct osmo_sub_auth_data *aud2g,
 		     struct osmo_sub_auth_data *aud3g,
@@ -111,49 +139,34 @@ int db_get_auth_data(struct db_context *dbc, const char *imsi,
 	/* obtain result values using sqlite3_column_*() */
 	if (sqlite3_column_type(stmt, 1) == SQLITE_INTEGER) {
 		/* we do have some 2G authentication data */
-		const uint8_t *ki;
-
-		aud2g->algo = sqlite3_column_int(stmt, 1);
-		ki = sqlite3_column_text(stmt, 2);
-#if 0
-		if (sqlite3_column_bytes(stmt, 2) != sizeof(aud2g->u.gsm.ki)) {
-			LOGAUC(imsi, LOGL_ERROR, "Error reading Ki: %d\n", rc);
+		if (hexparse_stmt(aud2g->u.gsm.ki, sizeof(aud2g->u.gsm.ki), stmt, 2, "Ki", imsi))
 			goto end_2g;
-		}
-#endif
-		osmo_hexparse((void*)ki, (void*)&aud2g->u.gsm.ki, sizeof(aud2g->u.gsm.ki));
+		aud2g->algo = sqlite3_column_int(stmt, 1);
 		aud2g->type = OSMO_AUTH_TYPE_GSM;
 	} else
 		LOGAUC(imsi, LOGL_DEBUG, "No 2G Auth Data\n");
-//end_2g:
+end_2g:
 	if (sqlite3_column_type(stmt, 3) == SQLITE_INTEGER) {
 		/* we do have some 3G authentication data */
-		const uint8_t *k, *op, *opc;
-
-		aud3g->algo = sqlite3_column_int(stmt, 3);
-		k = sqlite3_column_text(stmt, 4);
-		if (!k) {
-			LOGAUC(imsi, LOGL_ERROR, "Error reading K: %d\n", rc);
+		if (hexparse_stmt(aud3g->u.umts.k, sizeof(aud3g->u.umts.k), stmt, 4, "K", imsi)) {
 			ret = -EIO;
 			goto out;
 		}
-		osmo_hexparse((void*)k, (void*)&aud3g->u.umts.k, sizeof(aud3g->u.umts.k));
+		aud3g->algo = sqlite3_column_int(stmt, 3);
+
 		/* UMTS Subscribers can have either OP or OPC */
-		op = sqlite3_column_text(stmt, 5);
-		if (!op) {
-			opc = sqlite3_column_text(stmt, 6);
-			if (!opc) {
-				LOGAUC(imsi, LOGL_ERROR, "Error reading OPC: %d\n", rc);
+		if (sqlite3_column_text(stmt, 5)) {
+			if (hexparse_stmt(aud3g->u.umts.opc, sizeof(aud3g->u.umts.opc), stmt, 5, "OP", imsi)) {
 				ret = -EIO;
 				goto out;
 			}
-			osmo_hexparse((void*)opc, (void*)&aud3g->u.umts.opc,
-					sizeof(aud3g->u.umts.opc));
-			aud3g->u.umts.opc_is_op = 0;
-		} else {
-			osmo_hexparse((void*)op, (void*)&aud3g->u.umts.opc,
-					sizeof(aud3g->u.umts.opc));
 			aud3g->u.umts.opc_is_op = 1;
+		} else {
+			if (hexparse_stmt(aud3g->u.umts.opc, sizeof(aud3g->u.umts.opc), stmt, 6, "OPC", imsi)) {
+				ret = -EIO;
+				goto out;
+			}
+			aud3g->u.umts.opc_is_op = 0;
 		}
 		aud3g->u.umts.sqn = sqlite3_column_int64(stmt, 7);
 		aud3g->u.umts.ind_bitlen = sqlite3_column_int(stmt, 8);
@@ -163,19 +176,20 @@ int db_get_auth_data(struct db_context *dbc, const char *imsi,
 		LOGAUC(imsi, LOGL_DEBUG, "No 3G Auth Data\n");
 
 	if (aud2g->type == 0 && aud3g->type == 0)
-		ret = -ENOENT;
+		ret = -ENOKEY;
 
 out:
 	db_remove_reset(stmt);
 	return ret;
 }
 
-/* return -1 in case of error, 0 for unknown imsi, positive for number
- * of vectors generated */
+/* return number of vectors generated, negative value on error:
+ * -ENOENT if the IMSI is not known, -ENOKEY if the IMSI is known but has no auth data,
+ * -EIO on db failure */
 int db_get_auc(struct db_context *dbc, const char *imsi,
 	       unsigned int auc_3g_ind, struct osmo_auth_vector *vec,
 	       unsigned int num_vec, const uint8_t *rand_auts,
-	       const uint8_t *auts)
+	       const uint8_t *auts, bool separation_bit)
 {
 	struct osmo_sub_auth_data aud2g, aud3g;
 	int64_t subscr_id;
@@ -186,15 +200,22 @@ int db_get_auc(struct db_context *dbc, const char *imsi,
 	if (rc)
 		return rc;
 
+	/* modulo by the IND bitlen value range. For example, ind_bitlen == 5 would modulo 32:
+	 * 1 << 5 == 0b0100000 == 32
+	 * - 1 == 0b0011111 == bitmask of 5 lowest bits
+	 * x &= 0b0011111 == modulo 32
+	 * Why do this? osmo-hlr cannot possibly choose individual VLR INDs always matching all subscribers' IND_bitlen,
+	 * which might vary wildly. Instead, let hlr.c pass in an arbitrarily high number here, and the modulo does a
+	 * round-robin if the IND pools that this subscriber has available. */
+	auc_3g_ind &= (1U << aud3g.u.umts.ind_bitlen) - 1;
 	aud3g.u.umts.ind = auc_3g_ind;
-	if (aud3g.type == OSMO_AUTH_TYPE_UMTS
-	    && aud3g.u.umts.ind >= (1U << aud3g.u.umts.ind_bitlen)) {
-		LOGAUC(imsi, LOGL_NOTICE, "3G auth: SQN's IND bitlen %u is"
-		       " too small to hold an index of %u. Truncating. This"
-		       " may cause numerous additional AUTS resyncing.\n",
-		       aud3g.u.umts.ind_bitlen, aud3g.u.umts.ind);
-		aud3g.u.umts.ind &= (1U << aud3g.u.umts.ind_bitlen) - 1;
-	}
+
+	/* the first bit (bit0) cannot be used as AMF anymore, but has been
+	 * re-appropriated as the separation bit.  See 3GPP TS 33.102 Annex H
+	 * together with 3GPP TS 33.401 / 33.402 / 33.501 */
+	aud3g.u.umts.amf[0] = aud3g.u.umts.amf[0] & 0x7f;
+	if (separation_bit)
+		aud3g.u.umts.amf[0] |= 0x80;
 
 	LOGAUC(imsi, LOGL_DEBUG, "Calling to generate %u vectors\n", num_vec);
 	rc = auc_compute_vectors(vec, num_vec, &aud2g, &aud3g, rand_auts, auts);

src/db_debug.c

@@ -0,0 +1,86 @@
+/*
+ * libtalloc based memory allocator for SQLite3.
+ *
+ * (C) 2019 by Vadim Yanitskiy <axilirator@gmail.com>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include <sqlite3.h>
+#include <talloc.h>
+#include <errno.h>
+
+/* Dedicated talloc context for SQLite */
+static void *db_sqlite_ctx = NULL;
+
+static void *tall_xMalloc(int size)
+{
+	return talloc_size(db_sqlite_ctx, size);
+}
+
+static void tall_xFree(void *ptr)
+{
+	talloc_free(ptr);
+}
+
+static void *tall_xRealloc(void *ptr, int size)
+{
+	return talloc_realloc_fn(db_sqlite_ctx, ptr, size);
+}
+
+static int tall_xSize(void *ptr)
+{
+	return talloc_total_size(ptr);
+}
+
+/* DUMMY: talloc doesn't round up the allocation size */
+static int tall_xRoundup(int size) { return size; }
+
+/* DUMMY: nothing to initialize */
+static int tall_xInit(void *data) { return 0; }
+
+/* DUMMY: nothing to deinitialize */
+static void tall_xShutdown(void *data) {  }
+
+/* Interface between SQLite and talloc memory allocator */
+static const struct sqlite3_mem_methods tall_sqlite_if = {
+	/* Memory allocation function */
+	.xMalloc = &tall_xMalloc,
+	/* Free a prior allocation */
+	.xFree = &tall_xFree,
+	/* Resize an allocation */
+	.xRealloc = &tall_xRealloc,
+	/* Return the size of an allocation */
+	.xSize = &tall_xSize,
+	/* Round up request size to allocation size */
+	.xRoundup = &tall_xRoundup,
+	/* Initialize the memory allocator */
+	.xInit = &tall_xInit,
+	/* Deinitialize the memory allocator */
+	.xShutdown = &tall_xShutdown,
+	/* Argument to xInit() and xShutdown() */
+	.pAppData = NULL,
+};
+
+int db_sqlite3_use_talloc(void *ctx)
+{
+	if (db_sqlite_ctx != NULL)
+		return -EEXIST;
+
+	db_sqlite_ctx = talloc_named_const(ctx, 0, "SQLite3");
+	return sqlite3_config(SQLITE_CONFIG_MALLOC, &tall_sqlite_if);
+}

src/db_hlr.c

@@ -17,33 +17,37 @@
  *
  */
 
+#define _POSIX_C_SOURCE 200809L /* for strptime(3) */
+/* These are needed as well due to the above _POSIX_C_SOURCE definition: */
+#define _DEFAULT_SOURCE		/* for struct timezone */
+#define _XOPEN_SOURCE		/* for clockid_t */
+
 #include <string.h>
 #include <errno.h>
 #include <inttypes.h>
+#include <time.h>
 
 #include <osmocom/core/utils.h>
+#include <osmocom/core/timer.h>
 #include <osmocom/crypt/auth.h>
 #include <osmocom/gsm/gsm23003.h>
 
 #include <sqlite3.h>
 
-#include "logging.h"
-#include "hlr.h"
-#include "db.h"
-#include "gsup_server.h"
-#include "luop.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/hlr.h>
+#include <osmocom/hlr/db.h>
+#include <osmocom/gsupclient/cni_peer_id.h>
 
 #define LOGHLR(imsi, level, fmt, args ...)	LOGP(DAUC, level, "IMSI='%s': " fmt, imsi, ## args)
 
-#define SL3_TXT(x, stmt, idx) 					\
-	do {							\
-		const char *_txt = (const char *) sqlite3_column_text(stmt, idx);\
-		if (_txt)					\
-			strncpy(x, _txt, sizeof(x));		\
-		x[sizeof(x)-1] = '\0';				\
-	} while (0)
-
-int db_subscr_create(struct db_context *dbc, const char *imsi)
+/*! Add new subscriber record to the HLR database.
+ * \param[in,out] dbc  database context.
+ * \param[in] imsi  ASCII string of IMSI digits, is validated.
+ * \param[in] flags  Bitmask of DB_SUBSCR_FLAG_*.
+ * \returns 0 on success, -EINVAL on invalid IMSI, -EIO on database error.
+ */
+int db_subscr_create(struct db_context *dbc, const char *imsi, uint8_t flags)
 {
 	sqlite3_stmt *stmt;
 	int rc;
@@ -58,6 +62,10 @@ int db_subscr_create(struct db_context *dbc, const char *imsi)
 
 	if (!db_bind_text(stmt, "$imsi", imsi))
 		return -EIO;
+	if (!db_bind_int(stmt, "$nam_cs", (flags & DB_SUBSCR_FLAG_NAM_CS) != 0))
+		return -EIO;
+	if (!db_bind_int(stmt, "$nam_ps", (flags & DB_SUBSCR_FLAG_NAM_PS) != 0))
+		return -EIO;
 
 	/* execute the statement */
 	rc = sqlite3_step(stmt);
@@ -71,6 +79,15 @@ int db_subscr_create(struct db_context *dbc, const char *imsi)
 	return 0;
 }
 
+/*! Completely delete a subscriber record from the HLR database.
+ * Also remove authentication data.
+ * Future todo: also drop from all other database tables, which aren't used yet
+ * at the time of writing this.
+ * \param[in,out] dbc  database context.
+ * \param[in] subscr_id  ID of the subscriber in the HLR db.
+ * \returns if the subscriber was found and removed, -EIO on database error,
+ *          -ENOENT if no such subscriber data exists.
+ */
 int db_subscr_delete_by_id(struct db_context *dbc, int64_t subscr_id)
 {
 	int rc;
@@ -86,7 +103,7 @@ int db_subscr_delete_by_id(struct db_context *dbc, int64_t subscr_id)
 	rc = sqlite3_step(stmt);
 	if (rc != SQLITE_DONE) {
 		LOGP(DAUC, LOGL_ERROR,
-		       "Cannot delete subscriber ID=%"PRId64": SQL error: (%d) %s\n",
+		       "Cannot delete subscriber ID=%" PRId64 ": SQL error: (%d) %s\n",
 		       subscr_id, rc, sqlite3_errmsg(dbc->db));
 		db_remove_reset(stmt);
 		return -EIO;
@@ -95,11 +112,11 @@ int db_subscr_delete_by_id(struct db_context *dbc, int64_t subscr_id)
 	/* verify execution result */
 	rc = sqlite3_changes(dbc->db);
 	if (!rc) {
-		LOGP(DAUC, LOGL_ERROR, "Cannot delete: no such subscriber: ID=%"PRId64"\n",
+		LOGP(DAUC, LOGL_ERROR, "Cannot delete: no such subscriber: ID=%" PRId64 "\n",
 		     subscr_id);
 		ret = -ENOENT;
 	} else if (rc != 1) {
-		LOGP(DAUC, LOGL_ERROR, "Delete subscriber ID=%"PRId64
+		LOGP(DAUC, LOGL_ERROR, "Delete subscriber ID=%" PRId64
 		     ": SQL modified %d rows (expected 1)\n", subscr_id, rc);
 		ret = -EIO;
 	}
@@ -127,25 +144,35 @@ int db_subscr_delete_by_id(struct db_context *dbc, int64_t subscr_id)
 	return ret;
 }
 
+/*! Set a subscriber's MSISDN in the HLR database.
+ * \param[in,out] dbc  database context.
+ * \param[in] imsi  ASCII string of IMSI digits
+ * \param[in] msisdn  ASCII string of MSISDN digits, or NULL to remove the MSISDN.
+ * \returns 0 on success, -EINVAL in case of invalid MSISDN string, -EIO on
+ *          database failure, -ENOENT if no such subscriber exists.
+ */
 int db_subscr_update_msisdn_by_imsi(struct db_context *dbc, const char *imsi,
 				    const char *msisdn)
 {
 	int rc;
 	int ret = 0;
 
-	if (!osmo_msisdn_str_valid(msisdn)) {
+	if (msisdn && !osmo_msisdn_str_valid(msisdn)) {
 		LOGHLR(imsi, LOGL_ERROR,
 		       "Cannot update subscriber: invalid MSISDN: '%s'\n",
 		       msisdn);
 		return -EINVAL;
 	}
 
-	sqlite3_stmt *stmt = dbc->stmt[DB_STMT_SET_MSISDN_BY_IMSI];
+	sqlite3_stmt *stmt = dbc->stmt[
+		msisdn ? DB_STMT_SET_MSISDN_BY_IMSI : DB_STMT_DELETE_MSISDN_BY_IMSI];
 
 	if (!db_bind_text(stmt, "$imsi", imsi))
 		return -EIO;
-	if (!db_bind_text(stmt, "$msisdn", msisdn))
-		return -EIO;
+	if (msisdn) {
+		if (!db_bind_text(stmt, "$msisdn", msisdn))
+			return -EIO;
+	}
 
 	/* execute the statement */
 	rc = sqlite3_step(stmt);
@@ -175,14 +202,17 @@ out:
 
 }
 
-/* Insert or update 2G or 3G authentication tokens in the database.
+/*! Insert or update 2G or 3G authentication tokens in the database.
  * If aud->type is OSMO_AUTH_TYPE_GSM, the auc_2g table entry for the
  * subscriber will be added or modified; if aud->algo is OSMO_AUTH_ALG_NONE,
  * however, the auc_2g entry for the subscriber is deleted. If aud->type is
  * OSMO_AUTH_TYPE_UMTS, the auc_3g table is updated; again, if aud->algo is
  * OSMO_AUTH_ALG_NONE, the auc_3g entry is deleted.
- * Returns 0 if successful, -EINVAL for unknown aud->type, -ENOENT for unknown
- * subscr_id, -EIO for SQL errors.
+ * \param[in,out] dbc  database context.
+ * \param[in] subscr_id  DB ID of the subscriber.
+ * \param[in] aud  Pointer to new auth data (in ASCII string form).
+ * \returns 0 on success, -EINVAL for invalid aud, -ENOENT for unknown
+ *          subscr_id, -EIO for database errors.
  */
 int db_subscr_update_aud_by_id(struct db_context *dbc, int64_t subscr_id,
 			       const struct sub_auth_data_str *aud)
@@ -234,11 +264,11 @@ int db_subscr_update_aud_by_id(struct db_context *dbc, int64_t subscr_id,
 		switch (aud->algo) {
 		case OSMO_AUTH_ALG_NONE:
 		case OSMO_AUTH_ALG_MILENAGE:
+		case OSMO_AUTH_ALG_XOR:
 			break;
 		case OSMO_AUTH_ALG_COMP128v1:
 		case OSMO_AUTH_ALG_COMP128v2:
 		case OSMO_AUTH_ALG_COMP128v3:
-		case OSMO_AUTH_ALG_XOR:
 			LOGP(DAUC, LOGL_ERROR, "Cannot update auth tokens:"
 			     " auth algo not suited for 3G: %s\n",
 			     osmo_auth_alg_name(aud->algo));
@@ -296,7 +326,7 @@ int db_subscr_update_aud_by_id(struct db_context *dbc, int64_t subscr_id,
 		 * empty, and no entry is not an error then.*/
 		ret = -ENOENT;
 	else if (rc != 1) {
-		LOGP(DAUC, LOGL_ERROR, "Delete subscriber ID=%"PRId64
+		LOGP(DAUC, LOGL_ERROR, "Delete subscriber ID=%" PRId64
 		     " from %s: SQL modified %d rows (expected 1)\n",
 		     subscr_id, label, rc);
 		ret = -EIO;
@@ -361,6 +391,77 @@ out:
 	return ret;
 }
 
+/*! Set a subscriber's IMEI in the HLR database.
+ * \param[in,out] dbc  database context.
+ * \param[in] imsi  ASCII string of IMSI digits
+ * \param[in] imei  ASCII string of identifier digits, or NULL to remove the IMEI.
+ * \returns 0 on success, -ENOENT when the given subscriber does not exist,
+ *         -EIO on database errors.
+ */
+int db_subscr_update_imei_by_imsi(struct db_context *dbc, const char* imsi, const char *imei)
+{
+	int rc, ret = 0;
+	sqlite3_stmt *stmt = dbc->stmt[DB_STMT_UPD_IMEI_BY_IMSI];
+
+	if (imei && !osmo_imei_str_valid(imei, false)) {
+		LOGP(DAUC, LOGL_ERROR, "Cannot update subscriber IMSI='%s': invalid IMEI: '%s'\n", imsi, imei);
+		return -EINVAL;
+	}
+
+	if (!db_bind_text(stmt, "$imsi", imsi))
+		return -EIO;
+	if (imei && !db_bind_text(stmt, "$imei", imei))
+		return -EIO;
+
+	/* execute the statement */
+	rc = sqlite3_step(stmt);
+	if (rc != SQLITE_DONE) {
+		LOGP(DAUC, LOGL_ERROR, "Update IMEI for subscriber IMSI='%s': SQL Error: %s\n", imsi,
+		     sqlite3_errmsg(dbc->db));
+		ret = -EIO;
+		goto out;
+	}
+
+	/* verify execution result */
+	rc = sqlite3_changes(dbc->db);
+	if (!rc) {
+		LOGP(DAUC, LOGL_ERROR, "Cannot update IMEI for subscriber IMSI='%s': no such subscriber\n", imsi);
+		ret = -ENOENT;
+	} else if (rc != 1) {
+		LOGP(DAUC, LOGL_ERROR, "Update IMEI for subscriber IMSI='%s': SQL modified %d rows (expected 1)\n",
+		     imsi, rc);
+		ret = -EIO;
+	}
+
+out:
+	db_remove_reset(stmt);
+	return ret;
+}
+
+static void parse_last_lu_seen(time_t *dst, const char *last_lu_seen_str, const char *imsi, const char *label)
+{
+	struct tm tm = {0};
+	time_t val;
+	if (!last_lu_seen_str || last_lu_seen_str[0] == '\0')
+		return;
+
+	if (strptime(last_lu_seen_str, DB_LAST_LU_SEEN_FMT, &tm) == NULL) {
+		LOGP(DAUC, LOGL_ERROR, "IMSI-%s: Last LU Seen %s: Cannot parse timestamp '%s'\n",
+		     imsi, label, last_lu_seen_str);
+		return;
+	}
+
+	errno = 0;
+	val = mktime(&tm);
+	if (val == -1) {
+		LOGP(DAUC, LOGL_ERROR, "IMSI-%s: Last LU Seen %s: Cannot convert timestamp '%s' to time_t: %s\n",
+		     imsi, label, last_lu_seen_str, strerror(errno));
+		val = 0;
+	}
+
+	*dst = val;
+}
+
 /* Common code for db_subscr_get_by_*() functions. */
 static int db_sel(struct db_context *dbc, sqlite3_stmt *stmt, struct hlr_subscriber *subscr,
 		  const char **err)
@@ -386,19 +487,26 @@ static int db_sel(struct db_context *dbc, sqlite3_stmt *stmt, struct hlr_subscri
 
 	/* obtain the various columns */
 	subscr->id = sqlite3_column_int64(stmt, 0);
-	SL3_TXT(subscr->imsi, stmt, 1);
-	SL3_TXT(subscr->msisdn, stmt, 2);
+	copy_sqlite3_text_to_buf(subscr->imsi, stmt, 1);
+	copy_sqlite3_text_to_buf(subscr->msisdn, stmt, 2);
+	copy_sqlite3_text_to_buf(subscr->imei, stmt, 3);
 	/* FIXME: These should all be BLOBs as they might contain NUL */
-	SL3_TXT(subscr->vlr_number, stmt, 3);
-	SL3_TXT(subscr->sgsn_number, stmt, 4);
-	SL3_TXT(subscr->sgsn_address, stmt, 5);
-	subscr->periodic_lu_timer = sqlite3_column_int(stmt, 6);
-	subscr->periodic_rau_tau_timer = sqlite3_column_int(stmt, 7);
-	subscr->nam_cs = sqlite3_column_int(stmt, 8);
-	subscr->nam_ps = sqlite3_column_int(stmt, 9);
-	subscr->lmsi = sqlite3_column_int(stmt, 10);
-	subscr->ms_purged_cs = sqlite3_column_int(stmt, 11);
-	subscr->ms_purged_ps = sqlite3_column_int(stmt, 12);
+	copy_sqlite3_text_to_buf(subscr->vlr_number, stmt, 4);
+	copy_sqlite3_text_to_buf(subscr->sgsn_number, stmt, 5);
+	copy_sqlite3_text_to_buf(subscr->sgsn_address, stmt, 6);
+	subscr->periodic_lu_timer = sqlite3_column_int(stmt, 7);
+	subscr->periodic_rau_tau_timer = sqlite3_column_int(stmt, 8);
+	subscr->nam_cs = sqlite3_column_int(stmt, 9);
+	subscr->nam_ps = sqlite3_column_int(stmt, 10);
+	subscr->lmsi = sqlite3_column_int(stmt, 11);
+	subscr->ms_purged_cs = sqlite3_column_int(stmt, 12);
+	subscr->ms_purged_ps = sqlite3_column_int(stmt, 13);
+	parse_last_lu_seen(&subscr->last_lu_seen, (const char *)sqlite3_column_text(stmt, 14),
+			   subscr->imsi, "CS");
+	parse_last_lu_seen(&subscr->last_lu_seen_ps, (const char *)sqlite3_column_text(stmt, 15),
+			   subscr->imsi, "PS");
+	copy_sqlite3_text_to_ipa_name(&subscr->vlr_via_proxy, stmt, 16);
+	copy_sqlite3_text_to_ipa_name(&subscr->sgsn_via_proxy, stmt, 17);
 
 out:
 	db_remove_reset(stmt);
@@ -417,6 +525,38 @@ out:
 	return ret;
 }
 
+/*! Check if a subscriber exists in the HLR database.
+ * \param[in, out] dbc  database context.
+ * \param[in] imsi  ASCII string of IMSI digits.
+ * \returns 0 if it exists, -ENOENT if it does not exist, -EIO on database error.
+ */
+int db_subscr_exists_by_imsi(struct db_context *dbc, const char *imsi) {
+	sqlite3_stmt *stmt = dbc->stmt[DB_STMT_EXISTS_BY_IMSI];
+	const char *err;
+	int rc;
+
+	if (!db_bind_text(stmt, NULL, imsi))
+		return -EIO;
+
+	rc = sqlite3_step(stmt);
+	db_remove_reset(stmt);
+	if (rc == SQLITE_ROW)
+		return 0; /* exists */
+	if (rc == SQLITE_DONE)
+		return -ENOENT; /* does not exist */
+
+	err = sqlite3_errmsg(dbc->db);
+	LOGP(DAUC, LOGL_ERROR, "Failed to check if subscriber exists by IMSI='%s': %s\n", imsi, err);
+	return rc;
+}
+
+/*! Retrieve subscriber data from the HLR database.
+ * \param[in,out] dbc  database context.
+ * \param[in] imsi  ASCII string of IMSI digits.
+ * \param[out] subscr  place retrieved data in this struct.
+ * \returns 0 on success, -ENOENT if no such subscriber was found, -EIO on
+ *          database error.
+ */
 int db_subscr_get_by_imsi(struct db_context *dbc, const char *imsi,
 			  struct hlr_subscriber *subscr)
 {
@@ -428,12 +568,46 @@ int db_subscr_get_by_imsi(struct db_context *dbc, const char *imsi,
 		return -EIO;
 
 	rc = db_sel(dbc, stmt, subscr, &err);
-	if (rc)
+	if (rc && rc != -ENOENT)
 		LOGP(DAUC, LOGL_ERROR, "Cannot read subscriber from db: IMSI='%s': %s\n",
 		     imsi, err);
 	return rc;
 }
 
+/*! Check if a subscriber exists in the HLR database.
+ * \param[in, out] dbc  database context.
+ * \param[in] msisdn  ASCII string of MSISDN digits.
+ * \returns 0 if it exists, -ENOENT if it does not exist, -EIO on database error.
+ */
+int db_subscr_exists_by_msisdn(struct db_context *dbc, const char *msisdn)
+{
+	sqlite3_stmt *stmt = dbc->stmt[DB_STMT_EXISTS_BY_MSISDN];
+	const char *err;
+	int rc;
+
+	if (!db_bind_text(stmt, NULL, msisdn))
+		return -EIO;
+
+	rc = sqlite3_step(stmt);
+	db_remove_reset(stmt);
+	if (rc == SQLITE_ROW)
+		return 0; /* exists */
+	if (rc == SQLITE_DONE)
+		return -ENOENT; /* does not exist */
+
+	err = sqlite3_errmsg(dbc->db);
+	LOGP(DAUC, LOGL_ERROR, "Failed to check if subscriber exists "
+		"by MSISDN='%s': %s\n", msisdn, err);
+	return rc;
+}
+
+/*! Retrieve subscriber data from the HLR database.
+ * \param[in,out] dbc  database context.
+ * \param[in] msisdn  ASCII string of MSISDN digits.
+ * \param[out] subscr  place retrieved data in this struct.
+ * \returns 0 on success, -ENOENT if no such subscriber was found, -EIO on
+ *          database error.
+ */
 int db_subscr_get_by_msisdn(struct db_context *dbc, const char *msisdn,
 			    struct hlr_subscriber *subscr)
 {
@@ -445,12 +619,107 @@ int db_subscr_get_by_msisdn(struct db_context *dbc, const char *msisdn,
 		return -EIO;
 
 	rc = db_sel(dbc, stmt, subscr, &err);
-	if (rc)
+	if (rc && rc != -ENOENT)
 		LOGP(DAUC, LOGL_ERROR, "Cannot read subscriber from db: MSISDN='%s': %s\n",
 		     msisdn, err);
 	return rc;
 }
 
+/*! Retrieve subscriber data from the HLR database.
+ * \param[in,out] dbc  database context.
+ * \param[in] filter_type  ASCII string of identifier type to search.
+ * \param[in] filter  ASCII string to search.
+ * \param[in] get_cb  pointer to call back function for data.
+ * \param[in,out] data  pointer to pass to callback function.
+ * \param[in,out] count  counter for number of matched subscribers.
+ * \param[in,our] err
+ * \returns 0 on success, -ENOENT if no subscriber was found, -EIO on
+ *          database error.
+ */
+int db_subscrs_get(struct db_context *dbc, const char *filter_type, const char *filter,
+		   void (*get_cb)(struct hlr_subscriber *subscr, void *data), void *data,
+		   int *count, const char **err)
+{
+	sqlite3_stmt *stmt;
+	char search[256];
+	int rc;
+	struct hlr_subscriber subscr;
+	bool show_ls = false;
+
+	if (!filter_type) {
+		stmt = dbc->stmt[DB_STMT_SEL_ALL];
+	} else if (strcmp(filter_type, "imei") == 0) {
+		stmt = dbc->stmt[DB_STMT_SEL_FILTER_IMEI];
+	} else if (strcmp(filter_type, "imsi") == 0) {
+		stmt = dbc->stmt[DB_STMT_SEL_FILTER_IMSI];
+	} else if (strcmp(filter_type, "msisdn") == 0) {
+		stmt = dbc->stmt[DB_STMT_SEL_FILTER_MSISDN];
+	} else if (strcmp(filter_type, "cs") == 0) {
+		stmt = dbc->stmt[DB_STMT_SEL_FILTER_CS];
+	} else if (strcmp(filter_type, "ps") == 0) {
+		stmt = dbc->stmt[DB_STMT_SEL_FILTER_PS];
+	} else if (strcmp(filter_type, "last_lu_seen") == 0) {
+		show_ls = true;
+		stmt = dbc->stmt[DB_STMT_SEL_ALL_ORDER_LAST_SEEN];
+	} else {
+		return -EIO;
+	}
+
+	if (filter_type && filter && strcmp(filter_type, "last_lu_seen") != 0) {
+		if (strcmp(filter, "on") == 0) {
+			sprintf(search, "%s", "1");
+		} else if (strcmp(filter, "off") == 0) {
+			sprintf(search, "%s", "0");
+		} else {
+			sprintf(search, "%%%s%%", filter);
+		}
+		if (!db_bind_text(stmt, "$search", search)) {
+			*err = sqlite3_errmsg(dbc->db);
+			return -EIO;
+		}
+	}
+
+	rc = sqlite3_step(stmt);
+
+	if (rc == SQLITE_DONE) {
+		db_remove_reset(stmt);
+		*err = "No matching subscriber(s)";
+		return -ENOENT;
+	}
+
+	while (rc == SQLITE_ROW) {
+		subscr = (struct hlr_subscriber){
+			  .id = sqlite3_column_int64(stmt, 0),};
+		copy_sqlite3_text_to_buf(subscr.imsi, stmt, 1);
+		copy_sqlite3_text_to_buf(subscr.msisdn, stmt, 2);
+		copy_sqlite3_text_to_buf(subscr.imei, stmt, 3);
+		subscr.nam_cs = sqlite3_column_int(stmt, 9);
+		subscr.nam_ps = sqlite3_column_int(stmt, 10);
+		if (show_ls)
+			parse_last_lu_seen(&subscr.last_lu_seen, (const char *)sqlite3_column_text(stmt, 14),
+					   subscr.imsi, "CS");
+		get_cb(&subscr, data);
+		rc = sqlite3_step(stmt);
+		(*count)++;
+	}
+
+	db_remove_reset(stmt);
+	if (rc != SQLITE_DONE) {
+		*err = sqlite3_errmsg(dbc->db);
+		LOGP(DAUC, LOGL_ERROR, "Cannot read subscribers from db:: %s\n", *err);
+		return rc;
+	}
+	*err = NULL;
+	return 0;
+}
+
+/*! Retrieve subscriber data from the HLR database.
+ * \param[in,out] dbc  database context.
+ * \param[in] id  ID of the subscriber in the HLR db.
+ * \param[out] subscr  place retrieved data in this struct.
+ * \returns 0 on success, -ENOENT if no such subscriber was found, -EIO on
+ *          database error.
+ */
 int db_subscr_get_by_id(struct db_context *dbc, int64_t id,
 			struct hlr_subscriber *subscr)
 {
@@ -462,18 +731,42 @@ int db_subscr_get_by_id(struct db_context *dbc, int64_t id,
 		return -EIO;
 
 	rc = db_sel(dbc, stmt, subscr, &err);
-	if (rc)
-		LOGP(DAUC, LOGL_ERROR, "Cannot read subscriber from db: ID=%"PRId64": %s\n",
+	if (rc && rc != -ENOENT)
+		LOGP(DAUC, LOGL_ERROR, "Cannot read subscriber from db: ID=%" PRId64 ": %s\n",
 		     id, err);
 	return rc;
 }
 
-/* Enable or disable PS or CS for a subscriber.
- * For the subscriber with the given imsi, set nam_ps (when is_ps == true) or
- * nam_cs (when is_ps == false) to nam_val in the database.
- * Returns 0 on success, -ENOENT when the given IMSI does not exist, -EINVAL if
- * the SQL statement could not be composed, -ENOEXEC if running the SQL
- * statement failed, -EIO if the amount of rows modified is unexpected.
+/*! Retrieve subscriber data from the HLR database.
+ * \param[in,out] dbc  database context.
+ * \param[in] imei  ASCII string of identifier digits
+ * \param[out] subscr  place retrieved data in this struct.
+ * \returns 0 on success, -ENOENT if no such subscriber was found, -EIO on
+ *          database error.
+ */
+int db_subscr_get_by_imei(struct db_context *dbc, const char *imei, struct hlr_subscriber *subscr)
+{
+	sqlite3_stmt *stmt = dbc->stmt[DB_STMT_SEL_BY_IMEI];
+	const char *err;
+	int rc;
+
+	if (!db_bind_text(stmt, NULL, imei))
+		return -EIO;
+
+	rc = db_sel(dbc, stmt, subscr, &err);
+	if (rc && rc != -ENOENT)
+		LOGP(DAUC, LOGL_ERROR, "Cannot read subscriber from db: IMEI=%s: %s\n", imei, err);
+	return rc;
+}
+
+/*! You should use hlr_subscr_nam() instead; enable or disable PS or CS for a
+ * subscriber without notifying GSUP clients.
+ * \param[in,out] dbc  database context.
+ * \param[in] imsi  ASCII string of IMSI digits.
+ * \param[in] nam_val True to enable CS/PS, false to disable.
+ * \param[in] is_ps  when true, set nam_ps, else set nam_cs.
+ * \returns 0 on success, -ENOENT when the given IMSI does not exist, -EIO on
+ *          database errors.
  */
 int db_subscr_nam(struct db_context *dbc, const char *imsi, bool nam_val, bool is_ps)
 {
@@ -522,11 +815,21 @@ out:
 	return ret;
 }
 
+/*! Record a Location Updating in the database.
+ * \param[in,out] dbc  database context.
+ * \param[in] subscr_id  ID of the subscriber in the HLR db.
+ * \param[in] vlr_or_sgsn_number  ASCII string of identifier digits.
+ * \param[in] is_ps  when true, set sgsn_number, else set vlr_number.
+ * \returns 0 on success, -ENOENT when the given subscriber does not exist,
+ *         -EIO on database errors.
+ */
 int db_subscr_lu(struct db_context *dbc, int64_t subscr_id,
-		 const char *vlr_or_sgsn_number, bool is_ps)
+		 const struct osmo_ipa_name *vlr_name, bool is_ps,
+		 const struct osmo_ipa_name *via_proxy)
 {
 	sqlite3_stmt *stmt;
 	int rc, ret = 0;
+	struct timespec localtime;
 
 	stmt = dbc->stmt[is_ps ? DB_STMT_UPD_SGSN_BY_ID
 			       : DB_STMT_UPD_VLR_BY_ID];
@@ -534,13 +837,21 @@ int db_subscr_lu(struct db_context *dbc, int64_t subscr_id,
 	if (!db_bind_int64(stmt, "$subscriber_id", subscr_id))
 		return -EIO;
 
-	if (!db_bind_text(stmt, "$number", vlr_or_sgsn_number))
+	if (!db_bind_text(stmt, "$number", (char*)vlr_name->val))
 		return -EIO;
 
+	if (via_proxy && via_proxy->len) {
+		if (!db_bind_text(stmt, "$proxy", (char*)via_proxy->val))
+			return -EIO;
+	} else {
+		if (!db_bind_null(stmt, "$proxy"))
+			return -EIO;
+	}
+
 	/* execute the statement */
 	rc = sqlite3_step(stmt);
 	if (rc != SQLITE_DONE) {
-		LOGP(DAUC, LOGL_ERROR, "Update %s number for subscriber ID=%"PRId64": SQL Error: %s\n",
+		LOGP(DAUC, LOGL_ERROR, "Update %s number for subscriber ID=%" PRId64 ": SQL Error: %s\n",
 		     is_ps? "SGSN" : "VLR", subscr_id, sqlite3_errmsg(dbc->db));
 		ret = -EIO;
 		goto out;
@@ -549,22 +860,71 @@ int db_subscr_lu(struct db_context *dbc, int64_t subscr_id,
 	/* verify execution result */
 	rc = sqlite3_changes(dbc->db);
 	if (!rc) {
-		LOGP(DAUC, LOGL_ERROR, "Cannot update %s number for subscriber ID=%"PRId64
+		LOGP(DAUC, LOGL_ERROR, "Cannot update %s number for subscriber ID=%" PRId64
 		     ": no such subscriber\n",
 		     is_ps? "SGSN" : "VLR", subscr_id);
 		ret = -ENOENT;
+		goto out;
 	} else if (rc != 1) {
-		LOGP(DAUC, LOGL_ERROR, "Update %s number for subscriber ID=%"PRId64
+		LOGP(DAUC, LOGL_ERROR, "Update %s number for subscriber ID=%" PRId64
 		       ": SQL modified %d rows (expected 1)\n",
 		       is_ps? "SGSN" : "VLR", subscr_id, rc);
 		ret = -EIO;
+		goto out;
 	}
 
+	db_remove_reset(stmt);
+
+	if (osmo_clock_gettime(CLOCK_REALTIME, &localtime) != 0) {
+		LOGP(DAUC, LOGL_ERROR, "Cannot get the current time: (%d) %s\n", errno, strerror(errno));
+		ret = -errno;
+		goto out;
+	}
+
+	stmt = dbc->stmt[is_ps? DB_STMT_SET_LAST_LU_SEEN_PS : DB_STMT_SET_LAST_LU_SEEN];
+
+	if (!db_bind_int64(stmt, "$subscriber_id", subscr_id))
+		return -EIO;
+	/* The timestamp will be converted to UTC by SQLite. */
+	if (!db_bind_int64(stmt, "$val", (int64_t)localtime.tv_sec)) {
+		ret = -EIO;
+		goto out;
+	}
+
+	rc = sqlite3_step(stmt);
+	if (rc != SQLITE_DONE) {
+		LOGP(DAUC, LOGL_ERROR,
+		       "Cannot update LU timestamp for subscriber ID=%" PRId64 ": SQL error: (%d) %s\n",
+		       subscr_id, rc, sqlite3_errmsg(dbc->db));
+		ret = -EIO;
+		goto out;
+	}
+
+	/* verify execution result */
+	rc = sqlite3_changes(dbc->db);
+	if (!rc) {
+		LOGP(DAUC, LOGL_ERROR, "Cannot update LU timestamp for subscriber ID=%" PRId64
+		     ": no such subscriber\n", subscr_id);
+		ret = -ENOENT;
+		goto out;
+	} else if (rc != 1) {
+		LOGP(DAUC, LOGL_ERROR, "Update LU timestamp for subscriber ID=%" PRId64
+		     ": SQL modified %d rows (expected 1)\n", subscr_id, rc);
+		ret = -EIO;
+	}
 out:
 	db_remove_reset(stmt);
 	return ret;
 }
 
+/*! Set the ms_purged_cs or ms_purged_ps values in the database.
+ * \param[in,out] dbc  database context.
+ * \param[in] by_imsi  ASCII string of IMSI digits.
+ * \param[in] purge_val  true to purge, false to un-purge.
+ * \param[in] is_ps  when true, set ms_purged_ps, else set ms_purged_cs.
+ * \returns 0 on success, -ENOENT when the given IMSI does not exist, -EIO on
+ *          database errors.
+ */
 int db_subscr_purge(struct db_context *dbc, const char *by_imsi,
 		    bool purge_val, bool is_ps)
 {
@@ -612,51 +972,3 @@ out:
 
 	return ret;
 }
-
-/*! Update nam_cs/nam_ps in the db and trigger notifications to GSUP clients.
- * \param hlr     Global hlr context.
- * \param subscr  Subscriber from a fresh db_subscr_get_by_*() call.
- * \param nam_val True to enable CS/PS, false to disable.
- * \param is_ps   True to enable/disable PS, false for CS.
- * \returns 0 on success, ENOEXEC if there is no need to change, a negative
- *          value on error.
- */
-int hlr_subscr_nam(struct hlr *hlr, struct hlr_subscriber *subscr, bool nam_val, bool is_ps)
-{
-	int rc;
-        struct lu_operation *luop;
-        struct osmo_gsup_conn *co;
-	bool is_val = is_ps? subscr->nam_ps : subscr->nam_cs;
-
-	if (is_val == nam_val) {
-		LOGHLR(subscr->imsi, LOGL_DEBUG, "Already has the requested value when asked to %s %s\n",
-		       nam_val ? "enable" : "disable", is_ps ? "PS" : "CS");
-		return ENOEXEC;
-	}
-
-	rc = db_subscr_nam(hlr->dbc, subscr->imsi, nam_val, is_ps);
-	if (rc)
-		return rc > 0? -rc : rc;
-
-	/* If we're disabling, send a notice out to the GSUP client that is
-	 * responsible. Otherwise no need. */
-	if (nam_val)
-		return 0;
-
-	/* FIXME: only send to single SGSN where latest update for IMSI came from */
-	llist_for_each_entry(co, &hlr->gs->clients, list) {
-		luop = lu_op_alloc_conn(co);
-		if (!luop) {
-			LOGHLR(subscr->imsi, LOGL_ERROR,
-			       "Cannot notify GSUP client, cannot allocate lu_operation,"
-			       " for %s:%u\n",
-			       co && co->conn && co->conn->server? co->conn->server->addr : "unset",
-			       co && co->conn && co->conn->server? co->conn->server->port : 0);
-			continue;
-		}
-		luop->subscr = *subscr;
-		lu_op_tx_del_subscr_data(luop);
-		lu_op_free(luop);
-	}
-	return 0;
-}

src/db_sql2c.sed

@@ -0,0 +1,25 @@
+# Input to this are sql/*.sql files.
+#
+# We want each SQL statement line wrapped in "...\n", and each end (";") to
+# become a comma:
+#
+#   SOME SQL COMMAND (
+#     that may span )
+#   MULTIPLE LINES;
+#   MORE;
+#
+# -->
+#
+#   "SOME SQL COMMAND (\n"
+#   "  that may span )\n"
+#   "MULTIPLE LINES\n",   <--note the comma here
+#   "MORE\n",
+#
+# just replacing ';' with '\n,' won't work, since sed is bad in printing
+# multiple lines. Also, how to input newlines to sed is not portable across
+# platforms.
+
+# Match excluding a trailing ';' as \1, keep any trailing ';' in \2
+s/^\(.*[^;]\)\(;\|\)$/"\1\\n"\2/
+# Replace trailing ';' as ','
+s/;$/,/

src/db_test.c

@@ -1,87 +0,0 @@
-#include <string.h>
-
-#include <osmocom/core/utils.h>
-#include <osmocom/core/application.h>
-
-#include "db.h"
-#include "hlr.h"
-#include "rand.h"
-#include "logging.h"
-
-static struct hlr *g_hlr;
-
-static int test(const char *imsi, struct db_context *dbc)
-{
-	struct osmo_auth_vector vec[3];
-	int rc, i;
-
-	/* initialize all vectors with a known token pattern */
-	memset(vec, 0x55, sizeof(vec));
-	for (i = 0; i < ARRAY_SIZE(vec); i++)
-		vec[i].res_len = 0;
-
-	rc = db_get_auc(dbc, imsi, 0, vec, ARRAY_SIZE(vec), NULL, NULL);
-	if (rc <= 0) {
-		LOGP(DMAIN, LOGL_ERROR, "Cannot obtain auth tuples for '%s'\n", imsi);
-		return rc;
-	}
-	LOGP(DMAIN, LOGL_INFO, "Obtained %u tuples for subscriber IMSI %s\n",
-		rc, imsi);
-
-	for (i = 0; i < rc; i++) {
-		struct osmo_auth_vector *v = vec + i;
-		LOGP(DMAIN, LOGL_DEBUG, "Tuple %u, auth_types=0x%x\n", i, v->auth_types);
-		LOGP(DMAIN, LOGL_DEBUG, "RAND=%s\n", osmo_hexdump_nospc(v->rand, sizeof(v->rand)));
-		LOGP(DMAIN, LOGL_DEBUG, "AUTN=%s\n", osmo_hexdump_nospc(v->autn, sizeof(v->autn)));
-		LOGP(DMAIN, LOGL_DEBUG, "CK=%s\n", osmo_hexdump_nospc(v->ck, sizeof(v->ck)));
-		LOGP(DMAIN, LOGL_DEBUG, "IK=%s\n", osmo_hexdump_nospc(v->ik, sizeof(v->ik)));
-		LOGP(DMAIN, LOGL_DEBUG, "RES=%s\n", osmo_hexdump_nospc(v->res, v->res_len));
-		LOGP(DMAIN, LOGL_DEBUG, "Kc=%s\n", osmo_hexdump_nospc(v->kc, sizeof(v->kc)));
-		LOGP(DMAIN, LOGL_DEBUG, "SRES=%s\n", osmo_hexdump_nospc(v->sres, sizeof(v->sres)));
-	}
-
-	return rc;
-}
-
-int main(int argc, char **argv)
-{
-	int rc;
-
-	g_hlr = talloc_zero(NULL, struct hlr);
-
-	rc = osmo_init_logging(&hlr_log_info);
-	if (rc < 0) {
-		fprintf(stderr, "Error initializing logging\n");
-		exit(1);
-	}
-	LOGP(DMAIN, LOGL_NOTICE, "hlr starting\n");
-
-	rc = rand_init();
-	if (rc < 0) {
-		LOGP(DMAIN, LOGL_ERROR, "Error initializing random source\n");
-		exit(1);
-	}
-
-	g_hlr->dbc = db_open(NULL, "hlr.db");
-	if (!g_hlr->dbc) {
-		LOGP(DMAIN, LOGL_ERROR, "Error opening database\n");
-		exit(1);
-	}
-
-	/* non-existing subscriber */
-	rc = test("901990123456789", g_hlr->dbc);
-	/* 2G only AUC data (COMP128v1 / MILENAGE) */
-	rc = test("901990000000001", g_hlr->dbc);
-	/* 2G + 3G AUC data (COMP128v1 / MILENAGE) */
-	rc = test("901990000000002", g_hlr->dbc);
-	/* 3G AUC data (MILENAGE) */
-	rc = test("901990000000003", g_hlr->dbc);
-
-	LOGP(DMAIN, LOGL_NOTICE, "Exiting\n");
-
-	db_close(g_hlr->dbc);
-
-	log_fini();
-
-	exit(0);
-}

src/dbd_decode_binary.c

@@ -0,0 +1,42 @@
+/* This function is blatantly copied from libdbi, from
+ * https://sourceforge.net/p/libdbi/libdbi/ci/master/tree/src/dbd_helper.c
+ * to save having to depend on the entire libdbi just for KI BLOB decoding.
+ */
+
+/*
+ * libdbi - database independent abstraction layer for C.
+ * Copyright (C) 2001-2003, David Parker and Mark Tobenkin.
+ * http://libdbi.sourceforge.net
+ * 
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ * 
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ * 
+ * $Id: dbd_helper.c,v 1.44 2011/08/09 11:14:14 mhoenicka Exp $
+ */
+
+#include <sys/types.h>
+
+size_t _dbd_decode_binary(const unsigned char *in, unsigned char *out){
+  int i, e;
+  unsigned char c;
+  e = *(in++);
+  i = 0;
+  while( (c = *(in++))!=0 ){
+    if( c==1 ){
+      c = *(in++) - 1;
+    }
+    out[i++] = c + e;
+  }
+  return (size_t)i;
+}

src/dgsm.c

@@ -0,0 +1,247 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include <errno.h>
+#include <osmocom/core/logging.h>
+#include <osmocom/mslookup/mslookup_client.h>
+#include <osmocom/mslookup/mslookup_client_mdns.h>
+#include <osmocom/gsupclient/gsup_client.h>
+#include <osmocom/gsupclient/cni_peer_id.h>
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/hlr.h>
+#include <osmocom/hlr/db.h>
+#include <osmocom/hlr/gsup_router.h>
+#include <osmocom/hlr/gsup_server.h>
+#include <osmocom/hlr/dgsm.h>
+#include <osmocom/hlr/proxy.h>
+#include <osmocom/hlr/remote_hlr.h>
+#include <osmocom/hlr/mslookup_server.h>
+#include <osmocom/hlr/mslookup_server_mdns.h>
+#include <osmocom/hlr/dgsm.h>
+
+void *dgsm_ctx = NULL;
+
+static void resolve_hlr_result_cb(struct osmo_mslookup_client *client,
+				  uint32_t request_handle,
+				  const struct osmo_mslookup_query *query,
+				  const struct osmo_mslookup_result *result)
+{
+	struct proxy *proxy = g_hlr->gs->proxy;
+	struct proxy_subscr proxy_subscr;
+	const struct osmo_sockaddr_str *remote_hlr_addr;
+
+	/* A remote HLR is answering back, indicating that it is the home HLR for a given IMSI.
+	 * There should be a mostly empty proxy entry for that IMSI.
+	 * Add the remote address data in the proxy. */
+	if (query->id.type != OSMO_MSLOOKUP_ID_IMSI) {
+		LOGP(DDGSM, LOGL_ERROR, "Expected IMSI ID type in mslookup query+result: %s\n",
+		     osmo_mslookup_result_name_c(OTC_SELECT, query, result));
+		return;
+	}
+
+	if (result->rc != OSMO_MSLOOKUP_RC_RESULT) {
+		LOG_DGSM(query->id.imsi, LOGL_ERROR, "Failed to resolve remote HLR: %s\n",
+			 osmo_mslookup_result_name_c(OTC_SELECT, query, result));
+		proxy_subscr_del(proxy, query->id.imsi);
+		return;
+	}
+
+	if (osmo_sockaddr_str_is_nonzero(&result->host_v4))
+		remote_hlr_addr = &result->host_v4;
+	else if (osmo_sockaddr_str_is_nonzero(&result->host_v6))
+		remote_hlr_addr = &result->host_v6;
+	else {
+		LOG_DGSM(query->id.imsi, LOGL_ERROR, "Invalid address for remote HLR: %s\n",
+			 osmo_mslookup_result_name_c(OTC_SELECT, query, result));
+		proxy_subscr_del(proxy, query->id.imsi);
+		return;
+	}
+
+	if (proxy_subscr_get_by_imsi(&proxy_subscr, proxy, query->id.imsi)) {
+		LOG_DGSM(query->id.imsi, LOGL_ERROR, "No proxy entry for mslookup result: %s\n",
+			 osmo_mslookup_result_name_c(OTC_SELECT, query, result));
+		return;
+	}
+
+	proxy_subscr_remote_hlr_resolved(proxy, &proxy_subscr, remote_hlr_addr);
+}
+
+/* Return true when the message has been handled by D-GSM. */
+bool dgsm_check_forward_gsup_msg(struct osmo_gsup_req *req)
+{
+	struct proxy_subscr proxy_subscr;
+	struct proxy *proxy = g_hlr->gs->proxy;
+	struct osmo_mslookup_query query;
+	struct osmo_mslookup_query_handling handling;
+	uint32_t request_handle;
+
+	/* If the IMSI is known in the local HLR, then we won't proxy. */
+	if (db_subscr_exists_by_imsi(g_hlr->dbc, req->gsup.imsi) == 0)
+		return false;
+
+	/* Are we already forwarding this IMSI to a remote HLR? */
+	if (proxy_subscr_get_by_imsi(&proxy_subscr, proxy, req->gsup.imsi) == 0) {
+		proxy_subscr_forward_to_remote_hlr(proxy, &proxy_subscr, req);
+		return true;
+	}
+
+	/* The IMSI is not known locally, so we want to proxy to a remote HLR, but no proxy entry exists yet. We need to
+	 * look up the subscriber in remote HLRs via D-GSM mslookup, forward GSUP and reply once a result is back from
+	 * there.  Defer message and kick off MS lookup. */
+
+	/* Add a proxy entry without a remote address to indicate that we are busy querying for a remote HLR. */
+	proxy_subscr = (struct proxy_subscr){};
+	OSMO_STRLCPY_ARRAY(proxy_subscr.imsi, req->gsup.imsi);
+	if (proxy_subscr_create_or_update(proxy, &proxy_subscr)) {
+		osmo_gsup_req_respond_err(req, GMM_CAUSE_NET_FAIL, "Failed to create proxy entry\n");
+		return true;
+	}
+
+	/* Is a fixed gateway proxy configured? */
+	if (osmo_sockaddr_str_is_nonzero(&g_hlr->mslookup.client.gsup_gateway_proxy)) {
+		proxy_subscr_remote_hlr_resolved(proxy, &proxy_subscr, &g_hlr->mslookup.client.gsup_gateway_proxy);
+
+		/* Proxy database modified, update info */
+		if (proxy_subscr_get_by_imsi(&proxy_subscr, proxy, req->gsup.imsi)) {
+			osmo_gsup_req_respond_err(req, GMM_CAUSE_NET_FAIL, "Internal proxy error\n");
+			return true;
+		}
+
+		proxy_subscr_forward_to_remote_hlr(proxy, &proxy_subscr, req);
+		return true;
+	}
+
+	/* Kick off an mslookup for the remote HLR?  This check could be up first on the top, but do it only now so that
+	 * if the mslookup client disconnected, we still continue to service open proxy entries. */
+	if (!osmo_mslookup_client_active(g_hlr->mslookup.client.client)) {
+		LOG_GSUP_REQ(req, LOGL_DEBUG, "mslookup client not running, cannot query remote home HLR\n");
+		return false;
+	}
+
+	/* First spool message, then kick off mslookup. If the proxy denies this message type, then don't do anything. */
+	if (proxy_subscr_forward_to_remote_hlr(proxy, &proxy_subscr, req)) {
+		/* If the proxy denied forwarding, an error response was already generated. */
+		return true;
+	}
+
+	query = (struct osmo_mslookup_query){
+		.id = {
+			.type = OSMO_MSLOOKUP_ID_IMSI,
+		},
+	};
+	OSMO_STRLCPY_ARRAY(query.id.imsi, req->gsup.imsi);
+	OSMO_STRLCPY_ARRAY(query.service, OSMO_MSLOOKUP_SERVICE_HLR_GSUP);
+	handling = (struct osmo_mslookup_query_handling){
+		.min_wait_milliseconds = g_hlr->mslookup.client.result_timeout_milliseconds,
+		.result_cb = resolve_hlr_result_cb,
+	};
+	request_handle = osmo_mslookup_client_request(g_hlr->mslookup.client.client, &query, &handling);
+	if (!request_handle) {
+		LOG_DGSM(req->gsup.imsi, LOGL_ERROR, "Error dispatching mslookup query for home HLR: %s\n",
+			 osmo_mslookup_result_name_c(OTC_SELECT, &query, NULL));
+		proxy_subscr_del(proxy, req->gsup.imsi);
+		/* mslookup seems to not be working. Try handling it locally. */
+		return false;
+	}
+
+	return true;
+}
+
+void dgsm_init(void *ctx)
+{
+	dgsm_ctx = talloc_named_const(ctx, 0, "dgsm");
+	INIT_LLIST_HEAD(&g_hlr->mslookup.server.local_site_services);
+
+	g_hlr->mslookup.server.local_attach_max_age = 60 * 60;
+
+	g_hlr->mslookup.client.result_timeout_milliseconds = 2000;
+
+	g_hlr->gsup_unit_name.unit_name = "HLR";
+	g_hlr->gsup_unit_name.serno = "unnamed-HLR";
+	g_hlr->gsup_unit_name.swversion = PACKAGE_NAME "-" PACKAGE_VERSION;
+
+	osmo_sockaddr_str_from_str(&g_hlr->mslookup.server.mdns.bind_addr,
+				   OSMO_MSLOOKUP_MDNS_IP4, OSMO_MSLOOKUP_MDNS_PORT);
+	osmo_sockaddr_str_from_str(&g_hlr->mslookup.client.mdns.query_addr,
+				   OSMO_MSLOOKUP_MDNS_IP4, OSMO_MSLOOKUP_MDNS_PORT);
+}
+
+void dgsm_start(void *ctx)
+{
+	g_hlr->mslookup.client.client = osmo_mslookup_client_new(dgsm_ctx);
+	OSMO_ASSERT(g_hlr->mslookup.client.client);
+	g_hlr->mslookup.allow_startup = true;
+	mslookup_server_mdns_config_apply();
+	dgsm_mdns_client_config_apply();
+}
+
+void dgsm_stop()
+{
+	g_hlr->mslookup.allow_startup = false;
+	mslookup_server_mdns_config_apply();
+	dgsm_mdns_client_config_apply();
+}
+
+void dgsm_mdns_client_config_apply(void)
+{
+	/* Check whether to start/stop/restart mDNS client */
+	const struct osmo_sockaddr_str *current_bind_addr;
+	const char *current_domain_suffix;
+	current_bind_addr = osmo_mslookup_client_method_mdns_get_bind_addr(g_hlr->mslookup.client.mdns.running);
+	current_domain_suffix = osmo_mslookup_client_method_mdns_get_domain_suffix(g_hlr->mslookup.client.mdns.running);
+
+	bool should_run = g_hlr->mslookup.allow_startup
+		&& g_hlr->mslookup.client.enable && g_hlr->mslookup.client.mdns.enable;
+
+	bool should_stop = g_hlr->mslookup.client.mdns.running &&
+		(!should_run
+		 || osmo_sockaddr_str_cmp(&g_hlr->mslookup.client.mdns.query_addr,
+					  current_bind_addr)
+		 || strcmp(g_hlr->mslookup.client.mdns.domain_suffix,
+			   current_domain_suffix));
+
+	if (should_stop) {
+		osmo_mslookup_client_method_del(g_hlr->mslookup.client.client, g_hlr->mslookup.client.mdns.running);
+		g_hlr->mslookup.client.mdns.running = NULL;
+		LOGP(DDGSM, LOGL_NOTICE, "Stopped mslookup mDNS client\n");
+	}
+
+	if (should_run && !g_hlr->mslookup.client.mdns.running) {
+		g_hlr->mslookup.client.mdns.running =
+			osmo_mslookup_client_add_mdns(g_hlr->mslookup.client.client,
+						      g_hlr->mslookup.client.mdns.query_addr.ip,
+						      g_hlr->mslookup.client.mdns.query_addr.port,
+						      -1,
+						      g_hlr->mslookup.client.mdns.domain_suffix);
+		if (!g_hlr->mslookup.client.mdns.running)
+			LOGP(DDGSM, LOGL_ERROR, "Failed to start mslookup mDNS client with target "
+			     OSMO_SOCKADDR_STR_FMT "\n",
+			     OSMO_SOCKADDR_STR_FMT_ARGS(&g_hlr->mslookup.client.mdns.query_addr));
+		else
+			LOGP(DDGSM, LOGL_NOTICE, "Started mslookup mDNS client, sending mDNS requests to multicast "
+			     OSMO_SOCKADDR_STR_FMT "\n",
+			     OSMO_SOCKADDR_STR_FMT_ARGS(&g_hlr->mslookup.client.mdns.query_addr));
+	}
+
+	if (g_hlr->mslookup.client.enable && osmo_sockaddr_str_is_nonzero(&g_hlr->mslookup.client.gsup_gateway_proxy))
+			LOGP(DDGSM, LOGL_NOTICE,
+			     "mslookup client: all GSUP requests for unknown IMSIs will be forwarded to"
+			     " gateway-proxy " OSMO_SOCKADDR_STR_FMT "\n",
+			     OSMO_SOCKADDR_STR_FMT_ARGS(&g_hlr->mslookup.client.gsup_gateway_proxy));
+}

src/dgsm_vty.c

@@ -0,0 +1,580 @@
+/* Copyright 2019 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include <osmocom/vty/vty.h>
+#include <osmocom/vty/command.h>
+#include <osmocom/mslookup/mslookup_client_mdns.h>
+#include <osmocom/mslookup/mdns.h>
+#include <osmocom/hlr/hlr_vty.h>
+#include <osmocom/hlr/mslookup_server.h>
+#include <osmocom/hlr/mslookup_server_mdns.h>
+#include <osmocom/gsupclient/cni_peer_id.h>
+
+struct cmd_node mslookup_node = {
+	MSLOOKUP_NODE,
+	"%s(config-mslookup)# ",
+	1,
+};
+
+DEFUN(cfg_mslookup,
+      cfg_mslookup_cmd,
+      "mslookup",
+      "Configure Distributed GSM mslookup")
+{
+	vty->node = MSLOOKUP_NODE;
+	return CMD_SUCCESS;
+}
+
+static int mslookup_server_mdns_bind(struct vty *vty, int argc, const char **argv)
+{
+	const char *ip_str = argc > 0? argv[0] : g_hlr->mslookup.server.mdns.bind_addr.ip;
+	const char *port_str = argc > 1? argv[1] : NULL;
+	uint16_t port_nr = port_str ? atoi(port_str) : g_hlr->mslookup.server.mdns.bind_addr.port;
+	struct osmo_sockaddr_str addr;
+	if (osmo_sockaddr_str_from_str(&addr, ip_str, port_nr)
+	    || !osmo_sockaddr_str_is_nonzero(&addr)) {
+		vty_out(vty, "%% mslookup server: Invalid mDNS bind address: %s %u%s",
+			ip_str, port_nr, VTY_NEWLINE);
+		return CMD_WARNING;
+	}
+
+	g_hlr->mslookup.server.mdns.bind_addr = addr;
+	g_hlr->mslookup.server.mdns.enable = true;
+	g_hlr->mslookup.server.enable = true;
+	mslookup_server_mdns_config_apply();
+	return CMD_SUCCESS;
+}
+
+static int mslookup_client_mdns_to(struct vty *vty, int argc, const char **argv)
+{
+	const char *ip_str = argc > 0? argv[0] : g_hlr->mslookup.client.mdns.query_addr.ip;
+	const char *port_str = argc > 1? argv[1] : NULL;
+	uint16_t port_nr = port_str ? atoi(port_str) : g_hlr->mslookup.client.mdns.query_addr.port;
+	struct osmo_sockaddr_str addr;
+	if (osmo_sockaddr_str_from_str(&addr, ip_str, port_nr)
+	    || !osmo_sockaddr_str_is_nonzero(&addr)) {
+		vty_out(vty, "%% mslookup client: Invalid mDNS target address: %s %u%s",
+			ip_str, port_nr, VTY_NEWLINE);
+		return CMD_WARNING;
+	}
+
+	g_hlr->mslookup.client.mdns.query_addr = addr;
+	g_hlr->mslookup.client.mdns.enable = true;
+	g_hlr->mslookup.client.enable = true;
+	dgsm_mdns_client_config_apply();
+	return CMD_SUCCESS;
+}
+
+#define MDNS_STR "Multicast DNS related configuration\n"
+#define MDNS_IP46_STR "multicast IPv4 address like " OSMO_MSLOOKUP_MDNS_IP4 \
+			" or IPv6 address like " OSMO_MSLOOKUP_MDNS_IP6 "\n"
+#define MDNS_PORT_STR "mDNS UDP Port number\n"
+#define MDNS_DOMAIN_SUFFIX_STR "mDNS domain suffix (default: " OSMO_MDNS_DOMAIN_SUFFIX_DEFAULT "). This is appended" \
+				 " and stripped from mDNS packets during encoding/decoding, so we don't collide with" \
+				 " top-level domains administrated by IANA\n"
+#define IP46_STR "IPv4 address like 1.2.3.4 or IPv6 address like a:b:c:d::1\n"
+#define PORT_STR "Service-specific port number\n"
+
+DEFUN(cfg_mslookup_mdns,
+      cfg_mslookup_mdns_cmd,
+      "mdns bind [IP] [<1-65535>]",
+      MDNS_STR
+      "Convenience shortcut: enable and configure both server and client for mDNS mslookup\n"
+      MDNS_IP46_STR MDNS_PORT_STR)
+{
+	int rc1 = mslookup_server_mdns_bind(vty, argc, argv);
+	int rc2 = mslookup_client_mdns_to(vty, argc, argv);
+	if (rc1 != CMD_SUCCESS)
+		return rc1;
+	return rc2;
+}
+
+DEFUN(cfg_mslookup_mdns_domain_suffix,
+      cfg_mslookup_mdns_domain_suffix_cmd,
+      "mdns domain-suffix DOMAIN_SUFFIX",
+      MDNS_STR MDNS_DOMAIN_SUFFIX_STR MDNS_DOMAIN_SUFFIX_STR)
+{
+	osmo_talloc_replace_string(g_hlr, &g_hlr->mslookup.server.mdns.domain_suffix, argv[0]);
+	osmo_talloc_replace_string(g_hlr, &g_hlr->mslookup.client.mdns.domain_suffix, argv[0]);
+	mslookup_server_mdns_config_apply();
+	dgsm_mdns_client_config_apply();
+	return CMD_SUCCESS;
+}
+
+DEFUN(cfg_mslookup_no_mdns,
+      cfg_mslookup_no_mdns_cmd,
+      "no mdns bind",
+      NO_STR "Disable both server and client for mDNS mslookup\n")
+{
+	g_hlr->mslookup.server.mdns.enable = false;
+	g_hlr->mslookup.client.mdns.enable = false;
+	mslookup_server_mdns_config_apply();
+	dgsm_mdns_client_config_apply();
+	return CMD_SUCCESS;
+}
+
+struct cmd_node mslookup_server_node = {
+	MSLOOKUP_SERVER_NODE,
+	"%s(config-mslookup-server)# ",
+	1,
+};
+
+DEFUN(cfg_mslookup_server,
+      cfg_mslookup_server_cmd,
+      "server",
+      "Enable and configure Distributed GSM mslookup server")
+{
+	vty->node = MSLOOKUP_SERVER_NODE;
+	g_hlr->mslookup.server.enable = true;
+	mslookup_server_mdns_config_apply();
+	return CMD_SUCCESS;
+}
+
+DEFUN(cfg_mslookup_no_server,
+      cfg_mslookup_no_server_cmd,
+      "no server",
+      NO_STR "Disable Distributed GSM mslookup server")
+{
+	g_hlr->mslookup.server.enable = false;
+	mslookup_server_mdns_config_apply();
+	return CMD_SUCCESS;
+}
+
+DEFUN(cfg_mslookup_server_mdns_bind,
+      cfg_mslookup_server_mdns_bind_cmd,
+      "mdns bind [IP] [<1-65535>]",
+      MDNS_STR
+      "Configure where the mDNS server listens for mslookup requests\n"
+      MDNS_IP46_STR MDNS_PORT_STR)
+{
+	return mslookup_server_mdns_bind(vty, argc, argv);
+}
+
+DEFUN(cfg_mslookup_server_mdns_domain_suffix,
+      cfg_mslookup_server_mdns_domain_suffix_cmd,
+      "mdns domain-suffix DOMAIN_SUFFIX",
+      MDNS_STR
+      MDNS_DOMAIN_SUFFIX_STR
+      MDNS_DOMAIN_SUFFIX_STR)
+{
+	osmo_talloc_replace_string(g_hlr, &g_hlr->mslookup.server.mdns.domain_suffix, argv[0]);
+	mslookup_server_mdns_config_apply();
+	return CMD_SUCCESS;
+}
+
+DEFUN(cfg_mslookup_server_no_mdns_bind,
+      cfg_mslookup_server_no_mdns_bind_cmd,
+      "no mdns bind",
+      NO_STR "Disable server for mDNS mslookup (do not answer remote requests)\n")
+{
+	g_hlr->mslookup.server.mdns.enable = false;
+	mslookup_server_mdns_config_apply();
+	return CMD_SUCCESS;
+}
+
+struct cmd_node mslookup_server_msc_node = {
+	MSLOOKUP_SERVER_MSC_NODE,
+	"%s(config-mslookup-server-msc)# ",
+	1,
+};
+
+DEFUN(cfg_mslookup_server_msc,
+      cfg_mslookup_server_msc_cmd,
+      "msc ipa-name .IPA_NAME",
+      "Configure services for individual local MSCs\n"
+      "Identify locally connected MSC by IPA Unit Name\n"
+      "IPA Unit Name of the local MSC to configure\n")
+{
+	struct osmo_ipa_name msc_name;
+	struct mslookup_server_msc_cfg *msc;
+	osmo_ipa_name_set_str(&msc_name, argv_concat(argv, argc, 0));
+
+	msc = mslookup_server_msc_get(&msc_name, true);
+	if (!msc) {
+		vty_out(vty, "%% Error creating MSC %s%s", osmo_ipa_name_to_str(&msc_name), VTY_NEWLINE);
+		return CMD_WARNING;
+	}
+	vty->node = MSLOOKUP_SERVER_MSC_NODE;
+	vty->index = msc;
+	return CMD_SUCCESS;
+}
+
+#define SERVICE_NAME_STR \
+	"mslookup service name, e.g. sip.voice or smpp.sms\n"
+
+static struct mslookup_server_msc_cfg *msc_from_node(struct vty *vty)
+{
+	switch (vty->node) {
+	case MSLOOKUP_SERVER_NODE:
+		/* On the mslookup.server node, set services on the wildcard msc, without a particular name. */
+		return mslookup_server_msc_get(&mslookup_server_msc_wildcard, true);
+	case MSLOOKUP_SERVER_MSC_NODE:
+		return vty->index;
+	default:
+		return NULL;
+	}
+}
+
+DEFUN(cfg_mslookup_server_msc_service,
+      cfg_mslookup_server_msc_service_cmd,
+      "service NAME at IP <1-65535>",
+      "Configure addresses of local services, as sent in replies to remote mslookup requests.\n"
+      SERVICE_NAME_STR "at\n" IP46_STR PORT_STR)
+{
+	/* If this command is run on the 'server' node, it produces an empty unit name and serves as wildcard for all
+	 * MSCs. If on a 'server' / 'msc' node, set services only for that MSC Unit Name. */
+	struct mslookup_server_msc_cfg *msc = msc_from_node(vty);
+	const char *service = argv[0];
+	const char *ip_str = argv[1];
+	const char *port_str = argv[2];
+	struct osmo_sockaddr_str addr;
+
+	if (!msc) {
+		vty_out(vty, "%% Error: no MSC object on this node%s", VTY_NEWLINE);
+		return CMD_WARNING;
+	}
+
+	if (osmo_sockaddr_str_from_str(&addr, ip_str, atoi(port_str))
+	    || !osmo_sockaddr_str_is_nonzero(&addr)) {
+		vty_out(vty, "%% mslookup server: Invalid address for service %s: %s %s%s",
+			service, ip_str, port_str, VTY_NEWLINE);
+		return CMD_WARNING;
+	}
+
+	if (mslookup_server_msc_service_set(msc, service, &addr)) {
+		vty_out(vty, "%% mslookup server: Error setting service %s to %s %s%s",
+			service, ip_str, port_str, VTY_NEWLINE);
+		return CMD_WARNING;
+	}
+	return CMD_SUCCESS;
+}
+
+#define NO_SERVICE_AND_NAME_STR NO_STR "Remove one or more service address entries\n" SERVICE_NAME_STR
+
+DEFUN(cfg_mslookup_server_msc_no_service,
+      cfg_mslookup_server_msc_no_service_cmd,
+      "no service NAME",
+      NO_SERVICE_AND_NAME_STR)
+{
+	/* If this command is run on the 'server' node, it produces an empty unit name and serves as wildcard for all
+	 * MSCs. If on a 'server' / 'msc' node, set services only for that MSC Unit Name. */
+	struct mslookup_server_msc_cfg *msc = msc_from_node(vty);
+	const char *service = argv[0];
+
+	if (!msc) {
+		vty_out(vty, "%% Error: no MSC object on this node%s", VTY_NEWLINE);
+		return CMD_WARNING;
+	}
+
+	if (mslookup_server_msc_service_del(msc, service, NULL) < 1) {
+		vty_out(vty, "%% mslookup server: cannot remove service '%s'%s",
+			service, VTY_NEWLINE);
+		return CMD_WARNING;
+	}
+	return CMD_SUCCESS;
+}
+
+DEFUN(cfg_mslookup_server_msc_no_service_addr,
+      cfg_mslookup_server_msc_no_service_addr_cmd,
+      "no service NAME at IP <1-65535>",
+      NO_SERVICE_AND_NAME_STR "at\n" IP46_STR PORT_STR)
+{
+	/* If this command is run on the 'server' node, it produces an empty unit name and serves as wildcard for all
+	 * MSCs. If on a 'server' / 'msc' node, set services only for that MSC Unit Name. */
+	struct mslookup_server_msc_cfg *msc = msc_from_node(vty);
+	const char *service = argv[0];
+	const char *ip_str = argv[1];
+	const char *port_str = argv[2];
+	struct osmo_sockaddr_str addr;
+
+	if (!msc) {
+		vty_out(vty, "%% Error: no MSC object on this node%s", VTY_NEWLINE);
+		return CMD_WARNING;
+	}
+
+	if (osmo_sockaddr_str_from_str(&addr, ip_str, atoi(port_str))
+	    || !osmo_sockaddr_str_is_nonzero(&addr)) {
+		vty_out(vty, "%% mslookup server: Invalid address for 'no service' %s: %s %s%s",
+			service, ip_str, port_str, VTY_NEWLINE);
+		return CMD_WARNING;
+	}
+
+	if (mslookup_server_msc_service_del(msc, service, &addr) < 1) {
+		vty_out(vty, "%% mslookup server: cannot remove service '%s' to %s %s%s",
+			service, ip_str, port_str, VTY_NEWLINE);
+		return CMD_WARNING;
+	}
+	return CMD_SUCCESS;
+}
+
+struct cmd_node mslookup_client_node = {
+	MSLOOKUP_CLIENT_NODE,
+	"%s(config-mslookup-client)# ",
+	1,
+};
+
+DEFUN(cfg_mslookup_client,
+      cfg_mslookup_client_cmd,
+      "client",
+      "Enable and configure Distributed GSM mslookup client")
+{
+	vty->node = MSLOOKUP_CLIENT_NODE;
+	g_hlr->mslookup.client.enable = true;
+	dgsm_mdns_client_config_apply();
+	return CMD_SUCCESS;
+}
+
+DEFUN(cfg_mslookup_no_client,
+      cfg_mslookup_no_client_cmd,
+      "no client",
+      NO_STR "Disable Distributed GSM mslookup client")
+{
+	g_hlr->mslookup.client.enable = false;
+	dgsm_mdns_client_config_apply();
+	return CMD_SUCCESS;
+}
+
+DEFUN(cfg_mslookup_client_timeout,
+      cfg_mslookup_client_timeout_cmd,
+      "timeout <1-100000>",
+      "How long should the mslookup client wait for remote responses before evaluating received results\n"
+      "timeout in milliseconds\n")
+{
+	uint32_t val = atol(argv[0]);
+	g_hlr->mslookup.client.result_timeout_milliseconds = val;
+	return CMD_SUCCESS;
+}
+
+#define EXIT_HINT() \
+	if (vty->type != VTY_FILE) \
+		vty_out(vty, "%% 'exit' this node to apply changes%s", VTY_NEWLINE)
+
+
+DEFUN(cfg_mslookup_client_mdns_bind,
+      cfg_mslookup_client_mdns_bind_cmd,
+      "mdns bind [IP] [<1-65535>]",
+      MDNS_STR
+      "Enable mDNS client, and configure multicast address to send mDNS mslookup requests to\n"
+      MDNS_IP46_STR MDNS_PORT_STR)
+{
+	return mslookup_client_mdns_to(vty, argc, argv);
+}
+
+DEFUN(cfg_mslookup_client_mdns_domain_suffix,
+      cfg_mslookup_client_mdns_domain_suffix_cmd,
+      "mdns domain-suffix DOMAIN_SUFFIX",
+      MDNS_STR
+      MDNS_DOMAIN_SUFFIX_STR
+      MDNS_DOMAIN_SUFFIX_STR)
+{
+	osmo_talloc_replace_string(g_hlr, &g_hlr->mslookup.client.mdns.domain_suffix, argv[0]);
+	dgsm_mdns_client_config_apply();
+	return CMD_SUCCESS;
+}
+
+DEFUN(cfg_mslookup_client_no_mdns_bind,
+      cfg_mslookup_client_no_mdns_bind_cmd,
+      "no mdns bind",
+      NO_STR "Disable mDNS client, do not query remote services by mDNS\n")
+{
+	g_hlr->mslookup.client.mdns.enable = false;
+	dgsm_mdns_client_config_apply();
+	return CMD_SUCCESS;
+}
+
+void config_write_msc_services(struct vty *vty, const char *indent, struct mslookup_server_msc_cfg *msc)
+{
+	struct mslookup_service_host *e;
+
+	llist_for_each_entry(e, &msc->service_hosts, entry) {
+		if (osmo_sockaddr_str_is_nonzero(&e->host_v4))
+			vty_out(vty, "%sservice %s at %s %u%s", indent, e->service, e->host_v4.ip, e->host_v4.port,
+				VTY_NEWLINE);
+		if (osmo_sockaddr_str_is_nonzero(&e->host_v6))
+			vty_out(vty, "%sservice %s at %s %u%s", indent, e->service, e->host_v6.ip, e->host_v6.port,
+				VTY_NEWLINE);
+	}
+}
+
+int config_write_mslookup(struct vty *vty)
+{
+	if (!g_hlr->mslookup.server.enable
+	    && llist_empty(&g_hlr->mslookup.server.local_site_services)
+	    && !g_hlr->mslookup.client.enable)
+		return CMD_SUCCESS;
+
+	vty_out(vty, "mslookup%s", VTY_NEWLINE);
+
+	if (g_hlr->mslookup.server.enable || !llist_empty(&g_hlr->mslookup.server.local_site_services)) {
+		struct mslookup_server_msc_cfg *msc;
+
+		vty_out(vty, " server%s", VTY_NEWLINE);
+
+		if (g_hlr->mslookup.server.mdns.enable) {
+			vty_out(vty, "  mdns bind");
+			if (osmo_sockaddr_str_is_nonzero(&g_hlr->mslookup.server.mdns.bind_addr)) {
+				vty_out(vty, " %s %u",
+					g_hlr->mslookup.server.mdns.bind_addr.ip,
+					g_hlr->mslookup.server.mdns.bind_addr.port);
+			}
+			vty_out(vty, "%s", VTY_NEWLINE);
+		}
+		if (strcmp(g_hlr->mslookup.server.mdns.domain_suffix, OSMO_MDNS_DOMAIN_SUFFIX_DEFAULT))
+			vty_out(vty, "  mdns domain-suffix %s%s",
+				g_hlr->mslookup.server.mdns.domain_suffix,
+				VTY_NEWLINE);
+
+		msc = mslookup_server_msc_get(&mslookup_server_msc_wildcard, false);
+		if (msc)
+			config_write_msc_services(vty, "  ", msc);
+
+		llist_for_each_entry(msc, &g_hlr->mslookup.server.local_site_services, entry) {
+			if (!osmo_ipa_name_cmp(&mslookup_server_msc_wildcard, &msc->name))
+				continue;
+			vty_out(vty, " msc %s%s", osmo_ipa_name_to_str(&msc->name), VTY_NEWLINE);
+			config_write_msc_services(vty, "  ", msc);
+		}
+
+		/* If the server is disabled, still output the above to not lose the service config. */
+		if (!g_hlr->mslookup.server.enable)
+			vty_out(vty, " no server%s", VTY_NEWLINE);
+	}
+
+	if (g_hlr->mslookup.client.enable) {
+		vty_out(vty, " client%s", VTY_NEWLINE);
+
+		if (osmo_sockaddr_str_is_nonzero(&g_hlr->mslookup.client.gsup_gateway_proxy))
+			vty_out(vty, "  gateway-proxy %s %u%s",
+				g_hlr->mslookup.client.gsup_gateway_proxy.ip,
+				g_hlr->mslookup.client.gsup_gateway_proxy.port,
+				VTY_NEWLINE);
+
+		if (g_hlr->mslookup.client.mdns.enable
+		    && osmo_sockaddr_str_is_nonzero(&g_hlr->mslookup.client.mdns.query_addr))
+			vty_out(vty, "  mdns bind %s %u%s",
+				g_hlr->mslookup.client.mdns.query_addr.ip,
+				g_hlr->mslookup.client.mdns.query_addr.port,
+				VTY_NEWLINE);
+		if (strcmp(g_hlr->mslookup.client.mdns.domain_suffix, OSMO_MDNS_DOMAIN_SUFFIX_DEFAULT))
+			vty_out(vty, "  mdns domain-suffix %s%s",
+				g_hlr->mslookup.client.mdns.domain_suffix,
+				VTY_NEWLINE);
+	}
+
+	return CMD_SUCCESS;
+}
+
+DEFUN(cfg_mslookup_client_gateway_proxy,
+      cfg_mslookup_client_gateway_proxy_cmd,
+      "gateway-proxy IP [<1-65535>]",
+      "Configure a fixed IP address to send all GSUP requests for unknown IMSIs to, without invoking a lookup for IMSI\n"
+      "IP address of the remote HLR\n" "GSUP port number (omit for default " OSMO_STRINGIFY_VAL(OSMO_GSUP_PORT) ")\n")
+{
+	const char *ip_str = argv[0];
+	const char *port_str = argc > 1 ? argv[1] : NULL;
+	struct osmo_sockaddr_str addr;
+
+	if (osmo_sockaddr_str_from_str(&addr, ip_str, port_str ? atoi(port_str) : OSMO_GSUP_PORT)
+	    || !osmo_sockaddr_str_is_nonzero(&addr)) {
+		vty_out(vty, "%% mslookup client: Invalid address for gateway-proxy: %s %s%s",
+			ip_str, port_str ? : "", VTY_NEWLINE);
+		return CMD_WARNING;
+	}
+
+	g_hlr->mslookup.client.gsup_gateway_proxy = addr;
+	return CMD_SUCCESS;
+}
+
+DEFUN(cfg_mslookup_client_no_gateway_proxy,
+      cfg_mslookup_client_no_gateway_proxy_cmd,
+      "no gateway-proxy",
+      NO_STR "Disable gateway proxy for GSUP with unknown IMSIs\n")
+{
+	g_hlr->mslookup.client.gsup_gateway_proxy = (struct osmo_sockaddr_str){};
+	return CMD_SUCCESS;
+}
+
+DEFUN(do_mslookup_show_services,
+      do_mslookup_show_services_cmd,
+      "show mslookup services",
+      SHOW_STR "Distributed GSM / mslookup related information\n"
+      "List configured service addresses as sent to remote mslookup requests\n")
+{
+	struct mslookup_server_msc_cfg *msc;
+	const struct mslookup_service_host *local_hlr = mslookup_server_get_local_gsup_addr();
+
+	vty_out(vty, "Local GSUP HLR address returned in mslookup responses for local IMSIs:");
+	if (osmo_sockaddr_str_is_nonzero(&local_hlr->host_v4))
+		vty_out(vty, " " OSMO_SOCKADDR_STR_FMT,
+			OSMO_SOCKADDR_STR_FMT_ARGS(&local_hlr->host_v4));
+	if (osmo_sockaddr_str_is_nonzero(&local_hlr->host_v6))
+		vty_out(vty, " " OSMO_SOCKADDR_STR_FMT,
+			OSMO_SOCKADDR_STR_FMT_ARGS(&local_hlr->host_v6));
+	vty_out(vty, "%s", VTY_NEWLINE);
+
+	msc = mslookup_server_msc_get(&mslookup_server_msc_wildcard, false);
+	if (msc)
+		config_write_msc_services(vty, "", msc);
+
+	llist_for_each_entry(msc, &g_hlr->mslookup.server.local_site_services, entry) {
+		if (!osmo_ipa_name_cmp(&mslookup_server_msc_wildcard, &msc->name))
+			continue;
+		vty_out(vty, "msc ipa-name %s%s", osmo_ipa_name_to_str(&msc->name), VTY_NEWLINE);
+		config_write_msc_services(vty, " ", msc);
+	}
+	return CMD_SUCCESS;
+}
+
+void dgsm_vty_init(void)
+{
+	install_element(CONFIG_NODE, &cfg_mslookup_cmd);
+
+	install_node(&mslookup_node, config_write_mslookup);
+	install_element(MSLOOKUP_NODE, &cfg_mslookup_mdns_cmd);
+	install_element(MSLOOKUP_NODE, &cfg_mslookup_mdns_domain_suffix_cmd);
+	install_element(MSLOOKUP_NODE, &cfg_mslookup_no_mdns_cmd);
+	install_element(MSLOOKUP_NODE, &cfg_mslookup_server_cmd);
+	install_element(MSLOOKUP_NODE, &cfg_mslookup_no_server_cmd);
+
+	install_node(&mslookup_server_node, NULL);
+	install_element(MSLOOKUP_SERVER_NODE, &cfg_mslookup_server_mdns_bind_cmd);
+	install_element(MSLOOKUP_SERVER_NODE, &cfg_mslookup_server_mdns_domain_suffix_cmd);
+	install_element(MSLOOKUP_SERVER_NODE, &cfg_mslookup_server_no_mdns_bind_cmd);
+	install_element(MSLOOKUP_SERVER_NODE, &cfg_mslookup_server_msc_service_cmd);
+	install_element(MSLOOKUP_SERVER_NODE, &cfg_mslookup_server_msc_no_service_cmd);
+	install_element(MSLOOKUP_SERVER_NODE, &cfg_mslookup_server_msc_no_service_addr_cmd);
+	install_element(MSLOOKUP_SERVER_NODE, &cfg_mslookup_server_msc_cmd);
+
+	install_node(&mslookup_server_msc_node, NULL);
+	install_element(MSLOOKUP_SERVER_MSC_NODE, &cfg_mslookup_server_msc_service_cmd);
+	install_element(MSLOOKUP_SERVER_MSC_NODE, &cfg_mslookup_server_msc_no_service_cmd);
+	install_element(MSLOOKUP_SERVER_MSC_NODE, &cfg_mslookup_server_msc_no_service_addr_cmd);
+
+	install_element(MSLOOKUP_NODE, &cfg_mslookup_client_cmd);
+	install_element(MSLOOKUP_NODE, &cfg_mslookup_no_client_cmd);
+	install_node(&mslookup_client_node, NULL);
+	install_element(MSLOOKUP_CLIENT_NODE, &cfg_mslookup_client_timeout_cmd);
+	install_element(MSLOOKUP_CLIENT_NODE, &cfg_mslookup_client_mdns_bind_cmd);
+	install_element(MSLOOKUP_CLIENT_NODE, &cfg_mslookup_client_mdns_domain_suffix_cmd);
+	install_element(MSLOOKUP_CLIENT_NODE, &cfg_mslookup_client_no_mdns_bind_cmd);
+	install_element(MSLOOKUP_CLIENT_NODE, &cfg_mslookup_client_gateway_proxy_cmd);
+	install_element(MSLOOKUP_CLIENT_NODE, &cfg_mslookup_client_no_gateway_proxy_cmd);
+
+	install_element_ve(&do_mslookup_show_services_cmd);
+}

src/gsup_router.c

@@ -23,16 +23,17 @@
 #include <osmocom/core/linuxlist.h>
 #include <osmocom/core/talloc.h>
 
-#include "gsup_server.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/gsup_server.h>
+#include <osmocom/hlr/gsup_router.h>
 
-struct gsup_route {
-	struct llist_head list;
-
-	uint8_t *addr;
-	struct osmo_gsup_conn *conn;
-};
-
-/* find a route for the given address */
+/*! Find a route for the given address.
+ * \param[in] gs gsup server
+ * \param[in] addr IPA name of the client (SGSN, MSC/VLR). Although this is passed like a blob, together with the
+ *                 length, it must be nul-terminated! This is for legacy reasons, see the discussion here:
+ *                 https://gerrit.osmocom.org/#/c/osmo-hlr/+/13048/
+ * \param[in] addrlen length of addr, *including the nul-byte* (strlen(addr) + 1).
+ */
 struct osmo_gsup_conn *gsup_route_find(struct osmo_gsup_server *gs,
 					const uint8_t *addr, size_t addrlen)
 {
@@ -46,20 +47,48 @@ struct osmo_gsup_conn *gsup_route_find(struct osmo_gsup_server *gs,
 	return NULL;
 }
 
+struct osmo_gsup_conn *gsup_route_find_by_ipa_name(struct osmo_gsup_server *gs, const struct osmo_ipa_name *ipa_name)
+{
+	return gsup_route_find(gs, ipa_name->val, ipa_name->len);
+}
+
+/*! Find a GSUP connection's route (to read the IPA address from the route).
+ * \param[in] conn GSUP connection
+ * \return GSUP route
+ */
+struct gsup_route *gsup_route_find_by_conn(const struct osmo_gsup_conn *conn)
+{
+	struct gsup_route *gr;
+
+	llist_for_each_entry(gr, &conn->server->routes, list) {
+		if (gr->conn == conn)
+			return gr;
+	}
+
+	return NULL;
+}
+
 /* add a new route for the given address to the given conn */
 int gsup_route_add(struct osmo_gsup_conn *conn, const uint8_t *addr, size_t addrlen)
 {
 	struct gsup_route *gr;
+	struct osmo_gsup_conn *exists_on_conn;
 
 	/* Check if we already have a route for this address */
-	if (gsup_route_find(conn->server, addr, addrlen))
-		return -EEXIST;
+	exists_on_conn = gsup_route_find(conn->server, addr, addrlen);
+	if (exists_on_conn) {
+		if (exists_on_conn != conn)
+			return -EEXIST;
+		return 0;
+	}
 
 	/* allocate new route and populate it */
 	gr = talloc_zero(conn->server, struct gsup_route);
 	if (!gr)
 		return -ENOMEM;
 
+	LOGP(DMAIN, LOGL_INFO, "Adding GSUP route for %s via %s:%u\n", addr, conn->conn->addr, conn->conn->port);
+
 	gr->addr = talloc_memdup(gr, addr, addrlen);
 	gr->conn = conn;
 	llist_add_tail(&gr->list, &conn->server->routes);
@@ -67,6 +96,11 @@ int gsup_route_add(struct osmo_gsup_conn *conn, const uint8_t *addr, size_t addr
 	return 0;
 }
 
+int gsup_route_add_ipa_name(struct osmo_gsup_conn *conn, const struct osmo_ipa_name *ipa_name)
+{
+	return gsup_route_add(conn, ipa_name->val, ipa_name->len);
+}
+
 /* delete all routes for the given connection */
 int gsup_route_del_conn(struct osmo_gsup_conn *conn)
 {
@@ -75,6 +109,8 @@ int gsup_route_del_conn(struct osmo_gsup_conn *conn)
 
 	llist_for_each_entry_safe(gr, gr2, &conn->server->routes, list) {
 		if (gr->conn == conn) {
+			LOGP(DMAIN, LOGL_INFO, "Removing GSUP route for %s (GSUP disconnect)\n",
+			     osmo_quote_str_c(OTC_SELECT, (char*)gr->addr, talloc_total_size(gr->addr)));
 			llist_del(&gr->list);
 			talloc_free(gr);
 			num_deleted++;

src/gsup_router.h

@@ -1,8 +0,0 @@
-struct osmo_gsup_conn *gsup_route_find(struct osmo_gsup_server *gs,
-					const uint8_t *addr, size_t addrlen);
-
-/* add a new route for the given address to the given conn */
-int gsup_route_add(struct osmo_gsup_conn *conn, const uint8_t *addr, size_t addrlen);
-
-/* delete all routes for the given connection */
-int gsup_route_del_conn(struct osmo_gsup_conn *conn);

src/gsup_send.c

@@ -0,0 +1,91 @@
+/* (C) 2018 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/* This is kept separate to be able to override the actual sending functions from unit tests. */
+
+#include <errno.h>
+
+#include <osmocom/hlr/gsup_server.h>
+#include <osmocom/hlr/gsup_router.h>
+
+#include <osmocom/core/logging.h>
+
+/*! Send a msgb to a given address using routing.
+ * \param[in] gs gsup server
+ * \param[in] addr IPA name of the client (SGSN, MSC/VLR). Although this is passed like a blob, together with the
+ *                 length, it must be nul-terminated! This is for legacy reasons, see the discussion here:
+ *                 https://gerrit.osmocom.org/#/c/osmo-hlr/+/13048/
+ * \param[in] addrlen length of addr, *including the nul-byte* (strlen(addr) + 1).
+ * \param[in] msg message buffer
+ */
+int osmo_gsup_addr_send(struct osmo_gsup_server *gs,
+			const uint8_t *addr, size_t addrlen,
+			struct msgb *msg)
+{
+	struct osmo_gsup_conn *conn;
+
+	conn = gsup_route_find(gs, addr, addrlen);
+	if (!conn) {
+		LOGP(DLGSUP, LOGL_ERROR,
+		     "Cannot find route for addr %s\n", osmo_quote_str((const char*)addr, addrlen));
+		msgb_free(msg);
+		return -ENODEV;
+	}
+
+	return osmo_gsup_conn_send(conn, msg);
+}
+
+/*! Send a msgb to a given address using routing.
+ * \param[in] gs  gsup server
+ * \param[in] ipa_name  IPA unit name of the client (SGSN, MSC/VLR, proxy).
+ * \param[in] msg  message buffer
+ */
+int osmo_gsup_send_to_ipa_name(struct osmo_gsup_server *gs, const struct osmo_ipa_name *ipa_name, struct msgb *msg)
+{
+	if (ipa_name->val[ipa_name->len - 1]) {
+		/* Is not nul terminated. But for legacy reasons we (still) require that. */
+		if (ipa_name->len >= sizeof(ipa_name->val)) {
+			LOGP(DLGSUP, LOGL_ERROR, "IPA unit name is too long: %s\n",
+			     osmo_ipa_name_to_str(ipa_name));
+			return -EINVAL;
+		}
+		struct osmo_ipa_name ipa_name2 = *ipa_name;
+		ipa_name2.val[ipa_name->len] = '\0';
+		ipa_name2.len++;
+		return osmo_gsup_addr_send(gs, ipa_name2.val, ipa_name2.len, msg);
+	}
+	return osmo_gsup_addr_send(gs, ipa_name->val, ipa_name->len, msg);
+}
+
+int osmo_gsup_enc_send_to_ipa_name(struct osmo_gsup_server *gs, const struct osmo_ipa_name *ipa_name,
+			  const struct osmo_gsup_message *gsup)
+{
+	struct msgb *msg = osmo_gsup_msgb_alloc("GSUP Tx");
+	int rc;
+	rc = osmo_gsup_encode(msg, gsup);
+	if (rc) {
+		LOGP(DLGSUP, LOGL_ERROR, "IMSI-%s: Cannot encode GSUP: %s\n",
+		     gsup->imsi, osmo_gsup_message_type_name(gsup->message_type));
+		msgb_free(msg);
+		return -EINVAL;
+	}
+
+	LOGP(DLGSUP, LOGL_DEBUG, "IMSI-%s: Tx: %s\n", gsup->imsi, osmo_gsup_message_type_name(gsup->message_type));
+	return osmo_gsup_send_to_ipa_name(gs, ipa_name, msg);
+}

src/gsup_server.c

@@ -18,15 +18,31 @@
  */
 
 #include <errno.h>
+#include <netinet/tcp.h>
+#include <netinet/in.h>
 
 #include <osmocom/core/msgb.h>
 #include <osmocom/core/logging.h>
 #include <osmocom/core/linuxlist.h>
 #include <osmocom/abis/ipa.h>
 #include <osmocom/abis/ipaccess.h>
+#include <osmocom/gsm/gsm48_ie.h>
+#include <osmocom/gsm/apn.h>
+#include <osmocom/gsm/gsm23003.h>
 
-#include "gsup_server.h"
-#include "gsup_router.h"
+#include <osmocom/hlr/gsup_server.h>
+#include <osmocom/hlr/gsup_router.h>
+
+#define LOG_GSUP_CONN(conn, level, fmt, args...) \
+	LOGP(DLGSUP, level, "GSUP peer %s: " fmt, \
+	     (conn) ? osmo_ipa_name_to_str(&(conn)->peer_name) : "NULL", ##args)
+
+struct msgb *osmo_gsup_msgb_alloc(const char *label)
+{
+	struct msgb *msg = msgb_alloc_headroom(1024+512, 512, label);
+	OSMO_ASSERT(msg);
+	return msg;
+}
 
 static void osmo_gsup_server_send(struct osmo_gsup_conn *conn,
 			     int proto_ext, struct msgb *msg_tx)
@@ -48,6 +64,91 @@ int osmo_gsup_conn_send(struct osmo_gsup_conn *conn, struct msgb *msg)
 	return 0;
 }
 
+static void gsup_server_send_req_response(struct osmo_gsup_req *req, struct osmo_gsup_message *response)
+{
+	struct osmo_gsup_server *server = req->cb_data;
+	struct osmo_cni_peer_id *routing;
+	struct osmo_gsup_conn *conn = NULL;
+	struct msgb *msg = osmo_gsup_msgb_alloc("GSUP Tx");
+	int rc;
+
+	if (response->message_type == OSMO_GSUP_MSGT_ROUTING_ERROR
+	    && !osmo_cni_peer_id_is_empty(&req->via_proxy)) {
+		/* If a routing error occured, we need to route back via the immediate sending peer, not via the
+		 * intended final recipient -- because one source of routing errors is a duplicate name for a recipient.
+		 * If we resolve to req->source_name, we may send to a completely unrelated recipient. */
+		routing = &req->via_proxy;
+	} else {
+		routing = &req->source_name;
+	}
+	switch (routing->type) {
+	case OSMO_CNI_PEER_ID_IPA_NAME:
+		conn = gsup_route_find_by_ipa_name(server, &routing->ipa_name);
+		break;
+	default:
+		LOG_GSUP_REQ(req, LOGL_ERROR, "GSUP peer id kind not supported: %s\n",
+			     osmo_cni_peer_id_type_name(routing->type));
+		break;
+	}
+
+	if (!conn) {
+		LOG_GSUP_REQ(req, LOGL_ERROR, "GSUP client that sent this request not found, cannot respond\n");
+		msgb_free(msg);
+		return;
+	}
+
+	rc = osmo_gsup_encode(msg, response);
+	if (rc) {
+		LOG_GSUP_REQ(req, LOGL_ERROR, "Unable to encode: {%s}\n",
+			     osmo_gsup_message_to_str_c(OTC_SELECT, response));
+		msgb_free(msg);
+		return;
+	}
+
+	rc = osmo_gsup_conn_send(conn, msg);
+	if (rc)
+		LOG_GSUP_CONN(conn, LOGL_ERROR, "Unable to send: %s\n", osmo_gsup_message_to_str_c(OTC_SELECT, response));
+}
+
+struct osmo_gsup_req *osmo_gsup_conn_rx(struct osmo_gsup_conn *conn, struct msgb *msg)
+{
+	struct osmo_gsup_req *req;
+	struct osmo_cni_peer_id cpi = {
+		.type = OSMO_CNI_PEER_ID_IPA_NAME,
+		.ipa_name = conn->peer_name,
+	};
+
+	req = osmo_gsup_req_new(conn->server, &cpi, msg, gsup_server_send_req_response, conn->server, NULL);
+	if (!req)
+		return NULL;
+
+	if (!osmo_cni_peer_id_is_empty(&req->via_proxy)) {
+		switch (req->via_proxy.type) {
+		case OSMO_CNI_PEER_ID_IPA_NAME:
+			break;
+		default:
+			LOG_GSUP_REQ(req, LOGL_ERROR, "GSUP peer id kind not supported: %s\n",
+				     osmo_cni_peer_id_type_name(req->source_name.type));
+			osmo_gsup_req_respond_msgt(req, OSMO_GSUP_MSGT_ROUTING_ERROR, true);
+			return NULL;
+		}
+
+		/* The source of the GSUP message is not the immediate GSUP peer, but that peer is our proxy for that
+		 * source. Add it to the routes for this conn (so we can route responses back). */
+		if (gsup_route_add_ipa_name(conn, &req->source_name.ipa_name)) {
+			LOG_GSUP_REQ(req, LOGL_ERROR,
+				     "GSUP message received from %s via peer %s, but there already exists a"
+				     " different route to this source, message is not routable\n",
+				     osmo_cni_peer_id_to_str(&req->source_name),
+				     osmo_ipa_name_to_str(&conn->peer_name));
+			osmo_gsup_req_respond_msgt(req, OSMO_GSUP_MSGT_ROUTING_ERROR, true);
+			return NULL;
+		}
+	}
+
+	return req;
+}
+
 static int osmo_gsup_conn_oap_handle(struct osmo_gsup_conn *conn,
 				struct msgb *msg_rx)
 {
@@ -175,7 +276,7 @@ static int osmo_gsup_server_ccm_cb(struct ipa_server_conn *conn,
 {
 	struct osmo_gsup_conn *clnt = (struct osmo_gsup_conn *)conn->data;
 	uint8_t *addr = NULL;
-	size_t addr_len;
+	int addr_len;
 
 	LOGP(DLGSUP, LOGL_INFO, "CCM Callback\n");
 
@@ -193,10 +294,18 @@ static int osmo_gsup_server_ccm_cb(struct ipa_server_conn *conn,
 		return -EINVAL;
 	}
 
-	gsup_route_add(clnt, addr, addr_len);
+	osmo_ipa_name_set(&clnt->peer_name, addr, addr_len);
+	gsup_route_add_ipa_name(clnt, &clnt->peer_name);
 	return 0;
 }
 
+static void osmo_gsup_conn_free(struct osmo_gsup_conn *conn)
+{
+	gsup_route_del_conn(conn);
+	llist_del(&conn->list);
+	talloc_free(conn);
+}
+
 static int osmo_gsup_server_closed_cb(struct ipa_server_conn *conn)
 {
 	struct osmo_gsup_conn *clnt = (struct osmo_gsup_conn *)conn->data;
@@ -204,10 +313,7 @@ static int osmo_gsup_server_closed_cb(struct ipa_server_conn *conn)
 	LOGP(DLGSUP, LOGL_INFO, "Lost GSUP client %s:%d\n",
 		conn->addr, conn->port);
 
-	gsup_route_del_conn(clnt);
-	llist_del(&clnt->list);
-	talloc_free(clnt);
-
+	osmo_gsup_conn_free(clnt);
 	return 0;
 }
 
@@ -248,6 +354,19 @@ void osmo_gsup_server_add_conn(struct llist_head *clients,
 	llist_add(&conn->list, &prev_conn->list);
 }
 
+static void update_fd_settings(int fd)
+{
+	int ret;
+	int val;
+
+	/*TODO: Set keepalive settings here. See OS#4312 */
+
+	val = 1;
+	ret = setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &val, sizeof(val));
+	if (ret < 0)
+		LOGP(DLGSUP, LOGL_ERROR, "Failed to set TCP_NODELAY: %s\n", strerror(errno));
+}
+
 /* a client has connected to the server socket and we have accept()ed it */
 static int osmo_gsup_server_accept_cb(struct ipa_server_link *link, int fd)
 {
@@ -272,6 +391,8 @@ static int osmo_gsup_server_accept_cb(struct ipa_server_link *link, int fd)
 	LOGP(DLGSUP, LOGL_INFO, "New GSUP client %s:%d (IND=%u)\n",
 	     conn->conn->addr, conn->conn->port, conn->auc_3g_ind);
 
+	update_fd_settings(fd);
+
 	/* request the identity of the client */
 	rc = ipa_ccm_send_id_req(fd);
 	if (rc < 0)
@@ -289,8 +410,7 @@ failed:
 
 struct osmo_gsup_server *
 osmo_gsup_server_create(void *ctx, const char *ip_addr, uint16_t tcp_port,
-			osmo_gsup_read_cb_t read_cb,
-			struct llist_head *lu_op_lst)
+			osmo_gsup_read_cb_t read_cb, void *priv)
 {
 	struct osmo_gsup_server *gsups;
 	int rc;
@@ -310,13 +430,12 @@ osmo_gsup_server_create(void *ctx, const char *ip_addr, uint16_t tcp_port,
 		goto failed;
 
 	gsups->read_cb = read_cb;
+	gsups->priv = priv;
 
 	rc = ipa_server_link_open(gsups->link);
 	if (rc < 0)
 		goto failed;
 
-	gsups->luop = lu_op_lst;
-
 	return gsups;
 
 failed:
@@ -333,3 +452,117 @@ void osmo_gsup_server_destroy(struct osmo_gsup_server *gsups)
 	}
 	talloc_free(gsups);
 }
+
+/* Set GSUP message's pdp_infos[0] to a wildcard APN.
+ * Use the provided apn_buf to store the produced APN data. This must remain valid until
+ * osmo_gsup_encode() is done. Return 0 if an entry was added, -ENOMEM if the provided buffer is too
+ * small. */
+int osmo_gsup_configure_wildcard_apn(struct osmo_gsup_message *gsup,
+				     uint8_t *apn_buf, size_t apn_buf_size)
+{
+	int l;
+
+	l = osmo_apn_from_str(apn_buf, apn_buf_size, "*");
+	if (l <= 0)
+		return -ENOMEM;
+
+	gsup->pdp_infos[0].apn_enc = apn_buf;
+	gsup->pdp_infos[0].apn_enc_len = l;
+	gsup->pdp_infos[0].have_info = 1;
+	gsup->num_pdp_infos = 1;
+	/* FIXME: use real value: */
+	gsup->pdp_infos[0].context_id = 1;
+
+	return 0;
+}
+
+/**
+ * Populate a gsup message structure with an Insert Subscriber Data Message.
+ * All required memory buffers for data pointed to by pointers in struct osmo_gsup_message
+ * must be allocated by the caller and should have the same lifetime as the gsup parameter.
+ *
+ * \param[out] gsup  The gsup message to populate.
+ * \param[in] imsi  The subscriber's IMSI.
+ * \param[in] msisdn The subscriber's MSISDN.
+ * \param[out] msisdn_enc A buffer large enough to store the MSISDN in encoded form.
+ * \param[in] msisdn_enc_size Size of the buffer (must be >= OSMO_GSUP_MAX_CALLED_PARTY_BCD_LEN).
+ * \param[out] apn_buf A buffer large enough to store an APN (required if cn_domain is OSMO_GSUP_CN_DOMAIN_PS).
+ * \param[in] apn_buf_size Size of APN buffer (must be >= APN_MAXLEN).
+ * \param[in] cn_domain The CN Domain of the subscriber connection.
+ * \returns 0 on success, and negative on error.
+ */
+int osmo_gsup_create_insert_subscriber_data_msg(struct osmo_gsup_message *gsup, const char *imsi, const char *msisdn,
+						uint8_t *msisdn_enc, size_t msisdn_enc_size,
+						uint8_t *apn_buf, size_t apn_buf_size,
+						enum osmo_gsup_cn_domain cn_domain)
+{
+	int len;
+
+	OSMO_ASSERT(gsup);
+	*gsup = (struct osmo_gsup_message){
+		.message_type = OSMO_GSUP_MSGT_INSERT_DATA_REQUEST,
+	};
+
+	osmo_strlcpy(gsup->imsi, imsi, sizeof(gsup->imsi));
+
+	if (msisdn_enc_size < OSMO_GSUP_MAX_CALLED_PARTY_BCD_LEN)
+		return -ENOSPC;
+
+	OSMO_ASSERT(msisdn_enc);
+	len = gsm48_encode_bcd_number(msisdn_enc, msisdn_enc_size, 0, msisdn);
+	if (len < 1) {
+		LOGP(DLGSUP, LOGL_ERROR, "%s: Error: cannot encode MSISDN '%s'\n", imsi, msisdn);
+		return -ENOSPC;
+	}
+	gsup->msisdn_enc = msisdn_enc;
+	gsup->msisdn_enc_len = len;
+
+	#pragma message "FIXME: deal with encoding the following data: gsup.hlr_enc"
+
+	gsup->cn_domain = cn_domain;
+	if (gsup->cn_domain == OSMO_GSUP_CN_DOMAIN_PS) {
+		OSMO_ASSERT(apn_buf_size >= APN_MAXLEN);
+		OSMO_ASSERT(apn_buf);
+		/* FIXME: PDP infos - use more fine-grained access control
+		   instead of wildcard APN */
+		osmo_gsup_configure_wildcard_apn(gsup, apn_buf, apn_buf_size);
+	}
+
+	return 0;
+}
+
+int osmo_gsup_forward_to_local_peer(struct osmo_gsup_server *server, const struct osmo_cni_peer_id *to_peer,
+				    struct osmo_gsup_req *req, struct osmo_gsup_message *modified_gsup)
+{
+	int rc;
+	/* To forward to a remote entity (HLR, SMSC,...), we need to indicate the original source name in the Source
+	 * Name IE to make sure the reply can be routed back. Store the sender in gsup->source_name -- the remote entity
+	 * is required to return this as gsup->destination_name so that the reply gets routed to the original sender. */
+	struct osmo_gsup_message forward = *(modified_gsup? : &req->gsup);
+
+	if (req->source_name.type != OSMO_CNI_PEER_ID_IPA_NAME) {
+		osmo_gsup_req_respond_err(req, GMM_CAUSE_NET_FAIL, "Unsupported GSUP peer id type: %s",
+					  osmo_cni_peer_id_type_name(req->source_name.type));
+		rc = -ENOTSUP;
+		goto routing_error;
+	}
+	forward.source_name = req->source_name.ipa_name.val;
+	forward.source_name_len = req->source_name.ipa_name.len;
+
+	if (to_peer->type != OSMO_CNI_PEER_ID_IPA_NAME) {
+		osmo_gsup_req_respond_err(req, GMM_CAUSE_NET_FAIL, "Unsupported GSUP peer id type: %s",
+					  osmo_cni_peer_id_type_name(to_peer->type));
+		rc = -ENOTSUP;
+		goto routing_error;
+	}
+	LOG_GSUP_REQ(req, LOGL_INFO, "Forwarding to %s\n", osmo_cni_peer_id_to_str(to_peer));
+	rc = osmo_gsup_enc_send_to_ipa_name(server, &to_peer->ipa_name, &forward);
+	if (rc)
+		goto routing_error;
+	osmo_gsup_req_free(req);
+	return 0;
+
+routing_error:
+	osmo_gsup_req_respond_msgt(req, OSMO_GSUP_MSGT_ROUTING_ERROR, true);
+	return rc;
+}

src/gsup_server.h

@@ -1,50 +0,0 @@
-#pragma once
-
-#include <osmocom/core/linuxlist.h>
-#include <osmocom/core/msgb.h>
-#include <osmocom/abis/ipa.h>
-#include <osmocom/abis/ipaccess.h>
-
-struct osmo_gsup_conn;
-
-/* Expects message in msg->l2h */
-typedef int (*osmo_gsup_read_cb_t)(struct osmo_gsup_conn *conn, struct msgb *msg);
-
-struct osmo_gsup_server {
-	/* list of osmo_gsup_conn */
-	struct llist_head clients;
-
-	/* lu_operations list */
-	struct llist_head *luop;
-
-	struct ipa_server_link *link;
-	osmo_gsup_read_cb_t read_cb;
-	struct llist_head routes;
-};
-
-
-/* a single connection to a given client (SGSN, MSC) */
-struct osmo_gsup_conn {
-	struct llist_head list;
-
-	struct osmo_gsup_server *server;
-	struct ipa_server_conn *conn;
-	//struct oap_state oap_state;
-	struct tlv_parsed ccm;
-
-	unsigned int auc_3g_ind; /*!< IND index used for UMTS AKA SQN */
-};
-
-
-int osmo_gsup_conn_send(struct osmo_gsup_conn *conn, struct msgb *msg);
-int osmo_gsup_conn_ccm_get(const struct osmo_gsup_conn *clnt, uint8_t **addr,
-			   uint8_t tag);
-
-struct osmo_gsup_server *osmo_gsup_server_create(void *ctx,
-						 const char *ip_addr,
-						 uint16_t tcp_port,
-						 osmo_gsup_read_cb_t read_cb,
-						 struct llist_head *lu_op_lst);
-
-void osmo_gsup_server_destroy(struct osmo_gsup_server *gsups);
-