Add post-upgrade script for automatic db upgrade

Closes: OS#5138
Change-Id: I34e943e5020285f63a2162010eb42675af1520bf

.gitignore

@@ -26,6 +26,7 @@ m4
 *.m4
 missing
 .deps
+*~
 
 *.pc
 .libs
@@ -67,3 +68,5 @@ doc/manuals/generated/
 doc/manuals/osmomsc-usermanual.xml
 doc/manuals/common
 doc/manuals/build
+
+contrib/osmo-hlr.spec

Makefile.am

@@ -1,9 +1,9 @@
 AUTOMAKE_OPTIONS = foreign dist-bzip2
 
 SUBDIRS = \
-	doc \
 	src \
 	include \
+	doc \
 	sql \
 	contrib \
 	tests \
@@ -11,6 +11,8 @@ SUBDIRS = \
 
 EXTRA_DIST = \
 	.version \
+	contrib/osmo-hlr.spec.in \
+	debian \
 	$(NULL)
 
 AM_DISTCHECK_CONFIGURE_FLAGS = \

README.md

@@ -0,0 +1,66 @@
+osmo-hlr - Osmocom HLR Implementation
+=====================================
+
+This repository contains a C-language implementation of a GSM Home
+Location Register (HLR). It is part of the
+[Osmocom](https://osmocom.org/) Open Source Mobile Communications
+project.
+
+Warning: While the HLR logical functionality is implemented, OsmoHLR
+does not use the ETSI/3GPP TCAP/MAP protocol stack. Instead, a much
+simpler custom protocol (GSUP) is used.  This means, OsmoHLR is of
+no use outside the context of an Osmocom core network.  You can use
+it with OsmoMSC, OsmoSGSN etc. - but not with third party components.
+
+Homepage
+--------
+
+The official homepage of the project is
+https://osmocom.org/projects/osmo-hlr/wiki
+
+GIT Repository
+--------------
+
+You can clone from the official osmo-hlr.git repository using
+
+	git clone git://git.osmocom.org/osmo-hlr.git
+	git clone https://git.osmocom.org/osmo-hlr.git
+
+There is a cgit interface at https://git.osmocom.org/osmo-hlr/
+
+Documentation
+-------------
+
+User Manuals and VTY reference manuals are [optionally] built in PDF form
+as part of the build process.
+
+Pre-rendered PDF version of the current "master" can be found at
+[User Manual](https://ftp.osmocom.org/docs/latest/osmohlr-usermanual.pdf)
+as well as the VTY reference manuals
+* [VTY Reference Manual for osmo-hlr](https://ftp.osmocom.org/docs/latest/osmohlr-vty-reference.pdf)
+
+Mailing List
+------------
+
+Discussions related to osmo-hlr are happening on the
+openbsc@lists.osmocom.org mailing list, please see
+https://lists.osmocom.org/mailman/listinfo/openbsc for subscription
+options and the list archive.
+
+Please observe the [Osmocom Mailing List
+Rules](https://osmocom.org/projects/cellular-infrastructure/wiki/Mailing_List_Rules)
+when posting.
+
+Contributing
+------------
+
+Our coding standards are described at
+https://osmocom.org/projects/cellular-infrastructure/wiki/Coding_standards
+
+We us a gerrit based patch submission/review process for managing
+contributions.  Please see
+https://osmocom.org/projects/cellular-infrastructure/wiki/Gerrit for
+more details
+
+The current patch queue for osmo-hlr can be seen at
+https://gerrit.osmocom.org/#/q/project:osmo-hlr+status:open

TODO-RELEASE

@@ -0,0 +1,9 @@
+# When cleaning up this file: bump API version in corresponding Makefile.am and rename corresponding debian/lib*.install
+# according to https://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html#Updating-version-info
+# In short:
+# LIBVERSION=c:r:a
+# If the library source code has changed at all since the last update, then increment revision: c:r + 1:a.
+# If any interfaces have been added, removed, or changed since the last update: c + 1:0:0.
+# If any interfaces have been added since the last public release: c:r:a + 1.
+# If any interfaces have been removed or changed since the last public release: c:r:0.
+#library        what            description / commit summary line

configure.ac

@@ -12,6 +12,8 @@ AM_INIT_AUTOMAKE([foreign dist-bzip2 no-dist-gzip 1.9])
 
 AC_CONFIG_TESTDIR(tests)
 
+CFLAGS="$CFLAGS -std=gnu11"
+
 dnl kernel style compile messages
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 
@@ -25,7 +27,7 @@ AC_PROG_MKDIR_P
 AC_PROG_CC
 AC_PROG_INSTALL
 
-dnl patching ${archive_cmds} to affect generation of file "libtool" to fix linking with clang 
+dnl patching ${archive_cmds} to affect generation of file "libtool" to fix linking with clang
 AS_CASE(["$LD"],[*clang*],
   [AS_CASE(["${host_os}"],
      [*linux*],[archive_cmds='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-soname $wl$soname -o $lib'])])
@@ -39,11 +41,11 @@ PKG_PROG_PKG_CONFIG([0.20])
 
 PKG_CHECK_MODULES(TALLOC, [talloc >= 2.0.1])
 
-PKG_CHECK_MODULES(LIBOSMOCORE, libosmocore >= 1.3.0)
-PKG_CHECK_MODULES(LIBOSMOGSM, libosmogsm >= 1.3.0)
-PKG_CHECK_MODULES(LIBOSMOVTY, libosmovty >= 1.3.0)
-PKG_CHECK_MODULES(LIBOSMOCTRL, libosmoctrl >= 1.3.0)
-PKG_CHECK_MODULES(LIBOSMOABIS, libosmoabis >= 0.6.0)
+PKG_CHECK_MODULES(LIBOSMOCORE, libosmocore >= 1.5.0)
+PKG_CHECK_MODULES(LIBOSMOGSM, libosmogsm >= 1.5.0)
+PKG_CHECK_MODULES(LIBOSMOVTY, libosmovty >= 1.5.0)
+PKG_CHECK_MODULES(LIBOSMOCTRL, libosmoctrl >= 1.5.0)
+PKG_CHECK_MODULES(LIBOSMOABIS, libosmoabis >= 1.1.0)
 
 PKG_CHECK_MODULES(SQLITE3, sqlite3)
 
@@ -200,6 +202,7 @@ AC_OUTPUT(
 	contrib/Makefile
 	contrib/systemd/Makefile
 	contrib/dgsm/Makefile
+	contrib/osmo-hlr.spec
 	tests/Makefile
 	tests/auc/Makefile
 	tests/auc/gen_ts_55_205_test_sets/Makefile

contrib/Makefile.am

@@ -2,3 +2,13 @@ SUBDIRS = \
 	systemd \
 	dgsm \
 	$(NULL)
+
+EXTRA_DIST = osmo-hlr-post-upgrade.sh
+
+install-data-hook:
+	install -Dm755 $(srcdir)/osmo-hlr-post-upgrade.sh \
+		-t $(DESTDIR)$(datadir)/osmocom/
+
+uninstall-hook:
+	@$(PRE_UNINSTALL)
+	$(RM) $(DESTDIR)$(datadir)/osmocom/osmo-hlr-post-upgrade.sh

contrib/dgsm/esme_dgsm.py

@@ -100,6 +100,9 @@ def rx_deliver_sm(pdu):
         time.sleep(args.sleep)
         logging.info("Sleep done")
 
+    if args.always_fail is not None:
+        return args.always_fail
+
     result = query_mslookup("smpp.sms", msisdn)
     if 'v4' not in result or not result['v4']:
         logging.info('No IPv4 result from mslookup! This example only'
@@ -147,12 +150,35 @@ def main():
     parser.add_argument('--sleep', default=0, type=float,
                         help='sleep time in seconds before forwarding an SMS,'
                              ' to test multithreading (default: 0)')
+    parser.add_argument('--always-fail', default=None, metavar='SMPP_ESME_ERRCODE',
+                        help='test delivery failure: always return an error code on Deliver-SM,'
+                        ' pass an smpplib error code name like RDELIVERYFAILURE (see smpplib/consts.py),'
+                        ' or an SMPP error code in hex digits')
     args = parser.parse_args()
 
     logging.basicConfig(level=logging.INFO, format='[%(asctime)s]'
                         ' (%(threadName)s) %(message)s', datefmt="%H:%M:%S")
+
+    if args.always_fail:
+        resolved = None
+        name = 'SMPP_ESME_' + args.always_fail
+        if hasattr(smpplib.consts, name):
+            resolved = getattr(smpplib.consts, name)
+        if resolved is None:
+            try:
+                resolved = int(args.always_fail, 16)
+            except ValueError:
+                resolved = None
+        if resolved is None:
+            print('Invalid argument for --always-fail: %r' % args.always_fail)
+            exit(1)
+        args.always_fail = resolved
+        logging.info('--always-fail: returning error code %s to all Deliver-SM' % hex(args.always_fail))
+
     smpp_bind()
 
 
 if __name__ == "__main__":
     main()
+
+# vim: expandtab tabstop=4 shiftwidth=4

contrib/jenkins.sh

@@ -35,7 +35,6 @@ osmo-build-dep.sh libosmo-abis
 # Additional configure options and depends
 CONFIG=""
 if [ "$WITH_MANUALS" = "1" ]; then
-	osmo-build-dep.sh osmo-gsm-manuals
 	CONFIG="--enable-manuals"
 fi
 
@@ -57,11 +56,11 @@ autoreconf --install --force
 	$CONFIG
 $MAKE $PARALLEL_MAKE
 $MAKE check || cat-testlogs.sh
-DISTCHECK_CONFIGURE_FLAGS="$CONFIG" $MAKE distcheck || cat-testlogs.sh
+DISTCHECK_CONFIGURE_FLAGS="$CONFIG" $MAKE $PARALLEL_MAKE distcheck || cat-testlogs.sh
 
 if [ "$WITH_MANUALS" = "1" ] && [ "$PUBLISH" = "1" ]; then
 	make -C "$base/doc/manuals" publish
 fi
 
-$MAKE maintainer-clean
+$MAKE $PARALLEL_MAKE maintainer-clean
 osmo-clean-workspace.sh

contrib/osmo-hlr-post-upgrade.sh

@@ -0,0 +1,95 @@
+#!/bin/sh -e
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright 2021 sysmocom s.f.m.c GmbH <info@sysmocom.de>
+#
+# Packagers are supposed to call this script in post-upgrade, so it can safely
+# upgrade the database scheme if required.
+
+DB="/var/lib/osmocom/hlr.db"
+IS_ACTIVE=0
+
+msg() {
+	echo "osmo-hlr-post-upgrade: $@"
+}
+
+err() {
+	msg "ERROR: $@"
+}
+
+open_db() {
+	# Attempt to open the database with osmo-hlr-db-tool, it will fail if
+	# upgrading the schema is required
+	osmo-hlr-db-tool -s -l "$DB" create
+}
+
+check_upgrade_required() {
+	if ! [ -e "$DB" ]; then
+		msg "nothing to do (no existing database)"
+		exit 0
+	fi
+	
+	if open_db 2>/dev/null; then
+		msg "nothing to do (database version is up to date)"
+		exit 0
+	fi
+
+	msg "database upgrade is required"
+}
+
+stop_service() {
+	if systemctl is-active -q osmo-hlr; then
+		IS_ACTIVE=1
+		msg "stopping osmo-hlr service"
+		systemctl stop osmo-hlr
+
+		# Verify that it stopped
+		for i in $(seq 1 100); do
+			if ! systemctl is-active -q osmo-hlr; then
+				return
+			fi
+			sleep 0.1
+		done
+
+		err "failed to stop osmo-hlr service"
+		exit 1
+	else
+		msg "osmo-hlr service is not running"
+	fi
+}
+
+create_backup() {
+	backup="$DB.$(date +%Y%m%d%H%M%S).bak"
+	msg "creating backup: $backup"
+	if [ -e "$backup" ]; then
+		err "backup already exists: $backup"
+		exit 1
+	fi
+	cp "$DB" "$backup"
+}
+
+upgrade() {
+	msg "performing database upgrade"
+	osmo-hlr-db-tool -s -U -l "$DB" create
+	
+	if ! open_db 2>/dev/null; then
+		err "failed to open the database after upgrade"
+		err "osmo-hlr-db-tool output:"
+		open_db
+		# exit because of "set -e"
+	fi
+	
+	msg "database upgrade successful"
+}
+
+start_service() {
+	if [ "$IS_ACTIVE" = "1" ]; then
+		msg "starting osmo-hlr service"
+		systemctl start osmo-hlr
+	fi
+}
+
+check_upgrade_required
+stop_service
+create_backup
+upgrade
+start_service

contrib/osmo-hlr.spec.in

@@ -0,0 +1,195 @@
+#
+# spec file for package osmo-hlr
+#
+# Copyright (c) 2017 SUSE LINUX GmbH, Nuernberg, Germany.
+# Copyright (c) 2016, Martin Hauke <mardnh@gmx.de>
+#
+# All modifications and additions to the file contributed by third parties
+# remain the property of their copyright owners, unless otherwise agreed
+# upon. The license for this file, and modifications and additions to the
+# file, is the same license as for the pristine package itself (unless the
+# license for the pristine package is not an Open Source License, in which
+# case the license is the MIT License). An "Open Source License" is a
+# license that conforms to the Open Source Definition (Version 1.9)
+# published by the Open Source Initiative.
+
+Name:           osmo-hlr
+Version:        @VERSION@
+Release:        0
+Summary:        Osmocom Home Location Register for GSUP protocol towards OsmoSGSN and OsmoCSCN
+License:        AGPL-3.0-or-later AND GPL-2.0-or-later
+Group:          Productivity/Telephony/Servers
+URL:            https://osmocom.org/projects/osmo-hlr
+Source:         %{name}-%{version}.tar.xz
+BuildRequires:  autoconf
+BuildRequires:  automake
+BuildRequires:  libtool
+BuildRequires:  pkgconfig >= 0.20
+BuildRequires:  python3
+%if 0%{?suse_version}
+BuildRequires:  systemd-rpm-macros
+%endif
+BuildRequires:  pkgconfig(libosmoabis) >= 1.1.0
+BuildRequires:  pkgconfig(libosmocore) >= 1.5.0
+BuildRequires:  pkgconfig(libosmoctrl) >= 1.5.0
+BuildRequires:  pkgconfig(libosmogsm) >= 1.5.0
+BuildRequires:  pkgconfig(libosmovty) >= 1.5.0
+BuildRequires:  pkgconfig(sqlite3)
+BuildRequires:  pkgconfig(talloc) >= 2.0.1
+# only needed for populate_hlr_db.pl
+Requires:       libdbi-drivers-dbd-sqlite3
+%{?systemd_requires}
+
+%description
+The GSUP HLR is a stand-alone HLR (Home Location Register) for SIM
+and USIM based subscribers which exposes the GSUP protocol towards
+its users. OsmoSGSN supports this protocol.
+
+osmo-gsup-hlr is still very simplistic. It is a single-threaded
+architecture and uses only sqlite3 tables as back-end.  It is suitable
+for installations of the scale that OsmoNITB was able to handle.  It
+also lacks various features like fine-grained control of subscribed
+services (like supplementary services).
+
+%package -n libosmo-gsup-client0
+Summary:        Osmocom GSUP (General Subscriber Update Protocol) client library
+License:        GPL-2.0-or-later
+Group:          System/Libraries
+
+%description -n libosmo-gsup-client0
+This is a shared library that can be used to implement client programs for
+the GSUP protocol. The typical GSUP server is OsmoHLR, with OsmoMSC, OsmoSGSN
+and External USSD Entities (EUSEs) using this library to implement clients.
+
+%package -n libosmo-gsup-client-devel
+Summary:        Development files for the Osmocom GSUP client library
+License:        GPL-2.0-or-later
+Group:          Development/Libraries/C and C++
+Requires:       libosmo-gsup-client0 = %{version}
+
+%description -n libosmo-gsup-client-devel
+This is a shared library that can be used to implement client programs for
+the GSUP protocol. The typical GSUP server is OsmoHLR, with OsmoMSC, OsmoSGSN
+and External USSD Entities (EUSEs) using this library to implement clients.
+
+This subpackage contains libraries and header files for developing
+applications that want to make use of libosmo-gsup-client.
+
+%package -n libosmo-mslookup0
+Summary:        Osmocom MS lookup library
+License:        GPL-2.0-or-later
+Group:          System/Libraries
+
+%description -n libosmo-mslookup0
+This shared library contains routines for looking up mobile subscribers.
+
+%package -n libosmo-mslookup-devel
+Summary:        Development files for the Osmocom MS lookup library
+License:        GPL-2.0-or-later
+Group:          Development/Libraries/C and C++
+Requires:       libosmo-mslookup0 = %{version}
+
+%description -n libosmo-mslookup-devel
+This shared library contains routines for looking up mobile subscribers.
+
+This subpackage contains libraries and header files for developing
+applications that want to make use of libosmo-mslookup.
+
+
+%package -n osmo-mslookup-client
+Summary:        Standalone program using libosmo-mslookup
+License:        GPL-2.0-or-later
+Group:          Development/Libraries/C and C++
+
+%description -n osmo-mslookup-client
+Standalone program using libosmo-mslookup to easily integrate with programs
+that want to connect services (SIP, SMS,...) to the current location of a
+subscriber.
+
+%prep
+%setup -q
+
+%build
+echo "%{version}" >.tarball-version
+autoreconf -fi
+%configure \
+  --docdir="%{_docdir}/%{name}" \
+  --with-systemdsystemunitdir=%{_unitdir} \
+  --enable-shared \
+  --disable-static
+make V=1 %{?_smp_mflags}
+
+%install
+%make_install
+install -d "%{buildroot}/%{_localstatedir}/lib/osmocom"
+find %{buildroot} -type f -name "*.la" -delete -print
+
+%check
+make %{?_smp_mflags} check || (find . -name testsuite.log -exec cat {} +)
+
+%if 0%{?suse_version}
+%preun
+%service_del_preun %{name}.service
+
+%postun
+%service_del_postun %{name}.service
+
+%pre
+%service_add_pre %{name}.service
+%endif
+
+%post
+%if 0%{?suse_version}
+%service_add_post %{name}.service
+%endif
+/usr/share/osmocom/osmo-hlr-post-upgrade.sh
+
+%post   -n libosmo-gsup-client0 -p /sbin/ldconfig
+%postun -n libosmo-gsup-client0 -p /sbin/ldconfig
+%post   -n libosmo-mslookup0 -p /sbin/ldconfig
+%postun -n libosmo-mslookup0 -p /sbin/ldconfig
+
+%files
+%license COPYING
+%dir %{_docdir}/%{name}
+%dir %{_docdir}/%{name}/examples
+%{_docdir}/%{name}/examples/osmo-hlr.cfg
+%{_docdir}/%{name}/examples/osmo-hlr-dgsm.cfg
+%dir %{_docdir}/%{name}/sql
+%{_docdir}/%{name}/sql/hlr.sql
+%{_docdir}/%{name}/sql//hlr_data.sql
+%dir %{_sysconfdir}/osmocom
+%dir %{_localstatedir}/lib/osmocom
+%{_bindir}/osmo-hlr
+%{_bindir}/osmo-hlr-db-tool
+%dir %{_sysconfdir}/osmocom
+%config %{_sysconfdir}/osmocom/osmo-hlr.cfg
+%{_unitdir}/osmo-hlr.service
+%dir %{_datadir}/osmocom
+%{_datadir}/osmocom/osmo-hlr-post-upgrade.sh
+
+%files -n libosmo-gsup-client0
+%{_libdir}/libosmo-gsup-client.so.0*
+
+%files -n libosmo-gsup-client-devel
+%{_bindir}/osmo-euse-demo
+%dir %{_includedir}/osmocom
+%dir %{_includedir}/osmocom/gsupclient
+%{_includedir}/osmocom/gsupclient/*.h
+%{_libdir}/libosmo-gsup-client.so
+%{_libdir}/pkgconfig/libosmo-gsup-client.pc
+
+%files -n libosmo-mslookup0
+%{_libdir}/libosmo-mslookup.so.0*
+
+%files -n libosmo-mslookup-devel
+%dir %{_includedir}/osmocom
+%dir %{_includedir}/osmocom/mslookup
+%{_includedir}/osmocom/mslookup/*.h
+%{_libdir}/libosmo-mslookup.so
+%{_libdir}/pkgconfig/libosmo-mslookup.pc
+
+%files -n osmo-mslookup-client
+%{_bindir}/osmo-mslookup-client
+
+%changelog

debian/changelog

@@ -1,3 +1,103 @@
+osmo-hlr (1.3.0) unstable; urgency=medium
+
+  [ Alexander Couzens ]
+  * hlr: respect the num_auth_vectors requested
+  * hlr: remove unused internal USSD list
+
+  [ Oliver Smith ]
+  * add libosmo-mslookup abstract client
+  * add mDNS lookup method to libosmo-mslookup
+  * Makefile.am: fix pkgconfig_DATA
+  * add mDNS lookup method to libosmo-mslookup (#2)
+  * contrib/dgsm/ add example esme and dialplan
+  * mslookup_client.c: fix dereferencing null pointer
+  * mdns_msg.c: always call va_end
+  * mslookup_client_mdns.c: fix dereferencing null
+  * osmo-mslookup-client.c: fix dereferencing null
+  * osmo-mslookup-client: fix dereferencing null
+  * mdns_sock.c: fix resource leak of sock
+  * mdns_rfc.c: fix possible access of uninit. mem
+  * mslookup_client_mdns_test: disable by default
+  * mslookup_client_mdns_test: no automatic skip
+  * Cosmetic: mention OS#4491 in location cancel code
+  * hlr_vty_subscr: prettier output for last LU seen
+  * contrib: import RPM spec
+  * contrib: integrate RPM spec
+  * Makefile.am: EXTRA_DIST: debian, contrib/*.spec.in
+  * contrib/jenkins: don't build osmo-gsm-manuals
+  * configure.ac: set -std=gnu11
+
+  [ Neels Hofmeyr ]
+  * add osmo-mslookup-client program
+  * add osmo-mslookup-client program (#2)
+  * fix missing braces in LOGP_GSUP_FWD
+  * gsup_client.c: fix deprecation for client create func
+  * 1/2: refactor: add and use lu_fsm, osmo_gsup_req, osmo_ipa_name
+  * 2/2: wrap ipa_name in osmo_cni_peer_id with type enum and union
+  * gsup client: add up_down_cb(), add osmo_gsup_client_create3()
+  * db v5: prep for D-GSM: add vlr_via_proxy and sgsn_via_proxy
+  * enlarge the GSUP message headroom
+  * test_nodes.vty: remove cruft
+  * D-GSM 1/n: add mslookup server in osmo-hlr
+  * D-GSM 2/n: implement mDNS method of mslookup server
+  * D-GSM 3/n: implement roaming by mslookup in osmo-hlr
+  * gsup_server: send routing error back to the correct peer
+  * adoc: add D-GSM chapter to osmohlr-usermanual
+  * drop error log for when a subscriber does not exist
+  * vty: show subscriber: change format of 'last LU seen'
+  * vty: show subscriber: show lu d,h,m,s ago, not just seconds
+  * auc3g: officially wrap IND around IND_bitlen space
+  * make osmo_cni_peer_id_cmp() NULL safe
+  * osmo_gsup_req_new(): require from_peer != NULL
+  * gsup_server.c: properly handle negative rc from osmo_gsup_conn_ccm_get()
+  * osmo_mslookup_server_mdns_rx(): handle read() rc == 0
+  * hlr_subscr_nam(): fix condition to fix nam=false notifications
+  * esme_dgsm.py: add --always-fail option for debugging SMPP
+  * osmo-mslookup-client: fix segfault for respond_error() caller
+  * manual: describe subscriber import by SQL
+
+  [ Harald Welte ]
+  * Revert "add osmo-mslookup-client program"
+  * Revert "add mDNS lookup method to libosmo-mslookup"
+  * Use OSMO_FD_* instead of deprecated BSC_FD_*
+  * support the XOR algorithm for UMTS AKA
+  * auc_test.c: Add some comments on what the test cases actually do
+  * main: add --vty-ref-mode, use vty_dump_xml_ref_mode()
+  * manuals: generate vty reference xml at build time
+
+  [ Vadim Yanitskiy ]
+  * db: fix possible SQLite3 allocated memory leak in db_open()
+  * gsup_server: fix typo: s/omso_gsup_message/osmo_gsup_message/
+  * debian/control: change maintainer to the Osmocom team / mailing list
+  * cosmetic: fix spelling in logging message: existAnt -> existEnt
+  * doc/manuals: fix s/There/The/ in 'USSD Configuration'
+  * doc/manuals: re-organize description of internal USSD handlers
+  * USSD: fix handle_ussd(): do not free() unconditionally
+  * USSD: add special 'idle' handler to IUSE for testing
+
+  [ Eric ]
+  * configure.ac: fix libtool issue  with clang and sanitizer
+
+  [ Philipp Maier ]
+  * doc: do not use loglevel info for log category ss
+
+  [ Pau Espin Pedrol ]
+  * configure.ac: Fix trailing whitespace
+  * doc: Update VTY reference xml file
+  * Support setting rt-prio and cpu-affinity mask through VTY
+  * Set TCP NODELAY sockopt to GSUP cli and srv connections
+  * contrib/jenkins: Enable parallel make in make distcheck
+  * .gitignore: Ignore new autofoo tmp files
+  * tests: Replace deprecated API log_set_print_filename
+
+  [ Keith ]
+  * osmo-hlr-db-tool: Make import from osmo-nitb less "lossy"
+  * Correct vty inline help for show subscriber
+  * Add vty command to show summary of all or filtered subscribers
+  * Fix Coverity Warnings
+
+ -- Pau Espin Pedrol <pespin@sysmocom.de>  Tue, 23 Feb 2021 18:13:53 +0100
+
 osmo-hlr (1.2.0) unstable; urgency=medium
 
   [ Ruben Undheim ]

debian/control

@@ -1,19 +1,19 @@
 Source: osmo-hlr
 Section: net
 Priority: optional
-Maintainer: Max Suraev <msuraev@sysmocom.de>
+Maintainer: Osmocom team <openbsc@lists.osmocom.org>
 Build-Depends: debhelper (>= 9),
                pkg-config,
                dh-autoreconf,
                dh-systemd (>= 1.5),
                autotools-dev,
                python3-minimal,
-               libosmocore-dev,
-               libosmo-abis-dev,
-               libosmo-netif-dev,
+               libosmocore-dev (>= 1.5.0),
+               libosmo-abis-dev (>= 1.1.0),
+               libosmo-netif-dev (>= 1.1.0),
                libsqlite3-dev,
                sqlite3,
-               osmo-gsm-manuals-dev
+               osmo-gsm-manuals-dev (>= 1.1.0)
 Standards-Version: 3.9.6
 Vcs-Browser: http://cgit.osmocom.org/osmo-hlr
 Vcs-Git: git://git.osmocom.org/osmo-hlr

debian/osmo-hlr.install

@@ -5,4 +5,5 @@
 /usr/share/doc/osmo-hlr/sql/hlr.sql
 /usr/share/doc/osmo-hlr/sql/hlr_data.sql
 /usr/share/doc/osmo-hlr/examples/osmo-hlr.cfg
+/usr/share/osmocom/osmo-hlr-post-upgrade.sh
 /var/lib/osmocom

debian/postinst

@@ -0,0 +1,5 @@
+#!/bin/sh -e
+# Debian's postinst script is called on both installation and upgrade. Call the
+# post-upgrade script in both cases, it won't do anything if there is nothing
+# to do.
+/usr/share/osmocom/osmo-hlr-post-upgrade.sh

doc/examples/osmo-hlr.cfg

@@ -12,7 +12,7 @@ log stderr
  logging level main notice
  logging level db notice
  logging level auc notice
- logging level ss info
+ logging level ss notice
  logging level linp error
 !
 line vty

doc/manuals/Makefile.am

@@ -14,6 +14,12 @@ if BUILD_MANUALS
   include $(OSMO_GSM_MANUALS_DIR)/build/Makefile.asciidoc.inc
 
   VTY_REFERENCE = osmohlr-vty-reference.xml
+
+  BUILT_REFERENCE_XML = $(builddir)/vty/hlr_vty_reference.xml
+  $(builddir)/vty/hlr_vty_reference.xml: $(top_builddir)/src/osmo-hlr
+	mkdir -p $(builddir)/vty
+	$(top_builddir)/src/osmo-hlr --vty-ref-xml > $@
+
   include $(OSMO_GSM_MANUALS_DIR)/build/Makefile.vty-reference.inc
 
   OSMO_REPOSITORY = osmo-hlr

doc/manuals/chapters/running.adoc

@@ -54,7 +54,7 @@ this database file will be created in the current working directory.
 
 Alternatively, you may use the `osmo-hlr-db-tool`, which is installed along
 with `osmo-hlr`, to bootstrap an empty database, or to migrate subscriber data
-from an old 'OsmoNITB' database. See `osmo-hlr-db-tool --help`.
+from an old 'OsmoNITB' database. See <<db_import_nitb>>.
 
 === Multiple instances
 

doc/manuals/chapters/subscribers.adoc

@@ -127,3 +127,83 @@ OsmoHLR# subscriber imei 35761300444848 show
 ----
 <1> Randomly generated 5 digit MSISDN
 <2> Disabled CS and PS NAM prevent the subscriber from accessing the network
+
+
+=== Import Subscriber Data
+
+==== Scripted Import
+
+WARNING: It is not generally a good idea to depend on the HLR database's internal table structure, but in the lack of an
+automated import procedure, this example is provided as an ad-hoc method to aid automated subscriber import. This is not
+guaranteed to remain valid.
+
+NOTE: We may add CSV and other import methods to the `osmo-hlr-db-tool`, but so far that is not implemented. Contact the
+community if you are interested in such a feature being implemented.
+
+NOTE: `sqlite3` is available from your distribution packages or `sqlite.org`.
+
+Currently, probably the easiest way to automatically import subscribers to OsmoHLR is to write out a text file with SQL
+commands per subscriber, and feed that to `sqlite3`, as described below.
+
+A difficulty is to always choose subscriber IDs that are not yet in use. For an initial import, the subscriber ID may be
+incremented per subscriber record. If adding more subscribers to an existing database, it is necessary to choose
+subscriber IDs that are not yet in use. Get the highest ID in use with:
+
+----
+sqlite3 hlr.db 'select max(id) from subscriber'
+----
+
+A full SQL example of adding a single subscriber with id 23, IMSI 001010123456789, MSISDN 1234, Ki for COMP128v1, and K
+and OPC for Milenage:
+
+----
+INSERT subscriber (id, imsi, msisdn) VALUES (23, '001010123456789', '1234');
+
+INSERT INTO auc_2g (subscriber_id, algo_id_2g, ki)
+VALUES(23, 1, '0123456789abcdef0123456789abcdef');
+
+INSERT INTO auc_3g (subscriber_id, algo_id_3g, k, op, opc)
+VALUES(23, 5, '0123456789abcdef0123456789abcdef',NULL,'0123456789abcdef0123456789abcdef');
+----
+
+Table entries to `auc_2g` and/or `auc_3g` may be omitted if no such key material is required.
+
+UMTS Milenage auth (on both 2G and 3G RAN) is configured by the `auc_3g` table. `algo_id_3g` must currently always be 5
+(MILENAGE).
+
+The algorithm IDs for `algo_id_2g` and `algo_id_3g` are:
+
+.Algorithm IDs in OsmoHLR's database
+[options="header",width="50%",cols="40%,60%"]
+|===
+|`algo_id_2g` / `algo_id_3g` | Authentication Algorithm
+| 1 | COMP128v1
+| 2 | COMP128v2
+| 3 | COMP128v3
+| 4 | XOR
+| 5 | MILENAGE
+|===
+
+Create an empty HLR database with
+
+----
+osmo-hlr-db-tool -l hlr.db create
+----
+
+Repeat above SQL commands per subscriber, incrementing the subscriber ID for each block, then feed the SQL commands for
+the subscribers to be imported to the `sqlite3` command line tool:
+
+----
+sqlite3 hlr.db < subscribers.sql
+----
+
+[[db_import_nitb]]
+==== Import OsmoNITB database
+
+To upgrade from old OsmoNITB to OsmoHLR, use `osmo-hlr-db-tool`:
+
+----
+osmo-hlr-db-tool -l hlr.db import-nitb-db nitb.db
+----
+
+Be aware that the import is lossy, only the IMSI, MSISDN, nam_cs/ps and 2G auth data are set.

doc/manuals/chapters/ussd.adoc

@@ -50,15 +50,29 @@ prefix route to the named EUSE.  All USSD short codes starting with *123 will be
 routed to the named EUSE.
 
 `ussd route prefix *#100# internal own-msisdn` installs a prefix route
-to the named internal USSD handler.  There above command will restore
+to the named internal USSD handler.  The above command will restore
 the old behavior, in which *#100# will return a text message containing
-the subscribers own phone number.  There is one other handler called
-`own-imsi` which will return the IMSI instead of the MSISDN.
+the subscribers own phone number.  More information on internal USSD
+handlers can be found in <<iuse_handlers>>.
 
 `ussd default-route external foobar-00-00-00-00-00-00` installs a
 default route to the named EUSE.  This means that all USSD codes for
 which no more specific route exists will be routed to the named EUSE.
 
+[[iuse_handlers]]
+=== Built-in USSD handlers
+
+OsmoHLR has an Internal USSD Entity (IUSE) that allows to handle some
+USSD requests internally.  It features a set of simple handlers, which
+can be assigned to one or more USSD request prefixes:
+
+* `own-msisdn` returns subscriber's MSISDN (if assigned);
+* `own-imsi` returns subscriber's IMSI;
+* `test-idle` keeps the session idle until the MS terminates it, or
+  the guard timer expires (may be useful for testing).
+
+Additional handlers can be added on request.
+
 === Example EUSE program
 
 We have provided an example EUSE developed in C language using existing

doc/manuals/osmohlr-usermanual.adoc

@@ -28,6 +28,8 @@ include::{srcdir}/chapters/dgsm.adoc[]
 
 include::./common/chapters/gsup.adoc[]
 
+include::./common/chapters/vty_cpu_sched.adoc[]
+
 include::./common/chapters/port_numbers.adoc[]
 
 include::./common/chapters/bibliography.adoc[]
@@ -35,4 +37,3 @@ include::./common/chapters/bibliography.adoc[]
 include::./common/chapters/glossary.adoc[]
 
 include::./common/chapters/gfdl.adoc[]
-

include/osmocom/hlr/db.h

@@ -8,6 +8,13 @@
 struct hlr;
 
 enum stmt_idx {
+	DB_STMT_SEL_ALL,
+	DB_STMT_SEL_ALL_ORDER_LAST_SEEN,
+	DB_STMT_SEL_FILTER_MSISDN,
+	DB_STMT_SEL_FILTER_IMSI,
+	DB_STMT_SEL_FILTER_IMEI,
+	DB_STMT_SEL_FILTER_CS,
+	DB_STMT_SEL_FILTER_PS,
 	DB_STMT_SEL_BY_IMSI,
 	DB_STMT_SEL_BY_MSISDN,
 	DB_STMT_SEL_BY_ID,
@@ -148,6 +155,9 @@ int db_subscr_update_imei_by_imsi(struct db_context *dbc, const char* imsi, cons
 int db_subscr_exists_by_imsi(struct db_context *dbc, const char *imsi);
 int db_subscr_exists_by_msisdn(struct db_context *dbc, const char *msisdn);
 
+int db_subscrs_get(struct db_context *dbc, const char *filter_type, const char *filter,
+		   void (*get_cb)(struct hlr_subscriber *subscr, void *data), void *data,
+		   int *count, const char **err);
 int db_subscr_get_by_imsi(struct db_context *dbc, const char *imsi,
 			  struct hlr_subscriber *subscr);
 int db_subscr_get_by_msisdn(struct db_context *dbc, const char *msisdn,

src/db.c

@@ -51,6 +51,14 @@
 	"sgsn_via_proxy"
 
 static const char *stmt_sql[] = {
+	[DB_STMT_SEL_ALL] = "SELECT " SEL_COLUMNS " FROM subscriber;",
+	[DB_STMT_SEL_ALL_ORDER_LAST_SEEN] = "SELECT " SEL_COLUMNS " FROM subscriber "
+		"WHERE last_lu_seen IS NOT NULL ORDER BY last_lu_seen;",
+	[DB_STMT_SEL_FILTER_MSISDN] = "SELECT " SEL_COLUMNS " FROM subscriber WHERE msisdn LIKE $search ORDER BY msisdn",
+	[DB_STMT_SEL_FILTER_IMSI] = "SELECT " SEL_COLUMNS " FROM subscriber WHERE imsi LIKE $search ORDER BY imsi",
+	[DB_STMT_SEL_FILTER_IMEI] = "SELECT " SEL_COLUMNS " FROM subscriber WHERE imei LIKE $search ORDER BY imei",
+	[DB_STMT_SEL_FILTER_CS] = "SELECT " SEL_COLUMNS " FROM subscriber WHERE nam_cs = $search ORDER BY last_lu_seen",
+	[DB_STMT_SEL_FILTER_PS] = "SELECT " SEL_COLUMNS " FROM subscriber WHERE nam_ps = $search ORDER BY last_lu_seen",
 	[DB_STMT_SEL_BY_IMSI] = "SELECT " SEL_COLUMNS " FROM subscriber WHERE imsi = ?",
 	[DB_STMT_SEL_BY_MSISDN] = "SELECT " SEL_COLUMNS " FROM subscriber WHERE msisdn = ?",
 	[DB_STMT_SEL_BY_ID] = "SELECT " SEL_COLUMNS " FROM subscriber WHERE id = ?",

src/db_hlr.c

@@ -264,11 +264,11 @@ int db_subscr_update_aud_by_id(struct db_context *dbc, int64_t subscr_id,
 		switch (aud->algo) {
 		case OSMO_AUTH_ALG_NONE:
 		case OSMO_AUTH_ALG_MILENAGE:
+		case OSMO_AUTH_ALG_XOR:
 			break;
 		case OSMO_AUTH_ALG_COMP128v1:
 		case OSMO_AUTH_ALG_COMP128v2:
 		case OSMO_AUTH_ALG_COMP128v3:
-		case OSMO_AUTH_ALG_XOR:
 			LOGP(DAUC, LOGL_ERROR, "Cannot update auth tokens:"
 			     " auth algo not suited for 3G: %s\n",
 			     osmo_auth_alg_name(aud->algo));
@@ -625,6 +625,94 @@ int db_subscr_get_by_msisdn(struct db_context *dbc, const char *msisdn,
 	return rc;
 }
 
+/*! Retrieve subscriber data from the HLR database.
+ * \param[in,out] dbc  database context.
+ * \param[in] filter_type  ASCII string of identifier type to search.
+ * \param[in] filter  ASCII string to search.
+ * \param[in] get_cb  pointer to call back function for data.
+ * \param[in,out] data  pointer to pass to callback function.
+ * \param[in,out] count  counter for number of matched subscribers.
+ * \param[in,our] err
+ * \returns 0 on success, -ENOENT if no subscriber was found, -EIO on
+ *          database error.
+ */
+int db_subscrs_get(struct db_context *dbc, const char *filter_type, const char *filter,
+		   void (*get_cb)(struct hlr_subscriber *subscr, void *data), void *data,
+		   int *count, const char **err)
+{
+	sqlite3_stmt *stmt;
+	char search[256];
+	int rc;
+	struct hlr_subscriber subscr;
+	bool show_ls = false;
+
+	if (!filter_type) {
+		stmt = dbc->stmt[DB_STMT_SEL_ALL];
+	} else if (strcmp(filter_type, "imei") == 0) {
+		stmt = dbc->stmt[DB_STMT_SEL_FILTER_IMEI];
+	} else if (strcmp(filter_type, "imsi") == 0) {
+		stmt = dbc->stmt[DB_STMT_SEL_FILTER_IMSI];
+	} else if (strcmp(filter_type, "msisdn") == 0) {
+		stmt = dbc->stmt[DB_STMT_SEL_FILTER_MSISDN];
+	} else if (strcmp(filter_type, "cs") == 0) {
+		stmt = dbc->stmt[DB_STMT_SEL_FILTER_CS];
+	} else if (strcmp(filter_type, "ps") == 0) {
+		stmt = dbc->stmt[DB_STMT_SEL_FILTER_PS];
+	} else if (strcmp(filter_type, "last_lu_seen") == 0) {
+		show_ls = true;
+		stmt = dbc->stmt[DB_STMT_SEL_ALL_ORDER_LAST_SEEN];
+	} else {
+		return -EIO;
+	}
+
+	if (filter_type && filter && strcmp(filter_type, "last_lu_seen") != 0) {
+		if (strcmp(filter, "on") == 0) {
+			sprintf(search, "%s", "1");
+		} else if (strcmp(filter, "off") == 0) {
+			sprintf(search, "%s", "0");
+		} else {
+			sprintf(search, "%%%s%%", filter);
+		}
+		if (!db_bind_text(stmt, "$search", search)) {
+			*err = sqlite3_errmsg(dbc->db);
+			return -EIO;
+		}
+	}
+
+	rc = sqlite3_step(stmt);
+
+	if (rc == SQLITE_DONE) {
+		db_remove_reset(stmt);
+		*err = "No matching subscriber(s)";
+		return -ENOENT;
+	}
+
+	while (rc == SQLITE_ROW) {
+		subscr = (struct hlr_subscriber){
+			  .id = sqlite3_column_int64(stmt, 0),};
+		copy_sqlite3_text_to_buf(subscr.imsi, stmt, 1);
+		copy_sqlite3_text_to_buf(subscr.msisdn, stmt, 2);
+		copy_sqlite3_text_to_buf(subscr.imei, stmt, 3);
+		subscr.nam_cs = sqlite3_column_int(stmt, 9);
+		subscr.nam_ps = sqlite3_column_int(stmt, 10);
+		if (show_ls)
+			parse_last_lu_seen(&subscr.last_lu_seen, (const char *)sqlite3_column_text(stmt, 14),
+					   subscr.imsi, "CS");
+		get_cb(&subscr, data);
+		rc = sqlite3_step(stmt);
+		(*count)++;
+	}
+
+	db_remove_reset(stmt);
+	if (rc != SQLITE_DONE) {
+		*err = sqlite3_errmsg(dbc->db);
+		LOGP(DAUC, LOGL_ERROR, "Cannot read subscribers from db:: %s\n", *err);
+		return rc;
+	}
+	*err = NULL;
+	return 0;
+}
+
 /*! Retrieve subscriber data from the HLR database.
  * \param[in,out] dbc  database context.
  * \param[in] id  ID of the subscriber in the HLR db.

src/dgsm_vty.c

@@ -442,7 +442,7 @@ int config_write_mslookup(struct vty *vty)
 
 		msc = mslookup_server_msc_get(&mslookup_server_msc_wildcard, false);
 		if (msc)
-			config_write_msc_services(vty, " ", msc);
+			config_write_msc_services(vty, "  ", msc);
 
 		llist_for_each_entry(msc, &g_hlr->mslookup.server.local_site_services, entry) {
 			if (!osmo_ipa_name_cmp(&mslookup_server_msc_wildcard, &msc->name))

src/gsup_server.c

@@ -18,6 +18,8 @@
  */
 
 #include <errno.h>
+#include <netinet/tcp.h>
+#include <netinet/in.h>
 
 #include <osmocom/core/msgb.h>
 #include <osmocom/core/logging.h>
@@ -352,6 +354,19 @@ void osmo_gsup_server_add_conn(struct llist_head *clients,
 	llist_add(&conn->list, &prev_conn->list);
 }
 
+static void update_fd_settings(int fd)
+{
+	int ret;
+	int val;
+
+	/*TODO: Set keepalive settings here. See OS#4312 */
+
+	val = 1;
+	ret = setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &val, sizeof(val));
+	if (ret < 0)
+		LOGP(DLGSUP, LOGL_ERROR, "Failed to set TCP_NODELAY: %s\n", strerror(errno));
+}
+
 /* a client has connected to the server socket and we have accept()ed it */
 static int osmo_gsup_server_accept_cb(struct ipa_server_link *link, int fd)
 {
@@ -376,6 +391,8 @@ static int osmo_gsup_server_accept_cb(struct ipa_server_link *link, int fd)
 	LOGP(DLGSUP, LOGL_INFO, "New GSUP client %s:%d (IND=%u)\n",
 	     conn->conn->addr, conn->conn->port, conn->auc_3g_ind);
 
+	update_fd_settings(fd);
+
 	/* request the identity of the client */
 	rc = ipa_ccm_send_id_req(fd);
 	if (rc < 0)
@@ -461,7 +478,7 @@ int osmo_gsup_configure_wildcard_apn(struct osmo_gsup_message *gsup,
 
 /**
  * Populate a gsup message structure with an Insert Subscriber Data Message.
- * All required memory buffers for data pointed to by pointers in struct omso_gsup_message
+ * All required memory buffers for data pointed to by pointers in struct osmo_gsup_message
  * must be allocated by the caller and should have the same lifetime as the gsup parameter.
  *
  * \param[out] gsup  The gsup message to populate.

src/gsupclient/Makefile.am

@@ -1,7 +1,7 @@
 # This is _NOT_ the library release version, it's an API version.
 # Please read chapter "Library interface versions" of the libtool documentation
 # before making any modifications: https://www.gnu.org/software/libtool/manual/html_node/Versioning.html
-LIBVERSION=0:0:0
+LIBVERSION=1:0:1
 
 AM_CFLAGS = -Wall $(all_includes) -I$(top_srcdir)/include -I$(top_builddir)/include \
 	    $(TALLOC_CFLAGS) $(LIBOSMOCORE_CFLAGS) $(LIBOSMOABIS_CFLAGS)

src/gsupclient/gsup_client.c

@@ -31,6 +31,8 @@
 
 #include <errno.h>
 #include <string.h>
+#include <netinet/tcp.h>
+#include <netinet/in.h>
 
 static void start_test_procedure(struct osmo_gsup_client *gsupc);
 
@@ -129,6 +131,19 @@ static void gsup_client_oap_register(struct osmo_gsup_client *gsupc)
 	client_send(gsupc, IPAC_PROTO_EXT_OAP, msg_tx);
 }
 
+static void update_fd_settings(int fd)
+{
+	int ret;
+	int val;
+
+	/*TODO: Set keepalive settings here. See OS#4312 */
+
+	val = 1;
+	ret = setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &val, sizeof(val));
+	if (ret < 0)
+		LOGP(DLGSUP, LOGL_ERROR, "Failed to set TCP_NODELAY: %s\n", strerror(errno));
+}
+
 static void gsup_client_updown_cb(struct ipa_client_conn *link, int up)
 {
 	struct osmo_gsup_client *gsupc = link->data;
@@ -139,6 +154,7 @@ static void gsup_client_updown_cb(struct ipa_client_conn *link, int up)
 	gsupc->is_connected = up;
 
 	if (up) {
+		update_fd_settings(link->ofd->fd);
 		start_test_procedure(gsupc);
 
 		if (gsupc->oap_state.state == OSMO_OAP_INITIALIZED)

src/hlr.c

@@ -31,6 +31,7 @@
 #include <osmocom/vty/command.h>
 #include <osmocom/vty/telnet_interface.h>
 #include <osmocom/vty/ports.h>
+#include <osmocom/vty/cpu_sched_vty.h>
 #include <osmocom/ctrl/control_vty.h>
 #include <osmocom/gsm/apn.h>
 #include <osmocom/gsm/gsm48_ie.h>
@@ -576,6 +577,10 @@ static void print_help()
 	printf("  -U --db-upgrade            Allow HLR database schema upgrades.\n");
 	printf("  -C --db-check              Quit after opening (and upgrading) the database.\n");
 	printf("  -V --version               Print the version of OsmoHLR.\n");
+
+	printf("\nVTY reference generation:\n");
+	printf("     --vty-ref-mode MODE        VTY reference generation mode (e.g. 'expert').\n");
+	printf("     --vty-ref-xml              Generate the VTY reference XML output and exit.\n");
 }
 
 static struct {
@@ -591,10 +596,37 @@ static struct {
 	.db_upgrade = false,
 };
 
+static void handle_long_options(const char *prog_name, const int long_option)
+{
+	static int vty_ref_mode = VTY_REF_GEN_MODE_DEFAULT;
+
+	switch (long_option) {
+	case 1:
+		vty_ref_mode = get_string_value(vty_ref_gen_mode_names, optarg);
+		if (vty_ref_mode < 0) {
+			fprintf(stderr, "%s: Unknown VTY reference generation "
+				"mode '%s'\n", prog_name, optarg);
+			exit(2);
+		}
+		break;
+	case 2:
+		fprintf(stderr, "Generating the VTY reference in mode '%s' (%s)\n",
+			get_value_string(vty_ref_gen_mode_names, vty_ref_mode),
+			get_value_string(vty_ref_gen_mode_desc, vty_ref_mode));
+		vty_dump_xml_ref_mode(stdout, (enum vty_ref_gen_mode) vty_ref_mode);
+		exit(0);
+	default:
+		fprintf(stderr, "%s: error parsing cmdline options\n", prog_name);
+		exit(2);
+	}
+
+}
+
 static void handle_options(int argc, char **argv)
 {
 	while (1) {
 		int option_index = 0, c;
+		static int long_option = 0;
 		static struct option long_options[] = {
 			{"help", 0, 0, 'h'},
 			{"config-file", 1, 0, 'c'},
@@ -607,6 +639,8 @@ static void handle_options(int argc, char **argv)
 			{"db-upgrade", 0, 0, 'U' },
 			{"db-check", 0, 0, 'C' },
 			{"version", 0, 0, 'V' },
+			{"vty-ref-mode", 1, &long_option, 1},
+			{"vty-ref-xml", 0, &long_option, 2},
 			{0, 0, 0, 0}
 		};
 
@@ -616,6 +650,9 @@ static void handle_options(int argc, char **argv)
 			break;
 
 		switch (c) {
+		case 0:
+			handle_long_options(argv[0], long_option);
+			break;
 		case 'h':
 			print_usage();
 			print_help();
@@ -729,9 +766,10 @@ int main(int argc, char **argv)
 	osmo_stats_init(hlr_ctx);
 	vty_init(&vty_info);
 	ctrl_vty_init(hlr_ctx);
-	handle_options(argc, argv);
 	hlr_vty_init();
 	dgsm_vty_init();
+	osmo_cpu_sched_vty_init(hlr_ctx);
+	handle_options(argc, argv);
 
 	rc = vty_read_config_file(cmdline_opts.config_file, NULL);
 	if (rc < 0) {

src/hlr_db_tool.c

@@ -25,6 +25,7 @@
 #include <getopt.h>
 #include <inttypes.h>
 #include <string.h>
+#include <errno.h>
 
 #include <osmocom/core/logging.h>
 #include <osmocom/core/application.h>
@@ -70,8 +71,9 @@ static void print_help()
 	printf("                             (All commands imply this if none exists yet.)\n");
 	printf("\n");
 	printf("  import-nitb-db <nitb.db>   Add OsmoNITB db's subscribers to OsmoHLR db.\n");
-	printf("                             Be aware that the import is lossy, only the\n");
-	printf("                             IMSI, MSISDN, nam_cs/ps and 2G auth data are set.\n");
+	printf("                             Be aware that the import is somewhat lossy, only the IMSI,\n");
+	printf("                             MSISDN, IMEI, nam_cs/ps, 2G auth data and last seen LU are set.\n");
+	printf("                             The most recently associated IMEI from the Equipment table is used.\n");
 }
 
 static void print_version(int print_copyright)
@@ -212,9 +214,15 @@ enum nitb_stmt {
 
 static const char *nitb_stmt_sql[] = {
 	[NITB_SELECT_SUBSCR] =
-		"SELECT imsi, id, extension, authorized"
-		" FROM Subscriber"
-		" ORDER BY id",
+		"SELECT s.imsi, s.id, s.extension, s.authorized,"
+		" SUBSTR(e.imei,0,15), STRFTIME('%s', s.expire_lu)"
+		" FROM Subscriber s LEFT JOIN"
+		" (SELECT imei, subscriber_id, MAX(Equipment.updated) AS updated"
+		" FROM Equipment,EquipmentWatch"
+		" WHERE Equipment.id = EquipmentWatch.equipment_id"
+		" GROUP BY EquipmentWatch.subscriber_id) e"
+		" ON e.subscriber_id = s.id"
+		" ORDER by s.id",
 	[NITB_SELECT_AUTH_KEYS] =
 		"SELECT algorithm_id, a3a8_ki from authkeys"
 		" WHERE subscriber_id = $subscr_id",
@@ -222,8 +230,65 @@ static const char *nitb_stmt_sql[] = {
 
 sqlite3_stmt *nitb_stmt[ARRAY_SIZE(nitb_stmt_sql)] = {};
 
+enum hlr_db_stmt {
+	HLR_DB_STMT_SET_IMPLICIT_LU_BY_IMSI,
+};
+
+static const char *hlr_db_stmt_sql[] = {
+	[HLR_DB_STMT_SET_IMPLICIT_LU_BY_IMSI] =
+		"UPDATE subscriber SET last_lu_seen = datetime($last_lu, 'unixepoch') WHERE imsi = $imsi",
+};
+
+sqlite3_stmt *hlr_db_stmt[ARRAY_SIZE(hlr_db_stmt_sql)] = {};
+
 size_t _dbd_decode_binary(const unsigned char *in, unsigned char *out);
 
+/*! Set a subscriber's LU timestamp in the HLR database.
+ * In normal operations there is never any need to explicitly
+ * update the value of last_lu_seen, so this function can live here.
+ *
+ * \param[in,out] dbc  database context.
+ * \param[in] imsi  ASCII string of IMSI digits
+ * \param[in] imei  ASCII string of identifier digits, or NULL to remove the IMEI.
+ * \returns 0 on success, -ENOENT when the given subscriber does not exist,
+ *         -EIO on database errors.
+ */
+int db_subscr_update_lu_by_imsi(struct db_context *dbc, const char* imsi, const int last_lu)
+{
+	int rc, ret = 0;
+
+	sqlite3_stmt *stmt = hlr_db_stmt[HLR_DB_STMT_SET_IMPLICIT_LU_BY_IMSI];
+
+	if (!db_bind_text(stmt, "$imsi", imsi))
+		return -EIO;
+	if (last_lu && !db_bind_int(stmt, "$last_lu", last_lu))
+		return -EIO;
+
+	/* execute the statement */
+	rc = sqlite3_step(stmt);
+	if (rc != SQLITE_DONE) {
+		LOGP(DAUC, LOGL_ERROR, "Update last_lu_seen for subscriber IMSI='%s': SQL Error: %s\n", imsi,
+		     sqlite3_errmsg(dbc->db));
+		ret = -EIO;
+		goto out;
+	}
+
+	/* verify execution result */
+	rc = sqlite3_changes(dbc->db);
+	if (!rc) {
+		LOGP(DAUC, LOGL_ERROR, "Cannot update last_lu_seen for subscriber IMSI='%s': no such subscriber\n", imsi);
+		ret = -ENOENT;
+	} else if (rc != 1) {
+		LOGP(DAUC, LOGL_ERROR, "Update last_lu_seen for subscriber IMSI='%s': SQL modified %d rows (expected 1)\n",
+		     imsi, rc);
+		ret = -EIO;
+	}
+
+out:
+	db_remove_reset(stmt);
+	return ret;
+}
+
 void import_nitb_subscr_aud(sqlite3 *nitb_db, const char *imsi, int64_t nitb_id, int64_t hlr_id)
 {
 	int rc;
@@ -297,6 +362,7 @@ void import_nitb_subscr(sqlite3 *nitb_db, sqlite3_stmt *stmt)
 	int64_t imsi;
 	char imsi_str[32];
 	bool authorized;
+	int last_lu_int;
 
 	imsi = sqlite3_column_int64(stmt, 0);
 
@@ -315,8 +381,18 @@ void import_nitb_subscr(sqlite3 *nitb_db, sqlite3_stmt *stmt)
 	nitb_id = sqlite3_column_int64(stmt, 1);
 	copy_sqlite3_text_to_buf(subscr.msisdn, stmt, 2);
 	authorized = sqlite3_column_int(stmt, 3) ? true : false;
+	copy_sqlite3_text_to_buf(subscr.imei, stmt, 4);
+	/* Default periodic LU was 30 mins and the expire_lu
+	 * was twice that + 1 min
+	 */
+	last_lu_int = sqlite3_column_int(stmt, 5) - 3660;
 
 	db_subscr_update_msisdn_by_imsi(dbc, imsi_str, subscr.msisdn);
+	/* In case the subscriber was somehow never seen, invent an IMEI */
+	if (strlen(subscr.imei) == 14)
+		db_subscr_update_imei_by_imsi(dbc, imsi_str, subscr.imei);
+	db_subscr_update_lu_by_imsi(dbc, imsi_str, last_lu_int);
+
 	db_subscr_nam(dbc, imsi_str, authorized, true);
 	db_subscr_nam(dbc, imsi_str, authorized, false);
 
@@ -361,6 +437,17 @@ int import_nitb_db(void)
 		}
 	}
 
+	for (i = 0; i < ARRAY_SIZE(hlr_db_stmt_sql); i++) {
+		sql = hlr_db_stmt_sql[i];
+		rc = sqlite3_prepare_v2(g_hlr_db_tool_ctx->dbc->db, hlr_db_stmt_sql[i], -1,
+					&hlr_db_stmt[i], NULL);
+		if (rc != SQLITE_OK) {
+			LOGP(DDB, LOGL_ERROR, "OsmoHLR DB: Unable to prepare SQL statement '%s'\n", sql);
+			ret = -1;
+			goto out_free;
+		}
+	}
+
 	stmt = nitb_stmt[NITB_SELECT_SUBSCR];
 
 	while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) {
@@ -387,6 +474,7 @@ int main(int argc, char **argv)
 {
 	int rc;
 	int (*main_action)(void);
+	int i;
 	main_action = NULL;
 
 	g_hlr_db_tool_ctx = talloc_zero(NULL, struct hlr_db_tool_ctx);
@@ -430,6 +518,11 @@ int main(int argc, char **argv)
 	if (main_action)
 		rc = (*main_action)();
 
+	/* db_close will only finalize statments in g_hlr_db_tool_ctx->dbc->stmt
+	 * it is ok to call finalize on NULL */
+	for (i = 0; i < ARRAY_SIZE(hlr_db_stmt); i++) {
+		sqlite3_finalize(hlr_db_stmt[i]);
+	}
 	db_close(g_hlr_db_tool_ctx->dbc);
 	log_fini();
 	exit(rc ? EXIT_FAILURE : EXIT_SUCCESS);

src/hlr_ussd.c

@@ -279,19 +279,20 @@ static int ss_gsup_send_to_ms(struct ss_session *ss, struct osmo_gsup_server *gs
 }
 
 static int ss_tx_to_ms(struct ss_session *ss, enum osmo_gsup_message_type gsup_msg_type,
-			bool final, struct msgb *ss_msg)
+		       struct msgb *ss_msg)
 
 {
-	struct osmo_gsup_message resp = {0};
+	struct osmo_gsup_message resp;
 	int rc;
 
-	resp.message_type = gsup_msg_type;
+	resp = (struct osmo_gsup_message) {
+		.message_type = gsup_msg_type,
+		.session_id = ss->session_id,
+		.session_state = ss->state,
+	};
+
 	OSMO_STRLCPY_ARRAY(resp.imsi, ss->imsi);
-	if (final)
-		resp.session_state = OSMO_GSUP_SESSION_STATE_END;
-	else
-		resp.session_state = OSMO_GSUP_SESSION_STATE_CONTINUE;
-	resp.session_id = ss->session_id;
+
 	if (ss_msg) {
 		resp.ss_info = msgb_data(ss_msg);
 		resp.ss_info_len = msgb_length(ss_msg);
@@ -311,7 +312,8 @@ static int ss_tx_reject(struct ss_session *ss, int invoke_id, uint8_t problem_ta
 	LOGPSS(ss, LOGL_NOTICE, "Tx Reject(%u, 0x%02x, 0x%02x)\n", invoke_id,
 		problem_tag, problem_code);
 	OSMO_ASSERT(msg);
-	return ss_tx_to_ms(ss, OSMO_GSUP_MSGT_PROC_SS_RESULT, true, msg);
+	ss->state = OSMO_GSUP_SESSION_STATE_END;
+	return ss_tx_to_ms(ss, OSMO_GSUP_MSGT_PROC_SS_RESULT, msg);
 }
 #endif
 
@@ -320,15 +322,16 @@ static int ss_tx_to_ms_error(struct ss_session *ss, uint8_t invoke_id, uint8_t e
 	struct msgb *msg = gsm0480_gen_return_error(invoke_id, error_code);
 	LOGPSS(ss, LOGL_NOTICE, "Tx ReturnError(%u, 0x%02x)\n", invoke_id, error_code);
 	OSMO_ASSERT(msg);
-	return ss_tx_to_ms(ss, OSMO_GSUP_MSGT_PROC_SS_RESULT, true, msg);
+	ss->state = OSMO_GSUP_SESSION_STATE_END;
+	return ss_tx_to_ms(ss, OSMO_GSUP_MSGT_PROC_SS_RESULT, msg);
 }
 
-static int ss_tx_to_ms_ussd_7bit(struct ss_session *ss, bool final, uint8_t invoke_id, const char *text)
+static int ss_tx_to_ms_ussd_7bit(struct ss_session *ss, uint8_t invoke_id, const char *text)
 {
 	struct msgb *msg = gsm0480_gen_ussd_resp_7bit(invoke_id, text);
 	LOGPSS(ss, LOGL_INFO, "Tx USSD '%s'\n", text);
 	OSMO_ASSERT(msg);
-	return ss_tx_to_ms(ss, OSMO_GSUP_MSGT_PROC_SS_RESULT, final, msg);
+	return ss_tx_to_ms(ss, OSMO_GSUP_MSGT_PROC_SS_RESULT, msg);
 }
 
 /***********************************************************************
@@ -344,6 +347,8 @@ static int handle_ussd_own_msisdn(struct ss_session *ss,
 	char buf[GSM0480_USSD_7BIT_STRING_LEN+1];
 	int rc;
 
+	ss->state = OSMO_GSUP_SESSION_STATE_END;
+
 	rc = db_subscr_get_by_imsi(g_hlr->dbc, ss->imsi, &subscr);
 	switch (rc) {
 	case 0:
@@ -351,7 +356,7 @@ static int handle_ussd_own_msisdn(struct ss_session *ss,
 			snprintf(buf, sizeof(buf), "You have no MSISDN!");
 		else
 			snprintf(buf, sizeof(buf), "Your extension is %s", subscr.msisdn);
-		ss_tx_to_ms_ussd_7bit(ss, true, req->invoke_id, buf);
+		ss_tx_to_ms_ussd_7bit(ss, req->invoke_id, buf);
 		break;
 	case -ENOENT:
 		ss_tx_to_ms_error(ss, req->invoke_id, GSM0480_ERR_CODE_UNKNOWN_SUBSCRIBER);
@@ -369,7 +374,21 @@ static int handle_ussd_own_imsi(struct ss_session *ss,
 {
 	char buf[GSM0480_USSD_7BIT_STRING_LEN+1];
 	snprintf(buf, sizeof(buf), "Your IMSI is %s", ss->imsi);
-	ss_tx_to_ms_ussd_7bit(ss, true, req->invoke_id, buf);
+	ss->state = OSMO_GSUP_SESSION_STATE_END;
+	ss_tx_to_ms_ussd_7bit(ss, req->invoke_id, buf);
+	return 0;
+}
+
+/* This handler just keeps the session idle unless the guard timer expires. */
+static int handle_ussd_test_idle(struct ss_session *ss,
+				 const struct osmo_gsup_message *gsup,
+				 const struct ss_request *req)
+{
+	char buf[GSM0480_USSD_7BIT_STRING_LEN + 1];
+	snprintf(buf, sizeof(buf), "Keeping your session idle, it will expire "
+		 "at most in %u seconds.", g_hlr->ncss_guard_timeout);
+	ss->state = OSMO_GSUP_SESSION_STATE_CONTINUE;
+	ss_tx_to_ms_ussd_7bit(ss, req->invoke_id, buf);
 	return 0;
 }
 
@@ -383,6 +402,10 @@ static const struct hlr_iuse hlr_iuses[] = {
 		.name = "own-imsi",
 		.handle_ussd = handle_ussd_own_imsi,
 	},
+	{
+		.name = "test-idle",
+		.handle_ussd = handle_ussd_test_idle,
+	},
 };
 
 const struct hlr_iuse *iuse_find(const char *name)
@@ -496,8 +519,9 @@ static int handle_ussd(struct ss_session *ss, bool is_euse_originated, const str
 		} else {
 			/* Handle internally */
 			ss->u.iuse->handle_ussd(ss, gsup, req);
-			/* Release session immediately */
-			ss_session_free(ss);
+			/* Release session if the handler has changed its state to END */
+			if (ss->state == OSMO_GSUP_SESSION_STATE_END)
+				ss_session_free(ss);
 		}
 	}
 

src/hlr_vty.c

@@ -102,6 +102,8 @@ static int config_write_hlr_gsup(struct vty *vty)
 	vty_out(vty, " gsup%s", VTY_NEWLINE);
 	if (g_hlr->gsup_bind_addr)
 		vty_out(vty, "  bind ip %s%s", g_hlr->gsup_bind_addr, VTY_NEWLINE);
+	if (g_hlr->gsup_unit_name.serno)
+		vty_out(vty, "  ipa-name %s%s", g_hlr->gsup_unit_name.serno, VTY_NEWLINE);
 	return CMD_SUCCESS;
 }
 
@@ -174,10 +176,11 @@ DEFUN(cfg_hlr_gsup_ipa_name,
 #define UROUTE_STR "Routing Configuration\n"
 #define PREFIX_STR "Prefix-Matching Route\n" "USSD Prefix\n"
 
-#define INT_CHOICE "(own-msisdn|own-imsi)"
+#define INT_CHOICE "(own-msisdn|own-imsi|test-idle)"
 #define INT_STR "Internal USSD Handler\n" \
 		"Respond with subscribers' own MSISDN\n" \
-		"Respond with subscribers' own IMSI\n"
+		"Respond with subscribers' own IMSI\n" \
+		"Keep the session idle (useful for testing)\n"
 
 #define EXT_STR "External USSD Handler\n" \
 		"Name of External USSD Handler (IPA CCM ID)\n"
@@ -305,7 +308,7 @@ DEFUN(cfg_no_euse, cfg_no_euse_cmd,
 {
 	struct hlr_euse *euse = euse_find(g_hlr, argv[0]);
 	if (!euse) {
-		vty_out(vty, "%% Cannot remove non-existant EUSE %s%s", argv[0], VTY_NEWLINE);
+		vty_out(vty, "%% Cannot remove non-existent EUSE %s%s", argv[0], VTY_NEWLINE);
 		return CMD_WARNING;
 	}
 	if (g_hlr->euse_default == euse) {

src/hlr_vty_subscr.c

@@ -44,13 +44,14 @@ static char *get_datestr(const time_t *t, char *buf, size_t bufsize)
 	return buf;
 }
 
-static void dump_last_lu_seen(struct vty *vty, const char *domain_label, time_t last_lu_seen)
+static void dump_last_lu_seen(struct vty *vty, const char *domain_label, time_t last_lu_seen, bool only_age)
 {
 	uint32_t age;
 	char datebuf[32];
 	if (!last_lu_seen)
 		return;
-	vty_out(vty, "    last LU seen on %s: %s", domain_label, get_datestr(&last_lu_seen, datebuf, sizeof(datebuf)));
+	if (!only_age)
+		vty_out(vty, "    last LU seen on %s: %s", domain_label, get_datestr(&last_lu_seen, datebuf, sizeof(datebuf)));
 	if (!timestamp_age(&last_lu_seen, &age))
 		vty_out(vty, " (invalid timestamp)%s", VTY_NEWLINE);
 	else {
@@ -64,7 +65,10 @@ static void dump_last_lu_seen(struct vty *vty, const char *domain_label, time_t
 		UNIT_AGO("h", 60*60);
 		UNIT_AGO("m", 60);
 		UNIT_AGO("s", 1);
-		vty_out(vty, " ago)%s", VTY_NEWLINE);
+		if (!only_age)
+			vty_out(vty, " ago)%s", VTY_NEWLINE);
+		else
+			vty_out(vty, " ago)");
 #undef UNIT_AGO
 	}
 }
@@ -108,8 +112,8 @@ static void subscr_dump_full_vty(struct vty *vty, struct hlr_subscriber *subscr)
 		vty_out(vty, "    PS disabled%s", VTY_NEWLINE);
 	if (subscr->ms_purged_ps)
 		vty_out(vty, "    PS purged%s", VTY_NEWLINE);
-	dump_last_lu_seen(vty, "CS", subscr->last_lu_seen);
-	dump_last_lu_seen(vty, "PS", subscr->last_lu_seen_ps);
+	dump_last_lu_seen(vty, "CS", subscr->last_lu_seen, false);
+	dump_last_lu_seen(vty, "PS", subscr->last_lu_seen_ps, false);
 
 	if (!*subscr->imsi)
 		return;
@@ -159,6 +163,28 @@ static void subscr_dump_full_vty(struct vty *vty, struct hlr_subscriber *subscr)
 	}
 }
 
+static void subscr_dump_summary_vty(struct hlr_subscriber *subscr, void *data)
+{
+	struct vty *vty = data;
+	vty_out(vty, "%-5"PRIu64"  %-12s  %-16s", subscr->id,
+		*subscr->msisdn ? subscr->msisdn : "none",
+		*subscr->imsi ? subscr->imsi : "none");
+
+	if (*subscr->imei) {
+		char checksum = osmo_luhn(subscr->imei, 14);
+		if (checksum == -EINVAL)
+			vty_out(vty, "  %-14s (INVALID LENGTH!)", subscr->imei);
+		else
+			vty_out(vty, "  %-14s%c", subscr->imei, checksum);
+	} else {
+		vty_out(vty,"   ------------- ");
+	}
+	vty_out(vty, "   %-2s%-2s  ", subscr->nam_cs ? "CS" : "", subscr->nam_ps ? "PS" : "");
+	if (subscr->last_lu_seen)
+		dump_last_lu_seen(vty, "CS", subscr->last_lu_seen, true);
+	vty_out_newline(vty);
+}
+
 static int get_subscr_by_argv(struct vty *vty, const char *type, const char *id, struct hlr_subscriber *subscr)
 {
 	char imei_buf[GSM23003_IMEI_NUM_DIGITS_NO_CHK+1];
@@ -186,10 +212,52 @@ static int get_subscr_by_argv(struct vty *vty, const char *type, const char *id,
 	return rc;
 }
 
+static void dump_summary_table_vty(struct vty *vty, bool header, bool show_ls)
+{
+	const char *texts = "ID     MSISDN        IMSI              IMEI              NAM";
+	const char *lines = "-----  ------------  ----------------  ----------------  -----";
+	const char *ls_text = "    LAST SEEN";
+	const char *ls_line = "  ------------";
+	if (header) {
+		if (!show_ls)
+			vty_out(vty, "%s%s%s%s", texts, VTY_NEWLINE, lines, VTY_NEWLINE);
+		else
+			vty_out(vty, "%s%s%s%s%s%s", texts, ls_text, VTY_NEWLINE, lines, ls_line, VTY_NEWLINE);
+	} else {
+		if (!show_ls)
+			vty_out(vty, "%s%s%s%s", lines, VTY_NEWLINE, texts, VTY_NEWLINE);
+		else
+			vty_out(vty, "%s%s%s%s%s%s", lines, ls_line, VTY_NEWLINE, texts, ls_text, VTY_NEWLINE);
+	}
+}
+
+static int get_subscrs(struct vty *vty, const char *filter_type, const char *filter)
+{
+	int rc = -1;
+	int count = 0;
+	const char *err;
+	bool show_ls = (filter_type && strcmp(filter_type, "last_lu_seen") == 0);
+	dump_summary_table_vty(vty, true, show_ls);
+	rc = db_subscrs_get(g_hlr->dbc, filter_type, filter, subscr_dump_summary_vty, vty, &count, &err);
+	if (count > 40) {
+		dump_summary_table_vty(vty, false, show_ls);
+	}
+	if (count > 0)
+		vty_out(vty, " Subscribers Shown: %d%s", count, VTY_NEWLINE);
+	if (rc)
+		vty_out(vty, "%% %s%s", err, VTY_NEWLINE);
+	return rc;
+}
+
+
 #define SUBSCR_CMD "subscriber "
 #define SUBSCR_CMD_HELP "Subscriber management commands\n"
+#define SUBSCR_SHOW_HELP "Show subscriber information\n"
+#define SUBSCRS_SHOW_HELP "Show all subscribers (with filter possibility)\n"
 
 #define SUBSCR_ID "(imsi|msisdn|id|imei) IDENT"
+#define SUBSCR_FILTER "(imei|imsi|msisdn) FILTER"
+
 #define SUBSCR_ID_HELP \
 	"Identify subscriber by IMSI\n" \
 	"Identify subscriber by MSISDN (phone number)\n" \
@@ -207,7 +275,7 @@ static int get_subscr_by_argv(struct vty *vty, const char *type, const char *id,
 DEFUN(subscriber_show,
       subscriber_show_cmd,
       SUBSCR "show",
-      SUBSCR_HELP "Show subscriber information\n")
+      SUBSCR_HELP SUBSCR_SHOW_HELP)
 {
 	struct hlr_subscriber subscr;
 	const char *id_type = argv[0];
@@ -222,7 +290,50 @@ DEFUN(subscriber_show,
 
 ALIAS(subscriber_show, show_subscriber_cmd,
       "show " SUBSCR_CMD SUBSCR_ID,
-      SHOW_STR SUBSCR_CMD_HELP SUBSCR_ID_HELP);
+      SHOW_STR SUBSCR_SHOW_HELP SUBSCR_ID_HELP);
+
+DEFUN(show_subscriber_all,
+      show_subscriber_all_cmd,
+      "show subscribers all",
+      SHOW_STR SUBSCRS_SHOW_HELP "Show summary of all subscribers\n")
+{
+	if (get_subscrs(vty, NULL, NULL))
+		return CMD_WARNING;
+
+	return CMD_SUCCESS;
+}
+
+DEFUN(show_subscriber_filtered,
+      show_subscriber_filtered_cmd,
+      "show subscribers " SUBSCR_FILTER,
+      SHOW_STR SUBSCRS_SHOW_HELP
+      "Filter Subscribers by IMEI\n" "Filter Subscribers by IMSI\n" "Filter Subscribers by MSISDN\n"
+      "String to match in imei, imsi or msisdn\n")
+{
+	const char *filter_type = argv[0];
+	const char *filter = argv[1];
+
+	if (get_subscrs(vty, filter_type, filter))
+		return CMD_WARNING;
+
+	return CMD_SUCCESS;
+}
+
+ALIAS(show_subscriber_filtered, show_subscriber_filtered_cmd2,
+      "show subscribers (cs|ps) (on|off)",
+      SHOW_STR SUBSCR_SHOW_HELP
+      "Filter Subscribers by CS Network Access Mode\n" "Filter Subscribers by PS Network Access Mode\n"
+      "Authorised\n" "Not Authorised\n");
+
+DEFUN(show_subscriber_order_last_seen, show_subscriber_order_last_seen_cmd,
+      "show subscribers last-seen",
+      SHOW_STR SUBSCR_SHOW_HELP "Show Subscribers Ordered by Last Seen Time\n")
+{
+	if (get_subscrs(vty, "last_lu_seen", NULL))
+		return CMD_WARNING;
+
+	return CMD_SUCCESS;
+}
 
 DEFUN(subscriber_create,
       subscriber_create_cmd,
@@ -551,6 +662,55 @@ DEFUN(subscriber_aud3g,
 	return CMD_SUCCESS;
 }
 
+DEFUN(subscriber_aud3g_xor,
+      subscriber_aud3g_xor_cmd,
+      SUBSCR_UPDATE "aud3g xor k K"
+      " [ind-bitlen] [<0-28>]",
+      SUBSCR_UPDATE_HELP
+      "Set UMTS authentication data (3G, and 2G with UMTS AKA)\n"
+      "Use XOR algorithm\n"
+      "Set Encryption Key K\n" "K as 32 hexadecimal characters\n"
+      "Set IND bit length\n" "IND bit length value (default: 5)\n")
+{
+	struct hlr_subscriber subscr;
+	int minlen = 0;
+	int maxlen = 0;
+	int rc;
+	const char *id_type = argv[0];
+	const char *id = argv[1];
+	const char *k = argv[2];
+	int ind_bitlen = argc > 4? atoi(argv[4]) : 5;
+	struct sub_auth_data_str aud3g = {
+		.type = OSMO_AUTH_TYPE_UMTS,
+		.u.umts = {
+			.k = k,
+			.opc_is_op = 0,
+			.opc = "00000000000000000000000000000000",
+			.ind_bitlen = ind_bitlen,
+		},
+	};
+
+	if (!auth_algo_parse("xor", &aud3g.algo, &minlen, &maxlen)) {
+		vty_out(vty, "%% Unknown auth algorithm: '%s'%s", "xor", VTY_NEWLINE);
+		return CMD_WARNING;
+	}
+
+	if (!is_hexkey_valid(vty, "K", aud3g.u.umts.k, minlen, maxlen))
+		return CMD_WARNING;
+
+	if (get_subscr_by_argv(vty, id_type, id, &subscr))
+		return CMD_WARNING;
+
+	rc = db_subscr_update_aud_by_id(g_hlr->dbc, subscr.id, &aud3g);
+
+	if (rc) {
+		vty_out(vty, "%% Error: cannot set 3G auth data for IMSI='%s'%s",
+			subscr.imsi, VTY_NEWLINE);
+		return CMD_WARNING;
+	}
+	return CMD_SUCCESS;
+}
+
 DEFUN(subscriber_imei,
       subscriber_imei_cmd,
       SUBSCR_UPDATE "imei (none|IMEI)",
@@ -628,6 +788,10 @@ DEFUN(subscriber_nam,
 
 void hlr_vty_subscriber_init(void)
 {
+	install_element_ve(&show_subscriber_all_cmd);
+	install_element_ve(&show_subscriber_filtered_cmd);
+	install_element_ve(&show_subscriber_filtered_cmd2);
+	install_element_ve(&show_subscriber_order_last_seen_cmd);
 	install_element_ve(&subscriber_show_cmd);
 	install_element_ve(&show_subscriber_cmd);
 	install_element(ENABLE_NODE, &subscriber_create_cmd);
@@ -637,6 +801,7 @@ void hlr_vty_subscriber_init(void)
 	install_element(ENABLE_NODE, &subscriber_aud2g_cmd);
 	install_element(ENABLE_NODE, &subscriber_no_aud3g_cmd);
 	install_element(ENABLE_NODE, &subscriber_aud3g_cmd);
+	install_element(ENABLE_NODE, &subscriber_aud3g_xor_cmd);
 	install_element(ENABLE_NODE, &subscriber_imei_cmd);
 	install_element(ENABLE_NODE, &subscriber_nam_cmd);
 }

src/mslookup/osmo-mslookup-client.c

@@ -487,7 +487,7 @@ static int socket_cb(struct osmo_fd *ofd, unsigned int flags)
 {
 	int rc = 0;
 
-	if (flags & BSC_FD_READ)
+	if (flags & OSMO_FD_READ)
 		rc = socket_read_cb(ofd);
 	if (rc < 0)
 		return rc;
@@ -512,7 +512,7 @@ int socket_accept(struct osmo_fd *ofd, unsigned int flags)
 	c = talloc_zero(globals.ctx, struct socket_client);
 	OSMO_ASSERT(c);
 	c->ofd.fd = rc;
-	c->ofd.when = BSC_FD_READ;
+	c->ofd.when = OSMO_FD_READ;
 	c->ofd.cb = socket_cb;
 	c->ofd.data = c;
 
@@ -543,7 +543,7 @@ int socket_init(const char *sock_path)
 		return -1;
 	}
 
-	ofd->when = BSC_FD_READ;
+	ofd->when = OSMO_FD_READ;
 	ofd->cb = socket_accept;
 
 	rc = osmo_fd_register(ofd);
@@ -584,11 +584,11 @@ void respond_result(const char *query_str, const struct osmo_mslookup_result *r)
 	llist_for_each_entry_safe(c, n, &globals.socket_clients, entry) {
 		if (!strcmp(query_str, c->query_str)) {
 			socket_client_respond_result(c, g_buf);
-			if (r->last)
+			if (!r || r->last)
 				socket_client_close(c);
 		}
 	}
-	if (r->last)
+	if (!r || r->last)
 		globals.requests_handled++;
 }
 

tests/auc/auc_test.c

@@ -113,6 +113,7 @@ int rand_get(uint8_t *rand, unsigned int len)
 	return len;
 }
 
+/* Subscriber with 2G-only (COMP128v1) authentication data */
 static void test_gen_vectors_2g_only(void)
 {
 	struct osmo_sub_auth_data aud2g;
@@ -174,6 +175,8 @@ static void test_gen_vectors_2g_only(void)
 	comment_end();
 }
 
+/* Subscriber with separate 2G (COMP128v1) and 3G (MILENAGE) authentication data,
+ * reflects the default configuration of sysmoUSIM-SJS1 */
 static void test_gen_vectors_2g_plus_3g(void)
 {
 	struct osmo_sub_auth_data aud2g;
@@ -284,6 +287,9 @@ void _test_gen_vectors_3g_only__expect_vecs(struct osmo_auth_vector vecs[3])
 	      );
 }
 
+/* Subscriber with only 3G (MILENAGE) authentication data,
+ * reflects the default configuration of sysmoISIM-SJA2. Resulting
+ * tuples are suitable for both 2G and 3G authentication */
 static void test_gen_vectors_3g_only(void)
 {
 	struct osmo_sub_auth_data aud2g;
@@ -454,6 +460,55 @@ static void test_gen_vectors_3g_only(void)
 	comment_end();
 }
 
+/* Subscriber with only 3G (XOR) authentication data,
+ * reflects the default configuration of sysmoTSIM-SJAx as well
+ * as many "Test USIM" cards. Resulting tuples are suitable for both
+ * 2G and 3G authentication */
+static void test_gen_vectors_3g_xor(void)
+{
+	struct osmo_sub_auth_data aud2g;
+	struct osmo_sub_auth_data aud3g;
+	struct osmo_auth_vector vec;
+	int rc;
+
+	comment_start();
+
+	aud2g = (struct osmo_sub_auth_data){ 0 };
+
+	aud3g = (struct osmo_sub_auth_data){
+		.type = OSMO_AUTH_TYPE_UMTS,
+		.algo = OSMO_AUTH_ALG_XOR,
+		.u.umts.sqn = 0,
+	};
+
+	osmo_hexparse("000102030405060708090a0b0c0d0e0f",
+		      aud3g.u.umts.k, sizeof(aud3g.u.umts.k));
+	osmo_hexparse("00000000000000000000000000000000",
+		      aud3g.u.umts.opc, sizeof(aud3g.u.umts.opc));
+	next_rand("b5039c57e4a75051551d1a390a71ce48", true);
+
+	vec = (struct osmo_auth_vector){ {0} };
+	VERBOSE_ASSERT(aud3g.u.umts.sqn, == 0, "%"PRIu64);
+	rc = auc_compute_vectors(&vec, 1, &aud2g, &aud3g, NULL, NULL);
+	VERBOSE_ASSERT(rc, == 1, "%d");
+	VERBOSE_ASSERT(aud3g.u.umts.sqn, == 0, "%"PRIu64);
+
+	VEC_IS(&vec,
+	       "  rand: b5039c57e4a75051551d1a390a71ce48\n"
+	       "  autn: 54e0a256565d0000b5029e54e0a25656\n"
+	       "  ck: 029e54e0a256565d141032067cc047b5\n"
+	       "  ik: 9e54e0a256565d141032067cc047b502\n"
+	       "  res: b5029e54e0a256565d141032067cc047\n"
+	       "  res_len: 10\n"
+	       "  kc: 98e880384887f9fe\n"
+	       "  sres: 0ec81877\n"
+	       "  auth_types: 03000000\n"
+	      );
+
+	comment_end();
+}
+
+/* Test a variety of invalid authentication data combinations */
 void test_gen_vectors_bad_args()
 {
 	struct osmo_auth_vector vec;
@@ -613,15 +668,20 @@ int main(int argc, char **argv)
 	void *tall_ctx = talloc_named_const(NULL, 1, "auc_test");
 
 	osmo_init_logging2(tall_ctx, &hlr_log_info);
-	log_set_print_filename(osmo_stderr_target, cmdline_opts.verbose);
+	log_set_print_filename2(osmo_stderr_target,
+				cmdline_opts.verbose ?
+					LOG_FILENAME_BASENAME :
+					LOG_FILENAME_NONE);
 	log_set_print_timestamp(osmo_stderr_target, 0);
 	log_set_use_color(osmo_stderr_target, 0);
+	log_set_print_category_hex(osmo_stderr_target, 0);
 	log_set_print_category(osmo_stderr_target, 1);
 	log_parse_category_mask(osmo_stderr_target, "DMAIN,1:DDB,1:DAUC,1");
 
 	test_gen_vectors_2g_only();
 	test_gen_vectors_2g_plus_3g();
 	test_gen_vectors_3g_only();
+	test_gen_vectors_3g_xor();
 	test_gen_vectors_bad_args();
 
 	printf("Done\n");

tests/auc/auc_test.err

@@ -217,6 +217,29 @@ DAUC vector [2]: auth_types = 0x3
 ===== test_gen_vectors_3g_only: SUCCESS
 
 
+===== test_gen_vectors_3g_xor
+aud3g.u.umts.sqn == 0
+DAUC Computing 1 auth vector: 3G only (2G derived from 3G keys)
+DAUC 3G: k = 000102030405060708090a0b0c0d0e0f
+DAUC 3G: opc = 00000000000000000000000000000000
+DAUC 3G: for sqn ind 0, previous sqn was 0
+DAUC vector [0]: rand = b5039c57e4a75051551d1a390a71ce48
+DAUC vector [0]: sqn = 0
+DAUC vector [0]: autn = 54e0a256565d0000b5029e54e0a25656
+DAUC vector [0]: ck = 029e54e0a256565d141032067cc047b5
+DAUC vector [0]: ik = 9e54e0a256565d141032067cc047b502
+DAUC vector [0]: res = b5029e54e0a256565d141032067cc047
+DAUC vector [0]: res_len = 16
+DAUC vector [0]: deriving 2G from 3G
+DAUC vector [0]: kc = 98e880384887f9fe
+DAUC vector [0]: sres = 0ec81877
+DAUC vector [0]: auth_types = 0x3
+rc == 1
+aud3g.u.umts.sqn == 0
+vector matches expectations
+===== test_gen_vectors_3g_xor: SUCCESS
+
+
 ===== test_gen_vectors_bad_args
 
 - no auth data (a)

tests/auc/gen_ts_55_205_test_sets/main_template.c

@@ -106,9 +106,10 @@ int main()
 	void *tall_ctx = talloc_named_const(NULL, 1, "test");
 	msgb_talloc_ctx_init(tall_ctx, 0);
 	osmo_init_logging2(tall_ctx, &hlr_log_info);
-	log_set_print_filename(osmo_stderr_target, 0);
+	log_set_print_filename2(osmo_stderr_target, LOG_FILENAME_NONE);
 	log_set_print_timestamp(osmo_stderr_target, 0);
 	log_set_use_color(osmo_stderr_target, 0);
+	log_set_print_category_hex(osmo_stderr_target, 0);
 	log_set_print_category(osmo_stderr_target, 1);
 	log_parse_category_mask(osmo_stderr_target, "DMAIN,1:DDB,1:DAUC,1");
 

tests/db/db_test.c

@@ -980,9 +980,13 @@ int main(int argc, char **argv)
 	handle_options(argc, argv);
 
 	osmo_init_logging2(ctx, &hlr_log_info);
-	log_set_print_filename(osmo_stderr_target, cmdline_opts.verbose);
+	log_set_print_filename2(osmo_stderr_target,
+				cmdline_opts.verbose ?
+					LOG_FILENAME_BASENAME :
+					LOG_FILENAME_NONE);
 	log_set_print_timestamp(osmo_stderr_target, 0);
 	log_set_use_color(osmo_stderr_target, 0);
+	log_set_print_category_hex(osmo_stderr_target, 0);
 	log_set_print_category(osmo_stderr_target, 1);
 	log_parse_category_mask(osmo_stderr_target, "DMAIN,1:DDB,1:DAUC,1");
 

tests/gsup/gsup_test.c

@@ -101,7 +101,7 @@ int main(int argc, char **argv)
 {
 	ctx = talloc_named_const(NULL, 0, "gsup_test");
 	osmo_init_logging2(ctx, &info);
-	log_set_print_filename(osmo_stderr_target, 0);
+	log_set_print_filename2(osmo_stderr_target, LOG_FILENAME_NONE);
 	log_set_print_timestamp(osmo_stderr_target, 0);
 	log_set_use_color(osmo_stderr_target, 0);
 	log_set_print_category(osmo_stderr_target, 1);

tests/mslookup/mdns_test.c

@@ -583,7 +583,7 @@ int main()
 	void *ctx = talloc_named_const(NULL, 0, "main");
 	osmo_init_logging2(ctx, NULL);
 
-	log_set_print_filename(osmo_stderr_target, 0);
+	log_set_print_filename2(osmo_stderr_target, LOG_FILENAME_NONE);
 	log_set_print_level(osmo_stderr_target, 1);
 	log_set_print_category(osmo_stderr_target, 1);
 	log_set_print_category_hex(osmo_stderr_target, 0);

tests/mslookup/mslookup_client_mdns_test.c

@@ -233,7 +233,7 @@ int main()
 	ctx = talloc_named_const(NULL, 0, "main");
 	osmo_init_logging2(ctx, NULL);
 
-	log_set_print_filename(osmo_stderr_target, 0);
+	log_set_print_filename2(osmo_stderr_target, LOG_FILENAME_NONE);
 	log_set_print_level(osmo_stderr_target, 0);
 	log_set_print_category(osmo_stderr_target, 0);
 	log_set_print_category_hex(osmo_stderr_target, 0);

tests/mslookup/mslookup_client_test.c

@@ -174,7 +174,7 @@ int main()
 	ctx = talloc_named_const(NULL, 0, "main");
 	osmo_init_logging2(ctx, NULL);
 
-	log_set_print_filename(osmo_stderr_target, 0);
+	log_set_print_filename2(osmo_stderr_target, LOG_FILENAME_NONE);
 	log_set_print_level(osmo_stderr_target, 0);
 	log_set_print_category(osmo_stderr_target, 0);
 	log_set_print_category_hex(osmo_stderr_target, 0);

tests/mslookup/mslookup_test.c

@@ -73,7 +73,7 @@ int main()
 	ctx = talloc_named_const(NULL, 0, "main");
 	osmo_init_logging2(ctx, NULL);
 
-	log_set_print_filename(osmo_stderr_target, 0);
+	log_set_print_filename2(osmo_stderr_target, LOG_FILENAME_NONE);
 	log_set_print_level(osmo_stderr_target, 0);
 	log_set_print_category(osmo_stderr_target, 0);
 	log_set_print_category_hex(osmo_stderr_target, 0);

tests/test_nodes.vty

@@ -13,9 +13,14 @@ OsmoHLR> ?
 OsmoHLR> list
 ...
   show gsup-connections
+  show subscribers all
+  show subscribers (imei|imsi|msisdn) FILTER
+  show subscribers (cs|ps) (on|off)
+  show subscribers last-seen
   subscriber (imsi|msisdn|id|imei) IDENT show
   show subscriber (imsi|msisdn|id|imei) IDENT
   show mslookup services
+...
 
 OsmoHLR> enable
 OsmoHLR# ?
@@ -25,12 +30,15 @@ OsmoHLR# ?
 OsmoHLR# configure terminal
 OsmoHLR(config)# ?
 ...
-  hlr       Configure the HLR
-  mslookup  Configure Distributed GSM mslookup
+  hlr        Configure the HLR
+  mslookup   Configure Distributed GSM mslookup
+...
+
 OsmoHLR(config)# list
 ...
   hlr
   mslookup
+...
 
 OsmoHLR(config)# hlr
 OsmoHLR(config-hlr)# ?
@@ -47,7 +55,7 @@ OsmoHLR(config-hlr)# list
   database PATH
   euse NAME
   no euse NAME
-  ussd route prefix PREFIX internal (own-msisdn|own-imsi)
+  ussd route prefix PREFIX internal (own-msisdn|own-imsi|test-idle)
   ussd route prefix PREFIX external EUSE
   no ussd route prefix PREFIX
   ussd default-route external EUSE
@@ -89,7 +97,7 @@ log stderr
  logging level main notice
  logging level db notice
  logging level auc notice
- logging level ss info
+ logging level ss notice
  logging level mslookup notice
  logging level lu notice
  logging level dgsm notice
@@ -99,6 +107,7 @@ hlr
  database hlr_vty_test.db
  gsup
   bind ip 127.0.0.1
+  ipa-name unnamed-HLR
  ussd route prefix *#100# internal own-msisdn
  ussd route prefix *#101# internal own-imsi
 end
@@ -337,9 +346,9 @@ OsmoHLR(config-mslookup-server-msc)# show running-config
 mslookup
  server
   mdns bind 239.192.23.42 4266
- service foo.bar at 123.45.67.89 1011
- service baz.bar at 121.31.41.5 1617
- service baz.bar at a:b:c::d 1819
+  service foo.bar at 123.45.67.89 1011
+  service baz.bar at 121.31.41.5 1617
+  service baz.bar at a:b:c::d 1819
  msc MSC-1
  msc msc-901-70-23
   service foo.bar at 76.54.32.10 1234
@@ -393,8 +402,8 @@ OsmoHLR(config-mslookup-client)# show running-config
 mslookup
  server
   mdns bind 239.192.23.42 4266
- service foo.bar at 123.45.67.89 1011
- service baz.bar at 121.31.41.5 1617
+  service foo.bar at 123.45.67.89 1011
+  service baz.bar at 121.31.41.5 1617
  msc MSC-1
  msc msc-901-70-23
   service foo.bar at 76.54.32.10 1234
@@ -425,9 +434,9 @@ OsmoHLR(config-mslookup-server)# show running-config
 mslookup
  server
   mdns bind 239.192.23.42 4266
- service foo.bar at 123.45.67.89 1011
- service baz.bar at 121.31.41.5 1617
- service gsup.hlr at 23.42.17.11 4223
+  service foo.bar at 123.45.67.89 1011
+  service baz.bar at 121.31.41.5 1617
+  service gsup.hlr at 23.42.17.11 4223
  msc MSC-1
  msc msc-901-70-23
   service foo.bar at 76.54.32.10 1234

tests/test_subscriber.vty

@@ -11,9 +11,11 @@ OsmoHLR# list
   subscriber (imsi|msisdn|id|imei) IDENT update aud2g (comp128v1|comp128v2|comp128v3|xor) ki KI
   subscriber (imsi|msisdn|id|imei) IDENT update aud3g none
   subscriber (imsi|msisdn|id|imei) IDENT update aud3g milenage k K (op|opc) OP_C [ind-bitlen] [<0-28>]
+  subscriber (imsi|msisdn|id|imei) IDENT update aud3g xor k K [ind-bitlen] [<0-28>]
   subscriber (imsi|msisdn|id|imei) IDENT update imei (none|IMEI)
   subscriber (imsi|msisdn|id|imei) IDENT update network-access-mode (none|cs|ps|cs+ps)
   show mslookup services
+...
 
 OsmoHLR# subscriber?
   subscriber  Subscriber management commands
@@ -267,6 +269,7 @@ OsmoHLR# subscriber id 101 show
 OsmoHLR# subscriber imsi 123456789023000 update aud3g ?
   none      Delete 3G authentication data
   milenage  Use Milenage algorithm
+  xor       Use XOR algorithm
 
 OsmoHLR# subscriber imsi 123456789023000 update aud3g milenage ?
   k  Set Encryption Key K