tests/auc: change back to python3

Change the interpreter of the python script back to python3, as it was when the script was initially added in Idff9d757ab956179aa41ada2a223fd9f439aafbd. In the meantime, it had been changed to python2 to make it work with build slaves that were missing python3, but this is not necessary anymore. This should be merged shortly after osmo-python-tests was migrated to python3, and the jenkins build slaves were (automatically) updated to have the new osmo-python-tests installed. Related: OS#2819 Depends: osmo-python-tests I3ffc3519bf6c22536a49dad7a966188ddad351a7 Change-Id: Ifbb8f8f5428657a1c2d4d6d1217f8e374be43aba
hlr: exit(2) on unsupported positional arguments on command line
2025-11-03 05:33:28 +00:00 · 2019-12-10 14:37:25 +01:00 · 2019-12-03 22:17:00 +01:00 · 2019-12-03 14:51:38 +01:00 · 2019-12-01 16:09:16 +01:00 · 2019-11-27 03:19:06 +01:00
68 changed files with 1469 additions and 509 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,8 @@
 *.lo
 *.la
 *.db
+*.db-shm
+*.db-wal
 *.pyc
 .*.sw?
 .version
@@ -38,6 +40,7 @@ src/gsupclient/gsup-test-client
 tests/atconfig
 tests/testsuite
 tests/testsuite.log
+tests/testsuite.dir

 tests/auc/auc_3g_test
 tests/auc/auc_ts_55_205_test_sets.c
@@ -47,6 +50,7 @@ tests/gsup_server/gsup_server_test
 tests/gsup/gsup_test
 tests/db/db_test
 tests/hlr_vty_test.db*
+tests/db_upgrade/*.dump

 # manuals
 doc/manuals/*.html
--- a/configure.ac
+++ b/configure.ac
@@ -34,10 +34,10 @@ PKG_PROG_PKG_CONFIG([0.20])

 PKG_CHECK_MODULES(TALLOC, [talloc >= 2.0.1])

-PKG_CHECK_MODULES(LIBOSMOCORE, libosmocore >= 1.0.0)
-PKG_CHECK_MODULES(LIBOSMOGSM, libosmogsm >= 1.0.0)
-PKG_CHECK_MODULES(LIBOSMOVTY, libosmovty >= 1.0.0)
-PKG_CHECK_MODULES(LIBOSMOCTRL, libosmoctrl >= 1.0.0)
+PKG_CHECK_MODULES(LIBOSMOCORE, libosmocore >= 1.2.0)
+PKG_CHECK_MODULES(LIBOSMOGSM, libosmogsm >= 1.2.0)
+PKG_CHECK_MODULES(LIBOSMOVTY, libosmovty >= 1.2.0)
+PKG_CHECK_MODULES(LIBOSMOCTRL, libosmoctrl >= 1.2.0)
 PKG_CHECK_MODULES(LIBOSMOABIS, libosmoabis >= 0.6.0)

 PKG_CHECK_MODULES(SQLITE3, sqlite3)
@@ -59,6 +59,21 @@ then
 	CPPFLAGS="$CPPFLAGS -fsanitize=address -fsanitize=undefined"
 fi

+AC_ARG_ENABLE([sqlite_talloc],
+		AC_HELP_STRING([--enable-sqlite-talloc],
+				[Configure SQLite3 to use talloc memory allocator [default=no]]),
+		[sqlite_talloc="$enableval"],[sqlite_talloc="no"])
+if test "x$sqlite_talloc" = "xyes" ; then
+	# Older versions of SQLite3 (at least 3.8.2) become unstable with talloc.
+	# Feel free to relax to 3.24.0 > VER > 3.8.2 if it works for you.
+	# FIXME: PKG_CHECK_MODULES() may return cached result here!
+	PKG_CHECK_MODULES(SQLITE3, sqlite3 >= 3.24.0)
+	AC_DEFINE([SQLITE_USE_TALLOC], 1, [Use talloc for SQLite3])
+fi
+AC_MSG_CHECKING([whether to use talloc for SQLite3])
+AC_MSG_RESULT([$sqlite_talloc])
+AM_CONDITIONAL([DB_SQLITE_DEBUG], [test "x$sqlite_talloc" = "xyes"])
+
 AC_ARG_ENABLE(werror,
 	[AS_HELP_STRING(
 		[--enable-werror],
@@ -160,6 +175,8 @@ AC_OUTPUT(
 	src/Makefile
 	src/gsupclient/Makefile
 	include/Makefile
+	include/osmocom/Makefile
+	include/osmocom/hlr/Makefile
 	libosmo-gsup-client.pc
 	sql/Makefile
 	doc/manuals/Makefile
@@ -169,6 +186,6 @@ AC_OUTPUT(
 	tests/auc/Makefile
 	tests/auc/gen_ts_55_205_test_sets/Makefile
 	tests/gsup_server/Makefile
-	tests/gsup/Makefile
 	tests/db/Makefile
+	tests/db_upgrade/Makefile
 	)
--- a/contrib/jenkins.sh
+++ b/contrib/jenkins.sh
@@ -58,4 +58,5 @@ if [ "$WITH_MANUALS" = "1" ] && [ "$PUBLISH" = "1" ]; then
 	make -C "$base/doc/manuals" publish
 fi

+$MAKE maintainer-clean
 osmo-clean-workspace.sh
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,69 @@
+osmo-hlr (1.1.0) unstable; urgency=medium
+
+  [ Oliver Smith ]
+  * docs: running: document --db-upgrade
+  * Add IMEI column to subscriber table
+  * Optionally store IMEI in subscriber table
+  * VTY tests: fill DB before running test
+  * VTY: integrate IMEI
+  * hlr.c: replace deprecated osmo_gsup_get_err_msg_type()
+  * hlr.c: move hlr_ctx to the top
+  * tests: use -no-install libtool flag to avoid ./lt-* scripts
+  * Cosmetic: gsup_route_find: comment addr, addrlen
+  * USSD: save MO USSD's originating MSC's vlr_number
+  * USSD: don't use gsm0480_msgb_alloc_name()
+  * hlr.c: forward GSUP messages between clients
+  * db_hlr.c: db_subscr_create(): add flags argument
+  * db_hlr.c: add db_subscr_exists_by_imsi()
+  * Create subscribers on demand
+  * Document subscribers create on demand feature
+  * debian: create -doc subpackage with pdf manuals
+  * db_test: set timezone to work around mktime bug
+  * db_hlr: zero-initialize "struct tm"
+  * rx_check_imei_req(): fix IMEI bounds checking
+  * contrib/jenkins.sh: run "make maintainer-clean"
+  * VTY: add subscriber update network-access-mode
+  * manuals: improve subscribers create on demand
+  * gitignore: ignore everything generated in db_test
+  * db_auc.c: verify hex key sizes read from DB
+
+  [ Max ]
+  * Log ip:port when adding GSUP routes
+  * Add link to project wiki to .service file
+  * Enable statsd support
+
+  [ Vadim Yanitskiy ]
+  * hlr.c: properly terminate the process on SIGTERM
+  * hlr.c: fix: also store the session state in read_cb_forward()
+  * hlr.c: fix: properly print the original message type in read_cb_forward()
+  * hlr.c: check the presence of msgb->l2h in read_cb()
+  * hlr.c: fix possible msgb memleaks in read_cb()
+  * db_hlr.c: add db_subscr_exists_by_msisdn()
+  * src/db.h: use GSM23003_MSISDN_MAX_DIGITS for MSISDN buffer size
+  * src/hlr.c: fix deprecation warning: use gsm48_decode_bcd_number2()
+  * hlr_ussd.c: fix: properly pass invokeID in handle_ussd_own_msisdn()
+  * hlr_ussd.c: rx_proc_ss_req(): fix NULL pointer dereference
+  * build: fix mess with 'db_test_SOURCES' and 'db_test_LDADD'
+  * tests/db_test: close the database when test is finished
+  * src/db.c: integrate SQLite3 with talloc allocator
+
+  [ Neels Hofmeyr ]
+  * USSD: fix routing to multiple MSC
+  * fix error logging for GSUP route
+  * add missing error log: invalid IMSI
+  * osmo-hlr: allow configuring db path from cfg file
+  * use new OSMO_IMSI_BUF_SIZE
+
+  [ Daniel Willmann ]
+  * manuals: Add script to update vty/counter documentation from docker
+  * manuals: Update vty documentation
+
+  [ Pau Espin Pedrol ]
+  * Remove undefined param passed to logging_vty_add_cmds
+  * configure.ac: Require libosmocore 1.2.0
+
+ -- Pau Espin Pedrol <pespin@sysmocom.de>  Wed, 07 Aug 2019 16:14:23 +0200
+
 osmo-hlr (1.0.0) unstable; urgency=medium

  [ Stefan Sperling ]
--- a/debian/control
+++ b/debian/control
@@ -7,7 +7,7 @@ Build-Depends: debhelper (>= 9),
               dh-autoreconf,
               dh-systemd (>= 1.5),
               autotools-dev,
-               python-minimal,
+               python3-minimal,
               libosmocore-dev,
               libosmo-abis-dev,
               libosmo-netif-dev,
--- a/doc/manuals/chapters/subscribers.adoc
+++ b/doc/manuals/chapters/subscribers.adoc
@@ -52,7 +52,7 @@ transceiving only RAND and SRES, may be applicable. (See 3GPP TS 33.102, chapter
 |aud3g.ind_bitlen|5|Nr of index bits at lower SQN end
 |apn||
 |vlr_number||3GPP TS 23.008 chapter 2.4.5
-|hlr_number||3GPP TS 23.008 chapter 2.4.6
+|msc_number||3GPP TS 23.008 chapter 2.4.6
 |sgsn_number||3GPP TS 23.008 chapter 2.4.8.1
 |sgsn_address||3GPP TS 23.008 chapter 2.13.10
 |ggsn_number||3GPP TS 23.008 chapter 2.4.8.2
@@ -72,19 +72,58 @@ transceiving only RAND and SRES, may be applicable. (See 3GPP TS 33.102, chapter
 Usually a HLR will only allow mobile equipment (ME) on the network, if the HLR
 has a subscriber entry with the ME's IMSI. But OsmoHLR can also be configured to
 automatically create new entries for new IMSIs, with the
-subscriber-create-on-demand VTY option. The obvious use case is creating the
+`subscriber-create-on-demand` VTY option. The obvious use case is creating the
 new subscriber entry and then allowing the ME to use both the CS
 (Circuit Switched) and PS (Packet Switched) NAM (Network Access Mode).

+.osmo-hlr.cfg
+----
+hlr
+ subscriber-create-on-demand 5 cs+ps
+----
+
 On the other hand, operators might only want to give network access to IMSIs, of
 which they know the owner. In order to do that, one can set the default NAM to
-"none" and manually approve new subscribers by enabling their nam_cs and nam_ps
-parameters (e.g. over the VTY).
+`none` and manually approve new subscribers by changing the NAM (e.g. over the
+VTY, see the example below).

 Oftentimes it is hard to know, which IMSI belongs to which ME, but the IMEI is
 readily available. If you configure your MSC to send IMEI checking requests to
 the HLR, before sending location update requests, the subscribers created on
 demand can also have the IMEI stored in the HLR database. With OsmoMSC, this
-is done by writing "check-imei-rqd early" in the "msc" section of osmo-msc.cfg.
+is done by writing `check-imei-rqd early` in the `msc` section of osmo-msc.cfg.
 Then enable storing the IMEI when receiving check IMEI requests with
-"store-imei" in the OsmoHLR configuration.
+`store-imei` in the OsmoHLR configuration.
+
+.osmo-msc.cfg
+----
+msc
+ check-imei-rqd early
+----
+
+.osmo-hlr.cfg
+----
+hlr
+ subscriber-create-on-demand 5 none
+ store-imei
+----
+
+.Example: Enabling CS and PS NAM via VTY for a known IMEI
+----
+OsmoHLR> enable
+OsmoHLR# subscriber imei 35761300444848 show
+    ID: 1
+    IMSI: 123456789023000
+    MSISDN: 58192 <1>
+    IMEI: 35761300444848
+    CS disabled <2>
+    PS disabled <2>
+OsmoHLR# subscriber imei 35761300444848 update network-access-mode cs+ps
+OsmoHLR# subscriber imei 35761300444848 show
+    ID: 1
+    IMSI: 123456789023000
+    MSISDN: 58192
+    IMEI: 35761300444848
+----
+<1> Randomly generated 5 digit MSISDN
+<2> Disabled CS and PS NAM prevent the subscriber from accessing the network
--- a/doc/manuals/vty/hlr_vty_reference.xml
+++ b/doc/manuals/vty/hlr_vty_reference.xml
@@ -811,6 +811,22 @@
        <param name='IMEI' doc='Set IMEI (use for debug only!)' />
      </params>
    </command>
+    <command id='subscriber (imsi|msisdn|id|imei) IDENT update network-access-mode (none|cs|ps|cs+ps)'>
+      <params>
+        <param name='subscriber' doc='Subscriber management commands' />
+        <param name='imsi' doc='Identify subscriber by IMSI' />
+        <param name='msisdn' doc='Identify subscriber by MSISDN (phone number)' />
+        <param name='id' doc='Identify subscriber by database ID' />
+        <param name='imei' doc='Identify subscriber by IMEI' />
+        <param name='IDENT' doc='IMSI/MSISDN/ID/IMEI of the subscriber' />
+        <param name='update' doc='Set or update subscriber data' />
+        <param name='network-access-mode' doc='Set Network Access Mode (NAM) of the subscriber' />
+        <param name='none' doc='Do not allow access to circuit switched or packet switched services' />
+        <param name='cs' doc='Allow access to circuit switched services only' />
+        <param name='ps' doc='Allow access to packet switched services only' />
+        <param name='cs+ps' doc='Allow access to both circuit and packet switched services' />
+      </params>
+    </command>
  </node>
  <node id='config'>
    <name>config</name>
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -1,2 +1,3 @@
-nobase_include_HEADERS = osmocom/gsupclient/gsup_client.h
+SUBDIRS = osmocom

+nobase_include_HEADERS = osmocom/gsupclient/gsup_client.h
--- a/include/osmocom/Makefile.am
+++ b/include/osmocom/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = \
+	hlr \
+	$(NULL)
--- a/include/osmocom/gsupclient/gsup_client.h
+++ b/include/osmocom/gsupclient/gsup_client.h
@@ -6,18 +6,17 @@
 * Author: Jacob Erlbeck
 *
 * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * GNU General Public License for more details.
 *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 #pragma once

--- a/include/osmocom/hlr/Makefile.am
+++ b/include/osmocom/hlr/Makefile.am
@@ -0,0 +1,14 @@
+noinst_HEADERS = \
+	auc.h \
+	ctrl.h \
+	db.h \
+	gsup_router.h \
+	gsup_server.h \
+	hlr.h \
+	hlr_ussd.h \
+	hlr_vty.h \
+	hlr_vty_subscr.h \
+	logging.h \
+	luop.h \
+	rand.h \
+	$(NULL)
--- a/include/osmocom/hlr/auc.h
+++ b/include/osmocom/hlr/auc.h
--- a/include/osmocom/hlr/ctrl.h
+++ b/include/osmocom/hlr/ctrl.h
--- a/include/osmocom/hlr/db.h
+++ b/include/osmocom/hlr/db.h
@@ -28,6 +28,7 @@ enum stmt_idx {
 	DB_STMT_AUC_3G_INSERT,
 	DB_STMT_AUC_3G_DELETE,
 	DB_STMT_SET_LAST_LU_SEEN,
+	DB_STMT_SET_LAST_LU_SEEN_PS,
 	DB_STMT_EXISTS_BY_IMSI,
 	DB_STMT_EXISTS_BY_MSISDN,
 	_NUM_DB_STMT
@@ -39,6 +40,11 @@ struct db_context {
 	sqlite3_stmt *stmt[_NUM_DB_STMT];
 };

+/* Optional feature to make SQLite3 using talloc */
+#ifdef SQLITE_USE_TALLOC
+int db_sqlite3_use_talloc(void *ctx);
+#endif
+
 void db_remove_reset(sqlite3_stmt *stmt);
 bool db_bind_text(sqlite3_stmt *stmt, const char *param_name, const char *text);
 bool db_bind_int(sqlite3_stmt *stmt, const char *param_name, int nr);
@@ -60,7 +66,7 @@ int db_update_sqn(struct db_context *dbc, int64_t id,
 int db_get_auc(struct db_context *dbc, const char *imsi,
 	       unsigned int auc_3g_ind, struct osmo_auth_vector *vec,
 	       unsigned int num_vec, const uint8_t *rand_auts,
-	       const uint8_t *auts);
+	       const uint8_t *auts, bool separation_bit);

 #include <osmocom/core/linuxlist.h>
 #include <osmocom/gsm/protocol/gsm_23_003.h>
@@ -90,6 +96,7 @@ struct hlr_subscriber {
 	bool		ms_purged_cs;
 	bool		ms_purged_ps;
 	time_t		last_lu_seen;
+	time_t		last_lu_seen_ps;
 };

 /* A format string for use with strptime(3). This format string is
--- a/include/osmocom/hlr/gsup_router.h
+++ b/include/osmocom/hlr/gsup_router.h
@@ -1,7 +1,7 @@
 #pragma once

 #include <stdint.h>
-#include "gsup_server.h"
+#include <osmocom/hlr/gsup_server.h>

 struct gsup_route {
 	struct llist_head list;
--- a/include/osmocom/hlr/gsup_server.h
+++ b/include/osmocom/hlr/gsup_server.h
@@ -47,6 +47,7 @@ struct osmo_gsup_conn {
 	bool supports_ps; /* client supports OSMO_GSUP_CN_DOMAIN_PS */
 };

+struct msgb *osmo_gsup_msgb_alloc(const char *label);

 int osmo_gsup_conn_send(struct osmo_gsup_conn *conn, struct msgb *msg);
 int osmo_gsup_conn_ccm_get(const struct osmo_gsup_conn *clnt, uint8_t **addr,
--- a/include/osmocom/hlr/hlr.h
+++ b/include/osmocom/hlr/hlr.h
--- a/include/osmocom/hlr/hlr_ussd.h
+++ b/include/osmocom/hlr/hlr_ussd.h
@@ -5,7 +5,7 @@
 #include <osmocom/core/linuxlist.h>
 #include <osmocom/gsm/gsup.h>

-#include "gsup_server.h"
+#include <osmocom/hlr/gsup_server.h>

 #define NCSS_GUARD_TIMEOUT_DEFAULT 30

--- a/include/osmocom/hlr/hlr_vty.h
+++ b/include/osmocom/hlr/hlr_vty.h
@@ -25,7 +25,7 @@
 #include <osmocom/core/logging.h>
 #include <osmocom/vty/vty.h>
 #include <osmocom/vty/command.h>
-#include "hlr.h"
+#include <osmocom/hlr/hlr.h>

 enum hlr_vty_node {
 	HLR_NODE = _LAST_OSMOVTY_NODE + 1,
@@ -35,4 +35,4 @@ enum hlr_vty_node {

 int hlr_vty_is_config_node(struct vty *vty, int node);
 int hlr_vty_go_parent(struct vty *vty);
-void hlr_vty_init(const struct log_info *cat);
+void hlr_vty_init(void);
--- a/include/osmocom/hlr/hlr_vty_subscr.h
+++ b/include/osmocom/hlr/hlr_vty_subscr.h
--- a/include/osmocom/hlr/logging.h
+++ b/include/osmocom/hlr/logging.h
--- a/include/osmocom/hlr/luop.h
+++ b/include/osmocom/hlr/luop.h
@@ -27,8 +27,8 @@
 #include <osmocom/core/timer.h>
 #include <osmocom/gsm/gsup.h>

-#include "db.h"
-#include "gsup_server.h"
+#include <osmocom/hlr/db.h>
+#include <osmocom/hlr/gsup_server.h>

 #define CANCEL_TIMEOUT_SECS	30
 #define ISD_TIMEOUT_SECS	30
--- a/include/osmocom/hlr/rand.h
+++ b/include/osmocom/hlr/rand.h
--- a/sql/hlr.sql
+++ b/sql/hlr.sql
@@ -12,7 +12,7 @@ CREATE TABLE subscriber (
 	-- Chapter 2.4.5
 	vlr_number	VARCHAR(15),
 	-- Chapter 2.4.6
-	hlr_number	VARCHAR(15),
+	msc_number	VARCHAR(15),
 	-- Chapter 2.4.8.1
 	sgsn_number	VARCHAR(15),
 	-- Chapter 2.13.10
@@ -42,7 +42,8 @@ CREATE TABLE subscriber (

 	-- Timestamp of last location update seen from subscriber
 	-- The value is a string which encodes a UTC timestamp in granularity of seconds.
-	last_lu_seen TIMESTAMP default NULL
+	last_lu_seen TIMESTAMP default NULL,
+	last_lu_seen_ps TIMESTAMP default NULL
 );

 CREATE TABLE subscriber_apn (
@@ -69,11 +70,12 @@ CREATE TABLE auc_3g (
 	op		VARCHAR(32),		-- hex string: operator's secret key (128bit)
 	opc		VARCHAR(32),		-- hex string: derived from OP and K (128bit)
 	sqn		INTEGER NOT NULL DEFAULT 0,	-- sequence number of key usage
-	ind_bitlen	INTEGER NOT NULL DEFAULT 5	-- nr of index bits at lower SQN end
+	-- nr of index bits at lower SQN end
+	ind_bitlen	INTEGER NOT NULL DEFAULT 5
 );

 CREATE UNIQUE INDEX idx_subscr_imsi ON subscriber (imsi);

 -- Set HLR database schema version number
 -- Note: This constant is currently duplicated in src/db.c and must be kept in sync!
-PRAGMA user_version = 2;
+PRAGMA user_version = 4;
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -11,6 +11,7 @@ AM_CFLAGS = \
 	$(NULL)

 AM_CPPFLAGS = -I$(top_srcdir)/include \
+	-I$(top_builddir)/include \
 	$(NULL)

 EXTRA_DIST = \
@@ -24,18 +25,6 @@ BUILT_SOURCES = \
 CLEANFILES = $(BUILT_SOURCES)

 noinst_HEADERS = \
-	auc.h \
-	db.h \
-	hlr.h \
-	luop.h \
-	gsup_router.h \
-	gsup_server.h \
-	logging.h \
-	rand.h \
-	ctrl.h \
-	hlr_vty.h \
-	hlr_vty_subscr.h \
-	hlr_ussd.h \
 	db_bootstrap.h \
 	$(NULL)

@@ -87,21 +76,6 @@ osmo_hlr_db_tool_LDADD = \
 	$(SQLITE3_LIBS) \
 	$(NULL)

-db_test_SOURCES = \
-	auc.c \
-	db.c \
-	db_auc.c \
-	db_test.c \
-	logging.c \
-	rand_fake.c \
-	$(NULL)
-
-db_test_LDADD = \
-	$(LIBOSMOCORE_LIBS) \
-	$(LIBOSMOGSM_LIBS) \
-	$(SQLITE3_LIBS) \
-	$(NULL)
-
 osmo_euse_demo_SOURCES = \
 	osmo-euse-demo.c \
 	$(NULL)
@@ -112,6 +86,11 @@ osmo_euse_demo_LDADD = \
 	$(LIBOSMOGSM_LIBS) \
 	$(NULL)

+if DB_SQLITE_DEBUG
+osmo_hlr_SOURCES += db_debug.c
+osmo_hlr_db_tool_SOURCES += db_debug.c
+endif
+
 BOOTSTRAP_SQL = $(top_srcdir)/sql/hlr.sql

 db_bootstrap.h: $(BOOTSTRAP_SQL) $(srcdir)/db_sql2c.sed
--- a/src/auc.c
+++ b/src/auc.c
@@ -23,8 +23,8 @@
 #include <osmocom/core/utils.h>
 #include <osmocom/crypt/auth.h>

-#include "logging.h"
-#include "rand.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/rand.h>

 #define hexb(buf) osmo_hexdump_nospc((void*)buf, sizeof(buf))
 #define hex(buf,sz) osmo_hexdump_nospc((void*)buf, sz)
--- a/src/ctrl.c
+++ b/src/ctrl.c
@@ -28,9 +28,9 @@
 #include <osmocom/gsm/gsm23003.h>
 #include <osmocom/ctrl/ports.h>

-#include "hlr.h"
-#include "ctrl.h"
-#include "db.h"
+#include <osmocom/hlr/hlr.h>
+#include <osmocom/hlr/ctrl.h>
+#include <osmocom/hlr/db.h>

 #define SEL_BY "by-"
 #define SEL_BY_IMSI SEL_BY "imsi-"
@@ -95,7 +95,7 @@ static bool get_subscriber(struct db_context *dbc,
 		cmd->reply = "No such subscriber.";
 		return false;
 	default:
-		cmd->reply = "An unknown error has occured during get_subscriber().";
+		cmd->reply = "An unknown error has occurred during get_subscriber().";
 		return false;
 	}
 }
--- a/src/db.c
+++ b/src/db.c
@@ -23,12 +23,12 @@
 #include <sqlite3.h>
 #include <string.h>

-#include "logging.h"
-#include "db.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/db.h>
 #include "db_bootstrap.h"

 /* This constant is currently duplicated in sql/hlr.sql and must be kept in sync! */
-#define CURRENT_SCHEMA_VERSION	2
+#define CURRENT_SCHEMA_VERSION	4

 #define SEL_COLUMNS \
 	"id," \
@@ -45,7 +45,8 @@
 	"lmsi," \
 	"ms_purged_cs," \
 	"ms_purged_ps," \
-	"last_lu_seen"
+	"last_lu_seen," \
+	"last_lu_seen_ps" \

 static const char *stmt_sql[] = {
 	[DB_STMT_SEL_BY_IMSI] = "SELECT " SEL_COLUMNS " FROM subscriber WHERE imsi = ?",
@@ -79,6 +80,7 @@ static const char *stmt_sql[] = {
 		" VALUES($subscriber_id, $algo_id_3g, $k, $op, $opc, $ind_bitlen)",
 	[DB_STMT_AUC_3G_DELETE] = "DELETE FROM auc_3g WHERE subscriber_id = $subscriber_id",
 	[DB_STMT_SET_LAST_LU_SEEN] = "UPDATE subscriber SET last_lu_seen = datetime($val, 'unixepoch') WHERE id = $subscriber_id",
+	[DB_STMT_SET_LAST_LU_SEEN_PS] = "UPDATE subscriber SET last_lu_seen_ps = datetime($val, 'unixepoch') WHERE id = $subscriber_id",
 	[DB_STMT_EXISTS_BY_IMSI] = "SELECT 1 FROM subscriber WHERE imsi = $imsi",
 	[DB_STMT_EXISTS_BY_MSISDN] = "SELECT 1 FROM subscriber WHERE msisdn = $msisdn",
 };
@@ -201,28 +203,38 @@ void db_close(struct db_context *dbc)
 	talloc_free(dbc);
 }

-static int db_bootstrap(struct db_context *dbc)
+static int db_run_statements(struct db_context *dbc, const char **statements, size_t statements_count)
 {
+	int rc;
 	int i;
-	for (i = 0; i < ARRAY_SIZE(stmt_bootstrap_sql); i++) {
-		int rc;
+	for (i = 0; i < statements_count; i++) {
+		const char *stmt_str = statements[i];
 		sqlite3_stmt *stmt;
-		rc = sqlite3_prepare_v2(dbc->db, stmt_bootstrap_sql[i], -1, &stmt, NULL);
+
+		rc = sqlite3_prepare_v2(dbc->db, stmt_str, -1, &stmt, NULL);
 		if (rc != SQLITE_OK) {
-			LOGP(DDB, LOGL_ERROR, "Unable to prepare SQL statement '%s'\n", stmt_bootstrap_sql[i]);
+			LOGP(DDB, LOGL_ERROR, "Unable to prepare SQL statement '%s'\n", stmt_str);
 			return rc;
 		}
-
 		rc = sqlite3_step(stmt);
 		db_remove_reset(stmt);
 		sqlite3_finalize(stmt);
 		if (rc != SQLITE_DONE) {
-			LOGP(DDB, LOGL_ERROR, "Cannot bootstrap database: SQL error: (%d) %s,"
-			     " during stmt '%s'",
-			     rc, sqlite3_errmsg(dbc->db), stmt_bootstrap_sql[i]);
+			LOGP(DDB, LOGL_ERROR, "SQL error: (%d) %s, during stmt '%s'",
+			     rc, sqlite3_errmsg(dbc->db), stmt_str);
 			return rc;
 		}
 	}
+	return rc;
+}
+
+static int db_bootstrap(struct db_context *dbc)
+{
+	int rc = db_run_statements(dbc, stmt_bootstrap_sql, ARRAY_SIZE(stmt_bootstrap_sql));
+	if (rc != SQLITE_DONE) {
+		LOGP(DDB, LOGL_ERROR, "Cannot bootstrap database\n");
+		return rc;
+	}
 	return SQLITE_OK;
 }

@@ -263,72 +275,180 @@ static bool db_is_bootstrapped_v0(struct db_context *dbc)
 static int
 db_upgrade_v1(struct db_context *dbc)
 {
-	sqlite3_stmt *stmt;
 	int rc;
-	const char *update_stmt_sql = "ALTER TABLE subscriber ADD COLUMN last_lu_seen TIMESTAMP default NULL";
-	const char *set_schema_version_sql = "PRAGMA user_version = 1";
+	const char *statements[] = {
+		"ALTER TABLE subscriber ADD COLUMN last_lu_seen TIMESTAMP default NULL",
+		"PRAGMA user_version = 1",
+	};

-	rc = sqlite3_prepare_v2(dbc->db, update_stmt_sql, -1, &stmt, NULL);
-	if (rc != SQLITE_OK) {
-		LOGP(DDB, LOGL_ERROR, "Unable to prepare SQL statement '%s'\n", update_stmt_sql);
-		return rc;
-	}
-	rc = sqlite3_step(stmt);
-	db_remove_reset(stmt);
-	sqlite3_finalize(stmt);
+	rc = db_run_statements(dbc, statements, ARRAY_SIZE(statements));
 	if (rc != SQLITE_DONE) {
-		LOGP(DDB, LOGL_ERROR, "Unable to update HLR database schema to version %d\n", 1);
+		LOGP(DDB, LOGL_ERROR, "Unable to update HLR database schema to version 1\n");
 		return rc;
 	}
-
-	rc = sqlite3_prepare_v2(dbc->db, set_schema_version_sql, -1, &stmt, NULL);
-	if (rc != SQLITE_OK) {
-		LOGP(DDB, LOGL_ERROR, "Unable to prepare SQL statement '%s'\n", set_schema_version_sql);
-		return rc;
-	}
-	rc = sqlite3_step(stmt);
-	if (rc != SQLITE_DONE)
-		LOGP(DDB, LOGL_ERROR, "Unable to update HLR database schema to version %d\n", 1);
-
-	db_remove_reset(stmt);
-	sqlite3_finalize(stmt);
 	return rc;
 }

 static int db_upgrade_v2(struct db_context *dbc)
 {
-	sqlite3_stmt *stmt;
 	int rc;
-	const char *update_stmt_sql = "ALTER TABLE subscriber ADD COLUMN imei VARCHAR(14) default NULL";
-	const char *set_schema_version_sql = "PRAGMA user_version = 2";
+	const char *statements[] = {
+		"ALTER TABLE subscriber ADD COLUMN imei VARCHAR(14)",
+		"PRAGMA user_version = 2",
+	};

-	rc = sqlite3_prepare_v2(dbc->db, update_stmt_sql, -1, &stmt, NULL);
-	if (rc != SQLITE_OK) {
-		LOGP(DDB, LOGL_ERROR, "Unable to prepare SQL statement '%s'\n", update_stmt_sql);
-		return rc;
-	}
-	rc = sqlite3_step(stmt);
-	db_remove_reset(stmt);
-	sqlite3_finalize(stmt);
+	rc = db_run_statements(dbc, statements, ARRAY_SIZE(statements));
 	if (rc != SQLITE_DONE) {
-		LOGP(DDB, LOGL_ERROR, "Unable to update HLR database schema to version %d\n", 1);
+		LOGP(DDB, LOGL_ERROR, "Unable to update HLR database schema to version 2\n");
 		return rc;
 	}
-
-	rc = sqlite3_prepare_v2(dbc->db, set_schema_version_sql, -1, &stmt, NULL);
-	if (rc != SQLITE_OK) {
-		LOGP(DDB, LOGL_ERROR, "Unable to prepare SQL statement '%s'\n", set_schema_version_sql);
-		return rc;
-	}
-	rc = sqlite3_step(stmt);
-	if (rc != SQLITE_DONE)
-		LOGP(DDB, LOGL_ERROR, "Unable to update HLR database schema to version %d\n", 1);
-
-	db_remove_reset(stmt);
-	sqlite3_finalize(stmt);
 	return rc;
 }

+static int db_upgrade_v3(struct db_context *dbc)
+{
+	int rc;
+
+	/* A newer SQLite version would allow simply 'ATLER TABLE subscriber RENAME COLUMN hlr_number TO msc_number'.
+	 * This is a really expensive workaround for that in order to cover earlier SQLite versions as well:
+	 * Create a new table with the new column name and copy the data over (https://www.sqlite.org/faq.html#q11).
+	 */
+#define SUBSCR_V3_CREATE  \
+"(\n" \
+"-- OsmoHLR's DB scheme is modelled roughly after TS 23.008 version 13.3.0\n" \
+"	id		INTEGER PRIMARY KEY,\n" \
+"	-- Chapter 2.1.1.1\n" \
+"	imsi		VARCHAR(15) UNIQUE NOT NULL,\n" \
+"	-- Chapter 2.1.2\n" \
+"	msisdn		VARCHAR(15) UNIQUE,\n" \
+"	-- Chapter 2.2.3: Most recent / current IMEISV\n" \
+"	imeisv		VARCHAR,\n" \
+"	-- Chapter 2.1.9: Most recent / current IMEI\n" \
+"	imei		VARCHAR(14),\n" \
+"	-- Chapter 2.4.5\n" \
+"	vlr_number	VARCHAR(15),\n" \
+"	-- Chapter 2.4.6\n" \
+"	msc_number	VARCHAR(15),\n" \
+"	-- Chapter 2.4.8.1\n" \
+"	sgsn_number	VARCHAR(15),\n" \
+"	-- Chapter 2.13.10\n" \
+"	sgsn_address	VARCHAR,\n" \
+"	-- Chapter 2.4.8.2\n" \
+"	ggsn_number	VARCHAR(15),\n" \
+"	-- Chapter 2.4.9.2\n" \
+"	gmlc_number	VARCHAR(15),\n" \
+"	-- Chapter 2.4.23\n" \
+"	smsc_number	VARCHAR(15),\n" \
+"	-- Chapter 2.4.24\n" \
+"	periodic_lu_tmr	INTEGER,\n" \
+"	-- Chapter 2.13.115\n" \
+"	periodic_rau_tau_tmr INTEGER,\n" \
+"	-- Chapter 2.1.1.2: network access mode\n" \
+"	nam_cs		BOOLEAN NOT NULL DEFAULT 1,\n" \
+"	nam_ps		BOOLEAN NOT NULL DEFAULT 1,\n" \
+"	-- Chapter 2.1.8\n" \
+"	lmsi		INTEGER,\n" \
+ \
+"	-- The below purged flags might not even be stored non-volatile,\n" \
+"	-- refer to TS 23.012 Chapter 3.6.1.4\n" \
+"	-- Chapter 2.7.5\n" \
+"	ms_purged_cs	BOOLEAN NOT NULL DEFAULT 0,\n" \
+"	-- Chapter 2.7.6\n" \
+"	ms_purged_ps	BOOLEAN NOT NULL DEFAULT 0,\n" \
+ \
+"	-- Timestamp of last location update seen from subscriber\n" \
+"	-- The value is a string which encodes a UTC timestamp in granularity of seconds.\n" \
+"	last_lu_seen TIMESTAMP default NULL\n" \
+")\n"
+
+#define SUBSCR_V2_COLUMN_NAMES \
+	"id," \
+	"imsi," \
+	"msisdn," \
+	"imeisv," \
+	"imei," \
+	"vlr_number," \
+	"hlr_number," \
+	"sgsn_number," \
+	"sgsn_address," \
+	"ggsn_number," \
+	"gmlc_number," \
+	"smsc_number," \
+	"periodic_lu_tmr," \
+	"periodic_rau_tau_tmr," \
+	"nam_cs," \
+	"nam_ps," \
+	"lmsi," \
+	"ms_purged_cs," \
+	"ms_purged_ps," \
+	"last_lu_seen"
+
+#define SUBSCR_V3_COLUMN_NAMES \
+	"id," \
+	"imsi," \
+	"msisdn," \
+	"imeisv," \
+	"imei," \
+	"vlr_number," \
+	"msc_number," \
+	"sgsn_number," \
+	"sgsn_address," \
+	"ggsn_number," \
+	"gmlc_number," \
+	"smsc_number," \
+	"periodic_lu_tmr," \
+	"periodic_rau_tau_tmr," \
+	"nam_cs," \
+	"nam_ps," \
+	"lmsi," \
+	"ms_purged_cs," \
+	"ms_purged_ps," \
+	"last_lu_seen"
+
+	const char *statements[] = {
+		"BEGIN TRANSACTION",
+		"CREATE TEMPORARY TABLE subscriber_backup" SUBSCR_V3_CREATE,
+		"INSERT INTO subscriber_backup SELECT " SUBSCR_V2_COLUMN_NAMES " FROM subscriber",
+		"DROP TABLE subscriber",
+		"CREATE TABLE subscriber" SUBSCR_V3_CREATE,
+		"INSERT INTO subscriber SELECT " SUBSCR_V3_COLUMN_NAMES " FROM subscriber_backup",
+		"DROP TABLE subscriber_backup",
+		"COMMIT",
+		"PRAGMA user_version = 3",
+	};
+
+	rc = db_run_statements(dbc, statements, ARRAY_SIZE(statements));
+	if (rc != SQLITE_DONE) {
+		LOGP(DDB, LOGL_ERROR, "Unable to update HLR database schema to version 3\n");
+		return rc;
+	}
+	return rc;
+}
+
+static int db_upgrade_v4(struct db_context *dbc)
+{
+	int rc;
+	const char *statements[] = {
+		"ALTER TABLE subscriber ADD COLUMN last_lu_seen_ps TIMESTAMP default NULL",
+		"PRAGMA user_version = 4",
+	};
+
+	rc = db_run_statements(dbc, statements, ARRAY_SIZE(statements));
+	if (rc != SQLITE_DONE) {
+		LOGP(DDB, LOGL_ERROR, "Unable to update HLR database schema to version 4\n");
+		return rc;
+	}
+	return rc;
+}
+
+typedef int (*db_upgrade_func_t)(struct db_context *dbc);
+static db_upgrade_func_t db_upgrade_path[] = {
+	db_upgrade_v1,
+	db_upgrade_v2,
+	db_upgrade_v3,
+	db_upgrade_v4,
+};
+
 static int db_get_user_version(struct db_context *dbc)
 {
 	const char *user_version_sql = "PRAGMA user_version";
@@ -365,6 +485,17 @@ struct db_context *db_open(void *ctx, const char *fname, bool enable_sqlite_logg
 	LOGP(DDB, LOGL_INFO, "Compiled against SQLite3 lib version %s\n", SQLITE_VERSION);
 	LOGP(DDB, LOGL_INFO, "Running with SQLite3 lib version %s\n", sqlite3_libversion());

+#ifdef SQLITE_USE_TALLOC
+	/* Configure SQLite3 to use talloc memory allocator */
+	rc = db_sqlite3_use_talloc(ctx);
+	if (rc == SQLITE_OK) {
+		LOGP(DDB, LOGL_NOTICE, "SQLite3 is configured to use talloc\n");
+	} else {
+		LOGP(DDB, LOGL_ERROR, "Failed to configure SQLite3 "
+		     "to use talloc, using default memory allocator\n");
+	}
+#endif
+
 	dbc->fname = talloc_strdup(dbc, fname);

 	for (i = 0; i < 0xfffff; i++) {
@@ -428,32 +559,16 @@ struct db_context *db_open(void *ctx, const char *fname, bool enable_sqlite_logg

 	LOGP(DDB, LOGL_NOTICE, "Database '%s' has HLR DB schema version %d\n", dbc->fname, version);

-	if (version < CURRENT_SCHEMA_VERSION && allow_upgrade) {
-		switch (version) {
-		case 0:
-			rc = db_upgrade_v1(dbc);
-			if (rc != SQLITE_DONE) {
-				LOGP(DDB, LOGL_ERROR, "Failed to upgrade HLR DB schema to version 1: (rc=%d) %s\n",
-				     rc, sqlite3_errmsg(dbc->db));
-				goto out_free;
-			}
-			version = 1;
-			/* fall through */
-		case 1:
-			rc = db_upgrade_v2(dbc);
-			if (rc != SQLITE_DONE) {
-				LOGP(DDB, LOGL_ERROR, "Failed to upgrade HLR DB schema to version 2: (rc=%d) %s\n",
-				     rc, sqlite3_errmsg(dbc->db));
-				goto out_free;
-			}
-			version = 2;
-			/* fall through */
-		/* case N: ... */
-		default:
-			break;
+	for (; allow_upgrade && (version < ARRAY_SIZE(db_upgrade_path)); version++) {
+		db_upgrade_func_t upgrade_func = db_upgrade_path[version];
+		rc = upgrade_func(dbc);
+		if (rc != SQLITE_DONE) {
+			LOGP(DDB, LOGL_ERROR, "Failed to upgrade HLR DB schema to version %d: (rc=%d) %s\n",
+			     version+1, rc, sqlite3_errmsg(dbc->db));
+			goto out_free;
 		}
 		LOGP(DDB, LOGL_NOTICE, "Database '%s' has been upgraded to HLR DB schema version %d\n",
-		     dbc->fname, version);
+		     dbc->fname, version+1);
 	}

 	if (version != CURRENT_SCHEMA_VERSION) {
--- a/src/db_auc.c
+++ b/src/db_auc.c
@@ -26,10 +26,10 @@

 #include <sqlite3.h>

-#include "logging.h"
-#include "db.h"
-#include "auc.h"
-#include "rand.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/db.h>
+#include <osmocom/hlr/auc.h>
+#include <osmocom/hlr/rand.h>

 #define LOGAUC(imsi, level, fmt, args ...)	LOGP(DAUC, level, "IMSI='%s': " fmt, imsi, ## args)

@@ -73,6 +73,32 @@ out:
 	return ret;
 }

+/* hexparse a specific column of a sqlite prepared statement into dst (with length check)
+ * returns 0 for success, -EIO on error */
+static int hexparse_stmt(uint8_t *dst, size_t dst_len, sqlite3_stmt *stmt, int col, const char *col_name,
+			 const char *imsi)
+{
+	const uint8_t *text;
+	size_t col_len;
+
+	/* Bytes are stored as hex strings in database, hence divide length by two */
+	col_len = sqlite3_column_bytes(stmt, col) / 2;
+
+	if (col_len != dst_len) {
+		LOGAUC(imsi, LOGL_ERROR, "Error reading %s, expected length %lu but has length %lu\n", col_name,
+		       dst_len, col_len);
+		return -EIO;
+	}
+
+	text = sqlite3_column_text(stmt, col);
+	if (!text) {
+		LOGAUC(imsi, LOGL_ERROR, "Error reading %s\n", col_name);
+		return -EIO;
+	}
+	osmo_hexparse((void *)text, dst, dst_len);
+	return 0;
+}
+
 /* obtain the authentication data for a given imsi
 * returns 0 for success, negative value on error:
 * -ENOENT if the IMSI is not known, -ENOKEY if the IMSI is known but has no auth data,
@@ -113,49 +139,34 @@ int db_get_auth_data(struct db_context *dbc, const char *imsi,
 	/* obtain result values using sqlite3_column_*() */
 	if (sqlite3_column_type(stmt, 1) == SQLITE_INTEGER) {
 		/* we do have some 2G authentication data */
-		const uint8_t *ki;
-
-		aud2g->algo = sqlite3_column_int(stmt, 1);
-		ki = sqlite3_column_text(stmt, 2);
-#if 0
-		if (sqlite3_column_bytes(stmt, 2) != sizeof(aud2g->u.gsm.ki)) {
-			LOGAUC(imsi, LOGL_ERROR, "Error reading Ki: %d\n", rc);
+		if (hexparse_stmt(aud2g->u.gsm.ki, sizeof(aud2g->u.gsm.ki), stmt, 2, "Ki", imsi))
 			goto end_2g;
-		}
-#endif
-		osmo_hexparse((void*)ki, (void*)&aud2g->u.gsm.ki, sizeof(aud2g->u.gsm.ki));
+		aud2g->algo = sqlite3_column_int(stmt, 1);
 		aud2g->type = OSMO_AUTH_TYPE_GSM;
 	} else
 		LOGAUC(imsi, LOGL_DEBUG, "No 2G Auth Data\n");
-//end_2g:
+end_2g:
 	if (sqlite3_column_type(stmt, 3) == SQLITE_INTEGER) {
 		/* we do have some 3G authentication data */
-		const uint8_t *k, *op, *opc;
-
-		aud3g->algo = sqlite3_column_int(stmt, 3);
-		k = sqlite3_column_text(stmt, 4);
-		if (!k) {
-			LOGAUC(imsi, LOGL_ERROR, "Error reading K: %d\n", rc);
+		if (hexparse_stmt(aud3g->u.umts.k, sizeof(aud3g->u.umts.k), stmt, 4, "K", imsi)) {
 			ret = -EIO;
 			goto out;
 		}
-		osmo_hexparse((void*)k, (void*)&aud3g->u.umts.k, sizeof(aud3g->u.umts.k));
+		aud3g->algo = sqlite3_column_int(stmt, 3);
+
 		/* UMTS Subscribers can have either OP or OPC */
-		op = sqlite3_column_text(stmt, 5);
-		if (!op) {
-			opc = sqlite3_column_text(stmt, 6);
-			if (!opc) {
-				LOGAUC(imsi, LOGL_ERROR, "Error reading OPC: %d\n", rc);
+		if (sqlite3_column_text(stmt, 5)) {
+			if (hexparse_stmt(aud3g->u.umts.opc, sizeof(aud3g->u.umts.opc), stmt, 5, "OP", imsi)) {
 				ret = -EIO;
 				goto out;
 			}
-			osmo_hexparse((void*)opc, (void*)&aud3g->u.umts.opc,
-					sizeof(aud3g->u.umts.opc));
-			aud3g->u.umts.opc_is_op = 0;
-		} else {
-			osmo_hexparse((void*)op, (void*)&aud3g->u.umts.opc,
-					sizeof(aud3g->u.umts.opc));
 			aud3g->u.umts.opc_is_op = 1;
+		} else {
+			if (hexparse_stmt(aud3g->u.umts.opc, sizeof(aud3g->u.umts.opc), stmt, 6, "OPC", imsi)) {
+				ret = -EIO;
+				goto out;
+			}
+			aud3g->u.umts.opc_is_op = 0;
 		}
 		aud3g->u.umts.sqn = sqlite3_column_int64(stmt, 7);
 		aud3g->u.umts.ind_bitlen = sqlite3_column_int(stmt, 8);
@@ -178,7 +189,7 @@ out:
 int db_get_auc(struct db_context *dbc, const char *imsi,
 	       unsigned int auc_3g_ind, struct osmo_auth_vector *vec,
 	       unsigned int num_vec, const uint8_t *rand_auts,
-	       const uint8_t *auts)
+	       const uint8_t *auts, bool separation_bit)
 {
 	struct osmo_sub_auth_data aud2g, aud3g;
 	int64_t subscr_id;
@@ -198,6 +209,12 @@ int db_get_auc(struct db_context *dbc, const char *imsi,
 		       aud3g.u.umts.ind_bitlen, aud3g.u.umts.ind);
 		aud3g.u.umts.ind &= (1U << aud3g.u.umts.ind_bitlen) - 1;
 	}
+	/* the first bit (bit0) cannot be used as AMF anymore, but has been
+	 * re-appropriated as the separation bit.  See 3GPP TS 33.102 Annex H
+	 * together with 3GPP TS 33.401 / 33.402 / 33.501 */
+	aud3g.u.umts.amf[0] = aud3g.u.umts.amf[0] & 0x7f;
+	if (separation_bit)
+		aud3g.u.umts.amf[0] |= 0x80;

 	LOGAUC(imsi, LOGL_DEBUG, "Calling to generate %u vectors\n", num_vec);
 	rc = auc_compute_vectors(vec, num_vec, &aud2g, &aud3g, rand_auts, auts);
--- a/src/db_debug.c
+++ b/src/db_debug.c
@@ -0,0 +1,86 @@
+/*
+ * libtalloc based memory allocator for SQLite3.
+ *
+ * (C) 2019 by Vadim Yanitskiy <axilirator@gmail.com>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include <sqlite3.h>
+#include <talloc.h>
+#include <errno.h>
+
+/* Dedicated talloc context for SQLite */
+static void *db_sqlite_ctx = NULL;
+
+static void *tall_xMalloc(int size)
+{
+	return talloc_size(db_sqlite_ctx, size);
+}
+
+static void tall_xFree(void *ptr)
+{
+	talloc_free(ptr);
+}
+
+static void *tall_xRealloc(void *ptr, int size)
+{
+	return talloc_realloc_fn(db_sqlite_ctx, ptr, size);
+}
+
+static int tall_xSize(void *ptr)
+{
+	return talloc_total_size(ptr);
+}
+
+/* DUMMY: talloc doesn't round up the allocation size */
+static int tall_xRoundup(int size) { return size; }
+
+/* DUMMY: nothing to initialize */
+static int tall_xInit(void *data) { return 0; }
+
+/* DUMMY: nothing to deinitialize */
+static void tall_xShutdown(void *data) {  }
+
+/* Interface between SQLite and talloc memory allocator */
+static const struct sqlite3_mem_methods tall_sqlite_if = {
+	/* Memory allocation function */
+	.xMalloc = &tall_xMalloc,
+	/* Free a prior allocation */
+	.xFree = &tall_xFree,
+	/* Resize an allocation */
+	.xRealloc = &tall_xRealloc,
+	/* Return the size of an allocation */
+	.xSize = &tall_xSize,
+	/* Round up request size to allocation size */
+	.xRoundup = &tall_xRoundup,
+	/* Initialize the memory allocator */
+	.xInit = &tall_xInit,
+	/* Deinitialize the memory allocator */
+	.xShutdown = &tall_xShutdown,
+	/* Argument to xInit() and xShutdown() */
+	.pAppData = NULL,
+};
+
+int db_sqlite3_use_talloc(void *ctx)
+{
+	if (db_sqlite_ctx != NULL)
+		return -EEXIST;
+
+	db_sqlite_ctx = talloc_named_const(ctx, 0, "SQLite3");
+	return sqlite3_config(SQLITE_CONFIG_MALLOC, &tall_sqlite_if);
+}
--- a/src/db_hlr.c
+++ b/src/db_hlr.c
@@ -33,11 +33,11 @@

 #include <sqlite3.h>

-#include "logging.h"
-#include "hlr.h"
-#include "db.h"
-#include "gsup_server.h"
-#include "luop.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/hlr.h>
+#include <osmocom/hlr/db.h>
+#include <osmocom/hlr/gsup_server.h>
+#include <osmocom/hlr/luop.h>

 #define LOGHLR(imsi, level, fmt, args ...)	LOGP(DAUC, level, "IMSI='%s': " fmt, imsi, ## args)

@@ -438,14 +438,36 @@ out:
 	return ret;
 }

+static void parse_last_lu_seen(time_t *dst, const char *last_lu_seen_str, const char *imsi, const char *label)
+{
+	struct tm tm = {0};
+	time_t val;
+	if (!last_lu_seen_str || last_lu_seen_str[0] == '\0')
+		return;
+
+	if (strptime(last_lu_seen_str, DB_LAST_LU_SEEN_FMT, &tm) == NULL) {
+		LOGP(DAUC, LOGL_ERROR, "IMSI-%s: Last LU Seen %s: Cannot parse timestamp '%s'\n",
+		     imsi, label, last_lu_seen_str);
+		return;
+	}
+
+	errno = 0;
+	val = mktime(&tm);
+	if (val == -1) {
+		LOGP(DAUC, LOGL_ERROR, "IMSI-%s: Last LU Seen %s: Cannot convert timestamp '%s' to time_t: %s\n",
+		     imsi, label, last_lu_seen_str, strerror(errno));
+		val = 0;
+	}
+
+	*dst = val;
+}
+
 /* Common code for db_subscr_get_by_*() functions. */
 static int db_sel(struct db_context *dbc, sqlite3_stmt *stmt, struct hlr_subscriber *subscr,
 		  const char **err)
 {
 	int rc;
 	int ret = 0;
-	const char *last_lu_seen_str;
-	struct tm tm = {0};

 	/* execute the statement */
 	rc = sqlite3_step(stmt);
@@ -479,20 +501,10 @@ static int db_sel(struct db_context *dbc, sqlite3_stmt *stmt, struct hlr_subscri
 	subscr->lmsi = sqlite3_column_int(stmt, 11);
 	subscr->ms_purged_cs = sqlite3_column_int(stmt, 12);
 	subscr->ms_purged_ps = sqlite3_column_int(stmt, 13);
-	last_lu_seen_str = (const char *)sqlite3_column_text(stmt, 14);
-	if (last_lu_seen_str && last_lu_seen_str[0] != '\0') {
-		if (strptime(last_lu_seen_str, DB_LAST_LU_SEEN_FMT, &tm) == NULL) {
-			LOGP(DAUC, LOGL_ERROR, "Cannot parse last LU timestamp '%s' of subscriber with IMSI='%s': %s\n",
-			     last_lu_seen_str, subscr->imsi, strerror(errno));
-		} else {
-			subscr->last_lu_seen = mktime(&tm);
-			if (subscr->last_lu_seen == -1) {
-				LOGP(DAUC, LOGL_ERROR, "Cannot convert LU timestamp '%s' to time_t: %s\n",
-				     last_lu_seen_str, strerror(errno));
-				subscr->last_lu_seen = 0;
-			}
-		}
-	}
+	parse_last_lu_seen(&subscr->last_lu_seen, (const char *)sqlite3_column_text(stmt, 14),
+			   subscr->imsi, "CS");
+	parse_last_lu_seen(&subscr->last_lu_seen_ps, (const char *)sqlite3_column_text(stmt, 15),
+			   subscr->imsi, "PS");

 out:
 	db_remove_reset(stmt);
@@ -770,7 +782,7 @@ int db_subscr_lu(struct db_context *dbc, int64_t subscr_id,
 		goto out;
 	}

-	stmt = dbc->stmt[DB_STMT_SET_LAST_LU_SEEN];
+	stmt = dbc->stmt[is_ps? DB_STMT_SET_LAST_LU_SEEN_PS : DB_STMT_SET_LAST_LU_SEEN];

 	if (!db_bind_int64(stmt, "$subscriber_id", subscr_id))
 		return -EIO;
--- a/src/gsup_router.c
+++ b/src/gsup_router.c
@@ -23,9 +23,9 @@
 #include <osmocom/core/linuxlist.h>
 #include <osmocom/core/talloc.h>

-#include "logging.h"
-#include "gsup_server.h"
-#include "gsup_router.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/gsup_server.h>
+#include <osmocom/hlr/gsup_router.h>

 /*! Find a route for the given address.
 * \param[in] gs gsup server
@@ -40,14 +40,8 @@ struct osmo_gsup_conn *gsup_route_find(struct osmo_gsup_server *gs,
 	struct gsup_route *gr;

 	llist_for_each_entry(gr, &gs->routes, list) {
-		size_t gr_addrlen = talloc_total_size(gr->addr); /* gr->addr is a nul-terminated string */
-
-		/* FIXME: despite passing addrlen, a lot of code assumes that addr is also nul-terminated */
-		if (gr_addrlen == addrlen && !memcmp(gr->addr, addr, addrlen))
-			return gr->conn;
-
-		/* Compare addr as non-nul-terminated blob */
-		if (gr_addrlen - 1 == addrlen && !memcmp(gr->addr, addr, addrlen))
+		if (talloc_total_size(gr->addr) == addrlen &&
+		    !memcmp(gr->addr, addr, addrlen))
 			return gr->conn;
 	}
 	return NULL;
--- a/src/gsup_send.c
+++ b/src/gsup_send.c
@@ -21,8 +21,8 @@

 #include <errno.h>

-#include "gsup_server.h"
-#include "gsup_router.h"
+#include <osmocom/hlr/gsup_server.h>
+#include <osmocom/hlr/gsup_router.h>

 #include <osmocom/core/logging.h>

--- a/src/gsup_server.c
+++ b/src/gsup_server.c
@@ -27,8 +27,15 @@
 #include <osmocom/gsm/gsm48_ie.h>
 #include <osmocom/gsm/apn.h>

-#include "gsup_server.h"
-#include "gsup_router.h"
+#include <osmocom/hlr/gsup_server.h>
+#include <osmocom/hlr/gsup_router.h>
+
+struct msgb *osmo_gsup_msgb_alloc(const char *label)
+{
+	struct msgb *msg = msgb_alloc_headroom(1024+16, 16, label);
+	OSMO_ASSERT(msg);
+	return msg;
+}

 static void osmo_gsup_server_send(struct osmo_gsup_conn *conn,
 			     int proto_ext, struct msgb *msg_tx)
--- a/src/gsupclient/gsup_client.c
+++ b/src/gsupclient/gsup_client.c
@@ -386,7 +386,8 @@ int osmo_gsup_client_enc_send(struct osmo_gsup_client *gsupc,
 	rc = osmo_gsup_client_send(gsupc, gsup_msgb);
 	if (rc) {
 		LOGP(DLGSUP, LOGL_ERROR, "Couldn't send GSUP message\n");
-		goto error;
+		/* Do not free, osmo_gsup_client_send() already has. */
+		return rc;
 	}

 	return 0;
--- a/src/hlr.c
+++ b/src/hlr.c
@@ -37,16 +37,16 @@
 #include <osmocom/gsm/gsm_utils.h>
 #include <osmocom/gsm/protocol/gsm_23_003.h>

-#include "db.h"
-#include "hlr.h"
-#include "ctrl.h"
-#include "logging.h"
-#include "gsup_server.h"
-#include "gsup_router.h"
-#include "rand.h"
-#include "luop.h"
-#include "hlr_vty.h"
-#include "hlr_ussd.h"
+#include <osmocom/hlr/db.h>
+#include <osmocom/hlr/hlr.h>
+#include <osmocom/hlr/ctrl.h>
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/gsup_server.h>
+#include <osmocom/hlr/gsup_router.h>
+#include <osmocom/hlr/rand.h>
+#include <osmocom/hlr/luop.h>
+#include <osmocom/hlr/hlr_vty.h>
+#include <osmocom/hlr/hlr_ussd.h>

 struct hlr *g_hlr;
 static void *hlr_ctx = NULL;
@@ -128,7 +128,7 @@ osmo_hlr_subscriber_update_notify(struct hlr_subscriber *subscr)
 		}

 		/* Send ISD to MSC/SGSN */
-		msg_out = msgb_alloc_headroom(1024+16, 16, "GSUP ISD UPDATE");
+		msg_out = osmo_gsup_msgb_alloc("GSUP ISD UPDATE");
 		if (msg_out == NULL) {
 			LOGP(DLGSUP, LOGL_ERROR,
 			       "IMSI='%s': Cannot notify GSUP client; could not allocate msg buffer "
@@ -233,6 +233,7 @@ static int rx_send_auth_info(struct osmo_gsup_conn *conn,
 {
 	struct osmo_gsup_message gsup_out;
 	struct msgb *msg_out;
+	bool separation_bit = false;
 	int rc;

 	subscr_create_on_demand(gsup->imsi);
@@ -241,10 +242,13 @@ static int rx_send_auth_info(struct osmo_gsup_conn *conn,
 	memset(&gsup_out, 0, sizeof(gsup_out));
 	memcpy(&gsup_out.imsi, &gsup->imsi, sizeof(gsup_out.imsi));

+	if (gsup->current_rat_type == OSMO_RAT_EUTRAN_SGS)
+		separation_bit = true;
+
 	rc = db_get_auc(dbc, gsup->imsi, conn->auc_3g_ind,
 			gsup_out.auth_vectors,
 			ARRAY_SIZE(gsup_out.auth_vectors),
-			gsup->rand, gsup->auts);
+			gsup->rand, gsup->auts, separation_bit);
 	if (rc <= 0) {
 		gsup_out.message_type = OSMO_GSUP_MSGT_SEND_AUTH_INFO_ERROR;
 		switch (rc) {
@@ -271,7 +275,7 @@ static int rx_send_auth_info(struct osmo_gsup_conn *conn,
 		gsup_out.num_auth_vectors = rc;
 	}

-	msg_out = msgb_alloc_headroom(1024+16, 16, "GSUP AUC response");
+	msg_out = osmo_gsup_msgb_alloc("GSUP AUC response");
 	osmo_gsup_encode(msg_out, &gsup_out);
 	return osmo_gsup_conn_send(conn, msg_out);
 }
@@ -451,7 +455,7 @@ static int rx_purge_ms_req(struct osmo_gsup_conn *conn,
 		gsup_reply.cause = GMM_CAUSE_NET_FAIL;
 	}

-	msg_out = msgb_alloc_headroom(1024+16, 16, "GSUP AUC response");
+	msg_out = osmo_gsup_msgb_alloc("GSUP AUC response");
 	osmo_gsup_encode(msg_out, &gsup_reply);
 	return osmo_gsup_conn_send(conn, msg_out);
 }
@@ -466,8 +470,7 @@ static int gsup_send_err_reply(struct osmo_gsup_conn *conn, const char *imsi,
 	OSMO_STRLCPY_ARRAY(gsup_reply.imsi, imsi);
 	gsup_reply.message_type = type_err;
 	gsup_reply.cause = err_cause;
-	msg_out = msgb_alloc_headroom(1024+16, 16, "GSUP ERR response");
-	OSMO_ASSERT(msg_out);
+	msg_out = osmo_gsup_msgb_alloc("GSUP ERR response");
 	osmo_gsup_encode(msg_out, &gsup_reply);
 	LOGP(DMAIN, LOGL_NOTICE, "Tx %s\n", osmo_gsup_message_type_name(type_err));
 	return osmo_gsup_conn_send(conn, msg_out);
@@ -525,7 +528,7 @@ static int rx_check_imei_req(struct osmo_gsup_conn *conn, const struct osmo_gsup
 	/* Accept all IMEIs */
 	gsup_reply.imei_result = OSMO_GSUP_IMEI_RESULT_ACK;
 	gsup_reply.message_type = OSMO_GSUP_MSGT_CHECK_IMEI_RESULT;
-	msg_out = msgb_alloc_headroom(1024+16, 16, "GSUP Check_IMEI response");
+	msg_out = osmo_gsup_msgb_alloc("GSUP Check_IMEI response");
 	memcpy(gsup_reply.imsi, gsup->imsi, sizeof(gsup_reply.imsi));
 	osmo_gsup_encode(msg_out, &gsup_reply);
 	return osmo_gsup_conn_send(conn, msg_out);
@@ -591,8 +594,7 @@ static int read_cb_forward(struct osmo_gsup_conn *conn, struct msgb *msg, const
 end:
 	/* Send error back to source */
 	if (ret) {
-		struct msgb *msg_err = msgb_alloc_headroom(1024+16, 16, "GSUP forward ERR response");
-		OSMO_ASSERT(msg_err);
+		struct msgb *msg_err = osmo_gsup_msgb_alloc("GSUP forward ERR response");
 		gsup_err->message_type = OSMO_GSUP_MSGT_E_ROUTING_ERROR;
 		osmo_gsup_encode(msg_err, gsup_err);
 		LOGP_GSUP_FWD(gsup_err, LOGL_NOTICE, "Tx %s\n", osmo_gsup_message_type_name(gsup_err->message_type));
@@ -706,6 +708,7 @@ static void print_help()
 	printf("  -T --timestamp             Prefix every log line with a timestamp.\n");
 	printf("  -e --log-level number      Set a global loglevel.\n");
 	printf("  -U --db-upgrade            Allow HLR database schema upgrades.\n");
+	printf("  -C --db-check              Quit after opening (and upgrading) the database.\n");
 	printf("  -V --version               Print the version of OsmoHLR.\n");
 }

@@ -714,6 +717,7 @@ static struct {
 	const char *db_file;
 	bool daemonize;
 	bool db_upgrade;
+	bool db_check;
 } cmdline_opts = {
 	.config_file = "osmo-hlr.cfg",
 	.db_file = NULL,
@@ -735,6 +739,7 @@ static void handle_options(int argc, char **argv)
 			{"log-level", 1, 0, 'e'},
 			{"timestamp", 0, 0, 'T'},
 			{"db-upgrade", 0, 0, 'U' },
+			{"db-check", 0, 0, 'C' },
 			{"version", 0, 0, 'V' },
 			{0, 0, 0, 0}
 		};
@@ -773,6 +778,9 @@ static void handle_options(int argc, char **argv)
 		case 'U':
 			cmdline_opts.db_upgrade = true;
 			break;
+		case 'C':
+			cmdline_opts.db_check = true;
+			break;
 		case 'V':
 			print_version(1);
 			exit(0);
@@ -784,6 +792,11 @@ static void handle_options(int argc, char **argv)
 			break;
 		}
 	}
+
+	if (argc > optind) {
+		fprintf(stderr, "Unsupported positional arguments on command line\n");
+		exit(2);
+	}
 }

 static void signal_hdlr(int signal)
@@ -846,7 +859,7 @@ int main(int argc, char **argv)
 	vty_init(&vty_info);
 	ctrl_vty_init(hlr_ctx);
 	handle_options(argc, argv);
-	hlr_vty_init(&hlr_log_info);
+	hlr_vty_init();

 	rc = vty_read_config_file(cmdline_opts.config_file, NULL);
 	if (rc < 0) {
@@ -856,12 +869,6 @@ int main(int argc, char **argv)
 		return rc;
 	}

-	/* start telnet after reading config for vty_get_bind_addr() */
-	rc = telnet_init_dynif(hlr_ctx, NULL, vty_get_bind_addr(),
-			       OSMO_VTY_PORT_HLR);
-	if (rc < 0)
-		return rc;
-
 	LOGP(DMAIN, LOGL_NOTICE, "hlr starting\n");

 	rc = rand_init();
@@ -879,6 +886,23 @@ int main(int argc, char **argv)
 		exit(1);
 	}

+	if (cmdline_opts.db_check) {
+		LOGP(DMAIN, LOGL_NOTICE, "Cmdline option --db-check: Database was opened successfully, quitting.\n");
+		db_close(g_hlr->dbc);
+		log_fini();
+		talloc_free(hlr_ctx);
+		talloc_free(tall_vty_ctx);
+		talloc_disable_null_tracking();
+		exit(0);
+	}
+
+	/* start telnet after reading config for vty_get_bind_addr() */
+	rc = telnet_init_dynif(hlr_ctx, NULL, vty_get_bind_addr(),
+			       OSMO_VTY_PORT_HLR);
+	if (rc < 0)
+		return rc;
+
+
 	g_hlr->gs = osmo_gsup_server_create(hlr_ctx, g_hlr->gsup_bind_addr, OSMO_GSUP_PORT,
 					    read_cb, &g_lu_ops, g_hlr);
 	if (!g_hlr->gs) {
--- a/src/hlr_db_tool.c
+++ b/src/hlr_db_tool.c
@@ -29,9 +29,9 @@
 #include <osmocom/core/logging.h>
 #include <osmocom/core/application.h>

-#include "logging.h"
-#include "db.h"
-#include "rand.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/db.h>
+#include <osmocom/hlr/rand.h>

 struct hlr_db_tool_ctx {
 	/* DB context */
--- a/src/hlr_ussd.c
+++ b/src/hlr_ussd.c
@@ -29,12 +29,12 @@
 #include <string.h>
 #include <errno.h>

-#include "hlr.h"
-#include "hlr_ussd.h"
-#include "gsup_server.h"
-#include "gsup_router.h"
-#include "logging.h"
-#include "db.h"
+#include <osmocom/hlr/hlr.h>
+#include <osmocom/hlr/hlr_ussd.h>
+#include <osmocom/hlr/gsup_server.h>
+#include <osmocom/hlr/gsup_router.h>
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/db.h>

 /***********************************************************************
 * core data structures expressing config from VTY
@@ -317,7 +317,7 @@ static int ss_tx_ussd_7bit(struct ss_session *ss, bool final, uint8_t invoke_id,
 * Internal USSD Handlers
 ***********************************************************************/

-#include "db.h"
+#include <osmocom/hlr/db.h>

 static int handle_ussd_own_msisdn(struct osmo_gsup_conn *conn, struct ss_session *ss,
 				  const struct osmo_gsup_message *gsup, const struct ss_request *req)
@@ -336,11 +336,11 @@ static int handle_ussd_own_msisdn(struct osmo_gsup_conn *conn, struct ss_session
 		ss_tx_ussd_7bit(ss, true, req->invoke_id, buf);
 		break;
 	case -ENOENT:
-		ss_tx_error(ss, true, GSM0480_ERR_CODE_UNKNOWN_SUBSCRIBER);
+		ss_tx_error(ss, req->invoke_id, GSM0480_ERR_CODE_UNKNOWN_SUBSCRIBER);
 		break;
 	case -EIO:
 	default:
-		ss_tx_error(ss, true, GSM0480_ERR_CODE_SYSTEM_FAILURE);
+		ss_tx_error(ss, req->invoke_id, GSM0480_ERR_CODE_SYSTEM_FAILURE);
 		break;
 	}
 	return 0;
@@ -464,8 +464,7 @@ static int handle_ussd(struct osmo_gsup_conn *conn, struct ss_session *ss,
 	}

 	if (is_euse_originated) {
-		msg_out = msgb_alloc_headroom(1024+16, 16, "GSUP USSD FW");
-		OSMO_ASSERT(msg_out);
+		msg_out = osmo_gsup_msgb_alloc("GSUP USSD FW");
 		/* Received from EUSE, Forward to VLR */
 		osmo_gsup_encode(msg_out, gsup);
 		ss_gsup_send(ss, conn->server, msg_out);
@@ -481,8 +480,7 @@ static int handle_ussd(struct osmo_gsup_conn *conn, struct ss_session *ss,
 				LOGPSS(ss, LOGL_ERROR, "Cannot find conn for EUSE %s\n", addr);
 				ss_tx_error(ss, req->invoke_id, GSM0480_ERR_CODE_SYSTEM_FAILURE);
 			} else {
-				msg_out = msgb_alloc_headroom(1024+16, 16, "GSUP USSD FW");
-				OSMO_ASSERT(msg_out);
+				msg_out = osmo_gsup_msgb_alloc("GSUP USSD FW");
 				osmo_gsup_encode(msg_out, gsup);
 				osmo_gsup_conn_send(conn, msg_out);
 			}
@@ -519,6 +517,11 @@ int rx_proc_ss_req(struct osmo_gsup_conn *conn, const struct osmo_gsup_message *
 			/* FIXME: Send a Reject component? */
 			goto out_err;
 		}
+	} else if (gsup->session_state != OSMO_GSUP_SESSION_STATE_END) {
+		LOGP(DSS, LOGL_ERROR, "%s/0x%082x: Missing SS payload for '%s'\n",
+		     gsup->imsi, gsup->session_id,
+		     osmo_gsup_session_state_name(gsup->session_state));
+		goto out_err;
 	}

 	switch (gsup->session_state) {
@@ -606,13 +609,18 @@ int rx_proc_ss_req(struct osmo_gsup_conn *conn, const struct osmo_gsup_message *
 				gsup->imsi, gsup->session_id);
 			goto out_err;
 		}
-		if (ss_op_is_ussd(req.opcode)) {
-			/* dispatch unstructured SS to routing */
-			handle_ussd(conn, ss, gsup, &req);
-		} else {
-			/* dispatch non-call SS to internal code */
-			handle_ss(ss, gsup, &req);
+
+		/* SS payload is optional for END */
+		if (gsup->ss_info && gsup->ss_info_len) {
+			if (ss_op_is_ussd(req.opcode)) {
+				/* dispatch unstructured SS to routing */
+				handle_ussd(conn, ss, gsup, &req);
+			} else {
+				/* dispatch non-call SS to internal code */
+				handle_ss(ss, gsup, &req);
+			}
 		}
+
 		ss_session_free(ss);
 		break;
 	default:
--- a/src/hlr_vty.c
+++ b/src/hlr_vty.c
@@ -33,12 +33,12 @@
 #include <osmocom/vty/misc.h>
 #include <osmocom/abis/ipa.h>

-#include "db.h"
-#include "hlr.h"
-#include "hlr_vty.h"
-#include "hlr_vty_subscr.h"
-#include "hlr_ussd.h"
-#include "gsup_server.h"
+#include <osmocom/hlr/db.h>
+#include <osmocom/hlr/hlr.h>
+#include <osmocom/hlr/hlr_vty.h>
+#include <osmocom/hlr/hlr_vty_subscr.h>
+#include <osmocom/hlr/hlr_ussd.h>
+#include <osmocom/hlr/gsup_server.h>

 struct cmd_node hlr_node = {
 	HLR_NODE,
@@ -150,7 +150,7 @@ DEFUN(cfg_hlr_gsup_bind_ip,
 * USSD Entity
 ***********************************************************************/

-#include "hlr_ussd.h"
+#include <osmocom/hlr/hlr_ussd.h>

 #define USSD_STR "USSD Configuration\n"
 #define UROUTE_STR "Routing Configuration\n"
@@ -429,9 +429,9 @@ int hlr_vty_is_config_node(struct vty *vty, int node)
 	}
 }

-void hlr_vty_init(const struct log_info *cat)
+void hlr_vty_init(void)
 {
-	logging_vty_add_cmds(cat);
+	logging_vty_add_cmds();
 	osmo_talloc_vty_add_cmds();
 	osmo_stats_vty_add_cmds();

--- a/src/hlr_vty_subscr.c
+++ b/src/hlr_vty_subscr.c
@@ -28,8 +28,8 @@
 #include <osmocom/vty/command.h>
 #include <osmocom/core/utils.h>

-#include "hlr.h"
-#include "db.h"
+#include <osmocom/hlr/hlr.h>
+#include <osmocom/hlr/db.h>

 struct vty;

@@ -47,12 +47,20 @@ get_datestr(const time_t *t, char *datebuf)
 	return s;
 }

+static void dump_last_lu_seen(struct vty *vty, const char *domain_label, time_t last_lu_seen)
+{
+	char datebuf[26]; /* for ctime_r(3) */
+	if (!last_lu_seen)
+		return;
+	vty_out(vty, "    last LU seen on %s: %s UTC%s", domain_label, get_datestr(&last_lu_seen, datebuf),
+		VTY_NEWLINE);
+}
+
 static void subscr_dump_full_vty(struct vty *vty, struct hlr_subscriber *subscr)
 {
 	int rc;
 	struct osmo_sub_auth_data aud2g;
 	struct osmo_sub_auth_data aud3g;
-	char datebuf[26]; /* for ctime_r(3) */

 	vty_out(vty, "    ID: %"PRIu64"%s", subscr->id, VTY_NEWLINE);

@@ -87,8 +95,8 @@ static void subscr_dump_full_vty(struct vty *vty, struct hlr_subscriber *subscr)
 		vty_out(vty, "    PS disabled%s", VTY_NEWLINE);
 	if (subscr->ms_purged_ps)
 		vty_out(vty, "    PS purged%s", VTY_NEWLINE);
-	if (subscr->last_lu_seen)
-		vty_out(vty, "    last LU seen: %s UTC%s", get_datestr(&subscr->last_lu_seen, datebuf), VTY_NEWLINE);
+	dump_last_lu_seen(vty, "CS", subscr->last_lu_seen);
+	dump_last_lu_seen(vty, "PS", subscr->last_lu_seen_ps);

 	if (!*subscr->imsi)
 		return;
@@ -577,6 +585,33 @@ DEFUN(subscriber_imei,
 	return CMD_SUCCESS;
 }

+DEFUN(subscriber_nam,
+      subscriber_nam_cmd,
+      SUBSCR_UPDATE "network-access-mode (none|cs|ps|cs+ps)",
+      SUBSCR_UPDATE_HELP
+      "Set Network Access Mode (NAM) of the subscriber\n"
+      "Do not allow access to circuit switched or packet switched services\n"
+      "Allow access to circuit switched services only\n"
+      "Allow access to packet switched services only\n"
+      "Allow access to both circuit and packet switched services\n")
+{
+	struct hlr_subscriber subscr;
+	const char *id_type = argv[0];
+	const char *id = argv[1];
+	bool nam_cs = strstr(argv[2], "cs");
+	bool nam_ps = strstr(argv[2], "ps");
+
+	if (get_subscr_by_argv(vty, id_type, id, &subscr))
+		return CMD_WARNING;
+
+	if (nam_cs != subscr.nam_cs)
+		hlr_subscr_nam(g_hlr, &subscr, nam_cs, 0);
+	if (nam_ps != subscr.nam_ps)
+		hlr_subscr_nam(g_hlr, &subscr, nam_ps, 1);
+
+	return CMD_SUCCESS;
+}
+

 void hlr_vty_subscriber_init(void)
 {
@@ -590,4 +625,5 @@ void hlr_vty_subscriber_init(void)
 	install_element(ENABLE_NODE, &subscriber_no_aud3g_cmd);
 	install_element(ENABLE_NODE, &subscriber_aud3g_cmd);
 	install_element(ENABLE_NODE, &subscriber_imei_cmd);
+	install_element(ENABLE_NODE, &subscriber_nam_cmd);
 }
--- a/src/logging.c
+++ b/src/logging.c
@@ -1,5 +1,5 @@
 #include <osmocom/core/utils.h>
-#include "logging.h"
+#include <osmocom/hlr/logging.h>

 const struct log_info_cat hlr_log_info_cat[] = {
 	[DMAIN] = {
--- a/src/luop.c
+++ b/src/luop.c
@@ -28,10 +28,10 @@
 #include <osmocom/gsm/gsup.h>
 #include <osmocom/gsm/apn.h>

-#include "gsup_server.h"
-#include "gsup_router.h"
-#include "logging.h"
-#include "luop.h"
+#include <osmocom/hlr/gsup_server.h>
+#include <osmocom/hlr/gsup_router.h>
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/luop.h>

 const struct value_string lu_state_names[] = {
 	{ LU_S_NULL,			"NULL" },
@@ -50,8 +50,7 @@ static void _luop_tx_gsup(struct lu_operation *luop,
 {
 	struct msgb *msg_out;

-	msg_out = msgb_alloc_headroom(1024+16, 16, "GSUP LUOP");
-	OSMO_ASSERT(msg_out);
+	msg_out = osmo_gsup_msgb_alloc("GSUP LUOP");
 	osmo_gsup_encode(msg_out, gsup);

 	osmo_gsup_addr_send(luop->gsup_server, luop->peer,
--- a/src/osmo-euse-demo.c
+++ b/src/osmo-euse-demo.c
@@ -44,7 +44,7 @@

 #include <osmocom/gsupclient/gsup_client.h>

-#include "logging.h"
+#include <osmocom/hlr/logging.h>

 static struct osmo_gsup_client *g_gc;

--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -2,7 +2,7 @@ SUBDIRS = \
 	auc \
 	gsup_server \
 	db \
-	gsup \
+	db_upgrade \
 	$(NULL)

 # The `:;' works around a Bash 3.2 bug when the output is not writeable.
@@ -44,12 +44,14 @@ python-tests:
 # don't run vty and ctrl tests concurrently so that the ports don't conflict
 	$(MAKE) vty-test
 	$(MAKE) ctrl-test
+	$(MAKE) db-upgrade-equivalence-test
 else
 python-tests:
 	echo "Not running python-based external tests (determined at configure-time)"
 endif

 VTY_TEST_DB = hlr_vty_test.db
+VTY_TEST ?= *.vty

 # To update the VTY script from current application behavior,
 # pass -u to vty_script_runner.py by doing:
@@ -61,9 +63,8 @@ vty-test:
 	osmo_verify_transcript_vty.py -v \
 		-n OsmoHLR -p 4258 \
 		-r "$(top_builddir)/src/osmo-hlr -c $(top_srcdir)/doc/examples/osmo-hlr.cfg -l $(VTY_TEST_DB)" \
-		$(U) $(srcdir)/*.vty
-	-rm -f $(VTY_TEST_DB)
-	-rm $(VTY_TEST_DB)-*
+		$(U) $(srcdir)/$(VTY_TEST)
+	-rm -f $(VTY_TEST_DB) $(VTY_TEST_DB)-*

 CTRL_TEST_DB = hlr_ctrl_test.db

@@ -81,6 +82,9 @@ ctrl-test:
 	-rm -f $(CTRL_TEST_DB)
 	-rm $(CTRL_TEST_DB)-*

+db-upgrade-equivalence-test:
+	$(MAKE) -C db_upgrade upgrade-equivalence-test
+
 check-local: atconfig $(TESTSUITE)
 	$(SHELL) '$(TESTSUITE)' $(TESTSUITEFLAGS)
 	$(MAKE) $(AM_MAKEFLAGS) python-tests
--- a/tests/auc/Makefile.am
+++ b/tests/auc/Makefile.am
@@ -2,12 +2,12 @@ SUBDIRS = gen_ts_55_205_test_sets

 AM_CPPFLAGS = \
 	$(all_includes) \
-	-I$(top_srcdir)/src \
 	$(NULL)

 AM_CFLAGS = \
 	-Wall \
 	-ggdb3 \
+	-I$(top_srcdir)/include \
 	$(LIBOSMOCORE_CFLAGS) \
 	$(LIBOSMOGSM_CFLAGS) \
 	$(NULL)
--- a/tests/auc/auc_test.c
+++ b/tests/auc/auc_test.c
@@ -29,8 +29,8 @@

 #include <osmocom/crypt/auth.h>

-#include "logging.h"
-#include "auc.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/auc.h>

 #define comment_start() fprintf(stderr, "\n===== %s\n", __func__);
 #define comment_end() fprintf(stderr, "===== %s: SUCCESS\n\n", __func__);
--- a/tests/auc/gen_ts_55_205_test_sets/main_template.c
+++ b/tests/auc/gen_ts_55_205_test_sets/main_template.c
@@ -33,8 +33,8 @@

 #include <osmocom/crypt/auth.h>

-#include "logging.h"
-#include "auc.h"
+#include <osmocom/hlr/logging.h>
+#include <osmocom/hlr/auc.h>

 #define comment_start() fprintf(stderr, "\n===== %s\n", __func__);
 #define comment_end() fprintf(stderr, "===== %s: SUCCESS\n\n", __func__);
--- a/tests/auc/gen_ts_55_205_test_sets/pdftxt_2_c.py
+++ b/tests/auc/gen_ts_55_205_test_sets/pdftxt_2_c.py
@@ -1,5 +1,4 @@
-#!/usr/bin/env python
-# FIXME: use python3 once buildslaves are updated.
+#!/usr/bin/env python3
 # Convert test sets pasted from 3GPP TS 55.205 to C code.
 
 # (C) 2016 by sysmocom s.f.m.c. GmbH <info@sysmocom.de>
--- a/tests/db/Makefile.am
+++ b/tests/db/Makefile.am
@@ -1,7 +1,7 @@
 AM_CFLAGS = \
 	$(all_includes) \
-	-I$(top_srcdir)/src \
-	-I$(top_builddir)/src \
+	-I$(top_srcdir)/include \
+	-I$(top_builddir)/include \
 	-Wall \
 	-ggdb3 \
 	$(LIBOSMOCORE_CFLAGS) \
@@ -26,16 +26,20 @@ db_test_SOURCES = \
 	$(NULL)

 db_test_LDADD = \
-	$(top_srcdir)/src/db.c \
-	$(top_srcdir)/src/db_hlr.c \
-	$(top_srcdir)/src/db_auc.c \
-	$(top_srcdir)/src/logging.c \
+	$(top_builddir)/src/logging.o \
+	$(top_builddir)/src/db_auc.o \
+	$(top_builddir)/src/db_hlr.o \
+	$(top_builddir)/src/db.o \
 	$(LIBOSMOCORE_LIBS) \
 	$(LIBOSMOGSM_LIBS) \
 	$(LIBOSMOABIS_LIBS) \
 	$(SQLITE3_LIBS) \
 	$(NULL)

+if DB_SQLITE_DEBUG
+db_test_LDADD += $(top_builddir)/src/db_debug.o
+endif
+
 .PHONY: db_test.db update_exp manual manual-nonverbose manual-gdb
 db_test.db:
 	rm -f db_test.db
--- a/tests/db/db_test.c
+++ b/tests/db/db_test.c
@@ -27,8 +27,8 @@
 #include <osmocom/core/utils.h>
 #include <osmocom/core/logging.h>

-#include "db.h"
-#include "logging.h"
+#include <osmocom/hlr/db.h>
+#include <osmocom/hlr/logging.h>

 #define comment_start() fprintf(stderr, "\n===== %s\n", __func__);
 #define comment(fmt, args...) fprintf(stderr, "\n--- " fmt "\n\n", ## args);
@@ -51,7 +51,12 @@ static void _fill_invalid(void *dest, size_t size)
 * The return code is then available in g_rc. */
 #define ASSERT_RC(call, expect_rc) \
 	do { \
-		fprintf(stderr, #call " --> " #expect_rc "\n"); \
+    if ((expect_rc) == -ENOKEY) \
+		  fprintf(stderr, #call " --> -ENOKEY\n"); \
+    else if ((expect_rc) == -ENOTSUP) \
+		  fprintf(stderr, #call " --> -ENOTSUP\n"); \
+    else \
+		  fprintf(stderr, #call " --> " #expect_rc "\n"); \
 		g_rc = call; \
 		if (g_rc != (expect_rc)) \
 			fprintf(stderr, " MISMATCH: got rc = %d, expected: " \
@@ -67,7 +72,12 @@ static void _fill_invalid(void *dest, size_t size)
 	do { \
 		int rc; \
 		fill_invalid(g_subscr); \
-		fprintf(stderr, "db_subscr_get_by_" #by "(dbc, " #val ", &g_subscr) --> " \
+    if ((expect_rc) == -ENOKEY) \
+		  fprintf(stderr, "db_subscr_get_by_" #by "(dbc, " #val ", &g_subscr) --> -ENOKEY \n"); \
+    else if ((expect_rc) == -ENOTSUP) \
+		  fprintf(stderr, "db_subscr_get_by_" #by "(dbc, " #val ", &g_subscr) --> -ENOTSUP \n"); \
+    else \
+		  fprintf(stderr, "db_subscr_get_by_" #by "(dbc, " #val ", &g_subscr) --> " \
                                #expect_rc "\n"); \
 		rc = db_subscr_get_by_##by(dbc, val, &g_subscr); \
 		if (rc != (expect_rc)) \
@@ -105,7 +115,7 @@ static void _fill_invalid(void *dest, size_t size)
 #define ASSERT_DB_GET_AUC(imsi, expect_rc) \
 	do { \
 		struct osmo_auth_vector vec[N_VECTORS]; \
-		ASSERT_RC(db_get_auc(dbc, imsi, 3, vec, N_VECTORS, NULL, NULL), expect_rc); \
+		ASSERT_RC(db_get_auc(dbc, imsi, 3, vec, N_VECTORS, NULL, NULL, false), expect_rc); \
 	} while (0)

 /* Not linking the real auc_compute_vectors(), just returning num_vec.
@@ -208,6 +218,17 @@ void dump_aud(const char *label, struct osmo_sub_auth_data *aud)
 #undef Phex
 }

+void db_raw_sql(struct db_context *dbc, const char *sql)
+{
+	sqlite3_stmt *stmt;
+
+	fprintf(stderr, "raw SQL: %s\n", sql);
+	ASSERT_RC(sqlite3_prepare_v2(dbc->db, sql, -1, &stmt, NULL), SQLITE_OK);
+	ASSERT_RC(sqlite3_step(stmt), SQLITE_DONE);
+	db_remove_reset(stmt);
+	sqlite3_finalize(stmt);
+}
+
 static const char *imsi0 = "123456789000000";
 static const char *imsi1 = "123456789000001";
 static const char *imsi2 = "123456789000002";
@@ -749,6 +770,70 @@ static void test_subscr_aud()
 	comment_end();
 }

+/* Make each key too short in this test. Note that we can't set them longer than the allowed size without changing the
+ * table structure. */
+static void test_subscr_aud_invalid_len()
+{
+	int64_t id;
+
+	comment_start();
+	comment("Create subscriber");
+	ASSERT_RC(db_subscr_create(dbc, imsi0, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS), 0);
+	ASSERT_SEL(imsi, imsi0, 0);
+	id = g_subscr.id;
+
+
+	/* Invalid Ki length */
+	comment("Set auth data, 2G only, with invalid Ki length");
+	ASSERT_RC(db_subscr_update_aud_by_id(dbc, id,
+		  mk_aud_2g(OSMO_AUTH_ALG_COMP128v1, "0123456789abcdef0123456789abcdef")),
+		  0);
+	/* Use raw SQL to avoid length check in db_subscr_update_aud_by_id(). This changes all rows in the table, which
+         * is fine for this test (implicit WHERE 1). */
+	db_raw_sql(dbc, "UPDATE auc_2g SET ki = '0123456789abcdef0123456789abcde'");
+	ASSERT_SEL_AUD(imsi0, -ENOKEY, id);
+
+	comment("Remove 2G auth data");
+	ASSERT_RC(db_subscr_update_aud_by_id(dbc, id,
+		  mk_aud_2g(OSMO_AUTH_ALG_NONE, NULL)),
+		  0);
+
+	/* Invalid K length */
+	comment("Set auth data, 3G only, with invalid K length");
+	ASSERT_RC(db_subscr_update_aud_by_id(dbc, id,
+		mk_aud_3g(OSMO_AUTH_ALG_MILENAGE,
+			  "BeefedCafeFaceAcedAddedDecadeFee", true,
+			  "C01ffedC1cadaeAc1d1f1edAcac1aB0a", 5)),
+		0);
+	db_raw_sql(dbc, "UPDATE auc_3g SET k = 'C01ffedC1cadaeAc1d1f1edAcac1aB0'");
+	ASSERT_SEL_AUD(imsi0, -EIO, id);
+
+	/* Invalid OP length */
+	comment("Set auth data, 3G only, with invalid OP length");
+	ASSERT_RC(db_subscr_update_aud_by_id(dbc, id,
+		mk_aud_3g(OSMO_AUTH_ALG_MILENAGE,
+			  "BeefedCafeFaceAcedAddedDecadeFee", true,
+			  "C01ffedC1cadaeAc1d1f1edAcac1aB0a", 5)),
+		0);
+	db_raw_sql(dbc, "UPDATE auc_3g SET op = 'BeefedCafeFaceAcedAddedDecadeFe'");
+	ASSERT_SEL_AUD(imsi0, -EIO, id);
+
+	/* Invalid OPC length */
+	comment("Set auth data, 3G only, with invalid OPC length");
+	ASSERT_RC(db_subscr_update_aud_by_id(dbc, id,
+		mk_aud_3g(OSMO_AUTH_ALG_MILENAGE,
+			  "BeefedCafeFaceAcedAddedDecadeFee", false,
+			  "C01ffedC1cadaeAc1d1f1edAcac1aB0a", 5)),
+		0);
+	db_raw_sql(dbc, "UPDATE auc_3g SET opc = 'BeefedCafeFaceAcedAddedDecadeFe'");
+	ASSERT_SEL_AUD(imsi0, -EIO, id);
+
+
+	comment("Delete subscriber");
+	ASSERT_RC(db_subscr_delete_by_id(dbc, id), 0);
+	comment_end();
+}
+
 static void test_subscr_sqn()
 {
 	int64_t id;
@@ -900,9 +985,11 @@ int main(int argc, char **argv)

 	test_subscr_create_update_sel_delete();
 	test_subscr_aud();
+	test_subscr_aud_invalid_len();
 	test_subscr_sqn();

 	printf("Done\n");
+	db_close(dbc);
 	return 0;
 }

--- a/tests/db/db_test.err
+++ b/tests/db/db_test.err
@@ -814,7 +814,7 @@ db_get_auth_data(dbc, unknown_imsi, &g_aud2g, &g_aud3g, &g_id) --> -2
 DAUC IMSI='999999999': No such subscriber


-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> -2
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> -2
 DAUC IMSI='123456789000000': No such subscriber


@@ -828,12 +828,12 @@ struct hlr_subscriber {
  .imsi = '123456789000000',
 }

-db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -126
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data


-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> -126
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data

@@ -852,7 +852,7 @@ DAUC IMSI='123456789000000': No 3G Auth Data
 }
 3G: none

-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> 3
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> 3
 DAUC IMSI='123456789000000': No 3G Auth Data
 DAUC IMSI='123456789000000': Calling to generate 3 vectors
 DAUC IMSI='123456789000000': Generated 3 vectors
@@ -910,12 +910,12 @@ DAUC IMSI='123456789000000': No 3G Auth Data

 db_subscr_update_aud_by_id(dbc, id, mk_aud_2g(OSMO_AUTH_ALG_NONE, NULL)) --> 0

-db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -126
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data


-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> -126
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data

@@ -935,12 +935,12 @@ DAUC IMSI='123456789000000': No 3G Auth Data

 db_subscr_update_aud_by_id(dbc, id, mk_aud_2g(OSMO_AUTH_ALG_NONE, "f000000000000f00000000000f000000")) --> 0

-db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -126
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data


-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> -126
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data

@@ -963,7 +963,7 @@ DAUC IMSI='123456789000000': No 2G Auth Data
  .u.umts.ind_bitlen = 5,
 }

-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> 3
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> 3
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': Calling to generate 3 vectors
 DAUC IMSI='123456789000000': Generated 3 vectors
@@ -1037,12 +1037,12 @@ DAUC IMSI='123456789000000': No 2G Auth Data

 db_subscr_update_aud_by_id(dbc, id, mk_aud_3g(OSMO_AUTH_ALG_NONE, NULL, false, NULL, 0)) --> 0

-db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -126
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data


-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> -126
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data

@@ -1064,7 +1064,7 @@ DAUC IMSI='123456789000000': No 2G Auth Data
  .u.umts.ind_bitlen = 5,
 }

-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> 3
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> 3
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': Calling to generate 3 vectors
 DAUC IMSI='123456789000000': Generated 3 vectors
@@ -1072,12 +1072,12 @@ DAUC IMSI='123456789000000': Updating SQN=0 in DB

 db_subscr_update_aud_by_id(dbc, id, mk_aud_3g(OSMO_AUTH_ALG_NONE, "asdfasdfasd", false, "asdfasdfasdf", 99999)) --> 0

-db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -126
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data


-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> -126
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data

@@ -1105,7 +1105,7 @@ db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> 0
  .u.umts.ind_bitlen = 5,
 }

-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> 3
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> 3
 DAUC IMSI='123456789000000': Calling to generate 3 vectors
 DAUC IMSI='123456789000000': Generated 3 vectors
 DAUC IMSI='123456789000000': Updating SQN=0 in DB
@@ -1318,12 +1318,12 @@ struct hlr_subscriber {
  .imsi = '123456789000000',
 }

-db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -126
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data


-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> -126
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data

@@ -1332,12 +1332,97 @@ db_subscr_delete_by_id(dbc, id) --> 0
 db_subscr_get_by_imsi(dbc, imsi0, &g_subscr) --> -ENOENT
 DAUC Cannot read subscriber from db: IMSI='123456789000000': No such subscriber

-db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL) --> -2
+db_get_auc(dbc, imsi0, 3, vec, N_VECTORS, NULL, NULL, false) --> -2
 DAUC IMSI='123456789000000': No such subscriber

 ===== test_subscr_aud: SUCCESS


+===== test_subscr_aud_invalid_len
+
+--- Create subscriber
+
+db_subscr_create(dbc, imsi0, DB_SUBSCR_FLAG_NAM_CS | DB_SUBSCR_FLAG_NAM_PS) --> 0
+
+db_subscr_get_by_imsi(dbc, imsi0, &g_subscr) --> 0
+struct hlr_subscriber {
+  .id = 1,
+  .imsi = '123456789000000',
+}
+
+
+--- Set auth data, 2G only, with invalid Ki length
+
+db_subscr_update_aud_by_id(dbc, id, mk_aud_2g(OSMO_AUTH_ALG_COMP128v1, "0123456789abcdef0123456789abcdef")) --> 0
+
+raw SQL: UPDATE auc_2g SET ki = '0123456789abcdef0123456789abcde'
+sqlite3_prepare_v2(dbc->db, sql, -1, &stmt, NULL) --> SQLITE_OK
+
+sqlite3_step(stmt) --> SQLITE_DONE
+
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
+DAUC IMSI='123456789000000': Error reading Ki, expected length 16 but has length 15
+DAUC IMSI='123456789000000': No 3G Auth Data
+
+
+
+--- Remove 2G auth data
+
+db_subscr_update_aud_by_id(dbc, id, mk_aud_2g(OSMO_AUTH_ALG_NONE, NULL)) --> 0
+
+
+--- Set auth data, 3G only, with invalid K length
+
+db_subscr_update_aud_by_id(dbc, id, mk_aud_3g(OSMO_AUTH_ALG_MILENAGE, "BeefedCafeFaceAcedAddedDecadeFee", true, "C01ffedC1cadaeAc1d1f1edAcac1aB0a", 5)) --> 0
+
+raw SQL: UPDATE auc_3g SET k = 'C01ffedC1cadaeAc1d1f1edAcac1aB0'
+sqlite3_prepare_v2(dbc->db, sql, -1, &stmt, NULL) --> SQLITE_OK
+
+sqlite3_step(stmt) --> SQLITE_DONE
+
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -5
+DAUC IMSI='123456789000000': No 2G Auth Data
+DAUC IMSI='123456789000000': Error reading K, expected length 16 but has length 15
+
+
+
+--- Set auth data, 3G only, with invalid OP length
+
+db_subscr_update_aud_by_id(dbc, id, mk_aud_3g(OSMO_AUTH_ALG_MILENAGE, "BeefedCafeFaceAcedAddedDecadeFee", true, "C01ffedC1cadaeAc1d1f1edAcac1aB0a", 5)) --> 0
+
+raw SQL: UPDATE auc_3g SET op = 'BeefedCafeFaceAcedAddedDecadeFe'
+sqlite3_prepare_v2(dbc->db, sql, -1, &stmt, NULL) --> SQLITE_OK
+
+sqlite3_step(stmt) --> SQLITE_DONE
+
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -5
+DAUC IMSI='123456789000000': No 2G Auth Data
+DAUC IMSI='123456789000000': Error reading OP, expected length 16 but has length 15
+
+
+
+--- Set auth data, 3G only, with invalid OPC length
+
+db_subscr_update_aud_by_id(dbc, id, mk_aud_3g(OSMO_AUTH_ALG_MILENAGE, "BeefedCafeFaceAcedAddedDecadeFee", false, "C01ffedC1cadaeAc1d1f1edAcac1aB0a", 5)) --> 0
+
+raw SQL: UPDATE auc_3g SET opc = 'BeefedCafeFaceAcedAddedDecadeFe'
+sqlite3_prepare_v2(dbc->db, sql, -1, &stmt, NULL) --> SQLITE_OK
+
+sqlite3_step(stmt) --> SQLITE_DONE
+
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -5
+DAUC IMSI='123456789000000': No 2G Auth Data
+DAUC IMSI='123456789000000': Error reading OPC, expected length 16 but has length 15
+
+
+
+--- Delete subscriber
+
+db_subscr_delete_by_id(dbc, id) --> 0
+
+===== test_subscr_aud_invalid_len: SUCCESS
+
+
 ===== test_subscr_sqn

 --- Set SQN for unknown subscriber
@@ -1365,7 +1450,7 @@ struct hlr_subscriber {
  .imsi = '123456789000000',
 }

-db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -126
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data

@@ -1376,7 +1461,7 @@ DAUC IMSI='123456789000000': No 3G Auth Data
 db_update_sqn(dbc, id, 123) --> -ENOENT
 DAUC Cannot update SQN for subscriber ID=1: no auc_3g entry for such subscriber

-db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -126
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data

@@ -1384,7 +1469,7 @@ DAUC IMSI='123456789000000': No 3G Auth Data
 db_update_sqn(dbc, id, 543) --> -ENOENT
 DAUC Cannot update SQN for subscriber ID=1: no auc_3g entry for such subscriber

-db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -126
+db_get_auth_data(dbc, imsi0, &g_aud2g, &g_aud3g, &g_id) --> -ENOKEY
 DAUC IMSI='123456789000000': No 2G Auth Data
 DAUC IMSI='123456789000000': No 3G Auth Data

--- a/tests/db_upgrade/Makefile.am
+++ b/tests/db_upgrade/Makefile.am
@@ -0,0 +1,14 @@
+EXTRA_DIST = \
+	db_upgrade_test.sh \
+	db_upgrade_test.err \
+	db_upgrade_test.ok \
+	hlr_db_v0.sql \
+	osmo-hlr.cfg \
+	create_subscribers.vty \
+	$(NULL)
+
+update_exp:
+	$(srcdir)/db_upgrade_test.sh $(srcdir) $(builddir) >"$(srcdir)/db_upgrade_test.ok" 2>"$(srcdir)/db_upgrade_test.err"
+
+upgrade-equivalence-test:
+	$(srcdir)/db_upgrade_test.sh $(srcdir) $(builddir) do-equivalence-test
--- a/tests/db_upgrade/create_subscribers.vty
+++ b/tests/db_upgrade/create_subscribers.vty
@@ -0,0 +1,47 @@
+OsmoHLR> enable
+OsmoHLR# subscriber imsi 123456789012345 create
+% Created subscriber 123456789012345
+    ID: 1
+    IMSI: 123456789012345
+    MSISDN: none
+OsmoHLR# subscriber imsi 123456789012345 update msisdn 098765432109876
+% Updated subscriber IMSI='123456789012345' to MSISDN='098765432109876'
+OsmoHLR# subscriber imsi 123456789012345 update aud2g comp128v1 ki BeefedCafeFaceAcedAddedDecadeFee
+OsmoHLR# subscriber imsi 123456789012345 update aud3g milenage k C01ffedC1cadaeAc1d1f1edAcac1aB0a opc CededEffacedAceFacedBadFadedBeef
+OsmoHLR# subscriber imsi 111111111 create
+% Created subscriber 111111111
+    ID: 2
+    IMSI: 111111111
+    MSISDN: none
+OsmoHLR# subscriber imsi 222222222 create
+% Created subscriber 222222222
+    ID: 3
+    IMSI: 222222222
+    MSISDN: none
+OsmoHLR# subscriber imsi 222222222 update msisdn 22222
+% Updated subscriber IMSI='222222222' to MSISDN='22222'
+OsmoHLR# subscriber imsi 333333 create
+% Created subscriber 333333
+    ID: 4
+    IMSI: 333333
+    MSISDN: none
+OsmoHLR# subscriber imsi 333333 update msisdn 3
+% Updated subscriber IMSI='333333' to MSISDN='3'
+OsmoHLR# subscriber imsi 333333 update aud2g comp128v2 ki 33333333333333333333333333333333
+OsmoHLR# subscriber imsi 444444444444444 create
+% Created subscriber 444444444444444
+    ID: 5
+    IMSI: 444444444444444
+    MSISDN: none
+OsmoHLR# subscriber imsi 444444444444444 update msisdn 4444
+% Updated subscriber IMSI='444444444444444' to MSISDN='4444'
+OsmoHLR# subscriber imsi 444444444444444 update aud3g milenage k 44444444444444444444444444444444 op 44444444444444444444444444444444
+OsmoHLR# subscriber imsi 5555555 create
+% Created subscriber 5555555
+    ID: 6
+    IMSI: 5555555
+    MSISDN: none
+OsmoHLR# subscriber imsi 5555555 update msisdn 55555555555555
+% Updated subscriber IMSI='5555555' to MSISDN='55555555555555'
+OsmoHLR# subscriber imsi 5555555 update aud2g xor ki 55555555555555555555555555555555
+OsmoHLR# subscriber imsi 5555555 update aud3g milenage k 55555555555555555555555555555555 opc 55555555555555555555555555555555
--- a/tests/db_upgrade/create_subscribers_step2.sql
+++ b/tests/db_upgrade/create_subscribers_step2.sql
@@ -0,0 +1,6 @@
+update subscriber set vlr_number = 'MSC-1' where id = 1;
+update subscriber set ms_purged_cs = 1 where id = 2;
+update subscriber set ms_purged_ps = 1 where id = 3;
+update subscriber set nam_cs = 0 where id = 4;
+update subscriber set nam_ps = 0 where id = 5;
+update subscriber set nam_cs = 0, nam_ps = 0 where id = 6;
--- a/tests/db_upgrade/db_upgrade_test.err
+++ b/tests/db_upgrade/db_upgrade_test.err
--- a/tests/db_upgrade/db_upgrade_test.ok
+++ b/tests/db_upgrade/db_upgrade_test.ok
@@ -0,0 +1,172 @@
+Creating db in schema version 0
+
+Version 0 db:
+
+Table: auc_2g
+name|type|notnull|dflt_value|pk
+algo_id_2g|INTEGER|1||0
+ki|VARCHAR(32)|1||0
+subscriber_id|INTEGER|0||1
+
+Table auc_2g contents:
+algo_id_2g|ki|subscriber_id
+1|BeefedCafeFaceAcedAddedDecadeFee|1
+2|33333333333333333333333333333333|4
+4|55555555555555555555555555555555|6
+
+Table: auc_3g
+name|type|notnull|dflt_value|pk
+algo_id_3g|INTEGER|1||0
+ind_bitlen|INTEGER|1|5|0
+k|VARCHAR(32)|1||0
+op|VARCHAR(32)|0||0
+opc|VARCHAR(32)|0||0
+sqn|INTEGER|1|0|0
+subscriber_id|INTEGER|0||1
+
+Table auc_3g contents:
+algo_id_3g|ind_bitlen|k|op|opc|sqn|subscriber_id
+5|5|C01ffedC1cadaeAc1d1f1edAcac1aB0a||CededEffacedAceFacedBadFadedBeef|0|1
+5|5|44444444444444444444444444444444|44444444444444444444444444444444||0|5
+5|5|55555555555555555555555555555555||55555555555555555555555555555555|0|6
+
+Table: subscriber
+name|type|notnull|dflt_value|pk
+ggsn_number|VARCHAR(15)|0||0
+gmlc_number|VARCHAR(15)|0||0
+hlr_number|VARCHAR(15)|0||0
+id|INTEGER|0||1
+imeisv|VARCHAR|0||0
+imsi|VARCHAR(15)|1||0
+lmsi|INTEGER|0||0
+ms_purged_cs|BOOLEAN|1|0|0
+ms_purged_ps|BOOLEAN|1|0|0
+msisdn|VARCHAR(15)|0||0
+nam_cs|BOOLEAN|1|1|0
+nam_ps|BOOLEAN|1|1|0
+periodic_lu_tmr|INTEGER|0||0
+periodic_rau_tau_tmr|INTEGER|0||0
+sgsn_address|VARCHAR|0||0
+sgsn_number|VARCHAR(15)|0||0
+smsc_number|VARCHAR(15)|0||0
+vlr_number|VARCHAR(15)|0||0
+
+Table subscriber contents:
+ggsn_number|gmlc_number|hlr_number|id|imeisv|imsi|lmsi|ms_purged_cs|ms_purged_ps|msisdn|nam_cs|nam_ps|periodic_lu_tmr|periodic_rau_tau_tmr|sgsn_address|sgsn_number|smsc_number|vlr_number
+|||1||123456789012345||0|0|098765432109876|1|1||||||MSC-1
+|||2||111111111||1|0||1|1||||||
+|||3||222222222||0|1|22222|1|1||||||
+|||4||333333||0|0|3|0|1||||||
+|||5||444444444444444||0|0|4444|1|0||||||
+|||6||5555555||0|0|55555555555555|0|0||||||
+
+Table: subscriber_apn
+name|type|notnull|dflt_value|pk
+apn|VARCHAR(256)|1||0
+subscriber_id|INTEGER|0||0
+
+Table subscriber_apn contents:
+
+Table: subscriber_multi_msisdn
+name|type|notnull|dflt_value|pk
+msisdn|VARCHAR(15)|1||0
+subscriber_id|INTEGER|0||0
+
+Table subscriber_multi_msisdn contents:
+
+Launching osmo-hlr to upgrade db:
+osmo-hlr --database $db --db-upgrade --db-check --config-file $srcdir/osmo-hlr.cfg
+rc = 0
+DMAIN hlr starting
+DDB using database: <PATH>test.db
+DDB Database <PATH>test.db' has HLR DB schema version 0
+DDB Database <PATH>test.db' has been upgraded to HLR DB schema version 1
+DDB Database <PATH>test.db' has been upgraded to HLR DB schema version 2
+DDB Database <PATH>test.db' has been upgraded to HLR DB schema version 3
+DDB Database <PATH>test.db' has been upgraded to HLR DB schema version 4
+DMAIN Cmdline option --db-check: Database was opened successfully, quitting.
+
+Resulting db:
+
+Table: auc_2g
+name|type|notnull|dflt_value|pk
+algo_id_2g|INTEGER|1||0
+ki|VARCHAR(32)|1||0
+subscriber_id|INTEGER|0||1
+
+Table auc_2g contents:
+algo_id_2g|ki|subscriber_id
+1|BeefedCafeFaceAcedAddedDecadeFee|1
+2|33333333333333333333333333333333|4
+4|55555555555555555555555555555555|6
+
+Table: auc_3g
+name|type|notnull|dflt_value|pk
+algo_id_3g|INTEGER|1||0
+ind_bitlen|INTEGER|1|5|0
+k|VARCHAR(32)|1||0
+op|VARCHAR(32)|0||0
+opc|VARCHAR(32)|0||0
+sqn|INTEGER|1|0|0
+subscriber_id|INTEGER|0||1
+
+Table auc_3g contents:
+algo_id_3g|ind_bitlen|k|op|opc|sqn|subscriber_id
+5|5|C01ffedC1cadaeAc1d1f1edAcac1aB0a||CededEffacedAceFacedBadFadedBeef|0|1
+5|5|44444444444444444444444444444444|44444444444444444444444444444444||0|5
+5|5|55555555555555555555555555555555||55555555555555555555555555555555|0|6
+
+Table: subscriber
+name|type|notnull|dflt_value|pk
+ggsn_number|VARCHAR(15)|0||0
+gmlc_number|VARCHAR(15)|0||0
+id|INTEGER|0||1
+imei|VARCHAR(14)|0||0
+imeisv|VARCHAR|0||0
+imsi|VARCHAR(15)|1||0
+last_lu_seen|TIMESTAMP|0|NULL|0
+last_lu_seen_ps|TIMESTAMP|0|NULL|0
+lmsi|INTEGER|0||0
+ms_purged_cs|BOOLEAN|1|0|0
+ms_purged_ps|BOOLEAN|1|0|0
+msc_number|VARCHAR(15)|0||0
+msisdn|VARCHAR(15)|0||0
+nam_cs|BOOLEAN|1|1|0
+nam_ps|BOOLEAN|1|1|0
+periodic_lu_tmr|INTEGER|0||0
+periodic_rau_tau_tmr|INTEGER|0||0
+sgsn_address|VARCHAR|0||0
+sgsn_number|VARCHAR(15)|0||0
+smsc_number|VARCHAR(15)|0||0
+vlr_number|VARCHAR(15)|0||0
+
+Table subscriber contents:
+ggsn_number|gmlc_number|id|imei|imeisv|imsi|last_lu_seen|last_lu_seen_ps|lmsi|ms_purged_cs|ms_purged_ps|msc_number|msisdn|nam_cs|nam_ps|periodic_lu_tmr|periodic_rau_tau_tmr|sgsn_address|sgsn_number|smsc_number|vlr_number
+||1|||123456789012345||||0|0||098765432109876|1|1||||||MSC-1
+||2|||111111111||||1|0|||1|1||||||
+||3|||222222222||||0|1||22222|1|1||||||
+||4|||333333||||0|0||3|0|1||||||
+||5|||444444444444444||||0|0||4444|1|0||||||
+||6|||5555555||||0|0||55555555555555|0|0||||||
+
+Table: subscriber_apn
+name|type|notnull|dflt_value|pk
+apn|VARCHAR(256)|1||0
+subscriber_id|INTEGER|0||0
+
+Table subscriber_apn contents:
+
+Table: subscriber_multi_msisdn
+name|type|notnull|dflt_value|pk
+msisdn|VARCHAR(15)|1||0
+subscriber_id|INTEGER|0||0
+
+Table subscriber_multi_msisdn contents:
+
+Verify that osmo-hlr can open it:
+osmo-hlr --database $db --db-check --config-file $srcdir/osmo-hlr.cfg
+rc = 0
+DMAIN hlr starting
+DDB using database: <PATH>test.db
+DDB Database <PATH>test.db' has HLR DB schema version 4
+DMAIN Cmdline option --db-check: Database was opened successfully, quitting.
--- a/tests/db_upgrade/db_upgrade_test.sh
+++ b/tests/db_upgrade/db_upgrade_test.sh
@@ -0,0 +1,83 @@
+#!/bin/sh
+srcdir="${1:-.}"
+builddir="${2:-.}"
+do_equivalence_test="$3"
+
+set -e
+
+db="$builddir/test.db"
+osmo_hlr="$builddir/../../src/osmo-hlr"
+cfg="$srcdir/osmo-hlr.cfg"
+
+dump_sorted_schema(){
+	db_file="$1"
+	tables="$(sqlite3 -batch -noheader "$db_file" "SELECT name FROM sqlite_master WHERE type = 'table' order by name")"
+	for table in $tables; do
+		echo
+		echo "Table: $table"
+		sqlite3 -batch -header "$db_file" "SELECT name,type,\"notnull\",dflt_value,pk FROM PRAGMA_TABLE_INFO('$table') order by name;"
+		echo
+		echo "Table $table contents:"
+		columns="$(sqlite3 -batch -noheader "$db_file" "SELECT name FROM PRAGMA_TABLE_INFO('$table') order by name;")"
+		sqlite3 -batch -header "$db_file" "SELECT $(echo $columns | sed 's/ /,/g') from $table;"
+	done
+}
+
+rm -f "$db"
+echo "Creating db in schema version 0"
+sqlite3 -batch "$db" < "$srcdir/hlr_db_v0.sql"
+
+echo
+echo "Version 0 db:"
+dump_sorted_schema "$db"
+
+set +e
+
+echo
+echo "Launching osmo-hlr to upgrade db:"
+echo osmo-hlr --database '$db' --db-upgrade --db-check --config-file '$srcdir/osmo-hlr.cfg'
+"$osmo_hlr" --database "$db" --db-upgrade --db-check --config-file "$cfg" >log 2>&1
+echo "rc = $?"
+cat log | sed 's@[^ "]*/@<PATH>@g'
+
+echo
+echo "Resulting db:"
+dump_sorted_schema "$db"
+
+echo
+echo "Verify that osmo-hlr can open it:"
+echo osmo-hlr --database '$db' --db-check --config-file '$srcdir/osmo-hlr.cfg'
+"$osmo_hlr" --database "$db" --db-check --config-file "$cfg" >log 2>&1
+echo "rc = $?"
+cat log | sed 's@[^ "]*/@<PATH>@g'
+
+if [ -n "$do_equivalence_test" ]; then
+	# this part requires osmo_interact_vty.py, so this test is not part of the normal run
+	set -e -x
+	mint_db="$builddir/mint.db"
+	rm -f "$mint_db"
+
+	osmo_verify_transcript_vty.py -v \
+		-n OsmoHLR -p 4258 \
+		-r "$osmo_hlr -c $cfg -l $mint_db" \
+		"$srcdir/create_subscribers.vty"
+	sqlite3 -batch "$mint_db" < "$srcdir/create_subscribers_step2.sql"
+
+	set +x
+	test_dump="$builddir/test.dump"
+	mint_dump="$builddir/mint.dump"
+
+	dump_sorted_schema "$db" > "$test_dump"
+	dump_sorted_schema "$mint_db" > "$mint_dump"
+
+	echo
+	echo "Newly created sorted schema is:"
+	cat "$mint_dump"
+	echo
+	echo "Diff to upgraded schema:"
+	diff -u "$mint_dump" "$test_dump"
+	echo "rc=$?"
+fi
+
+rm -f log
+rm -f "$db"
--- a/tests/db_upgrade/hlr_db_v0.sql
+++ b/tests/db_upgrade/hlr_db_v0.sql
@@ -0,0 +1,80 @@
+PRAGMA foreign_keys=OFF;
+BEGIN TRANSACTION;
+CREATE TABLE subscriber (
+-- OsmoHLR's DB scheme is modelled roughly after TS 23.008 version 13.3.0
+	id		INTEGER PRIMARY KEY,
+	-- Chapter 2.1.1.1
+	imsi		VARCHAR(15) UNIQUE NOT NULL,
+	-- Chapter 2.1.2
+	msisdn		VARCHAR(15) UNIQUE,
+	-- Chapter 2.2.3: Most recent / current IMEI
+	imeisv		VARCHAR,
+	-- Chapter 2.4.5
+	vlr_number	VARCHAR(15),
+	-- Chapter 2.4.6
+	hlr_number	VARCHAR(15),
+	-- Chapter 2.4.8.1
+	sgsn_number	VARCHAR(15),
+	-- Chapter 2.13.10
+	sgsn_address	VARCHAR,
+	-- Chapter 2.4.8.2
+	ggsn_number	VARCHAR(15),
+	-- Chapter 2.4.9.2
+	gmlc_number	VARCHAR(15),
+	-- Chapter 2.4.23
+	smsc_number	VARCHAR(15),
+	-- Chapter 2.4.24
+	periodic_lu_tmr	INTEGER,
+	-- Chapter 2.13.115
+	periodic_rau_tau_tmr INTEGER,
+	-- Chapter 2.1.1.2: network access mode
+	nam_cs		BOOLEAN NOT NULL DEFAULT 1,
+	nam_ps		BOOLEAN NOT NULL DEFAULT 1,
+	-- Chapter 2.1.8
+	lmsi		INTEGER,
+	-- The below purged flags might not even be stored non-volatile,
+	-- refer to TS 23.012 Chapter 3.6.1.4
+	-- Chapter 2.7.5
+	ms_purged_cs	BOOLEAN NOT NULL DEFAULT 0,
+	-- Chapter 2.7.6
+	ms_purged_ps	BOOLEAN NOT NULL DEFAULT 0
+);
+INSERT INTO subscriber VALUES(1,'123456789012345','098765432109876',NULL,'MSC-1',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,1,NULL,0,0);
+INSERT INTO subscriber VALUES(2,'111111111',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,1,NULL,1,0);
+INSERT INTO subscriber VALUES(3,'222222222','22222',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,1,NULL,0,1);
+INSERT INTO subscriber VALUES(4,'333333','3',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,1,NULL,0,0);
+INSERT INTO subscriber VALUES(5,'444444444444444','4444',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,0,NULL,0,0);
+INSERT INTO subscriber VALUES(6,'5555555','55555555555555',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,NULL,0,0);
+CREATE TABLE subscriber_apn (
+	subscriber_id	INTEGER,		-- subscriber.id
+	apn		VARCHAR(256) NOT NULL
+);
+CREATE TABLE subscriber_multi_msisdn (
+-- Chapter 2.1.3
+	subscriber_id	INTEGER,		-- subscriber.id
+	msisdn		VARCHAR(15) NOT NULL
+);
+CREATE TABLE auc_2g (
+	subscriber_id	INTEGER PRIMARY KEY,	-- subscriber.id
+	algo_id_2g	INTEGER NOT NULL,	-- enum osmo_auth_algo value
+	ki		VARCHAR(32) NOT NULL	-- hex string: subscriber's secret key (128bit)
+);
+INSERT INTO auc_2g VALUES(1,1,'BeefedCafeFaceAcedAddedDecadeFee');
+INSERT INTO auc_2g VALUES(4,2,'33333333333333333333333333333333');
+INSERT INTO auc_2g VALUES(6,4,'55555555555555555555555555555555');
+CREATE TABLE auc_3g (
+	subscriber_id	INTEGER PRIMARY KEY,	-- subscriber.id
+	algo_id_3g	INTEGER NOT NULL,	-- enum osmo_auth_algo value
+	k		VARCHAR(32) NOT NULL,	-- hex string: subscriber's secret key (128bit)
+	op		VARCHAR(32),		-- hex string: operator's secret key (128bit)
+	opc		VARCHAR(32),		-- hex string: derived from OP and K (128bit)
+	sqn		INTEGER NOT NULL DEFAULT 0,	-- sequence number of key usage
+	-- nr of index bits at lower SQN end
+	ind_bitlen	INTEGER NOT NULL DEFAULT 5
+);
+INSERT INTO auc_3g VALUES(1,5,'C01ffedC1cadaeAc1d1f1edAcac1aB0a',NULL,'CededEffacedAceFacedBadFadedBeef',0,5);
+INSERT INTO auc_3g VALUES(5,5,'44444444444444444444444444444444','44444444444444444444444444444444',NULL,0,5);
+INSERT INTO auc_3g VALUES(6,5,'55555555555555555555555555555555',NULL,'55555555555555555555555555555555',0,5);
+CREATE UNIQUE INDEX idx_subscr_imsi ON subscriber (imsi)
+;
+COMMIT;
--- a/tests/db_upgrade/osmo-hlr.cfg
+++ b/tests/db_upgrade/osmo-hlr.cfg
@@ -0,0 +1,6 @@
+log stderr
+ logging level db notice
+ logging print category-hex 0
+ logging print file 0
+ logging print category 1
+ logging color 0
--- a/tests/gsup/Makefile.am
+++ b/tests/gsup/Makefile.am
@@ -1,42 +0,0 @@
-AM_CPPFLAGS = \
-	$(all_includes) \
-	-I$(top_srcdir)/src \
-	$(NULL)
-
-AM_CFLAGS = \
-	-Wall \
-	-ggdb3 \
-	$(LIBOSMOCORE_CFLAGS) \
-	$(LIBOSMOGSM_CFLAGS) \
-	$(LIBOSMOABIS_CFLAGS) \
-	$(NULL)
-
-AM_LDFLAGS = \
-	-no-install \
-	$(NULL)
-
-EXTRA_DIST = \
-	gsup_test.ok \
-	gsup_test.err \
-	$(NULL)
-
-noinst_PROGRAMS = \
-	gsup_test \
-	$(NULL)
-
-gsup_test_SOURCES = \
-	gsup_test.c \
-	$(NULL)
-
-gsup_test_LDADD = \
-	$(top_srcdir)/src/luop.c \
-	$(top_srcdir)/src/gsup_server.c \
-	$(top_srcdir)/src/gsup_router.c \
-	$(LIBOSMOCORE_LIBS) \
-	$(LIBOSMOGSM_LIBS) \
-	$(LIBOSMOABIS_LIBS) \
-	$(NULL)
-
-.PHONY: update_exp
-update_exp:
-	$(builddir)/gsup_test >"$(srcdir)/gsup_test.ok" 2>"$(srcdir)/gsup_test.err"
--- a/tests/gsup/gsup_test.c
+++ b/tests/gsup/gsup_test.c
@@ -1,91 +0,0 @@
-/* (C) 2018 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
- *
- * All Rights Reserved
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- */
-
-#include <string.h>
-
-#include <osmocom/core/logging.h>
-#include <osmocom/core/utils.h>
-#include <osmocom/core/application.h>
-#include <osmocom/gsm/gsup.h>
-
-#include "logging.h"
-#include "luop.h"
-
-struct osmo_gsup_server;
-
-/* override osmo_gsup_addr_send() to not actually send anything. */
-int osmo_gsup_addr_send(struct osmo_gsup_server *gs,
-			const uint8_t *addr, size_t addrlen,
-			struct msgb *msg)
-{
-	LOGP(DMAIN, LOGL_DEBUG, "%s\n", msgb_hexdump(msg));
-	msgb_free(msg);
-	return 0;
-}
-
-int db_subscr_get_by_imsi(struct db_context *dbc, const char *imsi,
-			  struct hlr_subscriber *subscr)
-{
-	return 0;
-}
-
-/* Verify that the internally allocated msgb is large enough */
-void test_gsup_tx_insert_subscr_data()
-{
-	struct lu_operation luop = {
-		.state = LU_S_LU_RECEIVED,
-		.subscr = {
-			.imsi = "123456789012345",
-			.msisdn = "987654321098765",
-			.nam_cs = true,
-			.nam_ps = true,
-		},
-		.is_ps = true,
-	};
-
-	lu_op_tx_insert_subscr_data(&luop);
-}
-
-const struct log_info_cat default_categories[] = {
-	[DMAIN] = {
-		.name = "DMAIN",
-		.description = "Main Program",
-		.enabled = 1, .loglevel = LOGL_DEBUG,
-	},
-};
-
-static struct log_info info = {
-	.cat = default_categories,
-	.num_cat = ARRAY_SIZE(default_categories),
-};
-
-int main(int argc, char **argv)
-{
-	void *ctx = talloc_named_const(NULL, 0, "gsup_test");
-	osmo_init_logging2(ctx, &info);
-	log_set_print_filename(osmo_stderr_target, 0);
-	log_set_print_timestamp(osmo_stderr_target, 0);
-	log_set_use_color(osmo_stderr_target, 0);
-	log_set_print_category(osmo_stderr_target, 1);
-
-	test_gsup_tx_insert_subscr_data();
-
-	printf("Done.\n");
-	return EXIT_SUCCESS;
-}
--- a/tests/gsup/gsup_test.err
+++ b/tests/gsup/gsup_test.err
@@ -1,2 +0,0 @@
-DMAIN 10 01 08 21 43 65 87 09 21 43 f5 08 09 08 89 67 45 23 01 89 67 f5 05 07 10 01 01 12 02 01 2a 28 01 01 
-DMAIN LU OP state change: LU RECEIVED -> ISD SENT
--- a/tests/gsup/gsup_test.ok
+++ b/tests/gsup/gsup_test.ok
@@ -1 +0,0 @@
-Done.
--- a/tests/gsup_server/Makefile.am
+++ b/tests/gsup_server/Makefile.am
@@ -1,11 +1,11 @@
 AM_CPPFLAGS = \
 	$(all_includes) \
-	-I$(top_srcdir)/src \
 	$(NULL)

 AM_CFLAGS = \
 	-Wall \
 	-ggdb3 \
+	-I$(top_srcdir)/include \
 	$(LIBOSMOCORE_CFLAGS) \
 	$(LIBOSMOGSM_CFLAGS) \
 	$(LIBOSMOABIS_CFLAGS) \
--- a/tests/gsup_server/gsup_server_test.c
+++ b/tests/gsup_server/gsup_server_test.c
@@ -20,7 +20,7 @@

 #include <stdio.h>
 #include <osmocom/core/utils.h>
-#include "gsup_server.h"
+#include <osmocom/hlr/gsup_server.h>

 #define comment_start() printf("\n===== %s\n", __func__)
 #define comment_end() printf("===== %s: SUCCESS\n\n", __func__)
--- a/tests/test_nodes.vty
+++ b/tests/test_nodes.vty
@@ -1,3 +1,15 @@
+OsmoHLR> ?
+  show        Show running system information
+  list        Print command list
+  exit        Exit current mode and down to previous mode
+  help        Description of the interactive help system
+  enable      Turn on privileged mode command
+  terminal    Set terminal line parameters
+  who         Display who is on vty
+  logging     Configure logging
+  no          Negate a command or set its defaults
+  logp        Print a message on all log outputs; useful for placing markers in test logs
+  subscriber  Subscriber management commands
 OsmoHLR> list
  show version
  show online-help
@@ -25,14 +37,24 @@ OsmoHLR> list
  show subscriber (imsi|msisdn|id|imei) IDENT

 OsmoHLR> enable
+OsmoHLR# ?
+  help        Description of the interactive help system
+  list        Print command list
+  write       Write running configuration to memory, network, or terminal
+  show        Show running system information
+  exit        Exit current mode and down to previous mode
+  disable     Turn off privileged mode command
+  configure   Configuration from vty interface
+  copy        Copy configuration
+  terminal    Set terminal line parameters
+  who         Display who is on vty
+  logging     Configure logging
+  no          Negate a command or set its defaults
+  logp        Print a message on all log outputs; useful for placing markers in test logs
+  subscriber  Subscriber management commands
 OsmoHLR# list
  help
-  list
-  write terminal
-  write file
-  write memory
-  write
-  show running-config
+...
  exit
  disable
  configure terminal
@@ -50,28 +72,35 @@ OsmoHLR# list
 ...

 OsmoHLR# configure terminal
+OsmoHLR(config)# ?
+...
+  banner    Set banner string
+  service   Set up miscellaneous service
+  line      Configure a terminal line
+  ctrl      Configure the Control Interface
+  log       Configure logging sub-system
+  stats     Configure stats sub-system
+  hlr       Configure the HLR
 OsmoHLR(config)# list
  help
-  list
-  write terminal
-  write file
-  write memory
-  write
-  show running-config
+...
  exit
  end
 ...
  hlr

 OsmoHLR(config)# hlr
+OsmoHLR(config-hlr)# ?
+...
+  euse                         Configure a particular External USSD Entity
+  no                           Negate a command or set its defaults
+  ussd                         USSD Configuration
+  ncss-guard-timeout           Set guard timer for NCSS (call independent SS) session activity
+  store-imei                   Save the IMEI in the database when receiving Check IMEI requests. Note that an MSC does not necessarily send Check IMEI requests (for OsmoMSC, you may want to set 'check-imei-rqd 1').
+  subscriber-create-on-demand  Make a new record when a subscriber is first seen.
 OsmoHLR(config-hlr)# list
  help
-  list
-  write terminal
-  write file
-  write memory
-  write
-  show running-config
+...
  exit
  end
  gsup
@@ -90,16 +119,11 @@ OsmoHLR(config-hlr)# list
  no subscriber-create-on-demand

 OsmoHLR(config-hlr)# gsup
+OsmoHLR(config-hlr-gsup)# ?
+...
+  bind   Listen/Bind related socket option
 OsmoHLR(config-hlr-gsup)# list
-  help
-  list
-  write terminal
-  write file
-  write memory
-  write
-  show running-config
-  exit
-  end
+...
  bind ip A.B.C.D

 OsmoHLR(config-hlr-gsup)# exit
--- a/tests/test_subscriber.vty
+++ b/tests/test_subscriber.vty
@@ -12,6 +12,7 @@ OsmoHLR# list
  subscriber (imsi|msisdn|id|imei) IDENT update aud3g none
  subscriber (imsi|msisdn|id|imei) IDENT update aud3g milenage k K (op|opc) OP_C [ind-bitlen] [<0-28>]
  subscriber (imsi|msisdn|id|imei) IDENT update imei (none|IMEI)
+  subscriber (imsi|msisdn|id|imei) IDENT update network-access-mode (none|cs|ps|cs+ps)

 OsmoHLR# subscriber?
  subscriber  Subscriber management commands
@@ -125,10 +126,11 @@ OsmoHLR# subscriber msisdn 423 show
    MSISDN: 423

 OsmoHLR# subscriber imsi 123456789023000 update ?
-  msisdn  Set MSISDN (phone number) of the subscriber
-  aud2g   Set 2G authentication data
-  aud3g   Set UMTS authentication data (3G, and 2G with UMTS AKA)
-  imei    Set IMEI of the subscriber (normally populated from MSC, no need to set this manually)
+  msisdn               Set MSISDN (phone number) of the subscriber
+  aud2g                Set 2G authentication data
+  aud3g                Set UMTS authentication data (3G, and 2G with UMTS AKA)
+  imei                 Set IMEI of the subscriber (normally populated from MSC, no need to set this manually)
+  network-access-mode  Set Network Access Mode (NAM) of the subscriber

 OsmoHLR# subscriber imsi 123456789023000 update msisdn ?
  none    Remove MSISDN (phone number)
@@ -437,3 +439,35 @@ OsmoHLR# show subscriber id 99
    IMSI: 000000000000099
    MSISDN: none
    IMEI: 12345 (INVALID LENGTH!)
+
+OsmoHLR# subscriber imsi 123456789023000 create
+% Created subscriber 123456789023000
+    ID: 101
+    IMSI: 123456789023000
+    MSISDN: none
+OsmoHLR# subscriber imsi 123456789023000 update network-access-mode none
+OsmoHLR# subscriber imsi 123456789023000 show
+    ID: 101
+    IMSI: 123456789023000
+    MSISDN: none
+    CS disabled
+    PS disabled
+OsmoHLR# subscriber imsi 123456789023000 update network-access-mode cs
+OsmoHLR# subscriber imsi 123456789023000 show
+    ID: 101
+    IMSI: 123456789023000
+    MSISDN: none
+    PS disabled
+OsmoHLR# subscriber imsi 123456789023000 update network-access-mode ps
+OsmoHLR# subscriber imsi 123456789023000 show
+    ID: 101
+    IMSI: 123456789023000
+    MSISDN: none
+    CS disabled
+OsmoHLR# subscriber imsi 123456789023000 update network-access-mode cs+ps
+OsmoHLR# subscriber imsi 123456789023000 show
+    ID: 101
+    IMSI: 123456789023000
+    MSISDN: none
+OsmoHLR# subscriber imsi 123456789023000 delete
+% Deleted subscriber for IMSI '123456789023000'
--- a/tests/testsuite.at
+++ b/tests/testsuite.at
@@ -15,13 +15,6 @@ cat $abs_srcdir/auc/auc_ts_55_205_test_sets.err > experr
 AT_CHECK([$abs_top_builddir/tests/auc/auc_ts_55_205_test_sets], [], [expout], [experr])
 AT_CLEANUP

-AT_SETUP([gsup])
-AT_KEYWORDS([gsup])
-cat $abs_srcdir/gsup/gsup_test.ok > expout
-cat $abs_srcdir/gsup/gsup_test.err > experr
-AT_CHECK([$abs_top_builddir/tests/gsup/gsup_test], [], [expout], [experr])
-AT_CLEANUP
-
 AT_SETUP([gsup_server])
 AT_KEYWORDS([gsup_server])
 cat $abs_srcdir/gsup_server/gsup_server_test.ok > expout
@@ -36,3 +29,13 @@ cat $abs_srcdir/db/db_test.err > experr
 sqlite3 db_test.db < $abs_top_srcdir/sql/hlr.sql
 AT_CHECK([$abs_top_builddir/tests/db/db_test], [], [expout], [experr])
 AT_CLEANUP
+
+# AT_SKIP_IF: disable for old sqlite versions, because the way we dump tables in the test doesn't work with it.
+# https://lists.osmocom.org/pipermail/openbsc/2019-November/013063.html
+AT_SETUP([db_upgrade])
+AT_SKIP_IF([ ! pkg-config sqlite3 --exists --atleast-version=3.16.2 ])
+AT_KEYWORDS([db_upgrade])
+cat $abs_srcdir/db_upgrade/db_upgrade_test.ok > expout
+cat $abs_srcdir/db_upgrade/db_upgrade_test.err > experr
+AT_CHECK([$abs_srcdir/db_upgrade/db_upgrade_test.sh $abs_srcdir/db_upgrade $abs_builddir/db_upgrade], [], [expout], [experr])
+AT_CLEANUP
Author	SHA1	Message	Date
Oliver Smith	276c5a7719	tests/auc: change back to python3 Change the interpreter of the python script back to python3, as it was when the script was initially added in Idff9d757ab956179aa41ada2a223fd9f439aafbd. In the meantime, it had been changed to python2 to make it work with build slaves that were missing python3, but this is not necessary anymore. This should be merged shortly after osmo-python-tests was migrated to python3, and the jenkins build slaves were (automatically) updated to have the new osmo-python-tests installed. Related: OS#2819 Depends: osmo-python-tests I3ffc3519bf6c22536a49dad7a966188ddad351a7 Change-Id: Ifbb8f8f5428657a1c2d4d6d1217f8e374be43aba	2019-12-10 14:37:25 +01:00
Harald Welte	80dc9ae4be	hlr: exit(2) on unsupported positional arguments on command line Change-Id: I0c2738d380a7e79622fb3db2062c17782555e82d	2019-12-03 22:17:00 +01:00
Oliver Smith	a377c41bd4	gsup_client.h: fix license header: GPLv2+ gsup_client.c is GPLv2 too, having AGPLv3 in the header file does not make sense. Change-Id: I3827a7c70d60137ffc654c1ca53c2652bb3df147	2019-12-03 14:51:38 +01:00
Harald Welte	06f5af22c8	AUC: Add support for setting the AMF separation bit to '1' for EUTRAN Despite LTE/EUTRAN using the same authentication procedure (UMTS AKA) as 3G, there's one difference: The "operator defined" 16bit AMF field is reduced to 15 bits, with the first bit now being used as 'separation bit'. That bit should be '0' for 2G/3G (as it is right now) and '1' for authentication vectores generated for authentication over EUTRAN/EPS. Depends: libosmocore I93850710ab55a605bf61b95063a69682a2899bb1 (OSMO_GSUP_RAT_TYPES_IE) Change-Id: Ic766bc40f6126bb479bd0a05b0e96bec3e240008	2019-12-01 16:09:16 +01:00
Neels Hofmeyr	07e1602d2d	db v4: add column last_lu_seen_ps Location Updating procedures from both CS and PS overwrite the same last_lu_seen field of a subscriber. For upcoming D-GSM it will be important to distinguish those, because only CS attaches qualify for MSISDN lookup. Add column last_lu_seen_ps, and upon PS LU, do not overwrite last_lu_seen, so that last_lu_seen now only reflects CS LU. In the VTY, dump both LU dates distinctively. Change-Id: Id7fc50567211a0870ac0524f6dee94d4513781ba	2019-11-27 03:19:06 +01:00
Neels Hofmeyr	abdfdb8a4a	test_nodes.vty: tweak: add some '?' checks I added these "by accident" when implementing D-GSM related VTY tests, now submitting them separately. Change-Id: I92a4245cae806270b00330403cc114017ab7af53	2019-11-25 13:58:51 +01:00
Neels Hofmeyr	7355d0ddfe	remove gsup_test The test doesn't do much: it encodes an Insert Subscriber Data request for the sole purpose to ensure the msgb is allocated large enough. A bug like that is easily avoided statically. Also, the lu functions will get refactored soon, it doesn't make sense to me to drag this test along. Change-Id: I42e1c72bf4cce8034f968cd4392773bf2b643c1b	2019-11-25 13:58:51 +01:00
Neels Hofmeyr	6b305b4c30	Makefile convenience: add VTY_TEST var to run only one test VTY transcript tests run all .vty test scripts, and it is not so trivial to figure out the test-db creation and cmdline to run only one of them when debugging. Add VTY_TEST var, useful to pick one test on the cmdline: cd tests make vty-test VTY_TEST=test_nodes.vty Not all VTY tests leave files behind that match hlr_vty_test.db-, so make sure that make does not fail it they can't be deleted (rm -f). Change-Id: I4ad7ddb31b2bfb668b3540cfef658417dd442375	2019-11-25 13:58:16 +01:00
Neels Hofmeyr	a7d0f87eb7	add osmo_gsup_msgb_alloc() Throughout osmo-hlr's code, the GSUP msgb allocation is duplicated as: msgb_alloc_headroom(1024+16, 16, "foo"); Instead, use one common function to keep the magic numbers in one place. Change-Id: I40e99b5bc4fd8f750da7643c03b2119ac3bfd95e	2019-11-25 13:13:19 +01:00
Neels Hofmeyr	981e126686	db upgrade: remove some code dup Instead of a switch() for each version number with identical switch cases except for the function name, use an array of function pointers and loop. Also print a success message after each individual version upgrade, instead of only one in the end (see change in db_upgrade_test.ok). Change-Id: I1736af3d9a3f02e29db836966ac15ce49f94737b	2019-11-25 13:13:18 +01:00
Neels Hofmeyr	7143f3a0cb	fix upgrade test in presence of ~/.sqliterc db_upgrade_test.sh: - If an ~/.sqliterc file exists, it causes output of '-- Loading resources from ~/.sqliterc'. Use -batch option to omit that. - To make sure that column headers are off when required, add -noheaders in some places. Change-Id: I279a39984563594a4a3914b2ce3d803ad9468fe8	2019-11-25 13:13:18 +01:00
Oliver Smith	f0968798a2	gitignore: add tests/db_upgrade/*.dump Change-Id: I0dca7a94883bbe69151d919ae204edfff12288ab	2019-11-25 13:13:05 +01:00
Neels Hofmeyr	2f75803e5d	move headers to include/osmocom/hlr Apply the same headers structure that we keep in most Osmocom source trees: Keep noinst_HEADERS in include/osmocom/hlr and include them using #include <osmocom/hlr/*.h> The only header kept in src/ is db_bootstrap.h, because it is generated during build time. If it was built in include/osmocom/hlr, we would need db.o to depend on db_bootstrap.h in a different subdir, which automake can't do well. Change-Id: Ic912fe27f545b85443c5fb713d8c3c8aac23c9ad	2019-11-20 01:25:39 +01:00
Neels Hofmeyr	7f4dd11682	db.c: code dup: add db_run_statements() for arrays of statements The db bootstrap as well as the upgrade code all execute a number of statements, and have massive code dup around each statement execution. Instead have one db_run_statements() that takes an array of strings and runs all. Change-Id: I2721dfc0a9aadcc7f5ac81a1c0fa87452098996f	2019-11-12 16:23:48 +00:00
Neels Hofmeyr	a8045daeef	hlr db schema 3: hlr_number -> msc_number The osmo-hlr DB schema indicates a hlr_number column and references it as 3GPP TS 23.008 chapter 2.4.6. However, chapter 2.4.6 refers to the "MSC number", while the "HLR number" is chapter 2.4.7. Taking a closer look, 2.4.6 says "The MSC number is [...] stored in the HLR", while 2.4.7 says "The HLR number may be stored in the VLR". As quite obvious, the HLR does not store the HLR number. This was a typo from the start. The osmo-hlr code base so far does not use the hlr_number column at all, so we get away with renaming the column without any effects on the code base. However, let's rather make this a new schema version to be safe. Change-Id: I527e8627b24b79f3e9eec32675c7f5a3a6d25440	2019-11-12 16:23:48 +00:00
Oliver Smith	4359b885d4	tests/db_upgrade: disable for old sqlite versions Skip the test if the installed sqlite version is older than 3.16.2 (current version of debian 9). This prevents test failures caused by the way we dump tables in the test, which does not work with older versions. This patch is a middle ground between reverting the db upgrade patch, and spending lots of time to replace the table dumping code with something that works with old sqlite versions to fix current build failures in OBS. Usually version checking is done in configure.ac, however I could not find a good way to pass the result to testsuite.at. So I decided to use pkg-config to do the test there. Fixes: `5b65461d68` ("add db_upgrade test") Related: https://lists.osmocom.org/pipermail/openbsc/2019-November/013063.html Change-Id: I348c133003a95badbd6807d1519aa669115872fb	2019-11-12 13:39:00 +01:00
Neels Hofmeyr	5b65461d68	add db_upgrade test We have a database schema upgrade path, but so far nothing that verifies that we don't break it. It almost seems like the user data weren't important to us!? Add a db upgrade test: - Create a db with an .sql dump taken from a db created with an old osmo-hlr, producing DB schema version 0. - Run osmo-hlr --db-upgrade --db-check - Verify that the upgrade exited successfully. - Verify that restarting with the upgraded DB works. If python tests are enabled, also: - create a new database using the new osmo-hlr (just built). - replay a VTY transcript to create subscribers as in the old snapshot. - replay some sql modifications as done in the old snapshot. - Get a list of sorted table names, - a list of their sorted columns with all their properties, - and dump the table contents in a column- and value-sorted way. - Compare the resulting dumps and error if there are any diffs. (This is how I found the difference in the imei column that was fixed in I68a00014a3d603fcba8781470bc5285f78b538d0) Change-Id: I0961bab0e17cfde5b030576c5bc243c2b51d9dc4	2019-10-31 21:32:58 +01:00
Neels Hofmeyr	f8ad67e7fc	hlr.sql: move comment Move a comment for ind_bitlen column to a separate line, so that it doesn't show in PRAGMA_TABLE_INFO('subscriber'). An upcoming patch introduces db_upgrade_test, which dumps a sorted db schema. In newer sqlite3 versions, a comment following a 'DEFAULT' keyword actually shows up in the PRAGMA_TABLE_INFO() results (on my machine), but older versions (on the build slaves) drop that comment. The ind_bitlen column is the only one producing this odd side effect, because it is the last column and has no comma between 'DEFAULT' and the comment. The easiest way to get matching results across sqlite3 client versions is to move the comment to above ind_bitlen instead of having it on the same line. (Adding a comma doesn't work.) Change-Id: Id66ad68dd3f22d533fc3a428223ea6ad0282bde0	2019-10-31 19:50:28 +01:00
Neels Hofmeyr	c3d40326ec	add --db-check option This allows starting osmo-hlr to merely open the database, do upgrades if necessary, and quit, without opening any ports. So that no ports are opened, move the telnet VTY startup to below the database check. Needed for upcoming patch that introduces a db_upgrade test, in I0961bab0e17cfde5b030576c5bc243c2b51d9dc4. Change-Id: I1a4b3360690acd2cd3cffdadffbb00a28d421316	2019-10-31 17:48:01 +01:00
Neels Hofmeyr	a9f8a4bdce	fix upgrade to version 2: imei column default value A subsequent commit will add a db_upgrade test, which verifies that the db resulting from an upgrade is identical to one created from scratch in the new version. That test currently would show a diff: an upgraded 'imei' column has 'default NULL', where a new db created in version 2 has no default value on the imei column. Fix the upgrade path to add an imei column without 'default NULL', so that adding the upgrade test will result in success. The test is added in I0961bab0e17cfde5b030576c5bc243c2b51d9dc4 Change-Id: I68a00014a3d603fcba8781470bc5285f78b538d0	2019-10-31 04:48:39 +01:00
Neels Hofmeyr	f5459de2e2	db upgrade to v2: log version 2, not 1 Change-Id: I9237b64e5748e693a5f039c5a5554d417eed3633	2019-10-31 04:48:39 +01:00
Neels Hofmeyr	7d2843df4c	fix double free in osmo_gsup_client_enc_send() Change-Id: Iee675fd498ab0867ac2411d9142358f513276182	2019-10-30 03:57:20 +01:00
Oliver Smith	2b0bf31183	tests/test_nodes.vty: check less libosmocore cmds Use three dots to avoid checking for exact commands between help and exit, which originate from libosmocore. This avoids test failues when we slightly change the commands, like the change from "write file" to "write file [PATH]" in [1] that is currently causing the vty tests to fail. [1] libosmocore I38edcf902a08b6bd0ebb9aa6fc1a7041421af525 Change-Id: I4a964b86195141e5a50705425206f3602f908999	2019-10-08 11:11:15 +02:00
Pau Espin Pedrol	28f0774e34	tests: Fix db_test err file to expect error code name instead of value Previous commit changed db_test to output code names to fix issues on some platforms (I guess due to different error code values), but somehow this log line was not updated. Fixes: `8b860e54be` Change-Id: I598de6f83a86d528174d3d188596314572f5d70d	2019-08-13 11:59:09 +02:00
Thorsten Alteholz	b07f33df41	fix spelling errors detected by lintian Change-Id: I2a1a21aceabc20fadc5dd28985a94689d2b873a2	2019-08-12 08:36:22 +00:00
Ruben Undheim	8b860e54be	Fix test for return codes on mipsel and alpha archs Change-Id: Ia64f1d9f39fe2b1fb704f7b6c4d9cce93ab708cd	2019-08-12 08:33:33 +00:00
Pau Espin Pedrol	9cf0030b6a	Bump version: 1.0.0.49-e493-dirty → 1.1.0 Change-Id: If53bdb2e216cb98df4d5a482cbbf24096db536ed	2019-08-07 16:14:24 +02:00
Pau Espin Pedrol	b9b224c7bd	configure.ac: Require libosmocore 1.2.0 Current code uses GSM23003_MSISDN_MAX_DIGITS, which is only available in libosmocore 1.2.0 onwards. Let's update configure.ac accordingly. Fixes: `2e403d6c3f` Change-Id: Iad03a8cf7a36bdc824ec2acc8fb8f9c27a1c0421	2019-08-07 16:08:59 +02:00
Pau Espin Pedrol	e49391bfc4	Remove undefined param passed to logging_vty_add_cmds Since March 15th 2017, libosmocore API logging_vty_add_cmds() had its parameter removed (c65c5b4ea075ef6cef11fff9442ae0b15c1d6af7). However, definition in C file doesn't contain "(void)", which means number of parameters is undefined and thus compiler doesn't complain. Let's remove parameters from all callers before enforcing "(void)" on it. Related: OS#4138 Change-Id: I6d0dbbd83ce17ee798bfb6e30378ed1dbae19134	2019-08-05 15:57:13 +02:00
Vadim Yanitskiy	fbd736ef37	src/db.c: integrate SQLite3 with talloc allocator This change introduces an optional feature that allows to make SQLite3 use talloc for all internal allocations. This would facilitate finding memleaks. OsmoHLR needs to be configured with '--enable-sqlite-talloc'. full talloc report on 'OsmoHLR' (total 292168 bytes in 449 blocks) struct osmo_gsup_server contains 162 bytes in 3 blocks (ref 0) ... struct db_context contains 288407 bytes in 420 blocks (ref 0) hlr.db contains 7 bytes in 1 blocks (ref 0) SQLite3 contains 288192 bytes in 418 blocks (ref 0) db.c:95 contains 48 bytes in 1 blocks (ref 0) db.c:95 contains 2 bytes in 1 blocks (ref 0) ... Unfortunately, old SQLite3 versions (such as 3.8.2) run out of memory when trying to initialize a new database: DDB ERROR db.c:88 (7) statement aborts at 3: [] DDB ERROR db.c:420 Unable to set Write-Ahead Logging: out of memory DDB ERROR db.c:88 (7) statement aborts at 3: [] DDB ERROR db.c:238 Unable to prepare SQL statement 'SELECT name FROM sqlite_master WHERE type='table' AND name=?' ... I've noticed a huge difference in heap usage footprint compared to generic malloc. At the same time, the recent versions (at least 3.24.0), work just fine. Change-Id: Icfe67ed0f063b63e6794f9516da3003d01cf20a7	2019-07-30 17:15:17 +00:00
Vadim Yanitskiy	dc30154fdf	tests/db_test: close the database when test is finished Change-Id: I96fedf9181e89e4d68815b04f494a9c2ecc0e057	2019-07-30 17:15:17 +00:00
Vadim Yanitskiy	37642177f9	build: fix mess with 'db_test_SOURCES' and 'db_test_LDADD' Somehow both 'db_test_SOURCES' and 'db_test_LDADD' ended up in 'src/Makefile.am'. This causes automake / autoconf to complain. Let's get rid of both useless declarations. Furthermore, the actual 'db_test_LDADD' in 'tests/Makefile.am' contained references to the source files from '$(top_srcdir)'. Most likely, the original intention was to depend on the object files in '$(top_builddir)'. Let's also fix this. Change-Id: Ib2e436ed91d9b7551dc5b205329d468c2b0ced04	2019-07-30 17:15:17 +00:00
Oliver Smith	6401b90574	db_auc.c: verify hex key sizes read from DB Replace commented out size check for Ki with a real check, and use it consistently for Ki, K, OP and OPC. Add a test that sets all keys to the wrong size and tries to read them. Related: OS#2565 Change-Id: Ib8e8e9394fb65c6e7932ce9f8bebc321b99f7696	2019-07-25 14:52:20 +02:00
Oliver Smith	5b5cac7e94	gitignore: ignore everything generated in db_test Change-Id: I3545be056cc7e4f72f6f86e772f9cc70a8e5c03c	2019-07-25 12:45:55 +00:00
Vadim Yanitskiy	937f583a7e	hlr_ussd.c: rx_proc_ss_req(): fix NULL pointer dereference The SS payload is mandatory for GSUP PROC_SS_{REQ,RSP} messages with session state BEGIN or CONTINUE, and optional for the END. Make sure that it's present for both BEGIN and CONTINUE, consider received message as incorrect otherwise. In case of the END, call handle_ussd() / handle_ss() only if SS payload is present. Change-Id: Ia71cabbf396bd1388e764a1749e953ac1782e307 Fixes: CID#188841	2019-07-24 19:14:44 +07:00
Vadim Yanitskiy	4ca7f6a17e	hlr_ussd.c: fix: properly pass invokeID in handle_ussd_own_msisdn() Change-Id: I06845c2c9ebee61671477ee1c9d82010f1f37b7b	2019-07-23 20:01:35 +07:00
Oliver Smith	b64cb27003	manuals: improve subscribers create on demand Write all VTY commands in monospace and add configuration example blocks. Add an example VTY session for enabling CS and PS NAM. Realted: OS#2542 Change-Id: I54f80810db3dac7a4a56ad42c5d2154b6006108c	2019-07-15 14:13:50 +02:00
Oliver Smith	3b33b01fb0	VTY: add subscriber update network-access-mode Allow updating the NAM (Network Access Mode) of subscribers with the VTY. This is important for the subscriber create on demand use case where subscribers get created without access to PS and CS NAM by default. Regenerate hlr_vty_reference.xml. Related: OS#2542 Change-Id: I231e03219355ebe6467d62ae2e40bef9d8303e3b	2019-07-15 14:13:46 +02:00
Oliver Smith	78abea6a0e	contrib/jenkins.sh: run "make maintainer-clean" Related: OS#3047 Change-Id: I63808c5c2724b8f4c3cf40db682f74eec54f1e76	2019-07-10 13:29:54 +02:00