Compare commits

...

4 Commits

Author SHA1 Message Date
Neels Hofmeyr
9d99d99137 add osmo-hlr-db-tool
Change-Id: I0dfa6ec033dd93161c1adc2ce1637195fe5b7a63
2017-10-25 04:14:32 +02:00
Neels Hofmeyr
8f063796e9 cosmetic: sql/hlr.sql: move comments
By moving the comments inside the table row definitions, they are dumped back
during 'sqlite3 hlr.db .dump'. When they are between SQL statements, the
comments are lost.

Tweak wording.

Change-Id: I280c2e2d3e9b7f1dc632722724d9e1c54d041820
2017-10-25 01:15:46 +02:00
Neels Hofmeyr
10df27f168 automatically create db tables on osmo-hlr invocation
If a database file is missing, osmo-hlr creates it, as is the default sqlite3
API behavior. Actually also create initial tables in it.

Note: it might be desirable to bail if the desired database file does not
exist. That is however a different semantic from this patch; this is not
changing the fact that a db file is created, this just creates a usable one.

Note: I am about to add osmo-hlr-db-tool to do database migration from
osmo-nitb, and possibly also CSV import. For that, it is also desirable to
bootstrap a usable database, which is the core reason for this patch.

Create db_bootstrap.h as a BUILT_SOURCE from reading in sql/hlr.sql and
mangling via sed to a list of SQL statement strings. On each db_open(), run
this bootstrap sequence.

In sql/hlr.sql:
* Add 'IF NOT EXISTS' to 'CREATE TABLE', so that the bootstrap
  sequence can be run on an already bootstrapped db. This is the same as
  osmo-nitb does it.
* Drop the final comment at the bottom, which ended up being an empty SQL
  statement and causing sqlite3 API errors, seemed to have no purpose anyway.

Note: by composing the statement strings as multiline and including the SQL
comments, sqlite3 actually retains the comments contained in table definitions
and prints them back during 'sqlite3 hlr.db .dump'.

Change-Id: If77dbbfe1af3e66aaec91cb6295b687f37678636
2017-10-25 01:13:45 +02:00
Neels Hofmeyr
1d13fe1673 db_test: fix *FLAGS
Change-Id: Ie57a04b7efc7a1e16cf0e3625d8ad2f0ef0089b0
2017-10-25 01:13:40 +02:00
8 changed files with 527 additions and 21 deletions

View File

@@ -1,6 +1,5 @@
--modelled roughly after TS 23.008 version 13.3.0
CREATE TABLE subscriber (
CREATE TABLE IF NOT EXISTS subscriber (
-- OsmoHLR's DB scheme is modelled roughly after TS 23.008 version 13.3.0
id INTEGER PRIMARY KEY,
-- Chapter 2.1.1.1
imsi VARCHAR(15) UNIQUE NOT NULL,
@@ -40,24 +39,24 @@ CREATE TABLE subscriber (
ms_purged_ps BOOLEAN NOT NULL DEFAULT 0
);
CREATE TABLE subscriber_apn (
CREATE TABLE IF NOT EXISTS subscriber_apn (
subscriber_id INTEGER, -- subscriber.id
apn VARCHAR(256) NOT NULL
);
CREATE TABLE IF NOT EXISTS subscriber_multi_msisdn (
-- Chapter 2.1.3
CREATE TABLE subscriber_multi_msisdn (
subscriber_id INTEGER, -- subscriber.id
msisdn VARCHAR(15) NOT NULL
);
CREATE TABLE auc_2g (
CREATE TABLE IF NOT EXISTS auc_2g (
subscriber_id INTEGER PRIMARY KEY, -- subscriber.id
algo_id_2g INTEGER NOT NULL, -- enum osmo_auth_algo value
ki VARCHAR(32) NOT NULL -- hex string: subscriber's secret key (128bit)
);
CREATE TABLE auc_3g (
CREATE TABLE IF NOT EXISTS auc_3g (
subscriber_id INTEGER PRIMARY KEY, -- subscriber.id
algo_id_3g INTEGER NOT NULL, -- enum osmo_auth_algo value
k VARCHAR(32) NOT NULL, -- hex string: subscriber's secret key (128bit)
@@ -68,4 +67,3 @@ CREATE TABLE auc_3g (
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_subscr_imsi ON subscriber (imsi);
-- SELECT algo_id_2g, ki, algo_id_3g, k, op, opc, sqn FROM subscriber LEFT JOIN auc_2g ON auc_2g.subscriber_id = subscriber.id LEFT JOIN auc_3g ON auc_3g.subscriber_id = subscriber.id WHERE imsi = ?

View File

@@ -10,8 +10,14 @@ AM_CFLAGS = \
EXTRA_DIST = \
populate_hlr_db.pl \
db_bootstrap.sed \
$(NULL)
BUILT_SOURCES = \
db_bootstrap.h \
$(NULL)
CLEANFILES = $(BUILT_SOURCES)
noinst_HEADERS = \
auc.h \
db.h \
@@ -24,10 +30,12 @@ noinst_HEADERS = \
ctrl.h \
hlr_vty.h \
hlr_vty_subscr.h \
db_bootstrap.h \
$(NULL)
bin_PROGRAMS = \
osmo-hlr \
osmo-hlr-db-tool \
$(NULL)
noinst_PROGRAMS = \
@@ -59,6 +67,20 @@ osmo_hlr_LDADD = \
$(SQLITE3_LIBS) \
$(NULL)
osmo_hlr_db_tool_SOURCES = \
hlr_db_tool.c \
db.c \
db_hlr.c \
logging.c \
rand_urandom.c \
$(NULL)
osmo_hlr_db_tool_LDADD = \
$(LIBOSMOCORE_LIBS) \
$(LIBOSMOGSM_LIBS) \
$(SQLITE3_LIBS) \
$(NULL)
db_test_SOURCES = \
auc.c \
db.c \
@@ -73,3 +95,14 @@ db_test_LDADD = \
$(LIBOSMOGSM_LIBS) \
$(SQLITE3_LIBS) \
$(NULL)
BOOTSTRAP_SQL = $(top_srcdir)/sql/hlr.sql
db_bootstrap.h: $(BOOTSTRAP_SQL) $(srcdir)/db_bootstrap.sed
echo "/* DO NOT EDIT THIS FILE. It is generated from osmo-hlr.git/sql/hlr.sql */" > "$@"
echo "#pragma once" >> "$@"
echo "static const char *stmt_bootstrap_sql[] = {" >> "$@"
cat "$(BOOTSTRAP_SQL)" \
| sed -f "$(srcdir)/db_bootstrap.sed" \
>> "$@"
echo "};" >> "$@"

View File

@@ -25,6 +25,7 @@
#include "logging.h"
#include "db.h"
#include "db_bootstrap.h"
#define SEL_COLUMNS \
"id," \
@@ -179,6 +180,35 @@ void db_close(struct db_context *dbc)
talloc_free(dbc);
}
static int db_bootstrap(struct db_context *dbc)
{
int i;
for (i = 0; i < ARRAY_SIZE(stmt_bootstrap_sql); i++) {
int rc;
sqlite3_stmt *stmt;
rc = sqlite3_prepare_v2(dbc->db, stmt_bootstrap_sql[i], -1,
&stmt, NULL);
if (rc != SQLITE_OK) {
LOGP(DDB, LOGL_ERROR, "Unable to prepare SQL statement '%s'\n",
stmt_bootstrap_sql[i]);
return -1;
}
/* execute the statement */
rc = sqlite3_step(stmt);
db_remove_reset(stmt);
if (rc != SQLITE_DONE) {
LOGP(DDB, LOGL_ERROR, "Cannot bootstrap database: SQL error: (%d) %s,"
" during stmt '%s'",
rc, sqlite3_errmsg(dbc->db),
stmt_bootstrap_sql[i]);
return -1;
}
}
return 0;
}
struct db_context *db_open(void *ctx, const char *fname)
{
struct db_context *dbc = talloc_zero(ctx, struct db_context);
@@ -231,6 +261,8 @@ struct db_context *db_open(void *ctx, const char *fname)
LOGP(DDB, LOGL_ERROR, "Unable to set Write-Ahead Logging: %s\n",
err_msg);
db_bootstrap(dbc);
/* prepare all SQL statements */
for (i = 0; i < ARRAY_SIZE(dbc->stmt); i++) {
rc = sqlite3_prepare_v2(dbc->db, stmt_sql[i], -1,

25
src/db_bootstrap.sed Normal file
View File

@@ -0,0 +1,25 @@
# Input to this is sql/hlr.sql.
#
# We want each SQL statement line wrapped in "...\n", and each end (";") to
# become a comma:
#
# SOME SQL COMMAND (
# that may span )
# MULTIPLE LINES;
# MORE;
#
# -->
#
# "SOME SQL COMMAND (\n"
# " that may span )\n"
# "MULTIPLE LINES\n",
# "MORE\n",
#
# just replacing ';' with '\n,' won't work, since sed is bad in printing
# multiple lines. Also, how to input newlines to sed is not portable across
# platforms.
# Match excluding a trailing ';' as \1, keep any trailing ';'
s/^\(.*[^;]\)\(;\|\)$/"\1\\n"\2/
# Replace trailing ';' as ','
s/;$/,/

View File

@@ -38,9 +38,7 @@
#define SL3_TXT(x, stmt, idx) \
do { \
const char *_txt = (const char *) sqlite3_column_text(stmt, idx);\
if (_txt) \
strncpy(x, _txt, sizeof(x)); \
x[sizeof(x)-1] = '\0'; \
osmo_strlcpy(x, _txt, sizeof(x)); \
} while (0)
int db_subscr_create(struct db_context *dbc, const char *imsi)

425
src/hlr_db_tool.c Normal file
View File

@@ -0,0 +1,425 @@
/* (C) 2017 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
*
* All Rights Reserved
*
* Author: Neels Hofmeyr <nhofmeyr@sysmocom.de>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
#include <stdlib.h>
#include <signal.h>
#include <stdio.h>
#include <getopt.h>
#include <inttypes.h>
#include <string.h>
#include <osmocom/core/logging.h>
#include <osmocom/core/application.h>
#include "logging.h"
#include "db.h"
#include "rand.h"
struct hlr_db_tool_ctx {
/* DB context */
struct db_context *dbc;
};
struct hlr_db_tool_ctx *g_hlr_db_tool_ctx;
static struct {
const char *db_file;
bool bootstrap;
const char *import_nitb_db;
} cmdline_opts = {
.db_file = "hlr.db",
};
static void print_help()
{
printf("Usage: osmo-hlr-db-tool [-l <hlr.db>] --import-nitb-db <nitb.db>\n");
printf(" -l --database db-name The OsmoHLR database to use, default '%s'.\n",
cmdline_opts.db_file);
printf(" -n --import-nitb-db db Add OsmoNITB db's subscribers to OsmoHLR db.\n");
printf(" Be aware that the import is lossy, only the\n");
printf(" IMSI, MSISDN, nam_cs/ps and 2G auth data are set.\n");
printf(" -h --help This text.\n");
printf(" -d option --debug=DRLL:DCC:DMM:DRR:DRSL:DNM Enable debugging.\n");
printf(" -s --disable-color Do not print ANSI colors in the log\n");
printf(" -T --timestamp Prefix every log line with a timestamp.\n");
printf(" -e --log-level number Set a global loglevel.\n");
printf(" -V --version Print the version of OsmoHLR-db-tool.\n");
}
static void print_version(int print_copyright)
{
printf("OsmoHLR-db-tool version %s\n", PACKAGE_VERSION);
if (print_copyright)
printf("\n"
"Copyright (C) 2017 by sysmocom - s.f.m.c. GmbH\n"
"License AGPLv3+: GNU AGPL version 3 or later <http://gnu.org/licenses/agpl-3.0.html>\n"
"This is free software: you are free to change and redistribute it.\n"
"There is NO WARRANTY, to the extent permitted by law.\n"
"\n");
}
static void handle_options(int argc, char **argv)
{
while (1) {
int option_index = 0, c;
static struct option long_options[] = {
{"help", 0, 0, 'h'},
{"database", 1, 0, 'l'},
{"import-nitb-db", 1, 0, 'n'},
{"debug", 1, 0, 'd'},
{"disable-color", 0, 0, 's'},
{"timestamp", 0, 0, 'T'},
{"log-level", 1, 0, 'e'},
{"version", 0, 0, 'V' },
{0, 0, 0, 0}
};
c = getopt_long(argc, argv, "hl:n:d:sTe:V",
long_options, &option_index);
if (c == -1)
break;
switch (c) {
case 'h':
print_help();
exit(0);
case 'l':
cmdline_opts.db_file = optarg;
break;
case 'n':
cmdline_opts.import_nitb_db = optarg;
break;
case 'd':
log_parse_category_mask(osmo_stderr_target, optarg);
break;
case 's':
log_set_use_color(osmo_stderr_target, 0);
break;
case 'T':
log_set_print_timestamp(osmo_stderr_target, 1);
break;
case 'e':
log_set_log_level(osmo_stderr_target, atoi(optarg));
break;
case 'V':
print_version(1);
exit(0);
break;
default:
/* catch unknown options *as well as* missing arguments. */
fprintf(stderr, "Error in command line options. Exiting.\n");
exit(-1);
break;
}
}
}
static void signal_hdlr(int signal)
{
switch (signal) {
case SIGINT:
LOGP(DMAIN, LOGL_NOTICE, "Terminating due to SIGINT\n");
db_close(g_hlr_db_tool_ctx->dbc);
log_fini();
talloc_report_full(g_hlr_db_tool_ctx, stderr);
exit(0);
break;
case SIGUSR1:
LOGP(DMAIN, LOGL_DEBUG, "Talloc Report due to SIGUSR1\n");
talloc_report_full(g_hlr_db_tool_ctx, stderr);
break;
}
}
sqlite3 *open_nitb_db(const char *filename)
{
int rc;
sqlite3 *nitb_db = NULL;
rc = sqlite3_open(filename, &nitb_db);
if (rc != SQLITE_OK) {
LOGP(DDB, LOGL_ERROR, "Unable to open OsmoNITB DB %s; rc = %d\n", filename, rc);
return NULL;
}
return nitb_db;
}
enum nitb_stmt {
NITB_SELECT_SUBSCR,
NITB_SELECT_AUTH_KEYS,
NITB_SELECT_IMEI,
};
static const char *nitb_stmt_sql[] = {
[NITB_SELECT_SUBSCR] =
"SELECT imsi, id, extension, authorized"
" FROM Subscriber"
" ORDER BY id",
[NITB_SELECT_AUTH_KEYS] =
"SELECT algorithm_id, a3a8_ki from authkeys"
" WHERE subscriber_id = $subscr_id",
[NITB_SELECT_IMEI] =
"SELECT imei"
" FROM Equipment"
" WHERE id = $subscr_id",
};
sqlite3_stmt *nitb_stmt[ARRAY_SIZE(nitb_stmt_sql)] = {};
size_t _dbd_decode_binary(const unsigned char *in, unsigned char *out){
int i, e;
unsigned char c;
e = *(in++);
i = 0;
while( (c = *(in++))!=0 ){
if( c==1 ){
c = *(in++) - 1;
}
out[i++] = c + e;
}
return (size_t)i;
}
#define SL3_TXT(x, stmt, idx) \
do { \
const char *_txt = (const char *) sqlite3_column_text(stmt, idx); \
osmo_strlcpy(x, _txt, sizeof(x)); \
} while (0)
void import_nitb_subscr_aud(sqlite3 *nitb_db, const char *imsi, int64_t nitb_id, int64_t hlr_id)
{
int rc;
struct db_context *dbc = g_hlr_db_tool_ctx->dbc;
sqlite3_stmt *stmt;
int count = 0;
stmt = nitb_stmt[NITB_SELECT_AUTH_KEYS];
if (!db_bind_int(stmt, NULL, nitb_id))
return;
while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) {
const void *blob;
unsigned int blob_size;
static unsigned char buf[4096];
static char ki[128];
int decoded_size;
struct sub_auth_data_str aud2g = {
.type = OSMO_AUTH_TYPE_GSM,
.algo = OSMO_AUTH_ALG_NONE,
.u.gsm.ki = ki,
};
aud2g.algo = sqlite3_column_int(stmt, 0);
if (count) {
LOGP(DDB, LOGL_ERROR,
"Warning: subscriber has more than one auth key,"
" importing only the first key, for IMSI=%s\n",
imsi);
break;
}
blob = sqlite3_column_blob(stmt, 1);
blob_size = sqlite3_column_bytes(stmt, 1);
if (blob_size > sizeof(buf)) {
LOGP(DDB, LOGL_ERROR,
"OsmoNITB import to %s: Cannot import auth data for IMSI %s:"
" too large blob: %u\n",
dbc->fname, imsi, blob_size);
db_remove_reset(stmt);
continue;
}
decoded_size = _dbd_decode_binary(blob, buf);
osmo_strlcpy(ki, osmo_hexdump_nospc(buf, decoded_size), sizeof(ki));
db_subscr_update_aud_by_id(dbc, hlr_id, &aud2g);
count ++;
}
if (rc != SQLITE_DONE && rc != SQLITE_ROW) {
LOGP(DDB, LOGL_ERROR, "OsmoNITB DB: SQL error: (%d) %s,"
" during stmt '%s'",
rc, sqlite3_errmsg(nitb_db),
nitb_stmt_sql[NITB_SELECT_AUTH_KEYS]);
}
db_remove_reset(stmt);
}
void import_nitb_subscr(sqlite3 *nitb_db, sqlite3_stmt *stmt)
{
struct db_context *dbc = g_hlr_db_tool_ctx->dbc;
int rc;
struct hlr_subscriber subscr;
int64_t nitb_id;
int64_t imsi;
char imsi_str[32];
bool authorized;
imsi = sqlite3_column_int64(stmt, 0);
snprintf(imsi_str, sizeof(imsi_str), "%"PRId64, imsi);
rc = db_subscr_create(dbc, imsi_str);
if (rc) {
LOGP(DDB, LOGL_ERROR, "OsmoNITB DB import to %s: failed to create IMSI %s: %d: %s\n",
dbc->fname,
imsi_str,
rc,
strerror(rc));
/* on error, still attempt to continue */
}
nitb_id = sqlite3_column_int64(stmt, 1);
SL3_TXT(subscr.msisdn, stmt, 2);
authorized = sqlite3_column_int(stmt, 3) ? true : false;
db_subscr_update_msisdn_by_imsi(dbc, imsi_str, subscr.msisdn);
db_subscr_nam(dbc, imsi_str, authorized, true);
db_subscr_nam(dbc, imsi_str, authorized, false);
/* find the just created id */
rc = db_subscr_get_by_imsi(dbc, imsi_str, &subscr);
if (rc) {
LOGP(DDB, LOGL_ERROR, "OsmoNITB DB import to %s: created IMSI %s,"
" but failed to get new subscriber id: %d: %s\n",
dbc->fname,
imsi_str,
rc,
strerror(rc));
return;
}
OSMO_ASSERT(!strcmp(imsi_str, subscr.imsi));
import_nitb_subscr_aud(nitb_db, imsi_str, nitb_id, subscr.id);
}
int import_nitb_db(void)
{
int i;
int ret;
int rc;
const char *sql;
sqlite3_stmt *stmt;
sqlite3 *nitb_db = open_nitb_db(cmdline_opts.import_nitb_db);
if (!nitb_db)
return -1;
ret = 0;
for (i = 0; i < ARRAY_SIZE(nitb_stmt_sql); i++) {
sql = nitb_stmt_sql[i];
rc = sqlite3_prepare_v2(nitb_db, sql, -1, &nitb_stmt[i], NULL);
if (rc != SQLITE_OK) {
LOGP(DDB, LOGL_ERROR, "OsmoNITB DB: Unable to prepare SQL statement '%s'\n", sql);
ret = -1;
goto out_free;
}
}
stmt = nitb_stmt[NITB_SELECT_SUBSCR];
while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) {
import_nitb_subscr(nitb_db, stmt);
/* On failure, carry on with the rest. */
}
if (rc != SQLITE_DONE) {
LOGP(DDB, LOGL_ERROR, "OsmoNITB DB: SQL error: (%d) %s,"
" during stmt '%s'",
rc, sqlite3_errmsg(nitb_db),
nitb_stmt_sql[NITB_SELECT_SUBSCR]);
goto out_free;
}
db_remove_reset(stmt);
sqlite3_finalize(stmt);
out_free:
sqlite3_close(nitb_db);
return ret;
}
int main(int argc, char **argv)
{
int rc;
int (*main_action)(void);
main_action = NULL;
g_hlr_db_tool_ctx = talloc_zero(NULL, struct hlr_db_tool_ctx);
OSMO_ASSERT(g_hlr_db_tool_ctx);
talloc_set_name_const(g_hlr_db_tool_ctx, "OsmoHLR-db-tool");
rc = osmo_init_logging(&hlr_log_info);
if (rc < 0) {
fprintf(stderr, "Error initializing logging\n");
exit(1);
}
handle_options(argc, argv);
if (cmdline_opts.import_nitb_db) {
if (main_action)
goto too_many_actions;
main_action = import_nitb_db;
}
/* Just in case any db actions need randomness */
rc = rand_init();
if (rc < 0) {
LOGP(DMAIN, LOGL_FATAL, "Error initializing random source\n");
exit(1);
}
g_hlr_db_tool_ctx->dbc = db_open(g_hlr_db_tool_ctx, cmdline_opts.db_file);
if (!g_hlr_db_tool_ctx->dbc) {
LOGP(DMAIN, LOGL_FATAL, "Error opening database\n");
exit(1);
}
osmo_init_ignore_signals();
signal(SIGINT, &signal_hdlr);
signal(SIGUSR1, &signal_hdlr);
rc = 0;
if (main_action)
rc = (*main_action)();
db_close(g_hlr_db_tool_ctx->dbc);
log_fini();
exit(rc);
too_many_actions:
fprintf(stderr, "Too many actions requested.\n");
log_fini();
exit(1);
}
/* stubs */
void lu_op_alloc_conn(void) { OSMO_ASSERT(0); }
void lu_op_tx_del_subscr_data(void) { OSMO_ASSERT(0); }
void lu_op_free(void) { OSMO_ASSERT(0); }

View File

@@ -5,19 +5,19 @@ const struct log_info_cat hlr_log_info_cat[] = {
[DMAIN] = {
.name = "DMAIN",
.description = "Main Program",
.enabled = 1, .loglevel = LOGL_DEBUG,
.enabled = 1, .loglevel = LOGL_NOTICE,
},
[DDB] = {
.name = "DDB",
.description = "Database Layer",
.color = "\033[1;31m",
.enabled = 1, .loglevel = LOGL_DEBUG,
.enabled = 1, .loglevel = LOGL_NOTICE,
},
[DAUC] = {
.name = "DAUC",
.description = "Authentication Center",
.color = "\033[1;33m",
.enabled = 1, .loglevel = LOGL_DEBUG,
.enabled = 1, .loglevel = LOGL_NOTICE,
},
};

View File

@@ -1,9 +1,7 @@
AM_CPPFLAGS = \
AM_CFLAGS = \
$(all_includes) \
-I$(top_srcdir)/src \
$(NULL)
AM_CFLAGS = \
-I$(top_builddir)/src \
-Wall \
-ggdb3 \
$(LIBOSMOCORE_CFLAGS) \
@@ -11,9 +9,6 @@ AM_CFLAGS = \
$(SQLITE3_CFLAGS) \
$(NULL)
AM_LDFLAGS = \
$(NULL)
EXTRA_DIST = \
db_test.ok \
db_test.err \