paulmataruso/osmo-mgw
1
0
Fork 0
mirror of https://gitea.osmocom.org/cellular-infrastructure/osmo-mgw.git synced 2025-10-23 08:12:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

meas_feed: Add tool to read meas_feed PCAP and write it to sqlite3

Browse Source 
In fact, the DB write code has been split out so we could later also
have a binary that would listen to realtime meas_feed packets and stuff
them into a database without any intermediate PCAP step.

The database schema contains a couple of convenience wrapper views, most
notably the "overview" view.
This commit is contained in:
Harald Welte
2012-11-15 00:12:56 +01:00
parent 7465e4c2e1
commit dc9b4e9ebf
4 changed files with 489 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
openbsc/src/utils/Makefile.am
View File

@@ -2,10 +2,12 @@ AM_CPPFLAGS = $(all_includes) -I$(top_srcdir)/include -I$(top_builddir)
AM_CFLAGS=-Wall $(LIBOSMOCORE_CFLAGS) $(LIBOSMOGSM_CFLAGS) $(LIBOSMOVTY_CFLAGS) $(LIBOSMOABIS_CFLAGS) $(COVERAGE_CFLAGS)
AM_LDFLAGS = $(COVERAGE_LDFLAGS)
noinst_HEADERS = meas_db.h
if HAVE_LIBCDK
bin_PROGRAMS = bs11_config isdnsync meas_vis
bin_PROGRAMS = bs11_config isdnsync osmo-meas-pcap2db meas_vis
else
bin_PROGRAMS = bs11_config isdnsync
bin_PROGRAMS = bs11_config isdnsync osmo-meas-pcap2db
endif
if BUILD_SMPP
@@ -27,3 +29,7 @@ smpp_mirror_LDADD = $(top_builddir)/src/libcommon/libcommon.a \
meas_vis_SOURCES = meas_vis.c
meas_vis_LDADD = $(LIBOSMOCORE_LIBS) $(LIBOSMOGSM_LIBS) -lcdk -lncurses
meas_vis_CFLAGS = $(LIBOSMOCORE_CFLAGS) $(LIBOSMOGSM_CFLAGS)
osmo_meas_pcap2db_SOURCES = meas_pcap2db.c meas_db.c
osmo_meas_pcap2db_LDADD = $(LIBOSMOCORE_LIBS) $(LIBOSMOGSM_LIBS) -lpcap -lsqlite3
osmo_meas_pcap2db_CFLAGS = $(LIBOSMOCORE_CFLAGS) $(LIBOSMOGSM_CFLAGS)

323
openbsc/src/utils/meas_db.c Normal file
View File

@@ -0,0 +1,323 @@
/* Routines for storing measurement reports in SQLite3 database */
/* (C) 2012 by Harald Welte <laforge@gnumonks.org>
*
* All Rights Reserved
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
#include <stdint.h>
#include <errno.h>
#include <string.h>
#include <sqlite3.h>
#include <osmocom/core/talloc.h>
#include <osmocom/core/utils.h>
#include <openbsc/meas_rep.h>
#include "meas_db.h"
#define INS_MR "INSERT INTO meas_rep (time, imsi, name, scenario, nr, bs_power, ms_timing_offset, fpc, ms_l1_pwr, ms_l1_ta) VALUES (?,?,?,?,?,?,?,?,?,?)"
#define INS_UD "INSERT INTO meas_rep_unidir (meas_id, rx_lev_full, rx_lev_sub, rx_qual_full, rx_qual_sub, dtx, uplink) VALUES (?,?,?,?,?,?,?)"
#define UPD_MR "UPDATE meas_rep SET ul_unidir=?, dl_unidir=? WHERE id=?"
struct meas_db_state {
sqlite3 *db;
sqlite3_stmt *stmt_ins_ud;
sqlite3_stmt *stmt_ins_mr;
sqlite3_stmt *stmt_upd_mr;
};
/* macros to check for SQLite3 result codes */
#define _SCK_OK(db, call, exp) \
do { \
int rc = call; \
if (rc != exp) { \
fprintf(stderr,"SQL Error in line %u: %s\n", \
__LINE__, sqlite3_errmsg(db)); \
goto err_io; \
} \
} while (0)
#define SCK_OK(db, call) _SCK_OK(db, call, SQLITE_OK)
#define SCK_DONE(db, call) _SCK_OK(db, call, SQLITE_DONE)
static int _insert_ud(struct meas_db_state *st, unsigned long meas_id, int dtx,
int uplink, const struct gsm_meas_rep_unidir *ud)
{
unsigned long rowid;
SCK_OK(st->db, sqlite3_bind_int(st->stmt_ins_ud, 1, meas_id));
SCK_OK(st->db, sqlite3_bind_int(st->stmt_ins_ud, 2,
rxlev2dbm(ud->full.rx_lev)));
SCK_OK(st->db, sqlite3_bind_int(st->stmt_ins_ud, 3,
rxlev2dbm(ud->sub.rx_lev)));
SCK_OK(st->db, sqlite3_bind_int(st->stmt_ins_ud, 4, ud->full.rx_qual));
SCK_OK(st->db, sqlite3_bind_int(st->stmt_ins_ud, 5, ud->sub.rx_qual));
SCK_OK(st->db, sqlite3_bind_int(st->stmt_ins_ud, 6, dtx));
SCK_OK(st->db, sqlite3_bind_int(st->stmt_ins_ud, 7, uplink));
SCK_DONE(st->db, sqlite3_step(st->stmt_ins_ud));
SCK_OK(st->db, sqlite3_reset(st->stmt_ins_ud));
return sqlite3_last_insert_rowid(st->db);
err_io:
exit(1);
}
/* insert a measurement report into the database */
int meas_db_insert(struct meas_db_state *st, const char *imsi,
const char *name, unsigned long timestamp,
const char *scenario,
const struct gsm_meas_rep *mr)
{
int rc;
sqlite3_int64 rowid, ul_rowid, dl_rowid;
SCK_OK(st->db, sqlite3_bind_int(st->stmt_ins_mr, 1, timestamp));
if (imsi)
SCK_OK(st->db, sqlite3_bind_text(st->stmt_ins_mr, 2,
imsi, -1, SQLITE_STATIC));
else
SCK_OK(st->db, sqlite3_bind_null(st->stmt_ins_mr, 2));
if (name)
SCK_OK(st->db, sqlite3_bind_text(st->stmt_ins_mr, 3,
name, -1, SQLITE_STATIC));
else
SCK_OK(st->db, sqlite3_bind_null(st->stmt_ins_mr, 3));
if (scenario)
SCK_OK(st->db, sqlite3_bind_text(st->stmt_ins_mr, 4,
scenario, -1, SQLITE_STATIC));
else
SCK_OK(st->db, sqlite3_bind_null(st->stmt_ins_mr, 4));
SCK_OK(st->db, sqlite3_bind_int(st->stmt_ins_mr, 5, mr->nr));
SCK_OK(st->db, sqlite3_bind_int(st->stmt_ins_mr, 6, mr->bs_power));
if (mr->flags & MEAS_REP_F_MS_TO)
SCK_OK(st->db, sqlite3_bind_int(st->stmt_ins_mr, 7,
mr->ms_timing_offset));
else
SCK_OK(st->db, sqlite3_bind_null(st->stmt_ins_mr, 7));
if (mr->flags & MEAS_REP_F_FPC)
SCK_OK(st->db, sqlite3_bind_int(st->stmt_ins_mr, 8, 1));
else
SCK_OK(st->db, sqlite3_bind_int(st->stmt_ins_mr, 8, 0));
if (mr->flags & MEAS_REP_F_MS_L1) {
SCK_OK(st->db, sqlite3_bind_int(st->stmt_ins_mr, 9,
mr->ms_l1.pwr));
SCK_OK(st->db, sqlite3_bind_int(st->stmt_ins_mr, 10,
mr->ms_l1.ta));
}
SCK_DONE(st->db, sqlite3_step(st->stmt_ins_mr));
SCK_OK(st->db, sqlite3_reset(st->stmt_ins_mr));
rowid = sqlite3_last_insert_rowid(st->db);
/* insert uplink measurement */
ul_rowid = _insert_ud(st, rowid, mr->flags & MEAS_REP_F_UL_DTX,
1, &mr->ul);
SCK_OK(st->db, sqlite3_bind_int(st->stmt_upd_mr, 1, ul_rowid));
/* insert downlink measurement, if present */
if (mr->flags & MEAS_REP_F_DL_VALID) {
dl_rowid = _insert_ud(st, rowid, mr->flags & MEAS_REP_F_DL_DTX,
0, &mr->dl);
SCK_OK(st->db, sqlite3_bind_int(st->stmt_upd_mr, 2, dl_rowid));
} else
SCK_OK(st->db, sqlite3_bind_null(st->stmt_upd_mr, 2));
/* update meas_rep with the id's of the unidirectional
* measurements */
SCK_OK(st->db, sqlite3_bind_int(st->stmt_upd_mr, 3, rowid));
SCK_DONE(st->db, sqlite3_step(st->stmt_upd_mr));
SCK_OK(st->db, sqlite3_reset(st->stmt_upd_mr));
return 0;
err_io:
return -EIO;
}
int meas_db_begin(struct meas_db_state *st)
{
SCK_OK(st->db, sqlite3_exec(st->db, "BEGIN", NULL, NULL, NULL));
return 0;
err_io:
return -EIO;
}
int meas_db_commit(struct meas_db_state *st)
{
SCK_OK(st->db, sqlite3_exec(st->db, "COMMIT", NULL, NULL, NULL));
return 0;
err_io:
return -EIO;
}
static const char *create_stmts[] = {
"CREATE TABLE IF NOT EXISTS meas_rep ("
"id INTEGER PRIMARY KEY AUTOINCREMENT,"
"time TIMESTAMP,"
"imsi TEXT,"
"name TEXT,"
"scenario TEXT,"
"nr INTEGER,"
"bs_power INTEGER NOT NULL,"
"ms_timing_offset INTEGER,"
"fpc INTEGER NOT NULL DEFAULT 0,"
"ul_unidir INTEGER REFERENCES meas_rep_unidir(id),"
"dl_unidir INTEGER REFERENCES meas_rep_unidir(id),"
"ms_l1_pwr INTEGER,"
"ms_l1_ta INTEGER"
")",
"CREATE TABLE IF NOT EXISTS meas_rep_unidir ("
"id INTEGER PRIMARY KEY AUTOINCREMENT,"
"meas_id INTEGER NOT NULL REFERENCES meas_rep(id),"
"rx_lev_full INTEGER NOT NULL,"
"rx_lev_sub INTEGER NOT NULL,"
"rx_qual_full INTEGER NOT NULL,"
"rx_qual_sub INTEGER NOT NULL,"
"dtx BOOLEAN NOT NULL DEFAULT 0,"
"uplink BOOLEAN NOT NULL"
")",
"CREATE VIEW IF NOT EXISTS path_loss AS "
"SELECT "
"meas_rep.id, "
"datetime(time,'unixepoch') AS timestamp, "
"imsi, "
"name, "
"scenario, "
"ms_timing_offset, "
"ms_l1_ta, "
"fpc, "
"ms_l1_pwr, "
"ud_ul.rx_lev_full AS ul_rx_lev_full, "
"ms_l1_pwr-ud_ul.rx_lev_full AS ul_path_loss_full, "
"ud_ul.rx_lev_sub ul_rx_lev_sub, "
"ms_l1_pwr-ud_ul.rx_lev_sub AS ul_path_loss_sub, "
"ud_ul.rx_qual_full AS ul_rx_qual_full, "
"ud_ul.rx_qual_sub AS ul_rx_qual_sub, "
"bs_power, "
"ud_dl.rx_lev_full AS dl_rx_lev_full, "
"bs_power-ud_dl.rx_lev_full AS dl_path_loss_full, "
"ud_dl.rx_lev_sub AS dl_rx_lev_sub, "
"bs_power-ud_dl.rx_lev_sub AS dl_path_loss_sub, "
"ud_dl.rx_qual_full AS dl_rx_qual_full, "
"ud_dl.rx_qual_sub AS dl_rx_qual_sub "
"FROM "
"meas_rep, "
"meas_rep_unidir AS ud_dl, "
"meas_rep_unidir AS ud_ul "
"WHERE "
"ud_ul.id = meas_rep.ul_unidir AND "
"ud_dl.id = meas_rep.dl_unidir",
"CREATE VIEW IF NOT EXISTS overview AS "
"SELECT "
"id,"
"timestamp,"
"imsi,"
"name,"
"scenario,"
"ms_l1_pwr,"
"ul_rx_lev_full,"
"ul_path_loss_full,"
"ul_rx_qual_full,"
"bs_power,"
"dl_rx_lev_full,"
"dl_path_loss_full,"
"dl_rx_qual_full "
"FROM path_loss",
};
static int check_create_tbl(struct meas_db_state *st)
{
int i, rc;
for (i = 0; i < ARRAY_SIZE(create_stmts); i++) {
SCK_OK(st->db, sqlite3_exec(st->db, create_stmts[i],
NULL, NULL, NULL));
}
return 0;
err_io:
return -EIO;
}
#define PREP_CHK(db, stmt, ptr) \
do { \
int rc; \
rc = sqlite3_prepare_v2(db, stmt, strlen(stmt)+1, \
ptr, NULL); \
if (rc != SQLITE_OK) { \
fprintf(stderr, "Error during prepare of '%s': %s\n", \
stmt, sqlite3_errmsg(db)); \
goto err_io; \
} \
} while (0)
struct meas_db_state *meas_db_open(void *ctx, const char *fname)
{
int rc;
struct meas_db_state *st = talloc_zero(ctx, struct meas_db_state);
if (!st)
return NULL;
rc = sqlite3_open_v2(fname, &st->db,
SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE,
NULL);
if (rc != SQLITE_OK) {
fprintf(stderr, "Unable to open DB: %s\n",
sqlite3_errmsg(st->db));
goto err_io;
}
rc = check_create_tbl(st);
PREP_CHK(st->db, INS_MR, &st->stmt_ins_mr);
PREP_CHK(st->db, INS_UD, &st->stmt_ins_ud);
PREP_CHK(st->db, UPD_MR, &st->stmt_upd_mr);
return st;
err_io:
talloc_free(st);
return NULL;
}
void meas_db_close(struct meas_db_state *st)
{
sqlite3_finalize(st->stmt_ins_mr);
sqlite3_finalize(st->stmt_ins_ud);
sqlite3_finalize(st->stmt_upd_mr);
sqlite3_close_v2(st->db);
talloc_free(st);
}

17
openbsc/src/utils/meas_db.h Normal file
View File

@@ -0,0 +1,17 @@
#ifndef OPENBSC_MEAS_DB_H
#define OPENBSC_MEAS_DB_H
struct meas_db_state;
struct meas_db_state *meas_db_open(void *ctx, const char *fname);
void meas_db_close(struct meas_db_state *st);
int meas_db_begin(struct meas_db_state *st);
int meas_db_commit(struct meas_db_state *st);
int meas_db_insert(struct meas_db_state *st, const char *imsi,
const char *name, unsigned long timestamp,
const char *scenario,
const struct gsm_meas_rep *mr);
#endif

141
openbsc/src/utils/meas_pcap2db.c Normal file
View File

@@ -0,0 +1,141 @@
/* read PCAP file with meas_feed data and write it to sqlite3 database */
/* (C) 2012 by Harald Welte <laforge@gnumonks.org>
*
* All Rights Reserved
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
#include <string.h>
#include <errno.h>
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <netinet/udp.h>
#include <cdk/cdk.h>
#include <osmocom/core/socket.h>
#include <osmocom/core/utils.h>
#include <osmocom/core/msgb.h>
#include <osmocom/core/select.h>
#include <osmocom/core/talloc.h>
#include <osmocom/gsm/gsm_utils.h>
#include <openbsc/meas_feed.h>
#include <pcap/pcap.h>
#include "meas_db.h"
static struct meas_db_state *db;
static void handle_mfm(const struct pcap_pkthdr *h,
const struct meas_feed_meas *mfm)
{
const char *scenario;
if (strlen(mfm->scenario))
scenario = mfm->scenario;
else
scenario = NULL;
meas_db_insert(db, mfm->imsi, mfm->name, h->ts.tv_sec,
scenario, &mfm->mr);
}
static void pcap_cb(u_char *user, const struct pcap_pkthdr *h,
const u_char *bytes)
{
const char *cur = bytes;
const struct iphdr *ip;
const struct udphdr *udp;
const struct meas_feed_meas *mfm;
uint16_t udplen;
if (h->caplen < 14+20+8)
return;
/* Check if there is IPv4 in the Ethernet */
if (cur[12] != 0x08 || cur[13] != 0x00)
return;
cur += 14; /* ethernet header */
ip = (struct iphdr *) cur;
if (ip->version != 4)
return;
cur += ip->ihl * 4;
if (ip->protocol != IPPROTO_UDP)
return;
udp = (struct udphdr *) cur;
if (udp->dest != htons(8888))
return;
udplen = ntohs(udp->len);
if (udplen != sizeof(*udp) + sizeof(*mfm))
return;
cur += sizeof(*udp);
mfm = (const struct meas_feed_meas *) cur;
handle_mfm(h, mfm);
}
int main(int argc, char **argv)
{
char errbuf[PCAP_ERRBUF_SIZE+1];
char *pcap_fname, *db_fname;
pcap_t *pc;
int rc;
if (argc < 3) {
fprintf(stderr, "You need to specify PCAP and database file\n");
exit(2);
}
pcap_fname = argv[1];
db_fname = argv[2];
pc = pcap_open_offline(pcap_fname, errbuf);
if (!pc) {
fprintf(stderr, "Cannot open %s: %s\n", pcap_fname, errbuf);
exit(1);
}
db = meas_db_open(NULL, db_fname);
if (!db)
exit(0);
rc = meas_db_begin(db);
if (rc < 0) {
fprintf(stderr, "Error during BEGIN\n");
exit(1);
}
pcap_loop(pc, 0 , pcap_cb, NULL);
meas_db_commit(db);
exit(0);
}