Inactive connection cleanup (disabled by default)

Add a watchdog timer to connections, and close these connections when
the watchdog timer expires. Kick the watchdog whenever RTP messages or
the relevant MGCP messages arrive. Add the currently remaining timeout
to "show mgcp stats" in the VTY.

This feature is disabled by default, as it is incompatible with LCLS
(connections in LCLS state appear to be inactive). Enable it with the
new "conn-timeout" VTY setting. In general, this feature can be used to
work around interoperability problems causing connections to stay open
forever, and slowly exhausting all available ports. This happened for
various reasons already.

MDCX is the only relevant MGCP message:
- CRCX creates the conn and timer
- DLCX deletes the conn and timer
- MDCX is the only remaining supported MGCP message that indicates a CI
- Can't easily generically parse a CI for all MGCP messages, parsing is
  done in handle_modify_con().

Related: OS#3429
Change-Id: I18886052e090466f73829133c24f011806cc1fe0

include/osmocom/mgcp/mgcp.h

@@ -274,6 +274,9 @@ struct mgcp_config {
 	uint16_t osmux_dummy;
 	/* domain name of the media gateway */
 	char domain[255+1];
+
+	/* time after which inactive connections (CIs) get closed */
+	int conn_timeout;
 };
 
 /* config management */

include/osmocom/mgcp/mgcp_internal.h

@@ -231,6 +231,9 @@ struct mgcp_conn {
 	/*! human readable name (vty, logging) */
 	char name[256];
 
+	/*! activity tracker (for cleaning up inactive connections) */
+	struct osmo_timer_list watchdog;
+
 	/*! union with connection description */
 	union {
 		struct mgcp_conn_rtp rtp;
@@ -328,3 +331,4 @@ enum {
 #define PTYPE_UNDEFINED (-1)
 
 void mgcp_get_local_addr(char *addr, struct mgcp_conn_rtp *conn);
+void mgcp_conn_watchdog_kick(struct mgcp_conn *conn);

src/libosmo-mgcp/mgcp_conn.c

@@ -29,6 +29,7 @@
 #include <osmocom/mgcp/mgcp_codec.h>
 #include <osmocom/gsm/gsm_utils.h>
 #include <osmocom/core/rate_ctr.h>
+#include <osmocom/core/timer.h>
 #include <ctype.h>
 
 const static struct rate_ctr_group_desc rate_ctr_group_desc = {
@@ -125,6 +126,23 @@ static void mgcp_rtp_conn_cleanup(struct mgcp_conn_rtp *conn_rtp)
 	rate_ctr_group_free(conn_rtp->rate_ctr_group);
 }
 
+void mgcp_conn_watchdog_cb(void *data)
+{
+	struct mgcp_conn *conn = data;
+	LOGP(DLMGCP, LOGL_ERROR, "endpoint:0x%x CI:%s connection timed out!\n", ENDPOINT_NUMBER(conn->endp), conn->id);
+	mgcp_conn_free(conn->endp, conn->id);
+}
+
+void mgcp_conn_watchdog_kick(struct mgcp_conn *conn)
+{
+	int timeout = conn->endp->cfg->conn_timeout;
+	if (!timeout)
+		return;
+
+	LOGP(DLMGCP, LOGL_DEBUG, "endpoint:0x%x CI:%s watchdog kicked\n", ENDPOINT_NUMBER(conn->endp), conn->id);
+	osmo_timer_schedule(&conn->watchdog, timeout, 0);
+}
+
 /*! allocate a new connection list entry.
  *  \param[in] ctx talloc context
  *  \param[in] endp associated endpoint
@@ -167,6 +185,9 @@ struct mgcp_conn *mgcp_conn_alloc(void *ctx, struct mgcp_endpoint *endp,
 		OSMO_ASSERT(false);
 	}
 
+	/* Initialize watchdog */
+	osmo_timer_setup(&conn->watchdog, mgcp_conn_watchdog_cb, conn);
+	mgcp_conn_watchdog_kick(conn);
 	llist_add(&conn->entry, &endp->conns);
 
 	return conn;
@@ -274,6 +295,7 @@ void mgcp_conn_free(struct mgcp_endpoint *endp, const char *id)
 		OSMO_ASSERT(false);
 	}
 
+	osmo_timer_del(&conn->watchdog);
 	llist_del(&conn->entry);
 	talloc_free(conn);
 }

src/libosmo-mgcp/mgcp_network.c

@@ -1246,6 +1246,8 @@ static int rtp_data_net(struct osmo_fd *fd, unsigned int what)
 	if (len < 0)
 		return -1;
 
+	mgcp_conn_watchdog_kick(conn_src->conn);
+
 	/* Check if the connection is in loopback mode, if yes, just send the
 	 * incoming data back to the origin */
 	if (conn_src->conn->mode == MGCP_CONN_LOOPBACK) {

src/libosmo-mgcp/mgcp_protocol.c

@@ -1141,6 +1141,8 @@ mgcp_header_done:
 		return create_err_response(endp, 400, "MDCX", p->trans);
 	}
 
+	mgcp_conn_watchdog_kick(conn->conn);
+
 	if (mode) {
 		if (mgcp_parse_conn_mode(mode, endp, conn->conn) != 0) {
 			rate_ctr_inc(&rate_ctrs->ctr[MGCP_MDCX_FAIL_INVALID_MODE]);

src/libosmo-mgcp/mgcp_vty.c

@@ -154,6 +154,10 @@ static int config_write_mgcp(struct vty *vty)
 		vty_out(vty, "  osmux dummy %s%s",
 			g_cfg->osmux_dummy ? "on" : "off", VTY_NEWLINE);
 	}
+
+	if (g_cfg->conn_timeout)
+		vty_out(vty, "  conn-timeout %u%s", g_cfg->conn_timeout, VTY_NEWLINE);
+
 	return CMD_SUCCESS;
 }
 
@@ -215,6 +219,13 @@ static void dump_endpoint(struct vty *vty, struct mgcp_endpoint *endp, int epidx
 		vty_out(vty, "   CONN: %s%s", mgcp_conn_dump(conn), VTY_NEWLINE);
 
 		if (show_stats) {
+			if (endp->cfg->conn_timeout) {
+				struct timeval remaining;
+				osmo_timer_remaining(&conn->watchdog, NULL, &remaining);
+				vty_out(vty, "   Currently remaining timeout (seconds): %d.%06d%s",
+					(int)remaining.tv_sec, (int)remaining.tv_usec, VTY_NEWLINE);
+			}
+
 			/* FIXME: Also add verbosity for other
 			 * connection types (E1) as soon as
 			 * the implementation is available */
@@ -1327,6 +1338,18 @@ DEFUN(cfg_mgcp_domain,
 	return CMD_SUCCESS;
 }
 
+DEFUN(cfg_mgcp_conn_timeout,
+      cfg_mgcp_conn_timeout_cmd,
+      "conn-timeout <1-65534>",
+      "Set a time after which inactive connections (CIs) are closed. This can be used to work around interoperability"
+      " problems causing connections to stay open forever, and slowly exhausting all available ports. Do not enable"
+      " when LCLS is used (connections in LCLS state appear to be inactive)!\n"
+      "Timeout value (sec.)\n")
+{
+	g_cfg->conn_timeout = strtoul(argv[0], NULL, 10);
+	return CMD_SUCCESS;
+}
+
 int mgcp_vty_init(void)
 {
 	install_element_ve(&show_mgcp_cmd);
@@ -1391,6 +1414,7 @@ int mgcp_vty_init(void)
 	install_element(MGCP_NODE, &cfg_mgcp_allow_transcoding_cmd);
 	install_element(MGCP_NODE, &cfg_mgcp_no_allow_transcoding_cmd);
 	install_element(MGCP_NODE, &cfg_mgcp_domain_cmd);
+	install_element(MGCP_NODE, &cfg_mgcp_conn_timeout_cmd);
 
 	install_element(MGCP_NODE, &cfg_mgcp_trunk_cmd);
 	install_node(&trunk_node, config_write_trunk);

tests/mgcp/mgcp_test.c

@@ -970,6 +970,7 @@ static void test_packet_loss_calc(void)
 {
 	int i;
 	struct mgcp_endpoint endp;
+	struct mgcp_config cfg = {0};
 	struct mgcp_trunk_config trunk;
 
 	printf("Testing packet loss calculation.\n");
@@ -977,6 +978,7 @@ static void test_packet_loss_calc(void)
 	memset(&endp, 0, sizeof(endp));
 	memset(&trunk, 0, sizeof(trunk));
 
+	endp.cfg = &cfg;
 	endp.type = &ep_typeset.rtp;
 	trunk.vty_number_endpoints = 1;
 	trunk.endpoints = &endp;
@@ -1197,6 +1199,7 @@ static void test_packet_error_detection(int patch_ssrc, int patch_ts)
 
 	struct mgcp_trunk_config trunk;
 	struct mgcp_endpoint endp;
+	struct mgcp_config cfg = {0};
 	struct mgcp_rtp_state state;
 	struct mgcp_rtp_end *rtp;
 	struct sockaddr_in addr = { 0 };
@@ -1224,6 +1227,7 @@ static void test_packet_error_detection(int patch_ssrc, int patch_ts)
 	state.in_stream.err_ts_ctr = &test_ctr_in;
 	state.out_stream.err_ts_ctr = &test_ctr_out;
 
+	endp.cfg = &cfg;
 	endp.type = &ep_typeset.rtp;
 
 	trunk.vty_number_endpoints = 1;