paulmataruso/goflow2
1
0
Fork 0
mirror of https://github.com/openobserve/goflow2.git synced 2025-10-23 07:11:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add new compose stack (#5)

Browse Source 
* add compose for ELK
* reorg compose directories, bind to docker versions and fix Clickhouse/Grafana configurations
This commit is contained in:
Louis
2021-05-31 22:10:05 -07:00
committed by GitHub
parent df01c790df
commit 80ef166b84
16 changed files with 126 additions and 631 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
README.md
View File

@@ -192,11 +192,16 @@ $ ./goflow2 -format=pb | ./enricher -db.asn path-to/GeoLite2-ASN.mmdb -db.countr
For a more scalable production setting, Kafka and protobuf are recommended.
Stream operations (aggregation and filtering) can be done with stream-processor tools.
For instance Flink, or the more recent Kafka Streams and kSQLdb.
Direct storage can be done with [Clickhouse](/docs/clickhouse.md). This database can also create materialized tables.
Direct storage can be done with data-warehouses like Clickhouse.
In some cases, the consumer will require protobuf messages to be prefixed by
length. To do this, use the flag `-format.protobuf.fixedlen=true`.
This repository contains [examples of pipelines](./compose) with docker-compose.
The available pipelines are:
* [Kafka+Clickhouse+Grafana](./compose/kcg)
* [Logstash+Elastic+Kibana](./compose/elk)
## User stories
Are you using GoFlow2 in production at scale? Add yourself here!

25
compose/elk/README.md Normal file
View File

@@ -0,0 +1,25 @@
# Flows + Logstash + Elastic + Kibana
Clickhouse is a powerful data warehouse.
A sample [docker-compose](./docker-compose.yml) is provided.
It's composed of:
* GoFlow2
* Logstash
* Elastic
* Kibana
To start the containers, use:
```bash
$ docker-compose up
```
This command will automatically build the GoFlow2 container.
GoFlow2 collects NetFlow v9/IPFIX and sFlow packets and logs them into a file (`/var/log/goflow/goflow.log`).
Logstash collects the log messages, parse the JSON and sends to Elastic.
Kibana can be used to visualize the data. You can access the dashboard at http://localhost:5601.
This stack requires to create an [index pattern](http://localhost:5601/app/management/kibana/indexPatterns/create).
Define the index pattern to be `logstash-*`. Select `@timestamp` to be the time filter.
You can then visualize flows in the [Discover](http://localhost:5601/app/discover) section.

51
compose/elk/docker-compose.yml Normal file
View File

@@ -0,0 +1,51 @@
version: "3"
services:
goflow:
build:
context: ../../
dockerfile: Dockerfile
user: root # because docker-compose mount as root
ports:
- '8080:8080'
- '6343:6343/udp'
- '2055:2055/udp'
command:
- -transport=file
- -transport.file=/var/log/goflow/goflow.log
- -format=json
restart: always
logging:
driver: gelf
options:
gelf-address: "udp://localhost:12201"
tag: "flows"
volumes:
- logs:/var/log/goflow
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.0
environment:
- discovery.type=single-node
ports:
- 9200:9200
kibana:
image: docker.elastic.co/kibana/kibana:7.13.0
ports:
- 5601:5601
depends_on:
- elasticsearch
- logstash
logstash:
image: docker.elastic.co/logstash/logstash:7.13.0
user: root # because docker-compose mount as root
links:
- elasticsearch
volumes:
- ./logstash.conf:/etc/logstash/logstash.conf
- logs:/var/log/goflow
command: logstash -f /etc/logstash/logstash.conf
ports:
- 12201:12201/udp
depends_on:
- elasticsearch
volumes:
logs:

22
compose/elk/logstash.conf Normal file
View File

@@ -0,0 +1,22 @@
input {
gelf {
port => 12201
}
file {
path => "/var/log/goflow/*.log"
type => "log"
}
}
filter {
json {
source => "message"
target => "flow"
remove_field => ["message"]
}
}
output {
elasticsearch {
hosts => ["elasticsearch:9200"]
index => "logstash-%{+YYYY-MM-dd}"
}
}

587
compose/grafana/dashboards/viz.json
View File

@@ -1,587 +0,0 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"id": 3,
"links": [],
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "PostgreSQL",
"fill": 1,
"gridPos": {
"h": 9,
"w": 24,
"x": 0,
"y": 0
},
"id": 2,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"format": "time_series",
"group": [],
"metricColumn": "none",
"rawQuery": true,
"rawSql": "SELECT\n (cast(extract(epoch from time_flow) as integer)/30)*30 AS \"time\",\n sum(bytes*sampling_rate*8)/30\nFROM flows\nWHERE\n $__timeFilter(date_inserted)\nGROUP BY \"time\"\nORDER BY \"time\"",
"refId": "A",
"select": [
[
{
"params": [
"bytes"
],
"type": "column"
}
]
],
"table": "flows",
"timeColumn": "date_inserted",
"timeColumnType": "timestamp",
"where": [
{
"name": "$__timeFilter",
"params": [],
"type": "macro"
}
]
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Instant traffic",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "bps",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"columns": [],
"datasource": "PostgreSQL",
"fontSize": "100%",
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 9
},
"id": 7,
"links": [],
"pageSize": null,
"scroll": true,
"showHeader": true,
"sort": {
"col": 2,
"desc": true
},
"styles": [
{
"alias": "Time",
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"pattern": "Time",
"type": "date"
},
{
"alias": "",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 0,
"mappingType": 1,
"pattern": ".*_port",
"thresholds": [],
"type": "number",
"unit": "none"
},
{
"alias": "",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"mappingType": 1,
"pattern": "sumbytes",
"thresholds": [],
"type": "number",
"unit": "decbytes"
},
{
"alias": "",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"decimals": 0,
"pattern": "/.*/",
"thresholds": [],
"type": "number",
"unit": "short"
}
],
"targets": [
{
"format": "table",
"group": [],
"metricColumn": "none",
"rawQuery": true,
"rawSql": "SELECT src_ip, count(*), sum(bytes) AS sumBytes FROM flows GROUP BY src_ip",
"refId": "A",
"select": [
[
{
"params": [
"value"
],
"type": "column"
}
]
],
"timeColumn": "time",
"where": [
{
"name": "$__timeFilter",
"params": [],
"type": "macro"
}
]
}
],
"title": "Top source IPs",
"transform": "table",
"type": "table"
},
{
"columns": [],
"datasource": "PostgreSQL",
"fontSize": "100%",
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 9
},
"id": 5,
"links": [],
"pageSize": null,
"scroll": true,
"showHeader": true,
"sort": {
"col": 2,
"desc": true
},
"styles": [
{
"alias": "Time",
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"pattern": "Time",
"type": "date"
},
{
"alias": "",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 0,
"mappingType": 1,
"pattern": ".*_port",
"thresholds": [],
"type": "number",
"unit": "none"
},
{
"alias": "",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"mappingType": 1,
"pattern": "sumbytes",
"thresholds": [],
"type": "number",
"unit": "decbytes"
},
{
"alias": "",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"decimals": 0,
"pattern": "/.*/",
"thresholds": [],
"type": "number",
"unit": "short"
}
],
"targets": [
{
"format": "table",
"group": [],
"metricColumn": "none",
"rawQuery": true,
"rawSql": "SELECT src_port, count(*), sum(bytes) AS sumBytes FROM flows GROUP BY src_port",
"refId": "A",
"select": [
[
{
"params": [
"value"
],
"type": "column"
}
]
],
"timeColumn": "time",
"where": [
{
"name": "$__timeFilter",
"params": [],
"type": "macro"
}
]
}
],
"title": "Top source ports",
"transform": "table",
"type": "table"
},
{
"columns": [],
"datasource": "PostgreSQL",
"fontSize": "100%",
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 18
},
"id": 8,
"links": [],
"pageSize": null,
"scroll": true,
"showHeader": true,
"sort": {
"col": 2,
"desc": true
},
"styles": [
{
"alias": "Time",
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"pattern": "Time",
"type": "date"
},
{
"alias": "",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 0,
"mappingType": 1,
"pattern": ".*_port",
"thresholds": [],
"type": "number",
"unit": "none"
},
{
"alias": "",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"mappingType": 1,
"pattern": "sumbytes",
"thresholds": [],
"type": "number",
"unit": "decbytes"
},
{
"alias": "",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"decimals": 0,
"pattern": "/.*/",
"thresholds": [],
"type": "number",
"unit": "short"
}
],
"targets": [
{
"format": "table",
"group": [],
"metricColumn": "none",
"rawQuery": true,
"rawSql": "SELECT dst_ip, count(*), sum(bytes) AS sumBytes FROM flows GROUP BY dst_ip",
"refId": "A",
"select": [
[
{
"params": [
"value"
],
"type": "column"
}
]
],
"timeColumn": "time",
"where": [
{
"name": "$__timeFilter",
"params": [],
"type": "macro"
}
]
}
],
"title": "Top destination IPs",
"transform": "table",
"type": "table"
},
{
"columns": [],
"datasource": "PostgreSQL",
"fontSize": "100%",
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 18
},
"id": 6,
"links": [],
"pageSize": null,
"scroll": true,
"showHeader": true,
"sort": {
"col": 2,
"desc": true
},
"styles": [
{
"alias": "Time",
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"pattern": "Time",
"type": "date"
},
{
"alias": "",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 0,
"mappingType": 1,
"pattern": ".*_port",
"thresholds": [],
"type": "number",
"unit": "none"
},
{
"alias": "",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"mappingType": 1,
"pattern": "sumbytes",
"thresholds": [],
"type": "number",
"unit": "decbytes"
},
{
"alias": "",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"decimals": 0,
"pattern": "/.*/",
"thresholds": [],
"type": "number",
"unit": "short"
}
],
"targets": [
{
"format": "table",
"group": [],
"metricColumn": "none",
"rawQuery": true,
"rawSql": "SELECT dst_port, count(*), sum(bytes) AS sumBytes FROM flows GROUP BY dst_port",
"refId": "A",
"select": [
[
{
"params": [
"value"
],
"type": "column"
}
]
],
"timeColumn": "time",
"where": [
{
"name": "$__timeFilter",
"params": [],
"type": "macro"
}
]
}
],
"title": "Top destination ports",
"transform": "table",
"type": "table"
}
],
"schemaVersion": 16,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-3h",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
]
},
"timezone": "",
"title": "Traffic",
"uid": "HdAEBnPiz",
"version": 3
}

23
compose/grafana/datasources.yml
View File

@@ -1,23 +0,0 @@
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
access: proxy
orgId: 1
url: http://prometheus:9090
version: 1
editable: true
- name: PostgreSQL
type: postgres
access: proxy
orgId: 1
url: postgres:5432
database: postgres
user: postgres
jsonData:
sslmode: disable
secureJsonData:
password: flows
version: 1
editable: true

11
docs/clickhouse.md → compose/kcg/README.md
View File

@@ -1,8 +1,8 @@
# Flows and Clickhouse
# Flows + Kafka + Clicklhouse + Grafana + Prometheus
Clickhouse is a powerful data warehouse.
A sample [docker-compose](../compose/docker-compose.yml) is provided.
A sample [docker-compose](./docker-compose.yml) is provided.
It's composed of:
* Apache Kafka
* Apache Zookeeper
@@ -22,6 +22,7 @@ GoFlow2 collects NetFlow v9/IPFIX and sFlow packets and sends as a protobuf into
Zookeeper coordinates Kafka and can also be used by Clickhouse to ensure replication.
Prometheus scrapes the metrics of the collector.
Clickhouse consumes from Kafka, stores raw data and aggregates over specific columns
using `MATERIALIZED TABLES` and `VIEWS` defined in a [schema file](../compose/clickhouse/create.sh).
Youj can visualize the data in Grafana (credentials: admin/admin) with the
pre-made dashboards.
using `MATERIALIZED TABLES` and `VIEWS` defined in a [schema file](./clickhouse/create.sh).
You can visualize the data in Grafana at http://localhost:3000 (credentials: admin/admin) with the
pre-made dashboards.

0
compose/clickhouse/create.sh → compose/kcg/clickhouse/create.sh
View File

2
compose/clickhouse/protocols.csv → compose/kcg/clickhouse/protocols.csv
View File

@@ -1,4 +1,4 @@
Proto,Name,Description
proto,name,description
0,HOPOPT,IPv6 Hop-by-Hop Option
1,ICMP,Internet Control Message
2,IGMP,Internet Group Management
1 Proto proto Name name Description description
2 0 HOPOPT IPv6 Hop-by-Hop Option
3 1 ICMP Internet Control Message
4 2 IGMP Internet Group Management

27
compose/docker-compose.yml → compose/kcg/docker-compose.yml
View File

@@ -1,16 +1,16 @@
version: "3"
services:
zookeeper:
image: 'bitnami/zookeeper:latest'
image: bitnami/zookeeper:3.6.3
ports:
- '2181:2181'
- 2181:2181
environment:
- ALLOW_ANONYMOUS_LOGIN=yes
restart: always
kafka:
image: 'bitnami/kafka:latest'
image: bitnami/kafka:2.8.0
ports:
- '9092:9092'
- 9092:9092
environment:
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
- ALLOW_PLAINTEXT_LISTENER=yes
@@ -23,29 +23,29 @@ services:
environment:
- GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=vertamedia-clickhouse-datasource
ports:
- '3000:3000'
- 3000:3000
restart: always
volumes:
- ./grafana/datasources-ch.yml:/etc/grafana/provisioning/datasources/datasources-ch.yml
- ./grafana/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml
- ./grafana/dashboards:/var/lib/grafana/dashboards
prometheus:
image: 'prom/prometheus'
image: prom/prometheus:v2.27.0
ports:
- '9090:9090'
- 9090:9090
restart: always
volumes:
- ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
goflow:
build:
context: ../
context: ../../
dockerfile: Dockerfile
depends_on:
- kafka
ports:
- '8080:8080'
- '6343:6343/udp'
- '2055:2055/udp'
- 8080:8080
- 6343:6343/udp
- 2055:2055/udp
restart: always
command:
- -transport.kafka.brokers=kafka:9092
@@ -54,11 +54,12 @@ services:
- -format=pb
- -format.protobuf.fixedlen=true
db:
image: yandex/clickhouse-server
image: yandex/clickhouse-server:21.5.6.6
ports:
- 8123:8123
volumes:
- ./clickhouse:/docker-entrypoint-initdb.d/
- ../pb/flow.proto:/var/lib/clickhouse/format_schemas/flow.proto
- ../../pb/flow.proto:/var/lib/clickhouse/format_schemas/flow.proto
- ./clickhouse/protocols.csv:/var/lib/clickhouse/user_files/protocols.csv
depends_on:
- kafka

2
compose/grafana/Dockerfile → compose/kcg/grafana/Dockerfile
View File

@@ -3,6 +3,6 @@ FROM ubuntu AS builder
RUN apt-get update && apt-get install -y git
RUN git clone https://github.com/Vertamedia/clickhouse-grafana.git
FROM grafana/grafana
FROM grafana/grafana:7.5.7
COPY --from=builder /clickhouse-grafana /var/lib/grafana/plugins

0
compose/grafana/dashboards.yml → compose/kcg/grafana/dashboards.yml
View File

0
compose/grafana/dashboards/perfs.json → compose/kcg/grafana/dashboards/perfs.json
View File

0
compose/grafana/dashboards/viz-ch.json → compose/kcg/grafana/dashboards/viz-ch.json
View File

0
compose/grafana/datasources-ch.yml → compose/kcg/grafana/datasources-ch.yml
View File

0
compose/prometheus/prometheus.yml → compose/kcg/prometheus/prometheus.yml
View File