init

.github/workflows/build.yml

@@ -0,0 +1,51 @@
+name: Build
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v2
+      with:
+        go-version: ^1.14
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v1
+      # note: @v2 seem to be https://github.com/actions/checkout/issues/290
+      # which only works IF the tags are pushed before/same time as the commit
+      # otherwise, display previous tag
+      with:
+        fetch-depth: 0
+
+    - name: Test & Vet
+      run: make test vet
+      
+    - name: Build
+      run: |
+        GOOS=linux make build
+        GOOS=darwin make build
+        GOOS=windows EXTENSION=.exe make build
+          
+    - name: Install fpm
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y rpm ruby ruby-dev
+        sudo gem install fpm
+        
+    - name: Package
+      run: make package-deb package-rpm
+
+    - name: Upload Artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: dist
+        path: dist/*
+        retention-days: 14

.github/workflows/docker-release.yaml

@@ -0,0 +1,40 @@
+name: DockerRelease
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  build:
+    name: DockerRelease
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v2
+      with:
+        go-version: ^1.14
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v1
+      with:
+        fetch-depth: 0
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v1
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+
+    - name: Login to DockerHub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+    - name: Build
+      run: |
+        export VERSION=$(git describe --tags --abbrev=0 HEAD)
+        make docker
+        make push-docker-release

.github/workflows/docker.yaml

@@ -0,0 +1,38 @@
+name: Docker
+
+on:
+  push:
+    branches: [ main ]
+
+jobs:
+  build:
+    name: Docker
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v2
+      with:
+        go-version: ^1.14
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v1
+      with:
+        fetch-depth: 0
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v1
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+
+    - name: Login to DockerHub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+    - name: Build
+      run: |
+        make docker
+        make push-docker

.github/workflows/release.yml

@@ -0,0 +1,64 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  build:
+    name: Release
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v2
+      with:
+        go-version: ^1.14
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v1
+      with:
+        fetch-depth: 0
+                
+    - name: Install fpm
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y rpm ruby ruby-dev
+        sudo gem install fpm
+        
+    - name: Build
+      run: |
+        export VERSION=$(git describe --tags --abbrev=0 HEAD)
+        GOOS=linux make build
+        GOOS=darwin make build
+        GOOS=windows EXTENSION=.exe make build
+        make package-deb package-rpm
+
+    - name: Create Release
+      id: create_release
+      uses: actions/create-release@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        tag_name: ${{ github.ref }}
+        release_name: Release ${{ github.ref }}
+        draft: false
+        prerelease: false
+        
+    - name: Upload Release Asset
+      uses: actions/github-script@v2
+      with:
+        github-token: ${{secrets.GITHUB_TOKEN}}
+        script: |
+          const fs = require('fs').promises;
+          const upload_url = '${{ steps.create_release.outputs.upload_url }}';
+          for (let file of await fs.readdir('./dist')) {
+            console.log('uploading', file);
+            await github.repos.uploadReleaseAsset({
+              url: upload_url,
+              name: file,
+              data: await fs.readFile(`./dist/${file}`)
+            }); 
+          }
+ 

Dockerfile

@@ -0,0 +1,36 @@
+FROM golang:alpine as builder
+ARG LDFLAGS=""
+
+RUN apk --update --no-cache add git build-base gcc
+
+COPY . /build
+WORKDIR /build
+
+RUN go build -ldflags "${LDFLAGS}" -o goflow2 cmd/goflow2/main.go
+
+FROM alpine:latest
+ARG src_dir
+ARG VERSION=""
+ARG CREATED=""
+ARG DESCRIPTION=""
+ARG NAME=""
+ARG MAINTAINER=""
+ARG URL=""
+ARG LICENSE=""
+ARG REV=""
+
+LABEL org.opencontainers.image.created="${CREATED}"
+LABEL org.opencontainers.image.authors="${MAINTAINER}"
+LABEL org.opencontainers.image.url="${URL}"
+LABEL org.opencontainers.image.title="${NAME}"
+LABEL org.opencontainers.image.version="${VERSION}"
+LABEL org.opencontainers.image.description="${DESCRIPTION}"
+LABEL org.opencontainers.image.licenses="${LICENSE}"
+LABEL org.opencontainers.image.revision="${REV}"
+
+RUN apk update --no-cache && \
+    adduser -S -D -H -h / flow
+USER flow
+COPY --from=builder /build/goflow2 /
+
+ENTRYPOINT ["./goflow2"]

Makefile

@@ -0,0 +1,100 @@
+EXTENSION ?= 
+DIST_DIR ?= dist/
+GOOS ?= linux
+ARCH ?= $(shell uname -m)
+BUILDINFOSDET ?= 
+
+DOCKER_REPO   := netsampler/
+NAME          := goflow2
+VERSION       ?= $(shell git describe --abbrev --long HEAD)
+ABBREV        ?= $(shell git rev-parse --short HEAD)
+COMMIT        ?= $(shell git rev-parse HEAD)
+TAG           ?= $(shell git describe --tags --abbrev=0 HEAD)
+VERSION_PKG   ?= $(shell echo $(VERSION) | sed 's/^v//g')
+ARCH          := x86_64
+LICENSE       := BSD-3-Clause
+URL           := https://github.com/netsampler/goflow2
+DESCRIPTION   := GoFlow2: Open-Source and Scalable Network Sample Collector
+DATE          :=  $(shell date +%FT%T%z)
+BUILDINFOS    ?=  ($(DATE)$(BUILDINFOSDET))
+LDFLAGS       ?= '-X main.version=$(VERSION) -X main.buildinfos=$(BUILDINFOS)'
+MAINTAINER    := lspgn@users.noreply.github.com
+
+OUTPUT := $(DIST_DIR)goflow2-$(VERSION_PKG)-$(GOOS)-$(ARCH)$(EXTENSION)
+
+.PHONY: proto
+proto:
+	@echo generating protobuf
+	protoc --go_out=. pb/*.proto
+	protoc --go_out=. cmd/enricher/pb/*.proto
+
+.PHONY: vet
+vet:
+	go vet cmd/goflow2/main.go
+
+.PHONY: test
+test:
+	go test -v ./...
+
+.PHONY: prepare
+prepare:
+	mkdir -p $(DIST_DIR)
+
+PHONY: clean
+clean:
+	rm -rf $(DIST_DIR)
+
+.PHONY: build
+build: prepare
+	go build -ldflags $(LDFLAGS) -o $(OUTPUT) cmd/goflow2/main.go 
+
+.PHONY: docker
+docker:
+	docker build \
+        --build-arg LDFLAGS=$(LDFLAGS) \
+        --build-arg CREATED="$(DATE)" \
+        --build-arg MAINTAINER="$(MAINTAINER)" \
+        --build-arg URL="$(URL)" \
+        --build-arg NAME="$(NAME)" \
+        --build-arg DESCRIPTION="$(DESCRIPTION)" \
+        --build-arg LICENSE="$(LICENSE)" \
+        --build-arg VERSION="$(VERSION)" \
+        --build-arg REV="$(COMMIT)" \
+        -t $(DOCKER_REPO)$(NAME):$(ABBREV) .
+
+.PHONY: push-docker
+push-docker:
+	docker push $(DOCKER_REPO)$(NAME):$(ABBREV)
+	docker tag $(DOCKER_REPO)$(NAME):$(ABBREV) $(DOCKER_REPO)$(NAME):latest
+	docker push $(DOCKER_REPO)$(NAME):latest
+
+.PHONY: push-docker-release
+push-docker-release:
+	docker tag $(DOCKER_REPO)$(NAME):$(ABBREV) $(DOCKER_REPO)$(NAME):$(VERSION)
+	docker push $(DOCKER_REPO)$(NAME):$(VERSION)
+
+.PHONY: package-deb
+package-deb: prepare
+	fpm -s dir -t deb -n $(NAME) -v $(VERSION_PKG) \
+        --maintainer "$(MAINTAINER)" \
+        --description "$(DESCRIPTION)"  \
+        --url "$(URL)" \
+        --architecture $(ARCH) \
+        --license "$(LICENSE)" \
+        --package $(DIST_DIR) \
+        $(OUTPUT)=/usr/bin/goflow2 \
+        package/goflow2.service=/lib/systemd/system/goflow2.service \
+        package/goflow2.env=/etc/default/goflow2
+
+.PHONY: package-rpm
+package-rpm: prepare
+	fpm -s dir -t rpm -n $(NAME) -v $(VERSION_PKG) \
+        --maintainer "$(MAINTAINER)" \
+        --description "$(DESCRIPTION)" \
+        --url "$(URL)" \
+        --architecture $(ARCH) \
+        --license "$(LICENSE) "\
+        --package $(DIST_DIR) \
+        $(OUTPUT)=/usr/bin/goflow2 \
+        package/goflow2.service=/lib/systemd/system/goflow2.service \
+        package/goflow2.env=/etc/default/goflow2

README.md

@@ -0,0 +1,216 @@
+# GoFlow2
+
+[![Build Status](https://github.com/netsampler/goflow2/workflows/Build/badge.svg)](https://github.com/netsampler/goflow2/actions?query=workflow%3ABuild)
+[![Go Reference](https://pkg.go.dev/badge/github.com/netsampler/goflow2.svg)](https://pkg.go.dev/github.com/netsampler/goflow2)
+
+This application is a NetFlow/IPFIX/sFlow collector in Go.
+
+It gathers network information (IP, interfaces, routers) from different flow protocols,
+serializes it in a common format.
+
+You will want to use GoFlow if:
+* You receive a decent amount of network samples and need horizontal scalability
+* Have protocol diversity and need a consistent format
+* Require raw samples and build aggregation and custom enrichment
+
+This software is the entry point of a pipeline. The storage, transport, enrichment, graphing, alerting are
+not provided.
+
+![GoFlow2 System diagram](/graphics/diagram.png)
+
+## Origins
+
+This work is a fork of a previous [open-source GoFlow code](https://github.com/cloudflare/goflow) built and used at Cloudflare.
+It lives in its own GitHub organization to be maintained more easily.
+
+Among the differences with the original code:
+The serializer and transport options have been revamped to make this program more user friendly.
+and target new use-cases like logging providers.
+Minimal changes in the decoding libraries.
+
+## Modularity
+
+In order to enable load-balancing and optimizations, the GoFlow library has a `decoder` which converts
+the payload of a flow packet into a Go structure.
+
+The `producer` functions (one per protocol) then converts those structures into a protobuf (`pb/flow.pb`)
+which contains the fields a network engineer is interested in.
+The flow packets usually contains multiples samples
+This acts as an abstraction of a sample.
+
+The `format` directory offers various utilities to process the protobuf. It can convert
+
+The `transport` provides different way of processing the protobuf. Either sending it via Kafka or 
+send it to a file (or stdout).
+
+GoFlow2 is a wrapper of all the functions and chains thems.
+
+You can build your own collector using this base and replace parts:
+* Use different transport (eg: RabbitMQ instead of Kafka)
+* Convert to another format (eg: Cap'n Proto, Avro, instead of protobuf)
+* Decode different samples (eg: not only IP networks, add MPLS)
+* Different metrics system (eg: [OpenTelemetry](https://opentelemetry.io/))
+
+### Protocol difference
+
+The sampling protocols have distinct features:
+
+**sFlow** is a stateless protocol which sends the full header of a packet with router information
+(interfaces, destination AS) while **NetFlow/IPFIX** rely on templates that contain fields (eg: source IPv6).
+
+The sampling rate in NetFlow/IPFIX is provided by **Option Data Sets**. This is why it can take a few minutes
+for the packets to be decoded until all the templates are received (**Option Template** and **Data Template**).
+
+Both of these protocols bundle multiple samples (**Data Set** in NetFlow/IPFIX and **Flow Sample** in sFlow)
+in one packet.
+
+The advantages of using an abstract network flow format, such as protobuf, is it enables summing over the
+protocols (eg: per ASN or per port, rather than per (ASN, router) and (port, router)).
+
+To read more about the protocols and how they are mapped inside, check out [page](/docs/protocols.md)
+
+### Features of GoFlow2
+
+Collection:
+* NetFlow v5
+* IPFIX/NetFlow v9 (sampling rate provided by the Option Data Set)
+* sFlow v5
+
+(adding NetFlow v1,7,8 is being evaluated)
+
+Production:
+* Convert to protobuf or json
+* Prints to the console/file
+* Sends to Kafka and partition
+
+Monitoring via Prometheus metrics
+
+## Get started
+
+To read about agents that samples network traffic, check this [page](/docs/agents.md).
+
+To setup the collector, download the latest release corresponding to your OS
+and run the following command (the binaries have a suffix with the version):
+
+```bash
+$ ./goflow2
+```
+
+By default, this command will launch an sFlow collector on port `:6343` and
+a NetFlowV9/IPFIX collector on port `:2055`.
+
+By default, the samples received will be printed in JSON format on the stdout.
+
+```json
+{
+  "Type": "SFLOW_5",
+  "TimeFlowEnd": 1621820000,
+  "TimeFlowStart": 1621820000,
+  "TimeReceived": 1621820000,
+  "Bytes": 70,
+  "Packets": 1,
+  "SamplingRate": 100,
+  "SamplerAddress": "192.168.1.254",
+  "DstAddr": "10.0.0.1",
+  "DstMac": "ff:ff:ff:ff:ff:ff",
+  "SrcAddr": "192.168.1.1",
+  "SrcMac": "ff:ff:ff:ff:ff:ff",
+  "InIf": 1,
+  "OutIf": 2,
+  "Etype": 2048,
+  "EtypeName": "IPv4",
+  "Proto": 6,
+  "ProtoName": "TCP",
+  "SrcPort": 443,
+  "DstPort": 46344,
+  "FragmentId": 54044,
+  "FragmentOffset": 16384,
+  ...
+  "IPTTL": 64,
+  "IPTos": 0,
+  "TCPFlags": 16,
+}
+```
+
+If you are using a log integration (eg: Loki with Promtail, Splunk, Fluentd, Google Cloud Logs, etc.),
+just send the output into a file.
+```bash
+$ ./goflow2 -transport.file /var/logs/goflow2.log
+```
+
+To enable Kafka and send protobuf, use the following arguments:
+```bash
+$ ./goflow2 -transport=kafka -transport.kafka.brokers=localhost:9092 -transport.kafka.topic=flows -format=pb
+```
+
+By default, the distribution will be randomized.
+To partition the feed (any field of the protobuf is available), the following options can be used:
+```
+-transport.kafka.hashing=true \
+-format.hash=SamplerAddress,DstAS
+```
+
+### Docker
+
+You can also run directly with a container:
+```
+$ sudo docker run -p 6343:6343/udp -p 2055:2055/udp -ti netsampler/goflow2:latest
+```
+
+### Output format considerations
+
+The JSON format is advised only when consuming a small amount of data directly.
+For bigger workloads, the protobuf output format provides a binary representation
+and is preferred.
+It can also be extended wtih enrichment as long as the user keep the same IDs.
+
+If you want to develop applications, build `pb/flow.proto` into the language you want:
+When adding custom fields, picking a field ID ≥ 1000 is suggested.
+
+You can compile the protobuf using the Makefile for Go.
+```
+make proto
+```
+
+For compiling the protobuf for other languages, refer to the [official guide](https://developers.google.com/protocol-buffers).
+
+## Flow Pipeline
+
+A basic enrichment tool is available in the `cmd/enricher` directory.
+You need to load the Maxmind GeoIP ASN and Country databases using `-db.asn` and `-db.country`.
+
+Running a flow enrichment system is as simple as a pipe.
+Once you plug the stdin of the enricher to the stdout of GoFlow in protobuf,
+the source and destination IP addresses will automatically be mapped 
+with a database for Autonomous System Number and Country.
+Similar output options as GoFlow are provided.
+
+```bash
+$ ./goflow2 -format=pb | ./enricher -db.asn path-to/GeoLite2-ASN.mmdb -db.country path-to/GeoLite2-Country.mmdb
+```
+
+For a more scalable production setting, Kafka and protobuf are recommended.
+Stream operations (aggregation and filtering) can be done with stream-processor tools.
+For instance Flink, or the more recent Kafka Streams and kSQLdb.
+Direct storage can be done with [Clickhouse](/docs/clickhouse.md). This database can also create materialized tables.
+
+In some cases, the consumer will require protobuf messages to be prefixed by
+length. To do this, use the flag `-format.protobuf.fixedlen=true`.
+
+## User stories
+
+Are you using GoFlow2 in production at scale? Add yourself here!
+
+### Contributions
+
+This project welcomes pull-requests, wether it's documentation,
+instrumentation (eg: docker-compose, metrics), internals (protocol libraries),
+integration (new CLI feature) or else!
+Just make sure to check for the use-cases via an issue.
+
+This software would not exist without the testing and commits from
+its users and [contributors](docs/contributors.md).
+
+## License
+
+Licensed under the BSD-3 License.

cmd/enricher/main.go

@@ -0,0 +1,177 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/oschwald/geoip2-golang"
+
+	"github.com/golang/protobuf/proto"
+	flowmessage "github.com/netsampler/goflow2/cmd/enricher/pb"
+
+	// import various formatters
+	"github.com/netsampler/goflow2/format"
+	"github.com/netsampler/goflow2/format/json"
+	_ "github.com/netsampler/goflow2/format/protobuf"
+
+	// import various transports
+	"github.com/netsampler/goflow2/transport"
+	_ "github.com/netsampler/goflow2/transport/file"
+	_ "github.com/netsampler/goflow2/transport/kafka"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	log "github.com/sirupsen/logrus"
+)
+
+var (
+	version    = ""
+	buildinfos = ""
+	AppVersion = "Enricher " + version + " " + buildinfos
+
+	DbAsn     = flag.String("db.asn", "", "IP->ASN database")
+	DbCountry = flag.String("db.country", "", "IP->Country database")
+
+	LogLevel = flag.String("loglevel", "info", "Log level")
+	LogFmt   = flag.String("logfmt", "normal", "Log formatter")
+
+	Format    = flag.String("format", "json", fmt.Sprintf("Choose the format (available: %s)", strings.Join(format.GetFormats(), ", ")))
+	Transport = flag.String("transport", "file", fmt.Sprintf("Choose the transport (available: %s)", strings.Join(transport.GetTransports(), ", ")))
+
+	MetricsAddr = flag.String("metrics.addr", ":8081", "Metrics address")
+	MetricsPath = flag.String("metrics.path", "/metrics", "Metrics path")
+
+	TemplatePath = flag.String("templates.path", "/templates", "NetFlow/IPFIX templates list")
+
+	Version = flag.Bool("v", false, "Print version")
+)
+
+func httpServer() {
+	http.Handle(*MetricsPath, promhttp.Handler())
+	log.Fatal(http.ListenAndServe(*MetricsAddr, nil))
+}
+
+func MapAsn(db *geoip2.Reader, addr []byte, dest *uint32) {
+	entry, err := db.ASN(net.IP(addr))
+	if err != nil {
+		return
+	}
+	*dest = uint32(entry.AutonomousSystemNumber)
+}
+func MapCountry(db *geoip2.Reader, addr []byte, dest *string) {
+	entry, err := db.Country(net.IP(addr))
+	if err != nil {
+		return
+	}
+	*dest = entry.Country.IsoCode
+}
+
+func MapFlow(dbAsn, dbCountry *geoip2.Reader, msg *flowmessage.FlowMessageExt) {
+	if dbAsn != nil {
+		MapAsn(dbAsn, msg.SrcAddr, &(msg.SrcAS))
+		MapAsn(dbAsn, msg.DstAddr, &(msg.DstAS))
+	}
+	if dbCountry != nil {
+		MapCountry(dbCountry, msg.SrcAddr, &(msg.SrcCountry))
+		MapCountry(dbCountry, msg.DstAddr, &(msg.DstCountry))
+	}
+}
+
+func init() {
+	json.AddJSONField("SrcCountry", json.FORMAT_TYPE_STRING)
+	json.AddJSONField("DstCountry", json.FORMAT_TYPE_STRING)
+}
+
+func main() {
+	flag.Parse()
+
+	if *Version {
+		fmt.Println(AppVersion)
+		os.Exit(0)
+	}
+
+	lvl, _ := log.ParseLevel(*LogLevel)
+	log.SetLevel(lvl)
+
+	var dbAsn, dbCountry *geoip2.Reader
+	var err error
+	if *DbAsn != "" {
+		dbAsn, err = geoip2.Open(*DbAsn)
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer dbAsn.Close()
+	}
+
+	if *DbCountry != "" {
+		dbCountry, err = geoip2.Open(*DbCountry)
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer dbCountry.Close()
+	}
+
+	ctx := context.Background()
+
+	formatter, err := format.FindFormat(ctx, *Format)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	transporter, err := transport.FindTransport(ctx, *Transport)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer transporter.Close(ctx)
+
+	switch *LogFmt {
+	case "json":
+		log.SetFormatter(&log.JSONFormatter{})
+	}
+
+	log.Info("Starting enricher")
+
+	go httpServer()
+
+	rdr := bufio.NewReader(os.Stdin)
+
+	msg := &flowmessage.FlowMessageExt{}
+	for {
+		line, err := rdr.ReadBytes('\n')
+		if err != nil && err != io.EOF {
+			log.Error(err)
+			continue
+		}
+		if len(line) == 0 {
+			continue
+		}
+		line = bytes.TrimSuffix(line, []byte("\n"))
+
+		err = proto.Unmarshal(line, msg)
+		if err != nil {
+			log.Error(err)
+			continue
+		}
+
+		MapFlow(dbAsn, dbCountry, msg)
+
+		key, data, err := formatter.Format(msg)
+		if err != nil {
+			log.Error(err)
+			continue
+		}
+
+		err = transporter.Send(key, data)
+		if err != nil {
+			log.Error(err)
+			continue
+		}
+	}
+}

cmd/enricher/pb/flowext.pb.go

@@ -0,0 +1,742 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.15.0
+// source: cmd/enricher/pb/flowext.proto
+
+package flowpb
+
+import (
+	proto "github.com/golang/protobuf/proto"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
+
+type FlowMessageExt_FlowType int32
+
+const (
+	FlowMessageExt_FLOWUNKNOWN FlowMessageExt_FlowType = 0
+	FlowMessageExt_SFLOW_5     FlowMessageExt_FlowType = 1
+	FlowMessageExt_NETFLOW_V5  FlowMessageExt_FlowType = 2
+	FlowMessageExt_NETFLOW_V9  FlowMessageExt_FlowType = 3
+	FlowMessageExt_IPFIX       FlowMessageExt_FlowType = 4
+)
+
+// Enum value maps for FlowMessageExt_FlowType.
+var (
+	FlowMessageExt_FlowType_name = map[int32]string{
+		0: "FLOWUNKNOWN",
+		1: "SFLOW_5",
+		2: "NETFLOW_V5",
+		3: "NETFLOW_V9",
+		4: "IPFIX",
+	}
+	FlowMessageExt_FlowType_value = map[string]int32{
+		"FLOWUNKNOWN": 0,
+		"SFLOW_5":     1,
+		"NETFLOW_V5":  2,
+		"NETFLOW_V9":  3,
+		"IPFIX":       4,
+	}
+)
+
+func (x FlowMessageExt_FlowType) Enum() *FlowMessageExt_FlowType {
+	p := new(FlowMessageExt_FlowType)
+	*p = x
+	return p
+}
+
+func (x FlowMessageExt_FlowType) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (FlowMessageExt_FlowType) Descriptor() protoreflect.EnumDescriptor {
+	return file_cmd_enricher_pb_flowext_proto_enumTypes[0].Descriptor()
+}
+
+func (FlowMessageExt_FlowType) Type() protoreflect.EnumType {
+	return &file_cmd_enricher_pb_flowext_proto_enumTypes[0]
+}
+
+func (x FlowMessageExt_FlowType) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use FlowMessageExt_FlowType.Descriptor instead.
+func (FlowMessageExt_FlowType) EnumDescriptor() ([]byte, []int) {
+	return file_cmd_enricher_pb_flowext_proto_rawDescGZIP(), []int{0, 0}
+}
+
+type FlowMessageExt struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Type          FlowMessageExt_FlowType `protobuf:"varint,1,opt,name=Type,proto3,enum=flowpb.FlowMessageExt_FlowType" json:"Type,omitempty"`
+	TimeReceived  uint64                  `protobuf:"varint,2,opt,name=TimeReceived,proto3" json:"TimeReceived,omitempty"`
+	SequenceNum   uint32                  `protobuf:"varint,4,opt,name=SequenceNum,proto3" json:"SequenceNum,omitempty"`
+	SamplingRate  uint64                  `protobuf:"varint,3,opt,name=SamplingRate,proto3" json:"SamplingRate,omitempty"`
+	FlowDirection uint32                  `protobuf:"varint,42,opt,name=FlowDirection,proto3" json:"FlowDirection,omitempty"`
+	// Sampler information
+	SamplerAddress []byte `protobuf:"bytes,11,opt,name=SamplerAddress,proto3" json:"SamplerAddress,omitempty"`
+	// Found inside packet
+	TimeFlowStart uint64 `protobuf:"varint,38,opt,name=TimeFlowStart,proto3" json:"TimeFlowStart,omitempty"`
+	TimeFlowEnd   uint64 `protobuf:"varint,5,opt,name=TimeFlowEnd,proto3" json:"TimeFlowEnd,omitempty"`
+	// Size of the sampled packet
+	Bytes   uint64 `protobuf:"varint,9,opt,name=Bytes,proto3" json:"Bytes,omitempty"`
+	Packets uint64 `protobuf:"varint,10,opt,name=Packets,proto3" json:"Packets,omitempty"`
+	// Source/destination addresses
+	SrcAddr []byte `protobuf:"bytes,6,opt,name=SrcAddr,proto3" json:"SrcAddr,omitempty"`
+	DstAddr []byte `protobuf:"bytes,7,opt,name=DstAddr,proto3" json:"DstAddr,omitempty"`
+	// Layer 3 protocol (IPv4/IPv6/ARP/MPLS...)
+	Etype uint32 `protobuf:"varint,30,opt,name=Etype,proto3" json:"Etype,omitempty"`
+	// Layer 4 protocol
+	Proto uint32 `protobuf:"varint,20,opt,name=Proto,proto3" json:"Proto,omitempty"`
+	// Ports for UDP and TCP
+	SrcPort uint32 `protobuf:"varint,21,opt,name=SrcPort,proto3" json:"SrcPort,omitempty"`
+	DstPort uint32 `protobuf:"varint,22,opt,name=DstPort,proto3" json:"DstPort,omitempty"`
+	// Interfaces
+	InIf  uint32 `protobuf:"varint,18,opt,name=InIf,proto3" json:"InIf,omitempty"`
+	OutIf uint32 `protobuf:"varint,19,opt,name=OutIf,proto3" json:"OutIf,omitempty"`
+	// Ethernet information
+	SrcMac uint64 `protobuf:"varint,27,opt,name=SrcMac,proto3" json:"SrcMac,omitempty"`
+	DstMac uint64 `protobuf:"varint,28,opt,name=DstMac,proto3" json:"DstMac,omitempty"`
+	// Vlan
+	SrcVlan uint32 `protobuf:"varint,33,opt,name=SrcVlan,proto3" json:"SrcVlan,omitempty"`
+	DstVlan uint32 `protobuf:"varint,34,opt,name=DstVlan,proto3" json:"DstVlan,omitempty"`
+	// 802.1q VLAN in sampled packet
+	VlanId uint32 `protobuf:"varint,29,opt,name=VlanId,proto3" json:"VlanId,omitempty"`
+	// VRF
+	IngressVrfID uint32 `protobuf:"varint,39,opt,name=IngressVrfID,proto3" json:"IngressVrfID,omitempty"`
+	EgressVrfID  uint32 `protobuf:"varint,40,opt,name=EgressVrfID,proto3" json:"EgressVrfID,omitempty"`
+	// IP and TCP special flags
+	IPTos            uint32 `protobuf:"varint,23,opt,name=IPTos,proto3" json:"IPTos,omitempty"`
+	ForwardingStatus uint32 `protobuf:"varint,24,opt,name=ForwardingStatus,proto3" json:"ForwardingStatus,omitempty"`
+	IPTTL            uint32 `protobuf:"varint,25,opt,name=IPTTL,proto3" json:"IPTTL,omitempty"`
+	TCPFlags         uint32 `protobuf:"varint,26,opt,name=TCPFlags,proto3" json:"TCPFlags,omitempty"`
+	IcmpType         uint32 `protobuf:"varint,31,opt,name=IcmpType,proto3" json:"IcmpType,omitempty"`
+	IcmpCode         uint32 `protobuf:"varint,32,opt,name=IcmpCode,proto3" json:"IcmpCode,omitempty"`
+	IPv6FlowLabel    uint32 `protobuf:"varint,37,opt,name=IPv6FlowLabel,proto3" json:"IPv6FlowLabel,omitempty"`
+	// Fragments (IPv4/IPv6)
+	FragmentId      uint32 `protobuf:"varint,35,opt,name=FragmentId,proto3" json:"FragmentId,omitempty"`
+	FragmentOffset  uint32 `protobuf:"varint,36,opt,name=FragmentOffset,proto3" json:"FragmentOffset,omitempty"`
+	BiFlowDirection uint32 `protobuf:"varint,41,opt,name=BiFlowDirection,proto3" json:"BiFlowDirection,omitempty"`
+	// Autonomous system information
+	SrcAS     uint32 `protobuf:"varint,14,opt,name=SrcAS,proto3" json:"SrcAS,omitempty"`
+	DstAS     uint32 `protobuf:"varint,15,opt,name=DstAS,proto3" json:"DstAS,omitempty"`
+	NextHop   []byte `protobuf:"bytes,12,opt,name=NextHop,proto3" json:"NextHop,omitempty"`
+	NextHopAS uint32 `protobuf:"varint,13,opt,name=NextHopAS,proto3" json:"NextHopAS,omitempty"`
+	// Prefix size
+	SrcNet uint32 `protobuf:"varint,16,opt,name=SrcNet,proto3" json:"SrcNet,omitempty"`
+	DstNet uint32 `protobuf:"varint,17,opt,name=DstNet,proto3" json:"DstNet,omitempty"`
+	// MPLS information
+	HasMPLS       bool   `protobuf:"varint,53,opt,name=HasMPLS,proto3" json:"HasMPLS,omitempty"`
+	MPLSCount     uint32 `protobuf:"varint,54,opt,name=MPLSCount,proto3" json:"MPLSCount,omitempty"`
+	MPLS1TTL      uint32 `protobuf:"varint,55,opt,name=MPLS1TTL,proto3" json:"MPLS1TTL,omitempty"`           // First TTL
+	MPLS1Label    uint32 `protobuf:"varint,56,opt,name=MPLS1Label,proto3" json:"MPLS1Label,omitempty"`       // First Label
+	MPLS2TTL      uint32 `protobuf:"varint,57,opt,name=MPLS2TTL,proto3" json:"MPLS2TTL,omitempty"`           // Second TTL
+	MPLS2Label    uint32 `protobuf:"varint,58,opt,name=MPLS2Label,proto3" json:"MPLS2Label,omitempty"`       // Second Label
+	MPLS3TTL      uint32 `protobuf:"varint,59,opt,name=MPLS3TTL,proto3" json:"MPLS3TTL,omitempty"`           // Third TTL
+	MPLS3Label    uint32 `protobuf:"varint,60,opt,name=MPLS3Label,proto3" json:"MPLS3Label,omitempty"`       // Third Label
+	MPLSLastTTL   uint32 `protobuf:"varint,61,opt,name=MPLSLastTTL,proto3" json:"MPLSLastTTL,omitempty"`     // Last TTL
+	MPLSLastLabel uint32 `protobuf:"varint,62,opt,name=MPLSLastLabel,proto3" json:"MPLSLastLabel,omitempty"` // Last Label
+	SrcCountry    string `protobuf:"bytes,1000,opt,name=SrcCountry,proto3" json:"SrcCountry,omitempty"`
+	DstCountry    string `protobuf:"bytes,1001,opt,name=DstCountry,proto3" json:"DstCountry,omitempty"`
+}
+
+func (x *FlowMessageExt) Reset() {
+	*x = FlowMessageExt{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_cmd_enricher_pb_flowext_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *FlowMessageExt) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*FlowMessageExt) ProtoMessage() {}
+
+func (x *FlowMessageExt) ProtoReflect() protoreflect.Message {
+	mi := &file_cmd_enricher_pb_flowext_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use FlowMessageExt.ProtoReflect.Descriptor instead.
+func (*FlowMessageExt) Descriptor() ([]byte, []int) {
+	return file_cmd_enricher_pb_flowext_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *FlowMessageExt) GetType() FlowMessageExt_FlowType {
+	if x != nil {
+		return x.Type
+	}
+	return FlowMessageExt_FLOWUNKNOWN
+}
+
+func (x *FlowMessageExt) GetTimeReceived() uint64 {
+	if x != nil {
+		return x.TimeReceived
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetSequenceNum() uint32 {
+	if x != nil {
+		return x.SequenceNum
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetSamplingRate() uint64 {
+	if x != nil {
+		return x.SamplingRate
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetFlowDirection() uint32 {
+	if x != nil {
+		return x.FlowDirection
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetSamplerAddress() []byte {
+	if x != nil {
+		return x.SamplerAddress
+	}
+	return nil
+}
+
+func (x *FlowMessageExt) GetTimeFlowStart() uint64 {
+	if x != nil {
+		return x.TimeFlowStart
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetTimeFlowEnd() uint64 {
+	if x != nil {
+		return x.TimeFlowEnd
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetBytes() uint64 {
+	if x != nil {
+		return x.Bytes
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetPackets() uint64 {
+	if x != nil {
+		return x.Packets
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetSrcAddr() []byte {
+	if x != nil {
+		return x.SrcAddr
+	}
+	return nil
+}
+
+func (x *FlowMessageExt) GetDstAddr() []byte {
+	if x != nil {
+		return x.DstAddr
+	}
+	return nil
+}
+
+func (x *FlowMessageExt) GetEtype() uint32 {
+	if x != nil {
+		return x.Etype
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetProto() uint32 {
+	if x != nil {
+		return x.Proto
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetSrcPort() uint32 {
+	if x != nil {
+		return x.SrcPort
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetDstPort() uint32 {
+	if x != nil {
+		return x.DstPort
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetInIf() uint32 {
+	if x != nil {
+		return x.InIf
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetOutIf() uint32 {
+	if x != nil {
+		return x.OutIf
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetSrcMac() uint64 {
+	if x != nil {
+		return x.SrcMac
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetDstMac() uint64 {
+	if x != nil {
+		return x.DstMac
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetSrcVlan() uint32 {
+	if x != nil {
+		return x.SrcVlan
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetDstVlan() uint32 {
+	if x != nil {
+		return x.DstVlan
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetVlanId() uint32 {
+	if x != nil {
+		return x.VlanId
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetIngressVrfID() uint32 {
+	if x != nil {
+		return x.IngressVrfID
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetEgressVrfID() uint32 {
+	if x != nil {
+		return x.EgressVrfID
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetIPTos() uint32 {
+	if x != nil {
+		return x.IPTos
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetForwardingStatus() uint32 {
+	if x != nil {
+		return x.ForwardingStatus
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetIPTTL() uint32 {
+	if x != nil {
+		return x.IPTTL
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetTCPFlags() uint32 {
+	if x != nil {
+		return x.TCPFlags
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetIcmpType() uint32 {
+	if x != nil {
+		return x.IcmpType
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetIcmpCode() uint32 {
+	if x != nil {
+		return x.IcmpCode
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetIPv6FlowLabel() uint32 {
+	if x != nil {
+		return x.IPv6FlowLabel
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetFragmentId() uint32 {
+	if x != nil {
+		return x.FragmentId
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetFragmentOffset() uint32 {
+	if x != nil {
+		return x.FragmentOffset
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetBiFlowDirection() uint32 {
+	if x != nil {
+		return x.BiFlowDirection
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetSrcAS() uint32 {
+	if x != nil {
+		return x.SrcAS
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetDstAS() uint32 {
+	if x != nil {
+		return x.DstAS
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetNextHop() []byte {
+	if x != nil {
+		return x.NextHop
+	}
+	return nil
+}
+
+func (x *FlowMessageExt) GetNextHopAS() uint32 {
+	if x != nil {
+		return x.NextHopAS
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetSrcNet() uint32 {
+	if x != nil {
+		return x.SrcNet
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetDstNet() uint32 {
+	if x != nil {
+		return x.DstNet
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetHasMPLS() bool {
+	if x != nil {
+		return x.HasMPLS
+	}
+	return false
+}
+
+func (x *FlowMessageExt) GetMPLSCount() uint32 {
+	if x != nil {
+		return x.MPLSCount
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetMPLS1TTL() uint32 {
+	if x != nil {
+		return x.MPLS1TTL
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetMPLS1Label() uint32 {
+	if x != nil {
+		return x.MPLS1Label
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetMPLS2TTL() uint32 {
+	if x != nil {
+		return x.MPLS2TTL
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetMPLS2Label() uint32 {
+	if x != nil {
+		return x.MPLS2Label
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetMPLS3TTL() uint32 {
+	if x != nil {
+		return x.MPLS3TTL
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetMPLS3Label() uint32 {
+	if x != nil {
+		return x.MPLS3Label
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetMPLSLastTTL() uint32 {
+	if x != nil {
+		return x.MPLSLastTTL
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetMPLSLastLabel() uint32 {
+	if x != nil {
+		return x.MPLSLastLabel
+	}
+	return 0
+}
+
+func (x *FlowMessageExt) GetSrcCountry() string {
+	if x != nil {
+		return x.SrcCountry
+	}
+	return ""
+}
+
+func (x *FlowMessageExt) GetDstCountry() string {
+	if x != nil {
+		return x.DstCountry
+	}
+	return ""
+}
+
+var File_cmd_enricher_pb_flowext_proto protoreflect.FileDescriptor
+
+var file_cmd_enricher_pb_flowext_proto_rawDesc = []byte{
+	0x0a, 0x1d, 0x63, 0x6d, 0x64, 0x2f, 0x65, 0x6e, 0x72, 0x69, 0x63, 0x68, 0x65, 0x72, 0x2f, 0x70,
+	0x62, 0x2f, 0x66, 0x6c, 0x6f, 0x77, 0x65, 0x78, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12,
+	0x06, 0x66, 0x6c, 0x6f, 0x77, 0x70, 0x62, 0x22, 0x98, 0x0d, 0x0a, 0x0e, 0x46, 0x6c, 0x6f, 0x77,
+	0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x45, 0x78, 0x74, 0x12, 0x33, 0x0a, 0x04, 0x54, 0x79,
+	0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1f, 0x2e, 0x66, 0x6c, 0x6f, 0x77, 0x70,
+	0x62, 0x2e, 0x46, 0x6c, 0x6f, 0x77, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x45, 0x78, 0x74,
+	0x2e, 0x46, 0x6c, 0x6f, 0x77, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12,
+	0x22, 0x0a, 0x0c, 0x54, 0x69, 0x6d, 0x65, 0x52, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x64, 0x18,
+	0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0c, 0x54, 0x69, 0x6d, 0x65, 0x52, 0x65, 0x63, 0x65, 0x69,
+	0x76, 0x65, 0x64, 0x12, 0x20, 0x0a, 0x0b, 0x53, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x4e,
+	0x75, 0x6d, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x53, 0x65, 0x71, 0x75, 0x65, 0x6e,
+	0x63, 0x65, 0x4e, 0x75, 0x6d, 0x12, 0x22, 0x0a, 0x0c, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x69, 0x6e,
+	0x67, 0x52, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0c, 0x53, 0x61, 0x6d,
+	0x70, 0x6c, 0x69, 0x6e, 0x67, 0x52, 0x61, 0x74, 0x65, 0x12, 0x24, 0x0a, 0x0d, 0x46, 0x6c, 0x6f,
+	0x77, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x2a, 0x20, 0x01, 0x28, 0x0d,
+	0x52, 0x0d, 0x46, 0x6c, 0x6f, 0x77, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12,
+	0x26, 0x0a, 0x0e, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73,
+	0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0e, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x72,
+	0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x54, 0x69, 0x6d, 0x65, 0x46,
+	0x6c, 0x6f, 0x77, 0x53, 0x74, 0x61, 0x72, 0x74, 0x18, 0x26, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0d,
+	0x54, 0x69, 0x6d, 0x65, 0x46, 0x6c, 0x6f, 0x77, 0x53, 0x74, 0x61, 0x72, 0x74, 0x12, 0x20, 0x0a,
+	0x0b, 0x54, 0x69, 0x6d, 0x65, 0x46, 0x6c, 0x6f, 0x77, 0x45, 0x6e, 0x64, 0x18, 0x05, 0x20, 0x01,
+	0x28, 0x04, 0x52, 0x0b, 0x54, 0x69, 0x6d, 0x65, 0x46, 0x6c, 0x6f, 0x77, 0x45, 0x6e, 0x64, 0x12,
+	0x14, 0x0a, 0x05, 0x42, 0x79, 0x74, 0x65, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05,
+	0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73,
+	0x18, 0x0a, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12,
+	0x18, 0x0a, 0x07, 0x53, 0x72, 0x63, 0x41, 0x64, 0x64, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c,
+	0x52, 0x07, 0x53, 0x72, 0x63, 0x41, 0x64, 0x64, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x44, 0x73, 0x74,
+	0x41, 0x64, 0x64, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x44, 0x73, 0x74, 0x41,
+	0x64, 0x64, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x45, 0x74, 0x79, 0x70, 0x65, 0x18, 0x1e, 0x20, 0x01,
+	0x28, 0x0d, 0x52, 0x05, 0x45, 0x74, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x50, 0x72, 0x6f,
+	0x74, 0x6f, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x12,
+	0x18, 0x0a, 0x07, 0x53, 0x72, 0x63, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x15, 0x20, 0x01, 0x28, 0x0d,
+	0x52, 0x07, 0x53, 0x72, 0x63, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x44, 0x73, 0x74,
+	0x50, 0x6f, 0x72, 0x74, 0x18, 0x16, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x44, 0x73, 0x74, 0x50,
+	0x6f, 0x72, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x49, 0x6e, 0x49, 0x66, 0x18, 0x12, 0x20, 0x01, 0x28,
+	0x0d, 0x52, 0x04, 0x49, 0x6e, 0x49, 0x66, 0x12, 0x14, 0x0a, 0x05, 0x4f, 0x75, 0x74, 0x49, 0x66,
+	0x18, 0x13, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x4f, 0x75, 0x74, 0x49, 0x66, 0x12, 0x16, 0x0a,
+	0x06, 0x53, 0x72, 0x63, 0x4d, 0x61, 0x63, 0x18, 0x1b, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x53,
+	0x72, 0x63, 0x4d, 0x61, 0x63, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x73, 0x74, 0x4d, 0x61, 0x63, 0x18,
+	0x1c, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x44, 0x73, 0x74, 0x4d, 0x61, 0x63, 0x12, 0x18, 0x0a,
+	0x07, 0x53, 0x72, 0x63, 0x56, 0x6c, 0x61, 0x6e, 0x18, 0x21, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07,
+	0x53, 0x72, 0x63, 0x56, 0x6c, 0x61, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x44, 0x73, 0x74, 0x56, 0x6c,
+	0x61, 0x6e, 0x18, 0x22, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x44, 0x73, 0x74, 0x56, 0x6c, 0x61,
+	0x6e, 0x12, 0x16, 0x0a, 0x06, 0x56, 0x6c, 0x61, 0x6e, 0x49, 0x64, 0x18, 0x1d, 0x20, 0x01, 0x28,
+	0x0d, 0x52, 0x06, 0x56, 0x6c, 0x61, 0x6e, 0x49, 0x64, 0x12, 0x22, 0x0a, 0x0c, 0x49, 0x6e, 0x67,
+	0x72, 0x65, 0x73, 0x73, 0x56, 0x72, 0x66, 0x49, 0x44, 0x18, 0x27, 0x20, 0x01, 0x28, 0x0d, 0x52,
+	0x0c, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x56, 0x72, 0x66, 0x49, 0x44, 0x12, 0x20, 0x0a,
+	0x0b, 0x45, 0x67, 0x72, 0x65, 0x73, 0x73, 0x56, 0x72, 0x66, 0x49, 0x44, 0x18, 0x28, 0x20, 0x01,
+	0x28, 0x0d, 0x52, 0x0b, 0x45, 0x67, 0x72, 0x65, 0x73, 0x73, 0x56, 0x72, 0x66, 0x49, 0x44, 0x12,
+	0x14, 0x0a, 0x05, 0x49, 0x50, 0x54, 0x6f, 0x73, 0x18, 0x17, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05,
+	0x49, 0x50, 0x54, 0x6f, 0x73, 0x12, 0x2a, 0x0a, 0x10, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64,
+	0x69, 0x6e, 0x67, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x18, 0x20, 0x01, 0x28, 0x0d, 0x52,
+	0x10, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x74, 0x61, 0x74, 0x75,
+	0x73, 0x12, 0x14, 0x0a, 0x05, 0x49, 0x50, 0x54, 0x54, 0x4c, 0x18, 0x19, 0x20, 0x01, 0x28, 0x0d,
+	0x52, 0x05, 0x49, 0x50, 0x54, 0x54, 0x4c, 0x12, 0x1a, 0x0a, 0x08, 0x54, 0x43, 0x50, 0x46, 0x6c,
+	0x61, 0x67, 0x73, 0x18, 0x1a, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x54, 0x43, 0x50, 0x46, 0x6c,
+	0x61, 0x67, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x49, 0x63, 0x6d, 0x70, 0x54, 0x79, 0x70, 0x65, 0x18,
+	0x1f, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x49, 0x63, 0x6d, 0x70, 0x54, 0x79, 0x70, 0x65, 0x12,
+	0x1a, 0x0a, 0x08, 0x49, 0x63, 0x6d, 0x70, 0x43, 0x6f, 0x64, 0x65, 0x18, 0x20, 0x20, 0x01, 0x28,
+	0x0d, 0x52, 0x08, 0x49, 0x63, 0x6d, 0x70, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x24, 0x0a, 0x0d, 0x49,
+	0x50, 0x76, 0x36, 0x46, 0x6c, 0x6f, 0x77, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x18, 0x25, 0x20, 0x01,
+	0x28, 0x0d, 0x52, 0x0d, 0x49, 0x50, 0x76, 0x36, 0x46, 0x6c, 0x6f, 0x77, 0x4c, 0x61, 0x62, 0x65,
+	0x6c, 0x12, 0x1e, 0x0a, 0x0a, 0x46, 0x72, 0x61, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x18,
+	0x23, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x46, 0x72, 0x61, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x49,
+	0x64, 0x12, 0x26, 0x0a, 0x0e, 0x46, 0x72, 0x61, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x4f, 0x66, 0x66,
+	0x73, 0x65, 0x74, 0x18, 0x24, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0e, 0x46, 0x72, 0x61, 0x67, 0x6d,
+	0x65, 0x6e, 0x74, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x28, 0x0a, 0x0f, 0x42, 0x69, 0x46,
+	0x6c, 0x6f, 0x77, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x29, 0x20, 0x01,
+	0x28, 0x0d, 0x52, 0x0f, 0x42, 0x69, 0x46, 0x6c, 0x6f, 0x77, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74,
+	0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x53, 0x72, 0x63, 0x41, 0x53, 0x18, 0x0e, 0x20, 0x01,
+	0x28, 0x0d, 0x52, 0x05, 0x53, 0x72, 0x63, 0x41, 0x53, 0x12, 0x14, 0x0a, 0x05, 0x44, 0x73, 0x74,
+	0x41, 0x53, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x44, 0x73, 0x74, 0x41, 0x53, 0x12,
+	0x18, 0x0a, 0x07, 0x4e, 0x65, 0x78, 0x74, 0x48, 0x6f, 0x70, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0c,
+	0x52, 0x07, 0x4e, 0x65, 0x78, 0x74, 0x48, 0x6f, 0x70, 0x12, 0x1c, 0x0a, 0x09, 0x4e, 0x65, 0x78,
+	0x74, 0x48, 0x6f, 0x70, 0x41, 0x53, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x4e, 0x65,
+	0x78, 0x74, 0x48, 0x6f, 0x70, 0x41, 0x53, 0x12, 0x16, 0x0a, 0x06, 0x53, 0x72, 0x63, 0x4e, 0x65,
+	0x74, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x53, 0x72, 0x63, 0x4e, 0x65, 0x74, 0x12,
+	0x16, 0x0a, 0x06, 0x44, 0x73, 0x74, 0x4e, 0x65, 0x74, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0d, 0x52,
+	0x06, 0x44, 0x73, 0x74, 0x4e, 0x65, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x48, 0x61, 0x73, 0x4d, 0x50,
+	0x4c, 0x53, 0x18, 0x35, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x48, 0x61, 0x73, 0x4d, 0x50, 0x4c,
+	0x53, 0x12, 0x1c, 0x0a, 0x09, 0x4d, 0x50, 0x4c, 0x53, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x36,
+	0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x4d, 0x50, 0x4c, 0x53, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12,
+	0x1a, 0x0a, 0x08, 0x4d, 0x50, 0x4c, 0x53, 0x31, 0x54, 0x54, 0x4c, 0x18, 0x37, 0x20, 0x01, 0x28,
+	0x0d, 0x52, 0x08, 0x4d, 0x50, 0x4c, 0x53, 0x31, 0x54, 0x54, 0x4c, 0x12, 0x1e, 0x0a, 0x0a, 0x4d,
+	0x50, 0x4c, 0x53, 0x31, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x18, 0x38, 0x20, 0x01, 0x28, 0x0d, 0x52,
+	0x0a, 0x4d, 0x50, 0x4c, 0x53, 0x31, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x1a, 0x0a, 0x08, 0x4d,
+	0x50, 0x4c, 0x53, 0x32, 0x54, 0x54, 0x4c, 0x18, 0x39, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x4d,
+	0x50, 0x4c, 0x53, 0x32, 0x54, 0x54, 0x4c, 0x12, 0x1e, 0x0a, 0x0a, 0x4d, 0x50, 0x4c, 0x53, 0x32,
+	0x4c, 0x61, 0x62, 0x65, 0x6c, 0x18, 0x3a, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x4d, 0x50, 0x4c,
+	0x53, 0x32, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x1a, 0x0a, 0x08, 0x4d, 0x50, 0x4c, 0x53, 0x33,
+	0x54, 0x54, 0x4c, 0x18, 0x3b, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x4d, 0x50, 0x4c, 0x53, 0x33,
+	0x54, 0x54, 0x4c, 0x12, 0x1e, 0x0a, 0x0a, 0x4d, 0x50, 0x4c, 0x53, 0x33, 0x4c, 0x61, 0x62, 0x65,
+	0x6c, 0x18, 0x3c, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x4d, 0x50, 0x4c, 0x53, 0x33, 0x4c, 0x61,
+	0x62, 0x65, 0x6c, 0x12, 0x20, 0x0a, 0x0b, 0x4d, 0x50, 0x4c, 0x53, 0x4c, 0x61, 0x73, 0x74, 0x54,
+	0x54, 0x4c, 0x18, 0x3d, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x4d, 0x50, 0x4c, 0x53, 0x4c, 0x61,
+	0x73, 0x74, 0x54, 0x54, 0x4c, 0x12, 0x24, 0x0a, 0x0d, 0x4d, 0x50, 0x4c, 0x53, 0x4c, 0x61, 0x73,
+	0x74, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x18, 0x3e, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, 0x4d, 0x50,
+	0x4c, 0x53, 0x4c, 0x61, 0x73, 0x74, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x1f, 0x0a, 0x0a, 0x53,
+	0x72, 0x63, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x72, 0x79, 0x18, 0xe8, 0x07, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x0a, 0x53, 0x72, 0x63, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x1f, 0x0a, 0x0a,
+	0x44, 0x73, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x72, 0x79, 0x18, 0xe9, 0x07, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x0a, 0x44, 0x73, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x72, 0x79, 0x22, 0x53, 0x0a,
+	0x08, 0x46, 0x6c, 0x6f, 0x77, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x46, 0x4c, 0x4f,
+	0x57, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x46,
+	0x4c, 0x4f, 0x57, 0x5f, 0x35, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x4e, 0x45, 0x54, 0x46, 0x4c,
+	0x4f, 0x57, 0x5f, 0x56, 0x35, 0x10, 0x02, 0x12, 0x0e, 0x0a, 0x0a, 0x4e, 0x45, 0x54, 0x46, 0x4c,
+	0x4f, 0x57, 0x5f, 0x56, 0x39, 0x10, 0x03, 0x12, 0x09, 0x0a, 0x05, 0x49, 0x50, 0x46, 0x49, 0x58,
+	0x10, 0x04, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_cmd_enricher_pb_flowext_proto_rawDescOnce sync.Once
+	file_cmd_enricher_pb_flowext_proto_rawDescData = file_cmd_enricher_pb_flowext_proto_rawDesc
+)
+
+func file_cmd_enricher_pb_flowext_proto_rawDescGZIP() []byte {
+	file_cmd_enricher_pb_flowext_proto_rawDescOnce.Do(func() {
+		file_cmd_enricher_pb_flowext_proto_rawDescData = protoimpl.X.CompressGZIP(file_cmd_enricher_pb_flowext_proto_rawDescData)
+	})
+	return file_cmd_enricher_pb_flowext_proto_rawDescData
+}
+
+var file_cmd_enricher_pb_flowext_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
+var file_cmd_enricher_pb_flowext_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
+var file_cmd_enricher_pb_flowext_proto_goTypes = []interface{}{
+	(FlowMessageExt_FlowType)(0), // 0: flowpb.FlowMessageExt.FlowType
+	(*FlowMessageExt)(nil),       // 1: flowpb.FlowMessageExt
+}
+var file_cmd_enricher_pb_flowext_proto_depIdxs = []int32{
+	0, // 0: flowpb.FlowMessageExt.Type:type_name -> flowpb.FlowMessageExt.FlowType
+	1, // [1:1] is the sub-list for method output_type
+	1, // [1:1] is the sub-list for method input_type
+	1, // [1:1] is the sub-list for extension type_name
+	1, // [1:1] is the sub-list for extension extendee
+	0, // [0:1] is the sub-list for field type_name
+}
+
+func init() { file_cmd_enricher_pb_flowext_proto_init() }
+func file_cmd_enricher_pb_flowext_proto_init() {
+	if File_cmd_enricher_pb_flowext_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_cmd_enricher_pb_flowext_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*FlowMessageExt); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_cmd_enricher_pb_flowext_proto_rawDesc,
+			NumEnums:      1,
+			NumMessages:   1,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_cmd_enricher_pb_flowext_proto_goTypes,
+		DependencyIndexes: file_cmd_enricher_pb_flowext_proto_depIdxs,
+		EnumInfos:         file_cmd_enricher_pb_flowext_proto_enumTypes,
+		MessageInfos:      file_cmd_enricher_pb_flowext_proto_msgTypes,
+	}.Build()
+	File_cmd_enricher_pb_flowext_proto = out.File
+	file_cmd_enricher_pb_flowext_proto_rawDesc = nil
+	file_cmd_enricher_pb_flowext_proto_goTypes = nil
+	file_cmd_enricher_pb_flowext_proto_depIdxs = nil
+}

cmd/enricher/pb/flowext.proto

@@ -0,0 +1,103 @@
+syntax = "proto3";
+package flowpb;
+
+message FlowMessageExt {
+
+  enum FlowType {
+    FLOWUNKNOWN = 0;
+    SFLOW_5 = 1;
+    NETFLOW_V5 = 2;
+    NETFLOW_V9 = 3;
+    IPFIX = 4;
+  }
+  FlowType Type = 1;
+
+  uint64 TimeReceived = 2;
+  uint32 SequenceNum = 4;
+  uint64 SamplingRate = 3;
+
+  uint32 FlowDirection = 42;
+
+  // Sampler information
+  bytes SamplerAddress = 11;
+
+  // Found inside packet
+  uint64 TimeFlowStart = 38;
+  uint64 TimeFlowEnd = 5;
+
+  // Size of the sampled packet
+  uint64 Bytes = 9;
+  uint64 Packets = 10;
+
+  // Source/destination addresses
+  bytes SrcAddr = 6;
+  bytes DstAddr = 7;
+
+  // Layer 3 protocol (IPv4/IPv6/ARP/MPLS...)
+  uint32 Etype = 30;
+
+  // Layer 4 protocol
+  uint32 Proto = 20;
+
+  // Ports for UDP and TCP
+  uint32 SrcPort = 21;
+  uint32 DstPort = 22;
+
+  // Interfaces
+  uint32 InIf = 18;
+  uint32 OutIf = 19;
+
+  // Ethernet information
+  uint64 SrcMac = 27;
+  uint64 DstMac = 28;
+
+  // Vlan
+  uint32 SrcVlan = 33;
+  uint32 DstVlan = 34;
+  // 802.1q VLAN in sampled packet
+  uint32 VlanId = 29;
+
+  // VRF
+  uint32 IngressVrfID = 39;
+  uint32 EgressVrfID = 40;
+
+  // IP and TCP special flags
+  uint32 IPTos = 23;
+  uint32 ForwardingStatus = 24;
+  uint32 IPTTL = 25;
+  uint32 TCPFlags = 26;
+  uint32 IcmpType = 31;
+  uint32 IcmpCode = 32;
+  uint32 IPv6FlowLabel = 37;
+  // Fragments (IPv4/IPv6)
+  uint32 FragmentId = 35;
+  uint32 FragmentOffset = 36;
+  uint32 BiFlowDirection = 41;
+
+  // Autonomous system information
+  uint32 SrcAS = 14;
+  uint32 DstAS = 15;
+
+  bytes NextHop = 12;
+  uint32 NextHopAS = 13;
+
+  // Prefix size
+  uint32 SrcNet = 16;
+  uint32 DstNet = 17;
+
+  // MPLS information
+  bool HasMPLS = 53;
+  uint32 MPLSCount = 54;
+  uint32 MPLS1TTL = 55; // First TTL
+  uint32 MPLS1Label = 56; // First Label
+  uint32 MPLS2TTL = 57; // Second TTL
+  uint32 MPLS2Label = 58; // Second Label
+  uint32 MPLS3TTL = 59; // Third TTL
+  uint32 MPLS3Label = 60; // Third Label
+  uint32 MPLSLastTTL = 61; // Last TTL
+  uint32 MPLSLastLabel = 62; // Last Label
+
+  string SrcCountry = 1000;
+  string DstCountry = 1001;
+
+}

cmd/goflow2/main.go

@@ -0,0 +1,154 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+
+	// import various formatters
+	"github.com/netsampler/goflow2/format"
+	_ "github.com/netsampler/goflow2/format/json"
+	_ "github.com/netsampler/goflow2/format/protobuf"
+
+	// import various transports
+	"github.com/netsampler/goflow2/transport"
+	_ "github.com/netsampler/goflow2/transport/file"
+	_ "github.com/netsampler/goflow2/transport/kafka"
+
+	"github.com/netsampler/goflow2/utils"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	log "github.com/sirupsen/logrus"
+)
+
+var (
+	version    = ""
+	buildinfos = ""
+	AppVersion = "GoFlow2 " + version + " " + buildinfos
+
+	ReusePort       = flag.Bool("reuseport", false, "Enable so_reuseport")
+	ListenAddresses = flag.String("listen", "sflow://:6343,netflow://:2055", "listen addresses")
+
+	Workers  = flag.Int("workers", 1, "Number of workers per collector")
+	LogLevel = flag.String("loglevel", "info", "Log level")
+	LogFmt   = flag.String("logfmt", "normal", "Log formatter")
+
+	Format    = flag.String("format", "json", fmt.Sprintf("Choose the format (available: %s)", strings.Join(format.GetFormats(), ", ")))
+	Transport = flag.String("transport", "file", fmt.Sprintf("Choose the transport (available: %s)", strings.Join(transport.GetTransports(), ", ")))
+	//FixedLength = flag.Bool("proto.fixedlen", false, "Enable fixed length protobuf")
+
+	MetricsAddr = flag.String("metrics.addr", ":8080", "Metrics address")
+	MetricsPath = flag.String("metrics.path", "/metrics", "Metrics path")
+
+	TemplatePath = flag.String("templates.path", "/templates", "NetFlow/IPFIX templates list")
+
+	Version = flag.Bool("v", false, "Print version")
+)
+
+func httpServer( /*state *utils.StateNetFlow*/ ) {
+	http.Handle(*MetricsPath, promhttp.Handler())
+	//http.HandleFunc(*TemplatePath, state.ServeHTTPTemplates)
+	log.Fatal(http.ListenAndServe(*MetricsAddr, nil))
+}
+
+func main() {
+	flag.Parse()
+
+	if *Version {
+		fmt.Println(AppVersion)
+		os.Exit(0)
+	}
+
+	lvl, _ := log.ParseLevel(*LogLevel)
+	log.SetLevel(lvl)
+
+	ctx := context.Background()
+
+	formatter, err := format.FindFormat(ctx, *Format)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	transporter, err := transport.FindTransport(ctx, *Transport)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer transporter.Close(ctx)
+
+	switch *LogFmt {
+	case "json":
+		log.SetFormatter(&log.JSONFormatter{})
+	}
+
+	log.Info("Starting GoFlow2")
+
+	go httpServer()
+	//go httpServer(sNF)
+
+	wg := &sync.WaitGroup{}
+
+	for _, listenAddress := range strings.Split(*ListenAddresses, ",") {
+		wg.Add(1)
+		go func(listenAddress string) {
+			defer wg.Done()
+			listenAddrUrl, err := url.Parse(listenAddress)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			hostname := listenAddrUrl.Hostname()
+			port, err := strconv.ParseUint(listenAddrUrl.Port(), 10, 64)
+			if err != nil {
+				log.Errorf("Port %s could not be converted to integer", listenAddrUrl.Port())
+				return
+			}
+
+			logFields := log.Fields{
+				"scheme":   listenAddrUrl.Scheme,
+				"hostname": hostname,
+				"port":     port,
+			}
+
+			log.WithFields(logFields).Info("Starting collection")
+
+			if listenAddrUrl.Scheme == "sflow" {
+				sSFlow := &utils.StateSFlow{
+					Format:    formatter,
+					Transport: transporter,
+					Logger:    log.StandardLogger(),
+				}
+				err = sSFlow.FlowRoutine(*Workers, hostname, int(port), *ReusePort)
+			} else if listenAddrUrl.Scheme == "netflow" {
+				sNF := &utils.StateNetFlow{
+					Format:    formatter,
+					Transport: transporter,
+					Logger:    log.StandardLogger(),
+				}
+				err = sNF.FlowRoutine(*Workers, hostname, int(port), *ReusePort)
+			} else if listenAddrUrl.Scheme == "nfl" {
+				sNFL := &utils.StateNFLegacy{
+					Format:    formatter,
+					Transport: transporter,
+					Logger:    log.StandardLogger(),
+				}
+				err = sNFL.FlowRoutine(*Workers, hostname, int(port), *ReusePort)
+			} else {
+				log.Errorf("scheme %s does not exist", listenAddrUrl.Scheme)
+				return
+			}
+
+			if err != nil {
+				log.WithFields(logFields).Fatal(err)
+			}
+
+		}(listenAddress)
+
+	}
+
+	wg.Wait()
+}

compose/clickhouse/create.sh

@@ -0,0 +1,125 @@
+#!/bin/bash
+set -e
+
+clickhouse client -n <<-EOSQL
+
+    CREATE DATABASE dictionaries;
+    
+    CREATE DICTIONARY dictionaries.protocols (
+        proto UInt8,
+        name String,
+        description String
+    )
+    PRIMARY KEY proto
+    LAYOUT(FLAT())
+    SOURCE (FILE(path '/var/lib/clickhouse/user_files/protocols.csv' format 'CSVWithNames'))
+    LIFETIME(3600);
+
+    CREATE TABLE IF NOT EXISTS flows
+    (
+        TimeReceived UInt64,
+        TimeFlowStart UInt64,
+
+        SequenceNum UInt32,
+        SamplingRate UInt64,
+        SamplerAddress FixedString(16),
+
+        SrcAddr FixedString(16),
+        DstAddr FixedString(16),
+
+        SrcAS UInt32,
+        DstAS UInt32,
+
+        EType UInt32,
+        Proto UInt32,
+
+        SrcPort UInt32,
+        DstPort UInt32,
+
+        Bytes UInt64,
+        Packets UInt64
+    ) ENGINE = Kafka()
+    SETTINGS
+        kafka_broker_list = 'kafka:9092',
+        kafka_topic_list = 'flows',
+        kafka_group_name = 'clickhouse',
+        kafka_format = 'Protobuf',
+        kafka_schema = './flow.proto:FlowMessage';
+
+    CREATE TABLE IF NOT EXISTS flows_raw
+    (
+        Date Date,
+        TimeReceived DateTime,
+        TimeFlowStart DateTime,
+
+        SequenceNum UInt32,
+        SamplingRate UInt64,
+        SamplerAddress FixedString(16),
+
+        SrcAddr FixedString(16),
+        DstAddr FixedString(16),
+
+        SrcAS UInt32,
+        DstAS UInt32,
+
+        EType UInt32,
+        Proto UInt32,
+
+        SrcPort UInt32,
+        DstPort UInt32,
+
+        Bytes UInt64,
+        Packets UInt64
+    ) ENGINE = MergeTree()
+    PARTITION BY Date
+    ORDER BY TimeReceived;
+
+    CREATE MATERIALIZED VIEW IF NOT EXISTS flows_raw_view TO flows_raw 
+    AS SELECT
+        toDate(TimeReceived) AS Date,
+        *
+       FROM flows;
+
+    CREATE TABLE IF NOT EXISTS flows_5m
+    (
+        Date Date,
+        Timeslot DateTime,
+
+        SrcAS UInt32,
+        DstAS UInt32,
+
+        ETypeMap Nested (
+            EType UInt32,
+            Bytes UInt64,
+            Packets UInt64,
+            Count UInt64
+        ),
+
+        Bytes UInt64,
+        Packets UInt64,
+        Count UInt64
+    ) ENGINE = SummingMergeTree()
+    PARTITION BY Date
+    ORDER BY (Date, Timeslot, SrcAS, DstAS, \`ETypeMap.EType\`);
+
+    CREATE MATERIALIZED VIEW IF NOT EXISTS flows_5m_view TO flows_5m 
+    AS
+        SELECT
+            Date,
+            toStartOfFiveMinute(TimeReceived) AS Timeslot,
+            SrcAS,
+            DstAS,
+
+            [EType] AS \`ETypeMap.EType\`,
+            [Bytes] AS \`ETypeMap.Bytes\`,
+            [Packets] AS \`ETypeMap.Packets\`,
+            [Count] AS \`ETypeMap.Count\`,
+
+            sum(Bytes) AS Bytes,
+            sum(Packets) AS Packets,
+            count() AS Count
+
+        FROM flows_raw
+        GROUP BY Date, Timeslot, SrcAS, DstAS, \`ETypeMap.EType\`;
+        
+EOSQL

compose/clickhouse/protocols.csv

@@ -0,0 +1,30 @@
+Proto,Name,Description
+0,HOPOPT,IPv6 Hop-by-Hop Option
+1,ICMP,Internet Control Message
+2,IGMP,Internet Group Management
+4,IPv4,IPv4 encapsulation
+6,TCP,Transmission Control Protocol
+8,EGP,Exterior Gateway Protocol
+9,IGP,Interior Gateway Protocol
+16,CHAOS,Chaos
+17,UDP,User Datagram Protocol
+27,RDP,Reliable Data Protocol
+41,IPv6,IPv6 encapsulation
+43,IPv6-Route,Routing Header for IPv6
+44,IPv6-Frag,Fragment Header for IPv6
+45,IDRP,Inter-Domain Routing Protocol
+46,RSVP,Reservation Protocol
+47,GRE,Generic Routing Encapsulation
+50,ESP,Encap Security Payload
+51,AH,Authentication Header
+55,MOBILE,IP Mobility
+58,IPv6-ICMP,ICMP for IPv6
+59,IPv6-NoNxt,No Next Header for IPv6
+60,IPv6-Opts,Destination Options for IPv6
+88,EIGRP,EIGRP
+89,OSPFIGP,OSPFIGP
+92,MTP,Multicast Transport Protocol
+94,IPIP,IP-within-IP Encapsulation Protocol
+97,ETHERIP,Ethernet-within-IP Encapsulation
+98,ENCAP,Encapsulation Header
+112,VRRP,Virtual Router Redundancy Protocol

compose/docker-compose.yml

@@ -0,0 +1,64 @@
+version: "3"
+services:
+  zookeeper:
+    image: 'bitnami/zookeeper:latest'
+    ports:
+      - '2181:2181'
+    environment:
+      - ALLOW_ANONYMOUS_LOGIN=yes
+    restart: always
+  kafka:
+    image: 'bitnami/kafka:latest'
+    ports:
+      - '9092:9092'
+    environment:
+      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+      - ALLOW_PLAINTEXT_LISTENER=yes
+      - KAFKA_DELETE_TOPIC_ENABLE=true
+    restart: always
+    depends_on:
+      - zookeeper
+  grafana:
+    build: grafana
+    environment:
+      - GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=vertamedia-clickhouse-datasource
+    ports:
+      - '3000:3000'
+    restart: always
+    volumes:
+      - ./grafana/datasources-ch.yml:/etc/grafana/provisioning/datasources/datasources-ch.yml
+      - ./grafana/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml
+      - ./grafana/dashboards:/var/lib/grafana/dashboards
+  prometheus:
+    image: 'prom/prometheus'
+    ports:
+      - '9090:9090'
+    restart: always
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+  goflow:
+    build:
+      context: ../
+      dockerfile: Dockerfile
+    depends_on:
+      - kafka
+    ports:
+      - '8080:8080'
+      - '6343:6343/udp'
+      - '2055:2055/udp'
+    restart: always
+    command: 
+    - -transport.kafka.brokers=kafka:9092
+    - -transport=kafka
+    - -transport.kafka.topic=flows
+    - -format=pb
+    - -format.protobuf.fixedlen=true
+  db:
+    image: yandex/clickhouse-server
+    ports:
+      - 8123:8123
+    volumes:
+      - ./clickhouse:/docker-entrypoint-initdb.d/
+      - ../pb/flow.proto:/var/lib/clickhouse/format_schemas/flow.proto
+    depends_on:
+      - kafka

compose/grafana/Dockerfile

@@ -0,0 +1,8 @@
+FROM ubuntu AS builder
+
+RUN apt-get update && apt-get install -y git
+RUN git clone https://github.com/Vertamedia/clickhouse-grafana.git
+
+FROM grafana/grafana
+
+COPY --from=builder /clickhouse-grafana /var/lib/grafana/plugins

compose/grafana/dashboards.yml

@@ -0,0 +1,6 @@
+- name: 'default'
+  org_id: 1
+  folder: ''
+  type: file
+  options:
+    folder: /var/lib/grafana/dashboards

compose/grafana/dashboards/viz-ch.json

@@ -0,0 +1,708 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "$$hashKey": "object:631",
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "links": [],
+  "panels": [
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "ClickHouse",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 9,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 2,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "database": "default",
+          "dateColDataType": "Date",
+          "dateLoading": false,
+          "dateTimeColDataType": "TimeFlowStart",
+          "dateTimeType": "DATETIME",
+          "datetimeLoading": false,
+          "format": "time_series",
+          "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t",
+          "group": [],
+          "intervalFactor": 1,
+          "metricColumn": "none",
+          "query": "SELECT\n    toUInt64(toStartOfMinute($dateTimeCol))*1000 as t,\n    sum(Bytes*SamplingRate) as sumbytes\nFROM $table\nWHERE $timeFilter\nGROUP BY t\nORDER BY t",
+          "rawQuery": "SELECT     toUInt64(toStartOfMinute(TimeFlowStart))*1000 as t,     sum(Bytes*SamplingRate) as sumbytes FROM default.flows_raw WHERE Date >= toDate(1585445405) AND TimeFlowStart >= toDateTime(1585445405) GROUP BY t ORDER BY t",
+          "rawSql": "SELECT\n  (cast(extract(epoch from time_flow) as integer)/30)*30 AS \"time\",\n  sum(bytes*sampling_rate*8)/30\nFROM flows\nWHERE\n  $__timeFilter(date_inserted)\nGROUP BY \"time\"\nORDER BY \"time\"",
+          "refId": "A",
+          "round": "0s",
+          "select": [
+            [
+              {
+                "params": [
+                  "bytes"
+                ],
+                "type": "column"
+              }
+            ]
+          ],
+          "table": "flows_raw",
+          "tableLoading": false,
+          "timeColumn": "date_inserted",
+          "timeColumnType": "timestamp",
+          "where": [
+            {
+              "name": "$__timeFilter",
+              "params": [],
+              "type": "macro"
+            }
+          ]
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Instant traffic",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bps",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "columns": [],
+      "datasource": "ClickHouse",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fontSize": "100%",
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 9
+      },
+      "id": 7,
+      "links": [],
+      "pageSize": null,
+      "scroll": true,
+      "showHeader": true,
+      "sort": {
+        "col": 1,
+        "desc": true
+      },
+      "styles": [
+        {
+          "alias": "Time",
+          "align": "auto",
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "pattern": "Time",
+          "type": "date"
+        },
+        {
+          "alias": "",
+          "align": "auto",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 0,
+          "mappingType": 1,
+          "pattern": ".*_port",
+          "thresholds": [],
+          "type": "number",
+          "unit": "none"
+        },
+        {
+          "alias": "",
+          "align": "auto",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "mappingType": 1,
+          "pattern": "sumbytes",
+          "thresholds": [],
+          "type": "number",
+          "unit": "decbytes"
+        },
+        {
+          "alias": "",
+          "align": "auto",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "decimals": 0,
+          "pattern": "/.*/",
+          "thresholds": [],
+          "type": "number",
+          "unit": "short"
+        }
+      ],
+      "targets": [
+        {
+          "database": "default",
+          "dateColDataType": "",
+          "dateLoading": false,
+          "dateTimeColDataType": "TimeFlowStart",
+          "dateTimeType": "DATETIME",
+          "datetimeLoading": false,
+          "format": "table",
+          "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t",
+          "group": [],
+          "intervalFactor": 1,
+          "metricColumn": "none",
+          "query": "SELECT\n    if(EType = 0x800, IPv4NumToString(reinterpretAsUInt32(substring(reverse(SrcAddr), 13,4))), IPv6NumToString(SrcAddr)) as srcip,\n    sum(Bytes*SamplingRate) AS sumbytes\nFROM $table\nWHERE $timeFilter\nGROUP BY srcip\nORDER BY sumbytes DESC",
+          "rawQuery": "SELECT     toUInt64(toStartOfMinute(TimeFlowStart))*1000 as t,     sum(Bytes*SamplingRate) as sumbytes FROM default.flows_raw WHERE Date >= toDate(1593315015) AND TimeFlowStart >= toDateTime(1593315015) GROUP BY t ORDER BY t",
+          "rawSql": "SELECT src_ip, count(*), sum(bytes) AS sumBytes FROM flows GROUP BY src_ip",
+          "refId": "A",
+          "round": "0s",
+          "select": [
+            [
+              {
+                "params": [
+                  "value"
+                ],
+                "type": "column"
+              }
+            ]
+          ],
+          "table": "flows_raw",
+          "tableLoading": false,
+          "timeColumn": "time",
+          "where": [
+            {
+              "name": "$__timeFilter",
+              "params": [],
+              "type": "macro"
+            }
+          ]
+        }
+      ],
+      "title": "Top source IPs",
+      "transform": "table",
+      "type": "table-old"
+    },
+    {
+      "columns": [],
+      "datasource": "ClickHouse",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fontSize": "100%",
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 9
+      },
+      "id": 9,
+      "links": [],
+      "pageSize": null,
+      "scroll": true,
+      "showHeader": true,
+      "sort": {
+        "col": 1,
+        "desc": true
+      },
+      "styles": [
+        {
+          "$$hashKey": "object:1506",
+          "alias": "Time",
+          "align": "auto",
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "pattern": "Time",
+          "type": "date"
+        },
+        {
+          "$$hashKey": "object:1507",
+          "alias": "",
+          "align": "auto",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 0,
+          "mappingType": 1,
+          "pattern": "port",
+          "thresholds": [],
+          "type": "number",
+          "unit": "none"
+        },
+        {
+          "$$hashKey": "object:1508",
+          "alias": "",
+          "align": "auto",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "mappingType": 1,
+          "pattern": "sumbytes",
+          "thresholds": [],
+          "type": "number",
+          "unit": "decbytes"
+        },
+        {
+          "$$hashKey": "object:1509",
+          "alias": "",
+          "align": "auto",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "decimals": 0,
+          "pattern": "/.*/",
+          "thresholds": [],
+          "type": "number",
+          "unit": "short"
+        }
+      ],
+      "targets": [
+        {
+          "database": "default",
+          "dateColDataType": "",
+          "dateLoading": false,
+          "dateTimeColDataType": "TimeFlowStart",
+          "dateTimeType": "DATETIME",
+          "datetimeLoading": false,
+          "extrapolate": true,
+          "format": "table",
+          "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t",
+          "group": [],
+          "intervalFactor": 1,
+          "metricColumn": "none",
+          "query": "WITH dictGetString('dictionaries.protocols', 'name', toUInt64(Proto)) AS protoName\nSELECT\n    if(protoName = '', toString(Proto), protoName) || '/' || toString(SrcPort) as port,\n    sum(Bytes*SamplingRate) AS sumbytes\nFROM $table\nWHERE $timeFilter\nGROUP BY port\nORDER BY sumbytes DESC",
+          "rawQuery": "WITH dictGetString('dictionaries.protocols', 'name', toUInt64(Proto)) AS protoName SELECT     if(protoName = '', toString(Proto), protoName) || '/' || toString(SrcPort) as port,     sum(Bytes*SamplingRate) AS sumbytes FROM default.flows_raw WHERE TimeFlowStart >= toDateTime(1593319741) GROUP BY port ORDER BY sumbytes DESC",
+          "rawSql": "SELECT src_ip, count(*), sum(bytes) AS sumBytes FROM flows GROUP BY src_ip",
+          "refId": "A",
+          "round": "0s",
+          "select": [
+            [
+              {
+                "params": [
+                  "value"
+                ],
+                "type": "column"
+              }
+            ]
+          ],
+          "table": "flows_raw",
+          "tableLoading": false,
+          "timeColumn": "time",
+          "where": [
+            {
+              "name": "$__timeFilter",
+              "params": [],
+              "type": "macro"
+            }
+          ]
+        }
+      ],
+      "title": "Top source ports",
+      "transform": "table",
+      "type": "table-old"
+    },
+    {
+      "columns": [],
+      "datasource": "ClickHouse",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fontSize": "100%",
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 18
+      },
+      "id": 10,
+      "links": [],
+      "pageSize": null,
+      "scroll": true,
+      "showHeader": true,
+      "sort": {
+        "col": 1,
+        "desc": true
+      },
+      "styles": [
+        {
+          "alias": "Time",
+          "align": "auto",
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "pattern": "Time",
+          "type": "date"
+        },
+        {
+          "alias": "",
+          "align": "auto",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 0,
+          "mappingType": 1,
+          "pattern": ".*_port",
+          "thresholds": [],
+          "type": "number",
+          "unit": "none"
+        },
+        {
+          "alias": "",
+          "align": "auto",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "mappingType": 1,
+          "pattern": "sumbytes",
+          "thresholds": [],
+          "type": "number",
+          "unit": "decbytes"
+        },
+        {
+          "alias": "",
+          "align": "auto",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "decimals": 0,
+          "pattern": "/.*/",
+          "thresholds": [],
+          "type": "number",
+          "unit": "short"
+        }
+      ],
+      "targets": [
+        {
+          "database": "default",
+          "dateColDataType": "",
+          "dateLoading": false,
+          "dateTimeColDataType": "TimeFlowStart",
+          "dateTimeType": "DATETIME",
+          "datetimeLoading": false,
+          "format": "table",
+          "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t",
+          "group": [],
+          "intervalFactor": 1,
+          "metricColumn": "none",
+          "query": "SELECT\n    if(EType = 0x800, IPv4NumToString(reinterpretAsUInt32(substring(reverse(DstAddr), 13,4))), IPv6NumToString(DstAddr)) as dstip,\n    sum(Bytes*SamplingRate) AS sumbytes\nFROM $table\nWHERE $timeFilter\nGROUP BY dstip\nORDER BY sumbytes DESC",
+          "rawQuery": "SELECT     toUInt64(toStartOfMinute(TimeFlowStart))*1000 as t,     sum(Bytes*SamplingRate) as sumbytes FROM default.flows_raw WHERE Date >= toDate(1593317660) AND TimeFlowStart >= toDateTime(1593317660) GROUP BY t ORDER BY t",
+          "rawSql": "SELECT src_ip, count(*), sum(bytes) AS sumBytes FROM flows GROUP BY src_ip",
+          "refId": "A",
+          "round": "0s",
+          "select": [
+            [
+              {
+                "params": [
+                  "value"
+                ],
+                "type": "column"
+              }
+            ]
+          ],
+          "table": "flows_raw",
+          "tableLoading": false,
+          "timeColumn": "time",
+          "where": [
+            {
+              "name": "$__timeFilter",
+              "params": [],
+              "type": "macro"
+            }
+          ]
+        }
+      ],
+      "title": "Top destination IPs",
+      "transform": "table",
+      "type": "table-old"
+    },
+    {
+      "columns": [],
+      "datasource": "ClickHouse",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fontSize": "100%",
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 18
+      },
+      "id": 11,
+      "links": [],
+      "pageSize": null,
+      "scroll": true,
+      "showHeader": true,
+      "sort": {
+        "col": 1,
+        "desc": false
+      },
+      "styles": [
+        {
+          "$$hashKey": "object:1428",
+          "alias": "Time",
+          "align": "auto",
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "pattern": "Time",
+          "type": "date"
+        },
+        {
+          "$$hashKey": "object:1429",
+          "alias": "",
+          "align": "auto",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 0,
+          "mappingType": 1,
+          "pattern": "port",
+          "thresholds": [],
+          "type": "number",
+          "unit": "none"
+        },
+        {
+          "$$hashKey": "object:1430",
+          "alias": "",
+          "align": "auto",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "mappingType": 1,
+          "pattern": "sumbytes",
+          "thresholds": [],
+          "type": "number",
+          "unit": "decbytes"
+        },
+        {
+          "$$hashKey": "object:1431",
+          "alias": "",
+          "align": "auto",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "decimals": 0,
+          "pattern": "/.*/",
+          "thresholds": [],
+          "type": "number",
+          "unit": "short"
+        }
+      ],
+      "targets": [
+        {
+          "database": "default",
+          "dateColDataType": "",
+          "dateLoading": false,
+          "dateTimeColDataType": "TimeFlowStart",
+          "dateTimeType": "DATETIME",
+          "datetimeLoading": false,
+          "extrapolate": true,
+          "format": "table",
+          "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t",
+          "group": [],
+          "intervalFactor": 1,
+          "metricColumn": "none",
+          "query": "WITH dictGetString('dictionaries.protocols', 'name', toUInt64(Proto)) AS protoName\nSELECT\n    if(protoName = '', toString(Proto), protoName) || '/' || toString(DstPort) as port,\n    sum(Bytes*SamplingRate) AS sumbytes\nFROM $table\nWHERE $timeFilter\nGROUP BY port\nORDER BY sumbytes DESC",
+          "rawQuery": "WITH dictGetString('dictionaries.protocols', 'name', toUInt64(Proto)) AS protoName SELECT     if(protoName = '', toString(Proto), protoName) || '/' || toString(DstPort) as port,     sum(Bytes*SamplingRate) AS sumbytes FROM default.flows_raw WHERE TimeFlowStart >= toDateTime(1593319708) GROUP BY port ORDER BY sumbytes DESC",
+          "rawSql": "SELECT src_ip, count(*), sum(bytes) AS sumBytes FROM flows GROUP BY src_ip",
+          "refId": "A",
+          "round": "0s",
+          "select": [
+            [
+              {
+                "params": [
+                  "value"
+                ],
+                "type": "column"
+              }
+            ]
+          ],
+          "table": "flows_raw",
+          "tableLoading": false,
+          "timeColumn": "time",
+          "where": [
+            {
+              "name": "$__timeFilter",
+              "params": [],
+              "type": "macro"
+            }
+          ]
+        }
+      ],
+      "title": "Top destination ports",
+      "transform": "table",
+      "type": "table-old"
+    }
+  ],
+  "refresh": false,
+  "schemaVersion": 25,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "",
+  "title": "Traffic (ClickHouse)",
+  "uid": "tkNEAd9Zk",
+  "version": 1
+}

compose/grafana/dashboards/viz.json

@@ -0,0 +1,587 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": 3,
+  "links": [],
+  "panels": [
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "PostgreSQL",
+      "fill": 1,
+      "gridPos": {
+        "h": 9,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 2,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "format": "time_series",
+          "group": [],
+          "metricColumn": "none",
+          "rawQuery": true,
+          "rawSql": "SELECT\n  (cast(extract(epoch from time_flow) as integer)/30)*30 AS \"time\",\n  sum(bytes*sampling_rate*8)/30\nFROM flows\nWHERE\n  $__timeFilter(date_inserted)\nGROUP BY \"time\"\nORDER BY \"time\"",
+          "refId": "A",
+          "select": [
+            [
+              {
+                "params": [
+                  "bytes"
+                ],
+                "type": "column"
+              }
+            ]
+          ],
+          "table": "flows",
+          "timeColumn": "date_inserted",
+          "timeColumnType": "timestamp",
+          "where": [
+            {
+              "name": "$__timeFilter",
+              "params": [],
+              "type": "macro"
+            }
+          ]
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Instant traffic",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bps",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "columns": [],
+      "datasource": "PostgreSQL",
+      "fontSize": "100%",
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 9
+      },
+      "id": 7,
+      "links": [],
+      "pageSize": null,
+      "scroll": true,
+      "showHeader": true,
+      "sort": {
+        "col": 2,
+        "desc": true
+      },
+      "styles": [
+        {
+          "alias": "Time",
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "pattern": "Time",
+          "type": "date"
+        },
+        {
+          "alias": "",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 0,
+          "mappingType": 1,
+          "pattern": ".*_port",
+          "thresholds": [],
+          "type": "number",
+          "unit": "none"
+        },
+        {
+          "alias": "",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "mappingType": 1,
+          "pattern": "sumbytes",
+          "thresholds": [],
+          "type": "number",
+          "unit": "decbytes"
+        },
+        {
+          "alias": "",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "decimals": 0,
+          "pattern": "/.*/",
+          "thresholds": [],
+          "type": "number",
+          "unit": "short"
+        }
+      ],
+      "targets": [
+        {
+          "format": "table",
+          "group": [],
+          "metricColumn": "none",
+          "rawQuery": true,
+          "rawSql": "SELECT src_ip, count(*), sum(bytes) AS sumBytes FROM flows GROUP BY src_ip",
+          "refId": "A",
+          "select": [
+            [
+              {
+                "params": [
+                  "value"
+                ],
+                "type": "column"
+              }
+            ]
+          ],
+          "timeColumn": "time",
+          "where": [
+            {
+              "name": "$__timeFilter",
+              "params": [],
+              "type": "macro"
+            }
+          ]
+        }
+      ],
+      "title": "Top source IPs",
+      "transform": "table",
+      "type": "table"
+    },
+    {
+      "columns": [],
+      "datasource": "PostgreSQL",
+      "fontSize": "100%",
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 9
+      },
+      "id": 5,
+      "links": [],
+      "pageSize": null,
+      "scroll": true,
+      "showHeader": true,
+      "sort": {
+        "col": 2,
+        "desc": true
+      },
+      "styles": [
+        {
+          "alias": "Time",
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "pattern": "Time",
+          "type": "date"
+        },
+        {
+          "alias": "",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 0,
+          "mappingType": 1,
+          "pattern": ".*_port",
+          "thresholds": [],
+          "type": "number",
+          "unit": "none"
+        },
+        {
+          "alias": "",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "mappingType": 1,
+          "pattern": "sumbytes",
+          "thresholds": [],
+          "type": "number",
+          "unit": "decbytes"
+        },
+        {
+          "alias": "",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "decimals": 0,
+          "pattern": "/.*/",
+          "thresholds": [],
+          "type": "number",
+          "unit": "short"
+        }
+      ],
+      "targets": [
+        {
+          "format": "table",
+          "group": [],
+          "metricColumn": "none",
+          "rawQuery": true,
+          "rawSql": "SELECT src_port, count(*), sum(bytes) AS sumBytes FROM flows GROUP BY src_port",
+          "refId": "A",
+          "select": [
+            [
+              {
+                "params": [
+                  "value"
+                ],
+                "type": "column"
+              }
+            ]
+          ],
+          "timeColumn": "time",
+          "where": [
+            {
+              "name": "$__timeFilter",
+              "params": [],
+              "type": "macro"
+            }
+          ]
+        }
+      ],
+      "title": "Top source ports",
+      "transform": "table",
+      "type": "table"
+    },
+    {
+      "columns": [],
+      "datasource": "PostgreSQL",
+      "fontSize": "100%",
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 18
+      },
+      "id": 8,
+      "links": [],
+      "pageSize": null,
+      "scroll": true,
+      "showHeader": true,
+      "sort": {
+        "col": 2,
+        "desc": true
+      },
+      "styles": [
+        {
+          "alias": "Time",
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "pattern": "Time",
+          "type": "date"
+        },
+        {
+          "alias": "",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 0,
+          "mappingType": 1,
+          "pattern": ".*_port",
+          "thresholds": [],
+          "type": "number",
+          "unit": "none"
+        },
+        {
+          "alias": "",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "mappingType": 1,
+          "pattern": "sumbytes",
+          "thresholds": [],
+          "type": "number",
+          "unit": "decbytes"
+        },
+        {
+          "alias": "",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "decimals": 0,
+          "pattern": "/.*/",
+          "thresholds": [],
+          "type": "number",
+          "unit": "short"
+        }
+      ],
+      "targets": [
+        {
+          "format": "table",
+          "group": [],
+          "metricColumn": "none",
+          "rawQuery": true,
+          "rawSql": "SELECT dst_ip, count(*), sum(bytes) AS sumBytes FROM flows GROUP BY dst_ip",
+          "refId": "A",
+          "select": [
+            [
+              {
+                "params": [
+                  "value"
+                ],
+                "type": "column"
+              }
+            ]
+          ],
+          "timeColumn": "time",
+          "where": [
+            {
+              "name": "$__timeFilter",
+              "params": [],
+              "type": "macro"
+            }
+          ]
+        }
+      ],
+      "title": "Top destination IPs",
+      "transform": "table",
+      "type": "table"
+    },
+    {
+      "columns": [],
+      "datasource": "PostgreSQL",
+      "fontSize": "100%",
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 18
+      },
+      "id": 6,
+      "links": [],
+      "pageSize": null,
+      "scroll": true,
+      "showHeader": true,
+      "sort": {
+        "col": 2,
+        "desc": true
+      },
+      "styles": [
+        {
+          "alias": "Time",
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "pattern": "Time",
+          "type": "date"
+        },
+        {
+          "alias": "",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 0,
+          "mappingType": 1,
+          "pattern": ".*_port",
+          "thresholds": [],
+          "type": "number",
+          "unit": "none"
+        },
+        {
+          "alias": "",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "mappingType": 1,
+          "pattern": "sumbytes",
+          "thresholds": [],
+          "type": "number",
+          "unit": "decbytes"
+        },
+        {
+          "alias": "",
+          "colorMode": null,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "decimals": 0,
+          "pattern": "/.*/",
+          "thresholds": [],
+          "type": "number",
+          "unit": "short"
+        }
+      ],
+      "targets": [
+        {
+          "format": "table",
+          "group": [],
+          "metricColumn": "none",
+          "rawQuery": true,
+          "rawSql": "SELECT dst_port, count(*), sum(bytes) AS sumBytes FROM flows GROUP BY dst_port",
+          "refId": "A",
+          "select": [
+            [
+              {
+                "params": [
+                  "value"
+                ],
+                "type": "column"
+              }
+            ]
+          ],
+          "timeColumn": "time",
+          "where": [
+            {
+              "name": "$__timeFilter",
+              "params": [],
+              "type": "macro"
+            }
+          ]
+        }
+      ],
+      "title": "Top destination ports",
+      "transform": "table",
+      "type": "table"
+    }
+  ],
+  "schemaVersion": 16,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-3h",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "",
+  "title": "Traffic",
+  "uid": "HdAEBnPiz",
+  "version": 3
+}

compose/grafana/datasources-ch.yml

@@ -0,0 +1,27 @@
+apiVersion: 1
+
+datasources:
+ - name: Prometheus
+   type: prometheus
+   access: proxy
+   orgId: 1
+   url: http://prometheus:9090
+   version: 1
+   editable: true
+ - name: ClickHouse
+   type: vertamedia-clickhouse-datasource
+   typeLogoUrl: ''
+   access: proxy
+   url: http://db:8123
+   password: ''
+   user: ''
+   database: ''
+   basicAuth: false
+   basicAuthUser: ''
+   basicAuthPassword: ''
+   withCredentials: false
+   isDefault: true
+   jsonData: {}
+   secureJsonFields: {}
+   version: 3
+   readOnly: false

compose/grafana/datasources.yml

@@ -0,0 +1,23 @@
+apiVersion: 1
+
+datasources:
+ - name: Prometheus
+   type: prometheus
+   access: proxy
+   orgId: 1
+   url: http://prometheus:9090
+   version: 1
+   editable: true
+ - name: PostgreSQL
+   type: postgres
+   access: proxy
+   orgId: 1
+   url: postgres:5432
+   database: postgres
+   user: postgres
+   jsonData:
+     sslmode: disable
+   secureJsonData:
+     password: flows
+   version: 1
+   editable: true

compose/prometheus/prometheus.yml

@@ -0,0 +1,14 @@
+global:
+  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+alerting:
+  alertmanagers:
+  - static_configs:
+    - targets:
+
+rule_files:
+
+scrape_configs:
+  - job_name: 'prometheus'
+    static_configs:
+    - targets: ['localhost:9090', 'goflow:8080']

decoders/decoder.go

@@ -0,0 +1,115 @@
+package decoder
+
+import (
+	"time"
+)
+
+type Message interface{}
+type MessageDecoded interface{}
+
+type DecoderFunc func(Message interface{}) error
+type DoneCallback func(string, int, time.Time, time.Time)
+type ErrorCallback func(string, int, time.Time, time.Time, error)
+
+// Worker structure
+type Worker struct {
+	Id            int
+	DecoderParams DecoderParams
+	WorkerPool    chan chan Message
+	Name          string
+	InMsg         chan Message
+	Quit          chan bool
+}
+
+// Create a worker and add it to the pool.
+func CreateWorker(workerPool chan chan Message, decoderParams DecoderParams, id int, name string) Worker {
+	return Worker{
+		Id:            id,
+		DecoderParams: decoderParams,
+		WorkerPool:    workerPool,
+		Name:          name,
+		InMsg:         make(chan Message),
+		Quit:          make(chan bool),
+	}
+}
+
+// Start the worker. Launches a goroutine to process NFv9 messages.
+// The worker will add its input channel of NFv9 messages to decode to the pool.
+func (w Worker) Start() {
+	go func() {
+		//log.Debugf("Worker %v started", w.Id)
+		for {
+			select {
+			case <-w.Quit:
+				break
+			case w.WorkerPool <- w.InMsg:
+				msg := <-w.InMsg
+				timeTrackStart := time.Now()
+				err := w.DecoderParams.DecoderFunc(msg)
+				timeTrackStop := time.Now()
+
+				if err != nil && w.DecoderParams.ErrorCallback != nil {
+					w.DecoderParams.ErrorCallback(w.Name, w.Id, timeTrackStart, timeTrackStop, err)
+				} else if err == nil && w.DecoderParams.DoneCallback != nil {
+					w.DecoderParams.DoneCallback(w.Name, w.Id, timeTrackStart, timeTrackStop)
+				}
+			}
+		}
+		//log.Debugf("Worker %v done", w.Id)
+	}()
+}
+
+// Stop the worker.
+func (w Worker) Stop() {
+	//log.Debugf("Stopping worker %v", w.Id)
+	w.Quit <- true
+}
+
+// Processor structure
+type Processor struct {
+	workerpool    chan chan Message
+	workerlist    []Worker
+	DecoderParams DecoderParams
+	Name          string
+}
+
+// Decoder structure. Define the function to call and the config specific to the type of packets.
+type DecoderParams struct {
+	DecoderFunc   DecoderFunc
+	DoneCallback  DoneCallback
+	ErrorCallback ErrorCallback
+}
+
+// Create a message processor which is going to create all the workers and set-up the pool.
+func CreateProcessor(numWorkers int, decoderParams DecoderParams, name string) Processor {
+	processor := Processor{
+		workerpool:    make(chan chan Message),
+		workerlist:    make([]Worker, numWorkers),
+		DecoderParams: decoderParams,
+		Name:          name,
+	}
+	for i := 0; i < numWorkers; i++ {
+		worker := CreateWorker(processor.workerpool, decoderParams, i, name)
+		processor.workerlist[i] = worker
+	}
+	return processor
+}
+
+// Start message processor
+func (p Processor) Start() {
+	for _, worker := range p.workerlist {
+		worker.Start()
+	}
+}
+
+func (p Processor) Stop() {
+	for _, worker := range p.workerlist {
+		worker.Stop()
+	}
+}
+
+// Send a message to be decoded to the pool.
+func (p Processor) ProcessMessage(msg Message) {
+	sendChannel := <-p.workerpool
+	sendChannel <- msg
+}

decoders/netflow/ipfix.go

@@ -0,0 +1,989 @@
+package netflow
+
+import (
+	"fmt"
+	"time"
+)
+
+const (
+	IPFIX_FIELD_Reserved                              = 0
+	IPFIX_FIELD_octetDeltaCount                       = 1
+	IPFIX_FIELD_packetDeltaCount                      = 2
+	IPFIX_FIELD_deltaFlowCount                        = 3
+	IPFIX_FIELD_protocolIdentifier                    = 4
+	IPFIX_FIELD_ipClassOfService                      = 5
+	IPFIX_FIELD_tcpControlBits                        = 6
+	IPFIX_FIELD_sourceTransportPort                   = 7
+	IPFIX_FIELD_sourceIPv4Address                     = 8
+	IPFIX_FIELD_sourceIPv4PrefixLength                = 9
+	IPFIX_FIELD_ingressInterface                      = 10
+	IPFIX_FIELD_destinationTransportPort              = 11
+	IPFIX_FIELD_destinationIPv4Address                = 12
+	IPFIX_FIELD_destinationIPv4PrefixLength           = 13
+	IPFIX_FIELD_egressInterface                       = 14
+	IPFIX_FIELD_ipNextHopIPv4Address                  = 15
+	IPFIX_FIELD_bgpSourceAsNumber                     = 16
+	IPFIX_FIELD_bgpDestinationAsNumber                = 17
+	IPFIX_FIELD_bgpNextHopIPv4Address                 = 18
+	IPFIX_FIELD_postMCastPacketDeltaCount             = 19
+	IPFIX_FIELD_postMCastOctetDeltaCount              = 20
+	IPFIX_FIELD_flowEndSysUpTime                      = 21
+	IPFIX_FIELD_flowStartSysUpTime                    = 22
+	IPFIX_FIELD_postOctetDeltaCount                   = 23
+	IPFIX_FIELD_postPacketDeltaCount                  = 24
+	IPFIX_FIELD_minimumIpTotalLength                  = 25
+	IPFIX_FIELD_maximumIpTotalLength                  = 26
+	IPFIX_FIELD_sourceIPv6Address                     = 27
+	IPFIX_FIELD_destinationIPv6Address                = 28
+	IPFIX_FIELD_sourceIPv6PrefixLength                = 29
+	IPFIX_FIELD_destinationIPv6PrefixLength           = 30
+	IPFIX_FIELD_flowLabelIPv6                         = 31
+	IPFIX_FIELD_icmpTypeCodeIPv4                      = 32
+	IPFIX_FIELD_igmpType                              = 33
+	IPFIX_FIELD_samplingInterval                      = 34
+	IPFIX_FIELD_samplingAlgorithm                     = 35
+	IPFIX_FIELD_flowActiveTimeout                     = 36
+	IPFIX_FIELD_flowIdleTimeout                       = 37
+	IPFIX_FIELD_engineType                            = 38
+	IPFIX_FIELD_engineId                              = 39
+	IPFIX_FIELD_exportedOctetTotalCount               = 40
+	IPFIX_FIELD_exportedMessageTotalCount             = 41
+	IPFIX_FIELD_exportedFlowRecordTotalCount          = 42
+	IPFIX_FIELD_ipv4RouterSc                          = 43
+	IPFIX_FIELD_sourceIPv4Prefix                      = 44
+	IPFIX_FIELD_destinationIPv4Prefix                 = 45
+	IPFIX_FIELD_mplsTopLabelType                      = 46
+	IPFIX_FIELD_mplsTopLabelIPv4Address               = 47
+	IPFIX_FIELD_samplerId                             = 48
+	IPFIX_FIELD_samplerMode                           = 49
+	IPFIX_FIELD_samplerRandomInterval                 = 50
+	IPFIX_FIELD_classId                               = 51
+	IPFIX_FIELD_minimumTTL                            = 52
+	IPFIX_FIELD_maximumTTL                            = 53
+	IPFIX_FIELD_fragmentIdentification                = 54
+	IPFIX_FIELD_postIpClassOfService                  = 55
+	IPFIX_FIELD_sourceMacAddress                      = 56
+	IPFIX_FIELD_postDestinationMacAddress             = 57
+	IPFIX_FIELD_vlanId                                = 58
+	IPFIX_FIELD_postVlanId                            = 59
+	IPFIX_FIELD_ipVersion                             = 60
+	IPFIX_FIELD_flowDirection                         = 61
+	IPFIX_FIELD_ipNextHopIPv6Address                  = 62
+	IPFIX_FIELD_bgpNextHopIPv6Address                 = 63
+	IPFIX_FIELD_ipv6ExtensionHeaders                  = 64
+	IPFIX_FIELD_mplsTopLabelStackSection              = 70
+	IPFIX_FIELD_mplsLabelStackSection2                = 71
+	IPFIX_FIELD_mplsLabelStackSection3                = 72
+	IPFIX_FIELD_mplsLabelStackSection4                = 73
+	IPFIX_FIELD_mplsLabelStackSection5                = 74
+	IPFIX_FIELD_mplsLabelStackSection6                = 75
+	IPFIX_FIELD_mplsLabelStackSection7                = 76
+	IPFIX_FIELD_mplsLabelStackSection8                = 77
+	IPFIX_FIELD_mplsLabelStackSection9                = 78
+	IPFIX_FIELD_mplsLabelStackSection10               = 79
+	IPFIX_FIELD_destinationMacAddress                 = 80
+	IPFIX_FIELD_postSourceMacAddress                  = 81
+	IPFIX_FIELD_interfaceName                         = 82
+	IPFIX_FIELD_interfaceDescription                  = 83
+	IPFIX_FIELD_samplerName                           = 84
+	IPFIX_FIELD_octetTotalCount                       = 85
+	IPFIX_FIELD_packetTotalCount                      = 86
+	IPFIX_FIELD_flagsAndSamplerId                     = 87
+	IPFIX_FIELD_fragmentOffset                        = 88
+	IPFIX_FIELD_forwardingStatus                      = 89
+	IPFIX_FIELD_mplsVpnRouteDistinguisher             = 90
+	IPFIX_FIELD_mplsTopLabelPrefixLength              = 91
+	IPFIX_FIELD_srcTrafficIndex                       = 92
+	IPFIX_FIELD_dstTrafficIndex                       = 93
+	IPFIX_FIELD_applicationDescription                = 94
+	IPFIX_FIELD_applicationId                         = 95
+	IPFIX_FIELD_applicationName                       = 96
+	IPFIX_FIELD_postIpDiffServCodePoint               = 98
+	IPFIX_FIELD_multicastReplicationFactor            = 99
+	IPFIX_FIELD_className                             = 100
+	IPFIX_FIELD_classificationEngineId                = 101
+	IPFIX_FIELD_layer2packetSectionOffset             = 102
+	IPFIX_FIELD_layer2packetSectionSize               = 103
+	IPFIX_FIELD_layer2packetSectionData               = 104
+	IPFIX_FIELD_bgpNextAdjacentAsNumber               = 128
+	IPFIX_FIELD_bgpPrevAdjacentAsNumber               = 129
+	IPFIX_FIELD_exporterIPv4Address                   = 130
+	IPFIX_FIELD_exporterIPv6Address                   = 131
+	IPFIX_FIELD_droppedOctetDeltaCount                = 132
+	IPFIX_FIELD_droppedPacketDeltaCount               = 133
+	IPFIX_FIELD_droppedOctetTotalCount                = 134
+	IPFIX_FIELD_droppedPacketTotalCount               = 135
+	IPFIX_FIELD_flowEndReason                         = 136
+	IPFIX_FIELD_commonPropertiesId                    = 137
+	IPFIX_FIELD_observationPointId                    = 138
+	IPFIX_FIELD_icmpTypeCodeIPv6                      = 139
+	IPFIX_FIELD_mplsTopLabelIPv6Address               = 140
+	IPFIX_FIELD_lineCardId                            = 141
+	IPFIX_FIELD_portId                                = 142
+	IPFIX_FIELD_meteringProcessId                     = 143
+	IPFIX_FIELD_exportingProcessId                    = 144
+	IPFIX_FIELD_templateId                            = 145
+	IPFIX_FIELD_wlanChannelId                         = 146
+	IPFIX_FIELD_wlanSSID                              = 147
+	IPFIX_FIELD_flowId                                = 148
+	IPFIX_FIELD_observationDomainId                   = 149
+	IPFIX_FIELD_flowStartSeconds                      = 150
+	IPFIX_FIELD_flowEndSeconds                        = 151
+	IPFIX_FIELD_flowStartMilliseconds                 = 152
+	IPFIX_FIELD_flowEndMilliseconds                   = 153
+	IPFIX_FIELD_flowStartMicroseconds                 = 154
+	IPFIX_FIELD_flowEndMicroseconds                   = 155
+	IPFIX_FIELD_flowStartNanoseconds                  = 156
+	IPFIX_FIELD_flowEndNanoseconds                    = 157
+	IPFIX_FIELD_flowStartDeltaMicroseconds            = 158
+	IPFIX_FIELD_flowEndDeltaMicroseconds              = 159
+	IPFIX_FIELD_systemInitTimeMilliseconds            = 160
+	IPFIX_FIELD_flowDurationMilliseconds              = 161
+	IPFIX_FIELD_flowDurationMicroseconds              = 162
+	IPFIX_FIELD_observedFlowTotalCount                = 163
+	IPFIX_FIELD_ignoredPacketTotalCount               = 164
+	IPFIX_FIELD_ignoredOctetTotalCount                = 165
+	IPFIX_FIELD_notSentFlowTotalCount                 = 166
+	IPFIX_FIELD_notSentPacketTotalCount               = 167
+	IPFIX_FIELD_notSentOctetTotalCount                = 168
+	IPFIX_FIELD_destinationIPv6Prefix                 = 169
+	IPFIX_FIELD_sourceIPv6Prefix                      = 170
+	IPFIX_FIELD_postOctetTotalCount                   = 171
+	IPFIX_FIELD_postPacketTotalCount                  = 172
+	IPFIX_FIELD_flowKeyIndicator                      = 173
+	IPFIX_FIELD_postMCastPacketTotalCount             = 174
+	IPFIX_FIELD_postMCastOctetTotalCount              = 175
+	IPFIX_FIELD_icmpTypeIPv4                          = 176
+	IPFIX_FIELD_icmpCodeIPv4                          = 177
+	IPFIX_FIELD_icmpTypeIPv6                          = 178
+	IPFIX_FIELD_icmpCodeIPv6                          = 179
+	IPFIX_FIELD_udpSourcePort                         = 180
+	IPFIX_FIELD_udpDestinationPort                    = 181
+	IPFIX_FIELD_tcpSourcePort                         = 182
+	IPFIX_FIELD_tcpDestinationPort                    = 183
+	IPFIX_FIELD_tcpSequenceNumber                     = 184
+	IPFIX_FIELD_tcpAcknowledgementNumber              = 185
+	IPFIX_FIELD_tcpWindowSize                         = 186
+	IPFIX_FIELD_tcpUrgentPointer                      = 187
+	IPFIX_FIELD_tcpHeaderLength                       = 188
+	IPFIX_FIELD_ipHeaderLength                        = 189
+	IPFIX_FIELD_totalLengthIPv4                       = 190
+	IPFIX_FIELD_payloadLengthIPv6                     = 191
+	IPFIX_FIELD_ipTTL                                 = 192
+	IPFIX_FIELD_nextHeaderIPv6                        = 193
+	IPFIX_FIELD_mplsPayloadLength                     = 194
+	IPFIX_FIELD_ipDiffServCodePoint                   = 195
+	IPFIX_FIELD_ipPrecedence                          = 196
+	IPFIX_FIELD_fragmentFlags                         = 197
+	IPFIX_FIELD_octetDeltaSumOfSquares                = 198
+	IPFIX_FIELD_octetTotalSumOfSquares                = 199
+	IPFIX_FIELD_mplsTopLabelTTL                       = 200
+	IPFIX_FIELD_mplsLabelStackLength                  = 201
+	IPFIX_FIELD_mplsLabelStackDepth                   = 202
+	IPFIX_FIELD_mplsTopLabelExp                       = 203
+	IPFIX_FIELD_ipPayloadLength                       = 204
+	IPFIX_FIELD_udpMessageLength                      = 205
+	IPFIX_FIELD_isMulticast                           = 206
+	IPFIX_FIELD_ipv4IHL                               = 207
+	IPFIX_FIELD_ipv4Options                           = 208
+	IPFIX_FIELD_tcpOptions                            = 209
+	IPFIX_FIELD_paddingOctets                         = 210
+	IPFIX_FIELD_collectorIPv4Address                  = 211
+	IPFIX_FIELD_collectorIPv6Address                  = 212
+	IPFIX_FIELD_exportInterface                       = 213
+	IPFIX_FIELD_exportProtocolVersion                 = 214
+	IPFIX_FIELD_exportTransportProtocol               = 215
+	IPFIX_FIELD_collectorTransportPort                = 216
+	IPFIX_FIELD_exporterTransportPort                 = 217
+	IPFIX_FIELD_tcpSynTotalCount                      = 218
+	IPFIX_FIELD_tcpFinTotalCount                      = 219
+	IPFIX_FIELD_tcpRstTotalCount                      = 220
+	IPFIX_FIELD_tcpPshTotalCount                      = 221
+	IPFIX_FIELD_tcpAckTotalCount                      = 222
+	IPFIX_FIELD_tcpUrgTotalCount                      = 223
+	IPFIX_FIELD_ipTotalLength                         = 224
+	IPFIX_FIELD_postNATSourceIPv4Address              = 225
+	IPFIX_FIELD_postNATDestinationIPv4Address         = 226
+	IPFIX_FIELD_postNAPTSourceTransportPort           = 227
+	IPFIX_FIELD_postNAPTDestinationTransportPort      = 228
+	IPFIX_FIELD_natOriginatingAddressRealm            = 229
+	IPFIX_FIELD_natEvent                              = 230
+	IPFIX_FIELD_initiatorOctets                       = 231
+	IPFIX_FIELD_responderOctets                       = 232
+	IPFIX_FIELD_firewallEvent                         = 233
+	IPFIX_FIELD_ingressVRFID                          = 234
+	IPFIX_FIELD_egressVRFID                           = 235
+	IPFIX_FIELD_VRFname                               = 236
+	IPFIX_FIELD_postMplsTopLabelExp                   = 237
+	IPFIX_FIELD_tcpWindowScale                        = 238
+	IPFIX_FIELD_biflowDirection                       = 239
+	IPFIX_FIELD_ethernetHeaderLength                  = 240
+	IPFIX_FIELD_ethernetPayloadLength                 = 241
+	IPFIX_FIELD_ethernetTotalLength                   = 242
+	IPFIX_FIELD_dot1qVlanId                           = 243
+	IPFIX_FIELD_dot1qPriority                         = 244
+	IPFIX_FIELD_dot1qCustomerVlanId                   = 245
+	IPFIX_FIELD_dot1qCustomerPriority                 = 246
+	IPFIX_FIELD_metroEvcId                            = 247
+	IPFIX_FIELD_metroEvcType                          = 248
+	IPFIX_FIELD_pseudoWireId                          = 249
+	IPFIX_FIELD_pseudoWireType                        = 250
+	IPFIX_FIELD_pseudoWireControlWord                 = 251
+	IPFIX_FIELD_ingressPhysicalInterface              = 252
+	IPFIX_FIELD_egressPhysicalInterface               = 253
+	IPFIX_FIELD_postDot1qVlanId                       = 254
+	IPFIX_FIELD_postDot1qCustomerVlanId               = 255
+	IPFIX_FIELD_ethernetType                          = 256
+	IPFIX_FIELD_postIpPrecedence                      = 257
+	IPFIX_FIELD_collectionTimeMilliseconds            = 258
+	IPFIX_FIELD_exportSctpStreamId                    = 259
+	IPFIX_FIELD_maxExportSeconds                      = 260
+	IPFIX_FIELD_maxFlowEndSeconds                     = 261
+	IPFIX_FIELD_messageMD5Checksum                    = 262
+	IPFIX_FIELD_messageScope                          = 263
+	IPFIX_FIELD_minExportSeconds                      = 264
+	IPFIX_FIELD_minFlowStartSeconds                   = 265
+	IPFIX_FIELD_opaqueOctets                          = 266
+	IPFIX_FIELD_sessionScope                          = 267
+	IPFIX_FIELD_maxFlowEndMicroseconds                = 268
+	IPFIX_FIELD_maxFlowEndMilliseconds                = 269
+	IPFIX_FIELD_maxFlowEndNanoseconds                 = 270
+	IPFIX_FIELD_minFlowStartMicroseconds              = 271
+	IPFIX_FIELD_minFlowStartMilliseconds              = 272
+	IPFIX_FIELD_minFlowStartNanoseconds               = 273
+	IPFIX_FIELD_collectorCertificate                  = 274
+	IPFIX_FIELD_exporterCertificate                   = 275
+	IPFIX_FIELD_dataRecordsReliability                = 276
+	IPFIX_FIELD_observationPointType                  = 277
+	IPFIX_FIELD_newConnectionDeltaCount               = 278
+	IPFIX_FIELD_connectionSumDurationSeconds          = 279
+	IPFIX_FIELD_connectionTransactionId               = 280
+	IPFIX_FIELD_postNATSourceIPv6Address              = 281
+	IPFIX_FIELD_postNATDestinationIPv6Address         = 282
+	IPFIX_FIELD_natPoolId                             = 283
+	IPFIX_FIELD_natPoolName                           = 284
+	IPFIX_FIELD_anonymizationFlags                    = 285
+	IPFIX_FIELD_anonymizationTechnique                = 286
+	IPFIX_FIELD_informationElementIndex               = 287
+	IPFIX_FIELD_p2pTechnology                         = 288
+	IPFIX_FIELD_tunnelTechnology                      = 289
+	IPFIX_FIELD_encryptedTechnology                   = 290
+	IPFIX_FIELD_basicList                             = 291
+	IPFIX_FIELD_subTemplateList                       = 292
+	IPFIX_FIELD_subTemplateMultiList                  = 293
+	IPFIX_FIELD_bgpValidityState                      = 294
+	IPFIX_FIELD_IPSecSPI                              = 295
+	IPFIX_FIELD_greKey                                = 296
+	IPFIX_FIELD_natType                               = 297
+	IPFIX_FIELD_initiatorPackets                      = 298
+	IPFIX_FIELD_responderPackets                      = 299
+	IPFIX_FIELD_observationDomainName                 = 300
+	IPFIX_FIELD_selectionSequenceId                   = 301
+	IPFIX_FIELD_selectorId                            = 302
+	IPFIX_FIELD_informationElementId                  = 303
+	IPFIX_FIELD_selectorAlgorithm                     = 304
+	IPFIX_FIELD_samplingPacketInterval                = 305
+	IPFIX_FIELD_samplingPacketSpace                   = 306
+	IPFIX_FIELD_samplingTimeInterval                  = 307
+	IPFIX_FIELD_samplingTimeSpace                     = 308
+	IPFIX_FIELD_samplingSize                          = 309
+	IPFIX_FIELD_samplingPopulation                    = 310
+	IPFIX_FIELD_samplingProbability                   = 311
+	IPFIX_FIELD_dataLinkFrameSize                     = 312
+	IPFIX_FIELD_ipHeaderPacketSection                 = 313
+	IPFIX_FIELD_ipPayloadPacketSection                = 314
+	IPFIX_FIELD_dataLinkFrameSection                  = 315
+	IPFIX_FIELD_mplsLabelStackSection                 = 316
+	IPFIX_FIELD_mplsPayloadPacketSection              = 317
+	IPFIX_FIELD_selectorIdTotalPktsObserved           = 318
+	IPFIX_FIELD_selectorIdTotalPktsSelected           = 319
+	IPFIX_FIELD_absoluteError                         = 320
+	IPFIX_FIELD_relativeError                         = 321
+	IPFIX_FIELD_observationTimeSeconds                = 322
+	IPFIX_FIELD_observationTimeMilliseconds           = 323
+	IPFIX_FIELD_observationTimeMicroseconds           = 324
+	IPFIX_FIELD_observationTimeNanoseconds            = 325
+	IPFIX_FIELD_digestHashValue                       = 326
+	IPFIX_FIELD_hashIPPayloadOffset                   = 327
+	IPFIX_FIELD_hashIPPayloadSize                     = 328
+	IPFIX_FIELD_hashOutputRangeMin                    = 329
+	IPFIX_FIELD_hashOutputRangeMax                    = 330
+	IPFIX_FIELD_hashSelectedRangeMin                  = 331
+	IPFIX_FIELD_hashSelectedRangeMax                  = 332
+	IPFIX_FIELD_hashDigestOutput                      = 333
+	IPFIX_FIELD_hashInitialiserValue                  = 334
+	IPFIX_FIELD_selectorName                          = 335
+	IPFIX_FIELD_upperCILimit                          = 336
+	IPFIX_FIELD_lowerCILimit                          = 337
+	IPFIX_FIELD_confidenceLevel                       = 338
+	IPFIX_FIELD_informationElementDataType            = 339
+	IPFIX_FIELD_informationElementDescription         = 340
+	IPFIX_FIELD_informationElementName                = 341
+	IPFIX_FIELD_informationElementRangeBegin          = 342
+	IPFIX_FIELD_informationElementRangeEnd            = 343
+	IPFIX_FIELD_informationElementSemantics           = 344
+	IPFIX_FIELD_informationElementUnits               = 345
+	IPFIX_FIELD_privateEnterpriseNumber               = 346
+	IPFIX_FIELD_virtualStationInterfaceId             = 347
+	IPFIX_FIELD_virtualStationInterfaceName           = 348
+	IPFIX_FIELD_virtualStationUUID                    = 349
+	IPFIX_FIELD_virtualStationName                    = 350
+	IPFIX_FIELD_layer2SegmentId                       = 351
+	IPFIX_FIELD_layer2OctetDeltaCount                 = 352
+	IPFIX_FIELD_layer2OctetTotalCount                 = 353
+	IPFIX_FIELD_ingressUnicastPacketTotalCount        = 354
+	IPFIX_FIELD_ingressMulticastPacketTotalCount      = 355
+	IPFIX_FIELD_ingressBroadcastPacketTotalCount      = 356
+	IPFIX_FIELD_egressUnicastPacketTotalCount         = 357
+	IPFIX_FIELD_egressBroadcastPacketTotalCount       = 358
+	IPFIX_FIELD_monitoringIntervalStartMilliSeconds   = 359
+	IPFIX_FIELD_monitoringIntervalEndMilliSeconds     = 360
+	IPFIX_FIELD_portRangeStart                        = 361
+	IPFIX_FIELD_portRangeEnd                          = 362
+	IPFIX_FIELD_portRangeStepSize                     = 363
+	IPFIX_FIELD_portRangeNumPorts                     = 364
+	IPFIX_FIELD_staMacAddress                         = 365
+	IPFIX_FIELD_staIPv4Address                        = 366
+	IPFIX_FIELD_wtpMacAddress                         = 367
+	IPFIX_FIELD_ingressInterfaceType                  = 368
+	IPFIX_FIELD_egressInterfaceType                   = 369
+	IPFIX_FIELD_rtpSequenceNumber                     = 370
+	IPFIX_FIELD_userName                              = 371
+	IPFIX_FIELD_applicationCategoryName               = 372
+	IPFIX_FIELD_applicationSubCategoryName            = 373
+	IPFIX_FIELD_applicationGroupName                  = 374
+	IPFIX_FIELD_originalFlowsPresent                  = 375
+	IPFIX_FIELD_originalFlowsInitiated                = 376
+	IPFIX_FIELD_originalFlowsCompleted                = 377
+	IPFIX_FIELD_distinctCountOfSourceIPAddress        = 378
+	IPFIX_FIELD_distinctCountOfDestinationIPAddress   = 379
+	IPFIX_FIELD_distinctCountOfSourceIPv4Address      = 380
+	IPFIX_FIELD_distinctCountOfDestinationIPv4Address = 381
+	IPFIX_FIELD_distinctCountOfSourceIPv6Address      = 382
+	IPFIX_FIELD_distinctCountOfDestinationIPv6Address = 383
+	IPFIX_FIELD_valueDistributionMethod               = 384
+	IPFIX_FIELD_rfc3550JitterMilliseconds             = 385
+	IPFIX_FIELD_rfc3550JitterMicroseconds             = 386
+	IPFIX_FIELD_rfc3550JitterNanoseconds              = 387
+	IPFIX_FIELD_dot1qDEI                              = 388
+	IPFIX_FIELD_dot1qCustomerDEI                      = 389
+	IPFIX_FIELD_flowSelectorAlgorithm                 = 390
+	IPFIX_FIELD_flowSelectedOctetDeltaCount           = 391
+	IPFIX_FIELD_flowSelectedPacketDeltaCount          = 392
+	IPFIX_FIELD_flowSelectedFlowDeltaCount            = 393
+	IPFIX_FIELD_selectorIDTotalFlowsObserved          = 394
+	IPFIX_FIELD_selectorIDTotalFlowsSelected          = 395
+	IPFIX_FIELD_samplingFlowInterval                  = 396
+	IPFIX_FIELD_samplingFlowSpacing                   = 397
+	IPFIX_FIELD_flowSamplingTimeInterval              = 398
+	IPFIX_FIELD_flowSamplingTimeSpacing               = 399
+	IPFIX_FIELD_hashFlowDomain                        = 400
+	IPFIX_FIELD_transportOctetDeltaCount              = 401
+	IPFIX_FIELD_transportPacketDeltaCount             = 402
+	IPFIX_FIELD_originalExporterIPv4Address           = 403
+	IPFIX_FIELD_originalExporterIPv6Address           = 404
+	IPFIX_FIELD_originalObservationDomainId           = 405
+	IPFIX_FIELD_intermediateProcessId                 = 406
+	IPFIX_FIELD_ignoredDataRecordTotalCount           = 407
+	IPFIX_FIELD_dataLinkFrameType                     = 408
+	IPFIX_FIELD_sectionOffset                         = 409
+	IPFIX_FIELD_sectionExportedOctets                 = 410
+	IPFIX_FIELD_dot1qServiceInstanceTag               = 411
+	IPFIX_FIELD_dot1qServiceInstanceId                = 412
+	IPFIX_FIELD_dot1qServiceInstancePriority          = 413
+	IPFIX_FIELD_dot1qCustomerSourceMacAddress         = 414
+	IPFIX_FIELD_dot1qCustomerDestinationMacAddress    = 415
+	IPFIX_FIELD_postLayer2OctetDeltaCount             = 417
+	IPFIX_FIELD_postMCastLayer2OctetDeltaCount        = 418
+	IPFIX_FIELD_postLayer2OctetTotalCount             = 420
+	IPFIX_FIELD_postMCastLayer2OctetTotalCount        = 421
+	IPFIX_FIELD_minimumLayer2TotalLength              = 422
+	IPFIX_FIELD_maximumLayer2TotalLength              = 423
+	IPFIX_FIELD_droppedLayer2OctetDeltaCount          = 424
+	IPFIX_FIELD_droppedLayer2OctetTotalCount          = 425
+	IPFIX_FIELD_ignoredLayer2OctetTotalCount          = 426
+	IPFIX_FIELD_notSentLayer2OctetTotalCount          = 427
+	IPFIX_FIELD_layer2OctetDeltaSumOfSquares          = 428
+	IPFIX_FIELD_layer2OctetTotalSumOfSquares          = 429
+	IPFIX_FIELD_layer2FrameDeltaCount                 = 430
+	IPFIX_FIELD_layer2FrameTotalCount                 = 431
+	IPFIX_FIELD_pseudoWireDestinationIPv4Address      = 432
+	IPFIX_FIELD_ignoredLayer2FrameTotalCount          = 433
+	IPFIX_FIELD_mibObjectValueInteger                 = 434
+	IPFIX_FIELD_mibObjectValueOctetString             = 435
+	IPFIX_FIELD_mibObjectValueOID                     = 436
+	IPFIX_FIELD_mibObjectValueBits                    = 437
+	IPFIX_FIELD_mibObjectValueIPAddress               = 438
+	IPFIX_FIELD_mibObjectValueCounter                 = 439
+	IPFIX_FIELD_mibObjectValueGauge                   = 440
+	IPFIX_FIELD_mibObjectValueTimeTicks               = 441
+	IPFIX_FIELD_mibObjectValueUnsigned                = 442
+	IPFIX_FIELD_mibObjectValueTable                   = 443
+	IPFIX_FIELD_mibObjectValueRow                     = 444
+	IPFIX_FIELD_mibObjectIdentifier                   = 445
+	IPFIX_FIELD_mibSubIdentifier                      = 446
+	IPFIX_FIELD_mibIndexIndicator                     = 447
+	IPFIX_FIELD_mibCaptureTimeSemantics               = 448
+	IPFIX_FIELD_mibContextEngineID                    = 449
+	IPFIX_FIELD_mibContextName                        = 450
+	IPFIX_FIELD_mibObjectName                         = 451
+	IPFIX_FIELD_mibObjectDescription                  = 452
+	IPFIX_FIELD_mibObjectSyntax                       = 453
+	IPFIX_FIELD_mibModuleName                         = 454
+	IPFIX_FIELD_mobileIMSI                            = 455
+	IPFIX_FIELD_mobileMSISDN                          = 456
+	IPFIX_FIELD_httpStatusCode                        = 457
+	IPFIX_FIELD_sourceTransportPortsLimit             = 458
+	IPFIX_FIELD_httpRequestMethod                     = 459
+	IPFIX_FIELD_httpRequestHost                       = 460
+	IPFIX_FIELD_httpRequestTarget                     = 461
+	IPFIX_FIELD_httpMessageVersion                    = 462
+	IPFIX_FIELD_natInstanceID                         = 463
+	IPFIX_FIELD_internalAddressRealm                  = 464
+	IPFIX_FIELD_externalAddressRealm                  = 465
+	IPFIX_FIELD_natQuotaExceededEvent                 = 466
+	IPFIX_FIELD_natThresholdEvent                     = 467
+)
+
+type IPFIXPacket struct {
+	Version             uint16
+	Length              uint16
+	ExportTime          uint32
+	SequenceNumber      uint32
+	ObservationDomainId uint32
+	FlowSets            []interface{}
+}
+
+type IPFIXOptionsTemplateFlowSet struct {
+	FlowSetHeader
+	Records []IPFIXOptionsTemplateRecord
+}
+
+type IPFIXOptionsTemplateRecord struct {
+	TemplateId      uint16
+	FieldCount      uint16
+	ScopeFieldCount uint16
+	Options         []Field
+	Scopes          []Field
+}
+
+func IPFIXTypeToString(typeId uint16) string {
+
+	nameList := map[uint16]string{
+		0:   "Reserved",
+		1:   "octetDeltaCount",
+		2:   "packetDeltaCount",
+		3:   "deltaFlowCount",
+		4:   "protocolIdentifier",
+		5:   "ipClassOfService",
+		6:   "tcpControlBits",
+		7:   "sourceTransportPort",
+		8:   "sourceIPv4Address",
+		9:   "sourceIPv4PrefixLength",
+		10:  "ingressInterface",
+		11:  "destinationTransportPort",
+		12:  "destinationIPv4Address",
+		13:  "destinationIPv4PrefixLength",
+		14:  "egressInterface",
+		15:  "ipNextHopIPv4Address",
+		16:  "bgpSourceAsNumber",
+		17:  "bgpDestinationAsNumber",
+		18:  "bgpNextHopIPv4Address",
+		19:  "postMCastPacketDeltaCount",
+		20:  "postMCastOctetDeltaCount",
+		21:  "flowEndSysUpTime",
+		22:  "flowStartSysUpTime",
+		23:  "postOctetDeltaCount",
+		24:  "postPacketDeltaCount",
+		25:  "minimumIpTotalLength",
+		26:  "maximumIpTotalLength",
+		27:  "sourceIPv6Address",
+		28:  "destinationIPv6Address",
+		29:  "sourceIPv6PrefixLength",
+		30:  "destinationIPv6PrefixLength",
+		31:  "flowLabelIPv6",
+		32:  "icmpTypeCodeIPv4",
+		33:  "igmpType",
+		34:  "samplingInterval",
+		35:  "samplingAlgorithm",
+		36:  "flowActiveTimeout",
+		37:  "flowIdleTimeout",
+		38:  "engineType",
+		39:  "engineId",
+		40:  "exportedOctetTotalCount",
+		41:  "exportedMessageTotalCount",
+		42:  "exportedFlowRecordTotalCount",
+		43:  "ipv4RouterSc",
+		44:  "sourceIPv4Prefix",
+		45:  "destinationIPv4Prefix",
+		46:  "mplsTopLabelType",
+		47:  "mplsTopLabelIPv4Address",
+		48:  "samplerId",
+		49:  "samplerMode",
+		50:  "samplerRandomInterval",
+		51:  "classId",
+		52:  "minimumTTL",
+		53:  "maximumTTL",
+		54:  "fragmentIdentification",
+		55:  "postIpClassOfService",
+		56:  "sourceMacAddress",
+		57:  "postDestinationMacAddress",
+		58:  "vlanId",
+		59:  "postVlanId",
+		60:  "ipVersion",
+		61:  "flowDirection",
+		62:  "ipNextHopIPv6Address",
+		63:  "bgpNextHopIPv6Address",
+		64:  "ipv6ExtensionHeaders",
+		65:  "Assigned for NetFlow v9 compatibility",
+		66:  "Assigned for NetFlow v9 compatibility",
+		67:  "Assigned for NetFlow v9 compatibility",
+		68:  "Assigned for NetFlow v9 compatibility",
+		69:  "Assigned for NetFlow v9 compatibility",
+		70:  "mplsTopLabelStackSection",
+		71:  "mplsLabelStackSection2",
+		72:  "mplsLabelStackSection3",
+		73:  "mplsLabelStackSection4",
+		74:  "mplsLabelStackSection5",
+		75:  "mplsLabelStackSection6",
+		76:  "mplsLabelStackSection7",
+		77:  "mplsLabelStackSection8",
+		78:  "mplsLabelStackSection9",
+		79:  "mplsLabelStackSection10",
+		80:  "destinationMacAddress",
+		81:  "postSourceMacAddress",
+		82:  "interfaceName",
+		83:  "interfaceDescription",
+		84:  "samplerName",
+		85:  "octetTotalCount",
+		86:  "packetTotalCount",
+		87:  "flagsAndSamplerId",
+		88:  "fragmentOffset",
+		89:  "forwardingStatus",
+		90:  "mplsVpnRouteDistinguisher",
+		91:  "mplsTopLabelPrefixLength",
+		92:  "srcTrafficIndex",
+		93:  "dstTrafficIndex",
+		94:  "applicationDescription",
+		95:  "applicationId",
+		96:  "applicationName",
+		97:  "Assigned for NetFlow v9 compatibility",
+		98:  "postIpDiffServCodePoint",
+		99:  "multicastReplicationFactor",
+		100: "className",
+		101: "classificationEngineId",
+		102: "layer2packetSectionOffset",
+		103: "layer2packetSectionSize",
+		104: "layer2packetSectionData",
+		128: "bgpNextAdjacentAsNumber",
+		129: "bgpPrevAdjacentAsNumber",
+		130: "exporterIPv4Address",
+		131: "exporterIPv6Address",
+		132: "droppedOctetDeltaCount",
+		133: "droppedPacketDeltaCount",
+		134: "droppedOctetTotalCount",
+		135: "droppedPacketTotalCount",
+		136: "flowEndReason",
+		137: "commonPropertiesId",
+		138: "observationPointId",
+		139: "icmpTypeCodeIPv6",
+		140: "mplsTopLabelIPv6Address",
+		141: "lineCardId",
+		142: "portId",
+		143: "meteringProcessId",
+		144: "exportingProcessId",
+		145: "templateId",
+		146: "wlanChannelId",
+		147: "wlanSSID",
+		148: "flowId",
+		149: "observationDomainId",
+		150: "flowStartSeconds",
+		151: "flowEndSeconds",
+		152: "flowStartMilliseconds",
+		153: "flowEndMilliseconds",
+		154: "flowStartMicroseconds",
+		155: "flowEndMicroseconds",
+		156: "flowStartNanoseconds",
+		157: "flowEndNanoseconds",
+		158: "flowStartDeltaMicroseconds",
+		159: "flowEndDeltaMicroseconds",
+		160: "systemInitTimeMilliseconds",
+		161: "flowDurationMilliseconds",
+		162: "flowDurationMicroseconds",
+		163: "observedFlowTotalCount",
+		164: "ignoredPacketTotalCount",
+		165: "ignoredOctetTotalCount",
+		166: "notSentFlowTotalCount",
+		167: "notSentPacketTotalCount",
+		168: "notSentOctetTotalCount",
+		169: "destinationIPv6Prefix",
+		170: "sourceIPv6Prefix",
+		171: "postOctetTotalCount",
+		172: "postPacketTotalCount",
+		173: "flowKeyIndicator",
+		174: "postMCastPacketTotalCount",
+		175: "postMCastOctetTotalCount",
+		176: "icmpTypeIPv4",
+		177: "icmpCodeIPv4",
+		178: "icmpTypeIPv6",
+		179: "icmpCodeIPv6",
+		180: "udpSourcePort",
+		181: "udpDestinationPort",
+		182: "tcpSourcePort",
+		183: "tcpDestinationPort",
+		184: "tcpSequenceNumber",
+		185: "tcpAcknowledgementNumber",
+		186: "tcpWindowSize",
+		187: "tcpUrgentPointer",
+		188: "tcpHeaderLength",
+		189: "ipHeaderLength",
+		190: "totalLengthIPv4",
+		191: "payloadLengthIPv6",
+		192: "ipTTL",
+		193: "nextHeaderIPv6",
+		194: "mplsPayloadLength",
+		195: "ipDiffServCodePoint",
+		196: "ipPrecedence",
+		197: "fragmentFlags",
+		198: "octetDeltaSumOfSquares",
+		199: "octetTotalSumOfSquares",
+		200: "mplsTopLabelTTL",
+		201: "mplsLabelStackLength",
+		202: "mplsLabelStackDepth",
+		203: "mplsTopLabelExp",
+		204: "ipPayloadLength",
+		205: "udpMessageLength",
+		206: "isMulticast",
+		207: "ipv4IHL",
+		208: "ipv4Options",
+		209: "tcpOptions",
+		210: "paddingOctets",
+		211: "collectorIPv4Address",
+		212: "collectorIPv6Address",
+		213: "exportInterface",
+		214: "exportProtocolVersion",
+		215: "exportTransportProtocol",
+		216: "collectorTransportPort",
+		217: "exporterTransportPort",
+		218: "tcpSynTotalCount",
+		219: "tcpFinTotalCount",
+		220: "tcpRstTotalCount",
+		221: "tcpPshTotalCount",
+		222: "tcpAckTotalCount",
+		223: "tcpUrgTotalCount",
+		224: "ipTotalLength",
+		225: "postNATSourceIPv4Address",
+		226: "postNATDestinationIPv4Address",
+		227: "postNAPTSourceTransportPort",
+		228: "postNAPTDestinationTransportPort",
+		229: "natOriginatingAddressRealm",
+		230: "natEvent",
+		231: "initiatorOctets",
+		232: "responderOctets",
+		233: "firewallEvent",
+		234: "ingressVRFID",
+		235: "egressVRFID",
+		236: "VRFname",
+		237: "postMplsTopLabelExp",
+		238: "tcpWindowScale",
+		239: "biflowDirection",
+		240: "ethernetHeaderLength",
+		241: "ethernetPayloadLength",
+		242: "ethernetTotalLength",
+		243: "dot1qVlanId",
+		244: "dot1qPriority",
+		245: "dot1qCustomerVlanId",
+		246: "dot1qCustomerPriority",
+		247: "metroEvcId",
+		248: "metroEvcType",
+		249: "pseudoWireId",
+		250: "pseudoWireType",
+		251: "pseudoWireControlWord",
+		252: "ingressPhysicalInterface",
+		253: "egressPhysicalInterface",
+		254: "postDot1qVlanId",
+		255: "postDot1qCustomerVlanId",
+		256: "ethernetType",
+		257: "postIpPrecedence",
+		258: "collectionTimeMilliseconds",
+		259: "exportSctpStreamId",
+		260: "maxExportSeconds",
+		261: "maxFlowEndSeconds",
+		262: "messageMD5Checksum",
+		263: "messageScope",
+		264: "minExportSeconds",
+		265: "minFlowStartSeconds",
+		266: "opaqueOctets",
+		267: "sessionScope",
+		268: "maxFlowEndMicroseconds",
+		269: "maxFlowEndMilliseconds",
+		270: "maxFlowEndNanoseconds",
+		271: "minFlowStartMicroseconds",
+		272: "minFlowStartMilliseconds",
+		273: "minFlowStartNanoseconds",
+		274: "collectorCertificate",
+		275: "exporterCertificate",
+		276: "dataRecordsReliability",
+		277: "observationPointType",
+		278: "newConnectionDeltaCount",
+		279: "connectionSumDurationSeconds",
+		280: "connectionTransactionId",
+		281: "postNATSourceIPv6Address",
+		282: "postNATDestinationIPv6Address",
+		283: "natPoolId",
+		284: "natPoolName",
+		285: "anonymizationFlags",
+		286: "anonymizationTechnique",
+		287: "informationElementIndex",
+		288: "p2pTechnology",
+		289: "tunnelTechnology",
+		290: "encryptedTechnology",
+		291: "basicList",
+		292: "subTemplateList",
+		293: "subTemplateMultiList",
+		294: "bgpValidityState",
+		295: "IPSecSPI",
+		296: "greKey",
+		297: "natType",
+		298: "initiatorPackets",
+		299: "responderPackets",
+		300: "observationDomainName",
+		301: "selectionSequenceId",
+		302: "selectorId",
+		303: "informationElementId",
+		304: "selectorAlgorithm",
+		305: "samplingPacketInterval",
+		306: "samplingPacketSpace",
+		307: "samplingTimeInterval",
+		308: "samplingTimeSpace",
+		309: "samplingSize",
+		310: "samplingPopulation",
+		311: "samplingProbability",
+		312: "dataLinkFrameSize",
+		313: "ipHeaderPacketSection",
+		314: "ipPayloadPacketSection",
+		315: "dataLinkFrameSection",
+		316: "mplsLabelStackSection",
+		317: "mplsPayloadPacketSection",
+		318: "selectorIdTotalPktsObserved",
+		319: "selectorIdTotalPktsSelected",
+		320: "absoluteError",
+		321: "relativeError",
+		322: "observationTimeSeconds",
+		323: "observationTimeMilliseconds",
+		324: "observationTimeMicroseconds",
+		325: "observationTimeNanoseconds",
+		326: "digestHashValue",
+		327: "hashIPPayloadOffset",
+		328: "hashIPPayloadSize",
+		329: "hashOutputRangeMin",
+		330: "hashOutputRangeMax",
+		331: "hashSelectedRangeMin",
+		332: "hashSelectedRangeMax",
+		333: "hashDigestOutput",
+		334: "hashInitialiserValue",
+		335: "selectorName",
+		336: "upperCILimit",
+		337: "lowerCILimit",
+		338: "confidenceLevel",
+		339: "informationElementDataType",
+		340: "informationElementDescription",
+		341: "informationElementName",
+		342: "informationElementRangeBegin",
+		343: "informationElementRangeEnd",
+		344: "informationElementSemantics",
+		345: "informationElementUnits",
+		346: "privateEnterpriseNumber",
+		347: "virtualStationInterfaceId",
+		348: "virtualStationInterfaceName",
+		349: "virtualStationUUID",
+		350: "virtualStationName",
+		351: "layer2SegmentId",
+		352: "layer2OctetDeltaCount",
+		353: "layer2OctetTotalCount",
+		354: "ingressUnicastPacketTotalCount",
+		355: "ingressMulticastPacketTotalCount",
+		356: "ingressBroadcastPacketTotalCount",
+		357: "egressUnicastPacketTotalCount",
+		358: "egressBroadcastPacketTotalCount",
+		359: "monitoringIntervalStartMilliSeconds",
+		360: "monitoringIntervalEndMilliSeconds",
+		361: "portRangeStart",
+		362: "portRangeEnd",
+		363: "portRangeStepSize",
+		364: "portRangeNumPorts",
+		365: "staMacAddress",
+		366: "staIPv4Address",
+		367: "wtpMacAddress",
+		368: "ingressInterfaceType",
+		369: "egressInterfaceType",
+		370: "rtpSequenceNumber",
+		371: "userName",
+		372: "applicationCategoryName",
+		373: "applicationSubCategoryName",
+		374: "applicationGroupName",
+		375: "originalFlowsPresent",
+		376: "originalFlowsInitiated",
+		377: "originalFlowsCompleted",
+		378: "distinctCountOfSourceIPAddress",
+		379: "distinctCountOfDestinationIPAddress",
+		380: "distinctCountOfSourceIPv4Address",
+		381: "distinctCountOfDestinationIPv4Address",
+		382: "distinctCountOfSourceIPv6Address",
+		383: "distinctCountOfDestinationIPv6Address",
+		384: "valueDistributionMethod",
+		385: "rfc3550JitterMilliseconds",
+		386: "rfc3550JitterMicroseconds",
+		387: "rfc3550JitterNanoseconds",
+		388: "dot1qDEI",
+		389: "dot1qCustomerDEI",
+		390: "flowSelectorAlgorithm",
+		391: "flowSelectedOctetDeltaCount",
+		392: "flowSelectedPacketDeltaCount",
+		393: "flowSelectedFlowDeltaCount",
+		394: "selectorIDTotalFlowsObserved",
+		395: "selectorIDTotalFlowsSelected",
+		396: "samplingFlowInterval",
+		397: "samplingFlowSpacing",
+		398: "flowSamplingTimeInterval",
+		399: "flowSamplingTimeSpacing",
+		400: "hashFlowDomain",
+		401: "transportOctetDeltaCount",
+		402: "transportPacketDeltaCount",
+		403: "originalExporterIPv4Address",
+		404: "originalExporterIPv6Address",
+		405: "originalObservationDomainId",
+		406: "intermediateProcessId",
+		407: "ignoredDataRecordTotalCount",
+		408: "dataLinkFrameType",
+		409: "sectionOffset",
+		410: "sectionExportedOctets",
+		411: "dot1qServiceInstanceTag",
+		412: "dot1qServiceInstanceId",
+		413: "dot1qServiceInstancePriority",
+		414: "dot1qCustomerSourceMacAddress",
+		415: "dot1qCustomerDestinationMacAddress",
+		416: "",
+		417: "postLayer2OctetDeltaCount",
+		418: "postMCastLayer2OctetDeltaCount",
+		419: "",
+		420: "postLayer2OctetTotalCount",
+		421: "postMCastLayer2OctetTotalCount",
+		422: "minimumLayer2TotalLength",
+		423: "maximumLayer2TotalLength",
+		424: "droppedLayer2OctetDeltaCount",
+		425: "droppedLayer2OctetTotalCount",
+		426: "ignoredLayer2OctetTotalCount",
+		427: "notSentLayer2OctetTotalCount",
+		428: "layer2OctetDeltaSumOfSquares",
+		429: "layer2OctetTotalSumOfSquares",
+		430: "layer2FrameDeltaCount",
+		431: "layer2FrameTotalCount",
+		432: "pseudoWireDestinationIPv4Address",
+		433: "ignoredLayer2FrameTotalCount",
+		434: "mibObjectValueInteger",
+		435: "mibObjectValueOctetString",
+		436: "mibObjectValueOID",
+		437: "mibObjectValueBits",
+		438: "mibObjectValueIPAddress",
+		439: "mibObjectValueCounter",
+		440: "mibObjectValueGauge",
+		441: "mibObjectValueTimeTicks",
+		442: "mibObjectValueUnsigned",
+		443: "mibObjectValueTable",
+		444: "mibObjectValueRow",
+		445: "mibObjectIdentifier",
+		446: "mibSubIdentifier",
+		447: "mibIndexIndicator",
+		448: "mibCaptureTimeSemantics",
+		449: "mibContextEngineID",
+		450: "mibContextName",
+		451: "mibObjectName",
+		452: "mibObjectDescription",
+		453: "mibObjectSyntax",
+		454: "mibModuleName",
+		455: "mobileIMSI",
+		456: "mobileMSISDN",
+		457: "httpStatusCode",
+		458: "sourceTransportPortsLimit",
+		459: "httpRequestMethod",
+		460: "httpRequestHost",
+		461: "httpRequestTarget",
+		462: "httpMessageVersion",
+		463: "natInstanceID",
+		464: "internalAddressRealm",
+		465: "externalAddressRealm",
+		466: "natQuotaExceededEvent",
+		467: "natThresholdEvent",
+	}
+
+	if typeId >= 105 && typeId <= 127 {
+		return "Assigned for NetFlow v9 compatibility"
+	} else if typeId >= 468 && typeId <= 32767 {
+		return "Unassigned"
+	} else {
+		return nameList[typeId]
+	}
+}
+
+func (flowSet IPFIXOptionsTemplateFlowSet) String(TypeToString func(uint16) string) string {
+	str := fmt.Sprintf("       Id %v\n", flowSet.Id)
+	str += fmt.Sprintf("       Length: %v\n", flowSet.Length)
+	str += fmt.Sprintf("       Records (%v records):\n", len(flowSet.Records))
+
+	for j, record := range flowSet.Records {
+		str += fmt.Sprintf("       - Record %v:\n", j)
+		str += fmt.Sprintf("            TemplateId: %v\n", record.TemplateId)
+		str += fmt.Sprintf("            FieldCount: %v\n", record.FieldCount)
+		str += fmt.Sprintf("            ScopeFieldCount: %v\n", record.ScopeFieldCount)
+
+		str += fmt.Sprintf("            Scopes (%v):\n", len(record.Scopes))
+
+		for k, field := range record.Scopes {
+			str += fmt.Sprintf("            - %v. %v (%v): %v\n", k, TypeToString(field.Type), field.Type, field.Length)
+		}
+
+		str += fmt.Sprintf("            Options (%v):\n", len(record.Options))
+
+		for k, field := range record.Options {
+			str += fmt.Sprintf("            - %v. %v (%v): %v\n", k, TypeToString(field.Type), field.Type, field.Length)
+		}
+
+	}
+
+	return str
+}
+
+func (p IPFIXPacket) String() string {
+	str := "Flow Packet\n"
+	str += "------------\n"
+	str += fmt.Sprintf("  Version: %v\n", p.Version)
+	str += fmt.Sprintf("  Length:  %v\n", p.Length)
+
+	exportTime := time.Unix(int64(p.ExportTime), 0)
+	str += fmt.Sprintf("  ExportTime: %v\n", exportTime.String())
+	str += fmt.Sprintf("  SequenceNumber: %v\n", p.SequenceNumber)
+	str += fmt.Sprintf("  ObservationDomainId: %v\n", p.ObservationDomainId)
+	str += fmt.Sprintf("  FlowSets (%v):\n", len(p.FlowSets))
+
+	for i, flowSet := range p.FlowSets {
+		switch flowSet := flowSet.(type) {
+		case TemplateFlowSet:
+			str += fmt.Sprintf("    - TemplateFlowSet %v:\n", i)
+			str += flowSet.String(IPFIXTypeToString)
+		case IPFIXOptionsTemplateFlowSet:
+			str += fmt.Sprintf("    - OptionsTemplateFlowSet %v:\n", i)
+			str += flowSet.String(IPFIXTypeToString)
+		case DataFlowSet:
+			str += fmt.Sprintf("    - DataFlowSet %v:\n", i)
+			str += flowSet.String(IPFIXTypeToString)
+		case OptionsDataFlowSet:
+			str += fmt.Sprintf("    - OptionsDataFlowSet %v:\n", i)
+			str += flowSet.String(IPFIXTypeToString, IPFIXTypeToString)
+		default:
+			str += fmt.Sprintf("    - (unknown type) %v: %v\n", i, flowSet)
+		}
+	}
+
+	return str
+}

decoders/netflow/netflow.go

@@ -0,0 +1,507 @@
+package netflow
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"sync"
+
+	"github.com/netsampler/goflow2/decoders/utils"
+)
+
+type FlowBaseTemplateSet map[uint16]map[uint32]map[uint16]interface{}
+
+type NetFlowTemplateSystem interface {
+	GetTemplate(version uint16, obsDomainId uint32, templateId uint16) (interface{}, error)
+	AddTemplate(version uint16, obsDomainId uint32, template interface{})
+}
+
+func DecodeNFv9OptionsTemplateSet(payload *bytes.Buffer) ([]NFv9OptionsTemplateRecord, error) {
+	records := make([]NFv9OptionsTemplateRecord, 0)
+	var err error
+	for payload.Len() >= 4 {
+		optsTemplateRecord := NFv9OptionsTemplateRecord{}
+		err = utils.BinaryDecoder(payload, &optsTemplateRecord.TemplateId, &optsTemplateRecord.ScopeLength, &optsTemplateRecord.OptionLength)
+		if err != nil {
+			break
+		}
+
+		sizeScope := int(optsTemplateRecord.ScopeLength) / 4
+		sizeOptions := int(optsTemplateRecord.OptionLength) / 4
+		if sizeScope < 0 || sizeOptions < 0 {
+			return records, NewErrorDecodingNetFlow("Error decoding OptionsTemplateSet: negative length.")
+		}
+
+		fields := make([]Field, sizeScope)
+		for i := 0; i < sizeScope; i++ {
+			field := Field{}
+			err = utils.BinaryDecoder(payload, &field)
+			fields[i] = field
+		}
+		optsTemplateRecord.Scopes = fields
+
+		fields = make([]Field, sizeOptions)
+		for i := 0; i < sizeOptions; i++ {
+			field := Field{}
+			err = utils.BinaryDecoder(payload, &field)
+			fields[i] = field
+		}
+		optsTemplateRecord.Options = fields
+
+		records = append(records, optsTemplateRecord)
+	}
+
+	return records, nil
+}
+
+func DecodeIPFIXOptionsTemplateSet(payload *bytes.Buffer) ([]IPFIXOptionsTemplateRecord, error) {
+	records := make([]IPFIXOptionsTemplateRecord, 0)
+	var err error
+	for payload.Len() >= 4 {
+		optsTemplateRecord := IPFIXOptionsTemplateRecord{}
+		err = utils.BinaryDecoder(payload, &optsTemplateRecord.TemplateId, &optsTemplateRecord.FieldCount, &optsTemplateRecord.ScopeFieldCount)
+		if err != nil {
+			break
+		}
+
+		fields := make([]Field, int(optsTemplateRecord.ScopeFieldCount))
+		for i := 0; i < int(optsTemplateRecord.ScopeFieldCount); i++ {
+			field := Field{}
+			err = utils.BinaryDecoder(payload, &field)
+			fields[i] = field
+		}
+		optsTemplateRecord.Scopes = fields
+
+		optionsSize := int(optsTemplateRecord.FieldCount) - int(optsTemplateRecord.ScopeFieldCount)
+		if optionsSize < 0 {
+			return records, NewErrorDecodingNetFlow("Error decoding OptionsTemplateSet: negative length.")
+		}
+		fields = make([]Field, optionsSize)
+		for i := 0; i < optionsSize; i++ {
+			field := Field{}
+			err = utils.BinaryDecoder(payload, &field)
+			fields[i] = field
+		}
+		optsTemplateRecord.Options = fields
+
+		records = append(records, optsTemplateRecord)
+	}
+
+	return records, nil
+}
+
+func DecodeTemplateSet(version uint16, payload *bytes.Buffer) ([]TemplateRecord, error) {
+	records := make([]TemplateRecord, 0)
+	var err error
+	for payload.Len() >= 4 {
+		templateRecord := TemplateRecord{}
+		err = utils.BinaryDecoder(payload, &templateRecord.TemplateId, &templateRecord.FieldCount)
+		if err != nil {
+			break
+		}
+
+		if int(templateRecord.FieldCount) < 0 {
+			return records, NewErrorDecodingNetFlow("Error decoding TemplateSet: zero count.")
+		}
+
+		fields := make([]Field, int(templateRecord.FieldCount))
+		for i := 0; i < int(templateRecord.FieldCount); i++ {
+			field := Field{}
+			err = utils.BinaryDecoder(payload, &field.Type)
+			err = utils.BinaryDecoder(payload, &field.Length)
+
+			if version == 10 && field.Type&0x8000 != 0 {
+				field.PenProvided = true
+				err = utils.BinaryDecoder(payload, &field.Pen)
+			}
+			fields[i] = field
+		}
+		templateRecord.Fields = fields
+		records = append(records, templateRecord)
+	}
+
+	return records, nil
+}
+
+func GetTemplateSize(template []Field) int {
+	sum := 0
+	for _, templateField := range template {
+		sum += int(templateField.Length)
+	}
+	return sum
+}
+
+func DecodeDataSetUsingFields(version uint16, payload *bytes.Buffer, listFields []Field) []DataField {
+	for payload.Len() >= GetTemplateSize(listFields) {
+
+		dataFields := make([]DataField, len(listFields))
+
+		for i, templateField := range listFields {
+
+			finalLength := int(templateField.Length)
+			if version == 10 && templateField.Length == 0xffff {
+				var variableLen8 byte
+				var variableLen16 uint16
+				utils.BinaryDecoder(payload, &variableLen8)
+				if variableLen8 == 0xff {
+					utils.BinaryDecoder(payload, &variableLen16)
+					finalLength = int(variableLen16)
+				} else {
+					finalLength = int(variableLen8)
+				}
+
+			}
+
+			value := payload.Next(finalLength)
+			nfvalue := DataField{
+				Type:  templateField.Type,
+				Value: value,
+			}
+			dataFields[i] = nfvalue
+		}
+		return dataFields
+	}
+	return []DataField{}
+}
+
+type ErrorTemplateNotFound struct {
+	version      uint16
+	obsDomainId  uint32
+	templateId   uint16
+	typeTemplate string
+}
+
+func NewErrorTemplateNotFound(version uint16, obsDomainId uint32, templateId uint16, typeTemplate string) *ErrorTemplateNotFound {
+	return &ErrorTemplateNotFound{
+		version:      version,
+		obsDomainId:  obsDomainId,
+		templateId:   templateId,
+		typeTemplate: typeTemplate,
+	}
+}
+
+func (e *ErrorTemplateNotFound) Error() string {
+	return fmt.Sprintf("No %v template %v found for and domain id %v", e.typeTemplate, e.templateId, e.obsDomainId)
+}
+
+type ErrorVersion struct {
+	version uint16
+}
+
+func NewErrorVersion(version uint16) *ErrorVersion {
+	return &ErrorVersion{
+		version: version,
+	}
+}
+
+func (e *ErrorVersion) Error() string {
+	return fmt.Sprintf("Unknown NetFlow version %v (only decodes v9 and v10/IPFIX)", e.version)
+}
+
+type ErrorFlowId struct {
+	id uint16
+}
+
+func NewErrorFlowId(id uint16) *ErrorFlowId {
+	return &ErrorFlowId{
+		id: id,
+	}
+}
+
+func (e *ErrorFlowId) Error() string {
+	return fmt.Sprintf("Unknown flow id %v (templates < 256, data >= 256)", e.id)
+}
+
+type ErrorDecodingNetFlow struct {
+	msg string
+}
+
+func NewErrorDecodingNetFlow(msg string) *ErrorDecodingNetFlow {
+	return &ErrorDecodingNetFlow{
+		msg: msg,
+	}
+}
+
+func (e *ErrorDecodingNetFlow) Error() string {
+	return fmt.Sprintf("Error decoding NetFlow: %v", e.msg)
+}
+
+func DecodeOptionsDataSet(version uint16, payload *bytes.Buffer, listFieldsScopes, listFieldsOption []Field) ([]OptionsDataRecord, error) {
+	records := make([]OptionsDataRecord, 0)
+
+	listFieldsScopesSize := GetTemplateSize(listFieldsScopes)
+	listFieldsOptionSize := GetTemplateSize(listFieldsOption)
+
+	for payload.Len() >= listFieldsScopesSize+listFieldsOptionSize {
+		payloadLim := bytes.NewBuffer(payload.Next(listFieldsScopesSize))
+		scopeValues := DecodeDataSetUsingFields(version, payloadLim, listFieldsScopes)
+		payloadLim = bytes.NewBuffer(payload.Next(listFieldsOptionSize))
+		optionValues := DecodeDataSetUsingFields(version, payloadLim, listFieldsOption)
+
+		record := OptionsDataRecord{
+			ScopesValues:  scopeValues,
+			OptionsValues: optionValues,
+		}
+
+		records = append(records, record)
+	}
+	return records, nil
+}
+
+func DecodeDataSet(version uint16, payload *bytes.Buffer, listFields []Field) ([]DataRecord, error) {
+	records := make([]DataRecord, 0)
+
+	listFieldsSize := GetTemplateSize(listFields)
+	for payload.Len() >= listFieldsSize {
+		payloadLim := bytes.NewBuffer(payload.Next(listFieldsSize))
+		values := DecodeDataSetUsingFields(version, payloadLim, listFields)
+
+		record := DataRecord{
+			Values: values,
+		}
+
+		records = append(records, record)
+	}
+	return records, nil
+}
+
+func (ts *BasicTemplateSystem) GetTemplates() map[uint16]map[uint32]map[uint16]interface{} {
+	ts.templateslock.RLock()
+	tmp := ts.templates
+	ts.templateslock.RUnlock()
+	return tmp
+}
+
+func (ts *BasicTemplateSystem) AddTemplate(version uint16, obsDomainId uint32, template interface{}) {
+	ts.templateslock.Lock()
+	_, exists := ts.templates[version]
+	if exists != true {
+		ts.templates[version] = make(map[uint32]map[uint16]interface{})
+	}
+	_, exists = ts.templates[version][obsDomainId]
+	if exists != true {
+		ts.templates[version][obsDomainId] = make(map[uint16]interface{})
+	}
+	var templateId uint16
+	switch templateIdConv := template.(type) {
+	case IPFIXOptionsTemplateRecord:
+		templateId = templateIdConv.TemplateId
+	case NFv9OptionsTemplateRecord:
+		templateId = templateIdConv.TemplateId
+	case TemplateRecord:
+		templateId = templateIdConv.TemplateId
+	}
+	ts.templates[version][obsDomainId][templateId] = template
+	ts.templateslock.Unlock()
+}
+
+func (ts *BasicTemplateSystem) GetTemplate(version uint16, obsDomainId uint32, templateId uint16) (interface{}, error) {
+	ts.templateslock.RLock()
+	templatesVersion, okver := ts.templates[version]
+	if okver {
+		templatesObsDom, okobs := templatesVersion[obsDomainId]
+		if okobs {
+			template, okid := templatesObsDom[templateId]
+			if okid {
+				ts.templateslock.RUnlock()
+				return template, nil
+			}
+			ts.templateslock.RUnlock()
+			return nil, NewErrorTemplateNotFound(version, obsDomainId, templateId, "info")
+		}
+		ts.templateslock.RUnlock()
+		return nil, NewErrorTemplateNotFound(version, obsDomainId, templateId, "info")
+	}
+	ts.templateslock.RUnlock()
+	return nil, NewErrorTemplateNotFound(version, obsDomainId, templateId, "info")
+}
+
+type BasicTemplateSystem struct {
+	templates     FlowBaseTemplateSet
+	templateslock *sync.RWMutex
+}
+
+func CreateTemplateSystem() *BasicTemplateSystem {
+	ts := &BasicTemplateSystem{
+		templates:     make(FlowBaseTemplateSet),
+		templateslock: &sync.RWMutex{},
+	}
+	return ts
+}
+
+func DecodeMessage(payload *bytes.Buffer, templates NetFlowTemplateSystem) (interface{}, error) {
+	var size uint16
+	packetNFv9 := NFv9Packet{}
+	packetIPFIX := IPFIXPacket{}
+	var returnItem interface{}
+
+	var version uint16
+	var obsDomainId uint32
+	binary.Read(payload, binary.BigEndian, &version)
+
+	if version == 9 {
+		utils.BinaryDecoder(payload, &packetNFv9.Count, &packetNFv9.SystemUptime, &packetNFv9.UnixSeconds, &packetNFv9.SequenceNumber, &packetNFv9.SourceId)
+		size = packetNFv9.Count
+		packetNFv9.Version = version
+		returnItem = *(&packetNFv9)
+		obsDomainId = packetNFv9.SourceId
+	} else if version == 10 {
+		utils.BinaryDecoder(payload, &packetIPFIX.Length, &packetIPFIX.ExportTime, &packetIPFIX.SequenceNumber, &packetIPFIX.ObservationDomainId)
+		size = packetIPFIX.Length
+		packetIPFIX.Version = version
+		returnItem = *(&packetIPFIX)
+		obsDomainId = packetIPFIX.ObservationDomainId
+	} else {
+		return nil, NewErrorVersion(version)
+	}
+
+	for i := 0; ((i < int(size) && version == 9) || version == 10) && payload.Len() > 0; i++ {
+		fsheader := FlowSetHeader{}
+		utils.BinaryDecoder(payload, &fsheader)
+
+		nextrelpos := int(fsheader.Length) - binary.Size(fsheader)
+		if nextrelpos < 0 {
+			return returnItem, NewErrorDecodingNetFlow("Error decoding packet: non-terminated stream.")
+		}
+
+		var flowSet interface{}
+
+		if fsheader.Id == 0 && version == 9 {
+			templateReader := bytes.NewBuffer(payload.Next(nextrelpos))
+			records, err := DecodeTemplateSet(version, templateReader)
+			if err != nil {
+				return returnItem, err
+			}
+			templatefs := TemplateFlowSet{
+				FlowSetHeader: fsheader,
+				Records:       records,
+			}
+
+			flowSet = templatefs
+
+			if templates != nil {
+				for _, record := range records {
+					templates.AddTemplate(version, obsDomainId, record)
+				}
+			}
+
+		} else if fsheader.Id == 1 && version == 9 {
+			templateReader := bytes.NewBuffer(payload.Next(nextrelpos))
+			records, err := DecodeNFv9OptionsTemplateSet(templateReader)
+			if err != nil {
+				return returnItem, err
+			}
+			optsTemplatefs := NFv9OptionsTemplateFlowSet{
+				FlowSetHeader: fsheader,
+				Records:       records,
+			}
+			flowSet = optsTemplatefs
+
+			if templates != nil {
+				for _, record := range records {
+					templates.AddTemplate(version, obsDomainId, record)
+				}
+			}
+
+		} else if fsheader.Id == 2 && version == 10 {
+			templateReader := bytes.NewBuffer(payload.Next(nextrelpos))
+			records, err := DecodeTemplateSet(version, templateReader)
+			if err != nil {
+				return returnItem, err
+			}
+			templatefs := TemplateFlowSet{
+				FlowSetHeader: fsheader,
+				Records:       records,
+			}
+			flowSet = templatefs
+
+			if templates != nil {
+				for _, record := range records {
+					templates.AddTemplate(version, obsDomainId, record)
+				}
+			}
+
+		} else if fsheader.Id == 3 && version == 10 {
+			templateReader := bytes.NewBuffer(payload.Next(nextrelpos))
+			records, err := DecodeIPFIXOptionsTemplateSet(templateReader)
+			if err != nil {
+				return returnItem, err
+			}
+			optsTemplatefs := IPFIXOptionsTemplateFlowSet{
+				FlowSetHeader: fsheader,
+				Records:       records,
+			}
+			flowSet = optsTemplatefs
+
+			if templates != nil {
+				for _, record := range records {
+					templates.AddTemplate(version, obsDomainId, record)
+				}
+			}
+
+		} else if fsheader.Id >= 256 {
+			dataReader := bytes.NewBuffer(payload.Next(nextrelpos))
+
+			if templates == nil {
+				continue
+			}
+
+			template, err := templates.GetTemplate(version, obsDomainId, fsheader.Id)
+
+			if err == nil {
+				switch templatec := template.(type) {
+				case TemplateRecord:
+					records, err := DecodeDataSet(version, dataReader, templatec.Fields)
+					if err != nil {
+						return returnItem, err
+					}
+					datafs := DataFlowSet{
+						FlowSetHeader: fsheader,
+						Records:       records,
+					}
+					flowSet = datafs
+				case IPFIXOptionsTemplateRecord:
+					records, err := DecodeOptionsDataSet(version, dataReader, templatec.Scopes, templatec.Options)
+					if err != nil {
+						return returnItem, err
+					}
+
+					datafs := OptionsDataFlowSet{
+						FlowSetHeader: fsheader,
+						Records:       records,
+					}
+					flowSet = datafs
+				case NFv9OptionsTemplateRecord:
+					records, err := DecodeOptionsDataSet(version, dataReader, templatec.Scopes, templatec.Options)
+					if err != nil {
+						return returnItem, err
+					}
+
+					datafs := OptionsDataFlowSet{
+						FlowSetHeader: fsheader,
+						Records:       records,
+					}
+					flowSet = datafs
+				}
+			} else {
+				return returnItem, err
+			}
+		} else {
+			return returnItem, NewErrorFlowId(fsheader.Id)
+		}
+
+		if version == 9 && flowSet != nil {
+			packetNFv9.FlowSets = append(packetNFv9.FlowSets, flowSet)
+		} else if version == 10 && flowSet != nil {
+			packetIPFIX.FlowSets = append(packetIPFIX.FlowSets, flowSet)
+		}
+	}
+
+	if version == 9 {
+		return packetNFv9, nil
+	} else if version == 10 {
+		return packetIPFIX, nil
+	} else {
+		return returnItem, NewErrorVersion(version)
+	}
+}

decoders/netflow/nfv9.go

@@ -0,0 +1,317 @@
+package netflow
+
+import (
+	"fmt"
+	"time"
+)
+
+const (
+	NFV9_FIELD_IN_BYTES                     = 1
+	NFV9_FIELD_IN_PKTS                      = 2
+	NFV9_FIELD_FLOWS                        = 3
+	NFV9_FIELD_PROTOCOL                     = 4
+	NFV9_FIELD_SRC_TOS                      = 5
+	NFV9_FIELD_TCP_FLAGS                    = 6
+	NFV9_FIELD_L4_SRC_PORT                  = 7
+	NFV9_FIELD_IPV4_SRC_ADDR                = 8
+	NFV9_FIELD_SRC_MASK                     = 9
+	NFV9_FIELD_INPUT_SNMP                   = 10
+	NFV9_FIELD_L4_DST_PORT                  = 11
+	NFV9_FIELD_IPV4_DST_ADDR                = 12
+	NFV9_FIELD_DST_MASK                     = 13
+	NFV9_FIELD_OUTPUT_SNMP                  = 14
+	NFV9_FIELD_IPV4_NEXT_HOP                = 15
+	NFV9_FIELD_SRC_AS                       = 16
+	NFV9_FIELD_DST_AS                       = 17
+	NFV9_FIELD_BGP_IPV4_NEXT_HOP            = 18
+	NFV9_FIELD_MUL_DST_PKTS                 = 19
+	NFV9_FIELD_MUL_DST_BYTES                = 20
+	NFV9_FIELD_LAST_SWITCHED                = 21
+	NFV9_FIELD_FIRST_SWITCHED               = 22
+	NFV9_FIELD_OUT_BYTES                    = 23
+	NFV9_FIELD_OUT_PKTS                     = 24
+	NFV9_FIELD_MIN_PKT_LNGTH                = 25
+	NFV9_FIELD_MAX_PKT_LNGTH                = 26
+	NFV9_FIELD_IPV6_SRC_ADDR                = 27
+	NFV9_FIELD_IPV6_DST_ADDR                = 28
+	NFV9_FIELD_IPV6_SRC_MASK                = 29
+	NFV9_FIELD_IPV6_DST_MASK                = 30
+	NFV9_FIELD_IPV6_FLOW_LABEL              = 31
+	NFV9_FIELD_ICMP_TYPE                    = 32
+	NFV9_FIELD_MUL_IGMP_TYPE                = 33
+	NFV9_FIELD_SAMPLING_INTERVAL            = 34
+	NFV9_FIELD_SAMPLING_ALGORITHM           = 35
+	NFV9_FIELD_FLOW_ACTIVE_TIMEOUT          = 36
+	NFV9_FIELD_FLOW_INACTIVE_TIMEOUT        = 37
+	NFV9_FIELD_ENGINE_TYPE                  = 38
+	NFV9_FIELD_ENGINE_ID                    = 39
+	NFV9_FIELD_TOTAL_BYTES_EXP              = 40
+	NFV9_FIELD_TOTAL_PKTS_EXP               = 41
+	NFV9_FIELD_TOTAL_FLOWS_EXP              = 42
+	NFV9_FIELD_IPV4_SRC_PREFIX              = 44
+	NFV9_FIELD_IPV4_DST_PREFIX              = 45
+	NFV9_FIELD_MPLS_TOP_LABEL_TYPE          = 46
+	NFV9_FIELD_MPLS_TOP_LABEL_IP_ADDR       = 47
+	NFV9_FIELD_FLOW_SAMPLER_ID              = 48
+	NFV9_FIELD_FLOW_SAMPLER_MODE            = 49
+	NFV9_FIELD_FLOW_SAMPLER_RANDOM_INTERVAL = 50
+	NFV9_FIELD_MIN_TTL                      = 52
+	NFV9_FIELD_MAX_TTL                      = 53
+	NFV9_FIELD_IPV4_IDENT                   = 54
+	NFV9_FIELD_DST_TOS                      = 55
+	NFV9_FIELD_IN_SRC_MAC                   = 56
+	NFV9_FIELD_OUT_DST_MAC                  = 57
+	NFV9_FIELD_SRC_VLAN                     = 58
+	NFV9_FIELD_DST_VLAN                     = 59
+	NFV9_FIELD_IP_PROTOCOL_VERSION          = 60
+	NFV9_FIELD_DIRECTION                    = 61
+	NFV9_FIELD_IPV6_NEXT_HOP                = 62
+	NFV9_FIELD_BGP_IPV6_NEXT_HOP            = 63
+	NFV9_FIELD_IPV6_OPTION_HEADERS          = 64
+	NFV9_FIELD_MPLS_LABEL_1                 = 70
+	NFV9_FIELD_MPLS_LABEL_2                 = 71
+	NFV9_FIELD_MPLS_LABEL_3                 = 72
+	NFV9_FIELD_MPLS_LABEL_4                 = 73
+	NFV9_FIELD_MPLS_LABEL_5                 = 74
+	NFV9_FIELD_MPLS_LABEL_6                 = 75
+	NFV9_FIELD_MPLS_LABEL_7                 = 76
+	NFV9_FIELD_MPLS_LABEL_8                 = 77
+	NFV9_FIELD_MPLS_LABEL_9                 = 78
+	NFV9_FIELD_MPLS_LABEL_10                = 79
+	NFV9_FIELD_IN_DST_MAC                   = 80
+	NFV9_FIELD_OUT_SRC_MAC                  = 81
+	NFV9_FIELD_IF_NAME                      = 82
+	NFV9_FIELD_IF_DESC                      = 83
+	NFV9_FIELD_SAMPLER_NAME                 = 84
+	NFV9_FIELD_IN_PERMANENT_BYTES           = 85
+	NFV9_FIELD_IN_PERMANENT_PKTS            = 86
+	NFV9_FIELD_FRAGMENT_OFFSET              = 88
+	NFV9_FIELD_FORWARDING_STATUS            = 89
+	NFV9_FIELD_MPLS_PAL_RD                  = 90
+	NFV9_FIELD_MPLS_PREFIX_LEN              = 91
+	NFV9_FIELD_SRC_TRAFFIC_INDEX            = 92
+	NFV9_FIELD_DST_TRAFFIC_INDEX            = 93
+	NFV9_FIELD_APPLICATION_DESCRIPTION      = 94
+	NFV9_FIELD_APPLICATION_TAG              = 95
+	NFV9_FIELD_APPLICATION_NAME             = 96
+	NFV9_FIELD_postipDiffServCodePoint      = 98
+	NFV9_FIELD_replication_factor           = 99
+	NFV9_FIELD_layer2packetSectionOffset    = 102
+	NFV9_FIELD_layer2packetSectionSize      = 103
+	NFV9_FIELD_layer2packetSectionData      = 104
+)
+
+type NFv9Packet struct {
+	Version        uint16
+	Count          uint16
+	SystemUptime   uint32
+	UnixSeconds    uint32
+	SequenceNumber uint32
+	SourceId       uint32
+	FlowSets       []interface{}
+}
+
+type NFv9OptionsTemplateFlowSet struct {
+	FlowSetHeader
+	Records []NFv9OptionsTemplateRecord
+}
+
+type NFv9OptionsTemplateRecord struct {
+	TemplateId   uint16
+	ScopeLength  uint16
+	OptionLength uint16
+	Scopes       []Field
+	Options      []Field
+}
+
+func NFv9TypeToString(typeId uint16) string {
+
+	nameList := map[uint16]string{
+		1:   "IN_BYTES",
+		2:   "IN_PKTS",
+		3:   "FLOWS",
+		4:   "PROTOCOL",
+		5:   "SRC_TOS",
+		6:   "TCP_FLAGS",
+		7:   "L4_SRC_PORT",
+		8:   "IPV4_SRC_ADDR",
+		9:   "SRC_MASK",
+		10:  "INPUT_SNMP",
+		11:  "L4_DST_PORT",
+		12:  "IPV4_DST_ADDR",
+		13:  "DST_MASK",
+		14:  "OUTPUT_SNMP",
+		15:  "IPV4_NEXT_HOP",
+		16:  "SRC_AS",
+		17:  "DST_AS",
+		18:  "BGP_IPV4_NEXT_HOP",
+		19:  "MUL_DST_PKTS",
+		20:  "MUL_DST_BYTES",
+		21:  "LAST_SWITCHED",
+		22:  "FIRST_SWITCHED",
+		23:  "OUT_BYTES",
+		24:  "OUT_PKTS",
+		25:  "MIN_PKT_LNGTH",
+		26:  "MAX_PKT_LNGTH",
+		27:  "IPV6_SRC_ADDR",
+		28:  "IPV6_DST_ADDR",
+		29:  "IPV6_SRC_MASK",
+		30:  "IPV6_DST_MASK",
+		31:  "IPV6_FLOW_LABEL",
+		32:  "ICMP_TYPE",
+		33:  "MUL_IGMP_TYPE",
+		34:  "SAMPLING_INTERVAL",
+		35:  "SAMPLING_ALGORITHM",
+		36:  "FLOW_ACTIVE_TIMEOUT",
+		37:  "FLOW_INACTIVE_TIMEOUT",
+		38:  "ENGINE_TYPE",
+		39:  "ENGINE_ID",
+		40:  "TOTAL_BYTES_EXP",
+		41:  "TOTAL_PKTS_EXP",
+		42:  "TOTAL_FLOWS_EXP",
+		43:  "*Vendor Proprietary*",
+		44:  "IPV4_SRC_PREFIX",
+		45:  "IPV4_DST_PREFIX",
+		46:  "MPLS_TOP_LABEL_TYPE",
+		47:  "MPLS_TOP_LABEL_IP_ADDR",
+		48:  "FLOW_SAMPLER_ID",
+		49:  "FLOW_SAMPLER_MODE",
+		50:  "FLOW_SAMPLER_RANDOM_INTERVAL",
+		51:  "*Vendor Proprietary*",
+		52:  "MIN_TTL",
+		53:  "MAX_TTL",
+		54:  "IPV4_IDENT",
+		55:  "DST_TOS",
+		56:  "IN_SRC_MAC",
+		57:  "OUT_DST_MAC",
+		58:  "SRC_VLAN",
+		59:  "DST_VLAN",
+		60:  "IP_PROTOCOL_VERSION",
+		61:  "DIRECTION",
+		62:  "IPV6_NEXT_HOP",
+		63:  "BPG_IPV6_NEXT_HOP",
+		64:  "IPV6_OPTION_HEADERS",
+		65:  "*Vendor Proprietary*",
+		66:  "*Vendor Proprietary*",
+		67:  "*Vendor Proprietary*",
+		68:  "*Vendor Proprietary*",
+		69:  "*Vendor Proprietary*",
+		70:  "MPLS_LABEL_1",
+		71:  "MPLS_LABEL_2",
+		72:  "MPLS_LABEL_3",
+		73:  "MPLS_LABEL_4",
+		74:  "MPLS_LABEL_5",
+		75:  "MPLS_LABEL_6",
+		76:  "MPLS_LABEL_7",
+		77:  "MPLS_LABEL_8",
+		78:  "MPLS_LABEL_9",
+		79:  "MPLS_LABEL_10",
+		80:  "IN_DST_MAC",
+		81:  "OUT_SRC_MAC",
+		82:  "IF_NAME",
+		83:  "IF_DESC",
+		84:  "SAMPLER_NAME",
+		85:  "IN_ PERMANENT _BYTES",
+		86:  "IN_ PERMANENT _PKTS",
+		87:  "*Vendor Proprietary*",
+		88:  "FRAGMENT_OFFSET",
+		89:  "FORWARDING STATUS",
+		90:  "MPLS PAL RD",
+		91:  "MPLS PREFIX LEN",
+		92:  "SRC TRAFFIC INDEX",
+		93:  "DST TRAFFIC INDEX",
+		94:  "APPLICATION DESCRIPTION",
+		95:  "APPLICATION TAG",
+		96:  "APPLICATION NAME",
+		98:  "postipDiffServCodePoint",
+		99:  "replication factor",
+		100: "DEPRECATED",
+		102: "layer2packetSectionOffset",
+		103: "layer2packetSectionSize",
+		104: "layer2packetSectionData",
+		234: "ingressVRFID",
+		235: "egressVRFID",
+	}
+
+	if typeId > 104 || typeId == 0 {
+		return "Unassigned"
+	} else {
+		return nameList[typeId]
+	}
+}
+
+func NFv9ScopeToString(scopeId uint16) string {
+	nameList := map[uint16]string{
+		1: "System",
+		2: "Interface",
+		3: "Line Card",
+		4: "NetFlow Cache",
+		5: "Template",
+	}
+
+	if scopeId >= 1 && scopeId <= 5 {
+		return nameList[scopeId]
+	} else {
+		return "Unassigned"
+	}
+}
+
+func (flowSet NFv9OptionsTemplateFlowSet) String(TypeToString func(uint16) string) string {
+	str := fmt.Sprintf("       Id %v\n", flowSet.Id)
+	str += fmt.Sprintf("       Length: %v\n", flowSet.Length)
+	str += fmt.Sprintf("       Records (%v records):\n", len(flowSet.Records))
+
+	for j, record := range flowSet.Records {
+		str += fmt.Sprintf("       - Record %v:\n", j)
+		str += fmt.Sprintf("            TemplateId: %v\n", record.TemplateId)
+		str += fmt.Sprintf("            ScopeLength: %v\n", record.ScopeLength)
+		str += fmt.Sprintf("            OptionLength: %v\n", record.OptionLength)
+		str += fmt.Sprintf("            Scopes (%v):\n", len(record.Scopes))
+
+		for k, field := range record.Scopes {
+			str += fmt.Sprintf("            - %v. %v (%v): %v\n", k, NFv9ScopeToString(field.Type), field.Type, field.Length)
+		}
+
+		str += fmt.Sprintf("            Options (%v):\n", len(record.Options))
+
+		for k, field := range record.Options {
+			str += fmt.Sprintf("            - %v. %v (%v): %v\n", k, TypeToString(field.Type), field.Type, field.Length)
+		}
+	}
+
+	return str
+}
+
+func (p NFv9Packet) String() string {
+	str := "Flow Packet\n"
+	str += "------------\n"
+	str += fmt.Sprintf("  Version: %v\n", p.Version)
+	str += fmt.Sprintf("  Count:  %v\n", p.Count)
+
+	unixSeconds := time.Unix(int64(p.UnixSeconds), 0)
+	str += fmt.Sprintf("  SystemUptime: %v\n", p.SystemUptime)
+	str += fmt.Sprintf("  UnixSeconds: %v\n", unixSeconds.String())
+	str += fmt.Sprintf("  SequenceNumber: %v\n", p.SequenceNumber)
+	str += fmt.Sprintf("  SourceId: %v\n", p.SourceId)
+	str += fmt.Sprintf("  FlowSets (%v):\n", len(p.FlowSets))
+
+	for i, flowSet := range p.FlowSets {
+		switch flowSet := flowSet.(type) {
+		case TemplateFlowSet:
+			str += fmt.Sprintf("    - TemplateFlowSet %v:\n", i)
+			str += flowSet.String(NFv9TypeToString)
+		case NFv9OptionsTemplateFlowSet:
+			str += fmt.Sprintf("    - OptionsTemplateFlowSet %v:\n", i)
+			str += flowSet.String(NFv9TypeToString)
+		case DataFlowSet:
+			str += fmt.Sprintf("    - DataFlowSet %v:\n", i)
+			str += flowSet.String(NFv9TypeToString)
+		case OptionsDataFlowSet:
+			str += fmt.Sprintf("    - OptionsDataFlowSet %v:\n", i)
+			str += flowSet.String(NFv9TypeToString, NFv9ScopeToString)
+		default:
+			str += fmt.Sprintf("    - (unknown type) %v: %v\n", i, flowSet)
+		}
+	}
+	return str
+}

decoders/netflow/packet.go

@@ -0,0 +1,156 @@
+package netflow
+
+import (
+	"fmt"
+)
+
+// FlowSetHeader contains fields shared by all Flow Sets (DataFlowSet,
+// TemplateFlowSet, OptionsTemplateFlowSet).
+type FlowSetHeader struct {
+	// FlowSet ID:
+	//    0 for TemplateFlowSet
+	//    1 for OptionsTemplateFlowSet
+	//    256-65535 for DataFlowSet (used as TemplateId)
+	Id uint16
+
+	// The total length of this FlowSet in bytes (including padding).
+	Length uint16
+}
+
+// TemplateFlowSet is a collection of templates that describe structure of Data
+// Records (actual NetFlow data).
+type TemplateFlowSet struct {
+	FlowSetHeader
+
+	// List of Template Records
+	Records []TemplateRecord
+}
+
+// DataFlowSet is a collection of Data Records (actual NetFlow data) and Options
+// Data Records (meta data).
+type DataFlowSet struct {
+	FlowSetHeader
+
+	Records []DataRecord
+}
+
+type OptionsDataFlowSet struct {
+	FlowSetHeader
+
+	Records []OptionsDataRecord
+}
+
+// TemplateRecord is a single template that describes structure of a Flow Record
+// (actual Netflow data).
+type TemplateRecord struct {
+	// Each of the newly generated Template Records is given a unique
+	// Template ID. This uniqueness is local to the Observation Domain that
+	// generated the Template ID. Template IDs of Data FlowSets are numbered
+	// from 256 to 65535.
+	TemplateId uint16
+
+	// Number of fields in this Template Record. Because a Template FlowSet
+	// usually contains multiple Template Records, this field allows the
+	// Collector to determine the end of the current Template Record and
+	// the start of the next.
+	FieldCount uint16
+
+	// List of fields in this Template Record.
+	Fields []Field
+}
+
+type DataRecord struct {
+	Values []DataField
+}
+
+// OptionsDataRecord is meta data sent alongide actual NetFlow data. Combined
+// with OptionsTemplateRecord it can be decoded to a single data row.
+type OptionsDataRecord struct {
+	// List of Scope values stored in raw format as []byte
+	ScopesValues []DataField
+
+	// List of Optons values stored in raw format as []byte
+	OptionsValues []DataField
+}
+
+// Field describes type and length of a single value in a Flow Data Record.
+// Field does not contain the record value itself it is just a description of
+// what record value will look like.
+type Field struct {
+	// A numeric value that represents the type of field.
+	PenProvided bool
+	Type        uint16
+
+	// The length (in bytes) of the field.
+	Length uint16
+
+	Pen uint32
+}
+
+type DataField struct {
+	// A numeric value that represents the type of field.
+	Type uint16
+
+	// The value (in bytes) of the field.
+	Value interface{}
+	//Value []byte
+}
+
+func (flowSet OptionsDataFlowSet) String(TypeToString func(uint16) string, ScopeToString func(uint16) string) string {
+	str := fmt.Sprintf("       Id %v\n", flowSet.Id)
+	str += fmt.Sprintf("       Length: %v\n", flowSet.Length)
+	str += fmt.Sprintf("       Records (%v records):\n", len(flowSet.Records))
+
+	for j, record := range flowSet.Records {
+		str += fmt.Sprintf("       - Record %v:\n", j)
+		str += fmt.Sprintf("            Scopes (%v):\n", len(record.ScopesValues))
+
+		for k, value := range record.ScopesValues {
+			str += fmt.Sprintf("            - %v. %v (%v): %v\n", k, ScopeToString(value.Type), value.Type, value.Value)
+		}
+
+		str += fmt.Sprintf("            Options (%v):\n", len(record.OptionsValues))
+
+		for k, value := range record.OptionsValues {
+			str += fmt.Sprintf("            - %v. %v (%v): %v\n", k, TypeToString(value.Type), value.Type, value.Value)
+		}
+	}
+
+	return str
+}
+
+func (flowSet DataFlowSet) String(TypeToString func(uint16) string) string {
+	str := fmt.Sprintf("       Id %v\n", flowSet.Id)
+	str += fmt.Sprintf("       Length: %v\n", flowSet.Length)
+	str += fmt.Sprintf("       Records (%v records):\n", len(flowSet.Records))
+
+	for j, record := range flowSet.Records {
+		str += fmt.Sprintf("       - Record %v:\n", j)
+		str += fmt.Sprintf("            Values (%v):\n", len(record.Values))
+
+		for k, value := range record.Values {
+			str += fmt.Sprintf("            - %v. %v (%v): %v\n", k, TypeToString(value.Type), value.Type, value.Value)
+		}
+	}
+
+	return str
+}
+
+func (flowSet TemplateFlowSet) String(TypeToString func(uint16) string) string {
+	str := fmt.Sprintf("       Id %v\n", flowSet.Id)
+	str += fmt.Sprintf("       Length: %v\n", flowSet.Length)
+	str += fmt.Sprintf("       Records (%v records):\n", len(flowSet.Records))
+
+	for j, record := range flowSet.Records {
+		str += fmt.Sprintf("       - %v. Record:\n", j)
+		str += fmt.Sprintf("            TemplateId: %v\n", record.TemplateId)
+		str += fmt.Sprintf("            FieldCount: %v\n", record.FieldCount)
+		str += fmt.Sprintf("            Fields (%v):\n", len(record.Fields))
+
+		for k, field := range record.Fields {
+			str += fmt.Sprintf("            - %v. %v (%v/%v): %v\n", k, TypeToString(field.Type), field.Type, field.PenProvided, field.Length)
+		}
+	}
+
+	return str
+}

decoders/netflowlegacy/netflow.go

@@ -0,0 +1,53 @@
+package netflowlegacy
+
+import (
+	"bytes"
+	"fmt"
+
+	"github.com/netsampler/goflow2/decoders/utils"
+)
+
+type ErrorVersion struct {
+	version uint16
+}
+
+func NewErrorVersion(version uint16) *ErrorVersion {
+	return &ErrorVersion{
+		version: version,
+	}
+}
+
+func (e *ErrorVersion) Error() string {
+	return fmt.Sprintf("Unknown NetFlow version %v (only decodes v5)", e.version)
+}
+
+func DecodeMessage(payload *bytes.Buffer) (interface{}, error) {
+	var version uint16
+	utils.BinaryDecoder(payload, &version)
+	packet := PacketNetFlowV5{}
+	if version == 5 {
+		packet.Version = version
+
+		utils.BinaryDecoder(payload,
+			&(packet.Count),
+			&(packet.SysUptime),
+			&(packet.UnixSecs),
+			&(packet.UnixNSecs),
+			&(packet.FlowSequence),
+			&(packet.EngineType),
+			&(packet.EngineId),
+			&(packet.SamplingInterval),
+		)
+
+		packet.Records = make([]RecordsNetFlowV5, int(packet.Count))
+		for i := 0; i < int(packet.Count) && payload.Len() >= 48; i++ {
+			record := RecordsNetFlowV5{}
+			utils.BinaryDecoder(payload, &record)
+			packet.Records[i] = record
+		}
+
+		return packet, nil
+	} else {
+		return nil, NewErrorVersion(version)
+	}
+}

decoders/netflowlegacy/netflow_test.go

@@ -0,0 +1,41 @@
+package netflowlegacy
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDecodeNetFlowV5(t *testing.T) {
+	data := []byte{
+		0x00, 0x05, 0x00, 0x06, 0x00, 0x82, 0xc3, 0x48, 0x5b, 0xcd, 0xba, 0x1b, 0x05, 0x97, 0x6d, 0xc7,
+		0x00, 0x00, 0x64, 0x3d, 0x08, 0x08, 0x00, 0x00, 0x0a, 0x80, 0x02, 0x79, 0x0a, 0x80, 0x02, 0x01,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x00, 0x02, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x02, 0x4e,
+		0x00, 0x82, 0x9b, 0x8c, 0x00, 0x82, 0x9b, 0x90, 0x1f, 0x90, 0xb9, 0x18, 0x00, 0x1b, 0x06, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x80, 0x02, 0x77, 0x0a, 0x81, 0x02, 0x01,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x94,
+		0x00, 0x82, 0x95, 0xa9, 0x00, 0x82, 0x9a, 0xfb, 0x1f, 0x90, 0xc1, 0x2c, 0x00, 0x12, 0x06, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x81, 0x02, 0x01, 0x0a, 0x80, 0x02, 0x77,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x07, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0xc2,
+		0x00, 0x82, 0x95, 0xa9, 0x00, 0x82, 0x9a, 0xfc, 0xc1, 0x2c, 0x1f, 0x90, 0x00, 0x16, 0x06, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x80, 0x02, 0x01, 0x0a, 0x80, 0x02, 0x79,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x09, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x01, 0xf1,
+		0x00, 0x82, 0x9b, 0x8c, 0x00, 0x82, 0x9b, 0x8f, 0xb9, 0x18, 0x1f, 0x90, 0x00, 0x1b, 0x06, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x80, 0x02, 0x01, 0x0a, 0x80, 0x02, 0x79,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x09, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x02, 0x2e,
+		0x00, 0x82, 0x9b, 0x90, 0x00, 0x82, 0x9b, 0x9d, 0xb9, 0x1a, 0x1f, 0x90, 0x00, 0x1b, 0x06, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x80, 0x02, 0x79, 0x0a, 0x80, 0x02, 0x01,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x00, 0x02, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x0b, 0xac,
+		0x00, 0x82, 0x9b, 0x90, 0x00, 0x82, 0x9b, 0x9d, 0x1f, 0x90, 0xb9, 0x1a, 0x00, 0x1b, 0x06, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	}
+	buf := bytes.NewBuffer(data)
+
+	dec, err := DecodeMessage(buf)
+	assert.Nil(t, err)
+	assert.NotNil(t, dec)
+	decNfv5 := dec.(PacketNetFlowV5)
+	assert.Equal(t, uint16(5), decNfv5.Version)
+	assert.Equal(t, uint16(9), decNfv5.Records[0].Input)
+}

decoders/netflowlegacy/packet.go

@@ -0,0 +1,96 @@
+package netflowlegacy
+
+import (
+	"encoding/binary"
+	"fmt"
+	"net"
+	"time"
+)
+
+type PacketNetFlowV5 struct {
+	Version          uint16
+	Count            uint16
+	SysUptime        uint32
+	UnixSecs         uint32
+	UnixNSecs        uint32
+	FlowSequence     uint32
+	EngineType       uint8
+	EngineId         uint8
+	SamplingInterval uint16
+	Records          []RecordsNetFlowV5
+}
+
+type RecordsNetFlowV5 struct {
+	SrcAddr  uint32
+	DstAddr  uint32
+	NextHop  uint32
+	Input    uint16
+	Output   uint16
+	DPkts    uint32
+	DOctets  uint32
+	First    uint32
+	Last     uint32
+	SrcPort  uint16
+	DstPort  uint16
+	Pad1     byte
+	TCPFlags uint8
+	Proto    uint8
+	Tos      uint8
+	SrcAS    uint16
+	DstAS    uint16
+	SrcMask  uint8
+	DstMask  uint8
+	Pad2     uint16
+}
+
+func (p PacketNetFlowV5) String() string {
+	str := "NetFlow v5 Packet\n"
+	str += "-----------------\n"
+	str += fmt.Sprintf("  Version: %v\n", p.Version)
+	str += fmt.Sprintf("  Count:  %v\n", p.Count)
+
+	unixSeconds := time.Unix(int64(p.UnixSecs), int64(p.UnixNSecs))
+	str += fmt.Sprintf("  SystemUptime: %v\n", time.Duration(p.SysUptime)*time.Millisecond)
+	str += fmt.Sprintf("  UnixSeconds: %v\n", unixSeconds.String())
+	str += fmt.Sprintf("  FlowSequence: %v\n", p.FlowSequence)
+	str += fmt.Sprintf("  EngineType: %v\n", p.EngineType)
+	str += fmt.Sprintf("  EngineId: %v\n", p.EngineId)
+	str += fmt.Sprintf("  SamplingInterval: %v\n", p.SamplingInterval)
+	str += fmt.Sprintf("  Records (%v):\n", len(p.Records))
+
+	for i, record := range p.Records {
+		str += fmt.Sprintf("    Record %v:\n", i)
+		str += record.String()
+	}
+	return str
+}
+
+func (r RecordsNetFlowV5) String() string {
+	srcaddr := make(net.IP, 4)
+	binary.BigEndian.PutUint32(srcaddr, r.SrcAddr)
+	dstaddr := make(net.IP, 4)
+	binary.BigEndian.PutUint32(dstaddr, r.DstAddr)
+	nexthop := make(net.IP, 4)
+	binary.BigEndian.PutUint32(nexthop, r.NextHop)
+
+	str := fmt.Sprintf("      SrcAddr: %v\n", srcaddr.String())
+	str += fmt.Sprintf("      DstAddr: %v\n", dstaddr.String())
+	str += fmt.Sprintf("      NextHop: %v\n", nexthop.String())
+	str += fmt.Sprintf("      Input: %v\n", r.Input)
+	str += fmt.Sprintf("      Output: %v\n", r.Output)
+	str += fmt.Sprintf("      DPkts: %v\n", r.DPkts)
+	str += fmt.Sprintf("      DOctets: %v\n", r.DOctets)
+	str += fmt.Sprintf("      First: %v\n", time.Duration(r.First)*time.Millisecond)
+	str += fmt.Sprintf("      Last: %v\n", time.Duration(r.Last)*time.Millisecond)
+	str += fmt.Sprintf("      SrcPort: %v\n", r.SrcPort)
+	str += fmt.Sprintf("      DstPort: %v\n", r.DstPort)
+	str += fmt.Sprintf("      TCPFlags: %v\n", r.TCPFlags)
+	str += fmt.Sprintf("      Proto: %v\n", r.Proto)
+	str += fmt.Sprintf("      Tos: %v\n", r.Tos)
+	str += fmt.Sprintf("      SrcAS: %v\n", r.SrcAS)
+	str += fmt.Sprintf("      DstAS: %v\n", r.DstAS)
+	str += fmt.Sprintf("      SrcMask: %v\n", r.SrcMask)
+	str += fmt.Sprintf("      DstMask: %v\n", r.DstMask)
+
+	return str
+}

decoders/sflow/datastructure.go

@@ -0,0 +1,103 @@
+package sflow
+
+type SampledHeader struct {
+	Protocol       uint32
+	FrameLength    uint32
+	Stripped       uint32
+	OriginalLength uint32
+	HeaderData     []byte
+}
+
+type SampledEthernet struct {
+	Length  uint32
+	SrcMac  []byte
+	DstMac  []byte
+	EthType uint32
+}
+
+type SampledIP_Base struct {
+	Length   uint32
+	Protocol uint32
+	SrcIP    []byte
+	DstIP    []byte
+	SrcPort  uint32
+	DstPort  uint32
+	TcpFlags uint32
+}
+
+type SampledIPv4 struct {
+	Base SampledIP_Base
+	Tos  uint32
+}
+
+type SampledIPv6 struct {
+	Base     SampledIP_Base
+	Priority uint32
+}
+
+type ExtendedSwitch struct {
+	SrcVlan     uint32
+	SrcPriority uint32
+	DstVlan     uint32
+	DstPriority uint32
+}
+
+type ExtendedRouter struct {
+	NextHopIPVersion uint32
+	NextHop          []byte
+	SrcMaskLen       uint32
+	DstMaskLen       uint32
+}
+
+type ExtendedGateway struct {
+	NextHopIPVersion  uint32
+	NextHop           []byte
+	AS                uint32
+	SrcAS             uint32
+	SrcPeerAS         uint32
+	ASDestinations    uint32
+	ASPathType        uint32
+	ASPathLength      uint32
+	ASPath            []uint32
+	CommunitiesLength uint32
+	Communities       []uint32
+	LocalPref         uint32
+}
+
+type IfCounters struct {
+	IfIndex            uint32
+	IfType             uint32
+	IfSpeed            uint64
+	IfDirection        uint32
+	IfStatus           uint32
+	IfInOctets         uint64
+	IfInUcastPkts      uint32
+	IfInMulticastPkts  uint32
+	IfInBroadcastPkts  uint32
+	IfInDiscards       uint32
+	IfInErrors         uint32
+	IfInUnknownProtos  uint32
+	IfOutOctets        uint64
+	IfOutUcastPkts     uint32
+	IfOutMulticastPkts uint32
+	IfOutBroadcastPkts uint32
+	IfOutDiscards      uint32
+	IfOutErrors        uint32
+	IfPromiscuousMode  uint32
+}
+
+type EthernetCounters struct {
+	Dot3StatsAlignmentErrors           uint32
+	Dot3StatsFCSErrors                 uint32
+	Dot3StatsSingleCollisionFrames     uint32
+	Dot3StatsMultipleCollisionFrames   uint32
+	Dot3StatsSQETestErrors             uint32
+	Dot3StatsDeferredTransmissions     uint32
+	Dot3StatsLateCollisions            uint32
+	Dot3StatsExcessiveCollisions       uint32
+	Dot3StatsInternalMacTransmitErrors uint32
+	Dot3StatsCarrierSenseErrors        uint32
+	Dot3StatsFrameTooLongs             uint32
+	Dot3StatsInternalMacReceiveErrors  uint32
+	Dot3StatsSymbolErrors              uint32
+}

decoders/sflow/packet.go

@@ -0,0 +1,69 @@
+package sflow
+
+type Packet struct {
+	Version        uint32
+	IPVersion      uint32
+	AgentIP        []byte
+	SubAgentId     uint32
+	SequenceNumber uint32
+	Uptime         uint32
+	SamplesCount   uint32
+	Samples        []interface{}
+}
+
+type SampleHeader struct {
+	Format uint32
+	Length uint32
+
+	SampleSequenceNumber uint32
+	SourceIdType         uint32
+	SourceIdValue        uint32
+}
+
+type FlowSample struct {
+	Header SampleHeader
+
+	SamplingRate     uint32
+	SamplePool       uint32
+	Drops            uint32
+	Input            uint32
+	Output           uint32
+	FlowRecordsCount uint32
+	Records          []FlowRecord
+}
+
+type CounterSample struct {
+	Header SampleHeader
+
+	CounterRecordsCount uint32
+	Records             []CounterRecord
+}
+
+type ExpandedFlowSample struct {
+	Header SampleHeader
+
+	SamplingRate     uint32
+	SamplePool       uint32
+	Drops            uint32
+	InputIfFormat    uint32
+	InputIfValue     uint32
+	OutputIfFormat   uint32
+	OutputIfValue    uint32
+	FlowRecordsCount uint32
+	Records          []FlowRecord
+}
+
+type RecordHeader struct {
+	DataFormat uint32
+	Length     uint32
+}
+
+type FlowRecord struct {
+	Header RecordHeader
+	Data   interface{}
+}
+
+type CounterRecord struct {
+	Header RecordHeader
+	Data   interface{}
+}

decoders/sflow/sflow.go

@@ -0,0 +1,397 @@
+package sflow
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+
+	"github.com/netsampler/goflow2/decoders/utils"
+)
+
+const (
+	FORMAT_EXT_SWITCH  = 1001
+	FORMAT_EXT_ROUTER  = 1002
+	FORMAT_EXT_GATEWAY = 1003
+	FORMAT_RAW_PKT     = 1
+	FORMAT_ETH         = 2
+	FORMAT_IPV4        = 3
+	FORMAT_IPV6        = 4
+)
+
+type ErrorDecodingSFlow struct {
+	msg string
+}
+
+func NewErrorDecodingSFlow(msg string) *ErrorDecodingSFlow {
+	return &ErrorDecodingSFlow{
+		msg: msg,
+	}
+}
+
+func (e *ErrorDecodingSFlow) Error() string {
+	return fmt.Sprintf("Error decoding sFlow: %v", e.msg)
+}
+
+type ErrorDataFormat struct {
+	dataformat uint32
+}
+
+func NewErrorDataFormat(dataformat uint32) *ErrorDataFormat {
+	return &ErrorDataFormat{
+		dataformat: dataformat,
+	}
+}
+
+func (e *ErrorDataFormat) Error() string {
+	return fmt.Sprintf("Unknown data format %v", e.dataformat)
+}
+
+type ErrorIPVersion struct {
+	version uint32
+}
+
+func NewErrorIPVersion(version uint32) *ErrorIPVersion {
+	return &ErrorIPVersion{
+		version: version,
+	}
+}
+
+func (e *ErrorIPVersion) Error() string {
+	return fmt.Sprintf("Unknown IP version: %v", e.version)
+}
+
+type ErrorVersion struct {
+	version uint32
+}
+
+func NewErrorVersion(version uint32) *ErrorVersion {
+	return &ErrorVersion{
+		version: version,
+	}
+}
+
+func (e *ErrorVersion) Error() string {
+	return fmt.Sprintf("Unknown sFlow version %v (supported v5)", e.version)
+}
+
+func DecodeCounterRecord(header *RecordHeader, payload *bytes.Buffer) (CounterRecord, error) {
+	counterRecord := CounterRecord{
+		Header: *header,
+	}
+	switch (*header).DataFormat {
+	case 1:
+		ifCounters := IfCounters{}
+		utils.BinaryDecoder(payload, &ifCounters)
+		counterRecord.Data = ifCounters
+	case 2:
+		ethernetCounters := EthernetCounters{}
+		utils.BinaryDecoder(payload, &ethernetCounters)
+		counterRecord.Data = ethernetCounters
+	default:
+		return counterRecord, NewErrorDataFormat((*header).DataFormat)
+	}
+
+	return counterRecord, nil
+}
+
+func DecodeIP(payload *bytes.Buffer) (uint32, []byte, error) {
+	var ipVersion uint32
+	utils.BinaryDecoder(payload, &ipVersion)
+	var ip []byte
+	if ipVersion == 1 {
+		ip = make([]byte, 4)
+	} else if ipVersion == 2 {
+		ip = make([]byte, 16)
+	} else {
+		return ipVersion, ip, NewErrorIPVersion(ipVersion)
+	}
+	if payload.Len() >= len(ip) {
+		utils.BinaryDecoder(payload, &ip)
+	} else {
+		return ipVersion, ip, NewErrorDecodingSFlow(fmt.Sprintf("Not enough data: %v, needs %v.", payload.Len(), len(ip)))
+	}
+	return ipVersion, ip, nil
+}
+
+func DecodeFlowRecord(header *RecordHeader, payload *bytes.Buffer) (FlowRecord, error) {
+	flowRecord := FlowRecord{
+		Header: *header,
+	}
+	switch (*header).DataFormat {
+	case FORMAT_EXT_SWITCH:
+		extendedSwitch := ExtendedSwitch{}
+		err := utils.BinaryDecoder(payload, &extendedSwitch)
+		if err != nil {
+			return flowRecord, err
+		}
+		flowRecord.Data = extendedSwitch
+	case FORMAT_RAW_PKT:
+		sampledHeader := SampledHeader{}
+		err := utils.BinaryDecoder(payload, &(sampledHeader.Protocol), &(sampledHeader.FrameLength), &(sampledHeader.Stripped), &(sampledHeader.OriginalLength))
+		if err != nil {
+			return flowRecord, err
+		}
+		sampledHeader.HeaderData = payload.Bytes()
+		flowRecord.Data = sampledHeader
+	case FORMAT_IPV4:
+		sampledIPBase := SampledIP_Base{
+			SrcIP: make([]byte, 4),
+			DstIP: make([]byte, 4),
+		}
+		err := utils.BinaryDecoder(payload, &sampledIPBase)
+		if err != nil {
+			return flowRecord, err
+		}
+		sampledIPv4 := SampledIPv4{
+			Base: sampledIPBase,
+		}
+		err = utils.BinaryDecoder(payload, &(sampledIPv4.Tos))
+		if err != nil {
+			return flowRecord, err
+		}
+		flowRecord.Data = sampledIPv4
+	case FORMAT_IPV6:
+		sampledIPBase := SampledIP_Base{
+			SrcIP: make([]byte, 16),
+			DstIP: make([]byte, 16),
+		}
+		err := utils.BinaryDecoder(payload, &sampledIPBase)
+		if err != nil {
+			return flowRecord, err
+		}
+		sampledIPv6 := SampledIPv6{
+			Base: sampledIPBase,
+		}
+		err = utils.BinaryDecoder(payload, &(sampledIPv6.Priority))
+		if err != nil {
+			return flowRecord, err
+		}
+		flowRecord.Data = sampledIPv6
+	case FORMAT_EXT_ROUTER:
+		extendedRouter := ExtendedRouter{}
+
+		ipVersion, ip, err := DecodeIP(payload)
+		if err != nil {
+			return flowRecord, err
+		}
+		extendedRouter.NextHopIPVersion = ipVersion
+		extendedRouter.NextHop = ip
+		err = utils.BinaryDecoder(payload, &(extendedRouter.SrcMaskLen), &(extendedRouter.DstMaskLen))
+		if err != nil {
+			return flowRecord, err
+		}
+		flowRecord.Data = extendedRouter
+	case FORMAT_EXT_GATEWAY:
+		extendedGateway := ExtendedGateway{}
+		ipVersion, ip, err := DecodeIP(payload)
+		if err != nil {
+			return flowRecord, err
+		}
+		extendedGateway.NextHopIPVersion = ipVersion
+		extendedGateway.NextHop = ip
+		err = utils.BinaryDecoder(payload, &(extendedGateway.AS), &(extendedGateway.SrcAS), &(extendedGateway.SrcPeerAS),
+			&(extendedGateway.ASDestinations))
+		if err != nil {
+			return flowRecord, err
+		}
+		asPath := make([]uint32, 0)
+		if extendedGateway.ASDestinations != 0 {
+			err := utils.BinaryDecoder(payload, &(extendedGateway.ASPathType), &(extendedGateway.ASPathLength))
+			if err != nil {
+				return flowRecord, err
+			}
+			if int(extendedGateway.ASPathLength) > payload.Len()-4 {
+				return flowRecord, errors.New(fmt.Sprintf("Invalid AS path length: %v.", extendedGateway.ASPathLength))
+			}
+			asPath = make([]uint32, extendedGateway.ASPathLength)
+			if len(asPath) > 0 {
+				err = utils.BinaryDecoder(payload, asPath)
+				if err != nil {
+					return flowRecord, err
+				}
+			}
+		}
+		extendedGateway.ASPath = asPath
+
+		err = utils.BinaryDecoder(payload, &(extendedGateway.CommunitiesLength))
+		if err != nil {
+			return flowRecord, err
+		}
+		if int(extendedGateway.CommunitiesLength) > payload.Len()-4 {
+			return flowRecord, errors.New(fmt.Sprintf("Invalid Communities length: %v.", extendedGateway.ASPathLength))
+		}
+		communities := make([]uint32, extendedGateway.CommunitiesLength)
+		if len(communities) > 0 {
+			err = utils.BinaryDecoder(payload, communities)
+			if err != nil {
+				return flowRecord, err
+			}
+		}
+		err = utils.BinaryDecoder(payload, &(extendedGateway.LocalPref))
+		if err != nil {
+			return flowRecord, err
+		}
+		extendedGateway.Communities = communities
+
+		flowRecord.Data = extendedGateway
+	default:
+		return flowRecord, errors.New(fmt.Sprintf("Unknown data format %v.", (*header).DataFormat))
+	}
+	return flowRecord, nil
+}
+
+func DecodeSample(header *SampleHeader, payload *bytes.Buffer) (interface{}, error) {
+	format := (*header).Format
+	var sample interface{}
+
+	err := utils.BinaryDecoder(payload, &((*header).SampleSequenceNumber))
+	if err != nil {
+		return sample, err
+	}
+	if format == FORMAT_RAW_PKT || format == FORMAT_ETH {
+		var sourceId uint32
+		err = utils.BinaryDecoder(payload, &sourceId)
+		if err != nil {
+			return sample, err
+		}
+
+		(*header).SourceIdType = sourceId >> 24
+		(*header).SourceIdValue = sourceId & 0x00ffffff
+	} else if format == FORMAT_IPV4 || format == FORMAT_IPV6 {
+		err = utils.BinaryDecoder(payload, &((*header).SourceIdType), &((*header).SourceIdValue))
+		if err != nil {
+			return sample, err
+		}
+	} else {
+		return nil, NewErrorDataFormat(format)
+	}
+
+	var recordsCount uint32
+	var flowSample FlowSample
+	var counterSample CounterSample
+	var expandedFlowSample ExpandedFlowSample
+	if format == FORMAT_RAW_PKT {
+		flowSample = FlowSample{
+			Header: *header,
+		}
+		err = utils.BinaryDecoder(payload, &(flowSample.SamplingRate), &(flowSample.SamplePool),
+			&(flowSample.Drops), &(flowSample.Input), &(flowSample.Output), &(flowSample.FlowRecordsCount))
+		if err != nil {
+			return sample, err
+		}
+		recordsCount = flowSample.FlowRecordsCount
+		flowSample.Records = make([]FlowRecord, recordsCount)
+		sample = flowSample
+	} else if format == FORMAT_ETH || format == FORMAT_IPV6 {
+		err = utils.BinaryDecoder(payload, &recordsCount)
+		if err != nil {
+			return sample, err
+		}
+		counterSample = CounterSample{
+			Header:              *header,
+			CounterRecordsCount: recordsCount,
+		}
+		counterSample.Records = make([]CounterRecord, recordsCount)
+		sample = counterSample
+	} else if format == FORMAT_IPV4 {
+		expandedFlowSample = ExpandedFlowSample{
+			Header: *header,
+		}
+		err = utils.BinaryDecoder(payload, &(expandedFlowSample.SamplingRate), &(expandedFlowSample.SamplePool),
+			&(expandedFlowSample.Drops), &(expandedFlowSample.InputIfFormat), &(expandedFlowSample.InputIfValue),
+			&(expandedFlowSample.OutputIfFormat), &(expandedFlowSample.OutputIfValue), &(expandedFlowSample.FlowRecordsCount))
+		if err != nil {
+			return sample, err
+		}
+		recordsCount = expandedFlowSample.FlowRecordsCount
+		expandedFlowSample.Records = make([]FlowRecord, recordsCount)
+		sample = expandedFlowSample
+	}
+	for i := 0; i < int(recordsCount) && payload.Len() >= 8; i++ {
+		recordHeader := RecordHeader{}
+		err = utils.BinaryDecoder(payload, &(recordHeader.DataFormat), &(recordHeader.Length))
+		if err != nil {
+			return sample, err
+		}
+		if int(recordHeader.Length) > payload.Len() {
+			break
+		}
+		recordReader := bytes.NewBuffer(payload.Next(int(recordHeader.Length)))
+		if format == FORMAT_RAW_PKT || format == FORMAT_IPV4 {
+			record, err := DecodeFlowRecord(&recordHeader, recordReader)
+			if err != nil {
+				continue
+			}
+			if format == FORMAT_RAW_PKT {
+				flowSample.Records[i] = record
+			} else if format == FORMAT_IPV4 {
+				expandedFlowSample.Records[i] = record
+			}
+		} else if format == FORMAT_ETH || format == FORMAT_IPV6 {
+			record, err := DecodeCounterRecord(&recordHeader, recordReader)
+			if err != nil {
+				continue
+			}
+			counterSample.Records[i] = record
+		}
+	}
+	return sample, nil
+}
+
+func DecodeMessage(payload *bytes.Buffer) (interface{}, error) {
+	var version uint32
+	err := utils.BinaryDecoder(payload, &version)
+	if err != nil {
+		return nil, err
+	}
+	packetV5 := Packet{}
+	if version == 5 {
+		packetV5.Version = version
+		err = utils.BinaryDecoder(payload, &(packetV5.IPVersion))
+		if err != nil {
+			return packetV5, err
+		}
+		var ip []byte
+		if packetV5.IPVersion == 1 {
+			ip = make([]byte, 4)
+			utils.BinaryDecoder(payload, ip)
+		} else if packetV5.IPVersion == 2 {
+			ip = make([]byte, 16)
+			err = utils.BinaryDecoder(payload, ip)
+			if err != nil {
+				return packetV5, err
+			}
+		} else {
+			return nil, NewErrorIPVersion(packetV5.IPVersion)
+		}
+
+		packetV5.AgentIP = ip
+		err = utils.BinaryDecoder(payload, &(packetV5.SubAgentId), &(packetV5.SequenceNumber), &(packetV5.Uptime), &(packetV5.SamplesCount))
+		if err != nil {
+			return packetV5, err
+		}
+		packetV5.Samples = make([]interface{}, int(packetV5.SamplesCount))
+		for i := 0; i < int(packetV5.SamplesCount) && payload.Len() >= 8; i++ {
+			header := SampleHeader{}
+			err = utils.BinaryDecoder(payload, &(header.Format), &(header.Length))
+			if err != nil {
+				return packetV5, err
+			}
+			if int(header.Length) > payload.Len() {
+				break
+			}
+			sampleReader := bytes.NewBuffer(payload.Next(int(header.Length)))
+
+			sample, err := DecodeSample(&header, sampleReader)
+			if err != nil {
+				continue
+			} else {
+				packetV5.Samples[i] = sample
+			}
+		}
+
+		return packetV5, nil
+	} else {
+		return nil, NewErrorVersion(version)
+	}
+}

decoders/sflow/sflow_test.go

@@ -0,0 +1,134 @@
+package sflow
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSFlowDecode(t *testing.T) {
+	data := []byte{
+		0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x01, 0xac, 0x10, 0x00, 0x11, 0x00, 0x00, 0x00, 0x01,
+		0x00, 0x00, 0x01, 0xaa, 0x67, 0xee, 0xaa, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01,
+		0x00, 0x00, 0x00, 0x88, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x04, 0x13, 0x00, 0x00, 0x08, 0x00,
+		0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0xaa, 0x00, 0x00, 0x04, 0x13,
+		0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, 0x01,
+		0x00, 0x00, 0x00, 0x52, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x4e, 0x00, 0xff, 0x12, 0x34,
+		0x35, 0x1b, 0xff, 0xab, 0xcd, 0xef, 0xab, 0x64, 0x81, 0x00, 0x00, 0x20, 0x08, 0x00, 0x45, 0x00,
+		0x00, 0x3c, 0x5c, 0x07, 0x00, 0x00, 0x7c, 0x01, 0x48, 0xa0, 0xac, 0x10, 0x20, 0xfe, 0xac, 0x10,
+		0x20, 0xf1, 0x08, 0x00, 0x97, 0x61, 0xa9, 0x48, 0x0c, 0xb2, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66,
+		0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76,
+		0x77, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x00, 0x00,
+	}
+	buf := bytes.NewBuffer(data)
+	_, err := DecodeMessage(buf)
+	assert.Nil(t, err)
+}
+
+func TestExpandedSFlowDecode(t *testing.T) {
+	data := getExpandedSFlowDecode()
+
+	buf := bytes.NewBuffer(data)
+	_, err := DecodeMessage(buf)
+	assert.Nil(t, err)
+}
+
+func getExpandedSFlowDecode() []byte {
+	return []byte{
+		0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x01, 0x01, 0x02, 0x03, 0x04, 0x00, 0x00, 0x00, 0x00,
+		0x0f, 0xa7, 0x72, 0xc2, 0x0f, 0x76, 0x73, 0x48, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x03,
+		0x00, 0x00, 0x00, 0xdc, 0x20, 0x90, 0x93, 0x26, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x42, 0xa4,
+		0x00, 0x00, 0x3f, 0xff, 0x04, 0x38, 0xec, 0xda, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x0f, 0x42, 0xa4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x42, 0x52, 0x00, 0x00, 0x00, 0x02,
+		0x00, 0x00, 0x03, 0xe9, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x1e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x90,
+		0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x05, 0xea, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x80,
+		0x08, 0xec, 0xf5, 0x2a, 0x8f, 0xbe, 0x74, 0x83, 0xef, 0x30, 0x65, 0xb7, 0x81, 0x00, 0x00, 0x1e,
+		0x08, 0x00, 0x45, 0x00, 0x05, 0xd4, 0x3b, 0xba, 0x40, 0x00, 0x3f, 0x06, 0xbd, 0x99, 0xb9, 0x3b,
+		0xdc, 0x93, 0x58, 0xee, 0x4e, 0x13, 0x01, 0xbb, 0xcf, 0xd6, 0x45, 0xb7, 0x1b, 0xc0, 0xd5, 0xb8,
+		0xff, 0x24, 0x80, 0x10, 0x00, 0x04, 0x01, 0x55, 0x00, 0x00, 0x01, 0x01, 0x08, 0x0a, 0xc8, 0xc8,
+		0x56, 0x95, 0x00, 0x34, 0xf6, 0x0f, 0xe8, 0x1d, 0xbd, 0x41, 0x45, 0x92, 0x4c, 0xc2, 0x71, 0xe0,
+		0xeb, 0x2e, 0x35, 0x17, 0x7c, 0x2f, 0xb9, 0xa8, 0x05, 0x92, 0x0e, 0x03, 0x1b, 0x50, 0x53, 0x0c,
+		0xe5, 0x7d, 0x86, 0x75, 0x32, 0x8a, 0xcc, 0xe2, 0x26, 0xa8, 0x90, 0x21, 0x78, 0xbf, 0xce, 0x7a,
+		0xf8, 0xb5, 0x8d, 0x48, 0xe4, 0xaa, 0xfe, 0x26, 0x34, 0xe0, 0xad, 0xb9, 0xec, 0x79, 0x74, 0xd8,
+		0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0xdc, 0x20, 0x90, 0x93, 0x27, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x0f, 0x42, 0xa4, 0x00, 0x00, 0x3f, 0xff, 0x04, 0x39, 0x2c, 0xd9, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x42, 0xa4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x42, 0x4b,
+		0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x03, 0xe9, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x17,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
+		0x00, 0x00, 0x00, 0x90, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x05, 0xca, 0x00, 0x00, 0x00, 0x04,
+		0x00, 0x00, 0x00, 0x80, 0xda, 0xb1, 0x22, 0xfb, 0xd9, 0xcf, 0x74, 0x83, 0xef, 0x30, 0x65, 0xb7,
+		0x81, 0x00, 0x00, 0x17, 0x08, 0x00, 0x45, 0x00, 0x05, 0xb4, 0xe2, 0x28, 0x40, 0x00, 0x3f, 0x06,
+		0x15, 0x0f, 0xc3, 0xb5, 0xaf, 0x26, 0x05, 0x92, 0xc6, 0x9e, 0x00, 0x50, 0x0f, 0xb3, 0x35, 0x8e,
+		0x36, 0x02, 0xa1, 0x01, 0xed, 0xb0, 0x80, 0x10, 0x00, 0x3b, 0xf7, 0xd4, 0x00, 0x00, 0x01, 0x01,
+		0x08, 0x0a, 0xd2, 0xe8, 0xac, 0xbe, 0x00, 0x36, 0xbc, 0x3c, 0x37, 0x36, 0xc4, 0x80, 0x3f, 0x66,
+		0x33, 0xc5, 0x50, 0xa6, 0x63, 0xb2, 0x92, 0xc3, 0x6a, 0x7a, 0x80, 0x65, 0x0b, 0x22, 0x62, 0xfe,
+		0x16, 0x9c, 0xab, 0x55, 0x03, 0x47, 0xa6, 0x54, 0x63, 0xa5, 0xbc, 0x17, 0x8e, 0x5a, 0xf6, 0xbc,
+		0x24, 0x52, 0xe9, 0xd2, 0x7b, 0x08, 0xe8, 0xc2, 0x6b, 0x05, 0x1c, 0xc0, 0x61, 0xb4, 0xe0, 0x43,
+		0x59, 0x62, 0xbf, 0x0a, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0xdc, 0x04, 0x12, 0xa0, 0x65,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x42, 0xa8, 0x00, 0x00, 0x3f, 0xff, 0xa4, 0x06, 0x9f, 0x9b,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x42, 0xa8, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x0f, 0x42, 0xa4, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x03, 0xe9, 0x00, 0x00, 0x00, 0x10,
+		0x00, 0x00, 0x05, 0x39, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x39, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x90, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x05, 0xf2,
+		0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x80, 0x74, 0x83, 0xef, 0x30, 0x65, 0xb7, 0x28, 0x99,
+		0x3a, 0x4e, 0x89, 0x27, 0x81, 0x00, 0x05, 0x39, 0x08, 0x00, 0x45, 0x18, 0x05, 0xdc, 0x8e, 0x5c,
+		0x40, 0x00, 0x3a, 0x06, 0x53, 0x77, 0x89, 0x4a, 0xcc, 0xd5, 0x59, 0xbb, 0xa9, 0x55, 0x07, 0x8f,
+		0xad, 0xdc, 0xf2, 0x9b, 0x09, 0xb4, 0xce, 0x1d, 0xbc, 0xee, 0x80, 0x10, 0x75, 0x40, 0x58, 0x02,
+		0x00, 0x00, 0x01, 0x01, 0x08, 0x0a, 0xb0, 0x18, 0x5b, 0x6f, 0xd7, 0xd6, 0x8b, 0x47, 0xee, 0x6a,
+		0x03, 0x0b, 0x9b, 0x52, 0xb1, 0xca, 0x61, 0x4b, 0x84, 0x57, 0x75, 0xc4, 0xb2, 0x18, 0x11, 0x39,
+		0xce, 0x5d, 0x2a, 0x38, 0x91, 0x29, 0x76, 0x11, 0x7d, 0xc1, 0xcc, 0x5c, 0x4b, 0x0a, 0xde, 0xbb,
+		0xa8, 0xad, 0x9d, 0x88, 0x36, 0x8b, 0xc0, 0x02, 0x87, 0xa7, 0xa5, 0x1c, 0xd9, 0x85, 0x71, 0x85,
+		0x68, 0x2b, 0x59, 0xc6, 0x2c, 0x3c, 0x84, 0x0c, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0xdc,
+		0x20, 0x90, 0x93, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x42, 0xa4, 0x00, 0x00, 0x3f, 0xff,
+		0x04, 0x39, 0x6c, 0xd8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x42, 0xa4,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x42, 0x4b, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x03, 0xe9,
+		0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x90, 0x00, 0x00, 0x00, 0x01,
+		0x00, 0x00, 0x05, 0xf2, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x80, 0xda, 0xb1, 0x22, 0xfb,
+		0xd9, 0xcf, 0x74, 0x83, 0xef, 0x30, 0x65, 0xb7, 0x81, 0x00, 0x00, 0x17, 0x08, 0x00, 0x45, 0x00,
+		0x05, 0xdc, 0x7e, 0x42, 0x40, 0x00, 0x3f, 0x06, 0x12, 0x4d, 0xb9, 0x66, 0xdb, 0x43, 0x67, 0xc2,
+		0xa9, 0x20, 0x63, 0x75, 0x57, 0xae, 0x6d, 0xbf, 0x59, 0x7c, 0x93, 0x71, 0x09, 0x67, 0x80, 0x10,
+		0x00, 0xeb, 0xfc, 0x16, 0x00, 0x00, 0x01, 0x01, 0x08, 0x0a, 0x40, 0x96, 0x88, 0x38, 0x36, 0xe1,
+		0x64, 0xc7, 0x1b, 0x43, 0xbc, 0x0e, 0x1f, 0x81, 0x6d, 0x39, 0xf6, 0x12, 0x0c, 0xea, 0xc0, 0xea,
+		0x7b, 0xc1, 0x77, 0xe2, 0x92, 0x6a, 0xbf, 0xbe, 0x84, 0xd9, 0x00, 0x18, 0x57, 0x49, 0x92, 0x72,
+		0x8f, 0xa3, 0x78, 0x45, 0x6f, 0xc6, 0x98, 0x8f, 0x71, 0xb0, 0xc5, 0x52, 0x7d, 0x8a, 0x82, 0xef,
+		0x52, 0xdb, 0xe9, 0xdc, 0x0a, 0x52, 0xdb, 0x06, 0x51, 0x80, 0x80, 0xa9, 0x00, 0x00, 0x00, 0x03,
+		0x00, 0x00, 0x00, 0xdc, 0x20, 0x90, 0x93, 0x29, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x42, 0xa4,
+		0x00, 0x00, 0x3f, 0xff, 0x04, 0x39, 0xac, 0xd7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x0f, 0x42, 0xa4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x42, 0xa5, 0x00, 0x00, 0x00, 0x02,
+		0x00, 0x00, 0x03, 0xe9, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x03, 0xbd, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x03, 0xbd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x90,
+		0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x05, 0xf2, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x80,
+		0x90, 0xe2, 0xba, 0x89, 0x21, 0xad, 0x74, 0x83, 0xef, 0x30, 0x65, 0xb7, 0x81, 0x00, 0x03, 0xbd,
+		0x08, 0x00, 0x45, 0x00, 0x05, 0xdc, 0x76, 0xa2, 0x40, 0x00, 0x38, 0x06, 0xac, 0x75, 0x33, 0x5b,
+		0x74, 0x6c, 0xc3, 0xb5, 0xae, 0x87, 0x1f, 0x40, 0x80, 0x68, 0xab, 0xbb, 0x2f, 0x90, 0x01, 0xee,
+		0x3a, 0xaf, 0x80, 0x10, 0x00, 0xeb, 0x8e, 0xf4, 0x00, 0x00, 0x01, 0x01, 0x08, 0x0a, 0x34, 0xc0,
+		0xff, 0x26, 0xac, 0x90, 0xd5, 0xc4, 0xcc, 0xd7, 0xa4, 0xa5, 0x5b, 0xa3, 0x79, 0x33, 0xc1, 0x25,
+		0xcd, 0x84, 0xdc, 0xaa, 0x37, 0xc9, 0xe3, 0xab, 0xc6, 0xb4, 0xeb, 0xe3, 0x8d, 0x72, 0x06, 0xd1,
+		0x5a, 0x1f, 0x9a, 0x8b, 0xe9, 0x9a, 0xf7, 0x33, 0x35, 0xe5, 0xca, 0x67, 0xba, 0x04, 0xf9, 0x3c,
+		0x27, 0xff, 0xa3, 0xca, 0x5e, 0x90, 0xf9, 0xc7, 0xd1, 0xe4, 0xf8, 0xf5, 0x7a, 0x14, 0xdc, 0x1c,
+		0xb1, 0xde, 0x63, 0x75, 0xb2, 0x65, 0x27, 0xf0, 0x0d, 0x29, 0xc5, 0x56, 0x60, 0x4a, 0x50, 0x10,
+		0x00, 0x77, 0xc0, 0xef, 0x00, 0x00, 0x74, 0xcf, 0x8a, 0x79, 0x87, 0x77, 0x75, 0x64, 0x75, 0xeb,
+		0xa4, 0x56, 0xb4, 0xd8, 0x70, 0xca, 0xe6, 0x11, 0xbb, 0x9f, 0xa1, 0x63, 0x95, 0xa1, 0xb4, 0x81,
+		0x8d, 0x50, 0xe0, 0xd5, 0xa9, 0x2c, 0xd7, 0x8f, 0xfe, 0x78, 0xce, 0xff, 0x5a, 0xa6, 0xb6, 0xb9,
+		0xf1, 0xe9, 0x5f, 0xda, 0xcb, 0xf3, 0x62, 0x61, 0x5f, 0x2b, 0x32, 0x95, 0x5d, 0x96, 0x2e, 0xef,
+		0x32, 0x04, 0xff, 0xcc, 0x76, 0xba, 0x49, 0xab, 0x92, 0xa7, 0xf1, 0xcc, 0x52, 0x68, 0xde, 0x94,
+		0x90, 0xdb, 0x1b, 0xa0, 0x28, 0x8a, 0xf8, 0x64, 0x55, 0x9c, 0x9b, 0xf6, 0x9c, 0x44, 0xd9, 0x68,
+		0xc0, 0xe5, 0x2c, 0xe1, 0x3d, 0x29, 0x19, 0xef, 0x8b, 0x0c, 0x9d, 0x0a, 0x7e, 0xcd, 0xc2, 0xe9,
+		0x85, 0x6b, 0x85, 0xb3, 0x97, 0xbe, 0xc6, 0x26, 0xd2, 0xe5, 0x2e, 0x90, 0xa9, 0xac, 0xe3, 0xd8,
+		0xef, 0xbd, 0x7b, 0x40, 0xf8, 0xb7, 0xe3, 0xc3, 0x8d, 0xa7, 0x38, 0x0f, 0x87, 0x7a, 0x50, 0x62,
+		0xc8, 0xb8, 0xa4, 0x52, 0x6e, 0xdc, 0x92, 0x7f, 0xe6, 0x8d, 0x45, 0x39, 0xfd, 0x06, 0x6e, 0xd9,
+		0xb5, 0x65, 0xac, 0xae, 0x2b, 0x8d, 0xea, 0xcf, 0xa2, 0x98, 0x0b, 0xc6, 0x43, 0x2e, 0xa7, 0x71,
+		0x99, 0x2b, 0xea, 0xc3, 0x9c, 0x27, 0x74, 0x9e, 0xd5, 0x11, 0x60, 0x7a, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x7b, 0xd6, 0x2a, 0x39, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	}
+}

decoders/utils/utils.go

@@ -0,0 +1,16 @@
+package utils
+
+import (
+	"encoding/binary"
+	"io"
+)
+
+func BinaryDecoder(payload io.Reader, dests ...interface{}) error {
+	for _, dest := range dests {
+		err := binary.Read(payload, binary.BigEndian, dest)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

docs/agents.md

@@ -0,0 +1,51 @@
+# Agents
+
+There are various agents that can send samples to a flow collector.
+
+## Hardware
+
+### Juniper
+
+In the latest versions, Juniper supports sFlow and IPFIX protocols.
+
+[Documentation](https://www.juniper.net/documentation/us/en/software/junos/network-mgmt/topics/topic-map/sflow-monitoring-technology.html).
+
+Sample configuration:
+```
+set protocols sflow collector 10.0.0.1
+set protocols sflow collector udp-port 6343
+set protocols sflow interface ge-0/0/0
+set protocols sflow sample-rate 2048
+```
+
+## Software
+
+### hsflowd
+
+[Documentation](https://sflow.net/host-sflow-linux-config.php).
+
+Sample packets using pcap, iptables nflog and many more. Uses sFlow.
+
+Sample configuration:
+```
+sflow {
+  collector { ip = 10.0.0.1 udpport = 6343 }
+  pcap { dev = eth0 }
+}
+```
+
+Run with
+```bash
+$ hsflowd -d -f hsflowd.conf
+```
+
+### nProbe
+
+[Documentation](https://www.ntop.org/guides/nprobe/)
+
+Sample packets using pcap, iptables nflog and many more. Uses NetFlow v9 or IPFIX.
+
+Run with
+```bash
+$ nprobe -i eth0 -n 10.0.0.1:2055 -V 10
+```

docs/clickhouse.md

@@ -0,0 +1,27 @@
+# Flows and Clickhouse
+
+Clickhouse is a powerful data warehouse.
+
+A sample [docker-compose](../compose/docker-compose.yml) is provided.
+It's composed of:
+* Apache Kafka
+* Apache Zookeeper
+* GoFlow2
+* Prometheus
+* Clickhouse
+* Grafana
+
+To start the containers, use:
+```bash
+$ docker-compose up
+```
+
+This command will automatically build Grafana and GoFlow2 containers.
+
+GoFlow2 collects NetFlow v9/IPFIX and sFlow packets and sends as a protobuf into Kafka.
+Zookeeper coordinates Kafka and can also be used by Clickhouse to ensure replication.
+Prometheus scrapes the metrics of the collector.
+Clickhouse consumes from Kafka, stores raw data and aggregates over specific columns
+using `MATERIALIZED TABLES` and `VIEWS` defined in a [schema file](../compose/clickhouse/create.sh).
+Youj can visualize the data in Grafana (credentials: admin/admin) with the
+pre-made dashboards.

docs/contributors.md

@@ -0,0 +1,13 @@
+# Contributors
+
+A special thank you to all the contributors of GoFlow.
+* [debugloop](https://github.com/debugloop)
+* [simPod](https://github.com/simPod)
+* [mmlb](https://github.com/mmlb)
+* [kanocz](https://github.com/kanocz)
+* [morrowc](https://github.com/morrowc)
+* [SuperQ](https://github.com/SuperQ)
+* [shyam334](https://github.com/shyam334)
+* [leoluk](https://github.com/leoluk)
+
+and many more!

docs/logs.md

@@ -0,0 +1,2 @@
+# Logs
+

docs/protocols.md

@@ -0,0 +1,58 @@
+# Protocols
+
+You can find information on the protocols in the links below:
+* [sFlow](https://sflow.org/developers/specifications.php)
+* [NetFlow v5](https://www.cisco.com/c/en/us/td/docs/net_mgmt/netflow_collection_engine/3-6/user/guide/format.html)
+* [NetFlow v9](https://www.cisco.com/en/US/technologies/tk648/tk362/technologies_white_paper09186a00800a3db9.html)
+* [IPFIX](https://www.iana.org/assignments/ipfix/ipfix.xhtml)
+
+The mapping to the protobuf format is listed in the table below.
+
+| Field | Description | NetFlow v5 | sFlow | NetFlow v9 | IPFIX |
+| - | - | - | - | - | - |
+|Type|Type of flow message|NETFLOW_V5|SFLOW_5|NETFLOW_V9|IPFIX|
+|TimeReceived|Timestamp of when the message was received|Included|Included|Included|Included|
+|SequenceNum|Sequence number of the flow packet|Included|Included|Included|Included|
+|SamplingRate|Sampling rate of the flow|Included|Included|Included|Included|
+|FlowDirection|Direction of the flow| | |DIRECTION (61)|flowDirection (61)|
+|SamplerAddress|Address of the device that generated the packet|IP source of packet|Agent IP|IP source of packet|IP source of packet|
+|TimeFlowStart|Time the flow started|System uptime and first|=TimeReceived|System uptime and FIRST_SWITCHED (22)|flowStartXXX (150, 152, 154, 156)|
+|TimeFlowEnd|Time the flow ended|System uptime and last|=TimeReceived|System uptime and LAST_SWITCHED (23)|flowEndXXX (151, 153, 155, 157)|
+|Bytes|Number of bytes in flow|dOctets|Length of sample|IN_BYTES (1) OUT_BYTES (23)|octetDeltaCount (1) postOctetDeltaCount (23)|
+|Packets|Number of packets in flow|dPkts|=1|IN_PKTS (2) OUT_PKTS (24)|packetDeltaCount (1) postPacketDeltaCount (24)|
+|SrcAddr|Source address (IP)|srcaddr (IPv4 only)|Included|Included|IPV4_SRC_ADDR (8) IPV6_SRC_ADDR (27)|sourceIPv4Address/sourceIPv6Address (8/27)|
+|DstAddr|Destination address (IP)|dstaddr (IPv4 only)|Included|Included|IPV4_DST_ADDR (12) IPV6_DST_ADDR (28)|destinationIPv4Address (12)destinationIPv6Address (28)|
+|Etype|Ethernet type (0x86dd for IPv6...)|IPv4|Included|Included|Included|
+|Proto|Protocol (UDP, TCP, ICMP...)|prot|Included|PROTOCOL (4)|protocolIdentifier (4)|
+|SrcPort|Source port (when UDP/TCP/SCTP)|srcport|Included|L4_DST_PORT (11)|destinationTransportPort (11)|
+|DstPort|Destination port (when UDP/TCP/SCTP)|dstport|Included|L4_SRC_PORT (7)|sourceTransportPort (7)|
+|InIf|Input interface|input|Included|INPUT_SNMP (10)|ingressInterface (10)|
+|OutIf|Output interface|output|Included|OUTPUT_SNMP (14)|egressInterface (14)|
+|SrcMac|Source mac address| |Included|IN_SRC_MAC (56)|sourceMacAddress (56)|
+|DstMac|Destination mac address| |Included|OUT_DST_MAC (57)|postDestinationMacAddress (57)|
+|SrcVlan|Source VLAN ID| |From ExtendedSwitch|SRC_VLAN (59)|vlanId (58)|
+|DstVlan|Destination VLAN ID| |From ExtendedSwitch|DST_VLAN (59)|postVlanId (59)|
+|VlanId|802.11q VLAN ID| |Included|SRC_VLAN (59)|postVlanId (59)|
+|IngressVrfID|VRF ID| | | |ingressVRFID (234)| 
+|EgressVrfID|VRF ID| | | |egressVRFID (235)|
+|IPTos|IP Type of Service|tos|Included|SRC_TOS (5)|ipClassOfService (5)|
+|ForwardingStatus|Forwarding status| | |FORWARDING_STATUS (89)|forwardingStatus (89)|
+|IPTTL|IP Time to Live| |Included|IPTTL (52)|minimumTTL (52|
+|TCPFlags|TCP flags|tcp_flags|Included|TCP_FLAGS (6)|tcpControlBits (6)|
+|IcmpType|ICMP Type| |Included|ICMP_TYPE (32)|icmpTypeXXX (176, 178) icmpTypeCodeXXX (32, 139)|
+|IcmpCode|ICMP Code| |Included|ICMP_TYPE (32)|icmpCodeXXX (177, 179) icmpTypeCodeXXX (32, 139)|
+|IPv6FlowLabel|IPv6 Flow Label| |Included|IPV6_FLOW_LABEL (31)|flowLabelIPv6 (31)|
+|FragmentId|IP Fragment ID| |Included|IPV4_IDENT (54)|fragmentIdentification (54)|
+|FragmentOffset|IP Fragment Offset| |Included|FRAGMENT_OFFSET (88)|fragmentOffset (88) and fragmentFlags (197)|
+|BiFlowDirection|BiFlow Identification| | | |biflowDirection (239)|
+|SrcAS|Source AS number|src_as|From ExtendedGateway|SRC_AS (16)|bgpSourceAsNumber (16)|
+|DstAS|Destination AS number|dst_as|From ExtendedGateway|DST_AS (17)|bgpDestinationAsNumber (17)|
+|NextHop|Nexthop address|nexthop|From ExtendedGateway|IPV4_NEXT_HOP (15) BGP_IPV4_NEXT_HOP (18) IPV6_NEXT_HOP (62) BGP_IPV6_NEXT_HOP (63)|ipNextHopIPv4Address (15) bgpNextHopIPv4Address (18) ipNextHopIPv6Address (62) bgpNextHopIPv6Address (63)|
+|NextHopAS|Nexthop AS number| |From ExtendedGateway| | |
+|SrcNet|Source address mask|src_mask|From ExtendedRouter|SRC_MASK (9) IPV6_SRC_MASK (29)|sourceIPv4PrefixLength (9) sourceIPv6PrefixLength (29)|
+|DstNet|Destination address mask|dst_mask|From ExtendedRouter|DST_MASK (13) IPV6_DST_MASK (30)|destinationIPv4PrefixLength (13) destinationIPv6PrefixLength (30)|
+|HasMPLS|Indicates the presence of MPLS header||Included|||
+|MPLSCount|Count of MPLS layers||Included|||
+|MPLSxTTL|TTL of the MPLS label||Included|||
+|MPLSxLabel|MPLS label||Included|||
+

format/format.go

@@ -0,0 +1,64 @@
+package format
+
+import (
+	"context"
+	"fmt"
+	"sync"
+)
+
+var (
+	formatDrivers = make(map[string]FormatDriver)
+	lock          = &sync.RWMutex{}
+)
+
+type FormatDriver interface {
+	Prepare() error                                  // Prepare driver (eg: flag registration)
+	Init(context.Context) error                      // Initialize driver (eg: parse keying)
+	Format(data interface{}) ([]byte, []byte, error) // Send a message
+}
+
+type FormatInterface interface {
+	Format(data interface{}) ([]byte, []byte, error)
+}
+
+type Format struct {
+	driver FormatDriver
+}
+
+func (t *Format) Format(data interface{}) ([]byte, []byte, error) {
+	return t.driver.Format(data)
+}
+
+func RegisterFormatDriver(name string, t FormatDriver) {
+	lock.Lock()
+	formatDrivers[name] = t
+	lock.Unlock()
+
+	if err := t.Prepare(); err != nil {
+		panic(err)
+	}
+}
+
+func FindFormat(ctx context.Context, name string) (*Format, error) {
+	lock.RLock()
+	t, ok := formatDrivers[name]
+	lock.RUnlock()
+	if !ok {
+		return nil, fmt.Errorf("Format %s not found", name)
+	}
+
+	err := t.Init(ctx)
+	return &Format{t}, err
+}
+
+func GetFormats() []string {
+	lock.RLock()
+	t := make([]string, len(formatDrivers))
+	var i int
+	for k, _ := range formatDrivers {
+		t[i] = k
+		i++
+	}
+	lock.RUnlock()
+	return t
+}

format/json/json.go

@@ -0,0 +1,372 @@
+package json
+
+import (
+	"context"
+	"encoding/binary"
+	"fmt"
+	"github.com/golang/protobuf/proto"
+	"github.com/netsampler/goflow2/format"
+	"github.com/netsampler/goflow2/format/protobuf"
+	flowmessage "github.com/netsampler/goflow2/pb"
+	"net"
+	"reflect"
+	"strings"
+)
+
+const (
+	FORMAT_TYPE_UNKNOWN = iota
+	FORMAT_TYPE_STRING_FUNC
+	FORMAT_TYPE_STRING
+	FORMAT_TYPE_INTEGER
+	FORMAT_TYPE_IP
+	FORMAT_TYPE_MAC
+)
+
+var (
+	EtypeName = map[uint32]string{
+		0x806:  "ARP",
+		0x800:  "IPv4",
+		0x86dd: "IPv6",
+	}
+	ProtoName = map[uint32]string{
+		1:  "ICMP",
+		6:  "TCP",
+		17: "UDP",
+		58: "ICMPv6",
+	}
+	IcmpTypeName = map[uint32]string{
+		0:  "EchoReply",
+		3:  "DestinationUnreachable",
+		8:  "Echo",
+		9:  "RouterAdvertisement",
+		10: "RouterSolicitation",
+		11: "TimeExceeded",
+	}
+	Icmp6TypeName = map[uint32]string{
+		1:   "DestinationUnreachable",
+		2:   "PacketTooBig",
+		3:   "TimeExceeded",
+		128: "EchoRequest",
+		129: "EchoReply",
+		133: "RouterSolicitation",
+		134: "RouterAdvertisement",
+	}
+
+	JsonFields = []string{
+		"Type",
+		"TimeReceived",
+		"SequenceNum",
+		"SamplingRate",
+		"SamplerAddress",
+		"TimeFlowStart",
+		"TimeFlowEnd",
+		"Bytes",
+		"Packets",
+		"SrcAddr",
+		"DstAddr",
+		"Etype",
+		"Proto",
+		"SrcPort",
+		"DstPort",
+		"InIf",
+		"OutIf",
+		"SrcMac",
+		"DstMac",
+		"SrcVlan",
+		"DstVlan",
+		"VlanId",
+		"IngressVrfID",
+		"EgressVrfID",
+		"IPTos",
+		"ForwardingStatus",
+		"IPTTL",
+		"TCPFlags",
+		"IcmpType",
+		"IcmpCode",
+		"IPv6FlowLabel",
+		"FragmentId",
+		"FragmentOffset",
+		"BiFlowDirection",
+		"SrcAS",
+		"DstAS",
+		"NextHop",
+		"NextHopAS",
+		"SrcNet",
+		"DstNet",
+	}
+	JsonFieldsTypes = []int{
+		FORMAT_TYPE_STRING_FUNC,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_IP,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_IP,
+		FORMAT_TYPE_IP,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_MAC,
+		FORMAT_TYPE_MAC,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_IP,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_INTEGER,
+		FORMAT_TYPE_INTEGER,
+	}
+	JsonExtras = []string{
+		"EtypeName",
+		"ProtoName",
+		"IcmpName",
+	}
+	JsonExtraCall = []JsonExtraFunction{
+		JsonExtraFunctionEtypeName,
+		JsonExtraFunctionProtoName,
+		JsonExtraFunctionIcmpName,
+	}
+)
+
+func AddJSONField(name string, jtype int) {
+	JsonFields = append(JsonFields, name)
+	JsonFieldsTypes = append(JsonFieldsTypes, jtype)
+}
+
+type JsonExtraFunction func(proto.Message) string
+
+func JsonExtraFetchNumbers(msg proto.Message, fields []string) []uint64 {
+	vfm := reflect.ValueOf(msg)
+	vfm = reflect.Indirect(vfm)
+
+	values := make([]uint64, len(fields))
+	for i, kf := range fields {
+		fieldValue := vfm.FieldByName(kf)
+		if fieldValue.IsValid() {
+			values[i] = fieldValue.Uint()
+		}
+	}
+
+	return values
+}
+
+func JsonExtraFunctionEtypeName(msg proto.Message) string {
+	num := JsonExtraFetchNumbers(msg, []string{"Etype"})
+	return EtypeName[uint32(num[0])]
+}
+func JsonExtraFunctionProtoName(msg proto.Message) string {
+	num := JsonExtraFetchNumbers(msg, []string{"Proto"})
+	return ProtoName[uint32(num[0])]
+}
+func JsonExtraFunctionIcmpName(msg proto.Message) string {
+	num := JsonExtraFetchNumbers(msg, []string{"Proto", "IcmpCode", "IcmpType"})
+	return IcmpCodeType(uint32(num[0]), uint32(num[1]), uint32(num[2]))
+}
+
+func IcmpCodeType(proto, icmpCode, icmpType uint32) string {
+	if proto == 1 {
+		return IcmpTypeName[icmpType]
+	} else if proto == 58 {
+		return Icmp6TypeName[icmpType]
+	}
+	return ""
+}
+
+type JsonDriver struct {
+	fieldsVar string
+	fields    []string // Hashing fields
+}
+
+func RenderIP(addr []byte) string {
+	if addr == nil || (len(addr) != 4 && len(addr) != 16) {
+		return ""
+	}
+
+	return net.IP(addr).String()
+}
+
+func (d *JsonDriver) Prepare() error {
+	return nil
+}
+
+func (d *JsonDriver) Init(context.Context) error {
+	return protobuf.ManualInit()
+}
+
+func (d *JsonDriver) Format(data interface{}) ([]byte, []byte, error) {
+	msg, ok := data.(proto.Message)
+	if !ok {
+		return nil, nil, fmt.Errorf("message is not protobuf")
+	}
+
+	key := protobuf.HashProtoLocal(msg)
+	return []byte(key), []byte(FormatMessageReflect(msg, "")), nil
+}
+
+func FormatMessageReflect(msg proto.Message, ext string) string {
+	fstr := make([]string, len(JsonFields)+len(JsonExtras))
+
+	vfm := reflect.ValueOf(msg)
+	vfm = reflect.Indirect(vfm)
+
+	for i, kf := range JsonFields {
+		fieldValue := vfm.FieldByName(kf)
+		if fieldValue.IsValid() {
+
+			switch JsonFieldsTypes[i] {
+			case FORMAT_TYPE_STRING_FUNC:
+				strMethod := fieldValue.MethodByName("String").Call([]reflect.Value{})
+				fstr[i] = fmt.Sprintf("\"%s\":\"%s\"", kf, strMethod[0].String())
+			case FORMAT_TYPE_STRING:
+				fstr[i] = fmt.Sprintf("\"%s\":\"%s\"", kf, fieldValue.String())
+			case FORMAT_TYPE_INTEGER:
+				fstr[i] = fmt.Sprintf("\"%s\":%d", kf, fieldValue.Uint())
+			case FORMAT_TYPE_IP:
+				ip := fieldValue.Bytes()
+				fstr[i] = fmt.Sprintf("\"%s\":\"%s\"", kf, RenderIP(ip))
+			case FORMAT_TYPE_MAC:
+				mac := make([]byte, 8)
+				binary.BigEndian.PutUint64(mac, fieldValue.Uint())
+				fstr[i] = fmt.Sprintf("\"%s\":\"%s\"", kf, net.HardwareAddr(mac[2:]).String())
+			default:
+				fstr[i] = fmt.Sprintf("\"%s\":null", kf)
+			}
+
+		} else {
+			fstr[i] = fmt.Sprintf("\"%s\":null", kf)
+		}
+	}
+
+	for i, e := range JsonExtras {
+		fstr[i+len(JsonFields)] = fmt.Sprintf("\"%s\":\"%s\"", e, JsonExtraCall[i](msg))
+	}
+
+	return fmt.Sprintf("{%s}", strings.Join(fstr, ","))
+}
+
+func FormatMessage(msg *flowmessage.FlowMessage, ext string) string {
+	srcmac := make([]byte, 8)
+	dstmac := make([]byte, 8)
+	binary.BigEndian.PutUint64(srcmac, msg.SrcMac)
+	binary.BigEndian.PutUint64(dstmac, msg.DstMac)
+	srcmac = srcmac[2:8]
+	dstmac = dstmac[2:8]
+
+	b := fmt.Sprintf(
+		"{"+
+			"\"Type\":\"%v\","+
+			"\"TimeReceived\":%d,"+
+			"\"SequenceNum\":%d,"+
+			"\"SamplingRate\":%d,"+
+			"\"SamplerAddress\":\"%v\","+
+			"\"TimeFlowStart\":%d,"+
+			"\"TimeFlowEnd\":%d,"+
+			"\"Bytes\":%d,"+
+			"\"Packets\":%d,"+
+			"\"SrcAddr\":\"%v\","+
+			"\"DstAddr\":\"%v\","+
+			"\"Etype\":%d,"+
+			"\"EtypeName\":\"%s\","+
+			"\"Proto\":%d,"+
+			"\"ProtoName\":\"%s\","+
+			"\"SrcPort\":%d,"+
+			"\"DstPort\":%d,"+
+			"\"InIf\":%d,"+
+			"\"OutIf\":%d,"+
+			"\"SrcMac\":\"%v\","+
+			"\"DstMac\":\"%v\","+
+			"\"SrcVlan\":%d,"+
+			"\"DstVlan\":%d,"+
+			"\"VlanId\":%d,"+
+			"\"IngressVrfID\":%d,"+
+			"\"EgressVrfID\":%d,"+
+			"\"IPTos\":%d,"+
+			"\"ForwardingStatus\":%d,"+
+			"\"IPTTL\":%d,"+
+			"\"TCPFlags\":%d,"+
+			"\"IcmpType\":%d,"+
+			"\"IcmpCode\":%d,"+
+			"\"IcmpName\":\"%s\","+
+			"\"IPv6FlowLabel\":%d,"+
+			"\"FragmentId\":%d,"+
+			"\"FragmentOffset\":%d,"+
+			"\"BiFlowDirection\":\"%v\","+
+			"\"SrcAS\":%d,"+
+			"\"DstAS\":%d,"+
+			"\"NextHop\":\"%v\","+
+			"\"NextHopAS\":%d,"+
+			"\"SrcNet\":%d,"+
+			"\"DstNet\":%d"+
+			"%s}",
+		msg.Type.String(),
+		msg.TimeReceived,
+		msg.SequenceNum,
+		msg.SamplingRate,
+		RenderIP(msg.SamplerAddress),
+		msg.TimeFlowStart,
+		msg.TimeFlowEnd,
+		msg.Bytes,
+		msg.Packets,
+		RenderIP(msg.SrcAddr),
+		RenderIP(msg.DstAddr),
+		msg.Etype,
+		EtypeName[msg.Etype],
+		msg.Proto,
+		ProtoName[msg.Proto],
+		msg.SrcPort,
+		msg.DstPort,
+		msg.InIf,
+		msg.OutIf,
+		net.HardwareAddr(srcmac).String(),
+		net.HardwareAddr(dstmac).String(),
+		msg.SrcVlan,
+		msg.DstVlan,
+		msg.VlanId,
+		msg.IngressVrfID,
+		msg.EgressVrfID,
+		msg.IPTos,
+		msg.ForwardingStatus,
+		msg.IPTTL,
+		msg.TCPFlags,
+		msg.IcmpType,
+		msg.IcmpCode,
+		IcmpCodeType(msg.Proto, msg.IcmpCode, msg.IcmpType),
+		msg.IPv6FlowLabel,
+		msg.FragmentId,
+		msg.FragmentOffset,
+		msg.BiFlowDirection,
+		msg.SrcAS,
+		msg.DstAS,
+		RenderIP(msg.NextHop),
+		msg.NextHopAS,
+		msg.SrcNet,
+		msg.DstNet,
+		ext)
+
+	return b
+}
+
+func init() {
+	d := &JsonDriver{}
+	format.RegisterFormatDriver("json", d)
+}

format/protobuf/protobuf.go

@@ -0,0 +1,79 @@
+package protobuf
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"github.com/golang/protobuf/proto"
+	"github.com/netsampler/goflow2/format"
+	"reflect"
+	"strings"
+)
+
+var (
+	fieldsVar string
+	fields    []string // Hashing fields
+)
+
+type ProtobufDriver struct {
+	fixedLen bool
+}
+
+func (d *ProtobufDriver) Prepare() error {
+	flag.StringVar(&fieldsVar, "format.hash", "SamplerAddress", "List of fields to do hashing, separated by commas")
+	flag.BoolVar(&d.fixedLen, "format.protobuf.fixedlen", false, "Prefix the protobuf with message length")
+	return nil
+}
+
+func ManualInit() error {
+	fields = strings.Split(fieldsVar, ",")
+	return nil
+}
+
+func (d *ProtobufDriver) Init(context.Context) error {
+	return ManualInit()
+}
+
+func (d *ProtobufDriver) Format(data interface{}) ([]byte, []byte, error) {
+	msg, ok := data.(proto.Message)
+	if !ok {
+		return nil, nil, fmt.Errorf("message is not protobuf")
+	}
+	key := HashProtoLocal(msg)
+
+	if !d.fixedLen {
+		b, err := proto.Marshal(msg)
+		return []byte(key), b, err
+	} else {
+		buf := proto.NewBuffer([]byte{})
+		err := buf.EncodeMessage(msg)
+		return []byte(key), buf.Bytes(), err
+	}
+}
+
+func init() {
+	d := &ProtobufDriver{}
+	format.RegisterFormatDriver("pb", d)
+}
+
+func HashProtoLocal(msg interface{}) string {
+	return HashProto(fields, msg)
+}
+
+func HashProto(fields []string, msg interface{}) string {
+	var keyStr string
+
+	if msg != nil {
+		vfm := reflect.ValueOf(msg)
+		vfm = reflect.Indirect(vfm)
+
+		for _, kf := range fields {
+			fieldValue := vfm.FieldByName(kf)
+			if fieldValue.IsValid() {
+				keyStr += fmt.Sprintf("%v-", fieldValue)
+			}
+		}
+	}
+
+	return keyStr
+}

go.mod

@@ -0,0 +1,14 @@
+module github.com/netsampler/goflow2
+
+go 1.15
+
+require (
+	github.com/Shopify/sarama v1.19.0
+	github.com/golang/protobuf v1.4.3
+	github.com/libp2p/go-reuseport v0.0.2
+	github.com/oschwald/geoip2-golang v1.5.0
+	github.com/prometheus/client_golang v1.10.0
+	github.com/sirupsen/logrus v1.8.1
+	github.com/stretchr/testify v1.7.0
+	google.golang.org/protobuf v1.23.0
+)

go.sum

@@ -0,0 +1,428 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
+github.com/Shopify/sarama v1.19.0 h1:9oksLxC6uxVPHPVYUmq6xhr1BOF/hHobWH2UzO67z1s=
+github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
+github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
+github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
+github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
+github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
+github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
+github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
+github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
+github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A=
+github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
+github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
+github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
+github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/eapache/go-resiliency v1.1.0 h1:1NtRmCAqadE2FN4ZcN6g90TP3uk8cg9rn9eNK2197aU=
+github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
+github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
+github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
+github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
+github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
+github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
+github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
+github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w=
+github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
+github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE=
+github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
+github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
+github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
+github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
+github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
+github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
+github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/libp2p/go-reuseport v0.0.2 h1:XSG94b1FJfGA01BUrT82imejHQyTxO4jEWqheyCXYvU=
+github.com/libp2p/go-reuseport v0.0.2/go.mod h1:SPD+5RwGC7rcnzngoYC86GjPzjSywuQyMVAheVBD9nQ=
+github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
+github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
+github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
+github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
+github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
+github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=
+github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU=
+github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k=
+github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w=
+github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
+github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
+github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
+github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs=
+github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
+github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
+github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis=
+github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
+github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA=
+github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
+github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
+github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
+github.com/oschwald/geoip2-golang v1.5.0 h1:igg2yQIrrcRccB1ytFXqBfOHCjXWIoMv85lVJ1ONZzw=
+github.com/oschwald/geoip2-golang v1.5.0/go.mod h1:xdvYt5xQzB8ORWFqPnqMwZpCpgNagttWdoZLlJQzg7s=
+github.com/oschwald/maxminddb-golang v1.8.0 h1:Uh/DSnGoxsyp/KYbY1AuP0tYEwfs0sCph9p/UMXK/Hk=
+github.com/oschwald/maxminddb-golang v1.8.0/go.mod h1:RXZtst0N6+FY/3qCNmZMBApR19cdQj43/NM9VkrNAis=
+github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM=
+github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
+github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
+github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
+github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
+github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_golang v1.10.0 h1:/o0BDeWzLWXNZ+4q5gXltUvaMpJqckTa+jTNoB+z4cg=
+github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/common v0.18.0 h1:WCVKW7aL6LEe1uryfI9dnEc2ZqNB1Fn0ok930v0iL1Y=
+github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
+github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ=
+github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
+github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
+github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
+github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
+go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
+go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
+go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY=
+golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
+sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU=

package/goflow2.env

@@ -0,0 +1 @@
+GOFLOW2_ARGS=

package/goflow2.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=GoFlow2
+After=network.target
+
+[Service]
+Type=simple
+EnvironmentFile=/etc/default/goflow2
+WorkingDirectory=/usr/share/goflow2
+ExecStart=/usr/bin/goflow2 $GOFLOW2_ARGS
+
+[Install]
+WantedBy=multi-user.target

pb/flow.pb.go

@@ -0,0 +1,721 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0
+// 	protoc        v3.15.0
+// source: pb/flow.proto
+
+package flowpb
+
+import (
+	proto "github.com/golang/protobuf/proto"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
+
+type FlowMessage_FlowType int32
+
+const (
+	FlowMessage_FLOWUNKNOWN FlowMessage_FlowType = 0
+	FlowMessage_SFLOW_5     FlowMessage_FlowType = 1
+	FlowMessage_NETFLOW_V5  FlowMessage_FlowType = 2
+	FlowMessage_NETFLOW_V9  FlowMessage_FlowType = 3
+	FlowMessage_IPFIX       FlowMessage_FlowType = 4
+)
+
+// Enum value maps for FlowMessage_FlowType.
+var (
+	FlowMessage_FlowType_name = map[int32]string{
+		0: "FLOWUNKNOWN",
+		1: "SFLOW_5",
+		2: "NETFLOW_V5",
+		3: "NETFLOW_V9",
+		4: "IPFIX",
+	}
+	FlowMessage_FlowType_value = map[string]int32{
+		"FLOWUNKNOWN": 0,
+		"SFLOW_5":     1,
+		"NETFLOW_V5":  2,
+		"NETFLOW_V9":  3,
+		"IPFIX":       4,
+	}
+)
+
+func (x FlowMessage_FlowType) Enum() *FlowMessage_FlowType {
+	p := new(FlowMessage_FlowType)
+	*p = x
+	return p
+}
+
+func (x FlowMessage_FlowType) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (FlowMessage_FlowType) Descriptor() protoreflect.EnumDescriptor {
+	return file_pb_flow_proto_enumTypes[0].Descriptor()
+}
+
+func (FlowMessage_FlowType) Type() protoreflect.EnumType {
+	return &file_pb_flow_proto_enumTypes[0]
+}
+
+func (x FlowMessage_FlowType) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use FlowMessage_FlowType.Descriptor instead.
+func (FlowMessage_FlowType) EnumDescriptor() ([]byte, []int) {
+	return file_pb_flow_proto_rawDescGZIP(), []int{0, 0}
+}
+
+type FlowMessage struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Type          FlowMessage_FlowType `protobuf:"varint,1,opt,name=Type,proto3,enum=flowpb.FlowMessage_FlowType" json:"Type,omitempty"`
+	TimeReceived  uint64               `protobuf:"varint,2,opt,name=TimeReceived,proto3" json:"TimeReceived,omitempty"`
+	SequenceNum   uint32               `protobuf:"varint,4,opt,name=SequenceNum,proto3" json:"SequenceNum,omitempty"`
+	SamplingRate  uint64               `protobuf:"varint,3,opt,name=SamplingRate,proto3" json:"SamplingRate,omitempty"`
+	FlowDirection uint32               `protobuf:"varint,42,opt,name=FlowDirection,proto3" json:"FlowDirection,omitempty"`
+	// Sampler information
+	SamplerAddress []byte `protobuf:"bytes,11,opt,name=SamplerAddress,proto3" json:"SamplerAddress,omitempty"`
+	// Found inside packet
+	TimeFlowStart uint64 `protobuf:"varint,38,opt,name=TimeFlowStart,proto3" json:"TimeFlowStart,omitempty"`
+	TimeFlowEnd   uint64 `protobuf:"varint,5,opt,name=TimeFlowEnd,proto3" json:"TimeFlowEnd,omitempty"`
+	// Size of the sampled packet
+	Bytes   uint64 `protobuf:"varint,9,opt,name=Bytes,proto3" json:"Bytes,omitempty"`
+	Packets uint64 `protobuf:"varint,10,opt,name=Packets,proto3" json:"Packets,omitempty"`
+	// Source/destination addresses
+	SrcAddr []byte `protobuf:"bytes,6,opt,name=SrcAddr,proto3" json:"SrcAddr,omitempty"`
+	DstAddr []byte `protobuf:"bytes,7,opt,name=DstAddr,proto3" json:"DstAddr,omitempty"`
+	// Layer 3 protocol (IPv4/IPv6/ARP/MPLS...)
+	Etype uint32 `protobuf:"varint,30,opt,name=Etype,proto3" json:"Etype,omitempty"`
+	// Layer 4 protocol
+	Proto uint32 `protobuf:"varint,20,opt,name=Proto,proto3" json:"Proto,omitempty"`
+	// Ports for UDP and TCP
+	SrcPort uint32 `protobuf:"varint,21,opt,name=SrcPort,proto3" json:"SrcPort,omitempty"`
+	DstPort uint32 `protobuf:"varint,22,opt,name=DstPort,proto3" json:"DstPort,omitempty"`
+	// Interfaces
+	InIf  uint32 `protobuf:"varint,18,opt,name=InIf,proto3" json:"InIf,omitempty"`
+	OutIf uint32 `protobuf:"varint,19,opt,name=OutIf,proto3" json:"OutIf,omitempty"`
+	// Ethernet information
+	SrcMac uint64 `protobuf:"varint,27,opt,name=SrcMac,proto3" json:"SrcMac,omitempty"`
+	DstMac uint64 `protobuf:"varint,28,opt,name=DstMac,proto3" json:"DstMac,omitempty"`
+	// Vlan
+	SrcVlan uint32 `protobuf:"varint,33,opt,name=SrcVlan,proto3" json:"SrcVlan,omitempty"`
+	DstVlan uint32 `protobuf:"varint,34,opt,name=DstVlan,proto3" json:"DstVlan,omitempty"`
+	// 802.1q VLAN in sampled packet
+	VlanId uint32 `protobuf:"varint,29,opt,name=VlanId,proto3" json:"VlanId,omitempty"`
+	// VRF
+	IngressVrfID uint32 `protobuf:"varint,39,opt,name=IngressVrfID,proto3" json:"IngressVrfID,omitempty"`
+	EgressVrfID  uint32 `protobuf:"varint,40,opt,name=EgressVrfID,proto3" json:"EgressVrfID,omitempty"`
+	// IP and TCP special flags
+	IPTos            uint32 `protobuf:"varint,23,opt,name=IPTos,proto3" json:"IPTos,omitempty"`
+	ForwardingStatus uint32 `protobuf:"varint,24,opt,name=ForwardingStatus,proto3" json:"ForwardingStatus,omitempty"`
+	IPTTL            uint32 `protobuf:"varint,25,opt,name=IPTTL,proto3" json:"IPTTL,omitempty"`
+	TCPFlags         uint32 `protobuf:"varint,26,opt,name=TCPFlags,proto3" json:"TCPFlags,omitempty"`
+	IcmpType         uint32 `protobuf:"varint,31,opt,name=IcmpType,proto3" json:"IcmpType,omitempty"`
+	IcmpCode         uint32 `protobuf:"varint,32,opt,name=IcmpCode,proto3" json:"IcmpCode,omitempty"`
+	IPv6FlowLabel    uint32 `protobuf:"varint,37,opt,name=IPv6FlowLabel,proto3" json:"IPv6FlowLabel,omitempty"`
+	// Fragments (IPv4/IPv6)
+	FragmentId      uint32 `protobuf:"varint,35,opt,name=FragmentId,proto3" json:"FragmentId,omitempty"`
+	FragmentOffset  uint32 `protobuf:"varint,36,opt,name=FragmentOffset,proto3" json:"FragmentOffset,omitempty"`
+	BiFlowDirection uint32 `protobuf:"varint,41,opt,name=BiFlowDirection,proto3" json:"BiFlowDirection,omitempty"`
+	// Autonomous system information
+	SrcAS     uint32 `protobuf:"varint,14,opt,name=SrcAS,proto3" json:"SrcAS,omitempty"`
+	DstAS     uint32 `protobuf:"varint,15,opt,name=DstAS,proto3" json:"DstAS,omitempty"`
+	NextHop   []byte `protobuf:"bytes,12,opt,name=NextHop,proto3" json:"NextHop,omitempty"`
+	NextHopAS uint32 `protobuf:"varint,13,opt,name=NextHopAS,proto3" json:"NextHopAS,omitempty"`
+	// Prefix size
+	SrcNet uint32 `protobuf:"varint,16,opt,name=SrcNet,proto3" json:"SrcNet,omitempty"`
+	DstNet uint32 `protobuf:"varint,17,opt,name=DstNet,proto3" json:"DstNet,omitempty"`
+	// MPLS information
+	HasMPLS       bool   `protobuf:"varint,53,opt,name=HasMPLS,proto3" json:"HasMPLS,omitempty"`
+	MPLSCount     uint32 `protobuf:"varint,54,opt,name=MPLSCount,proto3" json:"MPLSCount,omitempty"`
+	MPLS1TTL      uint32 `protobuf:"varint,55,opt,name=MPLS1TTL,proto3" json:"MPLS1TTL,omitempty"`           // First TTL
+	MPLS1Label    uint32 `protobuf:"varint,56,opt,name=MPLS1Label,proto3" json:"MPLS1Label,omitempty"`       // First Label
+	MPLS2TTL      uint32 `protobuf:"varint,57,opt,name=MPLS2TTL,proto3" json:"MPLS2TTL,omitempty"`           // Second TTL
+	MPLS2Label    uint32 `protobuf:"varint,58,opt,name=MPLS2Label,proto3" json:"MPLS2Label,omitempty"`       // Second Label
+	MPLS3TTL      uint32 `protobuf:"varint,59,opt,name=MPLS3TTL,proto3" json:"MPLS3TTL,omitempty"`           // Third TTL
+	MPLS3Label    uint32 `protobuf:"varint,60,opt,name=MPLS3Label,proto3" json:"MPLS3Label,omitempty"`       // Third Label
+	MPLSLastTTL   uint32 `protobuf:"varint,61,opt,name=MPLSLastTTL,proto3" json:"MPLSLastTTL,omitempty"`     // Last TTL
+	MPLSLastLabel uint32 `protobuf:"varint,62,opt,name=MPLSLastLabel,proto3" json:"MPLSLastLabel,omitempty"` // Last Label
+}
+
+func (x *FlowMessage) Reset() {
+	*x = FlowMessage{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_pb_flow_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *FlowMessage) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*FlowMessage) ProtoMessage() {}
+
+func (x *FlowMessage) ProtoReflect() protoreflect.Message {
+	mi := &file_pb_flow_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use FlowMessage.ProtoReflect.Descriptor instead.
+func (*FlowMessage) Descriptor() ([]byte, []int) {
+	return file_pb_flow_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *FlowMessage) GetType() FlowMessage_FlowType {
+	if x != nil {
+		return x.Type
+	}
+	return FlowMessage_FLOWUNKNOWN
+}
+
+func (x *FlowMessage) GetTimeReceived() uint64 {
+	if x != nil {
+		return x.TimeReceived
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetSequenceNum() uint32 {
+	if x != nil {
+		return x.SequenceNum
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetSamplingRate() uint64 {
+	if x != nil {
+		return x.SamplingRate
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetFlowDirection() uint32 {
+	if x != nil {
+		return x.FlowDirection
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetSamplerAddress() []byte {
+	if x != nil {
+		return x.SamplerAddress
+	}
+	return nil
+}
+
+func (x *FlowMessage) GetTimeFlowStart() uint64 {
+	if x != nil {
+		return x.TimeFlowStart
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetTimeFlowEnd() uint64 {
+	if x != nil {
+		return x.TimeFlowEnd
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetBytes() uint64 {
+	if x != nil {
+		return x.Bytes
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetPackets() uint64 {
+	if x != nil {
+		return x.Packets
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetSrcAddr() []byte {
+	if x != nil {
+		return x.SrcAddr
+	}
+	return nil
+}
+
+func (x *FlowMessage) GetDstAddr() []byte {
+	if x != nil {
+		return x.DstAddr
+	}
+	return nil
+}
+
+func (x *FlowMessage) GetEtype() uint32 {
+	if x != nil {
+		return x.Etype
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetProto() uint32 {
+	if x != nil {
+		return x.Proto
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetSrcPort() uint32 {
+	if x != nil {
+		return x.SrcPort
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetDstPort() uint32 {
+	if x != nil {
+		return x.DstPort
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetInIf() uint32 {
+	if x != nil {
+		return x.InIf
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetOutIf() uint32 {
+	if x != nil {
+		return x.OutIf
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetSrcMac() uint64 {
+	if x != nil {
+		return x.SrcMac
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetDstMac() uint64 {
+	if x != nil {
+		return x.DstMac
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetSrcVlan() uint32 {
+	if x != nil {
+		return x.SrcVlan
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetDstVlan() uint32 {
+	if x != nil {
+		return x.DstVlan
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetVlanId() uint32 {
+	if x != nil {
+		return x.VlanId
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetIngressVrfID() uint32 {
+	if x != nil {
+		return x.IngressVrfID
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetEgressVrfID() uint32 {
+	if x != nil {
+		return x.EgressVrfID
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetIPTos() uint32 {
+	if x != nil {
+		return x.IPTos
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetForwardingStatus() uint32 {
+	if x != nil {
+		return x.ForwardingStatus
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetIPTTL() uint32 {
+	if x != nil {
+		return x.IPTTL
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetTCPFlags() uint32 {
+	if x != nil {
+		return x.TCPFlags
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetIcmpType() uint32 {
+	if x != nil {
+		return x.IcmpType
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetIcmpCode() uint32 {
+	if x != nil {
+		return x.IcmpCode
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetIPv6FlowLabel() uint32 {
+	if x != nil {
+		return x.IPv6FlowLabel
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetFragmentId() uint32 {
+	if x != nil {
+		return x.FragmentId
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetFragmentOffset() uint32 {
+	if x != nil {
+		return x.FragmentOffset
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetBiFlowDirection() uint32 {
+	if x != nil {
+		return x.BiFlowDirection
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetSrcAS() uint32 {
+	if x != nil {
+		return x.SrcAS
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetDstAS() uint32 {
+	if x != nil {
+		return x.DstAS
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetNextHop() []byte {
+	if x != nil {
+		return x.NextHop
+	}
+	return nil
+}
+
+func (x *FlowMessage) GetNextHopAS() uint32 {
+	if x != nil {
+		return x.NextHopAS
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetSrcNet() uint32 {
+	if x != nil {
+		return x.SrcNet
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetDstNet() uint32 {
+	if x != nil {
+		return x.DstNet
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetHasMPLS() bool {
+	if x != nil {
+		return x.HasMPLS
+	}
+	return false
+}
+
+func (x *FlowMessage) GetMPLSCount() uint32 {
+	if x != nil {
+		return x.MPLSCount
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetMPLS1TTL() uint32 {
+	if x != nil {
+		return x.MPLS1TTL
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetMPLS1Label() uint32 {
+	if x != nil {
+		return x.MPLS1Label
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetMPLS2TTL() uint32 {
+	if x != nil {
+		return x.MPLS2TTL
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetMPLS2Label() uint32 {
+	if x != nil {
+		return x.MPLS2Label
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetMPLS3TTL() uint32 {
+	if x != nil {
+		return x.MPLS3TTL
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetMPLS3Label() uint32 {
+	if x != nil {
+		return x.MPLS3Label
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetMPLSLastTTL() uint32 {
+	if x != nil {
+		return x.MPLSLastTTL
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetMPLSLastLabel() uint32 {
+	if x != nil {
+		return x.MPLSLastLabel
+	}
+	return 0
+}
+
+var File_pb_flow_proto protoreflect.FileDescriptor
+
+var file_pb_flow_proto_rawDesc = []byte{
+	0x0a, 0x0d, 0x70, 0x62, 0x2f, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12,
+	0x06, 0x66, 0x6c, 0x6f, 0x77, 0x70, 0x62, 0x22, 0xd0, 0x0c, 0x0a, 0x0b, 0x46, 0x6c, 0x6f, 0x77,
+	0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x30, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1c, 0x2e, 0x66, 0x6c, 0x6f, 0x77, 0x70, 0x62, 0x2e, 0x46,
+	0x6c, 0x6f, 0x77, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x46, 0x6c, 0x6f, 0x77, 0x54,
+	0x79, 0x70, 0x65, 0x52, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x54, 0x69, 0x6d,
+	0x65, 0x52, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52,
+	0x0c, 0x54, 0x69, 0x6d, 0x65, 0x52, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x64, 0x12, 0x20, 0x0a,
+	0x0b, 0x53, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x4e, 0x75, 0x6d, 0x18, 0x04, 0x20, 0x01,
+	0x28, 0x0d, 0x52, 0x0b, 0x53, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x4e, 0x75, 0x6d, 0x12,
+	0x22, 0x0a, 0x0c, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x69, 0x6e, 0x67, 0x52, 0x61, 0x74, 0x65, 0x18,
+	0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0c, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x69, 0x6e, 0x67, 0x52,
+	0x61, 0x74, 0x65, 0x12, 0x24, 0x0a, 0x0d, 0x46, 0x6c, 0x6f, 0x77, 0x44, 0x69, 0x72, 0x65, 0x63,
+	0x74, 0x69, 0x6f, 0x6e, 0x18, 0x2a, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, 0x46, 0x6c, 0x6f, 0x77,
+	0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x26, 0x0a, 0x0e, 0x53, 0x61, 0x6d,
+	0x70, 0x6c, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28,
+	0x0c, 0x52, 0x0e, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73,
+	0x73, 0x12, 0x24, 0x0a, 0x0d, 0x54, 0x69, 0x6d, 0x65, 0x46, 0x6c, 0x6f, 0x77, 0x53, 0x74, 0x61,
+	0x72, 0x74, 0x18, 0x26, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0d, 0x54, 0x69, 0x6d, 0x65, 0x46, 0x6c,
+	0x6f, 0x77, 0x53, 0x74, 0x61, 0x72, 0x74, 0x12, 0x20, 0x0a, 0x0b, 0x54, 0x69, 0x6d, 0x65, 0x46,
+	0x6c, 0x6f, 0x77, 0x45, 0x6e, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x54, 0x69,
+	0x6d, 0x65, 0x46, 0x6c, 0x6f, 0x77, 0x45, 0x6e, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x42, 0x79, 0x74,
+	0x65, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12,
+	0x18, 0x0a, 0x07, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x04,
+	0x52, 0x07, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x72, 0x63,
+	0x41, 0x64, 0x64, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x53, 0x72, 0x63, 0x41,
+	0x64, 0x64, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x44, 0x73, 0x74, 0x41, 0x64, 0x64, 0x72, 0x18, 0x07,
+	0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x44, 0x73, 0x74, 0x41, 0x64, 0x64, 0x72, 0x12, 0x14, 0x0a,
+	0x05, 0x45, 0x74, 0x79, 0x70, 0x65, 0x18, 0x1e, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x45, 0x74,
+	0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x18, 0x14, 0x20, 0x01,
+	0x28, 0x0d, 0x52, 0x05, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x72, 0x63,
+	0x50, 0x6f, 0x72, 0x74, 0x18, 0x15, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x53, 0x72, 0x63, 0x50,
+	0x6f, 0x72, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x44, 0x73, 0x74, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x16,
+	0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x44, 0x73, 0x74, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x12, 0x0a,
+	0x04, 0x49, 0x6e, 0x49, 0x66, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x49, 0x6e, 0x49,
+	0x66, 0x12, 0x14, 0x0a, 0x05, 0x4f, 0x75, 0x74, 0x49, 0x66, 0x18, 0x13, 0x20, 0x01, 0x28, 0x0d,
+	0x52, 0x05, 0x4f, 0x75, 0x74, 0x49, 0x66, 0x12, 0x16, 0x0a, 0x06, 0x53, 0x72, 0x63, 0x4d, 0x61,
+	0x63, 0x18, 0x1b, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x53, 0x72, 0x63, 0x4d, 0x61, 0x63, 0x12,
+	0x16, 0x0a, 0x06, 0x44, 0x73, 0x74, 0x4d, 0x61, 0x63, 0x18, 0x1c, 0x20, 0x01, 0x28, 0x04, 0x52,
+	0x06, 0x44, 0x73, 0x74, 0x4d, 0x61, 0x63, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x72, 0x63, 0x56, 0x6c,
+	0x61, 0x6e, 0x18, 0x21, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x53, 0x72, 0x63, 0x56, 0x6c, 0x61,
+	0x6e, 0x12, 0x18, 0x0a, 0x07, 0x44, 0x73, 0x74, 0x56, 0x6c, 0x61, 0x6e, 0x18, 0x22, 0x20, 0x01,
+	0x28, 0x0d, 0x52, 0x07, 0x44, 0x73, 0x74, 0x56, 0x6c, 0x61, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x56,
+	0x6c, 0x61, 0x6e, 0x49, 0x64, 0x18, 0x1d, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x56, 0x6c, 0x61,
+	0x6e, 0x49, 0x64, 0x12, 0x22, 0x0a, 0x0c, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x56, 0x72,
+	0x66, 0x49, 0x44, 0x18, 0x27, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0c, 0x49, 0x6e, 0x67, 0x72, 0x65,
+	0x73, 0x73, 0x56, 0x72, 0x66, 0x49, 0x44, 0x12, 0x20, 0x0a, 0x0b, 0x45, 0x67, 0x72, 0x65, 0x73,
+	0x73, 0x56, 0x72, 0x66, 0x49, 0x44, 0x18, 0x28, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x45, 0x67,
+	0x72, 0x65, 0x73, 0x73, 0x56, 0x72, 0x66, 0x49, 0x44, 0x12, 0x14, 0x0a, 0x05, 0x49, 0x50, 0x54,
+	0x6f, 0x73, 0x18, 0x17, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x49, 0x50, 0x54, 0x6f, 0x73, 0x12,
+	0x2a, 0x0a, 0x10, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x74, 0x61,
+	0x74, 0x75, 0x73, 0x18, 0x18, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x10, 0x46, 0x6f, 0x72, 0x77, 0x61,
+	0x72, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x49,
+	0x50, 0x54, 0x54, 0x4c, 0x18, 0x19, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x49, 0x50, 0x54, 0x54,
+	0x4c, 0x12, 0x1a, 0x0a, 0x08, 0x54, 0x43, 0x50, 0x46, 0x6c, 0x61, 0x67, 0x73, 0x18, 0x1a, 0x20,
+	0x01, 0x28, 0x0d, 0x52, 0x08, 0x54, 0x43, 0x50, 0x46, 0x6c, 0x61, 0x67, 0x73, 0x12, 0x1a, 0x0a,
+	0x08, 0x49, 0x63, 0x6d, 0x70, 0x54, 0x79, 0x70, 0x65, 0x18, 0x1f, 0x20, 0x01, 0x28, 0x0d, 0x52,
+	0x08, 0x49, 0x63, 0x6d, 0x70, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x49, 0x63, 0x6d,
+	0x70, 0x43, 0x6f, 0x64, 0x65, 0x18, 0x20, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x49, 0x63, 0x6d,
+	0x70, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x24, 0x0a, 0x0d, 0x49, 0x50, 0x76, 0x36, 0x46, 0x6c, 0x6f,
+	0x77, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x18, 0x25, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, 0x49, 0x50,
+	0x76, 0x36, 0x46, 0x6c, 0x6f, 0x77, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x1e, 0x0a, 0x0a, 0x46,
+	0x72, 0x61, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x18, 0x23, 0x20, 0x01, 0x28, 0x0d, 0x52,
+	0x0a, 0x46, 0x72, 0x61, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x26, 0x0a, 0x0e, 0x46,
+	0x72, 0x61, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x24, 0x20,
+	0x01, 0x28, 0x0d, 0x52, 0x0e, 0x46, 0x72, 0x61, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x4f, 0x66, 0x66,
+	0x73, 0x65, 0x74, 0x12, 0x28, 0x0a, 0x0f, 0x42, 0x69, 0x46, 0x6c, 0x6f, 0x77, 0x44, 0x69, 0x72,
+	0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x29, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0f, 0x42, 0x69,
+	0x46, 0x6c, 0x6f, 0x77, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a,
+	0x05, 0x53, 0x72, 0x63, 0x41, 0x53, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x53, 0x72,
+	0x63, 0x41, 0x53, 0x12, 0x14, 0x0a, 0x05, 0x44, 0x73, 0x74, 0x41, 0x53, 0x18, 0x0f, 0x20, 0x01,
+	0x28, 0x0d, 0x52, 0x05, 0x44, 0x73, 0x74, 0x41, 0x53, 0x12, 0x18, 0x0a, 0x07, 0x4e, 0x65, 0x78,
+	0x74, 0x48, 0x6f, 0x70, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x4e, 0x65, 0x78, 0x74,
+	0x48, 0x6f, 0x70, 0x12, 0x1c, 0x0a, 0x09, 0x4e, 0x65, 0x78, 0x74, 0x48, 0x6f, 0x70, 0x41, 0x53,
+	0x18, 0x0d, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x4e, 0x65, 0x78, 0x74, 0x48, 0x6f, 0x70, 0x41,
+	0x53, 0x12, 0x16, 0x0a, 0x06, 0x53, 0x72, 0x63, 0x4e, 0x65, 0x74, 0x18, 0x10, 0x20, 0x01, 0x28,
+	0x0d, 0x52, 0x06, 0x53, 0x72, 0x63, 0x4e, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x73, 0x74,
+	0x4e, 0x65, 0x74, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x44, 0x73, 0x74, 0x4e, 0x65,
+	0x74, 0x12, 0x18, 0x0a, 0x07, 0x48, 0x61, 0x73, 0x4d, 0x50, 0x4c, 0x53, 0x18, 0x35, 0x20, 0x01,
+	0x28, 0x08, 0x52, 0x07, 0x48, 0x61, 0x73, 0x4d, 0x50, 0x4c, 0x53, 0x12, 0x1c, 0x0a, 0x09, 0x4d,
+	0x50, 0x4c, 0x53, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x36, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09,
+	0x4d, 0x50, 0x4c, 0x53, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x4d, 0x50, 0x4c,
+	0x53, 0x31, 0x54, 0x54, 0x4c, 0x18, 0x37, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x4d, 0x50, 0x4c,
+	0x53, 0x31, 0x54, 0x54, 0x4c, 0x12, 0x1e, 0x0a, 0x0a, 0x4d, 0x50, 0x4c, 0x53, 0x31, 0x4c, 0x61,
+	0x62, 0x65, 0x6c, 0x18, 0x38, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x4d, 0x50, 0x4c, 0x53, 0x31,
+	0x4c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x1a, 0x0a, 0x08, 0x4d, 0x50, 0x4c, 0x53, 0x32, 0x54, 0x54,
+	0x4c, 0x18, 0x39, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x4d, 0x50, 0x4c, 0x53, 0x32, 0x54, 0x54,
+	0x4c, 0x12, 0x1e, 0x0a, 0x0a, 0x4d, 0x50, 0x4c, 0x53, 0x32, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x18,
+	0x3a, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x4d, 0x50, 0x4c, 0x53, 0x32, 0x4c, 0x61, 0x62, 0x65,
+	0x6c, 0x12, 0x1a, 0x0a, 0x08, 0x4d, 0x50, 0x4c, 0x53, 0x33, 0x54, 0x54, 0x4c, 0x18, 0x3b, 0x20,
+	0x01, 0x28, 0x0d, 0x52, 0x08, 0x4d, 0x50, 0x4c, 0x53, 0x33, 0x54, 0x54, 0x4c, 0x12, 0x1e, 0x0a,
+	0x0a, 0x4d, 0x50, 0x4c, 0x53, 0x33, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x18, 0x3c, 0x20, 0x01, 0x28,
+	0x0d, 0x52, 0x0a, 0x4d, 0x50, 0x4c, 0x53, 0x33, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x20, 0x0a,
+	0x0b, 0x4d, 0x50, 0x4c, 0x53, 0x4c, 0x61, 0x73, 0x74, 0x54, 0x54, 0x4c, 0x18, 0x3d, 0x20, 0x01,
+	0x28, 0x0d, 0x52, 0x0b, 0x4d, 0x50, 0x4c, 0x53, 0x4c, 0x61, 0x73, 0x74, 0x54, 0x54, 0x4c, 0x12,
+	0x24, 0x0a, 0x0d, 0x4d, 0x50, 0x4c, 0x53, 0x4c, 0x61, 0x73, 0x74, 0x4c, 0x61, 0x62, 0x65, 0x6c,
+	0x18, 0x3e, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, 0x4d, 0x50, 0x4c, 0x53, 0x4c, 0x61, 0x73, 0x74,
+	0x4c, 0x61, 0x62, 0x65, 0x6c, 0x22, 0x53, 0x0a, 0x08, 0x46, 0x6c, 0x6f, 0x77, 0x54, 0x79, 0x70,
+	0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x46, 0x4c, 0x4f, 0x57, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e,
+	0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x46, 0x4c, 0x4f, 0x57, 0x5f, 0x35, 0x10, 0x01, 0x12,
+	0x0e, 0x0a, 0x0a, 0x4e, 0x45, 0x54, 0x46, 0x4c, 0x4f, 0x57, 0x5f, 0x56, 0x35, 0x10, 0x02, 0x12,
+	0x0e, 0x0a, 0x0a, 0x4e, 0x45, 0x54, 0x46, 0x4c, 0x4f, 0x57, 0x5f, 0x56, 0x39, 0x10, 0x03, 0x12,
+	0x09, 0x0a, 0x05, 0x49, 0x50, 0x46, 0x49, 0x58, 0x10, 0x04, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x33,
+}
+
+var (
+	file_pb_flow_proto_rawDescOnce sync.Once
+	file_pb_flow_proto_rawDescData = file_pb_flow_proto_rawDesc
+)
+
+func file_pb_flow_proto_rawDescGZIP() []byte {
+	file_pb_flow_proto_rawDescOnce.Do(func() {
+		file_pb_flow_proto_rawDescData = protoimpl.X.CompressGZIP(file_pb_flow_proto_rawDescData)
+	})
+	return file_pb_flow_proto_rawDescData
+}
+
+var file_pb_flow_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
+var file_pb_flow_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
+var file_pb_flow_proto_goTypes = []interface{}{
+	(FlowMessage_FlowType)(0), // 0: flowpb.FlowMessage.FlowType
+	(*FlowMessage)(nil),       // 1: flowpb.FlowMessage
+}
+var file_pb_flow_proto_depIdxs = []int32{
+	0, // 0: flowpb.FlowMessage.Type:type_name -> flowpb.FlowMessage.FlowType
+	1, // [1:1] is the sub-list for method output_type
+	1, // [1:1] is the sub-list for method input_type
+	1, // [1:1] is the sub-list for extension type_name
+	1, // [1:1] is the sub-list for extension extendee
+	0, // [0:1] is the sub-list for field type_name
+}
+
+func init() { file_pb_flow_proto_init() }
+func file_pb_flow_proto_init() {
+	if File_pb_flow_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_pb_flow_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*FlowMessage); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_pb_flow_proto_rawDesc,
+			NumEnums:      1,
+			NumMessages:   1,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_pb_flow_proto_goTypes,
+		DependencyIndexes: file_pb_flow_proto_depIdxs,
+		EnumInfos:         file_pb_flow_proto_enumTypes,
+		MessageInfos:      file_pb_flow_proto_msgTypes,
+	}.Build()
+	File_pb_flow_proto = out.File
+	file_pb_flow_proto_rawDesc = nil
+	file_pb_flow_proto_goTypes = nil
+	file_pb_flow_proto_depIdxs = nil
+}

pb/flow.proto

@@ -0,0 +1,103 @@
+syntax = "proto3";
+package flowpb;
+
+message FlowMessage {
+
+  enum FlowType {
+    FLOWUNKNOWN = 0;
+    SFLOW_5 = 1;
+    NETFLOW_V5 = 2;
+    NETFLOW_V9 = 3;
+    IPFIX = 4;
+  }
+  FlowType Type = 1;
+
+  uint64 TimeReceived = 2;
+  uint32 SequenceNum = 4;
+  uint64 SamplingRate = 3;
+
+  uint32 FlowDirection = 42;
+
+  // Sampler information
+  bytes SamplerAddress = 11;
+
+  // Found inside packet
+  uint64 TimeFlowStart = 38;
+  uint64 TimeFlowEnd = 5;
+
+  // Size of the sampled packet
+  uint64 Bytes = 9;
+  uint64 Packets = 10;
+
+  // Source/destination addresses
+  bytes SrcAddr = 6;
+  bytes DstAddr = 7;
+
+  // Layer 3 protocol (IPv4/IPv6/ARP/MPLS...)
+  uint32 Etype = 30;
+
+  // Layer 4 protocol
+  uint32 Proto = 20;
+
+  // Ports for UDP and TCP
+  uint32 SrcPort = 21;
+  uint32 DstPort = 22;
+
+  // Interfaces
+  uint32 InIf = 18;
+  uint32 OutIf = 19;
+
+  // Ethernet information
+  uint64 SrcMac = 27;
+  uint64 DstMac = 28;
+
+  // Vlan
+  uint32 SrcVlan = 33;
+  uint32 DstVlan = 34;
+  // 802.1q VLAN in sampled packet
+  uint32 VlanId = 29;
+
+  // VRF
+  uint32 IngressVrfID = 39;
+  uint32 EgressVrfID = 40;
+
+  // IP and TCP special flags
+  uint32 IPTos = 23;
+  uint32 ForwardingStatus = 24;
+  uint32 IPTTL = 25;
+  uint32 TCPFlags = 26;
+  uint32 IcmpType = 31;
+  uint32 IcmpCode = 32;
+  uint32 IPv6FlowLabel = 37;
+  // Fragments (IPv4/IPv6)
+  uint32 FragmentId = 35;
+  uint32 FragmentOffset = 36;
+  uint32 BiFlowDirection = 41;
+
+  // Autonomous system information
+  uint32 SrcAS = 14;
+  uint32 DstAS = 15;
+
+  bytes NextHop = 12;
+  uint32 NextHopAS = 13;
+
+  // Prefix size
+  uint32 SrcNet = 16;
+  uint32 DstNet = 17;
+
+  // MPLS information
+  bool HasMPLS = 53;
+  uint32 MPLSCount = 54;
+  uint32 MPLS1TTL = 55; // First TTL
+  uint32 MPLS1Label = 56; // First Label
+  uint32 MPLS2TTL = 57; // Second TTL
+  uint32 MPLS2Label = 58; // Second Label
+  uint32 MPLS3TTL = 59; // Third TTL
+  uint32 MPLS3Label = 60; // Third Label
+  uint32 MPLSLastTTL = 61; // Last TTL
+  uint32 MPLSLastLabel = 62; // Last Label
+
+  // Custom fields: start after ID 1000:
+  // uint32 MyCustomField = 1000;
+
+}

producer/producer_nf.go

@@ -0,0 +1,481 @@
+package producer
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"net"
+	"sync"
+	"time"
+
+	"github.com/netsampler/goflow2/decoders/netflow"
+	flowmessage "github.com/netsampler/goflow2/pb"
+)
+
+type SamplingRateSystem interface {
+	GetSamplingRate(version uint16, obsDomainId uint32) (uint32, error)
+	AddSamplingRate(version uint16, obsDomainId uint32, samplingRate uint32)
+}
+
+type basicSamplingRateSystem struct {
+	sampling     map[uint16]map[uint32]uint32
+	samplinglock *sync.RWMutex
+}
+
+func CreateSamplingSystem() SamplingRateSystem {
+	ts := &basicSamplingRateSystem{
+		sampling:     make(map[uint16]map[uint32]uint32),
+		samplinglock: &sync.RWMutex{},
+	}
+	return ts
+}
+
+func (s *basicSamplingRateSystem) AddSamplingRate(version uint16, obsDomainId uint32, samplingRate uint32) {
+	s.samplinglock.Lock()
+	_, exists := s.sampling[version]
+	if exists != true {
+		s.sampling[version] = make(map[uint32]uint32)
+	}
+	s.sampling[version][obsDomainId] = samplingRate
+	s.samplinglock.Unlock()
+}
+
+func (s *basicSamplingRateSystem) GetSamplingRate(version uint16, obsDomainId uint32) (uint32, error) {
+	s.samplinglock.RLock()
+	samplingVersion, okver := s.sampling[version]
+	if okver {
+		samplingRate, okid := samplingVersion[obsDomainId]
+		if okid {
+			s.samplinglock.RUnlock()
+			return samplingRate, nil
+		}
+		s.samplinglock.RUnlock()
+		return 0, errors.New("") // TBC
+	}
+	s.samplinglock.RUnlock()
+	return 0, errors.New("") // TBC
+}
+
+type SingleSamplingRateSystem struct {
+	Sampling uint32
+}
+
+func (s *SingleSamplingRateSystem) AddSamplingRate(version uint16, obsDomainId uint32, samplingRate uint32) {
+}
+
+func (s *SingleSamplingRateSystem) GetSamplingRate(version uint16, obsDomainId uint32) (uint32, error) {
+	return s.Sampling, nil
+}
+
+func NetFlowLookFor(dataFields []netflow.DataField, typeId uint16) (bool, interface{}) {
+	for _, dataField := range dataFields {
+		if dataField.Type == typeId {
+			return true, dataField.Value
+		}
+	}
+	return false, nil
+}
+
+func NetFlowPopulate(dataFields []netflow.DataField, typeId uint16, addr interface{}) bool {
+	exists, value := NetFlowLookFor(dataFields, typeId)
+	if exists && value != nil {
+		valueBytes, ok := value.([]byte)
+		valueReader := bytes.NewReader(valueBytes)
+		if ok {
+			switch addrt := addr.(type) {
+			case *(net.IP):
+				*addrt = valueBytes
+			case *(time.Time):
+				t := uint64(0)
+				binary.Read(valueReader, binary.BigEndian, &t)
+				t64 := int64(t / 1000)
+				*addrt = time.Unix(t64, 0)
+			default:
+				binary.Read(valueReader, binary.BigEndian, addr)
+			}
+		}
+	}
+	return exists
+}
+
+func DecodeUNumber(b []byte, out interface{}) error {
+	var o uint64
+	l := len(b)
+	switch l {
+	case 1:
+		o = uint64(b[0])
+	case 2:
+		o = uint64(binary.BigEndian.Uint16(b))
+	case 4:
+		o = uint64(binary.BigEndian.Uint32(b))
+	case 8:
+		o = binary.BigEndian.Uint64(b)
+	default:
+		if l < 8 {
+			var iter uint
+			for i := range b {
+				o |= uint64(b[i]) << uint(8*(uint(l)-iter-1))
+				iter++
+			}
+		} else {
+			return errors.New(fmt.Sprintf("Non-regular number of bytes for a number: %v", l))
+		}
+	}
+	switch t := out.(type) {
+	case *byte:
+		*t = byte(o)
+	case *uint16:
+		*t = uint16(o)
+	case *uint32:
+		*t = uint32(o)
+	case *uint64:
+		*t = o
+	default:
+		return errors.New("The parameter is not a pointer to a byte/uint16/uint32/uint64 structure")
+	}
+	return nil
+}
+
+func ConvertNetFlowDataSet(version uint16, baseTime uint32, uptime uint32, record []netflow.DataField) *flowmessage.FlowMessage {
+	flowMessage := &flowmessage.FlowMessage{}
+	var time uint64
+
+	if version == 9 {
+		flowMessage.Type = flowmessage.FlowMessage_NETFLOW_V9
+	} else if version == 10 {
+		flowMessage.Type = flowmessage.FlowMessage_IPFIX
+	}
+
+	for i := range record {
+		df := record[i]
+
+		v, ok := df.Value.([]byte)
+		if !ok {
+			continue
+		}
+
+		switch df.Type {
+
+		// Statistics
+		case netflow.NFV9_FIELD_IN_BYTES:
+			DecodeUNumber(v, &(flowMessage.Bytes))
+		case netflow.NFV9_FIELD_IN_PKTS:
+			DecodeUNumber(v, &(flowMessage.Packets))
+		case netflow.NFV9_FIELD_OUT_BYTES:
+			DecodeUNumber(v, &(flowMessage.Bytes))
+		case netflow.NFV9_FIELD_OUT_PKTS:
+			DecodeUNumber(v, &(flowMessage.Packets))
+
+		// L4
+		case netflow.NFV9_FIELD_L4_SRC_PORT:
+			DecodeUNumber(v, &(flowMessage.SrcPort))
+		case netflow.NFV9_FIELD_L4_DST_PORT:
+			DecodeUNumber(v, &(flowMessage.DstPort))
+		case netflow.NFV9_FIELD_PROTOCOL:
+			DecodeUNumber(v, &(flowMessage.Proto))
+
+		// Network
+		case netflow.NFV9_FIELD_SRC_AS:
+			DecodeUNumber(v, &(flowMessage.SrcAS))
+		case netflow.NFV9_FIELD_DST_AS:
+			DecodeUNumber(v, &(flowMessage.DstAS))
+
+		// Interfaces
+		case netflow.NFV9_FIELD_INPUT_SNMP:
+			DecodeUNumber(v, &(flowMessage.InIf))
+		case netflow.NFV9_FIELD_OUTPUT_SNMP:
+			DecodeUNumber(v, &(flowMessage.OutIf))
+
+		case netflow.NFV9_FIELD_FORWARDING_STATUS:
+			DecodeUNumber(v, &(flowMessage.ForwardingStatus))
+		case netflow.NFV9_FIELD_SRC_TOS:
+			DecodeUNumber(v, &(flowMessage.IPTos))
+		case netflow.NFV9_FIELD_TCP_FLAGS:
+			DecodeUNumber(v, &(flowMessage.TCPFlags))
+		case netflow.NFV9_FIELD_MIN_TTL:
+			DecodeUNumber(v, &(flowMessage.IPTTL))
+
+		// IP
+		case netflow.NFV9_FIELD_IPV4_SRC_ADDR:
+			flowMessage.SrcAddr = v
+			flowMessage.Etype = 0x800
+		case netflow.NFV9_FIELD_IPV4_DST_ADDR:
+			flowMessage.DstAddr = v
+			flowMessage.Etype = 0x800
+
+		case netflow.NFV9_FIELD_SRC_MASK:
+			DecodeUNumber(v, &(flowMessage.SrcNet))
+		case netflow.NFV9_FIELD_DST_MASK:
+			DecodeUNumber(v, &(flowMessage.DstNet))
+
+		case netflow.NFV9_FIELD_IPV6_SRC_ADDR:
+			flowMessage.SrcAddr = v
+			flowMessage.Etype = 0x86dd
+		case netflow.NFV9_FIELD_IPV6_DST_ADDR:
+			flowMessage.DstAddr = v
+			flowMessage.Etype = 0x86dd
+
+		case netflow.NFV9_FIELD_IPV6_SRC_MASK:
+			DecodeUNumber(v, &(flowMessage.SrcNet))
+		case netflow.NFV9_FIELD_IPV6_DST_MASK:
+			DecodeUNumber(v, &(flowMessage.DstNet))
+
+		case netflow.NFV9_FIELD_IPV4_NEXT_HOP:
+			flowMessage.NextHop = v
+		case netflow.NFV9_FIELD_BGP_IPV4_NEXT_HOP:
+			flowMessage.NextHop = v
+
+		case netflow.NFV9_FIELD_IPV6_NEXT_HOP:
+			flowMessage.NextHop = v
+		case netflow.NFV9_FIELD_BGP_IPV6_NEXT_HOP:
+			flowMessage.NextHop = v
+
+		// ICMP
+		case netflow.NFV9_FIELD_ICMP_TYPE:
+			var icmpTypeCode uint16
+			DecodeUNumber(v, &icmpTypeCode)
+			flowMessage.IcmpType = uint32(icmpTypeCode >> 8)
+			flowMessage.IcmpCode = uint32(icmpTypeCode & 0xff)
+		case netflow.IPFIX_FIELD_icmpTypeCodeIPv6:
+			var icmpTypeCode uint16
+			DecodeUNumber(v, &icmpTypeCode)
+			flowMessage.IcmpType = uint32(icmpTypeCode >> 8)
+			flowMessage.IcmpCode = uint32(icmpTypeCode & 0xff)
+		case netflow.IPFIX_FIELD_icmpTypeIPv4:
+			DecodeUNumber(v, &(flowMessage.IcmpType))
+		case netflow.IPFIX_FIELD_icmpTypeIPv6:
+			DecodeUNumber(v, &(flowMessage.IcmpType))
+		case netflow.IPFIX_FIELD_icmpCodeIPv4:
+			DecodeUNumber(v, &(flowMessage.IcmpCode))
+		case netflow.IPFIX_FIELD_icmpCodeIPv6:
+			DecodeUNumber(v, &(flowMessage.IcmpCode))
+
+		// Mac
+		case netflow.NFV9_FIELD_IN_SRC_MAC:
+			DecodeUNumber(v, &(flowMessage.SrcMac))
+		case netflow.NFV9_FIELD_OUT_DST_MAC:
+			DecodeUNumber(v, &(flowMessage.DstMac))
+
+		case netflow.NFV9_FIELD_SRC_VLAN:
+			DecodeUNumber(v, &(flowMessage.VlanId))
+			DecodeUNumber(v, &(flowMessage.SrcVlan))
+		case netflow.NFV9_FIELD_DST_VLAN:
+			DecodeUNumber(v, &(flowMessage.DstVlan))
+
+		case netflow.IPFIX_FIELD_ingressVRFID:
+			DecodeUNumber(v, &(flowMessage.IngressVrfID))
+		case netflow.IPFIX_FIELD_egressVRFID:
+			DecodeUNumber(v, &(flowMessage.EgressVrfID))
+
+		case netflow.NFV9_FIELD_IPV4_IDENT:
+			DecodeUNumber(v, &(flowMessage.FragmentId))
+		case netflow.NFV9_FIELD_FRAGMENT_OFFSET:
+			var fragOffset uint32
+			DecodeUNumber(v, &fragOffset)
+			flowMessage.FragmentOffset |= fragOffset
+		case netflow.IPFIX_FIELD_fragmentFlags:
+			var ipFlags uint32
+			DecodeUNumber(v, &ipFlags)
+			flowMessage.FragmentOffset |= ipFlags
+		case netflow.NFV9_FIELD_IPV6_FLOW_LABEL:
+			DecodeUNumber(v, &(flowMessage.IPv6FlowLabel))
+
+		case netflow.IPFIX_FIELD_biflowDirection:
+			DecodeUNumber(v, &(flowMessage.BiFlowDirection))
+
+		case netflow.NFV9_FIELD_DIRECTION:
+			DecodeUNumber(v, &(flowMessage.FlowDirection))
+
+		default:
+			if version == 9 {
+				// NetFlow v9 time works with a differential based on router's uptime
+				switch df.Type {
+				case netflow.NFV9_FIELD_FIRST_SWITCHED:
+					var timeFirstSwitched uint32
+					DecodeUNumber(v, &timeFirstSwitched)
+					timeDiff := (uptime - timeFirstSwitched) / 1000
+					flowMessage.TimeFlowStart = uint64(baseTime - timeDiff)
+				case netflow.NFV9_FIELD_LAST_SWITCHED:
+					var timeLastSwitched uint32
+					DecodeUNumber(v, &timeLastSwitched)
+					timeDiff := (uptime - timeLastSwitched) / 1000
+					flowMessage.TimeFlowEnd = uint64(baseTime - timeDiff)
+				}
+			} else if version == 10 {
+				switch df.Type {
+				case netflow.IPFIX_FIELD_flowStartSeconds:
+					DecodeUNumber(v, &time)
+					flowMessage.TimeFlowStart = time
+				case netflow.IPFIX_FIELD_flowStartMilliseconds:
+					DecodeUNumber(v, &time)
+					flowMessage.TimeFlowStart = time / 1000
+				case netflow.IPFIX_FIELD_flowStartMicroseconds:
+					DecodeUNumber(v, &time)
+					flowMessage.TimeFlowStart = time / 1000000
+				case netflow.IPFIX_FIELD_flowStartNanoseconds:
+					DecodeUNumber(v, &time)
+					flowMessage.TimeFlowStart = time / 1000000000
+				case netflow.IPFIX_FIELD_flowEndSeconds:
+					DecodeUNumber(v, &time)
+					flowMessage.TimeFlowEnd = time
+				case netflow.IPFIX_FIELD_flowEndMilliseconds:
+					DecodeUNumber(v, &time)
+					flowMessage.TimeFlowEnd = time / 1000
+				case netflow.IPFIX_FIELD_flowEndMicroseconds:
+					DecodeUNumber(v, &time)
+					flowMessage.TimeFlowEnd = time / 1000000
+				case netflow.IPFIX_FIELD_flowEndNanoseconds:
+					DecodeUNumber(v, &time)
+					flowMessage.TimeFlowEnd = time / 1000000000
+				}
+			}
+		}
+
+	}
+
+	return flowMessage
+}
+
+func SearchNetFlowDataSetsRecords(version uint16, baseTime uint32, uptime uint32, dataRecords []netflow.DataRecord) []*flowmessage.FlowMessage {
+	flowMessageSet := make([]*flowmessage.FlowMessage, 0)
+	for _, record := range dataRecords {
+		fmsg := ConvertNetFlowDataSet(version, baseTime, uptime, record.Values)
+		if fmsg != nil {
+			flowMessageSet = append(flowMessageSet, fmsg)
+		}
+	}
+	return flowMessageSet
+}
+
+func SearchNetFlowDataSets(version uint16, baseTime uint32, uptime uint32, dataFlowSet []netflow.DataFlowSet) []*flowmessage.FlowMessage {
+	flowMessageSet := make([]*flowmessage.FlowMessage, 0)
+	for _, dataFlowSetItem := range dataFlowSet {
+		fmsg := SearchNetFlowDataSetsRecords(version, baseTime, uptime, dataFlowSetItem.Records)
+		if fmsg != nil {
+			flowMessageSet = append(flowMessageSet, fmsg...)
+		}
+	}
+	return flowMessageSet
+}
+
+func SearchNetFlowOptionDataSets(dataFlowSet []netflow.OptionsDataFlowSet) (uint32, bool) {
+	var samplingRate uint32
+	var found bool
+	for _, dataFlowSetItem := range dataFlowSet {
+		for _, record := range dataFlowSetItem.Records {
+			b := NetFlowPopulate(record.OptionsValues, 305, &samplingRate)
+			if b {
+				return samplingRate, b
+			}
+			b = NetFlowPopulate(record.OptionsValues, 50, &samplingRate)
+			if b {
+				return samplingRate, b
+			}
+			b = NetFlowPopulate(record.OptionsValues, 34, &samplingRate)
+			if b {
+				return samplingRate, b
+			}
+		}
+	}
+	return samplingRate, found
+}
+
+func SplitNetFlowSets(packetNFv9 netflow.NFv9Packet) ([]netflow.DataFlowSet, []netflow.TemplateFlowSet, []netflow.NFv9OptionsTemplateFlowSet, []netflow.OptionsDataFlowSet) {
+	dataFlowSet := make([]netflow.DataFlowSet, 0)
+	templatesFlowSet := make([]netflow.TemplateFlowSet, 0)
+	optionsTemplatesFlowSet := make([]netflow.NFv9OptionsTemplateFlowSet, 0)
+	optionsDataFlowSet := make([]netflow.OptionsDataFlowSet, 0)
+	for _, flowSet := range packetNFv9.FlowSets {
+		switch flowSet.(type) {
+		case netflow.TemplateFlowSet:
+			templatesFlowSet = append(templatesFlowSet, flowSet.(netflow.TemplateFlowSet))
+		case netflow.NFv9OptionsTemplateFlowSet:
+			optionsTemplatesFlowSet = append(optionsTemplatesFlowSet, flowSet.(netflow.NFv9OptionsTemplateFlowSet))
+		case netflow.DataFlowSet:
+			dataFlowSet = append(dataFlowSet, flowSet.(netflow.DataFlowSet))
+		case netflow.OptionsDataFlowSet:
+			optionsDataFlowSet = append(optionsDataFlowSet, flowSet.(netflow.OptionsDataFlowSet))
+		}
+	}
+	return dataFlowSet, templatesFlowSet, optionsTemplatesFlowSet, optionsDataFlowSet
+}
+
+func SplitIPFIXSets(packetIPFIX netflow.IPFIXPacket) ([]netflow.DataFlowSet, []netflow.TemplateFlowSet, []netflow.IPFIXOptionsTemplateFlowSet, []netflow.OptionsDataFlowSet) {
+	dataFlowSet := make([]netflow.DataFlowSet, 0)
+	templatesFlowSet := make([]netflow.TemplateFlowSet, 0)
+	optionsTemplatesFlowSet := make([]netflow.IPFIXOptionsTemplateFlowSet, 0)
+	optionsDataFlowSet := make([]netflow.OptionsDataFlowSet, 0)
+	for _, flowSet := range packetIPFIX.FlowSets {
+		switch flowSet.(type) {
+		case netflow.TemplateFlowSet:
+			templatesFlowSet = append(templatesFlowSet, flowSet.(netflow.TemplateFlowSet))
+		case netflow.IPFIXOptionsTemplateFlowSet:
+			optionsTemplatesFlowSet = append(optionsTemplatesFlowSet, flowSet.(netflow.IPFIXOptionsTemplateFlowSet))
+		case netflow.DataFlowSet:
+			dataFlowSet = append(dataFlowSet, flowSet.(netflow.DataFlowSet))
+		case netflow.OptionsDataFlowSet:
+			optionsDataFlowSet = append(optionsDataFlowSet, flowSet.(netflow.OptionsDataFlowSet))
+		}
+	}
+	return dataFlowSet, templatesFlowSet, optionsTemplatesFlowSet, optionsDataFlowSet
+}
+
+// Convert a NetFlow datastructure to a FlowMessage protobuf
+// Does not put sampling rate
+func ProcessMessageNetFlow(msgDec interface{}, samplingRateSys SamplingRateSystem) ([]*flowmessage.FlowMessage, error) {
+	seqnum := uint32(0)
+	var baseTime uint32
+	var uptime uint32
+
+	flowMessageSet := make([]*flowmessage.FlowMessage, 0)
+
+	switch msgDecConv := msgDec.(type) {
+	case netflow.NFv9Packet:
+		dataFlowSet, _, _, optionDataFlowSet := SplitNetFlowSets(msgDecConv)
+
+		seqnum = msgDecConv.SequenceNumber
+		baseTime = msgDecConv.UnixSeconds
+		uptime = msgDecConv.SystemUptime
+		obsDomainId := msgDecConv.SourceId
+
+		flowMessageSet = SearchNetFlowDataSets(9, baseTime, uptime, dataFlowSet)
+		samplingRate, found := SearchNetFlowOptionDataSets(optionDataFlowSet)
+		if samplingRateSys != nil {
+			if found {
+				samplingRateSys.AddSamplingRate(9, obsDomainId, samplingRate)
+			} else {
+				samplingRate, _ = samplingRateSys.GetSamplingRate(9, obsDomainId)
+			}
+		}
+		for _, fmsg := range flowMessageSet {
+			fmsg.SequenceNum = seqnum
+			fmsg.SamplingRate = uint64(samplingRate)
+		}
+	case netflow.IPFIXPacket:
+		dataFlowSet, _, _, optionDataFlowSet := SplitIPFIXSets(msgDecConv)
+
+		seqnum = msgDecConv.SequenceNumber
+		baseTime = msgDecConv.ExportTime
+		obsDomainId := msgDecConv.ObservationDomainId
+
+		flowMessageSet = SearchNetFlowDataSets(10, baseTime, uptime, dataFlowSet)
+
+		samplingRate, found := SearchNetFlowOptionDataSets(optionDataFlowSet)
+		if samplingRateSys != nil {
+			if found {
+				samplingRateSys.AddSamplingRate(10, obsDomainId, samplingRate)
+			} else {
+				samplingRate, _ = samplingRateSys.GetSamplingRate(10, obsDomainId)
+			}
+		}
+		for _, fmsg := range flowMessageSet {
+			fmsg.SequenceNum = seqnum
+			fmsg.SamplingRate = uint64(samplingRate)
+		}
+	default:
+		return flowMessageSet, errors.New("Bad NetFlow/IPFIX version")
+	}
+
+	return flowMessageSet, nil
+}

producer/producer_nflegacy.go

@@ -0,0 +1,79 @@
+package producer
+
+import (
+	"encoding/binary"
+	"errors"
+	"net"
+
+	"github.com/netsampler/goflow2/decoders/netflowlegacy"
+	flowmessage "github.com/netsampler/goflow2/pb"
+)
+
+func ConvertNetFlowLegacyRecord(baseTime uint32, uptime uint32, record netflowlegacy.RecordsNetFlowV5) *flowmessage.FlowMessage {
+	flowMessage := &flowmessage.FlowMessage{}
+
+	flowMessage.Type = flowmessage.FlowMessage_NETFLOW_V5
+
+	timeDiffFirst := (uptime - record.First) / 1000
+	timeDiffLast := (uptime - record.Last) / 1000
+	flowMessage.TimeFlowStart = uint64(baseTime - timeDiffFirst)
+	flowMessage.TimeFlowEnd = uint64(baseTime - timeDiffLast)
+
+	v := make(net.IP, 4)
+	binary.BigEndian.PutUint32(v, record.NextHop)
+	flowMessage.NextHop = v
+	v = make(net.IP, 4)
+	binary.BigEndian.PutUint32(v, record.SrcAddr)
+	flowMessage.SrcAddr = v
+	v = make(net.IP, 4)
+	binary.BigEndian.PutUint32(v, record.DstAddr)
+	flowMessage.DstAddr = v
+
+	flowMessage.Etype = 0x800
+	flowMessage.SrcAS = uint32(record.SrcAS)
+	flowMessage.DstAS = uint32(record.DstAS)
+	flowMessage.SrcNet = uint32(record.SrcMask)
+	flowMessage.DstNet = uint32(record.DstMask)
+	flowMessage.Proto = uint32(record.Proto)
+	flowMessage.TCPFlags = uint32(record.TCPFlags)
+	flowMessage.IPTos = uint32(record.Tos)
+	flowMessage.InIf = uint32(record.Input)
+	flowMessage.OutIf = uint32(record.Output)
+	flowMessage.SrcPort = uint32(record.SrcPort)
+	flowMessage.DstPort = uint32(record.DstPort)
+	flowMessage.Packets = uint64(record.DPkts)
+	flowMessage.Bytes = uint64(record.DOctets)
+
+	return flowMessage
+}
+
+func SearchNetFlowLegacyRecords(baseTime uint32, uptime uint32, dataRecords []netflowlegacy.RecordsNetFlowV5) []*flowmessage.FlowMessage {
+	flowMessageSet := make([]*flowmessage.FlowMessage, 0)
+	for _, record := range dataRecords {
+		fmsg := ConvertNetFlowLegacyRecord(baseTime, uptime, record)
+		if fmsg != nil {
+			flowMessageSet = append(flowMessageSet, fmsg)
+		}
+	}
+	return flowMessageSet
+}
+
+func ProcessMessageNetFlowLegacy(msgDec interface{}) ([]*flowmessage.FlowMessage, error) {
+	switch packet := msgDec.(type) {
+	case netflowlegacy.PacketNetFlowV5:
+		seqnum := packet.FlowSequence
+		samplingRate := packet.SamplingInterval
+		baseTime := packet.UnixSecs
+		uptime := packet.SysUptime
+
+		flowMessageSet := SearchNetFlowLegacyRecords(baseTime, uptime, packet.Records)
+		for _, fmsg := range flowMessageSet {
+			fmsg.SequenceNum = seqnum
+			fmsg.SamplingRate = uint64(samplingRate)
+		}
+
+		return flowMessageSet, nil
+	default:
+		return []*flowmessage.FlowMessage{}, errors.New("Bad NetFlow v5 version")
+	}
+}

producer/producer_sf.go

@@ -0,0 +1,310 @@
+package producer
+
+import (
+	"encoding/binary"
+	"errors"
+	"net"
+
+	"github.com/netsampler/goflow2/decoders/sflow"
+	flowmessage "github.com/netsampler/goflow2/pb"
+)
+
+func GetSFlowFlowSamples(packet *sflow.Packet) []interface{} {
+	flowSamples := make([]interface{}, 0)
+	for _, sample := range packet.Samples {
+		switch sample.(type) {
+		case sflow.FlowSample:
+			flowSamples = append(flowSamples, sample)
+		case sflow.ExpandedFlowSample:
+			flowSamples = append(flowSamples, sample)
+		}
+	}
+	return flowSamples
+}
+
+type SFlowProducerConfig struct {
+}
+
+func ParseSampledHeader(flowMessage *flowmessage.FlowMessage, sampledHeader *sflow.SampledHeader) error {
+	return ParseSampledHeaderConfig(flowMessage, sampledHeader, nil)
+}
+
+func ParseSampledHeaderConfig(flowMessage *flowmessage.FlowMessage, sampledHeader *sflow.SampledHeader, config *SFlowProducerConfig) error {
+
+	data := (*sampledHeader).HeaderData
+	switch (*sampledHeader).Protocol {
+	case 1: // Ethernet
+		var hasMPLS bool
+		var countMpls uint32
+		var firstLabelMpls uint32
+		var firstTtlMpls uint8
+		var secondLabelMpls uint32
+		var secondTtlMpls uint8
+		var thirdLabelMpls uint32
+		var thirdTtlMpls uint8
+		var lastLabelMpls uint32
+		var lastTtlMpls uint8
+
+		var nextHeader byte
+		var tcpflags byte
+		srcIP := net.IP{}
+		dstIP := net.IP{}
+		offset := 14
+
+		var srcMac uint64
+		var dstMac uint64
+
+		var tos byte
+		var ttl byte
+		var identification uint16
+		var fragOffset uint16
+		var flowLabel uint32
+
+		var srcPort uint16
+		var dstPort uint16
+
+		etherType := data[12:14]
+
+		dstMac = binary.BigEndian.Uint64(append([]byte{0, 0}, data[0:6]...))
+		srcMac = binary.BigEndian.Uint64(append([]byte{0, 0}, data[6:12]...))
+		(*flowMessage).SrcMac = srcMac
+		(*flowMessage).DstMac = dstMac
+
+		encap := true
+		iterations := 0
+		for encap && iterations <= 1 {
+			encap = false
+
+			if etherType[0] == 0x81 && etherType[1] == 0x0 { // VLAN 802.1Q
+				(*flowMessage).VlanId = uint32(binary.BigEndian.Uint16(data[14:16]))
+				offset += 4
+				etherType = data[16:18]
+			}
+
+			if etherType[0] == 0x88 && etherType[1] == 0x47 { // MPLS
+				iterateMpls := true
+				hasMPLS = true
+				for iterateMpls {
+					if len(data) < offset+5 {
+						iterateMpls = false
+						break
+					}
+					label := binary.BigEndian.Uint32(append([]byte{0}, data[offset:offset+3]...)) >> 4
+					//exp := data[offset+2] > 1
+					bottom := data[offset+2] & 1
+					mplsTtl := data[offset+3]
+					offset += 4
+
+					if bottom == 1 || label <= 15 || offset > len(data) {
+						if data[offset]&0xf0>>4 == 4 {
+							etherType = []byte{0x8, 0x0}
+						} else if data[offset]&0xf0>>4 == 6 {
+							etherType = []byte{0x86, 0xdd}
+						}
+						iterateMpls = false
+					}
+
+					if countMpls == 0 {
+						firstLabelMpls = label
+						firstTtlMpls = mplsTtl
+					} else if countMpls == 1 {
+						secondLabelMpls = label
+						secondTtlMpls = mplsTtl
+					} else if countMpls == 2 {
+						thirdLabelMpls = label
+						thirdTtlMpls = mplsTtl
+					} else {
+						lastLabelMpls = label
+						lastTtlMpls = mplsTtl
+					}
+					countMpls++
+				}
+			}
+
+			if etherType[0] == 0x8 && etherType[1] == 0x0 { // IPv4
+				if len(data) >= offset+20 {
+					nextHeader = data[offset+9]
+					srcIP = data[offset+12 : offset+16]
+					dstIP = data[offset+16 : offset+20]
+					tos = data[offset+1]
+					ttl = data[offset+8]
+
+					identification = binary.BigEndian.Uint16(data[offset+4 : offset+6])
+					fragOffset = binary.BigEndian.Uint16(data[offset+6 : offset+8])
+
+					offset += 20
+				}
+			} else if etherType[0] == 0x86 && etherType[1] == 0xdd { // IPv6
+				if len(data) >= offset+40 {
+					nextHeader = data[offset+6]
+					srcIP = data[offset+8 : offset+24]
+					dstIP = data[offset+24 : offset+40]
+
+					tostmp := uint32(binary.BigEndian.Uint16(data[offset : offset+2]))
+					tos = uint8(tostmp & 0x0ff0 >> 4)
+					ttl = data[offset+7]
+
+					flowLabel = binary.BigEndian.Uint32(data[offset : offset+4])
+
+					offset += 40
+
+				}
+			} else if etherType[0] == 0x8 && etherType[1] == 0x6 { // ARP
+			} /*else {
+				return errors.New(fmt.Sprintf("Unknown EtherType: %v\n", etherType))
+			} */
+
+			if len(data) >= offset+4 && (nextHeader == 17 || nextHeader == 6) {
+				srcPort = binary.BigEndian.Uint16(data[offset+0 : offset+2])
+				dstPort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
+			}
+
+			if len(data) >= offset+13 && nextHeader == 6 {
+				tcpflags = data[offset+13]
+			}
+
+			// ICMP and ICMPv6
+			if len(data) >= offset+2 && (nextHeader == 1 || nextHeader == 58) {
+				(*flowMessage).IcmpType = uint32(data[offset+0])
+				(*flowMessage).IcmpCode = uint32(data[offset+1])
+			}
+
+			iterations++
+		}
+
+		(*flowMessage).HasMPLS = hasMPLS
+		(*flowMessage).MPLSCount = countMpls
+		(*flowMessage).MPLS1Label = firstLabelMpls
+		(*flowMessage).MPLS1TTL = uint32(firstTtlMpls)
+		(*flowMessage).MPLS2Label = secondLabelMpls
+		(*flowMessage).MPLS2TTL = uint32(secondTtlMpls)
+		(*flowMessage).MPLS3Label = thirdLabelMpls
+		(*flowMessage).MPLS3TTL = uint32(thirdTtlMpls)
+		(*flowMessage).MPLSLastLabel = lastLabelMpls
+		(*flowMessage).MPLSLastTTL = uint32(lastTtlMpls)
+
+		(*flowMessage).Etype = uint32(binary.BigEndian.Uint16(etherType[0:2]))
+		(*flowMessage).IPv6FlowLabel = flowLabel & 0xFFFFF
+
+		(*flowMessage).SrcPort = uint32(srcPort)
+		(*flowMessage).DstPort = uint32(dstPort)
+
+		(*flowMessage).SrcAddr = srcIP
+		(*flowMessage).DstAddr = dstIP
+		(*flowMessage).Proto = uint32(nextHeader)
+		(*flowMessage).IPTos = uint32(tos)
+		(*flowMessage).IPTTL = uint32(ttl)
+		(*flowMessage).TCPFlags = uint32(tcpflags)
+
+		(*flowMessage).FragmentId = uint32(identification)
+		(*flowMessage).FragmentOffset = uint32(fragOffset)
+	}
+	return nil
+}
+
+func SearchSFlowSamples(samples []interface{}) []*flowmessage.FlowMessage {
+	return SearchSFlowSamples(samples)
+}
+
+func SearchSFlowSamplesConfig(samples []interface{}, config *SFlowProducerConfig) []*flowmessage.FlowMessage {
+	flowMessageSet := make([]*flowmessage.FlowMessage, 0)
+
+	for _, flowSample := range samples {
+		var records []sflow.FlowRecord
+
+		flowMessage := &flowmessage.FlowMessage{}
+		flowMessage.Type = flowmessage.FlowMessage_SFLOW_5
+
+		switch flowSample := flowSample.(type) {
+		case sflow.FlowSample:
+			records = flowSample.Records
+			flowMessage.SamplingRate = uint64(flowSample.SamplingRate)
+			flowMessage.InIf = flowSample.Input
+			flowMessage.OutIf = flowSample.Output
+		case sflow.ExpandedFlowSample:
+			records = flowSample.Records
+			flowMessage.SamplingRate = uint64(flowSample.SamplingRate)
+			flowMessage.InIf = flowSample.InputIfValue
+			flowMessage.OutIf = flowSample.OutputIfValue
+		}
+
+		ipNh := net.IP{}
+		ipSrc := net.IP{}
+		ipDst := net.IP{}
+		flowMessage.Packets = 1
+		for _, record := range records {
+			switch recordData := record.Data.(type) {
+			case sflow.SampledHeader:
+				flowMessage.Bytes = uint64(recordData.FrameLength)
+				ParseSampledHeaderConfig(flowMessage, &recordData, config)
+			case sflow.SampledIPv4:
+				ipSrc = recordData.Base.SrcIP
+				ipDst = recordData.Base.DstIP
+				flowMessage.SrcAddr = ipSrc
+				flowMessage.DstAddr = ipDst
+				flowMessage.Bytes = uint64(recordData.Base.Length)
+				flowMessage.Proto = recordData.Base.Protocol
+				flowMessage.SrcPort = recordData.Base.SrcPort
+				flowMessage.DstPort = recordData.Base.DstPort
+				flowMessage.IPTos = recordData.Tos
+				flowMessage.Etype = 0x800
+			case sflow.SampledIPv6:
+				ipSrc = recordData.Base.SrcIP
+				ipDst = recordData.Base.DstIP
+				flowMessage.SrcAddr = ipSrc
+				flowMessage.DstAddr = ipDst
+				flowMessage.Bytes = uint64(recordData.Base.Length)
+				flowMessage.Proto = recordData.Base.Protocol
+				flowMessage.SrcPort = recordData.Base.SrcPort
+				flowMessage.DstPort = recordData.Base.DstPort
+				flowMessage.IPTos = recordData.Priority
+				flowMessage.Etype = 0x86dd
+			case sflow.ExtendedRouter:
+				ipNh = recordData.NextHop
+				flowMessage.NextHop = ipNh
+				flowMessage.SrcNet = recordData.SrcMaskLen
+				flowMessage.DstNet = recordData.DstMaskLen
+			case sflow.ExtendedGateway:
+				ipNh = recordData.NextHop
+				flowMessage.NextHop = ipNh
+				flowMessage.SrcAS = recordData.SrcAS
+				if len(recordData.ASPath) > 0 {
+					flowMessage.DstAS = recordData.ASPath[len(recordData.ASPath)-1]
+					flowMessage.NextHopAS = recordData.ASPath[0]
+					flowMessage.SrcAS = recordData.AS
+				} else {
+					flowMessage.DstAS = recordData.AS
+				}
+			case sflow.ExtendedSwitch:
+				flowMessage.SrcVlan = recordData.SrcVlan
+				flowMessage.DstVlan = recordData.DstVlan
+			}
+		}
+		flowMessageSet = append(flowMessageSet, flowMessage)
+	}
+	return flowMessageSet
+}
+
+func ProcessMessageSFlow(msgDec interface{}) ([]*flowmessage.FlowMessage, error) {
+	return ProcessMessageSFlowConfig(msgDec, nil)
+}
+
+func ProcessMessageSFlowConfig(msgDec interface{}, config *SFlowProducerConfig) ([]*flowmessage.FlowMessage, error) {
+	switch packet := msgDec.(type) {
+	case sflow.Packet:
+		seqnum := packet.SequenceNumber
+		var agent net.IP
+		agent = packet.AgentIP
+
+		flowSamples := GetSFlowFlowSamples(&packet)
+		flowMessageSet := SearchSFlowSamplesConfig(flowSamples, config)
+		for _, fmsg := range flowMessageSet {
+			fmsg.SamplerAddress = agent
+			fmsg.SequenceNum = seqnum
+		}
+
+		return flowMessageSet, nil
+	default:
+		return []*flowmessage.FlowMessage{}, errors.New("Bad sFlow version")
+	}
+}

producer/producer_test.go

@@ -0,0 +1,78 @@
+package producer
+
+import (
+	"testing"
+
+	"github.com/netsampler/goflow2/decoders/netflow"
+	"github.com/netsampler/goflow2/decoders/sflow"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestProcessMessageNetFlow(t *testing.T) {
+	records := []netflow.DataRecord{
+		netflow.DataRecord{
+			Values: []netflow.DataField{
+				netflow.DataField{
+					Type:  netflow.NFV9_FIELD_IPV4_SRC_ADDR,
+					Value: []byte{10, 0, 0, 1},
+				},
+			},
+		},
+	}
+	dfs := []interface{}{
+		netflow.DataFlowSet{
+			Records: records,
+		},
+	}
+
+	pktnf9 := netflow.NFv9Packet{
+		FlowSets: dfs,
+	}
+	testsr := &SingleSamplingRateSystem{1}
+	_, err := ProcessMessageNetFlow(pktnf9, testsr)
+	assert.Nil(t, err)
+
+	pktipfix := netflow.IPFIXPacket{
+		FlowSets: dfs,
+	}
+	_, err = ProcessMessageNetFlow(pktipfix, testsr)
+	assert.Nil(t, err)
+}
+
+func TestProcessMessageSFlow(t *testing.T) {
+	sh := sflow.SampledHeader{
+		FrameLength: 10,
+		Protocol:    1,
+		HeaderData: []byte{
+			0xff, 0xab, 0xcd, 0xef, 0xab, 0xcd, 0xff, 0xab, 0xcd, 0xef, 0xab, 0xbc, 0x86, 0xdd, 0x60, 0x2e,
+			0xc4, 0xec, 0x01, 0xcc, 0x06, 0x40, 0xfd, 0x01, 0x00, 0x00, 0xff, 0x01, 0x82, 0x10, 0xcd, 0xff,
+			0xff, 0x1c, 0x00, 0x00, 0x01, 0x50, 0xfd, 0x01, 0x00, 0x00, 0xff, 0x01, 0x00, 0x01, 0x02, 0xff,
+			0xff, 0x93, 0x00, 0x00, 0x02, 0x46, 0xcf, 0xca, 0x00, 0x50, 0x05, 0x15, 0x21, 0x6f, 0xa4, 0x9c,
+			0xf4, 0x59, 0x80, 0x18, 0x08, 0x09, 0x8c, 0x86, 0x00, 0x00, 0x01, 0x01, 0x08, 0x0a, 0x2a, 0x85,
+			0xee, 0x9e, 0x64, 0x5c, 0x27, 0x28,
+		},
+	}
+	pkt := sflow.Packet{
+		Version: 5,
+		Samples: []interface{}{
+			sflow.FlowSample{
+				SamplingRate: 1,
+				Records: []sflow.FlowRecord{
+					sflow.FlowRecord{
+						Data: sh,
+					},
+				},
+			},
+			sflow.ExpandedFlowSample{
+				SamplingRate: 1,
+				Records: []sflow.FlowRecord{
+					sflow.FlowRecord{
+						Data: sh,
+					},
+				},
+			},
+		},
+	}
+	_, err := ProcessMessageSFlow(pkt)
+	assert.Nil(t, err)
+}

transport/file/transport.go

@@ -0,0 +1,53 @@
+package file
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"github.com/netsampler/goflow2/transport"
+	"io"
+	"os"
+)
+
+type FileDriver struct {
+	fileDestination string
+	w               io.Writer
+	file            *os.File
+}
+
+func (d *FileDriver) Prepare() error {
+	flag.StringVar(&d.fileDestination, "transport.file", "", "File/console output (empty for stdout)")
+	// idea: add terminal coloring based on key partitioning (if any)
+	return nil
+}
+
+func (d *FileDriver) Init(context.Context) error {
+	if d.fileDestination == "" {
+		d.w = os.Stdout
+	} else {
+		var err error
+		d.file, err = os.OpenFile(d.fileDestination, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+		if err != nil {
+			return err
+		}
+		d.w = d.file
+	}
+	return nil
+}
+
+func (d *FileDriver) Send(key, data []byte) error {
+	fmt.Fprintln(d.w, string(data))
+	return nil
+}
+
+func (d *FileDriver) Close(context.Context) error {
+	if d.fileDestination != "" {
+		d.file.Close()
+	}
+	return nil
+}
+
+func init() {
+	d := &FileDriver{}
+	transport.RegisterTransportDriver("file", d)
+}

transport/kafka/kafka.go

@@ -0,0 +1,150 @@
+package kafka
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+
+	sarama "github.com/Shopify/sarama"
+	"github.com/netsampler/goflow2/transport"
+	"github.com/netsampler/goflow2/utils"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type KafkaState struct {
+	FixedLengthProto bool
+	producer         sarama.AsyncProducer
+	topic            string
+	hashing          bool
+	keying           []string
+}
+
+type KafkaDriver struct {
+	kafkaTLS   bool
+	kafkaSASL  bool
+	kafkaTopic string
+	kafkaSrv   string
+	kafkaBrk   string
+
+	kafkaLogErrors bool
+
+	kafkaHashing bool
+	kafkaVersion string
+
+	producer sarama.AsyncProducer
+
+	q chan bool
+}
+
+func (d *KafkaDriver) Prepare() error {
+	flag.BoolVar(&d.kafkaTLS, "transport.kafka.tls", false, "Use TLS to connect to Kafka")
+
+	flag.BoolVar(&d.kafkaSASL, "transport.kafka.sasl", false, "Use SASL/PLAIN data to connect to Kafka (TLS is recommended and the environment variables KAFKA_SASL_USER and KAFKA_SASL_PASS need to be set)")
+	flag.StringVar(&d.kafkaTopic, "transport.kafka.topic", "flow-messages", "Kafka topic to produce to")
+	flag.StringVar(&d.kafkaSrv, "transport.kafka.srv", "", "SRV record containing a list of Kafka brokers (or use brokers)")
+	flag.StringVar(&d.kafkaBrk, "transport.kafka.brokers", "127.0.0.1:9092,[::1]:9092", "Kafka brokers list separated by commas")
+
+	flag.BoolVar(&d.kafkaLogErrors, "transport.kafka.log.err", false, "Log Kafka errors")
+	flag.BoolVar(&d.kafkaHashing, "transport.kafka.hashing", false, "Enable partition hashing")
+
+	//flag.StringVar(&d.kafkaKeying, "transport.kafka.key", "SamplerAddress,DstAS", "Kafka list of fields to do hashing on (partition) separated by commas")
+	flag.StringVar(&d.kafkaVersion, "transport.kafka.version", "2.8.0", "Kafka version")
+
+	return nil
+}
+
+func (d *KafkaDriver) Init(context.Context) error {
+	kafkaConfigVersion, err := sarama.ParseKafkaVersion(d.kafkaVersion)
+	if err != nil {
+		return err
+	}
+
+	kafkaConfig := sarama.NewConfig()
+	kafkaConfig.Version = kafkaConfigVersion
+	kafkaConfig.Producer.Return.Successes = false
+	kafkaConfig.Producer.Return.Errors = d.kafkaLogErrors
+	if d.kafkaTLS {
+		rootCAs, err := x509.SystemCertPool()
+		if err != nil {
+			return errors.New(fmt.Sprintf("Error initializing TLS: %v", err))
+		}
+		kafkaConfig.Net.TLS.Enable = true
+		kafkaConfig.Net.TLS.Config = &tls.Config{RootCAs: rootCAs}
+	}
+
+	if d.kafkaHashing {
+		kafkaConfig.Producer.Partitioner = sarama.NewHashPartitioner
+	}
+
+	if d.kafkaSASL {
+		if !d.kafkaTLS /*&& log != nil*/ {
+			log.Warn("Using SASL without TLS will transmit the authentication in plaintext!")
+		}
+		kafkaConfig.Net.SASL.Enable = true
+		kafkaConfig.Net.SASL.User = os.Getenv("KAFKA_SASL_USER")
+		kafkaConfig.Net.SASL.Password = os.Getenv("KAFKA_SASL_PASS")
+		if kafkaConfig.Net.SASL.User == "" && kafkaConfig.Net.SASL.Password == "" {
+			return errors.New("Kafka SASL config from environment was unsuccessful. KAFKA_SASL_USER and KAFKA_SASL_PASS need to be set.")
+		} else /*if log != nil*/ {
+			log.Infof("Authenticating as user '%s'...", kafkaConfig.Net.SASL.User)
+		}
+	}
+
+	addrs := make([]string, 0)
+	if d.kafkaSrv != "" {
+		addrs, _ = utils.GetServiceAddresses(d.kafkaSrv)
+	} else {
+		addrs = strings.Split(d.kafkaBrk, ",")
+	}
+
+	kafkaProducer, err := sarama.NewAsyncProducer(addrs, kafkaConfig)
+	if err != nil {
+		return err
+	}
+	d.producer = kafkaProducer
+
+	d.q = make(chan bool)
+
+	if d.kafkaLogErrors {
+		go func() {
+			for {
+				select {
+				case msg := <-kafkaProducer.Errors():
+					//if log != nil {
+					log.Error(msg)
+					//}
+				case <-d.q:
+					return
+				}
+			}
+		}()
+	}
+
+	return err
+}
+
+func (d *KafkaDriver) Send(key, data []byte) error {
+	d.producer.Input() <- &sarama.ProducerMessage{
+		Topic: d.kafkaTopic,
+		Key:   sarama.ByteEncoder(key),
+		Value: sarama.ByteEncoder(data),
+	}
+	return nil
+}
+
+func (d *KafkaDriver) Close(context.Context) error {
+	d.producer.Close()
+	close(d.q)
+	return nil
+}
+
+func init() {
+	d := &KafkaDriver{}
+	transport.RegisterTransportDriver("kafka", d)
+}

transport/transport.go

@@ -0,0 +1,68 @@
+package transport
+
+import (
+	"context"
+	"fmt"
+	"sync"
+)
+
+var (
+	transportDrivers = make(map[string]TransportDriver)
+	lock             = &sync.RWMutex{}
+)
+
+type TransportDriver interface {
+	Prepare() error              // Prepare driver (eg: flag registration)
+	Init(context.Context) error  // Initialize driver (eg: start connections, open files...)
+	Close(context.Context) error // Close driver (eg: close connections and files...)
+	Send(key, data []byte) error // Send a formatted message
+}
+
+type TransportInterface interface {
+	Send(key, data []byte) error
+}
+
+type Transport struct {
+	driver TransportDriver
+}
+
+func (t *Transport) Close(ctx context.Context) {
+	t.driver.Close(ctx)
+}
+func (t *Transport) Send(key, data []byte) error {
+	return t.driver.Send(key, data)
+}
+
+func RegisterTransportDriver(name string, t TransportDriver) {
+	lock.Lock()
+	transportDrivers[name] = t
+	lock.Unlock()
+
+	if err := t.Prepare(); err != nil {
+		panic(err)
+	}
+}
+
+func FindTransport(ctx context.Context, name string) (*Transport, error) {
+	lock.RLock()
+	t, ok := transportDrivers[name]
+	lock.RUnlock()
+	if !ok {
+		return nil, fmt.Errorf("Transport %s not found", name)
+	}
+
+	err := t.Init(ctx)
+	return &Transport{t}, err
+}
+
+func GetTransports() []string {
+	lock.RLock()
+	t := make([]string, len(transportDrivers))
+	var i int
+	for k, _ := range transportDrivers {
+		t[i] = k
+		i++
+	}
+	lock.RUnlock()
+	return t
+}

utils/metrics.go

@@ -0,0 +1,171 @@
+package utils
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	MetricTrafficBytes = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "flow_traffic_bytes",
+			Help: "Bytes received by the application.",
+		},
+		[]string{"remote_ip", "local_ip", "local_port", "type"},
+	)
+	MetricTrafficPackets = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "flow_traffic_packets",
+			Help: "Packets received by the application.",
+		},
+		[]string{"remote_ip", "local_ip", "local_port", "type"},
+	)
+	MetricPacketSizeSum = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name:       "flow_traffic_summary_size_bytes",
+			Help:       "Summary of packet size.",
+			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		},
+		[]string{"remote_ip", "local_ip", "local_port", "type"},
+	)
+	DecoderStats = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "flow_decoder_count",
+			Help: "Decoder processed count.",
+		},
+		[]string{"worker", "name"},
+	)
+	DecoderErrors = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "flow_decoder_error_count",
+			Help: "Decoder processed error count.",
+		},
+		[]string{"worker", "name"},
+	)
+	DecoderTime = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name:       "flow_summary_decoding_time_us",
+			Help:       "Decoding time summary.",
+			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		},
+		[]string{"name"},
+	)
+	DecoderProcessTime = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name:       "flow_summary_processing_time_us",
+			Help:       "Processing time summary.",
+			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		},
+		[]string{"name"},
+	)
+	NetFlowStats = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "flow_process_nf_count",
+			Help: "NetFlows processed.",
+		},
+		[]string{"router", "version"},
+	)
+	NetFlowErrors = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "flow_process_nf_errors_count",
+			Help: "NetFlows processed errors.",
+		},
+		[]string{"router", "error"},
+	)
+	NetFlowSetRecordsStatsSum = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "flow_process_nf_flowset_records_sum",
+			Help: "NetFlows FlowSets sum of records.",
+		},
+		[]string{"router", "version", "type"}, // data-template, data, opts...
+	)
+	NetFlowSetStatsSum = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "flow_process_nf_flowset_sum",
+			Help: "NetFlows FlowSets sum.",
+		},
+		[]string{"router", "version", "type"}, // data-template, data, opts...
+	)
+	NetFlowTimeStatsSum = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name:       "flow_process_nf_delay_summary_seconds",
+			Help:       "NetFlows time difference between time of flow and processing.",
+			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		},
+		[]string{"router", "version"},
+	)
+	NetFlowTemplatesStats = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "flow_process_nf_templates_count",
+			Help: "NetFlows Template count.",
+		},
+		[]string{"router", "version", "obs_domain_id", "template_id", "type"}, // options/template
+	)
+	SFlowStats = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "flow_process_sf_count",
+			Help: "sFlows processed.",
+		},
+		[]string{"router", "agent", "version"},
+	)
+	SFlowErrors = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "flow_process_sf_errors_count",
+			Help: "sFlows processed errors.",
+		},
+		[]string{"router", "error"},
+	)
+	SFlowSampleStatsSum = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "flow_process_sf_samples_sum",
+			Help: "SFlows samples sum.",
+		},
+		[]string{"router", "agent", "version", "type"}, // counter, flow, expanded...
+	)
+	SFlowSampleRecordsStatsSum = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "flow_process_sf_samples_records_sum",
+			Help: "SFlows samples sum of records.",
+		},
+		[]string{"router", "agent", "version", "type"}, // data-template, data, opts...
+	)
+)
+
+func init() {
+	prometheus.MustRegister(MetricTrafficBytes)
+	prometheus.MustRegister(MetricTrafficPackets)
+	prometheus.MustRegister(MetricPacketSizeSum)
+
+	prometheus.MustRegister(DecoderStats)
+	prometheus.MustRegister(DecoderErrors)
+	prometheus.MustRegister(DecoderTime)
+	prometheus.MustRegister(DecoderProcessTime)
+
+	prometheus.MustRegister(NetFlowStats)
+	prometheus.MustRegister(NetFlowErrors)
+	prometheus.MustRegister(NetFlowSetRecordsStatsSum)
+	prometheus.MustRegister(NetFlowSetStatsSum)
+	prometheus.MustRegister(NetFlowTimeStatsSum)
+	prometheus.MustRegister(NetFlowTemplatesStats)
+
+	prometheus.MustRegister(SFlowStats)
+	prometheus.MustRegister(SFlowErrors)
+	prometheus.MustRegister(SFlowSampleStatsSum)
+	prometheus.MustRegister(SFlowSampleRecordsStatsSum)
+}
+
+func DefaultAccountCallback(name string, id int, start, end time.Time) {
+	DecoderProcessTime.With(
+		prometheus.Labels{
+			"name": name,
+		}).
+		Observe(float64((end.Sub(start)).Nanoseconds()) / 1000)
+	DecoderStats.With(
+		prometheus.Labels{
+			"worker": strconv.Itoa(id),
+			"name":   name,
+		}).
+		Inc()
+}

utils/netflow.go

@@ -0,0 +1,371 @@
+package utils
+
+import (
+	"bytes"
+	"encoding/json"
+	"net/http"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/netsampler/goflow2/decoders/netflow"
+	"github.com/netsampler/goflow2/format"
+	flowmessage "github.com/netsampler/goflow2/pb"
+	"github.com/netsampler/goflow2/producer"
+	"github.com/netsampler/goflow2/transport"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type TemplateSystem struct {
+	key       string
+	templates *netflow.BasicTemplateSystem
+}
+
+func (s *TemplateSystem) AddTemplate(version uint16, obsDomainId uint32, template interface{}) {
+	s.templates.AddTemplate(version, obsDomainId, template)
+
+	typeStr := "options_template"
+	var templateId uint16
+	switch templateIdConv := template.(type) {
+	case netflow.IPFIXOptionsTemplateRecord:
+		templateId = templateIdConv.TemplateId
+	case netflow.NFv9OptionsTemplateRecord:
+		templateId = templateIdConv.TemplateId
+	case netflow.TemplateRecord:
+		templateId = templateIdConv.TemplateId
+		typeStr = "template"
+	}
+	NetFlowTemplatesStats.With(
+		prometheus.Labels{
+			"router":        s.key,
+			"version":       strconv.Itoa(int(version)),
+			"obs_domain_id": strconv.Itoa(int(obsDomainId)),
+			"template_id":   strconv.Itoa(int(templateId)),
+			"type":          typeStr,
+		}).
+		Inc()
+}
+
+func (s *TemplateSystem) GetTemplate(version uint16, obsDomainId uint32, templateId uint16) (interface{}, error) {
+	return s.templates.GetTemplate(version, obsDomainId, templateId)
+}
+
+type StateNetFlow struct {
+	Format        format.FormatInterface
+	Transport     transport.TransportInterface
+	Logger        Logger
+	templateslock *sync.RWMutex
+	templates     map[string]*TemplateSystem
+
+	samplinglock *sync.RWMutex
+	sampling     map[string]producer.SamplingRateSystem
+}
+
+func (s *StateNetFlow) DecodeFlow(msg interface{}) error {
+	pkt := msg.(BaseMessage)
+	buf := bytes.NewBuffer(pkt.Payload)
+
+	key := pkt.Src.String()
+	samplerAddress := pkt.Src
+	if samplerAddress.To4() != nil {
+		samplerAddress = samplerAddress.To4()
+	}
+
+	s.templateslock.RLock()
+	templates, ok := s.templates[key]
+	s.templateslock.RUnlock()
+	if !ok {
+		templates = &TemplateSystem{
+			templates: netflow.CreateTemplateSystem(),
+			key:       key,
+		}
+		s.templateslock.Lock()
+		s.templates[key] = templates
+		s.templateslock.Unlock()
+	}
+	s.samplinglock.RLock()
+	sampling, ok := s.sampling[key]
+	s.samplinglock.RUnlock()
+	if !ok {
+		sampling = producer.CreateSamplingSystem()
+		s.samplinglock.Lock()
+		s.sampling[key] = sampling
+		s.samplinglock.Unlock()
+	}
+
+	ts := uint64(time.Now().UTC().Unix())
+	if pkt.SetTime {
+		ts = uint64(pkt.RecvTime.UTC().Unix())
+	}
+
+	timeTrackStart := time.Now()
+	msgDec, err := netflow.DecodeMessage(buf, templates)
+	if err != nil {
+		switch err.(type) {
+		case *netflow.ErrorVersion:
+			NetFlowErrors.With(
+				prometheus.Labels{
+					"router": key,
+					"error":  "error_version",
+				}).
+				Inc()
+		case *netflow.ErrorFlowId:
+			NetFlowErrors.With(
+				prometheus.Labels{
+					"router": key,
+					"error":  "error_flow_id",
+				}).
+				Inc()
+		case *netflow.ErrorTemplateNotFound:
+			NetFlowErrors.With(
+				prometheus.Labels{
+					"router": key,
+					"error":  "template_not_found",
+				}).
+				Inc()
+		default:
+			NetFlowErrors.With(
+				prometheus.Labels{
+					"router": key,
+					"error":  "error_decoding",
+				}).
+				Inc()
+		}
+		return err
+	}
+
+	flowMessageSet := make([]*flowmessage.FlowMessage, 0)
+
+	switch msgDecConv := msgDec.(type) {
+	case netflow.NFv9Packet:
+		NetFlowStats.With(
+			prometheus.Labels{
+				"router":  key,
+				"version": "9",
+			}).
+			Inc()
+
+		for _, fs := range msgDecConv.FlowSets {
+			switch fsConv := fs.(type) {
+			case netflow.TemplateFlowSet:
+				NetFlowSetStatsSum.With(
+					prometheus.Labels{
+						"router":  key,
+						"version": "9",
+						"type":    "TemplateFlowSet",
+					}).
+					Inc()
+
+				NetFlowSetRecordsStatsSum.With(
+					prometheus.Labels{
+						"router":  key,
+						"version": "9",
+						"type":    "OptionsTemplateFlowSet",
+					}).
+					Add(float64(len(fsConv.Records)))
+
+			case netflow.NFv9OptionsTemplateFlowSet:
+				NetFlowSetStatsSum.With(
+					prometheus.Labels{
+						"router":  key,
+						"version": "9",
+						"type":    "OptionsTemplateFlowSet",
+					}).
+					Inc()
+
+				NetFlowSetRecordsStatsSum.With(
+					prometheus.Labels{
+						"router":  key,
+						"version": "9",
+						"type":    "OptionsTemplateFlowSet",
+					}).
+					Add(float64(len(fsConv.Records)))
+
+			case netflow.OptionsDataFlowSet:
+				NetFlowSetStatsSum.With(
+					prometheus.Labels{
+						"router":  key,
+						"version": "9",
+						"type":    "OptionsDataFlowSet",
+					}).
+					Inc()
+
+				NetFlowSetRecordsStatsSum.With(
+					prometheus.Labels{
+						"router":  key,
+						"version": "9",
+						"type":    "OptionsDataFlowSet",
+					}).
+					Add(float64(len(fsConv.Records)))
+			case netflow.DataFlowSet:
+				NetFlowSetStatsSum.With(
+					prometheus.Labels{
+						"router":  key,
+						"version": "9",
+						"type":    "DataFlowSet",
+					}).
+					Inc()
+
+				NetFlowSetRecordsStatsSum.With(
+					prometheus.Labels{
+						"router":  key,
+						"version": "9",
+						"type":    "DataFlowSet",
+					}).
+					Add(float64(len(fsConv.Records)))
+			}
+		}
+		flowMessageSet, err = producer.ProcessMessageNetFlow(msgDecConv, sampling)
+
+		for _, fmsg := range flowMessageSet {
+			fmsg.TimeReceived = ts
+			fmsg.SamplerAddress = samplerAddress
+			timeDiff := fmsg.TimeReceived - fmsg.TimeFlowEnd
+			NetFlowTimeStatsSum.With(
+				prometheus.Labels{
+					"router":  key,
+					"version": "9",
+				}).
+				Observe(float64(timeDiff))
+		}
+	case netflow.IPFIXPacket:
+		NetFlowStats.With(
+			prometheus.Labels{
+				"router":  key,
+				"version": "10",
+			}).
+			Inc()
+
+		for _, fs := range msgDecConv.FlowSets {
+			switch fsConv := fs.(type) {
+			case netflow.TemplateFlowSet:
+				NetFlowSetStatsSum.With(
+					prometheus.Labels{
+						"router":  key,
+						"version": "10",
+						"type":    "TemplateFlowSet",
+					}).
+					Inc()
+
+				NetFlowSetRecordsStatsSum.With(
+					prometheus.Labels{
+						"router":  key,
+						"version": "10",
+						"type":    "TemplateFlowSet",
+					}).
+					Add(float64(len(fsConv.Records)))
+
+			case netflow.IPFIXOptionsTemplateFlowSet:
+				NetFlowSetStatsSum.With(
+					prometheus.Labels{
+						"router":  key,
+						"version": "10",
+						"type":    "OptionsTemplateFlowSet",
+					}).
+					Inc()
+
+				NetFlowSetRecordsStatsSum.With(
+					prometheus.Labels{
+						"router":  key,
+						"version": "10",
+						"type":    "OptionsTemplateFlowSet",
+					}).
+					Add(float64(len(fsConv.Records)))
+
+			case netflow.OptionsDataFlowSet:
+
+				NetFlowSetStatsSum.With(
+					prometheus.Labels{
+						"router":  key,
+						"version": "10",
+						"type":    "OptionsDataFlowSet",
+					}).
+					Inc()
+
+				NetFlowSetRecordsStatsSum.With(
+					prometheus.Labels{
+						"router":  key,
+						"version": "10",
+						"type":    "OptionsDataFlowSet",
+					}).
+					Add(float64(len(fsConv.Records)))
+
+			case netflow.DataFlowSet:
+				NetFlowSetStatsSum.With(
+					prometheus.Labels{
+						"router":  key,
+						"version": "10",
+						"type":    "DataFlowSet",
+					}).
+					Inc()
+
+				NetFlowSetRecordsStatsSum.With(
+					prometheus.Labels{
+						"router":  key,
+						"version": "10",
+						"type":    "DataFlowSet",
+					}).
+					Add(float64(len(fsConv.Records)))
+			}
+		}
+		flowMessageSet, err = producer.ProcessMessageNetFlow(msgDecConv, sampling)
+
+		for _, fmsg := range flowMessageSet {
+			fmsg.TimeReceived = ts
+			fmsg.SamplerAddress = samplerAddress
+			timeDiff := fmsg.TimeReceived - fmsg.TimeFlowEnd
+			NetFlowTimeStatsSum.With(
+				prometheus.Labels{
+					"router":  key,
+					"version": "10",
+				}).
+				Observe(float64(timeDiff))
+		}
+	}
+
+	timeTrackStop := time.Now()
+	DecoderTime.With(
+		prometheus.Labels{
+			"name": "NetFlow",
+		}).
+		Observe(float64((timeTrackStop.Sub(timeTrackStart)).Nanoseconds()) / 1000)
+
+	for _, fmsg := range flowMessageSet {
+		if s.Format != nil {
+			key, data, err := s.Format.Format(fmsg)
+
+			if err != nil && s.Logger != nil {
+				s.Logger.Error(err)
+			}
+			if err == nil && s.Transport != nil {
+				s.Transport.Send(key, data)
+			}
+		}
+	}
+
+	return nil
+}
+
+func (s *StateNetFlow) ServeHTTPTemplates(w http.ResponseWriter, r *http.Request) {
+	tmp := make(map[string]map[uint16]map[uint32]map[uint16]interface{})
+	s.templateslock.RLock()
+	for key, templatesrouterstr := range s.templates {
+		templatesrouter := templatesrouterstr.templates.GetTemplates()
+		tmp[key] = templatesrouter
+	}
+	s.templateslock.RUnlock()
+	enc := json.NewEncoder(w)
+	enc.Encode(tmp)
+}
+
+func (s *StateNetFlow) InitTemplates() {
+	s.templates = make(map[string]*TemplateSystem)
+	s.templateslock = &sync.RWMutex{}
+	s.sampling = make(map[string]producer.SamplingRateSystem)
+	s.samplinglock = &sync.RWMutex{}
+}
+
+func (s *StateNetFlow) FlowRoutine(workers int, addr string, port int, reuseport bool) error {
+	s.InitTemplates()
+	return UDPRoutine("NetFlow", s.DecodeFlow, workers, addr, port, reuseport, s.Logger)
+}

utils/nflegacy.go

@@ -0,0 +1,99 @@
+package utils
+
+import (
+	"bytes"
+	"time"
+
+	"github.com/netsampler/goflow2/decoders/netflowlegacy"
+	"github.com/netsampler/goflow2/format"
+	flowmessage "github.com/netsampler/goflow2/pb"
+	"github.com/netsampler/goflow2/producer"
+	"github.com/netsampler/goflow2/transport"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type StateNFLegacy struct {
+	Format    format.FormatInterface
+	Transport transport.TransportInterface
+	Logger    Logger
+}
+
+func (s *StateNFLegacy) DecodeFlow(msg interface{}) error {
+	pkt := msg.(BaseMessage)
+	buf := bytes.NewBuffer(pkt.Payload)
+	key := pkt.Src.String()
+	samplerAddress := pkt.Src
+	if samplerAddress.To4() != nil {
+		samplerAddress = samplerAddress.To4()
+	}
+
+	ts := uint64(time.Now().UTC().Unix())
+	if pkt.SetTime {
+		ts = uint64(pkt.RecvTime.UTC().Unix())
+	}
+
+	timeTrackStart := time.Now()
+	msgDec, err := netflowlegacy.DecodeMessage(buf)
+
+	if err != nil {
+		switch err.(type) {
+		case *netflowlegacy.ErrorVersion:
+			NetFlowErrors.With(
+				prometheus.Labels{
+					"router": key,
+					"error":  "error_version",
+				}).
+				Inc()
+		}
+		return err
+	}
+
+	switch msgDecConv := msgDec.(type) {
+	case netflowlegacy.PacketNetFlowV5:
+		NetFlowStats.With(
+			prometheus.Labels{
+				"router":  key,
+				"version": "5",
+			}).
+			Inc()
+		NetFlowSetStatsSum.With(
+			prometheus.Labels{
+				"router":  key,
+				"version": "5",
+				"type":    "DataFlowSet",
+			}).
+			Add(float64(msgDecConv.Count))
+	}
+
+	var flowMessageSet []*flowmessage.FlowMessage
+	flowMessageSet, err = producer.ProcessMessageNetFlowLegacy(msgDec)
+
+	timeTrackStop := time.Now()
+	DecoderTime.With(
+		prometheus.Labels{
+			"name": "NetFlowV5",
+		}).
+		Observe(float64((timeTrackStop.Sub(timeTrackStart)).Nanoseconds()) / 1000)
+
+	for _, fmsg := range flowMessageSet {
+		fmsg.TimeReceived = ts
+		fmsg.SamplerAddress = samplerAddress
+
+		if s.Format != nil {
+			key, data, err := s.Format.Format(fmsg)
+
+			if err != nil && s.Logger != nil {
+				s.Logger.Error(err)
+			}
+			if err == nil && s.Transport != nil {
+				s.Transport.Send(key, data)
+			}
+		}
+	}
+
+	return nil
+}
+
+func (s *StateNFLegacy) FlowRoutine(workers int, addr string, port int, reuseport bool) error {
+	return UDPRoutine("NetFlowV5", s.DecodeFlow, workers, addr, port, reuseport, s.Logger)
+}

utils/sflow.go

@@ -0,0 +1,152 @@
+package utils
+
+import (
+	"bytes"
+	"net"
+	"time"
+
+	"github.com/netsampler/goflow2/decoders/sflow"
+	"github.com/netsampler/goflow2/format"
+	flowmessage "github.com/netsampler/goflow2/pb"
+	"github.com/netsampler/goflow2/producer"
+	"github.com/netsampler/goflow2/transport"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type StateSFlow struct {
+	Format    format.FormatInterface
+	Transport transport.TransportInterface
+	Logger    Logger
+
+	Config *producer.SFlowProducerConfig
+}
+
+func (s *StateSFlow) DecodeFlow(msg interface{}) error {
+	pkt := msg.(BaseMessage)
+	buf := bytes.NewBuffer(pkt.Payload)
+	key := pkt.Src.String()
+
+	ts := uint64(time.Now().UTC().Unix())
+	if pkt.SetTime {
+		ts = uint64(pkt.RecvTime.UTC().Unix())
+	}
+
+	timeTrackStart := time.Now()
+	msgDec, err := sflow.DecodeMessage(buf)
+
+	if err != nil {
+		switch err.(type) {
+		case *sflow.ErrorVersion:
+			SFlowErrors.With(
+				prometheus.Labels{
+					"router": key,
+					"error":  "error_version",
+				}).
+				Inc()
+		case *sflow.ErrorIPVersion:
+			SFlowErrors.With(
+				prometheus.Labels{
+					"router": key,
+					"error":  "error_ip_version",
+				}).
+				Inc()
+		case *sflow.ErrorDataFormat:
+			SFlowErrors.With(
+				prometheus.Labels{
+					"router": key,
+					"error":  "error_data_format",
+				}).
+				Inc()
+		default:
+			SFlowErrors.With(
+				prometheus.Labels{
+					"router": key,
+					"error":  "error_decoding",
+				}).
+				Inc()
+		}
+		return err
+	}
+
+	switch msgDecConv := msgDec.(type) {
+	case sflow.Packet:
+		agentStr := net.IP(msgDecConv.AgentIP).String()
+		SFlowStats.With(
+			prometheus.Labels{
+				"router":  key,
+				"agent":   agentStr,
+				"version": "5",
+			}).
+			Inc()
+
+		for _, samples := range msgDecConv.Samples {
+			typeStr := "unknown"
+			countRec := 0
+			switch samplesConv := samples.(type) {
+			case sflow.FlowSample:
+				typeStr = "FlowSample"
+				countRec = len(samplesConv.Records)
+			case sflow.CounterSample:
+				typeStr = "CounterSample"
+				if samplesConv.Header.Format == 4 {
+					typeStr = "Expanded" + typeStr
+				}
+				countRec = len(samplesConv.Records)
+			case sflow.ExpandedFlowSample:
+				typeStr = "ExpandedFlowSample"
+				countRec = len(samplesConv.Records)
+			}
+			SFlowSampleStatsSum.With(
+				prometheus.Labels{
+					"router":  key,
+					"agent":   agentStr,
+					"version": "5",
+					"type":    typeStr,
+				}).
+				Inc()
+
+			SFlowSampleRecordsStatsSum.With(
+				prometheus.Labels{
+					"router":  key,
+					"agent":   agentStr,
+					"version": "5",
+					"type":    typeStr,
+				}).
+				Add(float64(countRec))
+		}
+
+	}
+
+	var flowMessageSet []*flowmessage.FlowMessage
+	flowMessageSet, err = producer.ProcessMessageSFlowConfig(msgDec, s.Config)
+
+	timeTrackStop := time.Now()
+	DecoderTime.With(
+		prometheus.Labels{
+			"name": "sFlow",
+		}).
+		Observe(float64((timeTrackStop.Sub(timeTrackStart)).Nanoseconds()) / 1000)
+
+	for _, fmsg := range flowMessageSet {
+		fmsg.TimeReceived = ts
+		fmsg.TimeFlowStart = ts
+		fmsg.TimeFlowEnd = ts
+
+		if s.Format != nil {
+			key, data, err := s.Format.Format(fmsg)
+
+			if err != nil && s.Logger != nil {
+				s.Logger.Error(err)
+			}
+			if err == nil && s.Transport != nil {
+				s.Transport.Send(key, data)
+			}
+		}
+	}
+
+	return nil
+}
+
+func (s *StateSFlow) FlowRoutine(workers int, addr string, port int, reuseport bool) error {
+	return UDPRoutine("sFlow", s.DecodeFlow, workers, addr, port, reuseport, s.Logger)
+}

utils/sflow_test.go

@@ -0,0 +1,93 @@
+package utils
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestDecodeFlowExpandedSFlow(t *testing.T) {
+	msg := BaseMessage{
+		Src:     []byte{},
+		Port:    1,
+		Payload: getExpandedSFlowDecode(),
+	}
+
+	s := &StateSFlow{
+	}
+
+	assert.Nil(t, s.DecodeFlow(msg))
+}
+
+func getExpandedSFlowDecode() []byte {
+	return []byte{
+		0, 0, 0, 5, 0, 0, 0, 1, 1, 2, 3, 4, 0, 0, 0, 0, 5, 167, 139, 219, 5, 118,
+		138, 184, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 220, 2, 144, 194, 214, 0, 0, 0, 0,
+		0, 5, 6, 164, 0, 0, 3, 255, 6, 6, 189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5,
+		6, 164, 0, 0, 0, 0, 0, 5, 6, 171, 0, 0, 0, 2, 0, 0, 3, 233, 0, 0, 0, 6,
+		0, 0, 5, 7, 0, 0, 0, 0, 0, 0, 5, 7, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0,
+		0, 144, 0, 0, 0, 1, 0, 0, 5, 234, 0, 0, 0, 4, 0, 0, 0, 128, 8, 6, 168, 250,
+		146, 253, 116, 131, 239, 8, 101, 183, 129, 0, 5, 7, 8, 0, 9, 0, 5, 212, 0, 2, 4, 0,
+		3, 6, 252, 8, 9, 187, 169, 1, 4, 7, 186, 201, 1, 187, 249, 6, 160, 7, 5, 240, 6, 4,
+		4, 0, 0, 6, 0, 123, 119, 210, 0, 0, 165, 105, 7, 171, 145, 234, 102, 0, 252, 187, 162, 227,
+		104, 188, 126, 232, 156, 164, 2, 115, 6, 100, 0, 185, 6, 4, 119, 5, 213, 1, 215, 208, 8, 4,
+		118, 183, 241, 225, 130, 186, 2, 250, 220, 153, 189, 3, 4, 4, 1, 8, 210, 119, 172, 9, 164, 233,
+		1, 8, 171, 226, 196, 195, 3, 152, 9, 5, 6, 181, 4, 7, 0, 0, 0, 3, 0, 0, 0, 220,
+		9, 107, 215, 156, 0, 0, 0, 0, 0, 5, 6, 165, 0, 0, 3, 255, 226, 123, 0, 100, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 5, 6, 165, 0, 0, 0, 0, 0, 5, 6, 164, 0, 0, 0, 2,
+		0, 0, 3, 233, 0, 0, 0, 6, 0, 0, 3, 184, 0, 0, 0, 0, 0, 0, 3, 184, 0, 0,
+		0, 0, 0, 0, 0, 1, 0, 0, 0, 144, 0, 0, 0, 1, 0, 0, 5, 190, 0, 0, 0, 4,
+		0, 0, 0, 128, 116, 131, 239, 8, 101, 183, 144, 226, 186, 134, 8, 1, 129, 0, 3, 184, 8, 0,
+		9, 0, 5, 168, 7, 127, 4, 0, 4, 6, 163, 211, 185, 9, 220, 7, 0, 254, 3, 8, 0, 9,
+		130, 136, 179, 1, 2, 2, 7, 5, 250, 4, 128, 6, 0, 1, 7, 1, 0, 0, 1, 1, 8, 0,
+		6, 9, 250, 9, 4, 113, 121, 4, 160, 125, 0, 4, 9, 209, 241, 194, 190, 148, 161, 186, 6, 192,
+		246, 190, 170, 2, 238, 190, 128, 221, 223, 1, 218, 225, 3, 9, 7, 226, 220, 231, 127, 3, 3, 252,
+		7, 9, 161, 247, 218, 8, 8, 174, 133, 4, 213, 245, 149, 218, 5, 4, 200, 128, 139, 5, 0, 115,
+		0, 0, 0, 3, 0, 0, 0, 220, 2, 144, 194, 215, 0, 0, 0, 0, 0, 5, 6, 164, 0, 0,
+		3, 255, 6, 6, 253, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 164, 0, 0, 0, 0,
+		0, 5, 6, 171, 0, 0, 0, 2, 0, 0, 3, 233, 0, 0, 0, 6, 0, 0, 0, 104, 0, 0,
+		0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 144, 0, 0, 0, 1,
+		0, 0, 5, 242, 0, 0, 0, 4, 0, 0, 0, 128, 116, 131, 239, 7, 9, 1, 116, 131, 239, 8,
+		101, 183, 129, 0, 0, 104, 8, 0, 9, 0, 5, 220, 152, 143, 4, 0, 1, 6, 5, 179, 9, 187,
+		191, 101, 190, 2, 144, 182, 0, 0, 130, 4, 252, 4, 160, 192, 138, 8, 219, 124, 128, 6, 0, 235,
+		180, 213, 0, 0, 1, 1, 8, 0, 9, 124, 6, 1, 9, 1, 252, 3, 194, 8, 195, 209, 115, 1,
+		5, 152, 204, 2, 6, 4, 1, 119, 254, 9, 1, 170, 0, 192, 2, 7, 190, 9, 149, 5, 101, 2,
+		128, 122, 0, 190, 1, 109, 188, 175, 4, 8, 152, 1, 142, 108, 2, 100, 2, 124, 125, 195, 5, 8,
+		233, 126, 7, 4, 243, 4, 3, 153, 0, 0, 0, 3, 0, 0, 0, 220, 5, 1, 150, 6, 0, 0,
+		0, 0, 0, 5, 6, 167, 0, 0, 3, 255, 6, 5, 105, 220, 0, 0, 0, 0, 0, 0, 0, 0,
+		0, 5, 6, 167, 0, 0, 0, 0, 0, 5, 6, 164, 0, 0, 0, 2, 0, 0, 3, 233, 0, 0,
+		0, 6, 0, 0, 5, 7, 0, 0, 0, 0, 0, 0, 5, 7, 0, 0, 0, 0, 0, 0, 0, 1,
+		0, 0, 0, 144, 0, 0, 0, 1, 0, 0, 2, 2, 0, 0, 0, 4, 0, 0, 0, 128, 116, 131,
+		239, 8, 101, 183, 152, 3, 130, 1, 196, 153, 129, 0, 5, 7, 8, 0, 9, 0, 2, 0, 0, 0,
+		4, 0, 126, 7, 119, 188, 185, 9, 221, 8, 2, 116, 144, 0, 9, 139, 3, 112, 2, 0, 8, 124,
+		255, 251, 0, 0, 131, 2, 0, 0, 0, 246, 3, 3, 107, 5, 0, 0, 0, 0, 9, 173, 2, 217,
+		6, 248, 0, 0, 9, 173, 2, 217, 8, 248, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+		255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 6, 9, 153,
+		215, 157, 0, 255, 0, 8, 1, 0, 9, 8, 9, 6, 164, 103, 9, 5, 0, 0, 0, 3, 0, 0,
+		0, 152, 5, 201, 2, 175, 0, 0, 0, 0, 0, 5, 6, 5, 0, 0, 3, 255, 1, 8, 9, 1,
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 5, 0, 0, 0, 0, 0, 5, 6, 164, 0, 0,
+		0, 2, 0, 0, 3, 233, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3,
+		0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0,
+		0, 4, 0, 0, 0, 0, 116, 131, 239, 8, 101, 183, 218, 177, 4, 251, 217, 207, 8, 0, 9, 0,
+		0, 8, 0, 0, 0, 0, 9, 7, 8, 161, 106, 3, 109, 6, 185, 9, 220, 215, 0, 123, 9, 184,
+		0, 8, 116, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 130, 6,
+		0, 0, 0, 3, 0, 0, 0, 220, 2, 144, 194, 216, 0, 0, 0, 0, 0, 5, 6, 164, 0, 0,
+		3, 255, 6, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 164, 0, 0, 0, 0,
+		0, 5, 6, 165, 0, 0, 0, 2, 0, 0, 3, 233, 0, 0, 0, 6, 0, 0, 3, 202, 0, 0,
+		0, 0, 0, 0, 3, 202, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 144, 0, 0, 0, 1,
+		0, 0, 5, 242, 0, 0, 0, 4, 0, 0, 0, 128, 144, 226, 186, 135, 4, 241, 116, 131, 239, 8,
+		101, 183, 129, 0, 3, 202, 8, 0, 9, 0, 5, 220, 147, 0, 4, 0, 7, 6, 225, 131, 1, 159,
+		7, 185, 195, 181, 170, 8, 9, 117, 7, 175, 8, 3, 191, 135, 190, 150, 196, 102, 0, 6, 0, 119,
+		116, 113, 0, 0, 201, 244, 240, 206, 2, 117, 4, 139, 8, 4, 240, 223, 247, 123, 6, 0, 239, 0,
+		9, 116, 152, 153, 191, 0, 124, 2, 7, 8, 3, 178, 166, 150, 3, 218, 163, 175, 121, 8, 4, 210,
+		4, 5, 166, 5, 178, 1, 6, 222, 172, 186, 6, 241, 232, 8, 188, 192, 2, 220, 128, 1, 8, 7,
+		194, 130, 220, 5, 2, 0, 158, 195, 0, 4, 3, 2, 160, 158, 157, 2, 102, 3, 7, 3, 0, 0,
+		1, 3, 3, 4, 1, 1, 4, 2, 187, 255, 188, 3, 4, 138, 9, 180, 104, 233, 212, 239, 123, 237,
+		112, 8, 133, 129, 152, 138, 7, 195, 8, 171, 237, 3, 4, 223, 116, 214, 151, 9, 151, 102, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		0, 0, 0, 0,
+	}
+}

utils/utils.go

@@ -0,0 +1,174 @@
+package utils
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"strconv"
+	"time"
+
+	reuseport "github.com/libp2p/go-reuseport"
+	decoder "github.com/netsampler/goflow2/decoders"
+	"github.com/netsampler/goflow2/decoders/netflow"
+	flowmessage "github.com/netsampler/goflow2/pb"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+func GetServiceAddresses(srv string) (addrs []string, err error) {
+	_, srvs, err := net.LookupSRV("", "", srv)
+	if err != nil {
+		return nil, errors.New(fmt.Sprintf("Service discovery: %v\n", err))
+	}
+	for _, srv := range srvs {
+		addrs = append(addrs, net.JoinHostPort(srv.Target, strconv.Itoa(int(srv.Port))))
+	}
+	return addrs, nil
+}
+
+type Logger interface {
+	Printf(string, ...interface{})
+	Errorf(string, ...interface{})
+	Warnf(string, ...interface{})
+	Warn(...interface{})
+	Error(...interface{})
+	Debug(...interface{})
+	Debugf(string, ...interface{})
+	Infof(string, ...interface{})
+	Fatalf(string, ...interface{})
+}
+
+type BaseMessage struct {
+	Src     net.IP
+	Port    int
+	Payload []byte
+
+	SetTime  bool
+	RecvTime time.Time
+}
+
+type Transport interface {
+	Send([]*flowmessage.FlowMessage)
+}
+
+type Formatter interface {
+	Format([]*flowmessage.FlowMessage)
+}
+
+/*
+type DefaultLogTransport struct {
+}
+
+func (s *DefaultLogTransport) Publish(msgs []*flowmessage.FlowMessage) {
+	for _, msg := range msgs {
+		fmt.Printf("%v\n", FlowMessageToString(msg))
+	}
+}
+
+type DefaultJSONTransport struct {
+}
+
+func (s *DefaultJSONTransport) Publish(msgs []*flowmessage.FlowMessage) {
+	for _, msg := range msgs {
+		fmt.Printf("%v\n", FlowMessageToJSON(msg))
+	}
+}
+*/
+type DefaultErrorCallback struct {
+	Logger Logger
+}
+
+func (cb *DefaultErrorCallback) Callback(name string, id int, start, end time.Time, err error) {
+	if _, ok := err.(*netflow.ErrorTemplateNotFound); ok {
+		return
+	}
+	if cb.Logger != nil {
+		cb.Logger.Errorf("Error from: %v (%v) duration: %v. %v", name, id, end.Sub(start), err)
+	}
+}
+
+func UDPRoutine(name string, decodeFunc decoder.DecoderFunc, workers int, addr string, port int, sockReuse bool, logger Logger) error {
+	ecb := DefaultErrorCallback{
+		Logger: logger,
+	}
+
+	decoderParams := decoder.DecoderParams{
+		DecoderFunc:   decodeFunc,
+		DoneCallback:  DefaultAccountCallback,
+		ErrorCallback: ecb.Callback,
+	}
+
+	processor := decoder.CreateProcessor(workers, decoderParams, name)
+	processor.Start()
+
+	addrUDP := net.UDPAddr{
+		IP:   net.ParseIP(addr),
+		Port: port,
+	}
+
+	var udpconn *net.UDPConn
+	var err error
+
+	if sockReuse {
+		pconn, err := reuseport.ListenPacket("udp", addrUDP.String())
+		defer pconn.Close()
+		if err != nil {
+			return err
+		}
+		var ok bool
+		udpconn, ok = pconn.(*net.UDPConn)
+		if !ok {
+			return err
+		}
+	} else {
+		udpconn, err = net.ListenUDP("udp", &addrUDP)
+		if err != nil {
+			return err
+		}
+		defer udpconn.Close()
+	}
+
+	payload := make([]byte, 9000)
+
+	localIP := addrUDP.IP.String()
+	if addrUDP.IP == nil {
+		localIP = ""
+	}
+
+	for {
+		size, pktAddr, _ := udpconn.ReadFromUDP(payload)
+		payloadCut := make([]byte, size)
+		copy(payloadCut, payload[0:size])
+
+		baseMessage := BaseMessage{
+			Src:     pktAddr.IP,
+			Port:    pktAddr.Port,
+			Payload: payloadCut,
+		}
+		processor.ProcessMessage(baseMessage)
+
+		MetricTrafficBytes.With(
+			prometheus.Labels{
+				"remote_ip":  pktAddr.IP.String(),
+				"local_ip":   localIP,
+				"local_port": strconv.Itoa(addrUDP.Port),
+				"type":       name,
+			}).
+			Add(float64(size))
+		MetricTrafficPackets.With(
+			prometheus.Labels{
+				"remote_ip":  pktAddr.IP.String(),
+				"local_ip":   localIP,
+				"local_port": strconv.Itoa(addrUDP.Port),
+				"type":       name,
+			}).
+			Inc()
+		MetricPacketSizeSum.With(
+			prometheus.Labels{
+				"remote_ip":  pktAddr.IP.String(),
+				"local_ip":   localIP,
+				"local_port": strconv.Itoa(addrUDP.Port),
+				"type":       name,
+			}).
+			Observe(float64(size))
+	}
+}