Merge branch 'netsampler:main' into main

Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.14.0 to 0.17.0.
- [Commits](https://github.com/golang/crypto/compare/v0.14.0...v0.17.0)

---
updated-dependencies:
- dependency-name: golang.org/x/crypto
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/workflows/docker-release.yml

@@ -1,41 +0,0 @@
-name: DockerRelease
-
-on:
-  push:
-    tags:
-      - 'v*'
-
-jobs:
-  build:
-    name: DockerRelease
-    runs-on: ubuntu-latest
-    steps:
-
-    - name: Set up Go 1.x
-      uses: actions/setup-go@v2
-      with:
-        go-version: ^1.21
-
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v1
-      with:
-        fetch-depth: 0
-
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v1
-
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v1
-
-    - name: Login to DockerHub
-      uses: docker/login-action@v1
-      with:
-        username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-    - name: Build
-      run: |
-        export VERSION=$(git describe --tags --abbrev=0 HEAD)
-        DOCKER_SUFFIX=-amd64 DOCKER_CMD='buildx build --push --platform linux/amd64' make docker
-        DOCKER_SUFFIX=-arm64 DOCKER_CMD='buildx build --push --platform linux/arm64/v8' make docker
-        make docker-manifest-release-buildx

.github/workflows/docker-release_registry_github.yml

@@ -29,11 +29,11 @@ jobs:
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v1
 
-    - name: Login to DockerHub
+    # - name: Login to DockerHub
-      uses: docker/login-action@v1
+    #   uses: docker/login-action@v1
-      with:
+    #   with:
-        username: ${{ secrets.DOCKERHUB_USERNAME }}
+    #     username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
+    #     password: ${{ secrets.DOCKERHUB_TOKEN }}
 
     - name: Log in to registry
       run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin

.github/workflows/docker.yml

@@ -26,11 +26,11 @@ jobs:
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v1
 
-    - name: Login to DockerHub
+    # - name: Login to DockerHub
-      uses: docker/login-action@v1
+    #   uses: docker/login-action@v1
-      with:
+    #   with:
-        username: ${{ secrets.DOCKERHUB_USERNAME }}
+    #     username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
+    #     password: ${{ secrets.DOCKERHUB_TOKEN }}
 
     - name: Build
       run: |

Makefile

@@ -12,7 +12,7 @@ COMMIT        ?= $(shell git rev-parse HEAD)
 TAG           ?= $(shell git describe --tags --abbrev=0 HEAD)
 VERSION_PKG   ?= $(shell echo $(VERSION) | sed 's/^v//g')
 LICENSE       := BSD-3-Clause
-URL           := https://github.com/netsampler/goflow2
+URL           := https://github.com/openobserve/goflow2
 DESCRIPTION   := GoFlow2: Open-Source and Scalable Network Sample Collector
 DATE          :=  $(shell date +%FT%T%z)
 BUILDINFOS    ?=  ($(DATE)$(BUILDINFOSDET))

README.md

@@ -1,3 +1,5 @@
+Notice: This is a fork of https://github.com/netsampler/goflow2 and adds an HTTP transport.
+
 # GoFlow2
 
 [![Build Status](https://github.com/netsampler/goflow2/workflows/Build/badge.svg)](https://github.com/netsampler/goflow2/actions?query=workflow%3ABuild)
@@ -85,6 +87,7 @@ Production:
 * Convert to protobuf or json
 * Prints to the console/file
 * Sends to Kafka and partition
+* Sends to HTTP endpoint
 
 Monitoring via Prometheus metrics
 
@@ -183,12 +186,23 @@ This will allow you to visualize the data in OpenObserve:
 
 ![OpenObserve](/graphics/netflow-top-items.png)
 
+Once you have run the above command, you can send sample data to the collector using the [nflow-generator](https://github.com/nerdalert/nflow-generator) :
+
+You can test the collector using the [nflow-generator](https://github.com/nerdalert/nflow-generator) to generate sample records:
+
+```shell
+./nflow-generator -t 127.0.0.1 -p 2055
+```
 
 ### Docker
 
 You can also run directly with a container:
 ```
-$ sudo docker run -p 6343:6343/udp -p 2055:2055/udp -ti netsampler/goflow2:latest
+docker run -p 6343:6343/udp -p 2055:2055/udp -ti ghcr.io/openobserve/goflow2:v100.0.1 -transport=http \
+  -transport.http.destination=http://192.168.86.75:5080/api/default/gflow4/_json \
+  -transport.http.batchSize=100 \
+  -transport.http.auth.header=Authorization \
+  -transport.http.auth.credentials="Basic cm9vdEBleGFtcGxlLmNvbTpDb21wbGV4cGFzcyMxMjM="
 ```
 
 ### Mapping extra fields

go.mod

@@ -42,9 +42,9 @@ require (
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
-	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/crypto v0.17.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/sys v0.15.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -88,8 +88,8 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
+golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
-golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -118,8 +118,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=

producer/proto/messages.go

@@ -2,7 +2,6 @@ package protoproducer
 
 import (
 	"bytes"
-	"encoding/hex"
 	"fmt"
 	"hash"
 	"hash/fnv"
@@ -121,7 +120,8 @@ func (m *ProtoProducerMessage) mapUnknown() map[string]interface{} {
 				value = v
 			} else if dataType == protowire.BytesType {
 				v, _ := protowire.ConsumeString(data)
-				value = hex.EncodeToString([]byte(v))
+				//value = hex.EncodeToString([]byte(v)) // removed, this conversion is left to the renderer
+				value = []byte(v)
 			} else {
 				continue
 			}

producer/proto/messages_test.go

@@ -0,0 +1,67 @@
+package protoproducer
+
+import (
+	"testing"
+
+	"google.golang.org/protobuf/encoding/protowire"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMarshalJSON(t *testing.T) {
+	var m ProtoProducerMessage
+
+	m.formatter = &FormatterConfigMapper{
+		fields: []string{"Etype", "test1", "test2", "test3"},
+		rename: map[string]string{
+			"Etype": "etype",
+		},
+		numToPb: map[int32]ProtobufFormatterConfig{
+			100: ProtobufFormatterConfig{
+				Name:  "test1",
+				Index: 100,
+				Type:  "varint",
+				Array: false,
+			},
+			101: ProtobufFormatterConfig{
+				Name:  "test2",
+				Index: 101,
+				Type:  "string",
+				Array: false,
+			},
+			102: ProtobufFormatterConfig{
+				Name:  "test3",
+				Index: 102,
+				Type:  "bytes",
+				Array: false,
+			},
+		},
+		render: map[string]RenderFunc{
+			"Etype": EtypeRenderer,
+			"test1": EtypeRenderer,
+			"test2": NilRenderer,
+			"test3": StringRenderer,
+		},
+	}
+
+	m.FlowMessage.Etype = 0x86dd
+
+	fmr := m.FlowMessage.ProtoReflect()
+	unk := fmr.GetUnknown()
+
+	unk = protowire.AppendTag(unk, protowire.Number(100), protowire.VarintType)
+	unk = protowire.AppendVarint(unk, 0x86dd)
+
+	unk = protowire.AppendTag(unk, protowire.Number(101), protowire.BytesType)
+	unk = protowire.AppendString(unk, string("testing"))
+
+	unk = protowire.AppendTag(unk, protowire.Number(102), protowire.BytesType)
+	unk = protowire.AppendString(unk, string([]byte{0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67}))
+
+	fmr.SetUnknown(unk)
+
+	out, err := m.MarshalJSON()
+	assert.Nil(t, err)
+	t.Log(string(out))
+	assert.Equal(t, "{\"etype\":\"IPv6\",\"test1\":\"IPv6\",\"test2\":\"74657374696e67\",\"test3\":\"testing\"}", string(out))
+}

producer/proto/render.go

@@ -22,6 +22,7 @@ const (
 	RendererNetwork      RendererID = "network"
 	RendererDateTime     RendererID = "datetime"
 	RendererDateTimeNano RendererID = "datetimenano"
+	RendererString       RendererID = "string"
 )
 
 var (
@@ -33,6 +34,7 @@ var (
 		RendererProto:        ProtoRenderer,
 		RendererDateTime:     DateTimeRenderer,
 		RendererDateTimeNano: DateTimeNanoRenderer,
+		RendererString:       StringRenderer,
 	}
 
 	defaultRenderers = map[string]RenderFunc{
@@ -95,6 +97,15 @@ func NilRenderer(msg *ProtoProducerMessage, fieldName string, data interface{})
 	return data
 }
 
+func StringRenderer(msg *ProtoProducerMessage, fieldName string, data interface{}) interface{} {
+	if dataC, ok := data.([]byte); ok {
+		return string(dataC)
+	} else if dataC, ok := data.(string); ok {
+		return string(dataC)
+	} // maybe should support uint64?
+	return NilRenderer(msg, fieldName, data)
+}
+
 func DateTimeRenderer(msg *ProtoProducerMessage, fieldName string, data interface{}) interface{} {
 	if dataC, ok := data.(uint64); ok {
 		ts := time.Unix(int64(dataC), 0).UTC()
@@ -151,6 +162,8 @@ func IPRenderer(msg *ProtoProducerMessage, fieldName string, data interface{}) i
 func EtypeRenderer(msg *ProtoProducerMessage, fieldName string, data interface{}) interface{} {
 	if dataC, ok := data.(uint32); ok {
 		return etypeName[dataC]
+	} else if dataC, ok := data.(uint64); ok { // supports protobuf mapped fields
+		return etypeName[uint32(dataC)]
 	}
 	return "unknown"
 }
@@ -158,6 +171,8 @@ func EtypeRenderer(msg *ProtoProducerMessage, fieldName string, data interface{}
 func ProtoRenderer(msg *ProtoProducerMessage, fieldName string, data interface{}) interface{} {
 	if dataC, ok := data.(uint32); ok {
 		return protoName[dataC]
+	} else if dataC, ok := data.(uint64); ok {
+		return protoName[uint32(dataC)]
 	}
 	return "unknown"
 }