Merge branch 'netsampler:main' into main

go.mod

@@ -42,9 +42,9 @@ require (
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
-	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/crypto v0.17.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/sys v0.15.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -88,8 +88,8 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
+golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
-golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -118,8 +118,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=

producer/proto/messages.go

@@ -2,7 +2,6 @@ package protoproducer
 
 import (
 	"bytes"
-	"encoding/hex"
 	"fmt"
 	"hash"
 	"hash/fnv"
@@ -121,7 +120,8 @@ func (m *ProtoProducerMessage) mapUnknown() map[string]interface{} {
 				value = v
 			} else if dataType == protowire.BytesType {
 				v, _ := protowire.ConsumeString(data)
-				value = hex.EncodeToString([]byte(v))
+				//value = hex.EncodeToString([]byte(v)) // removed, this conversion is left to the renderer
+				value = []byte(v)
 			} else {
 				continue
 			}

producer/proto/messages_test.go

@@ -0,0 +1,67 @@
+package protoproducer
+
+import (
+	"testing"
+
+	"google.golang.org/protobuf/encoding/protowire"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMarshalJSON(t *testing.T) {
+	var m ProtoProducerMessage
+
+	m.formatter = &FormatterConfigMapper{
+		fields: []string{"Etype", "test1", "test2", "test3"},
+		rename: map[string]string{
+			"Etype": "etype",
+		},
+		numToPb: map[int32]ProtobufFormatterConfig{
+			100: ProtobufFormatterConfig{
+				Name:  "test1",
+				Index: 100,
+				Type:  "varint",
+				Array: false,
+			},
+			101: ProtobufFormatterConfig{
+				Name:  "test2",
+				Index: 101,
+				Type:  "string",
+				Array: false,
+			},
+			102: ProtobufFormatterConfig{
+				Name:  "test3",
+				Index: 102,
+				Type:  "bytes",
+				Array: false,
+			},
+		},
+		render: map[string]RenderFunc{
+			"Etype": EtypeRenderer,
+			"test1": EtypeRenderer,
+			"test2": NilRenderer,
+			"test3": StringRenderer,
+		},
+	}
+
+	m.FlowMessage.Etype = 0x86dd
+
+	fmr := m.FlowMessage.ProtoReflect()
+	unk := fmr.GetUnknown()
+
+	unk = protowire.AppendTag(unk, protowire.Number(100), protowire.VarintType)
+	unk = protowire.AppendVarint(unk, 0x86dd)
+
+	unk = protowire.AppendTag(unk, protowire.Number(101), protowire.BytesType)
+	unk = protowire.AppendString(unk, string("testing"))
+
+	unk = protowire.AppendTag(unk, protowire.Number(102), protowire.BytesType)
+	unk = protowire.AppendString(unk, string([]byte{0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67}))
+
+	fmr.SetUnknown(unk)
+
+	out, err := m.MarshalJSON()
+	assert.Nil(t, err)
+	t.Log(string(out))
+	assert.Equal(t, "{\"etype\":\"IPv6\",\"test1\":\"IPv6\",\"test2\":\"74657374696e67\",\"test3\":\"testing\"}", string(out))
+}

producer/proto/render.go

@@ -22,6 +22,7 @@ const (
 	RendererNetwork      RendererID = "network"
 	RendererDateTime     RendererID = "datetime"
 	RendererDateTimeNano RendererID = "datetimenano"
+	RendererString       RendererID = "string"
 )
 
 var (
@@ -33,6 +34,7 @@ var (
 		RendererProto:        ProtoRenderer,
 		RendererDateTime:     DateTimeRenderer,
 		RendererDateTimeNano: DateTimeNanoRenderer,
+		RendererString:       StringRenderer,
 	}
 
 	defaultRenderers = map[string]RenderFunc{
@@ -95,6 +97,15 @@ func NilRenderer(msg *ProtoProducerMessage, fieldName string, data interface{})
 	return data
 }
 
+func StringRenderer(msg *ProtoProducerMessage, fieldName string, data interface{}) interface{} {
+	if dataC, ok := data.([]byte); ok {
+		return string(dataC)
+	} else if dataC, ok := data.(string); ok {
+		return string(dataC)
+	} // maybe should support uint64?
+	return NilRenderer(msg, fieldName, data)
+}
+
 func DateTimeRenderer(msg *ProtoProducerMessage, fieldName string, data interface{}) interface{} {
 	if dataC, ok := data.(uint64); ok {
 		ts := time.Unix(int64(dataC), 0).UTC()
@@ -151,6 +162,8 @@ func IPRenderer(msg *ProtoProducerMessage, fieldName string, data interface{}) i
 func EtypeRenderer(msg *ProtoProducerMessage, fieldName string, data interface{}) interface{} {
 	if dataC, ok := data.(uint32); ok {
 		return etypeName[dataC]
+	} else if dataC, ok := data.(uint64); ok { // supports protobuf mapped fields
+		return etypeName[uint32(dataC)]
 	}
 	return "unknown"
 }
@@ -158,6 +171,8 @@ func EtypeRenderer(msg *ProtoProducerMessage, fieldName string, data interface{}
 func ProtoRenderer(msg *ProtoProducerMessage, fieldName string, data interface{}) interface{} {
 	if dataC, ok := data.(uint32); ok {
 		return protoName[dataC]
+	} else if dataC, ok := data.(uint64); ok {
+		return protoName[uint32(dataC)]
 	}
 	return "unknown"
 }