paulmataruso/goflow2
1
0
Fork 0
mirror of https://github.com/openobserve/goflow2.git synced 2025-10-22 23:01:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'netsampler:main' into main

Browse Source
This commit is contained in:
Prabhat Sharma
2024-02-08 04:53:33 -08:00
committed by GitHub
parent c599adc412 ee095a980f
commit 80c7e5ddfe
5 changed files with 90 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
go.mod
View File

@@ -42,9 +42,9 @@ require (
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
github.com/xdg-go/stringprep v1.0.4 // indirect
golang.org/x/crypto v0.14.0 // indirect
golang.org/x/crypto v0.17.0 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/sys v0.15.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/text v0.14.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

8
go.sum
View File

@@ -88,8 +88,8 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -118,8 +118,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=

4
producer/proto/messages.go
View File

@@ -2,7 +2,6 @@ package protoproducer
import (
"bytes"
"encoding/hex"
"fmt"
"hash"
"hash/fnv"
@@ -121,7 +120,8 @@ func (m *ProtoProducerMessage) mapUnknown() map[string]interface{} {
value = v
} else if dataType == protowire.BytesType {
v, _ := protowire.ConsumeString(data)
value = hex.EncodeToString([]byte(v))
//value = hex.EncodeToString([]byte(v)) // removed, this conversion is left to the renderer
value = []byte(v)
} else {
continue
}

67
producer/proto/messages_test.go Normal file
View File

@@ -0,0 +1,67 @@
package protoproducer
import (
"testing"
"google.golang.org/protobuf/encoding/protowire"
"github.com/stretchr/testify/assert"
)
func TestMarshalJSON(t *testing.T) {
var m ProtoProducerMessage
m.formatter = &FormatterConfigMapper{
fields: []string{"Etype", "test1", "test2", "test3"},
rename: map[string]string{
"Etype": "etype",
},
numToPb: map[int32]ProtobufFormatterConfig{
100: ProtobufFormatterConfig{
Name: "test1",
Index: 100,
Type: "varint",
Array: false,
},
101: ProtobufFormatterConfig{
Name: "test2",
Index: 101,
Type: "string",
Array: false,
},
102: ProtobufFormatterConfig{
Name: "test3",
Index: 102,
Type: "bytes",
Array: false,
},
},
render: map[string]RenderFunc{
"Etype": EtypeRenderer,
"test1": EtypeRenderer,
"test2": NilRenderer,
"test3": StringRenderer,
},
}
m.FlowMessage.Etype = 0x86dd
fmr := m.FlowMessage.ProtoReflect()
unk := fmr.GetUnknown()
unk = protowire.AppendTag(unk, protowire.Number(100), protowire.VarintType)
unk = protowire.AppendVarint(unk, 0x86dd)
unk = protowire.AppendTag(unk, protowire.Number(101), protowire.BytesType)
unk = protowire.AppendString(unk, string("testing"))
unk = protowire.AppendTag(unk, protowire.Number(102), protowire.BytesType)
unk = protowire.AppendString(unk, string([]byte{0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67}))
fmr.SetUnknown(unk)
out, err := m.MarshalJSON()
assert.Nil(t, err)
t.Log(string(out))
assert.Equal(t, "{\"etype\":\"IPv6\",\"test1\":\"IPv6\",\"test2\":\"74657374696e67\",\"test3\":\"testing\"}", string(out))
}

15
producer/proto/render.go
View File

@@ -22,6 +22,7 @@ const (
RendererNetwork RendererID = "network"
RendererDateTime RendererID = "datetime"
RendererDateTimeNano RendererID = "datetimenano"
RendererString RendererID = "string"
)
var (
@@ -33,6 +34,7 @@ var (
RendererProto: ProtoRenderer,
RendererDateTime: DateTimeRenderer,
RendererDateTimeNano: DateTimeNanoRenderer,
RendererString: StringRenderer,
}
defaultRenderers = map[string]RenderFunc{
@@ -95,6 +97,15 @@ func NilRenderer(msg *ProtoProducerMessage, fieldName string, data interface{})
return data
}
func StringRenderer(msg *ProtoProducerMessage, fieldName string, data interface{}) interface{} {
if dataC, ok := data.([]byte); ok {
return string(dataC)
} else if dataC, ok := data.(string); ok {
return string(dataC)
} // maybe should support uint64?
return NilRenderer(msg, fieldName, data)
}
func DateTimeRenderer(msg *ProtoProducerMessage, fieldName string, data interface{}) interface{} {
if dataC, ok := data.(uint64); ok {
ts := time.Unix(int64(dataC), 0).UTC()
@@ -151,6 +162,8 @@ func IPRenderer(msg *ProtoProducerMessage, fieldName string, data interface{}) i
func EtypeRenderer(msg *ProtoProducerMessage, fieldName string, data interface{}) interface{} {
if dataC, ok := data.(uint32); ok {
return etypeName[dataC]
} else if dataC, ok := data.(uint64); ok { // supports protobuf mapped fields
return etypeName[uint32(dataC)]
}
return "unknown"
}
@@ -158,6 +171,8 @@ func EtypeRenderer(msg *ProtoProducerMessage, fieldName string, data interface{}
func ProtoRenderer(msg *ProtoProducerMessage, fieldName string, data interface{}) interface{} {
if dataC, ok := data.(uint32); ok {
return protoName[dataC]
} else if dataC, ok := data.(uint64); ok {
return protoName[uint32(dataC)]
}
return "unknown"
}