mirror of
https://github.com/openobserve/goflow2.git
synced 2025-10-23 07:11:57 +00:00
Custom map flow fields (#36)
* adds dataframe link decoding * can map NetFlow/IPFIX fields and bytes sections from sFlow/packets to any field inside the protobuf * add CLI argument for loading a mapping yaml file
This commit is contained in:
Louis
GitHub
@@ -47,6 +47,8 @@ var (
|
||||
|
||||
TemplatePath = flag.String("templates.path", "/templates", "NetFlow/IPFIX templates list")
|
||||
|
||||
MappingFile = flag.String("mapping", "", "Configuration file for custom mappings")
|
||||
|
||||
Version = flag.Bool("v", false, "Print version")
|
||||
)
|
||||
|
||||
@@ -67,6 +69,19 @@ func main() {
|
||||
lvl, _ := log.ParseLevel(*LogLevel)
|
||||
log.SetLevel(lvl)
|
||||
|
||||
var config utils.ProducerConfig
|
||||
if *MappingFile != "" {
|
||||
f, err := os.Open(*MappingFile)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
config, err = utils.LoadMapping(f)
|
||||
f.Close()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
formatter, err := format.FindFormat(ctx, *Format)
|
||||
@@ -121,6 +136,7 @@ func main() {
|
||||
Format: formatter,
|
||||
Transport: transporter,
|
||||
Logger: log.StandardLogger(),
|
||||
Config: config,
|
||||
}
|
||||
err = sSFlow.FlowRoutine(*Workers, hostname, int(port), *ReusePort)
|
||||
} else if listenAddrUrl.Scheme == "netflow" {
|
||||
@@ -128,6 +144,7 @@ func main() {
|
||||
Format: formatter,
|
||||
Transport: transporter,
|
||||
Logger: log.StandardLogger(),
|
||||
Config: config,
|
||||
}
|
||||
err = sNF.FlowRoutine(*Workers, hostname, int(port), *ReusePort)
|
||||
} else if listenAddrUrl.Scheme == "nfl" {
|
||||
|
||||
24
cmd/goflow2/mapping.yaml
Normal file
24
cmd/goflow2/mapping.yaml
Normal file
@@ -0,0 +1,24 @@
|
||||
ipfix:
|
||||
mapping:
|
||||
- field: 7 # IPFIX_FIELD_sourceTransportPort
|
||||
destination: CustomInteger1
|
||||
- field: 11 # IPFIX_FIELD_destinationTransportPort
|
||||
destination: CustomInteger2
|
||||
# penprovided: false
|
||||
# pen: 0
|
||||
netflowv9:
|
||||
mapping:
|
||||
- field: 7
|
||||
destination: CustomInteger1
|
||||
- field: 11
|
||||
destination: CustomInteger2
|
||||
sflow:
|
||||
mapping:
|
||||
- layer: 4 # Layer 4: TCP or UDP
|
||||
offset: 0 # Source port
|
||||
length: 16 # 2 bytes
|
||||
destination: CustomInteger1
|
||||
- layer: 4
|
||||
offset: 16 # Destination port
|
||||
length: 16 # 2 bytes
|
||||
destination: CustomInteger2
|
||||
@@ -121,6 +121,7 @@ func DecodeTemplateSet(version uint16, payload *bytes.Buffer) ([]TemplateRecord,
|
||||
|
||||
if version == 10 && field.Type&0x8000 != 0 {
|
||||
field.PenProvided = true
|
||||
field.Type = field.Type ^ 0x8000
|
||||
err = utils.BinaryDecoder(payload, &field.Pen)
|
||||
}
|
||||
fields[i] = field
|
||||
@@ -165,8 +166,10 @@ func DecodeDataSetUsingFields(version uint16, payload *bytes.Buffer, listFields
|
||||
|
||||
value := payload.Next(finalLength)
|
||||
nfvalue := DataField{
|
||||
Type: templateField.Type,
|
||||
Value: value,
|
||||
Type: templateField.Type,
|
||||
PenProvided: templateField.PenProvided,
|
||||
Pen: templateField.Pen,
|
||||
Value: value,
|
||||
}
|
||||
dataFields[i] = nfvalue
|
||||
}
|
||||
|
||||
@@ -89,7 +89,9 @@ type Field struct {
|
||||
|
||||
type DataField struct {
|
||||
// A numeric value that represents the type of field.
|
||||
Type uint16
|
||||
PenProvided bool
|
||||
Type uint16
|
||||
Pen uint32
|
||||
|
||||
// The value (in bytes) of the field.
|
||||
Value interface{}
|
||||
|
||||
@@ -56,3 +56,51 @@ The mapping to the protobuf format is listed in the table below.
|
||||
|MPLSxTTL|TTL of the MPLS label||Included|||
|
||||
|MPLSxLabel|MPLS label||Included|||
|
||||
|
||||
## Add new custom fields
|
||||
|
||||
If you are using enterprise fields that you need decoded
|
||||
or if you are looking for specific bytes inside the packet sample.
|
||||
|
||||
This feature is only available when sending Protobufs (no text output).
|
||||
|
||||
The [`mapping.yaml`](../cmd/goflow2/mapping.yaml) example file
|
||||
will collect source and destination port again, use it with `-mapping=mapping.yaml` in the CLI.
|
||||
|
||||
Data coming from the flows can be added to the protobuf either as an unsigned/signed integer a slice of bytes.
|
||||
|
||||
The `sflow` section allow to extract data from packet samples inside sFlow and inside IPFIX (dataframe).
|
||||
The following layers are available:
|
||||
* 0: no offset
|
||||
* 3: network layer, offsets to IP/IPv6 header
|
||||
* 4: transport layer, offsets to TCP/UDP header
|
||||
* 7: application layer, offsets to the TCP/UDP payload
|
||||
|
||||
|
||||
```yaml
|
||||
ipfix:
|
||||
mapping:
|
||||
- field: 7 # NetFlow or IPFIX field ID
|
||||
destination: CustomInteger1 # Name of the field inside the Protobuf
|
||||
penprovided: false # Has an enterprise number (optional)
|
||||
pen: 0 # Enterprise number (optional)
|
||||
netflowv9:
|
||||
mapping: []
|
||||
# ... similar to above, Enterprise number will not be supported
|
||||
sflow:
|
||||
mapping:
|
||||
- layer: 4 # Layer
|
||||
offset: 0 # Source port
|
||||
length: 16 # 2 bytes
|
||||
destination: CustomInteger1
|
||||
```
|
||||
|
||||
Without editing and recompiling the [protobuf](../pb/flow.proto), you can use up to 5 integers and 5 slices of bytes:
|
||||
|
||||
```protobuf
|
||||
// Custom allocations
|
||||
uint64 CustomInteger1 = 1001;
|
||||
[...]
|
||||
|
||||
bytes CustomBytes1 = 1011;
|
||||
[...]
|
||||
```
|
||||
@@ -16,6 +16,7 @@ const (
|
||||
FORMAT_TYPE_INTEGER
|
||||
FORMAT_TYPE_IP
|
||||
FORMAT_TYPE_MAC
|
||||
FORMAT_TYPE_BYTES
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -231,6 +232,8 @@ func FormatMessageReflectCustom(msg proto.Message, ext, quotes, sep, sign string
|
||||
mac := make([]byte, 8)
|
||||
binary.BigEndian.PutUint64(mac, fieldValue.Uint())
|
||||
fstr[i] = fmt.Sprintf("%s%s%s%s%q", quotes, kf, quotes, sign, net.HardwareAddr(mac[2:]).String())
|
||||
case FORMAT_TYPE_BYTES:
|
||||
fstr[i] = fmt.Sprintf("%s%s%s%s%.2x", quotes, kf, quotes, sign, fieldValue.Bytes())
|
||||
default:
|
||||
if null {
|
||||
fstr[i] = fmt.Sprintf("%s%s%s%snull", quotes, kf, quotes, sign)
|
||||
|
||||
1
go.mod
1
go.mod
@@ -11,4 +11,5 @@ require (
|
||||
github.com/sirupsen/logrus v1.8.1
|
||||
github.com/stretchr/testify v1.7.0
|
||||
google.golang.org/protobuf v1.23.0
|
||||
gopkg.in/yaml.v2 v2.3.0
|
||||
)
|
||||
|
||||
7
go.sum
7
go.sum
@@ -4,6 +4,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
|
||||
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
|
||||
github.com/Shopify/sarama v1.19.0 h1:9oksLxC6uxVPHPVYUmq6xhr1BOF/hHobWH2UzO67z1s=
|
||||
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
|
||||
github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc=
|
||||
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
|
||||
github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
|
||||
github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
|
||||
@@ -95,6 +96,7 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
|
||||
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
@@ -146,8 +148,10 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/libp2p/go-reuseport v0.0.2 h1:XSG94b1FJfGA01BUrT82imejHQyTxO4jEWqheyCXYvU=
|
||||
github.com/libp2p/go-reuseport v0.0.2/go.mod h1:SPD+5RwGC7rcnzngoYC86GjPzjSywuQyMVAheVBD9nQ=
|
||||
@@ -375,6 +379,7 @@ golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtn
|
||||
golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
@@ -404,6 +409,7 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
@@ -417,6 +423,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -148,6 +148,11 @@ type FlowMessage struct {
|
||||
MPLS3Label uint32 `protobuf:"varint,60,opt,name=MPLS3Label,proto3" json:"MPLS3Label,omitempty"` // Third Label
|
||||
MPLSLastTTL uint32 `protobuf:"varint,61,opt,name=MPLSLastTTL,proto3" json:"MPLSLastTTL,omitempty"` // Last TTL
|
||||
MPLSLastLabel uint32 `protobuf:"varint,62,opt,name=MPLSLastLabel,proto3" json:"MPLSLastLabel,omitempty"` // Last Label
|
||||
// Custom allocations
|
||||
CustomInteger1 uint64 `protobuf:"varint,1001,opt,name=CustomInteger1,proto3" json:"CustomInteger1,omitempty"`
|
||||
CustomInteger2 uint64 `protobuf:"varint,1002,opt,name=CustomInteger2,proto3" json:"CustomInteger2,omitempty"`
|
||||
CustomBytes1 []byte `protobuf:"bytes,1011,opt,name=CustomBytes1,proto3" json:"CustomBytes1,omitempty"`
|
||||
CustomBytes2 []byte `protobuf:"bytes,1012,opt,name=CustomBytes2,proto3" json:"CustomBytes2,omitempty"`
|
||||
}
|
||||
|
||||
func (x *FlowMessage) Reset() {
|
||||
@@ -539,11 +544,39 @@ func (x *FlowMessage) GetMPLSLastLabel() uint32 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *FlowMessage) GetCustomInteger1() uint64 {
|
||||
if x != nil {
|
||||
return x.CustomInteger1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *FlowMessage) GetCustomInteger2() uint64 {
|
||||
if x != nil {
|
||||
return x.CustomInteger2
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *FlowMessage) GetCustomBytes1() []byte {
|
||||
if x != nil {
|
||||
return x.CustomBytes1
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *FlowMessage) GetCustomBytes2() []byte {
|
||||
if x != nil {
|
||||
return x.CustomBytes2
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var File_pb_flow_proto protoreflect.FileDescriptor
|
||||
|
||||
var file_pb_flow_proto_rawDesc = []byte{
|
||||
0x0a, 0x0d, 0x70, 0x62, 0x2f, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12,
|
||||
0x06, 0x66, 0x6c, 0x6f, 0x77, 0x70, 0x62, 0x22, 0xd0, 0x0c, 0x0a, 0x0b, 0x46, 0x6c, 0x6f, 0x77,
|
||||
0x06, 0x66, 0x6c, 0x6f, 0x77, 0x70, 0x62, 0x22, 0xec, 0x0d, 0x0a, 0x0b, 0x46, 0x6c, 0x6f, 0x77,
|
||||
0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x30, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x18,
|
||||
0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1c, 0x2e, 0x66, 0x6c, 0x6f, 0x77, 0x70, 0x62, 0x2e, 0x46,
|
||||
0x6c, 0x6f, 0x77, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x46, 0x6c, 0x6f, 0x77, 0x54,
|
||||
@@ -639,15 +672,25 @@ var file_pb_flow_proto_rawDesc = []byte{
|
||||
0x28, 0x0d, 0x52, 0x0b, 0x4d, 0x50, 0x4c, 0x53, 0x4c, 0x61, 0x73, 0x74, 0x54, 0x54, 0x4c, 0x12,
|
||||
0x24, 0x0a, 0x0d, 0x4d, 0x50, 0x4c, 0x53, 0x4c, 0x61, 0x73, 0x74, 0x4c, 0x61, 0x62, 0x65, 0x6c,
|
||||
0x18, 0x3e, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, 0x4d, 0x50, 0x4c, 0x53, 0x4c, 0x61, 0x73, 0x74,
|
||||
0x4c, 0x61, 0x62, 0x65, 0x6c, 0x22, 0x53, 0x0a, 0x08, 0x46, 0x6c, 0x6f, 0x77, 0x54, 0x79, 0x70,
|
||||
0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x46, 0x4c, 0x4f, 0x57, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e,
|
||||
0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x46, 0x4c, 0x4f, 0x57, 0x5f, 0x35, 0x10, 0x01, 0x12,
|
||||
0x0e, 0x0a, 0x0a, 0x4e, 0x45, 0x54, 0x46, 0x4c, 0x4f, 0x57, 0x5f, 0x56, 0x35, 0x10, 0x02, 0x12,
|
||||
0x0e, 0x0a, 0x0a, 0x4e, 0x45, 0x54, 0x46, 0x4c, 0x4f, 0x57, 0x5f, 0x56, 0x39, 0x10, 0x03, 0x12,
|
||||
0x09, 0x0a, 0x05, 0x49, 0x50, 0x46, 0x49, 0x58, 0x10, 0x04, 0x42, 0x29, 0x5a, 0x27, 0x67, 0x69,
|
||||
0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x65, 0x74, 0x73, 0x61, 0x6d, 0x70,
|
||||
0x6c, 0x65, 0x72, 0x2f, 0x67, 0x6f, 0x66, 0x6c, 0x6f, 0x77, 0x32, 0x2f, 0x70, 0x62, 0x3b, 0x66,
|
||||
0x6c, 0x6f, 0x77, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
0x4c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x27, 0x0a, 0x0e, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x49,
|
||||
0x6e, 0x74, 0x65, 0x67, 0x65, 0x72, 0x31, 0x18, 0xe9, 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e,
|
||||
0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x49, 0x6e, 0x74, 0x65, 0x67, 0x65, 0x72, 0x31, 0x12, 0x27,
|
||||
0x0a, 0x0e, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x49, 0x6e, 0x74, 0x65, 0x67, 0x65, 0x72, 0x32,
|
||||
0x18, 0xea, 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x49,
|
||||
0x6e, 0x74, 0x65, 0x67, 0x65, 0x72, 0x32, 0x12, 0x23, 0x0a, 0x0c, 0x43, 0x75, 0x73, 0x74, 0x6f,
|
||||
0x6d, 0x42, 0x79, 0x74, 0x65, 0x73, 0x31, 0x18, 0xf3, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c,
|
||||
0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x42, 0x79, 0x74, 0x65, 0x73, 0x31, 0x12, 0x23, 0x0a, 0x0c,
|
||||
0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x42, 0x79, 0x74, 0x65, 0x73, 0x32, 0x18, 0xf4, 0x07, 0x20,
|
||||
0x01, 0x28, 0x0c, 0x52, 0x0c, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x42, 0x79, 0x74, 0x65, 0x73,
|
||||
0x32, 0x22, 0x53, 0x0a, 0x08, 0x46, 0x6c, 0x6f, 0x77, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0f, 0x0a,
|
||||
0x0b, 0x46, 0x4c, 0x4f, 0x57, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0b,
|
||||
0x0a, 0x07, 0x53, 0x46, 0x4c, 0x4f, 0x57, 0x5f, 0x35, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x4e,
|
||||
0x45, 0x54, 0x46, 0x4c, 0x4f, 0x57, 0x5f, 0x56, 0x35, 0x10, 0x02, 0x12, 0x0e, 0x0a, 0x0a, 0x4e,
|
||||
0x45, 0x54, 0x46, 0x4c, 0x4f, 0x57, 0x5f, 0x56, 0x39, 0x10, 0x03, 0x12, 0x09, 0x0a, 0x05, 0x49,
|
||||
0x50, 0x46, 0x49, 0x58, 0x10, 0x04, 0x42, 0x29, 0x5a, 0x27, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62,
|
||||
0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x65, 0x74, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x72, 0x2f,
|
||||
0x67, 0x6f, 0x66, 0x6c, 0x6f, 0x77, 0x32, 0x2f, 0x70, 0x62, 0x3b, 0x66, 0x6c, 0x6f, 0x77, 0x70,
|
||||
0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
@@ -101,4 +101,18 @@ message FlowMessage {
|
||||
// Custom fields: start after ID 1000:
|
||||
// uint32 MyCustomField = 1000;
|
||||
|
||||
|
||||
// Custom allocations
|
||||
uint64 CustomInteger1 = 1001;
|
||||
uint64 CustomInteger2 = 1002;
|
||||
uint64 CustomInteger3 = 1003;
|
||||
uint64 CustomInteger4 = 1004;
|
||||
uint64 CustomInteger5 = 1005;
|
||||
|
||||
bytes CustomBytes1 = 1011;
|
||||
bytes CustomBytes2 = 1012;
|
||||
bytes CustomBytes3 = 1013;
|
||||
bytes CustomBytes4 = 1014;
|
||||
bytes CustomBytes5 = 1015;
|
||||
|
||||
}
|
||||
|
||||
@@ -137,7 +137,45 @@ func DecodeUNumber(b []byte, out interface{}) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func ConvertNetFlowDataSet(version uint16, baseTime uint32, uptime uint32, record []netflow.DataField) *flowmessage.FlowMessage {
|
||||
func DecodeNumber(b []byte, out interface{}) error {
|
||||
var o int64
|
||||
l := len(b)
|
||||
switch l {
|
||||
case 1:
|
||||
o = int64(int8(b[0]))
|
||||
case 2:
|
||||
o = int64(int16(binary.BigEndian.Uint16(b)))
|
||||
case 4:
|
||||
o = int64(int32(binary.BigEndian.Uint32(b)))
|
||||
case 8:
|
||||
o = int64(binary.BigEndian.Uint64(b))
|
||||
default:
|
||||
if l < 8 {
|
||||
var iter int
|
||||
for i := range b {
|
||||
o |= int64(b[i]) << int(8*(int(l)-iter-1))
|
||||
iter++
|
||||
}
|
||||
} else {
|
||||
return errors.New(fmt.Sprintf("Non-regular number of bytes for a number: %v", l))
|
||||
}
|
||||
}
|
||||
switch t := out.(type) {
|
||||
case *int8:
|
||||
*t = int8(o)
|
||||
case *int16:
|
||||
*t = int16(o)
|
||||
case *int32:
|
||||
*t = int32(o)
|
||||
case *int64:
|
||||
*t = o
|
||||
default:
|
||||
return errors.New("The parameter is not a pointer to a int8/int16/int32/int64 structure")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ConvertNetFlowDataSet(version uint16, baseTime uint32, uptime uint32, record []netflow.DataField, mapperNetFlow *NetFlowMapper, mapperSFlow *SFlowMapper) *flowmessage.FlowMessage {
|
||||
flowMessage := &flowmessage.FlowMessage{}
|
||||
var time uint64
|
||||
|
||||
@@ -155,6 +193,12 @@ func ConvertNetFlowDataSet(version uint16, baseTime uint32, uptime uint32, recor
|
||||
continue
|
||||
}
|
||||
|
||||
MapCustomNetFlow(flowMessage, df, mapperNetFlow)
|
||||
|
||||
if df.PenProvided {
|
||||
continue
|
||||
}
|
||||
|
||||
switch df.Type {
|
||||
|
||||
// Statistics
|
||||
@@ -338,6 +382,16 @@ func ConvertNetFlowDataSet(version uint16, baseTime uint32, uptime uint32, recor
|
||||
case netflow.IPFIX_FIELD_flowEndDeltaMicroseconds:
|
||||
DecodeUNumber(v, &time)
|
||||
flowMessage.TimeFlowEnd = uint64(baseTime) - time/1000000
|
||||
// RFC7133
|
||||
case netflow.IPFIX_FIELD_dataLinkFrameSize:
|
||||
DecodeUNumber(v, &(flowMessage.Bytes))
|
||||
flowMessage.Packets = 1
|
||||
case netflow.IPFIX_FIELD_dataLinkFrameSection:
|
||||
ParseEthernetHeader(flowMessage, v, mapperSFlow)
|
||||
flowMessage.Packets = 1
|
||||
if flowMessage.Bytes == 0 {
|
||||
flowMessage.Bytes = uint64(len(v))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -347,10 +401,10 @@ func ConvertNetFlowDataSet(version uint16, baseTime uint32, uptime uint32, recor
|
||||
return flowMessage
|
||||
}
|
||||
|
||||
func SearchNetFlowDataSetsRecords(version uint16, baseTime uint32, uptime uint32, dataRecords []netflow.DataRecord) []*flowmessage.FlowMessage {
|
||||
func SearchNetFlowDataSetsRecords(version uint16, baseTime uint32, uptime uint32, dataRecords []netflow.DataRecord, mapperNetFlow *NetFlowMapper, mapperSFlow *SFlowMapper) []*flowmessage.FlowMessage {
|
||||
flowMessageSet := make([]*flowmessage.FlowMessage, 0)
|
||||
for _, record := range dataRecords {
|
||||
fmsg := ConvertNetFlowDataSet(version, baseTime, uptime, record.Values)
|
||||
fmsg := ConvertNetFlowDataSet(version, baseTime, uptime, record.Values, mapperNetFlow, mapperSFlow)
|
||||
if fmsg != nil {
|
||||
flowMessageSet = append(flowMessageSet, fmsg)
|
||||
}
|
||||
@@ -358,10 +412,10 @@ func SearchNetFlowDataSetsRecords(version uint16, baseTime uint32, uptime uint32
|
||||
return flowMessageSet
|
||||
}
|
||||
|
||||
func SearchNetFlowDataSets(version uint16, baseTime uint32, uptime uint32, dataFlowSet []netflow.DataFlowSet) []*flowmessage.FlowMessage {
|
||||
func SearchNetFlowDataSets(version uint16, baseTime uint32, uptime uint32, dataFlowSet []netflow.DataFlowSet, mapperNetFlow *NetFlowMapper, mapperSFlow *SFlowMapper) []*flowmessage.FlowMessage {
|
||||
flowMessageSet := make([]*flowmessage.FlowMessage, 0)
|
||||
for _, dataFlowSetItem := range dataFlowSet {
|
||||
fmsg := SearchNetFlowDataSetsRecords(version, baseTime, uptime, dataFlowSetItem.Records)
|
||||
fmsg := SearchNetFlowDataSetsRecords(version, baseTime, uptime, dataFlowSetItem.Records, mapperNetFlow, mapperSFlow)
|
||||
if fmsg != nil {
|
||||
flowMessageSet = append(flowMessageSet, fmsg...)
|
||||
}
|
||||
@@ -431,9 +485,13 @@ func SplitIPFIXSets(packetIPFIX netflow.IPFIXPacket) ([]netflow.DataFlowSet, []n
|
||||
return dataFlowSet, templatesFlowSet, optionsTemplatesFlowSet, optionsDataFlowSet
|
||||
}
|
||||
|
||||
func ProcessMessageNetFlow(msgDec interface{}, samplingRateSys SamplingRateSystem) ([]*flowmessage.FlowMessage, error) {
|
||||
return ProcessMessageNetFlowConfig(msgDec, samplingRateSys, nil)
|
||||
}
|
||||
|
||||
// Convert a NetFlow datastructure to a FlowMessage protobuf
|
||||
// Does not put sampling rate
|
||||
func ProcessMessageNetFlow(msgDec interface{}, samplingRateSys SamplingRateSystem) ([]*flowmessage.FlowMessage, error) {
|
||||
func ProcessMessageNetFlowConfig(msgDec interface{}, samplingRateSys SamplingRateSystem, config *ProducerConfigMapped) ([]*flowmessage.FlowMessage, error) {
|
||||
seqnum := uint32(0)
|
||||
var baseTime uint32
|
||||
var uptime uint32
|
||||
@@ -449,7 +507,11 @@ func ProcessMessageNetFlow(msgDec interface{}, samplingRateSys SamplingRateSyste
|
||||
uptime = msgDecConv.SystemUptime
|
||||
obsDomainId := msgDecConv.SourceId
|
||||
|
||||
flowMessageSet = SearchNetFlowDataSets(9, baseTime, uptime, dataFlowSet)
|
||||
var cfg *NetFlowMapper
|
||||
if config != nil {
|
||||
cfg = config.NetFlowV9
|
||||
}
|
||||
flowMessageSet = SearchNetFlowDataSets(9, baseTime, uptime, dataFlowSet, cfg, nil)
|
||||
samplingRate, found := SearchNetFlowOptionDataSets(optionDataFlowSet)
|
||||
if samplingRateSys != nil {
|
||||
if found {
|
||||
@@ -469,7 +531,13 @@ func ProcessMessageNetFlow(msgDec interface{}, samplingRateSys SamplingRateSyste
|
||||
baseTime = msgDecConv.ExportTime
|
||||
obsDomainId := msgDecConv.ObservationDomainId
|
||||
|
||||
flowMessageSet = SearchNetFlowDataSets(10, baseTime, uptime, dataFlowSet)
|
||||
var cfgIpfix *NetFlowMapper
|
||||
var cfgSflow *SFlowMapper
|
||||
if config != nil {
|
||||
cfgIpfix = config.IPFIX
|
||||
cfgSflow = config.SFlow
|
||||
}
|
||||
flowMessageSet = SearchNetFlowDataSets(10, baseTime, uptime, dataFlowSet, cfgIpfix, cfgSflow)
|
||||
|
||||
samplingRate, found := SearchNetFlowOptionDataSets(optionDataFlowSet)
|
||||
if samplingRateSys != nil {
|
||||
|
||||
@@ -22,182 +22,211 @@ func GetSFlowFlowSamples(packet *sflow.Packet) []interface{} {
|
||||
return flowSamples
|
||||
}
|
||||
|
||||
type SFlowProducerConfig struct {
|
||||
}
|
||||
|
||||
func ParseSampledHeader(flowMessage *flowmessage.FlowMessage, sampledHeader *sflow.SampledHeader) error {
|
||||
return ParseSampledHeaderConfig(flowMessage, sampledHeader, nil)
|
||||
}
|
||||
|
||||
func ParseSampledHeaderConfig(flowMessage *flowmessage.FlowMessage, sampledHeader *sflow.SampledHeader, config *SFlowProducerConfig) error {
|
||||
func ParseEthernetHeader(flowMessage *flowmessage.FlowMessage, data []byte, config *SFlowMapper) {
|
||||
var hasMPLS bool
|
||||
var countMpls uint32
|
||||
var firstLabelMpls uint32
|
||||
var firstTtlMpls uint8
|
||||
var secondLabelMpls uint32
|
||||
var secondTtlMpls uint8
|
||||
var thirdLabelMpls uint32
|
||||
var thirdTtlMpls uint8
|
||||
var lastLabelMpls uint32
|
||||
var lastTtlMpls uint8
|
||||
|
||||
var nextHeader byte
|
||||
var tcpflags byte
|
||||
srcIP := net.IP{}
|
||||
dstIP := net.IP{}
|
||||
offset := 14
|
||||
|
||||
var srcMac uint64
|
||||
var dstMac uint64
|
||||
|
||||
var tos byte
|
||||
var ttl byte
|
||||
var identification uint16
|
||||
var fragOffset uint16
|
||||
var flowLabel uint32
|
||||
|
||||
var srcPort uint16
|
||||
var dstPort uint16
|
||||
|
||||
for _, configLayer := range GetSFlowConfigLayer(config, 0) {
|
||||
extracted := GetBytes(data, configLayer.Offset, configLayer.Length)
|
||||
MapCustom(flowMessage, extracted, configLayer.Destination)
|
||||
}
|
||||
|
||||
etherType := data[12:14]
|
||||
|
||||
dstMac = binary.BigEndian.Uint64(append([]byte{0, 0}, data[0:6]...))
|
||||
srcMac = binary.BigEndian.Uint64(append([]byte{0, 0}, data[6:12]...))
|
||||
(*flowMessage).SrcMac = srcMac
|
||||
(*flowMessage).DstMac = dstMac
|
||||
|
||||
encap := true
|
||||
iterations := 0
|
||||
for encap && iterations <= 1 {
|
||||
encap = false
|
||||
|
||||
if etherType[0] == 0x81 && etherType[1] == 0x0 { // VLAN 802.1Q
|
||||
(*flowMessage).VlanId = uint32(binary.BigEndian.Uint16(data[14:16]))
|
||||
offset += 4
|
||||
etherType = data[16:18]
|
||||
}
|
||||
|
||||
if etherType[0] == 0x88 && etherType[1] == 0x47 { // MPLS
|
||||
iterateMpls := true
|
||||
hasMPLS = true
|
||||
for iterateMpls {
|
||||
if len(data) < offset+5 {
|
||||
iterateMpls = false
|
||||
break
|
||||
}
|
||||
label := binary.BigEndian.Uint32(append([]byte{0}, data[offset:offset+3]...)) >> 4
|
||||
//exp := data[offset+2] > 1
|
||||
bottom := data[offset+2] & 1
|
||||
mplsTtl := data[offset+3]
|
||||
offset += 4
|
||||
|
||||
if bottom == 1 || label <= 15 || offset > len(data) {
|
||||
if data[offset]&0xf0>>4 == 4 {
|
||||
etherType = []byte{0x8, 0x0}
|
||||
} else if data[offset]&0xf0>>4 == 6 {
|
||||
etherType = []byte{0x86, 0xdd}
|
||||
}
|
||||
iterateMpls = false
|
||||
}
|
||||
|
||||
if countMpls == 0 {
|
||||
firstLabelMpls = label
|
||||
firstTtlMpls = mplsTtl
|
||||
} else if countMpls == 1 {
|
||||
secondLabelMpls = label
|
||||
secondTtlMpls = mplsTtl
|
||||
} else if countMpls == 2 {
|
||||
thirdLabelMpls = label
|
||||
thirdTtlMpls = mplsTtl
|
||||
} else {
|
||||
lastLabelMpls = label
|
||||
lastTtlMpls = mplsTtl
|
||||
}
|
||||
countMpls++
|
||||
}
|
||||
}
|
||||
|
||||
for _, configLayer := range GetSFlowConfigLayer(config, 3) {
|
||||
extracted := GetBytes(data, offset*8+configLayer.Offset, configLayer.Length)
|
||||
MapCustom(flowMessage, extracted, configLayer.Destination)
|
||||
}
|
||||
|
||||
if etherType[0] == 0x8 && etherType[1] == 0x0 { // IPv4
|
||||
if len(data) >= offset+20 {
|
||||
nextHeader = data[offset+9]
|
||||
srcIP = data[offset+12 : offset+16]
|
||||
dstIP = data[offset+16 : offset+20]
|
||||
tos = data[offset+1]
|
||||
ttl = data[offset+8]
|
||||
|
||||
identification = binary.BigEndian.Uint16(data[offset+4 : offset+6])
|
||||
fragOffset = binary.BigEndian.Uint16(data[offset+6 : offset+8])
|
||||
|
||||
offset += 20
|
||||
}
|
||||
} else if etherType[0] == 0x86 && etherType[1] == 0xdd { // IPv6
|
||||
if len(data) >= offset+40 {
|
||||
nextHeader = data[offset+6]
|
||||
srcIP = data[offset+8 : offset+24]
|
||||
dstIP = data[offset+24 : offset+40]
|
||||
|
||||
tostmp := uint32(binary.BigEndian.Uint16(data[offset : offset+2]))
|
||||
tos = uint8(tostmp & 0x0ff0 >> 4)
|
||||
ttl = data[offset+7]
|
||||
|
||||
flowLabel = binary.BigEndian.Uint32(data[offset : offset+4])
|
||||
|
||||
offset += 40
|
||||
|
||||
}
|
||||
} else if etherType[0] == 0x8 && etherType[1] == 0x6 { // ARP
|
||||
} /*else {
|
||||
return errors.New(fmt.Sprintf("Unknown EtherType: %v\n", etherType))
|
||||
} */
|
||||
|
||||
for _, configLayer := range GetSFlowConfigLayer(config, 4) {
|
||||
extracted := GetBytes(data, offset*8+configLayer.Offset, configLayer.Length)
|
||||
MapCustom(flowMessage, extracted, configLayer.Destination)
|
||||
}
|
||||
|
||||
appOffset := 0
|
||||
if len(data) >= offset+4 && (nextHeader == 17 || nextHeader == 6) {
|
||||
srcPort = binary.BigEndian.Uint16(data[offset+0 : offset+2])
|
||||
dstPort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
|
||||
}
|
||||
|
||||
if nextHeader == 17 {
|
||||
appOffset = 8
|
||||
}
|
||||
|
||||
if len(data) >= offset+13 && nextHeader == 6 {
|
||||
tcpflags = data[offset+13]
|
||||
|
||||
appOffset = int(data[13]>>4) * 4
|
||||
}
|
||||
|
||||
// ICMP and ICMPv6
|
||||
if len(data) >= offset+2 && (nextHeader == 1 || nextHeader == 58) {
|
||||
(*flowMessage).IcmpType = uint32(data[offset+0])
|
||||
(*flowMessage).IcmpCode = uint32(data[offset+1])
|
||||
}
|
||||
|
||||
if appOffset > 0 {
|
||||
for _, configLayer := range GetSFlowConfigLayer(config, 7) {
|
||||
extracted := GetBytes(data, (offset+appOffset)*8+configLayer.Offset, configLayer.Length)
|
||||
MapCustom(flowMessage, extracted, configLayer.Destination)
|
||||
}
|
||||
}
|
||||
|
||||
iterations++
|
||||
}
|
||||
|
||||
(*flowMessage).HasMPLS = hasMPLS
|
||||
(*flowMessage).MPLSCount = countMpls
|
||||
(*flowMessage).MPLS1Label = firstLabelMpls
|
||||
(*flowMessage).MPLS1TTL = uint32(firstTtlMpls)
|
||||
(*flowMessage).MPLS2Label = secondLabelMpls
|
||||
(*flowMessage).MPLS2TTL = uint32(secondTtlMpls)
|
||||
(*flowMessage).MPLS3Label = thirdLabelMpls
|
||||
(*flowMessage).MPLS3TTL = uint32(thirdTtlMpls)
|
||||
(*flowMessage).MPLSLastLabel = lastLabelMpls
|
||||
(*flowMessage).MPLSLastTTL = uint32(lastTtlMpls)
|
||||
|
||||
(*flowMessage).Etype = uint32(binary.BigEndian.Uint16(etherType[0:2]))
|
||||
(*flowMessage).IPv6FlowLabel = flowLabel & 0xFFFFF
|
||||
|
||||
(*flowMessage).SrcPort = uint32(srcPort)
|
||||
(*flowMessage).DstPort = uint32(dstPort)
|
||||
|
||||
(*flowMessage).SrcAddr = srcIP
|
||||
(*flowMessage).DstAddr = dstIP
|
||||
(*flowMessage).Proto = uint32(nextHeader)
|
||||
(*flowMessage).IPTos = uint32(tos)
|
||||
(*flowMessage).IPTTL = uint32(ttl)
|
||||
(*flowMessage).TCPFlags = uint32(tcpflags)
|
||||
|
||||
(*flowMessage).FragmentId = uint32(identification)
|
||||
(*flowMessage).FragmentOffset = uint32(fragOffset)
|
||||
}
|
||||
|
||||
func ParseSampledHeaderConfig(flowMessage *flowmessage.FlowMessage, sampledHeader *sflow.SampledHeader, config *SFlowMapper) error {
|
||||
data := (*sampledHeader).HeaderData
|
||||
switch (*sampledHeader).Protocol {
|
||||
case 1: // Ethernet
|
||||
var hasMPLS bool
|
||||
var countMpls uint32
|
||||
var firstLabelMpls uint32
|
||||
var firstTtlMpls uint8
|
||||
var secondLabelMpls uint32
|
||||
var secondTtlMpls uint8
|
||||
var thirdLabelMpls uint32
|
||||
var thirdTtlMpls uint8
|
||||
var lastLabelMpls uint32
|
||||
var lastTtlMpls uint8
|
||||
|
||||
var nextHeader byte
|
||||
var tcpflags byte
|
||||
srcIP := net.IP{}
|
||||
dstIP := net.IP{}
|
||||
offset := 14
|
||||
|
||||
var srcMac uint64
|
||||
var dstMac uint64
|
||||
|
||||
var tos byte
|
||||
var ttl byte
|
||||
var identification uint16
|
||||
var fragOffset uint16
|
||||
var flowLabel uint32
|
||||
|
||||
var srcPort uint16
|
||||
var dstPort uint16
|
||||
|
||||
etherType := data[12:14]
|
||||
|
||||
dstMac = binary.BigEndian.Uint64(append([]byte{0, 0}, data[0:6]...))
|
||||
srcMac = binary.BigEndian.Uint64(append([]byte{0, 0}, data[6:12]...))
|
||||
(*flowMessage).SrcMac = srcMac
|
||||
(*flowMessage).DstMac = dstMac
|
||||
|
||||
encap := true
|
||||
iterations := 0
|
||||
for encap && iterations <= 1 {
|
||||
encap = false
|
||||
|
||||
if etherType[0] == 0x81 && etherType[1] == 0x0 { // VLAN 802.1Q
|
||||
(*flowMessage).VlanId = uint32(binary.BigEndian.Uint16(data[14:16]))
|
||||
offset += 4
|
||||
etherType = data[16:18]
|
||||
}
|
||||
|
||||
if etherType[0] == 0x88 && etherType[1] == 0x47 { // MPLS
|
||||
iterateMpls := true
|
||||
hasMPLS = true
|
||||
for iterateMpls {
|
||||
if len(data) < offset+5 {
|
||||
iterateMpls = false
|
||||
break
|
||||
}
|
||||
label := binary.BigEndian.Uint32(append([]byte{0}, data[offset:offset+3]...)) >> 4
|
||||
//exp := data[offset+2] > 1
|
||||
bottom := data[offset+2] & 1
|
||||
mplsTtl := data[offset+3]
|
||||
offset += 4
|
||||
|
||||
if bottom == 1 || label <= 15 || offset > len(data) {
|
||||
if data[offset]&0xf0>>4 == 4 {
|
||||
etherType = []byte{0x8, 0x0}
|
||||
} else if data[offset]&0xf0>>4 == 6 {
|
||||
etherType = []byte{0x86, 0xdd}
|
||||
}
|
||||
iterateMpls = false
|
||||
}
|
||||
|
||||
if countMpls == 0 {
|
||||
firstLabelMpls = label
|
||||
firstTtlMpls = mplsTtl
|
||||
} else if countMpls == 1 {
|
||||
secondLabelMpls = label
|
||||
secondTtlMpls = mplsTtl
|
||||
} else if countMpls == 2 {
|
||||
thirdLabelMpls = label
|
||||
thirdTtlMpls = mplsTtl
|
||||
} else {
|
||||
lastLabelMpls = label
|
||||
lastTtlMpls = mplsTtl
|
||||
}
|
||||
countMpls++
|
||||
}
|
||||
}
|
||||
|
||||
if etherType[0] == 0x8 && etherType[1] == 0x0 { // IPv4
|
||||
if len(data) >= offset+20 {
|
||||
nextHeader = data[offset+9]
|
||||
srcIP = data[offset+12 : offset+16]
|
||||
dstIP = data[offset+16 : offset+20]
|
||||
tos = data[offset+1]
|
||||
ttl = data[offset+8]
|
||||
|
||||
identification = binary.BigEndian.Uint16(data[offset+4 : offset+6])
|
||||
fragOffset = binary.BigEndian.Uint16(data[offset+6 : offset+8])
|
||||
|
||||
offset += 20
|
||||
}
|
||||
} else if etherType[0] == 0x86 && etherType[1] == 0xdd { // IPv6
|
||||
if len(data) >= offset+40 {
|
||||
nextHeader = data[offset+6]
|
||||
srcIP = data[offset+8 : offset+24]
|
||||
dstIP = data[offset+24 : offset+40]
|
||||
|
||||
tostmp := uint32(binary.BigEndian.Uint16(data[offset : offset+2]))
|
||||
tos = uint8(tostmp & 0x0ff0 >> 4)
|
||||
ttl = data[offset+7]
|
||||
|
||||
flowLabel = binary.BigEndian.Uint32(data[offset : offset+4])
|
||||
|
||||
offset += 40
|
||||
|
||||
}
|
||||
} else if etherType[0] == 0x8 && etherType[1] == 0x6 { // ARP
|
||||
} /*else {
|
||||
return errors.New(fmt.Sprintf("Unknown EtherType: %v\n", etherType))
|
||||
} */
|
||||
|
||||
if len(data) >= offset+4 && (nextHeader == 17 || nextHeader == 6) {
|
||||
srcPort = binary.BigEndian.Uint16(data[offset+0 : offset+2])
|
||||
dstPort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
|
||||
}
|
||||
|
||||
if len(data) >= offset+13 && nextHeader == 6 {
|
||||
tcpflags = data[offset+13]
|
||||
}
|
||||
|
||||
// ICMP and ICMPv6
|
||||
if len(data) >= offset+2 && (nextHeader == 1 || nextHeader == 58) {
|
||||
(*flowMessage).IcmpType = uint32(data[offset+0])
|
||||
(*flowMessage).IcmpCode = uint32(data[offset+1])
|
||||
}
|
||||
|
||||
iterations++
|
||||
}
|
||||
|
||||
(*flowMessage).HasMPLS = hasMPLS
|
||||
(*flowMessage).MPLSCount = countMpls
|
||||
(*flowMessage).MPLS1Label = firstLabelMpls
|
||||
(*flowMessage).MPLS1TTL = uint32(firstTtlMpls)
|
||||
(*flowMessage).MPLS2Label = secondLabelMpls
|
||||
(*flowMessage).MPLS2TTL = uint32(secondTtlMpls)
|
||||
(*flowMessage).MPLS3Label = thirdLabelMpls
|
||||
(*flowMessage).MPLS3TTL = uint32(thirdTtlMpls)
|
||||
(*flowMessage).MPLSLastLabel = lastLabelMpls
|
||||
(*flowMessage).MPLSLastTTL = uint32(lastTtlMpls)
|
||||
|
||||
(*flowMessage).Etype = uint32(binary.BigEndian.Uint16(etherType[0:2]))
|
||||
(*flowMessage).IPv6FlowLabel = flowLabel & 0xFFFFF
|
||||
|
||||
(*flowMessage).SrcPort = uint32(srcPort)
|
||||
(*flowMessage).DstPort = uint32(dstPort)
|
||||
|
||||
(*flowMessage).SrcAddr = srcIP
|
||||
(*flowMessage).DstAddr = dstIP
|
||||
(*flowMessage).Proto = uint32(nextHeader)
|
||||
(*flowMessage).IPTos = uint32(tos)
|
||||
(*flowMessage).IPTTL = uint32(ttl)
|
||||
(*flowMessage).TCPFlags = uint32(tcpflags)
|
||||
|
||||
(*flowMessage).FragmentId = uint32(identification)
|
||||
(*flowMessage).FragmentOffset = uint32(fragOffset)
|
||||
ParseEthernetHeader(flowMessage, data, config)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -206,7 +235,7 @@ func SearchSFlowSamples(samples []interface{}) []*flowmessage.FlowMessage {
|
||||
return SearchSFlowSamples(samples)
|
||||
}
|
||||
|
||||
func SearchSFlowSamplesConfig(samples []interface{}, config *SFlowProducerConfig) []*flowmessage.FlowMessage {
|
||||
func SearchSFlowSamplesConfig(samples []interface{}, config *SFlowMapper) []*flowmessage.FlowMessage {
|
||||
flowMessageSet := make([]*flowmessage.FlowMessage, 0)
|
||||
|
||||
for _, flowSample := range samples {
|
||||
@@ -289,15 +318,20 @@ func ProcessMessageSFlow(msgDec interface{}) ([]*flowmessage.FlowMessage, error)
|
||||
return ProcessMessageSFlowConfig(msgDec, nil)
|
||||
}
|
||||
|
||||
func ProcessMessageSFlowConfig(msgDec interface{}, config *SFlowProducerConfig) ([]*flowmessage.FlowMessage, error) {
|
||||
func ProcessMessageSFlowConfig(msgDec interface{}, config *ProducerConfigMapped) ([]*flowmessage.FlowMessage, error) {
|
||||
switch packet := msgDec.(type) {
|
||||
case sflow.Packet:
|
||||
seqnum := packet.SequenceNumber
|
||||
var agent net.IP
|
||||
agent = packet.AgentIP
|
||||
|
||||
var cfg *SFlowMapper
|
||||
if config != nil {
|
||||
cfg = config.SFlow
|
||||
}
|
||||
|
||||
flowSamples := GetSFlowFlowSamples(&packet)
|
||||
flowMessageSet := SearchSFlowSamplesConfig(flowSamples, config)
|
||||
flowMessageSet := SearchSFlowSamplesConfig(flowSamples, cfg)
|
||||
for _, fmsg := range flowMessageSet {
|
||||
fmsg.SamplerAddress = agent
|
||||
fmsg.SequenceNum = seqnum
|
||||
|
||||
185
producer/reflect.go
Normal file
185
producer/reflect.go
Normal file
@@ -0,0 +1,185 @@
|
||||
package producer
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
|
||||
"github.com/netsampler/goflow2/decoders/netflow"
|
||||
flowmessage "github.com/netsampler/goflow2/pb"
|
||||
)
|
||||
|
||||
func GetBytes(d []byte, offset int, length int) []byte {
|
||||
if length == 0 {
|
||||
return nil
|
||||
}
|
||||
leftBytes := offset / 8
|
||||
rightBytes := (offset + length) / 8
|
||||
if (offset+length)%8 != 0 {
|
||||
rightBytes += 1
|
||||
}
|
||||
if leftBytes >= len(d) {
|
||||
return nil
|
||||
}
|
||||
if rightBytes > len(d) {
|
||||
rightBytes = len(d)
|
||||
}
|
||||
chunk := make([]byte, rightBytes-leftBytes)
|
||||
|
||||
offsetMod8 := (offset % 8)
|
||||
shiftAnd := byte(0xff >> (8 - offsetMod8))
|
||||
|
||||
var shifted byte
|
||||
for i := range chunk {
|
||||
j := len(chunk) - 1 - i
|
||||
cur := d[j+leftBytes]
|
||||
chunk[j] = (cur << offsetMod8) | shifted
|
||||
shifted = shiftAnd & cur
|
||||
}
|
||||
last := len(chunk) - 1
|
||||
shiftAndLast := byte(0xff << ((8 - ((offset + length) % 8)) % 8))
|
||||
chunk[last] = chunk[last] & shiftAndLast
|
||||
return chunk
|
||||
}
|
||||
|
||||
func MapCustomNetFlow(flowMessage *flowmessage.FlowMessage, df netflow.DataField, mapper *NetFlowMapper) {
|
||||
if mapper == nil {
|
||||
return
|
||||
}
|
||||
mapped, ok := mapper.Map(df)
|
||||
if ok {
|
||||
v := df.Value.([]byte)
|
||||
MapCustom(flowMessage, v, mapped.Destination)
|
||||
}
|
||||
}
|
||||
|
||||
func MapCustom(flowMessage *flowmessage.FlowMessage, v []byte, destination string) {
|
||||
vfm := reflect.ValueOf(flowMessage)
|
||||
vfm = reflect.Indirect(vfm)
|
||||
|
||||
fieldValue := vfm.FieldByName(destination)
|
||||
|
||||
if fieldValue.IsValid() {
|
||||
typeDest := fieldValue.Type()
|
||||
fieldValueAddr := fieldValue.Addr()
|
||||
|
||||
if typeDest.Kind() == reflect.Slice && typeDest.Elem().Kind() == reflect.Uint8 {
|
||||
fieldValue.SetBytes(v)
|
||||
} else if fieldValueAddr.IsValid() && (typeDest.Kind() == reflect.Uint8 || typeDest.Kind() == reflect.Uint16 || typeDest.Kind() == reflect.Uint32 || typeDest.Kind() == reflect.Uint64) {
|
||||
DecodeUNumber(v, fieldValueAddr.Interface())
|
||||
} else if fieldValueAddr.IsValid() && (typeDest.Kind() == reflect.Int8 || typeDest.Kind() == reflect.Int16 || typeDest.Kind() == reflect.Int32 || typeDest.Kind() == reflect.Int64) {
|
||||
DecodeNumber(v, fieldValueAddr.Interface())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type NetFlowMapField struct {
|
||||
PenProvided bool `json:"penprovided"`
|
||||
Type uint16 `json:"field"`
|
||||
Pen uint32 `json:"pen"`
|
||||
|
||||
Destination string `json:"destination"`
|
||||
//DestinationLength uint8 `json:"dlen"` // could be used if populating a slice of uint16 that aren't in protobuf
|
||||
}
|
||||
|
||||
type IPFIXProducerConfig struct {
|
||||
Mapping []NetFlowMapField `json:"mapping"`
|
||||
//PacketMapping []SFlowMapField `json:"packet-mapping"` // for embedded frames: use sFlow configuration
|
||||
}
|
||||
|
||||
type NetFlowV9ProducerConfig struct {
|
||||
Mapping []NetFlowMapField `json:"mapping"`
|
||||
}
|
||||
|
||||
type SFlowMapField struct {
|
||||
Layer int `json:"layer"`
|
||||
Offset int `json:"offset"` // offset in bits
|
||||
Length int `json:"length"` // length in bits
|
||||
|
||||
Destination string `json:"destination"`
|
||||
//DestinationLength uint8 `json:"dlen"`
|
||||
}
|
||||
|
||||
type SFlowProducerConfig struct {
|
||||
Mapping []SFlowMapField `json:"mapping"`
|
||||
}
|
||||
|
||||
type ProducerConfig struct {
|
||||
IPFIX IPFIXProducerConfig `json:"ipfix"`
|
||||
NetFlowV9 NetFlowV9ProducerConfig `json:"netflowv9"`
|
||||
SFlow SFlowProducerConfig `json:"sflow"` // also used for IPFIX data frames
|
||||
|
||||
// should do a rename map list for when printing
|
||||
}
|
||||
|
||||
type DataMap struct {
|
||||
Destination string
|
||||
}
|
||||
|
||||
type NetFlowMapper struct {
|
||||
data map[string]DataMap // maps field to destination
|
||||
}
|
||||
|
||||
func (m *NetFlowMapper) Map(field netflow.DataField) (DataMap, bool) {
|
||||
mapped, found := m.data[fmt.Sprintf("%v-%d-%d", field.PenProvided, field.Pen, field.Type)]
|
||||
return mapped, found
|
||||
}
|
||||
|
||||
func MapFieldsNetFlow(fields []NetFlowMapField) *NetFlowMapper {
|
||||
ret := make(map[string]DataMap)
|
||||
for _, field := range fields {
|
||||
ret[fmt.Sprintf("%v-%d-%d", field.PenProvided, field.Pen, field.Type)] = DataMap{Destination: field.Destination}
|
||||
}
|
||||
return &NetFlowMapper{ret}
|
||||
}
|
||||
|
||||
type DataMapLayer struct {
|
||||
Offset int
|
||||
Length int
|
||||
Destination string
|
||||
}
|
||||
|
||||
type SFlowMapper struct {
|
||||
data map[int][]DataMapLayer // map layer to list of offsets
|
||||
}
|
||||
|
||||
func GetSFlowConfigLayer(m *SFlowMapper, layer int) []DataMapLayer {
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
return m.data[layer]
|
||||
}
|
||||
|
||||
func MapFieldsSFlow(fields []SFlowMapField) *SFlowMapper {
|
||||
ret := make(map[int][]DataMapLayer)
|
||||
for _, field := range fields {
|
||||
retLayerEntry := DataMapLayer{
|
||||
Offset: field.Offset,
|
||||
Length: field.Length,
|
||||
Destination: field.Destination,
|
||||
}
|
||||
retLayer, ok := ret[field.Layer]
|
||||
if !ok {
|
||||
retLayer = make([]DataMapLayer, 0)
|
||||
}
|
||||
retLayer = append(retLayer, retLayerEntry)
|
||||
ret[field.Layer] = retLayer
|
||||
|
||||
}
|
||||
return &SFlowMapper{ret}
|
||||
}
|
||||
|
||||
type ProducerConfigMapped struct {
|
||||
IPFIX *NetFlowMapper `json:"ipfix"`
|
||||
NetFlowV9 *NetFlowMapper `json:"netflowv9"`
|
||||
SFlow *SFlowMapper `json:"sflow"`
|
||||
}
|
||||
|
||||
func NewProducerConfigMapped(config *ProducerConfig) *ProducerConfigMapped {
|
||||
newCfg := &ProducerConfigMapped{}
|
||||
if config != nil {
|
||||
newCfg.IPFIX = MapFieldsNetFlow(config.IPFIX.Mapping)
|
||||
newCfg.NetFlowV9 = MapFieldsNetFlow(config.NetFlowV9.Mapping)
|
||||
newCfg.SFlow = MapFieldsSFlow(config.SFlow.Mapping)
|
||||
}
|
||||
return newCfg
|
||||
}
|
||||
@@ -59,6 +59,9 @@ type StateNetFlow struct {
|
||||
|
||||
samplinglock *sync.RWMutex
|
||||
sampling map[string]producer.SamplingRateSystem
|
||||
|
||||
Config *producer.ProducerConfig
|
||||
configMapped *producer.ProducerConfigMapped
|
||||
}
|
||||
|
||||
func (s *StateNetFlow) DecodeFlow(msg interface{}) error {
|
||||
@@ -215,7 +218,7 @@ func (s *StateNetFlow) DecodeFlow(msg interface{}) error {
|
||||
Add(float64(len(fsConv.Records)))
|
||||
}
|
||||
}
|
||||
flowMessageSet, err = producer.ProcessMessageNetFlow(msgDecConv, sampling)
|
||||
flowMessageSet, err = producer.ProcessMessageNetFlowConfig(msgDecConv, sampling, s.configMapped)
|
||||
|
||||
for _, fmsg := range flowMessageSet {
|
||||
fmsg.TimeReceived = ts
|
||||
@@ -308,7 +311,7 @@ func (s *StateNetFlow) DecodeFlow(msg interface{}) error {
|
||||
Add(float64(len(fsConv.Records)))
|
||||
}
|
||||
}
|
||||
flowMessageSet, err = producer.ProcessMessageNetFlow(msgDecConv, sampling)
|
||||
flowMessageSet, err = producer.ProcessMessageNetFlowConfig(msgDecConv, sampling, s.configMapped)
|
||||
|
||||
for _, fmsg := range flowMessageSet {
|
||||
fmsg.TimeReceived = ts
|
||||
@@ -365,7 +368,12 @@ func (s *StateNetFlow) InitTemplates() {
|
||||
s.samplinglock = &sync.RWMutex{}
|
||||
}
|
||||
|
||||
func (s *StateNetFlow) initConfig() {
|
||||
s.configMapped = producer.NewProducerConfigMapped(s.Config)
|
||||
}
|
||||
|
||||
func (s *StateNetFlow) FlowRoutine(workers int, addr string, port int, reuseport bool) error {
|
||||
s.InitTemplates()
|
||||
s.initConfig()
|
||||
return UDPRoutine("NetFlow", s.DecodeFlow, workers, addr, port, reuseport, s.Logger)
|
||||
}
|
||||
|
||||
@@ -18,7 +18,8 @@ type StateSFlow struct {
|
||||
Transport transport.TransportInterface
|
||||
Logger Logger
|
||||
|
||||
Config *producer.SFlowProducerConfig
|
||||
Config *producer.ProducerConfig
|
||||
configMapped *producer.ProducerConfigMapped
|
||||
}
|
||||
|
||||
func (s *StateSFlow) DecodeFlow(msg interface{}) error {
|
||||
@@ -118,7 +119,7 @@ func (s *StateSFlow) DecodeFlow(msg interface{}) error {
|
||||
}
|
||||
|
||||
var flowMessageSet []*flowmessage.FlowMessage
|
||||
flowMessageSet, err = producer.ProcessMessageSFlowConfig(msgDec, s.Config)
|
||||
flowMessageSet, err = producer.ProcessMessageSFlowConfig(msgDec, s.configMapped)
|
||||
|
||||
timeTrackStop := time.Now()
|
||||
DecoderTime.With(
|
||||
@@ -147,6 +148,11 @@ func (s *StateSFlow) DecodeFlow(msg interface{}) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *StateSFlow) initConfig() {
|
||||
s.configMapped = producer.NewProducerConfigMapped(s.Config)
|
||||
}
|
||||
|
||||
func (s *StateSFlow) FlowRoutine(workers int, addr string, port int, reuseport bool) error {
|
||||
s.initConfig()
|
||||
return UDPRoutine("sFlow", s.DecodeFlow, workers, addr, port, reuseport, s.Logger)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package utils
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"strconv"
|
||||
"time"
|
||||
@@ -11,9 +12,20 @@ import (
|
||||
decoder "github.com/netsampler/goflow2/decoders"
|
||||
"github.com/netsampler/goflow2/decoders/netflow"
|
||||
flowmessage "github.com/netsampler/goflow2/pb"
|
||||
"github.com/netsampler/goflow2/producer"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
type ProducerConfig *producer.ProducerConfig
|
||||
|
||||
func LoadMapping(f io.Reader) (ProducerConfig, error) {
|
||||
config := &producer.ProducerConfig{}
|
||||
dec := yaml.NewDecoder(f)
|
||||
err := dec.Decode(config)
|
||||
return config, err
|
||||
}
|
||||
|
||||
func GetServiceAddresses(srv string) (addrs []string, err error) {
|
||||
_, srvs, err := net.LookupSRV("", "", srv)
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user