Custom map flow fields (#36)

* adds dataframe link decoding
* can map NetFlow/IPFIX fields and bytes sections from sFlow/packets to any field inside the protobuf
* add CLI argument for loading a mapping yaml file

cmd/goflow2/main.go

@@ -47,6 +47,8 @@ var (
 
 	TemplatePath = flag.String("templates.path", "/templates", "NetFlow/IPFIX templates list")
 
+	MappingFile = flag.String("mapping", "", "Configuration file for custom mappings")
+
 	Version = flag.Bool("v", false, "Print version")
 )
 
@@ -67,6 +69,19 @@ func main() {
 	lvl, _ := log.ParseLevel(*LogLevel)
 	log.SetLevel(lvl)
 
+	var config utils.ProducerConfig
+	if *MappingFile != "" {
+		f, err := os.Open(*MappingFile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		config, err = utils.LoadMapping(f)
+		f.Close()
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+
 	ctx := context.Background()
 
 	formatter, err := format.FindFormat(ctx, *Format)
@@ -121,6 +136,7 @@ func main() {
 					Format:    formatter,
 					Transport: transporter,
 					Logger:    log.StandardLogger(),
+					Config:    config,
 				}
 				err = sSFlow.FlowRoutine(*Workers, hostname, int(port), *ReusePort)
 			} else if listenAddrUrl.Scheme == "netflow" {
@@ -128,6 +144,7 @@ func main() {
 					Format:    formatter,
 					Transport: transporter,
 					Logger:    log.StandardLogger(),
+					Config:    config,
 				}
 				err = sNF.FlowRoutine(*Workers, hostname, int(port), *ReusePort)
 			} else if listenAddrUrl.Scheme == "nfl" {

cmd/goflow2/mapping.yaml

@@ -0,0 +1,24 @@
+ipfix:
+  mapping:
+    - field: 7 # IPFIX_FIELD_sourceTransportPort
+      destination: CustomInteger1
+    - field: 11 # IPFIX_FIELD_destinationTransportPort
+      destination: CustomInteger2
+      # penprovided: false
+      # pen: 0
+netflowv9:
+  mapping:
+    - field: 7
+      destination: CustomInteger1
+    - field: 11
+      destination: CustomInteger2
+sflow:
+  mapping:
+    - layer: 4 # Layer 4: TCP or UDP
+      offset: 0 # Source port
+      length: 16 # 2 bytes
+      destination: CustomInteger1
+    - layer: 4
+      offset: 16 # Destination port
+      length: 16 # 2 bytes
+      destination: CustomInteger2

decoders/netflow/netflow.go

@@ -121,6 +121,7 @@ func DecodeTemplateSet(version uint16, payload *bytes.Buffer) ([]TemplateRecord,
 
 			if version == 10 && field.Type&0x8000 != 0 {
 				field.PenProvided = true
+				field.Type = field.Type ^ 0x8000
 				err = utils.BinaryDecoder(payload, &field.Pen)
 			}
 			fields[i] = field
@@ -165,8 +166,10 @@ func DecodeDataSetUsingFields(version uint16, payload *bytes.Buffer, listFields
 
 			value := payload.Next(finalLength)
 			nfvalue := DataField{
-				Type:  templateField.Type,
+				Type:        templateField.Type,
-				Value: value,
+				PenProvided: templateField.PenProvided,
+				Pen:         templateField.Pen,
+				Value:       value,
 			}
 			dataFields[i] = nfvalue
 		}

decoders/netflow/packet.go

@@ -89,7 +89,9 @@ type Field struct {
 
 type DataField struct {
 	// A numeric value that represents the type of field.
-	Type uint16
+	PenProvided bool
+	Type        uint16
+	Pen         uint32
 
 	// The value (in bytes) of the field.
 	Value interface{}

docs/protocols.md

@@ -56,3 +56,51 @@ The mapping to the protobuf format is listed in the table below.
 |MPLSxTTL|TTL of the MPLS label||Included|||
 |MPLSxLabel|MPLS label||Included|||
 
+## Add new custom fields
+
+If you are using enterprise fields that you need decoded
+or if you are looking for specific bytes inside the packet sample.
+
+This feature is only available when sending Protobufs (no text output).
+
+The [`mapping.yaml`](../cmd/goflow2/mapping.yaml) example file
+will collect source and destination port again, use it with `-mapping=mapping.yaml` in the CLI.
+
+Data coming from the flows can be added to the protobuf either as an unsigned/signed integer a slice of bytes.
+
+The `sflow` section allow to extract data from packet samples inside sFlow and inside IPFIX (dataframe).
+The following layers are available:
+* 0: no offset
+* 3: network layer, offsets to IP/IPv6 header
+* 4: transport layer, offsets to TCP/UDP header
+* 7: application layer, offsets to the TCP/UDP payload
+
+
+```yaml
+ipfix:
+  mapping:
+    - field: 7 # NetFlow or IPFIX field ID
+      destination: CustomInteger1 # Name of the field inside the Protobuf
+      penprovided: false # Has an enterprise number (optional)
+      pen: 0 # Enterprise number (optional)
+netflowv9:
+  mapping: []
+    # ... similar to above, Enterprise number will not be supported
+sflow:
+  mapping:
+    - layer: 4 # Layer
+      offset: 0 # Source port
+      length: 16 # 2 bytes
+      destination: CustomInteger1
+```
+
+Without editing and recompiling the [protobuf](../pb/flow.proto), you can use up to 5 integers and 5 slices of bytes:
+
+```protobuf
+  // Custom allocations
+  uint64 CustomInteger1 = 1001;
+  [...]
+
+  bytes CustomBytes1 = 1011;
+  [...]
+```

format/common/text.go

@@ -16,6 +16,7 @@ const (
 	FORMAT_TYPE_INTEGER
 	FORMAT_TYPE_IP
 	FORMAT_TYPE_MAC
+	FORMAT_TYPE_BYTES
 )
 
 var (
@@ -231,6 +232,8 @@ func FormatMessageReflectCustom(msg proto.Message, ext, quotes, sep, sign string
 				mac := make([]byte, 8)
 				binary.BigEndian.PutUint64(mac, fieldValue.Uint())
 				fstr[i] = fmt.Sprintf("%s%s%s%s%q", quotes, kf, quotes, sign, net.HardwareAddr(mac[2:]).String())
+			case FORMAT_TYPE_BYTES:
+				fstr[i] = fmt.Sprintf("%s%s%s%s%.2x", quotes, kf, quotes, sign, fieldValue.Bytes())
 			default:
 				if null {
 					fstr[i] = fmt.Sprintf("%s%s%s%snull", quotes, kf, quotes, sign)

go.mod

@@ -11,4 +11,5 @@ require (
 	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.0
 	google.golang.org/protobuf v1.23.0
+	gopkg.in/yaml.v2 v2.3.0
 )

go.sum

@@ -4,6 +4,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
 github.com/Shopify/sarama v1.19.0 h1:9oksLxC6uxVPHPVYUmq6xhr1BOF/hHobWH2UzO67z1s=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
+github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
 github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
 github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
@@ -95,6 +96,7 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
@@ -146,8 +148,10 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/libp2p/go-reuseport v0.0.2 h1:XSG94b1FJfGA01BUrT82imejHQyTxO4jEWqheyCXYvU=
 github.com/libp2p/go-reuseport v0.0.2/go.mod h1:SPD+5RwGC7rcnzngoYC86GjPzjSywuQyMVAheVBD9nQ=
@@ -375,6 +379,7 @@ golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
@@ -404,6 +409,7 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
@@ -417,6 +423,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pb/flow.pb.go

@@ -148,6 +148,11 @@ type FlowMessage struct {
 	MPLS3Label    uint32 `protobuf:"varint,60,opt,name=MPLS3Label,proto3" json:"MPLS3Label,omitempty"`       // Third Label
 	MPLSLastTTL   uint32 `protobuf:"varint,61,opt,name=MPLSLastTTL,proto3" json:"MPLSLastTTL,omitempty"`     // Last TTL
 	MPLSLastLabel uint32 `protobuf:"varint,62,opt,name=MPLSLastLabel,proto3" json:"MPLSLastLabel,omitempty"` // Last Label
+	// Custom allocations
+	CustomInteger1 uint64 `protobuf:"varint,1001,opt,name=CustomInteger1,proto3" json:"CustomInteger1,omitempty"`
+	CustomInteger2 uint64 `protobuf:"varint,1002,opt,name=CustomInteger2,proto3" json:"CustomInteger2,omitempty"`
+	CustomBytes1   []byte `protobuf:"bytes,1011,opt,name=CustomBytes1,proto3" json:"CustomBytes1,omitempty"`
+	CustomBytes2   []byte `protobuf:"bytes,1012,opt,name=CustomBytes2,proto3" json:"CustomBytes2,omitempty"`
 }
 
 func (x *FlowMessage) Reset() {
@@ -539,11 +544,39 @@ func (x *FlowMessage) GetMPLSLastLabel() uint32 {
 	return 0
 }
 
+func (x *FlowMessage) GetCustomInteger1() uint64 {
+	if x != nil {
+		return x.CustomInteger1
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetCustomInteger2() uint64 {
+	if x != nil {
+		return x.CustomInteger2
+	}
+	return 0
+}
+
+func (x *FlowMessage) GetCustomBytes1() []byte {
+	if x != nil {
+		return x.CustomBytes1
+	}
+	return nil
+}
+
+func (x *FlowMessage) GetCustomBytes2() []byte {
+	if x != nil {
+		return x.CustomBytes2
+	}
+	return nil
+}
+
 var File_pb_flow_proto protoreflect.FileDescriptor
 
 var file_pb_flow_proto_rawDesc = []byte{
 	0x0a, 0x0d, 0x70, 0x62, 0x2f, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12,
-	0x06, 0x66, 0x6c, 0x6f, 0x77, 0x70, 0x62, 0x22, 0xd0, 0x0c, 0x0a, 0x0b, 0x46, 0x6c, 0x6f, 0x77,
+	0x06, 0x66, 0x6c, 0x6f, 0x77, 0x70, 0x62, 0x22, 0xec, 0x0d, 0x0a, 0x0b, 0x46, 0x6c, 0x6f, 0x77,
 	0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x30, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x18,
 	0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1c, 0x2e, 0x66, 0x6c, 0x6f, 0x77, 0x70, 0x62, 0x2e, 0x46,
 	0x6c, 0x6f, 0x77, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x46, 0x6c, 0x6f, 0x77, 0x54,
@@ -639,15 +672,25 @@ var file_pb_flow_proto_rawDesc = []byte{
 	0x28, 0x0d, 0x52, 0x0b, 0x4d, 0x50, 0x4c, 0x53, 0x4c, 0x61, 0x73, 0x74, 0x54, 0x54, 0x4c, 0x12,
 	0x24, 0x0a, 0x0d, 0x4d, 0x50, 0x4c, 0x53, 0x4c, 0x61, 0x73, 0x74, 0x4c, 0x61, 0x62, 0x65, 0x6c,
 	0x18, 0x3e, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, 0x4d, 0x50, 0x4c, 0x53, 0x4c, 0x61, 0x73, 0x74,
-	0x4c, 0x61, 0x62, 0x65, 0x6c, 0x22, 0x53, 0x0a, 0x08, 0x46, 0x6c, 0x6f, 0x77, 0x54, 0x79, 0x70,
+	0x4c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x27, 0x0a, 0x0e, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x49,
-	0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x46, 0x4c, 0x4f, 0x57, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e,
+	0x6e, 0x74, 0x65, 0x67, 0x65, 0x72, 0x31, 0x18, 0xe9, 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e,
-	0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x46, 0x4c, 0x4f, 0x57, 0x5f, 0x35, 0x10, 0x01, 0x12,
+	0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x49, 0x6e, 0x74, 0x65, 0x67, 0x65, 0x72, 0x31, 0x12, 0x27,
-	0x0e, 0x0a, 0x0a, 0x4e, 0x45, 0x54, 0x46, 0x4c, 0x4f, 0x57, 0x5f, 0x56, 0x35, 0x10, 0x02, 0x12,
+	0x0a, 0x0e, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x49, 0x6e, 0x74, 0x65, 0x67, 0x65, 0x72, 0x32,
-	0x0e, 0x0a, 0x0a, 0x4e, 0x45, 0x54, 0x46, 0x4c, 0x4f, 0x57, 0x5f, 0x56, 0x39, 0x10, 0x03, 0x12,
+	0x18, 0xea, 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x49,
-	0x09, 0x0a, 0x05, 0x49, 0x50, 0x46, 0x49, 0x58, 0x10, 0x04, 0x42, 0x29, 0x5a, 0x27, 0x67, 0x69,
+	0x6e, 0x74, 0x65, 0x67, 0x65, 0x72, 0x32, 0x12, 0x23, 0x0a, 0x0c, 0x43, 0x75, 0x73, 0x74, 0x6f,
-	0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x65, 0x74, 0x73, 0x61, 0x6d, 0x70,
+	0x6d, 0x42, 0x79, 0x74, 0x65, 0x73, 0x31, 0x18, 0xf3, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c,
-	0x6c, 0x65, 0x72, 0x2f, 0x67, 0x6f, 0x66, 0x6c, 0x6f, 0x77, 0x32, 0x2f, 0x70, 0x62, 0x3b, 0x66,
+	0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x42, 0x79, 0x74, 0x65, 0x73, 0x31, 0x12, 0x23, 0x0a, 0x0c,
-	0x6c, 0x6f, 0x77, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x42, 0x79, 0x74, 0x65, 0x73, 0x32, 0x18, 0xf4, 0x07, 0x20,
+	0x01, 0x28, 0x0c, 0x52, 0x0c, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x42, 0x79, 0x74, 0x65, 0x73,
+	0x32, 0x22, 0x53, 0x0a, 0x08, 0x46, 0x6c, 0x6f, 0x77, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0f, 0x0a,
+	0x0b, 0x46, 0x4c, 0x4f, 0x57, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0b,
+	0x0a, 0x07, 0x53, 0x46, 0x4c, 0x4f, 0x57, 0x5f, 0x35, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x4e,
+	0x45, 0x54, 0x46, 0x4c, 0x4f, 0x57, 0x5f, 0x56, 0x35, 0x10, 0x02, 0x12, 0x0e, 0x0a, 0x0a, 0x4e,
+	0x45, 0x54, 0x46, 0x4c, 0x4f, 0x57, 0x5f, 0x56, 0x39, 0x10, 0x03, 0x12, 0x09, 0x0a, 0x05, 0x49,
+	0x50, 0x46, 0x49, 0x58, 0x10, 0x04, 0x42, 0x29, 0x5a, 0x27, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62,
+	0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x65, 0x74, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x72, 0x2f,
+	0x67, 0x6f, 0x66, 0x6c, 0x6f, 0x77, 0x32, 0x2f, 0x70, 0x62, 0x3b, 0x66, 0x6c, 0x6f, 0x77, 0x70,
+	0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

pb/flow.proto

@@ -101,4 +101,18 @@ message FlowMessage {
   // Custom fields: start after ID 1000:
   // uint32 MyCustomField = 1000;
 
+
+  // Custom allocations
+  uint64 CustomInteger1 = 1001;
+  uint64 CustomInteger2 = 1002;
+  uint64 CustomInteger3 = 1003;
+  uint64 CustomInteger4 = 1004;
+  uint64 CustomInteger5 = 1005;
+
+  bytes CustomBytes1 = 1011;
+  bytes CustomBytes2 = 1012;
+  bytes CustomBytes3 = 1013;
+  bytes CustomBytes4 = 1014;
+  bytes CustomBytes5 = 1015;
+
 }

producer/producer_nf.go

@@ -137,7 +137,45 @@ func DecodeUNumber(b []byte, out interface{}) error {
 	return nil
 }
 
-func ConvertNetFlowDataSet(version uint16, baseTime uint32, uptime uint32, record []netflow.DataField) *flowmessage.FlowMessage {
+func DecodeNumber(b []byte, out interface{}) error {
+	var o int64
+	l := len(b)
+	switch l {
+	case 1:
+		o = int64(int8(b[0]))
+	case 2:
+		o = int64(int16(binary.BigEndian.Uint16(b)))
+	case 4:
+		o = int64(int32(binary.BigEndian.Uint32(b)))
+	case 8:
+		o = int64(binary.BigEndian.Uint64(b))
+	default:
+		if l < 8 {
+			var iter int
+			for i := range b {
+				o |= int64(b[i]) << int(8*(int(l)-iter-1))
+				iter++
+			}
+		} else {
+			return errors.New(fmt.Sprintf("Non-regular number of bytes for a number: %v", l))
+		}
+	}
+	switch t := out.(type) {
+	case *int8:
+		*t = int8(o)
+	case *int16:
+		*t = int16(o)
+	case *int32:
+		*t = int32(o)
+	case *int64:
+		*t = o
+	default:
+		return errors.New("The parameter is not a pointer to a int8/int16/int32/int64 structure")
+	}
+	return nil
+}
+
+func ConvertNetFlowDataSet(version uint16, baseTime uint32, uptime uint32, record []netflow.DataField, mapperNetFlow *NetFlowMapper, mapperSFlow *SFlowMapper) *flowmessage.FlowMessage {
 	flowMessage := &flowmessage.FlowMessage{}
 	var time uint64
 
@@ -155,6 +193,12 @@ func ConvertNetFlowDataSet(version uint16, baseTime uint32, uptime uint32, recor
 			continue
 		}
 
+		MapCustomNetFlow(flowMessage, df, mapperNetFlow)
+
+		if df.PenProvided {
+			continue
+		}
+
 		switch df.Type {
 
 		// Statistics
@@ -338,6 +382,16 @@ func ConvertNetFlowDataSet(version uint16, baseTime uint32, uptime uint32, recor
 				case netflow.IPFIX_FIELD_flowEndDeltaMicroseconds:
 					DecodeUNumber(v, &time)
 					flowMessage.TimeFlowEnd = uint64(baseTime) - time/1000000
+				// RFC7133
+				case netflow.IPFIX_FIELD_dataLinkFrameSize:
+					DecodeUNumber(v, &(flowMessage.Bytes))
+					flowMessage.Packets = 1
+				case netflow.IPFIX_FIELD_dataLinkFrameSection:
+					ParseEthernetHeader(flowMessage, v, mapperSFlow)
+					flowMessage.Packets = 1
+					if flowMessage.Bytes == 0 {
+						flowMessage.Bytes = uint64(len(v))
+					}
 				}
 			}
 		}
@@ -347,10 +401,10 @@ func ConvertNetFlowDataSet(version uint16, baseTime uint32, uptime uint32, recor
 	return flowMessage
 }
 
-func SearchNetFlowDataSetsRecords(version uint16, baseTime uint32, uptime uint32, dataRecords []netflow.DataRecord) []*flowmessage.FlowMessage {
+func SearchNetFlowDataSetsRecords(version uint16, baseTime uint32, uptime uint32, dataRecords []netflow.DataRecord, mapperNetFlow *NetFlowMapper, mapperSFlow *SFlowMapper) []*flowmessage.FlowMessage {
 	flowMessageSet := make([]*flowmessage.FlowMessage, 0)
 	for _, record := range dataRecords {
-		fmsg := ConvertNetFlowDataSet(version, baseTime, uptime, record.Values)
+		fmsg := ConvertNetFlowDataSet(version, baseTime, uptime, record.Values, mapperNetFlow, mapperSFlow)
 		if fmsg != nil {
 			flowMessageSet = append(flowMessageSet, fmsg)
 		}
@@ -358,10 +412,10 @@ func SearchNetFlowDataSetsRecords(version uint16, baseTime uint32, uptime uint32
 	return flowMessageSet
 }
 
-func SearchNetFlowDataSets(version uint16, baseTime uint32, uptime uint32, dataFlowSet []netflow.DataFlowSet) []*flowmessage.FlowMessage {
+func SearchNetFlowDataSets(version uint16, baseTime uint32, uptime uint32, dataFlowSet []netflow.DataFlowSet, mapperNetFlow *NetFlowMapper, mapperSFlow *SFlowMapper) []*flowmessage.FlowMessage {
 	flowMessageSet := make([]*flowmessage.FlowMessage, 0)
 	for _, dataFlowSetItem := range dataFlowSet {
-		fmsg := SearchNetFlowDataSetsRecords(version, baseTime, uptime, dataFlowSetItem.Records)
+		fmsg := SearchNetFlowDataSetsRecords(version, baseTime, uptime, dataFlowSetItem.Records, mapperNetFlow, mapperSFlow)
 		if fmsg != nil {
 			flowMessageSet = append(flowMessageSet, fmsg...)
 		}
@@ -431,9 +485,13 @@ func SplitIPFIXSets(packetIPFIX netflow.IPFIXPacket) ([]netflow.DataFlowSet, []n
 	return dataFlowSet, templatesFlowSet, optionsTemplatesFlowSet, optionsDataFlowSet
 }
 
+func ProcessMessageNetFlow(msgDec interface{}, samplingRateSys SamplingRateSystem) ([]*flowmessage.FlowMessage, error) {
+	return ProcessMessageNetFlowConfig(msgDec, samplingRateSys, nil)
+}
+
 // Convert a NetFlow datastructure to a FlowMessage protobuf
 // Does not put sampling rate
-func ProcessMessageNetFlow(msgDec interface{}, samplingRateSys SamplingRateSystem) ([]*flowmessage.FlowMessage, error) {
+func ProcessMessageNetFlowConfig(msgDec interface{}, samplingRateSys SamplingRateSystem, config *ProducerConfigMapped) ([]*flowmessage.FlowMessage, error) {
 	seqnum := uint32(0)
 	var baseTime uint32
 	var uptime uint32
@@ -449,7 +507,11 @@ func ProcessMessageNetFlow(msgDec interface{}, samplingRateSys SamplingRateSyste
 		uptime = msgDecConv.SystemUptime
 		obsDomainId := msgDecConv.SourceId
 
-		flowMessageSet = SearchNetFlowDataSets(9, baseTime, uptime, dataFlowSet)
+		var cfg *NetFlowMapper
+		if config != nil {
+			cfg = config.NetFlowV9
+		}
+		flowMessageSet = SearchNetFlowDataSets(9, baseTime, uptime, dataFlowSet, cfg, nil)
 		samplingRate, found := SearchNetFlowOptionDataSets(optionDataFlowSet)
 		if samplingRateSys != nil {
 			if found {
@@ -469,7 +531,13 @@ func ProcessMessageNetFlow(msgDec interface{}, samplingRateSys SamplingRateSyste
 		baseTime = msgDecConv.ExportTime
 		obsDomainId := msgDecConv.ObservationDomainId
 
-		flowMessageSet = SearchNetFlowDataSets(10, baseTime, uptime, dataFlowSet)
+		var cfgIpfix *NetFlowMapper
+		var cfgSflow *SFlowMapper
+		if config != nil {
+			cfgIpfix = config.IPFIX
+			cfgSflow = config.SFlow
+		}
+		flowMessageSet = SearchNetFlowDataSets(10, baseTime, uptime, dataFlowSet, cfgIpfix, cfgSflow)
 
 		samplingRate, found := SearchNetFlowOptionDataSets(optionDataFlowSet)
 		if samplingRateSys != nil {

producer/producer_sf.go

@@ -22,182 +22,211 @@ func GetSFlowFlowSamples(packet *sflow.Packet) []interface{} {
 	return flowSamples
 }
 
-type SFlowProducerConfig struct {
-}
-
 func ParseSampledHeader(flowMessage *flowmessage.FlowMessage, sampledHeader *sflow.SampledHeader) error {
 	return ParseSampledHeaderConfig(flowMessage, sampledHeader, nil)
 }
 
-func ParseSampledHeaderConfig(flowMessage *flowmessage.FlowMessage, sampledHeader *sflow.SampledHeader, config *SFlowProducerConfig) error {
+func ParseEthernetHeader(flowMessage *flowmessage.FlowMessage, data []byte, config *SFlowMapper) {
+	var hasMPLS bool
+	var countMpls uint32
+	var firstLabelMpls uint32
+	var firstTtlMpls uint8
+	var secondLabelMpls uint32
+	var secondTtlMpls uint8
+	var thirdLabelMpls uint32
+	var thirdTtlMpls uint8
+	var lastLabelMpls uint32
+	var lastTtlMpls uint8
 
+	var nextHeader byte
+	var tcpflags byte
+	srcIP := net.IP{}
+	dstIP := net.IP{}
+	offset := 14
+
+	var srcMac uint64
+	var dstMac uint64
+
+	var tos byte
+	var ttl byte
+	var identification uint16
+	var fragOffset uint16
+	var flowLabel uint32
+
+	var srcPort uint16
+	var dstPort uint16
+
+	for _, configLayer := range GetSFlowConfigLayer(config, 0) {
+		extracted := GetBytes(data, configLayer.Offset, configLayer.Length)
+		MapCustom(flowMessage, extracted, configLayer.Destination)
+	}
+
+	etherType := data[12:14]
+
+	dstMac = binary.BigEndian.Uint64(append([]byte{0, 0}, data[0:6]...))
+	srcMac = binary.BigEndian.Uint64(append([]byte{0, 0}, data[6:12]...))
+	(*flowMessage).SrcMac = srcMac
+	(*flowMessage).DstMac = dstMac
+
+	encap := true
+	iterations := 0
+	for encap && iterations <= 1 {
+		encap = false
+
+		if etherType[0] == 0x81 && etherType[1] == 0x0 { // VLAN 802.1Q
+			(*flowMessage).VlanId = uint32(binary.BigEndian.Uint16(data[14:16]))
+			offset += 4
+			etherType = data[16:18]
+		}
+
+		if etherType[0] == 0x88 && etherType[1] == 0x47 { // MPLS
+			iterateMpls := true
+			hasMPLS = true
+			for iterateMpls {
+				if len(data) < offset+5 {
+					iterateMpls = false
+					break
+				}
+				label := binary.BigEndian.Uint32(append([]byte{0}, data[offset:offset+3]...)) >> 4
+				//exp := data[offset+2] > 1
+				bottom := data[offset+2] & 1
+				mplsTtl := data[offset+3]
+				offset += 4
+
+				if bottom == 1 || label <= 15 || offset > len(data) {
+					if data[offset]&0xf0>>4 == 4 {
+						etherType = []byte{0x8, 0x0}
+					} else if data[offset]&0xf0>>4 == 6 {
+						etherType = []byte{0x86, 0xdd}
+					}
+					iterateMpls = false
+				}
+
+				if countMpls == 0 {
+					firstLabelMpls = label
+					firstTtlMpls = mplsTtl
+				} else if countMpls == 1 {
+					secondLabelMpls = label
+					secondTtlMpls = mplsTtl
+				} else if countMpls == 2 {
+					thirdLabelMpls = label
+					thirdTtlMpls = mplsTtl
+				} else {
+					lastLabelMpls = label
+					lastTtlMpls = mplsTtl
+				}
+				countMpls++
+			}
+		}
+
+		for _, configLayer := range GetSFlowConfigLayer(config, 3) {
+			extracted := GetBytes(data, offset*8+configLayer.Offset, configLayer.Length)
+			MapCustom(flowMessage, extracted, configLayer.Destination)
+		}
+
+		if etherType[0] == 0x8 && etherType[1] == 0x0 { // IPv4
+			if len(data) >= offset+20 {
+				nextHeader = data[offset+9]
+				srcIP = data[offset+12 : offset+16]
+				dstIP = data[offset+16 : offset+20]
+				tos = data[offset+1]
+				ttl = data[offset+8]
+
+				identification = binary.BigEndian.Uint16(data[offset+4 : offset+6])
+				fragOffset = binary.BigEndian.Uint16(data[offset+6 : offset+8])
+
+				offset += 20
+			}
+		} else if etherType[0] == 0x86 && etherType[1] == 0xdd { // IPv6
+			if len(data) >= offset+40 {
+				nextHeader = data[offset+6]
+				srcIP = data[offset+8 : offset+24]
+				dstIP = data[offset+24 : offset+40]
+
+				tostmp := uint32(binary.BigEndian.Uint16(data[offset : offset+2]))
+				tos = uint8(tostmp & 0x0ff0 >> 4)
+				ttl = data[offset+7]
+
+				flowLabel = binary.BigEndian.Uint32(data[offset : offset+4])
+
+				offset += 40
+
+			}
+		} else if etherType[0] == 0x8 && etherType[1] == 0x6 { // ARP
+		} /*else {
+			return errors.New(fmt.Sprintf("Unknown EtherType: %v\n", etherType))
+		} */
+
+		for _, configLayer := range GetSFlowConfigLayer(config, 4) {
+			extracted := GetBytes(data, offset*8+configLayer.Offset, configLayer.Length)
+			MapCustom(flowMessage, extracted, configLayer.Destination)
+		}
+
+		appOffset := 0
+		if len(data) >= offset+4 && (nextHeader == 17 || nextHeader == 6) {
+			srcPort = binary.BigEndian.Uint16(data[offset+0 : offset+2])
+			dstPort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
+		}
+
+		if nextHeader == 17 {
+			appOffset = 8
+		}
+
+		if len(data) >= offset+13 && nextHeader == 6 {
+			tcpflags = data[offset+13]
+
+			appOffset = int(data[13]>>4) * 4
+		}
+
+		// ICMP and ICMPv6
+		if len(data) >= offset+2 && (nextHeader == 1 || nextHeader == 58) {
+			(*flowMessage).IcmpType = uint32(data[offset+0])
+			(*flowMessage).IcmpCode = uint32(data[offset+1])
+		}
+
+		if appOffset > 0 {
+			for _, configLayer := range GetSFlowConfigLayer(config, 7) {
+				extracted := GetBytes(data, (offset+appOffset)*8+configLayer.Offset, configLayer.Length)
+				MapCustom(flowMessage, extracted, configLayer.Destination)
+			}
+		}
+
+		iterations++
+	}
+
+	(*flowMessage).HasMPLS = hasMPLS
+	(*flowMessage).MPLSCount = countMpls
+	(*flowMessage).MPLS1Label = firstLabelMpls
+	(*flowMessage).MPLS1TTL = uint32(firstTtlMpls)
+	(*flowMessage).MPLS2Label = secondLabelMpls
+	(*flowMessage).MPLS2TTL = uint32(secondTtlMpls)
+	(*flowMessage).MPLS3Label = thirdLabelMpls
+	(*flowMessage).MPLS3TTL = uint32(thirdTtlMpls)
+	(*flowMessage).MPLSLastLabel = lastLabelMpls
+	(*flowMessage).MPLSLastTTL = uint32(lastTtlMpls)
+
+	(*flowMessage).Etype = uint32(binary.BigEndian.Uint16(etherType[0:2]))
+	(*flowMessage).IPv6FlowLabel = flowLabel & 0xFFFFF
+
+	(*flowMessage).SrcPort = uint32(srcPort)
+	(*flowMessage).DstPort = uint32(dstPort)
+
+	(*flowMessage).SrcAddr = srcIP
+	(*flowMessage).DstAddr = dstIP
+	(*flowMessage).Proto = uint32(nextHeader)
+	(*flowMessage).IPTos = uint32(tos)
+	(*flowMessage).IPTTL = uint32(ttl)
+	(*flowMessage).TCPFlags = uint32(tcpflags)
+
+	(*flowMessage).FragmentId = uint32(identification)
+	(*flowMessage).FragmentOffset = uint32(fragOffset)
+}
+
+func ParseSampledHeaderConfig(flowMessage *flowmessage.FlowMessage, sampledHeader *sflow.SampledHeader, config *SFlowMapper) error {
 	data := (*sampledHeader).HeaderData
 	switch (*sampledHeader).Protocol {
 	case 1: // Ethernet
-		var hasMPLS bool
+		ParseEthernetHeader(flowMessage, data, config)
-		var countMpls uint32
-		var firstLabelMpls uint32
-		var firstTtlMpls uint8
-		var secondLabelMpls uint32
-		var secondTtlMpls uint8
-		var thirdLabelMpls uint32
-		var thirdTtlMpls uint8
-		var lastLabelMpls uint32
-		var lastTtlMpls uint8
-
-		var nextHeader byte
-		var tcpflags byte
-		srcIP := net.IP{}
-		dstIP := net.IP{}
-		offset := 14
-
-		var srcMac uint64
-		var dstMac uint64
-
-		var tos byte
-		var ttl byte
-		var identification uint16
-		var fragOffset uint16
-		var flowLabel uint32
-
-		var srcPort uint16
-		var dstPort uint16
-
-		etherType := data[12:14]
-
-		dstMac = binary.BigEndian.Uint64(append([]byte{0, 0}, data[0:6]...))
-		srcMac = binary.BigEndian.Uint64(append([]byte{0, 0}, data[6:12]...))
-		(*flowMessage).SrcMac = srcMac
-		(*flowMessage).DstMac = dstMac
-
-		encap := true
-		iterations := 0
-		for encap && iterations <= 1 {
-			encap = false
-
-			if etherType[0] == 0x81 && etherType[1] == 0x0 { // VLAN 802.1Q
-				(*flowMessage).VlanId = uint32(binary.BigEndian.Uint16(data[14:16]))
-				offset += 4
-				etherType = data[16:18]
-			}
-
-			if etherType[0] == 0x88 && etherType[1] == 0x47 { // MPLS
-				iterateMpls := true
-				hasMPLS = true
-				for iterateMpls {
-					if len(data) < offset+5 {
-						iterateMpls = false
-						break
-					}
-					label := binary.BigEndian.Uint32(append([]byte{0}, data[offset:offset+3]...)) >> 4
-					//exp := data[offset+2] > 1
-					bottom := data[offset+2] & 1
-					mplsTtl := data[offset+3]
-					offset += 4
-
-					if bottom == 1 || label <= 15 || offset > len(data) {
-						if data[offset]&0xf0>>4 == 4 {
-							etherType = []byte{0x8, 0x0}
-						} else if data[offset]&0xf0>>4 == 6 {
-							etherType = []byte{0x86, 0xdd}
-						}
-						iterateMpls = false
-					}
-
-					if countMpls == 0 {
-						firstLabelMpls = label
-						firstTtlMpls = mplsTtl
-					} else if countMpls == 1 {
-						secondLabelMpls = label
-						secondTtlMpls = mplsTtl
-					} else if countMpls == 2 {
-						thirdLabelMpls = label
-						thirdTtlMpls = mplsTtl
-					} else {
-						lastLabelMpls = label
-						lastTtlMpls = mplsTtl
-					}
-					countMpls++
-				}
-			}
-
-			if etherType[0] == 0x8 && etherType[1] == 0x0 { // IPv4
-				if len(data) >= offset+20 {
-					nextHeader = data[offset+9]
-					srcIP = data[offset+12 : offset+16]
-					dstIP = data[offset+16 : offset+20]
-					tos = data[offset+1]
-					ttl = data[offset+8]
-
-					identification = binary.BigEndian.Uint16(data[offset+4 : offset+6])
-					fragOffset = binary.BigEndian.Uint16(data[offset+6 : offset+8])
-
-					offset += 20
-				}
-			} else if etherType[0] == 0x86 && etherType[1] == 0xdd { // IPv6
-				if len(data) >= offset+40 {
-					nextHeader = data[offset+6]
-					srcIP = data[offset+8 : offset+24]
-					dstIP = data[offset+24 : offset+40]
-
-					tostmp := uint32(binary.BigEndian.Uint16(data[offset : offset+2]))
-					tos = uint8(tostmp & 0x0ff0 >> 4)
-					ttl = data[offset+7]
-
-					flowLabel = binary.BigEndian.Uint32(data[offset : offset+4])
-
-					offset += 40
-
-				}
-			} else if etherType[0] == 0x8 && etherType[1] == 0x6 { // ARP
-			} /*else {
-				return errors.New(fmt.Sprintf("Unknown EtherType: %v\n", etherType))
-			} */
-
-			if len(data) >= offset+4 && (nextHeader == 17 || nextHeader == 6) {
-				srcPort = binary.BigEndian.Uint16(data[offset+0 : offset+2])
-				dstPort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
-			}
-
-			if len(data) >= offset+13 && nextHeader == 6 {
-				tcpflags = data[offset+13]
-			}
-
-			// ICMP and ICMPv6
-			if len(data) >= offset+2 && (nextHeader == 1 || nextHeader == 58) {
-				(*flowMessage).IcmpType = uint32(data[offset+0])
-				(*flowMessage).IcmpCode = uint32(data[offset+1])
-			}
-
-			iterations++
-		}
-
-		(*flowMessage).HasMPLS = hasMPLS
-		(*flowMessage).MPLSCount = countMpls
-		(*flowMessage).MPLS1Label = firstLabelMpls
-		(*flowMessage).MPLS1TTL = uint32(firstTtlMpls)
-		(*flowMessage).MPLS2Label = secondLabelMpls
-		(*flowMessage).MPLS2TTL = uint32(secondTtlMpls)
-		(*flowMessage).MPLS3Label = thirdLabelMpls
-		(*flowMessage).MPLS3TTL = uint32(thirdTtlMpls)
-		(*flowMessage).MPLSLastLabel = lastLabelMpls
-		(*flowMessage).MPLSLastTTL = uint32(lastTtlMpls)
-
-		(*flowMessage).Etype = uint32(binary.BigEndian.Uint16(etherType[0:2]))
-		(*flowMessage).IPv6FlowLabel = flowLabel & 0xFFFFF
-
-		(*flowMessage).SrcPort = uint32(srcPort)
-		(*flowMessage).DstPort = uint32(dstPort)
-
-		(*flowMessage).SrcAddr = srcIP
-		(*flowMessage).DstAddr = dstIP
-		(*flowMessage).Proto = uint32(nextHeader)
-		(*flowMessage).IPTos = uint32(tos)
-		(*flowMessage).IPTTL = uint32(ttl)
-		(*flowMessage).TCPFlags = uint32(tcpflags)
-
-		(*flowMessage).FragmentId = uint32(identification)
-		(*flowMessage).FragmentOffset = uint32(fragOffset)
 	}
 	return nil
 }
@@ -206,7 +235,7 @@ func SearchSFlowSamples(samples []interface{}) []*flowmessage.FlowMessage {
 	return SearchSFlowSamples(samples)
 }
 
-func SearchSFlowSamplesConfig(samples []interface{}, config *SFlowProducerConfig) []*flowmessage.FlowMessage {
+func SearchSFlowSamplesConfig(samples []interface{}, config *SFlowMapper) []*flowmessage.FlowMessage {
 	flowMessageSet := make([]*flowmessage.FlowMessage, 0)
 
 	for _, flowSample := range samples {
@@ -289,15 +318,20 @@ func ProcessMessageSFlow(msgDec interface{}) ([]*flowmessage.FlowMessage, error)
 	return ProcessMessageSFlowConfig(msgDec, nil)
 }
 
-func ProcessMessageSFlowConfig(msgDec interface{}, config *SFlowProducerConfig) ([]*flowmessage.FlowMessage, error) {
+func ProcessMessageSFlowConfig(msgDec interface{}, config *ProducerConfigMapped) ([]*flowmessage.FlowMessage, error) {
 	switch packet := msgDec.(type) {
 	case sflow.Packet:
 		seqnum := packet.SequenceNumber
 		var agent net.IP
 		agent = packet.AgentIP
 
+		var cfg *SFlowMapper
+		if config != nil {
+			cfg = config.SFlow
+		}
+
 		flowSamples := GetSFlowFlowSamples(&packet)
-		flowMessageSet := SearchSFlowSamplesConfig(flowSamples, config)
+		flowMessageSet := SearchSFlowSamplesConfig(flowSamples, cfg)
 		for _, fmsg := range flowMessageSet {
 			fmsg.SamplerAddress = agent
 			fmsg.SequenceNum = seqnum

producer/reflect.go

@@ -0,0 +1,185 @@
+package producer
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/netsampler/goflow2/decoders/netflow"
+	flowmessage "github.com/netsampler/goflow2/pb"
+)
+
+func GetBytes(d []byte, offset int, length int) []byte {
+	if length == 0 {
+		return nil
+	}
+	leftBytes := offset / 8
+	rightBytes := (offset + length) / 8
+	if (offset+length)%8 != 0 {
+		rightBytes += 1
+	}
+	if leftBytes >= len(d) {
+		return nil
+	}
+	if rightBytes > len(d) {
+		rightBytes = len(d)
+	}
+	chunk := make([]byte, rightBytes-leftBytes)
+
+	offsetMod8 := (offset % 8)
+	shiftAnd := byte(0xff >> (8 - offsetMod8))
+
+	var shifted byte
+	for i := range chunk {
+		j := len(chunk) - 1 - i
+		cur := d[j+leftBytes]
+		chunk[j] = (cur << offsetMod8) | shifted
+		shifted = shiftAnd & cur
+	}
+	last := len(chunk) - 1
+	shiftAndLast := byte(0xff << ((8 - ((offset + length) % 8)) % 8))
+	chunk[last] = chunk[last] & shiftAndLast
+	return chunk
+}
+
+func MapCustomNetFlow(flowMessage *flowmessage.FlowMessage, df netflow.DataField, mapper *NetFlowMapper) {
+	if mapper == nil {
+		return
+	}
+	mapped, ok := mapper.Map(df)
+	if ok {
+		v := df.Value.([]byte)
+		MapCustom(flowMessage, v, mapped.Destination)
+	}
+}
+
+func MapCustom(flowMessage *flowmessage.FlowMessage, v []byte, destination string) {
+	vfm := reflect.ValueOf(flowMessage)
+	vfm = reflect.Indirect(vfm)
+
+	fieldValue := vfm.FieldByName(destination)
+
+	if fieldValue.IsValid() {
+		typeDest := fieldValue.Type()
+		fieldValueAddr := fieldValue.Addr()
+
+		if typeDest.Kind() == reflect.Slice && typeDest.Elem().Kind() == reflect.Uint8 {
+			fieldValue.SetBytes(v)
+		} else if fieldValueAddr.IsValid() && (typeDest.Kind() == reflect.Uint8 || typeDest.Kind() == reflect.Uint16 || typeDest.Kind() == reflect.Uint32 || typeDest.Kind() == reflect.Uint64) {
+			DecodeUNumber(v, fieldValueAddr.Interface())
+		} else if fieldValueAddr.IsValid() && (typeDest.Kind() == reflect.Int8 || typeDest.Kind() == reflect.Int16 || typeDest.Kind() == reflect.Int32 || typeDest.Kind() == reflect.Int64) {
+			DecodeNumber(v, fieldValueAddr.Interface())
+		}
+	}
+}
+
+type NetFlowMapField struct {
+	PenProvided bool   `json:"penprovided"`
+	Type        uint16 `json:"field"`
+	Pen         uint32 `json:"pen"`
+
+	Destination string `json:"destination"`
+	//DestinationLength uint8  `json:"dlen"` // could be used if populating a slice of uint16 that aren't in protobuf
+}
+
+type IPFIXProducerConfig struct {
+	Mapping []NetFlowMapField `json:"mapping"`
+	//PacketMapping []SFlowMapField   `json:"packet-mapping"` // for embedded frames: use sFlow configuration
+}
+
+type NetFlowV9ProducerConfig struct {
+	Mapping []NetFlowMapField `json:"mapping"`
+}
+
+type SFlowMapField struct {
+	Layer  int `json:"layer"`
+	Offset int `json:"offset"` // offset in bits
+	Length int `json:"length"` // length in bits
+
+	Destination string `json:"destination"`
+	//DestinationLength uint8  `json:"dlen"`
+}
+
+type SFlowProducerConfig struct {
+	Mapping []SFlowMapField `json:"mapping"`
+}
+
+type ProducerConfig struct {
+	IPFIX     IPFIXProducerConfig     `json:"ipfix"`
+	NetFlowV9 NetFlowV9ProducerConfig `json:"netflowv9"`
+	SFlow     SFlowProducerConfig     `json:"sflow"` // also used for IPFIX data frames
+
+	// should do a rename map list for when printing
+}
+
+type DataMap struct {
+	Destination string
+}
+
+type NetFlowMapper struct {
+	data map[string]DataMap // maps field to destination
+}
+
+func (m *NetFlowMapper) Map(field netflow.DataField) (DataMap, bool) {
+	mapped, found := m.data[fmt.Sprintf("%v-%d-%d", field.PenProvided, field.Pen, field.Type)]
+	return mapped, found
+}
+
+func MapFieldsNetFlow(fields []NetFlowMapField) *NetFlowMapper {
+	ret := make(map[string]DataMap)
+	for _, field := range fields {
+		ret[fmt.Sprintf("%v-%d-%d", field.PenProvided, field.Pen, field.Type)] = DataMap{Destination: field.Destination}
+	}
+	return &NetFlowMapper{ret}
+}
+
+type DataMapLayer struct {
+	Offset      int
+	Length      int
+	Destination string
+}
+
+type SFlowMapper struct {
+	data map[int][]DataMapLayer // map layer to list of offsets
+}
+
+func GetSFlowConfigLayer(m *SFlowMapper, layer int) []DataMapLayer {
+	if m == nil {
+		return nil
+	}
+	return m.data[layer]
+}
+
+func MapFieldsSFlow(fields []SFlowMapField) *SFlowMapper {
+	ret := make(map[int][]DataMapLayer)
+	for _, field := range fields {
+		retLayerEntry := DataMapLayer{
+			Offset:      field.Offset,
+			Length:      field.Length,
+			Destination: field.Destination,
+		}
+		retLayer, ok := ret[field.Layer]
+		if !ok {
+			retLayer = make([]DataMapLayer, 0)
+		}
+		retLayer = append(retLayer, retLayerEntry)
+		ret[field.Layer] = retLayer
+
+	}
+	return &SFlowMapper{ret}
+}
+
+type ProducerConfigMapped struct {
+	IPFIX     *NetFlowMapper `json:"ipfix"`
+	NetFlowV9 *NetFlowMapper `json:"netflowv9"`
+	SFlow     *SFlowMapper   `json:"sflow"`
+}
+
+func NewProducerConfigMapped(config *ProducerConfig) *ProducerConfigMapped {
+	newCfg := &ProducerConfigMapped{}
+	if config != nil {
+		newCfg.IPFIX = MapFieldsNetFlow(config.IPFIX.Mapping)
+		newCfg.NetFlowV9 = MapFieldsNetFlow(config.NetFlowV9.Mapping)
+		newCfg.SFlow = MapFieldsSFlow(config.SFlow.Mapping)
+	}
+	return newCfg
+}

utils/netflow.go

@@ -59,6 +59,9 @@ type StateNetFlow struct {
 
 	samplinglock *sync.RWMutex
 	sampling     map[string]producer.SamplingRateSystem
+
+	Config       *producer.ProducerConfig
+	configMapped *producer.ProducerConfigMapped
 }
 
 func (s *StateNetFlow) DecodeFlow(msg interface{}) error {
@@ -215,7 +218,7 @@ func (s *StateNetFlow) DecodeFlow(msg interface{}) error {
 					Add(float64(len(fsConv.Records)))
 			}
 		}
-		flowMessageSet, err = producer.ProcessMessageNetFlow(msgDecConv, sampling)
+		flowMessageSet, err = producer.ProcessMessageNetFlowConfig(msgDecConv, sampling, s.configMapped)
 
 		for _, fmsg := range flowMessageSet {
 			fmsg.TimeReceived = ts
@@ -308,7 +311,7 @@ func (s *StateNetFlow) DecodeFlow(msg interface{}) error {
 					Add(float64(len(fsConv.Records)))
 			}
 		}
-		flowMessageSet, err = producer.ProcessMessageNetFlow(msgDecConv, sampling)
+		flowMessageSet, err = producer.ProcessMessageNetFlowConfig(msgDecConv, sampling, s.configMapped)
 
 		for _, fmsg := range flowMessageSet {
 			fmsg.TimeReceived = ts
@@ -365,7 +368,12 @@ func (s *StateNetFlow) InitTemplates() {
 	s.samplinglock = &sync.RWMutex{}
 }
 
+func (s *StateNetFlow) initConfig() {
+	s.configMapped = producer.NewProducerConfigMapped(s.Config)
+}
+
 func (s *StateNetFlow) FlowRoutine(workers int, addr string, port int, reuseport bool) error {
 	s.InitTemplates()
+	s.initConfig()
 	return UDPRoutine("NetFlow", s.DecodeFlow, workers, addr, port, reuseport, s.Logger)
 }

utils/sflow.go

@@ -18,7 +18,8 @@ type StateSFlow struct {
 	Transport transport.TransportInterface
 	Logger    Logger
 
-	Config *producer.SFlowProducerConfig
+	Config       *producer.ProducerConfig
+	configMapped *producer.ProducerConfigMapped
 }
 
 func (s *StateSFlow) DecodeFlow(msg interface{}) error {
@@ -118,7 +119,7 @@ func (s *StateSFlow) DecodeFlow(msg interface{}) error {
 	}
 
 	var flowMessageSet []*flowmessage.FlowMessage
-	flowMessageSet, err = producer.ProcessMessageSFlowConfig(msgDec, s.Config)
+	flowMessageSet, err = producer.ProcessMessageSFlowConfig(msgDec, s.configMapped)
 
 	timeTrackStop := time.Now()
 	DecoderTime.With(
@@ -147,6 +148,11 @@ func (s *StateSFlow) DecodeFlow(msg interface{}) error {
 	return nil
 }
 
+func (s *StateSFlow) initConfig() {
+	s.configMapped = producer.NewProducerConfigMapped(s.Config)
+}
+
 func (s *StateSFlow) FlowRoutine(workers int, addr string, port int, reuseport bool) error {
+	s.initConfig()
 	return UDPRoutine("sFlow", s.DecodeFlow, workers, addr, port, reuseport, s.Logger)
 }

utils/utils.go

@@ -3,6 +3,7 @@ package utils
 import (
 	"errors"
 	"fmt"
+	"io"
 	"net"
 	"strconv"
 	"time"
@@ -11,9 +12,20 @@ import (
 	decoder "github.com/netsampler/goflow2/decoders"
 	"github.com/netsampler/goflow2/decoders/netflow"
 	flowmessage "github.com/netsampler/goflow2/pb"
+	"github.com/netsampler/goflow2/producer"
 	"github.com/prometheus/client_golang/prometheus"
+	"gopkg.in/yaml.v2"
 )
 
+type ProducerConfig *producer.ProducerConfig
+
+func LoadMapping(f io.Reader) (ProducerConfig, error) {
+	config := &producer.ProducerConfig{}
+	dec := yaml.NewDecoder(f)
+	err := dec.Decode(config)
+	return config, err
+}
+
 func GetServiceAddresses(srv string) (addrs []string, err error) {
 	_, srvs, err := net.LookupSRV("", "", srv)
 	if err != nil {