Custom map flow fields (#36)

* adds dataframe link decoding
* can map NetFlow/IPFIX fields and bytes sections from sFlow/packets to any field inside the protobuf
* add CLI argument for loading a mapping yaml file

producer/producer_nf.go

@@ -137,7 +137,45 @@ func DecodeUNumber(b []byte, out interface{}) error {
 	return nil
 }
 
-func ConvertNetFlowDataSet(version uint16, baseTime uint32, uptime uint32, record []netflow.DataField) *flowmessage.FlowMessage {
+func DecodeNumber(b []byte, out interface{}) error {
+	var o int64
+	l := len(b)
+	switch l {
+	case 1:
+		o = int64(int8(b[0]))
+	case 2:
+		o = int64(int16(binary.BigEndian.Uint16(b)))
+	case 4:
+		o = int64(int32(binary.BigEndian.Uint32(b)))
+	case 8:
+		o = int64(binary.BigEndian.Uint64(b))
+	default:
+		if l < 8 {
+			var iter int
+			for i := range b {
+				o |= int64(b[i]) << int(8*(int(l)-iter-1))
+				iter++
+			}
+		} else {
+			return errors.New(fmt.Sprintf("Non-regular number of bytes for a number: %v", l))
+		}
+	}
+	switch t := out.(type) {
+	case *int8:
+		*t = int8(o)
+	case *int16:
+		*t = int16(o)
+	case *int32:
+		*t = int32(o)
+	case *int64:
+		*t = o
+	default:
+		return errors.New("The parameter is not a pointer to a int8/int16/int32/int64 structure")
+	}
+	return nil
+}
+
+func ConvertNetFlowDataSet(version uint16, baseTime uint32, uptime uint32, record []netflow.DataField, mapperNetFlow *NetFlowMapper, mapperSFlow *SFlowMapper) *flowmessage.FlowMessage {
 	flowMessage := &flowmessage.FlowMessage{}
 	var time uint64
 
@@ -155,6 +193,12 @@ func ConvertNetFlowDataSet(version uint16, baseTime uint32, uptime uint32, recor
 			continue
 		}
 
+		MapCustomNetFlow(flowMessage, df, mapperNetFlow)
+
+		if df.PenProvided {
+			continue
+		}
+
 		switch df.Type {
 
 		// Statistics
@@ -338,6 +382,16 @@ func ConvertNetFlowDataSet(version uint16, baseTime uint32, uptime uint32, recor
 				case netflow.IPFIX_FIELD_flowEndDeltaMicroseconds:
 					DecodeUNumber(v, &time)
 					flowMessage.TimeFlowEnd = uint64(baseTime) - time/1000000
+				// RFC7133
+				case netflow.IPFIX_FIELD_dataLinkFrameSize:
+					DecodeUNumber(v, &(flowMessage.Bytes))
+					flowMessage.Packets = 1
+				case netflow.IPFIX_FIELD_dataLinkFrameSection:
+					ParseEthernetHeader(flowMessage, v, mapperSFlow)
+					flowMessage.Packets = 1
+					if flowMessage.Bytes == 0 {
+						flowMessage.Bytes = uint64(len(v))
+					}
 				}
 			}
 		}
@@ -347,10 +401,10 @@ func ConvertNetFlowDataSet(version uint16, baseTime uint32, uptime uint32, recor
 	return flowMessage
 }
 
-func SearchNetFlowDataSetsRecords(version uint16, baseTime uint32, uptime uint32, dataRecords []netflow.DataRecord) []*flowmessage.FlowMessage {
+func SearchNetFlowDataSetsRecords(version uint16, baseTime uint32, uptime uint32, dataRecords []netflow.DataRecord, mapperNetFlow *NetFlowMapper, mapperSFlow *SFlowMapper) []*flowmessage.FlowMessage {
 	flowMessageSet := make([]*flowmessage.FlowMessage, 0)
 	for _, record := range dataRecords {
-		fmsg := ConvertNetFlowDataSet(version, baseTime, uptime, record.Values)
+		fmsg := ConvertNetFlowDataSet(version, baseTime, uptime, record.Values, mapperNetFlow, mapperSFlow)
 		if fmsg != nil {
 			flowMessageSet = append(flowMessageSet, fmsg)
 		}
@@ -358,10 +412,10 @@ func SearchNetFlowDataSetsRecords(version uint16, baseTime uint32, uptime uint32
 	return flowMessageSet
 }
 
-func SearchNetFlowDataSets(version uint16, baseTime uint32, uptime uint32, dataFlowSet []netflow.DataFlowSet) []*flowmessage.FlowMessage {
+func SearchNetFlowDataSets(version uint16, baseTime uint32, uptime uint32, dataFlowSet []netflow.DataFlowSet, mapperNetFlow *NetFlowMapper, mapperSFlow *SFlowMapper) []*flowmessage.FlowMessage {
 	flowMessageSet := make([]*flowmessage.FlowMessage, 0)
 	for _, dataFlowSetItem := range dataFlowSet {
-		fmsg := SearchNetFlowDataSetsRecords(version, baseTime, uptime, dataFlowSetItem.Records)
+		fmsg := SearchNetFlowDataSetsRecords(version, baseTime, uptime, dataFlowSetItem.Records, mapperNetFlow, mapperSFlow)
 		if fmsg != nil {
 			flowMessageSet = append(flowMessageSet, fmsg...)
 		}
@@ -431,9 +485,13 @@ func SplitIPFIXSets(packetIPFIX netflow.IPFIXPacket) ([]netflow.DataFlowSet, []n
 	return dataFlowSet, templatesFlowSet, optionsTemplatesFlowSet, optionsDataFlowSet
 }
 
+func ProcessMessageNetFlow(msgDec interface{}, samplingRateSys SamplingRateSystem) ([]*flowmessage.FlowMessage, error) {
+	return ProcessMessageNetFlowConfig(msgDec, samplingRateSys, nil)
+}
+
 // Convert a NetFlow datastructure to a FlowMessage protobuf
 // Does not put sampling rate
-func ProcessMessageNetFlow(msgDec interface{}, samplingRateSys SamplingRateSystem) ([]*flowmessage.FlowMessage, error) {
+func ProcessMessageNetFlowConfig(msgDec interface{}, samplingRateSys SamplingRateSystem, config *ProducerConfigMapped) ([]*flowmessage.FlowMessage, error) {
 	seqnum := uint32(0)
 	var baseTime uint32
 	var uptime uint32
@@ -449,7 +507,11 @@ func ProcessMessageNetFlow(msgDec interface{}, samplingRateSys SamplingRateSyste
 		uptime = msgDecConv.SystemUptime
 		obsDomainId := msgDecConv.SourceId
 
-		flowMessageSet = SearchNetFlowDataSets(9, baseTime, uptime, dataFlowSet)
+		var cfg *NetFlowMapper
+		if config != nil {
+			cfg = config.NetFlowV9
+		}
+		flowMessageSet = SearchNetFlowDataSets(9, baseTime, uptime, dataFlowSet, cfg, nil)
 		samplingRate, found := SearchNetFlowOptionDataSets(optionDataFlowSet)
 		if samplingRateSys != nil {
 			if found {
@@ -469,7 +531,13 @@ func ProcessMessageNetFlow(msgDec interface{}, samplingRateSys SamplingRateSyste
 		baseTime = msgDecConv.ExportTime
 		obsDomainId := msgDecConv.ObservationDomainId
 
-		flowMessageSet = SearchNetFlowDataSets(10, baseTime, uptime, dataFlowSet)
+		var cfgIpfix *NetFlowMapper
+		var cfgSflow *SFlowMapper
+		if config != nil {
+			cfgIpfix = config.IPFIX
+			cfgSflow = config.SFlow
+		}
+		flowMessageSet = SearchNetFlowDataSets(10, baseTime, uptime, dataFlowSet, cfgIpfix, cfgSflow)
 
 		samplingRate, found := SearchNetFlowOptionDataSets(optionDataFlowSet)
 		if samplingRateSys != nil {

producer/producer_sf.go

@@ -22,182 +22,211 @@ func GetSFlowFlowSamples(packet *sflow.Packet) []interface{} {
 	return flowSamples
 }
 
-type SFlowProducerConfig struct {
-}
-
 func ParseSampledHeader(flowMessage *flowmessage.FlowMessage, sampledHeader *sflow.SampledHeader) error {
 	return ParseSampledHeaderConfig(flowMessage, sampledHeader, nil)
 }
 
-func ParseSampledHeaderConfig(flowMessage *flowmessage.FlowMessage, sampledHeader *sflow.SampledHeader, config *SFlowProducerConfig) error {
+func ParseEthernetHeader(flowMessage *flowmessage.FlowMessage, data []byte, config *SFlowMapper) {
+	var hasMPLS bool
+	var countMpls uint32
+	var firstLabelMpls uint32
+	var firstTtlMpls uint8
+	var secondLabelMpls uint32
+	var secondTtlMpls uint8
+	var thirdLabelMpls uint32
+	var thirdTtlMpls uint8
+	var lastLabelMpls uint32
+	var lastTtlMpls uint8
 
+	var nextHeader byte
+	var tcpflags byte
+	srcIP := net.IP{}
+	dstIP := net.IP{}
+	offset := 14
+
+	var srcMac uint64
+	var dstMac uint64
+
+	var tos byte
+	var ttl byte
+	var identification uint16
+	var fragOffset uint16
+	var flowLabel uint32
+
+	var srcPort uint16
+	var dstPort uint16
+
+	for _, configLayer := range GetSFlowConfigLayer(config, 0) {
+		extracted := GetBytes(data, configLayer.Offset, configLayer.Length)
+		MapCustom(flowMessage, extracted, configLayer.Destination)
+	}
+
+	etherType := data[12:14]
+
+	dstMac = binary.BigEndian.Uint64(append([]byte{0, 0}, data[0:6]...))
+	srcMac = binary.BigEndian.Uint64(append([]byte{0, 0}, data[6:12]...))
+	(*flowMessage).SrcMac = srcMac
+	(*flowMessage).DstMac = dstMac
+
+	encap := true
+	iterations := 0
+	for encap && iterations <= 1 {
+		encap = false
+
+		if etherType[0] == 0x81 && etherType[1] == 0x0 { // VLAN 802.1Q
+			(*flowMessage).VlanId = uint32(binary.BigEndian.Uint16(data[14:16]))
+			offset += 4
+			etherType = data[16:18]
+		}
+
+		if etherType[0] == 0x88 && etherType[1] == 0x47 { // MPLS
+			iterateMpls := true
+			hasMPLS = true
+			for iterateMpls {
+				if len(data) < offset+5 {
+					iterateMpls = false
+					break
+				}
+				label := binary.BigEndian.Uint32(append([]byte{0}, data[offset:offset+3]...)) >> 4
+				//exp := data[offset+2] > 1
+				bottom := data[offset+2] & 1
+				mplsTtl := data[offset+3]
+				offset += 4
+
+				if bottom == 1 || label <= 15 || offset > len(data) {
+					if data[offset]&0xf0>>4 == 4 {
+						etherType = []byte{0x8, 0x0}
+					} else if data[offset]&0xf0>>4 == 6 {
+						etherType = []byte{0x86, 0xdd}
+					}
+					iterateMpls = false
+				}
+
+				if countMpls == 0 {
+					firstLabelMpls = label
+					firstTtlMpls = mplsTtl
+				} else if countMpls == 1 {
+					secondLabelMpls = label
+					secondTtlMpls = mplsTtl
+				} else if countMpls == 2 {
+					thirdLabelMpls = label
+					thirdTtlMpls = mplsTtl
+				} else {
+					lastLabelMpls = label
+					lastTtlMpls = mplsTtl
+				}
+				countMpls++
+			}
+		}
+
+		for _, configLayer := range GetSFlowConfigLayer(config, 3) {
+			extracted := GetBytes(data, offset*8+configLayer.Offset, configLayer.Length)
+			MapCustom(flowMessage, extracted, configLayer.Destination)
+		}
+
+		if etherType[0] == 0x8 && etherType[1] == 0x0 { // IPv4
+			if len(data) >= offset+20 {
+				nextHeader = data[offset+9]
+				srcIP = data[offset+12 : offset+16]
+				dstIP = data[offset+16 : offset+20]
+				tos = data[offset+1]
+				ttl = data[offset+8]
+
+				identification = binary.BigEndian.Uint16(data[offset+4 : offset+6])
+				fragOffset = binary.BigEndian.Uint16(data[offset+6 : offset+8])
+
+				offset += 20
+			}
+		} else if etherType[0] == 0x86 && etherType[1] == 0xdd { // IPv6
+			if len(data) >= offset+40 {
+				nextHeader = data[offset+6]
+				srcIP = data[offset+8 : offset+24]
+				dstIP = data[offset+24 : offset+40]
+
+				tostmp := uint32(binary.BigEndian.Uint16(data[offset : offset+2]))
+				tos = uint8(tostmp & 0x0ff0 >> 4)
+				ttl = data[offset+7]
+
+				flowLabel = binary.BigEndian.Uint32(data[offset : offset+4])
+
+				offset += 40
+
+			}
+		} else if etherType[0] == 0x8 && etherType[1] == 0x6 { // ARP
+		} /*else {
+			return errors.New(fmt.Sprintf("Unknown EtherType: %v\n", etherType))
+		} */
+
+		for _, configLayer := range GetSFlowConfigLayer(config, 4) {
+			extracted := GetBytes(data, offset*8+configLayer.Offset, configLayer.Length)
+			MapCustom(flowMessage, extracted, configLayer.Destination)
+		}
+
+		appOffset := 0
+		if len(data) >= offset+4 && (nextHeader == 17 || nextHeader == 6) {
+			srcPort = binary.BigEndian.Uint16(data[offset+0 : offset+2])
+			dstPort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
+		}
+
+		if nextHeader == 17 {
+			appOffset = 8
+		}
+
+		if len(data) >= offset+13 && nextHeader == 6 {
+			tcpflags = data[offset+13]
+
+			appOffset = int(data[13]>>4) * 4
+		}
+
+		// ICMP and ICMPv6
+		if len(data) >= offset+2 && (nextHeader == 1 || nextHeader == 58) {
+			(*flowMessage).IcmpType = uint32(data[offset+0])
+			(*flowMessage).IcmpCode = uint32(data[offset+1])
+		}
+
+		if appOffset > 0 {
+			for _, configLayer := range GetSFlowConfigLayer(config, 7) {
+				extracted := GetBytes(data, (offset+appOffset)*8+configLayer.Offset, configLayer.Length)
+				MapCustom(flowMessage, extracted, configLayer.Destination)
+			}
+		}
+
+		iterations++
+	}
+
+	(*flowMessage).HasMPLS = hasMPLS
+	(*flowMessage).MPLSCount = countMpls
+	(*flowMessage).MPLS1Label = firstLabelMpls
+	(*flowMessage).MPLS1TTL = uint32(firstTtlMpls)
+	(*flowMessage).MPLS2Label = secondLabelMpls
+	(*flowMessage).MPLS2TTL = uint32(secondTtlMpls)
+	(*flowMessage).MPLS3Label = thirdLabelMpls
+	(*flowMessage).MPLS3TTL = uint32(thirdTtlMpls)
+	(*flowMessage).MPLSLastLabel = lastLabelMpls
+	(*flowMessage).MPLSLastTTL = uint32(lastTtlMpls)
+
+	(*flowMessage).Etype = uint32(binary.BigEndian.Uint16(etherType[0:2]))
+	(*flowMessage).IPv6FlowLabel = flowLabel & 0xFFFFF
+
+	(*flowMessage).SrcPort = uint32(srcPort)
+	(*flowMessage).DstPort = uint32(dstPort)
+
+	(*flowMessage).SrcAddr = srcIP
+	(*flowMessage).DstAddr = dstIP
+	(*flowMessage).Proto = uint32(nextHeader)
+	(*flowMessage).IPTos = uint32(tos)
+	(*flowMessage).IPTTL = uint32(ttl)
+	(*flowMessage).TCPFlags = uint32(tcpflags)
+
+	(*flowMessage).FragmentId = uint32(identification)
+	(*flowMessage).FragmentOffset = uint32(fragOffset)
+}
+
+func ParseSampledHeaderConfig(flowMessage *flowmessage.FlowMessage, sampledHeader *sflow.SampledHeader, config *SFlowMapper) error {
 	data := (*sampledHeader).HeaderData
 	switch (*sampledHeader).Protocol {
 	case 1: // Ethernet
-		var hasMPLS bool
-		var countMpls uint32
-		var firstLabelMpls uint32
-		var firstTtlMpls uint8
-		var secondLabelMpls uint32
-		var secondTtlMpls uint8
-		var thirdLabelMpls uint32
-		var thirdTtlMpls uint8
-		var lastLabelMpls uint32
-		var lastTtlMpls uint8
-
-		var nextHeader byte
-		var tcpflags byte
-		srcIP := net.IP{}
-		dstIP := net.IP{}
-		offset := 14
-
-		var srcMac uint64
-		var dstMac uint64
-
-		var tos byte
-		var ttl byte
-		var identification uint16
-		var fragOffset uint16
-		var flowLabel uint32
-
-		var srcPort uint16
-		var dstPort uint16
-
-		etherType := data[12:14]
-
-		dstMac = binary.BigEndian.Uint64(append([]byte{0, 0}, data[0:6]...))
-		srcMac = binary.BigEndian.Uint64(append([]byte{0, 0}, data[6:12]...))
-		(*flowMessage).SrcMac = srcMac
-		(*flowMessage).DstMac = dstMac
-
-		encap := true
-		iterations := 0
-		for encap && iterations <= 1 {
-			encap = false
-
-			if etherType[0] == 0x81 && etherType[1] == 0x0 { // VLAN 802.1Q
-				(*flowMessage).VlanId = uint32(binary.BigEndian.Uint16(data[14:16]))
-				offset += 4
-				etherType = data[16:18]
-			}
-
-			if etherType[0] == 0x88 && etherType[1] == 0x47 { // MPLS
-				iterateMpls := true
-				hasMPLS = true
-				for iterateMpls {
-					if len(data) < offset+5 {
-						iterateMpls = false
-						break
-					}
-					label := binary.BigEndian.Uint32(append([]byte{0}, data[offset:offset+3]...)) >> 4
-					//exp := data[offset+2] > 1
-					bottom := data[offset+2] & 1
-					mplsTtl := data[offset+3]
-					offset += 4
-
-					if bottom == 1 || label <= 15 || offset > len(data) {
-						if data[offset]&0xf0>>4 == 4 {
-							etherType = []byte{0x8, 0x0}
-						} else if data[offset]&0xf0>>4 == 6 {
-							etherType = []byte{0x86, 0xdd}
-						}
-						iterateMpls = false
-					}
-
-					if countMpls == 0 {
-						firstLabelMpls = label
-						firstTtlMpls = mplsTtl
-					} else if countMpls == 1 {
-						secondLabelMpls = label
-						secondTtlMpls = mplsTtl
-					} else if countMpls == 2 {
-						thirdLabelMpls = label
-						thirdTtlMpls = mplsTtl
-					} else {
-						lastLabelMpls = label
-						lastTtlMpls = mplsTtl
-					}
-					countMpls++
-				}
-			}
-
-			if etherType[0] == 0x8 && etherType[1] == 0x0 { // IPv4
-				if len(data) >= offset+20 {
-					nextHeader = data[offset+9]
-					srcIP = data[offset+12 : offset+16]
-					dstIP = data[offset+16 : offset+20]
-					tos = data[offset+1]
-					ttl = data[offset+8]
-
-					identification = binary.BigEndian.Uint16(data[offset+4 : offset+6])
-					fragOffset = binary.BigEndian.Uint16(data[offset+6 : offset+8])
-
-					offset += 20
-				}
-			} else if etherType[0] == 0x86 && etherType[1] == 0xdd { // IPv6
-				if len(data) >= offset+40 {
-					nextHeader = data[offset+6]
-					srcIP = data[offset+8 : offset+24]
-					dstIP = data[offset+24 : offset+40]
-
-					tostmp := uint32(binary.BigEndian.Uint16(data[offset : offset+2]))
-					tos = uint8(tostmp & 0x0ff0 >> 4)
-					ttl = data[offset+7]
-
-					flowLabel = binary.BigEndian.Uint32(data[offset : offset+4])
-
-					offset += 40
-
-				}
-			} else if etherType[0] == 0x8 && etherType[1] == 0x6 { // ARP
-			} /*else {
-				return errors.New(fmt.Sprintf("Unknown EtherType: %v\n", etherType))
-			} */
-
-			if len(data) >= offset+4 && (nextHeader == 17 || nextHeader == 6) {
-				srcPort = binary.BigEndian.Uint16(data[offset+0 : offset+2])
-				dstPort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
-			}
-
-			if len(data) >= offset+13 && nextHeader == 6 {
-				tcpflags = data[offset+13]
-			}
-
-			// ICMP and ICMPv6
-			if len(data) >= offset+2 && (nextHeader == 1 || nextHeader == 58) {
-				(*flowMessage).IcmpType = uint32(data[offset+0])
-				(*flowMessage).IcmpCode = uint32(data[offset+1])
-			}
-
-			iterations++
-		}
-
-		(*flowMessage).HasMPLS = hasMPLS
-		(*flowMessage).MPLSCount = countMpls
-		(*flowMessage).MPLS1Label = firstLabelMpls
-		(*flowMessage).MPLS1TTL = uint32(firstTtlMpls)
-		(*flowMessage).MPLS2Label = secondLabelMpls
-		(*flowMessage).MPLS2TTL = uint32(secondTtlMpls)
-		(*flowMessage).MPLS3Label = thirdLabelMpls
-		(*flowMessage).MPLS3TTL = uint32(thirdTtlMpls)
-		(*flowMessage).MPLSLastLabel = lastLabelMpls
-		(*flowMessage).MPLSLastTTL = uint32(lastTtlMpls)
-
-		(*flowMessage).Etype = uint32(binary.BigEndian.Uint16(etherType[0:2]))
-		(*flowMessage).IPv6FlowLabel = flowLabel & 0xFFFFF
-
-		(*flowMessage).SrcPort = uint32(srcPort)
-		(*flowMessage).DstPort = uint32(dstPort)
-
-		(*flowMessage).SrcAddr = srcIP
-		(*flowMessage).DstAddr = dstIP
-		(*flowMessage).Proto = uint32(nextHeader)
-		(*flowMessage).IPTos = uint32(tos)
-		(*flowMessage).IPTTL = uint32(ttl)
-		(*flowMessage).TCPFlags = uint32(tcpflags)
-
-		(*flowMessage).FragmentId = uint32(identification)
-		(*flowMessage).FragmentOffset = uint32(fragOffset)
+		ParseEthernetHeader(flowMessage, data, config)
 	}
 	return nil
 }
@@ -206,7 +235,7 @@ func SearchSFlowSamples(samples []interface{}) []*flowmessage.FlowMessage {
 	return SearchSFlowSamples(samples)
 }
 
-func SearchSFlowSamplesConfig(samples []interface{}, config *SFlowProducerConfig) []*flowmessage.FlowMessage {
+func SearchSFlowSamplesConfig(samples []interface{}, config *SFlowMapper) []*flowmessage.FlowMessage {
 	flowMessageSet := make([]*flowmessage.FlowMessage, 0)
 
 	for _, flowSample := range samples {
@@ -289,15 +318,20 @@ func ProcessMessageSFlow(msgDec interface{}) ([]*flowmessage.FlowMessage, error)
 	return ProcessMessageSFlowConfig(msgDec, nil)
 }
 
-func ProcessMessageSFlowConfig(msgDec interface{}, config *SFlowProducerConfig) ([]*flowmessage.FlowMessage, error) {
+func ProcessMessageSFlowConfig(msgDec interface{}, config *ProducerConfigMapped) ([]*flowmessage.FlowMessage, error) {
 	switch packet := msgDec.(type) {
 	case sflow.Packet:
 		seqnum := packet.SequenceNumber
 		var agent net.IP
 		agent = packet.AgentIP
 
+		var cfg *SFlowMapper
+		if config != nil {
+			cfg = config.SFlow
+		}
+
 		flowSamples := GetSFlowFlowSamples(&packet)
-		flowMessageSet := SearchSFlowSamplesConfig(flowSamples, config)
+		flowMessageSet := SearchSFlowSamplesConfig(flowSamples, cfg)
 		for _, fmsg := range flowMessageSet {
 			fmsg.SamplerAddress = agent
 			fmsg.SequenceNum = seqnum

producer/reflect.go

@@ -0,0 +1,185 @@
+package producer
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/netsampler/goflow2/decoders/netflow"
+	flowmessage "github.com/netsampler/goflow2/pb"
+)
+
+func GetBytes(d []byte, offset int, length int) []byte {
+	if length == 0 {
+		return nil
+	}
+	leftBytes := offset / 8
+	rightBytes := (offset + length) / 8
+	if (offset+length)%8 != 0 {
+		rightBytes += 1
+	}
+	if leftBytes >= len(d) {
+		return nil
+	}
+	if rightBytes > len(d) {
+		rightBytes = len(d)
+	}
+	chunk := make([]byte, rightBytes-leftBytes)
+
+	offsetMod8 := (offset % 8)
+	shiftAnd := byte(0xff >> (8 - offsetMod8))
+
+	var shifted byte
+	for i := range chunk {
+		j := len(chunk) - 1 - i
+		cur := d[j+leftBytes]
+		chunk[j] = (cur << offsetMod8) | shifted
+		shifted = shiftAnd & cur
+	}
+	last := len(chunk) - 1
+	shiftAndLast := byte(0xff << ((8 - ((offset + length) % 8)) % 8))
+	chunk[last] = chunk[last] & shiftAndLast
+	return chunk
+}
+
+func MapCustomNetFlow(flowMessage *flowmessage.FlowMessage, df netflow.DataField, mapper *NetFlowMapper) {
+	if mapper == nil {
+		return
+	}
+	mapped, ok := mapper.Map(df)
+	if ok {
+		v := df.Value.([]byte)
+		MapCustom(flowMessage, v, mapped.Destination)
+	}
+}
+
+func MapCustom(flowMessage *flowmessage.FlowMessage, v []byte, destination string) {
+	vfm := reflect.ValueOf(flowMessage)
+	vfm = reflect.Indirect(vfm)
+
+	fieldValue := vfm.FieldByName(destination)
+
+	if fieldValue.IsValid() {
+		typeDest := fieldValue.Type()
+		fieldValueAddr := fieldValue.Addr()
+
+		if typeDest.Kind() == reflect.Slice && typeDest.Elem().Kind() == reflect.Uint8 {
+			fieldValue.SetBytes(v)
+		} else if fieldValueAddr.IsValid() && (typeDest.Kind() == reflect.Uint8 || typeDest.Kind() == reflect.Uint16 || typeDest.Kind() == reflect.Uint32 || typeDest.Kind() == reflect.Uint64) {
+			DecodeUNumber(v, fieldValueAddr.Interface())
+		} else if fieldValueAddr.IsValid() && (typeDest.Kind() == reflect.Int8 || typeDest.Kind() == reflect.Int16 || typeDest.Kind() == reflect.Int32 || typeDest.Kind() == reflect.Int64) {
+			DecodeNumber(v, fieldValueAddr.Interface())
+		}
+	}
+}
+
+type NetFlowMapField struct {
+	PenProvided bool   `json:"penprovided"`
+	Type        uint16 `json:"field"`
+	Pen         uint32 `json:"pen"`
+
+	Destination string `json:"destination"`
+	//DestinationLength uint8  `json:"dlen"` // could be used if populating a slice of uint16 that aren't in protobuf
+}
+
+type IPFIXProducerConfig struct {
+	Mapping []NetFlowMapField `json:"mapping"`
+	//PacketMapping []SFlowMapField   `json:"packet-mapping"` // for embedded frames: use sFlow configuration
+}
+
+type NetFlowV9ProducerConfig struct {
+	Mapping []NetFlowMapField `json:"mapping"`
+}
+
+type SFlowMapField struct {
+	Layer  int `json:"layer"`
+	Offset int `json:"offset"` // offset in bits
+	Length int `json:"length"` // length in bits
+
+	Destination string `json:"destination"`
+	//DestinationLength uint8  `json:"dlen"`
+}
+
+type SFlowProducerConfig struct {
+	Mapping []SFlowMapField `json:"mapping"`
+}
+
+type ProducerConfig struct {
+	IPFIX     IPFIXProducerConfig     `json:"ipfix"`
+	NetFlowV9 NetFlowV9ProducerConfig `json:"netflowv9"`
+	SFlow     SFlowProducerConfig     `json:"sflow"` // also used for IPFIX data frames
+
+	// should do a rename map list for when printing
+}
+
+type DataMap struct {
+	Destination string
+}
+
+type NetFlowMapper struct {
+	data map[string]DataMap // maps field to destination
+}
+
+func (m *NetFlowMapper) Map(field netflow.DataField) (DataMap, bool) {
+	mapped, found := m.data[fmt.Sprintf("%v-%d-%d", field.PenProvided, field.Pen, field.Type)]
+	return mapped, found
+}
+
+func MapFieldsNetFlow(fields []NetFlowMapField) *NetFlowMapper {
+	ret := make(map[string]DataMap)
+	for _, field := range fields {
+		ret[fmt.Sprintf("%v-%d-%d", field.PenProvided, field.Pen, field.Type)] = DataMap{Destination: field.Destination}
+	}
+	return &NetFlowMapper{ret}
+}
+
+type DataMapLayer struct {
+	Offset      int
+	Length      int
+	Destination string
+}
+
+type SFlowMapper struct {
+	data map[int][]DataMapLayer // map layer to list of offsets
+}
+
+func GetSFlowConfigLayer(m *SFlowMapper, layer int) []DataMapLayer {
+	if m == nil {
+		return nil
+	}
+	return m.data[layer]
+}
+
+func MapFieldsSFlow(fields []SFlowMapField) *SFlowMapper {
+	ret := make(map[int][]DataMapLayer)
+	for _, field := range fields {
+		retLayerEntry := DataMapLayer{
+			Offset:      field.Offset,
+			Length:      field.Length,
+			Destination: field.Destination,
+		}
+		retLayer, ok := ret[field.Layer]
+		if !ok {
+			retLayer = make([]DataMapLayer, 0)
+		}
+		retLayer = append(retLayer, retLayerEntry)
+		ret[field.Layer] = retLayer
+
+	}
+	return &SFlowMapper{ret}
+}
+
+type ProducerConfigMapped struct {
+	IPFIX     *NetFlowMapper `json:"ipfix"`
+	NetFlowV9 *NetFlowMapper `json:"netflowv9"`
+	SFlow     *SFlowMapper   `json:"sflow"`
+}
+
+func NewProducerConfigMapped(config *ProducerConfig) *ProducerConfigMapped {
+	newCfg := &ProducerConfigMapped{}
+	if config != nil {
+		newCfg.IPFIX = MapFieldsNetFlow(config.IPFIX.Mapping)
+		newCfg.NetFlowV9 = MapFieldsNetFlow(config.NetFlowV9.Mapping)
+		newCfg.SFlow = MapFieldsSFlow(config.SFlow.Mapping)
+	}
+	return newCfg
+}