Difference between revisions of "Logstash for IPFIX"

From Initech Technical Wiki
Jump to: navigation, search
 
(4 intermediate revisions by the same user not shown)
Line 1: Line 1:
−
Logstash for IPFIX (Netflow V10) is on the way but in the meantime i've been able to get it working thus:
+
<amp/>
 +
Native IPFIX (Netflow V10) for Logstash is still in development but in the meantime i've been able to get it working thus:
  
−
= [[Nprobe]] =
+
== [[Nprobe]] ==
 
I know, nprobe is commercial but i had it lying around so chose to use it.  It can output in a variety of ways but in the base license model only zeromq and tcp methods allow direct output to logstash.  My experience with zeromq was initially good but after a reinstall i couldn't get it working again, something wrong with the encoding or compression or something meant that logstash was seeing the data as garbage.
 
I know, nprobe is commercial but i had it lying around so chose to use it.  It can output in a variety of ways but in the base license model only zeromq and tcp methods allow direct output to logstash.  My experience with zeromq was initially good but after a reinstall i couldn't get it working again, something wrong with the encoding or compression or something meant that logstash was seeing the data as garbage.
  
−
Switching to tcp output in json format got it working in the end so this is the nprobe config file, you'll note that local interface sniffing is disabled to it only listens for flows on port 2100:
+
Switching to tcp output in json format got it working in the end so this is the nprobe config file, you'll note that local interface sniffing is disabled so it only listens for flows on port 2100:
  
 
<pre>
 
<pre>
Line 21: Line 22:
 
<pre>
 
<pre>
 
touch /etc/nprobe/nprobe-eth0.start
 
touch /etc/nprobe/nprobe-eth0.start
 +
</pre>
 +
 +
== Logstash ==
 +
The logstash config is easy unless you want to filter the flows as they arrive:
 +
 +
<pre>
 +
# cat /etc/logstash/conf.d/tcp.conf
 +
input {
 +
tcp {
 +
port => 5000
 +
type => "netflow"
 +
codec => "json"
 +
}
 +
}
 +
</pre>
 +
 +
I chose to do some filtering to assign nice names and tags to my SNMP interface ids
 +
 +
<pre>
 +
filter {
 +
if [INPUT_SNMP] == 863 {
 +
mutate {
 +
add_field =>  {
 +
"INPUT_DESC" => "International IP Transit - Vocus Auckland"
 +
}
 +
add_tag => [ "International IP Transit - Vocus Auckland", "International"]
 +
}
 +
}
 +
        if [OUTPUT_SNMP] == 863 {
 +
                mutate {
 +
                        add_field =>  {
 +
                                "OUTPUT_DESC" => "International IP Transit - Vocus Auckland"
 +
                        }
 +
                        add_tag => [ "International IP Transit - Vocus Auckland", "International"]
 +
                }
 +
        }
 +
        if [INPUT_SNMP] == 588 {
 +
                mutate {
 +
                        add_field =>  {
 +
                                "INPUT_DESC" => "Peering - APE"
 +
                        }
 +
                        add_tag => [ "Peering - APE", "Domestic"]
 +
                }
 +
        }
 +
        if [OUTPUT_SNMP] == 588 {
 +
                mutate {
 +
                        add_field =>  {
 +
                                "OUTPUT_DESC" => "Peering - APE"
 +
                        }
 +
                        add_tag => [ "Peering - APE", "Domestic"]
 +
                }
 +
        }
 +
        if [INPUT_SNMP] == 1248 {
 +
                mutate {
 +
                        add_field =>  {
 +
                                "INPUT_DESC" => "Peering - MegaIX Auckland"
 +
                        }
 +
                        add_tag => [ "Peering - MegaIX Auckland", "Domestic"]
 +
                }
 +
        }
 +
        if [OUTPUT_SNMP] == 1248 {
 +
                mutate {
 +
                        add_field =>  {
 +
                                "OUTPUT_DESC" => "Peering - MegaIX Auckland"
 +
                        }
 +
                        add_tag => [ "Peering - MegaIX Auckland", "Domestic"]
 +
                }
 +
        }
 +
        if [INPUT_SNMP] == 609 {
 +
                mutate {
 +
                        add_field =>  {
 +
                                "INPUT_DESC" => "Peering - WIX"
 +
                        }
 +
                        add_tag => [ "Peering - WIX", "Domestic"]
 +
                }
 +
        }
 +
        if [OUTPUT_SNMP] == 609 {
 +
                mutate {
 +
                        add_field =>  {
 +
                                "OUTPUT_DESC" => "Peering - WIX"
 +
                        }
 +
                        add_tag => [ "Peering - WIX", "Domestic"]
 +
                }
 +
        }
 +
        if [INPUT_SNMP] == 572 {
 +
                mutate {
 +
                        add_field =>  {
 +
                                "INPUT_DESC" => "International IP Transit - Vocus Sydney"
 +
                        }
 +
                        add_tag => [ "International IP Transit - Vocus Sydney", "International"]
 +
                }
 +
        }
 +
        if [OUTPUT_SNMP] == 572 {
 +
                mutate {
 +
                        add_field =>  {
 +
                                "OUTPUT_DESC" => "International IP Transit - Vocus Sydney"
 +
                        }
 +
                        add_tag => [ "International IP Transit - Vocus Sydney", "International"]
 +
                }
 +
        }
 +
        if [INPUT_SNMP] == 566 {
 +
                mutate {
 +
                        add_field =>  {
 +
                                "INPUT_DESC" => "Peering - MegaIX Sydney"
 +
                        }
 +
                        add_tag => [ "Peering - MegaIX Sydney", "Domestic"]
 +
                }
 +
        }
 +
        if [OUTPUT_SNMP] == 566 {
 +
                mutate {
 +
                        add_field =>  {
 +
                                "OUTPUT_DESC" => "Peering - MegaIX Sydney"
 +
                        }
 +
                        add_tag => [ "Peering - MegaIX Sydney", "Domestic"]
 +
                }
 +
        }
 +
}
 +
</pre>
 +
 +
== Kibana ==
 +
The dashboard and visualisations i built for Kibana suit me nicely and are included here in case you want to use them.
 +
 +
=== Dashboard ===
 +
<pre>
 +
[
 +
  {
 +
    "_id": "Default",
 +
    "_type": "dashboard",
 +
    "_source": {
 +
      "title": "Default",
 +
      "hits": 0,
 +
      "description": "",
 +
      "panelsJSON": "[{\"col\":4,\"id\":\"Top-10-Destination-Addresses\",\"panelIndex\":4,\"row\":7,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Top-10-Source-Addresses\",\"panelIndex\":5,\"row\":7,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Input-Interface-Distribution\",\"panelIndex\":6,\"row\":7,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":10,\"id\":\"Output-Interface-Distribution\",\"panelIndex\":10,\"row\":7,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":10,\"id\":\"Total-Data-Transferred\",\"panelIndex\":11,\"row\":1,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Traffic-Over-Time\",\"panelIndex\":12,\"row\":1,\"size_x\":9,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Top-10-Source-Ports\",\"panelIndex\":13,\"row\":11,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":4,\"id\":\"Top-10-Destination-Ports\",\"panelIndex\":14,\"row\":11,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Packets-Over-Time\",\"panelIndex\":15,\"row\":4,\"size_x\":9,\"size_y\":3,\"type\":\"visualization\"},{\"col\":10,\"id\":\"Protocol-Distribution\",\"panelIndex\":16,\"row\":4,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"id\":\"Source-AS-Distribution\",\"type\":\"visualization\",\"panelIndex\":17,\"size_x\":3,\"size_y\":4,\"col\":7,\"row\":11},{\"id\":\"Destination-AS-Distribution\",\"type\":\"visualization\",\"panelIndex\":18,\"size_x\":3,\"size_y\":4,\"col\":10,\"row\":11}]",
 +
      "optionsJSON": "{\"darkTheme\":false}",
 +
      "uiStateJSON": "{\"P-1\":{\"vis\":{\"legendOpen\":false}},\"P-10\":{\"vis\":{\"legendOpen\":false}},\"P-12\":{\"vis\":{\"legendOpen\":false}},\"P-15\":{\"vis\":{\"legendOpen\":false}},\"P-16\":{\"vis\":{\"legendOpen\":false}},\"P-18\":{\"vis\":{\"legendOpen\":false}},\"P-17\":{\"vis\":{\"legendOpen\":false}}}",
 +
      "version": 1,
 +
      "timeRestore": false,
 +
      "kibanaSavedObjectMeta": {
 +
        "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}"
 +
      }
 +
    }
 +
  }
 +
]
 +
</pre>
 +
 +
=== Visualisations ===
 +
<pre>
 +
[
 +
  {
 +
    "_id": "Traffic-Over-Time",
 +
    "_type": "visualization",
 +
    "_source": {
 +
      "title": "Traffic Over Time",
 +
      "visState": "{\"title\":\"Traffic Over Time\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"defaultYExtents\":false,\"mode\":\"stacked\",\"scale\":\"linear\",\"setYExtents\":false,\"shareYAxis\":true,\"times\":[],\"yAxis\":{}},\"aggs\":[{\"id\":\"2\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\",\"customLabel\":\"Bytes\"}},{\"id\":\"3\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"\"}},{\"id\":\"4\",\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"Domestic\",\"analyze_wildcard\":true}}},\"label\":\"Domestic\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"International\",\"analyze_wildcard\":true}}},\"label\":\"International\"}]}}],\"listeners\":{}}",
 +
      "uiStateJSON": "{\"spy\":{\"mode\":{\"name\":null,\"fill\":false}},\"vis\":{\"colors\":{\"Bytes\":\"#629E51\",\"Domestic\":\"#7EB26D\",\"International\":\"#EA6460\"},\"legendOpen\":true}}",
 +
      "description": "",
 +
      "version": 1,
 +
      "kibanaSavedObjectMeta": {
 +
        "searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
 +
      }
 +
    }
 +
  },
 +
  {
 +
    "_id": "Output-Interface-Distribution",
 +
    "_type": "visualization",
 +
    "_source": {
 +
      "title": "Output Interface Distribution",
 +
      "visState": "{\"title\":\"Output Interface Distribution\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\",\"customLabel\":\"Bytes\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"OUTPUT_DESC.raw\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
 +
      "uiStateJSON": "{\"vis\":{\"legendOpen\":false},\"spy\":{\"mode\":{\"name\":null,\"fill\":false}}}",
 +
      "description": "",
 +
      "version": 1,
 +
      "kibanaSavedObjectMeta": {
 +
        "searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
 +
      }
 +
    }
 +
  },
 +
  {
 +
    "_id": "Input-Interface-Distribution",
 +
    "_type": "visualization",
 +
    "_source": {
 +
      "title": "Input Interface Distribution",
 +
      "visState": "{\"title\":\"Input Interface Distribution\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\",\"customLabel\":\"Bytes\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"INPUT_DESC.raw\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
 +
      "uiStateJSON": "{\"vis\":{\"legendOpen\":false},\"spy\":{\"mode\":{\"name\":null,\"fill\":false}}}",
 +
      "description": "",
 +
      "version": 1,
 +
      "kibanaSavedObjectMeta": {
 +
        "searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
 +
      }
 +
    }
 +
  },
 +
  {
 +
    "_id": "Destination-AS-Distribution",
 +
    "_type": "visualization",
 +
    "_source": {
 +
      "title": "Destination AS Distribution",
 +
      "visState": "{\"title\":\"Destination AS Distribution\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"DST_AS\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
 +
      "uiStateJSON": "{\"vis\":{\"legendOpen\":true},\"spy\":{\"mode\":{\"name\":null,\"fill\":false}}}",
 +
      "description": "",
 +
      "version": 1,
 +
      "kibanaSavedObjectMeta": {
 +
        "searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
 +
      }
 +
    }
 +
  },
 +
  {
 +
    "_id": "Top-10-Destination-Ports",
 +
    "_type": "visualization",
 +
    "_source": {
 +
      "title": "Top 10 Destination Ports",
 +
      "visState": "{\"title\":\"New Visualization\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\",\"customLabel\":\"Total Traffic\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"L4_DST_PORT\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Port\"}}],\"listeners\":{}}",
 +
      "uiStateJSON": "{}",
 +
      "description": "",
 +
      "version": 1,
 +
      "kibanaSavedObjectMeta": {
 +
        "searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
 +
      }
 +
    }
 +
  },
 +
  {
 +
    "_id": "Source-AS-Distribution",
 +
    "_type": "visualization",
 +
    "_source": {
 +
      "title": "Source AS Distribution",
 +
      "visState": "{\"title\":\"Source AS Distribution\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"SRC_AS\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
 +
      "uiStateJSON": "{}",
 +
      "description": "",
 +
      "version": 1,
 +
      "kibanaSavedObjectMeta": {
 +
        "searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
 +
      }
 +
    }
 +
  },
 +
  {
 +
    "_id": "Total-Data-Transferred",
 +
    "_type": "visualization",
 +
    "_source": {
 +
      "title": "Total Data Transferred",
 +
      "visState": "{\"aggs\":[{\"id\":\"1\",\"params\":{\"customLabel\":\"Total Data Transferred\",\"field\":\"IN_BYTES\"},\"schema\":\"metric\",\"type\":\"sum\"}],\"listeners\":{},\"params\":{\"fontSize\":\"40\",\"handleNoResults\":true},\"title\":\"Total Data Transferred\",\"type\":\"metric\"}",
 +
      "uiStateJSON": "{}",
 +
      "description": "",
 +
      "version": 1,
 +
      "kibanaSavedObjectMeta": {
 +
        "searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
 +
      }
 +
    }
 +
  },
 +
  {
 +
    "_id": "Top-10-Source-Addresses",
 +
    "_type": "visualization",
 +
    "_source": {
 +
      "title": "Top 10 Source Addresses",
 +
      "visState": "{\"title\":\"Top 10 Destination Addresses\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"2\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\",\"customLabel\":\"Total Bytes\"}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"IPV4_SRC_ADDR.raw\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"2\",\"customLabel\":\"Source Address\"}}],\"listeners\":{}}",
 +
      "uiStateJSON": "{}",
 +
      "description": "",
 +
      "version": 1,
 +
      "kibanaSavedObjectMeta": {
 +
        "searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
 +
      }
 +
    }
 +
  },
 +
  {
 +
    "_id": "Top-10-Destination-Addresses",
 +
    "_type": "visualization",
 +
    "_source": {
 +
      "title": "Top 10 Destination Addresses",
 +
      "visState": "{\"title\":\"Top 10 Destination Addresses\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"2\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\",\"customLabel\":\"Total Bytes\"}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"IPV4_DST_ADDR.raw\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"2\",\"customLabel\":\"Destination Address\"}}],\"listeners\":{}}",
 +
      "uiStateJSON": "{}",
 +
      "description": "",
 +
      "version": 1,
 +
      "kibanaSavedObjectMeta": {
 +
        "searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
 +
      }
 +
    }
 +
  },
 +
  {
 +
    "_id": "Top-10-Source-Ports",
 +
    "_type": "visualization",
 +
    "_source": {
 +
      "title": "Top 10 Source Ports",
 +
      "visState": "{\"title\":\"Top 10 Destination Ports\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\",\"customLabel\":\"Total Traffic\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"L4_SRC_PORT\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Port\"}}],\"listeners\":{}}",
 +
      "uiStateJSON": "{}",
 +
      "description": "",
 +
      "version": 1,
 +
      "kibanaSavedObjectMeta": {
 +
        "searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
 +
      }
 +
    }
 +
  },
 +
  {
 +
    "_id": "Protocol-Distribution",
 +
    "_type": "visualization",
 +
    "_source": {
 +
      "title": "Protocol Distribution",
 +
      "visState": "{\"title\":\"New Visualization\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\",\"customLabel\":\"\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"PROTOCOL\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
 +
      "uiStateJSON": "{}",
 +
      "description": "",
 +
      "version": 1,
 +
      "kibanaSavedObjectMeta": {
 +
        "searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
 +
      }
 +
    }
 +
  },
 +
  {
 +
    "_id": "Packets-Over-Time",
 +
    "_type": "visualization",
 +
    "_source": {
 +
      "title": "Packets Over Time",
 +
      "visState": "{\"title\":\"Packets Over Time\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"defaultYExtents\":false,\"mode\":\"stacked\",\"scale\":\"linear\",\"setYExtents\":false,\"shareYAxis\":true,\"times\":[],\"yAxis\":{}},\"aggs\":[{\"id\":\"2\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_PKTS\",\"customLabel\":\"Packets\"}},{\"id\":\"3\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"\"}},{\"id\":\"4\",\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"tags: Domestic\",\"analyze_wildcard\":true}}},\"label\":\"Domestic\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"tags: International\",\"analyze_wildcard\":true}}},\"label\":\"International\"}]}}],\"listeners\":{}}",
 +
      "uiStateJSON": "{\"spy\":{\"mode\":{\"name\":null,\"fill\":false}},\"vis\":{\"colors\":{\"Bytes\":\"#629E51\",\"Domestic\":\"#447EBC\",\"International\":\"#65C5DB\"},\"legendOpen\":true}}",
 +
      "description": "",
 +
      "version": 1,
 +
      "kibanaSavedObjectMeta": {
 +
        "searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
 +
      }
 +
    }
 +
  }
 +
]
 
</pre>
 
</pre>

Latest revision as of 21:35, 11 January 2018

Native IPFIX (Netflow V10) for Logstash is still in development but in the meantime i've been able to get it working thus:

Nprobe

I know, nprobe is commercial but i had it lying around so chose to use it. It can output in a variety of ways but in the base license model only zeromq and tcp methods allow direct output to logstash. My experience with zeromq was initially good but after a reinstall i couldn't get it working again, something wrong with the encoding or compression or something meant that logstash was seeing the data as garbage.

Switching to tcp output in json format got it working in the end so this is the nprobe config file, you'll note that local interface sniffing is disabled so it only listens for flows on port 2100: 

# cat /etc/nprobe/nprobe-eth0.conf
-i none 
--collector-port 2100 
#--json-labels
#--zmq tcp://*:5000 
--tcp localhost:5000
-V10
-g=/var/run/nprobe-eth0.pid

And the nprobe init files: 

touch /etc/nprobe/nprobe-eth0.start

Logstash

The logstash config is easy unless you want to filter the flows as they arrive: 

# cat /etc/logstash/conf.d/tcp.conf 
input {
	tcp {
		port => 5000
		type => "netflow"
		codec => "json"
	}
}

I chose to do some filtering to assign nice names and tags to my SNMP interface ids 

filter {
	if [INPUT_SNMP] == 863 { 
		mutate { 
			add_field =>  { 
				"INPUT_DESC" => "International IP Transit - Vocus Auckland" 
			} 
			add_tag => [ "International IP Transit - Vocus Auckland", "International"]
		} 
	}
        if [OUTPUT_SNMP] == 863 {
                mutate {
                        add_field =>  {
                                "OUTPUT_DESC" => "International IP Transit - Vocus Auckland"
                        }
                        add_tag => [ "International IP Transit - Vocus Auckland", "International"]
                }
        }
        if [INPUT_SNMP] == 588 {
                mutate {
                        add_field =>  {
                                "INPUT_DESC" => "Peering - APE"
                        }
                        add_tag => [ "Peering - APE", "Domestic"]
                }
        }
        if [OUTPUT_SNMP] == 588 {
                mutate {
                        add_field =>  {
                                "OUTPUT_DESC" => "Peering - APE"
                        }
                        add_tag => [ "Peering - APE", "Domestic"]
                }
        }
        if [INPUT_SNMP] == 1248 {
                mutate {
                        add_field =>  {
                                "INPUT_DESC" => "Peering - MegaIX Auckland"
                        }
                        add_tag => [ "Peering - MegaIX Auckland", "Domestic"]
                }
        }
        if [OUTPUT_SNMP] == 1248 {
                mutate {
                        add_field =>  {
                                "OUTPUT_DESC" => "Peering - MegaIX Auckland"
                        }
                        add_tag => [ "Peering - MegaIX Auckland", "Domestic"]
                }
        }
        if [INPUT_SNMP] == 609 {
                mutate {
                        add_field =>  {
                                "INPUT_DESC" => "Peering - WIX"
                        }
                        add_tag => [ "Peering - WIX", "Domestic"]
                }
        }
        if [OUTPUT_SNMP] == 609 {
                mutate {
                        add_field =>  {
                                "OUTPUT_DESC" => "Peering - WIX"
                        }
                        add_tag => [ "Peering - WIX", "Domestic"]
                }
        }
        if [INPUT_SNMP] == 572 {
                mutate {
                        add_field =>  {
                                "INPUT_DESC" => "International IP Transit - Vocus Sydney"
                        }
                        add_tag => [ "International IP Transit - Vocus Sydney", "International"]
                }
        }
        if [OUTPUT_SNMP] == 572 {
                mutate {
                        add_field =>  {
                                "OUTPUT_DESC" => "International IP Transit - Vocus Sydney"
                        }
                        add_tag => [ "International IP Transit - Vocus Sydney", "International"]
                }
        }
        if [INPUT_SNMP] == 566 {
                mutate {
                        add_field =>  {
                                "INPUT_DESC" => "Peering - MegaIX Sydney"
                        }
                        add_tag => [ "Peering - MegaIX Sydney", "Domestic"]
                }
        }
        if [OUTPUT_SNMP] == 566 {
                mutate {
                        add_field =>  {
                                "OUTPUT_DESC" => "Peering - MegaIX Sydney"
                        }
                        add_tag => [ "Peering - MegaIX Sydney", "Domestic"]
                }
        }
}

Kibana

The dashboard and visualisations i built for Kibana suit me nicely and are included here in case you want to use them.

Dashboard

[
  {
    "_id": "Default",
    "_type": "dashboard",
    "_source": {
      "title": "Default",
      "hits": 0,
      "description": "",
      "panelsJSON": "[{\"col\":4,\"id\":\"Top-10-Destination-Addresses\",\"panelIndex\":4,\"row\":7,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Top-10-Source-Addresses\",\"panelIndex\":5,\"row\":7,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Input-Interface-Distribution\",\"panelIndex\":6,\"row\":7,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":10,\"id\":\"Output-Interface-Distribution\",\"panelIndex\":10,\"row\":7,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":10,\"id\":\"Total-Data-Transferred\",\"panelIndex\":11,\"row\":1,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Traffic-Over-Time\",\"panelIndex\":12,\"row\":1,\"size_x\":9,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Top-10-Source-Ports\",\"panelIndex\":13,\"row\":11,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":4,\"id\":\"Top-10-Destination-Ports\",\"panelIndex\":14,\"row\":11,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Packets-Over-Time\",\"panelIndex\":15,\"row\":4,\"size_x\":9,\"size_y\":3,\"type\":\"visualization\"},{\"col\":10,\"id\":\"Protocol-Distribution\",\"panelIndex\":16,\"row\":4,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"id\":\"Source-AS-Distribution\",\"type\":\"visualization\",\"panelIndex\":17,\"size_x\":3,\"size_y\":4,\"col\":7,\"row\":11},{\"id\":\"Destination-AS-Distribution\",\"type\":\"visualization\",\"panelIndex\":18,\"size_x\":3,\"size_y\":4,\"col\":10,\"row\":11}]",
      "optionsJSON": "{\"darkTheme\":false}",
      "uiStateJSON": "{\"P-1\":{\"vis\":{\"legendOpen\":false}},\"P-10\":{\"vis\":{\"legendOpen\":false}},\"P-12\":{\"vis\":{\"legendOpen\":false}},\"P-15\":{\"vis\":{\"legendOpen\":false}},\"P-16\":{\"vis\":{\"legendOpen\":false}},\"P-18\":{\"vis\":{\"legendOpen\":false}},\"P-17\":{\"vis\":{\"legendOpen\":false}}}",
      "version": 1,
      "timeRestore": false,
      "kibanaSavedObjectMeta": {
        "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}"
      }
    }
  }
]

Visualisations

[
  {
    "_id": "Traffic-Over-Time",
    "_type": "visualization",
    "_source": {
      "title": "Traffic Over Time",
      "visState": "{\"title\":\"Traffic Over Time\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"defaultYExtents\":false,\"mode\":\"stacked\",\"scale\":\"linear\",\"setYExtents\":false,\"shareYAxis\":true,\"times\":[],\"yAxis\":{}},\"aggs\":[{\"id\":\"2\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\",\"customLabel\":\"Bytes\"}},{\"id\":\"3\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"\"}},{\"id\":\"4\",\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"Domestic\",\"analyze_wildcard\":true}}},\"label\":\"Domestic\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"International\",\"analyze_wildcard\":true}}},\"label\":\"International\"}]}}],\"listeners\":{}}",
      "uiStateJSON": "{\"spy\":{\"mode\":{\"name\":null,\"fill\":false}},\"vis\":{\"colors\":{\"Bytes\":\"#629E51\",\"Domestic\":\"#7EB26D\",\"International\":\"#EA6460\"},\"legendOpen\":true}}",
      "description": "",
      "version": 1,
      "kibanaSavedObjectMeta": {
        "searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
      }
    }
  },
  {
    "_id": "Output-Interface-Distribution",
    "_type": "visualization",
    "_source": {
      "title": "Output Interface Distribution",
      "visState": "{\"title\":\"Output Interface Distribution\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\",\"customLabel\":\"Bytes\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"OUTPUT_DESC.raw\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
      "uiStateJSON": "{\"vis\":{\"legendOpen\":false},\"spy\":{\"mode\":{\"name\":null,\"fill\":false}}}",
      "description": "",
      "version": 1,
      "kibanaSavedObjectMeta": {
        "searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
      }
    }
  },
  {
    "_id": "Input-Interface-Distribution",
    "_type": "visualization",
    "_source": {
      "title": "Input Interface Distribution",
      "visState": "{\"title\":\"Input Interface Distribution\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\",\"customLabel\":\"Bytes\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"INPUT_DESC.raw\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
      "uiStateJSON": "{\"vis\":{\"legendOpen\":false},\"spy\":{\"mode\":{\"name\":null,\"fill\":false}}}",
      "description": "",
      "version": 1,
      "kibanaSavedObjectMeta": {
        "searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
      }
    }
  },
  {
    "_id": "Destination-AS-Distribution",
    "_type": "visualization",
    "_source": {
      "title": "Destination AS Distribution",
      "visState": "{\"title\":\"Destination AS Distribution\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"DST_AS\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
      "uiStateJSON": "{\"vis\":{\"legendOpen\":true},\"spy\":{\"mode\":{\"name\":null,\"fill\":false}}}",
      "description": "",
      "version": 1,
      "kibanaSavedObjectMeta": {
        "searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
      }
    }
  },
  {
    "_id": "Top-10-Destination-Ports",
    "_type": "visualization",
    "_source": {
      "title": "Top 10 Destination Ports",
      "visState": "{\"title\":\"New Visualization\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\",\"customLabel\":\"Total Traffic\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"L4_DST_PORT\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Port\"}}],\"listeners\":{}}",
      "uiStateJSON": "{}",
      "description": "",
      "version": 1,
      "kibanaSavedObjectMeta": {
        "searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
      }
    }
  },
  {
    "_id": "Source-AS-Distribution",
    "_type": "visualization",
    "_source": {
      "title": "Source AS Distribution",
      "visState": "{\"title\":\"Source AS Distribution\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"SRC_AS\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
      "uiStateJSON": "{}",
      "description": "",
      "version": 1,
      "kibanaSavedObjectMeta": {
        "searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
      }
    }
  },
  {
    "_id": "Total-Data-Transferred",
    "_type": "visualization",
    "_source": {
      "title": "Total Data Transferred",
      "visState": "{\"aggs\":[{\"id\":\"1\",\"params\":{\"customLabel\":\"Total Data Transferred\",\"field\":\"IN_BYTES\"},\"schema\":\"metric\",\"type\":\"sum\"}],\"listeners\":{},\"params\":{\"fontSize\":\"40\",\"handleNoResults\":true},\"title\":\"Total Data Transferred\",\"type\":\"metric\"}",
      "uiStateJSON": "{}",
      "description": "",
      "version": 1,
      "kibanaSavedObjectMeta": {
        "searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
      }
    }
  },
  {
    "_id": "Top-10-Source-Addresses",
    "_type": "visualization",
    "_source": {
      "title": "Top 10 Source Addresses",
      "visState": "{\"title\":\"Top 10 Destination Addresses\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"2\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\",\"customLabel\":\"Total Bytes\"}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"IPV4_SRC_ADDR.raw\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"2\",\"customLabel\":\"Source Address\"}}],\"listeners\":{}}",
      "uiStateJSON": "{}",
      "description": "",
      "version": 1,
      "kibanaSavedObjectMeta": {
        "searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
      }
    }
  },
  {
    "_id": "Top-10-Destination-Addresses",
    "_type": "visualization",
    "_source": {
      "title": "Top 10 Destination Addresses",
      "visState": "{\"title\":\"Top 10 Destination Addresses\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"2\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\",\"customLabel\":\"Total Bytes\"}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"IPV4_DST_ADDR.raw\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"2\",\"customLabel\":\"Destination Address\"}}],\"listeners\":{}}",
      "uiStateJSON": "{}",
      "description": "",
      "version": 1,
      "kibanaSavedObjectMeta": {
        "searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
      }
    }
  },
  {
    "_id": "Top-10-Source-Ports",
    "_type": "visualization",
    "_source": {
      "title": "Top 10 Source Ports",
      "visState": "{\"title\":\"Top 10 Destination Ports\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\",\"customLabel\":\"Total Traffic\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"L4_SRC_PORT\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Port\"}}],\"listeners\":{}}",
      "uiStateJSON": "{}",
      "description": "",
      "version": 1,
      "kibanaSavedObjectMeta": {
        "searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
      }
    }
  },
  {
    "_id": "Protocol-Distribution",
    "_type": "visualization",
    "_source": {
      "title": "Protocol Distribution",
      "visState": "{\"title\":\"New Visualization\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\",\"customLabel\":\"\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"PROTOCOL\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
      "uiStateJSON": "{}",
      "description": "",
      "version": 1,
      "kibanaSavedObjectMeta": {
        "searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
      }
    }
  },
  {
    "_id": "Packets-Over-Time",
    "_type": "visualization",
    "_source": {
      "title": "Packets Over Time",
      "visState": "{\"title\":\"Packets Over Time\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"defaultYExtents\":false,\"mode\":\"stacked\",\"scale\":\"linear\",\"setYExtents\":false,\"shareYAxis\":true,\"times\":[],\"yAxis\":{}},\"aggs\":[{\"id\":\"2\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_PKTS\",\"customLabel\":\"Packets\"}},{\"id\":\"3\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"\"}},{\"id\":\"4\",\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"tags: Domestic\",\"analyze_wildcard\":true}}},\"label\":\"Domestic\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"tags: International\",\"analyze_wildcard\":true}}},\"label\":\"International\"}]}}],\"listeners\":{}}",
      "uiStateJSON": "{\"spy\":{\"mode\":{\"name\":null,\"fill\":false}},\"vis\":{\"colors\":{\"Bytes\":\"#629E51\",\"Domestic\":\"#447EBC\",\"International\":\"#65C5DB\"},\"legendOpen\":true}}",
      "description": "",
      "version": 1,
      "kibanaSavedObjectMeta": {
        "searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
      }
    }
  }
]
Retrieved from ‘https://www.initech.nz/w/index.php?title=Logstash_for_IPFIX&oldid=116’