Logstash for IPFIX
From Initech Technical Wiki
Native IPFIX (Netflow V10) for Logstash is still in development but in the meantime i've been able to get it working thus:
Nprobe
I know, nprobe is commercial but i had it lying around so chose to use it. It can output in a variety of ways but in the base license model only zeromq and tcp methods allow direct output to logstash. My experience with zeromq was initially good but after a reinstall i couldn't get it working again, something wrong with the encoding or compression or something meant that logstash was seeing the data as garbage.
Switching to tcp output in json format got it working in the end so this is the nprobe config file, you'll note that local interface sniffing is disabled so it only listens for flows on port 2100:
# cat /etc/nprobe/nprobe-eth0.conf -i none --collector-port 2100 #--json-labels #--zmq tcp://*:5000 --tcp localhost:5000 -V10 -g=/var/run/nprobe-eth0.pid
And the nprobe init files:
touch /etc/nprobe/nprobe-eth0.start
Logstash
The logstash config is easy unless you want to filter the flows as they arrive:
# cat /etc/logstash/conf.d/tcp.conf
input {
tcp {
port => 5000
type => "netflow"
codec => "json"
}
}
I chose to do some filtering to assign nice names and tags to my SNMP interface ids
filter {
if [INPUT_SNMP] == 863 {
mutate {
add_field => {
"INPUT_DESC" => "International IP Transit - Vocus Auckland"
}
add_tag => [ "International IP Transit - Vocus Auckland", "International"]
}
}
if [OUTPUT_SNMP] == 863 {
mutate {
add_field => {
"OUTPUT_DESC" => "International IP Transit - Vocus Auckland"
}
add_tag => [ "International IP Transit - Vocus Auckland", "International"]
}
}
if [INPUT_SNMP] == 588 {
mutate {
add_field => {
"INPUT_DESC" => "Peering - APE"
}
add_tag => [ "Peering - APE", "Domestic"]
}
}
if [OUTPUT_SNMP] == 588 {
mutate {
add_field => {
"OUTPUT_DESC" => "Peering - APE"
}
add_tag => [ "Peering - APE", "Domestic"]
}
}
if [INPUT_SNMP] == 1248 {
mutate {
add_field => {
"INPUT_DESC" => "Peering - MegaIX Auckland"
}
add_tag => [ "Peering - MegaIX Auckland", "Domestic"]
}
}
if [OUTPUT_SNMP] == 1248 {
mutate {
add_field => {
"OUTPUT_DESC" => "Peering - MegaIX Auckland"
}
add_tag => [ "Peering - MegaIX Auckland", "Domestic"]
}
}
if [INPUT_SNMP] == 609 {
mutate {
add_field => {
"INPUT_DESC" => "Peering - WIX"
}
add_tag => [ "Peering - WIX", "Domestic"]
}
}
if [OUTPUT_SNMP] == 609 {
mutate {
add_field => {
"OUTPUT_DESC" => "Peering - WIX"
}
add_tag => [ "Peering - WIX", "Domestic"]
}
}
if [INPUT_SNMP] == 572 {
mutate {
add_field => {
"INPUT_DESC" => "International IP Transit - Vocus Sydney"
}
add_tag => [ "International IP Transit - Vocus Sydney", "International"]
}
}
if [OUTPUT_SNMP] == 572 {
mutate {
add_field => {
"OUTPUT_DESC" => "International IP Transit - Vocus Sydney"
}
add_tag => [ "International IP Transit - Vocus Sydney", "International"]
}
}
if [INPUT_SNMP] == 566 {
mutate {
add_field => {
"INPUT_DESC" => "Peering - MegaIX Sydney"
}
add_tag => [ "Peering - MegaIX Sydney", "Domestic"]
}
}
if [OUTPUT_SNMP] == 566 {
mutate {
add_field => {
"OUTPUT_DESC" => "Peering - MegaIX Sydney"
}
add_tag => [ "Peering - MegaIX Sydney", "Domestic"]
}
}
}
Kibana
The dashboard and visualisations i built for Kibana suit me nicely and are included here in case you want to use them.
Dashboard
[
{
"_id": "Default",
"_type": "dashboard",
"_source": {
"title": "Default",
"hits": 0,
"description": "",
"panelsJSON": "[{\"col\":4,\"id\":\"Top-10-Destination-Addresses\",\"panelIndex\":4,\"row\":7,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Top-10-Source-Addresses\",\"panelIndex\":5,\"row\":7,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Input-Interface-Distribution\",\"panelIndex\":6,\"row\":7,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":10,\"id\":\"Output-Interface-Distribution\",\"panelIndex\":10,\"row\":7,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":10,\"id\":\"Total-Data-Transferred\",\"panelIndex\":11,\"row\":1,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Traffic-Over-Time\",\"panelIndex\":12,\"row\":1,\"size_x\":9,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Top-10-Source-Ports\",\"panelIndex\":13,\"row\":11,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":4,\"id\":\"Top-10-Destination-Ports\",\"panelIndex\":14,\"row\":11,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Packets-Over-Time\",\"panelIndex\":15,\"row\":4,\"size_x\":9,\"size_y\":3,\"type\":\"visualization\"},{\"col\":10,\"id\":\"Protocol-Distribution\",\"panelIndex\":16,\"row\":4,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"id\":\"Source-AS-Distribution\",\"type\":\"visualization\",\"panelIndex\":17,\"size_x\":3,\"size_y\":4,\"col\":7,\"row\":11},{\"id\":\"Destination-AS-Distribution\",\"type\":\"visualization\",\"panelIndex\":18,\"size_x\":3,\"size_y\":4,\"col\":10,\"row\":11}]",
"optionsJSON": "{\"darkTheme\":false}",
"uiStateJSON": "{\"P-1\":{\"vis\":{\"legendOpen\":false}},\"P-10\":{\"vis\":{\"legendOpen\":false}},\"P-12\":{\"vis\":{\"legendOpen\":false}},\"P-15\":{\"vis\":{\"legendOpen\":false}},\"P-16\":{\"vis\":{\"legendOpen\":false}},\"P-18\":{\"vis\":{\"legendOpen\":false}},\"P-17\":{\"vis\":{\"legendOpen\":false}}}",
"version": 1,
"timeRestore": false,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}"
}
}
}
]
Visualisations
[
{
"_id": "Traffic-Over-Time",
"_type": "visualization",
"_source": {
"title": "Traffic Over Time",
"visState": "{\"title\":\"Traffic Over Time\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"defaultYExtents\":false,\"mode\":\"stacked\",\"scale\":\"linear\",\"setYExtents\":false,\"shareYAxis\":true,\"times\":[],\"yAxis\":{}},\"aggs\":[{\"id\":\"2\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\",\"customLabel\":\"Bytes\"}},{\"id\":\"3\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"\"}},{\"id\":\"4\",\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"Domestic\",\"analyze_wildcard\":true}}},\"label\":\"Domestic\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"International\",\"analyze_wildcard\":true}}},\"label\":\"International\"}]}}],\"listeners\":{}}",
"uiStateJSON": "{\"spy\":{\"mode\":{\"name\":null,\"fill\":false}},\"vis\":{\"colors\":{\"Bytes\":\"#629E51\",\"Domestic\":\"#7EB26D\",\"International\":\"#EA6460\"},\"legendOpen\":true}}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
}
}
},
{
"_id": "Output-Interface-Distribution",
"_type": "visualization",
"_source": {
"title": "Output Interface Distribution",
"visState": "{\"title\":\"Output Interface Distribution\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\",\"customLabel\":\"Bytes\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"OUTPUT_DESC.raw\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
"uiStateJSON": "{\"vis\":{\"legendOpen\":false},\"spy\":{\"mode\":{\"name\":null,\"fill\":false}}}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
}
}
},
{
"_id": "Input-Interface-Distribution",
"_type": "visualization",
"_source": {
"title": "Input Interface Distribution",
"visState": "{\"title\":\"Input Interface Distribution\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\",\"customLabel\":\"Bytes\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"INPUT_DESC.raw\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
"uiStateJSON": "{\"vis\":{\"legendOpen\":false},\"spy\":{\"mode\":{\"name\":null,\"fill\":false}}}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
}
}
},
{
"_id": "Destination-AS-Distribution",
"_type": "visualization",
"_source": {
"title": "Destination AS Distribution",
"visState": "{\"title\":\"Destination AS Distribution\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"DST_AS\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
"uiStateJSON": "{\"vis\":{\"legendOpen\":true},\"spy\":{\"mode\":{\"name\":null,\"fill\":false}}}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
}
}
},
{
"_id": "Top-10-Destination-Ports",
"_type": "visualization",
"_source": {
"title": "Top 10 Destination Ports",
"visState": "{\"title\":\"New Visualization\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\",\"customLabel\":\"Total Traffic\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"L4_DST_PORT\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Port\"}}],\"listeners\":{}}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
}
}
},
{
"_id": "Source-AS-Distribution",
"_type": "visualization",
"_source": {
"title": "Source AS Distribution",
"visState": "{\"title\":\"Source AS Distribution\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"SRC_AS\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
}
}
},
{
"_id": "Total-Data-Transferred",
"_type": "visualization",
"_source": {
"title": "Total Data Transferred",
"visState": "{\"aggs\":[{\"id\":\"1\",\"params\":{\"customLabel\":\"Total Data Transferred\",\"field\":\"IN_BYTES\"},\"schema\":\"metric\",\"type\":\"sum\"}],\"listeners\":{},\"params\":{\"fontSize\":\"40\",\"handleNoResults\":true},\"title\":\"Total Data Transferred\",\"type\":\"metric\"}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
}
}
},
{
"_id": "Top-10-Source-Addresses",
"_type": "visualization",
"_source": {
"title": "Top 10 Source Addresses",
"visState": "{\"title\":\"Top 10 Destination Addresses\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"2\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\",\"customLabel\":\"Total Bytes\"}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"IPV4_SRC_ADDR.raw\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"2\",\"customLabel\":\"Source Address\"}}],\"listeners\":{}}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
}
}
},
{
"_id": "Top-10-Destination-Addresses",
"_type": "visualization",
"_source": {
"title": "Top 10 Destination Addresses",
"visState": "{\"title\":\"Top 10 Destination Addresses\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"2\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\",\"customLabel\":\"Total Bytes\"}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"IPV4_DST_ADDR.raw\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"2\",\"customLabel\":\"Destination Address\"}}],\"listeners\":{}}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
}
}
},
{
"_id": "Top-10-Source-Ports",
"_type": "visualization",
"_source": {
"title": "Top 10 Source Ports",
"visState": "{\"title\":\"Top 10 Destination Ports\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\",\"customLabel\":\"Total Traffic\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"L4_SRC_PORT\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Port\"}}],\"listeners\":{}}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
}
}
},
{
"_id": "Protocol-Distribution",
"_type": "visualization",
"_source": {
"title": "Protocol Distribution",
"visState": "{\"title\":\"New Visualization\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_BYTES\",\"customLabel\":\"\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"PROTOCOL\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
}
}
},
{
"_id": "Packets-Over-Time",
"_type": "visualization",
"_source": {
"title": "Packets Over Time",
"visState": "{\"title\":\"Packets Over Time\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"defaultYExtents\":false,\"mode\":\"stacked\",\"scale\":\"linear\",\"setYExtents\":false,\"shareYAxis\":true,\"times\":[],\"yAxis\":{}},\"aggs\":[{\"id\":\"2\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"IN_PKTS\",\"customLabel\":\"Packets\"}},{\"id\":\"3\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"\"}},{\"id\":\"4\",\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"tags: Domestic\",\"analyze_wildcard\":true}}},\"label\":\"Domestic\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"tags: International\",\"analyze_wildcard\":true}}},\"label\":\"International\"}]}}],\"listeners\":{}}",
"uiStateJSON": "{\"spy\":{\"mode\":{\"name\":null,\"fill\":false}},\"vis\":{\"colors\":{\"Bytes\":\"#629E51\",\"Domestic\":\"#447EBC\",\"International\":\"#65C5DB\"},\"legendOpen\":true}}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"logstash-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
}
}
}
]