Logstash geoip隨機失敗

Question

我使用logstash處理來自dnsmasq的日誌。 在DNS響應中，我使用「geoip」過濾器來豐富具有位置信息的請求。不幸的是，一些請求充滿了地理位置信息，其他請求沒有。

地理位置代碼：

沒有地理位置樣品IP地址的

 geoip { 
     source => "serverip" 
     } 

列表

104.156.81.217 
104.156.85.217 
104.16.92.65 
104.16.93.65 
104.16.94.65 
104.16.95.65 
104.16.96.65 
104.20.5.131 
104.20.6.131 
104.20.77.18 
104.20.78.18 
104.244.43.135 
104.244.43.167 
104.244.43.231 
104.244.43.39 
104.244.43.7 
104.28.30.27 
104.28.31.27 
104.40.196.5 
104.41.231.130 
104.45.95.112 
104.47.151.128 
104.71.97.80 
104.84.200.206 
104.90.129.122 
104.90.176.199 
104.90.176.77 
104.94.60.210 
104.98.119.204 
104.98.150.212 
162.255.119.124 
185.118.208.20 
185.19.196.101 
185.54.150.54 
185.63.147.12 
191.232.139.13 
191.233.80.151 
191.239.8.125 
192.229.233.25 
23.101.51.170 
23.196.235.245 
23.196.247.114 
23.196.249.86 
23.196.255.139 
23.197.0.60 
23.199.209.223 
23.235.33.217 
23.235.37.217 
23.97.173.24 

成功進入：

{ 
     "message" => "May 27 18:17:16 dnsmasq[385]: reply www.google.com is 216.58.213.228", 
     "@version" => "1", 
     "@timestamp" => "2016-05-27T18:17:17.147Z", 
      "path" => "/var/log/dnsmasq.log", 
      "host" => "dns", 
      "type" => "dnsmasq", 
    "reqtimestamp" => "May 27 18:17:16", 
     "program" => "dnsmasq", 
      "pid" => "385", 
      "action" => "reply", 
      "domain" => "www.google.com", 
     "function" => "is", 
     "serverip" => "216.58.213.228", 
      "geoip" => { 
         "ip" => "216.58.213.228", 
      "country_code2" => "US", 
      "country_code3" => "USA", 
      "country_name" => "United States", 
      "continent_code" => "NA", 
      "region_name" => "CA", 
       "city_name" => "Mountain View", 
      "postal_code" => "94043", 
       "latitude" => 37.41919999999999, 
       "longitude" => -122.0574, 
       "dma_code" => 807, 
       "area_code" => 650, 
       "timezone" => "America/Los_Angeles", 
     "real_region_name" => "California", 
       "location" => [ 
      [0] -122.0574, 
      [1] 37.41919999999999 
     ] 
    } 
} 

失敗的條目：

{ 
     "message" => "May 27 18:15:50 dnsmasq[385]: reply e5884.d.akamaiedge.net is 23.197.8.251", 
     "@version" => "1", 
     "@timestamp" => "2016-05-27T18:15:51.697Z", 
      "path" => "/var/log/dnsmasq.log", 
      "host" => "dns", 
      "type" => "dnsmasq", 
    "reqtimestamp" => "May 27 18:15:50", 
     "program" => "dnsmasq", 
      "pid" => "385", 
      "action" => "reply", 
      "domain" => "e5884.d.akamaiedge.net", 
     "function" => "is", 
     "serverip" => "23.197.8.251" 
} 

完整Logstash配置：

input { 
    file { 
    path => "/var/log/dnsmasq.log" 
    start_position => "beginning" 
    type => "dnsmasq" 
    } 
} 

# Mar 15 20:13:05 dnsmasq[346]: query[A] imap.gmail.com from 192.168.0.140 
# Mar 2 20:38:45 dnsmasq-dhcp[11856]: DHCPACK(eth0) 192.168.0.152 60:67:20:72:df:00 E0199149 
# Mar 15 21:55:34 dnsmasq-dhcp[346]: 3806132383 DHCPACK(eth0) 192.168.0.80 04:0c:ce:d1:af:18 Air-de-irobot 
# Mar 16 08:54:31 dnsmasq-dhcp[346]: 4280587370 DHCPACK(eth0) 192.168.0.158 48:9d:24:ae:0e:00 BB-JP 
# Mar 16 08:18:49 dnsmasq[346]: /etc/pihole/gravity.list ssl.google-analytics.com is 192.168.0.2 

filter { 
    if [type] == "dnsmasq" { 
    grok { 
     match => [ "message", "%{SYSLOGTIMESTAMP:reqtimestamp} %{USER:program}\[%{NONNEGINT:pid}\]\: ?(%{NONNEGINT:num})?%{NOTSPACE:action} %{IP:clientip} %{MAC:clientmac} ?(%{HOSTNAME:clientname})?"] 
     match => [ "message", "%{SYSLOGTIMESTAMP:reqtimestamp} %{USER:program}\[%{NONNEGINT:pid}\]\: ?(%{NONNEGINT:num})?%{USER:action}?(\[%{USER:subaction}\])? %{NOTSPACE:domain} %{NOTSPACE:function} %{IP:clientip}"] 

     match => [ "message", "%{SYSLOGTIMESTAMP:reqtimestamp} %{USER:program}\[%{NONNEGINT:pid}\]\: %{NOTSPACE:action} %{DATA:data}"] 
    } 

    if [action] =~ "DHCPACK" { 
     if ![clientname] { 
     mutate { 
      add_field => { "clientname" => "No name" } 
     } 
     } 
     aggregate { 
     task_id => "%{clientip}" 
     code => "map['clientmac'] = event['clientmac']; map['clientname'] = event['clientname'];" 
     map_action => "create_or_update" 
     # timeout = 0 sets the timeout to the default value 1800 seconds. 
     timeout => 172800 
     } 
    } else if [action] == "query" { 
     aggregate { 
     task_id => "%{clientip}" 
     code => "event['clientmac'] = map['clientmac']; event['clientname'] = map['clientname']" 
     map_action => "update" 
     } 
     if ![clientname] { 
     mutate { 
      add_field => { "clientname" => "%{clientip}" } 
     } 
     } 
     if ![clientmac] { 
     mutate { 
      add_field => { "clientmac" => "%{clientip}" } 
     } 
     } 
    } else if [action] == "reply" { 
     mutate { 
      rename => { "clientip" => "serverip" } 
     } 
     geoip { 
     source => "serverip" 
     } 
    } else 
    { 
     drop{} 
    } 
    } 
} 
output { 
# elasticsearch { hosts => ["localhost:9200"] } 
stdout { codec => rubydebug } 
} 

Answer 1

最新版本Logstash（2.3.2）捆綁logstash geoip filter 2.0.7，其又包括從2013（GeoLiteCity-2013-01-18.dat）舊的MaxMind的GeoIP的數據庫如可從可見調試日誌

Using geoip database {:path=>"/usr/local/Cellar/logstash/2.3.2/vendor/bundle/jruby/1.9/gems/logstash-filter-geoip-2.0.7/vendor/GeoLiteCity-2013-01-18.dat", :level=>:info, :file=>"logstash/filters/geoip.rb", :line=>"97", :method=>"register"} 

的MaxMind有一個新的GeoIP2服務，您可以嘗試here並且將正確的地址解析所有從上面列表中的IP地址。

就Logstash geoip而言，從GeoIP切換到GeoIP2 has been made in March並將在next Logstash 5.0 version中可用。 OMG。