Elasticsearch從logstash映射配置

Question

我試圖在Windows上配置Elasticsearch從Logstash索引模板管理。

我有c:\ulyaoth\logstash-2.3.1\bin\logstash.json文件：

input { 
     beats { 
     port => 5044 
     type => "log" 
     } 
    } 

    filter { 
     grok { 
      match => ["message","%{TIMESTAMP_ISO8601:timestamp_match}"] 
      remove_field => ["_id","_index","_score","_type","beat.hostname","beat.name","count","fileds","host","input_type","offset","tags","type"] 
     } 

     mutate { 
      remove_field => ["_id","_index","_score","_type","beat.hostname","beat.name","count","fileds","host","input_type","offset","tags","type"] 
     } 

     date { 
      match => ["timestamp_match","YYYY-MM-dd HH:mm:ss.SSS"] 
      target => "timestamp_match" 
     } 
    } 

    output { 
     elasticsearch { 
     hosts => "localhost:9200" 
     index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}" 
     document_type => "%{[@metadata][type]}" 
     template => "c:/ulyaoth/logstash-2.3.1/bin/elasticsearch-template.custom.json" 
     template_name => "elasticsearch-template" 
     manage_template => true 
     template_overwrite => true 
     } 
    } 

和模板文件c:/ulyaoth/logstash-2.3.1/bin/elasticsearch-template.custom.json。我從c:\ulyaoth\logstash-2.3.1\vendor\bundle\jruby\1.9\gems\logstash-output-elasticsearch-2.5.5-java\lib\logstash\outputs\elasticsearch\挖出了這個文件，並修改它，以便：

"source":{"index": "not_analyzed"} 

這是整個文件：

{ 
     "template" : "logstash-*", 
     "settings" : { 
     "index.refresh_interval" : "5s" 
     }, 
     "mappings" : { 
     "_default_" : { 
      "_all" : {"enabled" : true, "omit_norms" : true}, 
      "dynamic_templates" : [ { 
      "message_field" : { 
       "match" : "message", 
       "match_mapping_type" : "string", 
       "mapping" : { 
       "type" : "string", "index" : "analyzed", "omit_norms" : true, 
       "fielddata" : { "format" : "disabled" } 
       } 
      } 
      }, { 
      "string_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "string", 
       "mapping" : { 
       "type" : "string", "index" : "analyzed", "omit_norms" : true, 
       "fielddata" : { "format" : "disabled" }, 
       "fields" : { 
        "raw" : {"type": "string", "index" : "not_analyzed", "doc_values" : true, "ignore_above" : 256} 
       } 
       } 
      } 
      }, { 
      "float_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "float", 
       "mapping" : { "type" : "float", "doc_values" : true } 
      } 
      }, { 
      "double_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "double", 
       "mapping" : { "type" : "double", "doc_values" : true } 
      } 
      }, { 
      "byte_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "byte", 
       "mapping" : { "type" : "byte", "doc_values" : true } 
      } 
      }, { 
      "short_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "short", 
       "mapping" : { "type" : "short", "doc_values" : true } 
      } 
      }, { 
      "integer_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "integer", 
       "mapping" : { "type" : "integer", "doc_values" : true } 
      } 
      }, { 
      "long_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "long", 
       "mapping" : { "type" : "long", "doc_values" : true } 
      } 
      }, { 
      "date_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "date", 
       "mapping" : { "type" : "date", "doc_values" : true } 
      } 
      }, { 
      "geo_point_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "geo_point", 
       "mapping" : { "type" : "geo_point", "doc_values" : true } 
      } 
      } ], 
      "properties" : { 
      "@timestamp": { "type": "date", "doc_values" : true }, 
      "@version": { "type": "string", "index": "not_analyzed", "doc_values" : true }, 
      "source":{"index": "not_analyzed"} 
      "geoip" : { 
       "type" : "object", 
       "dynamic": true, 
       "properties" : { 
       "ip": { "type": "ip", "doc_values" : true }, 
       "location" : { "type" : "geo_point", "doc_values" : true }, 
       "latitude" : { "type" : "float", "doc_values" : true }, 
       "longitude" : { "type" : "float", "doc_values" : true } 
       } 
      } 
      } 
     } 
     } 
    } 

我的問題是，該模板未註冊。 REST查詢返回空對象，並且我還看到仍在Kibana中分析的字段。

GET /_template HTTP/1.1 
    Host: 127.0.0.1:9200 

另一個問題是remove_field也不起作用 - 我仍然可以看到所有這些領域。

remove_field => ["_id","_index","_score","_type","beat.hostname","beat.name","count","fileds","host","input_type","offset","tags","type"] 

我沒有看到任何logstash日誌（具有諷刺意味的是），在ES日誌中我沒有看到任何錯誤或模板問題。

如何解決這些問題？

編輯：

最終工作配置是：

{ 
     "template" : "filebeat-*", 
     "settings" : { 
     "index.refresh_interval" : "5s" 
     }, 
     "mappings" : { 
     "_default_" : { 
      "_all" : {"enabled" : true, "omit_norms" : true}, 
      "dynamic_templates" : [ { 
      "message_field" : { 
       "match" : "message", 
       "match_mapping_type" : "string", 
       "mapping" : { 
       "type" : "string", "index" : "analyzed", "omit_norms" : true, 
       "fielddata" : { "format" : "disabled" } 
       } 
      } 
      }, { 
      "string_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "string", 
       "mapping" : { 
       "type" : "string", "index" : "analyzed", "omit_norms" : true, 
       "fielddata" : { "format" : "disabled" }, 
       "fields" : { 
        "raw" : {"type": "string", "index" : "not_analyzed", "doc_values" : true, "ignore_above" : 256} 
       } 
       } 
      } 
      }, { 
      "float_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "float", 
       "mapping" : { "type" : "float", "doc_values" : true } 
      } 
      }, { 
      "double_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "double", 
       "mapping" : { "type" : "double", "doc_values" : true } 
      } 
      }, { 
      "byte_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "byte", 
       "mapping" : { "type" : "byte", "doc_values" : true } 
      } 
      }, { 
      "short_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "short", 
       "mapping" : { "type" : "short", "doc_values" : true } 
      } 
      }, { 
      "integer_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "integer", 
       "mapping" : { "type" : "integer", "doc_values" : true } 
      } 
      }, { 
      "long_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "long", 
       "mapping" : { "type" : "long", "doc_values" : true } 
      } 
      }, { 
      "date_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "date", 
       "mapping" : { "type" : "date", "doc_values" : true } 
      } 
      }, { 
      "geo_point_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "geo_point", 
       "mapping" : { "type" : "geo_point", "doc_values" : true } 
      } 
      } ], 
      "properties" : { 
      "@timestamp": { "type": "date", "doc_values" : true }, 
      "@version": { "type": "string", "index": "not_analyzed", "doc_values" : true }, 
      "source":{ "type": "string", "index": "not_analyzed"} 
      "geoip" : { 
       "type" : "object", 
       "dynamic": true, 
       "properties" : { 
       "ip": { "type": "ip", "doc_values" : true }, 
       "location" : { "type" : "geo_point", "doc_values" : true }, 
       "latitude" : { "type" : "float", "doc_values" : true }, 
       "longitude" : { "type" : "float", "doc_values" : true } 
       } 
      } 
      } 
     } 
     } 
    } 

• 改變
  "template" : "filebeat-*"， 和"source":{ "type": "string", "index": "not_analyzed"}

Answer 1

的source FIEL d沒有type。也許你的意思是：

"source":{ "type": "string", "index": "not_analyzed"},