0
我正在嘗試將logstash 1.5.3升級到logstash 2.3.4。爲1.5.3Logstash 2.3.4試圖在elasticsearch中安裝模板時卡住
logstash配置看起來像如下： -
input {
kafka {
zk_connect => "kafka:2181"
group_id => "logstash"
topic_id => "logstash_logs2"
reset_beginning => false
consumer_threads => 3
}
}
filter {
if [app] == "walle_slowquery" or [app] == "walle_slowindex" {
ruby {
code => "event['timestamp'] = event['@timestamp']"
}
}
grok {
match => [
"timestamp", "^(?<app_log_time>%{YEAR}-%{MONTHNUM}-%{MONTHDAY})"
]
}
mutate {
rename => {
"app_log_time" => "[@metadata][app_log_time]"
}
}
}
output {
if [env] == "prod" or [env] == "common" {
elasticsearch {
index => "jabong-%{env}-%{app}-%{iver}-%{[@metadata][app_log_time]}"
cluster => "elasticsearch"
host => ["172.16.84.230:9300"]
protocol => "transport"
}
file {
path => "/var/log/shop/%{env}/%{app}/%{app}_%{host}_%{[@metadata][app_log_time]}.log"
}
stdout { codec => rubydebug }
}
}
我體改如下面2.3.4的logstash配置： -
input {
kafka {
zk_connect => "kafka:2181"
group_id => "logstash"
topic_id => "logstash_logs2"
reset_beginning => false
consumer_threads => 3
}
}
filter {
if [app] == "walle_slowquery" or [app] == "walle_slowindex" {
ruby {
code => "event['timestamp'] = event['@timestamp']"
}
}
grok {
match => [
"timestamp", "^(?<app_log_time>%{YEAR}-%{MONTHNUM}-%{MONTHDAY})"
]
}
mutate {
rename => {
"app_log_time" => "[@metadata][app_log_time]"
}
}
}
output {
if [env] == "prod" or [env] == "common" {
elasticsearch_java {
#For daily index creation used the time notation, Remove if not required.
index => "jabong-%{env}-%{app}-%{iver}-%{[@metadata][app_log_time]}"
cluster => "elasticsearch"
network_host => "172.16.84.230"
hosts => ["172.16.84.230:9300"]
protocol => "transport"
}
file {
path => "/var/log/shop/%{env}/%{app}/%{app}_%{host}_%{[@metadata][app_log_time]}.log"
}
stdout { codec => rubydebug }
}
}
在我使用自定義的模板指數jabong-* elasticsearch： -
{
"order": 0,
"template": "jabong-*",
"settings": {
"index.refresh_interval": "5s"
},
"mappings": {
"_default_": {
"dynamic_templates": [
{
"string_fields": {
"mapping": {
"index": "analyzed",
"omit_norms": true,
"type": "string",
"fields": {
"raw": {
"ignore_above": 256,
"index": "not_analyzed",
"type": "string"
}
}
},
"match_mapping_type": "string",
"match": "*"
}
}
],
"_all": {
"omit_norms": true,
"enabled": true
},
"properties": {
"geoip": {
"dynamic": true,
"type": "object",
"properties": {
"location": {
"type": "geo_point"
}
}
},
"@version": {
"index": "not_analyzed",
"type": "string"
}
}
}
},
"aliases": {}
}
我正在使用Kafka 0.8和logstash-input-kafka插件位於2.0.8，Elasticsearch的版本是1.7.1。
如果我開始在詳細模式logstash，logstash似乎與下面的消息被卡住： -
[email protected]:/opt/logstash-2.3.4# bin/logstash -f conf/logstash_indexer.conf --debug
Reading config file {:config_file=>"/opt/logstash-2.3.4/conf/logstash_indexer.conf", :level=>:debug, :file=>"logstash/config/loader.rb", :line=>"69", :method=>"local_config"}
Plugin not defined in namespace, checking for plugin file {:type=>"input", :name=>"kafka", :path=>"logstash/inputs/kafka", :level=>:debug, :file=>"logstash/plugin.rb", :line=>"76", :method=>"lookup"}
Plugin not defined in namespace, checking for plugin file {:type=>"codec", :name=>"json", :path=>"logstash/codecs/json", :level=>:debug, :file=>"logstash/plugin.rb", :line=>"76", :method=>"lookup"}
config LogStash::Codecs::JSON/@charset = "UTF-8" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@zk_connect = "kafka:2181" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@group_id = "logstash" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@topic_id = "logstash_logs2" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@reset_beginning = false {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@consumer_threads = 3 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@codec = <LogStash::Codecs::JSON charset=>"UTF-8"> {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@add_field = {} {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@white_list = nil {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@black_list = nil {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@auto_offset_reset = "largest" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@auto_commit_interval_ms = 1000 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@queue_size = 20 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@rebalance_max_retries = 4 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@rebalance_backoff_ms = 2000 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@consumer_timeout_ms = -1 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@consumer_restart_on_error = true {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@consumer_restart_sleep_ms = 0 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@decorate_events = false {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@consumer_id = nil {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@fetch_message_max_bytes = 1048576 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@decoder_class = "kafka.serializer.DefaultDecoder" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Inputs::Kafka/@key_decoder_class = "kafka.serializer.DefaultDecoder" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
Plugin not defined in namespace, checking for plugin file {:type=>"filter", :name=>"ruby", :path=>"logstash/filters/ruby", :level=>:debug, :file=>"logstash/plugin.rb", :line=>"76", :method=>"lookup"}
config LogStash::Filters::Ruby/@code = "event['timestamp'] = event['@timestamp']" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Ruby/@add_tag = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Ruby/@remove_tag = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Ruby/@add_field = {} {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Ruby/@remove_field = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Ruby/@periodic_flush = false {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
Plugin not defined in namespace, checking for plugin file {:type=>"filter", :name=>"grok", :path=>"logstash/filters/grok", :level=>:debug, :file=>"logstash/plugin.rb", :line=>"76", :method=>"lookup"}
config LogStash::Filters::Grok/@match = {"timestamp"=>"^(?<app_log_time>%{YEAR}-%{MONTHNUM}-%{MONTHDAY})"} {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Grok/@add_tag = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Grok/@remove_tag = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Grok/@add_field = {} {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Grok/@remove_field = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Grok/@periodic_flush = false {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Grok/@patterns_dir = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Grok/@patterns_files_glob = "*" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Grok/@break_on_match = true {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Grok/@named_captures_only = true {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Grok/@keep_empty_captures = false {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Grok/@singles = true {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Grok/@tag_on_failure = ["_grokparsefailure"] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Grok/@overwrite = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
Plugin not defined in namespace, checking for plugin file {:type=>"filter", :name=>"mutate", :path=>"logstash/filters/mutate", :level=>:debug, :file=>"logstash/plugin.rb", :line=>"76", :method=>"lookup"}
config LogStash::Filters::Mutate/@rename = {"app_log_time"=>"[@metadata][app_log_time]"} {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Mutate/@add_tag = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Mutate/@remove_tag = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Mutate/@add_field = {} {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Mutate/@remove_field = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Filters::Mutate/@periodic_flush = false {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
Plugin not defined in namespace, checking for plugin file {:type=>"output", :name=>"elasticsearch_java", :path=>"logstash/outputs/elasticsearch_java", :level=>:debug, :file=>"logstash/plugin.rb", :line=>"76", :method=>"lookup"}
Plugin not defined in namespace, checking for plugin file {:type=>"output", :name=>"file", :path=>"logstash/outputs/file", :level=>:debug, :file=>"logstash/plugin.rb", :line=>"76", :method=>"lookup"}
Plugin not defined in namespace, checking for plugin file {:type=>"output", :name=>"stdout", :path=>"logstash/outputs/stdout", :level=>:debug, :file=>"logstash/plugin.rb", :line=>"76", :method=>"lookup"}
starting agent {:level=>:info, :file=>"logstash/agent.rb", :line=>"207", :method=>"execute"}
starting pipeline {:id=>"main", :level=>:info, :file=>"logstash/agent.rb", :line=>"469", :method=>"start_pipeline"}
Settings: Default pipeline workers: 12
log4j java properties setup {:log4j_level=>"DEBUG", :level=>:debug, :file=>"logstash/logging.rb", :line=>"89", :method=>"setup_log4j"}
Registering kafka {:group_id=>"logstash", :topic_id=>"logstash_logs2", :zk_connect=>"kafka:2181", :level=>:info, :file=>"logstash/inputs/kafka.rb", :line=>"133", :method=>"register"}
Running kafka {:group_id=>"logstash", :topic_id=>"logstash_logs2", :zk_connect=>"kafka:2181", :level=>:info, :file=>"logstash/inputs/kafka.rb", :line=>"140", :method=>"run"}
Plugin not defined in namespace, checking for plugin file {:type=>"codec", :name=>"plain", :path=>"logstash/codecs/plain", :level=>:debug, :file=>"logstash/plugin.rb", :line=>"76", :method=>"lookup"}
config LogStash::Codecs::Plain/@charset = "UTF-8" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@index = "jabong-%{env}-%{app}-%{iver}-%{[@metadata][app_log_time]}" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@cluster = "elasticsearch" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@network_host = "172.16.84.230" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@hosts = ["172.16.84.230:9300"] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@protocol = "transport" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@codec = <LogStash::Codecs::Plain charset=>"UTF-8"> {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@workers = 1 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@manage_template = true {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@template_name = "logstash" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@template_overwrite = false {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@parent = nil {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@flush_size = 500 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@idle_flush_time = 1 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@upsert = "" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@doc_as_upsert = false {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@max_retries = 3 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@script = "" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@script_type = "inline" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@script_lang = "" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@script_var_name = "event" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@scripted_upsert = false {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@retry_max_interval = 2 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@retry_max_items = 500 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@retry_on_conflict = 1 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@pipeline = nil {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@action = "index" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@max_inflight_requests = 50 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
config LogStash::Outputs::ElasticSearchJava/@sniffing = false {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"153", :method=>"config_init"}
Using mapping template from {:path=>nil, :level=>:info, :file=>"logstash/outputs/elasticsearch/template_manager.rb", :line=>"6", :method=>"install_template"}
Attempting to install template {:manage_template=>{"template"=>"logstash-*", "settings"=>{"index.refresh_interval"=>"5s"}, "mappings"=>{"_default_"=>{"_all"=>{"enabled"=>true, "omit_norms"=>true}, "dynamic_templates"=>[{"message_field"=>{"match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"string", "index"=>"analyzed", "omit_norms"=>true, "fielddata"=>{"format"=>"disabled"}}}}, {"string_fields"=>{"match"=>"*", "match_mapping_type"=>"string", "mapping"=>{"type"=>"string", "index"=>"analyzed", "omit_norms"=>true, "fielddata"=>{"format"=>"disabled"}, "fields"=>{"raw"=>{"type"=>"string", "index"=>"not_analyzed", "ignore_above"=>256}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date"}, "@version"=>{"type"=>"string", "index"=>"not_analyzed"}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"float"}, "longitude"=>{"type"=>"float"}}}}}}}, :level=>:info, :file=>"logstash/outputs/elasticsearch/template_manager.rb", :line=>"8", :method=>"install_template"}
有人可以讓我知道了什麼錯誤？