我們遇到了一個問題，其中一個Elasticsearch數據節點在AWS中完全死亡。即使有第二個節點，我們也很難讓羣集重新聯機。它最初出現在yellow狀態，我們可以檢索數據但不寫入它。Logstash/Graylog不會將數據節點寫入到ES中
Elasticsearch錯誤
[2016年9月12日13：16：42312] [WARN] [discovery.zen.ping.unicast] [witness.domain.net]發送失敗平爲[{ ＃cloud-i-cb397257-0} {1.1.1.1} {1.1.1.1:9300}] RemoteTransportException [[datanode.domain.net] [1.1.1.1:9300] [internal：discovery/zen/unicast]];嵌套：IllegalStateException [未開始時接收到ping請求]; 所致：java.lang.IllegalStateException：接收ping請求而沒有啓動
Logstash錯誤
{：時間戳=> 「2016-09-12T15：48：15.898000-0400」：消息= >＃900592725，：events_consumed => 900732955，：worker_count => 4，：inflight_count => 0，：worker_states => [{：status =>「sleep」，：alive => true，：index => 0，：inflight_count => 0}，{：status =>「sleep」，：alive => true，：index => 1，：inflight_count => 0}，{：status =>「sleep」，：alive => true，：index => 2，：inflight_count => 0}，{：status =>「sleep」，：alive => true，：index => 3，：inflight_count => 0}]，：output_info => [{：type => 「gelf」，：config => {「host」=>「127.0.0.1」，「ALLOW_ENV」=> false}，：：is_m ulti_worker => false，：events_received => 900592725，：workers =>「127.0.0.1」，codec =>「UTF-8」>，workers => 1，port => 12201，chunksize => 1420，sender =>「 ％{host}「，level => [」％{{severity}「，」INFO「]，ship_metadata => true，ship_tags => true，ignore_metadata => [」@ timestamp「，」@version「，」severity「 「host_host」，「source_path」，「short_message」]，full_message =>「％{message}」，short_message =>「short_message」>]>，：：busy_workers => 0}]，：thread_info => [ {「thread_id」=> 17，「name」=>「[main] nil，」backtrace「=> [」[/ vendor/bundle/jruby/1.9/gems/jruby-openssl-0.9.13- java/lib/jopenssl19/openssl/ssl-internal.rb：106：在accept'", "[...]/vendor/bundle/jruby/1.9/gems/jruby-openssl-0.9.13-java/lib/jopenssl19/openssl/ssl-internal.rb:106:in接受'「，」[...]/vendor/bundle/jruby/1.9/gems/logstash -input-tcp-3.0.6/lib /logstash/inputs/tcp.rb:112:in run_server'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-input-tcp-3.0.6/lib/logstash/inputs/tcp.rb:84:in run'「，」[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3 -java/lib/logstash/pipeline。 rb：342：inputworker'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:336:in start_input'「]，」blocked_on「=> nil，」status「=>」sleep「，」current_call「=>」[...]/vendor/bundle/jruby/1.9/gems/jruby -openssl-0.9.13 -java/lib/jopenssl19/openssl/ssl-internal.rb：106：in accept'"}, {"thread_id"=>18, "name"=>"[main]>worker0", "plugin"=>["LogStash::Filters::Grok", {"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/", "match"=>{"message"=>"%{SYSLOGBASE2}"}}], "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in synchronize'「，」[...]/vendor/bundle/jruby/1.9/gems/logstash-core- 2.3.3 -java/lib/logstash/pipeline.rb：309：在inflight_batches_synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:234:in worker_loop'「，[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3 -java/lib /logstash/pipeline.rb:201:in start_workers'"], "blocked_on"=>nil, "status"=>"sleep", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in synchronize'「}，{」thread_id「=> 19，」name「=>」[main]> worker1「，」plugin「=> [」LogStash :: Filters :: Grok「，{」patterns_dir「=>」/ opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns /「，」match「=> {」message「=> 「％{SYSLOGBASE2}」}}]，「backtrace」=> [[] [/ vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3 -java/lib/logstash/pipeline.rb ：309：在synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in inflight_batches_synchronize'「，」[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3 -java/lib/logstash/pipeline.rb：234：in worker_loop'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:201:in start_workers' 「]，」blocked_on「=>無，」狀態「=>」睡眠「，」current_call「=>」[...]/vendor/bundle/jrub y/1.9/gems/logstash-core-2.3.3 -java/lib/logstash/pipeline.rb：309：在synchronize'"}, {"thread_id"=>20, "name"=>"[main]>worker2", "plugin"=>["LogStash::Filters::Grok", {"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/", "match"=>{"message"=>"%{SYSLOGBASE2}"}}], "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in同步'「，[...]/vendor/bundle/jruby/1.9/gems/logstash -core-2.3.3 -java/lib/logstash/pipeline.rb：309：inflight_batches_synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:234:in worker_loop'「，[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3- java/lib/logstash/pipeline.rb：201：在start_workers'"], "blocked_on"=>nil, "status"=>"sleep", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in synchronize'「}，{」thread_id「=> 21，」name「=>」[main]> worker3「，」plugin「=> [」LogStash :: Filters :: Grok「，{」patterns_dir「=>」/ opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns /「，」match「=> {」message 「=>」％{SYSLOGBASE2}「}}]，」backtrace「=> [」[.../vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3 -java/lib/logstash/pipeline.rb：309：in synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in inflight_batches_synchronize'「，」[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3 -java/lib/logstash/pipeline「。rb：234：in worker_loop'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:201:in start_workers'「]，」blocked_on「=> nil，」status「=>」sleep「，」current_call「=>」[/ vendor/bundle/jruby/1.9/gems/logstash -ore-2.3.3 -java/lib/logstash/pipeline.rb：309：synchronize'"}], :stalling_threads_info=>[{"thread_id"=>17, "name"=>"[main]<tcp", "plugin"=>nil, "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/jruby-openssl-0.9.13-java/lib/jopenssl19/openssl/ssl-internal.rb:106:in accept'「}，{」thread_id「=> 18，」name「=>」[main]> worker0「，」plugin「 => [「LogStash :: Filters :: Grok」，{「patterns_dir」=>「/ opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns /」，匹配「=> {」message「=>」％{SYSLOGBASE2}「}}]，」current_call「=>」[...]/vendor/bundle/jruby/1.9/gems/logstash-core- 2.3.3- java/lib/logstash/pipeline.rb：309：synchronize'"}, {"thread_id"=>19, "name"=>"[main]>worker1", "plugin"=>["LogStash::Filters::Grok", {"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/", "match"=>{"message"=>"%{SYSLOGBASE2}"}}], "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in synchronize'「}，{」thread_id「=> 20，」name「=>」[main]> worker2「，」plugin「=> [」LogStash :: Filters :: Grok「，{」patterns_dir「=>」/ opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns /「，」match「=> {」message 「=>」％{SYSLOGBASE2}「}}]，」current_call「=>」[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3 -java/lib/logstash/pipeline .rb：309：在synchronize'"}, {"thread_id"=>21, "name"=>"[main]>worker3", "plugin"=>["LogStash::Filters::Grok", {"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/", "match"=>{"message"=>"%{SYSLOGBASE2}"}}], "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in同步onize'「}]}>，：level =>：warn}