Hi

One of our ensemble nodes is showing an erratic behavior:

    Jul 13 20:16:00 host12 zookeeper[26293]: [2017-07-13 20:16:00,500]
    WARN Cannot open channel to 13 at election address /10.0.0.13:3888
    (org.apache.zookeeper.server.quorum.QuorumCnxManager)
    Jul 13 20:16:00 host12 zookeeper[26293]:
    java.net.SocketTimeoutException: connect timed out
    Jul 13 20:16:00 host12 zookeeper[26293]: #011at
    java.net.PlainSocketImpl.socketConnect(Native Method)
    Jul 13 20:16:00 host12 zookeeper[26293]: #011at
    java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
    Jul 13 20:16:00 host12 zookeeper[26293]: #011at
    java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
    Jul 13 20:16:00 host12 zookeeper[26293]: #011at
    java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
    Jul 13 20:16:00 host12 zookeeper[26293]: #011at
    java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
    Jul 13 20:16:00 host12 zookeeper[26293]: #011at
    java.net.Socket.connect(Socket.java:589)
    Jul 13 20:16:00 host12 zookeeper[26293]: #011at
    org.apache.zookeeper.server.quorum.QuorumCnxManager.connectOne(QuorumCnxManager.java:443)
    Jul 13 20:16:00 host12 zookeeper[26293]: #011at
    org.apache.zookeeper.server.quorum.QuorumCnxManager.connectOne(QuorumCnxManager.java:486)
    Jul 13 20:16:00 host12 zookeeper[26293]: #011at
    org.apache.zookeeper.server.quorum.QuorumCnxManager.toSend(QuorumCnxManager.java:421)
    Jul 13 20:16:00 host12 zookeeper[26293]: #011at
    org.apache.zookeeper.server.quorum.FastLeaderElection$Messenger$WorkerSender.process(FastLeaderElection.java:486)
    Jul 13 20:16:00 host12 zookeeper[26293]: #011at
    org.apache.zookeeper.server.quorum.FastLeaderElection$Messenger$WorkerSender.run(FastLeaderElection.java:465)
    Jul 13 20:16:00 host12 zookeeper[26293]: #011at
    java.lang.Thread.run(Thread.java:748)
    Jul 13 20:17:00 host12 zookeeper[26293]: [2017-07-13 20:17:00,513]
    WARN Cannot open channel to 13 at election address /10.0.0.13:3888
    (org.apache.zookeeper.server.quorum.QuorumCnxManager)
    Jul 13 20:17:00 host12 zookeeper[26293]:
    java.net.SocketTimeoutException: connect timed out
    Jul 13 20:17:00 host12 zookeeper[26293]: #011at
    java.net.PlainSocketImpl.socketConnect(Native Method)
    Jul 13 20:17:00 host12 zookeeper[26293]: #011at
    java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
    Jul 13 20:17:00 host12 zookeeper[26293]: #011at
    java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
    Jul 13 20:17:00 host12 zookeeper[26293]: #011at
    java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
    Jul 13 20:17:00 host12 zookeeper[26293]: #011at
    java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
    Jul 13 20:17:00 host12 zookeeper[26293]: #011at
    java.net.Socket.connect(Socket.java:589)
    Jul 13 20:17:00 host12 zookeeper[26293]: #011at
    org.apache.zookeeper.server.quorum.QuorumCnxManager.connectOne(QuorumCnxManager.java:443)
    Jul 13 20:17:00 host12 zookeeper[26293]: #011at
    org.apache.zookeeper.server.quorum.QuorumCnxManager.connectOne(QuorumCnxManager.java:486)
    Jul 13 20:17:00 host12 zookeeper[26293]: #011at
    org.apache.zookeeper.server.quorum.QuorumCnxManager.toSend(QuorumCnxManager.java:421)
    Jul 13 20:17:00 host12 zookeeper[26293]: #011at
    org.apache.zookeeper.server.quorum.FastLeaderElection$Messenger$WorkerSender.process(FastLeaderElection.java:486)
    Jul 13 20:17:00 host12 zookeeper[26293]: #011at
    org.apache.zookeeper.server.quorum.FastLeaderElection$Messenger$WorkerSender.run(FastLeaderElection.java:465)
    Jul 13 20:17:00 host12 zookeeper[26293]: #011at
    java.lang.Thread.run(Thread.java:748)
    Jul 13 20:17:26 host12 zookeeper[26293]: [2017-07-13 20:17:26,649]
    WARN Connection broken for id 12, my id = 11, error =
    (org.apache.zookeeper.server.quorum.QuorumCnxManager)
    Jul 13 20:17:26 host12 zookeeper[26293]: java.net.SocketException:
    Connection timed out (Read failed)
    Jul 13 20:17:26 host12 zookeeper[26293]: #011at
    java.net.SocketInputStream.socketRead0(Native Method)
    Jul 13 20:17:26 host12 zookeeper[26293]: #011at
    java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
    Jul 13 20:17:26 host12 zookeeper[26293]: #011at
    java.net.SocketInputStream.read(SocketInputStream.java:171)
    Jul 13 20:17:26 host12 zookeeper[26293]: #011at
    java.net.SocketInputStream.read(SocketInputStream.java:141)
    Jul 13 20:17:26 host12 zookeeper[26293]: #011at
    java.net.SocketInputStream.read(SocketInputStream.java:224)
    Jul 13 20:17:26 host12 zookeeper[26293]: #011at
    java.io.DataInputStream.readInt(DataInputStream.java:387)
    Jul 13 20:17:26 host12 zookeeper[26293]: #011at
    org.apache.zookeeper.server.quorum.QuorumCnxManager$RecvWorker.run(QuorumCnxManager.java:904)
    Jul 13 20:17:26 host12 zookeeper[26293]: [2017-07-13 20:17:26,649]
    WARN Interrupting SendWorker
    (org.apache.zookeeper.server.quorum.QuorumCnxManager)
    Jul 13 20:17:26 host12 zookeeper[26293]: [2017-07-13 20:17:26,650]
    WARN Interrupted while waiting for message on queue
    (org.apache.zookeeper.server.quorum.QuorumCnxManager)
    Jul 13 20:17:26 host12 zookeeper[26293]: java.lang.InterruptedException
    Jul 13 20:17:26 host12 zookeeper[26293]: #011at
    java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2014)
    Jul 13 20:17:26 host12 zookeeper[26293]: #011at
    java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2088)
    Jul 13 20:17:26 host12 zookeeper[26293]: #011at
    java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418)
    Jul 13 20:17:26 host12 zookeeper[26293]: #011at
    org.apache.zookeeper.server.quorum.QuorumC
NEW: Monitor These Apps!
elasticsearch, apache solr, apache hbase, hadoop, redis, casssandra, amazon cloudwatch, mysql, memcached, apache kafka, apache zookeeper, apache storm, ubuntu, centOS, red hat, debian, puppet labs, java, senseiDB