Home | About | Sematext search-lucene.com search-hadoop.com
 Search Hadoop and all its subprojects:

Switch to Threaded View
MapReduce >> mail # user >> secondary namenode is hang at post


Copy link to this message
-
Re: secondary namenode is hang at post
Thanks adam,

2013-12-11 14:43:07
Full thread dump Java HotSpot(TM) 64-Bit Server VM (20.7-b02 mixed mode):

"Attach Listener" daemon prio=10 tid=0x0000000041057000 nid=0x6b95 waiting
on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"DestroyJavaVM" prio=10 tid=0x00007fb6e0009800 nid=0x1b6f waiting on
condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"SecondaryNameNode Status
Name Node Address    : pnn/x.x.x.x:9000
Start Time           : Wed Dec 11 13:04:22 PST 2013
Last Checkpoint Time : --
Checkpoint Period    : 3600 seconds
Checkpoint Size      : 67108864 MB
Checkpoint Dirs      : [/hadoop/namesecondary, /hadoop/nfs_namesecondary]
Checkpoint Edits Dirs: [/hadoop/namesecondary, /hadoop/nfs_namesecondary]"
daemon prio=10 tid=0x00007fb6e0471800 nid=0x1b95 runnable
[0x00007fad08f43000]
   java.lang.Thread.State: RUNNABLE
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.read(SocketInputStream.java:129)
at java.io.BufferedInputStream.fill(BufferedInputStream.java:218)
at java.io.BufferedInputStream.read1(BufferedInputStream.java:258)
at java.io.BufferedInputStream.read(BufferedInputStream.java:317)
- locked <0x00007fb4077b5108> (a java.io.BufferedInputStream)
at sun.net.www.http.HttpClient.parseHTTPHeader(HttpClient.java:690)
at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:633)
at
sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1195)
- locked <0x00007fb4077a9218> (a
sun.net.www.protocol.http.HttpURLConnection)
at
sun.net.www.protocol.http.HttpURLConnection.getHeaderField(HttpURLConnection.java:2300)
at
org.apache.hadoop.hdfs.server.namenode.TransferFsImage.getFileClient(TransferFsImage.java:192)
at
org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode.putFSImage(SecondaryNameNode.java:404)
at
org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode.doCheckpoint(SecondaryNameNode.java:473)
at
org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode.doWork(SecondaryNameNode.java:333)
at
org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode.run(SecondaryNameNode.java:297)
at java.lang.Thread.run(Thread.java:662)

"Timer-0" daemon prio=10 tid=0x00007fb6e0484000 nid=0x1b94 in Object.wait()
[0x00007fad09044000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00007fad3e671138> (a java.util.TaskQueue)
at java.util.TimerThread.mainLoop(Timer.java:509)
- locked <0x00007fad3e671138> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:462)

"1408732008@qtp-421539177-1 - Acceptor0 SelectChannelConnector@snn:50090"
prio=10 tid=0x00007fb6e0478800 nid=0x1b93 runnable [0x00007fad09145000]
   java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
- locked <0x00007fad3e6c8f48> (a sun.nio.ch.Util$2)
- locked <0x00007fad3e6c8f60> (a java.util.Collections$UnmodifiableSet)
- locked <0x00007fad3e6c8ed0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
at
org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:511)
at org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:193)
at
org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124)
at
org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708)
at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)

"162291475@qtp-421539177-0" prio=10 tid=0x00007fb6e0477800 nid=0x1b92 in
Object.wait() [0x00007fad09246000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00007fad3e6711a0> (a
org.mortbay.thread.QueuedThreadPool$PoolThread)
at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626)
- locked <0x00007fad3e6711a0> (a
org.mortbay.thread.QueuedThreadPool$PoolThread)

"RMI TCP Accept-0" daemon prio=10 tid=0x00007fb6e0322000 nid=0x1b8e
runnable [0x00007fad09cd1000]
   java.lang.Thread.State: RUNNABLE
at java.net.PlainSocketImpl.socketAccept(Native Method)
at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:408)
- locked <0x00007fad3e671330> (a java.net.SocksSocketImpl)
at java.net.ServerSocket.implAccept(ServerSocket.java:462)
at java.net.ServerSocket.accept(ServerSocket.java:430)
at
sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:34)
at
sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:369)
at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:341)
at java.lang.Thread.run(Thread.java:662)

"Low Memory Detector" daemon prio=10 tid=0x00007fb6e00b8800 nid=0x1b8d
runnable [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread1" daemon prio=10 tid=0x00007fb6e00b6800 nid=0x1b8c
waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread0" daemon prio=10 tid=0x00007fb6e00b3800 nid=0x1b8b
waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"Signal Dispatcher" daemon prio=10 tid=0x00007fb6e00b1800 nid=0x1b8a
runnable [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"Finalizer" daemon prio=10 tid=0x00007fb6e0094800 nid=0x1b89 in
Object.wait() [0x00007fad0aa8d000]
   java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00007fad3e671898> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118)
- locked <0x00007fad3e671898> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134)
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159)

"Reference Handler" daemon prio=10 tid=0x00007