Home | About | Sematext search-lucene.com search-hadoop.com
NEW: Monitor These Apps!
elasticsearch, apache solr, apache hbase, hadoop, redis, casssandra, amazon cloudwatch, mysql, memcached, apache kafka, apache zookeeper, apache storm, ubuntu, centOS, red hat, debian, puppet labs, java, senseiDB
 Search Hadoop and all its subprojects:

Switch to Threaded View
HBase >> mail # user >> M/R scan problem


Copy link to this message
-
Re: M/R scan problem
I used kill -3, following the thread dump:

Full thread dump Java HotSpot(TM) 64-Bit Server VM (19.1-b02 mixed mode):

"IPC Client (47) connection to /127.0.0.1:59759 from hadoop" daemon
prio=10 tid=0x00002aaab05ca800 nid=0x4eaf in Object.wait()
[0x00000000403c1000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000000f9dba860> (a org.apache.hadoop.ipc.Client$Connection)
at org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:403)
- locked <0x00000000f9dba860> (a org.apache.hadoop.ipc.Client$Connection)
at org.apache.hadoop.ipc.Client$Connection.run(Client.java:445)

"SpillThread" daemon prio=10 tid=0x00002aaab0585000 nid=0x4c99 waiting
on condition [0x00000000404c2000]
   java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x00000000f9af0c38> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
at org.apache.hadoop.mapred.MapTask$MapOutputBuffer$SpillThread.run(MapTask.java:1169)

"main-EventThread" daemon prio=10 tid=0x00002aaab035d000 nid=0x4c95
waiting on condition [0x0000000041207000]
   java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x00000000f9af5f58> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)

"main-SendThread(hadoop09.infolinks.local:2181)" daemon prio=10
tid=0x00002aaab035c000 nid=0x4c94 runnable [0x0000000040815000]
   java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
- locked <0x00000000f9af61a8> (a sun.nio.ch.Util$2)
- locked <0x00000000f9af61b8> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000000f9af6160> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1107)

"communication thread" daemon prio=10 tid=0x000000004d020000
nid=0x4c93 waiting on condition [0x0000000042497000]
   java.lang.Thread.State: RUNNABLE
at java.util.Hashtable.put(Hashtable.java:420)
- locked <0x00000000f9dbaa58> (a java.util.Hashtable)
at org.apache.hadoop.ipc.Client$Connection.addCall(Client.java:225)
- locked <0x00000000f9dba860> (a org.apache.hadoop.ipc.Client$Connection)
at org.apache.hadoop.ipc.Client$Connection.access$1600(Client.java:176)
at org.apache.hadoop.ipc.Client.getConnection(Client.java:854)
at org.apache.hadoop.ipc.Client.call(Client.java:720)
at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:220)
at org.apache.hadoop.mapred.$Proxy0.ping(Unknown Source)
at org.apache.hadoop.mapred.Task$TaskReporter.run(Task.java:548)
at java.lang.Thread.run(Thread.java:662)

"Thread for syncLogs" daemon prio=10 tid=0x00002aaab02e9800 nid=0x4c90
runnable [0x0000000040714000]
   java.lang.Thread.State: RUNNABLE
at java.util.Arrays.copyOf(Arrays.java:2882)
at java.lang.AbstractStringBuilder.expandCapacity(AbstractStringBuilder.java:100)
at java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:390)
at java.lang.StringBuilder.append(StringBuilder.java:119)
at java.io.UnixFileSystem.resolve(UnixFileSystem.java:93)
at java.io.File.<init>(File.java:312)
at org.apache.hadoop.mapred.TaskLog.getTaskLogFile(TaskLog.java:72)
at org.apache.hadoop.mapred.TaskLog.writeToIndexFile(TaskLog.java:180)
at org.apache.hadoop.mapred.TaskLog.syncLogs(TaskLog.java:230)
- locked <0x00000000eea92fc0> (a java.lang.Class for
org.apache.hadoop.mapred.TaskLog)
at org.apache.hadoop.mapred.Child$2.run(Child.java:89)

"Low Memory Detector" daemon prio=10 tid=0x00002aaab0001800 nid=0x4c86
runnable [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"CompilerThread1" daemon prio=10 tid=0x000000004cb4e800 nid=0x4c85
waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"CompilerThread0" daemon prio=10 tid=0x000000004cb4b000 nid=0x4c84
waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"Signal Dispatcher" daemon prio=10 tid=0x000000004cb49000 nid=0x4c83
runnable [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"Finalizer" daemon prio=10 tid=0x000000004cb2c800 nid=0x4c82 in
Object.wait() [0x0000000041d7a000]
   java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000000f9c52630> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118)
- locked <0x00000000f9c52630> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134)
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159)

"Reference Handler" daemon prio=10 tid=0x000000004cb25000 nid=0x4c81
in Object.wait() [0x0000000041005000]
   java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000000f9af5ea0> (a java.lang.ref.Reference$Lock)
at java.lang.Object.wait(Object.java:485)
at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)
- locked <0x00000000f9af5ea0> (a java.lang.ref.Reference$Lock)

"main" prio=10 tid=0x000000004cab9000 nid=0x4c77 runnable [0x0000000040f04000]
   java.lang.Thread.State: RUNNABLE
at com.infolinks.hadoop
NEW: Monitor These Apps!
elasticsearch, apache solr, apache hbase, hadoop, redis, casssandra, amazon cloudwatch, mysql, memcached, apache kafka, apache zookeeper, apache storm, ubuntu, centOS, red hat, debian, puppet labs, java, senseiDB