Home | About | Sematext search-lucene.com search-hadoop.com
 Search Hadoop and all its subprojects:

Switch to Threaded View
Flume, mail # user - weird hbase sink behavior
Brock Noland
2013-01-09, 04:39
Xu
2013-01-09, 04:43
Xu
2013-01-09, 04:50
Brock Noland
2013-01-09, 05:10
Copy link to this message
-
weird hbase sink behavior
Xu 2013-01-09, 04:19
Hi folks,

I am using flume to sink into hbase. I had the set up working
yesterday, but when I restarted my flume agent, it for some reason
cannot dump to hbase anymore (too bad I don't have a version-control
for my config yet.)

Here is the configuration and execution output. The agent essentially
listens for objects via avro and dump to console for debugging and to
hbase. The last portion of the console output shows that an object is
indeed received, but for some reason "scan" on the hbase doesn't show
anything new. I tried Regex/Simple serializer and HBase and
AsyncHbase, none works now...

>From the log I don't see anything obviously wrong. I did a tcpdump on
the agent, and saw it established a couple of connections to the
regionservers, but closed them a few seconds later.

Any info on how to correctly configure this or troubleshoot would be
appreciated.

Thanks!
-Simon

root@flume-agent1:/usr/local/lib/flume# cat conf/test.conf
a1.channels = c1 c2
a1.sinks = k1 k2
a1.sources = r1

a1.sources.r1.type = avro
a1.sources.r1.bind = 0.0.0.0
a1.sources.r1.port = 8080

a1.sinks.k1.type = org.apache.flume.sink.hbase.HBaseSink
a1.sinks.k1.table = foo_table
a1.sinks.k1.columnFamily = bar_cf
a1.sinks.k1.serializer = org.apache.flume.sink.hbase.RegexHbaseEventSerializer

a1.sinks.k2.type = logger

#a1.channels.c1.type = file
#a1.channels.c1.checkpointDir = /mnt/flume/checkpoint
#a1.channels.c1.dataDirs = /mnt/flume/data
a1.channels.c1.type = memory
a1.channels.c2.type = memory

a1.sources.r1.channels = c1 c2
a1.sinks.k1.channel = c1
a1.sinks.k2.channel = c2

root@flume-agent1:/usr/local/lib/flume# bin/flume-ng agent --name a1
--conf conf --conf-file conf/test.conf
-Dflume.root.logger=INFO,console
Info: Including Hadoop libraries found via
(/usr/local/hadoop/bin/hadoop) for HDFS access
Warning: $HADOOP_HOME is deprecated.

Warning: $HADOOP_HOME is deprecated.

Info: Excluding /usr/local/hadoop/libexec/../lib/slf4j-api-1.4.3.jar
from classpath
Info: Excluding
/usr/local/hadoop/libexec/../lib/slf4j-log4j12-1.4.3.jar from
classpath
Info: Including HBASE libraries found via (/usr/local/hbase/bin/hbase)
for HBASE access
Info: Excluding /usr/local/hbase/lib/slf4j-api-1.4.3.jar from classpath
Info: Excluding /usr/local/hbase/lib/slf4j-log4j12-1.4.3.jar from classpath
Info: Excluding /usr/local/hadoop/libexec/../lib/slf4j-api-1.4.3.jar
from classpath
Info: Excluding
/usr/local/hadoop/libexec/../lib/slf4j-log4j12-1.4.3.jar from
classpath
+ exec /usr/lib/jvm/default-java/bin/java -Xmx20m
-Dflume.root.logger=INFO,console -cp
'/usr/local/lib/flume/conf:/usr/local/lib/flume/lib/*:/usr/local/hadoop/libexec/../conf:/usr/lib/jvm/default-java/lib/tools.jar:/usr/local/hadoop/libexec/..:/usr/local/hadoop/libexec/../hadoop-core-1.0.4.jar:/usr/local/hadoop/libexec/../lib/asm-3.2.jar:/usr/local/hadoop/libexec/../lib/aspectjrt-1.6.5.jar:/usr/local/hadoop/libexec/../lib/aspectjtools-1.6.5.jar:/usr/local/hadoop/libexec/../lib/commons-beanutils-1.7.0.jar:/usr/local/hadoop/libexec/../lib/commons-beanutils-core-1.8.0.jar:/usr/local/hadoop/libexec/../lib/commons-cli-1.2.jar:/usr/local/hadoop/libexec/../lib/commons-codec-1.4.jar:/usr/local/hadoop/libexec/../lib/commons-collections-3.2.1.jar:/usr/local/hadoop/libexec/../lib/commons-configuration-1.6.jar:/usr/local/hadoop/libexec/../lib/commons-daemon-1.0.1.jar:/usr/local/hadoop/libexec/../lib/commons-digester-1.8.jar:/usr/local/hadoop/libexec/../lib/commons-el-1.0.jar:/usr/local/hadoop/libexec/../lib/commons-httpclient-3.0.1.jar:/usr/local/hadoop/libexec/../lib/commons-io-2.1.jar:/usr/local/hadoop/libexec/../lib/commons-lang-2.4.jar:/usr/local/hadoop/libexec/../lib/commons-logging-1.1.1.jar:/usr/local/hadoop/libexec/../lib/commons-logging-api-1.0.4.jar:/usr/local/hadoop/libexec/../lib/commons-math-2.1.jar:/usr/local/hadoop/libexec/../lib/commons-net-1.4.1.jar:/usr/local/hadoop/libexec/../lib/core-3.1.1.jar:/usr/local/hadoop/libexec/../lib/hadoop-capacity-scheduler-1.0.4.jar:/usr/local/hadoop/libexec/../lib/hadoop-fairscheduler-1.0.4.jar:/usr/local/hadoop/libexec/../lib/hadoop-thriftfs-1.0.4.jar:/usr/local/hadoop/libexec/../lib/hsqldb-1.8.0.10.jar:/usr/local/hadoop/libexec/../lib/jackson-core-asl-1.8.8.jar:/usr/local/hadoop/libexec/../lib/jackson-mapper-asl-1.8.8.jar:/usr/local/hadoop/libexec/../lib/jasper-compiler-5.5.12.jar:/usr/local/hadoop/libexec/../lib/jasper-runtime-5.5.12.jar:/usr/local/hadoop/libexec/../lib/jdeb-0.8.jar:/usr/local/hadoop/libexec/../lib/jersey-core-1.8.jar:/usr/local/hadoop/libexec/../lib/jersey-json-1.8.jar:/usr/local/hadoop/libexec/../lib/jersey-server-1.8.jar:/usr/local/hadoop/libexec/../lib/jets3t-0.6.1.jar:/usr/local/hadoop/libexec/../lib/jetty-6.1.26.jar:/usr/local/hadoop/libexec/../lib/jetty-util-6.1.26.jar:/usr/local/hadoop/libexec/../lib/jsch-0.1.42.jar:/usr/local/hadoop/libexec/../lib/junit-4.5.jar:/usr/local/hadoop/libexec/../lib/kfs-0.2.2.jar:/usr/local/hadoop/libexec/../lib/log4j-1.2.15.jar:/usr/local/hadoop/libexec/../lib/mockito-all-1.8.5.jar:/usr/local/hadoop/libexec/../lib/oro-2.0.8.jar:/usr/local/hadoop/libexec/../lib/servlet-api-2.5-20081211.jar:/usr/local/hadoop/libexec/../lib/xmlenc-0.52.jar:/usr/local/hadoop/libexec/../lib/jsp-2.1/jsp-2.1.jar:/usr/local/hadoop/libexec/../lib/jsp-2.1/jsp-api-2.1.jar:/usr/local/hbase/conf:/usr/lib/tools.jar:/usr/local/hbase:/usr/local/hbase/hbase-0.94.3.jar:/usr/local/hbase/hbase-0.94.3-tests.jar:/usr/local/hbase/lib/activation-1.1.jar:/usr/local/hbase/lib/asm-3.1.jar:/usr/local/hbase/lib/avro-1.5.3.jar:/usr/local/hbase/lib/avro-ipc-1.5.3.jar:/usr/local/hbase/lib/commons-beanutils-1.7.0.jar:/usr/local/hbase/lib/commons-beanutils-core-1.8.0.jar:/usr/local/hbase/lib/commons-cli-1.2.jar:/usr/local/hbase/lib/commons-codec-1.4.jar:/usr/local/hbase/lib/commons-collections-3.2.1.jar:/usr/local/hbase/lib/commons-configuration-1.6.jar:/usr/local/hbase/lib/commons-digester-1.8.jar:/usr/local/hbase/lib/commons-el-1.0.jar:/usr/local/hbase/lib/commons-httpclient-3.1.jar:/usr/local/hbase/lib/commons-io-2.1.jar: