Home | About | Sematext search-lucene.com search-hadoop.com
 Search Hadoop and all its subprojects:

Switch to Plain View
Flume >> mail # user >> Lock contention in FileChannel


+
Pankaj Gupta 2013-08-13, 23:13
+
Hari Shreedharan 2013-08-13, 23:39
+
Pankaj Gupta 2013-08-14, 00:01
+
Hari Shreedharan 2013-08-14, 00:14
+
Brock Noland 2013-08-14, 00:51
+
Pankaj Gupta 2013-08-14, 02:06
+
Hari Shreedharan 2013-08-14, 02:18
+
Brock Noland 2013-08-14, 02:22
Copy link to this message
-
Re: Lock contention in FileChannel
Here's the config:
# define channels, one for each disk
agent1.channels.ch1.type = FILE
agent1.channels.ch1.checkpointDir = /flume1/checkpoint
agent1.channels.ch1.dataDirs = /flume1/data
agent1.channels.ch1.maxFileSize = 375809638400
agent1.channels.ch1.capacity = 75000000
agent1.channels.ch1.transactionCapacity = 4000

agent1.channels.ch2.type = FILE
agent1.channels.ch2.checkpointDir = /flume2/checkpoint
agent1.channels.ch2.dataDirs = /flume2/data
agent1.channels.ch2.maxFileSize = 375809638400
agent1.channels.ch2.capacity = 75000000
agent1.channels.ch2.transactionCapacity = 4000

# Define an Avro source named avroSource1
# Each sink can connect to only one channel.
# Connect it to channel ch1. Load balance it to 2 avroSinks
agent1.sources.avroSource1.channels = ch1
agent1.sources.avroSource1.type = avro
agent1.sources.avroSource1.bind = 0.0.0.0
agent1.sources.avroSource1.port = <port>
agent1.sinks.avroSink1-1-1.type = avro
agent1.sinks.avroSink1-1-1.channel = ch1
agent1.sinks.avroSink1-1-1.hostname = <hostname>
agent1.sinks.avroSink1-1-1.port = <port>
agent1.sinks.avroSink1-1-1.connect-timeout = 300000
agent1.sinks.avroSink1-1-1.batch-size = 4000
agent1.sinks.avroSink1-2-1.type = avro
agent1.sinks.avroSink1-2-1.channel = ch1
agent1.sinks.avroSink1-2-1.hostname = <hostname>
agent1.sinks.avroSink1-2-1.port = <port>
agent1.sinks.avroSink1-2-1.connect-timeout = 300000
agent1.sinks.avroSink1-2-1.batch-size = 4000
agent1.sinks.avroSink1-3-1.type = avro
agent1.sinks.avroSink1-3-1.channel = ch1
agent1.sinks.avroSink1-3-1.hostname = <hostname>
agent1.sinks.avroSink1-3-1.port = <port>
agent1.sinks.avroSink1-3-1.connect-timeout = 300000
agent1.sinks.avroSink1-3-1.batch-size = 4000
agent1.sinks.avroSink1-4-1.type = avro
agent1.sinks.avroSink1-4-1.channel = ch1
agent1.sinks.avroSink1-4-1.hostname = <hostname>
agent1.sinks.avroSink1-4-1.port = <port>
agent1.sinks.avroSink1-4-1.connect-timeout = 300000
agent1.sinks.avroSink1-4-1.batch-size = 4000

#Add the sink groups; load-balance between each group of sinks which round
robin between different hops
agent1.sinkgroups.group1.sinks = avroSink1-1-1 avroSink1-2-1 avroSink1-3-1
avroSink1-4-1
agent1.sinkgroups.group1.processor.type = load_balance
agent1.sinkgroups.group1.processor.selector = ROUND_ROBIN
agent1.sinkgroups.group1.processor.backoff = true
#End of set

# Define an Avro source named avroSource2
# Each sink can connect to only one channel.
# Connect it to channel ch2. Load balance it to 2 avroSinks
agent1.sources.avroSource2.channels = ch2
agent1.sources.avroSource2.type = avro
agent1.sources.avroSource2.bind = 0.0.0.0
agent1.sources.avroSource2.port = <port>
agent1.sinks.avroSink2-1-1.type = avro
agent1.sinks.avroSink2-1-1.channel = ch2
agent1.sinks.avroSink2-1-1.hostname = <hostname>
agent1.sinks.avroSink2-1-1.port = <port>
agent1.sinks.avroSink2-1-1.connect-timeout = 300000
agent1.sinks.avroSink2-1-1.batch-size = 4000
agent1.sinks.avroSink2-2-1.type = avro
agent1.sinks.avroSink2-2-1.channel = ch2
agent1.sinks.avroSink2-2-1.hostname = <hostname>
agent1.sinks.avroSink2-2-1.port = <port>
agent1.sinks.avroSink2-2-1.connect-timeout = 300000
agent1.sinks.avroSink2-2-1.batch-size = 4000
agent1.sinks.avroSink2-3-1.type = avro
agent1.sinks.avroSink2-3-1.channel = ch2
agent1.sinks.avroSink2-3-1.hostname = <hostname>
agent1.sinks.avroSink2-3-1.port = <port>
agent1.sinks.avroSink2-3-1.connect-timeout = 300000
agent1.sinks.avroSink2-3-1.batch-size = 4000
agent1.sinks.avroSink2-4-1.type = avro
agent1.sinks.avroSink2-4-1.channel = ch2
agent1.sinks.avroSink2-4-1.hostname = <hostname>
agent1.sinks.avroSink2-4-1.port = <port>
agent1.sinks.avroSink2-4-1.connect-timeout = 300000
agent1.sinks.avroSink2-4-1.batch-size = 4000

#Add the sink groups; load-balance between each group of sinks which round
robin between different hops
agent1.sinkgroups.group2.sinks = avroSink2-1-1 avroSink2-2-1 avroSink2-3-1
avroSink2-4-1
agent1.sinkgroups.group2.processor.type = load_balance
agent1.sinkgroups.group2.processor.selector = ROUND_ROBIN
agent1.sinkgroups.group2.processor.backoff = true
#End of set

# Define an Avro source named avroSource3
# Each sink can connect to only one channel.
# Connect it to channel ch1. Load balance it to 2 avroSinks
agent1.sources.avroSource3.channels = ch1
agent1.sources.avroSource3.type = avro
agent1.sources.avroSource3.bind = 0.0.0.0
agent1.sources.avroSource3.port = <port>
agent1.sinks.avroSink3-1-1.type = avro
agent1.sinks.avroSink3-1-1.channel = ch1
agent1.sinks.avroSink3-1-1.hostname = <hostname>
agent1.sinks.avroSink3-1-1.port = <port>
agent1.sinks.avroSink3-1-1.connect-timeout = 300000
agent1.sinks.avroSink3-1-1.batch-size = 4000
agent1.sinks.avroSink3-2-1.type = avro
agent1.sinks.avroSink3-2-1.channel = ch1
agent1.sinks.avroSink3-2-1.hostname = <hostname>
agent1.sinks.avroSink3-2-1.port = <port>
agent1.sinks.avroSink3-2-1.connect-timeout = 300000
agent1.sinks.avroSink3-2-1.batch-size = 4000
agent1.sinks.avroSink3-3-1.type = avro
agent1.sinks.avroSink3-3-1.channel = ch1
agent1.sinks.avroSink3-3-1.hostname = <hostname>
agent1.sinks.avroSink3-3-1.port = <port>
agent1.sinks.avroSink3-3-1.connect-timeout = 300000
agent1.sinks.avroSink3-3-1.batch-size = 4000
agent1.sinks.avroSink3-4-1.type = avro
agent1.sinks.avroSink3-4-1.channel = ch1
agent1.sinks.avroSink3-4-1.hostname = <hostname>
agent1.sinks.avroSink3-4-1.port = <port>
agent1.sinks.avroSink3-4-1.connect-timeout = 300000
agent1.sinks.avroSink3-4-1.batch-size = 4000

#Add the sink groups; load-balance between each group of sinks which round
robin between different hops
agent1.sinkgroups.group3.sinks = avroSink3-1-1 avroSink3-2-1 avroSink3-3-1
avroSink3-4-1
agent1.sinkgroups.group3.processor.type = load_balance
agent1.sinkgroups.group3.processor.selector = ROUND_ROBIN
agent1.sinkgroups.group3.processor.backoff = true
#End of set

# Define an Avro source named avroSource4
# Each sink can connect to only one channel.
# Conn
+
Brock Noland 2013-08-14, 02:41
+
Pankaj Gupta 2013-08-14, 02:46
+
Brock Noland 2013-08-14, 02:54
+
Pankaj Gupta 2013-08-14, 02:57
+
Brock Noland 2013-08-14, 03:06
+
Pankaj Gupta 2013-08-14, 03:16
+
Brock Noland 2013-08-14, 03:30
+
Pankaj Gupta 2013-08-14, 18:57
+
Pankaj Gupta 2013-08-14, 19:12
+
Pankaj Gupta 2013-08-14, 19:34
+
Hari Shreedharan 2013-08-14, 19:43
+
Pankaj Gupta 2013-08-14, 19:59
+
Pankaj Gupta 2013-08-15, 06:04
+
Pankaj Gupta 2013-08-18, 04:43
+
Hari Shreedharan 2013-08-14, 19:04
+
Pankaj Gupta 2013-08-14, 02:16