Home | About | Sematext search-lucene.com search-hadoop.com
 Search Hadoop and all its subprojects:

Switch to Threaded View
Flume, mail # user - Lock contention in FileChannel


Copy link to this message
-
Re: Lock contention in FileChannel
Pankaj Gupta 2013-08-14, 02:33
Here's the config:
# define channels, one for each disk
agent1.channels.ch1.type = FILE
agent1.channels.ch1.checkpointDir = /flume1/checkpoint
agent1.channels.ch1.dataDirs = /flume1/data
agent1.channels.ch1.maxFileSize = 375809638400
agent1.channels.ch1.capacity = 75000000
agent1.channels.ch1.transactionCapacity = 4000

agent1.channels.ch2.type = FILE
agent1.channels.ch2.checkpointDir = /flume2/checkpoint
agent1.channels.ch2.dataDirs = /flume2/data
agent1.channels.ch2.maxFileSize = 375809638400
agent1.channels.ch2.capacity = 75000000
agent1.channels.ch2.transactionCapacity = 4000

# Define an Avro source named avroSource1
# Each sink can connect to only one channel.
# Connect it to channel ch1. Load balance it to 2 avroSinks
agent1.sources.avroSource1.channels = ch1
agent1.sources.avroSource1.type = avro
agent1.sources.avroSource1.bind = 0.0.0.0
agent1.sources.avroSource1.port = <port>
agent1.sinks.avroSink1-1-1.type = avro
agent1.sinks.avroSink1-1-1.channel = ch1
agent1.sinks.avroSink1-1-1.hostname = <hostname>
agent1.sinks.avroSink1-1-1.port = <port>
agent1.sinks.avroSink1-1-1.connect-timeout = 300000
agent1.sinks.avroSink1-1-1.batch-size = 4000
agent1.sinks.avroSink1-2-1.type = avro
agent1.sinks.avroSink1-2-1.channel = ch1
agent1.sinks.avroSink1-2-1.hostname = <hostname>
agent1.sinks.avroSink1-2-1.port = <port>
agent1.sinks.avroSink1-2-1.connect-timeout = 300000
agent1.sinks.avroSink1-2-1.batch-size = 4000
agent1.sinks.avroSink1-3-1.type = avro
agent1.sinks.avroSink1-3-1.channel = ch1
agent1.sinks.avroSink1-3-1.hostname = <hostname>
agent1.sinks.avroSink1-3-1.port = <port>
agent1.sinks.avroSink1-3-1.connect-timeout = 300000
agent1.sinks.avroSink1-3-1.batch-size = 4000
agent1.sinks.avroSink1-4-1.type = avro
agent1.sinks.avroSink1-4-1.channel = ch1
agent1.sinks.avroSink1-4-1.hostname = <hostname>
agent1.sinks.avroSink1-4-1.port = <port>
agent1.sinks.avroSink1-4-1.connect-timeout = 300000
agent1.sinks.avroSink1-4-1.batch-size = 4000

#Add the sink groups; load-balance between each group of sinks which round
robin between different hops
agent1.sinkgroups.group1.sinks = avroSink1-1-1 avroSink1-2-1 avroSink1-3-1
avroSink1-4-1
agent1.sinkgroups.group1.processor.type = load_balance
agent1.sinkgroups.group1.processor.selector = ROUND_ROBIN
agent1.sinkgroups.group1.processor.backoff = true
#End of set

# Define an Avro source named avroSource2
# Each sink can connect to only one channel.
# Connect it to channel ch2. Load balance it to 2 avroSinks
agent1.sources.avroSource2.channels = ch2
agent1.sources.avroSource2.type = avro
agent1.sources.avroSource2.bind = 0.0.0.0
agent1.sources.avroSource2.port = <port>
agent1.sinks.avroSink2-1-1.type = avro
agent1.sinks.avroSink2-1-1.channel = ch2
agent1.sinks.avroSink2-1-1.hostname = <hostname>
agent1.sinks.avroSink2-1-1.port = <port>
agent1.sinks.avroSink2-1-1.connect-timeout = 300000
agent1.sinks.avroSink2-1-1.batch-size = 4000
agent1.sinks.avroSink2-2-1.type = avro
agent1.sinks.avroSink2-2-1.channel = ch2
agent1.sinks.avroSink2-2-1.hostname = <hostname>
agent1.sinks.avroSink2-2-1.port = <port>
agent1.sinks.avroSink2-2-1.connect-timeout = 300000
agent1.sinks.avroSink2-2-1.batch-size = 4000
agent1.sinks.avroSink2-3-1.type = avro
agent1.sinks.avroSink2-3-1.channel = ch2
agent1.sinks.avroSink2-3-1.hostname = <hostname>
agent1.sinks.avroSink2-3-1.port = <port>
agent1.sinks.avroSink2-3-1.connect-timeout = 300000
agent1.sinks.avroSink2-3-1.batch-size = 4000
agent1.sinks.avroSink2-4-1.type = avro
agent1.sinks.avroSink2-4-1.channel = ch2
agent1.sinks.avroSink2-4-1.hostname = <hostname>
agent1.sinks.avroSink2-4-1.port = <port>
agent1.sinks.avroSink2-4-1.connect-timeout = 300000
agent1.sinks.avroSink2-4-1.batch-size = 4000

#Add the sink groups; load-balance between each group of sinks which round
robin between different hops
agent1.sinkgroups.group2.sinks = avroSink2-1-1 avroSink2-2-1 avroSink2-3-1
avroSink2-4-1
agent1.sinkgroups.group2.processor.type = load_balance
agent1.sinkgroups.group2.processor.selector = ROUND_ROBIN
agent1.sinkgroups.group2.processor.backoff = true
#End of set

# Define an Avro source named avroSource3
# Each sink can connect to only one channel.
# Connect it to channel ch1. Load balance it to 2 avroSinks
agent1.sources.avroSource3.channels = ch1
agent1.sources.avroSource3.type = avro
agent1.sources.avroSource3.bind = 0.0.0.0
agent1.sources.avroSource3.port = <port>
agent1.sinks.avroSink3-1-1.type = avro
agent1.sinks.avroSink3-1-1.channel = ch1
agent1.sinks.avroSink3-1-1.hostname = <hostname>
agent1.sinks.avroSink3-1-1.port = <port>
agent1.sinks.avroSink3-1-1.connect-timeout = 300000
agent1.sinks.avroSink3-1-1.batch-size = 4000
agent1.sinks.avroSink3-2-1.type = avro
agent1.sinks.avroSink3-2-1.channel = ch1
agent1.sinks.avroSink3-2-1.hostname = <hostname>
agent1.sinks.avroSink3-2-1.port = <port>
agent1.sinks.avroSink3-2-1.connect-timeout = 300000
agent1.sinks.avroSink3-2-1.batch-size = 4000
agent1.sinks.avroSink3-3-1.type = avro
agent1.sinks.avroSink3-3-1.channel = ch1
agent1.sinks.avroSink3-3-1.hostname = <hostname>
agent1.sinks.avroSink3-3-1.port = <port>
agent1.sinks.avroSink3-3-1.connect-timeout = 300000
agent1.sinks.avroSink3-3-1.batch-size = 4000
agent1.sinks.avroSink3-4-1.type = avro
agent1.sinks.avroSink3-4-1.channel = ch1
agent1.sinks.avroSink3-4-1.hostname = <hostname>
agent1.sinks.avroSink3-4-1.port = <port>
agent1.sinks.avroSink3-4-1.connect-timeout = 300000
agent1.sinks.avroSink3-4-1.batch-size = 4000

#Add the sink groups; load-balance between each group of sinks which round
robin between different hops
agent1.sinkgroups.group3.sinks = avroSink3-1-1 avroSink3-2-1 avroSink3-3-1
avroSink3-4-1
agent1.sinkgroups.group3.processor.type = load_balance
agent1.sinkgroups.group3.processor.selector = ROUND_ROBIN
agent1.sinkgroups.group3.processor.backoff = true
#End of set

# Define an Avro source named avroSource4
# Each sink can connect to only one channel.
# Conn