Hi,
   I noticed that in one of the machines, the flume process does not run to kill [pid] or the TERM signal. We can shut it down using kill -9 [pid] alone. When running the process as a console application, it does not respond to Ctrl-C. I did not see any errors in the flume log.
I ran the java process with verbose logging turned on and the output is pasted below(the line which is bolded is when I sent a TERM signal to this process):
flume-ng-core-1.3.1.jar]
[Loaded com.google.gson.reflect.TypeToken from file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
[Loaded org.apache.flume.instrumentation.http.HTTPMetricsServer$HTTPMetricsHandler$1 from file:/etc/apache-flume-1.3.1-bin/lib/flume-ng-core-1.3.1.jar]
[Loaded org.jboss.netty.logging.InternalLoggerFactory$1 from file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
[Loaded org.jboss.netty.channel.socket.nio.AbstractNioWorkerPool from file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
[Loaded org.jboss.netty.channel.socket.nio.NioWorkerPool from file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
[Loaded org.jboss.netty.channel.socket.Worker from file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
[Loaded org.jboss.netty.channel.socket.nio.AbstractNioWorker from file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
[Loaded org.jboss.netty.channel.socket.nio.NioWorker from file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
[Loaded com.google.gson.internal.$Gson$Types from file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
[Loaded java.lang.reflect.WildcardType from /usr/lib/jvm/java-6-sun-1.6.0.38/jre/lib/rt.jar]
[Loaded java.nio.channels.NotYetConnectedException from /usr/lib/jvm/java-6-sun-1.6.0.38/jre/lib/rt.jar]
[Loaded com.google.gson.internal.$Gson$Types$GenericArrayTypeImpl from file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
[Loaded java.nio.channels.Selector from /usr/lib/jvm/java-6-sun-1.6.0.38/jre/lib/rt.jar]
[Loaded com.google.gson.internal.$Gson$Types$ParameterizedTypeImpl from file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
[Loaded org.jboss.netty.channel.ChannelException from file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
[Loaded com.google.gson.internal.$Gson$Preconditions from file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
[Loaded sun.misc.Cleaner from /usr/lib/jvm/java-6-sun-1.6.0.38/jre/lib/rt.jar]
[Loaded com.google.gson.Gson from file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
[Loaded java.util.concurrent.locks.ReadWriteLock from /usr/lib/jvm/java-6-sun-1.6.0.38/jre/lib/rt.jar]
[Loaded com.google.gson.stream.MalformedJsonException from file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
[Loaded com.google.gson.JsonParseException from file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
[Loaded com.google.gson.JsonIOException from file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
[Loaded com.google.gson.JsonSyntaxException from file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
[Loaded org.jboss.netty.channel.AbstractChannel from file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
[Loaded org.jboss.netty.channel.socket.nio.AbstractNioChannel from file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
[Loaded org.jboss.netty.channel.socket.nio.NioSocketChannel from file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
[Loaded com.google.gson.stream.JsonReader from file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
[Loaded com.google.gson.internal.bind.JsonTreeReader from file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
[Loaded java.io.EOFException from /usr/lib/jvm/java-6-sun-1.6.0.38/jre/lib/rt.jar]
[Loaded java.io.StringReader from /usr/lib/jvm/java-6-sun-1.6.0.38/jre/lib/rt.jar]
[Loaded org.jboss.netty.channel.socket.nio.NioProviderMetadata from file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
[Loaded com.google.gson.stream.JsonWriter from file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
[Loaded com.google.gson.internal.bind.JsonTreeWriter from file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
[Loaded com.google.gson.JsonElement from file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
[Loaded com.google.gson.JsonNull from file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
[Loaded com.google.gson.TypeAdapter from file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
[Loaded com.google.gson.Gson$FutureTypeAdapter from file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
[Loaded java.util.concurrent.locks.ReentrantReadWriteLock from /usr/lib/jvm/java-6-sun-1.6.0.38/jre/lib/rt.jar]
[Loaded java.util.concurrent.locks.ReentrantReadWriteLock$Sync from /usr/lib/jvm/java-6-sun-1.6.0.38/jre/lib/rt.jar]
[Loaded java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync from /usr/lib/jvm/java-6-sun-1.6.0.38/jre/lib/rt.jar]
[Loaded com.google.gson.Gson$6 from file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
[Loaded java.util.concurrent.locks.ReentrantReadWriteLock$Sync$ThreadLocalHoldCounter from /usr/lib/jvm/java-6-sun-1.6.0.38/jre/lib/rt.jar]
[Loaded java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock from /usr/lib/jvm/java-6-sun-1.6.0.38/jre/lib/rt.jar]
[Loaded java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock from /usr/lib/jvm/java-6-sun-1.6.0.38/jre/lib/rt.jar]
[Loaded com.google.gson.Gson$5 from file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
[Loaded org.jboss.netty.util.internal.QueueFactory from file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
[Loaded com.google.gson.Gson$4 from file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
[Loaded org.jboss.netty.util.internal.DetectionUtil from file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
[Loaded com.google.gson.Gson$1 from file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
[Loaded com.google.gson.JsonDeserializationContext from file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
[Loaded com.google.gson.JsonSerializationContext from file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]

I have had similar crashes, normally due to a poorly set-up
log4j.properties or some other issue that crashed the program badly enough.

Any more details on your configuration, or any ERROR messages in flume.log
or the like that could provide insight into what broke?

- Connor
On Mon, Feb 4, 2013 at 2:58 PM, Rahul Ravindran <[EMAIL PROTECTED]> wrote:

> Hi,
>    I noticed that in one of the machines, the flume process does not run
> to kill [pid] or the TERM signal. We can shut it down using kill -9 [pid]
> alone. When running the process as a console application, it does not
> respond to Ctrl-C. I did not see any errors in the flume log.
>
>
> I ran the java process with verbose logging turned on and the output is
> pasted below(the line which is bolded is when I sent a TERM signal to
> this process):
>
>
> flume-ng-core-1.3.1.jar]
> [Loaded com.google.gson.reflect.TypeToken from
> file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
> [Loaded
> org.apache.flume.instrumentation.http.HTTPMetricsServer$HTTPMetricsHandler$1
> from file:/etc/apache-flume-1.3.1-bin/lib/flume-ng-core-1.3.1.jar]
> [Loaded org.jboss.netty.logging.InternalLoggerFactory$1 from
> file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
> [Loaded org.jboss.netty.channel.socket.nio.AbstractNioWorkerPool from
> file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
> [Loaded org.jboss.netty.channel.socket.nio.NioWorkerPool from
> file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
> [Loaded org.jboss.netty.channel.socket.Worker from
> file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
> [Loaded org.jboss.netty.channel.socket.nio.AbstractNioWorker from
> file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
> [Loaded org.jboss.netty.channel.socket.nio.NioWorker from
> file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
> [Loaded com.google.gson.internal.$Gson$Types from
> file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
> [Loaded java.lang.reflect.WildcardType from
> /usr/lib/jvm/java-6-sun-1.6.0.38/jre/lib/rt.jar]
> [Loaded java.nio.channels.NotYetConnectedException from
> /usr/lib/jvm/java-6-sun-1.6.0.38/jre/lib/rt.jar]
> [Loaded com.google.gson.internal.$Gson$Types$GenericArrayTypeImpl from
> file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
> [Loaded java.nio.channels.Selector from
> /usr/lib/jvm/java-6-sun-1.6.0.38/jre/lib/rt.jar]
> [Loaded com.google.gson.internal.$Gson$Types$ParameterizedTypeImpl from
> file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
> [Loaded org.jboss.netty.channel.ChannelException from
> file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
> [Loaded com.google.gson.internal.$Gson$Preconditions from
> file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
> [Loaded sun.misc.Cleaner from
> /usr/lib/jvm/java-6-sun-1.6.0.38/jre/lib/rt.jar]
> [Loaded com.google.gson.Gson from
> file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
> [Loaded java.util.concurrent.locks.ReadWriteLock from
> /usr/lib/jvm/java-6-sun-1.6.0.38/jre/lib/rt.jar]
> [Loaded com.google.gson.stream.MalformedJsonException from
> file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
> [Loaded com.google.gson.JsonParseException from
> file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
> [Loaded com.google.gson.JsonIOException from
> file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
> [Loaded com.google.gson.JsonSyntaxException from
> file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
> [Loaded org.jboss.netty.channel.AbstractChannel from
> file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
> [Loaded org.jboss.netty.channel.socket.nio.AbstractNioChannel from
> file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
> [Loaded org.jboss.netty.channel.socket.nio.NioSocketChannel from
> file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
> [Loaded com.google.gson.stream.JsonReader from
> file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
> [Loaded com.google.gson.internal.bind.JsonTreeReader from

Rahul, the only way we can know what the process was hung on is if you
provide a stack trace.

Can you please issue a sudo kill -3 <pid> to the Flume java process the
next time this happens? Execute that 2 or 3 times, then look in the output
logs for Flume and you should see a stack trace.

Alternatively, using jstack to get the stack traces.

Regards
Mike
On Mon, Feb 4, 2013 at 5:45 PM, Connor Woodson <[EMAIL PROTECTED]>wrote:

> I have had similar crashes, normally due to a poorly set-up
> log4j.properties or some other issue that crashed the program badly enough.
>
> Any more details on your configuration, or any ERROR messages in flume.log
> or the like that could provide insight into what broke?
>
> - Connor
>
>
> On Mon, Feb 4, 2013 at 2:58 PM, Rahul Ravindran <[EMAIL PROTECTED]> wrote:
>
>> Hi,
>>    I noticed that in one of the machines, the flume process does not run
>> to kill [pid] or the TERM signal. We can shut it down using kill -9 [pid]
>> alone. When running the process as a console application, it does not
>> respond to Ctrl-C. I did not see any errors in the flume log.
>>
>>
>> I ran the java process with verbose logging turned on and the output is
>> pasted below(the line which is bolded is when I sent a TERM signal to
>> this process):
>>
>>
>> flume-ng-core-1.3.1.jar]
>> [Loaded com.google.gson.reflect.TypeToken from
>> file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
>> [Loaded
>> org.apache.flume.instrumentation.http.HTTPMetricsServer$HTTPMetricsHandler$1
>> from file:/etc/apache-flume-1.3.1-bin/lib/flume-ng-core-1.3.1.jar]
>> [Loaded org.jboss.netty.logging.InternalLoggerFactory$1 from
>> file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
>> [Loaded org.jboss.netty.channel.socket.nio.AbstractNioWorkerPool from
>> file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
>> [Loaded org.jboss.netty.channel.socket.nio.NioWorkerPool from
>> file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
>> [Loaded org.jboss.netty.channel.socket.Worker from
>> file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
>> [Loaded org.jboss.netty.channel.socket.nio.AbstractNioWorker from
>> file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
>> [Loaded org.jboss.netty.channel.socket.nio.NioWorker from
>> file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
>> [Loaded com.google.gson.internal.$Gson$Types from
>> file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
>> [Loaded java.lang.reflect.WildcardType from
>> /usr/lib/jvm/java-6-sun-1.6.0.38/jre/lib/rt.jar]
>> [Loaded java.nio.channels.NotYetConnectedException from
>> /usr/lib/jvm/java-6-sun-1.6.0.38/jre/lib/rt.jar]
>> [Loaded com.google.gson.internal.$Gson$Types$GenericArrayTypeImpl from
>> file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
>> [Loaded java.nio.channels.Selector from
>> /usr/lib/jvm/java-6-sun-1.6.0.38/jre/lib/rt.jar]
>> [Loaded com.google.gson.internal.$Gson$Types$ParameterizedTypeImpl from
>> file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
>> [Loaded org.jboss.netty.channel.ChannelException from
>> file:/etc/apache-flume-1.3.1-bin/lib/netty-3.4.0.Final.jar]
>> [Loaded com.google.gson.internal.$Gson$Preconditions from
>> file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
>> [Loaded sun.misc.Cleaner from
>> /usr/lib/jvm/java-6-sun-1.6.0.38/jre/lib/rt.jar]
>> [Loaded com.google.gson.Gson from
>> file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
>> [Loaded java.util.concurrent.locks.ReadWriteLock from
>> /usr/lib/jvm/java-6-sun-1.6.0.38/jre/lib/rt.jar]
>> [Loaded com.google.gson.stream.MalformedJsonException from
>> file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
>> [Loaded com.google.gson.JsonParseException from
>> file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
>> [Loaded com.google.gson.JsonIOException from
>> file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
>> [Loaded com.google.gson.JsonSyntaxException from
>> file:/etc/apache-flume-1.3.1-bin/lib/gson-2.2.2.jar]
>> [Loaded org.jboss.netty.channel.AbstractChannel from