Home | About | Sematext search-lucene.com search-hadoop.com
 Search Hadoop and all its subprojects:

Switch to Plain View
MapReduce >> mail # user >> Re: Why is my output directory owned by yarn?


+
Bill Sparks 2013-11-01, 14:30
Copy link to this message
-
Re: Why is my output directory owned by yarn?
The DefaultContainerExecutor isn't the one that can do setuid. The
LinuxContainerExecutor can do that.

On Fri, Nov 1, 2013 at 8:00 PM, Bill Sparks <[EMAIL PROTECTED]> wrote:
> We'll I thought I've set all this up correctly and on the NodeManager
> nodes can change to my user id, so general user authentication is working.
> But still the output is written as yarn. I guess my question is how to
> enable secure mode - I thought that was the default mode.
>
> When the containers are written they contain the correct user name
> (included).
>
> cat
> /tmp/hadoop-yarn/cache/yarn/nm-local-dir/usercache/jdoe/appcache/applicatio
> n_1383247324024_0005/container_1383247324024_0005_01_000001/launch_containe
> r.sh
> #!/bin/bash
>
> export
> YARN_LOCAL_DIRS="/tmp/hadoop-yarn/cache/yarn/nm-local-dir/usercache/jdoe/ap
> pcache/application_1383247324024_0005"
> export NM_HTTP_PORT="8042"
> export HADOOP_COMMON_HOME="/usr/lib/hadoop"
> export JAVA_HOME="/opt/java/jdk1.6.0_20"
> export HADOOP_YARN_HOME="/usr/lib/hadoop-yarn"
> export NM_HOST="nid00031"
> export
> CLASSPATH="$PWD:$HADOOP_CONF_DIR:$HADOOP_COMMON_HOME/*:$HADOOP_COMMON_HOME/
> lib/*:$HADOOP_HDFS_HOME/*:$HADOOP_HDFS_HOME/lib/*:$HADOOP_MAPRED_HOME/*:$HA
> DOOP_MAPRED_HOME/lib/*:$HADOOP_YARN_HOME/*:$HADOOP_YARN_HOME/lib/*:$HADOOP_
> MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapre
> duce/lib/*:job.jar/job.jar:job.jar/classes/:job.jar/lib/*:$PWD/*"
> export
> HADOOP_TOKEN_FILE_LOCATION="/tmp/hadoop-yarn/cache/yarn/nm-local-dir/userca
> che/jdoe/appcache/application_1383247324024_0005/container_1383247324024_00
> 05_01_000001/container_tokens"
> export APPLICATION_WEB_PROXY_BASE="/proxy/application_1383247324024_0005"
> export JVM_PID="$$"
> export USER="jdoe"
> export HADOOP_HDFS_HOME="/usr/lib/hadoop-hdfs"
> export
> PWD="/tmp/hadoop-yarn/cache/yarn/nm-local-dir/usercache/jdoe/appcache/appli
> cation_1383247324024_0005/container_1383247324024_0005_01_000001"
> export NM_PORT="36276"
> export HOME="/home/"
> export LOGNAME="jdoe"
> export APP_SUBMIT_TIME_ENV="1383312862021"
> export HADOOP_CONF_DIR="/etc/hadoop/conf"
> export MALLOC_ARENA_MAX="4"
> export AM_CONTAINER_ID="container_1383247324024_0005_01_000001"
> ln -sf
> "/tmp/hadoop-yarn/cache/yarn/nm-local-dir/usercache/jdoe/appcache/applicati
> on_1383247324024_0005/filecache/-300930022458385182/job.jar" "job.jar"
> mkdir -p jobSubmitDir
> ln -sf
> "/tmp/hadoop-yarn/cache/yarn/nm-local-dir/usercache/jdoe/appcache/applicati
> on_1383247324024_0005/filecache/-4297161085730400838/job.splitmetainfo"
> "jobSubmitDir/job.splitmetainfo"
> mkdir -p jobSubmitDir
> ln -sf
> "/tmp/hadoop-yarn/cache/yarn/nm-local-dir/usercache/jdoe/appcache/applicati
> on_1383247324024_0005/filecache/-3754219748389402012/job.split"
> "jobSubmitDir/job.split"
> ln -sf
> "/tmp/hadoop-yarn/cache/yarn/nm-local-dir/usercache/jdoe/appcache/applicati
> on_1383247324024_0005/filecache/233482461420248540/job.xml" "job.xml"
> mkdir -p jobSubmitDir
> ln -sf
> "/tmp/hadoop-yarn/cache/yarn/nm-local-dir/usercache/jdoe/appcache/applicati
> on_1383247324024_0005/filecache/-8903348211231085224/appTokens"
> "jobSubmitDir/appTokens"
> exec /bin/bash -c "$JAVA_HOME/bin/java
> -Dlog4j.configuration=container-log4j.properties
> -Dyarn.app.mapreduce.container.log.dir=/tmp/hadoop-yarn/containers/applicat
> ion_1383247324024_0005/container_1383247324024_0005_01_000001
> -Dyarn.app.mapreduce.container.log.filesize=0
> -Dhadoop.root.logger=INFO,CLA  -Xmx1024m
> org.apache.hadoop.mapreduce.v2.app.MRAppMaster
> 1>/tmp/hadoop-yarn/containers/application_1383247324024_0005/container_1383
> 247324024_0005_01_000001/stdout
> 2>/tmp/hadoop-yarn/containers/application_1383247324024_0005/container_1383
> 247324024_0005_01_000001/stderr  "
>
> # cat
> /tmp/hadoop-yarn/cache/yarn/nm-local-dir/usercache/jdoe/appcache/applicatio
> n_1383247324024_0005/container_1383247324024_0005_01_000001/default_contain
> er_executor.sh
> #!/bin/bash
>
> echo $$ >
> /tmp/hadoop-yarn/cache/yarn/nm-local-dir/nmPrivate/container_1383247324024_

Harsh J