Home | About | Sematext search-lucene.com search-hadoop.com
 Search Hadoop and all its subprojects:

Switch to Threaded View
Drill, mail # dev - [5/5] git commit: DRILL-214: if HADOOP_HOME is set, add jars in hadoop lib directory to classpath. Minor packaging and script fixes.


Copy link to this message
-
[5/5] git commit: DRILL-214: if HADOOP_HOME is set, add jars in hadoop lib directory to classpath. Minor packaging and script fixes.
jacques@... 2013-09-06, 02:56
DRILL-214: if HADOOP_HOME is set, add jars in hadoop lib directory to classpath. Minor packaging and script fixes.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/10866cf0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/10866cf0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/10866cf0

Branch: refs/heads/master
Commit: 10866cf06294842fce13c122724cee99c3f27a86
Parents: f5e2e61
Author: Steven Phillips <[EMAIL PROTECTED]>
Authored: Tue Sep 3 20:11:35 2013 -0700
Committer: Jacques Nadeau <[EMAIL PROTECTED]>
Committed: Thu Sep 5 19:51:09 2013 -0700

----------------------------------------------------------------------
 distribution/src/assemble/bin.xml               |  8 ++++++
 distribution/src/resources/drill-config.sh      | 19 +++++++++++++
 distribution/src/resources/hadoop-excludes.txt  | 13 +++++++++
 distribution/src/resources/runbit               |  5 ++--
 distribution/src/resources/sqlline              |  3 +-
 distribution/src/resources/storage-engines.json | 29 ++++++++++++++++++++
 distribution/src/resources/submit_plan          |  2 +-
 .../exec/store/parquet/ParquetGroupScan.java    |  6 +---
 .../store/parquet/ParquetSchemaProvider.java    |  2 +-
 .../apache/drill/exec/work/FragmentRunner.java  |  2 +-
 10 files changed, 77 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/10866cf0/distribution/src/assemble/bin.xml
----------------------------------------------------------------------
diff --git a/distribution/src/assemble/bin.xml b/distribution/src/assemble/bin.xml
index c00de72..8997a89 100644
--- a/distribution/src/assemble/bin.xml
+++ b/distribution/src/assemble/bin.xml
@@ -62,6 +62,10 @@
       <outputDirectory>bin</outputDirectory>
     </file>
     <file>
+      <source>src/resources/hadoop-excludes.txt</source>
+      <outputDirectory>bin</outputDirectory>
+    </file>
+    <file>
       <source>src/resources/drillbit.sh</source>
       <outputDirectory>bin</outputDirectory>
     </file>
@@ -89,5 +93,9 @@
       <source>src/resources/drill-env.sh</source>
       <outputDirectory>conf</outputDirectory>
     </file>
+    <file>
+      <source>src/resources/storage-engines.json</source>
+      <outputDirectory>conf</outputDirectory>
+    </file>
   </files>
 </assembly>

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/10866cf0/distribution/src/resources/drill-config.sh
----------------------------------------------------------------------
diff --git a/distribution/src/resources/drill-config.sh b/distribution/src/resources/drill-config.sh
index 20102fc..7be5bad 100644
--- a/distribution/src/resources/drill-config.sh
+++ b/distribution/src/resources/drill-config.sh
@@ -66,8 +66,27 @@ done
 # Allow alternate drill conf dir location.
 export DRILL_CONF_DIR="${DRILL_CONF_DIR:-/etc/drill/conf}"
 
+if [ ! -d $DRILL_CONF_DIR ]
+then
+  export DRILL_CONF_DIR=$DRILL_HOME/conf
+fi
+
 . "${DRILL_CONF_DIR}/drill-env.sh"
 
+if [ "${HADOOP_HOME}x" != "x" ]
+then
+  HADOOP_CLASSPATH=""
+  for jar in `ls $HADOOP_HOME/lib/*jar`
+  do
+    echo $jar | grep -v -f $DRILL_HOME/bin/hadoop-excludes.txt >/dev/null
+    if [ "$?" -eq "0" ]
+    then
+      HADOOP_CLASSPATH=$jar:$HADOOP_CLASSPATH
+    fi
+  done
+  export HADOOP_CLASSPATH=$HADOOP_HOME/conf:$HADOOP_CLASSPATH
+fi
+
 # Newer versions of glibc use an arena memory allocator that causes virtual
 # memory usage to explode. Tune the variable down to prevent vmem explosion.
 export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-4}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/10866cf0/distribution/src/resources/hadoop-excludes.txt
----------------------------------------------------------------------
diff --git a/distribution/src/resources/hadoop-excludes.txt b/distribution/src/resources/hadoop-excludes.txt
new file mode 100644
index 0000000..d72efeb
+++ b/distribution/src/resources/hadoop-excludes.txt
@@ -0,0 +1,13 @@
+asm
+jackson
+mockito
+jets3t
+log4j
+logging
+slf4j
+jetty
+jasper
+jersey
+eclipse
+common
+guava

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/10866cf0/distribution/src/resources/runbit
diff --git a/distribution/src/resources/runbit b/distribution/src/resources/runbit
index 77f28a3..8fcb138 100755
+++ b/distribution/src/resources/runbit
@@ -5,7 +5,7 @@ if [ -z $JAVA_HOME ]
 then
   JAVA=`which java`
 else
-  JAVA=`find $JAVA_HOME -name java`
+  JAVA=`find -L $JAVA_HOME -name java | head -n 1`
 fi
 
 if [ -e $JAVA ]; then
@@ -21,10 +21,9 @@ if [ $? -ne 0 ]; then
   exit 1
 fi
 
-
 CP=$DRILL_HOME/jars/*:$CP
 CP=$DRILL_HOME/lib/*:$CP
-
 CP=$CP:$DRILL_CONF_DIR
+CP=$HADOOP_CLASSPATH:$CP
 
 exec $JAVA $DRILL_JAVA_OPTS -cp $CP org.apache.drill.exec.server.Drillbit

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/10866cf0/distribution/src/resources/sqlline
diff --git a/distribution/src/resources/sqlline b/distribution/src/resources/sqlline
index 5d13646..3915929 100755
+++ b/distribution/src/resources/sqlline
@@ -24,7 +24,7 @@ if [ -z $JAVA_HOME ]
 then
   JAVA=`which java`
 else
-  JAVA=`find $JAVA_HOME -name java`
+  JAVA=`find -L $JAVA_HOME -name java | head -n 1`
 fi
 
 if [ -e $JAVA ]; then
@@ -49,6 +49,7 @@ fi
 CP=$DRILL_HOME/jars/*:$CP
 CP=$DRILL_HOME/lib/*:$CP
 CP=$DRILL_HOME/contrib/*:$CP
+CP=$HADOOP_CLASSPATH:$CP
 
 CP=$DRILL_CONF_DIR:$CP
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/10866cf0/distribution/src/resources/storage-engines.json
diff --git a/distribution/src/resources/storage-engines.json b/distribution/src/resources/storage-engines.json
new file mode 100644
index 0000000..d1d0413
+++ b/distribution/src/resources/storage-engines.json
@@ -0,0 +1,29 @@
+{
+  "storage":{
+    "parquet-local" :
+      {
+        "type":"parquet",
+        "dfsName" : "file:///"
+      },
+    "parquet-cp" :
+      {
+