Home | About | Sematext search-lucene.com search-hadoop.com
 Search Hadoop and all its subprojects:

Switch to Threaded View
Drill >> mail # dev >> [06/10] git commit: DRILL-274: Spooling batch buffer


Copy link to this message
-
[06/10] git commit: DRILL-274: Spooling batch buffer
DRILL-274: Spooling batch buffer
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/2c811a83
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/2c811a83
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/2c811a83

Branch: refs/heads/master
Commit: 2c811a83b30295fb39e1540ce76fbf54768ed50c
Parents: 6c0389f
Author: Steven Phillips <[EMAIL PROTECTED]>
Authored: Sun Dec 1 20:04:44 2013 -0800
Committer: Jacques Nadeau <[EMAIL PROTECTED]>
Committed: Sun Dec 1 20:04:44 2013 -0800

----------------------------------------------------------------------
 distribution/src/resources/drill-override.conf  |  35 ++-
 .../org/apache/drill/exec/ExecConstants.java    |   5 +
 .../apache/drill/exec/ops/FragmentContext.java  |  10 +-
 .../exec/physical/impl/WireRecordBatch.java     |  15 +-
 .../impl/mergereceiver/MergingRecordBatch.java  |  15 +-
 .../exec/record/RawFragmentBatchProvider.java   |   4 +-
 .../exec/store/LocalSyncableFileSystem.java     | 183 ++++++++++++++
 .../work/batch/AbstractFragmentCollector.java   |  22 +-
 .../drill/exec/work/batch/BatchCollector.java   |   4 +-
 .../exec/work/batch/BitComHandlerImpl.java      |   2 +-
 .../drill/exec/work/batch/IncomingBuffers.java  |  24 +-
 .../drill/exec/work/batch/MergingCollector.java |   5 +-
 .../exec/work/batch/PartitionedCollector.java   |   5 +-
 .../drill/exec/work/batch/RawBatchBuffer.java   |   4 +-
 .../exec/work/batch/SpoolingRawBatchBuffer.java | 250 +++++++++++++++++++
 .../work/batch/UnlimitedRawBatchBuffer.java     |   4 +
 .../work/foreman/RunningFragmentManager.java    |   6 +-
 .../work/fragment/RemoteFragmentHandler.java    |   5 +-
 .../src/main/resources/drill-module.conf        |  11 +-
 .../apache/drill/exec/client/DumpCatTest.java   |   2 +-
 .../exec/fn/impl/TestRepeatedFunction.java      |   2 +-
 .../physical/impl/TestComparisonFunctions.java  |   2 +-
 .../exec/physical/impl/TestOptiqPlans.java      |   4 +-
 .../exec/physical/impl/TestSimpleFunctions.java |  10 +-
 .../drill/exec/physical/impl/agg/TestAgg.java   |   2 +-
 .../physical/impl/filter/TestSimpleFilter.java  |   4 +-
 .../exec/physical/impl/join/TestMergeJoin.java  |   8 +-
 .../physical/impl/limit/TestSimpleLimit.java    |   4 +-
 .../impl/project/TestSimpleProjection.java      |   2 +-
 .../exec/physical/impl/sort/TestSimpleSort.java |   4 +-
 .../physical/impl/svremover/TestSVRemover.java  |   2 +-
 .../impl/trace/TestTraceMultiRecordBatch.java   |   2 +-
 .../impl/trace/TestTraceOutputDump.java         |   2 +-
 .../physical/impl/union/TestSimpleUnion.java    |   2 +-
 .../apache/drill/exec/work/batch/FileTest.java  |  66 +++++
 .../exec/work/batch/TestSpoolingBuffer.java     |  62 +++++
 .../src/test/resources/drill-module.conf        |   4 +-
 .../test/resources/drill-spool-test-module.conf |  83 ++++++
 .../resources/work/batch/multiple_exchange.json |  48 ++++
 39 files changed, 851 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2c811a83/distribution/src/resources/drill-override.conf
----------------------------------------------------------------------
diff --git a/distribution/src/resources/drill-override.conf b/distribution/src/resources/drill-override.conf
index 18a2a6a..5fe2362 100644
--- a/distribution/src/resources/drill-override.conf
+++ b/distribution/src/resources/drill-override.conf
@@ -16,6 +16,9 @@
 //  This file tells Drill to consider this module when class path scanning.  
 //  This file can also include any supplementary configuration information.  
 //  This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information.
+
+drill.logical.function.packages += "org.apache.drill.exec.expr.fn.impl"
+
 drill.exec: {
   cluster-id: "drillbits1"
   rpc: {
@@ -46,11 +49,19 @@ drill.exec: {
   optimizer: {
     implementation: "org.apache.drill.exec.opt.IdentityOptimizer"
   },
+  functions: ["org.apache.drill.expr.fn.impl"],
   storage: {
- packages += "org.apache.drill.exec.store"
-  }
-  metrics : {
-    context: "drillbit"
+    packages += "org.apache.drill.exec.store"  
+  },
+  metrics : {
+    context: "drillbit",
+    jmx: {
+      enabled : true
+    },
+    log: {
+      enabled : false,
+      interval : 60
+    }
   },
   zk: {
  connect: "localhost:2181",
@@ -60,7 +71,7 @@ drill.exec: {
   retry: {
    count: 7200,
    delay: 500
-   }
+   }    
   },
   functions: ["org.apache.drill.expr.fn.impl"],
   network: {
@@ -70,8 +81,18 @@ drill.exec: {
     max.width.per.endpoint: 5,
     global.max.width: 100,
     executor.threads: 4
-  }
+  },
   trace: {
-    directory: "/tmp"
+    directory: "/var/log/drill",
+    filesystem: "file:///"
+  },
+  tmp: {
+    directories: ["/tmp/drill"],
+    filesystem: "drill-local:///"
+  },
+  spooling: {
+    impl: "org.apache.drill.exec.work.batch.SpoolingRawBatchBuffer",
+    delete: false,
+    size: 100000000
   }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2c811a83/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index 36504f6..5336c0e 100644
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -45,4 +45,9 @@ public interface ExecConstants {
   public static final String USER_SERVER_RPC_THREADS = "drill.exec.rpc.user.server.threads";
   public static final String TRACE_DUMP_DIRECTORY = "drill.exec.trace.directory";
   public static final String TRACE_DUMP_FILESYSTEM = "drill.exec.trace.filesystem";
+  public static final String TEMP_DIRECTORIES = "drill.exec.tmp.directories";
+  public static final String TEMP_FILESYSTEM = "drill.exec.tmp.filesystem";
+