Home | About | Sematext search-lucene.com search-hadoop.com
 Search Hadoop and all its subprojects:

Switch to Threaded View
Drill, mail # dev - [1/7] git commit: DRILL-385: Implement Top-N sort operator


Copy link to this message
-
[1/7] git commit: DRILL-385: Implement Top-N sort operator
jacques@... 2014-03-29, 18:37
Repository: incubator-drill
Updated Branches:
  refs/heads/master f63292148 -> f63f47f29
DRILL-385: Implement Top-N sort operator
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/df587a46
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/df587a46
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/df587a46

Branch: refs/heads/master
Commit: df587a46fe1e55cd025ce586dc628788f34a2581
Parents: f632921
Author: Steven Phillips <[EMAIL PROTECTED]>
Authored: Mon Feb 3 19:42:53 2014 -0800
Committer: Jacques Nadeau <[EMAIL PROTECTED]>
Committed: Sat Mar 29 11:28:04 2014 -0700

 distribution/src/assemble/bin.xml               |   1 +
 distribution/src/resources/drill-override.conf  |   5 +-
 .../org/apache/drill/exec/ExecConstants.java    |   3 +-
 .../apache/drill/exec/physical/config/Sort.java |   4 +-
 .../apache/drill/exec/physical/config/TopN.java |  86 +++++
 .../exec/physical/impl/TopN/PriorityQueue.java  |  40 +++
 .../impl/TopN/PriorityQueueTemplate.java        | 203 +++++++++++
 .../exec/physical/impl/TopN/TopNBatch.java      | 350 +++++++++++++++++++
 .../impl/TopN/TopNSortBatchCreator.java         |  41 +++
 .../exec/physical/impl/sort/SortTemplate.java   |   3 +
 .../impl/svremover/CopierTemplate4.java         |   6 +-
 .../impl/svremover/RemovingRecordBatch.java     |  32 +-
 .../exec/record/ExpandableHyperContainer.java   |  58 +++
 .../drill/exec/record/HyperVectorWrapper.java   |  11 +
 .../apache/drill/exec/record/SchemaBuilder.java |   2 +-
 .../drill/exec/record/VectorContainer.java      |   4 +-
 .../exec/record/selection/SelectionVector4.java |   2 +-
 .../src/main/resources/drill-module.conf        |   5 +-
 .../exec/physical/impl/TopN/TestSimpleTopN.java | 116 ++++++
 .../src/test/resources/topN/one_key_sort.json   |  53 +++
 .../src/test/resources/topN/two_key_sort.json   |  55 +++
 21 files changed, 1050 insertions(+), 30 deletions(-)
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/df587a46/distribution/src/assemble/bin.xml
diff --git a/distribution/src/assemble/bin.xml b/distribution/src/assemble/bin.xml
index 8a80856..06ec639 100644
+++ b/distribution/src/assemble/bin.xml
@@ -22,6 +22,7 @@
   <id>binary-release</id>
   <formats>
     <format>tar.gz</format>
+    <format>dir</format>
   </formats>
   <includeBaseDirectory>true</includeBaseDirectory>
   <moduleSets>

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/df587a46/distribution/src/resources/drill-override.conf
diff --git a/distribution/src/resources/drill-override.conf b/distribution/src/resources/drill-override.conf
index 5fe2362..e837e74 100644
+++ b/distribution/src/resources/drill-override.conf
@@ -94,5 +94,6 @@ drill.exec: {
     impl: "org.apache.drill.exec.work.batch.SpoolingRawBatchBuffer",
     delete: false,
     size: 100000000
-  }
-}
\ No newline at end of file
+  },
+  sort.purge.threshold : 100
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/df587a46/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index bcc113f..eefae64 100644
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -51,5 +51,6 @@ public interface ExecConstants {
   public static final String INCOMING_BUFFER_SIZE = "drill.exec.buffer.size"; // incoming buffer size (number of batches)
   public static final String SPOOLING_BUFFER_DELETE = "drill.exec.buffer.spooling.delete";
   public static final String SPOOLING_BUFFER_MEMORY = "drill.exec.buffer.spooling.size";
-  
+  public static final String BATCH_PURGE_THRESHOLD = "drill.exec.sort.purge.threshold";
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/df587a46/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Sort.java
diff index 82aa830..507a960 100644
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Sort.java
@@ -34,8 +34,8 @@ import com.fasterxml.jackson.annotation.JsonTypeName;
 public class Sort extends AbstractSingle{
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Sort.class);
  
-  private final List<Ordering> orderings;
-  private boolean reverse = false;
+  protected final List<Ordering> orderings;
+  protected boolean reverse = false;
  
   @JsonCreator
   public Sort(@JsonProperty("child") PhysicalOperator child, @JsonProperty("orderings") List<Ordering> orderings, @JsonProperty("reverse") boolean reverse) {

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/df587a46/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/TopN.java
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/TopN.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/TopN.java
new file mode 100644
index 0000000..79c5782
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/TopN.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.config;
+
+import