Home | About | Sematext search-lucene.com search-hadoop.com
 Search Hadoop and all its subprojects:

Switch to Plain View
Drill >> mail # dev >> [1/2] git commit: Updates to Parquet for VariableLengthVectors and Bit fields


Copy link to this message
-
[1/2] git commit: Updates to Parquet for VariableLengthVectors and Bit fields
Updated Branches:
  refs/heads/master 47985bad0 -> c50135256
Updates to Parquet for VariableLengthVectors and Bit fields
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/556bd963
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/556bd963
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/556bd963

Branch: refs/heads/master
Commit: 556bd963aff5e01ab041db3529a7ae4f847f9bd0
Parents: 47985ba
Author: Jason Altekruse <[EMAIL PROTECTED]>
Authored: Tue Aug 13 18:22:49 2013 -0700
Committer: Jacques Nadeau <[EMAIL PROTECTED]>
Committed: Thu Aug 15 18:54:14 2013 -0700

----------------------------------------------------------------------
 .../templates/VariableLengthVectors.java        |  3 +-
 .../drill/exec/store/parquet/BitReader.java     |  3 +-
 .../drill/exec/store/parquet/ColumnReader.java  |  5 +--
 .../exec/store/parquet/PageReadStatus.java      |  2 +-
 .../exec/store/parquet/ParquetRecordReader.java |  6 +--
 .../drill/exec/vector/BaseDataValueVector.java  |  4 +-
 .../org/apache/drill/exec/vector/BitVector.java | 13 ++++++-
 .../parquet_scan_screen_read_entry_replace.json | 39 ++++++++++++++++++++
 sandbox/prototype/pom.xml                       |  2 +-
 9 files changed, 60 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/556bd963/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java
index 3be6dc2..e3c59fc 100644
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java
@@ -14,7 +14,6 @@ import java.io.Closeable;
 import java.nio.ByteBuffer;
 import java.util.Random;
 
-import org.apache.commons.lang3.ArrayUtils;
 import org.apache.drill.exec.memory.BufferAllocator;
 import org.apache.drill.exec.proto.SchemaDefProtos;
 import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
@@ -75,7 +74,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
  
   @Override
   public FieldMetadata getMetadata() {
-    int len = (valueCount+1) * ${type.width} + getVarByteLength();
+    int len = (valueCount + 1) * ${type.width} + getVarByteLength();
     return FieldMetadata.newBuilder()
              .setDef(getField().getDef())
              .setValueCount(valueCount)

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/556bd963/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/BitReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/BitReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/BitReader.java
index c85d4aa..8727341 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/BitReader.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/BitReader.java
@@ -52,11 +52,10 @@ public final class BitReader extends ColumnReader {
       vectorData = ((BaseDataValueVector) valueVecHolder.getValueVector()).getData();
       nextByte = bytes[(int) Math.max(0, Math.ceil(pageReadStatus.valuesRead / 8.0) - 1)];
       readLengthInBits = recordsReadInThisIteration + pageReadStatus.bitShift;
-      //recordsReadInThisIteration -= (8 - pageReadStatus.bitShift);
 
       int i = 0;
       // read individual bytes with appropriate shifting
-      for (; i <= (int) readLength; i++) {
+      for (; i < (int) readLength; i++) {
         currentByte = nextByte;
         currentByte = (byte) (currentByte >>> pageReadStatus.bitShift);
         // mask the bits about to be added from the next byte

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/556bd963/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnReader.java
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnReader.java
index 99b65e6..c62613a 100644
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnReader.java
@@ -39,8 +39,6 @@ public abstract class ColumnReader {
 
   long readPositionInBuffer;
 
-  int compressedSize;
-
   // quick reference to see if the field is fixed length (as this requires an instanceof)
   boolean isFixedLength;
   // counter for the total number of values read from one or more pages
@@ -89,7 +87,6 @@ public abstract class ColumnReader {
       // if no page has been read, or all of the records have been read out of a page, read the next one
       if (pageReadStatus.currentPage == null
           || pageReadStatus.valuesRead == pageReadStatus.currentPage.getValueCount()) {
-        totalValuesRead += pageReadStatus.valuesRead;
         if (!pageReadStatus.next()) {
           break;
         }
@@ -107,7 +104,7 @@ public abstract class ColumnReader {
       }
     }
     while (valuesReadInCurrentPass < recordsToReadInThisPass && pageReadStatus.currentPage != null);
-    ((BaseDataValueVector) valueVecHolder.getValueVector()).getMutator().setValueCount(
+    valueVecHolder.getValueVector().getMutator().setValueCount(
         valuesReadInCurrentPass);
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/556bd963/sandbox/prototype/exec/j