Home | About | Sematext search-lucene.com search-hadoop.com
 Search Hadoop and all its subprojects:

Switch to Threaded View
Drill, mail # dev - [08/17] git commit: DRILL-498: cast to varchar only returns one row


Copy link to this message
-
[08/17] git commit: DRILL-498: cast to varchar only returns one row
jacques@... 2014-04-20, 04:08
DRILL-498: cast to varchar only returns one row
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/e2bb5418
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/e2bb5418
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/e2bb5418

Branch: refs/heads/master
Commit: e2bb54188e060b79bc8bc478390af01a7ab91f10
Parents: 7ec8669
Author: Steven Phillips <[EMAIL PROTECTED]>
Authored: Wed Apr 9 20:03:37 2014 -0700
Committer: Jacques Nadeau <[EMAIL PROTECTED]>
Committed: Sat Apr 19 21:07:28 2014 -0700

 .../exec/store/parquet/VarLenBinaryReader.java  | 51 ++++++++++----------
 1 file changed, 25 insertions(+), 26 deletions(-)
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e2bb5418/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java
index d9e498e..09d19a8 100644
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java
@@ -87,20 +87,16 @@ public class VarLenBinaryReader {
     NullableVarBinaryVector currNullVec;
     // write the first 0 offset
     for (ColumnReader columnReader : columns) {
-      currVec = (VarBinaryVector) columnReader.valueVecHolder.getValueVector();
-      currVec.getAccessor().getOffsetVector().getData().writeInt(0);
       columnReader.bytesReadInCurrentPass = 0;
       columnReader.valuesReadInCurrentPass = 0;
     }
     // same for the nullable columns
     for (NullableVarLengthColumn columnReader : nullableColumns) {
-      currNullVec = (NullableVarBinaryVector) columnReader.valueVecHolder.getValueVector();
-      currNullVec.getMutator().getVectorWithValues().getAccessor().getOffsetVector().getData().writeInt(0);
       columnReader.bytesReadInCurrentPass = 0;
       columnReader.valuesReadInCurrentPass = 0;
       columnReader.nullsRead = 0;
     }
-    do {
+    outer: do {
       lengthVarFieldsInCurrentRecord = 0;
       for (ColumnReader columnReader : columns) {
         if (recordsReadInCurrentPass == columnReader.valueVecHolder.getValueVector().getValueCapacity()){
@@ -121,6 +117,11 @@ public class VarLenBinaryReader {
         columnReader.dataTypeLengthInBits = BytesUtils.readIntLittleEndian(bytes,
             (int) columnReader.pageReadStatus.readPosInBytes);
         lengthVarFieldsInCurrentRecord += columnReader.dataTypeLengthInBits;
+
+        if (columnReader.bytesReadInCurrentPass + columnReader.dataTypeLengthInBits > ((VarBinaryVector) columnReader.valueVecHolder.getValueVector()).getData().capacity()) {
+          break outer;
+        }
+
       }
       for (NullableVarLengthColumn columnReader : nullableColumns) {
         // check to make sure there is capacity for the next value (for nullables this is a check to see if there is
@@ -150,57 +151,55 @@ public class VarLenBinaryReader {
             (int) columnReader.pageReadStatus.readPosInBytes);
         lengthVarFieldsInCurrentRecord += columnReader.dataTypeLengthInBits;
 
+        if (columnReader.bytesReadInCurrentPass + columnReader.dataTypeLengthInBits > ((NullableVarBinaryVector) columnReader.valueVecHolder.getValueVector()).getData().capacity()) {
+          break outer;
+        }
       }
       // check that the next record will fit in the batch
       if (rowGroupFinished || (recordsReadInCurrentPass + 1) * parentReader.getBitWidthAllFixedFields() + lengthVarFieldsInCurrentRecord
           > parentReader.getBatchSize()){
-        break;
-      }
-      else{
-        recordsReadInCurrentPass++;
+        break outer;
       }
       for (ColumnReader columnReader : columns) {
         bytes = columnReader.pageReadStatus.pageDataByteArray;
         currVec = (VarBinaryVector) columnReader.valueVecHolder.getValueVector();
         // again, I am re-purposing the unused field here, it is a length n BYTES, not bits
-        currVec.getAccessor().getOffsetVector().getData().writeInt((int) columnReader.bytesReadInCurrentPass  +
-            columnReader.dataTypeLengthInBits - 4 * (int) columnReader.valuesReadInCurrentPass);
-        currVec.getData().writeBytes(bytes, (int) columnReader.pageReadStatus.readPosInBytes + 4,
-            columnReader.dataTypeLengthInBits);
+        boolean success = currVec.getMutator().setSafe(columnReader.valuesReadInCurrentPass, bytes,
+                (int) columnReader.pageReadStatus.readPosInBytes + 4, columnReader.dataTypeLengthInBits);
+        assert success;
         columnReader.pageReadStatus.readPosInBytes += columnReader.dataTypeLengthInBits + 4;
         columnReader.bytesReadInCurrentPass += columnReader.dataTypeLengthInBits + 4;
         columnReader.pageReadStatus.valuesRead++;
         columnReader.valuesReadInCurrentPass++;
-        currVec.getMutator().setValueCount((int)recordsReadInCurrentPass);
       }
       for (NullableVarLengthColumn columnReader : nullableColumns) {
         bytes = columnReader.pageReadStatus.pageDataByteArray;
         currNullVec = (NullableVarBinaryVector) columnReader.valueVecHolder.getValueVector();
         // again, I am re-purposing the unused field here, it is a length n BYTES, not bits
-        currNullVec.getMutator().getVectorWithValues().getAccessor().getOffsetVector().getData()
-            .writeInt(
-                (int) columnReader.bytesReadInCurrentPass  +
-                columnReader.dataTypeLengthInBits - 4 * (columnReader.valuesReadInCurrentPass -
-                    (columnReader.currentValNull ? Math.max (0, columnReader.nullsRead - 1) : columnReader.nullsRead)));
-        columnReader.currentValNull = false;
-        if (columnReader.dataTypeLengthInBits > 0){
-          currNullVec.getData().writeBytes(bytes, (int) columnReader.pageReadStatus.readPosInBytes + 4,
-              columnRea