Home | About | Sematext search-lucene.com search-hadoop.com
 Search Hadoop and all its subprojects:

Switch to Plain View
Drill >> mail # dev >> [2/2] git commit: Updates for Parquet varlen merge


Copy link to this message
-
[2/2] git commit: Updates for Parquet varlen merge
Updates for Parquet varlen merge
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/c5013525
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/c5013525
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/c5013525

Branch: refs/heads/master
Commit: c501352565d1c8e39e1f6e7e66d9fa3a0fe2bf9e
Parents: 556bd96
Author: Jacques Nadeau <[EMAIL PROTECTED]>
Authored: Thu Aug 15 20:38:04 2013 -0700
Committer: Jacques Nadeau <[EMAIL PROTECTED]>
Committed: Thu Aug 15 20:38:04 2013 -0700

----------------------------------------------------------------------
 .../exec/store/parquet/ParquetGroupScan.java    | 34 ++++++-----
 .../store/parquet/ParquetRecordReaderTest.java  | 61 +++++++++++++++-----
 .../exec/store/parquet/TestFileGenerator.java   |  8 ++-
 .../resources/parquet/parquet_scan_screen.json  | 44 ++++++++++++++
 .../parquet_scan_screen_read_entry_replace.json | 39 +++++++++++++
 .../parquet_scan_union_screen_physical.json     | 35 +++++++++++
 .../src/test/resources/parquet_scan_screen.json | 44 --------------
 .../parquet_scan_screen_read_entry_replace.json | 39 -------------
 .../parquet_scan_union_screen_physical.json     | 35 -----------
 9 files changed, 186 insertions(+), 153 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c5013525/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
index 66c1550..9e48d33 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
@@ -18,16 +18,14 @@
 package org.apache.drill.exec.store.parquet;
 
 import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
 
-import com.fasterxml.jackson.annotation.JacksonInject;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-import org.apache.drill.common.JSONOptions;
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.exec.exception.SetupException;
 import org.apache.drill.exec.physical.EndpointAffinity;
@@ -38,13 +36,9 @@ import org.apache.drill.exec.physical.base.AbstractGroupScan;
 import org.apache.drill.exec.physical.base.PhysicalOperator;
 import org.apache.drill.exec.physical.base.Size;
 import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
-
-import com.google.common.base.Preconditions;
-
-import org.apache.drill.exec.server.DrillbitContext;
-import org.apache.drill.exec.store.StorageEngineRegistry;
 import org.apache.drill.exec.store.AffinityCalculator;
-import org.apache.drill.exec.store.mock.MockGroupScanPOP;
+import org.apache.drill.exec.store.StorageEngineRegistry;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 
@@ -54,6 +48,12 @@ import parquet.hadoop.metadata.ColumnChunkMetaData;
 import parquet.hadoop.metadata.ParquetMetadata;
 import parquet.org.codehaus.jackson.annotate.JsonCreator;
 
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.base.Preconditions;
+
 
 @JsonTypeName("parquet-scan")
 public class ParquetGroupScan extends AbstractGroupScan {
@@ -116,8 +116,10 @@ public class ParquetGroupScan extends AbstractGroupScan {
     ColumnChunkMetaData columnChunkMetaData;
     for (ReadEntryWithPath readEntryWithPath : entries){
       Path path = new Path(readEntryWithPath.getPath());
-
       ParquetMetadata footer = ParquetFileReader.readFooter(this.storageEngine.getHadoopConfig(), path);
+//      FileSystem fs = FileSystem.get(this.storageEngine.getHadoopConfig());
+//      FileStatus status = fs.getFileStatus(path);
+//      ParquetMetadata footer = ParquetFileReader.readFooter(this.storageEngine.getHadoopConfig(), status);
       readEntryWithPath.getPath();
 
       int i = 0;

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c5013525/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
index 5628f50..1d91455 100644
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
@@ -27,6 +27,7 @@ import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.types.TypeProtos;
@@ -46,6 +47,7 @@ import org.apache.drill.exec.server.RemoteServiceSet;
 import org.apache.drill.exec.store.parquet.TestFileGenerator.FieldInfo;
 import org.apache.drill.exec.vector.BaseDataValueVector;
 import org.apache.drill.exec.vector.ValueVector;
+import org.junit.Befor