Home | About | Sematext search-lucene.com search-hadoop.com
 Search Hadoop and all its subprojects:

Switch to Threaded View
Drill >> mail # dev >> [1/7] git commit: DRILL-178 Creating JSON Storage engine


Copy link to this message
-
[1/7] git commit: DRILL-178 Creating JSON Storage engine
Updated Branches:
  refs/heads/master e43093d9e -> 4515263d3
DRILL-178 Creating JSON Storage engine
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/dddae743
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/dddae743
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/dddae743

Branch: refs/heads/master
Commit: dddae743c78a9b193fc4bf4d350f6e25f4e9484c
Parents: e43093d
Author: Timothy Chen <[EMAIL PROTECTED]>
Authored: Tue Aug 20 23:53:41 2013 -0700
Committer: Jacques Nadeau <[EMAIL PROTECTED]>
Committed: Wed Aug 28 20:32:45 2013 -0700

----------------------------------------------------------------------
 .../drill/exec/store/json/JSONGroupScan.java    | 206 ++++++++++---------
 .../drill/exec/store/json/JSONRecordReader.java |  25 +--
 .../exec/store/json/JSONScanBatchCreator.java   |   2 +-
 .../exec/store/json/JSONStorageEngine.java      |  48 +++++
 .../store/json/JSONStorageEngineConfig.java     |  37 ++++
 .../drill/exec/store/json/JSONSubScan.java      | 118 ++++++-----
 .../physical/impl/TestSimpleFragmentRun.java    |   6 +-
 .../drill/exec/store/JSONRecordReaderTest.java  |  36 +++-
 .../resources/physical_json_scan_test1.json     |  17 +-
 9 files changed, 316 insertions(+), 179 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/dddae743/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONGroupScan.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONGroupScan.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONGroupScan.java
index ff5f474..b44565b 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONGroupScan.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONGroupScan.java
@@ -18,128 +18,142 @@
 
 package org.apache.drill.exec.store.json;
 
-import static com.google.common.base.Preconditions.checkArgument;
-
-import java.io.File;
-import java.net.URI;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-
+import com.fasterxml.jackson.annotation.*;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.logical.StorageEngineConfig;
+import org.apache.drill.exec.exception.SetupException;
 import org.apache.drill.exec.physical.EndpointAffinity;
 import org.apache.drill.exec.physical.OperatorCost;
 import org.apache.drill.exec.physical.ReadEntry;
+import org.apache.drill.exec.physical.ReadEntryWithPath;
 import org.apache.drill.exec.physical.base.AbstractGroupScan;
 import org.apache.drill.exec.physical.base.PhysicalOperator;
 import org.apache.drill.exec.physical.base.Size;
 import org.apache.drill.exec.physical.base.SubScan;
 import org.apache.drill.exec.proto.CoordinationProtos;
+import org.apache.drill.exec.store.StorageEngineRegistry;
+import org.apache.drill.exec.store.parquet.ParquetStorageEngineConfig;
+
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
 
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
+import static com.google.common.base.Preconditions.checkArgument;
 
 @JsonTypeName("json-scan")
 public class JSONGroupScan extends AbstractGroupScan {
-    private static int ESTIMATED_RECORD_SIZE = 1024; // 1kb
-
-    private LinkedList<JSONGroupScan.ScanEntry>[] mappings;
-    protected final List<JSONGroupScan.ScanEntry> readEntries;
-    private final OperatorCost cost;
-    private final Size size;
-    
-    @JsonCreator
-    public JSONGroupScan(@JsonProperty("entries") List<JSONGroupScan.ScanEntry> readEntries) {
-        this.readEntries = readEntries;
-        OperatorCost cost = new OperatorCost(0,0,0,0);
-        Size size = new Size(0,0);
-        for(JSONGroupScan.ScanEntry r : readEntries){
-          cost = cost.add(r.getCost());
-          size = size.add(r.getSize());
-        }
-        this.cost = cost;
-        this.size = size;
-    }
-
-    @SuppressWarnings("unchecked")
-    @Override
-    public void applyAssignments(List<CoordinationProtos.DrillbitEndpoint> endpoints) {
-        checkArgument(endpoints.size() <= readEntries.size());
-
-        mappings = new LinkedList[endpoints.size()];
-
-        int i = 0;
-        for (ScanEntry e : readEntries) {
-            if (i == endpoints.size()) i = 0;
-            LinkedList entries = mappings[i];
-            if (entries == null) {
-                entries = new LinkedList<>();
-                mappings[i] = entries;
-            }
-            entries.add(e);
-            i++;
-        }
+  private static int ESTIMATED_RECORD_SIZE = 1024; // 1kb
+  private final StorageEngineRegistry registry;
+  private final StorageEngineConfig engineConfig;
+
+  private LinkedList<JSONGroupScan.ScanEntry>[] mappings;
+  private final List<JSONGroupScan.ScanEntry> readEntries;
+  private final OperatorCost cost;
+  private final Size size;
+
+  @JsonCreator
+  public JSONGroupScan(@JsonProperty("entries") List<ScanEntry> entries,
+                       @JsonProperty("storageengine") JSONStorageEngineConfig storageEngineConfig,
+                       @JacksonInject StorageEngineRegistry engineRegistry) throws SetupException {
+    engineRegistry.init(DrillConfig.create());
+    this.registry = engineRegistry;
+    this.engineConfig = storageEngineConfig;
+    this.readEntries = entries;
+    OperatorCost cost = new OperatorCost(0, 0, 0, 0);
+    Size size = new Size(0, 0);
+    for (JSONGroupScan.ScanEntry r : readEntries) {
+      cost = cost.add(r.getCost());
+      size = size.add(r.getSize());
     }
-
-    @