You are viewing a plain text version of this content. The canonical link for it is here.

Posted to dev@drill.apache.org by ja...@apache.org on 2013/07/20 03:57:22 UTC

[01/53] [abbrv] WIP fragmentation, physical plan, byte compiling, some vector work

Updated Branches:
  refs/heads/master 5052b64d9 -> 8186b5a1a


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/MockRecordReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/MockRecordReader.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/MockRecordReader.java
new file mode 100644
index 0000000..e1f56bd
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/MockRecordReader.java
@@ -0,0 +1,108 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.store;
+
+import org.apache.drill.common.expression.types.DataType;
+import org.apache.drill.exec.exception.ExecutionSetupException;
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.OutputMutator;
+import org.apache.drill.exec.record.BatchSchema;
+import org.apache.drill.exec.record.vector.Int16Vector;
+import org.apache.drill.exec.record.vector.Int32Vector;
+import org.apache.drill.exec.record.vector.ValueVector;
+
+public class MockRecordReader implements RecordReader {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MockRecordReader.class);
+
+  private BatchSchema expectedSchema;
+  private OutputMutator output;
+  private MockRecordConfig config;
+  private FragmentContext context;
+  private ValueVector<?>[] valueVectors;
+  private int recordsRead;
+
+  public MockRecordReader(FragmentContext context, MockRecordConfig config) {
+    this.config = config;
+  }
+
+  private int getEstimatedRecordSize(DataType[] types) {
+    int x = 0;
+    for (int i = 0; i < types.length; i++) {
+      x += getEstimatedColumnSize(i);
+    }
+    return x;
+  }
+
+  private int getEstimatedColumnSize(int fieldId) {
+    return 4;
+  }
+
+  private ValueVector<?> getVector(int fieldId, DataType dt, int length) {
+    ValueVector<?> v;
+    if (dt == DataType.INT16) {
+      v = new Int16Vector(fieldId, context.getAllocator());
+    } else if (dt == DataType.INT32) {
+      v = new Int32Vector(fieldId, context.getAllocator());
+    } else {
+      throw new UnsupportedOperationException();
+    }
+    v.allocateNew(length);
+    return v;
+
+  }
+
+  @Override
+  public void setup(BatchSchema expectedSchema, OutputMutator output) throws ExecutionSetupException {
+    try {
+      this.expectedSchema = expectedSchema;
+      this.output = output;
+      int estimateRowSize = getEstimatedRecordSize(config.getTypes());
+      valueVectors = new ValueVector<?>[config.getTypes().length];
+      int batchRecordCount = 250000 / estimateRowSize;
+
+      for (int i = 0; i < config.getTypes().length; i++) {
+        valueVectors[i] = getVector(i, config.getTypes()[i], batchRecordCount);
+        output.addField(i, valueVectors[i]);
+      }
+    } catch (SchemaChangeException e) {
+      throw new ExecutionSetupException("Failure while setting up fields", e);
+    }
+
+  }
+
+  @Override
+  public int next() {
+    int recordSetSize = Math.min(valueVectors[0].size(), this.config.getRecordCount()- recordsRead);
+    recordsRead += recordSetSize;
+    return recordSetSize;
+  }
+
+  @Override
+  public void cleanup() {
+    for (int i = 0; i < valueVectors.length; i++) {
+      try {
+        output.removeField(valueVectors[i].getField().getFieldId());
+      } catch (SchemaChangeException e) {
+        logger.warn("Failure while trying tremove field.", e);
+      }
+      valueVectors[i].close();
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/MockStorageEngine.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/MockStorageEngine.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/MockStorageEngine.java
new file mode 100644
index 0000000..cc82540
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/MockStorageEngine.java
@@ -0,0 +1,54 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.store;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.apache.drill.common.logical.data.Scan;
+import org.apache.drill.common.proto.CoordinationProtos.DrillbitEndpoint;
+import org.apache.drill.exec.ops.FragmentContext;
+
+import com.google.common.collect.ListMultimap;
+
+public class MockStorageEngine extends AbstractStorageEngine{
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MockStorageEngine.class);
+
+  @Override
+  public boolean supportsRead() {
+    return true;
+  }
+
+  @Override
+  public Collection<ReadEntry> getReadEntries(Scan scan) throws IOException {
+    return null;
+  }
+
+  @Override
+  public ListMultimap<ReadEntry, DrillbitEndpoint> getReadLocations(Collection<ReadEntry> entries) {
+    return null;
+  }
+
+  @Override
+  public RecordReader getReader(FragmentContext context, ReadEntry readEntry) throws IOException {
+    return null;
+  }
+
+  
+  
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/test/resources/drill-module.conf
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/drill-module.conf b/sandbox/prototype/exec/java-exec/src/test/resources/drill-module.conf
index 7c97f66..8785736 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/drill-module.conf
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/drill-module.conf
@@ -10,7 +10,12 @@ drill.exec: {
   optimizer: {
     implementation: "org.apache.drill.exec.opt.IdentityOptimizer"
   },
-  
+  storage: {
+	packages += "org.apache.drill.exec.store"  
+  }
+  metrics : { 
+  	context: "drillbit"
+  },
   zk: {
 	connect: "localhost:2181",
 	root: "/drill",

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/test/resources/physical_screen.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/physical_screen.json b/sandbox/prototype/exec/java-exec/src/test/resources/physical_screen.json
new file mode 100644
index 0000000..c63aef1
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/physical_screen.json
@@ -0,0 +1,25 @@
+{
+    head:{
+        type:"APACHE_DRILL_PHYSICAL",
+        version:"1",
+        generator:{
+            type:"manual"
+        }
+    },
+    graph:[
+        {
+            @id:1,
+            pop:"mock-scan",
+            url: "http://apache.org",
+            entries:[
+            	{id:1}
+            ],
+            cost: { disk: 1.0, memory: 1.0, cpu: 1.0, network: 1.0}
+        },
+        {
+            @id:2,
+            pop: "screen",
+            child: 1
+        }
+    ]
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/test/resources/physical_simpleexchange.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/physical_simpleexchange.json b/sandbox/prototype/exec/java-exec/src/test/resources/physical_simpleexchange.json
new file mode 100644
index 0000000..e332785
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/physical_simpleexchange.json
@@ -0,0 +1,41 @@
+{
+    head:{
+        type:"APACHE_DRILL_PHYSICAL",
+        version:"1",
+        generator:{
+            type:"manual"
+        }
+    },
+    graph:[
+        {
+            @id:1,
+            pop:"mock-scan",
+            url: "http://apache.org",
+            entries:[
+            	{id:1}
+            ],
+            cost: { disk: 1.0, memory: 1.0, cpu: 1.0, network: 1.0}
+        },
+        {
+            @id:2,
+            pop: "partition-to-random-exchange",
+            child: 1,
+            partition: {
+              mode: "DUPLICATE"
+            }
+        },
+        {
+            @id:3,
+            child: 2,
+            pop:"filter",
+            expr: "b > 5",
+            cost: { disk: 1.0, memory: 1.0, cpu: 1.0, network: 1.0}
+        },
+        {
+            @id: 4,
+            child: 3,
+            pop: "mock-store",
+            cost: { disk: 1.0, memory: 1.0, cpu: 1.0, network: 1.0}
+        }
+    ]
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/test/sh/runbit
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/sh/runbit b/sandbox/prototype/exec/java-exec/src/test/sh/runbit
index 10fc1d5..2885f7f 100755
--- a/sandbox/prototype/exec/java-exec/src/test/sh/runbit
+++ b/sandbox/prototype/exec/java-exec/src/test/sh/runbit
@@ -5,5 +5,5 @@ PROJECT_ROOT=../../../
 mvn dependency:build-classpath -f=$PROJECT_ROOT/pom.xml -Dmdep.outputFile=target/sh/cp.txt
 CP=`cat $PROJECT_ROOT/target/sh/cp.txt`
 CP=$CP:$PROJECT_ROOT/target/classes:$PROJECT_ROOT/target/test-classes
-java -cp $CP org.apache.drill.exec.server.Drillbit
+java -javaagent:/src/jrebel/jrebel.jar -cp $CP org.apache.drill.exec.server.Drillbit
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/pom.xml
----------------------------------------------------------------------
diff --git a/sandbox/prototype/pom.xml b/sandbox/prototype/pom.xml
index a4abf8b..fa89417 100644
--- a/sandbox/prototype/pom.xml
+++ b/sandbox/prototype/pom.xml
@@ -16,6 +16,20 @@
 
 	<name>prototype-parent</name>
 
+        <repositories>
+                <repository>
+                        <releases>
+                                <enabled>true</enabled>
+                                <updatePolicy>always</updatePolicy>
+                                <checksumPolicy>warn</checksumPolicy>
+                        </releases>
+                        <id>conjars</id>
+                        <name>Conjars</name>
+                        <url>http://conjars.org/repo</url>
+                        <layout>default</layout>
+                </repository>
+        </repositories>
+
 	<properties>
 		<dep.junit.version>4.11</dep.junit.version>
 		<dep.slf4j.version>1.7.2</dep.slf4j.version>
@@ -144,4 +158,4 @@
 		<module>planner</module>
 		<module>sqlparser</module>
 	</modules>
-</project>
\ No newline at end of file
+</project>