You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by at...@apache.org on 2009/02/04 23:41:58 UTC
svn commit: r740927 - in /hadoop/hive/branches/branch-0.2: ./
ql/src/java/org/apache/hadoop/hive/ql/metadata/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
Author: athusoo
Date: Wed Feb 4 22:41:58 2009
New Revision: 740927
URL: http://svn.apache.org/viewvc?rev=740927&view=rev
Log:
HIVE-189. Make sampling tests deterministic by sorting output. (athusoo)
Modified:
hadoop/hive/branches/branch-0.2/CHANGES.txt
hadoop/hive/branches/branch-0.2/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
hadoop/hive/branches/branch-0.2/ql/src/test/queries/clientpositive/input3_limit.q
hadoop/hive/branches/branch-0.2/ql/src/test/queries/clientpositive/sample3.q
hadoop/hive/branches/branch-0.2/ql/src/test/queries/clientpositive/sample5.q
hadoop/hive/branches/branch-0.2/ql/src/test/results/clientpositive/input3_limit.q.out
hadoop/hive/branches/branch-0.2/ql/src/test/results/clientpositive/sample3.q.out
hadoop/hive/branches/branch-0.2/ql/src/test/results/clientpositive/sample5.q.out
Modified: hadoop/hive/branches/branch-0.2/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.2/CHANGES.txt?rev=740927&r1=740926&r2=740927&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.2/CHANGES.txt (original)
+++ hadoop/hive/branches/branch-0.2/CHANGES.txt Wed Feb 4 22:41:58 2009
@@ -269,3 +269,5 @@
HIVE-262. Fix for joins wherein a merged join having and outer or
right outer join gave spurious rows in join. (Namit Jain via athusoo)
+
+ HIVE-189. Make sampling tests deterministic by sorting output. (athusoo)
Modified: hadoop/hive/branches/branch-0.2/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.2/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java?rev=740927&r1=740926&r2=740927&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.2/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java (original)
+++ hadoop/hive/branches/branch-0.2/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java Wed Feb 4 22:41:58 2009
@@ -21,6 +21,7 @@
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -46,142 +47,141 @@
*/
public class Partition {
- @SuppressWarnings("nls")
- static final private Log LOG = LogFactory.getLog("hive.ql.metadata.Partition");
+ @SuppressWarnings("nls")
+ static final private Log LOG = LogFactory.getLog("hive.ql.metadata.Partition");
- private Table table;
- private org.apache.hadoop.hive.metastore.api.Partition tPartition;
- /**
- * @return the tPartition
- */
- public org.apache.hadoop.hive.metastore.api.Partition getTPartition() {
- return tPartition;
- }
-
- private LinkedHashMap<String, String> spec;
-
- /**
- * @return
- * @see org.apache.hadoop.hive.metastore.api.Partition#getValues()
- */
- public List<String> getValues() {
- return tPartition.getValues();
- }
-
- private Path partPath;
- private URI partURI;
-
- public Partition(Table tbl, org.apache.hadoop.hive.metastore.api.Partition tp) throws HiveException {
- initialize(tbl, tp);
- }
+ private Table table;
+ private org.apache.hadoop.hive.metastore.api.Partition tPartition;
+ /**
+ * @return the tPartition
+ */
+ public org.apache.hadoop.hive.metastore.api.Partition getTPartition() {
+ return tPartition;
+ }
+
+ private LinkedHashMap<String, String> spec;
+
+ /**
+ * @return
+ * @see org.apache.hadoop.hive.metastore.api.Partition#getValues()
+ */
+ public List<String> getValues() {
+ return tPartition.getValues();
+ }
+
+ private Path partPath;
+ private URI partURI;
+
+ public Partition(Table tbl, org.apache.hadoop.hive.metastore.api.Partition tp) throws HiveException {
+ initialize(tbl, tp);
+ }
+
+ /**
+ * Create partition object with the given info.
+ * @param tbl Table the partition will be in.
+ * @param partSpec Partition specifications.
+ * @param location Location of the partition, relative to the table.
+ * @throws HiveException Thrown if we could not create the partition.
+ */
+ public Partition(Table tbl, Map<String, String> partSpec,
+ Path location) throws HiveException {
+
+ List<String> pvals = new ArrayList<String>();
+ for (FieldSchema field : tbl.getPartCols()) {
+ pvals.add(partSpec.get(field.getName()));
+ }
+
+ org.apache.hadoop.hive.metastore.api.Partition tpart =
+ new org.apache.hadoop.hive.metastore.api.Partition();
+ tpart.setDbName(tbl.getDbName());
+ tpart.setTableName(tbl.getName());
+ tpart.setValues(pvals);
+
+ StorageDescriptor sd = new StorageDescriptor();
+ try {
+ //replace with THRIFT-138
+ TMemoryBuffer buffer = new TMemoryBuffer(1024);
+ TBinaryProtocol prot = new TBinaryProtocol(buffer);
+ tbl.getTTable().getSd().write(prot);
+
+ sd.read(prot);
+ } catch (TException e) {
+ LOG.error("Could not create a copy of StorageDescription");
+ throw new HiveException("Could not create a copy of StorageDescription");
+ }
+
+ tpart.setSd(sd);
+ tpart.getSd().setLocation(location.toString());
+
+ initialize(tbl, tpart);
+ }
+
+ /**
+ * Initializes this object with the given variables
+ * @param tbl Table the partition belongs to
+ * @param tp Thrift Partition object
+ * @throws HiveException Thrown if we cannot initialize the partition
+ */
+ private void initialize(Table tbl,
+ org.apache.hadoop.hive.metastore.api.Partition tp)
+ throws HiveException {
+
+ this.table = tbl;
+ this.tPartition = tp;
+ this.partName = "";
- /**
- * Create partition object with the given info.
- * @param tbl Table the partition will be in.
- * @param partSpec Partition specifications.
- * @param location Location of the partition, relative to the table.
- * @throws HiveException Thrown if we could not create the partition.
- */
- public Partition(Table tbl, Map<String, String> partSpec,
- Path location) throws HiveException {
-
- List<String> pvals = new ArrayList<String>();
- for (FieldSchema field : tbl.getPartCols()) {
- pvals.add(partSpec.get(field.getName()));
- }
-
- org.apache.hadoop.hive.metastore.api.Partition tpart =
- new org.apache.hadoop.hive.metastore.api.Partition();
- tpart.setDbName(tbl.getDbName());
- tpart.setTableName(tbl.getName());
- tpart.setValues(pvals);
-
- StorageDescriptor sd = new StorageDescriptor();
+ if(tbl.isPartitioned()) {
try {
- //replace with THRIFT-138
- TMemoryBuffer buffer = new TMemoryBuffer(1024);
- TBinaryProtocol prot = new TBinaryProtocol(buffer);
- tbl.getTTable().getSd().write(prot);
-
- sd.read(prot);
- } catch (TException e) {
- LOG.error("Could not create a copy of StorageDescription");
- throw new HiveException("Could not create a copy of StorageDescription");
- }
-
- tpart.setSd(sd);
- tpart.getSd().setLocation(location.toString());
-
- initialize(tbl, tpart);
- }
-
- /**
- * Initializes this object with the given variables
- * @param tbl Table the partition belongs to
- * @param tp Thrift Partition object
- * @throws HiveException Thrown if we cannot initialize the partition
- */
- private void initialize(Table tbl,
- org.apache.hadoop.hive.metastore.api.Partition tp)
- throws HiveException {
-
- this.table = tbl;
- this.tPartition = tp;
- this.partName = "";
-
- if(tbl.isPartitioned()) {
- try {
- this.partName = Warehouse.makePartName(tbl.getPartCols(),
- tp.getValues());
- } catch (MetaException e) {
- throw new HiveException("Invalid partition for table " + tbl.getName(),
- e);
- }
- this.partPath = new Path(tp.getSd().getLocation());
- } else {
- // We are in the HACK territory.
- // SemanticAnalyzer expects a single partition whose schema
- // is same as the table partition.
- this.partPath = table.getPath();
+ this.partName = Warehouse.makePartName(tbl.getPartCols(),
+ tp.getValues());
+ } catch (MetaException e) {
+ throw new HiveException("Invalid partition for table " + tbl.getName(),
+ e);
}
-
- this.spec = new LinkedHashMap<String, String>(tbl.createSpec(tp));
- this.partURI = partPath.toUri();
- }
-
-
- public String getName() {
- return partName;
- }
-
- public Table getTable() {
- return (this.table);
- }
-
- public Path [] getPath() {
- Path [] ret = new Path [1];
- ret[0] = this.partPath;
- return(ret);
- }
-
- public Path getPartitionPath() {
- return this.partPath;
- }
-
- final public URI getDataLocation() {
- return this.partURI;
- }
-
- /**
- * The number of buckets is a property of the partition. However - internally we are just
- * storing it as a property of the table as a short term measure.
- */
- public int getBucketCount() {
- return this.table.getNumBuckets();
- /*
+ this.partPath = new Path(tp.getSd().getLocation());
+ } else {
+ // We are in the HACK territory.
+ // SemanticAnalyzer expects a single partition whose schema
+ // is same as the table partition.
+ this.partPath = table.getPath();
+ }
+
+ this.spec = new LinkedHashMap<String, String>(tbl.createSpec(tp));
+ this.partURI = partPath.toUri();
+ }
+
+ public String getName() {
+ return partName;
+ }
+
+ public Table getTable() {
+ return (this.table);
+ }
+
+ public Path [] getPath() {
+ Path [] ret = new Path [1];
+ ret[0] = this.partPath;
+ return(ret);
+ }
+
+ public Path getPartitionPath() {
+ return this.partPath;
+ }
+
+ final public URI getDataLocation() {
+ return this.partURI;
+ }
+
+ /**
+ * The number of buckets is a property of the partition. However - internally we are just
+ * storing it as a property of the table as a short term measure.
+ */
+ public int getBucketCount() {
+ return this.table.getNumBuckets();
+ /*
TODO: Keeping this code around for later use when we will support
sampling on tables which are not created with CLUSTERED INTO clause
-
+
// read from table meta data
int numBuckets = this.table.getNumBuckets();
if (numBuckets == -1) {
@@ -198,147 +198,148 @@
}
}
return numBuckets;
- */
- }
-
- public List<String> getBucketCols() {
- return this.table.getBucketCols();
- }
-
- /**
- * mapping from bucket number to bucket path
- */
- //TODO: add test case and clean it up
- @SuppressWarnings("nls")
- public Path getBucketPath(int bucketNum) {
- try {
- FileSystem fs = FileSystem.get(this.table.getDataLocation(), Hive.get().getConf());
- String pathPattern = this.partPath.toString();
- if (getBucketCount() > 0) {
- pathPattern = pathPattern + "/*";
- }
- LOG.info("Path pattern = " + pathPattern);
- FileStatus srcs[] = fs.globStatus(new Path(pathPattern));
- for (FileStatus src: srcs) {
- LOG.info("Got file: " + src.getPath());
- }
- return srcs[bucketNum].getPath();
- }
- catch (Exception e) {
- throw new RuntimeException("Cannot get bucket path for bucket " + bucketNum, e);
- }
- // return new Path(this.partPath, String.format("part-%1$05d", bucketNum));
- }
-
- /**
- * mapping from a Path to the bucket number if any
*/
- private static Pattern bpattern = Pattern.compile("part-([0-9][0-9][0-9][0-9][0-9])");
+ }
- private String partName;
- @SuppressWarnings("nls")
- public static int getBucketNum(Path p) {
- Matcher m = bpattern.matcher(p.getName());
- if(m.find()) {
- String bnum_str = m.group(1);
- try {
- return (Integer.parseInt(bnum_str));
- } catch (NumberFormatException e) {
- throw new RuntimeException("Unexpected error parsing: "+p.getName()+","+bnum_str);
- }
- }
- return 0;
- }
-
-
- @SuppressWarnings("nls")
- public Path [] getPath(Sample s) throws HiveException {
- if(s == null) {
- return getPath();
- } else {
- int bcount = this.getBucketCount();
- if(bcount == 0) {
- return getPath();
- }
-
- Dimension d = s.getSampleDimension();
- if(!d.getDimensionId().equals(this.table.getBucketingDimensionId())) {
- // if the bucket dimension is not the same as the sampling dimension
- // we must scan all the data
- return getPath();
- }
-
- int scount = s.getSampleFraction();
- ArrayList<Path> ret = new ArrayList<Path> ();
-
- if(bcount == scount) {
- ret.add(getBucketPath(s.getSampleNum()-1));
- } else if (bcount < scount) {
- if((scount/bcount)*bcount != scount) {
- throw new HiveException("Sample Count"+scount+" is not a multiple of bucket count " +
- bcount + " for table " + this.table.getName());
- }
- // undersampling a bucket
- ret.add(getBucketPath((s.getSampleNum()-1)%bcount));
- } else if (bcount > scount) {
- if((bcount/scount)*scount != bcount) {
- throw new HiveException("Sample Count"+scount+" is not a divisor of bucket count " +
- bcount + " for table " + this.table.getName());
- }
- // sampling multiple buckets
- for(int i=0; i<bcount/scount; i++) {
- ret.add(getBucketPath(i*scount + (s.getSampleNum()-1)));
- }
- }
- return(ret.toArray(new Path[ret.size()]));
- }
+ public List<String> getBucketCols() {
+ return this.table.getBucketCols();
+ }
+
+ /**
+ * mapping from bucket number to bucket path
+ */
+ //TODO: add test case and clean it up
+ @SuppressWarnings("nls")
+ public Path getBucketPath(int bucketNum) {
+ try {
+ FileSystem fs = FileSystem.get(this.table.getDataLocation(), Hive.get().getConf());
+ String pathPattern = this.partPath.toString();
+ if (getBucketCount() > 0) {
+ pathPattern = pathPattern + "/*";
+ }
+ LOG.info("Path pattern = " + pathPattern);
+ FileStatus srcs[] = fs.globStatus(new Path(pathPattern));
+ Arrays.sort(srcs);
+ for (FileStatus src: srcs) {
+ LOG.info("Got file: " + src.getPath());
+ }
+ return srcs[bucketNum].getPath();
}
-
- public LinkedHashMap<String, String> getSpec() {
- return this.spec;
+ catch (Exception e) {
+ throw new RuntimeException("Cannot get bucket path for bucket " + bucketNum, e);
}
+ // return new Path(this.partPath, String.format("part-%1$05d", bucketNum));
+ }
- /**
- * Replaces files in the partition with new data set specified by srcf. Works by moving files
- *
- * @param srcf Files to be moved. Leaf Directories or Globbed File Paths
- */
- @SuppressWarnings("nls")
- protected void replaceFiles(Path srcf) throws HiveException {
- FileSystem fs;
+ /**
+ * mapping from a Path to the bucket number if any
+ */
+ private static Pattern bpattern = Pattern.compile("part-([0-9][0-9][0-9][0-9][0-9])");
+
+ private String partName;
+ @SuppressWarnings("nls")
+ public static int getBucketNum(Path p) {
+ Matcher m = bpattern.matcher(p.getName());
+ if(m.find()) {
+ String bnum_str = m.group(1);
try {
- fs = FileSystem.get(table.getDataLocation(), Hive.get().getConf());
- Hive.get().replaceFiles(srcf, partPath, fs);
- } catch (IOException e) {
- throw new HiveException("addFiles: filesystem error in check phase", e);
+ return (Integer.parseInt(bnum_str));
+ } catch (NumberFormatException e) {
+ throw new RuntimeException("Unexpected error parsing: "+p.getName()+","+bnum_str);
}
}
+ return 0;
+ }
- /**
- * Inserts files specified into the partition. Works by moving files
- *
- * @param srcf Files to be moved. Leaf Directories or Globbed File Paths
- */
- @SuppressWarnings("nls")
- protected void copyFiles(Path srcf) throws HiveException {
- FileSystem fs;
- try {
- fs = FileSystem.get(table.getDataLocation(), Hive.get().getConf());
- Hive.get().copyFiles(srcf, partPath, fs);
- } catch (IOException e) {
- throw new HiveException("addFiles: filesystem error in check phase", e);
+
+ @SuppressWarnings("nls")
+ public Path [] getPath(Sample s) throws HiveException {
+ if(s == null) {
+ return getPath();
+ } else {
+ int bcount = this.getBucketCount();
+ if(bcount == 0) {
+ return getPath();
}
- }
- @SuppressWarnings("nls")
- @Override
- public String toString() {
- String pn = "Invalid Partition";
- try {
- pn = Warehouse.makePartName(spec);
- } catch (MetaException e) {
- // ignore as we most probably in an exception path already otherwise this error wouldn't occur
+ Dimension d = s.getSampleDimension();
+ if(!d.getDimensionId().equals(this.table.getBucketingDimensionId())) {
+ // if the bucket dimension is not the same as the sampling dimension
+ // we must scan all the data
+ return getPath();
}
- return table.toString() + "(" + pn + ")";
+
+ int scount = s.getSampleFraction();
+ ArrayList<Path> ret = new ArrayList<Path> ();
+
+ if(bcount == scount) {
+ ret.add(getBucketPath(s.getSampleNum()-1));
+ } else if (bcount < scount) {
+ if((scount/bcount)*bcount != scount) {
+ throw new HiveException("Sample Count"+scount+" is not a multiple of bucket count " +
+ bcount + " for table " + this.table.getName());
+ }
+ // undersampling a bucket
+ ret.add(getBucketPath((s.getSampleNum()-1)%bcount));
+ } else if (bcount > scount) {
+ if((bcount/scount)*scount != bcount) {
+ throw new HiveException("Sample Count"+scount+" is not a divisor of bucket count " +
+ bcount + " for table " + this.table.getName());
+ }
+ // sampling multiple buckets
+ for(int i=0; i<bcount/scount; i++) {
+ ret.add(getBucketPath(i*scount + (s.getSampleNum()-1)));
+ }
+ }
+ return(ret.toArray(new Path[ret.size()]));
+ }
+ }
+
+ public LinkedHashMap<String, String> getSpec() {
+ return this.spec;
+ }
+
+ /**
+ * Replaces files in the partition with new data set specified by srcf. Works by moving files
+ *
+ * @param srcf Files to be moved. Leaf Directories or Globbed File Paths
+ */
+ @SuppressWarnings("nls")
+ protected void replaceFiles(Path srcf) throws HiveException {
+ FileSystem fs;
+ try {
+ fs = FileSystem.get(table.getDataLocation(), Hive.get().getConf());
+ Hive.get().replaceFiles(srcf, partPath, fs);
+ } catch (IOException e) {
+ throw new HiveException("addFiles: filesystem error in check phase", e);
+ }
+ }
+
+ /**
+ * Inserts files specified into the partition. Works by moving files
+ *
+ * @param srcf Files to be moved. Leaf Directories or Globbed File Paths
+ */
+ @SuppressWarnings("nls")
+ protected void copyFiles(Path srcf) throws HiveException {
+ FileSystem fs;
+ try {
+ fs = FileSystem.get(table.getDataLocation(), Hive.get().getConf());
+ Hive.get().copyFiles(srcf, partPath, fs);
+ } catch (IOException e) {
+ throw new HiveException("addFiles: filesystem error in check phase", e);
+ }
+ }
+
+ @SuppressWarnings("nls")
+ @Override
+ public String toString() {
+ String pn = "Invalid Partition";
+ try {
+ pn = Warehouse.makePartName(spec);
+ } catch (MetaException e) {
+ // ignore as we most probably in an exception path already otherwise this error wouldn't occur
}
+ return table.toString() + "(" + pn + ")";
+ }
}
Modified: hadoop/hive/branches/branch-0.2/ql/src/test/queries/clientpositive/input3_limit.q
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.2/ql/src/test/queries/clientpositive/input3_limit.q?rev=740927&r1=740926&r2=740927&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.2/ql/src/test/queries/clientpositive/input3_limit.q (original)
+++ hadoop/hive/branches/branch-0.2/ql/src/test/queries/clientpositive/input3_limit.q Wed Feb 4 22:41:58 2009
@@ -7,12 +7,11 @@
CREATE TABLE T2(key STRING, value STRING);
EXPLAIN
-INSERT OVERWRITE TABLE T2 SELECT a.key, a.value from T1 a LIMIT 20;
+INSERT OVERWRITE TABLE T2 SELECT * FROM (SELECT * FROM T1 DISTRIBUTE BY key SORT BY key, value) T LIMIT 20;
+INSERT OVERWRITE TABLE T2 SELECT * FROM (SELECT * FROM T1 DISTRIBUTE BY key SORT BY key, value) T LIMIT 20;
-INSERT OVERWRITE TABLE T2 SELECT a.key, a.value from T1 a LIMIT 20;
-
-SELECT * FROM (SELECT * FROM T2 DISTRIBUTE BY key SORT BY key, value) T;
+SELECT * FROM T2;
DROP TABLE T1;
DROP TABLE T2;
Modified: hadoop/hive/branches/branch-0.2/ql/src/test/queries/clientpositive/sample3.q
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.2/ql/src/test/queries/clientpositive/sample3.q?rev=740927&r1=740926&r2=740927&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.2/ql/src/test/queries/clientpositive/sample3.q (original)
+++ hadoop/hive/branches/branch-0.2/ql/src/test/queries/clientpositive/sample3.q Wed Feb 4 22:41:58 2009
@@ -4,5 +4,5 @@
FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 5 on key) s;
SELECT s.key
-FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 5 on key) s;
+FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 5 on key) s SORT BY key;
Modified: hadoop/hive/branches/branch-0.2/ql/src/test/queries/clientpositive/sample5.q
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.2/ql/src/test/queries/clientpositive/sample5.q?rev=740927&r1=740926&r2=740927&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.2/ql/src/test/queries/clientpositive/sample5.q (original)
+++ hadoop/hive/branches/branch-0.2/ql/src/test/queries/clientpositive/sample5.q Wed Feb 4 22:41:58 2009
@@ -8,4 +8,4 @@
INSERT OVERWRITE TABLE dest1 SELECT s.* -- here's another test
FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 5 on key) s;
-SELECT dest1.* FROM dest1;
+SELECT dest1.* FROM dest1 SORT BY key;
Modified: hadoop/hive/branches/branch-0.2/ql/src/test/results/clientpositive/input3_limit.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.2/ql/src/test/results/clientpositive/input3_limit.q.out?rev=740927&r1=740926&r2=740927&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.2/ql/src/test/results/clientpositive/input3_limit.q.out (original)
+++ hadoop/hive/branches/branch-0.2/ql/src/test/results/clientpositive/input3_limit.q.out Wed Feb 4 22:41:58 2009
@@ -1,30 +1,66 @@
ABSTRACT SYNTAX TREE:
- (TOK_QUERY (TOK_FROM (TOK_TABREF T1 a)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB T2)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF a key)) (TOK_SELEXPR (TOK_COLREF a value))) (TOK_LIMIT 20)))
+ (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF T1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_DISTRIBUTEBY (TOK_COLREF key)) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (TOK_COLREF key)) (TOK_TABSORTCOLNAMEASC (TOK_COLREF value))))) T)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB T2)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_LIMIT 20)))
STAGE DEPENDENCIES:
Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
STAGE PLANS:
Stage: Stage-1
Map Reduce
Alias -> Map Operator Tree:
- a
+ t:t1
Select Operator
expressions:
expr: key
type: string
expr: value
type: string
- Limit
- Reduce Output Operator
- sort order:
- tag: -1
- value expressions:
- expr: 0
- type: string
- expr: 1
- type: string
+ Reduce Output Operator
+ key expressions:
+ expr: 0
+ type: string
+ expr: 1
+ type: string
+ sort order: ++
+ Map-reduce partition columns:
+ expr: 0
+ type: string
+ tag: -1
+ value expressions:
+ expr: 0
+ type: string
+ expr: 1
+ type: string
+ Reduce Operator Tree:
+ Extract
+ Select Operator
+ expressions:
+ expr: 0
+ type: string
+ expr: 1
+ type: string
+ Limit
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
+ name: binary_table
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ /data/users/athusoo/commits/hive_trunk_ws1/build/ql/tmp/156027535/510873534.10001
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: 0
+ type: string
+ expr: 1
+ type: string
Reduce Operator Tree:
Extract
Limit
@@ -47,23 +83,23 @@
name: t2
-128 val_128
-150 val_150
-165 val_165
-193 val_193
-213 val_213
-224 val_224
-238 val_238
-255 val_255
-265 val_265
-27 val_27
-273 val_273
-278 val_278
-311 val_311
-369 val_369
-401 val_401
-409 val_409
-484 val_484
-66 val_66
-86 val_86
-98 val_98
+0 val_0
+0 val_0
+0 val_0
+0 val_1
+0 val_1
+1 val_2
+10 val_10
+10 val_11
+100 val_100
+100 val_100
+100 val_101
+100 val_101
+101 val_102
+102 val_103
+103 val_103
+103 val_103
+104 val_104
+104 val_104
+104 val_105
+104 val_105
Modified: hadoop/hive/branches/branch-0.2/ql/src/test/results/clientpositive/sample3.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.2/ql/src/test/results/clientpositive/sample3.q.out?rev=740927&r1=740926&r2=740927&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.2/ql/src/test/results/clientpositive/sample3.q.out (original)
+++ hadoop/hive/branches/branch-0.2/ql/src/test/results/clientpositive/sample3.q.out Wed Feb 4 22:41:58 2009
@@ -33,210 +33,210 @@
limit: -1
-86
-27
-484
-150
+100
+100
+100
+100
+105
+105
+105
+114
+114
+114
+114
+119
+119
+119
+119
+119
+119
+123
+123
128
-213
-146
-281
-277
-209
-394
-466
-399
-489
-439
-367
-475
-155
128
-489
-72
-277
-399
-169
-498
-286
-187
-54
-213
+128
+128
+128
+132
+132
137
-489
-353
-272
-466
-411
-119
-100
-191
-308
-95
-196
-457
-187
-470
137
-169
-77
-72
-90
-277
-272
-452
-402
-317
-95
-498
-321
-119
-489
-218
-209
-367
-344
-263
-191
-128
-2
-321
-335
-466
-105
-114
-90
+137
+146
+146
+150
+155
+164
+164
164
164
-187
-119
-439
169
-443
-277
+169
+169
+169
+178
+178
178
-317
-493
-353
-407
18
-100
-498
-146
-362
18
-281
-344
-448
+182
+187
+187
+187
+191
+191
+191
+196
+196
+196
+196
+2
+2
+204
+209
+209
+209
+209
+213
+213
+213
+218
+22
222
-90
-169
-281
-245
-425
-272
+222
+227
231
-448
-31
-443
-371
-402
-128
240
-286
-335
-367
-59
-349
-123
-402
-128
-63
-344
+245
+245
+254
+259
+259
+263
+268
+27
+272
+272
+272
+277
+277
+277
+277
+277
277
-114
-209
281
-114
-254
-196
-86
-222
-411
281
-68
-119
-376
-213
-132
-191
-349
-475
-399
-77
-114
-443
-484
-407
-367
-2
-349
-164
-326
-308
-119
-358
-105
-416
-77
-461
-40
-385
-259
+281
+281
+281
+286
+286
+295
+295
303
-245
303
-277
-132
-443
-399
+308
+308
+308
+31
317
+317
+317
+321
+321
+326
330
-209
-489
-295
-105
-204
-439
-457
-470
-196
-119
-268
-295
-407
-439
335
-164
+335
+335
+344
+344
+344
349
-371
-123
-182
+349
+349
+349
+353
+353
353
+358
+362
+367
+367
+367
+367
+371
+371
+371
371
+376
+385
385
+394
+399
+399
+399
+399
+40
40
-259
-178
-100
-22
402
-137
-196
-100
+402
+402
+402
407
-371
-308
-178
+407
+407
+407
+411
+411
+416
+425
+439
+439
+439
+439
+443
+443
+443
+443
+448
+448
+452
457
-227
+457
+457
+461
+466
+466
+466
+470
+470
+475
+475
+484
+484
+489
+489
+489
+489
+489
+493
+498
+498
+498
+54
+59
+63
+68
+72
+72
+77
+77
+77
+86
+86
+90
+90
+90
+95
+95
Modified: hadoop/hive/branches/branch-0.2/ql/src/test/results/clientpositive/sample5.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.2/ql/src/test/results/clientpositive/sample5.q.out?rev=740927&r1=740926&r2=740927&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.2/ql/src/test/results/clientpositive/sample5.q.out (original)
+++ hadoop/hive/branches/branch-0.2/ql/src/test/results/clientpositive/sample5.q.out Wed Feb 4 22:41:58 2009
@@ -28,7 +28,7 @@
type: string
File Output Operator
compressed: false
- directory: /tmp/hive-zshao/211838955/159322479.10000.insclause-0
+ directory: /data/users/athusoo/commits/hive_trunk_ws1/ql/../build/ql/tmp/379123596/110097939.10000.insclause-0
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
@@ -41,14 +41,14 @@
serialization.lib org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
- location file:/data/users/zshao/sync/apache-trunk-HIVE-104/build/ql/test/data/warehouse/dest1
+ location file:/data/users/athusoo/commits/hive_trunk_ws1/build/ql/test/data/warehouse/dest1
serde: org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe
name: dest1
Needs Tagging: false
Path -> Alias:
- file:/data/users/zshao/sync/apache-trunk-HIVE-104/build/ql/test/data/warehouse/srcbucket
+ file:/data/users/athusoo/commits/hive_trunk_ws1/build/ql/test/data/warehouse/srcbucket
Path -> Partition:
- file:/data/users/zshao/sync/apache-trunk-HIVE-104/build/ql/test/data/warehouse/srcbucket
+ file:/data/users/athusoo/commits/hive_trunk_ws1/build/ql/test/data/warehouse/srcbucket
Partition
input format: org.apache.hadoop.mapred.TextInputFormat
@@ -63,7 +63,7 @@
serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
- location file:/data/users/zshao/sync/apache-trunk-HIVE-104/build/ql/test/data/warehouse/srcbucket
+ location file:/data/users/athusoo/commits/hive_trunk_ws1/build/ql/test/data/warehouse/srcbucket
serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe
name: srcbucket
@@ -71,7 +71,7 @@
Move Operator
tables:
replace: true
- source: /tmp/hive-zshao/211838955/159322479.10000.insclause-0
+ source: /data/users/athusoo/commits/hive_trunk_ws1/ql/../build/ql/tmp/379123596/110097939.10000.insclause-0
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
@@ -84,215 +84,215 @@
serialization.lib org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
- location file:/data/users/zshao/sync/apache-trunk-HIVE-104/build/ql/test/data/warehouse/dest1
+ location file:/data/users/athusoo/commits/hive_trunk_ws1/build/ql/test/data/warehouse/dest1
serde: org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe
name: dest1
-86 val_86
+2 val_2
+2 val_3
+18 val_18
+18 val_18
+22 val_23
27 val_27
-484 val_484
-150 val_150
-128 val_128
-213 val_213
-146 val_146
-281 val_281
-277 val_277
-209 val_209
-394 val_394
-466 val_466
-399 val_399
-489 val_489
-439 val_439
-367 val_367
-475 val_475
-155 val_155
-128 val_128
-489 val_489
-72 val_72
-277 val_277
-399 val_399
-169 val_169
-498 val_498
-286 val_286
-187 val_187
+31 val_32
+40 val_41
+40 val_41
54 val_54
-213 val_213
-137 val_137
-489 val_489
-353 val_353
-272 val_272
-466 val_466
-411 val_411
-119 val_119
-100 val_100
-191 val_191
-308 val_308
-95 val_95
-196 val_196
-457 val_457
-187 val_187
-470 val_470
-137 val_137
-169 val_169
-77 val_77
+59 val_60
+63 val_64
+68 val_69
72 val_72
+72 val_72
+77 val_77
+77 val_78
+77 val_78
+86 val_87
+86 val_86
+90 val_90
+90 val_90
90 val_90
-277 val_277
-272 val_272
-452 val_452
-402 val_402
-317 val_317
95 val_95
-498 val_498
-321 val_321
-119 val_119
-489 val_489
-218 val_218
-209 val_209
-367 val_367
-344 val_344
-263 val_263
-191 val_191
-128 val_128
-2 val_2
-321 val_321
-335 val_335
-466 val_466
+95 val_95
+100 val_100
+100 val_100
+100 val_101
+100 val_101
+105 val_106
+105 val_106
105 val_105
114 val_114
-90 val_90
+114 val_115
+114 val_115
+114 val_115
+119 val_120
+119 val_120
+119 val_120
+119 val_119
+119 val_119
+119 val_119
+123 val_124
+123 val_124
+128 val_129
+128 val_129
+128 val_128
+128 val_128
+128 val_128
+132 val_133
+132 val_133
+137 val_138
+137 val_137
+137 val_137
+146 val_146
+146 val_146
+150 val_150
+155 val_155
164 val_164
164 val_164
-187 val_187
-119 val_119
-439 val_439
+164 val_165
+164 val_165
+169 val_169
+169 val_169
+169 val_169
169 val_169
-443 val_443
-277 val_277
178 val_178
-317 val_317
-493 val_493
-353 val_353
-407 val_407
-18 val_18
-100 val_100
-498 val_498
-146 val_146
-362 val_362
-18 val_18
-281 val_281
-344 val_344
-448 val_448
+178 val_179
+178 val_179
+182 val_183
+187 val_187
+187 val_187
+187 val_187
+191 val_191
+191 val_191
+191 val_192
+196 val_197
+196 val_197
+196 val_197
+196 val_196
+204 val_205
+209 val_210
+209 val_210
+209 val_209
+209 val_209
+213 val_213
+213 val_213
+213 val_214
+218 val_218
222 val_222
-90 val_90
-169 val_169
-281 val_282
-245 val_246
-425 val_426
-272 val_273
+222 val_223
+227 val_228
231 val_232
-448 val_449
-31 val_32
-443 val_444
-371 val_372
-402 val_403
-128 val_129
240 val_241
-286 val_287
-335 val_336
-367 val_368
-59 val_60
-349 val_350
-123 val_124
-402 val_403
-128 val_129
-63 val_64
-344 val_345
+245 val_246
+245 val_246
+254 val_255
+259 val_260
+259 val_260
+263 val_263
+268 val_269
+272 val_273
+272 val_272
+272 val_272
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_278
277 val_278
-114 val_115
-209 val_210
281 val_282
-114 val_115
-254 val_255
-196 val_197
-86 val_87
-222 val_223
-411 val_412
281 val_282
-68 val_69
-119 val_120
-376 val_377
-213 val_214
-132 val_133
-191 val_192
-349 val_350
-475 val_476
-399 val_400
-77 val_78
-114 val_115
-443 val_444
-484 val_485
-407 val_408
-367 val_368
-2 val_3
-349 val_350
-164 val_165
-326 val_327
-308 val_309
-119 val_120
-358 val_359
-105 val_106
-416 val_417
-77 val_78
-461 val_462
-40 val_41
-385 val_386
-259 val_260
+281 val_282
+281 val_281
+281 val_281
+286 val_286
+286 val_287
+295 val_296
+295 val_296
303 val_304
-245 val_246
303 val_304
-277 val_278
-132 val_133
-443 val_444
-399 val_400
+308 val_309
+308 val_309
+308 val_308
+317 val_317
+317 val_317
317 val_318
+321 val_321
+321 val_321
+326 val_327
330 val_331
-209 val_210
-489 val_490
-295 val_296
-105 val_106
-204 val_205
-439 val_440
-457 val_458
-470 val_471
-196 val_197
-119 val_120
-268 val_269
-295 val_296
-407 val_408
-439 val_440
335 val_336
-164 val_165
+335 val_336
+335 val_335
+344 val_344
+344 val_344
+344 val_345
+349 val_350
+349 val_350
+349 val_350
349 val_350
-371 val_372
-123 val_124
-182 val_183
353 val_354
+353 val_353
+353 val_353
+358 val_359
+362 val_362
+367 val_367
+367 val_367
+367 val_368
+367 val_368
+371 val_372
371 val_372
+371 val_372
+371 val_372
+376 val_377
385 val_386
-40 val_41
-259 val_260
-178 val_179
-100 val_101
-22 val_23
+385 val_386
+394 val_394
+399 val_399
+399 val_399
+399 val_400
+399 val_400
402 val_403
-137 val_138
-196 val_197
-100 val_101
+402 val_403
+402 val_403
+402 val_402
+407 val_407
407 val_408
-371 val_372
-308 val_309
-178 val_179
+407 val_408
+407 val_408
+411 val_412
+411 val_411
+416 val_417
+425 val_426
+439 val_440
+439 val_440
+439 val_439
+439 val_439
+443 val_443
+443 val_444
+443 val_444
+443 val_444
+448 val_449
+448 val_448
+452 val_452
+457 val_457
457 val_458
-227 val_228
+457 val_458
+461 val_462
+466 val_466
+466 val_466
+466 val_466
+470 val_470
+470 val_471
+475 val_476
+475 val_475
+484 val_484
+484 val_485
+489 val_490
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+493 val_493
+498 val_498
+498 val_498
+498 val_498