svn commit: r1461586 - in /hive/trunk: common/src/java/org/apache/hadoop/hive/conf/ ql/src/java/org/apache/hadoop/hive/ql/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/ ql/src/java/org/apache/hadoop/...

[na...@apache.org]

Author: namit
Date: Wed Mar 27 14:07:10 2013
New Revision: 1461586

URL: http://svn.apache.org/r1461586
Log:
HIVE-3958 support partial scan for analyze command - RCFile
(Gang Tim Liu via namit)


Added:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanMapper.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanWork.java
    hive/trunk/ql/src/test/queries/clientnegative/stats_partialscan_autogether.q
    hive/trunk/ql/src/test/queries/clientnegative/stats_partialscan_non_external.q
    hive/trunk/ql/src/test/queries/clientnegative/stats_partialscan_non_native.q
    hive/trunk/ql/src/test/queries/clientnegative/stats_partscan_norcfile.q
    hive/trunk/ql/src/test/queries/clientpositive/stats_partscan_1.q
    hive/trunk/ql/src/test/results/clientnegative/stats_partialscan_autogether.q.out
    hive/trunk/ql/src/test/results/clientnegative/stats_partialscan_non_external.q.out
    hive/trunk/ql/src/test/results/clientnegative/stats_partialscan_non_native.q.out
    hive/trunk/ql/src/test/results/clientnegative/stats_partscan_norcfile.q.out
    hive/trunk/ql/src/test/results/clientpositive/stats_partscan_1.q.out
Modified:
    hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileKeyBufferWrapper.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java

Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1461586&r1=1461585&r2=1461586&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Wed Mar 27 14:07:10 2013
@@ -606,6 +606,7 @@ public class HiveConf extends Configurat
     // higher compute cost.
     HIVE_STATS_NDV_ERROR("hive.stats.ndv.error", (float)20.0),
     HIVE_STATS_KEY_PREFIX_MAX_LENGTH("hive.stats.key.prefix.max.length", 200),
+    HIVE_STATS_KEY_PREFIX("hive.stats.key.prefix", ""), // internal usage only
 
     // Concurrency
     HIVE_SUPPORT_CONCURRENCY("hive.support.concurrency", false),

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java?rev=1461586&r1=1461585&r2=1461586&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java Wed Mar 27 14:07:10 2013
@@ -328,6 +328,17 @@ public enum ErrorMsg {
   ANALYZE_TABLE_NOSCAN_NON_NATIVE(10228, "ANALYZE TABLE NOSCAN cannot be used for "
       + "a non-native table"),
 
+  ANALYZE_TABLE_PARTIALSCAN_NON_NATIVE(10229, "ANALYZE TABLE PARTIALSCAN cannot be used for "
+      + "a non-native table"),
+  ANALYZE_TABLE_PARTIALSCAN_NON_RCFILE(10230, "ANALYZE TABLE PARTIALSCAN doesn't "
+      + "support non-RCfile. "),
+  ANALYZE_TABLE_PARTIALSCAN_EXTERNAL_TABLE(10231, "ANALYZE TABLE PARTIALSCAN "
+      + "doesn't support external table: "),
+  ANALYZE_TABLE_PARTIALSCAN_AGGKEY(10232, "Analyze partialscan command "
+            + "fails to construct aggregation for the partition "),
+  ANALYZE_TABLE_PARTIALSCAN_AUTOGATHER(10233, "Analyze partialscan is not allowed " +
+            "if hive.stats.autogather is set to false"),
+
   SCRIPT_INIT_ERROR(20000, "Unable to initialize custom script."),
   SCRIPT_IO_ERROR(20001, "An error occurred while reading or writing to your custom script. "
       + "It may have crashed with an error."),

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java?rev=1461586&r1=1461585&r2=1461586&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java Wed Mar 27 14:07:10 2013
@@ -392,6 +392,7 @@ public class StatsTask extends Task<Stat
           if (statsAggregator != null) {
             updateStats(collectableStats, newPartStats, statsAggregator,
                 parameters, partitionID, atomic);
+            statsAggregator.cleanUp(partitionID);
           } else {
             for (String statType : collectableStats) {
               // The collectable stats for the aggregator needs to be cleared.
@@ -441,10 +442,6 @@ public class StatsTask extends Task<Stat
           String tableFullName = table.getDbName() + "." + table.getTableName();
           db.alterPartition(tableFullName, new Partition(table, tPart));
 
-          if (statsAggregator != null) {
-            statsAggregator.cleanUp(partitionID);
-          }
-
           console.printInfo("Partition " + tableFullName + partn.getSpec() +
               " stats: [" + newPartStats.toString() + ']');
         }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java?rev=1461586&r1=1461585&r2=1461586&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java Wed Mar 27 14:07:10 2013
@@ -25,6 +25,8 @@ import java.util.List;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.io.rcfile.merge.BlockMergeTask;
 import org.apache.hadoop.hive.ql.io.rcfile.merge.MergeWork;
+import org.apache.hadoop.hive.ql.io.rcfile.stats.PartialScanTask;
+import org.apache.hadoop.hive.ql.io.rcfile.stats.PartialScanWork;
 import org.apache.hadoop.hive.ql.plan.ColumnStatsWork;
 import org.apache.hadoop.hive.ql.plan.ConditionalWork;
 import org.apache.hadoop.hive.ql.plan.CopyWork;
@@ -83,6 +85,8 @@ public final class TaskFactory {
         BlockMergeTask.class));
     taskvec.add(new taskTuple<DependencyCollectionWork>(DependencyCollectionWork.class,
         DependencyCollectionTask.class));
+    taskvec.add(new taskTuple<PartialScanWork>(PartialScanWork.class,
+        PartialScanTask.class));
 
   }
 

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileKeyBufferWrapper.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileKeyBufferWrapper.java?rev=1461586&r1=1461585&r2=1461586&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileKeyBufferWrapper.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileKeyBufferWrapper.java Wed Mar 27 14:07:10 2013
@@ -62,4 +62,8 @@ public class RCFileKeyBufferWrapper impl
     return this.keyBuffer.compareTo(o.keyBuffer);
   }
 
+  public KeyBuffer getKeyBuffer() {
+    return keyBuffer;
+  }
+
 }

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanMapper.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanMapper.java?rev=1461586&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanMapper.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanMapper.java Wed Mar 27 14:07:10 2013
@@ -0,0 +1,166 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io.rcfile.stats;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.io.RCFile.KeyBuffer;
+import org.apache.hadoop.hive.ql.io.rcfile.merge.RCFileKeyBufferWrapper;
+import org.apache.hadoop.hive.ql.io.rcfile.merge.RCFileValueBufferWrapper;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.stats.StatsPublisher;
+import org.apache.hadoop.hive.ql.stats.StatsSetupConst;
+import org.apache.hadoop.hive.shims.CombineHiveKey;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+
+/**
+ *
+ * PartialScanMapper.
+ *
+ * It reads through block header and aggregates statistics at the end.
+ *
+ * https://issues.apache.org/jira/browse/HIVE-3958
+ */
+@SuppressWarnings("deprecation")
+public class PartialScanMapper extends MapReduceBase implements
+    Mapper<Object, RCFileValueBufferWrapper, Object, Object> {
+
+  private JobConf jc;
+  private String statsAggKeyPrefix;
+  private long uncompressedFileSize = 0;
+  private long rowNo = 0;
+  private boolean exception = false;
+
+  public final static Log LOG = LogFactory.getLog("PartialScanMapper");
+
+  public PartialScanMapper() {
+  }
+
+  @Override
+  public void configure(JobConf job) {
+    jc = job;
+    statsAggKeyPrefix = HiveConf.getVar(job,
+        HiveConf.ConfVars.HIVE_STATS_KEY_PREFIX);
+  }
+
+
+  @Override
+  public void map(Object k, RCFileValueBufferWrapper value,
+      OutputCollector<Object, Object> output, Reporter reporter)
+      throws IOException {
+    try {
+      //CombineHiveInputFormat is set in PartialScanTask.
+      RCFileKeyBufferWrapper key = (RCFileKeyBufferWrapper) ((CombineHiveKey) k).getKey();
+
+      // calculate rawdatasize
+      KeyBuffer keyBuffer = key.getKeyBuffer();
+      long[] uncompressedColumnSizes = new long[keyBuffer.getColumnNumber()];
+      for (int i = 0; i < keyBuffer.getColumnNumber(); i++) {
+        uncompressedColumnSizes[i] += keyBuffer.getEachColumnUncompressedValueLen()[i];
+      }
+      if (uncompressedColumnSizes != null) {
+        for (int i = 0; i < uncompressedColumnSizes.length; i++) {
+          uncompressedFileSize += uncompressedColumnSizes[i];
+        }
+      }
+
+      // calculate no. of rows
+      rowNo += keyBuffer.getNumberRows();
+    } catch (Throwable e) {
+      this.exception = true;
+      close();
+      throw new IOException(e);
+    }
+
+  }
+
+
+  @Override
+  public void close() throws IOException {
+    try {
+      // Only publish stats if this operator's flag was set to gather stats
+      if (!exception) {
+        publishStats();
+      }
+    } catch (HiveException e) {
+      this.exception = true;
+      throw new RuntimeException(e);
+    }
+  }
+
+
+  /**
+   * Publish statistics.
+   * similar to FileSinkOperator.java publishStats()
+   *
+   * @throws HiveException
+   */
+  private void publishStats() throws HiveException {
+    // Initializing a stats publisher
+    StatsPublisher statsPublisher = Utilities.getStatsPublisher(jc);
+
+    if (statsPublisher == null) {
+      // just return, stats gathering should not block the main query
+      LOG.error("StatsPublishing error: StatsPublisher is not initialized.");
+      throw new HiveException(ErrorMsg.STATSPUBLISHER_NOT_OBTAINED.getErrorCodedMsg());
+    }
+
+    if (!statsPublisher.connect(jc)) {
+      // should fail since stats gathering is main purpose of the job
+      LOG.error("StatsPublishing error: cannot connect to database");
+      throw new HiveException(ErrorMsg.STATSPUBLISHER_CONNECTION_ERROR.getErrorCodedMsg());
+    }
+
+    // construct key used to store stats in intermediate db
+    String taskID = Utilities.getTaskIdFromFilename(Utilities.getTaskId(jc));
+    String keyPrefix = Utilities.getHashedStatsPrefix(
+        statsAggKeyPrefix, HiveConf.getIntVar(jc, ConfVars.HIVE_STATS_KEY_PREFIX_MAX_LENGTH));
+    String key = keyPrefix + taskID;
+
+    // construct statistics to be stored
+    Map<String, String> statsToPublish = new HashMap<String, String>();
+    statsToPublish.put(StatsSetupConst.RAW_DATA_SIZE, Long.toString(uncompressedFileSize));
+    statsToPublish.put(StatsSetupConst.ROW_COUNT, Long.toString(rowNo));
+
+    if (!statsPublisher.publishStat(key, statsToPublish)) {
+      // The original exception is lost.
+      // Not changing the interface to maintain backward compatibility
+      throw new HiveException(ErrorMsg.STATSPUBLISHER_PUBLISHING_ERROR.getErrorCodedMsg());
+    }
+
+    if (!statsPublisher.closeConnection()) {
+      // The original exception is lost.
+      // Not changing the interface to maintain backward compatibility
+      throw new HiveException(ErrorMsg.STATSPUBLISHER_CLOSING_ERROR.getErrorCodedMsg());
+    }
+  }
+
+}

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java?rev=1461586&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java Wed Mar 27 14:07:10 2013
@@ -0,0 +1,389 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io.rcfile.stats;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.List;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.QueryPlan;
+import org.apache.hadoop.hive.ql.exec.HadoopJobExecHelper;
+import org.apache.hadoop.hive.ql.exec.HadoopJobExecHook;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.Throttle;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
+import org.apache.hadoop.hive.ql.io.HiveOutputFormatImpl;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.api.StageType;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
+import org.apache.hadoop.hive.ql.stats.StatsFactory;
+import org.apache.hadoop.hive.ql.stats.StatsPublisher;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapred.Counters;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RunningJob;
+import org.apache.log4j.Appender;
+import org.apache.log4j.FileAppender;
+import org.apache.log4j.LogManager;
+
+/**
+ * PartialScanTask.
+ * This task handles RCFile statics calculation via partial scan.
+ * Instead of reading all bytes, it reads block header and aggregates result.
+ * https://issues.apache.org/jira/browse/HIVE-3958
+ */
+@SuppressWarnings( { "deprecation"})
+public class PartialScanTask extends Task<PartialScanWork> implements
+    Serializable, HadoopJobExecHook {
+
+
+  private static final long serialVersionUID = 1L;
+
+  protected transient JobConf job;
+  protected HadoopJobExecHelper jobExecHelper;
+
+  @Override
+  public void initialize(HiveConf conf, QueryPlan queryPlan,
+      DriverContext driverContext) {
+    super.initialize(conf, queryPlan, driverContext);
+    job = new JobConf(conf, PartialScanTask.class);
+    jobExecHelper = new HadoopJobExecHelper(job, this.console, this, this);
+  }
+
+  @Override
+  public boolean requireLock() {
+    return true;
+  }
+
+  boolean success = true;
+
+  @Override
+  /**
+   * start a new map-reduce job to do partial scan to calculate Stats,
+   * almost the same as BlockMergeTask or ExecDriver.
+   */
+  public int execute(DriverContext driverContext) {
+    HiveConf.setVar(job, HiveConf.ConfVars.HIVEINPUTFORMAT,
+        CombineHiveInputFormat.class.getName());
+    success = true;
+    ShimLoader.getHadoopShims().prepareJobOutput(job);
+    job.setOutputFormat(HiveOutputFormatImpl.class);
+    job.setMapperClass(work.getMapperClass());
+
+    Context ctx = driverContext.getCtx();
+    boolean ctxCreated = false;
+    try {
+      if (ctx == null) {
+        ctx = new Context(job);
+        ctxCreated = true;
+      }
+    }catch (IOException e) {
+      e.printStackTrace();
+      console.printError("Error launching map-reduce job", "\n"
+          + org.apache.hadoop.util.StringUtils.stringifyException(e));
+      return 5;
+    }
+
+    job.setMapOutputKeyClass(NullWritable.class);
+    job.setMapOutputValueClass(NullWritable.class);
+    if(work.getNumMapTasks() != null) {
+      job.setNumMapTasks(work.getNumMapTasks());
+    }
+
+    // zero reducers
+    job.setNumReduceTasks(0);
+
+    if (work.getMinSplitSize() != null) {
+      HiveConf.setLongVar(job, HiveConf.ConfVars.MAPREDMINSPLITSIZE, work
+          .getMinSplitSize().longValue());
+    }
+
+    if (work.getInputformat() != null) {
+      HiveConf.setVar(job, HiveConf.ConfVars.HIVEINPUTFORMAT, work
+          .getInputformat());
+    }
+
+    String inpFormat = HiveConf.getVar(job, HiveConf.ConfVars.HIVEINPUTFORMAT);
+    if ((inpFormat == null) || (!StringUtils.isNotBlank(inpFormat))) {
+      inpFormat = ShimLoader.getHadoopShims().getInputFormatClassName();
+    }
+
+    LOG.info("Using " + inpFormat);
+
+    try {
+      job.setInputFormat((Class<? extends InputFormat>) (Class
+          .forName(inpFormat)));
+    } catch (ClassNotFoundException e) {
+      throw new RuntimeException(e.getMessage());
+    }
+
+    job.setOutputKeyClass(NullWritable.class);
+    job.setOutputValueClass(NullWritable.class);
+
+    int returnVal = 0;
+    RunningJob rj = null;
+    boolean noName = StringUtils.isEmpty(HiveConf.getVar(job,
+        HiveConf.ConfVars.HADOOPJOBNAME));
+
+    String jobName = null;
+    if (noName && this.getQueryPlan() != null) {
+      int maxlen = conf.getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH);
+      jobName = Utilities.abbreviate(this.getQueryPlan().getQueryStr(),
+          maxlen - 6);
+    }
+
+    if (noName) {
+      // This is for a special case to ensure unit tests pass
+      HiveConf.setVar(job, HiveConf.ConfVars.HADOOPJOBNAME,
+          jobName != null ? jobName : "JOB" + Utilities.randGen.nextInt());
+    }
+
+    // pass aggregation key to mapper
+    HiveConf.setVar(job,
+        HiveConf.ConfVars.HIVE_STATS_KEY_PREFIX,
+        work.getAggKey());
+
+    try {
+      addInputPaths(job, work);
+
+      Utilities.setMapRedWork(job, work, ctx.getMRTmpFileURI());
+
+      // remove the pwd from conf file so that job tracker doesn't show this
+      // logs
+      String pwd = HiveConf.getVar(job, HiveConf.ConfVars.METASTOREPWD);
+      if (pwd != null) {
+        HiveConf.setVar(job, HiveConf.ConfVars.METASTOREPWD, "HIVE");
+      }
+      JobClient jc = new JobClient(job);
+
+      String addedJars = Utilities.getResourceFiles(job, SessionState.ResourceType.JAR);
+      if (!addedJars.isEmpty()) {
+        job.set("tmpjars", addedJars);
+      }
+
+      // make this client wait if job trcker is not behaving well.
+      Throttle.checkJobTracker(job, LOG);
+
+      if (work.isGatheringStats()) {
+        // initialize stats publishing table
+        StatsPublisher statsPublisher;
+        String statsImplementationClass = HiveConf.getVar(job, HiveConf.ConfVars.HIVESTATSDBCLASS);
+        if (StatsFactory.setImplementation(statsImplementationClass, job)) {
+          statsPublisher = StatsFactory.getStatsPublisher();
+          if (!statsPublisher.init(job)) { // creating stats table if not exists
+            if (HiveConf.getBoolVar(job, HiveConf.ConfVars.HIVE_STATS_RELIABLE)) {
+              throw
+                new HiveException(ErrorMsg.STATSPUBLISHER_INITIALIZATION_ERROR.getErrorCodedMsg());
+            }
+          }
+        }
+      }
+
+      // Finally SUBMIT the JOB!
+      rj = jc.submitJob(job);
+
+      returnVal = jobExecHelper.progress(rj, jc);
+      success = (returnVal == 0);
+
+    } catch (Exception e) {
+      e.printStackTrace();
+      String mesg = " with exception '" + Utilities.getNameMessage(e) + "'";
+      if (rj != null) {
+        mesg = "Ended Job = " + rj.getJobID() + mesg;
+      } else {
+        mesg = "Job Submission failed" + mesg;
+      }
+
+      // Has to use full name to make sure it does not conflict with
+      // org.apache.commons.lang.StringUtils
+      console.printError(mesg, "\n"
+          + org.apache.hadoop.util.StringUtils.stringifyException(e));
+
+      success = false;
+      returnVal = 1;
+    } finally {
+      try {
+        if (ctxCreated) {
+          ctx.clear();
+        }
+        if (rj != null) {
+          if (returnVal != 0) {
+            rj.killJob();
+          }
+          HadoopJobExecHelper.runningJobKillURIs.remove(rj.getJobID());
+          jobID = rj.getID().toString();
+        }
+      } catch (Exception e) {
+      }
+    }
+
+    return (returnVal);
+  }
+
+  private void addInputPaths(JobConf job, PartialScanWork work) {
+    for (String path : work.getInputPaths()) {
+      FileInputFormat.addInputPath(job, new Path(path));
+    }
+  }
+
+  @Override
+  public String getName() {
+    return "RCFile Statistics Partial Scan";
+  }
+
+  public static String INPUT_SEPERATOR = ":";
+
+  public static void main(String[] args) {
+    String inputPathStr = null;
+    String outputDir = null;
+    String jobConfFileName = null;
+
+    try {
+      for (int i = 0; i < args.length; i++) {
+        if (args[i].equals("-input")) {
+          inputPathStr = args[++i];
+        } else if (args[i].equals("-jobconffile")) {
+          jobConfFileName = args[++i];
+        } else if (args[i].equals("-outputDir")) {
+          outputDir = args[++i];
+        }
+      }
+    } catch (IndexOutOfBoundsException e) {
+      System.err.println("Missing argument to option");
+      printUsage();
+    }
+
+    if (inputPathStr == null || outputDir == null
+        || outputDir.trim().equals("")) {
+      printUsage();
+    }
+
+    List<String> inputPaths = new ArrayList<String>();
+    String[] paths = inputPathStr.split(INPUT_SEPERATOR);
+    if (paths == null || paths.length == 0) {
+      printUsage();
+    }
+
+    FileSystem fs = null;
+    JobConf conf = new JobConf(PartialScanTask.class);
+    for (String path : paths) {
+      try {
+        Path pathObj = new Path(path);
+        if (fs == null) {
+          fs = FileSystem.get(pathObj.toUri(), conf);
+        }
+        FileStatus fstatus = fs.getFileStatus(pathObj);
+        if (fstatus.isDir()) {
+          FileStatus[] fileStatus = fs.listStatus(pathObj);
+          for (FileStatus st : fileStatus) {
+            inputPaths.add(st.getPath().toString());
+          }
+        } else {
+          inputPaths.add(fstatus.getPath().toString());
+        }
+      } catch (IOException e) {
+        e.printStackTrace(System.err);
+      }
+    }
+
+    if (jobConfFileName != null) {
+      conf.addResource(new Path(jobConfFileName));
+    }
+    HiveConf hiveConf = new HiveConf(conf, PartialScanTask.class);
+
+    Log LOG = LogFactory.getLog(PartialScanTask.class.getName());
+    boolean isSilent = HiveConf.getBoolVar(conf,
+        HiveConf.ConfVars.HIVESESSIONSILENT);
+    LogHelper console = new LogHelper(LOG, isSilent);
+
+    // print out the location of the log file for the user so
+    // that it's easy to find reason for local mode execution failures
+    for (Appender appender : Collections
+        .list((Enumeration<Appender>) LogManager.getRootLogger()
+            .getAllAppenders())) {
+      if (appender instanceof FileAppender) {
+        console.printInfo("Execution log at: "
+            + ((FileAppender) appender).getFile());
+      }
+    }
+
+    PartialScanWork mergeWork = new PartialScanWork(inputPaths);
+    DriverContext driverCxt = new DriverContext();
+    PartialScanTask taskExec = new PartialScanTask();
+    taskExec.initialize(hiveConf, null, driverCxt);
+    taskExec.setWork(mergeWork);
+    int ret = taskExec.execute(driverCxt);
+
+    if (ret != 0) {
+      System.exit(2);
+    }
+
+  }
+
+  private static void printUsage() {
+    System.exit(1);
+  }
+
+  @Override
+  public StageType getType() {
+    return StageType.MAPRED;
+  }
+
+
+  @Override
+  public boolean checkFatalErrors(Counters ctrs, StringBuilder errMsg) {
+    return false;
+  }
+
+  @Override
+  public void logPlanProgress(SessionState ss) throws IOException {
+    // no op
+  }
+
+  @Override
+  public void updateCounters(Counters ctrs, RunningJob rj) throws IOException {
+    // no op
+  }
+
+  @Override
+  protected void localizeMRTmpFilesImpl(Context ctx) {
+    // no op
+  }
+
+
+}

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanWork.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanWork.java?rev=1461586&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanWork.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanWork.java Wed Mar 27 14:07:10 2013
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io.rcfile.stats;
+
+import java.io.Serializable;
+import java.util.LinkedHashMap;
+import java.util.List;
+
+import org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
+import org.apache.hadoop.hive.ql.io.rcfile.merge.RCFileBlockMergeInputFormat;
+import org.apache.hadoop.hive.ql.plan.Explain;
+import org.apache.hadoop.hive.ql.plan.MapredWork;
+import org.apache.hadoop.hive.ql.plan.PartitionDesc;
+import org.apache.hadoop.mapred.Mapper;
+
+/**
+ * Partial Scan Work.
+ *
+ */
+@Explain(displayName = "Partial Scan Statistics")
+public class PartialScanWork extends MapredWork implements Serializable {
+
+  private static final long serialVersionUID = 1L;
+
+  private List<String> inputPaths;
+  private String aggKey;
+
+  public PartialScanWork() {
+  }
+
+  public PartialScanWork(List<String> inputPaths) {
+    super();
+    this.inputPaths = inputPaths;
+    PartitionDesc partDesc = new PartitionDesc();
+    partDesc.setInputFileFormatClass(RCFileBlockMergeInputFormat.class);
+    if(this.getPathToPartitionInfo() == null) {
+      this.setPathToPartitionInfo(new LinkedHashMap<String, PartitionDesc>());
+    }
+    if(this.getNumReduceTasks() == null) {
+      this.setNumReduceTasks(0);
+    }
+    for(String path: this.inputPaths) {
+      this.getPathToPartitionInfo().put(path, partDesc);
+    }
+  }
+
+  public List<String> getInputPaths() {
+    return inputPaths;
+  }
+
+  public void setInputPaths(List<String> inputPaths) {
+    this.inputPaths = inputPaths;
+  }
+
+  public Class<? extends Mapper> getMapperClass() {
+    return PartialScanMapper.class;
+  }
+
+  @Override
+  public Long getMinSplitSize() {
+    return null;
+  }
+
+  @Override
+  public String getInputformat() {
+    return CombineHiveInputFormat.class.getName();
+  }
+
+  @Override
+  public boolean isGatheringStats() {
+    return true;
+  }
+
+  /**
+   * @return the aggKey
+   */
+  @Explain(displayName = "Stats Aggregation Key Prefix", normalExplain = false)
+  public String getAggKey() {
+    return aggKey;
+  }
+
+  /**
+   * @param aggKey the aggKey to set
+   */
+  public void setAggKey(String aggKey) {
+    this.aggKey = aggKey;
+  }
+
+}

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java?rev=1461586&r1=1461585&r2=1461586&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java Wed Mar 27 14:07:10 2013
@@ -19,16 +19,23 @@
 package org.apache.hadoop.hive.ql.optimizer;
 
 import java.io.Serializable;
+import java.util.ArrayList;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.Stack;
 
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.ErrorMsg;
 import org.apache.hadoop.hive.ql.exec.Operator;
 import org.apache.hadoop.hive.ql.exec.TableScanOperator;
 import org.apache.hadoop.hive.ql.exec.Task;
 import org.apache.hadoop.hive.ql.exec.TaskFactory;
+import org.apache.hadoop.hive.ql.io.rcfile.stats.PartialScanWork;
 import org.apache.hadoop.hive.ql.lib.Node;
 import org.apache.hadoop.hive.ql.lib.NodeProcessor;
 import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
@@ -96,7 +103,7 @@ public class GenMRTableScan1 implements 
             ctx.getRootTasks().add(currTask);
           }
 
-          //        ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan;
+          // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan;
           // The plan consists of a StatsTask only.
           if (parseInfo.isNoScanAnalyzeCommand()) {
             statsTask.setParentTasks(null);
@@ -105,6 +112,11 @@ public class GenMRTableScan1 implements 
             ctx.getRootTasks().add(statsTask);
           }
 
+          // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS partialscan;
+          if (parseInfo.isPartialScanAnalyzeCommand()) {
+            handlePartialScanCommand(op, ctx, parseCtx, currTask, parseInfo, statsWork, statsTask);
+          }
+
           currWork.setGatheringStats(true);
           // NOTE: here we should use the new partition predicate pushdown API to get a list of pruned list,
           // and pass it to setTaskPlan as the last parameter
@@ -139,4 +151,64 @@ public class GenMRTableScan1 implements 
     return null;
   }
 
+  /**
+   * handle partial scan command.
+   *
+   * It is composed of PartialScanTask followed by StatsTask .
+   * @param op
+   * @param ctx
+   * @param parseCtx
+   * @param currTask
+   * @param parseInfo
+   * @param statsWork
+   * @param statsTask
+   * @throws SemanticException
+   */
+  private void handlePartialScanCommand(TableScanOperator op, GenMRProcContext ctx,
+      ParseContext parseCtx,
+      Task<? extends Serializable> currTask, QBParseInfo parseInfo, StatsWork statsWork,
+      Task<StatsWork> statsTask) throws SemanticException {
+    String aggregationKey = op.getConf().getStatsAggPrefix();
+    List<String> inputPaths = new ArrayList<String>();
+    switch (parseInfo.getTableSpec().specType) {
+    case TABLE_ONLY:
+      inputPaths.add(parseInfo.getTableSpec().tableHandle.getPath().toString());
+      break;
+    case STATIC_PARTITION:
+      Partition part = parseInfo.getTableSpec().partHandle;
+      try {
+        aggregationKey += Warehouse.makePartPath(part.getSpec());
+      } catch (MetaException e) {
+        throw new SemanticException(ErrorMsg.ANALYZE_TABLE_PARTIALSCAN_AGGKEY.getMsg(
+            part.getPartitionPath().toString() + e.getMessage()));
+      }
+      inputPaths.add(part.getPartitionPath().toString());
+      break;
+    default:
+      assert false;
+    }
+
+    // scan work
+    PartialScanWork scanWork = new PartialScanWork(inputPaths);
+    scanWork.setMapperCannotSpanPartns(true);
+    scanWork.setAggKey(aggregationKey);
+
+    // stats work
+    statsWork.setPartialScanAnalyzeCommand(true);
+
+    // partial scan task
+    DriverContext driverCxt = new DriverContext();
+    Task<PartialScanWork> psTask = TaskFactory.get(scanWork, parseCtx.getConf());
+    psTask.initialize(parseCtx.getConf(), null, driverCxt);
+    psTask.setWork(scanWork);
+
+    // task dependency
+    ctx.getRootTasks().remove(currTask);
+    ctx.getRootTasks().add(psTask);
+    psTask.addDependentTask(statsTask);
+    List<Task<? extends Serializable>> parentTasks = new ArrayList<Task<? extends Serializable>>();
+    parentTasks.add(psTask);
+    statsTask.setParentTasks(parentTasks);
+  }
+
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java?rev=1461586&r1=1461585&r2=1461586&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java Wed Mar 27 14:07:10 2013
@@ -506,6 +506,8 @@ public class ColumnStatsSemanticAnalyzer
     super(conf);
     // check if it is no scan. grammar prevents coexit noscan/columns
     super.processNoScanCommand(tree);
+    // check if it is partial scan. grammar prevents coexit partialscan/columns
+    super.processPartialScanCommand(tree);
     /* Rewrite only analyze table <> column <> compute statistics; Don't rewrite analyze table
      * command - table stats are collected by the table scan operator and is not rewritten to
      * an aggregation.

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g?rev=1461586&r1=1461585&r2=1461586&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g Wed Mar 27 14:07:10 2013
@@ -246,6 +246,7 @@ KW_GROUPING: 'GROUPING';
 KW_SETS: 'SETS';
 KW_TRUNCATE: 'TRUNCATE';
 KW_NOSCAN: 'NOSCAN';
+KW_PARTIALSCAN: 'PARTIALSCAN';
 KW_USER: 'USER';
 KW_ROLE: 'ROLE';
 KW_INNER: 'INNER';

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g?rev=1461586&r1=1461585&r2=1461586&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g Wed Mar 27 14:07:10 2013
@@ -1189,7 +1189,7 @@ descStatement
 analyzeStatement
 @init { msgs.push("analyze statement"); }
 @after { msgs.pop(); }
-    : KW_ANALYZE KW_TABLE (parttype=tableOrPartition) KW_COMPUTE KW_STATISTICS ((noscan=KW_NOSCAN) | (KW_FOR KW_COLUMNS statsColumnName=columnNameList))? -> ^(TOK_ANALYZE $parttype $noscan? $statsColumnName?)
+    : KW_ANALYZE KW_TABLE (parttype=tableOrPartition) KW_COMPUTE KW_STATISTICS ((noscan=KW_NOSCAN) | (partialscan=KW_PARTIALSCAN) | (KW_FOR KW_COLUMNS statsColumnName=columnNameList))? -> ^(TOK_ANALYZE $parttype $noscan? $partialscan? $statsColumnName?)
     ;
 
 showStatement

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java?rev=1461586&r1=1461585&r2=1461586&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java Wed Mar 27 14:07:10 2013
@@ -57,6 +57,8 @@ public class QBParseInfo {
   private boolean isAnalyzeCommand; // used for the analyze command (statistics)
   private boolean isInsertToTable;  // used for insert overwrite command (statistics)
   private boolean isNoScanAnalyzeCommand; // used for the analyze command (statistics) (noscan)
+  private boolean isPartialScanAnalyzeCommand; // used for the analyze command (statistics)
+                                               // (partialscan)
 
   private final HashMap<String, tableSpec> tableSpecs; // used for statistics
 
@@ -577,4 +579,18 @@ public class QBParseInfo {
   public void setNoScanAnalyzeCommand(boolean isNoScanAnalyzeCommand) {
     this.isNoScanAnalyzeCommand = isNoScanAnalyzeCommand;
   }
+
+  /**
+   * @return the isPartialScanAnalyzeCommand
+   */
+  public boolean isPartialScanAnalyzeCommand() {
+    return isPartialScanAnalyzeCommand;
+  }
+
+  /**
+   * @param isPartialScanAnalyzeCommand the isPartialScanAnalyzeCommand to set
+   */
+  public void setPartialScanAnalyzeCommand(boolean isPartialScanAnalyzeCommand) {
+    this.isPartialScanAnalyzeCommand = isPartialScanAnalyzeCommand;
+  }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1461586&r1=1461585&r2=1461586&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Wed Mar 27 14:07:10 2013
@@ -84,6 +84,7 @@ import org.apache.hadoop.hive.ql.hooks.R
 import org.apache.hadoop.hive.ql.hooks.WriteEntity;
 import org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
 import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
+import org.apache.hadoop.hive.ql.io.RCFileInputFormat;
 import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
 import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
 import org.apache.hadoop.hive.ql.lib.Dispatcher;
@@ -232,9 +233,12 @@ public class SemanticAnalyzer extends Ba
   // Max characters when auto generating the column name with func name
   private static final int AUTOGEN_COLALIAS_PRFX_MAXLENGTH = 20;
 
-  // flag to skip scan during analyze ... compute statistics
+  // flag for no scan during analyze ... compute statistics
   protected boolean noscan = false;
 
+  //flag for partial scan during analyze ... compute statistics
+  protected boolean partialscan = false;
+
   private static class Phase1Ctx {
     String dest;
     int nextNum;
@@ -886,6 +890,7 @@ public class SemanticAnalyzer extends Ba
         qb.addAlias(table_name);
         qb.getParseInfo().setIsAnalyzeCommand(true);
         qb.getParseInfo().setNoScanAnalyzeCommand(this.noscan);
+        qb.getParseInfo().setPartialScanAnalyzeCommand(this.partialscan);
         // Allow analyze the whole table and dynamic partitions
         HiveConf.setVar(conf, HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict");
         HiveConf.setVar(conf, HiveConf.ConfVars.HIVEMAPREDMODE, "nonstrict");
@@ -1067,6 +1072,26 @@ public class SemanticAnalyzer extends Ba
                   "Cannot get partitions for " + ts.partSpec), e);
             }
           }
+          // validate partial scan command
+          QBParseInfo qbpi = qb.getParseInfo();
+          if (qbpi.isPartialScanAnalyzeCommand()) {
+            Class<? extends InputFormat> inputFormatClass = null;
+            switch (ts.specType) {
+            case TABLE_ONLY:
+              inputFormatClass = ts.tableHandle.getInputFormatClass();
+              break;
+            case STATIC_PARTITION:
+              inputFormatClass = ts.partHandle.getInputFormatClass();
+              break;
+            default:
+              assert false;
+            }
+            // throw a HiveException for non-rcfile.
+            if (!inputFormatClass.equals(RCFileInputFormat.class)) {
+              throw new SemanticException(ErrorMsg.ANALYZE_TABLE_PARTIALSCAN_NON_RCFILE.getMsg());
+            }
+          }
+
           qb.getParseInfo().addTableSpec(alias, ts);
         }
       }
@@ -9495,6 +9520,24 @@ public class SemanticAnalyzer extends Ba
   }
 
   /**
+   * process analyze ... partial command
+   *
+   * separate it from noscan command process so that it provides us flexibility
+   *
+   * @param tree
+   * @throws SemanticException
+   */
+  protected void processPartialScanCommand (ASTNode tree) throws SemanticException {
+    // check if it is partial scan command
+    this.checkPartialScan(tree);
+
+    //validate partial scan
+    if (this.partialscan) {
+      validateAnalyzePartialscan(tree);
+    }
+  }
+
+  /**
    * process analyze ... noscan command
    * @param tree
    * @throws SemanticException
@@ -9535,6 +9578,43 @@ public class SemanticAnalyzer extends Ba
   }
 
   /**
+   * Validate partialscan command
+   *
+   * @param tree
+   * @throws SemanticException
+   */
+  private void validateAnalyzePartialscan(ASTNode tree) throws SemanticException {
+    // since it is partialscan, it is true table name in command
+    String tableName = getUnescapedName((ASTNode) tree.getChild(0).getChild(0));
+    Table tbl;
+    try {
+      tbl = db.getTable(tableName);
+    } catch (HiveException e) {
+      throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(tableName));
+    }
+    /* partialscan uses hdfs apis to retrieve such information from Namenode.      */
+    /* But that will be specific to hdfs. Through storagehandler mechanism,   */
+    /* storage of table could be on any storage system: hbase, cassandra etc. */
+    /* A nice error message should be given to user. */
+    if (tbl.isNonNative()) {
+      throw new SemanticException(ErrorMsg.ANALYZE_TABLE_PARTIALSCAN_NON_NATIVE.getMsg(tbl
+          .getTableName()));
+    }
+
+    /**
+     * Partial scan doesn't support external table.
+     */
+    if(tbl.getTableType().equals(TableType.EXTERNAL_TABLE)) {
+      throw new SemanticException(ErrorMsg.ANALYZE_TABLE_PARTIALSCAN_EXTERNAL_TABLE.getMsg(tbl
+          .getTableName()));
+    }
+
+    if (!HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {
+      throw new SemanticException(ErrorMsg.ANALYZE_TABLE_PARTIALSCAN_AUTOGATHER.getMsg());
+    }
+  }
+
+  /**
    * It will check if this is analyze ... compute statistics noscan
    * @param tree
    */
@@ -9554,6 +9634,26 @@ public class SemanticAnalyzer extends Ba
     }
   }
 
+  /**
+   * It will check if this is analyze ... compute statistics partialscan
+   * @param tree
+   */
+  private void checkPartialScan(ASTNode tree) {
+    if (tree.getChildCount() > 1) {
+      ASTNode child0 = (ASTNode) tree.getChild(0);
+      ASTNode child1;
+      if (child0.getToken().getType() == HiveParser.TOK_TAB) {
+        child0 = (ASTNode) child0.getChild(0);
+        if (child0.getToken().getType() == HiveParser.TOK_TABNAME) {
+          child1 = (ASTNode) tree.getChild(1);
+          if (child1.getToken().getType() == HiveParser.KW_PARTIALSCAN) {
+            this.partialscan = true;
+          }
+        }
+      }
+    }
+  }
+
 
   public QB getQB() {
     return qb;

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java?rev=1461586&r1=1461585&r2=1461586&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java Wed Mar 27 14:07:10 2013
@@ -47,6 +47,8 @@ public class StatsWork implements Serial
 
   private boolean isNoScanAnalyzeCommand = false;
 
+  private boolean isPartialScanAnalyzeCommand = false;
+
   public StatsWork() {
   }
 
@@ -124,4 +126,18 @@ public class StatsWork implements Serial
   public void setNoScanAnalyzeCommand(boolean isNoScanAnalyzeCommand) {
     this.isNoScanAnalyzeCommand = isNoScanAnalyzeCommand;
   }
+
+  /**
+   * @return the isPartialScanAnalyzeCommand
+   */
+  public boolean isPartialScanAnalyzeCommand() {
+    return isPartialScanAnalyzeCommand;
+  }
+
+  /**
+   * @param isPartialScanAnalyzeCommand the isPartialScanAnalyzeCommand to set
+   */
+  public void setPartialScanAnalyzeCommand(boolean isPartialScanAnalyzeCommand) {
+    this.isPartialScanAnalyzeCommand = isPartialScanAnalyzeCommand;
+  }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java?rev=1461586&r1=1461585&r2=1461586&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java Wed Mar 27 14:07:10 2013
@@ -224,6 +224,7 @@ public class JDBCStatsAggregator impleme
         } catch (SQLRecoverableException e) {
           // need to start from scratch (connection)
           if (failures >= maxRetries) {
+            LOG.error("Error during clean-up after " + maxRetries + " retries. " + e);
             return false;
           }
           // close the current connection

Added: hive/trunk/ql/src/test/queries/clientnegative/stats_partialscan_autogether.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/stats_partialscan_autogether.q?rev=1461586&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/stats_partialscan_autogether.q (added)
+++ hive/trunk/ql/src/test/queries/clientnegative/stats_partialscan_autogether.q Wed Mar 27 14:07:10 2013
@@ -0,0 +1,31 @@
+set datanucleus.cache.collections=false;
+set hive.stats.autogather=false;
+set hive.exec.dynamic.partition=true;
+set hive.exec.dynamic.partition.mode=nonstrict;
+set mapred.min.split.size=256;
+set mapred.min.split.size.per.node=256;
+set mapred.min.split.size.per.rack=256;
+set mapred.max.split.size=256;
+
+-- test analyze table ... compute statistics partialscan
+
+-- 1. prepare data
+CREATE table analyze_srcpart_partial_scan (key STRING, value STRING)
+partitioned by (ds string, hr string)
+stored as rcfile;
+insert overwrite table analyze_srcpart_partial_scan partition (ds, hr) select * from srcpart where ds is not null;
+describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11);
+
+
+-- 2. partialscan
+explain
+analyze table analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) compute statistics partialscan;
+analyze table analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) compute statistics partialscan;
+
+-- 3. confirm result
+describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11);
+describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-09',hr=11);
+drop table analyze_srcpart_partial_scan;
+
+
+

Added: hive/trunk/ql/src/test/queries/clientnegative/stats_partialscan_non_external.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/stats_partialscan_non_external.q?rev=1461586&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/stats_partialscan_non_external.q (added)
+++ hive/trunk/ql/src/test/queries/clientnegative/stats_partialscan_non_external.q Wed Mar 27 14:07:10 2013
@@ -0,0 +1,5 @@
+
+CREATE EXTERNAL TABLE external_table (key int, value string);
+
+-- we do not support analyze table ... partialscan on EXTERNAL tables yet
+analyze table external_table compute statistics partialscan;

Added: hive/trunk/ql/src/test/queries/clientnegative/stats_partialscan_non_native.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/stats_partialscan_non_native.q?rev=1461586&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/stats_partialscan_non_native.q (added)
+++ hive/trunk/ql/src/test/queries/clientnegative/stats_partialscan_non_native.q Wed Mar 27 14:07:10 2013
@@ -0,0 +1,6 @@
+
+CREATE TABLE non_native1(key int, value string) 
+STORED BY 'org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler';
+
+-- we do not support analyze table ... partialscan on non-native tables yet
+analyze table non_native1 compute statistics partialscan;

Added: hive/trunk/ql/src/test/queries/clientnegative/stats_partscan_norcfile.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/stats_partscan_norcfile.q?rev=1461586&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/stats_partscan_norcfile.q (added)
+++ hive/trunk/ql/src/test/queries/clientnegative/stats_partscan_norcfile.q Wed Mar 27 14:07:10 2013
@@ -0,0 +1,12 @@
+set datanucleus.cache.collections=false;
+set hive.stats.autogather=true;
+set hive.exec.dynamic.partition=true;
+set hive.exec.dynamic.partition.mode=nonstrict;
+
+-- test analyze table ... compute statistics partialscan
+
+create table analyze_srcpart_partial_scan like srcpart;
+insert overwrite table analyze_srcpart_partial_scan partition (ds, hr) select * from srcpart where ds is not null;
+analyze table analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) compute statistics partialscan;
+
+

Added: hive/trunk/ql/src/test/queries/clientpositive/stats_partscan_1.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/stats_partscan_1.q?rev=1461586&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/stats_partscan_1.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/stats_partscan_1.q Wed Mar 27 14:07:10 2013
@@ -0,0 +1,32 @@
+set datanucleus.cache.collections=false;
+set hive.stats.autogather=false;
+set hive.exec.dynamic.partition=true;
+set hive.exec.dynamic.partition.mode=nonstrict;
+set mapred.min.split.size=256;
+set mapred.min.split.size.per.node=256;
+set mapred.min.split.size.per.rack=256;
+set mapred.max.split.size=256;
+
+-- test analyze table ... compute statistics partialscan
+
+-- 1. prepare data
+CREATE table analyze_srcpart_partial_scan (key STRING, value STRING)
+partitioned by (ds string, hr string)
+stored as rcfile;
+insert overwrite table analyze_srcpart_partial_scan partition (ds, hr) select * from srcpart where ds is not null;
+describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11);
+
+set hive.stats.autogather=true;
+
+-- 2. partialscan
+explain
+analyze table analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) compute statistics partialscan;
+analyze table analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) compute statistics partialscan;
+
+-- 3. confirm result
+describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11);
+describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-09',hr=11);
+drop table analyze_srcpart_partial_scan;
+
+
+

Added: hive/trunk/ql/src/test/results/clientnegative/stats_partialscan_autogether.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/stats_partialscan_autogether.q.out?rev=1461586&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/stats_partialscan_autogether.q.out (added)
+++ hive/trunk/ql/src/test/results/clientnegative/stats_partialscan_autogether.q.out Wed Mar 27 14:07:10 2013
@@ -0,0 +1,86 @@
+PREHOOK: query: -- test analyze table ... compute statistics partialscan
+
+-- 1. prepare data
+CREATE table analyze_srcpart_partial_scan (key STRING, value STRING)
+partitioned by (ds string, hr string)
+stored as rcfile
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- test analyze table ... compute statistics partialscan
+
+-- 1. prepare data
+CREATE table analyze_srcpart_partial_scan (key STRING, value STRING)
+partitioned by (ds string, hr string)
+stored as rcfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@analyze_srcpart_partial_scan
+PREHOOK: query: insert overwrite table analyze_srcpart_partial_scan partition (ds, hr) select * from srcpart where ds is not null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+PREHOOK: Output: default@analyze_srcpart_partial_scan
+POSTHOOK: query: insert overwrite table analyze_srcpart_partial_scan partition (ds, hr) select * from srcpart where ds is not null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-09/hr=12
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11)
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11)
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+# col_name            	data_type           	comment             
+	 	 
+key                 	string              	None                
+value               	string              	None                
+	 	 
+# Partition Information	 	 
+# col_name            	data_type           	comment             
+	 	 
+ds                  	string              	None                
+hr                  	string              	None                
+	 	 
+# Detailed Partition Information	 	 
+Partition Value:    	[2008-04-08, 11]    	 
+Database:           	default             	 
+Table:              	analyze_srcpart_partial_scan	 
+#### A masked pattern was here ####
+Protect Mode:       	None                	 
+#### A masked pattern was here ####
+Partition Parameters:	 	 
+#### A masked pattern was here ####
+	 	 
+# Storage Information	 	 
+SerDe Library:      	org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe	 
+InputFormat:        	org.apache.hadoop.hive.ql.io.RCFileInputFormat	 
+OutputFormat:       	org.apache.hadoop.hive.ql.io.RCFileOutputFormat	 
+Compressed:         	No                  	 
+Num Buckets:        	-1                  	 
+Bucket Columns:     	[]                  	 
+Sort Columns:       	[]                  	 
+Storage Desc Params:	 	 
+	serialization.format	1                   
+FAILED: SemanticException [Error 10233]: Analyze partialscan is not allowed if hive.stats.autogather is set to false

Added: hive/trunk/ql/src/test/results/clientnegative/stats_partialscan_non_external.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/stats_partialscan_non_external.q.out?rev=1461586&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/stats_partialscan_non_external.q.out (added)
+++ hive/trunk/ql/src/test/results/clientnegative/stats_partialscan_non_external.q.out Wed Mar 27 14:07:10 2013
@@ -0,0 +1,6 @@
+PREHOOK: query: CREATE EXTERNAL TABLE external_table (key int, value string)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE EXTERNAL TABLE external_table (key int, value string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@external_table
+FAILED: SemanticException [Error 10231]: ANALYZE TABLE PARTIALSCAN doesn't support external table:  external_table

Added: hive/trunk/ql/src/test/results/clientnegative/stats_partialscan_non_native.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/stats_partialscan_non_native.q.out?rev=1461586&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/stats_partialscan_non_native.q.out (added)
+++ hive/trunk/ql/src/test/results/clientnegative/stats_partialscan_non_native.q.out Wed Mar 27 14:07:10 2013
@@ -0,0 +1,8 @@
+PREHOOK: query: CREATE TABLE non_native1(key int, value string) 
+STORED BY 'org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler'
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE non_native1(key int, value string) 
+STORED BY 'org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@non_native1
+FAILED: SemanticException [Error 10229]: ANALYZE TABLE PARTIALSCAN cannot be used for a non-native table non_native1

Added: hive/trunk/ql/src/test/results/clientnegative/stats_partscan_norcfile.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/stats_partscan_norcfile.q.out?rev=1461586&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/stats_partscan_norcfile.q.out (added)
+++ hive/trunk/ql/src/test/results/clientnegative/stats_partscan_norcfile.q.out Wed Mar 27 14:07:10 2013
@@ -0,0 +1,37 @@
+PREHOOK: query: -- test analyze table ... compute statistics partialscan
+
+create table analyze_srcpart_partial_scan like srcpart
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- test analyze table ... compute statistics partialscan
+
+create table analyze_srcpart_partial_scan like srcpart
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@analyze_srcpart_partial_scan
+PREHOOK: query: insert overwrite table analyze_srcpart_partial_scan partition (ds, hr) select * from srcpart where ds is not null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+PREHOOK: Output: default@analyze_srcpart_partial_scan
+POSTHOOK: query: insert overwrite table analyze_srcpart_partial_scan partition (ds, hr) select * from srcpart where ds is not null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-09/hr=12
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+FAILED: SemanticException [Error 10230]: ANALYZE TABLE PARTIALSCAN doesn't support non-RCfile. 

Added: hive/trunk/ql/src/test/results/clientpositive/stats_partscan_1.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/stats_partscan_1.q.out?rev=1461586&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/stats_partscan_1.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/stats_partscan_1.q.out Wed Mar 27 14:07:10 2013
@@ -0,0 +1,244 @@
+PREHOOK: query: -- test analyze table ... compute statistics partialscan
+
+-- 1. prepare data
+CREATE table analyze_srcpart_partial_scan (key STRING, value STRING)
+partitioned by (ds string, hr string)
+stored as rcfile
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- test analyze table ... compute statistics partialscan
+
+-- 1. prepare data
+CREATE table analyze_srcpart_partial_scan (key STRING, value STRING)
+partitioned by (ds string, hr string)
+stored as rcfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@analyze_srcpart_partial_scan
+PREHOOK: query: insert overwrite table analyze_srcpart_partial_scan partition (ds, hr) select * from srcpart where ds is not null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+PREHOOK: Output: default@analyze_srcpart_partial_scan
+POSTHOOK: query: insert overwrite table analyze_srcpart_partial_scan partition (ds, hr) select * from srcpart where ds is not null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-09/hr=12
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11)
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11)
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+# col_name            	data_type           	comment             
+	 	 
+key                 	string              	None                
+value               	string              	None                
+	 	 
+# Partition Information	 	 
+# col_name            	data_type           	comment             
+	 	 
+ds                  	string              	None                
+hr                  	string              	None                
+	 	 
+# Detailed Partition Information	 	 
+Partition Value:    	[2008-04-08, 11]    	 
+Database:           	default             	 
+Table:              	analyze_srcpart_partial_scan	 
+#### A masked pattern was here ####
+Protect Mode:       	None                	 
+#### A masked pattern was here ####
+Partition Parameters:	 	 
+#### A masked pattern was here ####
+	 	 
+# Storage Information	 	 
+SerDe Library:      	org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe	 
+InputFormat:        	org.apache.hadoop.hive.ql.io.RCFileInputFormat	 
+OutputFormat:       	org.apache.hadoop.hive.ql.io.RCFileOutputFormat	 
+Compressed:         	No                  	 
+Num Buckets:        	-1                  	 
+Bucket Columns:     	[]                  	 
+Sort Columns:       	[]                  	 
+Storage Desc Params:	 	 
+	serialization.format	1                   
+PREHOOK: query: -- 2. partialscan
+explain
+analyze table analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) compute statistics partialscan
+PREHOOK: type: QUERY
+POSTHOOK: query: -- 2. partialscan
+explain
+analyze table analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) compute statistics partialscan
+POSTHOOK: type: QUERY
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+ABSTRACT SYNTAX TREE:
+  (TOK_ANALYZE (TOK_TAB (TOK_TABNAME analyze_srcpart_partial_scan) (TOK_PARTSPEC (TOK_PARTVAL ds '2008-04-08') (TOK_PARTVAL hr 11))) partialscan)
+
+STAGE DEPENDENCIES:
+  Stage-2 is a root stage
+  Stage-1 depends on stages: Stage-2
+
+STAGE PLANS:
+  Stage: Stage-2
+    Partial Scan Statistics
+
+  Stage: Stage-1
+    Stats-Aggr Operator
+
+
+PREHOOK: query: analyze table analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) compute statistics partialscan
+PREHOOK: type: QUERY
+PREHOOK: Input: default@analyze_srcpart_partial_scan
+PREHOOK: Input: default@analyze_srcpart_partial_scan@ds=2008-04-08/hr=11
+PREHOOK: Output: default@analyze_srcpart_partial_scan
+PREHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-08/hr=11
+POSTHOOK: query: analyze table analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) compute statistics partialscan
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@analyze_srcpart_partial_scan
+POSTHOOK: Input: default@analyze_srcpart_partial_scan@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@analyze_srcpart_partial_scan
+POSTHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-08/hr=11
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: -- 3. confirm result
+describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11)
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: -- 3. confirm result
+describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11)
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+# col_name            	data_type           	comment             
+	 	 
+key                 	string              	None                
+value               	string              	None                
+	 	 
+# Partition Information	 	 
+# col_name            	data_type           	comment             
+	 	 
+ds                  	string              	None                
+hr                  	string              	None                
+	 	 
+# Detailed Partition Information	 	 
+Partition Value:    	[2008-04-08, 11]    	 
+Database:           	default             	 
+Table:              	analyze_srcpart_partial_scan	 
+#### A masked pattern was here ####
+Protect Mode:       	None                	 
+#### A masked pattern was here ####
+Partition Parameters:	 	 
+	numFiles            	1                   
+	numRows             	500                 
+	rawDataSize         	4812                
+	totalSize           	5293                
+#### A masked pattern was here ####
+	 	 
+# Storage Information	 	 
+SerDe Library:      	org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe	 
+InputFormat:        	org.apache.hadoop.hive.ql.io.RCFileInputFormat	 
+OutputFormat:       	org.apache.hadoop.hive.ql.io.RCFileOutputFormat	 
+Compressed:         	No                  	 
+Num Buckets:        	-1                  	 
+Bucket Columns:     	[]                  	 
+Sort Columns:       	[]                  	 
+Storage Desc Params:	 	 
+	serialization.format	1                   
+PREHOOK: query: describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-09',hr=11)
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-09',hr=11)
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+# col_name            	data_type           	comment             
+	 	 
+key                 	string              	None                
+value               	string              	None                
+	 	 
+# Partition Information	 	 
+# col_name            	data_type           	comment             
+	 	 
+ds                  	string              	None                
+hr                  	string              	None                
+	 	 
+# Detailed Partition Information	 	 
+Partition Value:    	[2008-04-09, 11]    	 
+Database:           	default             	 
+Table:              	analyze_srcpart_partial_scan	 
+#### A masked pattern was here ####
+Protect Mode:       	None                	 
+#### A masked pattern was here ####
+Partition Parameters:	 	 
+#### A masked pattern was here ####
+	 	 
+# Storage Information	 	 
+SerDe Library:      	org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe	 
+InputFormat:        	org.apache.hadoop.hive.ql.io.RCFileInputFormat	 
+OutputFormat:       	org.apache.hadoop.hive.ql.io.RCFileOutputFormat	 
+Compressed:         	No                  	 
+Num Buckets:        	-1                  	 
+Bucket Columns:     	[]                  	 
+Sort Columns:       	[]                  	 
+Storage Desc Params:	 	 
+	serialization.format	1                   
+PREHOOK: query: drop table analyze_srcpart_partial_scan
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@analyze_srcpart_partial_scan
+PREHOOK: Output: default@analyze_srcpart_partial_scan
+POSTHOOK: query: drop table analyze_srcpart_partial_scan
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@analyze_srcpart_partial_scan
+POSTHOOK: Output: default@analyze_srcpart_partial_scan
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]