You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pa...@apache.org on 2011/03/15 21:29:05 UTC
svn commit: r1081930 [1/5] - in /hive/trunk:
common/src/java/org/apache/hadoop/hive/conf/ conf/
ql/src/java/org/apache/hadoop/hive/ql/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/metadata/ ql/src/java/org/apache/ha...
Author: pauly
Date: Tue Mar 15 20:29:02 2011
New Revision: 1081930
URL: http://svn.apache.org/viewvc?rev=1081930&view=rev
Log:
HIVE-1918 Add export/import facilities to the hive system (Krishna Kumar via pauly)
Added:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/parse/TestEximUtil.java
hive/trunk/ql/src/test/queries/clientnegative/exim_00_unsupported_schema.q
hive/trunk/ql/src/test/queries/clientnegative/exim_01_nonpart_over_loaded.q
hive/trunk/ql/src/test/queries/clientnegative/exim_02_all_part_over_overlap.q
hive/trunk/ql/src/test/queries/clientnegative/exim_03_nonpart_noncompat_colschema.q
hive/trunk/ql/src/test/queries/clientnegative/exim_04_nonpart_noncompat_colnumber.q
hive/trunk/ql/src/test/queries/clientnegative/exim_05_nonpart_noncompat_coltype.q
hive/trunk/ql/src/test/queries/clientnegative/exim_06_nonpart_noncompat_storage.q
hive/trunk/ql/src/test/queries/clientnegative/exim_07_nonpart_noncompat_ifof.q
hive/trunk/ql/src/test/queries/clientnegative/exim_08_nonpart_noncompat_serde.q
hive/trunk/ql/src/test/queries/clientnegative/exim_09_nonpart_noncompat_serdeparam.q
hive/trunk/ql/src/test/queries/clientnegative/exim_10_nonpart_noncompat_bucketing.q
hive/trunk/ql/src/test/queries/clientnegative/exim_11_nonpart_noncompat_sorting.q
hive/trunk/ql/src/test/queries/clientnegative/exim_12_nonnative_export.q
hive/trunk/ql/src/test/queries/clientnegative/exim_13_nonnative_import.q
hive/trunk/ql/src/test/queries/clientnegative/exim_14_nonpart_part.q
hive/trunk/ql/src/test/queries/clientnegative/exim_15_part_nonpart.q
hive/trunk/ql/src/test/queries/clientnegative/exim_16_part_noncompat_schema.q
hive/trunk/ql/src/test/queries/clientnegative/exim_17_part_spec_underspec.q
hive/trunk/ql/src/test/queries/clientnegative/exim_18_part_spec_missing.q
hive/trunk/ql/src/test/queries/clientnegative/exim_19_external_over_existing.q
hive/trunk/ql/src/test/queries/clientnegative/exim_20_managed_location_over_existing.q
hive/trunk/ql/src/test/queries/clientnegative/exim_21_part_managed_external.q
hive/trunk/ql/src/test/queries/clientnegative/exim_22_export_authfail.q
hive/trunk/ql/src/test/queries/clientnegative/exim_23_import_exist_authfail.q
hive/trunk/ql/src/test/queries/clientnegative/exim_24_import_part_authfail.q
hive/trunk/ql/src/test/queries/clientnegative/exim_25_import_nonexist_authfail.q
hive/trunk/ql/src/test/queries/clientpositive/exim_00_nonpart_empty.q
hive/trunk/ql/src/test/queries/clientpositive/exim_01_nonpart.q
hive/trunk/ql/src/test/queries/clientpositive/exim_02_00_part_empty.q
hive/trunk/ql/src/test/queries/clientpositive/exim_02_part.q
hive/trunk/ql/src/test/queries/clientpositive/exim_03_nonpart_over_compat.q
hive/trunk/ql/src/test/queries/clientpositive/exim_04_all_part.q
hive/trunk/ql/src/test/queries/clientpositive/exim_04_evolved_parts.q
hive/trunk/ql/src/test/queries/clientpositive/exim_05_some_part.q
hive/trunk/ql/src/test/queries/clientpositive/exim_06_one_part.q
hive/trunk/ql/src/test/queries/clientpositive/exim_07_all_part_over_nonoverlap.q
hive/trunk/ql/src/test/queries/clientpositive/exim_08_nonpart_rename.q
hive/trunk/ql/src/test/queries/clientpositive/exim_09_part_spec_nonoverlap.q
hive/trunk/ql/src/test/queries/clientpositive/exim_10_external_managed.q
hive/trunk/ql/src/test/queries/clientpositive/exim_11_managed_external.q
hive/trunk/ql/src/test/queries/clientpositive/exim_12_external_location.q
hive/trunk/ql/src/test/queries/clientpositive/exim_13_managed_location.q
hive/trunk/ql/src/test/queries/clientpositive/exim_14_managed_location_over_existing.q
hive/trunk/ql/src/test/queries/clientpositive/exim_15_external_part.q
hive/trunk/ql/src/test/queries/clientpositive/exim_16_part_external.q
hive/trunk/ql/src/test/queries/clientpositive/exim_17_part_managed.q
hive/trunk/ql/src/test/queries/clientpositive/exim_18_part_external.q
hive/trunk/ql/src/test/queries/clientpositive/exim_19_part_external_location.q
hive/trunk/ql/src/test/queries/clientpositive/exim_20_part_managed_location.q
hive/trunk/ql/src/test/queries/clientpositive/exim_21_export_authsuccess.q
hive/trunk/ql/src/test/queries/clientpositive/exim_22_import_exist_authsuccess.q
hive/trunk/ql/src/test/queries/clientpositive/exim_23_import_part_authsuccess.q
hive/trunk/ql/src/test/queries/clientpositive/exim_24_import_nonexist_authsuccess.q
hive/trunk/ql/src/test/results/clientnegative/exim_00_unsupported_schema.q.out
hive/trunk/ql/src/test/results/clientnegative/exim_01_nonpart_over_loaded.q.out
hive/trunk/ql/src/test/results/clientnegative/exim_02_all_part_over_overlap.q.out
hive/trunk/ql/src/test/results/clientnegative/exim_03_nonpart_noncompat_colschema.q.out
hive/trunk/ql/src/test/results/clientnegative/exim_04_nonpart_noncompat_colnumber.q.out
hive/trunk/ql/src/test/results/clientnegative/exim_05_nonpart_noncompat_coltype.q.out
hive/trunk/ql/src/test/results/clientnegative/exim_06_nonpart_noncompat_storage.q.out
hive/trunk/ql/src/test/results/clientnegative/exim_07_nonpart_noncompat_ifof.q.out
hive/trunk/ql/src/test/results/clientnegative/exim_08_nonpart_noncompat_serde.q.out
hive/trunk/ql/src/test/results/clientnegative/exim_09_nonpart_noncompat_serdeparam.q.out
hive/trunk/ql/src/test/results/clientnegative/exim_10_nonpart_noncompat_bucketing.q.out
hive/trunk/ql/src/test/results/clientnegative/exim_11_nonpart_noncompat_sorting.q.out
hive/trunk/ql/src/test/results/clientnegative/exim_12_nonnative_export.q.out
hive/trunk/ql/src/test/results/clientnegative/exim_13_nonnative_import.q.out
hive/trunk/ql/src/test/results/clientnegative/exim_14_nonpart_part.q.out
hive/trunk/ql/src/test/results/clientnegative/exim_15_part_nonpart.q.out
hive/trunk/ql/src/test/results/clientnegative/exim_16_part_noncompat_schema.q.out
hive/trunk/ql/src/test/results/clientnegative/exim_17_part_spec_underspec.q.out
hive/trunk/ql/src/test/results/clientnegative/exim_18_part_spec_missing.q.out
hive/trunk/ql/src/test/results/clientnegative/exim_19_external_over_existing.q.out
hive/trunk/ql/src/test/results/clientnegative/exim_20_managed_location_over_existing.q.out
hive/trunk/ql/src/test/results/clientnegative/exim_21_part_managed_external.q.out
hive/trunk/ql/src/test/results/clientnegative/exim_22_export_authfail.q.out
hive/trunk/ql/src/test/results/clientnegative/exim_23_import_exist_authfail.q.out
hive/trunk/ql/src/test/results/clientnegative/exim_24_import_part_authfail.q.out
hive/trunk/ql/src/test/results/clientnegative/exim_25_import_nonexist_authfail.q.out
hive/trunk/ql/src/test/results/clientpositive/exim_00_nonpart_empty.q.out
hive/trunk/ql/src/test/results/clientpositive/exim_01_nonpart.q.out
hive/trunk/ql/src/test/results/clientpositive/exim_02_00_part_empty.q.out
hive/trunk/ql/src/test/results/clientpositive/exim_02_part.q.out
hive/trunk/ql/src/test/results/clientpositive/exim_03_nonpart_over_compat.q.out
hive/trunk/ql/src/test/results/clientpositive/exim_04_all_part.q.out
hive/trunk/ql/src/test/results/clientpositive/exim_04_evolved_parts.q.out
hive/trunk/ql/src/test/results/clientpositive/exim_05_some_part.q.out
hive/trunk/ql/src/test/results/clientpositive/exim_06_one_part.q.out
hive/trunk/ql/src/test/results/clientpositive/exim_07_all_part_over_nonoverlap.q.out
hive/trunk/ql/src/test/results/clientpositive/exim_08_nonpart_rename.q.out
hive/trunk/ql/src/test/results/clientpositive/exim_09_part_spec_nonoverlap.q.out
hive/trunk/ql/src/test/results/clientpositive/exim_10_external_managed.q.out
hive/trunk/ql/src/test/results/clientpositive/exim_11_managed_external.q.out
hive/trunk/ql/src/test/results/clientpositive/exim_12_external_location.q.out
hive/trunk/ql/src/test/results/clientpositive/exim_13_managed_location.q.out
hive/trunk/ql/src/test/results/clientpositive/exim_14_managed_location_over_existing.q.out
hive/trunk/ql/src/test/results/clientpositive/exim_15_external_part.q.out
hive/trunk/ql/src/test/results/clientpositive/exim_16_part_external.q.out
hive/trunk/ql/src/test/results/clientpositive/exim_17_part_managed.q.out
hive/trunk/ql/src/test/results/clientpositive/exim_18_part_external.q.out
hive/trunk/ql/src/test/results/clientpositive/exim_19_part_external_location.q.out
hive/trunk/ql/src/test/results/clientpositive/exim_20_part_managed_location.q.out
hive/trunk/ql/src/test/results/clientpositive/exim_21_export_authsuccess.q.out
hive/trunk/ql/src/test/results/clientpositive/exim_22_import_exist_authsuccess.q.out
hive/trunk/ql/src/test/results/clientpositive/exim_23_import_part_authsuccess.q.out
hive/trunk/ql/src/test/results/clientpositive/exim_24_import_nonexist_authsuccess.q.out
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/conf/hive-default.xml
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/AddPartitionDesc.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CopyWork.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1081930&r1=1081929&r2=1081930&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Tue Mar 15 20:29:02 2011
@@ -378,6 +378,7 @@ public class HiveConf extends Configurat
HIVE_INDEX_IGNORE_HDFS_LOC("hive.index.compact.file.ignore.hdfs", false),
+ HIVE_EXIM_URI_SCHEME_WL("hive.exim.uri.scheme.whitelist", "hdfs,pfile"),
// temporary variable for testing. This is added just to turn off this feature in case of a bug in
// deployment. It has not been documented in hive-default.xml intentionally, this should be removed
// once the feature is stable
Modified: hive/trunk/conf/hive-default.xml
URL: http://svn.apache.org/viewvc/hive/trunk/conf/hive-default.xml?rev=1081930&r1=1081929&r2=1081930&view=diff
==============================================================================
--- hive/trunk/conf/hive-default.xml (original)
+++ hive/trunk/conf/hive-default.xml Tue Mar 15 20:29:02 2011
@@ -977,4 +977,11 @@
If the data got moved or the name of the cluster got changed, the index data should still be usable.</description>
</property>
+<property>
+ <name>hive.exim.uri.scheme.whitelist</name>
+ <value>hdfs,pfile</value>
+ <description>A comma separated list of acceptable URI schemes for import and export.</description>
+</property>
+
+
</configuration>
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=1081930&r1=1081929&r2=1081930&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Tue Mar 15 20:29:02 2011
@@ -67,8 +67,8 @@ import org.apache.hadoop.hive.ql.lockmgr
import org.apache.hadoop.hive.ql.lockmgr.HiveLockMode;
import org.apache.hadoop.hive.ql.lockmgr.HiveLockObj;
import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject;
-import org.apache.hadoop.hive.ql.lockmgr.LockException;
import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject.HiveLockObjectData;
+import org.apache.hadoop.hive.ql.lockmgr.LockException;
import org.apache.hadoop.hive.ql.metadata.AuthorizationException;
import org.apache.hadoop.hive.ql.metadata.DummyPartition;
import org.apache.hadoop.hive.ql.metadata.Hive;
@@ -83,6 +83,7 @@ import org.apache.hadoop.hive.ql.parse.B
import org.apache.hadoop.hive.ql.parse.ErrorMsg;
import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHookContext;
import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHookContextImpl;
+import org.apache.hadoop.hive.ql.parse.ImportSemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.ParseContext;
import org.apache.hadoop.hive.ql.parse.ParseDriver;
import org.apache.hadoop.hive.ql.parse.ParseException;
@@ -433,6 +434,15 @@ public class Driver implements CommandPr
ss.getAuthorizer().authorize(
db.getDatabase(db.getCurrentDatabase()), null,
HiveOperation.CREATETABLE_AS_SELECT.getOutputRequiredPrivileges());
+ } else {
+ if (op.equals(HiveOperation.IMPORT)) {
+ ImportSemanticAnalyzer isa = (ImportSemanticAnalyzer) sem;
+ if (!isa.existsTable()) {
+ ss.getAuthorizer().authorize(
+ db.getDatabase(db.getCurrentDatabase()), null,
+ HiveOperation.CREATETABLE_AS_SELECT.getOutputRequiredPrivileges());
+ }
+ }
}
if (outputs != null && outputs.size() > 0) {
for (WriteEntity write : outputs) {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java?rev=1081930&r1=1081929&r2=1081930&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java Tue Mar 15 20:29:02 2011
@@ -59,8 +59,12 @@ public class CopyTask extends Task<CopyW
FileStatus[] srcs = LoadSemanticAnalyzer.matchFilesOrDir(srcFs, fromPath);
if (srcs == null || srcs.length == 0) {
- console.printError("No files matching path: " + fromPath.toString());
- return 3;
+ if (work.isErrorOnSrcEmpty()) {
+ console.printError("No files matching path: " + fromPath.toString());
+ return 3;
+ } else {
+ return 0;
+ }
}
if (!dstFs.mkdirs(toPath)) {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=1081930&r1=1081929&r2=1081930&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Tue Mar 15 20:29:02 2011
@@ -875,15 +875,35 @@ public class DDLTask extends Task<DDLWor
return 0;
}
+
+
if (addPartitionDesc.getLocation() == null) {
- db.createPartition(tbl, addPartitionDesc.getPartSpec());
+ db.createPartition(tbl, addPartitionDesc.getPartSpec(), null,
+ addPartitionDesc.getPartParams(),
+ addPartitionDesc.getInputFormat(),
+ addPartitionDesc.getOutputFormat(),
+ addPartitionDesc.getNumBuckets(),
+ addPartitionDesc.getCols(),
+ addPartitionDesc.getSerializationLib(),
+ addPartitionDesc.getSerdeParams(),
+ addPartitionDesc.getBucketCols(),
+ addPartitionDesc.getSortCols());
+
} else {
if (tbl.isView()) {
throw new HiveException("LOCATION clause illegal for view partition");
}
// set partition path relative to table
db.createPartition(tbl, addPartitionDesc.getPartSpec(), new Path(tbl
- .getPath(), addPartitionDesc.getLocation()));
+ .getPath(), addPartitionDesc.getLocation()), addPartitionDesc.getPartParams(),
+ addPartitionDesc.getInputFormat(),
+ addPartitionDesc.getOutputFormat(),
+ addPartitionDesc.getNumBuckets(),
+ addPartitionDesc.getCols(),
+ addPartitionDesc.getSerializationLib(),
+ addPartitionDesc.getSerdeParams(),
+ addPartitionDesc.getBucketCols(),
+ addPartitionDesc.getSortCols());
}
Partition part = db
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java?rev=1081930&r1=1081929&r2=1081930&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java Tue Mar 15 20:29:02 2011
@@ -265,7 +265,7 @@ public class MoveTask extends Task<MoveW
dc = null; // reset data container to prevent it being added again.
} else { // static partitions
db.loadPartition(new Path(tbd.getSourceDir()), tbd.getTable().getTableName(),
- tbd.getPartitionSpec(), tbd.getReplace(), tbd.getHoldDDLTime());
+ tbd.getPartitionSpec(), tbd.getReplace(), tbd.getHoldDDLTime(), tbd.getInheritTableSpecs());
Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false);
dc = new DataContainer(table.getTTable(), partn.getTPartition());
// add this partition to post-execution hook
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1081930&r1=1081929&r2=1081930&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Tue Mar 15 20:29:02 2011
@@ -1044,11 +1044,15 @@ public class Hive {
* @param replace
* if true - replace files in the partition, otherwise add files to
* the partition
+ * @param holdDDLTime if true, force [re]create the partition
+ * @param inheritTableSpecs if true, on [re]creating the partition, take the
+ * location/inputformat/outputformat/serde details from table spec
* @param tmpDirPath
* The temporary directory.
*/
public void loadPartition(Path loadPath, String tableName,
- Map<String, String> partSpec, boolean replace, boolean holdDDLTime)
+ Map<String, String> partSpec, boolean replace, boolean holdDDLTime,
+ boolean inheritTableSpecs)
throws HiveException {
Table tbl = getTable(tableName);
try {
@@ -1059,18 +1063,24 @@ public class Hive {
* processes might move forward with partial data
*/
- Partition oldPart = getPartition(tbl, partSpec, false, null);
+ Partition oldPart = getPartition(tbl, partSpec, false);
Path oldPartPath = null;
if(oldPart != null) {
oldPartPath = oldPart.getPartitionPath();
}
- Path partPath = new Path(tbl.getDataLocation().getPath(),
- Warehouse.makePartPath(partSpec));
-
- Path newPartPath = new Path(loadPath.toUri().getScheme(), loadPath
- .toUri().getAuthority(), partPath.toUri().getPath());
+ Path newPartPath = null;
+
+ if (inheritTableSpecs) {
+ Path partPath = new Path(tbl.getDataLocation().getPath(),
+ Warehouse.makePartPath(partSpec));
+ newPartPath = new Path(loadPath.toUri().getScheme(), loadPath.toUri().getAuthority(),
+ partPath.toUri().getPath());
+ } else {
+ newPartPath = oldPartPath;
+ }
+
if (replace) {
Hive.replaceFiles(loadPath, newPartPath, oldPartPath, getConf());
} else {
@@ -1080,7 +1090,7 @@ public class Hive {
// recreate the partition if it existed before
if (!holdDDLTime) {
- getPartition(tbl, partSpec, true, newPartPath.toString());
+ getPartition(tbl, partSpec, true, newPartPath.toString(), inheritTableSpecs);
}
} catch (IOException e) {
LOG.error(StringUtils.stringifyException(e));
@@ -1142,7 +1152,7 @@ public class Hive {
fullPartSpecs.add(fullPartSpec);
// finally load the partition -- move the file to the final table address
- loadPartition(partPath, tableName, fullPartSpec, replace, holdDDLTime);
+ loadPartition(partPath, tableName, fullPartSpec, replace, holdDDLTime, true);
LOG.info("New loading path = " + partPath + " with partSpec " + fullPartSpec);
}
return fullPartSpecs;
@@ -1198,7 +1208,8 @@ public class Hive {
*/
public Partition createPartition(Table tbl, Map<String, String> partSpec)
throws HiveException {
- return createPartition(tbl, partSpec, null);
+ return createPartition(tbl, partSpec, null, null, null, null, -1,
+ null, null, null, null, null);
}
/**
@@ -1210,12 +1221,26 @@ public class Hive {
* partition keys and their values
* @param location
* location of this partition
+ * @param partParams
+ * partition parameters
+ * @param inputFormat the inputformat class
+ * @param outputformat the outputformat class
+ * @param numBuckets the number of buckets
+ * @param cols the column schema
+ * @param serializationLib the serde class
+ * @param serdeParams the serde parameters
+ * @param bucketCols the bucketing columns
+ * @param sortCols sort columns and order
+ *
* @return created partition object
* @throws HiveException
* if table doesn't exist or partition already exists
*/
public Partition createPartition(Table tbl, Map<String, String> partSpec,
- Path location) throws HiveException {
+ Path location, Map<String, String> partParams, String inputFormat, String outputFormat,
+ int numBuckets, List<FieldSchema> cols,
+ String serializationLib, Map<String, String> serdeParams,
+ List<String> bucketCols, List<Order> sortCols) throws HiveException {
org.apache.hadoop.hive.metastore.api.Partition partition = null;
@@ -1231,7 +1256,36 @@ public class Hive {
Partition tmpPart = new Partition(tbl, partSpec, location);
// No need to clear DDL_TIME in parameters since we know it's
// not populated on construction.
- partition = getMSC().add_partition(tmpPart.getTPartition());
+ org.apache.hadoop.hive.metastore.api.Partition inPart
+ = tmpPart.getTPartition();
+ if (partParams != null) {
+ inPart.setParameters(partParams);
+ }
+ if (inputFormat != null) {
+ inPart.getSd().setInputFormat(inputFormat);
+ }
+ if (outputFormat != null) {
+ inPart.getSd().setOutputFormat(outputFormat);
+ }
+ if (numBuckets != -1) {
+ inPart.getSd().setNumBuckets(numBuckets);
+ }
+ if (cols != null) {
+ inPart.getSd().setCols(cols);
+ }
+ if (serializationLib != null) {
+ inPart.getSd().getSerdeInfo().setSerializationLib(serializationLib);
+ }
+ if (serdeParams != null) {
+ inPart.getSd().getSerdeInfo().setParameters(serdeParams);
+ }
+ if (bucketCols != null) {
+ inPart.getSd().setBucketCols(bucketCols);
+ }
+ if (sortCols != null) {
+ inPart.getSd().setSortCols(sortCols);
+ }
+ partition = getMSC().add_partition(inPart);
} catch (Exception e) {
LOG.error(StringUtils.stringifyException(e));
throw new HiveException(e);
@@ -1242,7 +1296,7 @@ public class Hive {
public Partition getPartition(Table tbl, Map<String, String> partSpec,
boolean forceCreate) throws HiveException {
- return getPartition(tbl, partSpec, forceCreate, null);
+ return getPartition(tbl, partSpec, forceCreate, null, true);
}
/**
@@ -1255,11 +1309,13 @@ public class Hive {
* @param forceCreate
* if this is true and partition doesn't exist then a partition is
* created
+ * @param partPath the path where the partition data is located
+ * @param inheritTableSpecs whether to copy over the table specs for if/of/serde
* @return result partition object or null if there is no partition
* @throws HiveException
*/
public Partition getPartition(Table tbl, Map<String, String> partSpec,
- boolean forceCreate, String partPath) throws HiveException {
+ boolean forceCreate, String partPath, boolean inheritTableSpecs) throws HiveException {
if (!tbl.isValidSpec(partSpec)) {
throw new HiveException("Invalid partition: " + partSpec);
}
@@ -1298,10 +1354,11 @@ public class Hive {
else {
LOG.debug("altering partition for table " + tbl.getTableName()
+ " with partition spec : " + partSpec);
-
- tpart.getSd().setOutputFormat(tbl.getTTable().getSd().getOutputFormat());
- tpart.getSd().setInputFormat(tbl.getTTable().getSd().getInputFormat());
- tpart.getSd().getSerdeInfo().setSerializationLib(tbl.getSerializationLib());
+ if (inheritTableSpecs) {
+ tpart.getSd().setOutputFormat(tbl.getTTable().getSd().getOutputFormat());
+ tpart.getSd().setInputFormat(tbl.getTTable().getSd().getInputFormat());
+ tpart.getSd().getSerdeInfo().setSerializationLib(tbl.getSerializationLib());
+ }
if (partPath == null || partPath.trim().equals("")) {
throw new HiveException("new partition path should not be null or empty.");
}
@@ -1800,7 +1857,7 @@ public class Hive {
throw new HiveException("Unable to move results from " + srcs[0].getPath()
+ " to destination directory: " + destf);
}
- LOG.debug("Renaming:" + srcf.toString() + ",Status:" + b);
+ LOG.debug("Renaming:" + srcf.toString() + " to " + destf.toString() + ",Status:" + b);
} else { // srcf is a file or pattern containing wildcards
if (!fs.exists(destf)) {
fs.mkdirs(destf);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=1081930&r1=1081929&r2=1081930&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java Tue Mar 15 20:29:02 2011
@@ -602,6 +602,12 @@ public abstract class BaseSemanticAnalyz
public tableSpec(Hive db, HiveConf conf, ASTNode ast)
throws SemanticException {
+ this(db, conf, ast, true, false);
+ }
+
+ public tableSpec(Hive db, HiveConf conf, ASTNode ast,
+ boolean allowDynamicPartitionsSpec, boolean allowPartialPartitionsSpec)
+ throws SemanticException {
assert (ast.getToken().getType() == HiveParser.TOK_TAB
|| ast.getToken().getType() == HiveParser.TOK_TABLE_PARTITION
@@ -639,7 +645,12 @@ public abstract class BaseSemanticAnalyz
String val = null;
String colName = unescapeIdentifier(partspec_val.getChild(0).getText().toLowerCase());
if (partspec_val.getChildCount() < 2) { // DP in the form of T partition (ds, hr)
- ++numDynParts;
+ if (allowDynamicPartitionsSpec) {
+ ++numDynParts;
+ } else {
+ throw new SemanticException(ErrorMsg.INVALID_PARTITION
+ .getMsg(" - Dynamic partitions not allowed"));
+ }
} else { // in the form of T partition (ds="2010-03-03")
val = stripQuotes(partspec_val.getChild(1).getText());
}
@@ -672,14 +683,18 @@ public abstract class BaseSemanticAnalyz
specType = SpecType.DYNAMIC_PARTITION;
} else {
try {
- // this doesn't create partition.
- partHandle = db.getPartition(tableHandle, partSpec, false);
- if (partHandle == null) {
- // if partSpec doesn't exists in DB, return a delegate one
- // and the actual partition is created in MoveTask
- partHandle = new Partition(tableHandle, partSpec, null);
+ if (allowPartialPartitionsSpec) {
+ partitions = db.getPartitions(tableHandle, partSpec);
} else {
- partitions.add(partHandle);
+ // this doesn't create partition.
+ partHandle = db.getPartition(tableHandle, partSpec, false);
+ if (partHandle == null) {
+ // if partSpec doesn't exists in DB, return a delegate one
+ // and the actual partition is created in MoveTask
+ partHandle = new Partition(tableHandle, partSpec, null);
+ } else {
+ partitions.add(partHandle);
+ }
}
} catch (HiveException e) {
throw new SemanticException(
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java?rev=1081930&r1=1081929&r2=1081930&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java Tue Mar 15 20:29:02 2011
@@ -173,6 +173,12 @@ public enum ErrorMsg {
OUTERJOIN_USES_FILTERS("The query results could be wrong. " +
"Turn on hive.outerjoin.supports.filters"),
NEED_PARTITION_SPECIFICATION("Table is partitioned and partition specification is needed"),
+ INVALID_METADATA("The metadata file could not be parsed "),
+ NEED_TABLE_SPECIFICATION("Table name could be determined; It should be specified "),
+ PARTITION_EXISTS("Partition already exists"),
+ TABLE_DATA_EXISTS("Table exists and contains data files"),
+ INCOMPATIBLE_SCHEMA("The existing table is not compatible with the import spec. "),
+ EXIM_FOR_NON_NATIVE("Export/Import cannot be done for a non-native table. "),
;
private String mesg;
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java?rev=1081930&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java Tue Mar 15 20:29:02 2011
@@ -0,0 +1,335 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.AbstractMap;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.StringTokenizer;
+import java.util.TreeMap;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.thrift.TDeserializer;
+import org.apache.thrift.TException;
+import org.apache.thrift.TSerializer;
+import org.apache.thrift.protocol.TJSONProtocol;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+/**
+ *
+ * EximUtil. Utility methods for the export/import semantic
+ * analyzers.
+ *
+ */
+public class EximUtil {
+
+ private static Log LOG = LogFactory.getLog(EximUtil.class);
+
+ private EximUtil() {
+ }
+
+ /**
+ * Initialize the URI where the exported data collection is
+ * to created for export, or is present for import
+ */
+ static URI getValidatedURI(HiveConf conf, String dcPath) throws SemanticException {
+ try {
+ boolean testMode = conf.getBoolVar(HiveConf.ConfVars.HIVETESTMODE);
+ URI uri = new Path(dcPath).toUri();
+ String scheme = uri.getScheme();
+ String authority = uri.getAuthority();
+ String path = uri.getPath();
+ LOG.debug("Path before norm :" + path);
+ // generate absolute path relative to home directory
+ if (!path.startsWith("/")) {
+ if (testMode) {
+ path = new Path(System.getProperty("build.dir.hive"),
+ path).toString();
+ } else {
+ path = new Path(new Path("/user/" + System.getProperty("user.name")),
+ path).toString();
+ }
+ }
+ // set correct scheme and authority
+ if (StringUtils.isEmpty(scheme)) {
+ if (testMode) {
+ scheme = "pfile";
+ } else {
+ scheme = "hdfs";
+ }
+ }
+
+ // if scheme is specified but not authority then use the default
+ // authority
+ if (StringUtils.isEmpty(authority)) {
+ URI defaultURI = FileSystem.get(conf).getUri();
+ authority = defaultURI.getAuthority();
+ }
+
+ LOG.debug("Scheme:" + scheme + ", authority:" + authority + ", path:" + path);
+ Collection<String> eximSchemes = conf.getStringCollection(
+ HiveConf.ConfVars.HIVE_EXIM_URI_SCHEME_WL.varname);
+ if (!eximSchemes.contains(scheme)) {
+ throw new SemanticException(
+ ErrorMsg.INVALID_PATH.getMsg(
+ "only the following file systems accepted for export/import : "
+ + conf.get(HiveConf.ConfVars.HIVE_EXIM_URI_SCHEME_WL.varname)));
+ }
+
+ try {
+ return new URI(scheme, authority, path, null, null);
+ } catch (URISyntaxException e) {
+ throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(), e);
+ }
+ } catch (IOException e) {
+ throw new SemanticException(ErrorMsg.GENERIC_ERROR.getMsg(), e);
+ }
+ }
+
+ static void validateTable(org.apache.hadoop.hive.ql.metadata.Table table) throws SemanticException {
+ if (table.isOffline()) {
+ throw new SemanticException(
+ ErrorMsg.OFFLINE_TABLE_OR_PARTITION.getMsg(":Table "
+ + table.getTableName()));
+ }
+ if (table.isView()) {
+ throw new SemanticException(ErrorMsg.DML_AGAINST_VIEW.getMsg());
+ }
+ if (table.isNonNative()) {
+ throw new SemanticException(ErrorMsg.EXIM_FOR_NON_NATIVE.getMsg());
+ }
+ }
+
+ public static String relativeToAbsolutePath(HiveConf conf, String location) throws SemanticException {
+ boolean testMode = conf.getBoolVar(HiveConf.ConfVars.HIVETESTMODE);
+ if (testMode) {
+ URI uri = new Path(location).toUri();
+ String scheme = uri.getScheme();
+ String authority = uri.getAuthority();
+ String path = uri.getPath();
+ if (!path.startsWith("/")) {
+ path = new Path(System.getProperty("build.dir.hive"),
+ path).toString();
+ }
+ if (StringUtils.isEmpty(scheme)) {
+ scheme = "pfile";
+ }
+ try {
+ uri = new URI(scheme, authority, path, null, null);
+ } catch (URISyntaxException e) {
+ throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(), e);
+ }
+ return uri.toString();
+ } else {
+ //no-op for non-test mode for now
+ return location;
+ }
+ }
+
+ /* major version number should match for backward compatibility */
+ public static final String METADATA_FORMAT_VERSION = "0.1";
+ /* If null, then the major version number should match */
+ public static final String METADATA_FORMAT_FORWARD_COMPATIBLE_VERSION = null;
+
+ static void createExportDump(FileSystem fs, Path metadataPath, org.apache.hadoop.hive.ql.metadata.Table tableHandle,
+ List<org.apache.hadoop.hive.ql.metadata.Partition> partitions) throws SemanticException, IOException {
+ try {
+ JSONObject jsonContainer = new JSONObject();
+ jsonContainer.put("version", METADATA_FORMAT_VERSION);
+ if (METADATA_FORMAT_FORWARD_COMPATIBLE_VERSION != null) {
+ jsonContainer.put("fcversion", METADATA_FORMAT_FORWARD_COMPATIBLE_VERSION);
+ }
+ TSerializer serializer = new TSerializer(new TJSONProtocol.Factory());
+ try {
+ String tableDesc = serializer.toString(tableHandle.getTTable(), "UTF-8");
+ jsonContainer.put("table", tableDesc);
+ JSONArray jsonPartitions = new JSONArray();
+ if (partitions != null) {
+ for (org.apache.hadoop.hive.ql.metadata.Partition partition : partitions) {
+ String partDesc = serializer.toString(partition.getTPartition(), "UTF-8");
+ jsonPartitions.put(partDesc);
+ }
+ }
+ jsonContainer.put("partitions", jsonPartitions);
+ } catch (TException e) {
+ throw new SemanticException(
+ ErrorMsg.GENERIC_ERROR
+ .getMsg("Exception while serializing the metastore objects"), e);
+ }
+ OutputStream out = fs.create(metadataPath);
+ out.write(jsonContainer.toString().getBytes("UTF-8"));
+ out.close();
+
+ } catch (JSONException e) {
+ throw new SemanticException(ErrorMsg.GENERIC_ERROR.getMsg("Error in serializing metadata"), e);
+ }
+ }
+
+ static Map.Entry<Table, List<Partition>>
+ readMetaData(FileSystem fs, Path metadataPath)
+ throws IOException, SemanticException {
+ try {
+ FSDataInputStream mdstream = fs.open(metadataPath);
+ byte[] buffer = new byte[1024];
+ ByteArrayOutputStream sb = new ByteArrayOutputStream();
+ int read = mdstream.read(buffer);
+ while (read != -1) {
+ sb.write(buffer, 0, read);
+ read = mdstream.read(buffer);
+ }
+ String md = new String(sb.toByteArray(), "UTF-8");
+ JSONObject jsonContainer = new JSONObject(md);
+ String version = jsonContainer.getString("version");
+ String fcversion = null;
+ try {
+ fcversion = jsonContainer.getString("fcversion");
+ } catch (JSONException ignored) {}
+ checkCompatibility(version, fcversion);
+ String tableDesc = jsonContainer.getString("table");
+ Table table = new Table();
+ TDeserializer deserializer = new TDeserializer(new TJSONProtocol.Factory());
+ deserializer.deserialize(table, tableDesc, "UTF-8");
+ JSONArray jsonPartitions = new JSONArray(jsonContainer.getString("partitions"));
+ List<Partition> partitionsList = new ArrayList<Partition>(jsonPartitions.length());
+ for (int i = 0; i < jsonPartitions.length(); ++i) {
+ String partDesc = jsonPartitions.getString(i);
+ Partition partition = new Partition();
+ deserializer.deserialize(partition, partDesc, "UTF-8");
+ partitionsList.add(partition);
+ }
+ return new AbstractMap.SimpleEntry<Table, List<Partition>>(table, partitionsList);
+ } catch (JSONException e) {
+ throw new SemanticException(ErrorMsg.GENERIC_ERROR.getMsg("Error in serializing metadata"), e);
+ } catch (TException e) {
+ throw new SemanticException(ErrorMsg.GENERIC_ERROR.getMsg("Error in serializing metadata"), e);
+ }
+ }
+
+ /* check the forward and backward compatibility */
+ private static void checkCompatibility(String version, String fcVersion) throws SemanticException {
+ doCheckCompatibility(
+ METADATA_FORMAT_VERSION,
+ version,
+ fcVersion);
+ }
+
+ /* check the forward and backward compatibility */
+ public static void doCheckCompatibility(String currVersion,
+ String version, String fcVersion) throws SemanticException {
+ if (version == null) {
+ throw new SemanticException(ErrorMsg.INVALID_METADATA.getMsg("Version number missing"));
+ }
+ StringTokenizer st = new StringTokenizer(version, ".");
+ int data_major = Integer.parseInt(st.nextToken());
+
+ StringTokenizer st2 = new StringTokenizer(currVersion, ".");
+ int code_major = Integer.parseInt(st2.nextToken());
+ int code_minor = Integer.parseInt(st2.nextToken());
+
+ if (code_major > data_major) {
+ throw new SemanticException(ErrorMsg.INVALID_METADATA.getMsg("Not backward compatible."
+ + " Producer version " + version + ", Consumer version " +
+ currVersion));
+ } else {
+ if ((fcVersion == null) || fcVersion.isEmpty()) {
+ if (code_major < data_major) {
+ throw new SemanticException(ErrorMsg.INVALID_METADATA.getMsg("Not forward compatible."
+ + "Producer version " + version + ", Consumer version " +
+ currVersion));
+ }
+ } else {
+ StringTokenizer st3 = new StringTokenizer(fcVersion, ".");
+ int fc_major = Integer.parseInt(st3.nextToken());
+ int fc_minor = Integer.parseInt(st3.nextToken());
+ if ((fc_major > code_major) || ((fc_major == code_major) && (fc_minor > code_minor))) {
+ throw new SemanticException(ErrorMsg.INVALID_METADATA.getMsg("Not forward compatible."
+ + "Minimum version " + fcVersion + ", Consumer version " +
+ currVersion));
+ }
+ }
+ }
+ }
+
+ /**
+ * Return the partition specification from the specified keys and values
+ *
+ * @param partCols
+ * the names of the partition keys
+ * @param partVals
+ * the values of the partition keys
+ *
+ * @return the partition specification as a map
+ */
+ public static Map<String, String> makePartSpec(List<FieldSchema> partCols, List<String> partVals) {
+ Map<String, String> partSpec = new TreeMap<String, String>();
+ for (int i = 0; i < partCols.size(); ++i) {
+ partSpec.put(partCols.get(i).getName(), partVals.get(i));
+ }
+ return partSpec;
+ }
+
+ /**
+ * Compares the schemas - names, types and order, but ignoring comments
+ *
+ * @param newSchema
+ * the new schema
+ * @param oldSchema
+ * the old schema
+ * @return a boolean indicating match
+ */
+ public static boolean schemaCompare(List<FieldSchema> newSchema, List<FieldSchema> oldSchema) {
+ Iterator<FieldSchema> newColIter = newSchema.iterator();
+ for (FieldSchema oldCol : oldSchema) {
+ FieldSchema newCol = null;
+ if (newColIter.hasNext()) {
+ newCol = newColIter.next();
+ } else {
+ return false;
+ }
+ // not using FieldSchema.equals as comments can be different
+ if (!oldCol.getName().equals(newCol.getName())
+ || !oldCol.getType().equals(newCol.getType())) {
+ return false;
+ }
+ }
+ if (newColIter.hasNext()) {
+ return false;
+ }
+ return true;
+ }
+}
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java?rev=1081930&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java Tue Mar 15 20:29:02 2011
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.Serializable;
+import java.net.URI;
+import java.util.List;
+
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.TransformerException;
+
+import org.antlr.runtime.tree.Tree;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.TaskFactory;
+import org.apache.hadoop.hive.ql.hooks.ReadEntity;
+import org.apache.hadoop.hive.ql.hooks.WriteEntity;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.CopyWork;
+import org.apache.thrift.TException;
+import org.apache.thrift.TSerializer;
+import org.apache.thrift.protocol.TJSONProtocol;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+/**
+ * ExportSemanticAnalyzer.
+ *
+ */
+public class ExportSemanticAnalyzer extends BaseSemanticAnalyzer {
+
+ public ExportSemanticAnalyzer(HiveConf conf) throws SemanticException {
+ super(conf);
+ }
+
+ @Override
+ public void analyzeInternal(ASTNode ast) throws SemanticException {
+ Tree tableTree = ast.getChild(0);
+ Tree toTree = ast.getChild(1);
+
+ // initialize export path
+ String tmpPath = stripQuotes(toTree.getText());
+ URI toURI = EximUtil.getValidatedURI(conf, tmpPath);
+
+ // initialize source table/partition
+ tableSpec ts = new tableSpec(db, conf, (ASTNode) tableTree, false, true);
+ EximUtil.validateTable(ts.tableHandle);
+ try {
+ FileSystem fs = FileSystem.get(toURI, conf);
+ Path toPath = new Path(toURI.getScheme(), toURI.getAuthority(), toURI.getPath());
+ try {
+ FileStatus tgt = fs.getFileStatus(toPath);
+ // target exists
+ if (!tgt.isDir()) {
+ throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(ast,
+ "Target is not a directory : " + toURI));
+ } else {
+ FileStatus[] files = fs.listStatus(toPath);
+ if (files != null) {
+ throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(ast,
+ "Target is not an empty directory : " + toURI));
+ }
+ }
+ } catch (FileNotFoundException e) {
+ }
+ } catch (IOException e) {
+ throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(ast), e);
+ }
+
+ List<Partition> partitions = null;
+ try {
+ partitions = null;
+ if (ts.tableHandle.isPartitioned()) {
+ partitions = (ts.partitions != null) ? ts.partitions : db.getPartitions(ts.tableHandle);
+ }
+ String tmpfile = ctx.getLocalTmpFileURI();
+ Path path = new Path(tmpfile, "_metadata");
+ EximUtil.createExportDump(FileSystem.getLocal(conf), path, ts.tableHandle, partitions);
+ Task<? extends Serializable> rTask = TaskFactory.get(new CopyWork(
+ path.toString(), toURI.toString(), false), conf);
+ rootTasks.add(rTask);
+ LOG.debug("_metadata file written into " + path.toString()
+ + " and then copied to " + toURI.toString());
+ } catch (Exception e) {
+ throw new SemanticException(
+ ErrorMsg.GENERIC_ERROR
+ .getMsg("Exception while writing out the local file"), e);
+ }
+
+ if (ts.tableHandle.isPartitioned()) {
+ for (Partition partition : partitions) {
+ URI fromURI = partition.getDataLocation();
+ Path toPartPath = new Path(toURI.toString(), partition.getName());
+ Task<? extends Serializable> rTask = TaskFactory.get(
+ new CopyWork(fromURI.toString(), toPartPath.toString(), false),
+ conf);
+ rootTasks.add(rTask);
+ inputs.add(new ReadEntity(partition));
+ }
+ } else {
+ URI fromURI = ts.tableHandle.getDataLocation();
+ Path toDataPath = new Path(toURI.toString(), "data");
+ Task<? extends Serializable> rTask = TaskFactory.get(new CopyWork(
+ fromURI.toString(), toDataPath.toString(), false), conf);
+ rootTasks.add(rTask);
+ inputs.add(new ReadEntity(ts.tableHandle));
+ }
+ outputs.add(new WriteEntity(toURI.toString(),
+ toURI.getScheme().equals("hdfs") ? true : false));
+ }
+
+}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g?rev=1081930&r1=1081929&r2=1081930&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g Tue Mar 15 20:29:02 2011
@@ -85,6 +85,8 @@ TOK_RIGHTOUTERJOIN;
TOK_FULLOUTERJOIN;
TOK_UNIQUEJOIN;
TOK_LOAD;
+TOK_EXPORT;
+TOK_IMPORT;
TOK_NULL;
TOK_ISNULL;
TOK_ISNOTNULL;
@@ -282,6 +284,8 @@ execStatement
@after { msgs.pop(); }
: queryStatementExpression
| loadStatement
+ | exportStatement
+ | importStatement
| ddlStatement
;
@@ -292,6 +296,20 @@ loadStatement
-> ^(TOK_LOAD $path $tab $islocal? $isoverwrite?)
;
+exportStatement
+@init { msgs.push("export statement"); }
+@after { msgs.pop(); }
+ : KW_EXPORT KW_TABLE (tab=tableOrPartition) KW_TO (path=StringLiteral)
+ -> ^(TOK_EXPORT $tab $path)
+ ;
+
+importStatement
+@init { msgs.push("import statement"); }
+@after { msgs.pop(); }
+ : KW_IMPORT ((ext=KW_EXTERNAL)? KW_TABLE (tab=tableOrPartition))? KW_FROM (path=StringLiteral) tableLocation?
+ -> ^(TOK_IMPORT $path $tab? $ext? tableLocation?)
+ ;
+
ddlStatement
@init { msgs.push("ddl statement"); }
@after { msgs.pop(); }
@@ -2074,6 +2092,8 @@ KW_DISTRIBUTE: 'DISTRIBUTE';
KW_SORT: 'SORT';
KW_UNION: 'UNION';
KW_LOAD: 'LOAD';
+KW_EXPORT: 'EXPORT';
+KW_IMPORT: 'IMPORT';
KW_DATA: 'DATA';
KW_INPATH: 'INPATH';
KW_IS: 'IS';
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java?rev=1081930&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java Tue Mar 15 20:29:02 2011
@@ -0,0 +1,537 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.antlr.runtime.tree.Tree;
+import org.apache.commons.lang.ObjectUtils;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Order;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.TaskFactory;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.hooks.WriteEntity;
+import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.AddPartitionDesc;
+import org.apache.hadoop.hive.ql.plan.CopyWork;
+import org.apache.hadoop.hive.ql.plan.CreateTableDesc;
+import org.apache.hadoop.hive.ql.plan.DDLWork;
+import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
+import org.apache.hadoop.hive.ql.plan.MoveWork;
+import org.apache.hadoop.hive.serde.Constants;
+
+/**
+ * ImportSemanticAnalyzer.
+ *
+ */
+public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
+
+ public ImportSemanticAnalyzer(HiveConf conf) throws SemanticException {
+ super(conf);
+ }
+
+ private boolean tableExists = false;
+
+ public boolean existsTable() {
+ return tableExists;
+ }
+
+ @Override
+ public void analyzeInternal(ASTNode ast) throws SemanticException {
+ try {
+ Tree fromTree = ast.getChild(0);
+ // initialize load path
+ String tmpPath = stripQuotes(fromTree.getText());
+ URI fromURI = EximUtil.getValidatedURI(conf, tmpPath);
+
+ FileSystem fs = FileSystem.get(fromURI, conf);
+ String dbname = null;
+ CreateTableDesc tblDesc = null;
+ List<AddPartitionDesc> partitionDescs = new ArrayList<AddPartitionDesc>();
+ Path fromPath = new Path(fromURI.getScheme(), fromURI.getAuthority(),
+ fromURI.getPath());
+ try {
+ Path metadataPath = new Path(fromPath, "_metadata");
+ Map.Entry<org.apache.hadoop.hive.metastore.api.Table,
+ List<Partition>> rv = EximUtil.readMetaData(fs, metadataPath);
+ dbname = db.getCurrentDatabase();
+ org.apache.hadoop.hive.metastore.api.Table table = rv.getKey();
+ tblDesc = new CreateTableDesc(
+ table.getTableName(),
+ false, // isExternal: set to false here, can be overwritten by the
+ // IMPORT stmt
+ table.getSd().getCols(),
+ table.getPartitionKeys(),
+ table.getSd().getBucketCols(),
+ table.getSd().getSortCols(),
+ table.getSd().getNumBuckets(),
+ null, null, null, null, null, // these 5 delims passed as serde params
+ null, // comment passed as table params
+ table.getSd().getInputFormat(),
+ table.getSd().getOutputFormat(),
+ null, // location: set to null here, can be
+ // overwritten by the IMPORT stmt
+ table.getSd().getSerdeInfo().getSerializationLib(),
+ null, // storagehandler passed as table params
+ table.getSd().getSerdeInfo().getParameters(),
+ table.getParameters(), false);
+
+
+ List<FieldSchema> partCols = tblDesc.getPartCols();
+ List<String> partColNames = new ArrayList<String>(partCols.size());
+ for (FieldSchema fsc : partCols) {
+ partColNames.add(fsc.getName());
+ }
+ List<Partition> partitions = rv.getValue();
+ for (Partition partition : partitions) {
+ AddPartitionDesc partDesc = new AddPartitionDesc(dbname, tblDesc.getTableName(),
+ EximUtil.makePartSpec(tblDesc.getPartCols(), partition.getValues()),
+ partition.getSd().getLocation(), partition.getParameters());
+ partDesc.setInputFormat(partition.getSd().getInputFormat());
+ partDesc.setOutputFormat(partition.getSd().getOutputFormat());
+ partDesc.setNumBuckets(partition.getSd().getNumBuckets());
+ partDesc.setCols(partition.getSd().getCols());
+ partDesc.setSerializationLib(partition.getSd().getSerdeInfo().getSerializationLib());
+ partDesc.setSerdeParams(partition.getSd().getSerdeInfo().getParameters());
+ partDesc.setBucketCols(partition.getSd().getBucketCols());
+ partDesc.setSortCols(partition.getSd().getSortCols());
+ partDesc.setLocation(new Path(fromPath,
+ Warehouse.makePartName(tblDesc.getPartCols(), partition.getValues())).toString());
+ partitionDescs.add(partDesc);
+ }
+ } catch (IOException e) {
+ throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(), e);
+ }
+ LOG.debug("metadata read and parsed");
+ for (int i = 1; i < ast.getChildCount(); ++i) {
+ ASTNode child = (ASTNode) ast.getChild(i);
+ switch (child.getToken().getType()) {
+ case HiveParser.KW_EXTERNAL:
+ tblDesc.setExternal(true);
+ break;
+ case HiveParser.TOK_TABLELOCATION:
+ String location = unescapeSQLString(child.getChild(0).getText());
+ location = EximUtil.relativeToAbsolutePath(conf, location);
+ tblDesc.setLocation(location);
+ break;
+ case HiveParser.TOK_TAB:
+ Tree tableTree = child.getChild(0);
+ // initialize destination table/partition
+ String tableName = getUnescapedName((ASTNode)tableTree);
+ tblDesc.setTableName(tableName);
+ // get partition metadata if partition specified
+ LinkedHashMap<String, String> partSpec = new LinkedHashMap<String, String>();
+ if (child.getChildCount() == 2) {
+ ASTNode partspec = (ASTNode) child.getChild(1);
+ // partSpec is a mapping from partition column name to its value.
+ for (int j = 0; j < partspec.getChildCount(); ++j) {
+ ASTNode partspec_val = (ASTNode) partspec.getChild(j);
+ String val = null;
+ String colName = unescapeIdentifier(partspec_val.getChild(0)
+ .getText().toLowerCase());
+ if (partspec_val.getChildCount() < 2) { // DP in the form of T
+ // partition (ds, hr)
+ throw new SemanticException(
+ ErrorMsg.INVALID_PARTITION
+ .getMsg(" - Dynamic partitions not allowed"));
+ } else { // in the form of T partition (ds="2010-03-03")
+ val = stripQuotes(partspec_val.getChild(1).getText());
+ }
+ partSpec.put(colName, val);
+ }
+ boolean found = false;
+ for (Iterator<AddPartitionDesc> partnIter = partitionDescs
+ .listIterator(); partnIter.hasNext();) {
+ AddPartitionDesc addPartitionDesc = partnIter.next();
+ if (!found && addPartitionDesc.getPartSpec().equals(partSpec)) {
+ found = true;
+ } else {
+ partnIter.remove();
+ }
+ }
+ if (!found) {
+ throw new SemanticException(
+ ErrorMsg.INVALID_PARTITION
+ .getMsg(" - Specified partition not found in import directory"));
+ }
+ }
+ }
+ }
+ if (tblDesc.getTableName() == null) {
+ throw new SemanticException(ErrorMsg.NEED_TABLE_SPECIFICATION.getMsg());
+ } else {
+ conf.set("import.destination.table", tblDesc.getTableName());
+ for (AddPartitionDesc addPartitionDesc : partitionDescs) {
+ addPartitionDesc.setTableName(tblDesc.getTableName());
+ }
+ }
+ Warehouse wh = new Warehouse(conf);
+ try {
+ Table table = db.getTable(tblDesc.getTableName());
+ checkTable(table, tblDesc);
+ LOG.debug("table " + tblDesc.getTableName()
+ + " exists: metadata checked");
+ tableExists = true;
+ conf.set("import.destination.dir", table.getDataLocation().toString());
+ if (table.isPartitioned()) {
+ LOG.debug("table partitioned");
+ for (AddPartitionDesc addPartitionDesc : partitionDescs) {
+ if (db.getPartition(table, addPartitionDesc.getPartSpec(), false) == null) {
+ rootTasks.add(addSinglePartition(fromURI, fs, tblDesc, table, wh, addPartitionDesc));
+ } else {
+ throw new SemanticException(
+ ErrorMsg.PARTITION_EXISTS
+ .getMsg(partSpecToString(addPartitionDesc.getPartSpec())));
+ }
+ }
+ } else {
+ LOG.debug("table non-partitioned");
+ checkTargetLocationEmpty(fs, new Path(table.getDataLocation()
+ .toString()));
+ loadTable(fromURI, table);
+ }
+ outputs.add(new WriteEntity(table));
+ } catch (InvalidTableException e) {
+ LOG.debug("table " + tblDesc.getTableName() + " does not exist");
+
+ Task<?> t = TaskFactory.get(new DDLWork(getInputs(), getOutputs(),
+ tblDesc), conf);
+ Table table = new Table(dbname, tblDesc.getTableName());
+ conf.set("import.destination.dir",
+ wh.getDnsPath(wh.getDefaultTablePath(
+ db.getCurrentDatabase(), tblDesc.getTableName())).toString());
+ if ((tblDesc.getPartCols() != null) && (tblDesc.getPartCols().size() != 0)) {
+ for (AddPartitionDesc addPartitionDesc : partitionDescs) {
+ t.addDependentTask(
+ addSinglePartition(fromURI, fs, tblDesc, table, wh, addPartitionDesc));
+ }
+ } else {
+ LOG.debug("adding dependent CopyWork/MoveWork for table");
+ if (tblDesc.isExternal() && (tblDesc.getLocation() == null)) {
+ LOG.debug("Importing in place, no emptiness check, no copying/loading");
+ Path dataPath = new Path(fromURI.toString(), "data");
+ tblDesc.setLocation(dataPath.toString());
+ } else {
+ Path tablePath = null;
+ if (tblDesc.getLocation() != null) {
+ tablePath = new Path(tblDesc.getLocation());
+ } else {
+ tablePath = wh.getDnsPath(wh.getDefaultTablePath(
+ db.getCurrentDatabase(), tblDesc.getTableName()));
+ }
+ checkTargetLocationEmpty(fs, tablePath);
+ t.addDependentTask(loadTable(fromURI, table));
+ }
+ }
+ rootTasks.add(t);
+ //inputs.add(new ReadEntity(fromURI.toString(),
+ // fromURI.getScheme().equals("hdfs") ? true : false));
+ }
+ } catch (SemanticException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new SemanticException(ErrorMsg.GENERIC_ERROR.getMsg(), e);
+ }
+ }
+
+ private Task<?> loadTable(URI fromURI, Table table) {
+ Path dataPath = new Path(fromURI.toString(), "data");
+ String tmpURI = ctx.getExternalTmpFileURI(fromURI);
+ Task<?> copyTask = TaskFactory.get(new CopyWork(dataPath.toString(),
+ tmpURI, false), conf);
+ LoadTableDesc loadTableWork = new LoadTableDesc(tmpURI.toString(),
+ ctx.getExternalTmpFileURI(fromURI),
+ Utilities.getTableDesc(table), new TreeMap<String, String>(),
+ false);
+ Task<?> loadTableTask = TaskFactory.get(new MoveWork(getInputs(),
+ getOutputs(), loadTableWork, null, false), conf);
+ copyTask.addDependentTask(loadTableTask);
+ rootTasks.add(copyTask);
+ return loadTableTask;
+ }
+
+ private Task<?> addSinglePartition(URI fromURI, FileSystem fs, CreateTableDesc tblDesc,
+ Table table, Warehouse wh,
+ AddPartitionDesc addPartitionDesc) throws MetaException, IOException, SemanticException {
+ if (tblDesc.isExternal() && tblDesc.getLocation() == null) {
+ LOG.debug("Importing in-place: adding AddPart for partition "
+ + partSpecToString(addPartitionDesc.getPartSpec()));
+ // addPartitionDesc already has the right partition location
+ Task<?> addPartTask = TaskFactory.get(new DDLWork(getInputs(),
+ getOutputs(), addPartitionDesc), conf);
+ return addPartTask;
+ } else {
+ String srcLocation = addPartitionDesc.getLocation();
+ Path tgtPath = null;
+ if (tblDesc.getLocation() == null) {
+ if (table.getDataLocation() != null) {
+ tgtPath = new Path(table.getDataLocation().toString(),
+ Warehouse.makePartPath(addPartitionDesc.getPartSpec()));
+ } else {
+ tgtPath = new Path(wh.getDnsPath(wh.getDefaultTablePath(
+ db.getCurrentDatabase(), tblDesc.getTableName())),
+ Warehouse.makePartPath(addPartitionDesc.getPartSpec()));
+ }
+ } else {
+ tgtPath = new Path(tblDesc.getLocation());
+ }
+ checkTargetLocationEmpty(fs, tgtPath);
+ addPartitionDesc.setLocation(tgtPath.toString());
+ LOG.debug("adding dependent CopyWork/AddPart/MoveWork for partition "
+ + partSpecToString(addPartitionDesc.getPartSpec())
+ + " with source location: " + srcLocation);
+ String tmpURI = ctx.getExternalTmpFileURI(fromURI);
+ Task<?> copyTask = TaskFactory.get(new CopyWork(srcLocation,
+ tmpURI, false), conf);
+ Task<?> addPartTask = TaskFactory.get(new DDLWork(getInputs(),
+ getOutputs(), addPartitionDesc), conf);
+ LoadTableDesc loadTableWork = new LoadTableDesc(tmpURI,
+ ctx.getExternalTmpFileURI(fromURI),
+ Utilities.getTableDesc(table),
+ addPartitionDesc.getPartSpec(), true);
+ loadTableWork.setInheritTableSpecs(false);
+ Task<?> loadPartTask = TaskFactory.get(new MoveWork(
+ getInputs(), getOutputs(), loadTableWork, null, false),
+ conf);
+ copyTask.addDependentTask(loadPartTask);
+ addPartTask.addDependentTask(loadPartTask);
+ rootTasks.add(copyTask);
+ return addPartTask;
+ }
+ }
+
+ private void checkTargetLocationEmpty(FileSystem fs, Path targetPath)
+ throws IOException, SemanticException {
+ LOG.debug("checking emptiness of " + targetPath.toString());
+ if (fs.exists(targetPath)) {
+ FileStatus[] status = fs.listStatus(targetPath);
+ if (status.length > 0) {
+ LOG.debug("Files inc. " + status[0].getPath().toString()
+ + " found in path : " + targetPath.toString());
+ throw new SemanticException(ErrorMsg.TABLE_DATA_EXISTS.getMsg());
+ }
+ }
+ }
+
+ private static String partSpecToString(Map<String, String> partSpec) {
+ StringBuilder sb = new StringBuilder();
+ boolean firstTime = true;
+ for (Map.Entry<String, String> entry : partSpec.entrySet()) {
+ if (!firstTime) {
+ sb.append(',');
+ }
+ firstTime = false;
+ sb.append(entry.getKey());
+ sb.append('=');
+ sb.append(entry.getValue());
+ }
+ return sb.toString();
+ }
+
+ private static void checkTable(Table table, CreateTableDesc tableDesc)
+ throws SemanticException, URISyntaxException {
+ {
+ EximUtil.validateTable(table);
+ if (!table.isPartitioned()) {
+ if (tableDesc.isExternal()) { // the import statement specified external
+ throw new SemanticException(
+ ErrorMsg.INCOMPATIBLE_SCHEMA
+ .getMsg(" External table cannot overwrite existing table."
+ + " Drop existing table first."));
+ }
+ } else {
+ if (tableDesc.isExternal()) { // the import statement specified external
+ if (!table.getTableType().equals(TableType.EXTERNAL_TABLE)) {
+ throw new SemanticException(
+ ErrorMsg.INCOMPATIBLE_SCHEMA
+ .getMsg(" External table cannot overwrite existing table."
+ + " Drop existing table first."));
+ }
+ }
+ }
+ }
+ {
+ if (!table.isPartitioned()) {
+ if (tableDesc.getLocation() != null) { // IMPORT statement specified
+ // location
+ if (!table.getDataLocation()
+ .equals(new URI(tableDesc.getLocation()))) {
+ throw new SemanticException(
+ ErrorMsg.INCOMPATIBLE_SCHEMA.getMsg(" Location does not match"));
+ }
+ }
+ }
+ }
+ {
+ // check column order and types
+ List<FieldSchema> existingTableCols = table.getCols();
+ List<FieldSchema> importedTableCols = tableDesc.getCols();
+ if (!EximUtil.schemaCompare(importedTableCols, existingTableCols)) {
+ throw new SemanticException(
+ ErrorMsg.INCOMPATIBLE_SCHEMA
+ .getMsg(" Column Schema does not match"));
+ }
+ }
+ {
+ // check partitioning column order and types
+ List<FieldSchema> existingTablePartCols = table.getPartCols();
+ List<FieldSchema> importedTablePartCols = tableDesc.getPartCols();
+ if (!EximUtil.schemaCompare(importedTablePartCols, existingTablePartCols)) {
+ throw new SemanticException(
+ ErrorMsg.INCOMPATIBLE_SCHEMA
+ .getMsg(" Partition Schema does not match"));
+ }
+ }
+ {
+ // check table params
+ Map<String, String> existingTableParams = table.getParameters();
+ Map<String, String> importedTableParams = tableDesc.getTblProps();
+ String error = checkParams(existingTableParams, importedTableParams,
+ new String[] { "howl.isd",
+ "howl.osd" });
+ if (error != null) {
+ throw new SemanticException(
+ ErrorMsg.INCOMPATIBLE_SCHEMA
+ .getMsg(" Table parameters do not match: " + error));
+ }
+ }
+ {
+ // check IF/OF/Serde
+ String existingifc = table.getInputFormatClass().getName();
+ String importedifc = tableDesc.getInputFormat();
+ String existingofc = table.getOutputFormatClass().getName();
+ String importedofc = tableDesc.getOutputFormat();
+ if ((!existingifc.equals(importedifc))
+ || (!existingofc.equals(importedofc))) {
+ throw new SemanticException(
+ ErrorMsg.INCOMPATIBLE_SCHEMA
+ .getMsg(" Table inputformat/outputformats do not match"));
+ }
+ String existingSerde = table.getSerializationLib();
+ String importedSerde = tableDesc.getSerName();
+ if (!existingSerde.equals(importedSerde)) {
+ throw new SemanticException(
+ ErrorMsg.INCOMPATIBLE_SCHEMA
+ .getMsg(" Table Serde class does not match"));
+ }
+ String existingSerdeFormat = table
+ .getSerdeParam(Constants.SERIALIZATION_FORMAT);
+ String importedSerdeFormat = tableDesc.getSerdeProps().get(
+ Constants.SERIALIZATION_FORMAT);
+ if (!ObjectUtils.equals(existingSerdeFormat, importedSerdeFormat)) {
+ throw new SemanticException(
+ ErrorMsg.INCOMPATIBLE_SCHEMA
+ .getMsg(" Table Serde format does not match"));
+ }
+ }
+ {
+ // check bucket/sort cols
+ if (!ObjectUtils.equals(table.getBucketCols(), tableDesc.getBucketCols())) {
+ throw new SemanticException(
+ ErrorMsg.INCOMPATIBLE_SCHEMA
+ .getMsg(" Table bucketing spec does not match"));
+ }
+ List<Order> existingOrder = table.getSortCols();
+ List<Order> importedOrder = tableDesc.getSortCols();
+ // safely sorting
+ final class OrderComparator implements Comparator<Order> {
+ @Override
+ public int compare(Order o1, Order o2) {
+ if (o1.getOrder() < o2.getOrder()) {
+ return -1;
+ } else {
+ if (o1.getOrder() == o2.getOrder()) {
+ return 0;
+ } else {
+ return 1;
+ }
+ }
+ }
+ }
+ if (existingOrder != null) {
+ if (importedOrder != null) {
+ Collections.sort(existingOrder, new OrderComparator());
+ Collections.sort(importedOrder, new OrderComparator());
+ if (!existingOrder.equals(importedOrder)) {
+ throw new SemanticException(
+ ErrorMsg.INCOMPATIBLE_SCHEMA
+ .getMsg(" Table sorting spec does not match"));
+ }
+ }
+ } else {
+ if (importedOrder != null) {
+ throw new SemanticException(
+ ErrorMsg.INCOMPATIBLE_SCHEMA
+ .getMsg(" Table sorting spec does not match"));
+ }
+ }
+ }
+ }
+
+ private static String checkParams(Map<String, String> map1,
+ Map<String, String> map2, String[] keys) {
+ if (map1 != null) {
+ if (map2 != null) {
+ for (String key : keys) {
+ String v1 = map1.get(key);
+ String v2 = map2.get(key);
+ if (!ObjectUtils.equals(v1, v2)) {
+ return "Mismatch for " + key;
+ }
+ }
+ } else {
+ for (String key : keys) {
+ if (map1.get(key) != null) {
+ return "Mismatch for " + key;
+ }
+ }
+ }
+ } else {
+ if (map2 != null) {
+ for (String key : keys) {
+ if (map2.get(key) != null) {
+ return "Mismatch for " + key;
+ }
+ }
+ }
+ }
+ return null;
+ }
+
+}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1081930&r1=1081929&r2=1081930&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Tue Mar 15 20:29:02 2011
@@ -7113,6 +7113,7 @@ public class SemanticAnalyzer extends Ba
break;
case HiveParser.TOK_TABLELOCATION:
location = unescapeSQLString(child.getChild(0).getText());
+ location = EximUtil.relativeToAbsolutePath(conf, location);
break;
case HiveParser.TOK_TABLEPROPERTIES:
tblProps = DDLSemanticAnalyzer.getProps((ASTNode) child.getChild(0));
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java?rev=1081930&r1=1081929&r2=1081930&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java Tue Mar 15 20:29:02 2011
@@ -36,6 +36,8 @@ public final class SemanticAnalyzerFacto
static {
commandType.put(HiveParser.TOK_EXPLAIN, HiveOperation.EXPLAIN);
commandType.put(HiveParser.TOK_LOAD, HiveOperation.LOAD);
+ commandType.put(HiveParser.TOK_EXPORT, HiveOperation.EXPORT);
+ commandType.put(HiveParser.TOK_IMPORT, HiveOperation.IMPORT);
commandType.put(HiveParser.TOK_CREATEDATABASE, HiveOperation.CREATEDATABASE);
commandType.put(HiveParser.TOK_DROPDATABASE, HiveOperation.DROPDATABASE);
commandType.put(HiveParser.TOK_SWITCHDATABASE, HiveOperation.SWITCHDATABASE);
@@ -118,6 +120,10 @@ public final class SemanticAnalyzerFacto
return new ExplainSemanticAnalyzer(conf);
case HiveParser.TOK_LOAD:
return new LoadSemanticAnalyzer(conf);
+ case HiveParser.TOK_EXPORT:
+ return new ExportSemanticAnalyzer(conf);
+ case HiveParser.TOK_IMPORT:
+ return new ImportSemanticAnalyzer(conf);
case HiveParser.TOK_CREATEDATABASE:
case HiveParser.TOK_DROPDATABASE:
case HiveParser.TOK_SWITCHDATABASE:
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/AddPartitionDesc.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/AddPartitionDesc.java?rev=1081930&r1=1081929&r2=1081930&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/AddPartitionDesc.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/AddPartitionDesc.java Tue Mar 15 20:29:02 2011
@@ -18,23 +18,35 @@
package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
-import java.util.HashMap;
import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Order;
+
/**
* Contains the information needed to add a partition.
*/
public class AddPartitionDesc extends DDLDesc implements Serializable {
private static final long serialVersionUID = 1L;
-
+
String tableName;
String dbName;
String location;
boolean ifNotExists;
boolean expectView;
LinkedHashMap<String, String> partSpec;
+ Map<String, String> partParams;
+ String inputFormat = null;
+ String outputFormat = null;
+ int numBuckets = -1;
+ List<FieldSchema> cols = null;
+ String serializationLib = null;
+ Map<String, String> serdeParams = null;
+ List<String> bucketCols = null;
+ List<Order> sortCols = null;
/**
* For serialization only.
@@ -51,7 +63,25 @@ public class AddPartitionDesc extends DD
* partition specification.
* @param location
* partition location, relative to table location.
- * @param ifNotExists
+ * @param params
+ * partition parameters.
+ */
+ public AddPartitionDesc(String dbName, String tableName,
+ Map<String, String> partSpec, String location, Map<String, String> params) {
+ this(dbName, tableName, partSpec, location, true, false);
+ this.partParams = params;
+ }
+
+ /**
+ * @param dbName
+ * database to add to.
+ * @param tableName
+ * table to add to.
+ * @param partSpec
+ * partition specification.
+ * @param location
+ * partition location, relative to table location.
+ * @param ifNotExists
* if true, the partition is only added if it doesn't exist
* @param expectView
* true for ALTER VIEW, false for ALTER TABLE
@@ -136,7 +166,7 @@ public class AddPartitionDesc extends DD
}
/**
- * @param ifNotExists
+ * @param ifNotExists
* if the part should be added only if it doesn't exist
*/
public void setIfNotExists(boolean ifNotExists) {
@@ -144,6 +174,86 @@ public class AddPartitionDesc extends DD
}
/**
+ * @return partition parameters.
+ */
+ public Map<String, String> getPartParams() {
+ return partParams;
+ }
+
+ /**
+ * @param partParams
+ * partition parameters
+ */
+
+ public void setPartParams(Map<String, String> partParams) {
+ this.partParams = partParams;
+ }
+
+ public int getNumBuckets() {
+ return numBuckets;
+ }
+
+ public void setNumBuckets(int numBuckets) {
+ this.numBuckets = numBuckets;
+ }
+
+ public List<FieldSchema> getCols() {
+ return cols;
+ }
+
+ public void setCols(List<FieldSchema> cols) {
+ this.cols = cols;
+ }
+
+ public String getSerializationLib() {
+ return serializationLib;
+ }
+
+ public void setSerializationLib(String serializationLib) {
+ this.serializationLib = serializationLib;
+ }
+
+ public Map<String, String> getSerdeParams() {
+ return serdeParams;
+ }
+
+ public void setSerdeParams(Map<String, String> serdeParams) {
+ this.serdeParams = serdeParams;
+ }
+
+ public List<String> getBucketCols() {
+ return bucketCols;
+ }
+
+ public void setBucketCols(List<String> bucketCols) {
+ this.bucketCols = bucketCols;
+ }
+
+ public List<Order> getSortCols() {
+ return sortCols;
+ }
+
+ public void setSortCols(List<Order> sortCols) {
+ this.sortCols = sortCols;
+ }
+
+ public String getInputFormat() {
+ return inputFormat;
+ }
+
+ public void setInputFormat(String inputFormat) {
+ this.inputFormat = inputFormat;
+ }
+
+ public String getOutputFormat() {
+ return outputFormat;
+ }
+
+ public void setOutputFormat(String outputFormat) {
+ this.outputFormat = outputFormat;
+ }
+
+ /*
* @return whether to expect a view being altered
*/
public boolean getExpectView() {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CopyWork.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CopyWork.java?rev=1081930&r1=1081929&r2=1081930&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CopyWork.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CopyWork.java Tue Mar 15 20:29:02 2011
@@ -29,13 +29,19 @@ public class CopyWork implements Seriali
private static final long serialVersionUID = 1L;
private String fromPath;
private String toPath;
+ private boolean errorOnSrcEmpty;
public CopyWork() {
}
public CopyWork(final String fromPath, final String toPath) {
+ this(fromPath, toPath, true);
+ }
+
+ public CopyWork(final String fromPath, final String toPath, boolean errorOnSrcEmpty) {
this.fromPath = fromPath;
this.toPath = toPath;
+ this.setErrorOnSrcEmpty(errorOnSrcEmpty);
}
@Explain(displayName = "source")
@@ -55,4 +61,12 @@ public class CopyWork implements Seriali
public void setToPath(final String toPath) {
this.toPath = toPath;
}
+
+ public void setErrorOnSrcEmpty(boolean errorOnSrcEmpty) {
+ this.errorOnSrcEmpty = errorOnSrcEmpty;
+ }
+
+ public boolean isErrorOnSrcEmpty() {
+ return errorOnSrcEmpty;
+ }
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java?rev=1081930&r1=1081929&r2=1081930&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java Tue Mar 15 20:29:02 2011
@@ -24,6 +24,8 @@ public enum HiveOperation {
EXPLAIN("EXPLAIN", null, null),
LOAD("LOAD", null, new Privilege[]{Privilege.ALTER_DATA}),
+ EXPORT("EXPORT", new Privilege[]{Privilege.SELECT}, null),
+ IMPORT("IMPORT", null, new Privilege[]{Privilege.ALTER_METADATA, Privilege.ALTER_DATA}),
CREATEDATABASE("CREATEDATABASE", null, null),
DROPDATABASE("DROPDATABASE", null, null),
SWITCHDATABASE("SWITCHDATABASE", null, null),
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java?rev=1081930&r1=1081929&r2=1081930&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java Tue Mar 15 20:29:02 2011
@@ -33,6 +33,8 @@ public class LoadTableDesc extends org.a
private String tmpDir;
private DynamicPartitionCtx dpCtx;
private boolean holdDDLTime;
+ private boolean inheritTableSpecs = true; //For partitions, flag controlling whether the current
+ //table specs are to be used
// TODO: the below seems like they should just be combined into partitionDesc
private org.apache.hadoop.hive.ql.plan.TableDesc table;
@@ -128,4 +130,12 @@ public class LoadTableDesc extends org.a
public void setDPCtx(final DynamicPartitionCtx dpCtx) {
this.dpCtx = dpCtx;
}
+
+ public boolean getInheritTableSpecs() {
+ return inheritTableSpecs;
+ }
+
+ public void setInheritTableSpecs(boolean inheritTableSpecs) {
+ this.inheritTableSpecs = inheritTableSpecs;
+ }
}