You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by br...@apache.org on 2015/01/13 01:18:40 UTC
svn commit: r1651241 - in /hive/trunk:
beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java
itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java
Author: brock
Date: Tue Jan 13 00:18:39 2015
New Revision: 1651241
URL: http://svn.apache.org/r1651241
Log:
HIVE-9309 - schematool fails on Postgres 8.1 (Mohit Sabharwal via Brock)
Modified:
hive/trunk/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java
hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java
Modified: hive/trunk/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java
URL: http://svn.apache.org/viewvc/hive/trunk/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java?rev=1651241&r1=1651240&r2=1651241&view=diff
==============================================================================
--- hive/trunk/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java (original)
+++ hive/trunk/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java Tue Jan 13 00:18:39 2015
@@ -17,6 +17,7 @@
*/
package org.apache.hive.beeline;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaException;
@@ -97,55 +98,65 @@ public class HiveSchemaHelper {
}
static final String DEFAUTL_DELIMITER = ";";
- /***
+
+ /**
* Find the type of given command
+ *
* @param dbCommand
* @return
*/
public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException;
- /** Parse the DB specific nesting format and extract the inner script name if any
+ /**
+ * Parse the DB specific nesting format and extract the inner script name if any
+ *
* @param dbCommand command from parent script
* @return
* @throws IllegalFormatException
*/
public String getScriptName(String dbCommand) throws IllegalArgumentException;
- /***
+ /**
* Find if the given command is a nested script execution
+ *
* @param dbCommand
* @return
*/
public boolean isNestedScript(String dbCommand);
- /***
+ /**
* Find if the given command should not be passed to DB
+ *
* @param dbCommand
* @return
*/
public boolean isNonExecCommand(String dbCommand);
- /***
+ /**
* Get the SQL statement delimiter
+ *
* @return
*/
public String getDelimiter();
- /***
+ /**
* Clear any client specific tags
+ *
* @return
*/
public String cleanseCommand(String dbCommand);
- /***
+ /**
* Does the DB required table/column names quoted
+ *
* @return
*/
public boolean needsQuotedIdentifier();
- /***
+ /**
* Flatten the nested upgrade script into a buffer
- * @param scriptDir upgrade script directory
+ *
+ * @param scriptDir upgrade script directory
* @param scriptFile upgrade script file
* @return string of sql commands
*/
@@ -258,6 +269,8 @@ public class HiveSchemaHelper {
private void setDbOpts(String dbOpts) {
if (dbOpts != null) {
this.dbOpts = Lists.newArrayList(dbOpts.split(","));
+ } else {
+ this.dbOpts = Lists.newArrayList();
}
}
@@ -369,6 +382,10 @@ public class HiveSchemaHelper {
// Postgres specific parser
public static class PostgresCommandParser extends AbstractCommandParser {
private static String POSTGRES_NESTING_TOKEN = "\\i";
+ @VisibleForTesting
+ public static String POSTGRES_STANDARD_STRINGS_OPT = "SET standard_conforming_strings";
+ @VisibleForTesting
+ public static String POSTGRES_SKIP_STANDARD_STRINGS_DBOPT = "postgres.filter.81";
public PostgresCommandParser(String dbOpts, String msUsername, String msPassword,
HiveConf hiveConf) {
@@ -394,6 +411,19 @@ public class HiveSchemaHelper {
public boolean needsQuotedIdentifier() {
return true;
}
+
+ @Override
+ public boolean isNonExecCommand(String dbCommand) {
+ // Skip "standard_conforming_strings" command which is read-only in older
+ // Postgres versions like 8.1
+ // See: http://www.postgresql.org/docs/8.2/static/release-8-1.html
+ if (getDbOpts().contains(POSTGRES_SKIP_STANDARD_STRINGS_DBOPT)) {
+ if (dbCommand.startsWith(POSTGRES_STANDARD_STRINGS_OPT)) {
+ return true;
+ }
+ }
+ return super.isNonExecCommand(dbCommand);
+ }
}
//Oracle specific parser
Modified: hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java?rev=1651241&r1=1651240&r2=1651241&view=diff
==============================================================================
--- hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java (original)
+++ hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java Tue Jan 13 00:18:39 2015
@@ -35,6 +35,7 @@ import org.apache.hadoop.hive.conf.HiveC
import org.apache.hadoop.hive.metastore.HiveMetaException;
import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfo;
import org.apache.hive.beeline.HiveSchemaHelper.NestedScriptParser;
+import org.apache.hive.beeline.HiveSchemaHelper.PostgresCommandParser;
public class TestSchemaTool extends TestCase {
private HiveSchemaTool schemaTool;
@@ -406,6 +407,56 @@ public class TestSchemaTool extends Test
assertTrue(flattenedSql.contains(parentTab));
}
+ /**
+ * Test script formatting
+ * @throws Exception
+ */
+ public void testPostgresFilter() throws Exception {
+ String testScript[] = {
+ "-- this is a comment",
+ "DROP TABLE IF EXISTS fooTab;",
+ HiveSchemaHelper.PostgresCommandParser.POSTGRES_STANDARD_STRINGS_OPT + ";",
+ "CREATE TABLE fooTab(id INTEGER);",
+ "DROP TABLE footab;",
+ "-- ending comment"
+ };
+
+ String expectedScriptWithOptionPresent[] = {
+ "DROP TABLE IF EXISTS fooTab",
+ HiveSchemaHelper.PostgresCommandParser.POSTGRES_STANDARD_STRINGS_OPT,
+ "CREATE TABLE fooTab(id INTEGER)",
+ "DROP TABLE footab",
+ };
+
+ NestedScriptParser noDbOptParser = HiveSchemaHelper
+ .getDbCommandParser("postgres");
+ String expectedSQL = StringUtils.join(
+ expectedScriptWithOptionPresent, System.getProperty("line.separator")) +
+ System.getProperty("line.separator");
+ File testScriptFile = generateTestScript(testScript);
+ String flattenedSql = noDbOptParser.buildCommand(
+ testScriptFile.getParentFile().getPath(), testScriptFile.getName());
+ assertEquals(expectedSQL, flattenedSql);
+
+ String expectedScriptWithOptionAbsent[] = {
+ "DROP TABLE IF EXISTS fooTab",
+ "CREATE TABLE fooTab(id INTEGER)",
+ "DROP TABLE footab",
+ };
+
+ NestedScriptParser dbOptParser = HiveSchemaHelper.getDbCommandParser(
+ "postgres",
+ PostgresCommandParser.POSTGRES_SKIP_STANDARD_STRINGS_DBOPT,
+ null, null, null);
+ expectedSQL = StringUtils.join(
+ expectedScriptWithOptionAbsent, System.getProperty("line.separator")) +
+ System.getProperty("line.separator");
+ testScriptFile = generateTestScript(testScript);
+ flattenedSql = dbOptParser.buildCommand(
+ testScriptFile.getParentFile().getPath(), testScriptFile.getName());
+ assertEquals(expectedSQL, flattenedSql);
+ }
+
private File generateTestScript(String [] stmts) throws IOException {
File testScriptFile = File.createTempFile("schematest", ".sql");
testScriptFile.deleteOnExit();