You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ga...@apache.org on 2018/02/12 03:51:26 UTC
[09/12] hive git commit: HIVE-17983 Make the standalone metastore
generate tarballs etc. (Alan Gates, reviewed by Thejas Nair)
http://git-wip-us.apache.org/repos/asf/hive/blob/47cac2d0/standalone-metastore/pom.xml
----------------------------------------------------------------------
diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml
index 58ed741..df769f5 100644
--- a/standalone-metastore/pom.xml
+++ b/standalone-metastore/pom.xml
@@ -38,6 +38,8 @@
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.useIncrementalCompilation>false</maven.compiler.useIncrementalCompilation>
<maven.repo.local>${settings.localRepository}</maven.repo.local>
+ <maven.assembly.plugin.version>2.3</maven.assembly.plugin.version>
+ <maven.exec.plugin.version>1.6.0</maven.exec.plugin.version>
<!-- Test Properties -->
<log4j.conf.dir>${project.basedir}/src/test/resources</log4j.conf.dir>
@@ -45,6 +47,8 @@
<test.warehouse.dir>${project.build.directory}/warehouse</test.warehouse.dir>
<test.warehouse.scheme>file://</test.warehouse.scheme>
<test.forkcount>1</test.forkcount>
+ <skipITests>true</skipITests>
+ <itest.jdbc.jars>set-this-to-colon-separated-full-path-list-of-jars-to-run-integration-tests</itest.jdbc.jars>
<!-- Plugin versions -->
<ant.contrib.version>1.0b3</ant.contrib.version>
@@ -75,6 +79,7 @@
<log4j2.version>2.8.2</log4j2.version>
<mockito-all.version>1.10.19</mockito-all.version>
<protobuf.version>2.5.0</protobuf.version>
+ <sqlline.version>1.3.0</sqlline.version>
<storage-api.version>3.0.0-SNAPSHOT</storage-api.version>
<!-- Thrift properties -->
@@ -277,10 +282,21 @@
<version>1.4.0</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>sqlline</groupId>
+ <artifactId>sqlline</artifactId>
+ <version>${sqlline.version}</version>
+ </dependency>
<!-- test scope dependencies -->
<dependency>
+ <groupId>com.microsoft.sqlserver</groupId>
+ <artifactId>mssql-jdbc</artifactId>
+ <version>6.2.1.jre8</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junit.version}</version>
@@ -292,6 +308,20 @@
<version>${mockito-all.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <!-- Note, this is LGPL. But we're only using it in a test and not changing it, so I
+ believe we are fine. -->
+ <groupId>org.mariadb.jdbc</groupId>
+ <artifactId>mariadb-java-client</artifactId>
+ <version>2.2.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.postgresql</groupId>
+ <artifactId>postgresql</artifactId>
+ <version>9.3-1102-jdbc41</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<profiles>
@@ -427,6 +457,11 @@
<artifactId>maven-checkstyle-plugin</artifactId>
<version>${maven.checkstyle.plugin.version}</version>
</plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>${maven.exec.plugin.version}</version>
+ </plugin>
</plugins>
</pluginManagement>
<plugins>
@@ -467,6 +502,21 @@
<goal>run</goal>
</goals>
</execution>
+ <execution>
+ <id>setup-metastore-scripts</id>
+ <phase>process-test-resources</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <mkdir dir="${test.tmp.dir}/scripts/metastore/upgrade" />
+ <copy todir="${test.tmp.dir}/scripts/metastore/upgrade">
+ <fileset dir="${basedir}/src/main/sql/"/>
+ </copy>
+ </target>
+ </configuration>
+ </execution>
</executions>
</plugin>
<plugin>
@@ -498,11 +548,62 @@
</execution>
</executions>
</plugin>
- <!-- TODO MS-SPLIT assembly plugin -->
<!-- TODO MS-SPLIT findbugs plugin -->
<!-- TODO MS-SPLIT javadoc plugin -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>${maven.assembly.plugin.version}</version>
+ <executions>
+ <execution>
+ <id>assemble</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <finalName>apache-hive-metastore-${project.version}</finalName>
+ <descriptors>
+ <descriptor>src/assembly/bin.xml</descriptor>
+ <descriptor>src/assembly/src.xml</descriptor>
+ </descriptors>
+ <tarLongFileMode>gnu</tarLongFileMode>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <version>2.20.1</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <redirectTestOutputToFile>true</redirectTestOutputToFile>
+ <reuseForks>false</reuseForks>
+ <argLine>-Xmx2048m</argLine>
+ <failIfNoTests>false</failIfNoTests>
+ <systemPropertyVariables>
+ <log4j.debug>true</log4j.debug>
+ <java.io.tmpdir>${test.tmp.dir}</java.io.tmpdir>
+ <test.tmp.dir>${test.tmp.dir}</test.tmp.dir>
+ <hive.in.test>true</hive.in.test>
+ </systemPropertyVariables>
+ <additionalClasspathElements>
+ <additionalClasspathElement>${log4j.conf.dir}</additionalClasspathElement>
+ <additionalClasspathElement>${itest.jdbc.jars}</additionalClasspathElement>
+ </additionalClasspathElements>
+ <skipITs>${skipITests}</skipITs> <!-- set this to false to run these tests -->
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.16</version>
<configuration>
@@ -583,6 +684,27 @@
</executions>
</plugin>
<plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <executable>java</executable>
+ <arguments>
+ <argument>-classpath</argument>
+ <classpath/>
+ <argument>org.apache.hadoop.hive.metastore.conf.ConfTemplatePrinter</argument>
+ <argument>${project.build.directory}/generated-sources/conf/metastore-site.xml.template</argument>
+ </arguments>
+ </configuration>
+ </plugin>
+ <plugin>
<groupId>org.datanucleus</groupId>
<artifactId>datanucleus-maven-plugin</artifactId>
<version>4.0.5</version>
http://git-wip-us.apache.org/repos/asf/hive/blob/47cac2d0/standalone-metastore/src/assembly/bin.xml
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/assembly/bin.xml b/standalone-metastore/src/assembly/bin.xml
new file mode 100644
index 0000000..81912d7
--- /dev/null
+++ b/standalone-metastore/src/assembly/bin.xml
@@ -0,0 +1,136 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<assembly
+ xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+
+ <id>bin</id>
+
+ <formats>
+ <format>dir</format>
+ <format>tar.gz</format>
+ </formats>
+
+ <baseDirectory>apache-hive-metastore-${project.version}-bin</baseDirectory>
+
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>lib</outputDirectory>
+ <unpack>false</unpack>
+ <useProjectArtifact>true</useProjectArtifact>
+ <useStrictFiltering>true</useStrictFiltering>
+ <useTransitiveFiltering>true</useTransitiveFiltering>
+ <excludes>
+ <exclude>org.apache.hadoop:*</exclude>
+ <exclude>org.slf4j:*</exclude>
+ <exclude>log4j:*</exclude>
+ </excludes>
+ </dependencySet>
+ </dependencySets>
+
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}</directory>
+ <excludes>
+ <exclude>target/**</exclude>
+ <exclude>.classpath</exclude>
+ <exclude>.project</exclude>
+ <exclude>.settings/**</exclude>
+ <exclude>lib/**</exclude>
+ </excludes>
+
+ <includes>
+ <include>README.txt</include>
+ <include>LICENSE</include>
+ <include>NOTICE</include>
+ </includes>
+ <outputDirectory>/</outputDirectory>
+ </fileSet>
+
+ <fileSet>
+ <directory>${project.basedir}/binary-package-licenses</directory>
+ <includes>
+ <include>/*</include>
+ </includes>
+ <excludes>
+ <exclude>/README</exclude>
+ </excludes>
+ <outputDirectory>binary-package-licenses</outputDirectory>
+ </fileSet>
+
+ <fileSet>
+ <fileMode>755</fileMode>
+ <directory>${project.basedir}/src/main/scripts</directory>
+ <includes>
+ <include>base</include>
+ <include>schematool</include>
+ <include>start-metastore</include>
+ <include>metastore-config.sh</include>
+ <include>ext/**/*</include>
+ </includes>
+ <outputDirectory>bin</outputDirectory>
+ </fileSet>
+
+ <fileSet>
+ <directory>${project.basedir}/src/main/sql</directory>
+ <includes>
+ <include>**/*</include>
+ </includes>
+ <outputDirectory>scripts/metastore/upgrade</outputDirectory>
+ </fileSet>
+
+ <fileSet>
+ <directory>${project.basedir}/src/gen/thrift/gen-php</directory>
+ <includes>
+ <include>**/*</include>
+ </includes>
+ <outputDirectory>lib/php/packages/hive_metastore</outputDirectory>
+ </fileSet>
+
+ <fileSet>
+ <directory>${project.basedir}/src/gen/thrift/gen-py/hive_metastore</directory>
+ <fileMode>755</fileMode>
+ <includes>
+ <include>**/*</include>
+ </includes>
+ <outputDirectory>lib/py/hive_metastore</outputDirectory>
+ </fileSet>
+
+ <fileSet>
+ <directory>${project.basedir}/src/main/resources/</directory>
+ <fileMode>644</fileMode>
+ <includes>
+ <include>metastore-site.xml</include>
+ <include>metastore-log4j2.properties</include>
+ </includes>
+ <outputDirectory>conf</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+ <files>
+ <file>
+ <source>${project.build.directory}/generated-sources/conf/metastore-site.xml.template</source>
+ <outputDirectory>conf</outputDirectory>
+ </file>
+ </files>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/hive/blob/47cac2d0/standalone-metastore/src/assembly/src.xml
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/assembly/src.xml b/standalone-metastore/src/assembly/src.xml
new file mode 100644
index 0000000..a240544
--- /dev/null
+++ b/standalone-metastore/src/assembly/src.xml
@@ -0,0 +1,53 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<assembly
+ xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+
+ <id>src</id>
+
+ <formats>
+ <format>tar.gz</format>
+ </formats>
+
+ <baseDirectory>apache-hive-metastore-${project.version}-src</baseDirectory>
+
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}</directory>
+
+ <excludes>
+ <exclude>target/**</exclude>
+ </excludes>
+
+ <includes>
+ <include>.checkstyle</include>
+ <include>.gitattributes</include>
+ <include>.gitignore</include>
+ <include>LICENSE</include>
+ <include>NOTICE</include>
+ <include>pom.xml</include>
+ <include>src/**/*</include>
+ </includes>
+ <outputDirectory>/</outputDirectory>
+ </fileSet>
+ </fileSets>
+</assembly>
http://git-wip-us.apache.org/repos/asf/hive/blob/47cac2d0/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java
index be89f9b..ed4a2ef 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java
@@ -55,6 +55,13 @@ public interface IMetaStoreSchemaInfo {
String generateInitFileName(String toVersion) throws HiveMetaException;
/**
+ * Get SQL script that will create the user and database for Metastore to use.
+ * @return filename
+ * @throws HiveMetaException if something goes wrong.
+ */
+ String getCreateUserScript() throws HiveMetaException;
+
+ /**
* Find the directory of metastore scripts
*
* @return the path of directory where the sql scripts are
http://git-wip-us.apache.org/repos/asf/hive/blob/47cac2d0/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
index 9b014da..88bd42d 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
@@ -39,12 +39,14 @@ import org.apache.hadoop.hive.metastore.utils.MetastoreVersionInfo;
public class MetaStoreSchemaInfo implements IMetaStoreSchemaInfo {
protected static final String UPGRADE_FILE_PREFIX = "upgrade-";
- private static final String INIT_FILE_PREFIX = "hive-schema-";
- private static final String VERSION_UPGRADE_LIST = "upgrade.order";
- private static final String PRE_UPGRADE_PREFIX = "pre-";
- protected final String dbType;
+ protected static final String INIT_FILE_PREFIX = "hive-schema-";
+ protected static final String VERSION_UPGRADE_LIST = "upgrade.order";
+ protected static final String PRE_UPGRADE_PREFIX = "pre-";
+ protected static final String CREATE_USER_PREFIX = "create-user";
+
private String[] hiveSchemaVersions;
- private final String hiveHome;
+ private final String metastoreHome;
+ protected final String dbType;
// Some version upgrades often don't change schema. So they are equivalent to
// a version
@@ -57,8 +59,8 @@ public class MetaStoreSchemaInfo implements IMetaStoreSchemaInfo {
"1.2.1", "1.2.0"
);
- public MetaStoreSchemaInfo(String hiveHome, String dbType) throws HiveMetaException {
- this.hiveHome = hiveHome;
+ public MetaStoreSchemaInfo(String metastoreHome, String dbType) throws HiveMetaException {
+ this.metastoreHome = metastoreHome;
this.dbType = dbType;
}
@@ -137,13 +139,24 @@ public class MetaStoreSchemaInfo implements IMetaStoreSchemaInfo {
return initScriptName;
}
+ @Override
+ public String getCreateUserScript() throws HiveMetaException {
+ String createScript = CREATE_USER_PREFIX + "." + dbType + SQL_FILE_EXTENSION;
+ // check if the file exists
+ if (!(new File(getMetaStoreScriptDir() + File.separatorChar +
+ createScript).exists())) {
+ throw new HiveMetaException("Unable to find create user file, expected: " + createScript);
+ }
+ return createScript;
+ }
+
/**
* Find the directory of metastore scripts
* @return
*/
@Override
public String getMetaStoreScriptDir() {
- return hiveHome + File.separatorChar +
+ return metastoreHome + File.separatorChar +
"scripts" + File.separatorChar + "metastore" +
File.separatorChar + "upgrade" + File.separatorChar + dbType;
}
@@ -207,8 +220,8 @@ public class MetaStoreSchemaInfo implements IMetaStoreSchemaInfo {
@Override
public String getMetaStoreSchemaVersion(MetaStoreConnectionInfo connectionInfo) throws HiveMetaException {
String versionQuery;
- boolean needsQuotedIdentifier =
- HiveSchemaHelper.getDbCommandParser(connectionInfo.getDbType(), connectionInfo.getMetaDbType()).needsQuotedIdentifier();
+ boolean needsQuotedIdentifier = HiveSchemaHelper.getDbCommandParser(connectionInfo.getDbType(),
+ connectionInfo.getMetaDbType(), false).needsQuotedIdentifier();
if (needsQuotedIdentifier) {
versionQuery = "select t.\"SCHEMA_VERSION\" from \"VERSION\" t";
} else {
http://git-wip-us.apache.org/repos/asf/hive/blob/47cac2d0/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/ConfTemplatePrinter.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/ConfTemplatePrinter.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/ConfTemplatePrinter.java
new file mode 100644
index 0000000..f57e2ce
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/ConfTemplatePrinter.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.conf;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Text;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+import java.io.File;
+
+public class ConfTemplatePrinter {
+
+ private Document generateTemplate() throws ParserConfigurationException {
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ DocumentBuilder docBuilder = dbf.newDocumentBuilder();
+ Document doc = docBuilder.newDocument();
+ doc.appendChild(doc.createProcessingInstruction(
+ "xml-stylesheet", "type=\"text/xsl\" href=\"configuration.xsl\""));
+
+ doc.appendChild(doc.createComment("\n" +
+ " Licensed to the Apache Software Foundation (ASF) under one or more\n" +
+ " contributor license agreements. See the NOTICE file distributed with\n" +
+ " this work for additional information regarding copyright ownership.\n" +
+ " The ASF licenses this file to You under the Apache License, Version 2.0\n" +
+ " (the \"License\"); you may not use this file except in compliance with\n" +
+ " the License. You may obtain a copy of the License at\n" +
+ "\n" +
+ " http://www.apache.org/licenses/LICENSE-2.0\n" +
+ "\n" +
+ " Unless required by applicable law or agreed to in writing, software\n" +
+ " distributed under the License is distributed on an \"AS IS\" BASIS,\n" +
+ " WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n" +
+ " See the License for the specific language governing permissions and\n" +
+ " limitations under the License.\n"));
+
+ Element root = doc.createElement("configuration");
+ doc.appendChild(root);
+
+ root.appendChild(doc.createComment(
+ " WARNING!!! This file is auto generated for documentation purposes ONLY! "));
+ root.appendChild(doc.createComment(
+ " WARNING!!! Any changes you make to this file will be ignored by the metastore. "));
+ root.appendChild(doc.createComment(
+ " WARNING!!! You must make your changes in metastore-site.xml instead. "));
+
+ root.appendChild(doc.createComment(" Metastore Execution Parameters "));
+
+ root.appendChild(doc.createComment("================================"));
+ root.appendChild(doc.createComment("All time unit values have a time unit abbreviation suffix"));
+ root.appendChild(doc.createComment("Any time value can take any of the units"));
+ root.appendChild(doc.createComment("d = day"));
+ root.appendChild(doc.createComment("h = hour"));
+ root.appendChild(doc.createComment("m = minute"));
+ root.appendChild(doc.createComment("s = second"));
+ root.appendChild(doc.createComment("ms = millisecond"));
+ root.appendChild(doc.createComment("us = microsecond"));
+ root.appendChild(doc.createComment("ns = nanosecond"));
+ root.appendChild(doc.createComment("================================"));
+
+ for (MetastoreConf.ConfVars confVars : MetastoreConf.ConfVars.values()) {
+ Element property = appendElement(root, "property", null);
+ appendElement(property, "name", confVars.getVarname());
+ appendElement(property, "value", confVars.getDefaultVal().toString());
+ appendElement(property, "description", normalize(confVars.getDescription()));
+ // wish to add new line here.
+ }
+ return doc;
+
+ }
+
+ private String normalize(String description) {
+ int index = description.indexOf('\n');
+ if (index < 0) {
+ return description;
+ }
+ int prev = 0;
+ StringBuilder builder = new StringBuilder(description.length() << 1);
+ for (;index > 0; index = description.indexOf('\n', prev = index + 1)) {
+ builder.append("\n ").append(description.substring(prev, index));
+ }
+ if (prev < description.length()) {
+ builder.append("\n ").append(description.substring(prev));
+ }
+ builder.append("\n ");
+ return builder.toString();
+ }
+
+ private void writeToFile(File template, Document document) throws TransformerException {
+ Transformer transformer = TransformerFactory.newInstance().newTransformer();
+ transformer.setOutputProperty(OutputKeys.INDENT, "yes");
+ transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2");
+ DOMSource source = new DOMSource(document);
+ StreamResult result = new StreamResult(template);
+ transformer.transform(source, result);
+ }
+
+ private Element appendElement(Element parent, String name, String text) {
+ Document document = parent.getOwnerDocument();
+ Element child = document.createElement(name);
+ parent.appendChild(child);
+ if (text != null) {
+ Text textNode = document.createTextNode(text);
+ child.appendChild(textNode);
+ }
+ return child;
+ }
+
+ private void print(String fileName) throws ParserConfigurationException, TransformerException {
+ Document doc = generateTemplate();
+ File file = new File(fileName);
+ File dir = file.getParentFile();
+ // Make certain the target directory exists.
+ dir.mkdirs();
+ writeToFile(file, doc);
+ }
+
+ public static void main(String[] args) throws Exception {
+ if (args.length != 1) {
+ String msg = "Usage: ConfTemplatePrinter filename";
+ System.err.println(msg);
+ throw new RuntimeException(msg);
+ }
+ ConfTemplatePrinter printer = new ConfTemplatePrinter();
+ printer.print(args[0]);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/47cac2d0/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index 5b8e5ca..699a649 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -96,7 +96,16 @@ public class MetastoreConf {
@Override
public String toString() {
- return Long.toString(unit.toNanos(val)) + "ns";
+ switch (unit) {
+ case NANOSECONDS: return Long.toString(val) + "ns";
+ case MICROSECONDS: return Long.toString(val) + "us";
+ case MILLISECONDS: return Long.toString(val) + "ms";
+ case SECONDS: return Long.toString(val) + "s";
+ case MINUTES: return Long.toString(val) + "m";
+ case HOURS: return Long.toString(val) + "h";
+ case DAYS: return Long.toString(val) + "d";
+ }
+ throw new RuntimeException("Unknown time unit " + unit);
}
}
@@ -537,7 +546,7 @@ public class MetastoreConf {
"The default value \"-1\" means no limit."),
LOG4J_FILE("metastore.log4j.file", "hive.log4j.file", "",
"Hive log4j configuration file.\n" +
- "If the property is not set, then logging will be initialized using hive-log4j2.properties found on the classpath.\n" +
+ "If the property is not set, then logging will be initialized using metastore-log4j2.properties found on the classpath.\n" +
"If the property is set, the value must be a valid URI (java.net.URI, e.g. \"file:///tmp/my-logging.xml\"), \n" +
"which you can then extract a URL from and pass to PropertyConfigurator.configure(URL)."),
MANAGER_FACTORY_CLASS("javax.jdo.PersistenceManagerFactoryClass",
@@ -893,81 +902,91 @@ public class MetastoreConf {
private final Object defaultVal;
private final Validator validator;
private final boolean caseSensitive;
+ private final String description;
- ConfVars(String varname, String hiveName, String defaultVal, String comment) {
+ ConfVars(String varname, String hiveName, String defaultVal, String description) {
this.varname = varname;
this.hiveName = hiveName;
this.defaultVal = defaultVal;
validator = null;
caseSensitive = false;
+ this.description = description;
}
ConfVars(String varname, String hiveName, String defaultVal, Validator validator,
- String comment) {
+ String description) {
this.varname = varname;
this.hiveName = hiveName;
this.defaultVal = defaultVal;
this.validator = validator;
caseSensitive = false;
+ this.description = description;
}
ConfVars(String varname, String hiveName, String defaultVal, boolean caseSensitive,
- String comment) {
+ String description) {
this.varname = varname;
this.hiveName = hiveName;
this.defaultVal = defaultVal;
validator = null;
this.caseSensitive = caseSensitive;
+ this.description = description;
}
- ConfVars(String varname, String hiveName, long defaultVal, String comment) {
+ ConfVars(String varname, String hiveName, long defaultVal, String description) {
this.varname = varname;
this.hiveName = hiveName;
this.defaultVal = defaultVal;
validator = null;
caseSensitive = false;
+ this.description = description;
}
ConfVars(String varname, String hiveName, long defaultVal, Validator validator,
- String comment) {
+ String description) {
this.varname = varname;
this.hiveName = hiveName;
this.defaultVal = defaultVal;
this.validator = validator;
caseSensitive = false;
+ this.description = description;
}
- ConfVars(String varname, String hiveName, boolean defaultVal, String comment) {
+ ConfVars(String varname, String hiveName, boolean defaultVal, String description) {
this.varname = varname;
this.hiveName = hiveName;
this.defaultVal = defaultVal;
validator = null;
caseSensitive = false;
+ this.description = description;
}
- ConfVars(String varname, String hiveName, double defaultVal, String comment) {
+ ConfVars(String varname, String hiveName, double defaultVal, String description) {
this.varname = varname;
this.hiveName = hiveName;
this.defaultVal = defaultVal;
validator = null;
caseSensitive = false;
+ this.description = description;
}
- ConfVars(String varname, String hiveName, long defaultVal, TimeUnit unit, String comment) {
+ ConfVars(String varname, String hiveName, long defaultVal, TimeUnit unit, String description) {
this.varname = varname;
this.hiveName = hiveName;
this.defaultVal = new TimeValue(defaultVal, unit);
validator = new Validator.TimeValidator(unit);
caseSensitive = false;
+ this.description = description;
}
ConfVars(String varname, String hiveName, long defaultVal, TimeUnit unit,
- Validator validator, String comment) {
+ Validator validator, String description) {
this.varname = varname;
this.hiveName = hiveName;
this.defaultVal = new TimeValue(defaultVal, unit);
this.validator = validator;
caseSensitive = false;
+ this.description = description;
}
public void validate(String value) throws IllegalArgumentException {
@@ -1008,6 +1027,10 @@ public class MetastoreConf {
return defaultVal;
}
+ public String getDescription() {
+ return description;
+ }
+
@Override
public String toString() {
return varname;
http://git-wip-us.apache.org/repos/asf/hive/blob/47cac2d0/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java
index 70007cd..37891b6 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java
@@ -22,6 +22,8 @@ import com.google.common.collect.Lists;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.HiveMetaException;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.File;
@@ -34,6 +36,8 @@ import java.util.IllegalFormatException;
import java.util.List;
public class HiveSchemaHelper {
+ private static final Logger LOG = LoggerFactory.getLogger(HiveSchemaHelper.class);
+
public static final String DB_DERBY = "derby";
public static final String DB_HIVE = "hive";
public static final String DB_MSSQL = "mssql";
@@ -43,10 +47,10 @@ public class HiveSchemaHelper {
/***
* Get JDBC connection to metastore db
- *
- * @param dbType the type of the meteastore database
* @param userName metastore connection username
* @param password metastore connection password
+ * @param url Metastore URL. If null will be read from config file.
+ * @param driver Driver class. If null will be read from config file.
* @param printInfo print connection parameters
* @param conf hive config object
* @param schema the schema to create the connection for
@@ -59,9 +63,12 @@ public class HiveSchemaHelper {
url = url == null ? getValidConfVar(MetastoreConf.ConfVars.CONNECTURLKEY, conf) : url;
driver = driver == null ? getValidConfVar(MetastoreConf.ConfVars.CONNECTION_DRIVER, conf) : driver;
if (printInfo) {
- System.out.println("Metastore connection URL:\t " + url);
- System.out.println("Metastore Connection Driver :\t " + driver);
- System.out.println("Metastore connection User:\t " + userName);
+ logAndPrintToStdout("Metastore connection URL:\t " + url);
+ logAndPrintToStdout("Metastore Connection Driver :\t " + driver);
+ logAndPrintToStdout("Metastore connection User:\t " + userName);
+ if (MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.HIVE_IN_TEST)) {
+ logAndPrintToStdout("Metastore connection Password:\t " + password);
+ }
}
if ((userName == null) || userName.isEmpty()) {
throw new HiveMetaException("UserName empty ");
@@ -79,6 +86,7 @@ public class HiveSchemaHelper {
} catch (IOException | SQLException e) {
throw new HiveMetaException("Failed to get schema version.", e);
} catch (ClassNotFoundException e) {
+ LOG.error("Unable to find driver class", e);
throw new HiveMetaException("Failed to load driver", e);
}
}
@@ -98,6 +106,11 @@ public class HiveSchemaHelper {
return confVarStr.trim();
}
+ private static void logAndPrintToStdout(String msg) {
+ LOG.info(msg);
+ System.out.println(msg);
+ }
+
public interface NestedScriptParser {
enum CommandType {
@@ -202,13 +215,17 @@ public class HiveSchemaHelper {
private String msUsername;
private String msPassword;
private Configuration conf;
+ // Depending on whether we are using beeline or sqlline the line endings have to be handled
+ // differently.
+ private final boolean usingSqlLine;
public AbstractCommandParser(String dbOpts, String msUsername, String msPassword,
- Configuration conf) {
+ Configuration conf, boolean usingSqlLine) {
setDbOpts(dbOpts);
this.msUsername = msUsername;
this.msPassword = msPassword;
this.conf = conf;
+ this.usingSqlLine = usingSqlLine;
}
@Override
@@ -301,6 +318,7 @@ public class HiveSchemaHelper {
// Now we have a complete statement, process it
// write the line to buffer
sb.append(currentCommand);
+ if (usingSqlLine) sb.append(";");
sb.append(System.getProperty("line.separator"));
}
}
@@ -340,8 +358,8 @@ public class HiveSchemaHelper {
private static final String DERBY_NESTING_TOKEN = "RUN";
public DerbyCommandParser(String dbOpts, String msUsername, String msPassword,
- Configuration conf) {
- super(dbOpts, msUsername, msPassword, conf);
+ Configuration conf, boolean usingSqlLine) {
+ super(dbOpts, msUsername, msPassword, conf, usingSqlLine);
}
@Override
@@ -370,9 +388,9 @@ public class HiveSchemaHelper {
private final NestedScriptParser nestedDbCommandParser;
public HiveCommandParser(String dbOpts, String msUsername, String msPassword,
- Configuration conf, String metaDbType) {
- super(dbOpts, msUsername, msPassword, conf);
- nestedDbCommandParser = getDbCommandParser(metaDbType);
+ Configuration conf, String metaDbType, boolean usingSqlLine) {
+ super(dbOpts, msUsername, msPassword, conf, usingSqlLine);
+ nestedDbCommandParser = getDbCommandParser(metaDbType, usingSqlLine);
}
@Override
@@ -406,8 +424,8 @@ public class HiveSchemaHelper {
private String delimiter = DEFAULT_DELIMITER;
public MySqlCommandParser(String dbOpts, String msUsername, String msPassword,
- Configuration conf) {
- super(dbOpts, msUsername, msPassword, conf);
+ Configuration conf, boolean usingSqlLine) {
+ super(dbOpts, msUsername, msPassword, conf, usingSqlLine);
}
@Override
@@ -472,8 +490,8 @@ public class HiveSchemaHelper {
public static final String POSTGRES_SKIP_STANDARD_STRINGS_DBOPT = "postgres.filter.81";
public PostgresCommandParser(String dbOpts, String msUsername, String msPassword,
- Configuration conf) {
- super(dbOpts, msUsername, msPassword, conf);
+ Configuration conf, boolean usingSqlLine) {
+ super(dbOpts, msUsername, msPassword, conf, usingSqlLine);
}
@Override
@@ -515,8 +533,8 @@ public class HiveSchemaHelper {
private static final String ORACLE_NESTING_TOKEN = "@";
public OracleCommandParser(String dbOpts, String msUsername, String msPassword,
- Configuration conf) {
- super(dbOpts, msUsername, msPassword, conf);
+ Configuration conf, boolean usingSqlLine) {
+ super(dbOpts, msUsername, msPassword, conf, usingSqlLine);
}
@Override
@@ -539,8 +557,8 @@ public class HiveSchemaHelper {
private static final String MSSQL_NESTING_TOKEN = ":r";
public MSSQLCommandParser(String dbOpts, String msUsername, String msPassword,
- Configuration conf) {
- super(dbOpts, msUsername, msPassword, conf);
+ Configuration conf, boolean usingSqlLine) {
+ super(dbOpts, msUsername, msPassword, conf, usingSqlLine);
}
@Override
@@ -558,29 +576,29 @@ public class HiveSchemaHelper {
}
}
- public static NestedScriptParser getDbCommandParser(String dbName) {
- return getDbCommandParser(dbName, null);
+ public static NestedScriptParser getDbCommandParser(String dbName, boolean usingSqlLine) {
+ return getDbCommandParser(dbName, null, usingSqlLine);
}
- public static NestedScriptParser getDbCommandParser(String dbName, String metaDbName) {
- return getDbCommandParser(dbName, null, null, null, null, metaDbName);
+ public static NestedScriptParser getDbCommandParser(String dbName, String metaDbName, boolean usingSqlLine) {
+ return getDbCommandParser(dbName, null, null, null, null, metaDbName, usingSqlLine);
}
public static NestedScriptParser getDbCommandParser(String dbName,
String dbOpts, String msUsername, String msPassword,
- Configuration conf, String metaDbType) {
+ Configuration conf, String metaDbType, boolean usingSqlLine) {
if (dbName.equalsIgnoreCase(DB_DERBY)) {
- return new DerbyCommandParser(dbOpts, msUsername, msPassword, conf);
+ return new DerbyCommandParser(dbOpts, msUsername, msPassword, conf, usingSqlLine);
} else if (dbName.equalsIgnoreCase(DB_HIVE)) {
- return new HiveCommandParser(dbOpts, msUsername, msPassword, conf, metaDbType);
+ return new HiveCommandParser(dbOpts, msUsername, msPassword, conf, metaDbType, usingSqlLine);
} else if (dbName.equalsIgnoreCase(DB_MSSQL)) {
- return new MSSQLCommandParser(dbOpts, msUsername, msPassword, conf);
+ return new MSSQLCommandParser(dbOpts, msUsername, msPassword, conf, usingSqlLine);
} else if (dbName.equalsIgnoreCase(DB_MYSQL)) {
- return new MySqlCommandParser(dbOpts, msUsername, msPassword, conf);
+ return new MySqlCommandParser(dbOpts, msUsername, msPassword, conf, usingSqlLine);
} else if (dbName.equalsIgnoreCase(DB_POSTGRACE)) {
- return new PostgresCommandParser(dbOpts, msUsername, msPassword, conf);
+ return new PostgresCommandParser(dbOpts, msUsername, msPassword, conf, usingSqlLine);
} else if (dbName.equalsIgnoreCase(DB_ORACLE)) {
- return new OracleCommandParser(dbOpts, msUsername, msPassword, conf);
+ return new OracleCommandParser(dbOpts, msUsername, msPassword, conf, usingSqlLine);
} else {
throw new IllegalArgumentException("Unknown dbType " + dbName);
}