You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by vi...@apache.org on 2018/07/19 19:55:17 UTC
[18/51] [partial] hive git commit: HIVE-20188 : Split server-specific
code outside of standalone metastore-common (Alexander Kolbasov reviewed by
Vihang Karajgaonkar)
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java
deleted file mode 100644
index 953c5fd..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java
+++ /dev/null
@@ -1,474 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore.security;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.framework.api.ACLProvider;
-import org.apache.curator.framework.imps.CuratorFrameworkState;
-import org.apache.curator.retry.ExponentialBackoffRetry;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.utils.SecurityUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager.DelegationTokenInformation;
-import org.apache.hadoop.security.token.delegation.MetastoreDelegationTokenSupport;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.ZooDefs;
-import org.apache.zookeeper.ZooDefs.Ids;
-import org.apache.zookeeper.ZooDefs.Perms;
-import org.apache.zookeeper.data.ACL;
-import org.apache.zookeeper.data.Id;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * ZooKeeper token store implementation.
- */
-public class ZooKeeperTokenStore implements DelegationTokenStore {
-
- private static final Logger LOGGER =
- LoggerFactory.getLogger(ZooKeeperTokenStore.class.getName());
-
- protected static final String ZK_SEQ_FORMAT = "%010d";
- private static final String NODE_KEYS = "/keys";
- private static final String NODE_TOKENS = "/tokens";
-
- private String rootNode = "";
- private volatile CuratorFramework zkSession;
- private String zkConnectString;
- private int connectTimeoutMillis;
- private List<ACL> newNodeAcl = Arrays.asList(new ACL(Perms.ALL, Ids.AUTH_IDS));
-
- /**
- * ACLProvider permissions will be used in case parent dirs need to be created
- */
- private final ACLProvider aclDefaultProvider = new ACLProvider() {
-
- @Override
- public List<ACL> getDefaultAcl() {
- return newNodeAcl;
- }
-
- @Override
- public List<ACL> getAclForPath(String path) {
- return getDefaultAcl();
- }
- };
-
-
- private final String WHEN_ZK_DSTORE_MSG = "when zookeeper based delegation token storage is enabled"
- + "(hive.cluster.delegation.token.store.class=" + ZooKeeperTokenStore.class.getName() + ")";
-
- private Configuration conf;
-
- private HadoopThriftAuthBridge.Server.ServerMode serverMode;
-
- /**
- * Default constructor for dynamic instantiation w/ Configurable
- * (ReflectionUtils does not support Configuration constructor injection).
- */
- protected ZooKeeperTokenStore() {
- }
-
- private CuratorFramework getSession() {
- if (zkSession == null || zkSession.getState() == CuratorFrameworkState.STOPPED) {
- synchronized (this) {
- if (zkSession == null || zkSession.getState() == CuratorFrameworkState.STOPPED) {
- zkSession =
- CuratorFrameworkFactory.builder().connectString(zkConnectString)
- .connectionTimeoutMs(connectTimeoutMillis).aclProvider(aclDefaultProvider)
- .retryPolicy(new ExponentialBackoffRetry(1000, 3)).build();
- zkSession.start();
- }
- }
- }
- return zkSession;
- }
-
- private void setupJAASConfig(Configuration conf) throws IOException {
- if (!UserGroupInformation.getLoginUser().isFromKeytab()) {
- // The process has not logged in using keytab
- // this should be a test mode, can't use keytab to authenticate
- // with zookeeper.
- LOGGER.warn("Login is not from keytab");
- return;
- }
-
- String principal;
- String keytab;
- switch (serverMode) {
- case METASTORE:
- principal = getNonEmptyConfVar(conf, "hive.metastore.kerberos.principal");
- keytab = getNonEmptyConfVar(conf, "hive.metastore.kerberos.keytab.file");
- break;
- case HIVESERVER2:
- principal = getNonEmptyConfVar(conf, "hive.server2.authentication.kerberos.principal");
- keytab = getNonEmptyConfVar(conf, "hive.server2.authentication.kerberos.keytab");
- break;
- default:
- throw new AssertionError("Unexpected server mode " + serverMode);
- }
- SecurityUtils.setZookeeperClientKerberosJaasConfig(principal, keytab);
- }
-
- private String getNonEmptyConfVar(Configuration conf, String param) throws IOException {
- String val = conf.get(param);
- if (val == null || val.trim().isEmpty()) {
- throw new IOException("Configuration parameter " + param + " should be set, "
- + WHEN_ZK_DSTORE_MSG);
- }
- return val;
- }
-
- /**
- * Create a path if it does not already exist ("mkdir -p")
- * @param path string with '/' separator
- * @param acl list of ACL entries
- * @throws TokenStoreException
- */
- public void ensurePath(String path, List<ACL> acl)
- throws TokenStoreException {
- try {
- CuratorFramework zk = getSession();
- String node = zk.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT)
- .withACL(acl).forPath(path);
- LOGGER.info("Created path: {} ", node);
- } catch (KeeperException.NodeExistsException e) {
- // node already exists
- } catch (Exception e) {
- throw new TokenStoreException("Error creating path " + path, e);
- }
- }
-
- /**
- * Parse ACL permission string, from ZooKeeperMain private method
- * @param permString
- * @return
- */
- public static int getPermFromString(String permString) {
- int perm = 0;
- for (int i = 0; i < permString.length(); i++) {
- switch (permString.charAt(i)) {
- case 'r':
- perm |= ZooDefs.Perms.READ;
- break;
- case 'w':
- perm |= ZooDefs.Perms.WRITE;
- break;
- case 'c':
- perm |= ZooDefs.Perms.CREATE;
- break;
- case 'd':
- perm |= ZooDefs.Perms.DELETE;
- break;
- case 'a':
- perm |= ZooDefs.Perms.ADMIN;
- break;
- default:
- LOGGER.error("Unknown perm type: " + permString.charAt(i));
- }
- }
- return perm;
- }
-
- /**
- * Parse comma separated list of ACL entries to secure generated nodes, e.g.
- * <code>sasl:hive/host1@MY.DOMAIN:cdrwa,sasl:hive/host2@MY.DOMAIN:cdrwa</code>
- * @param aclString
- * @return ACL list
- */
- public static List<ACL> parseACLs(String aclString) {
- String[] aclComps = StringUtils.splitByWholeSeparator(aclString, ",");
- List<ACL> acl = new ArrayList<ACL>(aclComps.length);
- for (String a : aclComps) {
- if (StringUtils.isBlank(a)) {
- continue;
- }
- a = a.trim();
- // from ZooKeeperMain private method
- int firstColon = a.indexOf(':');
- int lastColon = a.lastIndexOf(':');
- if (firstColon == -1 || lastColon == -1 || firstColon == lastColon) {
- LOGGER.error(a + " does not have the form scheme:id:perm");
- continue;
- }
- ACL newAcl = new ACL();
- newAcl.setId(new Id(a.substring(0, firstColon), a.substring(
- firstColon + 1, lastColon)));
- newAcl.setPerms(getPermFromString(a.substring(lastColon + 1)));
- acl.add(newAcl);
- }
- return acl;
- }
-
- private void initClientAndPaths() {
- if (this.zkSession != null) {
- this.zkSession.close();
- }
- try {
- ensurePath(rootNode + NODE_KEYS, newNodeAcl);
- ensurePath(rootNode + NODE_TOKENS, newNodeAcl);
- } catch (TokenStoreException e) {
- throw e;
- }
- }
-
- @Override
- public void setConf(Configuration conf) {
- if (conf == null) {
- throw new IllegalArgumentException("conf is null");
- }
- this.conf = conf;
- }
-
- @Override
- public Configuration getConf() {
- return null; // not required
- }
-
- private Map<Integer, byte[]> getAllKeys() throws KeeperException, InterruptedException {
-
- String masterKeyNode = rootNode + NODE_KEYS;
-
- // get children of key node
- List<String> nodes = zkGetChildren(masterKeyNode);
-
- // read each child node, add to results
- Map<Integer, byte[]> result = new HashMap<Integer, byte[]>();
- for (String node : nodes) {
- String nodePath = masterKeyNode + "/" + node;
- byte[] data = zkGetData(nodePath);
- if (data != null) {
- result.put(getSeq(node), data);
- }
- }
- return result;
- }
-
- private List<String> zkGetChildren(String path) {
- CuratorFramework zk = getSession();
- try {
- return zk.getChildren().forPath(path);
- } catch (Exception e) {
- throw new TokenStoreException("Error getting children for " + path, e);
- }
- }
-
- private byte[] zkGetData(String nodePath) {
- CuratorFramework zk = getSession();
- try {
- return zk.getData().forPath(nodePath);
- } catch (KeeperException.NoNodeException ex) {
- return null;
- } catch (Exception e) {
- throw new TokenStoreException("Error reading " + nodePath, e);
- }
- }
-
- private int getSeq(String path) {
- String[] pathComps = path.split("/");
- return Integer.parseInt(pathComps[pathComps.length-1]);
- }
-
- @Override
- public int addMasterKey(String s) {
- String keysPath = rootNode + NODE_KEYS + "/";
- CuratorFramework zk = getSession();
- String newNode;
- try {
- newNode = zk.create().withMode(CreateMode.PERSISTENT_SEQUENTIAL).withACL(newNodeAcl)
- .forPath(keysPath, s.getBytes());
- } catch (Exception e) {
- throw new TokenStoreException("Error creating new node with path " + keysPath, e);
- }
- LOGGER.info("Added key {}", newNode);
- return getSeq(newNode);
- }
-
- @Override
- public void updateMasterKey(int keySeq, String s) {
- CuratorFramework zk = getSession();
- String keyPath = rootNode + NODE_KEYS + "/" + String.format(ZK_SEQ_FORMAT, keySeq);
- try {
- zk.setData().forPath(keyPath, s.getBytes());
- } catch (Exception e) {
- throw new TokenStoreException("Error setting data in " + keyPath, e);
- }
- }
-
- @Override
- public boolean removeMasterKey(int keySeq) {
- String keyPath = rootNode + NODE_KEYS + "/" + String.format(ZK_SEQ_FORMAT, keySeq);
- zkDelete(keyPath);
- return true;
- }
-
- private void zkDelete(String path) {
- CuratorFramework zk = getSession();
- try {
- zk.delete().forPath(path);
- } catch (KeeperException.NoNodeException ex) {
- // already deleted
- } catch (Exception e) {
- throw new TokenStoreException("Error deleting " + path, e);
- }
- }
-
- @Override
- public String[] getMasterKeys() {
- try {
- Map<Integer, byte[]> allKeys = getAllKeys();
- String[] result = new String[allKeys.size()];
- int resultIdx = 0;
- for (byte[] keyBytes : allKeys.values()) {
- result[resultIdx++] = new String(keyBytes);
- }
- return result;
- } catch (KeeperException ex) {
- throw new TokenStoreException(ex);
- } catch (InterruptedException ex) {
- throw new TokenStoreException(ex);
- }
- }
-
-
- private String getTokenPath(DelegationTokenIdentifier tokenIdentifier) {
- try {
- return rootNode + NODE_TOKENS + "/"
- + TokenStoreDelegationTokenSecretManager.encodeWritable(tokenIdentifier);
- } catch (IOException ex) {
- throw new TokenStoreException("Failed to encode token identifier", ex);
- }
- }
-
- @Override
- public boolean addToken(DelegationTokenIdentifier tokenIdentifier,
- DelegationTokenInformation token) {
- byte[] tokenBytes = MetastoreDelegationTokenSupport.encodeDelegationTokenInformation(token);
- String tokenPath = getTokenPath(tokenIdentifier);
- CuratorFramework zk = getSession();
- String newNode;
- try {
- newNode = zk.create().withMode(CreateMode.PERSISTENT).withACL(newNodeAcl)
- .forPath(tokenPath, tokenBytes);
- } catch (Exception e) {
- throw new TokenStoreException("Error creating new node with path " + tokenPath, e);
- }
-
- LOGGER.info("Added token: {}", newNode);
- return true;
- }
-
- @Override
- public boolean removeToken(DelegationTokenIdentifier tokenIdentifier) {
- String tokenPath = getTokenPath(tokenIdentifier);
- zkDelete(tokenPath);
- return true;
- }
-
- @Override
- public DelegationTokenInformation getToken(DelegationTokenIdentifier tokenIdentifier) {
- byte[] tokenBytes = zkGetData(getTokenPath(tokenIdentifier));
- if(tokenBytes == null) {
- // The token is already removed.
- return null;
- }
- try {
- return MetastoreDelegationTokenSupport.decodeDelegationTokenInformation(tokenBytes);
- } catch (Exception ex) {
- throw new TokenStoreException("Failed to decode token", ex);
- }
- }
-
- @Override
- public List<DelegationTokenIdentifier> getAllDelegationTokenIdentifiers() {
- String containerNode = rootNode + NODE_TOKENS;
- final List<String> nodes = zkGetChildren(containerNode);
- List<DelegationTokenIdentifier> result = new java.util.ArrayList<DelegationTokenIdentifier>(
- nodes.size());
- for (String node : nodes) {
- DelegationTokenIdentifier id = new DelegationTokenIdentifier();
- try {
- TokenStoreDelegationTokenSecretManager.decodeWritable(id, node);
- result.add(id);
- } catch (Exception e) {
- LOGGER.warn("Failed to decode token '{}'", node);
- }
- }
- return result;
- }
-
- @Override
- public void close() throws IOException {
- if (this.zkSession != null) {
- this.zkSession.close();
- }
- }
-
- @Override
- public void init(Object hmsHandler, HadoopThriftAuthBridge.Server.ServerMode sMode) {
- this.serverMode = sMode;
- zkConnectString =
- conf.get(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR, null);
- if (zkConnectString == null || zkConnectString.trim().isEmpty()) {
- // try alternate config param
- zkConnectString =
- conf.get(
- MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR_ALTERNATE,
- null);
- if (zkConnectString == null || zkConnectString.trim().isEmpty()) {
- throw new IllegalArgumentException("Zookeeper connect string has to be specified through "
- + "either " + MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR
- + " or "
- + MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR_ALTERNATE
- + WHEN_ZK_DSTORE_MSG);
- }
- }
- connectTimeoutMillis =
- conf.getInt(
- MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_CONNECT_TIMEOUTMILLIS,
- CuratorFrameworkFactory.builder().getConnectionTimeoutMs());
- String aclStr = conf.get(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_ACL, null);
- if (StringUtils.isNotBlank(aclStr)) {
- this.newNodeAcl = parseACLs(aclStr);
- }
- rootNode =
- conf.get(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_ZNODE,
- MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_ZNODE_DEFAULT) + serverMode;
-
- try {
- // Install the JAAS Configuration for the runtime
- setupJAASConfig(conf);
- } catch (IOException e) {
- throw new TokenStoreException("Error setting up JAAS configuration for zookeeper client "
- + e.getMessage(), e);
- }
- initClientAndPaths();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java
deleted file mode 100644
index a50c0a3..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java
+++ /dev/null
@@ -1,490 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore.tools;
-
-import java.net.URI;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.metastore.ObjectStore;
-
-/**
- * This class provides Hive admins a tool to
- * - execute JDOQL against the metastore using DataNucleus
- * - perform HA name node upgrade
- */
-
-public class HiveMetaTool {
-
- private static final Logger LOG = LoggerFactory.getLogger(HiveMetaTool.class.getName());
- private final Options cmdLineOptions = new Options();
- private ObjectStore objStore;
- private boolean isObjStoreInitialized;
-
- public HiveMetaTool() {
- this.isObjStoreInitialized = false;
- }
-
- @SuppressWarnings("static-access")
- private void init() {
-
- System.out.println("Initializing HiveMetaTool..");
-
- Option help = new Option("help", "print this message");
- Option listFSRoot = new Option("listFSRoot", "print the current FS root locations");
- Option executeJDOQL =
- OptionBuilder.withArgName("query-string")
- .hasArgs()
- .withDescription("execute the given JDOQL query")
- .create("executeJDOQL");
-
- /* Ideally we want to specify the different arguments to updateLocation as separate argNames.
- * However if we did that, HelpFormatter swallows all but the last argument. Note that this is
- * a know issue with the HelpFormatter class that has not been fixed. We specify all arguments
- * with a single argName to workaround this HelpFormatter bug.
- */
- Option updateFSRootLoc =
- OptionBuilder
- .withArgName("new-loc> " + "<old-loc")
- .hasArgs(2)
- .withDescription(
- "Update FS root location in the metastore to new location.Both new-loc and " +
- "old-loc should be valid URIs with valid host names and schemes." +
- "When run with the dryRun option changes are displayed but are not " +
- "persisted. When run with the serdepropKey/tablePropKey option " +
- "updateLocation looks for the serde-prop-key/table-prop-key that is " +
- "specified and updates its value if found.")
- .create("updateLocation");
- Option dryRun = new Option("dryRun" , "Perform a dry run of updateLocation changes.When " +
- "run with the dryRun option updateLocation changes are displayed but not persisted. " +
- "dryRun is valid only with the updateLocation option.");
- Option serdePropKey =
- OptionBuilder.withArgName("serde-prop-key")
- .hasArgs()
- .withValueSeparator()
- .withDescription("Specify the key for serde property to be updated. serdePropKey option " +
- "is valid only with updateLocation option.")
- .create("serdePropKey");
- Option tablePropKey =
- OptionBuilder.withArgName("table-prop-key")
- .hasArg()
- .withValueSeparator()
- .withDescription("Specify the key for table property to be updated. tablePropKey option " +
- "is valid only with updateLocation option.")
- .create("tablePropKey");
- Option prepareAcidUpgrade =
- OptionBuilder.withArgName("find-compactions")
- .hasOptionalArg() //directory to output results to
- .withDescription("Generates a set Compaction commands to run to prepare for Hive 2.x" +
- " to 3.0 upgrade")
- .create("prepareAcidUpgrade");
-
- cmdLineOptions.addOption(help);
- cmdLineOptions.addOption(listFSRoot);
- cmdLineOptions.addOption(executeJDOQL);
- cmdLineOptions.addOption(updateFSRootLoc);
- cmdLineOptions.addOption(dryRun);
- cmdLineOptions.addOption(serdePropKey);
- cmdLineOptions.addOption(tablePropKey);
- cmdLineOptions.addOption(prepareAcidUpgrade);
- }
-
- private void initObjectStore(Configuration conf) {
- if (!isObjStoreInitialized) {
- objStore = new ObjectStore();
- objStore.setConf(conf);
- isObjStoreInitialized = true;
- }
- }
-
- private void shutdownObjectStore() {
- if (isObjStoreInitialized) {
- objStore.shutdown();
- isObjStoreInitialized = false;
- }
- }
-
- private void listFSRoot() {
- Configuration conf = MetastoreConf.newMetastoreConf();
- initObjectStore(conf);
-
- Set<String> hdfsRoots = objStore.listFSRoots();
- if (hdfsRoots != null) {
- System.out.println("Listing FS Roots..");
- for (String s : hdfsRoots) {
- System.out.println(s);
- }
- } else {
- System.err.println("Encountered error during listFSRoot - " +
- "commit of JDO transaction failed");
- }
- }
-
- private void executeJDOQLSelect(String query) {
- Configuration conf = MetastoreConf.newMetastoreConf();
- initObjectStore(conf);
-
- System.out.println("Executing query: " + query);
- try (ObjectStore.QueryWrapper queryWrapper = new ObjectStore.QueryWrapper()) {
- Collection<?> result = objStore.executeJDOQLSelect(query, queryWrapper);
- if (result != null) {
- Iterator<?> iter = result.iterator();
- while (iter.hasNext()) {
- Object o = iter.next();
- System.out.println(o.toString());
- }
- } else {
- System.err.println("Encountered error during executeJDOQLSelect -" +
- "commit of JDO transaction failed.");
- }
- }
- }
-
- private void executeJDOQLUpdate(String query) {
- Configuration conf = MetastoreConf.newMetastoreConf();
- initObjectStore(conf);
-
- System.out.println("Executing query: " + query);
- long numUpdated = objStore.executeJDOQLUpdate(query);
- if (numUpdated >= 0) {
- System.out.println("Number of records updated: " + numUpdated);
- } else {
- System.err.println("Encountered error during executeJDOQL -" +
- "commit of JDO transaction failed.");
- }
- }
-
- private int printUpdateLocations(Map<String, String> updateLocations) {
- int count = 0;
- for (String key: updateLocations.keySet()) {
- String value = updateLocations.get(key);
- System.out.println("old location: " + key + " new location: " + value);
- count++;
- }
- return count;
- }
-
- private void printTblURIUpdateSummary(ObjectStore.UpdateMStorageDescriptorTblURIRetVal retVal,
- boolean isDryRun) {
- String tblName = "SDS";
- String fieldName = "LOCATION";
-
- if (retVal == null) {
- System.err.println("Encountered error while executing updateMStorageDescriptorTblURI - " +
- "commit of JDO transaction failed. Failed to update FSRoot locations in " +
- fieldName + "field in " + tblName + " table.");
- } else {
- Map<String, String> updateLocations = retVal.getUpdateLocations();
- if (isDryRun) {
- System.out.println("Dry Run of updateLocation on table " + tblName + "..");
- } else {
- System.out.println("Successfully updated the following locations..");
- }
- int count = printUpdateLocations(updateLocations);
- if (isDryRun) {
- System.out.println("Found " + count + " records in " + tblName + " table to update");
- } else {
- System.out.println("Updated " + count + " records in " + tblName + " table");
- }
- List<String> badRecords = retVal.getBadRecords();
- if (badRecords.size() > 0) {
- System.err.println("Warning: Found records with bad " + fieldName + " in " +
- tblName + " table.. ");
- for (String badRecord:badRecords) {
- System.err.println("bad location URI: " + badRecord);
- }
- }
- int numNullRecords = retVal.getNumNullRecords();
- if (numNullRecords != 0) {
- LOG.debug("Number of NULL location URI: " + numNullRecords +
- ". This can happen for View or Index.");
- }
- }
- }
-
- private void printDatabaseURIUpdateSummary(ObjectStore.UpdateMDatabaseURIRetVal retVal,
- boolean isDryRun) {
- String tblName = "DBS";
- String fieldName = "LOCATION_URI";
-
- if (retVal == null) {
- System.err.println("Encountered error while executing updateMDatabaseURI - " +
- "commit of JDO transaction failed. Failed to update FSRoot locations in " +
- fieldName + "field in " + tblName + " table.");
- } else {
- Map<String, String> updateLocations = retVal.getUpdateLocations();
- if (isDryRun) {
- System.out.println("Dry Run of updateLocation on table " + tblName + "..");
- } else {
- System.out.println("Successfully updated the following locations..");
- }
- int count = printUpdateLocations(updateLocations);
- if (isDryRun) {
- System.out.println("Found " + count + " records in " + tblName + " table to update");
- } else {
- System.out.println("Updated " + count + " records in " + tblName + " table");
- }
- List<String> badRecords = retVal.getBadRecords();
- if (badRecords.size() > 0) {
- System.err.println("Warning: Found records with bad " + fieldName + " in " +
- tblName + " table.. ");
- for (String badRecord:badRecords) {
- System.err.println("bad location URI: " + badRecord);
- }
- }
- }
- }
-
- private void printPropURIUpdateSummary(ObjectStore.UpdatePropURIRetVal retVal, String
- tablePropKey, boolean isDryRun, String tblName, String methodName) {
- if (retVal == null) {
- System.err.println("Encountered error while executing " + methodName + " - " +
- "commit of JDO transaction failed. Failed to update FSRoot locations in " +
- "value field corresponding to" + tablePropKey + " in " + tblName + " table.");
- } else {
- Map<String, String> updateLocations = retVal.getUpdateLocations();
- if (isDryRun) {
- System.out.println("Dry Run of updateLocation on table " + tblName + "..");
- } else {
- System.out.println("Successfully updated the following locations..");
- }
- int count = printUpdateLocations(updateLocations);
- if (isDryRun) {
- System.out.println("Found " + count + " records in " + tblName + " table to update");
- } else {
- System.out.println("Updated " + count + " records in " + tblName + " table");
- }
- List<String> badRecords = retVal.getBadRecords();
- if (badRecords.size() > 0) {
- System.err.println("Warning: Found records with bad " + tablePropKey + " key in " +
- tblName + " table.. ");
- for (String badRecord:badRecords) {
- System.err.println("bad location URI: " + badRecord);
- }
- }
- }
- }
-
- private void printSerdePropURIUpdateSummary(ObjectStore.UpdateSerdeURIRetVal retVal,
- String serdePropKey, boolean isDryRun) {
- String tblName = "SERDE_PARAMS";
-
- if (retVal == null) {
- System.err.println("Encountered error while executing updateSerdeURI - " +
- "commit of JDO transaction failed. Failed to update FSRoot locations in " +
- "value field corresponding to " + serdePropKey + " in " + tblName + " table.");
- } else {
- Map<String, String> updateLocations = retVal.getUpdateLocations();
- if (isDryRun) {
- System.out.println("Dry Run of updateLocation on table " + tblName + "..");
- } else {
- System.out.println("Successfully updated the following locations..");
- }
- int count = printUpdateLocations(updateLocations);
- if (isDryRun) {
- System.out.println("Found " + count + " records in " + tblName + " table to update");
- } else {
- System.out.println("Updated " + count + " records in " + tblName + " table");
- }
- List<String> badRecords = retVal.getBadRecords();
- if (badRecords.size() > 0) {
- System.err.println("Warning: Found records with bad " + serdePropKey + " key in " +
- tblName + " table.. ");
- for (String badRecord:badRecords) {
- System.err.println("bad location URI: " + badRecord);
- }
- }
- }
- }
-
- public void updateFSRootLocation(URI oldURI, URI newURI, String serdePropKey, String
- tablePropKey, boolean isDryRun) {
- Configuration conf = MetastoreConf.newMetastoreConf();
- initObjectStore(conf);
-
- System.out.println("Looking for LOCATION_URI field in DBS table to update..");
- ObjectStore.UpdateMDatabaseURIRetVal updateMDBURIRetVal = objStore.updateMDatabaseURI(oldURI,
- newURI, isDryRun);
- printDatabaseURIUpdateSummary(updateMDBURIRetVal, isDryRun);
-
- System.out.println("Looking for LOCATION field in SDS table to update..");
- ObjectStore.UpdateMStorageDescriptorTblURIRetVal updateTblURIRetVal =
- objStore.updateMStorageDescriptorTblURI(oldURI, newURI, isDryRun);
- printTblURIUpdateSummary(updateTblURIRetVal, isDryRun);
-
- if (tablePropKey != null) {
- System.out.println("Looking for value of " + tablePropKey + " key in TABLE_PARAMS table " +
- "to update..");
- ObjectStore.UpdatePropURIRetVal updateTblPropURIRetVal =
- objStore.updateTblPropURI(oldURI, newURI,
- tablePropKey, isDryRun);
- printPropURIUpdateSummary(updateTblPropURIRetVal, tablePropKey, isDryRun, "TABLE_PARAMS",
- "updateTblPropURI");
-
- System.out.println("Looking for value of " + tablePropKey + " key in SD_PARAMS table " +
- "to update..");
- ObjectStore.UpdatePropURIRetVal updatePropURIRetVal = objStore
- .updateMStorageDescriptorTblPropURI(oldURI, newURI, tablePropKey, isDryRun);
- printPropURIUpdateSummary(updatePropURIRetVal, tablePropKey, isDryRun, "SD_PARAMS",
- "updateMStorageDescriptorTblPropURI");
- }
-
- if (serdePropKey != null) {
- System.out.println("Looking for value of " + serdePropKey + " key in SERDE_PARAMS table " +
- "to update..");
- ObjectStore.UpdateSerdeURIRetVal updateSerdeURIretVal = objStore.updateSerdeURI(oldURI,
- newURI, serdePropKey, isDryRun);
- printSerdePropURIUpdateSummary(updateSerdeURIretVal, serdePropKey, isDryRun);
- }
- }
- private static void printAndExit(HiveMetaTool metaTool) {
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("metatool", metaTool.cmdLineOptions);
- System.exit(1);
- }
-
- public static void main(String[] args) {
- HiveMetaTool metaTool = new HiveMetaTool();
- metaTool.init();
- CommandLineParser parser = new GnuParser();
- CommandLine line = null;
-
- try {
- try {
- line = parser.parse(metaTool.cmdLineOptions, args);
- } catch (ParseException e) {
- System.err.println("HiveMetaTool:Parsing failed. Reason: " + e.getLocalizedMessage());
- printAndExit(metaTool);
- }
-
- if (line.hasOption("help")) {
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("metatool", metaTool.cmdLineOptions);
- } else if (line.hasOption("listFSRoot")) {
- if (line.hasOption("dryRun")) {
- System.err.println("HiveMetaTool: dryRun is not valid with listFSRoot");
- printAndExit(metaTool);
- } else if (line.hasOption("serdePropKey")) {
- System.err.println("HiveMetaTool: serdePropKey is not valid with listFSRoot");
- printAndExit(metaTool);
- } else if (line.hasOption("tablePropKey")) {
- System.err.println("HiveMetaTool: tablePropKey is not valid with listFSRoot");
- printAndExit(metaTool);
- }
- metaTool.listFSRoot();
- } else if (line.hasOption("executeJDOQL")) {
- String query = line.getOptionValue("executeJDOQL");
- if (line.hasOption("dryRun")) {
- System.err.println("HiveMetaTool: dryRun is not valid with executeJDOQL");
- printAndExit(metaTool);
- } else if (line.hasOption("serdePropKey")) {
- System.err.println("HiveMetaTool: serdePropKey is not valid with executeJDOQL");
- printAndExit(metaTool);
- } else if (line.hasOption("tablePropKey")) {
- System.err.println("HiveMetaTool: tablePropKey is not valid with executeJDOQL");
- printAndExit(metaTool);
- }
- if (query.toLowerCase().trim().startsWith("select")) {
- metaTool.executeJDOQLSelect(query);
- } else if (query.toLowerCase().trim().startsWith("update")) {
- metaTool.executeJDOQLUpdate(query);
- } else {
- System.err.println("HiveMetaTool:Unsupported statement type");
- printAndExit(metaTool);
- }
- } else if (line.hasOption("updateLocation")) {
- String[] loc = line.getOptionValues("updateLocation");
- boolean isDryRun = false;
- String serdepropKey = null;
- String tablePropKey = null;
-
- if (loc.length != 2 && loc.length != 3) {
- System.err.println("HiveMetaTool:updateLocation takes in 2 required and 1 " +
- "optional arguments but " +
- "was passed " + loc.length + " arguments");
- printAndExit(metaTool);
- }
-
- Path newPath = new Path(loc[0]);
- Path oldPath = new Path(loc[1]);
-
- URI oldURI = oldPath.toUri();
- URI newURI = newPath.toUri();
-
- if (line.hasOption("dryRun")) {
- isDryRun = true;
- }
-
- if (line.hasOption("serdePropKey")) {
- serdepropKey = line.getOptionValue("serdePropKey");
- }
-
- if (line.hasOption("tablePropKey")) {
- tablePropKey = line.getOptionValue("tablePropKey");
- }
-
- /*
- * validate input - Both new and old URI should contain valid host names and valid schemes.
- * port is optional in both the URIs since HDFS HA NN URI doesn't have a port.
- */
- if (oldURI.getHost() == null || newURI.getHost() == null) {
- System.err.println("HiveMetaTool:A valid host is required in both old-loc and new-loc");
- } else if (oldURI.getScheme() == null || newURI.getScheme() == null) {
- System.err.println("HiveMetaTool:A valid scheme is required in both old-loc and new-loc");
- } else {
- metaTool.updateFSRootLocation(oldURI, newURI, serdepropKey, tablePropKey, isDryRun);
- }
- } else {
- if (line.hasOption("dryRun")) {
- System.err.println("HiveMetaTool: dryRun is not a valid standalone option");
- } else if (line.hasOption("serdePropKey")) {
- System.err.println("HiveMetaTool: serdePropKey is not a valid standalone option");
- } else if (line.hasOption("tablePropKey")) {
- System.err.println("HiveMetaTool: tablePropKey is not a valid standalone option");
- printAndExit(metaTool);
- } else {
- System.err.print("HiveMetaTool:Parsing failed. Reason: Invalid arguments: " );
- for (String s : line.getArgs()) {
- System.err.print(s + " ");
- }
- System.err.println();
- }
- printAndExit(metaTool);
- }
- } finally {
- metaTool.shutdownObjectStore();
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java
deleted file mode 100644
index 2da07a5..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java
+++ /dev/null
@@ -1,673 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.tools;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Lists;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.HiveMetaException;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.util.IllegalFormatException;
-import java.util.List;
-
-public class HiveSchemaHelper {
- private static final Logger LOG = LoggerFactory.getLogger(HiveSchemaHelper.class);
-
- public static final String DB_DERBY = "derby";
- public static final String DB_HIVE = "hive";
- public static final String DB_MSSQL = "mssql";
- public static final String DB_MYSQL = "mysql";
- public static final String DB_POSTGRACE = "postgres";
- public static final String DB_ORACLE = "oracle";
- public static final String EMBEDDED_HS2_URL =
- "jdbc:hive2://?hive.conf.restricted.list=;hive.security.authorization.sqlstd.confwhitelist=.*;"
- + "hive.security.authorization.sqlstd.confwhitelist.append=.*;hive.security.authorization.enabled=false;"
- + "hive.metastore.uris=;hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory;"
- + "hive.support.concurrency=false;hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager;"
- + "hive.metastore.rawstore.impl=org.apache.hadoop.hive.metastore.ObjectStore";
- public static final String HIVE_JDBC_DRIVER = "org.apache.hive.jdbc.HiveDriver";
-
- /***
- * Get JDBC connection to metastore db
- * @param userName metastore connection username
- * @param password metastore connection password
- * @param url Metastore URL. If null will be read from config file.
- * @param driver Driver class. If null will be read from config file.
- * @param printInfo print connection parameters
- * @param conf hive config object
- * @param schema the schema to create the connection for
- * @return metastore connection object
- * @throws org.apache.hadoop.hive.metastore.HiveMetaException
- */
- public static Connection getConnectionToMetastore(String userName, String password, String url,
- String driver, boolean printInfo, Configuration conf, String schema) throws HiveMetaException {
- try {
- url = url == null ? getValidConfVar(MetastoreConf.ConfVars.CONNECT_URL_KEY, conf) : url;
- driver = driver == null ? getValidConfVar(MetastoreConf.ConfVars.CONNECTION_DRIVER, conf) : driver;
- if (printInfo) {
- logAndPrintToStdout("Metastore connection URL:\t " + url);
- logAndPrintToStdout("Metastore Connection Driver :\t " + driver);
- logAndPrintToStdout("Metastore connection User:\t " + userName);
- if (MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.HIVE_IN_TEST)) {
- logAndPrintToStdout("Metastore connection Password:\t " + password);
- }
- }
- if ((userName == null) || userName.isEmpty()) {
- throw new HiveMetaException("UserName empty ");
- }
-
- // load required JDBC driver
- Class.forName(driver);
-
- // Connect using the JDBC URL and user/pass from conf
- Connection conn = DriverManager.getConnection(url, userName, password);
- if (schema != null) {
- conn.setSchema(schema);
- }
- return conn;
- } catch (IOException | SQLException e) {
- throw new HiveMetaException("Failed to get schema version.", e);
- } catch (ClassNotFoundException e) {
- LOG.error("Unable to find driver class", e);
- throw new HiveMetaException("Failed to load driver", e);
- }
- }
-
- public static Connection getConnectionToMetastore(MetaStoreConnectionInfo info, String schema)
- throws HiveMetaException {
- return getConnectionToMetastore(info.getUsername(), info.getPassword(), info.getUrl(), info.getDriver(),
- info.getPrintInfo(), info.getConf(), schema);
- }
-
- public static String getValidConfVar(MetastoreConf.ConfVars confVar, Configuration conf)
- throws IOException {
- String confVarStr = MetastoreConf.getAsString(conf, confVar);
- if (confVarStr == null || confVarStr.isEmpty()) {
- throw new IOException("Empty " + confVar.getVarname());
- }
- return confVarStr.trim();
- }
-
- private static void logAndPrintToStdout(String msg) {
- LOG.info(msg);
- System.out.println(msg);
- }
-
- public interface NestedScriptParser {
-
- enum CommandType {
- PARTIAL_STATEMENT,
- TERMINATED_STATEMENT,
- COMMENT
- }
-
- String DEFAULT_DELIMITER = ";";
- String DEFAULT_QUOTE = "\"";
-
- /**
- * Find the type of given command
- *
- * @param dbCommand
- * @return
- */
- boolean isPartialCommand(String dbCommand) throws IllegalArgumentException;
-
- /**
- * Parse the DB specific nesting format and extract the inner script name if any
- *
- * @param dbCommand command from parent script
- * @return
- * @throws IllegalFormatException
- */
- String getScriptName(String dbCommand) throws IllegalArgumentException;
-
- /**
- * Find if the given command is a nested script execution
- *
- * @param dbCommand
- * @return
- */
- boolean isNestedScript(String dbCommand);
-
- /**
- * Find if the given command should not be passed to DB
- *
- * @param dbCommand
- * @return
- */
- boolean isNonExecCommand(String dbCommand);
-
- /**
- * Get the SQL statement delimiter
- *
- * @return
- */
- String getDelimiter();
-
- /**
- * Get the SQL indentifier quotation character
- *
- * @return
- */
- String getQuoteCharacter();
-
- /**
- * Clear any client specific tags
- *
- * @return
- */
- String cleanseCommand(String dbCommand);
-
- /**
- * Does the DB required table/column names quoted
- *
- * @return
- */
- boolean needsQuotedIdentifier();
-
- /**
- * Flatten the nested upgrade script into a buffer
- *
- * @param scriptDir upgrade script directory
- * @param scriptFile upgrade script file
- * @return string of sql commands
- */
- String buildCommand(String scriptDir, String scriptFile)
- throws IllegalFormatException, IOException;
-
- /**
- * Flatten the nested upgrade script into a buffer
- *
- * @param scriptDir upgrade script directory
- * @param scriptFile upgrade script file
- * @param fixQuotes whether to replace quote characters
- * @return string of sql commands
- */
- String buildCommand(String scriptDir, String scriptFile, boolean fixQuotes)
- throws IllegalFormatException, IOException;
- }
-
- /***
- * Base implementation of NestedScriptParser
- * abstractCommandParser.
- *
- */
- private static abstract class AbstractCommandParser implements NestedScriptParser {
- private List<String> dbOpts;
- private String msUsername;
- private String msPassword;
- private Configuration conf;
- // Depending on whether we are using beeline or sqlline the line endings have to be handled
- // differently.
- private final boolean usingSqlLine;
-
- public AbstractCommandParser(String dbOpts, String msUsername, String msPassword,
- Configuration conf, boolean usingSqlLine) {
- setDbOpts(dbOpts);
- this.msUsername = msUsername;
- this.msPassword = msPassword;
- this.conf = conf;
- this.usingSqlLine = usingSqlLine;
- }
-
- @Override
- public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{
- if (dbCommand == null || dbCommand.isEmpty()) {
- throw new IllegalArgumentException("invalid command line " + dbCommand);
- }
- dbCommand = dbCommand.trim();
- if (dbCommand.endsWith(getDelimiter()) || isNonExecCommand(dbCommand)) {
- return false;
- } else {
- return true;
- }
- }
-
- @Override
- public boolean isNonExecCommand(String dbCommand) {
- return (dbCommand.startsWith("--") || dbCommand.startsWith("#"));
- }
-
- @Override
- public String getDelimiter() {
- return DEFAULT_DELIMITER;
- }
-
- @Override
- public String getQuoteCharacter() {
- return DEFAULT_QUOTE;
- }
-
-
- @Override
- public String cleanseCommand(String dbCommand) {
- // strip off the delimiter
- if (dbCommand.endsWith(getDelimiter())) {
- dbCommand = dbCommand.substring(0,
- dbCommand.length() - getDelimiter().length());
- }
- return dbCommand;
- }
-
- @Override
- public boolean needsQuotedIdentifier() {
- return false;
- }
-
- @Override
- public String buildCommand(
- String scriptDir, String scriptFile) throws IllegalFormatException, IOException {
- return buildCommand(scriptDir, scriptFile, false);
- }
-
- @Override
- public String buildCommand(
- String scriptDir, String scriptFile, boolean fixQuotes) throws IllegalFormatException, IOException {
- BufferedReader bfReader =
- new BufferedReader(new FileReader(scriptDir + File.separatorChar + scriptFile));
- String currLine;
- StringBuilder sb = new StringBuilder();
- String currentCommand = null;
- while ((currLine = bfReader.readLine()) != null) {
- currLine = currLine.trim();
-
- if (fixQuotes && !getQuoteCharacter().equals(DEFAULT_QUOTE)) {
- currLine = currLine.replace("\\\"", getQuoteCharacter());
- }
-
- if (currLine.isEmpty()) {
- continue; // skip empty lines
- }
-
- if (currentCommand == null) {
- currentCommand = currLine;
- } else {
- currentCommand = currentCommand + " " + currLine;
- }
- if (isPartialCommand(currLine)) {
- // if its a partial line, continue collecting the pieces
- continue;
- }
-
- // if this is a valid executable command then add it to the buffer
- if (!isNonExecCommand(currentCommand)) {
- currentCommand = cleanseCommand(currentCommand);
- if (isNestedScript(currentCommand)) {
- // if this is a nested sql script then flatten it
- String currScript = getScriptName(currentCommand);
- sb.append(buildCommand(scriptDir, currScript));
- } else {
- // Now we have a complete statement, process it
- // write the line to buffer
- sb.append(currentCommand);
- if (usingSqlLine) sb.append(";");
- sb.append(System.getProperty("line.separator"));
- }
- }
- currentCommand = null;
- }
- bfReader.close();
- return sb.toString();
- }
-
- private void setDbOpts(String dbOpts) {
- if (dbOpts != null) {
- this.dbOpts = Lists.newArrayList(dbOpts.split(","));
- } else {
- this.dbOpts = Lists.newArrayList();
- }
- }
-
- protected List<String> getDbOpts() {
- return dbOpts;
- }
-
- protected String getMsUsername() {
- return msUsername;
- }
-
- protected String getMsPassword() {
- return msPassword;
- }
-
- protected Configuration getConf() {
- return conf;
- }
- }
-
- // Derby commandline parser
- public static class DerbyCommandParser extends AbstractCommandParser {
- private static final String DERBY_NESTING_TOKEN = "RUN";
-
- public DerbyCommandParser(String dbOpts, String msUsername, String msPassword,
- Configuration conf, boolean usingSqlLine) {
- super(dbOpts, msUsername, msPassword, conf, usingSqlLine);
- }
-
- @Override
- public String getScriptName(String dbCommand) throws IllegalArgumentException {
-
- if (!isNestedScript(dbCommand)) {
- throw new IllegalArgumentException("Not a script format " + dbCommand);
- }
- String[] tokens = dbCommand.split(" ");
- if (tokens.length != 2) {
- throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
- }
- return tokens[1].replace(";", "").replaceAll("'", "");
- }
-
- @Override
- public boolean isNestedScript(String dbCommand) {
- // Derby script format is RUN '<file>'
- return dbCommand.startsWith(DERBY_NESTING_TOKEN);
- }
- }
-
- // Derby commandline parser
- public static class HiveCommandParser extends AbstractCommandParser {
- private static String HIVE_NESTING_TOKEN = "SOURCE";
- private final NestedScriptParser nestedDbCommandParser;
-
- public HiveCommandParser(String dbOpts, String msUsername, String msPassword,
- Configuration conf, String metaDbType, boolean usingSqlLine) {
- super(dbOpts, msUsername, msPassword, conf, usingSqlLine);
- nestedDbCommandParser = getDbCommandParser(metaDbType, usingSqlLine);
- }
-
- @Override
- public String getQuoteCharacter() {
- return nestedDbCommandParser.getQuoteCharacter();
- }
-
- @Override
- public String getScriptName(String dbCommand) throws IllegalArgumentException {
-
- if (!isNestedScript(dbCommand)) {
- throw new IllegalArgumentException("Not a script format " + dbCommand);
- }
- String[] tokens = dbCommand.split(" ");
- if (tokens.length != 2) {
- throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
- }
- return tokens[1].replace(";", "");
- }
-
- @Override
- public boolean isNestedScript(String dbCommand) {
- return dbCommand.startsWith(HIVE_NESTING_TOKEN);
- }
- }
-
- // MySQL parser
- public static class MySqlCommandParser extends AbstractCommandParser {
- private static final String MYSQL_NESTING_TOKEN = "SOURCE";
- private static final String DELIMITER_TOKEN = "DELIMITER";
- private String delimiter = DEFAULT_DELIMITER;
-
- public MySqlCommandParser(String dbOpts, String msUsername, String msPassword,
- Configuration conf, boolean usingSqlLine) {
- super(dbOpts, msUsername, msPassword, conf, usingSqlLine);
- }
-
- @Override
- public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{
- boolean isPartial = super.isPartialCommand(dbCommand);
- // if this is a delimiter directive, reset our delimiter
- if (dbCommand.startsWith(DELIMITER_TOKEN)) {
- String[] tokens = dbCommand.split(" ");
- if (tokens.length != 2) {
- throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
- }
- delimiter = tokens[1];
- }
- return isPartial;
- }
-
- @Override
- public String getScriptName(String dbCommand) throws IllegalArgumentException {
- String[] tokens = dbCommand.split(" ");
- if (tokens.length != 2) {
- throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
- }
- // remove ending ';'
- return tokens[1].replace(";", "");
- }
-
- @Override
- public boolean isNestedScript(String dbCommand) {
- return dbCommand.startsWith(MYSQL_NESTING_TOKEN);
- }
-
- @Override
- public String getDelimiter() {
- return delimiter;
- }
-
- @Override
- public String getQuoteCharacter() {
- return "`";
- }
-
- @Override
- public boolean isNonExecCommand(String dbCommand) {
- return super.isNonExecCommand(dbCommand) ||
- (dbCommand.startsWith("/*") && dbCommand.endsWith("*/")) ||
- dbCommand.startsWith(DELIMITER_TOKEN);
- }
-
- @Override
- public String cleanseCommand(String dbCommand) {
- return super.cleanseCommand(dbCommand).replaceAll("/\\*.*?\\*/[^;]", "");
- }
-
- }
-
- // Postgres specific parser
- public static class PostgresCommandParser extends AbstractCommandParser {
- private static final String POSTGRES_NESTING_TOKEN = "\\i";
- @VisibleForTesting
- public static final String POSTGRES_STANDARD_STRINGS_OPT = "SET standard_conforming_strings";
- @VisibleForTesting
- public static final String POSTGRES_SKIP_STANDARD_STRINGS_DBOPT = "postgres.filter.81";
-
- public PostgresCommandParser(String dbOpts, String msUsername, String msPassword,
- Configuration conf, boolean usingSqlLine) {
- super(dbOpts, msUsername, msPassword, conf, usingSqlLine);
- }
-
- @Override
- public String getScriptName(String dbCommand) throws IllegalArgumentException {
- String[] tokens = dbCommand.split(" ");
- if (tokens.length != 2) {
- throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
- }
- // remove ending ';'
- return tokens[1].replace(";", "");
- }
-
- @Override
- public boolean isNestedScript(String dbCommand) {
- return dbCommand.startsWith(POSTGRES_NESTING_TOKEN);
- }
-
- @Override
- public boolean needsQuotedIdentifier() {
- return true;
- }
-
- @Override
- public boolean isNonExecCommand(String dbCommand) {
- // Skip "standard_conforming_strings" command which is read-only in older
- // Postgres versions like 8.1
- // See: http://www.postgresql.org/docs/8.2/static/release-8-1.html
- if (getDbOpts().contains(POSTGRES_SKIP_STANDARD_STRINGS_DBOPT)) {
- if (dbCommand.startsWith(POSTGRES_STANDARD_STRINGS_OPT)) {
- return true;
- }
- }
- return super.isNonExecCommand(dbCommand);
- }
- }
-
- //Oracle specific parser
- public static class OracleCommandParser extends AbstractCommandParser {
- private static final String ORACLE_NESTING_TOKEN = "@";
-
- public OracleCommandParser(String dbOpts, String msUsername, String msPassword,
- Configuration conf, boolean usingSqlLine) {
- super(dbOpts, msUsername, msPassword, conf, usingSqlLine);
- }
-
- @Override
- public String getScriptName(String dbCommand) throws IllegalArgumentException {
- if (!isNestedScript(dbCommand)) {
- throw new IllegalArgumentException("Not a nested script format " + dbCommand);
- }
- // remove ending ';' and starting '@'
- return dbCommand.replace(";", "").replace(ORACLE_NESTING_TOKEN, "");
- }
-
- @Override
- public boolean isNestedScript(String dbCommand) {
- return dbCommand.startsWith(ORACLE_NESTING_TOKEN);
- }
- }
-
- //MSSQL specific parser
- public static class MSSQLCommandParser extends AbstractCommandParser {
- private static final String MSSQL_NESTING_TOKEN = ":r";
-
- public MSSQLCommandParser(String dbOpts, String msUsername, String msPassword,
- Configuration conf, boolean usingSqlLine) {
- super(dbOpts, msUsername, msPassword, conf, usingSqlLine);
- }
-
- @Override
- public String getScriptName(String dbCommand) throws IllegalArgumentException {
- String[] tokens = dbCommand.split(" ");
- if (tokens.length != 2) {
- throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
- }
- return tokens[1];
- }
-
- @Override
- public boolean isNestedScript(String dbCommand) {
- return dbCommand.startsWith(MSSQL_NESTING_TOKEN);
- }
- }
-
- public static NestedScriptParser getDbCommandParser(String dbName, boolean usingSqlLine) {
- return getDbCommandParser(dbName, null, usingSqlLine);
- }
-
- public static NestedScriptParser getDbCommandParser(String dbName, String metaDbName, boolean usingSqlLine) {
- return getDbCommandParser(dbName, null, null, null, null, metaDbName, usingSqlLine);
- }
-
- public static NestedScriptParser getDbCommandParser(String dbName,
- String dbOpts, String msUsername, String msPassword,
- Configuration conf, String metaDbType, boolean usingSqlLine) {
- if (dbName.equalsIgnoreCase(DB_DERBY)) {
- return new DerbyCommandParser(dbOpts, msUsername, msPassword, conf, usingSqlLine);
- } else if (dbName.equalsIgnoreCase(DB_HIVE)) {
- return new HiveCommandParser(dbOpts, msUsername, msPassword, conf, metaDbType, usingSqlLine);
- } else if (dbName.equalsIgnoreCase(DB_MSSQL)) {
- return new MSSQLCommandParser(dbOpts, msUsername, msPassword, conf, usingSqlLine);
- } else if (dbName.equalsIgnoreCase(DB_MYSQL)) {
- return new MySqlCommandParser(dbOpts, msUsername, msPassword, conf, usingSqlLine);
- } else if (dbName.equalsIgnoreCase(DB_POSTGRACE)) {
- return new PostgresCommandParser(dbOpts, msUsername, msPassword, conf, usingSqlLine);
- } else if (dbName.equalsIgnoreCase(DB_ORACLE)) {
- return new OracleCommandParser(dbOpts, msUsername, msPassword, conf, usingSqlLine);
- } else {
- throw new IllegalArgumentException("Unknown dbType " + dbName);
- }
- }
-
- public static class MetaStoreConnectionInfo {
- private final String userName;
- private final String password;
- private final String url;
- private final String driver;
- private final boolean printInfo;
- private final Configuration conf;
- private final String dbType;
- private final String metaDbType;
-
- public MetaStoreConnectionInfo(String userName, String password, String url, String driver,
- boolean printInfo, Configuration conf, String dbType, String metaDbType) {
- super();
- this.userName = userName;
- this.password = password;
- this.url = url;
- this.driver = driver;
- this.printInfo = printInfo;
- this.conf = conf;
- this.dbType = dbType;
- this.metaDbType = metaDbType;
- }
-
- public String getPassword() {
- return password;
- }
-
- public String getUrl() {
- return url;
- }
-
- public String getDriver() {
- return driver;
- }
-
- public boolean isPrintInfo() {
- return printInfo;
- }
-
- public Configuration getConf() {
- return conf;
- }
-
- public String getUsername() {
- return userName;
- }
-
- public boolean getPrintInfo() {
- return printInfo;
- }
-
- public String getDbType() {
- return dbType;
- }
-
- public String getMetaDbType() {
- return metaDbType;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/tools/MetastoreSchemaTool.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/tools/MetastoreSchemaTool.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/tools/MetastoreSchemaTool.java
deleted file mode 100644
index c2018f4..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/tools/MetastoreSchemaTool.java
+++ /dev/null
@@ -1,460 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.tools;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.cli.OptionGroup;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.io.output.NullOutputStream;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.metastore.HiveMetaException;
-import org.apache.hadoop.hive.metastore.IMetaStoreSchemaInfo;
-import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
-import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo;
-import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.NestedScriptParser;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import sqlline.SqlLine;
-
-import java.io.BufferedReader;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.PrintStream;
-import java.net.URI;
-import java.sql.Connection;
-import java.sql.SQLException;
-
-public class MetastoreSchemaTool {
- private static final Logger LOG = LoggerFactory.getLogger(MetastoreSchemaTool.class);
- private static final String PASSWD_MASK = "[passwd stripped]";
-
- protected Configuration conf;
-
- protected String dbOpts = null;
- protected String dbType;
- protected String driver = null;
- protected boolean dryRun = false;
- protected String hiveDb; // Hive database, for use when creating the user, not for connecting
- protected String hivePasswd; // Hive password, for use when creating the user, not for connecting
- protected String hiveUser; // Hive username, for use when creating the user, not for connecting
- protected String metaDbType;
- protected IMetaStoreSchemaInfo metaStoreSchemaInfo;
- protected boolean needsQuotedIdentifier;
- protected String quoteCharacter;
- protected String passWord = null;
- protected String url = null;
- protected String userName = null;
- protected URI[] validationServers = null; // The list of servers the database/partition/table can locate on
- protected boolean verbose = false;
- protected SchemaToolCommandLine cmdLine;
-
- private static String homeDir;
-
- private static String findHomeDir() {
- // If METASTORE_HOME is set, use it, else use HIVE_HOME for backwards compatibility.
- homeDir = homeDir == null ? System.getenv("METASTORE_HOME") : homeDir;
- return homeDir == null ? System.getenv("HIVE_HOME") : homeDir;
- }
-
- @VisibleForTesting
- public static void setHomeDirForTesting() {
- homeDir = System.getProperty("test.tmp.dir", "target/tmp");
- }
-
- @VisibleForTesting
- public MetastoreSchemaTool() {
-
- }
-
- @VisibleForTesting
- public void init(String metastoreHome, String[] args, OptionGroup additionalOptions,
- Configuration conf) throws HiveMetaException {
- try {
- cmdLine = new SchemaToolCommandLine(args, additionalOptions);
- } catch (ParseException e) {
- System.err.println("Failed to parse command line. ");
- throw new HiveMetaException(e);
- }
-
- if (metastoreHome == null || metastoreHome.isEmpty()) {
- throw new HiveMetaException("No Metastore home directory provided");
- }
- this.conf = conf;
- this.dbType = cmdLine.getDbType();
- this.metaDbType = cmdLine.getMetaDbType();
- NestedScriptParser parser = getDbCommandParser(dbType, metaDbType);
- this.needsQuotedIdentifier = parser.needsQuotedIdentifier();
- this.quoteCharacter = parser.getQuoteCharacter();
- this.metaStoreSchemaInfo = MetaStoreSchemaInfoFactory.get(conf, metastoreHome, dbType);
-
- if (cmdLine.hasOption("userName")) {
- setUserName(cmdLine.getOptionValue("userName"));
- } else {
- setUserName(getConf().get(MetastoreConf.ConfVars.CONNECTION_USER_NAME.getVarname()));
- }
- if (cmdLine.hasOption("passWord")) {
- setPassWord(cmdLine.getOptionValue("passWord"));
- } else {
- try {
- setPassWord(MetastoreConf.getPassword(getConf(), ConfVars.PWD));
- } catch (IOException err) {
- throw new HiveMetaException("Error getting metastore password", err);
- }
- }
- if (cmdLine.hasOption("url")) {
- setUrl(cmdLine.getOptionValue("url"));
- }
- if (cmdLine.hasOption("driver")) {
- setDriver(cmdLine.getOptionValue("driver"));
- }
- if (cmdLine.hasOption("dryRun")) {
- setDryRun(true);
- }
- if (cmdLine.hasOption("verbose")) {
- setVerbose(true);
- }
- if (cmdLine.hasOption("dbOpts")) {
- setDbOpts(cmdLine.getOptionValue("dbOpts"));
- }
- if (cmdLine.hasOption("validate") && cmdLine.hasOption("servers")) {
- setValidationServers(cmdLine.getOptionValue("servers"));
- }
- if (cmdLine.hasOption("hiveUser")) {
- setHiveUser(cmdLine.getOptionValue("hiveUser"));
- }
- if (cmdLine.hasOption("hivePassword")) {
- setHivePasswd(cmdLine.getOptionValue("hivePassword"));
- }
- if (cmdLine.hasOption("hiveDb")) {
- setHiveDb(cmdLine.getOptionValue("hiveDb"));
- }
- }
-
- public Configuration getConf() {
- return conf;
- }
-
- protected String getDbType() {
- return dbType;
- }
-
- protected void setUrl(String url) {
- this.url = url;
- }
-
- protected void setDriver(String driver) {
- this.driver = driver;
- }
-
- public void setUserName(String userName) {
- this.userName = userName;
- }
-
- public void setPassWord(String passWord) {
- this.passWord = passWord;
- }
-
- protected boolean isDryRun() {
- return dryRun;
- }
-
- protected void setDryRun(boolean dryRun) {
- this.dryRun = dryRun;
- }
-
- protected boolean isVerbose() {
- return verbose;
- }
-
- protected void setVerbose(boolean verbose) {
- this.verbose = verbose;
- }
-
- protected void setDbOpts(String dbOpts) {
- this.dbOpts = dbOpts;
- }
-
- protected URI[] getValidationServers() {
- return validationServers;
- }
-
- protected void setValidationServers(String servers) {
- if(StringUtils.isNotEmpty(servers)) {
- String[] strServers = servers.split(",");
- this.validationServers = new URI[strServers.length];
- for (int i = 0; i < validationServers.length; i++) {
- validationServers[i] = new Path(strServers[i]).toUri();
- }
- }
- }
-
- protected String getHiveUser() {
- return hiveUser;
- }
-
- protected void setHiveUser(String hiveUser) {
- this.hiveUser = hiveUser;
- }
-
- protected String getHivePasswd() {
- return hivePasswd;
- }
-
- protected void setHivePasswd(String hivePasswd) {
- this.hivePasswd = hivePasswd;
- }
-
- protected String getHiveDb() {
- return hiveDb;
- }
-
- protected void setHiveDb(String hiveDb) {
- this.hiveDb = hiveDb;
- }
-
- protected SchemaToolCommandLine getCmdLine() {
- return cmdLine;
- }
-
- public Connection getConnectionToMetastore(boolean printInfo) throws HiveMetaException {
- return HiveSchemaHelper.getConnectionToMetastore(userName,
- passWord, url, driver, printInfo, conf, null);
- }
-
- protected NestedScriptParser getDbCommandParser(String dbType, String metaDbType) {
- return HiveSchemaHelper.getDbCommandParser(dbType, dbOpts, userName,
- passWord, conf, null, true);
- }
-
- protected MetaStoreConnectionInfo getConnectionInfo(boolean printInfo) {
- return new MetaStoreConnectionInfo(userName, passWord, url, driver, printInfo, conf,
- dbType, hiveDb);
- }
-
- protected IMetaStoreSchemaInfo getMetaStoreSchemaInfo() {
- return metaStoreSchemaInfo;
- }
-
- /**
- * check if the current schema version in metastore matches the Hive version
- */
- @VisibleForTesting
- void verifySchemaVersion() throws HiveMetaException {
- // don't check version if its a dry run
- if (dryRun) {
- return;
- }
- String newSchemaVersion = metaStoreSchemaInfo.getMetaStoreSchemaVersion(getConnectionInfo(false));
- // verify that the new version is added to schema
- assertCompatibleVersion(metaStoreSchemaInfo.getHiveSchemaVersion(), newSchemaVersion);
- }
-
- protected void assertCompatibleVersion(String hiveSchemaVersion, String dbSchemaVersion)
- throws HiveMetaException {
- if (!metaStoreSchemaInfo.isVersionCompatible(hiveSchemaVersion, dbSchemaVersion)) {
- throw new HiveMetaException("Metastore schema version is not compatible. Hive Version: "
- + hiveSchemaVersion + ", Database Schema Version: " + dbSchemaVersion);
- }
- }
-
- /***
- * Execute a given metastore script. This default version uses sqlline to execute the files,
- * which requires only running one file. Subclasses can use other executors.
- * @param scriptDir directory script is in
- * @param scriptFile file in the directory to run
- * @throws IOException if it cannot read the file or directory
- * @throws HiveMetaException default implementation never throws this
- */
- protected void execSql(String scriptDir, String scriptFile) throws IOException, HiveMetaException {
-
- execSql(scriptDir + File.separatorChar + scriptFile);
- }
-
- // Generate the beeline args per hive conf and execute the given script
- protected void execSql(String sqlScriptFile) throws IOException {
- CommandBuilder builder = new CommandBuilder(conf, url, driver, userName, passWord, sqlScriptFile);
-
- // run the script using SqlLine
- SqlLine sqlLine = new SqlLine();
- ByteArrayOutputStream outputForLog = null;
- if (!verbose) {
- OutputStream out;
- if (LOG.isDebugEnabled()) {
- out = outputForLog = new ByteArrayOutputStream();
- } else {
- out = new NullOutputStream();
- }
- sqlLine.setOutputStream(new PrintStream(out));
- System.setProperty("sqlline.silent", "true");
- }
- LOG.info("Going to run command <" + builder.buildToLog() + ">");
- SqlLine.Status status = sqlLine.begin(builder.buildToRun(), null, false);
- if (LOG.isDebugEnabled() && outputForLog != null) {
- LOG.debug("Received following output from Sqlline:");
- LOG.debug(outputForLog.toString("UTF-8"));
- }
- if (status != SqlLine.Status.OK) {
- throw new IOException("Schema script failed, errorcode " + status);
- }
- }
-
- // test the connection metastore using the config property
- protected void testConnectionToMetastore() throws HiveMetaException {
- Connection conn = getConnectionToMetastore(true);
- try {
- conn.close();
- } catch (SQLException e) {
- throw new HiveMetaException("Failed to close metastore connection", e);
- }
- }
-
- // Quote if the database requires it
- protected String quote(String stmt) {
- stmt = stmt.replace("<q>", needsQuotedIdentifier ? quoteCharacter : "");
- stmt = stmt.replace("<qa>", quoteCharacter);
- return stmt;
- }
-
- protected static class CommandBuilder {
- protected final String userName;
- protected final String password;
- protected final String sqlScriptFile;
- protected final String driver;
- protected final String url;
-
- protected CommandBuilder(Configuration conf, String url, String driver, String userName,
- String password, String sqlScriptFile) throws IOException {
- this.userName = userName;
- this.password = password;
- this.url = url == null ?
- HiveSchemaHelper.getValidConfVar(MetastoreConf.ConfVars.CONNECT_URL_KEY, conf) : url;
- this.driver = driver == null ?
- HiveSchemaHelper.getValidConfVar(MetastoreConf.ConfVars.CONNECTION_DRIVER, conf) : driver;
- this.sqlScriptFile = sqlScriptFile;
- }
-
- public String[] buildToRun() throws IOException {
- return argsWith(password);
- }
-
- public String buildToLog() throws IOException {
- logScript();
- return StringUtils.join(argsWith(PASSWD_MASK), " ");
- }
-
- protected String[] argsWith(String password) throws IOException {
- return new String[]
- {
- "-u", url,
- "-d", driver,
- "-n", userName,
- "-p", password,
- "--isolation=TRANSACTION_READ_COMMITTED",
- "-f", sqlScriptFile
- };
- }
-
- private void logScript() throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Going to invoke file that contains:");
- try (BufferedReader reader = new BufferedReader(new FileReader(sqlScriptFile))) {
- String line;
- while ((line = reader.readLine()) != null) {
- LOG.debug("script: " + line);
- }
- }
- }
- }
- }
-
- // Create the required command line options
- private static void logAndPrintToError(String errmsg) {
- LOG.error(errmsg);
- System.err.println(errmsg);
- }
-
- public static void main(String[] args) {
- MetastoreSchemaTool tool = new MetastoreSchemaTool();
- System.exit(tool.run(args));
- }
-
- public int run(String[] args) {
- return run(findHomeDir(), args, null, MetastoreConf.newMetastoreConf());
- }
-
- public int run(String metastoreHome, String[] args, OptionGroup additionalOptions,
- Configuration conf) {
- try {
- init(metastoreHome, args, additionalOptions, conf);
- SchemaToolTask task;
- if (cmdLine.hasOption("info")) {
- task = new SchemaToolTaskInfo();
- } else if (cmdLine.hasOption("upgradeSchema") || cmdLine.hasOption("upgradeSchemaFrom")) {
- task = new SchemaToolTaskUpgrade();
- } else if (cmdLine.hasOption("initSchema") || cmdLine.hasOption("initSchemaTo")) {
- task = new SchemaToolTaskInit();
- } else if (cmdLine.hasOption("validate")) {
- task = new SchemaToolTaskValidate();
- } else if (cmdLine.hasOption("createCatalog")) {
- task = new SchemaToolTaskCreateCatalog();
- } else if (cmdLine.hasOption("alterCatalog")) {
- task = new SchemaToolTaskAlterCatalog();
- } else if (cmdLine.hasOption("moveDatabase")) {
- task = new SchemaToolTaskMoveDatabase();
- } else if (cmdLine.hasOption("moveTable")) {
- task = new SchemaToolTaskMoveTable();
- } else if (cmdLine.hasOption("createUser")) {
- task = new SchemaToolTaskCreateUser();
- } else {
- throw new HiveMetaException("No task defined!");
- }
-
- task.setHiveSchemaTool(this);
- task.setCommandLineArguments(cmdLine);
- task.execute();
- return 0;
- } catch (HiveMetaException e) {
- logAndPrintToError(e.getMessage());
- if (e.getCause() != null) {
- Throwable t = e.getCause();
- logAndPrintToError("Underlying cause: "
- + t.getClass().getName() + " : "
- + t.getMessage());
- if (e.getCause() instanceof SQLException) {
- logAndPrintToError("SQL Error code: " + ((SQLException) t).getErrorCode());
- }
- }
- if (cmdLine.hasOption("verbose")) {
- e.printStackTrace();
- } else {
- logAndPrintToError("Use --verbose for detailed stacktrace.");
- }
- logAndPrintToError("*** schemaTool failed ***");
- return 1;
-
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java
deleted file mode 100644
index d0ac7db..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.tools;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.DatabaseProduct;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * Helper class that generates SQL queries with syntax specific to target DB
- * todo: why throw MetaException?
- */
-@VisibleForTesting
-public final class SQLGenerator {
- static final private Logger LOG = LoggerFactory.getLogger(SQLGenerator.class.getName());
- private final DatabaseProduct dbProduct;
-
- private final Configuration conf;
-
- public SQLGenerator(DatabaseProduct dbProduct, Configuration conf) {
- this.dbProduct = dbProduct;
- this.conf = conf;
- }
-
- /**
- * Genereates "Insert into T(a,b,c) values(1,2,'f'),(3,4,'c')" for appropriate DB
- *
- * @param tblColumns e.g. "T(a,b,c)"
- * @param rows e.g. list of Strings like 3,4,'d'
- * @return fully formed INSERT INTO ... statements
- */
- public List<String> createInsertValuesStmt(String tblColumns, List<String> rows) {
- if (rows == null || rows.size() == 0) {
- return Collections.emptyList();
- }
- List<String> insertStmts = new ArrayList<>();
- StringBuilder sb = new StringBuilder();
- switch (dbProduct) {
- case ORACLE:
- if (rows.size() > 1) {
- //http://www.oratable.com/oracle-insert-all/
- //https://livesql.oracle.com/apex/livesql/file/content_BM1LJQ87M5CNIOKPOWPV6ZGR3.html
- for (int numRows = 0; numRows < rows.size(); numRows++) {
- if (numRows % MetastoreConf.getIntVar(conf, ConfVars.DIRECT_SQL_MAX_ELEMENTS_VALUES_CLAUSE) == 0) {
- if (numRows > 0) {
- sb.append(" select * from dual");
- insertStmts.add(sb.toString());
- }
- sb.setLength(0);
- sb.append("insert all ");
- }
- sb.append("into ").append(tblColumns).append(" values(").append(rows.get(numRows))
- .append(") ");
- }
- sb.append("select * from dual");
- insertStmts.add(sb.toString());
- return insertStmts;
- }
- //fall through
- case DERBY:
- case MYSQL:
- case POSTGRES:
- case SQLSERVER:
- for (int numRows = 0; numRows < rows.size(); numRows++) {
- if (numRows % MetastoreConf.getIntVar(conf, ConfVars.DIRECT_SQL_MAX_ELEMENTS_VALUES_CLAUSE) == 0) {
- if (numRows > 0) {
- insertStmts.add(sb.substring(0, sb.length() - 1));//exclude trailing comma
- }
- sb.setLength(0);
- sb.append("insert into ").append(tblColumns).append(" values");
- }
- sb.append('(').append(rows.get(numRows)).append("),");
- }
- insertStmts.add(sb.substring(0, sb.length() - 1));//exclude trailing comma
- return insertStmts;
- default:
- String msg = "Unrecognized database product name <" + dbProduct + ">";
- LOG.error(msg);
- throw new IllegalStateException(msg);
- }
- }
-
- /**
- * Given a {@code selectStatement}, decorated it with FOR UPDATE or semantically equivalent
- * construct. If the DB doesn't support, return original select.
- */
- public String addForUpdateClause(String selectStatement) throws MetaException {
- switch (dbProduct) {
- case DERBY:
- //https://db.apache.org/derby/docs/10.1/ref/rrefsqlj31783.html
- //sadly in Derby, FOR UPDATE doesn't meant what it should
- return selectStatement;
- case MYSQL:
- //http://dev.mysql.com/doc/refman/5.7/en/select.html
- case ORACLE:
- //https://docs.oracle.com/cd/E17952_01/refman-5.6-en/select.html
- case POSTGRES:
- //http://www.postgresql.org/docs/9.0/static/sql-select.html
- return selectStatement + " for update";
- case SQLSERVER:
- //https://msdn.microsoft.com/en-us/library/ms189499.aspx
- //https://msdn.microsoft.com/en-us/library/ms187373.aspx
- String modifier = " with (updlock)";
- int wherePos = selectStatement.toUpperCase().indexOf(" WHERE ");
- if (wherePos < 0) {
- return selectStatement + modifier;
- }
- return selectStatement.substring(0, wherePos) + modifier +
- selectStatement.substring(wherePos, selectStatement.length());
- default:
- String msg = "Unrecognized database product name <" + dbProduct + ">";
- LOG.error(msg);
- throw new MetaException(msg);
- }
- }
-
- /**
- * Suppose you have a query "select a,b from T" and you want to limit the result set
- * to the first 5 rows. The mechanism to do that differs in different DBs.
- * Make {@code noSelectsqlQuery} to be "a,b from T" and this method will return the
- * appropriately modified row limiting query.
- * <p>
- * Note that if {@code noSelectsqlQuery} contains a join, you must make sure that
- * all columns are unique for Oracle.
- */
- public String addLimitClause(int numRows, String noSelectsqlQuery) throws MetaException {
- switch (dbProduct) {
- case DERBY:
- //http://db.apache.org/derby/docs/10.7/ref/rrefsqljoffsetfetch.html
- return "select " + noSelectsqlQuery + " fetch first " + numRows + " rows only";
- case MYSQL:
- //http://www.postgresql.org/docs/7.3/static/queries-limit.html
- case POSTGRES:
- //https://dev.mysql.com/doc/refman/5.0/en/select.html
- return "select " + noSelectsqlQuery + " limit " + numRows;
- case ORACLE:
- //newer versions (12c and later) support OFFSET/FETCH
- return "select * from (select " + noSelectsqlQuery + ") where rownum <= " + numRows;
- case SQLSERVER:
- //newer versions (2012 and later) support OFFSET/FETCH
- //https://msdn.microsoft.com/en-us/library/ms189463.aspx
- return "select TOP(" + numRows + ") " + noSelectsqlQuery;
- default:
- String msg = "Unrecognized database product name <" + dbProduct + ">";
- LOG.error(msg);
- throw new MetaException(msg);
- }
- }
-
- public DatabaseProduct getDbProduct() {
- return dbProduct;
- }
-
- // This is required for SQL executed directly. If the SQL has double quotes then some dbs tend to
- // remove the escape characters and store the variable without double quote.
- public String addEscapeCharacters(String s) {
- if (dbProduct == DatabaseProduct.MYSQL) {
- return s.replaceAll("\\\\", "\\\\\\\\");
- }
- return s;
- }
-
-}