You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2013/12/20 05:06:34 UTC
[1/3] git commit: Merge branch '1.4.5-SNAPSHOT' of
https://git-wip-us.apache.org/repos/asf/accumulo into 1.4.5-SNAPSHOT
Updated Branches:
refs/heads/1.5.1-SNAPSHOT 76842b294 -> 6c966872f
Merge branch '1.4.5-SNAPSHOT' of https://git-wip-us.apache.org/repos/asf/accumulo into 1.4.5-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5a421c7a
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5a421c7a
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5a421c7a
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 5a421c7a06cfa2bf41df33bd77369f03849aaee9
Parents: 53a7d51 2235b50
Author: Josh Elser <el...@apache.org>
Authored: Thu Dec 19 15:50:21 2013 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Thu Dec 19 15:50:21 2013 -0500
----------------------------------------------------------------------
.../accumulo/server/master/state/MetaDataTableScanner.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
[2/3] git commit: ACCUMULO-1593 Applying Ted's patch to ensure UTF-8
String encoding applied to instance id pulled from ZK.
Posted by el...@apache.org.
ACCUMULO-1593 Applying Ted's patch to ensure UTF-8 String encoding applied to instance id pulled from ZK.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/041b482f
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/041b482f
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/041b482f
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 041b482f672bf6a147ac8c9d1646c5aee705d270
Parents: 5a421c7
Author: Josh Elser <el...@apache.org>
Authored: Thu Dec 19 22:23:03 2013 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Thu Dec 19 22:23:03 2013 -0500
----------------------------------------------------------------------
src/core/src/main/java/org/apache/accumulo/core/Constants.java | 3 +++
.../java/org/apache/accumulo/core/client/ZooKeeperInstance.java | 5 +++--
2 files changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/041b482f/src/core/src/main/java/org/apache/accumulo/core/Constants.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/Constants.java b/src/core/src/main/java/org/apache/accumulo/core/Constants.java
index c85649d..6578101 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/Constants.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/Constants.java
@@ -16,6 +16,8 @@
*/
package org.apache.accumulo.core;
+import java.nio.charset.Charset;
+
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
@@ -28,6 +30,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
public class Constants {
+ public static final Charset UTF8 = Charset.forName("UTF-8");
public static final String VERSION = "1.4.5-SNAPSHOT";
public static final int DATA_VERSION = 4;
public static final int PREV_DATA_VERSION = 3;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/041b482f/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java b/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
index 4cd4972..fcadd38 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
@@ -150,7 +150,7 @@ public class ZooKeeperInstance implements Instance {
throw new RuntimeException("Instance name " + instanceName
+ " does not exist in zookeeper. Run \"accumulo org.apache.accumulo.server.util.ListInstances\" to see a list.");
}
- instanceId = new String(iidb);
+ instanceId = new String(iidb, Constants.UTF8);
}
if (zooCache.get(Constants.ZROOT + "/" + instanceId) == null) {
@@ -260,7 +260,8 @@ public class ZooKeeperInstance implements Instance {
ArgumentChecker.notNull(zooCache, instanceId);
for (String name : zooCache.getChildren(Constants.ZROOT + Constants.ZINSTANCES)) {
String instanceNamePath = Constants.ZROOT + Constants.ZINSTANCES + "/" + name;
- UUID iid = UUID.fromString(new String(zooCache.get(instanceNamePath)));
+ byte[] bytes = zooCache.get(instanceNamePath);
+ UUID iid = UUID.fromString(new String(bytes, Constants.UTF8));
if (iid.equals(instanceId)) {
return name;
}
[3/3] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Posted by el...@apache.org.
Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Conflicts:
core/src/main/java/org/apache/accumulo/core/Constants.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6c966872
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6c966872
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6c966872
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 6c966872f9a83067adfa925f3080c5a6bdd8c9f4
Parents: 76842b2 041b482
Author: Josh Elser <el...@apache.org>
Authored: Thu Dec 19 23:03:34 2013 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Thu Dec 19 23:03:34 2013 -0500
----------------------------------------------------------------------
.../java/org/apache/accumulo/core/client/ZooKeeperInstance.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6c966872/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
index 6e1b660,0000000..c20d597
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
@@@ -1,352 -1,0 +1,353 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.impl.ConnectorImpl;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.file.FileUtil;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.OpTimer;
+import org.apache.accumulo.core.util.TextUtil;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * <p>
+ * An implementation of instance that looks in zookeeper to find information needed to connect to an instance of accumulo.
+ *
+ * <p>
+ * The advantage of using zookeeper to obtain information about accumulo is that zookeeper is highly available, very responsive, and supports caching.
+ *
+ * <p>
+ * Because it is possible for multiple instances of accumulo to share a single set of zookeeper servers, all constructors require an accumulo instance name.
+ *
+ * If you do not know the instance names then run accumulo org.apache.accumulo.server.util.ListInstances on an accumulo server.
+ *
+ */
+
+public class ZooKeeperInstance implements Instance {
+
+ private static final Logger log = Logger.getLogger(ZooKeeperInstance.class);
+
+ private String instanceId = null;
+ private String instanceName = null;
+
+ private final ZooCache zooCache;
+
+ private final String zooKeepers;
+
+ private final int zooKeepersSessionTimeOut;
+
+ private volatile boolean closed = false;
+
+ /**
+ *
+ * @param instanceName
+ * The name of specific accumulo instance. This is set at initialization time.
+ * @param zooKeepers
+ * A comma separated list of zoo keeper server locations. Each location can contain an optional port, of the format host:port.
+ */
+
+ public ZooKeeperInstance(String instanceName, String zooKeepers) {
+ this(instanceName, zooKeepers, (int) AccumuloConfiguration.getDefaultConfiguration().getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT));
+ }
+
+ /**
+ *
+ * @param instanceName
+ * The name of specific accumulo instance. This is set at initialization time.
+ * @param zooKeepers
+ * A comma separated list of zoo keeper server locations. Each location can contain an optional port, of the format host:port.
+ * @param sessionTimeout
+ * zoo keeper session time out in milliseconds.
+ */
+
+ public ZooKeeperInstance(String instanceName, String zooKeepers, int sessionTimeout) {
+ ArgumentChecker.notNull(instanceName, zooKeepers);
+ this.instanceName = instanceName;
+ this.zooKeepers = zooKeepers;
+ this.zooKeepersSessionTimeOut = sessionTimeout;
+ zooCache = ZooCache.getInstance(zooKeepers, sessionTimeout);
+ getInstanceID();
+ clientInstances.incrementAndGet();
+ }
+
+ /**
+ *
+ * @param instanceId
+ * The UUID that identifies the accumulo instance you want to connect to.
+ * @param zooKeepers
+ * A comma separated list of zoo keeper server locations. Each location can contain an optional port, of the format host:port.
+ */
+
+ public ZooKeeperInstance(UUID instanceId, String zooKeepers) {
+ this(instanceId, zooKeepers, (int) AccumuloConfiguration.getDefaultConfiguration().getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT));
+ }
+
+ /**
+ *
+ * @param instanceId
+ * The UUID that identifies the accumulo instance you want to connect to.
+ * @param zooKeepers
+ * A comma separated list of zoo keeper server locations. Each location can contain an optional port, of the format host:port.
+ * @param sessionTimeout
+ * zoo keeper session time out in milliseconds.
+ */
+
+ public ZooKeeperInstance(UUID instanceId, String zooKeepers, int sessionTimeout) {
+ ArgumentChecker.notNull(instanceId, zooKeepers);
+ this.instanceId = instanceId.toString();
+ this.zooKeepers = zooKeepers;
+ this.zooKeepersSessionTimeOut = sessionTimeout;
+ zooCache = ZooCache.getInstance(zooKeepers, sessionTimeout);
+ clientInstances.incrementAndGet();
+ }
+
+ @Override
+ public String getInstanceID() {
+ if (closed)
+ throw new RuntimeException("ZooKeeperInstance has been closed.");
+ if (instanceId == null) {
+ // want the instance id to be stable for the life of this instance object,
+ // so only get it once
+ String instanceNamePath = Constants.ZROOT + Constants.ZINSTANCES + "/" + instanceName;
+ byte[] iidb = zooCache.get(instanceNamePath);
+ if (iidb == null) {
+ throw new RuntimeException("Instance name " + instanceName
+ + " does not exist in zookeeper. Run \"accumulo org.apache.accumulo.server.util.ListInstances\" to see a list.");
+ }
- instanceId = new String(iidb);
++ instanceId = new String(iidb, Constants.UTF8);
+ }
+
+ if (zooCache.get(Constants.ZROOT + "/" + instanceId) == null) {
+ if (instanceName == null)
+ throw new RuntimeException("Instance id " + instanceId + " does not exist in zookeeper");
+ throw new RuntimeException("Instance id " + instanceId + " pointed to by the name " + instanceName + " does not exist in zookeeper");
+ }
+
+ return instanceId;
+ }
+
+ @Override
+ public List<String> getMasterLocations() {
+ if (closed)
+ throw new RuntimeException("ZooKeeperInstance has been closed.");
+ String masterLocPath = ZooUtil.getRoot(this) + Constants.ZMASTER_LOCK;
+
+ OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Looking up master location in zoocache.");
+ byte[] loc = ZooUtil.getLockData(zooCache, masterLocPath);
+ opTimer.stop("Found master at " + (loc == null ? null : new String(loc)) + " in %DURATION%");
+
+ if (loc == null) {
+ return Collections.emptyList();
+ }
+
+ return Collections.singletonList(new String(loc));
+ }
+
+ @Override
+ public String getRootTabletLocation() {
+ if (closed)
+ throw new RuntimeException("ZooKeeperInstance has been closed.");
+ String zRootLocPath = ZooUtil.getRoot(this) + Constants.ZROOT_TABLET_LOCATION;
+
+ OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Looking up root tablet location in zookeeper.");
+ byte[] loc = zooCache.get(zRootLocPath);
+ opTimer.stop("Found root tablet at " + (loc == null ? null : new String(loc)) + " in %DURATION%");
+
+ if (loc == null) {
+ return null;
+ }
+
+ return new String(loc).split("\\|")[0];
+ }
+
+ @Override
+ public String getInstanceName() {
+ if (closed)
+ throw new RuntimeException("ZooKeeperInstance has been closed.");
+ if (instanceName == null)
+ instanceName = lookupInstanceName(zooCache, UUID.fromString(getInstanceID()));
+
+ return instanceName;
+ }
+
+ @Override
+ public String getZooKeepers() {
+ return zooKeepers;
+ }
+
+ @Override
+ public int getZooKeepersSessionTimeOut() {
+ return zooKeepersSessionTimeOut;
+ }
+
+ @Override
+ @Deprecated
+ public Connector getConnector(String user, CharSequence pass) throws AccumuloException, AccumuloSecurityException {
+ return getConnector(user, TextUtil.getBytes(new Text(pass.toString())));
+ }
+
+ @Override
+ @Deprecated
+ public Connector getConnector(String user, ByteBuffer pass) throws AccumuloException, AccumuloSecurityException {
+ return getConnector(user, ByteBufferUtil.toBytes(pass));
+ }
+
+ @Override
+ public Connector getConnector(String principal, AuthenticationToken token) throws AccumuloException, AccumuloSecurityException {
+ return getConnector(CredentialHelper.create(principal, token, getInstanceID()));
+ }
+
+ @SuppressWarnings("deprecation")
+ private Connector getConnector(TCredentials credential) throws AccumuloException, AccumuloSecurityException {
+ return new ConnectorImpl(this, credential);
+ }
+
+ @Override
+ @Deprecated
+ public Connector getConnector(String principal, byte[] pass) throws AccumuloException, AccumuloSecurityException {
+ if (closed) {
+ throw new RuntimeException("ZooKeeperInstance has been closed.");
+ } else {
+ return getConnector(principal, new PasswordToken(pass));
+ }
+ }
+
+ private AccumuloConfiguration conf = null;
+
+ @Override
+ public AccumuloConfiguration getConfiguration() {
+ if (conf == null)
+ conf = AccumuloConfiguration.getDefaultConfiguration();
+ return conf;
+ }
+
+ @Override
+ public void setConfiguration(AccumuloConfiguration conf) {
+ this.conf = conf;
+ }
+
+ /**
+ * @deprecated Use {@link #lookupInstanceName(org.apache.accumulo.fate.zookeeper.ZooCache, UUID)} instead
+ */
+ @Deprecated
+ public static String lookupInstanceName(org.apache.accumulo.core.zookeeper.ZooCache zooCache, UUID instanceId) {
+ return lookupInstanceName((ZooCache) zooCache, instanceId);
+ }
+
+ /**
+ * Given a zooCache and instanceId, look up the instance name.
+ *
+ * @param zooCache
+ * @param instanceId
+ * @return the instance name
+ */
+ public static String lookupInstanceName(ZooCache zooCache, UUID instanceId) {
+ ArgumentChecker.notNull(zooCache, instanceId);
+ for (String name : zooCache.getChildren(Constants.ZROOT + Constants.ZINSTANCES)) {
+ String instanceNamePath = Constants.ZROOT + Constants.ZINSTANCES + "/" + name;
- UUID iid = UUID.fromString(new String(zooCache.get(instanceNamePath)));
++ byte[] bytes = zooCache.get(instanceNamePath);
++ UUID iid = UUID.fromString(new String(bytes, Constants.UTF8));
+ if (iid.equals(instanceId)) {
+ return name;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * To be moved to server code. Only lives here to support certain client side utilities to minimize command-line options.
+ */
+ @Deprecated
+ public static String getInstanceIDFromHdfs(Path instanceDirectory) {
+ try {
+ FileSystem fs = FileUtil.getFileSystem(CachedConfiguration.getInstance(), AccumuloConfiguration.getSiteConfiguration());
+ FileStatus[] files = null;
+ try {
+ files = fs.listStatus(instanceDirectory);
+ } catch (FileNotFoundException ex) {
+ // ignored
+ }
+ log.debug("Trying to read instance id from " + instanceDirectory);
+ if (files == null || files.length == 0) {
+ log.error("unable obtain instance id at " + instanceDirectory);
+ throw new RuntimeException("Accumulo not initialized, there is no instance id at " + instanceDirectory);
+ } else if (files.length != 1) {
+ log.error("multiple potential instances in " + instanceDirectory);
+ throw new RuntimeException("Accumulo found multiple possible instance ids in " + instanceDirectory);
+ } else {
+ String result = files[0].getPath().getName();
+ return result;
+ }
+ } catch (IOException e) {
+ throw new RuntimeException("Accumulo not initialized, there is no instance id at " + instanceDirectory, e);
+ }
+ }
+
+ @Deprecated
+ @Override
+ public Connector getConnector(org.apache.accumulo.core.security.thrift.AuthInfo auth) throws AccumuloException, AccumuloSecurityException {
+ return getConnector(auth.user, auth.password);
+ }
+
+ static private final AtomicInteger clientInstances = new AtomicInteger(0);
+
+ @Override
+ public synchronized void close() {
+ if (!closed && clientInstances.decrementAndGet() == 0) {
+ try {
+ zooCache.close();
+ ThriftUtil.close();
+ } catch (RuntimeException e) {
+ clientInstances.incrementAndGet();
+ throw e;
+ }
+ }
+ closed = true;
+ }
+
+ @Override
+ public void finalize() {
+ // This method intentionally left blank. Users need to explicitly close Instances if they want things cleaned up nicely.
+ if (!closed)
+ log.warn("ZooKeeperInstance being cleaned up without being closed. Please remember to call close() before dereferencing to clean up threads.");
+ }
+}