You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2014/03/14 22:11:27 UTC
[01/26] git commit: ACCUMULO-2319 Move MemoryManagementFramework
thread starts out of constructor
Repository: accumulo
Updated Branches:
refs/heads/ACCUMULO-2061 db76dd751 -> de5c29bc8
ACCUMULO-2319 Move MemoryManagementFramework thread starts out of constructor
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d11acbe5
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d11acbe5
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d11acbe5
Branch: refs/heads/ACCUMULO-2061
Commit: d11acbe5dc3f88efcb03b5b08832ecfa9fc15381
Parents: 117dce7
Author: Bill Havanki <bh...@cloudera.com>
Authored: Wed Mar 12 10:50:20 2014 -0400
Committer: Bill Havanki <bh...@cloudera.com>
Committed: Wed Mar 12 10:50:20 2014 -0400
----------------------------------------------------------------------
.../TabletServerResourceManager.java | 21 ++++++++++++--------
1 file changed, 13 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d11acbe5/src/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java b/src/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java
index f7f04c9..87d73b5 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java
@@ -187,6 +187,7 @@ public class TabletServerResourceManager {
}
memMgmt = new MemoryManagementFramework();
+ memMgmt.startThreads();
}
@@ -230,6 +231,8 @@ public class TabletServerResourceManager {
private LinkedBlockingQueue<TabletStateImpl> memUsageReports;
private long lastMemCheckTime = System.currentTimeMillis();
private long maxMem;
+ private Thread memoryGuardThread;
+ private Thread minorCompactionInitiatorThread;
MemoryManagementFramework() {
tabletReports = Collections.synchronizedMap(new HashMap<KeyExtent,TabletStateImpl>());
@@ -242,10 +245,9 @@ public class TabletServerResourceManager {
}
};
- Thread t1 = new Daemon(new LoggingRunnable(log, r1));
- t1.setPriority(Thread.NORM_PRIORITY + 1);
- t1.setName("Accumulo Memory Guard");
- t1.start();
+ memoryGuardThread = new Daemon(new LoggingRunnable(log, r1));
+ memoryGuardThread.setPriority(Thread.NORM_PRIORITY + 1);
+ memoryGuardThread.setName("Accumulo Memory Guard");
Runnable r2 = new Runnable() {
public void run() {
@@ -253,10 +255,13 @@ public class TabletServerResourceManager {
}
};
- Thread t2 = new Daemon(new LoggingRunnable(log, r2));
- t2.setName("Accumulo Minor Compaction Initiator");
- t2.start();
-
+ minorCompactionInitiatorThread = new Daemon(new LoggingRunnable(log, r2));
+ minorCompactionInitiatorThread.setName("Accumulo Minor Compaction Initiator");
+ }
+
+ void startThreads() {
+ memoryGuardThread.start();
+ minorCompactionInitiatorThread.start();
}
private long lastMemTotal = 0;
[19/26] git commit: ACCUMULO-2454 Improve Cobertura reporting
Posted by el...@apache.org.
ACCUMULO-2454 Improve Cobertura reporting
Coverage reports are now aggregated across modules. Also, the site reports exclude
Thrift-generated classes.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a99b0b67
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a99b0b67
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a99b0b67
Branch: refs/heads/ACCUMULO-2061
Commit: a99b0b67429aeb378f1e04fe8321d55e46893b50
Parents: 58535fc
Author: Bill Havanki <bh...@cloudera.com>
Authored: Thu Mar 13 17:02:23 2014 -0400
Committer: Bill Havanki <bh...@cloudera.com>
Committed: Thu Mar 13 17:02:23 2014 -0400
----------------------------------------------------------------------
pom.xml | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a99b0b67/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 9fd5bfa..9464bff 100644
--- a/pom.xml
+++ b/pom.xml
@@ -512,6 +512,7 @@
<artifactId>cobertura-maven-plugin</artifactId>
<version>2.5.2</version>
<configuration>
+ <aggregate>true</aggregate>
<formats>
<format>xml</format>
<format>html</format>
@@ -941,10 +942,16 @@
<artifactId>cobertura-maven-plugin</artifactId>
<version>2.5.2</version>
<configuration>
+ <aggregate>true</aggregate>
<formats>
<format>xml</format>
<format>html</format>
</formats>
+ <instrumentation>
+ <excludes>
+ <exclude>**/thrift/*.class</exclude>
+ </excludes>
+ </instrumentation>
</configuration>
</plugin>
</plugins>
[16/26] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by el...@apache.org.
Merge branch '1.4.5-SNAPSHOT' into 1.5.2-SNAPSHOT
Conflicts:
src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/security/SecurityFixture.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/58535fcb
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/58535fcb
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/58535fcb
Branch: refs/heads/ACCUMULO-2061
Commit: 58535fcb3e2dfe2bc5d424947924f95d97a645b5
Parents: bf0b7f7 ea86b44
Author: Bill Havanki <bh...@cloudera.com>
Authored: Thu Mar 13 10:40:15 2014 -0400
Committer: Bill Havanki <bh...@cloudera.com>
Committed: Thu Mar 13 10:40:15 2014 -0400
----------------------------------------------------------------------
.../apache/accumulo/test/randomwalk/security/SecurityFixture.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/58535fcb/test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityFixture.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityFixture.java
index a6661ea,0000000..26620a4
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityFixture.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityFixture.java
@@@ -1,101 -1,0 +1,103 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.randomwalk.security;
+
+import java.net.InetAddress;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.test.randomwalk.Fixture;
+import org.apache.accumulo.test.randomwalk.State;
+
+public class SecurityFixture extends Fixture {
+
+ @Override
+ public void setUp(State state) throws Exception {
+ String secTableName, systemUserName, tableUserName;
+ Connector conn = state.getConnector();
+
+ String hostname = InetAddress.getLocalHost().getHostName().replaceAll("[-.]", "_");
+
+ systemUserName = String.format("system_%s", hostname);
+ tableUserName = String.format("table_%s", hostname);
+ secTableName = String.format("security_%s", hostname);
+
+ if (conn.tableOperations().exists(secTableName))
+ conn.tableOperations().delete(secTableName);
+ Set<String> users = conn.securityOperations().listLocalUsers();
+ if (users.contains(tableUserName))
+ conn.securityOperations().dropLocalUser(tableUserName);
+ if (users.contains(systemUserName))
+ conn.securityOperations().dropLocalUser(systemUserName);
+
+ PasswordToken sysUserPass = new PasswordToken("sysUser");
+ conn.securityOperations().createLocalUser(systemUserName, sysUserPass);
+
+ WalkingSecurity.get(state).setTableName(secTableName);
+ state.set("rootUserPass", CredentialHelper.extractToken(state.getCredentials()));
+
+ WalkingSecurity.get(state).setSysUserName(systemUserName);
+ WalkingSecurity.get(state).createUser(systemUserName, sysUserPass);
+
+ WalkingSecurity.get(state).changePassword(tableUserName, new PasswordToken(new byte[0]));
+
+ WalkingSecurity.get(state).setTabUserName(tableUserName);
+
+ for (TablePermission tp : TablePermission.values()) {
+ WalkingSecurity.get(state).revokeTablePermission(systemUserName, secTableName, tp);
+ WalkingSecurity.get(state).revokeTablePermission(tableUserName, secTableName, tp);
+ }
+ for (SystemPermission sp : SystemPermission.values()) {
+ WalkingSecurity.get(state).revokeSystemPermission(systemUserName, sp);
+ WalkingSecurity.get(state).revokeSystemPermission(tableUserName, sp);
+ }
+ WalkingSecurity.get(state).changeAuthorizations(tableUserName, new Authorizations());
+ }
+
+ @Override
+ public void tearDown(State state) throws Exception {
+ log.debug("One last validate");
+ Validate.validate(state, log);
+ Connector conn = state.getConnector();
+
+ if (WalkingSecurity.get(state).getTableExists()) {
+ String secTableName = WalkingSecurity.get(state).getTableName();
+ log.debug("Dropping tables: " + secTableName);
+
+ conn.tableOperations().delete(secTableName);
+ }
+
+ if (WalkingSecurity.get(state).userExists(WalkingSecurity.get(state).getTabUserName())) {
+ String tableUserName = WalkingSecurity.get(state).getTabUserName();
+ log.debug("Dropping user: " + tableUserName);
+
+ conn.securityOperations().dropLocalUser(tableUserName);
+ }
+ String systemUserName = WalkingSecurity.get(state).getSysUserName();
+ log.debug("Dropping user: " + systemUserName);
+ conn.securityOperations().dropLocalUser(systemUserName);
+ WalkingSecurity.clearInstance();
+
++ // Allow user drops to propagate, in case a new security test starts
++ Thread.sleep(2000);
+ }
+}
[13/26] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by el...@apache.org.
Merge branch '1.4.5-SNAPSHOT' into 1.5.2-SNAPSHOT
Conflicts:
server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/bf0b7f78
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/bf0b7f78
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/bf0b7f78
Branch: refs/heads/ACCUMULO-2061
Commit: bf0b7f78b52f5d61e84792888ce479c758e4028d
Parents: 43cebf8 d11acbe
Author: Bill Havanki <bh...@cloudera.com>
Authored: Wed Mar 12 16:49:16 2014 -0400
Committer: Bill Havanki <bh...@cloudera.com>
Committed: Wed Mar 12 16:49:16 2014 -0400
----------------------------------------------------------------------
.../TabletServerResourceManager.java | 21 ++++++++++++--------
1 file changed, 13 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/bf0b7f78/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java
index e0dbead,0000000..57cd49b
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java
@@@ -1,803 -1,0 +1,808 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.tabletserver;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.accumulo.trace.instrument.TraceExecutorService;
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.file.blockfile.cache.LruBlockCache;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.LoggingRunnable;
+import org.apache.accumulo.core.util.MetadataTable.DataFileValue;
+import org.apache.accumulo.core.util.NamingThreadFactory;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.tabletserver.FileManager.ScanFileManager;
+import org.apache.accumulo.server.tabletserver.Tablet.MajorCompactionReason;
+import org.apache.accumulo.server.tabletserver.Tablet.MinorCompactionReason;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.log4j.Logger;
+
+/**
+ * ResourceManager is responsible for managing the resources of all tablets within a tablet server.
+ *
+ *
+ *
+ */
+public class TabletServerResourceManager {
+
+ private ExecutorService minorCompactionThreadPool;
+ private ExecutorService majorCompactionThreadPool;
+ private ExecutorService rootMajorCompactionThreadPool;
+ private ExecutorService defaultMajorCompactionThreadPool;
+ private ExecutorService splitThreadPool;
+ private ExecutorService defaultSplitThreadPool;
+ private ExecutorService defaultMigrationPool;
+ private ExecutorService migrationPool;
+ private ExecutorService assignmentPool;
+ private ExecutorService assignMetaDataPool;
+ private ExecutorService readAheadThreadPool;
+ private ExecutorService defaultReadAheadThreadPool;
+ private Map<String,ExecutorService> threadPools = new TreeMap<String,ExecutorService>();
+
+ private HashSet<TabletResourceManager> tabletResources;
+
+ private FileManager fileManager;
+
+ private MemoryManager memoryManager;
+
+ private MemoryManagementFramework memMgmt;
+
+ private final LruBlockCache _dCache;
+ private final LruBlockCache _iCache;
+ private final ServerConfiguration conf;
+
+ private static final Logger log = Logger.getLogger(TabletServerResourceManager.class);
+
+ private ExecutorService addEs(String name, ExecutorService tp) {
+ if (threadPools.containsKey(name)) {
+ throw new IllegalArgumentException("Cannot create two executor services with same name " + name);
+ }
+ tp = new TraceExecutorService(tp);
+ threadPools.put(name, tp);
+ return tp;
+ }
+
+ private ExecutorService addEs(final Property maxThreads, String name, final ThreadPoolExecutor tp) {
+ ExecutorService result = addEs(name, tp);
+ SimpleTimer.getInstance().schedule(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ int max = conf.getConfiguration().getCount(maxThreads);
+ if (tp.getMaximumPoolSize() != max) {
+ log.info("Changing " + maxThreads.getKey() + " to " + max);
+ tp.setCorePoolSize(max);
+ tp.setMaximumPoolSize(max);
+ }
+ } catch (Throwable t) {
+ log.error(t, t);
+ }
+ }
+
+ }, 1000, 10 * 1000);
+ return result;
+ }
+
+ private ExecutorService createEs(int max, String name) {
+ return addEs(name, Executors.newFixedThreadPool(max, new NamingThreadFactory(name)));
+ }
+
+ private ExecutorService createEs(Property max, String name) {
+ return createEs(max, name, new LinkedBlockingQueue<Runnable>());
+ }
+
+ private ExecutorService createEs(Property max, String name, BlockingQueue<Runnable> queue) {
+ int maxThreads = conf.getConfiguration().getCount(max);
+ ThreadPoolExecutor tp = new ThreadPoolExecutor(maxThreads, maxThreads, 0L, TimeUnit.MILLISECONDS, queue, new NamingThreadFactory(name));
+ return addEs(max, name, tp);
+ }
+
+ private ExecutorService createEs(int min, int max, int timeout, String name) {
+ return addEs(name, new ThreadPoolExecutor(min, max, timeout, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), new NamingThreadFactory(name)));
+ }
+
+ public TabletServerResourceManager(Instance instance, FileSystem fs) {
+ this.conf = new ServerConfiguration(instance);
+ final AccumuloConfiguration acuConf = conf.getConfiguration();
+
+ long maxMemory = acuConf.getMemoryInBytes(Property.TSERV_MAXMEM);
+ boolean usingNativeMap = acuConf.getBoolean(Property.TSERV_NATIVEMAP_ENABLED) && NativeMap.loadedNativeLibraries();
+
+ long blockSize = acuConf.getMemoryInBytes(Property.TSERV_DEFAULT_BLOCKSIZE);
+ long dCacheSize = acuConf.getMemoryInBytes(Property.TSERV_DATACACHE_SIZE);
+ long iCacheSize = acuConf.getMemoryInBytes(Property.TSERV_INDEXCACHE_SIZE);
+
+ _iCache = new LruBlockCache(iCacheSize, blockSize);
+ _dCache = new LruBlockCache(dCacheSize, blockSize);
+
+ Runtime runtime = Runtime.getRuntime();
+ if (!usingNativeMap && maxMemory + dCacheSize + iCacheSize > runtime.maxMemory()) {
+ throw new IllegalArgumentException(String.format(
+ "Maximum tablet server map memory %,d and block cache sizes %,d is too large for this JVM configuration %,d", maxMemory, dCacheSize + iCacheSize,
+ runtime.maxMemory()));
+ }
+ runtime.gc();
+
+ // totalMemory - freeMemory = memory in use
+ // maxMemory - memory in use = max available memory
+ if (!usingNativeMap && maxMemory > runtime.maxMemory() - (runtime.totalMemory() - runtime.freeMemory())) {
+ log.warn("In-memory map may not fit into local memory space.");
+ }
+
+ minorCompactionThreadPool = createEs(Property.TSERV_MINC_MAXCONCURRENT, "minor compactor");
+
+ // make this thread pool have a priority queue... and execute tablets with the most
+ // files first!
+ majorCompactionThreadPool = createEs(Property.TSERV_MAJC_MAXCONCURRENT, "major compactor", new CompactionQueue());
+ rootMajorCompactionThreadPool = createEs(0, 1, 300, "md root major compactor");
+ defaultMajorCompactionThreadPool = createEs(0, 1, 300, "md major compactor");
+
+ splitThreadPool = createEs(1, "splitter");
+ defaultSplitThreadPool = createEs(0, 1, 60, "md splitter");
+
+ defaultMigrationPool = createEs(0, 1, 60, "metadata tablet migration");
+ migrationPool = createEs(Property.TSERV_MIGRATE_MAXCONCURRENT, "tablet migration");
+
+ // not sure if concurrent assignments can run safely... even if they could there is probably no benefit at startup because
+ // individual tablet servers are already running assignments concurrently... having each individual tablet server run
+ // concurrent assignments would put more load on the metadata table at startup
+ assignmentPool = createEs(1, "tablet assignment");
+
+ assignMetaDataPool = createEs(0, 1, 60, "metadata tablet assignment");
+
+ readAheadThreadPool = createEs(Property.TSERV_READ_AHEAD_MAXCONCURRENT, "tablet read ahead");
+ defaultReadAheadThreadPool = createEs(Property.TSERV_METADATA_READ_AHEAD_MAXCONCURRENT, "metadata tablets read ahead");
+
+ tabletResources = new HashSet<TabletResourceManager>();
+
+ int maxOpenFiles = acuConf.getCount(Property.TSERV_SCAN_MAX_OPENFILES);
+
+ fileManager = new FileManager(conf, fs, maxOpenFiles, _dCache, _iCache);
+
+ try {
+ Class<? extends MemoryManager> clazz = AccumuloVFSClassLoader.loadClass(acuConf.get(Property.TSERV_MEM_MGMT), MemoryManager.class);
+ memoryManager = clazz.newInstance();
+ memoryManager.init(conf);
+ log.debug("Loaded memory manager : " + memoryManager.getClass().getName());
+ } catch (Exception e) {
+ log.error("Failed to find memory manger in config, using default", e);
+ }
+
+ if (memoryManager == null) {
+ memoryManager = new LargestFirstMemoryManager();
+ }
+
+ memMgmt = new MemoryManagementFramework();
++ memMgmt.startThreads();
+ }
+
+ private static class TabletStateImpl implements TabletState, Cloneable {
+
+ private long lct;
+ private Tablet tablet;
+ private long mts;
+ private long mcmts;
+
+ public TabletStateImpl(Tablet t, long mts, long lct, long mcmts) {
+ this.tablet = t;
+ this.mts = mts;
+ this.lct = lct;
+ this.mcmts = mcmts;
+ }
+
+ public KeyExtent getExtent() {
+ return tablet.getExtent();
+ }
+
+ Tablet getTablet() {
+ return tablet;
+ }
+
+ public long getLastCommitTime() {
+ return lct;
+ }
+
+ public long getMemTableSize() {
+ return mts;
+ }
+
+ public long getMinorCompactingMemTableSize() {
+ return mcmts;
+ }
+ }
+
+ private class MemoryManagementFramework {
+ private final Map<KeyExtent,TabletStateImpl> tabletReports;
+ private LinkedBlockingQueue<TabletStateImpl> memUsageReports;
+ private long lastMemCheckTime = System.currentTimeMillis();
+ private long maxMem;
++ private Thread memoryGuardThread;
++ private Thread minorCompactionInitiatorThread;
+
+ MemoryManagementFramework() {
+ tabletReports = Collections.synchronizedMap(new HashMap<KeyExtent,TabletStateImpl>());
+ memUsageReports = new LinkedBlockingQueue<TabletStateImpl>();
+ maxMem = conf.getConfiguration().getMemoryInBytes(Property.TSERV_MAXMEM);
+
+ Runnable r1 = new Runnable() {
+ public void run() {
+ processTabletMemStats();
+ }
+ };
+
- Thread t1 = new Daemon(new LoggingRunnable(log, r1));
- t1.setPriority(Thread.NORM_PRIORITY + 1);
- t1.setName("Accumulo Memory Guard");
- t1.start();
++ memoryGuardThread = new Daemon(new LoggingRunnable(log, r1));
++ memoryGuardThread.setPriority(Thread.NORM_PRIORITY + 1);
++ memoryGuardThread.setName("Accumulo Memory Guard");
+
+ Runnable r2 = new Runnable() {
+ public void run() {
+ manageMemory();
+ }
+ };
+
- Thread t2 = new Daemon(new LoggingRunnable(log, r2));
- t2.setName("Accumulo Minor Compaction Initiator");
- t2.start();
-
++ minorCompactionInitiatorThread = new Daemon(new LoggingRunnable(log, r2));
++ minorCompactionInitiatorThread.setName("Accumulo Minor Compaction Initiator");
++ }
++
++ void startThreads() {
++ memoryGuardThread.start();
++ minorCompactionInitiatorThread.start();
+ }
+
+ private long lastMemTotal = 0;
+
+ private void processTabletMemStats() {
+ while (true) {
+ try {
+
+ TabletStateImpl report = memUsageReports.take();
+
+ while (report != null) {
+ tabletReports.put(report.getExtent(), report);
+ report = memUsageReports.poll();
+ }
+
+ long delta = System.currentTimeMillis() - lastMemCheckTime;
+ if (holdCommits || delta > 50 || lastMemTotal > 0.90 * maxMem) {
+ lastMemCheckTime = System.currentTimeMillis();
+
+ long totalMemUsed = 0;
+
+ synchronized (tabletReports) {
+ for (TabletStateImpl tsi : tabletReports.values()) {
+ totalMemUsed += tsi.getMemTableSize();
+ totalMemUsed += tsi.getMinorCompactingMemTableSize();
+ }
+ }
+
+ if (totalMemUsed > 0.95 * maxMem) {
+ holdAllCommits(true);
+ } else {
+ holdAllCommits(false);
+ }
+
+ lastMemTotal = totalMemUsed;
+ }
+
+ } catch (InterruptedException e) {
+ log.warn(e, e);
+ }
+ }
+ }
+
+ private void manageMemory() {
+ while (true) {
+ MemoryManagementActions mma = null;
+
+ try {
+ ArrayList<TabletState> tablets;
+ synchronized (tabletReports) {
+ tablets = new ArrayList<TabletState>(tabletReports.values());
+ }
+ mma = memoryManager.getMemoryManagementActions(tablets);
+
+ } catch (Throwable t) {
+ log.error("Memory manager failed " + t.getMessage(), t);
+ }
+
+ try {
+ if (mma != null && mma.tabletsToMinorCompact != null && mma.tabletsToMinorCompact.size() > 0) {
+ for (KeyExtent keyExtent : mma.tabletsToMinorCompact) {
+ TabletStateImpl tabletReport = tabletReports.get(keyExtent);
+
+ if (tabletReport == null) {
+ log.warn("Memory manager asked to compact nonexistant tablet " + keyExtent);
+ continue;
+ }
+
+ if (!tabletReport.getTablet().initiateMinorCompaction(MinorCompactionReason.SYSTEM)) {
+ if (tabletReport.getTablet().isClosed()) {
+ tabletReports.remove(tabletReport.getExtent());
+ log.debug("Ignoring memory manager recommendation: not minor compacting closed tablet " + keyExtent);
+ } else {
+ log.info("Ignoring memory manager recommendation: not minor compacting " + keyExtent);
+ }
+ }
+ }
+
+ // log.debug("mma.tabletsToMinorCompact = "+mma.tabletsToMinorCompact);
+ }
+ } catch (Throwable t) {
+ log.error("Minor compactions for memory managment failed", t);
+ }
+
+ UtilWaitThread.sleep(250);
+ }
+ }
+
+ public void updateMemoryUsageStats(Tablet tablet, long size, long lastCommitTime, long mincSize) {
+ memUsageReports.add(new TabletStateImpl(tablet, size, lastCommitTime, mincSize));
+ }
+
+ public void tabletClosed(KeyExtent extent) {
+ tabletReports.remove(extent);
+ }
+ }
+
+ private final Object commitHold = new Object();
+ private volatile boolean holdCommits = false;
+ private long holdStartTime;
+
+ protected void holdAllCommits(boolean holdAllCommits) {
+ synchronized (commitHold) {
+ if (holdCommits != holdAllCommits) {
+ holdCommits = holdAllCommits;
+
+ if (holdCommits) {
+ holdStartTime = System.currentTimeMillis();
+ }
+
+ if (!holdCommits) {
+ log.debug(String.format("Commits held for %6.2f secs", (System.currentTimeMillis() - holdStartTime) / 1000.0));
+ commitHold.notifyAll();
+ }
+ }
+ }
+
+ }
+
+ void waitUntilCommitsAreEnabled() {
+ if (holdCommits) {
+ long timeout = System.currentTimeMillis() + conf.getConfiguration().getTimeInMillis(Property.GENERAL_RPC_TIMEOUT);
+ synchronized (commitHold) {
+ while (holdCommits) {
+ try {
+ if (System.currentTimeMillis() > timeout)
+ throw new HoldTimeoutException("Commits are held");
+ commitHold.wait(1000);
+ } catch (InterruptedException e) {}
+ }
+ }
+ }
+ }
+
+ public long holdTime() {
+ if (!holdCommits)
+ return 0;
+ synchronized (commitHold) {
+ return System.currentTimeMillis() - holdStartTime;
+ }
+ }
+
+ public void close() {
+ for (ExecutorService executorService : threadPools.values()) {
+ executorService.shutdown();
+ }
+
+ for (Entry<String,ExecutorService> entry : threadPools.entrySet()) {
+ while (true) {
+ try {
+ if (entry.getValue().awaitTermination(60, TimeUnit.SECONDS))
+ break;
+ log.info("Waiting for thread pool " + entry.getKey() + " to shutdown");
+ } catch (InterruptedException e) {
+ log.warn(e);
+ }
+ }
+ }
+ }
+
+ public synchronized TabletResourceManager createTabletResourceManager() {
+ TabletResourceManager trm = new TabletResourceManager();
+ return trm;
+ }
+
+ synchronized private void addTabletResource(TabletResourceManager tr) {
+ tabletResources.add(tr);
+ }
+
+ synchronized private void removeTabletResource(TabletResourceManager tr) {
+ tabletResources.remove(tr);
+ }
+
+ private class MapFileInfo {
+ private final String path;
+ private final long size;
+
+ MapFileInfo(String path, long size) {
+ this.path = path;
+ this.size = size;
+ }
+ }
+
+ public class TabletResourceManager {
+
+ private final long creationTime = System.currentTimeMillis();
+
+ private volatile boolean openFilesReserved = false;
+
+ private volatile boolean closed = false;
+
+ private Tablet tablet;
+
+ private AccumuloConfiguration tableConf;
+
+ TabletResourceManager() {}
+
+ void setTablet(Tablet tablet, AccumuloConfiguration tableConf) {
+ this.tablet = tablet;
+ this.tableConf = tableConf;
+ // TabletResourceManager is not really initialized until this
+ // function is called.... so do not make it publicly available
+ // until now
+
+ addTabletResource(this);
+ }
+
+ // BEGIN methods that Tablets call to manage their set of open map files
+
+ public void importedMapFiles() {
+ lastReportedCommitTime = System.currentTimeMillis();
+ }
+
+ synchronized ScanFileManager newScanFileManager() {
+ if (closed)
+ throw new IllegalStateException("closed");
+ return fileManager.newScanFileManager(tablet.getExtent());
+ }
+
+ // END methods that Tablets call to manage their set of open map files
+
+ // BEGIN methods that Tablets call to manage memory
+
+ private AtomicLong lastReportedSize = new AtomicLong();
+ private AtomicLong lastReportedMincSize = new AtomicLong();
+ private volatile long lastReportedCommitTime = 0;
+
+ public void updateMemoryUsageStats(long size, long mincSize) {
+
+ // do not want to update stats for every little change,
+ // so only do it under certain circumstances... the reason
+ // for this is that reporting stats acquires a lock, do
+ // not want all tablets locking on the same lock for every
+ // commit
+ long totalSize = size + mincSize;
+ long lrs = lastReportedSize.get();
+ long delta = totalSize - lrs;
+ long lrms = lastReportedMincSize.get();
+ boolean report = false;
+ // the atomic longs are considered independently, when one is set
+ // the other is not set intentionally because this method is not
+ // synchronized... therefore there are not transactional semantics
+ // for reading and writing two variables
+ if ((lrms > 0 && mincSize == 0 || lrms == 0 && mincSize > 0) && lastReportedMincSize.compareAndSet(lrms, mincSize)) {
+ report = true;
+ }
+
+ long currentTime = System.currentTimeMillis();
+ if ((delta > 32000 || delta < 0 || (currentTime - lastReportedCommitTime > 1000)) && lastReportedSize.compareAndSet(lrs, totalSize)) {
+ if (delta > 0)
+ lastReportedCommitTime = currentTime;
+ report = true;
+ }
+
+ if (report)
+ memMgmt.updateMemoryUsageStats(tablet, size, lastReportedCommitTime, mincSize);
+ }
+
+ // END methods that Tablets call to manage memory
+
+ // BEGIN methods that Tablets call to make decisions about major compaction
+ // when too many files are open, we may want tablets to compact down
+ // to one map file
+ Map<String,Long> findMapFilesToCompact(SortedMap<String,DataFileValue> tabletFiles, MajorCompactionReason reason) {
+ if (reason == MajorCompactionReason.USER) {
+ Map<String,Long> files = new HashMap<String,Long>();
+ for (Entry<String,DataFileValue> entry : tabletFiles.entrySet()) {
+ files.put(entry.getKey(), entry.getValue().getSize());
+ }
+ return files;
+ }
+
+ if (tabletFiles.size() <= 1)
+ return null;
+ TreeSet<MapFileInfo> candidateFiles = new TreeSet<MapFileInfo>(new Comparator<MapFileInfo>() {
+ @Override
+ public int compare(MapFileInfo o1, MapFileInfo o2) {
+ if (o1 == o2)
+ return 0;
+ if (o1.size < o2.size)
+ return -1;
+ if (o1.size > o2.size)
+ return 1;
+ return o1.path.compareTo(o2.path);
+ }
+ });
+
+ double ratio = tableConf.getFraction(Property.TABLE_MAJC_RATIO);
+ int maxFilesToCompact = tableConf.getCount(Property.TSERV_MAJC_THREAD_MAXOPEN);
+ int maxFilesPerTablet = tableConf.getMaxFilesPerTablet();
+
+ for (Entry<String,DataFileValue> entry : tabletFiles.entrySet()) {
+ candidateFiles.add(new MapFileInfo(entry.getKey(), entry.getValue().getSize()));
+ }
+
+ long totalSize = 0;
+ for (MapFileInfo mfi : candidateFiles) {
+ totalSize += mfi.size;
+ }
+
+ Map<String,Long> files = new HashMap<String,Long>();
+
+ while (candidateFiles.size() > 1) {
+ MapFileInfo max = candidateFiles.last();
+ if (max.size * ratio <= totalSize) {
+ files.clear();
+ for (MapFileInfo mfi : candidateFiles) {
+ files.put(mfi.path, mfi.size);
+ if (files.size() >= maxFilesToCompact)
+ break;
+ }
+
+ break;
+ }
+ totalSize -= max.size;
+ candidateFiles.remove(max);
+ }
+
+ int totalFilesToCompact = 0;
+ if (tabletFiles.size() > maxFilesPerTablet)
+ totalFilesToCompact = tabletFiles.size() - maxFilesPerTablet + 1;
+
+ totalFilesToCompact = Math.min(totalFilesToCompact, maxFilesToCompact);
+
+ if (files.size() < totalFilesToCompact) {
+
+ TreeMap<String,DataFileValue> tfc = new TreeMap<String,DataFileValue>(tabletFiles);
+ tfc.keySet().removeAll(files.keySet());
+
+ // put data in candidateFiles to sort it
+ candidateFiles.clear();
+ for (Entry<String,DataFileValue> entry : tfc.entrySet())
+ candidateFiles.add(new MapFileInfo(entry.getKey(), entry.getValue().getSize()));
+
+ for (MapFileInfo mfi : candidateFiles) {
+ files.put(mfi.path, mfi.size);
+ if (files.size() >= totalFilesToCompact)
+ break;
+ }
+ }
+
+ if (files.size() == 0)
+ return null;
+
+ return files;
+ }
+
+ boolean needsMajorCompaction(SortedMap<String,DataFileValue> tabletFiles, MajorCompactionReason reason) {
+ if (closed)
+ return false;// throw new IOException("closed");
+
+ // int threshold;
+
+ if (reason == MajorCompactionReason.USER)
+ return true;
+
+ if (reason == MajorCompactionReason.IDLE) {
+ // threshold = 1;
+ long idleTime;
+ if (lastReportedCommitTime == 0) {
+ // no commits, so compute how long the tablet has been assigned to the
+ // tablet server
+ idleTime = System.currentTimeMillis() - creationTime;
+ } else {
+ idleTime = System.currentTimeMillis() - lastReportedCommitTime;
+ }
+
+ if (idleTime < tableConf.getTimeInMillis(Property.TABLE_MAJC_COMPACTALL_IDLETIME)) {
+ return false;
+ }
+ }/*
+ * else{ threshold = tableConf.getCount(Property.TABLE_MAJC_THRESHOLD); }
+ */
+
+ return findMapFilesToCompact(tabletFiles, reason) != null;
+ }
+
+ // END methods that Tablets call to make decisions about major compaction
+
+ // tablets call this method to run minor compactions,
+ // this allows us to control how many minor compactions
+ // run concurrently in a tablet server
+ void executeMinorCompaction(final Runnable r) {
+ minorCompactionThreadPool.execute(new LoggingRunnable(log, r));
+ }
+
+ void close() throws IOException {
+ // always obtain locks in same order to avoid deadlock
+ synchronized (TabletServerResourceManager.this) {
+ synchronized (this) {
+ if (closed)
+ throw new IOException("closed");
+ if (openFilesReserved)
+ throw new IOException("tired to close files while open files reserved");
+
+ TabletServerResourceManager.this.removeTabletResource(this);
+
+ memMgmt.tabletClosed(tablet.getExtent());
+ memoryManager.tabletClosed(tablet.getExtent());
+
+ closed = true;
+ }
+ }
+ }
+
+ public TabletServerResourceManager getTabletServerResourceManager() {
+ return TabletServerResourceManager.this;
+ }
+
+ public void executeMajorCompaction(KeyExtent tablet, Runnable compactionTask) {
+ TabletServerResourceManager.this.executeMajorCompaction(tablet, compactionTask);
+ }
+
+ }
+
+ public void executeSplit(KeyExtent tablet, Runnable splitTask) {
+ if (tablet.isMeta()) {
+ if (tablet.isRootTablet()) {
+ log.warn("Saw request to split root tablet, ignoring");
+ return;
+ }
+ defaultSplitThreadPool.execute(splitTask);
+ } else {
+ splitThreadPool.execute(splitTask);
+ }
+ }
+
+ public void executeMajorCompaction(KeyExtent tablet, Runnable compactionTask) {
+ if (tablet.equals(Constants.ROOT_TABLET_EXTENT)) {
+ rootMajorCompactionThreadPool.execute(compactionTask);
+ } else if (tablet.isMeta()) {
+ defaultMajorCompactionThreadPool.execute(compactionTask);
+ } else {
+ majorCompactionThreadPool.execute(compactionTask);
+ }
+ }
+
+ public void executeReadAhead(KeyExtent tablet, Runnable task) {
+ if (tablet.isRootTablet()) {
+ task.run();
+ } else if (tablet.isMeta()) {
+ defaultReadAheadThreadPool.execute(task);
+ } else {
+ readAheadThreadPool.execute(task);
+ }
+ }
+
+ public void addAssignment(Runnable assignmentHandler) {
+ assignmentPool.execute(assignmentHandler);
+ }
+
+ public void addMetaDataAssignment(Runnable assignmentHandler) {
+ assignMetaDataPool.execute(assignmentHandler);
+ }
+
+ public void addMigration(KeyExtent tablet, Runnable migrationHandler) {
+ if (tablet.isRootTablet()) {
+ migrationHandler.run();
+ } else if (tablet.isMeta()) {
+ defaultMigrationPool.execute(migrationHandler);
+ } else {
+ migrationPool.execute(migrationHandler);
+ }
+ }
+
+ public void stopSplits() {
+ splitThreadPool.shutdown();
+ defaultSplitThreadPool.shutdown();
+ while (true) {
+ try {
+ while (!splitThreadPool.awaitTermination(1, TimeUnit.MINUTES)) {
+ log.info("Waiting for metadata split thread pool to stop");
+ }
+ while (!defaultSplitThreadPool.awaitTermination(1, TimeUnit.MINUTES)) {
+ log.info("Waiting for split thread pool to stop");
+ }
+ break;
+ } catch (InterruptedException ex) {
+ log.info(ex, ex);
+ }
+ }
+ }
+
+ public void stopNormalAssignments() {
+ assignmentPool.shutdown();
+ while (true) {
+ try {
+ while (!assignmentPool.awaitTermination(1, TimeUnit.MINUTES)) {
+ log.info("Waiting for assignment thread pool to stop");
+ }
+ break;
+ } catch (InterruptedException ex) {
+ log.info(ex, ex);
+ }
+ }
+ }
+
+ public void stopMetadataAssignments() {
+ assignMetaDataPool.shutdown();
+ while (true) {
+ try {
+ while (!assignMetaDataPool.awaitTermination(1, TimeUnit.MINUTES)) {
+ log.info("Waiting for metadata assignment thread pool to stop");
+ }
+ break;
+ } catch (InterruptedException ex) {
+ log.info(ex, ex);
+ }
+ }
+ }
+
+ public LruBlockCache getIndexCache() {
+ return _iCache;
+ }
+
+ public LruBlockCache getDataCache() {
+ return _dCache;
+ }
+
+}
[23/26] git commit: ACCUMULO-2475 recovery failed due to using the
wrong filesystem
Posted by el...@apache.org.
ACCUMULO-2475 recovery failed due to using the wrong filesystem
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ae750622
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ae750622
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ae750622
Branch: refs/heads/ACCUMULO-2061
Commit: ae750622c2baf4761afd9354540383f3b01ab6e2
Parents: 6dd1bd3
Author: Eric Newton <er...@gmail.com>
Authored: Fri Mar 14 12:55:32 2014 -0400
Committer: Eric Newton <er...@gmail.com>
Committed: Fri Mar 14 12:55:32 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ae750622/test/src/test/java/org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java b/test/src/test/java/org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java
index 86c5004..7cb1fe5 100644
--- a/test/src/test/java/org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java
@@ -46,6 +46,7 @@ import org.apache.accumulo.server.master.state.TabletLocationState;
import org.apache.accumulo.test.functional.ConfigurableMacIT;
import org.apache.accumulo.test.functional.FunctionalTestUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.Text;
import org.junit.Test;
@@ -56,6 +57,8 @@ public class MasterRepairsDualAssignmentIT extends ConfigurableMacIT {
public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
cfg.setProperty(Property.MASTER_RECOVERY_DELAY, "5s");
+ // use raw local file system so walogs sync and flush will work
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
}
@Test(timeout = 5 * 60 * 1000)
[10/26] git commit: ACCUMULO-2061 Propagate errors on delete or
mkdirs failure.
Posted by el...@apache.org.
ACCUMULO-2061 Propagate errors on delete or mkdirs failure.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/560af51c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/560af51c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/560af51c
Branch: refs/heads/ACCUMULO-2061
Commit: 560af51cac6e38da4829706292fa7b3f0d576336
Parents: e3adf78
Author: Josh Elser <el...@apache.org>
Authored: Wed Mar 12 16:19:19 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Wed Mar 12 16:19:19 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/server/util/ChangeSecret.java | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/560af51c/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
index f0dcd14..2926a3f 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
@@ -147,8 +147,14 @@ public class ChangeSecret {
// Need to recreate the instanceId on all of them to keep consistency
for (Volume v : fs.getVolumes()) {
final Path instanceId = ServerConstants.getInstanceIdLocation(v);
- v.getFileSystem().delete(instanceId, true);
- v.getFileSystem().mkdirs(instanceId);
+ if (!v.getFileSystem().delete(instanceId, true)) {
+ throw new IOException("Could not recursively delete " + instanceId);
+ }
+
+ if (!v.getFileSystem().mkdirs(instanceId)) {
+ throw new IOException("Could not create directory " + instanceId);
+ }
+
v.getFileSystem().create(new Path(instanceId, newInstanceId)).close();
}
}
[21/26] git commit: ACCUMULO-2472 update readme to list conditional
mutations as part of API
Posted by el...@apache.org.
ACCUMULO-2472 update readme to list conditional mutations as part of API
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a3172ac7
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a3172ac7
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a3172ac7
Branch: refs/heads/ACCUMULO-2061
Commit: a3172ac7d71a4227d2f0d55438387156ad5c27cc
Parents: ec36ce0
Author: Keith Turner <kt...@apache.org>
Authored: Thu Mar 13 19:27:07 2014 -0400
Committer: Keith Turner <kt...@apache.org>
Committed: Thu Mar 13 19:27:07 2014 -0400
----------------------------------------------------------------------
README | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3172ac7/README
----------------------------------------------------------------------
diff --git a/README b/README
index 69aad57..66c4df0 100644
--- a/README
+++ b/README
@@ -358,8 +358,9 @@ redirected to the log dir.
The public accumulo API is composed of :
* everything under org.apache.accumulo.core.client, excluding impl packages
- * Key, Mutation, Value, and Range in org.apache.accumulo.core.data.
- * org.apache.accumulo.server.mini
+ * Key, Mutation, Value, Range, Condition, and ConditionalMutation in
+ org.apache.accumulo.core.data.
+ * org.apache.accumulo.server.mini, excluding impl package
To get started using accumulo review the example and the javadoc for the
packages and classes mentioned above.
[15/26] git commit: ACCUMULO-2194 Add delay for randomwalk Security
teardown
Posted by el...@apache.org.
ACCUMULO-2194 Add delay for randomwalk Security teardown
If two Security randomwalk tests run back-to-back, the second test may see that the
table user still exists even though it was removed when the first test was torn down.
This can happen if the user drop does not propagate through Zookeeper quickly enough.
This commit adds a delay to the end of the Security test to give ZK some time.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ea86b44d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ea86b44d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ea86b44d
Branch: refs/heads/ACCUMULO-2061
Commit: ea86b44dfd6f46d389a630beb33ac49b65d87cd7
Parents: d11acbe
Author: Bill Havanki <bh...@cloudera.com>
Authored: Wed Mar 12 10:51:06 2014 -0400
Committer: Bill Havanki <bh...@cloudera.com>
Committed: Thu Mar 13 09:30:09 2014 -0400
----------------------------------------------------------------------
.../accumulo/server/test/randomwalk/security/SecurityFixture.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ea86b44d/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/security/SecurityFixture.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/security/SecurityFixture.java b/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/security/SecurityFixture.java
index c20e6db..144140b 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/security/SecurityFixture.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/security/SecurityFixture.java
@@ -94,5 +94,7 @@ public class SecurityFixture extends Fixture {
log.debug("Dropping user: " + systemUserName);
conn.securityOperations().dropUser(systemUserName);
+ // Allow user drops to propagate, in case a new security test starts
+ Thread.sleep(2000);
}
}
[04/26] git commit: ACCUMULO-2061 Formatting fixes
Posted by el...@apache.org.
ACCUMULO-2061 Formatting fixes
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/575e5418
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/575e5418
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/575e5418
Branch: refs/heads/ACCUMULO-2061
Commit: 575e54182f06ba19290e6a6f72c983dc189a42c1
Parents: 4ddaab8
Author: Josh Elser <el...@apache.org>
Authored: Wed Mar 12 12:12:05 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Wed Mar 12 12:12:05 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/volume/Volume.java | 4 +--
.../core/volume/VolumeConfiguration.java | 27 +++++++++++---------
.../apache/accumulo/core/volume/VolumeImpl.java | 19 +++++++-------
.../accumulo/server/fs/VolumeManager.java | 2 --
.../accumulo/server/fs/VolumeManagerImpl.java | 19 +++++++-------
.../apache/accumulo/server/fs/VolumeUtil.java | 2 +-
.../accumulo/server/fs/VolumeUtilTest.java | 2 +-
7 files changed, 38 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/575e5418/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/Volume.java b/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
index 08f61d4..487b699 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
@@ -21,7 +21,7 @@ import org.apache.hadoop.fs.Path;
/**
* Encapsulates a {@link FileSystem} and a base {@link Path} within that filesystem. This
- * also avoid the necessity to pass around a Configuration.
+ * also avoid the necessity to pass around a Configuration.
*/
public interface Volume {
@@ -36,7 +36,7 @@ public interface Volume {
* @return
*/
public String getBasePath();
-
+
/**
* Convert the given Path into a Path that is relative to the base path for this Volume
* @param p
http://git-wip-us.apache.org/repos/asf/accumulo/blob/575e5418/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
index 3005174..33d3c35 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
@@ -30,10 +30,10 @@ import org.apache.hadoop.fs.Path;
import com.google.common.base.Preconditions;
public class VolumeConfiguration {
-
+
public static Volume getVolume(String path, Configuration conf, AccumuloConfiguration acuconf) throws IOException {
Preconditions.checkNotNull(path);
-
+
if (path.contains(":")) {
// An absolute path
return create(new Path(path), conf);
@@ -64,7 +64,7 @@ public class VolumeConfiguration {
String singleNamespace = conf.get(Property.INSTANCE_DFS_DIR);
String dfsUri = conf.get(Property.INSTANCE_DFS_URI);
String baseDir;
-
+
if (dfsUri == null || dfsUri.isEmpty()) {
Configuration hadoopConfig = CachedConfiguration.getInstance();
try {
@@ -82,14 +82,15 @@ public class VolumeConfiguration {
/**
* Compute the URIs to be used by Accumulo
+ *
* @param conf
* @return
*/
public static String[] getVolumeUris(AccumuloConfiguration conf) {
String ns = conf.get(Property.INSTANCE_VOLUMES);
-
+
String configuredBaseDirs[];
-
+
if (ns == null || ns.isEmpty()) {
// Fall back to using the old config values
configuredBaseDirs = new String[] {getConfiguredBaseDir(conf)};
@@ -101,7 +102,7 @@ public class VolumeConfiguration {
if (!namespace.contains(":")) {
throw new IllegalArgumentException("Expected fully qualified URI for " + Property.INSTANCE_VOLUMES.getKey() + " got " + namespace);
}
-
+
try {
// pass through URI to unescape hex encoded chars (e.g. convert %2C to "," char)
configuredBaseDirs[i++] = new Path(new URI(namespace)).toString();
@@ -110,7 +111,7 @@ public class VolumeConfiguration {
}
}
}
-
+
return configuredBaseDirs;
}
@@ -126,24 +127,26 @@ public class VolumeConfiguration {
/**
* Create a Volume with the given FileSystem that writes to the default path
- * @param fs A FileSystem to write to
- * @return A Volume instance writing to the given FileSystem in the default path
+ *
+ * @param fs
+ * A FileSystem to write to
+ * @return A Volume instance writing to the given FileSystem in the default path
*/
@SuppressWarnings("deprecation")
public static <T extends FileSystem> Volume create(T fs, AccumuloConfiguration acuconf) {
String dfsDir = acuconf.get(Property.INSTANCE_DFS_DIR);
return new VolumeImpl(fs, null == dfsDir ? Property.INSTANCE_DFS_DIR.getDefaultValue() : dfsDir);
}
-
+
public static <T extends FileSystem> Volume create(T fs, String basePath) {
return new VolumeImpl(fs, basePath);
}
-
+
public static Volume create(String path, Configuration conf) throws IOException {
Preconditions.checkNotNull(path);
return create(new Path(path), conf);
}
-
+
public static Volume create(Path path, Configuration conf) throws IOException {
return new VolumeImpl(path, conf);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/575e5418/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java b/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
index 0aaf482..babdcfc 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
@@ -24,30 +24,29 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-
/**
*
*/
public class VolumeImpl implements Volume {
protected final FileSystem fs;
protected final String basePath;
-
+
public VolumeImpl(Path path, Configuration conf) throws IOException {
checkNotNull(path);
checkNotNull(conf);
-
+
this.fs = path.getFileSystem(conf);
this.basePath = path.toUri().getPath();
}
-
+
public VolumeImpl(FileSystem fs, String basePath) {
checkNotNull(fs);
checkNotNull(basePath);
-
+
this.fs = fs;
this.basePath = basePath;
}
-
+
@Override
public FileSystem getFileSystem() {
return fs;
@@ -66,20 +65,20 @@ public class VolumeImpl implements Volume {
@Override
public boolean isValidPath(Path p) {
checkNotNull(p);
-
+
return p.toUri().getPath().startsWith(basePath);
}
-
+
@Override
public boolean equals(Object o) {
if (o instanceof VolumeImpl) {
VolumeImpl other = (VolumeImpl) o;
return getFileSystem().equals(other.getFileSystem()) && getBasePath().equals(other.getBasePath());
}
-
+
return false;
}
-
+
@Override
public String toString() {
return getFileSystem() + " " + basePath;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/575e5418/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
index 9b8fb98..cbfdb5e 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
@@ -160,13 +160,11 @@ public interface VolumeManager {
/**
* Fetch the default Volume
- * @return
*/
public Volume getDefaultVolume();
/**
* Fetch the configured Volumes, excluding the default Volume
- * @return
*/
public Collection<Volume> getVolumes();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/575e5418/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
index ca5167d..b860f53 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
@@ -86,11 +86,12 @@ public class VolumeManagerImpl implements VolumeManager {
inverted.put(volume.getFileSystem(), volume);
}
}
-
+
public static org.apache.accumulo.server.fs.VolumeManager getLocal(String localBasePath) throws IOException {
AccumuloConfiguration accConf = DefaultConfiguration.getDefaultConfiguration();
Volume defaultLocalVolume = VolumeConfiguration.create(FileSystem.getLocal(CachedConfiguration.getInstance()), localBasePath);
-
+
+ // The default volume gets placed in the map, but local filesystem is only used for testing purposes
return new VolumeManagerImpl(Collections.singletonMap(DEFAULT, defaultLocalVolume), defaultLocalVolume, accConf);
}
@@ -112,16 +113,16 @@ public class VolumeManagerImpl implements VolumeManager {
@Override
public FSDataOutputStream create(Path path) throws IOException {
checkNotNull(path);
-
+
Volume v = getVolumeByPath(path);
-
+
return v.getFileSystem().create(path);
}
@Override
public FSDataOutputStream create(Path path, boolean overwrite) throws IOException {
checkNotNull(path);
-
+
Volume v = getVolumeByPath(path);
return v.getFileSystem().create(path, overwrite);
@@ -149,7 +150,7 @@ public class VolumeManagerImpl implements VolumeManager {
Volume v = getVolumeByPath(path);
FileSystem fs = v.getFileSystem();
-
+
if (bufferSize == 0) {
fs.getConf().getInt("io.file.buffer.size", 4096);
}
@@ -314,7 +315,7 @@ public class VolumeManagerImpl implements VolumeManager {
return defaultVolume;
}
-
+
log.debug("Could not determine volume for Path '" + path + "' from defined volumes");
} catch (IOException ex) {
throw new RuntimeException(ex);
@@ -404,7 +405,7 @@ public class VolumeManagerImpl implements VolumeManager {
// The "default" Volume for Accumulo (in case no volumes are specified)
for (String volumeUriOrDir : VolumeConfiguration.getVolumeUris(conf)) {
if (volumeUriOrDir.equals(DEFAULT))
- // Cannot re-define the default volume
+ // Cannot re-define the default volume
throw new IllegalArgumentException();
// We require a URI here, fail if it doesn't look like one
@@ -554,7 +555,7 @@ public class VolumeManagerImpl implements VolumeManager {
public Volume getDefaultVolume() {
return defaultVolume;
}
-
+
@Override
public Collection<Volume> getVolumes() {
return volumesByName.values();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/575e5418/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
index bcfb008..2ef438f 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
@@ -52,7 +52,7 @@ import org.apache.log4j.Logger;
public class VolumeUtil {
private static final Logger log = Logger.getLogger(VolumeUtil.class);
-
+
private static boolean isActiveVolume(Path dir) {
// consider relative path as active and take no action
http://git-wip-us.apache.org/repos/asf/accumulo/blob/575e5418/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
index 3b905c9..0013d04 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
@@ -202,7 +202,7 @@ public class VolumeUtilTest {
List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>();
replacements.add(new Pair<Path,Path>(new Path("file:/foo/v1"), new Path("file:/foo/v8")));
replacements.add(new Pair<Path,Path>(new Path("file:/foo/v2"), new Path("file:/foo/v9")));
-
+
FileType ft = FileType.TABLE;
Assert.assertEquals("file:/foo/v8/tables/+r/root_tablet",
[20/26] git commit: Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by el...@apache.org.
Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ec36ce07
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ec36ce07
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ec36ce07
Branch: refs/heads/ACCUMULO-2061
Commit: ec36ce07127678d8c1d3e27cb7fa181b578576ec
Parents: b66ee24 a99b0b6
Author: Bill Havanki <bh...@cloudera.com>
Authored: Thu Mar 13 17:04:13 2014 -0400
Committer: Bill Havanki <bh...@cloudera.com>
Committed: Thu Mar 13 17:04:13 2014 -0400
----------------------------------------------------------------------
pom.xml | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ec36ce07/pom.xml
----------------------------------------------------------------------
[12/26] git commit: ACCUMULO-2061 ref an actual ticket in the todo
Posted by el...@apache.org.
ACCUMULO-2061 ref an actual ticket in the todo
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5db49974
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5db49974
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5db49974
Branch: refs/heads/ACCUMULO-2061
Commit: 5db49974d2ba5e80103adeb7d6321d2b240285ec
Parents: 4b8a14e
Author: Josh Elser <el...@apache.org>
Authored: Wed Mar 12 16:40:35 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Wed Mar 12 16:40:35 2014 -0400
----------------------------------------------------------------------
.../main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5db49974/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
index 7c0f067..7ed8f34 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
@@ -52,7 +52,7 @@ public class PrintInfo {
@SuppressWarnings("deprecation")
AccumuloConfiguration aconf = AccumuloConfiguration.getSiteConfiguration();
- // TODO This will only work for RFiles in HDFS when the filesystem is defined in the core-site.xml
+ // TODO ACCUMULO-2462 This will only work for RFiles in HDFS when the filesystem is defined in the core-site.xml
// on the classpath if a path, and not a URI, is given
FileSystem hadoopFs = VolumeConfiguration.getDefaultVolume(conf, aconf).getFileSystem();
FileSystem localFs = FileSystem.getLocal(conf);
[17/26] git commit: Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by el...@apache.org.
Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/58ec7b1a
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/58ec7b1a
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/58ec7b1a
Branch: refs/heads/ACCUMULO-2061
Commit: 58ec7b1afa8a586ff8e96f08653f405d2be5a0ff
Parents: 7ed250a 58535fc
Author: Bill Havanki <bh...@cloudera.com>
Authored: Thu Mar 13 10:40:56 2014 -0400
Committer: Bill Havanki <bh...@cloudera.com>
Committed: Thu Mar 13 10:40:56 2014 -0400
----------------------------------------------------------------------
.../apache/accumulo/test/randomwalk/security/SecurityFixture.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/58ec7b1a/test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityFixture.java
----------------------------------------------------------------------
[09/26] git commit: ACCUMULO-2061 Javadoc
Posted by el...@apache.org.
ACCUMULO-2061 Javadoc
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e3adf78a
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e3adf78a
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e3adf78a
Branch: refs/heads/ACCUMULO-2061
Commit: e3adf78a722c131bba2de6d865664415f4601f4c
Parents: 96182f9
Author: Josh Elser <el...@apache.org>
Authored: Wed Mar 12 16:11:22 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Wed Mar 12 16:16:36 2014 -0400
----------------------------------------------------------------------
.../java/org/apache/accumulo/core/volume/Volume.java | 11 ++++-------
.../apache/accumulo/core/volume/VolumeConfiguration.java | 3 +++
.../java/org/apache/accumulo/core/volume/VolumeImpl.java | 3 ++-
3 files changed, 9 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e3adf78a/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/Volume.java b/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
index 487b699..17b2bf3 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
@@ -27,33 +27,30 @@ public interface Volume {
/**
* A {@link FileSystem} that Accumulo will use
- * @return
*/
public FileSystem getFileSystem();
/**
* The base path which Accumulo will use within the given {@link FileSystem}
- * @return
*/
public String getBasePath();
/**
* Convert the given Path into a Path that is relative to the base path for this Volume
- * @param p
- * @return
+ * @param p The suffix to use
+ * @return A Path for this Volume with the provided suffix
*/
public Path prefixChild(Path p);
/**
* Convert the given child path into a Path that is relative to the base path for this Volume
- * @param p
- * @return
+ * @param p The suffix to use
+ * @return A Path for this Volume with the provided suffix
*/
public Path prefixChild(String p);
/**
* Determine if the Path is valid on this Volume (contained by the basePath)
- * @param p
* @return True if path is contained within the basePath, false otherwise
*/
public boolean isValidPath(Path p);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e3adf78a/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
index 33d3c35..5db5bb2 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
@@ -59,6 +59,9 @@ public class VolumeConfiguration {
}
}
+ /**
+ * @see org.apache.accumulo.core.volume.VolumeConfiguration#getVolumeUris(AccumuloConfiguration)
+ */
@Deprecated
public static String getConfiguredBaseDir(AccumuloConfiguration conf) {
String singleNamespace = conf.get(Property.INSTANCE_DFS_DIR);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e3adf78a/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java b/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
index babdcfc..55ccfbc 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
@@ -25,7 +25,8 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
/**
- *
+ * Basic Volume implementation that contains a FileSystem and a base path
+ * that should be used within that filesystem.
*/
public class VolumeImpl implements Volume {
protected final FileSystem fs;
[03/26] git commit: ACCUMULO-2061 Add a warning about using paths
with PrintInfo
Posted by el...@apache.org.
ACCUMULO-2061 Add a warning about using paths with PrintInfo
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4ddaab80
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4ddaab80
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4ddaab80
Branch: refs/heads/ACCUMULO-2061
Commit: 4ddaab80828c2b0596cd6d63566b83844945d83a
Parents: 7086a7e
Author: Josh Elser <el...@apache.org>
Authored: Wed Mar 12 12:11:48 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Wed Mar 12 12:11:48 2014 -0400
----------------------------------------------------------------------
.../java/org/apache/accumulo/core/file/rfile/PrintInfo.java | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ddaab80/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
index 4e39fc7..7c0f067 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
@@ -31,10 +31,12 @@ import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
import com.beust.jcommander.Parameter;
public class PrintInfo {
+ private static final Logger log = Logger.getLogger(PrintInfo.class);
static class Opts extends Help {
@Parameter(names = {"-d", "--dump"}, description = "dump the key/value pairs")
@@ -50,6 +52,8 @@ public class PrintInfo {
@SuppressWarnings("deprecation")
AccumuloConfiguration aconf = AccumuloConfiguration.getSiteConfiguration();
+ // TODO This will only work for RFiles in HDFS when the filesystem is defined in the core-site.xml
+ // on the classpath if a path, and not a URI, is given
FileSystem hadoopFs = VolumeConfiguration.getDefaultVolume(conf, aconf).getFileSystem();
FileSystem localFs = FileSystem.getLocal(conf);
Opts opts = new Opts();
@@ -68,8 +72,11 @@ public class PrintInfo {
FileSystem fs;
if (arg.contains(":"))
fs = path.getFileSystem(conf);
- else
+ else {
+ // Recommend a URI is given for the above todo reason
+ log.warn("Attempting to find file across filesystems. Consider providing URI instead of path");
fs = hadoopFs.exists(path) ? hadoopFs : localFs; // fall back to local
+ }
CachableBlockFile.Reader _rdr = new CachableBlockFile.Reader(fs, path, conf, null, null, aconf);
Reader iter = new RFile.Reader(_rdr);
[05/26] git commit: ACCUMULO-2061 Stop recomputing the same thing 3
times
Posted by el...@apache.org.
ACCUMULO-2061 Stop recomputing the same thing 3 times
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7cb3b23a
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7cb3b23a
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7cb3b23a
Branch: refs/heads/ACCUMULO-2061
Commit: 7cb3b23ab4aed63547ad2d5be964e61b27e7fc47
Parents: 575e541
Author: Josh Elser <el...@apache.org>
Authored: Wed Mar 12 12:13:25 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Wed Mar 12 12:13:25 2014 -0400
----------------------------------------------------------------------
.../java/org/apache/accumulo/server/util/ChangeSecret.java | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7cb3b23a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
index 3f33a0e..f0dcd14 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
@@ -146,9 +146,10 @@ public class ChangeSecret {
private static void updateHdfs(VolumeManager fs, Instance inst, String newInstanceId) throws IOException {
// Need to recreate the instanceId on all of them to keep consistency
for (Volume v : fs.getVolumes()) {
- v.getFileSystem().delete(ServerConstants.getInstanceIdLocation(v), true);
- v.getFileSystem().mkdirs(ServerConstants.getInstanceIdLocation(v));
- v.getFileSystem().create(new Path(ServerConstants.getInstanceIdLocation(v), newInstanceId)).close();
+ final Path instanceId = ServerConstants.getInstanceIdLocation(v);
+ v.getFileSystem().delete(instanceId, true);
+ v.getFileSystem().mkdirs(instanceId);
+ v.getFileSystem().create(new Path(instanceId, newInstanceId)).close();
}
}
[25/26] git commit: Merge branch '1.6.0-SNAPSHOT' into ACCUMULO-2061
Posted by el...@apache.org.
Merge branch '1.6.0-SNAPSHOT' into ACCUMULO-2061
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0b4d5035
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0b4d5035
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0b4d5035
Branch: refs/heads/ACCUMULO-2061
Commit: 0b4d50358b3edfa0e42ecd9902c99ce2cdc8bedf
Parents: 5db4997 7c2a552
Author: Josh Elser <el...@apache.org>
Authored: Fri Mar 14 14:00:57 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Fri Mar 14 14:00:57 2014 -0400
----------------------------------------------------------------------
README | 5 +-
.../accumulo/core/data/ArrayByteSequence.java | 7 +-
.../apache/accumulo/core/data/KeyExtent.java | 3 +-
.../core/security/NamespacePermission.java | 1 +
.../core/data/ArrayByteSequenceTest.java | 107 +++++++++
.../accumulo/core/data/ByteSequenceTest.java | 41 ++++
.../apache/accumulo/core/data/ColumnTest.java | 70 ++++--
.../accumulo/core/data/KeyExtentTest.java | 240 ++++++++++++++-----
.../org/apache/accumulo/core/data/KeyTest.java | 33 +++
.../iterators/user/BigDecimalCombinerTest.java | 91 +++++--
.../core/iterators/user/CombinerTest.java | 2 +-
.../core/security/ColumnVisibilityTest.java | 34 ++-
.../core/security/NamespacePermissionsTest.java | 39 +++
.../core/security/VisibilityConstraintTest.java | 106 ++++++++
pom.xml | 7 +
.../tserver/TabletServerResourceManager.java | 19 +-
.../randomwalk/security/SecurityFixture.java | 2 +
.../test/MasterRepairsDualAssignmentIT.java | 3 +
18 files changed, 697 insertions(+), 113 deletions(-)
----------------------------------------------------------------------
[26/26] git commit: ACCUMULO-2061 Use URI instead of FileSystem as
the key to find correct Volumes and ensure that absolute URIs are still valid
even after they are not configured.
Posted by el...@apache.org.
ACCUMULO-2061 Use URI instead of FileSystem as the key to find correct Volumes and ensure that absolute URIs
are still valid even after they are not configured.
This will help ensure that FileSystem implementations' hashCode and equals don't
have the potential to collide but still provide unique access back to the Volumes
contained in the FileSystem. Added tests for the NonConfiguredVolume and also
for the no-longer-configured volumes.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/de5c29bc
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/de5c29bc
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/de5c29bc
Branch: refs/heads/ACCUMULO-2061
Commit: de5c29bc8f029f85f6abafa903b5dd7a9f84e27a
Parents: 0b4d503
Author: Josh Elser <el...@apache.org>
Authored: Fri Mar 14 17:06:32 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Fri Mar 14 17:06:32 2014 -0400
----------------------------------------------------------------------
.../core/volume/NonConfiguredVolume.java | 92 ++++++++++++++++++++
.../core/volume/NonConfiguredVolumeTest.java | 71 +++++++++++++++
.../accumulo/server/fs/VolumeManagerImpl.java | 25 +++---
.../java/org/apache/accumulo/test/VolumeIT.java | 55 +++++++++++-
4 files changed, 231 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/de5c29bc/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java b/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java
new file mode 100644
index 0000000..7dcbd88
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/volume/NonConfiguredVolume.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.volume;
+
+import java.io.IOException;
+
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Volume implementation which represents a Volume for which we have a FileSystem
+ * but no base path because it is not configured via {@link Property#INSTANCE_VOLUMES}
+ *
+ * This is useful to handle volumes that have been removed from accumulo-site.xml but references
+ * to these volumes have not been updated. This Volume should never be used to create new files,
+ * only to read existing files.
+ */
+public class NonConfiguredVolume implements Volume {
+ private static final Logger log = Logger.getLogger(NonConfiguredVolume.class);
+
+ private FileSystem fs;
+
+ public NonConfiguredVolume(FileSystem fs) {
+ this.fs = fs;
+ }
+
+ @Override
+ public FileSystem getFileSystem() {
+ return fs;
+ }
+
+ @Override
+ public String getBasePath() {
+ throw new UnsupportedOperationException("No base path known because this volume isn't configured in accumulo-site.xml");
+ }
+
+ @Override
+ public Path prefixChild(Path p) {
+ throw new UnsupportedOperationException("Cannot prefix path because this volume isn't configured in accumulo-site.xml");
+ }
+
+ @Override
+ public Path prefixChild(String p) {
+ throw new UnsupportedOperationException("Cannot prefix path because this volume isn't configured in accumulo-site.xml");
+ }
+
+ @Override
+ public boolean isValidPath(Path p) {
+ try {
+ return fs.equals(p.getFileSystem(CachedConfiguration.getInstance()));
+ } catch (IOException e) {
+ log.debug("Cannot determine FileSystem from path: " + p, e);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof NonConfiguredVolume) {
+ NonConfiguredVolume other = (NonConfiguredVolume) o;
+ return this.fs.equals(other.getFileSystem());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return "NonConfiguredVolume: " + this.fs.toString();
+ }
+
+ @Override
+ public int hashCode() {
+ return NonConfiguredVolume.class.hashCode() ^ this.fs.hashCode();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/accumulo/blob/de5c29bc/core/src/test/java/org/apache/accumulo/core/volume/NonConfiguredVolumeTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/volume/NonConfiguredVolumeTest.java b/core/src/test/java/org/apache/accumulo/core/volume/NonConfiguredVolumeTest.java
new file mode 100644
index 0000000..937baf8
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/volume/NonConfiguredVolumeTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.volume;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class NonConfiguredVolumeTest {
+
+ private NonConfiguredVolume volume;
+
+ @Before
+ public void create() throws IOException {
+ volume = new NonConfiguredVolume(FileSystem.getLocal(new Configuration()));
+ }
+
+ @Test
+ public void testSameFileSystem() throws IOException {
+ Assert.assertEquals(FileSystem.getLocal(new Configuration()), volume.getFileSystem());
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testGetBasePathFails() {
+ volume.getBasePath();
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testPrefixChildPath() {
+ volume.prefixChild(new Path("/foo"));
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testPrefixChildString() {
+ volume.prefixChild("/foo");
+ }
+
+ @Test
+ public void testEquality() throws IOException {
+ Volume newVolume = new NonConfiguredVolume(FileSystem.getLocal(new Configuration()));
+ Assert.assertEquals(volume, newVolume);
+ }
+
+ @Test
+ public void testHashCode() throws IOException {
+ Volume newVolume = new NonConfiguredVolume(FileSystem.getLocal(new Configuration()));
+ Assert.assertEquals(volume.hashCode(), newVolume.hashCode());
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/de5c29bc/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
index b860f53..64a6390 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
@@ -38,6 +38,7 @@ import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.volume.NonConfiguredVolume;
import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.server.client.HdfsZooInstance;
@@ -65,7 +66,7 @@ public class VolumeManagerImpl implements VolumeManager {
private static final Logger log = Logger.getLogger(VolumeManagerImpl.class);
Map<String,Volume> volumesByName;
- Multimap<FileSystem,Volume> volumesByFileSystem;
+ Multimap<URI,Volume> volumesByFileSystemUri;
Volume defaultVolume;
AccumuloConfiguration conf;
VolumeChooser chooser;
@@ -74,16 +75,16 @@ public class VolumeManagerImpl implements VolumeManager {
this.volumesByName = volumes;
this.defaultVolume = defaultVolume;
// We may have multiple directories used in a single FileSystem (e.g. testing)
- this.volumesByFileSystem = HashMultimap.create();
- invertVolumesByFileSystem(volumesByName, volumesByFileSystem);
+ this.volumesByFileSystemUri = HashMultimap.create();
+ invertVolumesByFileSystem(volumesByName, volumesByFileSystemUri);
this.conf = conf;
ensureSyncIsEnabled();
chooser = Property.createInstanceFromPropertyName(conf, Property.GENERAL_VOLUME_CHOOSER, VolumeChooser.class, new RandomVolumeChooser());
}
- private void invertVolumesByFileSystem(Map<String,Volume> forward, Multimap<FileSystem,Volume> inverted) {
+ private void invertVolumesByFileSystem(Map<String,Volume> forward, Multimap<URI,Volume> inverted) {
for (Volume volume : forward.values()) {
- inverted.put(volume.getFileSystem(), volume);
+ inverted.put(volume.getFileSystem().getUri(), volume);
}
}
@@ -299,8 +300,9 @@ public class VolumeManagerImpl implements VolumeManager {
public Volume getVolumeByPath(Path path) {
if (path.toString().contains(":")) {
try {
- FileSystem pathFs = path.getFileSystem(CachedConfiguration.getInstance());
- Collection<Volume> candidateVolumes = volumesByFileSystem.get(pathFs);
+ FileSystem desiredFs = path.getFileSystem(CachedConfiguration.getInstance());
+ URI desiredFsUri = desiredFs.getUri();
+ Collection<Volume> candidateVolumes = volumesByFileSystemUri.get(desiredFsUri);
if (null != candidateVolumes) {
for (Volume candidateVolume : candidateVolumes) {
if (candidateVolume.isValidPath(path)) {
@@ -309,11 +311,12 @@ public class VolumeManagerImpl implements VolumeManager {
}
// For the same reason as we can have multiple Volumes within a single filesystem
- // we could also not find a matching one. We should defer back to the defaultVolume
- // e.g. volume rename with old path references
- log.debug("Defaulting to " + defaultVolume + " as a valid volume could not be determined for " + path);
+ // we could also not find a matching one. We should still provide a Volume with the
+ // correct FileSystem even though we don't know what the proper base dir is
+ // e.g. Files on volumes that are now removed
+ log.debug("Found no configured Volume for the given path: " + path);
- return defaultVolume;
+ return new NonConfiguredVolume(desiredFs);
}
log.debug("Could not determine volume for Path '" + path + "' from defined volumes");
http://git-wip-us.apache.org/repos/asf/accumulo/blob/de5c29bc/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
index a7f7556..6c1da2c 100644
--- a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
@@ -307,6 +307,59 @@ public class VolumeIT extends ConfigurableMacIT {
}
+ @Test
+ public void testNonConfiguredVolumes() throws Exception {
+
+ String[] tableNames = getTableNames(2);
+
+ // grab this before shutting down cluster
+ String uuid = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()).getInstanceID();
+
+ verifyVolumesUsed(tableNames[0], false, v1, v2);
+
+ Assert.assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
+ cluster.stop();
+
+ Configuration conf = new Configuration(false);
+ conf.addResource(new Path(cluster.getConfig().getConfDir().toURI().toString(), "accumulo-site.xml"));
+
+ File v3f = new File(volDirBase, "v3");
+ v3f.mkdir();
+ Path v3 = new Path("file://" + v3f.getAbsolutePath());
+
+ conf.set(Property.INSTANCE_VOLUMES.getKey(), v2.toString() + "," + v3.toString());
+ BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(new File(cluster.getConfig().getConfDir(), "accumulo-site.xml")));
+ conf.writeXml(fos);
+ fos.close();
+
+ // initialize volume
+ Assert.assertEquals(0, cluster.exec(Initialize.class, "--add-volumes").waitFor());
+
+ // check that all volumes are initialized
+ for (Path volumePath : Arrays.asList(v1, v2, v3)) {
+ FileSystem fs = volumePath.getFileSystem(CachedConfiguration.getInstance());
+ Path vp = new Path(volumePath, ServerConstants.INSTANCE_ID_DIR);
+ FileStatus[] iids = fs.listStatus(vp);
+ Assert.assertEquals(1, iids.length);
+ Assert.assertEquals(uuid, iids[0].getPath().getName());
+ }
+
+ // start cluster and verify that new volume is used
+ cluster.start();
+
+ // Make sure we can still read the tables (tableNames[0] is very likely to have a file still on v1)
+ List<String> expected = new ArrayList<String>();
+ for (int i = 0; i < 100; i++) {
+ String row = String.format("%06d", i * 100 + 3);
+ expected.add(row + ":cf1:cq1:1");
+ }
+
+ verifyData(expected, getConnector().createScanner(tableNames[0], Authorizations.EMPTY));
+
+ // v1 should not have any data for tableNames[1]
+ verifyVolumesUsed(tableNames[1], false, v2, v3);
+ }
+
private void writeData(String tableName, Connector conn) throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException,
MutationsRejectedException {
TreeSet<Text> splits = new TreeSet<Text>();
@@ -331,7 +384,7 @@ public class VolumeIT extends ConfigurableMacIT {
private void verifyVolumesUsed(String tableName, boolean shouldExist, Path... paths) throws AccumuloException, AccumuloSecurityException,
TableExistsException, TableNotFoundException, MutationsRejectedException {
- Connector conn = cluster.getConnector("root", ROOT_PASSWORD);
+ Connector conn = getConnector();
List<String> expected = new ArrayList<String>();
for (int i = 0; i < 100; i++) {
[11/26] git commit: ACCUMULO-2061 Only append a trailing slash when
one doesn't exist on the base dir
Posted by el...@apache.org.
ACCUMULO-2061 Only append a trailing slash when one doesn't exist on the base dir
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4b8a14e3
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4b8a14e3
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4b8a14e3
Branch: refs/heads/ACCUMULO-2061
Commit: 4b8a14e38734acb5b8b89c1e1f48acc680b246ac
Parents: 560af51
Author: Josh Elser <el...@apache.org>
Authored: Wed Mar 12 16:25:19 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Wed Mar 12 16:25:19 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/volume/VolumeConfiguration.java | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4b8a14e3/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
index 5db5bb2..71ad611 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
@@ -123,7 +123,11 @@ public class VolumeConfiguration {
suffix = suffix.substring(1);
String result[] = new String[bases.length];
for (int i = 0; i < bases.length; i++) {
- result[i] = bases[i] + "/" + suffix;
+ if (bases[i].endsWith("/")) {
+ result[i] = bases[i] + suffix;
+ } else {
+ result[i] = bases[i] + "/" + suffix;
+ }
}
return result;
}
[14/26] git commit: Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by el...@apache.org.
Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Conflicts:
server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7ed250ab
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7ed250ab
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7ed250ab
Branch: refs/heads/ACCUMULO-2061
Commit: 7ed250ab99c233a4a709a4f59640cf54b921fcaa
Parents: c657c57 bf0b7f7
Author: Bill Havanki <bh...@cloudera.com>
Authored: Wed Mar 12 16:54:10 2014 -0400
Committer: Bill Havanki <bh...@cloudera.com>
Committed: Wed Mar 12 16:54:10 2014 -0400
----------------------------------------------------------------------
.../tserver/TabletServerResourceManager.java | 19 ++++++++++++-------
1 file changed, 12 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7ed250ab/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
index e958437,0000000..d4bc0fd
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
@@@ -1,717 -1,0 +1,722 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.file.blockfile.cache.LruBlockCache;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.LoggingRunnable;
+import org.apache.accumulo.core.util.NamingThreadFactory;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.tabletserver.LargestFirstMemoryManager;
+import org.apache.accumulo.server.tabletserver.MemoryManagementActions;
+import org.apache.accumulo.server.tabletserver.MemoryManager;
+import org.apache.accumulo.server.tabletserver.TabletState;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.trace.instrument.TraceExecutorService;
+import org.apache.accumulo.tserver.FileManager.ScanFileManager;
+import org.apache.accumulo.tserver.Tablet.MinorCompactionReason;
+import org.apache.accumulo.tserver.compaction.CompactionStrategy;
+import org.apache.accumulo.tserver.compaction.DefaultCompactionStrategy;
+import org.apache.accumulo.tserver.compaction.MajorCompactionReason;
+import org.apache.accumulo.tserver.compaction.MajorCompactionRequest;
+import org.apache.log4j.Logger;
+
+/**
+ * ResourceManager is responsible for managing the resources of all tablets within a tablet server.
+ *
+ *
+ *
+ */
+public class TabletServerResourceManager {
+
+ private ExecutorService minorCompactionThreadPool;
+ private ExecutorService majorCompactionThreadPool;
+ private ExecutorService rootMajorCompactionThreadPool;
+ private ExecutorService defaultMajorCompactionThreadPool;
+ private ExecutorService splitThreadPool;
+ private ExecutorService defaultSplitThreadPool;
+ private ExecutorService defaultMigrationPool;
+ private ExecutorService migrationPool;
+ private ExecutorService assignmentPool;
+ private ExecutorService assignMetaDataPool;
+ private ExecutorService readAheadThreadPool;
+ private ExecutorService defaultReadAheadThreadPool;
+ private Map<String,ExecutorService> threadPools = new TreeMap<String,ExecutorService>();
+
+ private HashSet<TabletResourceManager> tabletResources;
+
+ private final VolumeManager fs;
+
+ private FileManager fileManager;
+
+ private MemoryManager memoryManager;
+
+ private MemoryManagementFramework memMgmt;
+
+ private final LruBlockCache _dCache;
+ private final LruBlockCache _iCache;
+ private final ServerConfiguration conf;
+
+ private static final Logger log = Logger.getLogger(TabletServerResourceManager.class);
+
+ private ExecutorService addEs(String name, ExecutorService tp) {
+ if (threadPools.containsKey(name)) {
+ throw new IllegalArgumentException("Cannot create two executor services with same name " + name);
+ }
+ tp = new TraceExecutorService(tp);
+ threadPools.put(name, tp);
+ return tp;
+ }
+
+ private ExecutorService addEs(final Property maxThreads, String name, final ThreadPoolExecutor tp) {
+ ExecutorService result = addEs(name, tp);
+ SimpleTimer.getInstance().schedule(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ int max = conf.getConfiguration().getCount(maxThreads);
+ if (tp.getMaximumPoolSize() != max) {
+ log.info("Changing " + maxThreads.getKey() + " to " + max);
+ tp.setCorePoolSize(max);
+ tp.setMaximumPoolSize(max);
+ }
+ } catch (Throwable t) {
+ log.error(t, t);
+ }
+ }
+
+ }, 1000, 10 * 1000);
+ return result;
+ }
+
+ private ExecutorService createEs(int max, String name) {
+ return addEs(name, Executors.newFixedThreadPool(max, new NamingThreadFactory(name)));
+ }
+
+ private ExecutorService createEs(Property max, String name) {
+ return createEs(max, name, new LinkedBlockingQueue<Runnable>());
+ }
+
+ private ExecutorService createEs(Property max, String name, BlockingQueue<Runnable> queue) {
+ int maxThreads = conf.getConfiguration().getCount(max);
+ ThreadPoolExecutor tp = new ThreadPoolExecutor(maxThreads, maxThreads, 0L, TimeUnit.MILLISECONDS, queue, new NamingThreadFactory(name));
+ return addEs(max, name, tp);
+ }
+
+ private ExecutorService createEs(int min, int max, int timeout, String name) {
+ return addEs(name, new ThreadPoolExecutor(min, max, timeout, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), new NamingThreadFactory(name)));
+ }
+
+ public TabletServerResourceManager(Instance instance, VolumeManager fs) {
+ this.conf = new ServerConfiguration(instance);
+ this.fs = fs;
+ final AccumuloConfiguration acuConf = conf.getConfiguration();
+
+ long maxMemory = acuConf.getMemoryInBytes(Property.TSERV_MAXMEM);
+ boolean usingNativeMap = acuConf.getBoolean(Property.TSERV_NATIVEMAP_ENABLED) && NativeMap.isLoaded();
+
+ long blockSize = acuConf.getMemoryInBytes(Property.TSERV_DEFAULT_BLOCKSIZE);
+ long dCacheSize = acuConf.getMemoryInBytes(Property.TSERV_DATACACHE_SIZE);
+ long iCacheSize = acuConf.getMemoryInBytes(Property.TSERV_INDEXCACHE_SIZE);
+
+ _iCache = new LruBlockCache(iCacheSize, blockSize);
+ _dCache = new LruBlockCache(dCacheSize, blockSize);
+
+ Runtime runtime = Runtime.getRuntime();
+ if (!usingNativeMap && maxMemory + dCacheSize + iCacheSize > runtime.maxMemory()) {
+ throw new IllegalArgumentException(String.format(
+ "Maximum tablet server map memory %,d and block cache sizes %,d is too large for this JVM configuration %,d", maxMemory, dCacheSize + iCacheSize,
+ runtime.maxMemory()));
+ }
+ runtime.gc();
+
+ // totalMemory - freeMemory = memory in use
+ // maxMemory - memory in use = max available memory
+ if (!usingNativeMap && maxMemory > runtime.maxMemory() - (runtime.totalMemory() - runtime.freeMemory())) {
+ log.warn("In-memory map may not fit into local memory space.");
+ }
+
+ minorCompactionThreadPool = createEs(Property.TSERV_MINC_MAXCONCURRENT, "minor compactor");
+
+ // make this thread pool have a priority queue... and execute tablets with the most
+ // files first!
+ majorCompactionThreadPool = createEs(Property.TSERV_MAJC_MAXCONCURRENT, "major compactor", new CompactionQueue());
+ rootMajorCompactionThreadPool = createEs(0, 1, 300, "md root major compactor");
+ defaultMajorCompactionThreadPool = createEs(0, 1, 300, "md major compactor");
+
+ splitThreadPool = createEs(1, "splitter");
+ defaultSplitThreadPool = createEs(0, 1, 60, "md splitter");
+
+ defaultMigrationPool = createEs(0, 1, 60, "metadata tablet migration");
+ migrationPool = createEs(Property.TSERV_MIGRATE_MAXCONCURRENT, "tablet migration");
+
+ // not sure if concurrent assignments can run safely... even if they could there is probably no benefit at startup because
+ // individual tablet servers are already running assignments concurrently... having each individual tablet server run
+ // concurrent assignments would put more load on the metadata table at startup
+ assignmentPool = createEs(1, "tablet assignment");
+
+ assignMetaDataPool = createEs(0, 1, 60, "metadata tablet assignment");
+
+ readAheadThreadPool = createEs(Property.TSERV_READ_AHEAD_MAXCONCURRENT, "tablet read ahead");
+ defaultReadAheadThreadPool = createEs(Property.TSERV_METADATA_READ_AHEAD_MAXCONCURRENT, "metadata tablets read ahead");
+
+ tabletResources = new HashSet<TabletResourceManager>();
+
+ int maxOpenFiles = acuConf.getCount(Property.TSERV_SCAN_MAX_OPENFILES);
+
+ fileManager = new FileManager(conf, fs, maxOpenFiles, _dCache, _iCache);
+
+ memoryManager = Property.createInstanceFromPropertyName(acuConf, Property.TSERV_MEM_MGMT, MemoryManager.class, new LargestFirstMemoryManager());
+ memoryManager.init(conf);
+ memMgmt = new MemoryManagementFramework();
++ memMgmt.startThreads();
+ }
+
+ private static class TabletStateImpl implements TabletState, Cloneable {
+
+ private long lct;
+ private Tablet tablet;
+ private long mts;
+ private long mcmts;
+
+ public TabletStateImpl(Tablet t, long mts, long lct, long mcmts) {
+ this.tablet = t;
+ this.mts = mts;
+ this.lct = lct;
+ this.mcmts = mcmts;
+ }
+
+ @Override
+ public KeyExtent getExtent() {
+ return tablet.getExtent();
+ }
+
+ Tablet getTablet() {
+ return tablet;
+ }
+
+ @Override
+ public long getLastCommitTime() {
+ return lct;
+ }
+
+ @Override
+ public long getMemTableSize() {
+ return mts;
+ }
+
+ @Override
+ public long getMinorCompactingMemTableSize() {
+ return mcmts;
+ }
+ }
+
+ private class MemoryManagementFramework {
+ private final Map<KeyExtent,TabletStateImpl> tabletReports;
+ private LinkedBlockingQueue<TabletStateImpl> memUsageReports;
+ private long lastMemCheckTime = System.currentTimeMillis();
+ private long maxMem;
++ private Thread memoryGuardThread;
++ private Thread minorCompactionInitiatorThread;
+
+ MemoryManagementFramework() {
+ tabletReports = Collections.synchronizedMap(new HashMap<KeyExtent,TabletStateImpl>());
+ memUsageReports = new LinkedBlockingQueue<TabletStateImpl>();
+ maxMem = conf.getConfiguration().getMemoryInBytes(Property.TSERV_MAXMEM);
+
+ Runnable r1 = new Runnable() {
+ @Override
+ public void run() {
+ processTabletMemStats();
+ }
+ };
+
- Thread t1 = new Daemon(new LoggingRunnable(log, r1));
- t1.setPriority(Thread.NORM_PRIORITY + 1);
- t1.setName("Accumulo Memory Guard");
- t1.start();
++ memoryGuardThread = new Daemon(new LoggingRunnable(log, r1));
++ memoryGuardThread.setPriority(Thread.NORM_PRIORITY + 1);
++ memoryGuardThread.setName("Accumulo Memory Guard");
+
+ Runnable r2 = new Runnable() {
+ @Override
+ public void run() {
+ manageMemory();
+ }
+ };
+
- Thread t2 = new Daemon(new LoggingRunnable(log, r2));
- t2.setName("Accumulo Minor Compaction Initiator");
- t2.start();
++ minorCompactionInitiatorThread = new Daemon(new LoggingRunnable(log, r2));
++ minorCompactionInitiatorThread.setName("Accumulo Minor Compaction Initiator");
++ }
+
++ void startThreads() {
++ memoryGuardThread.start();
++ minorCompactionInitiatorThread.start();
+ }
+
+ private long lastMemTotal = 0;
+
+ private void processTabletMemStats() {
+ while (true) {
+ try {
+
+ TabletStateImpl report = memUsageReports.take();
+
+ while (report != null) {
+ tabletReports.put(report.getExtent(), report);
+ report = memUsageReports.poll();
+ }
+
+ long delta = System.currentTimeMillis() - lastMemCheckTime;
+ if (holdCommits || delta > 50 || lastMemTotal > 0.90 * maxMem) {
+ lastMemCheckTime = System.currentTimeMillis();
+
+ long totalMemUsed = 0;
+
+ synchronized (tabletReports) {
+ for (TabletStateImpl tsi : tabletReports.values()) {
+ totalMemUsed += tsi.getMemTableSize();
+ totalMemUsed += tsi.getMinorCompactingMemTableSize();
+ }
+ }
+
+ if (totalMemUsed > 0.95 * maxMem) {
+ holdAllCommits(true);
+ } else {
+ holdAllCommits(false);
+ }
+
+ lastMemTotal = totalMemUsed;
+ }
+
+ } catch (InterruptedException e) {
+ log.warn(e, e);
+ }
+ }
+ }
+
+ private void manageMemory() {
+ while (true) {
+ MemoryManagementActions mma = null;
+
+ try {
+ ArrayList<TabletState> tablets;
+ synchronized (tabletReports) {
+ tablets = new ArrayList<TabletState>(tabletReports.values());
+ }
+ mma = memoryManager.getMemoryManagementActions(tablets);
+
+ } catch (Throwable t) {
+ log.error("Memory manager failed " + t.getMessage(), t);
+ }
+
+ try {
+ if (mma != null && mma.tabletsToMinorCompact != null && mma.tabletsToMinorCompact.size() > 0) {
+ for (KeyExtent keyExtent : mma.tabletsToMinorCompact) {
+ TabletStateImpl tabletReport = tabletReports.get(keyExtent);
+
+ if (tabletReport == null) {
+ log.warn("Memory manager asked to compact nonexistant tablet " + keyExtent);
+ continue;
+ }
+
+ if (!tabletReport.getTablet().initiateMinorCompaction(MinorCompactionReason.SYSTEM)) {
+ if (tabletReport.getTablet().isClosed()) {
+ tabletReports.remove(tabletReport.getExtent());
+ log.debug("Ignoring memory manager recommendation: not minor compacting closed tablet " + keyExtent);
+ } else {
+ log.info("Ignoring memory manager recommendation: not minor compacting " + keyExtent);
+ }
+ }
+ }
+
+ // log.debug("mma.tabletsToMinorCompact = "+mma.tabletsToMinorCompact);
+ }
+ } catch (Throwable t) {
+ log.error("Minor compactions for memory managment failed", t);
+ }
+
+ UtilWaitThread.sleep(250);
+ }
+ }
+
+ public void updateMemoryUsageStats(Tablet tablet, long size, long lastCommitTime, long mincSize) {
+ memUsageReports.add(new TabletStateImpl(tablet, size, lastCommitTime, mincSize));
+ }
+
+ public void tabletClosed(KeyExtent extent) {
+ tabletReports.remove(extent);
+ }
+ }
+
+ private final Object commitHold = new Object();
+ private volatile boolean holdCommits = false;
+ private long holdStartTime;
+
+ protected void holdAllCommits(boolean holdAllCommits) {
+ synchronized (commitHold) {
+ if (holdCommits != holdAllCommits) {
+ holdCommits = holdAllCommits;
+
+ if (holdCommits) {
+ holdStartTime = System.currentTimeMillis();
+ }
+
+ if (!holdCommits) {
+ log.debug(String.format("Commits held for %6.2f secs", (System.currentTimeMillis() - holdStartTime) / 1000.0));
+ commitHold.notifyAll();
+ }
+ }
+ }
+
+ }
+
+ void waitUntilCommitsAreEnabled() {
+ if (holdCommits) {
+ long timeout = System.currentTimeMillis() + conf.getConfiguration().getTimeInMillis(Property.GENERAL_RPC_TIMEOUT);
+ synchronized (commitHold) {
+ while (holdCommits) {
+ try {
+ if (System.currentTimeMillis() > timeout)
+ throw new HoldTimeoutException("Commits are held");
+ commitHold.wait(1000);
+ } catch (InterruptedException e) {}
+ }
+ }
+ }
+ }
+
+ public long holdTime() {
+ if (!holdCommits)
+ return 0;
+ synchronized (commitHold) {
+ return System.currentTimeMillis() - holdStartTime;
+ }
+ }
+
+ public void close() {
+ for (ExecutorService executorService : threadPools.values()) {
+ executorService.shutdown();
+ }
+
+ for (Entry<String,ExecutorService> entry : threadPools.entrySet()) {
+ while (true) {
+ try {
+ if (entry.getValue().awaitTermination(60, TimeUnit.SECONDS))
+ break;
+ log.info("Waiting for thread pool " + entry.getKey() + " to shutdown");
+ } catch (InterruptedException e) {
+ log.warn(e);
+ }
+ }
+ }
+ }
+
+ public synchronized TabletResourceManager createTabletResourceManager() {
+ TabletResourceManager trm = new TabletResourceManager();
+ return trm;
+ }
+
+ synchronized private void addTabletResource(TabletResourceManager tr) {
+ tabletResources.add(tr);
+ }
+
+ synchronized private void removeTabletResource(TabletResourceManager tr) {
+ tabletResources.remove(tr);
+ }
+
+ public class TabletResourceManager {
+
+ private final long creationTime = System.currentTimeMillis();
+
+ private volatile boolean openFilesReserved = false;
+
+ private volatile boolean closed = false;
+
+ private Tablet tablet;
+
+ private AccumuloConfiguration tableConf;
+
+ TabletResourceManager() {}
+
+ void setTablet(Tablet tablet, AccumuloConfiguration tableConf) {
+ this.tablet = tablet;
+ this.tableConf = tableConf;
+ // TabletResourceManager is not really initialized until this
+ // function is called.... so do not make it publicly available
+ // until now
+
+ addTabletResource(this);
+ }
+
+ // BEGIN methods that Tablets call to manage their set of open map files
+
+ public void importedMapFiles() {
+ lastReportedCommitTime = System.currentTimeMillis();
+ }
+
+ synchronized ScanFileManager newScanFileManager() {
+ if (closed)
+ throw new IllegalStateException("closed");
+ return fileManager.newScanFileManager(tablet.getExtent());
+ }
+
+ // END methods that Tablets call to manage their set of open map files
+
+ // BEGIN methods that Tablets call to manage memory
+
+ private AtomicLong lastReportedSize = new AtomicLong();
+ private AtomicLong lastReportedMincSize = new AtomicLong();
+ private volatile long lastReportedCommitTime = 0;
+
+ public void updateMemoryUsageStats(long size, long mincSize) {
+
+ // do not want to update stats for every little change,
+ // so only do it under certain circumstances... the reason
+ // for this is that reporting stats acquires a lock, do
+ // not want all tablets locking on the same lock for every
+ // commit
+ long totalSize = size + mincSize;
+ long lrs = lastReportedSize.get();
+ long delta = totalSize - lrs;
+ long lrms = lastReportedMincSize.get();
+ boolean report = false;
+ // the atomic longs are considered independently, when one is set
+ // the other is not set intentionally because this method is not
+ // synchronized... therefore there are not transactional semantics
+ // for reading and writing two variables
+ if ((lrms > 0 && mincSize == 0 || lrms == 0 && mincSize > 0) && lastReportedMincSize.compareAndSet(lrms, mincSize)) {
+ report = true;
+ }
+
+ long currentTime = System.currentTimeMillis();
+ if ((delta > 32000 || delta < 0 || (currentTime - lastReportedCommitTime > 1000)) && lastReportedSize.compareAndSet(lrs, totalSize)) {
+ if (delta > 0)
+ lastReportedCommitTime = currentTime;
+ report = true;
+ }
+
+ if (report)
+ memMgmt.updateMemoryUsageStats(tablet, size, lastReportedCommitTime, mincSize);
+ }
+
+ // END methods that Tablets call to manage memory
+
+ // BEGIN methods that Tablets call to make decisions about major compaction
+ // when too many files are open, we may want tablets to compact down
+ // to one map file
+ boolean needsMajorCompaction(SortedMap<FileRef,DataFileValue> tabletFiles, MajorCompactionReason reason) {
+ if (closed)
+ return false;// throw new IOException("closed");
+
+ // int threshold;
+
+ if (reason == MajorCompactionReason.USER)
+ return true;
+
+ if (reason == MajorCompactionReason.IDLE) {
+ // threshold = 1;
+ long idleTime;
+ if (lastReportedCommitTime == 0) {
+ // no commits, so compute how long the tablet has been assigned to the
+ // tablet server
+ idleTime = System.currentTimeMillis() - creationTime;
+ } else {
+ idleTime = System.currentTimeMillis() - lastReportedCommitTime;
+ }
+
+ if (idleTime < tableConf.getTimeInMillis(Property.TABLE_MAJC_COMPACTALL_IDLETIME)) {
+ return false;
+ }
+ }
+ CompactionStrategy strategy = Property.createInstanceFromPropertyName(tableConf, Property.TABLE_COMPACTION_STRATEGY, CompactionStrategy.class,
+ new DefaultCompactionStrategy());
+ strategy.init(Property.getCompactionStrategyOptions(tableConf));
+ MajorCompactionRequest request = new MajorCompactionRequest(tablet.getExtent(), reason, TabletServerResourceManager.this.fs, tableConf);
+ request.setFiles(tabletFiles);
+ try {
+ return strategy.shouldCompact(request);
+ } catch (IOException ex) {
+ return false;
+ }
+ }
+
+ // END methods that Tablets call to make decisions about major compaction
+
+ // tablets call this method to run minor compactions,
+ // this allows us to control how many minor compactions
+ // run concurrently in a tablet server
+ void executeMinorCompaction(final Runnable r) {
+ minorCompactionThreadPool.execute(new LoggingRunnable(log, r));
+ }
+
+ void close() throws IOException {
+ // always obtain locks in same order to avoid deadlock
+ synchronized (TabletServerResourceManager.this) {
+ synchronized (this) {
+ if (closed)
+ throw new IOException("closed");
+ if (openFilesReserved)
+ throw new IOException("tired to close files while open files reserved");
+
+ TabletServerResourceManager.this.removeTabletResource(this);
+
+ memMgmt.tabletClosed(tablet.getExtent());
+ memoryManager.tabletClosed(tablet.getExtent());
+
+ closed = true;
+ }
+ }
+ }
+
+ public TabletServerResourceManager getTabletServerResourceManager() {
+ return TabletServerResourceManager.this;
+ }
+
+ public void executeMajorCompaction(KeyExtent tablet, Runnable compactionTask) {
+ TabletServerResourceManager.this.executeMajorCompaction(tablet, compactionTask);
+ }
+
+ }
+
+ public void executeSplit(KeyExtent tablet, Runnable splitTask) {
+ if (tablet.isMeta()) {
+ if (tablet.isRootTablet()) {
+ log.warn("Saw request to split root tablet, ignoring");
+ return;
+ }
+ defaultSplitThreadPool.execute(splitTask);
+ } else {
+ splitThreadPool.execute(splitTask);
+ }
+ }
+
+ public void executeMajorCompaction(KeyExtent tablet, Runnable compactionTask) {
+ if (tablet.isRootTablet()) {
+ rootMajorCompactionThreadPool.execute(compactionTask);
+ } else if (tablet.isMeta()) {
+ defaultMajorCompactionThreadPool.execute(compactionTask);
+ } else {
+ majorCompactionThreadPool.execute(compactionTask);
+ }
+ }
+
+ public void executeReadAhead(KeyExtent tablet, Runnable task) {
+ if (tablet.isRootTablet()) {
+ task.run();
+ } else if (tablet.isMeta()) {
+ defaultReadAheadThreadPool.execute(task);
+ } else {
+ readAheadThreadPool.execute(task);
+ }
+ }
+
+ public void addAssignment(Runnable assignmentHandler) {
+ assignmentPool.execute(assignmentHandler);
+ }
+
+ public void addMetaDataAssignment(Runnable assignmentHandler) {
+ assignMetaDataPool.execute(assignmentHandler);
+ }
+
+ public void addMigration(KeyExtent tablet, Runnable migrationHandler) {
+ if (tablet.isRootTablet()) {
+ migrationHandler.run();
+ } else if (tablet.isMeta()) {
+ defaultMigrationPool.execute(migrationHandler);
+ } else {
+ migrationPool.execute(migrationHandler);
+ }
+ }
+
+ public void stopSplits() {
+ splitThreadPool.shutdown();
+ defaultSplitThreadPool.shutdown();
+ while (true) {
+ try {
+ while (!splitThreadPool.awaitTermination(1, TimeUnit.MINUTES)) {
+ log.info("Waiting for metadata split thread pool to stop");
+ }
+ while (!defaultSplitThreadPool.awaitTermination(1, TimeUnit.MINUTES)) {
+ log.info("Waiting for split thread pool to stop");
+ }
+ break;
+ } catch (InterruptedException ex) {
+ log.info(ex, ex);
+ }
+ }
+ }
+
+ public void stopNormalAssignments() {
+ assignmentPool.shutdown();
+ while (true) {
+ try {
+ while (!assignmentPool.awaitTermination(1, TimeUnit.MINUTES)) {
+ log.info("Waiting for assignment thread pool to stop");
+ }
+ break;
+ } catch (InterruptedException ex) {
+ log.info(ex, ex);
+ }
+ }
+ }
+
+ public void stopMetadataAssignments() {
+ assignMetaDataPool.shutdown();
+ while (true) {
+ try {
+ while (!assignMetaDataPool.awaitTermination(1, TimeUnit.MINUTES)) {
+ log.info("Waiting for metadata assignment thread pool to stop");
+ }
+ break;
+ } catch (InterruptedException ex) {
+ log.info(ex, ex);
+ }
+ }
+ }
+
+ public LruBlockCache getIndexCache() {
+ return _iCache;
+ }
+
+ public LruBlockCache getDataCache() {
+ return _dCache;
+ }
+
+}
[24/26] git commit: Merge branch '1.6.0-SNAPSHOT' of
https://git-wip-us.apache.org/repos/asf/accumulo into 1.6.0-SNAPSHOT
Posted by el...@apache.org.
Merge branch '1.6.0-SNAPSHOT' of https://git-wip-us.apache.org/repos/asf/accumulo into 1.6.0-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7c2a5526
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7c2a5526
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7c2a5526
Branch: refs/heads/ACCUMULO-2061
Commit: 7c2a5526a1ce9b729e8a14ce9a34b457959bfa5e
Parents: ae75062 e68d748
Author: Eric Newton <er...@gmail.com>
Authored: Fri Mar 14 12:55:59 2014 -0400
Committer: Eric Newton <er...@gmail.com>
Committed: Fri Mar 14 12:55:59 2014 -0400
----------------------------------------------------------------------
README | 5 +-
.../accumulo/core/data/ArrayByteSequence.java | 7 +-
.../apache/accumulo/core/data/KeyExtent.java | 3 +-
.../core/security/NamespacePermission.java | 1 +
.../core/data/ArrayByteSequenceTest.java | 107 +++++++++
.../accumulo/core/data/ByteSequenceTest.java | 41 ++++
.../apache/accumulo/core/data/ColumnTest.java | 70 ++++--
.../accumulo/core/data/KeyExtentTest.java | 240 ++++++++++++++-----
.../org/apache/accumulo/core/data/KeyTest.java | 33 +++
.../iterators/user/BigDecimalCombinerTest.java | 91 +++++--
.../core/iterators/user/CombinerTest.java | 2 +-
.../core/security/ColumnVisibilityTest.java | 34 ++-
.../core/security/NamespacePermissionsTest.java | 39 +++
.../core/security/VisibilityConstraintTest.java | 106 ++++++++
pom.xml | 7 +
.../tserver/TabletServerResourceManager.java | 19 +-
.../randomwalk/security/SecurityFixture.java | 2 +
17 files changed, 694 insertions(+), 113 deletions(-)
----------------------------------------------------------------------
[22/26] git commit: ACCUMULO-2468 Add unit tests for o.a.a.core.data
Posted by el...@apache.org.
ACCUMULO-2468 Add unit tests for o.a.a.core.data
Add unit tests for several classes. Convert from JUnit 3 to 4 where
appropriate. Fix bugs where new unit tests uncovered them. Replaced
assertTrue with assertEquals where possible.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e68d748b
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e68d748b
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e68d748b
Branch: refs/heads/ACCUMULO-2061
Commit: e68d748b02c9d11f57767addb0daec905e10cd31
Parents: a3172ac
Author: Mike Drob <md...@cloudera.com>
Authored: Wed Mar 12 14:50:14 2014 -0400
Committer: Mike Drob <md...@cloudera.com>
Committed: Fri Mar 14 10:23:38 2014 -0400
----------------------------------------------------------------------
.../accumulo/core/data/ArrayByteSequence.java | 7 +-
.../apache/accumulo/core/data/KeyExtent.java | 3 +-
.../core/data/ArrayByteSequenceTest.java | 107 +++++++++
.../accumulo/core/data/ByteSequenceTest.java | 41 ++++
.../apache/accumulo/core/data/ColumnTest.java | 70 ++++--
.../accumulo/core/data/KeyExtentTest.java | 240 ++++++++++++++-----
.../org/apache/accumulo/core/data/KeyTest.java | 33 +++
7 files changed, 416 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e68d748b/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java b/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
index ff56c31..ca769cb 100644
--- a/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
@@ -52,12 +52,13 @@ public class ArrayByteSequence extends ByteSequence implements Serializable {
}
public ArrayByteSequence(ByteBuffer buffer) {
+ this.length = buffer.remaining();
+
if (buffer.hasArray()) {
this.data = buffer.array();
- this.offset = buffer.arrayOffset();
- this.length = buffer.limit();
+ this.offset = buffer.position();
} else {
- this.data = new byte[buffer.remaining()];
+ this.data = new byte[length];
this.offset = 0;
buffer.get(data);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e68d748b/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java b/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
index dda78fb..4c9978f 100644
--- a/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
@@ -301,9 +301,10 @@ public class KeyExtent implements WritableComparable<KeyExtent> {
/**
* Empty start or end rows tell the method there are no start or end rows, and to use all the keyextents that are before the end row if no start row etc.
*
+ * @deprecated this method not intended for public use and is likely to be removed in a future version.
* @return all the key extents that the rows cover
*/
-
+ @Deprecated
public static Collection<KeyExtent> getKeyExtentsForRange(Text startRow, Text endRow, Set<KeyExtent> kes) {
if (kes == null)
return Collections.emptyList();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e68d748b/core/src/test/java/org/apache/accumulo/core/data/ArrayByteSequenceTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/data/ArrayByteSequenceTest.java b/core/src/test/java/org/apache/accumulo/core/data/ArrayByteSequenceTest.java
new file mode 100644
index 0000000..70c40ed
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/data/ArrayByteSequenceTest.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.data;
+
+import static org.junit.Assert.assertEquals;
+
+import java.nio.ByteBuffer;
+
+import org.apache.accumulo.core.Constants;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ArrayByteSequenceTest {
+
+ ArrayByteSequence abs;
+ byte[] data;
+
+ @Before
+ public void setUp() {
+ data = new byte[] {'s', 'm', 'i', 'l', 'e', 's'};
+ abs = new ArrayByteSequence(data);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidByteBufferBounds0() {
+ abs = new ArrayByteSequence(data, -1, 0);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidByteBufferBounds1() {
+ abs = new ArrayByteSequence(data, data.length + 1, 0);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidByteBufferBounds2() {
+ abs = new ArrayByteSequence(data, 0, -1);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidByteBufferBounds3() {
+ abs = new ArrayByteSequence(data, 6, 2);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidByteAt0() {
+ abs.byteAt(-1);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidByteAt1() {
+ abs.byteAt(data.length);
+ }
+
+ @Test
+ public void testSubSequence() {
+ assertEquals(0, abs.subSequence(0, 0).length());
+ assertEquals("mile", abs.subSequence(1, 5).toString());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidSubsequence0() {
+ abs.subSequence(5, 1);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidSubsequence1() {
+ abs.subSequence(-1, 1);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidSubsequence3() {
+ abs.subSequence(0, 10);
+ }
+
+ @Test
+ public void testFromByteBuffer() {
+ ByteBuffer bb = ByteBuffer.wrap(data, 1, 4);
+ abs = new ArrayByteSequence(bb);
+
+ assertEquals("mile", abs.toString());
+
+ bb = bb.asReadOnlyBuffer();
+ abs = new ArrayByteSequence(bb);
+
+ assertEquals("mile", abs.toString());
+ }
+
+ @Test
+ public void testToString() {
+ assertEquals("String conversion should round trip correctly", "", new ArrayByteSequence("").toString());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e68d748b/core/src/test/java/org/apache/accumulo/core/data/ByteSequenceTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/data/ByteSequenceTest.java b/core/src/test/java/org/apache/accumulo/core/data/ByteSequenceTest.java
new file mode 100644
index 0000000..2cb60ab
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/data/ByteSequenceTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.data;
+
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+public class ByteSequenceTest {
+
+ @Test
+ public void testCompareBytes() {
+ ByteSequence a = new ArrayByteSequence("a");
+ ByteSequence b = new ArrayByteSequence("b");
+ ByteSequence abc = new ArrayByteSequence("abc");
+
+ assertLessThan(a, b);
+ assertLessThan(a, abc);
+ assertLessThan(abc, b);
+ }
+
+ private void assertLessThan(ByteSequence lhs, ByteSequence rhs) {
+ int result = ByteSequence.compareBytes(lhs, rhs);
+ assertTrue(result < 0);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e68d748b/core/src/test/java/org/apache/accumulo/core/data/ColumnTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/data/ColumnTest.java b/core/src/test/java/org/apache/accumulo/core/data/ColumnTest.java
index f040942..9071248 100644
--- a/core/src/test/java/org/apache/accumulo/core/data/ColumnTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/data/ColumnTest.java
@@ -16,11 +16,37 @@
*/
package org.apache.accumulo.core.data;
-import junit.framework.TestCase;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
-public class ColumnTest extends TestCase {
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.accumulo.core.data.thrift.TColumn;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class ColumnTest {
+ static Column col[];
+
+ @BeforeClass
+ public static void setup() {
+ col = new Column[5];
+ col[0] = new Column("colfam".getBytes(), "colq".getBytes(), "colv".getBytes());
+ col[1] = new Column("colfam".getBytes(), "colq".getBytes(), "colv".getBytes());
+ col[2] = new Column(new byte[0], new byte[0], new byte[0]);
+ col[3] = new Column(null, null, null);
+ col[4] = new Column("colfam".getBytes(), "cq".getBytes(), "cv".getBytes());
+ }
+
+
+ @Test
public void testEquals() {
- Column[] col = createColumns();
for (int i = 0; i < col.length; i++) {
for (int j = 0; j < col.length; j++) {
if (i == j || (i == 0 && j == 1) || (i == 1 && j == 0))
@@ -31,31 +57,43 @@ public class ColumnTest extends TestCase {
}
}
+ @Test
public void testCompare() {
- Column[] col = createColumns();
for (int i = 0; i < col.length; i++) {
for (int j = 0; j < col.length; j++) {
if (i == j || (i == 0 && j == 1) || (i == 1 && j == 0))
- assertTrue(col[i].compareTo(col[j]) == 0);
+ assertEquals(0, col[i].compareTo(col[j]));
else
- assertFalse(col[i].compareTo(col[j]) == 0);
+ assertNotEquals(0, col[i].compareTo(col[j]));
}
}
}
-
+
+ @Test
public void testEqualsCompare() {
- Column[] col = createColumns();
for (int i = 0; i < col.length; i++)
for (int j = 0; j < col.length; j++)
- assertTrue((col[i].compareTo(col[j]) == 0) == col[i].equals(col[j]));
+ assertEquals(col[i].equals(col[j]), col[i].compareTo(col[j]) == 0);
}
- public Column[] createColumns() {
- Column col[] = new Column[4];
- col[0] = new Column("colfam".getBytes(), "colq".getBytes(), "colv".getBytes());
- col[1] = new Column("colfam".getBytes(), "colq".getBytes(), "colv".getBytes());
- col[2] = new Column(new byte[0], new byte[0], new byte[0]);
- col[3] = new Column(null, null, null);
- return col;
+ @Test
+ public void testWriteReadFields() throws IOException {
+ for (Column c : col) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ c.write(new DataOutputStream(baos));
+
+ Column other = new Column();
+ other.readFields(new DataInputStream(new ByteArrayInputStream(baos.toByteArray())));
+
+ assertEquals(c, other);
+ }
+ }
+
+ @Test
+ public void testThriftRoundTrip() {
+ for (Column c : col) {
+ TColumn tc = c.toThrift();
+ assertEquals(c, new Column(tc));
+ }
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e68d748b/core/src/test/java/org/apache/accumulo/core/data/KeyExtentTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/data/KeyExtentTest.java b/core/src/test/java/org/apache/accumulo/core/data/KeyExtentTest.java
index 068ec89..ce4ad98 100644
--- a/core/src/test/java/org/apache/accumulo/core/data/KeyExtentTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/data/KeyExtentTest.java
@@ -16,86 +16,111 @@
*/
package org.apache.accumulo.core.data;
+import static org.hamcrest.CoreMatchers.hasItem;
+import static org.hamcrest.CoreMatchers.hasItems;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.TreeSet;
-import junit.framework.TestCase;
-
import org.apache.hadoop.io.Text;
+import org.junit.Before;
+import org.junit.Test;
-public class KeyExtentTest extends TestCase {
+public class KeyExtentTest {
KeyExtent nke(String t, String er, String per) {
return new KeyExtent(new Text(t), er == null ? null : new Text(er), per == null ? null : new Text(per));
}
+ KeyExtent ke;
+ TreeSet<KeyExtent> set0;
+
+ @Before
+ public void setup() {
+ set0 = new TreeSet<KeyExtent>();
+ }
+
+ @Test
public void testDecodingMetadataRow() {
Text flattenedExtent = new Text("foo;bar");
- KeyExtent ke = new KeyExtent(flattenedExtent, (Text) null);
+ ke = new KeyExtent(flattenedExtent, (Text) null);
- assertTrue(ke.getEndRow().equals(new Text("bar")));
- assertTrue(ke.getTableId().equals(new Text("foo")));
- assertTrue(ke.getPrevEndRow() == null);
+ assertEquals(new Text("bar"), ke.getEndRow());
+ assertEquals(new Text("foo"), ke.getTableId());
+ assertNull(ke.getPrevEndRow());
flattenedExtent = new Text("foo<");
ke = new KeyExtent(flattenedExtent, (Text) null);
- assertTrue(ke.getEndRow() == null);
- assertTrue(ke.getTableId().equals(new Text("foo")));
- assertTrue(ke.getPrevEndRow() == null);
+ assertNull(ke.getEndRow());
+ assertEquals(new Text("foo"), ke.getTableId());
+ assertNull(ke.getPrevEndRow());
flattenedExtent = new Text("foo;bar;");
ke = new KeyExtent(flattenedExtent, (Text) null);
- assertTrue(ke.getEndRow().equals(new Text("bar;")));
- assertTrue(ke.getTableId().equals(new Text("foo")));
- assertTrue(ke.getPrevEndRow() == null);
+ assertEquals(new Text("bar;"), ke.getEndRow());
+ assertEquals(new Text("foo"), ke.getTableId());
+ assertNull(ke.getPrevEndRow());
}
+ @Test
public void testFindContainingExtents() {
- TreeSet<KeyExtent> set0 = new TreeSet<KeyExtent>();
-
- assertTrue(KeyExtent.findContainingExtent(nke("t", null, null), set0) == null);
- assertTrue(KeyExtent.findContainingExtent(nke("t", "1", "0"), set0) == null);
- assertTrue(KeyExtent.findContainingExtent(nke("t", "1", null), set0) == null);
- assertTrue(KeyExtent.findContainingExtent(nke("t", null, "0"), set0) == null);
+ assertNull(KeyExtent.findContainingExtent(nke("t", null, null), set0));
+ assertNull(KeyExtent.findContainingExtent(nke("t", "1", "0"), set0));
+ assertNull(KeyExtent.findContainingExtent(nke("t", "1", null), set0));
+ assertNull(KeyExtent.findContainingExtent(nke("t", null, "0"), set0));
TreeSet<KeyExtent> set1 = new TreeSet<KeyExtent>();
set1.add(nke("t", null, null));
- assertTrue(KeyExtent.findContainingExtent(nke("t", null, null), set1).equals(nke("t", null, null)));
- assertTrue(KeyExtent.findContainingExtent(nke("t", "1", "0"), set1).equals(nke("t", null, null)));
- assertTrue(KeyExtent.findContainingExtent(nke("t", "1", null), set1).equals(nke("t", null, null)));
- assertTrue(KeyExtent.findContainingExtent(nke("t", null, "0"), set1).equals(nke("t", null, null)));
+ assertEquals(nke("t", null, null), KeyExtent.findContainingExtent(nke("t", null, null), set1));
+ assertEquals(nke("t", null, null), KeyExtent.findContainingExtent(nke("t", "1", "0"), set1));
+ assertEquals(nke("t", null, null), KeyExtent.findContainingExtent(nke("t", "1", null), set1));
+ assertEquals(nke("t", null, null), KeyExtent.findContainingExtent(nke("t", null, "0"), set1));
TreeSet<KeyExtent> set2 = new TreeSet<KeyExtent>();
set2.add(nke("t", "g", null));
set2.add(nke("t", null, "g"));
- assertTrue(KeyExtent.findContainingExtent(nke("t", null, null), set2) == null);
- assertTrue(KeyExtent.findContainingExtent(nke("t", "c", "a"), set2).equals(nke("t", "g", null)));
- assertTrue(KeyExtent.findContainingExtent(nke("t", "c", null), set2).equals(nke("t", "g", null)));
+ assertNull(KeyExtent.findContainingExtent(nke("t", null, null), set2));
+ assertEquals(nke("t", "g", null), KeyExtent.findContainingExtent(nke("t", "c", "a"), set2));
+ assertEquals(nke("t", "g", null), KeyExtent.findContainingExtent(nke("t", "c", null), set2));
- assertTrue(KeyExtent.findContainingExtent(nke("t", "g", "a"), set2).equals(nke("t", "g", null)));
- assertTrue(KeyExtent.findContainingExtent(nke("t", "g", null), set2).equals(nke("t", "g", null)));
+ assertEquals(nke("t", "g", null), KeyExtent.findContainingExtent(nke("t", "g", "a"), set2));
+ assertEquals(nke("t", "g", null), KeyExtent.findContainingExtent(nke("t", "g", null), set2));
- assertTrue(KeyExtent.findContainingExtent(nke("t", "h", "a"), set2) == null);
- assertTrue(KeyExtent.findContainingExtent(nke("t", "h", null), set2) == null);
+ assertNull(KeyExtent.findContainingExtent(nke("t", "h", "a"), set2));
+ assertNull(KeyExtent.findContainingExtent(nke("t", "h", null), set2));
- assertTrue(KeyExtent.findContainingExtent(nke("t", "z", "f"), set2) == null);
- assertTrue(KeyExtent.findContainingExtent(nke("t", null, "f"), set2) == null);
+ assertNull(KeyExtent.findContainingExtent(nke("t", "z", "f"), set2));
+ assertNull(KeyExtent.findContainingExtent(nke("t", null, "f"), set2));
- assertTrue(KeyExtent.findContainingExtent(nke("t", "z", "g"), set2).equals(nke("t", null, "g")));
- assertTrue(KeyExtent.findContainingExtent(nke("t", null, "g"), set2).equals(nke("t", null, "g")));
+ assertEquals(nke("t", null, "g"), KeyExtent.findContainingExtent(nke("t", "z", "g"), set2));
+ assertEquals(nke("t", null, "g"), KeyExtent.findContainingExtent(nke("t", null, "g"), set2));
- assertTrue(KeyExtent.findContainingExtent(nke("t", "z", "h"), set2).equals(nke("t", null, "g")));
- assertTrue(KeyExtent.findContainingExtent(nke("t", null, "h"), set2).equals(nke("t", null, "g")));
+ assertEquals(nke("t", null, "g"), KeyExtent.findContainingExtent(nke("t", "z", "h"), set2));
+ assertEquals(nke("t", null, "g"), KeyExtent.findContainingExtent(nke("t", null, "h"), set2));
TreeSet<KeyExtent> set3 = new TreeSet<KeyExtent>();
@@ -103,25 +128,25 @@ public class KeyExtentTest extends TestCase {
set3.add(nke("t", "s", "g"));
set3.add(nke("t", null, "s"));
- assertTrue(KeyExtent.findContainingExtent(nke("t", null, null), set3) == null);
+ assertNull(KeyExtent.findContainingExtent(nke("t", null, null), set3));
- assertTrue(KeyExtent.findContainingExtent(nke("t", "g", null), set3).equals(nke("t", "g", null)));
- assertTrue(KeyExtent.findContainingExtent(nke("t", "s", "g"), set3).equals(nke("t", "s", "g")));
- assertTrue(KeyExtent.findContainingExtent(nke("t", null, "s"), set3).equals(nke("t", null, "s")));
+ assertEquals(nke("t", "g", null), KeyExtent.findContainingExtent(nke("t", "g", null), set3));
+ assertEquals(nke("t", "s", "g"), KeyExtent.findContainingExtent(nke("t", "s", "g"), set3));
+ assertEquals(nke("t", null, "s"), KeyExtent.findContainingExtent(nke("t", null, "s"), set3));
- assertTrue(KeyExtent.findContainingExtent(nke("t", "t", "g"), set3) == null);
- assertTrue(KeyExtent.findContainingExtent(nke("t", "t", "f"), set3) == null);
- assertTrue(KeyExtent.findContainingExtent(nke("t", "s", "f"), set3) == null);
+ assertNull(KeyExtent.findContainingExtent(nke("t", "t", "g"), set3));
+ assertNull(KeyExtent.findContainingExtent(nke("t", "t", "f"), set3));
+ assertNull(KeyExtent.findContainingExtent(nke("t", "s", "f"), set3));
- assertTrue(KeyExtent.findContainingExtent(nke("t", "r", "h"), set3).equals(nke("t", "s", "g")));
- assertTrue(KeyExtent.findContainingExtent(nke("t", "s", "h"), set3).equals(nke("t", "s", "g")));
- assertTrue(KeyExtent.findContainingExtent(nke("t", "r", "g"), set3).equals(nke("t", "s", "g")));
+ assertEquals(nke("t", "s", "g"), KeyExtent.findContainingExtent(nke("t", "r", "h"), set3));
+ assertEquals(nke("t", "s", "g"), KeyExtent.findContainingExtent(nke("t", "s", "h"), set3));
+ assertEquals(nke("t", "s", "g"), KeyExtent.findContainingExtent(nke("t", "r", "g"), set3));
- assertTrue(KeyExtent.findContainingExtent(nke("t", null, "t"), set3).equals(nke("t", null, "s")));
- assertTrue(KeyExtent.findContainingExtent(nke("t", null, "r"), set3) == null);
+ assertEquals(nke("t", null, "s"), KeyExtent.findContainingExtent(nke("t", null, "t"), set3));
+ assertNull(KeyExtent.findContainingExtent(nke("t", null, "r"), set3));
- assertTrue(KeyExtent.findContainingExtent(nke("t", "f", null), set3).equals(nke("t", "g", null)));
- assertTrue(KeyExtent.findContainingExtent(nke("t", "h", null), set3) == null);
+ assertEquals(nke("t", "g", null), KeyExtent.findContainingExtent(nke("t", "f", null), set3));
+ assertNull(KeyExtent.findContainingExtent(nke("t", "h", null), set3));
TreeSet<KeyExtent> set4 = new TreeSet<KeyExtent>();
@@ -132,24 +157,24 @@ public class KeyExtentTest extends TestCase {
set4.add(nke("t2", "s", "g"));
set4.add(nke("t2", null, "s"));
- assertTrue(KeyExtent.findContainingExtent(nke("t", null, null), set4) == null);
- assertTrue(KeyExtent.findContainingExtent(nke("z", null, null), set4) == null);
- assertTrue(KeyExtent.findContainingExtent(nke("t11", null, null), set4) == null);
- assertTrue(KeyExtent.findContainingExtent(nke("t1", null, null), set4) == null);
- assertTrue(KeyExtent.findContainingExtent(nke("t2", null, null), set4) == null);
+ assertNull(KeyExtent.findContainingExtent(nke("t", null, null), set4));
+ assertNull(KeyExtent.findContainingExtent(nke("z", null, null), set4));
+ assertNull(KeyExtent.findContainingExtent(nke("t11", null, null), set4));
+ assertNull(KeyExtent.findContainingExtent(nke("t1", null, null), set4));
+ assertNull(KeyExtent.findContainingExtent(nke("t2", null, null), set4));
- assertTrue(KeyExtent.findContainingExtent(nke("t", "g", null), set4) == null);
- assertTrue(KeyExtent.findContainingExtent(nke("z", "g", null), set4) == null);
- assertTrue(KeyExtent.findContainingExtent(nke("t11", "g", null), set4) == null);
- assertTrue(KeyExtent.findContainingExtent(nke("t1", "g", null), set4) == null);
+ assertNull(KeyExtent.findContainingExtent(nke("t", "g", null), set4));
+ assertNull(KeyExtent.findContainingExtent(nke("z", "g", null), set4));
+ assertNull(KeyExtent.findContainingExtent(nke("t11", "g", null), set4));
+ assertNull(KeyExtent.findContainingExtent(nke("t1", "g", null), set4));
- assertTrue(KeyExtent.findContainingExtent(nke("t2", "g", null), set4).equals(nke("t2", "g", null)));
- assertTrue(KeyExtent.findContainingExtent(nke("t2", "s", "g"), set4).equals(nke("t2", "s", "g")));
- assertTrue(KeyExtent.findContainingExtent(nke("t2", null, "s"), set4).equals(nke("t2", null, "s")));
+ assertEquals(nke("t2", "g", null), KeyExtent.findContainingExtent(nke("t2", "g", null), set4));
+ assertEquals(nke("t2", "s", "g"), KeyExtent.findContainingExtent(nke("t2", "s", "g"), set4));
+ assertEquals(nke("t2", null, "s"), KeyExtent.findContainingExtent(nke("t2", null, "s"), set4));
- assertTrue(KeyExtent.findContainingExtent(nke("t1", "d", null), set4).equals(nke("t1", "d", null)));
- assertTrue(KeyExtent.findContainingExtent(nke("t1", "q", "d"), set4).equals(nke("t1", "q", "d")));
- assertTrue(KeyExtent.findContainingExtent(nke("t1", null, "q"), set4).equals(nke("t1", null, "q")));
+ assertEquals(nke("t1", "d", null), KeyExtent.findContainingExtent(nke("t1", "d", null), set4));
+ assertEquals(nke("t1", "q", "d"), KeyExtent.findContainingExtent(nke("t1", "q", "d"), set4));
+ assertEquals(nke("t1", null, "q"), KeyExtent.findContainingExtent(nke("t1", null, "q"), set4));
}
@@ -157,6 +182,7 @@ public class KeyExtentTest extends TestCase {
return !KeyExtent.findOverlapping(extent, extents).isEmpty();
}
+ @Test
public void testOverlaps() {
SortedMap<KeyExtent,Object> set0 = new TreeMap<KeyExtent,Object>();
set0.put(nke("a", null, null), null);
@@ -237,4 +263,88 @@ public class KeyExtentTest extends TestCase {
assertFalse(nke("t", null, "b").overlaps(nke("t", "b", "a")));
assertFalse(nke("t", null, "b").overlaps(nke("t", "b", null)));
}
+
+ @Test
+ public void testWriteReadFields() throws Exception {
+ ke = nke("t", "e", "b");
+ assertEquals(ke, writeAndReadFields(ke));
+
+ ke = nke("t", "e", null);
+ assertEquals(ke, writeAndReadFields(ke));
+
+ ke = nke("t", null, "b");
+ assertEquals(ke, writeAndReadFields(ke));
+
+ ke = nke("t", null, null);
+ assertEquals(ke, writeAndReadFields(ke));
+ }
+
+ private KeyExtent writeAndReadFields(KeyExtent in) throws IOException {
+ KeyExtent out = new KeyExtent();
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ in.write(new DataOutputStream(baos));
+
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ out.readFields(new DataInputStream(bais));
+
+ return out;
+ }
+
+ @Test
+ public void testKeyExtentsForSimpleRange() {
+ Collection<KeyExtent> results;
+
+ results = KeyExtent.getKeyExtentsForRange(null, null, null);
+ assertTrue("Non-empty set returned from no extents", results.isEmpty());
+
+ results = KeyExtent.getKeyExtentsForRange(null, null, Collections.<KeyExtent> emptySet());
+ assertTrue("Non-empty set returned from no extents", results.isEmpty());
+
+ KeyExtent t = nke("t", null, null);
+ results = KeyExtent.getKeyExtentsForRange(null, null, Collections.<KeyExtent> singleton(t));
+ assertEquals("Single tablet should always be returned", 1, results.size());
+ assertEquals(t, results.iterator().next());
+ }
+
+ @Test
+ public void testKeyExtentsForRange() {
+ KeyExtent b = nke("t", "b", null);
+ KeyExtent e = nke("t", "e", "b");
+ KeyExtent h = nke("t", "h", "e");
+ KeyExtent m = nke("t", "m", "h");
+ KeyExtent z = nke("t", null, "m");
+
+ set0.addAll(Arrays.asList(b, e, h, m, z));
+
+ Collection<KeyExtent> results;
+
+ results = KeyExtent.getKeyExtentsForRange(null, null, set0);
+ assertThat("infinite range should return full set", results.size(), is(5));
+ assertThat("infinite range should return full set", results, hasItems(b, e, h, m, z));
+
+ results = KeyExtent.getKeyExtentsForRange(new Text("a"), new Text("z"), set0);
+ assertThat("full overlap should return full set", results.size(), is(5));
+ assertThat("full overlap should return full set", results, hasItems(b, e, h, m, z));
+
+ results = KeyExtent.getKeyExtentsForRange(null, new Text("f"), set0);
+ assertThat("end row should return head set", results.size(), is(3));
+ assertThat("end row should return head set", results, hasItems(b, e, h));
+
+ results = KeyExtent.getKeyExtentsForRange(new Text("f"), null, set0);
+ assertThat("start row should return tail set", results.size(), is(3));
+ assertThat("start row should return tail set", results, hasItems(h, m, z));
+
+ results = KeyExtent.getKeyExtentsForRange(new Text("f"), new Text("g"), set0);
+ assertThat("slice should return correct subset", results.size(), is(1));
+ assertThat("slice should return correct subset", results, hasItem(h));
+ }
+
+ @Test
+ public void testDecodeEncode() {
+ assertNull(KeyExtent.decodePrevEndRow(KeyExtent.encodePrevEndRow(null)));
+
+ Text x = new Text();
+ assertEquals(x, KeyExtent.decodePrevEndRow(KeyExtent.encodePrevEndRow(x)));
+ }
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e68d748b/core/src/test/java/org/apache/accumulo/core/data/KeyTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/data/KeyTest.java b/core/src/test/java/org/apache/accumulo/core/data/KeyTest.java
index 56442a1..cc737db 100644
--- a/core/src/test/java/org/apache/accumulo/core/data/KeyTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/data/KeyTest.java
@@ -21,7 +21,12 @@ import static org.junit.Assert.assertNotSame;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
import org.apache.accumulo.core.data.thrift.TKey;
+import org.apache.accumulo.core.data.thrift.TKeyValue;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.hadoop.io.Text;
import org.junit.Test;
@@ -133,4 +138,32 @@ public class KeyTest {
tk.setRow((byte[]) null);
new Key(tk);
}
+
+ @Test
+ public void testCompressDecompress() {
+ List<KeyValue> kvs = new ArrayList<KeyValue>();
+ kvs.add(new KeyValue(new Key(), new byte[] {}));
+ kvs.add(new KeyValue(new Key("r"), new byte[] {}));
+ kvs.add(new KeyValue(new Key("r", "cf"), new byte[] {}));
+ kvs.add(new KeyValue(new Key("r2", "cf"), new byte[] {}));
+ kvs.add(new KeyValue(new Key("r", "cf", "cq"), new byte[] {}));
+ kvs.add(new KeyValue(new Key("r2", "cf2", "cq"), new byte[] {}));
+ kvs.add(new KeyValue(new Key("r", "cf", "cq", "cv"), new byte[] {}));
+ kvs.add(new KeyValue(new Key("r2", "cf2", "cq2", "cv"), new byte[] {}));
+ kvs.add(new KeyValue(new Key("r2", "cf2", "cq2", "cv"), new byte[] {}));
+ kvs.add(new KeyValue(new Key(), new byte[] {}));
+
+ List<TKeyValue> tkvs = Key.compress(kvs);
+ Key.decompress(tkvs);
+
+ assertEquals(kvs.size(), tkvs.size());
+ Iterator<KeyValue> kvi = kvs.iterator();
+ Iterator<TKeyValue> tkvi = tkvs.iterator();
+
+ while (kvi.hasNext()) {
+ KeyValue kv = kvi.next();
+ TKeyValue tkv = tkvi.next();
+ assertEquals(kv.getKey(), new Key(tkv.getKey()));
+ }
+ }
}
[02/26] git commit: ACCUMULO-2451 Update thet data version on all
volumes instead of just one
Posted by el...@apache.org.
ACCUMULO-2451 Update thet data version on all volumes instead of just one
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7086a7e7
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7086a7e7
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7086a7e7
Branch: refs/heads/ACCUMULO-2061
Commit: 7086a7e70c0c551ce41f9f55addaf3a1a22b1d06
Parents: db76dd7
Author: Josh Elser <el...@apache.org>
Authored: Wed Mar 12 12:10:44 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Wed Mar 12 12:10:44 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/server/Accumulo.java | 22 +++++++++++++-------
1 file changed, 14 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7086a7e7/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
index 48534f0..f7f2298 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
@@ -51,15 +51,21 @@ public class Accumulo {
private static final Logger log = Logger.getLogger(Accumulo.class);
public static synchronized void updateAccumuloVersion(VolumeManager fs) {
- // TODO ACCUMULO-2451 Should update all volumes, not one
- Volume volume = fs.getVolumes().iterator().next();
- try {
- if (getAccumuloPersistentVersion(fs) == ServerConstants.PREV_DATA_VERSION) {
- fs.create(new Path(ServerConstants.getDataVersionLocation(volume), Integer.toString(ServerConstants.DATA_VERSION)));
- fs.delete(new Path(ServerConstants.getDataVersionLocation(volume), Integer.toString(ServerConstants.PREV_DATA_VERSION)));
+ for (Volume volume : fs.getVolumes()) {
+ try {
+ if (getAccumuloPersistentVersion(fs) == ServerConstants.PREV_DATA_VERSION) {
+ log.debug("Attempting to upgrade " + volume);
+ Path dataVersionLocation = ServerConstants.getDataVersionLocation(volume);
+ fs.create(new Path(dataVersionLocation, Integer.toString(ServerConstants.DATA_VERSION))).close();
+
+ Path prevDataVersionLoc = new Path(dataVersionLocation, Integer.toString(ServerConstants.PREV_DATA_VERSION));
+ if (!fs.delete(prevDataVersionLoc)) {
+ throw new RuntimeException("Could not delete previous data version location (" + prevDataVersionLoc + ") for " + volume);
+ }
+ }
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to set accumulo version: an error occurred.", e);
}
- } catch (IOException e) {
- throw new RuntimeException("Unable to set accumulo version: an error occurred.", e);
}
}
[08/26] git commit: ACCUMULO-2444 unit tests for a.o.o.core.security
Posted by el...@apache.org.
ACCUMULO-2444 unit tests for a.o.o.core.security
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c657c575
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c657c575
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c657c575
Branch: refs/heads/ACCUMULO-2061
Commit: c657c5758a1ab1224abc3377066699eaf826dd62
Parents: 6dd1bd3
Author: Mike Drob <md...@cloudera.com>
Authored: Mon Mar 10 10:20:36 2014 -0400
Committer: Mike Drob <md...@cloudera.com>
Committed: Wed Mar 12 16:12:57 2014 -0400
----------------------------------------------------------------------
.../core/security/NamespacePermission.java | 1 +
.../core/security/ColumnVisibilityTest.java | 34 +++++-
.../core/security/NamespacePermissionsTest.java | 39 +++++++
.../core/security/VisibilityConstraintTest.java | 106 +++++++++++++++++++
4 files changed, 178 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c657c575/core/src/main/java/org/apache/accumulo/core/security/NamespacePermission.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/security/NamespacePermission.java b/core/src/main/java/org/apache/accumulo/core/security/NamespacePermission.java
index f9f7564..44cee0c 100644
--- a/core/src/main/java/org/apache/accumulo/core/security/NamespacePermission.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/NamespacePermission.java
@@ -80,6 +80,7 @@ public enum NamespacePermission {
* @throws IndexOutOfBoundsException
* if the byte ID is invalid
*/
+ // This method isn't used anywhere, why is it public API?
public static NamespacePermission getPermissionById(byte id) {
NamespacePermission result = mapping[id];
if (result != null)
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c657c575/core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java b/core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java
index 7a6a80d..931ff41 100644
--- a/core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java
@@ -19,10 +19,16 @@ package org.apache.accumulo.core.security;
import static org.apache.accumulo.core.security.ColumnVisibility.quote;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.util.Comparator;
+
+import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.security.ColumnVisibility.Node;
+import org.apache.accumulo.core.security.ColumnVisibility.NodeComparator;
import org.apache.accumulo.core.security.ColumnVisibility.NodeType;
+import org.apache.hadoop.io.Text;
import org.junit.Test;
public class ColumnVisibilityTest {
@@ -46,8 +52,14 @@ public class ColumnVisibilityTest {
@Test
public void testEmpty() {
// empty visibility is valid
- new ColumnVisibility();
- new ColumnVisibility(new byte[0]);
+ ColumnVisibility a = new ColumnVisibility();
+ ColumnVisibility b = new ColumnVisibility(new byte[0]);
+ ColumnVisibility c = new ColumnVisibility("");
+ ColumnVisibility d = new ColumnVisibility(new Text());
+
+ assertEquals(a, b);
+ assertEquals(a, c);
+ assertEquals(a, d);
}
@Test
@@ -205,6 +217,24 @@ public class ColumnVisibilityTest {
assertNode(node.getChildren().get(1).children.get(1), NodeType.TERM, 7, 8);
}
+ @Test
+ public void testEmptyParseTreesAreEqual() {
+ Comparator<Node> comparator = new NodeComparator(new byte[] {});
+ Node empty = new ColumnVisibility().getParseTree();
+ assertEquals(0, comparator.compare(empty, parse("")));
+ }
+
+ @Test
+ public void testParseTreesOrdering() {
+ byte[] expression = "(b&c&d)|((a|m)&y&z)|(e&f)".getBytes(Constants.UTF8);
+ byte[] flattened = new ColumnVisibility(expression).flatten();
+
+ // Convert to String for indexOf convenience
+ String flat = new String(flattened, Constants.UTF8);
+ assertTrue("shortest expressions sort first", flat.indexOf('e') < flat.indexOf('|'));
+ assertTrue("shortest children sort first", flat.indexOf('b') < flat.indexOf('a'));
+ }
+
private Node parse(String s) {
ColumnVisibility v = new ColumnVisibility(s);
return v.getParseTree();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c657c575/core/src/test/java/org/apache/accumulo/core/security/NamespacePermissionsTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/security/NamespacePermissionsTest.java b/core/src/test/java/org/apache/accumulo/core/security/NamespacePermissionsTest.java
new file mode 100644
index 0000000..223b3ca
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/security/NamespacePermissionsTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.security;
+
+import static org.junit.Assert.assertTrue;
+
+import java.util.EnumSet;
+
+import org.junit.Test;
+
+public class NamespacePermissionsTest {
+ @Test
+ public void testEnsureEquivalencies() {
+ EnumSet<NamespacePermission> set = EnumSet.allOf(NamespacePermission.class);
+
+ for (TablePermission permission : TablePermission.values()) {
+ set.remove(NamespacePermission.getEquivalent(permission));
+ }
+ for (SystemPermission permission : SystemPermission.values()) {
+ set.remove(NamespacePermission.getEquivalent(permission));
+ }
+
+ assertTrue("All namespace permissions should have equivalent table or system permissions.", set.isEmpty());
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c657c575/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java b/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
new file mode 100644
index 0000000..de6ca21
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.security;
+
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.constraints.Constraint.Environment;
+import org.apache.accumulo.core.data.ArrayByteSequence;
+import org.apache.accumulo.core.data.Mutation;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class VisibilityConstraintTest {
+
+ VisibilityConstraint vc;
+ Environment env;
+ Mutation mutation;
+
+ static final ColumnVisibility good = new ColumnVisibility("good");
+ static final ColumnVisibility bad = new ColumnVisibility("bad");
+
+ static final String D = "don't care";
+
+ static final List<Short> ENOAUTH = Arrays.asList((short) 2);
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @Before
+ public void setUp() throws Exception {
+ vc = new VisibilityConstraint();
+ mutation = new Mutation("r");
+
+ ArrayByteSequence bs = new ArrayByteSequence("good".getBytes(Constants.UTF8));
+
+ AuthorizationContainer ac = createNiceMock(AuthorizationContainer.class);
+ expect(ac.contains(bs)).andReturn(true);
+ replay(ac);
+
+ env = createMock(Environment.class);
+ expect(env.getAuthorizationsContainer()).andReturn(ac);
+ replay(env);
+ }
+
+ @Test
+ public void testNoVisibility() {
+ mutation.put(D, D, D);
+ assertNull("authorized", vc.check(env, mutation));
+ }
+
+ @Test
+ public void testVisibilityNoAuth() {
+ mutation.put(D, D, bad, D);
+ assertEquals("unauthorized", ENOAUTH, vc.check(env, mutation));
+ }
+
+ @Test
+ public void testGoodVisibilityAuth() {
+ mutation.put(D, D, good, D);
+ assertNull("authorized", vc.check(env, mutation));
+ }
+
+ @Test
+ public void testCachedVisibilities() {
+ mutation.put(D, D, good, "v");
+ mutation.put(D, D, good, "v2");
+ assertNull("authorized", vc.check(env, mutation));
+ }
+
+ @Test
+ public void testMixedVisibilities() {
+ mutation.put(D, D, bad, D);
+ mutation.put(D, D, good, D);
+ assertEquals("unauthorized", ENOAUTH, vc.check(env, mutation));
+ }
+
+ @Test
+ @Ignore
+ public void testMalformedVisibility() {
+ // TODO: ACCUMULO-1006 Should test for returning error code 1, but not sure how since ColumnVisibility won't let us construct a bad one in the first place
+ }
+}
[18/26] git commit: ACCUMULO-2465 Unit tests for BigDecimalCombiner
Posted by el...@apache.org.
ACCUMULO-2465 Unit tests for BigDecimalCombiner
Add tests for min and max. Refactor setup into @Before method.
Add keys that won't combine to make sure all values are still correct.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b66ee24c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b66ee24c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b66ee24c
Branch: refs/heads/ACCUMULO-2061
Commit: b66ee24cd101aeb1c49b5e6840bfa0e61cc60071
Parents: 58ec7b1
Author: Mike Drob <md...@cloudera.com>
Authored: Tue Mar 11 12:21:14 2014 -0400
Committer: Mike Drob <md...@cloudera.com>
Committed: Thu Mar 13 16:10:00 2014 -0400
----------------------------------------------------------------------
.../iterators/user/BigDecimalCombinerTest.java | 91 ++++++++++++++++----
.../core/iterators/user/CombinerTest.java | 2 +-
2 files changed, 76 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b66ee24c/core/src/test/java/org/apache/accumulo/core/iterators/user/BigDecimalCombinerTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/BigDecimalCombinerTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/BigDecimalCombinerTest.java
index 4cf4d5d..9e3d975 100644
--- a/core/src/test/java/org/apache/accumulo/core/iterators/user/BigDecimalCombinerTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/BigDecimalCombinerTest.java
@@ -25,9 +25,11 @@ import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.List;
import java.util.TreeMap;
import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.IteratorSetting.Column;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
@@ -35,39 +37,96 @@ import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.Combiner;
import org.apache.accumulo.core.iterators.SortedMapIterator;
import org.apache.accumulo.core.iterators.TypedValueCombiner.Encoder;
+import org.junit.Before;
import org.junit.Test;
public class BigDecimalCombinerTest {
private static final Collection<ByteSequence> EMPTY_COL_FAMS = new ArrayList<ByteSequence>();
private static double delta = 0.00001;
-
- @Test
- public void testSums() throws IOException {
- Encoder<BigDecimal> encoder = new BigDecimalCombiner.BigDecimalEncoder();
-
- TreeMap<Key,Value> tm1 = new TreeMap<Key,Value>();
-
- // keys that do not aggregate
+ Encoder<BigDecimal> encoder;
+ TreeMap<Key,Value> tm1;
+ List<Column> columns;
+ Combiner ai;
+
+
+ @Before
+ public void setup() {
+ encoder = new BigDecimalCombiner.BigDecimalEncoder();
+ tm1 = new TreeMap<Key,Value>();
+ columns = Collections.singletonList(new IteratorSetting.Column("cf001"));
+
+ // keys that will aggregate
CombinerTest.nkv(tm1, 1, 1, 1, 1, false, BigDecimal.valueOf(2), encoder);
CombinerTest.nkv(tm1, 1, 1, 1, 2, false, BigDecimal.valueOf(2.3), encoder);
CombinerTest.nkv(tm1, 1, 1, 1, 3, false, BigDecimal.valueOf(-1.4E1), encoder);
-
- Combiner ai = new BigDecimalCombiner.BigDecimalSummingCombiner();
+
+ // and keys that will not aggregate
+ CombinerTest.nkv(tm1, 1, 2, 1, 1, false, BigDecimal.valueOf(99), encoder);
+ CombinerTest.nkv(tm1, 1, 3, 1, 1, false, BigDecimal.valueOf(-88), encoder);
+ }
+
+ @Test
+ public void testSums() throws IOException {
+ ai = new BigDecimalCombiner.BigDecimalSummingCombiner();
IteratorSetting is = new IteratorSetting(1, BigDecimalCombiner.BigDecimalSummingCombiner.class);
- Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
+ Combiner.setColumns(is, columns);
+
+ ai.init(new SortedMapIterator(tm1), is.getOptions(), null);
+ ai.seek(new Range(), EMPTY_COL_FAMS, false);
+
+ assertTrue(ai.hasTop());
+ assertEquals(CombinerTest.nk(1, 1, 1, 3), ai.getTopKey());
+ assertEquals(-9.7, encoder.decode(ai.getTopValue().get()).doubleValue(), delta);
+
+ verify();
+ }
+
+ @Test
+ public void testMin() throws IOException {
+ ai = new BigDecimalCombiner.BigDecimalMinCombiner();
+ IteratorSetting is = new IteratorSetting(1, BigDecimalCombiner.BigDecimalMinCombiner.class);
+ Combiner.setColumns(is, columns);
ai.init(new SortedMapIterator(tm1), is.getOptions(), null);
ai.seek(new Range(), EMPTY_COL_FAMS, false);
-
+
assertTrue(ai.hasTop());
assertEquals(CombinerTest.nk(1, 1, 1, 3), ai.getTopKey());
- assertEquals(-9.7, encoder.decode(ai.getTopValue().get()).doubleValue(),delta);
-
+ assertEquals(-14.0, encoder.decode(ai.getTopValue().get()).doubleValue(), delta);
+
+ verify();
+ }
+
+ @Test
+ public void testMax() throws IOException {
+ ai = new BigDecimalCombiner.BigDecimalMaxCombiner();
+ IteratorSetting is = new IteratorSetting(1, BigDecimalCombiner.BigDecimalMaxCombiner.class);
+ Combiner.setColumns(is, columns);
+
+ ai.init(new SortedMapIterator(tm1), is.getOptions(), null);
+ ai.seek(new Range(), EMPTY_COL_FAMS, false);
+
+ assertTrue(ai.hasTop());
+ assertEquals(CombinerTest.nk(1, 1, 1, 3), ai.getTopKey());
+ assertEquals(2.3, encoder.decode(ai.getTopValue().get()).doubleValue(), delta);
+
+ verify();
+ }
+
+ private void verify() throws IOException {
+ ai.next(); // Skip the combined key, since we've already looked at it by now
+
+ // Should have exactly two more keys left over
+ assertEquals(CombinerTest.nk(1, 2, 1, 1), ai.getTopKey());
+ assertEquals(99.0, encoder.decode(ai.getTopValue().get()).doubleValue(), delta);
+ ai.next();
+
+ assertEquals(CombinerTest.nk(1, 3, 1, 1), ai.getTopKey());
+ assertEquals(-88.0, encoder.decode(ai.getTopValue().get()).doubleValue(), delta);
ai.next();
-
+
assertFalse(ai.hasTop());
}
-
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b66ee24c/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java
index 0c91ef7..41d6425 100644
--- a/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java
@@ -55,7 +55,7 @@ public class CombinerTest {
static Key nk(int row, int colf, int colq, long ts, boolean deleted) {
Key k = nk(row, colf, colq, ts);
- k.setDeleted(true);
+ k.setDeleted(deleted);
return k;
}
[07/26] git commit: Merge branch '1.6.0-SNAPSHOT' into ACCUMULO-2061
Posted by el...@apache.org.
Merge branch '1.6.0-SNAPSHOT' into ACCUMULO-2061
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/96182f93
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/96182f93
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/96182f93
Branch: refs/heads/ACCUMULO-2061
Commit: 96182f93f3105b047c63206c702a5313e9ca5535
Parents: 7cb3b23 6dd1bd3
Author: Josh Elser <el...@apache.org>
Authored: Wed Mar 12 13:01:05 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Wed Mar 12 13:01:05 2014 -0400
----------------------------------------------------------------------
init/src/main/packaging/etc/accumulo/tracers | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[06/26] git commit: ACCUMULO-2267 use the localhost convention for
the tracer, too
Posted by el...@apache.org.
ACCUMULO-2267 use the localhost convention for the tracer, too
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6dd1bd3e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6dd1bd3e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6dd1bd3e
Branch: refs/heads/ACCUMULO-2061
Commit: 6dd1bd3e9a51e4ca32fe6795f9448c7f12eaca2b
Parents: 0930732
Author: Eric Newton <er...@gmail.com>
Authored: Wed Mar 12 12:41:43 2014 -0400
Committer: Eric Newton <er...@gmail.com>
Committed: Wed Mar 12 12:41:43 2014 -0400
----------------------------------------------------------------------
init/src/main/packaging/etc/accumulo/tracers | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6dd1bd3e/init/src/main/packaging/etc/accumulo/tracers
----------------------------------------------------------------------
diff --git a/init/src/main/packaging/etc/accumulo/tracers b/init/src/main/packaging/etc/accumulo/tracers
index 09697dc..c319fb3 100644
--- a/init/src/main/packaging/etc/accumulo/tracers
+++ b/init/src/main/packaging/etc/accumulo/tracers
@@ -12,4 +12,4 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
+localhost