You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2013/12/13 23:56:10 UTC
[2/3] git commit: ACCUMULO-2010 Remove Closeable from interfaces
ACCUMULO-2010 Remove Closeable from interfaces
This gets around compiler warnings about resource leaks. This reverts
back to the old API semantics, where we are not advising that clients
close Instance objects. If it is determined in the future that we
should be advising clients to do this, and therefore want to
re-introduce the Closeable interface to Instance and the ZooKeeper
objects, it should be accompanied by changes that actually close these
resources in our own code. We should not be recommending this without
following our own recommendation, and adding Closeable to the
interface is a recommendation to call the close method.
Also clean up additional resource warnings.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/335f693a
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/335f693a
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/335f693a
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 335f693a4045d2c2501e2ed6ece0493734093143
Parents: 07a0433
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri Dec 13 17:15:13 2013 -0500
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri Dec 13 17:15:13 2013 -0500
----------------------------------------------------------------------
.../apache/accumulo/core/client/Instance.java | 26 +-
.../core/file/rfile/bcfile/PrintInfo.java | 8 +-
.../accumulo/core/zookeeper/ZooCache.java | 110 +++---
.../accumulo/core/zookeeper/ZooReader.java | 39 ++-
.../apache/accumulo/core/data/RangeTest.java | 337 ++++++++++---------
.../simple/filedata/FileDataIngest.java | 71 ++--
.../accumulo/server/logger/LogReader.java | 21 +-
.../accumulo/server/logger/LogService.java | 83 +++--
.../accumulo/server/test/MidPointPerfTest2.java | 118 +++----
.../test/continuous/UndefinedAnalyzer.java | 260 +++++++-------
.../accumulo/server/util/SendLogToChainsaw.java | 131 ++++---
11 files changed, 631 insertions(+), 573 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/335f693a/src/core/src/main/java/org/apache/accumulo/core/client/Instance.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/Instance.java b/src/core/src/main/java/org/apache/accumulo/core/client/Instance.java
index 3b2af18..b3ed056 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/Instance.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/Instance.java
@@ -16,7 +16,6 @@
*/
package org.apache.accumulo.core.client;
-import java.io.Closeable;
import java.nio.ByteBuffer;
import java.util.List;
@@ -27,49 +26,49 @@ import org.apache.accumulo.core.security.thrift.AuthInfo;
* This class represents the information a client needs to know to connect to an instance of accumulo.
*
*/
-public interface Instance extends Closeable {
+public interface Instance {
/**
* Returns the location of the tablet server that is serving the root tablet.
*
* @return location in "hostname:port" form
*/
public abstract String getRootTabletLocation();
-
+
/**
* Returns the location(s) of the accumulo master and any redundant servers.
*
* @return a list of locations in "hostname:port" form
*/
public abstract List<String> getMasterLocations();
-
+
/**
* Returns a unique string that identifies this instance of accumulo.
*
* @return a UUID
*/
public abstract String getInstanceID();
-
+
/**
* Returns the instance name given at system initialization time.
*
* @return current instance name
*/
public abstract String getInstanceName();
-
+
/**
* Returns a comma-separated list of zookeeper servers the instance is using.
*
* @return the zookeeper servers this instance is using in "hostname:port" form
*/
public abstract String getZooKeepers();
-
+
/**
* Returns the zookeeper connection timeout.
*
* @return the configured timeout to connect to zookeeper
*/
public abstract int getZooKeepersSessionTimeOut();
-
+
/**
* Returns a connection to accumulo.
*
@@ -84,7 +83,7 @@ public interface Instance extends Closeable {
* when a user's credentials are invalid
*/
public abstract Connector getConnector(String user, byte[] pass) throws AccumuloException, AccumuloSecurityException;
-
+
/**
* Returns a connection to accumulo.
*
@@ -97,7 +96,7 @@ public interface Instance extends Closeable {
* when a user's credentials are invalid
*/
public abstract Connector getConnector(AuthInfo auth) throws AccumuloException, AccumuloSecurityException;
-
+
/**
* Returns a connection to accumulo.
*
@@ -112,7 +111,7 @@ public interface Instance extends Closeable {
* when a user's credentials are invalid
*/
public abstract Connector getConnector(String user, ByteBuffer pass) throws AccumuloException, AccumuloSecurityException;
-
+
/**
* Returns a connection to this instance of accumulo.
*
@@ -132,16 +131,15 @@ public interface Instance extends Closeable {
* Closes up the instance to free up all associated resources. You should try to reuse an Instance as much as you can because there is some location caching
* stored which will enhance performance.
*/
- @Override
public abstract void close();
-
+
/**
* Returns the AccumuloConfiguration to use when interacting with this instance.
*
* @return the AccumuloConfiguration that specifies properties related to interacting with this instance
*/
public abstract AccumuloConfiguration getConfiguration();
-
+
/**
* Set the AccumuloConfiguration to use when interacting with this instance.
*
http://git-wip-us.apache.org/repos/asf/accumulo/blob/335f693a/src/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java b/src/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java
index 88e32d1..a29a53e 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java
@@ -32,9 +32,9 @@ public class PrintInfo {
public static void printMetaBlockInfo(Configuration conf, FileSystem fs, Path path) throws Exception {
FSDataInputStream fsin = fs.open(path);
BCFile.Reader bcfr = new BCFile.Reader(fsin, fs.getFileStatus(path).getLen(), conf);
-
+
Set<Entry<String,MetaIndexEntry>> es = bcfr.metaIndex.index.entrySet();
-
+
for (Entry<String,MetaIndexEntry> entry : es) {
PrintStream out = System.out;
out.println("Meta block : " + entry.getKey());
@@ -43,8 +43,10 @@ public class PrintInfo {
out.println(" Compression type : " + entry.getValue().getCompressionAlgorithm().getName());
out.println();
}
+
+ bcfr.close();
}
-
+
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
@SuppressWarnings("deprecation")
http://git-wip-us.apache.org/repos/asf/accumulo/blob/335f693a/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooCache.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooCache.java b/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooCache.java
index 1d55f6c..cbb3918 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooCache.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooCache.java
@@ -18,7 +18,6 @@ package org.apache.accumulo.core.zookeeper;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
-import java.io.Closeable;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
@@ -41,29 +40,29 @@ import org.apache.zookeeper.data.Stat;
* Caches values stored in zookeeper and keeps them up to date as they change in zookeeper.
*
*/
-public class ZooCache implements Closeable {
+public class ZooCache {
private static final Logger log = Logger.getLogger(ZooCache.class);
-
+
private ZCacheWatcher watcher = new ZCacheWatcher();
private Watcher externalWatcher = null;
-
+
private HashMap<String,byte[]> cache;
private HashMap<String,Stat> statCache;
private HashMap<String,List<String>> childrenCache;
-
+
private ZooReader zReader;
-
+
private ZooKeeper getZooKeeper() {
return zReader.getZooKeeper();
}
-
+
private class ZCacheWatcher implements Watcher {
@Override
public void process(WatchedEvent event) {
-
+
if (log.isTraceEnabled())
log.trace(event);
-
+
switch (event.getType()) {
case NodeDataChanged:
case NodeChildrenChanged:
@@ -92,21 +91,21 @@ public class ZooCache implements Closeable {
default:
log.warn("Unhandled: " + event);
}
-
+
if (externalWatcher != null) {
externalWatcher.process(event);
}
}
}
-
+
public ZooCache(String zooKeepers, int sessionTimeout) {
this(zooKeepers, sessionTimeout, null);
}
-
+
public ZooCache(String zooKeepers, int sessionTimeout, Watcher watcher) {
this(new ZooReader(zooKeepers, sessionTimeout), watcher);
}
-
+
public ZooCache(ZooReader reader, Watcher watcher) {
this.zReader = reader;
this.cache = new HashMap<String,byte[]>();
@@ -114,23 +113,23 @@ public class ZooCache implements Closeable {
this.childrenCache = new HashMap<String,List<String>>();
this.externalWatcher = watcher;
}
-
+
private static interface ZooRunnable {
void run(ZooKeeper zooKeeper) throws KeeperException, InterruptedException;
}
-
+
private synchronized void retry(ZooRunnable op) {
-
+
int sleepTime = 100;
-
+
while (true) {
-
+
ZooKeeper zooKeeper = getZooKeeper();
-
+
try {
op.run(zooKeeper);
return;
-
+
} catch (KeeperException e) {
if (e.code() == Code.NONODE) {
log.error("Looked up non existant node in cache " + e.getPath(), e);
@@ -141,7 +140,7 @@ public class ZooCache implements Closeable {
} catch (ConcurrentModificationException e) {
log.debug("Zookeeper was modified, will retry");
}
-
+
try {
// do not hold lock while sleeping
wait(sleepTime);
@@ -150,20 +149,20 @@ public class ZooCache implements Closeable {
}
if (sleepTime < 10000)
sleepTime = (int) (sleepTime + sleepTime * Math.random());
-
+
}
}
-
+
public synchronized List<String> getChildren(final String zPath) {
-
+
ZooRunnable zr = new ZooRunnable() {
-
+
@Override
public void run(ZooKeeper zooKeeper) throws KeeperException, InterruptedException {
-
+
if (childrenCache.containsKey(zPath))
return;
-
+
try {
List<String> children = zooKeeper.getChildren(zPath, watcher);
childrenCache.put(zPath, children);
@@ -173,31 +172,31 @@ public class ZooCache implements Closeable {
}
}
}
-
+
};
-
+
retry(zr);
-
+
List<String> children = childrenCache.get(zPath);
if (children == null) {
return null;
}
return Collections.unmodifiableList(children);
}
-
+
public synchronized byte[] get(final String zPath) {
return get(zPath, null);
}
-
+
public synchronized byte[] get(final String zPath, Stat stat) {
ZooRunnable zr = new ZooRunnable() {
-
+
@Override
public void run(ZooKeeper zooKeeper) throws KeeperException, InterruptedException {
-
+
if (cache.containsKey(zPath))
return;
-
+
/*
* The following call to exists() is important, since we are caching that a node does not exist. Once the node comes into existance, it will be added to
* the cache. But this notification of a node coming into existance will only be given if exists() was previously called.
@@ -205,11 +204,11 @@ public class ZooCache implements Closeable {
* If the call to exists() is bypassed and only getData() is called with a special case that looks for Code.NONODE in the KeeperException, then
* non-existance can not be cached.
*/
-
+
Stat stat = zooKeeper.exists(zPath, watcher);
-
+
byte[] data = null;
-
+
if (stat == null) {
if (log.isTraceEnabled())
log.trace("zookeeper did not contain " + zPath);
@@ -228,11 +227,11 @@ public class ZooCache implements Closeable {
log.trace("putting " + zPath + " " + (data == null ? null : new String(data)) + " in cache");
put(zPath, data, stat);
}
-
+
};
-
+
retry(zr);
-
+
if (stat != null) {
Stat cstat = statCache.get(zPath);
if (cstat != null) {
@@ -241,26 +240,26 @@ public class ZooCache implements Closeable {
DataOutputStream dos = new DataOutputStream(baos);
cstat.write(dos);
dos.close();
-
+
ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
DataInputStream dis = new DataInputStream(bais);
stat.readFields(dis);
-
+
dis.close();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
-
+
return cache.get(zPath);
}
-
+
private synchronized void put(String zPath, byte[] data, Stat stat) {
cache.put(zPath, data);
statCache.put(zPath, stat);
}
-
+
private synchronized void remove(String zPath) {
if (log.isTraceEnabled())
log.trace("removing " + zPath + " from cache");
@@ -268,36 +267,36 @@ public class ZooCache implements Closeable {
childrenCache.remove(zPath);
statCache.remove(zPath);
}
-
+
public synchronized void clear() {
cache.clear();
childrenCache.clear();
statCache.clear();
}
-
+
public synchronized void clear(String zPath) {
-
+
for (Iterator<String> i = cache.keySet().iterator(); i.hasNext();) {
String path = i.next();
if (path.startsWith(zPath))
i.remove();
}
-
+
for (Iterator<String> i = childrenCache.keySet().iterator(); i.hasNext();) {
String path = i.next();
if (path.startsWith(zPath))
i.remove();
}
-
+
for (Iterator<String> i = statCache.keySet().iterator(); i.hasNext();) {
String path = i.next();
if (path.startsWith(zPath))
i.remove();
}
}
-
+
private static Map<String,ZooCache> instances = new HashMap<String,ZooCache>();
-
+
public static synchronized ZooCache getInstance(String zooKeepers, int sessionTimeout) {
String key = zooKeepers + ":" + sessionTimeout;
ZooCache zc = instances.get(key);
@@ -305,11 +304,10 @@ public class ZooCache implements Closeable {
zc = new ZooCache(zooKeepers, sessionTimeout);
instances.put(key, zc);
}
-
+
return zc;
}
-
- @Override
+
public void close() {
cache.clear();
statCache.clear();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/335f693a/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooReader.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooReader.java b/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooReader.java
index ab02034..aabc0f2 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooReader.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooReader.java
@@ -16,67 +16,66 @@
*/
package org.apache.accumulo.core.zookeeper;
-import java.io.Closeable;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.accumulo.core.client.Instance;
+import org.apache.zookeeper.AsyncCallback.VoidCallback;
import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.Code;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooKeeper;
-import org.apache.zookeeper.AsyncCallback.VoidCallback;
-import org.apache.zookeeper.KeeperException.Code;
import org.apache.zookeeper.data.Stat;
-public class ZooReader implements IZooReader, Closeable {
-
+public class ZooReader implements IZooReader {
+
protected String keepers;
protected int timeout;
-
+
protected ZooKeeper getSession(String keepers, int timeout, String auth) {
return ZooSession.getSession(keepers, timeout, auth);
}
-
+
protected ZooKeeper getZooKeeper() {
return getSession(keepers, timeout, null);
}
-
+
@Override
public byte[] getData(String zPath, Stat stat) throws KeeperException, InterruptedException {
return getZooKeeper().getData(zPath, false, stat);
}
-
+
@Override
public Stat getStatus(String zPath) throws KeeperException, InterruptedException {
return getZooKeeper().exists(zPath, false);
}
-
+
@Override
public Stat getStatus(String zPath, Watcher watcher) throws KeeperException, InterruptedException {
return getZooKeeper().exists(zPath, watcher);
}
-
+
@Override
public List<String> getChildren(String zPath) throws KeeperException, InterruptedException {
return getZooKeeper().getChildren(zPath, false);
}
-
+
@Override
public List<String> getChildren(String zPath, Watcher watcher) throws KeeperException, InterruptedException {
return getZooKeeper().getChildren(zPath, watcher);
}
-
+
@Override
public boolean exists(String zPath) throws KeeperException, InterruptedException {
return getZooKeeper().exists(zPath, false) != null;
}
-
+
@Override
public boolean exists(String zPath, Watcher watcher) throws KeeperException, InterruptedException {
return getZooKeeper().exists(zPath, watcher) != null;
}
-
+
@Override
public void sync(final String path) throws KeeperException, InterruptedException {
final AtomicInteger rc = new AtomicInteger();
@@ -89,7 +88,8 @@ public class ZooReader implements IZooReader, Closeable {
waiter.set(true);
waiter.notifyAll();
}
- }}, null);
+ }
+ }, null);
synchronized (waiter) {
while (!waiter.get())
waiter.wait();
@@ -99,19 +99,18 @@ public class ZooReader implements IZooReader, Closeable {
throw KeeperException.create(code);
}
}
-
+
public ZooReader(String keepers, int timeout) {
this.keepers = keepers;
this.timeout = timeout;
}
-
+
public ZooReader(Instance instance) {
this(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
}
/**
- * Closes this reader. If closure of the underlying session is interrupted,
- * this method sets the calling thread's interrupt status.
+ * Closes this reader. If closure of the underlying session is interrupted, this method sets the calling thread's interrupt status.
*/
public void close() {
try {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/335f693a/src/core/src/test/java/org/apache/accumulo/core/data/RangeTest.java
----------------------------------------------------------------------
diff --git a/src/core/src/test/java/org/apache/accumulo/core/data/RangeTest.java b/src/core/src/test/java/org/apache/accumulo/core/data/RangeTest.java
index 68d9731..1f9a40d 100644
--- a/src/core/src/test/java/org/apache/accumulo/core/data/RangeTest.java
+++ b/src/core/src/test/java/org/apache/accumulo/core/data/RangeTest.java
@@ -35,554 +35,554 @@ public class RangeTest extends TestCase {
Key ik1 = null;
if (k1 != null)
ik1 = new Key(new Text(k1), 0l);
-
+
Key ik2 = null;
if (k2 != null)
ik2 = new Key(new Text(k2), 0l);
-
+
return new Range(ik1, ik2);
}
-
+
private List<Range> nrl(Range... ranges) {
return Arrays.asList(ranges);
}
-
+
private void check(List<Range> rl, List<Range> expected) {
HashSet<Range> s1 = new HashSet<Range>(rl);
HashSet<Range> s2 = new HashSet<Range>(expected);
-
+
assertTrue("got : " + rl + " expected : " + expected, s1.equals(s2));
}
-
+
public void testMergeOverlapping1() {
List<Range> rl = nrl(nr("a", "c"), nr("a", "b"));
List<Range> expected = nrl(nr("a", "c"));
check(Range.mergeOverlapping(rl), expected);
}
-
+
public void testMergeOverlapping2() {
List<Range> rl = nrl(nr("a", "c"), nr("d", "f"));
List<Range> expected = nrl(nr("a", "c"), nr("d", "f"));
check(Range.mergeOverlapping(rl), expected);
}
-
+
public void testMergeOverlapping3() {
List<Range> rl = nrl(nr("a", "e"), nr("b", "f"), nr("c", "r"), nr("g", "j"), nr("t", "x"));
List<Range> expected = nrl(nr("a", "r"), nr("t", "x"));
check(Range.mergeOverlapping(rl), expected);
}
-
+
public void testMergeOverlapping4() {
List<Range> rl = nrl(nr("a", "e"), nr("b", "f"), nr("c", "r"), nr("g", "j"));
List<Range> expected = nrl(nr("a", "r"));
check(Range.mergeOverlapping(rl), expected);
}
-
+
public void testMergeOverlapping5() {
List<Range> rl = nrl(nr("a", "e"));
List<Range> expected = nrl(nr("a", "e"));
check(Range.mergeOverlapping(rl), expected);
}
-
+
public void testMergeOverlapping6() {
List<Range> rl = nrl();
List<Range> expected = nrl();
check(Range.mergeOverlapping(rl), expected);
}
-
+
public void testMergeOverlapping7() {
List<Range> rl = nrl(nr("a", "e"), nr("g", "q"), nr("r", "z"));
List<Range> expected = nrl(nr("a", "e"), nr("g", "q"), nr("r", "z"));
check(Range.mergeOverlapping(rl), expected);
}
-
+
public void testMergeOverlapping8() {
List<Range> rl = nrl(nr("a", "c"), nr("a", "c"));
List<Range> expected = nrl(nr("a", "c"));
check(Range.mergeOverlapping(rl), expected);
}
-
+
public void testMergeOverlapping9() {
List<Range> rl = nrl(nr(null, null));
List<Range> expected = nrl(nr(null, null));
check(Range.mergeOverlapping(rl), expected);
}
-
+
public void testMergeOverlapping10() {
List<Range> rl = nrl(nr(null, null), nr("a", "c"));
List<Range> expected = nrl(nr(null, null));
check(Range.mergeOverlapping(rl), expected);
}
-
+
public void testMergeOverlapping11() {
List<Range> rl = nrl(nr("a", "c"), nr(null, null));
List<Range> expected = nrl(nr(null, null));
check(Range.mergeOverlapping(rl), expected);
}
-
+
public void testMergeOverlapping12() {
List<Range> rl = nrl(nr("b", "d"), nr("c", null));
List<Range> expected = nrl(nr("b", null));
check(Range.mergeOverlapping(rl), expected);
}
-
+
public void testMergeOverlapping13() {
List<Range> rl = nrl(nr("b", "d"), nr("a", null));
List<Range> expected = nrl(nr("a", null));
check(Range.mergeOverlapping(rl), expected);
}
-
+
public void testMergeOverlapping14() {
List<Range> rl = nrl(nr("b", "d"), nr("e", null));
List<Range> expected = nrl(nr("b", "d"), nr("e", null));
check(Range.mergeOverlapping(rl), expected);
}
-
+
public void testMergeOverlapping15() {
List<Range> rl = nrl(nr("b", "d"), nr("e", null), nr("c", "f"));
List<Range> expected = nrl(nr("b", null));
check(Range.mergeOverlapping(rl), expected);
}
-
+
public void testMergeOverlapping16() {
List<Range> rl = nrl(nr("b", "d"), nr("f", null), nr("c", "e"));
List<Range> expected = nrl(nr("b", "e"), nr("f", null));
check(Range.mergeOverlapping(rl), expected);
}
-
+
public void testMergeOverlapping17() {
List<Range> rl = nrl(nr("b", "d"), nr("r", null), nr("c", "e"), nr("g", "t"));
List<Range> expected = nrl(nr("b", "e"), nr("g", null));
check(Range.mergeOverlapping(rl), expected);
}
-
+
public void testMergeOverlapping18() {
List<Range> rl = nrl(nr(null, "d"), nr("r", null), nr("c", "e"), nr("g", "t"));
List<Range> expected = nrl(nr(null, "e"), nr("g", null));
check(Range.mergeOverlapping(rl), expected);
}
-
+
public void testMergeOverlapping19() {
List<Range> rl = nrl(nr(null, "d"), nr("r", null), nr("c", "e"), nr("g", "t"), nr("d", "h"));
List<Range> expected = nrl(nr(null, null));
check(Range.mergeOverlapping(rl), expected);
}
-
+
public void testMergeOverlapping20() {
-
+
List<Range> rl = nrl(new Range(new Text("a"), true, new Text("b"), false), new Range(new Text("b"), false, new Text("c"), false));
List<Range> expected = nrl(new Range(new Text("a"), true, new Text("b"), false), new Range(new Text("b"), false, new Text("c"), false));
check(Range.mergeOverlapping(rl), expected);
-
+
rl = nrl(new Range(new Text("a"), true, new Text("b"), false), new Range(new Text("b"), true, new Text("c"), false));
expected = nrl(new Range(new Text("a"), true, new Text("c"), false));
check(Range.mergeOverlapping(rl), expected);
-
+
rl = nrl(new Range(new Text("a"), true, new Text("b"), true), new Range(new Text("b"), false, new Text("c"), false));
expected = nrl(new Range(new Text("a"), true, new Text("c"), false));
check(Range.mergeOverlapping(rl), expected);
-
+
rl = nrl(new Range(new Text("a"), true, new Text("b"), true), new Range(new Text("b"), true, new Text("c"), false));
expected = nrl(new Range(new Text("a"), true, new Text("c"), false));
check(Range.mergeOverlapping(rl), expected);
-
+
}
-
+
public void testMergeOverlapping22() {
-
+
Range ke1 = new KeyExtent(new Text("tab1"), new Text("Bank"), null).toMetadataRange();
Range ke2 = new KeyExtent(new Text("tab1"), new Text("Fails"), new Text("Bank")).toMetadataRange();
Range ke3 = new KeyExtent(new Text("tab1"), new Text("Sam"), new Text("Fails")).toMetadataRange();
Range ke4 = new KeyExtent(new Text("tab1"), new Text("bails"), new Text("Sam")).toMetadataRange();
Range ke5 = new KeyExtent(new Text("tab1"), null, new Text("bails")).toMetadataRange();
-
+
List<Range> rl = nrl(ke1, ke2, ke3, ke4, ke5);
List<Range> expected = nrl(new KeyExtent(new Text("tab1"), null, null).toMetadataRange());
check(Range.mergeOverlapping(rl), expected);
-
+
rl = nrl(ke1, ke2, ke4, ke5);
expected = nrl(new KeyExtent(new Text("tab1"), new Text("Fails"), null).toMetadataRange(),
new KeyExtent(new Text("tab1"), null, new Text("Sam")).toMetadataRange());
check(Range.mergeOverlapping(rl), expected);
-
+
rl = nrl(ke2, ke3, ke4, ke5);
expected = nrl(new KeyExtent(new Text("tab1"), null, new Text("Bank")).toMetadataRange());
check(Range.mergeOverlapping(rl), expected);
-
+
rl = nrl(ke1, ke2, ke3, ke4);
expected = nrl(new KeyExtent(new Text("tab1"), new Text("bails"), null).toMetadataRange());
check(Range.mergeOverlapping(rl), expected);
-
+
rl = nrl(ke2, ke3, ke4);
expected = nrl(new KeyExtent(new Text("tab1"), new Text("bails"), new Text("Bank")).toMetadataRange());
check(Range.mergeOverlapping(rl), expected);
}
-
+
public void testMergeOverlapping21() {
for (boolean b1 : new boolean[] {true, false})
for (boolean b2 : new boolean[] {true, false})
for (boolean b3 : new boolean[] {true, false})
for (boolean b4 : new boolean[] {true, false}) {
-
+
// System.out.println("b1:"+b1+" b2:"+b2+" b3:"+b3+" b4:"+b4);
-
+
List<Range> rl = nrl(new Range(new Key(new Text("a")), b1, new Key(new Text("m")), b2), new Range(new Key(new Text("b")), b3,
new Key(new Text("n")), b4));
List<Range> expected = nrl(new Range(new Key(new Text("a")), b1, new Key(new Text("n")), b4));
check(Range.mergeOverlapping(rl), expected);
-
+
rl = nrl(new Range(new Key(new Text("a")), b1, new Key(new Text("m")), b2), new Range(new Key(new Text("a")), b3, new Key(new Text("n")), b4));
expected = nrl(new Range(new Key(new Text("a")), b1 || b3, new Key(new Text("n")), b4));
check(Range.mergeOverlapping(rl), expected);
-
+
rl = nrl(new Range(new Key(new Text("a")), b1, new Key(new Text("n")), b2), new Range(new Key(new Text("b")), b3, new Key(new Text("n")), b4));
expected = nrl(new Range(new Key(new Text("a")), b1, new Key(new Text("n")), b2 || b4));
check(Range.mergeOverlapping(rl), expected);
-
+
rl = nrl(new Range(new Key(new Text("a")), b1, new Key(new Text("n")), b2), new Range(new Key(new Text("a")), b3, new Key(new Text("n")), b4));
expected = nrl(new Range(new Key(new Text("a")), b1 || b3, new Key(new Text("n")), b2 || b4));
check(Range.mergeOverlapping(rl), expected);
}
-
+
}
-
+
public void testEqualsNull() {
-
+
assertTrue(nr(null, "d").equals(nr(null, "d")));
-
+
assertTrue(nr(null, null).equals(nr(null, null)));
-
+
assertTrue(nr("a", null).equals(nr("a", null)));
-
+
assertFalse(nr(null, "d").equals(nr("a", "d")));
assertFalse(nr("a", "d").equals(nr(null, "d")));
-
+
assertFalse(nr(null, null).equals(nr("a", "d")));
assertFalse(nr("a", "d").equals(nr(null, null)));
-
+
assertFalse(nr("a", null).equals(nr("a", "d")));
assertFalse(nr("a", "d").equals(nr("a", null)));
}
-
+
public void testEquals() {
assertFalse(nr("b", "d").equals(nr("a", "d")));
assertFalse(nr("a", "d").equals(nr("b", "d")));
-
+
assertFalse(nr("x", "y").equals(nr("a", "d")));
assertFalse(nr("a", "d").equals(nr("x", "y")));
-
+
assertFalse(nr("a", "z").equals(nr("a", "d")));
assertFalse(nr("a", "d").equals(nr("a", "z")));
-
+
assertTrue(nr("a", "z").equals(nr("a", "z")));
}
-
+
public void testRow1() {
Range rowRange = new Range(new Text("r1"));
-
+
assertTrue(rowRange.contains(new Key(new Text("r1"))));
assertTrue(rowRange.contains(new Key(new Text("r1"), new Text("cf1"))));
assertTrue(rowRange.contains(new Key(new Text("r1"), new Text("cf1"), new Text("cq1"))));
-
+
assertFalse(rowRange.contains(new Key(new Text("r1")).followingKey(PartialKey.ROW)));
assertFalse(rowRange.contains(new Key(new Text("r11"))));
assertFalse(rowRange.contains(new Key(new Text("r0"))));
}
-
+
public void testRow2() {
Range rowRange = new Range(new Text("r1"), new Text("r2"));
-
+
assertTrue(rowRange.contains(new Key(new Text("r1"))));
assertTrue(rowRange.contains(new Key(new Text("r1"), new Text("cf1"))));
assertTrue(rowRange.contains(new Key(new Text("r1"), new Text("cf1"), new Text("cq1"))));
-
+
assertTrue(rowRange.contains(new Key(new Text("r1")).followingKey(PartialKey.ROW)));
assertTrue(rowRange.contains(new Key(new Text("r11"))));
-
+
assertTrue(rowRange.contains(new Key(new Text("r2"))));
assertTrue(rowRange.contains(new Key(new Text("r2"), new Text("cf1"))));
assertTrue(rowRange.contains(new Key(new Text("r2"), new Text("cf1"), new Text("cq1"))));
-
+
assertFalse(rowRange.contains(new Key(new Text("r0"))));
assertFalse(rowRange.contains(new Key(new Text("r2")).followingKey(PartialKey.ROW)));
}
-
+
public void testRow3() {
Range rowRange = new Range(new Text("r1"), false, new Text("r2"), false);
-
+
assertFalse(rowRange.contains(new Key(new Text("r1"))));
assertFalse(rowRange.contains(new Key(new Text("r1"), new Text("cf1"))));
assertFalse(rowRange.contains(new Key(new Text("r1"), new Text("cf1"), new Text("cq1"))));
-
+
assertTrue(rowRange.contains(new Key(new Text("r1")).followingKey(PartialKey.ROW)));
assertTrue(rowRange.contains(new Key(new Text("r11"))));
-
+
assertFalse(rowRange.contains(new Key(new Text("r2"))));
assertFalse(rowRange.contains(new Key(new Text("r2"), new Text("cf1"))));
assertFalse(rowRange.contains(new Key(new Text("r2"), new Text("cf1"), new Text("cq1"))));
-
+
assertFalse(rowRange.contains(new Key(new Text("r0"))));
assertFalse(rowRange.contains(new Key(new Text("r2")).followingKey(PartialKey.ROW)));
}
-
+
public void testRow4() {
Range rowRange = new Range(new Text("r1"), true, new Text("r2"), false);
-
+
assertTrue(rowRange.contains(new Key(new Text("r1"))));
assertTrue(rowRange.contains(new Key(new Text("r1"), new Text("cf1"))));
assertTrue(rowRange.contains(new Key(new Text("r1"), new Text("cf1"), new Text("cq1"))));
-
+
assertTrue(rowRange.contains(new Key(new Text("r1")).followingKey(PartialKey.ROW)));
assertTrue(rowRange.contains(new Key(new Text("r11"))));
-
+
assertFalse(rowRange.contains(new Key(new Text("r2"))));
assertFalse(rowRange.contains(new Key(new Text("r2"), new Text("cf1"))));
assertFalse(rowRange.contains(new Key(new Text("r2"), new Text("cf1"), new Text("cq1"))));
-
+
assertFalse(rowRange.contains(new Key(new Text("r0"))));
assertFalse(rowRange.contains(new Key(new Text("r2")).followingKey(PartialKey.ROW)));
}
-
+
public void testRow5() {
Range rowRange = new Range(new Text("r1"), false, new Text("r2"), true);
-
+
assertFalse(rowRange.contains(new Key(new Text("r1"))));
assertFalse(rowRange.contains(new Key(new Text("r1"), new Text("cf1"))));
assertFalse(rowRange.contains(new Key(new Text("r1"), new Text("cf1"), new Text("cq1"))));
-
+
assertTrue(rowRange.contains(new Key(new Text("r1")).followingKey(PartialKey.ROW)));
assertTrue(rowRange.contains(new Key(new Text("r11"))));
-
+
assertTrue(rowRange.contains(new Key(new Text("r2"))));
assertTrue(rowRange.contains(new Key(new Text("r2"), new Text("cf1"))));
assertTrue(rowRange.contains(new Key(new Text("r2"), new Text("cf1"), new Text("cq1"))));
-
+
assertFalse(rowRange.contains(new Key(new Text("r0"))));
assertFalse(rowRange.contains(new Key(new Text("r2")).followingKey(PartialKey.ROW)));
}
-
+
public void testRow6() {
Range rowRange = new Range(new Text("r1"), true, null, true);
-
+
assertTrue(rowRange.contains(new Key(new Text("r1"))));
assertTrue(rowRange.contains(new Key(new Text("r1"), new Text("cf1"))));
assertTrue(rowRange.contains(new Key(new Text("r1"), new Text("cf1"), new Text("cq1"))));
-
+
assertTrue(rowRange.contains(new Key(new Text("r1")).followingKey(PartialKey.ROW)));
assertTrue(rowRange.contains(new Key(new Text("r11"))));
-
+
assertTrue(rowRange.contains(new Key(new Text("r2"))));
assertTrue(rowRange.contains(new Key(new Text("r2"), new Text("cf1"))));
assertTrue(rowRange.contains(new Key(new Text("r2"), new Text("cf1"), new Text("cq1"))));
-
+
assertFalse(rowRange.contains(new Key(new Text("r0"))));
assertTrue(rowRange.contains(new Key(new Text("r2")).followingKey(PartialKey.ROW)));
}
-
+
public void testRow7() {
Range rowRange = new Range(null, true, new Text("r2"), true);
-
+
assertTrue(rowRange.contains(new Key(new Text("r1"))));
assertTrue(rowRange.contains(new Key(new Text("r1"), new Text("cf1"))));
assertTrue(rowRange.contains(new Key(new Text("r1"), new Text("cf1"), new Text("cq1"))));
-
+
assertTrue(rowRange.contains(new Key(new Text("r1")).followingKey(PartialKey.ROW)));
assertTrue(rowRange.contains(new Key(new Text("r11"))));
-
+
assertTrue(rowRange.contains(new Key(new Text("r2"))));
assertTrue(rowRange.contains(new Key(new Text("r2"), new Text("cf1"))));
assertTrue(rowRange.contains(new Key(new Text("r2"), new Text("cf1"), new Text("cq1"))));
-
+
assertTrue(rowRange.contains(new Key(new Text("r0"))));
assertFalse(rowRange.contains(new Key(new Text("r2")).followingKey(PartialKey.ROW)));
}
-
+
public void testRow8() {
Range rowRange = new Range((Text) null);
-
+
assertTrue(rowRange.contains(new Key(new Text("r1"))));
assertTrue(rowRange.contains(new Key(new Text("r1"), new Text("cf1"))));
assertTrue(rowRange.contains(new Key(new Text("r1"), new Text("cf1"), new Text("cq1"))));
-
+
assertTrue(rowRange.contains(new Key(new Text("r1")).followingKey(PartialKey.ROW)));
assertTrue(rowRange.contains(new Key(new Text("r11"))));
-
+
assertTrue(rowRange.contains(new Key(new Text("r2"))));
assertTrue(rowRange.contains(new Key(new Text("r2"), new Text("cf1"))));
assertTrue(rowRange.contains(new Key(new Text("r2"), new Text("cf1"), new Text("cq1"))));
-
+
assertTrue(rowRange.contains(new Key(new Text("r0"))));
assertTrue(rowRange.contains(new Key(new Text("r2")).followingKey(PartialKey.ROW)));
}
-
+
private static Range nr(String r1, boolean r1i, String r2, boolean r2i) {
Text tr1 = null;
Text tr2 = null;
-
+
if (r1 != null)
tr1 = new Text(r1);
-
+
if (r2 != null)
tr2 = new Text(r2);
-
+
return new Range(tr1, r1i, tr2, r2i);
-
+
}
-
+
private static Key nk(String r) {
return new Key(new Text(r));
}
-
+
public void testClip1() {
Range fence = nr("a", false, "c", false);
-
+
runClipTest(fence, nr("a", false, "c", false), nr("a", false, "c", false));
runClipTest(fence, nr("a", true, "c", false), nr("a", false, "c", false));
runClipTest(fence, nr("a", false, "c", true), nr("a", false, "c", false));
runClipTest(fence, nr("a", true, "c", true), nr("a", false, "c", false));
-
+
fence = nr("a", true, "c", false);
-
+
runClipTest(fence, nr("a", false, "c", false), nr("a", false, "c", false));
runClipTest(fence, nr("a", true, "c", false), nr("a", true, "c", false));
runClipTest(fence, nr("a", false, "c", true), nr("a", false, "c", false));
runClipTest(fence, nr("a", true, "c", true), nr("a", true, "c", false));
-
+
fence = nr("a", false, "c", true);
-
+
runClipTest(fence, nr("a", false, "c", false), nr("a", false, "c", false));
runClipTest(fence, nr("a", true, "c", false), nr("a", false, "c", false));
runClipTest(fence, nr("a", false, "c", true), nr("a", false, "c", true));
runClipTest(fence, nr("a", true, "c", true), nr("a", false, "c", true));
-
+
fence = nr("a", true, "c", true);
-
+
runClipTest(fence, nr("a", false, "c", false), nr("a", false, "c", false));
runClipTest(fence, nr("a", true, "c", false), nr("a", true, "c", false));
runClipTest(fence, nr("a", false, "c", true), nr("a", false, "c", true));
runClipTest(fence, nr("a", true, "c", true), nr("a", true, "c", true));
}
-
+
public void testClip2() {
Range fence = nr("a", false, "c", false);
-
+
runClipTest(fence, nr(null, true, null, true), nr("a", false, "c", false));
runClipTest(fence, nr(null, true, "c", true), nr("a", false, "c", false));
runClipTest(fence, nr("a", true, null, true), nr("a", false, "c", false));
runClipTest(fence, nr("a", true, "c", true), nr("a", false, "c", false));
}
-
+
public void testClip3() {
Range fence = nr("a", false, "c", false);
-
+
runClipTest(fence, nr("0", false, "z", false), nr("a", false, "c", false));
runClipTest(fence, nr("0", true, "z", false), nr("a", false, "c", false));
runClipTest(fence, nr("0", false, "z", true), nr("a", false, "c", false));
runClipTest(fence, nr("0", true, "z", true), nr("a", false, "c", false));
-
+
runClipTest(fence, nr("0", false, "b", false), nr("a", false, "b", false));
runClipTest(fence, nr("0", true, "b", false), nr("a", false, "b", false));
runClipTest(fence, nr("0", false, "b", true), nr("a", false, "b", true));
runClipTest(fence, nr("0", true, "b", true), nr("a", false, "b", true));
-
+
runClipTest(fence, nr("a1", false, "z", false), nr("a1", false, "c", false));
runClipTest(fence, nr("a1", true, "z", false), nr("a1", true, "c", false));
runClipTest(fence, nr("a1", false, "z", true), nr("a1", false, "c", false));
runClipTest(fence, nr("a1", true, "z", true), nr("a1", true, "c", false));
-
+
runClipTest(fence, nr("a1", false, "b", false), nr("a1", false, "b", false));
runClipTest(fence, nr("a1", true, "b", false), nr("a1", true, "b", false));
runClipTest(fence, nr("a1", false, "b", true), nr("a1", false, "b", true));
runClipTest(fence, nr("a1", true, "b", true), nr("a1", true, "b", true));
}
-
+
public void testClip4() {
Range fence = new Range(nk("c"), false, nk("n"), false);
-
+
runClipTest(fence, new Range(nk("a"), false, nk("c"), false));
runClipTest(fence, new Range(nk("a"), false, nk("c"), true));
runClipTest(fence, new Range(nk("n"), false, nk("r"), false));
runClipTest(fence, new Range(nk("n"), true, nk("r"), false));
runClipTest(fence, new Range(nk("a"), true, nk("b"), false));
runClipTest(fence, new Range(nk("a"), true, nk("b"), true));
-
+
fence = new Range(nk("c"), true, nk("n"), true);
-
+
runClipTest(fence, new Range(nk("a"), false, nk("c"), false));
runClipTest(fence, new Range(nk("a"), false, nk("c"), true), new Range(nk("c"), true, nk("c"), true));
runClipTest(fence, new Range(nk("n"), false, nk("r"), false));
runClipTest(fence, new Range(nk("n"), true, nk("r"), false), new Range(nk("n"), true, nk("n"), true));
runClipTest(fence, new Range(nk("q"), false, nk("r"), false));
runClipTest(fence, new Range(nk("q"), true, nk("r"), false));
-
+
fence = nr("b", true, "b", true);
-
+
runClipTest(fence, nr("b", false, "c", false));
runClipTest(fence, nr("b", true, "c", false), nr("b", true, "b", true));
runClipTest(fence, nr("a", false, "b", false));
runClipTest(fence, nr("a", false, "b", true), nr("b", true, "b", true));
-
+
}
-
+
public void testBug1() {
-
+
// unit test related to a bug that was observed (bug was not in range, but want to ensure the following works)
-
+
// clip caught the scanner going to a tablet passed the end of the scan range
Range fence = new Range(new Text("10<"), false, new Text("~"), true);
-
+
Key k1 = new Key(new Text("10<"), new Text("~tab"), new Text("~pr"));
Range range = new Range(k1, true, k1.followingKey(PartialKey.ROW), false);
-
+
runClipTest(fence, range);
-
+
// scanner was not handling edge case properly...
Range scanRange = new Range(new Key("10;007cdc5b0".getBytes(), "~tab".getBytes(), "~pr".getBytes(), "".getBytes(), 130962, false), false, new Key(new Text(
"10<")).followingKey(PartialKey.ROW), false);
// below is the proper check the scanner now does instead of just comparing the row bytes
scanRange.afterEndKey(new Key(new Text("10<")).followingKey(PartialKey.ROW));
}
-
+
private void runClipTest(Range fence, Range range) {
try {
fence.clip(range);
assertFalse(true);
} catch (IllegalArgumentException e) {
-
+
}
-
+
}
-
+
private void runClipTest(Range fence, Range range, Range expected) {
Range clipped = fence.clip(range);
assertEquals(expected, clipped);
}
-
+
private static Key nk(String r, String cf, String cq) {
return new Key(new Text(r), new Text(cf), new Text(cq));
}
-
+
private static Key nk(String r, String cf, String cq, String cv) {
return new Key(new Text(r), new Text(cf), new Text(cq), new Text(cv));
}
-
+
private static Column nc(String cf, String cq) {
return new Column(cf.getBytes(), cq == null ? null : cq.getBytes(), null);
}
-
+
private static Column nc(String cf) {
return nc(cf, null);
}
-
+
private static Range nr(String row) {
return new Range(new Text(row));
}
-
+
public void testBound1() {
Range range1 = nr("row1");
-
+
Range range2 = range1.bound(nc("b"), nc("e"));
-
+
assertFalse(range2.contains(nk("row1")));
assertFalse(range2.contains(nk("row1", "a", "z")));
assertTrue(range2.contains(nk("row1", "b", "")));
@@ -593,19 +593,19 @@ public class RangeTest extends TestCase {
assertFalse(range2.contains(nk("row1", "e", "").followingKey(PartialKey.ROW_COLFAM)));
assertFalse(range2.contains(nk("row1", "f", "")));
assertFalse(range2.contains(nk("row1", "f", "z")));
-
+
}
-
+
public void testBound2() {
Range range1 = new Range(nk("row1", "b", "x"), true, nk("row1", "f", "x"), true);
-
+
Range range2 = range1.bound(nc("a"), nc("g"));
assertEquals(range1, range2);
assertFalse(range2.contains(nk("row1", "a", "x")));
assertTrue(range2.contains(nk("row1", "b", "x")));
assertTrue(range2.contains(nk("row1", "f", "x")));
assertFalse(range2.contains(nk("row1", "g", "")));
-
+
Range range3 = range1.bound(nc("c"), nc("d"));
assertFalse(range3.contains(nk("row1", "b", "x")));
assertTrue(range3.contains(nk("row1", "c", "")));
@@ -614,7 +614,7 @@ public class RangeTest extends TestCase {
assertTrue(range3.contains(nk("row1", "d", "z")));
assertFalse(range3.contains(nk("row1", "e", "")));
assertFalse(range3.contains(nk("row1", "f", "x")));
-
+
Range range4 = range1.bound(nc("c", "w"), nc("d", "z"));
assertFalse(range4.contains(nk("row1", "b", "x")));
assertTrue(range4.contains(nk("row1", "c", "w")));
@@ -625,14 +625,14 @@ public class RangeTest extends TestCase {
assertFalse(range4.contains(nk("row1", "d", "{", "")));
assertFalse(range4.contains(nk("row1", "d", "z", "a").followingKey(PartialKey.ROW_COLFAM_COLQUAL)));
assertFalse(range4.contains(nk("row1", "f", "x")));
-
+
Range range5 = range1.bound(nc("b", "w"), nc("f", "z"));
assertEquals(range1, range5);
assertFalse(range5.contains(nk("row1", "b", "w")));
assertTrue(range5.contains(nk("row1", "b", "x")));
assertTrue(range5.contains(nk("row1", "f", "x")));
assertFalse(range5.contains(nk("row1", "f", "z")));
-
+
Range range6 = range1.bound(nc("b", "y"), nc("f", "w"));
assertFalse(range6.contains(nk("row1", "b", "x")));
assertTrue(range6.contains(nk("row1", "b", "y")));
@@ -640,7 +640,7 @@ public class RangeTest extends TestCase {
assertTrue(range6.contains(nk("row1", "f", "w", "a")));
assertFalse(range6.contains(nk("row1", "f", "w").followingKey(PartialKey.ROW_COLFAM_COLQUAL)));
assertFalse(range6.contains(nk("row1", "f", "x")));
-
+
Range range7 = range1.bound(nc("a", "y"), nc("g", "w"));
assertEquals(range1, range7);
assertFalse(range7.contains(nk("row1", "b", "w")));
@@ -648,26 +648,26 @@ public class RangeTest extends TestCase {
assertTrue(range7.contains(nk("row1", "f", "x")));
assertFalse(range7.contains(nk("row1", "f", "z")));
}
-
+
public void testString() {
Range r1 = new Range(new Text("r1"));
Range r2 = new Range("r1");
assertEquals(r1, r2);
-
+
r1 = new Range(new Text("r1"), new Text("r2"));
r2 = new Range("r1", "r2");
assertEquals(r1, r2);
-
+
r1 = new Range(new Text("r1"), false, new Text("r2"), true);
r2 = new Range("r1", false, "r2", true);
assertEquals(r1, r2);
-
+
r1 = new Range(new Text("r1"), true, new Text("r2"), false);
r2 = new Range("r1", true, "r2", false);
assertEquals(r1, r2);
-
+
}
-
+
public void testExactRange() {
Range r = Range.exact("abc");
assertTrue(r.contains(new Key("abc")));
@@ -675,31 +675,31 @@ public class RangeTest extends TestCase {
assertFalse(r.contains(new Key("abcd")));
assertFalse(r.contains(new Key("abb")));
assertFalse(r.contains(new Key("abd")));
-
+
r = Range.exact("abc", "def");
assertTrue(r.contains(new Key("abc", "def", "ghi")));
assertFalse(r.contains(new Key("abc", "defg")));
assertFalse(r.contains(new Key("abc", "dee")));
assertFalse(r.contains(new Key("abc", "deg")));
-
+
r = Range.exact("abc", "def", "ghi");
assertTrue(r.contains(new Key("abc", "def", "ghi", "j&k")));
assertFalse(r.contains(new Key("abc", "def", "ghij")));
assertFalse(r.contains(new Key("abc", "def", "ghh")));
assertFalse(r.contains(new Key("abc", "def", "ghj")));
-
+
r = Range.exact("abc", "def", "ghi", "j&k");
assertTrue(r.contains(new Key("abc", "def", "ghi", "j&k", 7l)));
assertFalse(r.contains(new Key("abc", "def", "ghi", "j&kl")));
assertFalse(r.contains(new Key("abc", "def", "ghi", "j&j")));
assertFalse(r.contains(new Key("abc", "def", "ghi", "j&l")));
-
+
r = Range.exact("abc", "def", "ghi", "j&k", 7l);
assertTrue(r.contains(new Key("abc", "def", "ghi", "j&k", 7l)));
assertFalse(r.contains(new Key("abc", "def", "ghi", "j&k", 6l)));
assertFalse(r.contains(new Key("abc", "def", "ghi", "j&k", 8l)));
}
-
+
public void testPrefixRange() {
Range r = Range.prefix("abc");
assertTrue(r.contains(new Key("abc")));
@@ -707,59 +707,59 @@ public class RangeTest extends TestCase {
assertTrue(r.contains(new Key("abcd")));
assertFalse(r.contains(new Key("abb")));
assertFalse(r.contains(new Key("abd")));
-
+
r = Range.prefix("abc", "def");
assertTrue(r.contains(new Key("abc", "def", "ghi")));
assertTrue(r.contains(new Key("abc", "defg")));
assertFalse(r.contains(new Key("abc", "dee")));
assertFalse(r.contains(new Key("abc", "deg")));
-
+
r = Range.prefix("abc", "def", "ghi");
assertTrue(r.contains(new Key("abc", "def", "ghi", "j&k")));
assertTrue(r.contains(new Key("abc", "def", "ghij")));
assertFalse(r.contains(new Key("abc", "def", "ghh")));
assertFalse(r.contains(new Key("abc", "def", "ghj")));
-
+
r = Range.prefix("abc", "def", "ghi", "j&k");
assertTrue(r.contains(new Key("abc", "def", "ghi", "j&k", 7l)));
assertTrue(r.contains(new Key("abc", "def", "ghi", "j&kl")));
assertFalse(r.contains(new Key("abc", "def", "ghi", "j&j")));
assertFalse(r.contains(new Key("abc", "def", "ghi", "j&l")));
-
+
r = Range.prefix(makeText((byte) 0x07, (byte) 0xff));
assertTrue(r.contains(new Key(makeText((byte) 0x07, (byte) 0xff))));
assertTrue(r.contains(new Key(makeText((byte) 0x07, (byte) 0xff, (byte) 0x00))));
assertFalse(r.contains(new Key(makeText((byte) 0x07, (byte) 0xfe))));
assertFalse(r.contains(new Key(makeText((byte) 0x08))));
-
+
r = Range.prefix(makeText((byte) 0xff));
assertTrue(r.isInfiniteStopKey());
assertTrue(r.contains(new Key(makeText((byte) 0xff))));
assertTrue(r.contains(new Key(makeText((byte) 0xff, (byte) 0x07))));
-
+
r = Range.prefix(new Text("abc"), makeText((byte) 0xff));
assertTrue(r.contains(new Key(new Text("abc"), makeText((byte) 0xff))));
assertTrue(r.contains(new Key(new Text("abc"), makeText((byte) 0xff, (byte) 0x07))));
assertFalse(r.contains(new Key(new Text("abcd"))));
assertFalse(r.contains(new Key(new Text("abd"))));
-
+
r = Range.prefix(new Text("abc"), new Text("def"), makeText((byte) 0xff));
assertTrue(r.contains(new Key(new Text("abc"), new Text("def"), makeText((byte) 0xff))));
assertTrue(r.contains(new Key(new Text("abc"), new Text("def"), makeText((byte) 0xff, (byte) 0x07))));
assertFalse(r.contains(new Key(new Text("abc"), new Text("defg"))));
assertFalse(r.contains(new Key(new Text("abc"), new Text("deg"))));
-
+
r = Range.prefix(new Text("abc"), new Text("def"), new Text("ghi"), makeText((byte) 0xff));
assertTrue(r.contains(new Key(new Text("abc"), new Text("def"), new Text("ghi"), makeText((byte) 0xff))));
assertTrue(r.contains(new Key(new Text("abc"), new Text("def"), new Text("ghi"), makeText((byte) 0xff, (byte) 0x07))));
assertFalse(r.contains(new Key(new Text("abc"), new Text("def"), new Text("ghij"))));
assertFalse(r.contains(new Key(new Text("abc"), new Text("def"), new Text("ghj"))));
}
-
+
public static Text makeText(byte... b) {
return new Text(b);
}
-
+
public void testPrefix() {
assertEquals(Range.followingPrefix(makeText((byte) 0x07)), new Text(makeText((byte) 0x08)));
assertEquals(Range.followingPrefix(makeText((byte) 0xfe)), new Text(makeText((byte) 0xff)));
@@ -813,6 +813,7 @@ public class RangeTest extends TestCase {
Range r = new Range(new Key(new Text("soup")), true, false, new Key(new Text("nuts")), true, false);
TRange tr = r.toThrift();
try {
+ @SuppressWarnings("unused")
Range r2 = new Range(tr);
fail("Thrift constructor allowed invalid range");
} catch (IllegalArgumentException exc) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/335f693a/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataIngest.java
----------------------------------------------------------------------
diff --git a/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataIngest.java b/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataIngest.java
index 4ee1063..2fd0623 100644
--- a/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataIngest.java
+++ b/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataIngest.java
@@ -44,13 +44,13 @@ public class FileDataIngest {
public static final String REFS_FILE_EXT = "filext";
public static final ByteSequence CHUNK_CF_BS = new ArrayByteSequence(CHUNK_CF.getBytes(), 0, CHUNK_CF.getLength());
public static final ByteSequence REFS_CF_BS = new ArrayByteSequence(REFS_CF.getBytes(), 0, REFS_CF.getLength());
-
+
int chunkSize;
byte[] chunkSizeBytes;
byte[] buf;
MessageDigest md5digest;
ColumnVisibility cv;
-
+
public FileDataIngest(int chunkSize, ColumnVisibility colvis) {
this.chunkSize = chunkSize;
chunkSizeBytes = intToBytes(chunkSize);
@@ -62,13 +62,13 @@ public class FileDataIngest {
}
cv = colvis;
}
-
+
public String insertFileData(String filename, BatchWriter bw) throws MutationsRejectedException, IOException {
if (chunkSize == 0)
return "";
md5digest.reset();
String uid = hexString(md5digest.digest(filename.getBytes()));
-
+
// read through file once, calculating hashes
md5digest.reset();
InputStream fis = new FileInputStream(filename);
@@ -80,10 +80,10 @@ public class FileDataIngest {
numRead = fis.read(buf);
}
fis.close();
-
+
String hash = hexString(md5digest.digest());
Text row = new Text(hash);
-
+
// write info to accumulo
Mutation m = new Mutation(row);
m.put(REFS_CF, KeyUtil.buildNullSepText(uid, REFS_ORIG_FILE), cv, new Value(filename.getBytes()));
@@ -91,30 +91,35 @@ public class FileDataIngest {
if (fext != null)
m.put(REFS_CF, KeyUtil.buildNullSepText(uid, REFS_FILE_EXT), cv, new Value(fext.getBytes()));
bw.addMutation(m);
-
+
// read through file again, writing chunks to accumulo
- fis = new FileInputStream(filename);
- numRead = fis.read(buf);
int chunkCount = 0;
- while (numRead >= 0) {
- while (numRead < buf.length) {
- int moreRead = fis.read(buf, numRead, buf.length - numRead);
- if (moreRead > 0)
- numRead += moreRead;
- else if (moreRead < 0)
- break;
- }
- m = new Mutation(row);
- Text chunkCQ = new Text(chunkSizeBytes);
- chunkCQ.append(intToBytes(chunkCount), 0, 4);
- m.put(CHUNK_CF, chunkCQ, cv, new Value(buf, 0, numRead));
- bw.addMutation(m);
- if (chunkCount == Integer.MAX_VALUE)
- throw new RuntimeException("too many chunks for file " + filename + ", try raising chunk size");
- chunkCount++;
+ try {
+ fis = new FileInputStream(filename);
numRead = fis.read(buf);
+ while (numRead >= 0) {
+ while (numRead < buf.length) {
+ int moreRead = fis.read(buf, numRead, buf.length - numRead);
+ if (moreRead > 0)
+ numRead += moreRead;
+ else if (moreRead < 0)
+ break;
+ }
+ m = new Mutation(row);
+ Text chunkCQ = new Text(chunkSizeBytes);
+ chunkCQ.append(intToBytes(chunkCount), 0, 4);
+ m.put(CHUNK_CF, chunkCQ, cv, new Value(buf, 0, numRead));
+ bw.addMutation(m);
+ if (chunkCount == Integer.MAX_VALUE)
+ throw new RuntimeException("too many chunks for file " + filename + ", try raising chunk size");
+ chunkCount++;
+ numRead = fis.read(buf);
+ }
+ } finally {
+ if (fis != null) {
+ fis.close();
+ }
}
- fis.close();
m = new Mutation(row);
Text chunkCQ = new Text(chunkSizeBytes);
chunkCQ.append(intToBytes(chunkCount), 0, 4);
@@ -122,14 +127,14 @@ public class FileDataIngest {
bw.addMutation(m);
return hash;
}
-
+
public static int bytesToInt(byte[] b, int offset) {
if (b.length <= offset + 3)
throw new NumberFormatException("couldn't pull integer from bytes at offset " + offset);
int i = (((b[offset] & 255) << 24) + ((b[offset + 1] & 255) << 16) + ((b[offset + 2] & 255) << 8) + ((b[offset + 3] & 255) << 0));
return i;
}
-
+
public static byte[] intToBytes(int l) {
byte[] b = new byte[4];
b[0] = (byte) (l >>> 24);
@@ -138,13 +143,13 @@ public class FileDataIngest {
b[3] = (byte) (l >>> 0);
return b;
}
-
+
private static String getExt(String filename) {
if (filename.indexOf(".") == -1)
return null;
return filename.substring(filename.lastIndexOf(".") + 1);
}
-
+
public String hexString(byte[] bytes) {
StringBuilder sb = new StringBuilder();
for (byte b : bytes) {
@@ -152,14 +157,14 @@ public class FileDataIngest {
}
return sb.toString();
}
-
+
public static void main(String[] args) throws Exception {
if (args.length < 8) {
System.out.println("usage: " + FileDataIngest.class.getSimpleName()
+ " <instance> <zoo> <user> <pass> <data table> <visibility> <data chunk size> <file>{ <file>}");
System.exit(1);
}
-
+
String instance = args[0];
String zooKeepers = args[1];
String user = args[2];
@@ -167,7 +172,7 @@ public class FileDataIngest {
String dataTable = args[4];
ColumnVisibility colvis = new ColumnVisibility(args[5]);
int chunkSize = Integer.parseInt(args[6]);
-
+
Connector conn = new ZooKeeperInstance(instance, zooKeepers).getConnector(user, pass.getBytes());
if (!conn.tableOperations().exists(dataTable)) {
conn.tableOperations().create(dataTable);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/335f693a/src/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java b/src/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
index bf8c160..4eb6289 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
@@ -108,14 +108,26 @@ public class LogReader {
if (fs.isFile(path)) {
// read log entries from a simple hdfs file
org.apache.hadoop.io.SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(file), conf);
- while (reader.next(key, value)) {
- printLogEvent(key, value, row, rowMatcher, ke, tabletIds, max);
+ try {
+ while (reader.next(key, value)) {
+ printLogEvent(key, value, row, rowMatcher, ke, tabletIds, max);
+ }
+ } finally {
+ if (reader != null) {
+ reader.close();
+ }
}
} else if (local.isFile(path)) {
// read log entries from a simple file
org.apache.hadoop.io.SequenceFile.Reader reader = new SequenceFile.Reader(local, new Path(file), conf);
- while (reader.next(key, value)) {
- printLogEvent(key, value, row, rowMatcher, ke, tabletIds, max);
+ try {
+ while (reader.next(key, value)) {
+ printLogEvent(key, value, row, rowMatcher, ke, tabletIds, max);
+ }
+ } finally {
+ if (reader != null) {
+ reader.close();
+ }
}
} else {
try {
@@ -129,6 +141,7 @@ public class LogReader {
while (input.next(key, value)) {
printLogEvent(key, value, row, rowMatcher, ke, tabletIds, max);
}
+ input.close();
}
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/335f693a/src/server/src/main/java/org/apache/accumulo/server/logger/LogService.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/logger/LogService.java b/src/server/src/main/java/org/apache/accumulo/server/logger/LogService.java
index ebfc287..6fabd95 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/logger/LogService.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/logger/LogService.java
@@ -92,34 +92,34 @@ import org.apache.zookeeper.Watcher.Event.KeeperState;
*/
public class LogService implements MutationLogger.Iface, Watcher {
static final org.apache.log4j.Logger LOG = org.apache.log4j.Logger.getLogger(LogService.class);
-
+
private Configuration conf;
private Authenticator authenticator;
private TServer service;
private LogWriter writer_;
private MutationLogger.Iface writer;
private String ephemeralNode;
-
+
enum ShutdownState {
STARTED, REGISTERED, WAITING_FOR_HALT, HALT
};
-
+
private ShutdownState shutdownState = ShutdownState.STARTED;
-
+
synchronized void switchState(ShutdownState state) {
LOG.info("Switching from " + shutdownState + " to " + state);
shutdownState = state;
}
-
+
synchronized private void closedCheck() throws LoggerClosedException {
if (!shutdownState.equals(ShutdownState.REGISTERED))
throw new LoggerClosedException();
}
-
+
private List<FileLock> fileLocks = new ArrayList<FileLock>();
-
+
private final String addressString;
-
+
public static void main(String[] args) throws Exception {
LogService logService;
SecurityUtil.serverLogin();
@@ -136,16 +136,16 @@ public class LogService implements MutationLogger.Iface, Watcher {
LOG.error("Unexpected exception, exiting.", ex);
}
}
-
+
public LogService(String[] args) throws UnknownHostException, KeeperException, InterruptedException, IOException {
try {
Accumulo.init("logger");
} catch (UnknownHostException e1) {
LOG.error("Error reading logging configuration");
}
-
+
FileSystemMonitor.start(Property.LOGGER_MONITOR_FS);
-
+
conf = CachedConfiguration.getInstance();
FileSystem fs = null;
try {
@@ -168,34 +168,45 @@ public class LogService implements MutationLogger.Iface, Watcher {
}
rootDirs.add(root);
}
-
+
for (String root : rootDirs) {
File rootFile = new File(root);
rootFile.mkdirs();
+ @SuppressWarnings("resource")
FileOutputStream lockOutputStream = new FileOutputStream(root + "/.lock");
FileLock fileLock = lockOutputStream.getChannel().tryLock();
if (fileLock == null)
- throw new IOException("Failed to acquire lock file");
+ try {
+ throw new IOException("Failed to acquire lock file");
+ } finally {
+ if (lockOutputStream != null)
+ lockOutputStream.close();
+ }
fileLocks.add(fileLock);
-
+
try {
File test = new File(root, "test_writable");
if (!test.mkdir())
throw new RuntimeException("Unable to write to write-ahead log directory " + root);
test.delete();
} catch (Throwable t) {
- LOG.fatal("Unable to write to write-ahead log directory", t);
- throw new RuntimeException(t);
+ try {
+ LOG.fatal("Unable to write to write-ahead log directory", t);
+ throw new RuntimeException(t);
+ } finally {
+ if (lockOutputStream != null)
+ lockOutputStream.close();
+ }
}
LOG.info("Storing recovery logs at " + root);
}
-
+
authenticator = ZKAuthenticator.getInstance();
int poolSize = ServerConfiguration.getSystemConfiguration().getCount(Property.LOGGER_COPY_THREADPOOL_SIZE);
boolean archive = ServerConfiguration.getSystemConfiguration().getBoolean(Property.LOGGER_ARCHIVE);
AccumuloConfiguration acuConf = ServerConfiguration.getSystemConfiguration();
writer_ = new LogWriter(acuConf, fs, rootDirs, HdfsZooInstance.getInstance().getInstanceID(), poolSize, archive);
-
+
// call before putting this service online
removeIncompleteCopies(acuConf, fs, rootDirs);
@@ -241,7 +252,7 @@ public class LogService implements MutationLogger.Iface, Watcher {
this.switchState(ShutdownState.REGISTERED);
Accumulo.enableTracing(address.getHostName(), "logger");
}
-
+
/**
* @param acuConf
* @param fs
@@ -265,7 +276,7 @@ public class LogService implements MutationLogger.Iface, Watcher {
}
}
}
-
+
// look for .recovered that are not finished
for (String walog : walogs) {
Path path = new Path(ServerConstants.getRecoveryDir() + "/" + walog + ".recovered");
@@ -292,7 +303,7 @@ public class LogService implements MutationLogger.Iface, Watcher {
}
Runtime.getRuntime().halt(0);
}
-
+
void registerInZooKeeper(String zooDir) {
try {
IZooReaderWriter zoo = ZooReaderWriter.getInstance();
@@ -305,7 +316,7 @@ public class LogService implements MutationLogger.Iface, Watcher {
throw new RuntimeException("Unexpected error creating zookeeper entry " + zooDir);
}
}
-
+
private void checkForSystemPrivs(String request, AuthInfo credentials) throws ThriftSecurityException {
try {
if (!authenticator.hasSystemPermission(credentials, credentials.user, SystemPermission.SYSTEM)) {
@@ -317,13 +328,13 @@ public class LogService implements MutationLogger.Iface, Watcher {
throw e.asThriftException();
}
}
-
+
@Override
public void close(TInfo info, long id) throws NoSuchLogIDException, LoggerClosedException, TException {
closedCheck();
writer.close(info, id);
}
-
+
@Override
public LogCopyInfo startCopy(TInfo info, AuthInfo credentials, String localLog, String fullyQualifiedFileName, boolean sort) throws ThriftSecurityException,
TException {
@@ -332,50 +343,50 @@ public class LogService implements MutationLogger.Iface, Watcher {
lci.loggerZNode = ephemeralNode;
return lci;
}
-
+
@Override
public LogFile create(TInfo info, AuthInfo credentials, String tserverSession) throws ThriftSecurityException, LoggerClosedException, TException {
checkForSystemPrivs("create", credentials);
closedCheck();
return writer.create(info, credentials, tserverSession);
}
-
+
@Override
public void log(TInfo info, long id, long seq, int tid, TMutation mutation) throws NoSuchLogIDException, LoggerClosedException, TException {
closedCheck();
writer.log(info, id, seq, tid, mutation);
}
-
+
@Override
public void logManyTablets(TInfo info, long id, List<TabletMutations> mutations) throws NoSuchLogIDException, LoggerClosedException, TException {
closedCheck();
writer.logManyTablets(info, id, mutations);
}
-
+
@Override
public void minorCompactionFinished(TInfo info, long id, long seq, int tid, String fqfn) throws NoSuchLogIDException, LoggerClosedException, TException {
closedCheck();
writer.minorCompactionFinished(info, id, seq, tid, fqfn);
}
-
+
@Override
public void minorCompactionStarted(TInfo info, long id, long seq, int tid, String fqfn) throws NoSuchLogIDException, LoggerClosedException, TException {
closedCheck();
writer.minorCompactionStarted(info, id, seq, tid, fqfn);
}
-
+
@Override
public void defineTablet(TInfo info, long id, long seq, int tid, TKeyExtent tablet) throws NoSuchLogIDException, LoggerClosedException, TException {
closedCheck();
writer.defineTablet(info, id, seq, tid, tablet);
}
-
+
@Override
public void process(WatchedEvent event) {
LOG.debug("event " + event.getPath() + " " + event.getType() + " " + event.getState());
if (event.getState() == KeeperState.Disconnected)
return;
-
+
if (event.getState() == KeeperState.Expired) {
LOG.warn("Logger lost zookeeper registration at " + event.getPath());
service.stop();
@@ -400,13 +411,13 @@ public class LogService implements MutationLogger.Iface, Watcher {
service.stop();
}
}
-
+
@Override
public List<String> getClosedLogs(TInfo info, AuthInfo credentials) throws ThriftSecurityException, TException {
checkForSystemPrivs("getClosedLogs", credentials);
return writer.getClosedLogs(info, credentials);
}
-
+
@Override
public void remove(TInfo info, AuthInfo credentials, List<String> files) throws TException {
try {
@@ -416,7 +427,7 @@ public class LogService implements MutationLogger.Iface, Watcher {
LOG.error(ex, ex);
}
}
-
+
@Override
public void beginShutdown(TInfo tinfo, AuthInfo credentials) throws TException {
try {
@@ -427,7 +438,7 @@ public class LogService implements MutationLogger.Iface, Watcher {
LOG.error(ex, ex);
}
}
-
+
@Override
public void halt(TInfo tinfo, AuthInfo credentials) throws TException {
try {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/335f693a/src/server/src/main/java/org/apache/accumulo/server/test/MidPointPerfTest2.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/test/MidPointPerfTest2.java b/src/server/src/main/java/org/apache/accumulo/server/test/MidPointPerfTest2.java
index 2029b82..f55fcab 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/test/MidPointPerfTest2.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/test/MidPointPerfTest2.java
@@ -41,59 +41,60 @@ import org.apache.hadoop.io.WritableComparable;
*/
@Deprecated
class MultipleIndexIterator2 {
-
+
private MySequenceFile.Reader[] readers;
private boolean[] hasNextKey;
private Key[] nextKey;
private int currentMin = 0;
-
+
int findMinimum() {
int minIndex = -1;
for (int i = 0; i < nextKey.length; i++) {
if (!hasNextKey[i]) {
continue;
}
-
+
if (minIndex == -1) {
minIndex = i;
} else if (nextKey[i].compareTo(nextKey[minIndex]) < 0) {
minIndex = i;
}
}
-
+
return minIndex;
}
-
+
MultipleIndexIterator2(Configuration conf, FileSystem fs, List<Path> paths) throws IOException, InstantiationException, IllegalAccessException {
readers = new MySequenceFile.Reader[paths.size()];
-
+
int ri = 0;
for (Path path : paths) {
+ @SuppressWarnings("resource")
MySequenceFile.Reader index = new MySequenceFile.Reader(fs, path, conf);
readers[ri++] = index;
}
-
+
hasNextKey = new boolean[readers.length];
nextKey = new Key[readers.length];
for (int i = 0; i < readers.length; i++) {
nextKey[i] = (Key) readers[i].getKeyClass().newInstance();
hasNextKey[i] = readers[i].next(nextKey[i]);
}
-
+
currentMin = findMinimum();
}
-
+
boolean hasNext() {
return currentMin >= 0;
}
-
+
WritableComparable<?> next() {
if (currentMin < 0) {
throw new RuntimeException("There is no next");
}
-
+
WritableComparable<?> ret = nextKey[currentMin];
-
+
try {
nextKey[currentMin] = (Key) readers[currentMin].getKeyClass().newInstance();
hasNextKey[currentMin] = readers[currentMin].next(nextKey[currentMin]);
@@ -104,15 +105,15 @@ class MultipleIndexIterator2 {
} catch (IOException e) {
throw new RuntimeException(e);
}
-
+
currentMin = findMinimum();
-
+
return ret;
}
-
+
public void close() {
currentMin = -1;
-
+
for (Reader reader : readers) {
try {
reader.close();
@@ -136,27 +137,27 @@ public class MidPointPerfTest2 {
private static int maxFiles;
private static String tmpDir;
private static boolean createTestData = false;
-
+
public static void main(String[] args) {
-
+
for (String string : args) {
System.out.println("arg : " + string);
}
-
+
if (args[0].equals("-ctd")) {
dir = args[1];
numFiles = Integer.parseInt(args[2]);
numEntries = Integer.parseInt(args[3]);
min = Integer.parseInt(args[4]);
max = Integer.parseInt(args[5]);
-
+
createTestData = true;
} else {
dir = args[0];
maxFiles = Integer.parseInt(args[1]);
tmpDir = args[2];
}
-
+
try {
if (createTestData) {
Random r = new Random();
@@ -168,124 +169,125 @@ public class MidPointPerfTest2 {
throw new RuntimeException(e);
}
}
-
+
private static void timeIterate(String dir, int maxFiles, String tmpDir) throws Exception {
Configuration conf = CachedConfiguration.getInstance();
FileSystem fs = FileSystem.get(conf);
-
+
FileStatus[] files = fs.globStatus(new Path(dir + "/*/index"));
ArrayList<Path> paths = new ArrayList<Path>(files.length);
-
+
for (FileStatus fileStatus : files) {
paths.add(fileStatus.getPath());
}
-
+
long t1 = System.currentTimeMillis();
ArrayList<Path> rpaths = reduceFiles(conf, fs, paths, maxFiles, tmpDir, 0);
long t2 = System.currentTimeMillis();
-
+
MultipleIndexIterator2 mii = new MultipleIndexIterator2(conf, fs, rpaths);
-
+
int count = 0;
while (mii.hasNext()) {
mii.next();
count++;
}
-
+
long t3 = System.currentTimeMillis();
-
+
System.out.printf("reduce time : %6.2f secs \n", (t2 - t1) / 1000.0);
System.out.printf("iterate time : %6.2f secs \n", (t3 - t2) / 1000.0);
System.out.printf("total time : %6.2f secs \n", (t3 - t1) / 1000.0);
-
+
System.out.println("count " + count);
}
-
+
private static ArrayList<Path> reduceFiles(Configuration conf, FileSystem fs, ArrayList<Path> paths, int maxFiles, String tmpDir, int pass)
throws IOException, InstantiationException, IllegalAccessException {
if (paths.size() <= maxFiles) {
return paths;
}
-
+
String newDir = String.format("%s/pass_%04d", tmpDir, pass);
fs.mkdirs(new Path(newDir));
-
+
int start = 0;
-
+
ArrayList<Path> outFiles = new ArrayList<Path>();
-
+
int count = 0;
-
+
while (start < paths.size()) {
int end = Math.min(maxFiles + start, paths.size());
List<Path> inFiles = paths.subList(start, end);
-
+
start = end;
-
+
Path outFile = new Path(String.format("%s/index_%04d", newDir, count++));
outFiles.add(outFile);
-
+
long t1 = System.currentTimeMillis();
-
+
MySequenceFile.Writer writer = MySequenceFile.createWriter(fs, conf, outFile, Key.class, LongWritable.class, MySequenceFile.CompressionType.BLOCK);
MultipleIndexIterator2 mii = new MultipleIndexIterator2(conf, fs, inFiles);
-
+
while (mii.hasNext()) {
writer.append(mii.next(), new LongWritable(0));
}
-
+
mii.close();
writer.close();
-
+
long t2 = System.currentTimeMillis();
-
+
System.out.printf("out : %s num in : %d time : %6.2f secs\n", outFile, inFiles.size(), (t2 - t1) / 1000.0);
}
-
+
return reduceFiles(conf, fs, outFiles, maxFiles, tmpDir, pass + 1);
}
-
+
static class CompareKeys implements Comparator<Key> {
public CompareKeys() {}
-
+
public boolean equals(Key a, Key b) {
return a.equals(b);
}
-
+
+ @Override
public int compare(Key a, Key b) {
return a.compareTo(b);
}
}
-
+
private static void createTestData(String dir, int numFiles, int numEntries, int min, int max, Random r) throws Exception {
Configuration conf = CachedConfiguration.getInstance();
FileSystem fs = FileSystem.get(conf);
-
+
for (int i = 0; i < numFiles; i++) {
String newDir = String.format("%s/" + MyMapFile.EXTENSION + "_%06d", dir, i);
fs.mkdirs(new Path(newDir));
-
+
List<Key> keys = new ArrayList<Key>();
-
+
for (int j = 0; j < numEntries; j++) {
String row = String.format("row_%010d", r.nextInt() % (max - min) + min);
Key key1 = new Key(new Text(row), new Text(String.format("cf_%03d", r.nextInt() % 100)), new Text(String.format("cf_%05d", r.nextInt() % 10000)));
keys.add(key1);
}
-
+
Collections.sort(keys, new CompareKeys());
-
+
MySequenceFile.Writer writer = MySequenceFile.createWriter(fs, conf, new Path(newDir + "/index"), Key.class, LongWritable.class,
MySequenceFile.CompressionType.BLOCK);
-
+
System.out.println(new Path(newDir + "/index"));
-
+
for (Key key : keys) {
writer.append(key, new LongWritable(0));
}
-
+
writer.close();
}
}
-
+
}