You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2012/01/24 22:25:57 UTC
svn commit: r1235509 - in /incubator/accumulo/branches/1.4: ./ docs/
src/core/src/main/java/org/apache/accumulo/core/client/
src/core/src/main/java/org/apache/accumulo/core/client/admin/
src/core/src/main/java/org/apache/accumulo/core/client/impl/thrif...
Author: ecn
Date: Tue Jan 24 21:25:56 2012
New Revision: 1235509
URL: http://svn.apache.org/viewvc?rev=1235509&view=rev
Log:
ACCUMULO-334:
- switched bulk test over to using Combiners, not Aggregators
- cleaned up additional deprecation warnings
- added a marker column to the bulk test the aid in debugging lost data
- discovered how awkward the scopes were in IteratorSetting, so I removed them
- did not reproduce the bulk test failure
Modified:
incubator/accumulo/branches/1.4/docs/config.html
incubator/accumulo/branches/1.4/pom.xml
incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsHelper.java
incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/thrift/ClientService.java
incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/master/thrift/MasterClientService.java
incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/MutationLogger.java
incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java
incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java
incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/client/admin/TableOperationsHelperTest.java
incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormatTest.java
incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java
incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/client/mock/MockTableOperationsTest.java
incubator/accumulo/branches/1.4/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/InsertWithOutputFormat.java
incubator/accumulo/branches/1.4/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/RowHash.java
incubator/accumulo/branches/1.4/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java
incubator/accumulo/branches/1.4/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/WordCount.java
incubator/accumulo/branches/1.4/src/examples/wikisearch/query-war/pom.xml
incubator/accumulo/branches/1.4/src/examples/wikisearch/query/pom.xml
incubator/accumulo/branches/1.4/src/examples/wikisearch/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/TestQueryLogic.java
incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousMoru.java
incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/BulkPlusOne.java
incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/ConsistencyCheck.java
incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Setup.java
incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Verify.java
incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/multitable/CopyTool.java
incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/sequential/MapRedVerifyTool.java
incubator/accumulo/branches/1.4/test/system/randomwalk/conf/ (props changed)
Modified: incubator/accumulo/branches/1.4/docs/config.html
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/docs/config.html?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/docs/config.html (original)
+++ incubator/accumulo/branches/1.4/docs/config.html Tue Jan 24 21:25:56 2012
@@ -834,34 +834,41 @@ $HADOOP_HOME/lib/[^.].*.jar,
<td>Change the type of file a table writes</td>
</tr>
<tr >
+ <td>table.formatter</td>
+ <td><b><a href='#STRING'>string</a></b></td>
+ <td>yes</td>
+ <td><pre>org.apache.accumulo.core.util.format.DefaultFormatter</pre></td>
+ <td>The Formatter class to apply on results in the shell</td>
+ </tr>
+ <tr class='highlight'>
<td>table.groups.enabled</td>
<td><b><a href='#STRING'>string</a></b></td>
<td>yes</td>
<td><pre> </pre></td>
<td>A comma separated list of locality group names to enable for this table.</td>
</tr>
- <tr class='highlight'>
+ <tr >
<td>table.scan.max.memory</td>
<td><b><a href='#MEMORY'>memory</a></b></td>
<td>yes</td>
<td><pre>1M</pre></td>
<td>The maximum amount of memory that will be used to cache results of a client query/scan. Once this limit is reached, the buffered data is sent to the client.</td>
</tr>
- <tr >
+ <tr class='highlight'>
<td>table.security.scan.visibility.default</td>
<td><b><a href='#STRING'>string</a></b></td>
<td>yes</td>
<td><pre> </pre></td>
<td>The security label that will be assumed at scan time if an entry does not have a visibility set.<br />Note: An empty security label is displayed as []. The scan results will show an empty visibility even if the visibility from this setting is applied to the entry.<br />CAUTION: If a particular key has an empty security label AND its table's default visibility is also empty, access will ALWAYS be granted for users with permission to that table. Additionally, if this field is changed, all existing data with an empty visibility label will be interpreted with the new label on the next scan.</td>
</tr>
- <tr class='highlight'>
+ <tr >
<td>table.split.threshold</td>
<td><b><a href='#MEMORY'>memory</a></b></td>
<td>yes</td>
<td><pre>1G</pre></td>
<td>When combined size of files exceeds this amount a tablet is split.</td>
</tr>
- <tr >
+ <tr class='highlight'>
<td>table.walog.enabled</td>
<td><b><a href='#BOOLEAN'>boolean</a></b></td>
<td>yes</td>
Modified: incubator/accumulo/branches/1.4/pom.xml
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/pom.xml?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/pom.xml (original)
+++ incubator/accumulo/branches/1.4/pom.xml Tue Jan 24 21:25:56 2012
@@ -49,7 +49,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
- <version>1.0.1</version>
<executions>
<execution>
<id>enforce-mvn</id>
@@ -139,7 +138,6 @@
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
- <version>2.5</version>
<configuration>
<encoding>UTF-8</encoding>
</configuration>
@@ -163,7 +161,6 @@
</plugin>
<plugin>
<artifactId>maven-javadoc-plugin</artifactId>
- <version>2.8</version>
<configuration>
<encoding>UTF-8</encoding>
<quiet>true</quiet>
@@ -175,7 +172,6 @@
</plugin>
<plugin>
<artifactId>maven-source-plugin</artifactId>
- <version>2.1.2</version>
<configuration>
<outputDirectory>../../lib</outputDirectory>
</configuration>
Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java Tue Jan 24 21:25:56 2012
@@ -47,7 +47,6 @@ public class IteratorSetting {
private int priority;
private String name;
private String iteratorClass;
- private EnumSet<IteratorScope> scopes;
private Map<String,String> properties;
/**
@@ -111,28 +110,6 @@ public class IteratorSetting {
}
/**
- * Get the scopes under which this iterator will be configured.
- *
- * @return the scopes
- */
- public EnumSet<IteratorScope> getScopes() {
- return scopes;
- }
-
- /**
- * Set the scopes under which this iterator will be configured.
- *
- * @param scopes
- * the scopes to set
- */
- public void setScopes(EnumSet<IteratorScope> scopes) {
- ArgumentChecker.notNull(scopes);
- if (scopes.isEmpty())
- throw new IllegalArgumentException("empty scopes");
- this.scopes = scopes;
- }
-
- /**
* Get the configuration parameters for this iterator.
*
* @return the properties
@@ -170,7 +147,7 @@ public class IteratorSetting {
* the fully qualified class name for the iterator
*/
public IteratorSetting(int priority, String name, String iteratorClass) {
- this(priority, name, iteratorClass, EnumSet.of(IteratorScope.scan), new HashMap<String,String>());
+ this(priority, name, iteratorClass, new HashMap<String,String>());
}
/**
@@ -187,11 +164,10 @@ public class IteratorSetting {
* @param properties
* any properties for the iterator
*/
- public IteratorSetting(int priority, String name, String iteratorClass, EnumSet<IteratorScope> scopes, Map<String,String> properties) {
+ public IteratorSetting(int priority, String name, String iteratorClass, Map<String,String> properties) {
setPriority(priority);
setName(name);
setIteratorClass(iteratorClass);
- setScopes(scopes);
this.properties = new HashMap<String,String>();
setProperties(properties);
}
@@ -222,9 +198,9 @@ public class IteratorSetting {
* @param properties
* any properties for the iterator
*/
- public IteratorSetting(int priority, Class<? extends SortedKeyValueIterator<Key,Value>> iteratorClass, EnumSet<IteratorScope> scopes,
+ public IteratorSetting(int priority, Class<? extends SortedKeyValueIterator<Key,Value>> iteratorClass,
Map<String,String> properties) {
- this(priority, iteratorClass.getSimpleName(), iteratorClass.getName(), scopes, properties);
+ this(priority, iteratorClass.getSimpleName(), iteratorClass.getName(), properties);
}
/**
@@ -257,7 +233,7 @@ public class IteratorSetting {
*/
public IteratorSetting(int priority, String name, Class<? extends SortedKeyValueIterator<Key,Value>> iteratorClass, EnumSet<IteratorScope> scopes,
Map<String,String> properties) {
- this(priority, name, iteratorClass.getName(), scopes, properties);
+ this(priority, name, iteratorClass.getName(), properties);
}
/**
@@ -328,8 +304,6 @@ public class IteratorSetting {
sb.append(Integer.toString(priority));
sb.append(", class:");
sb.append(iteratorClass);
- sb.append(", scopes:");
- sb.append(scopes);
sb.append(", properties:");
sb.append(properties);
return sb.toString();
@@ -356,5 +330,13 @@ public class IteratorSetting {
super(new Text(columnFamily), null);
}
+ public Text getColumnFamily() {
+ return getFirst();
+ }
+
+ public Text getColumnQualifierf() {
+ return getSecond();
+ }
+
}
}
Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java Tue Jan 24 21:25:56 2012
@@ -460,7 +460,7 @@ public interface TableOperations {
public Map<String,String> tableIdMap();
/**
- * Add an iterator to a table.
+ * Add an iterator to a table on all scopes.
*
* @param tableName
* the name of the table
@@ -477,6 +477,24 @@ public interface TableOperations {
public void attachIterator(String tableName, IteratorSetting setting) throws AccumuloSecurityException, AccumuloException, TableNotFoundException;
/**
+ * Add an iterator to a table on the given scopes.
+ *
+ * @param tableName
+ * the name of the table
+ * @param setting
+ * object specifying the properties of the iterator
+ * @throws AccumuloSecurityException
+ * thrown if the user does not have the ability to set properties on the table
+ * @throws AccumuloException
+ * @throws TableNotFoundException
+ * throw if the table no longer exists
+ * @throws IllegalArgumentException
+ * if the setting conflicts with any existing iterators
+ */
+ public void attachIterator(String tableName, IteratorSetting setting, EnumSet<IteratorScope> scopes) throws AccumuloSecurityException, AccumuloException,
+ TableNotFoundException;
+
+ /**
* Remove an iterator from a table by name.
*
* @param tableName
@@ -538,5 +556,5 @@ public interface TableOperations {
* @throws IllegalStateException
* if the setting conflicts with any existing iterators
*/
- public void checkIteratorConflicts(String tableName, IteratorSetting setting) throws AccumuloException, TableNotFoundException;
+ public void checkIteratorConflicts(String tableName, IteratorSetting setting, EnumSet<IteratorScope> scopes) throws AccumuloException, TableNotFoundException;
}
Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsHelper.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsHelper.java?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsHelper.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsHelper.java Tue Jan 24 21:25:56 2012
@@ -30,11 +30,17 @@ import org.apache.accumulo.core.conf.Pro
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
public abstract class TableOperationsHelper implements TableOperations {
-
+
@Override
public void attachIterator(String tableName, IteratorSetting setting) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
- checkIteratorConflicts(tableName, setting);
- for (IteratorScope scope : setting.getScopes()) {
+ attachIterator(tableName, setting, EnumSet.allOf(IteratorScope.class));
+ }
+
+ @Override
+ public void attachIterator(String tableName, IteratorSetting setting, EnumSet<IteratorScope> scopes) throws AccumuloSecurityException, AccumuloException,
+ TableNotFoundException {
+ checkIteratorConflicts(tableName, setting, scopes);
+ for (IteratorScope scope : scopes) {
String root = String.format("%s%s.%s", Property.TABLE_ITERATOR_PREFIX, scope.name().toLowerCase(), setting.getName());
for (Entry<String,String> prop : setting.getProperties().entrySet()) {
this.setProperty(tableName, root + ".opt." + prop.getKey(), prop.getValue());
@@ -87,7 +93,7 @@ public abstract class TableOperationsHel
if (priority <= 0 || classname == null) {
return null;
}
- return new IteratorSetting(priority, name, classname, EnumSet.of(scope), settings);
+ return new IteratorSetting(priority, name, classname, settings);
}
@Override
@@ -111,10 +117,10 @@ public abstract class TableOperationsHel
}
@Override
- public void checkIteratorConflicts(String tableName, IteratorSetting setting) throws AccumuloException, TableNotFoundException {
+ public void checkIteratorConflicts(String tableName, IteratorSetting setting, EnumSet<IteratorScope> scopes) throws AccumuloException, TableNotFoundException {
if (!exists(tableName))
throw new TableNotFoundException(null, tableName, null);
- for (IteratorScope scope : setting.getScopes()) {
+ for (IteratorScope scope : scopes) {
String scopeStr = String.format("%s%s", Property.TABLE_ITERATOR_PREFIX, scope.name().toLowerCase());
String nameStr = String.format("%s.%s", scopeStr, setting.getName());
String optStr = String.format("%s.opt.", nameStr);
Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/thrift/ClientService.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/thrift/ClientService.java?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/thrift/ClientService.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/thrift/ClientService.java Tue Jan 24 21:25:56 2012
@@ -6262,8 +6262,6 @@ import org.slf4j.LoggerFactory;
private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bit_vector = new BitSet(1);
read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
} catch (org.apache.thrift.TException te) {
throw new java.io.IOException(te);
@@ -7821,8 +7819,6 @@ import org.slf4j.LoggerFactory;
private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bit_vector = new BitSet(1);
read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
} catch (org.apache.thrift.TException te) {
throw new java.io.IOException(te);
@@ -14037,8 +14033,6 @@ import org.slf4j.LoggerFactory;
private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bit_vector = new BitSet(1);
read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
} catch (org.apache.thrift.TException te) {
throw new java.io.IOException(te);
@@ -15168,8 +15162,6 @@ import org.slf4j.LoggerFactory;
private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bit_vector = new BitSet(1);
read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
} catch (org.apache.thrift.TException te) {
throw new java.io.IOException(te);
@@ -21131,8 +21123,6 @@ import org.slf4j.LoggerFactory;
private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bit_vector = new BitSet(1);
read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
} catch (org.apache.thrift.TException te) {
throw new java.io.IOException(te);
Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java Tue Jan 24 21:25:56 2012
@@ -210,8 +210,10 @@ public class IteratorUtil {
Collection<IterInfo> iters, Map<String,Map<String,String>> iterOpts, IteratorEnvironment env) throws IOException {
SortedKeyValueIterator<K,V> prev = source;
+ log.debug("Iterator options: " + iterOpts);
try {
for (IterInfo iterInfo : iters) {
+ log.debug("Loading " + iterInfo.className + " " + iterInfo.iterName + " " + iterInfo.priority);
@SuppressWarnings("unchecked")
Class<? extends SortedKeyValueIterator<K,V>> clazz = (Class<? extends SortedKeyValueIterator<K,V>>) AccumuloClassLoader.loadClass(iterInfo.className,
SortedKeyValueIterator.class);
Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/master/thrift/MasterClientService.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/master/thrift/MasterClientService.java?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/master/thrift/MasterClientService.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/master/thrift/MasterClientService.java Tue Jan 24 21:25:56 2012
@@ -2990,8 +2990,6 @@ import org.slf4j.LoggerFactory;
private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bit_vector = new BitSet(1);
read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
} catch (org.apache.thrift.TException te) {
throw new java.io.IOException(te);
@@ -13071,8 +13069,6 @@ import org.slf4j.LoggerFactory;
private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bit_vector = new BitSet(1);
read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
} catch (org.apache.thrift.TException te) {
throw new java.io.IOException(te);
Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/MutationLogger.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/MutationLogger.java?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/MutationLogger.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/MutationLogger.java Tue Jan 24 21:25:56 2012
@@ -9319,8 +9319,6 @@ import org.slf4j.LoggerFactory;
private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bit_vector = new BitSet(1);
read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
} catch (org.apache.thrift.TException te) {
throw new java.io.IOException(te);
Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java Tue Jan 24 21:25:56 2012
@@ -9884,8 +9884,6 @@ import org.slf4j.LoggerFactory;
private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bit_vector = new BitSet(1);
read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
} catch (org.apache.thrift.TException te) {
throw new java.io.IOException(te);
Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java Tue Jan 24 21:25:56 2012
@@ -36,8 +36,8 @@ import org.apache.accumulo.core.iterator
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.aggregation.Aggregator;
import org.apache.accumulo.core.iterators.user.AgeOffFilter;
-import org.apache.accumulo.core.iterators.user.ReqVisFilter;
import org.apache.accumulo.core.iterators.user.RegExFilter;
+import org.apache.accumulo.core.iterators.user.ReqVisFilter;
import org.apache.accumulo.core.iterators.user.VersioningIterator;
import org.apache.accumulo.core.util.shell.Shell;
import org.apache.accumulo.core.util.shell.Shell.Command;
@@ -121,8 +121,8 @@ public class SetIterCommand extends Comm
if (scopes.isEmpty())
throw new IllegalArgumentException("You must select at least one scope to configure");
- IteratorSetting setting = new IteratorSetting(priority, name, classname, scopes, options);
- shellState.getConnector().tableOperations().attachIterator(tableName, setting);
+ IteratorSetting setting = new IteratorSetting(priority, name, classname, options);
+ shellState.getConnector().tableOperations().attachIterator(tableName, setting, scopes);
}
private static String setUpOptions(ConsoleReader reader, String className, Map<String,String> options) throws IOException, ShellCommandException {
Modified: incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/client/admin/TableOperationsHelperTest.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/client/admin/TableOperationsHelperTest.java?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/client/admin/TableOperationsHelperTest.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/client/admin/TableOperationsHelperTest.java Tue Jan 24 21:25:56 2012
@@ -200,27 +200,25 @@ public class TableOperationsHelperTest {
public void testAttachIterator() throws Exception {
Tester t = new Tester();
Map<String,String> empty = Collections.emptyMap();
- t.attachIterator("table", new IteratorSetting(10, "someName", "foo.bar", EnumSet.of(IteratorScope.scan), empty));
+ t.attachIterator("table", new IteratorSetting(10, "someName", "foo.bar", empty), EnumSet.of(IteratorScope.scan));
t.check("table", new String[] {"table.iterator.scan.someName=10,foo.bar",});
t.removeIterator("table", "someName", EnumSet.of(IteratorScope.scan));
t.check("table", new String[] {});
IteratorSetting setting = new IteratorSetting(10, "someName", "foo.bar");
- setting.setScopes(EnumSet.of(IteratorScope.majc));
setting.addOptions(Collections.singletonMap("key", "value"));
- t.attachIterator("table", setting);
+ t.attachIterator("table", setting, EnumSet.of(IteratorScope.majc));
setting = new IteratorSetting(10, "someName", "foo.bar");
- t.attachIterator("table", setting);
+ t.attachIterator("table", setting, EnumSet.of(IteratorScope.scan));
t.check("table", new String[] {"table.iterator.majc.someName=10,foo.bar", "table.iterator.majc.someName.opt.key=value",
"table.iterator.scan.someName=10,foo.bar",});
t.removeIterator("table", "someName", EnumSet.of(IteratorScope.scan));
setting = new IteratorSetting(20, "otherName", "some.classname");
- setting.setScopes(EnumSet.of(IteratorScope.majc));
setting.addOptions(Collections.singletonMap("key", "value"));
- t.attachIterator("table", setting);
+ t.attachIterator("table", setting, EnumSet.of(IteratorScope.majc));
setting = new IteratorSetting(20, "otherName", "some.classname");
- t.attachIterator("table", setting);
+ t.attachIterator("table", setting, EnumSet.of(IteratorScope.scan));
Map<String,EnumSet<IteratorScope>> two = t.listIterators("table");
Assert.assertEquals(2, two.size());
Assert.assertTrue(two.containsKey("otherName"));
@@ -243,8 +241,7 @@ public class TableOperationsHelperTest {
Assert.assertEquals("some.classname", setting.getIteratorClass());
Assert.assertTrue(setting.hasProperties());
Assert.assertEquals(Collections.singletonMap("key", "value"), setting.getProperties());
- setting.setScopes(EnumSet.of(IteratorScope.minc));
- t.attachIterator("table", setting);
+ t.attachIterator("table", setting, EnumSet.of(IteratorScope.minc));
t.check("table", new String[] {"table.iterator.majc.otherName=20,some.classname", "table.iterator.majc.otherName.opt.key=value",
"table.iterator.minc.otherName=20,some.classname", "table.iterator.minc.otherName.opt.key=value", "table.iterator.scan.otherName=20,some.classname",});
Modified: incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormatTest.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormatTest.java?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormatTest.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormatTest.java Tue Jan 24 21:25:56 2012
@@ -62,7 +62,7 @@ public class AccumuloFileOutputFormatTes
@Test
public void testSet() throws IOException, InterruptedException {
- AccumuloFileOutputFormat.setBlockSize(job, 300);
+ AccumuloFileOutputFormat.setBlockSize(job.getConfiguration(), 300);
validate(300);
}
@@ -100,7 +100,7 @@ public class AccumuloFileOutputFormatTes
}
public void validate(int size) throws IOException, InterruptedException {
- AccumuloFileOutputFormat.handleBlockSize(job);
+ AccumuloFileOutputFormat.handleBlockSize(job.getConfiguration());
int detSize = job.getConfiguration().getInt("io.seqfile.compress.blocksize", -1);
assertEquals(size, detSize);
}
Modified: incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java Tue Jan 24 21:25:56 2012
@@ -33,6 +33,7 @@ import org.apache.accumulo.core.data.Key
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Job;
@@ -93,10 +94,11 @@ public class AccumuloOutputFormatTest {
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Mutation.class);
job.setNumReduceTasks(0);
- AccumuloInputFormat.setInputInfo(job.getConfiguration(), "root", "".getBytes(), "testtable1", new Authorizations());
- AccumuloInputFormat.setMockInstance(job.getConfiguration(), "testmrinstance");
- AccumuloOutputFormat.setOutputInfo(job, "root", "".getBytes(), false, "testtable2");
- AccumuloOutputFormat.setMockInstance(job, "testmrinstance");
+ Configuration conf = job.getConfiguration();
+ AccumuloInputFormat.setInputInfo(conf, "root", "".getBytes(), "testtable1", new Authorizations());
+ AccumuloInputFormat.setMockInstance(conf, "testmrinstance");
+ AccumuloOutputFormat.setOutputInfo(conf, "root", "".getBytes(), false, "testtable2");
+ AccumuloOutputFormat.setMockInstance(conf, "testmrinstance");
AccumuloInputFormat input = new AccumuloInputFormat();
List<InputSplit> splits = input.getSplits(job);
Modified: incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/client/mock/MockTableOperationsTest.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/client/mock/MockTableOperationsTest.java?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/client/mock/MockTableOperationsTest.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/client/mock/MockTableOperationsTest.java Tue Jan 24 21:25:56 2012
@@ -16,12 +16,15 @@
*/
package org.apache.accumulo.core.client.mock;
+import java.util.EnumSet;
+
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.junit.Assert;
import org.junit.Test;
@@ -41,7 +44,7 @@ public class MockTableOperationsTest {
Assert.fail();
} catch (TableNotFoundException e) {}
try {
- conn.tableOperations().checkIteratorConflicts(t, null);
+ conn.tableOperations().checkIteratorConflicts(t, null, EnumSet.allOf(IteratorScope.class));
Assert.fail();
} catch (TableNotFoundException e) {}
try {
Modified: incubator/accumulo/branches/1.4/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/InsertWithOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/InsertWithOutputFormat.java?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/InsertWithOutputFormat.java (original)
+++ incubator/accumulo/branches/1.4/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/InsertWithOutputFormat.java Tue Jan 24 21:25:56 2012
@@ -44,8 +44,8 @@ public class InsertWithOutputFormat exte
Text tableName = new Text(args[2]);
Job job = new Job(getConf());
Configuration conf = job.getConfiguration();
- AccumuloOutputFormat.setZooKeeperInstance(job, args[0], args[1]);
- AccumuloOutputFormat.setOutputInfo(job, args[3], args[4].getBytes(), true, null);
+ AccumuloOutputFormat.setZooKeeperInstance(conf, args[0], args[1]);
+ AccumuloOutputFormat.setOutputInfo(conf, args[3], args[4].getBytes(), true, null);
job.setOutputFormatClass(AccumuloOutputFormat.class);
// when running a mapreduce, you won't need to instantiate the output
Modified: incubator/accumulo/branches/1.4/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/RowHash.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/RowHash.java?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/RowHash.java (original)
+++ incubator/accumulo/branches/1.4/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/RowHash.java Tue Jan 24 21:25:56 2012
@@ -77,8 +77,8 @@ public class RowHash extends Configured
job.setNumReduceTasks(0);
job.setOutputFormatClass(AccumuloOutputFormat.class);
- AccumuloOutputFormat.setZooKeeperInstance(job, args[0], args[1]);
- AccumuloOutputFormat.setOutputInfo(job, args[2], args[3].getBytes(), true, args[6]);
+ AccumuloOutputFormat.setZooKeeperInstance(job.getConfiguration(), args[0], args[1]);
+ AccumuloOutputFormat.setOutputInfo(job.getConfiguration(), args[2], args[3].getBytes(), true, args[6]);
// AccumuloOutputFormat.setLogLevel(job, Level.TRACE);
job.waitForCompletion(true);
Modified: incubator/accumulo/branches/1.4/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java (original)
+++ incubator/accumulo/branches/1.4/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java Tue Jan 24 21:25:56 2012
@@ -356,11 +356,11 @@ public class TeraSortIngest extends Conf
job.setNumReduceTasks(0);
job.setOutputFormatClass(AccumuloOutputFormat.class);
- AccumuloOutputFormat.setZooKeeperInstance(job, args[6], args[7]);
- AccumuloOutputFormat.setOutputInfo(job, args[8], args[9].getBytes(), true, null);
- AccumuloOutputFormat.setMaxMutationBufferSize(job, 10L * 1000 * 1000);
-
Configuration conf = job.getConfiguration();
+ AccumuloOutputFormat.setZooKeeperInstance(conf, args[6], args[7]);
+ AccumuloOutputFormat.setOutputInfo(conf, args[8], args[9].getBytes(), true, null);
+ AccumuloOutputFormat.setMaxMutationBufferSize(conf, 10L * 1000 * 1000);
+
conf.setLong(NUMROWS, Long.parseLong(args[0]));
conf.setInt("cloudgen.minkeylength", Integer.parseInt(args[1]));
conf.setInt("cloudgen.maxkeylength", Integer.parseInt(args[2]));
Modified: incubator/accumulo/branches/1.4/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/WordCount.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/WordCount.java?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/WordCount.java (original)
+++ incubator/accumulo/branches/1.4/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/WordCount.java Tue Jan 24 21:25:56 2012
@@ -104,8 +104,8 @@ public class WordCount extends Configure
job.setOutputFormatClass(AccumuloOutputFormat.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Mutation.class);
- AccumuloOutputFormat.setOutputInfo(job, username, password.getBytes(), true, args[3]);
- AccumuloOutputFormat.setZooKeeperInstance(job, args[0], args[1]);
+ AccumuloOutputFormat.setOutputInfo(job.getConfiguration(), username, password.getBytes(), true, args[3]);
+ AccumuloOutputFormat.setZooKeeperInstance(job.getConfiguration(), args[0], args[1]);
job.waitForCompletion(true);
return 0;
}
Modified: incubator/accumulo/branches/1.4/src/examples/wikisearch/query-war/pom.xml
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/examples/wikisearch/query-war/pom.xml?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/examples/wikisearch/query-war/pom.xml (original)
+++ incubator/accumulo/branches/1.4/src/examples/wikisearch/query-war/pom.xml Tue Jan 24 21:25:56 2012
@@ -47,7 +47,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
- <version>2.3.1</version>
<configuration>
<source>1.6</source>
<target>1.6</target>
Modified: incubator/accumulo/branches/1.4/src/examples/wikisearch/query/pom.xml
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/examples/wikisearch/query/pom.xml?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/examples/wikisearch/query/pom.xml (original)
+++ incubator/accumulo/branches/1.4/src/examples/wikisearch/query/pom.xml Tue Jan 24 21:25:56 2012
@@ -128,7 +128,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
- <version>2.3.2</version>
<configuration>
<source>1.6</source>
<target>1.6</target>
Modified: incubator/accumulo/branches/1.4/src/examples/wikisearch/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/TestQueryLogic.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/examples/wikisearch/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/TestQueryLogic.java?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/examples/wikisearch/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/TestQueryLogic.java (original)
+++ incubator/accumulo/branches/1.4/src/examples/wikisearch/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/TestQueryLogic.java Tue Jan 24 21:25:56 2012
@@ -38,8 +38,6 @@ import org.apache.accumulo.core.data.Val
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.examples.wikisearch.ingest.WikipediaConfiguration;
import org.apache.accumulo.examples.wikisearch.ingest.WikipediaMapper;
-import org.apache.accumulo.examples.wikisearch.logic.AbstractQueryLogic;
-import org.apache.accumulo.examples.wikisearch.logic.QueryLogic;
import org.apache.accumulo.examples.wikisearch.parser.RangeCalculator;
import org.apache.accumulo.examples.wikisearch.reader.AggregatingRecordReader;
import org.apache.accumulo.examples.wikisearch.sample.Document;
@@ -165,7 +163,7 @@ public class TestQueryLogic {
}
- private void debugQuery(String tableName) throws Exception {
+ void debugQuery(String tableName) throws Exception {
Scanner s = c.createScanner(tableName, new Authorizations());
Range r = new Range();
s.setRange(r);
Modified: incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java (original)
+++ incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java Tue Jan 24 21:25:56 2012
@@ -3144,7 +3144,7 @@ public class Tablet {
copy.keySet().retainAll(smallestFiles);
- log.debug("Starting MajC " + extent + " " + datafileManager.abs2rel(datafileManager.string2path(copy.keySet())) + " --> "
+ log.debug("Starting MajC " + extent + " (" + reason + ") " + datafileManager.abs2rel(datafileManager.string2path(copy.keySet())) + " --> "
+ datafileManager.abs2rel(new Path(compactTmpName)));
Compactor compactor = new Compactor(conf, fs, copy, null, compactTmpName, filesToCompact.size() == 0 ? propogateDeletes : true, // always
Modified: incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousMoru.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousMoru.java?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousMoru.java (original)
+++ incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousMoru.java Tue Jan 24 21:25:56 2012
@@ -145,13 +145,13 @@ public class ContinuousMoru extends Conf
job.setNumReduceTasks(0);
job.setOutputFormatClass(AccumuloOutputFormat.class);
- AccumuloOutputFormat.setOutputInfo(job, user, pass.getBytes(), false, table);
- AccumuloOutputFormat.setZooKeeperInstance(job, instance, zookeepers);
- AccumuloOutputFormat.setMaxLatency(job, (int) (Integer.parseInt(maxLatency) / 1000.0));
- AccumuloOutputFormat.setMaxMutationBufferSize(job, Long.parseLong(maxMem));
- AccumuloOutputFormat.setMaxWriteThreads(job, Integer.parseInt(numThreads));
-
Configuration conf = job.getConfiguration();
+ AccumuloOutputFormat.setOutputInfo(conf, user, pass.getBytes(), false, table);
+ AccumuloOutputFormat.setZooKeeperInstance(conf, instance, zookeepers);
+ AccumuloOutputFormat.setMaxLatency(conf, (int) (Integer.parseInt(maxLatency) / 1000.0));
+ AccumuloOutputFormat.setMaxMutationBufferSize(conf, Long.parseLong(maxMem));
+ AccumuloOutputFormat.setMaxWriteThreads(conf, Integer.parseInt(numThreads));
+
conf.setLong(MIN, Long.parseLong(min));
conf.setLong(MAX, Long.parseLong(max));
conf.setInt(MAX_CF, Integer.parseInt(max_cf));
Modified: incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/BulkPlusOne.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/BulkPlusOne.java?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/BulkPlusOne.java (original)
+++ incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/BulkPlusOne.java Tue Jan 24 21:25:56 2012
@@ -22,8 +22,11 @@ import java.util.List;
import java.util.Random;
import java.util.TreeSet;
import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.accumulo.core.client.IteratorSetting.Column;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.file.FileOperations;
@@ -33,6 +36,7 @@ import org.apache.accumulo.server.test.r
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
public class BulkPlusOne extends BulkTest {
@@ -42,14 +46,24 @@ public class BulkPlusOne extends BulkTes
public static final int HEX_SIZE = (int) Math.ceil(Math.log(LOTS) / Math.log(16));
public static final String FMT = "r%0" + HEX_SIZE + "x";
static final Value one = new Value("1".getBytes());
+ public static final List<Column> COLNAMES = new ArrayList<Column>();
+ public static final Text CHECK_COLUMN_FAMILY = new Text("cf");
+ static {
+ for (int i = 0; i < COLS; i++) {
+ COLNAMES.add(new Column(CHECK_COLUMN_FAMILY, new Text(String.format("%03d", i))));
+ }
+ }
+ public static final Text MARKER_CF = new Text("marker");
+ private static final AtomicLong counter = new AtomicLong();
static void bulkLoadLots(Logger log, State state, Value value) throws Exception {
- Path dir = new Path("/tmp", "bulk_" + UUID.randomUUID().toString());
- Path fail = new Path(dir.toString() + "_fail");
- Random rand = (Random) state.get("rand");
- FileSystem fs = (FileSystem) state.get("fs");
+ final Path dir = new Path("/tmp", "bulk_" + UUID.randomUUID().toString());
+ final Path fail = new Path(dir.toString() + "_fail");
+ final DefaultConfiguration defaultConfiguration = AccumuloConfiguration.getDefaultConfiguration();
+ final Random rand = (Random) state.get("rand");
+ final FileSystem fs = (FileSystem) state.get("fs");
fs.mkdirs(fail);
- int parts = rand.nextInt(10) + 1;
+ final int parts = rand.nextInt(10) + 1;
TreeSet<Integer> startRows = new TreeSet<Integer>();
startRows.add(0);
@@ -60,26 +74,24 @@ public class BulkPlusOne extends BulkTes
for (Integer row : startRows)
printRows.add(String.format(FMT, row));
- log.debug("preparing bulk files with start rows " + printRows + " last row " + String.format(FMT, LOTS - 1));
- String cols[] = new String[COLS];
- for (int i = 0; i < cols.length; i++) {
- cols[i] = String.format("%03d", i);
- }
+ String markerColumnFamily = Long.toString(counter.incrementAndGet());
+ log.debug("preparing bulk files with start rows " + printRows + " last row " + String.format(FMT, LOTS - 1) + " marker " + markerColumnFamily);
List<Integer> rows = new ArrayList<Integer>(startRows);
rows.add(LOTS);
for (int i = 0; i < parts; i++) {
- FileSKVWriter f = FileOperations.getInstance().openWriter(dir + "/" + String.format("part_%d.", i) + RFile.EXTENSION, fs, fs.getConf(),
- AccumuloConfiguration.getDefaultConfiguration());
+ String fileName = dir + "/" + String.format("part_%d.", i) + RFile.EXTENSION;
+ FileSKVWriter f = FileOperations.getInstance().openWriter(fileName, fs, fs.getConf(), defaultConfiguration);
f.startDefaultLocalityGroup();
int start = rows.get(i);
int end = rows.get(i + 1);
for (int j = start; j < end; j++) {
- String row = String.format(FMT, j);
- for (String col : cols) {
- f.append(new Key(row, "cf", col), value);
+ Text row = new Text(String.format(FMT, j));
+ for (Column col : COLNAMES) {
+ f.append(new Key(row, col.getColumnFamily(), col.getColumnQualifierf()), value);
}
+ f.append(new Key(row, MARKER_CF, new Text(markerColumnFamily)), value);
}
f.close();
}
Modified: incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/ConsistencyCheck.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/ConsistencyCheck.java?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/ConsistencyCheck.java (original)
+++ incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/ConsistencyCheck.java Tue Jan 24 21:25:56 2012
@@ -40,6 +40,7 @@ public class ConsistencyCheck extends Bu
Scanner scanner = state.getConnector().createScanner(Setup.getTableName(), auths);
scanner = new IsolatedScanner(scanner);
scanner.setRange(new Range(row));
+ scanner.fetchColumnFamily(BulkPlusOne.CHECK_COLUMN_FAMILY);
Value v = null;
Key first = null;
for (Entry<Key,Value> entry : scanner) {
Modified: incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Setup.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Setup.java?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Setup.java (original)
+++ incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Setup.java Tue Jan 24 21:25:56 2012
@@ -17,8 +17,6 @@
package org.apache.accumulo.server.test.randomwalk.bulk;
import java.net.InetAddress;
-import java.util.Collections;
-import java.util.List;
import java.util.Properties;
import java.util.Random;
import java.util.concurrent.BlockingQueue;
@@ -27,17 +25,18 @@ import java.util.concurrent.ThreadFactor
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.TableExistsException;
-import org.apache.accumulo.core.iterators.conf.PerColumnIteratorConfig;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.iterators.LongCombiner;
+import org.apache.accumulo.core.iterators.user.SummingCombiner;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.Daemon;
import org.apache.accumulo.core.util.LoggingRunnable;
import org.apache.accumulo.server.test.randomwalk.State;
import org.apache.accumulo.server.test.randomwalk.Test;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.io.Text;
-@SuppressWarnings("deprecation")
public class Setup extends Test {
private static final int CORE_POOL_SIZE = 8;
@@ -52,12 +51,14 @@ public class Setup extends Test {
tableName = String.format("bulk_%s_%s_%d", hostname, pid, System.currentTimeMillis());
log.info("Starting bulk test on " + tableName);
- List<PerColumnIteratorConfig> aggregators = Collections.singletonList(new PerColumnIteratorConfig(new Text("cf".getBytes()), null,
- org.apache.accumulo.core.iterators.aggregation.StringSummation.class.getName()));
+ TableOperations tableOps = state.getConnector().tableOperations();
try {
- if (!state.getConnector().tableOperations().exists(getTableName())) {
- state.getConnector().tableOperations().create(getTableName());
- state.getConnector().tableOperations().addAggregators(getTableName(), aggregators);
+ if (!tableOps.exists(getTableName())) {
+ tableOps.create(getTableName());
+ IteratorSetting is = new IteratorSetting(10, org.apache.accumulo.core.iterators.user.SummingCombiner.class);
+ SummingCombiner.setEncodingType(is, LongCombiner.Type.STRING);
+ SummingCombiner.setColumns(is, BulkPlusOne.COLNAMES);
+ tableOps.attachIterator(getTableName(), is);
}
} catch (TableExistsException ex) {
// expected if there are multiple walkers
@@ -73,7 +74,6 @@ public class Setup extends Test {
}
};
ThreadPoolExecutor e = new ThreadPoolExecutor(CORE_POOL_SIZE, MAX_POOL_SIZE, 1, TimeUnit.SECONDS, q, factory);
-
state.set("pool", e);
}
Modified: incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Verify.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Verify.java?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Verify.java (original)
+++ incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Verify.java Tue Jan 24 21:25:56 2012
@@ -22,12 +22,18 @@ import java.util.Properties;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.server.test.randomwalk.State;
import org.apache.accumulo.server.test.randomwalk.Test;
+import org.apache.hadoop.io.Text;
public class Verify extends Test {
@@ -45,13 +51,68 @@ public class Verify extends Test {
String user = state.getConnector().whoami();
Authorizations auths = state.getConnector().securityOperations().getUserAuthorizations(user);
Scanner scanner = state.getConnector().createScanner(Setup.getTableName(), auths);
+ scanner.fetchColumnFamily(BulkPlusOne.CHECK_COLUMN_FAMILY);
for (Entry<Key,Value> entry : scanner) {
byte[] value = entry.getValue().get();
- if (!Arrays.equals(value, zero))
+ if (!Arrays.equals(value, zero)) {
throw new Exception("Bad key at " + entry);
+ }
}
log.info("Test successful on table " + Setup.getTableName());
state.getConnector().tableOperations().delete(Setup.getTableName());
}
+
+ public static void main(String args[]) throws Exception {
+ int i = 0;
+ String instance = args[i++];
+ String keepers = args[i++];
+ String username = args[i++];
+ String passwd = args[i++];
+ String tablename = args[i++];
+ int timeout = (int) DefaultConfiguration.getInstance().getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT);
+ Instance inst = new ZooKeeperInstance(instance, keepers, timeout);
+ Connector conn = inst.getConnector(username, passwd.getBytes());
+ Scanner scanner = conn.createScanner(tablename, conn.securityOperations().getUserAuthorizations(username));
+ scanner.fetchColumnFamily(BulkPlusOne.CHECK_COLUMN_FAMILY);
+ Text startBadRow = null;
+ Text lastBadRow = null;
+ Value currentBadValue = null;
+ for (Entry<Key,Value> entry : scanner) {
+ // System.out.println("Entry: " + entry);
+ byte[] value = entry.getValue().get();
+ if (!Arrays.equals(value, zero)) {
+ if (currentBadValue == null || entry.getValue().equals(currentBadValue)) {
+ // same value, keep skipping ahead
+ lastBadRow = new Text(entry.getKey().getRow());
+ if (startBadRow == null)
+ startBadRow = lastBadRow;
+ } else {
+ // new bad value, report
+ report(startBadRow, lastBadRow, currentBadValue);
+ startBadRow = lastBadRow = new Text(entry.getKey().getRow());
+ }
+ currentBadValue = new Value(entry.getValue());
+ } else {
+ // end of bad range, report
+ if (startBadRow != null) {
+ report(startBadRow, lastBadRow, currentBadValue);
+ }
+ startBadRow = lastBadRow = null;
+ currentBadValue = null;
+ }
+ }
+ if (startBadRow != null) {
+ report(startBadRow, lastBadRow, currentBadValue);
+ }
+ }
+
+ /**
+ * @param startBadEntry
+ * @param lastBadEntry
+ */
+ private static void report(Text startBadRow, Text lastBadRow, Value value) {
+ System.out.println("Bad value " + new String(value.get()));
+ System.out.println(" Range [" + startBadRow + " -> " + lastBadRow + "]");
+ }
}
Modified: incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/multitable/CopyTool.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/multitable/CopyTool.java?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/multitable/CopyTool.java (original)
+++ incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/multitable/CopyTool.java Tue Jan 24 21:25:56 2012
@@ -62,8 +62,8 @@ public class CopyTool extends Configured
job.setNumReduceTasks(0);
job.setOutputFormatClass(AccumuloOutputFormat.class);
- AccumuloOutputFormat.setOutputInfo(job, args[0], args[1].getBytes(), true, args[5]);
- AccumuloOutputFormat.setZooKeeperInstance(job, args[3], args[4]);
+ AccumuloOutputFormat.setOutputInfo(job.getConfiguration(), args[0], args[1].getBytes(), true, args[5]);
+ AccumuloOutputFormat.setZooKeeperInstance(job.getConfiguration(), args[3], args[4]);
job.waitForCompletion(true);
return job.isSuccessful() ? 0 : 1;
Modified: incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/sequential/MapRedVerifyTool.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/sequential/MapRedVerifyTool.java?rev=1235509&r1=1235508&r2=1235509&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/sequential/MapRedVerifyTool.java (original)
+++ incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/sequential/MapRedVerifyTool.java Tue Jan 24 21:25:56 2012
@@ -95,8 +95,8 @@ public class MapRedVerifyTool extends Co
job.setNumReduceTasks(1);
job.setOutputFormatClass(AccumuloOutputFormat.class);
- AccumuloOutputFormat.setOutputInfo(job, args[0], args[1].getBytes(), true, args[5]);
- AccumuloOutputFormat.setZooKeeperInstance(job, args[3], args[4]);
+ AccumuloOutputFormat.setOutputInfo(job.getConfiguration(), args[0], args[1].getBytes(), true, args[5]);
+ AccumuloOutputFormat.setZooKeeperInstance(job.getConfiguration(), args[3], args[4]);
job.waitForCompletion(true);
return job.isSuccessful() ? 0 : 1;
Propchange: incubator/accumulo/branches/1.4/test/system/randomwalk/conf/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Tue Jan 24 21:25:56 2012
@@ -0,0 +1,11 @@
+logger.xml
+
+tracers
+
+randomwalk.conf
+
+masters
+
+slaves
+
+walkers